Skip to content

Instantly share code, notes, and snippets.

@AndreyDodonov-EH

AndreyDodonov-EH/example.md

Last active July 12, 2025 15:02
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save AndreyDodonov-EH/fe04d4f0ee127e8bb0899a2d7b502efc to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save AndreyDodonov-EH/fe04d4f0ee127e8bb0899a2d7b502efc to your computer and use it in GitHub Desktop.
Download ZIP
Deduplicate console output live and/or show timestamps
Raw
example.md
Before After 
$ ./printer.sh 
Starting application...
Connecting to database...
Connecting to database...
Connecting to database...
Database connected successfully
Warning: High memory usage detected
Warning: High memory usage detected
Warning: High memory usage detected
Warning: High memory usage detected
Warning: High memory usage detected

$ ./printer.sh | lip -td -f "%Y-%m-%d %H:%M:%S"
[2025-07-06 17:44:13] Starting application...
[2025-07-06 17:44:14] Connecting to database... [3x]
[2025-07-06 17:44:15] Database connected successfully
[2025-07-06 17:44:15] Warning: High memory usage detected [5x]
Raw
lip
#!/usr/bin/env bash
#
# lip - Line Input Processor: Real-time line deduplication and timestamping utility
#
# A Unix pipeline utility for monitoring logs and command output with:
# - Live deduplication of consecutive identical lines with count
# - Optional timestamps with customizable format
# - Clean pipeline integration
#
# Usage: command | lip [-t] [-d] [-td] [-f FORMAT]
#
# Examples:
# python main.py | lip -td # Monitor Python script output
# dotnet run | lip -d # Deduplicate .NET app output
# docker logs -f app | lip -td # Monitor Docker logs
# npm run dev | lip -t # Timestamp dev server output
# make -j8 | lip -t -f "%Y-%m-%d %H:%M:%S" # Custom timestamp format
#
# # Monitor serial port output via minicom:
# minicom -C /tmp/serial.log # In one terminal
# tail -f /tmp/serial.log | lip -td # In another terminal
#
# Installation:
# # Option 1: Install as standalone script
# chmod +x lip
# sudo cp lip /usr/local/bin/
#
# # Option 2: Add function to .bashrc or .bash_profile
# echo "source /path/to/lip" >> ~/.bashrc
#
# # Option 3: Copy just the function to .bashrc
# # Copy the lip() function definition to your .bashrc
#
# # Option 4: Source directly when needed
# source /path/to/lip
# some_command | lip -td
# Main function
lip() {
# Parse options
local show_time=false
local show_dedup=false
local time_format="%H:%M:%S"
while [[ $# -gt 0 ]]; do
case "$1" in
-t|--time) show_time=true; shift ;;
-d|--dedup) show_dedup=true; shift ;;
-td|-dt|--time-dedup|--dedup-time) show_time=true; show_dedup=true; shift ;;
-f|--format)
if [[ -n "$2" && ! "$2" =~ ^- ]]; then
time_format="$2"
shift 2
else
echo "Error: -f/--format requires a format string" >&2
return 1
fi
;;
-h|--help)
cat << 'EOF'
lip - Line Input Processor: Real-time line deduplication and timestamping
Usage: lip [OPTIONS]
Options:
-t, --time Add timestamps to lines
-d, --dedup Deduplicate consecutive lines with count
-td, -dt Both timestamps and deduplication
-f, --format FMT Custom timestamp format (default: %H:%M:%S)
-h, --help Show this help
-v, --version Show version
Time format examples:
%H:%M:%S Default format: 14:23:01
%Y-%m-%d %H:%M:%S Full datetime: 2024-03-15 14:23:01
%H:%M:%S.%3N With milliseconds: 14:23:01.123
%s Unix timestamp: 1710512581
Examples:
# Deduplicate consecutive lines (default if no options)
python script.py | lip -d
# Add timestamps to program output
npm run dev | lip -t
# Both timestamps and deduplication
dotnet run | lip -td
# Custom timestamp format
./my_app | lip -t -f "%Y-%m-%d %H:%M:%S.%3N"
# Monitor build output
make -j8 | lip -td
# Debug applications
./my_app --verbose | lip -td
# Monitor serial port
cat /dev/ttyUSB0 | lip -td
# Monitor minicom output
# Terminal 1: minicom -C /tmp/serial.log
# Terminal 2: tail -f /tmp/serial.log | lip -td
Output format:
Without -t: repeated line [3x]
With -t: [14:23:01] repeated line [3x]
Note: For deduplication with timestamps, use 'lip -td' rather than
piping separate commands, as timestamps make every line unique.
EOF
return 0
;;
-v|--version)
echo "lip (Line Input Processor) version 1.0"
return 0
;;
*)
echo "Unknown option: $1 (try --help)" >&2
return 1
;;
esac
done
# If no options specified, default to dedup only
if [[ "$show_time" == false && "$show_dedup" == false ]]; then
show_dedup=true
fi
# Build the awk script based on options
if [[ "$show_dedup" == true && "$show_time" == true ]]; then
# Both dedup and timestamps
awk -v fmt="$time_format" '
BEGIN { ORS = "" }
{
if ($0 == prev) {
++n
printf "\r\033[2K%s%s \033[33m[%dx]\033[0m", timestamp, $0, n
} else {
if (NR > 1) printf "\n"
prev = $0
n = 1
cmd = "date +\"" fmt "\""
cmd | getline time
close(cmd)
timestamp = "\033[2m[" time "]\033[0m "
printf "%s%s", timestamp, $0
}
fflush()
}
END { if (NR > 0) printf "\n" }'
elif [[ "$show_dedup" == true ]]; then
# Just dedup
awk '
BEGIN { ORS = "" }
{
if ($0 == prev) {
++n
printf "\r\033[2K%s \033[33m[%dx]\033[0m", $0, n
} else {
if (NR > 1) printf "\n"
prev = $0
n = 1
printf "%s", $0
}
fflush()
}
END { if (NR > 0) printf "\n" }'
elif [[ "$show_time" == true ]]; then
# Just timestamps
awk -v fmt="$time_format" '{
cmd = "date +\"" fmt "\""
cmd | getline time
close(cmd)
printf "\033[2m[%s]\033[0m %s\n", time, $0
fflush()
}'
else
# Shouldn't happen, but just pass through
cat
fi
}
# Demo function for testing
lip_demo() {
echo "=== LIP (Line Input Processor) Demo ==="
echo
echo "1. Deduplication demo:"
echo "------------------------"
{
echo "Starting server..."
sleep 0.5
echo "Connecting to database..."
echo "Connecting to database..."
echo "Connecting to database..."
sleep 0.5
echo "Connected!"
echo "Processing requests..."
echo "Processing requests..."
} | lip -d
echo
echo "2. Timestamps demo:"
echo "------------------------"
{
echo "Server started"
sleep 1
echo "Request received"
sleep 1
echo "Request processed"
} | lip -t
echo
echo "3. Both features demo:"
echo "------------------------"
{
echo "Checking system..."
echo "Checking system..."
echo "Checking system..."
sleep 1
echo "System OK"
echo "Running diagnostics..."
echo "Running diagnostics..."
} | lip -td
echo
echo "4. Custom timestamp format demo:"
echo "------------------------"
{
echo "Event 1"
sleep 1
echo "Event 2"
sleep 1
echo "Event 3"
} | lip -t -f "%Y-%m-%d %H:%M:%S"
}
# If script is executed directly (not sourced)
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
# Check if stdin is a pipe
if [[ -p /dev/stdin ]]; then
lip "$@"
else
# No pipe input
if [[ $# -eq 0 ]]; then
echo "Error: No input provided. Use in a pipeline or try --help" >&2
echo "Example: python script.py | $(basename "$0") -td" >&2
exit 1
elif [[ "$1" == "demo" ]]; then
lip_demo
else
lip "$@"
fi
fi
fi
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment