Quick Answer: Search: grep "pattern" file. Replace: sed 's/old/new/g' file. Extract column: awk '{print $1}' file. Combine: cat log | grep ERROR | awk '{print $3}' | sort | uniq -c.
grep — Search Text
Basic Search
# Search for pattern in file
grep "error" logfile.txt
# Case-insensitive
grep -i "error" logfile.txt
# Recursive search in directory
grep -r "TODO" ./src/
# Show line numbers
grep -n "error" logfile.txt
# Show count of matches
grep -c "error" logfile.txt
# Show only filenames with matches
grep -l "error" *.log
# Show filenames WITHOUT matches
grep -L "error" *.log
Context
# Show 3 lines after match
grep -A 3 "error" logfile.txt
# Show 3 lines before match
grep -B 3 "error" logfile.txt
# Show 3 lines before and after
grep -C 3 "error" logfile.txt
Invert and Filter
# Show lines that DON'T match
grep -v "debug" logfile.txt
# Multiple patterns (OR)
grep -E "error|warning|critical" logfile.txt
# Match whole word only
grep -w "error" logfile.txt
# Match whole line only
grep -x "exact line content" file.txt
Regular Expressions
# Extended regex (-E or egrep)
grep -E "^[0-9]{4}-[0-9]{2}" logfile.txt # Lines starting with date
grep -E "error|warning" logfile.txt # Multiple patterns
grep -E "\b[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\b" file # IP addresses
# Perl-compatible regex (-P)
grep -P "\d{4}-\d{2}-\d{2}" logfile.txt # Date pattern
grep -P "(?<=user=)\w+" logfile.txt # Lookbehind
grep -oP "(?<=email=)[^\s&]+" logfile.txt # Extract email values
# Only show the matched part
grep -o "error.*$" logfile.txt
Useful grep One-Liners
# Find all IP addresses in a file
grep -oE "\b([0-9]{1,3}\.){3}[0-9]{1,3}\b" file
# Find empty lines
grep -n "^$" file
# Find lines longer than 80 characters
grep -n ".\{81\}" file
# Count unique errors
grep "ERROR" logfile.txt | sort | uniq -c | sort -rn
# Search compressed files
zgrep "pattern" file.gz
# Exclude directories from recursive search
grep -r --exclude-dir={.git,node_modules} "pattern" .
# Exclude file types
grep -r --include="*.py" "import" .
sed — Stream Editor
Substitution
# Replace first occurrence on each line
sed 's/old/new/' file
# Replace ALL occurrences on each line
sed 's/old/new/g' file
# Replace only on line 5
sed '5s/old/new/' file
# Replace on lines 10-20
sed '10,20s/old/new/g' file
# Replace on lines matching pattern
sed '/error/s/old/new/g' file
# Case-insensitive replace
sed 's/old/new/gi' file
# Edit file in-place
sed -i 's/old/new/g' file
# Edit in-place with backup
sed -i.bak 's/old/new/g' file
Delete Lines
# Delete line 5
sed '5d' file
# Delete lines 10-20
sed '10,20d' file
# Delete empty lines
sed '/^$/d' file
# Delete lines matching pattern
sed '/debug/d' file
# Delete lines NOT matching pattern
sed '/error/!d' file
# Delete first line
sed '1d' file
# Delete last line
sed '$d' file
Insert and Append
# Insert line before line 3
sed '3i\New line here' file
# Append line after line 3
sed '3a\New line here' file
# Insert before matching line
sed '/pattern/i\New line here' file
# Append after matching line
sed '/pattern/a\New line here' file
Print
# Print only matching lines (like grep)
sed -n '/pattern/p' file
# Print lines 10-20
sed -n '10,20p' file
# Print first line
sed -n '1p' file
# Print last line
sed -n '$p' file
Multiple Operations
# Chain multiple operations with -e
sed -e 's/foo/bar/g' -e 's/baz/qux/g' file
# Or use semicolons
sed 's/foo/bar/g; s/baz/qux/g' file
Useful sed One-Liners
# Remove trailing whitespace
sed 's/[[:space:]]*$//' file
# Remove leading whitespace
sed 's/^[[:space:]]*//' file
# Remove blank lines
sed '/^$/d' file
# Remove HTML tags
sed 's/<[^>]*>//g' file
# Add line numbers
sed = file | sed 'N; s/\n/\t/'
# Double-space a file
sed 'G' file
# Replace newlines with spaces
sed ':a;N;$!ba;s/\n/ /g' file
# Extract lines between two patterns
sed -n '/START/,/END/p' file
# Remove comments (# lines)
sed '/^#/d; /^$/d' file
# Convert DOS line endings to Unix
sed 's/\r$//' file
awk — Column Processing
Basic Usage
# Print entire line
awk '{print}' file
# Print first column
awk '{print $1}' file
# Print columns 1 and 3
awk '{print $1, $3}' file
# Print last column
awk '{print $NF}' file
# Print second-to-last column
awk '{print $(NF-1)}' file
# Custom separator for input
awk -F':' '{print $1}' /etc/passwd
# Custom separator for output
awk -F':' 'BEGIN{OFS="\t"} {print $1, $3}' /etc/passwd
Filtering
# Print lines matching pattern
awk '/error/' file
# Print lines NOT matching
awk '!/error/' file
# Print where column 3 > 100
awk '$3 > 100' file
# Print where column 1 equals "Sam"
awk '$1 == "Sam"' file
# Print lines longer than 80 chars
awk 'length > 80' file
# Print line numbers
awk '{print NR, $0}' file
Formatting
# Printf for formatted output
awk '{printf "%-20s %10d\n", $1, $2}' file
# Add header
awk 'BEGIN{print "Name\tScore"} {print $1, $2}' file
# Add header and footer
awk 'BEGIN{print "---START---"} {print} END{print "---END---"}' file
Math and Aggregation
# Sum a column
awk '{sum += $2} END{print sum}' file
# Average
awk '{sum += $2; n++} END{print sum/n}' file
# Min and max
awk 'NR==1{min=max=$2} $2>max{max=$2} $2<min{min=$2} END{print "min:", min, "max:", max}' file
# Count lines
awk 'END{print NR}' file
# Count matching lines
awk '/error/{count++} END{print count}' file
Built-in Variables
| Variable |
Meaning |
$0 |
Entire line |
$1, $2, ... |
Fields (columns) |
NR |
Current line number (total) |
NF |
Number of fields in current line |
FS |
Field separator (default: space) |
OFS |
Output field separator |
RS |
Record separator (default: newline) |
ORS |
Output record separator |
FILENAME |
Current filename |
Useful awk One-Liners
# Unique values in column 1
awk '!seen[$1]++' file
# Frequency count of column 1
awk '{count[$1]++} END{for(k in count) print count[k], k}' file | sort -rn
# Sum column 2 grouped by column 1
awk '{sum[$1] += $2} END{for(k in sum) print k, sum[k]}' file
# Print lines between two patterns
awk '/START/,/END/' file
# Remove duplicate lines (preserving order)
awk '!seen[$0]++' file
# Transpose rows to columns
awk '{for(i=1;i<=NF;i++) a[i]=a[i]" "$i} END{for(i=1;i<=NF;i++) print a[i]}' file
# Print fields in reverse order
awk '{for(i=NF;i>=1;i--) printf "%s ", $i; print ""}' file
# CSV to TSV
awk -F',' 'BEGIN{OFS="\t"} {$1=$1; print}' file.csv
# Extract specific field from key=value pairs
awk -F'=' '/^database_host/{print $2}' config.ini
Combining grep, sed, awk
Real-World Pipelines
# Top 10 IPs in access log
awk '{print $1}' access.log | sort | uniq -c | sort -rn | head -10
# Count HTTP status codes
awk '{print $9}' access.log | sort | uniq -c | sort -rn
# Find 500 errors with URLs
awk '$9 == 500 {print $7}' access.log | sort | uniq -c | sort -rn
# Extract emails from text
grep -oE "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" file
# Replace in all Python files
find . -name "*.py" -exec sed -i 's/old_function/new_function/g' {} +
# Kill processes by name
ps aux | grep "python" | grep -v grep | awk '{print $2}' | xargs kill
# Disk usage by directory (sorted)
du -sh */ | sort -rh | head -10
# Find large log files
find /var/log -name "*.log" -exec du -sh {} + | sort -rh | head -10
# Monitor log in real-time for errors
tail -f /var/log/syslog | grep --line-buffered "error"
# Parse JSON-like key=value logs
grep "status=500" app.log | sed 's/.*user=\([^ ]*\).*/\1/' | sort | uniq -c | sort -rn
Quick Reference
| Task |
Command |
| Search for pattern |
grep "pattern" file |
| Case-insensitive search |
grep -i "pattern" file |
| Recursive search |
grep -r "pattern" dir/ |
| Search with regex |
`grep -E "pat1 |
pat2" file` |
| Replace text |
sed 's/old/new/g' file |
| Replace in-place |
sed -i 's/old/new/g' file |
| Delete lines |
sed '/pattern/d' file |
| Print column |
awk '{print $1}' file |
| Custom delimiter |
awk -F':' '{print $1}' file |
| Sum column |
awk '{s+=$1} END{print s}' file |
| Unique values |
awk '!seen[$0]++' file |
| Count occurrences |
`sort \ |
uniq -c \ |
sort -rn` |
Related Guides