7. Text Processing

7.1 grep: Text Search

# Basic search
grep "error" /var/log/syslog

# Ignore case
grep -i "error" /var/log/syslog

# Show line numbers
grep -n "TODO" script.py

# Recursively search a directory
grep -r "password" /etc/ --include="*.conf"

# Invert match (exclude)
grep -v "^#" /etc/nginx/nginx.conf

# Show matching context (3 lines before and after)
grep -C 3 "panic" /var/log/kern.log

# Use regular expressions
grep -E "error|warning|critical" /var/log/syslog

# Count matching lines
grep -c "error" /var/log/syslog

$ grep -n "error" /var/log/syslog
42:Jun 20 09:15:23 server kernel: [ERROR] USB device not responding
107:Jun 20 09:30:45 server sshd[1234]: error: PAM: Authentication failure
256:Jun 20 10:05:12 server nginx: [error] connect() failed (111: Connection refused)

$ grep -c "error" /var/log/syslog
3

7.2 sed: Stream Editor

# Replace text (print only, do not modify the file)
echo "Hello World" | sed 's/World/Linux/'

# Replace and write back to the file (-i edits in place)
sed -i 's/old/new/g' config.txt

# Back up before replacing (-i.bak)
sed -i.bak 's/127.0.0.1/0.0.0.0/g' /etc/mysql/my.cnf

# Delete blank lines
sed '/^$/d' file.txt

# Delete lines containing a pattern
sed '/pattern/d' file.txt

# Print only lines 5-10
sed -n '5,10p' file.txt

# Insert content before line 3
sed '3i\new line content' file.txt

# Append content after line 3
sed '3a\new line content' file.txt

$ echo "Hello World" | sed 's/World/Linux/'
Hello Linux

$ cat config.txt
server_name example.com
listen 80
root /var/www/html

$ sed 's/listen 80/listen 8080/g' config.txt
server_name example.com
listen 8080
root /var/www/html

7.3 awk: Powerful Text Analysis

# Extract columns (whitespace delimiter by default)
awk '{print $1, $3}' file.txt

# Specify a delimiter
awk -F: '{print $1, $7}' /etc/passwd

# Conditional filtering
awk -F: '$3 >= 1000 {print $1}' /etc/passwd

# Calculate a total
ls -l | awk 'NR>1 {sum+=$5} END {print "Total:", sum}'

# Format output
df -h | awk '{printf "%-20s %s\n", $1, $5}'

$ awk -F: '$3 >= 1000 {print $1, $7}' /etc/passwd
nobody /usr/sbin/nologin
user /bin/bash
john /bin/bash
jane /bin/bash

$ df -h | awk '{printf "%-20s %s\n", $1, $5}'
Filesystem           Use%
/dev/sda1            45%
tmpfs                1%
/dev/sda2            78%

7.4 Other Text Tools

# cut: extract columns (useful for CSV-like text)
cut -d: -f1,3 /etc/passwd

# sort: sort lines
sort /etc/passwd             # Alphabetical sort
sort -t: -k3 -n /etc/passwd  # Numeric sort by the 3rd column
sort -u file.txt             # Sort and remove duplicates

# uniq: remove duplicates (sort first)
sort file.txt | uniq
sort file.txt | uniq -c      # Count duplicates

# wc: count
wc -l /etc/passwd            # Lines
wc -w file.txt               # Words
wc -c file.txt               # Bytes

# tr: replace or delete characters
echo "Hello World" | tr 'a-z' 'A-Z'     # Convert to uppercase
echo "Hello   World" | tr -s ' '        # Squeeze repeated spaces
echo "abc123" | tr -d '0-9'             # Delete digits

# tee: print to screen and write to a file
ls -l | tee file_list.txt

$ sort /etc/passwd | uniq -c | sort -rn | head -5
      1 nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin
      1 root:x:0:0:root:/root:/bin/bash
      1 sys:x:3:3:sys:/dev:/usr/sbin/nologin

$ wc -l /etc/passwd
35 /etc/passwd

$ echo "Hello World" | tr 'a-z' 'A-Z'
HELLO WORLD

7.5 Pipes and Redirection

Pipe (|): sends the output of one command into the input of the next command, like an assembly line.

# Pipeline example: count how many login-capable users exist in /etc/passwd
grep -v "nologin\|false" /etc/passwd | wc -l

# Find the top 5 largest directories under /home
du -sh /home/* 2>/dev/null | sort -rh | head -5

# Show the top 5 processes by memory usage
ps aux --sort=-%mem | head -6

# Redirect output to a file (overwrite)
echo "Hello" > output.txt

# Redirect output to a file (append)
echo "World" >> output.txt

# Redirect errors
find / -name "*.conf" 2> /dev/null

# Redirect stdout and stderr together
command &> all_output.txt

# Here Document (multi-line input)
cat << EOF
server {
    listen 80;
    server_name example.com;
}
EOF

$ grep -v "nologin\|false" /etc/passwd | wc -l
4

$ du -sh /home/* 2>/dev/null | sort -rh | head -5
2.1G	/home/user
850M	/home/john
320M	/home/jane
50M	/home/www

$ ps aux --sort=-%mem | head -6
USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
mysql     1234  2.3 12.5 1843652 512340 ?      Sl   Jun19  10:30 /usr/sbin/mysqld
www-data  2345  0.5  8.2 856432 335876 ?       S    Jun19   2:15 apache2 -k start
root       567  0.1  3.1 225432 127844 ?       Ss   Jun19   0:45 /usr/bin/dockerd
user      3456  1.2  2.8 987654 115234 pts/0   S+   10:00   0:15 vim largefile.txt

Symbol	Meaning	Example
>	Redirect stdout (overwrite)	echo hi > f.txt
>>	Redirect stdout (append)	echo hi >> f.txt
2>	Redirect stderr	cmd 2> err.log
&>	Redirect all output	cmd &> all.log
<<	Redirect stdin	wc -l < file.txt
|	Pipe output to another command	ls | grep txt