Display the total number of fields in file including all lines (In
example given below, we will see for one line and complete file also with
default delimiter and explicit delimiter):
For single line of a file with default delimiter (TAB)
# head -n 1 /etc/passwd |awk '{total +=NF }; END {print
total}'
1
For single line of a file with custom delimiter (:)
# head -n 1 /etc/passwd |awk -F ':' '{total +=NF }; END
{print total}'
7
For all lines of a file with default delimiter (TAB)
# cat /etc/passwd |awk '{total +=NF }; END {print total}'
56
For all lines of a file with custom delimiter (:)
# cat /etc/passwd |awk -F ':' '{total +=NF }; END {print
total}'
259
Display number of columns for each line in input file
With default delimiter (TAB)
# awk '{print NF}' /etc/passwd
1
1
1
1
1
1
1
1
1
1
1
1
1
1
2
1
With custom delimiter (:)
# awk -F ':' '{print NF}' /etc/passwd
7
7
7
7
7
7
7
7
7
7
7
7
7
Search the length of each line of input:
# awk ' {print length($0)}' /etc/passwd
31
32
39
36
40
31
44
32
46
27
47
44
45
47
43
Search the length of the longest line of input:
# awk ' { if (length($0) > max) max = length($0) } END
{print max}' /etc/passwd
69
Search fice (not so) random numbers with awk without any input:
# awk 'BEGIN { for (i = 1; i <= 5; i++) print int(101 *
rand()) }'
24
29
85
15
59
To store one specific value of the text input in a variable and Search
each result only once:
# awk -F ':' ' /^r/ { username = $1; print "Username :
",username };' /etc/passwd
Username : root
Username : rpm
Username : rpc
Username : rpcuser
Using the values of shell variables in awk:
# echo $TERM
xterm
# echo |awk -v myvar=$TERM '{print "The value of the
selected shell variable is: " myvar}'
The value of the selected shell variable is: xterm
Creating shell variables with awk:
# eval $(awk 'BEGIN { print "myvar=great" }')
# echo $myvar
Great
Working with fields/strings:
# awk -F ':' '{print $1.$4}' /etc/passwd
root0
bin1
daemon2
adm4
lp7
sync0
shutdown0
halt0
mail12
Remove empty lines from the input:
# cat /etc/login.defs |grep -v '#'
MAIL_DIR /var/spool/mail
PASS_MAX_DAYS 99999
PASS_MIN_DAYS 0
PASS_MIN_LEN 5
PASS_WARN_AGE 7
UID_MIN
500
UID_MAX
60000
GID_MIN
500
GID_MAX
60000
CREATE_HOME yes
UMASK 077
USERGROUPS_ENAB yes
# cat /etc/login.defs |grep -v '#' |awk 'NF > 0'
MAIL_DIR /var/spool/mail
PASS_MAX_DAYS 99999
PASS_MIN_DAYS 0
PASS_MIN_LEN 5
PASS_WARN_AGE 7
UID_MIN
500
UID_MAX
60000
GID_MIN
500
GID_MAX
60000
CREATE_HOME yes
UMASK 077
USERGROUPS_ENAB yes
Only Search parts of a file:
# awk '/^rp/ {print $0}; /^ss/ {print $0}; /^mail:/ {print
$0; print " "};' /etc/passwd
mail:x:8:12:mail:/var/spool/mail:/sbin/nologin
rpm:x:37:37::/var/lib/rpm:/sbin/nologin
rpc:x:32:32:Portmapper RPC user:/:/sbin/nologin
rpcuser:x:29:29:RPC Service User:/var/lib/nfs:/sbin/nologin
sshd:x:74:74:Privilege-separated
SSH:/var/empty/sshd:/sbin/nologin
Modifying a word using inbuilt function “sub” (here we will change
word mail with awk):
# awk '/^mail:/ {sub(/spool/, "XXXXX"); print $0;
print " "};' /etc/passwd
mail:x:8:12:mail:/var/XXXXX/mail:/sbin/nologin
Calculating the size of all files in the directory which were modified
in April (in bytes):
# ls -lrt
total 80
-rw-r--r-- 1 root root 4224 Feb 5 17:28
install.log.syslog
-rw-r--r-- 1 root root 38536 Feb 5 17:35 install.log
-rw------- 1 root root 955 Feb 5 17:35
anaconda-ks.cfg
drwxr-xr-x 2 root root 4096 Feb 5 20:03 Desktop
-rw-r--r-- 1 root root 119 Feb 5 20:28
ifcfg-eth0
-rw-r--r-- 1 root root 119 Feb 6 15:17
ifcfg-eth1
drwxr-xr-x 3 root root 4096 Mar 1 02:20 admin
# ls -l |awk '$2 == "1" {sum += $5 } END {print
sum}'
43953
Concatenate every 3 lines of input with a pipe operator.
# awk 'ORS=NR%3?"|":"\n"' /etc/passwd
root:x:0:0:root:/root:/bin/bash|bin:x:1:1:bin:/bin:/sbin/nologin|daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin|lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin|sync:x:5:0:sync:/sbin:/bin/sync
shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown|halt:x:7:0:halt:/sbin:/sbin/halt|mail:x:8:12:mail:/var/spool/mail:/sbin/nologin
news:x:9:13:news:/etc/news:|uucp:x:10:14:uucp:/var/spool/uucp:/sbin/nologin|operator:x:11:0:operator:/root:/sbin/nologin
games:x:12:100:games:/usr/games:/sbin/nologin|gopher:x:13:30:gopher:/var/gopher:/sbin/nologin|ftp:x:14:50:FTP
User:/var/ftp:/sbin/nologin
nobody:x:99:99:Nobody:/:/sbin/nologin|rpm:x:37:37::/var/lib/rpm:/sbin/nologin|dbus:x:81:81:System
message bus:/:/sbin/nologin
avahi:x:70:70:Avahi
daemon:/:/sbin/nologin|mailnull:x:47:47::/var/spool/mqueue:/sbin/nologin|smmsp:x:51:51::/var/spool/mqueue:/sbin/nologin
nscd:x:28:28:NSCD
Daemon:/:/sbin/nologin|vcsa:x:69:69:virtual console memory
owner:/dev:/sbin/nologin|haldaemon:x:68:68:HAL daemon:/:/sbin/nologin
rpc:x:32:32:Portmapper RPC
user:/:/sbin/nologin|rpcuser:x:29:29:RPC Service User:/var/lib/nfs:/sbin/nologin|nfsnobody:x:65534:65534:Anonymous
NFS User:/var/lib/nfs:/sbin/nologin
sshd:x:74:74:Privilege-separated
SSH:/var/empty/sshd:/sbin/nologin|pcap:x:77:77::/var/arpwatch:/sbin/nologin|ntp:x:38:38::/etc/ntp:/sbin/nologin
gdm:x:42:42::/var/gdm:/sbin/nologin|distcache:x:94:94:Distcache:/:/sbin/nologin|apache:x:48:48:Apache:/var/www:/sbin/nologin
webalizer:x:67:67:Webalizer:/var/www/usage:/sbin/nologin|squid:x:23:23::/var/spool/squid:/sbin/nologin|xfs:x:43:43:X
Font Server:/etc/X11/fs:/sbin/nologin
sabayon:x:86:86:Sabayon user:/home/sabayon:/sbin/nologin|
awk with "If Else If" to find the average and grade for
every user using a awk written in script
# cat grade.awk
{
total=$3+$4+$5;
avg=total/3;
if ( avg >= 90 ) grade="A";
else if ( avg >= 80) grade ="B";
else if (avg >= 70) grade ="C";
else grade="D";
Search $0,"=>",grade;
}
# awk -f grade.awk student-marks
Jones 2143 78 84 77 => C
Gondrol 2321 56 58 45 => D
RinRao 2122 38 37 => D
Edwin 2537 87 97 95 => A
Dayan 2415 30 47 => D
awk with "If Else" to find the PASS and FAIL results for
every user
# awk '{
if ($3 >=35 && $4 >= 35 && $5 >=
35)
Search
$0,"=>","Pass";
else
Search
$0,"=>","Fail";
}' student-marks
Jones 2143 78 84 77 => Pass
Gondrol 2321 56 58 45 => Pass
RinRao 2122 38 37 => Fail
Edwin 2537 87 97 95 => Pass
Dayan 2415 30 47 => Fail
awk with "If" to check all the marks are exist
# awk '{
if ($3 =="" || $4 == "" || $5 ==
"")
Search "Some score for the
student",$1,"is missing";'
}' student-marks
Some score for the student RinRao is missing
Some score for the student Dayan is missing
To Count number of users who is using /bin/sh shell
# awk -F ':' '$NF ~ /\/bin\/bash/ { n++ }; END { print n }'
/etc/passwd
1
To Find the line with highest largest value for a field (In example
given below, USER having maximum UID is calculated)
# awk -F ':' '$3 > maxuid { maxuid=$3; maxline=$0
}; END { print maxuid, maxline }' /etc/passwd
65534 nfsnobody:x:65534:65534:Anonymous NFS User:/var/lib/nfs:/sbin/nologin
Search the even-numbered lines
# awk 'NR % 2 == 0' /etc/passwd
bin:x:1:1:bin:/bin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
sync:x:5:0:sync:/sbin:/bin/sync
halt:x:7:0:halt:/sbin:/sbin/halt
news:x:9:13:news:/etc/news:
operator:x:11:0:operator:/root:/sbin/nologin
gopher:x:13:30:gopher:/var/gopher:/sbin/nologin
nobody:x:99:99:Nobody:/:/sbin/nologin
dbus:x:81:81:System message bus:/:/sbin/nologin
mailnull:x:47:47::/var/spool/mqueue:/sbin/nologin
nscd:x:28:28:NSCD Daemon:/:/sbin/nologin
Search then lines having same value for two fields
# awk -F ':' '$3==$4' /etc/passwd
root:x:0:0:root:/root:/bin/bash
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
nobody:x:99:99:Nobody:/:/sbin/nologin
rpm:x:37:37::/var/lib/rpm:/sbin/nologin
dbus:x:81:81:System message bus:/:/sbin/nologin
avahi:x:70:70:Avahi daemon:/:/sbin/nologin
mailnull:x:47:47::/var/spool/mqueue:/sbin/nologin
smmsp:x:51:51::/var/spool/mqueue:/sbin/nologin
Search Line that satisfy two check conditions
# awk -F ':' '$3>=100 && $NF ~
/\/sbin\/nologin/' /etc/passwd
nfsnobody:x:65534:65534:Anonymous NFS
User:/var/lib/nfs:/sbin/nologin
Search Line that have blank value in perticular field
# awk -F ':' '$5 == "" ' /etc/passwd
rpm:x:37:37::/var/lib/rpm:/sbin/nologin
mailnull:x:47:47::/var/spool/mqueue:/sbin/nologin
smmsp:x:51:51::/var/spool/mqueue:/sbin/nologin
pcap:x:77:77::/var/arpwatch:/sbin/nologin
ntp:x:38:38::/etc/ntp:/sbin/nologin
gdm:x:42:42::/var/gdm:/sbin/nologin
squid:x:23:23::/var/spool/squid:/sbin/nologin
Output Field Separator (OFS) Variable
Command without OFS
# awk -F':' '{print $3,$4;}' /etc/passwd
41 41
100 101
101 102
Command with OFS as "="
# awk -F':' 'BEGIN{OFS="=";} {print $3,$4;}'
/etc/passwd
41=41
100=101
101=102
Record Separator (RS) variable
# cat student.txt
Jones
2143
78
84
77
Gondrol
2321
56
58
45
----------------
# cat student.awk
BEGIN {
RS="\n\n";
FS="\n";
}
{
Search $1,$2;
}
# awk -f student.awk student.txt
Jones 2143
Gondrol 2321
RinRao 2122
Edwin 2537
Dayan 2415
Output Record Separator (ORS) Variable
# awk 'BEGIN{ORS="=";} {print;}' student.txt
Jones=2143=78=84=77==Gondrol=2321=56=58=45==
Number of Records (NR) Variable
# awk '{print "Processing Record - ",NR;}END
{print NR, " Records are processed";}' student.txt
Processing Record - 1
Processing Record - 2
Processing Record - 3
Processing Record - 4
Processing Record - 5
Processing Record - 6
Processing Record - 7
Processing Record - 8
Processing Record - 9
Processing Record - 10
Processing Record - 11
Processing Record - 12
12 Records are processed
Number of Fields (NF) in a record
# awk '{print NR,"->",NF}' student.txt
1 -> 1
2 -> 1
3 -> 1
4 -> 1
5 -> 1
6 -> 0
7 -> 1
8 -> 1
9 -> 1
10 -> 1
11 -> 1
12 -> 0
Name of the current input file
# awk '{print FILENAME}' /etc/passwd
/etc/passwd
/etc/passwd
/etc/passwd
/etc/passwd
/etc/passwd
/etc/passwd
Number of Records relative to the current input file
# awk '{print FILENAME, FNR;}' /etc/passwd
/etc/passwd 1
/etc/passwd 2
/etc/passwd 3
/etc/passwd 4
/etc/passwd 5
/etc/passwd 6
/etc/passwd 7
/etc/passwd 8
/etc/passwd 9
/etc/passwd 10
Search for the strings “Test” and “test” using a regular expression.
# cat test.file
Test.file
123
123
234
234
test.file
345
345
123
345
234
test.file
# awk '/[Tt]est/' test.file
Test.file
test.file
test.file
Search for lower case letters d-m your search would look like this:
# cat admin/test.file
Test.file
123
123
234
234
test.file
345
345
123
345
234
test.file
# awk '/[d-m]/' admin/test.file
Test.file
test.file
test.file
If you were searching for any number:
# cat admin/test.file
Test.file
123
123
234
234
test.file
345
345
123
345
234
test.file
# awk '/[0-9]/' admin/test.file
123
123
234
234
345
345
123
345
234
Combinations of searches using “&&” which requires and “AND”
as well as “||” which requires an “OR” operator.
# awk -F ':' '((NR > 10) && ($1 ==
"operator")) && ($NF == "/sbin/nologin")'
/etc/passwd
operator:x:11:0:operator:/root:/sbin/nologin
# awk -F ':' '((NR > 10) && ($1 ==
"operator")) || ($NF == "/sbin/nologin")' /etc/passwd
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
Text Conversion and Substitution
IN UNIX ENVIRONMENT: convert DOS newlines (CR/LF) to Unix format
awk '{sub(/\n$/,"");print}' # assumes EACH
line ends with Ctrl-M
IN UNIX ENVIRONMENT: convert Unix newlines (LF) to DOS format
awk '{sub(/$/,"\r");print}
IN DOS ENVIRONMENT: convert Unix newlines (LF) to DOS format
# awk 1 /etc/passwd |head
root:x:0:0:root:/root:/bin/bash
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
IN DOS ENVIRONMENT: convert DOS newlines (CR/LF) to Unix format Cannot
be done with DOS versions of awk, other than gawk:
# gawk -v BINMODE="w" '1' /etc/passwd
root:x:0:0:root:/root:/bin/bash
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
Use "tr" instead. “tr -d \r outfile”
Delete leading whitespace (spaces, tabs) from front of each line
aligns all text flush left
awk '{sub(/^[ \t]+/, ""); print}'
Delete trailing whitespace (spaces, tabs) from end of each line
awk '{sub(/[ \t]+$/, "");print}'
Delete BOTH leading and trailing whitespace from each line
awk '{gsub(/^[ \t]+|[ \t]+$/,"");print}'
Delete BOTH leading and trailing whitespace along with whitespaces
between the fields from each line
awk '{$1=$1;print}'
Insert 5 blank spaces at beginning of each line (make page offset)
awk '{sub(/^/, " ");print}'
Substitute (find and replace) "bash" with "XXXXX"
on each line
awk '{sub(/\//,"XXXXX");print}'
# replaces only 1st instance
# head -n 5 /etc/passwd |awk
'{sub(/\//,"XXXXX");print}'
root:x:0:0:root:XXXXXroot:/bin/bash
bin:x:1:1:bin:XXXXXbin:/sbin/nologin
daemon:x:2:2:daemon:XXXXXsbin:/sbin/nologin
adm:x:3:4:adm:XXXXXvar/adm:/sbin/nologin
lp:x:4:7:lp:XXXXXvar/spool/lpd:/sbin/nologin
gawk '{$0=gensub(/\//,"XXXXX",4);print}' #
replaces only 4th instance
# head -n 5 /etc/passwd |gawk
'{$0=gensub(/\//,"XXXXX",4);print}'
root:x:0:0:root:/root:/bin/bash
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbinXXXXXnologin
lp:x:4:7:lp:/var/spool/lpd:XXXXXsbin/nologin
awk '{gsub(/\//,"XXXXX");print}'
# replaces ALL instances in a line
# head -n 5 /etc/passwd |awk
'{gsub(/\//,"XXXXX");print}'
root:x:0:0:root:XXXXXroot:XXXXXbinXXXXXbash
bin:x:1:1:bin:XXXXXbin:XXXXXsbinXXXXXnologin
daemon:x:2:2:daemon:XXXXXsbin:XXXXXsbinXXXXXnologin
adm:x:3:4:adm:XXXXXvarXXXXXadm:XXXXXsbinXXXXXnologin
lp:x:4:7:lp:XXXXXvarXXXXXspoolXXXXXlpd:XXXXXsbinXXXXXnologin
Substitute "bin" with "YYYYYYY" ONLY for lines
which contain "nologin"
# head -n 5 /etc/passwd |awk '/nologin/{gsub(/bin/,
"YYYYYYY")};{print}'
root:x:0:0:root:/root:/bin/bash
YYYYYYY:x:1:1:YYYYYYY:/YYYYYYY:/sYYYYYYY/nologin
daemon:x:2:2:daemon:/sYYYYYYY:/sYYYYYYY/nologin
adm:x:3:4:adm:/var/adm:/sYYYYYYY/nologin
lp:x:4:7:lp:/var/spool/lpd:/sYYYYYYY/nologin
Substitute "bin" with "YYYYYYY" EXCEPT for lines
which contain "nologin"
# head -n 5 /etc/passwd |awk '!/nologin/{gsub(/bin/,
"YYYYYYY")};{print}'
root:x:0:0:root:/root:/YYYYYYY/bash
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
Change "bin" or "sbin" to "ZZZZZZ"
# head -n 5 /etc/passwd |awk '{gsub(/bin|sbin/,
"ZZZZZZ"); print}'
root:x:0:0:root:/root:/ZZZZZZ/bash
ZZZZZZ:x:1:1:ZZZZZZ:/ZZZZZZ:/ZZZZZZ/nologin
daemon:x:2:2:daemon:/ZZZZZZ:/ZZZZZZ/nologin
adm:x:3:4:adm:/var/adm:/ZZZZZZ/nologin
lp:x:4:7:lp:/var/spool/lpd:/ZZZZZZ/nologin
Reverse order of lines (simulate "tac")
# head -n 5 /etc/passwd
root:x:0:0:root:/root:/bin/bash
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
# head -n 5 /etc/passwd |awk '{a[i++]=$0} END {for (j=i-1;
j>=0;) print a[j--] }'
lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
bin:x:1:1:bin:/bin:/sbin/nologin
root:x:0:0:root:/root:/bin/bash
If a line ends with a backslash, append the next line to it (fails if
there are multiple lines ending with backslash...)
awk '/\\$/ {sub(/\\$/,""); getline t; print $0 t;
next}; 1' filename
Search and sort the login names of all users
# awk -F ":" '{ print $1 | "sort" }'
/etc/passwd
adm
apache
avahi
bin
daemon
dbus
distcache
ftp
games
gdm
gopher
haldaemon
halt
lp
mail
mailnull
news
nfsnobody
nobody
nscd
Search the first 2 fields, in opposite order, of every line
# awk -F ':' '{print $2, $1}' /etc/passwd
x root
x bin
x daemon
References:-
http://www.xenuser.org/simple-awk-cheat-sheet
http://awk.info/?awk1line
http://www.thegeekstuff.com/
No comments:
Post a Comment