Bash recipes
Dec 05, 2016
devtip , bash , shell script
Replace '\n' or '\r\n' with newline in a file
awk '{ gsub(/\\n/,"\n"); gsub(/\\r/,""); }1' target_file.txt
OR
sed 's/\\n/'$'\\n''/g' target_file.txt | sed 's/\\r//g'
#
Remove everything except printable characters (required sometimes when copying special character outputs)
sed $'s/[^[:print:]\t]//g' file.txt
Disable expansion of ! (exclamation mark)
# Option 1 - Disable Shell History set +H # Option 2 - Use single quotes around `!` to prevent expansion OR `\` to escape it # Below is an exmaple usecase for maven based build use case mvn install -Phive -Pyarn -Phive-thriftserver -pl '!':spark-yarn_2.12,'!':spark-hive-thriftserver_2.12,'!':spark-core_2.12,'!':spark-sql_2.12,'!':spark-mllib_2.12,'!':spark-hive_2.12,'!':spark-streaming-kafka-0-10_2.12 mvn install -Phive -Pyarn -Phive-thriftserver -pl \!:spark-yarn_2.12,\!:spark-hive-thriftserver_2.12,\!:spark-core_2.12,\!:spark-sql_2.12,\!:spark-mllib_2.12,\!:spark-hive_2.12,\!:spark-streaming-kafka-0-10_2.12
Using script name and path in the script
SCRIPT_DIR=$(dirname "$0") SCRIPT_NAME=$(basename "$0")
Iterate N times
for (( i=0; i<$N; i++ ));
do
# do whatever
done
Retry using while
while [ "$success" != "0" -a "$retryAttempt" -le "$MAX_RETRIES" ]; do retryAttempt=$((retryAttempt + 1)) sleep $SLEEP_INTERVAL ## do some work that returns with exit code 0 on success ## and non-zero exit code on error success="$?" done
Iterate over an array using indices
# declare an array as below, default field separator is space
myArray=("elem1" "elem2" "elem3")
desiredValue="elem2"
myArrayCount=${#myArray[@]}
for (( i=0; i<${myArrayCount}; i++ ));
do
if [ "${myArray[$i]}" == "$desiredValue" ]; then
# do something, for example just echo
echo "Found ${myArray[$i]}"
fi
done
Append element to a variable
myVar="elem1" myVar+="elem2" myVar+=" elem3" # now myVar is "elem1elem2 elem3"
Append element to an array
myArray=()
myArray+=("elem1")
myArray+=("elem2")
myArray+=("elem3")
# now myArray is ("elem1" "elem2" "elem3")
Join an array of strings on a delimiter
function joinStrings(){
local d=$1
shift
echo -n "$1"
shift
printf "%s" "${@/#/$d}"
}
# joinStrings ":::" a b c -> a:::b:::c
Taking array as argument for a function call
function myFunc(){
local array=("$@")
local i=0
for elem in ${array[@]};
do
echo "elem[$i]=${elem}"
i=$((i + 1))
done
## using indices
elemsCount=${#array[@]}
for (( i=0; i<${elemsCount}; i++ ));
do
echo "elem[$i]=${array[$i]}"
done
}
myFunc "elem1" "elem2" "elem3"
Taking fixed number of mixed arguments for a function call
# takes 3 arguments, third argument is an array
function myFunc(){
local param1=$1
local param2=$2
shift
shift
local array=("$@")
local i=0
for elem in ${array[@]};
do
echo "elem[$i]=${elem}"
i=$((i + 1))
done
## using indices
elemsCount=${#array[@]}
for (( i=0; i<${elemsCount}; i++ ));
do
echo "elem[$i]=${array[$i]}"
done
echo "param1=$param1"
echo "param2=$param2"
}
myFunc "param1" "param2" "elem1" "elem2" "elem3"
Taking multiple arrays as arguments for a function call
function myFunc(){
local array1=($1)
local array2=($2)
echo "array1=${array1[@]}"
echo "array2=${array2[@]}"
}
# everything in between quotes (" ") is treated as a single argument irrespective of spaces
myFunc "$(joinStrings ' ' a b c)" "$(joinStrings ' ' d e f)"
Log all script parameters (including the script name)
echo "$0 $@" >> ${LOG_FILE}
Trap interrupts and exit instead of continuing the loop
If there are loops in script logic and there are interrupts (like CTRL-C), the shell terminates the immediate operation but continues to the next iteration of the loop. To terminate the script completely, following statement must be used before executing the loop.
trap "echo Exited!; exit;" SIGINT SIGTERM
Running parallel operations in batches
Shell scripts don't have threads however they can leverage background jobs to run parallel operations. This works with both function calls and shell commands.
function waitForBackgroundProcesses(){
local pids=($@)
local pidCount=${#pids[@]}
for (( i=0; i<$pidCount; i++ ))
do
local bgPid=${pids[$i]}
wait $bgPid
echo "$bgPid exited with code: $?"
done
}
function doBusyTask(){
sleep $1
}
BATCH_SIZE=2
pidsInBatch=()
args=(2 4 5 2 1 3)
for arg in ${args[@]};
do
doBusyTask $arg &
processId="$!"
pidsInBatch+=($processId)
if [ "${#pidsInBatch[@]}" == "$BATCH_SIZE" ];then
waitForBackgroundProcesses ${pidsInBatch[@]}
pidsInBatch=()
fi
done
However when parent script exits, it will leave the background jobs running. To terminate background jobs as well, we need to do another trap handling for script exit:
trap 'bgPids=$(jobs -pr); [ -z "$bgPids" ] || kill "$bgPids";' EXIT
Compare two files line by line ignoring the ordering
# returns 1 if the match doesn't hold true
# otherwise 0
#
# return code can be verified as exit code after
# the function call
function compareFile1InFile2(){
local file1=$1
local file2=$2
local lineCountFile1=$(cat $file1|wc -l)
local numberOfLinesMatched=$(grep -f "$file1" "$file2" |sort -u|wc -l)
if [ "$numberOfLinesMatched" != "$lineCountFile1" ];then
return 1
else
return 0
fi
}
Diff between two variables
# example:
# diffVar1Var2 "$var1" "$var2"
function diffVar1Var2(){
local var1=$1
local var2=$2
diff <(echo "$var1") <(echo "$var2")
}
Show differences between two arrays
# space works as separator for elements inside a variable
# example:
# diffXVar1Var2 "${array1[@]}" "${array2[@]}"
function diffXVar1Var2(){
local var1=$1
local var2=$2
diff <(echo "$var1"|tr ' ' '\n'|sort) <(echo "$var2"|tr ' ' '\n'|sort)
}
Check if host is reachable on a given port
function isReachable(){
local host=$1
local port=$2
nc -w 1 -z "$host" "$port"
}
Multi-line echo statements
This is very useful for generating a templated output
cat<<EOL # can use shell variables as well in between the EOL, # the variables would be expanded. var1=$var1 EOL
Handling script inputs
# verify that parameters were passed
[ "$#" == "0" ] && showUsage
# ":" after a character indicates the parameter requires a value
# for example after a and p
while getopts "fa:p:" opt; do
case "$opt" in
f) flagEnabled="1" ;;
a) argsAsArray+=("$OPTARG") ;;
s) argAsParam="$OPTARG" ;;
*)
showUsage >&2
;;
esac
done
validateOptions
Log all script activity to a log file
# enable debugging, prints script lines to the stdout set -x # execute the rest of the script and append the output to # LOG_FILE exec &> >(tee -a "$LOG_FILE") # write stderr to stdout for the rest of the script exec 2>&1
Checking regex in if condition
# check if value of var starts with "MY_PREFIX" if [[ "$var" =~ ^MY_PREFIX ]];then # do something fi # check if value of var doesn't start with "MY_PREFIX" if [[ ! "$var" =~ ^MY_PREFIX ]];then # do something fi # check if value of var contains "MY_VALUE" if [[ "$var" =~ MY_VALUE ]];then # do something fi
Sharing variables and utility functions across multiple scripts
Define shared variables and utility functions in logically separate scripts and include them in every script where needed like below.
source path/library1.sh . path/library2.sh
Reading a file line by line
OLDIFS=$IFS
# changing field separator so that we can read line by line
IFS=$'\n'
i=1
allLines=$(cat ${FILE})
for line in $allLines;
do
# changing field separator back so that rest of the logic works normally
IFS=$OLDIFS
echo "line $i: $line"
i=$((i + 1))
done