Bash recipes

Dec 05, 2016

devtip , bash , shell script

Using script name and path in the script

SCRIPT_DIR=$(dirname "$0")
SCRIPT_NAME=$(basename "$0")

Iterate N times

        for (( i=0; i<$N; i++ ));
        do
				# do whatever
        done

Retry using while

		while [ "$success" != "0" -a "$retryAttempt" -le "$MAX_RETRIES" ];
		do
				retryAttempt=$((retryAttempt + 1))
				sleep $SLEEP_INTERVAL
				## do some work that returns with exit code 0 on success
				## and non-zero exit code on error
				success="$?"
		done

Iterate over an array using indices

        # declare an array as below, default field separator is space
        myArray=("elem1" "elem2" "elem3")
        desiredValue="elem2"
        myArrayCount=${#myArray[@]}
        for (( i=0; i<${myArrayCount}; i++ ));
        do
           if [ "${myArray[$i]}" == "$desiredValue" ]; then
				# do something, for example just echo
				echo "Found ${myArray[$i]}"
           fi
        done

Append element to a variable

		myVar="elem1"
		myVar+="elem2"
		myVar+=" elem3"
		# now myVar is "elem1elem2 elem3"

Append element to an array

		myArray=()
		myArray+=("elem1")
		myArray+=("elem2")
		myArray+=("elem3")
		# now myArray is ("elem1" "elem2" "elem3")

Join an array of strings on a delimiter

function joinStrings(){
	local d=$1
	shift
	echo -n "$1"
	shift
	printf "%s" "${@/#/$d}"
}

# joinStrings ":::" a b c -> a:::b:::c

Taking array as argument for a function call

		function myFunc(){
                local array=("$@")
                local i=0
                for elem in ${array[@]};
                do
                        echo "elem[$i]=${elem}"
                        i=$((i + 1))
                done

                ## using indices
                elemsCount=${#array[@]}
                for (( i=0; i<${elemsCount}; i++ ));
                do
                        echo "elem[$i]=${array[$i]}"
                done
		}

		myFunc "elem1" "elem2" "elem3"

Taking fixed number of mixed arguments for a function call

		# takes 3 arguments, third argument is an array
		function myFunc(){
				local param1=$1
				local param2=$2
				shift
				shift
				local array=("$@")
				local i=0
				for elem in ${array[@]};
				do
						echo "elem[$i]=${elem}"
						i=$((i + 1))
				done

				## using indices
				elemsCount=${#array[@]}
				for (( i=0; i<${elemsCount}; i++ ));
				do
						echo "elem[$i]=${array[$i]}"
				done
				echo "param1=$param1"
				echo "param2=$param2"
		}

		myFunc "param1" "param2" "elem1" "elem2" "elem3"

Taking multiple arrays as arguments for a function call

function myFunc(){
  local array1=($1)
  local array2=($2)

  echo "array1=${array1[@]}"
  echo "array2=${array2[@]}"
}
# everything in between quotes (" ") is treated as a single argument irrespective of spaces
myFunc "$(joinStrings ' ' a b c)" "$(joinStrings ' ' d e f)" 

Log all script parameters (including the script name)

echo "$0 $@" >> ${LOG_FILE}

Trap interrupts and exit instead of continuing the loop

If there are loops in script logic and there are interrupts (like CTRL-C), the shell terminates the immediate operation but continues to the next iteration of the loop. To terminate the script completely, following statement must be used before executing the loop.


trap "echo Exited!; exit;" SIGINT SIGTERM

Running parallel operations in batches

Shell scripts don't have threads however they can leverage background jobs to run parallel operations. This works with both function calls and shell commands.


		function waitForBackgroundProcesses(){
			local pids=($@)
			local pidCount=${#pids[@]}
			for (( i=0; i<$pidCount; i++ ))
			do  
				local bgPid=${pids[$i]}
				wait $bgPid
				echo "$bgPid exited with code: $?" 
			done
		}

		function doBusyTask(){
			sleep $1
		}

		BATCH_SIZE=2
		pidsInBatch=()
		args=(2 4 5 2 1 3)
		for arg in ${args[@]};
		do
			doBusyTask $arg &
			processId="$!"
			pidsInBatch+=($processId)
			if [ "${#pidsInBatch[@]}" == "$BATCH_SIZE" ];then
				waitForBackgroundProcesses ${pidsInBatch[@]}
				pidsInBatch=()
			fi  
		done

However when parent script exits, it will leave the background jobs running. To terminate background jobs as well, we need to do another trap handling for script exit:


trap 'bgPids=$(jobs -pr); [ -z "$bgPids" ] || kill "$bgPids";' EXIT

Compare two files line by line ignoring the ordering

# returns 1 if the match doesn't hold true
# otherwise 0
#
# return code can be verified as exit code after
# the function call
function compareFile1InFile2(){
	local file1=$1
	local file2=$2
	local lineCountFile1=$(cat $file1|wc -l)
	local numberOfLinesMatched=$(grep -f "$file1" "$file2" |sort -u|wc -l)
	if [ "$numberOfLinesMatched" != "$lineCountFile1" ];then
		return 1
	else
		return 0
	fi
}

Diff between two variables

# example:
#	diffVar1Var2 "$var1" "$var2"
function diffVar1Var2(){
        local var1=$1
        local var2=$2
        diff <(echo "$var1") <(echo "$var2")
}

Show differences between two arrays

# space works as separator for elements inside a variable
# example:
#	diffXVar1Var2 "${array1[@]}" "${array2[@]}"
function diffXVar1Var2(){
        local var1=$1
        local var2=$2
        diff <(echo "$var1"|tr ' ' '\n'|sort) <(echo "$var2"|tr ' ' '\n'|sort)
}

Check if host is reachable on a given port

function isReachable(){
	local host=$1
	local port=$2
	nc -w 1 -z "$host" "$port"
}

Multi-line echo statements

This is very useful for generating a templated output

cat<<EOL
# can use shell variables as well in between the EOL,
# the variables would be expanded.
var1=$var1
EOL

Handling script inputs

# verify that parameters were passed
[ "$#" == "0" ] && showUsage
# ":" after a character indicates the parameter requires a value
# for example after a and p
while getopts "fa:p:" opt; do
	case "$opt" in
		f) flagEnabled="1" ;;
		a) argsAsArray+=("$OPTARG") ;;
		s) argAsParam="$OPTARG" ;;
		*) 
		   showUsage >&2
		   ;;
	esac
done
validateOptions

Log all script activity to a log file

# enable debugging, prints script lines to the stdout
set -x
# execute the rest of the script and append the output to 
# LOG_FILE 
exec &> >(tee -a "$LOG_FILE")
# write stderr to stdout for the rest of the script
exec 2>&1

Checking regex in if condition

# check if value of var starts with "MY_PREFIX"
if [[ "$var" =~ ^MY_PREFIX ]];then
	# do something
fi
# check if value of var doesn't start with "MY_PREFIX"
if [[ ! "$var" =~ ^MY_PREFIX ]];then
	# do something
fi
# check if value of var contains "MY_VALUE"
if [[ "$var" =~ MY_VALUE ]];then
	# do something
fi

Sharing variables and utility functions across multiple scripts

Define shared variables and utility functions in logically separate scripts and include them in every script where needed like below.

source path/library1.sh
. path/library2.sh

Reading a file line by line

OLDIFS=$IFS
# changing field separator so that we can read line by line
IFS=$'\n'
i=1
allLines=$(cat ${FILE})
for line in $allLines;
do
	# changing field separator back so that rest of the logic works normally
	IFS=$OLDIFS
	echo "line $i: $line"
	i=$((i + 1))
done

Links Awards & recognitions Resume Personal Technical Miscellaneous My Blog
 
Last Updated: Dec 05 2016