12 — Error Handling Architecture: Strict Mode, Traps & Logging Internals
set -e: Exit on Error — The Full Picture
bash
# ── set -e: exit immediately when a command fails (non-zero exit) ──
set -e
# ── When -e triggers ──
false # ← script exits here (false returns 1)
echo "reached" # never reached
# ── When -e does NOT trigger (tested commands) ──
if false; then echo "if"; fi # OK — tested by if
false && echo "and" # OK — tested by &&
false || echo "or" # OK — tested by ||
! false # OK — negated
false; true # ✗ exits on false (untested command)
false || true # OK — false is tested by ||
# ── The pipeline trap (needs pipefail) ──
false | true # exit 0 (true is last) — -e doesn't catch false!
set -o pipefail
false | true # exit 1 (pipefail: any failure propagates) — -e catches it
# ── Functions and -e ──
maybe_fail() { return 1; }
maybe_fail # ✗ exits (simple command)
maybe_fail || true # OK — tested by ||
if maybe_fail; then ...; fi # OK — tested by if
# ── `local` masks command failures ──
set -e
bad() {
local x=$(false) # ✗ masked! local's exit status is 0 (always succeeds)
echo "reached" # "reached" — false's failure was eaten by local
}
bad
good() {
local x
x=$(false) # ✓ not masked — false's status propagates
echo "reached" # never reached — script exits
}
good
# ── Command substitution and -e (Bash 4.4+) ──
# Bash < 4.4: x=$(false); echo "ok" → "ok" (substitution failure masked!)
# Bash 4.4+: x=$(false); echo "ok" → exits (inherit_errexit makes it propagate)
shopt -s inherit_errexit # Bash 4.4+: subshells inherit set -e (default: OFF!)
# ── Subshell vs pipeline ──
set -e
(echo "subshell"; false) # ✗ subshell fails, parent's set -e catches it? YES.
(echo "subshell"; false) | cat # subshell exits 1, but pipefail needed for parent to catch
set -u: Unset Variable Guard
bash
# ── set -u: unset variable expansion is a FATAL error ──
set -u
echo "$UNDEFINED" # ✗ bash: UNDEFINED: unbound variable — script exits
echo "${UNDEFINED:-}" # OK — :- provides empty default
echo "${UNDEFINED:-default}" # OK — provides "default"
echo "${UNDEFINED:=default}" # OK — provides and ASSIGNS "default"
# ── Arrays and set -u ──
declare -a arr=()
echo "${arr[@]}" # Bash 4.4+: OK (empty array). Bash < 4.4: ✗ unbound!
echo "${arr[@]:-}" # OK — safe on all versions (the :- handles empty)
declare -A map=()
echo "${map[key]}" # ✗ if key doesn't exist: unbound (some versions)
echo "${map[key]:-}" # OK — safe access pattern
# ── ${var:-} is the idiom for "may be unset" ──
# Use it for optional variables, especially with set -u:
config_value="${CONFIG_VALUE:-default}"
# If CONFIG_VALUE is set: use it. If unset: use "default" without error.
set -o pipefail: Pipeline Failure Propagation
bash
# ── Without pipefail: only the LAST command's exit status matters ──
false | true # exit 0 — false is masked!
grep "x" file | head -1 # if grep fails (file not found), head still succeeds → exit 0!
# ── With pipefail: the pipeline fails if ANY command fails ──
set -o pipefail
false | true # exit 1 — false's failure propagates
grep "x" file | head -1 # if grep fails, pipeline exits 1
# ── pipefail returns the rightmost non-zero exit status ──
false | true # without pipefail: 0. With pipefail: 1 (false's status)
true | false # without pipefail: 1. With pipefail: 1 (false's status)
false | false # exit 1 (the rightmost non-zero, which is 1 for false)
(exit 3) | (exit 2) # exit 2 (rightmost non-zero)
(exit 2) | (exit 3) # exit 3 (rightmost non-zero)
# ── Production pattern: always use the full strict mode ──
set -euo pipefail
# -e: exit on error
# -u: error on unset variable
# -o pipefail: pipeline fails if any command fails
# This is the MINIMUM for a robust script.
trap ERR: Error Logging and Recovery
bash
#!/usr/bin/env bash
set -Eeuo pipefail # -E: ERR trap inherits into functions and subshells
# ── trap ERR fires when a command fails (with set -e) ──
# It fires BEFORE the script exits — you can log, clean up, or even recover.
on_error() {
local exit_code=$? # capture FIRST — any command overwrites $?
local line=$1
local func=${FUNCNAME[1]:-main}
local cmd="${BASH_COMMAND:-?}"
# Log the error with full context
printf '[%s] ERROR: %s failed in %s() at line %d (exit %d)\n' \
"$(date -Iseconds)" "$cmd" "$func" "$line" "$exit_code" >&2
# Optional: print the call stack
local i
for ((i = 1; i < ${#FUNCNAME[@]} - 1; i++)); do
printf ' → %s() at %s:%d\n' \
"${FUNCNAME[i]}" \
"$(basename "${BASH_SOURCE[i]}")" \
"${BASH_LINENO[i]}" >&2
done
# Clean up resources
[[ -n "${tmpdir:-}" ]] && rm -rf -- "$tmpdir"
exit "$exit_code" # re-exit with the original code
}
trap 'on_error $LINENO' ERR
# ── Test it ──
failing_function() {
false # triggers ERR trap
}
failing_function # trap fires: logs the error, cleans up, exits
trap EXIT: Cleanup on Any Exit Path
bash
# ── trap EXIT fires on ANY exit: normal, set -e abort, signal (if no specific trap) ──
# ── Clean up temp files ──
tmpfile=$(mktemp)
tmpdir=$(mktemp -d)
trap 'rm -f "$tmpfile"; rm -rf "$tmpdir"' EXIT
# EXIT fires on:
# - normal end of script
# - exit command
# - set -e abort (command failure)
# - signals (INT, TERM) if no specific trap for them
# It fires in the exiting shell — so $tmpfile is still in scope.
# ── Preserve exit code ──
cleanup() {
local exit_code=$? # capture FIRST — any command overwrites $?
[[ -n "${tmpfile:-}" ]] && rm -f "$tmpfile"
[[ -n "${tmpdir:-}" ]] && rm -rf "$tmpdir"
if ((exit_code != 0)); then
echo "script failed (exit $exit_code)" >&2
fi
exit "$exit_code" # preserve the original exit code
}
trap cleanup EXIT
# ── ⚠️ Multiple trap EXIT overwrite — last one wins ──
trap 'echo first' EXIT
trap 'echo second' EXIT # overwrites first — only "second" runs
# Combine all cleanup in ONE function:
trap 'cleanup_all' EXIT
# ── Combine ERR and EXIT ──
# ERR fires on command failure (for logging). EXIT fires on any exit (for cleanup).
trap 'on_error $LINENO' ERR
trap 'cleanup' EXIT
# On a set -e abort: ERR fires first (logs), then EXIT fires (cleans up), then exits.
Exit Code Conventions
bash
# ── Standard exit codes ──
exit 0 # success
exit 1 # general failure
exit 2 # usage error / misuse (shell builtin convention)
exit 126 # command found but not executable (permission denied)
exit 127 # command not found
exit 128 # invalid exit argument (exit 300 → 44, modulo 256)
exit 130 # terminated by SIGINT (128 + 2) — Ctrl-C
exit 137 # killed by SIGKILL (128 + 9) — kill -9
exit 143 # terminated by SIGTERM (128 + 15) — kill
# ── Signal exit codes: 128 + N (where N is the signal number) ──
# SIGINT = 2 → exit 130
# SIGKILL = 9 → exit 137
# SIGTERM = 15 → exit 143
# SIGSEGV = 11 → exit 139
# ── `exit` wraps to 0-255 ──
exit 300 # actual exit code: 44 (300 % 256)
exit -1 # actual exit code: 255 (-1 % 256 in bash)
exit 256 # actual exit code: 0 (256 % 256 = 0 — looks like success!)
# ── Define exit code constants ──
readonly EXIT_SUCCESS=0
readonly EXIT_FAILURE=1
readonly EXIT_USAGE=2
readonly EXIT_MISSING_DEP=3
readonly EXIT_NOT_FOUND=127
# Usage:
if ! command -v git &>/dev/null; then
echo "git not found" >&2
exit "$EXIT_NOT_FOUND"
fi
Structured Logging
bash
#!/usr/bin/env bash
# ── Log levels ──
declare -i LOG_LEVEL=${LOG_LEVEL:-3} # 0=trace 1=debug 2=info 3=warn 4=error
declare -A LOG_LEVELS=(
[TRACE]=0 [DEBUG]=1 [INFO]=2 [WARN]=3 [ERROR]=4 [FATAL]=5
)
declare -A LOG_COLORS=(
[TRACE]=$'\e[90m' # gray
[DEBUG]=$'\e[36m' # cyan
[INFO]=$'\e[32m' # green
[WARN]=$'\e[33m' # yellow
[ERROR]=$'\e[31m' # red
[FATAL]=$'\e[31m' # red
)
readonly RESET=$'\e[0m'
# ── Check if stderr is a terminal (for colors) ──
if [[ -t 2 ]]; then
USE_COLOR=true
else
USE_COLOR=false
fi
# ── Log function ──
log() {
local level=$1; shift
local message="$*"
# Check level threshold
(( LOG_LEVELS[$level] >= LOG_LEVEL )) || return 0
# Format: [ISO8601] [LEVEL] message
local timestamp
timestamp=$(date -Iseconds)
if $USE_COLOR; then
printf '%s[%s]%s %s[%-5s]%s %s\n' \
"${LOG_COLORS[$level]}" "$timestamp" "$RESET" \
"${LOG_COLORS[$level]}" "$level" "$RESET" \
"$message" >&2
else
printf '[%s] [%-5s] %s\n' "$timestamp" "$level" "$message" >&2
fi
}
# ── Convenience functions ──
log_trace() { log TRACE "$@"; }
log_debug() { log DEBUG "$@"; }
log_info() { log INFO "$@"; }
log_warn() { log WARN "$@"; }
log_error() { log ERROR "$@"; }
log_fatal() { log ERROR "$@"; exit 1; }
# ── Usage ──
log_info "starting deployment"
log_debug "config loaded: ${#config[@]} entries"
log_warn "deprecated option --old-flag"
log_error "failed to connect to database"
log_fatal "cannot continue without database" # logs ERROR, then exits 1
Anti-Pattern: Silent Failures
bash
# ❌ NAIVE — commands fail silently without set -e
mkdir "$dir" # if this fails (permission, exists), script continues silently
cd "$dir" # if mkdir failed, cd fails, script runs in wrong dir
cp file "$dir/" # copies to wrong location, no error
rm -rf * # removes wrong files!
# ✅ CORRECT — strict mode + explicit checks
set -euo pipefail
mkdir -p "$dir" || die "can't create $dir"
cd "$dir" || die "can't cd to $dir"
cp "$src" "$dir/" || die "can't copy $src"
# ❌ NAIVE — pipeline failure masked
grep "pattern" file | head -10
# If file doesn't exist: grep fails, but head succeeds (exit 0) — silent failure!
# ✅ CORRECT — pipefail catches it
set -o pipefail
grep "pattern" file | head -10
# grep fails → pipeline fails → set -e exits
# ❌ NAIVE — unset variable silently becomes empty
echo "config: $CONFIG_PATH" # if CONFIG_PATH is unset → "config: "
cp "$CONFIG_PATH" /etc/ # copies "" → "cp: missing destination"
# ✅ CORRECT — set -u catches it
set -u
echo "config: $CONFIG_PATH" # bash: CONFIG_PATH: unbound variable — exits
# Or with default:
echo "config: ${CONFIG_PATH:-/etc/default.conf}"
Production Pattern: Dependency Checks
bash
#!/usr/bin/env bash
set -euo pipefail
# ── Check for required commands ──
require() {
local cmd=$1
if ! command -v "$cmd" &>/dev/null; then
echo "ERROR: required command not found: $cmd" >&2
echo "Install: brew install $cmd (macOS) or apt install $cmd (Debian)" >&2
exit 127
fi
}
# ── Check for required files ──
require_file() {
local file=$1
[[ -f "$file" ]] || { echo "ERROR: required file not found: $file" >&2; exit 2; }
}
# ── Check for required env vars ──
require_env() {
local var=$1
[[ -n "${!var:-}" ]] || { echo "ERROR: required env var not set: $var" >&2; exit 2; }
}
# ── Check for required dirs ──
require_dir() {
local dir=$1
[[ -d "$dir" ]] || { echo "ERROR: required directory not found: $dir" >&2; exit 2; }
}
# ── Check all dependencies at startup ──
require git
require curl
require jq
require_env DATABASE_URL
require_file config.env
echo "all dependencies satisfied"
die: The Fatal Error Function
bash
# ── die: print error to stderr and exit 1 ──
die() {
printf '[%s] FATAL: %s\n' "$(date -Iseconds)" "$*" >&2
exit 1
}
# ── Usage patterns ──
[[ $# -ge 1 ]] || die "missing required argument. Usage: $0 <input>"
[[ -f "$1" ]] || die "file not found: $1"
command -v docker &>/dev/null || die "docker not installed"
cd "$BUILD_DIR" || die "can't cd to $BUILD_DIR"
# ── die with context (include the failing command) ──
die_cmd() {
printf '[%s] FATAL: '%s' failed\n' "$(date -Iseconds)" "$1" >&2
exit 1
}
# ── die with exit code ──
die_code() {
local code=$1; shift
printf '[%s] ERROR: %s\n' "$(date -Iseconds)" "$*" >&2
exit "$code"
}
die_code 2 "invalid usage" # exit 2 (usage error)
Debugging Techniques
bash
# ── set -x: trace every command ──
set -x # print each command BEFORE execution (with + prefix)
echo "hello" # + echo hello
set +x # turn off
# ── PS4: customize the trace prompt ──
PS4='+ $LINENO: '
set -x
# + 5: echo hello
set +x
# ── PS4 with full context ──
PS4='+ ${BASH_SOURCE[0]}:${LINENO}: ${FUNCNAME[0]:-main}: '
set -x
# + script.sh:5: main: echo hello
set +x
# ── Trace from the command line (no edit needed) ──
bash -x script.sh # trace entire script
bash -x script.sh 2>trace.log # trace to a file
# ── Trace only a section ──
{
set -x
# ... section to debug ...
set +x
} 2>&1 # capture trace
# ── set -v: print input lines as read (before expansion) ──
set -xv # -v: raw input lines, -x: expanded commands — shows both
# ── declare -p: inspect variable state ──
declare -p PATH # print PATH's definition (shows quoting, type)
declare -p arr # print array definition (for debugging)
declare -p config # print associative array
# ── Print variables for debugging ──
debug() {
printf '[DEBUG] %s\n' "$*" >&2
}
debug "count=$count, file=$file, mode=$mode"
Dry-Run Pattern
bash
#!/usr/bin/env bash
dry_run=false
# ── Parse --dry-run flag ──
while [[ $# -gt 0 ]]; do
case "$1" in
--dry-run) dry_run=true; shift ;;
*) break ;;
esac
done
# ── run wrapper: execute or echo ──
run() {
if $dry_run; then
printf 'DRY RUN: %s\n' "$*" >&2
else
"$@"
fi
}
# ── Usage ──
run rm -f "$file"
run cp "$src" "$dst"
run systemctl restart "$service"
# With --dry-run: prints each command without executing.
# Without --dry-run: executes each command.
💡 Tips & Tricks
bash
# ── `set -e` with `grep` (no match is not an error) ──
grep "pattern" file.txt || true # no match → exit 1 → || true → exit 0
# Or:
if grep -q "pattern" file.txt; then ...; fi # tested — no set -e issue
# ── `set -e` with `cd` ──
cd "$dir" || exit 1 # if cd fails, exit explicitly (set -e also catches it)
# But: cd in a subshell doesn't affect the parent: ( cd dir; cmd )
# ── `trap` DEBUG for step-by-step tracing ──
trap 'echo "line $LINENO: $BASH_COMMAND" >&2' DEBUG
# Fires BEFORE every command — very verbose but powerful for debugging.
# ── `trap` RETURN (fires when a function returns) ──
my_func() {
echo "doing work"
}
trap 'echo "my_func returned (status $?)"' RETURN
my_func # doing work\nmy_func returned (status 0)
# ── Capture and re-raise in trap ──
# If you want to do work in the trap but still exit with the original code:
trap 'cleanup; exit $?' EXIT # cleanup runs, then exit with the original status
# ── `ERR` trap with function context ──
# $BASH_LINENO[0] is the caller's line number (where the function was called)
# $LINENO is the current line (inside the function)
trap 'echo "error at ${BASH_LINENO[0]} (called from ${FUNCNAME[1]:-main})"' ERR
⚠️ Edge Cases & Gotchas
bash
# ── `set -e` doesn't catch everything ──
# Commands in if/while/&&/|| are "tested" — don't trigger -e.
# `local x=$(false)` masks the failure (local always returns 0).
# `$(false)` without inherit_errexit doesn't propagate (Bash < 4.4).
# ── `set -u` and arrays ──
# Bash < 4.4: "${arr[@]}" on an empty array errors. Use "${arr[@]:-}".
# `[[ -v arr[@] ]]` checks if array has any elements (Bash 4.2+).
# ── `pipefail` with `grep` ──
# grep returns 1 if no match. With pipefail: `grep ... | head` fails if no match.
# Fix: `grep ... | head -1 || true` or use `grep -q ... && ...`.
# ── `trap ERR` doesn't fire on syntax errors ──
# Syntax errors abort before any trap runs.
# `trap ERR` fires on command failures (with set -e), not parse errors.
# ── `exit` in a subshell doesn't exit the script ──
(exit 1) # subshell exits 1, parent continues
# Check: ( exit 1 ) || die "subshell failed"
# ── `trap EXIT` and signals ──
# EXIT fires on normal exit and set -e abort.
# For signals (INT, TERM), you need a specific trap: trap 'cleanup' INT TERM
# Or: trap 'cleanup' EXIT INT TERM (EXIT also fires after a signal trap)
# ── `inherit_errexit` is off by default ──
# Subshells don't inherit set -e by default (Bash 4.4+).
# Enable: shopt -s inherit_errexit
# Without it: ( false; echo "reached" ) → "reached" (subshell didn't exit!)
# ── Exit code wrapping ──
# exit 300 → exit 44 (300 % 256). exit 256 → exit 0 (looks like success!).
# Always use 0-255 for exit codes.
🧠 Quick Quiz
Why does this script print "reached" instead of exiting?
bash
set -e
f() {
local result=$(false)
echo "reached"
}
f
Answer
local result=$(false) masks the failure because local is a command that always returns exit status 0. The $(false) runs in a subshell and fails, but local's exit status (0) overrides it.
With set -e, the script sees exit 0 from local and doesn't exit.
The fix: separate declaration and assignment:
bash
set -e
f() {
local result
result=$(false) # assignment's exit status is false's (1) → set -e exits
echo "reached" # never reached
}
f
The lesson: local x=$(command) always returns 0 (local's status), masking the command's failure. With set -e, use local x; x=$(command) so the command's exit status propagates.