07 — File System, Redirection Internals & Safe I/O Patterns

# ── Every process has 3 standard file descriptors (fds) ──
# fd 0 = stdin  (input)
# fd 1 = stdout (output)
# fd 2 = stderr (errors)
# Additional fds (3-9) can be opened manually for advanced patterns.

# ── Redirection operators ──
cmd > file           # fd 1 (stdout) → file (truncate)
cmd >> file          # fd 1 → file (append)
cmd 2> file          # fd 2 (stderr) → file
cmd 2>> file         # fd 2 → file (append)
cmd &> file          # fd 1 + fd 2 → file (Bash shorthand)
cmd > file 2>&1     # fd 1 → file, then fd 2 → fd 1 (both to file)
cmd 2>&1 > file     # ⚠️ fd 2 → fd 1 (old stdout = terminal), then fd 1 → file. stderr stays on terminal!
cmd > /dev/null 2>&1  # discard stdout and stderr
cmd < file           # fd 0 (stdin) ← file
cmd <> file          # fd 0 ← file (open for read AND write — bidirectional)
cmd >&-              # close fd 1 (stdout)
cmd 2>&-             # close fd 2 (stderr)

# ── Opening custom fds ──
exec 3> /tmp/log.txt   # open fd 3 for writing to /tmp/log.txt (persists in the shell)
echo "log line" >&3    # write to fd 3
exec 3>&-              # close fd 3 (good practice — fd leak prevention)

exec 4< input.txt      # open fd 4 for reading from input.txt
read -r line <&4      # read from fd 4
exec 4<&-              # close fd 4

# ── Save and restore stdout ──
exec 5>&1              # save current stdout to fd 5
exec > /tmp/output.txt # redirect stdout to file
echo "goes to file"
exec 1>&5 5>&-         # restore stdout from fd 5, close fd 5
echo "goes to terminal"

# ── Redirections are processed LEFT-TO-RIGHT ──

# ❌ WRONG — stderr stays on terminal
cmd 2>&1 > output.txt
# Step 1: 2>&1 → fd 2 points to where fd 1 CURRENTLY points (terminal)
# Step 2: > output.txt → fd 1 points to output.txt
# Result: stdout → file, stderr → terminal (NOT what you wanted!)

# ✅ CORRECT — stdout first, then stderr follows
cmd > output.txt 2>&1
# Step 1: > output.txt → fd 1 points to output.txt
# Step 2: 2>&1 → fd 2 points to where fd 1 CURRENTLY points (output.txt)
# Result: both stdout and stderr → output.txt ✓

# ✅ CORRECT — Bash shorthand
cmd &> output.txt     # both → output.txt (cleaner, but Bash-only, not POSIX)
cmd &>> output.txt    # both → output.txt (append)

# ── Piping stderr only ──
cmd 2>&1 >/dev/null | grep "error"
# 2>&1: stderr → old stdout (which will be the pipe, after the pipe is set up... no, wait)
# >/dev/null: stdout → /dev/null
# The pipe `|` connects cmd's stdout to grep's stdin.
# 2>&1 makes stderr go to stdout, but stdout is redirected to /dev/null AFTER 2>&1.
# Actually: the pipe is set up first (stdout → pipe), then 2>&1 → stderr goes to the pipe,
# then >/dev/null → stdout goes to /dev/null (but stderr still goes to the pipe).
# Result: only stderr reaches grep, stdout is discarded. ✓

# ── Swap stdout and stderr ──
cmd 3>&1 1>&2 2>&3 3>&-
# 3>&1: save old stdout to fd 3
# 1>&2: stdout → old stderr
# 2>&3: stderr → old stdout (from fd 3)
# 3>&-: close fd 3
# Result: stdout and stderr are swapped.

# ── Heredoc: multi-line stdin ──
cat <<EOF
Line 1: $USER          # variable expansion (unquoted delimiter)
Line 2: $(date)
EOF

# ── Quoted delimiter: no expansion ──
cat <<'EOF'
$USER is literal       # no expansion — printed as-is
$(date) too
EOF

# ── Strip leading tabs (NOT spaces) with <<- ──
cat <<-EOF
        Indented line      # tabs stripped, spaces preserved
        $USER              # expansion works (unquoted)
EOF
# ⚠️ <<- strips TABS only. Configure editor to use tabs for heredoc indentation.
# Spaces are NOT stripped — the heredoc will include leading spaces.

# ── Heredoc to a variable ──
config=$(cat <<EOF
host=localhost
port=8080
EOF
)
echo "$config"

# ── Heredoc to a file ──
cat <<EOF > config.txt
host=localhost
port=8080
EOF

# ── Herestring: single string as stdin ──
read -r line <<< "hello world"     # line="hello world" (single string)
awk '{print $1}' <<< "hello world"  # hello (single string as stdin)
grep "pattern" <<< "$input_string"  # search in a variable

# ── Heredoc with command substitution ──
cat <<EOF
Date: $(date)
User: $USER
Home: $HOME
Files: $(ls | wc -l)
EOF

# ── Heredoc as a function body (write a file from a script) ──
write_config() {
    cat <<EOF > "$1"
# Generated by $0 on $(date -Iseconds)
host=${2:-localhost}
port=${3:-8080}
EOF
}
write_config config.txt "prod.example.com" 9090

# ── Basic search ──
find . -name "*.py"               # by name (case-sensitive)
find . -iname "*.py"              # case-insensitive
find . -type f                    # files only
find . -type d                    # directories only
find . -type l                    # symlinks only
find . -type f -name "*.py"       # combine: files AND name matches

# ── By time ──
find . -mtime -7                  # modified in last 7 days
find . -mtime +30                 # modified more than 30 days ago
find . -mmin -60                  # modified in last 60 minutes
find . -mmin +60                  # modified more than 60 min ago
find . -newer reference.txt       # newer than reference file
find . -newermt "2024-01-01"     # newer than specific date
find . -newermt "2024-01-01" ! -newermt "2024-02-01"  # between two dates

# ── By size ──
find . -size +10M                 # larger than 10MB
find . -size -1k                  # smaller than 1KB
find . -size +100M -size -1G     # between 100MB and 1GB
find . -empty                     # empty files and dirs

# ── By permissions ──
find . -perm 644                  # exactly 644
find . -perm -644                 # AT LEAST 644 (has rw-r--r--)
find . -perm /644                 # ANY of 644 bits set (r for user OR w for group OR r for other)
find . -perm -u+x                # executable by user

# ── Boolean operators ──
find . -name "*.py" -o -name "*.sh"          # OR (match .py OR .sh)
find . -name "*.py" -a -type f               # AND (default, but explicit with -a)
find . -not -name "*.py"                     # NOT (files not ending in .py)
find . \( -name "*.py" -o -name "*.sh" \) -type f  # group with parentheses (escape!)

# ── Actions: -exec vs -delete ──
find . -name "*.tmp" -delete              # delete matching files (fast, no subprocess)
find . -name "*.tmp" -exec rm {} \;       # run rm for EACH file (one subprocess per file)
find . -name "*.tmp" -exec rm {} +         # run rm with ALL files at once (batch — faster)
find . -name "*.log" -exec gzip {} +       # gzip all .log files (batch)
find . -name "*.py" -exec wc -l {} +      # count lines in all .py files (batch — single wc call)

# ── -exec ... \; vs -exec ... + ──
# \; — runs command ONCE PER FILE (one subprocess per file — slow for many files)
# +  — runs command ONCE with ALL files as args (batch — fast, like xargs)
# Use + when the command accepts multiple file args (rm, chmod, grep, wc).
# Use \; when the command takes one file at a time (or when order matters).

# ── -exec with shell features ──
# -exec runs a command directly, NOT through a shell — no pipes, no redirection.
# For shell features, use `bash -c`:
find . -name "*.py" -exec bash -c 'echo "$1: $(wc -l < "$1") lines"' _ {} \;
# _ is $0 (unused), {} becomes $1

# ── -execdir (safer than -exec) ──
find . -name "*.log" -execdir rm {} \;
# -execdir runs the command from the directory containing the file (CWD changes).
# Safer: avoids race conditions where {} could be a path with special chars.

# ❌ NAIVE — breaks on filenames with spaces, newlines, or special chars
for file in $(find . -name "*.txt"); do
    cp "$file" /backup/
done
# $(find ...) captures all output as one string, word-splits on spaces.
# "my file.txt" becomes "my" and "file.txt" — two iterations, both wrong.

# ❌ ALSO BAD — find | xargs (without -0)
find . -name "*.txt" | xargs cp -t /backup/
# xargs splits on whitespace too — same problem.

# ✅ CORRECT — null-delimited (handles ALL filenames, including newlines)
while IFS= read -r -d '' file; do
    # -d '' = read until null byte
    # -print0 = separate filenames with null bytes
    # This is the ONLY safe way to handle arbitrary filenames
    cp "$file" /backup/
done < <(find . -name "*.txt" -print0)

# ✅ CORRECT (xargs) — null-delimited
find . -name "*.txt" -print0 | xargs -0 cp -t /backup/

# ✅ CORRECT (exec batch) — no need for null delimiters, find handles it
find . -name "*.txt" -exec cp {} /backup/ +

# ── Why null bytes are the only safe delimiter ──
# Filenames can contain ANY byte EXCEPT null (\x00):
#   - spaces:  "my file.txt"
#   - newlines: "my\nfile.txt"  (yes, this is legal on Unix)
#   - dashes:  "-file.txt" (looks like an option flag)
#   - stars:   "*.txt"
# Null bytes (\x00) are the ONLY character not allowed in filenames.
# So -print0 + read -d '' is the only universally safe approach.

# ── Octal notation: each digit = owner, group, other ──
# 7 = rwx (4+2+1)    6 = rw- (4+2)    5 = r-x (4+1)    4 = r-- (4)    0 = ---
chmod 755 file    # rwxr-xr-x (owner: rwx, group: r-x, other: r-x)
chmod 644 file    # rw-r--r-- (typical file)
chmod 600 file    # rw------- (private file, e.g. SSH keys)
chmod 700 dir     # rwx------ (private directory)
chmod 755 dir     # rwxr-xr-x (typical directory — needs execute to enter)
chmod 777 file    # rwxrwxrwx (EVERYONE can read/write/exec — usually a mistake)

# ── Symbolic notation: who + action + what ──
# who:  u (user/owner), g (group), o (other), a (all — default)
# action: + (add), - (remove), = (set exactly)
# what:  r (read), w (write), x (execute), X (execute if dir or already exec), s (setuid/setgid), t (sticky)
chmod u+x file        # add execute for owner
chmod g-w file        # remove write for group
chmod a+r file        # add read for all (same as +r)
chmod u=rwx,go=rx file  # set explicitly: owner rwx, group+other rx
chmod +x file         # add execute for all (default who = a)
chmod u+s file        # setuid bit (run as owner — dangerous, use sparingly)
chmod g+s dir         # setgid bit (new files inherit group — shared dirs)
chmod +t dir          # sticky bit (only owner can delete files — /tmp uses this)

# ── Recursive ──
chmod -R 755 dir     # recursive (dirs AND files get 755 — usually wrong for files!)
# Best practice: separate dirs (need execute) from files (don't need execute):
find dir -type d -exec chmod 755 {} +    # dirs: rwxr-xr-x
find dir -type f -exec chmod 644 {} +    # files: rw-r--r--

# ── Special permissions ──
# 4000 = setuid (chmod u+s)  — runs executable as the file's owner
# 2000 = setgid (chmod g+s)  — runs as group / new files inherit group
# 1000 = sticky  (chmod +t)  — only file owner can delete (used on /tmp)
chmod 4755 /usr/bin/sudo    # setuid root (rwsr-xr-x — the 's' replaces 'x')
chmod 2755 shared_dir       # setgid (rwxr-sr-x — 's' in group execute position)
chmod 1777 /tmp             # sticky (rwxrwxrwt — 't' in other execute position)

# ── umask: default permission mask ──
umask   # current umask (e.g. 022)
umask 022   # files created with 644 (666 - 022), dirs with 755 (777 - 022)
umask 077   # files: 600, dirs: 700 (more restrictive — private)
umask 000   # files: 666, dirs: 777 (least restrictive — dangerous)

# ── mktemp creates unique temp files/dirs atomically (race-free) ──
tmpfile=$(mktemp)           # creates a file in $TMPDIR (or /tmp)
tmpdir=$(mktemp -d)         # creates a directory (for multiple temp files)
trap 'rm -f "$tmpfile"; rm -rf "$tmpdir"' EXIT  # cleanup on any exit

# ── Why not /tmp/myscript.$$? ──
# $$ is the shell PID — predictable (attacker can pre-create a symlink).
# mktemp uses random names + atomic creation (O_CREAT|O_EXCL) — race-free.

# ── Custom template (GNU mktemp) ──
tmpfile=$(mktemp /tmp/myapp.XXXXXX)    # XXXXXX is replaced with random chars
tmpdir=$(mktemp -d /tmp/myapp.XXXXXX)

# ── Custom template (BSD/macOS mktemp) ──
tmpfile=$(mktemp /tmp/myapp.XXXXXX)    # same syntax on BSD

# ── Temp file in a specific directory ──
tmpfile=$(mktemp -p "$SCRIPT_DIR/tmp")  # create in a specific directory

# ── Use $TMPDIR if available (respects user's preference) ──
echo "${TMPDIR:-/tmp}"   # TMPDIR on macOS: /var/folders/...; Linux: often unset, fallback /tmp

# ❌ DANGEROUS — predictable name, symlink attack risk
tmpfile="/tmp/myapp.$$"   # $$ is the PID — attacker can guess it
touch "$tmpfile"           # if attacker pre-created a symlink /tmp/myapp.1234 → /etc/passwd,
#                          your script overwrites /etc/passwd! (symlink attack)

# ❌ ALSO DANGEROUS — race condition
tmpfile="/tmp/myapp_$(date +%s).tmp"
echo "data" > "$tmpfile"   # window between generating name and creating file

# ✅ CORRECT — mktemp (atomic, race-free, unpredictable)
tmpfile=$(mktemp)          # atomically created with O_CREAT|O_EXCL — no race
trap 'rm -f "$tmpfile"' EXIT

# ── du: disk usage ──
du -sh dir               # total size (human-readable: 1.2G)
du -sh *                 # size of each item in current dir
du -sh * | sort -rh      # largest first
du -sh --max-depth=1 .   # size of immediate subdirs (GNU du)
du -d 1 -h .             # same on BSD/macOS du
du -ah dir               # all files (not just dirs)
du -ch file1 file2 | tail -1  # total of multiple files (-c: grand total)
du -sh --exclude=node_modules .  # exclude directories (GNU du)

# ── df: disk space ──
df -h                    # human-readable sizes
df -h /                  # space on root filesystem
df -h .                  # space on current filesystem
df -i                    # inode usage (can run out of inodes before space!)

# ── stat: file metadata ──
stat file.txt            # full metadata: size, perms, owner, timestamps, inode
stat -c '%s' file.txt    # size in bytes (GNU)
stat -f '%z' file.txt    # size in bytes (BSD/macOS)
stat -c '%y' file.txt    # last modification time (GNU)
stat -c '%a' file.txt    # octal permissions (GNU)
stat -f '%A' file.txt    # octal permissions (BSD/macOS)

# ── file: detect file type ──
file file.txt            # "ASCII text"
file image.png           # "PNG image data, 1920 x 1080, ..."
file --mime-type file   # MIME type: "text/plain"
file -b file.txt         # brief (no filename in output)

# ── Atomic file writes (crash-safe) ──
# Write to a temp file, then rename (rename is atomic on most filesystems):
tmpfile=$(mktemp "$file.XXXXXX")
echo "content" > "$tmpfile"
mv "$tmpfile" "$file"    # atomic rename — readers never see a partial write
# If the script crashes during echo, the original file is intact (temp file is orphaned).

# ── Lock file to prevent concurrent execution ──
lockfile=$(mktemp /tmp/myapp.lock.XXXXXX)
exec 9> "$lockfile"        # open fd 9 for writing
if ! flock -n 9; then     # try to acquire exclusive lock (non-blocking)
    echo "another instance is running" >&2
    exit 1
fi
# Lock is held until fd 9 is closed (script exits or exec 9>&-)
# flock is Linux-only; macOS: brew install flock or use mkdir for locking

# ── mkdir-based locking (portable, no flock needed) ──
lockdir="/tmp/myapp.lock"
mkdir "$lockdir" 2>/dev/null || { echo "already running" >&2; exit 1; }
trap 'rmdir "$lockdir" 2>/dev/null' EXIT
# mkdir is atomic — only one process can create the dir (race-free)

# ── Copy with progress (rsync) ──
rsync -ah --progress src/ dest/   # -a archive, -h human-readable, --progress

# ── Safe rm with a trash function ──
trash() {
    local trash_dir="${TRASH:-$HOME/.trash}"
    mkdir -p "$trash_dir"
    for f in "$@"; do
        mv "$f" "$trash_dir/"  # move instead of delete
    done
}

# ── `rm -rf $var` with empty $var is catastrophic ──
# rm -rf "$var" with var="" → rm -rf "" (no-op, safe with quotes)
# rm -rf $var with var="" → rm -rf (no arg!) → deletes CWD!
# Always quote: rm -rf "$var"  AND  check: [[ -n "$var" ]] || die "empty path"

# ── `cp`/`mv` overwrite without warning ──
cp src dst    # silently overwrites dst!
# Use: cp -i (interactive) or cp -n (no overwrite)
# Or check: [[ ! -e "$dst" ]] || { echo "exists"; exit 1; }

# ── macOS `cp`/`mv` differ from GNU ──
# BSD cp doesn't have --backup. BSD mv has different -i behavior.
# Install GNU coreutils: brew install coreutils → gcp, gmv, etc.

# ── `ln -s` relative symlinks ──
ln -s target link          # if target is relative, it's relative to the LINK's dir, not CWD
ln -s /absolute/path link  # absolute target — always works
ln -s ../file.txt link     # relative target — resolves relative to link's location

# ── `find` default action is -print (even with -exec) ──
find . -name "*.txt" -exec echo {} \;   # ALSO prints each filename (default -print)
find . -name "*.txt" -exec echo {} \; -print  # explicit (same)
find . -name "*.txt" -exec echo {} \; -not -print  # no — can't negate default
# To suppress: find . -name "*.txt" -exec echo {} \; | grep -v '^\.\/'  (or just don't worry)

# ── `du` on sparse files reports apparent size, not disk usage ──
# A 100GB sparse file that uses 0 blocks: du reports 0 (actual), du --apparent-size reports 100G

set -e
find . -name "*.tmp" -exec rm {} \;
echo "cleaned up"