[{"data":1,"prerenderedAt":2122},["ShallowReactive",2],{"page-\u002Flinux\u002F18-troubleshooting-and-recovery":3},{"id":4,"title":5,"body":6,"description":16,"extension":2116,"meta":2117,"navigation":298,"path":2118,"seo":2119,"stem":2120,"__hash__":2121},"content\u002Flinux\u002F18-troubleshooting-and-recovery.md","18 — Troubleshooting & Recovery",{"type":7,"value":8,"toc":2082},"minimark",[9,13,17,22,85,89,255,259,264,476,480,573,576,618,622,699,707,783,787,907,911,985,989,993,996,1024,1044,1051,1106,1110,1294,1301,1368,1372,1379,1466,1473,1531,1538,1625,1632,1635,1663,1670,1708,1719,1723,1795,1799,1893,1897,1906,2078],[10,11,5],"h1",{"id":12},"_18-troubleshooting-recovery",[14,15,16],"p",{},"Things break. A service won't start, a disk fills up, boot fails, performance degrades. This chapter is your field guide for diagnosing and fixing Linux problems — the tools, the methodology, and the recovery procedures for when a system is truly stuck.",[18,19,21],"h2",{"id":20},"the-troubleshooting-methodology","The Troubleshooting Methodology",[23,24,25,38,55,61,67,73,79],"ol",{},[26,27,28,32,33,37],"li",{},[29,30,31],"strong",{},"Don't panic, don't reboot (yet)."," Rebooting destroys evidence (logs, process state, ",[34,35,36],"code",{},"\u002Fproc","). Gather information first.",[26,39,40,43,44,47,48,47,51,54],{},[29,41,42],{},"Check the logs."," The system tells you what's wrong — ",[34,45,46],{},"journalctl",", ",[34,49,50],{},"dmesg",[34,52,53],{},"\u002Fvar\u002Flog\u002F",". Read the error.",[26,56,57,60],{},[29,58,59],{},"Isolate the layer."," Is it the network? Disk? The app? The kernel? Test each layer independently.",[26,62,63,66],{},[29,64,65],{},"Reproduce."," If you can reproduce the issue, you can test fixes.",[26,68,69,72],{},[29,70,71],{},"Change one thing at a time."," Multiple changes obscure which fix worked (or which broke something else).",[26,74,75,78],{},[29,76,77],{},"Have a rollback plan."," Before changing a config, back it up.",[26,80,81,84],{},[29,82,83],{},"Document."," Once fixed, record the cause and solution.",[18,86,88],{"id":87},"the-first-questions","The First Questions",[90,91,93],"code-wrapper",{"language":92},"bash",[94,95,99],"pre",{"className":96,"code":97,"language":92,"meta":98,"style":98},"language-bash shiki shiki-themes github-light github-dark","# Complex Implementation: system health snapshot in 10 commands\nsystemctl list-units --state=failed  # failed services\njournalctl -b -p err                 # current boot, errors only\njournalctl -b -1 -p err              # previous boot, errors (if rebooted)\ndmesg --level=err,crit               # kernel errors\nuptime                               # load average\nfree -h                              # memory\ndf -h                                # disk space\ndf -i                                # inodes\nss -tlnp                             # listening ports\ntop -b -n 1 | head -20               # top processes\n","",[34,100,101,110,128,145,162,173,182,194,205,216,228],{"__ignoreMap":98},[102,103,106],"span",{"class":104,"line":105},"line",1,[102,107,109],{"class":108},"sdCPZ","# Complex Implementation: system health snapshot in 10 commands\n",[102,111,113,117,121,125],{"class":104,"line":112},2,[102,114,116],{"class":115},"sIsaT","systemctl",[102,118,120],{"class":119},"sJ6F3"," list-units",[102,122,124],{"class":123},"snvgF"," --state=failed",[102,126,127],{"class":108},"  # failed services\n",[102,129,131,133,136,139,142],{"class":104,"line":130},3,[102,132,46],{"class":115},[102,134,135],{"class":123}," -b",[102,137,138],{"class":123}," -p",[102,140,141],{"class":119}," err",[102,143,144],{"class":108},"                 # current boot, errors only\n",[102,146,148,150,152,155,157,159],{"class":104,"line":147},4,[102,149,46],{"class":115},[102,151,135],{"class":123},[102,153,154],{"class":123}," -1",[102,156,138],{"class":123},[102,158,141],{"class":119},[102,160,161],{"class":108},"              # previous boot, errors (if rebooted)\n",[102,163,165,167,170],{"class":104,"line":164},5,[102,166,50],{"class":115},[102,168,169],{"class":123}," --level=err,crit",[102,171,172],{"class":108},"               # kernel errors\n",[102,174,176,179],{"class":104,"line":175},6,[102,177,178],{"class":115},"uptime",[102,180,181],{"class":108},"                               # load average\n",[102,183,185,188,191],{"class":104,"line":184},7,[102,186,187],{"class":115},"free",[102,189,190],{"class":123}," -h",[102,192,193],{"class":108},"                              # memory\n",[102,195,197,200,202],{"class":104,"line":196},8,[102,198,199],{"class":115},"df",[102,201,190],{"class":123},[102,203,204],{"class":108},"                                # disk space\n",[102,206,208,210,213],{"class":104,"line":207},9,[102,209,199],{"class":115},[102,211,212],{"class":123}," -i",[102,214,215],{"class":108},"                                # inodes\n",[102,217,219,222,225],{"class":104,"line":218},10,[102,220,221],{"class":115},"ss",[102,223,224],{"class":123}," -tlnp",[102,226,227],{"class":108},"                             # listening ports\n",[102,229,231,234,236,239,242,246,249,252],{"class":104,"line":230},11,[102,232,233],{"class":115},"top",[102,235,135],{"class":123},[102,237,238],{"class":123}," -n",[102,240,241],{"class":123}," 1",[102,243,245],{"class":244},"svdQ7"," |",[102,247,248],{"class":115}," head",[102,250,251],{"class":123}," -20",[102,253,254],{"class":108},"               # top processes\n",[18,256,258],{"id":257},"common-problems-solutions","Common Problems & Solutions",[260,261,263],"h3",{"id":262},"no-space-left-on-device","\"No space left on device\"",[90,265,266],{"language":92},[94,267,269],{"className":96,"code":268,"language":92,"meta":98,"style":98},"# Complex Implementation: full disk-space diagnosis\ndf -h                    # which filesystem is full?\ndf -i                    # maybe inodes are full (not space)?\n\n# Find the biggest consumers:\nsudo du -sh \u002F* 2>\u002Fdev\u002Fnull | sort -rh | head\nsudo du -sh \u002Fvar\u002F* 2>\u002Fdev\u002Fnull | sort -rh | head\nsudo ncdu \u002F              # interactive (if installed)\n\n# Common culprits:\njournalctl --disk-usage   # journald size\nsudo du -sh \u002Fvar\u002Flog \u002Fvar\u002Fcache \u002Fvar\u002Flib\u002Fdocker 2>\u002Fdev\u002Fnull\n\n# Find deleted-but-open files (space not freed until fd closes):\nsudo lsof +L1\n# COMMAND   PID  USER  FD  SIZE  NLINK  NAME\n# nginx    1234  root  9w  50G   0      \u002Fvar\u002Flog\u002Fnginx\u002Faccess.log (deleted)\n# Fix: restart the process (closes the fd, frees the space)\nsudo systemctl restart nginx\n",[34,270,271,276,285,294,300,305,341,368,380,384,389,399,422,427,433,444,450,456,462],{"__ignoreMap":98},[102,272,273],{"class":104,"line":105},[102,274,275],{"class":108},"# Complex Implementation: full disk-space diagnosis\n",[102,277,278,280,282],{"class":104,"line":112},[102,279,199],{"class":115},[102,281,190],{"class":123},[102,283,284],{"class":108},"                    # which filesystem is full?\n",[102,286,287,289,291],{"class":104,"line":130},[102,288,199],{"class":115},[102,290,212],{"class":123},[102,292,293],{"class":108},"                    # maybe inodes are full (not space)?\n",[102,295,296],{"class":104,"line":147},[102,297,299],{"emptyLinePlaceholder":298},true,"\n",[102,301,302],{"class":104,"line":164},[102,303,304],{"class":108},"# Find the biggest consumers:\n",[102,306,307,310,313,316,319,322,325,328,330,333,336,338],{"class":104,"line":175},[102,308,309],{"class":115},"sudo",[102,311,312],{"class":119}," du",[102,314,315],{"class":123}," -sh",[102,317,318],{"class":119}," \u002F",[102,320,321],{"class":123},"*",[102,323,324],{"class":244}," 2>",[102,326,327],{"class":119},"\u002Fdev\u002Fnull",[102,329,245],{"class":244},[102,331,332],{"class":115}," sort",[102,334,335],{"class":123}," -rh",[102,337,245],{"class":244},[102,339,340],{"class":115}," head\n",[102,342,343,345,347,349,352,354,356,358,360,362,364,366],{"class":104,"line":184},[102,344,309],{"class":115},[102,346,312],{"class":119},[102,348,315],{"class":123},[102,350,351],{"class":119}," \u002Fvar\u002F",[102,353,321],{"class":123},[102,355,324],{"class":244},[102,357,327],{"class":119},[102,359,245],{"class":244},[102,361,332],{"class":115},[102,363,335],{"class":123},[102,365,245],{"class":244},[102,367,340],{"class":115},[102,369,370,372,375,377],{"class":104,"line":196},[102,371,309],{"class":115},[102,373,374],{"class":119}," ncdu",[102,376,318],{"class":119},[102,378,379],{"class":108},"              # interactive (if installed)\n",[102,381,382],{"class":104,"line":207},[102,383,299],{"emptyLinePlaceholder":298},[102,385,386],{"class":104,"line":218},[102,387,388],{"class":108},"# Common culprits:\n",[102,390,391,393,396],{"class":104,"line":230},[102,392,46],{"class":115},[102,394,395],{"class":123}," --disk-usage",[102,397,398],{"class":108},"   # journald size\n",[102,400,402,404,406,408,411,414,417,419],{"class":104,"line":401},12,[102,403,309],{"class":115},[102,405,312],{"class":119},[102,407,315],{"class":123},[102,409,410],{"class":119}," \u002Fvar\u002Flog",[102,412,413],{"class":119}," \u002Fvar\u002Fcache",[102,415,416],{"class":119}," \u002Fvar\u002Flib\u002Fdocker",[102,418,324],{"class":244},[102,420,421],{"class":119},"\u002Fdev\u002Fnull\n",[102,423,425],{"class":104,"line":424},13,[102,426,299],{"emptyLinePlaceholder":298},[102,428,430],{"class":104,"line":429},14,[102,431,432],{"class":108},"# Find deleted-but-open files (space not freed until fd closes):\n",[102,434,436,438,441],{"class":104,"line":435},15,[102,437,309],{"class":115},[102,439,440],{"class":119}," lsof",[102,442,443],{"class":119}," +L1\n",[102,445,447],{"class":104,"line":446},16,[102,448,449],{"class":108},"# COMMAND   PID  USER  FD  SIZE  NLINK  NAME\n",[102,451,453],{"class":104,"line":452},17,[102,454,455],{"class":108},"# nginx    1234  root  9w  50G   0      \u002Fvar\u002Flog\u002Fnginx\u002Faccess.log (deleted)\n",[102,457,459],{"class":104,"line":458},18,[102,460,461],{"class":108},"# Fix: restart the process (closes the fd, frees the space)\n",[102,463,465,467,470,473],{"class":104,"line":464},19,[102,466,309],{"class":115},[102,468,469],{"class":119}," systemctl",[102,471,472],{"class":119}," restart",[102,474,475],{"class":119}," nginx\n",[260,477,479],{"id":478},"permission-denied-but-permissions-look-right","\"Permission denied\" (but permissions look right)",[90,481,482],{"language":92},[94,483,485],{"className":96,"code":484,"language":92,"meta":98,"style":98},"# Complex Implementation: multi-layer permission diagnosis\nls -l file                      # check owner, group, mode\nnamei -l \u002Fpath\u002Fto\u002Ffile          # permissions on EACH path component (parent dirs!)\ngetfacl file                    # ACLs (might deny even if base perms allow)\nls -Z file                      # SELinux label (RHEL — SELinux might be blocking)\nsudo ausearch -m AVC -ts recent # SELinux denials\nlsattr file                     # extended attributes (immutable?)\n",[34,486,487,492,506,519,529,541,563],{"__ignoreMap":98},[102,488,489],{"class":104,"line":105},[102,490,491],{"class":108},"# Complex Implementation: multi-layer permission diagnosis\n",[102,493,494,497,500,503],{"class":104,"line":112},[102,495,496],{"class":115},"ls",[102,498,499],{"class":123}," -l",[102,501,502],{"class":119}," file",[102,504,505],{"class":108},"                      # check owner, group, mode\n",[102,507,508,511,513,516],{"class":104,"line":130},[102,509,510],{"class":115},"namei",[102,512,499],{"class":123},[102,514,515],{"class":119}," \u002Fpath\u002Fto\u002Ffile",[102,517,518],{"class":108},"          # permissions on EACH path component (parent dirs!)\n",[102,520,521,524,526],{"class":104,"line":147},[102,522,523],{"class":115},"getfacl",[102,525,502],{"class":119},[102,527,528],{"class":108},"                    # ACLs (might deny even if base perms allow)\n",[102,530,531,533,536,538],{"class":104,"line":164},[102,532,496],{"class":115},[102,534,535],{"class":123}," -Z",[102,537,502],{"class":119},[102,539,540],{"class":108},"                      # SELinux label (RHEL — SELinux might be blocking)\n",[102,542,543,545,548,551,554,557,560],{"class":104,"line":175},[102,544,309],{"class":115},[102,546,547],{"class":119}," ausearch",[102,549,550],{"class":123}," -m",[102,552,553],{"class":119}," AVC",[102,555,556],{"class":123}," -ts",[102,558,559],{"class":119}," recent",[102,561,562],{"class":108}," # SELinux denials\n",[102,564,565,568,570],{"class":104,"line":184},[102,566,567],{"class":115},"lsattr",[102,569,502],{"class":119},[102,571,572],{"class":108},"                     # extended attributes (immutable?)\n",[14,574,575],{},"Common causes:",[577,578,579,592,602],"ul",{},[26,580,581,587,588,591],{},[29,582,583,584],{},"Parent directory lacks ",[34,585,586],{},"x",": you can't access a file if you can't traverse its parent. ",[34,589,590],{},"namei -l"," reveals this.",[26,593,594,597,598,601],{},[29,595,596],{},"SELinux",": \"Permission denied\" with correct Unix perms = SELinux. Check ",[34,599,600],{},"ausearch -m AVC",".",[26,603,604,607,608,611,612,614,615,601],{},[29,605,606],{},"Immutable attribute",": ",[34,609,610],{},"chattr +i file"," makes it undeletable. Check with ",[34,613,567],{},", remove with ",[34,616,617],{},"chattr -i",[260,619,621],{"id":620},"service-wont-start","\"Service won't start\"",[90,623,624],{"language":92},[94,625,627],{"className":96,"code":626,"language":92,"meta":98,"style":98},"systemctl status nginx              # status + last log lines\njournalctl -u nginx -n 50           # full logs\nsudo nginx -t                       # test config (if it has a config test)\nsystemctl cat nginx                 # see the unit file (ExecStart, etc.)\n# Run the ExecStart command manually to see the error:\n\u002Fusr\u002Fsbin\u002Fnginx -g 'daemon off;'\n",[34,628,629,642,659,671,683,688],{"__ignoreMap":98},[102,630,631,633,636,639],{"class":104,"line":105},[102,632,116],{"class":115},[102,634,635],{"class":119}," status",[102,637,638],{"class":119}," nginx",[102,640,641],{"class":108},"              # status + last log lines\n",[102,643,644,646,649,651,653,656],{"class":104,"line":112},[102,645,46],{"class":115},[102,647,648],{"class":123}," -u",[102,650,638],{"class":119},[102,652,238],{"class":123},[102,654,655],{"class":123}," 50",[102,657,658],{"class":108},"           # full logs\n",[102,660,661,663,665,668],{"class":104,"line":130},[102,662,309],{"class":115},[102,664,638],{"class":119},[102,666,667],{"class":123}," -t",[102,669,670],{"class":108},"                       # test config (if it has a config test)\n",[102,672,673,675,678,680],{"class":104,"line":147},[102,674,116],{"class":115},[102,676,677],{"class":119}," cat",[102,679,638],{"class":119},[102,681,682],{"class":108},"                 # see the unit file (ExecStart, etc.)\n",[102,684,685],{"class":104,"line":164},[102,686,687],{"class":108},"# Run the ExecStart command manually to see the error:\n",[102,689,690,693,696],{"class":104,"line":175},[102,691,692],{"class":115},"\u002Fusr\u002Fsbin\u002Fnginx",[102,694,695],{"class":123}," -g",[102,697,698],{"class":119}," 'daemon off;'\n",[260,700,702,703,706],{"id":701},"disk-full-but-du-doesnt-show-it","\"Disk full\" but ",[34,704,705],{},"du"," Doesn't Show It",[90,708,709],{"language":92},[94,710,712],{"className":96,"code":711,"language":92,"meta":98,"style":98},"# Complex Implementation: find deleted-but-open files hogging disk space\n# — a process holds a large file OPEN but DELETED; space not freed until fd closes\nsudo lsof +L1               # list open files with link count 0 (deleted but open)\n# COMMAND   PID  USER  FD  SIZE  NLINK  NAME\n# nginx    1234  root  9w  50G   0      \u002Fvar\u002Flog\u002Fnginx\u002Faccess.log (deleted)\n\n# Fix: restart the process (closes the fd, frees the space)\nsudo systemctl restart nginx\n# Or truncate the file via \u002Fproc (without restarting):\nsudo truncate -s 0 \u002Fproc\u002F1234\u002Ffd\u002F9\n",[34,713,714,719,724,736,740,744,748,752,762,767],{"__ignoreMap":98},[102,715,716],{"class":104,"line":105},[102,717,718],{"class":108},"# Complex Implementation: find deleted-but-open files hogging disk space\n",[102,720,721],{"class":104,"line":112},[102,722,723],{"class":108},"# — a process holds a large file OPEN but DELETED; space not freed until fd closes\n",[102,725,726,728,730,733],{"class":104,"line":130},[102,727,309],{"class":115},[102,729,440],{"class":119},[102,731,732],{"class":119}," +L1",[102,734,735],{"class":108},"               # list open files with link count 0 (deleted but open)\n",[102,737,738],{"class":104,"line":147},[102,739,449],{"class":108},[102,741,742],{"class":104,"line":164},[102,743,455],{"class":108},[102,745,746],{"class":104,"line":175},[102,747,299],{"emptyLinePlaceholder":298},[102,749,750],{"class":104,"line":184},[102,751,461],{"class":108},[102,753,754,756,758,760],{"class":104,"line":196},[102,755,309],{"class":115},[102,757,469],{"class":119},[102,759,472],{"class":119},[102,761,475],{"class":119},[102,763,764],{"class":104,"line":207},[102,765,766],{"class":108},"# Or truncate the file via \u002Fproc (without restarting):\n",[102,768,769,771,774,777,780],{"class":104,"line":218},[102,770,309],{"class":115},[102,772,773],{"class":119}," truncate",[102,775,776],{"class":123}," -s",[102,778,779],{"class":123}," 0",[102,781,782],{"class":119}," \u002Fproc\u002F1234\u002Ffd\u002F9\n",[260,784,786],{"id":785},"dns-not-resolving","DNS Not Resolving",[90,788,789],{"language":92},[94,790,792],{"className":96,"code":791,"language":92,"meta":98,"style":98},"# Complex Implementation: full DNS diagnostic chain\ndig example.com                     # direct DNS query (bypasses cache)\ngetent hosts example.com            # via NSS (checks \u002Fetc\u002Fhosts, then DNS)\ncat \u002Fetc\u002Fresolv.conf                # nameservers\ncat \u002Fetc\u002Fhosts                      # stale override?\ncat \u002Fetc\u002Fnsswitch.conf | grep hosts # order: files dns?\nping 8.8.8.8                        # is the network up?\ndig @8.8.8.8 example.com            # query a specific server\nresolvectl status                   # systemd-resolved\nsudo resolvectl flush-caches        # flush DNS cache\n",[34,793,794,799,810,823,834,844,861,872,884,894],{"__ignoreMap":98},[102,795,796],{"class":104,"line":105},[102,797,798],{"class":108},"# Complex Implementation: full DNS diagnostic chain\n",[102,800,801,804,807],{"class":104,"line":112},[102,802,803],{"class":115},"dig",[102,805,806],{"class":119}," example.com",[102,808,809],{"class":108},"                     # direct DNS query (bypasses cache)\n",[102,811,812,815,818,820],{"class":104,"line":130},[102,813,814],{"class":115},"getent",[102,816,817],{"class":119}," hosts",[102,819,806],{"class":119},[102,821,822],{"class":108},"            # via NSS (checks \u002Fetc\u002Fhosts, then DNS)\n",[102,824,825,828,831],{"class":104,"line":147},[102,826,827],{"class":115},"cat",[102,829,830],{"class":119}," \u002Fetc\u002Fresolv.conf",[102,832,833],{"class":108},"                # nameservers\n",[102,835,836,838,841],{"class":104,"line":164},[102,837,827],{"class":115},[102,839,840],{"class":119}," \u002Fetc\u002Fhosts",[102,842,843],{"class":108},"                      # stale override?\n",[102,845,846,848,851,853,856,858],{"class":104,"line":175},[102,847,827],{"class":115},[102,849,850],{"class":119}," \u002Fetc\u002Fnsswitch.conf",[102,852,245],{"class":244},[102,854,855],{"class":115}," grep",[102,857,817],{"class":119},[102,859,860],{"class":108}," # order: files dns?\n",[102,862,863,866,869],{"class":104,"line":184},[102,864,865],{"class":115},"ping",[102,867,868],{"class":123}," 8.8.8.8",[102,870,871],{"class":108},"                        # is the network up?\n",[102,873,874,876,879,881],{"class":104,"line":196},[102,875,803],{"class":115},[102,877,878],{"class":119}," @8.8.8.8",[102,880,806],{"class":119},[102,882,883],{"class":108},"            # query a specific server\n",[102,885,886,889,891],{"class":104,"line":207},[102,887,888],{"class":115},"resolvectl",[102,890,635],{"class":119},[102,892,893],{"class":108},"                   # systemd-resolved\n",[102,895,896,898,901,904],{"class":104,"line":218},[102,897,309],{"class":115},[102,899,900],{"class":119}," resolvectl",[102,902,903],{"class":119}," flush-caches",[102,905,906],{"class":108},"        # flush DNS cache\n",[260,908,910],{"id":909},"read-only-file-system","\"Read-only file system\"",[90,912,913],{"language":92},[94,914,916],{"className":96,"code":915,"language":92,"meta":98,"style":98},"# The filesystem remounted itself read-only (ext4 does this on journal errors to prevent corruption)\nmount | grep \"ro,\"               # which is read-only?\ndmesg | grep -i \"error\\|ext4\"    # why? (disk errors, corruption)\nsudo fsck \u002Fdev\u002Fsda1              # check + repair (must be unmounted)\nsudo mount -o remount,rw \u002F       # try to remount read-write (if safe)\n",[34,917,918,923,938,954,967],{"__ignoreMap":98},[102,919,920],{"class":104,"line":105},[102,921,922],{"class":108},"# The filesystem remounted itself read-only (ext4 does this on journal errors to prevent corruption)\n",[102,924,925,928,930,932,935],{"class":104,"line":112},[102,926,927],{"class":115},"mount",[102,929,245],{"class":244},[102,931,855],{"class":115},[102,933,934],{"class":119}," \"ro,\"",[102,936,937],{"class":108},"               # which is read-only?\n",[102,939,940,942,944,946,948,951],{"class":104,"line":130},[102,941,50],{"class":115},[102,943,245],{"class":244},[102,945,855],{"class":115},[102,947,212],{"class":123},[102,949,950],{"class":119}," \"error\\|ext4\"",[102,952,953],{"class":108},"    # why? (disk errors, corruption)\n",[102,955,956,958,961,964],{"class":104,"line":147},[102,957,309],{"class":115},[102,959,960],{"class":119}," fsck",[102,962,963],{"class":119}," \u002Fdev\u002Fsda1",[102,965,966],{"class":108},"              # check + repair (must be unmounted)\n",[102,968,969,971,974,977,980,982],{"class":104,"line":164},[102,970,309],{"class":115},[102,972,973],{"class":119}," mount",[102,975,976],{"class":123}," -o",[102,978,979],{"class":119}," remount,rw",[102,981,318],{"class":119},[102,983,984],{"class":108},"       # try to remount read-write (if safe)\n",[18,986,988],{"id":987},"recovery-procedures","Recovery Procedures",[260,990,992],{"id":991},"boot-to-rescue-mode","Boot to Rescue Mode",[14,994,995],{},"At the GRUB menu:",[23,997,998,1005,1012,1018],{},[26,999,1000,1001,1004],{},"Press ",[34,1002,1003],{},"e"," to edit the default entry.",[26,1006,1007,1008,1011],{},"Find the ",[34,1009,1010],{},"linux"," line.",[26,1013,1014,1015,601],{},"Append ",[34,1016,1017],{},"systemd.unit=rescue.target",[26,1019,1020,1023],{},[34,1021,1022],{},"Ctrl+X"," to boot.",[90,1025,1026],{"language":92},[94,1027,1029],{"className":96,"code":1028,"language":92,"meta":98,"style":98},"mount -o remount,rw \u002F     # if root is read-only\n",[34,1030,1031],{"__ignoreMap":98},[102,1032,1033,1035,1037,1039,1041],{"class":104,"line":105},[102,1034,927],{"class":115},[102,1036,976],{"class":123},[102,1038,979],{"class":119},[102,1040,318],{"class":119},[102,1042,1043],{"class":108},"     # if root is read-only\n",[260,1045,1047,1050],{"id":1046},"initbinbash-when-systemd-is-broken",[34,1048,1049],{},"init=\u002Fbin\u002Fbash"," (When systemd is broken)",[90,1052,1053],{"language":92},[94,1054,1056],{"className":96,"code":1055,"language":92,"meta":98,"style":98},"# No systemd, no services, root mounted read-only. Raw shell as PID 1:\nmount -o remount,rw \u002F\n# fix the problem (e.g., edit \u002Fetc\u002Ffstab, reinstall a package)\nsync\nmount -o remount,ro \u002F\nexec \u002Fsbin\u002Finit             # start systemd normally\n",[34,1057,1058,1063,1074,1079,1084,1095],{"__ignoreMap":98},[102,1059,1060],{"class":104,"line":105},[102,1061,1062],{"class":108},"# No systemd, no services, root mounted read-only. Raw shell as PID 1:\n",[102,1064,1065,1067,1069,1071],{"class":104,"line":112},[102,1066,927],{"class":115},[102,1068,976],{"class":123},[102,1070,979],{"class":119},[102,1072,1073],{"class":119}," \u002F\n",[102,1075,1076],{"class":104,"line":130},[102,1077,1078],{"class":108},"# fix the problem (e.g., edit \u002Fetc\u002Ffstab, reinstall a package)\n",[102,1080,1081],{"class":104,"line":147},[102,1082,1083],{"class":115},"sync\n",[102,1085,1086,1088,1090,1093],{"class":104,"line":164},[102,1087,927],{"class":115},[102,1089,976],{"class":123},[102,1091,1092],{"class":119}," remount,ro",[102,1094,1073],{"class":119},[102,1096,1097,1100,1103],{"class":104,"line":175},[102,1098,1099],{"class":123},"exec",[102,1101,1102],{"class":119}," \u002Fsbin\u002Finit",[102,1104,1105],{"class":108},"             # start systemd normally\n",[260,1107,1109],{"id":1108},"live-usb-rescue-iso-chroot","Live USB \u002F Rescue ISO + chroot",[90,1111,1112],{"language":92},[94,1113,1115],{"className":96,"code":1114,"language":92,"meta":98,"style":98},"# Complex Implementation: chroot to fix an installed system from a live USB\nsudo mount \u002Fdev\u002Fsda2 \u002Fmnt               # root partition\nsudo mount \u002Fdev\u002Fsda1 \u002Fmnt\u002Fboot\u002Fefi      # ESP (if UEFI)\n# For LVM:\nsudo vgchange -a y                      # activate all LVM volumes\nsudo mount \u002Fdev\u002Fvg_root\u002Flv_root \u002Fmnt\n\n# Bind-mount virtual filesystems for chroot:\nsudo mount --bind \u002Fdev \u002Fmnt\u002Fdev\nsudo mount --bind \u002Fdev\u002Fpts \u002Fmnt\u002Fdev\u002Fpts\nsudo mount --bind \u002Fproc \u002Fmnt\u002Fproc\nsudo mount --bind \u002Fsys \u002Fmnt\u002Fsys\nsudo mount --bind \u002Frun \u002Fmnt\u002Frun\n\nsudo chroot \u002Fmnt\n# Now you're \"inside\" the installed system — apt\u002Fdnf, fsck, etc. work\nexit\nsudo reboot\n",[34,1116,1117,1122,1137,1151,1156,1172,1184,1188,1193,1208,1222,1236,1250,1264,1268,1277,1282,1287],{"__ignoreMap":98},[102,1118,1119],{"class":104,"line":105},[102,1120,1121],{"class":108},"# Complex Implementation: chroot to fix an installed system from a live USB\n",[102,1123,1124,1126,1128,1131,1134],{"class":104,"line":112},[102,1125,309],{"class":115},[102,1127,973],{"class":119},[102,1129,1130],{"class":119}," \u002Fdev\u002Fsda2",[102,1132,1133],{"class":119}," \u002Fmnt",[102,1135,1136],{"class":108},"               # root partition\n",[102,1138,1139,1141,1143,1145,1148],{"class":104,"line":130},[102,1140,309],{"class":115},[102,1142,973],{"class":119},[102,1144,963],{"class":119},[102,1146,1147],{"class":119}," \u002Fmnt\u002Fboot\u002Fefi",[102,1149,1150],{"class":108},"      # ESP (if UEFI)\n",[102,1152,1153],{"class":104,"line":147},[102,1154,1155],{"class":108},"# For LVM:\n",[102,1157,1158,1160,1163,1166,1169],{"class":104,"line":164},[102,1159,309],{"class":115},[102,1161,1162],{"class":119}," vgchange",[102,1164,1165],{"class":123}," -a",[102,1167,1168],{"class":119}," y",[102,1170,1171],{"class":108},"                      # activate all LVM volumes\n",[102,1173,1174,1176,1178,1181],{"class":104,"line":175},[102,1175,309],{"class":115},[102,1177,973],{"class":119},[102,1179,1180],{"class":119}," \u002Fdev\u002Fvg_root\u002Flv_root",[102,1182,1183],{"class":119}," \u002Fmnt\n",[102,1185,1186],{"class":104,"line":184},[102,1187,299],{"emptyLinePlaceholder":298},[102,1189,1190],{"class":104,"line":196},[102,1191,1192],{"class":108},"# Bind-mount virtual filesystems for chroot:\n",[102,1194,1195,1197,1199,1202,1205],{"class":104,"line":207},[102,1196,309],{"class":115},[102,1198,973],{"class":119},[102,1200,1201],{"class":123}," --bind",[102,1203,1204],{"class":119}," \u002Fdev",[102,1206,1207],{"class":119}," \u002Fmnt\u002Fdev\n",[102,1209,1210,1212,1214,1216,1219],{"class":104,"line":218},[102,1211,309],{"class":115},[102,1213,973],{"class":119},[102,1215,1201],{"class":123},[102,1217,1218],{"class":119}," \u002Fdev\u002Fpts",[102,1220,1221],{"class":119}," \u002Fmnt\u002Fdev\u002Fpts\n",[102,1223,1224,1226,1228,1230,1233],{"class":104,"line":230},[102,1225,309],{"class":115},[102,1227,973],{"class":119},[102,1229,1201],{"class":123},[102,1231,1232],{"class":119}," \u002Fproc",[102,1234,1235],{"class":119}," \u002Fmnt\u002Fproc\n",[102,1237,1238,1240,1242,1244,1247],{"class":104,"line":401},[102,1239,309],{"class":115},[102,1241,973],{"class":119},[102,1243,1201],{"class":123},[102,1245,1246],{"class":119}," \u002Fsys",[102,1248,1249],{"class":119}," \u002Fmnt\u002Fsys\n",[102,1251,1252,1254,1256,1258,1261],{"class":104,"line":424},[102,1253,309],{"class":115},[102,1255,973],{"class":119},[102,1257,1201],{"class":123},[102,1259,1260],{"class":119}," \u002Frun",[102,1262,1263],{"class":119}," \u002Fmnt\u002Frun\n",[102,1265,1266],{"class":104,"line":429},[102,1267,299],{"emptyLinePlaceholder":298},[102,1269,1270,1272,1275],{"class":104,"line":435},[102,1271,309],{"class":115},[102,1273,1274],{"class":119}," chroot",[102,1276,1183],{"class":119},[102,1278,1279],{"class":104,"line":446},[102,1280,1281],{"class":108},"# Now you're \"inside\" the installed system — apt\u002Fdnf, fsck, etc. work\n",[102,1283,1284],{"class":104,"line":452},[102,1285,1286],{"class":123},"exit\n",[102,1288,1289,1291],{"class":104,"line":458},[102,1290,309],{"class":115},[102,1292,1293],{"class":119}," reboot\n",[260,1295,1297,1298],{"id":1296},"fixing-a-broken-fstab","Fixing a Broken ",[34,1299,1300],{},"fstab",[90,1302,1303],{"language":92},[94,1304,1306],{"className":96,"code":1305,"language":92,"meta":98,"style":98},"# In emergency mode:\nmount -o remount,rw \u002F\njournalctl -b | grep -i \"mount\\|fstab\"\nnano \u002Fetc\u002Ffstab                # comment out the bad line\nmount -a                       # test\nsystemctl default              # continue boot\n",[34,1307,1308,1313,1323,1338,1349,1358],{"__ignoreMap":98},[102,1309,1310],{"class":104,"line":105},[102,1311,1312],{"class":108},"# In emergency mode:\n",[102,1314,1315,1317,1319,1321],{"class":104,"line":112},[102,1316,927],{"class":115},[102,1318,976],{"class":123},[102,1320,979],{"class":119},[102,1322,1073],{"class":119},[102,1324,1325,1327,1329,1331,1333,1335],{"class":104,"line":130},[102,1326,46],{"class":115},[102,1328,135],{"class":123},[102,1330,245],{"class":244},[102,1332,855],{"class":115},[102,1334,212],{"class":123},[102,1336,1337],{"class":119}," \"mount\\|fstab\"\n",[102,1339,1340,1343,1346],{"class":104,"line":147},[102,1341,1342],{"class":115},"nano",[102,1344,1345],{"class":119}," \u002Fetc\u002Ffstab",[102,1347,1348],{"class":108},"                # comment out the bad line\n",[102,1350,1351,1353,1355],{"class":104,"line":164},[102,1352,927],{"class":115},[102,1354,1165],{"class":123},[102,1356,1357],{"class":108},"                       # test\n",[102,1359,1360,1362,1365],{"class":104,"line":175},[102,1361,116],{"class":115},[102,1363,1364],{"class":119}," default",[102,1366,1367],{"class":108},"              # continue boot\n",[18,1369,1371],{"id":1370},"diagnostic-tools","Diagnostic Tools",[260,1373,1375,1378],{"id":1374},"strace-trace-syscalls",[34,1376,1377],{},"strace"," — Trace Syscalls",[90,1380,1381],{"language":92},[94,1382,1384],{"className":96,"code":1383,"language":92,"meta":98,"style":98},"# Complex Implementation: find why a program can't open a file\nstrace -e trace=open,openat myapp 2>&1 | grep ENOENT\n# openat(AT_FDCWD, \"\u002Fetc\u002Fmyapp\u002Fconfig.yaml\", O_RDONLY) = -1 ENOENT (No such file or directory)\n\n# Count syscalls (summary):\nstrace -c command\n# Attach to a running process:\nstrace -p 1234\n# Follow child processes:\nstrace -f command\n",[34,1385,1386,1391,1414,1419,1423,1428,1438,1443,1452,1457],{"__ignoreMap":98},[102,1387,1388],{"class":104,"line":105},[102,1389,1390],{"class":108},"# Complex Implementation: find why a program can't open a file\n",[102,1392,1393,1395,1398,1401,1404,1407,1409,1411],{"class":104,"line":112},[102,1394,1377],{"class":115},[102,1396,1397],{"class":123}," -e",[102,1399,1400],{"class":119}," trace=open,openat",[102,1402,1403],{"class":119}," myapp",[102,1405,1406],{"class":244}," 2>&1",[102,1408,245],{"class":244},[102,1410,855],{"class":115},[102,1412,1413],{"class":119}," ENOENT\n",[102,1415,1416],{"class":104,"line":130},[102,1417,1418],{"class":108},"# openat(AT_FDCWD, \"\u002Fetc\u002Fmyapp\u002Fconfig.yaml\", O_RDONLY) = -1 ENOENT (No such file or directory)\n",[102,1420,1421],{"class":104,"line":147},[102,1422,299],{"emptyLinePlaceholder":298},[102,1424,1425],{"class":104,"line":164},[102,1426,1427],{"class":108},"# Count syscalls (summary):\n",[102,1429,1430,1432,1435],{"class":104,"line":175},[102,1431,1377],{"class":115},[102,1433,1434],{"class":123}," -c",[102,1436,1437],{"class":119}," command\n",[102,1439,1440],{"class":104,"line":184},[102,1441,1442],{"class":108},"# Attach to a running process:\n",[102,1444,1445,1447,1449],{"class":104,"line":196},[102,1446,1377],{"class":115},[102,1448,138],{"class":123},[102,1450,1451],{"class":123}," 1234\n",[102,1453,1454],{"class":104,"line":207},[102,1455,1456],{"class":108},"# Follow child processes:\n",[102,1458,1459,1461,1464],{"class":104,"line":218},[102,1460,1377],{"class":115},[102,1462,1463],{"class":123}," -f",[102,1465,1437],{"class":119},[260,1467,1469,1472],{"id":1468},"lsof-list-open-files",[34,1470,1471],{},"lsof"," — List Open Files",[90,1474,1475],{"language":92},[94,1476,1478],{"className":96,"code":1477,"language":92,"meta":98,"style":98},"sudo lsof \u002Fvar\u002Flog\u002Fsyslog          # who has this file open?\nsudo lsof -i :80                   # what's using port 80?\nsudo lsof -p 1234                  # all files opened by PID 1234\nsudo lsof +L1                     # deleted but still open (disk space mystery)\n",[34,1479,1480,1492,1506,1520],{"__ignoreMap":98},[102,1481,1482,1484,1486,1489],{"class":104,"line":105},[102,1483,309],{"class":115},[102,1485,440],{"class":119},[102,1487,1488],{"class":119}," \u002Fvar\u002Flog\u002Fsyslog",[102,1490,1491],{"class":108},"          # who has this file open?\n",[102,1493,1494,1496,1498,1500,1503],{"class":104,"line":112},[102,1495,309],{"class":115},[102,1497,440],{"class":119},[102,1499,212],{"class":123},[102,1501,1502],{"class":119}," :80",[102,1504,1505],{"class":108},"                   # what's using port 80?\n",[102,1507,1508,1510,1512,1514,1517],{"class":104,"line":130},[102,1509,309],{"class":115},[102,1511,440],{"class":119},[102,1513,138],{"class":123},[102,1515,1516],{"class":123}," 1234",[102,1518,1519],{"class":108},"                  # all files opened by PID 1234\n",[102,1521,1522,1524,1526,1528],{"class":104,"line":147},[102,1523,309],{"class":115},[102,1525,440],{"class":119},[102,1527,732],{"class":119},[102,1529,1530],{"class":108},"                     # deleted but still open (disk space mystery)\n",[260,1532,1534,1537],{"id":1533},"tcpdump-packet-capture",[34,1535,1536],{},"tcpdump"," — Packet Capture",[90,1539,1540],{"language":92},[94,1541,1543],{"className":96,"code":1542,"language":92,"meta":98,"style":98},"sudo tcpdump -i eth0 port 80               # capture port 80\nsudo tcpdump -i eth0 host 8.8.8.8          # capture to\u002Ffrom a host\nsudo tcpdump -i eth0 -w capture.pcap       # save (for Wireshark)\nsudo tcpdump -i any -n -A 'tcp port 80'    # all interfaces, numeric, ASCII\n",[34,1544,1545,1566,1584,1603],{"__ignoreMap":98},[102,1546,1547,1549,1552,1554,1557,1560,1563],{"class":104,"line":105},[102,1548,309],{"class":115},[102,1550,1551],{"class":119}," tcpdump",[102,1553,212],{"class":123},[102,1555,1556],{"class":119}," eth0",[102,1558,1559],{"class":119}," port",[102,1561,1562],{"class":123}," 80",[102,1564,1565],{"class":108},"               # capture port 80\n",[102,1567,1568,1570,1572,1574,1576,1579,1581],{"class":104,"line":112},[102,1569,309],{"class":115},[102,1571,1551],{"class":119},[102,1573,212],{"class":123},[102,1575,1556],{"class":119},[102,1577,1578],{"class":119}," host",[102,1580,868],{"class":123},[102,1582,1583],{"class":108},"          # capture to\u002Ffrom a host\n",[102,1585,1586,1588,1590,1592,1594,1597,1600],{"class":104,"line":130},[102,1587,309],{"class":115},[102,1589,1551],{"class":119},[102,1591,212],{"class":123},[102,1593,1556],{"class":119},[102,1595,1596],{"class":123}," -w",[102,1598,1599],{"class":119}," capture.pcap",[102,1601,1602],{"class":108},"       # save (for Wireshark)\n",[102,1604,1605,1607,1609,1611,1614,1616,1619,1622],{"class":104,"line":147},[102,1606,309],{"class":115},[102,1608,1551],{"class":119},[102,1610,212],{"class":123},[102,1612,1613],{"class":119}," any",[102,1615,238],{"class":123},[102,1617,1618],{"class":123}," -A",[102,1620,1621],{"class":119}," 'tcp port 80'",[102,1623,1624],{"class":108},"    # all interfaces, numeric, ASCII\n",[260,1626,1628,1631],{"id":1627},"sysrq-magic-sysrq-reisub",[34,1629,1630],{},"sysrq"," (Magic SysRq) — REISUB",[14,1633,1634],{},"For when the system is completely frozen:",[90,1636,1637],{"language":92},[94,1638,1640],{"className":96,"code":1639,"language":92,"meta":98,"style":98},"echo 1 | sudo tee \u002Fproc\u002Fsys\u002Fkernel\u002Fsysrq   # enable\n",[34,1641,1642],{"__ignoreMap":98},[102,1643,1644,1647,1649,1651,1654,1657,1660],{"class":104,"line":105},[102,1645,1646],{"class":123},"echo",[102,1648,241],{"class":123},[102,1650,245],{"class":244},[102,1652,1653],{"class":115}," sudo",[102,1655,1656],{"class":119}," tee",[102,1658,1659],{"class":119}," \u002Fproc\u002Fsys\u002Fkernel\u002Fsysrq",[102,1661,1662],{"class":108},"   # enable\n",[14,1664,1665,1666,1669],{},"Mnemonic: ",[29,1667,1668],{},"REISUB"," (R-E-I-S-U-B) — the safe reboot sequence when frozen:",[577,1671,1672,1678,1684,1690,1696,1702],{},[26,1673,1674,1677],{},[34,1675,1676],{},"Alt+SysRq+r"," — take back control of keyboard from X.",[26,1679,1680,1683],{},[34,1681,1682],{},"Alt+SysRq+e"," — SIGTERM all processes (except PID 1).",[26,1685,1686,1689],{},[34,1687,1688],{},"Alt+SysRq+i"," — SIGKILL all processes.",[26,1691,1692,1695],{},[34,1693,1694],{},"Alt+SysRq+s"," — sync all filesystems.",[26,1697,1698,1701],{},[34,1699,1700],{},"Alt+SysRq+u"," — remount all filesystems read-only.",[26,1703,1704,1707],{},[34,1705,1706],{},"Alt+SysRq+b"," — reboot (force).",[14,1709,1710,1711,1714,1715,1718],{},"Always ",[34,1712,1713],{},"s"," (sync) before ",[34,1716,1717],{},"b"," (reboot) to avoid data loss.",[18,1720,1722],{"id":1721},"tips-tricks","💡 Tips & Tricks",[577,1724,1725,1739,1747,1758,1766,1777,1786],{},[26,1726,1727,1730,1731,1734,1735,1738],{},[29,1728,1729],{},"Idiom",": use ",[34,1732,1733],{},"journalctl -b -p err"," as your first \"what went wrong?\" query — limits to current boot, errors only. Add ",[34,1736,1737],{},"--since"," for a time window.",[26,1740,1741,1730,1743,1746],{},[29,1742,1729],{},[34,1744,1745],{},"journalctl -b -1 -p err"," for previous-boot debugging — when a reboot happened (crash, OOM), shows what happened before.",[26,1748,1749,1730,1751,1754,1755,1757],{},[29,1750,1729],{},[34,1752,1753],{},"lsof +L1"," to find deleted-but-open files — explains \"disk full but ",[34,1756,705],{}," doesn't show it.\" A process holds a large deleted file open; space not freed until fd closes.",[26,1759,1760,1730,1762,1765],{},[29,1761,1729],{},[34,1763,1764],{},"strace -e trace=open,openat \u003Ccmd> 2>&1 | grep ENOENT"," to find missing files — reveals exactly which path a program can't find.",[26,1767,1768,1730,1770,1773,1774,1776],{},[29,1769,1729],{},[34,1771,1772],{},"namei -l \u002Fpath\u002Fto\u002Ffile"," to see permissions on each path component — reveals when a \"Permission denied\" is due to a parent directory lacking ",[34,1775,586],{},", not the file itself.",[26,1778,1779,1730,1782,1785],{},[29,1780,1781],{},"Debug",[34,1783,1784],{},"sudo ausearch -m AVC -ts recent"," when SELinux blocks something — shows the exact denial. Common on RHEL.",[26,1787,1788,1790,1791,1714,1793,1718],{},[29,1789,1781],{},": REISUB is the safe reboot sequence when the system is frozen — always ",[34,1792,1713],{},[34,1794,1717],{},[18,1796,1798],{"id":1797},"️-edge-cases-gotchas","⚠️ Edge Cases & Gotchas",[577,1800,1801,1809,1815,1827,1850,1868,1880],{},[26,1802,1803,1806,1807,37],{},[29,1804,1805],{},"Don't reboot immediately",": rebooting destroys evidence (logs, process state, ",[34,1808,36],{},[26,1810,1811,1814],{},[29,1812,1813],{},"\"Connection refused\" vs \"Connection timed out\"",": refused = something is listening but refused you (port closed, or firewall REJECT). Timed out = nothing responded (firewall DROP, or network issue). The distinction matters.",[26,1816,1817,1823,1824,1826],{},[29,1818,1819,1822],{},[34,1820,1821],{},"fsck"," on a mounted filesystem can corrupt it",": never ",[34,1825,1821],{}," a mounted ext4\u002Fxfs. Unmount first, or boot from a live USB.",[26,1828,1829,1835,1836,1838,1839,1842,1843,1846,1847,601],{},[29,1830,1831,1834],{},[34,1832,1833],{},"\u002Fvar\u002Flog"," can fill up",": if logs aren't rotated, ",[34,1837,1833],{}," fills up, causing services to fail. Set ",[34,1840,1841],{},"SystemMaxUse="," in ",[34,1844,1845],{},"journald.conf"," and configure ",[34,1848,1849],{},"logrotate",[26,1851,1852,1855,1856,1859,1860,1863,1864,1867],{},[29,1853,1854],{},"Auth log locations differ by distro",": Debian\u002FUbuntu uses ",[34,1857,1858],{},"\u002Fvar\u002Flog\u002Fauth.log","; RHEL\u002FRocky uses ",[34,1861,1862],{},"\u002Fvar\u002Flog\u002Fsecure",". Use ",[34,1865,1866],{},"journalctl -u ssh"," for portability.",[26,1869,1870,1875,1876,1879],{},[29,1871,1872,1874],{},[34,1873,50],{}," is cleared on reboot",": for persistent kernel logs, rely on ",[34,1877,1878],{},"journalctl -k"," (which persists if journald is persistent).",[26,1881,1882,607,1885,1888,1889,1892],{},[29,1883,1884],{},"Locked package manager",[34,1886,1887],{},"sudo rm \u002Fvar\u002Flib\u002Fdpkg\u002Flock-frontend"," only if you're SURE no apt is running. Check with ",[34,1890,1891],{},"lsof \u002Fvar\u002Flib\u002Fdpkg\u002Flock-frontend"," first.",[18,1894,1896],{"id":1895},"quick-quiz","🧠 Quick Quiz",[14,1898,1899,1900,1902,1903,1905],{},"A production server is suddenly unresponsive — SSH hangs, ",[34,1901,233],{}," freezes, even ",[34,1904,496],{}," takes 30 seconds. You can't get a shell. The system doesn't respond to Ctrl+Alt+Del. What tool can save you, and what's the procedure?",[1907,1908,1909,1913,1919,1936,1941,1995,2007,2010,2072],"details",{},[1910,1911,1912],"summary",{},"Answer",[14,1914,1915,1918],{},[29,1916,1917],{},"Magic SysRq (REISUB)"," — the kernel's built-in emergency recovery mechanism, accessible even when the system is completely frozen.",[14,1920,1921,607,1924,1927,1928,1931,1932,1935],{},[29,1922,1923],{},"Prerequisites",[34,1925,1926],{},"kernel.sysrq"," must be enabled (",[34,1929,1930],{},"echo 1 | sudo tee \u002Fproc\u002Fsys\u002Fkernel\u002Fsysrq","). If it's not enabled, you may not be able to use it (enable it in ",[34,1933,1934],{},"\u002Fetc\u002Fsysctl.d\u002F"," before you need it).",[14,1937,1938],{},[29,1939,1940],{},"Procedure — REISUB:",[23,1942,1943,1952,1960,1968,1979,1987],{},[26,1944,1945,1948,1949,1951],{},[29,1946,1947],{},"R"," (",[34,1950,1676],{},") — take back control of the keyboard from X\u002Fdisplay manager.",[26,1953,1954,1948,1957,1959],{},[29,1955,1956],{},"E",[34,1958,1682],{},") — SIGTERM all processes (except PID 1). Gives processes a chance to shut down gracefully.",[26,1961,1962,1948,1965,1967],{},[29,1963,1964],{},"I",[34,1966,1688],{},") — SIGKILL all processes that didn't respond to SIGTERM.",[26,1969,1970,1948,1973,1975,1976],{},[29,1971,1972],{},"S",[34,1974,1694],{},") — sync all filesystems (flush dirty pages to disk). ",[29,1977,1978],{},"Wait for the disk light to stop.",[26,1980,1981,1948,1984,1986],{},[29,1982,1983],{},"U",[34,1985,1700],{},") — remount all filesystems read-only (clean state for reboot).",[26,1988,1989,1948,1992,1994],{},[29,1990,1991],{},"B",[34,1993,1706],{},") — reboot.",[14,1996,1997,1998,2000,2001,2003,2004,2006],{},"The critical steps are ",[29,1999,1972],{}," (sync) and ",[29,2002,1983],{}," (remount read-only) before ",[29,2005,1991],{}," (reboot) — this prevents filesystem corruption.",[14,2008,2009],{},"If you don't have a SysRq key (some laptops don't), you can trigger via command line (if you have ANY shell access, even a serial console):",[90,2011,2012],{"language":92},[94,2013,2015],{"className":96,"code":2014,"language":92,"meta":98,"style":98},"echo s | sudo tee \u002Fproc\u002Fsysrq-trigger   # sync\necho u | sudo tee \u002Fproc\u002Fsysrq-trigger   # remount ro\necho b | sudo tee \u002Fproc\u002Fsysrq-trigger   # reboot\n",[34,2016,2017,2036,2054],{"__ignoreMap":98},[102,2018,2019,2021,2024,2026,2028,2030,2033],{"class":104,"line":105},[102,2020,1646],{"class":123},[102,2022,2023],{"class":119}," s",[102,2025,245],{"class":244},[102,2027,1653],{"class":115},[102,2029,1656],{"class":119},[102,2031,2032],{"class":119}," \u002Fproc\u002Fsysrq-trigger",[102,2034,2035],{"class":108},"   # sync\n",[102,2037,2038,2040,2043,2045,2047,2049,2051],{"class":104,"line":112},[102,2039,1646],{"class":123},[102,2041,2042],{"class":119}," u",[102,2044,245],{"class":244},[102,2046,1653],{"class":115},[102,2048,1656],{"class":119},[102,2050,2032],{"class":119},[102,2052,2053],{"class":108},"   # remount ro\n",[102,2055,2056,2058,2061,2063,2065,2067,2069],{"class":104,"line":130},[102,2057,1646],{"class":123},[102,2059,2060],{"class":119}," b",[102,2062,245],{"class":244},[102,2064,1653],{"class":115},[102,2066,1656],{"class":119},[102,2068,2032],{"class":119},[102,2070,2071],{"class":108},"   # reboot\n",[14,2073,2074,2077],{},[29,2075,2076],{},"Why this matters",": yanking the power on a frozen system can corrupt the filesystem (dirty pages not flushed). REISUB ensures a clean shutdown even when nothing else works.",[2079,2080,2081],"style",{},"html pre.shiki code .sdCPZ, html code.shiki .sdCPZ{--shiki-default:#6A737D;--shiki-github-dark:#6A737D}html pre.shiki code .sIsaT, html code.shiki .sIsaT{--shiki-default:#6F42C1;--shiki-github-dark:#B392F0}html pre.shiki code .sJ6F3, html code.shiki .sJ6F3{--shiki-default:#032F62;--shiki-github-dark:#9ECBFF}html pre.shiki code .snvgF, html code.shiki .snvgF{--shiki-default:#005CC5;--shiki-github-dark:#79B8FF}html pre.shiki code .svdQ7, html code.shiki .svdQ7{--shiki-default:#D73A49;--shiki-github-dark:#F97583}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .github-dark .shiki span {color: var(--shiki-github-dark);background: var(--shiki-github-dark-bg);font-style: var(--shiki-github-dark-font-style);font-weight: var(--shiki-github-dark-font-weight);text-decoration: var(--shiki-github-dark-text-decoration);}html.github-dark .shiki span {color: var(--shiki-github-dark);background: var(--shiki-github-dark-bg);font-style: var(--shiki-github-dark-font-style);font-weight: var(--shiki-github-dark-font-weight);text-decoration: var(--shiki-github-dark-text-decoration);}",{"title":98,"searchDepth":112,"depth":112,"links":2083},[2084,2085,2086,2095,2103,2113,2114,2115],{"id":20,"depth":112,"text":21},{"id":87,"depth":112,"text":88},{"id":257,"depth":112,"text":258,"children":2087},[2088,2089,2090,2091,2093,2094],{"id":262,"depth":130,"text":263},{"id":478,"depth":130,"text":479},{"id":620,"depth":130,"text":621},{"id":701,"depth":130,"text":2092},"\"Disk full\" but du Doesn't Show It",{"id":785,"depth":130,"text":786},{"id":909,"depth":130,"text":910},{"id":987,"depth":112,"text":988,"children":2096},[2097,2098,2100,2101],{"id":991,"depth":130,"text":992},{"id":1046,"depth":130,"text":2099},"init=\u002Fbin\u002Fbash (When systemd is broken)",{"id":1108,"depth":130,"text":1109},{"id":1296,"depth":130,"text":2102},"Fixing a Broken fstab",{"id":1370,"depth":112,"text":1371,"children":2104},[2105,2107,2109,2111],{"id":1374,"depth":130,"text":2106},"strace — Trace Syscalls",{"id":1468,"depth":130,"text":2108},"lsof — List Open Files",{"id":1533,"depth":130,"text":2110},"tcpdump — Packet Capture",{"id":1627,"depth":130,"text":2112},"sysrq (Magic SysRq) — REISUB",{"id":1721,"depth":112,"text":1722},{"id":1797,"depth":112,"text":1798},{"id":1895,"depth":112,"text":1896},"md",{},"\u002Flinux\u002F18-troubleshooting-and-recovery",{"title":5,"description":16},"linux\u002F18-troubleshooting-and-recovery","G0BkhvMYmmdYSPhUs2kW2at4ZzBxyAqugzHYnFQG8v0",1789924650156]