[{"data":1,"prerenderedAt":2102},["ShallowReactive",2],{"page-\u002Flinux\u002F12-logging-and-monitoring":3},{"id":4,"title":5,"body":6,"description":2095,"extension":2096,"meta":2097,"navigation":473,"path":2098,"seo":2099,"stem":2100,"__hash__":2101},"content\u002Flinux\u002F12-logging-and-monitoring.md","12 — Logging & Monitoring",{"type":7,"value":8,"toc":2059},"minimark",[9,13,22,27,209,216,354,359,443,447,526,534,634,638,699,703,763,769,918,926,1011,1018,1075,1082,1149,1153,1270,1276,1390,1397,1436,1443,1500,1504,1507,1542,1585,1589,1621,1625,1765,1769,1888,1892,1899,1918,1928,2055],[10,11,5],"h1",{"id":12},"_12-logging-monitoring",[14,15,16,17,21],"p",{},"Logs are how you know what's happening on your system. This chapter covers systemd-journald (the modern default), traditional syslog, ",[18,19,20],"code",{},"logrotate",", kernel logs, and real-time monitoring tools — through the lens of production diagnostics and post-incident forensics.",[23,24,26],"h2",{"id":25},"the-logging-landscape","The Logging Landscape",[28,29,31],"code-wrapper",{"language":30},"bash",[32,33,37],"pre",{"className":34,"code":35,"language":30,"meta":36,"style":36},"language-bash shiki shiki-themes github-light github-dark","┌────────────────────────────────────────────────┐\n│  Application \u002F Service                          │\n│     ↓ (stdout\u002Fstderr → journald, or syslog API) │\n├────────────────────────────────────────────────┤\n│  journald (systemd)   ← \u002Fvar\u002Flog\u002Fjournal\u002F       │\n│     ↓ (can forward to rsyslog)                  │\n├────────────────────────────────────────────────┤\n│  rsyslog (traditional) ← \u002Fvar\u002Flog\u002F*.log         │\n│     ↓ (can forward to remote)                   │\n├────────────────────────────────────────────────┤\n│  logrotate (rotates\u002Farchives old logs)          │\n└────────────────────────────────────────────────┘\n","",[18,38,39,48,67,97,103,114,136,141,159,178,183,203],{"__ignoreMap":36},[40,41,44],"span",{"class":42,"line":43},"line",1,[40,45,47],{"class":46},"sIsaT","┌────────────────────────────────────────────────┐\n",[40,49,51,54,58,61,64],{"class":42,"line":50},2,[40,52,53],{"class":46},"│",[40,55,57],{"class":56},"sJ6F3","  Application",[40,59,60],{"class":56}," \u002F",[40,62,63],{"class":56}," Service",[40,65,66],{"class":56},"                          │\n",[40,68,70,72,75,79,82,85,88,91,94],{"class":42,"line":69},3,[40,71,53],{"class":46},[40,73,74],{"class":56},"     ↓",[40,76,78],{"class":77},"ssxIu"," (stdout\u002Fstderr ",[40,80,81],{"class":56},"→",[40,83,84],{"class":56}," journald,",[40,86,87],{"class":56}," or",[40,89,90],{"class":56}," syslog",[40,92,93],{"class":56}," API",[40,95,96],{"class":77},") │\n",[40,98,100],{"class":42,"line":99},4,[40,101,102],{"class":46},"├────────────────────────────────────────────────┤\n",[40,104,106,108,111],{"class":42,"line":105},5,[40,107,53],{"class":46},[40,109,110],{"class":56},"  journald",[40,112,113],{"class":77}," (systemd)   ← \u002Fvar\u002Flog\u002Fjournal\u002F       │\n",[40,115,117,119,121,124,127,130,133],{"class":42,"line":116},6,[40,118,53],{"class":46},[40,120,74],{"class":56},[40,122,123],{"class":77}," (can ",[40,125,126],{"class":56},"forward",[40,128,129],{"class":56}," to",[40,131,132],{"class":56}," rsyslog",[40,134,135],{"class":77},")                  │\n",[40,137,139],{"class":42,"line":138},7,[40,140,102],{"class":46},[40,142,144,146,149,152,156],{"class":42,"line":143},8,[40,145,53],{"class":46},[40,147,148],{"class":56},"  rsyslog",[40,150,151],{"class":77}," (traditional) ← \u002Fvar\u002Flog\u002F",[40,153,155],{"class":154},"svdQ7","*",[40,157,158],{"class":77},".log         │\n",[40,160,162,164,166,168,170,172,175],{"class":42,"line":161},9,[40,163,53],{"class":46},[40,165,74],{"class":56},[40,167,123],{"class":77},[40,169,126],{"class":56},[40,171,129],{"class":56},[40,173,174],{"class":56}," remote",[40,176,177],{"class":77},")                   │\n",[40,179,181],{"class":42,"line":180},10,[40,182,102],{"class":46},[40,184,186,188,191,194,197,200],{"class":42,"line":185},11,[40,187,53],{"class":46},[40,189,190],{"class":56},"  logrotate",[40,192,193],{"class":77}," (rotates\u002Farchives ",[40,195,196],{"class":56},"old",[40,198,199],{"class":56}," logs",[40,201,202],{"class":77},")          │\n",[40,204,206],{"class":42,"line":205},12,[40,207,208],{"class":46},"└────────────────────────────────────────────────┘\n",[23,210,212,215],{"id":211},"journalctl-the-log-viewer",[18,213,214],{},"journalctl"," — The Log Viewer",[28,217,218],{"language":30},[32,219,221],{"className":34,"code":220,"language":30,"meta":36,"style":36},"# Complex Implementation: targeted post-incident log analysis\njournalctl -b -p err                            # current boot, errors only\njournalctl -b -1 -p err                         # PREVIOUS boot, errors (after crash\u002Freboot)\njournalctl --list-boots                         # all boots with timestamps\njournalctl -u nginx -f                          # follow nginx logs (like tail -f)\njournalctl --since \"2026-09-10 10:00\" --until \"2026-09-10 12:00\"\njournalctl -k                                    # kernel logs only (like dmesg)\njournalctl -p warning..err                      # priority range\njournalctl --vacuum-size=500M                    # keep logs under 500 MB\njournalctl -o json -u myapp                      # JSON output (for log shippers)\n",[18,222,223,229,246,262,272,288,304,314,326,336],{"__ignoreMap":36},[40,224,225],{"class":42,"line":43},[40,226,228],{"class":227},"sdCPZ","# Complex Implementation: targeted post-incident log analysis\n",[40,230,231,233,237,240,243],{"class":42,"line":50},[40,232,214],{"class":46},[40,234,236],{"class":235},"snvgF"," -b",[40,238,239],{"class":235}," -p",[40,241,242],{"class":56}," err",[40,244,245],{"class":227},"                            # current boot, errors only\n",[40,247,248,250,252,255,257,259],{"class":42,"line":69},[40,249,214],{"class":46},[40,251,236],{"class":235},[40,253,254],{"class":235}," -1",[40,256,239],{"class":235},[40,258,242],{"class":56},[40,260,261],{"class":227},"                         # PREVIOUS boot, errors (after crash\u002Freboot)\n",[40,263,264,266,269],{"class":42,"line":99},[40,265,214],{"class":46},[40,267,268],{"class":235}," --list-boots",[40,270,271],{"class":227},"                         # all boots with timestamps\n",[40,273,274,276,279,282,285],{"class":42,"line":105},[40,275,214],{"class":46},[40,277,278],{"class":235}," -u",[40,280,281],{"class":56}," nginx",[40,283,284],{"class":235}," -f",[40,286,287],{"class":227},"                          # follow nginx logs (like tail -f)\n",[40,289,290,292,295,298,301],{"class":42,"line":116},[40,291,214],{"class":46},[40,293,294],{"class":235}," --since",[40,296,297],{"class":56}," \"2026-09-10 10:00\"",[40,299,300],{"class":235}," --until",[40,302,303],{"class":56}," \"2026-09-10 12:00\"\n",[40,305,306,308,311],{"class":42,"line":138},[40,307,214],{"class":46},[40,309,310],{"class":235}," -k",[40,312,313],{"class":227},"                                    # kernel logs only (like dmesg)\n",[40,315,316,318,320,323],{"class":42,"line":143},[40,317,214],{"class":46},[40,319,239],{"class":235},[40,321,322],{"class":56}," warning..err",[40,324,325],{"class":227},"                      # priority range\n",[40,327,328,330,333],{"class":42,"line":161},[40,329,214],{"class":46},[40,331,332],{"class":235}," --vacuum-size=500M",[40,334,335],{"class":227},"                    # keep logs under 500 MB\n",[40,337,338,340,343,346,348,351],{"class":42,"line":180},[40,339,214],{"class":46},[40,341,342],{"class":235}," -o",[40,344,345],{"class":56}," json",[40,347,278],{"class":235},[40,349,350],{"class":56}," myapp",[40,352,353],{"class":227},"                      # JSON output (for log shippers)\n",[355,356,358],"h3",{"id":357},"priority-levels","Priority Levels",[360,361,362,375],"table",{},[363,364,365],"thead",{},[366,367,368,372],"tr",{},[369,370,371],"th",{},"Code",[369,373,374],{},"Level",[376,377,378,387,395,403,411,419,427,435],"tbody",{},[366,379,380,384],{},[381,382,383],"td",{},"0",[381,385,386],{},"emerg",[366,388,389,392],{},[381,390,391],{},"1",[381,393,394],{},"alert",[366,396,397,400],{},[381,398,399],{},"2",[381,401,402],{},"crit",[366,404,405,408],{},[381,406,407],{},"3",[381,409,410],{},"err",[366,412,413,416],{},[381,414,415],{},"4",[381,417,418],{},"warning",[366,420,421,424],{},[381,422,423],{},"5",[381,425,426],{},"notice",[366,428,429,432],{},[381,430,431],{},"6",[381,433,434],{},"info",[366,436,437,440],{},[381,438,439],{},"7",[381,441,442],{},"debug",[355,444,446],{"id":445},"edge-case-journald-logs-are-lost-on-reboot-by-default","Edge Case: journald Logs Are Lost on Reboot by Default",[28,448,449],{"language":30},[32,450,452],{"className":34,"code":451,"language":30,"meta":36,"style":36},"# NAIVE: expect pre-crash logs to survive reboot\n# By default, logs are in \u002Frun\u002Flog\u002Fjournal\u002F (RAM, volatile)\n# After a crash + reboot, pre-crash logs are GONE\n\n# PRODUCTION: make logs persistent\nsudo mkdir -p \u002Fvar\u002Flog\u002Fjournal\nsudo systemd-tmpfiles --create --prefix \u002Fvar\u002Flog\u002Fjournal\nsudo systemctl restart systemd-journald\n# Or set Storage=persistent in \u002Fetc\u002Fsystemd\u002Fjournald.conf\n",[18,453,454,459,464,469,475,480,493,508,521],{"__ignoreMap":36},[40,455,456],{"class":42,"line":43},[40,457,458],{"class":227},"# NAIVE: expect pre-crash logs to survive reboot\n",[40,460,461],{"class":42,"line":50},[40,462,463],{"class":227},"# By default, logs are in \u002Frun\u002Flog\u002Fjournal\u002F (RAM, volatile)\n",[40,465,466],{"class":42,"line":69},[40,467,468],{"class":227},"# After a crash + reboot, pre-crash logs are GONE\n",[40,470,471],{"class":42,"line":99},[40,472,474],{"emptyLinePlaceholder":473},true,"\n",[40,476,477],{"class":42,"line":105},[40,478,479],{"class":227},"# PRODUCTION: make logs persistent\n",[40,481,482,485,488,490],{"class":42,"line":116},[40,483,484],{"class":46},"sudo",[40,486,487],{"class":56}," mkdir",[40,489,239],{"class":235},[40,491,492],{"class":56}," \u002Fvar\u002Flog\u002Fjournal\n",[40,494,495,497,500,503,506],{"class":42,"line":138},[40,496,484],{"class":46},[40,498,499],{"class":56}," systemd-tmpfiles",[40,501,502],{"class":235}," --create",[40,504,505],{"class":235}," --prefix",[40,507,492],{"class":56},[40,509,510,512,515,518],{"class":42,"line":143},[40,511,484],{"class":46},[40,513,514],{"class":56}," systemctl",[40,516,517],{"class":56}," restart",[40,519,520],{"class":56}," systemd-journald\n",[40,522,523],{"class":42,"line":161},[40,524,525],{"class":227},"# Or set Storage=persistent in \u002Fetc\u002Fsystemd\u002Fjournald.conf\n",[23,527,529,530,533],{"id":528},"traditional-syslog-varlog","Traditional Syslog (",[18,531,532],{},"\u002Fvar\u002Flog\u002F",")",[360,535,536,546],{},[363,537,538],{},[366,539,540,543],{},[369,541,542],{},"File",[369,544,545],{},"Contents",[376,547,548,569,588,598,608,621],{},[366,549,550,559],{},[381,551,552,555,556],{},[18,553,554],{},"\u002Fvar\u002Flog\u002Fsyslog"," or ",[18,557,558],{},"\u002Fvar\u002Flog\u002Fmessages",[381,560,561,562,565,566,533],{},"General system log (Debian: ",[18,563,564],{},"syslog",", RHEL: ",[18,567,568],{},"messages",[366,570,571,579],{},[381,572,573,555,576],{},[18,574,575],{},"\u002Fvar\u002Flog\u002Fauth.log",[18,577,578],{},"\u002Fvar\u002Flog\u002Fsecure",[381,580,581,582,565,585,533],{},"Auth: logins, sudo, su (Debian: ",[18,583,584],{},"auth.log",[18,586,587],{},"secure",[366,589,590,595],{},[381,591,592],{},[18,593,594],{},"\u002Fvar\u002Flog\u002Fkern.log",[381,596,597],{},"Kernel messages",[366,599,600,605],{},[381,601,602],{},[18,603,604],{},"\u002Fvar\u002Flog\u002Fcron",[381,606,607],{},"Cron job execution",[366,609,610,615],{},[381,611,612],{},[18,613,614],{},"\u002Fvar\u002Flog\u002Fwtmp",[381,616,617,618,533],{},"Login records (binary — use ",[18,619,620],{},"last",[366,622,623,628],{},[381,624,625],{},[18,626,627],{},"\u002Fvar\u002Flog\u002Fbtmp",[381,629,630,631,533],{},"Failed login attempts (binary — use ",[18,632,633],{},"lastb",[355,635,637],{"id":636},"edge-case-auth-log-locations-differ-by-distro","Edge Case: Auth Log Locations Differ by Distro",[28,639,640],{"language":30},[32,641,643],{"className":34,"code":642,"language":30,"meta":36,"style":36},"# NAIVE: script hardcodes \u002Fvar\u002Flog\u002Fauth.log\n# → fails on RHEL (which uses \u002Fvar\u002Flog\u002Fsecure)\ngrep \"Failed password\" \u002Fvar\u002Flog\u002Fauth.log\n\n# PRODUCTION: use journalctl (portable across distros)\njournalctl -u ssh -u sudo --since today\n# Or: journalctl _COMM=sudo\n",[18,644,645,650,655,666,670,675,694],{"__ignoreMap":36},[40,646,647],{"class":42,"line":43},[40,648,649],{"class":227},"# NAIVE: script hardcodes \u002Fvar\u002Flog\u002Fauth.log\n",[40,651,652],{"class":42,"line":50},[40,653,654],{"class":227},"# → fails on RHEL (which uses \u002Fvar\u002Flog\u002Fsecure)\n",[40,656,657,660,663],{"class":42,"line":69},[40,658,659],{"class":46},"grep",[40,661,662],{"class":56}," \"Failed password\"",[40,664,665],{"class":56}," \u002Fvar\u002Flog\u002Fauth.log\n",[40,667,668],{"class":42,"line":99},[40,669,474],{"emptyLinePlaceholder":473},[40,671,672],{"class":42,"line":105},[40,673,674],{"class":227},"# PRODUCTION: use journalctl (portable across distros)\n",[40,676,677,679,681,684,686,689,691],{"class":42,"line":116},[40,678,214],{"class":46},[40,680,278],{"class":235},[40,682,683],{"class":56}," ssh",[40,685,278],{"class":235},[40,687,688],{"class":56}," sudo",[40,690,294],{"class":235},[40,692,693],{"class":56}," today\n",[40,695,696],{"class":42,"line":138},[40,697,698],{"class":227},"# Or: journalctl _COMM=sudo\n",[355,700,702],{"id":701},"reading-binary-logs","Reading Binary Logs",[28,704,705],{"language":30},[32,706,708],{"className":34,"code":707,"language":30,"meta":36,"style":36},"last                  # login history (from wtmp)\nlast -n 20            # last 20 logins\nlast reboot           # reboot history\nlastb                 # failed login attempts (from btmp)\nwho                   # currently logged in (from utmp)\nw                     # who + what they're doing\n",[18,709,710,717,730,740,747,755],{"__ignoreMap":36},[40,711,712,714],{"class":42,"line":43},[40,713,620],{"class":46},[40,715,716],{"class":227},"                  # login history (from wtmp)\n",[40,718,719,721,724,727],{"class":42,"line":50},[40,720,620],{"class":46},[40,722,723],{"class":235}," -n",[40,725,726],{"class":235}," 20",[40,728,729],{"class":227},"            # last 20 logins\n",[40,731,732,734,737],{"class":42,"line":69},[40,733,620],{"class":46},[40,735,736],{"class":56}," reboot",[40,738,739],{"class":227},"           # reboot history\n",[40,741,742,744],{"class":42,"line":99},[40,743,633],{"class":46},[40,745,746],{"class":227},"                 # failed login attempts (from btmp)\n",[40,748,749,752],{"class":42,"line":105},[40,750,751],{"class":46},"who",[40,753,754],{"class":227},"                   # currently logged in (from utmp)\n",[40,756,757,760],{"class":42,"line":116},[40,758,759],{"class":46},"w",[40,761,762],{"class":227},"                     # who + what they're doing\n",[23,764,766,768],{"id":765},"logrotate-manage-log-size",[18,767,20],{}," — Manage Log Size",[28,770,771],{"language":30},[32,772,774],{"className":34,"code":773,"language":30,"meta":36,"style":36},"# Complex Implementation: logrotate config for nginx\n# \u002Fetc\u002Flogrotate.d\u002Fnginx\nsudo tee \u002Fetc\u002Flogrotate.d\u002Fnginx \u003C\u003C'EOF'\n\u002Fvar\u002Flog\u002Fnginx\u002F*.log {\n    daily\n    rotate 14\n    missingok\n    compress\n    delaycompress\n    notifempty\n    create 640 www-data adm\n    sharedscripts\n    postrotate\n        systemctl reload nginx >\u002Fdev\u002Fnull 2>&1 || true\n    endscript\n}\nEOF\n\n# Dry-run (debug) before relying on it:\nsudo logrotate -d \u002Fetc\u002Flogrotate.d\u002Fnginx    # shows what WOULD happen (no changes)\nsudo logrotate -f \u002Fetc\u002Flogrotate.d\u002Fnginx    # force rotation now\n",[18,775,776,781,786,802,807,812,817,822,827,832,837,842,847,853,859,865,871,877,882,888,904],{"__ignoreMap":36},[40,777,778],{"class":42,"line":43},[40,779,780],{"class":227},"# Complex Implementation: logrotate config for nginx\n",[40,782,783],{"class":42,"line":50},[40,784,785],{"class":227},"# \u002Fetc\u002Flogrotate.d\u002Fnginx\n",[40,787,788,790,793,796,799],{"class":42,"line":69},[40,789,484],{"class":46},[40,791,792],{"class":56}," tee",[40,794,795],{"class":56}," \u002Fetc\u002Flogrotate.d\u002Fnginx",[40,797,798],{"class":154}," \u003C\u003C",[40,800,801],{"class":56},"'EOF'\n",[40,803,804],{"class":42,"line":99},[40,805,806],{"class":56},"\u002Fvar\u002Flog\u002Fnginx\u002F*.log {\n",[40,808,809],{"class":42,"line":105},[40,810,811],{"class":56},"    daily\n",[40,813,814],{"class":42,"line":116},[40,815,816],{"class":56},"    rotate 14\n",[40,818,819],{"class":42,"line":138},[40,820,821],{"class":56},"    missingok\n",[40,823,824],{"class":42,"line":143},[40,825,826],{"class":56},"    compress\n",[40,828,829],{"class":42,"line":161},[40,830,831],{"class":56},"    delaycompress\n",[40,833,834],{"class":42,"line":180},[40,835,836],{"class":56},"    notifempty\n",[40,838,839],{"class":42,"line":185},[40,840,841],{"class":56},"    create 640 www-data adm\n",[40,843,844],{"class":42,"line":205},[40,845,846],{"class":56},"    sharedscripts\n",[40,848,850],{"class":42,"line":849},13,[40,851,852],{"class":56},"    postrotate\n",[40,854,856],{"class":42,"line":855},14,[40,857,858],{"class":56},"        systemctl reload nginx >\u002Fdev\u002Fnull 2>&1 || true\n",[40,860,862],{"class":42,"line":861},15,[40,863,864],{"class":56},"    endscript\n",[40,866,868],{"class":42,"line":867},16,[40,869,870],{"class":56},"}\n",[40,872,874],{"class":42,"line":873},17,[40,875,876],{"class":56},"EOF\n",[40,878,880],{"class":42,"line":879},18,[40,881,474],{"emptyLinePlaceholder":473},[40,883,885],{"class":42,"line":884},19,[40,886,887],{"class":227},"# Dry-run (debug) before relying on it:\n",[40,889,891,893,896,899,901],{"class":42,"line":890},20,[40,892,484],{"class":46},[40,894,895],{"class":56}," logrotate",[40,897,898],{"class":235}," -d",[40,900,795],{"class":56},[40,902,903],{"class":227},"    # shows what WOULD happen (no changes)\n",[40,905,907,909,911,913,915],{"class":42,"line":906},21,[40,908,484],{"class":46},[40,910,895],{"class":56},[40,912,284],{"class":235},[40,914,795],{"class":56},[40,916,917],{"class":227},"    # force rotation now\n",[355,919,921,922,925],{"id":920},"caveat-copytruncate-loses-lines","Caveat: ",[18,923,924],{},"copytruncate"," Loses Lines",[28,927,928],{"language":30},[32,929,931],{"className":34,"code":930,"language":30,"meta":36,"style":36},"# NAIVE: copytruncate for apps that hold the file open\n# copytruncate copies the log then truncates the original (app keeps writing to same fd)\n# BUT: lines written between copy and truncate are LOST\ncopytruncate\n\n# PRODUCTION: use create + postrotate (app reopens the new file)\ncreate 640 www-data adm\npostrotate\n    systemctl reload nginx >\u002Fdev\u002Fnull 2>&1 || true\nendscript\n",[18,932,933,938,943,948,953,957,962,976,981,1006],{"__ignoreMap":36},[40,934,935],{"class":42,"line":43},[40,936,937],{"class":227},"# NAIVE: copytruncate for apps that hold the file open\n",[40,939,940],{"class":42,"line":50},[40,941,942],{"class":227},"# copytruncate copies the log then truncates the original (app keeps writing to same fd)\n",[40,944,945],{"class":42,"line":69},[40,946,947],{"class":227},"# BUT: lines written between copy and truncate are LOST\n",[40,949,950],{"class":42,"line":99},[40,951,952],{"class":46},"copytruncate\n",[40,954,955],{"class":42,"line":105},[40,956,474],{"emptyLinePlaceholder":473},[40,958,959],{"class":42,"line":116},[40,960,961],{"class":227},"# PRODUCTION: use create + postrotate (app reopens the new file)\n",[40,963,964,967,970,973],{"class":42,"line":138},[40,965,966],{"class":46},"create",[40,968,969],{"class":235}," 640",[40,971,972],{"class":56}," www-data",[40,974,975],{"class":56}," adm\n",[40,977,978],{"class":42,"line":143},[40,979,980],{"class":46},"postrotate\n",[40,982,983,986,989,991,994,997,1000,1003],{"class":42,"line":161},[40,984,985],{"class":46},"    systemctl",[40,987,988],{"class":56}," reload",[40,990,281],{"class":56},[40,992,993],{"class":154}," >",[40,995,996],{"class":56},"\u002Fdev\u002Fnull",[40,998,999],{"class":154}," 2>&1",[40,1001,1002],{"class":154}," ||",[40,1004,1005],{"class":235}," true\n",[40,1007,1008],{"class":42,"line":180},[40,1009,1010],{"class":46},"endscript\n",[23,1012,1014,1015],{"id":1013},"kernel-logs-dmesg","Kernel Logs — ",[18,1016,1017],{},"dmesg",[28,1019,1020],{"language":30},[32,1021,1023],{"className":34,"code":1022,"language":30,"meta":36,"style":36},"# Complex Implementation: kernel error diagnosis\ndmesg -T --level=err,warn         # with timestamps, errors + warnings only\ndmesg | grep -i \"error\\|fail\"    # filter for errors\njournalctl -k -p err             # same, from journald (persists if journald is persistent)\n",[18,1024,1025,1030,1043,1062],{"__ignoreMap":36},[40,1026,1027],{"class":42,"line":43},[40,1028,1029],{"class":227},"# Complex Implementation: kernel error diagnosis\n",[40,1031,1032,1034,1037,1040],{"class":42,"line":50},[40,1033,1017],{"class":46},[40,1035,1036],{"class":235}," -T",[40,1038,1039],{"class":235}," --level=err,warn",[40,1041,1042],{"class":227},"         # with timestamps, errors + warnings only\n",[40,1044,1045,1047,1050,1053,1056,1059],{"class":42,"line":69},[40,1046,1017],{"class":46},[40,1048,1049],{"class":154}," |",[40,1051,1052],{"class":46}," grep",[40,1054,1055],{"class":235}," -i",[40,1057,1058],{"class":56}," \"error\\|fail\"",[40,1060,1061],{"class":227},"    # filter for errors\n",[40,1063,1064,1066,1068,1070,1072],{"class":42,"line":99},[40,1065,214],{"class":46},[40,1067,310],{"class":235},[40,1069,239],{"class":235},[40,1071,242],{"class":56},[40,1073,1074],{"class":227},"             # same, from journald (persists if journald is persistent)\n",[355,1076,1078,1079,1081],{"id":1077},"common-dmesg-messages","Common ",[18,1080,1017],{}," Messages",[360,1083,1084,1094],{},[363,1085,1086],{},[366,1087,1088,1091],{},[369,1089,1090],{},"Message",[369,1092,1093],{},"Meaning",[376,1095,1096,1109,1119,1129,1139],{},[366,1097,1098,1103],{},[381,1099,1100],{},[18,1101,1102],{},"EXT4-fs error",[381,1104,1105,1106],{},"Filesystem corruption — check with ",[18,1107,1108],{},"fsck",[366,1110,1111,1116],{},[381,1112,1113],{},[18,1114,1115],{},"Out of memory: Killed process",[381,1117,1118],{},"OOM killer activated",[366,1120,1121,1126],{},[381,1122,1123],{},[18,1124,1125],{},"segfault at ...",[381,1127,1128],{},"A program crashed (bad memory access)",[366,1130,1131,1136],{},[381,1132,1133],{},[18,1134,1135],{},"I\u002FO error, dev sda",[381,1137,1138],{},"Disk failing or bad sector",[366,1140,1141,1146],{},[381,1142,1143],{},[18,1144,1145],{},"hung_task",[381,1147,1148],{},"A process is stuck waiting for I\u002FO",[23,1150,1152],{"id":1151},"real-time-monitoring","Real-Time Monitoring",[28,1154,1155],{"language":30},[32,1156,1158],{"className":34,"code":1157,"language":30,"meta":36,"style":36},"# Complex Implementation: multi-tool diagnostic dashboard\ntop -b -n 1 | head -20           # process snapshot\nvmstat 1 5                       # 5 samples, 1s apart (r=runnable, b=blocked, wa=I\u002FO wait)\niostat -x 1 3                    # disk I\u002FO, 3 samples (%util, await)\nsar -u 1 3                       # CPU usage over time\nmpstat -P ALL 1                  # per-CPU\niftop                             # per-connection bandwidth\nwatch -n 1 'ss -tlnp | grep :80' # watch port 80 every 1s\n",[18,1159,1160,1165,1188,1201,1217,1231,1247,1255],{"__ignoreMap":36},[40,1161,1162],{"class":42,"line":43},[40,1163,1164],{"class":227},"# Complex Implementation: multi-tool diagnostic dashboard\n",[40,1166,1167,1170,1172,1174,1177,1179,1182,1185],{"class":42,"line":50},[40,1168,1169],{"class":46},"top",[40,1171,236],{"class":235},[40,1173,723],{"class":235},[40,1175,1176],{"class":235}," 1",[40,1178,1049],{"class":154},[40,1180,1181],{"class":46}," head",[40,1183,1184],{"class":235}," -20",[40,1186,1187],{"class":227},"           # process snapshot\n",[40,1189,1190,1193,1195,1198],{"class":42,"line":69},[40,1191,1192],{"class":46},"vmstat",[40,1194,1176],{"class":235},[40,1196,1197],{"class":235}," 5",[40,1199,1200],{"class":227},"                       # 5 samples, 1s apart (r=runnable, b=blocked, wa=I\u002FO wait)\n",[40,1202,1203,1206,1209,1211,1214],{"class":42,"line":99},[40,1204,1205],{"class":46},"iostat",[40,1207,1208],{"class":235}," -x",[40,1210,1176],{"class":235},[40,1212,1213],{"class":235}," 3",[40,1215,1216],{"class":227},"                    # disk I\u002FO, 3 samples (%util, await)\n",[40,1218,1219,1222,1224,1226,1228],{"class":42,"line":105},[40,1220,1221],{"class":46},"sar",[40,1223,278],{"class":235},[40,1225,1176],{"class":235},[40,1227,1213],{"class":235},[40,1229,1230],{"class":227},"                       # CPU usage over time\n",[40,1232,1233,1236,1239,1242,1244],{"class":42,"line":116},[40,1234,1235],{"class":46},"mpstat",[40,1237,1238],{"class":235}," -P",[40,1240,1241],{"class":56}," ALL",[40,1243,1176],{"class":235},[40,1245,1246],{"class":227},"                  # per-CPU\n",[40,1248,1249,1252],{"class":42,"line":138},[40,1250,1251],{"class":46},"iftop",[40,1253,1254],{"class":227},"                             # per-connection bandwidth\n",[40,1256,1257,1260,1262,1264,1267],{"class":42,"line":143},[40,1258,1259],{"class":46},"watch",[40,1261,723],{"class":235},[40,1263,1176],{"class":235},[40,1265,1266],{"class":56}," 'ss -tlnp | grep :80'",[40,1268,1269],{"class":227}," # watch port 80 every 1s\n",[355,1271,1273,1275],{"id":1272},"sar-historical-performance",[18,1274,1221],{}," — Historical Performance",[28,1277,1278],{"language":30},[32,1279,1281],{"className":34,"code":1280,"language":30,"meta":36,"style":36},"# Complex Implementation: post-incident \"what was the load at 3 AM?\"\n# sar data is only collected if sysstat is ENABLED — install before you need it\nsudo apt install sysstat\nsudo systemctl enable --now sysstat   # start data collection\n\n# Query historical data:\nsar -u -f \u002Fvar\u002Flog\u002Fsysstat\u002Fsa10 -s 03:00:00 -e 03:30:00   # 10th of month, 3:00-3:30\nsar -r   # memory\nsar -d   # disk\nsar -n DEV  # network\n",[18,1282,1283,1288,1293,1306,1324,1328,1333,1359,1369,1378],{"__ignoreMap":36},[40,1284,1285],{"class":42,"line":43},[40,1286,1287],{"class":227},"# Complex Implementation: post-incident \"what was the load at 3 AM?\"\n",[40,1289,1290],{"class":42,"line":50},[40,1291,1292],{"class":227},"# sar data is only collected if sysstat is ENABLED — install before you need it\n",[40,1294,1295,1297,1300,1303],{"class":42,"line":69},[40,1296,484],{"class":46},[40,1298,1299],{"class":56}," apt",[40,1301,1302],{"class":56}," install",[40,1304,1305],{"class":56}," sysstat\n",[40,1307,1308,1310,1312,1315,1318,1321],{"class":42,"line":99},[40,1309,484],{"class":46},[40,1311,514],{"class":56},[40,1313,1314],{"class":56}," enable",[40,1316,1317],{"class":235}," --now",[40,1319,1320],{"class":56}," sysstat",[40,1322,1323],{"class":227},"   # start data collection\n",[40,1325,1326],{"class":42,"line":105},[40,1327,474],{"emptyLinePlaceholder":473},[40,1329,1330],{"class":42,"line":116},[40,1331,1332],{"class":227},"# Query historical data:\n",[40,1334,1335,1337,1339,1341,1344,1347,1350,1353,1356],{"class":42,"line":138},[40,1336,1221],{"class":46},[40,1338,278],{"class":235},[40,1340,284],{"class":235},[40,1342,1343],{"class":56}," \u002Fvar\u002Flog\u002Fsysstat\u002Fsa10",[40,1345,1346],{"class":235}," -s",[40,1348,1349],{"class":56}," 03:00:00",[40,1351,1352],{"class":235}," -e",[40,1354,1355],{"class":56}," 03:30:00",[40,1357,1358],{"class":227},"   # 10th of month, 3:00-3:30\n",[40,1360,1361,1363,1366],{"class":42,"line":143},[40,1362,1221],{"class":46},[40,1364,1365],{"class":235}," -r",[40,1367,1368],{"class":227},"   # memory\n",[40,1370,1371,1373,1375],{"class":42,"line":161},[40,1372,1221],{"class":46},[40,1374,898],{"class":235},[40,1376,1377],{"class":227},"   # disk\n",[40,1379,1380,1382,1384,1387],{"class":42,"line":180},[40,1381,1221],{"class":46},[40,1383,723],{"class":235},[40,1385,1386],{"class":56}," DEV",[40,1388,1389],{"class":227},"  # network\n",[355,1391,1393,1394,1396],{"id":1392},"edge-case-sar-shows-no-data-if-not-enabled","Edge Case: ",[18,1395,1221],{}," Shows \"No Data\" If Not Enabled",[28,1398,1399],{"language":30},[32,1400,1402],{"className":34,"code":1401,"language":30,"meta":36,"style":36},"# Installing sysstat isn't enough — you must enable data collection\n# NAIVE: apt install sysstat → sar shows \"No data\"\n# PRODUCTION:\nsudo systemctl enable --now sysstat\n# On Debian, also set ENABLED=\"true\" in \u002Fetc\u002Fdefault\u002Fsysstat\n",[18,1403,1404,1409,1414,1419,1431],{"__ignoreMap":36},[40,1405,1406],{"class":42,"line":43},[40,1407,1408],{"class":227},"# Installing sysstat isn't enough — you must enable data collection\n",[40,1410,1411],{"class":42,"line":50},[40,1412,1413],{"class":227},"# NAIVE: apt install sysstat → sar shows \"No data\"\n",[40,1415,1416],{"class":42,"line":69},[40,1417,1418],{"class":227},"# PRODUCTION:\n",[40,1420,1421,1423,1425,1427,1429],{"class":42,"line":99},[40,1422,484],{"class":46},[40,1424,514],{"class":56},[40,1426,1314],{"class":56},[40,1428,1317],{"class":235},[40,1430,1305],{"class":56},[40,1432,1433],{"class":42,"line":105},[40,1434,1435],{"class":227},"# On Debian, also set ENABLED=\"true\" in \u002Fetc\u002Fdefault\u002Fsysstat\n",[23,1437,1439,1442],{"id":1438},"atop-historical-per-process",[18,1440,1441],{},"atop"," — Historical Per-Process",[28,1444,1445],{"language":30},[32,1446,1448],{"className":34,"code":1447,"language":30,"meta":36,"style":36},"# Complex Implementation: \"what was running at time X?\"\n# atop logs per-process snapshots to \u002Fvar\u002Flog\u002Fatop\u002F\nsudo apt install atop\nsudo systemctl enable --now atop\n# Replay:\natop -r \u002Fvar\u002Flog\u002Fatop\u002Fatop_20260910   # press t (forward), T (backward) to navigate\n",[18,1449,1450,1455,1460,1471,1483,1488],{"__ignoreMap":36},[40,1451,1452],{"class":42,"line":43},[40,1453,1454],{"class":227},"# Complex Implementation: \"what was running at time X?\"\n",[40,1456,1457],{"class":42,"line":50},[40,1458,1459],{"class":227},"# atop logs per-process snapshots to \u002Fvar\u002Flog\u002Fatop\u002F\n",[40,1461,1462,1464,1466,1468],{"class":42,"line":69},[40,1463,484],{"class":46},[40,1465,1299],{"class":56},[40,1467,1302],{"class":56},[40,1469,1470],{"class":56}," atop\n",[40,1472,1473,1475,1477,1479,1481],{"class":42,"line":99},[40,1474,484],{"class":46},[40,1476,514],{"class":56},[40,1478,1314],{"class":56},[40,1480,1317],{"class":235},[40,1482,1470],{"class":56},[40,1484,1485],{"class":42,"line":105},[40,1486,1487],{"class":227},"# Replay:\n",[40,1489,1490,1492,1494,1497],{"class":42,"line":116},[40,1491,1441],{"class":46},[40,1493,1365],{"class":235},[40,1495,1496],{"class":56}," \u002Fvar\u002Flog\u002Fatop\u002Fatop_20260910",[40,1498,1499],{"class":227},"   # press t (forward), T (backward) to navigate\n",[23,1501,1503],{"id":1502},"writing-logs-from-your-services","Writing Logs from Your Services",[14,1505,1506],{},"systemd captures stdout\u002Fstderr automatically:",[28,1508,1510],{"language":1509},"ini",[32,1511,1514],{"className":1512,"code":1513,"language":1509,"meta":36,"style":36},"language-ini shiki shiki-themes github-light github-dark","[Service]\nStandardOutput=journal      # default\nStandardError=journal       # default\n",[18,1515,1516,1521,1532],{"__ignoreMap":36},[40,1517,1518],{"class":42,"line":43},[40,1519,1520],{"class":46},"[Service]\n",[40,1522,1523,1526,1529],{"class":42,"line":50},[40,1524,1525],{"class":154},"StandardOutput",[40,1527,1528],{"class":77},"=journal      ",[40,1530,1531],{"class":227},"# default\n",[40,1533,1534,1537,1540],{"class":42,"line":69},[40,1535,1536],{"class":154},"StandardError",[40,1538,1539],{"class":77},"=journal       ",[40,1541,1531],{"class":227},[28,1543,1544],{"language":30},[32,1545,1547],{"className":34,"code":1546,"language":30,"meta":36,"style":36},"# From shell:\nlogger -t myapp \"Starting up\"                    # to syslog (user facility)\nlogger -p local0.info \"Info message\"             # custom facility + priority\n",[18,1548,1549,1554,1570],{"__ignoreMap":36},[40,1550,1551],{"class":42,"line":43},[40,1552,1553],{"class":227},"# From shell:\n",[40,1555,1556,1559,1562,1564,1567],{"class":42,"line":50},[40,1557,1558],{"class":46},"logger",[40,1560,1561],{"class":235}," -t",[40,1563,350],{"class":56},[40,1565,1566],{"class":56}," \"Starting up\"",[40,1568,1569],{"class":227},"                    # to syslog (user facility)\n",[40,1571,1572,1574,1576,1579,1582],{"class":42,"line":69},[40,1573,1558],{"class":46},[40,1575,239],{"class":235},[40,1577,1578],{"class":56}," local0.info",[40,1580,1581],{"class":56}," \"Info message\"",[40,1583,1584],{"class":227},"             # custom facility + priority\n",[23,1586,1588],{"id":1587},"centralized-logging","Centralized Logging",[28,1590,1591],{"language":30},[32,1592,1594],{"className":34,"code":1593,"language":30,"meta":36,"style":36},"# rsyslog forward to remote (TCP, reliable):\n# \u002Fetc\u002Frsyslog.d\u002F60-forward.conf\n*.*  @@logserver.example.com:514    # TCP (@@ = TCP, @ = UDP)\n",[18,1595,1596,1601,1606],{"__ignoreMap":36},[40,1597,1598],{"class":42,"line":43},[40,1599,1600],{"class":227},"# rsyslog forward to remote (TCP, reliable):\n",[40,1602,1603],{"class":42,"line":50},[40,1604,1605],{"class":227},"# \u002Fetc\u002Frsyslog.d\u002F60-forward.conf\n",[40,1607,1608,1610,1613,1615,1618],{"class":42,"line":69},[40,1609,155],{"class":154},[40,1611,1612],{"class":77},".",[40,1614,155],{"class":154},[40,1616,1617],{"class":77},"  @@logserver.example.com:514    ",[40,1619,1620],{"class":227},"# TCP (@@ = TCP, @ = UDP)\n",[23,1622,1624],{"id":1623},"tips-tricks","💡 Tips & Tricks",[1626,1627,1628,1652,1668,1684,1708,1716,1736,1747],"ul",{},[1629,1630,1631,1635,1636,1639,1640,1643,1644,1647,1648,1651],"li",{},[1632,1633,1634],"strong",{},"Idiom",": use ",[18,1637,1638],{},"journalctl -u \u003Cservice> -f"," for live log tailing — the ",[18,1641,1642],{},"-f"," flag follows (like ",[18,1645,1646],{},"tail -f","). Pair with ",[18,1649,1650],{},"--since \"10 min ago\""," to skip old context.",[1629,1653,1654,1635,1656,1659,1660,1663,1664,1667],{},[1632,1655,1634],{},[18,1657,1658],{},"journalctl -b -p err"," to find boot errors fast — ",[18,1661,1662],{},"-b"," limits to current boot, ",[18,1665,1666],{},"-p err"," shows errors and worse.",[1629,1669,1670,1635,1672,1675,1676,1679,1680,1683],{},[1632,1671,1634],{},[18,1673,1674],{},"journalctl --list-boots"," + ",[18,1677,1678],{},"journalctl -b -1"," for previous-boot debugging — when a reboot happened, ",[18,1681,1682],{},"-b -1"," shows the last boot's logs.",[1629,1685,1686,1635,1688,1690,1691,1693,1694,1696,1697,1700,1701,1703,1704,1707],{},[1632,1687,1634],{},[18,1689,620],{}," and ",[18,1692,633],{}," for login forensics — ",[18,1695,620],{}," shows who logged in (from ",[18,1698,1699],{},"wtmp","); ",[18,1702,633],{}," shows failed attempts (from ",[18,1705,1706],{},"btmp",").",[1629,1709,1710,1635,1712,1715],{},[1632,1711,1634],{},[18,1713,1714],{},"logrotate -d"," to dry-run before relying on it — catches config errors without losing logs.",[1629,1717,1718,1720,1721,1724,1725,1729,1730,1732,1733,1735],{},[1632,1719,1634],{},": install ",[18,1722,1723],{},"sysstat"," and enable it ",[1726,1727,1728],"em",{},"before"," you need it — ",[18,1731,1221],{}," data is only collected if ",[18,1734,1723],{}," is running. If you install it after an incident, there's no historical data.",[1629,1737,1738,1635,1741,1743,1744,1612],{},[1632,1739,1740],{},"Debug",[18,1742,1441],{}," for historical \"what was running at time X?\" — replay with ",[18,1745,1746],{},"atop -r \u002Fvar\u002Flog\u002Fatop\u002Fatop_YYYYMMDD",[1629,1748,1749,1635,1751,1754,1755,1690,1758,1761,1762,1764],{},[1632,1750,1740],{},[18,1752,1753],{},"iostat -x 1"," and watch ",[18,1756,1757],{},"%util",[18,1759,1760],{},"await"," — ",[18,1763,1757],{}," near 100% = disk saturated. The most common cause of \"the app is slow\" is disk I\u002FO, not CPU.",[23,1766,1768],{"id":1767},"️-edge-cases-gotchas","⚠️ Edge Cases & Gotchas",[1626,1770,1771,1795,1806,1823,1843,1856,1872,1882],{},[1629,1772,1773,1776,1777,1780,1781,1784,1785,1787,1788,1791,1792,1612],{},[1632,1774,1775],{},"journald logs are lost on reboot by default",": if ",[18,1778,1779],{},"\u002Fvar\u002Flog\u002Fjournal\u002F"," doesn't exist, logs are in ",[18,1782,1783],{},"\u002Frun\u002Flog\u002Fjournal\u002F"," (RAM, volatile). Create ",[18,1786,1779],{}," or set ",[18,1789,1790],{},"Storage=persistent"," in ",[18,1793,1794],{},"journald.conf",[1629,1796,1797,1802,1803,1612],{},[1632,1798,1799,1801],{},[18,1800,1017],{}," is cleared on reboot",": the kernel ring buffer is in RAM. For persistent kernel logs, rely on ",[18,1804,1805],{},"journalctl -k",[1629,1807,1808,1816,1817,1675,1819,1822],{},[1632,1809,1810,1812,1813,1815],{},[18,1811,20],{}," with ",[18,1814,924],{}," loses lines",": lines written between the copy and truncate are lost. Prefer ",[18,1818,966],{},[18,1820,1821],{},"postrotate"," (app reopens the new file).",[1629,1824,1825,1831,1832,1834,1835,1791,1838,1840,1841,1612],{},[1632,1826,1827,1830],{},[18,1828,1829],{},"\u002Fvar\u002Flog"," can fill up",": if logs aren't rotated or journald isn't size-limited, ",[18,1833,1829],{}," fills up. Set ",[18,1836,1837],{},"SystemMaxUse=",[18,1839,1794],{}," and configure ",[18,1842,20],{},[1629,1844,1845,1852,1853,1855],{},[1632,1846,1847,1732,1849,1851],{},[18,1848,1221],{},[18,1850,1723],{}," is enabled",": installing isn't enough — you must enable data collection. If you forget, ",[18,1854,1221],{}," shows \"No data.\"",[1629,1857,1858,1861,1862,1864,1865,1867,1868,1871],{},[1632,1859,1860],{},"Auth log locations differ by distro",": Debian\u002FUbuntu uses ",[18,1863,575],{},"; RHEL\u002FRocky uses ",[18,1866,578],{},". Use ",[18,1869,1870],{},"journalctl -u ssh"," for portability.",[1629,1873,1874,1877,1878,1881],{},[1632,1875,1876],{},"Remote syslog can leak sensitive data",": syslog is plain text over the network (unless you use TLS). Use TLS (",[18,1879,1880],{},"rsyslog"," supports it) or a VPN for log forwarding.",[1629,1883,1884,1887],{},[1632,1885,1886],{},"High-frequency logging can overwhelm journald",": journald rate-limits by default. If a service logs thousands of lines\u002Fsec, you may see \"Suppressed X messages.\"",[23,1889,1891],{"id":1890},"spot-the-bug","🧠 Spot the Bug",[14,1893,1894,1895,1898],{},"An admin sets up a web app as a systemd service. Logs are visible via ",[18,1896,1897],{},"journalctl -u myapp",". They want logs in a file too, so they add to the unit:",[28,1900,1901],{"language":1509},[32,1902,1904],{"className":1512,"code":1903,"language":1509,"meta":36,"style":36},"[Service]\nExecStart=\u002Fusr\u002Fbin\u002Fnode \u002Fopt\u002Fmyapp\u002Fserver.js >> \u002Fvar\u002Flog\u002Fmyapp.log 2>&1\n",[18,1905,1906,1910],{"__ignoreMap":36},[40,1907,1908],{"class":42,"line":43},[40,1909,1520],{"class":46},[40,1911,1912,1915],{"class":42,"line":50},[40,1913,1914],{"class":154},"ExecStart",[40,1916,1917],{"class":77},"=\u002Fusr\u002Fbin\u002Fnode \u002Fopt\u002Fmyapp\u002Fserver.js >> \u002Fvar\u002Flog\u002Fmyapp.log 2>&1\n",[14,1919,1920,1921,1923,1924,1927],{},"After restart, ",[18,1922,1897],{}," shows nothing, and ",[18,1925,1926],{},"\u002Fvar\u002Flog\u002Fmyapp.log"," is owned by root. What went wrong?",[1929,1930,1931,1935,1938,1968,1973,2016,2023,2044],"details",{},[1932,1933,1934],"summary",{},"Answer",[14,1936,1937],{},"Two issues:",[1939,1940,1941,1955],"ol",{},[1629,1942,1943,1950,1951,1954],{},[1632,1944,1945,1946,1949],{},"The ",[18,1947,1948],{},">>"," redirect consumed stdout\u002Fstderr."," systemd captures stdout\u002Fstderr and sends them to journald. By redirecting ",[18,1952,1953],{},">> \u002Fvar\u002Flog\u002Fmyapp.log 2>&1",", nothing goes to stdout (it all goes to the file), so journald gets nothing.",[1629,1956,1957,1960,1961,1963,1964,1967],{},[1632,1958,1959],{},"The file is owned by root."," The ",[18,1962,1948],{}," redirect is performed by the shell systemd spawns (running as root or the specified user). If the app runs as ",[18,1965,1966],{},"appuser"," but the redirect created the file as root, the app can't write to it later.",[14,1969,1970],{},[1632,1971,1972],{},"Better approach — use systemd's built-in redirection:",[28,1974,1975],{"language":1509},[32,1976,1978],{"className":1512,"code":1977,"language":1509,"meta":36,"style":36},"[Service]\nExecStart=\u002Fusr\u002Fbin\u002Fnode \u002Fopt\u002Fmyapp\u002Fserver.js\nStandardOutput=journal\nStandardError=journal\n# Also append to a file:\nStandardOutput=append:\u002Fvar\u002Flog\u002Fmyapp.log\n",[18,1979,1980,1984,1991,1998,2004,2009],{"__ignoreMap":36},[40,1981,1982],{"class":42,"line":43},[40,1983,1520],{"class":46},[40,1985,1986,1988],{"class":42,"line":50},[40,1987,1914],{"class":154},[40,1989,1990],{"class":77},"=\u002Fusr\u002Fbin\u002Fnode \u002Fopt\u002Fmyapp\u002Fserver.js\n",[40,1992,1993,1995],{"class":42,"line":69},[40,1994,1525],{"class":154},[40,1996,1997],{"class":77},"=journal\n",[40,1999,2000,2002],{"class":42,"line":99},[40,2001,1536],{"class":154},[40,2003,1997],{"class":77},[40,2005,2006],{"class":42,"line":105},[40,2007,2008],{"class":227},"# Also append to a file:\n",[40,2010,2011,2013],{"class":42,"line":116},[40,2012,1525],{"class":154},[40,2014,2015],{"class":77},"=append:\u002Fvar\u002Flog\u002Fmyapp.log\n",[14,2017,2018,2019,2022],{},"Or use ",[18,2020,2021],{},"tee"," to split the stream:",[28,2024,2025],{"language":1509},[32,2026,2028],{"className":1512,"code":2027,"language":1509,"meta":36,"style":36},"[Service]\nExecStart=\u002Fbin\u002Fsh -c '\u002Fusr\u002Fbin\u002Fnode \u002Fopt\u002Fmyapp\u002Fserver.js 2>&1 | tee -a \u002Fvar\u002Flog\u002Fmyapp.log'\n",[18,2029,2030,2034],{"__ignoreMap":36},[40,2031,2032],{"class":42,"line":43},[40,2033,1520],{"class":46},[40,2035,2036,2038,2041],{"class":42,"line":50},[40,2037,1914],{"class":154},[40,2039,2040],{"class":77},"=\u002Fbin\u002Fsh -c ",[40,2042,2043],{"class":56},"'\u002Fusr\u002Fbin\u002Fnode \u002Fopt\u002Fmyapp\u002Fserver.js 2>&1 | tee -a \u002Fvar\u002Flog\u002Fmyapp.log'\n",[14,2045,2046,2047,2050,2051,2054],{},"But the cleanest production approach is to let journald handle logging and use ",[18,2048,2049],{},"journalctl -u myapp -f"," for live viewing, ",[18,2052,2053],{},"journalctl -u myapp --since today -o cat > \u002Fvar\u002Flog\u002Fmyapp.log"," for export.",[2056,2057,2058],"style",{},"html pre.shiki code .sIsaT, html code.shiki .sIsaT{--shiki-default:#6F42C1;--shiki-github-dark:#B392F0}html pre.shiki code .sJ6F3, html code.shiki .sJ6F3{--shiki-default:#032F62;--shiki-github-dark:#9ECBFF}html pre.shiki code .ssxIu, html code.shiki .ssxIu{--shiki-default:#24292E;--shiki-github-dark:#E1E4E8}html pre.shiki code .svdQ7, html code.shiki .svdQ7{--shiki-default:#D73A49;--shiki-github-dark:#F97583}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .github-dark .shiki span {color: var(--shiki-github-dark);background: var(--shiki-github-dark-bg);font-style: var(--shiki-github-dark-font-style);font-weight: var(--shiki-github-dark-font-weight);text-decoration: var(--shiki-github-dark-text-decoration);}html.github-dark .shiki span {color: var(--shiki-github-dark);background: var(--shiki-github-dark-bg);font-style: var(--shiki-github-dark-font-style);font-weight: var(--shiki-github-dark-font-weight);text-decoration: var(--shiki-github-dark-text-decoration);}html pre.shiki code .sdCPZ, html code.shiki .sdCPZ{--shiki-default:#6A737D;--shiki-github-dark:#6A737D}html pre.shiki code .snvgF, html code.shiki .snvgF{--shiki-default:#005CC5;--shiki-github-dark:#79B8FF}",{"title":36,"searchDepth":50,"depth":50,"links":2060},[2061,2062,2067,2072,2077,2082,2088,2090,2091,2092,2093,2094],{"id":25,"depth":50,"text":26},{"id":211,"depth":50,"text":2063,"children":2064},"journalctl — The Log Viewer",[2065,2066],{"id":357,"depth":69,"text":358},{"id":445,"depth":69,"text":446},{"id":528,"depth":50,"text":2068,"children":2069},"Traditional Syslog (\u002Fvar\u002Flog\u002F)",[2070,2071],{"id":636,"depth":69,"text":637},{"id":701,"depth":69,"text":702},{"id":765,"depth":50,"text":2073,"children":2074},"logrotate — Manage Log Size",[2075],{"id":920,"depth":69,"text":2076},"Caveat: copytruncate Loses Lines",{"id":1013,"depth":50,"text":2078,"children":2079},"Kernel Logs — dmesg",[2080],{"id":1077,"depth":69,"text":2081},"Common dmesg Messages",{"id":1151,"depth":50,"text":1152,"children":2083},[2084,2086],{"id":1272,"depth":69,"text":2085},"sar — Historical Performance",{"id":1392,"depth":69,"text":2087},"Edge Case: sar Shows \"No Data\" If Not Enabled",{"id":1438,"depth":50,"text":2089},"atop — Historical Per-Process",{"id":1502,"depth":50,"text":1503},{"id":1587,"depth":50,"text":1588},{"id":1623,"depth":50,"text":1624},{"id":1767,"depth":50,"text":1768},{"id":1890,"depth":50,"text":1891},"Logs are how you know what's happening on your system. This chapter covers systemd-journald (the modern default), traditional syslog, logrotate, kernel logs, and real-time monitoring tools — through the lens of production diagnostics and post-incident forensics.","md",{},"\u002Flinux\u002F12-logging-and-monitoring",{"title":5,"description":2095},"linux\u002F12-logging-and-monitoring","XEiJ9_Em7uHSXJm9gP75P_16iRaEmNbY19HXx_kdCMQ",1789924650063]