[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill:01M6FP46RDPSHKY8662D2ZPA4V":3,"skill-same:01M6FP46RDPSHKY8662D2ZPA4V":89},{"skillNo":4,"name":5,"nameZh":6,"description":7,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":15,"categories":16,"keywords":30,"publishedAt":50,"linkOnly":51,"markdownContent":52,"markdownBody":53,"frontMatter":54,"refName":35,"commitSha":63,"skillDirectory":64,"forksCount":65,"repositoryHtmlUrl":66,"repositoryHomepageUrl":6,"ownerHtmlUrl":67,"ownerBlogUrl":6,"ownerDescription":6,"updatedAt":68,"install":69,"resources":82},"01M6FP46RDPSHKY8662D2ZPA4V","emergency-rescue",null,"Recover from developer disasters. Use when someone force-pushed to main, leaked credentials in git, ran out of disk space, killed the wrong process, corrupted a database, broke a deploy, locked themselves out of SSH, lost commits after a bad rebase, or hit any other \"oh no\" moment that needs immediate, calm, step-by-step recovery.","kbarbel640-del\u002Fskills","kbarbel640-del","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F255315490?v=4","01M6CJPPQGCCESXMG3K7BBGQ5J","01M6CJPPX2PB2BCZERFVM2C2B3",1,"MIT","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fgitgoodordietrying\u002Femergency-rescue\u002FSKILL.md",[17,22,25],{"categoryNo":18,"categoryName":19,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000010","后端与接口","SCENARIO","使用场景",{"categoryNo":23,"categoryName":24,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000011","运维与部署",{"categoryNo":26,"categoryName":27,"taxonomyType":28,"taxonomyTypeDesc":29},"01JBSEEDCAT000000000000030","开发工程师","PROFESSION","适用职业",[31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49],"emergency rescue","rescue","disk","bad","main","use","process","rebase","Bash","commits","linux","credentials","ssh","recovery","developer","database","win32","git","deploy","2026-11-10 10:12:30",false,"---\nname: emergency-rescue\ndescription: Recover from developer disasters. Use when someone force-pushed to main, leaked credentials in git, ran out of disk space, killed the wrong process, corrupted a database, broke a deploy, locked themselves out of SSH, lost commits after a bad rebase, or hit any other \"oh no\" moment that needs immediate, calm, step-by-step recovery.\nmetadata: {\"clawdbot\":{\"emoji\":\"?\",\"requires\":{\"anyBins\":[\"git\",\"bash\"]},\"os\":[\"linux\",\"darwin\",\"win32\"]}}\n---\n\n# Emergency Rescue Kit\n\nStep-by-step recovery procedures for the worst moments in a developer's day. Every section follows the same pattern: **diagnose → fix → verify**. Commands are non-destructive by default. Destructive steps are flagged.\n\nWhen something has gone wrong, find your situation below and follow the steps in order.\n\n## When to Use\n\n- Someone force-pushed to main and overwrote history\n- Credentials were committed to a public repository\n- A rebase or reset destroyed commits you need\n- Disk is full and nothing works\n- A process is consuming all memory or won't die\n- A database migration failed halfway through\n- A deploy needs to be rolled back immediately\n- SSH access is locked out\n- SSL certificates expired in production\n- You don't know what went wrong, but it's broken\n\n---\n\n## Git Disasters\n\n### Force-pushed to main (or any shared branch)\n\nSomeone ran `git push --force` and overwrote remote history.\n\n```bash\n# DIAGNOSE: Check the reflog on any machine that had the old state\ngit reflog show origin\u002Fmain\n# Look for the last known-good commit hash\n\n# FIX (if you have the old state locally):\ngit push origin \u003Cgood-commit-hash>:main --force-with-lease\n# --force-with-lease is safer than --force: it fails if remote changed again\n\n# FIX (if you DON'T have the old state locally):\n# GitHub\u002FGitLab retain force-pushed refs temporarily\n\n# GitHub: check the \"push\" event in the audit log or use the API\ngh api repos\u002F{owner}\u002F{repo}\u002Fevents --jq '.[] | select(.type==\"PushEvent\") | .payload.before'\n\n# GitLab: check the reflog on the server (admin access needed)\n# Or restore from any CI runner or team member's local clone\n\n# VERIFY:\ngit log --oneline -10 origin\u002Fmain\n# Confirm the history looks correct\n```\n\n### Lost commits after rebase or reset --hard\n\nYou ran `git rebase` or `git reset --hard` and commits disappeared.\n\n```bash\n# DIAGNOSE: Your commits are NOT gone. Git keeps everything for 30+ days.\ngit reflog\n# Find the commit hash from BEFORE the rebase\u002Freset\n# Look for entries like \"rebase (start)\" or \"reset: moving to\"\n\n# FIX: Reset back to the pre-disaster state\ngit reset --hard \u003Ccommit-hash-before-disaster>\n\n# FIX (alternative): Cherry-pick specific lost commits\ngit cherry-pick \u003Clost-commit-hash>\n\n# FIX (if reflog is empty — rare, usually means different repo):\ngit fsck --lost-found\n# Look in .git\u002Flost-found\u002Fcommit\u002F for dangling commits\nls .git\u002Flost-found\u002Fcommit\u002F\ngit show \u003Chash>  # Inspect each one\n\n# VERIFY:\ngit log --oneline -10\n# Your commits should be back\n```\n\n### Committed to the wrong branch\n\nYou made commits on `main` that should be on a feature branch.\n\n```bash\n# DIAGNOSE: Check where you are and what you committed\ngit log --oneline -5\ngit branch\n\n# FIX: Create the feature branch at current position, then reset main\ngit branch feature-branch          # Create branch pointing at current commit\ngit reset --hard HEAD~\u003CN>          # Move main back N commits (⚠️ destructive)\ngit checkout feature-branch        # Switch to the new branch\n\n# FIX (safer alternative using cherry-pick):\ngit checkout -b feature-branch     # Create and switch to new branch\ngit checkout main\ngit reset --hard origin\u002Fmain       # Reset main to remote state\n# Your commits are safely on feature-branch\n\n# VERIFY:\ngit log --oneline main -5\ngit log --oneline feature-branch -5\n```\n\n### Merge gone wrong (conflicts everywhere, wrong result)\n\nA merge produced a bad result and you want to start over.\n\n```bash\n# FIX (merge not yet committed — still in conflict state):\ngit merge --abort\n\n# FIX (merge was committed but not pushed):\ngit reset --hard HEAD~1\n\n# FIX (merge was already pushed): Create a revert commit\ngit revert -m 1 \u003Cmerge-commit-hash>\n# -m 1 means \"keep the first parent\" (your branch before merge)\ngit push\n\n# VERIFY:\ngit log --oneline --graph -10\ngit diff HEAD~1  # Review what changed\n```\n\n### Corrupted git repository\n\nGit commands fail with \"bad object\", \"corrupt\", or \"broken link\" errors.\n\n```bash\n# DIAGNOSE: Check repository integrity\ngit fsck --full\n\n# FIX (if remote is intact — most common):\n# Save any uncommitted work first\ncp -r . ..\u002Frepo-backup\n\n# Re-clone and restore local work\ncd ..\ngit clone \u003Cremote-url> repo-fresh\ncp -r repo-backup\u002Fpath\u002Fto\u002Funcommitted\u002Ffiles repo-fresh\u002F\n\n# FIX (repair without re-cloning):\n# Remove corrupt objects and fetch them again\ngit fsck --full 2>&1 | grep \"corrupt\\|missing\" | awk '{print $NF}'\n# For each corrupt object:\nrm .git\u002Fobjects\u002F\u003Cfirst-2-chars>\u002F\u003Cremaining-hash>\ngit fetch origin  # Re-download from remote\n\n# VERIFY:\ngit fsck --full  # Should report no errors\ngit log --oneline -5\n```\n\n---\n\n## Credential Leaks\n\n### Secret committed to git (API key, password, token)\n\nA credential is in the git history. Every second counts — automated scrapers monitor public GitHub repos for leaked keys.\n\n```bash\n# STEP 1: REVOKE THE CREDENTIAL IMMEDIATELY\n# Do this FIRST, before cleaning git history.\n# The credential is already compromised the moment it was pushed publicly.\n\n# AWS keys:\naws iam delete-access-key --access-key-id AKIAXXXXXXXXXXXXXXXX --user-name \u003Cuser>\n# Then create a new key pair\n\n# GitHub tokens:\n# Go to github.com → Settings → Developer settings → Tokens → Revoke\n\n# Database passwords:\n# Change the password in the database immediately\n# ALTER USER myuser WITH PASSWORD 'new-secure-password';\n\n# Generic API tokens:\n# Revoke in the provider's dashboard, generate new ones\n\n# STEP 2: Remove from current branch\ngit rm --cached \u003Cfile-with-secret>    # If the whole file is secret\n# OR edit the file to remove the secret, then:\ngit add \u003Cfile>\n\n# STEP 3: Add to .gitignore\necho \".env\" >> .gitignore\necho \"credentials.json\" >> .gitignore\ngit add .gitignore\n\n# STEP 4: Remove from git history (⚠️ rewrites history)\n# Option A: git-filter-repo (recommended, install with pip install git-filter-repo)\ngit filter-repo --path \u003Cfile-with-secret> --invert-paths\n\n# Option B: BFG Repo Cleaner (faster for large repos)\n# Download from https:\u002F\u002Frtyley.github.io\u002Fbfg-repo-cleaner\u002F\njava -jar bfg.jar --delete-files \u003Cfilename> .\ngit reflog expire --expire=now --all\ngit gc --prune=now --aggressive\n\n# STEP 5: Force push the cleaned history\ngit push origin --force --all\ngit push origin --force --tags\n\n# STEP 6: Notify all collaborators to re-clone\n# Their local copies still have the secret in reflog\n\n# VERIFY:\ngit log --all -p -S '\u003Cthe-secret-string>' --diff-filter=A\n# Should return nothing\n```\n\n### .env file pushed to public repo\n\n```bash\n# STEP 1: Revoke ALL credentials in that .env file. All of them. Now.\n\n# STEP 2: Remove and ignore\ngit rm --cached .env\necho \".env\" >> .gitignore\ngit add .gitignore\ngit commit -m \"Remove .env from tracking\"\n\n# STEP 3: Remove from history (see credential removal above)\ngit filter-repo --path .env --invert-paths\n\n# STEP 4: Check what was exposed\n# List every variable that was in the .env:\ngit show HEAD~1:.env 2>\u002Fdev\u002Fnull || git log --all -p -- .env | head -50\n# Rotate every single value.\n\n# PREVENTION: Add a pre-commit hook\ncat > .git\u002Fhooks\u002Fpre-commit \u003C\u003C 'HOOK'\n#!\u002Fbin\u002Fbash\nif git diff --cached --name-only | grep -qE '\\.env$|\\.env\\.local$|credentials'; then\n    echo \"ERROR: Attempting to commit potential secrets file\"\n    echo \"Files: $(git diff --cached --name-only | grep -E '\\.env|credentials')\"\n    exit 1\nfi\nHOOK\nchmod +x .git\u002Fhooks\u002Fpre-commit\n```\n\n### Secret visible in CI\u002FCD logs\n\n```bash\n# STEP 1: Revoke the credential immediately\n\n# STEP 2: Delete the CI run\u002Flogs if possible\n# GitHub Actions:\ngh run delete \u003Crun-id>\n# Or: Settings → Actions → delete specific run\n\n# STEP 3: Fix the pipeline\n# Never echo secrets. Mask them:\n# GitHub Actions: echo \"::add-mask::$MY_SECRET\"\n# GitLab CI: variables are masked if marked as \"Masked\" in settings\n\n# STEP 4: Audit what was exposed\n# Check the log output for patterns like:\n# AKIAXXXXXXXXX (AWS)\n# ghp_XXXXXXXXX (GitHub)\n# sk-XXXXXXXXXXX (OpenAI\u002FStripe)\n# Any connection strings with passwords\n```\n\n---\n\n## Disk Full Emergencies\n\n### System or container disk is full\n\nNothing works — builds fail, logs can't write, services crash.\n\n```bash\n# DIAGNOSE: What's using space?\ndf -h                          # Which filesystem is full?\ndu -sh \u002F* 2>\u002Fdev\u002Fnull | sort -rh | head -20    # Biggest top-level dirs\ndu -sh \u002Fvar\u002Flog\u002F* | sort -rh | head -10        # Log bloat?\n\n# QUICK WINS (safe to run immediately):\n\n# 1. Docker cleanup (often the #1 cause)\ndocker system df               # See Docker disk usage\ndocker system prune -a -f      # Remove all unused images, containers, networks\ndocker volume prune -f          # Remove unused volumes\ndocker builder prune -a -f      # Remove build cache\n# ⚠️ This removes ALL unused Docker data. Safe if you can re-pull\u002Frebuild.\n\n# 2. Package manager caches\n# npm\nnpm cache clean --force\nrm -rf ~\u002F.npm\u002F_cacache\n\n# pip\npip cache purge\n\n# apt\nsudo apt-get clean\nsudo apt-get autoremove -y\n\n# brew\nbrew cleanup --prune=all\n\n# 3. Log rotation (immediate)\n# Truncate (not delete) large log files to free space instantly\nsudo truncate -s 0 \u002Fvar\u002Flog\u002Fsyslog\nsudo truncate -s 0 \u002Fvar\u002Flog\u002Fjournal\u002F*\u002F*.journal  # systemd journals\nfind \u002Fvar\u002Flog -name \"*.log\" -size +100M -exec truncate -s 0 {} \\;\n# Truncate preserves the file handle so services don't break\n\n# 4. Old build artifacts\nfind . -name \"node_modules\" -type d -prune -exec rm -rf {} + 2>\u002Fdev\u002Fnull\nfind . -name \".next\" -type d -exec rm -rf {} + 2>\u002Fdev\u002Fnull\nfind . -name \"dist\" -type d -exec rm -rf {} + 2>\u002Fdev\u002Fnull\nfind \u002Ftmp -type f -mtime +7 -delete 2>\u002Fdev\u002Fnull\n\n# 5. Find the actual culprit\nfind \u002F -xdev -type f -size +100M -exec ls -lh {} \\; 2>\u002Fdev\u002Fnull | sort -k5 -rh | head -20\n# Shows files over 100MB, sorted by size\n\n# VERIFY:\ndf -h  # Check free space increased\n```\n\n### Docker-specific disk full\n\n```bash\n# DIAGNOSE:\ndocker system df -v\n\n# Common culprits:\n# 1. Dangling images from builds\ndocker image prune -f\n\n# 2. Stopped containers accumulating\ndocker container prune -f\n\n# 3. Build cache (often the biggest)\ndocker builder prune -a -f\n\n# 4. Volumes from old containers\ndocker volume ls -qf dangling=true\ndocker volume prune -f\n\n# NUCLEAR OPTION (⚠️ removes EVERYTHING):\ndocker system prune -a --volumes -f\n# You will need to re-pull all images and recreate all volumes\n\n# VERIFY:\ndocker system df\ndf -h\n```\n\n---\n\n## Process Emergencies\n\n### Port already in use\n\n```bash\n# DIAGNOSE: What's using the port?\n# Linux:\nlsof -i :8080\nss -tlnp | grep 8080\n# macOS:\nlsof -i :8080\n# Windows:\nnetstat -ano | findstr :8080\n\n# FIX: Kill the process\nkill $(lsof -t -i :8080)           # Graceful\nkill -9 $(lsof -t -i :8080)       # Force (if graceful didn't work)\n\n# FIX (Windows):\n# Find PID from netstat output, then:\ntaskkill \u002FPID \u003Cpid> \u002FF\n\n# FIX (if it's a leftover Docker container):\ndocker ps | grep 8080\ndocker stop \u003Ccontainer-id>\n\n# VERIFY:\nlsof -i :8080  # Should return nothing\n```\n\n### Process won't die\n\n```bash\n# DIAGNOSE:\nps aux | grep \u003Cprocess-name>\n# Note the PID\n\n# ESCALATION LADDER:\nkill \u003Cpid>                # SIGTERM (graceful shutdown)\nsleep 5\nkill -9 \u003Cpid>             # SIGKILL (cannot be caught, immediate death)\n\n# If SIGKILL doesn't work, it's a zombie or kernel-stuck process:\n# Check if zombie:\nps aux | grep \u003Cpid>\n# State \"Z\" = zombie. The parent must reap it:\nkill -SIGCHLD $(ps -o ppid= -p \u003Cpid>)\n# Or kill the parent process\n\n# If truly stuck in kernel (state \"D\"):\n# Only a reboot will fix it. The process is stuck in an I\u002FO syscall.\n\n# MASS CLEANUP: Kill all processes matching a name\npkill -f \u003Cpattern>          # Graceful\npkill -9 -f \u003Cpattern>      # Force\n```\n\n### Out of memory (OOM killed)\n\n```bash\n# DIAGNOSE: Was your process OOM-killed?\ndmesg | grep -i \"oom\\|killed process\" | tail -20\njournalctl -k | grep -i \"oom\\|killed\" | tail -20\n\n# Check what's using memory right now:\nps aux --sort=-%mem | head -20        # Top memory consumers\nfree -h                                 # System memory overview\n\n# FIX: Free memory immediately\n# 1. Kill the biggest consumer (if safe to do so)\nkill $(ps aux --sort=-%mem | awk 'NR==2{print $2}')\n\n# 2. Drop filesystem caches (safe, no data loss)\nsync && echo 3 | sudo tee \u002Fproc\u002Fsys\u002Fvm\u002Fdrop_caches\n\n# 3. Disable swap thrashing (if swap is full)\nsudo swapoff -a && sudo swapon -a\n\n# PREVENT: Set memory limits\n# Docker:\ndocker run --memory=512m --memory-swap=1g myapp\n\n# Systemd service:\n# Add to [Service] section:\n# MemoryMax=512M\n# MemoryHigh=400M\n\n# Node.js:\nnode --max-old-space-size=512 app.js\n\n# VERIFY:\nfree -h\nps aux --sort=-%mem | head -5\n```\n\n---\n\n## Database Emergencies\n\n### Failed migration (partially applied)\n\n```bash\n# DIAGNOSE: What state is the database in?\n# Check which migrations have run:\n\n# Rails:\nrails db:migrate:status\n\n# Django:\npython manage.py showmigrations\n\n# Knex\u002FNode:\nnpx knex migrate:status\n\n# Prisma:\nnpx prisma migrate status\n\n# Raw SQL — check migration table:\n# PostgreSQL\u002FMySQL:\nSELECT * FROM schema_migrations ORDER BY version DESC LIMIT 10;\n# Or: SELECT * FROM _migrations ORDER BY id DESC LIMIT 10;\n\n# FIX: Roll back the failed migration\n# Most frameworks track migration state. Roll back to last good state:\n\n# Rails:\nrails db:rollback STEP=1\n\n# Django:\npython manage.py migrate \u003Capp_name> \u003Cprevious_migration_number>\n\n# Knex:\nnpx knex migrate:rollback\n\n# FIX (manual): If the framework is confused about state:\n# 1. Check what the migration actually did\n# 2. Manually undo partial changes\n# 3. Delete the migration record from the migrations table\n# 4. Fix the migration code\n# 5. Re-run\n\n# VERIFY:\n# Run the migration again and confirm it applies cleanly\n# Check the affected tables\u002Fcolumns exist correctly\n```\n\n### Accidentally dropped a table or database\n\n```bash\n# PostgreSQL:\n# If you have WAL archiving \u002F point-in-time recovery configured:\npg_restore -d mydb \u002Fbackups\u002Flatest.dump -t dropped_table\n\n# If no backup exists, check if the transaction is still open:\n# (Only works if you haven't committed yet)\n# Just run ROLLBACK; in your SQL session.\n\n# MySQL:\n# If binary logging is enabled:\nmysqlbinlog \u002Fvar\u002Flog\u002Fmysql\u002Fmysql-bin.000001 \\\n  --start-datetime=\"2026-02-03 10:00:00\" \\\n  --stop-datetime=\"2026-02-03 10:30:00\" > recovery.sql\n# Review recovery.sql, then apply\n\n# SQLite:\n# If the file still exists, it's fine — SQLite DROP TABLE is within the file\n# Restore from backup:\ncp \u002Fbackups\u002Fdb.sqlite3 .\u002Fdb.sqlite3\n\n# PREVENTION: Always run destructive SQL in a transaction\nBEGIN;\nDROP TABLE users;  -- oops\nROLLBACK;          -- saved\n```\n\n### Database locked \u002F deadlocked\n\n```bash\n# PostgreSQL:\n-- Find blocking queries\nSELECT pid, usename, state, query, wait_event_type, query_start\nFROM pg_stat_activity\nWHERE state != 'idle'\nORDER BY query_start;\n\n-- Find locks\nSELECT blocked_locks.pid AS blocked_pid,\n       blocking_locks.pid AS blocking_pid,\n       blocked_activity.query AS blocked_query,\n       blocking_activity.query AS blocking_query\nFROM pg_catalog.pg_locks blocked_locks\nJOIN pg_catalog.pg_stat_activity blocked_activity ON blocked_activity.pid = blocked_locks.pid\nJOIN pg_catalog.pg_locks blocking_locks ON blocking_locks.locktype = blocked_locks.locktype\nJOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid\nWHERE NOT blocked_locks.granted;\n\n-- Kill blocking query\nSELECT pg_terminate_backend(\u003Cblocking_pid>);\n\n# MySQL:\nSHOW PROCESSLIST;\nSHOW ENGINE INNODB STATUS\\G  -- Look for \"LATEST DETECTED DEADLOCK\"\nKILL \u003Cprocess_id>;\n\n# SQLite:\n# SQLite uses file-level locking. Common fix:\n# 1. Find and close all connections\n# 2. Check for .db-journal or .db-wal files (active transactions)\n# 3. If stuck: cp database.db database-fixed.db && mv database-fixed.db database.db\n# This forces SQLite to release the lock by creating a fresh file handle\n\n# VERIFY:\n# Run a simple query to confirm database is responsive\nSELECT 1;\n```\n\n### Connection pool exhausted\n\n```bash\n# DIAGNOSE:\n# Error messages like: \"too many connections\", \"connection pool exhausted\",\n# \"FATAL: remaining connection slots are reserved for superuser\"\n\n# PostgreSQL — check connection count:\nSELECT count(*), state FROM pg_stat_activity GROUP BY state;\nSELECT max_conn, used, max_conn - used AS available\nFROM (SELECT count(*) AS used FROM pg_stat_activity) t,\n     (SELECT setting::int AS max_conn FROM pg_settings WHERE name='max_connections') m;\n\n# FIX: Kill idle connections\n-- Terminate idle connections older than 5 minutes\nSELECT pg_terminate_backend(pid)\nFROM pg_stat_activity\nWHERE state = 'idle'\nAND query_start \u003C now() - interval '5 minutes';\n\n# FIX: Increase max connections (requires restart)\n# postgresql.conf:\n# max_connections = 200  (default is 100)\n\n# BETTER FIX: Use a connection pooler\n# PgBouncer or pgcat in front of PostgreSQL\n# Application-level: set pool size to match your needs\n# Node.js (pg): { max: 20 }\n# Python (SQLAlchemy): pool_size=20, max_overflow=10\n# Go (database\u002Fsql): db.SetMaxOpenConns(20)\n\n# VERIFY:\nSELECT count(*) FROM pg_stat_activity;\n# Should be well below max_connections\n```\n\n---\n\n## Deploy Emergencies\n\n### Quick rollback\n\n```bash\n# Git-based deploys:\ngit log --oneline -5 origin\u002Fmain\ngit revert HEAD                    # Create a revert commit\ngit push origin main               # Deploy the revert\n# Revert is safer than reset because it preserves history\n\n# Docker\u002Fcontainer deploys:\n# Roll back to previous image tag\ndocker pull myapp:previous-tag\ndocker stop myapp-current\ndocker run -d --name myapp myapp:previous-tag\n\n# Kubernetes:\nkubectl rollout undo deployment\u002Fmyapp\nkubectl rollout status deployment\u002Fmyapp    # Watch rollback progress\n\n# Heroku:\nheroku releases\nheroku rollback v\u003Cprevious-version>\n\n# AWS ECS:\naws ecs update-service --cluster mycluster --service myservice \\\n  --task-definition myapp:\u003Cprevious-revision>\n\n# VERIFY:\n# Hit the health check endpoint\ncurl -s -o \u002Fdev\u002Fnull -w \"%{http_code}\" https:\u002F\u002Fmyapp.example.com\u002Fhealth\n# Should return 200\n```\n\n### Container won't start\n\n```bash\n# DIAGNOSE: Why did it fail?\ndocker logs \u003Ccontainer-id> --tail 100\ndocker inspect \u003Ccontainer-id> | grep -A5 \"State\"\n\n# Common causes and fixes:\n\n# 1. \"exec format error\" — wrong platform (built for arm64, running on amd64)\ndocker build --platform linux\u002Famd64 -t myapp .\n\n# 2. \"permission denied\" — file not executable or wrong user\n# In Dockerfile:\nRUN chmod +x \u002Fapp\u002Fentrypoint.sh\n# Or: USER root before the command, then drop back\n\n# 3. \"port already allocated\" — another container or process on that port\ndocker ps -a | grep \u003Cport>\ndocker stop \u003Cconflicting-container>\n\n# 4. \"no such file or directory\" — entrypoint or CMD path is wrong\ndocker run -it --entrypoint sh myapp  # Get a shell to debug\nls -la \u002Fapp\u002F                           # Check what's actually there\n\n# 5. Healthcheck failing → container keeps restarting\ndocker inspect \u003Ccontainer-id> --format='{{json .State.Health}}'\n# Temporarily disable healthcheck to get logs:\ndocker run --no-healthcheck myapp\n\n# 6. Out of memory — container OOM killed\ndocker inspect \u003Ccontainer-id> --format='{{.State.OOMKilled}}'\n# If true: docker run --memory=1g myapp\n\n# VERIFY:\ndocker ps  # Container should show \"Up\" status\ndocker logs \u003Ccontainer-id> --tail 5  # No errors\n```\n\n### SSL certificate expired\n\n```bash\n# DIAGNOSE: Check certificate expiry\necho | openssl s_client -connect mysite.com:443 -servername mysite.com 2>\u002Fdev\u002Fnull | \\\n  openssl x509 -noout -dates\n# notAfter shows expiry date\n\n# FIX (Let's Encrypt — most common):\nsudo certbot renew --force-renewal\nsudo systemctl reload nginx   # or: sudo systemctl reload apache2\n\n# FIX (manual certificate):\n# 1. Get new certificate from your CA\n# 2. Replace files:\nsudo cp new-cert.pem \u002Fetc\u002Fssl\u002Fcerts\u002Fmysite.pem\nsudo cp new-key.pem \u002Fetc\u002Fssl\u002Fprivate\u002Fmysite.key\n# 3. Reload web server\nsudo nginx -t && sudo systemctl reload nginx\n\n# FIX (AWS ACM):\n# ACM auto-renews if DNS validation is configured.\n# If email validation: check the admin email for renewal link\n# If stuck: request a new certificate in ACM and update the load balancer\n\n# PREVENTION: Auto-renewal with monitoring\n# Cron job to check expiry and alert:\necho '0 9 * * 1 echo | openssl s_client -connect mysite.com:443 2>\u002Fdev\u002Fnull | openssl x509 -checkend 604800 -noout || echo \"CERT EXPIRES WITHIN 7 DAYS\" | mail -s \"SSL ALERT\" admin@example.com' | crontab -\n\n# VERIFY:\ncurl -sI https:\u002F\u002Fmysite.com | head -5\n# Should return HTTP\u002F2 200, not certificate errors\n```\n\n---\n\n## Access Emergencies\n\n### SSH locked out\n\n```bash\n# DIAGNOSE: Why can't you connect?\nssh -vvv user@host  # Verbose output shows where it fails\n\n# Common causes:\n\n# 1. Key not accepted — wrong key, permissions, or authorized_keys issue\nssh -i ~\u002F.ssh\u002Fspecific_key user@host  # Try explicit key\nchmod 600 ~\u002F.ssh\u002Fid_rsa               # Fix key permissions\nchmod 700 ~\u002F.ssh                       # Fix .ssh dir permissions\n\n# 2. \"Connection refused\" — sshd not running or firewall blocking\n# If you have console access (cloud provider's web console):\nsudo systemctl start sshd\nsudo systemctl status sshd\n\n# 3. Firewall blocking port 22\n# Cloud console:\nsudo ufw allow 22\u002Ftcp       # Ubuntu\nsudo firewall-cmd --add-service=ssh --permanent && sudo firewall-cmd --reload  # CentOS\n\n# 4. Changed SSH port and forgot\n# Try common alternate ports:\nssh -p 2222 user@host\nssh -p 22222 user@host\n# Or check from console: grep -i port \u002Fetc\u002Fssh\u002Fsshd_config\n\n# 5. IP changed \u002F DNS stale\nping hostname    # Verify IP resolution\nssh user@\u003Cdirect-ip>  # Try IP instead of hostname\n\n# 6. Locked out after too many attempts (fail2ban)\n# From console:\nsudo fail2ban-client set sshd unbanip \u003Cyour-ip>\n# Or wait for the ban to expire (usually 10 min)\n\n# CLOUD PROVIDER ESCAPE HATCHES:\n# AWS: EC2 → Instance → Connect → Session Manager (no SSH needed)\n# GCP: Compute → VM instances → SSH (browser-based)\n# Azure: VM → Serial console\n# DigitalOcean: Droplet → Access → Console\n\n# VERIFY:\nssh user@host echo \"connection works\"\n```\n\n### Lost sudo access\n\n```bash\n# If you have physical\u002Fconsole access:\n# 1. Boot into single-user\u002Frecovery mode\n#    - Reboot, hold Shift (GRUB), select \"recovery mode\"\n#    - Or add init=\u002Fbin\u002Fbash to kernel command line\n\n# 2. Remount filesystem read-write\nmount -o remount,rw \u002F\n\n# 3. Fix sudo access\nusermod -aG sudo \u003Cusername>    # Debian\u002FUbuntu\nusermod -aG wheel \u003Cusername>   # CentOS\u002FRHEL\n# Or edit directly:\nvisudo\n# Add: username ALL=(ALL:ALL) ALL\n\n# 4. Reboot normally\nreboot\n\n# If you have another sudo\u002Froot user:\nsu - other-admin\nsudo usermod -aG sudo \u003Clocked-user>\n\n# CLOUD: Use the provider's console or reset the instance\n# AWS: Create an AMI, launch new instance, mount old root volume, fix\n```\n\n---\n\n## Network Emergencies\n\n### Nothing connects (total network failure)\n\n```bash\n# DIAGNOSE: Isolate the layer\n# 1. Is the network interface up?\nip addr show         # or: ifconfig\nping 127.0.0.1       # Loopback works?\n\n# 2. Can you reach the gateway?\nip route | grep default\nping \u003Cgateway-ip>\n\n# 3. Can you reach the internet by IP?\nping 8.8.8.8          # Google DNS\nping 1.1.1.1          # Cloudflare DNS\n\n# 4. Is DNS working?\nnslookup google.com\ndig google.com\n\n# DECISION TREE:\n# ping 127.0.0.1 fails      → network stack broken, restart networking\n# ping gateway fails         → local network issue (cable, wifi, DHCP)\n# ping 8.8.8.8 fails        → routing\u002Ffirewall issue\n# ping 8.8.8.8 works but    → DNS issue\n#   nslookup fails\n\n# FIX: DNS broken\necho \"nameserver 8.8.8.8\" | sudo tee \u002Fetc\u002Fresolv.conf\n# Or: sudo systemd-resolve --flush-caches\n\n# FIX: Interface down\nsudo ip link set eth0 up\nsudo dhclient eth0        # Request new DHCP lease\n\n# FIX: Restart networking entirely\nsudo systemctl restart NetworkManager    # Desktop Linux\nsudo systemctl restart networking        # Server\nsudo systemctl restart systemd-networkd  # Systemd-based\n\n# Docker: Container can't reach the internet\ndocker run --rm alpine ping 8.8.8.8  # Test from container\n# If fails:\nsudo systemctl restart docker    # Often fixes Docker networking\n# Or: docker network prune\n```\n\n### DNS not propagating after change\n\n```bash\n# DIAGNOSE: Check what different DNS servers see\ndig @8.8.8.8 mysite.com        # Google\ndig @1.1.1.1 mysite.com        # Cloudflare\ndig @ns1.yourdns.com mysite.com # Authoritative nameserver\n\n# Check TTL (time remaining before caches expire):\ndig mysite.com | grep -i ttl\n\n# REALITY CHECK:\n# DNS propagation takes time. TTL controls this.\n# TTL 300 = 5 minutes. TTL 86400 = 24 hours.\n# You cannot speed this up. You can only wait.\n\n# FIX: If authoritative nameserver has wrong records\n# Update the record at your DNS provider (Cloudflare, Route53, etc.)\n# Then flush your local cache:\n# macOS:\nsudo dscacheutil -flushcache && sudo killall -HUP mDNSResponder\n# Linux:\nsudo systemd-resolve --flush-caches\n# Windows:\nipconfig \u002Fflushdns\n\n# WORKAROUND: While waiting for propagation\n# Add to \u002Fetc\u002Fhosts for immediate local effect:\necho \"93.184.216.34 mysite.com\" | sudo tee -a \u002Fetc\u002Fhosts\n# Remove this after propagation completes!\n\n# VERIFY:\ndig +short mysite.com  # Should show new IP\u002Frecord\n```\n\n---\n\n## File Emergencies\n\n### Accidentally deleted files (not in git)\n\n```bash\n# DIAGNOSE: Are the files recoverable?\n\n# If the process still has the file open:\nlsof | grep deleted\n# Then recover from \u002Fproc:\ncp \u002Fproc\u002F\u003Cpid>\u002Ffd\u002F\u003Cfd-number> \u002Fpath\u002Fto\u002Frestored-file\n\n# If recently deleted on ext4 (Linux):\n# Install extundelete or testdisk\nsudo extundelete \u002Fdev\u002Fsda1 --restore-file path\u002Fto\u002Ffile\n# Or use testdisk interactively for a better UI\n\n# macOS:\n# Check Trash first: ~\u002F.Trash\u002F\n# Time Machine: tmutil restore \u002Fpath\u002Fto\u002Ffile\n\n# PREVENTION:\n# Use trash-cli instead of rm:\n# npm install -g trash-cli\n# trash file.txt  (moves to trash instead of permanent delete)\n# Or alias: alias rm='echo \"Use trash instead\"; false'\n```\n\n### Wrong permissions applied recursively\n\n```bash\n# \"I ran chmod -R 777 \u002F\" or \"chmod -R 000 \u002Fimportant\u002Fdir\"\n\n# FIX: Common default permissions\n# For a web project:\nfind \u002Fpath -type d -exec chmod 755 {} \\;  # Directories: rwxr-xr-x\nfind \u002Fpath -type f -exec chmod 644 {} \\;  # Files: rw-r--r--\nfind \u002Fpath -name \"*.sh\" -exec chmod 755 {} \\;  # Scripts: executable\n\n# For SSH:\nchmod 700 ~\u002F.ssh\nchmod 600 ~\u002F.ssh\u002Fid_rsa\nchmod 644 ~\u002F.ssh\u002Fid_rsa.pub\nchmod 600 ~\u002F.ssh\u002Fauthorized_keys\nchmod 644 ~\u002F.ssh\u002Fconfig\n\n# For a system directory (⚠️ serious — may need rescue boot):\n# If \u002Fetc permissions are broken:\n# Boot from live USB, mount the drive, fix permissions\n# Reference: dpkg --verify (Debian) or rpm -Va (RHEL) to compare against package defaults\n\n# VERIFY:\nls -la \u002Fpath\u002Fto\u002Ffixed\u002Fdirectory\n```\n\n---\n\n## The Universal Diagnostic\n\nWhen you don't know what's wrong, run this sequence:\n\n```bash\n#!\u002Fbin\u002Fbash\n# emergency-diagnostic.sh — Quick system health check\n\necho \"=== DISK ===\"\ndf -h | grep -E '^\u002F|Filesystem'\n\necho -e \"\\n=== MEMORY ===\"\nfree -h\n\necho -e \"\\n=== CPU \u002F LOAD ===\"\nuptime\n\necho -e \"\\n=== TOP PROCESSES (by CPU) ===\"\nps aux --sort=-%cpu | head -6\n\necho -e \"\\n=== TOP PROCESSES (by MEM) ===\"\nps aux --sort=-%mem | head -6\n\necho -e \"\\n=== NETWORK ===\"\nping -c 1 -W 2 8.8.8.8 > \u002Fdev\u002Fnull 2>&1 && echo \"Internet: OK\" || echo \"Internet: UNREACHABLE\"\nping -c 1 -W 2 $(ip route | awk '\u002Fdefault\u002F{print $3}') > \u002Fdev\u002Fnull 2>&1 && echo \"Gateway: OK\" || echo \"Gateway: UNREACHABLE\"\n\necho -e \"\\n=== RECENT ERRORS ===\"\njournalctl -p err --since \"1 hour ago\" --no-pager | tail -20 2>\u002Fdev\u002Fnull || \\\n  dmesg | tail -20\n\necho -e \"\\n=== DOCKER (if running) ===\"\ndocker ps --format \"table {{.Names}}\\t{{.Status}}\\t{{.Ports}}\" 2>\u002Fdev\u002Fnull || echo \"Docker not running\"\ndocker system df 2>\u002Fdev\u002Fnull || true\n\necho -e \"\\n=== LISTENING PORTS ===\"\nss -tlnp 2>\u002Fdev\u002Fnull | head -15 || netstat -tlnp 2>\u002Fdev\u002Fnull | head -15\n\necho -e \"\\n=== FAILED SERVICES ===\"\nsystemctl --failed 2>\u002Fdev\u002Fnull || true\n```\n\nRun it, read the output, then jump to the relevant section above.\n\n## Tips\n\n- **Revoke credentials before cleaning git history.** The moment a secret is pushed publicly, automated scrapers have it within minutes. Cleaning the history is important but secondary to revocation.\n- **`git reflog` is your undo button.** It records every HEAD movement for 30+ days. Lost commits, bad rebases, accidental resets — the reflog has the recovery hash. Learn to read it before you need it.\n- **Truncate log files, don't delete them.** `truncate -s 0 file.log` frees disk space instantly while keeping the file handle open. Deleting a log file that a process has open won't free space until the process restarts.\n- **`--force-with-lease` instead of `--force`.** Always. It fails if someone else has pushed, preventing you from overwriting their work on top of your recovery.\n- **Every recovery operation should end with verification.** Run the diagnostic command, check the output, confirm the fix worked. Don't assume — confirm.\n- **Docker is the #1 disk space thief on developer machines.** `docker system prune -a` is almost always safe on development machines and can recover tens of gigabytes.\n- **Database emergencies: wrap destructive operations in transactions.** `BEGIN; DROP TABLE users; ROLLBACK;` costs nothing and saves everything. Make it muscle memory.\n- **When SSH is locked out, every cloud provider has a console escape hatch.** AWS Session Manager, GCP browser SSH, Azure Serial Console. Know where yours is *before* you need it.\n- **The order matters: diagnose → fix → verify.** Skipping diagnosis leads to wrong fixes. Skipping verification leads to false confidence. Follow the sequence every time.\n- **Keep this skill installed.** You won't need it most days. The day you do need it, you'll need it immediately.\n","# Emergency Rescue Kit\n\nStep-by-step recovery procedures for the worst moments in a developer's day. Every section follows the same pattern: **diagnose → fix → verify**. Commands are non-destructive by default. Destructive steps are flagged.\n\nWhen something has gone wrong, find your situation below and follow the steps in order.\n\n## When to Use\n\n- Someone force-pushed to main and overwrote history\n- Credentials were committed to a public repository\n- A rebase or reset destroyed commits you need\n- Disk is full and nothing works\n- A process is consuming all memory or won't die\n- A database migration failed halfway through\n- A deploy needs to be rolled back immediately\n- SSH access is locked out\n- SSL certificates expired in production\n- You don't know what went wrong, but it's broken\n\n---\n\n## Git Disasters\n\n### Force-pushed to main (or any shared branch)\n\nSomeone ran `git push --force` and overwrote remote history.\n\n```bash\n# DIAGNOSE: Check the reflog on any machine that had the old state\ngit reflog show origin\u002Fmain\n# Look for the last known-good commit hash\n\n# FIX (if you have the old state locally):\ngit push origin \u003Cgood-commit-hash>:main --force-with-lease\n# --force-with-lease is safer than --force: it fails if remote changed again\n\n# FIX (if you DON'T have the old state locally):\n# GitHub\u002FGitLab retain force-pushed refs temporarily\n\n# GitHub: check the \"push\" event in the audit log or use the API\ngh api repos\u002F{owner}\u002F{repo}\u002Fevents --jq '.[] | select(.type==\"PushEvent\") | .payload.before'\n\n# GitLab: check the reflog on the server (admin access needed)\n# Or restore from any CI runner or team member's local clone\n\n# VERIFY:\ngit log --oneline -10 origin\u002Fmain\n# Confirm the history looks correct\n```\n\n### Lost commits after rebase or reset --hard\n\nYou ran `git rebase` or `git reset --hard` and commits disappeared.\n\n```bash\n# DIAGNOSE: Your commits are NOT gone. Git keeps everything for 30+ days.\ngit reflog\n# Find the commit hash from BEFORE the rebase\u002Freset\n# Look for entries like \"rebase (start)\" or \"reset: moving to\"\n\n# FIX: Reset back to the pre-disaster state\ngit reset --hard \u003Ccommit-hash-before-disaster>\n\n# FIX (alternative): Cherry-pick specific lost commits\ngit cherry-pick \u003Clost-commit-hash>\n\n# FIX (if reflog is empty — rare, usually means different repo):\ngit fsck --lost-found\n# Look in .git\u002Flost-found\u002Fcommit\u002F for dangling commits\nls .git\u002Flost-found\u002Fcommit\u002F\ngit show \u003Chash>  # Inspect each one\n\n# VERIFY:\ngit log --oneline -10\n# Your commits should be back\n```\n\n### Committed to the wrong branch\n\nYou made commits on `main` that should be on a feature branch.\n\n```bash\n# DIAGNOSE: Check where you are and what you committed\ngit log --oneline -5\ngit branch\n\n# FIX: Create the feature branch at current position, then reset main\ngit branch feature-branch          # Create branch pointing at current commit\ngit reset --hard HEAD~\u003CN>          # Move main back N commits (⚠️ destructive)\ngit checkout feature-branch        # Switch to the new branch\n\n# FIX (safer alternative using cherry-pick):\ngit checkout -b feature-branch     # Create and switch to new branch\ngit checkout main\ngit reset --hard origin\u002Fmain       # Reset main to remote state\n# Your commits are safely on feature-branch\n\n# VERIFY:\ngit log --oneline main -5\ngit log --oneline feature-branch -5\n```\n\n### Merge gone wrong (conflicts everywhere, wrong result)\n\nA merge produced a bad result and you want to start over.\n\n```bash\n# FIX (merge not yet committed — still in conflict state):\ngit merge --abort\n\n# FIX (merge was committed but not pushed):\ngit reset --hard HEAD~1\n\n# FIX (merge was already pushed): Create a revert commit\ngit revert -m 1 \u003Cmerge-commit-hash>\n# -m 1 means \"keep the first parent\" (your branch before merge)\ngit push\n\n# VERIFY:\ngit log --oneline --graph -10\ngit diff HEAD~1  # Review what changed\n```\n\n### Corrupted git repository\n\nGit commands fail with \"bad object\", \"corrupt\", or \"broken link\" errors.\n\n```bash\n# DIAGNOSE: Check repository integrity\ngit fsck --full\n\n# FIX (if remote is intact — most common):\n# Save any uncommitted work first\ncp -r . ..\u002Frepo-backup\n\n# Re-clone and restore local work\ncd ..\ngit clone \u003Cremote-url> repo-fresh\ncp -r repo-backup\u002Fpath\u002Fto\u002Funcommitted\u002Ffiles repo-fresh\u002F\n\n# FIX (repair without re-cloning):\n# Remove corrupt objects and fetch them again\ngit fsck --full 2>&1 | grep \"corrupt\\|missing\" | awk '{print $NF}'\n# For each corrupt object:\nrm .git\u002Fobjects\u002F\u003Cfirst-2-chars>\u002F\u003Cremaining-hash>\ngit fetch origin  # Re-download from remote\n\n# VERIFY:\ngit fsck --full  # Should report no errors\ngit log --oneline -5\n```\n\n---\n\n## Credential Leaks\n\n### Secret committed to git (API key, password, token)\n\nA credential is in the git history. Every second counts — automated scrapers monitor public GitHub repos for leaked keys.\n\n```bash\n# STEP 1: REVOKE THE CREDENTIAL IMMEDIATELY\n# Do this FIRST, before cleaning git history.\n# The credential is already compromised the moment it was pushed publicly.\n\n# AWS keys:\naws iam delete-access-key --access-key-id AKIAXXXXXXXXXXXXXXXX --user-name \u003Cuser>\n# Then create a new key pair\n\n# GitHub tokens:\n# Go to github.com → Settings → Developer settings → Tokens → Revoke\n\n# Database passwords:\n# Change the password in the database immediately\n# ALTER USER myuser WITH PASSWORD 'new-secure-password';\n\n# Generic API tokens:\n# Revoke in the provider's dashboard, generate new ones\n\n# STEP 2: Remove from current branch\ngit rm --cached \u003Cfile-with-secret>    # If the whole file is secret\n# OR edit the file to remove the secret, then:\ngit add \u003Cfile>\n\n# STEP 3: Add to .gitignore\necho \".env\" >> .gitignore\necho \"credentials.json\" >> .gitignore\ngit add .gitignore\n\n# STEP 4: Remove from git history (⚠️ rewrites history)\n# Option A: git-filter-repo (recommended, install with pip install git-filter-repo)\ngit filter-repo --path \u003Cfile-with-secret> --invert-paths\n\n# Option B: BFG Repo Cleaner (faster for large repos)\n# Download from https:\u002F\u002Frtyley.github.io\u002Fbfg-repo-cleaner\u002F\njava -jar bfg.jar --delete-files \u003Cfilename> .\ngit reflog expire --expire=now --all\ngit gc --prune=now --aggressive\n\n# STEP 5: Force push the cleaned history\ngit push origin --force --all\ngit push origin --force --tags\n\n# STEP 6: Notify all collaborators to re-clone\n# Their local copies still have the secret in reflog\n\n# VERIFY:\ngit log --all -p -S '\u003Cthe-secret-string>' --diff-filter=A\n# Should return nothing\n```\n\n### .env file pushed to public repo\n\n```bash\n# STEP 1: Revoke ALL credentials in that .env file. All of them. Now.\n\n# STEP 2: Remove and ignore\ngit rm --cached .env\necho \".env\" >> .gitignore\ngit add .gitignore\ngit commit -m \"Remove .env from tracking\"\n\n# STEP 3: Remove from history (see credential removal above)\ngit filter-repo --path .env --invert-paths\n\n# STEP 4: Check what was exposed\n# List every variable that was in the .env:\ngit show HEAD~1:.env 2>\u002Fdev\u002Fnull || git log --all -p -- .env | head -50\n# Rotate every single value.\n\n# PREVENTION: Add a pre-commit hook\ncat > .git\u002Fhooks\u002Fpre-commit \u003C\u003C 'HOOK'\n#!\u002Fbin\u002Fbash\nif git diff --cached --name-only | grep -qE '\\.env$|\\.env\\.local$|credentials'; then\n    echo \"ERROR: Attempting to commit potential secrets file\"\n    echo \"Files: $(git diff --cached --name-only | grep -E '\\.env|credentials')\"\n    exit 1\nfi\nHOOK\nchmod +x .git\u002Fhooks\u002Fpre-commit\n```\n\n### Secret visible in CI\u002FCD logs\n\n```bash\n# STEP 1: Revoke the credential immediately\n\n# STEP 2: Delete the CI run\u002Flogs if possible\n# GitHub Actions:\ngh run delete \u003Crun-id>\n# Or: Settings → Actions → delete specific run\n\n# STEP 3: Fix the pipeline\n# Never echo secrets. Mask them:\n# GitHub Actions: echo \"::add-mask::$MY_SECRET\"\n# GitLab CI: variables are masked if marked as \"Masked\" in settings\n\n# STEP 4: Audit what was exposed\n# Check the log output for patterns like:\n# AKIAXXXXXXXXX (AWS)\n# ghp_XXXXXXXXX (GitHub)\n# sk-XXXXXXXXXXX (OpenAI\u002FStripe)\n# Any connection strings with passwords\n```\n\n---\n\n## Disk Full Emergencies\n\n### System or container disk is full\n\nNothing works — builds fail, logs can't write, services crash.\n\n```bash\n# DIAGNOSE: What's using space?\ndf -h                          # Which filesystem is full?\ndu -sh \u002F* 2>\u002Fdev\u002Fnull | sort -rh | head -20    # Biggest top-level dirs\ndu -sh \u002Fvar\u002Flog\u002F* | sort -rh | head -10        # Log bloat?\n\n# QUICK WINS (safe to run immediately):\n\n# 1. Docker cleanup (often the #1 cause)\ndocker system df               # See Docker disk usage\ndocker system prune -a -f      # Remove all unused images, containers, networks\ndocker volume prune -f          # Remove unused volumes\ndocker builder prune -a -f      # Remove build cache\n# ⚠️ This removes ALL unused Docker data. Safe if you can re-pull\u002Frebuild.\n\n# 2. Package manager caches\n# npm\nnpm cache clean --force\nrm -rf ~\u002F.npm\u002F_cacache\n\n# pip\npip cache purge\n\n# apt\nsudo apt-get clean\nsudo apt-get autoremove -y\n\n# brew\nbrew cleanup --prune=all\n\n# 3. Log rotation (immediate)\n# Truncate (not delete) large log files to free space instantly\nsudo truncate -s 0 \u002Fvar\u002Flog\u002Fsyslog\nsudo truncate -s 0 \u002Fvar\u002Flog\u002Fjournal\u002F*\u002F*.journal  # systemd journals\nfind \u002Fvar\u002Flog -name \"*.log\" -size +100M -exec truncate -s 0 {} \\;\n# Truncate preserves the file handle so services don't break\n\n# 4. Old build artifacts\nfind . -name \"node_modules\" -type d -prune -exec rm -rf {} + 2>\u002Fdev\u002Fnull\nfind . -name \".next\" -type d -exec rm -rf {} + 2>\u002Fdev\u002Fnull\nfind . -name \"dist\" -type d -exec rm -rf {} + 2>\u002Fdev\u002Fnull\nfind \u002Ftmp -type f -mtime +7 -delete 2>\u002Fdev\u002Fnull\n\n# 5. Find the actual culprit\nfind \u002F -xdev -type f -size +100M -exec ls -lh {} \\; 2>\u002Fdev\u002Fnull | sort -k5 -rh | head -20\n# Shows files over 100MB, sorted by size\n\n# VERIFY:\ndf -h  # Check free space increased\n```\n\n### Docker-specific disk full\n\n```bash\n# DIAGNOSE:\ndocker system df -v\n\n# Common culprits:\n# 1. Dangling images from builds\ndocker image prune -f\n\n# 2. Stopped containers accumulating\ndocker container prune -f\n\n# 3. Build cache (often the biggest)\ndocker builder prune -a -f\n\n# 4. Volumes from old containers\ndocker volume ls -qf dangling=true\ndocker volume prune -f\n\n# NUCLEAR OPTION (⚠️ removes EVERYTHING):\ndocker system prune -a --volumes -f\n# You will need to re-pull all images and recreate all volumes\n\n# VERIFY:\ndocker system df\ndf -h\n```\n\n---\n\n## Process Emergencies\n\n### Port already in use\n\n```bash\n# DIAGNOSE: What's using the port?\n# Linux:\nlsof -i :8080\nss -tlnp | grep 8080\n# macOS:\nlsof -i :8080\n# Windows:\nnetstat -ano | findstr :8080\n\n# FIX: Kill the process\nkill $(lsof -t -i :8080)           # Graceful\nkill -9 $(lsof -t -i :8080)       # Force (if graceful didn't work)\n\n# FIX (Windows):\n# Find PID from netstat output, then:\ntaskkill \u002FPID \u003Cpid> \u002FF\n\n# FIX (if it's a leftover Docker container):\ndocker ps | grep 8080\ndocker stop \u003Ccontainer-id>\n\n# VERIFY:\nlsof -i :8080  # Should return nothing\n```\n\n### Process won't die\n\n```bash\n# DIAGNOSE:\nps aux | grep \u003Cprocess-name>\n# Note the PID\n\n# ESCALATION LADDER:\nkill \u003Cpid>                # SIGTERM (graceful shutdown)\nsleep 5\nkill -9 \u003Cpid>             # SIGKILL (cannot be caught, immediate death)\n\n# If SIGKILL doesn't work, it's a zombie or kernel-stuck process:\n# Check if zombie:\nps aux | grep \u003Cpid>\n# State \"Z\" = zombie. The parent must reap it:\nkill -SIGCHLD $(ps -o ppid= -p \u003Cpid>)\n# Or kill the parent process\n\n# If truly stuck in kernel (state \"D\"):\n# Only a reboot will fix it. The process is stuck in an I\u002FO syscall.\n\n# MASS CLEANUP: Kill all processes matching a name\npkill -f \u003Cpattern>          # Graceful\npkill -9 -f \u003Cpattern>      # Force\n```\n\n### Out of memory (OOM killed)\n\n```bash\n# DIAGNOSE: Was your process OOM-killed?\ndmesg | grep -i \"oom\\|killed process\" | tail -20\njournalctl -k | grep -i \"oom\\|killed\" | tail -20\n\n# Check what's using memory right now:\nps aux --sort=-%mem | head -20        # Top memory consumers\nfree -h                                 # System memory overview\n\n# FIX: Free memory immediately\n# 1. Kill the biggest consumer (if safe to do so)\nkill $(ps aux --sort=-%mem | awk 'NR==2{print $2}')\n\n# 2. Drop filesystem caches (safe, no data loss)\nsync && echo 3 | sudo tee \u002Fproc\u002Fsys\u002Fvm\u002Fdrop_caches\n\n# 3. Disable swap thrashing (if swap is full)\nsudo swapoff -a && sudo swapon -a\n\n# PREVENT: Set memory limits\n# Docker:\ndocker run --memory=512m --memory-swap=1g myapp\n\n# Systemd service:\n# Add to [Service] section:\n# MemoryMax=512M\n# MemoryHigh=400M\n\n# Node.js:\nnode --max-old-space-size=512 app.js\n\n# VERIFY:\nfree -h\nps aux --sort=-%mem | head -5\n```\n\n---\n\n## Database Emergencies\n\n### Failed migration (partially applied)\n\n```bash\n# DIAGNOSE: What state is the database in?\n# Check which migrations have run:\n\n# Rails:\nrails db:migrate:status\n\n# Django:\npython manage.py showmigrations\n\n# Knex\u002FNode:\nnpx knex migrate:status\n\n# Prisma:\nnpx prisma migrate status\n\n# Raw SQL — check migration table:\n# PostgreSQL\u002FMySQL:\nSELECT * FROM schema_migrations ORDER BY version DESC LIMIT 10;\n# Or: SELECT * FROM _migrations ORDER BY id DESC LIMIT 10;\n\n# FIX: Roll back the failed migration\n# Most frameworks track migration state. Roll back to last good state:\n\n# Rails:\nrails db:rollback STEP=1\n\n# Django:\npython manage.py migrate \u003Capp_name> \u003Cprevious_migration_number>\n\n# Knex:\nnpx knex migrate:rollback\n\n# FIX (manual): If the framework is confused about state:\n# 1. Check what the migration actually did\n# 2. Manually undo partial changes\n# 3. Delete the migration record from the migrations table\n# 4. Fix the migration code\n# 5. Re-run\n\n# VERIFY:\n# Run the migration again and confirm it applies cleanly\n# Check the affected tables\u002Fcolumns exist correctly\n```\n\n### Accidentally dropped a table or database\n\n```bash\n# PostgreSQL:\n# If you have WAL archiving \u002F point-in-time recovery configured:\npg_restore -d mydb \u002Fbackups\u002Flatest.dump -t dropped_table\n\n# If no backup exists, check if the transaction is still open:\n# (Only works if you haven't committed yet)\n# Just run ROLLBACK; in your SQL session.\n\n# MySQL:\n# If binary logging is enabled:\nmysqlbinlog \u002Fvar\u002Flog\u002Fmysql\u002Fmysql-bin.000001 \\\n  --start-datetime=\"2026-02-03 10:00:00\" \\\n  --stop-datetime=\"2026-02-03 10:30:00\" > recovery.sql\n# Review recovery.sql, then apply\n\n# SQLite:\n# If the file still exists, it's fine — SQLite DROP TABLE is within the file\n# Restore from backup:\ncp \u002Fbackups\u002Fdb.sqlite3 .\u002Fdb.sqlite3\n\n# PREVENTION: Always run destructive SQL in a transaction\nBEGIN;\nDROP TABLE users;  -- oops\nROLLBACK;          -- saved\n```\n\n### Database locked \u002F deadlocked\n\n```bash\n# PostgreSQL:\n-- Find blocking queries\nSELECT pid, usename, state, query, wait_event_type, query_start\nFROM pg_stat_activity\nWHERE state != 'idle'\nORDER BY query_start;\n\n-- Find locks\nSELECT blocked_locks.pid AS blocked_pid,\n       blocking_locks.pid AS blocking_pid,\n       blocked_activity.query AS blocked_query,\n       blocking_activity.query AS blocking_query\nFROM pg_catalog.pg_locks blocked_locks\nJOIN pg_catalog.pg_stat_activity blocked_activity ON blocked_activity.pid = blocked_locks.pid\nJOIN pg_catalog.pg_locks blocking_locks ON blocking_locks.locktype = blocked_locks.locktype\nJOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid\nWHERE NOT blocked_locks.granted;\n\n-- Kill blocking query\nSELECT pg_terminate_backend(\u003Cblocking_pid>);\n\n# MySQL:\nSHOW PROCESSLIST;\nSHOW ENGINE INNODB STATUS\\G  -- Look for \"LATEST DETECTED DEADLOCK\"\nKILL \u003Cprocess_id>;\n\n# SQLite:\n# SQLite uses file-level locking. Common fix:\n# 1. Find and close all connections\n# 2. Check for .db-journal or .db-wal files (active transactions)\n# 3. If stuck: cp database.db database-fixed.db && mv database-fixed.db database.db\n# This forces SQLite to release the lock by creating a fresh file handle\n\n# VERIFY:\n# Run a simple query to confirm database is responsive\nSELECT 1;\n```\n\n### Connection pool exhausted\n\n```bash\n# DIAGNOSE:\n# Error messages like: \"too many connections\", \"connection pool exhausted\",\n# \"FATAL: remaining connection slots are reserved for superuser\"\n\n# PostgreSQL — check connection count:\nSELECT count(*), state FROM pg_stat_activity GROUP BY state;\nSELECT max_conn, used, max_conn - used AS available\nFROM (SELECT count(*) AS used FROM pg_stat_activity) t,\n     (SELECT setting::int AS max_conn FROM pg_settings WHERE name='max_connections') m;\n\n# FIX: Kill idle connections\n-- Terminate idle connections older than 5 minutes\nSELECT pg_terminate_backend(pid)\nFROM pg_stat_activity\nWHERE state = 'idle'\nAND query_start \u003C now() - interval '5 minutes';\n\n# FIX: Increase max connections (requires restart)\n# postgresql.conf:\n# max_connections = 200  (default is 100)\n\n# BETTER FIX: Use a connection pooler\n# PgBouncer or pgcat in front of PostgreSQL\n# Application-level: set pool size to match your needs\n# Node.js (pg): { max: 20 }\n# Python (SQLAlchemy): pool_size=20, max_overflow=10\n# Go (database\u002Fsql): db.SetMaxOpenConns(20)\n\n# VERIFY:\nSELECT count(*) FROM pg_stat_activity;\n# Should be well below max_connections\n```\n\n---\n\n## Deploy Emergencies\n\n### Quick rollback\n\n```bash\n# Git-based deploys:\ngit log --oneline -5 origin\u002Fmain\ngit revert HEAD                    # Create a revert commit\ngit push origin main               # Deploy the revert\n# Revert is safer than reset because it preserves history\n\n# Docker\u002Fcontainer deploys:\n# Roll back to previous image tag\ndocker pull myapp:previous-tag\ndocker stop myapp-current\ndocker run -d --name myapp myapp:previous-tag\n\n# Kubernetes:\nkubectl rollout undo deployment\u002Fmyapp\nkubectl rollout status deployment\u002Fmyapp    # Watch rollback progress\n\n# Heroku:\nheroku releases\nheroku rollback v\u003Cprevious-version>\n\n# AWS ECS:\naws ecs update-service --cluster mycluster --service myservice \\\n  --task-definition myapp:\u003Cprevious-revision>\n\n# VERIFY:\n# Hit the health check endpoint\ncurl -s -o \u002Fdev\u002Fnull -w \"%{http_code}\" https:\u002F\u002Fmyapp.example.com\u002Fhealth\n# Should return 200\n```\n\n### Container won't start\n\n```bash\n# DIAGNOSE: Why did it fail?\ndocker logs \u003Ccontainer-id> --tail 100\ndocker inspect \u003Ccontainer-id> | grep -A5 \"State\"\n\n# Common causes and fixes:\n\n# 1. \"exec format error\" — wrong platform (built for arm64, running on amd64)\ndocker build --platform linux\u002Famd64 -t myapp .\n\n# 2. \"permission denied\" — file not executable or wrong user\n# In Dockerfile:\nRUN chmod +x \u002Fapp\u002Fentrypoint.sh\n# Or: USER root before the command, then drop back\n\n# 3. \"port already allocated\" — another container or process on that port\ndocker ps -a | grep \u003Cport>\ndocker stop \u003Cconflicting-container>\n\n# 4. \"no such file or directory\" — entrypoint or CMD path is wrong\ndocker run -it --entrypoint sh myapp  # Get a shell to debug\nls -la \u002Fapp\u002F                           # Check what's actually there\n\n# 5. Healthcheck failing → container keeps restarting\ndocker inspect \u003Ccontainer-id> --format='{{json .State.Health}}'\n# Temporarily disable healthcheck to get logs:\ndocker run --no-healthcheck myapp\n\n# 6. Out of memory — container OOM killed\ndocker inspect \u003Ccontainer-id> --format='{{.State.OOMKilled}}'\n# If true: docker run --memory=1g myapp\n\n# VERIFY:\ndocker ps  # Container should show \"Up\" status\ndocker logs \u003Ccontainer-id> --tail 5  # No errors\n```\n\n### SSL certificate expired\n\n```bash\n# DIAGNOSE: Check certificate expiry\necho | openssl s_client -connect mysite.com:443 -servername mysite.com 2>\u002Fdev\u002Fnull | \\\n  openssl x509 -noout -dates\n# notAfter shows expiry date\n\n# FIX (Let's Encrypt — most common):\nsudo certbot renew --force-renewal\nsudo systemctl reload nginx   # or: sudo systemctl reload apache2\n\n# FIX (manual certificate):\n# 1. Get new certificate from your CA\n# 2. Replace files:\nsudo cp new-cert.pem \u002Fetc\u002Fssl\u002Fcerts\u002Fmysite.pem\nsudo cp new-key.pem \u002Fetc\u002Fssl\u002Fprivate\u002Fmysite.key\n# 3. Reload web server\nsudo nginx -t && sudo systemctl reload nginx\n\n# FIX (AWS ACM):\n# ACM auto-renews if DNS validation is configured.\n# If email validation: check the admin email for renewal link\n# If stuck: request a new certificate in ACM and update the load balancer\n\n# PREVENTION: Auto-renewal with monitoring\n# Cron job to check expiry and alert:\necho '0 9 * * 1 echo | openssl s_client -connect mysite.com:443 2>\u002Fdev\u002Fnull | openssl x509 -checkend 604800 -noout || echo \"CERT EXPIRES WITHIN 7 DAYS\" | mail -s \"SSL ALERT\" admin@example.com' | crontab -\n\n# VERIFY:\ncurl -sI https:\u002F\u002Fmysite.com | head -5\n# Should return HTTP\u002F2 200, not certificate errors\n```\n\n---\n\n## Access Emergencies\n\n### SSH locked out\n\n```bash\n# DIAGNOSE: Why can't you connect?\nssh -vvv user@host  # Verbose output shows where it fails\n\n# Common causes:\n\n# 1. Key not accepted — wrong key, permissions, or authorized_keys issue\nssh -i ~\u002F.ssh\u002Fspecific_key user@host  # Try explicit key\nchmod 600 ~\u002F.ssh\u002Fid_rsa               # Fix key permissions\nchmod 700 ~\u002F.ssh                       # Fix .ssh dir permissions\n\n# 2. \"Connection refused\" — sshd not running or firewall blocking\n# If you have console access (cloud provider's web console):\nsudo systemctl start sshd\nsudo systemctl status sshd\n\n# 3. Firewall blocking port 22\n# Cloud console:\nsudo ufw allow 22\u002Ftcp       # Ubuntu\nsudo firewall-cmd --add-service=ssh --permanent && sudo firewall-cmd --reload  # CentOS\n\n# 4. Changed SSH port and forgot\n# Try common alternate ports:\nssh -p 2222 user@host\nssh -p 22222 user@host\n# Or check from console: grep -i port \u002Fetc\u002Fssh\u002Fsshd_config\n\n# 5. IP changed \u002F DNS stale\nping hostname    # Verify IP resolution\nssh user@\u003Cdirect-ip>  # Try IP instead of hostname\n\n# 6. Locked out after too many attempts (fail2ban)\n# From console:\nsudo fail2ban-client set sshd unbanip \u003Cyour-ip>\n# Or wait for the ban to expire (usually 10 min)\n\n# CLOUD PROVIDER ESCAPE HATCHES:\n# AWS: EC2 → Instance → Connect → Session Manager (no SSH needed)\n# GCP: Compute → VM instances → SSH (browser-based)\n# Azure: VM → Serial console\n# DigitalOcean: Droplet → Access → Console\n\n# VERIFY:\nssh user@host echo \"connection works\"\n```\n\n### Lost sudo access\n\n```bash\n# If you have physical\u002Fconsole access:\n# 1. Boot into single-user\u002Frecovery mode\n#    - Reboot, hold Shift (GRUB), select \"recovery mode\"\n#    - Or add init=\u002Fbin\u002Fbash to kernel command line\n\n# 2. Remount filesystem read-write\nmount -o remount,rw \u002F\n\n# 3. Fix sudo access\nusermod -aG sudo \u003Cusername>    # Debian\u002FUbuntu\nusermod -aG wheel \u003Cusername>   # CentOS\u002FRHEL\n# Or edit directly:\nvisudo\n# Add: username ALL=(ALL:ALL) ALL\n\n# 4. Reboot normally\nreboot\n\n# If you have another sudo\u002Froot user:\nsu - other-admin\nsudo usermod -aG sudo \u003Clocked-user>\n\n# CLOUD: Use the provider's console or reset the instance\n# AWS: Create an AMI, launch new instance, mount old root volume, fix\n```\n\n---\n\n## Network Emergencies\n\n### Nothing connects (total network failure)\n\n```bash\n# DIAGNOSE: Isolate the layer\n# 1. Is the network interface up?\nip addr show         # or: ifconfig\nping 127.0.0.1       # Loopback works?\n\n# 2. Can you reach the gateway?\nip route | grep default\nping \u003Cgateway-ip>\n\n# 3. Can you reach the internet by IP?\nping 8.8.8.8          # Google DNS\nping 1.1.1.1          # Cloudflare DNS\n\n# 4. Is DNS working?\nnslookup google.com\ndig google.com\n\n# DECISION TREE:\n# ping 127.0.0.1 fails      → network stack broken, restart networking\n# ping gateway fails         → local network issue (cable, wifi, DHCP)\n# ping 8.8.8.8 fails        → routing\u002Ffirewall issue\n# ping 8.8.8.8 works but    → DNS issue\n#   nslookup fails\n\n# FIX: DNS broken\necho \"nameserver 8.8.8.8\" | sudo tee \u002Fetc\u002Fresolv.conf\n# Or: sudo systemd-resolve --flush-caches\n\n# FIX: Interface down\nsudo ip link set eth0 up\nsudo dhclient eth0        # Request new DHCP lease\n\n# FIX: Restart networking entirely\nsudo systemctl restart NetworkManager    # Desktop Linux\nsudo systemctl restart networking        # Server\nsudo systemctl restart systemd-networkd  # Systemd-based\n\n# Docker: Container can't reach the internet\ndocker run --rm alpine ping 8.8.8.8  # Test from container\n# If fails:\nsudo systemctl restart docker    # Often fixes Docker networking\n# Or: docker network prune\n```\n\n### DNS not propagating after change\n\n```bash\n# DIAGNOSE: Check what different DNS servers see\ndig @8.8.8.8 mysite.com        # Google\ndig @1.1.1.1 mysite.com        # Cloudflare\ndig @ns1.yourdns.com mysite.com # Authoritative nameserver\n\n# Check TTL (time remaining before caches expire):\ndig mysite.com | grep -i ttl\n\n# REALITY CHECK:\n# DNS propagation takes time. TTL controls this.\n# TTL 300 = 5 minutes. TTL 86400 = 24 hours.\n# You cannot speed this up. You can only wait.\n\n# FIX: If authoritative nameserver has wrong records\n# Update the record at your DNS provider (Cloudflare, Route53, etc.)\n# Then flush your local cache:\n# macOS:\nsudo dscacheutil -flushcache && sudo killall -HUP mDNSResponder\n# Linux:\nsudo systemd-resolve --flush-caches\n# Windows:\nipconfig \u002Fflushdns\n\n# WORKAROUND: While waiting for propagation\n# Add to \u002Fetc\u002Fhosts for immediate local effect:\necho \"93.184.216.34 mysite.com\" | sudo tee -a \u002Fetc\u002Fhosts\n# Remove this after propagation completes!\n\n# VERIFY:\ndig +short mysite.com  # Should show new IP\u002Frecord\n```\n\n---\n\n## File Emergencies\n\n### Accidentally deleted files (not in git)\n\n```bash\n# DIAGNOSE: Are the files recoverable?\n\n# If the process still has the file open:\nlsof | grep deleted\n# Then recover from \u002Fproc:\ncp \u002Fproc\u002F\u003Cpid>\u002Ffd\u002F\u003Cfd-number> \u002Fpath\u002Fto\u002Frestored-file\n\n# If recently deleted on ext4 (Linux):\n# Install extundelete or testdisk\nsudo extundelete \u002Fdev\u002Fsda1 --restore-file path\u002Fto\u002Ffile\n# Or use testdisk interactively for a better UI\n\n# macOS:\n# Check Trash first: ~\u002F.Trash\u002F\n# Time Machine: tmutil restore \u002Fpath\u002Fto\u002Ffile\n\n# PREVENTION:\n# Use trash-cli instead of rm:\n# npm install -g trash-cli\n# trash file.txt  (moves to trash instead of permanent delete)\n# Or alias: alias rm='echo \"Use trash instead\"; false'\n```\n\n### Wrong permissions applied recursively\n\n```bash\n# \"I ran chmod -R 777 \u002F\" or \"chmod -R 000 \u002Fimportant\u002Fdir\"\n\n# FIX: Common default permissions\n# For a web project:\nfind \u002Fpath -type d -exec chmod 755 {} \\;  # Directories: rwxr-xr-x\nfind \u002Fpath -type f -exec chmod 644 {} \\;  # Files: rw-r--r--\nfind \u002Fpath -name \"*.sh\" -exec chmod 755 {} \\;  # Scripts: executable\n\n# For SSH:\nchmod 700 ~\u002F.ssh\nchmod 600 ~\u002F.ssh\u002Fid_rsa\nchmod 644 ~\u002F.ssh\u002Fid_rsa.pub\nchmod 600 ~\u002F.ssh\u002Fauthorized_keys\nchmod 644 ~\u002F.ssh\u002Fconfig\n\n# For a system directory (⚠️ serious — may need rescue boot):\n# If \u002Fetc permissions are broken:\n# Boot from live USB, mount the drive, fix permissions\n# Reference: dpkg --verify (Debian) or rpm -Va (RHEL) to compare against package defaults\n\n# VERIFY:\nls -la \u002Fpath\u002Fto\u002Ffixed\u002Fdirectory\n```\n\n---\n\n## The Universal Diagnostic\n\nWhen you don't know what's wrong, run this sequence:\n\n```bash\n#!\u002Fbin\u002Fbash\n# emergency-diagnostic.sh — Quick system health check\n\necho \"=== DISK ===\"\ndf -h | grep -E '^\u002F|Filesystem'\n\necho -e \"\\n=== MEMORY ===\"\nfree -h\n\necho -e \"\\n=== CPU \u002F LOAD ===\"\nuptime\n\necho -e \"\\n=== TOP PROCESSES (by CPU) ===\"\nps aux --sort=-%cpu | head -6\n\necho -e \"\\n=== TOP PROCESSES (by MEM) ===\"\nps aux --sort=-%mem | head -6\n\necho -e \"\\n=== NETWORK ===\"\nping -c 1 -W 2 8.8.8.8 > \u002Fdev\u002Fnull 2>&1 && echo \"Internet: OK\" || echo \"Internet: UNREACHABLE\"\nping -c 1 -W 2 $(ip route | awk '\u002Fdefault\u002F{print $3}') > \u002Fdev\u002Fnull 2>&1 && echo \"Gateway: OK\" || echo \"Gateway: UNREACHABLE\"\n\necho -e \"\\n=== RECENT ERRORS ===\"\njournalctl -p err --since \"1 hour ago\" --no-pager | tail -20 2>\u002Fdev\u002Fnull || \\\n  dmesg | tail -20\n\necho -e \"\\n=== DOCKER (if running) ===\"\ndocker ps --format \"table {{.Names}}\\t{{.Status}}\\t{{.Ports}}\" 2>\u002Fdev\u002Fnull || echo \"Docker not running\"\ndocker system df 2>\u002Fdev\u002Fnull || true\n\necho -e \"\\n=== LISTENING PORTS ===\"\nss -tlnp 2>\u002Fdev\u002Fnull | head -15 || netstat -tlnp 2>\u002Fdev\u002Fnull | head -15\n\necho -e \"\\n=== FAILED SERVICES ===\"\nsystemctl --failed 2>\u002Fdev\u002Fnull || true\n```\n\nRun it, read the output, then jump to the relevant section above.\n\n## Tips\n\n- **Revoke credentials before cleaning git history.** The moment a secret is pushed publicly, automated scrapers have it within minutes. Cleaning the history is important but secondary to revocation.\n- **`git reflog` is your undo button.** It records every HEAD movement for 30+ days. Lost commits, bad rebases, accidental resets — the reflog has the recovery hash. Learn to read it before you need it.\n- **Truncate log files, don't delete them.** `truncate -s 0 file.log` frees disk space instantly while keeping the file handle open. Deleting a log file that a process has open won't free space until the process restarts.\n- **`--force-with-lease` instead of `--force`.** Always. It fails if someone else has pushed, preventing you from overwriting their work on top of your recovery.\n- **Every recovery operation should end with verification.** Run the diagnostic command, check the output, confirm the fix worked. Don't assume — confirm.\n- **Docker is the #1 disk space thief on developer machines.** `docker system prune -a` is almost always safe on development machines and can recover tens of gigabytes.\n- **Database emergencies: wrap destructive operations in transactions.** `BEGIN; DROP TABLE users; ROLLBACK;` costs nothing and saves everything. Make it muscle memory.\n- **When SSH is locked out, every cloud provider has a console escape hatch.** AWS Session Manager, GCP browser SSH, Azure Serial Console. Know where yours is *before* you need it.\n- **The order matters: diagnose → fix → verify.** Skipping diagnosis leads to wrong fixes. Skipping verification leads to false confidence. Follow the sequence every time.\n- **Keep this skill installed.** You won't need it most days. The day you do need it, you'll need it immediately.\n",{"name":5,"metadata":55,"description":7},{"clawdbot":56},{"os":57,"emoji":59,"requires":60},[41,58,47],"darwin","?",{"anyBins":61},[48,62],"bash","e7e336ac628c3e3bd771aff1aaa469c93778f94a","skills\u002Fgitgoodordietrying\u002Femergency-rescue",0,"https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del","2026-08-23 22:39:37",{"source":8,"skillName":5,"commands":70,"prompt":80,"zipAvailable":81},[71,74,77],{"tool":72,"command":73},"npx","npx skills add kbarbel640-del\u002Fskills --skill emergency-rescue",{"tool":75,"command":76},"bunx","bunx skills add kbarbel640-del\u002Fskills --skill emergency-rescue",{"tool":78,"command":79},"pnpm","pnpm dlx skills add kbarbel640-del\u002Fskills --skill emergency-rescue","安装技能，技能地址：https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fgitgoodordietrying\u002Femergency-rescue\u002FSKILL.md",true,[83],{"path":84,"resourceType":85,"resourceTypeDesc":86,"sizeBytes":87,"sourceHtmlUrl":88},"_meta.json","REFERENCE","参考文档",300,"https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fgitgoodordietrying\u002Femergency-rescue\u002F_meta.json",[90,119,145,166,172,188],{"skillNo":91,"name":92,"nameZh":6,"description":93,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":94,"categories":95,"keywords":99,"publishedAt":50},"01M6G8J55967D6281ACMKAKTEQ","crowd-prompting","A marketplace where AI agents improve prompts, system instructions, tool descriptions, and other text-based content with domain expertise from real-world operations — and earn tokens for valuable contributions.","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzzadrian\u002Fcrowd-prompting\u002FSKILL.md",[96],{"categoryNo":97,"categoryName":98,"taxonomyType":28,"taxonomyTypeDesc":29},"01JBSEEDCAT000000000000036","内容创作者",[100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118],"crowd prompting","prompting","api","with","tokens","domain","world","prompts","operations","instructions","text","tool","system","improve","productivity","marketplace","https","content","ai agents",{"skillNo":120,"name":121,"nameZh":6,"description":122,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":123,"categories":124,"keywords":128,"publishedAt":50},"01M6G8J0PKPAYVET6BWE8W4C26","dilemm-ai","Compete in DilemmAI, the prisoner's dilemma AI arena at dilemm.ai. Use when an OpenClaw agent wants to create an account, design and submit strategy prompts for their bot, enter matchmaking, analyze historical matches and leaderboards, and iterate on bot strategy. Requires headless browser automation (no API available yet).","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzyori\u002Fdilemm-ai\u002FSKILL.md",[125],{"categoryNo":126,"categoryName":127,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000023","Skill与智能体开发",[129,102,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,107,36],"dilemm ai","arena","analyze","browser","openclaw agent","create","automation","browser automation","bot","openclaw","design","strategy","headless","compete","submit","matchmaking",{"skillNo":146,"name":147,"nameZh":6,"description":148,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":149,"categories":150,"keywords":154,"publishedAt":50},"01M6G8HWAF3K9SW7NP3Z83CPJ6","arxiv-paper-reviews","Interact with arXiv Crawler API to fetch papers, read reviews, and submit comments. Use when working with arXiv papers, fetching paper lists by date\u002Fcategory\u002Finterest, viewing paper details with comments, or submitting paper reviews via API at http:\u002F\u002F150.158.152.82:8000.","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzxrys\u002Fweak-accept\u002FSKILL.md",[151],{"categoryNo":152,"categoryName":153,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000021","资料检索与研究",[155,156,157,158,159,160,161,103,162,143,36,163,102,164,165],"arxiv paper reviews","reviews","arxiv","paper","crawler","papers","comments","fetch","read","http","via",{"skillNo":167,"name":147,"nameZh":6,"description":148,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":168,"categories":169,"keywords":171,"publishedAt":50},"01M6G8HQJ7N7WMY6YV7D7SSJ16","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzxrys\u002Farxiv-paper-reviews\u002FSKILL.md",[170],{"categoryNo":152,"categoryName":153,"taxonomyType":20,"taxonomyTypeDesc":21},[158,157,156,155,165,160,161,103,159,143,36,162,163,102,164],{"skillNo":173,"name":174,"nameZh":6,"description":175,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":176,"categories":177,"keywords":178,"publishedAt":50},"01M6G8HM95Y64JNVFC2HN4E2BG","git-summary","Get a quick summary of the current Git repository including status, recent commits, branches, and contributors.","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzweack\u002Fgit-summary\u002FSKILL.md",[],[48,179,180,40,181,47,182,41,183,184,185,186,187],"summary","git summary","repository","status","contributors","true","quick","get","branches",{"skillNo":189,"name":190,"nameZh":6,"description":191,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":192,"categories":193,"keywords":198,"publishedAt":50},"01M6G8HGEGF05CNEJCGVYFSC4R","agent-constitution","Interact with AgentConstitution governance contracts on Base Sepolia. Check compliance, read rules, log actions, query governance state.","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fztsalexey\u002Fagent-constitution\u002FSKILL.md",[194,195],{"categoryNo":126,"categoryName":127,"taxonomyType":20,"taxonomyTypeDesc":21},{"categoryNo":196,"categoryName":197,"taxonomyType":28,"taxonomyTypeDesc":29},"01JBSEEDCAT000000000000037","律师与法务",[199,200,201,202,103,203,204,35,205,163,206,207,208,209,210,116,211,212,213],"agent constitution","constitution","contracts","tree","query","base","log","check","Rules","projects","compliance","actions","state","governance","github"]