[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill:01M6CBSM80EJVEMWS4M4N0ZRD9":3,"skill-same:01M6CBSM80EJVEMWS4M4N0ZRD9":64},{"skillNo":4,"name":5,"nameZh":6,"description":7,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":15,"categories":16,"keywords":22,"publishedAt":28,"linkOnly":29,"markdownContent":30,"markdownBody":31,"frontMatter":32,"refName":33,"commitSha":34,"skillDirectory":35,"forksCount":36,"repositoryHtmlUrl":37,"repositoryHomepageUrl":6,"ownerHtmlUrl":38,"ownerBlogUrl":6,"ownerDescription":6,"updatedAt":39,"install":40,"resources":53},"01M6CBSM80EJVEMWS4M4N0ZRD9","disaster-recovery-testing",null,"Execute comprehensive disaster recovery tests, validate recovery procedures, and document lessons learned from DR exercises.","ray-manaloto\u002Fmacos-development-environment","ray-manaloto","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F227192835?v=4","01M6CBR7J13ZA01X5YB1A1QRJP","01M6CBR7P0S3NS8ZV81JYNST14",1,"MIT","https:\u002F\u002Fgithub.com\u002Fray-manaloto\u002Fmacos-development-environment\u002Fblob\u002Fmain\u002F.agents\u002Fskills\u002Fdisaster-recovery-testing\u002FSKILL.md",[17],{"categoryNo":18,"categoryName":19,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000008","测试与质量","SCENARIO","使用场景",[23,24,25,26,27],"testing","recovery","disaster recovery testing","validate","execute","2026-11-10 10:12:30",false,"---\nname: disaster-recovery-testing\ndescription: Execute comprehensive disaster recovery tests, validate recovery procedures, and document lessons learned from DR exercises.\n---\n\n# Disaster Recovery Testing\n\n## Overview\n\nImplement systematic disaster recovery testing to validate recovery procedures, measure RTO\u002FRPO, identify gaps, and ensure team readiness for actual incidents.\n\n## When to Use\n\n- Annual DR exercises\n- Infrastructure changes\n- New service deployments\n- Compliance requirements\n- Team training\n- Recovery procedure validation\n- Cross-region failover testing\n\n## Implementation Examples\n\n### 1. **DR Test Plan and Execution**\n\n```yaml\n# dr-test-plan.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: dr-test-procedures\n  namespace: operations\ndata:\n  dr-test-plan.md: |\n    # Disaster Recovery Test Plan\n\n    ## Test Objectives\n    - Validate backup restoration procedures\n    - Verify failover mechanisms\n    - Test DNS failover\n    - Validate data integrity post-recovery\n    - Measure RTO and RPO\n    - Train incident response team\n\n    ## Pre-Test Checklist\n    - [ ] Notify stakeholders\n    - [ ] Schedule 4-6 hour window\n    - [ ] Disable alerting to prevent noise\n    - [ ] Backup production data\n    - [ ] Ensure DR environment is isolated\n    - [ ] Have rollback plan ready\n\n    ## Test Scope\n    - Primary database failover to standby\n    - Application failover to DR site\n    - DNS resolution update\n    - Load balancer health checks\n    - Data synchronization verification\n\n    ## Success Criteria\n    - RTO: \u003C 1 hour\n    - RPO: \u003C 15 minutes\n    - Zero data loss\n    - All services operational\n    - Alerts functional\n\n    ## Post-Test Activities\n    - Document timeline\n    - Identify gaps\n    - Update procedures\n    - Schedule post-mortem\n    - Update team documentation\n\n---\napiVersion: batch\u002Fv1\nkind: Job\nmetadata:\n  name: dr-test-executor\n  namespace: operations\nspec:\n  template:\n    spec:\n      serviceAccountName: dr-test-sa\n      containers:\n        - name: executor\n          image: alpine:latest\n          env:\n            - name: TEST_ID\n              value: \"dr-test-$(date +%s)\"\n            - name: BACKUP_BUCKET\n              value: \"s3:\u002F\u002Fmy-backups\"\n            - name: DR_NAMESPACE\n              value: \"dr-test\"\n          command:\n            - sh\n            - -c\n            - |\n              apk add --no-cache aws-cli kubectl jq postgresql-client mysql-client\n\n              echo \"Starting DR Test: $TEST_ID\"\n\n              # Step 1: Create test namespace\n              echo \"Creating isolated test environment...\"\n              kubectl create namespace \"$DR_NAMESPACE\" --dry-run=client -o yaml | kubectl apply -f -\n\n              # Step 2: Restore database from backup\n              echo \"Restoring database from latest backup...\"\n              LATEST_BACKUP=$(aws s3 ls \"$BACKUP_BUCKET\u002Fdatabases\u002F\" | \\\n                sort | tail -n 1 | awk '{print $4}')\n\n              aws s3 cp \"$BACKUP_BUCKET\u002Fdatabases\u002F$LATEST_BACKUP\" - | \\\n                gunzip | psql postgres:\u002F\u002Fuser:pass@dr-db:5432\u002Ftestdb\n\n              # Step 3: Deploy application to DR namespace\n              echo \"Deploying application to DR environment...\"\n              kubectl set image deployment\u002Fmyapp \\\n                myapp=myrepo\u002Fmyapp:production \\\n                -n \"$DR_NAMESPACE\"\n\n              # Step 4: Run health checks\n              echo \"Running health checks...\"\n              for i in {1..30}; do\n                if curl -sf http:\u002F\u002Fmyapp-dr\u002Fhealth > \u002Fdev\u002Fnull; then\n                  echo \"Health check passed\"\n                  break\n                fi\n                echo \"Waiting for service to be healthy... ($i\u002F30)\"\n                sleep 10\n              done\n\n              # Step 5: Run smoke tests\n              echo \"Running smoke tests...\"\n              kubectl exec -it deployment\u002Fmyapp -n \"$DR_NAMESPACE\" -- \\\n                npm run test:smoke || exit 1\n\n              # Step 6: Validate data integrity\n              echo \"Validating data integrity...\"\n              PROD_RECORD_COUNT=$(psql postgres:\u002F\u002Fuser:pass@prod-db:5432\u002Fmydb \\\n                -t -c \"SELECT COUNT(*) FROM users;\")\n              DR_RECORD_COUNT=$(psql postgres:\u002F\u002Fuser:pass@dr-db:5432\u002Ftestdb \\\n                -t -c \"SELECT COUNT(*) FROM users;\")\n\n              if [ \"$PROD_RECORD_COUNT\" -eq \"$DR_RECORD_COUNT\" ]; then\n                echo \"Data integrity verified\"\n              else\n                echo \"Data integrity check failed\"\n                exit 1\n              fi\n\n              # Step 7: Record metrics\n              echo \"Recording DR test metrics...\"\n              kubectl logs deployment\u002Fmyapp -n \"$DR_NAMESPACE\" | \\\n                grep \"startup_time\" | jq '.' > \u002Ftmp\u002Fdr-metrics-$TEST_ID.json\n\n              echo \"DR Test Complete: $TEST_ID\"\n\n          restartPolicy: Never\n\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: dr-test-sa\n  namespace: operations\n\n---\napiVersion: rbac.authorization.k8s.io\u002Fv1\nkind: ClusterRole\nmetadata:\n  name: dr-test\nrules:\n  - apiGroups: [\"\"]\n    resources: [\"namespaces\"]\n    verbs: [\"create\", \"get\", \"list\"]\n  - apiGroups: [\"apps\"]\n    resources: [\"deployments\"]\n    verbs: [\"create\", \"get\", \"list\", \"patch\", \"set\"]\n  - apiGroups: [\"\"]\n    resources: [\"pods\", \"pods\u002Flog\"]\n    verbs: [\"get\", \"list\"]\n  - apiGroups: [\"\"]\n    resources: [\"pods\u002Fexec\"]\n    verbs: [\"create\", \"get\"]\n\n---\napiVersion: rbac.authorization.k8s.io\u002Fv1\nkind: ClusterRoleBinding\nmetadata:\n  name: dr-test\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: dr-test\nsubjects:\n  - kind: ServiceAccount\n    name: dr-test-sa\n    namespace: operations\n```\n\n### 2. **DR Test Script**\n\n```bash\n#!\u002Fbin\u002Fbash\n# execute-dr-test.sh - Comprehensive DR test execution\n\nset -euo pipefail\n\nTEST_ID=\"dr-test-$(date +%Y%m%d-%H%M%S)\"\nLOG_FILE=\"\u002Ftmp\u002Fdr-test-${TEST_ID}.log\"\nMETRICS_FILE=\"\u002Ftmp\u002Fdr-metrics-${TEST_ID}.json\"\n\n# Logging\nexec 1> >(tee -a \"$LOG_FILE\")\nexec 2>&1\n\nlog_info() {\n    echo \"[INFO] $(date '+%Y-%m-%d %H:%M:%S') $1\"\n}\n\nlog_error() {\n    echo \"[ERROR] $(date '+%Y-%m-%d %H:%M:%S') $1\"\n}\n\n# Start time\nSTART_TIME=$(date +%s)\n\nlog_info \"Starting DR Test: $TEST_ID\"\n\n# Disable production monitoring\nlog_info \"Disabling production alerts...\"\naws sns set-topic-attributes \\\n    --topic-arn \"arn:aws:sns:us-east-1:123456789012:prod-alerts\" \\\n    --attribute-name DisplayName \\\n    --attribute-value \"DR Test - Alerts Disabled\"\n\n# Phase 1: Backup Validation\nlog_info \"Phase 1: Validating backups...\"\nif ! aws s3 ls s3:\u002F\u002Fmy-backups\u002Fdatabases\u002F | grep -q \"sql.gz\"; then\n    log_error \"No valid backups found\"\n    exit 1\nfi\n\n# Phase 2: Environment Setup\nlog_info \"Phase 2: Setting up DR environment...\"\nLATEST_BACKUP=$(aws s3 ls s3:\u002F\u002Fmy-backups\u002Fdatabases\u002F | \\\n    sort | tail -n 1 | awk '{print $4}')\n\nlog_info \"Using backup: $LATEST_BACKUP\"\naws s3 cp \"s3:\u002F\u002Fmy-backups\u002Fdatabases\u002F$LATEST_BACKUP\" - | gunzip > \u002Ftmp\u002Frestore.sql\n\n# Phase 3: Database Restoration\nlog_info \"Phase 3: Restoring database...\"\npsql -h dr-db.internal -U postgres -d postgres -f \u002Ftmp\u002Frestore.sql > \u002Fdev\u002Fnull 2>&1\n\n# Phase 4: Application Deployment\nlog_info \"Phase 4: Deploying application...\"\nkubectl create namespace dr-test --dry-run=client -o yaml | kubectl apply -f -\nkubectl apply -f dr-deployment.yaml -n dr-test\nkubectl rollout status deployment\u002Fmyapp -n dr-test --timeout=10m\n\n# Phase 5: Health Checks\nlog_info \"Phase 5: Running health checks...\"\nHEALTH_CHECK_START=$(date +%s)\n\nfor i in {1..60}; do\n    if curl -sf --max-time 5 http:\u002F\u002Fmyapp-dr.internal\u002Fhealth > \u002Fdev\u002Fnull 2>&1; then\n        HEALTH_CHECK_TIME=$(($(date +%s) - HEALTH_CHECK_START))\n        log_info \"Health check passed in ${HEALTH_CHECK_TIME}s\"\n        break\n    fi\n    if [ $i -eq 60 ]; then\n        log_error \"Health check timeout\"\n        exit 1\n    fi\n    sleep 10\ndone\n\n# Phase 6: Data Integrity\nlog_info \"Phase 6: Validating data integrity...\"\nPROD_HASH=$(psql -h prod-db.internal -U postgres -d mydb -t -c \\\n    \"SELECT md5(string_agg(CAST(id AS text), ',')) FROM users ORDER BY id;\")\nDR_HASH=$(psql -h dr-db.internal -U postgres -d mydb -t -c \\\n    \"SELECT md5(string_agg(CAST(id AS text), ',')) FROM users ORDER BY id;\")\n\nif [ \"$PROD_HASH\" = \"$DR_HASH\" ]; then\n    log_info \"Data integrity verified\"\nelse\n    log_error \"Data integrity check failed: $PROD_HASH != $DR_HASH\"\nfi\n\n# Phase 7: Smoke Tests\nlog_info \"Phase 7: Running smoke tests...\"\nkubectl exec -it deployment\u002Fmyapp -n dr-test -- npm run test:smoke || \\\n    log_error \"Smoke tests failed\"\n\n# Record metrics\nEND_TIME=$(date +%s)\nTOTAL_TIME=$((END_TIME - START_TIME))\nRTO=$TOTAL_TIME\nRPO=$(date -d \"$(aws s3api head-object --bucket my-backups --key databases\u002F$LATEST_BACKUP --query 'LastModified' --output text)\" +%s)\n\nlog_info \"DR Test Complete\"\nlog_info \"Total time: ${TOTAL_TIME}s\"\nlog_info \"RTO: ${RTO}s (target: 3600s)\"\nlog_info \"RPO: $(date -d @$RPO)\"\n\n# Generate report\ncat > \"$METRICS_FILE\" \u003C\u003CEOF\n{\n  \"test_id\": \"$TEST_ID\",\n  \"start_time\": $START_TIME,\n  \"end_time\": $END_TIME,\n  \"rto_seconds\": $RTO,\n  \"rpo_timestamp\": $RPO,\n  \"data_integrity\": \"PASS\",\n  \"health_check\": \"PASS\",\n  \"smoke_tests\": \"PASS\"\n}\nEOF\n\nlog_info \"Metrics saved to: $METRICS_FILE\"\n\n# Re-enable monitoring\nlog_info \"Re-enabling production alerts...\"\naws sns set-topic-attributes \\\n    --topic-arn \"arn:aws:sns:us-east-1:123456789012:prod-alerts\" \\\n    --attribute-name DisplayName \\\n    --attribute-value \"Production Alerts\"\n\nlog_info \"Test artifacts: $LOG_FILE, $METRICS_FILE\"\n```\n\n### 3. **DR Test Automation**\n\n```yaml\n# scheduled-dr-tests.yaml\napiVersion: batch\u002Fv1\nkind: CronJob\nmetadata:\n  name: quarterly-dr-test\n  namespace: operations\nspec:\n  # Run quarterly on first Monday of each quarter at 2 AM\n  schedule: \"0 2 2-8 1,4,7,10 MON\"\n  jobTemplate:\n    spec:\n      backoffLimit: 0\n      template:\n        spec:\n          serviceAccountName: dr-test-sa\n          containers:\n            - name: dr-test\n              image: myrepo\u002Fdr-test:latest\n              command:\n                - \u002Fusr\u002Flocal\u002Fbin\u002Fexecute-dr-test.sh\n              env:\n                - name: SLACK_WEBHOOK\n                  valueFrom:\n                    secretKeyRef:\n                      name: dr-notifications\n                      key: slack-webhook\n                - name: TEST_MODE\n                  value: \"full\"\n          restartPolicy: Never\n```\n\n## Best Practices\n\n### ✅ DO\n- Schedule regular DR tests\n- Document procedures in advance\n- Test in isolated environments\n- Measure actual RTO\u002FRPO\n- Involve all teams\n- Automate validation\n- Record findings\n- Update procedures based on results\n\n### ❌ DON'T\n- Skip DR testing\n- Test during business hours\n- Test against production\n- Ignore test failures\n- Neglect post-test analysis\n- Forget to re-enable monitoring\n- Use stale backup processes\n- Test only once a year\n\n## DR Test Levels\n\n- **Tabletop**: Documentation and discussion\n- **Simulation**: Controlled partial failover\n- **Full DR**: Complete system failover\n- **Continuous**: Ongoing shadow operations\n\n## Key Metrics\n\n- **RTO**: Recovery Time Objective\n- **RPO**: Recovery Point Objective\n- **MTPD**: Mean Time to Detect\n- **MTTR**: Mean Time to Recover\n\n## Resources\n\n- [AWS Disaster Recovery](https:\u002F\u002Faws.amazon.com\u002Fdisaster-recovery\u002F)\n- [Disaster Recovery Testing Best Practices](https:\u002F\u002Fwww.gartner.com\u002Fsmarterwithgartner\u002F5-best-practices-for-disaster-recovery-testing)\n- [NIST Disaster Recovery Planning](https:\u002F\u002Fcsrc.nist.gov\u002Fpublications\u002Fdetail\u002Fsp\u002F800-34\u002Frev-1\u002Ffinal)\n","# Disaster Recovery Testing\n\n## Overview\n\nImplement systematic disaster recovery testing to validate recovery procedures, measure RTO\u002FRPO, identify gaps, and ensure team readiness for actual incidents.\n\n## When to Use\n\n- Annual DR exercises\n- Infrastructure changes\n- New service deployments\n- Compliance requirements\n- Team training\n- Recovery procedure validation\n- Cross-region failover testing\n\n## Implementation Examples\n\n### 1. **DR Test Plan and Execution**\n\n```yaml\n# dr-test-plan.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: dr-test-procedures\n  namespace: operations\ndata:\n  dr-test-plan.md: |\n    # Disaster Recovery Test Plan\n\n    ## Test Objectives\n    - Validate backup restoration procedures\n    - Verify failover mechanisms\n    - Test DNS failover\n    - Validate data integrity post-recovery\n    - Measure RTO and RPO\n    - Train incident response team\n\n    ## Pre-Test Checklist\n    - [ ] Notify stakeholders\n    - [ ] Schedule 4-6 hour window\n    - [ ] Disable alerting to prevent noise\n    - [ ] Backup production data\n    - [ ] Ensure DR environment is isolated\n    - [ ] Have rollback plan ready\n\n    ## Test Scope\n    - Primary database failover to standby\n    - Application failover to DR site\n    - DNS resolution update\n    - Load balancer health checks\n    - Data synchronization verification\n\n    ## Success Criteria\n    - RTO: \u003C 1 hour\n    - RPO: \u003C 15 minutes\n    - Zero data loss\n    - All services operational\n    - Alerts functional\n\n    ## Post-Test Activities\n    - Document timeline\n    - Identify gaps\n    - Update procedures\n    - Schedule post-mortem\n    - Update team documentation\n\n---\napiVersion: batch\u002Fv1\nkind: Job\nmetadata:\n  name: dr-test-executor\n  namespace: operations\nspec:\n  template:\n    spec:\n      serviceAccountName: dr-test-sa\n      containers:\n        - name: executor\n          image: alpine:latest\n          env:\n            - name: TEST_ID\n              value: \"dr-test-$(date +%s)\"\n            - name: BACKUP_BUCKET\n              value: \"s3:\u002F\u002Fmy-backups\"\n            - name: DR_NAMESPACE\n              value: \"dr-test\"\n          command:\n            - sh\n            - -c\n            - |\n              apk add --no-cache aws-cli kubectl jq postgresql-client mysql-client\n\n              echo \"Starting DR Test: $TEST_ID\"\n\n              # Step 1: Create test namespace\n              echo \"Creating isolated test environment...\"\n              kubectl create namespace \"$DR_NAMESPACE\" --dry-run=client -o yaml | kubectl apply -f -\n\n              # Step 2: Restore database from backup\n              echo \"Restoring database from latest backup...\"\n              LATEST_BACKUP=$(aws s3 ls \"$BACKUP_BUCKET\u002Fdatabases\u002F\" | \\\n                sort | tail -n 1 | awk '{print $4}')\n\n              aws s3 cp \"$BACKUP_BUCKET\u002Fdatabases\u002F$LATEST_BACKUP\" - | \\\n                gunzip | psql postgres:\u002F\u002Fuser:pass@dr-db:5432\u002Ftestdb\n\n              # Step 3: Deploy application to DR namespace\n              echo \"Deploying application to DR environment...\"\n              kubectl set image deployment\u002Fmyapp \\\n                myapp=myrepo\u002Fmyapp:production \\\n                -n \"$DR_NAMESPACE\"\n\n              # Step 4: Run health checks\n              echo \"Running health checks...\"\n              for i in {1..30}; do\n                if curl -sf http:\u002F\u002Fmyapp-dr\u002Fhealth > \u002Fdev\u002Fnull; then\n                  echo \"Health check passed\"\n                  break\n                fi\n                echo \"Waiting for service to be healthy... ($i\u002F30)\"\n                sleep 10\n              done\n\n              # Step 5: Run smoke tests\n              echo \"Running smoke tests...\"\n              kubectl exec -it deployment\u002Fmyapp -n \"$DR_NAMESPACE\" -- \\\n                npm run test:smoke || exit 1\n\n              # Step 6: Validate data integrity\n              echo \"Validating data integrity...\"\n              PROD_RECORD_COUNT=$(psql postgres:\u002F\u002Fuser:pass@prod-db:5432\u002Fmydb \\\n                -t -c \"SELECT COUNT(*) FROM users;\")\n              DR_RECORD_COUNT=$(psql postgres:\u002F\u002Fuser:pass@dr-db:5432\u002Ftestdb \\\n                -t -c \"SELECT COUNT(*) FROM users;\")\n\n              if [ \"$PROD_RECORD_COUNT\" -eq \"$DR_RECORD_COUNT\" ]; then\n                echo \"Data integrity verified\"\n              else\n                echo \"Data integrity check failed\"\n                exit 1\n              fi\n\n              # Step 7: Record metrics\n              echo \"Recording DR test metrics...\"\n              kubectl logs deployment\u002Fmyapp -n \"$DR_NAMESPACE\" | \\\n                grep \"startup_time\" | jq '.' > \u002Ftmp\u002Fdr-metrics-$TEST_ID.json\n\n              echo \"DR Test Complete: $TEST_ID\"\n\n          restartPolicy: Never\n\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: dr-test-sa\n  namespace: operations\n\n---\napiVersion: rbac.authorization.k8s.io\u002Fv1\nkind: ClusterRole\nmetadata:\n  name: dr-test\nrules:\n  - apiGroups: [\"\"]\n    resources: [\"namespaces\"]\n    verbs: [\"create\", \"get\", \"list\"]\n  - apiGroups: [\"apps\"]\n    resources: [\"deployments\"]\n    verbs: [\"create\", \"get\", \"list\", \"patch\", \"set\"]\n  - apiGroups: [\"\"]\n    resources: [\"pods\", \"pods\u002Flog\"]\n    verbs: [\"get\", \"list\"]\n  - apiGroups: [\"\"]\n    resources: [\"pods\u002Fexec\"]\n    verbs: [\"create\", \"get\"]\n\n---\napiVersion: rbac.authorization.k8s.io\u002Fv1\nkind: ClusterRoleBinding\nmetadata:\n  name: dr-test\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: dr-test\nsubjects:\n  - kind: ServiceAccount\n    name: dr-test-sa\n    namespace: operations\n```\n\n### 2. **DR Test Script**\n\n```bash\n#!\u002Fbin\u002Fbash\n# execute-dr-test.sh - Comprehensive DR test execution\n\nset -euo pipefail\n\nTEST_ID=\"dr-test-$(date +%Y%m%d-%H%M%S)\"\nLOG_FILE=\"\u002Ftmp\u002Fdr-test-${TEST_ID}.log\"\nMETRICS_FILE=\"\u002Ftmp\u002Fdr-metrics-${TEST_ID}.json\"\n\n# Logging\nexec 1> >(tee -a \"$LOG_FILE\")\nexec 2>&1\n\nlog_info() {\n    echo \"[INFO] $(date '+%Y-%m-%d %H:%M:%S') $1\"\n}\n\nlog_error() {\n    echo \"[ERROR] $(date '+%Y-%m-%d %H:%M:%S') $1\"\n}\n\n# Start time\nSTART_TIME=$(date +%s)\n\nlog_info \"Starting DR Test: $TEST_ID\"\n\n# Disable production monitoring\nlog_info \"Disabling production alerts...\"\naws sns set-topic-attributes \\\n    --topic-arn \"arn:aws:sns:us-east-1:123456789012:prod-alerts\" \\\n    --attribute-name DisplayName \\\n    --attribute-value \"DR Test - Alerts Disabled\"\n\n# Phase 1: Backup Validation\nlog_info \"Phase 1: Validating backups...\"\nif ! aws s3 ls s3:\u002F\u002Fmy-backups\u002Fdatabases\u002F | grep -q \"sql.gz\"; then\n    log_error \"No valid backups found\"\n    exit 1\nfi\n\n# Phase 2: Environment Setup\nlog_info \"Phase 2: Setting up DR environment...\"\nLATEST_BACKUP=$(aws s3 ls s3:\u002F\u002Fmy-backups\u002Fdatabases\u002F | \\\n    sort | tail -n 1 | awk '{print $4}')\n\nlog_info \"Using backup: $LATEST_BACKUP\"\naws s3 cp \"s3:\u002F\u002Fmy-backups\u002Fdatabases\u002F$LATEST_BACKUP\" - | gunzip > \u002Ftmp\u002Frestore.sql\n\n# Phase 3: Database Restoration\nlog_info \"Phase 3: Restoring database...\"\npsql -h dr-db.internal -U postgres -d postgres -f \u002Ftmp\u002Frestore.sql > \u002Fdev\u002Fnull 2>&1\n\n# Phase 4: Application Deployment\nlog_info \"Phase 4: Deploying application...\"\nkubectl create namespace dr-test --dry-run=client -o yaml | kubectl apply -f -\nkubectl apply -f dr-deployment.yaml -n dr-test\nkubectl rollout status deployment\u002Fmyapp -n dr-test --timeout=10m\n\n# Phase 5: Health Checks\nlog_info \"Phase 5: Running health checks...\"\nHEALTH_CHECK_START=$(date +%s)\n\nfor i in {1..60}; do\n    if curl -sf --max-time 5 http:\u002F\u002Fmyapp-dr.internal\u002Fhealth > \u002Fdev\u002Fnull 2>&1; then\n        HEALTH_CHECK_TIME=$(($(date +%s) - HEALTH_CHECK_START))\n        log_info \"Health check passed in ${HEALTH_CHECK_TIME}s\"\n        break\n    fi\n    if [ $i -eq 60 ]; then\n        log_error \"Health check timeout\"\n        exit 1\n    fi\n    sleep 10\ndone\n\n# Phase 6: Data Integrity\nlog_info \"Phase 6: Validating data integrity...\"\nPROD_HASH=$(psql -h prod-db.internal -U postgres -d mydb -t -c \\\n    \"SELECT md5(string_agg(CAST(id AS text), ',')) FROM users ORDER BY id;\")\nDR_HASH=$(psql -h dr-db.internal -U postgres -d mydb -t -c \\\n    \"SELECT md5(string_agg(CAST(id AS text), ',')) FROM users ORDER BY id;\")\n\nif [ \"$PROD_HASH\" = \"$DR_HASH\" ]; then\n    log_info \"Data integrity verified\"\nelse\n    log_error \"Data integrity check failed: $PROD_HASH != $DR_HASH\"\nfi\n\n# Phase 7: Smoke Tests\nlog_info \"Phase 7: Running smoke tests...\"\nkubectl exec -it deployment\u002Fmyapp -n dr-test -- npm run test:smoke || \\\n    log_error \"Smoke tests failed\"\n\n# Record metrics\nEND_TIME=$(date +%s)\nTOTAL_TIME=$((END_TIME - START_TIME))\nRTO=$TOTAL_TIME\nRPO=$(date -d \"$(aws s3api head-object --bucket my-backups --key databases\u002F$LATEST_BACKUP --query 'LastModified' --output text)\" +%s)\n\nlog_info \"DR Test Complete\"\nlog_info \"Total time: ${TOTAL_TIME}s\"\nlog_info \"RTO: ${RTO}s (target: 3600s)\"\nlog_info \"RPO: $(date -d @$RPO)\"\n\n# Generate report\ncat > \"$METRICS_FILE\" \u003C\u003CEOF\n{\n  \"test_id\": \"$TEST_ID\",\n  \"start_time\": $START_TIME,\n  \"end_time\": $END_TIME,\n  \"rto_seconds\": $RTO,\n  \"rpo_timestamp\": $RPO,\n  \"data_integrity\": \"PASS\",\n  \"health_check\": \"PASS\",\n  \"smoke_tests\": \"PASS\"\n}\nEOF\n\nlog_info \"Metrics saved to: $METRICS_FILE\"\n\n# Re-enable monitoring\nlog_info \"Re-enabling production alerts...\"\naws sns set-topic-attributes \\\n    --topic-arn \"arn:aws:sns:us-east-1:123456789012:prod-alerts\" \\\n    --attribute-name DisplayName \\\n    --attribute-value \"Production Alerts\"\n\nlog_info \"Test artifacts: $LOG_FILE, $METRICS_FILE\"\n```\n\n### 3. **DR Test Automation**\n\n```yaml\n# scheduled-dr-tests.yaml\napiVersion: batch\u002Fv1\nkind: CronJob\nmetadata:\n  name: quarterly-dr-test\n  namespace: operations\nspec:\n  # Run quarterly on first Monday of each quarter at 2 AM\n  schedule: \"0 2 2-8 1,4,7,10 MON\"\n  jobTemplate:\n    spec:\n      backoffLimit: 0\n      template:\n        spec:\n          serviceAccountName: dr-test-sa\n          containers:\n            - name: dr-test\n              image: myrepo\u002Fdr-test:latest\n              command:\n                - \u002Fusr\u002Flocal\u002Fbin\u002Fexecute-dr-test.sh\n              env:\n                - name: SLACK_WEBHOOK\n                  valueFrom:\n                    secretKeyRef:\n                      name: dr-notifications\n                      key: slack-webhook\n                - name: TEST_MODE\n                  value: \"full\"\n          restartPolicy: Never\n```\n\n## Best Practices\n\n### ✅ DO\n- Schedule regular DR tests\n- Document procedures in advance\n- Test in isolated environments\n- Measure actual RTO\u002FRPO\n- Involve all teams\n- Automate validation\n- Record findings\n- Update procedures based on results\n\n### ❌ DON'T\n- Skip DR testing\n- Test during business hours\n- Test against production\n- Ignore test failures\n- Neglect post-test analysis\n- Forget to re-enable monitoring\n- Use stale backup processes\n- Test only once a year\n\n## DR Test Levels\n\n- **Tabletop**: Documentation and discussion\n- **Simulation**: Controlled partial failover\n- **Full DR**: Complete system failover\n- **Continuous**: Ongoing shadow operations\n\n## Key Metrics\n\n- **RTO**: Recovery Time Objective\n- **RPO**: Recovery Point Objective\n- **MTPD**: Mean Time to Detect\n- **MTTR**: Mean Time to Recover\n\n## Resources\n\n- [AWS Disaster Recovery](https:\u002F\u002Faws.amazon.com\u002Fdisaster-recovery\u002F)\n- [Disaster Recovery Testing Best Practices](https:\u002F\u002Fwww.gartner.com\u002Fsmarterwithgartner\u002F5-best-practices-for-disaster-recovery-testing)\n- [NIST Disaster Recovery Planning](https:\u002F\u002Fcsrc.nist.gov\u002Fpublications\u002Fdetail\u002Fsp\u002F800-34\u002Frev-1\u002Ffinal)\n",{"name":5,"description":7},"main","716b17df3ef6719d88aeac180015008b7ddfff6f",".agents\u002Fskills\u002Fdisaster-recovery-testing",0,"https:\u002F\u002Fgithub.com\u002Fray-manaloto\u002Fmacos-development-environment","https:\u002F\u002Fgithub.com\u002Fray-manaloto","2026-08-03 16:55:50",{"source":8,"skillName":5,"commands":41,"prompt":51,"zipAvailable":52},[42,45,48],{"tool":43,"command":44},"npx","npx skills add ray-manaloto\u002Fmacos-development-environment --skill disaster-recovery-testing",{"tool":46,"command":47},"bunx","bunx skills add ray-manaloto\u002Fmacos-development-environment --skill disaster-recovery-testing",{"tool":49,"command":50},"pnpm","pnpm dlx skills add ray-manaloto\u002Fmacos-development-environment --skill disaster-recovery-testing","安装技能，技能地址：https:\u002F\u002Fgithub.com\u002Fray-manaloto\u002Fmacos-development-environment\u002Fblob\u002Fmain\u002F.agents\u002Fskills\u002Fdisaster-recovery-testing\u002FSKILL.md",true,[54,60],{"path":55,"resourceType":56,"resourceTypeDesc":57,"sizeBytes":58,"sourceHtmlUrl":59},"SOURCE.md","REFERENCE","参考文档",182,"https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fdefi\u002Fcow-swap-mev-protection\u002FSOURCE.md",{"path":61,"resourceType":56,"resourceTypeDesc":57,"sizeBytes":62,"sourceHtmlUrl":63},"TRUST.auto.yaml",2217,"https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fdefi\u002Fcow-swap-mev-protection\u002FTRUST.auto.yaml",[65,86,104,123,143,157],{"skillNo":66,"name":67,"nameZh":6,"description":68,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":69,"categories":70,"keywords":74,"publishedAt":28},"01M6CC8DXNJ39YWERYTWCGGCGG","next-workflow-step","Use this skill when the user wants to transition to the next workflow step, wrap up a session cleanly, prepare a handoff, or says things like \"next step\", \"continue the workflow\", \"wrap up\", \"hand off\", \"move to the next phase\", \"what's next in embrace\", or \"prepare for \u002Fclear\". Also triggers when the current octo:embrace phase appears complete and no explicit next action is set.","https:\u002F\u002Fgithub.com\u002Fray-manaloto\u002Fmacos-development-environment\u002Fblob\u002Fmain\u002Frsm-subagents\u002Fplugins\u002Fworkflow-toolkit\u002Fskills\u002Fnext-workflow-step\u002FSKILL.md",[71],{"categoryNo":72,"categoryName":73,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000023","Skill与智能体开发",[75,76,77,78,79,80,81,82,83,84,85],"next workflow step","workflow","up","use","session","user","wrap up","continue","handoff","prepare","SET",{"skillNo":87,"name":88,"nameZh":6,"description":89,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":90,"categories":91,"keywords":92,"publishedAt":28},"01M6CC8BRZBQHEBXGMKPDM422P","source-discovery","This skill should be used when the user asks to \"find sources\", \"discover URLs\", \"catalog a source\", \"classify a URL\", \"search for libraries\", \"check what exists\", or mentions source discovery, URL cataloging, source classification, or research source management.","https:\u002F\u002Fgithub.com\u002Fray-manaloto\u002Fmacos-development-environment\u002Fblob\u002Fmain\u002Frsm-subagents\u002Fplugins\u002Fresearch-review-toolkit\u002Fskills\u002Fsource-discovery\u002FSKILL.md",[],[93,94,95,96,97,98,99,100,101,80,102,103],"discovery","source discovery","research","libraries","search","classification","find","check","management","catalog","find sources",{"skillNo":105,"name":106,"nameZh":6,"description":107,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":108,"categories":109,"keywords":110,"publishedAt":28},"01M6CC89CGJP69A0PS22G2H1FV","review-synthesis","This skill should be used when the user asks to \"summarize review findings\", \"prioritize fixes\", \"rank review issues\", \"combine reviewer feedback\", \"create action items from review\", or mentions review synthesis, finding prioritization, severity ranking, or review result summarization.","https:\u002F\u002Fgithub.com\u002Fray-manaloto\u002Fmacos-development-environment\u002Fblob\u002Fmain\u002Frsm-subagents\u002Fplugins\u002Fresearch-review-toolkit\u002Fskills\u002Freview-synthesis\u002FSKILL.md",[],[111,112,113,114,115,116,117,80,118,119,120,121,122],"review","review synthesis","Synthesis","prioritize","create action","feedback","reviewer","summarize","prioritization","create","issues","severity",{"skillNo":124,"name":125,"nameZh":6,"description":126,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":127,"categories":128,"keywords":132,"publishedAt":28},"01M6CC872EZ9GRJY0HRCG0RT2M","research-scoring","This skill should be used when the user asks to \"score research\", \"check research quality\", \"evaluate coverage\", \"measure improvement\", \"assess research completeness\", or mentions research scoring, quality metrics, coverage gaps, improvement baseline, or research pipeline status.","https:\u002F\u002Fgithub.com\u002Fray-manaloto\u002Fmacos-development-environment\u002Fblob\u002Fmain\u002Frsm-subagents\u002Fplugins\u002Fresearch-review-toolkit\u002Fskills\u002Fresearch-scoring\u002FSKILL.md",[129],{"categoryNo":130,"categoryName":131,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000021","资料检索与研究",[133,95,134,135,136,137,138,139,100,80,140,141,142],"research scoring","status","coverage gaps","metrics","pipeline","baseline","quality","coverage","pipeline status","research quality",{"skillNo":144,"name":145,"nameZh":6,"description":146,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":147,"categories":148,"keywords":149,"publishedAt":28},"01M6CC852HSDS1FE4EJAT0RHGR","notebooklm-integration","This skill should be used when the user asks to \"ingest into NotebookLM\", \"query NotebookLM\", \"add sources to notebook\", \"cross-source synthesis\", \"check stale sources\", or mentions NotebookLM, notebook integration, source ingestion, or knowledge consolidation.","https:\u002F\u002Fgithub.com\u002Fray-manaloto\u002Fmacos-development-environment\u002Fblob\u002Fmain\u002Frsm-subagents\u002Fplugins\u002Fresearch-review-toolkit\u002Fskills\u002Fnotebooklm-integration\u002FSKILL.md",[],[150,151,152,153,100,80,154,155,156,113],"notebooklm integration","notebooklm","integration","knowledge","ingestion","ingest","stale",{"skillNo":158,"name":159,"nameZh":6,"description":160,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":161,"categories":162,"keywords":164,"publishedAt":28},"01M6CC82Z4V3205KCJWQZPS2VG","evidence-synthesis","This skill should be used when the user asks to \"synthesize findings\", \"combine research\", \"cross-reference sources\", \"summarize evidence\", \"write a deep review\", or mentions evidence synthesis, finding consolidation, cross-source analysis, or research summarization.","https:\u002F\u002Fgithub.com\u002Fray-manaloto\u002Fmacos-development-environment\u002Fblob\u002Fmain\u002Frsm-subagents\u002Fplugins\u002Fresearch-review-toolkit\u002Fskills\u002Fevidence-synthesis\u002FSKILL.md",[163],{"categoryNo":130,"categoryName":131,"taxonomyType":20,"taxonomyTypeDesc":21},[165,113,166,118,95,167,168,169,80,170,111],"evidence synthesis","evidence","deep review","analysis","write","reference"]