[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill:01M718CK7ZZRDM5KS0P8XEPK79":3,"skill-same:01M718CK7ZZRDM5KS0P8XEPK79":70},{"skillNo":4,"name":5,"nameZh":6,"description":7,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":16,"categories":17,"keywords":23,"publishedAt":44,"linkOnly":45,"markdownContent":46,"markdownBody":47,"frontMatter":48,"refName":50,"commitSha":51,"skillDirectory":52,"forksCount":14,"repositoryHtmlUrl":53,"repositoryHomepageUrl":6,"ownerHtmlUrl":54,"ownerBlogUrl":6,"ownerDescription":6,"updatedAt":55,"install":56,"resources":69},"01M718CK7ZZRDM5KS0P8XEPK79","data-engineering-data-pipeline",null,"You are a data pipeline architecture expert specializing in scalable, reliable, and cost-effective data pipelines for batch and streaming data processing.","sapsapshen\u002Fai-memory-router","sapsapshen","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F20347959?v=4","01M6XDQEF1DP50DKF37H81J3J5","shenmian","01M6XDQEJPE41Y3DT2763X8ZJN",0,"MIT","https:\u002F\u002Fgithub.com\u002Fsapsapshen\u002Fai-memory-router\u002Fblob\u002Fmain\u002Fskills-store\u002Fglobal\u002Fdata-engineering-data-pipeline\u002FSKILL.md",[18],{"categoryNo":19,"categoryName":20,"taxonomyType":21,"taxonomyTypeDesc":22},"01JBSEEDCAT000000000000033","数据分析师","PROFESSION","适用职业",[24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43],"data pipeline","data engineering","engineering","data","pipeline","data engineering data pipeline","2026","architecture","unknown","for","you","Scalable","processing","community","expert","cost","data processing","streaming","batch","data pipelines","2026-11-10 10:12:30",false,"---\nname: data-engineering-data-pipeline\ndescription: \"You are a data pipeline architecture expert specializing in scalable, reliable, and cost-effective data pipelines for batch and streaming data processing.\"\nrisk: unknown\nsource: community\ndate_added: \"2026-02-27\"\n---\n\n# Data Pipeline Architecture\n\nYou are a data pipeline architecture expert specializing in scalable, reliable, and cost-effective data pipelines for batch and streaming data processing.\n\n## Use this skill when\n\n- Working on data pipeline architecture tasks or workflows\n- Needing guidance, best practices, or checklists for data pipeline architecture\n\n## Do not use this skill when\n\n- The task is unrelated to data pipeline architecture\n- You need a different domain or tool outside this scope\n\n## Requirements\n\n$ARGUMENTS\n\n## Core Capabilities\n\n- Design ETL\u002FELT, Lambda, Kappa, and Lakehouse architectures\n- Implement batch and streaming data ingestion\n- Build workflow orchestration with Airflow\u002FPrefect\n- Transform data using dbt and Spark\n- Manage Delta Lake\u002FIceberg storage with ACID transactions\n- Implement data quality frameworks (Great Expectations, dbt tests)\n- Monitor pipelines with CloudWatch\u002FPrometheus\u002FGrafana\n- Optimize costs through partitioning, lifecycle policies, and compute optimization\n\n## Instructions\n\n### 1. Architecture Design\n- Assess: sources, volume, latency requirements, targets\n- Select pattern: ETL (transform before load), ELT (load then transform), Lambda (batch + speed layers), Kappa (stream-only), Lakehouse (unified)\n- Design flow: sources → ingestion → processing → storage → serving\n- Add observability touchpoints\n\n### 2. Ingestion Implementation\n**Batch**\n- Incremental loading with watermark columns\n- Retry logic with exponential backoff\n- Schema validation and dead letter queue for invalid records\n- Metadata tracking (_extracted_at, _source)\n\n**Streaming**\n- Kafka consumers with exactly-once semantics\n- Manual offset commits within transactions\n- Windowing for time-based aggregations\n- Error handling and replay capability\n\n### 3. Orchestration\n**Airflow**\n- Task groups for logical organization\n- XCom for inter-task communication\n- SLA monitoring and email alerts\n- Incremental execution with execution_date\n- Retry with exponential backoff\n\n**Prefect**\n- Task caching for idempotency\n- Parallel execution with .submit()\n- Artifacts for visibility\n- Automatic retries with configurable delays\n\n### 4. Transformation with dbt\n- Staging layer: incremental materialization, deduplication, late-arriving data handling\n- Marts layer: dimensional models, aggregations, business logic\n- Tests: unique, not_null, relationships, accepted_values, custom data quality tests\n- Sources: freshness checks, loaded_at_field tracking\n- Incremental strategy: merge or delete+insert\n\n### 5. Data Quality Framework\n**Great Expectations**\n- Table-level: row count, column count\n- Column-level: uniqueness, nullability, type validation, value sets, ranges\n- Checkpoints for validation execution\n- Data docs for documentation\n- Failure notifications\n\n**dbt Tests**\n- Schema tests in YAML\n- Custom data quality tests with dbt-expectations\n- Test results tracked in metadata\n\n### 6. Storage Strategy\n**Delta Lake**\n- ACID transactions with append\u002Foverwrite\u002Fmerge modes\n- Upsert with predicate-based matching\n- Time travel for historical queries\n- Optimize: compact small files, Z-order clustering\n- Vacuum to remove old files\n\n**Apache Iceberg**\n- Partitioning and sort order optimization\n- MERGE INTO for upserts\n- Snapshot isolation and time travel\n- File compaction with binpack strategy\n- Snapshot expiration for cleanup\n\n### 7. Monitoring & Cost Optimization\n**Monitoring**\n- Track: records processed\u002Ffailed, data size, execution time, success\u002Ffailure rates\n- CloudWatch metrics and custom namespaces\n- SNS alerts for critical\u002Fwarning\u002Finfo events\n- Data freshness checks\n- Performance trend analysis\n\n**Cost Optimization**\n- Partitioning: date\u002Fentity-based, avoid over-partitioning (keep >1GB)\n- File sizes: 512MB-1GB for Parquet\n- Lifecycle policies: hot (Standard) → warm (IA) → cold (Glacier)\n- Compute: spot instances for batch, on-demand for streaming, serverless for adhoc\n- Query optimization: partition pruning, clustering, predicate pushdown\n\n## Example: Minimal Batch Pipeline\n\n```python\n# Batch ingestion with validation\nfrom batch_ingestion import BatchDataIngester\nfrom storage.delta_lake_manager import DeltaLakeManager\nfrom data_quality.expectations_suite import DataQualityFramework\n\ningester = BatchDataIngester(config={})\n\n# Extract with incremental loading\ndf = ingester.extract_from_database(\n    connection_string='postgresql:\u002F\u002Fhost:5432\u002Fdb',\n    query='SELECT * FROM orders',\n    watermark_column='updated_at',\n    last_watermark=last_run_timestamp\n)\n\n# Validate\nschema = {'required_fields': ['id', 'user_id'], 'dtypes': {'id': 'int64'}}\ndf = ingester.validate_and_clean(df, schema)\n\n# Data quality checks\ndq = DataQualityFramework()\nresult = dq.validate_dataframe(df, suite_name='orders_suite', data_asset_name='orders')\n\n# Write to Delta Lake\ndelta_mgr = DeltaLakeManager(storage_path='s3:\u002F\u002Flake')\ndelta_mgr.create_or_update_table(\n    df=df,\n    table_name='orders',\n    partition_columns=['order_date'],\n    mode='append'\n)\n\n# Save failed records\ningester.save_dead_letter_queue('s3:\u002F\u002Flake\u002Fdlq\u002Forders')\n```\n\n## Output Deliverables\n\n### 1. Architecture Documentation\n- Architecture diagram with data flow\n- Technology stack with justification\n- Scalability analysis and growth patterns\n- Failure modes and recovery strategies\n\n### 2. Implementation Code\n- Ingestion: batch\u002Fstreaming with error handling\n- Transformation: dbt models (staging → marts) or Spark jobs\n- Orchestration: Airflow\u002FPrefect DAGs with dependencies\n- Storage: Delta\u002FIceberg table management\n- Data quality: Great Expectations suites and dbt tests\n\n### 3. Configuration Files\n- Orchestration: DAG definitions, schedules, retry policies\n- dbt: models, sources, tests, project config\n- Infrastructure: Docker Compose, K8s manifests, Terraform\n- Environment: dev\u002Fstaging\u002Fprod configs\n\n### 4. Monitoring & Observability\n- Metrics: execution time, records processed, quality scores\n- Alerts: failures, performance degradation, data freshness\n- Dashboards: Grafana\u002FCloudWatch for pipeline health\n- Logging: structured logs with correlation IDs\n\n### 5. Operations Guide\n- Deployment procedures and rollback strategy\n- Troubleshooting guide for common issues\n- Scaling guide for increased volume\n- Cost optimization strategies and savings\n- Disaster recovery and backup procedures\n\n## Success Criteria\n- Pipeline meets defined SLA (latency, throughput)\n- Data quality checks pass with >99% success rate\n- Automatic retry and alerting on failures\n- Comprehensive monitoring shows health and performance\n- Documentation enables team maintenance\n- Cost optimization reduces infrastructure costs by 30-50%\n- Schema evolution without downtime\n- End-to-end data lineage tracked\n","# Data Pipeline Architecture\n\nYou are a data pipeline architecture expert specializing in scalable, reliable, and cost-effective data pipelines for batch and streaming data processing.\n\n## Use this skill when\n\n- Working on data pipeline architecture tasks or workflows\n- Needing guidance, best practices, or checklists for data pipeline architecture\n\n## Do not use this skill when\n\n- The task is unrelated to data pipeline architecture\n- You need a different domain or tool outside this scope\n\n## Requirements\n\n$ARGUMENTS\n\n## Core Capabilities\n\n- Design ETL\u002FELT, Lambda, Kappa, and Lakehouse architectures\n- Implement batch and streaming data ingestion\n- Build workflow orchestration with Airflow\u002FPrefect\n- Transform data using dbt and Spark\n- Manage Delta Lake\u002FIceberg storage with ACID transactions\n- Implement data quality frameworks (Great Expectations, dbt tests)\n- Monitor pipelines with CloudWatch\u002FPrometheus\u002FGrafana\n- Optimize costs through partitioning, lifecycle policies, and compute optimization\n\n## Instructions\n\n### 1. Architecture Design\n- Assess: sources, volume, latency requirements, targets\n- Select pattern: ETL (transform before load), ELT (load then transform), Lambda (batch + speed layers), Kappa (stream-only), Lakehouse (unified)\n- Design flow: sources → ingestion → processing → storage → serving\n- Add observability touchpoints\n\n### 2. Ingestion Implementation\n**Batch**\n- Incremental loading with watermark columns\n- Retry logic with exponential backoff\n- Schema validation and dead letter queue for invalid records\n- Metadata tracking (_extracted_at, _source)\n\n**Streaming**\n- Kafka consumers with exactly-once semantics\n- Manual offset commits within transactions\n- Windowing for time-based aggregations\n- Error handling and replay capability\n\n### 3. Orchestration\n**Airflow**\n- Task groups for logical organization\n- XCom for inter-task communication\n- SLA monitoring and email alerts\n- Incremental execution with execution_date\n- Retry with exponential backoff\n\n**Prefect**\n- Task caching for idempotency\n- Parallel execution with .submit()\n- Artifacts for visibility\n- Automatic retries with configurable delays\n\n### 4. Transformation with dbt\n- Staging layer: incremental materialization, deduplication, late-arriving data handling\n- Marts layer: dimensional models, aggregations, business logic\n- Tests: unique, not_null, relationships, accepted_values, custom data quality tests\n- Sources: freshness checks, loaded_at_field tracking\n- Incremental strategy: merge or delete+insert\n\n### 5. Data Quality Framework\n**Great Expectations**\n- Table-level: row count, column count\n- Column-level: uniqueness, nullability, type validation, value sets, ranges\n- Checkpoints for validation execution\n- Data docs for documentation\n- Failure notifications\n\n**dbt Tests**\n- Schema tests in YAML\n- Custom data quality tests with dbt-expectations\n- Test results tracked in metadata\n\n### 6. Storage Strategy\n**Delta Lake**\n- ACID transactions with append\u002Foverwrite\u002Fmerge modes\n- Upsert with predicate-based matching\n- Time travel for historical queries\n- Optimize: compact small files, Z-order clustering\n- Vacuum to remove old files\n\n**Apache Iceberg**\n- Partitioning and sort order optimization\n- MERGE INTO for upserts\n- Snapshot isolation and time travel\n- File compaction with binpack strategy\n- Snapshot expiration for cleanup\n\n### 7. Monitoring & Cost Optimization\n**Monitoring**\n- Track: records processed\u002Ffailed, data size, execution time, success\u002Ffailure rates\n- CloudWatch metrics and custom namespaces\n- SNS alerts for critical\u002Fwarning\u002Finfo events\n- Data freshness checks\n- Performance trend analysis\n\n**Cost Optimization**\n- Partitioning: date\u002Fentity-based, avoid over-partitioning (keep >1GB)\n- File sizes: 512MB-1GB for Parquet\n- Lifecycle policies: hot (Standard) → warm (IA) → cold (Glacier)\n- Compute: spot instances for batch, on-demand for streaming, serverless for adhoc\n- Query optimization: partition pruning, clustering, predicate pushdown\n\n## Example: Minimal Batch Pipeline\n\n```python\n# Batch ingestion with validation\nfrom batch_ingestion import BatchDataIngester\nfrom storage.delta_lake_manager import DeltaLakeManager\nfrom data_quality.expectations_suite import DataQualityFramework\n\ningester = BatchDataIngester(config={})\n\n# Extract with incremental loading\ndf = ingester.extract_from_database(\n    connection_string='postgresql:\u002F\u002Fhost:5432\u002Fdb',\n    query='SELECT * FROM orders',\n    watermark_column='updated_at',\n    last_watermark=last_run_timestamp\n)\n\n# Validate\nschema = {'required_fields': ['id', 'user_id'], 'dtypes': {'id': 'int64'}}\ndf = ingester.validate_and_clean(df, schema)\n\n# Data quality checks\ndq = DataQualityFramework()\nresult = dq.validate_dataframe(df, suite_name='orders_suite', data_asset_name='orders')\n\n# Write to Delta Lake\ndelta_mgr = DeltaLakeManager(storage_path='s3:\u002F\u002Flake')\ndelta_mgr.create_or_update_table(\n    df=df,\n    table_name='orders',\n    partition_columns=['order_date'],\n    mode='append'\n)\n\n# Save failed records\ningester.save_dead_letter_queue('s3:\u002F\u002Flake\u002Fdlq\u002Forders')\n```\n\n## Output Deliverables\n\n### 1. Architecture Documentation\n- Architecture diagram with data flow\n- Technology stack with justification\n- Scalability analysis and growth patterns\n- Failure modes and recovery strategies\n\n### 2. Implementation Code\n- Ingestion: batch\u002Fstreaming with error handling\n- Transformation: dbt models (staging → marts) or Spark jobs\n- Orchestration: Airflow\u002FPrefect DAGs with dependencies\n- Storage: Delta\u002FIceberg table management\n- Data quality: Great Expectations suites and dbt tests\n\n### 3. Configuration Files\n- Orchestration: DAG definitions, schedules, retry policies\n- dbt: models, sources, tests, project config\n- Infrastructure: Docker Compose, K8s manifests, Terraform\n- Environment: dev\u002Fstaging\u002Fprod configs\n\n### 4. Monitoring & Observability\n- Metrics: execution time, records processed, quality scores\n- Alerts: failures, performance degradation, data freshness\n- Dashboards: Grafana\u002FCloudWatch for pipeline health\n- Logging: structured logs with correlation IDs\n\n### 5. Operations Guide\n- Deployment procedures and rollback strategy\n- Troubleshooting guide for common issues\n- Scaling guide for increased volume\n- Cost optimization strategies and savings\n- Disaster recovery and backup procedures\n\n## Success Criteria\n- Pipeline meets defined SLA (latency, throughput)\n- Data quality checks pass with >99% success rate\n- Automatic retry and alerting on failures\n- Comprehensive monitoring shows health and performance\n- Documentation enables team maintenance\n- Cost optimization reduces infrastructure costs by 30-50%\n- Schema evolution without downtime\n- End-to-end data lineage tracked\n",{"name":5,"risk":32,"source":37,"date_added":49,"description":7},"2026-02-27","main","7a38d8e7073e61c16eda85681cb32fb68a8657a5","skills-store\u002Fglobal\u002Fdata-engineering-data-pipeline","https:\u002F\u002Fgithub.com\u002Fsapsapshen\u002Fai-memory-router","https:\u002F\u002Fgithub.com\u002Fsapsapshen","2026-04-14 21:12:12",{"source":8,"skillName":5,"commands":57,"prompt":67,"zipAvailable":68},[58,61,64],{"tool":59,"command":60},"npx","npx skills add sapsapshen\u002Fai-memory-router --skill data-engineering-data-pipeline",{"tool":62,"command":63},"bunx","bunx skills add sapsapshen\u002Fai-memory-router --skill data-engineering-data-pipeline",{"tool":65,"command":66},"pnpm","pnpm dlx skills add sapsapshen\u002Fai-memory-router --skill data-engineering-data-pipeline","安装技能，技能地址：https:\u002F\u002Fgithub.com\u002Fsapsapshen\u002Fai-memory-router\u002Fblob\u002Fmain\u002Fskills-store\u002Fglobal\u002Fdata-engineering-data-pipeline\u002FSKILL.md",true,[],[71,93,118,128,141,160],{"skillNo":72,"name":73,"nameZh":6,"description":74,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":75,"categories":76,"keywords":80,"publishedAt":44},"01M71ATQRKG48SH06FQ697T5G9","wechat-search","Search WeChat Official Account articles using OpenClaw's web search and fetch capabilities with compliance-focused design.","https:\u002F\u002Fgithub.com\u002Fsapsapshen\u002Fai-memory-router\u002Fblob\u002Fmain\u002Fskills-store\u002Fopenclaw\u002Fwechat-search-release\u002FSKILL.md",[77],{"categoryNo":78,"categoryName":79,"taxonomyType":21,"taxonomyTypeDesc":22},"01JBSEEDCAT000000000000037","律师与法务",[81,82,83,84,85,86,87,88,89,90,91,92],"wechat search","wechat","search","capabilities","openclaw","design","web search","compliance","web","official","fetch","with",{"skillNo":94,"name":95,"nameZh":6,"description":96,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":97,"categories":98,"keywords":99,"publishedAt":44},"01M71ATNAVGZB35PFYM2GC5N00","trend-tap","Real-time trending topics aggregator across 7 platforms (X\u002FTwitter, Reddit, Google Trends, Hacker News, Zhihu, Bilibili, Weibo). Trigger: when user says 'trend', 'trends', 'trend tap', '热点', '热搜', or asks what is trending\u002Fhot. Do NOT use web_search — this skill already fetches live data from all platforms with zero API keys.","https:\u002F\u002Fgithub.com\u002Fsapsapshen\u002Fai-memory-router\u002Fblob\u002Fmain\u002Fskills-store\u002Fopenclaw\u002Ftrend-tap\u002FSKILL.md",[],[100,101,102,103,104,105,83,106,107,108,109,110,111,112,27,113,114,115,116,117],"trend","trend tap","tap","api keys","Bash","time","trending","hacker","aggregator","do","when","brew","live","google","install","read","across","api",{"skillNo":119,"name":120,"nameZh":6,"description":121,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":122,"categories":123,"keywords":124,"publishedAt":44},"01M71ATJ9AM5KM15AA8PS302EB","trend-radar","Real-time trending topics aggregator across 7 platforms (X\u002FTwitter, Reddit, Google Trends, Hacker News, Zhihu, Bilibili, Weibo). Trigger: when user says 'trend', 'trends', 'trend pulse', '热点', '热搜', or asks what is trending\u002Fhot. Do NOT use web_search — this skill already fetches live data from all platforms with zero API keys.","https:\u002F\u002Fgithub.com\u002Fsapsapshen\u002Fai-memory-router\u002Fblob\u002Fmain\u002Fskills-store\u002Fopenclaw\u002Ftrend-radar\u002FSKILL.md",[],[125,126,100,111,112,27,113,104,83,115,127,116,117,110,108,107,106,114,105,103],"trend radar","radar","pulse",{"skillNo":129,"name":130,"nameZh":6,"description":131,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":132,"categories":133,"keywords":134,"publishedAt":44},"01M71ATGF095N3EJM9G8SD0BB5","tech-news-daily","每日科技新闻汇——从全球多个 RSS 源抓取最新科技新闻，翻译（英→中）、去重、排版成每日简报。当用户询问科技新闻、科技新鲜事、每日科技、技术动态、Daily Tech News 或类似表达时触发此技能。","https:\u002F\u002Fgithub.com\u002Fsapsapshen\u002Fai-memory-router\u002Fblob\u002Fmain\u002Fskills-store\u002Fopenclaw\u002Ftech-news-daily\u002FSKILL.md",[],[135,136,137,138,139,140],"news","tech","tech news daily","daily","rss","翻译",{"skillNo":142,"name":143,"nameZh":6,"description":144,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":145,"categories":146,"keywords":152,"publishedAt":44},"01M71ATFAEAMQ17M70NQNY7SR8","summarize","Summarize URLs or files with the summarize CLI (web, PDFs, images, audio, YouTube).","https:\u002F\u002Fgithub.com\u002Fsapsapshen\u002Fai-memory-router\u002Fblob\u002Fmain\u002Fskills-store\u002Fopenclaw\u002Fsummarize\u002FSKILL.md",[147],{"categoryNo":148,"categoryName":149,"taxonomyType":150,"taxonomyTypeDesc":151},"01JBSEEDCAT000000000000021","资料检索与研究","SCENARIO","使用场景",[143,89,153,154,155,92,111,156,157,158,102,159,114],"Audio","cli","youtube","images","https","files","pdfs",{"skillNo":161,"name":162,"nameZh":6,"description":163,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":164,"categories":165,"keywords":166,"publishedAt":44},"01M71ATDNS3R0M4H936PACQAWZ","sonoscli","Control Sonos speakers (discover\u002Fstatus\u002Fplay\u002Fvolume\u002Fgroup).","https:\u002F\u002Fgithub.com\u002Fsapsapshen\u002Fai-memory-router\u002Fblob\u002Fmain\u002Fskills-store\u002Fopenclaw\u002Fsonoscli\u002FSKILL.md",[],[162,167,168,169,114,170,157,171,172,173,174,175],"status","play","go","latest","github","discover","control","cmd","volume"]