[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill:01M6EEBM4QWJM1HT99B061HXP3":3,"skill-same:01M6EEBM4QWJM1HT99B061HXP3":48},{"skillNo":4,"name":5,"nameZh":6,"description":6,"descriptionZh":6,"repositoryFullName":7,"ownerLogin":8,"ownerAvatarUrl":9,"ownerAccountNo":10,"ownerDisplayName":11,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":15,"categories":16,"keywords":17,"publishedAt":21,"linkOnly":22,"markdownContent":23,"markdownBody":23,"frontMatter":24,"refName":25,"commitSha":26,"skillDirectory":27,"forksCount":28,"repositoryHtmlUrl":29,"repositoryHomepageUrl":30,"ownerHtmlUrl":31,"ownerBlogUrl":6,"ownerDescription":32,"updatedAt":33,"install":34,"resources":47},"01M6EEBM4QWJM1HT99B061HXP3","apache-spark",null,"ComeOnOliver\u002Fskillshub","ComeOnOliver","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F43215937?v=4","01M6E5JNFECJAAVFE0W7SMWWD2","Haoran Shu","01M6E5JNNDQS7AJZDYFQ8TPZCK",67,"MIT","https:\u002F\u002Fgithub.com\u002FComeOnOliver\u002Fskillshub\u002Fblob\u002Fmain\u002Fskills\u002FTerminalSkills\u002Fskills\u002Fapache-spark\u002FSKILL.md",[],[18,19,20],"apache spark","spark","apache","2026-11-10 10:12:30",false,"# Apache Spark\n\n## Overview\n\nApache Spark is the standard for distributed data processing. It handles batch processing, streaming, SQL, machine learning, and graph processing. PySpark provides a Python API. Runs on standalone clusters, YARN, Kubernetes, or managed services (Databricks, EMR, Dataproc).\n\n## Instructions\n\n### Step 1: PySpark Setup\n\n```bash\npip install pyspark\n```\n\n### Step 2: DataFrame Operations\n\n```python\n# etl\u002Fprocess.py — PySpark data processing\nfrom pyspark.sql import SparkSession\nfrom pyspark.sql import functions as F\n\nspark = SparkSession.builder \\\n    .appName(\"DataPipeline\") \\\n    .config(\"spark.sql.adaptive.enabled\", \"true\") \\\n    .getOrCreate()\n\n# Read data\ndf = spark.read.parquet(\"s3:\u002F\u002Fbucket\u002Fraw\u002Fevents\u002F\")\n\n# Transform\nprocessed = (df\n    .filter(F.col(\"event_type\").isin([\"purchase\", \"signup\"]))\n    .withColumn(\"date\", F.to_date(\"timestamp\"))\n    .withColumn(\"revenue\", F.col(\"amount\") * F.col(\"quantity\"))\n    .groupBy(\"date\", \"event_type\")\n    .agg(\n        F.count(\"*\").alias(\"event_count\"),\n        F.sum(\"revenue\").alias(\"total_revenue\"),\n        F.countDistinct(\"user_id\").alias(\"unique_users\"),\n    )\n    .orderBy(\"date\")\n)\n\n# Write results\nprocessed.write \\\n    .mode(\"overwrite\") \\\n    .partitionBy(\"date\") \\\n    .parquet(\"s3:\u002F\u002Fbucket\u002Fprocessed\u002Fdaily_metrics\u002F\")\n```\n\n### Step 3: SQL Interface\n\n```python\n# Register as SQL table\ndf.createOrReplaceTempView(\"events\")\n\nresult = spark.sql(\"\"\"\n    SELECT\n        date_trunc('month', timestamp) as month,\n        COUNT(DISTINCT user_id) as monthly_active_users,\n        SUM(CASE WHEN event_type = 'purchase' THEN amount ELSE 0 END) as revenue\n    FROM events\n    WHERE timestamp >= '2025-01-01'\n    GROUP BY 1\n    ORDER BY 1\n\"\"\")\nresult.show()\n```\n\n### Step 4: Structured Streaming\n\n```python\n# Real-time processing from Kafka\nstream = spark.readStream \\\n    .format(\"kafka\") \\\n    .option(\"kafka.bootstrap.servers\", \"kafka:9092\") \\\n    .option(\"subscribe\", \"events\") \\\n    .load()\n\nparsed = stream.select(\n    F.from_json(F.col(\"value\").cast(\"string\"), schema).alias(\"data\")\n).select(\"data.*\")\n\nquery = parsed \\\n    .groupBy(F.window(\"timestamp\", \"5 minutes\"), \"event_type\") \\\n    .count() \\\n    .writeStream \\\n    .outputMode(\"update\") \\\n    .format(\"console\") \\\n    .start()\n```\n\n## Guidelines\n\n- Use DataFrames (not RDDs) for most work — they're optimized by Catalyst query optimizer.\n- Partitioning is critical for performance — partition by date or high-cardinality columns.\n- For managed Spark, consider Databricks (easiest), AWS EMR, or GCP Dataproc.\n- PySpark syntax mirrors Pandas but executes distributed — think in columns, not rows.\n",{},"main","def8531e65114c0fca8fb8551c1871ee0eed705c","skills\u002FTerminalSkills\u002Fskills\u002Fapache-spark",24,"https:\u002F\u002Fgithub.com\u002FComeOnOliver\u002Fskillshub","https:\u002F\u002Fskillshub.wtf","https:\u002F\u002Fgithub.com\u002FComeOnOliver","AI & Blockchain Engineer\r\n","2026-06-24 22:07:19",{"source":7,"skillName":5,"commands":35,"prompt":45,"zipAvailable":46},[36,39,42],{"tool":37,"command":38},"npx","npx skills add ComeOnOliver\u002Fskillshub --skill apache-spark",{"tool":40,"command":41},"bunx","bunx skills add ComeOnOliver\u002Fskillshub --skill apache-spark",{"tool":43,"command":44},"pnpm","pnpm dlx skills add ComeOnOliver\u002Fskillshub --skill apache-spark","安装技能，技能地址：https:\u002F\u002Fgithub.com\u002FComeOnOliver\u002Fskillshub\u002Fblob\u002Fmain\u002Fskills\u002FTerminalSkills\u002Fskills\u002Fapache-spark\u002FSKILL.md",true,[],[49,63,73,82,88,95],{"skillNo":50,"name":51,"nameZh":6,"description":6,"descriptionZh":6,"repositoryFullName":7,"ownerLogin":8,"ownerAvatarUrl":9,"ownerAccountNo":10,"ownerDisplayName":11,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":52,"categories":53,"keywords":59,"publishedAt":21},"01M6FKJN13GMG4KY57J3QD8GY5","model-hierarchy-skill","https:\u002F\u002Fgithub.com\u002FComeOnOliver\u002Fskillshub\u002Fblob\u002Fmain\u002Fskills\u002Fzscole\u002Fmodel-hierarchy-skill\u002Fmodel-hierarchy-skill\u002FSKILL.md",[54],{"categoryNo":55,"categoryName":56,"taxonomyType":57,"taxonomyTypeDesc":58},"01JBSEEDCAT000000000000023","Skill与智能体开发","SCENARIO","使用场景",[60,61,62],"model","model hierarchy","model hierarchy skill",{"skillNo":64,"name":65,"nameZh":6,"description":6,"descriptionZh":6,"repositoryFullName":7,"ownerLogin":8,"ownerAvatarUrl":9,"ownerAccountNo":10,"ownerDisplayName":11,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":66,"categories":67,"keywords":69,"publishedAt":21},"01M6FKJJJ6GCAMR4PPPRD12M5X","zoom-mcp","https:\u002F\u002Fgithub.com\u002FComeOnOliver\u002Fskillshub\u002Fblob\u002Fmain\u002Fskills\u002Fzoom\u002Fskills\u002Fzoom-mcp\u002FSKILL.md",[68],{"categoryNo":55,"categoryName":56,"taxonomyType":57,"taxonomyTypeDesc":58},[70,71,72],"MCP","zoom","zoom mcp",{"skillNo":74,"name":75,"nameZh":6,"description":6,"descriptionZh":6,"repositoryFullName":7,"ownerLogin":8,"ownerAvatarUrl":9,"ownerAccountNo":10,"ownerDisplayName":11,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":76,"categories":77,"keywords":78,"publishedAt":21},"01M6FKJE3KYT3KPEZMEEHYKA6K","zoom-apps-sdk","https:\u002F\u002Fgithub.com\u002FComeOnOliver\u002Fskillshub\u002Fblob\u002Fmain\u002Fskills\u002Fzoom\u002Fskills\u002Fzoom-apps-sdk\u002FSKILL.md",[],[79,80,81,71],"sdk","zoom apps sdk","apps",{"skillNo":83,"name":84,"nameZh":6,"description":6,"descriptionZh":6,"repositoryFullName":7,"ownerLogin":8,"ownerAvatarUrl":9,"ownerAccountNo":10,"ownerDisplayName":11,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":85,"categories":86,"keywords":87,"publishedAt":21},"01M6FKJ9EDH9Q5VWDF8HAJRSP0","windows","https:\u002F\u002Fgithub.com\u002FComeOnOliver\u002Fskillshub\u002Fblob\u002Fmain\u002Fskills\u002Fzoom\u002Fskills\u002Fwindows\u002FSKILL.md",[],[84],{"skillNo":89,"name":90,"nameZh":6,"description":6,"descriptionZh":6,"repositoryFullName":7,"ownerLogin":8,"ownerAvatarUrl":9,"ownerAccountNo":10,"ownerDisplayName":11,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":91,"categories":92,"keywords":94,"publishedAt":21},"01M6FKJ666BEEFC3AE9GJ72Q93","whiteboard","https:\u002F\u002Fgithub.com\u002FComeOnOliver\u002Fskillshub\u002Fblob\u002Fmain\u002Fskills\u002Fzoom\u002Fskills\u002Fwhiteboard\u002FSKILL.md",[93],{"categoryNo":55,"categoryName":56,"taxonomyType":57,"taxonomyTypeDesc":58},[90],{"skillNo":96,"name":97,"nameZh":6,"description":6,"descriptionZh":6,"repositoryFullName":7,"ownerLogin":8,"ownerAvatarUrl":9,"ownerAccountNo":10,"ownerDisplayName":11,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":98,"categories":99,"keywords":100,"publishedAt":21},"01M6FKJ21WFCDGTJQ2Q91T9DVW","websockets","https:\u002F\u002Fgithub.com\u002FComeOnOliver\u002Fskillshub\u002Fblob\u002Fmain\u002Fskills\u002Fzoom\u002Fskills\u002Fwebsockets\u002FSKILL.md",[],[97]]