[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill:01M6H077BCMY3EXC65DM834T6C":3,"skill-same:01M6H077BCMY3EXC65DM834T6C":72},{"skillNo":4,"name":5,"nameZh":6,"description":7,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":16,"sourceHtmlUrl":17,"categories":18,"keywords":19,"publishedAt":30,"linkOnly":31,"markdownContent":32,"markdownBody":33,"frontMatter":34,"refName":35,"commitSha":36,"skillDirectory":37,"forksCount":38,"repositoryHtmlUrl":39,"repositoryHomepageUrl":6,"ownerHtmlUrl":40,"ownerBlogUrl":41,"ownerDescription":42,"updatedAt":43,"install":44,"resources":57},"01M6H077BCMY3EXC65DM834T6C","scrape-legally-and-respect-robots-txt",null,"Use when collecting data from websites. Checks robots.txt, terms of service, and applicable law before crawling, and stops when they forbid it.","jiayaoqijia\u002Fcryptoskill","jiayaoqijia","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F1453879?v=4","01M6GMYTX7Q7S504VC6E756ENR","YQ","01M6GMYV1QKJ1CC75MTTGT9PR0",81,"MIT","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fpayments\u002F0xklen-scrape-legally-and-respect-robots-txt\u002FLICENSE","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fpayments\u002F0xklen-scrape-legally-and-respect-robots-txt\u002FSKILL.md",[],[20,21,22,23,24,25,26,27,28,29],"scrape","scrape legally and respect robots txt","data","terms","use","service","it","terms of service","crawling","when","2026-11-10 10:12:30",false,"---\nname: scrape-legally-and-respect-robots-txt\ndescription: Use when collecting data from websites. Checks robots.txt, terms of service, and applicable law before crawling, and stops when they forbid it.\n---\n\n# Scrape Legally and Respect robots.txt\n\nThe technical ability to fetch a page is not permission to fetch it. Read the rules first; a crawl that violates them is a liability regardless of how clean the code is.\n\n## Procedure\n\n1. Read `robots.txt` before the first request: `curl -s https:\u002F\u002Fexample.com\u002Frobots.txt`. Honour `Disallow` for your user-agent and check `Crawl-delay`. Do not fetch a path a rule excludes.\n2. Identify yourself honestly. Set a descriptive UA with a contact: `curl -A \"research-bot\u002F1.0 (+mailto:you@example.com)\" ...`. Never spoof a browser to slip past a block.\n3. Read the Terms of Service for the specific prohibition. Search the page: `curl -s https:\u002F\u002Fexample.com\u002Ftos | grep -inE 'scrap|crawl|robot|automated|bulk'`. \"No automated access\" means stop.\n4. Check the legal frame that applies: unauthorised access can implicate the CFAA (US) and ToS breach can be a contract claim; personal data triggers GDPR\u002FCCPA obligations. When in doubt, use an official API or a licensed dataset instead.\n5. Do not defeat controls. Bypassing a login wall, CAPTCHA, or IP block is the line between scraping and unauthorised access.\n6. Prefer the API, the bulk export, or the public dataset dump (`data.gov`, Common Crawl) over crawling an HTML site.\n7. Log the robots.txt decision and its timestamp in your notes so the choice is auditable.\n\n```bash\ncurl -s https:\u002F\u002Fexample.com\u002Frobots.txt | grep -iE 'User-agent|Disallow|Crawl-delay' \n# then verify your path is permitted before building the crawler\n```\n\n## Pitfalls\n\n- robots.txt is per-user-agent and per-path; a global `Disallow: \u002F` is not overridden by a friendlier section for another bot.\n- A `403` is an answer, not a barrier to route around with a proxy; stop and report it.\n- Public availability is not the same as a redistribution licence — the licence lives in the ToS or a header.\n- Scraping personal data (names, emails, profiles) has stricter rules than scraping product prices.\n- Rate is part of compliance: hammering an allowed path can still breach a `Crawl-delay`.\n- A site can change its robots.txt between your crawl planning and your run; re-fetch it at crawl start, not from a cached copy.\n\n## Verification\n\n    curl -s https:\u002F\u002Fexample.com\u002Frobots.txt; echo \"decision logged: allowed=\u002Fpath per $(date -u)\"\n\nReport: \"robots.txt permits \u002Fpath for this UA; ToS has no anti-scraping clause; using official API for the rate-limited endpoint.\"\n","# Scrape Legally and Respect robots.txt\n\nThe technical ability to fetch a page is not permission to fetch it. Read the rules first; a crawl that violates them is a liability regardless of how clean the code is.\n\n## Procedure\n\n1. Read `robots.txt` before the first request: `curl -s https:\u002F\u002Fexample.com\u002Frobots.txt`. Honour `Disallow` for your user-agent and check `Crawl-delay`. Do not fetch a path a rule excludes.\n2. Identify yourself honestly. Set a descriptive UA with a contact: `curl -A \"research-bot\u002F1.0 (+mailto:you@example.com)\" ...`. Never spoof a browser to slip past a block.\n3. Read the Terms of Service for the specific prohibition. Search the page: `curl -s https:\u002F\u002Fexample.com\u002Ftos | grep -inE 'scrap|crawl|robot|automated|bulk'`. \"No automated access\" means stop.\n4. Check the legal frame that applies: unauthorised access can implicate the CFAA (US) and ToS breach can be a contract claim; personal data triggers GDPR\u002FCCPA obligations. When in doubt, use an official API or a licensed dataset instead.\n5. Do not defeat controls. Bypassing a login wall, CAPTCHA, or IP block is the line between scraping and unauthorised access.\n6. Prefer the API, the bulk export, or the public dataset dump (`data.gov`, Common Crawl) over crawling an HTML site.\n7. Log the robots.txt decision and its timestamp in your notes so the choice is auditable.\n\n```bash\ncurl -s https:\u002F\u002Fexample.com\u002Frobots.txt | grep -iE 'User-agent|Disallow|Crawl-delay' \n# then verify your path is permitted before building the crawler\n```\n\n## Pitfalls\n\n- robots.txt is per-user-agent and per-path; a global `Disallow: \u002F` is not overridden by a friendlier section for another bot.\n- A `403` is an answer, not a barrier to route around with a proxy; stop and report it.\n- Public availability is not the same as a redistribution licence — the licence lives in the ToS or a header.\n- Scraping personal data (names, emails, profiles) has stricter rules than scraping product prices.\n- Rate is part of compliance: hammering an allowed path can still breach a `Crawl-delay`.\n- A site can change its robots.txt between your crawl planning and your run; re-fetch it at crawl start, not from a cached copy.\n\n## Verification\n\n    curl -s https:\u002F\u002Fexample.com\u002Frobots.txt; echo \"decision logged: allowed=\u002Fpath per $(date -u)\"\n\nReport: \"robots.txt permits \u002Fpath for this UA; ToS has no anti-scraping clause; using official API for the rate-limited endpoint.\"\n",{"name":5,"description":7},"main","3df1d0d67b30ff15236561f8c0f9f06d33f42dfd","skills\u002Fpayments\u002F0xklen-scrape-legally-and-respect-robots-txt",17,"https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia","https:\u002F\u002Feth2030.com","Hardcore infra. Building @alt-research and applied cryptography.","2026-11-12 02:28:42",{"source":8,"skillName":5,"commands":45,"prompt":55,"zipAvailable":56},[46,49,52],{"tool":47,"command":48},"npx","npx skills add jiayaoqijia\u002Fcryptoskill --skill scrape-legally-and-respect-robots-txt",{"tool":50,"command":51},"bunx","bunx skills add jiayaoqijia\u002Fcryptoskill --skill scrape-legally-and-respect-robots-txt",{"tool":53,"command":54},"pnpm","pnpm dlx skills add jiayaoqijia\u002Fcryptoskill --skill scrape-legally-and-respect-robots-txt","安装技能，技能地址：https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fpayments\u002F0xklen-scrape-legally-and-respect-robots-txt\u002FSKILL.md",true,[58,62,68],{"path":59,"resourceType":59,"resourceTypeDesc":60,"sizeBytes":61,"sourceHtmlUrl":16},"LICENSE","许可证",1063,{"path":63,"resourceType":64,"resourceTypeDesc":65,"sizeBytes":66,"sourceHtmlUrl":67},"SOURCE.md","REFERENCE","参考文档",363,"https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fpayments\u002F0xklen-scrape-legally-and-respect-robots-txt\u002FSOURCE.md",{"path":69,"resourceType":64,"resourceTypeDesc":65,"sizeBytes":70,"sourceHtmlUrl":71},"TRUST.auto.yaml",2197,"https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fpayments\u002F0xklen-scrape-legally-and-respect-robots-txt\u002FTRUST.auto.yaml",[73,101,124,141,154,164],{"skillNo":74,"name":75,"nameZh":6,"description":76,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":77,"sourceHtmlUrl":78,"categories":79,"keywords":80,"publishedAt":30},"01M712C436EQYE43342Q4E7GG1","recipe-performance","Diagnose a specific MetaMask user flow's real-app performance and Sentry critical-user-flow telemetry with deterministic mm-harness recipes. Use for CUF boundaries, cold\u002Fwarm\u002Fbackground-resume behavior, latency regressions, lifecycle tags, duplicate or blocking network requests, long tasks, layout\u002Fpaint cost, screen redraws, React rerenders, or browser trace analysis where tests alone cannot prove the real app.","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fwallets\u002Fmetamask-official-recipe-performance\u002FLICENSE","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fwallets\u002Fmetamask-official-recipe-performance\u002FSKILL.md",[],[81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100],"performance","recipe performance","recipe","mm","sentry","harness","app","deterministic","recipes","flow","cold","critical","analysis","tasks","react","browser","layout","screen","network","metamask",{"skillNo":102,"name":103,"nameZh":6,"description":104,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":105,"licenseFileUrl":106,"sourceHtmlUrl":107,"categories":108,"keywords":109,"publishedAt":30},"01M70XT2F8A5CQZ81JBP98KSW3","mexc-spot-websocket","MEXC Spot WebSocket workflows for public streams, protobuf handling, depth maintenance, listen keys, and private streams without exposing API keys.","Apache-2.0","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fexchanges\u002Fmncrftfrcnm-mexc-spot-websocket\u002FLICENSE","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fexchanges\u002Fmncrftfrcnm-mexc-spot-websocket\u002FSKILL.md",[],[110,111,112,113,114,115,116,117,118,119,120,121,122,123],"mexc spot websocket","spot","websocket","api","workflows","public","listen","streams","depth","keys","for","api keys","maintenance","protobuf",{"skillNo":125,"name":126,"nameZh":6,"description":127,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":105,"licenseFileUrl":128,"sourceHtmlUrl":129,"categories":130,"keywords":131,"publishedAt":30},"01M70XT0MBXN3WZY1BKN0CPNKR","mexc-spot-rest","MEXC Spot REST workflows for public data, signed account requests, orders, wallet\u002Fadmin actions, and secure signing without exposing API keys.","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fexchanges\u002Fmncrftfrcnm-mexc-spot-rest\u002FLICENSE","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fexchanges\u002Fmncrftfrcnm-mexc-spot-rest\u002FSKILL.md",[],[132,111,133,134,135,136,137,121,120,119,138,139,22,115,140,114,113],"mexc spot rest","REST","admin","signing","Wallet","requests","actions","secure","orders",{"skillNo":142,"name":143,"nameZh":6,"description":144,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":105,"licenseFileUrl":145,"sourceHtmlUrl":146,"categories":147,"keywords":148,"publishedAt":30},"01M70XSYSVZ2N0N91XCAR2CAK0","mexc-futures-websocket","MEXC Futures WebSocket workflows for public streams, private account\u002Forder\u002Fposition streams, ping\u002Fpong, subscriptions, and depth maintenance.","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fexchanges\u002Fmncrftfrcnm-mexc-futures-websocket\u002FLICENSE","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fexchanges\u002Fmncrftfrcnm-mexc-futures-websocket\u002FSKILL.md",[],[149,150,112,151,152,122,120,153,118,117,115,114],"mexc futures websocket","futures","position","ping","subscriptions",{"skillNo":155,"name":156,"nameZh":6,"description":6,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":157,"sourceHtmlUrl":158,"categories":159,"keywords":160,"publishedAt":30},"01M6H0ZWHVE5W04WG2DKQE2SQC","0xklen-version-and-refresh-sanctions-lists","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fpayments\u002F0xklen-version-and-refresh-sanctions-lists\u002FLICENSE","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fpayments\u002F0xklen-version-and-refresh-sanctions-lists\u002FSKILL.md",[],[161,162,163],"refresh","0xklen version and refresh sanctions lists","version",{"skillNo":165,"name":166,"nameZh":6,"description":167,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":168,"sourceHtmlUrl":169,"categories":170,"keywords":171,"publishedAt":30},"01M6H0Z28Q3R8J0S799GJQWXM3","verify-tests-cover-changed-lines","Use when a PR adds code that the existing suite may not exercise. Measures patch coverage — the share of newly changed lines hit by tests — before approving.","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fpayments\u002F0xklen-verify-tests-cover-changed-lines\u002FLICENSE","https:\u002F\u002Fgithub.com\u002Fjiayaoqijia\u002Fcryptoskill\u002Fblob\u002Fmain\u002Fskills\u002Fpayments\u002F0xklen-verify-tests-cover-changed-lines\u002FSKILL.md",[],[172,173,174,175,176,177,178,24,179,180,181],"tests","cover","verify tests cover changed lines","verify","coverage","pr","exercise","patch","code","share"]