[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill:01M77SVYC9V16P09M8FGGRDEDV":3,"skill-same:01M77SVYC9V16P09M8FGGRDEDV":59},{"skillNo":4,"name":5,"nameZh":6,"description":7,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":16,"categories":17,"keywords":18,"publishedAt":19,"linkOnly":20,"markdownContent":21,"markdownBody":22,"frontMatter":23,"refName":36,"commitSha":37,"skillDirectory":38,"forksCount":39,"repositoryHtmlUrl":40,"repositoryHomepageUrl":41,"ownerHtmlUrl":42,"ownerBlogUrl":6,"ownerDescription":43,"updatedAt":44,"install":45,"resources":58},"01M77SVYC9V16P09M8FGGRDEDV","Real Estate Spider",null,"专业爬取中国房产中介网站（安居客、搜房网、贝壳找房、链家）数据的通用爬虫技能，包含反爬虫策略和自动数据提取功能","Lord1Egypt\u002Fawesome-skill-forge","Lord1Egypt","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F99746359?v=4","01M7592YDNT0XK9WJ0T2MQVPDN","Mohamed Mounir Ali AL Keshita","01M75GRFNHC8VZNWEB8DKDE433",3,"MIT","https:\u002F\u002Fgithub.com\u002FLord1Egypt\u002Fawesome-skill-forge\u002Fblob\u002Fmaster\u002Fcommunity\u002Fclawhub\u002Fr\u002Freal-estate-spider\u002FSKILL.md",[],[],"2026-11-15 14:19:17",false,"---\nname: Real Estate Spider\ndescription: 专业爬取中国房产中介网站（安居客、搜房网、贝壳找房、链家）数据的通用爬虫技能，包含反爬虫策略和自动数据提取功能\nread_when:\n  - 需要爬取房产中介网站数据\n  - 需要绕过网站反爬虫机制\n  - 需要提取房产信息（价格、面积、位置、户型等）\nmetadata: {\"clawdbot\":{\"emoji\":\"🏠\",\"requires\":{\"bins\":[\"python3\",\"agent-browser\"]}}}\nallowed-tools: Bash(agent-browser:*)\n---\n\n# 房产中介网站爬虫技能\n\n## 简介\n\n本技能专门用于爬取中国主流房产中介网站数据，包括：\n1. 安居客 (anjuke.com)\n2. 搜房网 (soufun.com)\n3. 贝壳找房 (ke.com)\n4. 链家 (lianjia.com)\n\n## 前置要求\n\n- Python 3.x\n- agent-browser 技能已安装\n- requests 库 (可以通过 pip 安装)\n\n## 安装依赖\n\n```bash\n# 安装 Python requests 库\npip install requests beautifulsoup4 lxml\n```\n\n## 主要功能\n\n### 1. 反爬虫绕过策略\n- 模拟真实浏览器指纹\n- 随机延迟避免频率检测\n- Cookie 和会话管理\n- 代理 IP 支持（可选）\n- 验证码处理机制\n\n### 2. 数据提取功能\n- 提取房价信息\n- 提取房产面积\n- 提取地理位置\n- 提取户型信息\n- 提取装修状态\n- 提取建筑年代\n\n### 3. 导出格式\n- JSON 格式\n- CSV 格式\n- Excel 格式\n- 可视化图表\n\n## 使用方法\n\n### 基本爬虫脚本\n\n```bash\n# 使用 Python 脚本爬取安居客数据\npython3 ~\u002F.openclaw\u002Fworkspace\u002Fskills\u002Freal-estate-spider\u002Fscripts\u002Fanjuke_crawler.py\n\n# 使用 Shell 脚本配合 agent-browser\nbash ~\u002F.openclaw\u002Fworkspace\u002Fskills\u002Freal-estate-spider\u002Fscripts\u002Fbypass_anjuke.sh\n```\n\n### 配置网站选择\n\n```python\n# 配置文件示例\n# ~\u002F.openclaw\u002Fworkspace\u002Fskills\u002Freal-estate-spider\u002Fconfig\u002Freal_estate_config.py\n\nimport json\n\nCONFIG = {\n    \"anjuke\": {\n        \"url\": \"https:\u002F\u002Fwww.anjuke.com\",\n        \"data_selectors\": {\n            \"price\": \".property-price\",\n            \"area\": \".property-area\",\n            \"location\": \".property-location\",\n            \"type\": \".property-type\"\n        }\n    },\n    \"ke\": {\n        \"url\": \"https:\u002F\u002Fke.com\",\n        \"data_selectors\": {\n            \"price\": \".price-text\",\n            \"area\": \".area-text\",\n            \"location\": \".location-text\",\n            \"type\": \".type-text\"\n        }\n    },\n    \"lianjia\": {\n        \"url\": \"https:\u002F\u002Fwww.lianjia.com\",\n        \"data_selectors\": {\n            \"price\": \".total-price\",\n  4         \"area\": \".area-num\",\n            \"location\": \".location-text\",\n            \"type\": \".house-type\"\n        }\n    },\n    \"soufun\": {\n        \"url\": \"https:\u002F\u002Fwww.soufun.com\",\n        \"data_selectors\": {\n            \"price\": \".price-num\",\n            \"area\": \".area-num\",\n            \"location\": \".location-text\",\n            \"type\": \".type-text\"\n        }\n    }\n}\n```\n\n### 通用爬虫模板\n\n```python\n# 通用爬虫脚本模板\nimport time\nimport random\nimport json\nfrom dataclasses import dataclass\n\n@dataclass\nclass PropertyData:\n    title: str\n    price: str\n    area: str\n    location: str\n    house_type: str\n    age: str\n    orientation: str\n    decoration: str\n\nclass RealEstateSpider:\n    def __init__(self, website_name):\n        self.website_name = website_name\n        self.config = CONFIG[website_name]\n        self.base_url = self.config[\"url\"]\n        self.selectors = self.config[\"data_selectors\"]\n        \n    def crawl(self, city=\"北京\", district=None):\n        \"\"\"爬取指定城市和区域的房产数据\"\"\"\n        # 构建URL\n        url = self.build_url(city, district)\n        \n        # 发送请求\n        data = self.send_request(url)\n        \n        # 解析数据\n        properties = self.parse_data(data)\n        \n        # 返回结果\n        return properties\n    \n    def build_url(self, city, district):\n        \"\"\"构建目标URL\"\"\"\n        if self.website_name == \"anjuke\":\n            return f\"{self.base_url}\u002Ffangyuan\u002F{city}\"\n        elif self.website_name == \"ke\":\n            return f\"{self.base_url}\u002Fcity\u002F{city}\"\n        elif self.website_name == \"lianjia\":\n            return f\"{self.base_url}\u002Fershoufang\u002F{city}\"\n        elif self.website_name == \"soufun\":\n            return f\"{self.base_url}\u002Fesf\u002F{city}\"\n        else:\n            return self.base_url\n    \n    def send_request(self, url):\n        \"\"\"发送请求，处理反爬虫\"\"\"\n        headers = {\n            'User-Agent': 'Mozilla\u002F5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\u002F537.36 (KHTML, like Gecko) Chrome\u002F146.0.0.0 Safari\u002F537.36',\n            'Accept': 'text\u002Fhtml,application\u002Fxhtml+xml,application\u002Fxml;q=0.9,image\u002Favif,image\u002Fwebp,image\u002Fapng,*\u002F*;q=0.8',\n            'Accept-Language': 'zh-CN,zh;q=0.9',\n            'Accept-Encoding': 'gzip, deflate, br',\n            'Connection': 'keep-alive',\n            'Cache-Control': 'no-cache',\n            'Upgrade-Insecure-Requests': '1'\n        }\n        \n        # 随机延迟避免频率检测\n        sleep_time = random.uniform(2, 5)\n        time.sleep(sleep_time)\n        \n        # 发送请求（此处为简化示例，实际需要根据网站调整）\n        import requests\n        response = requests.get(url, headers=headers)\n        return response.text\n    \n    def parse_data(self, html_data):\n        \"\"\"解析HTML数据\"\"\"\n        # 这里需要根据具体网站的HTML结构实现解析逻辑\n        properties = []\n        \n        # 示例解析逻辑\n        import re\n        pattern = r'\"price\":\"([\\d\\.]+)\",.*\"avg_price\":\"([\\d\\.]+)\",.*\"area_num\":\"([\\d\\.]+)\",.*\"house_age\":\"([\\d年]+)\",.*\"orient\":\"([^\"]+)\",.*\"fitment_name\":\"([^\"]+)\",.*\"title\":\"([^\"]+)\"'\n        matches = re.findall(pattern, html_data)\n        \n        for match in matches:\n            property = PropertyData(\n                title=match[6],\n                price=match[0],\n                area=match[2],\n                location=\"\",  # 需要根据网站调整\n                house_type=\"\",  # 需要根据网站调整\n                age=match[3],\n                orientation=match[4],\n                decoration=match[5]\n            )\n            properties.append(property)\n            \n        return properties\n    \n    def save_data(self, properties, format=\"json\"):\n        \"\"\"保存数据\"\"\"\n        if format == \"json\":\n            with open(f'{self.website_name}_properties.json', 'w', encoding='utf-8') as f:\n                json.dump([prop.__dict__ for prop in properties], f, ensure_ascii=False, indent=2)\n        elif format == \"csv\":\n            import csv\n            with open(f'{self.website_name}_properties.csv', 'w', newline='', encoding='utf-8') as f:\n                writer = csv.writer(f)\n                writer.writerow(['title', 'price', 'area', 'location', 'house_type', 'age', 'orientation', 'decoration'])\n                for prop in properties:\n                    writer.writerow([prop.title, prop.price, prop.area, prop.location, prop.house_type, prop.age, prop.orientation, prop.decoration])\n\nif __name__ == \"__main__\":\n    # 示例：爬取安居客数据\n    spider = RealEstateSpider(\"anjuke\")\n    properties = spider.crawl(city=\"南京\")\n    spider.save_data(properties, format=\"json\")\n```\n\n### 使用 agent-browser 进行浏览器自动化\n\n```bash\n# 使用 agent-browser 绕过JavaScript检测\nagent-browser set viewport 1920 1080\nagent-browser set headers '{\n    \"User-Agent\": \"Mozilla\u002F5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\u002F537.36 (KHTML, like Gecko) Chrome\u002F146.0.0.0 Safari\u002F537.36\",\n    \"Accept\": \"text\u002Fhtml,application\u002Fxhtml+xml,application\u002Fxml;q=0.9,image\u002Favif,image\u002Fwebp,image\u002Fapng,*\u002F*;q=0.8\",\n    \"Accept-Language\": \"zh-CN,zh;q=0.9,en;q=0.8\",\n    \"Accept-Encoding\": \"gzip, deflate, br\",\n    \"Cache-Control\": \"no-cache\",\n    \"Connection\": \"keep-alive\",\n    \"Upgrade-Insecure-Requests\": \"1\"\n}'\n\n# 访问房产网站\nagent-browser open \"https:\u002F\u002Fwww.anjuke.com\"\nagent-browser wait 3000\nagent-browser snapshot -i\n\n# 模拟人类浏览行为\nagent-browser scroll down 300\nagent-browser wait 1000\nagent-browser click \"link:二手房\"\nagent-browser wait 2000\n\n# 获取页面数据\nagent-browser snapshot -c --scope \".property-list\"\nagent-browser screenshot \"anjuke_properties.png\"\n```\n\n## 反爬虫策略\n\n### 1. 浏览器指纹伪装\n```bash\n# 设置设备指纹\nagent-browser set device \"iPhone 14\"\nagent-browser set geo 39.9042 116.4074  # 北京地理位置\nagent-browser set media dark\n```\n\n### 2. 会话管理\n```bash\n# 保存会话状态\nagent-browser cookies set \"session_id\" \"your_session_value\"\nagent-browser state save \"real_estate_session.json\"\n\n# 恢复会话状态\nagent-browser state load \"real_estate_session.json\"\n```\n\n### 3. 请求频率控制\n```python\nimport time\nimport random\n\n# 随机延迟\ndelay = random.uniform(3, 8)\ntime.sleep(delay)\n\n# 批次处理，每批次后长时间休息\nbatch_size = 10\nfor i in range(batch_size):\n    # 处理请求\n    # 小延迟\n    time.sleep(random.uniform(1, 2))\n    \n# 批次结束后大延迟\ntime.sleep(random.uniform(30, 60))\n```\n\n### 4. 代理IP轮换\n```python\n# 代理IP池示例\nproxy_pool = [\n    \"http:\u002F\u002Fproxy1.example.com:8080\",\n    \"http:\u002F\u002Fproxy2.example.com:8080\",\n    \"http:\u002F\u002Fproxy3.example.com:8080\",\n]\n\nimport requests\n\nfor url in urls_to_crawl:\n    proxy = proxy_pool[i % len(proxy_pool)]\n    response = requests.get(url, proxies={\"http\": proxy})\n```\n\n## 验证码处理\n\n### 手动处理验证码\n```bash\n# 遇到验证码时暂停脚本，人工处理\necho \"验证码出现，请手动处理！\"\nagent-browser screenshot \"captcha.png\"\nagent-browser pause\n# 手动处理后继续\nagent-browser continue\n```\n\n### 验证码识别服务（可选）\n```python\n# 使用第三方验证码识别服务（需要API密钥）\nimport requests\n\ndef solve_captcha(image_path):\n    # 上传验证码图片到识别服务\n    response = requests.post(\n        \"https:\u002F\u002Fcaptcha.service.com\u002Fapi\u002Fsolve\",\n        files={\"image\": open(image_path, \"rb\")}\n    )\n    return response.json()[\"solution\"]\n```\n\n## 数据导出与分析\n\n```python\n# 数据分析示例\nimport json\nimport pandas as pd\n\n# 加载数据\nwith open('anjuke_properties.json', 'r', encoding='utf-8') as f:\n    data = json.load(f)\n\n# 转换为DataFrame\ndf = pd.DataFrame(data)\n\n# 统计分析\navg_price = df['price'].mean()\navg_area = df['area'].mean()\nprice_range = df['price'].min(), df['price'].max()\n\nprint(f\"平均价格: {avg_price}\")\nprint(f\"平均面积: {avg_area}\")\nprint(f\"价格范围: {price_range}\")\n```\n\n## 常见问题与解决方案\n\n### 1. 网站封锁\n- **解决方案**: 使用代理IP轮换\n- **解决方案**: 降低请求频率\n- **解决方案**: 更换用户代理\n\n### 2. 验证码频繁出现\n- **解决方案**: 增加随机延迟\n- **解决方案**: 模拟真实用户行为（滚动、点击）\n- **解决方案**: 使用验证码识别服务\n\n### 3. 数据提取失败\n- **解决方案**: 更新HTML选择器\n- **解决方案**: 检查网站结构变化\n- **解决方案**: 使用备用解析方法\n\n### 4. JavaScript渲染问题\n- **解决方案**: 使用agent-browser等待页面完全加载\n- **解决方案**: 等待网络空闲状态\n- **解决方案**: 截图检查页面状态\n\n## 法律与伦理注意事项\n\n### 使用本技能时请遵守：\n1. **网站条款**: 遵守房产网站的爬虫政策\n2. **数据使用**: 仅用于合法目的\n3. **频率控制**: 避免影响网站正常运营\n4. **数据保护**: 保护用户隐私信息\n\n### 建议使用频率：\n- 每批次不超过100个请求\n- 每小时不超过1000个请求\n- 每天不超过5000个请求\n\n## 持续改进\n\n本技能会根据房产网站的反爬虫策略更新，持续改进爬取方法和数据提取逻辑。\n\n### 更新日志：\n- 2026-03-26: 创建技能，包含通用房产爬虫框架\n- 后续更新: 将根据网站变化调整选择器和反爬虫策略","# 房产中介网站爬虫技能\n\n## 简介\n\n本技能专门用于爬取中国主流房产中介网站数据，包括：\n1. 安居客 (anjuke.com)\n2. 搜房网 (soufun.com)\n3. 贝壳找房 (ke.com)\n4. 链家 (lianjia.com)\n\n## 前置要求\n\n- Python 3.x\n- agent-browser 技能已安装\n- requests 库 (可以通过 pip 安装)\n\n## 安装依赖\n\n```bash\n# 安装 Python requests 库\npip install requests beautifulsoup4 lxml\n```\n\n## 主要功能\n\n### 1. 反爬虫绕过策略\n- 模拟真实浏览器指纹\n- 随机延迟避免频率检测\n- Cookie 和会话管理\n- 代理 IP 支持（可选）\n- 验证码处理机制\n\n### 2. 数据提取功能\n- 提取房价信息\n- 提取房产面积\n- 提取地理位置\n- 提取户型信息\n- 提取装修状态\n- 提取建筑年代\n\n### 3. 导出格式\n- JSON 格式\n- CSV 格式\n- Excel 格式\n- 可视化图表\n\n## 使用方法\n\n### 基本爬虫脚本\n\n```bash\n# 使用 Python 脚本爬取安居客数据\npython3 ~\u002F.openclaw\u002Fworkspace\u002Fskills\u002Freal-estate-spider\u002Fscripts\u002Fanjuke_crawler.py\n\n# 使用 Shell 脚本配合 agent-browser\nbash ~\u002F.openclaw\u002Fworkspace\u002Fskills\u002Freal-estate-spider\u002Fscripts\u002Fbypass_anjuke.sh\n```\n\n### 配置网站选择\n\n```python\n# 配置文件示例\n# ~\u002F.openclaw\u002Fworkspace\u002Fskills\u002Freal-estate-spider\u002Fconfig\u002Freal_estate_config.py\n\nimport json\n\nCONFIG = {\n    \"anjuke\": {\n        \"url\": \"https:\u002F\u002Fwww.anjuke.com\",\n        \"data_selectors\": {\n            \"price\": \".property-price\",\n            \"area\": \".property-area\",\n            \"location\": \".property-location\",\n            \"type\": \".property-type\"\n        }\n    },\n    \"ke\": {\n        \"url\": \"https:\u002F\u002Fke.com\",\n        \"data_selectors\": {\n            \"price\": \".price-text\",\n            \"area\": \".area-text\",\n            \"location\": \".location-text\",\n            \"type\": \".type-text\"\n        }\n    },\n    \"lianjia\": {\n        \"url\": \"https:\u002F\u002Fwww.lianjia.com\",\n        \"data_selectors\": {\n            \"price\": \".total-price\",\n  4         \"area\": \".area-num\",\n            \"location\": \".location-text\",\n            \"type\": \".house-type\"\n        }\n    },\n    \"soufun\": {\n        \"url\": \"https:\u002F\u002Fwww.soufun.com\",\n        \"data_selectors\": {\n            \"price\": \".price-num\",\n            \"area\": \".area-num\",\n            \"location\": \".location-text\",\n            \"type\": \".type-text\"\n        }\n    }\n}\n```\n\n### 通用爬虫模板\n\n```python\n# 通用爬虫脚本模板\nimport time\nimport random\nimport json\nfrom dataclasses import dataclass\n\n@dataclass\nclass PropertyData:\n    title: str\n    price: str\n    area: str\n    location: str\n    house_type: str\n    age: str\n    orientation: str\n    decoration: str\n\nclass RealEstateSpider:\n    def __init__(self, website_name):\n        self.website_name = website_name\n        self.config = CONFIG[website_name]\n        self.base_url = self.config[\"url\"]\n        self.selectors = self.config[\"data_selectors\"]\n        \n    def crawl(self, city=\"北京\", district=None):\n        \"\"\"爬取指定城市和区域的房产数据\"\"\"\n        # 构建URL\n        url = self.build_url(city, district)\n        \n        # 发送请求\n        data = self.send_request(url)\n        \n        # 解析数据\n        properties = self.parse_data(data)\n        \n        # 返回结果\n        return properties\n    \n    def build_url(self, city, district):\n        \"\"\"构建目标URL\"\"\"\n        if self.website_name == \"anjuke\":\n            return f\"{self.base_url}\u002Ffangyuan\u002F{city}\"\n        elif self.website_name == \"ke\":\n            return f\"{self.base_url}\u002Fcity\u002F{city}\"\n        elif self.website_name == \"lianjia\":\n            return f\"{self.base_url}\u002Fershoufang\u002F{city}\"\n        elif self.website_name == \"soufun\":\n            return f\"{self.base_url}\u002Fesf\u002F{city}\"\n        else:\n            return self.base_url\n    \n    def send_request(self, url):\n        \"\"\"发送请求，处理反爬虫\"\"\"\n        headers = {\n            'User-Agent': 'Mozilla\u002F5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\u002F537.36 (KHTML, like Gecko) Chrome\u002F146.0.0.0 Safari\u002F537.36',\n            'Accept': 'text\u002Fhtml,application\u002Fxhtml+xml,application\u002Fxml;q=0.9,image\u002Favif,image\u002Fwebp,image\u002Fapng,*\u002F*;q=0.8',\n            'Accept-Language': 'zh-CN,zh;q=0.9',\n            'Accept-Encoding': 'gzip, deflate, br',\n            'Connection': 'keep-alive',\n            'Cache-Control': 'no-cache',\n            'Upgrade-Insecure-Requests': '1'\n        }\n        \n        # 随机延迟避免频率检测\n        sleep_time = random.uniform(2, 5)\n        time.sleep(sleep_time)\n        \n        # 发送请求（此处为简化示例，实际需要根据网站调整）\n        import requests\n        response = requests.get(url, headers=headers)\n        return response.text\n    \n    def parse_data(self, html_data):\n        \"\"\"解析HTML数据\"\"\"\n        # 这里需要根据具体网站的HTML结构实现解析逻辑\n        properties = []\n        \n        # 示例解析逻辑\n        import re\n        pattern = r'\"price\":\"([\\d\\.]+)\",.*\"avg_price\":\"([\\d\\.]+)\",.*\"area_num\":\"([\\d\\.]+)\",.*\"house_age\":\"([\\d年]+)\",.*\"orient\":\"([^\"]+)\",.*\"fitment_name\":\"([^\"]+)\",.*\"title\":\"([^\"]+)\"'\n        matches = re.findall(pattern, html_data)\n        \n        for match in matches:\n            property = PropertyData(\n                title=match[6],\n                price=match[0],\n                area=match[2],\n                location=\"\",  # 需要根据网站调整\n                house_type=\"\",  # 需要根据网站调整\n                age=match[3],\n                orientation=match[4],\n                decoration=match[5]\n            )\n            properties.append(property)\n            \n        return properties\n    \n    def save_data(self, properties, format=\"json\"):\n        \"\"\"保存数据\"\"\"\n        if format == \"json\":\n            with open(f'{self.website_name}_properties.json', 'w', encoding='utf-8') as f:\n                json.dump([prop.__dict__ for prop in properties], f, ensure_ascii=False, indent=2)\n        elif format == \"csv\":\n            import csv\n            with open(f'{self.website_name}_properties.csv', 'w', newline='', encoding='utf-8') as f:\n                writer = csv.writer(f)\n                writer.writerow(['title', 'price', 'area', 'location', 'house_type', 'age', 'orientation', 'decoration'])\n                for prop in properties:\n                    writer.writerow([prop.title, prop.price, prop.area, prop.location, prop.house_type, prop.age, prop.orientation, prop.decoration])\n\nif __name__ == \"__main__\":\n    # 示例：爬取安居客数据\n    spider = RealEstateSpider(\"anjuke\")\n    properties = spider.crawl(city=\"南京\")\n    spider.save_data(properties, format=\"json\")\n```\n\n### 使用 agent-browser 进行浏览器自动化\n\n```bash\n# 使用 agent-browser 绕过JavaScript检测\nagent-browser set viewport 1920 1080\nagent-browser set headers '{\n    \"User-Agent\": \"Mozilla\u002F5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\u002F537.36 (KHTML, like Gecko) Chrome\u002F146.0.0.0 Safari\u002F537.36\",\n    \"Accept\": \"text\u002Fhtml,application\u002Fxhtml+xml,application\u002Fxml;q=0.9,image\u002Favif,image\u002Fwebp,image\u002Fapng,*\u002F*;q=0.8\",\n    \"Accept-Language\": \"zh-CN,zh;q=0.9,en;q=0.8\",\n    \"Accept-Encoding\": \"gzip, deflate, br\",\n    \"Cache-Control\": \"no-cache\",\n    \"Connection\": \"keep-alive\",\n    \"Upgrade-Insecure-Requests\": \"1\"\n}'\n\n# 访问房产网站\nagent-browser open \"https:\u002F\u002Fwww.anjuke.com\"\nagent-browser wait 3000\nagent-browser snapshot -i\n\n# 模拟人类浏览行为\nagent-browser scroll down 300\nagent-browser wait 1000\nagent-browser click \"link:二手房\"\nagent-browser wait 2000\n\n# 获取页面数据\nagent-browser snapshot -c --scope \".property-list\"\nagent-browser screenshot \"anjuke_properties.png\"\n```\n\n## 反爬虫策略\n\n### 1. 浏览器指纹伪装\n```bash\n# 设置设备指纹\nagent-browser set device \"iPhone 14\"\nagent-browser set geo 39.9042 116.4074  # 北京地理位置\nagent-browser set media dark\n```\n\n### 2. 会话管理\n```bash\n# 保存会话状态\nagent-browser cookies set \"session_id\" \"your_session_value\"\nagent-browser state save \"real_estate_session.json\"\n\n# 恢复会话状态\nagent-browser state load \"real_estate_session.json\"\n```\n\n### 3. 请求频率控制\n```python\nimport time\nimport random\n\n# 随机延迟\ndelay = random.uniform(3, 8)\ntime.sleep(delay)\n\n# 批次处理，每批次后长时间休息\nbatch_size = 10\nfor i in range(batch_size):\n    # 处理请求\n    # 小延迟\n    time.sleep(random.uniform(1, 2))\n    \n# 批次结束后大延迟\ntime.sleep(random.uniform(30, 60))\n```\n\n### 4. 代理IP轮换\n```python\n# 代理IP池示例\nproxy_pool = [\n    \"http:\u002F\u002Fproxy1.example.com:8080\",\n    \"http:\u002F\u002Fproxy2.example.com:8080\",\n    \"http:\u002F\u002Fproxy3.example.com:8080\",\n]\n\nimport requests\n\nfor url in urls_to_crawl:\n    proxy = proxy_pool[i % len(proxy_pool)]\n    response = requests.get(url, proxies={\"http\": proxy})\n```\n\n## 验证码处理\n\n### 手动处理验证码\n```bash\n# 遇到验证码时暂停脚本，人工处理\necho \"验证码出现，请手动处理！\"\nagent-browser screenshot \"captcha.png\"\nagent-browser pause\n# 手动处理后继续\nagent-browser continue\n```\n\n### 验证码识别服务（可选）\n```python\n# 使用第三方验证码识别服务（需要API密钥）\nimport requests\n\ndef solve_captcha(image_path):\n    # 上传验证码图片到识别服务\n    response = requests.post(\n        \"https:\u002F\u002Fcaptcha.service.com\u002Fapi\u002Fsolve\",\n        files={\"image\": open(image_path, \"rb\")}\n    )\n    return response.json()[\"solution\"]\n```\n\n## 数据导出与分析\n\n```python\n# 数据分析示例\nimport json\nimport pandas as pd\n\n# 加载数据\nwith open('anjuke_properties.json', 'r', encoding='utf-8') as f:\n    data = json.load(f)\n\n# 转换为DataFrame\ndf = pd.DataFrame(data)\n\n# 统计分析\navg_price = df['price'].mean()\navg_area = df['area'].mean()\nprice_range = df['price'].min(), df['price'].max()\n\nprint(f\"平均价格: {avg_price}\")\nprint(f\"平均面积: {avg_area}\")\nprint(f\"价格范围: {price_range}\")\n```\n\n## 常见问题与解决方案\n\n### 1. 网站封锁\n- **解决方案**: 使用代理IP轮换\n- **解决方案**: 降低请求频率\n- **解决方案**: 更换用户代理\n\n### 2. 验证码频繁出现\n- **解决方案**: 增加随机延迟\n- **解决方案**: 模拟真实用户行为（滚动、点击）\n- **解决方案**: 使用验证码识别服务\n\n### 3. 数据提取失败\n- **解决方案**: 更新HTML选择器\n- **解决方案**: 检查网站结构变化\n- **解决方案**: 使用备用解析方法\n\n### 4. JavaScript渲染问题\n- **解决方案**: 使用agent-browser等待页面完全加载\n- **解决方案**: 等待网络空闲状态\n- **解决方案**: 截图检查页面状态\n\n## 法律与伦理注意事项\n\n### 使用本技能时请遵守：\n1. **网站条款**: 遵守房产网站的爬虫政策\n2. **数据使用**: 仅用于合法目的\n3. **频率控制**: 避免影响网站正常运营\n4. **数据保护**: 保护用户隐私信息\n\n### 建议使用频率：\n- 每批次不超过100个请求\n- 每小时不超过1000个请求\n- 每天不超过5000个请求\n\n## 持续改进\n\n本技能会根据房产网站的反爬虫策略更新，持续改进爬取方法和数据提取逻辑。\n\n### 更新日志：\n- 2026-03-26: 创建技能，包含通用房产爬虫框架\n- 后续更新: 将根据网站变化调整选择器和反爬虫策略",{"name":5,"metadata":24,"read_when":31,"description":7,"allowed-tools":35},{"clawdbot":25},{"emoji":26,"requires":27},"🏠",{"bins":28},[29,30],"python3","agent-browser",[32,33,34],"需要爬取房产中介网站数据","需要绕过网站反爬虫机制","需要提取房产信息（价格、面积、位置、户型等）","Bash(agent-browser:*)","master","4aa004da461d820f4ea5499e58eb8d196cf99016","community\u002Fclawhub\u002Fr\u002Freal-estate-spider",0,"https:\u002F\u002Fgithub.com\u002FLord1Egypt\u002Fawesome-skill-forge","https:\u002F\u002Fskill-explorer-rho.vercel.app","https:\u002F\u002Fgithub.com\u002FLord1Egypt","Unbreakable. Unstoppable. Unmatched⚡.\r\n Speaker of machine tongues 🤖  | World-builder of decentralized dreams 🌐 | Defender of the digital realm🛡️.","2026-06-10 08:21:01",{"source":8,"skillName":5,"commands":46,"prompt":56,"zipAvailable":57},[47,50,53],{"tool":48,"command":49},"npx","npx skills add Lord1Egypt\u002Fawesome-skill-forge --skill \"Real Estate Spider\"",{"tool":51,"command":52},"bunx","bunx skills add Lord1Egypt\u002Fawesome-skill-forge --skill \"Real Estate Spider\"",{"tool":54,"command":55},"pnpm","pnpm dlx skills add Lord1Egypt\u002Fawesome-skill-forge --skill \"Real Estate Spider\"","安装技能，技能地址：https:\u002F\u002Fgithub.com\u002FLord1Egypt\u002Fawesome-skill-forge\u002Fblob\u002Fmaster\u002Fcommunity\u002Fclawhub\u002Fr\u002Freal-estate-spider\u002FSKILL.md",true,[],[60,67,74,81,88,95],{"skillNo":61,"name":62,"nameZh":6,"description":63,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":64,"categories":65,"keywords":66,"publishedAt":19},"01M785WKJB99G3STR1NF9NRTJW","youtube-content","YouTube transcripts to summaries, threads, blogs.","https:\u002F\u002Fgithub.com\u002FLord1Egypt\u002Fawesome-skill-forge\u002Fblob\u002Fmaster\u002Fskills\u002Fyoutube-content\u002FSKILL.md",[],[],{"skillNo":68,"name":69,"nameZh":6,"description":70,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":71,"categories":72,"keywords":73,"publishedAt":19},"01M785WEFHB0G2S9YA6EPR3H0E","xurl","X\u002FTwitter via xurl CLI: post, search, DM, media, v2 API.","https:\u002F\u002Fgithub.com\u002FLord1Egypt\u002Fawesome-skill-forge\u002Fblob\u002Fmaster\u002Fskills\u002Fxurl\u002FSKILL.md",[],[],{"skillNo":75,"name":76,"nameZh":6,"description":77,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":78,"categories":79,"keywords":80,"publishedAt":19},"01M785W98EDN4EFJXJPMQEHCRC","touchdesigner-mcp","Control a running TouchDesigner instance via twozero MCP — create operators, set parameters, wire connections, execute Python, build real-time visuals. 36 native tools.","https:\u002F\u002Fgithub.com\u002FLord1Egypt\u002Fawesome-skill-forge\u002Fblob\u002Fmaster\u002Fskills\u002Ftouchdesigner-mcp\u002FSKILL.md",[],[],{"skillNo":82,"name":83,"nameZh":6,"description":84,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":85,"categories":86,"keywords":87,"publishedAt":19},"01M785W5S9FP3QBH2V191E025P","test-driven-development","TDD: enforce RED-GREEN-REFACTOR, tests before code.","https:\u002F\u002Fgithub.com\u002FLord1Egypt\u002Fawesome-skill-forge\u002Fblob\u002Fmaster\u002Fskills\u002Ftest-driven-development\u002FSKILL.md",[],[],{"skillNo":89,"name":90,"nameZh":6,"description":91,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":92,"categories":93,"keywords":94,"publishedAt":19},"01M785W2AS08NSSH9YATCV6FJB","teams-meeting-pipeline","Operate the Teams meeting summary pipeline via Hermes CLI — summarize meetings, inspect pipeline status, replay jobs, manage Microsoft Graph subscriptions.","https:\u002F\u002Fgithub.com\u002FLord1Egypt\u002Fawesome-skill-forge\u002Fblob\u002Fmaster\u002Fskills\u002Fteams-meeting-pipeline\u002FSKILL.md",[],[],{"skillNo":96,"name":97,"nameZh":6,"description":98,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":99,"categories":100,"keywords":101,"publishedAt":19},"01M785VXKW4T34BTP1SCHW1R7F","systematic-debugging","4-phase root cause debugging: understand bugs before fixing.","https:\u002F\u002Fgithub.com\u002FLord1Egypt\u002Fawesome-skill-forge\u002Fblob\u002Fmaster\u002Fskills\u002Fsystematic-debugging\u002FSKILL.md",[],[]]