(async function crawlAll() { const allNames new Set(); const totalPages 19; // 已知一共19页 const baseUrl window.location.origin /xiaoqu/xuanwu/; for (let page 1; page totalPages; page) { try { // 构建第 N 页的 URL const url page 1 ? baseUrl : ${baseUrl}pg${page}; console.log(正在抓取第 ${page} 页...); // 模拟请求 const response await fetch(url); if (!response.ok) throw new Error(请求失败); const text await response.text(); const parser new DOMParser(); const doc parser.parseFromString(text, text/html); // 提取小区名 const links doc.querySelectorAll(.xiaoquListItem .title a); links.forEach(link { const name link.title.trim(); if (name) allNames.add(name); }); // 延时防封 await new Promise(resolve setTimeout(resolve, 1000)); } catch (e) { console.log(第 ${page} 页跳过或失败:, e.message); } } // 一次性输出所有 const finalList Array.from(allNames).join(\n); console.log(\n 抓取完成以下是完整列表\n); console.log(finalList); // 一行一个完整输出 })();获取单页的小区名称// 获取所有小区列表项 const items document.querySelectorAll(.xiaoquListItem); // 遍历并提取小区名称 const names []; items.forEach(item { // 从 title 属性或文本获取小区名 const name item.querySelector(.title a).title.trim(); names.push(name); }); // 一行一个输出 console.log(names.join(\n)); // 同时返回结果方便复制 names;自动点击小区列表下一页 获取小区名单导出excel// 贝壳小区采集器 v18浏览器控制台版 // 使用在贝壳“小区列表页”打开开发者工具把本文件完整粘贴到 Console 后回车。 // 说明同源 fetch 后台取页不刷新当前页面以小区 ID 去重并保存到 localStorage。 (function () { use strict; const APP_KEY __bk_xiaoqu_v18__; const STORAGE_KEY bk_xiaoqu_data_v18; const CONFIG_KEY bk_xiaoqu_config_v18; const old window[APP_KEY]; if (old typeof old.destroy function) old.destroy(); const state { running: false, page: 0, maxPage: 0, failCount: 0, captchaCount: 0, verifyWindow: null, controller: null, activeFrame: null, data: new Map(), host: null, root: null }; const sleep (ms) new Promise((resolve) setTimeout(resolve, ms)); const textOf (el) (el (el.innerText || el.textContent) || ).trim(); const intOf (value) parseInt(String(value || ).replace(/[^0-9]/g, ), 10) || 0; function loadConfig() { try { return Object.assign({ captchaSeconds: 10, pageMinMs: 1200, pageMaxMs: 2600 }, JSON.parse(localStorage.getItem(CONFIG_KEY) || {})); } catch (_) { return { captchaSeconds: 10, pageMinMs: 1200, pageMaxMs: 2600 }; } } const config loadConfig(); function loadData() { const keys [STORAGE_KEY, bk_xiaoqu_data_v17, bk_xiaoqu_data_v16]; for (const key of keys) { try { const list JSON.parse(localStorage.getItem(key) || []); if (!Array.isArray(list) || !list.length) continue; for (const item of list) { if (item item.resblockId) state.data.set(String(item.resblockId), item); } break; } catch (_) {} } } function saveData() { try { localStorage.setItem(STORAGE_KEY, JSON.stringify(Array.from(state.data.values()))); } catch (error) { log(本地保存失败 error.message, err); } } function isListPage() { return /^\/xiaoqu(?:\/|$)/.test(location.pathname); } function baseListPath() { const parts location.pathname.split(/).filter(Boolean).filter((part) !/^pg\d$/i.test(part)); return / parts.join(/) /; } function pageUrl(pageNo) { const path baseListPath(); return new URL(path (pageNo 1 ? pg pageNo / : ) location.search, location.origin).href; } function currentPageNo() { const match location.pathname.match(/\/pg(\d)(?:\/|$)/i); return match ? Number(match[1]) || 1 : 1; } function maxPageOf(doc) { try { const el doc.querySelector([page-data]); if (el) { const value JSON.parse(el.getAttribute(page-data) || {}); const n Number(value.totalPage); if (n 0) return n; } } catch (_) {} const nums Array.from(doc.querySelectorAll(.page-box a, .house-lst-page-box a)) .map((a) intOf(textOf(a))).filter((n) n 0); return nums.length ? Math.max(...nums) : 1; } function isCaptcha(doc, rawHtml) { if (doc doc.querySelector(#captcha, .geetest_captcha, .geetest_holder, [class*geetest_])) return true; const html String(rawHtml || ).slice(0, 200000); return /geetest_captcha|geetest_holder|BlockedScreen|verify_code|captcha_v4|点击按钮开始验证/i.test(html); } function parseItems(doc) { const result []; for (const li of doc.querySelectorAll(.listContent li)) { try { const nameEl li.querySelector(.info .title a, .title a); if (!nameEl) continue; const rawUrl nameEl.getAttribute(href) || ; const url rawUrl ? new URL(rawUrl, location.origin).href : ; const urlMatch url.match(/\/xiaoqu\/(\d)(?:\.html)?\/?(?:[?#]|$)/i); const resblockId String(li.dataset.id || li.dataset.housecode || (urlMatch urlMatch[1]) || ); if (!resblockId) continue; const links li.querySelectorAll(.positionInfo a); const houseInfo textOf(li.querySelector(.houseInfo, .sub)); const saleMatch houseInfo.match(/90\s*天成交\s*(\d)\s*套/); const rentMatch houseInfo.match(/(\d)\s*套正在出租/); result.push({ resblockId, name: textOf(nameEl), district: textOf(li.querySelector(.positionInfo .district)) || textOf(links[0]), bizcircle: textOf(li.querySelector(.positionInfo .bizcircle)) || textOf(links[1]), price: intOf(textOf(li.querySelector(.unitPrice span, .totalPrice span))), sale90: saleMatch ? Number(saleMatch[1]) : 0, rentCount: rentMatch ? Number(rentMatch[1]) : 0, sellCount: intOf(textOf(li.querySelector(.totalSellCount span))), tag: Array.from(li.querySelectorAll(.tagList span)).map(textOf).filter(Boolean).join(), url }); } catch (error) { log(单条解析失败 error.message, warn); } } return result; } // 不能用 fetch贝壳触发风控时会跨域跳到 hip.ke.com/captcha // 带 cookie 的跨域重定向会被 CORS 拦截JS 只能得到 Failed to fetch。 // iframe 导航不受 fetch CORS 限制跳到跨域页后无法读取 document正好可据此识别验证码。 function fetchPage(pageNo) { const url pageUrl(pageNo); return new Promise((resolve) { const frame document.createElement(iframe); state.activeFrame frame; frame.style.cssText position:fixed;left:-10000px;top:-10000px;width:1200px;height:800px;border:0;visibility:hidden; let finished false; const finish (value) { if (finished) return; finished true; clearTimeout(timer); try { frame.remove(); } catch (_) {} if (state.activeFrame frame) state.activeFrame null; resolve(value); }; const timer setTimeout(() { finish({ kind: error, url, message: 后台页面加载超时 }); }, 35000); frame.onload () { setTimeout(() { if (finished) return; try { const finalUrl frame.contentWindow.location.href; const doc frame.contentDocument || frame.contentWindow.document; if (!doc) { finish({ kind: error, url, message: 后台页面 document 为空 }); return; } if (isCaptcha(doc, doc.documentElement doc.documentElement.outerHTML)) { finish({ kind: captcha, url: finalUrl || url }); return; } const items parseItems(doc); if (!items.length) { finish({ kind: error, url, message: 页面没有找到 .listContent li }); return; } finish({ kind: ok, url, doc, items }); } catch (error) { // wh.ke.com - hip.ke.com/captcha 后读取 iframe 会抛 SecurityError。 finish({ kind: captcha, url, crossOrigin: true }); } }, 600); }; frame.onerror () finish({ kind: error, url, message: 后台页面加载失败 }); frame.src url; document.body.appendChild(frame); }); } async function interruptibleWait(ms) { const end Date.now() ms; while (state.running Date.now() end) await sleep(Math.min(250, end - Date.now())); } function openVerifyPage(url) { try { if (!state.verifyWindow || state.verifyWindow.closed) state.verifyWindow window.open(url, bk_xiaoqu_verify); else { state.verifyWindow.location.href url; state.verifyWindow.focus(); } if (!state.verifyWindow) log(验证标签被浏览器拦截请点击“打开验证页”按钮。, warn); } catch (error) { log(无法打开验证页 error.message, warn); } } function insertItems(items) { let added 0; let duplicate 0; for (const item of items) { const key String(item.resblockId); if (state.data.has(key)) duplicate; else { state.data.set(key, item); added; } } saveData(); return { added, duplicate }; } async function start() { if (state.running) return; if (!isListPage()) { setStatus(当前不是贝壳小区列表页, err); return; } readSettings(); state.running true; state.failCount 0; state.captchaCount 0; state.maxPage maxPageOf(document); let page 1; updateUi(); log(开始采集共 state.maxPage 页, ok); while (page state.maxPage state.running) { state.page page; setStatus(后台请求第 page / state.maxPage 页, ok); updateUi(); const result await fetchPage(page); if (!state.running) break; if (result.kind captcha) { state.captchaCount; updateUi(); setStatus(验证码暂停在第 page 页每 config.captchaSeconds 秒重试, warn); log(第 page 页出现验证码不会跳页。请在验证标签完成验证。, warn); openVerifyPage(result.url); await interruptibleWait(config.captchaSeconds * 1000); continue; } if (result.kind error) { state.failCount; updateUi(); log(第 page 页请求失败 result.message, err); if (state.failCount 3) { state.running false; setStatus(连续失败 3 次已停在第 page 页, err); break; } await interruptibleWait(3000); continue; } state.failCount 0; const count insertItems(result.items); if (page 1) state.maxPage Math.max(state.maxPage, maxPageOf(result.doc)); log(第 page 页新增 count.added 重复 count.duplicate 累计 state.data.size, ok); updateUi(); page; if (page state.maxPage) { const delay config.pageMinMs Math.floor(Math.random() * Math.max(1, config.pageMaxMs - config.pageMinMs)); await interruptibleWait(delay); } } const completed page state.maxPage; state.running false; updateUi(); if (completed) { setStatus(采集完成共 state.data.size 条, ok); log(全部采集完成共 state.data.size 条。, ok); } } function stop() { state.running false; if (state.controller) state.controller.abort(); if (state.activeFrame) { try { state.activeFrame.remove(); } catch (_) {} state.activeFrame null; } setStatus(已停止, warn); log(用户停止采集。, warn); } function csvCell(value) { const s String(value null ? : value); return s.replace(//g, ) ; } function exportCsv() { const header [小区ID, 小区名称, 行政区, 商圈, 均价(元/㎡), 90天成交(套), 正在出租(套), 在售二手房(套), 标签, URL]; const rows [header]; for (const o of state.data.values()) rows.push([o.resblockId, o.name, o.district, o.bizcircle, o.price, o.sale90, o.rentCount, o.sellCount, o.tag, o.url]); const csv \uFEFF rows.map((row) row.map(csvCell).join(,)).join(\r\n); const blobUrl URL.createObjectURL(new Blob([csv], { type: text/csv;charsetutf-8 })); const a document.createElement(a); a.href blobUrl; a.download 贝壳小区_ new Date().toISOString().slice(0, 19).replace(/[T:]/g, -) .csv; a.click(); setTimeout(() URL.revokeObjectURL(blobUrl), 1000); log(已导出 state.data.size 条Excel 可直接打开。, ok); } function clearData() { if (!confirm(确定清空已保存的 state.data.size 条数据吗)) return; state.data.clear(); saveData(); updateUi(); log(已清空数据。, warn); } function readSettings() { const input state.root.getElementById(captchaSeconds); config.captchaSeconds Math.max(2, Math.min(300, Number(input.value) || 10)); input.value String(config.captchaSeconds); localStorage.setItem(CONFIG_KEY, JSON.stringify(config)); } function log(message, type) { const box state.root state.root.getElementById(log); const line [ new Date().toLocaleTimeString() ] message; console[type err ? error : type warn ? warn : log]([贝壳采集器] message); if (!box) return; const div document.createElement(div); div.className type || ; div.textContent line; box.appendChild(div); while (box.children.length 250) box.firstChild.remove(); box.scrollTop box.scrollHeight; } function setStatus(message, type) { const el state.root state.root.getElementById(status); if (el) { el.textContent message; el.className type || ; } } function updateUi() { if (!state.root) return; state.root.getElementById(page).textContent state.page / (state.maxPage || ?); state.root.getElementById(count).textContent state.data.size; state.root.getElementById(captchaCount).textContent state.captchaCount; state.root.getElementById(failCount).textContent state.failCount; state.root.getElementById(start).disabled state.running; state.root.getElementById(stop).disabled !state.running; } function buildUi() { const host document.createElement(div); host.style.cssText position:fixed;right:12px;top:12px;width:330px;z-index:2147483647; const root host.attachShadow({ mode: open }); root.innerHTML style *{box-sizing:border-box} .panel{font:13px/1.45 -apple-system,BlinkMacSystemFont,Segoe UI,Microsoft YaHei,sans-serif;color:#eef2ff;background:#172554;border:1px solid #3b82f6;border-radius:10px;padding:12px;box-shadow:0 8px 28px #0008} h3{font-size:15px;margin:0 0 8px}.stats{display:grid;grid-template-columns:1fr 1fr;gap:5px;background:#ffffff12;padding:7px;border-radius:6px}.stats b{color:#fbbf24} .setting{display:flex;align-items:center;gap:6px;margin:8px 0}.setting input{width:64px;padding:4px;border-radius:4px;border:1px solid #94a3b8} .buttons{display:grid;grid-template-columns:1fr 1fr;gap:6px;margin-top:7px}button{border:0;border-radius:5px;padding:7px 5px;color:white;background:#2563eb;cursor:pointer}button:disabled{opacity:.45;cursor:not-allowed}#stop{background:#dc2626}#verify{background:#d97706}#clear{background:#64748b} #status{margin:8px 0;padding:6px;background:#0004;border-radius:5px}.ok{color:#86efac}.warn{color:#fde047}.err{color:#fca5a5} #log{height:155px;overflow:auto;background:#0f172a;padding:6px;border-radius:5px;font:11px/1.45 Consolas,monospace}.tip{color:#cbd5e1;font-size:11px;margin-top:7px} /style div classpanel h3 贝壳小区采集器 v18/h3 div classstatsspan页码b idpage0/?/b/spanspan已采b idcount0/b/spanspan验证码b idcaptchaCount0/b/spanspan连续失败b idfailCount0/b/span/div label classsetting验证码重试间隔 input idcaptchaSeconds typenumber min2 max300 value10 秒/label div idstatus初始化.../div div classbuttonsbutton idstart▶ 从第1页开始/buttonbutton idstop⏹ 停止/buttonbutton idverify 打开验证页/buttonbutton idexport 导出Excel(CSV)/buttonbutton idparse 解析当前页/buttonbutton idclear 清空数据/button/div div classtip后台 fetch不刷新本页按小区 ID 去重。验证码通过后会自动重试原页。/div div idlog/div /div; document.body.appendChild(host); state.host host; state.root root; root.getElementById(captchaSeconds).value String(config.captchaSeconds); root.getElementById(start).onclick start; root.getElementById(stop).onclick stop; root.getElementById(verify).onclick () openVerifyPage(pageUrl(state.page || currentPageNo())); root.getElementById(export).onclick exportCsv; root.getElementById(clear).onclick clearData; root.getElementById(parse).onclick () { const count insertItems(parseItems(document)); log(当前页新增 count.added 重复 count.duplicate, ok); updateUi(); }; root.getElementById(captchaSeconds).onchange readSettings; } function destroy() { stop(); if (state.host) state.host.remove(); } loadData(); buildUi(); state.page currentPageNo(); state.maxPage maxPageOf(document); updateUi(); setStatus(isListPage() ? 就绪 : 请先打开贝壳小区列表页, isListPage() ? ok : err); log(已加载历史数据 state.data.size 条当前检测到 state.maxPage 页。, ok); window[APP_KEY] { destroy, state }; })();导入到excel 里面// UserScript // name 贝壳小区采集【暴力猴兼容版】 // namespace http://tampermonkey.net/ // version 1.5 // description 兼容Violentmonkey修复悬浮窗不出现 // author You // match *://*.ke.com/xiaoqu/* // match *://*.ke.com/*xiaoqu* // grant none // run-at document-end // /UserScript (function() { use strict; const DATA_STORE_KEY bk_v4_data; const TASK_RUN_FLAG bk_v4_task_active; function loadDataMap() { const raw localStorage.getItem(DATA_STORE_KEY); if (!raw) return new Map(); const arr JSON.parse(raw); return new Map(arr.map(o [o.resblockId, o])); } function saveDataMap(map) { const arr Array.from(map.values()); localStorage.setItem(DATA_STORE_KEY, JSON.stringify(arr)); } function getTaskActive() { return localStorage.getItem(TASK_RUN_FLAG) true; } function setTaskActive(enable) { if (enable) localStorage.setItem(TASK_RUN_FLAG, true); else localStorage.removeItem(TASK_RUN_FLAG); } let dataMap loadDataMap(); let uiDom null; const sleep ms new Promise(ressetTimeout(res,ms)); function buildPanel() { if(document.getElementById(bk_auto_spider)) return; uiDom document.createElement(div); uiDom.id bk_auto_spider; uiDom.style.cssText position:fixed;top:10px;left:10px;z-index:9999999;background:#fff;border:2px solid #2378dd;border-radius:8px;padding:12px;min-width:420px;box-shadow:0 4px 16px #0003;font-size:13px;color:#000;; document.body.appendChild(uiDom); } function getPageNum(){ const m location.href.match(/pg(\d)/); return m ? Number(m[1]) : 1; } function renderUi(page, total, add, skip, statusText){ buildPanel(); uiDom.innerHTML div stylefont-weight:bold;font-size:15px;margin-bottom:8px贝壳全自动采集器/div div当前页码${page}/div div累计采集${total} 条/div div本页新增${add}跳过重复${skip}/div div stylecolor:#d62828;margin:6px 0状态${statusText}/div div styledisplay:flex;gap:6px;flex-wrap:wrap;margin-top:8px button idstartBtn▶开始全自动采集/button button idstopBtn⏹停止任务/button button idexportBtn导出CSV/button button idclearBtn清空数据/button /div; document.querySelector(#startBtn).onclick runMain; document.querySelector(#stopBtn).onclick (){ setTaskActive(false); renderUi(getPageNum(), dataMap.size,0,0,⏹任务已手动停止); }; document.querySelector(#exportBtn).onclick downloadCsv; document.querySelector(#clearBtn).onclick (){ if(confirm(确认清空全部采集数据不可恢复)){ localStorage.removeItem(DATA_STORE_KEY); setTaskActive(false); dataMap new Map(); renderUi(getPageNum(),0,0,0,✅数据已清空); } }; } function isCaptchaShow(){ const capEl document.getElementById(captcha); if(!capEl) return false; const css getComputedStyle(capEl); return !(css.display none || css.visibility hidden); } function parseCurrentPage(){ const listItems document.querySelectorAll(li.xiaoquListItem[data-id]); console.log(本页小区数量:${listItems.length}); let add 0, skip 0; listItems.forEach(li{ try{ const rid li.dataset.id.trim(); if(!rid) return; if(dataMap.has(rid)){ skip; return; } const titleA li.querySelector(.title a); const name titleA?.title?.trim()||; const url titleA?.href||; const dist li.querySelector(.positionInfo .district)?.innerText.trim()||; const biz li.querySelector(.positionInfo .bizcircle)?.innerText.trim()||; const price li.querySelector(.xiaoquListItemPrice .totalPrice span)?.innerText.trim()||暂无数据; const jiaoYiLink li.querySelector(.houseInfo a[href*chengjiao]); const zuFangLink li.querySelector(.houseInfo a[href*zufang]); let sale90 0, rentCnt0; if(jiaoYiLink){ const m jiaoYiLink.innerText.match(/(\d)/); if(m) sale90 m[1]; } if(zuFangLink){ const m zuFangLink.innerText.match(/(\d)/); if(m) rentCnt m[1]; } const sellTotal li.querySelector(.xiaoquListItemSellCount .totalSellCount span)?.innerText.trim()||0; const subwayText li.querySelector(.tagList span)?.innerText.trim()||; dataMap.set(rid, { resblockId:rid,name,district:dist,bizcircle:biz,price, sale90,rentCount:rentCnt,sellCount:sellTotal,subwayTag:subwayText,url }); add; }catch(err){ console.warn(解析单条异常,err); } }); saveDataMap(dataMap); return {add,skip}; } function getNextPageHref(){ const pageWrap document.querySelector(.house-lst-page-box); if(!pageWrap) return null; const links pageWrap.querySelectorAll(a); for(let a of links){ const txt a.innerText.trim(); if(txt 下一页 !a.classList.contains(disabled)){ return a.href; } } return null; } async function runMain(){ setTaskActive(true); while(true){ while(isCaptchaShow()){ renderUi(getPageNum(), dataMap.size,--,--,⚠️请手动完成滑块验证码); await sleep(800); } const {add,skip} parseCurrentPage(); const currPg getPageNum(); renderUi(currPg, dataMap.size, add, skip,✅本页解析完成); const nextUrl getNextPageHref(); if(!nextUrl){ renderUi(currPg, dataMap.size, add, skip,全部页面采集完毕请导出); setTaskActive(false); return; } renderUi(currPg, dataMap.size, add, skip,⏳等待跳转下一页); await sleep(2800); location.href nextUrl; return; } } function downloadCsv(){ const headers [小区ID,小区名称,行政区,商圈,二手房参考均价,90天成交套数,正在出租套数,在售二手房套数,地铁标签,页面url]; const rows [headers]; for(const obj of dataMap.values()){ const line [ obj.resblockId,obj.name,obj.district,obj.bizcircle,obj.price, obj.sale90,obj.rentCount,obj.sellCount,obj.subwayTag,obj.url ].map(c${String(c).replace(//g,)}); rows.push(line.join(,)); } const blob new Blob([\uFEFFrows.join(\n)], {type:text/csv;charsetutf-8}); const a document.createElement(a); a.href URL.createObjectURL(blob); a.download 贝壳小区采集_${Date.now()}.csv; document.body.appendChild(a); a.click(); a.remove(); } //定时守护强制重建面板 setInterval((){ if(!document.getElementById(bk_auto_spider) document.body){ renderUi(getPageNum(), dataMap.size,0,0,UI自动恢复); } },1000); //暴力猴专用延迟启动 async function init(){ let cnt0; while(!document.body cnt30){ await sleep(200); cnt; } await sleep(1200); renderUi(getPageNum(), dataMap.size,0,0,✅页面就绪); if(getTaskActive()){ await sleep(1500); runMain(); } } init(); })();
贝壳获取小区的名称
(async function crawlAll() { const allNames new Set(); const totalPages 19; // 已知一共19页 const baseUrl window.location.origin /xiaoqu/xuanwu/; for (let page 1; page totalPages; page) { try { // 构建第 N 页的 URL const url page 1 ? baseUrl : ${baseUrl}pg${page}; console.log(正在抓取第 ${page} 页...); // 模拟请求 const response await fetch(url); if (!response.ok) throw new Error(请求失败); const text await response.text(); const parser new DOMParser(); const doc parser.parseFromString(text, text/html); // 提取小区名 const links doc.querySelectorAll(.xiaoquListItem .title a); links.forEach(link { const name link.title.trim(); if (name) allNames.add(name); }); // 延时防封 await new Promise(resolve setTimeout(resolve, 1000)); } catch (e) { console.log(第 ${page} 页跳过或失败:, e.message); } } // 一次性输出所有 const finalList Array.from(allNames).join(\n); console.log(\n 抓取完成以下是完整列表\n); console.log(finalList); // 一行一个完整输出 })();获取单页的小区名称// 获取所有小区列表项 const items document.querySelectorAll(.xiaoquListItem); // 遍历并提取小区名称 const names []; items.forEach(item { // 从 title 属性或文本获取小区名 const name item.querySelector(.title a).title.trim(); names.push(name); }); // 一行一个输出 console.log(names.join(\n)); // 同时返回结果方便复制 names;自动点击小区列表下一页 获取小区名单导出excel// 贝壳小区采集器 v18浏览器控制台版 // 使用在贝壳“小区列表页”打开开发者工具把本文件完整粘贴到 Console 后回车。 // 说明同源 fetch 后台取页不刷新当前页面以小区 ID 去重并保存到 localStorage。 (function () { use strict; const APP_KEY __bk_xiaoqu_v18__; const STORAGE_KEY bk_xiaoqu_data_v18; const CONFIG_KEY bk_xiaoqu_config_v18; const old window[APP_KEY]; if (old typeof old.destroy function) old.destroy(); const state { running: false, page: 0, maxPage: 0, failCount: 0, captchaCount: 0, verifyWindow: null, controller: null, activeFrame: null, data: new Map(), host: null, root: null }; const sleep (ms) new Promise((resolve) setTimeout(resolve, ms)); const textOf (el) (el (el.innerText || el.textContent) || ).trim(); const intOf (value) parseInt(String(value || ).replace(/[^0-9]/g, ), 10) || 0; function loadConfig() { try { return Object.assign({ captchaSeconds: 10, pageMinMs: 1200, pageMaxMs: 2600 }, JSON.parse(localStorage.getItem(CONFIG_KEY) || {})); } catch (_) { return { captchaSeconds: 10, pageMinMs: 1200, pageMaxMs: 2600 }; } } const config loadConfig(); function loadData() { const keys [STORAGE_KEY, bk_xiaoqu_data_v17, bk_xiaoqu_data_v16]; for (const key of keys) { try { const list JSON.parse(localStorage.getItem(key) || []); if (!Array.isArray(list) || !list.length) continue; for (const item of list) { if (item item.resblockId) state.data.set(String(item.resblockId), item); } break; } catch (_) {} } } function saveData() { try { localStorage.setItem(STORAGE_KEY, JSON.stringify(Array.from(state.data.values()))); } catch (error) { log(本地保存失败 error.message, err); } } function isListPage() { return /^\/xiaoqu(?:\/|$)/.test(location.pathname); } function baseListPath() { const parts location.pathname.split(/).filter(Boolean).filter((part) !/^pg\d$/i.test(part)); return / parts.join(/) /; } function pageUrl(pageNo) { const path baseListPath(); return new URL(path (pageNo 1 ? pg pageNo / : ) location.search, location.origin).href; } function currentPageNo() { const match location.pathname.match(/\/pg(\d)(?:\/|$)/i); return match ? Number(match[1]) || 1 : 1; } function maxPageOf(doc) { try { const el doc.querySelector([page-data]); if (el) { const value JSON.parse(el.getAttribute(page-data) || {}); const n Number(value.totalPage); if (n 0) return n; } } catch (_) {} const nums Array.from(doc.querySelectorAll(.page-box a, .house-lst-page-box a)) .map((a) intOf(textOf(a))).filter((n) n 0); return nums.length ? Math.max(...nums) : 1; } function isCaptcha(doc, rawHtml) { if (doc doc.querySelector(#captcha, .geetest_captcha, .geetest_holder, [class*geetest_])) return true; const html String(rawHtml || ).slice(0, 200000); return /geetest_captcha|geetest_holder|BlockedScreen|verify_code|captcha_v4|点击按钮开始验证/i.test(html); } function parseItems(doc) { const result []; for (const li of doc.querySelectorAll(.listContent li)) { try { const nameEl li.querySelector(.info .title a, .title a); if (!nameEl) continue; const rawUrl nameEl.getAttribute(href) || ; const url rawUrl ? new URL(rawUrl, location.origin).href : ; const urlMatch url.match(/\/xiaoqu\/(\d)(?:\.html)?\/?(?:[?#]|$)/i); const resblockId String(li.dataset.id || li.dataset.housecode || (urlMatch urlMatch[1]) || ); if (!resblockId) continue; const links li.querySelectorAll(.positionInfo a); const houseInfo textOf(li.querySelector(.houseInfo, .sub)); const saleMatch houseInfo.match(/90\s*天成交\s*(\d)\s*套/); const rentMatch houseInfo.match(/(\d)\s*套正在出租/); result.push({ resblockId, name: textOf(nameEl), district: textOf(li.querySelector(.positionInfo .district)) || textOf(links[0]), bizcircle: textOf(li.querySelector(.positionInfo .bizcircle)) || textOf(links[1]), price: intOf(textOf(li.querySelector(.unitPrice span, .totalPrice span))), sale90: saleMatch ? Number(saleMatch[1]) : 0, rentCount: rentMatch ? Number(rentMatch[1]) : 0, sellCount: intOf(textOf(li.querySelector(.totalSellCount span))), tag: Array.from(li.querySelectorAll(.tagList span)).map(textOf).filter(Boolean).join(), url }); } catch (error) { log(单条解析失败 error.message, warn); } } return result; } // 不能用 fetch贝壳触发风控时会跨域跳到 hip.ke.com/captcha // 带 cookie 的跨域重定向会被 CORS 拦截JS 只能得到 Failed to fetch。 // iframe 导航不受 fetch CORS 限制跳到跨域页后无法读取 document正好可据此识别验证码。 function fetchPage(pageNo) { const url pageUrl(pageNo); return new Promise((resolve) { const frame document.createElement(iframe); state.activeFrame frame; frame.style.cssText position:fixed;left:-10000px;top:-10000px;width:1200px;height:800px;border:0;visibility:hidden; let finished false; const finish (value) { if (finished) return; finished true; clearTimeout(timer); try { frame.remove(); } catch (_) {} if (state.activeFrame frame) state.activeFrame null; resolve(value); }; const timer setTimeout(() { finish({ kind: error, url, message: 后台页面加载超时 }); }, 35000); frame.onload () { setTimeout(() { if (finished) return; try { const finalUrl frame.contentWindow.location.href; const doc frame.contentDocument || frame.contentWindow.document; if (!doc) { finish({ kind: error, url, message: 后台页面 document 为空 }); return; } if (isCaptcha(doc, doc.documentElement doc.documentElement.outerHTML)) { finish({ kind: captcha, url: finalUrl || url }); return; } const items parseItems(doc); if (!items.length) { finish({ kind: error, url, message: 页面没有找到 .listContent li }); return; } finish({ kind: ok, url, doc, items }); } catch (error) { // wh.ke.com - hip.ke.com/captcha 后读取 iframe 会抛 SecurityError。 finish({ kind: captcha, url, crossOrigin: true }); } }, 600); }; frame.onerror () finish({ kind: error, url, message: 后台页面加载失败 }); frame.src url; document.body.appendChild(frame); }); } async function interruptibleWait(ms) { const end Date.now() ms; while (state.running Date.now() end) await sleep(Math.min(250, end - Date.now())); } function openVerifyPage(url) { try { if (!state.verifyWindow || state.verifyWindow.closed) state.verifyWindow window.open(url, bk_xiaoqu_verify); else { state.verifyWindow.location.href url; state.verifyWindow.focus(); } if (!state.verifyWindow) log(验证标签被浏览器拦截请点击“打开验证页”按钮。, warn); } catch (error) { log(无法打开验证页 error.message, warn); } } function insertItems(items) { let added 0; let duplicate 0; for (const item of items) { const key String(item.resblockId); if (state.data.has(key)) duplicate; else { state.data.set(key, item); added; } } saveData(); return { added, duplicate }; } async function start() { if (state.running) return; if (!isListPage()) { setStatus(当前不是贝壳小区列表页, err); return; } readSettings(); state.running true; state.failCount 0; state.captchaCount 0; state.maxPage maxPageOf(document); let page 1; updateUi(); log(开始采集共 state.maxPage 页, ok); while (page state.maxPage state.running) { state.page page; setStatus(后台请求第 page / state.maxPage 页, ok); updateUi(); const result await fetchPage(page); if (!state.running) break; if (result.kind captcha) { state.captchaCount; updateUi(); setStatus(验证码暂停在第 page 页每 config.captchaSeconds 秒重试, warn); log(第 page 页出现验证码不会跳页。请在验证标签完成验证。, warn); openVerifyPage(result.url); await interruptibleWait(config.captchaSeconds * 1000); continue; } if (result.kind error) { state.failCount; updateUi(); log(第 page 页请求失败 result.message, err); if (state.failCount 3) { state.running false; setStatus(连续失败 3 次已停在第 page 页, err); break; } await interruptibleWait(3000); continue; } state.failCount 0; const count insertItems(result.items); if (page 1) state.maxPage Math.max(state.maxPage, maxPageOf(result.doc)); log(第 page 页新增 count.added 重复 count.duplicate 累计 state.data.size, ok); updateUi(); page; if (page state.maxPage) { const delay config.pageMinMs Math.floor(Math.random() * Math.max(1, config.pageMaxMs - config.pageMinMs)); await interruptibleWait(delay); } } const completed page state.maxPage; state.running false; updateUi(); if (completed) { setStatus(采集完成共 state.data.size 条, ok); log(全部采集完成共 state.data.size 条。, ok); } } function stop() { state.running false; if (state.controller) state.controller.abort(); if (state.activeFrame) { try { state.activeFrame.remove(); } catch (_) {} state.activeFrame null; } setStatus(已停止, warn); log(用户停止采集。, warn); } function csvCell(value) { const s String(value null ? : value); return s.replace(//g, ) ; } function exportCsv() { const header [小区ID, 小区名称, 行政区, 商圈, 均价(元/㎡), 90天成交(套), 正在出租(套), 在售二手房(套), 标签, URL]; const rows [header]; for (const o of state.data.values()) rows.push([o.resblockId, o.name, o.district, o.bizcircle, o.price, o.sale90, o.rentCount, o.sellCount, o.tag, o.url]); const csv \uFEFF rows.map((row) row.map(csvCell).join(,)).join(\r\n); const blobUrl URL.createObjectURL(new Blob([csv], { type: text/csv;charsetutf-8 })); const a document.createElement(a); a.href blobUrl; a.download 贝壳小区_ new Date().toISOString().slice(0, 19).replace(/[T:]/g, -) .csv; a.click(); setTimeout(() URL.revokeObjectURL(blobUrl), 1000); log(已导出 state.data.size 条Excel 可直接打开。, ok); } function clearData() { if (!confirm(确定清空已保存的 state.data.size 条数据吗)) return; state.data.clear(); saveData(); updateUi(); log(已清空数据。, warn); } function readSettings() { const input state.root.getElementById(captchaSeconds); config.captchaSeconds Math.max(2, Math.min(300, Number(input.value) || 10)); input.value String(config.captchaSeconds); localStorage.setItem(CONFIG_KEY, JSON.stringify(config)); } function log(message, type) { const box state.root state.root.getElementById(log); const line [ new Date().toLocaleTimeString() ] message; console[type err ? error : type warn ? warn : log]([贝壳采集器] message); if (!box) return; const div document.createElement(div); div.className type || ; div.textContent line; box.appendChild(div); while (box.children.length 250) box.firstChild.remove(); box.scrollTop box.scrollHeight; } function setStatus(message, type) { const el state.root state.root.getElementById(status); if (el) { el.textContent message; el.className type || ; } } function updateUi() { if (!state.root) return; state.root.getElementById(page).textContent state.page / (state.maxPage || ?); state.root.getElementById(count).textContent state.data.size; state.root.getElementById(captchaCount).textContent state.captchaCount; state.root.getElementById(failCount).textContent state.failCount; state.root.getElementById(start).disabled state.running; state.root.getElementById(stop).disabled !state.running; } function buildUi() { const host document.createElement(div); host.style.cssText position:fixed;right:12px;top:12px;width:330px;z-index:2147483647; const root host.attachShadow({ mode: open }); root.innerHTML style *{box-sizing:border-box} .panel{font:13px/1.45 -apple-system,BlinkMacSystemFont,Segoe UI,Microsoft YaHei,sans-serif;color:#eef2ff;background:#172554;border:1px solid #3b82f6;border-radius:10px;padding:12px;box-shadow:0 8px 28px #0008} h3{font-size:15px;margin:0 0 8px}.stats{display:grid;grid-template-columns:1fr 1fr;gap:5px;background:#ffffff12;padding:7px;border-radius:6px}.stats b{color:#fbbf24} .setting{display:flex;align-items:center;gap:6px;margin:8px 0}.setting input{width:64px;padding:4px;border-radius:4px;border:1px solid #94a3b8} .buttons{display:grid;grid-template-columns:1fr 1fr;gap:6px;margin-top:7px}button{border:0;border-radius:5px;padding:7px 5px;color:white;background:#2563eb;cursor:pointer}button:disabled{opacity:.45;cursor:not-allowed}#stop{background:#dc2626}#verify{background:#d97706}#clear{background:#64748b} #status{margin:8px 0;padding:6px;background:#0004;border-radius:5px}.ok{color:#86efac}.warn{color:#fde047}.err{color:#fca5a5} #log{height:155px;overflow:auto;background:#0f172a;padding:6px;border-radius:5px;font:11px/1.45 Consolas,monospace}.tip{color:#cbd5e1;font-size:11px;margin-top:7px} /style div classpanel h3 贝壳小区采集器 v18/h3 div classstatsspan页码b idpage0/?/b/spanspan已采b idcount0/b/spanspan验证码b idcaptchaCount0/b/spanspan连续失败b idfailCount0/b/span/div label classsetting验证码重试间隔 input idcaptchaSeconds typenumber min2 max300 value10 秒/label div idstatus初始化.../div div classbuttonsbutton idstart▶ 从第1页开始/buttonbutton idstop⏹ 停止/buttonbutton idverify 打开验证页/buttonbutton idexport 导出Excel(CSV)/buttonbutton idparse 解析当前页/buttonbutton idclear 清空数据/button/div div classtip后台 fetch不刷新本页按小区 ID 去重。验证码通过后会自动重试原页。/div div idlog/div /div; document.body.appendChild(host); state.host host; state.root root; root.getElementById(captchaSeconds).value String(config.captchaSeconds); root.getElementById(start).onclick start; root.getElementById(stop).onclick stop; root.getElementById(verify).onclick () openVerifyPage(pageUrl(state.page || currentPageNo())); root.getElementById(export).onclick exportCsv; root.getElementById(clear).onclick clearData; root.getElementById(parse).onclick () { const count insertItems(parseItems(document)); log(当前页新增 count.added 重复 count.duplicate, ok); updateUi(); }; root.getElementById(captchaSeconds).onchange readSettings; } function destroy() { stop(); if (state.host) state.host.remove(); } loadData(); buildUi(); state.page currentPageNo(); state.maxPage maxPageOf(document); updateUi(); setStatus(isListPage() ? 就绪 : 请先打开贝壳小区列表页, isListPage() ? ok : err); log(已加载历史数据 state.data.size 条当前检测到 state.maxPage 页。, ok); window[APP_KEY] { destroy, state }; })();导入到excel 里面// UserScript // name 贝壳小区采集【暴力猴兼容版】 // namespace http://tampermonkey.net/ // version 1.5 // description 兼容Violentmonkey修复悬浮窗不出现 // author You // match *://*.ke.com/xiaoqu/* // match *://*.ke.com/*xiaoqu* // grant none // run-at document-end // /UserScript (function() { use strict; const DATA_STORE_KEY bk_v4_data; const TASK_RUN_FLAG bk_v4_task_active; function loadDataMap() { const raw localStorage.getItem(DATA_STORE_KEY); if (!raw) return new Map(); const arr JSON.parse(raw); return new Map(arr.map(o [o.resblockId, o])); } function saveDataMap(map) { const arr Array.from(map.values()); localStorage.setItem(DATA_STORE_KEY, JSON.stringify(arr)); } function getTaskActive() { return localStorage.getItem(TASK_RUN_FLAG) true; } function setTaskActive(enable) { if (enable) localStorage.setItem(TASK_RUN_FLAG, true); else localStorage.removeItem(TASK_RUN_FLAG); } let dataMap loadDataMap(); let uiDom null; const sleep ms new Promise(ressetTimeout(res,ms)); function buildPanel() { if(document.getElementById(bk_auto_spider)) return; uiDom document.createElement(div); uiDom.id bk_auto_spider; uiDom.style.cssText position:fixed;top:10px;left:10px;z-index:9999999;background:#fff;border:2px solid #2378dd;border-radius:8px;padding:12px;min-width:420px;box-shadow:0 4px 16px #0003;font-size:13px;color:#000;; document.body.appendChild(uiDom); } function getPageNum(){ const m location.href.match(/pg(\d)/); return m ? Number(m[1]) : 1; } function renderUi(page, total, add, skip, statusText){ buildPanel(); uiDom.innerHTML div stylefont-weight:bold;font-size:15px;margin-bottom:8px贝壳全自动采集器/div div当前页码${page}/div div累计采集${total} 条/div div本页新增${add}跳过重复${skip}/div div stylecolor:#d62828;margin:6px 0状态${statusText}/div div styledisplay:flex;gap:6px;flex-wrap:wrap;margin-top:8px button idstartBtn▶开始全自动采集/button button idstopBtn⏹停止任务/button button idexportBtn导出CSV/button button idclearBtn清空数据/button /div; document.querySelector(#startBtn).onclick runMain; document.querySelector(#stopBtn).onclick (){ setTaskActive(false); renderUi(getPageNum(), dataMap.size,0,0,⏹任务已手动停止); }; document.querySelector(#exportBtn).onclick downloadCsv; document.querySelector(#clearBtn).onclick (){ if(confirm(确认清空全部采集数据不可恢复)){ localStorage.removeItem(DATA_STORE_KEY); setTaskActive(false); dataMap new Map(); renderUi(getPageNum(),0,0,0,✅数据已清空); } }; } function isCaptchaShow(){ const capEl document.getElementById(captcha); if(!capEl) return false; const css getComputedStyle(capEl); return !(css.display none || css.visibility hidden); } function parseCurrentPage(){ const listItems document.querySelectorAll(li.xiaoquListItem[data-id]); console.log(本页小区数量:${listItems.length}); let add 0, skip 0; listItems.forEach(li{ try{ const rid li.dataset.id.trim(); if(!rid) return; if(dataMap.has(rid)){ skip; return; } const titleA li.querySelector(.title a); const name titleA?.title?.trim()||; const url titleA?.href||; const dist li.querySelector(.positionInfo .district)?.innerText.trim()||; const biz li.querySelector(.positionInfo .bizcircle)?.innerText.trim()||; const price li.querySelector(.xiaoquListItemPrice .totalPrice span)?.innerText.trim()||暂无数据; const jiaoYiLink li.querySelector(.houseInfo a[href*chengjiao]); const zuFangLink li.querySelector(.houseInfo a[href*zufang]); let sale90 0, rentCnt0; if(jiaoYiLink){ const m jiaoYiLink.innerText.match(/(\d)/); if(m) sale90 m[1]; } if(zuFangLink){ const m zuFangLink.innerText.match(/(\d)/); if(m) rentCnt m[1]; } const sellTotal li.querySelector(.xiaoquListItemSellCount .totalSellCount span)?.innerText.trim()||0; const subwayText li.querySelector(.tagList span)?.innerText.trim()||; dataMap.set(rid, { resblockId:rid,name,district:dist,bizcircle:biz,price, sale90,rentCount:rentCnt,sellCount:sellTotal,subwayTag:subwayText,url }); add; }catch(err){ console.warn(解析单条异常,err); } }); saveDataMap(dataMap); return {add,skip}; } function getNextPageHref(){ const pageWrap document.querySelector(.house-lst-page-box); if(!pageWrap) return null; const links pageWrap.querySelectorAll(a); for(let a of links){ const txt a.innerText.trim(); if(txt 下一页 !a.classList.contains(disabled)){ return a.href; } } return null; } async function runMain(){ setTaskActive(true); while(true){ while(isCaptchaShow()){ renderUi(getPageNum(), dataMap.size,--,--,⚠️请手动完成滑块验证码); await sleep(800); } const {add,skip} parseCurrentPage(); const currPg getPageNum(); renderUi(currPg, dataMap.size, add, skip,✅本页解析完成); const nextUrl getNextPageHref(); if(!nextUrl){ renderUi(currPg, dataMap.size, add, skip,全部页面采集完毕请导出); setTaskActive(false); return; } renderUi(currPg, dataMap.size, add, skip,⏳等待跳转下一页); await sleep(2800); location.href nextUrl; return; } } function downloadCsv(){ const headers [小区ID,小区名称,行政区,商圈,二手房参考均价,90天成交套数,正在出租套数,在售二手房套数,地铁标签,页面url]; const rows [headers]; for(const obj of dataMap.values()){ const line [ obj.resblockId,obj.name,obj.district,obj.bizcircle,obj.price, obj.sale90,obj.rentCount,obj.sellCount,obj.subwayTag,obj.url ].map(c${String(c).replace(//g,)}); rows.push(line.join(,)); } const blob new Blob([\uFEFFrows.join(\n)], {type:text/csv;charsetutf-8}); const a document.createElement(a); a.href URL.createObjectURL(blob); a.download 贝壳小区采集_${Date.now()}.csv; document.body.appendChild(a); a.click(); a.remove(); } //定时守护强制重建面板 setInterval((){ if(!document.getElementById(bk_auto_spider) document.body){ renderUi(getPageNum(), dataMap.size,0,0,UI自动恢复); } },1000); //暴力猴专用延迟启动 async function init(){ let cnt0; while(!document.body cnt30){ await sleep(200); cnt; } await sleep(1200); renderUi(getPageNum(), dataMap.size,0,0,✅页面就绪); if(getTaskActive()){ await sleep(1500); runMain(); } } init(); })();