publishport-opencli 1.8.4-pp.1

package/clis/juejin/utils.js

@@ -0,0 +1,199 @@
+// Shared helpers for the Juejin (`api.juejin.cn`) adapter.
+//
+// Juejin is a Chinese developer community (similar to Dev.to). The public
+// REST API is unauthenticated; all read endpoints are reachable without a
+// browser session. Article URLs round-trip as `https://juejin.cn/post/<id>`.
+import { ArgumentError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
+
+export const JUEJIN_API_BASE = 'https://api.juejin.cn';
+export const JUEJIN_POST_URL = 'https://juejin.cn/post';
+export const JUEJIN_USER_URL = 'https://juejin.cn/user';
+const UA = 'opencli-juejin-adapter (+https://github.com/jackwener/opencli)';
+
+// Juejin content / article IDs are 19-digit numeric strings.
+const JUEJIN_ID = /^\d{16,20}$/;
+
+// Top-level categories surfaced by `query_category_briefs`. The slugs are
+// stable so the adapter accepts a friendly name in addition to the raw id.
+export const CATEGORY_ALIASES = {
+    backend: '6809637769959178254',
+    frontend: '6809637767543259144',
+    android: '6809635626879549454',
+    ios: '6809635626661445640',
+    ai: '6809637773935378440',
+};
+
+export function requireString(value, label) {
+    const s = String(value ?? '').trim();
+    if (!s) {
+        throw new ArgumentError(`juejin ${label} cannot be empty`);
+    }
+    return s;
+}
+
+export function requireBoundedInt(value, defaultValue, maxValue, label = 'limit') {
+    const raw = value ?? defaultValue;
+    let n;
+    if (typeof raw === 'number') {
+        n = raw;
+    }
+    else if (typeof raw === 'string' && /^[1-9]\d*$/.test(raw)) {
+        n = Number(raw);
+    }
+    else {
+        throw new ArgumentError(`juejin ${label} must be a positive decimal integer`);
+    }
+    if (!Number.isSafeInteger(n) || n <= 0) {
+        throw new ArgumentError(`juejin ${label} must be a positive integer`);
+    }
+    if (n > maxValue) {
+        throw new ArgumentError(`juejin ${label} must be <= ${maxValue}`);
+    }
+    return n;
+}
+
+export function requireCursor(value) {
+    const raw = value ?? '0';
+    if (typeof raw === 'number') {
+        if (Number.isSafeInteger(raw) && raw >= 0) return String(raw);
+        throw new ArgumentError('juejin cursor must be a non-negative decimal integer');
+    }
+    if (typeof raw === 'string' && /^(0|[1-9]\d*)$/.test(raw)) {
+        return raw;
+    }
+    throw new ArgumentError('juejin cursor must be a non-negative decimal integer');
+}
+
+/** Resolve a `--category` arg to the underlying numeric category id. */
+export function resolveCategory(value) {
+    if (value == null) return '';
+    const raw = String(value).trim();
+    if (!raw) return '';
+    if (JUEJIN_ID.test(raw)) return raw;
+    const slug = raw.toLowerCase();
+    if (CATEGORY_ALIASES[slug]) return CATEGORY_ALIASES[slug];
+    throw new ArgumentError(
+        `juejin category "${value}" is not recognised`,
+        `Use a category id (e.g. "${CATEGORY_ALIASES.backend}") or one of: ${Object.keys(CATEGORY_ALIASES).join(', ')}.`,
+    );
+}
+
+/**
+ * POST JSON to a Juejin endpoint. The API returns `{ err_no, err_msg, data }`;
+ * a non-zero `err_no` is surfaced as a typed `CommandExecutionError`.
+ */
+export async function juejinFetch(path, body, label, method = 'POST') {
+    const url = `${JUEJIN_API_BASE}${path}`;
+    let resp;
+    try {
+        const init = {
+            method,
+            headers: { 'user-agent': UA, accept: 'application/json' },
+        };
+        if (method === 'POST') {
+            init.headers['content-type'] = 'application/json';
+            init.body = JSON.stringify(body ?? {});
+        }
+        resp = await fetch(url, init);
+    } catch (err) {
+        throw new CommandExecutionError(
+            `${label} request failed: ${err?.message ?? err}`,
+            'Check that api.juejin.cn is reachable from this network.',
+        );
+    }
+    if (resp.status === 429) {
+        throw new CommandExecutionError(
+            `${label} returned HTTP 429 (rate limited)`,
+            'Juejin throttles bursty traffic; wait a few seconds and retry.',
+        );
+    }
+    if (!resp.ok) {
+        throw new CommandExecutionError(`${label} returned HTTP ${resp.status}`);
+    }
+    let payload;
+    try {
+        payload = await resp.json();
+    } catch (err) {
+        throw new CommandExecutionError(`${label} returned malformed JSON: ${err?.message ?? err}`);
+    }
+    if (!payload || typeof payload !== 'object' || Array.isArray(payload) || !Object.hasOwn(payload, 'err_no')) {
+        throw new CommandExecutionError(`${label} returned a malformed API envelope`);
+    }
+    if (payload.err_no !== 0) {
+        throw new CommandExecutionError(`${label} returned err_no ${payload.err_no}: ${payload.err_msg ?? ''}`);
+    }
+    return payload;
+}
+
+export function readDataArray(payload, label) {
+    if (!payload || typeof payload !== 'object' || Array.isArray(payload) || !Object.hasOwn(payload, 'data')) {
+        throw new CommandExecutionError(`${label} returned a malformed payload`);
+    }
+    if (!Array.isArray(payload.data)) {
+        throw new CommandExecutionError(`${label} returned a non-array data field`);
+    }
+    if (payload.data.length === 0) {
+        throw new EmptyResultError(label, `${label} returned no articles.`);
+    }
+    return payload.data;
+}
+
+function readArticleId(value, label) {
+    const id = String(value ?? '').trim();
+    if (!JUEJIN_ID.test(id)) {
+        throw new CommandExecutionError(`${label} returned a malformed article id`);
+    }
+    return id;
+}
+
+function readOptionalNumber(value, label) {
+    if (value == null) return null;
+    const n = Number(value);
+    if (!Number.isFinite(n)) {
+        throw new CommandExecutionError(`${label} returned a malformed numeric field`);
+    }
+    return n;
+}
+
+/** Map a recommend-feed row (`item_info.article_info` / `author_user_info`) to a flat shape. */
+export function mapFeedItem(row, rank) {
+    const info = row?.item_info ?? {};
+    const article = info.article_info ?? {};
+    const author = info.author_user_info ?? {};
+    const tags = Array.isArray(info.tags)
+        ? info.tags.map(t => t?.tag_name).filter(Boolean).slice(0, 6).join(', ')
+        : '';
+    const articleId = readArticleId(article.article_id, 'juejin recommend');
+    return {
+        rank,
+        article_id: articleId,
+        title: String(article.title ?? '').trim(),
+        brief: String(article.brief_content ?? '').trim(),
+        views: readOptionalNumber(article.view_count, 'juejin recommend'),
+        likes: readOptionalNumber(article.digg_count, 'juejin recommend'),
+        comments: readOptionalNumber(article.comment_count, 'juejin recommend'),
+        author: String(author.user_name ?? '').trim(),
+        tags,
+        url: articleId ? `${JUEJIN_POST_URL}/${articleId}` : '',
+    };
+}
+
+/** Map a hot-list row (different envelope from the feed endpoint) to the same flat shape. */
+export function mapHotItem(row, rank) {
+    const content = row?.content ?? {};
+    const counter = row?.content_counter ?? {};
+    const author = row?.author ?? {};
+    const articleId = readArticleId(content.content_id, 'juejin hot');
+    return {
+        rank,
+        article_id: articleId,
+        title: String(content.title ?? '').trim(),
+        brief: String(content.brief ?? '').trim(),
+        views: readOptionalNumber(counter.view, 'juejin hot'),
+        likes: readOptionalNumber(counter.like, 'juejin hot'),
+        comments: readOptionalNumber(counter.comment_count, 'juejin hot'),
+        hot_rank: readOptionalNumber(counter.hot_rank, 'juejin hot'),
+        author: String(author.name ?? '').trim(),
+        url: articleId ? `${JUEJIN_POST_URL}/${articleId}` : '',
+    };
+}

package/clis/ke/auth.js

@@ -0,0 +1,46 @@
+import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
+import { registerSiteAuthCommands } from '../_shared/site-auth.js';
+
+async function hasKeSessionCookie(page) {
+  const cookies = await page.getCookies({ url: 'https://www.ke.com' });
+  return cookies.some(c => c.name === 'lianjia_token' && c.value);
+}
+
+async function verifyKeIdentity(page) {
+  if (!await hasKeSessionCookie(page)) {
+    throw new AuthRequiredError('ke.com', 'Ke lianjia_token cookie missing — anonymous');
+  }
+  await page.goto('https://www.ke.com/');
+  await page.wait(2);
+  const probe = await page.evaluate(`
+    (() => {
+      const loginBtn = document.querySelector('.btn-login, a[class*=actLoginBtn], .login-btn');
+      if (loginBtn && /登录|登陆/.test(loginBtn.innerText || '')) {
+        return { kind: 'auth', detail: 'Ke shows 登录 button — anonymous session' };
+      }
+      const el = document.querySelector('.userNick, .user-name, .myInfo a, [class*=userNick]');
+      const username = (el?.innerText || '').trim();
+      if (!username) {
+        return { kind: 'auth', detail: 'Ke no user-name DOM anchor — anonymous or SSR failed' };
+      }
+      return { ok: true, username };
+    })()
+  `);
+  if (probe?.kind === 'auth') throw new AuthRequiredError('ke.com', probe.detail);
+  if (!probe?.ok) throw new CommandExecutionError(`Unexpected Ke probe: ${JSON.stringify(probe)}`);
+  return { username: probe.username };
+}
+
+registerSiteAuthCommands({
+  site: 'ke',
+  domain: 'ke.com',
+  loginUrl: 'https://clogin.ke.com/login/?service=https%3A%2F%2Fwww.ke.com',
+  columns: ['username'],
+  verify: verifyKeIdentity,
+  poll: async (page) => {
+    if (!await hasKeSessionCookie(page)) {
+      throw new AuthRequiredError('ke.com', 'Waiting for Ke lianjia_token cookie');
+    }
+    return verifyKeIdentity(page);
+  },
+});

package/clis/ke/chengjiao.js

@@ -0,0 +1,78 @@
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import { cityUrl, gotoKe } from './utils.js';
+
+cli({
+    site: 'ke',
+    name: 'chengjiao',
+    access: 'read',
+    description: '贝壳找房成交记录',
+    domain: 'ke.com',
+    strategy: Strategy.COOKIE,
+    browser: true,
+    args: [
+        { name: 'city', default: 'bj', help: '城市代码，如 bj(北京), sh(上海), gz(广州), sz(深圳), zs(中山)' },
+        { name: 'district', help: '区域拼音，如 chaoyang, haidian' },
+        { name: 'limit', type: 'int', default: 20, help: '返回数量' },
+    ],
+    columns: ['title', 'community', 'layout', 'area', 'deal_price', 'unit_price', 'deal_date'],
+    func: async (page, kwargs) => {
+        const city = kwargs.city || 'bj';
+        const limit = Number(kwargs.limit) || 20;
+        const base = cityUrl(city);
+
+        let path = '/chengjiao/';
+        if (kwargs.district) {
+            path = `/chengjiao/${kwargs.district}/`;
+        }
+
+        await gotoKe(page, base + path);
+
+        const items = await page.evaluate(`(async () => {
+  // chengjiao page uses .listContent li or similar structure
+  const selectors = [
+    '.listContent li',
+    'ul.listContent li',
+    '.sellListContent li.clear',
+    'li.clear',
+  ];
+  let cards = [];
+  for (const sel of selectors) {
+    cards = document.querySelectorAll(sel);
+    if (cards.length > 0) break;
+  }
+
+  const results = [];
+  for (const card of cards) {
+    const titleEl = card.querySelector('.title a, a.VIEWDATA');
+    if (!titleEl) continue;
+
+    const houseInfoEl = card.querySelector('.houseInfo');
+    const communityEl = card.querySelector('.positionInfo a');
+    const priceEl = card.querySelector('.totalPrice span');
+    const unitPriceEl = card.querySelector('.unitPrice span');
+    const dateEl = card.querySelector('.dealDate');
+    const dealCycleEl = card.querySelector('.dealCycleTxt span');
+
+    const houseText = (houseInfoEl ? houseInfoEl.textContent : '').replace(/\\s+/g, ' ').trim();
+    const houseParts = houseText.split('|').map(s => s.trim());
+
+    const layoutMatch = (houseParts[0] || '').match(/(\\d室\\d厅)/);
+    const layout = layoutMatch ? layoutMatch[1] : (houseParts[0] || '');
+
+    results.push({
+      title: (titleEl.textContent || '').trim(),
+      url: titleEl.href || '',
+      community: (communityEl ? communityEl.textContent : '').trim(),
+      layout: layout,
+      area: (houseParts[1] || '').trim(),
+      deal_price: ((priceEl ? priceEl.textContent : '').trim() || '') + '万',
+      unit_price: (unitPriceEl ? unitPriceEl.textContent : '').trim(),
+      deal_date: (dateEl ? dateEl.textContent : '').replace(/\\s+/g, ' ').trim(),
+    });
+  }
+  return results;
+})()`);
+
+        return (items || []).slice(0, limit);
+    },
+});

package/clis/ke/ershoufang.js

@@ -0,0 +1,101 @@
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import { cityUrl, gotoKe } from './utils.js';
+
+cli({
+    site: 'ke',
+    name: 'ershoufang',
+    access: 'read',
+    description: '贝壳找房二手房列表',
+    domain: 'ke.com',
+    strategy: Strategy.COOKIE,
+    browser: true,
+    args: [
+        { name: 'city', default: 'bj', help: '城市代码，如 bj(北京), sh(上海), gz(广州), sz(深圳), zs(中山)' },
+        { name: 'district', help: '区域拼音，如 chaoyang, haidian, tianhe' },
+        { name: 'min-price', type: 'int', help: '最低总价（万元）' },
+        { name: 'max-price', type: 'int', help: '最高总价（万元）' },
+        { name: 'rooms', type: 'int', help: '几居室 (1-5)' },
+        { name: 'limit', type: 'int', default: 20, help: '返回数量' },
+    ],
+    columns: ['title', 'community', 'layout', 'area', 'direction', 'total_price', 'unit_price', 'url'],
+    func: async (page, kwargs) => {
+        const city = kwargs.city || 'bj';
+        const limit = Number(kwargs.limit) || 20;
+        const base = cityUrl(city);
+
+        let path = '/ershoufang/';
+        if (kwargs.district) {
+            path = `/ershoufang/${kwargs.district}/`;
+        }
+
+        const priceParts = [];
+        if (kwargs['min-price'] || kwargs['max-price']) {
+            const min = kwargs['min-price'] || '';
+            const max = kwargs['max-price'] || '';
+            priceParts.push(`p${min}t${max}`);
+        }
+
+        const roomParts = [];
+        if (kwargs.rooms) {
+            roomParts.push(`l${kwargs.rooms}`);
+        }
+
+        const filters = [...priceParts, ...roomParts].join('');
+        const url = base + path + (filters ? filters + '/' : '');
+
+        await gotoKe(page, url);
+
+        const items = await page.evaluate(`(async () => {
+  const cards = document.querySelectorAll('.sellListContent li.clear');
+  const results = [];
+  for (const card of cards) {
+    const titleEl = card.querySelector('.title a');
+    const communityEl = card.querySelector('.positionInfo a');
+    const houseInfoEl = card.querySelector('.houseInfo');
+    const priceEl = card.querySelector('.totalPrice span');
+    const unitPriceEl = card.querySelector('.unitPrice span');
+
+    if (!titleEl) continue;
+
+    // houseInfo text varies:
+    //   "中楼层 (共24层) 4室2厅 | 133.99平米 | 东南"
+    //   "高楼层 (共32层) | 2022年 | 4室2厅 | 110平米"
+    const houseText = (houseInfoEl ? houseInfoEl.textContent : '').replace(/\\s+/g, ' ').trim();
+    const houseParts = houseText.split('|').map(s => s.trim());
+
+    // Extract structured fields from all parts
+    let layout = '', area = '', direction = '', floor = '';
+    for (const part of houseParts) {
+      if (/\\d室\\d厅/.test(part)) {
+        layout = part.match(/(\\d室\\d厅)/)[1];
+      } else if (/平米|㎡/.test(part)) {
+        area = part;
+      } else if (/^[东南西北]+$/.test(part.replace(/\\s/g, ''))) {
+        direction = part;
+      } else if (/楼层/.test(part)) {
+        floor = part;
+      }
+    }
+    // layout might be embedded in the floor part: "中楼层 (共24层) 4室2厅"
+    if (!layout) {
+      const m = houseText.match(/(\\d室\\d厅)/);
+      if (m) layout = m[1];
+    }
+
+    results.push({
+      title: (titleEl.textContent || '').trim(),
+      url: titleEl.href || '',
+      community: (communityEl ? communityEl.textContent : '').trim(),
+      layout: layout,
+      area: area,
+      direction: direction,
+      total_price: ((priceEl ? priceEl.textContent : '').trim() || '') + '万',
+      unit_price: (unitPriceEl ? unitPriceEl.textContent : '').trim(),
+    });
+  }
+  return results;
+})()`);
+
+        return (items || []).slice(0, limit);
+    },
+});

package/clis/ke/utils.js

@@ -0,0 +1,104 @@
+import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
+
+const CAPTCHA_TEXT_PATTERNS = [
+    '请拖动下方滑块完成验证',
+    '请按住滑块',
+    '验证码',
+    '安全验证',
+    '访问验证',
+    '滑动验证',
+];
+
+const LOGIN_TEXT_PATTERNS = [
+    '请登录',
+    '登录后',
+    '账号登录',
+    '手机登录',
+    '立即登录',
+    '扫码登录',
+];
+
+function cleanText(value) {
+    return typeof value === 'string'
+        ? value.replace(/\u00a0/g, ' ').replace(/\s+/g, ' ').trim()
+        : '';
+}
+
+export async function readPageState(page) {
+    const result = await page.evaluate(`
+    (() => {
+      try {
+        return {
+          href: window.location.href || '',
+          title: document.title || '',
+          body_text: document.body ? (document.body.innerText || '').substring(0, 2000) : '',
+        };
+      } catch(e) {
+        return { href: '', title: '', body_text: '' };
+      }
+    })()
+  `);
+    if (!result) {
+        return { href: '', title: '', body_text: '' };
+    }
+    return {
+        href: cleanText(result.href),
+        title: cleanText(result.title),
+        body_text: cleanText(result.body_text),
+    };
+}
+
+export function assertNotBlocked(state) {
+    const { href, title, body_text } = state;
+    if (href.includes('hip.ke.com/captcha') || href.includes('/captcha')) {
+        throw new AuthRequiredError('ke.com', '触发了验证码，请先在浏览器中完成验证');
+    }
+    if (CAPTCHA_TEXT_PATTERNS.some(p => title.includes(p) || body_text.includes(p))) {
+        throw new AuthRequiredError('ke.com', '触发了验证码，请先在浏览器中完成滑块验证');
+    }
+    if (LOGIN_TEXT_PATTERNS.some(p => title.includes(p))) {
+        throw new AuthRequiredError('ke.com', '未登录，请先在浏览器中登录贝壳找房');
+    }
+}
+
+export async function gotoKe(page, url) {
+    await page.goto(url, { settleMs: 2500 });
+    await page.wait(2);
+    const state = await readPageState(page);
+    assertNotBlocked(state);
+    return state;
+}
+
+/**
+ * Fetch a ke.com JSON API from inside the browser context (credentials included).
+ */
+export async function fetchKeJson(page, url) {
+    const result = await page.evaluate(`(async () => {
+    const res = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
+    if (!res.ok) return { __keErr: res.status };
+    try {
+      return await res.json();
+    } catch {
+      return { __keErr: 'parse' };
+    }
+  })()`);
+    const r = result;
+    if (r?.__keErr !== undefined) {
+        const code = r.__keErr;
+        if (code === 401 || code === 403) {
+            throw new AuthRequiredError('ke.com', '未登录或登录已过期，请先在浏览器中登录贝壳找房');
+        }
+        if (code === 'parse') {
+            throw new CommandExecutionError('响应不是有效 JSON', '可能触发了风控，请检查登录状态或稍后重试');
+        }
+        throw new CommandExecutionError(`HTTP ${code}`, '请检查网络连接或登录状态');
+    }
+    return result;
+}
+
+/**
+ * Build a ke.com city URL prefix. Default city is 'bj' (Beijing).
+ */
+export function cityUrl(city) {
+    return `https://${city}.ke.com`;
+}

package/clis/ke/xiaoqu.js

@@ -0,0 +1,78 @@
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import { cityUrl, gotoKe } from './utils.js';
+
+cli({
+    site: 'ke',
+    name: 'xiaoqu',
+    access: 'read',
+    description: '贝壳找房小区列表',
+    domain: 'ke.com',
+    strategy: Strategy.COOKIE,
+    browser: true,
+    args: [
+        { name: 'city', default: 'bj', help: '城市代码，如 bj(北京), sh(上海), gz(广州), sz(深圳), zs(中山)' },
+        { name: 'district', help: '区域拼音，如 chaoyang, haidian' },
+        { name: 'limit', type: 'int', default: 20, help: '返回数量' },
+    ],
+    columns: ['name', 'district', 'avg_price', 'year', 'on_sale'],
+    func: async (page, kwargs) => {
+        const city = kwargs.city || 'bj';
+        const limit = Number(kwargs.limit) || 20;
+        const base = cityUrl(city);
+
+        let path = '/xiaoqu/';
+        if (kwargs.district) {
+            path = `/xiaoqu/${kwargs.district}/`;
+        }
+
+        await gotoKe(page, base + path);
+
+        const items = await page.evaluate(`(async () => {
+  const selectors = [
+    '.xiaoquListItem',
+    'li.xiaoquListItem',
+    '.listContent li',
+    'ul.listContent li',
+  ];
+  let cards = [];
+  for (const sel of selectors) {
+    cards = document.querySelectorAll(sel);
+    if (cards.length > 0) break;
+  }
+
+  const results = [];
+  for (const card of cards) {
+    // Name is in a.img[title] or .title a
+    const imgLink = card.querySelector('a.img[title], a[title]');
+    const titleLink = card.querySelector('.title a');
+    const nameEl = titleLink || imgLink;
+    if (!nameEl) continue;
+
+    const name = (titleLink ? titleLink.textContent : imgLink.getAttribute('title')) || '';
+    const url = nameEl.href || '';
+
+    const priceEl = card.querySelector('.totalPrice span');
+    const districtEl = card.querySelector('.positionInfo a, .district a');
+    const infoEl = card.querySelector('.positionInfo, .houseInfo, .xiaoquListItemInfo');
+    const saleEl = card.querySelector('.xiaoquListItemSellCount a, .houseInfo a');
+
+    const infoText = infoEl ? infoEl.textContent : '';
+    const yearMatch = infoText.match(/(\\d{4})年/);
+
+    const priceText = (priceEl ? priceEl.textContent : '').trim();
+
+    results.push({
+      name: name.trim(),
+      url: url,
+      district: (districtEl ? districtEl.textContent : '').trim(),
+      avg_price: priceText ? priceText + '元/平' : '暂无',
+      year: yearMatch ? yearMatch[1] : '',
+      on_sale: (saleEl ? saleEl.textContent : '').trim(),
+    });
+  }
+  return results;
+})()`);
+
+        return (items || []).slice(0, limit);
+    },
+});