Second commit

.gitignore

@@ -0,0 +1 @@
+node_modules

auto-scrape.mjs

@@ -0,0 +1,639 @@
+#!/usr/bin/env node
+/**
+ * 一堂创业课文档采集工具
+ *
+ * 用法：
+ *   node auto-scrape.mjs <URL> [输出文件名]
+ *
+ * 示例：
+ *   node auto-scrape.mjs https://yitang.top/fs-doc/abc123/docId
+ *   node auto-scrape.mjs https://yitanger.feishu.cn/wiki/R9NywV2i9i...
+ *
+ * Cookie 持久化：首次登录后自动保存，之后无需重复扫码。
+ */
+import { chromium } from 'playwright';
+import CryptoJS from 'crypto-js';
+import { writeFileSync, readFileSync, existsSync } from 'fs';
+import { resolve, basename, dirname } from 'path';
+import { execSync } from 'child_process';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+// ── 打包模式：自动检测 Playwright 浏览器路径 ────────────────────────────────────
+// 当通过 pkg 打包后，process.execPath 指向可执行文件本身
+// 浏览器目录应位于可执行文件旁边的 browsers/ 目录中
+const execDir = dirname(process.execPath);
+const bundledBrowsers = resolve(execDir, 'browsers');
+if (existsSync(bundledBrowsers)) {
+  process.env.PLAYWRIGHT_BROWSERS_PATH = bundledBrowsers;
+}
+
+// ── CLI 参数解析 ──────────────────────────────────────────────────────────────
+const args = process.argv.slice(2);
+if (!args[0] || args[0] === '--help' || args[0] === '-h') {
+  console.log(`
+用法：node auto-scrape.mjs <URL> [输出文件名]
+
+  URL         yitang.top/fs-doc/... 或 *.feishu.cn/wiki/... 文档链接（必填）
+  输出文件名  保存路径，默认为文档标题.md
+
+示例：
+  node auto-scrape.mjs https://yitang.top/fs-doc/abc/docId
+  node auto-scrape.mjs https://yitanger.feishu.cn/wiki/R9NywV2i9izjuqkPd63cWuafn4g
+  node auto-scrape.mjs https://yitang.top/fs-doc/abc/docId 课程笔记.md
+`);
+  process.exit(0);
+}
+
+const TARGET_URL  = args[0].trim();
+const customOutput = args[1];
+const COOKIE_FILE  = resolve('cookies.json');
+
+// ── 判断 URL 类型 ──────────────────────────────────────────────────────────────
+const isFeishuWiki = /feishu\.cn\/wiki\//.test(TARGET_URL);
+const isFeishuDocx = /feishu\.cn\/docx\//.test(TARGET_URL);
+const isYitang = TARGET_URL.includes('yitang.top/fs-doc/');
+
+if (!isFeishuWiki && !isFeishuDocx && !isYitang) {
+  console.error('❌ URL 格式不正确，支持 yitang.top/fs-doc/<acl>/<docId> 或 *.feishu.cn/wiki/<token>');
+  process.exit(1);
+}
+
+// ── Feishu Wiki：直接调用 fetch-feishu-wiki.mjs ────────────────────────────────
+if (isFeishuWiki) {
+  console.log('🔄 检测到飞书 Wiki 文档，使用 lark-cli API 获取...\n');
+  try {
+    const fetchScript = resolve(__dirname, 'fetch-feishu-wiki.mjs');
+    execSync(`node "${fetchScript}" "${TARGET_URL}"`, {
+      stdio: 'inherit',
+      encoding: 'utf-8'
+    });
+    process.exit(0);
+  } catch (error) {
+    console.error('❌ 调用 fetch-feishu-wiki.mjs 失败');
+    process.exit(1);
+  }
+}
+
+// yitang 专用字段
+let ACL, DOC_ID, LOGIN_URL;
+if (isYitang) {
+  const match = TARGET_URL.match(/\/fs-doc\/([^/]+)\/([^/?#]+)/);
+  if (!match) {
+    console.error('❌ yitang URL 格式不正确，应为 https://yitang.top/fs-doc/<acl>/<docId>');
+    process.exit(1);
+  }
+  ACL      = match[1];
+  DOC_ID   = match[2];
+  LOGIN_URL = `https://yitang.top/login?after_login=%2Ffs-doc%2F${ACL}%2F${DOC_ID}`;
+}
+
+// ── Feishu block → Markdown ──────────────────────────────────────────────────
+const T = { PAGE:1, TEXT:2, H1:3, H2:4, H3:5, H4:6, H5:7, H6:8,
+            ORDERED:13, BULLET:12, TODO:17, CODE:14, QUOTE:15, DIVIDER:22,
+            IMAGE:27, TABLE:31, TABLE_CELL:32, QUOTE_CONTAINER:34,
+            GRID:24, GRID_COL:25 };
+
+function textRuns(elements) {
+  if (!Array.isArray(elements)) return '';
+  return elements.map(el => {
+    const run = el.text_run;
+    if (!run) return '';
+    let t = run.content || '';
+    const s = run.text_element_style || {};
+    if (s.bold)          t = `**${t}**`;
+    if (s.italic)        t = `*${t}*`;
+    if (s.strikethrough) t = `~~${t}~~`;
+    if (s.inline_code)   t = `\`${t}\``;
+    if (s.link?.url)     t = `[${t}](${s.link.url})`;
+    return t;
+  }).join('');
+}
+
+function blockToMd(block, depth = 0) {
+  if (!block) return '';
+  const attr   = block.blockAttr || {};
+  const indent = '  '.repeat(Math.max(0, depth - 1));
+  switch (block.type) {
+    case T.PAGE:     return `# ${textRuns(attr.page?.elements)}\n\n`;
+    case T.H1:       return `# ${textRuns(attr.heading1?.elements)}\n\n`;
+    case T.H2:       return `## ${textRuns(attr.heading2?.elements)}\n\n`;
+    case T.H3:       return `### ${textRuns(attr.heading3?.elements)}\n\n`;
+    case T.H4:       return `#### ${textRuns(attr.heading4?.elements)}\n\n`;
+    case T.H5:       return `##### ${textRuns(attr.heading5?.elements)}\n\n`;
+    case T.H6:       return `###### ${textRuns(attr.heading6?.elements)}\n\n`;
+    case T.TEXT: {
+      const txt = textRuns(attr.text?.elements);
+      return txt ? `${txt}\n\n` : '\n';
+    }
+    case T.BULLET:   return `${indent}- ${textRuns(attr.bullet?.elements)}\n`;
+    case T.ORDERED:  return `${indent}1. ${textRuns(attr.ordered?.elements)}\n`;
+    case T.TODO: {
+      const done = attr.todo?.style?.done ? '[x]' : '[ ]';
+      return `${indent}- ${done} ${textRuns(attr.todo?.elements)}\n`;
+    }
+    case T.QUOTE:    return `> ${textRuns(attr.quote?.elements)}\n\n`;
+    case T.CODE: {
+      const language = attr.code?.style?.language;
+      const lang = typeof language === 'string' ? language.toLowerCase().replace('plain text','') : '';
+      const code = (attr.code?.elements || []).map(e => e.text_run?.content || '').join('');
+      return `\`\`\`${lang}\n${code}\n\`\`\``;
+    }
+    case T.DIVIDER:  return `---\n\n`;
+    case T.IMAGE: {
+      const caption = (attr.textArea || []).map(a => a.text).filter(Boolean).join(' ');
+      const url = attr.cdnUrl || attr.image?.token || '';
+      return `![${caption}](${url})\n\n`;
+    }
+    case T.QUOTE_CONTAINER: return '';
+    case T.GRID:            return '';
+    case T.GRID_COL:        return '';
+    case T.TABLE:           return '\n';
+    case T.TABLE_CELL: {
+      const text = (block.childrens || []).map(c => {
+        const a = c.blockAttr || {};
+        return textRuns(a.text?.elements || a.heading1?.elements || a.heading2?.elements || []);
+      }).filter(Boolean).join(' | ');
+      return text ? `| ${text} ` : '';
+    }
+    default: return '';
+  }
+}
+
+function blocksToMarkdown(rootBlock) {
+  const lines = [];
+  function walk(block, depth) {
+    if (!block) return;
+    lines.push(blockToMd(block, depth));
+    if (block.type !== T.TABLE_CELL && block.childrens) {
+      block.childrens.forEach(c => walk(c, depth + 1));
+    }
+    if (block.type === T.TABLE) lines.push('\n');
+  }
+  walk(rootBlock, 0);
+  return lines.join('').replace(/\n{3,}/g, '\n\n').trim();
+}
+
+// ── Feishu 原生 docx block (block_type schema) → Markdown ────────────────────
+// 飞书 Open Platform block_type: 1=page 2=text 3-8=h1-h6 12=bullet 13=ordered
+//   14=code 15=quote 16=todo 21=divider 26=image 30=table 31=table_cell
+
+const FT_HEADING = { 3: 1, 4: 2, 5: 3, 6: 4, 7: 5, 8: 6 };
+const FT_CODE_LANG = {
+  1: '', 7: 'bash', 9: 'cpp', 10: 'c', 12: 'css', 22: 'go',
+  28: 'json', 29: 'java', 30: 'javascript', 49: 'python',
+  52: 'ruby', 53: 'rust', 56: 'sql', 60: 'shell', 63: 'typescript',
+};
+
+function ftTextElements(elements = []) {
+  return elements.map(el => {
+    if (el.text_run) {
+      const content = el.text_run.content || '';
+      const s = el.text_run.text_element_style || {};
+      let t = content;
+      if (s.inline_code) return `\`${t}\``;
+      if (s.bold)          t = `**${t}**`;
+      if (s.italic)        t = `*${t}*`;
+      if (s.strikethrough) t = `~~${t}~~`;
+      if (s.link?.url)     t = `[${t}](${decodeURIComponent(s.link.url)})`;
+      return t;
+    }
+    if (el.mention_doc) {
+      const { url = '', title = '' } = el.mention_doc;
+      return url ? `[${title || url}](${url})` : title;
+    }
+    if (el.mention_user) return `@${el.mention_user.user_id || 'user'}`;
+    return '';
+  }).join('');
+}
+
+function ftBlockToMd(block, index, depth, orderedNum = 1) {
+  const t = block.block_type;
+  const indent = '  '.repeat(depth);
+  if (t === 1) return ftRenderChildren(block.children || [], index, depth);
+  if (t === 2) {
+    const text = ftTextElements(block.text?.elements);
+    return text ? text + '\n' : '\n';
+  }
+  if (FT_HEADING[t] !== undefined) {
+    return `${'#'.repeat(FT_HEADING[t])} ${ftTextElements(block.text?.elements)}\n\n`;
+  }
+  if (t === 12) {
+    const nested = block.children?.length ? '\n' + ftRenderChildren(block.children, index, depth + 1) : '';
+    return `${indent}- ${ftTextElements(block.text?.elements)}${nested}\n`;
+  }
+  if (t === 13) {
+    const nested = block.children?.length ? '\n' + ftRenderChildren(block.children, index, depth + 1) : '';
+    return `${indent}${orderedNum}. ${ftTextElements(block.text?.elements)}${nested}\n`;
+  }
+  if (t === 14) {
+    const langId = block.code?.style?.language ?? 1;
+    const code = ftTextElements(block.code?.elements || []);
+    return `\`\`\`${FT_CODE_LANG[langId] ?? ''}\n${code}\n\`\`\`\n`;
+  }
+  if (t === 15) return `> ${ftTextElements(block.text?.elements)}\n\n`;
+  if (t === 16) {
+    const done = block.text?.style?.done ? '[x]' : '[ ]';
+    return `- ${done} ${ftTextElements(block.text?.elements)}\n`;
+  }
+  if (t === 21) return '\n---\n\n';
+  if (t === 26) {
+    const token = block.image?.token || '';
+    const src = token
+      ? `https://internal-api-drive-stream.feishu.cn/space/api/box/stream/download/all/${token}`
+      : '';
+    return `![${token}](${src})\n\n`;
+  }
+  if (t === 30) return ftRenderTable(block, index) + '\n';
+  if (t === 31) return null;
+  const fallback = ftTextElements(block.text?.elements || []);
+  return fallback ? fallback + '\n' : null;
+}
+
+function ftRenderChildren(childIds, index, depth) {
+  const lines = [];
+  let n = 1;
+  for (const id of childIds) {
+    const block = index.get(id);
+    if (!block) continue;
+    if (block.block_type === 13) {
+      const line = ftBlockToMd(block, index, depth, n++);
+      if (line != null) lines.push(line);
+    } else {
+      n = 1;
+      const line = ftBlockToMd(block, index, depth);
+      if (line != null) lines.push(line);
+    }
+  }
+  return lines.join('');
+}
+
+function ftRenderTable(block, index) {
+  const { row_size = 0, column_size = 0 } = block.table?.property || {};
+  const cellIds = block.table?.cells || [];
+  if (!row_size || !column_size) return '';
+  const rows = Array.from({ length: row_size }, () => Array(column_size).fill(''));
+  for (let r = 0; r < row_size; r++) {
+    for (let c = 0; c < column_size; c++) {
+      const cell = index.get(cellIds[r * column_size + c]);
+      if (!cell) continue;
+      rows[r][c] = ftRenderChildren(cell.children || [], index, 0).replace(/\n+/g, ' ').trim();
+    }
+  }
+  const out = ['| ' + rows[0].join(' | ') + ' |', '| ' + Array(column_size).fill('---').join(' | ') + ' |'];
+  for (let r = 1; r < row_size; r++) out.push('| ' + rows[r].join(' | ') + ' |');
+  return out.join('\n');
+}
+
+function feishuBlocksToMd(blocks) {
+  const index = new Map();
+  for (const b of blocks) if (b.block_id) index.set(b.block_id, b);
+  const root = blocks.find(b => b.block_type === 1) || blocks[0];
+  if (!root) return blocks.map(b => ftBlockToMd(b, index, 0)).filter(Boolean).join('\n');
+  return ftRenderChildren(root.children || [], index, 0).replace(/\n{3,}/g, '\n\n').trim();
+}
+
+// 从 API 响应 JSON 中递归查找 block 数组（block_type schema）
+function findFeishuBlocks(obj, depth = 0) {
+  if (depth > 5 || !obj || typeof obj !== 'object') return [];
+  if (obj.block_id && typeof obj.block_type === 'number') return [obj];
+  if (Array.isArray(obj)) {
+    const flat = [];
+    for (const item of obj) flat.push(...findFeishuBlocks(item, depth + 1));
+    return flat;
+  }
+  for (const c of [obj.items, obj.blocks, obj.data?.items, obj.data?.blocks,
+                    obj.data?.document?.blocks, obj.result?.data?.items]) {
+    if (c) { const found = findFeishuBlocks(c, depth + 1); if (found.length) return found; }
+  }
+  return [];
+}
+
+// 从 wiki node API 响应中找出真实 docx token 和标题
+function findWikiObjToken(obj, wikiToken, depth = 0) {
+  if (depth > 6 || !obj || typeof obj !== 'object') return null;
+  if (obj.obj_token && (obj.node_token === wikiToken || obj.wiki_token === wikiToken || depth > 0)) return obj;
+  if (obj.obj_token && obj.title) return obj;
+  if (Array.isArray(obj)) {
+    for (const item of obj) { const f = findWikiObjToken(item, wikiToken, depth + 1); if (f) return f; }
+    return null;
+  }
+  for (const key of Object.keys(obj)) {
+    const f = findWikiObjToken(obj[key], wikiToken, depth + 1);
+    if (f) return f;
+  }
+  return null;
+}
+
+// ── Main ─────────────────────────────────────────────────────────────────────
+console.log('╔════════════════════════════════════════════╗');
+console.log('║  一堂文档采集工具                          ║');
+console.log('╚════════════════════════════════════════════╝');
+console.log(`📎 URL:    ${TARGET_URL}`);
+if (isYitang) {
+  console.log(`📄 ACL:    ${ACL}`);
+  console.log(`🔖 DocID:  ${DOC_ID}`);
+} else {
+  const ftToken = TARGET_URL.match(/\/(wiki|docx)\/([A-Za-z0-9_-]+)/);
+  console.log(`🔖 Type:   feishu ${ftToken?.[1] || ''}  Token: ${ftToken?.[2] || ''}`);
+}
+console.log('');
+
+const hasSavedCookies = existsSync(COOKIE_FILE);
+const browser = await chromium.launch({ headless: false, args: ['--window-size=820,720'] });
+const context  = await browser.newContext();
+
+if (hasSavedCookies) {
+  const saved = JSON.parse(readFileSync(COOKIE_FILE, 'utf-8'));
+  await context.addCookies(saved);
+  console.log(`✅ 已加载保存的 Cookie（${saved.length} 条），跳过扫码登录`);
+} else {
+  console.log('首次运行，请在弹出的浏览器中完成登录...');
+}
+
+const page = await context.newPage();
+
+let apiResponseText = null;
+const feishuApiResponses = [];
+
+// 飞书原生 docx/wiki API 匹配模式
+const FEISHU_DOC_RE = [
+  /\/docx\/v\d+\/documents\/[A-Za-z0-9]+\/blocks/,
+  /\/space\/api\/docs\/v\d+\/raw_api/,
+  /apiName=DocxV\d+Document/i,
+];
+const FEISHU_WIKI_RE = [
+  /\/wiki\/v\d+\/spaces?\/[^/]+\/nodes?/,
+  /\/wiki\/api\/.*\/(get_?node|node_meta|node_info)/,
+  /\/space\/api\/wiki\/v\d+\/.*node/,
+  /apiName=WikiV\d+/i,
+];
+
+page.on('response', async (resp) => {
+  const u = resp.url();
+
+  if (isFeishu) {
+    const isDoc  = FEISHU_DOC_RE.some(p => p.test(u));
+    const isWiki = FEISHU_WIKI_RE.some(p => p.test(u));
+    // 跳过非 JSON、静态资源、已知无用的请求
+    const ct = resp.headers()['content-type'] || '';
+    if (!ct.includes('json') && !ct.includes('text/plain')) return;
+    if (resp.status() < 200 || resp.status() >= 300) return;
+
+    try {
+      const text = await resp.text();
+      let json; try { json = JSON.parse(text); } catch { return; }
+
+      if (isDoc || isWiki) {
+        console.log(`\n🎯 捕获飞书 API (${isWiki ? 'wiki' : 'doc'})：${u.split('?')[0]}  (${text.length} 字节)`);
+        feishuApiResponses.push({ url: u, data: json, kind: isWiki ? 'wiki' : 'doc' });
+        return;
+      }
+
+      // 兜底：扫描任意 JSON 响应中是否含有 block 数据
+      const blocks = findFeishuBlocks(json);
+      if (blocks.length > 0) {
+        console.log(`\n📦 兜底捕获 block 数据：${u.split('?')[0]}  (${blocks.length} 个 block)`);
+        feishuApiResponses.push({ url: u, data: json, kind: 'doc' });
+      }
+    } catch {}
+    return;
+  }
+
+  // yitang 路径
+  if (u.includes('get-doc-blocks') || u.includes('get-blocks')) {
+    try {
+      const text = await resp.text();
+      // 忽略"未登录"错误响应（code 10）
+      try {
+        const parsed = JSON.parse(text);
+        if (parsed.code === 10) {
+          console.log(`\n⚠️  API 返回未登录，等待真实响应...`);
+          return;
+        }
+      } catch {}
+      apiResponseText = text;
+      console.log(`\n🎯 捕获 API 响应：${u.split('?')[0]}  (${text.length} 字节)`);
+    } catch {}
+  }
+});
+
+console.log('🌐 正在打开文档页面...');
+await page.goto(TARGET_URL, { waitUntil: 'load', timeout: 30000 });
+
+// 检测是否需要登录
+const needLogin = async () => {
+  const u = page.url();
+  if (isFeishu) {
+    // 飞书登录页：passport.feishu.cn / accounts.feishu.cn / /login
+    return u.includes('passport.feishu') || u.includes('accounts.feishu') || u.includes('/login');
+  }
+  // yitang 路径
+  if (u.includes('/login') || u.includes('sso.')) return true;
+  const tok = await readToken();
+  if (!tok && hasSavedCookies) return true;
+  return false;
+};
+
+if (await needLogin()) {
+  console.log('⚠️  需要登录，正在跳转...');
+  if (isFeishu) {
+    if (!page.url().includes('passport') && !page.url().includes('accounts') && !page.url().includes('/login')) {
+      await page.goto(TARGET_URL, { waitUntil: 'load', timeout: 30000 });
+    }
+    console.log('请在浏览器中完成飞书登录...');
+    await page.waitForFunction(
+      () => !location.href.includes('passport') && !location.href.includes('accounts') && !location.href.includes('/login'),
+      { timeout: 300000 }
+    );
+  } else {
+    if (!page.url().includes('/login') && !page.url().includes('sso.')) {
+      await page.goto(LOGIN_URL, { waitUntil: 'load', timeout: 30000 });
+    }
+    console.log('请在浏览器中完成登录（微信扫码或手机验证码）...');
+    await page.waitForURL('**/fs-doc/**', { timeout: 300000 });
+  }
+  console.log('\n✅ 登录成功');
+  await page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => {});
+}
+
+// 保存最新 Cookie
+const cookies = await context.cookies();
+writeFileSync(COOKIE_FILE, JSON.stringify(cookies, null, 2));
+console.log(`💾 Cookie 已保存`);
+
+// ── 飞书原生路径 ───────────────────────────────────────────────────────────────
+if (isFeishu) {
+  if (feishuApiResponses.length === 0) {
+    console.log('⏳ 等待飞书 API 数据加载...');
+    await page.waitForTimeout(8000);
+  }
+  if (feishuApiResponses.length === 0) {
+    console.log('🔄 重新加载页面...');
+    await page.goto(TARGET_URL, { waitUntil: 'networkidle', timeout: 30000 });
+    await page.waitForTimeout(5000);
+  }
+  if (feishuApiResponses.length === 0) {
+    console.error('❌ 未捕获到飞书 API 响应，请检查登录状态或文档权限');
+    await browser.close();
+    process.exit(1);
+  }
+
+  // 提取 wiki node 元信息（wiki 页面需先拿 obj_token）
+  let wikiMeta = null;
+  const ftToken = TARGET_URL.match(/\/(wiki|docx)\/([A-Za-z0-9_-]+)/)?.[2] || '';
+  if (TARGET_URL.includes('/wiki/')) {
+    for (const item of feishuApiResponses) {
+      if (item.kind === 'wiki') {
+        wikiMeta = findWikiObjToken(item.data, ftToken);
+        if (wikiMeta) {
+          console.log(`📎 wiki → docx token: ${wikiMeta.obj_token}  标题: ${wikiMeta.title || '(未知)'}`);
+          break;
+        }
+      }
+    }
+
+    // 如果是 wiki 页面且找到了 docx token，需要导航到 docx URL 来获取实际内容
+    if (wikiMeta?.obj_token) {
+      const docxUrl = TARGET_URL.replace(/\/wiki\/[^/]+/, `/docx/${wikiMeta.obj_token}`);
+      console.log(`🔄 正在加载文档内容: ${docxUrl}`);
+      feishuApiResponses.length = 0; // 清空之前的 wiki API 响应
+      await page.goto(docxUrl, { waitUntil: 'load', timeout: 30000 });
+      await page.waitForTimeout(3000);
+
+      if (feishuApiResponses.length === 0) {
+        console.log('⏳ 等待文档 API 数据加载...');
+        await page.waitForTimeout(5000);
+      }
+    }
+  }
+
+  // 提取 blocks（去重）
+  let docxBlocks = [];
+  for (const item of feishuApiResponses) {
+    if (item.kind !== 'doc') continue;
+    const found = findFeishuBlocks(item.data);
+    if (found.length) docxBlocks.push(...found);
+  }
+  const seenIds = new Set();
+  docxBlocks = docxBlocks.filter(b => {
+    if (!b.block_id || seenIds.has(b.block_id)) return false;
+    seenIds.add(b.block_id);
+    return true;
+  });
+  console.log(`📦 共提取 ${docxBlocks.length} 个 block`);
+
+  if (docxBlocks.length === 0) {
+    console.error('❌ 未获取到有效 block 数据');
+    await browser.close();
+    process.exit(1);
+  }
+
+  const pageTitle = wikiMeta?.title || await page.title();
+  const markdown  = feishuBlocksToMd(docxBlocks);
+  const output    = `# ${pageTitle}\n\n> Source: ${TARGET_URL}\n\n${markdown}`;
+  const safeTitle = pageTitle.replace(/[/\\:*?"<>|]/g, '_').trim();
+  const outDir    = resolve('output');
+  if (!existsSync(outDir)) mkdirSync(outDir, { recursive: true });
+  const outputPath = resolve(customOutput || `output/${safeTitle || 'output'}.md`);
+  writeFileSync(outputPath, output, 'utf-8');
+
+  console.log('');
+  console.log('╔════════════════════════════════════════════╗');
+  console.log(`║  ✅ 完成！共 ${output.length} 字符`);
+  console.log(`║  📄 ${outputPath}`);
+  console.log('╚════════════════════════════════════════════╝');
+  await browser.close();
+  process.exit(0);
+}
+
+// 读取 TOKEN（函数，方便重登后复用）
+async function readToken() {
+  return page.evaluate(() => {
+    try {
+      const raw = localStorage.getItem('#|g.TOKEN')
+        || Object.entries(localStorage).find(([k]) => k.includes('TOKEN'))?.[1]
+        || null;
+      if (!raw) return null;
+      return raw.startsWith('{') ? JSON.parse(raw).$v || null : raw;
+    } catch { return null; }
+  }).catch(() => null);
+}
+
+// 读取 TOKEN
+let capturedToken = await page.evaluate(() => {
+  try {
+    const raw = localStorage.getItem('#|g.TOKEN')
+      || Object.entries(localStorage).find(([k]) => k.includes('TOKEN'))?.[1]
+      || null;
+    if (!raw) return null;
+    return raw.startsWith('{') ? JSON.parse(raw).$v || null : raw;
+  } catch { return null; }
+}).catch(() => null);
+console.log('🔑 TOKEN:', capturedToken ? capturedToken.substring(0, 8) + '...' : '未找到');
+
+if (!apiResponseText) {
+  console.log('⏳ 等待 API 数据加载...');
+  await page.waitForTimeout(10000);
+}
+if (!apiResponseText) {
+  console.log('🔄 重新加载页面...');
+  await page.goto(TARGET_URL, { waitUntil: 'networkidle', timeout: 30000 });
+  await page.waitForTimeout(5000);
+}
+if (!apiResponseText) {
+  console.error('❌ 未捕获到 API 响应，可能未登录或文档受限');
+  await browser.close();
+  process.exit(1);
+}
+
+if (!apiResponseText) {
+  console.error('❌ 仍未获取到有效数据，请检查是否已完成登录或文档是否有访问权限');
+  await browser.close();
+  process.exit(1);
+}
+
+// ── 解密 ──────────────────────────────────────────────────────────────────────
+let docData;
+const envelope = JSON.parse(apiResponseText);
+const payload  = envelope.data;
+
+if (capturedToken && typeof payload === 'string') {
+  try {
+    const key = CryptoJS.enc.Utf8.parse(CryptoJS.MD5(capturedToken).toString());
+    const iv  = CryptoJS.enc.Utf8.parse(capturedToken);
+    const dec = CryptoJS.AES.decrypt(payload.trim(), key, {
+      iv, mode: CryptoJS.mode.CBC, padding: CryptoJS.pad.Pkcs7,
+    });
+    docData = JSON.parse(dec.toString(CryptoJS.enc.Utf8));
+    console.log('🔓 AES 解密成功');
+  } catch (e) {
+    console.log('⚠️  AES 解密失败：', e.message);
+    docData = typeof payload === 'object' ? payload : envelope;
+  }
+} else {
+  docData = typeof payload === 'object' ? payload : envelope;
+  console.log('📄 直接使用响应数据');
+}
+
+// ── 转换并保存 ────────────────────────────────────────────────────────────────
+const pageTitle  = await page.title();
+const markdown   = blocksToMarkdown(docData.blocks);
+const output     = `# ${pageTitle}\n\n> Source: ${TARGET_URL}\n\n${markdown}`;
+
+// 输出文件名：优先用参数，其次用标题，最后回退 output.md
+const safeTitle  = pageTitle.replace(/[/\\:*?"<>|]/g, '_').trim();
+const outDir2    = resolve('output');
+if (!existsSync(outDir2)) mkdirSync(outDir2, { recursive: true });
+const outputPath = resolve(customOutput || `output/${safeTitle || 'output'}.md`);
+writeFileSync(outputPath, output, 'utf-8');
+
+console.log('');
+console.log('╔════════════════════════════════════════════╗');
+console.log(`║  ✅ 完成！共 ${output.length} 字符`);
+console.log(`║  📄 ${outputPath}`);
+console.log('╚════════════════════════════════════════════╝');
+
+await browser.close();

clean-html.mjs

@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+
+import fs from 'fs';
+
+const filePath = process.argv[2];
+if (!filePath) {
+  console.error('Usage: node clean-html.mjs <file-path>');
+  process.exit(1);
+}
+
+let content = fs.readFileSync(filePath, 'utf-8');
+
+// 1. Remove <title> tags but keep content
+content = content.replace(/<title>(.*?)<\/title>/gs, '# $1');
+
+// 2. Convert <callout emoji="..."> to blockquote
+content = content.replace(/<callout emoji="([^"]+)">\s*/g, '> 📌 ');
+content = content.replace(/<\/callout>/g, '');
+
+// 3. Remove complex grid/column structures with embedded images
+// These are usually image galleries that we can't display properly in markdown
+content = content.replace(/<grid>.*?<\/grid>/gs, (match) => {
+  // Try to extract any alt text or descriptions
+  const imgMatches = match.matchAll(/alt="([^"]+)"/g);
+  const descriptions = [];
+  for (const m of imgMatches) {
+    descriptions.push(m[1]);
+  }
+
+  if (descriptions.length > 0) {
+    return '\n> 📷 [图片集]\n\n';
+  }
+  return '';
+});
+
+// 4. Remove any remaining HTML tags (but preserve content where possible)
+content = content.replace(/<figure[^>]*>.*?<\/figure>/gs, '');
+content = content.replace(/<source[^>]*\/?>/g, '');
+content = content.replace(/<column[^>]*>/g, '');
+content = content.replace(/<\/column>/g, '');
+
+// 5. Clean up excessive blank lines (more than 2 consecutive)
+content = content.replace(/\n{4,}/g, '\n\n\n');
+
+// 6. Fix any escaped underscores in URLs
+content = content.replace(/\\_/g, '_');
+
+fs.writeFileSync(filePath, content, 'utf-8');
+console.log(`✅ Cleaned HTML tags from: ${filePath}`);

feishu-copy.mjs

@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+/**
+ * 飞书文档复制工具
+ *
+ * 功能：
+ *   1. 读取源飞书文档（wiki/docx）的完整内容
+ *   2. 在你的飞书账号下创建新文档
+ *   3. 复制所有文本、格式、图片到新文档
+ *
+ * 用法：node feishu-copy.mjs <source_wiki_url> [目标文件夹token]
+ *
+ * 示例：
+ *   node feishu-copy.mjs https://xxx.feishu.cn/wiki/ABC123
+ *   node feishu-copy.mjs https://xxx.feishu.cn/wiki/ABC123 fldcnXXXXXXX
+ */
+import { execSync } from 'child_process';
+import { unlinkSync } from 'fs';
+import { basename } from 'path';
+
+const argv = process.argv.slice(2);
+if (!argv[0] || argv[0] === '--help' || argv[0] === '-h') {
+  console.log(`
+用法：node feishu-copy.mjs <source_url> [目标文件夹token]
+
+参数：
+  source_url        源文档 URL（wiki 或 docx）
+  目标文件夹token    可选，新文档放在指定文件夹下
+
+示例：
+  node feishu-copy.mjs https://yitanger.feishu.cn/wiki/R9NywV2i9izjuqkPd63cWuafn4g
+  node feishu-copy.mjs https://yitanger.feishu.cn/wiki/ABC123 fldcnXXXXXXX
+`);
+  process.exit(0);
+}
+
+const sourceUrl = argv[0].trim();
+const targetFolder = argv[1]?.trim();
+
+console.log('╔════════════════════════════════════════════╗');
+console.log('║  飞书文档复制工具                          ║');
+console.log('╚════════════════════════════════════════════╝');
+console.log(`📎 源文档: ${sourceUrl}\n`);
+
+// ── 1. 解析源文档 token ───────────────────────────────────────────────────────
+console.log('🔍 正在解析源文档...');
+let obj_token, title;
+
+if (sourceUrl.includes('/wiki/')) {
+  // wiki → 先解析成 docx token
+  const nodeInfo = larkJson(`lark-cli wiki +node-get --node-token "${sourceUrl}"`);
+  if (!nodeInfo.ok) die('获取 wiki 节点失败', nodeInfo);
+  obj_token = nodeInfo.data.obj_token;
+  title = nodeInfo.data.title;
+} else if (sourceUrl.includes('/docx/') || sourceUrl.match(/[A-Za-z0-9]{20,}/)) {
+  // docx URL 或直接是 token
+  obj_token = sourceUrl.match(/[A-Za-z0-9]{20,}/)?.[0] || sourceUrl;
+  // 先 fetch 一次拿标题
+  const docInfo = larkJson(`lark-cli docs +fetch --doc ${obj_token} --api-version v2`);
+  if (!docInfo.ok) die('获取文档失败', docInfo);
+  title = docInfo.data.document.title || '未命名文档';
+} else {
+  die('不支持的 URL 格式', { url: sourceUrl });
+}
+
+console.log(`✅ 标题：${title}`);
+console.log(`✅ obj_token：${obj_token}\n`);
+
+// ── 2. 导出源文档为 Markdown ─────────────────────────────────────────────────
+console.log('📄 正在导出源文档为 Markdown...');
+const exportResult = larkJson(
+  `lark-cli drive +export --token ${obj_token} --doc-type docx --file-extension markdown --output-dir . --overwrite`
+);
+if (!exportResult.ok) die('导出失败', exportResult);
+
+const mdPath = exportResult.data.saved_path;
+const mdFile = basename(mdPath); // 相对文件名，lark-cli 只接受相对路径
+console.log(`✅ 已导出（${mdFile}）\n`);
+
+// ── 3. 导入为新文档 ──────────────────────────────────────────────────────────
+console.log('📝 正在导入为新文档...');
+const importCmd = targetFolder
+  ? `lark-cli drive +import --file "@${mdFile}" --type markdown --folder-token ${targetFolder}`
+  : `lark-cli drive +import --file "@${mdFile}" --type markdown`;
+
+const cwd = process.cwd();
+const importResult = larkJson(importCmd, { cwd });
+if (!importResult.ok) die('导入失败', importResult);
+
+const newDocToken = importResult.data?.token || importResult.data?.document_id;
+if (!newDocToken) die('导入返回异常', importResult);
+
+// 删除临时导出文件
+try { unlinkSync(mdPath); } catch {}
+
+console.log(`✅ 新文档已创建并写入内容`);
+console.log(`✅ 新文档 token：${newDocToken}\n`);
+
+console.log('╔════════════════════════════════════════════╗');
+console.log(`║  ✅ 复制完成！`);
+console.log(`║  📄 新文档: https://feishu.cn/docx/${newDocToken}`);
+console.log(`║  💡 图片为飞书链接，需登录后可见`);
+console.log('╚════════════════════════════════════════════╝');
+
+// ── helpers ──────────────────────────────────────────────────────────────────
+function larkJson(cmd, opts = {}) {
+  try {
+    return JSON.parse(execSync(cmd, { encoding: 'utf-8', ...opts }));
+  } catch (e) {
+    try { return JSON.parse(e.stderr || e.stdout || '{}'); } catch { return {}; }
+  }
+}
+
+function die(msg, info) {
+  console.error(`❌ ${msg}:`, info?.error?.message || JSON.stringify(info));
+  process.exit(1);
+}

feishu-to-md.mjs

@@ -0,0 +1,125 @@
+#!/usr/bin/env node
+/**
+ * 飞书文档 → Markdown 转换工具（含图片本地化）
+ *
+ * 流程：
+ *   1. wiki +node-get  → 解析 obj_token
+ *   2. drive +export markdown → 拿到含 feishu.cn/file/<token> 的 Markdown
+ *   3. docs +media-preview  → 逐张下载图片到本地
+ *   4. 替换图片链接为相对路径，保存最终 Markdown
+ *
+ * 用法：node feishu-to-md.mjs <wiki_url_or_token> [输出目录]
+ */
+import { execSync } from 'child_process';
+import { writeFileSync, readFileSync, mkdirSync, existsSync, unlinkSync } from 'fs';
+import { resolve, join, basename } from 'path';
+
+const argv = process.argv.slice(2);
+if (!argv[0] || argv[0] === '--help' || argv[0] === '-h') {
+  console.log(`
+用法：node feishu-to-md.mjs <wiki_url_or_token> [输出目录]
+
+示例：
+  node feishu-to-md.mjs https://yitanger.feishu.cn/wiki/R9NywV2i9izjuqkPd63cWuafn4g
+  node feishu-to-md.mjs R9NywV2i9izjuqkPd63cWuafn4g
+`);
+  process.exit(0);
+}
+
+const input   = argv[0].trim();
+const outDir  = argv[1] ? resolve(argv[1]) : resolve('./output');
+const wikiUrl = input.startsWith('http') ? input : `https://yitanger.feishu.cn/wiki/${input}`;
+
+// lark-cli 命令必须在 outDir 下执行（+media-preview 只接受相对路径）
+process.chdir(outDir);
+
+console.log('╔════════════════════════════════════════════╗');
+console.log('║  飞书文档 → Markdown 转换工具              ║');
+console.log('╚════════════════════════════════════════════╝');
+console.log(`📎 URL: ${wikiUrl}\n`);
+
+// ── 1. 解析 wiki node → obj_token ────────────────────────────────────────────
+console.log('🔍 正在获取 wiki 节点信息...');
+const nodeInfo = larkJson(`lark-cli wiki +node-get --node-token "${wikiUrl}"`);
+if (!nodeInfo.ok) die('获取 wiki 节点失败', nodeInfo);
+
+const { obj_token, title } = nodeInfo.data;
+console.log(`✅ 标题：${title}`);
+console.log(`✅ obj_token：${obj_token}\n`);
+
+const safeTitle = title.replace(/[/\\:*?"<>|]/g, '_').trim();
+const imgDir    = join(outDir, `${safeTitle}_files`);
+if (!existsSync(imgDir)) mkdirSync(imgDir, { recursive: true });
+
+// ── 2. 导出 Markdown ──────────────────────────────────────────────────────────
+console.log('📄 正在导出 Markdown...');
+const exportResult = larkJson(
+  `lark-cli drive +export --token ${obj_token} --doc-type docx --file-extension markdown --output-dir . --overwrite`
+);
+if (!exportResult.ok) die('导出失败', exportResult);
+
+const mdPath = exportResult.data.saved_path;
+let mdContent = readFileSync(mdPath, 'utf-8');
+console.log(`✅ 导出成功（${mdContent.length} 字符）\n`);
+
+// ── 3. 下载图片 ────────────────────────────────────────────────────────────────
+const TOKEN_RE = /!\[[^\]]*\]\(https:\/\/feishu\.cn\/file\/([A-Za-z0-9]+)\)/g;
+const tokens = [...new Set([...mdContent.matchAll(TOKEN_RE)].map(m => m[1]))];
+console.log(`🖼️  发现 ${tokens.length} 张图片，开始下载...`);
+
+const tokenToLocal = new Map();
+let n = 0;
+for (const token of tokens) {
+  n++;
+  const relPath  = `${safeTitle}_files/image_${n}.png`;
+  try {
+    execSync(`lark-cli docs +media-preview --token ${token} --output "${relPath}" --overwrite`, {
+      stdio: 'ignore',
+    });
+    tokenToLocal.set(token, relPath);
+    console.log(`  ✓ image_${n}.png`);
+  } catch {
+    tokenToLocal.set(token, `https://feishu.cn/file/${token}`);
+    console.log(`  ✗ image_${n}.png（下载失败，保留原链接）`);
+  }
+}
+console.log('');
+
+// ── 4. 替换图片链接并清理 ─────────────────────────────────────────────────────
+mdContent = mdContent.replace(
+  /!\[([^\]]*)\]\(https:\/\/feishu\.cn\/file\/([A-Za-z0-9]+)\)/g,
+  (_, alt, token) => `![${alt}](${tokenToLocal.get(token) ?? `https://feishu.cn/file/${token}`})`
+);
+
+// 清理 <title> 标签和 <grid>/<column> 残留
+mdContent = mdContent
+  .replace(/<title>[\s\S]*?<\/title>\n?/g, '')
+  .replace(/<\/?grid[^>]*>\n?/g, '')
+  .replace(/<\/?column[^>]*>\n?/g, '')
+  .replace(/\n{3,}/g, '\n\n')
+  .trim();
+
+// 加首行标题和来源
+const finalContent = `# ${title}\n\n> Source: ${wikiUrl}\n\n${mdContent}`;
+writeFileSync(mdPath, finalContent, 'utf-8');
+
+console.log('╔════════════════════════════════════════════╗');
+console.log(`║  ✅ 完成！共 ${finalContent.length} 字符`);
+console.log(`║  📄 ${mdPath}`);
+console.log(`║  🖼️  ${imgDir} (${tokens.length} 张)`);
+console.log('╚════════════════════════════════════════════╝');
+
+// ── helpers ──────────────────────────────────────────────────────────────────
+function larkJson(cmd) {
+  try {
+    return JSON.parse(execSync(cmd, { encoding: 'utf-8' }));
+  } catch (e) {
+    // lark-cli 失败时把 JSON 输出到 stderr
+    try { return JSON.parse(e.stderr || e.stdout || '{}'); } catch { return {}; }
+  }
+}
+
+function die(msg, info) {
+  console.error(`❌ ${msg}:`, info?.error?.message || JSON.stringify(info));
+  process.exit(1);
+}

fetch-feishu-wiki.mjs

@@ -0,0 +1,126 @@
+#!/usr/bin/env node
+/**
+ * Feishu Wiki 文档获取工具
+ *
+ * 用法：node fetch-feishu-wiki.mjs <WIKI_URL>
+ *
+ * 图片下载：使用 lark-cli docs +media-preview --as user 逐张下载
+ */
+
+import { execSync } from 'child_process';
+import { writeFileSync, existsSync, mkdirSync, readFileSync, renameSync } from 'fs';
+import { resolve } from 'path';
+
+// ── CLI ────────────────────────────────────────────────────────────────────────
+const args = process.argv.slice(2);
+if (!args[0] || args[0] === '--help' || args[0] === '-h') {
+  console.log('用法：node fetch-feishu-wiki.mjs <WIKI_URL>');
+  process.exit(0);
+}
+
+const WIKI_URL = args[0].trim();
+
+// ── 1. 获取 wiki 节点 ─────────────────────────────────────────────────────────
+console.log('🔍 正在获取 wiki 节点信息...');
+let docToken, docTitle;
+try {
+  const r = execSync(`lark-cli wiki +node-get --as bot --node-token ${WIKI_URL}`,
+    { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 });
+  const { ok, data, error } = JSON.parse(r);
+  if (!ok) { console.error('❌', error.message); process.exit(1); }
+  docToken = data.obj_token;
+  docTitle = (data.title || 'untitled').trim();
+} catch (e) { console.error('❌ 获取 wiki 节点失败:', e.message); process.exit(1); }
+console.log(`📄 文档标题: ${docTitle}`);
+console.log(`🔑 文档 token: ${docToken}`);
+
+// ── 2. 获取 Markdown ──────────────────────────────────────────────────────────
+console.log('📥 正在获取文档内容...');
+let markdown;
+try {
+  const r = execSync(
+    `lark-cli docs +fetch --as bot --api-version v2 --doc ${docToken} --doc-format markdown`,
+    { encoding: 'utf-8', maxBuffer: 50 * 1024 * 1024 });
+  const { ok, data, error } = JSON.parse(r);
+  if (!ok) { console.error('❌', error.message); process.exit(1); }
+  markdown = data.document.content;
+} catch (e) { console.error('❌ 获取文档内容失败:', e.message); process.exit(1); }
+console.log(`✅ Markdown 获取成功，共 ${markdown.length} 字符`);
+
+// ── 3. 提取图片 ───────────────────────────────────────────────────────────────
+const imageRegex = /!\[([^\]]*)\]\((https:\/\/feishu\.cn\/file\/([^)]+))\)/g;
+const images = [];
+let m;
+while ((m = imageRegex.exec(markdown)) !== null) {
+  images.push({ alt: m[1], url: m[2], token: m[3] });
+}
+console.log(`🖼️  找到 ${images.length} 张图片`);
+
+// ── 4. 目录准备 ───────────────────────────────────────────────────────────────
+const safeTitle = docTitle.replace(/[/\\:*?"<>|]/g, '_').trim();
+const outputDir = resolve('output');
+const imagesDir = resolve(outputDir, safeTitle);
+if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
+if (images.length > 0 && !existsSync(imagesDir)) mkdirSync(imagesDir, { recursive: true });
+
+// ── 5. 下载图片 ───────────────────────────────────────────────────────────────
+let updatedMarkdown = markdown;
+
+if (images.length > 0) {
+  console.log('   📥 下载图片中...');
+
+  for (let i = 0; i < images.length; i++) {
+    const { token, url } = images[i];
+    const imageFilename = `${i + 1}_${token}.png`;
+    const imageLocalPath = resolve(imagesDir, imageFilename);
+    const imageRelativePath = `./${safeTitle}/${imageFilename}`;
+
+    process.stdout.write(`   [${i + 1}/${images.length}] ${token.substring(0, 14)}... `);
+
+    try {
+      // lark-cli docs +media-preview 输出到当前目录
+      const tmpOutput = `_scraper_tmp_${i}.png`;
+      const r = execSync(
+        `lark-cli docs +media-preview --as user --token ${token} --output ./${tmpOutput} --overwrite`,
+        { encoding: 'utf-8', maxBuffer: 5 * 1024 * 1024, timeout: 30000 }
+      );
+      const parsed = JSON.parse(r);
+
+      if (parsed.ok) {
+        // 将临时文件移动到目标位置
+        const tmpPath = resolve(tmpOutput);
+        if (existsSync(tmpPath)) {
+          renameSync(tmpPath, imageLocalPath);
+          updatedMarkdown = updatedMarkdown.replace(url, imageRelativePath);
+          console.log(`✓ (${(parsed.data?.size_bytes || 0).toLocaleString()} bytes)`);
+        } else {
+          console.log('✗ (文件未生成)');
+        }
+      } else {
+        console.log(`✗ (${parsed.error?.message || 'unknown'})`);
+      }
+    } catch (e) {
+      console.log(`✗ (${e.message.split('\n')[0]})`);
+      // 清理可能残留的临时文件
+      try { const tmpPath = resolve(`_scraper_tmp_${i}.png`); if (existsSync(tmpPath)) renameSync(tmpPath, imageLocalPath); } catch {}
+    }
+  }
+
+  // 清理可能的残留临时文件
+  for (let i = 0; i < images.length; i++) {
+    try { const p = resolve(`_scraper_tmp_${i}.png`); if (existsSync(p)) { const dest = resolve(imagesDir, `${i + 1}_${images[i].token}.png`); renameSync(p, dest); } } catch {}
+  }
+}
+
+// ── 6. 保存 Markdown ──────────────────────────────────────────────────────────
+const outputPath = resolve(outputDir, `${safeTitle}.md`);
+writeFileSync(outputPath, updatedMarkdown, 'utf-8');
+
+console.log('');
+console.log('╔════════════════════════════════════════════╗');
+console.log(`║  ✅ 完成！`);
+console.log(`║  📄 ${outputPath}`);
+if (images.length > 0) {
+  console.log(`║  🖼️  图片: ${imagesDir}`);
+}
+console.log('╚════════════════════════════════════════════╝');

package.json

@@ -0,0 +1,23 @@
+{
+  "name": "scraper",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "type": "commonjs",
+  "dependencies": {
+    "crawlee": "^3.16.0",
+    "crypto-js": "^4.2.0",
+    "playwright": "^1.58.2",
+    "turndown": "^7.2.2"
+  },
+  "devDependencies": {
+    "@yao-pkg/pkg": "^6.14.2",
+    "esbuild": "^0.28.0"
+  }
+}