优化文档采集工具

- 输出文件名添加 YYMMDD_ 日期前缀，便于按时间排序
- 改用 storageState 完整保存登录态（cookies + localStorage），大幅减少重复扫码
- 增加 TOKEN 轮询宽限期，避免 SPA 异步加载时误判未登录
- 更新 README 文档记录

README.md

@@ -126,3 +126,24 @@ node auto-scrape.mjs <url>
 **Q: 图片下载失败？**
 
 飞书图片需要有效的个人授权。确认 `lark-cli auth status` 显示已登录，且包含 `docs:document.media:download` 权限。
+
+---
+
+## 更新记录
+
+### 2024-09-13
+
+**文件名优化**
+- 自动生成的 Markdown 文件名现在带有 `YYMMDD_` 日期前缀
+- 输出示例：`output/260913_《战略笃定》探索营1.md`
+- 便于按采集时间排序，同一文档的多次采集版本可区分
+
+**登录态持久化优化（大幅减少重复扫码）**
+- 改用 Playwright `storageState` 完整保存登录态（cookies + localStorage）
+- 修复了原先只存 cookie 导致每次都要重新扫码的问题（一堂平台的 API 鉴权 TOKEN 存在 localStorage 中）
+- 首次登录后，后续运行基本无需再扫码，直到服务端会话过期（通常数天）
+- `cookies.json` 从旧格式自动升级为新格式（兼容已有文件）
+
+**登录判定改进**
+- 增加 5 秒轮询宽限期，避免 SPA 异步写入 TOKEN 时误判为未登录
+- 减少因页面加载时序导致的不必要登录提示

auto-scrape.mjs

@@ -17,6 +17,7 @@ import { writeFileSync, readFileSync, existsSync, mkdirSync } from 'fs';
 import { resolve, dirname } from 'path';
 import { execSync } from 'child_process';
 import { fileURLToPath } from 'url';
+import { createInterface } from 'readline/promises';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -31,7 +32,12 @@ if (existsSync(bundledBrowsers)) {
 }
 
 // ── CLI 参数解析 ──────────────────────────────────────────────────────────────
-const args = process.argv.slice(2);
+let args = process.argv.slice(2);
+if (!args[0]) {
+  const rl = createInterface({ input: process.stdin, output: process.stdout });
+  args = [(await rl.question('请输入一堂文档链接：')).trim()];
+  rl.close();
+}
 if (!args[0] || args[0] === '--help' || args[0] === '-h') {
   console.log(`
 用法：node auto-scrape.mjs <URL> [输出文件名]
@@ -43,14 +49,34 @@ if (!args[0] || args[0] === '--help' || args[0] === '-h') {
   node auto-scrape.mjs https://yitang.top/fs-doc/abc/docId
   node auto-scrape.mjs https://your-tenant.feishu.cn/wiki/WikiNodeTokenExample
   node auto-scrape.mjs https://yitang.top/fs-doc/abc/docId 课程笔记.md
+  node auto-scrape.mjs --clean 已有文档.md
 `);
   process.exit(0);
 }
 
+// ── --clean：对已生成的 Markdown 文件执行清洗（不启动浏览器）──────────────────
+if (args[0] === '--clean') {
+  const file = resolve(args[1] || '');
+  if (!args[1] || !existsSync(file)) {
+    console.error('用法：node auto-scrape.mjs --clean <markdown文件>');
+    process.exit(1);
+  }
+  const md = readFileSync(file, 'utf-8');
+  writeFileSync(file, cleanMarkdown(md), 'utf-8');
+  console.log(`✅ 已清洗：${file}`);
+  process.exit(0);
+}
+
 const TARGET_URL  = args[0].trim();
 const customOutput = args[1];
 const COOKIE_FILE  = resolve('cookies.json');
 
+// ── 文件名日期前缀（YYMMDD_），便于按采集时间排序 ─────────────────────────────
+function datePrefix(d = new Date()) {
+  const p = n => String(n).padStart(2, '0');
+  return `${p(d.getFullYear() % 100)}${p(d.getMonth() + 1)}${p(d.getDate())}`;
+}
+
 // ── 判断 URL 类型 ──────────────────────────────────────────────────────────────
 const isFeishuWiki = /feishu\.cn\/wiki\//.test(TARGET_URL);
 const isFeishuDocx = /feishu\.cn\/docx\//.test(TARGET_URL);
@@ -91,6 +117,60 @@ if (isYitang) {
   LOGIN_URL = `https://yitang.top/login?after_login=%2Ffs-doc%2F${ACL}%2F${DOC_ID}`;
 }
 
+// ── Markdown 清洗（CommonMark 定界符合规）────────────────────────────────────
+// 1) 相邻同样式文本块合并：**甲****乙** → **甲乙**
+// 2) 加粗/斜体/删除线以中文标点结束、后接汉字/字母/数字时，闭合标记后补空格：**标签：** 研发
+// 3) 标题行去除 **、*、~~ 等格式标记残留
+function stripHeadingMarkers(line) {
+  const m = line.match(/^(#{1,6}\s+)(.*)$/);
+  if (!m) return line;
+  let inner = m[2], prev;
+  do {
+    prev = inner;
+    inner = inner
+      .replace(/\*\*([^*]+)\*\*/g, '$1')
+      .replace(/__([^_]+)__/g, '$1')
+      .replace(/~~([^~]+)~~/g, '$1')
+      .replace(/(^|[^*])\*([^*]+)\*/g, '$1$2');
+  } while (inner !== prev);
+  return m[1] + inner.trimEnd();
+}
+
+function mergeAdjacentSpans(line) {
+  let prev;
+  do {
+    prev = line;
+    line = line
+      .replace(/\*\*([^*\n]+)\*\*\*\*([^*\n]+)\*\*/g, '**$1$2**')
+      .replace(/\*\*\*([^*\n]+?)\*\*\*\*\*\*([^*\n]+?)\*\*\*/g, '***$1$2***')
+      .replace(/~~([^~\n]+)~~~~([^~\n]+)~~/g, '~~$1$2~~');
+  } while (line !== prev);
+  return line;
+}
+
+function fixClosingDelimiters(line) {
+  // CommonMark 侧翼规则：闭合定界符前为标点、后紧跟字母/数字（含汉字）时无法闭合，需补空格
+  const addSpace = (whole, marker, inner) =>
+    /[\p{P}\p{S}]/u.test(inner.slice(-1)) ? `${marker}${inner}${marker} ` : whole;
+  line = line.replace(/(?<!\*)\*\*\*([^*\n]+?)\*\*\*(?!\*)(?=[\p{L}\p{N}])/gu, (m, inner) => addSpace(m, '***', inner));
+  line = line.replace(/\*\*([^*\n]+)\*\*(?=[\p{L}\p{N}])/gu, (m, inner) => addSpace(m, '**', inner));
+  line = line.replace(/~~([^~\n]+)~~(?=[\p{L}\p{N}])/gu, (m, inner) => addSpace(m, '~~', inner));
+  line = line.replace(/(?<!\*)\*([^*\n]+)\*(?!\*)(?=[\p{L}\p{N}])/gu, (m, inner) => addSpace(m, '*', inner));
+  return line;
+}
+
+function cleanMarkdown(md) {
+  // 迭代至不动点，确保单遍扫描因定界符交叠漏掉的模式也被处理
+  let prev;
+  do {
+    prev = md;
+    md = md.split('\n')
+      .map(l => fixClosingDelimiters(mergeAdjacentSpans(stripHeadingMarkers(l))))
+      .join('\n');
+  } while (md !== prev);
+  return md;
+}
+
 // ── Feishu block → Markdown ──────────────────────────────────────────────────
 const T = { PAGE:1, TEXT:2, H1:3, H2:4, H3:5, H4:6, H5:7, H6:8,
             ORDERED:13, BULLET:12, TODO:17, CODE:14, QUOTE:15, DIVIDER:22,
@@ -99,11 +179,18 @@ const T = { PAGE:1, TEXT:2, H1:3, H2:4, H3:5, H4:6, H5:7, H6:8,
 
 function textRuns(elements) {
   if (!Array.isArray(elements)) return '';
-  return elements.map(el => {
+  // 相同样式的相邻 run 先合并，避免生成 **甲****乙** 这类相邻定界符
+  const runs = [];
+  for (const el of elements) {
     const run = el.text_run;
-    if (!run) return '';
+    if (!run) continue;
-    let t = run.content || '';
     const s = run.text_element_style || {};
+    const key = `${!!s.bold}|${!!s.italic}|${!!s.strikethrough}|${!!s.inline_code}|${s.link?.url || ''}`;
+    const last = runs[runs.length - 1];
+    if (last && last.key === key) last.content += run.content || '';
+    else runs.push({ key, content: run.content || '', style: s });
+  }
+  return runs.map(({ content: t, style: s }) => {
     if (s.bold)          t = `**${t}**`;
     if (s.italic)        t = `*${t}*`;
     if (s.strikethrough) t = `~~${t}~~`;
@@ -189,24 +276,34 @@ const FT_CODE_LANG = {
 };
 
 function ftTextElements(elements = []) {
-  return elements.map(el => {
+  // 相同样式的相邻 text_run 先合并，避免生成 **甲****乙** 这类相邻定界符
+  const segs = [];
+  for (const el of elements) {
     if (el.text_run) {
-      const content = el.text_run.content || '';
       const s = el.text_run.text_element_style || {};
-      let t = content;
+      const key = `${!!s.inline_code}|${!!s.bold}|${!!s.italic}|${!!s.strikethrough}|${s.link?.url || ''}`;
+      const last = segs[segs.length - 1];
+      if (last && last.key === key) last.content += el.text_run.content || '';
+      else segs.push({ key, content: el.text_run.content || '', style: s });
+    } else if (el.mention_doc) {
+      const { url = '', title = '' } = el.mention_doc;
+      segs.push({ key: null, text: url ? `[${title || url}](${url})` : title });
+    } else if (el.mention_user) {
+      segs.push({ key: null, text: `@${el.mention_user.user_id || 'user'}` });
+    } else {
+      segs.push({ key: null, text: '' });
+    }
+  }
+  return segs.map(seg => {
+    if (seg.key === null) return seg.text;
+    let t = seg.content;
+    const s = seg.style;
     if (s.inline_code) return `\`${t}\``;
     if (s.bold)          t = `**${t}**`;
     if (s.italic)        t = `*${t}*`;
     if (s.strikethrough) t = `~~${t}~~`;
     if (s.link?.url)     t = `[${t}](${decodeURIComponent(s.link.url)})`;
     return t;
-    }
-    if (el.mention_doc) {
-      const { url = '', title = '' } = el.mention_doc;
-      return url ? `[${title || url}](${url})` : title;
-    }
-    if (el.mention_user) return `@${el.mention_user.user_id || 'user'}`;
-    return '';
   }).join('');
 }
 
@@ -342,14 +439,47 @@ if (isYitang) {
 }
 console.log('');
 
-const hasSavedCookies = existsSync(COOKIE_FILE);
+// ── 加载已保存的登录态（cookies + localStorage）──────────────────────────────
-const browser = await chromium.launch({ headless: false, args: ['--window-size=820,720'] });
+// yitang 的 API 鉴权 TOKEN 存在 localStorage（#|g.TOKEN），不在 cookie 里，
-const context  = await browser.newContext();
+// 所以必须用 Playwright 的 storageState 整体持久化，只存 cookie 会导致每次都要重新扫码。
-
+function loadStorageState() {
-if (hasSavedCookies) {
+  if (!existsSync(COOKIE_FILE)) return null;
+  try {
     const saved = JSON.parse(readFileSync(COOKIE_FILE, 'utf-8'));
-  await context.addCookies(saved);
+    // 兼容旧格式：文件是纯 cookie 数组
-  console.log(`✅ 已加载保存的 Cookie（${saved.length} 条），跳过扫码登录`);
+    if (Array.isArray(saved)) return { cookies: saved, origins: [] };
+    if (Array.isArray(saved.cookies)) {
+      return { cookies: saved.cookies, origins: saved.origins || [] };
+    }
+  } catch (e) {
+    console.log(`⚠️  登录态文件解析失败（${e.message}），将重新登录`);
+  }
+  return null;
+}
+
+// 把当前 context 的 cookies + localStorage 落盘
+async function saveStorageState(ctx) {
+  try {
+    const state = await ctx.storageState();
+    writeFileSync(COOKIE_FILE, JSON.stringify(state, null, 2), 'utf-8');
+    const lsCount = (state.origins || []).reduce((n, o) => n + (o.localStorage?.length || 0), 0);
+    console.log(`💾 登录态已保存（Cookie ${state.cookies.length} 条，localStorage ${lsCount} 项）`);
+  } catch (e) {
+    console.log(`⚠️  登录态保存失败：${e.message}`);
+  }
+}
+
+const savedState = loadStorageState();
+const hasSavedCookies = !!savedState;
+const browser = await chromium.launch({ headless: false, args: ['--window-size=820,720'] });
+const context = await browser.newContext(savedState ? { storageState: savedState } : {});
+
+if (savedState) {
+  const lsCount = (savedState.origins || []).reduce((n, o) => n + (o.localStorage?.length || 0), 0);
+  console.log(`✅ 已加载保存的登录态（Cookie ${savedState.cookies.length} 条，localStorage ${lsCount} 项）`);
+  if (lsCount === 0) {
+    console.log('   ⚠️  旧版 cookie 文件不含 localStorage，本次可能仍需登录；登录后会自动升级格式');
+  }
 } else {
   console.log('首次运行，请在弹出的浏览器中完成登录...');
 }
@@ -433,7 +563,15 @@ const needLogin = async () => {
   }
   // yitang 路径
   if (u.includes('/login') || u.includes('sso.')) return true;
-  const tok = await readToken();
+  // TOKEN 由 SPA 异步写入 localStorage，给已有会话一点时间恢复，避免误判为未登录而重新扫码
+  let tok = await readToken();
+  if (!tok && hasSavedCookies) {
+    for (let i = 0; i < 10 && !tok; i++) {
+      await page.waitForTimeout(500);
+      if (page.url().includes('/login') || page.url().includes('sso.')) return true;
+      tok = await readToken();
+    }
+  }
   if (!tok && hasSavedCookies) return true;
   return false;
 };
@@ -460,10 +598,8 @@ if (await needLogin()) {
   await page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => {});
 }
 
-// 保存最新 Cookie
+// 保存最新登录态（cookies + localStorage）
-const cookies = await context.cookies();
+await saveStorageState(context);
-writeFileSync(COOKIE_FILE, JSON.stringify(cookies, null, 2));
-console.log(`💾 Cookie 已保存`);
 
 // ── 飞书原生路径 ───────────────────────────────────────────────────────────────
 if (isFeishu) {
@@ -533,12 +669,12 @@ if (isFeishu) {
   }
 
   const pageTitle = wikiMeta?.title || await page.title();
-  const markdown  = feishuBlocksToMd(docxBlocks);
+  const markdown  = cleanMarkdown(feishuBlocksToMd(docxBlocks));
   const output    = `# ${pageTitle}\n\n> Source: ${TARGET_URL}\n\n${markdown}`;
   const safeTitle = pageTitle.replace(/[/\\:*?"<>|]/g, '_').trim();
   const outDir    = resolve('output');
   if (!existsSync(outDir)) mkdirSync(outDir, { recursive: true });
-  const outputPath = resolve(customOutput || `output/${safeTitle || 'output'}.md`);
+  const outputPath = resolve(customOutput || `output/${datePrefix()}_${safeTitle || 'output'}.md`);
   writeFileSync(outputPath, output, 'utf-8');
 
   console.log('');
@@ -621,14 +757,14 @@ if (capturedToken && typeof payload === 'string') {
 
 // ── 转换并保存 ────────────────────────────────────────────────────────────────
 const pageTitle  = await page.title();
-const markdown   = blocksToMarkdown(docData.blocks);
+const markdown   = cleanMarkdown(blocksToMarkdown(docData.blocks));
 const output     = `# ${pageTitle}\n\n> Source: ${TARGET_URL}\n\n${markdown}`;
 
 // 输出文件名：优先用参数，其次用标题，最后回退 output.md
 const safeTitle  = pageTitle.replace(/[/\\:*?"<>|]/g, '_').trim();
 const outDir2    = resolve('output');
 if (!existsSync(outDir2)) mkdirSync(outDir2, { recursive: true });
-const outputPath = resolve(customOutput || `output/${safeTitle || 'output'}.md`);
+const outputPath = resolve(customOutput || `output/${datePrefix()}_${safeTitle || 'output'}.md`);
 writeFileSync(outputPath, output, 'utf-8');
 
 console.log('');

fetch-feishu-wiki.mjs

@@ -113,7 +113,10 @@ if (images.length > 0) {
 }
 
 // ── 6. 保存 Markdown ──────────────────────────────────────────────────────────
-const outputPath = resolve(outputDir, `${safeTitle}.md`);
+// 文件名加 YYMMDD_ 日期前缀，便于按采集时间排序
+const _d = new Date(), _p = n => String(n).padStart(2, '0');
+const datePrefix = `${_p(_d.getFullYear() % 100)}${_p(_d.getMonth() + 1)}${_p(_d.getDate())}`;
+const outputPath = resolve(outputDir, `${datePrefix}_${safeTitle}.md`);
 writeFileSync(outputPath, updatedMarkdown, 'utf-8');
 
 console.log('');

package.json

@@ -4,7 +4,8 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build:mac": "mkdir -p dist && PLAYWRIGHT_BROWSERS_PATH=dist/browsers npx playwright install chromium && npx pkg auto-scrape.mjs --targets node22-macos-arm64 --output dist/yitang-scraper"
   },
   "keywords": [],
   "author": "",