diff --git a/README.md b/README.md index c72fc19..0e98ebe 100644 --- a/README.md +++ b/README.md @@ -126,3 +126,24 @@ node auto-scrape.mjs **Q: 图片下载失败？** 飞书图片需要有效的个人授权。确认 `lark-cli auth status` 显示已登录，且包含 `docs:document.media:download` 权限。 + +--- + +## 更新记录 + +### 2024-09-13 + +**文件名优化** +- 自动生成的 Markdown 文件名现在带有 `YYMMDD_` 日期前缀 +- 输出示例：`output/260913_《战略笃定》探索营1.md` +- 便于按采集时间排序，同一文档的多次采集版本可区分 + +**登录态持久化优化（大幅减少重复扫码）** +- 改用 Playwright `storageState` 完整保存登录态（cookies + localStorage） +- 修复了原先只存 cookie 导致每次都要重新扫码的问题（一堂平台的 API 鉴权 TOKEN 存在 localStorage 中） +- 首次登录后，后续运行基本无需再扫码，直到服务端会话过期（通常数天） +- `cookies.json` 从旧格式自动升级为新格式（兼容已有文件） + +**登录判定改进** +- 增加 5 秒轮询宽限期，避免 SPA 异步写入 TOKEN 时误判为未登录 +- 减少因页面加载时序导致的不必要登录提示 diff --git a/auto-scrape.mjs b/auto-scrape.mjs index 8200156..a84c66a 100644 --- a/auto-scrape.mjs +++ b/auto-scrape.mjs @@ -17,6 +17,7 @@ import { writeFileSync, readFileSync, existsSync, mkdirSync } from 'fs'; import { resolve, dirname } from 'path'; import { execSync } from 'child_process'; import { fileURLToPath } from 'url'; +import { createInterface } from 'readline/promises'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -31,7 +32,12 @@ if (existsSync(bundledBrowsers)) { } // ── CLI 参数解析 ────────────────────────────────────────────────────────────── -const args = process.argv.slice(2); +let args = process.argv.slice(2); +if (!args[0]) { + const rl = createInterface({ input: process.stdin, output: process.stdout }); + args = [(await rl.question('请输入一堂文档链接：')).trim()]; + rl.close(); +} if (!args[0] || args[0] === '--help' || args[0] === '-h') { console.log(` 用法：node auto-scrape.mjs [输出文件名] @@ -43,14 +49,34 @@ if (!args[0] || args[0] === '--help' || args[0] === '-h') { node auto-scrape.mjs https://yitang.top/fs-doc/abc/docId node auto-scrape.mjs https://your-tenant.feishu.cn/wiki/WikiNodeTokenExample node auto-scrape.mjs https://yitang.top/fs-doc/abc/docId 课程笔记.md + node auto-scrape.mjs --clean 已有文档.md `); process.exit(0); } +// ── --clean：对已生成的 Markdown 文件执行清洗（不启动浏览器）────────────────── +if (args[0] === '--clean') { + const file = resolve(args[1] || ''); + if (!args[1] || !existsSync(file)) { + console.error('用法：node auto-scrape.mjs --clean '); + process.exit(1); + } + const md = readFileSync(file, 'utf-8'); + writeFileSync(file, cleanMarkdown(md), 'utf-8'); + console.log(`✅ 已清洗：${file}`); + process.exit(0); +} + const TARGET_URL = args[0].trim(); const customOutput = args[1]; const COOKIE_FILE = resolve('cookies.json'); +// ── 文件名日期前缀（YYMMDD_），便于按采集时间排序 ───────────────────────────── +function datePrefix(d = new Date()) { + const p = n => String(n).padStart(2, '0'); + return `${p(d.getFullYear() % 100)}${p(d.getMonth() + 1)}${p(d.getDate())}`; +} + // ── 判断 URL 类型 ────────────────────────────────────────────────────────────── const isFeishuWiki = /feishu\.cn\/wiki\//.test(TARGET_URL); const isFeishuDocx = /feishu\.cn\/docx\//.test(TARGET_URL); @@ -91,6 +117,60 @@ if (isYitang) { LOGIN_URL = `https://yitang.top/login?after_login=%2Ffs-doc%2F${ACL}%2F${DOC_ID}`; } +// ── Markdown 清洗（CommonMark 定界符合规）──────────────────────────────────── +// 1) 相邻同样式文本块合并：**甲****乙** → **甲乙** +// 2) 加粗/斜体/删除线以中文标点结束、后接汉字/字母/数字时，闭合标记后补空格：**标签：** 研发 +// 3) 标题行去除 **、*、~~ 等格式标记残留 +function stripHeadingMarkers(line) { + const m = line.match(/^(#{1,6}\s+)(.*)$/); + if (!m) return line; + let inner = m[2], prev; + do { + prev = inner; + inner = inner + .replace(/\*\*([^*]+)\*\*/g, '$1') + .replace(/__([^_]+)__/g, '$1') + .replace(/~~([^~]+)~~/g, '$1') + .replace(/(^|[^*])\*([^*]+)\*/g, '$1$2'); + } while (inner !== prev); + return m[1] + inner.trimEnd(); +} + +function mergeAdjacentSpans(line) { + let prev; + do { + prev = line; + line = line + .replace(/\*\*([^*\n]+)\*\*\*\*([^*\n]+)\*\*/g, '**$1$2**') + .replace(/\*\*\*([^*\n]+?)\*\*\*\*\*\*([^*\n]+?)\*\*\*/g, '***$1$2***') + .replace(/~~([^~\n]+)~~~~([^~\n]+)~~/g, '~~$1$2~~'); + } while (line !== prev); + return line; +} + +function fixClosingDelimiters(line) { + // CommonMark 侧翼规则：闭合定界符前为标点、后紧跟字母/数字（含汉字）时无法闭合，需补空格 + const addSpace = (whole, marker, inner) => + /[\p{P}\p{S}]/u.test(inner.slice(-1)) ? `${marker}${inner}${marker} ` : whole; + line = line.replace(/(? addSpace(m, '***', inner)); + line = line.replace(/\*\*([^*\n]+)\*\*(?=[\p{L}\p{N}])/gu, (m, inner) => addSpace(m, '**', inner)); + line = line.replace(/~~([^~\n]+)~~(?=[\p{L}\p{N}])/gu, (m, inner) => addSpace(m, '~~', inner)); + line = line.replace(/(? addSpace(m, '*', inner)); + return line; +} + +function cleanMarkdown(md) { + // 迭代至不动点，确保单遍扫描因定界符交叠漏掉的模式也被处理 + let prev; + do { + prev = md; + md = md.split('\n') + .map(l => fixClosingDelimiters(mergeAdjacentSpans(stripHeadingMarkers(l)))) + .join('\n'); + } while (md !== prev); + return md; +} + // ── Feishu block → Markdown ────────────────────────────────────────────────── const T = { PAGE:1, TEXT:2, H1:3, H2:4, H3:5, H4:6, H5:7, H6:8, ORDERED:13, BULLET:12, TODO:17, CODE:14, QUOTE:15, DIVIDER:22, @@ -99,11 +179,18 @@ const T = { PAGE:1, TEXT:2, H1:3, H2:4, H3:5, H4:6, H5:7, H6:8, function textRuns(elements) { if (!Array.isArray(elements)) return ''; - return elements.map(el => { + // 相同样式的相邻 run 先合并，避免生成 **甲****乙** 这类相邻定界符 + const runs = []; + for (const el of elements) { const run = el.text_run; - if (!run) return ''; - let t = run.content || ''; + if (!run) continue; const s = run.text_element_style || {}; + const key = `${!!s.bold}|${!!s.italic}|${!!s.strikethrough}|${!!s.inline_code}|${s.link?.url || ''}`; + const last = runs[runs.length - 1]; + if (last && last.key === key) last.content += run.content || ''; + else runs.push({ key, content: run.content || '', style: s }); + } + return runs.map(({ content: t, style: s }) => { if (s.bold) t = `**${t}**`; if (s.italic) t = `*${t}*`; if (s.strikethrough) t = `~~${t}~~`; @@ -189,24 +276,34 @@ const FT_CODE_LANG = { }; function ftTextElements(elements = []) { - return elements.map(el => { + // 相同样式的相邻 text_run 先合并，避免生成 **甲****乙** 这类相邻定界符 + const segs = []; + for (const el of elements) { if (el.text_run) { - const content = el.text_run.content || ''; const s = el.text_run.text_element_style || {}; - let t = content; - if (s.inline_code) return `\`${t}\``; - if (s.bold) t = `**${t}**`; - if (s.italic) t = `*${t}*`; - if (s.strikethrough) t = `~~${t}~~`; - if (s.link?.url) t = `[${t}](${decodeURIComponent(s.link.url)})`; - return t; - } - if (el.mention_doc) { + const key = `${!!s.inline_code}|${!!s.bold}|${!!s.italic}|${!!s.strikethrough}|${s.link?.url || ''}`; + const last = segs[segs.length - 1]; + if (last && last.key === key) last.content += el.text_run.content || ''; + else segs.push({ key, content: el.text_run.content || '', style: s }); + } else if (el.mention_doc) { const { url = '', title = '' } = el.mention_doc; - return url ? `[${title || url}](${url})` : title; + segs.push({ key: null, text: url ? `[${title || url}](${url})` : title }); + } else if (el.mention_user) { + segs.push({ key: null, text: `@${el.mention_user.user_id || 'user'}` }); + } else { + segs.push({ key: null, text: '' }); } - if (el.mention_user) return `@${el.mention_user.user_id || 'user'}`; - return ''; + } + return segs.map(seg => { + if (seg.key === null) return seg.text; + let t = seg.content; + const s = seg.style; + if (s.inline_code) return `\`${t}\``; + if (s.bold) t = `**${t}**`; + if (s.italic) t = `*${t}*`; + if (s.strikethrough) t = `~~${t}~~`; + if (s.link?.url) t = `[${t}](${decodeURIComponent(s.link.url)})`; + return t; }).join(''); } @@ -342,14 +439,47 @@ if (isYitang) { } console.log(''); -const hasSavedCookies = existsSync(COOKIE_FILE); -const browser = await chromium.launch({ headless: false, args: ['--window-size=820,720'] }); -const context = await browser.newContext(); +// ── 加载已保存的登录态（cookies + localStorage）────────────────────────────── +// yitang 的 API 鉴权 TOKEN 存在 localStorage（#|g.TOKEN），不在 cookie 里， +// 所以必须用 Playwright 的 storageState 整体持久化，只存 cookie 会导致每次都要重新扫码。 +function loadStorageState() { + if (!existsSync(COOKIE_FILE)) return null; + try { + const saved = JSON.parse(readFileSync(COOKIE_FILE, 'utf-8')); + // 兼容旧格式：文件是纯 cookie 数组 + if (Array.isArray(saved)) return { cookies: saved, origins: [] }; + if (Array.isArray(saved.cookies)) { + return { cookies: saved.cookies, origins: saved.origins || [] }; + } + } catch (e) { + console.log(`⚠️ 登录态文件解析失败（${e.message}），将重新登录`); + } + return null; +} -if (hasSavedCookies) { - const saved = JSON.parse(readFileSync(COOKIE_FILE, 'utf-8')); - await context.addCookies(saved); - console.log(`✅ 已加载保存的 Cookie（${saved.length} 条），跳过扫码登录`); +// 把当前 context 的 cookies + localStorage 落盘 +async function saveStorageState(ctx) { + try { + const state = await ctx.storageState(); + writeFileSync(COOKIE_FILE, JSON.stringify(state, null, 2), 'utf-8'); + const lsCount = (state.origins || []).reduce((n, o) => n + (o.localStorage?.length || 0), 0); + console.log(`💾 登录态已保存（Cookie ${state.cookies.length} 条，localStorage ${lsCount} 项）`); + } catch (e) { + console.log(`⚠️ 登录态保存失败：${e.message}`); + } +} + +const savedState = loadStorageState(); +const hasSavedCookies = !!savedState; +const browser = await chromium.launch({ headless: false, args: ['--window-size=820,720'] }); +const context = await browser.newContext(savedState ? { storageState: savedState } : {}); + +if (savedState) { + const lsCount = (savedState.origins || []).reduce((n, o) => n + (o.localStorage?.length || 0), 0); + console.log(`✅ 已加载保存的登录态（Cookie ${savedState.cookies.length} 条，localStorage ${lsCount} 项）`); + if (lsCount === 0) { + console.log(' ⚠️ 旧版 cookie 文件不含 localStorage，本次可能仍需登录；登录后会自动升级格式'); + } } else { console.log('首次运行，请在弹出的浏览器中完成登录...'); } @@ -433,7 +563,15 @@ const needLogin = async () => { } // yitang 路径 if (u.includes('/login') || u.includes('sso.')) return true; - const tok = await readToken(); + // TOKEN 由 SPA 异步写入 localStorage，给已有会话一点时间恢复，避免误判为未登录而重新扫码 + let tok = await readToken(); + if (!tok && hasSavedCookies) { + for (let i = 0; i < 10 && !tok; i++) { + await page.waitForTimeout(500); + if (page.url().includes('/login') || page.url().includes('sso.')) return true; + tok = await readToken(); + } + } if (!tok && hasSavedCookies) return true; return false; }; @@ -460,10 +598,8 @@ if (await needLogin()) { await page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => {}); } -// 保存最新 Cookie -const cookies = await context.cookies(); -writeFileSync(COOKIE_FILE, JSON.stringify(cookies, null, 2)); -console.log(`💾 Cookie 已保存`); +// 保存最新登录态（cookies + localStorage） +await saveStorageState(context); // ── 飞书原生路径 ─────────────────────────────────────────────────────────────── if (isFeishu) { @@ -533,12 +669,12 @@ if (isFeishu) { } const pageTitle = wikiMeta?.title || await page.title(); - const markdown = feishuBlocksToMd(docxBlocks); + const markdown = cleanMarkdown(feishuBlocksToMd(docxBlocks)); const output = `# ${pageTitle}\n\n> Source: ${TARGET_URL}\n\n${markdown}`; const safeTitle = pageTitle.replace(/[/\\:*?"<>|]/g, '_').trim(); const outDir = resolve('output'); if (!existsSync(outDir)) mkdirSync(outDir, { recursive: true }); - const outputPath = resolve(customOutput || `output/${safeTitle || 'output'}.md`); + const outputPath = resolve(customOutput || `output/${datePrefix()}_${safeTitle || 'output'}.md`); writeFileSync(outputPath, output, 'utf-8'); console.log(''); @@ -621,14 +757,14 @@ if (capturedToken && typeof payload === 'string') { // ── 转换并保存 ──────────────────────────────────────────────────────────────── const pageTitle = await page.title(); -const markdown = blocksToMarkdown(docData.blocks); +const markdown = cleanMarkdown(blocksToMarkdown(docData.blocks)); const output = `# ${pageTitle}\n\n> Source: ${TARGET_URL}\n\n${markdown}`; // 输出文件名：优先用参数，其次用标题，最后回退 output.md const safeTitle = pageTitle.replace(/[/\\:*?"<>|]/g, '_').trim(); const outDir2 = resolve('output'); if (!existsSync(outDir2)) mkdirSync(outDir2, { recursive: true }); -const outputPath = resolve(customOutput || `output/${safeTitle || 'output'}.md`); +const outputPath = resolve(customOutput || `output/${datePrefix()}_${safeTitle || 'output'}.md`); writeFileSync(outputPath, output, 'utf-8'); console.log(''); diff --git a/fetch-feishu-wiki.mjs b/fetch-feishu-wiki.mjs index 90dbd87..5ed98b6 100755 --- a/fetch-feishu-wiki.mjs +++ b/fetch-feishu-wiki.mjs @@ -113,7 +113,10 @@ if (images.length > 0) { } // ── 6. 保存 Markdown ────────────────────────────────────────────────────────── -const outputPath = resolve(outputDir, `${safeTitle}.md`); +// 文件名加 YYMMDD_ 日期前缀，便于按采集时间排序 +const _d = new Date(), _p = n => String(n).padStart(2, '0'); +const datePrefix = `${_p(_d.getFullYear() % 100)}${_p(_d.getMonth() + 1)}${_p(_d.getDate())}`; +const outputPath = resolve(outputDir, `${datePrefix}_${safeTitle}.md`); writeFileSync(outputPath, updatedMarkdown, 'utf-8'); console.log(''); diff --git a/package.json b/package.json index 238d805..7b150c3 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "description": "", "main": "index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "build:mac": "mkdir -p dist && PLAYWRIGHT_BROWSERS_PATH=dist/browsers npx playwright install chromium && npx pkg auto-scrape.mjs --targets node22-macos-arm64 --output dist/yitang-scraper" }, "keywords": [], "author": "",