beyondx123
ced9c7bdb3
- 输出文件名添加 YYMMDD_ 日期前缀，便于按时间排序 - 改用 storageState 完整保存登录态（cookies + localStorage），大幅减少重复扫码 - 增加 TOKEN 轮询宽限期，避免 SPA 异步加载时误判未登录 - 更新 README 文档记录
777 lines
32 KiB
JavaScript
777 lines
32 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* 一堂创业课文档采集工具
|
||
*
|
||
* 用法：
|
||
* node auto-scrape.mjs <URL> [输出文件名]
|
||
*
|
||
* 示例：
|
||
* node auto-scrape.mjs https://yitang.top/fs-doc/abc123/docId
|
||
* node auto-scrape.mjs https://your-tenant.feishu.cn/wiki/WikiNodeTokenExample...
|
||
*
|
||
* Cookie 持久化：首次登录后自动保存，之后无需重复扫码。
|
||
*/
|
||
import { chromium } from 'playwright';
|
||
import CryptoJS from 'crypto-js';
|
||
import { writeFileSync, readFileSync, existsSync, mkdirSync } from 'fs';
|
||
import { resolve, dirname } from 'path';
|
||
import { execSync } from 'child_process';
|
||
import { fileURLToPath } from 'url';
|
||
import { createInterface } from 'readline/promises';
|
||
|
||
const __filename = fileURLToPath(import.meta.url);
|
||
const __dirname = dirname(__filename);
|
||
|
||
// ── 打包模式：自动检测 Playwright 浏览器路径 ────────────────────────────────────
|
||
// 当通过 pkg 打包后，process.execPath 指向可执行文件本身
|
||
// 浏览器目录应位于可执行文件旁边的 browsers/ 目录中
|
||
const execDir = dirname(process.execPath);
|
||
const bundledBrowsers = resolve(execDir, 'browsers');
|
||
if (existsSync(bundledBrowsers)) {
|
||
process.env.PLAYWRIGHT_BROWSERS_PATH = bundledBrowsers;
|
||
}
|
||
|
||
// ── CLI 参数解析 ──────────────────────────────────────────────────────────────
|
||
let args = process.argv.slice(2);
|
||
if (!args[0]) {
|
||
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
||
args = [(await rl.question('请输入一堂文档链接：')).trim()];
|
||
rl.close();
|
||
}
|
||
if (!args[0] || args[0] === '--help' || args[0] === '-h') {
|
||
console.log(`
|
||
用法：node auto-scrape.mjs <URL> [输出文件名]
|
||
|
||
URL yitang.top/fs-doc/... 或 *.feishu.cn/wiki/... 文档链接（必填）
|
||
输出文件名 保存路径，默认为文档标题.md
|
||
|
||
示例：
|
||
node auto-scrape.mjs https://yitang.top/fs-doc/abc/docId
|
||
node auto-scrape.mjs https://your-tenant.feishu.cn/wiki/WikiNodeTokenExample
|
||
node auto-scrape.mjs https://yitang.top/fs-doc/abc/docId 课程笔记.md
|
||
node auto-scrape.mjs --clean 已有文档.md
|
||
`);
|
||
process.exit(0);
|
||
}
|
||
|
||
// ── --clean：对已生成的 Markdown 文件执行清洗（不启动浏览器）──────────────────
|
||
if (args[0] === '--clean') {
|
||
const file = resolve(args[1] || '');
|
||
if (!args[1] || !existsSync(file)) {
|
||
console.error('用法：node auto-scrape.mjs --clean <markdown文件>');
|
||
process.exit(1);
|
||
}
|
||
const md = readFileSync(file, 'utf-8');
|
||
writeFileSync(file, cleanMarkdown(md), 'utf-8');
|
||
console.log(`✅ 已清洗：${file}`);
|
||
process.exit(0);
|
||
}
|
||
|
||
const TARGET_URL = args[0].trim();
|
||
const customOutput = args[1];
|
||
const COOKIE_FILE = resolve('cookies.json');
|
||
|
||
// ── 文件名日期前缀（YYMMDD_），便于按采集时间排序 ─────────────────────────────
|
||
function datePrefix(d = new Date()) {
|
||
const p = n => String(n).padStart(2, '0');
|
||
return `${p(d.getFullYear() % 100)}${p(d.getMonth() + 1)}${p(d.getDate())}`;
|
||
}
|
||
|
||
// ── 判断 URL 类型 ──────────────────────────────────────────────────────────────
|
||
const isFeishuWiki = /feishu\.cn\/wiki\//.test(TARGET_URL);
|
||
const isFeishuDocx = /feishu\.cn\/docx\//.test(TARGET_URL);
|
||
const isYitang = TARGET_URL.includes('yitang.top/fs-doc/');
|
||
const isFeishu = isFeishuWiki || isFeishuDocx;
|
||
|
||
if (!isFeishuWiki && !isFeishuDocx && !isYitang) {
|
||
console.error('❌ URL 格式不正确，支持 yitang.top/fs-doc/<acl>/<docId> 或 *.feishu.cn/wiki/<token>');
|
||
process.exit(1);
|
||
}
|
||
|
||
// ── Feishu Wiki：直接调用 fetch-feishu-wiki.mjs ────────────────────────────────
|
||
if (isFeishuWiki) {
|
||
console.log('🔄 检测到飞书 Wiki 文档，使用 lark-cli API 获取...\n');
|
||
try {
|
||
const fetchScript = resolve(__dirname, 'fetch-feishu-wiki.mjs');
|
||
execSync(`node "${fetchScript}" "${TARGET_URL}"`, {
|
||
stdio: 'inherit',
|
||
encoding: 'utf-8'
|
||
});
|
||
process.exit(0);
|
||
} catch (error) {
|
||
console.error('❌ 调用 fetch-feishu-wiki.mjs 失败');
|
||
process.exit(1);
|
||
}
|
||
}
|
||
|
||
// yitang 专用字段
|
||
let ACL, DOC_ID, LOGIN_URL;
|
||
if (isYitang) {
|
||
const match = TARGET_URL.match(/\/fs-doc\/([^/]+)\/([^/?#]+)/);
|
||
if (!match) {
|
||
console.error('❌ yitang URL 格式不正确，应为 https://yitang.top/fs-doc/<acl>/<docId>');
|
||
process.exit(1);
|
||
}
|
||
ACL = match[1];
|
||
DOC_ID = match[2];
|
||
LOGIN_URL = `https://yitang.top/login?after_login=%2Ffs-doc%2F${ACL}%2F${DOC_ID}`;
|
||
}
|
||
|
||
// ── Markdown 清洗（CommonMark 定界符合规）────────────────────────────────────
|
||
// 1) 相邻同样式文本块合并：**甲****乙** → **甲乙**
|
||
// 2) 加粗/斜体/删除线以中文标点结束、后接汉字/字母/数字时，闭合标记后补空格：**标签：** 研发
|
||
// 3) 标题行去除 **、*、~~ 等格式标记残留
|
||
function stripHeadingMarkers(line) {
|
||
const m = line.match(/^(#{1,6}\s+)(.*)$/);
|
||
if (!m) return line;
|
||
let inner = m[2], prev;
|
||
do {
|
||
prev = inner;
|
||
inner = inner
|
||
.replace(/\*\*([^*]+)\*\*/g, '$1')
|
||
.replace(/__([^_]+)__/g, '$1')
|
||
.replace(/~~([^~]+)~~/g, '$1')
|
||
.replace(/(^|[^*])\*([^*]+)\*/g, '$1$2');
|
||
} while (inner !== prev);
|
||
return m[1] + inner.trimEnd();
|
||
}
|
||
|
||
function mergeAdjacentSpans(line) {
|
||
let prev;
|
||
do {
|
||
prev = line;
|
||
line = line
|
||
.replace(/\*\*([^*\n]+)\*\*\*\*([^*\n]+)\*\*/g, '**$1$2**')
|
||
.replace(/\*\*\*([^*\n]+?)\*\*\*\*\*\*([^*\n]+?)\*\*\*/g, '***$1$2***')
|
||
.replace(/~~([^~\n]+)~~~~([^~\n]+)~~/g, '~~$1$2~~');
|
||
} while (line !== prev);
|
||
return line;
|
||
}
|
||
|
||
function fixClosingDelimiters(line) {
|
||
// CommonMark 侧翼规则：闭合定界符前为标点、后紧跟字母/数字（含汉字）时无法闭合，需补空格
|
||
const addSpace = (whole, marker, inner) =>
|
||
/[\p{P}\p{S}]/u.test(inner.slice(-1)) ? `${marker}${inner}${marker} ` : whole;
|
||
line = line.replace(/(?<!\*)\*\*\*([^*\n]+?)\*\*\*(?!\*)(?=[\p{L}\p{N}])/gu, (m, inner) => addSpace(m, '***', inner));
|
||
line = line.replace(/\*\*([^*\n]+)\*\*(?=[\p{L}\p{N}])/gu, (m, inner) => addSpace(m, '**', inner));
|
||
line = line.replace(/~~([^~\n]+)~~(?=[\p{L}\p{N}])/gu, (m, inner) => addSpace(m, '~~', inner));
|
||
line = line.replace(/(?<!\*)\*([^*\n]+)\*(?!\*)(?=[\p{L}\p{N}])/gu, (m, inner) => addSpace(m, '*', inner));
|
||
return line;
|
||
}
|
||
|
||
function cleanMarkdown(md) {
|
||
// 迭代至不动点，确保单遍扫描因定界符交叠漏掉的模式也被处理
|
||
let prev;
|
||
do {
|
||
prev = md;
|
||
md = md.split('\n')
|
||
.map(l => fixClosingDelimiters(mergeAdjacentSpans(stripHeadingMarkers(l))))
|
||
.join('\n');
|
||
} while (md !== prev);
|
||
return md;
|
||
}
|
||
|
||
// ── Feishu block → Markdown ──────────────────────────────────────────────────
|
||
const T = { PAGE:1, TEXT:2, H1:3, H2:4, H3:5, H4:6, H5:7, H6:8,
|
||
ORDERED:13, BULLET:12, TODO:17, CODE:14, QUOTE:15, DIVIDER:22,
|
||
IMAGE:27, TABLE:31, TABLE_CELL:32, QUOTE_CONTAINER:34,
|
||
GRID:24, GRID_COL:25 };
|
||
|
||
function textRuns(elements) {
|
||
if (!Array.isArray(elements)) return '';
|
||
// 相同样式的相邻 run 先合并，避免生成 **甲****乙** 这类相邻定界符
|
||
const runs = [];
|
||
for (const el of elements) {
|
||
const run = el.text_run;
|
||
if (!run) continue;
|
||
const s = run.text_element_style || {};
|
||
const key = `${!!s.bold}|${!!s.italic}|${!!s.strikethrough}|${!!s.inline_code}|${s.link?.url || ''}`;
|
||
const last = runs[runs.length - 1];
|
||
if (last && last.key === key) last.content += run.content || '';
|
||
else runs.push({ key, content: run.content || '', style: s });
|
||
}
|
||
return runs.map(({ content: t, style: s }) => {
|
||
if (s.bold) t = `**${t}**`;
|
||
if (s.italic) t = `*${t}*`;
|
||
if (s.strikethrough) t = `~~${t}~~`;
|
||
if (s.inline_code) t = `\`${t}\``;
|
||
if (s.link?.url) t = `[${t}](${s.link.url})`;
|
||
return t;
|
||
}).join('');
|
||
}
|
||
|
||
function blockToMd(block, depth = 0) {
|
||
if (!block) return '';
|
||
const attr = block.blockAttr || {};
|
||
const indent = ' '.repeat(Math.max(0, depth - 1));
|
||
switch (block.type) {
|
||
case T.PAGE: return `# ${textRuns(attr.page?.elements)}\n\n`;
|
||
case T.H1: return `# ${textRuns(attr.heading1?.elements)}\n\n`;
|
||
case T.H2: return `## ${textRuns(attr.heading2?.elements)}\n\n`;
|
||
case T.H3: return `### ${textRuns(attr.heading3?.elements)}\n\n`;
|
||
case T.H4: return `#### ${textRuns(attr.heading4?.elements)}\n\n`;
|
||
case T.H5: return `##### ${textRuns(attr.heading5?.elements)}\n\n`;
|
||
case T.H6: return `###### ${textRuns(attr.heading6?.elements)}\n\n`;
|
||
case T.TEXT: {
|
||
const txt = textRuns(attr.text?.elements);
|
||
return txt ? `${txt}\n\n` : '\n';
|
||
}
|
||
case T.BULLET: return `${indent}- ${textRuns(attr.bullet?.elements)}\n`;
|
||
case T.ORDERED: return `${indent}1. ${textRuns(attr.ordered?.elements)}\n`;
|
||
case T.TODO: {
|
||
const done = attr.todo?.style?.done ? '[x]' : '[ ]';
|
||
return `${indent}- ${done} ${textRuns(attr.todo?.elements)}\n`;
|
||
}
|
||
case T.QUOTE: return `> ${textRuns(attr.quote?.elements)}\n\n`;
|
||
case T.CODE: {
|
||
const language = attr.code?.style?.language;
|
||
const lang = typeof language === 'string' ? language.toLowerCase().replace('plain text','') : '';
|
||
const code = (attr.code?.elements || []).map(e => e.text_run?.content || '').join('');
|
||
return `\`\`\`${lang}\n${code}\n\`\`\``;
|
||
}
|
||
case T.DIVIDER: return `---\n\n`;
|
||
case T.IMAGE: {
|
||
const caption = (attr.textArea || []).map(a => a.text).filter(Boolean).join(' ');
|
||
const url = attr.cdnUrl || attr.image?.token || '';
|
||
return `\n\n`;
|
||
}
|
||
case T.QUOTE_CONTAINER: return '';
|
||
case T.GRID: return '';
|
||
case T.GRID_COL: return '';
|
||
case T.TABLE: return '\n';
|
||
case T.TABLE_CELL: {
|
||
const text = (block.childrens || []).map(c => {
|
||
const a = c.blockAttr || {};
|
||
return textRuns(a.text?.elements || a.heading1?.elements || a.heading2?.elements || []);
|
||
}).filter(Boolean).join(' | ');
|
||
return text ? `| ${text} ` : '';
|
||
}
|
||
default: return '';
|
||
}
|
||
}
|
||
|
||
function blocksToMarkdown(rootBlock) {
|
||
const lines = [];
|
||
function walk(block, depth) {
|
||
if (!block) return;
|
||
lines.push(blockToMd(block, depth));
|
||
if (block.type !== T.TABLE_CELL && block.childrens) {
|
||
block.childrens.forEach(c => walk(c, depth + 1));
|
||
}
|
||
if (block.type === T.TABLE) lines.push('\n');
|
||
}
|
||
walk(rootBlock, 0);
|
||
return lines.join('').replace(/\n{3,}/g, '\n\n').trim();
|
||
}
|
||
|
||
// ── Feishu 原生 docx block (block_type schema) → Markdown ────────────────────
|
||
// 飞书 Open Platform block_type: 1=page 2=text 3-8=h1-h6 12=bullet 13=ordered
|
||
// 14=code 15=quote 16=todo 21=divider 26=image 30=table 31=table_cell
|
||
|
||
const FT_HEADING = { 3: 1, 4: 2, 5: 3, 6: 4, 7: 5, 8: 6 };
|
||
const FT_CODE_LANG = {
|
||
1: '', 7: 'bash', 9: 'cpp', 10: 'c', 12: 'css', 22: 'go',
|
||
28: 'json', 29: 'java', 30: 'javascript', 49: 'python',
|
||
52: 'ruby', 53: 'rust', 56: 'sql', 60: 'shell', 63: 'typescript',
|
||
};
|
||
|
||
function ftTextElements(elements = []) {
|
||
// 相同样式的相邻 text_run 先合并，避免生成 **甲****乙** 这类相邻定界符
|
||
const segs = [];
|
||
for (const el of elements) {
|
||
if (el.text_run) {
|
||
const s = el.text_run.text_element_style || {};
|
||
const key = `${!!s.inline_code}|${!!s.bold}|${!!s.italic}|${!!s.strikethrough}|${s.link?.url || ''}`;
|
||
const last = segs[segs.length - 1];
|
||
if (last && last.key === key) last.content += el.text_run.content || '';
|
||
else segs.push({ key, content: el.text_run.content || '', style: s });
|
||
} else if (el.mention_doc) {
|
||
const { url = '', title = '' } = el.mention_doc;
|
||
segs.push({ key: null, text: url ? `[${title || url}](${url})` : title });
|
||
} else if (el.mention_user) {
|
||
segs.push({ key: null, text: `@${el.mention_user.user_id || 'user'}` });
|
||
} else {
|
||
segs.push({ key: null, text: '' });
|
||
}
|
||
}
|
||
return segs.map(seg => {
|
||
if (seg.key === null) return seg.text;
|
||
let t = seg.content;
|
||
const s = seg.style;
|
||
if (s.inline_code) return `\`${t}\``;
|
||
if (s.bold) t = `**${t}**`;
|
||
if (s.italic) t = `*${t}*`;
|
||
if (s.strikethrough) t = `~~${t}~~`;
|
||
if (s.link?.url) t = `[${t}](${decodeURIComponent(s.link.url)})`;
|
||
return t;
|
||
}).join('');
|
||
}
|
||
|
||
function ftBlockToMd(block, index, depth, orderedNum = 1) {
|
||
const t = block.block_type;
|
||
const indent = ' '.repeat(depth);
|
||
if (t === 1) return ftRenderChildren(block.children || [], index, depth);
|
||
if (t === 2) {
|
||
const text = ftTextElements(block.text?.elements);
|
||
return text ? text + '\n' : '\n';
|
||
}
|
||
if (FT_HEADING[t] !== undefined) {
|
||
return `${'#'.repeat(FT_HEADING[t])} ${ftTextElements(block.text?.elements)}\n\n`;
|
||
}
|
||
if (t === 12) {
|
||
const nested = block.children?.length ? '\n' + ftRenderChildren(block.children, index, depth + 1) : '';
|
||
return `${indent}- ${ftTextElements(block.text?.elements)}${nested}\n`;
|
||
}
|
||
if (t === 13) {
|
||
const nested = block.children?.length ? '\n' + ftRenderChildren(block.children, index, depth + 1) : '';
|
||
return `${indent}${orderedNum}. ${ftTextElements(block.text?.elements)}${nested}\n`;
|
||
}
|
||
if (t === 14) {
|
||
const langId = block.code?.style?.language ?? 1;
|
||
const code = ftTextElements(block.code?.elements || []);
|
||
return `\`\`\`${FT_CODE_LANG[langId] ?? ''}\n${code}\n\`\`\`\n`;
|
||
}
|
||
if (t === 15) return `> ${ftTextElements(block.text?.elements)}\n\n`;
|
||
if (t === 16) {
|
||
const done = block.text?.style?.done ? '[x]' : '[ ]';
|
||
return `- ${done} ${ftTextElements(block.text?.elements)}\n`;
|
||
}
|
||
if (t === 21) return '\n---\n\n';
|
||
if (t === 26) {
|
||
const token = block.image?.token || '';
|
||
const src = token
|
||
? `https://internal-api-drive-stream.feishu.cn/space/api/box/stream/download/all/${token}`
|
||
: '';
|
||
return `\n\n`;
|
||
}
|
||
if (t === 30) return ftRenderTable(block, index) + '\n';
|
||
if (t === 31) return null;
|
||
const fallback = ftTextElements(block.text?.elements || []);
|
||
return fallback ? fallback + '\n' : null;
|
||
}
|
||
|
||
function ftRenderChildren(childIds, index, depth) {
|
||
const lines = [];
|
||
let n = 1;
|
||
for (const id of childIds) {
|
||
const block = index.get(id);
|
||
if (!block) continue;
|
||
if (block.block_type === 13) {
|
||
const line = ftBlockToMd(block, index, depth, n++);
|
||
if (line != null) lines.push(line);
|
||
} else {
|
||
n = 1;
|
||
const line = ftBlockToMd(block, index, depth);
|
||
if (line != null) lines.push(line);
|
||
}
|
||
}
|
||
return lines.join('');
|
||
}
|
||
|
||
function ftRenderTable(block, index) {
|
||
const { row_size = 0, column_size = 0 } = block.table?.property || {};
|
||
const cellIds = block.table?.cells || [];
|
||
if (!row_size || !column_size) return '';
|
||
const rows = Array.from({ length: row_size }, () => Array(column_size).fill(''));
|
||
for (let r = 0; r < row_size; r++) {
|
||
for (let c = 0; c < column_size; c++) {
|
||
const cell = index.get(cellIds[r * column_size + c]);
|
||
if (!cell) continue;
|
||
rows[r][c] = ftRenderChildren(cell.children || [], index, 0).replace(/\n+/g, ' ').trim();
|
||
}
|
||
}
|
||
const out = ['| ' + rows[0].join(' | ') + ' |', '| ' + Array(column_size).fill('---').join(' | ') + ' |'];
|
||
for (let r = 1; r < row_size; r++) out.push('| ' + rows[r].join(' | ') + ' |');
|
||
return out.join('\n');
|
||
}
|
||
|
||
function feishuBlocksToMd(blocks) {
|
||
const index = new Map();
|
||
for (const b of blocks) if (b.block_id) index.set(b.block_id, b);
|
||
const root = blocks.find(b => b.block_type === 1) || blocks[0];
|
||
if (!root) return blocks.map(b => ftBlockToMd(b, index, 0)).filter(Boolean).join('\n');
|
||
return ftRenderChildren(root.children || [], index, 0).replace(/\n{3,}/g, '\n\n').trim();
|
||
}
|
||
|
||
// 从 API 响应 JSON 中递归查找 block 数组（block_type schema）
|
||
function findFeishuBlocks(obj, depth = 0) {
|
||
if (depth > 5 || !obj || typeof obj !== 'object') return [];
|
||
if (obj.block_id && typeof obj.block_type === 'number') return [obj];
|
||
if (Array.isArray(obj)) {
|
||
const flat = [];
|
||
for (const item of obj) flat.push(...findFeishuBlocks(item, depth + 1));
|
||
return flat;
|
||
}
|
||
for (const c of [obj.items, obj.blocks, obj.data?.items, obj.data?.blocks,
|
||
obj.data?.document?.blocks, obj.result?.data?.items]) {
|
||
if (c) { const found = findFeishuBlocks(c, depth + 1); if (found.length) return found; }
|
||
}
|
||
return [];
|
||
}
|
||
|
||
// 从 wiki node API 响应中找出真实 docx token 和标题
|
||
function findWikiObjToken(obj, wikiToken, depth = 0) {
|
||
if (depth > 6 || !obj || typeof obj !== 'object') return null;
|
||
if (obj.obj_token && (obj.node_token === wikiToken || obj.wiki_token === wikiToken || depth > 0)) return obj;
|
||
if (obj.obj_token && obj.title) return obj;
|
||
if (Array.isArray(obj)) {
|
||
for (const item of obj) { const f = findWikiObjToken(item, wikiToken, depth + 1); if (f) return f; }
|
||
return null;
|
||
}
|
||
for (const key of Object.keys(obj)) {
|
||
const f = findWikiObjToken(obj[key], wikiToken, depth + 1);
|
||
if (f) return f;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
// ── Main ─────────────────────────────────────────────────────────────────────
|
||
console.log('╔════════════════════════════════════════════╗');
|
||
console.log('║ 一堂文档采集工具 ║');
|
||
console.log('╚════════════════════════════════════════════╝');
|
||
console.log(`📎 URL: ${TARGET_URL}`);
|
||
if (isYitang) {
|
||
console.log(`📄 ACL: ${ACL}`);
|
||
console.log(`🔖 DocID: ${DOC_ID}`);
|
||
} else {
|
||
const ftToken = TARGET_URL.match(/\/(wiki|docx)\/([A-Za-z0-9_-]+)/);
|
||
console.log(`🔖 Type: feishu ${ftToken?.[1] || ''} Token: ${ftToken?.[2] || ''}`);
|
||
}
|
||
console.log('');
|
||
|
||
// ── 加载已保存的登录态（cookies + localStorage）──────────────────────────────
|
||
// yitang 的 API 鉴权 TOKEN 存在 localStorage（#|g.TOKEN），不在 cookie 里，
|
||
// 所以必须用 Playwright 的 storageState 整体持久化，只存 cookie 会导致每次都要重新扫码。
|
||
function loadStorageState() {
|
||
if (!existsSync(COOKIE_FILE)) return null;
|
||
try {
|
||
const saved = JSON.parse(readFileSync(COOKIE_FILE, 'utf-8'));
|
||
// 兼容旧格式：文件是纯 cookie 数组
|
||
if (Array.isArray(saved)) return { cookies: saved, origins: [] };
|
||
if (Array.isArray(saved.cookies)) {
|
||
return { cookies: saved.cookies, origins: saved.origins || [] };
|
||
}
|
||
} catch (e) {
|
||
console.log(`⚠️ 登录态文件解析失败（${e.message}），将重新登录`);
|
||
}
|
||
return null;
|
||
}
|
||
|
||
// 把当前 context 的 cookies + localStorage 落盘
|
||
async function saveStorageState(ctx) {
|
||
try {
|
||
const state = await ctx.storageState();
|
||
writeFileSync(COOKIE_FILE, JSON.stringify(state, null, 2), 'utf-8');
|
||
const lsCount = (state.origins || []).reduce((n, o) => n + (o.localStorage?.length || 0), 0);
|
||
console.log(`💾 登录态已保存（Cookie ${state.cookies.length} 条，localStorage ${lsCount} 项）`);
|
||
} catch (e) {
|
||
console.log(`⚠️ 登录态保存失败：${e.message}`);
|
||
}
|
||
}
|
||
|
||
const savedState = loadStorageState();
|
||
const hasSavedCookies = !!savedState;
|
||
const browser = await chromium.launch({ headless: false, args: ['--window-size=820,720'] });
|
||
const context = await browser.newContext(savedState ? { storageState: savedState } : {});
|
||
|
||
if (savedState) {
|
||
const lsCount = (savedState.origins || []).reduce((n, o) => n + (o.localStorage?.length || 0), 0);
|
||
console.log(`✅ 已加载保存的登录态（Cookie ${savedState.cookies.length} 条，localStorage ${lsCount} 项）`);
|
||
if (lsCount === 0) {
|
||
console.log(' ⚠️ 旧版 cookie 文件不含 localStorage，本次可能仍需登录；登录后会自动升级格式');
|
||
}
|
||
} else {
|
||
console.log('首次运行，请在弹出的浏览器中完成登录...');
|
||
}
|
||
|
||
const page = await context.newPage();
|
||
|
||
let apiResponseText = null;
|
||
const feishuApiResponses = [];
|
||
|
||
// 飞书原生 docx/wiki API 匹配模式
|
||
const FEISHU_DOC_RE = [
|
||
/\/docx\/v\d+\/documents\/[A-Za-z0-9]+\/blocks/,
|
||
/\/space\/api\/docs\/v\d+\/raw_api/,
|
||
/apiName=DocxV\d+Document/i,
|
||
];
|
||
const FEISHU_WIKI_RE = [
|
||
/\/wiki\/v\d+\/spaces?\/[^/]+\/nodes?/,
|
||
/\/wiki\/api\/.*\/(get_?node|node_meta|node_info)/,
|
||
/\/space\/api\/wiki\/v\d+\/.*node/,
|
||
/apiName=WikiV\d+/i,
|
||
];
|
||
|
||
page.on('response', async (resp) => {
|
||
const u = resp.url();
|
||
|
||
if (isFeishu) {
|
||
const isDoc = FEISHU_DOC_RE.some(p => p.test(u));
|
||
const isWiki = FEISHU_WIKI_RE.some(p => p.test(u));
|
||
// 跳过非 JSON、静态资源、已知无用的请求
|
||
const ct = resp.headers()['content-type'] || '';
|
||
if (!ct.includes('json') && !ct.includes('text/plain')) return;
|
||
if (resp.status() < 200 || resp.status() >= 300) return;
|
||
|
||
try {
|
||
const text = await resp.text();
|
||
let json; try { json = JSON.parse(text); } catch { return; }
|
||
|
||
if (isDoc || isWiki) {
|
||
console.log(`\n🎯 捕获飞书 API (${isWiki ? 'wiki' : 'doc'})：${u.split('?')[0]} (${text.length} 字节)`);
|
||
feishuApiResponses.push({ url: u, data: json, kind: isWiki ? 'wiki' : 'doc' });
|
||
return;
|
||
}
|
||
|
||
// 兜底：扫描任意 JSON 响应中是否含有 block 数据
|
||
const blocks = findFeishuBlocks(json);
|
||
if (blocks.length > 0) {
|
||
console.log(`\n📦 兜底捕获 block 数据：${u.split('?')[0]} (${blocks.length} 个 block)`);
|
||
feishuApiResponses.push({ url: u, data: json, kind: 'doc' });
|
||
}
|
||
} catch {}
|
||
return;
|
||
}
|
||
|
||
// yitang 路径
|
||
if (u.includes('get-doc-blocks') || u.includes('get-blocks')) {
|
||
try {
|
||
const text = await resp.text();
|
||
// 忽略"未登录"错误响应（code 10）
|
||
try {
|
||
const parsed = JSON.parse(text);
|
||
if (parsed.code === 10) {
|
||
console.log(`\n⚠️ API 返回未登录，等待真实响应...`);
|
||
return;
|
||
}
|
||
} catch {}
|
||
apiResponseText = text;
|
||
console.log(`\n🎯 捕获 API 响应：${u.split('?')[0]} (${text.length} 字节)`);
|
||
} catch {}
|
||
}
|
||
});
|
||
|
||
console.log('🌐 正在打开文档页面...');
|
||
await page.goto(TARGET_URL, { waitUntil: 'load', timeout: 30000 });
|
||
|
||
// 检测是否需要登录
|
||
const needLogin = async () => {
|
||
const u = page.url();
|
||
if (isFeishu) {
|
||
// 飞书登录页：passport.feishu.cn / accounts.feishu.cn / /login
|
||
return u.includes('passport.feishu') || u.includes('accounts.feishu') || u.includes('/login');
|
||
}
|
||
// yitang 路径
|
||
if (u.includes('/login') || u.includes('sso.')) return true;
|
||
// TOKEN 由 SPA 异步写入 localStorage，给已有会话一点时间恢复，避免误判为未登录而重新扫码
|
||
let tok = await readToken();
|
||
if (!tok && hasSavedCookies) {
|
||
for (let i = 0; i < 10 && !tok; i++) {
|
||
await page.waitForTimeout(500);
|
||
if (page.url().includes('/login') || page.url().includes('sso.')) return true;
|
||
tok = await readToken();
|
||
}
|
||
}
|
||
if (!tok && hasSavedCookies) return true;
|
||
return false;
|
||
};
|
||
|
||
if (await needLogin()) {
|
||
console.log('⚠️ 需要登录，正在跳转...');
|
||
if (isFeishu) {
|
||
if (!page.url().includes('passport') && !page.url().includes('accounts') && !page.url().includes('/login')) {
|
||
await page.goto(TARGET_URL, { waitUntil: 'load', timeout: 30000 });
|
||
}
|
||
console.log('请在浏览器中完成飞书登录...');
|
||
await page.waitForFunction(
|
||
() => !location.href.includes('passport') && !location.href.includes('accounts') && !location.href.includes('/login'),
|
||
{ timeout: 300000 }
|
||
);
|
||
} else {
|
||
if (!page.url().includes('/login') && !page.url().includes('sso.')) {
|
||
await page.goto(LOGIN_URL, { waitUntil: 'load', timeout: 30000 });
|
||
}
|
||
console.log('请在浏览器中完成登录（微信扫码或手机验证码）...');
|
||
await page.waitForURL('**/fs-doc/**', { timeout: 300000 });
|
||
}
|
||
console.log('\n✅ 登录成功');
|
||
await page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => {});
|
||
}
|
||
|
||
// 保存最新登录态（cookies + localStorage）
|
||
await saveStorageState(context);
|
||
|
||
// ── 飞书原生路径 ───────────────────────────────────────────────────────────────
|
||
if (isFeishu) {
|
||
if (feishuApiResponses.length === 0) {
|
||
console.log('⏳ 等待飞书 API 数据加载...');
|
||
await page.waitForTimeout(8000);
|
||
}
|
||
if (feishuApiResponses.length === 0) {
|
||
console.log('🔄 重新加载页面...');
|
||
await page.goto(TARGET_URL, { waitUntil: 'networkidle', timeout: 30000 });
|
||
await page.waitForTimeout(5000);
|
||
}
|
||
if (feishuApiResponses.length === 0) {
|
||
console.error('❌ 未捕获到飞书 API 响应，请检查登录状态或文档权限');
|
||
await browser.close();
|
||
process.exit(1);
|
||
}
|
||
|
||
// 提取 wiki node 元信息（wiki 页面需先拿 obj_token）
|
||
let wikiMeta = null;
|
||
const ftToken = TARGET_URL.match(/\/(wiki|docx)\/([A-Za-z0-9_-]+)/)?.[2] || '';
|
||
if (TARGET_URL.includes('/wiki/')) {
|
||
for (const item of feishuApiResponses) {
|
||
if (item.kind === 'wiki') {
|
||
wikiMeta = findWikiObjToken(item.data, ftToken);
|
||
if (wikiMeta) {
|
||
console.log(`📎 wiki → docx token: ${wikiMeta.obj_token} 标题: ${wikiMeta.title || '(未知)'}`);
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
// 如果是 wiki 页面且找到了 docx token，需要导航到 docx URL 来获取实际内容
|
||
if (wikiMeta?.obj_token) {
|
||
const docxUrl = TARGET_URL.replace(/\/wiki\/[^/]+/, `/docx/${wikiMeta.obj_token}`);
|
||
console.log(`🔄 正在加载文档内容: ${docxUrl}`);
|
||
feishuApiResponses.length = 0; // 清空之前的 wiki API 响应
|
||
await page.goto(docxUrl, { waitUntil: 'load', timeout: 30000 });
|
||
await page.waitForTimeout(3000);
|
||
|
||
if (feishuApiResponses.length === 0) {
|
||
console.log('⏳ 等待文档 API 数据加载...');
|
||
await page.waitForTimeout(5000);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 提取 blocks（去重）
|
||
let docxBlocks = [];
|
||
for (const item of feishuApiResponses) {
|
||
if (item.kind !== 'doc') continue;
|
||
const found = findFeishuBlocks(item.data);
|
||
if (found.length) docxBlocks.push(...found);
|
||
}
|
||
const seenIds = new Set();
|
||
docxBlocks = docxBlocks.filter(b => {
|
||
if (!b.block_id || seenIds.has(b.block_id)) return false;
|
||
seenIds.add(b.block_id);
|
||
return true;
|
||
});
|
||
console.log(`📦 共提取 ${docxBlocks.length} 个 block`);
|
||
|
||
if (docxBlocks.length === 0) {
|
||
console.error('❌ 未获取到有效 block 数据');
|
||
await browser.close();
|
||
process.exit(1);
|
||
}
|
||
|
||
const pageTitle = wikiMeta?.title || await page.title();
|
||
const markdown = cleanMarkdown(feishuBlocksToMd(docxBlocks));
|
||
const output = `# ${pageTitle}\n\n> Source: ${TARGET_URL}\n\n${markdown}`;
|
||
const safeTitle = pageTitle.replace(/[/\\:*?"<>|]/g, '_').trim();
|
||
const outDir = resolve('output');
|
||
if (!existsSync(outDir)) mkdirSync(outDir, { recursive: true });
|
||
const outputPath = resolve(customOutput || `output/${datePrefix()}_${safeTitle || 'output'}.md`);
|
||
writeFileSync(outputPath, output, 'utf-8');
|
||
|
||
console.log('');
|
||
console.log('╔════════════════════════════════════════════╗');
|
||
console.log(`║ ✅ 完成！共 ${output.length} 字符`);
|
||
console.log(`║ 📄 ${outputPath}`);
|
||
console.log('╚════════════════════════════════════════════╝');
|
||
await browser.close();
|
||
process.exit(0);
|
||
}
|
||
|
||
// 读取 TOKEN（函数，方便重登后复用）
|
||
async function readToken() {
|
||
return page.evaluate(() => {
|
||
try {
|
||
const raw = localStorage.getItem('#|g.TOKEN')
|
||
|| Object.entries(localStorage).find(([k]) => k.includes('TOKEN'))?.[1]
|
||
|| null;
|
||
if (!raw) return null;
|
||
return raw.startsWith('{') ? JSON.parse(raw).$v || null : raw;
|
||
} catch { return null; }
|
||
}).catch(() => null);
|
||
}
|
||
|
||
// 读取 TOKEN
|
||
let capturedToken = await page.evaluate(() => {
|
||
try {
|
||
const raw = localStorage.getItem('#|g.TOKEN')
|
||
|| Object.entries(localStorage).find(([k]) => k.includes('TOKEN'))?.[1]
|
||
|| null;
|
||
if (!raw) return null;
|
||
return raw.startsWith('{') ? JSON.parse(raw).$v || null : raw;
|
||
} catch { return null; }
|
||
}).catch(() => null);
|
||
console.log('🔑 TOKEN:', capturedToken ? capturedToken.substring(0, 8) + '...' : '未找到');
|
||
|
||
if (!apiResponseText) {
|
||
console.log('⏳ 等待 API 数据加载...');
|
||
await page.waitForTimeout(10000);
|
||
}
|
||
if (!apiResponseText) {
|
||
console.log('🔄 重新加载页面...');
|
||
await page.goto(TARGET_URL, { waitUntil: 'networkidle', timeout: 30000 });
|
||
await page.waitForTimeout(5000);
|
||
}
|
||
if (!apiResponseText) {
|
||
console.error('❌ 未捕获到 API 响应，可能未登录或文档受限');
|
||
await browser.close();
|
||
process.exit(1);
|
||
}
|
||
|
||
if (!apiResponseText) {
|
||
console.error('❌ 仍未获取到有效数据，请检查是否已完成登录或文档是否有访问权限');
|
||
await browser.close();
|
||
process.exit(1);
|
||
}
|
||
|
||
// ── 解密 ──────────────────────────────────────────────────────────────────────
|
||
let docData;
|
||
const envelope = JSON.parse(apiResponseText);
|
||
const payload = envelope.data;
|
||
|
||
if (capturedToken && typeof payload === 'string') {
|
||
try {
|
||
const key = CryptoJS.enc.Utf8.parse(CryptoJS.MD5(capturedToken).toString());
|
||
const iv = CryptoJS.enc.Utf8.parse(capturedToken);
|
||
const dec = CryptoJS.AES.decrypt(payload.trim(), key, {
|
||
iv, mode: CryptoJS.mode.CBC, padding: CryptoJS.pad.Pkcs7,
|
||
});
|
||
docData = JSON.parse(dec.toString(CryptoJS.enc.Utf8));
|
||
console.log('🔓 AES 解密成功');
|
||
} catch (e) {
|
||
console.log('⚠️ AES 解密失败：', e.message);
|
||
docData = typeof payload === 'object' ? payload : envelope;
|
||
}
|
||
} else {
|
||
docData = typeof payload === 'object' ? payload : envelope;
|
||
console.log('📄 直接使用响应数据');
|
||
}
|
||
|
||
// ── 转换并保存 ────────────────────────────────────────────────────────────────
|
||
const pageTitle = await page.title();
|
||
const markdown = cleanMarkdown(blocksToMarkdown(docData.blocks));
|
||
const output = `# ${pageTitle}\n\n> Source: ${TARGET_URL}\n\n${markdown}`;
|
||
|
||
// 输出文件名：优先用参数，其次用标题，最后回退 output.md
|
||
const safeTitle = pageTitle.replace(/[/\\:*?"<>|]/g, '_').trim();
|
||
const outDir2 = resolve('output');
|
||
if (!existsSync(outDir2)) mkdirSync(outDir2, { recursive: true });
|
||
const outputPath = resolve(customOutput || `output/${datePrefix()}_${safeTitle || 'output'}.md`);
|
||
writeFileSync(outputPath, output, 'utf-8');
|
||
|
||
console.log('');
|
||
console.log('╔════════════════════════════════════════════╗');
|
||
console.log(`║ ✅ 完成！共 ${output.length} 字符`);
|
||
console.log(`║ 📄 ${outputPath}`);
|
||
console.log('╚════════════════════════════════════════════╝');
|
||
|
||
await browser.close();
|