Second commit
This commit is contained in:
+639
@@ -0,0 +1,639 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* 一堂创业课文档采集工具
|
||||
*
|
||||
* 用法:
|
||||
* node auto-scrape.mjs <URL> [输出文件名]
|
||||
*
|
||||
* 示例:
|
||||
* node auto-scrape.mjs https://yitang.top/fs-doc/abc123/docId
|
||||
* node auto-scrape.mjs https://yitanger.feishu.cn/wiki/R9NywV2i9i...
|
||||
*
|
||||
* Cookie 持久化:首次登录后自动保存,之后无需重复扫码。
|
||||
*/
|
||||
import { chromium } from 'playwright';
|
||||
import CryptoJS from 'crypto-js';
|
||||
import { writeFileSync, readFileSync, existsSync } from 'fs';
|
||||
import { resolve, basename, dirname } from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// ── 打包模式:自动检测 Playwright 浏览器路径 ────────────────────────────────────
|
||||
// 当通过 pkg 打包后,process.execPath 指向可执行文件本身
|
||||
// 浏览器目录应位于可执行文件旁边的 browsers/ 目录中
|
||||
const execDir = dirname(process.execPath);
|
||||
const bundledBrowsers = resolve(execDir, 'browsers');
|
||||
if (existsSync(bundledBrowsers)) {
|
||||
process.env.PLAYWRIGHT_BROWSERS_PATH = bundledBrowsers;
|
||||
}
|
||||
|
||||
// ── CLI 参数解析 ──────────────────────────────────────────────────────────────
|
||||
const args = process.argv.slice(2);
|
||||
if (!args[0] || args[0] === '--help' || args[0] === '-h') {
|
||||
console.log(`
|
||||
用法:node auto-scrape.mjs <URL> [输出文件名]
|
||||
|
||||
URL yitang.top/fs-doc/... 或 *.feishu.cn/wiki/... 文档链接(必填)
|
||||
输出文件名 保存路径,默认为文档标题.md
|
||||
|
||||
示例:
|
||||
node auto-scrape.mjs https://yitang.top/fs-doc/abc/docId
|
||||
node auto-scrape.mjs https://yitanger.feishu.cn/wiki/R9NywV2i9izjuqkPd63cWuafn4g
|
||||
node auto-scrape.mjs https://yitang.top/fs-doc/abc/docId 课程笔记.md
|
||||
`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const TARGET_URL = args[0].trim();
|
||||
const customOutput = args[1];
|
||||
const COOKIE_FILE = resolve('cookies.json');
|
||||
|
||||
// ── 判断 URL 类型 ──────────────────────────────────────────────────────────────
|
||||
const isFeishuWiki = /feishu\.cn\/wiki\//.test(TARGET_URL);
|
||||
const isFeishuDocx = /feishu\.cn\/docx\//.test(TARGET_URL);
|
||||
const isYitang = TARGET_URL.includes('yitang.top/fs-doc/');
|
||||
|
||||
if (!isFeishuWiki && !isFeishuDocx && !isYitang) {
|
||||
console.error('❌ URL 格式不正确,支持 yitang.top/fs-doc/<acl>/<docId> 或 *.feishu.cn/wiki/<token>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// ── Feishu Wiki:直接调用 fetch-feishu-wiki.mjs ────────────────────────────────
|
||||
if (isFeishuWiki) {
|
||||
console.log('🔄 检测到飞书 Wiki 文档,使用 lark-cli API 获取...\n');
|
||||
try {
|
||||
const fetchScript = resolve(__dirname, 'fetch-feishu-wiki.mjs');
|
||||
execSync(`node "${fetchScript}" "${TARGET_URL}"`, {
|
||||
stdio: 'inherit',
|
||||
encoding: 'utf-8'
|
||||
});
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('❌ 调用 fetch-feishu-wiki.mjs 失败');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// yitang 专用字段
|
||||
let ACL, DOC_ID, LOGIN_URL;
|
||||
if (isYitang) {
|
||||
const match = TARGET_URL.match(/\/fs-doc\/([^/]+)\/([^/?#]+)/);
|
||||
if (!match) {
|
||||
console.error('❌ yitang URL 格式不正确,应为 https://yitang.top/fs-doc/<acl>/<docId>');
|
||||
process.exit(1);
|
||||
}
|
||||
ACL = match[1];
|
||||
DOC_ID = match[2];
|
||||
LOGIN_URL = `https://yitang.top/login?after_login=%2Ffs-doc%2F${ACL}%2F${DOC_ID}`;
|
||||
}
|
||||
|
||||
// ── Feishu block → Markdown ──────────────────────────────────────────────────
|
||||
const T = { PAGE:1, TEXT:2, H1:3, H2:4, H3:5, H4:6, H5:7, H6:8,
|
||||
ORDERED:13, BULLET:12, TODO:17, CODE:14, QUOTE:15, DIVIDER:22,
|
||||
IMAGE:27, TABLE:31, TABLE_CELL:32, QUOTE_CONTAINER:34,
|
||||
GRID:24, GRID_COL:25 };
|
||||
|
||||
function textRuns(elements) {
|
||||
if (!Array.isArray(elements)) return '';
|
||||
return elements.map(el => {
|
||||
const run = el.text_run;
|
||||
if (!run) return '';
|
||||
let t = run.content || '';
|
||||
const s = run.text_element_style || {};
|
||||
if (s.bold) t = `**${t}**`;
|
||||
if (s.italic) t = `*${t}*`;
|
||||
if (s.strikethrough) t = `~~${t}~~`;
|
||||
if (s.inline_code) t = `\`${t}\``;
|
||||
if (s.link?.url) t = `[${t}](${s.link.url})`;
|
||||
return t;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
function blockToMd(block, depth = 0) {
|
||||
if (!block) return '';
|
||||
const attr = block.blockAttr || {};
|
||||
const indent = ' '.repeat(Math.max(0, depth - 1));
|
||||
switch (block.type) {
|
||||
case T.PAGE: return `# ${textRuns(attr.page?.elements)}\n\n`;
|
||||
case T.H1: return `# ${textRuns(attr.heading1?.elements)}\n\n`;
|
||||
case T.H2: return `## ${textRuns(attr.heading2?.elements)}\n\n`;
|
||||
case T.H3: return `### ${textRuns(attr.heading3?.elements)}\n\n`;
|
||||
case T.H4: return `#### ${textRuns(attr.heading4?.elements)}\n\n`;
|
||||
case T.H5: return `##### ${textRuns(attr.heading5?.elements)}\n\n`;
|
||||
case T.H6: return `###### ${textRuns(attr.heading6?.elements)}\n\n`;
|
||||
case T.TEXT: {
|
||||
const txt = textRuns(attr.text?.elements);
|
||||
return txt ? `${txt}\n\n` : '\n';
|
||||
}
|
||||
case T.BULLET: return `${indent}- ${textRuns(attr.bullet?.elements)}\n`;
|
||||
case T.ORDERED: return `${indent}1. ${textRuns(attr.ordered?.elements)}\n`;
|
||||
case T.TODO: {
|
||||
const done = attr.todo?.style?.done ? '[x]' : '[ ]';
|
||||
return `${indent}- ${done} ${textRuns(attr.todo?.elements)}\n`;
|
||||
}
|
||||
case T.QUOTE: return `> ${textRuns(attr.quote?.elements)}\n\n`;
|
||||
case T.CODE: {
|
||||
const language = attr.code?.style?.language;
|
||||
const lang = typeof language === 'string' ? language.toLowerCase().replace('plain text','') : '';
|
||||
const code = (attr.code?.elements || []).map(e => e.text_run?.content || '').join('');
|
||||
return `\`\`\`${lang}\n${code}\n\`\`\``;
|
||||
}
|
||||
case T.DIVIDER: return `---\n\n`;
|
||||
case T.IMAGE: {
|
||||
const caption = (attr.textArea || []).map(a => a.text).filter(Boolean).join(' ');
|
||||
const url = attr.cdnUrl || attr.image?.token || '';
|
||||
return `\n\n`;
|
||||
}
|
||||
case T.QUOTE_CONTAINER: return '';
|
||||
case T.GRID: return '';
|
||||
case T.GRID_COL: return '';
|
||||
case T.TABLE: return '\n';
|
||||
case T.TABLE_CELL: {
|
||||
const text = (block.childrens || []).map(c => {
|
||||
const a = c.blockAttr || {};
|
||||
return textRuns(a.text?.elements || a.heading1?.elements || a.heading2?.elements || []);
|
||||
}).filter(Boolean).join(' | ');
|
||||
return text ? `| ${text} ` : '';
|
||||
}
|
||||
default: return '';
|
||||
}
|
||||
}
|
||||
|
||||
function blocksToMarkdown(rootBlock) {
|
||||
const lines = [];
|
||||
function walk(block, depth) {
|
||||
if (!block) return;
|
||||
lines.push(blockToMd(block, depth));
|
||||
if (block.type !== T.TABLE_CELL && block.childrens) {
|
||||
block.childrens.forEach(c => walk(c, depth + 1));
|
||||
}
|
||||
if (block.type === T.TABLE) lines.push('\n');
|
||||
}
|
||||
walk(rootBlock, 0);
|
||||
return lines.join('').replace(/\n{3,}/g, '\n\n').trim();
|
||||
}
|
||||
|
||||
// ── Feishu 原生 docx block (block_type schema) → Markdown ────────────────────
|
||||
// 飞书 Open Platform block_type: 1=page 2=text 3-8=h1-h6 12=bullet 13=ordered
|
||||
// 14=code 15=quote 16=todo 21=divider 26=image 30=table 31=table_cell
|
||||
|
||||
const FT_HEADING = { 3: 1, 4: 2, 5: 3, 6: 4, 7: 5, 8: 6 };
|
||||
const FT_CODE_LANG = {
|
||||
1: '', 7: 'bash', 9: 'cpp', 10: 'c', 12: 'css', 22: 'go',
|
||||
28: 'json', 29: 'java', 30: 'javascript', 49: 'python',
|
||||
52: 'ruby', 53: 'rust', 56: 'sql', 60: 'shell', 63: 'typescript',
|
||||
};
|
||||
|
||||
function ftTextElements(elements = []) {
|
||||
return elements.map(el => {
|
||||
if (el.text_run) {
|
||||
const content = el.text_run.content || '';
|
||||
const s = el.text_run.text_element_style || {};
|
||||
let t = content;
|
||||
if (s.inline_code) return `\`${t}\``;
|
||||
if (s.bold) t = `**${t}**`;
|
||||
if (s.italic) t = `*${t}*`;
|
||||
if (s.strikethrough) t = `~~${t}~~`;
|
||||
if (s.link?.url) t = `[${t}](${decodeURIComponent(s.link.url)})`;
|
||||
return t;
|
||||
}
|
||||
if (el.mention_doc) {
|
||||
const { url = '', title = '' } = el.mention_doc;
|
||||
return url ? `[${title || url}](${url})` : title;
|
||||
}
|
||||
if (el.mention_user) return `@${el.mention_user.user_id || 'user'}`;
|
||||
return '';
|
||||
}).join('');
|
||||
}
|
||||
|
||||
function ftBlockToMd(block, index, depth, orderedNum = 1) {
|
||||
const t = block.block_type;
|
||||
const indent = ' '.repeat(depth);
|
||||
if (t === 1) return ftRenderChildren(block.children || [], index, depth);
|
||||
if (t === 2) {
|
||||
const text = ftTextElements(block.text?.elements);
|
||||
return text ? text + '\n' : '\n';
|
||||
}
|
||||
if (FT_HEADING[t] !== undefined) {
|
||||
return `${'#'.repeat(FT_HEADING[t])} ${ftTextElements(block.text?.elements)}\n\n`;
|
||||
}
|
||||
if (t === 12) {
|
||||
const nested = block.children?.length ? '\n' + ftRenderChildren(block.children, index, depth + 1) : '';
|
||||
return `${indent}- ${ftTextElements(block.text?.elements)}${nested}\n`;
|
||||
}
|
||||
if (t === 13) {
|
||||
const nested = block.children?.length ? '\n' + ftRenderChildren(block.children, index, depth + 1) : '';
|
||||
return `${indent}${orderedNum}. ${ftTextElements(block.text?.elements)}${nested}\n`;
|
||||
}
|
||||
if (t === 14) {
|
||||
const langId = block.code?.style?.language ?? 1;
|
||||
const code = ftTextElements(block.code?.elements || []);
|
||||
return `\`\`\`${FT_CODE_LANG[langId] ?? ''}\n${code}\n\`\`\`\n`;
|
||||
}
|
||||
if (t === 15) return `> ${ftTextElements(block.text?.elements)}\n\n`;
|
||||
if (t === 16) {
|
||||
const done = block.text?.style?.done ? '[x]' : '[ ]';
|
||||
return `- ${done} ${ftTextElements(block.text?.elements)}\n`;
|
||||
}
|
||||
if (t === 21) return '\n---\n\n';
|
||||
if (t === 26) {
|
||||
const token = block.image?.token || '';
|
||||
const src = token
|
||||
? `https://internal-api-drive-stream.feishu.cn/space/api/box/stream/download/all/${token}`
|
||||
: '';
|
||||
return `\n\n`;
|
||||
}
|
||||
if (t === 30) return ftRenderTable(block, index) + '\n';
|
||||
if (t === 31) return null;
|
||||
const fallback = ftTextElements(block.text?.elements || []);
|
||||
return fallback ? fallback + '\n' : null;
|
||||
}
|
||||
|
||||
function ftRenderChildren(childIds, index, depth) {
|
||||
const lines = [];
|
||||
let n = 1;
|
||||
for (const id of childIds) {
|
||||
const block = index.get(id);
|
||||
if (!block) continue;
|
||||
if (block.block_type === 13) {
|
||||
const line = ftBlockToMd(block, index, depth, n++);
|
||||
if (line != null) lines.push(line);
|
||||
} else {
|
||||
n = 1;
|
||||
const line = ftBlockToMd(block, index, depth);
|
||||
if (line != null) lines.push(line);
|
||||
}
|
||||
}
|
||||
return lines.join('');
|
||||
}
|
||||
|
||||
function ftRenderTable(block, index) {
|
||||
const { row_size = 0, column_size = 0 } = block.table?.property || {};
|
||||
const cellIds = block.table?.cells || [];
|
||||
if (!row_size || !column_size) return '';
|
||||
const rows = Array.from({ length: row_size }, () => Array(column_size).fill(''));
|
||||
for (let r = 0; r < row_size; r++) {
|
||||
for (let c = 0; c < column_size; c++) {
|
||||
const cell = index.get(cellIds[r * column_size + c]);
|
||||
if (!cell) continue;
|
||||
rows[r][c] = ftRenderChildren(cell.children || [], index, 0).replace(/\n+/g, ' ').trim();
|
||||
}
|
||||
}
|
||||
const out = ['| ' + rows[0].join(' | ') + ' |', '| ' + Array(column_size).fill('---').join(' | ') + ' |'];
|
||||
for (let r = 1; r < row_size; r++) out.push('| ' + rows[r].join(' | ') + ' |');
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
function feishuBlocksToMd(blocks) {
|
||||
const index = new Map();
|
||||
for (const b of blocks) if (b.block_id) index.set(b.block_id, b);
|
||||
const root = blocks.find(b => b.block_type === 1) || blocks[0];
|
||||
if (!root) return blocks.map(b => ftBlockToMd(b, index, 0)).filter(Boolean).join('\n');
|
||||
return ftRenderChildren(root.children || [], index, 0).replace(/\n{3,}/g, '\n\n').trim();
|
||||
}
|
||||
|
||||
// 从 API 响应 JSON 中递归查找 block 数组(block_type schema)
|
||||
function findFeishuBlocks(obj, depth = 0) {
|
||||
if (depth > 5 || !obj || typeof obj !== 'object') return [];
|
||||
if (obj.block_id && typeof obj.block_type === 'number') return [obj];
|
||||
if (Array.isArray(obj)) {
|
||||
const flat = [];
|
||||
for (const item of obj) flat.push(...findFeishuBlocks(item, depth + 1));
|
||||
return flat;
|
||||
}
|
||||
for (const c of [obj.items, obj.blocks, obj.data?.items, obj.data?.blocks,
|
||||
obj.data?.document?.blocks, obj.result?.data?.items]) {
|
||||
if (c) { const found = findFeishuBlocks(c, depth + 1); if (found.length) return found; }
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
// 从 wiki node API 响应中找出真实 docx token 和标题
|
||||
function findWikiObjToken(obj, wikiToken, depth = 0) {
|
||||
if (depth > 6 || !obj || typeof obj !== 'object') return null;
|
||||
if (obj.obj_token && (obj.node_token === wikiToken || obj.wiki_token === wikiToken || depth > 0)) return obj;
|
||||
if (obj.obj_token && obj.title) return obj;
|
||||
if (Array.isArray(obj)) {
|
||||
for (const item of obj) { const f = findWikiObjToken(item, wikiToken, depth + 1); if (f) return f; }
|
||||
return null;
|
||||
}
|
||||
for (const key of Object.keys(obj)) {
|
||||
const f = findWikiObjToken(obj[key], wikiToken, depth + 1);
|
||||
if (f) return f;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── Main ─────────────────────────────────────────────────────────────────────
|
||||
console.log('╔════════════════════════════════════════════╗');
|
||||
console.log('║ 一堂文档采集工具 ║');
|
||||
console.log('╚════════════════════════════════════════════╝');
|
||||
console.log(`📎 URL: ${TARGET_URL}`);
|
||||
if (isYitang) {
|
||||
console.log(`📄 ACL: ${ACL}`);
|
||||
console.log(`🔖 DocID: ${DOC_ID}`);
|
||||
} else {
|
||||
const ftToken = TARGET_URL.match(/\/(wiki|docx)\/([A-Za-z0-9_-]+)/);
|
||||
console.log(`🔖 Type: feishu ${ftToken?.[1] || ''} Token: ${ftToken?.[2] || ''}`);
|
||||
}
|
||||
console.log('');
|
||||
|
||||
const hasSavedCookies = existsSync(COOKIE_FILE);
|
||||
const browser = await chromium.launch({ headless: false, args: ['--window-size=820,720'] });
|
||||
const context = await browser.newContext();
|
||||
|
||||
if (hasSavedCookies) {
|
||||
const saved = JSON.parse(readFileSync(COOKIE_FILE, 'utf-8'));
|
||||
await context.addCookies(saved);
|
||||
console.log(`✅ 已加载保存的 Cookie(${saved.length} 条),跳过扫码登录`);
|
||||
} else {
|
||||
console.log('首次运行,请在弹出的浏览器中完成登录...');
|
||||
}
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
let apiResponseText = null;
|
||||
const feishuApiResponses = [];
|
||||
|
||||
// 飞书原生 docx/wiki API 匹配模式
|
||||
const FEISHU_DOC_RE = [
|
||||
/\/docx\/v\d+\/documents\/[A-Za-z0-9]+\/blocks/,
|
||||
/\/space\/api\/docs\/v\d+\/raw_api/,
|
||||
/apiName=DocxV\d+Document/i,
|
||||
];
|
||||
const FEISHU_WIKI_RE = [
|
||||
/\/wiki\/v\d+\/spaces?\/[^/]+\/nodes?/,
|
||||
/\/wiki\/api\/.*\/(get_?node|node_meta|node_info)/,
|
||||
/\/space\/api\/wiki\/v\d+\/.*node/,
|
||||
/apiName=WikiV\d+/i,
|
||||
];
|
||||
|
||||
page.on('response', async (resp) => {
|
||||
const u = resp.url();
|
||||
|
||||
if (isFeishu) {
|
||||
const isDoc = FEISHU_DOC_RE.some(p => p.test(u));
|
||||
const isWiki = FEISHU_WIKI_RE.some(p => p.test(u));
|
||||
// 跳过非 JSON、静态资源、已知无用的请求
|
||||
const ct = resp.headers()['content-type'] || '';
|
||||
if (!ct.includes('json') && !ct.includes('text/plain')) return;
|
||||
if (resp.status() < 200 || resp.status() >= 300) return;
|
||||
|
||||
try {
|
||||
const text = await resp.text();
|
||||
let json; try { json = JSON.parse(text); } catch { return; }
|
||||
|
||||
if (isDoc || isWiki) {
|
||||
console.log(`\n🎯 捕获飞书 API (${isWiki ? 'wiki' : 'doc'}):${u.split('?')[0]} (${text.length} 字节)`);
|
||||
feishuApiResponses.push({ url: u, data: json, kind: isWiki ? 'wiki' : 'doc' });
|
||||
return;
|
||||
}
|
||||
|
||||
// 兜底:扫描任意 JSON 响应中是否含有 block 数据
|
||||
const blocks = findFeishuBlocks(json);
|
||||
if (blocks.length > 0) {
|
||||
console.log(`\n📦 兜底捕获 block 数据:${u.split('?')[0]} (${blocks.length} 个 block)`);
|
||||
feishuApiResponses.push({ url: u, data: json, kind: 'doc' });
|
||||
}
|
||||
} catch {}
|
||||
return;
|
||||
}
|
||||
|
||||
// yitang 路径
|
||||
if (u.includes('get-doc-blocks') || u.includes('get-blocks')) {
|
||||
try {
|
||||
const text = await resp.text();
|
||||
// 忽略"未登录"错误响应(code 10)
|
||||
try {
|
||||
const parsed = JSON.parse(text);
|
||||
if (parsed.code === 10) {
|
||||
console.log(`\n⚠️ API 返回未登录,等待真实响应...`);
|
||||
return;
|
||||
}
|
||||
} catch {}
|
||||
apiResponseText = text;
|
||||
console.log(`\n🎯 捕获 API 响应:${u.split('?')[0]} (${text.length} 字节)`);
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
|
||||
console.log('🌐 正在打开文档页面...');
|
||||
await page.goto(TARGET_URL, { waitUntil: 'load', timeout: 30000 });
|
||||
|
||||
// 检测是否需要登录
|
||||
const needLogin = async () => {
|
||||
const u = page.url();
|
||||
if (isFeishu) {
|
||||
// 飞书登录页:passport.feishu.cn / accounts.feishu.cn / /login
|
||||
return u.includes('passport.feishu') || u.includes('accounts.feishu') || u.includes('/login');
|
||||
}
|
||||
// yitang 路径
|
||||
if (u.includes('/login') || u.includes('sso.')) return true;
|
||||
const tok = await readToken();
|
||||
if (!tok && hasSavedCookies) return true;
|
||||
return false;
|
||||
};
|
||||
|
||||
if (await needLogin()) {
|
||||
console.log('⚠️ 需要登录,正在跳转...');
|
||||
if (isFeishu) {
|
||||
if (!page.url().includes('passport') && !page.url().includes('accounts') && !page.url().includes('/login')) {
|
||||
await page.goto(TARGET_URL, { waitUntil: 'load', timeout: 30000 });
|
||||
}
|
||||
console.log('请在浏览器中完成飞书登录...');
|
||||
await page.waitForFunction(
|
||||
() => !location.href.includes('passport') && !location.href.includes('accounts') && !location.href.includes('/login'),
|
||||
{ timeout: 300000 }
|
||||
);
|
||||
} else {
|
||||
if (!page.url().includes('/login') && !page.url().includes('sso.')) {
|
||||
await page.goto(LOGIN_URL, { waitUntil: 'load', timeout: 30000 });
|
||||
}
|
||||
console.log('请在浏览器中完成登录(微信扫码或手机验证码)...');
|
||||
await page.waitForURL('**/fs-doc/**', { timeout: 300000 });
|
||||
}
|
||||
console.log('\n✅ 登录成功');
|
||||
await page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => {});
|
||||
}
|
||||
|
||||
// 保存最新 Cookie
|
||||
const cookies = await context.cookies();
|
||||
writeFileSync(COOKIE_FILE, JSON.stringify(cookies, null, 2));
|
||||
console.log(`💾 Cookie 已保存`);
|
||||
|
||||
// ── 飞书原生路径 ───────────────────────────────────────────────────────────────
|
||||
if (isFeishu) {
|
||||
if (feishuApiResponses.length === 0) {
|
||||
console.log('⏳ 等待飞书 API 数据加载...');
|
||||
await page.waitForTimeout(8000);
|
||||
}
|
||||
if (feishuApiResponses.length === 0) {
|
||||
console.log('🔄 重新加载页面...');
|
||||
await page.goto(TARGET_URL, { waitUntil: 'networkidle', timeout: 30000 });
|
||||
await page.waitForTimeout(5000);
|
||||
}
|
||||
if (feishuApiResponses.length === 0) {
|
||||
console.error('❌ 未捕获到飞书 API 响应,请检查登录状态或文档权限');
|
||||
await browser.close();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// 提取 wiki node 元信息(wiki 页面需先拿 obj_token)
|
||||
let wikiMeta = null;
|
||||
const ftToken = TARGET_URL.match(/\/(wiki|docx)\/([A-Za-z0-9_-]+)/)?.[2] || '';
|
||||
if (TARGET_URL.includes('/wiki/')) {
|
||||
for (const item of feishuApiResponses) {
|
||||
if (item.kind === 'wiki') {
|
||||
wikiMeta = findWikiObjToken(item.data, ftToken);
|
||||
if (wikiMeta) {
|
||||
console.log(`📎 wiki → docx token: ${wikiMeta.obj_token} 标题: ${wikiMeta.title || '(未知)'}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 如果是 wiki 页面且找到了 docx token,需要导航到 docx URL 来获取实际内容
|
||||
if (wikiMeta?.obj_token) {
|
||||
const docxUrl = TARGET_URL.replace(/\/wiki\/[^/]+/, `/docx/${wikiMeta.obj_token}`);
|
||||
console.log(`🔄 正在加载文档内容: ${docxUrl}`);
|
||||
feishuApiResponses.length = 0; // 清空之前的 wiki API 响应
|
||||
await page.goto(docxUrl, { waitUntil: 'load', timeout: 30000 });
|
||||
await page.waitForTimeout(3000);
|
||||
|
||||
if (feishuApiResponses.length === 0) {
|
||||
console.log('⏳ 等待文档 API 数据加载...');
|
||||
await page.waitForTimeout(5000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 提取 blocks(去重)
|
||||
let docxBlocks = [];
|
||||
for (const item of feishuApiResponses) {
|
||||
if (item.kind !== 'doc') continue;
|
||||
const found = findFeishuBlocks(item.data);
|
||||
if (found.length) docxBlocks.push(...found);
|
||||
}
|
||||
const seenIds = new Set();
|
||||
docxBlocks = docxBlocks.filter(b => {
|
||||
if (!b.block_id || seenIds.has(b.block_id)) return false;
|
||||
seenIds.add(b.block_id);
|
||||
return true;
|
||||
});
|
||||
console.log(`📦 共提取 ${docxBlocks.length} 个 block`);
|
||||
|
||||
if (docxBlocks.length === 0) {
|
||||
console.error('❌ 未获取到有效 block 数据');
|
||||
await browser.close();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const pageTitle = wikiMeta?.title || await page.title();
|
||||
const markdown = feishuBlocksToMd(docxBlocks);
|
||||
const output = `# ${pageTitle}\n\n> Source: ${TARGET_URL}\n\n${markdown}`;
|
||||
const safeTitle = pageTitle.replace(/[/\\:*?"<>|]/g, '_').trim();
|
||||
const outDir = resolve('output');
|
||||
if (!existsSync(outDir)) mkdirSync(outDir, { recursive: true });
|
||||
const outputPath = resolve(customOutput || `output/${safeTitle || 'output'}.md`);
|
||||
writeFileSync(outputPath, output, 'utf-8');
|
||||
|
||||
console.log('');
|
||||
console.log('╔════════════════════════════════════════════╗');
|
||||
console.log(`║ ✅ 完成!共 ${output.length} 字符`);
|
||||
console.log(`║ 📄 ${outputPath}`);
|
||||
console.log('╚════════════════════════════════════════════╝');
|
||||
await browser.close();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// 读取 TOKEN(函数,方便重登后复用)
|
||||
async function readToken() {
|
||||
return page.evaluate(() => {
|
||||
try {
|
||||
const raw = localStorage.getItem('#|g.TOKEN')
|
||||
|| Object.entries(localStorage).find(([k]) => k.includes('TOKEN'))?.[1]
|
||||
|| null;
|
||||
if (!raw) return null;
|
||||
return raw.startsWith('{') ? JSON.parse(raw).$v || null : raw;
|
||||
} catch { return null; }
|
||||
}).catch(() => null);
|
||||
}
|
||||
|
||||
// 读取 TOKEN
|
||||
let capturedToken = await page.evaluate(() => {
|
||||
try {
|
||||
const raw = localStorage.getItem('#|g.TOKEN')
|
||||
|| Object.entries(localStorage).find(([k]) => k.includes('TOKEN'))?.[1]
|
||||
|| null;
|
||||
if (!raw) return null;
|
||||
return raw.startsWith('{') ? JSON.parse(raw).$v || null : raw;
|
||||
} catch { return null; }
|
||||
}).catch(() => null);
|
||||
console.log('🔑 TOKEN:', capturedToken ? capturedToken.substring(0, 8) + '...' : '未找到');
|
||||
|
||||
if (!apiResponseText) {
|
||||
console.log('⏳ 等待 API 数据加载...');
|
||||
await page.waitForTimeout(10000);
|
||||
}
|
||||
if (!apiResponseText) {
|
||||
console.log('🔄 重新加载页面...');
|
||||
await page.goto(TARGET_URL, { waitUntil: 'networkidle', timeout: 30000 });
|
||||
await page.waitForTimeout(5000);
|
||||
}
|
||||
if (!apiResponseText) {
|
||||
console.error('❌ 未捕获到 API 响应,可能未登录或文档受限');
|
||||
await browser.close();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!apiResponseText) {
|
||||
console.error('❌ 仍未获取到有效数据,请检查是否已完成登录或文档是否有访问权限');
|
||||
await browser.close();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// ── 解密 ──────────────────────────────────────────────────────────────────────
|
||||
let docData;
|
||||
const envelope = JSON.parse(apiResponseText);
|
||||
const payload = envelope.data;
|
||||
|
||||
if (capturedToken && typeof payload === 'string') {
|
||||
try {
|
||||
const key = CryptoJS.enc.Utf8.parse(CryptoJS.MD5(capturedToken).toString());
|
||||
const iv = CryptoJS.enc.Utf8.parse(capturedToken);
|
||||
const dec = CryptoJS.AES.decrypt(payload.trim(), key, {
|
||||
iv, mode: CryptoJS.mode.CBC, padding: CryptoJS.pad.Pkcs7,
|
||||
});
|
||||
docData = JSON.parse(dec.toString(CryptoJS.enc.Utf8));
|
||||
console.log('🔓 AES 解密成功');
|
||||
} catch (e) {
|
||||
console.log('⚠️ AES 解密失败:', e.message);
|
||||
docData = typeof payload === 'object' ? payload : envelope;
|
||||
}
|
||||
} else {
|
||||
docData = typeof payload === 'object' ? payload : envelope;
|
||||
console.log('📄 直接使用响应数据');
|
||||
}
|
||||
|
||||
// ── 转换并保存 ────────────────────────────────────────────────────────────────
|
||||
const pageTitle = await page.title();
|
||||
const markdown = blocksToMarkdown(docData.blocks);
|
||||
const output = `# ${pageTitle}\n\n> Source: ${TARGET_URL}\n\n${markdown}`;
|
||||
|
||||
// 输出文件名:优先用参数,其次用标题,最后回退 output.md
|
||||
const safeTitle = pageTitle.replace(/[/\\:*?"<>|]/g, '_').trim();
|
||||
const outDir2 = resolve('output');
|
||||
if (!existsSync(outDir2)) mkdirSync(outDir2, { recursive: true });
|
||||
const outputPath = resolve(customOutput || `output/${safeTitle || 'output'}.md`);
|
||||
writeFileSync(outputPath, output, 'utf-8');
|
||||
|
||||
console.log('');
|
||||
console.log('╔════════════════════════════════════════════╗');
|
||||
console.log(`║ ✅ 完成!共 ${output.length} 字符`);
|
||||
console.log(`║ 📄 ${outputPath}`);
|
||||
console.log('╚════════════════════════════════════════════╝');
|
||||
|
||||
await browser.close();
|
||||
Reference in New Issue
Block a user