更新采集脚本：CHROME_PATH 自定义浏览器支持、tools/yitang-scraper
auto-scrape.mjs 支持 CHROME_PATH 环境变量指定浏览器可执行路径，解决本地 playwright 浏览器版本不匹配问题
This commit is contained in:
admin
+117
-23
@@ -13,10 +13,11 @@
|
||||
*/
|
||||
import { chromium } from 'playwright';
|
||||
import CryptoJS from 'crypto-js';
|
||||
import { writeFileSync, readFileSync, existsSync } from 'fs';
|
||||
import { resolve, basename, dirname } from 'path';
|
||||
import { writeFileSync, readFileSync, existsSync, mkdirSync } from 'fs';
|
||||
import { resolve, dirname } from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { createInterface } from 'readline/promises';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
@@ -31,7 +32,12 @@ if (existsSync(bundledBrowsers)) {
|
||||
}
|
||||
|
||||
// ── CLI 参数解析 ──────────────────────────────────────────────────────────────
|
||||
const args = process.argv.slice(2);
|
||||
let args = process.argv.slice(2);
|
||||
if (!args[0]) {
|
||||
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
||||
args = [(await rl.question('请输入一堂文档链接：')).trim()];
|
||||
rl.close();
|
||||
}
|
||||
if (!args[0] || args[0] === '--help' || args[0] === '-h') {
|
||||
console.log(`
|
||||
用法：node auto-scrape.mjs <URL> [输出文件名]
|
||||
@@ -43,10 +49,24 @@ if (!args[0] || args[0] === '--help' || args[0] === '-h') {
|
||||
node auto-scrape.mjs https://yitang.top/fs-doc/abc/docId
|
||||
node auto-scrape.mjs https://your-tenant.feishu.cn/wiki/WikiNodeTokenExample
|
||||
node auto-scrape.mjs https://yitang.top/fs-doc/abc/docId 课程笔记.md
|
||||
node auto-scrape.mjs --clean 已有文档.md
|
||||
`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ── --clean：对已生成的 Markdown 文件执行清洗（不启动浏览器）──────────────────
|
||||
if (args[0] === '--clean') {
|
||||
const file = resolve(args[1] || '');
|
||||
if (!args[1] || !existsSync(file)) {
|
||||
console.error('用法：node auto-scrape.mjs --clean <markdown文件>');
|
||||
process.exit(1);
|
||||
}
|
||||
const md = readFileSync(file, 'utf-8');
|
||||
writeFileSync(file, cleanMarkdown(md), 'utf-8');
|
||||
console.log(`✅ 已清洗：${file}`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const TARGET_URL = args[0].trim();
|
||||
const customOutput = args[1];
|
||||
const COOKIE_FILE = resolve('cookies.json');
|
||||
@@ -55,6 +75,7 @@ const COOKIE_FILE = resolve('cookies.json');
|
||||
const isFeishuWiki = /feishu\.cn\/wiki\//.test(TARGET_URL);
|
||||
const isFeishuDocx = /feishu\.cn\/docx\//.test(TARGET_URL);
|
||||
const isYitang = TARGET_URL.includes('yitang.top/fs-doc/');
|
||||
const isFeishu = isFeishuWiki || isFeishuDocx;
|
||||
|
||||
if (!isFeishuWiki && !isFeishuDocx && !isYitang) {
|
||||
console.error('❌ URL 格式不正确，支持 yitang.top/fs-doc/<acl>/<docId> 或 *.feishu.cn/wiki/<token>');
|
||||
@@ -90,6 +111,60 @@ if (isYitang) {
|
||||
LOGIN_URL = `https://yitang.top/login?after_login=%2Ffs-doc%2F${ACL}%2F${DOC_ID}`;
|
||||
}
|
||||
|
||||
// ── Markdown 清洗（CommonMark 定界符合规）────────────────────────────────────
|
||||
// 1) 相邻同样式文本块合并：**甲****乙** → **甲乙**
|
||||
// 2) 加粗/斜体/删除线以中文标点结束、后接汉字/字母/数字时，闭合标记后补空格：**标签：** 研发
|
||||
// 3) 标题行去除 **、*、~~ 等格式标记残留
|
||||
function stripHeadingMarkers(line) {
|
||||
const m = line.match(/^(#{1,6}\s+)(.*)$/);
|
||||
if (!m) return line;
|
||||
let inner = m[2], prev;
|
||||
do {
|
||||
prev = inner;
|
||||
inner = inner
|
||||
.replace(/\*\*([^*]+)\*\*/g, '$1')
|
||||
.replace(/__([^_]+)__/g, '$1')
|
||||
.replace(/~~([^~]+)~~/g, '$1')
|
||||
.replace(/(^|[^*])\*([^*]+)\*/g, '$1$2');
|
||||
} while (inner !== prev);
|
||||
return m[1] + inner.trimEnd();
|
||||
}
|
||||
|
||||
function mergeAdjacentSpans(line) {
|
||||
let prev;
|
||||
do {
|
||||
prev = line;
|
||||
line = line
|
||||
.replace(/\*\*([^*\n]+)\*\*\*\*([^*\n]+)\*\*/g, '**$1$2**')
|
||||
.replace(/\*\*\*([^*\n]+?)\*\*\*\*\*\*([^*\n]+?)\*\*\*/g, '***$1$2***')
|
||||
.replace(/~~([^~\n]+)~~~~([^~\n]+)~~/g, '~~$1$2~~');
|
||||
} while (line !== prev);
|
||||
return line;
|
||||
}
|
||||
|
||||
function fixClosingDelimiters(line) {
|
||||
// CommonMark 侧翼规则：闭合定界符前为标点、后紧跟字母/数字（含汉字）时无法闭合，需补空格
|
||||
const addSpace = (whole, marker, inner) =>
|
||||
/[\p{P}\p{S}]/u.test(inner.slice(-1)) ? `${marker}${inner}${marker} ` : whole;
|
||||
line = line.replace(/(?<!\*)\*\*\*([^*\n]+?)\*\*\*(?!\*)(?=[\p{L}\p{N}])/gu, (m, inner) => addSpace(m, '***', inner));
|
||||
line = line.replace(/\*\*([^*\n]+)\*\*(?=[\p{L}\p{N}])/gu, (m, inner) => addSpace(m, '**', inner));
|
||||
line = line.replace(/~~([^~\n]+)~~(?=[\p{L}\p{N}])/gu, (m, inner) => addSpace(m, '~~', inner));
|
||||
line = line.replace(/(?<!\*)\*([^*\n]+)\*(?!\*)(?=[\p{L}\p{N}])/gu, (m, inner) => addSpace(m, '*', inner));
|
||||
return line;
|
||||
}
|
||||
|
||||
function cleanMarkdown(md) {
|
||||
// 迭代至不动点，确保单遍扫描因定界符交叠漏掉的模式也被处理
|
||||
let prev;
|
||||
do {
|
||||
prev = md;
|
||||
md = md.split('\n')
|
||||
.map(l => fixClosingDelimiters(mergeAdjacentSpans(stripHeadingMarkers(l))))
|
||||
.join('\n');
|
||||
} while (md !== prev);
|
||||
return md;
|
||||
}
|
||||
|
||||
// ── Feishu block → Markdown ──────────────────────────────────────────────────
|
||||
const T = { PAGE:1, TEXT:2, H1:3, H2:4, H3:5, H4:6, H5:7, H6:8,
|
||||
ORDERED:13, BULLET:12, TODO:17, CODE:14, QUOTE:15, DIVIDER:22,
|
||||
@@ -98,11 +173,18 @@ const T = { PAGE:1, TEXT:2, H1:3, H2:4, H3:5, H4:6, H5:7, H6:8,
|
||||
|
||||
function textRuns(elements) {
|
||||
if (!Array.isArray(elements)) return '';
|
||||
return elements.map(el => {
|
||||
// 相同样式的相邻 run 先合并，避免生成 **甲****乙** 这类相邻定界符
|
||||
const runs = [];
|
||||
for (const el of elements) {
|
||||
const run = el.text_run;
|
||||
if (!run) return '';
|
||||
let t = run.content || '';
|
||||
if (!run) continue;
|
||||
const s = run.text_element_style || {};
|
||||
const key = `${!!s.bold}|${!!s.italic}|${!!s.strikethrough}|${!!s.inline_code}|${s.link?.url || ''}`;
|
||||
const last = runs[runs.length - 1];
|
||||
if (last && last.key === key) last.content += run.content || '';
|
||||
else runs.push({ key, content: run.content || '', style: s });
|
||||
}
|
||||
return runs.map(({ content: t, style: s }) => {
|
||||
if (s.bold) t = `**${t}**`;
|
||||
if (s.italic) t = `*${t}*`;
|
||||
if (s.strikethrough) t = `~~${t}~~`;
|
||||
@@ -188,24 +270,34 @@ const FT_CODE_LANG = {
|
||||
};
|
||||
|
||||
function ftTextElements(elements = []) {
|
||||
return elements.map(el => {
|
||||
// 相同样式的相邻 text_run 先合并，避免生成 **甲****乙** 这类相邻定界符
|
||||
const segs = [];
|
||||
for (const el of elements) {
|
||||
if (el.text_run) {
|
||||
const content = el.text_run.content || '';
|
||||
const s = el.text_run.text_element_style || {};
|
||||
let t = content;
|
||||
if (s.inline_code) return `\`${t}\``;
|
||||
if (s.bold) t = `**${t}**`;
|
||||
if (s.italic) t = `*${t}*`;
|
||||
if (s.strikethrough) t = `~~${t}~~`;
|
||||
if (s.link?.url) t = `[${t}](${decodeURIComponent(s.link.url)})`;
|
||||
return t;
|
||||
}
|
||||
if (el.mention_doc) {
|
||||
const key = `${!!s.inline_code}|${!!s.bold}|${!!s.italic}|${!!s.strikethrough}|${s.link?.url || ''}`;
|
||||
const last = segs[segs.length - 1];
|
||||
if (last && last.key === key) last.content += el.text_run.content || '';
|
||||
else segs.push({ key, content: el.text_run.content || '', style: s });
|
||||
} else if (el.mention_doc) {
|
||||
const { url = '', title = '' } = el.mention_doc;
|
||||
return url ? `[${title || url}](${url})` : title;
|
||||
segs.push({ key: null, text: url ? `[${title || url}](${url})` : title });
|
||||
} else if (el.mention_user) {
|
||||
segs.push({ key: null, text: `@${el.mention_user.user_id || 'user'}` });
|
||||
} else {
|
||||
segs.push({ key: null, text: '' });
|
||||
}
|
||||
if (el.mention_user) return `@${el.mention_user.user_id || 'user'}`;
|
||||
return '';
|
||||
}
|
||||
return segs.map(seg => {
|
||||
if (seg.key === null) return seg.text;
|
||||
let t = seg.content;
|
||||
const s = seg.style;
|
||||
if (s.inline_code) return `\`${t}\``;
|
||||
if (s.bold) t = `**${t}**`;
|
||||
if (s.italic) t = `*${t}*`;
|
||||
if (s.strikethrough) t = `~~${t}~~`;
|
||||
if (s.link?.url) t = `[${t}](${decodeURIComponent(s.link.url)})`;
|
||||
return t;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
@@ -342,7 +434,9 @@ if (isYitang) {
|
||||
console.log('');
|
||||
|
||||
const hasSavedCookies = existsSync(COOKIE_FILE);
|
||||
const browser = await chromium.launch({ headless: false, args: ['--window-size=820,720'] });
|
||||
const launchOpts = { headless: false, args: ['--window-size=820,720'] };
|
||||
if (process.env.CHROME_PATH) launchOpts.executablePath = process.env.CHROME_PATH;
|
||||
const browser = await chromium.launch(launchOpts);
|
||||
const context = await browser.newContext();
|
||||
|
||||
if (hasSavedCookies) {
|
||||
@@ -532,7 +626,7 @@ if (isFeishu) {
|
||||
}
|
||||
|
||||
const pageTitle = wikiMeta?.title || await page.title();
|
||||
const markdown = feishuBlocksToMd(docxBlocks);
|
||||
const markdown = cleanMarkdown(feishuBlocksToMd(docxBlocks));
|
||||
const output = `# ${pageTitle}\n\n> Source: ${TARGET_URL}\n\n${markdown}`;
|
||||
const safeTitle = pageTitle.replace(/[/\\:*?"<>|]/g, '_').trim();
|
||||
const outDir = resolve('output');
|
||||
@@ -620,7 +714,7 @@ if (capturedToken && typeof payload === 'string') {
|
||||
|
||||
// ── 转换并保存 ────────────────────────────────────────────────────────────────
|
||||
const pageTitle = await page.title();
|
||||
const markdown = blocksToMarkdown(docData.blocks);
|
||||
const markdown = cleanMarkdown(blocksToMarkdown(docData.blocks));
|
||||
const output = `# ${pageTitle}\n\n> Source: ${TARGET_URL}\n\n${markdown}`;
|
||||
|
||||
// 输出文件名：优先用参数，其次用标题，最后回退 output.md
|
||||
|
||||
Reference in New Issue
Block a user