You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
134 lines
5.4 KiB
134 lines
5.4 KiB
// 拉取 A7 全部页面（正式树 31 + backup 数据共享 3）到本地可渲染目录结构。
|
|
// 用法: node fetch-a7-pages.mjs [--only=a7-3-1]
|
|
// 产物: .scratch/a7-parity/pages/<页面url去.html>/<原相对引用路径> （file:// 可直接渲染）
|
|
// .scratch/a7-parity/oss-cache/<sign_md5> （全局去重缓存）
|
|
import fs from 'fs';
|
|
import path from 'path';
|
|
import crypto from 'crypto';
|
|
|
|
const ROOT = 'e:/code/crm-backend-matt/.scratch/a7-parity';
|
|
const LANHU = 'e:/code/crm-backend-matt/.scratch/lanhu-latest';
|
|
const OSS = 'https://lanhu-axure-file.oss-cn-beijing.aliyuncs.com/';
|
|
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)';
|
|
const CONCURRENCY = 8;
|
|
|
|
const only = process.argv.find(a => a.startsWith('--only='));
|
|
const onlyKey = only ? only.split('=')[1] : null;
|
|
|
|
const doc = JSON.parse(fs.readFileSync(path.join(LANHU, 'axure-v29.json'), 'utf-8'));
|
|
const pages = doc.pages;
|
|
|
|
// A7 页面（key = 页面名.html）
|
|
const a7 = Object.entries(pages).filter(([k]) => k.toUpperCase().startsWith('A7'));
|
|
console.log(`A7 pages in manifest: ${a7.length}`);
|
|
if (onlyKey) {
|
|
const filtered = a7.filter(([k]) => k.includes(onlyKey));
|
|
console.log(`--only=${onlyKey} matched: ${filtered.length}`);
|
|
if (filtered.length) { a7.length = 0; a7.push(...filtered); }
|
|
}
|
|
|
|
const cacheDir = path.join(ROOT, 'oss-cache');
|
|
fs.mkdirSync(cacheDir, { recursive: true });
|
|
|
|
const stats = { downloaded: 0, cacheHit: 0, failed: [], bytes: 0 };
|
|
|
|
async function fetchBuf(url, tries = 3) {
|
|
let lastErr;
|
|
for (let i = 0; i < tries; i++) {
|
|
try {
|
|
const res = await fetch(url, { headers: { 'User-Agent': UA } });
|
|
if (!res.ok) throw new Error(`HTTP ${res.status} ${url}`);
|
|
return Buffer.from(await res.arrayBuffer());
|
|
} catch (e) {
|
|
lastErr = e;
|
|
await new Promise(r => setTimeout(r, 800 * (i + 1)));
|
|
}
|
|
}
|
|
throw lastErr;
|
|
}
|
|
|
|
// 下载 sign_md5 指向的 OSS 对象，优先 oss-cache
|
|
async function fetchSigned(signMd5) {
|
|
const isUrl = signMd5.startsWith('http');
|
|
const cacheKey = isUrl
|
|
? 'url_' + crypto.createHash('md5').update(signMd5).digest('hex')
|
|
: signMd5;
|
|
const cachePath = path.join(cacheDir, cacheKey);
|
|
if (fs.existsSync(cachePath)) {
|
|
stats.cacheHit++;
|
|
return fs.readFileSync(cachePath);
|
|
}
|
|
const url = isUrl ? signMd5 : OSS + signMd5;
|
|
const buf = await fetchBuf(url);
|
|
fs.writeFileSync(cachePath, buf);
|
|
stats.downloaded++;
|
|
stats.bytes += buf.length;
|
|
return buf;
|
|
}
|
|
|
|
function safeName(relKey) {
|
|
// Windows 路径合法化（mapping key 已是解码后相对路径；替换非法字符）
|
|
return relKey.split('/').map(seg => seg.replace(/[<>:"|?*]/g, '_')).join('/');
|
|
}
|
|
|
|
// 单页：下载 mapping → html（data-src→src）→ 全部资源
|
|
async function fetchPage(key, entry) {
|
|
const pageDir = path.join(ROOT, 'pages', safeName(key.replace(/\.html$/, '')));
|
|
const mappingBuf = await fetchSigned(entry.mapping_md5);
|
|
const mapping = JSON.parse(mappingBuf.toString('utf-8'));
|
|
|
|
const jobs = []; // [relKey, signMd5]
|
|
for (const [rel, v] of Object.entries(mapping.styles || {})) jobs.push([rel, v.sign_md5]);
|
|
for (const [rel, v] of Object.entries(mapping.scripts || {})) jobs.push([rel, v.sign_md5]);
|
|
for (const [rel, v] of Object.entries(mapping.images || {})) jobs.push([rel, v.sign_md5]);
|
|
if (mapping.dataJs?.src) jobs.push([mapping.dataJs.src, mapping.dataJs.sign_md5]);
|
|
if (mapping.html?.src) jobs.push([mapping.html.src, mapping.html.sign_md5]);
|
|
|
|
let htmlBuf = null;
|
|
for (const [rel, sign] of jobs) {
|
|
const buf = await fetchSigned(sign);
|
|
const dest = path.join(pageDir, safeName(rel));
|
|
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
if (rel === mapping.html.src) {
|
|
// 主页面 HTML：蓝湖的三处改写需还原 ——
|
|
// ① <img src> 被改成 data-src 懒加载 → 还原为 src；
|
|
// ② <link> 的 href 被改成 src（浏览器忽略 link 的 src，CSS 全部不加载）→ 还原为 href；
|
|
// ③ <body style="display:none;opacity:0"> 防闪烁隐藏，解除逻辑在蓝湖 web 端注入代码 → 直接剥掉
|
|
htmlBuf = buf.toString('utf-8')
|
|
.replace(/data-src="/g, 'src="')
|
|
.replace(/ rel="stylesheet" src="/g, ' rel="stylesheet" href="')
|
|
.replace(/<body[^>]*\sstyle="[^"]*display:\s*none[^"]*"[^>]*>/, '<body>')
|
|
.replace(/srcset="/g, 'data-srcset-disabled="');
|
|
fs.writeFileSync(dest, htmlBuf, 'utf-8');
|
|
} else {
|
|
fs.writeFileSync(dest, buf);
|
|
}
|
|
}
|
|
return { key, dir: pageDir, assets: jobs.length };
|
|
}
|
|
|
|
// 并发队列
|
|
const queue = [...a7];
|
|
const results = [];
|
|
async function worker(id) {
|
|
while (queue.length) {
|
|
const [key, entry] = queue.shift();
|
|
try {
|
|
const r = await fetchPage(key, entry);
|
|
console.log(`[w${id}] OK ${key} (${r.assets} assets)`);
|
|
results.push({ key, ok: true, assets: r.assets });
|
|
} catch (e) {
|
|
console.error(`[w${id}] FAIL ${key}: ${e.message}`);
|
|
results.push({ key, ok: false, error: e.message });
|
|
stats.failed.push(key);
|
|
}
|
|
}
|
|
}
|
|
await Promise.all(Array.from({ length: CONCURRENCY }, (_, i) => worker(i)));
|
|
|
|
console.log('=== summary ===');
|
|
console.log(`pages ok: ${results.filter(r => r.ok).length}/${a7.length}`);
|
|
console.log(`oss downloads: ${stats.downloaded} (cache hits: ${stats.cacheHit}), ${(stats.bytes / 1048576).toFixed(1)} MB`);
|
|
if (stats.failed.length) console.log('failed:', stats.failed.join(', '));
|
|
fs.writeFileSync(path.join(ROOT, 'fetch-a7.log'),
|
|
JSON.stringify({ results, stats }, null, 2));
|
|
|