crm-backend-matt/.scratch/opportunity-bugfix/fetch-all-pages.mjs

// 票 01：v29 原型 182 页全量拉取 → lanhu-pages/ 文本库
// 通路：.scratch/lanhu-latest/RESEARCH-axure-pipeline.md（Step 1 清单已有，直接 Step 3/4）
// 用法：node fetch-all-pages.mjs [only=<url关键词>]   仅拉匹配页（调试用）
import fs from 'node:fs';
import path from 'node:path';

const ROOT = 'e:/code/crm-backend-matt';
const OSS = 'https://lanhu-axure-file.oss-cn-beijing.aliyuncs.com/';
const IN_JSON = `${ROOT}/.scratch/lanhu-latest/axure-v29.json`;
const OUT_DIR = `${ROOT}/.scratch/opportunity-bugfix/lanhu-pages`;
const SLEEP_MS = 200;

const data = JSON.parse(fs.readFileSync(IN_JSON, 'utf8'));
fs.mkdirSync(OUT_DIR, { recursive: true });

// ---- sitemap 树 walk：url → { name, path } ----
const pageMeta = new Map(); // url -> {name, path}
const walk = (nodes, folder) => {
  for (const n of nodes || []) {
    // 带 url 的 Folder：自身也算页面，再递归 children（Axure sitemap 常见形态，漏了会少 44 页）
    if (n.url) pageMeta.set(n.url, { name: n.pageName, path: [...folder, n.pageName].join(' > ') });
    if (Array.isArray(n.children) && n.children.length) walk(n.children, [...folder, n.pageName]);
  }
};
walk(data.sitemap.rootNodes, []);

// sitemap 里没有、但 pages 里有的 key：兜底用 key 本身
const urls = Object.keys(data.pages);
for (const u of urls) {
  if (!pageMeta.has(u)) pageMeta.set(u, { name: u.replace(/\.html$/, ''), path: u });
}

// ---- 文件名安全化 + 冲突处理 ----
const used = new Set();
const safeName = (name) => {
  let s = name.replace(/[\\/:*?"<>|\s]+/g, '_');
  let base = s, i = 2;
  while (used.has(s.toLowerCase())) s = `${base}_${i++}`;
  used.add(s.toLowerCase());
  return s;
};
const fileOf = new Map(); // url -> out file
for (const u of urls) fileOf.set(u, path.join(OUT_DIR, safeName(pageMeta.get(u).name) + '.md'));

// ---- 文本提取（继承 extract-html-text.mjs 已验证算法 + 批注区增强）----
const decode = (s) => s
  .replace(/&#x([0-9A-Fa-f]+);/g, (_, h) => String.fromCodePoint(parseInt(h, 16)))
  .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10)))
  .replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/&lt;/g, '<')
  .replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#39;/g, "'");

const cell = (s) => s.replace(/\|/g, '\\|').replace(/\r?\n/g, ' ').trim();

function extractControls(html) {
  const re = /<!--\s*(.*?)\s*-->\s*<div id="(u\d+)"[^>]*>/g;
  const items = [];
  let m;
  while ((m = re.exec(html)) !== null) {
    const [, comment, id] = m;
    const typeMatch = comment.match(/^(.*?)\s*[（(](.+?)[）)]$/);
    const name = typeMatch ? typeMatch[1] : comment;
    const type = typeMatch ? typeMatch[2] : '其他';
    // 窗口 20000：id 精确锚定无跨控件误配风险；3000 会截断超长批注 cell（闭合 </div> 落在窗外整段丢失，A1X-1-1 实证）
    const after = html.slice(m.index, m.index + 20000);
    const textRe = new RegExp(`id="${id}_text"[^>]*>[\\s\\S]*?</div>`);
    const tm = after.match(textRe);
    let text = '';
    if (tm) {
      text = (tm[0].match(/<p[^>]*>\s*<span[^>]*>([\s\S]*?)<\/span>\s*<\/p>/g) || [])
        .map(p => decode(
          p.replace(/<br\s*\/?>/gi, ' ')
           .replace(/<\/?p[^>]*>/g, '').replace(/<\/?span[^>]*>/g, '')
           .replace(/<[^>]+>/g, '')
        ))
        .filter(t => t.length > 0)
        .join(' | ');
    }
    items.push({ id, name: decode(name).trim(), type: decode(type).trim(), text: text.trim() });
  }
  return items;
}

// ---- 下载（重试一次）----
async function fetchText(url) {
  for (let i = 0; i < 2; i++) {
    try {
      const res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0' } });
      if (res.ok) return await res.text();
      console.log(`  HTTP ${res.status} (try ${i + 1}) ${url.slice(-30)}`);
    } catch (e) {
      console.log(`  ERR ${e.message} (try ${i + 1})`);
    }
    if (i === 0) await new Promise(r => setTimeout(r, 800));
  }
  return null;
}

// ---- 主流程 ----
const only = process.argv.find(a => a.startsWith('only='))?.slice(5);
const todo = urls.filter(u => !only || u.includes(only));
console.log(`pages total: ${urls.length}, todo: ${todo.length}${only ? ` (filter: ${only})` : ''}`);

let ok = 0, skip = 0, fail = 0;
const failed = [], emptyPages = [];
for (const u of todo) {
  const meta = pageMeta.get(u);
  const file = fileOf.get(u);
  if (fs.existsSync(file) && fs.statSync(file).size > 100) { skip++; continue; }
  const entry = data.pages[u];
  const html = await fetchText(OSS + entry.html.sign_md5);
  if (!html) { fail++; failed.push(u); continue; }
  const items = extractControls(html);
  const withText = items.filter(i => i.text);
  if (withText.length === 0) emptyPages.push(u);
  const lines = [
    `# ${meta.name}`,
    '',
    `- sitemap: ${meta.path}`,
    `- 源文件: ${u} (v29)`,
    `- 控件: ${items.length} / 有文本: ${withText.length}`,
    '',
    '## 控件文本',
    '',
    '| id | 类型 | 名称 | 文本 |',
    '|---|---|---|---|',
    ...withText.map(i => `| ${i.id} | ${cell(i.type)} | ${cell(i.name)} | ${cell(i.text)} |`),
    '',
  ];
  fs.writeFileSync(file, lines.join('\n'), 'utf8');
  ok++;
  process.stdout.write(`[${ok + skip + fail}/${todo.length}] ${meta.name} (${withText.length} texts)\n`);
  await new Promise(r => setTimeout(r, SLEEP_MS));
}

console.log(`\n=== done: ok=${ok} skip=${skip} fail=${fail} empty=${emptyPages.length}`);
if (failed.length) console.log('failed pages:\n' + failed.join('\n'));
if (emptyPages.length) console.log('empty pages (0 text): ' + emptyPages.join(', '));