"""Fetch Lanhu prototype pages via MCP HTTP+SSE endpoint. The MCP tool `lanhu_get_ai_analyze_page_result` takes page **names** (not ids) and can accept a list, so we batch several pages per call. Usage: python fetch.py stage1 # text_only scan of all pages (small) python fetch.py batch --names "A,B" # analyze specific page names (mode=full, developer) python fetch.py auto --batch 4 # walk final-scope.json, N names per call python fetch.py auto --batch 4 --force Cache layout: stage1.json # STAGE-1 global text scan pages/.json # analyze result per page (full mode, developer) pages/.meta.json """ from __future__ import annotations import argparse import json import os import re import sys import time import urllib.request import urllib.error MCP = "http://127.0.0.1:8000/mcp" TID = "b013dded-642f-4899-b829-daa5c093fdf0" PID = "e528bb49-1eea-4d76-b7e0-4055bc7c1b70" DOC_ID = "bade4454-52aa-44db-8ba2-dec594732ecb" DOC_URL = (f"https://lanhuapp.com/web/#/item/project/product" f"?tid={TID}&pid={PID}&docId={DOC_ID}") HERE = os.path.dirname(os.path.abspath(__file__)) SCOPE = os.path.join(HERE, "final-scope.json") PAGES_DIR = os.path.join(HERE, "pages") STAGE1 = os.path.join(HERE, "stage1.json") def safe_name(name: str) -> str: s = re.sub(r"[\\/:*?\"<>|]", "_", name) s = re.sub(r"\s+", " ", s).strip() return s[:120] def http_post(payload: dict, sid: str | None = None, timeout: int = 180): data = json.dumps(payload).encode("utf-8") headers = { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", } if sid: headers["mcp-session-id"] = sid req = urllib.request.Request(MCP, data=data, headers=headers, method="POST") with urllib.request.urlopen(req, timeout=timeout) as r: return r.read().decode("utf-8"), r.headers def parse_sse(raw: str) -> dict: m = re.search(r"data: (\{.*)", raw, re.S) if not m: raise RuntimeError(f"no SSE data frame: {raw[:200]!r}") return json.loads(m.group(1)) def initialize() -> str: body, hdr = http_post({ "jsonrpc": "2.0", "id": 1, "method": "initialize", "params": { "protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "verify-fetch", "version": "0.1"}, } }) sid = hdr.get("mcp-session-id") if not sid: raise RuntimeError(f"no session id: {hdr}") http_post({"jsonrpc": "2.0", "method": "notifications/initialized"}, sid=sid) return sid def call_analyze(sid: str, page_names, mode: str = "full", analysis_mode: str = "developer", timeout: int = 240) -> dict: body, _ = http_post({ "jsonrpc": "2.0", "id": 100, "method": "tools/call", "params": { "name": "lanhu_get_ai_analyze_page_result", "arguments": { "url": DOC_URL, "page_names": page_names, "mode": mode, "analysis_mode": analysis_mode, }, }, }, sid=sid, timeout=timeout) env = parse_sse(body) if env.get("result", {}).get("isError"): raise RuntimeError(env["result"]["content"][0]["text"][:400]) if "error" in env: raise RuntimeError(str(env["error"])[:400]) text = env["result"]["content"][0]["text"] try: return json.loads(text) except json.JSONDecodeError: return {"_raw_text": text} def cmd_stage1(args): sid = initialize() print(f"sid={sid}") print("STAGE 1: text_only scan of ALL pages ...", flush=True) t0 = time.time() result = call_analyze(sid, "all", mode="text_only") with open(STAGE1, "w", encoding="utf-8") as f: json.dump(result, f, ensure_ascii=False, indent=2) print(f" done in {time.time()-t0:.1f}s -> {STAGE1} ({os.path.getsize(STAGE1)} B)") def cmd_batch(args): names = [n.strip() for n in args.names.split(",") if n.strip()] sid = initialize() print(f"sid={sid}") print(f"batch analyze: {names}", flush=True) t0 = time.time() result = call_analyze(sid, names, mode="full", analysis_mode=args.mode) dt = time.time() - t0 os.makedirs(PAGES_DIR, exist_ok=True) tag = safe_name("__".join(names)) out = os.path.join(PAGES_DIR, f"{tag}.json") with open(out, "w", encoding="utf-8") as f: json.dump(result, f, ensure_ascii=False, indent=2) print(f" done in {dt:.1f}s -> {out} ({os.path.getsize(out)} B)") def pretty_name(path: str) -> str: # scope path is like "顶层文件夹/子文件夹/A3-0-1 最近访问列表" # the last segment is the page name in Lanhu return path.rsplit("/", 1)[-1].strip() def cmd_auto(args): scope = json.load(open(SCOPE, "r", encoding="utf-8")) os.makedirs(PAGES_DIR, exist_ok=True) todo = [] for p in scope["final"]: name = pretty_name(p["path"]) cache = os.path.join(PAGES_DIR, f"{safe_name(name)}.json") if not args.force and os.path.exists(cache): continue todo.append((name, p["path"])) if args.only: todo = [t for t in todo if args.only in t[1]] if not todo: print("nothing to fetch (all cached)") return print(f"initializing MCP session ...") sid = initialize() print(f" sid={sid}") print(f"fetching {len(todo)} page(s) in batches of {args.batch}") ok = fail = 0 for i in range(0, len(todo), args.batch): chunk = todo[i:i+args.batch] names = [c[0] for c in chunk] t0 = time.time() try: result = call_analyze(sid, names, mode="full", analysis_mode="developer") # split result by page if it's keyed by page name; otherwise store as one bundle wrote = 0 if isinstance(result, dict) and any(n in result for n in names): for n in names: if n in result: out = os.path.join(PAGES_DIR, f"{safe_name(n)}.json") with open(out, "w", encoding="utf-8") as f: json.dump(result[n], f, ensure_ascii=False, indent=2) with open(os.path.join(PAGES_DIR, f"{safe_name(n)}.meta.json"), "w", encoding="utf-8") as f: json.dump({"name": n, "status": "ok", "fetched_at": time.strftime("%Y-%m-%dT%H:%M:%S")}, f, ensure_ascii=False, indent=2) wrote += 1 else: # unknown shape — bundle write tag = safe_name("__".join(names)) out = os.path.join(PAGES_DIR, f"__bundle__{tag}.json") with open(out, "w", encoding="utf-8") as f: json.dump(result, f, ensure_ascii=False, indent=2) for n in names: with open(os.path.join(PAGES_DIR, f"{safe_name(n)}.meta.json"), "w", encoding="utf-8") as f: json.dump({"name": n, "status": "bundle", "bundle": os.path.basename(out), "fetched_at": time.strftime("%Y-%m-%dT%H:%M:%S")}, f, ensure_ascii=False, indent=2) wrote = len(names) ok += wrote dt = time.time() - t0 print(f" [{i+1:>2}-{i+len(chunk):>2}/{len(todo)}] ok {dt:5.1f}s wrote={wrote} names={names}", flush=True) except Exception as e: fail += len(chunk) for n in names: with open(os.path.join(PAGES_DIR, f"{safe_name(n)}.meta.json"), "w", encoding="utf-8") as f: json.dump({"name": n, "status": "error", "error": str(e)[:400], "fetched_at": time.strftime("%Y-%m-%dT%H:%M:%S")}, f, ensure_ascii=False, indent=2) print(f" [{i+1:>2}-{i+len(chunk):>2}/{len(todo)}] FAIL names={names} :: {e}", flush=True) time.sleep(args.sleep) print(f"done. ok={ok} fail={fail}") def main() -> int: ap = argparse.ArgumentParser() sub = ap.add_subparsers(dest="cmd", required=True) s1 = sub.add_parser("stage1") s1.set_defaults(func=cmd_stage1) sb = sub.add_parser("batch") sb.add_argument("--names", required=True, help="comma-separated page names") sb.add_argument("--mode", default="developer", choices=["developer", "tester", "explorer"]) sb.set_defaults(func=cmd_batch) sa = sub.add_parser("auto") sa.add_argument("--batch", type=int, default=4) sa.add_argument("--sleep", type=float, default=1.0) sa.add_argument("--force", action="store_true") sa.add_argument("--only") sa.set_defaults(func=cmd_auto) args = ap.parse_args() args.func(args) return 0 if __name__ == "__main__": sys.exit(main())