You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
235 lines
8.8 KiB
235 lines
8.8 KiB
2 weeks ago
|
"""Fetch Lanhu prototype pages via MCP HTTP+SSE endpoint.
|
||
|
|
|
||
|
|
The MCP tool `lanhu_get_ai_analyze_page_result` takes page **names** (not ids)
|
||
|
|
and can accept a list, so we batch several pages per call.
|
||
|
|
|
||
|
|
Usage:
|
||
|
|
python fetch.py stage1 # text_only scan of all pages (small)
|
||
|
|
python fetch.py batch --names "A,B" # analyze specific page names (mode=full, developer)
|
||
|
|
python fetch.py auto --batch 4 # walk final-scope.json, N names per call
|
||
|
|
python fetch.py auto --batch 4 --force
|
||
|
|
|
||
|
|
Cache layout:
|
||
|
|
stage1.json # STAGE-1 global text scan
|
||
|
|
pages/<safe_name>.json # analyze result per page (full mode, developer)
|
||
|
|
pages/<safe_name>.meta.json
|
||
|
|
"""
|
||
|
|
from __future__ import annotations
|
||
|
|
import argparse
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
import urllib.request
|
||
|
|
import urllib.error
|
||
|
|
|
||
|
|
MCP = "http://127.0.0.1:8000/mcp"
|
||
|
|
TID = "b013dded-642f-4899-b829-daa5c093fdf0"
|
||
|
|
PID = "e528bb49-1eea-4d76-b7e0-4055bc7c1b70"
|
||
|
|
DOC_ID = "bade4454-52aa-44db-8ba2-dec594732ecb"
|
||
|
|
DOC_URL = (f"https://lanhuapp.com/web/#/item/project/product"
|
||
|
|
f"?tid={TID}&pid={PID}&docId={DOC_ID}")
|
||
|
|
|
||
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
|
|
SCOPE = os.path.join(HERE, "final-scope.json")
|
||
|
|
PAGES_DIR = os.path.join(HERE, "pages")
|
||
|
|
STAGE1 = os.path.join(HERE, "stage1.json")
|
||
|
|
|
||
|
|
|
||
|
|
def safe_name(name: str) -> str:
|
||
|
|
s = re.sub(r"[\\/:*?\"<>|]", "_", name)
|
||
|
|
s = re.sub(r"\s+", " ", s).strip()
|
||
|
|
return s[:120]
|
||
|
|
|
||
|
|
|
||
|
|
def http_post(payload: dict, sid: str | None = None, timeout: int = 180):
|
||
|
|
data = json.dumps(payload).encode("utf-8")
|
||
|
|
headers = {
|
||
|
|
"Content-Type": "application/json",
|
||
|
|
"Accept": "application/json, text/event-stream",
|
||
|
|
}
|
||
|
|
if sid:
|
||
|
|
headers["mcp-session-id"] = sid
|
||
|
|
req = urllib.request.Request(MCP, data=data, headers=headers, method="POST")
|
||
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||
|
|
return r.read().decode("utf-8"), r.headers
|
||
|
|
|
||
|
|
|
||
|
|
def parse_sse(raw: str) -> dict:
|
||
|
|
m = re.search(r"data: (\{.*)", raw, re.S)
|
||
|
|
if not m:
|
||
|
|
raise RuntimeError(f"no SSE data frame: {raw[:200]!r}")
|
||
|
|
return json.loads(m.group(1))
|
||
|
|
|
||
|
|
|
||
|
|
def initialize() -> str:
|
||
|
|
body, hdr = http_post({
|
||
|
|
"jsonrpc": "2.0", "id": 1, "method": "initialize",
|
||
|
|
"params": {
|
||
|
|
"protocolVersion": "2024-11-05",
|
||
|
|
"capabilities": {},
|
||
|
|
"clientInfo": {"name": "verify-fetch", "version": "0.1"},
|
||
|
|
}
|
||
|
|
})
|
||
|
|
sid = hdr.get("mcp-session-id")
|
||
|
|
if not sid:
|
||
|
|
raise RuntimeError(f"no session id: {hdr}")
|
||
|
|
http_post({"jsonrpc": "2.0", "method": "notifications/initialized"}, sid=sid)
|
||
|
|
return sid
|
||
|
|
|
||
|
|
|
||
|
|
def call_analyze(sid: str, page_names, mode: str = "full",
|
||
|
|
analysis_mode: str = "developer", timeout: int = 240) -> dict:
|
||
|
|
body, _ = http_post({
|
||
|
|
"jsonrpc": "2.0", "id": 100, "method": "tools/call",
|
||
|
|
"params": {
|
||
|
|
"name": "lanhu_get_ai_analyze_page_result",
|
||
|
|
"arguments": {
|
||
|
|
"url": DOC_URL,
|
||
|
|
"page_names": page_names,
|
||
|
|
"mode": mode,
|
||
|
|
"analysis_mode": analysis_mode,
|
||
|
|
},
|
||
|
|
},
|
||
|
|
}, sid=sid, timeout=timeout)
|
||
|
|
env = parse_sse(body)
|
||
|
|
if env.get("result", {}).get("isError"):
|
||
|
|
raise RuntimeError(env["result"]["content"][0]["text"][:400])
|
||
|
|
if "error" in env:
|
||
|
|
raise RuntimeError(str(env["error"])[:400])
|
||
|
|
text = env["result"]["content"][0]["text"]
|
||
|
|
try:
|
||
|
|
return json.loads(text)
|
||
|
|
except json.JSONDecodeError:
|
||
|
|
return {"_raw_text": text}
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_stage1(args):
|
||
|
|
sid = initialize()
|
||
|
|
print(f"sid={sid}")
|
||
|
|
print("STAGE 1: text_only scan of ALL pages ...", flush=True)
|
||
|
|
t0 = time.time()
|
||
|
|
result = call_analyze(sid, "all", mode="text_only")
|
||
|
|
with open(STAGE1, "w", encoding="utf-8") as f:
|
||
|
|
json.dump(result, f, ensure_ascii=False, indent=2)
|
||
|
|
print(f" done in {time.time()-t0:.1f}s -> {STAGE1} ({os.path.getsize(STAGE1)} B)")
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_batch(args):
|
||
|
|
names = [n.strip() for n in args.names.split(",") if n.strip()]
|
||
|
|
sid = initialize()
|
||
|
|
print(f"sid={sid}")
|
||
|
|
print(f"batch analyze: {names}", flush=True)
|
||
|
|
t0 = time.time()
|
||
|
|
result = call_analyze(sid, names, mode="full", analysis_mode=args.mode)
|
||
|
|
dt = time.time() - t0
|
||
|
|
os.makedirs(PAGES_DIR, exist_ok=True)
|
||
|
|
tag = safe_name("__".join(names))
|
||
|
|
out = os.path.join(PAGES_DIR, f"{tag}.json")
|
||
|
|
with open(out, "w", encoding="utf-8") as f:
|
||
|
|
json.dump(result, f, ensure_ascii=False, indent=2)
|
||
|
|
print(f" done in {dt:.1f}s -> {out} ({os.path.getsize(out)} B)")
|
||
|
|
|
||
|
|
|
||
|
|
def pretty_name(path: str) -> str:
|
||
|
|
# scope path is like "顶层文件夹/子文件夹/A3-0-1 最近访问列表"
|
||
|
|
# the last segment is the page name in Lanhu
|
||
|
|
return path.rsplit("/", 1)[-1].strip()
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_auto(args):
|
||
|
|
scope = json.load(open(SCOPE, "r", encoding="utf-8"))
|
||
|
|
os.makedirs(PAGES_DIR, exist_ok=True)
|
||
|
|
todo = []
|
||
|
|
for p in scope["final"]:
|
||
|
|
name = pretty_name(p["path"])
|
||
|
|
cache = os.path.join(PAGES_DIR, f"{safe_name(name)}.json")
|
||
|
|
if not args.force and os.path.exists(cache):
|
||
|
|
continue
|
||
|
|
todo.append((name, p["path"]))
|
||
|
|
if args.only:
|
||
|
|
todo = [t for t in todo if args.only in t[1]]
|
||
|
|
if not todo:
|
||
|
|
print("nothing to fetch (all cached)")
|
||
|
|
return
|
||
|
|
print(f"initializing MCP session ...")
|
||
|
|
sid = initialize()
|
||
|
|
print(f" sid={sid}")
|
||
|
|
print(f"fetching {len(todo)} page(s) in batches of {args.batch}")
|
||
|
|
ok = fail = 0
|
||
|
|
for i in range(0, len(todo), args.batch):
|
||
|
|
chunk = todo[i:i+args.batch]
|
||
|
|
names = [c[0] for c in chunk]
|
||
|
|
t0 = time.time()
|
||
|
|
try:
|
||
|
|
result = call_analyze(sid, names, mode="full", analysis_mode="developer")
|
||
|
|
# split result by page if it's keyed by page name; otherwise store as one bundle
|
||
|
|
wrote = 0
|
||
|
|
if isinstance(result, dict) and any(n in result for n in names):
|
||
|
|
for n in names:
|
||
|
|
if n in result:
|
||
|
|
out = os.path.join(PAGES_DIR, f"{safe_name(n)}.json")
|
||
|
|
with open(out, "w", encoding="utf-8") as f:
|
||
|
|
json.dump(result[n], f, ensure_ascii=False, indent=2)
|
||
|
|
with open(os.path.join(PAGES_DIR, f"{safe_name(n)}.meta.json"), "w", encoding="utf-8") as f:
|
||
|
|
json.dump({"name": n, "status": "ok",
|
||
|
|
"fetched_at": time.strftime("%Y-%m-%dT%H:%M:%S")},
|
||
|
|
f, ensure_ascii=False, indent=2)
|
||
|
|
wrote += 1
|
||
|
|
else:
|
||
|
|
# unknown shape — bundle write
|
||
|
|
tag = safe_name("__".join(names))
|
||
|
|
out = os.path.join(PAGES_DIR, f"__bundle__{tag}.json")
|
||
|
|
with open(out, "w", encoding="utf-8") as f:
|
||
|
|
json.dump(result, f, ensure_ascii=False, indent=2)
|
||
|
|
for n in names:
|
||
|
|
with open(os.path.join(PAGES_DIR, f"{safe_name(n)}.meta.json"), "w", encoding="utf-8") as f:
|
||
|
|
json.dump({"name": n, "status": "bundle",
|
||
|
|
"bundle": os.path.basename(out),
|
||
|
|
"fetched_at": time.strftime("%Y-%m-%dT%H:%M:%S")},
|
||
|
|
f, ensure_ascii=False, indent=2)
|
||
|
|
wrote = len(names)
|
||
|
|
ok += wrote
|
||
|
|
dt = time.time() - t0
|
||
|
|
print(f" [{i+1:>2}-{i+len(chunk):>2}/{len(todo)}] ok {dt:5.1f}s wrote={wrote} names={names}", flush=True)
|
||
|
|
except Exception as e:
|
||
|
|
fail += len(chunk)
|
||
|
|
for n in names:
|
||
|
|
with open(os.path.join(PAGES_DIR, f"{safe_name(n)}.meta.json"), "w", encoding="utf-8") as f:
|
||
|
|
json.dump({"name": n, "status": "error", "error": str(e)[:400],
|
||
|
|
"fetched_at": time.strftime("%Y-%m-%dT%H:%M:%S")},
|
||
|
|
f, ensure_ascii=False, indent=2)
|
||
|
|
print(f" [{i+1:>2}-{i+len(chunk):>2}/{len(todo)}] FAIL names={names} :: {e}", flush=True)
|
||
|
|
time.sleep(args.sleep)
|
||
|
|
print(f"done. ok={ok} fail={fail}")
|
||
|
|
|
||
|
|
|
||
|
|
def main() -> int:
|
||
|
|
ap = argparse.ArgumentParser()
|
||
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
||
|
|
|
||
|
|
s1 = sub.add_parser("stage1")
|
||
|
|
s1.set_defaults(func=cmd_stage1)
|
||
|
|
|
||
|
|
sb = sub.add_parser("batch")
|
||
|
|
sb.add_argument("--names", required=True, help="comma-separated page names")
|
||
|
|
sb.add_argument("--mode", default="developer", choices=["developer", "tester", "explorer"])
|
||
|
|
sb.set_defaults(func=cmd_batch)
|
||
|
|
|
||
|
|
sa = sub.add_parser("auto")
|
||
|
|
sa.add_argument("--batch", type=int, default=4)
|
||
|
|
sa.add_argument("--sleep", type=float, default=1.0)
|
||
|
|
sa.add_argument("--force", action="store_true")
|
||
|
|
sa.add_argument("--only")
|
||
|
|
sa.set_defaults(func=cmd_auto)
|
||
|
|
|
||
|
|
args = ap.parse_args()
|
||
|
|
args.func(args)
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
sys.exit(main())
|
||
|
|
|