"""Phase 5: record runtime dependencies and saved state gaps for included sites. Read only; nothing is executed. * Netlify Functions metadata (GET /sites/{id}/functions) and per deploy function and edge function flags. * Environment variable NAMES, scopes and contexts only (values are discarded in memory and never written). * Static scan of recovered deployed files and recovered source for Functions, Blobs, external APIs, WebSockets, databases, model and state files, and secret shaped strings (file names only, never values).""" import glob import json import os import re import subprocess import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from nlr_common import ROOT, api_json, checkpoint_update, log, now, write_json PATTERNS = { "netlify_functions": r"\.netlify/functions/|netlify/functions/|functions_dir|exports\.handler|export default async", "netlify_edge_functions": r"edge-functions|\.netlify/edge-functions|EdgeFunction", "netlify_blobs": r"@netlify/blobs|getStore\(|getDeployStore\(", "netlify_db_or_neon": r"@netlify/neon|@neondatabase|NETLIFY_DATABASE_URL", "websocket": r"wss?://[A-Za-z0-9.\-]+", "firebase": r"firebaseio\.com|firebase\.google|initializeApp\(", "supabase": r"supabase\.co|createClient\(", "mongodb": r"mongodb(\+srv)?://", "postgres_mysql": r"postgres(ql)?://|mysql://", "redis": r"redis(s)?://", "llm_api": r"api\.openai\.com|api\.anthropic\.com|generativelanguage\.googleapis|api\.groq\.com|openrouter\.ai|huggingface\.co|/v1/chat/completions|ollama", "localhost_backend": r"(?:localhost|127\.0\.0\.1|0\.0\.0\.0):\d{2,5}", "local_storage_state": r"localStorage\.|indexedDB\.|sessionStorage\.", } SECRET_SHAPES = { "openai_key": r"sk-(?:proj-)?[A-Za-z0-9_\-]{20,}", "anthropic_key": r"sk-ant-[A-Za-z0-9_\-]{20,}", "google_api_key": r"AIza[0-9A-Za-z_\-]{35}", "github_token": r"gh[pousr]_[A-Za-z0-9]{30,}", "aws_access_key": r"AKIA[0-9A-Z]{16}", "private_key_block": r"-----BEGIN [A-Z ]*PRIVATE KEY-----", "slack_token": r"xox[baprs]-[A-Za-z0-9\-]{10,}", "stripe_secret": r"sk_live_[A-Za-z0-9]{20,}", } STATE_EXT = (".onnx", ".gguf", ".safetensors", ".pt", ".pth", ".ckpt", ".h5", ".tflite", ".pb", ".bin", ".npy", ".npz", ".pkl", ".sqlite", ".db", ".parquet", ".arrow", ".faiss", ".index", ".dmg", ".zip") HOST_RE = re.compile(r"https?://([A-Za-z0-9.\-]+\.[A-Za-z]{2,})") IGNORE_HOSTS = re.compile(r"(fonts\.googleapis|fonts\.gstatic|w3\.org|schema\.org|unpkg|jsdelivr|cdnjs|reactjs\.org|" r"github\.com/facebook|googletagmanager|google-analytics|images\.unsplash|creativecommons|" r"mozilla\.org|tailwindcss|netlify\.app$)") def scan_tree(paths): found = {k: set() for k in PATTERNS} secrets, hosts, state_files = {}, {}, [] for p, rel in paths: if os.path.islink(p): continue if rel.lower().endswith(STATE_EXT): state_files.append({"file": rel, "bytes": os.path.getsize(p)}) if os.path.getsize(p) > 20_000_000 or rel.lower().endswith((".png", ".jpg", ".jpeg", ".gif", ".webp", ".woff", ".woff2", ".ico", ".mp4", ".mp3", ".dmg", ".zip")): continue try: t = open(p, "rb").read().decode("utf-8", "ignore") except OSError: continue for k, pat in PATTERNS.items(): if re.search(pat, t): found[k].add(rel) for k, pat in SECRET_SHAPES.items(): if re.search(pat, t): secrets.setdefault(k, set()).add(rel) for h in HOST_RE.findall(t): if not IGNORE_HOSTS.search(h): hosts.setdefault(h.lower(), set()).add(rel) return ({k: sorted(v) for k, v in found.items() if v}, {k: sorted(v) for k, v in secrets.items()}, {h: sorted(v)[:5] for h, v in sorted(hosts.items())}, state_files) def source_paths(site_dir): """Files from git archives of deployed commits, read straight from the bare mirror (no checkout).""" out = [] for mirror in glob.glob(os.path.join(site_dir, "source", "*.git")): rec = json.load(open(os.path.join(site_dir, "metadata", "source_recovery.json"))) for c in rec.get("commits", {}): tmpd = os.path.join(ROOT, "tmp", "srcscan", os.path.basename(site_dir), c) if not os.path.isdir(tmpd): os.makedirs(tmpd, exist_ok=True) a = subprocess.run(["git", "archive", c], cwd=mirror, capture_output=True) subprocess.run(["tar", "-x", "-C", tmpd], input=a.stdout, capture_output=True) for root, _, files in os.walk(tmpd): for f in files: p = os.path.join(root, f) out.append((p, f"source@{c[:10]}/" + os.path.relpath(p, tmpd))) return out def main(): cls = json.load(open(os.path.join(ROOT, "checkpoints", "classification.json")))["sites"] acct = json.load(open(os.path.join(ROOT, "metadata_accounts.json")))[0]["id"] for sid, v in sorted(cls.items(), key=lambda x: x[1]["name"]): if not v["include"]: continue d = v["dir"] rec = {"site_id": sid, "site_name": v["name"], "checked_at": now()} st, fn = api_json(f"/sites/{sid}/functions") rec["functions_api_http"] = st if isinstance(fn, list): rec["functions"] = [{k: f.get(k) for k in ("n", "name", "d", "display_name", "a", "runtime", "s", "oid", "g", "generator", "schedule") if k in f} for f in fn] elif isinstance(fn, dict): rec["functions"] = {k: (len(v_) if isinstance(v_, list) else v_) for k, v_ in fn.items() if k not in ("token", "secret")} st, env = api_json(f"/accounts/{acct}/env", {"site_id": sid}) rec["env_api_http"] = st rec["env_var_names"] = [] for e in env or []: rec["env_var_names"].append({"key": e.get("key"), "scopes": e.get("scopes"), "is_secret": e.get("is_secret"), "contexts": sorted({x.get("context") for x in (e.get("values") or []) if x.get("context")})}) env = None # values never persisted try: deps = json.load(open(os.path.join(d, "metadata", "deploys.json")))["deploys"] except FileNotFoundError: deps = [] rec["deploy_runtime_flags"] = {x["id"]: {"edge_functions_present": x.get("edge_functions_present"), "available_functions": x.get("available_functions"), "required_functions_count": x.get("required_functions_count"), "function_schedules": x.get("function_schedules"), "blobs_region": x.get("blobs_region"), "database_branch_present": x.get("database_branch_present"), "agent_runner_present": x.get("agent_runner_present"), "has_source_zip": x.get("has_source_zip")} for x in deps} paths = [] for tree in glob.glob(os.path.join(d, "originals", "deploy_*")): for root, _, files in os.walk(tree): for f in files: p = os.path.join(root, f) paths.append((p, os.path.basename(tree) + "/" + os.path.relpath(p, tree))) paths += source_paths(d) found, secrets, hosts, state_files = scan_tree(paths) rec["static_scan"] = {"files_scanned": len(paths), "references": found, "external_hosts": hosts, "model_state_or_binary_files": state_files} rec["secret_shaped_strings"] = {"note": "File names only. Values are not recorded. Keep these archives private.", "by_type": secrets} gaps = [] if rec.get("functions") or found.get("netlify_functions"): gaps.append("Serverless Functions are referenced. Netlify's documented API does not export deployed function bundles; function code is recoverable only where it appears in deployed files or linked source.") if any(x.get("edge_functions_present") for x in rec["deploy_runtime_flags"].values()) or found.get("netlify_edge_functions"): gaps.append("Edge Functions are referenced or present; edge bundles are not exportable through the documented API.") server_fns = [f for x in rec["deploy_runtime_flags"].values() for f in (x.get("available_functions") or [])] if any("nextjs" in str(f.get("g") or "") or f.get("n") == "___netlify-server-handler" for f in server_fns): gaps.append("Server rendered framework site (Next.js runtime): page HTML is produced inside the " "___netlify-server-handler function and is not part of the deployed static files. The function " "bundle is not exportable through the documented API. See public_mirror/ if present.") if found.get("netlify_blobs"): gaps.append("Code references Netlify Blobs (@netlify/blobs or getStore). Store names are not discoverable " "from deployed files alone; no Blobs export was attempted. Note: every deploy carries a default " "blobs_region, which by itself does not mean Blobs are used.") if any(x.get("has_source_zip") for x in rec["deploy_runtime_flags"].values()): gaps.append("Netlify reports has_source_zip=true for at least one deploy, but no documented API exports it.") for k in ("firebase", "supabase", "mongodb", "postgres_mysql", "redis", "llm_api", "websocket", "localhost_backend", "netlify_db_or_neon"): if found.get(k): gaps.append(f"External dependency '{k}' referenced in {len(found[k])} file(s); needs its own authorized export. Not accessed.") if found.get("local_storage_state"): gaps.append("App state is kept in the visitor's browser storage (localStorage or IndexedDB); that state lives on visitors' devices and is not on Netlify.") rec["gaps"] = gaps rec["runtime_data_status"] = "NO_HOSTED_STATE_FOUND" if not gaps else "GAPS_RECORDED_NO_EXPORT" write_json(os.path.join(d, "metadata", "runtime_dependencies.json"), rec) checkpoint_update(lambda c: c["sites"].setdefault(sid, {}).update({"runtime_status": rec["runtime_data_status"], "secret_flags": sorted(secrets)})) log(f"runtime {v['name']}: functions_http={rec['functions_api_http']} env_names={len(rec['env_var_names'])} " f"refs={sorted(found)} secrets={sorted(secrets)} state_files={len(state_files)}") if __name__ == "__main__": main()