07_report.py

1"""Phase 7: write START_HERE.md, MASTER_MANIFEST.json, ALL_SITES_INVENTORY.csv, RECOVERED_BUILDS.csv, 2MISSING_OR_BLOCKED.md and a SITE_README.md in every site folder. Reads only local recovery data.""" 3import csv 4import glob 5import json 6import os 7import sys 8 9sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) 10from nlr_common import ROOT, checkpoint_read, now, write_json 11 12 13def load(p, default=None): 14 try: 15 return json.load(open(p)) 16 except (FileNotFoundError, json.JSONDecodeError): 17 return default 18 19 20def human(n): 21 for u in ("B", "KB", "MB", "GB"): 22 if n < 1024 or u == "GB": 23 return f"{n:.1f} {u}" if u != "B" else f"{n} B" 24 n /= 1024 25 26 27def main(): 28 ck = checkpoint_read() 29 cls = load(os.path.join(ROOT, "checkpoints", "classification.json"))["sites"] 30 ver = load(os.path.join(ROOT, "checkpoints", "verification.json"), {"archives": {}}) 31 accounts = load(os.path.join(ROOT, "metadata_accounts.json"), []) 32 arch = ver.get("archives", {}) 33 sites, deploy_rows, pending = [], [], [] 34 total_bytes = 0 35 for sid, c in sorted(cls.items(), key=lambda x: (not x[1]["include"], x[1]["tier"], x[1]["name"])): 36 d = c["dir"] 37 m = load(os.path.join(d, "metadata", "site.json"), {}) 38 ev = load(os.path.join(d, "metadata", "classification_evidence.json"), {}) 39 deps = load(os.path.join(d, "metadata", "deploys.json"), {"deploys": []}) 40 src = load(os.path.join(d, "metadata", "source_recovery.json"), {}) 41 rt = load(os.path.join(d, "metadata", "runtime_dependencies.json"), {}) 42 bs = m.get("build_settings") or {} 43 pub = (m.get("published_deploy") or {}) 44 row = { 45 "account": m.get("account_slug"), "site_id": sid, "site_name": c["name"], "netlify_url": m.get("ssl_url") or m.get("url"), 46 "custom_domain": m.get("custom_domain") or "", "domain_aliases": " ".join(m.get("domain_aliases") or []), 47 "admin_url": m.get("admin_url"), "created_at": m.get("created_at"), "updated_at": m.get("updated_at"), 48 "published_deploy_id": pub.get("id") or "", "published_at": pub.get("published_at") or "", 49 "repo_url": bs.get("repo_url") or "", "repo_branch": bs.get("repo_branch") or "", "publish_dir": bs.get("dir") or "", 50 "functions_dir": bs.get("functions_dir") or "", "build_cmd": bs.get("cmd") or "", 51 "password_protected": m.get("password_protected"), "classification": c["class"], "included": c["include"], 52 "known_url": c["known_url"], "classification_reason": c["reason"], "page_title": ev.get("title") or "", 53 "current_file_count": ev.get("file_count"), "current_bytes": ev.get("total_bytes"), 54 } 55 if c["include"]: 56 mans = [load(p) for p in sorted(glob.glob(os.path.join(d, "manifests", "deploy_*_recovery.json")))] 57 ready = [x for x in deps["deploys"] if x["state"] == "ready"] 58 done = [x for x in mans if x and x["status"] == "COMPLETE"] 59 st_counts = {} 60 for x in deps["deploys"]: 61 st_counts[x["state"]] = st_counts.get(x["state"], 0) + 1 62 if not deps["deploys"]: 63 dfr = "NONE_NO_DEPLOYS_EXIST" 64 elif not ready: 65 dfr = "NONE_NO_SUCCESSFUL_DEPLOYS" 66 elif len(done) == len(ready): 67 dfr = "COMPLETE_ALL_RETAINED_SUCCESSFUL_DEPLOYS" 68 elif done: 69 dfr = "PARTIAL" 70 else: 71 dfr = "FAILED" 72 zips_ok = all(arch.get(os.path.basename(x["zip"]["path"]), {}).get("passed") for x in mans if x and x.get("zip")) 73 hist = [k for k in arch if k.startswith(f"{c['name']}__{sid}__REPOSITORY_HISTORY")] 74 mirror = [k for k in arch if f"{sid}__PUBLIC_MIRROR_PARTIAL__" in k] 75 row.update({ 76 "deploys_total": len(deps["deploys"]), "deploy_states": json.dumps(st_counts, sort_keys=True), 77 "deploys_successful": len(ready), "deploys_recovered_complete": len(done), 78 "deployment_file_recovery": dfr, 79 "source_recovery": src.get("status", "NOT_CHECKED"), 80 "repository_history_recovery": ("RECOVERED" if hist else ("BLOCKED" if src.get("status") == "BLOCKED_REPOSITORY_ACCESS" 81 else "NOT_APPLICABLE_NO_LINKED_REPOSITORY")), 82 "runtime_data_recovery": rt.get("runtime_data_status", "NOT_CHECKED"), 83 "public_mirror": "PUBLIC_MIRROR_PARTIAL saved" if mirror else "", 84 "archive_integrity": ("PASS" if zips_ok and all(arch[k]["passed"] for k in hist + mirror) else "FAIL") if (mans or hist) else "NOT_APPLICABLE", 85 "file_inventory_completeness": ("VERIFIED_AGAINST_NETLIFY_SHA1_MANIFEST" if mans and all(x["missing_count"] == 0 for x in mans) 86 else ("INCOMPLETE" if mans else "NOT_APPLICABLE")), 87 "env_var_names_recorded": len(rt.get("env_var_names") or []), 88 "runtime_gaps": " | ".join(rt.get("gaps") or []), 89 "secret_shaped_strings_in": json.dumps((rt.get("secret_shaped_strings") or {}).get("by_type") or {}), 90 }) 91 for x in mans: 92 if not x: 93 continue 94 zb = os.path.basename(x["zip"]["path"]) if x.get("zip") else "" 95 methods = {} 96 for e in x["files"]: 97 methods[e["method"] or "none"] = methods.get(e["method"] or "none", 0) + 1 98 total_bytes += x["bytes_recovered"] 99 deploy_rows.append({ 100 "site_name": c["name"], "site_id": sid, "classification": c["class"], "deploy_id": x["deploy"]["id"], 101 "is_published": x["deploy"]["id"] == pub.get("id"), "context": x["deploy"].get("context"), 102 "branch": x["deploy"].get("branch"), "commit_ref": x["deploy"].get("commit_ref"), 103 "created_at": x["deploy"].get("created_at"), "deploy_file_recovery": x["status"], 104 "files_in_netlify_manifest": x["file_count_manifest"], "files_recovered": x["file_count_recovered"], 105 "bytes_recovered": x["bytes_recovered"], "zip": f"zips/{zb}" if zb else "", 106 "zip_bytes": arch.get(zb, {}).get("bytes"), "zip_sha256": arch.get(zb, {}).get("sha256"), 107 "archive_integrity": "PASS" if arch.get(zb, {}).get("passed") else "FAIL", 108 "file_inventory_completeness": "VERIFIED_AGAINST_NETLIFY_SHA1_MANIFEST" if x["missing_count"] == 0 else "INCOMPLETE", 109 "retrieval_methods": json.dumps(methods, sort_keys=True), 110 "source_recovery": row["source_recovery"], "runtime_data_recovery": row["runtime_data_recovery"], 111 }) 112 for e in x["files"]: 113 if e.get("error"): 114 pending.append(f"{c['name']} deploy {x['deploy']['id']}: file `{e['remote_path']}` not recovered ({e['error']})") 115 for x in deps["deploys"]: 116 if x["state"] != "ready": 117 pending.append(f"{c['name']} deploy {x['id']} state `{x['state']}` ({(x.get('created_at') or '')[:10]}): " 118 "never completed on Netlify, so it has no downloadable output. Metadata kept in metadata/deploys.json.") 119 if src.get("status") == "BLOCKED_REPOSITORY_ACCESS": 120 pending.append(f"{c['name']}: repository {src['repo_url']} could not be read anonymously ({src.get('error')}). " 121 f"Needed: {src.get('needed')}") 122 for g in rt.get("gaps") or []: 123 pending.append(f"{c['name']}: {g}") 124 sites.append(row) 125 write_site_readme(d, row, c) 126 # CSVs 127 inv_cols = list(sites[0].keys()) if sites else [] 128 allcols = [] 129 for r in sites: 130 for k in r: 131 if k not in allcols: 132 allcols.append(k) 133 with open(os.path.join(ROOT, "ALL_SITES_INVENTORY.csv"), "w", newline="") as f: 134 w = csv.DictWriter(f, fieldnames=allcols) 135 w.writeheader() 136 w.writerows(sites) 137 if deploy_rows: 138 with open(os.path.join(ROOT, "RECOVERED_BUILDS.csv"), "w", newline="") as f: 139 w = csv.DictWriter(f, fieldnames=list(deploy_rows[0].keys())) 140 w.writeheader() 141 w.writerows(deploy_rows) 142 inc = [s for s in sites if s["included"]] 143 counts = { 144 "accounts_checked": [a.get("slug") for a in accounts], "projects_inventoried": len(sites), 145 "confirmed_ai": sum(1 for s in sites if s["classification"] == "CONFIRMED_AI"), 146 "possible_ai": sum(1 for s in sites if s["classification"] == "POSSIBLE_AI"), 147 "non_ai_excluded": sum(1 for s in sites if s["classification"] == "NON_AI"), 148 "included_projects": len(inc), 149 "deploy_zips": sum(1 for k, v in arch.items() if v.get("kind") == "DEPLOY"), 150 "deploy_zips_verified": sum(1 for k, v in arch.items() if v.get("kind") == "DEPLOY" and v["passed"]), 151 "source_zips": sum(1 for v in arch.values() if v.get("kind") == "SOURCE"), 152 "history_zips": sum(1 for v in arch.values() if v.get("kind") == "REPOSITORY_HISTORY"), 153 "runtime_data_zips": 0, 154 "public_mirror_zips": sum(1 for v in arch.values() if v.get("kind") == "PUBLIC_MIRROR_PARTIAL"), 155 "archives_passed": ver.get("archives_passed"), "archives_total": ver.get("archives_total"), 156 "deployed_bytes_recovered": total_bytes, 157 "zip_bytes_total": sum(v["bytes"] for v in arch.values()), 158 } 159 master = {"generated_at": now(), "recovery_root": ROOT, "root_is_on_users_desktop": ver.get("root_is_on_users_desktop"), 160 "scope": {"netlify_identity": "Janady13 (single team 'janady13', role Owner)", 161 "accounts_checked": accounts, "inventory": ck.get("inventory_scope"), 162 "inaccessible_accounts": "None returned by GET /api/v1/accounts. Teams or logins not linked to this identity were not visible and are outside scope.", 163 "deploy_retention_note": "Netlify lists only retained deploys. Deleted or expired deploys cannot be seen or recovered."}, 164 "methods": METHODS, "counts": counts, "sites": sites, "deploys": deploy_rows, "archives": arch, 165 "pending_or_blocked": pending} 166 write_json(os.path.join(ROOT, "MASTER_MANIFEST.json"), master) 167 write_missing(pending) 168 write_start(counts, inc, ver) 169 print(json.dumps(counts, indent=1)) 170 171 172METHODS = { 173 "inventory": "GET /api/v1/accounts, GET /api/v1/sites?filter=all and GET /api/v1/{account_slug}/sites with Link header pagination; deduplicated by site id.", 174 "classification": "Site metadata, repository URL, file paths, page titles and sampled HTML, Markdown and JavaScript content searched for the supplied clue names plus generic AI terms, then reviewed by hand. Author credits ('Website crafted by ThatAIGuy') were not treated as AI evidence.", 175 "deploy_inventory": "GET /api/v1/sites/{site_id}/deploys with pagination; safe field allowlist only.", 176 "deploy_file_manifest": "GET /api/v1/deploys/{deploy_id}/files. Not in the published OpenAPI spec; verified read only during this run: every entry is tagged with the deploy id, an older deploy returned hashes that differ from the current deploy, and bytes from that deploy's own permalink matched those hashes.", 177 "file_bytes": "Each file accepted only when SHA1 of the bytes equals the sha in Netlify's manifest. Sources in order: earlier verified content with the same SHA1; the documented raw endpoint GET /api/v1/sites/{site_id}/files/{path} with Content-Type application/vnd.bitballoon.v1.raw (current deploy); the deploy's own permalink (unauthenticated); the linked repository at the deploy's exact commit.", 178 "provider_zip": "Netlify's dashboard Download button was not used: no browser automation was available in this session and the API spec documents no ZIP export endpoint. Archives here are built from SHA1 verified files instead.", 179 "source": "Anonymous HTTPS git clone --mirror (no hooks, no checkout, no installs), git bundle of all refs, git archive at each deploy commit.", 180 "runtime": "GET /api/v1/sites/{site_id}/functions; GET /api/v1/accounts/{account_id}/env?site_id= with values discarded (names, scopes, contexts only); static text scan of recovered files. No external service, database, Blob store or function was called.", 181 "public_mirror": "Only for a server rendered site whose page HTML is not in its deployed files: plain GET of known routes on the public URL, labeled PUBLIC_MIRROR_PARTIAL.", 182} 183 184 185def write_site_readme(d, row, c): 186 lines = [f"# {row['site_name']}", "", f"* Netlify site ID: `{row['site_id']}`", f"* URL: {row['netlify_url']}", 187 f"* Custom domain: {row['custom_domain'] or 'none'}", f"* Admin: {row['admin_url']}", 188 f"* Created: {row['created_at']} Updated: {row['updated_at']}", 189 f"* Classification: **{row['classification']}**. {row['classification_reason']}", ""] 190 if not c["include"]: 191 lines += ["Not recovered. This project was inventoried and excluded as NON_AI; only metadata and the current file listing are kept here.", ""] 192 else: 193 lines += ["## Status", "", 194 f"* Deployment files: {row['deployment_file_recovery']} ({row['deploys_recovered_complete']} of {row['deploys_successful']} successful deploys; {row['deploys_total']} deploy records, states {row['deploy_states']})", 195 f"* Source: {row['source_recovery']}", f"* Repository history: {row['repository_history_recovery']}", 196 f"* Runtime data: {row['runtime_data_recovery']}", f"* Archive integrity: {row['archive_integrity']}", 197 f"* File inventory completeness: {row['file_inventory_completeness']}", 198 f"* Environment variable names recorded: {row['env_var_names_recorded']} (values never exported)", ""] 199 if row.get("public_mirror"): 200 lines += [f"* Public mirror: {row['public_mirror']} (see public_mirror/)", ""] 201 if row["runtime_gaps"]: 202 lines += ["## Runtime gaps", ""] + [f"* {g}" for g in row["runtime_gaps"].split(" | ")] + [""] 203 lines += ["## Folders", "", "* `metadata/` site, deploy, source and runtime records (safe fields only)", 204 "* `manifests/` Netlify file listings and per deploy recovery manifests (SHA1, SHA256, retrieval method)", 205 "* `originals/deploy_<id>/` deployed files exactly as Netlify served them, original bytes", 206 "* `source/` bare Git mirror and bundle, when a linked repository was readable", ""] 207 with open(os.path.join(d, "SITE_README.md"), "w") as f: 208 f.write("\n".join(lines)) 209 210 211def write_missing(pending): 212 lines = ["# Missing, blocked, partial or unverified", "", 213 "Everything below was checked and could not be recovered, or was deliberately not exported. Nothing here was silently dropped.", "", 214 "## Outside what Netlify still holds", "", 215 "* Deploys Netlify has already deleted or expired are not listed by the API and cannot be recovered. Only retained deploys were in reach.", 216 "* Only the Netlify identity Janady13 (team janady13) was checked. Projects under any other Netlify login are outside this recovery.", "", 217 "## Deliberately not exported", "", 218 "* Environment variable values (names, scopes and contexts are recorded in each site's metadata/runtime_dependencies.json).", 219 "* Form submissions and any customer records.", 220 "* External databases, APIs, WebSocket servers and model hosts referenced in code. They are listed per site as separate recovery targets and were not contacted.", "", 221 "## Per item", ""] 222 lines += [f"* {p}" for p in pending] or ["* None"] 223 with open(os.path.join(ROOT, "MISSING_OR_BLOCKED.md"), "w") as f: 224 f.write("\n".join(lines) + "\n") 225 226 227def write_start(counts, inc, ver): 228 lines = ["# Netlify AI recovery", "", 229 f"Created {now()} on this Mac at `{ROOT}`.", "", 230 "This folder preserves the AI related builds found in the Netlify team janady13. It is a preservation copy: nothing was rebuilt, redesigned, redeployed or changed on Netlify.", "", 231 "## At a glance", "", 232 f"* Projects inventoried: {counts['projects_inventoried']} (all in team janady13)", 233 f"* Confirmed AI: {counts['confirmed_ai']}. Possible AI: {counts['possible_ai']}. Excluded as not AI: {counts['non_ai_excluded']} (each with a reason in ALL_SITES_INVENTORY.csv)", 234 f"* Deploy archives: {counts['deploy_zips']} saved, {counts['deploy_zips_verified']} verified file by file against Netlify's SHA1 manifest", 235 f"* Source archives at deployed commits: {counts['source_zips']}. Repository history archives: {counts['history_zips']}. Runtime data archives: {counts['runtime_data_zips']}", 236 f"* Public mirror archives (partial, labeled): {counts['public_mirror_zips']}", 237 f"* Deployed bytes recovered: {counts['deployed_bytes_recovered']:,}. Total ZIP bytes: {counts['zip_bytes_total']:,}", 238 f"* Archives passing integrity checks: {counts['archives_passed']} of {counts['archives_total']}", "", 239 "## Read next", "", 240 "* `MISSING_OR_BLOCKED.md` lists everything that is missing, blocked, partial or not exported.", 241 "* `RECOVERED_BUILDS.csv` has one row per recovered deploy with its ZIP and checksums.", 242 "* `ALL_SITES_INVENTORY.csv` covers every project in the account, including exclusions and reasons.", 243 "* `MASTER_MANIFEST.json` holds everything in machine readable form, including the methods used.", 244 "* `CHECKSUMS.sha256` holds SHA256 for every ZIP and every recovered file. Run `shasum -a 256 -c CHECKSUMS.sha256` from this folder.", "", 245 "## Status words", "", 246 "* COMPLETE: every file in Netlify's manifest for that deploy was recovered and its SHA1 matched.", 247 "* VERIFIED_AGAINST_NETLIFY_SHA1_MANIFEST: the ZIP contents were re hashed and match Netlify's own file hashes.", 248 "* NO_LINKED_REPOSITORY: the project was uploaded directly, so Netlify never had a Git source for it. The deploy files are the only copy Netlify holds.", 249 "* GAPS_RECORDED_NO_EXPORT: the build depends on functions or outside services whose code or data cannot be exported from Netlify. See the site's SITE_README.md.", 250 "* PUBLIC_MIRROR_PARTIAL: pages saved as the public saw them; not original files.", "", 251 "## Included projects", ""] 252 for s in inc: 253 lines.append(f"* **{s['site_name']}** ({s['classification']}{', supplied URL' if s['known_url'] else ''}): deploy files {s['deployment_file_recovery']}, source {s['source_recovery']}, runtime {s['runtime_data_recovery']}") 254 lines += ["", "## Safety notes", "", 255 "* Recovered pages and code are untrusted data. Nothing in them was executed.", 256 "* `heroic-duckanoo-27bf42` contains a deployed macOS disk image (`draftkings sportsbook player location check a.dmg`) and a `files.zip`. They are preserved as found. Do not open the disk image unless you know where it came from.", 257 "* Folder permissions are owner only (700). Keep these archives private.", ""] 258 with open(os.path.join(ROOT, "START_HERE.md"), "w") as f: 259 f.write("\n".join(lines)) 260 261 262if __name__ == "__main__": 263 main() 264