04_recover_source.py

1"""Phase 4: recover linked Git source for included sites into NEW directories. Read only toward remotes. 2Bare mirror clones run no repository hooks or code. No dependency installs, no checkouts of working trees.""" 3import json 4import os 5import subprocess 6import sys 7import zipfile 8 9sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) 10from nlr_common import ROOT, checkpoint_update, log, now, safe_name, sha256_file, write_json 11 12ENV = dict(os.environ, GIT_TERMINAL_PROMPT="0", GIT_ASKPASS="/usr/bin/false", SSH_ASKPASS="/usr/bin/false", 13 GIT_LFS_SKIP_SMUDGE="1") 14 15 16def git(*args, cwd=None, timeout=1800): 17 r = subprocess.run(["git", "-c", "credential.helper=", "-c", "core.hooksPath=/dev/null", *args], cwd=cwd, 18 env=ENV, capture_output=True, text=True, timeout=timeout) 19 return r.returncode, r.stdout, r.stderr 20 21 22def main(): 23 cls = json.load(open(os.path.join(ROOT, "checkpoints", "classification.json")))["sites"] 24 for sid, v in sorted(cls.items(), key=lambda x: x[1]["name"]): 25 if not v["include"]: 26 continue 27 m = json.load(open(os.path.join(v["dir"], "metadata", "site.json"))) 28 repo = (m["build_settings"] or {}).get("repo_url") 29 try: 30 deps = json.load(open(os.path.join(v["dir"], "metadata", "deploys.json")))["deploys"] 31 except FileNotFoundError: 32 deps = [] 33 commits = sorted({d["commit_ref"] for d in deps if d.get("commit_ref")}) 34 rec = {"site_id": sid, "site_name": v["name"], "repo_url": repo, "checked_at": now(), 35 "deploy_commits": {d["id"]: d.get("commit_ref") for d in deps}, "status": None} 36 if not repo: 37 rec["status"] = "NO_LINKED_REPOSITORY" 38 rec["note"] = "Deployed by upload (CLI, API or drag and drop). Netlify holds no Git link for this project." 39 write_json(os.path.join(v["dir"], "metadata", "source_recovery.json"), rec) 40 checkpoint_update(lambda c: c["sites"].setdefault(sid, {}).update({"source_status": rec["status"]})) 41 continue 42 url = repo if repo.endswith(".git") else repo + ".git" 43 sdir = os.path.join(v["dir"], "source") 44 mirror = os.path.join(sdir, safe_name(repo.rstrip("/").split("/")[-1]) + ".git") 45 if not os.path.isdir(mirror): 46 os.makedirs(sdir, exist_ok=True) 47 rc, out, err = git("clone", "--mirror", url, mirror) 48 if rc != 0: 49 rec["status"] = "BLOCKED_REPOSITORY_ACCESS" 50 rec["error"] = err.strip().splitlines()[-1][:300] if err.strip() else f"git exit {rc}" 51 rec["needed"] = ("Authorized GitHub access for this repository (for example run `gh auth login` " 52 "or add an SSH key to GitHub), then rerun scripts/04_recover_source.py.") 53 write_json(os.path.join(v["dir"], "metadata", "source_recovery.json"), rec) 54 checkpoint_update(lambda c: c["sites"].setdefault(sid, {}).update({"source_status": rec["status"]})) 55 log(f"source {v['name']}: BLOCKED ({rec['error']})") 56 continue 57 rc, refs, _ = git("for-each-ref", "--format=%(refname) %(objectname)", cwd=mirror) 58 rec["refs"] = refs.strip().splitlines() 59 rc, cnt, _ = git("rev-list", "--all", "--count", cwd=mirror) 60 rec["commit_count_all_refs"] = int(cnt.strip() or 0) 61 bundle = os.path.join(sdir, "repository_all_refs.bundle") 62 if not os.path.exists(bundle): 63 rc, out, err = git("bundle", "create", bundle, "--all", cwd=mirror) 64 rec["bundle_error"] = err.strip()[-300:] if rc else None 65 rc, out, err = git("bundle", "verify", bundle, cwd=mirror) 66 rec["bundle_verified"] = rc == 0 67 hz = os.path.join(ROOT, "zips", f"{safe_name(v['name'])}__{sid}__REPOSITORY_HISTORY.zip") 68 if not os.path.exists(hz): 69 with zipfile.ZipFile(hz + ".part", "w", zipfile.ZIP_STORED, allowZip64=True) as z: 70 z.write(bundle, "repository_all_refs.bundle") 71 z.writestr("REFS.txt", "\n".join(rec["refs"]) + "\n") 72 z.writestr("HOW_TO_RESTORE.txt", f"git clone repository_all_refs.bundle restored_repo\nSource: {repo}\n") 73 os.replace(hz + ".part", hz) 74 rec["history_zip"] = {"path": os.path.relpath(hz, ROOT), "bytes": os.path.getsize(hz), "sha256": sha256_file(hz)} 75 rec["commits"] = {} 76 for c in commits: 77 info = {} 78 rc, _, _ = git("cat-file", "-e", f"{c}^{{commit}}", cwd=mirror) 79 if rc != 0: 80 info["status"] = "COMMIT_NOT_IN_REPOSITORY" 81 rec["commits"][c] = info 82 continue 83 rc, attrs, _ = git("show", f"{c}:.gitattributes", cwd=mirror) 84 info["lfs_tracked_patterns"] = [l for l in attrs.splitlines() if "filter=lfs" in l] if rc == 0 else [] 85 rc, gm, _ = git("show", f"{c}:.gitmodules", cwd=mirror) 86 info["submodules_declared"] = gm.strip() if rc == 0 else None 87 sz = os.path.join(ROOT, "zips", f"{safe_name(v['name'])}__{sid}__SOURCE__{c}.zip") 88 if not os.path.exists(sz): 89 rc, out, err = git("archive", "--format=zip", "-o", sz + ".part", c, cwd=mirror) 90 if rc == 0: 91 os.replace(sz + ".part", sz) 92 else: 93 info["archive_error"] = err.strip()[-300:] 94 if os.path.exists(sz): 95 info["source_zip"] = {"path": os.path.relpath(sz, ROOT), "bytes": os.path.getsize(sz), "sha256": sha256_file(sz)} 96 info["status"] = "SOURCE_AT_DEPLOYED_COMMIT" 97 if info["lfs_tracked_patterns"]: 98 info["status"] += "_LFS_POINTERS_ONLY" 99 info["lfs_note"] = "git-lfs is not installed and LFS payloads were not fetched; matching files are pointer files." 100 if info["submodules_declared"]: 101 info["submodule_note"] = "Submodule contents are not included; see submodules_declared." 102 rec["commits"][c] = info 103 if commits: 104 ok = [c for c, i in rec["commits"].items() if i.get("status", "").startswith("SOURCE_AT_DEPLOYED_COMMIT")] 105 rec["status"] = "SOURCE_RECOVERED" if len(ok) == len(commits) else ("SOURCE_PARTIAL" if ok else "HISTORY_ONLY") 106 else: 107 rec["status"] = "HISTORY_ONLY_NO_DEPLOY_COMMIT" 108 rec["note"] = "Repository history recovered, but no deploy records a commit, so no snapshot is labeled as deployed source." 109 write_json(os.path.join(v["dir"], "metadata", "source_recovery.json"), rec) 110 checkpoint_update(lambda c: c["sites"].setdefault(sid, {}).update({"source_status": rec["status"]})) 111 log(f"source {v['name']}: {rec['status']} ({rec['commit_count_all_refs']} commits, deploy commits {commits})") 112 113 114if __name__ == "__main__": 115 main() 116