"""Fill files missing from a deploy using the linked repository at that deploy's exact commit. A repository file is accepted only if its bytes have the SHA1 Netlify recorded for the deployed file, so the result is byte identical to what was deployed. The deploy ZIP is then rebuilt to include it.""" import glob import json import os import subprocess import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import importlib rd = importlib.import_module("03_recover_deploys") from nlr_common import ROOT, checkpoint_update, log, sha1, sha256_file, write_bytes_atomic, write_json def main(): for man_path in sorted(glob.glob(os.path.join(ROOT, "sites", "*", "manifests", "deploy_*_recovery.json"))): man = json.load(open(man_path)) commit = man["deploy"].get("commit_ref") if not man["missing_count"] or not commit: continue site_dir = os.path.dirname(os.path.dirname(man_path)) mirrors = glob.glob(os.path.join(site_dir, "source", "*.git")) if not mirrors: continue bs = json.load(open(os.path.join(site_dir, "metadata", "site.json")))["build_settings"] prefixes = [] for p in [bs.get("dir"), os.path.join(bs.get("base") or "", bs.get("dir") or ""), ""]: p = (p or "").strip("/").removeprefix("./") if p not in prefixes: prefixes.append(p) sr = rd.SiteRecovery(man["site_id"], {"dir": site_dir}) tree = os.path.join(site_dir, "originals", f"deploy_{man['deploy']['id']}") fixed = 0 for e in man["files"]: if not e.get("error"): continue for pre in prefixes: cand = (pre + "/" if pre else "") + e["remote_path"].lstrip("/") r = subprocess.run(["git", "show", f"{commit}:{cand}"], cwd=mirrors[0], capture_output=True) if r.returncode == 0 and sha1(r.stdout) == e["sha1_manifest"]: write_bytes_atomic(sr.cas_path(e["sha1_manifest"]), r.stdout) dst = os.path.join(tree, e["local_path"]) os.makedirs(os.path.dirname(dst), exist_ok=True) if not os.path.exists(dst): os.link(sr.cas_path(e["sha1_manifest"]), dst) e["previous_error"] = e.pop("error") e["method"] = "git_source_at_deploy_commit_sha1_verified" e["source_path_in_repo"] = cand e["size_local"] = os.path.getsize(dst) e["sha256"] = sha256_file(dst) fixed += 1 break if not fixed: log(f"backfill {man['site_name']} {man['deploy']['id']}: nothing matched at {commit[:10]}") continue man["missing_count"] = sum(1 for e in man["files"] if e.get("error")) man["file_count_recovered"] = man["file_count_manifest"] - man["missing_count"] man["bytes_recovered"] = sum(e.get("size_local") or 0 for e in man["files"]) man["status"] = "COMPLETE" if man["missing_count"] == 0 else "PARTIAL" sr.name = man["site_name"] man["zip"] = sr.make_zip(man["deploy"], tree, man["files"]) write_json(man_path, man) res = {k: man[k] for k in ("status", "file_count_manifest", "file_count_recovered", "missing_count", "bytes_recovered", "zip")} checkpoint_update(lambda c: c["deploys"].setdefault(man["deploy"]["id"], {}).update(res)) log(f"backfill {man['site_name']} {man['deploy']['id']}: +{fixed} files from {commit[:10]}, now {man['status']} " f"{man['file_count_recovered']}/{man['file_count_manifest']}") if __name__ == "__main__": main()