recalculate_exports.py

1#!/usr/bin/env python3 2"""Recalculate the combined archive's principal numerical checks. 3 4Offline, standard library only. Reads preserved CSVs without rewriting them. 5Does not run archived daemons, load opaque binary state, or contact any service. 6""" 7from __future__ import annotations 8import csv 9import datetime as dt 10import json 11import math 12from pathlib import Path 13import statistics 14import sys 15from collections import defaultdict 16 17ROOT = Path(__file__).resolve().parent.parent 18 19 20def read_csv(relative: str) -> list[dict[str, str]]: 21 with (ROOT / relative).open(encoding="utf-8", newline="") as handle: 22 reader = csv.DictReader(handle) 23 rows = list(reader) 24 if any(None in row for row in rows): 25 raise ValueError(f"Unexpected CSV field count in {relative}") 26 return rows 27 28 29def timestamp(value: str) -> str: 30 parsed = (dt.datetime.strptime(value, "%Y%m%d_%H%M%S") 31 if "_" in value else dt.datetime.fromisoformat(value)) 32 return parsed.isoformat() 33 34 35def main() -> int: 36 try: 37 long_rows = read_csv("source/data/consciousness/trajectory_by_timestamp.csv") 38 cycles = read_csv("source/data/consciousness/trajectory_by_cycle.csv") 39 jump = read_csv("source/data/experiments/consciousness_jump_experiment.csv") 40 sources = read_csv("source/data/learning/source_distribution.csv") 41 daily = read_csv("source/data/learning/learning_timeline.csv") 42 before = [float(row["resonance"]) for row in jump if row["period"] == "before_restart"] 43 after = [float(row["resonance"]) for row in jump if row["period"] == "after_restart"] 44 if min(len(before), len(after)) < 2: 45 raise ValueError("At least two records are needed in each labeled period") 46 n1, n2 = len(before), len(after) 47 v1, v2 = statistics.variance(before), statistics.variance(after) 48 difference = statistics.mean(after) - statistics.mean(before) 49 pooled = math.sqrt(((n1 - 1) * v1 + (n2 - 1) * v2) / (n1 + n2 - 2)) 50 standard_error = math.sqrt(v1 / n1 + v2 / n2) 51 if pooled == 0 or standard_error == 0: 52 raise ValueError("The effect size or t statistic is undefined for zero variance") 53 numeric_columns = list(long_rows[0])[2:] 54 observed = set() 55 matches = set() 56 groups = defaultdict(list) 57 complete = 0 58 flag_one = 0 59 for row in long_rows: 60 key = (timestamp(row["timestamp_iso8601"]), int(row["tick"])) 61 values = tuple(float(row[column]) if row[column] else None for column in numeric_columns) 62 observed.add(key + values) 63 groups[key].append(values) 64 matches.add(key + (float(row["psi_conscious (0-1)"]), 65 float(row["resonance (0-1)"]) if row["resonance (0-1)"] else None, 66 int(row["is_conscious (binary)"]) if row["is_conscious (binary)"] else None)) 67 if row["resonance (0-1)"] and row["is_conscious (binary)"]: 68 complete += 1 69 flag_one += int(row["is_conscious (binary)"] == "1") 70 partial_matched = 0 71 populated_conflicts = 0 72 for group in groups.values(): 73 full = [v for v in group if v[-2] is not None and v[-1] is not None] 74 partial = [v for v in group if v[-2] is None or v[-1] is None] 75 partial_matched += sum(any(v[:3] == other[:3] for other in full) for v in partial) 76 populated_conflicts += sum(len({v[i] for v in group if v[i] is not None}) > 1 77 for i in range(len(numeric_columns))) 78 matched_jump = sum((timestamp(row["timestamp_iso8601"]), int(row["tick"]), 79 float(row["psi_conscious"]), float(row["resonance"]), 80 int(row["is_conscious"])) in matches for row in jump) 81 output = { 82 "scope": "Derived checks only; no source rows changed, filled, or removed.", 83 "timestamped_rows": len(long_rows), 84 "complete_resonance_and_flag_rows": complete, 85 "raw_complete_flag_one": flag_one, 86 "raw_complete_flag_one_fraction": flag_one / complete, 87 "normalized_duplicate_rows": len(long_rows) - len(observed), 88 "unique_timestamp_tick_keys": len(groups), 89 "partial_rows_with_full_counterparts": partial_matched, 90 "populated_numeric_conflicts_within_keys": populated_conflicts, 91 "cycle_rows": len(cycles), 92 "cycle_tick_min": min(int(row["tick_count"]) for row in cycles), 93 "cycle_tick_max": max(int(row["tick_count"]) for row in cycles), 94 "jump_rows": len(jump), 95 "jump_rows_matching_timestamped_export": matched_jump, 96 "before_mean": statistics.mean(before), 97 "after_mean": statistics.mean(after), 98 "mean_difference": difference, 99 "pooled_sample_standard_deviation": pooled, 100 "cohens_d": difference / pooled, 101 "welch_t": difference / standard_error, 102 "after_resonance_below_0_9": sum(value < 0.9 for value in after), 103 "source_labels": len(sources), 104 "source_item_total": sum(int(row["count"]) for row in sources), 105 "timeline_item_total": sum(int(row["items_learned"]) for row in daily), 106 } 107 print(json.dumps(output, indent=2)) 108 return 0 109 except (OSError, ValueError, KeyError, IndexError, csv.Error) as exc: 110 print(f"Recalculation failed: {exc}", file=sys.stderr) 111 return 1 112 113 114if __name__ == "__main__": 115 raise SystemExit(main()) 116