#!/usr/bin/env python3
"""Rebuild the three raw-run archives left out of the records of "Cancer Research: The Whole Arc", and check them.

Version 1, October 16, 2026. Uses only public files: records from the page's Records section and the Spiral
Staircase page's nudge_test.py. Linux, Python 3.11 and NumPy (tested with Python 3.11.14 and NumPy 2.4.3). Keep this
file in the folder that holds coherence-growth-2026-10-11/, cause-test-2026-10-12/ and nudge-test-2026-10-10/.

    python3 -B rebuild_archives.py inputs              check every input file against its SHA-256
    python3 -B rebuild_archives.py fresh               sim_fresh4000.npz: Test 17's test, starts 7 to 12
    python3 -B rebuild_archives.py replication         sim_replication4000.npz: Test 17's replication, starts 13 to 18
    python3 -B rebuild_archives.py cause [PIECES]      sim_cause4000.npz: the Cause Test, starts 19 to 60
    python3 -B rebuild_archives.py verify ARCHIVE      compare an archive with the original, array by array

"cause" runs pieces of 7 base runs or 14 moments until all 462 tasks are done, then writes the archive. With PIECES,
it stops after that many pieces; running it again carries on from its run log.
"""
import hashlib
import json
import shutil
import sys
from pathlib import Path

TOP = Path(__file__).resolve().parent
CG, CAUSE = TOP / "coherence-growth-2026-10-11", TOP / "cause-test-2026-10-12"
sys.dont_write_bytecode = True                                   # leave the record folders as they are

# Every input file and its SHA-256: the records list's, and the Staircase page's for nudge_test.py.
INPUTS = {
    "nudge-test-2026-10-10/nudge_test.py":
        "b06730e7772be1f719cadae7924cdca3c2e0ebb89efb17104969363431f782e7",
    "coherence-growth-2026-10-11/coherence_growth.py":
        "7738b55b8a3757f7fb186e11611251de0f5eddc8a6a746f23c9d51784e3b5b3a",
    "coherence-growth-2026-10-11/earlier_growth_check.py":
        "a5b5eab338d2e473097b707c562998c3b6e9eb7ceb450e5e31e4c5d11db032c8",
    "coherence-growth-2026-10-11/replication.py":
        "99381ca8c95a4e17b3c17cf01acc580c6203f15242b4e1c9443b50bf78253c0e",
    "coherence-growth-2026-10-11/replication_lock.json":
        "b36bd2c03ddecd9e8dc2237f06041311b6704d93a48257e2d75630cd665384c2",
    "coherence-growth-2026-10-11/replication_prereg.md":
        "d261cd665a70bb7fb8323abd9249f40c774653fafc5d1addc9ce245de34d1a70",
    "coherence-growth-2026-10-11/coherence_growth.json":
        "27db2ce0a1346b5f1de198145a2793f7f531ffc6849fa53cc69d74462d05e858",
    "coherence-growth-2026-10-11/coherence_growth.txt":
        "f3acff1e85e580d8fb20dfba210efaf8ded8011ec87df556c18653ca27992231",
    "coherence-growth-2026-10-11/check.json":
        "95e76112e2d32769e6fe5430de3d0b7d2e3a609d2ec8bd6161d74c95f21558a7",
    "coherence-growth-2026-10-11/earlier_growth_check.json":
        "506ef2b72e01afcb837d88717e91d315ea9e9aa5e06091ad2372f47f3f4a7a06",
    "coherence-growth-2026-10-11/earlier_growth_check.txt":
        "92fd74db1a8e3f655ab21ae018952aa2d6a4bba2f2c5c99ca644b0a4f3cf1e29",
    "cause-test-2026-10-12/cause_test.py":
        "e182e564ca92c7401fc958da9698a93fdece8663eacba485e1c2f41cf69e8f54",
    "cause-test-2026-10-12/cause_rule.py":
        "5fdd1c90df371675d1113a5eca799abdc2c639c00adbeaeb0339c05b02f487ca",
    "cause-test-2026-10-12/design_numbers.py":
        "103241b2fb361649f3e53cea900fedb1ee475c11200e8e8e8feeb2c2af4fecd3",
    "cause-test-2026-10-12/design_numbers.json":
        "39bc39762c551631327d5120b69a6b5daf27f7a6c5cdb5f06fe59a22b281af28",
    "cause-test-2026-10-12/design_numbers.txt":
        "76d68062720f21e7c5f62ae14092e6c135ae207b036564ccb30e1a9770edca39",
    "cause-test-2026-10-12/cause_moments.csv":
        "1a1d72113ab035b14b9e80e413e7030291915d8d1a365ad1f658f95c277709a8",
    "cause-test-2026-10-12/cause_targets.json":
        "225058b7c11159c412ced9b2f1d8c7ea48d9e48d2805b913be4bdc2278acbe19",
    "cause-test-2026-10-12/cause_targets.txt":
        "5e3f8a8dba33a11bad8a0693e65803cfbfdb3bd83bfbaa7e868d41b02882d30c",
    "cause-test-2026-10-12/cause_test_prereg.md":
        "1c44034e64f4ec74d5afba7080cf1df1e96262680583f785447b153f3ce61112",
    "cause-test-2026-10-12/cause_lock.json":
        "32d72d1e2a4b67a3642e3cdce0207b43e0af670683609c8d84fb1b00366da527",
}

# The inputs each archive needs. The lock files and the files they cover are checked too, where public.
NEEDS = dict(
    fresh=(
        "nudge-test-2026-10-10/nudge_test.py",
        "coherence-growth-2026-10-11/coherence_growth.py",
    ),
    replication=(
        "nudge-test-2026-10-10/nudge_test.py",
        "coherence-growth-2026-10-11/coherence_growth.py",
        "coherence-growth-2026-10-11/earlier_growth_check.py",
        "coherence-growth-2026-10-11/replication.py",
        "coherence-growth-2026-10-11/replication_lock.json",
        "coherence-growth-2026-10-11/replication_prereg.md",
        "coherence-growth-2026-10-11/coherence_growth.json",
        "coherence-growth-2026-10-11/coherence_growth.txt",
        "coherence-growth-2026-10-11/check.json",
        "coherence-growth-2026-10-11/earlier_growth_check.json",
        "coherence-growth-2026-10-11/earlier_growth_check.txt",
    ),
    cause=(
        "nudge-test-2026-10-10/nudge_test.py",
        "coherence-growth-2026-10-11/coherence_growth.py",
        "coherence-growth-2026-10-11/earlier_growth_check.py",
        "cause-test-2026-10-12/cause_test.py",
        "cause-test-2026-10-12/cause_rule.py",
        "cause-test-2026-10-12/design_numbers.py",
        "cause-test-2026-10-12/design_numbers.json",
        "cause-test-2026-10-12/design_numbers.txt",
        "cause-test-2026-10-12/cause_moments.csv",
        "cause-test-2026-10-12/cause_targets.json",
        "cause-test-2026-10-12/cause_targets.txt",
        "cause-test-2026-10-12/cause_test_prereg.md",
        "cause-test-2026-10-12/cause_lock.json",
    ),
)

# Files the scripts' locks cover that aren't public. The lock check skips them and checks every other file it covers.
NOT_PUBLIC = dict(
    replication=("test16_results", "fresh_runs", "earlier_runs", "test_figure", "check_figure"),
    cause=("test16_results", "earlier_runs", "fresh_runs", "replication_runs"))

# The original archives: SHA-256 and size of each file, and each array's type, shape and the SHA-256 of its bytes in
# C order. The wall-clock arrays (seconds each run took) differ in every run, so they are listed but never compared.
EXPECTED = {
    "sim_fresh4000.npz": dict(
        sha256="a008e1c61a9ef4215a6b40b867d1b577e947bd0dfa296ab2a6676c204bb11ca0", bytes=592016,
        wall_clock=('seconds',),
        arrays={
            "starts": ("<i8", [6], "3d8291de915fe2a5ed9d08d415b79b0c537259f7dea4d47c73a28f4fcbda87c5"),
            "n": ("<i8", [], "6fff9aef0d88c32b1e16a50833d2698afd0363ad3495673f8f92adfc4caad4f7"),
            "k": ("<f8", [], "9d9191449bcffb888c9d5981e1dea8034f8bde1ed9cdd6e011bfb69d28600120"),
            "dt": ("<f8", [], "743d526cd0e45eed73e8f0b2308a404561eada75bd4f213c1af94057eef75450"),
            "settle": ("<f8", [], "f39bbd95f59e971080a9f011457d4ca9e0770b8d906c5bf909e3e4998adceb0b"),
            "run": ("<f8", [], "5b78c1203e68492ba4690f7aa928e7f4b253dd3382f0433b9a6f243fbc458817"),
            "every": ("<f8", [], "a7ca1349517e14c0690dccc98bf9a515019b2018116a3191f6c7d53831c0432a"),
            "gap": ("<f8", [], "71b06a26c199718d33fb623a3848b9c2b9733e369271a5451258bb7c081ff81c"),
            "seed": ("<i8", [], "be751d82e9a233c3b0480dad058b627a35a8a4118c369f39a7f3c39cd5d5b887"),
            "z": ("<c8", [6, 2, 6001], "5eaf3cbe2f70d975329fd066676ee6614fac739536876a5bef8fa2d9625e7c21"),
            "logs": ("<f8", [6, 1200], "a255c11bf0fb64070eda4f4dbf9c902e44ff41421d6e83ee628bf551fc7e5144"),
            "trace": ("<f4", [6, 2, 61], "bcd34dfaafbae44a5c99e43a5ffa6a6272e586d41fb8656fb46d2244e319b324"),
        }),
    "sim_replication4000.npz": dict(
        sha256="2ebd7e66313d8ff2710f5aebaa2c8cb5fdc1ddc16cb3d767dd0fe75d27c61e0c", bytes=592435,
        wall_clock=('seconds',),
        arrays={
            "starts": ("<i8", [6], "71cebbaaae9bec207a67c4f0c9b622fc41f4a3b5e00a5b64e8c4d7cdb475dd0d"),
            "n": ("<i8", [], "6fff9aef0d88c32b1e16a50833d2698afd0363ad3495673f8f92adfc4caad4f7"),
            "k": ("<f8", [], "9d9191449bcffb888c9d5981e1dea8034f8bde1ed9cdd6e011bfb69d28600120"),
            "dt": ("<f8", [], "743d526cd0e45eed73e8f0b2308a404561eada75bd4f213c1af94057eef75450"),
            "settle": ("<f8", [], "f39bbd95f59e971080a9f011457d4ca9e0770b8d906c5bf909e3e4998adceb0b"),
            "run": ("<f8", [], "5b78c1203e68492ba4690f7aa928e7f4b253dd3382f0433b9a6f243fbc458817"),
            "every": ("<f8", [], "a7ca1349517e14c0690dccc98bf9a515019b2018116a3191f6c7d53831c0432a"),
            "gap": ("<f8", [], "71b06a26c199718d33fb623a3848b9c2b9733e369271a5451258bb7c081ff81c"),
            "seed": ("<i8", [], "be751d82e9a233c3b0480dad058b627a35a8a4118c369f39a7f3c39cd5d5b887"),
            "z": ("<c8", [6, 2, 6001], "51a8e4f67553a8036d7d4e3a6dcfa12543c41542b6cdf30e8ef923ed8311d2b2"),
            "logs": ("<f8", [6, 1200], "ebb2b50e9e75febd9cedb7f9f6925a634cd6b18e3a2be114484c1756b129f078"),
            "trace": ("<f4", [6, 2, 61], "9a2a1d76bab766f8980e71042e40096d86bf2a26d270d61d82dc817415b00f66"),
        }),
    "sim_cause4000.npz": dict(
        sha256="541a2f8009d6c97c805faf12a1ce8aef4a34dfebb692a8ad1ca0cbd650491218", bytes=25074738,
        wall_clock=('base_seconds', 'moment_seconds'),
        arrays={
            "starts": ("<i8", [42], "2e9669ef773d7dd139efb0a5e049ae43ec929a7e5c3e9005f52de583eeef76b3"),
            "T": ("<i8", [42, 10], "1c0999c607fe4c2b3ced3f78c10246038d3599de6739b233c8cbc644b0d80a69"),
            "z": ("<c8", [42, 2, 6001], "03312bfe1e4b2772080e131507b67b9216bfa9d81125f3303a5273e3023e8bc8"),
            "logs": ("<f8", [42, 1200], "3bfd4ee6e1a97c361e318cf1e8cc0459f28b071092d80bbb8b7db9d4737626e5"),
            "trace": ("<f4", [42, 2, 61], "01ae963acbf15dae72f27f0b566003b755362a21eeae8f1e17342704133a668f"),
            "n": ("<i8", [], "6fff9aef0d88c32b1e16a50833d2698afd0363ad3495673f8f92adfc4caad4f7"),
            "k": ("<f8", [], "9d9191449bcffb888c9d5981e1dea8034f8bde1ed9cdd6e011bfb69d28600120"),
            "dt": ("<f8", [], "743d526cd0e45eed73e8f0b2308a404561eada75bd4f213c1af94057eef75450"),
            "settle": ("<f8", [], "f39bbd95f59e971080a9f011457d4ca9e0770b8d906c5bf909e3e4998adceb0b"),
            "run": ("<f8", [], "5b78c1203e68492ba4690f7aa928e7f4b253dd3382f0433b9a6f243fbc458817"),
            "every": ("<f8", [], "a7ca1349517e14c0690dccc98bf9a515019b2018116a3191f6c7d53831c0432a"),
            "gap": ("<f8", [], "71b06a26c199718d33fb623a3848b9c2b9733e369271a5451258bb7c081ff81c"),
            "seed": ("<i8", [], "be751d82e9a233c3b0480dad058b627a35a8a4118c369f39a7f3c39cd5d5b887"),
            "draws": ("<i8", [], "cc9144be0886c1b806bc7f6de058f9ff1572bd52e238bf80b90c857a95cf2eb0"),
            "after": ("<f8", [], "35ba779625877df3fea7b5cc8f8c8b7842a6fb8ebb44f00d39aaa18c342c4575"),
            "R0": ("<f8", [42, 10, 2], "c17571028842b313bc5fb4455508b450f046836d3d667e307d4702208521c1a4"),
            "psi0": ("<f8", [42, 10, 2], "625017c08b35598191a43e207c0d5d2213ad0a1f8a1d39cee47a7169f2304e26"),
            "z2_0": ("<f8", [42, 10, 2], "9896b29f32ab84401273be0b586d0cf5a602362916dda8fa41420c145df1021b"),
            "z3_0": ("<f8", [42, 10, 2], "44faf01b976803c28e4f31d011e7f4e346ebae3bc4f0e39f644e6fc3ce525bd0"),
            "z4_0": ("<f8", [42, 10, 2], "48df8f1ab108b850a7dfa6a298d1aacd1bed597cab58b635a2b20984388ddbc3"),
            "C0": ("<f8", [42, 10], "0a59e038ce623bad4bbcc547fd9e64b5fbb32849cdaf9b0e2477f36904ad652d"),
            "ab0": ("<f8", [42, 10], "93f02b7b667dd8ab4e2e2f6806169f4a0dd5a295b38bb9c83f7688066c97ff44"),
            "gap0": ("<f8", [42, 10], "55e7592ded504893e86b112415f5cc2f05f70059fe2b976ae2c1213cbc794ccc"),
            "R1": ("<f8", [42, 10, 6, 2], "9d5ff7df0394da871e78e9aaf6f8634346482f2f876f95bb9a67e64bda7451ff"),
            "psi1": ("<f8", [42, 10, 6, 2], "ad2485b261fb83308b16b19014aa9da68effc8912fc418a0eab4584a8bf61662"),
            "z2_1": ("<f8", [42, 10, 6, 2], "882b678188c792bc115427b54974a4815861639d36795c466d7fe5e6020022ec"),
            "z3_1": ("<f8", [42, 10, 6, 2], "d075c130f96233db4bc44287d8f84e08bb4e70c7d035aece90cdb5b68853b088"),
            "z4_1": ("<f8", [42, 10, 6, 2], "991278d09691b9bb0287ee90f58328250aaed28d22742308ec00411b222f0248"),
            "ratio": ("<f8", [42, 10, 6, 2], "9b354ccda764a5fa027c2bf0a71a40fafc70f6e0d2637e27cad29d3ef7de4be0"),
            "inband": ("|b1", [42, 10, 6, 2], "0347498b6c916c5a0117c1b56b62665234e8b4032d1fd8c075beb25152c29c0c"),
            "C1": ("<f8", [42, 10, 6], "5b0da053e8a59c98cf81c4c7030a0dfd539ab35d1e620b377f4790ab82a0251f"),
            "ab1": ("<f8", [42, 10, 6], "e4b3fd1e799c7257d8e5f2909ae728eb23ead98ac284613593acb47f0e818418"),
            "sumsq": ("<f8", [42, 10, 6], "d9126bfbf948e57ddc173b0acb54180865937dfa28c3bd4c212145e989887955"),
            "moved": ("<f8", [42, 10, 6], "70dffd076f68b13a3a7366946081f6beccc94dd15ed48f8b13d73c565c483a01"),
            "largest": ("<f8", [42, 10, 6], "7e42b3c78569ab18bca35eb9a9fa64f97c843fc23ba1a9efac18180ab332fdf6"),
            "rms": ("<f8", [42, 10, 6], "ed40de0bb1cac7be5a310ab7ca5b7931e63675c858709040ddff19c88565c920"),
            "gap1": ("<f8", [42, 10, 6], "013e5a6ee1c2ba10481e0124706453fde70c3236b3bfca09b4ec629827622e6e"),
            "gaprel": ("<f8", [42, 10, 6], "5c48e265a8c95c7cf691a2bacebf509c0df8fd676af60b0b3cd482264da7891c"),
            "amp": ("<f8", [42, 10, 2, 2], "1c386ad12a0cdeb58a28cc47793a9f438c2b22691d7583734f0eda3754674ce4"),
            "status": ("<i8", [42, 10, 2, 2], "9e8e4d1848b6146e3a4ad37c38a8de9ea2dcc5cfec426df9bc9136268477c3dc"),
            "built": ("|b1", [42, 10, 2], "5aa11e05ea0095307a08e76d2f5725989b38e45625156f63ee6b0d28cae6d3bf"),
            "same": ("|b1", [42, 10, 2], "5aa11e05ea0095307a08e76d2f5725989b38e45625156f63ee6b0d28cae6d3bf"),
            "match": ("<f8", [42, 10, 2], "53192a12b178687771e5a607727c1a7102e095fc05515a6e827a4eeff529a280"),
            "carried": ("|b1", [42, 10], "6c2ec9b1535cbaaa8c402cfda1dd38dd234214cfd7a55405efed4255fa04f0f2"),
            "code_error": ("|b1", [42, 10], "de1da8f3791deb5ea5a11c9a67179b1e240aa14979524a3cc28add0e3612c0fe"),
            "reason": ("<U1", [42, 10], "065cc6b2b996ca729f6aa0208e13ac4b494dd0d74a4c4df6053d08b0c11da865"),
            "branch_z": ("<c8", [42, 10, 6, 2, 501], "c04e97b5b565dfc1ae8a7d856029851ad9b1db4084b1fae0fc10f78dcffae7e1"),
            "branch_logs": ("<f8", [42, 10, 6, 100], "175da25266a44b6ad99438f7c2eb506503d084b406d1b77bce3b170f1a6e7c1b"),
        }),
}


def sha256(path):
    return hashlib.sha256(Path(path).read_bytes()).hexdigest()


def inputs(names=None):
    """Each input file against its SHA-256; stops if any is missing or different."""
    problems = 0
    for name in names or INPUTS:
        path = TOP / name
        state = "MISSING" if not path.exists() else "OK" if sha256(path) == INPUTS[name] else "DIFFERENT"
        problems += state != "OK"
        print(f"  {state:10s} {name}")
    if problems:
        sys.exit(f"{problems} input file(s) missing or different. Stopping.")


def lock_checked(module, skip):
    """The script's own lock, after checking the settings and every file it covers except those in `skip`."""
    lock = json.loads(module.LOCK.read_text())
    for name, path in module.LOCKED.items():
        if name not in skip and sha256(path) != lock["sha256"][name]:
            sys.exit(f"{Path(path).name} differs from the lock ({name}). Stopping.")
    if lock["settings"] != json.loads(json.dumps(module.SETTINGS)):
        sys.exit("The settings differ from the lock. Stopping.")
    return lock


def unbroken(nt):
    """Every start in one unbroken piece: no time limit, and no progress left over from an earlier, paused run."""
    folder = nt.LYAP_PARTS
    if folder.exists() and any(folder.iterdir()):
        sys.exit(f"{folder.relative_to(TOP)} holds progress files from an earlier run. Remove that folder, so that every"
                 " start runs unbroken, and run the same command again.")
    nt.LYAP_BUDGET = float("inf")


def fresh():
    inputs(NEEDS["fresh"])
    sys.path.insert(0, str(CG))
    import coherence_growth as cg
    unbroken(cg.nt)
    cg.run()


def replication():
    inputs(NEEDS["replication"])
    sys.path.insert(0, str(CG))
    import replication as r
    lock = lock_checked(r, NOT_PUBLIC["replication"])
    r.unchanged = lambda: lock
    r.RUNLOG = CG / "rebuild_replication_runlog.json"            # the record's own run log stays as it is
    unbroken(r.nt)
    r.run()


def cause(pieces=None):
    inputs(NEEDS["cause"])
    sys.path.insert(0, str(CAUSE))
    import cause_test as c
    lock = lock_checked(c, NOT_PUBLIC["cause"])
    c.unchanged = lambda: lock
    c.RUNLOG = CAUSE / "rebuild_cause_runlog.json"               # the record's own run log stays as it is
    if c.SAVED.exists():
        print(f"  nothing left to run: {c.SAVED.name} is saved")
        return
    todo, ran = c.tasks(), 0

    def done():
        return json.loads(c.RUNLOG.read_text())["parts"] if c.RUNLOG.exists() else {}

    while len(done()) < len(todo):
        if pieces is not None and ran == pieces:
            print(f"  stopped after {ran} piece(s); run the same command again to carry on")
            return
        before = len(done())
        c.run()
        ran += 1
        if len(done()) == before:
            sys.exit("A whole piece finished no task. Stopping.")
    parts = done()
    for task in todo:                                            # as the Cause Test's report checks before assembling
        if sha256(c.PARTS / f"{task['name']}.npz") != parts[task["name"]]["sha256"]:
            sys.exit(f"{task['name']}.npz differs from the run log. Stopping.")
    c.assemble(c.PARTS, [s + 1 for s in c.NEW], c.dn.moment_table(), c.SAVED, run=c.RUN, after=c.AFTER)
    shutil.rmtree(c.PARTS)
    print(f"  saved {c.SAVED.name}")


def verify(path):
    """An archive against the original's fingerprints: the whole file, then array by array."""
    import numpy as np
    path = Path(path)
    want = EXPECTED.get(path.name)
    if want is None:
        sys.exit(f"Expected one of: {', '.join(EXPECTED)}")
    whole = sha256(path)
    print(f"{path.name}: {path.stat().st_size:,} bytes, SHA-256 {whole}")
    print("  the original, byte for byte" if whole == want["sha256"] else
          f"  not byte for byte the original ({want['bytes']:,} bytes, {want['sha256']}); array by array:")
    same = 0
    with np.load(path) as data:
        for key, (dtype, shape, digest) in want["arrays"].items():
            if key not in data.files:
                state = "MISSING"
            else:
                a = data[key]
                if (a.dtype.str, list(a.shape)) != (dtype, shape):
                    state = f"DIFFERENT (type {a.dtype.str}, shape {list(a.shape)}; the original's: {dtype}, {shape})"
                else:
                    state = "SAME" if hashlib.sha256(np.ascontiguousarray(a).tobytes()).hexdigest() == digest else "DIFFERENT"
            same += state == "SAME"
            print(f"  {state:10s} {key}")
        for key in want["wall_clock"]:
            print(f"  {'-':10s} {key} (wall-clock seconds, not compared)")
        for key in sorted(set(data.files) - set(want["arrays"]) - set(want["wall_clock"])):
            print(f"  {'EXTRA':10s} {key}")
    print(f"  {same} of {len(want['arrays'])} arrays the same as the original's")
    return same == len(want["arrays"])


if __name__ == "__main__":
    command, rest = (sys.argv[1] if len(sys.argv) > 1 else ""), sys.argv[2:]
    if command == "inputs" and not rest:
        inputs()
    elif command == "fresh" and not rest:
        fresh()
    elif command == "replication" and not rest:
        replication()
    elif command == "cause" and len(rest) <= 1 and all(arg.isdigit() and int(arg) > 0 for arg in rest):
        cause(int(rest[0]) if rest else None)
    elif command == "verify" and len(rest) == 1:
        sys.exit(0 if verify(rest[0]) else 1)
    else:
        sys.exit(__doc__)