#!/usr/bin/env python3 """Build the published e64 settling instrument. Spec: #p8v2_e64_viewer_goal_20260805. Prerequisites and corrections: #hutter_e64_viewer_prereq_20260805 (in ../cmpr-src). PUBLICATION IS ../hutter's. This script runs no compression: it reads the eleven retained e64 models and the eleven .pos dumps, both cmpr-src's, and bakes what the page needs into one self-contained HTML file. docs/pprog/build-p8v2-e64 [--cmpr-src PATH] """ import json import os import re import struct import subprocess import sys HERE = os.path.dirname(os.path.abspath(__file__)) OUT = os.path.join(HERE, "p8v2-e64") SAMPLE = "64" # GENERATION 2 (#pprog_p8v2_gen2_report_20260806). The baseline is generation 1's # v002, and every generation-2 variant differs from it on exactly one axis; there is # no v012, because "A3 off the new baseline" IS the vector 311111, which is v003, and # the report runs that rather than minting a second id for the same vector. # # The variant set, the baseline and every axis label below are READ FROM axes.json, # not restated here: the report puts them there precisely so this builder stops # carrying a hand-copied dict that drifts from the design. GEN = 2 POSDIR = "gen%d-pos" % GEN VARIANTS = ["wordsv2-v002", "wordsv2-v003"] + \ ["wordsv2-v%03d" % i for i in range(13, 21)] # model layout, from #pp_wordsv2: 48-byte header, then the k=1 argmax table, the # backward LPP, the token section (4 bytes per rule), the gap section, the trace. # # THE k=0 BACKGROUND IS GONE as of 2026-08-06 (#pp_wordsv2). It used to occupy 256 # bytes at offset 48 and every offset after the header has moved down by 256. A model # built before that date is 256 bytes longer and read_model rejects it on the size # identity rather than misreading the argmax table as a background. HDR = 48 K1 = 48 BWD0 = 304 # FROZEN 2026-08-10 as the generation-2 record (#viz_standard, "Versioning a viewer"). # This builder still runs -- the page must stay rebuildable, or "frozen" would mean # "unmaintainable" -- but it is not where generation 3 work goes; that is # build-p8v2-gen3-e64. BLOCKS = ["p8v2_e64_viewer_goal_20260805", "viz_standard"] # ours # The generation-2 report and the goal it answers are published beside the page rather # than paraphrased on it (#viz_standard: prefer publishing the source document, and a # block id is not a definition). The generation-1 report stays: the page cites it for # what the axes looked like before the k=0 removal. SRC_BLOCKS = ["hutter_e64_viewer_prereq_20260805", "hutter_publication_handoff", "pprog_p8v2_gen2_report_20260806", "pprog_p8v2_gen2_goal_20260806", "pprog_p8v2_gen1_report_20260804", "variant_protocol"] LINKS = [("gen1-pos", "gen1-pos"), ("gen2-pos", "gen2-pos"), ("models", "models"), ("p8v2-words.md", "p8v2-words.md"), ("gen1.tsv", "gen1.tsv"), ("gen2.tsv", "gen2.tsv"), ("axes.json", "axes.json")] def read_model(path): d = open(path, "rb").read() if d[:4] != b"P8V2": sys.exit("build-p8v2-e64: %s is not a P8V2 model" % path) n, m, tc, tb, g, sc, bwd, rounds = struct.unpack("<8I", d[4:36]) axes = d[36:42].decode("ascii") want = 48 + 256 + bwd + tb + g + sc if len(d) != want: sys.exit("build-p8v2-e64: %s is %d bytes, its header says %d. A model built " "before the k=0 removal of 2026-08-06 is 256 bytes longer; regenerate " "it with cmpr-src's tests/pprog/acceptance-p8v2." % (path, len(d), want)) k1 = list(d[K1:K1 + 256]) lpp = list(d[BWD0:BWD0 + bwd]) if bwd else None toff = BWD0 + bwd toks = [] for i in range(tc): a, b, c, w = d[toff + 4 * i: toff + 4 * i + 4] toks.append([a, b, c, w]) return dict(axes=axes, M=m, TC=tc, ROUNDS=rounds, k1=k1, lpp=lpp, toks=toks, size=len(d), bwd=bwd) BYTE_RE = re.compile(r"0x([0-9A-F]{2})|(.)", re.S) def bytes_in(name): """The byte values a printed event name spells. cmpr writes a byte either literally or as 0xNN. Nothing marks which, so "0x20" is in principle a literal '0','x','2','0' -- every name parsed here is checked against the model bytes, so a wrong reading fails the build instead of reaching the page. """ out = [] for m in BYTE_RE.finditer(name): out.append(int(m.group(1), 16) if m.group(1) else ord(m.group(2))) return out def sn_patterns(cmpr, cmpr_src, model_path, mod): """The SN atoms for the patterns this page draws, verbatim from the query layer. "SN means that every pattern is interpretable" (#p8v2_gen3, item 2 of #p8v2_gen3_viz_goal_20260810). The page used to name a pattern as a pair of byte values, which is the implementation's view of it and not a pattern. THIS SELECTS, IT DOES NOT DUMP. --patterns emits the whole structural line -- 66152 lines for a generation-2 model at e64, of which 32385 groups are the memory chain, which lies outside the window of indeterminacy and is neither drawn nor stepped. Two families are drawn and only those are kept: the learned k=1 markov (input to input) and the k=2 token rules (2-token to input). IN A DUMP THE UNINDENTED LINE IS THE FROM EVENT and the line indented under it is the TO event with its strength. The memory chain settles the direction: only one reading makes "1 time steps ago" -> "2 time steps ago" a shift. Every atom is cross-checked against the model bytes, in both directions: the k=1 family must arrive in from-byte order 0..255 with each to-byte equal to k1[a], and the token family must be exactly the model's token section. WHAT IS RETURNED IS THE EVENT NAMES, NOT THE PATTERNS. A pattern is a pair of events and a strength; the events are the part the query layer names and the pairing is already in the model tables the page carries. Baking both texts of every pattern of every variant instead cost 208 kB of duplicated string for ten models that share 256 event names between them. """ p = subprocess.run([cmpr, "--run", "wordsv2", "--ofra", os.path.join("tests", "pprog", "p8v2-words.md"), "--patterns", "--model", model_path], cwd=cmpr_src, capture_output=True, text=True) if p.returncode != 0: sys.exit("build-p8v2-e64: the query layer failed on %s:\n%s" % (model_path, p.stderr.strip())) k1, tok, frm = [], [], None for line in p.stdout.split("\n"): if not line.strip(): continue if not line.startswith(" "): frm = line.strip() continue to = line.strip() m = re.match(r'^(".*\.") (\d+)\.$', to) if not m: sys.exit("build-p8v2-e64: cannot read an SN line: %r" % to) to_ev, w = m.group(1), int(m.group(2)) f_in = re.match(r'^"The input byte is (.*)\."$', frm) f_tk = re.match(r'^"The 2-token is "(.*)"\."$', frm) t_in = re.match(r'^"The input byte is (.*)\."$', to_ev) if f_in and t_in: k1.append([bytes_in(f_in.group(1)), bytes_in(t_in.group(1)), w, frm, to_ev]) elif f_tk and t_in: tok.append([bytes_in(f_tk.group(1)), bytes_in(t_in.group(1)), w, frm, to_ev]) if len(k1) != 256: sys.exit("build-p8v2-e64: the k=1 family has %d atoms, expected 256, in %s" % (len(k1), model_path)) event, k1w = [None] * 256, [0] * 256 for a, (fb, tb_, w, fa, ta) in enumerate(k1): k1w[a] = w if fb != [a] or len(tb_) != 1 or tb_[0] != mod["k1"][a]: sys.exit("build-p8v2-e64: SN atom %d of the k=1 family reads %r -> %r, but the " "model says %d -> %d. Either the dump is not in from-byte order or a " "byte name was misread." % (a, fa, ta, a, mod["k1"][a])) event[a] = fa if event[tb_[0]] not in (None, ta): sys.exit("build-p8v2-e64: byte %d is named both %r and %r" % (tb_[0], event[tb_[0]], ta)) event[tb_[0]] = ta tok_out, seen = {}, [] for ab, c, w, fa, ta in tok: if len(ab) != 2 or len(c) != 1: sys.exit("build-p8v2-e64: a token atom does not spell two bytes: %r" % fa) seen.append([ab[0], ab[1], c[0], w]) tok_out["%d,%d" % (ab[0], ab[1])] = fa if sorted(seen) != sorted(mod["toks"]): sys.exit("build-p8v2-e64: the token atoms in %s are not the model's token section " "(%d atoms against %d rules)" % (model_path, len(seen), len(mod["toks"]))) if None in event: sys.exit("build-p8v2-e64: the k=1 family did not name every byte") # k1w IS NOT CHECKED AGAINST THE MODEL, because there is nothing to check it # against: the k=1 section is 256 bytes of argmax and stores no weight at all. # The query layer reports every one of these patterns at strength 1 while the # settling shell applies them at 255 -- see "One pattern, two strengths" on the # page. The token weights, which the model does store, agree exactly. return {"event": event, "token": tok_out, "k1w": k1w} def sn_merge(data): """One set of event names for the page. The ten models were each queried and each verified against its own bytes; the NAMES are a property of the event system, not of a model, so they merge into one table and disagreement is a build failure rather than something the page picks between. """ event, token, k1w = None, {}, None for v, d in sorted(data.items()): if event is None: event, k1w = d["sn"]["event"], d["sn"]["k1w"] elif d["sn"]["event"] != event: sys.exit("build-p8v2-e64: %s names the bytes differently from the other " "models" % v) elif d["sn"]["k1w"] != k1w: sys.exit("build-p8v2-e64: %s reports different k=1 pattern strengths. The " "page carries one set; give it one per variant instead." % v) for k, atom in d["sn"]["token"].items(): if token.setdefault(k, atom) != atom: sys.exit("build-p8v2-e64: token %s is named both %r and %r" % (k, token[k], atom)) return {"event": event, "token": token, "k1w": k1w} def read_pos(path): rows = [] for line in open(path): if line.startswith("#") or not line.strip(): continue f = line.rstrip("\n").split("\t") rows.append(dict(pos=int(f[0]), byte=int(f[1]), rec=int(f[2]), causal=int(f[3]), tok_fired=int(f[4]), tok_a=int(f[5]), tok_b=int(f[6]), tok_c=int(f[7]), settled=int(f[8]), s1=int(f[9]), w1=int(f[10]), s2=int(f[11]), w2=int(f[12]), conv=int(f[13]))) return rows def main(argv): cmpr_src = os.path.abspath(os.path.join(HERE, "..", "..", "..", "cmpr-src")) if "--cmpr-src" in argv: cmpr_src = os.path.abspath(argv[argv.index("--cmpr-src") + 1]) pprog = os.path.join(cmpr_src, "tests", "pprog") mdir = os.path.join(pprog, "models", "p8v2", "enwik9", SAMPLE) if not os.path.isdir(mdir): sys.exit("build-p8v2-e64: no %s -- run acceptance-p8v2 in cmpr-src first" % mdir) os.makedirs(OUT, exist_ok=True) rel = os.path.relpath(pprog, OUT) for name, target in LINKS: link = os.path.join(OUT, name) if not os.path.exists(os.path.join(pprog, target)): continue if os.path.islink(link) or os.path.exists(link): os.remove(link) os.symlink(os.path.join(rel, target), link) # The axis document is the page's source for every label, every alternative and # every definition. It is emitted by cmpr-src's tests/pprog/p8v2-axes.py and gated # by tests/pprog/acceptance-p8v2-axes, which fails if any alternative's prose is not # the text of its OFRA block byte for byte. Read it; do not restate it. axes_path = os.path.join(pprog, "axes.json") if not os.path.exists(axes_path): sys.exit("build-p8v2-e64: no %s -- run cmpr-src's tests/pprog/p8v2-axes.py first" % axes_path) axes_doc = json.load(open(axes_path)) if axes_doc.get("schema") != "p8v2-axes/1": sys.exit("build-p8v2-e64: axes.json schema is %r, expected 'p8v2-axes/1'" % axes_doc.get("schema")) # A FROZEN PAGE CANNOT PIN axes.json'S generation FIELD, and this check used to try. # axes.json is a living file: it moved to generation 3 on 2026-08-11 and this builder # -- for a page frozen as the generation-2 record -- stopped running altogether, which # is the failure mode "frozen must not mean unmaintainable" was meant to exclude. The # file is CUMULATIVE, and says so: `generations` lists every generation it describes. # So the test is that the generation this page renders is one of them. gens = axes_doc.get("generations") or [axes_doc.get("generation")] if GEN not in gens: sys.exit("build-p8v2-e64: axes.json describes generations %r, not %d, so it cannot " "label this page's runs. Regenerate it with cmpr-src's " "tests/pprog/p8v2-axes.py." % (gens, GEN)) baseline = axes_doc["baseline"]["variant"] if baseline not in VARIANTS: sys.exit("build-p8v2-e64: axes.json says the baseline is %s, which is not in the " "variant set" % baseline) cmpr = os.environ.get("CMPR") or os.path.join(cmpr_src, "cmpr", "dist", "cmpr") if not os.access(cmpr, os.X_OK): cmpr = "cmpr" data = {} for v in VARIANTS: mod = read_model(os.path.join(mdir, v + ".m")) mod["pos"] = read_pos(os.path.join(pprog, POSDIR, "%s.e64.pos" % v)) mod["sn"] = sn_patterns(cmpr, cmpr_src, os.path.join(mdir, v + ".m"), mod) data[v] = mod # Every variant's axis vector must be the one axes.json gives it, or the page would # attach a label to a run that is not that run. byvec = {} for a in axes_doc["axes"]: for alt in a["alternatives"]: byvec.setdefault(a["letter"], {})[alt["digit"]] = alt if data[baseline]["axes"] != axes_doc["baseline"]["axes"]: sys.exit("build-p8v2-e64: %s carries axes %s but axes.json calls the baseline %s" % (baseline, data[baseline]["axes"], axes_doc["baseline"]["axes"])) base = data[baseline] M = base["M"] sample = [r["byte"] for r in base["pos"]] for v, d in data.items(): # the sample is one sample assert [r["byte"] for r in d["pos"]] == sample, "%s: different bytes" % v assert d["M"] == M # Which variants share the baseline's learned model? The prereq says B2's stored # LPP is only valid for those, and that E relearns k=1. Check rather than assume. def learned(d): return (d["k1"], d["toks"]) shares = sorted(v for v, d in data.items() if learned(d) == learned(base)) lpp_owner = [v for v, d in data.items() if d["lpp"]] if len(lpp_owner) != 1: sys.exit("build-p8v2-e64: expected exactly one model with a backward LPP, got %r" % lpp_owner) payload = { "M": M, "W": M, # one window at e64: ws=0, we=M, so W = M = 64 "gen": GEN, "baseline": baseline, "baseline_axes": axes_doc["baseline"]["axes"], "sample": sample, "shares_baseline_model": shares, "lpp_owner": lpp_owner[0], "lpp": data[lpp_owner[0]]["lpp"], "axes_doc": axes_doc, # labels, alternatives, definitions, worked examples # sn is the query layer's answer EXCEPT k1w, pinned below: generation 4 # extirpated the literal 1 that --patterns printed when this page was the # generation-2 record, and this FROZEN page renders that record, so the # live reading (the stated 8) is replaced by the recorded one. "sn": dict(sn_merge(data), k1w=[1] * 256), "variants": {}, } for v, d in data.items(): payload["variants"][v] = { "axes": d["axes"], "rounds": d["ROUNDS"], "size": d["size"], "k1": d["k1"], "toks": d["toks"], "rec": [r["rec"] for r in d["pos"]], "causal": [r["causal"] for r in d["pos"]], "endpoint": [[r["settled"], r["s1"], r["w1"], r["s2"], r["w2"]] for r in d["pos"]], "conv": d["pos"][0]["conv"], # per-window, all rows repeat it } # block dumps beside the page for b, cwd in ([(x, cmpr_src) for x in SRC_BLOCKS] + [(x, os.path.join(HERE, "..", "..")) for x in BLOCKS]): p = subprocess.run([cmpr, "--print-block", "#" + b], cwd=cwd, capture_output=True, text=True) if p.returncode == 0 and p.stdout.strip(): open(os.path.join(OUT, b + ".txt"), "w").write(p.stdout) else: sys.stderr.write("build-p8v2-e64: could not dump #%s\n" % b) tpl = open(os.path.join(HERE, "p8v2-e64.tpl.html")).read() page = tpl.replace("/*DATA*/null", json.dumps(payload, separators=(",", ":"))) open(os.path.join(OUT, "index.html"), "w").write(page) sys.stderr.write("build-p8v2-e64: wrote %s (generation %d, baseline %s, %d variants, " "M=%d, W=%d, %d share the baseline model, LPP in %s)\n" % (OUT, GEN, baseline, len(data), M, payload["W"], len(shares), lpp_owner[0])) return 0 if __name__ == "__main__": sys.exit(main(sys.argv))