#!/usr/bin/env python3
"""Build the published GENERATION 5 e64 settling instrument.
Spec: #p8v2_gen3_viz_goal_20260810 (the standing instrument spec) applied to
generation 5, #pprog_p8v2_gen5_goal_20260815 and its report (../cmpr-src). The
generation-4 instrument is docs/pprog/build-p8v2-gen4-e64 and is FROZEN; one page
per generation, per #viz_standard "Versioning a viewer".
WHAT GENERATION 5 CHANGES FOR THIS PAGE. f is SPLIT. What a k=1 application
DELIVERS (#f-p8-forward, axis G) and what a position DOES with the applications that
reach it over time (#f-p8-decay, axis H) are now two axes at model header bytes 42
and 43, and generation 4's axis A is frozen at its recorded pairs. So the single
"A update" control of the last two pages becomes TWO controls, and the seven
alternatives the page drives are the decay rules H1..H7 -- the baseline plus the six
retired A ideas restated on the cap-8 delivery.
PUBLICATION IS ../hutter's. This script runs no compression: it reads the eight
retained e64 models and the eight .pos dumps, both cmpr-src's, and bakes what the
page needs into one self-contained HTML file.
docs/pprog/build-p8v2-gen5-e64 [--cmpr-src PATH]
"""
import json
import os
import re
import struct
import subprocess
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, "p8v2-gen5-e64")
SAMPLE = "64"
# GENERATION 5 (#pprog_p8v2_gen5_report_20260817). v024 (711111) is the generation's
# baseline -- the cap-8 delivery with a constant fall-off of 1, which is (G1, H1) --
# and v026..v031 (71111112 .. 71111117) are the six restated decay rules off it.
# v025 (721111) is the B2 GROUND-TRUTH COLUMN: the same delivery read from the stored
# 65536-byte count matrix instead of the stated 8, and the only variant here that
# serializes that matrix, which is what the smax-driven rules H5 and H6 read.
#
# THE AXIS STRING IS NOW VARIABLE LENGTH. The two new digits are written only when a
# variant names the split, so v024 and v025 still carry six characters and reproduce
# their generation-4 rows byte for byte; v026..v031 carry eight. read_model reads the
# whole NUL-padded field rather than a fixed six bytes.
#
# Axis labels and alternative prose are READ FROM axes.json, not restated here.
GEN = 5
POSDIR = "gen%d-pos" % GEN
VARIANTS = ["wordsv2-v%03d" % i for i in (24, 25, 26, 27, 28, 29, 30, 31)]
# model layout, from #pp_wordsv2: 48-byte header, then the k=1 argmax table, the
# backward LPP, the token section (4 bytes per rule), the gap section, the trace.
#
# THE k=0 BACKGROUND IS GONE as of 2026-08-06 (#pp_wordsv2). It used to occupy 256
# bytes at offset 48 and every offset after the header has moved down by 256. A model
# built before that date is 256 bytes longer and read_model rejects it on the size
# identity rather than misreading the argmax table as a background.
HDR = 48
K1 = 48
BWD0 = 304
BLOCKS = ["p8v2_gen3_viz_goal_20260810", "viz_standard",
"p8v2_latd_probability_ask_20260815",
# The ring surface's own goal, with MJC's four answers written into it, and
# the review it answers. The page quotes both -- for what the AND ring is
# for, for what the bpc row is and is not, and for the answer to gap 4 -- so
# under #viz_standard the source document is published, not paraphrased.
"p8v2_ring_viz_goal_20260817", "commentary_gen5_e64"] # ours
# The generation-2 report and the goal it answers are published beside the page rather
# than paraphrased on it (#viz_standard: prefer publishing the source document, and a
# block id is not a definition). The generation-1 report stays: the page cites it for
# what the axes looked like before the k=0 removal.
SRC_BLOCKS = ["p8v2_gen3", "pprog_p8v2_gen5_goal_20260815",
"pprog_p8v2_gen5_report_20260817", "f-p8", "f-p8-cap8",
"f-p8-forward", "f-p8-decay", "f-p8-decay-assign",
"f-p8-decay-const2", "f-p8-decay-renorm", "f-p8-decay-deficit",
"f-p8-decay-period", "f-p8-decay-indegree",
"hutter_publication_handoff", "variant_protocol"]
LINKS = [("gen5-pos", "gen5-pos"), ("models", "models"),
("p8v2-words.md", "p8v2-words.md"), ("gen5.tsv", "gen5.tsv"),
("gen4.tsv", "gen4.tsv"), ("axes.json", "axes.json")]
# LSA.md is at the ROOT of cmpr-src, not under tests/pprog, so it does not go through
# LINKS above. The page's "Is any of this probability?" section quotes it for what an
# LSA value means -- a log count with a stochastic, uncorrelated error -- which is the
# link in the chain that explains why a pattern's stored weight and the count the
# concordance shows are allowed to disagree. Publish the source, do not paraphrase it
# (#viz_standard).
ROOT_LINKS = [("LSA.md", "LSA.md")]
def read_model(path):
d = open(path, "rb").read()
if d[:4] != b"P8V2":
sys.exit("build-p8v2-gen5-e64: %s is not a P8V2 model" % path)
n, m, tc, tb, g, sc, bwd, rounds = struct.unpack("<8I", d[4:36])
# SINCE GENERATION 5 THE AXIS FIELD IS VARIABLE LENGTH. The forward and decay
# digits sit at bytes 42 and 43 and are written only when a variant names the
# split, so a generation-4 vector is six characters NUL-padded and a generation-5
# one is eight. Read the whole field and strip; a fixed slice of six would
# silently render v026 as the baseline.
axes = d[36:48].rstrip(b"\0").decode("ascii")
if not axes.isdigit() or len(axes) not in (6, 8):
sys.exit("build-p8v2-gen5-e64: %s carries an axis field %r that is neither a "
"six-digit vector nor an eight-digit split one" % (path, axes))
want = 48 + 256 + bwd + tb + g + sc
if len(d) != want:
sys.exit("build-p8v2-gen5-e64: %s is %d bytes, its header says %d. A model built "
"before the k=0 removal of 2026-08-06 is 256 bytes longer; regenerate "
"it with cmpr-src's tests/pprog/acceptance-p8v2." % (path, len(d), want))
k1 = list(d[K1:K1 + 256])
lpp = list(d[BWD0:BWD0 + bwd]) if bwd else None
toff = BWD0 + bwd
toks = []
for i in range(tc):
a, b, c, w = d[toff + 4 * i: toff + 4 * i + 4]
toks.append([a, b, c, w])
return dict(axes=axes, M=m, TC=tc, ROUNDS=rounds, k1=k1, lpp=lpp,
toks=toks, size=len(d), bwd=bwd)
BYTE_RE = re.compile(r"0x([0-9A-F]{2})|(.)", re.S)
def bytes_in(name):
"""The byte values a printed event name spells.
cmpr writes a byte either literally or as 0xNN. Nothing marks which, so
"0x20" is in principle a literal '0','x','2','0' -- every name parsed here is
checked against the model bytes, so a wrong reading fails the build instead of
reaching the page.
"""
out = []
for m in BYTE_RE.finditer(name):
out.append(int(m.group(1), 16) if m.group(1) else ord(m.group(2)))
return out
def sn_patterns(cmpr, cmpr_src, model_path, mod):
"""The SN atoms for the patterns this page draws, verbatim from the query layer.
"SN means that every pattern is interpretable" (#p8v2_gen3, item 2 of
#p8v2_gen3_viz_goal_20260810). The page used to name a pattern as a pair of
byte values, which is the implementation's view of it and not a pattern.
THIS SELECTS, IT DOES NOT DUMP. --patterns emits the whole structural line --
66152 lines for a generation-2 model at e64, of which 32385 groups are the
memory chain, which lies outside the window of indeterminacy and is neither
drawn nor stepped. Two families are drawn and only those are kept: the learned
k=1 markov (input to input) and the k=2 token rules (2-token to input).
IN A DUMP THE UNINDENTED LINE IS THE FROM EVENT and the line indented under it
is the TO event with its strength. The memory chain settles the direction:
only one reading makes "1 time steps ago" -> "2 time steps ago" a shift.
Every atom is cross-checked against the model bytes, in both directions: the
k=1 family must arrive in from-byte order 0..255 with each to-byte equal to
k1[a], and the token family must be exactly the model's token section.
WHAT IS RETURNED IS THE EVENT NAMES, NOT THE PATTERNS. A pattern is a pair of
events and a strength; the events are the part the query layer names and the
pairing is already in the model tables the page carries. Baking both texts of
every pattern of every variant instead cost 208 kB of duplicated string for ten
models that share 256 event names between them.
"""
p = subprocess.run([cmpr, "--run", "wordsv2", "--ofra",
os.path.join("tests", "pprog", "p8v2-words.md"),
"--patterns", "--model", model_path],
cwd=cmpr_src, capture_output=True, text=True)
if p.returncode != 0:
sys.exit("build-p8v2-gen5-e64: the query layer failed on %s:\n%s"
% (model_path, p.stderr.strip()))
k1, tok, frm = [], [], None
for line in p.stdout.split("\n"):
if not line.strip():
continue
if not line.startswith(" "):
frm = line.strip()
continue
to = line.strip()
m = re.match(r'^(".*\.") (\d+)\.$', to)
if not m:
sys.exit("build-p8v2-gen5-e64: cannot read an SN line: %r" % to)
to_ev, w = m.group(1), int(m.group(2))
f_in = re.match(r'^"The input byte is (.*)\."$', frm)
f_tk = re.match(r'^"The 2-token is "(.*)"\."$', frm)
t_in = re.match(r'^"The input byte is (.*)\."$', to_ev)
if f_in and t_in:
k1.append([bytes_in(f_in.group(1)), bytes_in(t_in.group(1)), w, frm, to_ev])
elif f_tk and t_in:
tok.append([bytes_in(f_tk.group(1)), bytes_in(t_in.group(1)), w, frm, to_ev])
if len(k1) != 256:
sys.exit("build-p8v2-gen5-e64: the k=1 family has %d atoms, expected 256, in %s"
% (len(k1), model_path))
event, k1w = [None] * 256, [0] * 256
for a, (fb, tb_, w, fa, ta) in enumerate(k1):
k1w[a] = w
if fb != [a] or len(tb_) != 1 or tb_[0] != mod["k1"][a]:
sys.exit("build-p8v2-gen5-e64: SN atom %d of the k=1 family reads %r -> %r, but the "
"model says %d -> %d. Either the dump is not in from-byte order or a "
"byte name was misread." % (a, fa, ta, a, mod["k1"][a]))
event[a] = fa
if event[tb_[0]] not in (None, ta):
sys.exit("build-p8v2-gen5-e64: byte %d is named both %r and %r"
% (tb_[0], event[tb_[0]], ta))
event[tb_[0]] = ta
tok_out, seen = {}, []
for ab, c, w, fa, ta in tok:
if len(ab) != 2 or len(c) != 1:
sys.exit("build-p8v2-gen5-e64: a token atom does not spell two bytes: %r" % fa)
seen.append([ab[0], ab[1], c[0], w])
tok_out["%d,%d" % (ab[0], ab[1])] = fa
if sorted(seen) != sorted(mod["toks"]):
sys.exit("build-p8v2-gen5-e64: the token atoms in %s are not the model's token section "
"(%d atoms against %d rules)" % (model_path, len(seen), len(mod["toks"])))
if None in event:
sys.exit("build-p8v2-gen5-e64: the k=1 family did not name every byte")
# k1w CANNOT BE CHECKED AGAINST THE MODEL -- the k=1 section is 256 bytes of
# argmax and stores no weight -- but since generation 4 it is checked against
# axes.json's stated k1_strength in main(): the strength is a READING, and the
# reading is now single-sourced. The token weights, which the model does store,
# agree exactly.
return {"event": event, "token": tok_out, "k1w": k1w}
def check_latd(cmpr, cmpr_src, model_path, mod, sn, sample):
"""Check the page's own LATD scan against `cmpr --latd`, pattern by pattern.
THE PAGE REIMPLEMENTS LATD IN JAVASCRIPT, and it has to: the expansion has to
follow the selection, and shipping 309 baked answers would be a copy of a fact
the query layer already owns. But #viz_standard's rule for a builder is that it
IMPORTS the upstream derivation rather than reimplementing it, so that the page
cannot drift from the numbers -- and a reimplementation is exactly a drift risk.
The resolution is to reimplement it and then PROVE the copy: the same scan the
JS does, in Python, against `cmpr --run wordsv2 --latd` for every pattern the
browser lists. A disagreement fails the build. The scan is cheap on both sides
(about 4 ms per query, 309 queries) and it makes the page's claim -- that this is
the support set and not an estimate of it -- checkable rather than asserted.
Returns what was checked, so the page can say so instead of the reader taking it
on trust.
"""
M = len(sample)
checked, skipped = 0, []
def scan(pred, n):
"""positions i with pred(i) true and a successor at i+n -- the JS scan"""
return [(i, sample[i + n]) for i in range(M - n) if pred(i)]
want = []
for a in range(256):
want.append((sn["event"][a], scan(lambda i, a=a: sample[i] == a, 1)))
for t in mod["toks"]:
key = "%d,%d" % (t[0], t[1])
want.append((sn["token"][key],
scan(lambda i, t=t: i + 1 < M and sample[i] == t[0]
and sample[i + 1] == t[1], 2)))
for atom, sup in want:
# --latd names a pattern by its ANTECEDENT SENTENCE exactly as --patterns
# prints it, and that is what sn_patterns kept.
p = subprocess.run([cmpr, "--run", "wordsv2", "--ofra",
os.path.join("tests", "pprog", "p8v2-words.md"),
"--model", model_path, "--latd", atom.strip('"')],
cwd=cmpr_src, capture_output=True, text=True)
if p.returncode != 0:
# An unkept 2-token is a documented refusal, not a failure; anything else is.
if "no k=2 rule" in p.stderr:
skipped.append(atom)
continue
sys.exit("build-p8v2-gen5-e64: --latd failed on %s:\n%s"
% (atom, p.stderr.strip()))
m = re.search(r"^ positions:(.*)$", p.stdout, re.M)
if m is None:
sys.exit("build-p8v2-gen5-e64: --latd printed no positions line for %s" % atom)
got = [int(x) for x in m.group(1).split()]
if got != [i for i, _ in sup]:
sys.exit("build-p8v2-gen5-e64: the page's LATD scan disagrees with the query "
"layer on %s: --latd says %r, the scan says %r. The scan is the "
"page's own and the query layer is the authority; fix the scan."
% (atom, got, [i for i, _ in sup]))
# and the successor counts, which are what the concordance's disagreement
# column is read off
c = {}
for _, s in sup:
c[s] = c.get(s, 0) + 1
m = re.search(r"^ observed successors:(.*)$", p.stdout, re.M)
f = (m.group(1).split() if m else [])
# Compare BYTE VALUES, not the printed names: the query layer writes a space as
# 0x20 and a full stop as '.', and reconstructing that convention here would be
# a second place for it to be wrong. bytes_in is the reader that already exists.
cli = {}
for i in range(0, len(f) - 1, 2):
b = bytes_in(f[i])
if len(b) != 1:
sys.exit("build-p8v2-gen5-e64: --latd named a successor %r that is not one "
"byte, expanding %s" % (f[i], atom))
cli[b[0]] = int(f[i + 1])
ours = dict(c)
if cli != ours:
sys.exit("build-p8v2-gen5-e64: the page's successor counts disagree with the "
"query layer on %s: --latd says %r, the scan says %r" % (atom, cli, ours))
checked += 1
return {"checked": checked, "skipped": len(skipped), "model": os.path.basename(model_path)}
def sn_merge(data):
"""One set of event names for the page.
The ten models were each queried and each verified against its own bytes; the
NAMES are a property of the event system, not of a model, so they merge into
one table and disagreement is a build failure rather than something the page
picks between.
"""
event, token, k1w = None, {}, None
for v, d in sorted(data.items()):
if event is None:
event, k1w = d["sn"]["event"], d["sn"]["k1w"]
elif d["sn"]["event"] != event:
sys.exit("build-p8v2-gen5-e64: %s names the bytes differently from the other "
"models" % v)
elif d["sn"]["k1w"] != k1w:
sys.exit("build-p8v2-gen5-e64: %s reports different k=1 pattern strengths. The "
"page carries one set; give it one per variant instead." % v)
for k, atom in d["sn"]["token"].items():
if token.setdefault(k, atom) != atom:
sys.exit("build-p8v2-gen5-e64: token %s is named both %r and %r"
% (k, token[k], atom))
return {"event": event, "token": token, "k1w": k1w}
def read_pos(path):
rows = []
for line in open(path):
if line.startswith("#") or not line.strip():
continue
f = line.rstrip("\n").split("\t")
rows.append(dict(pos=int(f[0]), byte=int(f[1]), rec=int(f[2]),
causal=int(f[3]), tok_fired=int(f[4]),
tok_a=int(f[5]), tok_b=int(f[6]), tok_c=int(f[7]),
settled=int(f[8]), s1=int(f[9]), w1=int(f[10]),
s2=int(f[11]), w2=int(f[12]), conv=int(f[13])))
return rows
def main(argv):
cmpr_src = os.path.abspath(os.path.join(HERE, "..", "..", "..", "cmpr-src"))
if "--cmpr-src" in argv:
cmpr_src = os.path.abspath(argv[argv.index("--cmpr-src") + 1])
pprog = os.path.join(cmpr_src, "tests", "pprog")
mdir = os.path.join(pprog, "models", "p8v2", "enwik9", SAMPLE)
if not os.path.isdir(mdir):
sys.exit("build-p8v2-gen5-e64: no %s -- run acceptance-p8v2 in cmpr-src first" % mdir)
os.makedirs(OUT, exist_ok=True)
rel = os.path.relpath(pprog, OUT)
relroot = os.path.relpath(cmpr_src, OUT)
for base, relbase, links, need in ((pprog, rel, LINKS, False),
(cmpr_src, relroot, ROOT_LINKS, True)):
for name, target in links:
link = os.path.join(OUT, name)
if not os.path.exists(os.path.join(base, target)):
# The pprog artifacts are optional -- a missing TSV is reported later and
# the page says so. LSA.md is not: the probability section links it as
# its source, and a section that quotes a document nobody can open is
# the paraphrase #viz_standard is written against.
if need:
sys.exit("build-p8v2-gen5-e64: no %s to publish beside the page"
% os.path.join(base, target))
continue
if os.path.islink(link) or os.path.exists(link):
os.remove(link)
os.symlink(os.path.join(relbase, target), link)
# The axis document is the page's source for every label, every alternative and
# every definition. It is emitted by cmpr-src's tests/pprog/p8v2-axes.py and gated
# by tests/pprog/acceptance-p8v2-axes, which fails if any alternative's prose is not
# the text of its OFRA block byte for byte. Read it; do not restate it.
axes_path = os.path.join(pprog, "axes.json")
if not os.path.exists(axes_path):
sys.exit("build-p8v2-gen5-e64: no %s -- run cmpr-src's tests/pprog/p8v2-axes.py first"
% axes_path)
axes_doc = json.load(open(axes_path))
if axes_doc.get("schema") != "p8v2-axes/1":
sys.exit("build-p8v2-gen5-e64: axes.json schema is %r, expected 'p8v2-axes/1'"
% axes_doc.get("schema"))
# THE CONCESSION IS GONE: axes.json said generation 2 while carrying A4, A5 and A6, so
# this builder checked only that the alternatives were present. cmpr-src corrected the
# field on 2026-08-11 and added `generations`, which lists every generation the file
# describes -- the right shape, because a frozen page renders an older generation out
# of the same living file. Both checks now run: the generation is one the file
# describes, AND the alternatives this page draws are in it.
#
# AND IT IS ONE GENERATION BEHIND AGAIN, 2026-08-17. The regenerated file says
# generation 5 in `generation` and lists [1, 2, 3, 4] in `generations`: the list is
# a literal in cmpr-src's tests/pprog/p8v2-axes.py and was not extended when the
# generation was. Membership is still the right test -- a frozen page renders an
# old generation out of a living file -- so the two fields are taken TOGETHER, the
# file's current generation being one it describes by definition. Reported
# upstream; when the literal is fixed this union collapses to `generations` alone.
gens = list(axes_doc.get("generations") or [])
if axes_doc.get("generation") is not None and axes_doc["generation"] not in gens:
gens.append(axes_doc["generation"])
if GEN not in gens:
sys.exit("build-p8v2-gen5-e64: axes.json describes generations %r, not %d, so it "
"cannot label this page's runs. Regenerate it with cmpr-src's "
"tests/pprog/p8v2-axes.py." % (gens, GEN))
# The alternatives this page DRIVES: the cap-8 pair it inherits, the forward axis
# and all seven decay rules. A missing one would leave a control with no label.
need = {"A": [7], "G": [1], "H": [1, 2, 3, 4, 5, 6, 7]}
have = {a["letter"]: [alt["digit"] for alt in a["alternatives"]] for a in axes_doc["axes"]}
for letter, digits in need.items():
missing = [d for d in digits if d not in have.get(letter, [])]
if missing:
sys.exit("build-p8v2-gen5-e64: axes.json has no alternative %s%s. Regenerate it "
"with cmpr-src's tests/pprog/p8v2-axes.py."
% (letter, ",".join(str(m) for m in missing)))
# axes.json's baseline field names v002/211111 -- the vector every generation's
# digits are read against. The GENERATION'S baseline is v024, which is (G1, H1)
# written without the split digits; that identity is what lets generation 4's rows
# be reproduced byte for byte, and it is checked against the model below.
baseline = "wordsv2-v024"
if baseline not in VARIANTS:
sys.exit("build-p8v2-gen5-e64: the generation-5 baseline %s is not in the "
"variant set" % baseline)
cmpr = os.environ.get("CMPR") or os.path.join(cmpr_src, "cmpr", "dist", "cmpr")
if not os.access(cmpr, os.X_OK):
cmpr = "cmpr"
data = {}
for v in VARIANTS:
mod = read_model(os.path.join(mdir, v + ".m"))
mod["pos"] = read_pos(os.path.join(pprog, POSDIR, "%s.e64.pos" % v))
mod["sn"] = sn_patterns(cmpr, cmpr_src, os.path.join(mdir, v + ".m"), mod)
data[v] = mod
# Every variant's axis vector must be the one axes.json gives it, or the page would
# attach a label to a run that is not that run.
byvec = {}
for a in axes_doc["axes"]:
for alt in a["alternatives"]:
byvec.setdefault(a["letter"], {})[alt["digit"]] = alt
if data[baseline]["axes"] != "711111":
sys.exit("build-p8v2-gen5-e64: %s carries axes %s, expected 711111 -- the cap-8 "
"pair written without the split digits, which is (G1, H1)"
% (baseline, data[baseline]["axes"]))
# THE SPLIT, CHECKED AGAINST THE MODELS. The six restatements must each carry an
# eight-digit vector whose forward digit is the baseline 1 and whose decay digit is
# 2..7, and the two carried-over variants must still carry six digits -- otherwise
# "v024 reproduces generation 4 byte for byte" is not a claim this page may make.
want_decay = dict(zip(["wordsv2-v%03d" % i for i in range(26, 32)], range(2, 8)))
for v, d in sorted(data.items()):
ax = d["axes"]
if v in want_decay:
if len(ax) != 8 or ax[6] != "1" or int(ax[7]) != want_decay[v]:
sys.exit("build-p8v2-gen5-e64: %s carries axes %s; generation 5 expects "
"an eight-digit vector at forward 1, decay %d"
% (v, ax, want_decay[v]))
elif len(ax) != 6:
sys.exit("build-p8v2-gen5-e64: %s carries the eight-digit vector %s, but it "
"is one of the two variants carried over from generation 4, which "
"do not name the split" % (v, ax))
base = data[baseline]
M = base["M"]
sample = [r["byte"] for r in base["pos"]]
for v, d in data.items(): # the sample is one sample
assert [r["byte"] for r in d["pos"]] == sample, "%s: different bytes" % v
assert d["M"] == M
# Which variants share the baseline's learned model? The prereq says B2's stored
# LPP is only valid for those, and that E relearns k=1. Check rather than assume.
def learned(d):
return (d["k1"], d["toks"])
shares = sorted(v for v, d in data.items() if learned(d) == learned(base))
if shares != sorted(data):
sys.exit("build-p8v2-gen5-e64: generation 5 varies axes B, G and H only, none of "
"which changes learning, so every variant must carry the same learned "
"model; these do not: %r" % shares)
# THE COUNT MATRIX, AND WHY IT IS ON THE PAGE LEGITIMATELY THIS GENERATION.
#
# H5 and H6 set a pattern's rate from ITS OWN SUPPORT, and B2 settles with the
# stored weights: both read the 65536-byte learned count matrix, which a B1 model
# does not serialize. It is IN THE RUN SET -- v025 is the generation's B2
# ground-truth column and stores it -- so nothing is borrowed from outside the
# generation, which is what the generation-3 page had to do. There is exactly one
# such variant here, so there is nothing to pick between; the learned-model identity
# checked above is what makes reading it under v024's siblings honest.
owner = "wordsv2-v025"
c1mod = data[owner]
if not c1mod["lpp"]:
sys.exit("build-p8v2-gen5-e64: %s stores no LPP; the B2 preset and the "
"support-driven decay rules H5 and H6 cannot be driven" % owner)
others = sorted(v for v, d in data.items() if d["lpp"] and v != owner)
for v in others:
if data[v]["lpp"] != c1mod["lpp"]:
sys.exit("build-p8v2-gen5-e64: %s and %s store different count matrices over "
"the same learned model; refusing to pick one" % (owner, v))
c1 = c1mod["lpp"]
# THE CONSTANTS THE PAGE READS (generation 4's c5): k1_strength and the per-
# alternative fall-offs live in axes.json's constants section, and the template
# reads them from the baked axes_doc rather than holding a copy. Absent constants
# would make the page invent f's numbers, so they are checked here.
cst = axes_doc.get("constants") or {}
prog = cst.get("program") or {}
pa = cst.get("per_alternative") or {}
# SINCE THE SPLIT THE CONSTANTS COME FROM THE TWO NEW BLOCKS: #f-p8-forward carries
# the delivery's k1_strength and #f-p8-decay the baseline fall-off, with
# #f-p8-decay-const2 carrying the one-constant control's 2. The page reads all
# three from the baked axes_doc; a missing one would make it invent f's numbers.
missing = [n for n, k in (("f-p8-forward.k1_strength", ("f-p8-forward", "k1_strength")),
("f-p8-decay.falloff", ("f-p8-decay", "falloff")),
("f-p8-decay-const2.falloff", ("f-p8-decay-const2", "falloff")))
if k[1] not in (pa.get(k[0]) or {})]
if "k1_strength" not in prog or missing:
sys.exit("build-p8v2-gen5-e64: axes.json carries no constants for %s. Regenerate "
"it with cmpr-src's tests/pprog/p8v2-axes.py."
% (", ".join(missing) or "program.k1_strength"))
if pa["f-p8-forward"]["k1_strength"] != prog["k1_strength"]["value"]:
sys.exit("build-p8v2-gen5-e64: axes.json states the delivery as %r on "
"#f-p8-forward and %r as the program constant; the page carries one "
"number and will not pick"
% (pa["f-p8-forward"]["k1_strength"], prog["k1_strength"]["value"]))
# sup1[x] is the learned support of x's own argmax rule, and smax the reference
# support -- both exactly as #f-p8-deficit builds them, including its floor of 1.
sup1 = [c1[x * 256 + base["k1"][x]] for x in range(256)]
tokw = [t[3] for t in base["toks"]]
smax_model = max([1] + sup1 + tokw)
# THE SETTLING DIAGNOSTICS, READ FROM gen5.tsv. The TSV is the generation's own
# published record and carries every column the page quotes; axes.json's
# settling_columns.numbers historically lagged the newest generation, so the TSV
# has been the source since the generation-3 page and stays it.
tsv = os.path.join(pprog, "gen%d.tsv" % GEN)
settling = []
if os.path.exists(tsv):
with open(tsv) as fh:
head = fh.readline().rstrip("\n").split("\t")
for line in fh:
r = dict(zip(head, line.rstrip("\n").split("\t")))
if r.get("sample") == "e64" and r.get("variant") in data:
settling.append({k: r.get(k) for k in
("variant", "axes", "sample", "settled_ok",
"mean_sweeps", "settle_apps", "roundtrip",
"p_trace", "tokens_kept")})
if len(settling) != len(data):
sys.stderr.write("build-p8v2-gen5-e64: gen%d.tsv has e64 rows for %d of the %d "
"variants; the per-axis table will say so rather than guess\n"
% (GEN, len(settling), len(data)))
# THE EXTIRPATION, CHECKED: since generation 4 the query layer reports every k=1
# pattern at the stated strength (constants.program.k1_strength), not a literal 1.
# A different value here means the installed cmpr predates generation 4 -- the
# acceptance-p8v2 CMPR trap, seen from this side.
sn0 = sn_merge(data)
k1s = prog["k1_strength"]["value"]
if any(w != k1s for w in sn0["k1w"]):
sys.exit("build-p8v2-gen5-e64: the query layer reports k=1 strengths %r, but "
"axes.json states k1_strength = %d. The cmpr being run predates the "
"generation-4 extirpation; point CMPR at cmpr-src's dist build."
% (sorted(set(sn0["k1w"])), k1s))
# THE PAGE'S LATD SCAN, PROVED AGAINST THE QUERY LAYER. Checked once, on the
# baseline: the four variants share one learned model byte for byte, which is
# asserted above, so one check covers the set.
sn = sn0
latd = check_latd(cmpr, cmpr_src, os.path.join(mdir, baseline + ".m"), base, sn, sample)
# THE LADDER, WHICH IS THE REPORT'S HEADLINE TABLE AND IS COMPUTED HERE RATHER THAN
# COPIED. settled_ok in the TSV counts the recorded positions too, and those are
# clamped and correct by construction, so it is bounded below by a number that has
# nothing to do with settling. What the generation is actually measured on is the
# UNRECORDED positions that settle back to their true byte, at each rung of the
# prefix ladder -- #viz_standard: a rate on one prefix is a point on a curve, so
# publish the trend. Both numbers go on the page, side by side, from the dumps.
ladder = {}
for v in VARIANTS:
for smp in ("e64", "e1k", "e10k"):
f = os.path.join(pprog, POSDIR, "%s.%s.pos" % (v, smp))
if not os.path.exists(f):
continue
rows = read_pos(f)
unrec = [r for r in rows if not r["rec"]]
ladder.setdefault(v, {})[smp] = {
"rows": len(rows),
"unrec": len(unrec),
"unrec_ok": sum(1 for r in unrec if r["settled"] == r["byte"]),
"ok": sum(1 for r in rows if r["settled"] == r["byte"]),
}
payload = {
"M": M,
"ladder": ladder,
"latd_check": latd,
"settling": settling,
"W": M, # one window at e64: ws=0, we=M, so W = M = 64
"matrix_owner_is_in_generation": True,
"gen": GEN,
"baseline": baseline,
"baseline_axes": axes_doc["baseline"]["axes"],
"sample": sample,
"shares_baseline_model": shares,
"lpp_owner": owner,
"lpp": c1,
"sup1": sup1, # the learned support of each byte's own argmax rule
"smax_model": smax_model, # the reference support, as #f-p8-deficit computes it
"axes_doc": axes_doc, # labels, alternatives, definitions, worked examples
"sn": sn, # the event names, verbatim from the query layer
"variants": {},
}
for v, d in data.items():
payload["variants"][v] = {
"axes": d["axes"], "rounds": d["ROUNDS"], "size": d["size"],
"k1": d["k1"], "toks": d["toks"],
"rec": [r["rec"] for r in d["pos"]],
"causal": [r["causal"] for r in d["pos"]],
"endpoint": [[r["settled"], r["s1"], r["w1"], r["s2"], r["w2"]]
for r in d["pos"]],
"conv": d["pos"][0]["conv"], # per-window, all rows repeat it
}
# block dumps beside the page
for b, cwd in ([(x, cmpr_src) for x in SRC_BLOCKS] +
[(x, os.path.join(HERE, "..", "..")) for x in BLOCKS]):
p = subprocess.run([cmpr, "--print-block", "#" + b], cwd=cwd,
capture_output=True, text=True)
if p.returncode == 0 and p.stdout.strip():
open(os.path.join(OUT, b + ".txt"), "w").write(p.stdout)
else:
sys.stderr.write("build-p8v2-gen5-e64: could not dump #%s\n" % b)
tpl = open(os.path.join(HERE, "p8v2-gen5-e64.tpl.html")).read()
page = tpl.replace("/*DATA*/null", json.dumps(payload, separators=(",", ":")))
open(os.path.join(OUT, "index.html"), "w").write(page)
# THE NOTES PAGE, AND WHY IT IS A SECOND PAGE. Everything discursive -- what
# generation 3 changed, where the support is borrowed from, what is still open --
# used to sit between the reader and the instrument on one long page. MJC, 2026-08-15:
# the instrument is where the effort goes, not the prose. So the commentary moved
# here and links back INTO the instrument by fragment (v/p/s/t), which is the deep
# link #p8v2_gen3_viz_goal_20260810 item 4c asked for.
#
# IT SHARES THE INSTRUMENT'S STYLESHEET RATHER THAN CARRYING A COPY. Two hand-kept
# copies of a stylesheet is the gen1-viz.py fork in miniature (#viz_surfaces), so
# the ", tpl, re.S)
if not m:
sys.exit("build-p8v2-gen5-e64: no