#!/usr/bin/env python3
"""Build the three PATTERN pages: seeing what a p8v2 model holds.
Spec: cmpr-src's tests/pprog/p8v2-pattern-viz-specs.md, written for generation 5
(#pprog_p8v2_gen5_goal_20260815, choice A: "write two more specs that do not use any of
the same ideas from this one, and then build all three"). cmpr-src specifies; ../hutter
builds, and nothing there is implemented on that side.
WHY THREE AND NOT ONE. Each spec is built on a DIFFERENT organizing principle, so that
the three disagree about what a pattern is: A treats the model as an inventory laid out
in data space, B as a topology, C as something that changes as the sample grows. They
share no encoding, no interaction and no layout, and that is the point -- so this script
renders three page families rather than one page with three tabs.
A the pattern page per (model, sample). 16x16 of source bytes, the argmax
successor printed as the datum, ink for stored support; the
k=2 rules as support-ordered small multiples on one scale
with them; selecting anything dims to its footprint in the
actual text. This is the only interactive one.
B the transition arc per (model, sample). 256 unlabelled ticks on a baseline,
one hairline arc per k=1 rule, support as ARC HEIGHT ALONE
so the page has exactly one visual variable; a second,
fainter baseline of the 65536 k=2 contexts beneath it.
Nothing is selectable: a page to print.
C the learning slopegraph what the model learned at e64, e1k and e10k as three
columns, one line per source byte, the argmax FLIPS labelled
with both successors and the counts behind them. A second
panel puts each rule's support against sample size.
PUBLICATION IS ../hutter's. This script runs no compression. It reads the retained
models and the .pos dumps, both cmpr-src's, reconstructs every pattern THROUGH THE QUERY
LAYER (#pprog_pattern_query_goal_20260706 -- never a de-optimized dump), and bakes what
each page needs into one self-contained HTML file.
docs/pprog/build-p8v2-patterns [--cmpr-src PATH] [--no-latd]
"""
import json
import math
import os
import re
import struct
import subprocess
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, "p8v2-patterns")
# THE PAIR THE SPEC IS ABOUT. "B1 vs B2 is the variant pair (#wordsv2-v024,
# #wordsv2-v025) at the same sample" -- v024 does not serialize the count matrix and
# v025 does, over the same learned model, so the two render as the same page and the
# visual difference between them IS what B1 discards.
B1, B2 = "wordsv2-v024", "wordsv2-v025"
SAMPLES = [("e64", "64", "64"), ("e1k", "1000", "1,000"), ("e10k", "10000", "10,000")]
GEN = 5
POSDIR = "gen%d-pos" % GEN
# model layout, from #pp_wordsv2: 48-byte header (the axis field running to byte 48
# since the generation-5 split), the k=1 argmax table at 48, the backward LPP at 304,
# then four bytes per k=2 rule. There is no k=0 background as of 2026-08-06.
K1 = 48
BWD0 = 304
SRC_BLOCKS = ["pprog_p8v2_gen5_goal_20260815", "pprog_p8v2_gen5_report_20260817",
"f-p8", "f-p8-forward", "f-p8-decay", "pp_wordsv2", "variant_protocol",
"hutter_publication_handoff", "hutter_metrics"]
BLOCKS = ["viz_standard"]
LINKS = [("gen5-pos", "gen5-pos"), ("models", "models"), ("gen5.tsv", "gen5.tsv"),
("axes.json", "axes.json"), ("p8v2-words.md", "p8v2-words.md"),
("p8v2-pattern-viz-specs.md", "p8v2-pattern-viz-specs.md")]
ROOT_LINKS = [("LSA.md", "LSA.md")]
def die(msg):
sys.exit("build-p8v2-patterns: " + msg)
def read_model(path):
d = open(path, "rb").read()
if d[:4] != b"P8V2":
die("%s is not a P8V2 model" % path)
n, m, tc, tb, g, sc, bwd, rounds = struct.unpack("<8I", d[4:36])
axes = d[36:48].rstrip(b"\0").decode("ascii")
want = 48 + 256 + bwd + tb + g + sc
if len(d) != want:
die("%s is %d bytes, its header says %d. A model built before the k=0 removal "
"of 2026-08-06 is 256 bytes longer; regenerate it with cmpr-src's "
"tests/pprog/acceptance-p8v2." % (path, len(d), want))
toks = []
off = BWD0 + bwd
for i in range(tc):
a, b, c, w = d[off + 4 * i: off + 4 * i + 4]
toks.append([a, b, c, w])
return dict(axes=axes, M=m, k1=list(d[K1:K1 + 256]),
lpp=list(d[BWD0:BWD0 + bwd]) if bwd else None,
toks=toks, size=len(d))
def read_pos(path):
"""The sample bytes, from the generation's own dump rather than from enwik9.
The dump is what the runs were measured on and it carries the byte at every
position, so reading it here means the text under a concordance is the same text
the .pos rows are about -- one source, not two that have to agree.
"""
out = []
for line in open(path):
if line.startswith("#") or not line.strip():
continue
f = line.rstrip("\n").split("\t")
out.append(int(f[1]))
return out
BYTE_RE = re.compile(r"0x([0-9A-F]{2})|(.)", re.S)
def bytes_in(name):
"""The byte values a printed event name spells; see build-p8v2-gen5-e64."""
return [int(m.group(1), 16) if m.group(1) else ord(m.group(2))
for m in BYTE_RE.finditer(name)]
def sn_patterns(cmpr, cmpr_src, model_path, mod):
"""Every pattern these pages draw, RECONSTRUCTED THROUGH THE QUERY LAYER.
The spec is explicit that all three pages read only (program text, model file),
reconstructed on demand through the query layer and never a de-optimized dump. So
the model bytes are read above only to CHECK this: the k=1 family must arrive in
from-byte order with each consequent equal to the table's argmax, and the k=2 family
must be exactly the model's token section. A disagreement fails the build rather
than reaching a page.
"""
p = subprocess.run([cmpr, "--run", "wordsv2", "--ofra",
os.path.join("tests", "pprog", "p8v2-words.md"),
"--patterns", "--model", model_path],
cwd=cmpr_src, capture_output=True, text=True)
if p.returncode != 0:
die("the query layer failed on %s:\n%s" % (model_path, p.stderr.strip()))
k1, tok, frm = [], [], None
for line in p.stdout.split("\n"):
if not line.strip():
continue
if not line.startswith(" "):
frm = line.strip()
continue
m = re.match(r'^(".*\.") (\d+)\.$', line.strip())
if not m:
die("cannot read an SN line: %r" % line.strip())
to_ev, w = m.group(1), int(m.group(2))
f_in = re.match(r'^"The input byte is (.*)\."$', frm)
f_tk = re.match(r'^"The 2-token is "(.*)"\."$', frm)
t_in = re.match(r'^"The input byte is (.*)\."$', to_ev)
if f_in and t_in:
k1.append([bytes_in(f_in.group(1)), bytes_in(t_in.group(1)), w, frm, to_ev])
elif f_tk and t_in:
tok.append([bytes_in(f_tk.group(1)), bytes_in(t_in.group(1)), w, frm, to_ev])
if len(k1) != 256:
die("the k=1 family has %d atoms, expected 256, in %s" % (len(k1), model_path))
event, k1w = [None] * 256, [0] * 256
for a, (fb, tb_, w, fa, ta) in enumerate(k1):
k1w[a] = w
if fb != [a] or len(tb_) != 1 or tb_[0] != mod["k1"][a]:
die("SN atom %d of the k=1 family reads %r -> %r but the model says %d -> %d"
% (a, fa, ta, a, mod["k1"][a]))
event[a] = fa
if event[tb_[0]] not in (None, ta):
die("byte %d is named both %r and %r" % (tb_[0], event[tb_[0]], ta))
event[tb_[0]] = ta
if None in event:
die("the k=1 family did not name every byte in %s" % model_path)
tok_out, seen = {}, []
for ab, c, w, fa, ta in tok:
if len(ab) != 2 or len(c) != 1:
die("a token atom does not spell two bytes: %r" % fa)
seen.append([ab[0], ab[1], c[0], w])
tok_out["%d,%d" % (ab[0], ab[1])] = fa
if sorted(seen) != sorted(mod["toks"]):
die("the token atoms in %s are not the model's token section (%d against %d)"
% (model_path, len(seen), len(mod["toks"])))
return {"event": event, "token": tok_out, "k1w": k1w}
def check_latd(cmpr, cmpr_src, model_path, mod, sn, sample, limit):
"""Prove the pages' own footprint scan against `cmpr --latd`, pattern by pattern.
SPEC A DIMS TO "the positions in the sample where it fired", which the page computes
in JavaScript by scanning the text -- it has to, because the selection has to follow
the click and shipping every answer would be a copy of a fact the query layer owns.
#viz_standard's rule for a builder is that it IMPORTS the upstream derivation rather
than reimplementing it, and a reimplementation is exactly a drift risk. So the same
scan runs here in Python and is compared against the query layer's own answer; a
disagreement fails the build.
`limit` caps how many patterns are checked, because the query is a process spawn per
pattern and there are 1074 kept tokens at e10k. WHAT WAS CHECKED IS RETURNED AND
PRINTED ON THE PAGE -- a partial proof stated as partial, never a silent cap.
"""
M = len(sample)
checked, skipped, tried = 0, 0, 0
def scan(pred, n):
return [i for i in range(M - n) if pred(i)]
want = [(sn["event"][a], scan(lambda i, a=a: sample[i] == a, 1)) for a in range(256)]
for t in mod["toks"]:
want.append((sn["token"]["%d,%d" % (t[0], t[1])],
scan(lambda i, t=t: i + 1 < M and sample[i] == t[0]
and sample[i + 1] == t[1], 2)))
for atom, sup in want:
if tried >= limit:
break
tried += 1
p = subprocess.run([cmpr, "--run", "wordsv2", "--ofra",
os.path.join("tests", "pprog", "p8v2-words.md"),
"--model", model_path, "--latd", atom.strip('"')],
cwd=cmpr_src, capture_output=True, text=True)
if p.returncode != 0:
if "no k=2 rule" in p.stderr:
skipped += 1
continue
die("--latd failed on %s:\n%s" % (atom, p.stderr.strip()))
m = re.search(r"^ positions:(.*)$", p.stdout, re.M)
if m is None:
die("--latd printed no positions line for %s" % atom)
got = [int(x) for x in m.group(1).split()]
if got != sup:
die("the pages' footprint scan disagrees with the query layer on %s: --latd "
"says %d positions, the scan says %d. The query layer is the authority; "
"fix the scan." % (atom, len(got), len(sup)))
checked += 1
return {"checked": checked, "skipped": skipped, "of": len(want),
"model": os.path.basename(model_path)}
def true_counts(sample):
"""The bigram counts in the sample itself -- the only counts on these pages that
are counts of anything. A stored LSA weight is a stochastic estimate of one of
these (LSA.md), which is exactly why spec C can show the two disagreeing."""
cnt = {}
for j in range(1, len(sample)):
k = sample[j - 1] * 256 + sample[j]
cnt[k] = cnt.get(k, 0) + 1
return cnt
def argmax_share(cnt):
"""P(argmax): over every context that occurs, the argmax successor's count over the
total. This is the generation's central measured fact about the k=1 strength -- it
falls as the sample grows, so a stated constant is right at small samples and the
stored matrix is right at large ones."""
tot, best = 0, 0
rows = {}
for k, c in cnt.items():
rows.setdefault(k >> 8, []).append(c)
for a, cs in rows.items():
tot += sum(cs)
best += max(cs)
return (best / tot) if tot else 0.0
# --------------------------------------------------------------- page furniture
CSS = """
:root{--bg:#fbfaf7;--fg:#1a1a18;--sub:#6b6a64;--rule:#ddd9d0;--panel:#f4f2ec;
--c1:#2a5d8f;--c2:#8f4a2a;--ok:#2c6e49;--bad:#a8321e}
@media (prefers-color-scheme: dark){
:root{--bg:#16171a;--fg:#e8e6e1;--sub:#9b9a94;--rule:#33353a;--panel:#1e2024;
--c1:#7fb2e0;--c2:#e0a07f;--ok:#7fc7a0;--bad:#e08a7a}}
*{box-sizing:border-box}
body{margin:0;background:var(--bg);color:var(--fg);
font:15px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif}
.wrap{max-width:1180px;margin:0 auto;padding:1.6rem 1rem 4rem}
h1{font-size:1.5rem;margin:0 0 .3rem;font-weight:600}
h2{font-size:1.1rem;margin:2.2rem 0 .5rem;font-weight:600;
border-top:1px solid var(--rule);padding-top:1rem}
h3{font-size:.95rem;margin:1.4rem 0 .3rem;font-weight:600}
p.sub{color:var(--sub);font-size:.86rem;margin:.35rem 0 .9rem}
a{color:var(--c1)}
code,.m{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:.82em}
.panel{background:var(--panel);border:1px solid var(--rule);border-radius:4px;
padding:.7rem .9rem;margin:.9rem 0;font-size:.86rem}
table.dat{border-collapse:collapse;width:100%;font-size:.82rem}
table.dat th,table.dat td{border-bottom:1px solid var(--rule);padding:.28rem .5rem;
text-align:left;vertical-align:top}
table.dat th{color:var(--sub);font-weight:600}
.foot{color:var(--sub);font-size:.78rem;margin-top:2.5rem;border-top:1px solid var(--rule);
padding-top:.8rem}
.tag{font-size:.68rem;padding:.05rem .35rem;border-radius:3px;border:1px solid var(--rule);
color:var(--sub);white-space:nowrap}
.tag.ok{color:var(--ok);border-color:var(--ok)}
.tag.unrun{color:var(--c2);border-color:var(--c2)}
.nav{font-size:.82rem;color:var(--sub);margin:.2rem 0 1.2rem}
.nav a{margin-right:.6rem}
.nav b{color:var(--fg)}
.scroll{overflow-x:auto}
"""
# spec A's own furniture: the grid, the small multiples, the strip
CSS_A = """
.grid{border-collapse:collapse;margin:.4rem 0 .2rem;user-select:none}
.grid td{width:2.05rem;height:2.05rem;padding:0;text-align:center;position:relative;
border:1px solid transparent;cursor:pointer;line-height:1}
.grid td:hover{border-color:var(--c1)}
.grid td.on{border-color:var(--c1);background:var(--panel)}
.grid .to{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:.95rem}
.grid .from{position:absolute;left:.12rem;top:.02rem;font-size:.52rem;color:var(--sub);
opacity:.55;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}
.dim .grid td:not(.on) .to{opacity:.12!important}
.dim .tok:not(.on){opacity:.15}
.tok{display:flex;align-items:center;gap:.4rem;font-size:.78rem;padding:.06rem .2rem;
cursor:pointer;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}
.tok:hover{background:var(--panel)}
.tok.on{background:var(--panel);outline:1px solid var(--c1)}
.tok .bar{height:.42rem;background:currentColor;opacity:.75;flex:none}
.tok .lab{white-space:pre;flex:none}
.toks{columns:3;column-gap:1.4rem;max-height:26rem;overflow-y:auto;
border:1px solid var(--rule);border-radius:4px;padding:.4rem}
.strip{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:.78rem;
white-space:pre;overflow-x:auto;border:1px solid var(--rule);border-radius:4px;
padding:.5rem;background:var(--panel);line-height:1.5}
.strip i{font-style:normal;opacity:.22}
.strip b{font-weight:600;background:var(--c1);color:var(--bg);border-radius:2px}
.strip u{text-decoration:none;background:var(--c2);color:var(--bg);border-radius:2px;
opacity:.85}
"""
CSS_B = """
svg{display:block;max-width:100%;height:auto;margin:.6rem 0}
svg .base{stroke:var(--fg);stroke-width:1;opacity:.5}
svg .tick{stroke:var(--fg);stroke-width:1;opacity:.35}
svg .arc{fill:none;stroke:var(--fg);stroke-width:.6;opacity:.55}
svg .arc2{fill:none;stroke:var(--fg);stroke-width:.6;opacity:.28}
svg text{fill:var(--sub);font-size:9px;
font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}
@media print{body{background:#fff;color:#000}
svg .base,svg .tick,svg .arc,svg .arc2{stroke:#000}}
"""
CSS_C = """
svg{display:block;max-width:100%;height:auto;margin:.6rem 0}
svg .flat{fill:none;stroke:var(--sub);stroke-width:.6;opacity:.28}
svg .flip{fill:none;stroke:var(--c2);stroke-width:1.1;opacity:.9}
svg .col{stroke:var(--fg);stroke-width:1;opacity:.35}
svg .sup{fill:none;stroke:var(--c1);stroke-width:.7;opacity:.35}
svg text{fill:var(--fg);font-size:9px;
font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}
svg text.s{fill:var(--sub);font-size:8px}
@media print{body{background:#fff;color:#000}}
"""
def esc(s):
return (str(s).replace("&", "&").replace("<", "<").replace(">", ">"))
def glyph(b):
"""A byte as itself where it can be, and as its hex otherwise. 'bytes label
themselves' is spec A's whole legend, so this is the only naming convention any of
the three pages uses."""
if 33 <= b < 127:
return esc(chr(b))
if b == 32:
return "·"
return "•"
def bname(b):
if 33 <= b < 127:
return esc(chr(b))
if b == 32:
return "SP"
return "%02X" % b
def head(title, css, extra=""):
return ("\n\n\n"
"\n"
"%s\n\n\n
\n"
% (esc(title), CSS, css)) + extra
def nav(which, variant, sample):
"""Every page names the other two and the other rungs of the ladder, because a
reader who wants "the same thing one size up" should not have to guess a URL."""
h = ['
the three pattern pages · ']
for key, lab in (("a", "A inventory"), ("b", "B topology"), ("c", "C change")):
if key == "c":
h.append('C change ' if which == "c"
else 'C change ')
continue
tgt = "%s-%s-%s.html" % (key, variant.replace("wordsv2-", ""), sample or "e64")
h.append(('%s ' % lab) if which == key else
('%s ' % (tgt, lab)))
if which in ("a", "b"):
h.append('· ')
for s, _, lab in SAMPLES:
t = "%s-%s-%s.html" % (which, variant.replace("wordsv2-", ""), s)
h.append(('%s ' % s) if s == sample else
('%s ' % (t, s)))
h.append('· ')
for v in (B1, B2):
t = "%s-%s-%s.html" % (which, v.replace("wordsv2-", ""), sample)
lab = v.replace("wordsv2-", "") + (" (B1)" if v == B1 else " (B2)")
h.append(('%s ' % lab) if v == variant else
('%s ' % (t, lab)))
h.append('
')
return "".join(h)
FOOT = ('
Published under '
'#viz_standard. Self-contained: no external stylesheet, no '
'font, no image, nothing fetched. Built from the spec in '
'p8v2-pattern-viz-specs.md by '
'docs/pprog/build-p8v2-patterns, never hand-edited. Every pattern '
'drawn was reconstructed through the query layer from (program text, model '
'file); no page reads a de-optimized dump. Nothing here is a result and '
'no page carries a chart point — the picks are the programmer\'s '
'(#variant_protocol).
\n'
'
\n')
# ================================================================ SPEC A
def page_a(d, other):
"""The pattern page: the inventory, laid out in data space.
THE WHOLE k=1 MAP IS THE GRID. Position in the 16x16 is the SOURCE byte -- high
nibble down, low nibble across -- and what a cell prints is that source's argmax
SUCCESSOR, as the datum itself. Ink weight is the stored support. So a B1 page and
a B2 page are the same page, and the difference between them is what B1 discards:
B1 stores 256 bytes of argmax and no weight at all, so the only ink its file can
justify is the bare glyph and every cell is at the same stated strength.
NO AXES AND NO LEGEND, per the spec. The source byte labels itself in the corner of
its own cell; nothing here needs a caption to decode.
"""
v, smp = d["variant"], d["sample"]
sup, k1, toks = d["sup1"], d["k1"], d["toks"]
stored = d["stored"]
smax = max([1] + sup + [t[3] for t in toks])
k1s = d["k1_strength"]
def ink(s):
# INK WEIGHT IS THE STORED SUPPORT, on one scale shared with the k=2 bars. A
# support of 0 is ONE observation and not none (LSA.md), so the floor is visible
# rather than invisible.
#
# UNDER B1 THERE IS NOTHING TO SCALE. The file stores no k=1 weight, so every
# cell is at one flat weight and that flatness IS the page's subject -- it is
# set here explicitly rather than falling out of a stated constant that happens
# to exceed this model's smax, which is what it would do at every sample.
if not stored:
return 1.0
return 0.30 + 0.70 * (s / smax if smax else 0)
h = [head("p8v2 patterns A — %s at %s" % (v, smp), CSS_A)]
h.append("
Everything the model holds, laid out in data space. '
'Position in the grid is the source byte (high nibble down, low '
'nibble across, named in the corner of its own cell); what the cell '
'prints is that source\'s argmax successor, as the datum itself. '
'Ink is the stored support. Click any pattern — a cell or a '
'k=2 rule — and everything else dims to the positions in the sample '
'where it fired, shown in place in the actual text.
')
if stored:
h.append('
This is the B2 page: the support is in the '
'file.%s stores the whole 65536-byte LSA matrix over '
'ordered byte pairs, so every cell here is inked at the weight the '
'model actually learned. The B1 page is the '
'same model — the same 256 argmax bytes, the same %d k=2 rules, '
'byte for byte — rendered from a file that does not carry the '
'weights. The visual difference between the two pages is what B1 '
'discards.
This is the B1 page, and the flat ink is the '
'point.%s keeps p8v2\'s 256-byte forward argmax table '
'and nothing else: the k=1 section stores which byte follows '
'and no weight at all. So the only ink this file can justify is the '
'bare glyph, and every cell is drawn at the same strength — the '
'stated constant %d from '
'axes.json, not a learned value '
'(#f-p8-forward). Put the '
'B2 page beside it: same model, same %d k=2 '
'rules, and the difference in the grid is exactly what B1 discards.
")
for lo in range(16):
a = hi * 16 + lo
s = sup[a] if stored else k1s
h.append('
%s'
'%s
'
% (a, bname(a), bname(k1[a]),
("stored support %d of %d" % (sup[a], smax)) if stored
else ("stated strength %d — nothing is stored" % k1s),
glyph(a), ink(s), glyph(k1[a])))
h.append("
")
h.append("
")
# ---- the k=2 small multiples, on the same scale
h.append('
The k=2 rules, in support order
')
h.append('
%d kept token rules, one line each: the token\'s bytes, an '
'arrow, the consequent, and a length-true bar in LSA units on the same '
'scale as the grid\'s ink — so strength is comparable across k by '
'eye. Unlike k=1, the k=2 rules carry their strength in the file under B1 '
'and B2 alike.
')
for i in order:
a, b, c, w = toks[i]
h.append('
%s%s→%s'
''
'%d
'
% (a, b, glyph(a), glyph(b), glyph(c),
6 + 94 * (w / smax if smax else 0), w))
h.append('
')
# ---- the footprint strip
h.append('
Where it fired
')
h.append('
Nothing selected. Click a cell or a k=2 rule '
'above.
')
h.append('')
h.append('
The whole sample, in place. Highlighted where the selected '
'pattern\'s antecedent occurs, and in the second colour where the byte that '
'followed is the one the rule names. One rule and its footprint in the '
'data are a single reading here, not two charts. This scan is the page\'s '
'own, and it is proved against cmpr --latd at build time — '
'%s.
' % d["latd_note"])
h.append('
')
h.append('
What is on this page, and what is not
')
h.append('
'
'
the datum
what it is
'
'
a grid cell
the k=1 pattern out of that source '
'byte: its argmax successor. There are always exactly 256, one per byte, '
'whether or not the byte ever occurs in the sample.
'
'
the ink
%s
'
'
a k=2 line
a kept token rule: the two-byte '
'context, the successor it names, and its stored LSA weight.
'
'
the bar
that weight, on the same scale as the '
'ink. The reference is the largest weight in this model, which is %d.
'
'
the highlight
the positions the pattern was '
'learned from — its LATD expansion, which is the pattern\'s own '
'definition read back and not a debugging view.
'
'
'
% (('the stored LSA support of that rule, which is a log count with a '
'stochastic, uncorrelated error (LSA.md) — '
'not a count. The counts are in the strip.') if stored else
('nothing. Every cell is at the stated constant %d, because a B1 model '
'stores no k=1 weight; the flatness is the fact.' % k1s), smax))
h.append('
No page here carries a rate or a chart point. '
'A per-byte rate over a whole model file is forbidden '
'(#hutter_metrics) and k is '
'measured end to end at a real size, which these prefixes are not '
'(#viz_standard). What is here '
'is an inventory.
')
h.append("\n" % A_JS)
h.append('\n'
% json.dumps({"sample": d["sample_bytes"], "k1": k1,
"toks": toks, "sup": sup, "stored": stored,
"k1s": k1s, "smax": smax}, separators=(",", ":")))
h.append('\n')
h.append(FOOT)
return "".join(h)
A_JS = r"""
function pr(b){ return (b>=33&&b<127)?String.fromCharCode(b):(b===32?"·":"•"); }
function nm(b){ return (b>=33&&b<127)?String.fromCharCode(b):(b===32?"SP":
("0x"+(b<16?"0":"")+b.toString(16).toUpperCase())); }
function esc(s){ return String(s).replace(/&/g,"&").replace(/" + PD.sup[a] + " of " + PD.smax
: "stated strength " + PD.k1s + ", nothing stored")
: "stored weight " + tokW(a, b) + " of " + PD.smax;
var name = kind === "k1"
? "" + esc(nm(a)) + " → " + esc(nm(to)) + "(k=1)"
: "" + esc(nm(a)) + esc(nm(b)) + " → " + esc(nm(to)) +
"(k=2)";
if (!occ.length)
return name + " — " + w + ". It never fires in this sample. The k=1 " +
"family has one pattern per byte value whether or not the byte occurs; that is " +
"what an argmax table is.";
var obs = top.slice(0, 5).map(function(x){
return esc(nm(x)) + " " + dist[x] + (x === to ? " ← the rule" : ""); }).join(", ");
var wrong = top.length && top[0] !== to;
return name + " — " + w + ". Fires at " + occ.length + " position" +
(occ.length === 1 ? "" : "s") + " here; the byte that actually followed was " + obs +
(top.length > 5 ? ", …" : "") + ". " +
(wrong
? "The rule names the less frequent successor, in the only data there is. " +
"A stored weight is a log count with a stochastic, uncorrelated error " +
"(LSA.md), so the estimate can invert the order of two " +
"small counts — the property LSA is designed around, seen from the data side."
: "The rule names the most frequent successor here.");
}
/* THE SAMPLE, IN PLACE. Not an extract: the strip is the whole text, dimmed away from
the pattern's support set, so the footprint is read against everything it is not. */
function strip(occ, to){
var S = PD.sample, M = S.length, mark = null, i, out = [];
if (occ){
mark = new Uint8Array(M);
occ.forEach(function(o){
for (var j = 0; j < o[1]; j++) mark[o[0]+j] = 1;
if (o[0]+o[1] < M) mark[o[0]+o[1]] = (o[2] === to) ? 2 : 3;
});
}
var run = -1, buf = "";
function flush(){
if (!buf) return;
var t = esc(buf);
out.push(run === 0 ? "" + t + "" : run === 1 ? "" + t + ""
: run === 2 ? "" + t + "" : run === 3 ? "" + t + "" : t);
buf = "";
}
for (i = 0; i < M; i++){
var k = mark ? mark[i] : -1;
if (k !== run){ flush(); run = k; }
buf += pr(S[i]);
}
flush();
return out.join("");
}
"""
# ================================================================ SPEC B
#
# THE ONE VISUAL VARIABLE. Every arc is the same hairline; support is encoded as ARC
# HEIGHT ALONE. So height is NOT a function of span -- the arcs are quadratic Beziers
# whose control point is set by support and nothing else, which is what lets a reader
# take height off the page as one quantity. A semicircle would encode span in the
# height and there would be two variables pretending to be one.
BW, BH, BPAD = 1100.0, 300.0, 26.0
def arc(x0, x1, s, smax, up, h0):
"""One hairline from tick x0 to tick x1, rising with support and NOTHING ELSE.
The apex of a quadratic Bezier is (P0 + 2C + P2) / 4, so putting the control point
at twice the wanted offset makes the apex land exactly on it: height is the support
and is independent of the span. A semicircle would put the span into the height and
the page would have two visual variables pretending to be one.
s is passed as None where the model stores no weight -- a B1 k=1 rule -- and every
such arc is drawn at full height. Scaling a STATED constant against this model's
smax is meaningless (8 against an smax of 2 is a ratio of 4, which used to run the
arcs off the top of the viewbox), and the flatness is the fact the page is showing.
"""
r = 1.0 if s is None else ((s / smax) if smax else 0.0)
hgt = 6.0 + (h0 - 6.0) * r
y = BPAD + (- hgt if up else hgt)
return 'M%.1f %.1f Q%.1f %.1f %.1f %.1f' % (x0, BPAD, (x0 + x1) / 2.0, BPAD + 2 * (y - BPAD), x1, BPAD)
def page_b(d):
"""The transition arc: the topology.
It answers a question the inventory cannot: what SHAPE does the learned relation
have? The 256 byte values are unlabelled ticks on one baseline in byte order, and
every k=1 rule is an arc from its source tick to its successor tick, above the line
for a forward step in byte order and below for a backward one. Read that way the
hubs, the short arcs of the alphabet's local runs, the long arcs into punctuation
and the high bytes, and the CYCLES -- pairs whose arcs mirror each other above and
below, which are exactly the positions where settling can oscillate rather than
converge -- are immediate and are unavailable in any table.
"""
v, smp = d["variant"], d["sample"]
k1, toks, sup, stored = d["k1"], d["toks"], d["sup1"], d["stored"]
smax = max([1] + sup + [t[3] for t in toks])
k1s = d["k1_strength"]
def X(b):
return BPAD + (BW - 2 * BPAD) * b / 255.0
# --- the k=1 baseline
up, dn, cyc = [], [], []
back = {}
for a in range(256):
back.setdefault(k1[a], []).append(a)
for a in range(256):
b = k1[a]
s = sup[a] if stored else None
(up if b >= a else dn).append(arc(X(a), X(b), s, smax, b >= a, BH / 2 - BPAD))
if k1[b] == a and a < b:
cyc.append((a, b))
hubs = sorted(back.items(), key=lambda kv: -len(kv[1]))[:6]
g = ['')
# --- the k=2 baseline, one order up
#
# ITS TICKS ARE THE 65536 CONTEXTS IN LEXICOGRAPHIC ORDER, and an arc goes from the
# context a rule fires in, (a,b), to the context the step LANDS IN, (b,c). That is
# the k=2 order's own transition structure, so the two baselines are the same
# construction at two orders and can be compared as two skylines without either
# being aggregated into the other.
def X2(c):
return BPAD + (BW - 2 * BPAD) * c / 65535.0
g2 = ['')
h = [head("p8v2 patterns B — %s at %s" % (v, smp), CSS_B)]
h.append("
The shape of the learned relation. The 256 byte values are '
'ticks on one baseline in byte order. Every k=1 rule is an arc from its '
'source tick to its successor tick, above the line for a forward step '
'in byte order and below for a backward one. Every arc is the same '
'hairline: support is arc height and nothing else, so this page has '
'exactly one visual variable and height can be read off it as one quantity. '
'Nothing is selectable and nothing moves.
')
h.append("".join(g))
h.append('
%d forward arcs above, %d backward below.%s
'
% (len(up), len(dn),
('' if stored else
' Under B1 every arc is the same height, because the file stores '
'no k=1 weight at all — the height here is the stated constant '
'%d. Put the B2 page beside it and the '
'skyline appears.' % (k1s, B2.replace("wordsv2-", ""), smp))))
h.append("
What the shape says
")
h.append('
read for
what it is, and where
')
h.append('
hubs
ticks many arcs land on. Here: %s. These are '
'the bytes the model expects after most things, and they are why a window '
'can settle to one answer from several directions.
'
% ", ".join("%s (%d arcs in)" % (bname(b), len(srcs))
for b, srcs in hubs if srcs))
h.append('
short arcs
the alphabet\'s local runs — a '
'letter naming a neighbouring letter. They sit low and tight against the '
'baseline near the printable range.
')
h.append('
long arcs
rules that cross the whole range, into '
'punctuation and the high bytes. A long arc from a high byte is usually a '
'byte that never occurs in the sample: the argmax table has an entry for '
'every byte value whether or not it was ever seen.
')
h.append('
cycles
%d pairs whose arcs mirror each '
'other above and below the line: %s. These are exactly the positions '
'where settling can oscillate rather than converge — each byte '
'names the other, so a forward application and a backward one can push a '
'position back and forth forever. That is what the conv column '
'of gen5.tsv is about when it reads W.
'
% (len(cyc), ", ".join("%s↔%s" % (bname(a), bname(b))
for a, b in cyc[:14]) + (", …" if len(cyc) > 14 else "")
if cyc else "none at this sample"))
h.append('
')
h.append("
The k=2 rules, one order up
")
h.append('
A second, fainter baseline whose ticks are the 65536 '
'two-byte contexts in lexicographic order. An arc runs from the context '
'a rule fires in, (a, b), to the context the step lands in, '
'(b, c) — the same construction as above, one order up, so the '
'two orders of the model can be compared as two skylines without either '
'being aggregated into the other. %d rules are kept; 65536 − %d '
'contexts carry none.
The k=2 rules DO carry their strength in the file, under B1 '
'and B2 alike, so this skyline is the same on both pages. Where it clusters '
'is where the sample\'s two-byte contexts are: the printable band, and '
'nowhere else.
')
h.append('
What is not encoded here. No colour, no line '
'weight, no label, no axis and no aggregate. Height is support and position '
'is byte order; there is nothing else to decode. A support of 0 is one '
'observation and not none (LSA.md), so the lowest '
'arcs are still real rules. Nothing on this page is a rate or a chart point '
'(#hutter_metrics).
')
h.append(FOOT)
return "".join(h)
# ================================================================ SPEC C
CW, CH = 1080.0, 620.0
def page_c(d):
"""The learning slopegraph: what CHANGED as the sample grew.
Neither an inventory nor a structure but a DIFFERENCE. Three columns, one line per
source byte joining its argmax successor at e64, e1k and e10k. Most lines are flat
and grey and carry no information; the page exists for the ones that are not, and
each flip is labelled with both successors and the counts behind them, so the reader
sees whether the model corrected itself or was pushed by LSA's stochastic increment.
"""
per = d["per"] # sample key -> {k1, sup1, toks, cnt, share, M}
keys = [s for s, _, _ in SAMPLES]
labs = {s: lab for s, _, lab in SAMPLES}
def X(i):
return 150.0 + i * (CW - 300.0) / (len(keys) - 1)
def Y(b):
return 40.0 + (CH - 90.0) * b / 255.0
flips, flat = [], 0
for a in range(256):
seq = [per[s]["k1"][a] for s in keys]
if len(set(seq)) == 1:
flat += 1
else:
flips.append((a, seq))
g = ['')
# --- panel 2: support against sample size, on a shared axis
P2H = 300.0
smaxall = max(1, max(max(per[s]["sup1"]) for s in keys))
def X2(i):
return 150.0 + i * (CW - 300.0) / (len(keys) - 1)
def Y2(w):
return P2H - 40.0 - (P2H - 80.0) * w / smaxall
g2 = ['')
v = d["variant"]
h = [head("p8v2 patterns C — what the model learned", CSS_C)]
h.append("
The learning slopegraph — what changed as the sample grew
")
h.append(nav("c", v, None))
h.append('
Neither an inventory nor a structure but a difference: '
'what the model learned at 64, 1,000 and 10,000 bytes, as three columns with '
'one line per source byte joining its argmax successor at each size. '
'Vertical position is the successor\'s byte value, so a flat line is a '
'rule that did not change. %d of the 256 are flat and grey and carry no '
'information; this page exists for the other %d.
Every rule that changed its mind, and what the data actually '
'says at each size. The counts are counts; the stored weight beside '
'them is a log count with a stochastic, uncorrelated error '
'(LSA.md), so the two are not required to agree. Where '
'the rule names a successor the counts do not, the estimate inverted the '
'order of two small counts — the property LSA is designed around, '
'seen from the data side.
')
h.append('
source
')
for s in keys:
h.append('
%s — the rule
what the data says
' % labs[s])
h.append('
')
for a, seq in sorted(flips):
h.append('
%s
' % bname(a))
for i, s in enumerate(keys):
cnt, k1 = per[s]["cnt"], per[s]["k1"]
row = sorted(((c, b) for (k, c) in cnt.items() if (k >> 8) == a
for b in [k & 255]), reverse=True)
tot = sum(c for c, _ in row)
best = row[0][1] if row else None
h.append('
%s w %d
'
% (bname(k1[a]), per[s]["sup1"][a]))
h.append('
%s
'
% (("".join("%s %d " % (bname(b), c) for c, b in row[:4])
+ ("…" if len(row) > 4 else "")
+ ("" if best == k1[a] else
' rule ≠ data'))
if tot else 'never occurs'))
h.append('
')
h.append('
')
h.append("
Support against sample size
")
h.append('
Each of the 256 k=1 rules\' stored support at each size, on '
'one shared axis in LSA units — which are already a log scale, so this '
'is a log axis by construction and not by choice. Read underneath it the '
'number the whole generation turns on: the argmax\'s share of its '
'context, over every context that occurs.
')
h.append("".join(g2))
h.append('
The share falls: %s. That is the generation\'s '
'central measured fact about the k=1 strength, and it decides the axis it '
'is on. A stated constant delivery is right at the small samples, '
'where the argmax carries about half its context\'s mass and 8 is the weight '
'that says so; at 10,000 bytes the same contexts have recurred with '
'different successors, the share is down to %.3f, and the stored '
'matrix already knows better than any constant '
'(#f-p8-forward). It is the same direction '
'#viz_standard records for the trace rate: a '
'model learned from the bytes it is measured on has memorised a small '
'prefix, and every measure of its sharpness is optimistic there.
'
% (", ".join("%.3f at %s" % (per[s]["share"], labs[s]) for s in keys),
per[keys[-1]]["share"]))
h.append('
One page, not eight, and here is why that is honest. '
'All eight generation-5 variants carry one learned model, byte for byte '
'— axes B, G and H change how a model is read and settled, never how it '
'is learned — and the builder refuses to build unless that identity '
'holds. So there is one thing to draw here. The stored supports are read from '
'%s, the only variant in the generation that serializes the count '
'matrix; a B1 model would leave the second panel empty, which is what the B1 '
'and B2 inventory pages are for.
'
% (esc(B2), B1.replace("wordsv2-", "")))
h.append(FOOT)
return "".join(h)
# ================================================================ the front page
def page_index(meta):
h = [head("p8v2 — seeing the patterns", "")]
h.append("
Seeing the patterns
")
h.append('
Three pages over the same p8v2 models, each built on a '
'different organizing principle, so that the three disagree about what a '
'pattern is. They share no encoding, no interaction and no layout. The '
'spec is p8v2-pattern-viz-specs.md, '
'written on the generating side; these are the builds.
')
h.append('
What a pattern is here. A p8v2 model holds two '
'families. The k=1 family is 256 rules, one per byte value, each '
'naming the byte the model expects to follow it; under B1 the file stores '
'the argmax and no weight, and under B2 it stores the whole 65536 '
'-entry LSA matrix over ordered byte pairs. The k=2 family is the '
'kept two-byte token rules, which carry their strength in the file either '
'way. Every pattern on these pages was reconstructed through the query layer '
'from (program text, model file) — never a de-optimized dump.
')
h.append("
A — the pattern page: the inventory, in data space
")
h.append('
One page per (model, sample). The whole k=1 map as a 16×16 '
'grid of source bytes, each cell printing its argmax successor as the datum '
'itself with ink weight proportional to the stored support; the k=2 rules '
'below as a support-ordered column of small multiples on the same scale. '
'Selecting any pattern dims everything except the positions in the sample '
'where it fired, shown in place in the actual text. B1 and B2 render as '
'the same page, and the visual difference between them is what B1 '
'discards. The only interactive one.
')
h.append(_grid_links("a", meta))
h.append("
B — the transition arc: the topology
")
h.append('
One page per (model, sample), answering a question A cannot: '
'what shape does the learned relation have? The 256 byte values as '
'unlabelled ticks on one baseline, every k=1 rule an arc from source to '
'successor, above the line for a forward step and below for a backward one; '
'support is arc height alone, every arc the same hairline, so the '
'page has exactly one visual variable. The hubs, the alphabet\'s local runs, '
'the long arcs into the high bytes and the cycles — the pairs '
'where settling can oscillate rather than converge — are immediate and '
'are unavailable in any table. Nothing is selectable: a page to print.
')
h.append(_grid_links("b", meta))
h.append("
C — the learning slopegraph: the change
")
h.append('
Neither an inventory nor a structure but a difference: '
'what the model learned at 64, 1,000 and 10,000 bytes, as three columns of a '
'slopegraph with one line per source byte joining its argmax successor at '
'each size. Most lines are flat and grey; the page exists for the argmax '
'flips, each labelled with both successors and the counts behind them. A '
'second panel puts each rule\'s support against sample size, under which sits '
'the number the generation turns on: the argmax\'s share of its context, '
'falling as the sample grows.
')
h.append('
The learning slopegraph — one page: '
'all eight generation-5 variants share one learned model byte for byte, and '
'the builder refuses to build unless that holds.
')
h.append("
What none of the three do
")
h.append('
None aggregates a rate into a single number, none shows a '
'per-byte compression rate over a whole model file '
'(#hutter_metrics forbids it), '
'and none invents a strength the model does not store: under B1 the k=1 '
'strength is the stated constant from axes.json and '
'is labelled as stated, not as learned. Nothing here is a result and no '
'page carries a chart point.
what an LSA value is: a log count with a stochastic, uncorrelated '
'error — the reason a stored weight and a concordance count are '
'allowed to disagree
')
h.append(FOOT)
return "".join(h)
def _grid_links(which, meta):
h = ['
']
for s, _, lab in SAMPLES:
h.append("
%s bytes
" % lab)
h.append("
")
for v in (B1, B2):
h.append('
%s %s
'
% (v.replace("wordsv2-", ""),
"(B1 — no stored k=1 weight)" if v == B1
else "(B2 — the count matrix is in the file)"))
for s, _, lab in SAMPLES:
k = (v, s)
h.append('
")
return "".join(h)
# ================================================================ main
def main(argv):
cmpr_src = os.path.abspath(os.path.join(HERE, "..", "..", "..", "cmpr-src"))
if "--cmpr-src" in argv:
cmpr_src = os.path.abspath(argv[argv.index("--cmpr-src") + 1])
no_latd = "--no-latd" in argv
pprog = os.path.join(cmpr_src, "tests", "pprog")
if not os.path.isdir(os.path.join(pprog, "models", "p8v2", "enwik9")):
die("no models under %s -- run acceptance-p8v2 in cmpr-src first" % pprog)
os.makedirs(OUT, exist_ok=True)
rel = os.path.relpath(pprog, OUT)
relroot = os.path.relpath(cmpr_src, OUT)
for base, relbase, links, need in ((pprog, rel, LINKS, True),
(cmpr_src, relroot, ROOT_LINKS, True)):
for name, target in links:
link = os.path.join(OUT, name)
if not os.path.exists(os.path.join(base, target)):
if need:
die("no %s to publish beside the pages" % os.path.join(base, target))
continue
if os.path.islink(link) or os.path.exists(link):
os.remove(link)
os.symlink(os.path.join(relbase, target), link)
axes_doc = json.load(open(os.path.join(pprog, "axes.json")))
cst = ((axes_doc.get("constants") or {}).get("per_alternative") or {})
if "k1_strength" not in (cst.get("f-p8-forward") or {}):
die("axes.json carries no per_alternative['f-p8-forward'].k1_strength. The B1 "
"pages state that constant and will not invent it; regenerate axes.json "
"with cmpr-src's tests/pprog/p8v2-axes.py.")
k1s = cst["f-p8-forward"]["k1_strength"]
cmpr = os.environ.get("CMPR") or os.path.join(cmpr_src, "cmpr", "dist", "cmpr")
if not os.access(cmpr, os.X_OK):
cmpr = "cmpr"
data, meta, learned = {}, {}, {}
for v in (B1, B2):
for s, mdir_name, lab in SAMPLES:
mp = os.path.join(pprog, "models", "p8v2", "enwik9", mdir_name, v + ".m")
if not os.path.exists(mp):
die("no %s -- the pages need both variants at every sample" % mp)
mod = read_model(mp)
sample = read_pos(os.path.join(pprog, POSDIR, "%s.%s.pos" % (v, s)))
if len(sample) != mod["M"]:
die("%s dumps %d positions but its model header says M = %d"
% (v, len(sample), mod["M"]))
sn = sn_patterns(cmpr, cmpr_src, mp, mod)
if any(w != k1s for w in sn["k1w"]):
die("the query layer reports k=1 strengths %r, but axes.json states "
"k1_strength = %d. The cmpr being run predates the generation-4 "
"extirpation; point CMPR at cmpr-src's dist build."
% (sorted(set(sn["k1w"])), k1s))
# ONE LEARNED MODEL PER SAMPLE, checked rather than assumed: axes B, G and H
# change how a model is read and settled, never how it is learned, so B1 and
# B2 must agree byte for byte on the argmax table and the token section.
key = (mod["k1"], sorted(mod["toks"]))
if learned.setdefault(s, key) != key:
die("%s at %s carries a different learned model from its pair; the pages' "
"whole claim is that B1 and B2 are the same model rendered from "
"files that store different amounts of it" % (v, s))
latd = ({"checked": 0, "skipped": 0, "of": 0, "model": ""} if no_latd else
check_latd(cmpr, cmpr_src, mp, mod, sn, sample,
303 if s == "e64" else 64))
cnt = true_counts(sample)
d = {"variant": v, "sample": s, "sample_bytes": sample, "k1": mod["k1"],
"toks": mod["toks"], "stored": mod["lpp"] is not None,
"k1_strength": k1s, "cnt": cnt, "share": argmax_share(cnt),
"M": mod["M"], "axes": mod["axes"]}
# THE SUPPORT IS ALWAYS THE LEARNED ONE, and it always comes from the model
# that stores it. A B1 page does not draw it -- that is the page's whole
# subject -- so this is filled in after both variants of a sample are read.
d["lpp"] = mod["lpp"]
data[(v, s)] = d
meta[(v, s)] = {"ntok": len(mod["toks"])}
latd_of = latd["of"]
d["latd"] = latd
d["latd_note"] = (
"the check was skipped (--no-latd)" if no_latd else
("all %d patterns of this model agree" % latd["checked"]
if latd["checked"] >= latd_of else
"%d of this model's %d patterns were checked and all agree; the query "
"is a process spawn per pattern, so the check is capped above e64"
% (latd["checked"], latd_of)))
for s, _, _ in SAMPLES:
lpp = data[(B2, s)]["lpp"]
if not lpp:
die("%s at %s stores no count matrix; the pages have no learned support to "
"draw" % (B2, s))
for v in (B1, B2):
d = data[(v, s)]
d["sup1"] = [lpp[x * 256 + d["k1"][x]] for x in range(256)]
for (v, s), d in sorted(data.items()):
open(os.path.join(OUT, "a-%s-%s.html" % (v.replace("wordsv2-", ""), s)),
"w").write(page_a(d, data[(B2 if v == B1 else B1, s)]))
open(os.path.join(OUT, "b-%s-%s.html" % (v.replace("wordsv2-", ""), s)),
"w").write(page_b(d))
cd = dict(data[(B2, "e64")])
cd["per"] = {s: {"k1": data[(B2, s)]["k1"], "sup1": data[(B2, s)]["sup1"],
"cnt": data[(B2, s)]["cnt"], "share": data[(B2, s)]["share"],
"M": data[(B2, s)]["M"]}
for s, _, _ in SAMPLES}
open(os.path.join(OUT, "c.html"), "w").write(page_c(cd))
open(os.path.join(OUT, "index.html"), "w").write(page_index(meta))
for b, cwd in ([(x, cmpr_src) for x in SRC_BLOCKS] +
[(x, os.path.join(HERE, "..", "..")) for x in BLOCKS]):
p = subprocess.run([cmpr, "--print-block", "#" + b], cwd=cwd,
capture_output=True, text=True)
if p.returncode == 0 and p.stdout.strip():
open(os.path.join(OUT, b + ".txt"), "w").write(p.stdout)
else:
sys.stderr.write("build-p8v2-patterns: could not dump #%s\n" % b)
sys.stderr.write("build-p8v2-patterns: wrote %s (%d pages: A and B for %s and %s at "
"%s, C once; k=1 strength %d; LATD %s)\n"
% (OUT, 2 * len(data) + 2, B1, B2,
"/".join(s for s, _, _ in SAMPLES), k1s,
"not checked" if no_latd else
"checked on %d patterns" % sum(d["latd"]["checked"]
for d in data.values())))
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv))