#!/bin/bash
# Acceptance test for P8v2 / wordsv2 -- the k=2 token model above the p7 byte-level surprise trace --
# run as a GENERATION OF VARIANTS under #variant_protocol, via the REAL strategy -> --rewritepl ->
# compile chain:
#
#   cmpr --run wordsv2-v001 --ofra tests/pprog/p8v2-words.md <N> --compress <model> < in
#   cmpr --run wordsv2-v001 --ofra tests/pprog/p8v2-words.md      --decompress <model> > out
#
# Generation 1 is the baseline #wordsv2-v001 (axes A1 B1 C1 D1 E1 F1) plus one variant per
# non-baseline alternative on the six axes, eleven runs in all.
#
# Generation 2 (#pprog_p8v2_gen2_goal_20260806) is #variant_protocol's ADVANCE step: the baseline
# moves to #wordsv2-v002 (A2, the generation-1 pick) and every surviving non-baseline alternative is
# re-run off it -- v002, v003 and v013..v020, ten runs. Select it with:
#
#   P8V2_GEN=2 tests/pprog/acceptance-p8v2
#   P8V2_GEN=3 tests/pprog/acceptance-p8v2   (generation 3: the baseline and the a5 family,
#                                             #wordsv2-v021/-v022/-v023 = A4/A5/A6)
#   P8V2_GEN=4 tests/pprog/acceptance-p8v2   (generation 4: the f-p8-cap8 pair -- baseline
#                                             #wordsv2-v024 = A7, #wordsv2-v025 = A7 B2, plus
#                                             the v002 and v013 bridge runs)
#
#   P8V2_GEN=5 tests/pprog/acceptance-p8v2   (generation 5: the f SPLIT. Delivery held at the
#                                             cap-8 forward pass, decay varied on its own axis --
#                                             baseline #wordsv2-v024, ground truth #wordsv2-v025,
#                                             and #wordsv2-v026..-v031 = decay 2..7)
#
# which writes gen2.tsv and gen2-pos/ instead of gen1.tsv and gen1-pos/. Both generations write into
# the same models/ tree, because variant ids are global and never reused.
#
# THE TWO TSVs ARE ONLY COMPARABLE WITH EACH OTHER IF BOTH ARE POST-2026-08-06. The k=0 background
# was removed on that date, which changed the model format AND the LSA entropy stream (learn_k0's
# draws are gone), so every number moved. gen1.tsv must be regenerated whenever gen2.tsv is.
#
# THIS SCRIPT GENERATES; IT DOES NOT DISPLAY. Per #hutter_metrics every byte of S belongs to one of
# three size classes -- (a) fixed, (b) sub-linear and capped by the architecture, (c) linear in the
# DSS -- and mixing them into a single per-byte rate produces a number that is an artifact of the
# sample size. So the TSV this writes carries MEASURED FACTS ONLY: byte counts per class, the k=2
# layer's diagnostics, the cost. Every rate, extrapolation and ranking is derived by the display
# step, tests/pprog/gen1-report, which runs no compressions -- so a change of reporting policy costs
# a re-render and not a re-run. Rendering and publication belong to ../hutter.
#
# Like acceptance-p7 this hits the LLM on the first run for whichever blocks are new (a variant that
# moves one axis re-rolls one of the fifteen blocks and splices the other fourteen from the cache the
# baseline filled) and caches the generated PL under .cmpr/pprog/blocks/ thereafter. Needs the
# project's .cmpr/conf (model + API key). The build under test must be the `cmpr` that runs the
# parent --run (set CMPR or `sudo make install`).
#
# The model file is P8V2: a 48-byte header (magic, N, M, TC, TB, G, SC, BWD, ROUNDS as u32 LE, then
# the six axis digits) + 256 k1 table bytes + BWD backward-LPP bytes + TB token bytes + G gap bytes
# + SC trace bytes. Classes: (a) = binary + 48 + 256 + BWD, (b) = TB, (c) = G + SC.
#
# Round-trip is NOT required of every variant. Step 1 explicitly expects it to fail somewhere, and
# the TSV carries a round-trip column rather than a pass/fail gate; only the BASELINE is gated,
# because it is the causal composite rule and round-trips by construction.
#
# SAMPLES are real enwik9 prefixes, taken from $ENWIK9 (default ../hutter/enwik9). The default set
# is the cheap end -- e64, e1k, e10k -- which is a correctness run of a couple of minutes. The
# measurement ladder is opt-in because it is expensive and #hutter_run_costs says why:
#
#   P8V2_LADDER="e64 e1k e10k e100k e1m" tests/pprog/acceptance-p8v2
#
# The two settling alternatives (D2 #wordsv2-v006, D3 #wordsv2-v007) settle one full window per
# candidate recorded byte, which is quadratic in M: 40 s at 1e4, about an hour at 1e5. They are run
# only up to SLOW_MAX and every skipped row is written into the TSV with its reason, never dropped.
#
# When $ENWIK9 is absent the script falls back to repo-derived samples so it still runs, and marks
# every row so no one reads those numbers as enwik9 measurements. They are NOT: the old fallback
# t10k sample has 66 distinct 2-grams in 10000 bytes where a real enwik9 prefix has 1074.

set -e
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
CMPR="${CMPR:-cmpr}"
OFRA="$SCRIPT_DIR/p8v2-words.md"
P7OFRA="$SCRIPT_DIR/p7-hutter-order-1-markov.md"
ENWIK9="${ENWIK9:-$ROOT/../hutter/enwik9}"
cd "$ROOT"

GEN="${P8V2_GEN:-1}"
case "$GEN" in
  1) GEN_VARIANTS="wordsv2-v001 wordsv2-v002 wordsv2-v003 wordsv2-v004 wordsv2-v005 wordsv2-v006 wordsv2-v007 wordsv2-v008 wordsv2-v009 wordsv2-v010 wordsv2-v011" ;;
  2) GEN_VARIANTS="wordsv2-v002 wordsv2-v003 wordsv2-v013 wordsv2-v014 wordsv2-v015 wordsv2-v016 wordsv2-v017 wordsv2-v018 wordsv2-v019 wordsv2-v020" ;;
  3) GEN_VARIANTS="wordsv2-v002 wordsv2-v021 wordsv2-v022 wordsv2-v023" ;;
  4) GEN_VARIANTS="wordsv2-v002 wordsv2-v013 wordsv2-v024 wordsv2-v025" ;;
  5) GEN_VARIANTS="wordsv2-v024 wordsv2-v025 wordsv2-v026 wordsv2-v027 wordsv2-v028 wordsv2-v029 wordsv2-v030 wordsv2-v031" ;;
  *) echo "acceptance-p8v2: P8V2_GEN must be 1, 2, 3, 4 or 5, got '$GEN'"; exit 1 ;;
esac
VARIANTS="${P8V2_VARIANTS:-$GEN_VARIANTS}"
LADDER="${P8V2_LADDER:-e64 e1k e10k}"
# The quadratic ones are the D2/D3 vectors of both generations.
SLOW_VARIANTS="wordsv2-v006 wordsv2-v007 wordsv2-v015 wordsv2-v016"
SLOW_MAX="${P8V2_SLOW_MAX:-10000}"     # D2/D3 are quadratic in M; above this they are skipped
DUMP_MAX="${P8V2_DUMP_MAX:-10000}"     # keep .pos dumps only for samples this size or smaller
TSV="${P8V2_TSV:-$SCRIPT_DIR/gen$GEN.tsv}"
POSDIR="$SCRIPT_DIR/gen$GEN-pos"

tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
fail() { echo "FAIL: $1"; exit 1; }

u32le() { od -An -tu4 -j "$2" -N 4 "$1" | tr -d ' '; }   # u32le <file> <offset>

# ---------------------------------------------------------------- the samples
REAL=yes
if [ -r "$ENWIK9" ]; then
    head -c 64      "$ENWIK9" > "$tmp/e64.in"
    head -c 1000    "$ENWIK9" > "$tmp/e1k.in"
    head -c 10000   "$ENWIK9" > "$tmp/e10k.in"
    head -c 100000  "$ENWIK9" > "$tmp/e100k.in"
    head -c 1000000 "$ENWIK9" > "$tmp/e1m.in"
    head -c 10000000 "$ENWIK9" > "$tmp/e10m.in"
    SRC="enwik9 prefix ($ENWIK9)"
else
    REAL=no
    echo "NOTE: $ENWIK9 not readable; falling back to repo-derived samples. These are NOT enwik9"
    echo "      and their numbers must not be read as measurements of it."
    printf '%s' '<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.3/" xmln' > "$tmp/e64.in"
    head -c 1000 "$P7OFRA" > "$tmp/e1k.in"
    head -c 10000 "$P7OFRA" > "$tmp/e10k.in"
    head -c 100000 "$P7OFRA" > "$tmp/e100k.in" 2>/dev/null || cp "$tmp/e10k.in" "$tmp/e100k.in"
    cp "$tmp/e100k.in" "$tmp/e1m.in"
    cp "$tmp/e100k.in" "$tmp/e10m.in"
    SRC="FALLBACK (not enwik9)"
fi

# ---------------------------------------------------------------- run helpers
compress() {  # compress <variant> <N> <infile> <modelfile> <errfile>
    "$CMPR" --run "$1" --ofra "$OFRA" "$2" --compress "$4" < "$3" > /dev/null 2>"$5" \
        || fail "$1: compress exited nonzero: $(tail -1 "$5")"
}
decompress() {  # decompress <variant> <modelfile> <outfile> <errfile>
    "$CMPR" --run "$1" --ofra "$OFRA" --decompress "$2" > "$3" 2>"$4" \
        || fail "$1: decompress exited nonzero: $(tail -1 "$4")"
}

# ------------------------------------------------------- 1. the baseline gates
BASE=wordsv2-v001
roundtrip() {  # roundtrip <N> <infile> <label>
    compress "$BASE" "$1" "$2" "$tmp/rt.m" "$tmp/rt.err"
    decompress "$BASE" "$tmp/rt.m" "$tmp/rt.out" "$tmp/rtd.err"
    cmp -s "$2" "$tmp/rt.out" \
        || fail "$3: baseline round-trip mismatch ($(wc -c <"$2") in, $(wc -c <"$tmp/rt.out") out)"
}
roundtrip 10000 "$tmp/e10k.in" "text-10k"
head -c 10000 /dev/urandom > "$tmp/rand.in"
roundtrip 10000 "$tmp/rand.in" "random-10k"
: > "$tmp/empty.in"
roundtrip 5 "$tmp/empty.in" "empty-N5"
printf 'XYZ' > "$tmp/part.in"
roundtrip 10 "$tmp/part.in" "partial-3-N10"

# 2. a zero-size input has no class (b) and no class (c): no distinct 2-byte context exists in an
# empty sample, and there is nothing to record. (The p7 KNOWN RED invariant, restated per class.)
compress "$BASE" 5 "$tmp/empty.in" "$tmp/z.m" "$tmp/z.err"
grep -q "class (c) trace 0 bytes" "$tmp/z.err" \
    || fail "zero-size input must report zero class (c), got: $(grep -o 'class (c).*' "$tmp/z.err")"
grep -q "class (b) model 0 bytes" "$tmp/z.err" \
    || fail "zero-size input must report zero class (b), got: $(grep -o 'class (b).*' "$tmp/z.err")"
[ "$(u32le "$tmp/z.m" 12)" -eq 0 ] || fail "empty sample must keep zero tokens"

# 3. the model format, on a real sample
compress "$BASE" 10000 "$tmp/e10k.in" "$tmp/b.m" "$tmp/b.err"
magic=$(head -c 4 "$tmp/b.m")
[ "$magic" = "P8V2" ] || fail "model magic is '$magic', expected P8V2"
hN=$(u32le "$tmp/b.m" 4);  hM=$(u32le "$tmp/b.m" 8);  hTC=$(u32le "$tmp/b.m" 12)
hTB=$(u32le "$tmp/b.m" 16); hG=$(u32le "$tmp/b.m" 20); hSC=$(u32le "$tmp/b.m" 24)
hBWD=$(u32le "$tmp/b.m" 28); hR=$(u32le "$tmp/b.m" 32)
msz=$(wc -c < "$tmp/b.m")
[ "$hN" -eq 10000 ] && [ "$hM" -eq 10000 ] || fail "header N/M are $hN/$hM, expected 10000/10000"
[ "$hTB" -eq $((4 * hTC)) ] || fail "token section $hTB != 4 * TC($hTC)"
[ "$msz" -eq $((48 + 256 + hBWD + hTB + hG + hSC)) ] \
    || fail "model size $msz != 48 + 256 + BWD($hBWD) + TB($hTB) + G($hG) + SC($hSC)"
[ "$hTC" -gt 0 ] || fail "no k=2 tokens learned on a 10000-byte text sample"
[ "$hBWD" -eq 0 ] || fail "baseline is B1 and must store no backward LPP, got $hBWD"
# The class (b) cap is architectural, not a measurement: 65536 possible k=2 contexts, 4 bytes each.
[ "$hTB" -le 262144 ] || fail "token section $hTB exceeds the k=2 architectural cap 262144"

# 4. the k=2 model has to earn its place in class (c), which is the only class that survives to 1e9.
# Compared against p7 on the same bytes, whose P is entirely class (c).
"$CMPR" --run order-1-markov --ofra "$P7OFRA" 10000 --compress "$tmp/p7.m" < "$tmp/e10k.in" \
    > /dev/null 2>"$tmp/p7.err" || fail "p7 comparison run failed"
p7G=$(u32le "$tmp/p7.m" 16); p7SC=$(u32le "$tmp/p7.m" 12)
a8=$(awk "BEGIN{printf \"%.6f\", ($hG + $hSC)/$hM}")
a7=$(awk "BEGIN{printf \"%.6f\", ($p7G + $p7SC)/$hM}")
awk "BEGIN{exit !($a8 < $a7)}" \
    || fail "p8v2 class (c) rate a = $a8 is not better than p7's $a7 on the same sample"

# 5. the reporting lines, in BOTH modes -- measured facts, and no derived quantity
decompress "$BASE" "$tmp/b.m" "$tmp/b.out" "$tmp/bd.err"
for err in "$tmp/b.err" "$tmp/bd.err"; do
    grep -Eq "p8v2: class \(a\) fixed [0-9]+ bytes = binary [0-9]+ \+ header 48 \+ k1 table 256 \+ backward 0" "$err" \
        || fail "class (a) line missing/incorrect in $err"
    grep -q "p8v2: class (b) model $hTB bytes = $hTC kept k=2 rules x 4 (architectural cap 262144 = 65536 contexts x 4)" "$err" \
        || fail "class (b) line missing/incorrect in $err"
    grep -q "p8v2: class (c) trace $((hG + hSC)) bytes = gap section $hG + recorded $hSC of 10000 positions" "$err" \
        || fail "class (c) line missing/incorrect in $err"
    grep -q "no rate and no extrapolation is printed here" "$err" \
        || fail "the generating step must not print a derived quantity ($err)"
    grep -Eq "hutter: |achieved per-byte rate|extrapolated whole-enwik9" "$err" \
        && fail "a derived quantity leaked into the generating step's output ($err)"
    grep -q "structural: total ESs 20002, total atomic patterns $((2560000 + 256 + hTC + hSC)) (memchain 2560000 + k1 table 256 + backward 0 + token $hTC + top_recall $hSC), maximum pattern length 10000" "$err" \
        || fail "wordsv2 structural line missing/incorrect in $err"
    grep -q "p8v2: variant $BASE ; axes 111111 ; sample M 10000 bytes ; replay rounds $hR" "$err" \
        || fail "p8v2 variant line missing/incorrect in $err"
done
grep -q "learning: tokens $hTC kept of " "$tmp/b.err" || fail "token learning report missing"
grep -q "trace: $hSC surprising of 10000 bytes" "$tmp/b.err" || fail "trace report missing or inconsistent with header SC"
grep -Eq "settling: mean sweeps [0-9.]+ ; settled argmax correct [0-9]+ of 10000 ; pattern applications [0-9]+" "$tmp/b.err" \
    || fail "settling report missing"

# 6. a mode is required, exactly as for the trace kinds
if "$CMPR" --run "$BASE" --ofra "$OFRA" 5 < "$tmp/part.in" > /dev/null 2>"$tmp/mode.err"; then
    fail "wordsv2 program without a mode should have failed"
fi
grep -q "requires a mode" "$tmp/mode.err" || fail "missing-mode error message wrong"

# 7. QUERY LAYER. The standing discipline (#pprog_pattern_query_goal_20260706) is that every
# optimization admitted comes with its unfolding, so the k=2 rules have to be readable in SN and a
# position where one fired has to name it.
"$CMPR" --run "$BASE" --ofra "$OFRA" --patterns-from token --model "$tmp/b.m" > "$tmp/q.out" 2>"$tmp/q.err" \
    || fail "--patterns-from token failed: $(tail -1 "$tmp/q.err")"
qn=$(grep -c '^"The 2-token is ' "$tmp/q.out" || true)
[ "$qn" -eq "$hTC" ] || fail "--patterns printed $qn token rules, header says $hTC kept"
grep -Eq '^  "The input byte is .*" [0-9]+\.$' "$tmp/q.out" || fail "token rule consequent not in SN"
"$CMPR" --run "$BASE" --ofra "$OFRA" --patterns --model "$tmp/b.m" > "$tmp/qall.out" 2>/dev/null \
    || fail "bare --patterns failed"
grep -q '^"The 2-token is ' "$tmp/qall.out" || fail "bare --patterns omits the (token, byte) pair"
grep -q '^"The top-level memory event is present."' "$tmp/qall.out" || fail "bare --patterns omits top_recall"
"$CMPR" --run "$BASE" --ofra "$OFRA" --model "$tmp/b.m" --explain byte --where "The position is 0." \
    > "$tmp/x0.out" 2>/dev/null || fail "--explain at position 0 failed"
grep -q '^"The top-level memory event is present."' "$tmp/x0.out" \
    || fail "--explain at a recorded position must be the enwik9 recall"
"$CMPR" --run "$BASE" --ofra "$OFRA" --model "$tmp/b.m" --explain byte --where "The position is 40." \
    > "$tmp/x40.out" 2>/dev/null || fail "--explain at position 40 failed"
grep -Eq '^"(The 2-token at position [0-9]+ is |The input byte 1 time steps ago was )' "$tmp/x40.out" \
    || fail "--explain must name either the k=2 token event or the k=1 antecedent"
# The memchain is stored append-only and never shifted (#um_optimizations), so not one of its
# 256*N identity copies is ever performed. Condition (3) of that block is that the collapse unfolds
# back into the atoms it stands for, and this is where that is held: the family still enumerates in
# full, and the structural line above already asserted 256*N atomic patterns for it.
"$CMPR" --run "$BASE" --ofra "$OFRA" --patterns-from input --patterns-to mem_cell_1 \
    --model "$tmp/b.m" > "$tmp/mc.out" 2>/dev/null || fail "memchain query failed"
mcn=$(grep -c '^"' "$tmp/mc.out")
[ "$mcn" -eq 256 ] || fail "memchain input->mem_cell_1 unfolded to $mcn atoms, expected 256"
grep -q '"The input byte is 0x00."' "$tmp/mc.out" \
    || fail "memchain unfolding lost an atom the chain never copies"

if "$CMPR" --run "$BASE" --ofra "$OFRA" --model "$tmp/b.m" --explain byte \
      --where "The position is 99999999." > /dev/null 2>"$tmp/xbad.err"; then
    fail "--explain past the sample should have failed"
fi
grep -q "past the sample" "$tmp/xbad.err" || fail "out-of-range --explain message wrong"

# 8. LATD. #p8v2_gen3: "every pattern can be expanded to the set of input positions which it
# records". The expansion is reconstructed from (program text, model file) like the rest of the
# query layer, so what it must hold is that the SUPPORT SET IS THE DATA and not a restatement of
# the stored weight: the two are allowed to disagree, and at these sample sizes they do.
"$CMPR" --run "$BASE" --ofra "$OFRA" --model "$tmp/b.m" --latd 'The input byte is m.' \
    > "$tmp/latd.out" 2>"$tmp/latd.err" || fail "--latd on a k=1 pattern failed: $(tail -1 "$tmp/latd.err")"
grep -q '^"The input byte is m."' "$tmp/latd.out" || fail "--latd did not print the atom it expanded"
grep -Eq '^  positions:( [0-9]+)+$' "$tmp/latd.out" || fail "--latd printed no support positions"
grep -q '"The position is ' "$tmp/latd.out" || fail "--latd cluster is not in SN"
grep -Eq 'holds [0-9]+ of [0-9]+ times' "$tmp/latd.out" \
    || fail "--latd must say how often the argmax it names actually holds"
# every position it names really does carry the antecedent byte, checked against the sample itself
"$CMPR" --run "$BASE" --ofra "$OFRA" --decompress "$tmp/b.m" > "$tmp/latd.sample" 2>/dev/null \
    || fail "could not replay the sample to check --latd"
for lp in $(sed -n 's/^  positions://p' "$tmp/latd.out"); do
    got=$(dd if="$tmp/latd.sample" bs=1 skip="$lp" count=1 2>/dev/null)
    [ "$got" = "m" ] || fail "--latd named position $lp, but the sample has '$got' there, not 'm'"
done
# a pattern that does not exist has no expansion, and that is not the empty set
if "$CMPR" --run "$BASE" --ofra "$OFRA" --model "$tmp/b.m" --latd 'The 2-token is "QZ".' \
      > /dev/null 2>"$tmp/latdbad.err"; then
    fail "--latd on an unkept 2-token should have failed"
fi
grep -q "no k=2 rule" "$tmp/latdbad.err" || fail "unkept-token --latd message wrong"
if "$CMPR" --run "$BASE" --ofra "$OFRA" --model "$tmp/b.m" --latd 'The input byte is m.' --patterns \
      > /dev/null 2>"$tmp/latdmix.err"; then
    fail "--latd combined with --patterns should have failed"
fi
grep -q "cannot be combined" "$tmp/latdmix.err" || fail "--latd/--patterns conflict message wrong"

# ------------------------------------------------------------------- THE TSV
# One row per (variant, sample), MEASURED FACTS ONLY. No rate, no extrapolation, no ranking: those
# are tests/pprog/gen1-report's, per #hutter_metrics. cost_shape/cost_why are required of anything
# expensive by #hutter_run_costs -- a wall clock alone does not say whether the next size up is
# twice the cost or a hundred times.
printf 'variant\taxes\tsample\tsample_src\tN\tU\tbinary_bytes\tp_header\tp_k1\tp_backward\tp_token\tp_gap\tp_trace\troundtrip\ttokens_kept\trounds\tsettle_apps\tsettled_ok\tmean_sweeps\tcov_positions\tcoverage\thits\tremoved\tadded\tsecs\tcost_shape\tcost_why\n' > "$TSV"

mkdir -p "$POSDIR"

# The models are retained, not discarded with $tmp, so the query layer (--patterns, --explain) can
# be run against them afterwards by anything that publishes: without them a rendering can show what
# the model DID at a position but cannot name the rule that did it. Layout is by program version,
# then dataset, then prefix length in bytes -- models/p8v2/enwik9/64/wordsv2-v001.m -- so a second
# dataset or a p8v3 does not collide. The fallback samples are NOT enwik9 and go under their own
# dataset name so nothing can read them as it.
MODELS="$SCRIPT_DIR/models/p8v2"
DATASET=$([ "$REAL" = yes ] && echo enwik9 || echo fallback-not-enwik9)

# The p7 baseline, one row per ladder sample. p7 fits the same three-class schema exactly: its
# 256-byte table is class (a), it has no class (b) at all, and its gap section plus trace is class
# (c). So the comparison that matters -- what the k=2 layer buys in the only class that survives to
# 1e9 -- is a row in this file rather than something computed by hand afterwards.
# Capped: p7 still implements the memchain shift literally (#um_optimizations applies the
# append-only collapse to the wordsv2 kind only), so a p7 run is quadratic in M and 1e6 takes
# upwards of ten minutes.
P7_MAX="${P8V2_P7_MAX:-100000}"
for s in $LADDER; do
    n=$(wc -c < "$tmp/$s.in")
    if [ "$n" -gt "$P7_MAX" ]; then
        printf 'p7-order-1-markov\t------\t%s\t%s\tSKIPPED: p7 still shifts the chain literally, O(M*N) = O(M^2); %d bytes is over the %d limit. #um_optimizations records the collapse that would lift it\n' \
            "$s" "$SRC" "$n" "$P7_MAX" >> "$TSV"
        echo "  p7 $s: SKIPPED (quadratic shift)"
        continue
    fi
    "$CMPR" --run order-1-markov --ofra "$P7OFRA" "$n" --compress "$tmp/p7.$s.m" \
        < "$tmp/$s.in" > /dev/null 2>"$tmp/p7.$s.err" || fail "p7 baseline at $s failed"
    pM=$(u32le "$tmp/p7.$s.m" 8); pSC=$(u32le "$tmp/p7.$s.m" 12); pG=$(u32le "$tmp/p7.$s.m" 16)
    pbin=$(grep -oE "binary [0-9]+" "$tmp/p7.$s.err" | grep -oE "[0-9]+$" | head -1)
    printf 'p7-order-1-markov\t------\t%s\t%s\t%s\t%s\t%s\t20\t256\t0\t0\t%s\t%s\tn/a\t0\t1\t0\t0\t0\t0\t0\t0\t0\t0\t0\tO(M*N) shift + O(M) learn\tp7 has no settling; the cost is the literal chain shift\n' \
        "$s" "$SRC" "$n" "$pM" "$pbin" "$pG" "$pSC" >> "$TSV"
    echo "  p7 $s: trace=$((pG + pSC)) (the class (c) baseline)"
done

for v in $VARIANTS; do
    for s in $LADDER; do
        n=$(wc -c < "$tmp/$s.in")
        case " $SLOW_VARIANTS " in
          *" $v "*)
            if [ "$n" -gt "$SLOW_MAX" ]; then
                # Recorded, not dropped: #hutter_run_costs says a cap is reported with its reason.
                printf '%s\t?\t%s\t%s\tSKIPPED: D2/D3 settle one full window per candidate recorded byte, O(SC*W^2*256), quadratic in M; %d bytes is over the %d limit\n' \
                    "$v" "$s" "$SRC" "$n" "$SLOW_MAX" >> "$TSV"
                echo "  $v $s: SKIPPED (quadratic settling)"
                continue
            fi ;;
        esac
        m="$tmp/$v.$s.m"
        t0=$(date +%s.%N)
        compress "$v" "$n" "$tmp/$s.in" "$m" "$tmp/$v.$s.cerr"
        t1=$(date +%s.%N)
        secs=$(awk "BEGIN{printf \"%.2f\", $t1 - $t0}")
        decompress "$v" "$m" "$tmp/$v.$s.out" "$tmp/$v.$s.derr"
        if cmp -s "$tmp/$s.in" "$tmp/$v.$s.out"; then rt=yes; else rt=no; fi
        hM=$(u32le "$m" 8);  hTC=$(u32le "$m" 12); hTB=$(u32le "$m" 16)
        hG=$(u32le "$m" 20); hSC=$(u32le "$m" 24); hBWD=$(u32le "$m" 28); hR=$(u32le "$m" 32)
        # 8 since generation 5 split f; a pre-gen5 model has NULs at 42..43, so strip them
        ax=$(dd if="$m" bs=1 skip=36 count=8 2>/dev/null | tr -d '\0')
        bin=$(grep -oE "binary [0-9]+" "$tmp/$v.$s.cerr" | grep -oE "[0-9]+$" | head -1)
        apps=$(grep -oE "pattern applications [0-9]+" "$tmp/$v.$s.cerr" | grep -oE "[0-9]+$")
        ok=$(grep -oE "settled argmax correct [0-9]+" "$tmp/$v.$s.cerr" | grep -oE "[0-9]+$")
        sw=$(grep -oE "mean sweeps [0-9.]+" "$tmp/$v.$s.cerr" | grep -oE "[0-9.]+$")
        # The k=2 diagnostics are DERIVED from (sample, model) and computed here rather than in the
        # generated child, which does not need to know about them.
        d=$(python3 "$SCRIPT_DIR/p8v2-diag.py" "$tmp/$s.in" "$m")
        getd() { echo "$d" | tr '\t' '\n' | grep "^$1=" | cut -d= -f2; }
        case "$ax" in
          ???1??) shape="O(min(M,dumpcap)*W*256)"
                  why="the trace is one causal pass; all settling is the diagnostic dump" ;;
          *)      shape="O(SC*W^2*256), quadratic in M"
                  why="one full window settled per candidate recorded byte" ;;
        esac
        printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t48\t256\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
            "$v" "$ax" "$s" "$SRC" "$n" "$hM" "$bin" "$hBWD" "$hTB" "$hG" "$hSC" "$rt" \
            "$hTC" "$hR" "$apps" "$ok" "$sw" \
            "$(getd positions)" "$(getd coverage)" "$(getd hits)" "$(getd removed)" "$(getd added)" \
            "$secs" "$shape" "$why" >> "$TSV"
        echo "  $v $s: trace=$((hG + hSC)) model=$hTB tokens=$hTC roundtrip=$rt ${secs}s"
        # Keep the per-position dump only for the small samples: at 1e6 it is ~50 MB per variant and
        # nothing reads past the first few hundred positions.
        if [ -f "$m.pos" ] && [ "$n" -le "$DUMP_MAX" ]; then
            cp "$m.pos" "$POSDIR/$v.$s.pos"
        fi
        # The model itself, under the same gate. A model is 762 bytes at 64 and under 11 KB at 1e4,
        # except v004, which carries the 65536-byte backward LPP under B2; all 33 are about 350 KB.
        if [ "$n" -le "$DUMP_MAX" ]; then
            mkdir -p "$MODELS/$DATASET/$n"
            cp "$m" "$MODELS/$DATASET/$n/$v.m"
        fi
    done
done

# --------------------------------------------- replay conformance (gen4 goal, choice C)
# The reference implementation may not fall behind the runs: for every variant of this
# generation that retained an e64 model, tests/pprog/p8v2-replay.py must replay it and
# verify against the retained artifacts, including the .pos dump just written. Only
# real-enwik9 runs are checked, because the replay reads the enwik9 prefix itself.
if [ "$REAL" = yes ]; then
    for v in $VARIANTS; do
        [ -f "$MODELS/$DATASET/64/$v.m" ] || continue
        python3 "$SCRIPT_DIR/p8v2-replay.py" "$v" 64 > "$tmp/replay.$v.out" 2>&1 \
            || fail "p8v2-replay.py does not conform for $v at e64: $(tail -3 "$tmp/replay.$v.out" | tr '\n' ' ')"
    done
    echo "  replay conformance: every retained e64 model of this generation verified by p8v2-replay.py"
fi

echo "PASS: P8v2/wordsv2 generation 1 via real chain (baseline round-trips text/random/empty/partial,"
echo "      P8V2 format, zero input has no class (b) or (c), class (c) rate beats p7 on the same"
echo "      sample, both-mode class reporting with no derived quantity leaking, mode validation,"
echo "      k=2 rules and positions readable in the query layer, every pattern expandable by
      --latd to the input positions that formed it)"
echo "      samples: $SRC ; ladder: $LADDER ; $(( $(wc -l < "$TSV") - 1 )) TSV rows in $TSV"
echo "      derived view: python3 tests/pprog/gen1-report $TSV"
echo "      axis documentation (needs BOTH generations on disk, runs no compression):"
echo "        python3 tests/pprog/p8v2-axes.py  ;  tests/pprog/acceptance-p8v2-axes"