#!/usr/bin/env python3
"""Build the published p8v2 generation-1 page.

PUBLICATION IS ../hutter's (#hutter_publication_handoff).  cmpr-src generates the
measured facts and derives the rates; this script renders them and nothing else.
It runs no compression: everything comes from tests/pprog/gen1.tsv and the .pos
dumps that are already on disk.

The derived quantities are NOT recomputed here.  This script imports
tests/pprog/gen1-report and calls its derive(), so the page cannot drift from the
DISPLAY step that #hutter_metrics makes authoritative.

  docs/pprog/build-p8v2-gen1 [--cmpr-src PATH]

Writes docs/pprog/p8v2-gen1/: index.html, gen1.html and gen2.html (both
re-rendered from the current dumps), gen1-report.txt, the cited block dumps, and
the symlinks to the data.

THE PANEL RENDERER IS cmpr-src's, AND THERE IS ONLY ONE COPY OF IT.  This script
used to run a copy of gen1-viz.py that sat next to it in docs/pprog/, and that
copy silently forked: cmpr-src's gained the decomposed divergence strip, the
ladder ordering and the strike/conv gutter, and the published panel had none of
them for five days because nothing compared the two files (2026-08-10).  The
renderer now comes from tests/pprog/gen1-viz.py and is symlinked into the output
so the page can still link the source it was rendered by.  Do not reintroduce a
local copy -- #viz_surfaces says the renderer lives in cmpr-src and is ours.

BOTH GENERATIONS RENDER HERE, and the directory name is a legacy URL.  gen1-viz.py
takes the generation off the posdir name, so every generation with a posdir gets a
panel with no edit to this script.  The output directory stays p8v2-gen1/ because
it is a published path; what it holds is every generation panel, not only the
first.
"""

import importlib.util
import json
import os
import subprocess
import sys
import html as H

HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, "p8v2-gen1")
DSS = 1e9

# What each generation's panel is, for the navigation.  A generation with no entry
# here STILL GETS A CARD -- one that says the prose is missing -- because a link that
# reads badly is recoverable and a panel nothing links is not.
PANEL_NOTE = {
    1: "Every variant over the same input positions, with a divergence strip marking "
       "the columns where they disagree — decomposed into recorded, level and "
       "settled, because the combined strip saturates and a saturated strip is not a "
       "list of positions worth looking at. Self-contained inline SVG, both surfaces, "
       "no external request. Spec: section 15 of "
       '<a href="p8v2-viz-spec.md">p8v2-viz-spec.md</a>; renderer '
       '<a href="gen1-viz.py">gen1-viz.py</a>, standard library only.',
    2: "The same panel over generation 2's ten variants, post-<code>k=0</code> removal "
       "(<code>#pprog_p8v2_gen2_report_20260806</code>). Generation 1's runs were re-run "
       "after the same removal, so the two panels are comparable with each other and "
       "with nothing published before 2026-08-06. Same renderer: it takes the "
       "generation off the posdir name.",
    3: "The baseline against generation 3's three axis-A alternatives, the first that "
       "set a pattern's fall-off from its own support rather than from a constant "
       "(<code>#pprog_p8v2_gen3_report_20260810</code>). Read the <code>conv</code> "
       "gutter carefully here: under a rule that fires a pattern only on some time "
       "steps, a window can look settled because nothing has happened yet, and one "
       "alternative reports a mean of 1.00 time steps for exactly that reason. The "
       "column measures the wrong thing for a scheduled rule, and generation 3 says so "
       "rather than correcting it silently.",
    4: "The A7 pair &mdash; the k=1 argmax delivered at min(8, w<sub>s</sub>) with "
       "fall-off 1, one alternative because the two constants only make sense together "
       "&mdash; against the baseline and against B2, where the stored learned weights "
       "are the delivery and every B1 strategy has an exact fact to be compared with "
       "(<code>#pprog_p8v2_gen4_report_20260815</code>). The ground truth cuts both "
       "ways: at e64 the stated 8 beats the stored truth, by e10k the stored matrix "
       "wins. The instrument for this generation is "
       '<a href="/hutter/pprog/p8v2-gen4-e64/">p8v2-gen4-e64</a>.',
}

BLOCKS = ["hutter_publication_handoff", "hutter_metrics", "hutter_run_costs",
          "um_optimizations", "pprog_p8v2_gen1_report_20260804",
          "pprog_p8v2_impl_goal_20260804", "variant_protocol",
          "pprog_p8v2_questions_20260727", "pprog_p8v2_gen1_choices",
          "pprog_p8v2_gen2_goal_20260806", "pp_wordsv2",
          "pprog_p8v2_gen3_goal_20260810", "pprog_p8v2_gen3_report_20260810",
          "p8v2_gen3", "f-p8", "pprog_p8v2_gen4_goal_20260815",
          "pprog_p8v2_gen4_report_20260815", "f-p8-cap8"]

# A GENERATION'S OWN DATA IS LINKED BY DISCOVERY, NOT BY THIS LIST.  gen<N>-pos and
# gen<N>.tsv are added for every generation found on disk (see main), because the
# hand-written form of this list is how generation 3 came to have a rendered panel
# that nothing linked and no data beside it -- exactly the "a page nobody links"
# failure #viz_surfaces warns about, one day after that warning was written.
LINKS = [("axes.json", "axes.json"),
         ("acceptance-p8v2", "acceptance-p8v2"), ("gen1-report", "gen1-report"),
         ("p8v2-viz-spec.md", "p8v2-viz-spec.md"), ("p8v2-words.md", "p8v2-words.md"),
         ("p8v2-diag.py", "p8v2-diag.py"), ("p8v2-replay.py", "p8v2-replay.py"),
         ("p8v2-axes.py", "p8v2-axes.py"),
         # the renderer itself, so the page links the file it was rendered by and
         # not a copy that can disagree with it
         ("gen1-viz.py", "gen1-viz.py")]

# What each variant varies used to be a hand-written dict here, one line per variant,
# copied out of a goal block. It was a copy: it went stale the moment an axis moved
# and it could not carry a worked example. It is now DERIVED, from cmpr-src's
# tests/pprog/axes.json (emitted by tests/pprog/p8v2-axes.py, whose prose is lifted
# straight out of the alternatives' OFRA blocks) crossed with each row's own axis
# vector. #pprog_p8v2_gen2_goal_20260806.
GEN1_BASELINE = "111111"


def load_axes(pprog):
    p = os.path.join(pprog, "axes.json")
    if not os.path.exists(p):
        sys.stderr.write("build-p8v2-gen1: no %s -- run cmpr-src's "
                         "tests/pprog/p8v2-axes.py\n" % p)
        return None
    with open(p) as f:
        return json.load(f)


def what(ax, digits, baseline=GEN1_BASELINE):
    """One line saying what this axis vector varies, from axes.json and nothing else."""
    if ax is None or not digits or len(digits) != 6 or not digits.isdigit():
        return "the p7 order-1 Markov baseline"
    parts = []
    for i, a in enumerate(ax["axes"]):
        if digits[i] == baseline[i]:
            continue
        for alt in a["alternatives"]:
            if str(alt["digit"]) == digits[i]:
                parts.append("%s%s &mdash; %s"
                             % (a["letter"], digits[i], H.escape(alt["label"])))
    if not parts:
        return "the baseline &mdash; every axis at its baseline alternative"
    return "; ".join(parts)


def load_report_module(cmpr_src):
    path = os.path.join(cmpr_src, "tests", "pprog", "gen1-report")
    spec = importlib.util.spec_from_loader(
        "gen1report", importlib.machinery.SourceFileLoader("gen1report", path))
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod


def short(v):
    return v.replace("wordsv2-", "").replace("p7-order-1-markov", "p7")


def f3(x):
    return "%.4f" % x


def main(argv):
    cmpr_src = os.path.abspath(os.path.join(HERE, "..", "..", "..", "cmpr-src"))
    if "--cmpr-src" in argv:
        cmpr_src = os.path.abspath(argv[argv.index("--cmpr-src") + 1])
    pprog = os.path.join(cmpr_src, "tests", "pprog")
    if not os.path.isdir(pprog):
        sys.exit("build-p8v2-gen1: no %s" % pprog)

    os.makedirs(OUT, exist_ok=True)
    axes_doc = load_axes(pprog)
    rpt = load_report_module(cmpr_src)
    rows = rpt.load(os.path.join(pprog, "gen1.tsv"))

    # --- data links: the measured facts stay cmpr-src's, symlinked not copied ---
    # Every generation with a posdir, in order.  One list drives the symlinks, the
    # panels and the navigation, so a generation cannot land with a panel and no way
    # to reach it.
    gens = sorted(int(d[3:-4]) for d in os.listdir(pprog)
                  if d.startswith("gen") and d.endswith("-pos") and d[3:-4].isdigit()
                  and os.path.isdir(os.path.join(pprog, d)))
    links = [(x, y) for x, y in LINKS]
    for g in gens:
        links.append(("gen%d-pos" % g, "gen%d-pos" % g))
        links.append(("gen%d.tsv" % g, "gen%d.tsv" % g))

    rel = os.path.relpath(pprog, OUT)
    for name, target in links:
        link = os.path.join(OUT, name)
        src = os.path.join(rel, target)
        if not os.path.exists(os.path.join(pprog, target)):
            continue
        if os.path.islink(link) or os.path.exists(link):
            os.remove(link)
        os.symlink(src, link)

    # --- the panels: re-rendered here, from the current dumps, one per generation ---
    # cmpr-src's renderer, not a copy of it (see the module docstring). Every posdir
    # it finds gets a panel, so generation 3 needs no edit here either.
    viz = os.path.join(pprog, "gen1-viz.py")
    for g in gens:
        subprocess.run([sys.executable, viz, os.path.join(pprog, "gen%d-pos" % g),
                        os.path.join(OUT, "gen%d.html" % g)], check=True)

    # --- the derived view, verbatim, so the page can be checked against it ---
    with open(os.path.join(OUT, "gen1-report.txt"), "w") as f:
        f.write(subprocess.run([sys.executable, os.path.join(pprog, "gen1-report")],
                               capture_output=True, text=True, check=True).stdout)

    # --- the cited blocks ---
    cmpr = os.environ.get("CMPR") or os.path.join(cmpr_src, "cmpr", "dist", "cmpr")
    if not os.access(cmpr, os.X_OK):
        cmpr = "cmpr"
    for b in BLOCKS:
        p = subprocess.run([cmpr, "--print-block", "#" + b],
                           cwd=cmpr_src, capture_output=True, text=True)
        if p.returncode == 0 and p.stdout.strip():
            with open(os.path.join(OUT, b + ".txt"), "w") as f:
                f.write(p.stdout)
        else:
            sys.stderr.write("build-p8v2-gen1: could not dump #%s\n" % b)

    # --- group rows by sample, in ladder order ---
    samples, by_sample = [], {}
    for r in rows:
        s = r["sample"]
        if s not in samples:
            samples.append(s)
            by_sample[s] = []
        by_sample[s].append(r)

    def done(sub):
        return [r for r in sub if r.get("U", "").strip().isdigit()]

    # per-sample tables, ranked by a exactly as gen1-report ranks them
    per_sample = []
    for s in samples:
        d_rows = done(by_sample[s])
        if not d_rows:
            continue
        M = int(d_rows[0]["U"])
        scored = sorted((rpt.derive(r, DSS)["a"], i, r) for i, r in enumerate(d_rows))
        body = []
        for a, _, r in scored:
            d = rpt.derive(r, DSS)
            v = short(r["variant"])
            rt = r.get("roundtrip", "")
            rtc = "" if rt in ("yes", "n/a", "") else ' class="no"'
            body.append(
                "<tr%s><td>%s</td><td>%s</td><td>%d</td><td>%d</td><td>%d</td>"
                "<td><b>%s</b></td><td>%d</td><td>%s</td><td>%.1f</td><td%s>%s</td></tr>"
                % (' class="p7"' if v == "p7" else "", v, r["axes"],
                   d["fixed"], d["model_b"], d["trace"], f3(d["a"]),
                   d["model_b_dss"], "%.6f" % d["su"], d["k"], rtc, rt))
        skipped = [r for r in by_sample[s] if r not in d_rows]
        note = ""
        if skipped:
            why = (skipped[0].get("roundtrip") or "").strip()
            for k in skipped[0]:
                if "SKIPPED" in str(skipped[0][k]):
                    why = str(skipped[0][k]).split("SKIPPED:", 1)[-1].strip()
                    break
            note = ("<p class=\"sub\" style=\"font-size:.9rem\">%s skipped at this "
                    "size: %s</p>" % (", ".join(short(r["variant"]) for r in skipped),
                                      H.escape(why)))
        per_sample.append((s, M, "\n".join(body), note))

    # the a-against-M trend
    variants = []
    for r in rows:
        v = short(r["variant"])
        if v not in variants:
            variants.append(v)
    variants = [v for v in variants if v != "p7"] + ["p7"]
    digits_of = {}
    for r in rows:
        digits_of.setdefault(short(r["variant"]), (r.get("axes") or "").strip())
    trend = []
    for v in variants:
        cells = []
        for s in samples:
            hit = [r for r in done(by_sample[s]) if short(r["variant"]) == v]
            cells.append("<td>%s</td>" % (f3(rpt.derive(hit[0], DSS)["a"]) if hit
                                          else "<span class=sk>&mdash;</span>"))
        trend.append("<tr%s><td>%s</td>%s<td class=\"what\">%s</td></tr>"
                     % (' class="p7"' if v == "p7" else "", v, "".join(cells),
                        what(axes_doc, digits_of.get(v, ""))))

    # baseline against p7, the headline comparison
    head = []
    for s in samples:
        d_rows = done(by_sample[s])
        b = [r for r in d_rows if short(r["variant"]) == "v001"]
        p = [r for r in d_rows if short(r["variant"]) == "p7"]
        if not b:
            continue
        db = rpt.derive(b[0], DSS)
        M = int(b[0]["U"])
        hits = rpt.num(b[0], "hits")
        pos = rpt.num(b[0], "cov_positions")
        net = rpt.num(b[0], "removed") - rpt.num(b[0], "added")
        cell_p7 = f3(rpt.derive(p[0], DSS)["a"]) if p else "&mdash;"
        ratio = ("%.2f" % (db["a"] / rpt.derive(p[0], DSS)["a"])) if p else "&mdash;"
        head.append("<tr><td>%d</td><td><b>%s</b></td><td>%s</td><td>%s</td>"
                    "<td>%d</td><td>%s</td><td>%.2f</td></tr>"
                    % (M, f3(db["a"]), cell_p7, ratio, db["model_b"],
                       ("%d/%d" % (hits, pos)) if pos else "&mdash;",
                       net / db["model_b"] if db["model_b"] else 0))

    # cost shapes
    shapes = []
    for r in rows:
        sh, why = r.get("cost_shape", ""), r.get("cost_why", "")
        if sh and (sh, why) not in shapes:
            shapes.append((sh, why))
    costs = "\n".join("<tr><td>%s</td><td class=\"what\">%s</td></tr>"
                      % (H.escape(sh), H.escape(why)) for sh, why in shapes)

    panels = "\n".join(
        '<a class="big" href="gen%d.html">\n'
        '  <b>Open the generation %d panel &rarr;</b>\n  <span>%s</span>\n</a>'
        % (g, g, PANEL_NOTE.get(g,
           "Generation %d's variants over the same input positions. No description "
           "has been written for this panel yet; it is linked anyway, because a "
           "rendered panel nothing links is the worse failure." % g))
        for g in gens)

    tpl = open(os.path.join(HERE, "p8v2-gen1.tpl.html")).read()
    page = tpl.replace("<!--PANELS-->", panels) \
              .replace("<!--HEADLINE-->", "\n".join(head)) \
              .replace("<!--TREND-->", "\n".join(trend)) \
              .replace("<!--COSTS-->", costs) \
              .replace("<!--SAMPLES-->", "\n".join(
                  '<h3>M = %d bytes <small>(%s)</small></h3>\n<div class="scroll">'
                  '<table class="nums"><tr><th>variant</th><th>axes</th>'
                  '<th>fixed B</th><th>model B</th><th>trace B</th><th>a</th>'
                  '<th>model B @1e9</th><th>S/U</th><th>k</th><th>round-trip</th></tr>\n'
                  '%s</table></div>%s' % (M, s, body, note)
                  for s, M, body, note in per_sample))
    with open(os.path.join(OUT, "index.html"), "w") as f:
        f.write(page)
    sys.stderr.write("build-p8v2-gen1: wrote %s (%d samples, %d rows)\n"
                     % (OUT, len(per_sample), len(rows)))
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))