\documentclass[11pt]{article}
\usepackage[margin=1in]{geometry}
\usepackage{amsmath,amssymb,amsthm,booktabs}
\usepackage{hyperref}

\newtheorem{theorem}{Theorem}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{example}[theorem]{Example}
\newtheorem{remark}[theorem]{Remark}
\newtheorem{conjecture}[theorem]{Conjecture}

\title{Kneser--Ney as Quotient:\\Pulling $n$-gram Smoothing into the Universal Model}
\author{Claude and MJC}
\date{February 16, 2026}

\begin{document}
\maketitle

\begin{abstract}
Interpolated Kneser--Ney (KN) smoothing achieves 1.784\,bpc on enwik9
with byte-level 6-grams and zero structure~\cite{kn-scaling}.
We have tried to combine KN with other models via mixture
approaches, with ``mixed results''~\cite{match-model}---the
combination problem is unsolved because KN lives outside the UM
framework, requiring ad-hoc mixing.
We pull KN back into the UM by showing that its three core operations---the
discount, the backoff distribution, and the interpolation
recursion---are quotient operations on the joint count table
$c(\text{context}, \text{output})$.
The discount $D$ approximates the removal of \emph{common evidence}
(the GCD of the Bayes-from-counting framework~\cite{bayes-counting}):
it subtracts evidence that the lower-order model already explains.
The KN continuation count is the column-side type count---a quotient
measuring each output's \emph{generality} across contexts.
The interpolation recursion is the hierarchical combination
rule~\cite{pattern-space} applied to a tower of decreasing-order
models, each a projection of the one above.
With KN inside the UM, the combination problem dissolves:
KN patterns at different orders coexist in a single pattern table $P$,
and extending $E$ with lexical events~\cite{extended-es} gives them
the word-level structure they currently lack.
\end{abstract}


\section{The Event Spaces of $n$-gram Prediction}\label{sec:event-spaces}

An order-$k$ byte-level $n$-gram model has event space
$E_k = I_k \times O$, where:
\begin{align}
I_k &= \{0, \ldots, 255\}^{k-1} \quad \text{(context: previous $k{-}1$ bytes)}, \\
O &= \{0, \ldots, 255\} \quad \text{(output: next byte)}.
\end{align}
The joint event $(c, o) \in E_k$ records ``context $c$ was followed by
byte $o$.''  The count table is:
\begin{equation}
c_k(c, o) = |\{t : (b_{t-k+1}, \ldots, b_{t-1}) = c,\; b_t = o\}|.
\end{equation}
This is exactly the log product pattern (LPP) of the UM applied to
$E_k$: each entry $s_k(c, o) = \lfloor \log_2 c_k(c, o) \rfloor$
is an atomic pattern weight.

\begin{proposition}[The $n$-gram count table is an LPP]
The order-$k$ count table $c_k: I_k \times O \to \mathbb{N}$ is the
output of the standard learning function $\omega_0$ applied to the
event space $E_k$.  Each nonzero entry is an atomic pattern
$(c, o, s_k(c,o))$ in the pattern space $P_k \subseteq E_k^2 \times T$.
\end{proposition}

For a full KN model of order $K$, we have a tower of event spaces:
\begin{equation}\label{eq:tower}
E_K \twoheadrightarrow E_{K-1} \twoheadrightarrow \cdots
\twoheadrightarrow E_2 \twoheadrightarrow E_1,
\end{equation}
where each projection $\pi_k: E_k \to E_{k-1}$ drops the oldest byte
from the context:
\begin{equation}
\pi_k(b_1, \ldots, b_{k-1}, o) = (b_2, \ldots, b_{k-1}, o).
\end{equation}


\section{The Order Projection as Quotient}\label{sec:projection}

\begin{definition}[Order projection]
The projection $\pi_k: I_k \to I_{k-1}$ identifies contexts that
differ only in their oldest byte:
\begin{equation}
[c]_{k-1} = \{c' \in I_k : \pi_k(c') = \pi_k(c)\}
= \{(b, c_2, \ldots, c_{k-1}) : b \in \{0..255\}\}.
\end{equation}
Each equivalence class has $|[c]_{k-1}| = 256$ elements (one per
value of the dropped byte).
\end{definition}

\begin{proposition}[Projection as UM quotient]
The order projection $\pi_k$ is a quotient in the UM sense: it maps
a finer event space ($I_k$, which distinguishes 256 more contexts)
onto a coarser one ($I_{k-1}$, which pools them).  The quotient
$Q_k = |I_k| / |I_{k-1}| = 256$ measures the compression at each
step.

The count tables are related by marginalization:
\begin{equation}\label{eq:margin}
c_{k-1}(\pi_k(c), o) = \sum_{c' \in [c]_{k-1}} c_k(c', o).
\end{equation}
The lower-order count is the sum over the equivalence class---the
standard UM marginalization over a projected event space.
\end{proposition}

\begin{remark}[The tower is the factorization tower]
The tower~\eqref{eq:tower} is a factorization tower in the sense of
the nested model~\cite{nested-model}: each level is a UM with its own
$E_k = I_k \times O$, and the projection $\pi_k$ is a factor map
between adjacent levels.  The full KN model interpolates predictions
from all levels---but in the UM, these should be patterns in a single
unified pattern table, not separate models mixed externally.
\end{remark}


\section{The Discount as Common Evidence Removal}\label{sec:discount}

The interpolated KN prediction at order $k$ is:
\begin{equation}\label{eq:kn}
P_k(o \mid c) = \frac{\max(c_k(c, o) - D, 0)}{c_k(c, \cdot)}
+ \frac{D \cdot \tau_k(c)}{c_k(c, \cdot)} \cdot P_{k-1}(o \mid \pi_k(c)),
\end{equation}
where $c_k(c, \cdot) = \sum_o c_k(c, o)$ is the total count for
context $c$, $\tau_k(c) = |\{o : c_k(c, o) > 0\}|$ is the type
count, and $D \in [0, 1]$ is the discount.

The discount subtracts $D$ from each nonzero count.  What does this
mean in quotient terms?

\begin{definition}[Common evidence in the GCD framework]
The Bayes-from-counting framework~\cite{bayes-counting} decomposes
each row of the count table via the GCD:
\begin{equation}
c_k(c, o) = g(c) \cdot r(c, o),
\qquad g(c) = \gcd_{o: c_k(c,o)>0} c_k(c,o),
\end{equation}
where $g(c)$ is the \emph{common evidence}---the largest integer
that divides every count in the row---and $r(c, o)$ is the
\emph{differential evidence} (the reduced counts, coprime across
the row).

The conditional probability depends only on the reduced counts:
$P(o \mid c) = r(c, o) / R(c)$ where $R(c) = \sum_o r(c, o)$.
The GCD cancels.
\end{definition}

\begin{proposition}[The discount approximates GCD removal]
\label{prop:discount-gcd}
The KN discount $D$ approximates the removal of common evidence in
the following sense.  Subtracting $D$ from each nonzero count gives:
\begin{equation}
c_k(c, o) - D = g(c) \cdot r(c, o) - D.
\end{equation}
When $D = g(c)$ (the per-row GCD), this removes exactly one unit of
common evidence:
\begin{equation}
c_k(c, o) - g(c) = g(c) \cdot (r(c, o) - 1).
\end{equation}
For contexts where $\min_o c_k(c, o) = 1$ (the minimum count is 1,
so $g(c) = 1$), this removes one occurrence of each continuation,
leaving the reduced counts minus one.

KN uses a single global $D$ rather than the per-row $g(c)$, which is
an approximation.  The optimal $D$ (Chen \& Goodman, 1999) is:
\begin{equation}
D^* = \frac{n_1}{n_1 + 2n_2},
\end{equation}
where $n_1, n_2$ are the global counts of $n$-grams occurring exactly
once or twice.  For natural language, $D^* \approx 0.75$--$0.9$,
which is close to the mean GCD: most rows in a byte-level count table
have $g(c) = 1$ (because most contexts have at least one continuation
with count 1), so the global $D$ tracks the typical GCD well.
\end{proposition}

\begin{remark}[Why $D$ is not exactly the GCD]
The GCD is an integer; $D$ is a real number $\in (0, 1)$.  The GCD
removes evidence in integer units (``one packet of common evidence'');
$D$ removes a fractional amount.  This fractional removal has no
direct UM interpretation---it is an artifact of the estimation
procedure.

The UM-native version would use $g(c)$ directly: subtract the GCD
from each count, leaving the reduced counts.  But this creates a
problem: for contexts with $g(c) > 1$, removing the full GCD can
remove too much evidence, and the reduced counts may be too sparse for
reliable estimation.  KN's fractional $D$ is a regularization that
trades optimality for robustness.
\end{remark}


\section{The Continuation Count as Column Quotient}\label{sec:continuation}

Standard KN differs from simple interpolation in its backoff
distribution.  Instead of using the raw unigram $P(o) = c(o) / N$,
KN uses the \emph{continuation count}:
\begin{equation}
c_{\mathrm{KN}}(o) = |\{c : c_k(c, o) > 0\}|,
\end{equation}
the number of distinct contexts in which output $o$ has appeared.

\begin{proposition}[The continuation count is the column type count]
\label{prop:continuation}
In the count table $c_k: I_k \times O \to \mathbb{N}$, the
continuation count $c_{\mathrm{KN}}(o)$ is the column type count: the
number of nonzero entries in column $o$.  This is the column-side
analogue of the row type count $\tau_k(c)$.
\end{proposition}

\begin{theorem}[The continuation count as column quotient]
\label{thm:continuation-quotient}
The continuation count measures the \emph{generality} of output $o$
across contexts---a quotient in the UM sense:
\begin{equation}
Q_{\mathrm{gen}}(o) = \frac{|I_k|}{c_{\mathrm{KN}}(o)}.
\end{equation}
Low generality quotient (many distinct contexts) means $o$ is
``general''---it appears in diverse situations, like the space
character.  High generality quotient (few distinct contexts) means
$o$ is ``specific''---it appears only in restricted contexts, like
a rare byte value.

The KN backoff distribution normalizes the continuation count:
\begin{equation}
P_{\mathrm{KN}}(o) = \frac{c_{\mathrm{KN}}(o)}{\sum_{o'} c_{\mathrm{KN}}(o')},
\end{equation}
which is the type-count-based estimate of the marginal.  This
estimates how likely $o$ is to appear in a \emph{new} context---i.e.,
one not yet in the count table---rather than how often it appears
overall.
\end{theorem}

\begin{remark}[Why continuation counts work]
Raw counts are dominated by frequency: the space character has the
highest $c(o)$ because it appears most often.  But much of this count
comes from a few very common contexts (``the~'', ``of~'', etc.).
The continuation count gives each context equal vote, regardless of
how many times it appeared.  This is a de-biasing operation---it
removes the effect of context frequency, leaving only the diversity
of contexts in which $o$ appears.

In UM quotient terms: the raw count $c(o)$ conflates the
\emph{common evidence} (how often each context occurs) with the
\emph{differential evidence} (how broadly $o$ is distributed).
The continuation count extracts the differential evidence by
counting types rather than tokens.  This is a column-side GCD-like
operation: it factors out the context-frequency effect.
\end{remark}


\section{Interpolation as Hierarchical Combination}\label{sec:interpolation}

The KN recursion~\eqref{eq:kn} interpolates between orders.  We now
show this is the hierarchical combination rule from the pattern-space
framework~\cite{pattern-space}.

\begin{theorem}[KN interpolation as hierarchical combination]
\label{thm:hierarchical}
The interpolated KN recursion has the form:
\begin{equation}
P_k(o \mid c) = \underbrace{\frac{\max(c_k(c,o) - D, 0)}{c_k(c, \cdot)}}_{\text{high-order residual}}
+ \underbrace{\frac{D \cdot \tau_k(c)}{c_k(c, \cdot)}}_{\text{backoff weight}}
\cdot P_{k-1}(o \mid \pi_k(c)).
\end{equation}
This is a two-stage hierarchical combination:
\begin{enumerate}
\item \textbf{Stage 1 (high-order residual)}: Use the high-order
  count, discounted by $D$, to predict $o$.  This is the
  \emph{differential evidence} from the full context $c$---the part
  that the lower-order model does not capture.

\item \textbf{Stage 2 (backoff)}: For the remaining probability
  mass ($D \cdot \tau_k(c) / c_k(c, \cdot)$), delegate to the
  lower-order model $P_{k-1}$.  This is the \emph{common evidence}
  that is equally well explained by the shorter context $\pi_k(c)$.
\end{enumerate}
The two stages condition on different information: Stage~1 uses the
full context $c$ (all $k{-}1$ bytes); Stage~2 uses only the shorter
context $\pi_k(c)$ ($k{-}2$ bytes).  The dropped byte
$c_1$---the oldest byte in the context---is the information that
separates them.
\end{theorem}

\begin{proof}
The backoff weight $\alpha_k(c) = D \cdot \tau_k(c) / c_k(c, \cdot)$
sums to $1 - \sum_o \max(c_k(c,o) - D, 0) / c_k(c, \cdot)$, so the
two terms form a proper probability distribution.  The high-order term
uses the full context $c$; the backoff term marginalizes out the oldest
byte by delegating to $P_{k-1}$.

This matches the hierarchical combination rule: information flows from
the full context through the high-order model (Stage~1), and the
residual (unexplained by the full context) is handled by the
lower-order model (Stage~2), which has absorbed the shorter context
via its own counting.  The two stages are approximately independent
because the high-order residual captures what the oldest byte adds,
while the backoff captures what is predictable without it.
\end{proof}

\begin{remark}[The full recursion is a factorization tower]
Applying the recursion from order $K$ down to order 1 gives a tower:
\begin{equation}
P_K \xrightarrow{\;\text{residual + backoff}\;}
P_{K-1} \xrightarrow{\;\text{residual + backoff}\;}
\cdots \xrightarrow{\;\text{residual + backoff}\;}
P_1.
\end{equation}
Each step projects to a coarser event space (shorter context) and
uses the residual/backoff decomposition to separate high-order from
low-order evidence.  The full prediction is a weighted sum over all
orders, with each order contributing its differential evidence.

This tower is the \emph{order-projection factorization tower} of the
$n$-gram model.  In the UM, the whole tower should be a single
pattern table $P$ over the union of event spaces
$\bigcup_k E_k$, with patterns at each order contributing to the
same forward pass.
\end{remark}


\section{The Unified Pattern Table}\label{sec:unified}

\begin{definition}[The KN pattern table]
\label{def:kn-pattern-table}
We define the UM pattern table for KN as the union of patterns at
all orders:
\begin{equation}
P_{\mathrm{KN}} = \bigcup_{k=1}^{K} P_k,
\end{equation}
where $P_k$ contains atomic patterns $(c, o, w_k(c, o))$ for each
context $c \in I_k$ and output $o \in O$ with $c_k(c, o) > 0$.

The weight $w_k(c, o)$ is NOT the raw log count.  It is the
\emph{residual} weight after the quotient decomposition:
\begin{equation}\label{eq:residual-weight}
w_k(c, o) = \log_2 \max(c_k(c, o) - D, 0),
\end{equation}
which is the log of the discounted count---the differential evidence
from order $k$ after removing the common evidence $D$.
\end{definition}

\begin{proposition}[No double-counting in the unified table]
\label{prop:no-double}
The unified pattern table $P_{\mathrm{KN}}$ avoids double-counting
because each order contributes only its \emph{residual} evidence:
\begin{itemize}
\item Order $k$ contributes what the full context $c \in I_k$ explains
  \emph{above} the shorter context $\pi_k(c) \in I_{k-1}$.
\item The discount $D$ removes the common evidence between adjacent
  orders.
\item The backoff weight distributes the removed mass to the
  lower-order patterns.
\end{itemize}
This is the safe combination principle~\cite{pattern-space}: the
evidence at each order is the residual after accounting for
lower-order evidence, and residuals at different orders are
(approximately) independent because they condition on disjoint
information (the oldest byte at each level).
\end{proposition}

\begin{remark}[The problem with the current implementation]
Our current KN implementation~\cite{kn-scaling} stores counts in
a hash table and computes the interpolated prediction procedurally
at each position.  It is not a UM: there is no event space, no
pattern table, and no forward pass.  The match-model
experiments~\cite{match-model} tried to combine KN with context
matching via external mixing---this is the ``mixing $\neq$ extending''
problem~\cite{extended-es} applied to the counting model itself.

Pulling KN into a unified pattern table solves the combination
problem: the KN patterns and the match-model patterns (and
eventually the word-level patterns) all live in the same $P$, and
the forward pass combines them via $\max$-$\min$ without ad-hoc
mixing weights.
\end{remark}


\section{The Discount--GCD Gap}\label{sec:gap}

The central open question: what exactly is the relationship between
$D$ and the per-row GCD $g(c)$?

\begin{conjecture}[The discount--GCD correspondence]
\label{conj:discount-gcd}
The optimal KN discount $D^*$ minimizes the expected divergence
between the discounted distribution and the GCD-reduced distribution:
\begin{equation}
D^* = \arg\min_D \sum_c P(c) \cdot
\mathrm{KL}\!\left(\frac{c_k(c,\cdot) - D}{c_k(c,\cdot) - D\tau_k(c)}
\;\middle\|\;
\frac{r(c, \cdot)}{R(c)}\right),
\end{equation}
where $r(c, o) = c_k(c,o) / g(c)$ are the GCD-reduced counts and
$R(c) = c_k(c, \cdot) / g(c)$ is the reduced total.

When the discount equals the GCD ($D = g(c)$ per-row), the KL
divergence is zero for that row.  The global $D$ is the best single
number that approximates the per-row GCDs.
\end{conjecture}

\begin{proposition}[Empirical GCD distribution]
\label{prop:gcd-dist}
For byte-level $n$-gram count tables on natural language:
\begin{itemize}
\item The vast majority of rows have $g(c) = 1$ (at least one
  continuation appears exactly once).
\item Rows with $g(c) > 1$ are typically high-frequency contexts
  (``th'', ``he'', etc.) where all continuations are well-observed.
\item The optimal global $D \approx 0.8$--$0.9$ is consistent with
  $g(c) = 1$ for most rows, since subtracting $D < 1$ from a count
  of 1 gives $\max(1 - D, 0) = 1 - D > 0$---the continuation
  survives with reduced weight.
\end{itemize}

The GCD-based discount would use $D = g(c)$ per row, which equals 1
for most rows and $> 1$ for high-frequency contexts.  This is
essentially modified KN (which uses different $D$ values depending
on count), but derived from the quotient structure rather than
estimated from held-out data.
\end{proposition}

\begin{remark}[The un-worked-out detail]
The exact mapping from KN's discount to the quotient framework has
subtleties:
\begin{enumerate}
\item The GCD operates on \emph{integer} counts; the discount operates
  on counts as \emph{real} numbers.  The tropical approximation
  (min vs GCD) from~\cite{bayes-counting} mediates between these, but
  the gap is nonzero.
\item The discount is applied \emph{before} normalization; the GCD
  cancels \emph{after} normalization.  These are not the same operation.
  Subtracting $D$ and then normalizing gives a different distribution
  than dividing by $g(c)$ and then normalizing (the latter gives the
  same distribution---the GCD cancels).
\item The continuation count in the backoff distribution has no direct
  GCD analogue.  It is a type count (column-side), while the GCD
  operates on the count magnitudes.  The connection between type counts
  and GCDs is unclear.
\end{enumerate}
These are the details that need to be resolved to make the
discount--quotient correspondence precise.  We suspect the resolution
involves the \emph{two-sided} GCD decomposition from~\cite{bayes-counting}:
the row GCDs $g_I$ and column GCDs $g_O$ together determine the
full structure, and the discount/continuation-count pair approximates
the $(g_I, g_O)$ pair.
\end{remark}


\section{The UM-Native KN Model}\label{sec:um-native}

We now sketch the target: a UM that achieves KN-level performance
while remaining inside the framework.

\begin{definition}[The UM-native $n$-gram model]
\label{def:um-native}
The event space is the full tower:
\begin{equation}
E = I_K \times O,
\end{equation}
where $I_K = \{0..255\}^{K-1}$ is the order-$K$ context.
Lower-order contexts are projections: $I_k = \pi_{k+1} \circ \cdots
\circ \pi_K(I_K)$.

The pattern table $P$ contains atomic patterns at all orders:
\begin{equation}
P = \{(c_k, o, w_k(c_k, o)) : k = 1, \ldots, K,\;
c_k \in I_k,\; c_k(c_k, o) > 0\}.
\end{equation}

The weight $w_k$ encodes the \emph{residual} evidence at order $k$:
the part of the joint count that is not explained by order $k{-}1$.
\end{definition}

\begin{proposition}[The forward pass recovers KN]
\label{prop:forward-pass}
With appropriate weights, the UM forward pass over $P$ recovers the
interpolated KN prediction.  The $\max$ operation in the forward pass
selects the best-supported syllogism at each order; the hierarchical
structure of the weights (residuals at each order) ensures that
evidence is not double-counted.

Specifically, the prediction for output $o$ given context $c$ is:
\begin{equation}
t_{\mathrm{out}}(o) = \max_k \min(t_{\mathrm{in}}(c_k), w_k(c_k, o)),
\end{equation}
where $t_{\mathrm{in}}(c_k) = 255$ (the context is observed with
certainty) and the $\max$ ranges over all orders that have a pattern
for this context-output pair.
\end{proposition}

\begin{remark}[The $\max$ vs interpolation gap]
The UM forward pass uses $\max$ (take the best syllogism); KN uses
interpolation (weighted average over all orders).  These are not the
same.  The $\max$ operation selects the single best-supported order
for each output $o$, while KN averages over all orders.

For outputs that are well-supported at high order, $\max$ and
interpolation agree (the high-order evidence dominates).  For outputs
supported only at low order, $\max$ gives the low-order weight
directly, while KN gives a weighted average that includes zero
contributions from higher orders.

The gap between $\max$ and interpolation is the ``tropical--integer
gap'' of~\cite{bayes-counting}: the tropical semiring's $\max$
approximates the integer arithmetic's sum.  For compression, the
gap matters: interpolation spreads probability mass more evenly,
which is better for arithmetic coding.

This gap is the first implementation question: can we close it by
using the log-stochastic form (sum of log-supports, i.e., the
standard softmax) instead of the tropical $\max$-$\min$?  The
answer should be yes---the log-stochastic UM forward pass uses
$\log \sum \exp$ in place of $\max$, recovering the interpolated
form.
\end{remark}


\section{Extending with Lexical Events}\label{sec:lexical}

Once KN lives inside the UM, extending it with lexical events
becomes natural.

\begin{proposition}[KN + words in a single $P$]
The extended event space~\cite{extended-es} adds position,
accumulator, bag-of-letters, and word events to $H$.  The pattern
table grows by the four families of new patterns~\cite{pattern-space}:
recognition, prediction, language-model, and structural.

The KN patterns ($n$-gram byte contexts $\to$ output bytes) coexist
with the word-level patterns (word identity $\to$ predicted bytes,
word bigrams $\to$ word identity) in a single $P$.  The forward pass
combines them via the hierarchical combination rule:
\begin{enumerate}
\item Within-word prediction: the word-level patterns provide strong
  evidence for the remaining bytes once the word is identified.  This
  evidence is \emph{independent} of the byte-level KN patterns at
  distant offsets (different information sources).
\item Cross-word prediction: word bigram patterns provide evidence for
  the next word's identity, which then cascades into byte-level
  predictions via the prediction patterns.  This evidence is
  independent of within-word byte patterns (the word boundary is a
  Markov blanket~\cite{pattern-space}).
\item Byte-level fallback: for positions where no word event is active
  (rare words, non-word tokens, XML markup), the KN patterns provide
  the baseline prediction.
\end{enumerate}

No external mixing is needed.  The combination is internal to the
forward pass over the unified $P$.
\end{proposition}


\section{Research Agenda}\label{sec:agenda}

\begin{enumerate}
\item \textbf{Compute the GCD distribution.}  For our existing
  order-6 byte count table on enwik9, compute $g(c)$ for every
  context $c$.  Characterize the distribution.  Verify that most
  rows have $g(c) = 1$ and that the global $D^* = 0.9$ tracks the
  empirical GCD.

\item \textbf{Per-row discount.}  Implement a version of KN that uses
  $D = g(c)$ per row (integer GCD discount) instead of a global $D$.
  Compare bpc against standard KN.  This tests whether the quotient
  decomposition is a better discount than the global estimate.

\item \textbf{Build the unified pattern table.}  Construct $P$ as a
  hash table of (context, output, weight) triples, with weights set to
  the residual log-counts at each order.  Implement the forward pass
  as a lookup + $\max$ over orders.  Compare against the procedural KN
  implementation.

\item \textbf{Log-stochastic forward pass.}  Implement $\log \sum \exp$
  in place of $\max$ to close the tropical--integer gap.  This should
  recover interpolated KN exactly (or within numerical precision).

\item \textbf{Add word events.}  Inject the top-$N$ words as events
  in $H'$, with recognition, prediction, and language-model
  patterns.  Measure $\Delta$bpc from the word-level patterns above
  the KN baseline.  This is the first test of whether the unified
  approach beats the best ad-hoc mixture (1.634\,bpc from
  match-model v12~\cite{match-model}).

\item \textbf{Resolve the discount--GCD gap.}  Work out the exact
  relationship between the two-sided GCD decomposition $(g_I, g_O)$
  and the KN discount/continuation-count pair $(D, c_{\mathrm{KN}})$.
  This is the theoretical core.
\end{enumerate}


\section{Discussion}

\subsection{Why this matters}

The combination problem has been our bottleneck since the match-model
paper~\cite{match-model}: five of nine mixture strategies made things
worse, and the best valid strategy gained only 0.05\,bpc.  The root
cause is that external mixing is the wrong approach---it is the same
mistake as the \texttt{the\_inject} experiments~\cite{extended-es},
applied to the counting model rather than the lexical model.

Pulling KN into the UM dissolves the combination problem.  Models
don't need to be ``combined''---they are all patterns in a single $P$,
and the forward pass handles combination natively.  The discount is not
an ad-hoc parameter but a quotient operation that separates residual
from common evidence.  The continuation count is not a heuristic but a
column-side type quotient measuring output generality.

\subsection{The path to 1.0\,bpc}

Our current best is 1.634\,bpc (KN-6 + extended match, logistic
mixer~\cite{match-model}).  The gap to 1.0\,bpc decomposes as:
\begin{itemize}
\item $\sim$0.08\,bpc: hash table saturation and higher order
  (addressable by larger HT or better data structure).
\item $\sim$0.10\,bpc: word-level patterns (addressable by the
  extended event space with word events in $H'$).
\item $\sim$0.45\,bpc: longer-range dependencies, syntax, semantics
  (requires the factorization tower beyond word bigrams).
\end{itemize}
The UM-native KN + word events should capture the first two
components, reaching $\sim$1.45\,bpc.  The third requires deeper
nesting (the factorization tower~\cite{nested-model}) and is the
subject of future work.


\begin{thebibliography}{99}
\bibitem{cmp}
Michaeljohn Clement.
\newblock \emph{CMP}.
\newblock \url{https://cmpr.ai/cmp.pdf}, 2026.

\bibitem{kn-scaling}
Claude and MJC.
\newblock \emph{Scaling Byte-Level Kneser--Ney to 1.78\,bpc on enwik9}.
\newblock Hutter archive, 12 Feb 2026.

\bibitem{match-model}
Claude and MJC.
\newblock \emph{Match Models and the Combination Problem}.
\newblock Hutter archive, 16 Feb 2026.

\bibitem{bayes-counting}
Claude and MJC.
\newblock \emph{Bayes from Counting: Partial Quotients, GCD, and the
Symmetric Learning Function on $E = I \times O$}.
\newblock Hutter archive, 12 Feb 2026.

\bibitem{pattern-space}
Claude and MJC.
\newblock \emph{Patterns in the Extended Event Space: Independence,
Correlation, and the New Synapses}.
\newblock Hutter archive, 15 Feb 2026.

\bibitem{extended-es}
Claude and MJC.
\newblock \emph{The Extended Event Space: Injecting Lexical Structure
into $H$}.
\newblock Hutter archive, 15 Feb 2026.

\bibitem{nested-model}
Claude and MJC.
\newblock \emph{The Nested Model: Self-Similar Architecture in the
Extended Event Space}.
\newblock Hutter archive, 15 Feb 2026.
\end{thebibliography}

\end{document}
