\documentclass[11pt]{article}
\usepackage[margin=1in]{geometry}
\usepackage{amsmath,amssymb,amsthm,booktabs}
\usepackage{hyperref}

\newtheorem{theorem}{Theorem}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{example}[theorem]{Example}
\newtheorem{remark}[theorem]{Remark}

\title{Information Geometry of the Count Table:\\
The Statistical Manifold of the Universal Model}
\author{Claude and MJC}
\date{February 12, 2026}

\begin{document}
\maketitle

\begin{abstract}
The count table $c: I \times O \to \mathbb{N}$, normalized to a probability
distribution, defines a point on the statistical manifold of distributions
over $I \times O$.  We study the geometry of this manifold equipped
with the Fisher information metric.  The conditional distributions
$P(o \mid i)$ form an $|I|$-parameter exponential family.  The natural
parameters are the log-odds ratios $\log(c(i,o)/c(i,o'))$, which are
the pattern strengths of the UM.  The Fisher metric on this family
measures how distinguishable two models are from finite data.
We show: (1) the KL divergence between two count tables equals the
relative entropy, connecting to the UM's cross-entropy loss;
(2) the geodesics of the Fisher metric correspond to exponential
interpolations between models, which is the EM algorithm's update path;
(3) the tock step moves along a \emph{transverse} direction (changing
the manifold itself, not the point on it), and the tick step moves
along a geodesic (updating the point).  The geometry explains why
KN smoothing works (it moves the estimate toward the ``center'' of
the manifold) and why the R\textsuperscript{2} $\approx$ 0.83
invariant emerges (it is a geometric property of the
Fisher metric on high-dimensional exponential families).
\end{abstract}


\section{The Statistical Manifold}

\begin{definition}[Count table manifold]
For event space $E = I \times O$ with $|I| = m$ and $|O| = n$,
the \emph{count table manifold} is:
\[
\mathcal{M} = \left\{ P \in \mathbb{R}^{m \times n}_+ : \sum_o P(i,o) = P(i,\cdot) > 0 \;\forall i, \;\sum_{i,o} P(i,o) = 1 \right\}.
\]
This is the interior of the probability simplex $\Delta^{mn-1}$
restricted to distributions with full support on $I$.
\end{definition}

\begin{definition}[Fisher information metric]
The \emph{Fisher information metric} on $\mathcal{M}$ is:
\[
g_{\mu\nu}(\theta) = E_\theta\left[ \frac{\partial \log P(x;\theta)}{\partial \theta_\mu} \cdot \frac{\partial \log P(x;\theta)}{\partial \theta_\nu} \right],
\]
where $\theta$ are the parameters of the distribution.
\end{definition}

\begin{proposition}[Exponential family structure]
The conditional distributions $\{P(o \mid i)\}_{i \in I}$ form an
exponential family with natural parameters
$\eta_{io} = \log P(o \mid i) - \log P(n \mid i)$ (log-odds relative
to a reference event $n$) and sufficient statistics
$T_{io}(x) = \mathbf{1}[X_I = i, X_O = o]$ (event indicators).
\end{proposition}


\section{The Pattern Matrix as Natural Parameters}

\begin{proposition}[Patterns are log-odds]
The UM's pattern strengths, in the continuous (non-SN-quantized) limit,
are:
\[
p_{ij} \propto \log \frac{c(i,j)}{c(i,\cdot)} = \log P(j \mid i).
\]
The log-conditional probabilities are the natural parameters of the
exponential family.  The pattern matrix IS the natural parameterization
of the statistical manifold.
\end{proposition}

\begin{corollary}
The forward pass in the continuous limit:
\[
(f_p(t))_j = \max_i (t_i + p_{ij})
\]
(where $t_i = \log P(i)$ and $p_{ij} = \log P(j \mid i)$) computes
the MAP estimate on the joint manifold:
\[
\hat{o} = \arg\max_j \max_i \log P(i,j) = \arg\max_j \max_i (t_i + p_{ij}).
\]
\end{corollary}


\section{KL Divergence and Cross-Entropy}

\begin{proposition}[KL = relative entropy = excess loss]
The KL divergence between the true distribution $P$ and the model's
estimate $\hat{P}$ is:
\[
D_{\text{KL}}(P \| \hat{P}) = \sum_{i,o} P(i,o) \log \frac{P(i,o)}{\hat{P}(i,o)} = H(P, \hat{P}) - H(P),
\]
where $H(P, \hat{P})$ is the cross-entropy (the UM's loss in bpc)
and $H(P)$ is the entropy (the irreducible loss).
\end{proposition}

\begin{remark}
$D_{\text{KL}}$ measures the ``distance'' from $\hat{P}$ to $P$ on the
statistical manifold.  It is not a true distance (not symmetric, doesn't
satisfy triangle inequality) but it is the Bregman divergence associated
with the negative entropy, which makes it the natural ``distance'' for
exponential families.
\end{remark}

\begin{proposition}[The Pythagorean theorem]
For the exponential family of count-table distributions, the KL
divergence satisfies a Pythagorean identity.  If $P$ is the true
distribution, $\hat{P}_{\text{MLE}}$ is the MLE (the empirical
distribution from counting), and $\hat{P}_{\text{smooth}}$ is a
smoothed estimate (e.g., KN smoothing), then:
\[
D_{\text{KL}}(P \| \hat{P}_{\text{smooth}}) =
D_{\text{KL}}(P \| \hat{P}_{\text{MLE}}) +
D_{\text{KL}}(\hat{P}_{\text{MLE}} \| \hat{P}_{\text{smooth}}).
\]
Smoothing ALWAYS increases the KL divergence from the truth, but
it may decrease the \emph{expected} KL divergence over finite
samples (bias-variance trade-off).
\end{proposition}


\section{Geodesics and the EM Path}

\begin{proposition}[Geodesics in the Fisher metric]
The geodesics on $\mathcal{M}$ (with the Fisher metric) are
\emph{exponential geodesics}: they interpolate between two distributions
by exponentially weighting their log-probabilities:
\[
P_\lambda(i,o) \propto P_0(i,o)^{1-\lambda} \cdot P_1(i,o)^\lambda,
\quad \lambda \in [0, 1].
\]
\end{proposition}

\begin{proposition}[EM updates follow geodesics]
The EM algorithm (tick-tock iteration, per the fixed-point paper)
updates the distribution along an exponential geodesic in the
natural parameter space.  Each E-step (counting) computes the expected
sufficient statistics; each M-step (event space optimization) moves
to the maximum-likelihood point.  The M-step path is a geodesic in
the $e$-flat (natural parameter) geometry.
\end{proposition}


\section{Tick vs.\ Tock on the Manifold}

\begin{proposition}[Tick = motion on the manifold]
The tick step (updating the count table from new data) moves the
model's position on the manifold $\mathcal{M}$ without changing the
manifold itself.  As data accumulates, the position converges to the
true distribution $P$:
\[
\hat{P}_N \xrightarrow{N \to \infty} P \quad \text{on } \mathcal{M}.
\]
\end{proposition}

\begin{proposition}[Tock = change of manifold]
The tock step (changing the event space) replaces the manifold
$\mathcal{M}$ with a different manifold $\mathcal{M}'$ (corresponding
to the new event space).  This is a \emph{transverse} motion: it
changes the space of possible distributions, not the position within
a fixed space.
\end{proposition}

\begin{remark}
The tick-tock distinction maps to the distinction between:
\begin{itemize}
\item \textbf{Estimation} (finding the best point on a fixed manifold):
  this is statistics (tick).
\item \textbf{Model selection} (choosing the right manifold): this is
  architecture (tock).
\end{itemize}
The Fisher metric is relevant for estimation (it measures how hard it
is to distinguish nearby points).  For model selection, the relevant
geometry is the ``manifold of manifolds''---the lattice of event spaces
with the MI ordering.
\end{remark}


\section{KN Smoothing as Geometric Centering}

\begin{proposition}[KN smoothing as shrinkage]
KN smoothing with discount $\delta$ shrinks the empirical distribution
toward the lower-order distribution:
\[
P_{\text{KN}}(o \mid i) = \frac{\max(c(i,o) - \delta, 0)}{\sum_{o'} c(i,o')} + \gamma(i) \cdot P_{\text{lower}}(o),
\]
where $\gamma(i)$ is a normalization factor.  On the statistical manifold,
this moves the estimate from the MLE toward the ``center'' (uniform or
lower-order distribution).
\end{proposition}

\begin{remark}
Geometrically, KN smoothing is a form of \emph{regularization}: it
prevents the estimate from being too close to the boundary of the
simplex (where some probabilities are zero).  The $\delta$ parameter
controls the shrinkage amount.  The ``no support $\neq$ disbelief''
principle says: points on the boundary (zero probabilities) are
\emph{not} valid model positions, because zero probability means
``certainty of non-occurrence,'' which requires positive evidence for
the alternative.  KN smoothing keeps the model in the interior of the
manifold, where the Fisher metric is well-defined and ``no support''
remains distinguishable from ``disbelief.''
\end{remark}


\section{The R\textsuperscript{2} Invariant as a Geometric Property}

\begin{proposition}[R\textsuperscript{2} from Fisher metric]
The conjunction R\textsuperscript{2} $\approx$ 0.83 for the 128-hidden
tanh RNN is a geometric property of the Fisher metric on a
128-dimensional Gaussian manifold.

The argument (from the conjunction invariant paper):
\begin{enumerate}
\item Each neuron $h_k$ is approximately the product of two independent
  sub-Gaussian variables (the contributions from offsets $d_1$ and $d_2$).
\item The R\textsuperscript{2} of the best 2-factor fit depends on the
  ratio of the order statistics of $|h_k|$ to the norm $\|h\|$.
\item For a 128-dimensional Gaussian, the expected max-to-norm ratio is
  $\sqrt{2 \log 128}/\sqrt{128} = \sqrt{14}/\sqrt{128} \approx 0.331$.
\item $R^2 = \mathrm{erf}^2(\sqrt{2 \log H}/\sqrt{H})$.  At $H = 128$:
  $R^2 \approx 0.83$.
\end{enumerate}
This ratio is a property of the Fisher metric on the Gaussian manifold:
it measures how ``peaked'' the distribution of neuron activations is,
which depends on the dimension (128) and the geometry (Gaussian).
\end{proposition}


\section{The Tropical Metric}

\begin{definition}[Tropical distance]
On the support lattice $\{0, \ldots, 255\}^n$, define the
\emph{tropical distance}:
\[
d_{\text{trop}}(s, s') = \max_i |s_i - s'_i|.
\]
This is the $\ell^\infty$ metric---the maximum difference in support
across all events.
\end{definition}

\begin{proposition}[Forward pass is 1-Lipschitz]
The forward pass is 1-Lipschitz in the tropical metric:
\[
d_{\text{trop}}(f_p(t), f_p(t')) \leq d_{\text{trop}}(t, t').
\]
\end{proposition}

\begin{proof}
For each output $j$:
\begin{align}
|(f_p(t))_j - (f_p(t'))_j|
&= |\max_i \min(t_i, p_{ij}) - \max_i \min(t'_i, p_{ij})| \\
&\leq \max_i |\min(t_i, p_{ij}) - \min(t'_i, p_{ij})| \\
&\leq \max_i |t_i - t'_i| = d_{\text{trop}}(t, t').
\end{align}
The first inequality uses $|\max f - \max g| \leq \max |f - g|$.
The second uses the 1-Lipschitz property of $\min(\cdot, p)$.
\end{proof}

\begin{corollary}
The forward pass is a \emph{contraction} in the tropical metric:
small changes in input support produce small changes in output support.
This is the UM's version of ``stability'': the model's predictions
change continuously with the evidence.
\end{corollary}


\section{Discussion}

The information-geometric perspective reveals:

\begin{enumerate}
\item \textbf{The pattern matrix IS the natural parameterization.}
  Log-conditional probabilities are the natural coordinates on the
  statistical manifold.  The UM's pattern strengths are (discretized
  versions of) these coordinates.

\item \textbf{KL divergence IS the UM's loss.}  The cross-entropy loss
  in bpc is the KL divergence plus the entropy.  Minimizing bpc is
  minimizing KL divergence: finding the closest point on the manifold
  to the true distribution.

\item \textbf{KN smoothing IS geometric regularization.}  Smoothing
  keeps the model in the interior of the manifold, where the geometry
  is well-defined and the epistemology of zero (no support $\neq$
  disbelief) is respected.

\item \textbf{The R\textsuperscript{2} invariant IS geometric.}
  The 0.83 value is determined by the dimension and the metric,
  not by the specific data or training procedure.

\item \textbf{The forward pass IS a contraction.}  In the tropical
  metric, predictions change smoothly with evidence.  This is the
  geometric content of the UM's ``caution'': the $\min$ operation
  ensures that predictions never exceed the weakest evidence.
\end{enumerate}


\begin{thebibliography}{99}
\bibitem{cmp}
Michaeljohn Clement.
\newblock \emph{CMP}.
\newblock \url{https://cmpr.ai/cmp.pdf}, 2026.

\bibitem{amari}
Shun-ichi Amari.
\newblock \emph{Information Geometry and Its Applications}.
\newblock Springer, 2016.

\bibitem{conjunction}
Claude and MJC.
\newblock \emph{The Conjunction Invariant: Why $R^2 \approx 0.83$ Is a
Fixed Point}.
\newblock Hutter archive, 12 Feb 2026.

\bibitem{no-support}
Claude and MJC.
\newblock \emph{No Support Is Not Disbelief}.
\newblock Hutter archive, 12 Feb 2026.
\end{thebibliography}

\end{document}
