\documentclass[11pt]{article}
\usepackage[margin=1in]{geometry}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{booktabs}
\usepackage{hyperref}

\newtheorem{observation}{Observation}

\title{Q1 Exact Results: f32 vs MPFR-256 at $t = 42$}
\author{Claude and MJC}
\date{11 February 2026}

\begin{document}
\maketitle

\begin{abstract}
We ran Programs 1--4 from Q1 Exact, comparing the sat-rnn's f32
forward pass and BPTT backward trace against MPFR-256 exact arithmetic.
The results are more dramatic than predicted: the gradient diverges
at $d = 1$ (not $d \approx 10$), the mantissa carries zero information
in the backward pass, and the f32 error ratio stabilizes at
$\|g^{\text{f32}} - g^{\text{exact}}\| / \|g^{\text{exact}}\| \approx 3.44$
---a dynamical constant of the Jacobian. Bit-level sampling confirms
the sign bit carries $52\times$ more leverage per bit than the exponent,
and the exponent carries $52\times$ more than the mantissa. The mantissa
is not overengineered: it is the memory.
\end{abstract}

\section{Program 1: Forward Pass}

Hidden state $h_{42}$ after 42 exact timesteps vs f32:

\begin{center}
\begin{tabular}{lr}
  \toprule
  Metric & Value \\
  \midrule
  Sign agreement & 128/128 \\
  Exponent agreement & 124/128 \\
  Mean mantissa bits & 21.5 / 23 \\
  Max absolute error & $4.74 \times 10^{-1}$ \\
  Mean absolute error & $8.30 \times 10^{-3}$ \\
  Max relative error & $1.23$ \\
  Near-zero neurons & 0 \\
  \bottomrule
\end{tabular}
\end{center}

\paragraph{Saturation distribution.}
$|h| \geq 0.95$: 125 neurons. $|h| \in [0.5, 0.75)$: 1.
$|h| \in [0.25, 0.5)$: 1. Only 3 neurons are unsaturated.

\paragraph{Exponent distribution.} 112 neurons have $E = 127$
(value in $[1, 2)$---i.e.\ $|h| = 1.0$ exactly in f32).
16 neurons have $E = 126$ (value in $[0.5, 1)$). Two exponent
values cover the entire hidden state.

\begin{observation}[Forward pass preserves structure]
After 42 steps, sign and exponent are essentially preserved
(128/128 sign, 124/128 exponent). The 4 exponent disagreements
come from neurons near the $|h| = 1.0$ boundary where
$\text{f32}(\tanh) = 1.0$ but $\text{exact}(\tanh) = 0.999\ldots$.
Mantissa has 21.5/23 bits agreement---the forward pass is well-behaved.
\end{observation}

\paragraph{Output predictions.}
$P_{\text{f32}}(\texttt{x}) = 9.79 \times 10^{-5}$,
$P_{\text{exact}}(\texttt{x}) = 8.33 \times 10^{-5}$.
$\text{bpc}_{\text{f32}} = 13.32$,
$\text{bpc}_{\text{exact}} = 13.55$.
$D_{\text{KL}}(\text{exact} \| \text{f32}) = 5.8 \times 10^{-3}$ bits.

Top predictions agree: both rank \texttt{t}, \texttt{[}, \texttt{n},
\texttt{a}, space as the top 5, with probabilities within 2\% of
each other (except \texttt{[}: 8.6\% f32 vs 10.7\% exact).

\section{Program 2: Output Gradient}

The gradient $g_{42} = \partial \log P(y) / \partial h$ is where
f32 error first becomes significant:

\begin{center}
\begin{tabular}{lr}
  \toprule
  Metric & Value \\
  \midrule
  Sign agreement & 128/128 \\
  Exponent agreement & 111/128 \\
  Mean mantissa bits & \textbf{4.8} / 23 \\
  $\|g_{\text{f32}}\|$ & 1.878 \\
  $\|g_{\text{exact}}\|$ & 1.886 \\
  $\|g_{\text{f32}} - g_{\text{exact}}\| / \|g_{\text{exact}}\|$ & 0.037 \\
  \bottomrule
\end{tabular}
\end{center}

\begin{observation}[Softmax destroys mantissa]
The hidden state has 21.5 mantissa bits of agreement; the gradient
has 4.8 bits. The softmax (256-term sum, subtraction for the gradient)
destroys $\sim$17 mantissa bits in a single operation. Sign is still
perfect (128/128), and the overall relative error is only 3.7\%, but
the per-neuron mantissa is devastated.
\end{observation}

Gradient magnitudes: 78/128 neurons in $[0.1, 0.5)$, 48 in
$[0.01, 0.1)$, 2 in $[0.001, 0.01)$, none above 0.5 or below 0.001.

\section{Programs 3+4: BPTT Trace}

The backward Jacobian trace from $d = 1$ to $d = 42$:

\begin{center}
\small
\begin{tabular}{rrrrrrrr}
  \toprule
  $d$ & sign & exp & mant & $\|g^{\text{f32}}\|$ & $\|g^{\text{ex}}\|$ & $\|{\Delta}\|/\|g^{\text{ex}}\|$ & gates \\
  \midrule
  0  & 128 & 111 & 4.8 & 1.88 & 1.89 & 0.037 & --- \\
  1  & 90 & 23 & 0.7 & 4.73 & 7.02 & 0.80 & 14 \\
  2  & 105 & 30 & 1.1 & 8.82 & 15.5 & 0.58 & 14 \\
  3  & 128 & 1 & 0.8 & 39.0 & 90.1 & 0.57 & 9 \\
  5  & 59 & 30 & 0.6 & 51.6 & 37.1 & 1.66 & 9 \\
  7  & 19 & 21 & 0.2 & 116 & 54.1 & 3.07 & 10 \\
  10 & 1 & 3 & 0.0 & 596 & 248 & 3.41 & 10 \\
  15 & 0 & 0 & 0.0 & $8.7 \times 10^5$ & $3.6 \times 10^5$ & 3.43 & 16 \\
  20 & 0 & 0 & 0.0 & $3.6 \times 10^6$ & $1.5 \times 10^6$ & 3.44 & 7 \\
  30 & 0 & 0 & 0.0 & $1.9 \times 10^3$ & $7.9 \times 10^2$ & 3.44 & 11 \\
  42 & 0 & 0 & 0.0 & $2.0 \times 10^5$ & $8.5 \times 10^4$ & 3.41 & 27 \\
  \bottomrule
\end{tabular}
\end{center}

\begin{observation}[Error ratio converges to a dynamical constant]
The relative error $\|\Delta g\| / \|g^{\text{exact}}\|$ converges
to $\approx 3.44$ by $d \approx 7$ and stays there through $d = 42$.
This is not a random walk---it is a fixed-point property of the
Jacobian dynamics. The f32 error vector has aligned with the dominant
backward direction.
\end{observation}

\begin{observation}[Sign and exponent die simultaneously]
Sign agreement drops from 128/128 (at $d = 0$) to 90 (at $d = 1$)
to 0 (at $d = 11$). Exponent agreement drops from 111 to 0 over
the same range. There is no ``exponent survives deeper'' hierarchy
in the backward pass. All three channels die together because the
Jacobian mixes them at every step.
\end{observation}

\begin{observation}[Mantissa is zero from $d = 1$]
Mean mantissa bits of agreement: 0.7 at $d = 1$, 0.0 from $d = 8$
onward. The mantissa decorrelates in a single BPTT step. This
confirms the channel coupling: the 128-term dot product in the
Jacobian immediately mixes mantissa noise into all channels.
\end{observation}

\begin{observation}[Exponent bits partially stable]
Despite zero mantissa and sign agreement, $\sim$4--5 of 8 exponent
bits remain stable (agree between f32 and exact) through $d = 42$.
These are the high-order exponent bits that encode the \emph{scale}
of the gradient---not the direction, not the magnitude to precision,
just the order-of-magnitude. The pattern exponents survive.
\end{observation}

\section{Bit-Level Sampling}

Flipping each bit in each neuron at $t = 42$ and measuring the KL
divergence of the output distribution:

\begin{center}
\begin{tabular}{llrrr}
  \toprule
  Bits & Channel & Mean KL (bits) & Per bit & Flips $> 0.1$ bpc \\
  \midrule
  0--4 & mantissa (low) & $< 10^{-6}$ & $< 10^{-7}$ & 0 \\
  5--14 & mantissa (mid) & $< 10^{-4}$ & $< 10^{-5}$ & 0 \\
  15--22 & mantissa (high) & 0.0035 & 0.00044 & 61 \\
  23--30 & exponent & 0.064 & 0.0080 & $\sim$90 \\
  31 & sign & 0.046 & 0.046 & 110 \\
  \bottomrule
\end{tabular}
\end{center}

\begin{observation}[Per-bit leverage: $52\times$ hierarchy]
Each exponent bit carries $52\times$ the leverage of each mantissa
bit ($0.0080 / 0.00015 = 52$). The sign bit carries $5.7\times$ the
leverage of each exponent bit ($0.046 / 0.0080 = 5.7$). The total
hierarchy is sign : exponent : mantissa $= 300 : 52 : 1$ per bit.
\end{observation}

\begin{observation}[Mantissa leverage doubles per bit position]
Mean bpc change from flipping bit $b$: $0.000001 \times 2^{b-5}$ for
$b = 5, \ldots, 22$. The mantissa has perfect geometric scaling:
each bit is exactly $2\times$ the leverage of the bit below it.
This is the ``mantissa is memory'' structure---each bit carries a
well-defined quantum of information, and the 23 bits together form
a 23-level hierarchy.
\end{observation}

\paragraph{Propagation through time.}
Flipping the sign bit of neuron 8 ($h = 0.855$, unsaturated) causes:

\begin{center}
\begin{tabular}{lrrrr}
  \toprule
  $t$ & bit 0 KL & bit 11 KL & bit 22 KL & bit 31 KL \\
  \midrule
  43 & 0 & 0 & 0.0004 & 0.125 \\
  44 & 0 & 0 & 0.000001 & 0.334 \\
  46 & 0 & 0 & 0.00005 & 0.729 \\
  48 & 0 & 0 & 0.00002 & 0.329 \\
  50 & 0 & 0 & 0.0001 & 0.328 \\
  \bottomrule
\end{tabular}
\end{center}

The sign bit propagates and \emph{amplifies} ($0.125 \to 0.729$ by
$t = 46$). The MSB mantissa (bit 22) stays at $\sim 10^{-4}$.
The LSB (bit 0) and bit 11 are zero at all times.

\begin{observation}[The mantissa is the memory]
The 23 mantissa bits look overengineered for a single prediction
(bits 0--14 contribute $< 10^{-4}$ to KL). But through the
forward dynamics, they are the memory: the state at $t + 50$
depends on mantissa bits at $t$, integrated through $\sim$50
Jacobian steps. The mantissa bits do not propagate as individual
bits---they propagate as a collective through the mixing of the
Jacobian, where they can become exponent and sign information at
later timesteps. This is the entropy smuggling.
\end{observation}

\section{$H = 2^{32}$: The State Space}

Setting $H = 2^{32}$ per neuron, the entire hidden state is an
element of $H^{128}$. At $t = 42$:

\begin{itemize}
  \item 112/128 neurons have exponent 127 ($|h| \in [1, 2)$, i.e.\
    exactly $\pm 1.0$ in f32). 16 neurons have exponent 126
    ($|h| \in [0.5, 1)$).
  \item Only 17 unique f32 bit patterns among the 128 neurons.
  \item Sign: 68 positive, 60 negative.
\end{itemize}

The state space is nominally $2^{32 \times 128} = 2^{4096}$ but the
$\tanh$ constraint and saturation reduce the effective state to
$\sim$26 bits/neuron $\times$ 128 neurons $= 3328$ bits. The
sign/exponent/mantissa factorization is one decomposition of this
space; the dynamical factorization (stable/unstable bits under the
Jacobian) is another. They diverge after one BPTT step.

The weight matrices define an immutable map on this space. Training
selected this particular map from the $\epsilon$-field of random
initialization.

\section{Lyapunov Structure Across Positions}

The 3.44 ratio at $t = 42$ is not universal. Running the same analysis
at d=10 BPTT depth across all positions reveals three regimes:

\begin{center}
\small
\begin{tabular}{rrrrrrr}
  \toprule
  $t$ & $h$ mant & $g_0$ mant & ratio $d{=}0$ & ratio $d{=}10$ & $\cos(g^{f32}, g^{ex})$ & $\|g^{f32}\| / \|g^{ex}\|$ \\
  \midrule
  10 & 22.9 & 22.5 & 0.00 & 0.01 & +1.000 & 0.99 \\
  20 & 23.0 & 22.2 & 0.00 & 0.00 & +1.000 & 1.00 \\
  35 & 22.0 & 10.2 & 0.00 & 0.04 & +1.000 & 0.96 \\
  45 & 21.7 & 5.7 & 0.02 & 1.08 & $-0.694$ & 0.11 \\
  60 & 18.0 & 1.9 & 0.32 & 1.00 & $-0.082$ & 0.03 \\
  100 & 6.1 & 1.6 & 0.42 & 1.00 & +0.050 & 0.00 \\
  200 & 10.4 & 1.1 & 0.56 & 1.00 & $-0.229$ & 0.00 \\
  500 & 14.0 & 2.1 & 0.29 & 1.00 & $-0.131$ & 0.00 \\
  1000 & 14.5 & 2.2 & 0.21 & 0.99 & +0.197 & 0.04 \\
  \bottomrule
\end{tabular}
\end{center}

Three regimes:
\begin{enumerate}
  \item \textbf{Agreement ($t < 40$):} $\cos = +1.0$, ratio $\approx 0$.
    f32 and exact track perfectly. The forward states agree to 22+
    mantissa bits, the gradient to 10+ bits.

  \item \textbf{Transition ($t \approx 40$--$60$):} Cosine drops from
    $+1$ to $\sim 0$. The 3.44 ratio at $t = 42$ is a transient of this
    transition---the f32 error vector temporarily anti-aligns with the
    exact gradient.

  \item \textbf{Decorrelation ($t > 60$):} $\cos \approx 0$, ratio
    $\approx 1.0$. The f32 and exact gradients at $d = 10$ are
    \emph{uncorrelated}. They point in random directions relative to
    each other. The norm ratio $\|g^{f32}\| / \|g^{ex}\|$ fluctuates
    wildly (0.00 to $> 10^6$)---at some positions the exact gradient
    is near zero while f32 gives finite values, or vice versa.
\end{enumerate}

\begin{observation}[BPTT gradients are f32 noise after $t \approx 60$]
At depth $d = 10$, the f32 and exact gradients share zero mutual
information beyond $t \approx 60$. Cosine $\approx 0$ means the
128-dimensional gradient vectors point in completely different
directions. Yet the model learns ($0.079$ bpc). This means either:
(a) the output gradient at $d = 0$ contains enough signal, or
(b) the Adam optimizer extracts signal from the f32 noise's
statistical structure across positions.
\end{observation}

\section{Program 5: Pattern Attributions}

\paragraph{W\_y (output layer).}
Spearman rank correlation between f32 and exact: $\rho = 0.997$.
At $\tau = 0.01$: 125 agree, 0 f32-only, 1 exact-only.

\paragraph{W\_x (input layer, by BPTT depth).}

\begin{center}
\begin{tabular}{rrrrrr}
  \toprule
  $d$ & $\rho$ & agree & f32-only & exact-only & mean ratio \\
  \midrule
  1  & +0.52 & 126 & 0 & 2 & 0.69 \\
  2  & +0.74 & 128 & 0 & 0 & 0.61 \\
  5  & +0.50 & 128 & 0 & 0 & 1.30 \\
  8  & +0.91 & 128 & 0 & 0 & 2.53 \\
  10 & +0.99 & 128 & 0 & 0 & 2.40 \\
  11 & +1.00 & 128 & 0 & 0 & 2.44 \\
  15 & +1.00 & 128 & 0 & 0 & 2.43 \\
  20 & +1.00 & 128 & 0 & 0 & 2.44 \\
  \bottomrule
\end{tabular}
\end{center}

\begin{observation}[Pattern ranking is perfect at depth]
At $d \geq 11$, the Spearman correlation between f32 and exact
attributions is \textbf{exactly 1.000}: the ranking is perfectly
preserved. There are zero phantom patterns at any depth.
The f32 quotient preserves the topology of the pattern space
(which patterns matter, in what order) while distorting the
magnitudes by a factor of $\sim$2.44.
\end{observation}

\begin{observation}[Pattern exponents are fine]
The magnitudes diverge ($\text{mean}_{\text{f32}} / \text{mean}_{\text{exact}}
\to 2.44$ at depth), but the \emph{exponents} of the attributions---the
importance ranking---are perfectly preserved. This is because the
f32 error vector aligns with the dominant backward direction (the
same one that carries the actual gradient). The error is proportional
to the signal, not orthogonal to it. At depth, f32 is wrong about
``how much'' but right about ``what matters.''
\end{observation}

\section{Program 6: Entropy of the Quotient}

Full forward pass in f32 and MPFR-256 over all 1023 positions:

\begin{center}
\begin{tabular}{lrrrr}
  \toprule
  Phase & bpc (f32) & bpc (exact) & diff & $n$ \\
  \midrule
  $t < 50$ & 6.612 & 6.642 & $-0.031$ & 50 \\
  $50$--$200$ & 6.805 & 6.652 & $+0.153$ & 150 \\
  $200$--$500$ & 5.469 & 5.351 & $+0.118$ & 300 \\
  $500$+ & 5.469 & 5.439 & $+0.031$ & 523 \\
  \midrule
  Overall & 5.721 & 5.650 & $+0.071$ & 1023 \\
  \bottomrule
\end{tabular}
\end{center}

\begin{observation}[f32 costs 0.071 bpc]
The f32 quotient adds $+0.071$ bpc relative to exact arithmetic.
The hidden states diverge completely after $t \approx 45$ (max error
$= 2.0$, i.e.\ full sign flips), yet the bpc cost is modest because:
(a)~the sign vector $\mathrm{sgn}(h_t)$ is the dominant information
channel, and many sign bits happen to agree between f32 and exact
even when the mantissa has decorrelated; (b)~the exact model's bpc
of 5.65 shows the trained weights are suboptimal in exact
arithmetic---the training in f32 found a local minimum that is
tuned to the f32 dynamics, not the exact ones.
\end{observation}

\begin{observation}[f32 bpc is lower in phase 1]
For $t < 50$, f32 is 0.031 bpc \emph{better} than exact
($6.612$ vs $6.642$). This is not noise: the model was trained in
f32, so it optimized for the f32 dynamics. The exact computation
is a different dynamical system that the model was not trained on.
\end{observation}

\section{Summary of Predictions vs Results}

\begin{center}
\begin{tabular}{lll}
  \toprule
  Prediction & Expected & Actual \\
  \midrule
  P1: Sign agreement & 128/128 & \textbf{128/128} \checkmark \\
  P1: Exponent agreement & 128/128 & 124/128 (close) \\
  P1: Mantissa bits & $\sim$18/23 & 21.5/23 (better) \\
  P2: Gradient sign & 128/128 & \textbf{128/128} \checkmark \\
  P2: Gradient mantissa & 15--18 bits & \textbf{4.8 bits} (much worse) \\
  P3: Gates killed & 60--80 & \textbf{114} ($= 128 - 14$) \\
  P4: Sign dies at $d \approx$ & 10--15 & \textbf{7} (faster) \\
  P4: Mantissa decay & logarithmic & \textbf{instant} (0.7 at $d{=}1$) \\
  P4: Error ratio & growing & \textbf{3.44 (transient), 1.0 (steady)} \\
  \bottomrule
\end{tabular}
\end{center}

The predictions underestimated how fast the backward pass decorrelates.
The key surprise is not a constant ratio but a \emph{phase transition}:
before $t \approx 40$, f32 BPTT is exact; after $t \approx 60$, it is
pure noise. The transition is sharp (20 positions) and the 3.44 ratio
at $t = 42$ is a transient phenomenon during this transition.

The gradient at $d = 0$ (output layer, no BPTT) has ratio $\sim$0.3
at steady state: 30\% error, 70\% signal. This is where learning
actually happens. BPTT at depth 10+ contributes noise, not signal.

\end{document}
