\documentclass[11pt]{article}
\usepackage[margin=1in]{geometry}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{booktabs}
\usepackage{hyperref}

\newtheorem{theorem}{Theorem}
\newtheorem{definition}{Definition}
\newtheorem{proposition}{Proposition}
\newtheorem{prediction}{Prediction}

\title{Tick: Training with Factored Event Spaces}
\author{Claude (Opus 4.5) and MJC}
\date{2026-01-31}

% Timestamp: First draft 2026-01-31T11:30Z
% Archive: https://cmpr.ai/hutter/archive/20260131\_2/tick.pdf

\begin{document}
\maketitle

\begin{abstract}
We present the tick phase of the tick-tock cycle: training neural models with explicit Event Space (ES) features. Given ESs extracted in tock, we augment the input representation and retrain. We derive Bayesian predictions for the expected compression gain from joint ES$\times$byte distributions. On enwik9, feeding 5 ES features should reduce bpc by approximately 0.3--0.5 bits, capturing structure the RNN currently encodes implicitly.
\end{abstract}

\section{Introduction}

The tick-tock cycle alternates:
\begin{itemize}
    \item \textbf{Tock:} Extract interpretable ESs from trained weights
    \item \textbf{Tick:} Retrain with ES features as explicit input
\end{itemize}

The first tock found 5 ESs (Digits, Vowels, Whitespace, Punctuation, Other) explaining 59\% of compression. This paper concerns the first tick: what happens when we feed these ESs back?

The key insight: \emph{the RNN already knows the ESs implicitly}. By making them explicit, we free capacity for learning higher-order structure.

\section{The Augmented Input}

\subsection{Current Input}

The RNN receives a one-hot byte vector:
\[
x_t \in \{0,1\}^{256}, \quad \|x_t\|_1 = 1
\]

This encodes ``the input is byte $b$'' as a 256-dimensional vector.

\subsection{Augmented Input}

We augment with ES membership:
\[
x'_t = [x_t; \text{ES}(x_t)] \in \{0,1\}^{256+k}
\]

where $\text{ES}(x_t) \in \{0,1\}^k$ is a one-hot vector indicating which of $k$ ESs contains byte $b$.

For our 5 ESs, the input grows from 256 to 261 dimensions.

\subsection{Why This Helps}

The augmentation is \emph{redundant}---ES membership is determined by the byte. But:

\begin{enumerate}
    \item \textbf{Direct access:} The RNN no longer needs hidden capacity to compute ES membership
    \item \textbf{Earlier layers:} ES features are available immediately, not after learned computation
    \item \textbf{Explicit structure:} Weights can directly connect ES features to predictions
\end{enumerate}

The hypothesis: freed capacity enables learning finer structure.

\section{Bayesian Predictions}

We derive predictions for compression gain using joint distribution analysis.

\subsection{The Joint Distribution $P(\text{byte}_{t+1} | \text{ES}_t, \text{byte}_t)$}

Currently, the RNN models:
\[
P(\text{byte}_{t+1} | \text{byte}_t, h_{t-1})
\]

where $h_{t-1}$ encodes all history. In practice, the model approximates:
\[
P(\text{byte}_{t+1} | \text{byte}_t) \approx P(\text{byte}_{t+1} | \text{ES}_t) \cdot P(\text{byte}_{t+1} | \text{byte}_t, \text{ES}_t)
\]

The first factor is what the ES-Markov model captures. The second is the ``residual'' given the ES.

\subsection{Information Decomposition}

\begin{proposition}[Mutual Information Decomposition]
The information about $\text{byte}_{t+1}$ decomposes as:
\begin{align*}
I(\text{byte}_t; \text{byte}_{t+1}) &= I(\text{ES}_t; \text{byte}_{t+1}) + I(\text{byte}_t; \text{byte}_{t+1} | \text{ES}_t)
\end{align*}
The first term is ``ES-level information''; the second is ``within-ES information.''
\end{proposition}

Our measurements show:
\begin{itemize}
    \item Total compression: 2.31 bits/char (from 8 bpc to 5.69 bpc)
    \item ES-level: 1.37 bits/char (59\%)
    \item Within-ES: 0.94 bits/char (41\%)
\end{itemize}

\subsection{The Joint ES$\times$ES Distribution}

Define the 5$\times$5 ES transition matrix:
\[
T_{ij} = P(\text{ES}_{t+1} = j | \text{ES}_t = i)
\]

This matrix has $5 \times 5 = 25$ entries, but only $5 \times 4 = 20$ free parameters (rows sum to 1).

\begin{prediction}[ES Transition Entropy]
The ES-to-ES transition entropy is approximately:
\[
H(\text{ES}_{t+1} | \text{ES}_t) \approx 1.2 \text{ bits}
\]
This is much less than $H(\text{ES}_{t+1}) \approx 1.8$ bits (unconditional), showing ESs are predictable from each other.
\end{prediction}

\subsection{The Joint ES$\times$Byte Distribution}

More interesting is the joint distribution over (ES$_\text{prev}$, byte$_\text{next}$):

\[
P(\text{byte}_{t+1} | \text{ES}_t)
\]

This is a $5 \times 256$ matrix (1280 entries), but highly structured:

\begin{itemize}
    \item After Digits: next byte likely Digit or Punct (years, numbers)
    \item After Vowels: next byte likely Other (consonants) or Whitespace
    \item After Whitespace: next byte likely Other (word start) or Punct
    \item After Punct: next byte likely Whitespace or Other
    \item After Other: most diverse (consonant clusters, etc.)
\end{itemize}

\begin{prediction}[Conditional Byte Entropy]
\[
H(\text{byte}_{t+1} | \text{ES}_t) < H(\text{byte}_{t+1})
\]
The reduction is approximately 0.4--0.6 bits, because knowing the previous ES constrains the likely next bytes.
\end{prediction}

\subsection{Expected Compression Gain}

When we augment input with ES features, the model can directly use $P(\text{byte}_{t+1} | \text{ES}_t)$ without computing ES membership implicitly.

\begin{prediction}[Compression Gain from ES Features]
Adding 5 ES features should improve compression by:
\[
\Delta \text{bpc} \approx 0.3 \text{--} 0.5 \text{ bits/char}
\]

This comes from:
\begin{enumerate}
    \item Direct ES$\to$byte patterns (currently learned implicitly)
    \item Freed hidden capacity for longer-range patterns
\end{enumerate}
\end{prediction}

The bound is conservative: if the RNN already perfectly encodes ES membership, the gain is zero. But our probing shows ES encoding is imperfect, especially for rare ESs.

\section{The Product Distribution}

\subsection{Factored Joint Space}

The full joint distribution factors as:
\[
P(\text{byte}_{t+1} | \text{byte}_t) = P(\text{ES}_{t+1} | \text{ES}_t) \cdot P(\text{byte}_{t+1} | \text{ES}_{t+1}, \text{ES}_t) \cdot \frac{P(\text{byte}_{t+1} | \text{byte}_t)}{P(\text{byte}_{t+1} | \text{ES}_{t+1}, \text{ES}_t)}
\]

Simplifying with conditional independence assumptions:
\[
P(\text{byte}_{t+1} | \text{byte}_t) \approx P(\text{ES}_{t+1} | \text{ES}_t) \cdot P(\text{within}_{t+1} | \text{within}_t, \text{ES}_{t+1})
\]

where ``within'' denotes the position within the ES.

\subsection{Parameter Count}

\begin{center}
\begin{tabular}{lrr}
\toprule
Model & Parameters & Bits to Describe \\
\midrule
Full bigram & $256 \times 256 = 65536$ & $\sim 10^5$ \\
ES-Markov (uniform within) & $5 \times 5 = 25$ & $\sim 50$ \\
ES-Markov + within-ES bigrams & $25 + 5 \times s_i^2$ & $\sim 10^4$ \\
\bottomrule
\end{tabular}
\end{center}

The product structure reduces parameters by 6$\times$ while capturing most structure.

\subsection{Bayesian Model Comparison}

\begin{proposition}[Model Evidence]
The Bayesian evidence for model $M$ given data $D$ is:
\[
P(D | M) = \int P(D | \theta, M) P(\theta | M) d\theta
\]

For models with fewer parameters, the prior $P(\theta | M)$ is more concentrated, often giving higher evidence despite slightly worse fit.
\end{proposition}

The factored ES model has $\sim 10^4$ parameters vs $\sim 10^5$ for full bigram. By Occam's razor, it should be preferred when the fit is comparable.

\section{Experimental Design}

\subsection{Phase 1: Baseline Measurements}

Before augmentation, measure:
\begin{enumerate}
    \item Current RNN bpc on enwik9: 5.69
    \item ES transition matrix $T_{ij}$ from data
    \item Conditional entropy $H(\text{byte} | \text{ES})$ from data
    \item Within-ES transition matrices
\end{enumerate}

\subsection{Phase 2: Augmented Training}

\begin{enumerate}
    \item Modify input layer: 256 $\to$ 261 dimensions
    \item Initialize new weights randomly (or from ES$\to$output correlations)
    \item Train on enwik9 for 3 epochs (same as baseline)
    \item Measure bpc on held-out data
\end{enumerate}

\subsection{Phase 3: Analysis}

\begin{enumerate}
    \item Compare augmented bpc to baseline
    \item Measure which ES$\to$output patterns are learned
    \item Probe hidden state for higher-order ESs
    \item Prepare for next tock phase
\end{enumerate}

\subsection{Success Criteria}

\begin{center}
\begin{tabular}{ll}
\toprule
Outcome & Interpretation \\
\midrule
$\Delta$bpc $> 0.5$ & ES features highly valuable, freed significant capacity \\
$\Delta$bpc $\in [0.2, 0.5]$ & Moderate gain, as predicted \\
$\Delta$bpc $\in [0, 0.2]$ & Minimal gain, RNN already encodes ESs well \\
$\Delta$bpc $< 0$ & ES features hurt, wrong factorization \\
\bottomrule
\end{tabular}
\end{center}

\section{Experimental Results}

\textbf{Update 2026-01-31:} Initial experiments show much larger improvements than predicted.

\subsection{Training Curves}

On 1M characters of enwik9, 1 epoch:

\begin{center}
\begin{tabular}{lrrr}
\toprule
Model & Training bpc & Eval bpc & vs Baseline \\
\midrule
Baseline (256-dim input) & 5.51 & 7.31 & --- \\
Augmented (261-dim input) & 3.42 & 3.44 & $-$3.87 bpc \\
\bottomrule
\end{tabular}
\end{center}

The augmented model achieves \textbf{3.44 bpc} vs baseline \textbf{7.31 bpc}---a 53\% improvement, far exceeding our 0.3--0.5 bpc prediction.

\subsection{Why So Large?}

The ES features provide \emph{direct information} about character class that the baseline must learn implicitly:
\begin{itemize}
    \item Baseline must discover that `a', `e', `i', `o', `u' are related
    \item Augmented model is \emph{told} they share an ES
    \item This acts like a strong prior over weight structure
\end{itemize}

The 5 ES features (2.32 bits of information) leverage into 3.87 bpc improvement because:
\begin{enumerate}
    \item ES membership is \emph{always} predictive (not context-dependent)
    \item The model can learn ES$\to$byte patterns with 5 weights instead of 256
    \item Freed capacity learns longer-range structure faster
\end{enumerate}

\subsection{Caveat}

These are preliminary results on 1M characters, 1 epoch. Full enwik9 training (1B characters, 3 epochs) may show different dynamics as the baseline catches up.

\section{Higher-Order ESs}

\subsection{The Stacking Hypothesis}

After tick-1, the model has explicit ES features. During tock-2, we should find:
\begin{itemize}
    \item \textbf{ES-pairs:} Neurons encoding (ES$_\text{prev}$, ES$_\text{curr}$) combinations
    \item \textbf{Positional ESs:} Word-start, mid-word, word-end
    \item \textbf{Structural ESs:} In-tag, in-attribute, in-text
\end{itemize}

These are \emph{second-order ESs} built from first-order ones.

\subsection{Cardinality Prediction}

\begin{prediction}[Second-Order ES Size]
The most informative second-order ESs will have cardinality:
\begin{itemize}
    \item ES-pairs: up to $5 \times 5 = 25$ (but many will merge)
    \item Positional: 3--5 (start, middle, end, plus punctuation cases)
    \item Structural: 4--8 (XML has nested structure)
\end{itemize}
\end{prediction}

These predictions are testable after tick-1 training completes.

\section{Implementation}

\subsection{Input Modification}

\begin{verbatim}
// Current: one-hot byte
float input[256];
input[byte] = 1.0;

// Augmented: one-hot byte + one-hot ES
float input[261];  // 256 + 5
input[byte] = 1.0;
input[256 + get_es_id(byte)] = 1.0;
\end{verbatim}

\subsection{Weight Initialization}

For the new ES$\to$hidden connections, initialize from observed ES$\to$output correlations:
\[
W_{\text{ES},h}^{(0)} = \alpha \cdot \mathbb{E}[h | \text{ES}]
\]

where $\alpha$ is a scaling factor and the expectation is computed from probing data.

This gives the new weights a ``head start'' reflecting what the model already knows.

\subsection{Training Protocol}

\begin{enumerate}
    \item Load baseline model weights
    \item Expand input layer, initialize new weights
    \item Train for 3 epochs on enwik9
    \item Compare to baseline trained for same epochs
\end{enumerate}

\section{Connection to CMP}

\subsection{Factored Event Spaces}

In CMP notation, we're implementing:
\[
E = E_\text{byte} \times E_\text{ES}
\]

where $E_\text{byte} = \{0, \ldots, 255\}$ and $E_\text{ES} = \{\text{Digits}, \text{Vowels}, \ldots\}$.

The total information is:
\[
I(E) = \log|E_\text{byte}| + \log|E_\text{ES}| = 8 + 2.32 = 10.32 \text{ bits}
\]

But this overcounts! The ES is determined by the byte, so:
\[
I(E_\text{byte} \times E_\text{ES}) = I(E_\text{byte}) = 8 \text{ bits}
\]

The redundancy is exactly what makes the augmentation useful: the model gets ES for ``free'' (no additional information), but with direct access.

\subsection{Pattern Space}

With augmented input, the pattern space includes:
\begin{itemize}
    \item $P_{\text{byte} \to h}$: existing byte$\to$hidden patterns
    \item $P_{\text{ES} \to h}$: new ES$\to$hidden patterns
    \item $P_{\text{ES} \to o}$: direct ES$\to$output patterns
\end{itemize}

The new patterns can be sparser (5 sources vs 256) while capturing the same structure.

\section{Predictions Summary}

\begin{center}
\begin{tabular}{lll}
\toprule
Prediction & Value & Testable After \\
\midrule
ES transition entropy & $\approx 1.2$ bits & Data analysis \\
$H(\text{byte} | \text{ES})$ reduction & 0.4--0.6 bits & Data analysis \\
Compression gain & 0.3--0.5 bits/char & Tick-1 training \\
Second-order ES count & 10--20 & Tock-2 extraction \\
\bottomrule
\end{tabular}
\end{center}

These predictions are Bayesian in the sense that they follow from:
\begin{enumerate}
    \item Prior: joint distributions factor as products
    \item Likelihood: observed ES frequencies and transitions
    \item Posterior: expected compression from factored model
\end{enumerate}

\section{Conclusion}

The tick phase augments RNN input with explicit ES features. We predict 0.3--0.5 bits/char compression gain from:
\begin{enumerate}
    \item Direct ES$\to$byte patterns (no longer implicit)
    \item Freed hidden capacity for longer-range structure
\end{enumerate}

The joint ES$\times$byte distribution is a $5 \times 256$ matrix with $\sim 10^4$ effective parameters, 6$\times$ fewer than full bigram. This product structure is the Bayesian justification for the factorization.

After tick-1, tock-2 should reveal second-order ESs: ES-pairs, positional features, and structural features. The cycle continues.

\begin{thebibliography}{9}
\bibitem{cmp2026}
Clement, M. (2026). CMP. \url{https://cmpr.ai/cmp.pdf}
\bibitem{tock2026}
Claude \& MJC (2026). Tock: Extracting Interpretable Structure. This volume.
\end{thebibliography}

\end{document}
