\documentclass[11pt]{article}
\usepackage{amsmath,amssymb}
\usepackage[margin=1in]{geometry}
\usepackage{booktabs}

\title{Embeddings and the Atomic Time Step}
\author{Hutter RNN Project}
\date{January 31, 2026}

\begin{document}
\maketitle

\section{The Atomic Time Step}

The Universal Model update:
\[
t_{n+1} = p \times t_n
\]

Each application of pattern $p$ advances thought by one atomic timestep. This is the fundamental ``tick'' of the system.

\subsection{Cost Per Tick}

Each tick consumes entropy:
\[
H_{\text{tick}} = -\log_2 p_{\text{avg}}
\]

After $n$ ticks:
\[
H_{\text{total}} = n \cdot H_{\text{tick}}
\]

The depth limit:
\[
n_{\max} = \frac{24}{H_{\text{tick}}}
\]

\section{Embeddings}

An embedding maps events to vectors:
\[
\phi: E \to \mathbb{R}^d
\]

The dimension $d$ is the number of basis functions---like Fourier coefficients.

\subsection{Thought as Distribution}

Thought $t \in T$ is a distribution over events:
\[
t: E \to [0, 255] \quad \text{(log-support values)}
\]

In probability form:
\[
p(e) = \frac{\exp(t(e))}{\sum_{e'} \exp(t(e'))}
\]

\subsection{Embedding the Thought}

Project thought onto $d$ basis functions:
\[
h = \phi(t) \in \mathbb{R}^d
\]

This is like taking $d$ Fourier coefficients of the distribution $t$.

\section{Time Evolution in Embedding Space}

\subsection{Abstract Form}
In thought space:
\[
t_{n+1} = p \times t_n
\]

\subsection{Embedded Form}
In embedding space:
\[
h_{n+1} = W \cdot h_n
\]

The pattern $p$ becomes a matrix $W \in \mathbb{R}^{d \times d}$.

\subsection{The RNN Case}
For an Elman RNN:
\[
h_{n+1} = \tanh(W_{hh} \cdot h_n + W_{ih} \cdot x_n)
\]

Here:
\begin{itemize}
\item $W_{hh}$ is the embedded pattern (time evolution)
\item $W_{ih}$ incorporates new input $x_n$
\item $\tanh$ is normalization (keeps values bounded)
\end{itemize}

Without input ($x_n = 0$) and for small values ($\tanh \approx \text{id}$):
\[
h_{n+1} \approx W_{hh} \cdot h_n
\]

This is exactly $t_{n+1} = p \times t_n$ in the embedding.

\section{Embedding Dimension as Bandwidth}

\subsection{Fourier Analogy}
\begin{center}
\begin{tabular}{ll}
\toprule
\textbf{Fourier} & \textbf{Embedding} \\
\midrule
$d$ frequency components & $d$ hidden dimensions \\
Basis functions $e^{2\pi i k t}$ & Basis vectors in $\mathbb{R}^d$ \\
Bandwidth $= d \cdot f_0$ & Representable complexity \\
\bottomrule
\end{tabular}
\end{center}

\subsection{What $d$ Controls}

Higher $d$:
\begin{itemize}
\item More frequencies / finer resolution
\item More complex patterns representable
\item More parameters ($W$ is $d \times d$)
\item More energy cost
\end{itemize}

Lower $d$:
\begin{itemize}
\item Fewer frequencies / coarser resolution
\item Simpler patterns only
\item Fewer parameters
\item Lower energy cost
\end{itemize}

\subsection{SVD and Dimension}

From our SVD analysis:
\begin{itemize}
\item Component 0: 97.5\% of variance (baseline frequency)
\item Components 1--5: 2.5\% (interpretable structure)
\item Components 6--63: diminishing returns
\item Components 64--255: noise
\end{itemize}

Truncating to rank-64 keeps the signal, discards the noise. The ``bandwidth'' of English bigrams is approximately 64 dimensions.

\section{The Full Picture}

\begin{center}
\begin{tabular}{lll}
\toprule
\textbf{Level} & \textbf{Object} & \textbf{Time Step} \\
\midrule
Events & $e \in E$ & --- \\
Thought & $t \in T = [0,255]^{|E|}$ & $t_{n+1} = p \times t_n$ \\
Embedding & $h \in \mathbb{R}^d$ & $h_{n+1} = W \cdot h_n$ \\
RNN & $h \in \mathbb{R}^d$ & $h_{n+1} = \tanh(W_{hh} h_n + W_{ih} x_n)$ \\
\bottomrule
\end{tabular}
\end{center}

The atomic time step $t_{n+1} = p \times t_n$ is the same operation at every level, just in different representations.

\section{Back to Empirical}

\subsection{Pattern Injection Revisited}

We injected bigram statistics into $W_{ih}$ and $W_{ho}$ via SVD:
\[
P^T = U \cdot S \cdot V^T \implies W_{ho} = U\sqrt{S}, \quad W_{ih} = \sqrt{S}V^T
\]

This embeds the pattern $P$ (bigram log-probs) into the RNN weights.

Result: 1 bit/char head start (5.46 $\to$ 4.47 bpc).

\subsection{Why $W_{hh}$ Injection Fails}

$W_{hh}$ encodes the time evolution---how to carry information from $h_n$ to $h_{n+1}$.

This requires patterns that span multiple ticks. But:
\[
\text{bits per pattern} = k \cdot H_{\text{tick}}
\]

For $k$-step patterns, we need $k \cdot H_{\text{tick}}$ bits of precision.

Trigrams ($k=2$) at $H \approx 2$ bits/char need 4 bits per pattern. Seems fine.

But: the patterns must be \emph{carried} through $W_{hh}$ multiplication, which accumulates precision loss. After a few steps, the signal is washed out.

\subsection{Prediction}

Effective memory depth of RNN:
\[
d_{\text{memory}} \approx \frac{24}{H_{\text{avg}}} \approx \frac{24}{2} = 12 \text{ steps}
\]

For English text at $\sim$2 bits/char, the RNN should effectively ``remember'' about 12 characters back.

This is testable: probe how RNN predictions depend on context at various distances.

\section{Summary}

\begin{center}
\fbox{\parbox{0.85\textwidth}{
\begin{itemize}
\item The atomic time step $t_{n+1} = p \times t_n$ becomes $h_{n+1} = W \cdot h_n$ in embedding space
\item Embedding dimension $d$ = bandwidth = number of frequencies
\item SVD shows English bigrams have effective dimension $\sim$64
\item $W_{hh}$ carries information through time, limited by precision
\item Predicted memory depth: $\sim$12 characters for English
\end{itemize}
}}
\end{center}

\end{document}
