\documentclass{article}
\usepackage{amsmath,amssymb}
\usepackage{booktabs}

\title{Pattern Injection: UM $\to$ RNN}
\author{Hutter RNN Project}
\date{2026-01-31}

\begin{document}
\maketitle

\section{The Isomorphism}

We have two representations:
\begin{align}
\text{UM} &= (E, T, P, f, \omega) \\
\text{RNN} &= (W_{ih}, W_{hh}, W_{ho}, b_h, b_o)
\end{align}

\textbf{Tock} (extraction): Read patterns from trained weights.

\textbf{Tick} (injection): Write patterns into weights, skip training.

\section{The Mapping}

\subsection{Forward: RNN $\to$ Predictions}

\begin{align}
h_t &= \tanh(W_{ih} x_t + W_{hh} h_{t-1} + b_h) \\
o_t &= W_{ho} h_t + b_o \\
P(e_{t+1} | h_t) &= \text{softmax}(o_t)
\end{align}

\subsection{UM Patterns as Target}

We have pattern matrix $P \in \mathbb{R}^{|E| \times |E|}$:
\[
P[e_1, e_2] = \log \text{count}(e_1 \to e_2) - \log \text{count}(e_1)
\]

This is the log-probability $\log P(e_2 | e_1)$.

\subsection{The Injection Problem}

\textbf{Goal:} Set weights such that:
\[
W_{ho} h_{e_1} + b_o \approx P[e_1, :]
\]

where $h_{e_1}$ is the hidden state after seeing $e_1$.

\section{Simple Case: 1-Markov}

For a 1-Markov model (only previous byte matters):

\subsection{One-Hot Encoding}

If $x_t$ is one-hot for byte $e_1$, we want:
\[
o_t = P[e_1, :] \quad \text{(the } e_1 \text{ row of pattern matrix)}
\]

\subsection{Direct Injection}

Set:
\begin{align}
W_{ih} &= I \quad \text{(identity, or encoding matrix)} \\
W_{hh} &= 0 \quad \text{(no recurrence for 1-Markov)} \\
W_{ho} &= P^T \quad \text{(pattern matrix, transposed)} \\
b_o &= 0
\end{align}

Then:
\begin{align}
h_t &= \tanh(x_t) \approx x_t \quad \text{(for one-hot)} \\
o_t &= P^T x_t = P[e_1, :]
\end{align}

\textbf{Result:} Network outputs correct bigram probabilities with no training.

\section{General Case: Beyond 1-Markov}

\subsection{The Hidden State Problem}

For $k$-Markov or full context:
\begin{itemize}
\item Hidden state $h_t$ encodes context $(e_1, \ldots, e_t)$
\item We need: $W_{ho} h_t = P[\text{context}, :]$
\item But $h_t$ is a learned representation
\end{itemize}

\subsection{Factored Injection}

Factor the pattern matrix through a bottleneck:
\[
P \approx U V^T
\]

where $U \in \mathbb{R}^{|E| \times H}$ and $V \in \mathbb{R}^{|E| \times H}$.

This is like SVD or matrix factorization.

Set:
\begin{align}
W_{ih} &= U \quad \text{(input embedding)} \\
W_{ho} &= V^T \quad \text{(output embedding)}
\end{align}

\subsection{The Recurrence Problem}

What about $W_{hh}$? This encodes how context evolves.

For ES-augmented model:
\[
W_{hh} = \text{block structure respecting ES}
\]

Inject ES transition patterns into the recurrence.

\section{Practical Algorithm}

\subsection{Step 1: Compute Patterns from Data}

\begin{verbatim}
for each (e1, e2) in corpus:
    count[e1][e2]++
P[e1, e2] = log(count[e1][e2] / sum(count[e1]))
\end{verbatim}

\subsection{Step 2: Factorize}

\begin{verbatim}
U, S, V = svd(P)
U_h = U[:, :H] * sqrt(S[:H])
V_h = V[:, :H] * sqrt(S[:H])
\end{verbatim}

\subsection{Step 3: Inject}

\begin{verbatim}
W_ih = U_h      # input projection
W_ho = V_h.T    # output projection
W_hh = ?        # harder - need recurrence structure
b_o = row_means(P)  # bias = marginal
\end{verbatim}

\subsection{Step 4: Fine-tune}

Train normally, but starting from injected weights.

\section{What This Buys Us}

\begin{center}
\begin{tabular}{ll}
\toprule
Benefit & How \\
\midrule
Faster convergence & Start near good solution \\
Interpretable init & Know what patterns are encoded \\
Transfer learning & Inject patterns from other data \\
Debugging & Inject known patterns, verify behavior \\
Hybrid models & Mix learned + injected patterns \\
\bottomrule
\end{tabular}
\end{center}

\section{The ES Advantage}

For ES-augmented model:

\begin{enumerate}
\item Compute ES-level patterns: $P_{ES}[es_1, es_2]$
\item Compute within-ES patterns: $P_w[e | es]$
\item Inject ES patterns into ES-specific weights
\item Inject within-ES patterns into within-ES weights
\end{enumerate}

The factored structure makes injection cleaner.

\section{Open Questions}

\begin{enumerate}
\item How to inject recurrence patterns ($W_{hh}$)?
\item What's the right factorization rank $H$?
\item How much does injection help vs. random init?
\item Can we inject higher-order patterns ($k$-grams)?
\end{enumerate}

\section{Summary}

\begin{verbatim}
       TOCK                    TICK
   (extraction)            (injection)

   RNN weights  ───────>  UM patterns
        W                      P

   RNN weights  <───────  UM patterns
        W                      P
\end{verbatim}

The isomorphism goes both ways. We can:
\begin{itemize}
\item Read patterns out (interpretability)
\item Write patterns in (initialization)
\end{itemize}

\end{document}
