\documentclass[11pt]{article}
\usepackage{amsmath,amssymb}
\usepackage[margin=1in]{geometry}
\usepackage{booktabs}

\title{RNN Memory Depth: Experiment and Results}
\author{Hutter RNN Project}
\date{January 31, 2026}

\begin{document}
\maketitle

\section{Motivation}

From the time-energy analysis, we derived a prediction for RNN memory depth:
\[
d_{\max} = \frac{24}{H_{\text{avg}}} \approx \frac{24}{2} = 12 \text{ characters}
\]

The reasoning: float32 has 24 bits of mantissa. Each timestep ``costs'' $H_{\text{avg}}$ bits of entropy. After $d_{\max}$ steps, precision is exhausted and earlier information is washed out.

This experiment tests whether the RNN's effective memory matches this prediction.

\section{Method 1: Perturbation (Failed)}

\subsection{Approach}
For each sample position:
\begin{enumerate}
\item Run RNN on context, get prediction $P_{\text{orig}}$
\item Flip byte at distance $k$ back
\item Run RNN on modified context, get prediction $P_{\text{mod}}$
\item Measure KL divergence $D_{KL}(P_{\text{orig}} \| P_{\text{mod}})$
\end{enumerate}

\subsection{Results}
Dependency \emph{increased} with distance---the opposite of expected.

\subsection{Problem}
When you flip a byte at position $-k$, the perturbation propagates through $k$ steps of hidden state evolution. Flipping farther back = more accumulated effect, not less.

This measures \textbf{perturbation propagation}, not memory depth.

\section{Method 2: Conditional Variance}

\subsection{Approach}
For each distance $k$:
\begin{enumerate}
\item Collect many samples with their full context
\item Group samples by the byte value at position $-k$
\item Compute mean prediction for each group
\item Measure variance of means across groups
\end{enumerate}

If the RNN ``remembers'' position $-k$, then predictions should vary depending on what byte was there. Higher variance = more dependency on that position.

\subsection{Implementation}

\begin{verbatim}
for k in range(1, max_dist + 1):
    predictions_by_byte = defaultdict(list)

    for pos in sample_positions:
        context = data[pos - context_len : pos]
        byte_at_k = context[-k]
        _, probs = forward(model, context)
        predictions_by_byte[byte_at_k].append(probs)

    # Compute variance of group means
    group_means = [mean(preds) for preds in predictions_by_byte.values()]
    variance[k] = var(group_means)
\end{verbatim}

\subsection{Results}

\begin{center}
\begin{tabular}{rcc}
\toprule
Distance $k$ & Variance & Normalized \\
\midrule
1 & 0.000610 & 0.76 \\
2 & 0.000767 & 0.96 \\
3 & 0.000713 & 0.89 \\
4 & 0.000799 & 1.00 \\
5 & 0.000667 & 0.83 \\
10 & 0.000797 & 1.00 \\
15 & 0.000613 & 0.77 \\
20 & 0.000500 & 0.63 \\
25 & 0.000471 & 0.59 \\
30 & 0.000470 & 0.59 \\
\bottomrule
\end{tabular}
\end{center}

\textbf{Observation}: No clear exponential decay. Dependency is roughly uniform out to 30 characters, with some noise.

\section{Analysis}

\subsection{Expected vs Observed}

\begin{center}
\begin{tabular}{ll}
\toprule
\textbf{Predicted} & Exponential decay, 1/e at $k \approx 12$ \\
\textbf{Observed} & Roughly flat, no clear decay to $k = 30$ \\
\bottomrule
\end{tabular}
\end{center}

\subsection{Possible Explanations}

\textbf{1. The 24-bit limit applies to gradients, not inference.}

The depth limit derivation assumed that carrying information through $W_{hh}$ multiplication loses precision. But:
\begin{itemize}
\item During inference, we only need relative magnitudes
\item The hidden state is normalized by $\tanh$
\item Information might be encoded in directions, not magnitudes
\end{itemize}

\textbf{2. The RNN has learned efficient encoding.}

The trained $W_{hh}$ might have learned to preserve information more efficiently than a random matrix would. The singular value structure of $W_{hh}$ could concentrate information in stable subspaces.

\textbf{3. English has redundancy that extends memory.}

Natural language has long-range correlations. The RNN might be exploiting statistical regularities (word boundaries, sentence structure) that effectively extend its memory beyond what raw precision would allow.

\textbf{4. The measurement method has issues.}

Variance of group means might not be the right statistic. The signal might be too weak relative to noise. Need more samples or a different approach.

\section{Connection to Theory}

The depth limit formula:
\[
d_{\max} = \frac{24}{-\log_2 p_{\text{avg}}}
\]

This was derived for arithmetic coding intervals, where precision loss is exact and cumulative. For RNNs:
\begin{itemize}
\item $\tanh$ normalization prevents unbounded growth
\item The hidden state is a distributed representation
\item Information can be encoded redundantly across dimensions
\end{itemize}

The formula might be a \emph{lower bound} (you can't do worse than this) rather than an \emph{exact prediction}.

\section{Future Work}

\begin{enumerate}
\item \textbf{Probe hidden state directly}: Instead of looking at output predictions, look at how hidden state $h_n$ depends on input at position $-k$.

\item \textbf{Analyze $W_{hh}$ eigenvalues}: The spectral radius of $W_{hh}$ determines how quickly information decays. If $|\lambda_{\max}| \approx 1$, information persists.

\item \textbf{Synthetic experiment}: Train on data with known memory requirements (e.g., matching brackets at distance $k$). See if performance degrades at $k > 12$.

\item \textbf{Compare with untrained model}: Does a randomly initialized RNN show the predicted decay?
\end{enumerate}

\section{Conclusion}

The predicted memory depth of $\sim$12 characters was not confirmed. Observed dependency is roughly flat out to 30 characters.

This suggests either:
\begin{itemize}
\item The precision limit is not the dominant factor
\item The RNN has learned to work around it
\item The measurement method needs refinement
\end{itemize}

The experiment is documented here for future investigation.

\section{Files}

\begin{itemize}
\item \texttt{memory\_depth.py} --- Method 1 (perturbation, failed)
\item \texttt{memory\_depth2.py} --- Method 2 (conditional variance)
\item \texttt{memory-depth.html} --- Interactive visualization
\item \texttt{memory\_depth\_results.npz} --- Raw data
\end{itemize}

\end{document}
