\documentclass{article}
\usepackage{amsmath,amssymb}
\usepackage{booktabs}

\title{Open Research Questions for Archive 20260131\_3}
\author{Hutter RNN Project}
\date{2026-01-31}

\begin{document}
\maketitle

\section{What We Learned}

\begin{enumerate}
\item ES features provide $\leq 0.19$ bits/char (3.7\%), not 53\%
\item ES captures 15.9\% of byte-level Markov mutual information
\item Max entropy renormalization loses 2.09 bits/char within-ES
\item Pattern extraction must use data statistics, not weight products
\item Current ES definition is poor: ``Other'' wastes 3 bits, ``Whitespace'' wastes 1.6 bits
\end{enumerate}

\section{Open Questions}

\subsection{Q1: Better ES Definitions}

Current ESs: Digit(10), Punct(6), Vowel(10), Whitespace(4), Other(226).

``Other'' is 88\% of the alphabet but only captures residual structure. Should we split:
\begin{itemize}
\item Consonants (21) vs Other-other?
\item Uppercase (26) vs lowercase?
\item Brackets/parens vs other punctuation?
\item High-frequency consonants (t,n,s,r) vs low-frequency?
\end{itemize}

\textbf{Hypothesis:} Optimal ES count is $\approx \log_2(256) = 8$ classes, balancing granularity vs complexity.

\subsection{Q2: Hierarchical Coding}

Instead of flat $P(\text{byte})$, use:
\[
P(\text{byte}) = P(\text{ES}) \times P(\text{byte}|\text{ES})
\]

\textbf{Question:} Does hierarchical prediction help the RNN, or is the bottleneck elsewhere?

\subsection{Q3: Converged Performance}

Our experiments used 1M chars (0.1\% of data). Training is now at 1.6\%.

\textbf{Question:} What is the actual improvement from ES features at convergence?

\textbf{Prediction:} $\leq 0.19$ bits/char improvement, per Bayesian bound.

\subsection{Q4: Learning Within-ES Structure}

The 2.09 bit gap is within-ES structure that must be learned.

\textbf{Question:} How efficiently can an RNN learn within-ES patterns vs across-ES patterns?

\textbf{Experiment:} Compare learning curves for:
\begin{itemize}
\item Predicting ES given prev-ES (should be fast)
\item Predicting byte-within-ES given byte-within-ES (should be slower)
\end{itemize}

\subsection{Q5: Second-Order ESs}

ES-pairs $(ES_t, ES_{t+1})$ have 25 states vs 5 for first-order.

\textbf{Question:} Do ES-pairs capture significantly more structure?

From our analysis:
\[
H(ES_{t+1} | ES_t) = 1.46 \text{ bits} \quad \text{vs} \quad H(ES) = 1.65 \text{ bits}
\]

Only 0.19 bits gained. But what about:
\[
H(\text{byte} | ES_t, ES_{t+1}) \quad \text{vs} \quad H(\text{byte} | ES_t)?
\]

\subsection{Q6: Positional ESs}

Hypothesis: Position within word matters. ``e'' at word-end behaves differently than ``e'' mid-word.

\textbf{Question:} Can we learn positional ESs automatically?

\textbf{Approach:} Cluster hidden states by (byte, position) and look for separable clusters.

\subsection{Q7: The Actual Tock Problem}

We assumed ESs are given. The real challenge: extract ESs from trained RNN.

\textbf{Question:} Which hidden neurons encode ES-like structure?

\textbf{Approach:}
\begin{enumerate}
\item Find neurons whose activation predicts byte class
\item Cluster neurons by what they predict
\item Interpret clusters as candidate ESs
\end{enumerate}

\subsection{Q8: Why Did Training Dynamics Differ?}

The 53\% early-training gap was real, even if not information-theoretic.

\textbf{Question:} Why did ES features stabilize training?

\textbf{Hypotheses:}
\begin{itemize}
\item Gradient normalization (ES features add constant-magnitude input)
\item Easier credit assignment (ES → output is shorter path than byte → hidden → output)
\item Regularization (ES features constrain hidden representations)
\end{itemize}

\section{Proposed Experiments for 20260131\_3}

\begin{enumerate}
\item \textbf{Wait for convergence:} Let current training finish (ETA: 60 hours)
\item \textbf{Measure final gap:} Compare converged aug vs baseline bpc
\item \textbf{Try better ESs:} Split Other into 4 subcategories
\item \textbf{Second-order test:} Add ES-pair features, measure improvement
\item \textbf{Hidden state analysis:} Cluster activations, look for emergent ESs
\item \textbf{Ablation:} Remove individual ES features, measure impact
\end{enumerate}

\section{Summary Table}

\begin{table}[h]
\centering
\begin{tabular}{lcc}
\toprule
Claim & This Archive & Next Steps \\
\midrule
ES improvement & 3.7\% max & Verify at convergence \\
ES explains & 15.9\% of MI & Try better ES defs \\
Within-ES gap & 2.09 bits & Learn to close it \\
Pattern extraction & Data-based & Extend to trigrams? \\
Tock (ES from RNN) & Not attempted & Hidden state clustering \\
\bottomrule
\end{tabular}
\end{table}

\end{document}
