#!/usr/bin/env python3  
"""    
  
Compare:  
  A) Manzini loop-lay: raw grid, greedy DP (max_rep=5)  
  B) Zattera-trained 1-loop state machine  
  C) Zattera-trained 2-loop state machine ("looped")  
  
All use the same Manzini 7-column grid. Same Nbits formula.  
  
Optimized: prefix-filtered forward DP for word generation.  
  
Usage:  
    python3 side_by_side.py <ivtff_file>  
"""  
  
import sys, re, math, heapq  
from collections import Counter, defaultdict  
from itertools import product  
  
  
# ══════════════════════════════════════════════════════════════════════════  
# CORPUS  
# ══════════════════════════════════════════════════════════════════════════  
  
RARE = set('gxvzbju')  
VCH = ['cth', 'ckh', 'cph', 'cfh', 'ch', 'sh']  
  
  
def bad(w):  
    if set(w) & RARE:  
        return True  
    t = w  
    for c in sorted(VCH, key=len, reverse=True):  
        t = t.replace(c, ' ' * len(c))  
    return 'c' in t or 'h' in t  
  
  
def parse_ivtff(fn):  
    tok = []  
    with open(fn, 'r', encoding='utf-8', errors='replace') as f:  
        for line in f:  
            line = line.rstrip('\n')  
            if line.startswith('#') or line.startswith(';') or not line.strip():  
                continue  
            if not re.match(r'<(f\d+[rv]\d?)\.(\d+\w*)', line):  
                continue  
            text = re.sub(r'<->', '.', line)  
            text = re.sub(r'<[^>]*>', '', text)  
            text = re.sub(r'\{[^}]*\}', '', text)  
            text = re.sub(r'@\d+;?', '', text)  
            text = re.sub(r'[!%*]', '', text)  
            words = [re.sub(r'[^a-z]', '', w.strip())  
                     for w in text.split('.') if w.strip()]  
            tok.extend([w for w in words if w])  
    return tok  
  
  
def filt(tok):  
    return Counter(w for w in tok if w and not bad(w))  
  
  
# ══════════════════════════════════════════════════════════════════════════  
# PREFIX TRIE for fast prefix checking  
# ══════════════════════════════════════════════════════════════════════════  
  
class PrefixTrie:  
    """Fast prefix membership test."""  
    def __init__(self, words):  
        self.root = {}  
        self.max_len = 0  
        for w in words:  
            if len(w) > self.max_len:  
                self.max_len = len(w) 
            node = self.root  
            for ch in w:  
                if ch not in node:  
                    node[ch] = {}  
                node = node[ch]  
            node['$'] = True  # marks end of word  
        self.max_len = max((len(w) for w in words), default=0)  
  
    def has_prefix(self, prefix):  
        """Return True if prefix is a prefix of (or equal to) any word."""  
        node = self.root  
        for ch in prefix:  
            if ch not in node:  
                return False  
            node = node[ch]  
        return True  
  
    def is_word(self, w):  
        node = self.root  
        for ch in w:  
            if ch not in node:  
                return False  
            node = node[ch]  
        return '$' in node  
  
  
# ══════════════════════════════════════════════════════════════════════════  
# HUFFMAN & NBITS  
# ══════════════════════════════════════════════════════════════════════════  
  
class HN:  
    __slots__ = ('s', 'f', 'l', 'r')  
    def __init__(s, sy, fr, l=None, r=None):  
        s.s, s.f, s.l, s.r = sy, fr, l, r  
    def __lt__(s, o):  
        return s.f < o.f  
  
  
def hc(fr):  
    if not fr:  
        return {}  
    if len(fr) == 1:  
        return {list(fr)[0]: '0'}  
    h = [HN(s, f) for s, f in fr.items()]  
    heapq.heapify(h)  
    while len(h) > 1:  
        a, b = heapq.heappop(h), heapq.heappop(h)  
        heapq.heappush(h, HN(None, a.f + b.f, a, b))  
    cd = {}  
    def _w(n, p):  
        if n.s is not None:  
            cd[n.s] = p or '0'  
            return  
        if n.l: _w(n.l, p + '0')  
        if n.r: _w(n.r, p + '1')  
    _w(h[0], '')  
    return cd  
  
  
def hb(cd, fr):  
    return sum(len(cd[s]) * f for s, f in fr.items() if s in cd)  
  
  
def nbits_chunks(wc, segs, csize):  
    log2c = math.log2(csize)  
    cf = Counter()  
    total_chunks = 0  
    for w, s in segs.items():  
        c = wc[w]  
        for ch in s:  
            cf[ch] += c  
        total_chunks += len(s) * c  
    codes = hc(dict(cf))  
    nb_t = hb(codes, cf)  
    nb_d = sum(log2c * len(ch) + len(codes[ch]) for ch in cf)  
    Tc = sum(wc[w] for w in segs)  
    return {  
        'nb': int(round(nb_d + nb_t)),  
        'bpt': (nb_d + nb_t) / Tc if Tc else 0,  
        'T': Tc,  
        'n': len(cf),  
        'nb_t': int(round(nb_t)),  
        'nb_d': int(round(nb_d)),  
        'total_chunks': total_chunks,  
    }  
  
  
def nbits_flat(wc, words, csize):  
    log2c = math.log2(csize)  
    sub = {w: wc[w] for w in words}  
    codes = hc(sub)  
    nb_t = hb(codes, sub)  
    nb_d = sum(log2c * len(w) + len(codes[w]) for w in sub)  
    Tc = sum(sub.values())  
    return {  
        'nb': int(round(nb_d + nb_t)),  
        'bpt': (nb_d + nb_t) / Tc if Tc else 0,  
        'T': Tc,  
    }  
  
  
# ══════════════════════════════════════════════════════════════════════════  
# MANZINI GRID  
# ══════════════════════════════════════════════════════════════════════════  
  
GRID = [  
    ['q', 'ch', 'sh', 'cth', 'ckh', 'cph', 'cfh', ''],  
    ['e', ''],  
    ['e', ''],  
    ['o', 'a', ''],  
    ['i', 'ii', 'iii', 'iiii', ''],  
    ['l', 'r', 'd', 'n', 'm', 's', 't', 'k', 'p', 'f', ''],  
    ['y', ''],  
]  
  
  
def gen_chunks(cols):  
    out = set()  
    for combo in product(*cols):  
        c = ''.join(combo)  
        if c:  
            out.add(c)  
    return out  
  
  
# ══════════════════════════════════════════════════════════════════════════  
# STATE MACHINE with prefix-filtered generation  
# ══════════════════════════════════════════════════════════════════════════  
  
class StateMachine:  
  
    def __init__(self):  
        self.states = {}  
        self.initial = None  
        self.topo_order = []  
        self._trie = None       # set by caller for prefix filtering  
        self._all_words = None   # full vocabulary (unfiltered gen)  
  
    def set_prefix_trie(self, trie):  
        self._trie = trie  
  
    def add_state(self, name, token='', end_state=False):  
        self.states[name] = {  
            'token': token,  
            'end': end_state,  
            'next': set(),  
            'prev': set(),  
        }  
  
    def set_initial(self, name):  
        self.initial = name  
  
    def add_edge(self, a, b):  
        self.states[a]['next'].add(b)  
        self.states[b]['prev'].add(a)  
  
    def remove_edge(self, a, b):  
        self.states[a]['next'].discard(b)  
        self.states[b]['prev'].discard(a)  
  
    def compute_topo_order(self):  
        in_deg = {s: len(self.states[s]['prev']) for s in self.states}  
        queue = [s for s, d in in_deg.items() if d == 0]  
        order = []  
        while queue:  
            s = queue.pop(0)  
            order.append(s)  
            for n in self.states[s]['next']:  
                in_deg[n] -= 1  
                if in_deg[n] == 0:  
                    queue.append(n)  
        self.topo_order = order  
  
    def emit_all(self, use_filter=False):  
        """  
        Forward DP along topological order.  
          
        If use_filter and self._trie is set, only propagate strings  
        that are prefixes of known words. This dramatically prunes  
        the search space for 2-loop machines.  
          
        Returns Counter: {word: path_count}  
        """  
        self.compute_topo_order()  
  
        trie = self._trie if use_filter else None  
        max_len = trie.max_len if trie else 999  
  
        # incoming[state] = set of strings arriving (before state emits)  
        # Using sets instead of Counters — we only need word membership  
        # for training, not path counts  
        incoming = defaultdict(set)  
        incoming[self.initial].add('')  
  
        generated = set()  
  
        for s in self.topo_order:  
            if not incoming[s]:  
                continue  
  
            tok = self.states[s]['token']  
            st = self.states[s]  
  
            # Compute outgoing = {str + tok for str in incoming[s]}  
            # with optional prefix filtering  
            if tok:  
                outgoing = set()  
                for string in incoming[s]:  
                    ns = string + tok  
                    if len(ns) > max_len:  
                        continue  
                    if trie and not trie.has_prefix(ns):  
                        continue  
                    outgoing.add(ns)  
            else:  
                # Empty token — just pass through  
                outgoing = incoming[s]  
  
            # Collect at end states  
            if st['end']:  
                for string in outgoing:  
                    if string:  
                        generated.add(string)  
  
            # Propagate to next states  
            for nxt in st['next']:  
                incoming[nxt].update(outgoing)  
  
            # Free memory  
            del incoming[s]  
  
        return generated  
  
    def emit_all_unfiltered(self):  
        """Generate all words without filtering. For 1-loop machines."""  
        return self.emit_all(use_filter=False)  
  
    def emit_all_filtered(self):  
        """Generate words with prefix filtering. For 2-loop machines."""  
        return self.emit_all(use_filter=True)  
  
    def train(self, language, mode='F1', verbose=True, use_filter=False):  
        """  
        Zattera-style training: iteratively remove edges that improve F1.  
          
        For each candidate edge removal, temporarily remove it,  
        regenerate, and check improvement.  
        """  
        iteration = 0  
  
        while True:  
            iteration += 1  
  
            generated = self.emit_all(use_filter=use_filter)  
  
            tp = len(generated & language)  
            fp = len(generated - language)  
  
            if tp + fp == 0:  
                break  
  
            precision = tp / (tp + fp)  
            recall = tp / len(language)  
            f1 = (2 * precision * recall / (precision + recall)  
                   if (precision + recall) > 0 else 0)  
  
            if verbose and (iteration <= 5 or iteration % 25 == 0):  
                print(f"    Iter {iteration:>4}: |gen|={len(generated):>6,}, "  
                      f"TP={tp:>4}, FP={fp:>5,}, P={precision:.3f}, "  
                      f"R={recall:.3f}, F1={f1:.3f}", flush=True)  
  
            # Collect prunable edges  
            all_edges = []  
            for s_name, s in self.states.items():  
                # Skip edges from non-emitting initial state  
                if s_name == self.initial and s['token'] == '':  
                    for nxt in s['next']:  
                        ns = self.states[nxt]  
                        if ns['end'] and ns['token'] == '':  
                            continue  
                        # We DO include edges from BEGIN to emitting states  
                        # but Zattera excludes them. Let's follow Zattera:  
                        # skip edges FROM non-emitting initial  
                    continue  
                for nxt in s['next']:  
                    ns = self.states[nxt]  
                    # Skip edges TO non-emitting end state  
                    if ns['end'] and ns['token'] == '':  
                        continue  
                    all_edges.append((s_name, nxt))  
  
            candidate = None  
            best_p, best_r, best_f1 = precision, recall, f1  
  
            for a, b in all_edges:  
                # Temporarily remove  
                self.states[a]['next'].discard(b)  
                self.states[b]['prev'].discard(a)  
  
                new_gen = self.emit_all(use_filter=use_filter)  
  
                # Restore  
                self.states[a]['next'].add(b)  
                self.states[b]['prev'].add(a)  
  
                new_tp = len(new_gen & language)  
                new_fp = len(new_gen - language)  
  
                if new_tp + new_fp == 0:  
                    new_p = 1.0  
                else:  
                    new_p = new_tp / (new_tp + new_fp)  
                new_r = new_tp / len(language)  
                new_f1 = (2 * new_p * new_r / (new_p + new_r)  
                          if (new_p + new_r) > 0 else 0)  
  
                improved = False  
                if mode == 'PRECISION' and new_p >= best_p:  
                    improved = True  
                elif mode == 'RECALL' and new_r >= best_r:  
                    improved = True  
                elif mode == 'F1' and new_f1 >= best_f1:  
                    improved = True  
  
                if improved:  
                    candidate = (a, b)  
                    best_p, best_r, best_f1 = new_p, new_r, new_f1  
  
            if candidate is None:  
                break  
  
            self.remove_edge(candidate[0], candidate[1])  
  
        self._remove_unreachable()  
        generated = self.emit_all(use_filter=use_filter)  
  
        if verbose:  
            tp = len(generated & language)  
            fp = len(generated - language)  
            p = tp / (tp + fp) if (tp + fp) > 0 else 0  
            r = tp / len(language)  
            f = 2 * p * r / (p + r) if (p + r) > 0 else 0  
            print(f"    Final: {iteration} iters, |gen|={len(generated):,}, "  
                  f"TP={tp}, FP={fp}, P={p:.3f}, R={r:.3f}, F1={f:.3f}")  
  
        return generated  
  
    def _remove_unreachable(self):  
        changed = True  
        while changed:  
            changed = False  
            to_remove = []  
            for name, s in self.states.items():  
                if name == self.initial:  
                    continue  
                if len(s['prev']) == 0:  
                    to_remove.append(name)  
                    changed = True  
                elif len(s['next']) == 0 and not s['end']:  
                    to_remove.append(name)  
                    changed = True  
            for name in to_remove:  
                s = self.states[name]  
                for nxt in list(s['next']):  
                    self.remove_edge(name, nxt)  
                for prv in list(s['prev']):  
                    self.remove_edge(prv, name)  
                del self.states[name]  
  
    def count_edges(self):  
        return sum(len(self.states[s]['next']) for s in self.states)  
  
    def count_states(self):  
        return len(self.states)  
  
  
# ══════════════════════════════════════════════════════════════════════════  
# BUILD STATE MACHINE FROM GRID  
# ══════════════════════════════════════════════════════════════════════════  
  
def build_machine_from_grid(grid, n_loops=1):  
    m = StateMachine()  
    ncols = len(grid)  
  
    m.add_state('BEGIN', token='', end_state=False)  
    m.set_initial('BEGIN')  
    m.add_state('END', token='', end_state=True)  
  
    for loop in range(n_loops):  
        for col_idx, col in enumerate(grid):  
            for option in col:  
                if option == '':  
                    name = f'L{loop}_C{col_idx}_skip'  
                else:  
                    name = f'L{loop}_C{col_idx}_{option}'  
                m.add_state(name, token=option, end_state=False)  
  
    # BEGIN -> first column of first loop  
    for option in grid[0]:  
        tag = option if option else 'skip'  
        m.add_edge('BEGIN', f'L0_C0_{tag}')  
  
    # Within each loop: col i -> col i+1  
    for loop in range(n_loops):  
        for col_idx in range(ncols - 1):  
            for opt_a in grid[col_idx]:  
                tag_a = opt_a if opt_a else 'skip'  
                for opt_b in grid[col_idx + 1]:  
                    tag_b = opt_b if opt_b else 'skip'  
                    m.add_edge(f'L{loop}_C{col_idx}_{tag_a}',  
                               f'L{loop}_C{col_idx+1}_{tag_b}')  
  
    # Between loops: last col of loop i -> first col of loop i+1  
    for loop in range(n_loops - 1):  
        for opt_a in grid[ncols - 1]:  
            tag_a = opt_a if opt_a else 'skip'  
            for opt_b in grid[0]:  
                tag_b = opt_b if opt_b else 'skip'  
                m.add_edge(f'L{loop}_C{ncols-1}_{tag_a}',  
                           f'L{loop+1}_C0_{tag_b}')  
  
    # Last column of each loop -> END  
    for loop in range(n_loops):  
        for option in grid[ncols - 1]:  
            tag = option if option else 'skip'  
            m.add_edge(f'L{loop}_C{ncols-1}_{tag}', 'END')  
  
    return m  
  
  
# ══════════════════════════════════════════════════════════════════════════  
# GREEDY DP SEGMENTER  
# ══════════════════════════════════════════════════════════════════════════  
  
def build_seg(chunks, max_chunks):  
    cs = set(chunks)  
    mx = max((len(c) for c in cs), default=0)  
  
    def seg(word):  
        n = len(word)  
        dp = [None] * (n + 1)  
        dp[0] = (0, [])  
        for i in range(n):  
            if dp[i] is None:  
                continue  
            p, sq = dp[i]  
            if p >= max_chunks:  
                continue  
            for e in range(i + 1, min(i + mx + 1, n + 1)):  
                c = word[i:e]  
                if c not in cs:  
                    continue  
                np_ = p + 1  
                ns = sq + [c]  
                if dp[e] is None:  
                    dp[e] = (np_, ns)  
                elif np_ < dp[e][0]:  
                    dp[e] = (np_, ns)  
                elif np_ == dp[e][0]:  
                    os = dp[e][1]  
                    for j in range(min(len(ns), len(os))):  
                        if len(ns[j]) > len(os[j]):  
                            dp[e] = (np_, ns)  
                            break  
                        elif len(ns[j]) < len(os[j]):  
                            break  
        return dp[n][1] if dp[n] else None  
  
    return seg  
  
  
# ══════════════════════════════════════════════════════════════════════════  
# MAIN  
# ══════════════════════════════════════════════════════════════════════════  
  
def main():  
    if len(sys.argv) < 2:  
        print("Usage: python3 manzini_v17_trained_vs_raw.py <ivtff>")  
        sys.exit(1)  
  
    W = 88  
    D = '─' * W  
  
    print(f"\n{'=' * W}")  
    print("  MANZINI RAW GRID vs ZATTERA TRAINED STATE MACHINE")  
    print(f"{'=' * W}")  
  
    tok = parse_ivtff(sys.argv[1])  
    wc = filt(tok)  
    T = sum(wc.values())  
    N = len(wc)  
    language = set(wc.keys())  
    chars = set()  
    for w in wc:  
        chars.update(w)  
    CS = len(chars)  
  
    chunks = gen_chunks(GRID)  
    flat = nbits_flat(wc, wc.keys(), CS)  
  
    # Build prefix trie from all corpus words (for filtered generation)  
    trie = PrefixTrie(language)  
  
    print(f"\n  Corpus     : {T:,} tokens, {N:,} types, Csize={CS}")  
    print(f"  Grid chunks: {len(chunks):,}")  
    print(f"  Max word len: {trie.max_len}")  
    print(f"  Flat Huffman: {flat['nb']:,} bits, {flat['bpt']:.2f} b/tok")  
  
    # ══════════════════════════════════════════════════════════════════  
    # MODEL A: Manzini raw grid, greedy DP  
    # ══════════════════════════════════════════════════════════════════  
  
    print(f"\n{'=' * W}")  
    print("  MODEL A: Manzini raw grid (greedy DP, max_rep=5)")  
    print(D)  
  
    seg_a = build_seg(chunks, 5)  
    segs_a = {}  
    for w in wc:  
        s = seg_a(w)  
        if s:  
            segs_a[w] = s  
    nb_a = nbits_chunks(wc, segs_a, CS)  
  
    print(f"  Types covered: {len(segs_a):,} / {N:,} ({len(segs_a)/N:.1%})")  
    print(f"  Nb: {nb_a['nb']:,} bits, {nb_a['bpt']:.2f} b/tok")  
  
    # ══════════════════════════════════════════════════════════════════  
    # MODEL B: Zattera trained 1-loop  
    # ══════════════════════════════════════════════════════════════════  
  
    print(f"\n{'=' * W}")  
    print("  MODEL B: Zattera trained state machine (1 loop)")  
    print(D)  
  
    print("  Building 1-loop machine...", flush=True)  
    m1 = build_machine_from_grid(GRID, n_loops=1)  
    m1.set_prefix_trie(trie)  
    print(f"    States: {m1.count_states()}, Edges: {m1.count_edges()}")  
  
    print("  Generating all words (before training)...", flush=True)  
    pre_gen1 = m1.emit_all(use_filter=False)  
    pre_tp1 = len(pre_gen1 & language)  
    pre_fp1 = len(pre_gen1 - language)  
    print(f"    Generated: {len(pre_gen1):,} words")  
    print(f"    TP: {pre_tp1:,}, FP: {pre_fp1:,}")  
    if pre_tp1 + pre_fp1 > 0:  
        print(f"    Precision: {pre_tp1/(pre_tp1+pre_fp1):.3f}, "  
              f"Recall: {pre_tp1/len(language):.3f}")  
  
    print("  Training (F1 mode)...", flush=True)  
    gen1 = m1.train(language, mode='F1', use_filter=False)  
    trained_vocab_1 = set(gen1)  
  
    tp1 = len(trained_vocab_1 & language)  
    fp1 = len(trained_vocab_1 - language)  
    p1 = tp1 / (tp1 + fp1) if (tp1 + fp1) > 0 else 0  
    r1 = tp1 / len(language)  
    f1_1 = 2 * p1 * r1 / (p1 + r1) if (p1 + r1) > 0 else 0  
  
    seg_b = build_seg(trained_vocab_1, 5)  
    segs_b = {}  
    for w in wc:  
        s = seg_b(w)  
        if s:  
            segs_b[w] = s  
    nb_b = nbits_chunks(wc, segs_b, CS)  
  
    print(f"\n  Trained vocabulary: {len(trained_vocab_1):,} chunks")  
    print(f"  Types covered: {len(segs_b):,} / {N:,} ({len(segs_b)/N:.1%})")  
    print(f"  Nb: {nb_b['nb']:,} bits, {nb_b['bpt']:.2f} b/tok")  
  
    # ══════════════════════════════════════════════════════════════════  
    # MODEL C: Zattera trained 2-loop  
    # ══════════════════════════════════════════════════════════════════  
  
    print(f"\n{'=' * W}")  
    print("  MODEL C: Zattera trained state machine (2 loops)")  
    print(D)  
  
    print("  Building 2-loop machine...", flush=True)  
    m2 = build_machine_from_grid(GRID, n_loops=2)  
    m2.set_prefix_trie(trie)  
    print(f"    States: {m2.count_states()}, Edges: {m2.count_edges()}")  
  
    print("  Generating all words with prefix filter (before training)...",  
          flush=True)  
    pre_gen2 = m2.emit_all(use_filter=True)  
    pre_tp2 = len(pre_gen2 & language)  
    pre_fp2 = len(pre_gen2 - language)  
    print(f"    Generated: {len(pre_gen2):,} words")  
    print(f"    TP: {pre_tp2:,}, FP: {pre_fp2:,}")  
    if pre_tp2 + pre_fp2 > 0:  
        print(f"    Precision: {pre_tp2/(pre_tp2+pre_fp2):.3f}, "  
              f"Recall: {pre_tp2/len(language):.3f}")  
  
    print("  Training (F1 mode, prefix-filtered)...", flush=True)  
    gen2 = m2.train(language, mode='F1', use_filter=True)  
    trained_vocab_2 = set(gen2)  
  
    tp2 = len(trained_vocab_2 & language)  
    fp2 = len(trained_vocab_2 - language)  
    p2 = tp2 / (tp2 + fp2) if (tp2 + fp2) > 0 else 0  
    r2 = tp2 / len(language)  
    f1_2 = 2 * p2 * r2 / (p2 + r2) if (p2 + r2) > 0 else 0  
  
    seg_c = build_seg(trained_vocab_2, 5)  
    segs_c = {}  
    for w in wc:  
        s = seg_c(w)  
        if s:  
            segs_c[w] = s  
    nb_c = nbits_chunks(wc, segs_c, CS)  
  
    print(f"\n  Trained vocabulary: {len(trained_vocab_2):,} chunks")  
    print(f"  Types covered: {len(segs_c):,} / {N:,} ({len(segs_c)/N:.1%})")  
    print(f"  Nb: {nb_c['nb']:,} bits, {nb_c['bpt']:.2f} b/tok")  
  
    # ══════════════════════════════════════════════════════════════════  
    # SIDE-BY-SIDE  
    # ══════════════════════════════════════════════════════════════════  
  
    print(f"\n{'=' * W}")  
    print("  SIDE-BY-SIDE COMPARISON")  
    print(D)  
  
    cpw_a = nb_a['total_chunks'] / nb_a['T'] if nb_a['T'] else 0  
    cpw_b = nb_b['total_chunks'] / nb_b['T'] if nb_b['T'] else 0  
    cpw_c = nb_c['total_chunks'] / nb_c['T'] if nb_c['T'] else 0  
  
    print(f"""  
  All models built from the same Manzini 7-column grid.  
  
  A = Raw grid chunks ({len(chunks):,}), greedy DP (max_rep=5)  
  B = Zattera-trained 1-loop → {len(trained_vocab_1):,} words as chunks → greedy DP  
  C = Zattera-trained 2-loop → {len(trained_vocab_2):,} words as chunks → greedy DP  
  
  ┌──────────────────┬────────────┬────────────┬────────────┐  
  │ Metric           │  A: Raw DP │ B: Train×1 │ C: Train×2 │  
  ├──────────────────┼────────────┼────────────┼────────────┤  
  │ Chunk vocab size │ {len(chunks):>10,} │ {len(trained_vocab_1):>10,} │ {len(trained_vocab_2):>10,} │  
  │ Types covered    │ {len(segs_a):>10,} │ {len(segs_b):>10,} │ {len(segs_c):>10,} │  
  │ Type coverage    │ {len(segs_a)/N:>9.1%}  │ {len(segs_b)/N:>9.1%}  │ {len(segs_c)/N:>9.1%}  │  
  │ Token coverage   │ {nb_a['T']/T:>9.1%}  │ {nb_b['T']/T:>9.1%}  │ {nb_c['T']/T:>9.1%}  │  
  │ Chunks used      │ {nb_a['n']:>10,} │ {nb_b['n']:>10,} │ {nb_c['n']:>10,} │  
  │ Avg ch/word      │ {cpw_a:>10.3f} │ {cpw_b:>10.3f} │ {cpw_c:>10.3f} │  
  │ Nb_dict          │ {nb_a['nb_d']:>10,} │ {nb_b['nb_d']:>10,} │ {nb_c['nb_d']:>10,} │  
  │ Nb_text          │ {nb_a['nb_t']:>10,} │ {nb_b['nb_t']:>10,} │ {nb_c['nb_t']:>10,} │  
  │ Nb_total         │ {nb_a['nb']:>10,} │ {nb_b['nb']:>10,} │ {nb_c['nb']:>10,} │  
  │ b/tok            │ {nb_a['bpt']:>10.2f} │ {nb_b['bpt']:>10.2f} │ {nb_c['bpt']:>10.2f} │  
  │ vs Flat          │ {nb_a['nb']/flat['nb']:>10.3f}× │ {nb_b['nb']/flat['nb']:>10.3f}× │ {nb_c['nb']/flat['nb']:>10.3f}× │  
  └──────────────────┴────────────┴────────────┴────────────┘  
  
  Machine training stats:  
  ┌──────────────────┬────────────┬────────────┐  
  │ Metric           │ B: Train×1 │ C: Train×2 │  
  ├──────────────────┼────────────┼────────────┤  
  │ Precision        │ {p1:>10.3f} │ {p2:>10.3f} │  
  │ Recall           │ {r1:>10.3f} │ {r2:>10.3f} │  
  │ F1               │ {f1_1:>10.3f} │ {f1_2:>10.3f} │  
  │ True positives   │ {tp1:>10,} │ {tp2:>10,} │  
  │ False positives  │ {fp1:>10,} │ {fp2:>10,} │  
  └──────────────────┴────────────┴────────────┘""")  
  
    # ══════════════════════════════════════════════════════════════════  
    # EXAMPLES  
    # ══════════════════════════════════════════════════════════════════  
  
    print(f"\n{'=' * W}")  
    print("  DECOMPOSITION EXAMPLES")  
    print(D)  
  
    test_words = [  
        'daiin', 'chey', 'qokeey', 'chol', 'shey', 'chedy',  
        'qokain', 'dar', 'qokaiin', 'cheey', 'cheol',  
        'ol', 'ar', 'shol', 'shor', 'okeey', 'lkaiin',  
        'chodaiin', 'chokain', 'daly', 'otaly',  
    ]  
  
    print(f"\n  {'Word':<14} {'Cnt':>4} │ {'A: Raw DP':<22} │ "  
          f"{'B: Train×1':<22} │ {'C: Train×2':<22}")  
    print(f"  {'─' * 90}")  
  
    for w in test_words:  
        if w not in wc:  
            continue  
        sa = segs_a.get(w)  
        sb = segs_b.get(w)  
        sc = segs_c.get(w)  
        sa_s = '+'.join(sa) if sa else '—'  
        sb_s = '+'.join(sb) if sb else '—'  
        sc_s = '+'.join(sc) if sc else '—'  
        print(f"  {w:<14} {wc[w]:>4} │ {sa_s:<22} │ "  
              f"{sb_s:<22} │ {sc_s:<22}")  
  
    # ══════════════════════════════════════════════════════════════════  
    # COVERAGE GAPS  
    # ══════════════════════════════════════════════════════════════════  
  
    print(f"\n{'=' * W}")  
    print("  COVERAGE GAPS")  
    print(D)  
  
    for name, segs, nb in [('A: Raw DP', segs_a, nb_a),  
                            ('B: Train×1', segs_b, nb_b),  
                            ('C: Train×2', segs_c, nb_c)]:  
        failed = [w for w in wc if w not in segs]  
        fail_tok = sum(wc[w] for w in failed)  
        print(f"\n  {name}: {len(failed)} failed types, "  
              f"{fail_tok:,} tokens ({fail_tok/T:.1%})")  
        for w in sorted(failed, key=lambda x: wc[x], reverse=True)[:10]:  
            print(f"    {w:<20} {wc[w]:>4}")  
  
    # ══════════════════════════════════════════════════════════════════  
    # WORDS GAINED BY 2-LOOP TRAINING  
    # ══════════════════════════════════════════════════════════════════  
  
    print(f"\n{'=' * W}")  
    print("  WORDS IN C (2-loop trained) BUT NOT B (1-loop trained)")  
    print(D)  
  
    only_c = trained_vocab_2 - trained_vocab_1  
    in_lang = only_c & language  
    not_lang = only_c - language  
    print(f"\n  {len(only_c)} words unique to 2-loop training")  
    print(f"    {len(in_lang)} are in corpus (true positives)")  
    print(f"    {len(not_lang)} are not in corpus (false positives)")  
  
    if in_lang:  
        in_lang_sorted = sorted(in_lang, key=lambda w: wc.get(w, 0),  
                                 reverse=True)  
        print(f"\n  Top corpus words gained by 2-loop:")  
        for w in in_lang_sorted[:20]:  
            print(f"    {w:<20} {wc.get(w,0):>4}")  
  
    # ══════════════════════════════════════════════════════════════════  
    # DELTAS  
    # ══════════════════════════════════════════════════════════════════  
  
    print(f"\n{'=' * W}")  
    print("  PAIRWISE DELTAS")  
    print(D)  
  
    for label, nb_x, nb_y in [('B vs A', nb_b, nb_a),  
                                ('C vs A', nb_c, nb_a),  
                                ('C vs B', nb_c, nb_b)]:  
        d = nb_x['bpt'] - nb_y['bpt']  
        db = nb_x['nb'] - nb_y['nb']  
        winner = label[0] if d < 0 else label[-1]  
        print(f"  {label}: {d:+.2f} b/tok ({db:+,} bits) → {winner} wins")  
  
    print(f"\n{'=' * W}")  
  
  
if __name__ == '__main__':  
    main()