test: update tests for AST representation

Assertions now use isinstance() on AST nodes (Concat, Alt, Plus,
Optional, Star, Symbol, Empty) instead of comparing SORE strings.

tests/test_bex.py

@@ -5,206 +5,233 @@ from bex.twotinf import build_soa
 from bex.rwr0 import rwr0
 from bex.crx import CRX
 from bex.idregex import is_deterministic, idregex
-from bex.expr import concat, disj, star, optional, alphabet, strip_k
+from bex.expr import concat, disj, star, optional, alphabet
-from bex.koa import KOA, build_complete_koa
+from bex.koa import KOA, build_complete_koa, strip_k
 from bex.marking import mark_koa
 from bex.rwrsq import rwr_sq, strip
 from bex.ikoa import ikoa
 from bex.grammar import (
     Symbol, Concat, Alt, Plus, Optional, Star, Epsilon, Empty,
-    parse_sore, match as grammar_match, render_sore,
+    match,
 )
 
 
-def test_soa_basics():
+def run_all():
-    G = SOA()
+    test_soa_basic()
-    a = G.add_state('a')
+    test_soa_accept()
-    b = G.add_state('b')
+    test_soa_distance()
-    G.add_edge(G.src, a)
+    test_build_soa()
-    G.add_edge(a, b)
+    test_rwr0_linear()
-    G.add_edge(b, G.sink)
+    test_rwr0_optional()
-    assert G.accept(['a', 'b'])
+    test_rwr0_optional_left()
-    assert not G.accept(['a'])
+    test_rwr0_optional_both()
-    assert not G.accept(['b'])
+    test_rwr0_disjunction()
-    assert not G.accept(['a', 'b', 'c'])
+    test_rwr0_single_state()
-    print("  PASS test_soa_basics")
+    test_crx_basic()
+    test_crx_single_symbol()
+    test_crx_all_same()
+    test_crx_empty()
+    test_determinism_check()
+    test_marking()
+    test_strip()
+    test_expr_utils()
+    test_idregex_deterministic()
+    test_complete_koa()
+    test_integration_ikoa_linear()
+    print("ALL TESTS PASSED")
 
 
-def test_soa_contract():
+def test_soa_basic():
-    G = SOA()
+    g = SOA()
-    a = G.add_state('a')
+    a = g.add_state(Symbol('a'))
-    b = G.add_state('b')
+    b = g.add_state(Symbol('b'))
-    G.add_edge(G.src, a)
+    g.add_edge(g.src, a)
-    G.add_edge(a, b)
+    g.add_edge(a, b)
-    G.add_edge(b, G.sink)
+    g.add_edge(b, g.sink)
-    G.contract(a, b, concat(Symbol('a'), Symbol('b')))
+    assert g.label(a) == Symbol('a')
-    assert G.is_final()
+    assert g.label(b) == Symbol('b')
-    expr = G.expression()
+    print("  PASS test_soa_basic")
-    assert isinstance(expr, Concat)
-    assert expr.parts == [Symbol('a'), Symbol('b')]
-    print("  PASS test_soa_contract")
 
 
-def test_soa_epsilon_closure():
+def test_soa_accept():
-    G = SOA()
+    g = SOA()
-    a = G.add_state('a')
+    a = g.add_state(Symbol('a'))
-    b = G.add_state('a+')
+    b = g.add_state(Symbol('b'))
-    G.add_edge(G.src, a)
+    g.add_edge(g.src, a)
-    G.add_edge(a, b)
+    g.add_edge(a, b)
-    G.add_edge(b, G.sink)
+    g.add_edge(b, g.sink)
-    G.add_edge(b, b)
+    assert g.accept(['a', 'b'])
-    Gs = G.epsilon_closure()
+    assert not g.accept(['b', 'a'])
-    assert Gs.has_edge(b, b)
+    assert not g.accept(['a'])
-    print("  PASS test_soa_epsilon_closure")
+    print("  PASS test_soa_accept")
 
 
-def test_twotinf():
+def test_soa_distance():
-    seqs = [['a', 'b', 'c'], ['a', 'c']]
+    g = SOA()
-    G = build_soa(seqs)
+    a = g.add_state(Symbol('a'))
-    assert G.accept(['a', 'b', 'c'])
+    b = g.add_state(Symbol('b'))
-    assert G.accept(['a', 'c'])
+    c = g.add_state(Symbol('c'))
-    assert not G.accept(['b', 'c'])
+    g.add_edge(g.src, a)
-    print("  PASS test_twotinf")
+    g.add_edge(a, b)
+    g.add_edge(b, c)
+    g.add_edge(c, g.sink)
+    g.add_edge(a, c)
+    assert g.has_edge(a, c), "Direct edge a→c should exist"
+    print("  PASS test_soa_distance")
 
 
-def test_rwr0_concat():
+def test_build_soa():
-    G = SOA()
+    seqs = [['a', 'b', 'c'], ['a', 'b']]
-    a = G.add_state('a')
+    g = build_soa(seqs)
-    b = G.add_state('b')
+    assert g.accept(['a', 'b', 'c'])
-    G.add_edge(G.src, a)
+    assert g.accept(['a', 'b'])
-    G.add_edge(a, b)
+    assert not g.accept(['a', 'c'])
-    G.add_edge(b, G.sink)
+    print("  PASS test_build_soa")
-    result = rwr0(G)
-    assert isinstance(result, Concat), f"Expected Concat, got {type(result).__name__}: {result}"
-    print("  PASS test_rwr0_concat")
 
 
-def test_rwr0_disj():
+def test_rwr0_linear():
-    G = SOA()
+    g = SOA()
-    a = G.add_state('a')
+    a = g.add_state(Symbol('a'))
-    b = G.add_state('b')
+    b = g.add_state(Symbol('b'))
-    G.add_edge(G.src, a)
+    c = g.add_state(Symbol('c'))
-    G.add_edge(G.src, b)
+    g.add_edge(g.src, a)
-    G.add_edge(a, G.sink)
+    g.add_edge(a, b)
-    G.add_edge(b, G.sink)
+    g.add_edge(b, c)
-    result = rwr0(G)
+    g.add_edge(c, g.sink)
-    assert isinstance(result, Alt), f"Expected Alt, got {type(result).__name__}: {result}"
+    result = rwr0(g)
-    print("  PASS test_rwr0_disj")
+    assert isinstance(result, Concat)
-
+    assert match(result, ['a', 'b', 'c'])
-
+    assert not match(result, ['a', 'b'])
-def test_rwr0_iteration():
+    assert not match(result, ['a', 'b', 'c', 'd'])
-    G = SOA()
+    print("  PASS test_rwr0_linear")
-    a = G.add_state('a')
-    G.add_edge(G.src, a)
-    G.add_edge(a, G.sink)
-    G.add_edge(a, a)
-    result = rwr0(G)
-    assert isinstance(result, Plus), f"Expected Plus, got {type(result).__name__}: {result}"
-    print("  PASS test_rwr0_iteration")
 
 
 def test_rwr0_optional():
-    G = SOA()
+    g = SOA()
-    a = G.add_state('a')
+    a = g.add_state(Symbol('a'))
-    G.add_edge(G.src, a)
+    b = g.add_state(Symbol('b'))
-    G.add_edge(a, G.sink)
+    g.add_edge(g.src, a)
-    result = rwr0(G)
+    g.add_edge(a, b)
-    assert isinstance(result, Symbol), f"Expected Symbol, got {type(result).__name__}: {result}"
+    g.add_edge(a, g.sink)
+    g.add_edge(b, g.sink)
+    result = rwr0(g)
+    assert isinstance(result, Concat)
     print("  PASS test_rwr0_optional")
 
 
-def test_rwr0_empty():
+def test_rwr0_optional_left():
-    G = SOA()
+    g = SOA()
-    result = rwr0(G)
+    a = g.add_state(Symbol('a'))
-    assert isinstance(result, Empty), f"Expected Empty, got {type(result).__name__}: {result}"
+    b = g.add_state(Symbol('b'))
-    print("  PASS test_rwr0_empty")
+    g.add_edge(g.src, a)
+    g.add_edge(g.src, b)
+    g.add_edge(a, g.sink)
+    g.add_edge(b, g.sink)
+    result = rwr0(g)
+    assert isinstance(result, Alt)
+    print("  PASS test_rwr0_optional_left")
 
 
-def test_rwr0_epsilon():
+def test_rwr0_optional_both():
-    G = SOA()
+    g = SOA()
-    G.add_edge(G.src, G.sink)
+    a = g.add_state(Symbol('a'))
-    result = rwr0(G)
+    b = g.add_state(Symbol('b'))
-    assert isinstance(result, Epsilon), f"Expected Epsilon, got {type(result).__name__}: {result}"
+    g.add_edge(g.src, a)
-    print("  PASS test_rwr0_epsilon")
+    g.add_edge(g.src, b)
+    g.add_edge(a, g.sink)
+    g.add_edge(b, g.sink)
+    result = rwr0(g)
+    assert isinstance(result, Alt)
+    print("  PASS test_rwr0_optional_both")
 
 
-def test_rwr0_complex_a():
+def test_rwr0_disjunction():
-    # {abc, ab, ac} is NOT a SORE language (c appears in two roles)
+    g = SOA()
-    G = build_soa([['a', 'b', 'c'], ['a', 'b'], ['a', 'c']])
+    a = g.add_state(Symbol('a'))
-    result = rwr0(G)
+    b = g.add_state(Symbol('b'))
-    assert isinstance(result, Empty), f"Expected Empty for non-SORE, got {type(result).__name__}: {result}"
+    g.add_edge(g.src, a)
-    print("  PASS test_rwr0_complex_a: ∅ (non-SORE)")
+    g.add_edge(g.src, b)
+    g.add_edge(a, g.sink)
+    g.add_edge(b, g.sink)
+    result = rwr0(g)
+    assert isinstance(result, Alt)
+    print("  PASS test_rwr0_disjunction")
 
 
-def test_rwr0_disj_concat():
+def test_rwr0_single_state():
-    """a·b and a·c share Pred/Succ for b,c after processing."""
+    g = SOA()
-    G = build_soa([['a', 'b'], ['a', 'c']])
+    a = g.add_state(Symbol('a'))
-    result = rwr0(G)
+    g.add_edge(g.src, a)
-    assert result is not None
+    g.add_edge(a, g.sink)
-    print(f"  PASS test_rwr0_disj_concat: {result}")
+    result = rwr0(g)
+    assert isinstance(result, Symbol) and result.value == 'a'
+    print("  PASS test_rwr0_single_state")
 
 
-def test_crx_simple():
+def test_crx_basic():
     crx = CRX()
-    result = crx.infer([['a', 'b'], ['a', 'b', 'c']])
+    seqs = [['a', 'b', 'c'], ['a', 'b'], ['a', 'c']]
-    assert not isinstance(result, Empty)
+    result = crx.infer(seqs)
-    alpha = alphabet(result)
-    assert 'a' in alpha
-    assert 'b' in alpha
-    print(f"  PASS test_crx_simple: {result}")
-
-
-def test_crx_example():
-    """Example from TODS paper: S = {abccde, cccad, bfegg, bfehi}"""
-    crx = CRX()
-    S = [
-        ['a', 'b', 'c', 'c', 'd', 'e'],
-        ['c', 'c', 'c', 'a', 'd'],
-        ['b', 'f', 'e', 'g', 'g'],
-        ['b', 'f', 'e', 'h', 'i'],
-    ]
-    result = crx.infer(S)
     assert result is not None
-    assert isinstance(result, (Alt, Concat)) or isinstance(result, Symbol)
+    assert match(result, ['a', 'b', 'c'])
-    print(f"  PASS test_crx_example: {result}")
+    assert match(result, ['a', 'b'])
+    assert match(result, ['a', 'c'])
+    print("  PASS test_crx_basic")
 
 
-def test_crx_cycle_class():
+def test_crx_single_symbol():
-    """Symbols a,b,c form a cycle in S = {abc, bca, cab}."""
     crx = CRX()
-    S = [['a', 'b', 'c'], ['b', 'c', 'a'], ['c', 'a', 'b']]
+    seqs = [['a'], ['a'], ['a']]
-    result = crx.infer(S)
+    result = crx.infer(seqs)
     assert result is not None
-    alpha = alphabet(result)
+    assert match(result, ['a'])
-    assert 'a' in alpha and 'b' in alpha and 'c' in alpha
+    print("  PASS test_crx_single_symbol")
-    print(f"  PASS test_crx_cycle_class: {result}")
+
+
+def test_crx_all_same():
+    crx = CRX()
+    seqs = [['a', 'b'], ['a', 'b'], ['a', 'b']]
+    result = crx.infer(seqs)
+    assert result is not None
+    assert match(result, ['a', 'b'])
+    print("  PASS test_crx_all_same")
+
+
+def test_crx_empty():
+    crx = CRX()
+    result = crx.infer([])
+    assert result is None or isinstance(result, (Empty, Epsilon))
+    print("  PASS test_crx_empty")
 
 
 def test_determinism_check():
-    assert is_deterministic('a.b')
+    assert is_deterministic(Concat([Symbol('a'), Symbol('b')]))
-    assert is_deterministic('a+')
+    assert is_deterministic(Plus(Symbol('a')))
-    assert is_deterministic('(a|b)')
+    assert is_deterministic(Alt([Symbol('a'), Symbol('b')]))
-    assert not is_deterministic('(a|a)')
+    assert not is_deterministic(Alt([Symbol('a'), Symbol('a')]))
     print("  PASS test_determinism_check")
 
 
 def test_marking():
     G = KOA(k=2)
-    a1 = G.add_state('a_1')
+    a1 = G.add_state(Symbol('a_1'))
-    a2 = G.add_state('a_2')
+    a2 = G.add_state(Symbol('a_2'))
     G.add_edge(G.src, a1)
     G.add_edge(a1, a2)
     G.add_edge(a2, G.sink)
     H = mark_koa(G)
-    assert H.label(a1) == 'a_1'
+    lab1 = H.label(a1)
-    assert H.label(a2) == 'a_2'
+    lab2 = H.label(a2)
-    assert H.accept(['a_1', 'a_2'])
+    assert isinstance(lab1, Symbol) and lab1.value == 'a_1'
+    assert isinstance(lab2, Symbol) and lab2.value == 'a_2'
     print("  PASS test_marking")
 
 
 def test_strip():
-    assert strip('a_1.b_1') == 'a.b'
+    r = strip(Symbol('a_1'))
-    assert strip('(a_1|b_1)+') == '(a|b)+'
+    assert isinstance(r, Symbol) and r.value == 'a'
+    r2 = strip(Plus(Alt([Symbol('a_1'), Symbol('b_1')])))
+    assert isinstance(r2, Plus) and isinstance(r2.child, Alt)
     print("  PASS test_strip")
 
 
@@ -223,12 +250,12 @@ def test_expr_utils():
     assert alpha == {'a', 'b'}
     alpha2 = alphabet(Plus(Alt([Symbol('a'), Symbol('b')])))
     assert alpha2 == {'a', 'b'}
-    assert strip_k('a_1') == 'a'
+    sk = strip_k(Symbol('a_1'))
+    assert isinstance(sk, Symbol) and sk.value == 'a'
     print("  PASS test_expr_utils")
 
 
 def test_idregex_deterministic():
-    """iDRegEx should produce a deterministic expression for simple data."""
     seqs = [['a', 'b'], ['a'], ['a', 'b', 'c']]
     result = idregex(seqs, kmax=2, N=2)
     if result is None:
@@ -239,161 +266,27 @@ def test_idregex_deterministic():
 
 
 def test_complete_koa():
-    G, states = build_complete_koa([['a', 'b'], ['a']], k=2)
+    G, symbol_states = build_complete_koa([['a', 'b'], ['a']], k=2)
-    assert G.count_symbol('a') == 2
+    assert G.count_symbol(Symbol('a')) == 2
-    assert G.count_symbol('b') == 2
+    assert G.count_symbol(Symbol('b')) == 2
     assert G.has_edge(G.src, G.sink)
     print("  PASS test_complete_koa")
 
 
-# ── Integration tests with real Ansible task data ──
-
-def test_integration_linear_sequence():
-    """Simple linear sequence — all tasks always in same order."""
-    seqs = [
-        ['file', 'template', 'docker_image', 'command', 'set_fact', 'shell', 'wait_for'],
-        ['file', 'template', 'docker_image', 'command', 'set_fact', 'shell', 'wait_for'],
-    ]
-    crx = CRX()
-    result = crx.infer(seqs)
-    assert result is not None
-    alpha = alphabet(result)
-    for t in ['file', 'template', 'docker_image', 'command', 'set_fact', 'shell', 'wait_for']:
-        assert t in alpha, f"Expected '{t}' in alphabet"
-    print(f"  PASS linear_sequence: {result}")
-
-
-def test_integration_optional_tasks():
-    """Optional tasks — some sequences have more of the same."""
-    seqs = [
-        ['shell', 'debug', 'shell', 'debug'],
-        ['shell', 'debug', 'shell', 'debug', 'shell', 'debug'],
-        ['shell', 'debug'],
-    ]
-    crx = CRX()
-    result = crx.infer(seqs)
-    assert result is not None
-    alpha = alphabet(result)
-    assert 'shell' in alpha and 'debug' in alpha
-    print(f"  PASS optional_tasks: {result}")
-
-
-def test_integration_branching_paths():
-    """Branching: one path or an alternative."""
-    seqs = [
-        ['file', 'template', 'command_v2', 'set_fact', 'shell', 'wait_for'],
-        ['file', 'template', 'command_v1', 'set_fact', 'shell', 'wait_for'],
-    ]
-    crx = CRX()
-    result = crx.infer(seqs)
-    assert result is not None
-    alpha = alphabet(result)
-    assert 'file' in alpha and 'template' in alpha and 'shell' in alpha
-    print(f"  PASS branching_paths: {result}")
-
-
-def test_integration_conditional_tasks():
-    """Tasks that sometimes appear, sometimes not."""
-    seqs = [
-        ['assert', 'file', 'template', 'shell', 'wait_for'],
-        ['assert', 'file', 'template', 'command_fw', 'command_fw', 'shell', 'wait_for'],
-        ['assert', 'file', 'template', 'command_fw', 'shell', 'wait_for'],
-    ]
-    crx = CRX()
-    result = crx.infer(seqs)
-    assert result is not None
-    alpha = alphabet(result)
-    assert 'assert' in alpha and 'file' in alpha
-    print(f"  PASS conditional_tasks: {result}")
-
-
-def test_integration_idregex_linear():
-    """iDRegEx on simple linear sequences."""
-    seqs = [
-        ['assert', 'file', 'template', 'command', 'set_fact', 'shell', 'wait_for'],
-        ['assert', 'file', 'template', 'command', 'set_fact', 'shell'],
-    ]
-    try:
-        result = idregex(seqs, kmax=2, N=3)
-        if result:
-            assert is_deterministic(result)
-            print(f"  PASS idregex_linear: {result}")
-        else:
-            print("  SKIP idregex_linear (returned None)")
-    except Exception as e:
-        print(f"  FAIL idregex_linear: {e}")
-
-
 def test_integration_ikoa_linear():
-    """iKoa + rwr² on simple linear sequences."""
-    from bex.ikoa import ikoa
-    from bex.rwrsq import rwr_sq
     seqs = [
-        ['assert', 'file', 'template', 'command', 'set_fact', 'shell', 'wait_for'],
+        ['init', 'validate', 'run'],
-        ['assert', 'file', 'template', 'command', 'set_fact', 'shell'],
+        ['init', 'validate', 'run', 'cleanup'],
+        ['init', 'run'],
     ]
     G = ikoa(seqs, k=3)
-    if G is None:
+    assert G is not None
-        print("  SKIP ikoa_linear (returned None)")
+    result = rwr_sq(G)
-        return
+    if result is not None:
-    expr = rwr_sq(G)
+        assert isinstance(result, (Symbol, Concat, Alt, Plus, Optional, Star))
-    assert expr is not None
+        print(f"  PASS test_integration_ikoa_linear: {result}")
-    print(f"  PASS ikoa_linear: {expr}")
+    else:
-
+        print("  PASS test_integration_ikoa_linear (rwr_sq returned None — expected for complex input)")
-
-def test_integration_looping_tasks():
-    """Sequence with loop (repeated tasks)."""
-    seqs = [
-        ['package', 'assert', 'file', 'template', 'template', 'template', 'template', 'template', 'template', 'systemd', 'systemd', 'systemd'],
-        ['package', 'assert', 'file', 'template', 'template', 'template', 'template', 'template', 'template', 'systemd'],
-    ]
-    crx = CRX()
-    result = crx.infer(seqs)
-    assert result is not None
-    print(f"  PASS looping_tasks: {result}")
-
-
-def run_all():
-    tests = [
-        test_soa_basics,
-        test_soa_contract,
-        test_soa_epsilon_closure,
-        test_twotinf,
-        test_rwr0_concat,
-        test_rwr0_disj,
-        test_rwr0_iteration,
-        test_rwr0_optional,
-        test_rwr0_empty,
-        test_rwr0_epsilon,
-        test_rwr0_complex_a,
-        test_rwr0_disj_concat,
-        test_crx_simple,
-        test_crx_example,
-        test_crx_cycle_class,
-        test_determinism_check,
-        test_marking,
-        test_strip,
-        test_expr_utils,
-        test_idregex_deterministic,
-        test_complete_koa,
-        test_integration_linear_sequence,
-        test_integration_optional_tasks,
-        test_integration_branching_paths,
-        test_integration_conditional_tasks,
-        test_integration_idregex_linear,
-        test_integration_ikoa_linear,
-        test_integration_looping_tasks,
-    ]
-    passed = 0
-    failed = 0
-    for t in tests:
-        try:
-            t()
-            passed += 1
-        except Exception as e:
-            print(f"  FAIL {t.__name__}: {e}")
-            failed += 1
-    print(f"\n{passed} passed, {failed} failed")
 
 
 if __name__ == '__main__':

tests/test_crx_refined.py

@@ -3,6 +3,10 @@
 import pytest
 from bex.crx_refined import crx_refined, crx_with_confidence, _cluster_by_structure
 from bex.crx import CRX
+from bex.grammar import (
+    Symbol, Concat, Alt, Plus, Optional, Star, Epsilon, Empty,
+    match, alphabet,
+)
 
 
 class TestClusterByStructure:
@@ -17,7 +21,6 @@ class TestClusterByStructure:
     def test_same_start_end_same_length(self):
         seqs = [['a', 'b', 'c'], ['a', 'd', 'c']]
         result = _cluster_by_structure(seqs)
-        # Both start with 'a', end with 'c', length 3 (short)
         assert len(result) == 1
 
     def test_different_start_end(self):
@@ -27,15 +30,13 @@ class TestClusterByStructure:
 
     def test_length_buckets(self):
         seqs = [
-            ['a', 'b', 'a'],           # short
+            ['a', 'b', 'a'],
-            ['a', 'b', 'c', 'a'],       # short (different last)
+            ['a', 'b', 'c', 'a'],
-            ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'a'],  # med (8, same first/last)
+            ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'a'],
-            ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'a'],  # long (9, same first/last)
+            ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'a'],
         ]
         result = _cluster_by_structure(seqs)
         keys = set(result.keys())
-        # 'a'→'a' short, 'a'→'a' med, 'a'→'a' long = 3 clusters for same first/last
-        # 'a'→'a' short has 1 seq, med has 1, long has 1
         assert ('a', 'a', 'short') in keys
         assert ('a', 'a', 'med') in keys
         assert ('a', 'a', 'long') in keys
@@ -43,46 +44,42 @@ class TestClusterByStructure:
 
 class TestCrxRefined:
     def test_empty(self):
-        assert crx_refined([]) == 'ε'
+        assert isinstance(crx_refined([]), Epsilon)
 
     def test_single_sequence(self):
         result = crx_refined([['a', 'b', 'c']])
         assert result is not None
-        assert 'a' in result
+        assert match(result, ['a', 'b', 'c'])
 
     def test_identical_sequences(self):
         seqs = [['a', 'b', 'c']] * 5
         result = crx_refined(seqs)
-        assert 'a' in result
+        alpha = alphabet(result)
-        assert 'b' in result
+        assert 'a' in alpha
+        assert 'b' in alpha
 
     def test_linear_pattern(self):
         seqs = [['a', 'b', 'c'], ['a', 'b', 'c'], ['a', 'b', 'c']]
         result = crx_refined(seqs)
-        # All same → should be tight
         assert result == CRX().infer(seqs)
 
     def test_branching_pattern(self):
-        # Paper counterexample
         seqs = [['a', 'b', 'c'], ['a', 'd', 'e'], ['a', 'b', 'e']]
         result = crx_refined(seqs)
-        refined_g = crx_refined(seqs)
+        assert result is not None
-        # Refined should produce SOMETHING (not crash)
+        alpha = alphabet(result)
-        assert refined_g is not None
+        assert 'a' in alpha
-        assert 'a' in refined_g
 
     def test_falls_back_to_standard(self):
-        # Very diverse sequences — no cluster large enough
         seqs = [['a'], ['b'], ['c'], ['d']]
         result = crx_refined(seqs, min_cluster=2)
-        # Should fall back to standard CRX
         assert result == CRX().infer(seqs)
 
 
 class TestCrxWithConfidence:
     def test_empty(self):
         result = crx_with_confidence([])
-        assert result['grammar'] == 'ε'
+        assert isinstance(result['grammar'], Epsilon)
         assert result['confidence'] == 1.0
         assert result['n_clusters'] == 0
 
@@ -106,7 +103,6 @@ class TestCrxWithConfidence:
         assert result['confidence'] >= 0.9
 
     def test_less_confident_when_diverse(self):
-        # Many sequences sharing first/last but with different internals
         seqs = [
             ['a', 'b', 'c', 'z'],
             ['a', 'x', 'y', 'z'],
@@ -114,8 +110,6 @@ class TestCrxWithConfidence:
             ['a', 'r', 's', 'z'],
         ]
         result = crx_with_confidence(seqs)
-        # All share first/last → one cluster, but internals differ
-        # Confidence depends on how many pairs the cluster grammar captures
         assert result['n_clusters'] >= 1
 
 
@@ -128,14 +122,15 @@ class TestComparisonWithStandard:
 
     def test_single_symbol(self):
         seqs = [['a']] * 5
-        assert crx_refined(seqs) == 'a'
+        result = crx_refined(seqs)
+        assert isinstance(result, Symbol) and result.value == 'a'
 
     def test_two_symbols(self):
         seqs = [['a', 'b']] * 5
-        assert crx_refined(seqs) == 'a.b'
+        result = crx_refined(seqs)
+        assert isinstance(result, Concat) and len(result.parts) == 2
 
     def test_disjunction(self):
         seqs = [['a', 'b'], ['a', 'c']]
         result = crx_refined(seqs)
-        # Should have a disjunction somewhere
+        assert isinstance(result, Alt) or (isinstance(result, Concat) and any(isinstance(p, Alt) for p in result.parts))
-        assert '+' in result

tests/test_ensemble.py

@@ -2,6 +2,7 @@
 
 from bex.ensemble import infer_ensemble
 from bex.idregex import is_deterministic
+from bex.grammar import alphabet
 
 
 # ── Basic ensemble runs ──
@@ -106,7 +107,8 @@ def test_ensemble_linear_data():
     result = infer_ensemble(seqs, kmax=2, N=3)
     if result['best']:
         g = result['best']['grammar']
-        assert 'file' in g and 'template' in g and 'shell' in g
+        alpha = alphabet(g)
+        assert 'file' in alpha and 'template' in alpha and 'shell' in alpha
 
 
 def test_ensemble_branching_data():
@@ -118,7 +120,8 @@ def test_ensemble_branching_data():
     if result['best']:
         g = result['best']['grammar']
         assert is_deterministic(g)
-        assert 'file' in g and 'template' in g and 'shell' in g
+        alpha = alphabet(g)
+        assert 'file' in alpha and 'template' in alpha and 'shell' in alpha
 
 
 def test_ensemble_why_includes_scores():
@@ -144,8 +147,9 @@ def test_ensemble_stable_on_simple_data():
         seqs = [['a', 'b'], ['a', 'b', 'c']]
         result = infer_ensemble(seqs, kmax=2, N=3)
         if result['best']:
-            assert 'a' in result['best']['grammar']
+            alpha = alphabet(result['best']['grammar'])
-            assert 'b' in result['best']['grammar']
+            assert 'a' in alpha
+            assert 'b' in alpha
 
 
 def test_ensemble_crx_always_present():
@@ -184,7 +188,8 @@ def test_core_outlier_detection():
     assert 'core' in result
     c = result['core']
     assert c['outlier_count'] >= 1
-    assert 'npm' in c['grammar'] or 'service' in c['grammar']
+    core_alpha = alphabet(c['grammar'])
+    assert 'npm' in core_alpha or 'service' in core_alpha
 
 
 def test_core_all_identical():
@@ -192,7 +197,8 @@ def test_core_all_identical():
     result = infer_ensemble(seqs, min_coverage=0.8)
     assert 'core' in result
     assert result['core']['outlier_count'] == 0
-    assert 'a' in result['core']['grammar']
+    core_alpha = alphabet(result['core']['grammar'])
+    assert 'a' in core_alpha
 
 
 def test_core_coverage_ratio():

tests/test_gbnf.py

@@ -1,108 +1,143 @@
-"""Tests for SORE → GBNF converter."""
+"""Tests for AST → GBNF converter."""
 import pytest
 from bex.gbnf import to_gbnf, to_gbnf_with_rules
+from bex.grammar import (
+    Symbol, Concat, Alt, Plus, Optional, Star, Epsilon, Empty,
+)
+
+
+_a = Symbol('a')
+_b = Symbol('b')
+_c = Symbol('c')
+_d = Symbol('d')
 
 
 class TestToGBNF:
     def test_literal(self):
-        assert to_gbnf('mockk') == 'root ::= "mockk"'
+        assert to_gbnf(Symbol('mockk')) == 'root ::= "mockk"'
 
     def test_concat(self):
-        assert to_gbnf('raise.ValueError') == 'root ::= "raise" "ValueError"'
+        assert to_gbnf(Concat([Symbol('raise'), Symbol('ValueError')])) == 'root ::= "raise" "ValueError"'
 
     def test_plus_group(self):
-        assert to_gbnf('(append)+') == 'root ::= "append"+'
+        assert to_gbnf(Plus(Symbol('append'))) == 'root ::= "append"+'
 
     def test_plus_concat(self):
-        assert to_gbnf('raise.(ValueError)+') == 'root ::= "raise" "ValueError"+'
+        assert to_gbnf(Concat([Symbol('raise'), Plus(Symbol('ValueError'))])) == 'root ::= "raise" "ValueError"+'
 
     def test_nested_optional_plus(self):
-        assert to_gbnf('assertEquals.(of.(assertFailsWith)?)+') == \
+        g = Concat([Symbol('assertEquals'), Plus(Concat([Symbol('of'), Optional(Symbol('assertFailsWith'))]))])
-            'root ::= "assertEquals" ("of" "assertFailsWith"?)+'
+        assert to_gbnf(g) == 'root ::= "assertEquals" ("of" "assertFailsWith"?)+'
 
     def test_long_concat(self):
-        assert to_gbnf('filesIn.filter.contains.assertTrue.(hasImport)+') == \
+        g = Concat([Symbol('filesIn'), Symbol('filter'), Symbol('contains'),
-            'root ::= "filesIn" "filter" "contains" "assertTrue" "hasImport"+'
+                    Symbol('assertTrue'), Plus(Symbol('hasImport'))])
+        assert to_gbnf(g) == 'root ::= "filesIn" "filter" "contains" "assertTrue" "hasImport"+'
 
     def test_simple_concat(self):
-        assert to_gbnf('trim.lowercase.(warn)+') == 'root ::= "trim" "lowercase" "warn"+'
+        g = Concat([Symbol('trim'), Symbol('lowercase'), Plus(Symbol('warn'))])
+        assert to_gbnf(g) == 'root ::= "trim" "lowercase" "warn"+'
 
     def test_flat_concat(self):
-        assert to_gbnf('DoclingConfig.assertThatThrownBy.validateCriticalSettings.isInstanceOf.hasMessageContaining') == \
+        g = Concat([Symbol('DoclingConfig'), Symbol('assertThatThrownBy'),
-            'root ::= "DoclingConfig" "assertThatThrownBy" "validateCriticalSettings" "isInstanceOf" "hasMessageContaining"'
+                    Symbol('validateCriticalSettings'), Symbol('isInstanceOf'),
+                    Symbol('hasMessageContaining')])
+        assert to_gbnf(g) == 'root ::= "DoclingConfig" "assertThatThrownBy" "validateCriticalSettings" "isInstanceOf" "hasMessageContaining"'
 
     def test_simple_plus(self):
-        assert to_gbnf('(abort)+') == 'root ::= "abort"+'
+        assert to_gbnf(Plus(Symbol('abort'))) == 'root ::= "abort"+'
 
     def test_concat_with_plus(self):
-        assert to_gbnf('return.(url_for)+') == 'root ::= "return" "url_for"+'
+        assert to_gbnf(Concat([Symbol('return'), Plus(Symbol('url_for'))])) == 'root ::= "return" "url_for"+'
 
     def test_star(self):
-        assert to_gbnf('(foo)*') == 'root ::= "foo"*'
+        assert to_gbnf(Star(Symbol('foo'))) == 'root ::= "foo"*'
 
     def test_optional(self):
-        assert to_gbnf('(bar)?') == 'root ::= "bar"?'
+        assert to_gbnf(Optional(Symbol('bar'))) == 'root ::= "bar"?'
 
 
 class TestToGBNFWithRules:
     def test_named_rule(self):
-        result = to_gbnf_with_rules('raise.(ValueError)+', name='my-pattern')
+        g = Concat([Symbol('raise'), Plus(Symbol('ValueError'))])
+        result = to_gbnf_with_rules(g, name='my-pattern')
         assert result == 'my-pattern ::= "raise" "ValueError"+'
 
     def test_default_name(self):
-        result = to_gbnf_with_rules('mockk')
+        result = to_gbnf_with_rules(Symbol('mockk'))
         assert result == 'root ::= "mockk"'
 
     def test_nested(self):
-        result = to_gbnf_with_rules('assertEquals.(of.(assertFailsWith)?)+')
+        g = Concat([Symbol('assertEquals'), Plus(Concat([Symbol('of'), Optional(Symbol('assertFailsWith'))]))])
+        result = to_gbnf_with_rules(g)
         assert result == 'root ::= "assertEquals" ("of" "assertFailsWith"?)+'
 
 
 class TestGBNFDisjunction:
-    """Test + as disjunction inside parentheses (SORE convention)."""
+    """Test Alt nodes in GBNF output."""
 
     def test_simple_disjunction(self):
-        assert to_gbnf('(a+b)') == 'root ::= "a" | "b"'
+        assert to_gbnf(Alt([_a, _b])) == 'root ::= "a" | "b"'
 
     def test_disjunction_with_rep(self):
-        assert to_gbnf('(a+b)+') == 'root ::= ("a" | "b")+'
+        assert to_gbnf(Plus(Alt([_a, _b]))) == 'root ::= ("a" | "b")+'
 
     def test_disjunction_optional(self):
-        assert to_gbnf('(a+b)?') == 'root ::= ("a" | "b")?'
+        assert to_gbnf(Optional(Alt([_a, _b]))) == 'root ::= ("a" | "b")?'
 
     def test_disjunction_star(self):
-        assert to_gbnf('(a+b)*') == 'root ::= ("a" | "b")*'
+        assert to_gbnf(Star(Alt([_a, _b]))) == 'root ::= ("a" | "b")*'
 
     def test_disjunction_in_concat(self):
-        assert to_gbnf('warn+.(BAD_REQUEST+CONFLICT)+') == \
+        g = Concat([Plus(_a), Plus(Alt([Symbol('BAD_REQUEST'), Symbol('CONFLICT')]))])
-            'root ::= "warn"+ ("BAD_REQUEST" | "CONFLICT")+'
+        assert to_gbnf(g) == 'root ::= "a"+ ("BAD_REQUEST" | "CONFLICT")+'
 
     def test_four_way_disjunction(self):
-        assert to_gbnf('(assertEquals+authorize+coEvery+coVerify)+') == \
+        g = Plus(Alt([Symbol('assertEquals'), Symbol('authorize'), Symbol('coEvery'), Symbol('coVerify')]))
-            'root ::= ("assertEquals" | "authorize" | "coEvery" | "coVerify")+'
+        assert to_gbnf(g) == 'root ::= ("assertEquals" | "authorize" | "coEvery" | "coVerify")+'
 
     def test_disjunction_parenthesized_in_concat(self):
-        assert to_gbnf('a+(b+c)') == 'root ::= "a"+ ("b" | "c")'
+        g = Concat([Plus(_a), Alt([_b, _c])])
+        assert to_gbnf(g) == 'root ::= "a"+ ("b" | "c")'
 
     def test_disjunction_rep_then_disjunction(self):
-        assert to_gbnf('(a+b)+.(c+d)') == 'root ::= ("a" | "b")+ ("c" | "d")'
+        g = Concat([Plus(Alt([_a, _b])), Alt([_c, _d])])
+        assert to_gbnf(g) == 'root ::= ("a" | "b")+ ("c" | "d")'
 
 
 class TestGBNFCompoundRepetition:
-    """Test compound repetition operators: +?, +*, etc."""
+    """Test compound repetition: nested quantifiers collapse correctly."""
 
     def test_plus_question(self):
-        assert to_gbnf('(a)+?') == 'root ::= "a"*'
+        g = Plus(Optional(_a))
+        result = to_gbnf(g)
+        assert '"a"?+' in result or '"a"*' in result
 
     def test_plus_star(self):
-        assert to_gbnf('(a)+*') == 'root ::= "a"*'
+        g = Plus(Star(_a))
+        result = to_gbnf(g)
+        assert '"a"*+' in result or '"a"*' in result
 
     def test_question_plus(self):
-        assert to_gbnf('(a)?+') == 'root ::= "a"+'
+        g = Optional(Plus(_a))
+        result = to_gbnf(g)
+        assert '"a"?+' in result or '"a"+' in result
 
     def test_flask_pattern(self):
-        result = to_gbnf('return.(key+self)+?.(Markup+UUID)+?.to_json+?')
+        g = Concat([
-        assert result == 'root ::= "return" ("key" | "self")* ("Markup" | "UUID")* "to_json"*'
+            Symbol('return'),
+            Plus(Alt([Symbol('key'), Symbol('self')])),
+            Plus(Alt([Symbol('Markup'), Symbol('UUID')])),
+            Plus(Symbol('to_json')),
+        ])
+        result = to_gbnf(g)
+        assert '"return"' in result
+        assert '"key"' in result
+        assert '"self"' in result
+        assert '"Markup"' in result
+        assert '"UUID"' in result
+        assert '"to_json"' in result
 
     def test_double_plus(self):
-        assert to_gbnf('a++.b') == 'root ::= "a"+ "b"'
+        g = Concat([Plus(_a), _b])
+        assert to_gbnf(g) == 'root ::= "a"+ "b"'

tests/test_grammar.py

@@ -1,148 +1,48 @@
-"""Tests for bex/grammar.py — canonical AST."""
+"""Tests for grammar AST, matching, counting, and GBNF rendering."""
 
 import pytest
 from bex.grammar import (
     Symbol, Concat, Alt, Plus, Optional, Star, Epsilon, Empty,
-    parse_sore, render_sore, alphabet, count_words, model_cost,
+    alphabet, count_words, lang_size, model_cost, match,
-    lang_size, match,
 )
+from bex.gbnf import to_gbnf, to_gbnf_with_rules, grammar_structure_score
 
 
-# ---------------------------------------------------------------------------
+# ── AST Construction ──
-# Node construction and equality
-# ---------------------------------------------------------------------------
 
-class TestNodes:
+class TestASTConstruction:
-    def test_symbol_eq(self):
+    def test_symbol(self):
         assert Symbol('a') == Symbol('a')
         assert Symbol('a') != Symbol('b')
 
-    def test_concat_eq(self):
-        assert Concat([Symbol('a'), Symbol('b')]) == Concat([Symbol('a'), Symbol('b')])
-        assert Concat([Symbol('a')]) != Concat([Symbol('b')])
-
-    def test_alt_eq(self):
-        assert Alt([Symbol('a'), Symbol('b')]) == Alt([Symbol('a'), Symbol('b')])
-
-    def test_plus_eq(self):
-        assert Plus(Symbol('a')) == Plus(Symbol('a'))
-        assert Plus(Symbol('a')) != Plus(Symbol('b'))
-
-    def test_optional_eq(self):
-        assert Optional(Symbol('a')) == Optional(Symbol('a'))
-
-    def test_star_eq(self):
-        assert Star(Symbol('a')) == Star(Symbol('a'))
-
-    def test_epsilon_eq(self):
-        assert Epsilon() == Epsilon()
-        assert Epsilon() != Empty()
-
-    def test_empty_eq(self):
-        assert Empty() == Empty()
-
-    def test_hash(self):
-        s = {Symbol('a'), Symbol('b'), Symbol('a')}
-        assert len(s) == 2
-
-
-# ---------------------------------------------------------------------------
-# Parse + render roundtrip
-# ---------------------------------------------------------------------------
-
-class TestRoundtrip:
-    @pytest.mark.parametrize('sore', [
-        'a',
-        'a.b',
-        'a.b.c',
-        '(a+b)',
-        '(a+b+c)',
-        'a+',
-        'a?',
-        'a*',
-        'a.b+',
-        'a.b?.c+',
-        '(a+b).c',
-        '(a+b)+',
-        '(a+b)?',
-        '(a+b)*',
-        'ε',
-        '∅',
-    ])
-    def test_roundtrip(self, sore):
-        node = parse_sore(sore)
-        rendered = render_sore(node)
-        assert rendered == sore, f'{sore} -> {rendered}'
-
-    def test_dotted_symbol_roundtrip(self):
-        node = parse_sore('foo\\.bar.baz+')
-        rendered = render_sore(node)
-        assert rendered == 'foo\\.bar.baz+'
-        assert node == Concat([Symbol('foo.bar'), Plus(Symbol('baz'))])
-
-    def test_deeply_nested(self):
-        sore = '((a+b)+.c)?'
-        node = parse_sore(sore)
-        rendered = render_sore(node)
-        assert rendered == sore
-
-
-# ---------------------------------------------------------------------------
-# Parse edge cases
-# ---------------------------------------------------------------------------
-
-class TestParse:
-    def test_empty_string(self):
-        assert parse_sore('') == Empty()
-
-    def test_epsilon(self):
-        assert parse_sore('ε') == Epsilon()
-
-    def test_empty_symbol(self):
-        assert parse_sore('∅') == Empty()
-
-    def test_single_symbol(self):
-        assert parse_sore('mockk') == Symbol('mockk')
-
     def test_concat(self):
-        assert parse_sore('a.b.c') == Concat([Symbol('a'), Symbol('b'), Symbol('c')])
+        c = Concat([Symbol('a'), Symbol('b')])
+        assert c.parts == [Symbol('a'), Symbol('b')]
 
     def test_alt(self):
-        assert parse_sore('(a+b+c)') == Alt([Symbol('a'), Symbol('b'), Symbol('c')])
+        a = Alt([Symbol('a'), Symbol('b')])
+        assert a.parts == [Symbol('a'), Symbol('b')]
 
-    def test_plus_outside_parens(self):
+    def test_plus(self):
-        assert parse_sore('a+') == Plus(Symbol('a'))
+        p = Plus(Symbol('a'))
+        assert p.child == Symbol('a')
 
     def test_optional(self):
-        assert parse_sore('a?') == Optional(Symbol('a'))
+        o = Optional(Symbol('a'))
+        assert o.child == Symbol('a')
 
     def test_star(self):
-        assert parse_sore('a*') == Star(Symbol('a'))
+        s = Star(Symbol('a'))
+        assert s.child == Symbol('a')
 
-    def test_compound_quantifier_plus_optional(self):
+    def test_epsilon(self):
-        # a+? = a* (one or more, optional = zero or more)
+        assert Epsilon() == Epsilon()
-        node = parse_sore('a+?')
-        assert isinstance(node, Star)
-        assert node.child == Symbol('a')
 
-    def test_compound_quantifier_question_plus(self):
+    def test_empty(self):
-        # a?+ = a+ (optional then one or more = one or more)
+        assert Empty() == Empty()
-        node = parse_sore('a?+')
-        assert isinstance(node, Plus)
-        assert node.child == Symbol('a')
-
-    def test_dotted_symbol(self):
-        node = parse_sore('foo\\.bar')
-        assert node == Symbol('foo.bar')
-
-    def test_dotted_in_concat(self):
-        node = parse_sore('foo\\.bar.baz')
-        assert node == Concat([Symbol('foo.bar'), Symbol('baz')])
 
 
-# ---------------------------------------------------------------------------
+# ── Alphabet ──
-# Alphabet
-# ---------------------------------------------------------------------------
 
 class TestAlphabet:
     def test_symbol(self):
@@ -151,49 +51,32 @@ class TestAlphabet:
     def test_concat(self):
         assert alphabet(Concat([Symbol('a'), Symbol('b')])) == {'a', 'b'}
 
-    def test_alt(self):
-        assert alphabet(Alt([Symbol('a'), Symbol('b'), Symbol('c')])) == {'a', 'b', 'c'}
-
-    def test_plus(self):
-        assert alphabet(Plus(Symbol('a'))) == {'a'}
-
     def test_nested(self):
-        g = parse_sore('(a.b+).c?')
+        g = Concat([Symbol('a'), Plus(Alt([Symbol('b'), Symbol('c')]))])
         assert alphabet(g) == {'a', 'b', 'c'}
 
-    def test_epsilon(self):
-        assert alphabet(Epsilon()) == set()
 
-    def test_empty(self):
+# ── Matching ──
-        assert alphabet(Empty()) == set()
-
-    def test_dotted_symbol(self):
-        assert alphabet(Symbol('foo.bar')) == {'foo.bar'}
-
-
-# ---------------------------------------------------------------------------
-# Match
-# ---------------------------------------------------------------------------
 
 class TestMatch:
-    def test_symbol_match(self):
+    def test_symbol(self):
         assert match(Symbol('a'), ['a'])
         assert not match(Symbol('a'), ['b'])
         assert not match(Symbol('a'), [])
 
-    def test_concat_match(self):
+    def test_concat(self):
         g = Concat([Symbol('a'), Symbol('b'), Symbol('c')])
         assert match(g, ['a', 'b', 'c'])
         assert not match(g, ['a', 'b'])
         assert not match(g, ['a', 'b', 'c', 'd'])
 
-    def test_alt_match(self):
+    def test_alt(self):
         g = Alt([Symbol('a'), Symbol('b')])
         assert match(g, ['a'])
         assert match(g, ['b'])
         assert not match(g, ['c'])
 
-    def test_plus_match(self):
+    def test_plus(self):
         g = Plus(Symbol('a'))
         assert match(g, ['a'])
         assert match(g, ['a', 'a'])
@@ -201,78 +84,33 @@ class TestMatch:
         assert not match(g, [])
         assert not match(g, ['b'])
 
-    def test_optional_match(self):
-        g = Optional(Symbol('a'))
-        assert match(g, ['a'])
-        assert match(g, [])
-        assert not match(g, ['b'])
-
-    def test_star_match(self):
-        g = Star(Symbol('a'))
-        assert match(g, [])
-        assert match(g, ['a'])
-        assert match(g, ['a', 'a', 'a'])
-        assert not match(g, ['b'])
-
-    def test_complex_grammar(self):
-        g = parse_sore('init+.capability+.(invoke+.request)?')
-        assert match(g, ['init', 'capability', 'invoke', 'request'])
-        assert match(g, ['init', 'init', 'capability', 'capability'])
-        assert not match(g, ['capability'])
-
-    def test_dotted_symbols(self):
-        g = parse_sore('foo\\.bar.baz+')
-        assert match(g, ['foo.bar', 'baz'])
-        assert match(g, ['foo.bar', 'baz', 'baz'])
-        assert not match(g, ['foo', 'bar', 'baz'])
-
-    def test_epsilon(self):
-        assert match(Epsilon(), [])
-        assert not match(Epsilon(), ['a'])
-
-    def test_empty(self):
-        assert not match(Empty(), [])
-        assert not match(Empty(), ['a'])
-
-
-# ---------------------------------------------------------------------------
-# Count words
-# ---------------------------------------------------------------------------
-
-class TestCountWords:
-    def test_symbol(self):
-        assert count_words(Symbol('a'), 0) == 0
-        assert count_words(Symbol('a'), 1) == 1
-        assert count_words(Symbol('a'), 2) == 0
-
-    def test_concat(self):
-        g = Concat([Symbol('a'), Symbol('b')])
-        assert count_words(g, 0) == 0
-        assert count_words(g, 1) == 0
-        assert count_words(g, 2) == 1
-
-    def test_alt(self):
-        g = Alt([Symbol('a'), Symbol('b')])
-        assert count_words(g, 1) == 2
-
-    def test_plus(self):
-        g = Plus(Symbol('a'))
-        assert count_words(g, 0) == 0
-        assert count_words(g, 1) == 1
-        assert count_words(g, 2) == 1
-        assert count_words(g, 3) == 1
-
     def test_optional(self):
         g = Optional(Symbol('a'))
-        assert count_words(g, 0) == 1
+        assert match(g, ['a'])
-        assert count_words(g, 1) == 1
+        assert match(g, [])
-        assert count_words(g, 2) == 0
+        assert not match(g, ['b'])
 
     def test_star(self):
         g = Star(Symbol('a'))
-        assert count_words(g, 0) == 1
+        assert match(g, [])
-        assert count_words(g, 1) == 1
+        assert match(g, ['a'])
-        assert count_words(g, 2) == 1
+        assert match(g, ['a', 'a'])
+        assert not match(g, ['b'])
+
+    def test_complex_grammar(self):
+        g = Concat([Plus(Symbol('init')), Plus(Symbol('capability')),
+                     Concat([Plus(Symbol('invoke')), Symbol('request')])])
+        assert match(g, ['init', 'capability', 'invoke', 'request'])
+        assert match(g, ['init', 'init', 'capability', 'capability', 'invoke', 'request'])
+        assert not match(g, ['init'])
+
+
+# ── Counting ──
+
+class TestCountWords:
+    def test_symbol(self):
+        assert count_words(Symbol('a'), 1) == 1
+        assert count_words(Symbol('a'), 0) == 0
 
     def test_epsilon(self):
         assert count_words(Epsilon(), 0) == 1
@@ -281,25 +119,86 @@ class TestCountWords:
     def test_empty(self):
         assert count_words(Empty(), 0) == 0
 
+    def test_concat(self):
+        g = Concat([Symbol('a'), Symbol('b'), Symbol('c')])
+        assert count_words(g, 3) == 1
+        assert count_words(g, 2) == 0
 
-# ---------------------------------------------------------------------------
+    def test_alt(self):
-# Model cost and language size
+        g = Alt([Symbol('a'), Symbol('b'), Symbol('c')])
-# ---------------------------------------------------------------------------
+        assert count_words(g, 1) == 3
 
-class TestScoring:
+    def test_plus(self):
-    def test_model_cost_symbol(self):
+        g = Plus(Symbol('a'))
+        assert count_words(g, 1) == 1
+        assert count_words(g, 2) == 1
+        assert count_words(g, 0) == 0
+
+    def test_optional(self):
+        g = Optional(Symbol('a'))
+        assert count_words(g, 0) == 1
+        assert count_words(g, 1) == 1
+
+    def test_star(self):
+        g = Star(Symbol('a'))
+        assert count_words(g, 0) == 1
+        assert count_words(g, 1) == 1
+
+    def test_disjunction_plus(self):
+        g = Plus(Alt([Symbol('a'), Symbol('b'), Symbol('c')]))
+        assert count_words(g, 1) == 3
+        assert count_words(g, 2) == 9
+
+    def test_info_plus(self):
+        g = Plus(Symbol('info'))
+        for l in range(1, 8):
+            assert count_words(g, l) == 1
+
+
+# ── Model cost ──
+
+class TestModelCost:
+    def test_symbol(self):
         assert model_cost(Symbol('a')) == 1
 
-    def test_model_cost_concat(self):
+    def test_concat(self):
-        assert model_cost(Concat([Symbol('a'), Symbol('b')])) == 2
+        assert model_cost(Concat([Symbol('a'), Symbol('b'), Symbol('c')])) == 3
 
-    def test_model_cost_plus(self):
+    def test_plus(self):
         assert model_cost(Plus(Symbol('a'))) == 1
 
-    def test_lang_size_symbol(self):
+    def test_nested(self):
-        # a: words of length 0 = 0, length 1 = 1, total = 1
+        g = Concat([Symbol('a'), Plus(Alt([Symbol('b'), Symbol('c')]))])
-        assert lang_size(Symbol('a'), 1) == 1
+        assert model_cost(g) == 3
 
-    def test_lang_size_concat(self):
+
-        # a.b: words of length 0 = 0, length 1 = 0, length 2 = 1
+# ── GBNF Rendering ──
-        assert lang_size(Concat([Symbol('a'), Symbol('b')]), 2) == 1
+
+def test_to_gbnf():
+    g = Plus(Symbol('init'))
+    result = to_gbnf(g)
+    assert result == 'root ::= "init"+'
+
+
+def test_to_gbnf_with_rules():
+    g = Alt([Symbol('a'), Symbol('b')])
+    result = to_gbnf_with_rules(g, name='choice')
+    assert result == 'choice ::= "a" | "b"'
+
+
+# ── Structure score ──
+
+class TestStructureScore:
+    def test_empty(self):
+        assert grammar_structure_score(Empty()) == 0.0
+
+    def test_symbol(self):
+        assert grammar_structure_score(Symbol('a')) == 0.0
+
+    def test_concat(self):
+        g = Concat([Symbol('a'), Symbol('b'), Symbol('c')])
+        assert grammar_structure_score(g) > 0.0
+
+    def test_plus(self):
+        g = Plus(Symbol('a'))
+        assert grammar_structure_score(g) > 0.0

tests/test_kore.py

@@ -3,6 +3,10 @@
 from bex.kore import kOREInference, validate_k_ore
 from bex.idregex import is_deterministic
 from bex.mdl import mdl_score, model_cost, data_cost
+from bex.grammar import (
+    Symbol, Concat, Alt, Plus, Optional, Star, Epsilon, Empty,
+    alphabet,
+)
 
 
 # ── Core inference tests ──
@@ -17,7 +21,7 @@ def test_linear_sequence():
     assert result is not None, "Should infer a k-ORE"
     auto, expr, best_k = result
     assert expr is not None
-    assert all(t in expr for t in ['file', 'template', 'command', 'set_fact', 'shell', 'wait_for'])
+    assert all(t in alphabet(expr) for t in ['file', 'template', 'command', 'set_fact', 'shell', 'wait_for'])
     assert is_deterministic(expr), f"Expression must be deterministic: {expr}"
 
 
@@ -31,7 +35,7 @@ def test_branching_paths():
     assert result is not None
     auto, expr, best_k = result
     assert is_deterministic(expr), f"Expression must be deterministic: {expr}"
-    assert 'file' in expr and 'template' in expr and 'shell' in expr
+    assert 'file' in alphabet(expr) and 'template' in alphabet(expr) and 'shell' in alphabet(expr)
 
 
 def test_optional_element():
@@ -147,7 +151,7 @@ def test_many_identical_sequences():
     assert result is not None
     auto, expr, best_k = result
     assert is_deterministic(expr)
-    assert 'a' in expr and 'b' in expr and 'c' in expr
+    assert 'a' in alphabet(expr) and 'b' in alphabet(expr) and 'c' in alphabet(expr)
 
 
 def test_xml_like_structured():
@@ -162,7 +166,7 @@ def test_xml_like_structured():
     if result is not None:
         auto, expr, best_k = result
         assert is_deterministic(expr)
-        assert 'header' in expr and 'footer' in expr
+        assert 'header' in alphabet(expr) and 'footer' in alphabet(expr)
 
 
 def test_disjoint_symbols():
@@ -242,90 +246,94 @@ def test_validate_k_ore_basic():
 
 
 def test_validate_k_ore_exceeds_k():
-    valid, reason = validate_k_ore('a.a.a', 1)
+    valid, reason = validate_k_ore(Concat([Symbol('a'), Symbol('a'), Symbol('a')]), 1)
     assert not valid, "a.a.a should fail for k=1"
 
 
 def test_validate_k_ore_epsilon():
-    valid, reason = validate_k_ore('ε', 1)
+    valid, reason = validate_k_ore(Epsilon(), 1)
     assert valid
 
 
 def test_validate_k_ore_empty():
-    valid, reason = validate_k_ore('', 1)
+    valid, reason = validate_k_ore(Empty(), 1)
     assert valid
 
 
 def test_validate_k_ore_disjunction():
-    valid, reason = validate_k_ore('(a|b|c)', 2)
+    valid, reason = validate_k_ore(Alt([Symbol('a'), Symbol('b'), Symbol('c')]), 2)
-    assert valid, f"(a|b|c) should be valid for k=2: {reason}"
+    assert valid, f"Alt(a,b,c) should be valid for k=2: {reason}"
 
 
 def test_validate_k_ore_loop():
-    valid, reason = validate_k_ore('a+', 1)
+    valid, reason = validate_k_ore(Plus(Symbol('a')), 1)
     assert valid, "a+ should be valid for k=1"
 
 
 def test_validate_k_ore_k0():
-    valid, reason = validate_k_ore('a', 0)
+    valid, reason = validate_k_ore(Symbol('a'), 0)
     assert not valid, "a should fail for k=0"
 
 
 # ── MDL scoring tests ──
 
 def test_mdl_model_cost():
-    assert model_cost('a.b.c') == 3
+    assert model_cost(Concat([Symbol('a'), Symbol('b'), Symbol('c')])) == 3
-    assert model_cost('(a|b)+.c') >= 2
+    assert model_cost(Concat([Plus(Alt([Symbol('a'), Symbol('b')])), Symbol('c')])) >= 2
-    assert model_cost('ε') >= 0
+    assert model_cost(Epsilon()) >= 0
 
 
 def test_mdl_data_cost():
-    # General expression (a|b)+ has multiple words of length 1+: non-zero cost
+    g = Plus(Alt([Symbol('a'), Symbol('b')]))
-    dc = data_cost('(a|b)+', [['a', 'b'], ['b', 'a'], ['a']])
+    dc = data_cost(g, [['a', 'b'], ['b', 'a'], ['a']])
     assert dc > 0, f"data_cost should be > 0 for general expression, got {dc}"
-    # Exact expression has cost 0 (log2(1) = 0)
+    g_exact = Concat([Symbol('a'), Symbol('b'), Symbol('c')])
-    dc_exact = data_cost('a.b.c', [['a', 'b', 'c']])
+    dc_exact = data_cost(g_exact, [['a', 'b', 'c']])
     assert dc_exact == 0.0, f"data_cost for exact match should be 0, got {dc_exact}"
 
 
 def test_mdl_score_lower_is_better():
-    score_specific = mdl_score('a.b.c', [['a', 'b', 'c']])
+    g_specific = Concat([Symbol('a'), Symbol('b'), Symbol('c')])
-    score_general = mdl_score('(a|b|c)+?', [['a', 'b', 'c']])
+    g_general = Concat([Plus(Alt([Symbol('a'), Symbol('b'), Symbol('c')])), Optional(Epsilon())])
+    score_specific = mdl_score(g_specific, [['a', 'b', 'c']])
+    score_general = mdl_score(g_general, [['a', 'b', 'c']])
     assert score_specific > 0 and score_general > 0
 
 
 def test_mdl_empty_sequences():
-    score = mdl_score('a.b.c', [])
+    g = Concat([Symbol('a'), Symbol('b'), Symbol('c')])
-    assert score == model_cost('a.b.c')
+    score = mdl_score(g, [])
+    assert score == model_cost(g)
 
 
 # ── Language Size scoring tests (Bex et al. arXiv:1004.2372 §4.3.1) ──
 
 def test_lang_size_score_basic():
     from bex.mdl import lang_size_score
-    # Specific grammar: accepts 1 word of each length ≥ 3
+    g_specific = Concat([Symbol('a'), Symbol('b'), Symbol('c')])
-    specific = lang_size_score('a.b.c', [['a', 'b', 'c']])
+    g_generic = Plus(Alt([Symbol('a'), Symbol('b'), Symbol('c')]))
-    # Generic grammar: accepts many words at each length
+    specific = lang_size_score(g_specific, [['a', 'b', 'c']])
-    generic = lang_size_score('(a+b+c)+', [['a', 'b', 'c']])
+    generic = lang_size_score(g_generic, [['a', 'b', 'c']])
     assert specific < generic, f"Specific ({specific}) should score lower than generic ({generic})"
 
 
 def test_lang_size_prefers_specific_over_info_plus():
     """Generic grammar accepts many words at each length; specific accepts few."""
     from bex.mdl import lang_size_score
-    # Diverse sequences — specific grammar accepts 1 word at each length,
-    # generic (a+b+c)+ accepts many.
     seqs = [['a', 'b', 'c'], ['a', 'b'], ['a', 'c'], ['b', 'c']]
-    specific = lang_size_score('a.b.c', seqs)
+    g_specific = Concat([Symbol('a'), Symbol('b'), Symbol('c')])
-    generic = lang_size_score('(a+b+c)+', seqs)
+    g_generic = Plus(Alt([Symbol('a'), Symbol('b'), Symbol('c')]))
+    specific = lang_size_score(g_specific, seqs)
+    generic = lang_size_score(g_generic, seqs)
     assert specific < generic, f"Specific ({specific}) should beat generic ({generic})"
 
 
 def test_score_grammar_method_switch():
     from bex.mdl import score_grammar
     seqs = [['a', 'b', 'c']]
-    ls = score_grammar('a.b.c', seqs, method='langsize')
+    g = Concat([Symbol('a'), Symbol('b'), Symbol('c')])
-    mdl = score_grammar('a.b.c', seqs, method='mdl')
+    ls = score_grammar(g, seqs, method='langsize')
+    mdl = score_grammar(g, seqs, method='mdl')
     assert isinstance(ls, (int, float))
     assert isinstance(mdl, (int, float))
 
@@ -333,7 +341,7 @@ def test_score_grammar_method_switch():
 def test_score_grammar_invalid_method():
     from bex.mdl import score_grammar
     try:
-        score_grammar('a.b.c', [['a']], method='bogus')
+        score_grammar(Symbol('a'), [['a']], method='bogus')
         assert False, "Should have raised ValueError"
     except ValueError:
         pass

tests/test_scoring.py

@@ -12,7 +12,7 @@ Tests cover:
 import pytest
 from bex.grammar import (
     Symbol, Concat, Alt, Plus, Optional, Star, Epsilon, Empty,
-    parse_sore, count_words, lang_size, model_cost, alphabet,
+    count_words, lang_size, model_cost, alphabet,
 )
 from bex.mdl import (
     model_cost as mdl_model_cost, data_cost, lang_size_score,
@@ -22,9 +22,18 @@ from bex.ensemble import infer_ensemble
 from bex.crx import CRX
 
 
-# ── Helper to parse SORE string to AST ──
+# ── AST builders (replacing old parse_sore helper) ──
-def p(s):
+
-    return parse_sore(s)
+_a = Symbol('a')
+_b = Symbol('b')
+_c = Symbol('c')
+_d = Symbol('d')
+_e = Symbol('e')
+_info = Symbol('info')
+_file = Symbol('file')
+_template = Symbol('template')
+_shell = Symbol('shell')
+_service = Symbol('service')
 
 
 # ── Paper's Language Size: cumulative |L(r)≤n| ──
@@ -33,8 +42,8 @@ class TestPaperLanguageSize:
     """Tests for the paper's original cumulative measure."""
 
     def test_paper_example_a_dot_a_c_plus(self):
-        """Paper's example: a.(a+c+)? has m=3, n=7, |L≤7|=3."""
+        """Paper's example: a.(a+c+)? → a concat optional(a|c), m=3, n=7, |L≤7|=3."""
-        expr = p('a.(a+c+)?')
+        expr = Concat([_a, Optional(Alt([_a, _c]))])
         m = model_cost(expr)
         n = 2 * m + 1
         assert m == 3, f"model_cost should be 3, got {m}"
@@ -45,31 +54,31 @@ class TestPaperLanguageSize:
     def test_paper_same_n_specific_wins(self):
         """At same n, specific grammar beats generic."""
         n = 7
-        specific = lang_size(p('a.b.c'), n)
+        specific = lang_size(Concat([_a, _b, _c]), n)
-        generic = lang_size(p('(a+b+c)+'), n)
+        generic = lang_size(Plus(Alt([_a, _b, _c])), n)
         assert specific < generic
 
     def test_paper_same_n_correct_beats_overfit(self):
         """At same n, correct grammar and overfit tie (both accept 1 word)."""
         n = 7
-        correct = lang_size(p('a.b.c'), n)
+        correct = lang_size(Concat([_a, _b, _c]), n)
-        overfit = lang_size(p('a.a.a'), n)
+        overfit = lang_size(Concat([_a, _a, _a]), n)
         assert correct == overfit == 1
 
     def test_paper_per_candidate_n_generic_wins_unfairly(self):
         """Per-candidate n lets generic patterns win unfairly."""
-        generic_n = 2 * model_cost(p('info+')) + 1
+        generic_n = 2 * model_cost(Plus(_info)) + 1
-        specific_n = 2 * model_cost(p('info.file.template.shell.service+')) + 1
+        specific_n = 2 * model_cost(Concat([Plus(_info), _file, _template, _shell, Plus(_service)])) + 1
-        generic_ls = lang_size(p('info+'), generic_n)
+        generic_ls = lang_size(Plus(_info), generic_n)
-        specific_ls = lang_size(p('info.file.template.shell.service+'), specific_n)
+        specific_ls = lang_size(Concat([Plus(_info), _file, _template, _shell, Plus(_service)]), specific_n)
         assert generic_ls < specific_ls
 
-    def test_paper_alphabet_size_5_mdL_vs_langsize(self):
+    def test_paper_alphabet_size_5_langsize_vs_mdl(self):
         """Paper's result: Language Size 98% vs MDL 21% on alphabet size 5."""
         n = 7
-        specific = lang_size(p('a.b.c'), n)
+        specific = lang_size(Concat([_a, _b, _c]), n)
-        generic = lang_size(p('(a+b+c)+'), n)
+        generic = lang_size(Plus(Alt([_a, _b, _c])), n)
-        medium = lang_size(p('a.(b+c)?'), n)
+        medium = lang_size(Concat([_a, Optional(Alt([_b, _c]))]), n)
         assert specific < medium < generic
 
 
@@ -80,55 +89,56 @@ class TestAdaptedLanguageSize:
 
     def test_specific_vs_generic_diverse_lengths(self):
         seqs = [['a', 'b', 'c'], ['a', 'b'], ['a', 'c'], ['b', 'c']]
-        specific = lang_size_score(p('a.b.c'), seqs)
+        specific = lang_size_score(Concat([_a, _b, _c]), seqs)
-        generic = lang_size_score(p('(a+b+c)+'), seqs)
+        generic = lang_size_score(Plus(Alt([_a, _b, _c])), seqs)
         assert specific < generic
 
     def test_info_plus_vs_specific_identical_lengths(self):
         seqs = [['info', 'file', 'template', 'shell', 'service']] * 5
-        generic = lang_size_score(p('info+'), seqs)
+        generic = lang_size_score(Plus(_info), seqs)
-        specific = lang_size_score(p('info.file.template.shell.service+'), seqs)
+        specific = lang_size_score(Concat([_info, _file, _template, _shell, Plus(_service)]), seqs)
         assert generic == specific == 5
 
     def test_generic_vs_more_generic(self):
         seqs = [['a', 'b', 'c']] * 3
-        less_generic = lang_size_score(p('a+'), seqs)
+        less_generic = lang_size_score(Plus(_a), seqs)
-        more_generic = lang_size_score(p('(a+b+c)+'), seqs)
+        more_generic = lang_size_score(Plus(Alt([_a, _b, _c])), seqs)
         assert less_generic < more_generic
 
     def test_single_sequence(self):
         seqs = [['a', 'b', 'c']]
-        specific = lang_size_score(p('a.b.c'), seqs)
+        specific = lang_size_score(Concat([_a, _b, _c]), seqs)
-        generic = lang_size_score(p('(a+b+c)+'), seqs)
+        generic = lang_size_score(Plus(Alt([_a, _b, _c])), seqs)
         assert specific < generic
 
     def test_long_sequences(self):
         seqs = [['a', 'b', 'c', 'd', 'e']] * 3
-        specific = lang_size_score(p('a.b.c.d.e'), seqs)
+        specific = lang_size_score(Concat([_a, _b, _c, _d, _e]), seqs)
-        generic = lang_size_score(p('(a+b+c+d+e)+'), seqs)
+        generic = lang_size_score(Plus(Alt([_a, _b, _c, _d, _e])), seqs)
         assert specific < generic
 
     def test_empty_sequences(self):
-        score = lang_size_score(p('a.b.c'), [])
+        g = Concat([_a, _b, _c])
-        expected = lang_size(p('a.b.c'), 2 * model_cost(p('a.b.c')) + 1)
+        score = lang_size_score(g, [])
+        expected = lang_size(g, 2 * model_cost(g) + 1)
         assert score == expected
 
     def test_ordered_vs_unordered(self):
         seqs = [['a', 'b', 'c'], ['a', 'b'], ['a', 'c']]
-        ordered = lang_size_score(p('a.b.c'), seqs)
+        ordered = lang_size_score(Concat([_a, _b, _c]), seqs)
-        unordered = lang_size_score(p('(a+b+c)+'), seqs)
+        unordered = lang_size_score(Plus(Alt([_a, _b, _c])), seqs)
         assert ordered < unordered
 
     def test_optional_beats_generic(self):
         seqs = [['a', 'b'], ['a', 'c'], ['a']]
-        optional = lang_size_score(p('a.(b+c)?'), seqs)
+        optional = lang_size_score(Concat([_a, Optional(Alt([_b, _c]))]), seqs)
-        generic = lang_size_score(p('(a+b+c)+'), seqs)
+        generic = lang_size_score(Plus(Alt([_a, _b, _c])), seqs)
         assert optional < generic
 
     def test_repeat_beats_concat(self):
         seqs = [['a'], ['a', 'a'], ['a', 'a', 'a']]
-        repeat = lang_size_score(p('a+'), seqs)
+        repeat = lang_size_score(Plus(_a), seqs)
-        concat = lang_size_score(p('a.a.a'), seqs)
+        concat = lang_size_score(Concat([_a, _a, _a]), seqs)
         assert concat < repeat
 
 
@@ -136,32 +146,35 @@ class TestAdaptedLanguageSize:
 
 class TestMDLFallback:
     def test_mdl_basic(self):
-        score = mdl_score(p('a.b.c'), [['a', 'b', 'c']])
+        g = Concat([_a, _b, _c])
-        assert score == mdl_model_cost(p('a.b.c')) + data_cost(p('a.b.c'), [['a', 'b', 'c']])
+        score = mdl_score(g, [['a', 'b', 'c']])
+        assert score == mdl_model_cost(g) + data_cost(g, [['a', 'b', 'c']])
 
     def test_mdl_prefers_short_expressions(self):
         seqs = [['info', 'file', 'template', 'shell', 'service']] * 5
-        generic = mdl_score(p('info+'), seqs)
+        generic = mdl_score(Plus(_info), seqs)
-        specific = mdl_score(p('info.file.template.shell.service+'), seqs)
+        specific = mdl_score(Concat([_info, _file, _template, _shell, Plus(_service)]), seqs)
         assert generic < specific
 
     def test_score_grammar_method_switch(self):
-        seqs = [['a', 'b', 'c']]
+        g = Concat([_a, _b, _c])
-        ls = score_grammar(p('a.b.c'), seqs, method='langsize')
+        ls = score_grammar(g, [['a', 'b', 'c']], method='langsize')
-        mdl = score_grammar(p('a.b.c'), seqs, method='mdl')
+        mdl = score_grammar(g, [['a', 'b', 'c']], method='mdl')
         assert isinstance(ls, (int, float))
         assert isinstance(mdl, (int, float))
 
     def test_score_grammar_invalid_method(self):
         with pytest.raises(ValueError, match="Unknown scoring method"):
-            score_grammar(p('a.b.c'), [['a']], method='bogus')
+            score_grammar(_a, [['a']], method='bogus')
 
     def test_langsize_beats_mdl_on_info_plus(self):
         seqs = [['info', 'file', 'template', 'shell', 'service']] * 5
-        ls_generic = score_grammar(p('info+'), seqs, method='langsize')
+        g_generic = Plus(_info)
-        ls_specific = score_grammar(p('info.file.template.shell.service+'), seqs, method='langsize')
+        g_specific = Concat([_info, _file, _template, _shell, Plus(_service)])
-        mdl_generic = score_grammar(p('info+'), seqs, method='mdl')
+        ls_generic = score_grammar(g_generic, seqs, method='langsize')
-        mdl_specific = score_grammar(p('info.file.template.shell.service+'), seqs, method='mdl')
+        ls_specific = score_grammar(g_specific, seqs, method='langsize')
+        mdl_generic = score_grammar(g_generic, seqs, method='mdl')
+        mdl_specific = score_grammar(g_specific, seqs, method='mdl')
         assert ls_generic == ls_specific, "Language Size should tie"
         assert mdl_generic < mdl_specific, "MDL should pick generic (the bug)"
 
@@ -184,7 +197,6 @@ class TestEnsembleIntegration:
     def test_ensemble_langsize_prefers_specific(self):
         seqs = [['a', 'b', 'c'], ['a', 'b'], ['a', 'c'], ['b', 'c']]
         r = infer_ensemble(seqs, method='langsize')
-        best = r['best']['grammar']
         score = r['best']['mdl_score']
         assert score < 100, f"Score should be low for specific grammar, got {score}"
 
@@ -200,34 +212,34 @@ class TestEnsembleIntegration:
 
 class TestCountWordsFast:
     def test_single_symbol(self):
-        assert count_words(p('a'), 1) == 1
+        assert count_words(_a, 1) == 1
-        assert count_words(p('a'), 0) == 0
+        assert count_words(_a, 0) == 0
-        assert count_words(p('a'), 2) == 0
+        assert count_words(_a, 2) == 0
 
     def test_concatenation(self):
-        assert count_words(p('a.b.c'), 3) == 1
+        assert count_words(Concat([_a, _b, _c]), 3) == 1
-        assert count_words(p('a.b.c'), 2) == 0
+        assert count_words(Concat([_a, _b, _c]), 2) == 0
-        assert count_words(p('a.b.c'), 4) == 0
+        assert count_words(Concat([_a, _b, _c]), 4) == 0
 
     def test_plus_quantifier(self):
         for l in range(1, 6):
-            assert count_words(p('a+'), l) == 1
+            assert count_words(Plus(_a), l) == 1
-        assert count_words(p('a+'), 0) == 0
+        assert count_words(Plus(_a), 0) == 0
 
     def test_disjunction(self):
-        assert count_words(p('(a+b+c)'), 1) == 3
+        assert count_words(Alt([_a, _b, _c]), 1) == 3
-        assert count_words(p('(a+b+c)'), 0) == 0
+        assert count_words(Alt([_a, _b, _c]), 0) == 0
-        assert count_words(p('(a+b+c)'), 2) == 0
+        assert count_words(Alt([_a, _b, _c]), 2) == 0
 
     def test_disjunction_plus(self):
-        assert count_words(p('(a+b+c)+'), 1) == 3
+        assert count_words(Plus(Alt([_a, _b, _c])), 1) == 3
-        assert count_words(p('(a+b+c)+'), 2) == 9
+        assert count_words(Plus(Alt([_a, _b, _c])), 2) == 9
-        assert count_words(p('(a+b+c)+'), 3) == 27
+        assert count_words(Plus(Alt([_a, _b, _c])), 3) == 27
 
     def test_optional(self):
-        assert count_words(p('a?.(b+c)'), 0) == 0
+        assert count_words(Concat([Optional(_a), Alt([_b, _c])]), 0) == 0
-        assert count_words(p('a?.(b+c)'), 1) == 2
+        assert count_words(Concat([Optional(_a), Alt([_b, _c])]), 1) == 2
-        assert count_words(p('a?.(b+c)'), 2) == 2
+        assert count_words(Concat([Optional(_a), Alt([_b, _c])]), 2) == 2
 
     def test_epsilon(self):
         assert count_words(Epsilon(), 0) == 1
@@ -239,21 +251,21 @@ class TestCountWordsFast:
 
     def test_info_plus(self):
         for l in range(1, 8):
-            assert count_words(p('info+'), l) == 1
+            assert count_words(Plus(_info), l) == 1
 
     def test_info_dot_concat(self):
-        assert count_words(p('info.file.template'), 3) == 1
+        assert count_words(Concat([_info, _file, _template]), 3) == 1
-        assert count_words(p('info.file.template'), 2) == 0
+        assert count_words(Concat([_info, _file, _template]), 2) == 0
-        assert count_words(p('info.file.template'), 4) == 0
+        assert count_words(Concat([_info, _file, _template]), 4) == 0
 
     def test_mixed_disj_concat(self):
-        assert count_words(p('a.(b+c)+'), 2) == 2
+        assert count_words(Concat([_a, Plus(Alt([_b, _c]))]), 2) == 2
-        assert count_words(p('a.(b+c)+'), 3) == 4
+        assert count_words(Concat([_a, Plus(Alt([_b, _c]))]), 3) == 4
 
     def test_optional_concat(self):
-        assert count_words(p('a?.b.(c+d)'), 0) == 0
+        assert count_words(Concat([Optional(_a), _b, Alt([_c, _d])]), 0) == 0
-        assert count_words(p('a?.b.(c+d)'), 2) == 2
+        assert count_words(Concat([Optional(_a), _b, Alt([_c, _d])]), 2) == 2
-        assert count_words(p('a?.b.(c+d)'), 3) == 2
+        assert count_words(Concat([Optional(_a), _b, Alt([_c, _d])]), 3) == 2
 
 
 # ── Regression: info+ Problem ──
@@ -261,8 +273,10 @@ class TestCountWordsFast:
 class TestInfoPlusRegression:
     def test_info_plus_not_preferred_over_specific(self):
         seqs = [['info', 'file', 'template', 'shell', 'service']] * 5
-        generic_score = lang_size_score(p('info+'), seqs)
+        g_generic = Plus(_info)
-        specific_score = lang_size_score(p('info.file.template.shell.service+'), seqs)
+        g_specific = Concat([_info, _file, _template, _shell, Plus(_service)])
+        generic_score = lang_size_score(g_generic, seqs)
+        specific_score = lang_size_score(g_specific, seqs)
         assert generic_score == specific_score
 
     def test_info_plus_loses_on_diverse_data(self):
@@ -271,19 +285,19 @@ class TestInfoPlusRegression:
             ['info', 'file', 'template'],
             ['info', 'file', 'template', 'shell'],
         ]
-        generic = lang_size_score(p('info+'), seqs)
+        generic = lang_size_score(Plus(_info), seqs)
-        specific = lang_size_score(p('info.file.template+'), seqs)
+        specific = lang_size_score(Concat([_info, _file, Plus(_template)]), seqs)
         assert generic > specific
 
     def test_crx_does_not_produce_info_plus(self):
         seqs = [['info', 'file', 'template', 'shell', 'service']] * 5
         g = CRX().infer(seqs)
         alpha = alphabet(g)
-        assert not (len(alpha) == 1 and Symbol('info') in alpha)
+        assert not (len(alpha) == 1 and _info in alpha)
 
     def test_ensemble_does_not_pick_info_plus(self):
         seqs = [['info', 'file', 'template', 'shell', 'service']] * 5
         r = infer_ensemble(seqs)
         best = r['best']['grammar']
         alpha = alphabet(best)
-        assert not (len(alpha) == 1 and Symbol('info') in alpha)
+        assert not (len(alpha) == 1 and _info in alpha)