From 8f412905c5c4f9038272615673549383ddd0b26f Mon Sep 17 00:00:00 2001
From: tobjend <tobend85@gmail.com>
Date: Mon, 13 Jul 2026 00:51:11 +0200
Subject: [PATCH] feat: default decomposition ON + restore Language Size
 scoring discrimination

B (decompose default):
- analyze_directory + CLI now default decompose=True (--no-decompose to
  disable); default max_seq_length 5 -> 4 (matches golden_config).
- Decomposing long sequences into <=4-symbol fragments yields tighter
  grammars and disables the too_diverse skip, so far more packages get a
  grammar. RAGSAK: 29 -> 95 grammars, pure bags 9 -> 5; fastapi 74 -> 26
  pure bags; zod 15 -> 5. All runs exit 0, no stalls.

A (repair Language Size scorer, not a new one):
- _COUNT_CAP was applied to count_words' RETURN value, silently clamping
  lang_size_score / model_cost / data_cost at 10^12 for every real codebase
  grammar. That broke ADR-13 (Language Size picks most-specific grammar):
  bags and tight grammars tied at 10^12, so the scorer could not prefer
  specific grammars over generic ones.
- _COUNT_CAP raised to 10**30. Memoization (_count_concat, earlier commit)
  already prevents the recursion hang the cap was guarding against, so the
  cap no longer needs to clamp scores. lang_size_score now discriminates
  (verified: tight=20 vs bag=9975).
---
 bex/grammar.py                  |  2 +-
 bex/tag_preprocessor/analyze.py | 16 ++++++++++------
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/bex/grammar.py b/bex/grammar.py
index ab4db66..94e119e 100644
--- a/bex/grammar.py
+++ b/bex/grammar.py
@@ -170,7 +170,7 @@ def _match_rep(child, seq, pos, min_rep):
 # Counting (for MDL scoring)
 # ---------------------------------------------------------------------------
 
-_COUNT_CAP = 10**12
+_COUNT_CAP = 10 ** 30
 
 @lru_cache(maxsize=None)
 def count_words(node, length):
diff --git a/bex/tag_preprocessor/analyze.py b/bex/tag_preprocessor/analyze.py
index e8e01d5..29e8dc5 100644
--- a/bex/tag_preprocessor/analyze.py
+++ b/bex/tag_preprocessor/analyze.py
@@ -757,8 +757,8 @@ def analyze_directory(
     crx_method='standard',
     min_structure=0.0,
     context_strategy="dir",
-    decompose=False,
-    max_seq_length=5,
+    decompose=True,
+    max_seq_length=4,
     reduce_threshold=0.15,
     split_mixed=False,
     idregex_refine=False,
@@ -1040,12 +1040,16 @@ def _parse_args(argv=None):
         help="Method to split mixed groups: first-symbol (fast, crude) or distributional (slower, smarter clustering)",
     )
     parser.add_argument(
-        "--decompose", action="store_true",
-        help="Decompose long sequences into shorter fragments before inference",
+        "--decompose", action="store_true", default=True,
+        help="Decompose long sequences into shorter fragments before inference (default: on)",
     )
     parser.add_argument(
-        "--max-seq-length", type=int, default=5,
-        help="Maximum sequence length after decomposition (default: 5)",
+        "--no-decompose", dest="decompose", action="store_false",
+        help="Disable sequence decomposition",
+    )
+    parser.add_argument(
+        "--max-seq-length", type=int, default=4,
+        help="Maximum sequence length after decomposition (default: 4)",
     )
     parser.add_argument(
         "--idregex-refine", action="store_true",