bex/tag_preprocessor/analyze.py

@@ -331,7 +331,54 @@ def _recursive_split(symbol_seqs, min_subgroup=3, max_depth=3, _depth=0):
     return result
 
 
-def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, split_mixed=False):
+def _count_optionals(grammar):
+    """Count optional parts in a SORE grammar.
+
+    Returns (n_optional, n_concat_parts) counting only top-level parts
+    of concatenations that end with ?. A grammar like a?.b?.c?.d?.e?
+    has 5 optional parts out of 5 concat parts = 1.0 ratio (over-approx).
+    A grammar like a.(b|c).(d|e) has 0 optional parts = 0.0 ratio (structured).
+    """
+    import re
+    # Split on top-level dots (concatenation)
+    parts = []
+    depth = 0
+    cur = []
+    for ch in grammar:
+        if ch == '(':
+            depth += 1
+            cur.append(ch)
+        elif ch == ')':
+            depth -= 1
+            cur.append(ch)
+        elif ch == '.' and depth == 0:
+            parts.append(''.join(cur))
+            cur = []
+        else:
+            cur.append(ch)
+    parts.append(''.join(cur))
+
+    # Count parts ending with ? (but not +?)
+    n_optional = sum(1 for p in parts if p.endswith('?') and not p.endswith('+?'))
+    n_total = len(parts)
+    return n_optional, n_total
+
+
+def _should_try_idregex(grammar, n_methods):
+    """Decide if iDRegEx refinement is worth trying.
+
+    Heuristic: only try if the group is small (≤10 methods) AND
+    the CRX grammar is a flat optional chain (high optional ratio).
+    """
+    if n_methods > 10:
+        return False
+    n_optional, n_total = _count_optionals(grammar)
+    if n_total < 3:
+        return False
+    return n_optional / n_total > 0.5
+
+
+def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, split_mixed=False, idregex_refine=False):
     """Infer grammar for one package group. Module-level for ProcessPoolExecutor."""
     filtered = frequency_filter(group_seqs, min_coverage=min_coverage)
     imports = _extract_imports(group_files)
@@ -401,11 +448,30 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre
             meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "low_structure", "structure_score": grammar_structure_score(grammar)}
             return (label, None, len(filtered), meta)
 
+    # iDRegEx refinement: try on small groups with many optionals
+    if idregex_refine and result and result.get('best') and result['best'].get('grammar'):
+        grammar = result['best']['grammar']
+        if _should_try_idregex(grammar, len(symbol_seqs)):
+            from ..idregex import idregex
+            from ..mdl import lang_size_score, model_cost
+            idr_g = idregex(symbol_seqs, kmax=kmax, N=N)
+            if idr_g and idr_g != '∅':
+                ok_idr, _ = validate_sore(idr_g)
+                if ok_idr and model_cost(idr_g) >= 2:
+                    crx_lang = lang_size_score(grammar, symbol_seqs)
+                    idr_lang = lang_size_score(idr_g, symbol_seqs)
+                    if crx_lang > 0 and idr_lang > 0 and crx_lang / idr_lang > 10:
+                        result = {
+                            'best': {'algorithm': 'iDRegEx', 'grammar': idr_g, 'mdl_score': idr_lang},
+                            'all': [result['best'], {'algorithm': 'iDRegEx', 'grammar': idr_g, 'mdl_score': idr_lang}],
+                            'why': f"iDRefined: {crx_lang/idr_lang:.0f}x tighter by lang_size",
+                        }
+
     meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
     return (label, result, len(filtered), meta)
 
 
-def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, split_mixed=False):
+def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, split_mixed=False, idregex_refine=False):
     """Preprocess and group by package directory, infer per group.
 
     Groups methods by their file's relative directory path, merging
@@ -440,7 +506,7 @@ def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFA
             gs = [sequences[i] for i in indices]
             gf = set(seq_files[i] for i in indices)
             f = ex.submit(_infer_group, label, gs, gf, project_root,
-                           min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure, split_mixed)
+                           min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure, split_mixed, idregex_refine)
             futures[f] = label
 
         for f in as_completed(futures):
@@ -539,7 +605,7 @@ def _filter_glob(files, include=None, exclude=None):
     return files
 
 
-def analyze_by_reduce(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, reduce_threshold=0.15):
+def analyze_by_reduce(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, reduce_threshold=0.15, idregex_refine=False):
     """Reduce-style analysis: group by directory, then merge similar groups.
 
     Uses Algorithm 4 (Reduce, TODS 2010) to merge directories with similar
@@ -579,7 +645,7 @@ def analyze_by_reduce(file_paths, extension, project_root="", min_coverage=DEFAU
         futures = {}
         for label, seqs in result['merged'].items():
             f = ex.submit(_infer_group, label, seqs, set(), project_root,
-                           min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure)
+                           min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure, False, idregex_refine)
             futures[f] = label
 
         done = 0
@@ -592,7 +658,7 @@ def analyze_by_reduce(file_paths, extension, project_root="", min_coverage=DEFAU
     return results
 
 
-def analyze_by_ilocal(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, context_strategy="dir", reduce=True):
+def analyze_by_ilocal(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, context_strategy="dir", reduce=True, idregex_refine=False):
     """iLocal-style analysis: extract (context, sequence) pairs, reduce, infer.
 
     Instead of hard-coding directory as grouping key, this extracts contexts
@@ -640,7 +706,7 @@ def analyze_by_ilocal(file_paths, extension, project_root="", min_coverage=DEFAU
         futures = {}
         for label, seqs in context_groups.items():
             f = ex.submit(_infer_group, label, seqs, set(), project_root,
-                           min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure)
+                           min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure, False, idregex_refine)
             futures[f] = label
 
         done = 0
@@ -671,6 +737,7 @@ def analyze_directory(
     context_strategy="dir",
     reduce_threshold=0.15,
     split_mixed=False,
+    idregex_refine=False,
 ):
     """Scan a directory and run analysis for each language found.
 
@@ -711,6 +778,7 @@ def analyze_directory(
                 crx_method=crx_method,
                 min_structure=min_structure,
                 split_mixed=split_mixed,
+                idregex_refine=idregex_refine,
             )
         elif slice == "reduce":
             results[ext] = analyze_by_reduce(
@@ -726,6 +794,7 @@ def analyze_directory(
                 crx_method=crx_method,
                 min_structure=min_structure,
                 reduce_threshold=reduce_threshold,
+                idregex_refine=idregex_refine,
             )
         elif slice == "ilocal":
             results[ext] = analyze_by_ilocal(
@@ -741,6 +810,7 @@ def analyze_directory(
                 crx_method=crx_method,
                 min_structure=min_structure,
                 context_strategy=context_strategy,
+                idregex_refine=idregex_refine,
             )
         else:
             results[ext] = analyze_clusters(
@@ -942,6 +1012,10 @@ def _parse_args(argv=None):
         "--split-mixed", action="store_true",
         help="Split groups with mixed first symbols before CRX inference (produces tighter grammars)",
     )
+    parser.add_argument(
+        "--idregex-refine", action="store_true",
+        help="Run iDRegEx on small groups (≤10 methods) where CRX grammar has many optionals — picks tighter grammar by lang_size",
+    )
     return parser.parse_args(argv)
 
 
@@ -969,6 +1043,7 @@ def main():
         context_strategy=args.context_strategy,
         reduce_threshold=args.reduce_threshold,
         split_mixed=args.split_mixed,
+        idregex_refine=args.idregex_refine,
     )
 
     if args.json_flag or args.format == "json":

experiments/EXPERIMENT_LOG.md

@@ -685,3 +685,40 @@ For the structured groups, CRX already captures the ordering well (score ≥ 0.5
 (they return None) and not needed for structured groups (CRX already works).
 The pipeline's existing filtering (min_structure, split_mixed) is the right
 approach to handle diversity.
+
+---
+
+## Round 16: iDRegEx Refinement for CRX Flat Bags (commit pending)
+
+**Hypothesis:** CRX over-approximates on small groups with many optional parts
+(flat chains like `a?.b?.c?.d?.e?`). iDRegEx produces tighter nested
+disjunctions on these groups. We can detect the flat bags with a heuristic
+and refine them with iDRegEx, getting >10x tighter grammars at minimal cost.
+
+**Method:**
+1. After CRX produces a grammar, count top-level optional parts
+2. If `n_methods ≤ 10` AND `optionals/total_parts > 0.5` → CRX produced a flat bag
+3. Run iDRegEx on the same sequences
+4. Compare by `lang_size_score` — if >10x improvement, use iDRegEx
+
+**Heuristic (`_count_optionals`):** Splits grammar on top-level dots, counts
+parts ending with `?`. `a?.b?.c?.d?` → 4/4 optionals. `a.(b|c).(d|e)` → 0/3.
+
+**Key insight:** `lang_size_score` (Bex et al.) is the right metric for comparing
+grammars — it counts how many words the grammar accepts at each input length.
+- CRX flat chains accept exponentially many words (e.g., 9432)
+- iDRegEx nested disjunctions accept only the actual sequences (e.g., 60)
+- `lang_size_score` naturally prefers iDRegEx when it produces something
+
+**RAGSAK results:**
+| Package | Methods | CRX optionals | iDRegEx result | lang_size improvement |
+|---------|---------|---------------|----------------|----------------------|
+| agents/capability | 5 | 75% | `(defaultCapabilityId\|summarize)` | 477x tighter |
+
+**Speed cost:** 1 candidate × ~700ms = negligible (0.7s on 74s pipeline).
+
+**Why kORE is dropped:** kORE produces the same or worse output as iDRegEx,
+is sometimes slower, and returns None more often. iDRegEx supersedes kORE.
+
+**Decision:** `--idregex-refine` flag enables this. Default: off.
+When enabled, ~1 candidate per RAGSAK run gets refined. Cost is negligible.

tests/test_analyze.py

@@ -8,7 +8,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent))
 
 from bex.tag_preprocessor.analyze import (
     scan_directory, frequency_filter, infer, analyze_directory, _filter_glob,
-    _group_by_package,
+    _group_by_package, _count_optionals, _should_try_idregex,
 )
 
 
@@ -168,6 +168,46 @@ def test_infer_low_coverage_filters_noise():
     print("  PASS test_infer_low_coverage_filters_noise")
 
 
+def test_count_optionals_flat_chain():
+    n_opt, n_total = _count_optionals("a?.b?.c?.d?.e?")
+    assert n_opt == 5
+    assert n_total == 5
+
+
+def test_count_optionals_no_optionals():
+    n_opt, n_total = _count_optionals("a.b.c")
+    assert n_opt == 0
+    assert n_total == 3
+
+
+def test_count_optionals_mixed():
+    n_opt, n_total = _count_optionals("return.error?.(request+response)?.data?")
+    assert n_opt == 3  # error?, (request+response)?, data?
+    assert n_total == 4  # return.error?.(request+response)?.data?
+
+
+def test_count_optionals_repetition_not_optional():
+    n_opt, n_total = _count_optionals("a.b+.c?")
+    assert n_opt == 1  # only c? is optional
+    assert n_total == 3
+
+
+def test_should_try_idregex_small_many_optionals():
+    assert _should_try_idregex("a?.b?.c?.d?.e?", 5) is True
+
+
+def test_should_try_idregex_large_group():
+    assert _should_try_idregex("a?.b?.c?.d?.e?", 15) is False
+
+
+def test_should_try_idregex_few_optionals():
+    assert _should_try_idregex("a.b.c.d.e", 5) is False
+
+
+def test_should_short_concat():
+    assert _should_try_idregex("a?.b", 5) is False  # too few parts
+
+
 def run_all():
     tests = [
         test_scan_directory_empty,
@@ -182,6 +222,14 @@ def run_all():
         test_frequency_filter_edge_empty_sequences,
         test_infer_returns_ensemble_dict,
         test_infer_low_coverage_filters_noise,
+        test_count_optionals_flat_chain,
+        test_count_optionals_no_optionals,
+        test_count_optionals_mixed,
+        test_count_optionals_repetition_not_optional,
+        test_should_try_idregex_small_many_optionals,
+        test_should_try_idregex_large_group,
+        test_should_try_idregex_few_optionals,
+        test_should_short_concat,
     ]
     passed = 0
     failed = 0