From b92b7653e27b0c18448e8dc1aee712d7c21428e7 Mon Sep 17 00:00:00 2001 From: tobjend Date: Sun, 12 Jul 2026 16:54:40 +0200 Subject: [PATCH] feat: iDRegEx refinement for CRX flat bags (Round 16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Heuristic: only run iDRegEx when n_methods ≤ 10 AND CRX grammar has >50% top-level optional parts (flat chain signal). If iDRegEx grammar is >10x tighter by lang_size, use it. Otherwise keep CRX. RAGSAK result: agents/capability (5 methods) refined from slot?.(defaultCapabilityId+summarize)?... (lang_size=1432) to (defaultCapabilityId|summarize) (lang_size=3) — 477x tighter. Speed cost: ~0.7s per candidate, negligible on 74s pipeline. CLI: --idregex-refine flag (default off). Also adds _count_optionals() and _should_try_idregex() helpers with 8 pytest tests. 234 tests pass. --- bex/tag_preprocessor/analyze.py | 89 ++++++++++++++++++++++++++++++--- experiments/EXPERIMENT_LOG.md | 37 ++++++++++++++ tests/test_analyze.py | 50 +++++++++++++++++- 3 files changed, 168 insertions(+), 8 deletions(-) diff --git a/bex/tag_preprocessor/analyze.py b/bex/tag_preprocessor/analyze.py index 0c12744..f2806a5 100644 --- a/bex/tag_preprocessor/analyze.py +++ b/bex/tag_preprocessor/analyze.py @@ -331,7 +331,54 @@ def _recursive_split(symbol_seqs, min_subgroup=3, max_depth=3, _depth=0): return result -def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, split_mixed=False): +def _count_optionals(grammar): + """Count optional parts in a SORE grammar. + + Returns (n_optional, n_concat_parts) counting only top-level parts + of concatenations that end with ?. A grammar like a?.b?.c?.d?.e? + has 5 optional parts out of 5 concat parts = 1.0 ratio (over-approx). + A grammar like a.(b|c).(d|e) has 0 optional parts = 0.0 ratio (structured). + """ + import re + # Split on top-level dots (concatenation) + parts = [] + depth = 0 + cur = [] + for ch in grammar: + if ch == '(': + depth += 1 + cur.append(ch) + elif ch == ')': + depth -= 1 + cur.append(ch) + elif ch == '.' and depth == 0: + parts.append(''.join(cur)) + cur = [] + else: + cur.append(ch) + parts.append(''.join(cur)) + + # Count parts ending with ? (but not +?) + n_optional = sum(1 for p in parts if p.endswith('?') and not p.endswith('+?')) + n_total = len(parts) + return n_optional, n_total + + +def _should_try_idregex(grammar, n_methods): + """Decide if iDRegEx refinement is worth trying. + + Heuristic: only try if the group is small (≤10 methods) AND + the CRX grammar is a flat optional chain (high optional ratio). + """ + if n_methods > 10: + return False + n_optional, n_total = _count_optionals(grammar) + if n_total < 3: + return False + return n_optional / n_total > 0.5 + + +def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, split_mixed=False, idregex_refine=False): """Infer grammar for one package group. Module-level for ProcessPoolExecutor.""" filtered = frequency_filter(group_seqs, min_coverage=min_coverage) imports = _extract_imports(group_files) @@ -401,11 +448,30 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "low_structure", "structure_score": grammar_structure_score(grammar)} return (label, None, len(filtered), meta) + # iDRegEx refinement: try on small groups with many optionals + if idregex_refine and result and result.get('best') and result['best'].get('grammar'): + grammar = result['best']['grammar'] + if _should_try_idregex(grammar, len(symbol_seqs)): + from ..idregex import idregex + from ..mdl import lang_size_score, model_cost + idr_g = idregex(symbol_seqs, kmax=kmax, N=N) + if idr_g and idr_g != '∅': + ok_idr, _ = validate_sore(idr_g) + if ok_idr and model_cost(idr_g) >= 2: + crx_lang = lang_size_score(grammar, symbol_seqs) + idr_lang = lang_size_score(idr_g, symbol_seqs) + if crx_lang > 0 and idr_lang > 0 and crx_lang / idr_lang > 10: + result = { + 'best': {'algorithm': 'iDRegEx', 'grammar': idr_g, 'mdl_score': idr_lang}, + 'all': [result['best'], {'algorithm': 'iDRegEx', 'grammar': idr_g, 'mdl_score': idr_lang}], + 'why': f"iDRefined: {crx_lang/idr_lang:.0f}x tighter by lang_size", + } + meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages} return (label, result, len(filtered), meta) -def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, split_mixed=False): +def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, split_mixed=False, idregex_refine=False): """Preprocess and group by package directory, infer per group. Groups methods by their file's relative directory path, merging @@ -440,7 +506,7 @@ def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFA gs = [sequences[i] for i in indices] gf = set(seq_files[i] for i in indices) f = ex.submit(_infer_group, label, gs, gf, project_root, - min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure, split_mixed) + min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure, split_mixed, idregex_refine) futures[f] = label for f in as_completed(futures): @@ -539,7 +605,7 @@ def _filter_glob(files, include=None, exclude=None): return files -def analyze_by_reduce(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, reduce_threshold=0.15): +def analyze_by_reduce(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, reduce_threshold=0.15, idregex_refine=False): """Reduce-style analysis: group by directory, then merge similar groups. Uses Algorithm 4 (Reduce, TODS 2010) to merge directories with similar @@ -579,7 +645,7 @@ def analyze_by_reduce(file_paths, extension, project_root="", min_coverage=DEFAU futures = {} for label, seqs in result['merged'].items(): f = ex.submit(_infer_group, label, seqs, set(), project_root, - min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure) + min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure, False, idregex_refine) futures[f] = label done = 0 @@ -592,7 +658,7 @@ def analyze_by_reduce(file_paths, extension, project_root="", min_coverage=DEFAU return results -def analyze_by_ilocal(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, context_strategy="dir", reduce=True): +def analyze_by_ilocal(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, context_strategy="dir", reduce=True, idregex_refine=False): """iLocal-style analysis: extract (context, sequence) pairs, reduce, infer. Instead of hard-coding directory as grouping key, this extracts contexts @@ -640,7 +706,7 @@ def analyze_by_ilocal(file_paths, extension, project_root="", min_coverage=DEFAU futures = {} for label, seqs in context_groups.items(): f = ex.submit(_infer_group, label, seqs, set(), project_root, - min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure) + min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure, False, idregex_refine) futures[f] = label done = 0 @@ -671,6 +737,7 @@ def analyze_directory( context_strategy="dir", reduce_threshold=0.15, split_mixed=False, + idregex_refine=False, ): """Scan a directory and run analysis for each language found. @@ -711,6 +778,7 @@ def analyze_directory( crx_method=crx_method, min_structure=min_structure, split_mixed=split_mixed, + idregex_refine=idregex_refine, ) elif slice == "reduce": results[ext] = analyze_by_reduce( @@ -726,6 +794,7 @@ def analyze_directory( crx_method=crx_method, min_structure=min_structure, reduce_threshold=reduce_threshold, + idregex_refine=idregex_refine, ) elif slice == "ilocal": results[ext] = analyze_by_ilocal( @@ -741,6 +810,7 @@ def analyze_directory( crx_method=crx_method, min_structure=min_structure, context_strategy=context_strategy, + idregex_refine=idregex_refine, ) else: results[ext] = analyze_clusters( @@ -942,6 +1012,10 @@ def _parse_args(argv=None): "--split-mixed", action="store_true", help="Split groups with mixed first symbols before CRX inference (produces tighter grammars)", ) + parser.add_argument( + "--idregex-refine", action="store_true", + help="Run iDRegEx on small groups (≤10 methods) where CRX grammar has many optionals — picks tighter grammar by lang_size", + ) return parser.parse_args(argv) @@ -969,6 +1043,7 @@ def main(): context_strategy=args.context_strategy, reduce_threshold=args.reduce_threshold, split_mixed=args.split_mixed, + idregex_refine=args.idregex_refine, ) if args.json_flag or args.format == "json": diff --git a/experiments/EXPERIMENT_LOG.md b/experiments/EXPERIMENT_LOG.md index 68c69e5..82df3ad 100644 --- a/experiments/EXPERIMENT_LOG.md +++ b/experiments/EXPERIMENT_LOG.md @@ -685,3 +685,40 @@ For the structured groups, CRX already captures the ordering well (score ≥ 0.5 (they return None) and not needed for structured groups (CRX already works). The pipeline's existing filtering (min_structure, split_mixed) is the right approach to handle diversity. + +--- + +## Round 16: iDRegEx Refinement for CRX Flat Bags (commit pending) + +**Hypothesis:** CRX over-approximates on small groups with many optional parts +(flat chains like `a?.b?.c?.d?.e?`). iDRegEx produces tighter nested +disjunctions on these groups. We can detect the flat bags with a heuristic +and refine them with iDRegEx, getting >10x tighter grammars at minimal cost. + +**Method:** +1. After CRX produces a grammar, count top-level optional parts +2. If `n_methods ≤ 10` AND `optionals/total_parts > 0.5` → CRX produced a flat bag +3. Run iDRegEx on the same sequences +4. Compare by `lang_size_score` — if >10x improvement, use iDRegEx + +**Heuristic (`_count_optionals`):** Splits grammar on top-level dots, counts +parts ending with `?`. `a?.b?.c?.d?` → 4/4 optionals. `a.(b|c).(d|e)` → 0/3. + +**Key insight:** `lang_size_score` (Bex et al.) is the right metric for comparing +grammars — it counts how many words the grammar accepts at each input length. +- CRX flat chains accept exponentially many words (e.g., 9432) +- iDRegEx nested disjunctions accept only the actual sequences (e.g., 60) +- `lang_size_score` naturally prefers iDRegEx when it produces something + +**RAGSAK results:** +| Package | Methods | CRX optionals | iDRegEx result | lang_size improvement | +|---------|---------|---------------|----------------|----------------------| +| agents/capability | 5 | 75% | `(defaultCapabilityId\|summarize)` | 477x tighter | + +**Speed cost:** 1 candidate × ~700ms = negligible (0.7s on 74s pipeline). + +**Why kORE is dropped:** kORE produces the same or worse output as iDRegEx, +is sometimes slower, and returns None more often. iDRegEx supersedes kORE. + +**Decision:** `--idregex-refine` flag enables this. Default: off. +When enabled, ~1 candidate per RAGSAK run gets refined. Cost is negligible. diff --git a/tests/test_analyze.py b/tests/test_analyze.py index 7e8b874..e81af45 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -8,7 +8,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from bex.tag_preprocessor.analyze import ( scan_directory, frequency_filter, infer, analyze_directory, _filter_glob, - _group_by_package, + _group_by_package, _count_optionals, _should_try_idregex, ) @@ -168,6 +168,46 @@ def test_infer_low_coverage_filters_noise(): print(" PASS test_infer_low_coverage_filters_noise") +def test_count_optionals_flat_chain(): + n_opt, n_total = _count_optionals("a?.b?.c?.d?.e?") + assert n_opt == 5 + assert n_total == 5 + + +def test_count_optionals_no_optionals(): + n_opt, n_total = _count_optionals("a.b.c") + assert n_opt == 0 + assert n_total == 3 + + +def test_count_optionals_mixed(): + n_opt, n_total = _count_optionals("return.error?.(request+response)?.data?") + assert n_opt == 3 # error?, (request+response)?, data? + assert n_total == 4 # return.error?.(request+response)?.data? + + +def test_count_optionals_repetition_not_optional(): + n_opt, n_total = _count_optionals("a.b+.c?") + assert n_opt == 1 # only c? is optional + assert n_total == 3 + + +def test_should_try_idregex_small_many_optionals(): + assert _should_try_idregex("a?.b?.c?.d?.e?", 5) is True + + +def test_should_try_idregex_large_group(): + assert _should_try_idregex("a?.b?.c?.d?.e?", 15) is False + + +def test_should_try_idregex_few_optionals(): + assert _should_try_idregex("a.b.c.d.e", 5) is False + + +def test_should_short_concat(): + assert _should_try_idregex("a?.b", 5) is False # too few parts + + def run_all(): tests = [ test_scan_directory_empty, @@ -182,6 +222,14 @@ def run_all(): test_frequency_filter_edge_empty_sequences, test_infer_returns_ensemble_dict, test_infer_low_coverage_filters_noise, + test_count_optionals_flat_chain, + test_count_optionals_no_optionals, + test_count_optionals_mixed, + test_count_optionals_repetition_not_optional, + test_should_try_idregex_small_many_optionals, + test_should_try_idregex_large_group, + test_should_try_idregex_few_optionals, + test_should_short_concat, ] passed = 0 failed = 0