feat: iDRegEx refinement for CRX flat bags (Round 16)
Heuristic: only run iDRegEx when n_methods ≤ 10 AND CRX grammar has >50% top-level optional parts (flat chain signal). If iDRegEx grammar is >10x tighter by lang_size, use it. Otherwise keep CRX. RAGSAK result: agents/capability (5 methods) refined from slot?.(defaultCapabilityId+summarize)?... (lang_size=1432) to (defaultCapabilityId|summarize) (lang_size=3) — 477x tighter. Speed cost: ~0.7s per candidate, negligible on 74s pipeline. CLI: --idregex-refine flag (default off). Also adds _count_optionals() and _should_try_idregex() helpers with 8 pytest tests. 234 tests pass.
This commit is contained in:
tobjend
parent
e9f672cba4
commit
b92b7653e2
3 changed files with 168 additions and 8 deletions
|
|
@ -331,7 +331,54 @@ def _recursive_split(symbol_seqs, min_subgroup=3, max_depth=3, _depth=0):
|
|||
return result
|
||||
|
||||
|
||||
def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, split_mixed=False):
|
||||
def _count_optionals(grammar):
|
||||
"""Count optional parts in a SORE grammar.
|
||||
|
||||
Returns (n_optional, n_concat_parts) counting only top-level parts
|
||||
of concatenations that end with ?. A grammar like a?.b?.c?.d?.e?
|
||||
has 5 optional parts out of 5 concat parts = 1.0 ratio (over-approx).
|
||||
A grammar like a.(b|c).(d|e) has 0 optional parts = 0.0 ratio (structured).
|
||||
"""
|
||||
import re
|
||||
# Split on top-level dots (concatenation)
|
||||
parts = []
|
||||
depth = 0
|
||||
cur = []
|
||||
for ch in grammar:
|
||||
if ch == '(':
|
||||
depth += 1
|
||||
cur.append(ch)
|
||||
elif ch == ')':
|
||||
depth -= 1
|
||||
cur.append(ch)
|
||||
elif ch == '.' and depth == 0:
|
||||
parts.append(''.join(cur))
|
||||
cur = []
|
||||
else:
|
||||
cur.append(ch)
|
||||
parts.append(''.join(cur))
|
||||
|
||||
# Count parts ending with ? (but not +?)
|
||||
n_optional = sum(1 for p in parts if p.endswith('?') and not p.endswith('+?'))
|
||||
n_total = len(parts)
|
||||
return n_optional, n_total
|
||||
|
||||
|
||||
def _should_try_idregex(grammar, n_methods):
|
||||
"""Decide if iDRegEx refinement is worth trying.
|
||||
|
||||
Heuristic: only try if the group is small (≤10 methods) AND
|
||||
the CRX grammar is a flat optional chain (high optional ratio).
|
||||
"""
|
||||
if n_methods > 10:
|
||||
return False
|
||||
n_optional, n_total = _count_optionals(grammar)
|
||||
if n_total < 3:
|
||||
return False
|
||||
return n_optional / n_total > 0.5
|
||||
|
||||
|
||||
def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, split_mixed=False, idregex_refine=False):
|
||||
"""Infer grammar for one package group. Module-level for ProcessPoolExecutor."""
|
||||
filtered = frequency_filter(group_seqs, min_coverage=min_coverage)
|
||||
imports = _extract_imports(group_files)
|
||||
|
|
@ -401,11 +448,30 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre
|
|||
meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "low_structure", "structure_score": grammar_structure_score(grammar)}
|
||||
return (label, None, len(filtered), meta)
|
||||
|
||||
# iDRegEx refinement: try on small groups with many optionals
|
||||
if idregex_refine and result and result.get('best') and result['best'].get('grammar'):
|
||||
grammar = result['best']['grammar']
|
||||
if _should_try_idregex(grammar, len(symbol_seqs)):
|
||||
from ..idregex import idregex
|
||||
from ..mdl import lang_size_score, model_cost
|
||||
idr_g = idregex(symbol_seqs, kmax=kmax, N=N)
|
||||
if idr_g and idr_g != '∅':
|
||||
ok_idr, _ = validate_sore(idr_g)
|
||||
if ok_idr and model_cost(idr_g) >= 2:
|
||||
crx_lang = lang_size_score(grammar, symbol_seqs)
|
||||
idr_lang = lang_size_score(idr_g, symbol_seqs)
|
||||
if crx_lang > 0 and idr_lang > 0 and crx_lang / idr_lang > 10:
|
||||
result = {
|
||||
'best': {'algorithm': 'iDRegEx', 'grammar': idr_g, 'mdl_score': idr_lang},
|
||||
'all': [result['best'], {'algorithm': 'iDRegEx', 'grammar': idr_g, 'mdl_score': idr_lang}],
|
||||
'why': f"iDRefined: {crx_lang/idr_lang:.0f}x tighter by lang_size",
|
||||
}
|
||||
|
||||
meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
|
||||
return (label, result, len(filtered), meta)
|
||||
|
||||
|
||||
def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, split_mixed=False):
|
||||
def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, split_mixed=False, idregex_refine=False):
|
||||
"""Preprocess and group by package directory, infer per group.
|
||||
|
||||
Groups methods by their file's relative directory path, merging
|
||||
|
|
@ -440,7 +506,7 @@ def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFA
|
|||
gs = [sequences[i] for i in indices]
|
||||
gf = set(seq_files[i] for i in indices)
|
||||
f = ex.submit(_infer_group, label, gs, gf, project_root,
|
||||
min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure, split_mixed)
|
||||
min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure, split_mixed, idregex_refine)
|
||||
futures[f] = label
|
||||
|
||||
for f in as_completed(futures):
|
||||
|
|
@ -539,7 +605,7 @@ def _filter_glob(files, include=None, exclude=None):
|
|||
return files
|
||||
|
||||
|
||||
def analyze_by_reduce(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, reduce_threshold=0.15):
|
||||
def analyze_by_reduce(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, reduce_threshold=0.15, idregex_refine=False):
|
||||
"""Reduce-style analysis: group by directory, then merge similar groups.
|
||||
|
||||
Uses Algorithm 4 (Reduce, TODS 2010) to merge directories with similar
|
||||
|
|
@ -579,7 +645,7 @@ def analyze_by_reduce(file_paths, extension, project_root="", min_coverage=DEFAU
|
|||
futures = {}
|
||||
for label, seqs in result['merged'].items():
|
||||
f = ex.submit(_infer_group, label, seqs, set(), project_root,
|
||||
min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure)
|
||||
min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure, False, idregex_refine)
|
||||
futures[f] = label
|
||||
|
||||
done = 0
|
||||
|
|
@ -592,7 +658,7 @@ def analyze_by_reduce(file_paths, extension, project_root="", min_coverage=DEFAU
|
|||
return results
|
||||
|
||||
|
||||
def analyze_by_ilocal(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, context_strategy="dir", reduce=True):
|
||||
def analyze_by_ilocal(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, context_strategy="dir", reduce=True, idregex_refine=False):
|
||||
"""iLocal-style analysis: extract (context, sequence) pairs, reduce, infer.
|
||||
|
||||
Instead of hard-coding directory as grouping key, this extracts contexts
|
||||
|
|
@ -640,7 +706,7 @@ def analyze_by_ilocal(file_paths, extension, project_root="", min_coverage=DEFAU
|
|||
futures = {}
|
||||
for label, seqs in context_groups.items():
|
||||
f = ex.submit(_infer_group, label, seqs, set(), project_root,
|
||||
min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure)
|
||||
min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure, False, idregex_refine)
|
||||
futures[f] = label
|
||||
|
||||
done = 0
|
||||
|
|
@ -671,6 +737,7 @@ def analyze_directory(
|
|||
context_strategy="dir",
|
||||
reduce_threshold=0.15,
|
||||
split_mixed=False,
|
||||
idregex_refine=False,
|
||||
):
|
||||
"""Scan a directory and run analysis for each language found.
|
||||
|
||||
|
|
@ -711,6 +778,7 @@ def analyze_directory(
|
|||
crx_method=crx_method,
|
||||
min_structure=min_structure,
|
||||
split_mixed=split_mixed,
|
||||
idregex_refine=idregex_refine,
|
||||
)
|
||||
elif slice == "reduce":
|
||||
results[ext] = analyze_by_reduce(
|
||||
|
|
@ -726,6 +794,7 @@ def analyze_directory(
|
|||
crx_method=crx_method,
|
||||
min_structure=min_structure,
|
||||
reduce_threshold=reduce_threshold,
|
||||
idregex_refine=idregex_refine,
|
||||
)
|
||||
elif slice == "ilocal":
|
||||
results[ext] = analyze_by_ilocal(
|
||||
|
|
@ -741,6 +810,7 @@ def analyze_directory(
|
|||
crx_method=crx_method,
|
||||
min_structure=min_structure,
|
||||
context_strategy=context_strategy,
|
||||
idregex_refine=idregex_refine,
|
||||
)
|
||||
else:
|
||||
results[ext] = analyze_clusters(
|
||||
|
|
@ -942,6 +1012,10 @@ def _parse_args(argv=None):
|
|||
"--split-mixed", action="store_true",
|
||||
help="Split groups with mixed first symbols before CRX inference (produces tighter grammars)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--idregex-refine", action="store_true",
|
||||
help="Run iDRegEx on small groups (≤10 methods) where CRX grammar has many optionals — picks tighter grammar by lang_size",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
|
|
@ -969,6 +1043,7 @@ def main():
|
|||
context_strategy=args.context_strategy,
|
||||
reduce_threshold=args.reduce_threshold,
|
||||
split_mixed=args.split_mixed,
|
||||
idregex_refine=args.idregex_refine,
|
||||
)
|
||||
|
||||
if args.json_flag or args.format == "json":
|
||||
|
|
|
|||
|
|
@ -685,3 +685,40 @@ For the structured groups, CRX already captures the ordering well (score ≥ 0.5
|
|||
(they return None) and not needed for structured groups (CRX already works).
|
||||
The pipeline's existing filtering (min_structure, split_mixed) is the right
|
||||
approach to handle diversity.
|
||||
|
||||
---
|
||||
|
||||
## Round 16: iDRegEx Refinement for CRX Flat Bags (commit pending)
|
||||
|
||||
**Hypothesis:** CRX over-approximates on small groups with many optional parts
|
||||
(flat chains like `a?.b?.c?.d?.e?`). iDRegEx produces tighter nested
|
||||
disjunctions on these groups. We can detect the flat bags with a heuristic
|
||||
and refine them with iDRegEx, getting >10x tighter grammars at minimal cost.
|
||||
|
||||
**Method:**
|
||||
1. After CRX produces a grammar, count top-level optional parts
|
||||
2. If `n_methods ≤ 10` AND `optionals/total_parts > 0.5` → CRX produced a flat bag
|
||||
3. Run iDRegEx on the same sequences
|
||||
4. Compare by `lang_size_score` — if >10x improvement, use iDRegEx
|
||||
|
||||
**Heuristic (`_count_optionals`):** Splits grammar on top-level dots, counts
|
||||
parts ending with `?`. `a?.b?.c?.d?` → 4/4 optionals. `a.(b|c).(d|e)` → 0/3.
|
||||
|
||||
**Key insight:** `lang_size_score` (Bex et al.) is the right metric for comparing
|
||||
grammars — it counts how many words the grammar accepts at each input length.
|
||||
- CRX flat chains accept exponentially many words (e.g., 9432)
|
||||
- iDRegEx nested disjunctions accept only the actual sequences (e.g., 60)
|
||||
- `lang_size_score` naturally prefers iDRegEx when it produces something
|
||||
|
||||
**RAGSAK results:**
|
||||
| Package | Methods | CRX optionals | iDRegEx result | lang_size improvement |
|
||||
|---------|---------|---------------|----------------|----------------------|
|
||||
| agents/capability | 5 | 75% | `(defaultCapabilityId\|summarize)` | 477x tighter |
|
||||
|
||||
**Speed cost:** 1 candidate × ~700ms = negligible (0.7s on 74s pipeline).
|
||||
|
||||
**Why kORE is dropped:** kORE produces the same or worse output as iDRegEx,
|
||||
is sometimes slower, and returns None more often. iDRegEx supersedes kORE.
|
||||
|
||||
**Decision:** `--idregex-refine` flag enables this. Default: off.
|
||||
When enabled, ~1 candidate per RAGSAK run gets refined. Cost is negligible.
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent))
|
|||
|
||||
from bex.tag_preprocessor.analyze import (
|
||||
scan_directory, frequency_filter, infer, analyze_directory, _filter_glob,
|
||||
_group_by_package,
|
||||
_group_by_package, _count_optionals, _should_try_idregex,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -168,6 +168,46 @@ def test_infer_low_coverage_filters_noise():
|
|||
print(" PASS test_infer_low_coverage_filters_noise")
|
||||
|
||||
|
||||
def test_count_optionals_flat_chain():
|
||||
n_opt, n_total = _count_optionals("a?.b?.c?.d?.e?")
|
||||
assert n_opt == 5
|
||||
assert n_total == 5
|
||||
|
||||
|
||||
def test_count_optionals_no_optionals():
|
||||
n_opt, n_total = _count_optionals("a.b.c")
|
||||
assert n_opt == 0
|
||||
assert n_total == 3
|
||||
|
||||
|
||||
def test_count_optionals_mixed():
|
||||
n_opt, n_total = _count_optionals("return.error?.(request+response)?.data?")
|
||||
assert n_opt == 3 # error?, (request+response)?, data?
|
||||
assert n_total == 4 # return.error?.(request+response)?.data?
|
||||
|
||||
|
||||
def test_count_optionals_repetition_not_optional():
|
||||
n_opt, n_total = _count_optionals("a.b+.c?")
|
||||
assert n_opt == 1 # only c? is optional
|
||||
assert n_total == 3
|
||||
|
||||
|
||||
def test_should_try_idregex_small_many_optionals():
|
||||
assert _should_try_idregex("a?.b?.c?.d?.e?", 5) is True
|
||||
|
||||
|
||||
def test_should_try_idregex_large_group():
|
||||
assert _should_try_idregex("a?.b?.c?.d?.e?", 15) is False
|
||||
|
||||
|
||||
def test_should_try_idregex_few_optionals():
|
||||
assert _should_try_idregex("a.b.c.d.e", 5) is False
|
||||
|
||||
|
||||
def test_should_short_concat():
|
||||
assert _should_try_idregex("a?.b", 5) is False # too few parts
|
||||
|
||||
|
||||
def run_all():
|
||||
tests = [
|
||||
test_scan_directory_empty,
|
||||
|
|
@ -182,6 +222,14 @@ def run_all():
|
|||
test_frequency_filter_edge_empty_sequences,
|
||||
test_infer_returns_ensemble_dict,
|
||||
test_infer_low_coverage_filters_noise,
|
||||
test_count_optionals_flat_chain,
|
||||
test_count_optionals_no_optionals,
|
||||
test_count_optionals_mixed,
|
||||
test_count_optionals_repetition_not_optional,
|
||||
test_should_try_idregex_small_many_optionals,
|
||||
test_should_try_idregex_large_group,
|
||||
test_should_try_idregex_few_optionals,
|
||||
test_should_short_concat,
|
||||
]
|
||||
passed = 0
|
||||
failed = 0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue