tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix: wire crx_refined into pipeline, fix hardcoded min_coverage

Browse source 
- Add --crx-method (standard|refined) to CLI and analyze_directory
- Pass crx_method through analyze_by_package → _infer_group
- Fix hardcoded min_coverage=0.2 in 3 places (was ignoring parameter)
- Refined CRX wraps crx_with_confidence into standard result format
This commit is contained in:
tobjend 2026-07-12 02:09:52 +02:00
parent 739000e8c6
commit 6912841b9e
1 changed files with 23 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
bex/tag_preprocessor/analyze.py
View file

@ -256,7 +256,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
if not sequences:
return []
sequences = frequency_filter(sequences, min_coverage=0.2)
sequences = frequency_filter(sequences, min_coverage=min_coverage)
cluster_fps = set(seq_files)
imports = _extract_imports(cluster_fps)
@ -282,9 +282,9 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
return [("(all methods)", result, len(sequences), meta)]
def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, method='langsize', min_methods=5):
def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, method='langsize', min_methods=5, crx_method='standard'):
"""Infer grammar for one package group. Module-level for ProcessPoolExecutor."""
filtered = frequency_filter(group_seqs, min_coverage=0.2)
filtered = frequency_filter(group_seqs, min_coverage=min_coverage)
imports = _extract_imports(group_files)
arg_patterns = _build_arg_patterns(group_files)
packages = _top_packages(group_files, project_root)
@ -302,12 +302,21 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre
meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_diverse"}
return (label, None, len(filtered), meta)
result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore)
if crx_method == 'refined':
from ..crx_refined import crx_with_confidence
info = crx_with_confidence(symbol_seqs)
result = {
'best': {'algorithm': 'CRX-refined', 'grammar': info['grammar'], 'mdl_score': info['confidence']},
'all': [{'algorithm': 'CRX-refined', 'grammar': info['grammar'], 'mdl_score': info['confidence']}],
'why': f"CRX-refined (confidence={info['confidence']:.2f})",
}
else:
result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore)
meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
return (label, result, len(filtered), meta)
def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, method='langsize', min_methods=5):
def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, method='langsize', min_methods=5, crx_method='standard'):
"""Preprocess and group by package directory, infer per group.
Groups methods by their file's relative directory path, merging
@ -342,7 +351,7 @@ def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFA
gs = [sequences[i] for i in indices]
gf = set(seq_files[i] for i in indices)
f = ex.submit(_infer_group, label, gs, gf, project_root,
min_coverage, prefer, kmax, N, include_kore, method, min_methods)
min_coverage, prefer, kmax, N, include_kore, method, min_methods, crx_method)
futures[f] = label
for f in as_completed(futures):
@ -375,7 +384,7 @@ def infer(file_paths, extension, min_coverage=DEFAULT_COVERAGE, prefer=None, kma
Ensemble result dict from infer_ensemble.
"""
sequences, _ = _preprocess_files(file_paths)
sequences = frequency_filter(sequences, min_coverage=0.2)
sequences = frequency_filter(sequences, min_coverage=min_coverage)
symbol_seqs = [[text for _, text, _ in seq] for seq in sequences]
@ -453,6 +462,7 @@ def analyze_directory(
include_kore=False,
method='langsize',
min_methods=5,
crx_method='standard',
):
"""Scan a directory and run analysis for each language found.
@ -489,6 +499,7 @@ def analyze_directory(
include_kore=include_kore,
method=method,
min_methods=min_methods,
crx_method=crx_method,
)
else:
results[ext] = analyze_clusters(
@ -650,6 +661,10 @@ def _parse_args(argv=None):
"--min-methods", type=int, default=5,
help="Minimum methods per group to infer grammar (default: 5). Groups with fewer are skipped.",
)
parser.add_argument(
"--crx-method", choices=["standard", "refined"], default="standard",
help="CRX method: standard (default) or refined (cluster-then-infer, tighter grammars)",
)
return parser.parse_args(argv)
@ -671,6 +686,7 @@ def main():
include_kore=args.kore,
method=args.scoring_method,
min_methods=args.min_methods,
crx_method=args.crx_method,
)
if args.json_flag or args.format == "json":