fix: wire crx_refined into pipeline, fix hardcoded min_coverage

- Add --crx-method (standard|refined) to CLI and analyze_directory
- Pass crx_method through analyze_by_package → _infer_group
- Fix hardcoded min_coverage=0.2 in 3 places (was ignoring parameter)
- Refined CRX wraps crx_with_confidence into standard result format
This commit is contained in:
tobjend 2026-07-12 02:09:52 +02:00
parent 739000e8c6
commit 6912841b9e

View file

@ -256,7 +256,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
if not sequences: if not sequences:
return [] return []
sequences = frequency_filter(sequences, min_coverage=0.2) sequences = frequency_filter(sequences, min_coverage=min_coverage)
cluster_fps = set(seq_files) cluster_fps = set(seq_files)
imports = _extract_imports(cluster_fps) imports = _extract_imports(cluster_fps)
@ -282,9 +282,9 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
return [("(all methods)", result, len(sequences), meta)] return [("(all methods)", result, len(sequences), meta)]
def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, method='langsize', min_methods=5): def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, method='langsize', min_methods=5, crx_method='standard'):
"""Infer grammar for one package group. Module-level for ProcessPoolExecutor.""" """Infer grammar for one package group. Module-level for ProcessPoolExecutor."""
filtered = frequency_filter(group_seqs, min_coverage=0.2) filtered = frequency_filter(group_seqs, min_coverage=min_coverage)
imports = _extract_imports(group_files) imports = _extract_imports(group_files)
arg_patterns = _build_arg_patterns(group_files) arg_patterns = _build_arg_patterns(group_files)
packages = _top_packages(group_files, project_root) packages = _top_packages(group_files, project_root)
@ -302,12 +302,21 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre
meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_diverse"} meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_diverse"}
return (label, None, len(filtered), meta) return (label, None, len(filtered), meta)
if crx_method == 'refined':
from ..crx_refined import crx_with_confidence
info = crx_with_confidence(symbol_seqs)
result = {
'best': {'algorithm': 'CRX-refined', 'grammar': info['grammar'], 'mdl_score': info['confidence']},
'all': [{'algorithm': 'CRX-refined', 'grammar': info['grammar'], 'mdl_score': info['confidence']}],
'why': f"CRX-refined (confidence={info['confidence']:.2f})",
}
else:
result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore) result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore)
meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages} meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
return (label, result, len(filtered), meta) return (label, result, len(filtered), meta)
def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, method='langsize', min_methods=5): def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, method='langsize', min_methods=5, crx_method='standard'):
"""Preprocess and group by package directory, infer per group. """Preprocess and group by package directory, infer per group.
Groups methods by their file's relative directory path, merging Groups methods by their file's relative directory path, merging
@ -342,7 +351,7 @@ def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFA
gs = [sequences[i] for i in indices] gs = [sequences[i] for i in indices]
gf = set(seq_files[i] for i in indices) gf = set(seq_files[i] for i in indices)
f = ex.submit(_infer_group, label, gs, gf, project_root, f = ex.submit(_infer_group, label, gs, gf, project_root,
min_coverage, prefer, kmax, N, include_kore, method, min_methods) min_coverage, prefer, kmax, N, include_kore, method, min_methods, crx_method)
futures[f] = label futures[f] = label
for f in as_completed(futures): for f in as_completed(futures):
@ -375,7 +384,7 @@ def infer(file_paths, extension, min_coverage=DEFAULT_COVERAGE, prefer=None, kma
Ensemble result dict from infer_ensemble. Ensemble result dict from infer_ensemble.
""" """
sequences, _ = _preprocess_files(file_paths) sequences, _ = _preprocess_files(file_paths)
sequences = frequency_filter(sequences, min_coverage=0.2) sequences = frequency_filter(sequences, min_coverage=min_coverage)
symbol_seqs = [[text for _, text, _ in seq] for seq in sequences] symbol_seqs = [[text for _, text, _ in seq] for seq in sequences]
@ -453,6 +462,7 @@ def analyze_directory(
include_kore=False, include_kore=False,
method='langsize', method='langsize',
min_methods=5, min_methods=5,
crx_method='standard',
): ):
"""Scan a directory and run analysis for each language found. """Scan a directory and run analysis for each language found.
@ -489,6 +499,7 @@ def analyze_directory(
include_kore=include_kore, include_kore=include_kore,
method=method, method=method,
min_methods=min_methods, min_methods=min_methods,
crx_method=crx_method,
) )
else: else:
results[ext] = analyze_clusters( results[ext] = analyze_clusters(
@ -650,6 +661,10 @@ def _parse_args(argv=None):
"--min-methods", type=int, default=5, "--min-methods", type=int, default=5,
help="Minimum methods per group to infer grammar (default: 5). Groups with fewer are skipped.", help="Minimum methods per group to infer grammar (default: 5). Groups with fewer are skipped.",
) )
parser.add_argument(
"--crx-method", choices=["standard", "refined"], default="standard",
help="CRX method: standard (default) or refined (cluster-then-infer, tighter grammars)",
)
return parser.parse_args(argv) return parser.parse_args(argv)
@ -671,6 +686,7 @@ def main():
include_kore=args.kore, include_kore=args.kore,
method=args.scoring_method, method=args.scoring_method,
min_methods=args.min_methods, min_methods=args.min_methods,
crx_method=args.crx_method,
) )
if args.json_flag or args.format == "json": if args.json_flag or args.format == "json":