diff --git a/bex/tag_preprocessor/analyze.py b/bex/tag_preprocessor/analyze.py index 440f964..6ac4013 100644 --- a/bex/tag_preprocessor/analyze.py +++ b/bex/tag_preprocessor/analyze.py @@ -256,7 +256,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL if not sequences: return [] - sequences = frequency_filter(sequences, min_coverage=0.2) + sequences = frequency_filter(sequences, min_coverage=min_coverage) cluster_fps = set(seq_files) imports = _extract_imports(cluster_fps) @@ -282,9 +282,9 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL return [("(all methods)", result, len(sequences), meta)] -def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, method='langsize', min_methods=5): +def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, method='langsize', min_methods=5, crx_method='standard'): """Infer grammar for one package group. Module-level for ProcessPoolExecutor.""" - filtered = frequency_filter(group_seqs, min_coverage=0.2) + filtered = frequency_filter(group_seqs, min_coverage=min_coverage) imports = _extract_imports(group_files) arg_patterns = _build_arg_patterns(group_files) packages = _top_packages(group_files, project_root) @@ -302,12 +302,21 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_diverse"} return (label, None, len(filtered), meta) - result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore) + if crx_method == 'refined': + from ..crx_refined import crx_with_confidence + info = crx_with_confidence(symbol_seqs) + result = { + 'best': {'algorithm': 'CRX-refined', 'grammar': info['grammar'], 'mdl_score': info['confidence']}, + 'all': [{'algorithm': 'CRX-refined', 'grammar': info['grammar'], 'mdl_score': info['confidence']}], + 'why': f"CRX-refined (confidence={info['confidence']:.2f})", + } + else: + result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore) meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages} return (label, result, len(filtered), meta) -def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, method='langsize', min_methods=5): +def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, method='langsize', min_methods=5, crx_method='standard'): """Preprocess and group by package directory, infer per group. Groups methods by their file's relative directory path, merging @@ -342,7 +351,7 @@ def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFA gs = [sequences[i] for i in indices] gf = set(seq_files[i] for i in indices) f = ex.submit(_infer_group, label, gs, gf, project_root, - min_coverage, prefer, kmax, N, include_kore, method, min_methods) + min_coverage, prefer, kmax, N, include_kore, method, min_methods, crx_method) futures[f] = label for f in as_completed(futures): @@ -375,7 +384,7 @@ def infer(file_paths, extension, min_coverage=DEFAULT_COVERAGE, prefer=None, kma Ensemble result dict from infer_ensemble. """ sequences, _ = _preprocess_files(file_paths) - sequences = frequency_filter(sequences, min_coverage=0.2) + sequences = frequency_filter(sequences, min_coverage=min_coverage) symbol_seqs = [[text for _, text, _ in seq] for seq in sequences] @@ -453,6 +462,7 @@ def analyze_directory( include_kore=False, method='langsize', min_methods=5, + crx_method='standard', ): """Scan a directory and run analysis for each language found. @@ -489,6 +499,7 @@ def analyze_directory( include_kore=include_kore, method=method, min_methods=min_methods, + crx_method=crx_method, ) else: results[ext] = analyze_clusters( @@ -650,6 +661,10 @@ def _parse_args(argv=None): "--min-methods", type=int, default=5, help="Minimum methods per group to infer grammar (default: 5). Groups with fewer are skipped.", ) + parser.add_argument( + "--crx-method", choices=["standard", "refined"], default="standard", + help="CRX method: standard (default) or refined (cluster-then-infer, tighter grammars)", + ) return parser.parse_args(argv) @@ -671,6 +686,7 @@ def main(): include_kore=args.kore, method=args.scoring_method, min_methods=args.min_methods, + crx_method=args.crx_method, ) if args.json_flag or args.format == "json":