From ab20b03256e61037006e24132edeb017ad77d808 Mon Sep 17 00:00:00 2001 From: tobjend Date: Sat, 11 Jul 2026 23:11:54 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20add=20diversity=20threshold=20=E2=80=94?= =?UTF-8?q?=20skip=20groups=20with=20unique=5Fratio=20>=200.9=20or=20metho?= =?UTF-8?q?ds=20<=20min=5Fmethods?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds --min-methods CLI flag (default 5). Groups that are too diverse or too small are skipped with skip_reason in meta. Prevents noisy/meaningless grammars from diverse packages. --- bex/tag_preprocessor/analyze.py | 47 ++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/bex/tag_preprocessor/analyze.py b/bex/tag_preprocessor/analyze.py index daa01e5..440f964 100644 --- a/bex/tag_preprocessor/analyze.py +++ b/bex/tag_preprocessor/analyze.py @@ -245,7 +245,7 @@ def _preprocess_files(file_paths): return sequences, seq_files -def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, method='langsize'): +def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, method='langsize', min_methods=5): """Run full pipeline: preprocess → frequency filter → ensemble infer. Returns: @@ -264,25 +264,50 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL packages = _top_packages(cluster_fps, project_root) symbol_seqs = [[text for _, text, _ in seq] for seq in sequences] - result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore, method=method) + + # Diversity threshold + n_methods = len(symbol_seqs) meta = {"files": cluster_fps, "imports": imports, "arg_patterns": arg_patterns, "packages": packages} + if n_methods < min_methods: + meta["skip_reason"] = "too_few_methods" + return [("(all methods)", None, len(sequences), meta)] + unique_seqs = len(set(tuple(s) for s in symbol_seqs)) + unique_ratio = unique_seqs / n_methods + if unique_ratio > 0.9: + meta["skip_reason"] = "too_diverse" + return [("(all methods)", None, len(sequences), meta)] + + result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore, method=method) return [("(all methods)", result, len(sequences), meta)] -def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, method='langsize'): +def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, method='langsize', min_methods=5): """Infer grammar for one package group. Module-level for ProcessPoolExecutor.""" filtered = frequency_filter(group_seqs, min_coverage=0.2) imports = _extract_imports(group_files) arg_patterns = _build_arg_patterns(group_files) packages = _top_packages(group_files, project_root) symbol_seqs = [[text for _, text, _ in seq] for seq in filtered] + + # Diversity threshold: skip if too few methods or too diverse + n_methods = len(symbol_seqs) + if n_methods < min_methods: + meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_few_methods"} + return (label, None, len(filtered), meta) + + unique_seqs = len(set(tuple(s) for s in symbol_seqs)) + unique_ratio = unique_seqs / n_methods + if unique_ratio > 0.9: + meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_diverse"} + return (label, None, len(filtered), meta) + result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore) meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages} return (label, result, len(filtered), meta) -def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, method='langsize'): +def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, method='langsize', min_methods=5): """Preprocess and group by package directory, infer per group. Groups methods by their file's relative directory path, merging @@ -317,7 +342,7 @@ def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFA gs = [sequences[i] for i in indices] gf = set(seq_files[i] for i in indices) f = ex.submit(_infer_group, label, gs, gf, project_root, - min_coverage, prefer, kmax, N, include_kore, method) + min_coverage, prefer, kmax, N, include_kore, method, min_methods) futures[f] = label for f in as_completed(futures): @@ -427,6 +452,7 @@ def analyze_directory( main_only=False, include_kore=False, method='langsize', + min_methods=5, ): """Scan a directory and run analysis for each language found. @@ -462,6 +488,7 @@ def analyze_directory( kmax=kmax, include_kore=include_kore, method=method, + min_methods=min_methods, ) else: results[ext] = analyze_clusters( @@ -472,6 +499,7 @@ def analyze_directory( kmax=kmax, include_kore=include_kore, method=method, + min_methods=min_methods, ) return results @@ -618,6 +646,10 @@ def _parse_args(argv=None): "--scoring-method", choices=["langsize", "mdl"], default="langsize", help="Scoring method: langsize (default, Bex et al.) or mdl (fallback)", ) + parser.add_argument( + "--min-methods", type=int, default=5, + help="Minimum methods per group to infer grammar (default: 5). Groups with fewer are skipped.", + ) return parser.parse_args(argv) @@ -638,6 +670,7 @@ def main(): main_only=args.main_only, include_kore=args.kore, method=args.scoring_method, + min_methods=args.min_methods, ) if args.json_flag or args.format == "json": @@ -654,7 +687,9 @@ def main(): print(f" Grammar: {best['grammar']}") print(f" Score: {best['mdl_score']}") else: - print(f" ╰─ {label} ({count} methods) — no grammar") + reason = meta.get("skip_reason", "") + suffix = f" — {reason}" if reason else "" + print(f" ╰─ {label} ({count} methods) — no grammar{suffix}") imps = meta.get("imports", []) if imps: joined = " | ".join(imps[:6])