feat: add diversity threshold — skip groups with unique_ratio > 0.9 or methods < min_methods

Adds --min-methods CLI flag (default 5). Groups that are too diverse or
too small are skipped with skip_reason in meta. Prevents noisy/meaningless
grammars from diverse packages.

bex/tag_preprocessor/analyze.py

@@ -245,7 +245,7 @@ def _preprocess_files(file_paths):
     return sequences, seq_files
 
 
-def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, method='langsize'):
+def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, method='langsize', min_methods=5):
     """Run full pipeline: preprocess → frequency filter → ensemble infer.
 
     Returns:
@@ -264,25 +264,50 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
     packages = _top_packages(cluster_fps, project_root)
 
     symbol_seqs = [[text for _, text, _ in seq] for seq in sequences]
-    result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore, method=method)
+
+    # Diversity threshold
+    n_methods = len(symbol_seqs)
     meta = {"files": cluster_fps, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
+    if n_methods < min_methods:
+        meta["skip_reason"] = "too_few_methods"
+        return [("(all methods)", None, len(sequences), meta)]
+    unique_seqs = len(set(tuple(s) for s in symbol_seqs))
+    unique_ratio = unique_seqs / n_methods
+    if unique_ratio > 0.9:
+        meta["skip_reason"] = "too_diverse"
+        return [("(all methods)", None, len(sequences), meta)]
+
+    result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore, method=method)
 
     return [("(all methods)", result, len(sequences), meta)]
 
 
-def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, method='langsize'):
+def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, method='langsize', min_methods=5):
     """Infer grammar for one package group. Module-level for ProcessPoolExecutor."""
     filtered = frequency_filter(group_seqs, min_coverage=0.2)
     imports = _extract_imports(group_files)
     arg_patterns = _build_arg_patterns(group_files)
     packages = _top_packages(group_files, project_root)
     symbol_seqs = [[text for _, text, _ in seq] for seq in filtered]
+
+    # Diversity threshold: skip if too few methods or too diverse
+    n_methods = len(symbol_seqs)
+    if n_methods < min_methods:
+        meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_few_methods"}
+        return (label, None, len(filtered), meta)
+
+    unique_seqs = len(set(tuple(s) for s in symbol_seqs))
+    unique_ratio = unique_seqs / n_methods
+    if unique_ratio > 0.9:
+        meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_diverse"}
+        return (label, None, len(filtered), meta)
+
     result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore)
     meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
     return (label, result, len(filtered), meta)
 
 
-def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, method='langsize'):
+def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, method='langsize', min_methods=5):
     """Preprocess and group by package directory, infer per group.
 
     Groups methods by their file's relative directory path, merging
@@ -317,7 +342,7 @@ def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFA
             gs = [sequences[i] for i in indices]
             gf = set(seq_files[i] for i in indices)
             f = ex.submit(_infer_group, label, gs, gf, project_root,
-                           min_coverage, prefer, kmax, N, include_kore, method)
+                           min_coverage, prefer, kmax, N, include_kore, method, min_methods)
             futures[f] = label
 
         for f in as_completed(futures):
@@ -427,6 +452,7 @@ def analyze_directory(
     main_only=False,
     include_kore=False,
     method='langsize',
+    min_methods=5,
 ):
     """Scan a directory and run analysis for each language found.
 
@@ -462,6 +488,7 @@ def analyze_directory(
                 kmax=kmax,
                 include_kore=include_kore,
                 method=method,
+                min_methods=min_methods,
             )
         else:
             results[ext] = analyze_clusters(
@@ -472,6 +499,7 @@ def analyze_directory(
                 kmax=kmax,
                 include_kore=include_kore,
                 method=method,
+                min_methods=min_methods,
             )
     return results
 
@@ -618,6 +646,10 @@ def _parse_args(argv=None):
         "--scoring-method", choices=["langsize", "mdl"], default="langsize",
         help="Scoring method: langsize (default, Bex et al.) or mdl (fallback)",
     )
+    parser.add_argument(
+        "--min-methods", type=int, default=5,
+        help="Minimum methods per group to infer grammar (default: 5). Groups with fewer are skipped.",
+    )
     return parser.parse_args(argv)
 
 
@@ -638,6 +670,7 @@ def main():
         main_only=args.main_only,
         include_kore=args.kore,
         method=args.scoring_method,
+        min_methods=args.min_methods,
     )
 
     if args.json_flag or args.format == "json":
@@ -654,7 +687,9 @@ def main():
                 print(f"     Grammar:   {best['grammar']}")
                 print(f"     Score:     {best['mdl_score']}")
             else:
-                print(f"  ╰─ {label} ({count} methods) — no grammar")
+                reason = meta.get("skip_reason", "")
+                suffix = f" — {reason}" if reason else ""
+                print(f"  ╰─ {label} ({count} methods) — no grammar{suffix}")
             imps = meta.get("imports", [])
             if imps:
                 joined = "  |  ".join(imps[:6])