tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feat: add diversity threshold — skip groups with unique_ratio > 0.9 or methods < min_methods

Browse source 
Adds --min-methods CLI flag (default 5). Groups that are too diverse or
too small are skipped with skip_reason in meta. Prevents noisy/meaningless
grammars from diverse packages.
This commit is contained in:
tobjend 2026-07-11 23:11:54 +02:00
parent dfb56a083a
commit ab20b03256
1 changed files with 41 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
bex/tag_preprocessor/analyze.py
View file

@ -245,7 +245,7 @@ def _preprocess_files(file_paths):
return sequences, seq_files
def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, method='langsize'):
def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, method='langsize', min_methods=5):
"""Run full pipeline: preprocess → frequency filter → ensemble infer.
Returns:
@ -264,25 +264,50 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
packages = _top_packages(cluster_fps, project_root)
symbol_seqs = [[text for _, text, _ in seq] for seq in sequences]
result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore, method=method)
# Diversity threshold
n_methods = len(symbol_seqs)
meta = {"files": cluster_fps, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
if n_methods < min_methods:
meta["skip_reason"] = "too_few_methods"
return [("(all methods)", None, len(sequences), meta)]
unique_seqs = len(set(tuple(s) for s in symbol_seqs))
unique_ratio = unique_seqs / n_methods
if unique_ratio > 0.9:
meta["skip_reason"] = "too_diverse"
return [("(all methods)", None, len(sequences), meta)]
result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore, method=method)
return [("(all methods)", result, len(sequences), meta)]
def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, method='langsize'):
def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, method='langsize', min_methods=5):
"""Infer grammar for one package group. Module-level for ProcessPoolExecutor."""
filtered = frequency_filter(group_seqs, min_coverage=0.2)
imports = _extract_imports(group_files)
arg_patterns = _build_arg_patterns(group_files)
packages = _top_packages(group_files, project_root)
symbol_seqs = [[text for _, text, _ in seq] for seq in filtered]
# Diversity threshold: skip if too few methods or too diverse
n_methods = len(symbol_seqs)
if n_methods < min_methods:
meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_few_methods"}
return (label, None, len(filtered), meta)
unique_seqs = len(set(tuple(s) for s in symbol_seqs))
unique_ratio = unique_seqs / n_methods
if unique_ratio > 0.9:
meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_diverse"}
return (label, None, len(filtered), meta)
result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore)
meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
return (label, result, len(filtered), meta)
def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, method='langsize'):
def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, method='langsize', min_methods=5):
"""Preprocess and group by package directory, infer per group.
Groups methods by their file's relative directory path, merging
@ -317,7 +342,7 @@ def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFA
gs = [sequences[i] for i in indices]
gf = set(seq_files[i] for i in indices)
f = ex.submit(_infer_group, label, gs, gf, project_root,
min_coverage, prefer, kmax, N, include_kore, method)
min_coverage, prefer, kmax, N, include_kore, method, min_methods)
futures[f] = label
for f in as_completed(futures):
@ -427,6 +452,7 @@ def analyze_directory(
main_only=False,
include_kore=False,
method='langsize',
min_methods=5,
):
"""Scan a directory and run analysis for each language found.
@ -462,6 +488,7 @@ def analyze_directory(
kmax=kmax,
include_kore=include_kore,
method=method,
min_methods=min_methods,
)
else:
results[ext] = analyze_clusters(
@ -472,6 +499,7 @@ def analyze_directory(
kmax=kmax,
include_kore=include_kore,
method=method,
min_methods=min_methods,
)
return results
@ -618,6 +646,10 @@ def _parse_args(argv=None):
"--scoring-method", choices=["langsize", "mdl"], default="langsize",
help="Scoring method: langsize (default, Bex et al.) or mdl (fallback)",
)
parser.add_argument(
"--min-methods", type=int, default=5,
help="Minimum methods per group to infer grammar (default: 5). Groups with fewer are skipped.",
)
return parser.parse_args(argv)
@ -638,6 +670,7 @@ def main():
main_only=args.main_only,
include_kore=args.kore,
method=args.scoring_method,
min_methods=args.min_methods,
)
if args.json_flag or args.format == "json":
@ -654,7 +687,9 @@ def main():
print(f" Grammar: {best['grammar']}")
print(f" Score: {best['mdl_score']}")
else:
print(f" ╰─ {label} ({count} methods) — no grammar")
reason = meta.get("skip_reason", "")
suffix = f" — {reason}" if reason else ""
print(f" ╰─ {label} ({count} methods) — no grammar{suffix}")
imps = meta.get("imports", [])
if imps:
joined = " | ".join(imps[:6])