feat: loosen filtering thresholds to keep more grammars
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
ci/woodpecker/pr/woodpecker Pipeline failed

- min_methods: 3→2 (keep groups with 2+ methods)
- unique_ratio: 0.9→0.95 (keep groups with up to 95% unique sequences)
- max_mdl: 200→500 (keep higher-MDL grammars)

This prevents throwing away good grammars that have genuine structure
but happen to have many unique sequences or moderate MDL scores.
This commit is contained in:
tobjend 2026-07-13 02:15:45 +02:00
parent 36f9e9173d
commit 9b3dca46eb

View file

@ -280,7 +280,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
return [("(all methods)", None, len(sequences), meta)]
unique_seqs = len(set(tuple(s) for s in symbol_seqs))
unique_ratio = unique_seqs / n_methods
if unique_ratio > 0.9:
if unique_ratio > 0.95:
meta["skip_reason"] = "too_diverse"
return [("(all methods)", None, len(sequences), meta)]
@ -437,7 +437,7 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre
unique_seqs = len(set(tuple(s) for s in symbol_seqs))
unique_ratio = unique_seqs / n_methods
# Skip diversity check when decomposing (decomposition creates diverse fragments)
if unique_ratio > 0.9 and not decompose:
if unique_ratio > 0.95 and not decompose:
meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_diverse"}
return (label, None, len(filtered), meta)
@ -902,7 +902,7 @@ def _build_json_output(results):
return json.dumps(output, indent=2)
def _build_yaml_output(results, dir_path, max_mdl=200.0, min_structure=0.0):
def _build_yaml_output(results, dir_path, max_mdl=500.0, min_structure=0.0):
"""Build YAML output grouped by top-level module, sorted by MDL.
Filters out (other), no-grammar groups, groups above max_mdl,
@ -1038,8 +1038,8 @@ def _parse_args(argv=None):
help="Scoring method: langsize (default, Bex et al.) or mdl (fallback)",
)
parser.add_argument(
"--min-methods", type=int, default=3,
help="Minimum methods per group to infer grammar (default: 3). Groups with fewer are skipped.",
"--min-methods", type=int, default=2,
help="Minimum methods per group to infer grammar (default: 2). Groups with fewer are skipped.",
)
parser.add_argument(
"--crx-method", choices=["standard", "refined"], default="standard",