From 9b3dca46eb2a45686bca2d483230ea18b8dbc16d Mon Sep 17 00:00:00 2001 From: tobjend Date: Mon, 13 Jul 2026 02:15:45 +0200 Subject: [PATCH] feat: loosen filtering thresholds to keep more grammars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - min_methods: 3→2 (keep groups with 2+ methods) - unique_ratio: 0.9→0.95 (keep groups with up to 95% unique sequences) - max_mdl: 200→500 (keep higher-MDL grammars) This prevents throwing away good grammars that have genuine structure but happen to have many unique sequences or moderate MDL scores. --- bex/tag_preprocessor/analyze.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bex/tag_preprocessor/analyze.py b/bex/tag_preprocessor/analyze.py index 27a6137..d35692e 100644 --- a/bex/tag_preprocessor/analyze.py +++ b/bex/tag_preprocessor/analyze.py @@ -280,7 +280,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL return [("(all methods)", None, len(sequences), meta)] unique_seqs = len(set(tuple(s) for s in symbol_seqs)) unique_ratio = unique_seqs / n_methods - if unique_ratio > 0.9: + if unique_ratio > 0.95: meta["skip_reason"] = "too_diverse" return [("(all methods)", None, len(sequences), meta)] @@ -437,7 +437,7 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre unique_seqs = len(set(tuple(s) for s in symbol_seqs)) unique_ratio = unique_seqs / n_methods # Skip diversity check when decomposing (decomposition creates diverse fragments) - if unique_ratio > 0.9 and not decompose: + if unique_ratio > 0.95 and not decompose: meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_diverse"} return (label, None, len(filtered), meta) @@ -902,7 +902,7 @@ def _build_json_output(results): return json.dumps(output, indent=2) -def _build_yaml_output(results, dir_path, max_mdl=200.0, min_structure=0.0): +def _build_yaml_output(results, dir_path, max_mdl=500.0, min_structure=0.0): """Build YAML output grouped by top-level module, sorted by MDL. Filters out (other), no-grammar groups, groups above max_mdl, @@ -1038,8 +1038,8 @@ def _parse_args(argv=None): help="Scoring method: langsize (default, Bex et al.) or mdl (fallback)", ) parser.add_argument( - "--min-methods", type=int, default=3, - help="Minimum methods per group to infer grammar (default: 3). Groups with fewer are skipped.", + "--min-methods", type=int, default=2, + help="Minimum methods per group to infer grammar (default: 2). Groups with fewer are skipped.", ) parser.add_argument( "--crx-method", choices=["standard", "refined"], default="standard",