feat: loosen filtering thresholds to keep more grammars
- min_methods: 3→2 (keep groups with 2+ methods) - unique_ratio: 0.9→0.95 (keep groups with up to 95% unique sequences) - max_mdl: 200→500 (keep higher-MDL grammars) This prevents throwing away good grammars that have genuine structure but happen to have many unique sequences or moderate MDL scores.
This commit is contained in:
parent
36f9e9173d
commit
9b3dca46eb
1 changed files with 5 additions and 5 deletions
|
|
@ -280,7 +280,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
|
|||
return [("(all methods)", None, len(sequences), meta)]
|
||||
unique_seqs = len(set(tuple(s) for s in symbol_seqs))
|
||||
unique_ratio = unique_seqs / n_methods
|
||||
if unique_ratio > 0.9:
|
||||
if unique_ratio > 0.95:
|
||||
meta["skip_reason"] = "too_diverse"
|
||||
return [("(all methods)", None, len(sequences), meta)]
|
||||
|
||||
|
|
@ -437,7 +437,7 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre
|
|||
unique_seqs = len(set(tuple(s) for s in symbol_seqs))
|
||||
unique_ratio = unique_seqs / n_methods
|
||||
# Skip diversity check when decomposing (decomposition creates diverse fragments)
|
||||
if unique_ratio > 0.9 and not decompose:
|
||||
if unique_ratio > 0.95 and not decompose:
|
||||
meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_diverse"}
|
||||
return (label, None, len(filtered), meta)
|
||||
|
||||
|
|
@ -902,7 +902,7 @@ def _build_json_output(results):
|
|||
return json.dumps(output, indent=2)
|
||||
|
||||
|
||||
def _build_yaml_output(results, dir_path, max_mdl=200.0, min_structure=0.0):
|
||||
def _build_yaml_output(results, dir_path, max_mdl=500.0, min_structure=0.0):
|
||||
"""Build YAML output grouped by top-level module, sorted by MDL.
|
||||
|
||||
Filters out (other), no-grammar groups, groups above max_mdl,
|
||||
|
|
@ -1038,8 +1038,8 @@ def _parse_args(argv=None):
|
|||
help="Scoring method: langsize (default, Bex et al.) or mdl (fallback)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-methods", type=int, default=3,
|
||||
help="Minimum methods per group to infer grammar (default: 3). Groups with fewer are skipped.",
|
||||
"--min-methods", type=int, default=2,
|
||||
help="Minimum methods per group to infer grammar (default: 2). Groups with fewer are skipped.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--crx-method", choices=["standard", "refined"], default="standard",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue