tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

tune: min_methods 5→3, fix DEFAULT_COVERAGE help text

Browse source 
min_methods=3 gains 9 extra FastAPI grammars (3-4 method groups with
clear sequential structure) without quality loss. No change for Flask
or RAGSAK (their small groups fail other gates anyway).

Multi-codebase validation (Round 13): tested Flask, FastAPI, httpx,
Pydantic, SQLAlchemy. Key finding: CRX works on codebases with small
focused sequential modules (FastAPI docs_src=22 grammars) but not on
large utility libraries (Flask=0, Pydantic=0, SQLAlchemy=1).
This commit is contained in:
tobjend 2026-07-12 03:54:35 +02:00
parent 11364151a6
commit 52c90bfdd8
1 changed files with 6 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
bex/tag_preprocessor/analyze.py
View file

@ -246,7 +246,7 @@ def _preprocess_files(file_paths):
return sequences, seq_files
def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, method='langsize', min_methods=5):
def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, method='langsize', min_methods=3):
"""Run full pipeline: preprocess → frequency filter → ensemble infer.
Returns:
@ -283,7 +283,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
return [("(all methods)", result, len(sequences), meta)]
def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=5, crx_method='standard', min_structure=0.0):
def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0):
"""Infer grammar for one package group. Module-level for ProcessPoolExecutor."""
filtered = frequency_filter(group_seqs, min_coverage=min_coverage)
imports = _extract_imports(group_files)
@ -329,7 +329,7 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre
return (label, result, len(filtered), meta)
def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=5, crx_method='standard', min_structure=0.0):
def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0):
"""Preprocess and group by package directory, infer per group.
Groups methods by their file's relative directory path, merging
@ -475,7 +475,7 @@ def analyze_directory(
include_kore=False,
include_idregex=False,
method='langsize',
min_methods=5,
min_methods=3,
crx_method='standard',
min_structure=0.0,
):
@ -679,8 +679,8 @@ def _parse_args(argv=None):
help="Scoring method: langsize (default, Bex et al.) or mdl (fallback)",
)
parser.add_argument(
"--min-methods", type=int, default=5,
help="Minimum methods per group to infer grammar (default: 5). Groups with fewer are skipped.",
"--min-methods", type=int, default=3,
help="Minimum methods per group to infer grammar (default: 3). Groups with fewer are skipped.",
)
parser.add_argument(
"--crx-method", choices=["standard", "refined"], default="standard",