tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

tune: min_methods 5→3, fix DEFAULT_COVERAGE help text

Browse source 
min_methods=3 gains 9 extra FastAPI grammars (3-4 method groups with
clear sequential structure) without quality loss. No change for Flask
or RAGSAK (their small groups fail other gates anyway).

Multi-codebase validation (Round 13): tested Flask, FastAPI, httpx,
Pydantic, SQLAlchemy. Key finding: CRX works on codebases with small
focused sequential modules (FastAPI docs_src=22 grammars) but not on
large utility libraries (Flask=0, Pydantic=0, SQLAlchemy=1).
This commit is contained in:
tobjend 2026-07-12 03:54:35 +02:00
parent 11364151a6
commit 52c90bfdd8
1 changed files with 6 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
bex/tag_preprocessor/analyze.py
View file

@ -246,7 +246,7 @@ def _preprocess_files(file_paths):
return sequences, seq_files return sequences, seq_files
def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, method='langsize', min_methods=5): def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, method='langsize', min_methods=3):
"""Run full pipeline: preprocess → frequency filter → ensemble infer. """Run full pipeline: preprocess → frequency filter → ensemble infer.
Returns: Returns:
@ -283,7 +283,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
return [("(all methods)", result, len(sequences), meta)] return [("(all methods)", result, len(sequences), meta)]
def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=5, crx_method='standard', min_structure=0.0): def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0):
"""Infer grammar for one package group. Module-level for ProcessPoolExecutor.""" """Infer grammar for one package group. Module-level for ProcessPoolExecutor."""
filtered = frequency_filter(group_seqs, min_coverage=min_coverage) filtered = frequency_filter(group_seqs, min_coverage=min_coverage)
imports = _extract_imports(group_files) imports = _extract_imports(group_files)
@ -329,7 +329,7 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre
return (label, result, len(filtered), meta) return (label, result, len(filtered), meta)
def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=5, crx_method='standard', min_structure=0.0): def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0):
"""Preprocess and group by package directory, infer per group. """Preprocess and group by package directory, infer per group.
Groups methods by their file's relative directory path, merging Groups methods by their file's relative directory path, merging
@ -475,7 +475,7 @@ def analyze_directory(
include_kore=False, include_kore=False,
include_idregex=False, include_idregex=False,
method='langsize', method='langsize',
min_methods=5, min_methods=3,
crx_method='standard', crx_method='standard',
min_structure=0.0, min_structure=0.0,
): ):
@ -679,8 +679,8 @@ def _parse_args(argv=None):
help="Scoring method: langsize (default, Bex et al.) or mdl (fallback)", help="Scoring method: langsize (default, Bex et al.) or mdl (fallback)",
) )
parser.add_argument( parser.add_argument(
"--min-methods", type=int, default=5, "--min-methods", type=int, default=3,
help="Minimum methods per group to infer grammar (default: 5). Groups with fewer are skipped.", help="Minimum methods per group to infer grammar (default: 3). Groups with fewer are skipped.",
) )
parser.add_argument( parser.add_argument(
"--crx-method", choices=["standard", "refined"], default="standard", "--crx-method", choices=["standard", "refined"], default="standard",