tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feature/treesitter-tag-queries #2

Open
tobi wants to merge 78 commits from feature/treesitter-tag-queries into main
pull from: feature/treesitter-tag-queries
merge into: tobi:main
Conversation 0 Commits 78 Files changed 246 +6 -6
1 changed files with 6 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 52c90bfdd8 - Show all commits

12
bex/tag_preprocessor/analyze.py
View file

@ -246,7 +246,7 @@ def _preprocess_files(file_paths):
return sequences, seq_files return sequences, seq_files
def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, method='langsize', min_methods=5): def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, method='langsize', min_methods=3):
"""Run full pipeline: preprocess → frequency filter → ensemble infer. """Run full pipeline: preprocess → frequency filter → ensemble infer.
Returns: Returns:
@ -283,7 +283,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
return [("(all methods)", result, len(sequences), meta)] return [("(all methods)", result, len(sequences), meta)]
def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=5, crx_method='standard', min_structure=0.0): def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0):
"""Infer grammar for one package group. Module-level for ProcessPoolExecutor.""" """Infer grammar for one package group. Module-level for ProcessPoolExecutor."""
filtered = frequency_filter(group_seqs, min_coverage=min_coverage) filtered = frequency_filter(group_seqs, min_coverage=min_coverage)
imports = _extract_imports(group_files) imports = _extract_imports(group_files)
@ -329,7 +329,7 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre
return (label, result, len(filtered), meta) return (label, result, len(filtered), meta)
def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=5, crx_method='standard', min_structure=0.0): def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0):
"""Preprocess and group by package directory, infer per group. """Preprocess and group by package directory, infer per group.
Groups methods by their file's relative directory path, merging Groups methods by their file's relative directory path, merging
@ -475,7 +475,7 @@ def analyze_directory(
include_kore=False, include_kore=False,
include_idregex=False, include_idregex=False,
method='langsize', method='langsize',
min_methods=5, min_methods=3,
crx_method='standard', crx_method='standard',
min_structure=0.0, min_structure=0.0,
): ):
@ -679,8 +679,8 @@ def _parse_args(argv=None):
help="Scoring method: langsize (default, Bex et al.) or mdl (fallback)", help="Scoring method: langsize (default, Bex et al.) or mdl (fallback)",
) )
parser.add_argument( parser.add_argument(
"--min-methods", type=int, default=5, "--min-methods", type=int, default=3,
help="Minimum methods per group to infer grammar (default: 5). Groups with fewer are skipped.", help="Minimum methods per group to infer grammar (default: 3). Groups with fewer are skipped.",
) )
parser.add_argument( parser.add_argument(
"--crx-method", choices=["standard", "refined"], default="standard", "--crx-method", choices=["standard", "refined"], default="standard",