tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feature/treesitter-tag-queries #2

Open
tobi wants to merge 78 commits from feature/treesitter-tag-queries into main
pull from: feature/treesitter-tag-queries
merge into: tobi:main
Conversation 0 Commits 78 Files changed 246 +6 -6
1 changed files with 6 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 52c90bfdd8 - Show all commits

12
bex/tag_preprocessor/analyze.py
View file

@ -246,7 +246,7 @@ def _preprocess_files(file_paths):
return sequences, seq_files
def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, method='langsize', min_methods=5):
def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, method='langsize', min_methods=3):
"""Run full pipeline: preprocess → frequency filter → ensemble infer.
Returns:
@ -283,7 +283,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
return [("(all methods)", result, len(sequences), meta)]
def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=5, crx_method='standard', min_structure=0.0):
def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0):
"""Infer grammar for one package group. Module-level for ProcessPoolExecutor."""
filtered = frequency_filter(group_seqs, min_coverage=min_coverage)
imports = _extract_imports(group_files)
@ -329,7 +329,7 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre
return (label, result, len(filtered), meta)
def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=5, crx_method='standard', min_structure=0.0):
def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0):
"""Preprocess and group by package directory, infer per group.
Groups methods by their file's relative directory path, merging
@ -475,7 +475,7 @@ def analyze_directory(
include_kore=False,
include_idregex=False,
method='langsize',
min_methods=5,
min_methods=3,
crx_method='standard',
min_structure=0.0,
):
@ -679,8 +679,8 @@ def _parse_args(argv=None):
help="Scoring method: langsize (default, Bex et al.) or mdl (fallback)",
)
parser.add_argument(
"--min-methods", type=int, default=5,
help="Minimum methods per group to infer grammar (default: 5). Groups with fewer are skipped.",
"--min-methods", type=int, default=3,
help="Minimum methods per group to infer grammar (default: 3). Groups with fewer are skipped.",
)
parser.add_argument(
"--crx-method", choices=["standard", "refined"], default="standard",