tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feature/treesitter-tag-queries #2

Open
tobi wants to merge 78 commits from feature/treesitter-tag-queries into main
pull from: feature/treesitter-tag-queries
merge into: tobi:main
Conversation 0 Commits 78 Files changed 246 +23 -7
1 changed files with 23 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 6912841b9e - Show all commits

30
bex/tag_preprocessor/analyze.py
View file

@ -256,7 +256,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
if not sequences: if not sequences:
return [] return []
sequences = frequency_filter(sequences, min_coverage=0.2) sequences = frequency_filter(sequences, min_coverage=min_coverage)
cluster_fps = set(seq_files) cluster_fps = set(seq_files)
imports = _extract_imports(cluster_fps) imports = _extract_imports(cluster_fps)
@ -282,9 +282,9 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
return [("(all methods)", result, len(sequences), meta)] return [("(all methods)", result, len(sequences), meta)]
def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, method='langsize', min_methods=5): def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, method='langsize', min_methods=5, crx_method='standard'):
"""Infer grammar for one package group. Module-level for ProcessPoolExecutor.""" """Infer grammar for one package group. Module-level for ProcessPoolExecutor."""
filtered = frequency_filter(group_seqs, min_coverage=0.2) filtered = frequency_filter(group_seqs, min_coverage=min_coverage)
imports = _extract_imports(group_files) imports = _extract_imports(group_files)
arg_patterns = _build_arg_patterns(group_files) arg_patterns = _build_arg_patterns(group_files)
packages = _top_packages(group_files, project_root) packages = _top_packages(group_files, project_root)
@ -302,12 +302,21 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre
meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_diverse"} meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "too_diverse"}
return (label, None, len(filtered), meta) return (label, None, len(filtered), meta)
result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore) if crx_method == 'refined':
from ..crx_refined import crx_with_confidence
info = crx_with_confidence(symbol_seqs)
result = {
'best': {'algorithm': 'CRX-refined', 'grammar': info['grammar'], 'mdl_score': info['confidence']},
'all': [{'algorithm': 'CRX-refined', 'grammar': info['grammar'], 'mdl_score': info['confidence']}],
'why': f"CRX-refined (confidence={info['confidence']:.2f})",
}
else:
result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore)
meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages} meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
return (label, result, len(filtered), meta) return (label, result, len(filtered), meta)
def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, method='langsize', min_methods=5): def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, method='langsize', min_methods=5, crx_method='standard'):
"""Preprocess and group by package directory, infer per group. """Preprocess and group by package directory, infer per group.
Groups methods by their file's relative directory path, merging Groups methods by their file's relative directory path, merging
@ -342,7 +351,7 @@ def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFA
gs = [sequences[i] for i in indices] gs = [sequences[i] for i in indices]
gf = set(seq_files[i] for i in indices) gf = set(seq_files[i] for i in indices)
f = ex.submit(_infer_group, label, gs, gf, project_root, f = ex.submit(_infer_group, label, gs, gf, project_root,
min_coverage, prefer, kmax, N, include_kore, method, min_methods) min_coverage, prefer, kmax, N, include_kore, method, min_methods, crx_method)
futures[f] = label futures[f] = label
for f in as_completed(futures): for f in as_completed(futures):
@ -375,7 +384,7 @@ def infer(file_paths, extension, min_coverage=DEFAULT_COVERAGE, prefer=None, kma
Ensemble result dict from infer_ensemble. Ensemble result dict from infer_ensemble.
""" """
sequences, _ = _preprocess_files(file_paths) sequences, _ = _preprocess_files(file_paths)
sequences = frequency_filter(sequences, min_coverage=0.2) sequences = frequency_filter(sequences, min_coverage=min_coverage)
symbol_seqs = [[text for _, text, _ in seq] for seq in sequences] symbol_seqs = [[text for _, text, _ in seq] for seq in sequences]
@ -453,6 +462,7 @@ def analyze_directory(
include_kore=False, include_kore=False,
method='langsize', method='langsize',
min_methods=5, min_methods=5,
crx_method='standard',
): ):
"""Scan a directory and run analysis for each language found. """Scan a directory and run analysis for each language found.
@ -489,6 +499,7 @@ def analyze_directory(
include_kore=include_kore, include_kore=include_kore,
method=method, method=method,
min_methods=min_methods, min_methods=min_methods,
crx_method=crx_method,
) )
else: else:
results[ext] = analyze_clusters( results[ext] = analyze_clusters(
@ -650,6 +661,10 @@ def _parse_args(argv=None):
"--min-methods", type=int, default=5, "--min-methods", type=int, default=5,
help="Minimum methods per group to infer grammar (default: 5). Groups with fewer are skipped.", help="Minimum methods per group to infer grammar (default: 5). Groups with fewer are skipped.",
) )
parser.add_argument(
"--crx-method", choices=["standard", "refined"], default="standard",
help="CRX method: standard (default) or refined (cluster-then-infer, tighter grammars)",
)
return parser.parse_args(argv) return parser.parse_args(argv)
@ -671,6 +686,7 @@ def main():
include_kore=args.kore, include_kore=args.kore,
method=args.scoring_method, method=args.scoring_method,
min_methods=args.min_methods, min_methods=args.min_methods,
crx_method=args.crx_method,
) )
if args.json_flag or args.format == "json": if args.json_flag or args.format == "json":