tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feature/treesitter-tag-queries #2

Open
tobi wants to merge 78 commits from feature/treesitter-tag-queries into main
pull from: feature/treesitter-tag-queries
merge into: tobi:main
Conversation 0 Commits 78 Files changed 246 +17 -32
2 changed files with 17 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit ee60b62854 - Show all commits

21
bex/tag_preprocessor/analyze.py
View file

@ -20,7 +20,7 @@ import pathspec
from .code import preprocess_by_method, extract_arg_info, _summarize_arg_info from .code import preprocess_by_method, extract_arg_info, _summarize_arg_info
from bex.ensemble import infer_ensemble from bex.ensemble import infer_ensemble
from bex.gbnf import validate_sore, grammar_structure_score from bex.gbnf import validate_sore, grammar_structure_score, to_gbnf
from bex.distributional import distributional_split from bex.distributional import distributional_split
from bex.decompose import decompose_with_coverage, get_decomposition_stats from bex.decompose import decompose_with_coverage, get_decomposition_stats
@ -916,27 +916,12 @@ def _build_yaml_output(results, dir_path, max_mdl=200.0, min_structure=0.0):
parts = label.replace(os.sep, "/").split("/") parts = label.replace(os.sep, "/").split("/")
module = parts[0] if len(parts) > 1 else "(root)" module = parts[0] if len(parts) > 1 else "(root)"
# When recursive split produced leaf grammars, output each one # Output the best grammar per parent group (don't expand leaves)
all_grammars = meta.get("all_grammars", [])
if all_grammars:
for leaf_label, leaf_grammar, leaf_score, leaf_count in all_grammars:
if grammar_structure_score(leaf_grammar) < min_structure:
continue
entry = {
"package": leaf_label,
"methods": leaf_count,
"grammar": leaf_grammar,
"score": round(leaf_score, 3),
"algorithm": best["algorithm"],
"mdl": round(best["mdl_score"], 1),
}
modules.setdefault(module, []).append(entry)
total_patterns += 1
else:
entry = { entry = {
"package": label, "package": label,
"methods": count, "methods": count,
"grammar": best["grammar"], "grammar": best["grammar"],
"gbnf": to_gbnf(best["grammar"]),
"score": round(best.get("mdl_score", 0), 3), "score": round(best.get("mdl_score", 0), 3),
"algorithm": best["algorithm"], "algorithm": best["algorithm"],
"mdl": round(best["mdl_score"], 1), "mdl": round(best["mdl_score"], 1),

8
tests/test_grammar_index.py
View file

@ -153,10 +153,10 @@ class TestLoadGrammarIndex:
pytest.skip("RAGSAK grammars.yml not generated yet") pytest.skip("RAGSAK grammars.yml not generated yet")
idx = load_grammar_index("/home/tobi/Desktop/kesai/RAGSAK") idx = load_grammar_index("/home/tobi/Desktop/kesai/RAGSAK")
assert len(idx.get_all()) > 0 assert len(idx.get_all()) > 0
# Should be able to resolve a file in the storage package # Should be able to resolve a file in a package with a grammar
g = idx.get( g = idx.get(
"/home/tobi/Desktop/kesai/RAGSAK/infrastructure/adapters/search/src/main/kotlin/eu/corentic/springrag/service/storage/GcsStorage.kt", "/home/tobi/Desktop/kesai/RAGSAK/agents/capability/support/src/main/kotlin/eu/corentic/springrag/agent/capability/support/AgentCapabilityConfiguration.kt",
context_symbol="return", context_symbol="id",
) )
assert g is not None assert g is not None
assert "return" in g assert "id" in g