tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

chore: update experiment scripts for AST representation

Browse source
This commit is contained in:
tobjend 2026-07-12 23:46:25 +02:00
parent 19a1db48ef
commit f8dda557d2
4 changed files with 50 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
experiments/coarsen_eval.py
View file

@ -17,6 +17,8 @@ from bex.tag_preprocessor.code import (
) )
from bex.twotinf import build_soa from bex.twotinf import build_soa
from bex.rwr0 import rwr0 from bex.rwr0 import rwr0
from bex.grammar import Empty
from bex.gbnf import to_gbnf
RESULTS_DIR = Path(__file__).parent / "results" RESULTS_DIR = Path(__file__).parent / "results"
@ -80,8 +82,8 @@ def infer_sore(seqs):
return None return None
try: try:
soa = build_soa(clean) soa = build_soa(clean)
sore = rwr0(soa) grammar = rwr0(soa)
return sore if sore != "∅" else None return grammar if not isinstance(grammar, Empty) else None
except Exception: except Exception:
return None return None
@ -103,8 +105,8 @@ def measure_group(ctx, items, label):
"unique": unique, "unique": unique,
"unique_ratio": round(unique_ratio, 3), "unique_ratio": round(unique_ratio, 3),
"alphabet_size": len(alphabet), "alphabet_size": len(alphabet),
"sore": sore[:200] if sore else None, "grammar": to_gbnf(sore)[:200] if sore else None,
"sore_success": sore is not None, "grammar_success": sore is not None,
} }
@ -136,7 +138,7 @@ def run_comparison(name, raw_seqs, coarse_seqs, packages, k_values=(1, 2, 3)):
# Measure all groups # Measure all groups
group_results = [] group_results = []
sore_successes = 0 grammar_successes = 0
methods_in_good = 0 methods_in_good = 0
total_methods = 0 total_methods = 0
@ -144,8 +146,8 @@ def run_comparison(name, raw_seqs, coarse_seqs, packages, k_values=(1, 2, 3)):
m = measure_group(ctx, items, f"{label}_k{k}") m = measure_group(ctx, items, f"{label}_k{k}")
group_results.append(m) group_results.append(m)
total_methods += m["methods"] total_methods += m["methods"]
if m["sore_success"]: if m["grammar_success"]:
sore_successes += 1 grammar_successes += 1
methods_in_good += m["methods"] methods_in_good += m["methods"]
# Cross-package analysis # Cross-package analysis
@ -153,7 +155,7 @@ def run_comparison(name, raw_seqs, coarse_seqs, packages, k_values=(1, 2, 3)):
results[label][f"k{k}"] = { results[label][f"k{k}"] = {
"contexts": len(groups), "contexts": len(groups),
"sore_successes": sore_successes, "grammar_successes": grammar_successes,
"total_methods": total_methods, "total_methods": total_methods,
"methods_in_good": methods_in_good, "methods_in_good": methods_in_good,
"coverage": round(methods_in_good / total_methods * 100, 1) if total_methods else 0, "coverage": round(methods_in_good / total_methods * 100, 1) if total_methods else 0,
@ -189,7 +191,7 @@ def print_results(results):
print(f"\n --- k={k} ---") print(f"\n --- k={k} ---")
print(f" {'':20s} {'Raw':>10s} {'Coarsened':>10s}") print(f" {'':20s} {'Raw':>10s} {'Coarsened':>10s}")
print(f" {'Contexts':20s} {rk.get('contexts',0):10d} {ck.get('contexts',0):10d}") print(f" {'Contexts':20s} {rk.get('contexts',0):10d} {ck.get('contexts',0):10d}")
print(f" {'SORE successes':20s} {rk.get('sore_successes',0):10d} {ck.get('sore_successes',0):10d}") print(f" {'Grammar successes':20s} {rk.get('grammar_successes',0):10d} {ck.get('grammar_successes',0):10d}")
print(f" {'Coverage':20s} {rk.get('coverage',0):9.1f}% {ck.get('coverage',0):9.1f}%") print(f" {'Coverage':20s} {rk.get('coverage',0):9.1f}% {ck.get('coverage',0):9.1f}%")
print(f" {'Cross-pkg contexts':20s} {rk.get('cross_package_contexts',0):10d} {ck.get('cross_package_contexts',0):10d}") print(f" {'Cross-pkg contexts':20s} {rk.get('cross_package_contexts',0):10d} {ck.get('cross_package_contexts',0):10d}")

32
experiments/context_eval.py
View file

@ -15,6 +15,8 @@ from bex.tag_preprocessor.analyze import (
from bex.twotinf import build_soa from bex.twotinf import build_soa
from bex.rwr0 import rwr0 from bex.rwr0 import rwr0
from bex.reduce import reduce_contexts, soa_distance, build_soa_with_support from bex.reduce import reduce_contexts, soa_distance, build_soa_with_support
from bex.grammar import Empty
from bex.gbnf import to_gbnf
RESULTS_DIR = Path(__file__).parent / "results" RESULTS_DIR = Path(__file__).parent / "results"
@ -115,8 +117,8 @@ def evaluate_context(contexts, label, min_methods=3, max_methods=50,
"meaningful_contexts": 0, "meaningful_contexts": 0,
"total_methods": 0, "total_methods": 0,
"methods_in_good_groups": 0, "methods_in_good_groups": 0,
"sore_successes": 0, "grammar_successes": 0,
"sore_failures": 0, "grammar_failures": 0,
"skip_reasons": {}, "skip_reasons": {},
"groups": [], "groups": [],
} }
@ -136,7 +138,7 @@ def evaluate_context(contexts, label, min_methods=3, max_methods=50,
results["groups"].append({ results["groups"].append({
"context": str(ctx), "methods": n, "unique": unique, "context": str(ctx), "methods": n, "unique": unique,
"unique_ratio": round(unique_ratio, 3), "unique_ratio": round(unique_ratio, 3),
"sore": "SKIP(too_large)", "sore_success": False, "grammar": "SKIP(too_large)", "grammar_success": False,
}) })
continue continue
@ -145,7 +147,7 @@ def evaluate_context(contexts, label, min_methods=3, max_methods=50,
results["groups"].append({ results["groups"].append({
"context": str(ctx), "methods": n, "unique": unique, "context": str(ctx), "methods": n, "unique": unique,
"unique_ratio": round(unique_ratio, 3), "unique_ratio": round(unique_ratio, 3),
"sore": "SKIP(too_diverse)", "sore_success": False, "grammar": "SKIP(too_diverse)", "grammar_success": False,
}) })
continue continue
@ -157,7 +159,7 @@ def evaluate_context(contexts, label, min_methods=3, max_methods=50,
results["groups"].append({ results["groups"].append({
"context": str(ctx), "methods": n, "unique": unique, "context": str(ctx), "methods": n, "unique": unique,
"unique_ratio": round(unique_ratio, 3), "unique_ratio": round(unique_ratio, 3),
"sore": "SKIP(large_alphabet)", "sore_success": False, "grammar": "SKIP(large_alphabet)", "grammar_success": False,
}) })
continue continue
@ -179,26 +181,26 @@ def evaluate_context(contexts, label, min_methods=3, max_methods=50,
results["groups"].append({ results["groups"].append({
"context": str(ctx), "methods": n, "unique": unique, "context": str(ctx), "methods": n, "unique": unique,
"unique_ratio": round(unique_ratio, 3), "unique_ratio": round(unique_ratio, 3),
"sore": "SKIP(complex_soa)", "sore_success": False, "grammar": "SKIP(complex_soa)", "grammar_success": False,
}) })
continue continue
sore = rwr0(soa) grammar = rwr0(soa)
group_info = { group_info = {
"context": str(ctx), "context": str(ctx),
"methods": n, "methods": n,
"unique": unique, "unique": unique,
"unique_ratio": round(unique_ratio, 3), "unique_ratio": round(unique_ratio, 3),
"sore": sore[:200] if sore not in ("∅",) else sore, "grammar": to_gbnf(grammar)[:200] if not isinstance(grammar, Empty) else "∅",
"sore_success": sore != "∅", "grammar_success": not isinstance(grammar, Empty),
} }
if sore != "∅": if not isinstance(grammar, Empty):
results["sore_successes"] += 1 results["grammar_successes"] += 1
results["methods_in_good_groups"] += n results["methods_in_good_groups"] += n
else: else:
results["sore_failures"] += 1 results["grammar_failures"] += 1
results["groups"].append(group_info) results["groups"].append(group_info)
@ -222,7 +224,7 @@ def run_experiment(name, contexts, label, codebase_name, merge_info=None):
results["merge_info"] = merge_info results["merge_info"] = merge_info
print(f" Contexts: {results['total_contexts']} Meaningful: {results['meaningful_contexts']} " print(f" Contexts: {results['total_contexts']} Meaningful: {results['meaningful_contexts']} "
f"SOREs: {results['sore_successes']} Coverage: {results['coverage']}% " f"Grammars: {results['grammar_successes']} Coverage: {results['coverage']}% "
f"Time: {elapsed:.2f}s") f"Time: {elapsed:.2f}s")
if merge_info: if merge_info:
print(f" Merges: {merge_info['merges']} Threshold: {merge_info['threshold']}") print(f" Merges: {merge_info['merges']} Threshold: {merge_info['threshold']}")
@ -306,14 +308,14 @@ def run_codebase(codebase_key):
print(f"\n{'=' * 70}") print(f"\n{'=' * 70}")
print(f" {name} SUMMARY") print(f" {name} SUMMARY")
print(f"{'=' * 70}") print(f"{'=' * 70}")
print(f"{'Strategy':<42} {'Ctx':>5} {'SORE':>5} {'Cov%':>6} {'Merge':>5}") print(f"{'Strategy':<42} {'Ctx':>5} {'Grms':>5} {'Cov%':>6} {'Merge':>5}")
print("-" * 70) print("-" * 70)
for r in all_results: for r in all_results:
merge_str = "" merge_str = ""
if r.get("merge_info"): if r.get("merge_info"):
merge_str = str(r["merge_info"]["merges"]) merge_str = str(r["merge_info"]["merges"])
print(f"{r['strategy']:<42} {r['meaningful_contexts']:>5} " print(f"{r['strategy']:<42} {r['meaningful_contexts']:>5} "
f"{r['sore_successes']:>5} {r['coverage']:>5.1f}% {merge_str:>5}") f"{r['grammar_successes']:>5} {r['coverage']:>5.1f}% {merge_str:>5}")
summary = [ summary = [
{k: v for k, v in r.items() if k != "groups"} {k: v for k, v in r.items() if k != "groups"}

21
experiments/freq_eval.py
View file

@ -15,6 +15,7 @@ from bex.tag_preprocessor.analyze import (
from bex.tag_preprocessor.code import _extract_call_tokens, _extract_coarsened_tokens from bex.tag_preprocessor.code import _extract_call_tokens, _extract_coarsened_tokens
from bex.twotinf import build_soa from bex.twotinf import build_soa
from bex.rwr0 import rwr0 from bex.rwr0 import rwr0
from bex.grammar import Empty
RESULTS_DIR = Path(__file__).parent / "results" RESULTS_DIR = Path(__file__).parent / "results"
@ -67,7 +68,7 @@ def measure_at_threshold(raw_seqs, packages, threshold):
ctx = tuple(seq[:3]) ctx = tuple(seq[:3])
contexts[ctx].append(seq) contexts[ctx].append(seq)
sore_successes = 0 grammar_successes = 0
methods_in_good = 0 methods_in_good = 0
total_methods = 0 total_methods = 0
top_patterns = [] top_patterns = []
@ -87,15 +88,17 @@ def measure_at_threshold(raw_seqs, packages, threshold):
continue continue
try: try:
soa = build_soa(seqs) soa = build_soa(seqs)
sore = rwr0(soa) grammar = rwr0(soa)
if sore != "∅": if not isinstance(grammar, Empty):
sore_successes += 1 grammar_successes += 1
methods_in_good += n methods_in_good += n
from bex.gbnf import to_gbnf
gbnf_str = to_gbnf(grammar)
top_patterns.append({ top_patterns.append({
"context": str(ctx), "context": str(ctx),
"methods": n, "methods": n,
"unique": unique, "unique": unique,
"sore": sore[:150], "grammar": gbnf_str[:150],
}) })
except Exception: except Exception:
pass pass
@ -107,7 +110,7 @@ def measure_at_threshold(raw_seqs, packages, threshold):
"seqs_total": total, "seqs_total": total,
"seqs_surviving": surviving, "seqs_surviving": surviving,
"contexts": len(contexts), "contexts": len(contexts),
"sore_successes": sore_successes, "grammar_successes": grammar_successes,
"total_methods": total_methods, "total_methods": total_methods,
"methods_in_good": methods_in_good, "methods_in_good": methods_in_good,
"coverage": round(methods_in_good / total_methods * 100, 1) if total_methods else 0, "coverage": round(methods_in_good / total_methods * 100, 1) if total_methods else 0,
@ -132,7 +135,7 @@ def run_codebase(key):
r["elapsed"] = round(elapsed, 2) r["elapsed"] = round(elapsed, 2)
results.append(r) results.append(r)
print(f" thresh={thresh:.2f} syms={r['symbols_kept']:4d} seqs={r['seqs_surviving']:5d} " print(f" thresh={thresh:.2f} syms={r['symbols_kept']:4d} seqs={r['seqs_surviving']:5d} "
f"ctxs={r['contexts']:4d} SOREs={r['sore_successes']:3d} cov={r['coverage']:5.1f}% " f"ctxs={r['contexts']:4d} grms={r['grammar_successes']:3d} cov={r['coverage']:5.1f}% "
f"({elapsed:.1f}s)") f"({elapsed:.1f}s)")
out_path = RESULTS_DIR / f"freq_{key}.json" out_path = RESULTS_DIR / f"freq_{key}.json"
@ -152,13 +155,13 @@ def main():
print(f"\n{'=' * 80}") print(f"\n{'=' * 80}")
print(" SUMMARY") print(" SUMMARY")
print(f"{'=' * 80}") print(f"{'=' * 80}")
print(f" {'Codebase':20s} {'Thresh':7s} {'Syms':5s} {'Seqs':6s} {'SOREs':6s} {'Cov':7s}") print(f" {'Codebase':20s} {'Thresh':7s} {'Syms':5s} {'Seqs':6s} {'Grms':6s} {'Cov':7s}")
print(f" {'-'*55}") print(f" {'-'*55}")
for key, results in all_results.items(): for key, results in all_results.items():
name = CODEBASES[key]["name"] name = CODEBASES[key]["name"]
for r in results: for r in results:
print(f" {name:20s} {r['threshold']:7.2f} {r['symbols_kept']:5d} " print(f" {name:20s} {r['threshold']:7.2f} {r['symbols_kept']:5d} "
f"{r['seqs_surviving']:6d} {r['sore_successes']:6d} {r['coverage']:6.1f}%") f"{r['seqs_surviving']:6d} {r['grammar_successes']:6d} {r['coverage']:6.1f}%")
return all_results return all_results

25
experiments/gbnf_eval.py
View file

@ -6,6 +6,7 @@ import time
from pathlib import Path from pathlib import Path
from bex.tag_preprocessor.analyze import analyze_directory from bex.tag_preprocessor.analyze import analyze_directory
from bex.gbnf import to_gbnf from bex.gbnf import to_gbnf
from bex.grammar import Empty
def run(codebase_name, dir_path): def run(codebase_name, dir_path):
print(f"\n{'='*60}") print(f"\n{'='*60}")
@ -20,33 +21,27 @@ def run(codebase_name, dir_path):
elapsed = time.time() - t0 elapsed = time.time() - t0
output = [] output = []
sore_count = 0
gbnf_ok = 0 gbnf_ok = 0
gbnf_fail = 0 gbnf_fail = 0
total_pkgs = 0 total_pkgs = 0
for ext, pkgs in results.items(): for ext, pkgs in results.items():
for pkg, info in sorted(pkgs.items()): for pkg, info in sorted(pkgs.items()):
grammar = info.get('grammar', '') grammar_str = info.get('grammar', '')
if grammar and grammar not in ('∅', 'ε', ''): if grammar_str:
sore_count += 1
total_pkgs += 1 total_pkgs += 1
entry = {'package': pkg, 'ext': ext, 'sore': grammar, 'methods': info.get('methods', 0)} entry = {'package': pkg, 'ext': ext, 'grammar': grammar_str, 'methods': info.get('methods', 0)}
try: try:
gbnf = to_gbnf(grammar) gbnf = to_gbnf(grammar_str)
entry['gbnf'] = gbnf entry['gbnf'] = gbnf
gbnf_ok += 1 gbnf_ok += 1
except Exception as e: except Exception as e:
entry['gbnf_error'] = str(e) entry['gbnf_error'] = str(e)
gbnf_fail += 1 gbnf_fail += 1
output.append(entry) output.append(entry)
elif grammar in ('∅', 'ε', ''):
pass # skip trivial
else:
total_pkgs += 1
print(f"\nTime: {elapsed:.1f}s") print(f"\nTime: {elapsed:.1f}s")
print(f"Packages with grammar: {sore_count}") print(f"Packages with grammar: {total_pkgs}")
print(f"GBNF OK: {gbnf_ok}, FAIL: {gbnf_fail}") print(f"GBNF OK: {gbnf_ok}, FAIL: {gbnf_fail}")
# Print all conversions # Print all conversions
@ -54,12 +49,12 @@ def run(codebase_name, dir_path):
for e in output: for e in output:
if 'gbnf' in e: if 'gbnf' in e:
print(f" {e['package']}") print(f" {e['package']}")
print(f" SORE: {e['sore']}") print(f" Grammar: {e['grammar']}")
print(f" GBNF: {e['gbnf']}") print(f" GBNF: {e['gbnf']}")
elif 'gbnf_error' in e: elif 'gbnf_error' in e:
print(f" {e['package']}") print(f" {e['package']}")
print(f" SORE: {e['sore']}") print(f" Grammar: {e['grammar']}")
print(f" ERR: {e['gbnf_error']}") print(f" ERR: {e['gbnf_error']}")
# Save to file # Save to file
out_path = Path(f"/tmp/gbnf_{codebase_name.lower().replace(' ','_')}.json") out_path = Path(f"/tmp/gbnf_{codebase_name.lower().replace(' ','_')}.json")