From f8dda557d2a7442a06955ccf728ac3b780552f50 Mon Sep 17 00:00:00 2001 From: tobjend Date: Sun, 12 Jul 2026 23:46:25 +0200 Subject: [PATCH] chore: update experiment scripts for AST representation --- experiments/coarsen_eval.py | 20 +++++++++++--------- experiments/context_eval.py | 32 +++++++++++++++++--------------- experiments/freq_eval.py | 21 ++++++++++++--------- experiments/gbnf_eval.py | 25 ++++++++++--------------- 4 files changed, 50 insertions(+), 48 deletions(-) diff --git a/experiments/coarsen_eval.py b/experiments/coarsen_eval.py index 654ded8..6ec3d7a 100644 --- a/experiments/coarsen_eval.py +++ b/experiments/coarsen_eval.py @@ -17,6 +17,8 @@ from bex.tag_preprocessor.code import ( ) from bex.twotinf import build_soa from bex.rwr0 import rwr0 +from bex.grammar import Empty +from bex.gbnf import to_gbnf RESULTS_DIR = Path(__file__).parent / "results" @@ -80,8 +82,8 @@ def infer_sore(seqs): return None try: soa = build_soa(clean) - sore = rwr0(soa) - return sore if sore != "∅" else None + grammar = rwr0(soa) + return grammar if not isinstance(grammar, Empty) else None except Exception: return None @@ -103,8 +105,8 @@ def measure_group(ctx, items, label): "unique": unique, "unique_ratio": round(unique_ratio, 3), "alphabet_size": len(alphabet), - "sore": sore[:200] if sore else None, - "sore_success": sore is not None, + "grammar": to_gbnf(sore)[:200] if sore else None, + "grammar_success": sore is not None, } @@ -136,7 +138,7 @@ def run_comparison(name, raw_seqs, coarse_seqs, packages, k_values=(1, 2, 3)): # Measure all groups group_results = [] - sore_successes = 0 + grammar_successes = 0 methods_in_good = 0 total_methods = 0 @@ -144,8 +146,8 @@ def run_comparison(name, raw_seqs, coarse_seqs, packages, k_values=(1, 2, 3)): m = measure_group(ctx, items, f"{label}_k{k}") group_results.append(m) total_methods += m["methods"] - if m["sore_success"]: - sore_successes += 1 + if m["grammar_success"]: + grammar_successes += 1 methods_in_good += m["methods"] # Cross-package analysis @@ -153,7 +155,7 @@ def run_comparison(name, raw_seqs, coarse_seqs, packages, k_values=(1, 2, 3)): results[label][f"k{k}"] = { "contexts": len(groups), - "sore_successes": sore_successes, + "grammar_successes": grammar_successes, "total_methods": total_methods, "methods_in_good": methods_in_good, "coverage": round(methods_in_good / total_methods * 100, 1) if total_methods else 0, @@ -189,7 +191,7 @@ def print_results(results): print(f"\n --- k={k} ---") print(f" {'':20s} {'Raw':>10s} {'Coarsened':>10s}") print(f" {'Contexts':20s} {rk.get('contexts',0):10d} {ck.get('contexts',0):10d}") - print(f" {'SORE successes':20s} {rk.get('sore_successes',0):10d} {ck.get('sore_successes',0):10d}") + print(f" {'Grammar successes':20s} {rk.get('grammar_successes',0):10d} {ck.get('grammar_successes',0):10d}") print(f" {'Coverage':20s} {rk.get('coverage',0):9.1f}% {ck.get('coverage',0):9.1f}%") print(f" {'Cross-pkg contexts':20s} {rk.get('cross_package_contexts',0):10d} {ck.get('cross_package_contexts',0):10d}") diff --git a/experiments/context_eval.py b/experiments/context_eval.py index b729ae0..89c84b5 100644 --- a/experiments/context_eval.py +++ b/experiments/context_eval.py @@ -15,6 +15,8 @@ from bex.tag_preprocessor.analyze import ( from bex.twotinf import build_soa from bex.rwr0 import rwr0 from bex.reduce import reduce_contexts, soa_distance, build_soa_with_support +from bex.grammar import Empty +from bex.gbnf import to_gbnf RESULTS_DIR = Path(__file__).parent / "results" @@ -115,8 +117,8 @@ def evaluate_context(contexts, label, min_methods=3, max_methods=50, "meaningful_contexts": 0, "total_methods": 0, "methods_in_good_groups": 0, - "sore_successes": 0, - "sore_failures": 0, + "grammar_successes": 0, + "grammar_failures": 0, "skip_reasons": {}, "groups": [], } @@ -136,7 +138,7 @@ def evaluate_context(contexts, label, min_methods=3, max_methods=50, results["groups"].append({ "context": str(ctx), "methods": n, "unique": unique, "unique_ratio": round(unique_ratio, 3), - "sore": "SKIP(too_large)", "sore_success": False, + "grammar": "SKIP(too_large)", "grammar_success": False, }) continue @@ -145,7 +147,7 @@ def evaluate_context(contexts, label, min_methods=3, max_methods=50, results["groups"].append({ "context": str(ctx), "methods": n, "unique": unique, "unique_ratio": round(unique_ratio, 3), - "sore": "SKIP(too_diverse)", "sore_success": False, + "grammar": "SKIP(too_diverse)", "grammar_success": False, }) continue @@ -157,7 +159,7 @@ def evaluate_context(contexts, label, min_methods=3, max_methods=50, results["groups"].append({ "context": str(ctx), "methods": n, "unique": unique, "unique_ratio": round(unique_ratio, 3), - "sore": "SKIP(large_alphabet)", "sore_success": False, + "grammar": "SKIP(large_alphabet)", "grammar_success": False, }) continue @@ -179,26 +181,26 @@ def evaluate_context(contexts, label, min_methods=3, max_methods=50, results["groups"].append({ "context": str(ctx), "methods": n, "unique": unique, "unique_ratio": round(unique_ratio, 3), - "sore": "SKIP(complex_soa)", "sore_success": False, + "grammar": "SKIP(complex_soa)", "grammar_success": False, }) continue - sore = rwr0(soa) + grammar = rwr0(soa) group_info = { "context": str(ctx), "methods": n, "unique": unique, "unique_ratio": round(unique_ratio, 3), - "sore": sore[:200] if sore not in ("∅",) else sore, - "sore_success": sore != "∅", + "grammar": to_gbnf(grammar)[:200] if not isinstance(grammar, Empty) else "∅", + "grammar_success": not isinstance(grammar, Empty), } - if sore != "∅": - results["sore_successes"] += 1 + if not isinstance(grammar, Empty): + results["grammar_successes"] += 1 results["methods_in_good_groups"] += n else: - results["sore_failures"] += 1 + results["grammar_failures"] += 1 results["groups"].append(group_info) @@ -222,7 +224,7 @@ def run_experiment(name, contexts, label, codebase_name, merge_info=None): results["merge_info"] = merge_info print(f" Contexts: {results['total_contexts']} Meaningful: {results['meaningful_contexts']} " - f"SOREs: {results['sore_successes']} Coverage: {results['coverage']}% " + f"Grammars: {results['grammar_successes']} Coverage: {results['coverage']}% " f"Time: {elapsed:.2f}s") if merge_info: print(f" Merges: {merge_info['merges']} Threshold: {merge_info['threshold']}") @@ -306,14 +308,14 @@ def run_codebase(codebase_key): print(f"\n{'=' * 70}") print(f" {name} SUMMARY") print(f"{'=' * 70}") - print(f"{'Strategy':<42} {'Ctx':>5} {'SORE':>5} {'Cov%':>6} {'Merge':>5}") + print(f"{'Strategy':<42} {'Ctx':>5} {'Grms':>5} {'Cov%':>6} {'Merge':>5}") print("-" * 70) for r in all_results: merge_str = "" if r.get("merge_info"): merge_str = str(r["merge_info"]["merges"]) print(f"{r['strategy']:<42} {r['meaningful_contexts']:>5} " - f"{r['sore_successes']:>5} {r['coverage']:>5.1f}% {merge_str:>5}") + f"{r['grammar_successes']:>5} {r['coverage']:>5.1f}% {merge_str:>5}") summary = [ {k: v for k, v in r.items() if k != "groups"} diff --git a/experiments/freq_eval.py b/experiments/freq_eval.py index 02dcd76..ca1eac6 100644 --- a/experiments/freq_eval.py +++ b/experiments/freq_eval.py @@ -15,6 +15,7 @@ from bex.tag_preprocessor.analyze import ( from bex.tag_preprocessor.code import _extract_call_tokens, _extract_coarsened_tokens from bex.twotinf import build_soa from bex.rwr0 import rwr0 +from bex.grammar import Empty RESULTS_DIR = Path(__file__).parent / "results" @@ -67,7 +68,7 @@ def measure_at_threshold(raw_seqs, packages, threshold): ctx = tuple(seq[:3]) contexts[ctx].append(seq) - sore_successes = 0 + grammar_successes = 0 methods_in_good = 0 total_methods = 0 top_patterns = [] @@ -87,15 +88,17 @@ def measure_at_threshold(raw_seqs, packages, threshold): continue try: soa = build_soa(seqs) - sore = rwr0(soa) - if sore != "∅": - sore_successes += 1 + grammar = rwr0(soa) + if not isinstance(grammar, Empty): + grammar_successes += 1 methods_in_good += n + from bex.gbnf import to_gbnf + gbnf_str = to_gbnf(grammar) top_patterns.append({ "context": str(ctx), "methods": n, "unique": unique, - "sore": sore[:150], + "grammar": gbnf_str[:150], }) except Exception: pass @@ -107,7 +110,7 @@ def measure_at_threshold(raw_seqs, packages, threshold): "seqs_total": total, "seqs_surviving": surviving, "contexts": len(contexts), - "sore_successes": sore_successes, + "grammar_successes": grammar_successes, "total_methods": total_methods, "methods_in_good": methods_in_good, "coverage": round(methods_in_good / total_methods * 100, 1) if total_methods else 0, @@ -132,7 +135,7 @@ def run_codebase(key): r["elapsed"] = round(elapsed, 2) results.append(r) print(f" thresh={thresh:.2f} syms={r['symbols_kept']:4d} seqs={r['seqs_surviving']:5d} " - f"ctxs={r['contexts']:4d} SOREs={r['sore_successes']:3d} cov={r['coverage']:5.1f}% " + f"ctxs={r['contexts']:4d} grms={r['grammar_successes']:3d} cov={r['coverage']:5.1f}% " f"({elapsed:.1f}s)") out_path = RESULTS_DIR / f"freq_{key}.json" @@ -152,13 +155,13 @@ def main(): print(f"\n{'=' * 80}") print(" SUMMARY") print(f"{'=' * 80}") - print(f" {'Codebase':20s} {'Thresh':7s} {'Syms':5s} {'Seqs':6s} {'SOREs':6s} {'Cov':7s}") + print(f" {'Codebase':20s} {'Thresh':7s} {'Syms':5s} {'Seqs':6s} {'Grms':6s} {'Cov':7s}") print(f" {'-'*55}") for key, results in all_results.items(): name = CODEBASES[key]["name"] for r in results: print(f" {name:20s} {r['threshold']:7.2f} {r['symbols_kept']:5d} " - f"{r['seqs_surviving']:6d} {r['sore_successes']:6d} {r['coverage']:6.1f}%") + f"{r['seqs_surviving']:6d} {r['grammar_successes']:6d} {r['coverage']:6.1f}%") return all_results diff --git a/experiments/gbnf_eval.py b/experiments/gbnf_eval.py index 61eb543..ce4352e 100644 --- a/experiments/gbnf_eval.py +++ b/experiments/gbnf_eval.py @@ -6,6 +6,7 @@ import time from pathlib import Path from bex.tag_preprocessor.analyze import analyze_directory from bex.gbnf import to_gbnf +from bex.grammar import Empty def run(codebase_name, dir_path): print(f"\n{'='*60}") @@ -20,33 +21,27 @@ def run(codebase_name, dir_path): elapsed = time.time() - t0 output = [] - sore_count = 0 gbnf_ok = 0 gbnf_fail = 0 total_pkgs = 0 for ext, pkgs in results.items(): for pkg, info in sorted(pkgs.items()): - grammar = info.get('grammar', '') - if grammar and grammar not in ('∅', 'ε', ''): - sore_count += 1 + grammar_str = info.get('grammar', '') + if grammar_str: total_pkgs += 1 - entry = {'package': pkg, 'ext': ext, 'sore': grammar, 'methods': info.get('methods', 0)} + entry = {'package': pkg, 'ext': ext, 'grammar': grammar_str, 'methods': info.get('methods', 0)} try: - gbnf = to_gbnf(grammar) + gbnf = to_gbnf(grammar_str) entry['gbnf'] = gbnf gbnf_ok += 1 except Exception as e: entry['gbnf_error'] = str(e) gbnf_fail += 1 output.append(entry) - elif grammar in ('∅', 'ε', ''): - pass # skip trivial - else: - total_pkgs += 1 print(f"\nTime: {elapsed:.1f}s") - print(f"Packages with grammar: {sore_count}") + print(f"Packages with grammar: {total_pkgs}") print(f"GBNF OK: {gbnf_ok}, FAIL: {gbnf_fail}") # Print all conversions @@ -54,12 +49,12 @@ def run(codebase_name, dir_path): for e in output: if 'gbnf' in e: print(f" {e['package']}") - print(f" SORE: {e['sore']}") - print(f" GBNF: {e['gbnf']}") + print(f" Grammar: {e['grammar']}") + print(f" GBNF: {e['gbnf']}") elif 'gbnf_error' in e: print(f" {e['package']}") - print(f" SORE: {e['sore']}") - print(f" ERR: {e['gbnf_error']}") + print(f" Grammar: {e['grammar']}") + print(f" ERR: {e['gbnf_error']}") # Save to file out_path = Path(f"/tmp/gbnf_{codebase_name.lower().replace(' ','_')}.json")