bc7d3b6ca1
- iDRegEx now opt-in via --idregex flag (was running on every group, causing 55s+ on Flask alone — src/flask/json took 55s in iDRegEx) - GBNF tokenizer strips newlines from literals (multi-line symbols) - Fix OverflowError: lang_size_score produces huge ints for large disjunctions, format as string not float - Flask: 2.7s (was 55s+), RAGSAK: 13s (was 74s)
75 lines
2.3 KiB
Python
75 lines
2.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Run pipeline + GBNF conversion on a codebase. Output results to JSON."""
|
|
import sys
|
|
import json
|
|
import time
|
|
from pathlib import Path
|
|
from bex.tag_preprocessor.analyze import analyze_directory
|
|
from bex.gbnf import to_gbnf
|
|
|
|
def run(codebase_name, dir_path):
|
|
print(f"\n{'='*60}")
|
|
print(f" {codebase_name}: {dir_path}")
|
|
print(f"{'='*60}")
|
|
|
|
t0 = time.time()
|
|
results = analyze_directory(
|
|
dir_path, slice='package', method='langsize',
|
|
min_coverage=0.05, min_methods=3,
|
|
)
|
|
elapsed = time.time() - t0
|
|
|
|
output = []
|
|
sore_count = 0
|
|
gbnf_ok = 0
|
|
gbnf_fail = 0
|
|
total_pkgs = 0
|
|
|
|
for ext, pkgs in results.items():
|
|
for pkg, info in sorted(pkgs.items()):
|
|
grammar = info.get('grammar', '')
|
|
if grammar and grammar not in ('∅', 'ε', ''):
|
|
sore_count += 1
|
|
total_pkgs += 1
|
|
entry = {'package': pkg, 'ext': ext, 'sore': grammar, 'methods': info.get('methods', 0)}
|
|
try:
|
|
gbnf = to_gbnf(grammar)
|
|
entry['gbnf'] = gbnf
|
|
gbnf_ok += 1
|
|
except Exception as e:
|
|
entry['gbnf_error'] = str(e)
|
|
gbnf_fail += 1
|
|
output.append(entry)
|
|
elif grammar in ('∅', 'ε', ''):
|
|
pass # skip trivial
|
|
else:
|
|
total_pkgs += 1
|
|
|
|
print(f"\nTime: {elapsed:.1f}s")
|
|
print(f"Packages with grammar: {sore_count}")
|
|
print(f"GBNF OK: {gbnf_ok}, FAIL: {gbnf_fail}")
|
|
|
|
# Print all conversions
|
|
print(f"\n{'─'*60}")
|
|
for e in output:
|
|
if 'gbnf' in e:
|
|
print(f" {e['package']}")
|
|
print(f" SORE: {e['sore']}")
|
|
print(f" GBNF: {e['gbnf']}")
|
|
elif 'gbnf_error' in e:
|
|
print(f" {e['package']}")
|
|
print(f" SORE: {e['sore']}")
|
|
print(f" ERR: {e['gbnf_error']}")
|
|
|
|
# Save to file
|
|
out_path = Path(f"/tmp/gbnf_{codebase_name.lower().replace(' ','_')}.json")
|
|
with open(out_path, 'w') as f:
|
|
json.dump(output, f, indent=2)
|
|
print(f"\nSaved to {out_path}")
|
|
|
|
return output
|
|
|
|
if __name__ == '__main__':
|
|
name = sys.argv[1]
|
|
path = sys.argv[2]
|
|
run(name, path)
|