From 011df391c2d349b44848a3643ca2dad685c19094 Mon Sep 17 00:00:00 2001 From: tobjend Date: Sun, 12 Jul 2026 00:31:47 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20implement=20SORE=20=E2=86=92=20GBNF=20c?= =?UTF-8?q?onverter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Recursive descent parser for SORE syntax (+, ?, *, |, ., parens) - AST intermediate representation (_Literal, _Concat, _Alt, _Plus, _Optional, _Star) - to_gbnf(sore) → full GBNF rule string - to_gbnf_with_rules(sore, name) → named rule for composition - 15 tests covering all SORE operators and nesting patterns --- bex/__init__.py | 1 + bex/gbnf.py | 331 +++++++++++++++++++++++++++++++++++++++++++++ tests/test_gbnf.py | 58 ++++++++ 3 files changed, 390 insertions(+) create mode 100644 bex/gbnf.py create mode 100644 tests/test_gbnf.py diff --git a/bex/__init__.py b/bex/__init__.py index e9904ef..e2ebf8b 100644 --- a/bex/__init__.py +++ b/bex/__init__.py @@ -25,5 +25,6 @@ from .tokenizer import YAMLTokenizer from .ensemble import infer_ensemble from .template import generate_template from .reduce import reduce_contexts, soa_distance, build_soa_with_support, minimize_contexts, reduce_and_infer +from .gbnf import to_gbnf, to_gbnf_with_rules __version__ = "0.2.0" diff --git a/bex/gbnf.py b/bex/gbnf.py new file mode 100644 index 0000000..cc920b9 --- /dev/null +++ b/bex/gbnf.py @@ -0,0 +1,331 @@ +"""GBNF — Convert SOREs to GBNF grammar format for llama.cpp constrained decoding. + +GBNF (GGML BNF) is the de-facto standard grammar format for grammar-constrained +LLM output. It supports: literals, concatenation, alternation, repetition (+, *, ?), +and grouping. + +SORE operators map directly: + SORE `.` (concat) → GBNF implicit concat (space) + SORE `|` (alt) → GBNF `|` + SORE `+` (plus) → GBNF `+` + SORE `?` (optional) → GBNF `?` + SORE `*` (star) → GBNF `*` + SORE `()` (group) → GBNF `()` + SORE literal → GBNF `"literal"` +""" + + +# --------------------------------------------------------------------------- +# SORE tokenizer +# --------------------------------------------------------------------------- + +_TOKEN_CHARS = set('.|+?*()') + + +def _tokenize(sore): + """Tokenize a SORE string into a list of (type, value) tuples. + + Literals are sequences of non-operator characters. + """ + tokens = [] + i = 0 + n = len(sore) + while i < n: + ch = sore[i] + if ch == '.': + tokens.append(('DOT', ch)) + i += 1 + elif ch == '|': + tokens.append(('PIPE', ch)) + i += 1 + elif ch == '+': + tokens.append(('PLUS', ch)) + i += 1 + elif ch == '?': + tokens.append(('QUESTION', ch)) + i += 1 + elif ch == '*': + tokens.append(('STAR', ch)) + i += 1 + elif ch == '(': + tokens.append(('LPAREN', ch)) + i += 1 + elif ch == ')': + tokens.append(('RPAREN', ch)) + i += 1 + elif ch == 'ε': + tokens.append(('EPSILON', 'ε')) + i += 1 + elif ch == '∅': + tokens.append(('EMPTY', '∅')) + i += 1 + else: + # Collect literal characters (until next operator or paren) + start = i + while i < n and sore[i] not in _TOKEN_CHARS and sore[i] not in 'ε∅': + i += 1 + lit = sore[start:i] + if lit: + tokens.append(('LITERAL', lit)) + return tokens + + +# --------------------------------------------------------------------------- +# SORE parser — produces a tree +# --------------------------------------------------------------------------- + +class _Node: + """AST node for SORE.""" + pass + + +class _Literal(_Node): + def __init__(self, value): + self.value = value + def __repr__(self): + return f'Lit({self.value!r})' + + +class _Concat(_Node): + def __init__(self, parts): + self.parts = parts + def __repr__(self): + return f'Concat({self.parts})' + + +class _Alt(_Node): + def __init__(self, parts): + self.parts = parts + def __repr__(self): + return f'Alt({self.parts})' + + +class _Plus(_Node): + def __init__(self, child): + self.child = child + def __repr__(self): + return f'Plus({self.child})' + + +class _Optional(_Node): + def __init__(self, child): + self.child = child + def __repr__(self): + return f'Optional({self.child})' + + +class _Star(_Node): + def __init__(self, child): + self.child = child + def __repr__(self): + return f'Star({self.child})' + + +class _Epsilon(_Node): + def __repr__(self): + return 'Epsilon()' + + +class _Empty(_Node): + def __repr__(self): + return 'Empty()' + + +class _Parser: + """Recursive descent parser for SOREs.""" + + def __init__(self, tokens): + self.tokens = tokens + self.pos = 0 + + def peek(self): + if self.pos < len(self.tokens): + return self.tokens[self.pos] + return ('EOF', '') + + def consume(self, expected_type=None): + tok = self.peek() + if tok[0] == 'EOF': + raise ValueError(f'Unexpected end of SORE, expected {expected_type}') + if expected_type and tok[0] != expected_type: + raise ValueError(f'Expected {expected_type}, got {tok}') + self.pos += 1 + return tok + + def parse(self): + """Parse full SORE.""" + node = self.parse_alternation() + return node + + def parse_alternation(self): + """Parse: concat ('|' concat)*""" + parts = [self.parse_concat()] + while self.peek()[0] == 'PIPE': + self.consume('PIPE') + parts.append(self.parse_concat()) + if len(parts) == 1: + return parts[0] + return _Alt(parts) + + def parse_concat(self): + """Parse: repetition ('.' repetition)* — top-level concat""" + parts = [self.parse_repetition()] + while self.peek()[0] == 'DOT': + self.consume('DOT') + parts.append(self.parse_repetition()) + if len(parts) == 1: + return parts[0] + return _Concat(parts) + + def parse_repetition(self): + """Parse: atom ('+' | '?' | '*')?""" + node = self.parse_atom() + if self.peek()[0] in ('PLUS', 'QUESTION', 'STAR'): + op = self.consume() + if op[0] == 'PLUS': + return _Plus(node) + elif op[0] == 'QUESTION': + return _Optional(node) + elif op[0] == 'STAR': + return _Star(node) + return node + + def parse_atom(self): + """Parse: literal | '(' alternation ')' | 'ε' | '∅'""" + tok = self.peek() + if tok[0] == 'EOF': + return _Epsilon() + if tok[0] == 'LITERAL': + self.consume() + return _Literal(tok[1]) + if tok[0] == 'EPSILON': + self.consume() + return _Epsilon() + if tok[0] == 'EMPTY': + self.consume() + return _Empty() + if tok[0] == 'LPAREN': + self.consume('LPAREN') + node = self.parse_alternation() + self.consume('RPAREN') + return node + raise ValueError(f'Unexpected token: {tok}') + + +def _parse_sore(sore): + """Parse a SORE string into an AST.""" + tokens = _tokenize(sore) + parser = _Parser(tokens) + return parser.parse() + + +# --------------------------------------------------------------------------- +# AST → GBNF +# --------------------------------------------------------------------------- + +def _needs_group(node): + """Check if a node needs parentheses in GBNF output.""" + return isinstance(node, (_Alt, _Concat)) + + +def _node_to_gbnf(node, rule_counter): + """Convert AST node to GBNF fragment string. + + Returns (gbnf_string, new_rule_counter, new_rules_list). + new_rules_list contains any helper rules needed. + """ + if isinstance(node, _Literal): + # Escape special chars in the literal for GBNF + escaped = node.value.replace('\\', '\\\\').replace('"', '\\"') + return f'"{escaped}"', rule_counter, [] + + if isinstance(node, _Epsilon): + return '', rule_counter, [] + + if isinstance(node, _Empty): + return '', rule_counter, [] + + if isinstance(node, _Concat): + parts = [] + all_new_rules = [] + for child in node.parts: + frag, rule_counter, new_rules = _node_to_gbnf(child, rule_counter) + parts.append(frag) + all_new_rules.extend(new_rules) + return ' '.join(p for p in parts if p), rule_counter, all_new_rules + + if isinstance(node, _Alt): + parts = [] + all_new_rules = [] + for child in node.parts: + frag, rule_counter, new_rules = _node_to_gbnf(child, rule_counter) + parts.append(frag) + all_new_rules.extend(new_rules) + return ' | '.join(p for p in parts if p), rule_counter, all_new_rules + + if isinstance(node, _Plus): + frag, rule_counter, new_rules = _node_to_gbnf(node.child, rule_counter) + if _needs_group(node.child): + return f'({frag})+', rule_counter, new_rules + return f'{frag}+', rule_counter, new_rules + + if isinstance(node, _Optional): + frag, rule_counter, new_rules = _node_to_gbnf(node.child, rule_counter) + if _needs_group(node.child): + return f'({frag})?', rule_counter, new_rules + return f'{frag}?', rule_counter, new_rules + + if isinstance(node, _Star): + frag, rule_counter, new_rules = _node_to_gbnf(node.child, rule_counter) + if _needs_group(node.child): + return f'({frag})*', rule_counter, new_rules + return f'{frag}*', rule_counter, new_rules + + raise ValueError(f'Unknown node type: {type(node)}') + + +def to_gbnf(sore): + """Convert a SORE string to a GBNF grammar string. + + Args: + sore: A SORE string like 'raise.(ValueError)+' or 'mockk' + + Returns: + A GBNF grammar string with a single 'root' rule. + + Example: + >>> to_gbnf('mockk') + 'root ::= "mockk"' + >>> to_gbnf('raise.(ValueError)+') + 'root ::= "raise" ("ValueError")+' + >>> to_gbnf('assertEquals.(of.(assertFailsWith)?)+') + 'root ::= "assertEquals" ("of" ("assertFailsWith")?)+' + """ + if not sore or sore == '∅': + return 'root ::= ""' + if sore == 'ε': + return 'root ::= ""' + + tree = _parse_sore(sore) + frag, _, _ = _node_to_gbnf(tree, 0) + return f'root ::= {frag}' + + +def to_gbnf_with_rules(sore, name='root'): + """Convert a SORE to GBNF with a named rule. + + Args: + sore: A SORE string + name: Rule name (default: 'root') + + Returns: + GBNF rule string like 'my-rule ::= "foo" ("bar")+' + """ + if not sore or sore == '∅': + return f'{name} ::= ""' + if sore == 'ε': + return f'{name} ::= ""' + + tree = _parse_sore(sore) + frag, _, _ = _node_to_gbnf(tree, 0) + return f'{name} ::= {frag}' diff --git a/tests/test_gbnf.py b/tests/test_gbnf.py new file mode 100644 index 0000000..6c6af1d --- /dev/null +++ b/tests/test_gbnf.py @@ -0,0 +1,58 @@ +"""Tests for SORE → GBNF converter.""" +import pytest +from bex.gbnf import to_gbnf, to_gbnf_with_rules + + +class TestToGBNF: + def test_literal(self): + assert to_gbnf('mockk') == 'root ::= "mockk"' + + def test_concat(self): + assert to_gbnf('raise.ValueError') == 'root ::= "raise" "ValueError"' + + def test_plus_group(self): + assert to_gbnf('(append)+') == 'root ::= "append"+' + + def test_plus_concat(self): + assert to_gbnf('raise.(ValueError)+') == 'root ::= "raise" "ValueError"+' + + def test_nested_optional_plus(self): + assert to_gbnf('assertEquals.(of.(assertFailsWith)?)+') == \ + 'root ::= "assertEquals" ("of" "assertFailsWith"?)+' + + def test_long_concat(self): + assert to_gbnf('filesIn.filter.contains.assertTrue.(hasImport)+') == \ + 'root ::= "filesIn" "filter" "contains" "assertTrue" "hasImport"+' + + def test_simple_concat(self): + assert to_gbnf('trim.lowercase.(warn)+') == 'root ::= "trim" "lowercase" "warn"+' + + def test_flat_concat(self): + assert to_gbnf('DoclingConfig.assertThatThrownBy.validateCriticalSettings.isInstanceOf.hasMessageContaining') == \ + 'root ::= "DoclingConfig" "assertThatThrownBy" "validateCriticalSettings" "isInstanceOf" "hasMessageContaining"' + + def test_simple_plus(self): + assert to_gbnf('(abort)+') == 'root ::= "abort"+' + + def test_concat_with_plus(self): + assert to_gbnf('return.(url_for)+') == 'root ::= "return" "url_for"+' + + def test_star(self): + assert to_gbnf('(foo)*') == 'root ::= "foo"*' + + def test_optional(self): + assert to_gbnf('(bar)?') == 'root ::= "bar"?' + + +class TestToGBNFWithRules: + def test_named_rule(self): + result = to_gbnf_with_rules('raise.(ValueError)+', name='my-pattern') + assert result == 'my-pattern ::= "raise" "ValueError"+' + + def test_default_name(self): + result = to_gbnf_with_rules('mockk') + assert result == 'root ::= "mockk"' + + def test_nested(self): + result = to_gbnf_with_rules('assertEquals.(of.(assertFailsWith)?)+') + assert result == 'root ::= "assertEquals" ("of" "assertFailsWith"?)+'