fix(gbnf): handle disjunction inside parens and compound repetition

The GBNF parser now correctly handles SORE's overloaded + operator:
- + inside (a+b+c) → alternation (not repetition)
- + outside parens → repetition
- +? and +* compound operators → normalized to Star

Also adds implicit concatenation when LPAREN follows a repetition,
so a+(b+c) parses as a+ followed by (b|c).

28 tests pass (13 new disjunction/compound tests). Full suite: 212 passed.
This commit is contained in:
tobjend 2026-07-12 02:20:58 +02:00
parent 6912841b9e
commit f57c302c91
2 changed files with 102 additions and 11 deletions

View file

@ -132,11 +132,17 @@ class _Empty(_Node):
class _Parser:
"""Recursive descent parser for SOREs."""
"""Recursive descent parser for SOREs.
Handles the overloaded + operator:
- (a+b+c) disjunction (inside parens)
- r+ one-or-more repetition (outside parens)
"""
def __init__(self, tokens):
self.tokens = tokens
self.pos = 0
self.paren_depth = 0
def peek(self):
if self.pos < len(self.tokens):
@ -158,36 +164,67 @@ class _Parser:
return node
def parse_alternation(self):
"""Parse: concat ('|' concat)*"""
"""Parse: concat (('|' | '+') concat)* — + is alternation inside parens"""
parts = [self.parse_concat()]
while self.peek()[0] == 'PIPE':
self.consume('PIPE')
while self.peek()[0] in ('PIPE', 'PLUS'):
if self.peek()[0] == 'PLUS' and self.paren_depth == 0:
break # + outside parens is repetition, not alternation
self.consume()
parts.append(self.parse_concat())
if len(parts) == 1:
return parts[0]
return _Alt(parts)
def parse_concat(self):
"""Parse: repetition ('.' repetition)* — top-level concat"""
"""Parse: repetition (('.' | LPAREN) repetition)* — implicit concat"""
parts = [self.parse_repetition()]
while self.peek()[0] == 'DOT':
while self.peek()[0] in ('DOT', 'LPAREN'):
if self.peek()[0] == 'DOT':
self.consume('DOT')
# LPAREN = implicit concat (no separator)
parts.append(self.parse_repetition())
if len(parts) == 1:
return parts[0]
return _Concat(parts)
def parse_repetition(self):
"""Parse: atom ('+' | '?' | '*')?"""
"""Parse: atom ('+' | '?' | '*')?
Inside parens, + is alternation (consumed by parse_alternation),
not repetition. Outside parens, always consume + as repetition.
Handles compound: +?, +*, ?+, *+ etc.
"""
node = self.parse_atom()
if self.peek()[0] in ('PLUS', 'QUESTION', 'STAR'):
if self.peek()[0] == 'PLUS' and self.paren_depth > 0:
return node # + inside parens is alternation, handled by caller
op = self.consume()
if op[0] == 'PLUS':
return _Plus(node)
node = _Plus(node)
elif op[0] == 'QUESTION':
return _Optional(node)
node = _Optional(node)
elif op[0] == 'STAR':
return _Star(node)
node = _Star(node)
# Handle compound repetition: +?, +*, ?+ etc.
# Normalize: Optional(Plus(x)) → Star(x)
if self.peek()[0] in ('PLUS', 'QUESTION', 'STAR'):
if self.peek()[0] == 'PLUS' and self.paren_depth > 0:
return node
op2 = self.consume()
if op2[0] == 'QUESTION':
if isinstance(node, _Plus):
node = _Star(node.child)
else:
node = _Optional(node)
elif op2[0] == 'STAR':
node = _Star(node.child if isinstance(node, (_Plus, _Optional)) else node)
elif op2[0] == 'PLUS':
if isinstance(node, _Optional):
node = _Plus(node.child)
elif isinstance(node, (_Plus, _Star)):
node = node # ++ is idempotent
else:
node = _Plus(node)
return node
def parse_atom(self):
@ -206,8 +243,10 @@ class _Parser:
return _Empty()
if tok[0] == 'LPAREN':
self.consume('LPAREN')
self.paren_depth += 1
node = self.parse_alternation()
self.consume('RPAREN')
self.paren_depth -= 1
return node
raise ValueError(f'Unexpected token: {tok}')
@ -250,6 +289,8 @@ def _node_to_gbnf(node, rule_counter):
all_new_rules = []
for child in node.parts:
frag, rule_counter, new_rules = _node_to_gbnf(child, rule_counter)
if isinstance(child, _Alt):
frag = f'({frag})'
parts.append(frag)
all_new_rules.extend(new_rules)
return ' '.join(p for p in parts if p), rule_counter, all_new_rules

View file

@ -56,3 +56,53 @@ class TestToGBNFWithRules:
def test_nested(self):
result = to_gbnf_with_rules('assertEquals.(of.(assertFailsWith)?)+')
assert result == 'root ::= "assertEquals" ("of" "assertFailsWith"?)+'
class TestGBNFDisjunction:
"""Test + as disjunction inside parentheses (SORE convention)."""
def test_simple_disjunction(self):
assert to_gbnf('(a+b)') == 'root ::= "a" | "b"'
def test_disjunction_with_rep(self):
assert to_gbnf('(a+b)+') == 'root ::= ("a" | "b")+'
def test_disjunction_optional(self):
assert to_gbnf('(a+b)?') == 'root ::= ("a" | "b")?'
def test_disjunction_star(self):
assert to_gbnf('(a+b)*') == 'root ::= ("a" | "b")*'
def test_disjunction_in_concat(self):
assert to_gbnf('warn+.(BAD_REQUEST+CONFLICT)+') == \
'root ::= "warn"+ ("BAD_REQUEST" | "CONFLICT")+'
def test_four_way_disjunction(self):
assert to_gbnf('(assertEquals+authorize+coEvery+coVerify)+') == \
'root ::= ("assertEquals" | "authorize" | "coEvery" | "coVerify")+'
def test_disjunction_parenthesized_in_concat(self):
assert to_gbnf('a+(b+c)') == 'root ::= "a"+ ("b" | "c")'
def test_disjunction_rep_then_disjunction(self):
assert to_gbnf('(a+b)+.(c+d)') == 'root ::= ("a" | "b")+ ("c" | "d")'
class TestGBNFCompoundRepetition:
"""Test compound repetition operators: +?, +*, etc."""
def test_plus_question(self):
assert to_gbnf('(a)+?') == 'root ::= "a"*'
def test_plus_star(self):
assert to_gbnf('(a)+*') == 'root ::= "a"*'
def test_question_plus(self):
assert to_gbnf('(a)?+') == 'root ::= "a"+'
def test_flask_pattern(self):
result = to_gbnf('return.(key+self)+?.(Markup+UUID)+?.to_json+?')
assert result == 'root ::= "return" ("key" | "self")* ("Markup" | "UUID")* "to_json"*'
def test_double_plus(self):
assert to_gbnf('a++.b') == 'root ::= "a"+ "b"'