fix(gbnf): handle disjunction inside parens and compound repetition
The GBNF parser now correctly handles SORE's overloaded + operator: - + inside (a+b+c) → alternation (not repetition) - + outside parens → repetition - +? and +* compound operators → normalized to Star Also adds implicit concatenation when LPAREN follows a repetition, so a+(b+c) parses as a+ followed by (b|c). 28 tests pass (13 new disjunction/compound tests). Full suite: 212 passed.
This commit is contained in:
parent
6912841b9e
commit
f57c302c91
2 changed files with 102 additions and 11 deletions
63
bex/gbnf.py
63
bex/gbnf.py
|
|
@ -132,11 +132,17 @@ class _Empty(_Node):
|
|||
|
||||
|
||||
class _Parser:
|
||||
"""Recursive descent parser for SOREs."""
|
||||
"""Recursive descent parser for SOREs.
|
||||
|
||||
Handles the overloaded + operator:
|
||||
- (a+b+c) → disjunction (inside parens)
|
||||
- r+ → one-or-more repetition (outside parens)
|
||||
"""
|
||||
|
||||
def __init__(self, tokens):
|
||||
self.tokens = tokens
|
||||
self.pos = 0
|
||||
self.paren_depth = 0
|
||||
|
||||
def peek(self):
|
||||
if self.pos < len(self.tokens):
|
||||
|
|
@ -158,36 +164,67 @@ class _Parser:
|
|||
return node
|
||||
|
||||
def parse_alternation(self):
|
||||
"""Parse: concat ('|' concat)*"""
|
||||
"""Parse: concat (('|' | '+') concat)* — + is alternation inside parens"""
|
||||
parts = [self.parse_concat()]
|
||||
while self.peek()[0] == 'PIPE':
|
||||
self.consume('PIPE')
|
||||
while self.peek()[0] in ('PIPE', 'PLUS'):
|
||||
if self.peek()[0] == 'PLUS' and self.paren_depth == 0:
|
||||
break # + outside parens is repetition, not alternation
|
||||
self.consume()
|
||||
parts.append(self.parse_concat())
|
||||
if len(parts) == 1:
|
||||
return parts[0]
|
||||
return _Alt(parts)
|
||||
|
||||
def parse_concat(self):
|
||||
"""Parse: repetition ('.' repetition)* — top-level concat"""
|
||||
"""Parse: repetition (('.' | LPAREN) repetition)* — implicit concat"""
|
||||
parts = [self.parse_repetition()]
|
||||
while self.peek()[0] == 'DOT':
|
||||
self.consume('DOT')
|
||||
while self.peek()[0] in ('DOT', 'LPAREN'):
|
||||
if self.peek()[0] == 'DOT':
|
||||
self.consume('DOT')
|
||||
# LPAREN = implicit concat (no separator)
|
||||
parts.append(self.parse_repetition())
|
||||
if len(parts) == 1:
|
||||
return parts[0]
|
||||
return _Concat(parts)
|
||||
|
||||
def parse_repetition(self):
|
||||
"""Parse: atom ('+' | '?' | '*')?"""
|
||||
"""Parse: atom ('+' | '?' | '*')?
|
||||
|
||||
Inside parens, + is alternation (consumed by parse_alternation),
|
||||
not repetition. Outside parens, always consume + as repetition.
|
||||
Handles compound: +?, +*, ?+, *+ etc.
|
||||
"""
|
||||
node = self.parse_atom()
|
||||
if self.peek()[0] in ('PLUS', 'QUESTION', 'STAR'):
|
||||
if self.peek()[0] == 'PLUS' and self.paren_depth > 0:
|
||||
return node # + inside parens is alternation, handled by caller
|
||||
op = self.consume()
|
||||
if op[0] == 'PLUS':
|
||||
return _Plus(node)
|
||||
node = _Plus(node)
|
||||
elif op[0] == 'QUESTION':
|
||||
return _Optional(node)
|
||||
node = _Optional(node)
|
||||
elif op[0] == 'STAR':
|
||||
return _Star(node)
|
||||
node = _Star(node)
|
||||
# Handle compound repetition: +?, +*, ?+ etc.
|
||||
# Normalize: Optional(Plus(x)) → Star(x)
|
||||
if self.peek()[0] in ('PLUS', 'QUESTION', 'STAR'):
|
||||
if self.peek()[0] == 'PLUS' and self.paren_depth > 0:
|
||||
return node
|
||||
op2 = self.consume()
|
||||
if op2[0] == 'QUESTION':
|
||||
if isinstance(node, _Plus):
|
||||
node = _Star(node.child)
|
||||
else:
|
||||
node = _Optional(node)
|
||||
elif op2[0] == 'STAR':
|
||||
node = _Star(node.child if isinstance(node, (_Plus, _Optional)) else node)
|
||||
elif op2[0] == 'PLUS':
|
||||
if isinstance(node, _Optional):
|
||||
node = _Plus(node.child)
|
||||
elif isinstance(node, (_Plus, _Star)):
|
||||
node = node # ++ is idempotent
|
||||
else:
|
||||
node = _Plus(node)
|
||||
return node
|
||||
|
||||
def parse_atom(self):
|
||||
|
|
@ -206,8 +243,10 @@ class _Parser:
|
|||
return _Empty()
|
||||
if tok[0] == 'LPAREN':
|
||||
self.consume('LPAREN')
|
||||
self.paren_depth += 1
|
||||
node = self.parse_alternation()
|
||||
self.consume('RPAREN')
|
||||
self.paren_depth -= 1
|
||||
return node
|
||||
raise ValueError(f'Unexpected token: {tok}')
|
||||
|
||||
|
|
@ -250,6 +289,8 @@ def _node_to_gbnf(node, rule_counter):
|
|||
all_new_rules = []
|
||||
for child in node.parts:
|
||||
frag, rule_counter, new_rules = _node_to_gbnf(child, rule_counter)
|
||||
if isinstance(child, _Alt):
|
||||
frag = f'({frag})'
|
||||
parts.append(frag)
|
||||
all_new_rules.extend(new_rules)
|
||||
return ' '.join(p for p in parts if p), rule_counter, all_new_rules
|
||||
|
|
|
|||
|
|
@ -56,3 +56,53 @@ class TestToGBNFWithRules:
|
|||
def test_nested(self):
|
||||
result = to_gbnf_with_rules('assertEquals.(of.(assertFailsWith)?)+')
|
||||
assert result == 'root ::= "assertEquals" ("of" "assertFailsWith"?)+'
|
||||
|
||||
|
||||
class TestGBNFDisjunction:
|
||||
"""Test + as disjunction inside parentheses (SORE convention)."""
|
||||
|
||||
def test_simple_disjunction(self):
|
||||
assert to_gbnf('(a+b)') == 'root ::= "a" | "b"'
|
||||
|
||||
def test_disjunction_with_rep(self):
|
||||
assert to_gbnf('(a+b)+') == 'root ::= ("a" | "b")+'
|
||||
|
||||
def test_disjunction_optional(self):
|
||||
assert to_gbnf('(a+b)?') == 'root ::= ("a" | "b")?'
|
||||
|
||||
def test_disjunction_star(self):
|
||||
assert to_gbnf('(a+b)*') == 'root ::= ("a" | "b")*'
|
||||
|
||||
def test_disjunction_in_concat(self):
|
||||
assert to_gbnf('warn+.(BAD_REQUEST+CONFLICT)+') == \
|
||||
'root ::= "warn"+ ("BAD_REQUEST" | "CONFLICT")+'
|
||||
|
||||
def test_four_way_disjunction(self):
|
||||
assert to_gbnf('(assertEquals+authorize+coEvery+coVerify)+') == \
|
||||
'root ::= ("assertEquals" | "authorize" | "coEvery" | "coVerify")+'
|
||||
|
||||
def test_disjunction_parenthesized_in_concat(self):
|
||||
assert to_gbnf('a+(b+c)') == 'root ::= "a"+ ("b" | "c")'
|
||||
|
||||
def test_disjunction_rep_then_disjunction(self):
|
||||
assert to_gbnf('(a+b)+.(c+d)') == 'root ::= ("a" | "b")+ ("c" | "d")'
|
||||
|
||||
|
||||
class TestGBNFCompoundRepetition:
|
||||
"""Test compound repetition operators: +?, +*, etc."""
|
||||
|
||||
def test_plus_question(self):
|
||||
assert to_gbnf('(a)+?') == 'root ::= "a"*'
|
||||
|
||||
def test_plus_star(self):
|
||||
assert to_gbnf('(a)+*') == 'root ::= "a"*'
|
||||
|
||||
def test_question_plus(self):
|
||||
assert to_gbnf('(a)?+') == 'root ::= "a"+'
|
||||
|
||||
def test_flask_pattern(self):
|
||||
result = to_gbnf('return.(key+self)+?.(Markup+UUID)+?.to_json+?')
|
||||
assert result == 'root ::= "return" ("key" | "self")* ("Markup" | "UUID")* "to_json"*'
|
||||
|
||||
def test_double_plus(self):
|
||||
assert to_gbnf('a++.b') == 'root ::= "a"+ "b"'
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue