bex/gbnf.py

@@ -383,3 +383,48 @@ def validate_sore(sore):
         return True, None
     except ValueError as e:
         return False, str(e)
+
+
+def grammar_structure_score(sore):
+    """Quantify how structured a SORE is (0.0 = flat bag, 1.0 = fully structured).
+
+    Structured grammars have ordering (concatenation, optional, repetition)
+    that tells you the SEQUENCE things happen. Flat bags just list symbols.
+    """
+    import re
+    if not sore or sore in ('∅', 'ε'):
+        return 0.0
+    depth = 0
+    dots = 0
+    pluses_outside = 0
+    questions = 0
+    stars = 0
+    for ch in sore:
+        if ch == '(':
+            depth += 1
+        elif ch == ')':
+            depth -= 1
+        elif ch == '.' and depth == 0:
+            dots += 1
+        elif ch == '+' and depth == 0:
+            pluses_outside += 1
+        elif ch == '?' and depth == 0:
+            questions += 1
+        elif ch == '*' and depth == 0:
+            stars += 1
+    disj_parts = 0
+    for m in re.finditer(r'\(([^)]+)\)', sore):
+        inner = m.group(1)
+        if '+' in inner:
+            disj_parts = max(disj_parts, inner.count('+') + 1)
+    symbols_only = re.sub(r'[.?*+()]', '', sore)
+    sym_len = len(symbols_only)
+    if sym_len == 0:
+        return 0.0
+    struct_ops = dots + questions + stars + pluses_outside
+    struct_ratio = struct_ops / max(sym_len, 1)
+    disj_ratio = disj_parts / max(sym_len, 1)
+    score = min(1.0, struct_ratio * 3)
+    if disj_ratio > 0.5 and dots == 0:
+        score *= 0.3
+    return score

bex/tag_preprocessor/analyze.py

@@ -20,7 +20,7 @@ import pathspec
 
 from .code import preprocess_by_method, extract_arg_info, _summarize_arg_info
 from bex.ensemble import infer_ensemble
-from bex.gbnf import validate_sore
+from bex.gbnf import validate_sore, grammar_structure_score
 
 SUPPORTED_EXTENSIONS = {
     ".py", ".js", ".ts", ".kt", ".rb", ".go", ".rs", ".java", ".c", ".cpp",
@@ -283,7 +283,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAUL
     return [("(all methods)", result, len(sequences), meta)]
 
 
-def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=5, crx_method='standard'):
+def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False, include_idregex=False, method='langsize', min_methods=5, crx_method='standard', min_structure=0.0):
     """Infer grammar for one package group. Module-level for ProcessPoolExecutor."""
     filtered = frequency_filter(group_seqs, min_coverage=min_coverage)
     imports = _extract_imports(group_files)
@@ -316,16 +316,20 @@ def _infer_group(label, group_seqs, group_files, project_root, min_coverage, pre
 
     # Validate grammar is parseable
     if result and result.get('best') and result['best'].get('grammar'):
-        ok, err = validate_sore(result['best']['grammar'])
+        grammar = result['best']['grammar']
+        ok, err = validate_sore(grammar)
         if not ok:
             meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "malformed_grammar", "skip_detail": err}
             return (label, None, len(filtered), meta)
+        if min_structure > 0 and grammar_structure_score(grammar) < min_structure:
+            meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages, "skip_reason": "low_structure", "structure_score": grammar_structure_score(grammar)}
+            return (label, None, len(filtered), meta)
 
     meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
     return (label, result, len(filtered), meta)
 
 
-def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=5, crx_method='standard'):
+def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False, include_idregex=False, method='langsize', min_methods=5, crx_method='standard', min_structure=0.0):
     """Preprocess and group by package directory, infer per group.
 
     Groups methods by their file's relative directory path, merging
@@ -360,7 +364,7 @@ def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFA
             gs = [sequences[i] for i in indices]
             gf = set(seq_files[i] for i in indices)
             f = ex.submit(_infer_group, label, gs, gf, project_root,
-                           min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method)
+                           min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure)
             futures[f] = label
 
         for f in as_completed(futures):
@@ -473,6 +477,7 @@ def analyze_directory(
     method='langsize',
     min_methods=5,
     crx_method='standard',
+    min_structure=0.0,
 ):
     """Scan a directory and run analysis for each language found.
 
@@ -511,6 +516,7 @@ def analyze_directory(
                 method=method,
                 min_methods=min_methods,
                 crx_method=crx_method,
+                min_structure=min_structure,
             )
         else:
             results[ext] = analyze_clusters(
@@ -680,6 +686,10 @@ def _parse_args(argv=None):
         "--crx-method", choices=["standard", "refined"], default="standard",
         help="CRX method: standard (default) or refined (cluster-then-infer, tighter grammars)",
     )
+    parser.add_argument(
+        "--min-structure", type=float, default=0.0,
+        help="Minimum structure score (0.0-1.0) to keep grammar. Flat bags of symbols below this are dropped (default: 0, keep all)",
+    )
     return parser.parse_args(argv)
 
 
@@ -703,6 +713,7 @@ def main():
         method=args.scoring_method,
         min_methods=args.min_methods,
         crx_method=args.crx_method,
+        min_structure=args.min_structure,
     )
 
     if args.json_flag or args.format == "json":

experiments/results/flask_gbnf.json

@@ -3,66 +3,71 @@
     "package": "examples/celery/src/task_app",
     "ext": ".py",
     "methods": 11,
-    "skip": "too_diverse"
+    "skip": "too_diverse",
+    "structure_score": 0
   },
   {
     "package": "examples/javascript/tests",
     "ext": ".py",
     "methods": 5,
-    "skip": "too_diverse"
+    "skip": "too_diverse",
+    "structure_score": 0
   },
   {
     "package": "examples/tutorial/flaskr",
     "ext": ".py",
     "methods": 18,
-    "skip": "too_diverse"
+    "skip": "too_diverse",
+    "structure_score": 0
   },
   {
     "package": "examples/tutorial/tests",
     "ext": ".py",
     "methods": 25,
-    "skip": "too_diverse"
+    "skip": "too_diverse",
+    "structure_score": 0
   },
   {
     "package": "src/flask",
     "ext": ".py",
     "methods": 216,
-    "skip": "too_diverse"
+    "skip": "too_diverse",
+    "structure_score": 0
   },
   {
     "package": "src/flask/json",
     "ext": ".py",
     "methods": 50,
-    "sore": "(TypeError+UUID+__html__+_app+_default+_json+_prepare_response_obj+_untag_scan+and+args+current_app+dict+dumps+else+for+fp+http_date+if+in+is+isinstance+item+items+iter+json+k+key+kwargs+len+list+loads+mimetype+next+not+obj+or+order+raise+response_class+return+s+self+serializer+setdefault+str+tag+tags+tuple+v+value)+.NotImplementedError+?",
+    "skip": "low_structure",
-    "gbnf": "root ::= (\"TypeError\" | \"UUID\" | \"__html__\" | \"_app\" | \"_default\" | \"_json\" | \"_prepare_response_obj\" | \"_untag_scan\" | \"and\" | \"args\" | \"current_app\" | \"dict\" | \"dumps\" | \"else\" | \"for\" | \"fp\" | \"http_date\" | \"if\" | \"in\" | \"is\" | \"isinstance\" | \"item\" | \"items\" | \"iter\" | \"json\" | \"k\" | \"key\" | \"kwargs\" | \"len\" | \"list\" | \"loads\" | \"mimetype\" | \"next\" | \"not\" | \"obj\" | \"or\" | \"order\" | \"raise\" | \"response_class\" | \"return\" | \"s\" | \"self\" | \"serializer\" | \"setdefault\" | \"str\" | \"tag\" | \"tags\" | \"tuple\" | \"v\" | \"value\")+ \"NotImplementedError\"*",
+    "structure_score": 0.042105263157894736
-    "gbnf_ok": true
   },
   {
     "package": "src/flask/sansio",
     "ext": ".py",
     "methods": 102,
-    "sore": "(Any+BlueprintSetupState+ValueError+__name__+_method_route+add_url_rule+and+app+append+callable+code+config+decorator+def+dict+else+endpoint+f+for+func+if+import_name+in+is+is not+lambda+name+not+options+or+os+path+raise+record_once+return+root_path+rstrip+rule+s+self+setdefault+state+static_folder+str+t+url_default_functions+value+values+view_func+view_functions)+",
+    "skip": "low_structure",
-    "gbnf": "root ::= (\"Any\" | \"BlueprintSetupState\" | \"ValueError\" | \"__name__\" | \"_method_route\" | \"add_url_rule\" | \"and\" | \"app\" | \"append\" | \"callable\" | \"code\" | \"config\" | \"decorator\" | \"def\" | \"dict\" | \"else\" | \"endpoint\" | \"f\" | \"for\" | \"func\" | \"if\" | \"import_name\" | \"in\" | \"is\" | \"is not\" | \"lambda\" | \"name\" | \"not\" | \"options\" | \"or\" | \"os\" | \"path\" | \"raise\" | \"record_once\" | \"return\" | \"root_path\" | \"rstrip\" | \"rule\" | \"s\" | \"self\" | \"setdefault\" | \"state\" | \"static_folder\" | \"str\" | \"t\" | \"url_default_functions\" | \"value\" | \"values\" | \"view_func\" | \"view_functions\")+",
+    "structure_score": 0.009523809523809525
-    "gbnf_ok": true
   },
   {
     "package": "tests",
     "ext": ".py",
     "methods": 962,
-    "sore": "(@+Blueprint+__name__+app+append+class+client+config+data+def+e+flask+get+in+index+isinstance+not+raise+render_template+request+return+route+rv+s+self+session+value)+",
+    "skip": "low_structure",
-    "gbnf": "root ::= (\"@\" | \"Blueprint\" | \"__name__\" | \"app\" | \"append\" | \"class\" | \"client\" | \"config\" | \"data\" | \"def\" | \"e\" | \"flask\" | \"get\" | \"in\" | \"index\" | \"isinstance\" | \"not\" | \"raise\" | \"render_template\" | \"request\" | \"return\" | \"route\" | \"rv\" | \"s\" | \"self\" | \"session\" | \"value\")+",
+    "structure_score": 0.021897810218978103
-    "gbnf_ok": true
   },
   {
     "package": "tests/test_apps",
     "ext": ".py",
     "methods": 3,
-    "skip": "too_diverse"
+    "skip": "too_diverse",
+    "structure_score": 0
   },
   {
     "package": "tests/test_apps/blueprintapp/apps",
     "ext": ".py",
     "methods": 4,
+    "skip": "",
+    "structure_score": 0.2857142857142857,
     "sore": "return.render_template+",
     "gbnf": "root ::= \"return\" \"render_template\"+",
     "gbnf_ok": true
@@ -71,13 +76,17 @@
     "package": "tests/type_check",
     "ext": ".py",
     "methods": 24,
-    "sore": "def?.(Generator+encode+for+in+iter+range+return+show+str+t+x+yield)+?.(Response+code+jsonify)+?.render_template+?.HTTPStatus?.stream_template+?.self?.OK?.name+?.template_name+?",
+    "skip": "",
-    "gbnf": "root ::= \"def\"? (\"Generator\" | \"encode\" | \"for\" | \"in\" | \"iter\" | \"range\" | \"return\" | \"show\" | \"str\" | \"t\" | \"x\" | \"yield\")* (\"Response\" | \"code\" | \"jsonify\")* \"render_template\"* \"HTTPStatus\"? \"stream_template\"* \"self\"? \"OK\"? \"name\"* \"template_name\"*",
+    "structure_score": 0.5597014925373135,
+    "sore": "def?.(Generator+encode+for+in+iter+range+return+show+str+t+x+yield)+?.stream_template+?.(Response+code+jsonify)+?.render_template+?.HTTPStatus?.name+?.self?.OK?.template_name+?",
+    "gbnf": "root ::= \"def\"? (\"Generator\" | \"encode\" | \"for\" | \"in\" | \"iter\" | \"range\" | \"return\" | \"show\" | \"str\" | \"t\" | \"x\" | \"yield\")* \"stream_template\"* (\"Response\" | \"code\" | \"jsonify\")* \"render_template\"* \"HTTPStatus\"? \"name\"* \"self\"? \"OK\"? \"template_name\"*",
     "gbnf_ok": true
   },
   {
     "package": "(other)",
     "ext": ".py",
-    "methods": 4
+    "methods": 4,
+    "skip": "",
+    "structure_score": 0
   }
 ]