fix: sanitize symbol extraction, fix malformed grammars

- Add sanitize_symbol() to extract clean identifiers from AST nodes
- Strip SORE special characters ((),+?*[]{}|\^$.) from symbols
- Extract just function name from multi-line expressions
- Skip single-char fragments (truncated identifiers)
- Filter empty symbols in preprocessing
- Wire iLocal context extraction into pipeline
- Result: RAGSAK malformed grammars 6→0, FastAPI clean
- 212 tests pass

bex/tag_preprocessor/analyze.py

@@ -463,6 +463,67 @@ def _filter_glob(files, include=None, exclude=None):
     return files
 
 
+def analyze_by_ilocal(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, context_strategy="dir", reduce=True):
+    """iLocal-style analysis: extract (context, sequence) pairs, reduce, infer.
+
+    Instead of hard-coding directory as grouping key, this extracts contexts
+    using a configurable strategy, then uses iLocal's reduce to merge contexts
+    with identical signatures.
+
+    Args:
+        context_strategy: one of 'dir', 'file', 'parent_dir', 'depth_2',
+                         'depth_3', 'imports', 'symbol_overlap'
+        reduce: whether to run iLocal reduce to merge similar contexts
+    """
+    from .ilocal_source import CONTEXT_STRATEGIES, reduce_contexts
+
+    t0 = time.time()
+    sequences, seq_files = _preprocess_files(file_paths)
+    if not sequences:
+        return []
+    _vprint(f"Preprocess: {len(sequences)} methods from {len(file_paths)} {extension} files ({time.time()-t0:.1f}s)")
+
+    strategy_fn = CONTEXT_STRATEGIES.get(context_strategy)
+    if not strategy_fn:
+        _vprint(f"Unknown context strategy: {context_strategy}, falling back to 'dir'")
+        strategy_fn = CONTEXT_STRATEGIES["dir"]
+
+    t1 = time.time()
+    context_groups = strategy_fn(sequences, seq_files, project_root)
+    _vprint(f"Context extraction ({context_strategy}): {len(context_groups)} contexts ({time.time()-t1:.1f}s)")
+
+    if reduce:
+        t2 = time.time()
+        before = len(context_groups)
+        context_groups = reduce_contexts(context_groups)
+        _vprint(f"Reduce: {before} → {len(context_groups)} contexts ({time.time()-t2:.1f}s)")
+
+    _vprint(f"Contexts: {len(context_groups)} groups")
+    for label, seqs in sorted(context_groups.items(), key=lambda x: -len(x[1]))[:10]:
+        _vprint(f"  ├ {label[:60]} ({len(seqs)} methods)")
+    if len(context_groups) > 10:
+        _vprint(f"  └ ... and {len(context_groups) - 10} more")
+
+    results = []
+    n_workers = os.cpu_count()
+    _vprint(f"Inferring {len(context_groups)} contexts across {n_workers} workers ...")
+    with ProcessPoolExecutor(max_workers=n_workers) as ex:
+        futures = {}
+        for label, seqs in context_groups.items():
+            f = ex.submit(_infer_group, label, seqs, set(), project_root,
+                           min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure)
+            futures[f] = label
+
+        done = 0
+        for f in as_completed(futures):
+            done += 1
+            if done % 20 == 0 or done == len(futures):
+                _vprint(f"  [{done}/{len(futures)}]")
+            results.append(f.result())
+
+    return results
+
+
 def analyze_directory(
     dir_path,
     min_coverage=DEFAULT_COVERAGE,
@@ -478,6 +539,7 @@ def analyze_directory(
     min_methods=3,
     crx_method='standard',
     min_structure=0.0,
+    context_strategy="dir",
 ):
     """Scan a directory and run analysis for each language found.
 
@@ -518,6 +580,21 @@ def analyze_directory(
                 crx_method=crx_method,
                 min_structure=min_structure,
             )
+        elif slice == "ilocal":
+            results[ext] = analyze_by_ilocal(
+                files, ext,
+                project_root=dir_path,
+                min_coverage=min_coverage,
+                prefer=prefer,
+                kmax=kmax,
+                include_kore=include_kore,
+                include_idregex=include_idregex,
+                method=method,
+                min_methods=min_methods,
+                crx_method=crx_method,
+                min_structure=min_structure,
+                context_strategy=context_strategy,
+            )
         else:
             results[ext] = analyze_clusters(
                 files, ext,
@@ -647,8 +724,12 @@ def _parse_args(argv=None):
         help="BEX core coverage threshold — outlier methods are removed until this fraction remains (default: 0.8)",
     )
     parser.add_argument(
-        "--slice", choices=["flat", "package"], default="flat",
-        help="Grouping strategy: flat (one per language) or package (per directory) (default: flat)",
+        "--slice", choices=["flat", "package", "ilocal"], default="flat",
+        help="Grouping strategy: flat (one per language), package (per directory), or ilocal (iLocal context-based) (default: flat)",
+    )
+    parser.add_argument(
+        "--context-strategy", choices=["dir", "file", "parent_dir", "depth_2", "depth_3", "imports", "symbol_overlap"], default="dir",
+        help="Context extraction strategy for ilocal slicing (default: dir)",
     )
     parser.add_argument(
         "--include",
@@ -714,6 +795,7 @@ def main():
         min_methods=args.min_methods,
         crx_method=args.crx_method,
         min_structure=args.min_structure,
+        context_strategy=args.context_strategy,
     )
 
     if args.json_flag or args.format == "json":

bex/tag_preprocessor/code.py

@@ -13,6 +13,51 @@ import re
 import sys
 from pathlib import Path
 
+# Pattern to extract the first identifier from a call expression.
+# Matches: "foo", "Foo.bar", "foo.Bar.baz" — stops at first ( or whitespace.
+_CALL_NAME_RE = re.compile(r"^([A-Za-z_][\w.]*)")
+
+
+# SORE special characters that break grammar parsing
+_SORE_SPECIAL = set("()+?*[]{}|\\^$.")
+
+
+def sanitize_symbol(text, capname):
+    """Sanitize extracted symbol text for grammar inference.
+
+    Multi-line expressions (constructor calls, if-blocks) break CRX and
+    produce malformed SOREs. This function:
+    - For call-like captures: extracts just the function/method name
+    - For others: extracts the first identifier-like token
+    - Strips all SORE special characters
+    - Falls back to empty string if nothing usable remains
+    """
+    text = text.strip()
+    if not text:
+        return text
+
+    # Collapse whitespace first
+    text = re.sub(r"\s+", " ", text)
+
+    # Extract first identifier (handles "foo", "Foo.bar", "foo.Bar.baz")
+    m = _CALL_NAME_RE.match(text)
+    if m:
+        name = m.group(1).rstrip(".")
+        if name:
+            return name
+
+    # Fallback: take first line, strip special chars
+    text = text.split("\n", 1)[0].strip()
+    # Remove any SORE special characters
+    cleaned = "".join(c for c in text if c not in _SORE_SPECIAL)
+    cleaned = cleaned.strip()
+    if len(cleaned) > 80:
+        cleaned = cleaned[:80]
+    # Skip single-char fragments (likely truncated identifiers)
+    if len(cleaned) <= 1:
+        return ""
+    return cleaned
+
 from tree_sitter import Language, Parser, Query, QueryCursor
 
 QUERIES_DIR = Path(__file__).parent / "queries"
@@ -191,7 +236,7 @@ def extract_arg_info(file_path, code):
         if not capname.startswith(BEHAVIORAL_PREFIXES):
             continue
         for node in nodes:
-            text = code[node.start_byte:node.end_byte].strip()
+            text = sanitize_symbol(code[node.start_byte:node.end_byte], capname)
             parent = node.parent
             if not parent:
                 continue
@@ -337,8 +382,9 @@ def preprocess_by_method(file_path: str, code: str):
         if not capname.startswith(BEHAVIORAL_PREFIXES):
             continue
         for node in nodes:
-            text = code[node.start_byte:node.end_byte].strip()
-            items.append((node.start_byte, capname, node, text))
+            text = sanitize_symbol(code[node.start_byte:node.end_byte], capname)
+            if text:  # skip empty/fragment symbols
+                items.append((node.start_byte, capname, node, text))
 
     items.sort(key=lambda x: x[0])
 
@@ -379,7 +425,9 @@ def preprocess(file_path: str, code: str):
         if not capname.startswith(BEHAVIORAL_PREFIXES):
             continue
         for node in nodes:
-            items.append((node.start_byte, capname, node, code[node.start_byte:node.end_byte].strip()))
+            text = sanitize_symbol(code[node.start_byte:node.end_byte], capname)
+            if text:  # skip empty/fragment symbols
+                items.append((node.start_byte, capname, node, text))
 
     items.sort(key=lambda x: x[0])
 

bex/tag_preprocessor/ilocal_source.py

@@ -0,0 +1,196 @@
+"""iLocal-style context extraction for source code.
+
+Adapts Bex 2007's iLocal algorithm from YAML/XML to source code.
+Instead of hard-coding directory as the grouping key, this module
+extracts (context, sequence) pairs and lets reduce find natural groupings.
+
+Context = structural position that determines the call sequence.
+For YAML: context = YAML key.
+For source code: context = file path component, class name, import set, etc.
+"""
+
+import os
+from collections import Counter, defaultdict
+
+
+def extract_contexts_by_dir(sequences, seq_files, project_root):
+    """Context = relative directory path (like current package grouping).
+
+    Returns dict: {context_key: [original_sequence, ...]}
+    """
+    contexts = defaultdict(list)
+    for seq, fp in zip(sequences, seq_files):
+        rel = os.path.relpath(os.path.dirname(fp), project_root)
+        if rel == ".":
+            rel = ""
+        contexts[rel].append(seq)
+    return dict(contexts)
+
+
+def extract_contexts_by_file(sequences, seq_files, project_root):
+    """Context = individual file path.
+
+    Returns dict: {context_key: [original_sequence, ...]}
+    """
+    contexts = defaultdict(list)
+    for seq, fp in zip(sequences, seq_files):
+        rel = os.path.relpath(fp, project_root)
+        contexts[rel].append(seq)
+    return dict(contexts)
+
+
+def extract_contexts_by_parent_dir(sequences, seq_files, project_root):
+    """Context = parent of directory (one level coarser than dir).
+
+    Returns dict: {context_key: [original_sequence, ...]}
+    """
+    contexts = defaultdict(list)
+    for seq, fp in zip(sequences, seq_files):
+        rel_dir = os.path.relpath(os.path.dirname(fp), project_root)
+        if rel_dir == ".":
+            rel_dir = ""
+        parts = rel_dir.replace(os.sep, "/").rstrip("/").split("/")
+        parent = "/".join(parts[:-1]) if len(parts) > 1 else ""
+        contexts[parent].append(seq)
+    return dict(contexts)
+
+
+def extract_contexts_by_file_depth(sequences, seq_files, project_root, depth=2):
+    """Context = first N components of file path.
+
+    Returns dict: {context_key: [original_sequence, ...]}
+    """
+    contexts = defaultdict(list)
+    for seq, fp in zip(sequences, seq_files):
+        rel = os.path.relpath(os.path.dirname(fp), project_root)
+        if rel == ".":
+            rel = ""
+        parts = rel.replace(os.sep, "/").rstrip("/").split("/")
+        key = "/".join(parts[:depth]) if parts and parts[0] else ""
+        contexts[key].append(seq)
+    return dict(contexts)
+
+
+def extract_contexts_by_import_set(sequences, seq_files, project_root):
+    """Context = unique import set of the file.
+
+    Methods from files with identical imports share a context.
+    """
+    contexts = defaultdict(list)
+    for seq, fp in zip(sequences, seq_files):
+        imports = _file_import_signature(fp)
+        contexts[imports].append(seq)
+    return dict(contexts)
+
+
+def _file_import_signature(fp):
+    """Build a canonical import signature for a file."""
+    try:
+        with open(fp) as f:
+            lines = [f.readline() for _ in range(100)]
+    except OSError:
+        return ""
+    imports = []
+    for line in lines:
+        stripped = line.strip()
+        if stripped.startswith("import ") or stripped.startswith("from "):
+            imports.append(stripped)
+    return "|".join(sorted(set(imports)))
+
+
+def extract_contexts_by_symbol_overlap(sequences, seq_files, project_root, min_overlap=0.5):
+    """Context = cluster of methods that share >= min_overlap of their symbols.
+
+    Uses union-find for transitive clustering.
+    """
+    symbol_seqs = [[text for _, text, _ in seq] for seq in sequences]
+    n = len(symbol_seqs)
+    if n == 0:
+        return {}
+
+    # Build symbol sets
+    sym_sets = [set(s) for s in symbol_seqs]
+
+    # Union-find
+    parent = list(range(n))
+
+    def find(x):
+        while parent[x] != x:
+            parent[x] = parent[parent[x]]
+            x = parent[x]
+        return x
+
+    def union(x, y):
+        px, py = find(x), find(y)
+        if px != py:
+            parent[px] = py
+
+    # Cluster by symbol overlap
+    for i in range(n):
+        for j in range(i + 1, n):
+            if not sym_sets[i] or not sym_sets[j]:
+                continue
+            overlap = len(sym_sets[i] & sym_sets[j]) / max(len(sym_sets[i] | sym_sets[j]), 1)
+            if overlap >= min_overlap:
+                union(i, j)
+
+    # Group by cluster — keep original tuples
+    clusters = defaultdict(list)
+    for i in range(n):
+        root = find(i)
+        clusters[root].append(sequences[i])
+
+    return {f"cluster_{k}": v for k, v in clusters.items()}
+
+
+def reduce_contexts(context_groups):
+    """iLocal reduce: merge contexts with identical signatures.
+
+    Signature = sorted set of (length, first_symbol, last_symbol) across
+    all sequences in the context.
+
+    Args:
+        context_groups: dict of {context_key: [original_sequence, ...]}
+
+    Returns:
+        dict: {generalized_context: [original_sequence, ...]}
+    """
+    if not context_groups:
+        return {}
+
+    signature_map = {}
+    for ctx, seqs in context_groups.items():
+        sig_parts = []
+        for s in seqs:
+            symbol_seq = [text for _, text, _ in s] if s and isinstance(s[0], tuple) else s
+            first = symbol_seq[0] if symbol_seq else "∅"
+            last = symbol_seq[-1] if symbol_seq else "∅"
+            sig_parts.append((len(symbol_seq), first, last))
+        signature = tuple(sorted(set(sig_parts)))
+        if signature not in signature_map:
+            signature_map[signature] = []
+        signature_map[signature].append(ctx)
+
+    result = {}
+    for sig, ctx_list in signature_map.items():
+        merged_ctx = "|".join(sorted(ctx_list)[:3])  # cap label length
+        if len(ctx_list) > 3:
+            merged_ctx += f"|+{len(ctx_list)-3}"
+        merged_seqs = []
+        for ctx in ctx_list:
+            merged_seqs.extend(context_groups[ctx])
+        result[merged_ctx] = merged_seqs
+
+    return result
+
+
+# Context extraction strategies registry
+CONTEXT_STRATEGIES = {
+    "dir": extract_contexts_by_dir,
+    "file": extract_contexts_by_file,
+    "parent_dir": extract_contexts_by_parent_dir,
+    "depth_2": lambda s, f, p: extract_contexts_by_file_depth(s, f, p, depth=2),
+    "depth_3": lambda s, f, p: extract_contexts_by_file_depth(s, f, p, depth=3),
+    "imports": extract_contexts_by_import_set,
+    "symbol_overlap": extract_contexts_by_symbol_overlap,
+}