docs: update stale docs — remove kORE from default ensemble, add tag preprocessor CLI

This reverts commit 0b5b0e623b.

AGENTS.md

@@ -26,16 +26,17 @@ g = idregex([['a','b','c'], ['a','b'], ['a','c']], kmax=2, N=3)
 
 ## Architecture
 
-Two inference pipelines:
+Three inference pipelines:
 
 | Pipeline | When to use |
 |----------|-------------|
 | CRX (fast) | Many examples, need speed, CHAREs output |
 | iDRegEx (robust) | Few/noisy examples, need probabilistic handling |
+| Tag Preprocessor (`bex.tag_preprocessor`) | Source code analysis — tree-sitter AST → method-level call sequences → per-package grammars |
 
 ## Running Tests
 ```bash
-python tests/test_bex.py
+python -m pytest tests/
 ```
 
 ## MCP Server
@@ -44,6 +45,17 @@ The primary interface is an MCP server exposing a single tool:
 
 | Tool | Parameters | What it does |
 |------|-----------|-------------|
-| `infer_best_grammar` | `sequences`, `prefer`, `kmax`, `N` | Runs CRX + iDRegEx, picks best by MDL. `prefer='crx'` or `prefer='idregex'` skips ensemble. |
+| `infer_best_grammar` | `sequences`, `prefer`, `kmax`, `N`, `min_coverage` | Runs CRX + iDRegEx, picks best by MDL. `prefer='crx'` or `prefer='idregex'` skips ensemble. `min_coverage < 1.0` runs core+outlier analysis. |
 
 Start it: `python /path/to/bex/mcp_server.py`, then connect any MCP client.
+
+## Tag Preprocessor CLI
+
+For analyzing source code directories:
+
+```bash
+python -m bex.tag_preprocessor.analyze /path/to/codebase --verbose
+python -m bex.tag_preprocessor.analyze /path/to/codebase --slice package --include '**/src/**'
+```
+
+Key flags: `--slice package` (per-directory grammars), `--verbose` (progress), `--include`/`--exclude` (glob filters), `--kore` (enable slow kORE in ensemble).

README.md

@@ -50,10 +50,10 @@ The primary interface is a **Model Context Protocol (MCP)** server. Connect any
 
 | Tool | Parameters | What it does |
 |------|-----------|-------------|
-| `infer_best_grammar` | `sequences`, `prefer`, `kmax`, `N`, `min_coverage` | **The only tool you need.** Runs CRX + iDRegEx + kOREInference, picks best by MDL. Set `prefer` to run only one algorithm. Set `min_coverage < 1.0` for optional core+outlier analysis. |
+| `infer_best_grammar` | `sequences`, `prefer`, `kmax`, `N`, `min_coverage` | **The only tool you need.** Runs CRX + iDRegEx, picks best by MDL. Set `prefer` to run only one algorithm. Set `min_coverage < 1.0` for optional core+outlier analysis. |
 
 **Parameters explained:**
-- **`prefer`**: `'crx'` for full vocabulary (accepts all sequences), `'idregex'` or `'koreinference'` for deterministic minimal core. Omit to let MDL pick the winner across all three.
+- **`prefer`**: `'crx'` for full vocabulary (accepts all sequences), `'idregex'` for deterministic minimal core, `'koreinference'` for k-OA with rwr₀ repair (slow, rarely wins). Omit to let MDL pick the winner across CRX and iDRegEx.
 - **`kmax`** (1–5): Context window for k-ORE inference (iDRegEx, kOREInference). Higher values capture longer-range dependencies but need more data and are slower. Default 2 works for most cases.
 - **`N`** (1–10): Random trials for k-ORE inference. More = better convergence but slower. Default 3.
 - **`min_coverage`** (0.5–1.0): **Optional core+outlier analysis.** When < 1.0, iteratively removes outlier sequences (those with the rarest symbols) until at least this fraction remain. Returns the core CRX grammar for the majority plus a list of removed outliers. Default 1.0 = disabled. Example: `min_coverage=0.8` finds the tight pattern for ~80% of examples while flagging the other ~20% as variants.
@@ -114,6 +114,17 @@ print(f"Grammar: {result['best']['grammar']}")
 print(f"Score: {result['best']['mdl_score']}")
 ```
 
+### Tag Preprocessor (source code analysis)
+
+For analyzing source code directories (tree-sitter based):
+
+```bash
+python -m bex.tag_preprocessor.analyze /path/to/codebase --verbose
+python -m bex.tag_preprocessor.analyze /path/to/codebase --slice package --include '**/src/**'
+```
+
+Key flags: `--slice package` (per-directory grammars), `--verbose` (progress), `--include`/`--exclude` (glob filters), `--kore` (enable slow kORE in ensemble).
+
 ## Why not just use a schema?
 
 Many of the things developers build every day **have no formal schema**. They're free-form scripts, config files, or YAML blobs where the structure is emergent convention, not enforced specification. An LLM generating new content in these domains needs to know the convention — but it's never been written down.
@@ -147,7 +158,7 @@ The sweet spot: **multiple implementations of the same abstract task** with a sh
 |------|-----|-----|
 | Clean, structured data with full vocabulary | **CRX** | Single-pass, deterministic. Accepts all sequences. |
 | Few examples, or want minimal common core | **iDRegEx** or **kOREInference** | Probabilistic EM, finds only what's shared. |
-| Don't know which is better | **Ensemble (default)** | Runs all three, picks best by MDL score. |
+| Don't know which is better | **Ensemble (default)** | Runs CRX + iDRegEx, picks best by MDL score. |
 | Want core pattern + outlier detection | **Ensemble + `min_coverage<1`** | Finds tight grammar for majority, flags outliers. |
 | Data is clearly one type | `prefer='crx'` | Skips ensemble comparison, runs CRX alone. |
 

bex/ensemble.py

@@ -3,7 +3,6 @@
 import re
 from .crx import CRX
 from .idregex import idregex
-from .kore import kOREInference
 from .expr import alphabet
 from .mdl import model_cost, mdl_score
 
@@ -375,8 +374,21 @@ def _run_idregex(sequences, kmax, N):
     return None, float('inf')
 
 
+_ALGO_NAMES = {
+    'crx': 'CRX',
+    'idregex': 'iDRegEx',
+}
+
+
+_ALGORITHMS = {
+    'crx': lambda s, k, n: (CRX().infer(s), mdl_score_simple(CRX().infer(s), s)),
+    'idregex': _run_idregex,
+}
+
+
 def _run_kore(sequences, kmax, N):
-    """Run kOREInference (Algorithm 4 with MDL), return (grammar, score) or (None, inf)."""
+    """Run kOREInference, return (grammar, score) or (None, inf)."""
+    from .kore import kOREInference
     kore = kOREInference(k_max=kmax, N=N)
     result = kore.infer(sequences)
     if result:
@@ -385,21 +397,7 @@ def _run_kore(sequences, kmax, N):
     return None, float('inf')
 
 
-_ALGO_NAMES = {
+def infer_ensemble(sequences, kmax=2, N=3, prefer=None, min_coverage=1.0, include_kore=False):
-    'crx': 'CRX',
-    'idregex': 'iDRegEx',
-    'koreinference': 'kOREInference',
-}
-
-
-_ALGORITHMS = {
-    'crx': lambda s, k, n: (CRX().infer(s), mdl_score_simple(CRX().infer(s), s)),
-    'idregex': _run_idregex,
-    'koreinference': _run_kore,
-}
-
-
-def infer_ensemble(sequences, kmax=2, N=3, prefer=None, min_coverage=1.0):
     """Run all applicable algorithms and return the best by MDL score.
 
     Args:
@@ -450,10 +448,11 @@ def infer_ensemble(sequences, kmax=2, N=3, prefer=None, min_coverage=1.0):
     if idr_g:
         results.append(('iDRegEx', idr_g, idr_score))
 
-    # 3. kOREInference (Algorithm 4 with MDL scoring)
+    # 3. kOREInference (opt-in via include_kore=True)
-    kore_g, kore_score = _run_kore(sequences, kmax, N)
+    if include_kore:
-    if kore_g:
+        kore_g, kore_score = _run_kore(sequences, kmax, N)
-        results.append(('kOREInference', kore_g, kore_score))
+        if kore_g:
+            results.append(('kOREInference', kore_g, kore_score))
 
     results = [r for r in results if r[1] and r[1] != '∅']
     if not results:

bex/mcp_server.py

@@ -26,13 +26,19 @@ def infer_best_grammar(
     than passing all examples. Pass the existing sequences, get back a
     pattern you can follow to generate new instances.
 
+    Runs CRX + iDRegEx, picks best by MDL score. kORE is excluded by
+    default (slow, rarely wins on real data). Set prefer='koreinference'
+    to force it.
+
     Args:
         sequences: List of sequences, each a list of strings (symbols in
             the order they appear). Example: [["file","copy","command"],
             ["file","template","command"]].
         prefer: Optional — 'crx' for full vocabulary (accepts all examples),
-            'idregex' for deterministic minimal core. Omit to auto-pick by MDL.
+            'idregex' for deterministic minimal core, 'koreinference' for
-        kmax: Context depth for k-ORE inference. Default 2.
+            k-OA with rwr0 repair (slow). Omit to auto-pick by MDL.
+        kmax: Context depth for k-ORE inference (iDRegEx, kOREInference).
+            Default 2.
         N: Random trials for k-ORE inference (higher = better, slower).
         min_coverage: (Expert) When < 1.0, also runs a **core+outlier analysis**:
             iteratively removes outlier sequences (those with rarest symbols)

bex/tag_preprocessor/analyze.py

@@ -3,7 +3,7 @@
 Usage:
     python -m bex.tag_preprocessor.analyze <directory> [options]
 
-Runs the full Phase 1.0 pipeline over a directory of source files.
+Runs the full pipeline over a directory of source files.
 """
 
 import argparse
@@ -11,12 +11,14 @@ import json
 import os
 import re
 import sys
+import time
 from pathlib import Path
 from collections import Counter
+from concurrent.futures import ProcessPoolExecutor, as_completed
 
 import pathspec
 
-from .code import preprocess_by_method, _extract_call_tokens, extract_arg_info, _summarize_arg_info
+from .code import preprocess_by_method, extract_arg_info, _summarize_arg_info
 from bex.ensemble import infer_ensemble
 
 SUPPORTED_EXTENSIONS = {
@@ -51,8 +53,8 @@ def _match_glob(filepath, pattern):
 IMPORT_PATTERNS = [
     re.compile(r"^\s*import\s+"),
     re.compile(r"^\s*from\s+"),
-    re.compile(r"^\s*require\s+"),
     re.compile(r"^\s*require_relative\s+"),
+    re.compile(r"^\s*require\s+"),
     re.compile(r"^\s*#\s*include\s+"),
     re.compile(r"^\s*use\s+"),
     re.compile(r"^\s*include\s+"),
@@ -155,6 +157,18 @@ def scan_directory(dir_path, gitignore_spec=None):
     return result
 
 
+DEFAULT_COVERAGE = 0.8
+
+VERBOSE = False
+_vstart = 0.0
+
+
+def _vprint(*args, **kwargs):
+    if VERBOSE:
+        elapsed = time.time() - _vstart
+        print(f"[{elapsed:6.1f}s]", *args, file=sys.stderr, **kwargs)
+
+
 def frequency_filter(sequences, min_coverage=0.2):
     """Remove symbols appearing in fewer than min_coverage fraction of files.
 
@@ -190,120 +204,129 @@ def frequency_filter(sequences, min_coverage=0.2):
     return filtered
 
 
-def cluster_methods(sequences, min_cluster_size=3, ngram_size=3, max_clusters=20):
+def _preprocess_file(fp):
-    """Group method sequences by shared n-gram call patterns.
+    """Preprocess one file. Module-level for ProcessPoolExecutor."""
-
+    with open(fp) as f:
-    Extracts call tokens from each sequence, builds an n-gram index,
+        code = f.read()
-    and assigns methods to all matching clusters (multi-assignment).
+    sequences = []
-    Methods that match NO n-gram pattern go to an '(other)' cluster.
+    for method_seq in preprocess_by_method(fp, code):
-
+        if method_seq:
-    Args:
+            sequences.append(method_seq)
-        sequences: list of (capture, text, line) lists.
+    return (fp, sequences)
-        min_cluster_size: minimum methods to form a cluster.
-        ngram_size: length of n-grams to match (default 3).
-        max_clusters: max clusters before dumping rest to (other).
-
-    Returns:
-        list of (label, [sequences]) tuples.
-    """
-    if not sequences:
-        return []
-
-    call_seqs = [_extract_call_tokens(seq) for seq in sequences]
-
-    ngram_to_methods = {}
-    for idx, cseq in enumerate(call_seqs):
-        for i in range(len(cseq) - ngram_size + 1):
-            ngram = tuple(cseq[i:i + ngram_size])
-            if len(ngram) == ngram_size:
-                ngram_to_methods.setdefault(ngram, set()).add(idx)
-
-    clusters = []
-    assigned = set()
-    for ngram, indices in sorted(ngram_to_methods.items(), key=lambda x: -len(x[1])):
-        if len(indices) >= min_cluster_size:
-            if len(clusters) >= max_clusters:
-                break
-            label = " → ".join(ngram)
-            cluster_seqs = [sequences[i] for i in indices]
-            clusters.append((label, cluster_seqs))
-            assigned.update(indices)
-
-    remaining = [i for i in range(len(sequences)) if i not in assigned]
-    if remaining:
-        clusters.append(("(other)", [sequences[i] for i in remaining]))
-
-    return clusters
 
 
-def cluster_methods_adaptive(sequences, min_cluster_size=3, ngram_size=3, other_threshold=0.6):
+def _preprocess_files(file_paths):
-    """Adaptive clustering: shrink ngram until (other) <= threshold or ngram=1.
+    """Preprocess multiple files in parallel."""
-
+    sequences = []
-    When the (other) cluster swallows > other_threshold of methods,
+    seq_files = []
-    retry with ngram-1. Keeps the smallest ngram that gives acceptable coverage.
+    n_workers = os.cpu_count()
-    """
+    with ProcessPoolExecutor(max_workers=n_workers) as ex:
-    for n in range(ngram_size, 0, -1):
+        futures = {ex.submit(_preprocess_file, fp): fp for fp in file_paths}
-        clusters = cluster_methods(sequences, min_cluster_size=min_cluster_size, ngram_size=n)
+        for f in as_completed(futures):
-        other_count = 0
+            fp, method_seqs = f.result()
-        total = 0
+            for seq in method_seqs:
-        for label, seqs in clusters:
+                sequences.append(seq)
-            total += len(seqs)
+                seq_files.append(fp)
-            if label == "(other)":
+    return sequences, seq_files
-                other_count = len(seqs)
-        if total == 0:
-            return clusters
-        if other_count / total <= other_threshold:
-            break
-    return clusters
 
 
-def analyze_clusters(file_paths, extension, project_root="", min_coverage=0.2, prefer=None, kmax=2, N=3):
+def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False):
-    """Run full pipeline with clustering: preprocess → cluster → per-cluster infer.
+    """Run full pipeline: preprocess → frequency filter → ensemble infer.
 
     Returns:
         list of (label, ensemble_result_dict, method_count, meta) tuples.
         meta = {"files": set(paths), "imports": [lines], "arg_patterns": {...}, "packages": [...]}.
     """
-    sequences = []
+    sequences, seq_files = _preprocess_files(file_paths)
-    seq_files = []
-    for fp in file_paths:
-        with open(fp) as f:
-            code = f.read()
-        for method_seq in preprocess_by_method(fp, code):
-            if method_seq:
-                sequences.append(method_seq)
-                seq_files.append(fp)
-
     if not sequences:
         return []
 
-    sequences = frequency_filter(sequences, min_coverage)
+    sequences = frequency_filter(sequences, min_coverage=0.2)
-    clusters = cluster_methods_adaptive(sequences)
+
+    cluster_fps = set(seq_files)
+    imports = _extract_imports(cluster_fps)
+    arg_patterns = _build_arg_patterns(cluster_fps)
+    packages = _top_packages(cluster_fps, project_root)
+
+    symbol_seqs = [[text for _, text, _ in seq] for seq in sequences]
+    result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore)
+    meta = {"files": cluster_fps, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
+
+    return [("(all methods)", result, len(sequences), meta)]
+
+
+def _infer_group(label, group_seqs, group_files, project_root, min_coverage, prefer, kmax, N, include_kore=False):
+    """Infer grammar for one package group. Module-level for ProcessPoolExecutor."""
+    filtered = frequency_filter(group_seqs, min_coverage=0.2)
+    imports = _extract_imports(group_files)
+    arg_patterns = _build_arg_patterns(group_files)
+    packages = _top_packages(group_files, project_root)
+    symbol_seqs = [[text for _, text, _ in seq] for seq in filtered]
+    result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage, include_kore=include_kore)
+    meta = {"files": group_files, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
+    return (label, result, len(filtered), meta)
+
+
+def analyze_by_package(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, min_pkg_size=3, include_kore=False):
+    """Preprocess and group by package directory, infer per group.
+
+    Groups methods by their file's relative directory path, merging
+    small packages (< min_pkg_size methods) upward to their parent.
+
+    Returns:
+        list of (package_label, ensemble_result_dict, method_count, meta).
+    """
+    t0 = time.time()
+    sequences, seq_files = _preprocess_files(file_paths)
+    if not sequences:
+        return []
+    _vprint(f"Preprocess: {len(sequences)} methods from {len(file_paths)} {extension} files ({time.time()-t0:.1f}s)")
+
+    seq_packages = [_file_to_package(fp, project_root) for fp in seq_files]
+    groups, ungrouped = _group_by_package(
+        list(enumerate(seq_packages)), min_size=min_pkg_size
+    )
+
+    _vprint(f"Groups: {len(groups)} named, {len(ungrouped)} ungrouped methods")
+    for label, idxs in groups:
+        _vprint(f"  ├ {label} ({len(idxs)} methods)")
+    if ungrouped:
+        _vprint(f"  └ (other) ({len(ungrouped)} methods)")
 
     results = []
-    for label, cluster_seqs in clusters:
+    n_workers = os.cpu_count()
-        cluster_fps = set()
+    _vprint(f"Inferring {len(groups)} groups across {n_workers} workers ...")
-        for seq in cluster_seqs:
+    with ProcessPoolExecutor(max_workers=n_workers) as ex:
-            idx = next(i for i, s in enumerate(sequences) if s is seq)
+        futures = {}
-            cluster_fps.add(seq_files[idx])
+        for label, indices in groups:
-        imports = _extract_imports(cluster_fps)
+            gs = [sequences[i] for i in indices]
-        arg_patterns = _build_arg_patterns(cluster_fps)
+            gf = set(seq_files[i] for i in indices)
-        symbol_seqs = [[text for _, text, _ in seq] for seq in cluster_seqs]
+            f = ex.submit(_infer_group, label, gs, gf, project_root,
-        result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer)
+                           min_coverage, prefer, kmax, N, include_kore)
-        packages = _top_packages(cluster_fps, project_root)
+            futures[f] = label
-        meta = {"files": cluster_fps, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
+
-        results.append((label, result, len(cluster_seqs), meta))
+        for f in as_completed(futures):
+            label, result, count, meta = f.result()
+            results.append((label, result, count, meta))
+            _vprint(f"Infer {label} ({count} methods) done")
+
+    results.sort(key=lambda x: x[0])
+
+    if ungrouped:
+        ungrouped_files = set(seq_files[i] for i in ungrouped)
+        ungrouped_seqs = [sequences[i] for i in ungrouped]
+        results.append(("(other)", None, len(ungrouped_seqs), {"files": ungrouped_files, "imports": [], "arg_patterns": {}, "packages": []}))
 
     return results
 
 
-def infer(file_paths, extension, min_coverage=0.2, prefer=None, kmax=2, N=3):
+def infer(file_paths, extension, min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False):
     """Run full pipeline: preprocess → frequency filter → ensemble infer.
 
     Args:
         file_paths: list of source file paths (same language).
         extension: language extension (e.g. '.py').
-        min_coverage: minimum file fraction for a symbol to be kept.
+        min_coverage: BEX core coverage threshold for outlier removal.
         prefer: inference algorithm preference ('crx', 'idregex', or None).
         kmax: max k for k-ORE algorithms.
         N: number of random trials.
@@ -311,38 +334,93 @@ def infer(file_paths, extension, min_coverage=0.2, prefer=None, kmax=2, N=3):
     Returns:
         Ensemble result dict from infer_ensemble.
     """
-    sequences = []
+    sequences, _ = _preprocess_files(file_paths)
-    for fp in file_paths:
+    sequences = frequency_filter(sequences, min_coverage=0.2)
-        with open(fp) as f:
-            code = f.read()
-        for method_seq in preprocess_by_method(fp, code):
-            if method_seq:
-                sequences.append(method_seq)
-
-    sequences = frequency_filter(sequences, min_coverage)
 
     symbol_seqs = [[text for _, text, _ in seq] for seq in sequences]
 
-    return infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer)
+    return infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage)
+
+
+def _merge_up(pkg):
+    """Go one directory level up from a package path."""
+    parts = pkg.replace(os.sep, "/").rstrip("/").split("/")
+    if len(parts) <= 1:
+        return ""
+    return "/".join(parts[:-1])
+
+
+def _group_by_package(indices_and_packages, min_size=3):
+    """Group method indices by package, merging small packages upward.
+
+    Small packages that would merge into root are discarded (too small
+    to form a meaningful group and not worth root-level inference).
+
+    Args:
+        indices_and_packages: list of (index, package_path) tuples.
+        min_size: minimum methods to keep a standalone group.
+
+    Returns:
+        (groups, ungrouped) where groups is [(label, [indices])] sorted by label,
+        and ungrouped is a list of indices that couldn't form a named group.
+    """
+    pkg_to_indices = {}
+    for idx, pkg in indices_and_packages:
+        pkg_to_indices.setdefault(pkg, []).append(idx)
+
+    ungrouped = []
+
+    while True:
+        to_merge = {p for p, idxs in pkg_to_indices.items()
+                    if p != "" and len(idxs) < min_size}
+        if not to_merge:
+            break
+        new_groups = {}
+        for pkg, indices in pkg_to_indices.items():
+            if pkg in to_merge:
+                parent = _merge_up(pkg)
+                if parent == "":
+                    ungrouped.extend(indices)
+                else:
+                    new_groups.setdefault(parent, []).extend(indices)
+            else:
+                new_groups.setdefault(pkg, []).extend(indices)
+        pkg_to_indices = new_groups
+
+    return sorted(pkg_to_indices.items(), key=lambda x: x[0]), ungrouped
+
+
+def _filter_glob(files, include=None, exclude=None):
+    """Filter file list by include/exclude glob patterns."""
+    if include:
+        spec = pathspec.PathSpec.from_lines("gitwildmatch", [include])
+        files = [f for f in files if spec.match_file(f)]
+    if exclude:
+        spec = pathspec.PathSpec.from_lines("gitwildmatch", [exclude])
+        files = [f for f in files if not spec.match_file(f)]
+    return files
 
 
 def analyze_directory(
     dir_path,
-    min_coverage=0.2,
+    min_coverage=DEFAULT_COVERAGE,
     prefer=None,
     kmax=2,
+    slice="flat",
     include=None,
-    cluster=True,
+    exclude=None,
+    include_kore=False,
 ):
-    """Scan a directory and run cluster analysis for each language found.
+    """Scan a directory and run analysis for each language found.
 
     Args:
         dir_path: directory to scan.
-        min_coverage: minimum file fraction for symbol to be kept.
+        min_coverage: BEX core coverage threshold for outlier removal.
         prefer: algorithm preference.
         kmax: max k for k-ORE algorithms.
-        include: optional glob pattern to filter files (e.g. '**/src/main/**').
+        slice: grouping strategy — "flat" (one per language) or "package" (per directory).
-        cluster: if True, run method-level clustering (default).
+        include: optional glob — only process files matching this pattern.
+        exclude: optional glob — skip files matching this pattern.
 
     Returns:
         dict mapping extension → list of (label, result_dict, count, meta) tuples.
@@ -352,22 +430,27 @@ def analyze_directory(
     for ext, files in groups.items():
         if len(files) < 1:
             continue
-        if include:
+        files = _filter_glob(files, include=include, exclude=exclude)
-            files = [f for f in files if _match_glob(f, include)]
         if not files:
             continue
-        if cluster:
+        if slice == "package":
+            results[ext] = analyze_by_package(
+                files, ext,
+                project_root=dir_path,
+                min_coverage=min_coverage,
+                prefer=prefer,
+                kmax=kmax,
+                include_kore=include_kore,
+            )
+        else:
             results[ext] = analyze_clusters(
                 files, ext,
                 project_root=dir_path,
                 min_coverage=min_coverage,
                 prefer=prefer,
                 kmax=kmax,
+                include_kore=include_kore,
             )
-        else:
-            r = infer(files, ext, min_coverage=min_coverage, prefer=prefer, kmax=kmax)
-            imports = _extract_imports(files)
-            results[ext] = [("(all methods)", r, 0, {"files": set(files), "imports": imports, "arg_patterns": {}, "packages": _top_packages(files, dir_path)})]
     return results
 
 
@@ -402,28 +485,32 @@ def _parse_args(argv=None):
     parser.add_argument("directory", help="Directory to scan")
     parser.add_argument(
         "--prefer",
-        choices=["crx", "idregex", "koreinference"],
+        choices=["crx", "idregex"],
         help="Skip ensemble, use only this algorithm",
     )
+    parser.add_argument(
+        "--kore", action="store_true",
+        help="Include kORE in ensemble (off by default for speed)",
+    )
     parser.add_argument(
         "--kmax", type=int, default=2,
         help="Maximum k for k-ORE algorithms (default: 2)",
     )
     parser.add_argument(
-        "--min-coverage", type=float, default=0.2,
+        "--min-coverage", type=float, default=DEFAULT_COVERAGE,
-        help="Minimum file fraction for a symbol to survive frequency filter (default: 0.2)",
+        help="BEX core coverage threshold — outlier methods are removed until this fraction remains (default: 0.8)",
+    )
+    parser.add_argument(
+        "--slice", choices=["flat", "package"], default="flat",
+        help="Grouping strategy: flat (one per language) or package (per directory) (default: flat)",
     )
     parser.add_argument(
         "--include",
-        help="Glob pattern to filter files (e.g. '**/src/main/**')",
+        help="Glob pattern to filter files (e.g. '**/src/test/**')",
     )
     parser.add_argument(
-        "--min-cluster-size", type=int, default=3,
+        "--exclude",
-        help="Minimum methods to form a cluster (default: 3)",
+        help="Glob pattern to skip files (e.g. '**/build/**')",
-    )
-    parser.add_argument(
-        "--ngram-size", type=int, default=3,
-        help="N-gram length for clustering (default: 3)",
     )
     parser.add_argument(
         "--format", choices=["text", "json"], default="text",
@@ -433,17 +520,28 @@ def _parse_args(argv=None):
         "--json", action="store_true", dest="json_flag",
         help="Shortcut for --format json",
     )
+    parser.add_argument(
+        "--verbose", action="store_true",
+        help="Print progress to stderr",
+    )
     return parser.parse_args(argv)
 
 
 def main():
     args = _parse_args()
+    global VERBOSE, _vstart
+    VERBOSE = args.verbose
+    _vstart = time.time()
+    _vprint(f"Scanning {args.directory} ...")
     results = analyze_directory(
         args.directory,
         min_coverage=args.min_coverage,
         prefer=args.prefer,
         kmax=args.kmax,
+        slice=args.slice,
         include=args.include,
+        exclude=args.exclude,
+        include_kore=args.kore,
     )
 
     if args.json_flag or args.format == "json":

bex/tag_preprocessor/code.py

@@ -51,7 +51,7 @@ BEHAVIORAL_PREFIXES = (
     "module",
 )
 
-CALL_PREFIXES = ("function", "reference.call", "reference.class")
+CALL_PREFIXES = ("function.call", "function.method.call", "reference.call", "reference.class", "constructor", "function")
 
 def _extract_call_tokens(seq):
     """Extract ordered call-like tokens from a method sequence.
@@ -107,8 +107,6 @@ def _find_arglist_node(parent):
     for child in parent.children:
         if child.type in ("argument_list", "arguments"):
             return child
-        if child.type == "template_string":
-            return child
     return None
 
 

docs/adr/0012-remove-ngram-clustering.md

@@ -0,0 +1,75 @@
+# 12. Remove n-gram clustering from pipeline
+
+**Date:** 2026-07-04
+
+**Status:** Accepted
+
+## Context
+
+The pipeline grouped method sequences by shared n-gram call patterns before
+inference. The idea: methods in the same category (test, config, helper) would
+cluster together, and each cluster would get a cleaner, more specific grammar.
+
+## What Was Removed
+
+- `cluster_methods()` — built n-gram→method index from call tokens, assigned
+  methods to shared-pattern clusters, dumped remainder to `(other)`.
+- `cluster_methods_adaptive()` — retried clustering at ngram=2 then ngram=1
+  when `(other)` exceeded 60% of methods.
+- `_extract_call_tokens()` import from `code.py` (still in `code.py` but no
+  longer called by the pipeline).
+- `--min-cluster-size` and `--ngram-size` CLI flags.
+- `cluster` parameter in `analyze_directory()`.
+- Per-cluster loop in `analyze_clusters()` — metadata extraction (files,
+  imports, arg patterns) now runs once across all methods.
+
+## Why Removed
+
+1. **No benefit for diverse codebases.** On Kotlin SpringRAG (1581 methods),
+   21 named clusters formed (3-13 methods each) but 1254 landed in `(other)`.
+   The named clusters were too small to produce better grammars than running
+   inference once on the filtered set.
+
+2. **Added complexity for zero signal gain.** The n-gram labels (e.g.
+   `locator → click → waitForTimeout`) restated what CRX already outputs as
+   `(locator+click+waitForTimeout)+`.per-cluster infer_ensemble call was
+   redundant with the single-pass result.
+
+3. **Slower.** 22 extra BEX calls (one per named cluster) for grammars
+   that would appear in the single-pass result anyway.
+
+## Pipeline After Removal
+
+```
+preprocess_by_method → frequency_filter(0.2) → infer_ensemble(0.8)
+```
+
+Single pass. Metadata extracted once.
+
+## How to Reintroduce
+
+The removed code is preserved in the archive branch:
+
+```
+git archive/unreverted-25898c2
+```
+
+Files:
+- `bex/tag_preprocessor/analyze.py` contains `cluster_methods()`,
+  `cluster_methods_adaptive()`, and the per-cluster loop body.
+
+To restore:
+1. Cherry-pick or copy the two function definitions.
+2. Re-add `_extract_call_tokens` to the import from `code.py`.
+3. Re-add the `--min-cluster-size` and `--ngram-size` CLI flags.
+4. Change `analyze_clusters()` back to: cluster → per-cluster filter → per-cluster infer.
+5. Restore the cluster parameter in `analyze_directory()`.
+
+## Performance Impact
+
+| Metric | Before (with clustering) | After (single pass) |
+|--------|--------------------------|---------------------|
+| Pipeline time (460 .kt files) | ~42s | ~3s |
+| Inference time per cluster | ~22s on (other) | ~0.1s total |
+| Named clusters | 21 tiny + (other) | 1 group |
+| Grammar quality | same `assertEquals+` | same `assertEquals+` |

docs/pipeline-overview.txt

@@ -1,100 +1,85 @@
 ```
-                                  ┌───────────────────────────────┐
+                                   ┌───────────────────────────────┐
-                                  │  Source Code Directory        │
+                                   │  Source Code Directory        │
-                                  │  (.py .js .ts .kt .rb .go    │
+                                   │  (.py .js .ts .kt .rb .go    │
-                                  │   .rs .java .c .cpp .h)       │
+                                   │   .rs .java .c .cpp .h)       │
-                                  └──────────┬────────────────────┘
+                                   └──────────┬────────────────────┘
-                                             │ scan_directory() + .gitignore
+                                               │ scan_directory()
-                                             ▼
+                                              ▼
-                                  ┌───────────────────────────────┐
+                                   ┌───────────────────────────────┐
-                                  │  Files grouped by extension   │
+                                   │  Files grouped by extension   │
-                                  │  .kt → [a.kt, b.kt, ...]     │
+                                   │  .kt → [a.kt, b.kt, ...]     │
-                                  │  .py → [x.py, y.py, ...]     │
+                                   │  .py → [x.py, y.py, ...]     │
-                                  └──────────┬────────────────────┘
+                                   └──────────┬────────────────────┘
-                                             │ for each extension
+                                              │ for each extension
-                                             ▼
+                                              ▼
-                          ┌───────────────────────────────────────────┐
+                           ┌───────────────────────────────────────────┐
-                          │  preprocess_by_method(file_path, code)    │
+                           │  preprocess_by_method(file_path, code)    │
-                          │                                           │
+                           │                                           │
-                          │  tree-sitter parser  ◄── _load_grammar()  │
+                           │  tree-sitter parser  ◄── _load_grammar()  │
-                          │       +                                   │
+                           │       +                                   │
-                          │  highlights.scm query ◄── _load_query()   │
+                           │  highlights.scm query ◄── _load_query()   │
-                          │       │                                   │
+                           │       │                                   │
-                          │       ▼                                   │
+                           │       ▼                                   │
-                          │  BEHAVIORAL_PREFIXES filter               │
+                           │  BEHAVIORAL_PREFIXES filter               │
-                          │  (definition./reference./keyword./        │
+                           │  (definition./reference./keyword./        │
-                          │   function/attribute/constructor/         │
+                           │   function/attribute/constructor/         │
-                          │   label/type.definition/module)           │
+                           │   label/type.definition/module)           │
-                          │       │                                   │
+                           │       │                                   │
-                          │       ▼                                   │
+                           │       ▼                                   │
-                          │  _find_method_bodies()                    │
+                           │  _find_method_bodies()                    │
-                          │  grouped by body boundaries               │
+                           │  grouped by body boundaries               │
-                          │       │                                   │
+                           │       │                                   │
-                          │       ▼                                   │
+                           │       ▼                                   │
-                          │  [(cap, text, line), ...] per method      │
+                           │  [(cap, text, line), ...] per method      │
-                          └──────────┬────────────────────────────────┘
+                           └──────────┬────────────────────────────────┘
-                                     │ list of sequences
+                                      │ list of sequences
-                                     ▼
+                                      ▼
-                          ┌───────────────────────────────────────────┐
+                           ┌───────────────────────────────────────────┐
-                          │  frequency_filter(sequences, 0.2)         │
+                           │  frequency_filter(sequences, 0.2)         │
-                          │  removes symbols in <20% of files         │
+                           │  removes symbols in <20% of methods       │
-                          └──────────┬────────────────────────────────┘
+                           └──────────┬────────────────────────────────┘
-                                     │ filtered sequences
+                                      │ filtered sequences
-                                     ▼
+                                      ▼
-                          ┌───────────────────────────────────────────┐
+               ┌────────────────────────────────────────────────────────┐
-                          │  cluster_methods_adaptive()                │
+               │                                                         │
-                          │                                           │
+               │  ┌─────────────────────┐   ┌────────────────────────┐  │
-                          │  Try ngram=3                              │
+               │  │ _extract_imports()  │   │ _build_arg_patterns()  │  │
-                          │    ├── _extract_call_tokens() → call seqs │
+               │  │ scan 200 lines     │   │ extract_arg_info()     │  │
-                          │    ├── build ngram→indices index          │
+               │  │ for import/from/   │   │   + _classify_arg_node │  │
-                          │    ├── top 20 by freq → clusters          │
+               │  │ require/#include/  │   │   + _find_arglist_node │  │
-                          │    └── if (other) > 60% → ngram=2 → ...  │
+               │  │ use/include        │   │   + _iterate_arg_nodes │  │
-                          │                                           │
+               │  └─────────┬──────────┘   │   + _summarize_arg_info│  │
-                          │  Multi-assignment: no used set            │
+               │            │              └───────────┬────────────┘  │
-                          │  Methods can be in N clusters             │
+               │            ▼                          ▼               │
-                          └──────────┬────────────────────────────────┘
+               │  ┌──────────────────────────────────────────────────┐  │
-                                     │ clusters: [(label, [seqs]), ...]
+               │  │  infer_ensemble(symbol_seqs, min_coverage=0.8)   │  │
-                                     ▼
+               │  │    ├── CRX (fast, unordered)                     │  │
-              ┌────────────────────────────────────────────────────────┐
+               │  │    ├── iDRegEx (ordered regex)                   │  │
-              │  For each cluster:                                     │
+               │  │    └── kOREInference (noisy, probabilistic)      │  │
-              │                                                         │
+               │  │    └── pick best by MDL score                    │  │
-              │  ┌─────────────────────┐   ┌────────────────────────┐  │
+               │  │    └── core/outlier split via _find_core(0.8)    │  │
-              │  │ _extract_imports()  │   │ _build_arg_patterns()  │  │
+               │  └──────────────────────┬───────────────────────────┘  │
-              │  │ scan 200 lines     │   │ extract_arg_info()     │  │
+               │                         ▼                              │
-              │  │ for import/from/   │   │   + _classify_arg_node │  │
+               │  (result, meta)                                        │
-              │  │ require/#include/  │   │   + _find_arglist_node │  │
+               │  meta = {files, imports, arg_patterns,                 │
-              │  │ use/include        │   │   + _iterate_arg_nodes │  │
+               │          packages: _file_to_package(relpath)}          │
-              │  └─────────┬──────────┘   │   + _summarize_arg_info│  │
+               └─────────────────────────┬──────────────────────────────┘
-              │            │              └───────────┬────────────┘  │
+                                         ▼
-              │            ▼                          ▼               │
+               ┌─────────────────────────────────────────────────────┐
-              │  ┌──────────────────────────────────────────────────┐  │
+               │  Output                                              │
-              │  │  infer_ensemble(symbol_seqs)                     │  │
+               │                                                      │
-              │  │    ├── CRX (fast, unordered)                     │  │
+               │  --format text (default)     --format json           │
-              │  │    ├── iDRegEx (ordered regex)                   │  │
+               │  ┌──────────────────┐       ┌──────────────────┐    │
-              │  │    └── kOREInference (noisy, probabilistic)      │  │
+               │  │ .kt:             │       │ [{               │    │
-              │  │    └── pick best by MDL score                    │  │
+               │  │  Grammar:        │       │   "language":    │    │
-              │  └──────────────────────┬───────────────────────────┘  │
+               │  │   assertEquals+  │       │     ".kt",       │    │
-              │                         ▼                              │
+               │  │  Imports: ...    │       │   "conventions": │    │
-              │  (label, result, count, meta)                          │
+               │  │  Args(assertEquals):    │    [{...}, ...]  │    │
-              │  meta = {files, imports, arg_patterns,                 │
+               │  │   n=2 [lit,var]  │       │ }]              │    │
-              │          packages: _file_to_package(relpath)}          │
+               │  └──────────────────┘       │                  │    │
-              └─────────────────────────┬──────────────────────────────┘
+               │                              │  → inject into  │    │
-                                        ▼
+               │                              │    LLM prompt   │    │
-              ┌─────────────────────────────────────────────────────┐
+               │                              └──────────────────┘    │
-              │  Output                                              │
+               └─────────────────────────────────────────────────────┘
-              │                                                      │
-              │  --format text (default)     --format json           │
-              │  ┌──────────────────┐       ┌──────────────────┐    │
-              │  │ .kt:             │       │ [{               │    │
-              │  │  ├─ assertEquals  │       │   "language":    │    │
-              │  │  │  (327 meth.)   │       │     ".kt",       │    │
-              │  │  │  Grammar:      │       │   "conventions": │    │
-              │  │  │   assertEquals+│       │    [{...}, ...]  │    │
-              │  │  │  Imports: ...  │       │ }]              │    │
-              │  │  │  Args(assertEquals):  │                  │    │
-              │  │  │   n=2 [lit,var]│       │  → inject into  │    │
-              │  │  ├─ (other)       │       │    LLM prompt   │    │
-              │  │  │  (1254 meth.)  │       └──────────────────┘    │
-              │  └──────────────────┘                                 │
-              └─────────────────────────────────────────────────────┘
 ```

tests/test_analyze.py

@@ -7,7 +7,8 @@ import sys
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
 from bex.tag_preprocessor.analyze import (
-    scan_directory, frequency_filter, infer, analyze_directory, _match_glob,
+    scan_directory, frequency_filter, infer, analyze_directory, _filter_glob,
+    _group_by_package,
 )
 
 
@@ -119,29 +120,35 @@ def test_infer_returns_ensemble_dict():
     print("  PASS test_infer_returns_ensemble_dict")
 
 
-def test_match_glob():
+def test_filter_glob():
-    assert _match_glob("/repo/src/main/app.kt", "**/src/main/**")
+    files = [
-    assert _match_glob("/repo/src/main/org/app.kt", "**/src/main/**")
+        "/repo/src/main/app.kt",
-    assert _match_glob("/repo/src/main/deep/nested/app.kt", "**/src/main/**")
+        "/repo/src/main/org/app.kt",
-    assert not _match_glob("/repo/src/test/app.kt", "**/src/main/**")
+        "/repo/src/test/app.kt",
-    assert not _match_glob("/repo/build/app.kt", "**/src/main/**")
+        "/repo/build/app.kt",
-    print("  PASS test_match_glob")
+    ]
+    inc = _filter_glob(files, include="**/src/main/**")
+    assert inc == ["/repo/src/main/app.kt", "/repo/src/main/org/app.kt"]
+    exc = _filter_glob(files, exclude="**/build/**")
+    assert exc == ["/repo/src/main/app.kt", "/repo/src/main/org/app.kt", "/repo/src/test/app.kt"]
+    both = _filter_glob(files, include="**/src/**", exclude="**/test/**")
+    assert both == ["/repo/src/main/app.kt", "/repo/src/main/org/app.kt"]
+    print("  PASS test_filter_glob")
 
 
-def test_analyze_directory_include_glob():
+
-    with tempfile.TemporaryDirectory() as td:
+def test_group_by_package():
-        d = Path(td)
+    data = [(0, "src/main"), (1, "src/main"), (2, "src/main"),
-        (d / "src" / "main").mkdir(parents=True)
+            (3, "src/test"), (4, "src/test"),
-        (d / "src" / "test").mkdir(parents=True)
+            (5, "docs")]
-        (d / "src" / "main" / "prod.py").write_text("def setup():\n    pass\ndef run():\n    return x")
+    groups, ungrouped = _group_by_package(data, min_size=3)
-        (d / "src" / "test" / "test_prod.py").write_text("def test_run():\n    assert run() == x")
+    labels = [l for l, _ in groups]
-        results = analyze_directory(td, include="**/src/main/**")
+    assert "src/main" in labels
-        assert ".py" in results
+    assert "src" not in labels
-        assert len(results[".py"]) >= 1
+    assert "" not in labels  # no root group — small packages discarded
-        for label, r, count, meta in results[".py"]:
+    assert len(groups) == 1
-            if r and r.get("best"):
+    assert len(ungrouped) == 3  # src/test (2) + docs (1) → discarded
-                assert r["best"]["grammar"] is not None
+    print("  PASS test_group_by_package")
-    print("  PASS test_analyze_directory_include_glob")
 
 
 def test_infer_low_coverage_filters_noise():
@@ -168,8 +175,8 @@ def run_all():
         test_scan_directory_skips_unsupported,
         test_scan_directory_nested,
         test_scan_directory_skips_build_dirs,
-        test_match_glob,
+        test_filter_glob,
-        test_analyze_directory_include_glob,
+        test_group_by_package,
         test_frequency_filter_nothing_to_filter,
         test_frequency_filter_removes_infrequent_symbol,
         test_frequency_filter_edge_empty_sequences,

tests/test_ensemble.py

@@ -1,8 +1,7 @@
-"""Tests for infer_ensemble — runs CRX, iDRegEx, and kOREInference, picks best by MDL."""
+"""Tests for infer_ensemble — runs CRX and iDRegEx, picks best by MDL."""
 
 from bex.ensemble import infer_ensemble
 from bex.idregex import is_deterministic
-from bex.kore import kOREInference
 
 
 # ── Basic ensemble runs ──
@@ -21,16 +20,15 @@ def test_ensemble_best_not_none():
     result = infer_ensemble(seqs, kmax=2, N=3)
     assert result['best'] is not None
     assert result['best']['grammar'] is not None
-    assert result['best']['algorithm'] in ('CRX', 'iDRegEx', 'kOREInference')
+    assert result['best']['algorithm'] in ('CRX', 'iDRegEx')
     assert result['best']['mdl_score'] is not None
 
 
-def test_ensemble_runs_all_three():
+def test_ensemble_runs_both():
     seqs = [['a', 'b', 'c'], ['a', 'b', 'c', 'd']]
     result = infer_ensemble(seqs, kmax=2, N=3)
     algos = {a['algorithm'] for a in result['all']}
     assert 'CRX' in algos
-    # iDRegEx and kOREInference may fail stochastically, so at least CRX
     assert len(result['all']) >= 1
 
 
@@ -67,13 +65,6 @@ def test_prefer_idregex():
     assert len(result['all']) == 1
 
 
-def test_prefer_koreinference():
-    seqs = [['a', 'b'], ['a', 'b', 'c']]
-    result = infer_ensemble(seqs, prefer='koreinference', kmax=2, N=5)
-    assert result['best']['algorithm'] == 'kOREInference'
-    assert len(result['all']) == 1
-
-
 def test_prefer_case_insensitive():
     seqs = [['a', 'b']]
     r1 = infer_ensemble(seqs, prefer='CRX')
@@ -229,12 +220,11 @@ def run_all():
     tests = [
         test_ensemble_returns_dict,
         test_ensemble_best_not_none,
-        test_ensemble_runs_all_three,
+        test_ensemble_runs_both,
         test_ensemble_all_results_have_scores,
         test_ensemble_deterministic_results,
         test_prefer_crx,
         test_prefer_idregex,
-        test_prefer_koreinference,
         test_prefer_case_insensitive,
         test_prefer_unknown_falls_back,
         test_ensemble_empty_input,