feat: method-level sequence extraction via child_by_field_name('body')

- _find_method_bodies uses tree-sitter's universal body field (9/10 grammars)
- Kotlin fallback: scan children for body-like types
- preprocess_by_method groups highlight captures by enclosing method body
- Returns per-method sequences for k-ORE ordering analysis
- analyze.py infer() now uses preprocess_by_method
- Document findings in ANALYSIS.md
- 97 tests pass

bex/tag_preprocessor/analyze.py

@@ -1,32 +1,81 @@
 """Orchestrator: directory scan → preprocess → frequency filter → ensemble infer.
 
 Usage:
-    python -m bex.tag_preprocessor.analyze <directory>
+    python -m bex.tag_preprocessor.analyze <directory> [options]
 
 Runs the full Phase 1.0 pipeline over a directory of source files.
 """
 
+import argparse
 import os
 import sys
-from pathlib import Path
+from pathlib import Path, PurePath
 from collections import Counter
 
-from .code import preprocess
+import pathspec
+
+from .code import preprocess_by_method
 from bex.ensemble import infer_ensemble
 
 SUPPORTED_EXTENSIONS = {
     ".py", ".js", ".ts", ".kt", ".rb", ".go", ".rs", ".java", ".c", ".cpp",
 }
 
+FALLBACK_SKIP = {
+    ".git", "node_modules", "build", "dist", "target", "bin", "obj",
+    "__pycache__", ".gradle", ".mvn", ".idea", ".vscode", ".tox",
+    ".venv", "venv", "env", ".env", "out",
+}
 
-def scan_directory(dir_path):
+
+def _load_gitignore(dir_path):
+    """Load .gitignore from dir_path root, return PathSpec or None."""
+    path = os.path.join(dir_path, ".gitignore")
+    if os.path.isfile(path):
+        with open(path) as f:
+            return pathspec.PathSpec.from_lines("gitwildmatch", f)
+    return None
+
+
+def _match_glob(filepath, pattern):
+    """Match filepath against a gitignore-style glob pattern.
+
+    Uses pathspec for proper **/ recursion support.
+    """
+    spec = pathspec.PathSpec.from_lines("gitwildmatch", [pattern])
+    return spec.match_file(filepath)
+
+
+def scan_directory(dir_path, gitignore_spec=None):
     """Walk dir_path, return dict mapping extension → [file paths].
 
-    Only includes supported extensions. Walks recursively.
+    Uses .gitignore patterns from the target directory to skip
+    ignored files/dirs. Falls back to FALLBACK_SKIP for common
+    build/dependency dirs when no .gitignore exists.
     """
+    if gitignore_spec is None:
+        gitignore_spec = _load_gitignore(dir_path)
+
     result = {}
-    for root, _, files in os.walk(dir_path):
+    for root, dirs, files in os.walk(dir_path):
+        rel_root = os.path.relpath(root, dir_path)
+        if rel_root == ".":
+            rel_root = ""
+
+        pruned = []
+        for d in dirs:
+            rel_dir = os.path.join(rel_root, d) if rel_root else d
+            if gitignore_spec and gitignore_spec.match_file(rel_dir):
+                continue
+            if gitignore_spec is None and d in FALLBACK_SKIP:
+                continue
+            pruned.append(d)
+        dirs[:] = pruned
+
         for f in files:
+            rel_file = os.path.join(rel_root, f) if rel_root else f
+            if gitignore_spec and gitignore_spec.match_file(rel_file):
+                continue
             ext = os.path.splitext(f)[1].lower()
             if ext in SUPPORTED_EXTENSIONS:
                 result.setdefault(ext, []).append(os.path.join(root, f))
@@ -86,9 +135,9 @@ def infer(file_paths, extension, min_coverage=0.2, prefer=None, kmax=2, N=3):
     for fp in file_paths:
         with open(fp) as f:
             code = f.read()
-        seq = preprocess(fp, code)
-        if seq:
-            sequences.append(seq)
+        for method_seq in preprocess_by_method(fp, code):
+            if method_seq:
+                sequences.append(method_seq)
 
     sequences = frequency_filter(sequences, min_coverage)
 
@@ -97,13 +146,21 @@ def infer(file_paths, extension, min_coverage=0.2, prefer=None, kmax=2, N=3):
     return infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer)
 
 
-def analyze_directory(dir_path, min_coverage=0.2, prefer=None):
+def analyze_directory(
+    dir_path,
+    min_coverage=0.2,
+    prefer=None,
+    kmax=2,
+    include=None,
+):
     """Scan a directory and run infer for each language found.
 
     Args:
         dir_path: directory to scan.
         min_coverage: minimum file fraction for symbol to be kept.
         prefer: algorithm preference.
+        kmax: max k for k-ORE algorithms.
+        include: optional glob pattern to filter files (e.g. '**/src/main/**').
 
     Returns:
         dict mapping extension → ensemble result dict.
@@ -113,16 +170,53 @@ def analyze_directory(dir_path, min_coverage=0.2, prefer=None):
     for ext, files in groups.items():
         if len(files) < 1:
             continue
-        results[ext] = infer(files, ext, min_coverage=min_coverage, prefer=prefer)
+        if include:
+            files = [f for f in files if _match_glob(f, include)]
+        if not files:
+            continue
+        results[ext] = infer(
+            files, ext,
+            min_coverage=min_coverage,
+            prefer=prefer,
+            kmax=kmax,
+        )
     return results
 
 
+def _parse_args(argv=None):
+    parser = argparse.ArgumentParser(
+        description="Analyze a directory of source code for behavioral conventions.",
+    )
+    parser.add_argument("directory", help="Directory to scan")
+    parser.add_argument(
+        "--prefer",
+        choices=["crx", "idregex", "koreinference"],
+        help="Skip ensemble, use only this algorithm",
+    )
+    parser.add_argument(
+        "--kmax", type=int, default=2,
+        help="Maximum k for k-ORE algorithms (default: 2)",
+    )
+    parser.add_argument(
+        "--min-coverage", type=float, default=0.2,
+        help="Minimum file fraction for a symbol to survive frequency filter (default: 0.2)",
+    )
+    parser.add_argument(
+        "--include",
+        help="Glob pattern to filter files (e.g. '**/src/main/**')",
+    )
+    return parser.parse_args(argv)
+
+
 def main():
-    if len(sys.argv) < 2:
-        print("Usage: python -m bex.tag_preprocessor.analyze <directory>", file=sys.stderr)
-        sys.exit(1)
-    dir_path = sys.argv[1]
-    results = analyze_directory(dir_path)
+    args = _parse_args()
+    results = analyze_directory(
+        args.directory,
+        min_coverage=args.min_coverage,
+        prefer=args.prefer,
+        kmax=args.kmax,
+        include=args.include,
+    )
     for ext, result in results.items():
         best = result.get("best")
         if best:

bex/tag_preprocessor/code.py

@@ -112,6 +112,77 @@ def _load_query(query_name):
     return src
 
 
+def _find_method_bodies(tree):
+    """Find function/method body nodes across all languages.
+
+    Uses tree-sitter's named field 'body' (universal across 9/10 grammars).
+    Kotlin fallback: scan children for body-like types when the field is absent.
+    Parent filter: 'function' or 'method' in type name avoids class/loop bodies.
+    """
+    bodies = []
+
+    def walk(node):
+        body = node.child_by_field_name("body")
+        if not body:
+            for child in node.children:
+                ctype = child.type.lower()
+                if "body" in ctype or "block" in ctype or ctype == "compound_statement":
+                    body = child
+                    break
+        if body:
+            ptype = node.type.lower()
+            if "function" in ptype or "method" in ptype:
+                bodies.append(body)
+        for child in node.children:
+            walk(child)
+
+    walk(tree.root_node)
+    return bodies
+
+
+def preprocess_by_method(file_path: str, code: str):
+    """Preprocess and group behavioral tokens by enclosing method body.
+
+    Returns list of sequences, one per function/method found.
+    Each sequence is [(capture_name, text, line_number), ...].
+    """
+    ext = os.path.splitext(file_path)[1].lower()
+    lang, query_name = _load_grammar(ext)
+    query_src = _load_query(query_name)
+
+    parser = Parser(lang)
+    tree = parser.parse(code.encode())
+
+    query = Query(lang, query_src)
+
+    cursor = QueryCursor(query)
+    captures = cursor.captures(tree.root_node)
+
+    items = []
+    for capname, nodes in captures.items():
+        if not capname.startswith(BEHAVIORAL_PREFIXES):
+            continue
+        for node in nodes:
+            text = code[node.start_byte:node.end_byte].strip()
+            items.append((node.start_byte, capname, node, text))
+
+    items.sort(key=lambda x: x[0])
+
+    method_bodies = _find_method_bodies(tree)
+    method_bodies.sort(key=lambda b: b.start_byte)
+
+    sequences = []
+    for body_node in method_bodies:
+        seq = []
+        for start, capname, node, text in items:
+            if body_node.start_byte <= start < body_node.end_byte:
+                seq.append((capname, text, code[:start].count("\n") + 1))
+        if seq:
+            sequences.append(seq)
+
+    return sequences
+
+
 def preprocess(file_path: str, code: str):
     ext = os.path.splitext(file_path)[1].lower()
     lang, query_name = _load_grammar(ext)

references/tags-queries/ANALYSIS.md

@@ -227,3 +227,31 @@ Two languages required deviations from the plain nvim-treesitter approach:
 **Community tags.scm alone are insufficient. But nvim-treesitter highlights.scm — used as-is with two caveats — covers 4/5 gaps.** The remaining gap (no sequencing/traversal mechanism) is handled by tree-sitter's built-in ordered cursor iteration over captures, which naturally preserves source order.
 
 A universal query file is no longer needed. The `code.py` preprocessor loads highlights.scm per-language, resolves `; inherits:` directives, handles the two known deviations (Kotlin bundled query, jsx `#set!` stripping), and filters by behavioral capture prefix — zero adapters, zero per-language branches.
+
+## Method-Level Sequencing
+
+### The Problem
+
+Flat per-file sequences are too diverse for k-ORE to produce ordered grammars. Each file mixes multiple function bodies into one flat token list — the ordering across methods is meaningless.
+
+### The Solution: `child_by_field_name("body")`
+
+Research across all 10 grammar `node-types.json` files revealed that every tree-sitter grammar stores function/method bodies in a field called **`body`**:
+
+| Language | Function node type | `body` field type |
+|----------|-------------------|-------------------|
+| Python | `function_definition` | `block` |
+| Go | `function_declaration`, `method_declaration` | `block` |
+| Rust | `function_item` | `block` |
+| JavaScript | `function_declaration`, `method_definition`, `arrow_function` | `statement_block` |
+| TypeScript | `function_declaration`, `method_signature` | `statement_block` |
+| Ruby | `method` | `body_statement` |
+| Java | `method_declaration` | `block` |
+| C/C++ | `function_definition` | `compound_statement` |
+| Kotlin (fwcd) | `function_declaration` *(no field — positional child)* | `function_body` |
+
+**9/10 grammars** expose the body via `node.child_by_field_name("body")` in the tree-sitter API. Kotlin (fwcd grammar) is the exception — its `function_declaration` doesn't use a named `body` field. For Kotlin, we scan children and match by type name (`function_body`).
+
+The parent filter `"function" in node.type or "method" in node.type` ensures we don't capture class bodies, loop bodies, or other block-like constructs.
+
+This gives us **per-method sequences**: each function/method body becomes its own token list. k-ORE can then find actual call-order conventions (e.g. `validate → process → respond`).

tests/test_analyze.py

@@ -6,7 +6,9 @@ import sys
 
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
-from bex.tag_preprocessor.analyze import scan_directory, frequency_filter, infer
+from bex.tag_preprocessor.analyze import (
+    scan_directory, frequency_filter, infer, analyze_directory, _match_glob,
+)
 
 
 def test_scan_directory_empty():
@@ -57,6 +59,22 @@ def test_scan_directory_nested():
     print("  PASS test_scan_directory_nested")
 
 
+def test_scan_directory_skips_build_dirs():
+    with tempfile.TemporaryDirectory() as td:
+        d = Path(td)
+        (d / "src" / "main").mkdir(parents=True)
+        (d / "build" / "reports").mkdir(parents=True)
+        (d / "node_modules" / "pkg").mkdir(parents=True)
+        (d / "src" / "main" / "app.py").write_text("x = 1")
+        (d / "build" / "reports" / "report.js").write_text("let x = 1;")
+        (d / "node_modules" / "pkg" / "index.js").write_text("let y = 2;")
+        result = scan_directory(td)
+        assert ".py" in result
+        assert ".js" not in result
+        assert len(result[".py"]) == 1
+    print("  PASS test_scan_directory_skips_build_dirs")
+
+
 def test_frequency_filter_nothing_to_filter():
     sequences = [
         [("function", "foo", 1), ("keyword.return", "return", 2)],
@@ -101,6 +119,28 @@ def test_infer_returns_ensemble_dict():
     print("  PASS test_infer_returns_ensemble_dict")
 
 
+def test_match_glob():
+    assert _match_glob("/repo/src/main/app.kt", "**/src/main/**")
+    assert _match_glob("/repo/src/main/org/app.kt", "**/src/main/**")
+    assert _match_glob("/repo/src/main/deep/nested/app.kt", "**/src/main/**")
+    assert not _match_glob("/repo/src/test/app.kt", "**/src/main/**")
+    assert not _match_glob("/repo/build/app.kt", "**/src/main/**")
+    print("  PASS test_match_glob")
+
+
+def test_analyze_directory_include_glob():
+    with tempfile.TemporaryDirectory() as td:
+        d = Path(td)
+        (d / "src" / "main").mkdir(parents=True)
+        (d / "src" / "test").mkdir(parents=True)
+        (d / "src" / "main" / "prod.py").write_text("x = 1")
+        (d / "src" / "test" / "test_prod.py").write_text("y = 2")
+        results = analyze_directory(td, include="**/src/main/**")
+        assert ".py" in results
+        assert len(results[".py"]["all"]) >= 1
+    print("  PASS test_analyze_directory_include_glob")
+
+
 def test_infer_low_coverage_filters_noise():
     with tempfile.TemporaryDirectory() as td:
         d = Path(td)
@@ -124,6 +164,9 @@ def run_all():
         test_scan_directory_groups_by_extension,
         test_scan_directory_skips_unsupported,
         test_scan_directory_nested,
+        test_scan_directory_skips_build_dirs,
+        test_match_glob,
+        test_analyze_directory_include_glob,
         test_frequency_filter_nothing_to_filter,
         test_frequency_filter_removes_infrequent_symbol,
         test_frequency_filter_edge_empty_sequences,