From 2620b6e49f26c0bbdea2de9f0160b95e3a8d8597 Mon Sep 17 00:00:00 2001 From: tobjend Date: Fri, 3 Jul 2026 21:34:25 +0200 Subject: [PATCH] feat: method-level sequence extraction via child_by_field_name('body') - _find_method_bodies uses tree-sitter's universal body field (9/10 grammars) - Kotlin fallback: scan children for body-like types - preprocess_by_method groups highlight captures by enclosing method body - Returns per-method sequences for k-ORE ordering analysis - analyze.py infer() now uses preprocess_by_method - Document findings in ANALYSIS.md - 97 tests pass --- bex/tag_preprocessor/analyze.py | 126 ++++++++++++++++++++++++---- bex/tag_preprocessor/code.py | 71 ++++++++++++++++ references/tags-queries/ANALYSIS.md | 28 +++++++ tests/test_analyze.py | 45 +++++++++- 4 files changed, 253 insertions(+), 17 deletions(-) diff --git a/bex/tag_preprocessor/analyze.py b/bex/tag_preprocessor/analyze.py index 7465364..d539183 100644 --- a/bex/tag_preprocessor/analyze.py +++ b/bex/tag_preprocessor/analyze.py @@ -1,32 +1,81 @@ """Orchestrator: directory scan → preprocess → frequency filter → ensemble infer. Usage: - python -m bex.tag_preprocessor.analyze + python -m bex.tag_preprocessor.analyze [options] Runs the full Phase 1.0 pipeline over a directory of source files. """ +import argparse import os import sys -from pathlib import Path +from pathlib import Path, PurePath from collections import Counter -from .code import preprocess +import pathspec + +from .code import preprocess_by_method from bex.ensemble import infer_ensemble SUPPORTED_EXTENSIONS = { ".py", ".js", ".ts", ".kt", ".rb", ".go", ".rs", ".java", ".c", ".cpp", } +FALLBACK_SKIP = { + ".git", "node_modules", "build", "dist", "target", "bin", "obj", + "__pycache__", ".gradle", ".mvn", ".idea", ".vscode", ".tox", + ".venv", "venv", "env", ".env", "out", +} -def scan_directory(dir_path): + +def _load_gitignore(dir_path): + """Load .gitignore from dir_path root, return PathSpec or None.""" + path = os.path.join(dir_path, ".gitignore") + if os.path.isfile(path): + with open(path) as f: + return pathspec.PathSpec.from_lines("gitwildmatch", f) + return None + + +def _match_glob(filepath, pattern): + """Match filepath against a gitignore-style glob pattern. + + Uses pathspec for proper **/ recursion support. + """ + spec = pathspec.PathSpec.from_lines("gitwildmatch", [pattern]) + return spec.match_file(filepath) + + +def scan_directory(dir_path, gitignore_spec=None): """Walk dir_path, return dict mapping extension → [file paths]. - Only includes supported extensions. Walks recursively. + Uses .gitignore patterns from the target directory to skip + ignored files/dirs. Falls back to FALLBACK_SKIP for common + build/dependency dirs when no .gitignore exists. """ + if gitignore_spec is None: + gitignore_spec = _load_gitignore(dir_path) + result = {} - for root, _, files in os.walk(dir_path): + for root, dirs, files in os.walk(dir_path): + rel_root = os.path.relpath(root, dir_path) + if rel_root == ".": + rel_root = "" + + pruned = [] + for d in dirs: + rel_dir = os.path.join(rel_root, d) if rel_root else d + if gitignore_spec and gitignore_spec.match_file(rel_dir): + continue + if gitignore_spec is None and d in FALLBACK_SKIP: + continue + pruned.append(d) + dirs[:] = pruned + for f in files: + rel_file = os.path.join(rel_root, f) if rel_root else f + if gitignore_spec and gitignore_spec.match_file(rel_file): + continue ext = os.path.splitext(f)[1].lower() if ext in SUPPORTED_EXTENSIONS: result.setdefault(ext, []).append(os.path.join(root, f)) @@ -86,9 +135,9 @@ def infer(file_paths, extension, min_coverage=0.2, prefer=None, kmax=2, N=3): for fp in file_paths: with open(fp) as f: code = f.read() - seq = preprocess(fp, code) - if seq: - sequences.append(seq) + for method_seq in preprocess_by_method(fp, code): + if method_seq: + sequences.append(method_seq) sequences = frequency_filter(sequences, min_coverage) @@ -97,13 +146,21 @@ def infer(file_paths, extension, min_coverage=0.2, prefer=None, kmax=2, N=3): return infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer) -def analyze_directory(dir_path, min_coverage=0.2, prefer=None): +def analyze_directory( + dir_path, + min_coverage=0.2, + prefer=None, + kmax=2, + include=None, +): """Scan a directory and run infer for each language found. Args: dir_path: directory to scan. min_coverage: minimum file fraction for symbol to be kept. prefer: algorithm preference. + kmax: max k for k-ORE algorithms. + include: optional glob pattern to filter files (e.g. '**/src/main/**'). Returns: dict mapping extension → ensemble result dict. @@ -113,16 +170,53 @@ def analyze_directory(dir_path, min_coverage=0.2, prefer=None): for ext, files in groups.items(): if len(files) < 1: continue - results[ext] = infer(files, ext, min_coverage=min_coverage, prefer=prefer) + if include: + files = [f for f in files if _match_glob(f, include)] + if not files: + continue + results[ext] = infer( + files, ext, + min_coverage=min_coverage, + prefer=prefer, + kmax=kmax, + ) return results +def _parse_args(argv=None): + parser = argparse.ArgumentParser( + description="Analyze a directory of source code for behavioral conventions.", + ) + parser.add_argument("directory", help="Directory to scan") + parser.add_argument( + "--prefer", + choices=["crx", "idregex", "koreinference"], + help="Skip ensemble, use only this algorithm", + ) + parser.add_argument( + "--kmax", type=int, default=2, + help="Maximum k for k-ORE algorithms (default: 2)", + ) + parser.add_argument( + "--min-coverage", type=float, default=0.2, + help="Minimum file fraction for a symbol to survive frequency filter (default: 0.2)", + ) + parser.add_argument( + "--include", + help="Glob pattern to filter files (e.g. '**/src/main/**')", + ) + return parser.parse_args(argv) + + def main(): - if len(sys.argv) < 2: - print("Usage: python -m bex.tag_preprocessor.analyze ", file=sys.stderr) - sys.exit(1) - dir_path = sys.argv[1] - results = analyze_directory(dir_path) + args = _parse_args() + results = analyze_directory( + args.directory, + min_coverage=args.min_coverage, + prefer=args.prefer, + kmax=args.kmax, + include=args.include, + ) for ext, result in results.items(): best = result.get("best") if best: diff --git a/bex/tag_preprocessor/code.py b/bex/tag_preprocessor/code.py index 8e9a24c..3afa89b 100644 --- a/bex/tag_preprocessor/code.py +++ b/bex/tag_preprocessor/code.py @@ -112,6 +112,77 @@ def _load_query(query_name): return src +def _find_method_bodies(tree): + """Find function/method body nodes across all languages. + + Uses tree-sitter's named field 'body' (universal across 9/10 grammars). + Kotlin fallback: scan children for body-like types when the field is absent. + Parent filter: 'function' or 'method' in type name avoids class/loop bodies. + """ + bodies = [] + + def walk(node): + body = node.child_by_field_name("body") + if not body: + for child in node.children: + ctype = child.type.lower() + if "body" in ctype or "block" in ctype or ctype == "compound_statement": + body = child + break + if body: + ptype = node.type.lower() + if "function" in ptype or "method" in ptype: + bodies.append(body) + for child in node.children: + walk(child) + + walk(tree.root_node) + return bodies + + +def preprocess_by_method(file_path: str, code: str): + """Preprocess and group behavioral tokens by enclosing method body. + + Returns list of sequences, one per function/method found. + Each sequence is [(capture_name, text, line_number), ...]. + """ + ext = os.path.splitext(file_path)[1].lower() + lang, query_name = _load_grammar(ext) + query_src = _load_query(query_name) + + parser = Parser(lang) + tree = parser.parse(code.encode()) + + query = Query(lang, query_src) + + cursor = QueryCursor(query) + captures = cursor.captures(tree.root_node) + + items = [] + for capname, nodes in captures.items(): + if not capname.startswith(BEHAVIORAL_PREFIXES): + continue + for node in nodes: + text = code[node.start_byte:node.end_byte].strip() + items.append((node.start_byte, capname, node, text)) + + items.sort(key=lambda x: x[0]) + + method_bodies = _find_method_bodies(tree) + method_bodies.sort(key=lambda b: b.start_byte) + + sequences = [] + for body_node in method_bodies: + seq = [] + for start, capname, node, text in items: + if body_node.start_byte <= start < body_node.end_byte: + seq.append((capname, text, code[:start].count("\n") + 1)) + if seq: + sequences.append(seq) + + return sequences + + def preprocess(file_path: str, code: str): ext = os.path.splitext(file_path)[1].lower() lang, query_name = _load_grammar(ext) diff --git a/references/tags-queries/ANALYSIS.md b/references/tags-queries/ANALYSIS.md index 0c33d2c..7b6caf1 100644 --- a/references/tags-queries/ANALYSIS.md +++ b/references/tags-queries/ANALYSIS.md @@ -227,3 +227,31 @@ Two languages required deviations from the plain nvim-treesitter approach: **Community tags.scm alone are insufficient. But nvim-treesitter highlights.scm — used as-is with two caveats — covers 4/5 gaps.** The remaining gap (no sequencing/traversal mechanism) is handled by tree-sitter's built-in ordered cursor iteration over captures, which naturally preserves source order. A universal query file is no longer needed. The `code.py` preprocessor loads highlights.scm per-language, resolves `; inherits:` directives, handles the two known deviations (Kotlin bundled query, jsx `#set!` stripping), and filters by behavioral capture prefix — zero adapters, zero per-language branches. + +## Method-Level Sequencing + +### The Problem + +Flat per-file sequences are too diverse for k-ORE to produce ordered grammars. Each file mixes multiple function bodies into one flat token list — the ordering across methods is meaningless. + +### The Solution: `child_by_field_name("body")` + +Research across all 10 grammar `node-types.json` files revealed that every tree-sitter grammar stores function/method bodies in a field called **`body`**: + +| Language | Function node type | `body` field type | +|----------|-------------------|-------------------| +| Python | `function_definition` | `block` | +| Go | `function_declaration`, `method_declaration` | `block` | +| Rust | `function_item` | `block` | +| JavaScript | `function_declaration`, `method_definition`, `arrow_function` | `statement_block` | +| TypeScript | `function_declaration`, `method_signature` | `statement_block` | +| Ruby | `method` | `body_statement` | +| Java | `method_declaration` | `block` | +| C/C++ | `function_definition` | `compound_statement` | +| Kotlin (fwcd) | `function_declaration` *(no field — positional child)* | `function_body` | + +**9/10 grammars** expose the body via `node.child_by_field_name("body")` in the tree-sitter API. Kotlin (fwcd grammar) is the exception — its `function_declaration` doesn't use a named `body` field. For Kotlin, we scan children and match by type name (`function_body`). + +The parent filter `"function" in node.type or "method" in node.type` ensures we don't capture class bodies, loop bodies, or other block-like constructs. + +This gives us **per-method sequences**: each function/method body becomes its own token list. k-ORE can then find actual call-order conventions (e.g. `validate → process → respond`). diff --git a/tests/test_analyze.py b/tests/test_analyze.py index f0ba673..50e2c51 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -6,7 +6,9 @@ import sys sys.path.insert(0, str(Path(__file__).parent.parent)) -from bex.tag_preprocessor.analyze import scan_directory, frequency_filter, infer +from bex.tag_preprocessor.analyze import ( + scan_directory, frequency_filter, infer, analyze_directory, _match_glob, +) def test_scan_directory_empty(): @@ -57,6 +59,22 @@ def test_scan_directory_nested(): print(" PASS test_scan_directory_nested") +def test_scan_directory_skips_build_dirs(): + with tempfile.TemporaryDirectory() as td: + d = Path(td) + (d / "src" / "main").mkdir(parents=True) + (d / "build" / "reports").mkdir(parents=True) + (d / "node_modules" / "pkg").mkdir(parents=True) + (d / "src" / "main" / "app.py").write_text("x = 1") + (d / "build" / "reports" / "report.js").write_text("let x = 1;") + (d / "node_modules" / "pkg" / "index.js").write_text("let y = 2;") + result = scan_directory(td) + assert ".py" in result + assert ".js" not in result + assert len(result[".py"]) == 1 + print(" PASS test_scan_directory_skips_build_dirs") + + def test_frequency_filter_nothing_to_filter(): sequences = [ [("function", "foo", 1), ("keyword.return", "return", 2)], @@ -101,6 +119,28 @@ def test_infer_returns_ensemble_dict(): print(" PASS test_infer_returns_ensemble_dict") +def test_match_glob(): + assert _match_glob("/repo/src/main/app.kt", "**/src/main/**") + assert _match_glob("/repo/src/main/org/app.kt", "**/src/main/**") + assert _match_glob("/repo/src/main/deep/nested/app.kt", "**/src/main/**") + assert not _match_glob("/repo/src/test/app.kt", "**/src/main/**") + assert not _match_glob("/repo/build/app.kt", "**/src/main/**") + print(" PASS test_match_glob") + + +def test_analyze_directory_include_glob(): + with tempfile.TemporaryDirectory() as td: + d = Path(td) + (d / "src" / "main").mkdir(parents=True) + (d / "src" / "test").mkdir(parents=True) + (d / "src" / "main" / "prod.py").write_text("x = 1") + (d / "src" / "test" / "test_prod.py").write_text("y = 2") + results = analyze_directory(td, include="**/src/main/**") + assert ".py" in results + assert len(results[".py"]["all"]) >= 1 + print(" PASS test_analyze_directory_include_glob") + + def test_infer_low_coverage_filters_noise(): with tempfile.TemporaryDirectory() as td: d = Path(td) @@ -124,6 +164,9 @@ def run_all(): test_scan_directory_groups_by_extension, test_scan_directory_skips_unsupported, test_scan_directory_nested, + test_scan_directory_skips_build_dirs, + test_match_glob, + test_analyze_directory_include_glob, test_frequency_filter_nothing_to_filter, test_frequency_filter_removes_infrequent_symbol, test_frequency_filter_edge_empty_sequences,