docs: add 8 architecture decision records

ADR 0001: nvim-treesitter highlights.scm as capture source
ADR 0002: language-agnostic method extraction via child_by_field_name
ADR 0003: method-level n-gram clustering before inference
ADR 0004: frequency filter with min_coverage threshold
ADR 0005: import extraction per cluster
ADR 0006: argument pattern extraction via AST node classification
ADR 0007: JSON output for LLM prompt injection
ADR 0008: BEX ensemble for grammar inference

bex/tag_preprocessor/analyze.py

@@ -0,0 +1,477 @@
+"""Orchestrator: directory scan → preprocess → frequency filter → ensemble infer.
+
+Usage:
+    python -m bex.tag_preprocessor.analyze <directory> [options]
+
+Runs the full Phase 1.0 pipeline over a directory of source files.
+"""
+
+import argparse
+import json
+import os
+import re
+import sys
+from pathlib import Path, PurePath
+from collections import Counter
+
+import pathspec
+
+from .code import preprocess_by_method, _extract_call_tokens, extract_arg_info, _summarize_arg_info
+from bex.ensemble import infer_ensemble
+
+IMPORT_PATTERNS = [
+    re.compile(r"^\s*import\s+"),
+    re.compile(r"^\s*from\s+"),
+    re.compile(r"^\s*require\s+"),
+    re.compile(r"^\s*require_relative\s+"),
+    re.compile(r"^\s*#\s*include\s+"),
+    re.compile(r"^\s*use\s+"),
+    re.compile(r"^\s*include\s+"),
+]
+
+SUPPORTED_EXTENSIONS = {
+    ".py", ".js", ".ts", ".kt", ".rb", ".go", ".rs", ".java", ".c", ".cpp",
+}
+
+FALLBACK_SKIP = {
+    ".git", "node_modules", "build", "dist", "target", "bin", "obj",
+    "__pycache__", ".gradle", ".mvn", ".idea", ".vscode", ".tox",
+    ".venv", "venv", "env", ".env", "out",
+}
+
+
+def _load_gitignore(dir_path):
+    """Load .gitignore from dir_path root, return PathSpec or None."""
+    path = os.path.join(dir_path, ".gitignore")
+    if os.path.isfile(path):
+        with open(path) as f:
+            return pathspec.PathSpec.from_lines("gitwildmatch", f)
+    return None
+
+
+def _match_glob(filepath, pattern):
+    """Match filepath against a gitignore-style glob pattern.
+
+    Uses pathspec for proper **/ recursion support.
+    """
+    spec = pathspec.PathSpec.from_lines("gitwildmatch", [pattern])
+    return spec.match_file(filepath)
+
+
+def _file_to_package(fp, ext):
+    """Infer the package/directory from a file path.
+
+    For Kotlin/Java: derives from `src/main/kotlin/` or `src/test/kotlin/` tree.
+    For other: shows the relative directory path.
+    """
+    p = PurePath(fp)
+    try:
+        idx = p.parts.index("kotlin")
+        return "/".join(p.parts[idx + 1:-1])
+    except ValueError:
+        pass
+    try:
+        idx = p.parts.index("java")
+        return "/".join(p.parts[idx + 1:-1])
+    except ValueError:
+        pass
+    try:
+        idx = p.parts.index("python")
+        return "/".join(p.parts[idx + 1:-1])
+    except ValueError:
+        pass
+    # Fallback: show parent dirs relative to first language-agnostic src
+    parts = p.parts
+    for keyword in ("src", "lib", "app"):
+        try:
+            idx = parts.index(keyword)
+            return "/".join(parts[idx:-1])
+        except ValueError:
+            continue
+    return str(p.parent)
+
+
+def _top_packages(file_paths, ext, top_n=3):
+    """Return the most common packages among a set of files."""
+    pkg_counts = Counter()
+    for fp in file_paths:
+        pkg = _file_to_package(fp, ext)
+        pkg_counts[pkg] += 1
+    return [pkg for pkg, _ in pkg_counts.most_common(top_n)]
+
+
+def _build_arg_patterns(file_paths):
+    """Extract merged argument patterns across a set of files."""
+    merged = {}
+    for fp in file_paths:
+        try:
+            with open(fp) as f:
+                code = f.read()
+        except OSError:
+            continue
+        info = extract_arg_info(fp, code)
+        for call_name, observations in info.items():
+            merged.setdefault(call_name, []).extend(observations)
+    return _summarize_arg_info(merged)
+
+
+def _extract_imports(file_paths):
+    """Extract unique import lines from source files.
+
+    Scans top 200 lines of each file for common import patterns
+    across all 10 supported languages. Deduplicates across files.
+    """
+    seen = set()
+    result = []
+    for fp in sorted(file_paths):
+        try:
+            with open(fp) as f:
+                for i, line in enumerate(f):
+                    if i >= 200:
+                        break
+                    stripped = line.strip()
+                    if any(p.match(stripped) for p in IMPORT_PATTERNS):
+                        if stripped not in seen:
+                            seen.add(stripped)
+                            result.append(stripped)
+        except OSError:
+            continue
+    return result
+
+
+def scan_directory(dir_path, gitignore_spec=None):
+    """Walk dir_path, return dict mapping extension → [file paths].
+
+    Uses .gitignore patterns from the target directory to skip
+    ignored files/dirs. Falls back to FALLBACK_SKIP for common
+    build/dependency dirs when no .gitignore exists.
+    """
+    if gitignore_spec is None:
+        gitignore_spec = _load_gitignore(dir_path)
+
+    result = {}
+    for root, dirs, files in os.walk(dir_path):
+        rel_root = os.path.relpath(root, dir_path)
+        if rel_root == ".":
+            rel_root = ""
+
+        pruned = []
+        for d in dirs:
+            rel_dir = os.path.join(rel_root, d) if rel_root else d
+            if gitignore_spec and gitignore_spec.match_file(rel_dir):
+                continue
+            if gitignore_spec is None and d in FALLBACK_SKIP:
+                continue
+            pruned.append(d)
+        dirs[:] = pruned
+
+        for f in files:
+            rel_file = os.path.join(rel_root, f) if rel_root else f
+            if gitignore_spec and gitignore_spec.match_file(rel_file):
+                continue
+            ext = os.path.splitext(f)[1].lower()
+            if ext in SUPPORTED_EXTENSIONS:
+                result.setdefault(ext, []).append(os.path.join(root, f))
+    return result
+
+
+def frequency_filter(sequences, min_coverage=0.2):
+    """Remove symbols appearing in fewer than min_coverage fraction of files.
+
+    Args:
+        sequences: list of lists of (capture_name, text, line_number) tuples.
+        min_coverage: minimum fraction of files a symbol must appear in.
+
+    Returns:
+        Filtered sequences with rare symbols removed.
+    """
+    if not sequences:
+        return sequences
+
+    n_files = len(sequences)
+    threshold = max(1, int(n_files * min_coverage))
+
+    symbol_file_count = Counter()
+    for seq in sequences:
+        seen = set()
+        for _, text, _ in seq:
+            if text not in seen:
+                symbol_file_count[text] += 1
+                seen.add(text)
+
+    keep = {text for text, count in symbol_file_count.items()
+            if count >= threshold}
+
+    filtered = []
+    for seq in sequences:
+        new_seq = [(cap, text, line) for cap, text, line in seq if text in keep]
+        filtered.append(new_seq)
+
+    return filtered
+
+
+def cluster_methods(sequences, min_cluster_size=3, ngram_size=3):
+    """Group method sequences by shared n-gram call patterns.
+
+    Extracts call tokens from each sequence, builds an n-gram index,
+    and assigns methods to the largest matching clusters first.
+    Remaining methods go to an '(other)' cluster.
+
+    Args:
+        sequences: list of (capture, text, line) lists.
+        min_cluster_size: minimum methods to form a cluster.
+        ngram_size: length of n-grams to match (default 3).
+
+    Returns:
+        list of (label, [sequences]) tuples.
+    """
+    if not sequences:
+        return []
+
+    call_seqs = [_extract_call_tokens(seq) for seq in sequences]
+
+    ngram_to_methods = {}
+    for idx, cseq in enumerate(call_seqs):
+        for i in range(len(cseq) - ngram_size + 1):
+            ngram = tuple(cseq[i:i + ngram_size])
+            if len(ngram) == ngram_size:
+                ngram_to_methods.setdefault(ngram, set()).add(idx)
+
+    used = set()
+    clusters = []
+    for ngram, indices in sorted(ngram_to_methods.items(), key=lambda x: -len(x[1])):
+        indices = indices - used
+        if len(indices) >= min_cluster_size:
+            label = " → ".join(ngram)
+            cluster_seqs = [sequences[i] for i in indices]
+            clusters.append((label, cluster_seqs))
+            used.update(indices)
+
+    remaining = [i for i in range(len(sequences)) if i not in used]
+    if remaining:
+        clusters.append(("(other)", [sequences[i] for i in remaining]))
+
+    return clusters
+
+
+def analyze_clusters(file_paths, extension, min_coverage=0.2, prefer=None, kmax=2, N=3):
+    """Run full pipeline with clustering: preprocess → cluster → per-cluster infer.
+
+    Returns:
+        list of (label, ensemble_result_dict, method_count, meta) tuples.
+        meta = {"files": set(paths), "imports": [sorted_import_lines]}.
+    """
+    sequences = []
+    seq_files = []
+    for fp in file_paths:
+        with open(fp) as f:
+            code = f.read()
+        for method_seq in preprocess_by_method(fp, code):
+            if method_seq:
+                sequences.append(method_seq)
+                seq_files.append(fp)
+
+    if not sequences:
+        return []
+
+    sequences = frequency_filter(sequences, min_coverage)
+    clusters = cluster_methods(sequences)
+
+    results = []
+    for label, cluster_seqs in clusters:
+        cluster_fps = set()
+        for seq in cluster_seqs:
+            idx = next(i for i, s in enumerate(sequences) if s is seq)
+            cluster_fps.add(seq_files[idx])
+        imports = _extract_imports(cluster_fps)
+        arg_patterns = _build_arg_patterns(cluster_fps)
+        symbol_seqs = [[text for _, text, _ in seq] for seq in cluster_seqs]
+        result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer)
+        packages = _top_packages(cluster_fps, extension)
+        meta = {"files": cluster_fps, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
+        results.append((label, result, len(cluster_seqs), meta))
+
+    return results
+
+
+def infer(file_paths, extension, min_coverage=0.2, prefer=None, kmax=2, N=3):
+    """Run full pipeline: preprocess → frequency filter → ensemble infer.
+
+    Args:
+        file_paths: list of source file paths (same language).
+        extension: language extension (e.g. '.py').
+        min_coverage: minimum file fraction for a symbol to be kept.
+        prefer: inference algorithm preference ('crx', 'idregex', or None).
+        kmax: max k for k-ORE algorithms.
+        N: number of random trials.
+
+    Returns:
+        Ensemble result dict from infer_ensemble.
+    """
+    sequences = []
+    for fp in file_paths:
+        with open(fp) as f:
+            code = f.read()
+        for method_seq in preprocess_by_method(fp, code):
+            if method_seq:
+                sequences.append(method_seq)
+
+    sequences = frequency_filter(sequences, min_coverage)
+
+    symbol_seqs = [[text for _, text, _ in seq] for seq in sequences]
+
+    return infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer)
+
+
+def analyze_directory(
+    dir_path,
+    min_coverage=0.2,
+    prefer=None,
+    kmax=2,
+    include=None,
+    cluster=True,
+):
+    """Scan a directory and run cluster analysis for each language found.
+
+    Args:
+        dir_path: directory to scan.
+        min_coverage: minimum file fraction for symbol to be kept.
+        prefer: algorithm preference.
+        kmax: max k for k-ORE algorithms.
+        include: optional glob pattern to filter files (e.g. '**/src/main/**').
+        cluster: if True, run method-level clustering (default).
+
+    Returns:
+        dict mapping extension → list of (label, result_dict, count) tuples.
+    """
+    groups = scan_directory(dir_path)
+    results = {}
+    for ext, files in groups.items():
+        if len(files) < 1:
+            continue
+        if include:
+            files = [f for f in files if _match_glob(f, include)]
+        if not files:
+            continue
+        if cluster:
+            results[ext] = analyze_clusters(
+                files, ext,
+                min_coverage=min_coverage,
+                prefer=prefer,
+                kmax=kmax,
+            )
+        else:
+            r = infer(files, ext, min_coverage=min_coverage, prefer=prefer, kmax=kmax)
+            results[ext] = [("(all methods)", r, 0, {"files": set(files), "imports": _extract_imports(files)})]
+    return results
+
+
+def _parse_args(argv=None):
+    parser = argparse.ArgumentParser(
+        description="Analyze a directory of source code for behavioral conventions.",
+    )
+    parser.add_argument("directory", help="Directory to scan")
+    parser.add_argument(
+        "--prefer",
+        choices=["crx", "idregex", "koreinference"],
+        help="Skip ensemble, use only this algorithm",
+    )
+    parser.add_argument(
+        "--kmax", type=int, default=2,
+        help="Maximum k for k-ORE algorithms (default: 2)",
+    )
+    parser.add_argument(
+        "--min-coverage", type=float, default=0.2,
+        help="Minimum file fraction for a symbol to survive frequency filter (default: 0.2)",
+    )
+    parser.add_argument(
+        "--include",
+        help="Glob pattern to filter files (e.g. '**/src/main/**')",
+    )
+    parser.add_argument(
+        "--min-cluster-size", type=int, default=3,
+        help="Minimum methods to form a cluster (default: 3)",
+    )
+    parser.add_argument(
+        "--ngram-size", type=int, default=3,
+        help="N-gram length for clustering (default: 3)",
+    )
+    parser.add_argument(
+        "--format", choices=["text", "json"], default="text",
+        help="Output format (default: text)",
+    )
+    parser.add_argument(
+        "--json", action="store_true", dest="json_flag",
+        help="Shortcut for --format json",
+    )
+    return parser.parse_args(argv)
+
+
+def _build_json_output(results):
+    """Convert results dict to a compact JSON structure for prompt injection."""
+    output = []
+    for ext, clusters in results.items():
+        lang = {"language": ext, "conventions": []}
+        total_methods = 0
+        for label, result, count, meta in clusters:
+            total_methods += count
+            entry = {
+                "label": label,
+                "method_count": count,
+            }
+            if result and result.get("best"):
+                entry["algorithm"] = result["best"]["algorithm"]
+                entry["grammar"] = result["best"]["grammar"]
+                entry["mdl_score"] = round(result["best"]["mdl_score"], 1)
+            entry["imports"] = meta.get("imports", [])
+            entry["arg_patterns"] = meta.get("arg_patterns", {})
+            lang["conventions"].append(entry)
+        lang["total_methods"] = total_methods
+        output.append(lang)
+    return json.dumps(output, indent=2)
+
+
+def main():
+    args = _parse_args()
+    results = analyze_directory(
+        args.directory,
+        min_coverage=args.min_coverage,
+        prefer=args.prefer,
+        kmax=args.kmax,
+        include=args.include,
+    )
+
+    if args.json_flag or args.format == "json":
+        print(_build_json_output(results))
+        return
+
+    for ext, clusters in results.items():
+        print(f"\n{ext}:")
+        for label, result, count, meta in clusters:
+            if result and result.get("best"):
+                best = result["best"]
+                print(f"  ╰─ {label} ({count} methods)")
+                print(f"     Algorithm: {best['algorithm']}")
+                print(f"     Grammar:   {best['grammar']}")
+                print(f"     MDL:       {best['mdl_score']}")
+            else:
+                print(f"  ╰─ {label} ({count} methods) — no grammar")
+            imps = meta.get("imports", [])
+            if imps:
+                joined = "  |  ".join(imps[:6])
+                print(f"     Imports:  {joined}")
+                if len(imps) > 6:
+                    print(f"               ... and {len(imps) - 6} more")
+            argp = meta.get("arg_patterns", {})
+            if argp:
+                top_calls = sorted(argp.items(), key=lambda x: -x[1]["occurrences"])[:4]
+                for call_name, asum in top_calls:
+                    ac = asum["arg_count"]
+                    pats = asum["patterns"][:2]
+                    pat_strs = [f"{p['args']}:{','.join(p['types'])}" for p in pats]
+                    print(f"     Args({call_name}): n={ac['common']} "
+                          f"[{'; '.join(pat_strs)}]")
+
+
+if __name__ == "__main__":
+    main()

bex/tag_preprocessor/code.py

@@ -0,0 +1,406 @@
+"""Universal tree-sitter tag preprocessor.
+
+Usage:
+    python -m bex.tag_preprocessor.code <file>
+
+Emits an ordered sequence of behavioral tokens using community highlights.scm
+queries from nvim-treesitter. One code path for all languages.
+"""
+
+import importlib
+import os
+import re
+import sys
+from pathlib import Path
+
+from tree_sitter import Language, Parser, Query, QueryCursor
+
+QUERIES_DIR = Path(__file__).parent / "queries"
+
+# (query_name, module_name, func_name)
+EXTENSION_MAP = {
+    ".py": ("python", "tree_sitter_python", "language"),
+    ".js": ("javascript", "tree_sitter_javascript", "language"),
+    ".mjs": ("javascript", "tree_sitter_javascript", "language"),
+    ".cjs": ("javascript", "tree_sitter_javascript", "language"),
+    ".ts": ("typescript", "tree_sitter_typescript", "language_typescript"),
+    ".tsx": ("typescript", "tree_sitter_typescript", "language_tsx"),
+    ".rb": ("ruby", "tree_sitter_ruby", "language"),
+    ".go": ("go", "tree_sitter_go", "language"),
+    ".rs": ("rust", "tree_sitter_rust", "language"),
+    ".java": ("java", "tree_sitter_java", "language"),
+    ".kt": ("kotlin", "tree_sitter_kotlin", "language"),
+    ".kts": ("kotlin", "tree_sitter_kotlin", "language"),
+    ".c": ("c", "tree_sitter_c", "language"),
+    ".h": ("c", "tree_sitter_c", "language"),
+    ".cpp": ("cpp", "tree_sitter_cpp", "language"),
+    ".cc": ("cpp", "tree_sitter_cpp", "language"),
+    ".cxx": ("cpp", "tree_sitter_cpp", "language"),
+    ".hpp": ("cpp", "tree_sitter_cpp", "language"),
+}
+
+BEHAVIORAL_PREFIXES = (
+    "definition.",
+    "reference.",
+    "keyword.",
+    "function",
+    "attribute",
+    "constructor",
+    "label",
+    "type.definition",
+    "module",
+)
+
+CALL_PREFIXES = ("function", "reference.call", "reference.class")
+
+def _extract_call_tokens(seq):
+    """Extract ordered call-like tokens from a method sequence.
+
+    Filters to captures representing function calls, constructors,
+    or references — the 'what happens in what order'.
+    Returns list of text values.
+    """
+    result = []
+    for capname, text, _ in seq:
+        for prefix in CALL_PREFIXES:
+            if capname.startswith(prefix):
+                result.append(text)
+                break
+    return result
+
+
+_grammar_cache = {}
+_query_cache = {}
+
+ARG_LITERAL_TYPES = {
+    "string", "string_literal", "number", "integer", "float",
+    "decimal", "true", "false", "null", "nil", "none",
+}
+
+LAMBDA_TYPES = {
+    "lambda", "do_block", "block",
+}
+
+
+def _classify_arg_node(node):
+    """Classify a tree-sitter argument node by structural role."""
+    t = node.type
+    if t in ARG_LITERAL_TYPES:
+        return "lit"
+    if t in LAMBDA_TYPES or "ambda" in t or "block" in t:
+        return "lambda"
+    if t == "keyword_argument" or t.endswith("named_argument"):
+        return "kwarg"
+    if t.endswith("call_expression") or t in ("call", "method_invocation"):
+        return "call"
+    if t.endswith("identifier") or t.endswith("name"):
+        return "var"
+    if "subscript" in t:
+        return "subscript"
+    if "binary" in t or "unary" in t or "ternary" in t or "operator" in t:
+        return "expr"
+    if t in ("interpolation", "template_string"):
+        return "template"
+    return "other"
+
+
+def _find_arglist_node(parent):
+    """Find the argument container node in a call expression, language-agnostic."""
+    args = parent.child_by_field_name("arguments")
+    if args:
+        return args
+    for child in parent.children:
+        if child.type in ("argument_list", "arguments", "call_suffix"):
+            return child
+        if child.type == "template_string":
+            return child
+    return None
+
+
+def _iterate_arg_nodes(arglist):
+    """Yield individual argument expression nodes from an arglist."""
+    if arglist.type == "call_suffix":
+        for child in arglist.children:
+            if child.is_named and ("ambda" in child.type or "block" in child.type):
+                yield child
+                return
+        for child in arglist.children:
+            if child.type == "value_arguments":
+                for arg in child.children:
+                    if arg.is_named and arg.type == "value_argument":
+                        for expr in arg.children:
+                            if expr.is_named:
+                                yield expr
+                                break
+        return
+    for child in arglist.children:
+        if child.is_named:
+            yield child
+
+
+def extract_arg_info(file_path, code):
+    """For each behavioral call in the file, record argument structure.
+
+    Returns dict: call_text -> [ (arg_count, (type1, type2, ...)), ... ]
+    where types are 'var','lit','call','lambda','kwarg','expr','other'.
+    """
+    ext = os.path.splitext(file_path)[1].lower()
+    lang, query_name = _load_grammar(ext)
+    query_src = _load_query(query_name)
+
+    parser = Parser(lang)
+    tree = parser.parse(code.encode())
+    query = Query(lang, query_src)
+
+    cursor = QueryCursor(query)
+    captures = cursor.captures(tree.root_node)
+
+    info = {}
+    for capname, nodes in captures.items():
+        if not capname.startswith(BEHAVIORAL_PREFIXES):
+            continue
+        for node in nodes:
+            text = code[node.start_byte:node.end_byte].strip()
+            parent = node.parent
+            if not parent:
+                continue
+            arglist = _find_arglist_node(parent)
+            if not arglist:
+                continue
+            types = [_classify_arg_node(c) for c in _iterate_arg_nodes(arglist)]
+            info.setdefault(text, []).append((len(types), tuple(types)))
+    return info
+
+
+def _summarize_arg_info(info):
+    """Aggregate extract_arg_info across files into compact summaries.
+
+    Returns dict: call_text -> {
+        occurrences: int,
+        arg_count: {min, max, common},
+        patterns: [(count, (types..)), ...],  # top-5 by frequency
+    }
+    """
+    from collections import Counter
+    summary = {}
+    for call_name, observations in info.items():
+        counts = [c for c, _ in observations]
+        pattern_counts = Counter(observations)
+        top_patterns = pattern_counts.most_common(5)
+        summary[call_name] = {
+            "occurrences": len(observations),
+            "arg_count": {
+                "min": min(counts),
+                "max": max(counts),
+                "common": max(set(counts), key=counts.count),
+            },
+            "patterns": [
+                {"count": c, "args": n, "types": list(t)}
+                for (n, t), c in top_patterns
+            ],
+        }
+    return summary
+
+
+def _load_grammar(ext):
+    entry = EXTENSION_MAP.get(ext)
+    if entry is None:
+        raise ValueError(f"Unsupported extension: {ext}")
+    query_name, module_name, func_name = entry
+
+    cache_key = f"{module_name}.{func_name}"
+    if cache_key in _grammar_cache:
+        return _grammar_cache[cache_key], query_name
+
+    mod = importlib.import_module(module_name)
+    lang = Language(getattr(mod, func_name)())
+    _grammar_cache[cache_key] = lang
+    return lang, query_name
+
+
+INHERIT_RE = re.compile(r"^;\s*inherits:\s*(.+)$", re.MULTILINE)
+
+
+def _resolve_inherits(src, query_name, seen=None):
+    if seen is None:
+        seen = set()
+    if query_name in seen:
+        return ""
+    seen.add(query_name)
+
+    qpath = QUERIES_DIR / f"{query_name}.scm"
+    if not qpath.exists():
+        return ""
+
+    content = qpath.read_text()
+    m = INHERIT_RE.search(content)
+    if m:
+        parents = m.group(1)
+        parent_parts = []
+        for parent in parents.split(","):
+            parent = parent.strip().strip("()")
+            if parent:
+                parent_parts.append(_resolve_inherits(parent, parent, seen))
+        parent_src = "\n".join(p for p in parent_parts if p)
+        body = INHERIT_RE.sub("", content)
+        return parent_src + "\n" + body if parent_src else body
+    return content
+
+
+def _load_query(query_name):
+    if query_name in _query_cache:
+        return _query_cache[query_name]
+
+    src = _resolve_inherits(query_name, query_name)
+    if not src:
+        raise FileNotFoundError(f"Query file not found: {query_name}")
+
+    _query_cache[query_name] = src
+    return src
+
+
+def _find_method_bodies(tree):
+    """Find function/method body nodes across all languages.
+
+    Uses tree-sitter's named field 'body' (universal across 9/10 grammars).
+    Kotlin fallback: scan children for body-like types when the field is absent.
+    Parent filter: 'function' or 'method' in type name avoids class/loop bodies.
+    """
+    bodies = []
+
+    def walk(node):
+        body = node.child_by_field_name("body")
+        if not body:
+            for child in node.children:
+                ctype = child.type.lower()
+                if "body" in ctype or "block" in ctype or ctype == "compound_statement":
+                    body = child
+                    break
+        if body:
+            ptype = node.type.lower()
+            if "function" in ptype or "method" in ptype:
+                bodies.append(body)
+        for child in node.children:
+            walk(child)
+
+    walk(tree.root_node)
+    return bodies
+
+
+def preprocess_by_method(file_path: str, code: str):
+    """Preprocess and group behavioral tokens by enclosing method body.
+
+    Returns list of sequences, one per function/method found.
+    Each sequence is [(capture_name, text, line_number), ...].
+    """
+    ext = os.path.splitext(file_path)[1].lower()
+    lang, query_name = _load_grammar(ext)
+    query_src = _load_query(query_name)
+
+    parser = Parser(lang)
+    tree = parser.parse(code.encode())
+
+    query = Query(lang, query_src)
+
+    cursor = QueryCursor(query)
+    captures = cursor.captures(tree.root_node)
+
+    items = []
+    for capname, nodes in captures.items():
+        if not capname.startswith(BEHAVIORAL_PREFIXES):
+            continue
+        for node in nodes:
+            text = code[node.start_byte:node.end_byte].strip()
+            items.append((node.start_byte, capname, node, text))
+
+    items.sort(key=lambda x: x[0])
+
+    method_bodies = _find_method_bodies(tree)
+    method_bodies.sort(key=lambda b: b.start_byte)
+
+    sequences = []
+    for body_node in method_bodies:
+        seq = []
+        for start, capname, node, text in items:
+            if body_node.start_byte <= start < body_node.end_byte:
+                seq.append((capname, text, code[:start].count("\n") + 1))
+        if seq:
+            sequences.append(seq)
+
+    return sequences
+
+
+def preprocess(file_path: str, code: str):
+    ext = os.path.splitext(file_path)[1].lower()
+    lang, query_name = _load_grammar(ext)
+    query_src = _load_query(query_name)
+
+    parser = Parser(lang)
+    tree = parser.parse(code.encode())
+
+    try:
+        query = Query(lang, query_src)
+    except Exception as e:
+        print(f"Query error for {query_name}: {e}", file=sys.stderr)
+        return
+
+    cursor = QueryCursor(query)
+    captures = cursor.captures(tree.root_node)
+
+    items = []
+    for capname, nodes in captures.items():
+        if not capname.startswith(BEHAVIORAL_PREFIXES):
+            continue
+        for node in nodes:
+            items.append((node.start_byte, capname, node, code[node.start_byte:node.end_byte].strip()))
+
+    items.sort(key=lambda x: x[0])
+
+    return [(capname, text, code[:start].count("\n") + 1) for start, capname, _, text in items]
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python -m bex.tag-preprocessor.code <file>", file=sys.stderr)
+        sys.exit(1)
+
+    file_path = sys.argv[1]
+
+    if not os.path.exists(file_path):
+        print(f"File not found: {file_path}", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        with open(file_path) as f:
+            code = f.read()
+    except Exception as e:
+        print(f"Error reading {file_path}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        seq = preprocess(file_path, code)
+    except ValueError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+    except FileNotFoundError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error preprocessing {file_path}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    if not seq:
+        print("(no behavioral tokens)")
+        return
+
+    print(f"{'CAPTURE':35s} {'TEXT':50s} LINE")
+    print("-" * 88)
+    for capname, text, line in seq:
+        print(f"{capname:35s} '{text[:48]:48s}' L{line}")
+
+    print(f"\nSequence ({len(seq)} tokens):")
+    print(" -> ".join(c for c, _, _ in seq))
+
+
+if __name__ == "__main__":
+    main()

bex/tag_preprocessor/nvim-reference/c.scm

@@ -0,0 +1,341 @@
+; Lower priority to prefer @variable.parameter when identifier appears in parameter_declaration.
+((identifier) @variable
+  (#set! priority 95))
+
+(preproc_def
+  (preproc_arg) @variable)
+
+[
+  "default"
+  "goto"
+  "asm"
+  "__asm__"
+] @keyword
+
+[
+  "enum"
+  "struct"
+  "union"
+  "typedef"
+] @keyword.type
+
+[
+  "sizeof"
+  "offsetof"
+] @keyword.operator
+
+(alignof_expression
+  .
+  _ @keyword.operator)
+
+"return" @keyword.return
+
+[
+  "while"
+  "for"
+  "do"
+  "continue"
+  "break"
+] @keyword.repeat
+
+[
+  "if"
+  "else"
+  "case"
+  "switch"
+] @keyword.conditional
+
+[
+  "#if"
+  "#ifdef"
+  "#ifndef"
+  "#else"
+  "#elif"
+  "#endif"
+  "#elifdef"
+  "#elifndef"
+  (preproc_directive)
+] @keyword.directive
+
+"#define" @keyword.directive.define
+
+"#include" @keyword.import
+
+[
+  ";"
+  ":"
+  ","
+  "."
+  "::"
+] @punctuation.delimiter
+
+"..." @punctuation.special
+
+[
+  "("
+  ")"
+  "["
+  "]"
+  "{"
+  "}"
+] @punctuation.bracket
+
+[
+  "="
+  "-"
+  "*"
+  "/"
+  "+"
+  "%"
+  "~"
+  "|"
+  "&"
+  "^"
+  "<<"
+  ">>"
+  "->"
+  "<"
+  "<="
+  ">="
+  ">"
+  "=="
+  "!="
+  "!"
+  "&&"
+  "||"
+  "-="
+  "+="
+  "*="
+  "/="
+  "%="
+  "|="
+  "&="
+  "^="
+  ">>="
+  "<<="
+  "--"
+  "++"
+] @operator
+
+; Make sure the comma operator is given a highlight group after the comma
+; punctuator so the operator is highlighted properly.
+(comma_expression
+  "," @operator)
+
+[
+  (true)
+  (false)
+] @boolean
+
+(conditional_expression
+  [
+    "?"
+    ":"
+  ] @keyword.conditional.ternary)
+
+(string_literal) @string
+
+(system_lib_string) @string
+
+(escape_sequence) @string.escape
+
+(null) @constant.builtin
+
+(number_literal) @number
+
+(char_literal) @character
+
+(preproc_defined) @function.macro
+
+((field_expression
+  (field_identifier) @property) @_parent
+  (#not-has-parent? @_parent template_method function_declarator call_expression))
+
+(field_designator) @property
+
+((field_identifier) @property
+  (#has-ancestor? @property field_declaration)
+  (#not-has-ancestor? @property function_declarator))
+
+(statement_identifier) @label
+
+(declaration
+  type: (type_identifier) @_type
+  declarator: (identifier) @label
+  (#eq? @_type "__label__"))
+
+[
+  (type_identifier)
+  (type_descriptor)
+] @type
+
+(storage_class_specifier) @keyword.modifier
+
+[
+  (type_qualifier)
+  (gnu_asm_qualifier)
+  "__extension__"
+] @keyword.modifier
+
+(linkage_specification
+  "extern" @keyword.modifier)
+
+(type_definition
+  declarator: (type_identifier) @type.definition)
+
+(primitive_type) @type.builtin
+
+(sized_type_specifier
+  _ @type.builtin
+  type: _?)
+
+((identifier) @constant
+  (#lua-match? @constant "^[A-Z][A-Z0-9_]+$"))
+
+(preproc_def
+  (preproc_arg) @constant
+  (#lua-match? @constant "^[A-Z][A-Z0-9_]+$"))
+
+(enumerator
+  name: (identifier) @constant)
+
+(case_statement
+  value: (identifier) @constant)
+
+((identifier) @constant.builtin
+  ; format-ignore
+  (#any-of? @constant.builtin
+    "stderr" "stdin" "stdout"
+    "__FILE__" "__LINE__" "__DATE__" "__TIME__"
+    "__STDC__" "__STDC_VERSION__" "__STDC_HOSTED__"
+    "__cplusplus" "__OBJC__" "__ASSEMBLER__"
+    "__BASE_FILE__" "__FILE_NAME__" "__INCLUDE_LEVEL__"
+    "__TIMESTAMP__" "__clang__" "__clang_major__"
+    "__clang_minor__" "__clang_patchlevel__"
+    "__clang_version__" "__clang_literal_encoding__"
+    "__clang_wide_literal_encoding__"
+    "__FUNCTION__" "__func__" "__PRETTY_FUNCTION__"
+    "__VA_ARGS__" "__VA_OPT__"))
+
+(preproc_def
+  (preproc_arg) @constant.builtin
+  ; format-ignore
+  (#any-of? @constant.builtin
+    "stderr" "stdin" "stdout"
+    "__FILE__" "__LINE__" "__DATE__" "__TIME__"
+    "__STDC__" "__STDC_VERSION__" "__STDC_HOSTED__"
+    "__cplusplus" "__OBJC__" "__ASSEMBLER__"
+    "__BASE_FILE__" "__FILE_NAME__" "__INCLUDE_LEVEL__"
+    "__TIMESTAMP__" "__clang__" "__clang_major__"
+    "__clang_minor__" "__clang_patchlevel__"
+    "__clang_version__" "__clang_literal_encoding__"
+    "__clang_wide_literal_encoding__"
+    "__FUNCTION__" "__func__" "__PRETTY_FUNCTION__"
+    "__VA_ARGS__" "__VA_OPT__"))
+
+(attribute_specifier
+  (argument_list
+    (identifier) @variable.builtin))
+
+(attribute_specifier
+  (argument_list
+    (call_expression
+      function: (identifier) @variable.builtin)))
+
+((call_expression
+  function: (identifier) @function.builtin)
+  (#lua-match? @function.builtin "^__builtin_"))
+
+((call_expression
+  function: (identifier) @function.builtin)
+  (#has-ancestor? @function.builtin attribute_specifier))
+
+; Preproc def / undef
+(preproc_def
+  name: (_) @constant.macro)
+
+(preproc_call
+  directive: (preproc_directive) @_u
+  argument: (_) @constant.macro
+  (#eq? @_u "#undef"))
+
+(preproc_ifdef
+  name: (identifier) @constant.macro)
+
+(preproc_elifdef
+  name: (identifier) @constant.macro)
+
+(preproc_defined
+  (identifier) @constant.macro)
+
+(call_expression
+  function: (identifier) @function.call)
+
+(call_expression
+  function: (field_expression
+    field: (field_identifier) @function.call))
+
+(function_declarator
+  declarator: (identifier) @function)
+
+(function_declarator
+  declarator: (parenthesized_declarator
+    (pointer_declarator
+      declarator: (field_identifier) @function)))
+
+(preproc_function_def
+  name: (identifier) @function.macro)
+
+(comment) @comment @spell
+
+((comment) @comment.documentation
+  (#lua-match? @comment.documentation "^/[*][*][^*].*[*]/$"))
+
+; Parameters
+(parameter_declaration
+  declarator: (identifier) @variable.parameter)
+
+(parameter_declaration
+  declarator: (array_declarator) @variable.parameter)
+
+(parameter_declaration
+  declarator: (pointer_declarator) @variable.parameter)
+
+; K&R functions
+; To enable support for K&R functions,
+; add the following lines to your own query config and uncomment them.
+; They are commented out as they'll conflict with C++
+; Note that you'll need to have `; extends` at the top of your query file.
+;
+; (parameter_list (identifier) @variable.parameter)
+;
+; (function_definition
+;   declarator: _
+;   (declaration
+;     declarator: (identifier) @variable.parameter))
+;
+; (function_definition
+;   declarator: _
+;   (declaration
+;     declarator: (array_declarator) @variable.parameter))
+;
+; (function_definition
+;   declarator: _
+;   (declaration
+;     declarator: (pointer_declarator) @variable.parameter))
+(preproc_params
+  (identifier) @variable.parameter)
+
+[
+  "__attribute__"
+  "__declspec"
+  "__based"
+  "__cdecl"
+  "__clrcall"
+  "__stdcall"
+  "__fastcall"
+  "__thiscall"
+  "__vectorcall"
+  (ms_pointer_modifier)
+  (attribute_declaration)
+] @attribute

bex/tag_preprocessor/nvim-reference/cpp.scm

@@ -0,0 +1,268 @@
+; inherits: c
+
+((identifier) @variable.member
+  (#lua-match? @variable.member "^m_.*$"))
+
+(parameter_declaration
+  declarator: (reference_declarator) @variable.parameter)
+
+; function(Foo ...foo)
+(variadic_parameter_declaration
+  declarator: (variadic_declarator
+    (_) @variable.parameter))
+
+; int foo = 0
+(optional_parameter_declaration
+  declarator: (_) @variable.parameter)
+
+;(field_expression) @variable.parameter ;; How to highlight this?
+((field_expression
+  (field_identifier) @function.method) @_parent
+  (#has-parent? @_parent template_method function_declarator))
+
+(field_declaration
+  (field_identifier) @variable.member)
+
+(field_initializer
+  (field_identifier) @property)
+
+(function_declarator
+  declarator: (field_identifier) @function.method)
+
+(concept_definition
+  name: (identifier) @type.definition)
+
+(alias_declaration
+  name: (type_identifier) @type.definition)
+
+(auto) @type.builtin
+
+(namespace_identifier) @module
+
+((namespace_identifier) @type
+  (#lua-match? @type "^[%u]"))
+
+(case_statement
+  value: (qualified_identifier
+    (identifier) @constant))
+
+(using_declaration
+  .
+  "using"
+  .
+  "namespace"
+  .
+  [
+    (qualified_identifier)
+    (identifier)
+  ] @module)
+
+(destructor_name
+  (identifier) @function.method)
+
+; functions
+(function_declarator
+  (qualified_identifier
+    (identifier) @function))
+
+(function_declarator
+  (qualified_identifier
+    (qualified_identifier
+      (identifier) @function)))
+
+(function_declarator
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (identifier) @function))))
+
+((qualified_identifier
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (identifier) @function)))) @_parent
+  (#has-ancestor? @_parent function_declarator))
+
+(function_declarator
+  (template_function
+    (identifier) @function))
+
+(operator_name) @function
+
+"operator" @function
+
+"static_assert" @function.builtin
+
+(call_expression
+  (qualified_identifier
+    (identifier) @function.call))
+
+(call_expression
+  (qualified_identifier
+    (qualified_identifier
+      (identifier) @function.call)))
+
+(call_expression
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (identifier) @function.call))))
+
+((qualified_identifier
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (identifier) @function.call)))) @_parent
+  (#has-ancestor? @_parent call_expression))
+
+(call_expression
+  (template_function
+    (identifier) @function.call))
+
+(call_expression
+  (qualified_identifier
+    (template_function
+      (identifier) @function.call)))
+
+(call_expression
+  (qualified_identifier
+    (qualified_identifier
+      (template_function
+        (identifier) @function.call))))
+
+(call_expression
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (template_function
+          (identifier) @function.call)))))
+
+((qualified_identifier
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (template_function
+          (identifier) @function.call))))) @_parent
+  (#has-ancestor? @_parent call_expression))
+
+; methods
+(function_declarator
+  (template_method
+    (field_identifier) @function.method))
+
+(call_expression
+  (field_expression
+    (field_identifier) @function.method.call))
+
+; constructors
+((function_declarator
+  (qualified_identifier
+    (identifier) @constructor))
+  (#lua-match? @constructor "^%u"))
+
+((call_expression
+  function: (identifier) @constructor)
+  (#lua-match? @constructor "^%u"))
+
+((call_expression
+  function: (qualified_identifier
+    name: (identifier) @constructor))
+  (#lua-match? @constructor "^%u"))
+
+((call_expression
+  function: (field_expression
+    field: (field_identifier) @constructor))
+  (#lua-match? @constructor "^%u"))
+
+; constructing a type in an initializer list: Constructor ():  **SuperType (1)**
+((field_initializer
+  (field_identifier) @constructor
+  (argument_list))
+  (#lua-match? @constructor "^%u"))
+
+; Constants
+(this) @variable.builtin
+
+(null
+  "nullptr" @constant.builtin)
+
+(true) @boolean
+
+(false) @boolean
+
+; Literals
+(raw_string_literal) @string
+
+; Keywords
+[
+  "try"
+  "catch"
+  "noexcept"
+  "throw"
+] @keyword.exception
+
+[
+  "decltype"
+  "explicit"
+  "friend"
+  "override"
+  "using"
+  "requires"
+  "constexpr"
+] @keyword
+
+[
+  "class"
+  "namespace"
+  "template"
+  "typename"
+  "concept"
+] @keyword.type
+
+[
+  "co_await"
+  "co_yield"
+  "co_return"
+] @keyword.coroutine
+
+[
+  "public"
+  "private"
+  "protected"
+  "final"
+  "virtual"
+] @keyword.modifier
+
+[
+  "new"
+  "delete"
+  "xor"
+  "bitand"
+  "bitor"
+  "compl"
+  "not"
+  "xor_eq"
+  "and_eq"
+  "or_eq"
+  "not_eq"
+  "and"
+  "or"
+] @keyword.operator
+
+"<=>" @operator
+
+"::" @punctuation.delimiter
+
+(template_argument_list
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(template_parameter_list
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(literal_suffix) @operator

bex/tag_preprocessor/nvim-reference/go.scm

@@ -0,0 +1,254 @@
+; Forked from tree-sitter-go
+; Copyright (c) 2014 Max Brunsfeld (The MIT License)
+;
+; Identifiers
+(type_identifier) @type
+
+(type_spec
+  name: (type_identifier) @type.definition)
+
+(field_identifier) @property
+
+(identifier) @variable
+
+(package_identifier) @module
+
+(parameter_declaration
+  (identifier) @variable.parameter)
+
+(variadic_parameter_declaration
+  (identifier) @variable.parameter)
+
+(label_name) @label
+
+(const_spec
+  name: (identifier) @constant)
+
+; Function calls
+(call_expression
+  function: (identifier) @function.call)
+
+(call_expression
+  function: (selector_expression
+    field: (field_identifier) @function.method.call))
+
+; Function definitions
+(function_declaration
+  name: (identifier) @function)
+
+(method_declaration
+  name: (field_identifier) @function.method)
+
+(method_elem
+  name: (field_identifier) @function.method)
+
+; Constructors
+((call_expression
+  (identifier) @constructor)
+  (#lua-match? @constructor "^[nN]ew.+$"))
+
+((call_expression
+  (identifier) @constructor)
+  (#lua-match? @constructor "^[mM]ake.+$"))
+
+; Operators
+[
+  "--"
+  "-"
+  "-="
+  ":="
+  "!"
+  "!="
+  "..."
+  "*"
+  "*"
+  "*="
+  "/"
+  "/="
+  "&"
+  "&&"
+  "&="
+  "&^"
+  "&^="
+  "%"
+  "%="
+  "^"
+  "^="
+  "+"
+  "++"
+  "+="
+  "<-"
+  "<"
+  "<<"
+  "<<="
+  "<="
+  "="
+  "=="
+  ">"
+  ">="
+  ">>"
+  ">>="
+  "|"
+  "|="
+  "||"
+  "~"
+] @operator
+
+; Keywords
+[
+  "break"
+  "const"
+  "continue"
+  "default"
+  "defer"
+  "goto"
+  "range"
+  "select"
+  "var"
+  "fallthrough"
+] @keyword
+
+[
+  "type"
+  "struct"
+  "interface"
+] @keyword.type
+
+"func" @keyword.function
+
+"return" @keyword.return
+
+"go" @keyword.coroutine
+
+"for" @keyword.repeat
+
+[
+  "import"
+  "package"
+] @keyword.import
+
+[
+  "else"
+  "case"
+  "switch"
+  "if"
+] @keyword.conditional
+
+; Builtin types
+[
+  "chan"
+  "map"
+] @type.builtin
+
+((type_identifier) @type.builtin
+  (#any-of? @type.builtin
+    "any" "bool" "byte" "comparable" "complex128" "complex64" "error" "float32" "float64" "int"
+    "int16" "int32" "int64" "int8" "rune" "string" "uint" "uint16" "uint32" "uint64" "uint8"
+    "uintptr"))
+
+; Builtin functions
+((identifier) @function.builtin
+  (#any-of? @function.builtin
+    "append" "cap" "clear" "close" "complex" "copy" "delete" "imag" "len" "make" "max" "min" "new"
+    "panic" "print" "println" "real" "recover"))
+
+; Delimiters
+"." @punctuation.delimiter
+
+"," @punctuation.delimiter
+
+":" @punctuation.delimiter
+
+";" @punctuation.delimiter
+
+"(" @punctuation.bracket
+
+")" @punctuation.bracket
+
+"{" @punctuation.bracket
+
+"}" @punctuation.bracket
+
+"[" @punctuation.bracket
+
+"]" @punctuation.bracket
+
+; Literals
+(interpreted_string_literal) @string
+
+(raw_string_literal) @string
+
+(rune_literal) @string
+
+(escape_sequence) @string.escape
+
+(int_literal) @number
+
+(float_literal) @number.float
+
+(imaginary_literal) @number
+
+[
+  (true)
+  (false)
+] @boolean
+
+[
+  (nil)
+  (iota)
+] @constant.builtin
+
+(keyed_element
+  .
+  (literal_element
+    (identifier) @variable.member))
+
+(field_declaration
+  name: (field_identifier) @variable.member)
+
+; Comments
+(comment) @comment @spell
+
+; Doc Comments
+(source_file
+  .
+  (comment)+ @comment.documentation)
+
+(source_file
+  (comment)+ @comment.documentation
+  .
+  (const_declaration))
+
+(source_file
+  (comment)+ @comment.documentation
+  .
+  (function_declaration))
+
+(source_file
+  (comment)+ @comment.documentation
+  .
+  (type_declaration))
+
+(source_file
+  (comment)+ @comment.documentation
+  .
+  (var_declaration))
+
+; Spell
+((interpreted_string_literal) @spell
+  (#not-has-parent? @spell import_spec))
+
+; Regex
+(call_expression
+  (selector_expression) @_function
+  (#any-of? @_function
+    "regexp.Match" "regexp.MatchReader" "regexp.MatchString" "regexp.Compile" "regexp.CompilePOSIX"
+    "regexp.MustCompile" "regexp.MustCompilePOSIX")
+  (argument_list
+    .
+    [
+      (raw_string_literal
+        (raw_string_literal_content) @string.regexp)
+      (interpreted_string_literal
+        (interpreted_string_literal_content) @string.regexp)
+    ]))

bex/tag_preprocessor/nvim-reference/java.scm

@@ -0,0 +1,330 @@
+; CREDITS @maxbrunsfeld (maxbrunsfeld@gmail.com)
+; Variables
+(identifier) @variable
+
+(underscore_pattern) @character.special
+
+; Methods
+(method_declaration
+  name: (identifier) @function.method)
+
+(method_invocation
+  name: (identifier) @function.method.call)
+
+(super) @function.builtin
+
+; Parameters
+(formal_parameter
+  name: (identifier) @variable.parameter)
+
+(spread_parameter
+  (variable_declarator
+    name: (identifier) @variable.parameter)) ; int... foo
+
+; Lambda parameter
+(inferred_parameters
+  (identifier) @variable.parameter) ; (x,y) -> ...
+
+(lambda_expression
+  parameters: (identifier) @variable.parameter) ; x -> ...
+
+; Operators
+[
+  "+"
+  ":"
+  "++"
+  "-"
+  "--"
+  "&"
+  "&&"
+  "|"
+  "||"
+  "!"
+  "!="
+  "=="
+  "*"
+  "/"
+  "%"
+  "<"
+  "<="
+  ">"
+  ">="
+  "="
+  "-="
+  "+="
+  "*="
+  "/="
+  "%="
+  "->"
+  "^"
+  "^="
+  "&="
+  "|="
+  "~"
+  ">>"
+  ">>>"
+  "<<"
+  "::"
+] @operator
+
+; Types
+(interface_declaration
+  name: (identifier) @type)
+
+(annotation_type_declaration
+  name: (identifier) @type)
+
+(class_declaration
+  name: (identifier) @type)
+
+(record_declaration
+  name: (identifier) @type)
+
+(enum_declaration
+  name: (identifier) @type)
+
+(constructor_declaration
+  name: (identifier) @type)
+
+(compact_constructor_declaration
+  name: (identifier) @type)
+
+(type_identifier) @type
+
+((type_identifier) @type.builtin
+  (#eq? @type.builtin "var"))
+
+((method_invocation
+  object: (identifier) @type)
+  (#lua-match? @type "^[A-Z]"))
+
+((method_reference
+  .
+  (identifier) @type)
+  (#lua-match? @type "^[A-Z]"))
+
+((field_access
+  object: (identifier) @type)
+  (#lua-match? @type "^[A-Z]"))
+
+(scoped_identifier
+  (identifier) @type
+  (#lua-match? @type "^[A-Z]"))
+
+; Fields
+(field_declaration
+  declarator: (variable_declarator
+    name: (identifier) @variable.member))
+
+(field_access
+  field: (identifier) @variable.member)
+
+[
+  (boolean_type)
+  (integral_type)
+  (floating_point_type)
+  (void_type)
+] @type.builtin
+
+; Variables
+((identifier) @constant
+  (#lua-match? @constant "^[A-Z_][A-Z%d_]+$"))
+
+(this) @variable.builtin
+
+; Annotations
+(annotation
+  "@" @attribute
+  name: (identifier) @attribute)
+
+(marker_annotation
+  "@" @attribute
+  name: (identifier) @attribute)
+
+; Literals
+(string_literal) @string
+
+(escape_sequence) @string.escape
+
+(character_literal) @character
+
+[
+  (hex_integer_literal)
+  (decimal_integer_literal)
+  (octal_integer_literal)
+  (binary_integer_literal)
+] @number
+
+[
+  (decimal_floating_point_literal)
+  (hex_floating_point_literal)
+] @number.float
+
+[
+  (true)
+  (false)
+] @boolean
+
+(null_literal) @constant.builtin
+
+; Keywords
+[
+  "assert"
+  "default"
+  "extends"
+  "implements"
+  "instanceof"
+  "@interface"
+  "permits"
+  "to"
+  "with"
+] @keyword
+
+[
+  "record"
+  "class"
+  "enum"
+  "interface"
+] @keyword.type
+
+(synchronized_statement
+  "synchronized" @keyword)
+
+[
+  "abstract"
+  "final"
+  "native"
+  "non-sealed"
+  "open"
+  "private"
+  "protected"
+  "public"
+  "sealed"
+  "static"
+  "strictfp"
+  "transitive"
+] @keyword.modifier
+
+(modifiers
+  "synchronized" @keyword.modifier)
+
+[
+  "transient"
+  "volatile"
+] @keyword.modifier
+
+[
+  "return"
+  "yield"
+] @keyword.return
+
+"new" @keyword.operator
+
+; Conditionals
+[
+  "if"
+  "else"
+  "switch"
+  "case"
+  "when"
+] @keyword.conditional
+
+(ternary_expression
+  [
+    "?"
+    ":"
+  ] @keyword.conditional.ternary)
+
+; Loops
+[
+  "for"
+  "while"
+  "do"
+  "continue"
+  "break"
+] @keyword.repeat
+
+; Includes
+[
+  "exports"
+  "import"
+  "module"
+  "opens"
+  "package"
+  "provides"
+  "requires"
+  "uses"
+] @keyword.import
+
+(import_declaration
+  (asterisk
+    "*" @character.special))
+
+; Punctuation
+[
+  ";"
+  "."
+  "..."
+  ","
+] @punctuation.delimiter
+
+[
+  "{"
+  "}"
+] @punctuation.bracket
+
+[
+  "["
+  "]"
+] @punctuation.bracket
+
+[
+  "("
+  ")"
+] @punctuation.bracket
+
+(type_arguments
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(type_parameters
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(string_interpolation
+  [
+    "\\{"
+    "}"
+  ] @punctuation.special)
+
+; Exceptions
+[
+  "throw"
+  "throws"
+  "finally"
+  "try"
+  "catch"
+] @keyword.exception
+
+; Labels
+(labeled_statement
+  (identifier) @label)
+
+; Comments
+[
+  (line_comment)
+  (block_comment)
+] @comment @spell
+
+((block_comment) @comment.documentation
+  (#lua-match? @comment.documentation "^/[*][*][^*].*[*]/$"))
+
+((line_comment) @comment.documentation
+  (#lua-match? @comment.documentation "^///[^/]"))
+
+((line_comment) @comment.documentation
+  (#lua-match? @comment.documentation "^///$"))

bex/tag_preprocessor/nvim-reference/javascript.scm

@@ -0,0 +1,56 @@
+; inherits: ecma,jsx
+
+; Parameters
+(formal_parameters
+  (identifier) @variable.parameter)
+
+(formal_parameters
+  (rest_pattern
+    (identifier) @variable.parameter))
+
+; ({ a }) => null
+(formal_parameters
+  (object_pattern
+    (shorthand_property_identifier_pattern) @variable.parameter))
+
+; ({ a = b }) => null
+(formal_parameters
+  (object_pattern
+    (object_assignment_pattern
+      (shorthand_property_identifier_pattern) @variable.parameter)))
+
+; ({ a: b }) => null
+(formal_parameters
+  (object_pattern
+    (pair_pattern
+      value: (identifier) @variable.parameter)))
+
+; ([ a ]) => null
+(formal_parameters
+  (array_pattern
+    (identifier) @variable.parameter))
+
+; ({ a } = { a }) => null
+(formal_parameters
+  (assignment_pattern
+    (object_pattern
+      (shorthand_property_identifier_pattern) @variable.parameter)))
+
+; ({ a = b } = { a }) => null
+(formal_parameters
+  (assignment_pattern
+    (object_pattern
+      (object_assignment_pattern
+        (shorthand_property_identifier_pattern) @variable.parameter))))
+
+; a => null
+(arrow_function
+  parameter: (identifier) @variable.parameter)
+
+; optional parameters
+(formal_parameters
+  (assignment_pattern
+    left: (identifier) @variable.parameter))
+
+; punctuation
+(optional_chain) @punctuation.delimiter

bex/tag_preprocessor/nvim-reference/kotlin.scm

@@ -0,0 +1,398 @@
+; Identifiers
+(simple_identifier) @variable
+
+; `it` keyword inside lambdas
+; FIXME: This will highlight the keyword outside of lambdas since tree-sitter
+;        does not allow us to check for arbitrary nestation
+((simple_identifier) @variable.builtin
+  (#eq? @variable.builtin "it"))
+
+; `field` keyword inside property getter/setter
+; FIXME: This will highlight the keyword outside of getters and setters
+;        since tree-sitter does not allow us to check for arbitrary nestation
+((simple_identifier) @variable.builtin
+  (#eq? @variable.builtin "field"))
+
+[
+  "this"
+  "super"
+  "this@"
+  "super@"
+] @variable.builtin
+
+; NOTE: for consistency with "super@"
+(super_expression
+  "@" @variable.builtin)
+
+(class_parameter
+  (simple_identifier) @variable.member)
+
+; NOTE: temporary fix for treesitter bug that causes delay in file opening
+;(class_body
+;  (property_declaration
+;    (variable_declaration
+;      (simple_identifier) @variable.member)))
+; id_1.id_2.id_3: `id_2` and `id_3` are assumed as object properties
+(_
+  (navigation_suffix
+    (simple_identifier) @variable.member))
+
+; SCREAMING CASE identifiers are assumed to be constants
+((simple_identifier) @constant
+  (#lua-match? @constant "^[A-Z][A-Z0-9_]*$"))
+
+(_
+  (navigation_suffix
+    (simple_identifier) @constant
+    (#lua-match? @constant "^[A-Z][A-Z0-9_]*$")))
+
+(enum_entry
+  (simple_identifier) @constant)
+
+(type_identifier) @type
+
+; '?' operator, replacement for Java @Nullable
+(nullable_type) @punctuation.special
+
+(type_alias
+  (type_identifier) @type.definition)
+
+((type_identifier) @type.builtin
+  (#any-of? @type.builtin
+    "Byte" "Short" "Int" "Long" "UByte" "UShort" "UInt" "ULong" "Float" "Double" "Boolean" "Char"
+    "String" "Array" "ByteArray" "ShortArray" "IntArray" "LongArray" "UByteArray" "UShortArray"
+    "UIntArray" "ULongArray" "FloatArray" "DoubleArray" "BooleanArray" "CharArray" "Map" "Set"
+    "List" "EmptyMap" "EmptySet" "EmptyList" "MutableMap" "MutableSet" "MutableList"))
+
+(package_header
+  "package" @keyword
+  .
+  (identifier
+    (simple_identifier) @module))
+
+(import_header
+  "import" @keyword.import)
+
+(wildcard_import) @character.special
+
+; The last `simple_identifier` in a `import_header` will always either be a function
+; or a type. Classes can appear anywhere in the import path, unlike functions
+(import_header
+  (identifier
+    (simple_identifier) @type @_import)
+  (import_alias
+    (type_identifier) @type.definition)?
+  (#lua-match? @_import "^[A-Z]"))
+
+(import_header
+  (identifier
+    (simple_identifier) @function @_import .)
+  (import_alias
+    (type_identifier) @function)?
+  (#lua-match? @_import "^[a-z]"))
+
+(label) @label
+
+; Function definitions
+(function_declaration
+  (simple_identifier) @function)
+
+(getter
+  "get" @function.builtin)
+
+(setter
+  "set" @function.builtin)
+
+(primary_constructor) @constructor
+
+(secondary_constructor
+  "constructor" @constructor)
+
+(constructor_invocation
+  (user_type
+    (type_identifier) @constructor))
+
+(anonymous_initializer
+  "init" @constructor)
+
+(parameter
+  (simple_identifier) @variable.parameter)
+
+(parameter_with_optional_type
+  (simple_identifier) @variable.parameter)
+
+; lambda parameters
+(lambda_literal
+  (lambda_parameters
+    (variable_declaration
+      (simple_identifier) @variable.parameter)))
+
+; Function calls
+; function()
+(call_expression
+  .
+  (simple_identifier) @function.call)
+
+; ::function
+(callable_reference
+  .
+  (simple_identifier) @function.call)
+
+; object.function() or object.property.function()
+(call_expression
+  (navigation_expression
+    (navigation_suffix
+      (simple_identifier) @function.call) .))
+
+(call_expression
+  .
+  (simple_identifier) @function.builtin
+  (#any-of? @function.builtin
+    "arrayOf" "arrayOfNulls" "byteArrayOf" "shortArrayOf" "intArrayOf" "longArrayOf" "ubyteArrayOf"
+    "ushortArrayOf" "uintArrayOf" "ulongArrayOf" "floatArrayOf" "doubleArrayOf" "booleanArrayOf"
+    "charArrayOf" "emptyArray" "mapOf" "setOf" "listOf" "emptyMap" "emptySet" "emptyList"
+    "mutableMapOf" "mutableSetOf" "mutableListOf" "print" "println" "error" "TODO" "run"
+    "runCatching" "repeat" "lazy" "lazyOf" "enumValues" "enumValueOf" "assert" "check"
+    "checkNotNull" "require" "requireNotNull" "with" "suspend" "synchronized"))
+
+; Literals
+[
+  (line_comment)
+  (multiline_comment)
+] @comment @spell
+
+((multiline_comment) @comment.documentation
+  (#lua-match? @comment.documentation "^/[*][*][^*].*[*]/$"))
+
+(shebang_line) @keyword.directive
+
+(real_literal) @number.float
+
+[
+  (integer_literal)
+  (long_literal)
+  (hex_literal)
+  (bin_literal)
+  (unsigned_literal)
+] @number
+
+[
+  (null_literal)
+  ; should be highlighted the same as booleans
+  (boolean_literal)
+] @boolean
+
+(character_literal) @character
+
+(string_literal) @string
+
+; NOTE: Escapes not allowed in multi-line strings
+(character_literal
+  (character_escape_seq) @string.escape)
+
+; There are 3 ways to define a regex
+;    - "[abc]?".toRegex()
+(call_expression
+  (navigation_expression
+    (string_literal) @string.regexp
+    (navigation_suffix
+      ((simple_identifier) @_function
+        (#eq? @_function "toRegex")))))
+
+;    - Regex("[abc]?")
+(call_expression
+  ((simple_identifier) @_function
+    (#eq? @_function "Regex"))
+  (call_suffix
+    (value_arguments
+      (value_argument
+        (string_literal) @string.regexp))))
+
+;    - Regex.fromLiteral("[abc]?")
+(call_expression
+  (navigation_expression
+    ((simple_identifier) @_class
+      (#eq? @_class "Regex"))
+    (navigation_suffix
+      ((simple_identifier) @_function
+        (#eq? @_function "fromLiteral"))))
+  (call_suffix
+    (value_arguments
+      (value_argument
+        (string_literal) @string.regexp))))
+
+; Keywords
+(type_alias
+  "typealias" @keyword)
+
+(companion_object
+  "companion" @keyword)
+
+[
+  (class_modifier)
+  (member_modifier)
+  (function_modifier)
+  (property_modifier)
+  (platform_modifier)
+  (variance_modifier)
+  (parameter_modifier)
+  (visibility_modifier)
+  (reification_modifier)
+  (inheritance_modifier)
+] @keyword.modifier
+
+[
+  "val"
+  "var"
+  ;	"typeof" ; NOTE: It is reserved for future use
+] @keyword
+
+[
+  "enum"
+  "class"
+  "object"
+  "interface"
+] @keyword.type
+
+[
+  "return"
+  "return@"
+] @keyword.return
+
+"suspend" @keyword.coroutine
+
+"fun" @keyword.function
+
+[
+  "if"
+  "else"
+  "when"
+] @keyword.conditional
+
+[
+  "for"
+  "do"
+  "while"
+  "continue"
+  "continue@"
+  "break"
+  "break@"
+] @keyword.repeat
+
+[
+  "try"
+  "catch"
+  "throw"
+  "finally"
+] @keyword.exception
+
+(annotation
+  "@" @attribute
+  (use_site_target)? @attribute)
+
+(annotation
+  (user_type
+    (type_identifier) @attribute))
+
+(annotation
+  (constructor_invocation
+    (user_type
+      (type_identifier) @attribute)))
+
+(file_annotation
+  "@" @attribute
+  "file" @attribute
+  ":" @attribute)
+
+(file_annotation
+  (user_type
+    (type_identifier) @attribute))
+
+(file_annotation
+  (constructor_invocation
+    (user_type
+      (type_identifier) @attribute)))
+
+; Operators & Punctuation
+[
+  "!"
+  "!="
+  "!=="
+  "="
+  "=="
+  "==="
+  ">"
+  ">="
+  "<"
+  "<="
+  "||"
+  "&&"
+  "+"
+  "++"
+  "+="
+  "-"
+  "--"
+  "-="
+  "*"
+  "*="
+  "/"
+  "/="
+  "%"
+  "%="
+  "?."
+  "?:"
+  "!!"
+  "is"
+  "!is"
+  "in"
+  "!in"
+  "as"
+  "as?"
+  ".."
+  "->"
+] @operator
+
+[
+  "("
+  ")"
+  "["
+  "]"
+  "{"
+  "}"
+] @punctuation.bracket
+
+[
+  "."
+  ","
+  ";"
+  ":"
+  "::"
+] @punctuation.delimiter
+
+(super_expression
+  [
+    "<"
+    ">"
+  ] @punctuation.delimiter)
+
+(type_arguments
+  [
+    "<"
+    ">"
+  ] @punctuation.delimiter)
+
+(type_parameters
+  [
+    "<"
+    ">"
+  ] @punctuation.delimiter)
+
+; NOTE: `interpolated_identifier`s can be highlighted in any way
+(string_literal
+  "$" @punctuation.special
+  (interpolated_identifier) @none @variable)
+
+(string_literal
+  "${" @punctuation.special
+  (interpolated_expression) @none
+  "}" @punctuation.special)

bex/tag_preprocessor/nvim-reference/python.scm

@@ -0,0 +1,443 @@
+; From tree-sitter-python licensed under MIT License
+; Copyright (c) 2016 Max Brunsfeld
+; Variables
+(identifier) @variable
+
+; Reset highlighting in f-string interpolations
+(interpolation) @none
+
+; Identifier naming conventions
+((identifier) @type
+  (#lua-match? @type "^[A-Z].*[a-z]"))
+
+((identifier) @constant
+  (#lua-match? @constant "^[A-Z][A-Z_0-9]*$"))
+
+((identifier) @constant.builtin
+  (#lua-match? @constant.builtin "^__[a-zA-Z0-9_]*__$"))
+
+((identifier) @constant.builtin
+  (#any-of? @constant.builtin
+    ; https://docs.python.org/3/library/constants.html
+    "NotImplemented" "Ellipsis" "quit" "exit" "copyright" "credits" "license"))
+
+"_" @character.special ; match wildcard
+
+((assignment
+  left: (identifier) @type.definition
+  (type
+    (identifier) @_annotation))
+  (#eq? @_annotation "TypeAlias"))
+
+((assignment
+  left: (identifier) @type.definition
+  right: (call
+    function: (identifier) @_func))
+  (#any-of? @_func "TypeVar" "NewType"))
+
+; Function definitions
+(function_definition
+  name: (identifier) @function)
+
+(type
+  (identifier) @type)
+
+(type
+  (subscript
+    (identifier) @type)) ; type subscript: Tuple[int]
+
+((call
+  function: (identifier) @_isinstance
+  arguments: (argument_list
+    (_)
+    (identifier) @type))
+  (#eq? @_isinstance "isinstance"))
+
+; Literals
+(none) @constant.builtin
+
+[
+  (true)
+  (false)
+] @boolean
+
+(integer) @number
+
+(float) @number.float
+
+(comment) @comment @spell
+
+((module
+  .
+  (comment) @keyword.directive @nospell)
+  (#lua-match? @keyword.directive "^#!/"))
+
+(string) @string
+
+[
+  (escape_sequence)
+  (escape_interpolation)
+] @string.escape
+
+; doc-strings
+(expression_statement
+  (string
+    (string_content) @spell) @string.documentation)
+
+; Tokens
+[
+  "-"
+  "-="
+  ":="
+  "!="
+  "*"
+  "**"
+  "**="
+  "*="
+  "/"
+  "//"
+  "//="
+  "/="
+  "&"
+  "&="
+  "%"
+  "%="
+  "^"
+  "^="
+  "+"
+  "+="
+  "<"
+  "<<"
+  "<<="
+  "<="
+  "<>"
+  "="
+  "=="
+  ">"
+  ">="
+  ">>"
+  ">>="
+  "@"
+  "@="
+  "|"
+  "|="
+  "~"
+  "->"
+] @operator
+
+; Keywords
+[
+  "and"
+  "in"
+  "is"
+  "not"
+  "or"
+  "is not"
+  "not in"
+  "del"
+] @keyword.operator
+
+[
+  "def"
+  "lambda"
+] @keyword.function
+
+[
+  "assert"
+  "exec"
+  "global"
+  "nonlocal"
+  "pass"
+  "print"
+  "with"
+  "as"
+] @keyword
+
+[
+  "type"
+  "class"
+] @keyword.type
+
+[
+  "async"
+  "await"
+] @keyword.coroutine
+
+[
+  "return"
+  "yield"
+] @keyword.return
+
+(yield
+  "from" @keyword.return)
+
+(future_import_statement
+  "from" @keyword.import
+  "__future__" @module.builtin)
+
+(import_from_statement
+  "from" @keyword.import)
+
+"import" @keyword.import
+
+(aliased_import
+  "as" @keyword.import)
+
+(wildcard_import
+  "*" @character.special)
+
+(import_statement
+  name: (dotted_name
+    (identifier) @module))
+
+(import_statement
+  name: (aliased_import
+    name: (dotted_name
+      (identifier) @module)
+    alias: (identifier) @module))
+
+(import_from_statement
+  module_name: (dotted_name
+    (identifier) @module))
+
+(import_from_statement
+  module_name: (relative_import
+    (dotted_name
+      (identifier) @module)))
+
+[
+  "if"
+  "elif"
+  "else"
+  "match"
+  "case"
+] @keyword.conditional
+
+[
+  "for"
+  "while"
+  "break"
+  "continue"
+] @keyword.repeat
+
+[
+  "try"
+  "except"
+  "except*"
+  "raise"
+  "finally"
+] @keyword.exception
+
+(raise_statement
+  "from" @keyword.exception)
+
+(try_statement
+  (else_clause
+    "else" @keyword.exception))
+
+[
+  "("
+  ")"
+  "["
+  "]"
+  "{"
+  "}"
+] @punctuation.bracket
+
+(interpolation
+  "{" @punctuation.special
+  "}" @punctuation.special)
+
+(type_conversion) @function.macro
+
+[
+  ","
+  "."
+  ":"
+  ";"
+  (ellipsis)
+] @punctuation.delimiter
+
+((identifier) @type.builtin
+  (#any-of? @type.builtin
+    ; https://docs.python.org/3/library/exceptions.html
+    "BaseException" "Exception" "ArithmeticError" "BufferError" "LookupError" "AssertionError"
+    "AttributeError" "EOFError" "FloatingPointError" "GeneratorExit" "ImportError"
+    "ModuleNotFoundError" "IndexError" "KeyError" "KeyboardInterrupt" "MemoryError" "NameError"
+    "NotImplementedError" "OSError" "OverflowError" "RecursionError" "ReferenceError" "RuntimeError"
+    "StopIteration" "StopAsyncIteration" "SyntaxError" "IndentationError" "TabError" "SystemError"
+    "SystemExit" "TypeError" "UnboundLocalError" "UnicodeError" "UnicodeEncodeError"
+    "UnicodeDecodeError" "UnicodeTranslateError" "ValueError" "ZeroDivisionError" "EnvironmentError"
+    "IOError" "WindowsError" "BlockingIOError" "ChildProcessError" "ConnectionError"
+    "BrokenPipeError" "ConnectionAbortedError" "ConnectionRefusedError" "ConnectionResetError"
+    "FileExistsError" "FileNotFoundError" "InterruptedError" "IsADirectoryError"
+    "NotADirectoryError" "PermissionError" "ProcessLookupError" "TimeoutError" "Warning"
+    "UserWarning" "DeprecationWarning" "PendingDeprecationWarning" "SyntaxWarning" "RuntimeWarning"
+    "FutureWarning" "ImportWarning" "UnicodeWarning" "BytesWarning" "ResourceWarning"
+    ; https://docs.python.org/3/library/stdtypes.html
+    "bool" "int" "float" "complex" "list" "tuple" "range" "str" "bytes" "bytearray" "memoryview"
+    "set" "frozenset" "dict" "type" "object"))
+
+; Normal parameters
+(parameters
+  (identifier) @variable.parameter)
+
+; Lambda parameters
+(lambda_parameters
+  (identifier) @variable.parameter)
+
+(lambda_parameters
+  (tuple_pattern
+    (identifier) @variable.parameter))
+
+; Default parameters
+(keyword_argument
+  name: (identifier) @variable.parameter)
+
+; Naming parameters on call-site
+(default_parameter
+  name: (identifier) @variable.parameter)
+
+(typed_parameter
+  (identifier) @variable.parameter)
+
+(typed_default_parameter
+  name: (identifier) @variable.parameter)
+
+; Variadic parameters *args, **kwargs
+(parameters
+  (list_splat_pattern ; *args
+    (identifier) @variable.parameter))
+
+(parameters
+  (dictionary_splat_pattern ; **kwargs
+    (identifier) @variable.parameter))
+
+; Typed variadic parameters
+(parameters
+  (typed_parameter
+    (list_splat_pattern ; *args: type
+      (identifier) @variable.parameter)))
+
+(parameters
+  (typed_parameter
+    (dictionary_splat_pattern ; *kwargs: type
+      (identifier) @variable.parameter)))
+
+; Lambda parameters
+(lambda_parameters
+  (list_splat_pattern
+    (identifier) @variable.parameter))
+
+(lambda_parameters
+  (dictionary_splat_pattern
+    (identifier) @variable.parameter))
+
+((identifier) @variable.builtin
+  (#eq? @variable.builtin "self"))
+
+((identifier) @variable.builtin
+  (#eq? @variable.builtin "cls"))
+
+; After @type.builtin bacause builtins (such as `type`) are valid as attribute name
+((attribute
+  attribute: (identifier) @variable.member)
+  (#lua-match? @variable.member "^[%l_].*$"))
+
+; Class definitions
+(class_definition
+  name: (identifier) @type)
+
+(class_definition
+  body: (block
+    (function_definition
+      name: (identifier) @function.method)))
+
+(class_definition
+  superclasses: (argument_list
+    (identifier) @type))
+
+((class_definition
+  body: (block
+    (expression_statement
+      (assignment
+        left: (identifier) @variable.member))))
+  (#lua-match? @variable.member "^[%l_].*$"))
+
+((class_definition
+  body: (block
+    (expression_statement
+      (assignment
+        left: (_
+          (identifier) @variable.member)))))
+  (#lua-match? @variable.member "^[%l_].*$"))
+
+((class_definition
+  (block
+    (function_definition
+      name: (identifier) @constructor)))
+  (#any-of? @constructor "__new__" "__init__"))
+
+; Function calls
+(call
+  function: (identifier) @function.call)
+
+(call
+  function: (attribute
+    attribute: (identifier) @function.method.call))
+
+((call
+  function: (identifier) @constructor)
+  (#lua-match? @constructor "^%u"))
+
+((call
+  function: (attribute
+    attribute: (identifier) @constructor))
+  (#lua-match? @constructor "^%u"))
+
+; Builtin functions
+((call
+  function: (identifier) @function.builtin)
+  (#any-of? @function.builtin
+    "abs" "all" "any" "ascii" "bin" "bool" "breakpoint" "bytearray" "bytes" "callable" "chr"
+    "classmethod" "compile" "complex" "delattr" "dict" "dir" "divmod" "enumerate" "eval" "exec"
+    "filter" "float" "format" "frozenset" "getattr" "globals" "hasattr" "hash" "help" "hex" "id"
+    "input" "int" "isinstance" "issubclass" "iter" "len" "list" "locals" "map" "max" "memoryview"
+    "min" "next" "object" "oct" "open" "ord" "pow" "print" "property" "range" "repr" "reversed"
+    "round" "set" "setattr" "slice" "sorted" "staticmethod" "str" "sum" "super" "tuple" "type"
+    "vars" "zip" "__import__"))
+
+; Regex from the `re` module
+(call
+  function: (attribute
+    object: (identifier) @_re)
+  arguments: (argument_list
+    .
+    (string
+      (string_content) @string.regexp))
+  (#eq? @_re "re"))
+
+; Decorators
+((decorator
+  "@" @attribute)
+  (#set! priority 101))
+
+(decorator
+  (identifier) @attribute)
+
+(decorator
+  (attribute
+    attribute: (identifier) @attribute))
+
+(decorator
+  (call
+    (identifier) @attribute))
+
+(decorator
+  (call
+    (attribute
+      attribute: (identifier) @attribute)))
+
+((decorator
+  (identifier) @attribute.builtin)
+  (#any-of? @attribute.builtin "classmethod" "property" "staticmethod"))

bex/tag_preprocessor/nvim-reference/ruby.scm

@@ -0,0 +1,309 @@
+; Variables
+[
+  (identifier)
+  (global_variable)
+] @variable
+
+; Keywords
+[
+  "alias"
+  "begin"
+  "do"
+  "end"
+  "ensure"
+  "module"
+  "rescue"
+  "then"
+] @keyword
+
+"class" @keyword.type
+
+[
+  "return"
+  "yield"
+] @keyword.return
+
+[
+  "and"
+  "or"
+  "in"
+  "not"
+] @keyword.operator
+
+[
+  "def"
+  "undef"
+] @keyword.function
+
+(method
+  "end" @keyword.function)
+
+[
+  "case"
+  "else"
+  "elsif"
+  "if"
+  "unless"
+  "when"
+  "then"
+] @keyword.conditional
+
+(if
+  "end" @keyword.conditional)
+
+[
+  "for"
+  "until"
+  "while"
+  "break"
+  "redo"
+  "retry"
+  "next"
+] @keyword.repeat
+
+(constant) @constant
+
+((identifier) @keyword.modifier
+  (#any-of? @keyword.modifier "private" "protected" "public"))
+
+[
+  "rescue"
+  "ensure"
+] @keyword.exception
+
+; Function calls
+"defined?" @function
+
+(call
+  receiver: (constant)? @type
+  method: [
+    (identifier)
+    (constant)
+  ] @function.call)
+
+(program
+  (call
+    (identifier) @keyword.import)
+  (#any-of? @keyword.import "require" "require_relative" "load"))
+
+; Function definitions
+(alias
+  (identifier) @function)
+
+(setter
+  (identifier) @function)
+
+(method
+  name: [
+    (identifier) @function
+    (constant) @type
+  ])
+
+(singleton_method
+  name: [
+    (identifier) @function
+    (constant) @type
+  ])
+
+(class
+  name: (constant) @type)
+
+(module
+  name: (constant) @type)
+
+(superclass
+  (constant) @type)
+
+; Identifiers
+[
+  (class_variable)
+  (instance_variable)
+] @variable.member
+
+((identifier) @constant.builtin
+  (#any-of? @constant.builtin
+    "__callee__" "__dir__" "__id__" "__method__" "__send__" "__ENCODING__" "__FILE__" "__LINE__"))
+
+((identifier) @function.builtin
+  (#any-of? @function.builtin "attr_reader" "attr_writer" "attr_accessor" "module_function"))
+
+((call
+  !receiver
+  method: (identifier) @function.builtin)
+  (#any-of? @function.builtin "include" "extend" "prepend" "refine" "using"))
+
+((identifier) @keyword.exception
+  (#any-of? @keyword.exception "raise" "fail" "catch" "throw"))
+
+((constant) @type
+  (#not-lua-match? @type "^[A-Z0-9_]+$"))
+
+[
+  (self)
+  (super)
+] @variable.builtin
+
+(method_parameters
+  (identifier) @variable.parameter)
+
+(lambda_parameters
+  (identifier) @variable.parameter)
+
+(block_parameters
+  (identifier) @variable.parameter)
+
+(splat_parameter
+  (identifier) @variable.parameter)
+
+(hash_splat_parameter
+  (identifier) @variable.parameter)
+
+(optional_parameter
+  (identifier) @variable.parameter)
+
+(destructured_parameter
+  (identifier) @variable.parameter)
+
+(block_parameter
+  (identifier) @variable.parameter)
+
+(keyword_parameter
+  (identifier) @variable.parameter)
+
+; TODO: Re-enable this once it is supported
+; ((identifier) @function
+;  (#is-not? local))
+; Literals
+[
+  (string_content)
+  (heredoc_content)
+  "\""
+  "`"
+] @string
+
+[
+  (heredoc_beginning)
+  (heredoc_end)
+] @label
+
+[
+  (bare_symbol)
+  (simple_symbol)
+  (delimited_symbol)
+  (hash_key_symbol)
+] @string.special.symbol
+
+(regex
+  (string_content) @string.regexp)
+
+(escape_sequence) @string.escape
+
+(integer) @number
+
+(float) @number.float
+
+[
+  (true)
+  (false)
+] @boolean
+
+(nil) @constant.builtin
+
+(comment) @comment @spell
+
+((program
+  .
+  (comment) @keyword.directive @nospell)
+  (#lua-match? @keyword.directive "^#!/"))
+
+(program
+  (comment)+ @comment.documentation
+  (class))
+
+(module
+  (comment)+ @comment.documentation
+  (body_statement
+    (class)))
+
+(class
+  (comment)+ @comment.documentation
+  (body_statement
+    (method)))
+
+(body_statement
+  (comment)+ @comment.documentation
+  (method))
+
+; Operators
+[
+  "!"
+  "="
+  "=="
+  "==="
+  "<=>"
+  "=>"
+  "->"
+  ">>"
+  "<<"
+  ">"
+  "<"
+  ">="
+  "<="
+  "**"
+  "*"
+  "/"
+  "%"
+  "+"
+  "-"
+  "&"
+  "|"
+  "^"
+  "&&"
+  "||"
+  "||="
+  "&&="
+  "!="
+  "%="
+  "+="
+  "-="
+  "*="
+  "/="
+  "=~"
+  "!~"
+  "?"
+  ":"
+  ".."
+  "..."
+] @operator
+
+[
+  ","
+  ";"
+  "."
+  "&."
+  "::"
+] @punctuation.delimiter
+
+(regex
+  "/" @punctuation.bracket)
+
+(pair
+  ":" @punctuation.delimiter)
+
+[
+  "("
+  ")"
+  "["
+  "]"
+  "{"
+  "}"
+  "%w("
+  "%i("
+] @punctuation.bracket
+
+(block_parameters
+  "|" @punctuation.bracket)
+
+(interpolation
+  "#{" @punctuation.special
+  "}" @punctuation.special)

bex/tag_preprocessor/nvim-reference/rust.scm

@@ -0,0 +1,531 @@
+; Forked from https://github.com/tree-sitter/tree-sitter-rust
+; Copyright (c) 2017 Maxim Sokolov
+; Licensed under the MIT license.
+; Identifier conventions
+(shebang) @keyword.directive
+
+(identifier) @variable
+
+((identifier) @type
+  (#lua-match? @type "^[A-Z]"))
+
+(const_item
+  name: (identifier) @constant)
+
+; Assume all-caps names are constants
+((identifier) @constant
+  (#lua-match? @constant "^[A-Z][A-Z%d_]*$"))
+
+; Other identifiers
+(type_identifier) @type
+
+(primitive_type) @type.builtin
+
+(field_identifier) @variable.member
+
+(shorthand_field_identifier) @variable.member
+
+(shorthand_field_initializer
+  (identifier) @variable.member)
+
+(mod_item
+  name: (identifier) @module)
+
+(self) @variable.builtin
+
+"_" @character.special
+
+(label
+  [
+    "'"
+    (identifier)
+  ] @label)
+
+; Function definitions
+(function_item
+  (identifier) @function)
+
+(function_signature_item
+  (identifier) @function)
+
+(parameter
+  [
+    (identifier)
+    "_"
+  ] @variable.parameter)
+
+(parameter
+  (ref_pattern
+    [
+      (mut_pattern
+        (identifier) @variable.parameter)
+      (identifier) @variable.parameter
+    ]))
+
+(closure_parameters
+  (_) @variable.parameter)
+
+; Function calls
+(call_expression
+  function: (identifier) @function.call)
+
+(call_expression
+  function: (scoped_identifier
+    (identifier) @function.call .))
+
+(call_expression
+  function: (field_expression
+    field: (field_identifier) @function.call))
+
+(generic_function
+  function: (identifier) @function.call)
+
+(generic_function
+  function: (scoped_identifier
+    name: (identifier) @function.call))
+
+(generic_function
+  function: (field_expression
+    field: (field_identifier) @function.call))
+
+; Assume other uppercase names are enum constructors
+((field_identifier) @constant
+  (#lua-match? @constant "^[A-Z]"))
+
+(enum_variant
+  name: (identifier) @constant)
+
+; Assume that uppercase names in paths are types
+(scoped_identifier
+  path: (identifier) @module)
+
+(scoped_identifier
+  (scoped_identifier
+    name: (identifier) @module))
+
+(scoped_type_identifier
+  path: (identifier) @module)
+
+(scoped_type_identifier
+  path: (identifier) @type
+  (#lua-match? @type "^[A-Z]"))
+
+(scoped_type_identifier
+  (scoped_identifier
+    name: (identifier) @module))
+
+((scoped_identifier
+  path: (identifier) @type)
+  (#lua-match? @type "^[A-Z]"))
+
+((scoped_identifier
+  name: (identifier) @type)
+  (#lua-match? @type "^[A-Z]"))
+
+((scoped_identifier
+  name: (identifier) @constant)
+  (#lua-match? @constant "^[A-Z][A-Z%d_]*$"))
+
+((scoped_identifier
+  path: (identifier) @type
+  name: (identifier) @constant)
+  (#lua-match? @type "^[A-Z]")
+  (#lua-match? @constant "^[A-Z]"))
+
+((scoped_type_identifier
+  path: (identifier) @type
+  name: (type_identifier) @constant)
+  (#lua-match? @type "^[A-Z]")
+  (#lua-match? @constant "^[A-Z]"))
+
+[
+  (crate)
+  (super)
+] @module
+
+(scoped_use_list
+  path: (identifier) @module)
+
+(scoped_use_list
+  path: (scoped_identifier
+    (identifier) @module))
+
+(use_list
+  (scoped_identifier
+    (identifier) @module
+    .
+    (_)))
+
+(use_list
+  (identifier) @type
+  (#lua-match? @type "^[A-Z]"))
+
+(use_as_clause
+  alias: (identifier) @type
+  (#lua-match? @type "^[A-Z]"))
+
+; Correct enum constructors
+(call_expression
+  function: (scoped_identifier
+    "::"
+    name: (identifier) @constant)
+  (#lua-match? @constant "^[A-Z]"))
+
+; Assume uppercase names in a match arm are constants.
+((match_arm
+  pattern: (match_pattern
+    (identifier) @constant))
+  (#lua-match? @constant "^[A-Z]"))
+
+((match_arm
+  pattern: (match_pattern
+    (scoped_identifier
+      name: (identifier) @constant)))
+  (#lua-match? @constant "^[A-Z]"))
+
+((identifier) @constant.builtin
+  (#any-of? @constant.builtin "Some" "None" "Ok" "Err"))
+
+; Macro definitions
+"$" @function.macro
+
+(metavariable) @function.macro
+
+(macro_definition
+  "macro_rules!" @function.macro)
+
+; Attribute macros
+(attribute_item
+  (attribute
+    (identifier) @function.macro))
+
+(inner_attribute_item
+  (attribute
+    (identifier) @function.macro))
+
+(attribute
+  (scoped_identifier
+    (identifier) @function.macro .))
+
+; Derive macros (assume all arguments are types)
+; (attribute
+;   (identifier) @_name
+;   arguments: (attribute (attribute (identifier) @type))
+;   (#eq? @_name "derive"))
+; Function-like macros
+(macro_invocation
+  macro: (identifier) @function.macro)
+
+(macro_invocation
+  macro: (scoped_identifier
+    (identifier) @function.macro .))
+
+; Literals
+(boolean_literal) @boolean
+
+(integer_literal) @number
+
+(float_literal) @number.float
+
+[
+  (raw_string_literal)
+  (string_literal)
+] @string
+
+(escape_sequence) @string.escape
+
+(char_literal) @character
+
+; Keywords
+[
+  "use"
+  "mod"
+] @keyword.import
+
+(use_as_clause
+  "as" @keyword.import)
+
+[
+  "default"
+  "impl"
+  "let"
+  "move"
+  "unsafe"
+  "where"
+] @keyword
+
+[
+  "enum"
+  "struct"
+  "union"
+  "trait"
+  "type"
+] @keyword.type
+
+[
+  "async"
+  "await"
+  "gen"
+] @keyword.coroutine
+
+"try" @keyword.exception
+
+[
+  "ref"
+  "pub"
+  "raw"
+  (mutable_specifier)
+  "const"
+  "static"
+  "dyn"
+  "extern"
+] @keyword.modifier
+
+(lifetime
+  "'" @keyword.modifier)
+
+(lifetime
+  (identifier) @attribute)
+
+(lifetime
+  (identifier) @attribute.builtin
+  (#any-of? @attribute.builtin "static" "_"))
+
+"fn" @keyword.function
+
+[
+  "return"
+  "yield"
+] @keyword.return
+
+(type_cast_expression
+  "as" @keyword.operator)
+
+(qualified_type
+  "as" @keyword.operator)
+
+(use_list
+  (self) @module)
+
+(scoped_use_list
+  (self) @module)
+
+(scoped_identifier
+  [
+    (crate)
+    (super)
+    (self)
+  ] @module)
+
+(visibility_modifier
+  [
+    (crate)
+    (super)
+    (self)
+  ] @module)
+
+[
+  "if"
+  "else"
+  "match"
+] @keyword.conditional
+
+[
+  "break"
+  "continue"
+  "in"
+  "loop"
+  "while"
+] @keyword.repeat
+
+"for" @keyword
+
+(for_expression
+  "for" @keyword.repeat)
+
+; Operators
+[
+  "!"
+  "!="
+  "%"
+  "%="
+  "&"
+  "&&"
+  "&="
+  "*"
+  "*="
+  "+"
+  "+="
+  "-"
+  "-="
+  ".."
+  "..="
+  "..."
+  "/"
+  "/="
+  "<"
+  "<<"
+  "<<="
+  "<="
+  "="
+  "=="
+  ">"
+  ">="
+  ">>"
+  ">>="
+  "?"
+  "@"
+  "^"
+  "^="
+  "|"
+  "|="
+  "||"
+] @operator
+
+(use_wildcard
+  "*" @character.special)
+
+(remaining_field_pattern
+  ".." @character.special)
+
+(range_pattern
+  [
+    ".."
+    "..="
+    "..."
+  ] @character.special)
+
+; Punctuation
+[
+  "("
+  ")"
+  "["
+  "]"
+  "{"
+  "}"
+] @punctuation.bracket
+
+(closure_parameters
+  "|" @punctuation.bracket)
+
+(type_arguments
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(type_parameters
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(bracketed_type
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(for_lifetimes
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+[
+  ","
+  "."
+  ":"
+  "::"
+  ";"
+  "->"
+  "=>"
+] @punctuation.delimiter
+
+(attribute_item
+  "#" @punctuation.special)
+
+(inner_attribute_item
+  [
+    "!"
+    "#"
+  ] @punctuation.special)
+
+(macro_invocation
+  "!" @function.macro)
+
+(never_type
+  "!" @type.builtin)
+
+(macro_invocation
+  macro: (identifier) @_identifier @keyword.exception
+  "!" @keyword.exception
+  (#eq? @_identifier "panic"))
+
+(macro_invocation
+  macro: (identifier) @_identifier @keyword.exception
+  "!" @keyword.exception
+  (#contains? @_identifier "assert"))
+
+(macro_invocation
+  macro: (identifier) @_identifier @keyword.debug
+  "!" @keyword.debug
+  (#eq? @_identifier "dbg"))
+
+; Comments
+[
+  (line_comment)
+  (block_comment)
+  (outer_doc_comment_marker)
+  (inner_doc_comment_marker)
+] @comment @spell
+
+(line_comment
+  (doc_comment)) @comment.documentation
+
+(block_comment
+  (doc_comment)) @comment.documentation
+
+(call_expression
+  function: (scoped_identifier
+    path: (identifier) @_regex
+    (#any-of? @_regex "Regex" "ByteRegexBuilder")
+    name: (identifier) @_new
+    (#eq? @_new "new"))
+  arguments: (arguments
+    (raw_string_literal
+      (string_content) @string.regexp)))
+
+(call_expression
+  function: (scoped_identifier
+    path: (scoped_identifier
+      (identifier) @_regex
+      (#any-of? @_regex "Regex" "ByteRegexBuilder") .)
+    name: (identifier) @_new
+    (#eq? @_new "new"))
+  arguments: (arguments
+    (raw_string_literal
+      (string_content) @string.regexp)))
+
+(call_expression
+  function: (scoped_identifier
+    path: (identifier) @_regex
+    (#any-of? @_regex "RegexSet" "RegexSetBuilder")
+    name: (identifier) @_new
+    (#eq? @_new "new"))
+  arguments: (arguments
+    (array_expression
+      (raw_string_literal
+        (string_content) @string.regexp))))
+
+(call_expression
+  function: (scoped_identifier
+    path: (scoped_identifier
+      (identifier) @_regex
+      (#any-of? @_regex "RegexSet" "RegexSetBuilder") .)
+    name: (identifier) @_new
+    (#eq? @_new "new"))
+  arguments: (arguments
+    (array_expression
+      (raw_string_literal
+        (string_content) @string.regexp))))

bex/tag_preprocessor/nvim-reference/typescript.scm

@@ -0,0 +1,208 @@
+; inherits: ecma
+
+"require" @keyword.import
+
+(import_require_clause
+  source: (string) @string.special.url)
+
+[
+  "declare"
+  "implements"
+  "type"
+  "override"
+  "module"
+  "asserts"
+  "infer"
+  "is"
+  "using"
+] @keyword
+
+[
+  "namespace"
+  "interface"
+  "enum"
+] @keyword.type
+
+[
+  "keyof"
+  "satisfies"
+] @keyword.operator
+
+(as_expression
+  "as" @keyword.operator)
+
+(mapped_type_clause
+  "as" @keyword.operator)
+
+[
+  "abstract"
+  "private"
+  "protected"
+  "public"
+  "readonly"
+] @keyword.modifier
+
+; types
+(type_identifier) @type
+
+(predefined_type) @type.builtin
+
+(import_statement
+  "type"
+  (import_clause
+    (named_imports
+      (import_specifier
+        name: (identifier) @type))))
+
+(template_literal_type) @string
+
+(non_null_expression
+  "!" @operator)
+
+; punctuation
+(type_arguments
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(type_parameters
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(object_type
+  [
+    "{|"
+    "|}"
+  ] @punctuation.bracket)
+
+(union_type
+  "|" @punctuation.delimiter)
+
+(intersection_type
+  "&" @punctuation.delimiter)
+
+(type_annotation
+  ":" @punctuation.delimiter)
+
+(type_predicate_annotation
+  ":" @punctuation.delimiter)
+
+(index_signature
+  ":" @punctuation.delimiter)
+
+(omitting_type_annotation
+  "-?:" @punctuation.delimiter)
+
+(adding_type_annotation
+  "+?:" @punctuation.delimiter)
+
+(opting_type_annotation
+  "?:" @punctuation.delimiter)
+
+"?." @punctuation.delimiter
+
+(abstract_method_signature
+  "?" @punctuation.special)
+
+(method_signature
+  "?" @punctuation.special)
+
+(method_definition
+  "?" @punctuation.special)
+
+(property_signature
+  "?" @punctuation.special)
+
+(optional_parameter
+  "?" @punctuation.special)
+
+(optional_type
+  "?" @punctuation.special)
+
+(public_field_definition
+  [
+    "?"
+    "!"
+  ] @punctuation.special)
+
+(flow_maybe_type
+  "?" @punctuation.special)
+
+(template_type
+  [
+    "${"
+    "}"
+  ] @punctuation.special)
+
+(conditional_type
+  [
+    "?"
+    ":"
+  ] @keyword.conditional.ternary)
+
+; Parameters
+(required_parameter
+  pattern: (identifier) @variable.parameter)
+
+(optional_parameter
+  pattern: (identifier) @variable.parameter)
+
+(required_parameter
+  (rest_pattern
+    (identifier) @variable.parameter))
+
+; ({ a }) => null
+(required_parameter
+  (object_pattern
+    (shorthand_property_identifier_pattern) @variable.parameter))
+
+; ({ a = b }) => null
+(required_parameter
+  (object_pattern
+    (object_assignment_pattern
+      (shorthand_property_identifier_pattern) @variable.parameter)))
+
+; ({ a: b }) => null
+(required_parameter
+  (object_pattern
+    (pair_pattern
+      value: (identifier) @variable.parameter)))
+
+; ([ a ]) => null
+(required_parameter
+  (array_pattern
+    (identifier) @variable.parameter))
+
+; a => null
+(arrow_function
+  parameter: (identifier) @variable.parameter)
+
+; global declaration
+(ambient_declaration
+  "global" @module)
+
+; function signatures
+(ambient_declaration
+  (function_signature
+    name: (identifier) @function))
+
+; method signatures
+(method_signature
+  name: (_) @function.method)
+
+(abstract_method_signature
+  name: (property_identifier) @function.method)
+
+; property signatures
+(property_signature
+  name: (property_identifier) @function.method
+  type: (type_annotation
+    [
+      (union_type
+        (parenthesized_type
+          (function_type)))
+      (function_type)
+    ]))

bex/tag_preprocessor/queries/c.scm

@@ -0,0 +1,341 @@
+; Lower priority to prefer @variable.parameter when identifier appears in parameter_declaration.
+((identifier) @variable
+  (#set! priority 95))
+
+(preproc_def
+  (preproc_arg) @variable)
+
+[
+  "default"
+  "goto"
+  "asm"
+  "__asm__"
+] @keyword
+
+[
+  "enum"
+  "struct"
+  "union"
+  "typedef"
+] @keyword.type
+
+[
+  "sizeof"
+  "offsetof"
+] @keyword.operator
+
+(alignof_expression
+  .
+  _ @keyword.operator)
+
+"return" @keyword.return
+
+[
+  "while"
+  "for"
+  "do"
+  "continue"
+  "break"
+] @keyword.repeat
+
+[
+  "if"
+  "else"
+  "case"
+  "switch"
+] @keyword.conditional
+
+[
+  "#if"
+  "#ifdef"
+  "#ifndef"
+  "#else"
+  "#elif"
+  "#endif"
+  "#elifdef"
+  "#elifndef"
+  (preproc_directive)
+] @keyword.directive
+
+"#define" @keyword.directive.define
+
+"#include" @keyword.import
+
+[
+  ";"
+  ":"
+  ","
+  "."
+  "::"
+] @punctuation.delimiter
+
+"..." @punctuation.special
+
+[
+  "("
+  ")"
+  "["
+  "]"
+  "{"
+  "}"
+] @punctuation.bracket
+
+[
+  "="
+  "-"
+  "*"
+  "/"
+  "+"
+  "%"
+  "~"
+  "|"
+  "&"
+  "^"
+  "<<"
+  ">>"
+  "->"
+  "<"
+  "<="
+  ">="
+  ">"
+  "=="
+  "!="
+  "!"
+  "&&"
+  "||"
+  "-="
+  "+="
+  "*="
+  "/="
+  "%="
+  "|="
+  "&="
+  "^="
+  ">>="
+  "<<="
+  "--"
+  "++"
+] @operator
+
+; Make sure the comma operator is given a highlight group after the comma
+; punctuator so the operator is highlighted properly.
+(comma_expression
+  "," @operator)
+
+[
+  (true)
+  (false)
+] @boolean
+
+(conditional_expression
+  [
+    "?"
+    ":"
+  ] @keyword.conditional.ternary)
+
+(string_literal) @string
+
+(system_lib_string) @string
+
+(escape_sequence) @string.escape
+
+(null) @constant.builtin
+
+(number_literal) @number
+
+(char_literal) @character
+
+(preproc_defined) @function.macro
+
+((field_expression
+  (field_identifier) @property) @_parent
+  (#not-has-parent? @_parent function_declarator call_expression))
+
+(field_designator) @property
+
+((field_identifier) @property
+  (#has-ancestor? @property field_declaration)
+  (#not-has-ancestor? @property function_declarator))
+
+(statement_identifier) @label
+
+(declaration
+  type: (type_identifier) @_type
+  declarator: (identifier) @label
+  (#eq? @_type "__label__"))
+
+[
+  (type_identifier)
+  (type_descriptor)
+] @type
+
+(storage_class_specifier) @keyword.modifier
+
+[
+  (type_qualifier)
+  (gnu_asm_qualifier)
+  "__extension__"
+] @keyword.modifier
+
+(linkage_specification
+  "extern" @keyword.modifier)
+
+(type_definition
+  declarator: (type_identifier) @type.definition)
+
+(primitive_type) @type.builtin
+
+(sized_type_specifier
+  _ @type.builtin
+  type: _?)
+
+((identifier) @constant
+  (#lua-match? @constant "^[A-Z][A-Z0-9_]+$"))
+
+(preproc_def
+  (preproc_arg) @constant
+  (#lua-match? @constant "^[A-Z][A-Z0-9_]+$"))
+
+(enumerator
+  name: (identifier) @constant)
+
+(case_statement
+  value: (identifier) @constant)
+
+((identifier) @constant.builtin
+  ; format-ignore
+  (#any-of? @constant.builtin
+    "stderr" "stdin" "stdout"
+    "__FILE__" "__LINE__" "__DATE__" "__TIME__"
+    "__STDC__" "__STDC_VERSION__" "__STDC_HOSTED__"
+    "__cplusplus" "__OBJC__" "__ASSEMBLER__"
+    "__BASE_FILE__" "__FILE_NAME__" "__INCLUDE_LEVEL__"
+    "__TIMESTAMP__" "__clang__" "__clang_major__"
+    "__clang_minor__" "__clang_patchlevel__"
+    "__clang_version__" "__clang_literal_encoding__"
+    "__clang_wide_literal_encoding__"
+    "__FUNCTION__" "__func__" "__PRETTY_FUNCTION__"
+    "__VA_ARGS__" "__VA_OPT__"))
+
+(preproc_def
+  (preproc_arg) @constant.builtin
+  ; format-ignore
+  (#any-of? @constant.builtin
+    "stderr" "stdin" "stdout"
+    "__FILE__" "__LINE__" "__DATE__" "__TIME__"
+    "__STDC__" "__STDC_VERSION__" "__STDC_HOSTED__"
+    "__cplusplus" "__OBJC__" "__ASSEMBLER__"
+    "__BASE_FILE__" "__FILE_NAME__" "__INCLUDE_LEVEL__"
+    "__TIMESTAMP__" "__clang__" "__clang_major__"
+    "__clang_minor__" "__clang_patchlevel__"
+    "__clang_version__" "__clang_literal_encoding__"
+    "__clang_wide_literal_encoding__"
+    "__FUNCTION__" "__func__" "__PRETTY_FUNCTION__"
+    "__VA_ARGS__" "__VA_OPT__"))
+
+(attribute_specifier
+  (argument_list
+    (identifier) @variable.builtin))
+
+(attribute_specifier
+  (argument_list
+    (call_expression
+      function: (identifier) @variable.builtin)))
+
+((call_expression
+  function: (identifier) @function.builtin)
+  (#lua-match? @function.builtin "^__builtin_"))
+
+((call_expression
+  function: (identifier) @function.builtin)
+  (#has-ancestor? @function.builtin attribute_specifier))
+
+; Preproc def / undef
+(preproc_def
+  name: (_) @constant.macro)
+
+(preproc_call
+  directive: (preproc_directive) @_u
+  argument: (_) @constant.macro
+  (#eq? @_u "#undef"))
+
+(preproc_ifdef
+  name: (identifier) @constant.macro)
+
+(preproc_elifdef
+  name: (identifier) @constant.macro)
+
+(preproc_defined
+  (identifier) @constant.macro)
+
+(call_expression
+  function: (identifier) @function.call)
+
+(call_expression
+  function: (field_expression
+    field: (field_identifier) @function.call))
+
+(function_declarator
+  declarator: (identifier) @function)
+
+(function_declarator
+  declarator: (parenthesized_declarator
+    (pointer_declarator
+      declarator: (field_identifier) @function)))
+
+(preproc_function_def
+  name: (identifier) @function.macro)
+
+(comment) @comment @spell
+
+((comment) @comment.documentation
+  (#lua-match? @comment.documentation "^/[*][*][^*].*[*]/$"))
+
+; Parameters
+(parameter_declaration
+  declarator: (identifier) @variable.parameter)
+
+(parameter_declaration
+  declarator: (array_declarator) @variable.parameter)
+
+(parameter_declaration
+  declarator: (pointer_declarator) @variable.parameter)
+
+; K&R functions
+; To enable support for K&R functions,
+; add the following lines to your own query config and uncomment them.
+; They are commented out as they'll conflict with C++
+; Note that you'll need to have `; extends` at the top of your query file.
+;
+; (parameter_list (identifier) @variable.parameter)
+;
+; (function_definition
+;   declarator: _
+;   (declaration
+;     declarator: (identifier) @variable.parameter))
+;
+; (function_definition
+;   declarator: _
+;   (declaration
+;     declarator: (array_declarator) @variable.parameter))
+;
+; (function_definition
+;   declarator: _
+;   (declaration
+;     declarator: (pointer_declarator) @variable.parameter))
+(preproc_params
+  (identifier) @variable.parameter)
+
+[
+  "__attribute__"
+  "__declspec"
+  "__based"
+  "__cdecl"
+  "__clrcall"
+  "__stdcall"
+  "__fastcall"
+  "__thiscall"
+  "__vectorcall"
+  (ms_pointer_modifier)
+  (attribute_declaration)
+] @attribute

bex/tag_preprocessor/queries/cpp.scm

@@ -0,0 +1,273 @@
+; inherits: c
+
+((identifier) @variable.member
+  (#lua-match? @variable.member "^m_.*$"))
+
+(parameter_declaration
+  declarator: (reference_declarator) @variable.parameter)
+
+; function(Foo ...foo)
+(variadic_parameter_declaration
+  declarator: (variadic_declarator
+    (_) @variable.parameter))
+
+; int foo = 0
+(optional_parameter_declaration
+  declarator: (_) @variable.parameter)
+
+;(field_expression) @variable.parameter ;; How to highlight this?
+((field_expression
+  (field_identifier) @function.method) @_parent
+  (#has-parent? @_parent template_method function_declarator))
+
+(field_declaration
+  (field_identifier) @variable.member)
+
+(field_initializer
+  (field_identifier) @property)
+
+(function_declarator
+  declarator: (field_identifier) @function.method)
+
+(concept_definition
+  name: (identifier) @type.definition)
+
+(alias_declaration
+  name: (type_identifier) @type.definition)
+
+(auto) @type.builtin
+
+(namespace_identifier) @module
+
+((namespace_identifier) @type
+  (#lua-match? @type "^[%u]"))
+
+(case_statement
+  value: (qualified_identifier
+    (identifier) @constant))
+
+(using_declaration
+  .
+  "using"
+  .
+  "namespace"
+  .
+  [
+    (qualified_identifier)
+    (identifier)
+  ] @module)
+
+(destructor_name
+  (identifier) @function.method)
+
+; functions
+(function_declarator
+  (qualified_identifier
+    (identifier) @function))
+
+(function_declarator
+  (qualified_identifier
+    (qualified_identifier
+      (identifier) @function)))
+
+(function_declarator
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (identifier) @function))))
+
+((qualified_identifier
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (identifier) @function)))) @_parent
+  (#has-ancestor? @_parent function_declarator))
+
+(function_declarator
+  (template_function
+    (identifier) @function))
+
+(operator_name) @function
+
+"operator" @function
+
+"static_assert" @function.builtin
+
+(call_expression
+  (qualified_identifier
+    (identifier) @function.call))
+
+(call_expression
+  (qualified_identifier
+    (qualified_identifier
+      (identifier) @function.call)))
+
+(call_expression
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (identifier) @function.call))))
+
+((qualified_identifier
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (identifier) @function.call)))) @_parent
+  (#has-ancestor? @_parent call_expression))
+
+(call_expression
+  (template_function
+    (identifier) @function.call))
+
+(call_expression
+  (qualified_identifier
+    (template_function
+      (identifier) @function.call)))
+
+(call_expression
+  (qualified_identifier
+    (qualified_identifier
+      (template_function
+        (identifier) @function.call))))
+
+(call_expression
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (template_function
+          (identifier) @function.call)))))
+
+((qualified_identifier
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (template_function
+          (identifier) @function.call))))) @_parent
+  (#has-ancestor? @_parent call_expression))
+
+; methods
+(function_declarator
+  (template_method
+    (field_identifier) @function.method))
+
+(call_expression
+  (field_expression
+    (field_identifier) @function.method.call))
+
+(call_expression
+  (field_expression
+    (template_method
+      (field_identifier) @function.method.call)))
+
+; constructors
+((function_declarator
+  (qualified_identifier
+    (identifier) @constructor))
+  (#lua-match? @constructor "^%u"))
+
+((call_expression
+  function: (identifier) @constructor)
+  (#lua-match? @constructor "^%u"))
+
+((call_expression
+  function: (qualified_identifier
+    name: (identifier) @constructor))
+  (#lua-match? @constructor "^%u"))
+
+((call_expression
+  function: (field_expression
+    field: (field_identifier) @constructor))
+  (#lua-match? @constructor "^%u"))
+
+; constructing a type in an initializer list: Constructor ():  **SuperType (1)**
+((field_initializer
+  (field_identifier) @constructor
+  (argument_list))
+  (#lua-match? @constructor "^%u"))
+
+; Constants
+(this) @variable.builtin
+
+(null
+  "nullptr" @constant.builtin)
+
+(true) @boolean
+
+(false) @boolean
+
+; Literals
+(raw_string_literal) @string
+
+; Keywords
+[
+  "try"
+  "catch"
+  "noexcept"
+  "throw"
+] @keyword.exception
+
+[
+  "decltype"
+  "explicit"
+  "friend"
+  "override"
+  "using"
+  "requires"
+  "constexpr"
+] @keyword
+
+[
+  "class"
+  "namespace"
+  "template"
+  "typename"
+  "concept"
+] @keyword.type
+
+[
+  "co_await"
+  "co_yield"
+  "co_return"
+] @keyword.coroutine
+
+[
+  "public"
+  "private"
+  "protected"
+  "final"
+  "virtual"
+] @keyword.modifier
+
+[
+  "new"
+  "delete"
+  "xor"
+  "bitand"
+  "bitor"
+  "compl"
+  "not"
+  "xor_eq"
+  "and_eq"
+  "or_eq"
+  "not_eq"
+  "and"
+  "or"
+] @keyword.operator
+
+"<=>" @operator
+
+"::" @punctuation.delimiter
+
+(template_argument_list
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(template_parameter_list
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(literal_suffix) @operator

bex/tag_preprocessor/queries/ecma.scm

@@ -0,0 +1,392 @@
+; Types
+; Javascript
+; Variables
+;-----------
+(identifier) @variable
+
+; Properties
+;-----------
+(property_identifier) @variable.member
+
+(shorthand_property_identifier) @variable.member
+
+(private_property_identifier) @variable.member
+
+(object_pattern
+  (shorthand_property_identifier_pattern) @variable)
+
+(object_pattern
+  (object_assignment_pattern
+    (shorthand_property_identifier_pattern) @variable))
+
+; Special identifiers
+;--------------------
+((identifier) @type
+  (#lua-match? @type "^[A-Z]"))
+
+((identifier) @constant
+  (#lua-match? @constant "^_*[A-Z][A-Z%d_]*$"))
+
+((shorthand_property_identifier) @constant
+  (#lua-match? @constant "^_*[A-Z][A-Z%d_]*$"))
+
+((identifier) @variable.builtin
+  (#any-of? @variable.builtin "arguments" "module" "console" "window" "document"))
+
+((identifier) @type.builtin
+  (#any-of? @type.builtin
+    "Object" "Function" "Boolean" "Symbol" "Number" "Math" "Date" "String" "RegExp" "Map" "Set"
+    "WeakMap" "WeakSet" "Promise" "Array" "Int8Array" "Uint8Array" "Uint8ClampedArray" "Int16Array"
+    "Uint16Array" "Int32Array" "Uint32Array" "Float32Array" "Float64Array" "ArrayBuffer" "DataView"
+    "Error" "EvalError" "InternalError" "RangeError" "ReferenceError" "SyntaxError" "TypeError"
+    "URIError"))
+
+(statement_identifier) @label
+
+; Function and method definitions
+;--------------------------------
+(function_expression
+  name: (identifier) @function)
+
+(function_declaration
+  name: (identifier) @function)
+
+(generator_function
+  name: (identifier) @function)
+
+(generator_function_declaration
+  name: (identifier) @function)
+
+(method_definition
+  name: [
+    (property_identifier)
+    (private_property_identifier)
+  ] @function.method)
+
+(method_definition
+  name: (property_identifier) @constructor
+  (#eq? @constructor "constructor"))
+
+(pair
+  key: (property_identifier) @function.method
+  value: (function_expression))
+
+(pair
+  key: (property_identifier) @function.method
+  value: (arrow_function))
+
+(assignment_expression
+  left: (member_expression
+    property: (property_identifier) @function.method)
+  right: (arrow_function))
+
+(assignment_expression
+  left: (member_expression
+    property: (property_identifier) @function.method)
+  right: (function_expression))
+
+(variable_declarator
+  name: (identifier) @function
+  value: (arrow_function))
+
+(variable_declarator
+  name: (identifier) @function
+  value: (function_expression))
+
+(assignment_expression
+  left: (identifier) @function
+  right: (arrow_function))
+
+(assignment_expression
+  left: (identifier) @function
+  right: (function_expression))
+
+; Function and method calls
+;--------------------------
+(call_expression
+  function: (identifier) @function.call)
+
+(call_expression
+  function: (member_expression
+    property: [
+      (property_identifier)
+      (private_property_identifier)
+    ] @function.method.call))
+
+(call_expression
+  function: (await_expression
+    (identifier) @function.call))
+
+(call_expression
+  function: (await_expression
+    (member_expression
+      property: [
+        (property_identifier)
+        (private_property_identifier)
+      ] @function.method.call)))
+
+; Builtins
+;---------
+((identifier) @module.builtin
+  (#eq? @module.builtin "Intl"))
+
+((identifier) @function.builtin
+  (#any-of? @function.builtin
+    "eval" "isFinite" "isNaN" "parseFloat" "parseInt" "decodeURI" "decodeURIComponent" "encodeURI"
+    "encodeURIComponent" "require"))
+
+; Constructor
+;------------
+(new_expression
+  constructor: (identifier) @constructor)
+
+; Decorators
+;----------
+(decorator
+  "@" @attribute
+  (identifier) @attribute)
+
+(decorator
+  "@" @attribute
+  (call_expression
+    (identifier) @attribute))
+
+(decorator
+  "@" @attribute
+  (member_expression
+    (property_identifier) @attribute))
+
+(decorator
+  "@" @attribute
+  (call_expression
+    (member_expression
+      (property_identifier) @attribute)))
+
+; Literals
+;---------
+[
+  (this)
+  (super)
+] @variable.builtin
+
+((identifier) @variable.builtin
+  (#eq? @variable.builtin "self"))
+
+[
+  (true)
+  (false)
+] @boolean
+
+[
+  (null)
+  (undefined)
+] @constant.builtin
+
+[
+  (comment)
+  (html_comment)
+] @comment @spell
+
+((comment) @comment.documentation
+  (#lua-match? @comment.documentation "^/[*][*][^*].*[*]/$"))
+
+(hash_bang_line) @keyword.directive
+
+((string_fragment) @keyword.directive
+  (#eq? @keyword.directive "use strict"))
+
+(string) @string
+
+(template_string) @string
+
+(escape_sequence) @string.escape
+
+(regex_pattern) @string.regexp
+
+(regex_flags) @character.special
+
+(regex
+  "/" @punctuation.bracket) ; Regex delimiters
+
+(number) @number
+
+((identifier) @number
+  (#any-of? @number "NaN" "Infinity"))
+
+; Punctuation
+;------------
+[
+  ";"
+  "."
+  ","
+  ":"
+] @punctuation.delimiter
+
+[
+  "--"
+  "-"
+  "-="
+  "&&"
+  "+"
+  "++"
+  "+="
+  "&="
+  "/="
+  "**="
+  "<<="
+  "<"
+  "<="
+  "<<"
+  "="
+  "=="
+  "==="
+  "!="
+  "!=="
+  "=>"
+  ">"
+  ">="
+  ">>"
+  "||"
+  "%"
+  "%="
+  "*"
+  "**"
+  ">>>"
+  "&"
+  "|"
+  "^"
+  "??"
+  "*="
+  ">>="
+  ">>>="
+  "^="
+  "|="
+  "&&="
+  "||="
+  "??="
+  "..."
+] @operator
+
+(binary_expression
+  "/" @operator)
+
+(ternary_expression
+  [
+    "?"
+    ":"
+  ] @keyword.conditional.ternary)
+
+(unary_expression
+  [
+    "!"
+    "~"
+    "-"
+    "+"
+  ] @operator)
+
+(unary_expression
+  [
+    "delete"
+    "void"
+  ] @keyword.operator)
+
+[
+  "("
+  ")"
+  "["
+  "]"
+  "{"
+  "}"
+] @punctuation.bracket
+
+(template_substitution
+  [
+    "${"
+    "}"
+  ] @punctuation.special) @none
+
+; Imports
+;----------
+(namespace_import
+  "*" @character.special
+  (identifier) @module)
+
+(namespace_export
+  "*" @character.special
+  (identifier) @module)
+
+(export_statement
+  "*" @character.special)
+
+; Keywords
+;----------
+[
+  "if"
+  "else"
+  "switch"
+  "case"
+] @keyword.conditional
+
+[
+  "import"
+  "from"
+  "as"
+  "export"
+] @keyword.import
+
+[
+  "for"
+  "of"
+  "do"
+  "while"
+  "continue"
+] @keyword.repeat
+
+[
+  "break"
+  "const"
+  "debugger"
+  "extends"
+  "get"
+  "let"
+  "set"
+  "static"
+  "target"
+  "var"
+  "with"
+] @keyword
+
+"class" @keyword.type
+
+[
+  "async"
+  "await"
+] @keyword.coroutine
+
+[
+  "return"
+  "yield"
+] @keyword.return
+
+"function" @keyword.function
+
+[
+  "new"
+  "delete"
+  "in"
+  "instanceof"
+  "typeof"
+] @keyword.operator
+
+[
+  "throw"
+  "try"
+  "catch"
+  "finally"
+] @keyword.exception
+
+(export_statement
+  "default" @keyword)
+
+(switch_default
+  "default" @keyword.conditional)

bex/tag_preprocessor/queries/go.scm

@@ -0,0 +1,249 @@
+; Forked from tree-sitter-go
+; Copyright (c) 2014 Max Brunsfeld (The MIT License)
+;
+; Identifiers
+(type_identifier) @type
+
+(type_spec
+  name: (type_identifier) @type.definition)
+
+(field_identifier) @property
+
+(identifier) @variable
+
+(package_identifier) @module
+
+(parameter_declaration
+  (identifier) @variable.parameter)
+
+(variadic_parameter_declaration
+  (identifier) @variable.parameter)
+
+(label_name) @label
+
+(const_spec
+  name: (identifier) @constant)
+
+; Function calls
+(call_expression
+  function: (identifier) @function.call)
+
+(call_expression
+  function: (selector_expression
+    field: (field_identifier) @function.method.call))
+
+; Function definitions
+(function_declaration
+  name: (identifier) @function)
+
+(method_declaration
+  name: (field_identifier) @function.method)
+
+(method_elem
+  name: (field_identifier) @function.method)
+
+; Constructors
+((call_expression
+  (identifier) @constructor)
+  (#lua-match? @constructor "^[nN]ew.+$"))
+
+((call_expression
+  (identifier) @constructor)
+  (#lua-match? @constructor "^[mM]ake.+$"))
+
+; Operators
+[
+  "--"
+  "-"
+  "-="
+  ":="
+  "!"
+  "!="
+  "..."
+  "*"
+  "*="
+  "/"
+  "/="
+  "&"
+  "&&"
+  "&="
+  "&^"
+  "&^="
+  "%"
+  "%="
+  "^"
+  "^="
+  "+"
+  "++"
+  "+="
+  "<-"
+  "<"
+  "<<"
+  "<<="
+  "<="
+  "="
+  "=="
+  ">"
+  ">="
+  ">>"
+  ">>="
+  "|"
+  "|="
+  "||"
+  "~"
+] @operator
+
+; Keywords
+[
+  "break"
+  "const"
+  "continue"
+  "default"
+  "defer"
+  "goto"
+  "range"
+  "select"
+  "var"
+  "fallthrough"
+] @keyword
+
+[
+  "type"
+  "struct"
+  "interface"
+] @keyword.type
+
+"func" @keyword.function
+
+"return" @keyword.return
+
+"go" @keyword.coroutine
+
+"for" @keyword.repeat
+
+[
+  "import"
+  "package"
+] @keyword.import
+
+[
+  "else"
+  "case"
+  "switch"
+  "if"
+] @keyword.conditional
+
+; Builtin types
+[
+  "chan"
+  "map"
+] @type.builtin
+
+((type_identifier) @type.builtin
+  (#any-of? @type.builtin
+    "any" "bool" "byte" "comparable" "complex128" "complex64" "error" "float32" "float64" "int"
+    "int16" "int32" "int64" "int8" "rune" "string" "uint" "uint16" "uint32" "uint64" "uint8"
+    "uintptr"))
+
+; Builtin functions
+((identifier) @function.builtin
+  (#any-of? @function.builtin
+    "append" "cap" "clear" "close" "complex" "copy" "delete" "imag" "len" "make" "max" "min" "new"
+    "panic" "print" "println" "real" "recover"))
+
+; Delimiters
+[
+  "."
+  ","
+  ":"
+  ";"
+] @punctuation.delimiter
+
+[
+  "("
+  ")"
+  "{"
+  "}"
+  "["
+  "]"
+] @punctuation.bracket
+
+; Literals
+(interpreted_string_literal) @string
+
+(raw_string_literal) @string
+
+(rune_literal) @character
+
+(escape_sequence) @string.escape
+
+(int_literal) @number
+
+(float_literal) @number.float
+
+(imaginary_literal) @number
+
+[
+  (true)
+  (false)
+] @boolean
+
+[
+  (nil)
+  (iota)
+] @constant.builtin
+
+(keyed_element
+  .
+  (literal_element
+    (identifier) @variable.member))
+
+(field_declaration
+  name: (field_identifier) @variable.member)
+
+; Comments
+(comment) @comment @spell
+
+; Doc Comments
+(source_file
+  .
+  (comment)+ @comment.documentation)
+
+(source_file
+  (comment)+ @comment.documentation
+  .
+  (const_declaration))
+
+(source_file
+  (comment)+ @comment.documentation
+  .
+  (function_declaration))
+
+(source_file
+  (comment)+ @comment.documentation
+  .
+  (type_declaration))
+
+(source_file
+  (comment)+ @comment.documentation
+  .
+  (var_declaration))
+
+; Spell
+((interpreted_string_literal) @spell
+  (#not-has-parent? @spell import_spec))
+
+; Regex
+(call_expression
+  (selector_expression) @_function
+  (#any-of? @_function
+    "regexp.Match" "regexp.MatchReader" "regexp.MatchString" "regexp.Compile" "regexp.CompilePOSIX"
+    "regexp.MustCompile" "regexp.MustCompilePOSIX")
+  (argument_list
+    .
+    [
+      (raw_string_literal
+        (raw_string_literal_content) @string.regexp)
+      (interpreted_string_literal
+        (interpreted_string_literal_content) @string.regexp)
+    ]))

bex/tag_preprocessor/queries/java.scm

@@ -0,0 +1,333 @@
+; CREDITS @maxbrunsfeld (maxbrunsfeld@gmail.com)
+; Variables
+(identifier) @variable
+
+(underscore_pattern) @character.special
+
+; Methods
+(method_declaration
+  name: (identifier) @function.method)
+
+(method_invocation
+  name: (identifier) @function.method.call)
+
+(super) @function.builtin
+
+; Parameters
+(formal_parameter
+  name: (identifier) @variable.parameter)
+
+(spread_parameter
+  (variable_declarator
+    name: (identifier) @variable.parameter)) ; int... foo
+
+; Lambda parameter
+(inferred_parameters
+  (identifier) @variable.parameter) ; (x,y) -> ...
+
+(lambda_expression
+  parameters: (identifier) @variable.parameter) ; x -> ...
+
+; Operators
+[
+  "+"
+  ":"
+  "++"
+  "-"
+  "--"
+  "&"
+  "&&"
+  "|"
+  "||"
+  "!"
+  "!="
+  "=="
+  "*"
+  "/"
+  "%"
+  "<"
+  "<="
+  ">"
+  ">="
+  "="
+  "-="
+  "+="
+  "*="
+  "/="
+  "%="
+  "->"
+  "^"
+  "^="
+  "&="
+  "|="
+  "~"
+  ">>"
+  ">>>"
+  "<<"
+  "::"
+] @operator
+
+; Types
+(interface_declaration
+  name: (identifier) @type)
+
+(annotation_type_declaration
+  name: (identifier) @type)
+
+(class_declaration
+  name: (identifier) @type)
+
+(record_declaration
+  name: (identifier) @type)
+
+(enum_declaration
+  name: (identifier) @type)
+
+(constructor_declaration
+  name: (identifier) @type)
+
+(compact_constructor_declaration
+  name: (identifier) @type)
+
+(type_identifier) @type
+
+((type_identifier) @type.builtin
+  (#eq? @type.builtin "var"))
+
+((method_invocation
+  object: (identifier) @type)
+  (#lua-match? @type "^[A-Z]"))
+
+((method_reference
+  .
+  (identifier) @type)
+  (#lua-match? @type "^[A-Z]"))
+
+((field_access
+  object: (identifier) @type)
+  (#lua-match? @type "^[A-Z]"))
+
+(scoped_identifier
+  (identifier) @type
+  (#lua-match? @type "^[A-Z]"))
+
+; Fields
+(field_declaration
+  declarator: (variable_declarator
+    name: (identifier) @variable.member))
+
+(field_access
+  field: (identifier) @variable.member)
+
+[
+  (boolean_type)
+  (integral_type)
+  (floating_point_type)
+  (void_type)
+] @type.builtin
+
+; Variables
+((identifier) @constant
+  (#lua-match? @constant "^[A-Z_][A-Z%d_]+$"))
+
+(this) @variable.builtin
+
+; Annotations
+(annotation
+  "@" @attribute
+  name: (identifier) @attribute)
+
+(marker_annotation
+  "@" @attribute
+  name: (identifier) @attribute)
+
+; Literals
+(string_literal) @string
+
+(escape_sequence) @string.escape
+
+(character_literal) @character
+
+[
+  (hex_integer_literal)
+  (decimal_integer_literal)
+  (octal_integer_literal)
+  (binary_integer_literal)
+] @number
+
+[
+  (decimal_floating_point_literal)
+  (hex_floating_point_literal)
+] @number.float
+
+[
+  (true)
+  (false)
+] @boolean
+
+(null_literal) @constant.builtin
+
+; Keywords
+[
+  "assert"
+  "default"
+  "extends"
+  "implements"
+  "instanceof"
+  "@interface"
+  "permits"
+  "to"
+  "with"
+] @keyword
+
+[
+  "record"
+  "class"
+  "enum"
+  "interface"
+] @keyword.type
+
+(synchronized_statement
+  "synchronized" @keyword)
+
+[
+  "abstract"
+  "final"
+  "native"
+  "non-sealed"
+  "open"
+  "private"
+  "protected"
+  "public"
+  "sealed"
+  "static"
+  "strictfp"
+  "transitive"
+] @keyword.modifier
+
+(modifiers
+  "synchronized" @keyword.modifier)
+
+[
+  "transient"
+  "volatile"
+] @keyword.modifier
+
+[
+  "return"
+  "yield"
+] @keyword.return
+
+"new" @keyword.operator
+
+; Conditionals
+[
+  "if"
+  "else"
+  "switch"
+  "case"
+  "when"
+] @keyword.conditional
+
+(ternary_expression
+  [
+    "?"
+    ":"
+  ] @keyword.conditional.ternary)
+
+(wildcard
+  "?" @character.special)
+
+; Loops
+[
+  "for"
+  "while"
+  "do"
+  "continue"
+  "break"
+] @keyword.repeat
+
+; Includes
+[
+  "exports"
+  "import"
+  "module"
+  "opens"
+  "package"
+  "provides"
+  "requires"
+  "uses"
+] @keyword.import
+
+(import_declaration
+  (asterisk
+    "*" @character.special))
+
+; Punctuation
+[
+  ";"
+  "."
+  "..."
+  ","
+] @punctuation.delimiter
+
+[
+  "{"
+  "}"
+] @punctuation.bracket
+
+[
+  "["
+  "]"
+] @punctuation.bracket
+
+[
+  "("
+  ")"
+] @punctuation.bracket
+
+(type_arguments
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(type_parameters
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(string_interpolation
+  [
+    "\\{"
+    "}"
+  ] @punctuation.special)
+
+; Exceptions
+[
+  "throw"
+  "throws"
+  "finally"
+  "try"
+  "catch"
+] @keyword.exception
+
+; Labels
+(labeled_statement
+  (identifier) @label)
+
+; Comments
+[
+  (line_comment)
+  (block_comment)
+] @comment @spell
+
+((block_comment) @comment.documentation
+  (#lua-match? @comment.documentation "^/[*][*][^*].*[*]/$"))
+
+((line_comment) @comment.documentation
+  (#lua-match? @comment.documentation "^///[^/]"))
+
+((line_comment) @comment.documentation
+  (#lua-match? @comment.documentation "^///$"))

bex/tag_preprocessor/queries/javascript.scm

@@ -0,0 +1,56 @@
+; inherits: ecma,jsx
+
+; Parameters
+(formal_parameters
+  (identifier) @variable.parameter)
+
+(formal_parameters
+  (rest_pattern
+    (identifier) @variable.parameter))
+
+; ({ a }) => null
+(formal_parameters
+  (object_pattern
+    (shorthand_property_identifier_pattern) @variable.parameter))
+
+; ({ a = b }) => null
+(formal_parameters
+  (object_pattern
+    (object_assignment_pattern
+      (shorthand_property_identifier_pattern) @variable.parameter)))
+
+; ({ a: b }) => null
+(formal_parameters
+  (object_pattern
+    (pair_pattern
+      value: (identifier) @variable.parameter)))
+
+; ([ a ]) => null
+(formal_parameters
+  (array_pattern
+    (identifier) @variable.parameter))
+
+; ({ a } = { a }) => null
+(formal_parameters
+  (assignment_pattern
+    (object_pattern
+      (shorthand_property_identifier_pattern) @variable.parameter)))
+
+; ({ a = b } = { a }) => null
+(formal_parameters
+  (assignment_pattern
+    (object_pattern
+      (object_assignment_pattern
+        (shorthand_property_identifier_pattern) @variable.parameter))))
+
+; a => null
+(arrow_function
+  parameter: (identifier) @variable.parameter)
+
+; optional parameters
+(formal_parameters
+  (assignment_pattern
+    left: (identifier) @variable.parameter))
+
+; punctuation
+(optional_chain) @punctuation.delimiter

bex/tag_preprocessor/queries/jsx.scm

@@ -0,0 +1,153 @@
+(jsx_element
+  open_tag: (jsx_opening_element
+    [
+      "<"
+      ">"
+    ] @tag.delimiter))
+
+(jsx_element
+  close_tag: (jsx_closing_element
+    [
+      "</"
+      ">"
+    ] @tag.delimiter))
+
+(jsx_self_closing_element
+  [
+    "<"
+    "/>"
+  ] @tag.delimiter)
+
+(jsx_attribute
+  (property_identifier) @tag.attribute)
+
+(jsx_opening_element
+  name: (identifier) @tag.builtin)
+
+(jsx_closing_element
+  name: (identifier) @tag.builtin)
+
+(jsx_self_closing_element
+  name: (identifier) @tag.builtin)
+
+(jsx_opening_element
+  ((identifier) @tag
+    (#lua-match? @tag "^[A-Z]")))
+
+; Handle the dot operator effectively - <My.Component>
+(jsx_opening_element
+  (member_expression
+    (identifier) @tag.builtin
+    (property_identifier) @tag))
+
+(jsx_closing_element
+  ((identifier) @tag
+    (#lua-match? @tag "^[A-Z]")))
+
+; Handle the dot operator effectively - </My.Component>
+(jsx_closing_element
+  (member_expression
+    (identifier) @tag.builtin
+    (property_identifier) @tag))
+
+(jsx_self_closing_element
+  ((identifier) @tag
+    (#lua-match? @tag "^[A-Z]")))
+
+; Handle the dot operator effectively - <My.Component />
+(jsx_self_closing_element
+  (member_expression
+    (identifier) @tag.builtin
+    (property_identifier) @tag))
+
+(html_character_reference) @tag
+
+(jsx_text) @none @spell
+
+(html_character_reference) @character.special
+
+((jsx_element
+  (jsx_opening_element
+    name: (identifier) @_tag)
+  (jsx_text) @markup.heading)
+  (#eq? @_tag "title"))
+
+((jsx_element
+  (jsx_opening_element
+    name: (identifier) @_tag)
+  (jsx_text) @markup.heading.1)
+  (#eq? @_tag "h1"))
+
+((jsx_element
+  (jsx_opening_element
+    name: (identifier) @_tag)
+  (jsx_text) @markup.heading.2)
+  (#eq? @_tag "h2"))
+
+((jsx_element
+  (jsx_opening_element
+    name: (identifier) @_tag)
+  (jsx_text) @markup.heading.3)
+  (#eq? @_tag "h3"))
+
+((jsx_element
+  (jsx_opening_element
+    name: (identifier) @_tag)
+  (jsx_text) @markup.heading.4)
+  (#eq? @_tag "h4"))
+
+((jsx_element
+  (jsx_opening_element
+    name: (identifier) @_tag)
+  (jsx_text) @markup.heading.5)
+  (#eq? @_tag "h5"))
+
+((jsx_element
+  (jsx_opening_element
+    name: (identifier) @_tag)
+  (jsx_text) @markup.heading.6)
+  (#eq? @_tag "h6"))
+
+((jsx_element
+  (jsx_opening_element
+    name: (identifier) @_tag)
+  (jsx_text) @markup.strong)
+  (#any-of? @_tag "strong" "b"))
+
+((jsx_element
+  (jsx_opening_element
+    name: (identifier) @_tag)
+  (jsx_text) @markup.italic)
+  (#any-of? @_tag "em" "i"))
+
+((jsx_element
+  (jsx_opening_element
+    name: (identifier) @_tag)
+  (jsx_text) @markup.strikethrough)
+  (#any-of? @_tag "s" "del"))
+
+((jsx_element
+  (jsx_opening_element
+    name: (identifier) @_tag)
+  (jsx_text) @markup.underline)
+  (#eq? @_tag "u"))
+
+((jsx_element
+  (jsx_opening_element
+    name: (identifier) @_tag)
+  (jsx_text) @markup.raw)
+  (#any-of? @_tag "code" "kbd"))
+
+((jsx_element
+  (jsx_opening_element
+    name: (identifier) @_tag)
+  (jsx_text) @markup.link.label)
+  (#eq? @_tag "a"))
+
+((jsx_attribute
+  (property_identifier) @_attr
+  (string
+    (string_fragment) @string.special.url))
+  (#any-of? @_attr "href" "src"))
+
+

bex/tag_preprocessor/queries/kotlin.scm

@@ -0,0 +1,380 @@
+;; Based on the nvim-treesitter highlighting, which is under the Apache license.
+;; See https://github.com/nvim-treesitter/nvim-treesitter/blob/f8ab59861eed4a1c168505e3433462ed800f2bae/queries/kotlin/highlights.scm
+;;
+;; The only difference in this file is that queries using #lua-match?
+;; have been removed.
+
+;;; Identifiers
+
+(simple_identifier) @variable
+
+; `it` keyword inside lambdas
+; FIXME: This will highlight the keyword outside of lambdas since tree-sitter
+;        does not allow us to check for arbitrary nestation
+((simple_identifier) @variable.builtin
+(#eq? @variable.builtin "it"))
+
+; `field` keyword inside property getter/setter
+; FIXME: This will highlight the keyword outside of getters and setters
+;        since tree-sitter does not allow us to check for arbitrary nestation
+((simple_identifier) @variable.builtin
+(#eq? @variable.builtin "field"))
+
+; `this` this keyword inside classes
+(this_expression) @variable.builtin
+
+; `super` keyword inside classes
+(super_expression) @variable.builtin
+
+(class_parameter
+	(simple_identifier) @property)
+
+(class_body
+	(property_declaration
+		(variable_declaration
+			(simple_identifier) @property)))
+
+; id_1.id_2.id_3: `id_2` and `id_3` are assumed as object properties
+(_
+	(navigation_suffix
+		(simple_identifier) @property))
+
+(enum_entry
+	(simple_identifier) @constant)
+
+(type_identifier) @type
+
+((type_identifier) @type.builtin
+	(#any-of? @type.builtin
+		"Byte"
+		"Short"
+		"Int"
+		"Long"
+		"UByte"
+		"UShort"
+		"UInt"
+		"ULong"
+		"Float"
+		"Double"
+		"Boolean"
+		"Char"
+		"String"
+		"Array"
+		"ByteArray"
+		"ShortArray"
+		"IntArray"
+		"LongArray"
+		"UByteArray"
+		"UShortArray"
+		"UIntArray"
+		"ULongArray"
+		"FloatArray"
+		"DoubleArray"
+		"BooleanArray"
+		"CharArray"
+		"Map"
+		"Set"
+		"List"
+		"EmptyMap"
+		"EmptySet"
+		"EmptyList"
+		"MutableMap"
+		"MutableSet"
+		"MutableList"
+))
+
+(package_header
+	. (identifier)) @namespace
+
+(import_header
+	"import" @include)
+
+
+; TODO: Seperate labeled returns/breaks/continue/super/this
+;       Must be implemented in the parser first
+(label) @label
+
+;;; Function definitions
+
+(function_declaration
+	. (simple_identifier) @function)
+
+(getter
+	("get") @function.builtin)
+(setter
+	("set") @function.builtin)
+
+(primary_constructor) @constructor
+(secondary_constructor
+	("constructor") @constructor)
+
+(constructor_invocation
+	(user_type
+		(type_identifier) @constructor))
+
+(anonymous_initializer
+	("init") @constructor)
+
+(parameter
+	(simple_identifier) @parameter)
+
+(parameter_with_optional_type
+	(simple_identifier) @parameter)
+
+; lambda parameters
+(lambda_literal
+	(lambda_parameters
+		(variable_declaration
+			(simple_identifier) @parameter)))
+
+;;; Function calls
+
+; function()
+(call_expression
+	. (simple_identifier) @function)
+
+; object.function() or object.property.function()
+(call_expression
+	(navigation_expression
+		(navigation_suffix
+			(simple_identifier) @function) . ))
+
+(call_expression
+	. (simple_identifier) @function.builtin
+    (#any-of? @function.builtin
+		"arrayOf"
+		"arrayOfNulls"
+		"byteArrayOf"
+		"shortArrayOf"
+		"intArrayOf"
+		"longArrayOf"
+		"ubyteArrayOf"
+		"ushortArrayOf"
+		"uintArrayOf"
+		"ulongArrayOf"
+		"floatArrayOf"
+		"doubleArrayOf"
+		"booleanArrayOf"
+		"charArrayOf"
+		"emptyArray"
+		"mapOf"
+		"setOf"
+		"listOf"
+		"emptyMap"
+		"emptySet"
+		"emptyList"
+		"mutableMapOf"
+		"mutableSetOf"
+		"mutableListOf"
+		"print"
+		"println"
+		"error"
+		"TODO"
+		"run"
+		"runCatching"
+		"repeat"
+		"lazy"
+		"lazyOf"
+		"enumValues"
+		"enumValueOf"
+		"assert"
+		"check"
+		"checkNotNull"
+		"require"
+		"requireNotNull"
+		"with"
+		"suspend"
+		"synchronized"
+))
+
+;;; Literals
+
+[
+	(line_comment)
+	(multiline_comment)
+	(shebang_line)
+] @comment
+
+(real_literal) @float
+[
+	(integer_literal)
+	(long_literal)
+	(hex_literal)
+	(bin_literal)
+	(unsigned_literal)
+] @number
+
+[
+	(null_literal) ; should be highlighted the same as booleans
+	(boolean_literal)
+] @boolean
+
+(character_literal) @character
+
+(string_literal) @string
+
+(character_escape_seq) @string.escape
+
+; There are 3 ways to define a regex
+;    - "[abc]?".toRegex()
+(call_expression
+	(navigation_expression
+		((string_literal) @string.regex)
+		(navigation_suffix
+			((simple_identifier) @_function
+			(#eq? @_function "toRegex")))))
+
+;    - Regex("[abc]?")
+(call_expression
+	((simple_identifier) @_function
+	(#eq? @_function "Regex"))
+	(call_suffix
+		(value_arguments
+			(value_argument
+				(string_literal) @string.regex))))
+
+;   - Regex.fromLiteral("[abc]?")
+(call_expression
+	(navigation_expression
+		((simple_identifier) @_class
+		(#eq? @_class "Regex"))
+		(navigation_suffix
+			((simple_identifier) @_function
+			(#eq? @_function "fromLiteral"))))
+	(call_suffix
+		(value_arguments
+			(value_argument
+				(string_literal) @string.regex))))
+
+;;; Keywords
+
+(type_alias "typealias" @keyword)
+[
+	(class_modifier)
+	(member_modifier)
+	(function_modifier)
+	(property_modifier)
+	(platform_modifier)
+	(variance_modifier)
+	(parameter_modifier)
+	(visibility_modifier)
+	(reification_modifier)
+	(inheritance_modifier)
+]@keyword
+
+[
+	"val"
+	"var"
+	"enum"
+	"class"
+	"object"
+	"interface"
+;	"typeof" ; NOTE: It is reserved for future use
+] @keyword
+
+("fun") @keyword.function
+
+(jump_expression) @keyword.return
+
+[
+	"if"
+	"else"
+	"when"
+] @conditional
+
+[
+	"for"
+	"do"
+	"while"
+] @repeat
+
+[
+	"try"
+	"catch"
+	"throw"
+	"finally"
+] @exception
+
+
+(annotation
+	"@" @attribute (use_site_target)? @attribute)
+(annotation
+	(user_type
+		(type_identifier) @attribute))
+(annotation
+	(constructor_invocation
+		(user_type
+			(type_identifier) @attribute)))
+
+(file_annotation
+	"@" @attribute "file" @attribute ":" @attribute)
+(file_annotation
+	(user_type
+		(type_identifier) @attribute))
+(file_annotation
+	(constructor_invocation
+		(user_type
+			(type_identifier) @attribute)))
+
+;;; Operators & Punctuation
+
+[
+	"!"
+	"!="
+	"!=="
+	"="
+	"=="
+	"==="
+	">"
+	">="
+	"<"
+	"<="
+	"||"
+	"&&"
+	"+"
+	"++"
+	"+="
+	"-"
+	"--"
+	"-="
+	"*"
+	"*="
+	"/"
+	"/="
+	"%"
+	"%="
+	"?."
+	"?:"
+	"!!"
+	"is"
+	"!is"
+	"in"
+	"!in"
+	"as"
+	"as?"
+	".."
+	"->"
+] @operator
+
+[
+	"(" ")"
+	"[" "]"
+	"{" "}"
+] @punctuation.bracket
+
+[
+	"."
+	","
+	";"
+	":"
+	"::"
+] @punctuation.delimiter
+
+; NOTE: `interpolated_identifier`s can be highlighted in any way
+(string_literal
+	"$" @punctuation.special
+	(interpolated_identifier) @none)
+(string_literal
+	"${" @punctuation.special
+	(interpolated_expression) @none
+	"}" @punctuation.special)

bex/tag_preprocessor/queries/python.scm

@@ -0,0 +1,456 @@
+; From tree-sitter-python licensed under MIT License
+; Copyright (c) 2016 Max Brunsfeld
+; Variables
+(identifier) @variable
+
+; Reset highlighting in f-string interpolations
+(interpolation) @none @nospell
+
+; Identifier naming conventions
+((identifier) @type
+  (#lua-match? @type "^[A-Z].*[a-z]"))
+
+((identifier) @constant
+  (#lua-match? @constant "^[A-Z][A-Z_0-9]*$"))
+
+((identifier) @constant.builtin
+  (#lua-match? @constant.builtin "^__[a-zA-Z0-9_]*__$"))
+
+((identifier) @constant.builtin
+  (#any-of? @constant.builtin
+    ; https://docs.python.org/3/library/constants.html
+    "NotImplemented" "Ellipsis" "quit" "exit" "copyright" "credits" "license"))
+
+"_" @character.special ; match wildcard
+
+((assignment
+  left: (identifier) @type.definition
+  (type
+    (identifier) @_annotation))
+  (#eq? @_annotation "TypeAlias"))
+
+((assignment
+  left: (identifier) @type.definition
+  right: (call
+    function: (identifier) @_func))
+  (#any-of? @_func "TypeVar" "NewType"))
+
+; Function definitions
+(function_definition
+  name: (identifier) @function)
+
+(type
+  (identifier) @type)
+
+(type
+  (subscript
+    (identifier) @type)) ; type subscript: Tuple[int]
+
+((call
+  function: (identifier) @_isinstance
+  arguments: (argument_list
+    (_)
+    (identifier) @type))
+  (#eq? @_isinstance "isinstance"))
+
+; Literals
+(none) @constant.builtin
+
+[
+  (true)
+  (false)
+] @boolean
+
+(integer) @number
+
+(float) @number.float
+
+(comment) @comment @spell
+
+((module
+  .
+  (comment) @keyword.directive @nospell)
+  (#lua-match? @keyword.directive "^#!/"))
+
+(string) @string
+
+[
+  (escape_sequence)
+  (escape_interpolation)
+] @string.escape
+
+; doc-strings
+(expression_statement
+  (string
+    (string_content) @spell) @string.documentation)
+
+; Tokens
+[
+  "-"
+  "-="
+  ":="
+  "!="
+  "*"
+  "**"
+  "**="
+  "*="
+  "/"
+  "//"
+  "//="
+  "/="
+  "&"
+  "&="
+  "%"
+  "%="
+  "^"
+  "^="
+  "+"
+  "+="
+  "<"
+  "<<"
+  "<<="
+  "<="
+  "<>"
+  "="
+  "=="
+  ">"
+  ">="
+  ">>"
+  ">>="
+  "@"
+  "@="
+  "|"
+  "|="
+  "~"
+  "->"
+] @operator
+
+; Keywords
+[
+  "and"
+  "in"
+  "is"
+  "not"
+  "or"
+  "is not"
+  "not in"
+  "del"
+] @keyword.operator
+
+[
+  "def"
+  "lambda"
+] @keyword.function
+
+[
+  "assert"
+  "exec"
+  "global"
+  "nonlocal"
+  "pass"
+  "print"
+  "with"
+  "as"
+] @keyword
+
+[
+  "type"
+  "class"
+] @keyword.type
+
+[
+  "async"
+  "await"
+] @keyword.coroutine
+
+[
+  "return"
+  "yield"
+] @keyword.return
+
+(yield
+  "from" @keyword.return)
+
+(future_import_statement
+  "from" @keyword.import
+  "__future__" @module.builtin)
+
+(import_from_statement
+  "from" @keyword.import)
+
+"import" @keyword.import
+
+(aliased_import
+  "as" @keyword.import)
+
+(wildcard_import
+  "*" @character.special)
+
+(import_statement
+  name: (dotted_name
+    (identifier) @module))
+
+(import_statement
+  name: (aliased_import
+    name: (dotted_name
+      (identifier) @module)
+    alias: (identifier) @module))
+
+(import_from_statement
+  module_name: (dotted_name
+    (identifier) @module))
+
+(import_from_statement
+  module_name: (relative_import
+    (dotted_name
+      (identifier) @module)))
+
+[
+  "if"
+  "elif"
+  "else"
+  "match"
+  "case"
+] @keyword.conditional
+
+[
+  "for"
+  "while"
+  "break"
+  "continue"
+] @keyword.repeat
+
+[
+  "try"
+  "except"
+  "raise"
+  "finally"
+] @keyword.exception
+
+(raise_statement
+  "from" @keyword.exception)
+
+(try_statement
+  (else_clause
+    "else" @keyword.exception))
+
+[
+  "("
+  ")"
+  "["
+  "]"
+  "{"
+  "}"
+] @punctuation.bracket
+
+(interpolation
+  "{" @punctuation.special
+  "}" @punctuation.special)
+
+(format_expression
+  "{" @punctuation.special
+  "}" @punctuation.special)
+
+(line_continuation) @punctuation.special
+
+(type_conversion) @function.macro
+
+[
+  ","
+  "."
+  ":"
+  ";"
+  (ellipsis)
+] @punctuation.delimiter
+
+((identifier) @type.builtin
+  (#any-of? @type.builtin
+    ; https://docs.python.org/3/library/exceptions.html
+    "BaseException" "Exception" "ArithmeticError" "BufferError" "LookupError" "AssertionError"
+    "AttributeError" "EOFError" "FloatingPointError" "GeneratorExit" "ImportError"
+    "ModuleNotFoundError" "IndexError" "KeyError" "KeyboardInterrupt" "MemoryError" "NameError"
+    "NotImplementedError" "OSError" "OverflowError" "RecursionError" "ReferenceError" "RuntimeError"
+    "StopIteration" "StopAsyncIteration" "SyntaxError" "IndentationError" "TabError" "SystemError"
+    "SystemExit" "TypeError" "UnboundLocalError" "UnicodeError" "UnicodeEncodeError"
+    "UnicodeDecodeError" "UnicodeTranslateError" "ValueError" "ZeroDivisionError" "EnvironmentError"
+    "IOError" "WindowsError" "BlockingIOError" "ChildProcessError" "ConnectionError"
+    "BrokenPipeError" "ConnectionAbortedError" "ConnectionRefusedError" "ConnectionResetError"
+    "FileExistsError" "FileNotFoundError" "InterruptedError" "IsADirectoryError"
+    "NotADirectoryError" "PermissionError" "ProcessLookupError" "TimeoutError" "Warning"
+    "UserWarning" "DeprecationWarning" "PendingDeprecationWarning" "SyntaxWarning" "RuntimeWarning"
+    "FutureWarning" "ImportWarning" "UnicodeWarning" "BytesWarning" "ResourceWarning"
+    ; https://docs.python.org/3/library/stdtypes.html
+    "bool" "int" "float" "complex" "list" "tuple" "range" "str" "bytes" "bytearray" "memoryview"
+    "set" "frozenset" "dict" "type" "object"))
+
+; Normal parameters
+(parameters
+  (identifier) @variable.parameter)
+
+; Lambda parameters
+(lambda_parameters
+  (identifier) @variable.parameter)
+
+(lambda_parameters
+  (tuple_pattern
+    (identifier) @variable.parameter))
+
+; Default parameters
+(keyword_argument
+  name: (identifier) @variable.parameter)
+
+; Naming parameters on call-site
+(default_parameter
+  name: (identifier) @variable.parameter)
+
+(typed_parameter
+  (identifier) @variable.parameter)
+
+(typed_default_parameter
+  name: (identifier) @variable.parameter)
+
+; Variadic parameters *args, **kwargs
+(parameters
+  (list_splat_pattern ; *args
+    (identifier) @variable.parameter))
+
+(parameters
+  (dictionary_splat_pattern ; **kwargs
+    (identifier) @variable.parameter))
+
+; Typed variadic parameters
+(parameters
+  (typed_parameter
+    (list_splat_pattern ; *args: type
+      (identifier) @variable.parameter)))
+
+(parameters
+  (typed_parameter
+    (dictionary_splat_pattern ; *kwargs: type
+      (identifier) @variable.parameter)))
+
+; Lambda parameters
+(lambda_parameters
+  (list_splat_pattern
+    (identifier) @variable.parameter))
+
+(lambda_parameters
+  (dictionary_splat_pattern
+    (identifier) @variable.parameter))
+
+((identifier) @variable.builtin
+  (#eq? @variable.builtin "self"))
+
+((identifier) @variable.builtin
+  (#eq? @variable.builtin "cls"))
+
+; After @type.builtin bacause builtins (such as `type`) are valid as attribute name
+((attribute
+  attribute: (identifier) @variable.member)
+  (#lua-match? @variable.member "^[%l_].*$"))
+
+; Class definitions
+(class_definition
+  name: (identifier) @type)
+
+(class_definition
+  body: (block
+    (function_definition
+      name: (identifier) @function.method)))
+
+(class_definition
+  superclasses: (argument_list
+    (identifier) @type))
+
+((class_definition
+  body: (block
+    (expression_statement
+      (assignment
+        left: (identifier) @variable.member))))
+  (#lua-match? @variable.member "^[%l_].*$"))
+
+((class_definition
+  body: (block
+    (expression_statement
+      (assignment
+        left: (_
+          (identifier) @variable.member)))))
+  (#lua-match? @variable.member "^[%l_].*$"))
+
+((class_definition
+  (block
+    (function_definition
+      name: (identifier) @constructor)))
+  (#any-of? @constructor "__new__" "__init__"))
+
+; Function calls
+(call
+  function: (identifier) @function.call)
+
+(call
+  function: (attribute
+    attribute: (identifier) @function.method.call))
+
+((call
+  function: (identifier) @constructor)
+  (#lua-match? @constructor "^%u"))
+
+((call
+  function: (attribute
+    attribute: (identifier) @constructor))
+  (#lua-match? @constructor "^%u"))
+
+; Builtin functions
+((call
+  function: (identifier) @function.builtin)
+  (#any-of? @function.builtin
+    "abs" "all" "any" "ascii" "bin" "bool" "breakpoint" "bytearray" "bytes" "callable" "chr"
+    "classmethod" "compile" "complex" "delattr" "dict" "dir" "divmod" "enumerate" "eval" "exec"
+    "filter" "float" "format" "frozenset" "getattr" "globals" "hasattr" "hash" "help" "hex" "id"
+    "input" "int" "isinstance" "issubclass" "iter" "len" "list" "locals" "map" "max" "memoryview"
+    "min" "next" "object" "oct" "open" "ord" "pow" "print" "property" "range" "repr" "reversed"
+    "round" "set" "setattr" "slice" "sorted" "staticmethod" "str" "sum" "super" "tuple" "type"
+    "vars" "zip" "__import__"))
+
+; Regex from the `re` module
+(call
+  function: (attribute
+    object: (identifier) @_re)
+  arguments: (argument_list
+    (string
+      (string_content) @string.regexp))
+  (#eq? @_re "re"))
+
+(call
+  function: (attribute
+    object: (identifier) @_re)
+  arguments: (argument_list
+    (concatenated_string
+      (string
+        (string_content) @string.regexp)))
+  (#eq? @_re "re"))
+
+; Decorators
+((decorator
+  "@" @attribute)
+  (#set! priority 101))
+
+(decorator
+  (identifier) @attribute)
+
+(decorator
+  (attribute
+    attribute: (identifier) @attribute))
+
+(decorator
+  (call
+    (identifier) @attribute))
+
+(decorator
+  (call
+    (attribute
+      attribute: (identifier) @attribute)))
+
+((decorator
+  (identifier) @attribute.builtin)
+  (#any-of? @attribute.builtin "classmethod" "property" "staticmethod"))

bex/tag_preprocessor/queries/ruby.scm

@@ -0,0 +1,319 @@
+; Variables
+[
+  (identifier)
+  (global_variable)
+] @variable
+
+; Keywords
+[
+  "alias"
+  "begin"
+  "do"
+  "end"
+  "ensure"
+  "module"
+  "rescue"
+  "then"
+] @keyword
+
+"class" @keyword.type
+
+[
+  "return"
+  "yield"
+] @keyword.return
+
+[
+  "and"
+  "or"
+  "in"
+  "not"
+] @keyword.operator
+
+[
+  "def"
+  "undef"
+] @keyword.function
+
+(method
+  "end" @keyword.function)
+
+[
+  "case"
+  "else"
+  "elsif"
+  "if"
+  "unless"
+  "when"
+  "then"
+] @keyword.conditional
+
+(in_clause
+  "in" @keyword.conditional)
+
+(if
+  "end" @keyword.conditional)
+
+[
+  "for"
+  "until"
+  "while"
+  "break"
+  "redo"
+  "retry"
+  "next"
+] @keyword.repeat
+
+(in
+  "in" @keyword.repeat)
+
+(constant) @constant
+
+((identifier) @keyword.modifier
+  (#any-of? @keyword.modifier "private" "protected" "public"))
+
+[
+  "rescue"
+  "ensure"
+] @keyword.exception
+
+; Function calls
+"defined?" @function
+
+(call
+  receiver: (constant)? @type
+  method: [
+    (identifier)
+    (constant)
+  ] @function.call)
+
+(program
+  (call
+    (identifier) @keyword.import)
+  (#any-of? @keyword.import "require" "require_relative" "load"))
+
+; Function definitions
+(alias
+  (identifier) @function)
+
+(setter
+  (identifier) @function)
+
+(method
+  name: [
+    (identifier) @function
+    (constant) @type
+  ])
+
+(singleton_method
+  name: [
+    (identifier) @function
+    (constant) @type
+  ])
+
+(class
+  name: (constant) @type)
+
+(module
+  name: (constant) @type)
+
+(superclass
+  (constant) @type)
+
+; Identifiers
+[
+  (class_variable)
+  (instance_variable)
+] @variable.member
+
+((identifier) @constant.builtin
+  (#any-of? @constant.builtin
+    "__callee__" "__dir__" "__id__" "__method__" "__send__" "__ENCODING__" "__FILE__" "__LINE__"))
+
+((identifier) @function.builtin
+  (#any-of? @function.builtin "attr_reader" "attr_writer" "attr_accessor" "module_function"))
+
+((call
+  !receiver
+  method: (identifier) @function.builtin)
+  (#any-of? @function.builtin "include" "extend" "prepend" "refine" "using"))
+
+((identifier) @keyword.exception
+  (#any-of? @keyword.exception "raise" "fail" "catch" "throw"))
+
+((constant) @type
+  (#not-lua-match? @type "^[A-Z0-9_]+$"))
+
+[
+  (self)
+  (super)
+] @variable.builtin
+
+(method_parameters
+  (identifier) @variable.parameter)
+
+(lambda_parameters
+  (identifier) @variable.parameter)
+
+(block_parameters
+  (identifier) @variable.parameter)
+
+(splat_parameter
+  (identifier) @variable.parameter)
+
+(hash_splat_parameter
+  (identifier) @variable.parameter)
+
+(optional_parameter
+  (identifier) @variable.parameter)
+
+(destructured_parameter
+  (identifier) @variable.parameter)
+
+(block_parameter
+  (identifier) @variable.parameter)
+
+(keyword_parameter
+  (identifier) @variable.parameter)
+
+; Literals
+[
+  (string_content)
+  (heredoc_content)
+  "\""
+  "`"
+] @string
+
+[
+  (heredoc_beginning)
+  (heredoc_end)
+] @label
+
+[
+  (bare_symbol)
+  (simple_symbol)
+  (hash_key_symbol)
+] @string.special.symbol
+
+(delimited_symbol
+  ":\"" @string.special.symbol
+  (string_content) @string.special.symbol
+  "\"" @string.special.symbol)
+
+(regex
+  (string_content) @string.regexp)
+
+(escape_sequence) @string.escape
+
+(integer) @number
+
+(float) @number.float
+
+[
+  (true)
+  (false)
+] @boolean
+
+(nil) @constant.builtin
+
+(comment) @comment @spell
+
+((program
+  .
+  (comment) @keyword.directive @nospell)
+  (#lua-match? @keyword.directive "^#!/"))
+
+(program
+  (comment)+ @comment.documentation
+  (class))
+
+(module
+  (comment)+ @comment.documentation
+  (body_statement
+    (class)))
+
+(class
+  (comment)+ @comment.documentation
+  (body_statement
+    (method)))
+
+(body_statement
+  (comment)+ @comment.documentation
+  (method))
+
+; Operators
+[
+  "!"
+  "="
+  "=="
+  "==="
+  "<=>"
+  "=>"
+  "->"
+  ">>"
+  "<<"
+  ">"
+  "<"
+  ">="
+  "<="
+  "**"
+  "*"
+  "/"
+  "%"
+  "+"
+  "-"
+  "&"
+  "|"
+  "^"
+  "&&"
+  "||"
+  "||="
+  "&&="
+  "!="
+  "%="
+  "+="
+  "-="
+  "*="
+  "/="
+  "=~"
+  "!~"
+  "?"
+  ":"
+  ".."
+  "..."
+] @operator
+
+[
+  ","
+  ";"
+  "."
+  "&."
+  "::"
+] @punctuation.delimiter
+
+(regex
+  "/" @punctuation.bracket)
+
+(pair
+  ":" @punctuation.delimiter)
+
+(keyword_pattern
+  ":" @punctuation.delimiter)
+
+[
+  "("
+  ")"
+  "["
+  "]"
+  "{"
+  "}"
+  "%w("
+  "%i("
+] @punctuation.bracket
+
+(block_parameters
+  "|" @punctuation.bracket)
+
+(interpolation
+  "#{" @punctuation.special
+  "}" @punctuation.special)

bex/tag_preprocessor/queries/rust.scm

@@ -0,0 +1,534 @@
+; Forked from https://github.com/tree-sitter/tree-sitter-rust
+; Copyright (c) 2017 Maxim Sokolov
+; Licensed under the MIT license.
+; Identifier conventions
+(shebang) @keyword.directive
+
+(identifier) @variable
+
+((identifier) @type
+  (#lua-match? @type "^[A-Z]"))
+
+(const_item
+  name: (identifier) @constant)
+
+; Assume all-caps names are constants
+((identifier) @constant
+  (#lua-match? @constant "^[A-Z][A-Z%d_]*$"))
+
+; Other identifiers
+(type_identifier) @type
+
+(primitive_type) @type.builtin
+
+(field_identifier) @variable.member
+
+(shorthand_field_identifier) @variable.member
+
+(shorthand_field_initializer
+  (identifier) @variable.member)
+
+(mod_item
+  name: (identifier) @module)
+
+(self) @variable.builtin
+
+"_" @character.special
+
+(label
+  [
+    "'"
+    (identifier)
+  ] @label)
+
+; Function definitions
+(function_item
+  (identifier) @function)
+
+(function_signature_item
+  (identifier) @function)
+
+(parameter
+  [
+    (identifier)
+    "_"
+  ] @variable.parameter)
+
+(parameter
+  (ref_pattern
+    [
+      (mut_pattern
+        (identifier) @variable.parameter)
+      (identifier) @variable.parameter
+    ]))
+
+(closure_parameters
+  (_) @variable.parameter)
+
+; Function calls
+(call_expression
+  function: (identifier) @function.call)
+
+(call_expression
+  function: (scoped_identifier
+    (identifier) @function.call .))
+
+(call_expression
+  function: (field_expression
+    field: (field_identifier) @function.call))
+
+(generic_function
+  function: (identifier) @function.call)
+
+(generic_function
+  function: (scoped_identifier
+    name: (identifier) @function.call))
+
+(generic_function
+  function: (field_expression
+    field: (field_identifier) @function.call))
+
+; Assume other uppercase names are enum constructors
+((field_identifier) @constant
+  (#lua-match? @constant "^[A-Z]"))
+
+(enum_variant
+  name: (identifier) @constant)
+
+; Assume that uppercase names in paths are types
+(scoped_identifier
+  path: (identifier) @module)
+
+(scoped_identifier
+  (scoped_identifier
+    name: (identifier) @module))
+
+(scoped_type_identifier
+  path: (identifier) @module)
+
+(scoped_type_identifier
+  path: (identifier) @type
+  (#lua-match? @type "^[A-Z]"))
+
+(scoped_type_identifier
+  (scoped_identifier
+    name: (identifier) @module))
+
+((scoped_identifier
+  path: (identifier) @type)
+  (#lua-match? @type "^[A-Z]"))
+
+((scoped_identifier
+  name: (identifier) @type)
+  (#lua-match? @type "^[A-Z]"))
+
+((scoped_identifier
+  name: (identifier) @constant)
+  (#lua-match? @constant "^[A-Z][A-Z%d_]*$"))
+
+((scoped_identifier
+  path: (identifier) @type
+  name: (identifier) @constant)
+  (#lua-match? @type "^[A-Z]")
+  (#lua-match? @constant "^[A-Z]"))
+
+((scoped_type_identifier
+  path: (identifier) @type
+  name: (type_identifier) @constant)
+  (#lua-match? @type "^[A-Z]")
+  (#lua-match? @constant "^[A-Z]"))
+
+[
+  (crate)
+  (super)
+] @module
+
+(scoped_use_list
+  path: (identifier) @module)
+
+(scoped_use_list
+  path: (scoped_identifier
+    (identifier) @module))
+
+(use_list
+  (scoped_identifier
+    (identifier) @module
+    .
+    (_)))
+
+(use_list
+  (identifier) @type
+  (#lua-match? @type "^[A-Z]"))
+
+(use_as_clause
+  alias: (identifier) @type
+  (#lua-match? @type "^[A-Z]"))
+
+; Correct enum constructors
+(call_expression
+  function: (scoped_identifier
+    "::"
+    name: (identifier) @constant)
+  (#lua-match? @constant "^[A-Z]"))
+
+; Assume uppercase names in a match arm are constants.
+((match_arm
+  pattern: (match_pattern
+    (identifier) @constant))
+  (#lua-match? @constant "^[A-Z]"))
+
+((match_arm
+  pattern: (match_pattern
+    (scoped_identifier
+      name: (identifier) @constant)))
+  (#lua-match? @constant "^[A-Z]"))
+
+((identifier) @constant.builtin
+  (#any-of? @constant.builtin "Some" "None" "Ok" "Err"))
+
+; Macro definitions
+"$" @function.macro
+
+(metavariable) @function.macro
+
+(macro_definition
+  "macro_rules!" @function.macro)
+
+; Attribute macros
+(attribute_item
+  (attribute
+    (identifier) @function.macro))
+
+(inner_attribute_item
+  (attribute
+    (identifier) @function.macro))
+
+(attribute
+  (scoped_identifier
+    (identifier) @function.macro .))
+
+; Derive macros (assume all arguments are types)
+; (attribute
+;   (identifier) @_name
+;   arguments: (attribute (attribute (identifier) @type))
+;   (#eq? @_name "derive"))
+; Function-like macros
+(macro_invocation
+  macro: (identifier) @function.macro)
+
+(macro_invocation
+  macro: (scoped_identifier
+    (identifier) @function.macro .))
+
+; Literals
+(boolean_literal) @boolean
+
+(integer_literal) @number
+
+(float_literal) @number.float
+
+[
+  (raw_string_literal)
+  (string_literal)
+] @string
+
+(escape_sequence) @string.escape
+
+(char_literal) @character
+
+; Keywords
+[
+  "use"
+  "mod"
+] @keyword.import
+
+(use_as_clause
+  "as" @keyword.import)
+
+[
+  "default"
+  "impl"
+  "let"
+  "move"
+  "unsafe"
+  "where"
+] @keyword
+
+[
+  "enum"
+  "struct"
+  "union"
+  "trait"
+  "type"
+] @keyword.type
+
+[
+  "async"
+  "await"
+  "gen"
+] @keyword.coroutine
+
+"try" @keyword.exception
+
+[
+  "ref"
+  "pub"
+  "raw"
+  (mutable_specifier)
+  "const"
+  "static"
+  "dyn"
+  "extern"
+] @keyword.modifier
+
+(lifetime
+  "'" @keyword.modifier)
+
+(lifetime
+  (identifier) @attribute)
+
+(lifetime
+  (identifier) @attribute.builtin
+  (#any-of? @attribute.builtin "static" "_"))
+
+"fn" @keyword.function
+
+[
+  "return"
+  "yield"
+] @keyword.return
+
+(type_cast_expression
+  "as" @keyword.operator)
+
+(qualified_type
+  "as" @keyword.operator)
+
+(use_list
+  (self) @module)
+
+(scoped_use_list
+  (self) @module)
+
+(scoped_identifier
+  [
+    (crate)
+    (super)
+    (self)
+  ] @module)
+
+(visibility_modifier
+  [
+    (crate)
+    (super)
+    (self)
+  ] @module)
+
+[
+  "if"
+  "else"
+  "match"
+] @keyword.conditional
+
+[
+  "break"
+  "continue"
+  "in"
+  "loop"
+  "while"
+] @keyword.repeat
+
+"for" @keyword
+
+(for_expression
+  "for" @keyword.repeat)
+
+; Operators
+[
+  "!"
+  "!="
+  "%"
+  "%="
+  "&"
+  "&&"
+  "&="
+  "*"
+  "*="
+  "+"
+  "+="
+  "-"
+  "-="
+  ".."
+  "..="
+  "..."
+  "/"
+  "/="
+  "<"
+  "<<"
+  "<<="
+  "<="
+  "="
+  "=="
+  ">"
+  ">="
+  ">>"
+  ">>="
+  "?"
+  "@"
+  "^"
+  "^="
+  "|"
+  "|="
+  "||"
+] @operator
+
+(use_wildcard
+  "*" @character.special)
+
+(remaining_field_pattern
+  ".." @character.special)
+
+(range_pattern
+  [
+    ".."
+    "..="
+    "..."
+  ] @character.special)
+
+; Punctuation
+[
+  "("
+  ")"
+  "["
+  "]"
+  "{"
+  "}"
+] @punctuation.bracket
+
+(closure_parameters
+  "|" @punctuation.bracket)
+
+(type_arguments
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(type_parameters
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(bracketed_type
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(for_lifetimes
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+[
+  ","
+  "."
+  ":"
+  "::"
+  ";"
+  "->"
+  "=>"
+] @punctuation.delimiter
+
+(attribute_item
+  "#" @punctuation.special)
+
+(inner_attribute_item
+  [
+    "!"
+    "#"
+  ] @punctuation.special)
+
+(macro_invocation
+  "!" @function.macro)
+
+(never_type
+  "!" @type.builtin)
+
+(macro_invocation
+  macro: (identifier) @_identifier @keyword.exception
+  "!" @keyword.exception
+  (#eq? @_identifier "panic"))
+
+(macro_invocation
+  macro: (identifier) @_identifier @keyword.exception
+  "!" @keyword.exception
+  (#contains? @_identifier "assert"))
+
+(macro_invocation
+  macro: (identifier) @_identifier @keyword.debug
+  "!" @keyword.debug
+  (#eq? @_identifier "dbg"))
+
+; Comments
+[
+  (line_comment)
+  (block_comment)
+] @comment @spell
+
+[
+  (outer_doc_comment_marker)
+  (inner_doc_comment_marker)
+] @comment.documentation
+
+(line_comment
+  (doc_comment)) @comment.documentation
+
+(block_comment
+  (doc_comment)) @comment.documentation
+
+(call_expression
+  function: (scoped_identifier
+    path: (identifier) @_regex
+    (#any-of? @_regex "Regex" "ByteRegexBuilder")
+    name: (identifier) @_new
+    (#eq? @_new "new"))
+  arguments: (arguments
+    (raw_string_literal
+      (string_content) @string.regexp)))
+
+(call_expression
+  function: (scoped_identifier
+    path: (scoped_identifier
+      (identifier) @_regex
+      (#any-of? @_regex "Regex" "ByteRegexBuilder") .)
+    name: (identifier) @_new
+    (#eq? @_new "new"))
+  arguments: (arguments
+    (raw_string_literal
+      (string_content) @string.regexp)))
+
+(call_expression
+  function: (scoped_identifier
+    path: (identifier) @_regex
+    (#any-of? @_regex "RegexSet" "RegexSetBuilder")
+    name: (identifier) @_new
+    (#eq? @_new "new"))
+  arguments: (arguments
+    (array_expression
+      (raw_string_literal
+        (string_content) @string.regexp))))
+
+(call_expression
+  function: (scoped_identifier
+    path: (scoped_identifier
+      (identifier) @_regex
+      (#any-of? @_regex "RegexSet" "RegexSetBuilder") .)
+    name: (identifier) @_new
+    (#eq? @_new "new"))
+  arguments: (arguments
+    (array_expression
+      (raw_string_literal
+        (string_content) @string.regexp))))

bex/tag_preprocessor/queries/typescript.scm

@@ -0,0 +1,208 @@
+; inherits: ecma
+
+"require" @keyword.import
+
+(import_require_clause
+  source: (string) @string.special.url)
+
+[
+  "declare"
+  "implements"
+  "type"
+  "override"
+  "module"
+  "asserts"
+  "infer"
+  "is"
+  "using"
+] @keyword
+
+[
+  "namespace"
+  "interface"
+  "enum"
+] @keyword.type
+
+[
+  "keyof"
+  "satisfies"
+] @keyword.operator
+
+(as_expression
+  "as" @keyword.operator)
+
+(mapped_type_clause
+  "as" @keyword.operator)
+
+[
+  "abstract"
+  "private"
+  "protected"
+  "public"
+  "readonly"
+] @keyword.modifier
+
+; types
+(type_identifier) @type
+
+(predefined_type) @type.builtin
+
+(import_statement
+  "type"
+  (import_clause
+    (named_imports
+      (import_specifier
+        name: (identifier) @type))))
+
+(template_literal_type) @string
+
+(non_null_expression
+  "!" @operator)
+
+; punctuation
+(type_arguments
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(type_parameters
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(object_type
+  [
+    "{|"
+    "|}"
+  ] @punctuation.bracket)
+
+(union_type
+  "|" @punctuation.delimiter)
+
+(intersection_type
+  "&" @punctuation.delimiter)
+
+(type_annotation
+  ":" @punctuation.delimiter)
+
+(type_predicate_annotation
+  ":" @punctuation.delimiter)
+
+(index_signature
+  ":" @punctuation.delimiter)
+
+(omitting_type_annotation
+  "-?:" @punctuation.delimiter)
+
+(adding_type_annotation
+  "+?:" @punctuation.delimiter)
+
+(opting_type_annotation
+  "?:" @punctuation.delimiter)
+
+"?." @punctuation.delimiter
+
+(abstract_method_signature
+  "?" @punctuation.special)
+
+(method_signature
+  "?" @punctuation.special)
+
+(method_definition
+  "?" @punctuation.special)
+
+(property_signature
+  "?" @punctuation.special)
+
+(optional_parameter
+  "?" @punctuation.special)
+
+(optional_type
+  "?" @punctuation.special)
+
+(public_field_definition
+  [
+    "?"
+    "!"
+  ] @punctuation.special)
+
+(flow_maybe_type
+  "?" @punctuation.special)
+
+(template_type
+  [
+    "${"
+    "}"
+  ] @punctuation.special)
+
+(conditional_type
+  [
+    "?"
+    ":"
+  ] @keyword.conditional.ternary)
+
+; Parameters
+(required_parameter
+  pattern: (identifier) @variable.parameter)
+
+(optional_parameter
+  pattern: (identifier) @variable.parameter)
+
+(required_parameter
+  (rest_pattern
+    (identifier) @variable.parameter))
+
+; ({ a }) => null
+(required_parameter
+  (object_pattern
+    (shorthand_property_identifier_pattern) @variable.parameter))
+
+; ({ a = b }) => null
+(required_parameter
+  (object_pattern
+    (object_assignment_pattern
+      (shorthand_property_identifier_pattern) @variable.parameter)))
+
+; ({ a: b }) => null
+(required_parameter
+  (object_pattern
+    (pair_pattern
+      value: (identifier) @variable.parameter)))
+
+; ([ a ]) => null
+(required_parameter
+  (array_pattern
+    (identifier) @variable.parameter))
+
+; a => null
+(arrow_function
+  parameter: (identifier) @variable.parameter)
+
+; global declaration
+(ambient_declaration
+  "global" @module)
+
+; function signatures
+(ambient_declaration
+  (function_signature
+    name: (identifier) @function))
+
+; method signatures
+(method_signature
+  name: (_) @function.method)
+
+(abstract_method_signature
+  name: (property_identifier) @function.method)
+
+; property signatures
+(property_signature
+  name: (property_identifier) @function.method
+  type: (type_annotation
+    [
+      (union_type
+        (parenthesized_type
+          (function_type)))
+      (function_type)
+    ]))

docs/adr/0001-use-nvim-treesitter-highlights-scm.md

@@ -0,0 +1,41 @@
+# 1. Use nvim-treesitter `highlights.scm` as behavioral capture source
+
+**Date:** 2026-07-03
+
+**Status:** Accepted
+
+## Context
+
+We need a universal source of behavioral code tokens (function calls, references, definitions) across multiple programming languages. Options:
+
+- **`tags.scm`** (nvim-treesitter): Purpose-built for symbol tagging. Covers definitions and references.
+- **`highlights.scm`** (nvim-treesitter): Built for syntax highlighting. Covers a wider range of tokens including keywords, operators, and built-ins.
+- **Custom per-language queries**: Write and maintain our own query files for each language.
+
+We need tokens that represent *what the code does at runtime* — not just structure.
+
+## Decision
+
+Use nvim-treesitter `highlights.scm` as the capture source for all 10 languages.
+
+We filter captures to a `BEHAVIORAL_PREFIXES` set: `definition.`, `reference.`, `keyword.`, `function`, `attribute`, `constructor`, `label`, `type.definition`, `module`.
+
+For Kotlin, use the `ts-kotlin` (fwcd fork) bundled `highlights.scm` instead of nvim-treesitter's, because nvim-treesitter's Kotlin query references a duplicate `annotation` node type that doesn't exist in the grammar.
+
+## Consequences
+
+**Positive:**
+- `highlights.scm` covers 4 out of 5 behavioral capture types that `tags.scm` misses, across all 10 languages.
+- No per-language custom code or adapters needed.
+- Community-maintained queries stay fresh with language evolution.
+- Same query files work for both parsing and tokenizing.
+
+**Negative:**
+- `highlights.scm` includes non-behavioral captures (comments, punctuation, operators) — we filter these out.
+- Two `jsx` captures use `#set!` with 3 arguments, which `py-tree-sitter` 0.26 rejects. Strip these 2 patterns.
+- Kotlin requires a separate grammar package (`ts-kotlin`) because the nvim-treesitter Kotlin grammar is incompatible.
+
+## Alternatives Considered
+
+- **`tags.scm`**: Cleaner signal-to-noise ratio, but misses `function`, `attribute`, `constructor`, `module` captures that are essential for behavioral understanding.
+- **Custom queries**: Would give full control but require per-language maintenance — violates our universal-preprocessor constraint.

docs/adr/0002-language-agnostic-method-extraction.md

@@ -0,0 +1,50 @@
+# 2. Language-agnostic method extraction via `child_by_field_name("body")`
+
+**Date:** 2026-07-03
+
+**Status:** Accepted
+
+## Context
+
+To analyze method-level behavioral conventions, we must extract the body of each function/method from the AST. The standard tree-sitter approach is `node.child_by_field_name("body")`, but this named field is not universal across all language grammars.
+
+We need one code path that works for all 10 supported languages without per-language branches.
+
+## Decision
+
+Use `node.child_by_field_name("body")` as the primary extraction method. When it returns `None`, fall back to scanning the node's children for any child with a type containing `body`, `block`, or `compound_statement`.
+
+Parent nodes are further filtered to only include nodes whose type contains `function` or `method` — avoiding class bodies, loop bodies, and conditional blocks.
+
+This logic lives in `_find_method_bodies()` in `code.py`:
+
+```python
+def walk(node):
+    body = node.child_by_field_name("body")
+    if not body:
+        for child in node.children:
+            ctype = child.type.lower()
+            if "body" in ctype or "block" in ctype or ctype == "compound_statement":
+                body = child; break
+    if body:
+        ptype = node.type.lower()
+        if "function" in ptype or "method" in ptype:
+            bodies.append(body)
+    for child in node.children: walk(child)
+```
+
+## Consequences
+
+**Positive:**
+- Works for 9/10 grammars via `child_by_field_name("body")` alone (Python, Go, Rust, JS, TS, Ruby, Java, C, C++).
+- Kotlin fallback works because the fwcd Kotlin grammar uses `function_body` as a child node type.
+- Zero per-language case analysis — just pattern matching on type strings.
+
+**Negative:**
+- Fallback relies on string matching (`"body" in ctype`) which could produce false positives if future grammar versions introduce new body-like types.
+- C/C++ `function_definition` uses `declarator` field for the function name, not `name` — affects name extraction but not body extraction.
+
+## Alternatives Considered
+
+- **Grammar-specific field names**: Map each language to its body field name. Rejected because it creates a maintenance burden and violates the zero-adapters constraint.
+- **Top-down sibling traversal**: Walk from node start to next sibling to find the body. Fragile across grammars with different compound statement structures.

docs/adr/0003-method-level-n-gram-clustering.md

@@ -0,0 +1,44 @@
+# 3. Method-level n-gram clustering before inference
+
+**Date:** 2026-07-03
+
+**Status:** Accepted
+
+## Context
+
+The BEX ensemble (CRX, iDRegEx, kORE) infers grammars from sets of symbol sequences. When we run inference on *all methods in a codebase*, the sequences are too diverse — each file has different conventions, and the ensemble produces only a flat vocabulary bag like `(any+assertEquals+assertTrue+every+listOf+verify)+`.
+
+This doesn't capture the *ordering* of calls or the distinct methodological styles present in the codebase.
+
+## Decision
+
+Group methods by shared n-gram (default: 3-gram) call patterns *before* running inference.
+
+Pipeline: `preprocess_by_method` → `frequency_filter` → `cluster_methods` → per-cluster `infer_ensemble`
+
+The clustering algorithm:
+1. Extract call tokens from each method sequence (filter to `function`, `reference.call`, `reference.class` captures).
+2. Build an n-gram index: for each method, for each sliding window of size N, record the n-gram.
+3. Sort n-grams by frequency (most shared first).
+4. Assign each method to the largest matching cluster, then remove assigned methods.
+5. Remaining unclustered methods go to `(other)`.
+
+This produces 10-30 clusters for a typical test suite, each with 3-100+ methods sharing a call-order pattern.
+
+## Consequences
+
+**Positive:**
+- iDRegEx and kOREInference now produce ordered grammars (e.g. `every+.assertEquals.verify+.any?`) because small, focused clusters have enough signal.
+- Each cluster reveals a distinct *methodological style* in the codebase (mockist TDD vs data-driven testing vs pure assertion).
+- The `(other)` cluster still captures the full vocabulary bag for diverse methods.
+
+**Negative:**
+- Clustering adds a hyperparameter (`ngram_size`, default 3). Wrong value can produce too many tiny clusters or one giant cluster.
+- `min_cluster_size` (default 3) filters out tiny but potentially interesting patterns.
+- Methods in `(other)` never get ordered grammar inference — just vocabulary.
+
+## Alternatives Considered
+
+- **Infer on all methods (no clustering)**: Produces flat vocabulary only. CRX works at 100% coverage, but iDRegEx and kORE fail on diverse inputs.
+- **Infer per file**: Too fine-grained — most files have 1-5 methods, not enough for inference.
+- **Infer per directory**: Better, but directories mix unrelated conventions (setup/teardown vs actual test logic).

docs/adr/0004-frequency-filter-with-min-coverage.md

@@ -0,0 +1,51 @@
+# 4. Frequency filter with `min_coverage` threshold
+
+**Date:** 2026-07-03
+
+**Status:** Accepted
+
+## Context
+
+A raw method sequence can contain hundreds of unique call tokens, many of which appear in only 1-2 methods. These rare symbols are noise — they inflate grammar size, confuse the inference algorithm, and dilute the signal of common conventions.
+
+We need a principled way to discard rare symbols while keeping the behavioral patterns that define the codebase.
+
+## Decision
+
+Apply a frequency filter *before* clustering: remove any symbol that appears in fewer than `min_coverage` fraction of method sequences.
+
+Default threshold: `0.2` (20% of methods must contain the symbol).
+
+Filtering is done by `frequency_filter()` in `analyze.py`:
+```python
+n_files = len(sequences)
+threshold = max(1, int(n_files * min_coverage))
+symbol_file_count = Counter()
+for seq in sequences:
+    seen = set()
+    for _, text, _ in seq:
+        symbol_file_count[text] += 1 if text not in seen else 0
+        seen.add(text)
+keep = {text for text, count in symbol_file_count.items() if count >= threshold}
+```
+
+A symbol is counted once per file (not once per occurrence) to avoid skew from files that repeat the same symbol many times.
+
+## Consequences
+
+**Positive:**
+- Removes noise before clustering, improving cluster quality.
+- Prevents rare one-off function calls from creating spurious n-gram matches.
+- The `common` vs `other` cluster distinction is sharper because the filter removes tokens that would appear in neither.
+
+**Negative:**
+- With large codebases (600+ methods), 20% threshold may be too aggressive — a symbol needs 120+ occurrences to survive.
+  - Mitigation: `--min-coverage` flag lets users tune per codebase.
+  - Production code often needs lower values (`0.05`) because methods are more diverse than tests.
+- The threshold is relative, not absolute. A 3-file project keeps anything in 1+ files (`max(1, 3*0.2)` = 1).
+
+## Alternatives Considered
+
+- **No filter**: CRX produces `(a+b+c+d+e+f+g+h+i+j+...)+` — the vocabulary is too large to be informative.
+- **Absolute threshold**: `min_occurrences=5`. Doesn't scale — works for small projects, wrong for large ones.
+- **TF-IDF style weighting**: More sophisticated but adds complexity. The simple coverage filter works well in practice.

docs/adr/0005-import-extraction-per-cluster.md

@@ -0,0 +1,45 @@
+# 5. Import extraction per cluster
+
+**Date:** 2026-07-03
+
+**Status:** Accepted
+
+## Context
+
+An LLM prompted with a behavioral convention like `every → assertEquals → verify` still needs to know *which imports to use*. Without imports, it will guess the wrong library — writing `from unittest.mock import patch` instead of `import io.mockk.every`, or importing from `jest` instead of `vitest`.
+
+Imports are the bridge between abstract conventions and actionable code.
+
+## Decision
+
+For each cluster, scan the source files whose methods belong to that cluster and extract all unique import lines.
+
+Language-agnostic approach: match lines against common import patterns:
+- `import ...` (Java, Kotlin, Python, Go, JS/TS)
+- `from ... import ...` (Python)
+- `require ...` / `require_relative ...` (Ruby, JS)
+- `#include ...` (C/C++)
+- `use ...` (Rust)
+- `include ...` (Ruby)
+
+Scan the first 200 lines of each file (imports are always at the top), deduplicate across files, and sort the result.
+
+File-to-cluster mapping is preserved by tracking `(file_path, sequence)` pairs through the pipeline. After `frequency_filter` (which preserves order and count), we use object identity to map each clustered sequence back to its source file.
+
+## Consequences
+
+**Positive:**
+- Each cluster shows exact import lines used by its methods.
+- An LLM can copy these directly — no guessing.
+- Reveals *library choice conventions*: `kotlin.test.*` vs `org.junit.jupiter.api.*`, `io.mockk.coEvery` vs `io.mockk.every`.
+
+**Negative:**
+- Import scanning re-reads files (second pass). Negligible cost since files are small and OS-cached.
+- 200-line scan limit might miss imports in files with very long license headers.
+- Lines containing `import` in prose (comments, strings) may produce false positives — rare in practice.
+
+## Alternatives Considered
+
+- **Single global import list**: Simpler but useless — conflates imports from unrelated clusters.
+- **No imports**: LLM must guess. Leads to wrong imports and broken code.
+- **Per-file imports (not per-cluster)**: Too granular — mixes test imports with production imports in the same file.

docs/adr/0006-argument-pattern-extraction.md

@@ -0,0 +1,53 @@
+# 6. Argument pattern extraction via AST node classification
+
+**Date:** 2026-07-03
+
+**Status:** Accepted
+
+## Context
+
+A behavioral token like `assertEquals` tells the LLM that the function is called, but not *how*. Two codebases both use `assertEquals` — one writes `assertEquals(expected, actual)` and the other writes `assertEquals(actual, expected)` with swapped argument order. An LLM guessing the wrong order writes broken tests.
+
+The highlights.scm captures tell us *that* a function is called. We need the argument *structure* — number of arguments, their types, and the common patterns.
+
+## Decision
+
+For each behavioral capture node, walk up to its parent `call_expression` (or equivalent), find the argument list node, and classify each argument by structural role.
+
+Argument classification is language-agnostic:
+
+| Classification | Matches |
+|---|---|
+| `lit` | string, number, boolean, null |
+| `var` | identifiers, names |
+| `call` | nested call expressions, method invocations |
+| `lambda` | lambda expressions, blocks, do-blocks |
+| `kwarg` | keyword/named arguments |
+| `expr` | binary/unary/ternary/operator expressions |
+| `template` | string interpolation, template literals |
+| `other` | anything else (fallback) |
+
+Argument list node detection uses a tiered approach:
+1. `child_by_field_name("arguments")` — works for Python, JS, TS, Java, Go, Ruby, Rust.
+2. Fallback: scan children for `argument_list`, `arguments`, `call_suffix` (Kotlin), `template_string` (JS tagged templates).
+3. Kotlin special case: `call_suffix` may contain a direct `lambda_expression` child (for `every { ... }` syntax) or a `value_arguments → value_argument` chain (for `func(a, b)` syntax).
+
+Results are aggregated per cluster into a summary showing min/max/common arg counts and the top argument-type patterns.
+
+## Consequences
+
+**Positive:**
+- Reveals argument ordering conventions: `assertEquals: n=2 [lit,var]` means expected-first.
+- Reveals calling convention variance: `verify: n=0 [] | n=1 [lambda] | n=1 [var]` means three styles coexist.
+- No per-language branches — the tiered arglist detection handles all 10 grammars.
+
+**Negative:**
+- `kwarg` detection only covers named arguments, not default values or spread operators.
+- Nested destructuring patterns fall into `other` bucket — no granularity for complex argument shapes.
+- `other` is a catch-all that can hide meaningful distinctions we haven't classified yet.
+
+## Alternatives Considered
+
+- **Extract raw argument text**: Language-agnostic but fragile — variable names change per test, producing high variance and low signal.
+- **No argument extraction**: The LLM sees `assertEquals` but doesn't know argument order. Leads to wrong code.
+- **Per-language argument extractors**: Would be more precise but violate the zero-adapters constraint.

docs/adr/0007-json-output-for-llm-prompt-injection.md

@@ -0,0 +1,63 @@
+# 7. JSON output for LLM prompt injection
+
+**Date:** 2026-07-03
+
+**Status:** Accepted
+
+## Context
+
+The text table output is human-readable but not directly usable by an LLM. To use Dervish conventions in another agent or coding session, the output must be parsed, reformatted, and injected into a prompt — an extra friction step.
+
+An LLM consuming conventions needs:
+- Structured data it can read directly (no parsing).
+- All metadata per convention (grammar, imports, args, files, packages).
+- Compact enough to fit in context without overflow.
+
+## Decision
+
+Add a `--json` flag that outputs a structured JSON array instead of the text table.
+
+JSON structure:
+```json
+[{
+  "language": ".kt",
+  "conventions": [{
+    "label": "every → assertEquals → verify",
+    "method_count": 16,
+    "algorithm": "CRX",
+    "grammar": "every+.assertEquals.verify+.any?",
+    "mdl_score": 8.64,
+    "imports": ["import io.mockk.every", "..."],
+    "packages": ["eu/corentic/springrag/agent/capability"],
+    "arg_patterns": {
+      "assertEquals": {
+        "occurrences": 42,
+        "arg_count": {"min": 2, "max": 3, "common": 2},
+        "patterns": [{"count": 30, "args": 2, "types": ["lit", "var"]}]
+      }
+    }
+  }],
+  "total_methods": 665
+}]
+```
+
+Also accepts `--format json` and `--format text` for explicit control.
+
+## Consequences
+
+**Positive:**
+- LLM consumes the JSON directly — no parsing step needed.
+- All metadata in one object per convention — imports, args, files, packages all together.
+- `--json` is a single flag — the default text output remains for human review.
+
+**Negative:**
+- JSON is more verbose than text (full import list instead of truncated preview).
+- No easy way to limit output size — a large codebase produces JSON that may overflow context.
+  - Mitigation: `--include` flag filters files before analysis.
+
+## Alternatives Considered
+
+- **YAML output**: More readable, but less universally parseable by LLMs.
+- **CSV output**: Too flat for nested data (arg_patterns, imports list).
+- **Custom prompt template**: Would need per-framework templates. JSON is framework-agnostic.
+- **No structured output**: User must pipe through `jq` or manual reformatting. Bad UX.

docs/adr/0008-bex-ensemble-for-grammar-inference.md

@@ -0,0 +1,60 @@
+# 8. BEX ensemble for grammar inference
+
+**Date:** 2026-07-03
+
+**Status:** Accepted
+
+## Context
+
+Given a set of symbol sequences (e.g. `["every", "assertEquals", "verify"]`), we need to infer a grammar that concisely describes the pattern. Three algorithms are available:
+
+- **CRX**: Fast, produces unordered CHAREs (e.g. `(a+b+c)+`). Best for vocabulary discovery.
+- **iDRegEx**: Slower, produces ordered regex with alternation and optionality (e.g. `a.b.(c|d)?`). Best for small, clean sequences.
+- **kOREInference**: Probabilistic, handles noise well (e.g. `a.b.(b?(a|c))`). Best for diverse sequences with outliers.
+
+No single algorithm works best for all codebases. We need to pick the right one for each cluster automatically.
+
+## Decision
+
+Run all three algorithms (ensemble), compute MDL (Minimum Description Length) for each, and select the one with the lowest MDL score.
+
+MDL = grammar_length + sum of per-example encoding costs. Lower is better — the grammar explains the data most compactly.
+
+Ensemble logic in `infer_ensemble()`:
+```python
+def infer_ensemble(sequences, kmax=2, N=3, prefer=None):
+    best = None
+    best_score = float('inf')
+    for name, fn in [('CRX', crx), ('iDRegEx', idregex), ('kOREInference', kore)]:
+        if prefer and name.lower() != prefer.lower():
+            continue
+        grammar = fn(sequences, ...)
+        mdl = compute_mdl(grammar, sequences)
+        if mdl < best_score:
+            best_score = mdl
+            best = {'algorithm': name, 'grammar': grammar, 'mdl_score': mdl}
+    return {'best': best, 'all': all_results, 'why': {...}}
+```
+
+Default `kmax=2`, `N=3` (max k for k-ORE, random trials).
+
+## Consequences
+
+**Positive:**
+- CRX handles large clusters with diverse vocabulary — produces useful vocabulary bags.
+- iDRegEx fires on small, focused clusters (3-12 methods) — produces ordered grammars with exact subsequences.
+- kOREInference handles noisy clusters where methods share a theme but vary in exact call order.
+- MDL provides a principled, automatic selection criterion.
+
+**Negative:**
+- k-ORE algorithms fail on real code when sequences are too diverse (per-file sequences differ more than per-log sequences they were designed for).
+  - Clustering helps by grouping similar methods before inference.
+- iDRegEx can produce overfit grammars on very small clusters (3 methods) — e.g. `every.every.verify.(assertEquals)?` for 3 methods that happen to share an exact sequence.
+- MDL comparison assumes grammars are comparable — CRX CHAREs and iDRegEx regex use different notation, so length comparison is approximate.
+
+## Alternatives Considered
+
+- **Single algorithm (CRX only)**: Fast but produces only unordered vocab — misses ordering conventions entirely.
+- **Single algorithm (iDRegEx only)**: Produces ordered grammars but fails on diverse inputs (returns `ε`).
+- **Single algorithm (kORE only)**: Most robust to noise but slowest, and still fails on highly diverse code sequences.
+- **Algorithm per cluster size**: Manual heuristic (CRX for >20 methods, iDRegEx for <10). Harder to tune than MDL-driven selection.

references/COMMUNITY_TAGS_FINDINGS.md

@@ -0,0 +1,119 @@
+# Community tags.scm — Source Audit
+
+Audit date: July 3, 2026
+Source: Individual tree-sitter grammar repos (NOT nvim-treesitter)
+
+## Correction
+
+Existing `references/tags-queries/` files match grammar-repo content, NOT
+nvim-treesitter. The ANALYSIS.md report checked only nvim-treesitter (404).
+All grammar repos have live tags.scm.
+
+## Availability
+
+| Language | Grammar Repo | Has tags.scm? |
+|----------|-------------|---------------|
+| Python | tree-sitter/tree-sitter-python | YES |
+| JavaScript | tree-sitter/tree-sitter-javascript | YES |
+| Ruby | tree-sitter/tree-sitter-ruby | YES |
+| Go | tree-sitter/tree-sitter-go | YES |
+| Rust | tree-sitter/tree-sitter-rust | YES |
+| Java | tree-sitter/tree-sitter-java | YES |
+| C | tree-sitter/tree-sitter-c | YES |
+| C++ | tree-sitter/tree-sitter-cpp | YES |
+| TypeScript | tree-sitter/tree-sitter-typescript | YES |
+| Kotlin | fwcd/tree-sitter-kotlin | YES |
+
+## Captures Per Language
+
+### Python
+- `@definition.class` — class_definition name
+- `@definition.function` — function_definition name
+- `@definition.constant` — module-level assignment LHS
+- `@reference.call` — call function (identifier or attribute)
+
+### JavaScript
+- `@definition.class` — class / class_declaration
+- `@definition.function` — function_declaration, function_expression, arrow_function
+- `@definition.method` — method_definition
+- `@definition.constant` — export_statement with value
+- `@reference.call` — call_expression function name
+- `@reference.class` — new_expression constructor
+
+### Ruby
+- `@definition.class` — class / singleton_class
+- `@definition.method` — method / singleton_method / alias
+- `@definition.module` — module
+- `@reference.call` — call method / identifier
+
+### Go
+- `@definition.function` — function_declaration
+- `@definition.method` — method_declaration
+- `@definition.type` — type_spec
+- `@reference.call` — call_expression (direct + selector)
+- `@reference.type` — type_identifier references
+- Plus: package, import, var, const, struct, interface declarations
+
+### Rust
+- `@definition.class` — struct / enum / union / type_item
+- `@definition.function` — function_item
+- `@definition.method` — function_item in declaration_list
+- `@definition.interface` — trait_item
+- `@definition.module` — mod_item
+- `@definition.macro` — macro_definition
+- `@reference.call` — call_expression (direct + field + macro)
+- `@reference.implementation` — impl_item
+
+### Java
+- `@definition.class` — class_declaration
+- `@definition.method` — method_declaration
+- `@definition.interface` — interface_declaration
+- `@reference.call` — method_invocation with argument_list
+- `@reference.implementation` — type_list in implements
+- `@reference.class` — object_creation_expression, superclass
+
+### C
+- `@definition.class` — struct_specifier
+- `@definition.function` — function_declarator
+- `@definition.type` — type_definition, enum_specifier
+
+### C++
+- `@definition.class` — struct_specifier, union_specifier, class_specifier
+- `@definition.function` — function_declarator (identifier)
+- `@definition.method` — function_declarator (qualified_identifier)
+- `@definition.type` — type_definition, enum_specifier
+
+### TypeScript
+- `@definition.function` — function_signature
+- `@definition.method` — method_signature, abstract_method_signature
+- `@definition.class` — abstract_class_declaration
+- `@definition.module` — module
+- `@definition.interface` — interface_declaration
+- `@reference.type` — type_annotation
+- `@reference.class` — new_expression
+
+### Kotlin
+- `@definition.class` — class_declaration, object_declaration, companion_object
+- `@definition.function` — function_declaration (simple_identifier)
+- `@definition.constant` — property_declaration, enum_entry
+- `@definition.type` — type_alias
+- `@reference.call` — call_expression, navigation_expression
+- `@reference.class` — constructor_invocation
+
+## Core Vocabulary (present in all)
+
+- `@definition.*` — declaration/definition sites
+- `@reference.*` — usage/reference sites
+
+## Gaps vs Dervish Sequence Needs
+
+| Need | Present in any community tags.scm? |
+|------|-----------------------------------|
+| Call tracking | Java, Python, Go, Rust, Kotlin, JS, Ruby |
+| Constructor tracking | Java, TypeScript, Kotlin, JS |
+| Class/method/function defs | All |
+| Control flow (if/else/when) | **0 languages** |
+| Error handling (try/catch/throw) | **0 languages** |
+| Return statements | **0 languages** |
+| Annotations/decorators | **0 languages** |
+| Ordered sequences | **0 languages** |

references/gemini-conversation-summary.md

@@ -0,0 +1,207 @@
+# Dervish / BEX — Complete Conversation Summary
+
+## Overview
+
+4 Gemini conversations from 2026-07-03 (all from same session, branching at different points).
+All revolve around BEX grammar inference algorithms and the Dervish MCP tool.
+
+---
+
+## Chat 1: "LLMs, Agenten und Schema-Inferenz" (32 turns)
+**Model:** 3.5 Flash | **Link:** share/8fce4fbdf14a
+
+### Flow
+1. Starts with arXiv:1004.2372 (Bex et al. — XML schema inference via k-OREs)
+2. Gemini proposes 4 application areas for LLM agents:
+   - **① Workflow Discovery** — infer state machines from agent traces
+   - **② Dynamic Schema Generation** — distill JSON/XML schemas from unstructured data for grammar-guided decoding
+   - **③ API Reverse Engineering** — infer API structure from probe calls
+   - **④ Prompt Injection Defense** — detect structural anomalies in incoming data
+3. User picks **#3** (API reverse engineering), explores MCP tools
+4. Shifts focus to **#1 extended: Code Generation patterns** (IaC, CI/CD pipelines)
+5. **Key insight:** BEX can learn the *unwritten conventions* of existing codebases
+6. User requests Python implementation of all BEX algorithms from paper pseudocode
+7. Multiple BEX papers explored:
+   - Bex et al. 2010 (k-OREs, the main paper)
+   - Bex, Neven, Vansummeren 2008 (DTD inference — simpler, less powerful)
+   - Bex, Neven, Schwentick, Vansummeren 2010 (concise regex + DTDs)
+8. **Paradigm shift:** XML → **YAML-native**. YAML is trees too, no XML intermediate needed
+9. User requests actual pseudocode extraction from PDFs (not AI-generated hallucinations)
+10. **Escalation:** User frustrated when Gemini can't find real papers about formal grammars helping agents (hallucinates fake sources). Ends with user abandoning Gemini as search engine
+
+### Key Decisions
+- Focus on **code generation patterns** as primary application
+- **YAML-native** approach (no XML intermediate)
+- Multiple BEX papers needed for full algorithm coverage
+
+---
+
+## Chat 2: "Regex Power Beyond Formal Definitions" (14 turns)
+**Model:** 3.5 Flash | **Link:** share/d100a2d42200
+
+### Flow
+1. Regex primitives (concatenation, disjunction, iteration) vs modern regex
+2. **Grammar Induction** from positive examples only (Gold's Theorem — impossible in general case)
+3. BEX algorithms as the practical solution to this problem
+4. Improvements on BEX: interleaving (.), higher k-values
+5. User: "I have implemented BEX but I'm intrigued by interleaving"
+6. Dervish README walkthrough
+7. **embabel agent** discussion: JVM-based agent framework with typed actions
+8. GOAP (Goal-Oriented Action Planning) + Dervish — infer action preconditions from examples
+9. **Convention monitoring idea:** Dervish observes agent, detects when patterns become conventions
+
+### Key Decisions
+- BEX improvements needed: interleaving, higher k
+- Dervish could act as **passive convention observer** for agents
+
+---
+
+## Chat 3: "Dervish: Grammar Inference for LLM Agents" (32 turns)
+**Model:** 3.1 Pro | **Link:** share/0f836d3c25ba
+
+### Flow (Turns 1-27 — shared with Chat 4)
+
+#### Phase: README Polish (Turns 1-5)
+- README paste → improve "MDL" wording
+- Add **generative schema** section (infer grammar → generate sample data)
+- Add "Why not just use a schema?" section
+
+#### Phase: Architecture Discovery (Turns 6-10)
+- Question: **Can Dervish analyze Java/Kotlin codebases?**
+- Answer: Language structure ≠ code conventions. Dervish needs *behavioral* sequences
+- **Turn 7: DomainRouter pre-detection** idea — detect context (source code vs YAML) before choosing algorithm
+- **Turn 8: MDL Ensemble insight** — Dervish already runs all algorithms and picks best by MDL. No need for pre-detection!
+- **Turn 9: Single-pass heuristic** — maybe we don't need all algorithms, can derive coverage levels from one pass
+
+#### Phase: TreeSitter Revolution (Turns 10-17)
+- **Turn 10: Language-agnostic breakthrough** — Dervish is absolutely language-agnostic, can analyze any codebase
+- **Turn 11: Prior art needed** — is there existing research on grammar inference from ASTs?
+- **Turn 12: TreeSitter AST pipeline** — extract AST nodes → flatten to sequences → Dervish infers grammar
+- **Turn 13-14: Concrete Java examples** — spring boot controllers, which AST nodes matter
+- **Turn 15: Repository vs directory scoping** — how does Dervish know where conventions boundaries are?
+- **Turn 16: Avoiding per-language heuristics** — must be zero-config
+- **Turn 17: Novelty confirmed** — no prior work on grammar inference from TreeSitter ASTs. This is a genuine research contribution
+
+#### Phase: Roadmap Definition (Turns 18-20)
+- **Formal roadmap defined:**
+  - **1.0: Code Repository Analysis** — TreeSitter → AST nodes → Dervish grammar
+  - **1.1: Structured Data** (YAML, XML, JSON) — existing feature, enhance
+  - **1.2: Markdown/Document AST** — DISCUSSED THEN **DROPPED** (too fuzzy)
+  - **2.0: Cross-file conventions** — class hierarchies, test patterns, file-level relationships
+- **1.2 dropped** — documents too fuzzy
+
+#### Phase: Implementation Design (Turns 21-27)
+- **Features first, combined later** — code analysis separate from structured data
+- **Interesting nodes** — NOT per-language heuristic! Use **TreeSitter's built-in tag queries** (tags.scm)
+- Universal query approach: a single query works across languages (annotation→@meta, call_expression→@call, try_statement→@block)
+- **Community tags.scm** for 100% coverage (Zero maintenance)
+- **Single-pass frequency analysis** — count symbol frequency across files, filter outliers < threshold
+- **Concrete example with Spring Boot:**
+  - Raw AST: `['@RestController', '@RequestMapping', '@PostMapping', 'log.info', 'dto.getItems', 'dto.getItems.isEmpty', 'orderService.process', 'ResponseEntity.ok']`
+  - After frequency filter: remove `dto.getItems` (occurs in 1/10 files)
+  - Result: convention grammar for Spring Boot controllers
+- **Critical concern (Turn 27):** Will Dervish find genuinely novel insights or just obvious patterns?
+
+#### Phase: Broader Vision (Turns 28-32 — UNIQUE TO CHAT 3)
+- **BEX in LLM training/design** — structural tokenization (replace BPE with grammar-guided), Skeleton-of-Thought patterns
+- **Paper title proposals:**
+  1. "DERVISH: Neuro-Symbolic Code Generation via MDL-Optimized Structural Grammars"
+  2. "AST2Regex: Mining Implicit Conventions from Abstract Syntax Trees"
+  3. "Grammar is All You Need: Zero-Shot Convention Learning for LLMs"
+- **Terminology debate:** "skeleton" bad, prefer "grammar" or "schema"
+- **Grammar-Constrained Decoding (GCD)** research — position Dervish as bridging structured output generation
+- Final task: deep research into GCD + neuro-symbolic generation literature
+
+### Key Decisions
+- **TreeSitter AST → Dervish pipeline** is the core architecture
+- **No per-language heuristics** — use TreeSitter's universal node types and community tag queries
+- **Single-pass frequency analysis** filters noise
+- **1.2 (documents) dropped** — focus on code repos + structured data
+- **Features isolated** before integration
+- **This is novel research** — no prior work on grammar inference from TreeSitter ASTs
+
+---
+
+## Chat 4: "Dervish: Grammar Inference for LLM Agents" (27 turns)
+**Model:** 3.1 Pro | **Link:** share/a5ff288e0fdf
+
+### Relationship to Chat 3
+- **IDENTICAL to Chat 3 for Turns 1-27** (same conversation, same user messages, same AI responses)
+- **Ends at Turn 27** — Chat 3 continues with 5 additional turns (28-32)
+- Chat 4 = the "short branch" of the conversation
+
+---
+
+## Divergence Map
+
+```
+Chat 1 (32 turns) — LLMs/Agenten/Schema
+─────────────────────────────────
+Separate conversation, different focus (MCP, YAML-native, PDF algorithms)
+
+Chat 2 (14 turns) — Regex theory
+─────────────────────────────────
+Separate conversation, different focus (regex theory, embabel, GOAP)
+
+Chat 3 (32 turns) — Dervish: Grammar Inference
+Chat 4 (27 turns) — Dervish: Grammar Inference
+│
+├── Turns 1-5:  README polish (MDL wording, generative schema section)
+├── Turns 6-10: Architecture (DomainRouter, MDL ensemble, TreeSitter idea)
+├── Turns 11-17: TreeSitter deep dive (language-agnostic, prior art, roadmapping)
+├── Turns 18-20: Formal roadmap, 1.2 dropped
+├── Turns 21-27: Implementation design (tags queries, frequency analysis, Spring Boot example)
+│
+└── Turn 27: "Will it find genuine insights?" (SAME question in both)
+    ├── Chat 3 continues (Turns 28-32)
+    │   ├── 28: BEX in LLM training (structural tokenization, SoT)
+    │   ├── 29: Paper search on GCD approaches
+    │   ├── 30: arXiv paper title proposals
+    │   ├── 31: Terminology debate (grammar vs skeleton)
+    │   └── 32: GCD research assignment
+    │
+    └── Chat 4 ENDS at Turn 27
+```
+
+---
+
+## Planned Feature Roadmap (from Chats 3-4)
+
+### Phase 1.0: Code Repository Analysis (TreeSitter AST → Dervish)
+1. **Language detection** (file extension, MIME type)
+2. **TreeSitter parsing** with language-specific grammar
+3. **UNIVERSAL_STRUCTURE_QUERY** (single query for all languages):
+   - `(call_expression) @call`
+   - `(annotation) @meta` / `(decorator) @meta`
+   - `(try_statement) @block` / `(catch_clause) @block`
+   - Alternatively: load community `tags.scm` for each language
+4. **Single-pass frequency analysis** — count symbol frequency across files, filter outliers below threshold
+5. **Dervish inference** on cleaned sequences → compact grammar
+6. **Output:** ~60-200 token rule representing codebase conventions
+
+### Phase 1.1: Structured Data (existing feature, enhance)
+- YAML/XML/JSON sequences → grammar inference
+- Already partially implemented in Dervish
+
+### Phase 1.2: Markdown/Document AST (DROPPED)
+- Too fuzzy, documents vary too much
+- Not a priority
+
+### Phase 2.0: Cross-File Conventions (future)
+- Class hierarchies, test patterns, file-level relationships
+- Beyond single-file AST analysis
+
+### Broader Research (Chat 3 extra turns)
+- **Structural tokenization** — replace BPE with grammar-guided tokenization
+- **Grammar-Constrained Decoding (GCD)** — position Dervish in the GCD ecosystem
+- **arXiv paper** — "DERVISH: Neuro-Symbolic Code Generation via MDL-Optimized Structural Grammars"
+- **Literature review** — deep search on GCD + neuro-symbolic generation
+
+---
+
+## Repository State
+- **Main repo:** grammar-inference-engine (git submodule at `projects/grammar-inference-engine/`)
+- **Branch:** `feature/dervish-2`
+- **Remote:** `origin → https://forgejo.corentic.eu/tobi/grammar-inference-engine`
+- **Current code:** BEX algorithms (CRX, iDRegEx), MCP server, basic ORES/SORE/CHARE types
+- **Need 5th Gemini share link** — user mentioned 5 chats, we only have 4

references/tags-queries/ANALYSIS.md

@@ -0,0 +1,257 @@
+# Community Tree-Sitter Query Analysis
+
+Analyzed: July 3, 2026
+Source: nvim-treesitter (master branch) tag queries + highlight queries
+
+## Key Finding
+
+**Community TAGS queries are for code navigation (Go to Definition), NOT for Dervish.**
+
+Dervish needs **behavioral sequences**: what operations happen in what order.
+Tags queries only capture declaration sites and reference locations.
+
+## Language-by-Language Gap Analysis
+
+### JAVA
+
+**Tags captures:**
+- `@definition.class` — class declaration sites
+- `@definition.method` — method declaration sites
+- `@definition.interface` — interface declaration sites
+- `@reference.call` — method invocations (ONLY with `argument_list`)
+- `@reference.implementation` — implements clauses
+- `@reference.class` — `new Type()` expressions + superclass refs
+
+**What is MISSING for Dervish:**
+- `@annotation` / `@decorator` — annotations only exist in highlights as `@attribute`, not in tags
+- `@reference.call` excludes parameterless calls like `foo()` — requires `argument_list`
+- No `throw_statement`, `try_statement`, `catch_clause` in tags
+- No `if_statement`, `return_statement` captures
+- No mechanism to extract ORDERED sequences
+- Highlights collapse all control-flow keywords into `@keyword`
+
+### KOTLIN (from highlights.scm since no tags.scm exists at that path)
+
+**Available captures (highlights only):**
+- `@function.call` — `call_expression` with `simple_identifier`
+- `@constructor` — `constructor_invocation`, `primary_constructor`, secondary
+- `@attribute` — `annotation` with `user_type`
+- `@keyword.conditional` — `if`/`else`/`when`
+- `@keyword.repeat` — `for`/`do`/`while`
+- `@keyword.exception` — `try`/`catch`/`throw`/`finally`
+- `@keyword.return` — `return`
+- `@function` — `function_declaration`
+
+**What is MISSING for Dervish:**
+- No tags.scm available on nvim-treesitter master for Kotlin
+- Call expressions captured as `@function.call` in highlights, not structured for sequence extraction
+- Navigation expressions (`obj.method()`) captured but as `@function.call` on the suffix
+- No `when_structure` / `when_condition` structural capture
+- No `navigation_expression` itself as a token — only the method name inside
+- No sequencing mechanism
+
+### PYTHON
+
+**Tags captures:**
+- `@definition.constant` — module-level assignments
+- `@definition.class` — class definitions
+- `@definition.function` — function definitions
+- `@reference.call` — direct function calls + method calls via attribute
+
+**What is MISSING for Dervish:**
+- No `decorator` capture in tags (only in highlights as `@function`)
+- `call` only captured as reference, not as behavioral sequence token
+- No `raise_statement`, `try_statement`, `except_clause` in tags
+- No `if_statement`, `return_statement`, `with_statement` in tags
+- Highlights `@function` conflates decorators, definitions, and calls
+
+### GO
+
+**Tags captures:**
+- `@definition.function` — function declarations (with doc comments)
+- `@definition.method` — method declarations
+- `@definition.type` — type specs
+- `@reference.call` — direct calls + selector calls (`obj.Method()`)
+- `@reference.type` — type identifier references
+- Plus: var/const/import/struct/interface tracking
+
+**What is MISSING for Dervish:**
+- No `defer_statement` capture (only `defer` keyword in highlights)
+- No `go` (goroutine launch) as structural capture
+- No `if_statement`, `return_statement`, `for_statement`, `switch_statement` in tags
+- `call_expression` well captured but only function name as `@name`, not whole expression
+- Highlights distinguish builtins (`append`, `len`, `make`, `panic`, etc.) but not structured
+
+### RUST
+
+**Tags captures:**
+- `@definition.class` — struct/enum/union/type alias
+- `@definition.method` — methods in impl blocks
+- `@definition.function` — free functions
+- `@definition.interface` — trait definitions
+- `@definition.module` — module definitions
+- `@definition.macro` — macro definitions
+- `@reference.call` — direct calls + method calls + macro invocations
+- `@reference.implementation` — trait/inherent impl tracking
+
+**What is MISSING for Dervish:**
+- Best coverage of any language — captures calls, macros, implementations
+- Still: no `if_expression` / `match_expression` / `return` in tags
+- No `unsafe` block capture
+- `@attribute` (`#[derive]`) only in highlights, not tags
+- No sequencing mechanism
+
+### TYPESCRIPT / JAVASCRIPT
+
+**Tags captures:**
+- `@definition.function` — function signatures only (definitions)
+- `@definition.method` — method signatures
+- `@definition.class` — class + abstract class declarations
+- `@definition.module` — module declarations
+- `@definition.interface` — interface declarations
+- `@reference.type` — type annotations
+- `@reference.class` — `new Foo()` constructor calls
+
+**What is MISSING for Dervish:**
+- **NO `call_expression` capture at all** — neither in tags nor highlights
+- **NO `@reference.call`** — critical gap
+- No function/method call tracking
+- Highlight query is extremely minimal (35 lines)
+- Missing: `arrow_function`, `call_expression`, `method_invocation`
+- Query file seems designed for `.d.ts` type definitions, not implementation code
+
+### C++
+
+**Tags captures:**
+- `@definition.class` — struct/union/class specifiers
+- `@definition.function` — function declarators
+- `@definition.method` — qualified methods (with namespace scope)
+- `@definition.type` — typedef/enum
+
+**What is MISSING for Dervish:**
+- **NO `call_expression` capture in tags** — call tracking only in highlights as `@function`
+- **NO `@reference.call`** in tags at all
+- No `template_function`, `template_method` in tags
+- No `new_expression`, `delete_expression` in tags
+- No `throw_statement`, `try_statement`, `catch_clause` in tags
+- No sequencing mechanism
+
+### YAML
+
+**What is MISSING for Dervish:**
+- No tags query exists at all
+- YAML is data-oriented, not code-oriented
+- Highlights only capture data types (strings, numbers, booleans) and key names
+- YAML sequence extraction (for Ansible/Helm) needs a different approach entirely
+
+## Cross-Cutting Findings
+
+### What community tags DO capture (for Dervish, partially useful):
+
+| Capture | Languages | Dervish Value |
+|---------|-----------|--------------|
+| `@reference.call` | Java, Python, Go, Rust | **Partial** — identifies call sites but per-language inconsistencies |
+| `@reference.class` / `new` | Java, TypeScript | **Partial** — constructor calls |
+| `@reference.implementation` | Java, Rust | Low — inheritance tracking |
+| macro invocations (as `@reference.call`) | Rust | **Useful** — Rust macros ARE behavioral tokens |
+
+### What community tags do NOT capture (GAPS):
+
+| Dervish Need | In Tags? | In Highlights? |
+|-------------|----------|---------------|
+| `@annotation` / `@decorator` / `@attribute` | **0 languages** | Java `@attribute`, Python `@function` (decorator), Rust `@attribute`, Kotlin `@attribute` |
+| Control flow (`if`/`else`/`when`) | **0 languages** | All: collapsed in `@keyword` |
+| Error handling (`try`/`catch`/`throw`) | **0 languages** | All: collapsed in `@keyword` |
+| `@return` | **0 languages** | All: collapsed in `@keyword` |
+| Call expressions | Java, Python, Go, Rust | All except TS |
+| Ordered sequences | **0 languages** | **0 languages** |
+| Variable declarations | Go only | Not structural |
+| Resource management (`defer`/`with`) | **0 languages** | Go: `defer` in keyword; Python: `with` in keyword |
+| Goroutines / async / coroutines | **0 languages** | Go `go` in keyword; Python async/await in keyword; Kotlin `suspend` as `@keyword.coroutine` |
+
+### The Naming Inconsistency Problem
+
+Same tree-sitter node types have DIFFERENT capture names across languages:
+
+| Node | Java | Python | Go | Rust | C++ | TS |
+|------|------|--------|----|------|-----|----|
+| call expr | `@reference.call` | `@reference.call` | `@reference.call` | `@reference.call` | NONE | NONE |
+| class def | `@definition.class` | `@definition.class` | -- | `@definition.class` | `@definition.class` | `@definition.class` |
+| function def | -- | `@definition.function` | `@definition.function` | `@definition.function` | `@definition.function` | `@definition.function` |
+
+Dervish cannot rely on capture names alone. Must either:
+1. Use universal query with Dervish-specific capture names, OR
+2. Map language-specific capture names to canonical Dervish vocabulary
+
+### Community tags.scm availability on nvim-treesitter master
+
+Checked July 3, 2026 via `raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/master/queries/<lang>/tags.scm`:
+
+| Language | tags.scm exists? | highlights.scm? |
+|----------|-----------------|-----------------|
+| Java | 404 | yes (in refs) |
+| Kotlin | 404 | yes (newly downloaded) |
+| Python | 404 | yes (in refs) |
+| Go | 404 | yes (in refs) |
+| Rust | 404 | yes (in refs) |
+| TypeScript | 404 | yes (in refs) |
+| C++ | 404 | yes (in refs) |
+| YAML | never | yes (in refs) |
+
+All tags.scm returned 404 — nvim-treesitter may have restructured or moved to a different branch/tag.
+
+## Resolution
+
+This analysis was performed against community **tags.scm** queries. The actual implementation uses **nvim-treesitter `highlights.scm`** instead, which resolves the critical gaps:
+
+| Gap in tags.scm | highlights.scm outcome |
+|---|---|
+| Missing control flow (`if`/`else`/`when`) | `@keyword.conditional` — all languages |
+| Missing error handling (`try`/`catch`/`throw`) | `@keyword.exception` — all languages |
+| Missing `@return` | `@keyword.return` — all languages |
+| Missing annotations/decorators | Java/Kotlin `@attribute`, Rust `@attribute`, Python `@function` (decorator) |
+| Naming inconsistency across langs | Mitigated by **prefix-based filter** (`definition.*`, `keyword.*`, `reference.*`, etc.) — exact capture names don't need to match, only prefixes |
+
+Highlights.scm proved richer than tags.scm for behavioral sequence extraction, because highlight queries deliberately distinguish keywords, functions, calls, and attributes — exactly what Dervish needs.
+
+### Special Cases
+
+Two languages required deviations from the plain nvim-treesitter approach:
+
+**Kotlin** — nvim-treesitter `queries/kotlin/highlights.scm` fails at row 296 due to a conflict between the grammar's duplicate `annotation` node type (id 92 named=False, id 304 named=True). Resolution: use `ts-kotlin`'s bundled `queries/highlights.scm` instead (`pip install ts-kotlin`). This is the fwcd grammar fork, which also includes the `simple_identifier` node needed by highlight queries.
+
+**JavaScript/TypeScript** — JS query uses `; inherits: ecma, jsx` and TS uses `; inherits: ecma`. The `code.py` preprocessor resolves inheritance by concatenating parent query content before child content. Additionally, `jsx.scm` contains two `#set!` predicates with 3 arguments (`#set! @_capture property value`) which py-tree-sitter 0.26 rejects (expects 1-2 args). Resolution: those two `bo.commentstring` patterns were stripped from `jsx.scm`.
+
+## Verdict
+
+**Community tags.scm alone are insufficient. But nvim-treesitter highlights.scm — used as-is with two caveats — covers 4/5 gaps.** The remaining gap (no sequencing/traversal mechanism) is handled by tree-sitter's built-in ordered cursor iteration over captures, which naturally preserves source order.
+
+A universal query file is no longer needed. The `code.py` preprocessor loads highlights.scm per-language, resolves `; inherits:` directives, handles the two known deviations (Kotlin bundled query, jsx `#set!` stripping), and filters by behavioral capture prefix — zero adapters, zero per-language branches.
+
+## Method-Level Sequencing
+
+### The Problem
+
+Flat per-file sequences are too diverse for k-ORE to produce ordered grammars. Each file mixes multiple function bodies into one flat token list — the ordering across methods is meaningless.
+
+### The Solution: `child_by_field_name("body")`
+
+Research across all 10 grammar `node-types.json` files revealed that every tree-sitter grammar stores function/method bodies in a field called **`body`**:
+
+| Language | Function node type | `body` field type |
+|----------|-------------------|-------------------|
+| Python | `function_definition` | `block` |
+| Go | `function_declaration`, `method_declaration` | `block` |
+| Rust | `function_item` | `block` |
+| JavaScript | `function_declaration`, `method_definition`, `arrow_function` | `statement_block` |
+| TypeScript | `function_declaration`, `method_signature` | `statement_block` |
+| Ruby | `method` | `body_statement` |
+| Java | `method_declaration` | `block` |
+| C/C++ | `function_definition` | `compound_statement` |
+| Kotlin (fwcd) | `function_declaration` *(no field — positional child)* | `function_body` |
+
+**9/10 grammars** expose the body via `node.child_by_field_name("body")` in the tree-sitter API. Kotlin (fwcd grammar) is the exception — its `function_declaration` doesn't use a named `body` field. For Kotlin, we scan children and match by type name (`function_body`).
+
+The parent filter `"function" in node.type or "method" in node.type` ensures we don't capture class bodies, loop bodies, or other block-like constructs.
+
+This gives us **per-method sequences**: each function/method body becomes its own token list. k-ORE can then find actual call-order conventions (e.g. `validate → process → respond`).

references/tags-queries/cpp_highlights_query.scm

@@ -0,0 +1,70 @@
+; Functions
+
+(call_expression
+  function: (qualified_identifier
+    name: (identifier) @function))
+
+(template_function
+  name: (identifier) @function)
+
+(template_method
+  name: (field_identifier) @function)
+
+(template_function
+  name: (identifier) @function)
+
+(function_declarator
+  declarator: (qualified_identifier
+    name: (identifier) @function))
+
+(function_declarator
+  declarator: (field_identifier) @function)
+
+; Types
+
+((namespace_identifier) @type
+ (#match? @type "^[A-Z]"))
+
+(auto) @type
+
+; Constants
+
+(this) @variable.builtin
+(null "nullptr" @constant)
+
+; Keywords
+
+[
+ "catch"
+ "class"
+ "co_await"
+ "co_return"
+ "co_yield"
+ "constexpr"
+ "constinit"
+ "consteval"
+ "delete"
+ "explicit"
+ "final"
+ "friend"
+ "mutable"
+ "namespace"
+ "noexcept"
+ "new"
+ "override"
+ "private"
+ "protected"
+ "public"
+ "template"
+ "throw"
+ "try"
+ "typename"
+ "using"
+ "concept"
+ "requires"
+ "virtual"
+] @keyword
+
+; Strings
+
+(raw_string_literal) @string

references/tags-queries/cpp_injections_query.scm

@@ -0,0 +1,3 @@
+(raw_string_literal
+  delimiter: (raw_string_delimiter) @injection.language
+  (raw_string_content) @injection.content)

references/tags-queries/cpp_tags_query.scm

@@ -0,0 +1,15 @@
+(struct_specifier name: (type_identifier) @name body:(_)) @definition.class
+
+(declaration type: (union_specifier name: (type_identifier) @name)) @definition.class
+
+(function_declarator declarator: (identifier) @name) @definition.function
+
+(function_declarator declarator: (field_identifier) @name) @definition.function
+
+(function_declarator declarator: (qualified_identifier scope: (namespace_identifier) @local.scope name: (identifier) @name)) @definition.method
+
+(type_definition declarator: (type_identifier) @name) @definition.type
+
+(enum_specifier name: (type_identifier) @name) @definition.type
+
+(class_specifier name: (type_identifier) @name) @definition.class

references/tags-queries/go_highlights_query.scm

@@ -0,0 +1,123 @@
+; Function calls
+
+(call_expression
+  function: (identifier) @function)
+
+(call_expression
+  function: (identifier) @function.builtin
+  (#match? @function.builtin "^(append|cap|close|complex|copy|delete|imag|len|make|new|panic|print|println|real|recover)$"))
+
+(call_expression
+  function: (selector_expression
+    field: (field_identifier) @function.method))
+
+; Function definitions
+
+(function_declaration
+  name: (identifier) @function)
+
+(method_declaration
+  name: (field_identifier) @function.method)
+
+; Identifiers
+
+(type_identifier) @type
+(field_identifier) @property
+(identifier) @variable
+
+; Operators
+
+[
+  "--"
+  "-"
+  "-="
+  ":="
+  "!"
+  "!="
+  "..."
+  "*"
+  "*"
+  "*="
+  "/"
+  "/="
+  "&"
+  "&&"
+  "&="
+  "%"
+  "%="
+  "^"
+  "^="
+  "+"
+  "++"
+  "+="
+  "<-"
+  "<"
+  "<<"
+  "<<="
+  "<="
+  "="
+  "=="
+  ">"
+  ">="
+  ">>"
+  ">>="
+  "|"
+  "|="
+  "||"
+  "~"
+] @operator
+
+; Keywords
+
+[
+  "break"
+  "case"
+  "chan"
+  "const"
+  "continue"
+  "default"
+  "defer"
+  "else"
+  "fallthrough"
+  "for"
+  "func"
+  "go"
+  "goto"
+  "if"
+  "import"
+  "interface"
+  "map"
+  "package"
+  "range"
+  "return"
+  "select"
+  "struct"
+  "switch"
+  "type"
+  "var"
+] @keyword
+
+; Literals
+
+[
+  (interpreted_string_literal)
+  (raw_string_literal)
+  (rune_literal)
+] @string
+
+(escape_sequence) @escape
+
+[
+  (int_literal)
+  (float_literal)
+  (imaginary_literal)
+] @number
+
+[
+  (true)
+  (false)
+  (nil)
+  (iota)
+] @constant.builtin
+
+(comment) @comment

references/tags-queries/go_tags_query.scm

@@ -0,0 +1,42 @@
+(
+  (comment)* @doc
+  .
+  (function_declaration
+    name: (identifier) @name) @definition.function
+  (#strip! @doc "^//\\s*")
+  (#set-adjacent! @doc @definition.function)
+)
+
+(
+  (comment)* @doc
+  .
+  (method_declaration
+    name: (field_identifier) @name) @definition.method
+  (#strip! @doc "^//\\s*")
+  (#set-adjacent! @doc @definition.method)
+)
+
+(call_expression
+  function: [
+    (identifier) @name
+    (parenthesized_expression (identifier) @name)
+    (selector_expression field: (field_identifier) @name)
+    (parenthesized_expression (selector_expression field: (field_identifier) @name))
+  ]) @reference.call
+
+(type_spec
+  name: (type_identifier) @name) @definition.type
+
+(type_identifier) @name @reference.type
+
+(package_clause "package" (package_identifier) @name)
+
+(type_declaration (type_spec name: (type_identifier) @name type: (interface_type)))
+
+(type_declaration (type_spec name: (type_identifier) @name type: (struct_type)))
+
+(import_declaration (import_spec) @name)
+
+(var_declaration (var_spec name: (identifier) @name))
+
+(const_declaration (const_spec name: (identifier) @name))

references/tags-queries/java_highlights_query.scm

@@ -0,0 +1,149 @@
+; Variables
+
+(identifier) @variable
+
+; Methods
+
+(method_declaration
+  name: (identifier) @function.method)
+(method_invocation
+  name: (identifier) @function.method)
+(super) @function.builtin
+
+; Annotations
+
+(annotation
+  name: (identifier) @attribute)
+(marker_annotation
+  name: (identifier) @attribute)
+
+"@" @operator
+
+; Types
+
+(type_identifier) @type
+
+(interface_declaration
+  name: (identifier) @type)
+(class_declaration
+  name: (identifier) @type)
+(enum_declaration
+  name: (identifier) @type)
+
+((field_access
+  object: (identifier) @type)
+ (#match? @type "^[A-Z]"))
+((scoped_identifier
+  scope: (identifier) @type)
+ (#match? @type "^[A-Z]"))
+((method_invocation
+  object: (identifier) @type)
+ (#match? @type "^[A-Z]"))
+((method_reference
+  . (identifier) @type)
+ (#match? @type "^[A-Z]"))
+
+(constructor_declaration
+  name: (identifier) @type)
+
+[
+  (boolean_type)
+  (integral_type)
+  (floating_point_type)
+  (floating_point_type)
+  (void_type)
+] @type.builtin
+
+; Constants
+
+((identifier) @constant
+ (#match? @constant "^_*[A-Z][A-Z\\d_]+$"))
+
+; Builtins
+
+(this) @variable.builtin
+
+; Literals
+
+[
+  (hex_integer_literal)
+  (decimal_integer_literal)
+  (octal_integer_literal)
+  (decimal_floating_point_literal)
+  (hex_floating_point_literal)
+] @number
+
+[
+  (character_literal)
+  (string_literal)
+] @string
+(escape_sequence) @string.escape
+
+[
+  (true)
+  (false)
+  (null_literal)
+] @constant.builtin
+
+[
+  (line_comment)
+  (block_comment)
+] @comment
+
+; Keywords
+
+[
+  "abstract"
+  "assert"
+  "break"
+  "case"
+  "catch"
+  "class"
+  "continue"
+  "default"
+  "do"
+  "else"
+  "enum"
+  "exports"
+  "extends"
+  "final"
+  "finally"
+  "for"
+  "if"
+  "implements"
+  "import"
+  "instanceof"
+  "interface"
+  "module"
+  "native"
+  "new"
+  "non-sealed"
+  "open"
+  "opens"
+  "package"
+  "permits"
+  "private"
+  "protected"
+  "provides"
+  "public"
+  "requires"
+  "record"
+  "return"
+  "sealed"
+  "static"
+  "strictfp"
+  "switch"
+  "synchronized"
+  "throw"
+  "throws"
+  "to"
+  "transient"
+  "transitive"
+  "try"
+  "uses"
+  "volatile"
+  "when"
+  "while"
+  "with"
+  "yield"
+] @keyword

references/tags-queries/java_tags_query.scm

@@ -0,0 +1,20 @@
+(class_declaration
+  name: (identifier) @name) @definition.class
+
+(method_declaration
+  name: (identifier) @name) @definition.method
+
+(method_invocation
+  name: (identifier) @name
+  arguments: (argument_list) @reference.call)
+
+(interface_declaration
+  name: (identifier) @name) @definition.interface
+
+(type_list
+  (type_identifier) @name) @reference.implementation
+
+(object_creation_expression
+  type: (type_identifier) @name) @reference.class
+
+(superclass (type_identifier) @name) @reference.class

references/tags-queries/kotlin_tags_query.scm

@@ -0,0 +1,43 @@
+; Classes
+(class_declaration
+  (type_identifier) @name) @definition.class
+
+; Objects
+(object_declaration
+  (type_identifier) @name) @definition.class
+
+; Functions (top-level and member)
+(function_declaration
+  (simple_identifier) @name) @definition.function
+
+; Properties
+(property_declaration
+  (variable_declaration
+    (simple_identifier) @name)) @definition.constant
+
+; Enum entries
+(enum_entry
+  (simple_identifier) @name) @definition.constant
+
+; Type aliases
+(type_alias
+  (type_identifier) @name) @definition.type
+
+; Companion objects (only named ones)
+(companion_object
+  (type_identifier) @name) @definition.class
+
+; Function calls
+(call_expression
+  (simple_identifier) @name) @reference.call
+
+; Method calls via navigation
+(call_expression
+  (navigation_expression
+    (navigation_suffix
+      (simple_identifier) @name))) @reference.call
+
+; Constructor invocations (class references)
+(constructor_invocation
+  (user_type
+    (type_identifier) @name)) @reference.class

references/tags-queries/python_highlights_query.scm

@@ -0,0 +1,137 @@
+; Identifier naming conventions
+
+(identifier) @variable
+
+((identifier) @constructor
+ (#match? @constructor "^[A-Z]"))
+
+((identifier) @constant
+ (#match? @constant "^[A-Z][A-Z_]*$"))
+
+; Function calls
+
+(decorator) @function
+(decorator
+  (identifier) @function)
+
+(call
+  function: (attribute attribute: (identifier) @function.method))
+(call
+  function: (identifier) @function)
+
+; Builtin functions
+
+((call
+  function: (identifier) @function.builtin)
+ (#match?
+   @function.builtin
+   "^(abs|all|any|ascii|bin|bool|breakpoint|bytearray|bytes|callable|chr|classmethod|compile|complex|delattr|dict|dir|divmod|enumerate|eval|exec|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|isinstance|issubclass|iter|len|list|locals|map|max|memoryview|min|next|object|oct|open|ord|pow|print|property|range|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|vars|zip|__import__)$"))
+
+; Function definitions
+
+(function_definition
+  name: (identifier) @function)
+
+(attribute attribute: (identifier) @property)
+(type (identifier) @type)
+
+; Literals
+
+[
+  (none)
+  (true)
+  (false)
+] @constant.builtin
+
+[
+  (integer)
+  (float)
+] @number
+
+(comment) @comment
+(string) @string
+(escape_sequence) @escape
+
+(interpolation
+  "{" @punctuation.special
+  "}" @punctuation.special) @embedded
+
+[
+  "-"
+  "-="
+  "!="
+  "*"
+  "**"
+  "**="
+  "*="
+  "/"
+  "//"
+  "//="
+  "/="
+  "&"
+  "&="
+  "%"
+  "%="
+  "^"
+  "^="
+  "+"
+  "->"
+  "+="
+  "<"
+  "<<"
+  "<<="
+  "<="
+  "<>"
+  "="
+  ":="
+  "=="
+  ">"
+  ">="
+  ">>"
+  ">>="
+  "|"
+  "|="
+  "~"
+  "@="
+  "and"
+  "in"
+  "is"
+  "not"
+  "or"
+  "is not"
+  "not in"
+] @operator
+
+[
+  "as"
+  "assert"
+  "async"
+  "await"
+  "break"
+  "class"
+  "continue"
+  "def"
+  "del"
+  "elif"
+  "else"
+  "except"
+  "exec"
+  "finally"
+  "for"
+  "from"
+  "global"
+  "if"
+  "import"
+  "lambda"
+  "nonlocal"
+  "pass"
+  "print"
+  "raise"
+  "return"
+  "try"
+  "while"
+  "with"
+  "yield"
+  "match"
+  "case"
+] @keyword

references/tags-queries/python_tags_query.scm

@@ -0,0 +1,14 @@
+(module (expression_statement (assignment left: (identifier) @name) @definition.constant))
+
+(class_definition
+  name: (identifier) @name) @definition.class
+
+(function_definition
+  name: (identifier) @name) @definition.function
+
+(call
+  function: [
+      (identifier) @name
+      (attribute
+        attribute: (identifier) @name)
+  ]) @reference.call

references/tags-queries/rust_highlights_query.scm

@@ -0,0 +1,161 @@
+; Identifiers
+
+(type_identifier) @type
+(primitive_type) @type.builtin
+(field_identifier) @property
+
+; Identifier conventions
+
+; Assume all-caps names are constants
+((identifier) @constant
+ (#match? @constant "^[A-Z][A-Z\\d_]+$'"))
+
+; Assume uppercase names are enum constructors
+((identifier) @constructor
+ (#match? @constructor "^[A-Z]"))
+
+; Assume that uppercase names in paths are types
+((scoped_identifier
+  path: (identifier) @type)
+ (#match? @type "^[A-Z]"))
+((scoped_identifier
+  path: (scoped_identifier
+    name: (identifier) @type))
+ (#match? @type "^[A-Z]"))
+((scoped_type_identifier
+  path: (identifier) @type)
+ (#match? @type "^[A-Z]"))
+((scoped_type_identifier
+  path: (scoped_identifier
+    name: (identifier) @type))
+ (#match? @type "^[A-Z]"))
+
+; Assume all qualified names in struct patterns are enum constructors. (They're
+; either that, or struct names; highlighting both as constructors seems to be
+; the less glaring choice of error, visually.)
+(struct_pattern
+  type: (scoped_type_identifier
+    name: (type_identifier) @constructor))
+
+; Function calls
+
+(call_expression
+  function: (identifier) @function)
+(call_expression
+  function: (field_expression
+    field: (field_identifier) @function.method))
+(call_expression
+  function: (scoped_identifier
+    "::"
+    name: (identifier) @function))
+
+(generic_function
+  function: (identifier) @function)
+(generic_function
+  function: (scoped_identifier
+    name: (identifier) @function))
+(generic_function
+  function: (field_expression
+    field: (field_identifier) @function.method))
+
+(macro_invocation
+  macro: (identifier) @function.macro
+  "!" @function.macro)
+
+; Function definitions
+
+(function_item (identifier) @function)
+(function_signature_item (identifier) @function)
+
+(line_comment) @comment
+(block_comment) @comment
+
+(line_comment (doc_comment)) @comment.documentation
+(block_comment (doc_comment)) @comment.documentation
+
+"(" @punctuation.bracket
+")" @punctuation.bracket
+"[" @punctuation.bracket
+"]" @punctuation.bracket
+"{" @punctuation.bracket
+"}" @punctuation.bracket
+
+(type_arguments
+  "<" @punctuation.bracket
+  ">" @punctuation.bracket)
+(type_parameters
+  "<" @punctuation.bracket
+  ">" @punctuation.bracket)
+
+"::" @punctuation.delimiter
+":" @punctuation.delimiter
+"." @punctuation.delimiter
+"," @punctuation.delimiter
+";" @punctuation.delimiter
+
+(parameter (identifier) @variable.parameter)
+
+(lifetime (identifier) @label)
+
+"as" @keyword
+"async" @keyword
+"await" @keyword
+"break" @keyword
+"const" @keyword
+"continue" @keyword
+"default" @keyword
+"dyn" @keyword
+"else" @keyword
+"enum" @keyword
+"extern" @keyword
+"fn" @keyword
+"for" @keyword
+"gen" @keyword
+"if" @keyword
+"impl" @keyword
+"in" @keyword
+"let" @keyword
+"loop" @keyword
+"macro_rules!" @keyword
+"match" @keyword
+"mod" @keyword
+"move" @keyword
+"pub" @keyword
+"raw" @keyword
+"ref" @keyword
+"return" @keyword
+"static" @keyword
+"struct" @keyword
+"trait" @keyword
+"type" @keyword
+"union" @keyword
+"unsafe" @keyword
+"use" @keyword
+"where" @keyword
+"while" @keyword
+"yield" @keyword
+(crate) @keyword
+(mutable_specifier) @keyword
+(use_list (self) @keyword)
+(scoped_use_list (self) @keyword)
+(scoped_identifier (self) @keyword)
+(super) @keyword
+
+(self) @variable.builtin
+
+(char_literal) @string
+(string_literal) @string
+(raw_string_literal) @string
+
+(boolean_literal) @constant.builtin
+(integer_literal) @constant.builtin
+(float_literal) @constant.builtin
+
+(escape_sequence) @escape
+
+(attribute_item) @attribute
+(inner_attribute_item) @attribute
+
+"*" @operator
+"&" @operator
+"'" @operator

references/tags-queries/rust_injections_query.scm

@@ -0,0 +1,9 @@
+((macro_invocation
+  (token_tree) @injection.content)
+ (#set! injection.language "rust")
+ (#set! injection.include-children))
+
+((macro_rule
+  (token_tree) @injection.content)
+ (#set! injection.language "rust")
+ (#set! injection.include-children))

references/tags-queries/rust_tags_query.scm

@@ -0,0 +1,60 @@
+; ADT definitions
+
+(struct_item
+    name: (type_identifier) @name) @definition.class
+
+(enum_item
+    name: (type_identifier) @name) @definition.class
+
+(union_item
+    name: (type_identifier) @name) @definition.class
+
+; type aliases
+
+(type_item
+    name: (type_identifier) @name) @definition.class
+
+; method definitions
+
+(declaration_list
+    (function_item
+        name: (identifier) @name) @definition.method)
+
+; function definitions
+
+(function_item
+    name: (identifier) @name) @definition.function
+
+; trait definitions
+(trait_item
+    name: (type_identifier) @name) @definition.interface
+
+; module definitions
+(mod_item
+    name: (identifier) @name) @definition.module
+
+; macro definitions
+
+(macro_definition
+    name: (identifier) @name) @definition.macro
+
+; references
+
+(call_expression
+    function: (identifier) @name) @reference.call
+
+(call_expression
+    function: (field_expression
+        field: (field_identifier) @name)) @reference.call
+
+(macro_invocation
+    macro: (identifier) @name) @reference.call
+
+; implementations
+
+(impl_item
+    trait: (type_identifier) @name) @reference.implementation
+
+(impl_item
+    type: (type_identifier) @name
+    !trait) @reference.implementation

references/tags-queries/typescript_highlights_query.scm

@@ -0,0 +1,35 @@
+; Types
+
+(type_identifier) @type
+(predefined_type) @type.builtin
+
+((identifier) @type
+ (#match? @type "^[A-Z]"))
+
+(type_arguments
+  "<" @punctuation.bracket
+  ">" @punctuation.bracket)
+
+; Variables
+
+(required_parameter (identifier) @variable.parameter)
+(optional_parameter (identifier) @variable.parameter)
+
+; Keywords
+
+[ "abstract"
+  "declare"
+  "enum"
+  "export"
+  "implements"
+  "interface"
+  "keyof"
+  "namespace"
+  "private"
+  "protected"
+  "public"
+  "type"
+  "readonly"
+  "override"
+  "satisfies"
+] @keyword

references/tags-queries/typescript_tags_query.scm

@@ -0,0 +1,23 @@
+(function_signature
+  name: (identifier) @name) @definition.function
+
+(method_signature
+  name: (property_identifier) @name) @definition.method
+
+(abstract_method_signature
+  name: (property_identifier) @name) @definition.method
+
+(abstract_class_declaration
+  name: (type_identifier) @name) @definition.class
+
+(module
+  name: (identifier) @name) @definition.module
+
+(interface_declaration
+  name: (type_identifier) @name) @definition.interface
+
+(type_annotation
+  (type_identifier) @name) @reference.type
+
+(new_expression
+  constructor: (identifier) @name) @reference.class

references/tags-queries/yaml_highlights_query.scm

@@ -0,0 +1,79 @@
+(boolean_scalar) @boolean
+
+(null_scalar) @constant.builtin
+
+[
+  (double_quote_scalar)
+  (single_quote_scalar)
+  (block_scalar)
+  (string_scalar)
+] @string
+
+[
+  (integer_scalar)
+  (float_scalar)
+] @number
+
+(comment) @comment
+
+[
+  (anchor_name)
+  (alias_name)
+] @label
+
+(tag) @type
+
+[
+  (yaml_directive)
+  (tag_directive)
+  (reserved_directive)
+] @attribute
+
+(block_mapping_pair
+  key: (flow_node
+    [
+      (double_quote_scalar)
+      (single_quote_scalar)
+    ] @property))
+
+(block_mapping_pair
+  key: (flow_node
+    (plain_scalar
+      (string_scalar) @property)))
+
+(flow_mapping
+  (_
+    key: (flow_node
+      [
+        (double_quote_scalar)
+        (single_quote_scalar)
+      ] @property)))
+
+(flow_mapping
+  (_
+    key: (flow_node
+      (plain_scalar
+        (string_scalar) @property))))
+
+[
+  ","
+  "-"
+  ":"
+  ">"
+  "?"
+  "|"
+] @punctuation.delimiter
+
+[
+  "["
+  "]"
+  "{"
+  "}"
+] @punctuation.bracket
+
+[
+  "*"
+  "&"
+  "---"
+  "..."
+] @punctuation.special

tests/test_analyze.py

@@ -0,0 +1,299 @@
+"""Tests for tag-preprocessor orchestrator (analyze.py)."""
+
+from pathlib import Path, PurePath
+import tempfile
+import sys
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from bex.tag_preprocessor.analyze import (
+    scan_directory, frequency_filter, infer, analyze_directory, _match_glob,
+    _file_to_package, _top_packages, _extract_imports,
+)
+from bex.tag_preprocessor.code import _summarize_arg_info, _classify_arg_node
+
+
+def test_scan_directory_empty():
+    with tempfile.TemporaryDirectory() as td:
+        result = scan_directory(td)
+        assert result == {}
+    print("  PASS test_scan_directory_empty")
+
+
+def test_scan_directory_groups_by_extension():
+    with tempfile.TemporaryDirectory() as td:
+        d = Path(td)
+        (d / "a.py").write_text("x = 1")
+        (d / "b.py").write_text("y = 2")
+        (d / "c.js").write_text("let x = 1;")
+        (d / "d.rs").write_text("fn main() {}")
+        result = scan_directory(td)
+        assert ".py" in result
+        assert ".js" in result
+        assert ".rs" in result
+        assert len(result[".py"]) == 2
+        assert len(result[".js"]) == 1
+        assert len(result[".rs"]) == 1
+    print("  PASS test_scan_directory_groups_by_extension")
+
+
+def test_scan_directory_skips_unsupported():
+    with tempfile.TemporaryDirectory() as td:
+        d = Path(td)
+        (d / "f.py").write_text("x = 1")
+        (d / "f.txt").write_text("hello")
+        (d / "f.md").write_text("# doc")
+        result = scan_directory(td)
+        assert ".py" in result
+        assert ".txt" not in result
+        assert ".md" not in result
+    print("  PASS test_scan_directory_skips_unsupported")
+
+
+def test_scan_directory_nested():
+    with tempfile.TemporaryDirectory() as td:
+        d = Path(td)
+        (d / "sub").mkdir()
+        (d / "sub" / "a.py").write_text("x = 1")
+        (d / "b.py").write_text("y = 2")
+        result = scan_directory(td)
+        assert len(result[".py"]) == 2
+    print("  PASS test_scan_directory_nested")
+
+
+def test_scan_directory_skips_build_dirs():
+    with tempfile.TemporaryDirectory() as td:
+        d = Path(td)
+        (d / "src" / "main").mkdir(parents=True)
+        (d / "build" / "reports").mkdir(parents=True)
+        (d / "node_modules" / "pkg").mkdir(parents=True)
+        (d / "src" / "main" / "app.py").write_text("x = 1")
+        (d / "build" / "reports" / "report.js").write_text("let x = 1;")
+        (d / "node_modules" / "pkg" / "index.js").write_text("let y = 2;")
+        result = scan_directory(td)
+        assert ".py" in result
+        assert ".js" not in result
+        assert len(result[".py"]) == 1
+    print("  PASS test_scan_directory_skips_build_dirs")
+
+
+def test_frequency_filter_nothing_to_filter():
+    sequences = [
+        [("function", "foo", 1), ("keyword.return", "return", 2)],
+        [("function", "bar", 1), ("keyword.return", "return", 2)],
+    ]
+    filtered = frequency_filter(sequences, min_coverage=0.5)
+    assert len(filtered) == 2
+    assert filtered == sequences
+    print("  PASS test_frequency_filter_nothing_to_filter")
+
+
+def test_frequency_filter_removes_infrequent_symbol():
+    sequences = [
+        [("keyword.return", "return", 1)],
+        [("keyword.return", "return", 1)],
+        [("function", "rare_fn", 1)],
+    ]
+    filtered = frequency_filter(sequences, min_coverage=0.67)
+    assert len(filtered) == 3
+    assert len(filtered[0]) == 1  # "return" kept
+    assert len(filtered[1]) == 1  # "return" kept
+    assert len(filtered[2]) == 0  # "rare_fn" removed
+    print("  PASS test_frequency_filter_removes_infrequent_symbol")
+
+
+def test_frequency_filter_edge_empty_sequences():
+    assert frequency_filter([], min_coverage=0.5) == []
+    assert frequency_filter([[], []], min_coverage=0.5) == [[], []]
+    print("  PASS test_frequency_filter_edge_empty_sequences")
+
+
+def test_infer_returns_ensemble_dict():
+    with tempfile.TemporaryDirectory() as td:
+        d = Path(td)
+        (d / "a.py").write_text("def foo():\n    return 1")
+        (d / "b.py").write_text("def bar():\n    return 2")
+        result = infer([str(d / "a.py"), str(d / "b.py")], ".py", min_coverage=0.5)
+        assert isinstance(result, dict)
+        assert "best" in result
+        assert "all" in result
+        assert "why" in result
+    print("  PASS test_infer_returns_ensemble_dict")
+
+
+def test_match_glob():
+    assert _match_glob("/repo/src/main/app.kt", "**/src/main/**")
+    assert _match_glob("/repo/src/main/org/app.kt", "**/src/main/**")
+    assert _match_glob("/repo/src/main/deep/nested/app.kt", "**/src/main/**")
+    assert not _match_glob("/repo/src/test/app.kt", "**/src/main/**")
+    assert not _match_glob("/repo/build/app.kt", "**/src/main/**")
+    print("  PASS test_match_glob")
+
+
+def test_analyze_directory_include_glob():
+    with tempfile.TemporaryDirectory() as td:
+        d = Path(td)
+        (d / "src" / "main").mkdir(parents=True)
+        (d / "src" / "test").mkdir(parents=True)
+        (d / "src" / "main" / "prod.py").write_text("def setup():\n    pass\ndef run():\n    return x")
+        (d / "src" / "test" / "test_prod.py").write_text("def test_run():\n    assert run() == x")
+        results = analyze_directory(td, include="**/src/main/**")
+        assert ".py" in results
+        assert len(results[".py"]) >= 1
+        for label, r, count, meta in results[".py"]:
+            if r and r.get("best"):
+                assert r["best"]["grammar"] is not None
+    print("  PASS test_analyze_directory_include_glob")
+
+
+def test_infer_low_coverage_filters_noise():
+    with tempfile.TemporaryDirectory() as td:
+        d = Path(td)
+        (d / "common.py").write_text(
+            "def setup():\n    pass\ndef teardown():\n    pass"
+        )
+        (d / "rare.py").write_text(
+            "def setup():\n    pass\ndef one_off():\n    raise Exception('boom')"
+        )
+        result = infer(
+            [str(d / "common.py"), str(d / "rare.py")], ".py", min_coverage=0.6
+        )
+        assert result["best"] is not None
+        assert result["best"]["grammar"] is not None
+    print("  PASS test_infer_low_coverage_filters_noise")
+
+
+def test_file_to_package():
+    assert _file_to_package(
+        "/repo/src/main/kotlin/com/example/app/Foo.kt", ".kt"
+    ) == "com/example/app"
+    assert _file_to_package(
+        "/repo/src/test/java/com/example/FooTest.java", ".java"
+    ) == "com/example"
+    assert _file_to_package(
+        "/repo/src/main/python/mypackage/module.py", ".py"
+    ) == "mypackage"
+    assert _file_to_package("/repo/lib/foo.py", ".py") == "lib"
+    print("  PASS test_file_to_package")
+
+
+def test_top_packages():
+    fps = {
+        "/repo/src/main/kotlin/com/example/a/Foo.kt",
+        "/repo/src/main/kotlin/com/example/a/Bar.kt",
+        "/repo/src/main/kotlin/com/example/b/Baz.kt",
+    }
+    pkgs = _top_packages(fps, ".kt")
+    assert pkgs[0] == "com/example/a"
+    assert pkgs[1] == "com/example/b"
+    print("  PASS test_top_packages")
+
+
+def test_extract_imports():
+    with tempfile.TemporaryDirectory() as td:
+        d = Path(td)
+        (d / "a.kt").write_text(
+            "package com.example\nimport io.mockk.every\nimport io.mockk.verify\n\nclass Foo"
+        )
+        (d / "b.kt").write_text(
+            "package com.example\nimport org.junit.Test\n\nclass Bar"
+        )
+        result = _extract_imports({str(d / "a.kt"), str(d / "b.kt")})
+        assert "import io.mockk.every" in result
+        assert "import io.mockk.verify" in result
+        assert "import org.junit.Test" in result
+        assert len(result) == 3
+    print("  PASS test_extract_imports")
+
+
+def test_extract_imports_empty():
+    assert _extract_imports(set()) == []
+    print("  PASS test_extract_imports_empty")
+
+
+def test_extract_imports_no_imports():
+    with tempfile.TemporaryDirectory() as td:
+        d = Path(td)
+        (d / "a.py").write_text("x = 1\ny = 2")
+        result = _extract_imports({str(d / "a.py")})
+        assert result == []
+    print("  PASS test_extract_imports_no_imports")
+
+
+def test_extract_arg_info_python():
+    with tempfile.TemporaryDirectory() as td:
+        d = Path(td)
+        (d / "a.py").write_text(
+            "def test_foo():\n"
+            "    result = compute(42)\n"
+            "    assert result == 0\n"
+            "    run(mock, times=1)\n"
+        )
+        from bex.tag_preprocessor.code import extract_arg_info
+        fp = str(d / "a.py")
+        code = Path(fp).read_text()
+        info = extract_arg_info(fp, code)
+        # compute(42) -> 1 arg: literal
+        assert "compute" in info
+        obs = info["compute"]
+        assert any(n == 1 and ts == ("lit",) for n, ts in obs)
+        # run(mock, times=1) -> 2 args: var + kwarg
+        assert "run" in info
+        obs2 = info["run"]
+        assert any(n == 2 for n, ts in obs2)
+        assert any(ts == ("var", "kwarg") for n, ts in obs2)
+    print("  PASS test_extract_arg_info_python")
+
+
+def test_summarize_arg_info():
+    info = {
+        "foo": [(2, ("var", "lit")), (2, ("var", "lit")), (3, ("var", "lit", "lit"))],
+        "bar": [(0, ()), (1, ("lambda",))],
+    }
+    summary = _summarize_arg_info(info)
+    assert summary["foo"]["occurrences"] == 3
+    assert summary["foo"]["arg_count"]["min"] == 2
+    assert summary["foo"]["arg_count"]["max"] == 3
+    assert summary["foo"]["arg_count"]["common"] == 2
+    assert len(summary["foo"]["patterns"]) == 2
+    assert summary["bar"]["occurrences"] == 2
+    assert summary["bar"]["arg_count"]["min"] == 0
+    print("  PASS test_summarize_arg_info")
+
+
+def run_all():
+    tests = [
+        test_scan_directory_empty,
+        test_scan_directory_groups_by_extension,
+        test_scan_directory_skips_unsupported,
+        test_scan_directory_nested,
+        test_scan_directory_skips_build_dirs,
+        test_match_glob,
+        test_analyze_directory_include_glob,
+        test_frequency_filter_nothing_to_filter,
+        test_frequency_filter_removes_infrequent_symbol,
+        test_frequency_filter_edge_empty_sequences,
+        test_infer_returns_ensemble_dict,
+        test_infer_low_coverage_filters_noise,
+        test_file_to_package,
+        test_top_packages,
+        test_extract_imports,
+        test_extract_imports_empty,
+        test_extract_imports_no_imports,
+        test_extract_arg_info_python,
+        test_summarize_arg_info,
+    ]
+    passed = 0
+    failed = 0
+    for t in tests:
+        try:
+            t()
+            passed += 1
+        except Exception as e:
+            print(f"  FAIL {t.__name__}: {e}")
+            failed += 1
+    print(f"\n{passed} passed, {failed} failed")
+
+
+if __name__ == "__main__":
+    run_all()