bex/tag_preprocessor/analyze.py

@@ -220,17 +220,12 @@ def frequency_filter(sequences, min_coverage=0.2):
     return filtered
 
 
-_CALL_ONLY_EXTS = {".ts", ".tsx", ".js", ".jsx", ".mts", ".mjs"}
-
-
 def _preprocess_file(fp):
     """Preprocess one file. Module-level for ProcessPoolExecutor."""
-    ext = os.path.splitext(fp)[1].lower()
-    call_only = ext in _CALL_ONLY_EXTS
     with open(fp) as f:
         code = f.read()
     sequences = []
-    for method_seq in preprocess_by_method(fp, code, call_only=call_only):
+    for method_seq in preprocess_by_method(fp, code):
         if method_seq:
             sequences.append(method_seq)
     return (fp, sequences)

bex/tag_preprocessor/code.py

@@ -13,51 +13,8 @@ import re
 import sys
 from pathlib import Path
 
-# Pattern to extract the first identifier from a call expression.
-# Matches: "foo", "Foo.bar", "foo.Bar.baz" — stops at first ( or whitespace.
-_CALL_NAME_RE = re.compile(r"^([A-Za-z_][\w.]*)")
 
 
-# SORE special characters that break grammar parsing
-_SORE_SPECIAL = set("()+?*[]{}|\\^$.")
-
-
-def sanitize_symbol(text, capname):
-    """Sanitize extracted symbol text for grammar inference.
-
-    Multi-line expressions (constructor calls, if-blocks) break CRX and
-    produce malformed SOREs. This function:
-    - For call-like captures: extracts just the function/method name
-    - For others: extracts the first identifier-like token
-    - Strips all SORE special characters
-    - Falls back to empty string if nothing usable remains
-    """
-    text = text.strip()
-    if not text:
-        return text
-
-    # Collapse whitespace first
-    text = re.sub(r"\s+", " ", text)
-
-    # Extract first identifier (handles "foo", "Foo.bar", "foo.Bar.baz")
-    m = _CALL_NAME_RE.match(text)
-    if m:
-        name = m.group(1).rstrip(".")
-        if name:
-            return name
-
-    # Fallback: take first line, strip special chars
-    text = text.split("\n", 1)[0].strip()
-    # Remove any SORE special characters
-    cleaned = "".join(c for c in text if c not in _SORE_SPECIAL)
-    cleaned = cleaned.strip()
-    if len(cleaned) > 80:
-        cleaned = cleaned[:80]
-    # Skip single-char fragments (likely truncated identifiers)
-    if len(cleaned) <= 1:
-        return ""
-    return cleaned
-
 from tree_sitter import Language, Parser, Query, QueryCursor
 
 QUERIES_DIR = Path(__file__).parent / "queries"
@@ -224,8 +181,9 @@ def extract_arg_info(file_path, code):
     lang, query_name = _load_grammar(ext)
     query_src = _load_query(query_name)
 
+    code_bytes = code.encode()
     parser = Parser(lang)
-    tree = parser.parse(code.encode())
+    tree = parser.parse(code_bytes)
     query = Query(lang, query_src)
 
     cursor = QueryCursor(query)
@@ -236,7 +194,7 @@ def extract_arg_info(file_path, code):
         if not capname.startswith(BEHAVIORAL_PREFIXES):
             continue
         for node in nodes:
-            text = sanitize_symbol(code[node.start_byte:node.end_byte], capname)
+            text = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
             parent = node.parent
             if not parent:
                 continue
@@ -359,37 +317,32 @@ def _find_method_bodies(tree):
     return bodies
 
 
-def preprocess_by_method(file_path: str, code: str, call_only=False):
+def preprocess_by_method(file_path: str, code: str):
     """Preprocess and group behavioral tokens by enclosing method body.
 
     Returns list of sequences, one per function/method found.
     Each sequence is [(capture_name, text, line_number), ...].
-
-    Args:
-        call_only: When True, only keep CALL_PREFIXES captures (function
-            calls, method calls). Use for JS/TS where keyword captures
-            produce truncated text due to tree-sitter node boundary issues.
     """
     ext = os.path.splitext(file_path)[1].lower()
     lang, query_name = _load_grammar(ext)
     query_src = _load_query(query_name)
 
+    code_bytes = code.encode()
     parser = Parser(lang)
-    tree = parser.parse(code.encode())
+    tree = parser.parse(code_bytes)
 
     query = Query(lang, query_src)
 
     cursor = QueryCursor(query)
     captures = cursor.captures(tree.root_node)
 
-    prefix_filter = CALL_PREFIXES if call_only else BEHAVIORAL_PREFIXES
     items = []
     for capname, nodes in captures.items():
-        if not capname.startswith(prefix_filter):
+        if not capname.startswith(BEHAVIORAL_PREFIXES):
             continue
         for node in nodes:
-            text = sanitize_symbol(code[node.start_byte:node.end_byte], capname)
-            if text:  # skip empty/fragment symbols
+            text = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
+            if text:
                 items.append((node.start_byte, capname, node, text))
 
     items.sort(key=lambda x: x[0])
@@ -402,20 +355,21 @@ def preprocess_by_method(file_path: str, code: str, call_only=False):
         seq = []
         for start, capname, node, text in items:
             if body_node.start_byte <= start < body_node.end_byte:
-                seq.append((capname, text, code[:start].count("\n") + 1))
+                seq.append((capname, text, code_bytes[:start].count(b"\n") + 1))
         if seq:
             sequences.append(seq)
 
     return sequences
 
 
-def preprocess(file_path: str, code: str, call_only=False):
+def preprocess(file_path: str, code: str):
     ext = os.path.splitext(file_path)[1].lower()
     lang, query_name = _load_grammar(ext)
     query_src = _load_query(query_name)
 
+    code_bytes = code.encode()
     parser = Parser(lang)
-    tree = parser.parse(code.encode())
+    tree = parser.parse(code_bytes)
 
     try:
         query = Query(lang, query_src)
@@ -426,19 +380,18 @@ def preprocess(file_path: str, code: str, call_only=False):
     cursor = QueryCursor(query)
     captures = cursor.captures(tree.root_node)
 
-    prefix_filter = CALL_PREFIXES if call_only else BEHAVIORAL_PREFIXES
     items = []
     for capname, nodes in captures.items():
-        if not capname.startswith(prefix_filter):
+        if not capname.startswith(BEHAVIORAL_PREFIXES):
             continue
         for node in nodes:
-            text = sanitize_symbol(code[node.start_byte:node.end_byte], capname)
-            if text:  # skip empty/fragment symbols
+            text = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
+            if text:
                 items.append((node.start_byte, capname, node, text))
 
     items.sort(key=lambda x: x[0])
 
-    return [(capname, text, code[:start].count("\n") + 1) for start, capname, _, text in items]
+    return [(capname, text, code_bytes[:start].count(b"\n") + 1) for start, capname, _, text in items]
 
 
 def main():

experiments/EXPERIMENT_LOG.md

@@ -552,3 +552,72 @@ useful here — recursive split already does the separation work.
 
 **Decision:** Recursive split is the right default. `_recursive_split()` replaces
 single-level `_split_by_first_symbol()` when `split_mixed=True`.
+
+---
+
+## Round 14: Byte/Char Offset Fix — Tree-sitter Text Extraction
+
+**Hypothesis:** Tree-sitter returns byte offsets, but we indexed into Python strings
+(character offsets). Non-ASCII characters create cumulative drift, truncating symbols.
+
+**Root cause:** `parser.parse(code.encode())` feeds bytes → tree-sitter returns
+byte offsets. `code[node.start_byte:node.end_byte]` indexes a **string** with
+**byte** offsets. Every multi-byte char shifts the index further.
+
+**Example (Kotlin):**
+```python
+# Before fix: byte offset 498 in string "override fun ddCheck(..." 
+# "ddCheck" at chars 498–505, but byte 498 lands at 'd' only after accounting
+# for emoji/non-ASCII in earlier strings. Result: truncated to "ddCheck" (OK here)
+# but in practice: "ddCheck" → "ddCheck" ( lucky), "ddCheck" → "ddChe" ( unlucky)
+```
+
+**Evidence:**
+- Zod (TypeScript): 653 truncated symbols → **0 after fix**
+- RAGSAK (Kotlin): 653 truncated symbols → **0 after fix**
+
+**Method:** Store `code_bytes = code.encode()`, index into that, decode only final
+text. Applied to `preprocess_by_method()`, `preprocess()`, and `extract_arg_info()`.
+
+**Results (all codebases, `--slice package --min-coverage 0.05 --min-methods 3 --split-mixed`):**
+
+| Codebase | Language | Files | Before (kept) | After (kept) | Malformed | Too diverse |
+|----------|----------|-------|---------------|--------------|-----------|-------------|
+| RAGSAK   | Kotlin   | 462   | 8             | **27**       | 2         | 95          |
+| Zod      | TypeScript| 372  | —             | **10**       | 1         | 1           |
+| Flask    | Python   | 83    | 2             | **5**        | 0         | —           |
+| FastAPI  | Python   | 1129  | 33            | **111**      | 0         | —           |
+
+**RAGSAK (27 kept grammars):**
+- Clean symbol names: `ToolInvocationRequest`, `ToolingRequest`, `ToolInvocationResult`
+- Previous: `ToolingReq`, `ToolInvocationRe` (truncated)
+- Score range: 3–3.97e+100 (wide spread, some very large from CRX over-approximation)
+
+**Zod (10 kept grammars):**
+- `safeParse.(expect+toEqual)+?.as+?.z?` (score 41)
+- `return.i+?.new?.test?.Date+?.DATA?` (score 48)
+- `(Object+typeof)+.map?.(Error+Promise+Set+any+as+def+for+if+key+new+of+return+then+throw+util)+?` (score 282393216)
+- Truncation gone: previous `ddCheck`, `arse`, `rty` → now full names
+
+**FastAPI (111 kept grammars):**
+- Largest jump: 33 → 111 (3.4× more grammars)
+- Many test files now produce grammars: `(test_create_existing_item+test_create_item+...)` patterns
+
+**Malformed grammars:**
+- RAGSAK: 2 (down from 6 in Round 12)
+- Zod: 1 (`src/v4/locales` — 214 methods, too diverse)
+- FastAPI/Flask: 0
+
+**Chain of thought:**
+1. Symbols were truncated → CRX saw incomplete names → merged wrong things or produced flat bags
+2. With clean symbols, CRX can distinguish `ddCheck` from `ddChecker` → tighter grammars
+3. More methods survive the `min_coverage` filter → more groups produce grammars
+4. The 3.4× jump in FastAPI confirms: truncation was the primary bottleneck, not the algorithm
+
+**Verdict:** This was the single highest-impact fix in the pipeline. Tree-sitter
+byte/char mismatch was silently corrupting every symbol extraction. The
+`sanitize_symbol()` function (removed in this round) was a band-aid that hid
+the real problem. Clean symbols → clean grammars.
+
+**Decision:** Keep `code_bytes = code.encode()` pattern permanently. Remove
+`sanitize_symbol()` (dead code). Remove `call_only` parameter (no longer needed).