diff --git a/bex/tag_preprocessor/analyze.py b/bex/tag_preprocessor/analyze.py index 0c12744..0326896 100644 --- a/bex/tag_preprocessor/analyze.py +++ b/bex/tag_preprocessor/analyze.py @@ -220,12 +220,17 @@ def frequency_filter(sequences, min_coverage=0.2): return filtered +_CALL_ONLY_EXTS = {".ts", ".tsx", ".js", ".jsx", ".mts", ".mjs"} + + def _preprocess_file(fp): """Preprocess one file. Module-level for ProcessPoolExecutor.""" + ext = os.path.splitext(fp)[1].lower() + call_only = ext in _CALL_ONLY_EXTS with open(fp) as f: code = f.read() sequences = [] - for method_seq in preprocess_by_method(fp, code): + for method_seq in preprocess_by_method(fp, code, call_only=call_only): if method_seq: sequences.append(method_seq) return (fp, sequences) diff --git a/bex/tag_preprocessor/code.py b/bex/tag_preprocessor/code.py index 45d1b69..9ddb598 100644 --- a/bex/tag_preprocessor/code.py +++ b/bex/tag_preprocessor/code.py @@ -359,11 +359,16 @@ def _find_method_bodies(tree): return bodies -def preprocess_by_method(file_path: str, code: str): +def preprocess_by_method(file_path: str, code: str, call_only=False): """Preprocess and group behavioral tokens by enclosing method body. Returns list of sequences, one per function/method found. Each sequence is [(capture_name, text, line_number), ...]. + + Args: + call_only: When True, only keep CALL_PREFIXES captures (function + calls, method calls). Use for JS/TS where keyword captures + produce truncated text due to tree-sitter node boundary issues. """ ext = os.path.splitext(file_path)[1].lower() lang, query_name = _load_grammar(ext) @@ -377,9 +382,10 @@ def preprocess_by_method(file_path: str, code: str): cursor = QueryCursor(query) captures = cursor.captures(tree.root_node) + prefix_filter = CALL_PREFIXES if call_only else BEHAVIORAL_PREFIXES items = [] for capname, nodes in captures.items(): - if not capname.startswith(BEHAVIORAL_PREFIXES): + if not capname.startswith(prefix_filter): continue for node in nodes: text = sanitize_symbol(code[node.start_byte:node.end_byte], capname) @@ -403,7 +409,7 @@ def preprocess_by_method(file_path: str, code: str): return sequences -def preprocess(file_path: str, code: str): +def preprocess(file_path: str, code: str, call_only=False): ext = os.path.splitext(file_path)[1].lower() lang, query_name = _load_grammar(ext) query_src = _load_query(query_name) @@ -420,9 +426,10 @@ def preprocess(file_path: str, code: str): cursor = QueryCursor(query) captures = cursor.captures(tree.root_node) + prefix_filter = CALL_PREFIXES if call_only else BEHAVIORAL_PREFIXES items = [] for capname, nodes in captures.items(): - if not capname.startswith(BEHAVIORAL_PREFIXES): + if not capname.startswith(prefix_filter): continue for node in nodes: text = sanitize_symbol(code[node.start_byte:node.end_byte], capname)