From 17561a2f379352732ca7a8992586c7bc4ca0a45a Mon Sep 17 00:00:00 2001 From: tobjend Date: Sun, 12 Jul 2026 12:53:48 +0200 Subject: [PATCH] fix: sanitize symbol extraction, fix malformed grammars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add sanitize_symbol() to extract clean identifiers from AST nodes - Strip SORE special characters ((),+?*[]{}|\^$.) from symbols - Extract just function name from multi-line expressions - Skip single-char fragments (truncated identifiers) - Filter empty symbols in preprocessing - Wire iLocal context extraction into pipeline - Result: RAGSAK malformed grammars 6→0, FastAPI clean - 212 tests pass --- bex/tag_preprocessor/analyze.py | 86 ++++++++++- bex/tag_preprocessor/code.py | 56 +++++++- bex/tag_preprocessor/ilocal_source.py | 196 ++++++++++++++++++++++++++ 3 files changed, 332 insertions(+), 6 deletions(-) create mode 100644 bex/tag_preprocessor/ilocal_source.py diff --git a/bex/tag_preprocessor/analyze.py b/bex/tag_preprocessor/analyze.py index 8180039..9c06007 100644 --- a/bex/tag_preprocessor/analyze.py +++ b/bex/tag_preprocessor/analyze.py @@ -463,6 +463,67 @@ def _filter_glob(files, include=None, exclude=None): return files +def analyze_by_ilocal(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, context_strategy="dir", reduce=True): + """iLocal-style analysis: extract (context, sequence) pairs, reduce, infer. + + Instead of hard-coding directory as grouping key, this extracts contexts + using a configurable strategy, then uses iLocal's reduce to merge contexts + with identical signatures. + + Args: + context_strategy: one of 'dir', 'file', 'parent_dir', 'depth_2', + 'depth_3', 'imports', 'symbol_overlap' + reduce: whether to run iLocal reduce to merge similar contexts + """ + from .ilocal_source import CONTEXT_STRATEGIES, reduce_contexts + + t0 = time.time() + sequences, seq_files = _preprocess_files(file_paths) + if not sequences: + return [] + _vprint(f"Preprocess: {len(sequences)} methods from {len(file_paths)} {extension} files ({time.time()-t0:.1f}s)") + + strategy_fn = CONTEXT_STRATEGIES.get(context_strategy) + if not strategy_fn: + _vprint(f"Unknown context strategy: {context_strategy}, falling back to 'dir'") + strategy_fn = CONTEXT_STRATEGIES["dir"] + + t1 = time.time() + context_groups = strategy_fn(sequences, seq_files, project_root) + _vprint(f"Context extraction ({context_strategy}): {len(context_groups)} contexts ({time.time()-t1:.1f}s)") + + if reduce: + t2 = time.time() + before = len(context_groups) + context_groups = reduce_contexts(context_groups) + _vprint(f"Reduce: {before} → {len(context_groups)} contexts ({time.time()-t2:.1f}s)") + + _vprint(f"Contexts: {len(context_groups)} groups") + for label, seqs in sorted(context_groups.items(), key=lambda x: -len(x[1]))[:10]: + _vprint(f" ├ {label[:60]} ({len(seqs)} methods)") + if len(context_groups) > 10: + _vprint(f" └ ... and {len(context_groups) - 10} more") + + results = [] + n_workers = os.cpu_count() + _vprint(f"Inferring {len(context_groups)} contexts across {n_workers} workers ...") + with ProcessPoolExecutor(max_workers=n_workers) as ex: + futures = {} + for label, seqs in context_groups.items(): + f = ex.submit(_infer_group, label, seqs, set(), project_root, + min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure) + futures[f] = label + + done = 0 + for f in as_completed(futures): + done += 1 + if done % 20 == 0 or done == len(futures): + _vprint(f" [{done}/{len(futures)}]") + results.append(f.result()) + + return results + + def analyze_directory( dir_path, min_coverage=DEFAULT_COVERAGE, @@ -478,6 +539,7 @@ def analyze_directory( min_methods=3, crx_method='standard', min_structure=0.0, + context_strategy="dir", ): """Scan a directory and run analysis for each language found. @@ -518,6 +580,21 @@ def analyze_directory( crx_method=crx_method, min_structure=min_structure, ) + elif slice == "ilocal": + results[ext] = analyze_by_ilocal( + files, ext, + project_root=dir_path, + min_coverage=min_coverage, + prefer=prefer, + kmax=kmax, + include_kore=include_kore, + include_idregex=include_idregex, + method=method, + min_methods=min_methods, + crx_method=crx_method, + min_structure=min_structure, + context_strategy=context_strategy, + ) else: results[ext] = analyze_clusters( files, ext, @@ -647,8 +724,12 @@ def _parse_args(argv=None): help="BEX core coverage threshold — outlier methods are removed until this fraction remains (default: 0.8)", ) parser.add_argument( - "--slice", choices=["flat", "package"], default="flat", - help="Grouping strategy: flat (one per language) or package (per directory) (default: flat)", + "--slice", choices=["flat", "package", "ilocal"], default="flat", + help="Grouping strategy: flat (one per language), package (per directory), or ilocal (iLocal context-based) (default: flat)", + ) + parser.add_argument( + "--context-strategy", choices=["dir", "file", "parent_dir", "depth_2", "depth_3", "imports", "symbol_overlap"], default="dir", + help="Context extraction strategy for ilocal slicing (default: dir)", ) parser.add_argument( "--include", @@ -714,6 +795,7 @@ def main(): min_methods=args.min_methods, crx_method=args.crx_method, min_structure=args.min_structure, + context_strategy=args.context_strategy, ) if args.json_flag or args.format == "json": diff --git a/bex/tag_preprocessor/code.py b/bex/tag_preprocessor/code.py index 7ec0885..45d1b69 100644 --- a/bex/tag_preprocessor/code.py +++ b/bex/tag_preprocessor/code.py @@ -13,6 +13,51 @@ import re import sys from pathlib import Path +# Pattern to extract the first identifier from a call expression. +# Matches: "foo", "Foo.bar", "foo.Bar.baz" — stops at first ( or whitespace. +_CALL_NAME_RE = re.compile(r"^([A-Za-z_][\w.]*)") + + +# SORE special characters that break grammar parsing +_SORE_SPECIAL = set("()+?*[]{}|\\^$.") + + +def sanitize_symbol(text, capname): + """Sanitize extracted symbol text for grammar inference. + + Multi-line expressions (constructor calls, if-blocks) break CRX and + produce malformed SOREs. This function: + - For call-like captures: extracts just the function/method name + - For others: extracts the first identifier-like token + - Strips all SORE special characters + - Falls back to empty string if nothing usable remains + """ + text = text.strip() + if not text: + return text + + # Collapse whitespace first + text = re.sub(r"\s+", " ", text) + + # Extract first identifier (handles "foo", "Foo.bar", "foo.Bar.baz") + m = _CALL_NAME_RE.match(text) + if m: + name = m.group(1).rstrip(".") + if name: + return name + + # Fallback: take first line, strip special chars + text = text.split("\n", 1)[0].strip() + # Remove any SORE special characters + cleaned = "".join(c for c in text if c not in _SORE_SPECIAL) + cleaned = cleaned.strip() + if len(cleaned) > 80: + cleaned = cleaned[:80] + # Skip single-char fragments (likely truncated identifiers) + if len(cleaned) <= 1: + return "" + return cleaned + from tree_sitter import Language, Parser, Query, QueryCursor QUERIES_DIR = Path(__file__).parent / "queries" @@ -191,7 +236,7 @@ def extract_arg_info(file_path, code): if not capname.startswith(BEHAVIORAL_PREFIXES): continue for node in nodes: - text = code[node.start_byte:node.end_byte].strip() + text = sanitize_symbol(code[node.start_byte:node.end_byte], capname) parent = node.parent if not parent: continue @@ -337,8 +382,9 @@ def preprocess_by_method(file_path: str, code: str): if not capname.startswith(BEHAVIORAL_PREFIXES): continue for node in nodes: - text = code[node.start_byte:node.end_byte].strip() - items.append((node.start_byte, capname, node, text)) + text = sanitize_symbol(code[node.start_byte:node.end_byte], capname) + if text: # skip empty/fragment symbols + items.append((node.start_byte, capname, node, text)) items.sort(key=lambda x: x[0]) @@ -379,7 +425,9 @@ def preprocess(file_path: str, code: str): if not capname.startswith(BEHAVIORAL_PREFIXES): continue for node in nodes: - items.append((node.start_byte, capname, node, code[node.start_byte:node.end_byte].strip())) + text = sanitize_symbol(code[node.start_byte:node.end_byte], capname) + if text: # skip empty/fragment symbols + items.append((node.start_byte, capname, node, text)) items.sort(key=lambda x: x[0]) diff --git a/bex/tag_preprocessor/ilocal_source.py b/bex/tag_preprocessor/ilocal_source.py new file mode 100644 index 0000000..0de602f --- /dev/null +++ b/bex/tag_preprocessor/ilocal_source.py @@ -0,0 +1,196 @@ +"""iLocal-style context extraction for source code. + +Adapts Bex 2007's iLocal algorithm from YAML/XML to source code. +Instead of hard-coding directory as the grouping key, this module +extracts (context, sequence) pairs and lets reduce find natural groupings. + +Context = structural position that determines the call sequence. +For YAML: context = YAML key. +For source code: context = file path component, class name, import set, etc. +""" + +import os +from collections import Counter, defaultdict + + +def extract_contexts_by_dir(sequences, seq_files, project_root): + """Context = relative directory path (like current package grouping). + + Returns dict: {context_key: [original_sequence, ...]} + """ + contexts = defaultdict(list) + for seq, fp in zip(sequences, seq_files): + rel = os.path.relpath(os.path.dirname(fp), project_root) + if rel == ".": + rel = "" + contexts[rel].append(seq) + return dict(contexts) + + +def extract_contexts_by_file(sequences, seq_files, project_root): + """Context = individual file path. + + Returns dict: {context_key: [original_sequence, ...]} + """ + contexts = defaultdict(list) + for seq, fp in zip(sequences, seq_files): + rel = os.path.relpath(fp, project_root) + contexts[rel].append(seq) + return dict(contexts) + + +def extract_contexts_by_parent_dir(sequences, seq_files, project_root): + """Context = parent of directory (one level coarser than dir). + + Returns dict: {context_key: [original_sequence, ...]} + """ + contexts = defaultdict(list) + for seq, fp in zip(sequences, seq_files): + rel_dir = os.path.relpath(os.path.dirname(fp), project_root) + if rel_dir == ".": + rel_dir = "" + parts = rel_dir.replace(os.sep, "/").rstrip("/").split("/") + parent = "/".join(parts[:-1]) if len(parts) > 1 else "" + contexts[parent].append(seq) + return dict(contexts) + + +def extract_contexts_by_file_depth(sequences, seq_files, project_root, depth=2): + """Context = first N components of file path. + + Returns dict: {context_key: [original_sequence, ...]} + """ + contexts = defaultdict(list) + for seq, fp in zip(sequences, seq_files): + rel = os.path.relpath(os.path.dirname(fp), project_root) + if rel == ".": + rel = "" + parts = rel.replace(os.sep, "/").rstrip("/").split("/") + key = "/".join(parts[:depth]) if parts and parts[0] else "" + contexts[key].append(seq) + return dict(contexts) + + +def extract_contexts_by_import_set(sequences, seq_files, project_root): + """Context = unique import set of the file. + + Methods from files with identical imports share a context. + """ + contexts = defaultdict(list) + for seq, fp in zip(sequences, seq_files): + imports = _file_import_signature(fp) + contexts[imports].append(seq) + return dict(contexts) + + +def _file_import_signature(fp): + """Build a canonical import signature for a file.""" + try: + with open(fp) as f: + lines = [f.readline() for _ in range(100)] + except OSError: + return "" + imports = [] + for line in lines: + stripped = line.strip() + if stripped.startswith("import ") or stripped.startswith("from "): + imports.append(stripped) + return "|".join(sorted(set(imports))) + + +def extract_contexts_by_symbol_overlap(sequences, seq_files, project_root, min_overlap=0.5): + """Context = cluster of methods that share >= min_overlap of their symbols. + + Uses union-find for transitive clustering. + """ + symbol_seqs = [[text for _, text, _ in seq] for seq in sequences] + n = len(symbol_seqs) + if n == 0: + return {} + + # Build symbol sets + sym_sets = [set(s) for s in symbol_seqs] + + # Union-find + parent = list(range(n)) + + def find(x): + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + + def union(x, y): + px, py = find(x), find(y) + if px != py: + parent[px] = py + + # Cluster by symbol overlap + for i in range(n): + for j in range(i + 1, n): + if not sym_sets[i] or not sym_sets[j]: + continue + overlap = len(sym_sets[i] & sym_sets[j]) / max(len(sym_sets[i] | sym_sets[j]), 1) + if overlap >= min_overlap: + union(i, j) + + # Group by cluster — keep original tuples + clusters = defaultdict(list) + for i in range(n): + root = find(i) + clusters[root].append(sequences[i]) + + return {f"cluster_{k}": v for k, v in clusters.items()} + + +def reduce_contexts(context_groups): + """iLocal reduce: merge contexts with identical signatures. + + Signature = sorted set of (length, first_symbol, last_symbol) across + all sequences in the context. + + Args: + context_groups: dict of {context_key: [original_sequence, ...]} + + Returns: + dict: {generalized_context: [original_sequence, ...]} + """ + if not context_groups: + return {} + + signature_map = {} + for ctx, seqs in context_groups.items(): + sig_parts = [] + for s in seqs: + symbol_seq = [text for _, text, _ in s] if s and isinstance(s[0], tuple) else s + first = symbol_seq[0] if symbol_seq else "∅" + last = symbol_seq[-1] if symbol_seq else "∅" + sig_parts.append((len(symbol_seq), first, last)) + signature = tuple(sorted(set(sig_parts))) + if signature not in signature_map: + signature_map[signature] = [] + signature_map[signature].append(ctx) + + result = {} + for sig, ctx_list in signature_map.items(): + merged_ctx = "|".join(sorted(ctx_list)[:3]) # cap label length + if len(ctx_list) > 3: + merged_ctx += f"|+{len(ctx_list)-3}" + merged_seqs = [] + for ctx in ctx_list: + merged_seqs.extend(context_groups[ctx]) + result[merged_ctx] = merged_seqs + + return result + + +# Context extraction strategies registry +CONTEXT_STRATEGIES = { + "dir": extract_contexts_by_dir, + "file": extract_contexts_by_file, + "parent_dir": extract_contexts_by_parent_dir, + "depth_2": lambda s, f, p: extract_contexts_by_file_depth(s, f, p, depth=2), + "depth_3": lambda s, f, p: extract_contexts_by_file_depth(s, f, p, depth=3), + "imports": extract_contexts_by_import_set, + "symbol_overlap": extract_contexts_by_symbol_overlap, +}