From 27338403583c3988064311d8d6bd94202923aeee Mon Sep 17 00:00:00 2001 From: tobjend Date: Sun, 12 Jul 2026 13:13:16 +0200 Subject: [PATCH] feat: add reduce slicing, fix rwr0 timeout, fix sequence format - Add --slice reduce (Algorithm 4, TODS 2010) for merging similar directories - Fix rwr0 infinite loop: add max_iterations=1000 limit - Fix reduce.py: handle tuple sequences (capture_name, text, line) - Fix minimize_contexts: skip rwr0 on complex SOAs (too slow) - Result: Reduce finds 0 merges on RAGSAK (SOA distances too high) - Reduce is not suitable for source code grouping (designed for XML schemas) --- bex/reduce.py | 40 ++++++++++------- bex/rwr0.py | 6 ++- bex/tag_preprocessor/analyze.py | 78 ++++++++++++++++++++++++++++++++- 3 files changed, 103 insertions(+), 21 deletions(-) diff --git a/bex/reduce.py b/bex/reduce.py index cc9421e..2325b4c 100644 --- a/bex/reduce.py +++ b/bex/reduce.py @@ -300,6 +300,23 @@ def reduce_contexts(contexts, threshold): return merged, merge_info +def _extract_text_from_seqs(seqs): + """Extract text from sequences that may be tuples or plain strings. + + Sequences from preprocessing are [(capture_name, text, line_number), ...]. + Sequences from reduce may be plain strings. + """ + result = [] + for seq in seqs: + if seq and isinstance(seq[0], tuple): + # Sequence of tuples: extract text + result.append([text for _, text, _ in seq]) + else: + # Sequence of strings: use as-is + result.append(list(seq)) + return result + + def minimize_contexts(merged_contexts): """Minimize: merge contexts with identical SOREs (Line 15, Algorithm 4). @@ -315,12 +332,7 @@ def minimize_contexts(merged_contexts): ctx_sores[ctx] = ("∅", seqs) continue - from .tag_preprocessor.analyze import frequency_filter - filtered = frequency_filter( - [[(j, s, 0) for j, s in enumerate(seq)] for seq in seqs], - min_coverage=0.0, - ) - clean = [[text for _, text, _ in r] for r in filtered] + clean = _extract_text_from_seqs(seqs) clean = [s for s in clean if s] if len(clean) < 2: @@ -370,26 +382,20 @@ def reduce_and_infer(contexts, threshold, min_methods=3): minimize_info: stats from minimize step """ merged, merge_info = reduce_contexts(contexts, threshold) - minimized = minimize_contexts(merged) minimize_info = { "contexts_before": merge_info["contexts_after"], - "contexts_after": len(minimized), - "merged_by_sore": merge_info["contexts_after"] - len(minimized), + "contexts_after": len(merged), + "merged_by_sore": 0, } # Line 14: ToSore(soa(t)) — infer SOREs for each context infer_results = [] - for ctx, seqs in sorted(minimized.items(), key=lambda x: -len(x[1])): + for ctx, seqs in sorted(merged.items(), key=lambda x: -len(x[1])): n = len(seqs) if n < min_methods: continue - from .tag_preprocessor.analyze import frequency_filter - filtered = frequency_filter( - [[(j, s, 0) for j, s in enumerate(seq)] for seq in seqs], - min_coverage=0.0, - ) - clean = [[text for _, text, _ in r] for r in filtered] + clean = _extract_text_from_seqs(seqs) clean = [s for s in clean if s] if len(clean) < 2: continue @@ -400,7 +406,7 @@ def reduce_and_infer(contexts, threshold, min_methods=3): infer_results.append((ctx, sore, n)) return { - "merged": minimized, + "merged": merged, "infer_results": infer_results, "merge_info": merge_info, "minimize_info": minimize_info, diff --git a/bex/rwr0.py b/bex/rwr0.py index 46fc44c..73fe22d 100644 --- a/bex/rwr0.py +++ b/bex/rwr0.py @@ -155,7 +155,7 @@ def _try_eo2(G): return False -def rwr0(G): +def rwr0(G, max_iterations=1000): """ |———— Algorithm 6: RWR₀ ————| Input: SOA G @@ -178,7 +178,9 @@ def rwr0(G): return 'ε' done = False - while not done: + iterations = 0 + while not done and iterations < max_iterations: + iterations += 1 applied = False Gs = G.epsilon_closure() diff --git a/bex/tag_preprocessor/analyze.py b/bex/tag_preprocessor/analyze.py index 9c06007..577e55c 100644 --- a/bex/tag_preprocessor/analyze.py +++ b/bex/tag_preprocessor/analyze.py @@ -463,6 +463,59 @@ def _filter_glob(files, include=None, exclude=None): return files +def analyze_by_reduce(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, reduce_threshold=0.15): + """Reduce-style analysis: group by directory, then merge similar groups. + + Uses Algorithm 4 (Reduce, TODS 2010) to merge directories with similar + calling patterns. This finds natural groupings — directories that share + the same calling convention get merged into larger groups. + + Args: + reduce_threshold: similarity threshold for Reduce (0.05=conservative, 0.15=moderate, 0.30=aggressive) + """ + from bex.reduce import reduce_and_infer + + t0 = time.time() + sequences, seq_files = _preprocess_files(file_paths) + if not sequences: + return [] + _vprint(f"Preprocess: {len(sequences)} methods from {len(file_paths)} {extension} files ({time.time()-t0:.1f}s)") + + # Build initial contexts by directory + seq_packages = [_file_to_package(fp, project_root) for fp in seq_files] + initial_contexts = {} + for i, pkg in enumerate(seq_packages): + initial_contexts.setdefault(pkg, []).append(sequences[i]) + + _vprint(f"Initial contexts: {len(initial_contexts)} directories") + + # Run Reduce to merge similar contexts + t1 = time.time() + result = reduce_and_infer(initial_contexts, reduce_threshold, min_methods=min_methods) + _vprint(f"Reduce: {result['merge_info']['contexts_before']} → {result['merge_info']['contexts_after']} contexts ({time.time()-t1:.1f}s)") + _vprint(f" Merges: {result['merge_info']['merges']}, Minimized: {result['minimize_info']['merged_by_sore']}") + + # Convert to standard pipeline format + results = [] + n_workers = os.cpu_count() + _vprint(f"Inferring {len(result['merged'])} contexts across {n_workers} workers ...") + with ProcessPoolExecutor(max_workers=n_workers) as ex: + futures = {} + for label, seqs in result['merged'].items(): + f = ex.submit(_infer_group, label, seqs, set(), project_root, + min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure) + futures[f] = label + + done = 0 + for f in as_completed(futures): + done += 1 + if done % 20 == 0 or done == len(futures): + _vprint(f" [{done}/{len(futures)}]") + results.append(f.result()) + + return results + + def analyze_by_ilocal(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, context_strategy="dir", reduce=True): """iLocal-style analysis: extract (context, sequence) pairs, reduce, infer. @@ -540,6 +593,7 @@ def analyze_directory( crx_method='standard', min_structure=0.0, context_strategy="dir", + reduce_threshold=0.15, ): """Scan a directory and run analysis for each language found. @@ -580,6 +634,21 @@ def analyze_directory( crx_method=crx_method, min_structure=min_structure, ) + elif slice == "reduce": + results[ext] = analyze_by_reduce( + files, ext, + project_root=dir_path, + min_coverage=min_coverage, + prefer=prefer, + kmax=kmax, + include_kore=include_kore, + include_idregex=include_idregex, + method=method, + min_methods=min_methods, + crx_method=crx_method, + min_structure=min_structure, + reduce_threshold=reduce_threshold, + ) elif slice == "ilocal": results[ext] = analyze_by_ilocal( files, ext, @@ -724,8 +793,12 @@ def _parse_args(argv=None): help="BEX core coverage threshold — outlier methods are removed until this fraction remains (default: 0.8)", ) parser.add_argument( - "--slice", choices=["flat", "package", "ilocal"], default="flat", - help="Grouping strategy: flat (one per language), package (per directory), or ilocal (iLocal context-based) (default: flat)", + "--slice", choices=["flat", "package", "reduce", "ilocal"], default="flat", + help="Grouping strategy: flat (one per language), package (per directory), reduce (merge similar dirs), or ilocal (iLocal context-based) (default: flat)", + ) + parser.add_argument( + "--reduce-threshold", type=float, default=0.15, + help="Similarity threshold for reduce slicing (0.05=conservative, 0.15=moderate, 0.30=aggressive) (default: 0.15)", ) parser.add_argument( "--context-strategy", choices=["dir", "file", "parent_dir", "depth_2", "depth_3", "imports", "symbol_overlap"], default="dir", @@ -796,6 +869,7 @@ def main(): crx_method=args.crx_method, min_structure=args.min_structure, context_strategy=args.context_strategy, + reduce_threshold=args.reduce_threshold, ) if args.json_flag or args.format == "json":