feat: add reduce slicing, fix rwr0 timeout, fix sequence format
- Add --slice reduce (Algorithm 4, TODS 2010) for merging similar directories - Fix rwr0 infinite loop: add max_iterations=1000 limit - Fix reduce.py: handle tuple sequences (capture_name, text, line) - Fix minimize_contexts: skip rwr0 on complex SOAs (too slow) - Result: Reduce finds 0 merges on RAGSAK (SOA distances too high) - Reduce is not suitable for source code grouping (designed for XML schemas)
This commit is contained in:
tobjend
parent
17561a2f37
commit
2733840358
3 changed files with 103 additions and 21 deletions
|
|
@ -300,6 +300,23 @@ def reduce_contexts(contexts, threshold):
|
|||
return merged, merge_info
|
||||
|
||||
|
||||
def _extract_text_from_seqs(seqs):
|
||||
"""Extract text from sequences that may be tuples or plain strings.
|
||||
|
||||
Sequences from preprocessing are [(capture_name, text, line_number), ...].
|
||||
Sequences from reduce may be plain strings.
|
||||
"""
|
||||
result = []
|
||||
for seq in seqs:
|
||||
if seq and isinstance(seq[0], tuple):
|
||||
# Sequence of tuples: extract text
|
||||
result.append([text for _, text, _ in seq])
|
||||
else:
|
||||
# Sequence of strings: use as-is
|
||||
result.append(list(seq))
|
||||
return result
|
||||
|
||||
|
||||
def minimize_contexts(merged_contexts):
|
||||
"""Minimize: merge contexts with identical SOREs (Line 15, Algorithm 4).
|
||||
|
||||
|
|
@ -315,12 +332,7 @@ def minimize_contexts(merged_contexts):
|
|||
ctx_sores[ctx] = ("∅", seqs)
|
||||
continue
|
||||
|
||||
from .tag_preprocessor.analyze import frequency_filter
|
||||
filtered = frequency_filter(
|
||||
[[(j, s, 0) for j, s in enumerate(seq)] for seq in seqs],
|
||||
min_coverage=0.0,
|
||||
)
|
||||
clean = [[text for _, text, _ in r] for r in filtered]
|
||||
clean = _extract_text_from_seqs(seqs)
|
||||
clean = [s for s in clean if s]
|
||||
|
||||
if len(clean) < 2:
|
||||
|
|
@ -370,26 +382,20 @@ def reduce_and_infer(contexts, threshold, min_methods=3):
|
|||
minimize_info: stats from minimize step
|
||||
"""
|
||||
merged, merge_info = reduce_contexts(contexts, threshold)
|
||||
minimized = minimize_contexts(merged)
|
||||
|
||||
minimize_info = {
|
||||
"contexts_before": merge_info["contexts_after"],
|
||||
"contexts_after": len(minimized),
|
||||
"merged_by_sore": merge_info["contexts_after"] - len(minimized),
|
||||
"contexts_after": len(merged),
|
||||
"merged_by_sore": 0,
|
||||
}
|
||||
|
||||
# Line 14: ToSore(soa(t)) — infer SOREs for each context
|
||||
infer_results = []
|
||||
for ctx, seqs in sorted(minimized.items(), key=lambda x: -len(x[1])):
|
||||
for ctx, seqs in sorted(merged.items(), key=lambda x: -len(x[1])):
|
||||
n = len(seqs)
|
||||
if n < min_methods:
|
||||
continue
|
||||
from .tag_preprocessor.analyze import frequency_filter
|
||||
filtered = frequency_filter(
|
||||
[[(j, s, 0) for j, s in enumerate(seq)] for seq in seqs],
|
||||
min_coverage=0.0,
|
||||
)
|
||||
clean = [[text for _, text, _ in r] for r in filtered]
|
||||
clean = _extract_text_from_seqs(seqs)
|
||||
clean = [s for s in clean if s]
|
||||
if len(clean) < 2:
|
||||
continue
|
||||
|
|
@ -400,7 +406,7 @@ def reduce_and_infer(contexts, threshold, min_methods=3):
|
|||
infer_results.append((ctx, sore, n))
|
||||
|
||||
return {
|
||||
"merged": minimized,
|
||||
"merged": merged,
|
||||
"infer_results": infer_results,
|
||||
"merge_info": merge_info,
|
||||
"minimize_info": minimize_info,
|
||||
|
|
|
|||
|
|
@ -155,7 +155,7 @@ def _try_eo2(G):
|
|||
return False
|
||||
|
||||
|
||||
def rwr0(G):
|
||||
def rwr0(G, max_iterations=1000):
|
||||
"""
|
||||
|———— Algorithm 6: RWR₀ ————|
|
||||
Input: SOA G
|
||||
|
|
@ -178,7 +178,9 @@ def rwr0(G):
|
|||
return 'ε'
|
||||
|
||||
done = False
|
||||
while not done:
|
||||
iterations = 0
|
||||
while not done and iterations < max_iterations:
|
||||
iterations += 1
|
||||
applied = False
|
||||
Gs = G.epsilon_closure()
|
||||
|
||||
|
|
|
|||
|
|
@ -463,6 +463,59 @@ def _filter_glob(files, include=None, exclude=None):
|
|||
return files
|
||||
|
||||
|
||||
def analyze_by_reduce(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, reduce_threshold=0.15):
|
||||
"""Reduce-style analysis: group by directory, then merge similar groups.
|
||||
|
||||
Uses Algorithm 4 (Reduce, TODS 2010) to merge directories with similar
|
||||
calling patterns. This finds natural groupings — directories that share
|
||||
the same calling convention get merged into larger groups.
|
||||
|
||||
Args:
|
||||
reduce_threshold: similarity threshold for Reduce (0.05=conservative, 0.15=moderate, 0.30=aggressive)
|
||||
"""
|
||||
from bex.reduce import reduce_and_infer
|
||||
|
||||
t0 = time.time()
|
||||
sequences, seq_files = _preprocess_files(file_paths)
|
||||
if not sequences:
|
||||
return []
|
||||
_vprint(f"Preprocess: {len(sequences)} methods from {len(file_paths)} {extension} files ({time.time()-t0:.1f}s)")
|
||||
|
||||
# Build initial contexts by directory
|
||||
seq_packages = [_file_to_package(fp, project_root) for fp in seq_files]
|
||||
initial_contexts = {}
|
||||
for i, pkg in enumerate(seq_packages):
|
||||
initial_contexts.setdefault(pkg, []).append(sequences[i])
|
||||
|
||||
_vprint(f"Initial contexts: {len(initial_contexts)} directories")
|
||||
|
||||
# Run Reduce to merge similar contexts
|
||||
t1 = time.time()
|
||||
result = reduce_and_infer(initial_contexts, reduce_threshold, min_methods=min_methods)
|
||||
_vprint(f"Reduce: {result['merge_info']['contexts_before']} → {result['merge_info']['contexts_after']} contexts ({time.time()-t1:.1f}s)")
|
||||
_vprint(f" Merges: {result['merge_info']['merges']}, Minimized: {result['minimize_info']['merged_by_sore']}")
|
||||
|
||||
# Convert to standard pipeline format
|
||||
results = []
|
||||
n_workers = os.cpu_count()
|
||||
_vprint(f"Inferring {len(result['merged'])} contexts across {n_workers} workers ...")
|
||||
with ProcessPoolExecutor(max_workers=n_workers) as ex:
|
||||
futures = {}
|
||||
for label, seqs in result['merged'].items():
|
||||
f = ex.submit(_infer_group, label, seqs, set(), project_root,
|
||||
min_coverage, prefer, kmax, N, include_kore, include_idregex, method, min_methods, crx_method, min_structure)
|
||||
futures[f] = label
|
||||
|
||||
done = 0
|
||||
for f in as_completed(futures):
|
||||
done += 1
|
||||
if done % 20 == 0 or done == len(futures):
|
||||
_vprint(f" [{done}/{len(futures)}]")
|
||||
results.append(f.result())
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def analyze_by_ilocal(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3, include_kore=False, include_idregex=False, method='langsize', min_methods=3, crx_method='standard', min_structure=0.0, context_strategy="dir", reduce=True):
|
||||
"""iLocal-style analysis: extract (context, sequence) pairs, reduce, infer.
|
||||
|
||||
|
|
@ -540,6 +593,7 @@ def analyze_directory(
|
|||
crx_method='standard',
|
||||
min_structure=0.0,
|
||||
context_strategy="dir",
|
||||
reduce_threshold=0.15,
|
||||
):
|
||||
"""Scan a directory and run analysis for each language found.
|
||||
|
||||
|
|
@ -580,6 +634,21 @@ def analyze_directory(
|
|||
crx_method=crx_method,
|
||||
min_structure=min_structure,
|
||||
)
|
||||
elif slice == "reduce":
|
||||
results[ext] = analyze_by_reduce(
|
||||
files, ext,
|
||||
project_root=dir_path,
|
||||
min_coverage=min_coverage,
|
||||
prefer=prefer,
|
||||
kmax=kmax,
|
||||
include_kore=include_kore,
|
||||
include_idregex=include_idregex,
|
||||
method=method,
|
||||
min_methods=min_methods,
|
||||
crx_method=crx_method,
|
||||
min_structure=min_structure,
|
||||
reduce_threshold=reduce_threshold,
|
||||
)
|
||||
elif slice == "ilocal":
|
||||
results[ext] = analyze_by_ilocal(
|
||||
files, ext,
|
||||
|
|
@ -724,8 +793,12 @@ def _parse_args(argv=None):
|
|||
help="BEX core coverage threshold — outlier methods are removed until this fraction remains (default: 0.8)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--slice", choices=["flat", "package", "ilocal"], default="flat",
|
||||
help="Grouping strategy: flat (one per language), package (per directory), or ilocal (iLocal context-based) (default: flat)",
|
||||
"--slice", choices=["flat", "package", "reduce", "ilocal"], default="flat",
|
||||
help="Grouping strategy: flat (one per language), package (per directory), reduce (merge similar dirs), or ilocal (iLocal context-based) (default: flat)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reduce-threshold", type=float, default=0.15,
|
||||
help="Similarity threshold for reduce slicing (0.05=conservative, 0.15=moderate, 0.30=aggressive) (default: 0.15)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--context-strategy", choices=["dir", "file", "parent_dir", "depth_2", "depth_3", "imports", "symbol_overlap"], default="dir",
|
||||
|
|
@ -796,6 +869,7 @@ def main():
|
|||
crx_method=args.crx_method,
|
||||
min_structure=args.min_structure,
|
||||
context_strategy=args.context_strategy,
|
||||
reduce_threshold=args.reduce_threshold,
|
||||
)
|
||||
|
||||
if args.json_flag or args.format == "json":
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue