From eb2173442b0ae0e0c772078b7de3325ee04c7934 Mon Sep 17 00:00:00 2001 From: tobjend Date: Sat, 4 Jul 2026 01:13:48 +0200 Subject: [PATCH] fix: restore frequency_filter at 0.2, add dual 0.8 BEX coverage - frequency_filter(min_coverage=0.2) strips rare symbols before clustering/inference - DEFAULT_COVERAGE=0.8 passed to infer_ensemble for sequence-level core/outlier detection - Both filters active: symbol-level (0.2) then BEX sequence-level (0.8) - Fix seq_of_file tracking broken by filter creating new list objects --- bex/tag_preprocessor/analyze.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/bex/tag_preprocessor/analyze.py b/bex/tag_preprocessor/analyze.py index feb46d7..7b8e4dd 100644 --- a/bex/tag_preprocessor/analyze.py +++ b/bex/tag_preprocessor/analyze.py @@ -155,6 +155,9 @@ def scan_directory(dir_path, gitignore_spec=None): return result +DEFAULT_COVERAGE = 0.8 + + def frequency_filter(sequences, min_coverage=0.2): """Remove symbols appearing in fewer than min_coverage fraction of files. @@ -257,9 +260,12 @@ def cluster_methods_adaptive(sequences, min_cluster_size=3, ngram_size=3, other_ return clusters -def analyze_clusters(file_paths, extension, project_root="", min_coverage=0.2, prefer=None, kmax=2, N=3): +def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3): """Run full pipeline with clustering: preprocess → cluster → per-cluster infer. + Each cluster gets its own ensemble inference with BEX core/outlier + detection at the given min_coverage threshold. + Returns: list of (label, ensemble_result_dict, method_count, meta) tuples. meta = {"files": set(paths), "imports": [lines], "arg_patterns": {...}, "packages": [...]}. @@ -277,7 +283,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=0.2, p if not sequences: return [] - sequences = frequency_filter(sequences, min_coverage) + sequences = frequency_filter(sequences, min_coverage=0.2) clusters = cluster_methods_adaptive(sequences) results = [] @@ -289,7 +295,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=0.2, p imports = _extract_imports(cluster_fps) arg_patterns = _build_arg_patterns(cluster_fps) symbol_seqs = [[text for _, text, _ in seq] for seq in cluster_seqs] - result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer) + result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage) packages = _top_packages(cluster_fps, project_root) meta = {"files": cluster_fps, "imports": imports, "arg_patterns": arg_patterns, "packages": packages} results.append((label, result, len(cluster_seqs), meta)) @@ -297,13 +303,13 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=0.2, p return results -def infer(file_paths, extension, min_coverage=0.2, prefer=None, kmax=2, N=3): +def infer(file_paths, extension, min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3): """Run full pipeline: preprocess → frequency filter → ensemble infer. Args: file_paths: list of source file paths (same language). extension: language extension (e.g. '.py'). - min_coverage: minimum file fraction for a symbol to be kept. + min_coverage: BEX core coverage threshold for outlier removal. prefer: inference algorithm preference ('crx', 'idregex', or None). kmax: max k for k-ORE algorithms. N: number of random trials. @@ -319,16 +325,16 @@ def infer(file_paths, extension, min_coverage=0.2, prefer=None, kmax=2, N=3): if method_seq: sequences.append(method_seq) - sequences = frequency_filter(sequences, min_coverage) + sequences = frequency_filter(sequences, min_coverage=0.2) symbol_seqs = [[text for _, text, _ in seq] for seq in sequences] - return infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer) + return infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage) def analyze_directory( dir_path, - min_coverage=0.2, + min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, include=None, @@ -410,8 +416,8 @@ def _parse_args(argv=None): help="Maximum k for k-ORE algorithms (default: 2)", ) parser.add_argument( - "--min-coverage", type=float, default=0.2, - help="Minimum file fraction for a symbol to survive frequency filter (default: 0.2)", + "--min-coverage", type=float, default=DEFAULT_COVERAGE, + help="BEX core coverage threshold — outlier methods are removed until this fraction remains (default: 0.8)", ) parser.add_argument( "--include",