tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix: restore frequency_filter at 0.2, add dual 0.8 BEX coverage

Browse source 
- frequency_filter(min_coverage=0.2) strips rare symbols before clustering/inference
- DEFAULT_COVERAGE=0.8 passed to infer_ensemble for sequence-level core/outlier detection
- Both filters active: symbol-level (0.2) then BEX sequence-level (0.8)
- Fix seq_of_file tracking broken by filter creating new list objects
This commit is contained in:
tobjend 2026-07-04 01:13:48 +02:00
parent 55d21af48e
commit eb2173442b
1 changed files with 16 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
bex/tag_preprocessor/analyze.py
View file

@ -155,6 +155,9 @@ def scan_directory(dir_path, gitignore_spec=None):
return result
DEFAULT_COVERAGE = 0.8
def frequency_filter(sequences, min_coverage=0.2):
"""Remove symbols appearing in fewer than min_coverage fraction of files.
@ -257,9 +260,12 @@ def cluster_methods_adaptive(sequences, min_cluster_size=3, ngram_size=3, other_
return clusters
def analyze_clusters(file_paths, extension, project_root="", min_coverage=0.2, prefer=None, kmax=2, N=3):
def analyze_clusters(file_paths, extension, project_root="", min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3):
"""Run full pipeline with clustering: preprocess → cluster → per-cluster infer.
Each cluster gets its own ensemble inference with BEX core/outlier
detection at the given min_coverage threshold.
Returns:
list of (label, ensemble_result_dict, method_count, meta) tuples.
meta = {"files": set(paths), "imports": [lines], "arg_patterns": {...}, "packages": [...]}.
@ -277,7 +283,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=0.2, p
if not sequences:
return []
sequences = frequency_filter(sequences, min_coverage)
sequences = frequency_filter(sequences, min_coverage=0.2)
clusters = cluster_methods_adaptive(sequences)
results = []
@ -289,7 +295,7 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=0.2, p
imports = _extract_imports(cluster_fps)
arg_patterns = _build_arg_patterns(cluster_fps)
symbol_seqs = [[text for _, text, _ in seq] for seq in cluster_seqs]
result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer)
result = infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage)
packages = _top_packages(cluster_fps, project_root)
meta = {"files": cluster_fps, "imports": imports, "arg_patterns": arg_patterns, "packages": packages}
results.append((label, result, len(cluster_seqs), meta))
@ -297,13 +303,13 @@ def analyze_clusters(file_paths, extension, project_root="", min_coverage=0.2, p
return results
def infer(file_paths, extension, min_coverage=0.2, prefer=None, kmax=2, N=3):
def infer(file_paths, extension, min_coverage=DEFAULT_COVERAGE, prefer=None, kmax=2, N=3):
"""Run full pipeline: preprocess → frequency filter → ensemble infer.
Args:
file_paths: list of source file paths (same language).
extension: language extension (e.g. '.py').
min_coverage: minimum file fraction for a symbol to be kept.
min_coverage: BEX core coverage threshold for outlier removal.
prefer: inference algorithm preference ('crx', 'idregex', or None).
kmax: max k for k-ORE algorithms.
N: number of random trials.
@ -319,16 +325,16 @@ def infer(file_paths, extension, min_coverage=0.2, prefer=None, kmax=2, N=3):
if method_seq:
sequences.append(method_seq)
sequences = frequency_filter(sequences, min_coverage)
sequences = frequency_filter(sequences, min_coverage=0.2)
symbol_seqs = [[text for _, text, _ in seq] for seq in sequences]
return infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer)
return infer_ensemble(symbol_seqs, kmax=kmax, N=N, prefer=prefer, min_coverage=min_coverage)
def analyze_directory(
dir_path,
min_coverage=0.2,
min_coverage=DEFAULT_COVERAGE,
prefer=None,
kmax=2,
include=None,
@ -410,8 +416,8 @@ def _parse_args(argv=None):
help="Maximum k for k-ORE algorithms (default: 2)",
)
parser.add_argument(
"--min-coverage", type=float, default=0.2,
help="Minimum file fraction for a symbol to survive frequency filter (default: 0.2)",
"--min-coverage", type=float, default=DEFAULT_COVERAGE,
help="BEX core coverage threshold — outlier methods are removed until this fraction remains (default: 0.8)",
)
parser.add_argument(
"--include",