diff --git a/experiments/EXPERIMENT_LOG.md b/experiments/EXPERIMENT_LOG.md index f9acd98..78bce2c 100644 --- a/experiments/EXPERIMENT_LOG.md +++ b/experiments/EXPERIMENT_LOG.md @@ -210,6 +210,37 @@ structural tokens (response/client patterns) are highly repetitive. --- +## Round 8: Frequency Threshold Sweep + +**Hypothesis:** The fixed `min_coverage=0.2` is too aggressive. Lower thresholds +reveal more patterns while still filtering noise. + +**Method:** Test thresholds 0.00–0.20 on all 4 codebases. Measure symbol count, +surviving sequences, SORE success, coverage. + +**Results:** + +| Codebase | Thresh | Syms | Seqs | SOREs | Coverage | +|----------|--------|------|------|-------|----------| +| RAGSAK | 0.01 | 130 | 1270 | 61 | 26.1% | +| RAGSAK | 0.05 | 16 | 919 | 40 | 45.4% | +| RAGSAK | 0.10 | 5 | 657 | 19 | 69.1% | +| Flask | 0.01 | 47 | 784 | 28 | 35.2% | +| Flask | 0.05 | 5 | 414 | 7 | 44.4% | +| Flask | 0.10 | 2 | 237 | 5 | 100.0% | +| Coroutines | 0.01 | 76 | 4802 | 175 | 40.4% | +| FastAPI | 0.01 | 23 | 2678 | 14 | 17.0% | + +Key findings: +- Coverage increases with threshold (trivial: 1 symbol = 100% coverage) +- Sweet spot: 0.01–0.05. Enough symbols for meaningful patterns, enough + filtering to remove noise. +- At 0.01: RAGSAK gets `warn.status.body.ErrorResponse` (real convention) +- At 0.05: that pattern disappears (too aggressive) +- Flask dies at 0.15+ (0 symbols survive) + +--- + ## What We Learned (Summary) 1. **Per-package grouping is too sparse.** 1-3 sequences per package isn't diff --git a/experiments/freq_eval.py b/experiments/freq_eval.py new file mode 100644 index 0000000..02dcd76 --- /dev/null +++ b/experiments/freq_eval.py @@ -0,0 +1,167 @@ +"""Frequency Threshold Experiment — Test different min_coverage values. + +The current pipeline uses min_coverage=0.2 fixed. This tests what happens +at 0.01, 0.05, 0.10, 0.15, 0.20 on all 4 codebases. +""" + +import json +import time +from collections import Counter, defaultdict +from pathlib import Path + +from bex.tag_preprocessor.analyze import ( + _preprocess_files, _file_to_package, scan_directory, frequency_filter +) +from bex.tag_preprocessor.code import _extract_call_tokens, _extract_coarsened_tokens +from bex.twotinf import build_soa +from bex.rwr0 import rwr0 + + +RESULTS_DIR = Path(__file__).parent / "results" + + +CODEBASES = { + "ragsak": {"name": "RAGSAK", "path": "/home/tobi/Desktop/kesai/RAGSAK", "ext": ".kt"}, + "flask": {"name": "Flask", "path": "/home/tobi/Desktop/dervish/external_refs/flask", "ext": ".py"}, + "coroutines": {"name": "Kotlin Coroutines", "path": "/home/tobi/Desktop/dervish/projects/grammar-inference-engine/external_refs/kotlinx.coroutines", "ext": ".kt"}, + "fastapi": {"name": "FastAPI", "path": "/home/tobi/Desktop/dervish/projects/grammar-inference-engine/external_refs/fastapi", "ext": ".py"}, +} + +THRESHOLDS = [0.0, 0.01, 0.02, 0.05, 0.10, 0.15, 0.20] + + +def load_data(base, ext): + groups = scan_directory(base) + files = groups.get(ext, []) + sequences, seq_files = _preprocess_files(files) + raw_seqs = [_extract_call_tokens(seq) for seq in sequences] + packages = [_file_to_package(fp, base) for fp in seq_files] + return raw_seqs, packages, seq_files + + +def measure_at_threshold(raw_seqs, packages, threshold): + """Measure what happens at a given frequency threshold.""" + # Count symbol frequency + sym_file = Counter() + for seq in raw_seqs: + for sym in set(seq): + sym_file[sym] += 1 + + total = len(raw_seqs) + n_keep = sum(1 for c in sym_file.values() if c >= max(1, int(total * threshold))) + keep_syms = {s for s, c in sym_file.items() if c >= max(1, int(total * threshold))} + surviving = sum(1 for seq in raw_seqs if any(s in keep_syms for s in seq)) + + # Filter sequences + wrapped = [[(j, s, 0) for j, s in enumerate(seq)] for seq in raw_seqs] + filtered = frequency_filter(wrapped, min_coverage=threshold) + clean = [[text for _, text, _ in r] for r in filtered] + clean = [s for s in clean if s] + + # Group by first 3 symbols for inference + contexts = defaultdict(list) + for i, seq in enumerate(clean): + if not seq: + contexts[("_eps",)].append(seq) + else: + ctx = tuple(seq[:3]) + contexts[ctx].append(seq) + + sore_successes = 0 + methods_in_good = 0 + total_methods = 0 + top_patterns = [] + + for ctx, seqs in sorted(contexts.items(), key=lambda x: -len(x[1])): + n = len(seqs) + total_methods += n + if n < 3: + continue + unique = len(set(tuple(s) for s in seqs)) + if unique / n > 0.9: + continue + alphabet = set() + for s in seqs: + alphabet.update(s) + if len(alphabet) > 20: + continue + try: + soa = build_soa(seqs) + sore = rwr0(soa) + if sore != "∅": + sore_successes += 1 + methods_in_good += n + top_patterns.append({ + "context": str(ctx), + "methods": n, + "unique": unique, + "sore": sore[:150], + }) + except Exception: + pass + + return { + "threshold": threshold, + "symbols_total": len(sym_file), + "symbols_kept": n_keep, + "seqs_total": total, + "seqs_surviving": surviving, + "contexts": len(contexts), + "sore_successes": sore_successes, + "total_methods": total_methods, + "methods_in_good": methods_in_good, + "coverage": round(methods_in_good / total_methods * 100, 1) if total_methods else 0, + "top_patterns": top_patterns[:5], + } + + +def run_codebase(key): + cfg = CODEBASES[key] + print(f"\n{'=' * 60}") + print(f" {cfg['name']}") + print(f"{'=' * 60}") + + raw_seqs, packages, seq_files = load_data(cfg["path"], cfg["ext"]) + print(f" {len(raw_seqs)} methods from {len(set(packages))} packages") + + results = [] + for thresh in THRESHOLDS: + t0 = time.time() + r = measure_at_threshold(raw_seqs, packages, thresh) + elapsed = time.time() - t0 + r["elapsed"] = round(elapsed, 2) + results.append(r) + print(f" thresh={thresh:.2f} syms={r['symbols_kept']:4d} seqs={r['seqs_surviving']:5d} " + f"ctxs={r['contexts']:4d} SOREs={r['sore_successes']:3d} cov={r['coverage']:5.1f}% " + f"({elapsed:.1f}s)") + + out_path = RESULTS_DIR / f"freq_{key}.json" + with open(out_path, "w") as f: + json.dump({"codebase": cfg["name"], "results": results}, f, indent=2) + print(f" Saved to {out_path}") + + return results + + +def main(): + all_results = {} + for key in CODEBASES: + all_results[key] = run_codebase(key) + + # Summary table + print(f"\n{'=' * 80}") + print(" SUMMARY") + print(f"{'=' * 80}") + print(f" {'Codebase':20s} {'Thresh':7s} {'Syms':5s} {'Seqs':6s} {'SOREs':6s} {'Cov':7s}") + print(f" {'-'*55}") + for key, results in all_results.items(): + name = CODEBASES[key]["name"] + for r in results: + print(f" {name:20s} {r['threshold']:7.2f} {r['symbols_kept']:5d} " + f"{r['seqs_surviving']:6d} {r['sore_successes']:6d} {r['coverage']:6.1f}%") + + return all_results + + +if __name__ == "__main__": + main() diff --git a/experiments/results/freq_coroutines.json b/experiments/results/freq_coroutines.json new file mode 100644 index 0000000..c63ae41 --- /dev/null +++ b/experiments/results/freq_coroutines.json @@ -0,0 +1,302 @@ +{ + "codebase": "Kotlin Coroutines", + "results": [ + { + "threshold": 0.0, + "symbols_total": 2962, + "symbols_kept": 2962, + "seqs_total": 6720, + "seqs_surviving": 6559, + "contexts": 3433, + "sore_successes": 207, + "total_methods": 6559, + "methods_in_good": 1384, + "coverage": 21.1, + "top_patterns": [ + { + "context": "('expectUnreached',)", + "methods": 76, + "unique": 1, + "sore": "expectUnreached" + }, + { + "context": "('test', 'main', 'verifyLines')", + "methods": 56, + "unique": 1, + "sore": "test.main.verifyLines" + }, + { + "context": "('expect',)", + "methods": 42, + "unique": 1, + "sore": "expect" + }, + { + "context": "('close',)", + "methods": 40, + "unique": 1, + "sore": "close" + }, + { + "context": "('noImpl',)", + "methods": 34, + "unique": 1, + "sore": "noImpl" + } + ], + "elapsed": 0.82 + }, + { + "threshold": 0.01, + "symbols_total": 2962, + "symbols_kept": 76, + "seqs_total": 6720, + "seqs_surviving": 4802, + "contexts": 1297, + "sore_successes": 175, + "total_methods": 4802, + "methods_in_good": 1939, + "coverage": 40.4, + "top_patterns": [ + { + "context": "('test', 'main')", + "methods": 85, + "unique": 1, + "sore": "test.main" + }, + { + "context": "('expectUnreached',)", + "methods": 81, + "unique": 1, + "sore": "expectUnreached" + }, + { + "context": "('expect',)", + "methods": 73, + "unique": 1, + "sore": "expect" + }, + { + "context": "('block',)", + "methods": 59, + "unique": 1, + "sore": "block" + }, + { + "context": "('compareAndSet',)", + "methods": 55, + "unique": 1, + "sore": "compareAndSet" + } + ], + "elapsed": 0.85 + }, + { + "threshold": 0.02, + "symbols_total": 2962, + "symbols_kept": 38, + "seqs_total": 6720, + "seqs_surviving": 3954, + "contexts": 826, + "sore_successes": 148, + "total_methods": 3954, + "methods_in_good": 1635, + "coverage": 41.4, + "top_patterns": [ + { + "context": "('expectUnreached',)", + "methods": 92, + "unique": 1, + "sore": "expectUnreached" + }, + { + "context": "('runTest',)", + "methods": 82, + "unique": 1, + "sore": "runTest" + }, + { + "context": "('error', 'error')", + "methods": 80, + "unique": 1, + "sore": "(error)+" + }, + { + "context": "('expect',)", + "methods": 75, + "unique": 1, + "sore": "expect" + }, + { + "context": "('assertEquals',)", + "methods": 73, + "unique": 1, + "sore": "assertEquals" + } + ], + "elapsed": 0.72 + }, + { + "threshold": 0.05, + "symbols_total": 2962, + "symbols_kept": 17, + "seqs_total": 6720, + "seqs_surviving": 3511, + "contexts": 430, + "sore_successes": 104, + "total_methods": 3511, + "methods_in_good": 1674, + "coverage": 47.7, + "top_patterns": [ + { + "context": "('runTest',)", + "methods": 220, + "unique": 1, + "sore": "runTest" + }, + { + "context": "('expectUnreached',)", + "methods": 99, + "unique": 1, + "sore": "expectUnreached" + }, + { + "context": "('assertEquals',)", + "methods": 91, + "unique": 1, + "sore": "assertEquals" + }, + { + "context": "('runTest', 'assertEquals')", + "methods": 88, + "unique": 1, + "sore": "runTest.assertEquals" + }, + { + "context": "('expect',)", + "methods": 81, + "unique": 1, + "sore": "expect" + } + ], + "elapsed": 0.36 + }, + { + "threshold": 0.1, + "symbols_total": 2962, + "symbols_kept": 5, + "seqs_total": 6720, + "seqs_surviving": 2675, + "contexts": 61, + "sore_successes": 28, + "total_methods": 2675, + "methods_in_good": 1633, + "coverage": 61.0, + "top_patterns": [ + { + "context": "('runTest',)", + "methods": 428, + "unique": 1, + "sore": "runTest" + }, + { + "context": "('assertEquals',)", + "methods": 236, + "unique": 1, + "sore": "assertEquals" + }, + { + "context": "('runTest', 'assertEquals')", + "methods": 197, + "unique": 1, + "sore": "runTest.assertEquals" + }, + { + "context": "('runTest', 'expect', 'launch')", + "methods": 197, + "unique": 72, + "sore": "runTest.((expect)+|((assertEquals)+|(finish|(launch)+))+)+" + }, + { + "context": "('launch',)", + "methods": 145, + "unique": 1, + "sore": "launch" + } + ], + "elapsed": 0.09 + }, + { + "threshold": 0.15, + "symbols_total": 2962, + "symbols_kept": 4, + "seqs_total": 6720, + "seqs_surviving": 2499, + "contexts": 34, + "sore_successes": 19, + "total_methods": 2499, + "methods_in_good": 1309, + "coverage": 52.4, + "top_patterns": [ + { + "context": "('runTest',)", + "methods": 549, + "unique": 1, + "sore": "runTest" + }, + { + "context": "('assertEquals',)", + "methods": 253, + "unique": 1, + "sore": "assertEquals" + }, + { + "context": "('runTest', 'assertEquals')", + "methods": 233, + "unique": 1, + "sore": "runTest.assertEquals" + }, + { + "context": "('expect',)", + "methods": 102, + "unique": 1, + "sore": "expect" + }, + { + "context": "('assertEquals', 'assertEquals')", + "methods": 55, + "unique": 1, + "sore": "(assertEquals)+" + } + ], + "elapsed": 0.05 + }, + { + "threshold": 0.2, + "symbols_total": 2962, + "symbols_kept": 1, + "seqs_total": 6720, + "seqs_surviving": 1736, + "contexts": 3, + "sore_successes": 2, + "total_methods": 1736, + "methods_in_good": 1734, + "coverage": 99.9, + "top_patterns": [ + { + "context": "('runTest',)", + "methods": 1728, + "unique": 1, + "sore": "runTest" + }, + { + "context": "('runTest', 'runTest')", + "methods": 6, + "unique": 1, + "sore": "(runTest)+" + } + ], + "elapsed": 0.03 + } + ] +} \ No newline at end of file diff --git a/experiments/results/freq_fastapi.json b/experiments/results/freq_fastapi.json new file mode 100644 index 0000000..c210e6d --- /dev/null +++ b/experiments/results/freq_fastapi.json @@ -0,0 +1,320 @@ +{ + "codebase": "FastAPI", + "results": [ + { + "threshold": 0.0, + "symbols_total": 1503, + "symbols_kept": 1503, + "seqs_total": 4811, + "seqs_surviving": 3657, + "contexts": 795, + "sore_successes": 53, + "total_methods": 3657, + "methods_in_good": 518, + "coverage": 14.2, + "top_patterns": [ + { + "context": "('get', 'get')", + "methods": 89, + "unique": 1, + "sore": "(get)+" + }, + { + "context": "('HTTPException', 'HTTPException')", + "methods": 38, + "unique": 1, + "sore": "(HTTPException)+" + }, + { + "context": "('update', 'update')", + "methods": 33, + "unique": 1, + "sore": "(update)+" + }, + { + "context": "('post', 'post')", + "methods": 30, + "unique": 1, + "sore": "(post)+" + }, + { + "context": "('User', 'User')", + "methods": 23, + "unique": 1, + "sore": "(User)+" + } + ], + "elapsed": 0.52 + }, + { + "threshold": 0.01, + "symbols_total": 1503, + "symbols_kept": 23, + "seqs_total": 4811, + "seqs_surviving": 2678, + "contexts": 78, + "sore_successes": 14, + "total_methods": 2678, + "methods_in_good": 456, + "coverage": 17.0, + "top_patterns": [ + { + "context": "('get', 'get')", + "methods": 123, + "unique": 1, + "sore": "(get)+" + }, + { + "context": "('put', 'put', 'json')", + "methods": 67, + "unique": 2, + "sore": "((put)+|(json)+)+" + }, + { + "context": "('HTTPException', 'HTTPException')", + "methods": 53, + "unique": 1, + "sore": "(HTTPException)+" + }, + { + "context": "('update', 'update')", + "methods": 41, + "unique": 1, + "sore": "(update)+" + }, + { + "context": "('post', 'post')", + "methods": 32, + "unique": 1, + "sore": "(post)+" + } + ], + "elapsed": 0.16 + }, + { + "threshold": 0.02, + "symbols_total": 1503, + "symbols_kept": 8, + "seqs_total": 4811, + "seqs_surviving": 2331, + "contexts": 32, + "sore_successes": 9, + "total_methods": 2331, + "methods_in_good": 430, + "coverage": 18.4, + "top_patterns": [ + { + "context": "('get', 'get')", + "methods": 173, + "unique": 1, + "sore": "(get)+" + }, + { + "context": "('json', 'json')", + "methods": 73, + "unique": 1, + "sore": "(json)+" + }, + { + "context": "('snapshot', 'snapshot')", + "methods": 42, + "unique": 1, + "sore": "(snapshot)+" + }, + { + "context": "('post', 'post')", + "methods": 33, + "unique": 1, + "sore": "(post)+" + }, + { + "context": "('raises', 'raises', 'get')", + "methods": 33, + "unique": 2, + "sore": "((raises)+|(get)+)+" + } + ], + "elapsed": 0.09 + }, + { + "threshold": 0.05, + "symbols_total": 1503, + "symbols_kept": 5, + "seqs_total": 4811, + "seqs_surviving": 2277, + "contexts": 17, + "sore_successes": 5, + "total_methods": 2277, + "methods_in_good": 574, + "coverage": 25.2, + "top_patterns": [ + { + "context": "('get', 'get')", + "methods": 234, + "unique": 1, + "sore": "(get)+" + }, + { + "context": "('TestClient', 'TestClient')", + "methods": 176, + "unique": 1, + "sore": "(TestClient)+" + }, + { + "context": "('json', 'json')", + "methods": 73, + "unique": 1, + "sore": "(json)+" + }, + { + "context": "('post', 'post')", + "methods": 48, + "unique": 1, + "sore": "(post)+" + }, + { + "context": "('snapshot', 'snapshot')", + "methods": 43, + "unique": 1, + "sore": "(snapshot)+" + } + ], + "elapsed": 0.08 + }, + { + "threshold": 0.1, + "symbols_total": 1503, + "symbols_kept": 3, + "seqs_total": 4811, + "seqs_surviving": 2182, + "contexts": 10, + "sore_successes": 8, + "total_methods": 2182, + "methods_in_good": 1997, + "coverage": 91.5, + "top_patterns": [ + { + "context": "('get', 'get', 'json')", + "methods": 951, + "unique": 6, + "sore": "((get)+|(json)+)+" + }, + { + "context": "('TestClient', 'TestClient', 'get')", + "methods": 274, + "unique": 11, + "sore": "((TestClient)+|((get)+|(json)+)+)+" + }, + { + "context": "('json', 'json')", + "methods": 248, + "unique": 1, + "sore": "(json)+" + }, + { + "context": "('get', 'get')", + "methods": 235, + "unique": 1, + "sore": "(get)+" + }, + { + "context": "('TestClient', 'TestClient')", + "methods": 184, + "unique": 1, + "sore": "(TestClient)+" + } + ], + "elapsed": 0.05 + }, + { + "threshold": 0.15, + "symbols_total": 1503, + "symbols_kept": 2, + "seqs_total": 4811, + "seqs_surviving": 1998, + "contexts": 6, + "sore_successes": 5, + "total_methods": 1998, + "methods_in_good": 1987, + "coverage": 99.4, + "top_patterns": [ + { + "context": "('get', 'get', 'json')", + "methods": 1161, + "unique": 7, + "sore": "((get)+|(json)+)+" + }, + { + "context": "('json', 'json')", + "methods": 425, + "unique": 1, + "sore": "(json)+" + }, + { + "context": "('get', 'get')", + "methods": 302, + "unique": 1, + "sore": "(get)+" + }, + { + "context": "('get', 'get', 'get')", + "methods": 96, + "unique": 20, + "sore": "((get)+|(json)+)+" + }, + { + "context": "('json', 'json', 'get')", + "methods": 3, + "unique": 2, + "sore": "((json)+|(get)+)+" + } + ], + "elapsed": 0.04 + }, + { + "threshold": 0.2, + "symbols_total": 1503, + "symbols_kept": 2, + "seqs_total": 4811, + "seqs_surviving": 1998, + "contexts": 6, + "sore_successes": 5, + "total_methods": 1998, + "methods_in_good": 1987, + "coverage": 99.4, + "top_patterns": [ + { + "context": "('get', 'get', 'json')", + "methods": 1161, + "unique": 7, + "sore": "((get)+|(json)+)+" + }, + { + "context": "('json', 'json')", + "methods": 425, + "unique": 1, + "sore": "(json)+" + }, + { + "context": "('get', 'get')", + "methods": 302, + "unique": 1, + "sore": "(get)+" + }, + { + "context": "('get', 'get', 'get')", + "methods": 96, + "unique": 20, + "sore": "((get)+|(json)+)+" + }, + { + "context": "('json', 'json', 'get')", + "methods": 3, + "unique": 2, + "sore": "((json)+|(get)+)+" + } + ], + "elapsed": 0.04 + } + ] +} \ No newline at end of file diff --git a/experiments/results/freq_flask.json b/experiments/results/freq_flask.json new file mode 100644 index 0000000..50d12e5 --- /dev/null +++ b/experiments/results/freq_flask.json @@ -0,0 +1,258 @@ +{ + "codebase": "Flask", + "results": [ + { + "threshold": 0.0, + "symbols_total": 1055, + "symbols_kept": 1055, + "seqs_total": 1424, + "seqs_surviving": 1109, + "contexts": 598, + "sore_successes": 25, + "total_methods": 1109, + "methods_in_good": 207, + "coverage": 18.7, + "top_patterns": [ + { + "context": "('append', 'append')", + "methods": 51, + "unique": 1, + "sore": "(append)+" + }, + { + "context": "('render_template', 'render_template')", + "methods": 31, + "unique": 1, + "sore": "(render_template)+" + }, + { + "context": "('abort', 'abort')", + "methods": 16, + "unique": 1, + "sore": "(abort)+" + }, + { + "context": "('get', 'get')", + "methods": 10, + "unique": 1, + "sore": "(get)+" + }, + { + "context": "('pop', 'pop')", + "methods": 10, + "unique": 1, + "sore": "(pop)+" + } + ], + "elapsed": 0.05 + }, + { + "threshold": 0.01, + "symbols_total": 1055, + "symbols_kept": 47, + "seqs_total": 1424, + "seqs_surviving": 784, + "contexts": 173, + "sore_successes": 28, + "total_methods": 784, + "methods_in_good": 276, + "coverage": 35.2, + "top_patterns": [ + { + "context": "('append', 'append')", + "methods": 54, + "unique": 1, + "sore": "(append)+" + }, + { + "context": "('render_template', 'render_template')", + "methods": 33, + "unique": 1, + "sore": "(render_template)+" + }, + { + "context": "('get', 'get')", + "methods": 19, + "unique": 1, + "sore": "(get)+" + }, + { + "context": "('abort', 'abort')", + "methods": 16, + "unique": 1, + "sore": "(abort)+" + }, + { + "context": "('pop', 'pop')", + "methods": 15, + "unique": 1, + "sore": "(pop)+" + } + ], + "elapsed": 0.13 + }, + { + "threshold": 0.02, + "symbols_total": 1055, + "symbols_kept": 20, + "seqs_total": 1424, + "seqs_surviving": 621, + "contexts": 86, + "sore_successes": 11, + "total_methods": 621, + "methods_in_good": 213, + "coverage": 34.3, + "top_patterns": [ + { + "context": "('append', 'append')", + "methods": 64, + "unique": 1, + "sore": "(append)+" + }, + { + "context": "('render_template', 'render_template')", + "methods": 35, + "unique": 1, + "sore": "(render_template)+" + }, + { + "context": "('get', 'get')", + "methods": 34, + "unique": 1, + "sore": "(get)+" + }, + { + "context": "('pop', 'pop')", + "methods": 17, + "unique": 1, + "sore": "(pop)+" + }, + { + "context": "('abort', 'abort')", + "methods": 16, + "unique": 1, + "sore": "(abort)+" + } + ], + "elapsed": 0.07 + }, + { + "threshold": 0.05, + "symbols_total": 1055, + "symbols_kept": 5, + "seqs_total": 1424, + "seqs_surviving": 414, + "contexts": 16, + "sore_successes": 7, + "total_methods": 414, + "methods_in_good": 184, + "coverage": 44.4, + "top_patterns": [ + { + "context": "('append', 'append')", + "methods": 83, + "unique": 1, + "sore": "(append)+" + }, + { + "context": "('get', 'get')", + "methods": 53, + "unique": 1, + "sore": "(get)+" + }, + { + "context": "('get', 'get', 'get')", + "methods": 23, + "unique": 4, + "sore": "(get)+" + }, + { + "context": "('route', 'route')", + "methods": 11, + "unique": 1, + "sore": "(route)+" + }, + { + "context": "('route', 'route', 'get')", + "methods": 8, + "unique": 3, + "sore": "((route)+|(get)+)+" + } + ], + "elapsed": 0.02 + }, + { + "threshold": 0.1, + "symbols_total": 1055, + "symbols_kept": 2, + "seqs_total": 1424, + "seqs_surviving": 237, + "contexts": 5, + "sore_successes": 5, + "total_methods": 237, + "methods_in_good": 237, + "coverage": 100.0, + "top_patterns": [ + { + "context": "('route', 'route', 'get')", + "methods": 83, + "unique": 5, + "sore": "((route)+|(get)+)+" + }, + { + "context": "('get', 'get')", + "methods": 60, + "unique": 1, + "sore": "(get)+" + }, + { + "context": "('route', 'route', 'route')", + "methods": 38, + "unique": 14, + "sore": "((route)+|(get)+)+" + }, + { + "context": "('get', 'get', 'get')", + "methods": 33, + "unique": 6, + "sore": "(get)+" + }, + { + "context": "('route', 'route')", + "methods": 23, + "unique": 1, + "sore": "(route)+" + } + ], + "elapsed": 0.01 + }, + { + "threshold": 0.15, + "symbols_total": 1055, + "symbols_kept": 0, + "seqs_total": 1424, + "seqs_surviving": 0, + "contexts": 0, + "sore_successes": 0, + "total_methods": 0, + "methods_in_good": 0, + "coverage": 0, + "top_patterns": [], + "elapsed": 0.01 + }, + { + "threshold": 0.2, + "symbols_total": 1055, + "symbols_kept": 0, + "seqs_total": 1424, + "seqs_surviving": 0, + "contexts": 0, + "sore_successes": 0, + "total_methods": 0, + "methods_in_good": 0, + "coverage": 0, + "top_patterns": [], + "elapsed": 0.01 + } + ] +} \ No newline at end of file diff --git a/experiments/results/freq_ragsak.json b/experiments/results/freq_ragsak.json new file mode 100644 index 0000000..fdc8873 --- /dev/null +++ b/experiments/results/freq_ragsak.json @@ -0,0 +1,308 @@ +{ + "codebase": "RAGSAK", + "results": [ + { + "threshold": 0.0, + "symbols_total": 1724, + "symbols_kept": 1724, + "seqs_total": 1609, + "seqs_surviving": 1578, + "contexts": 1065, + "sore_successes": 49, + "total_methods": 1578, + "methods_in_good": 217, + "coverage": 13.8, + "top_patterns": [ + { + "context": "('warn', 'status', 'body')", + "methods": 22, + "unique": 1, + "sore": "warn.status.body.ErrorResponse" + }, + { + "context": "('ery {', 'stKnowledgeBases()', 'gRequest(m')", + "methods": 12, + "unique": 4, + "sore": "ery {.stKnowledgeBases().gRequest(m.(ckKnowledgeBase(re.ertEquals(Kn.\"k|eckKnowledgeBase(r.(sertEquals(K.(\"|(sertEquals(t|sertNull(o)))" + }, + { + "context": "('mockk',)", + "methods": 11, + "unique": 1, + "sore": "mockk" + }, + { + "context": "('filesIn', 'assertTrue', 'hasImport')", + "methods": 7, + "unique": 2, + "sore": "filesIn.assertTrue.(hasImport)+" + }, + { + "context": "('of',)", + "methods": 7, + "unique": 1, + "sore": "of" + } + ], + "elapsed": 0.09 + }, + { + "threshold": 0.01, + "symbols_total": 1724, + "symbols_kept": 130, + "seqs_total": 1609, + "seqs_surviving": 1270, + "contexts": 633, + "sore_successes": 61, + "total_methods": 1270, + "methods_in_good": 331, + "coverage": 26.1, + "top_patterns": [ + { + "context": "('assertEquals', 'assertEquals')", + "methods": 23, + "unique": 1, + "sore": "(assertEquals)+" + }, + { + "context": "('warn', 'status', 'body')", + "methods": 22, + "unique": 1, + "sore": "warn.status.body.ErrorResponse" + }, + { + "context": "('map',)", + "methods": 16, + "unique": 1, + "sore": "map" + }, + { + "context": "('mockk',)", + "methods": 12, + "unique": 1, + "sore": "mockk" + }, + { + "context": "('build',)", + "methods": 11, + "unique": 1, + "sore": "build" + } + ], + "elapsed": 0.08 + }, + { + "threshold": 0.02, + "symbols_total": 1724, + "symbols_kept": 54, + "seqs_total": 1609, + "seqs_surviving": 1185, + "contexts": 456, + "sore_successes": 56, + "total_methods": 1185, + "methods_in_good": 405, + "coverage": 34.2, + "top_patterns": [ + { + "context": "('assertEquals', 'assertEquals')", + "methods": 28, + "unique": 1, + "sore": "(assertEquals)+" + }, + { + "context": "('warn', 'status', 'body')", + "methods": 25, + "unique": 1, + "sore": "warn.status.body" + }, + { + "context": "('assertTrue',)", + "methods": 18, + "unique": 1, + "sore": "assertTrue" + }, + { + "context": "('map',)", + "methods": 17, + "unique": 1, + "sore": "map" + }, + { + "context": "('of',)", + "methods": 17, + "unique": 1, + "sore": "of" + } + ], + "elapsed": 0.21 + }, + { + "threshold": 0.05, + "symbols_total": 1724, + "symbols_kept": 16, + "seqs_total": 1609, + "seqs_surviving": 919, + "contexts": 182, + "sore_successes": 40, + "total_methods": 919, + "methods_in_good": 417, + "coverage": 45.4, + "top_patterns": [ + { + "context": "('build',)", + "methods": 40, + "unique": 1, + "sore": "build" + }, + { + "context": "('assertEquals', 'assertEquals')", + "methods": 34, + "unique": 1, + "sore": "(assertEquals)+" + }, + { + "context": "('of',)", + "methods": 34, + "unique": 1, + "sore": "of" + }, + { + "context": "('map',)", + "methods": 33, + "unique": 1, + "sore": "map" + }, + { + "context": "('assertTrue',)", + "methods": 29, + "unique": 1, + "sore": "assertTrue" + } + ], + "elapsed": 0.16 + }, + { + "threshold": 0.1, + "symbols_total": 1724, + "symbols_kept": 5, + "seqs_total": 1609, + "seqs_surviving": 657, + "contexts": 44, + "sore_successes": 19, + "total_methods": 657, + "methods_in_good": 454, + "coverage": 69.1, + "top_patterns": [ + { + "context": "('assertTrue',)", + "methods": 68, + "unique": 1, + "sore": "assertTrue" + }, + { + "context": "('every',)", + "methods": 57, + "unique": 1, + "sore": "every" + }, + { + "context": "('assertEquals',)", + "methods": 44, + "unique": 1, + "sore": "assertEquals" + }, + { + "context": "('assertEquals', 'assertEquals')", + "methods": 43, + "unique": 1, + "sore": "(assertEquals)+" + }, + { + "context": "('every', 'listOf', 'listOf')", + "methods": 41, + "unique": 30, + "sore": "(((assertEquals)+|(assertTrue)+)+|((every)+|((listOf)+|(any)+)+)+)+" + } + ], + "elapsed": 0.1 + }, + { + "threshold": 0.15, + "symbols_total": 1724, + "symbols_kept": 2, + "seqs_total": 1609, + "seqs_surviving": 473, + "contexts": 9, + "sore_successes": 8, + "total_methods": 473, + "methods_in_good": 431, + "coverage": 91.1, + "top_patterns": [ + { + "context": "('every',)", + "methods": 95, + "unique": 1, + "sore": "every" + }, + { + "context": "('assertEquals',)", + "methods": 72, + "unique": 1, + "sore": "assertEquals" + }, + { + "context": "('every', 'every', 'every')", + "methods": 59, + "unique": 20, + "sore": "((every)+|(assertEquals)+)+" + }, + { + "context": "('assertEquals', 'assertEquals')", + "methods": 56, + "unique": 1, + "sore": "(assertEquals)+" + }, + { + "context": "('every', 'assertEquals')", + "methods": 47, + "unique": 1, + "sore": "every.assertEquals" + } + ], + "elapsed": 0.02 + }, + { + "threshold": 0.2, + "symbols_total": 1724, + "symbols_kept": 1, + "seqs_total": 1609, + "seqs_surviving": 326, + "contexts": 3, + "sore_successes": 3, + "total_methods": 326, + "methods_in_good": 326, + "coverage": 100.0, + "top_patterns": [ + { + "context": "('assertEquals',)", + "methods": 162, + "unique": 1, + "sore": "assertEquals" + }, + { + "context": "('assertEquals', 'assertEquals')", + "methods": 87, + "unique": 1, + "sore": "(assertEquals)+" + }, + { + "context": "('assertEquals', 'assertEquals', 'assertEquals')", + "methods": 77, + "unique": 10, + "sore": "(assertEquals)+" + } + ], + "elapsed": 0.01 + } + ] +} \ No newline at end of file