diff --git a/bex/tag_preprocessor/code.py b/bex/tag_preprocessor/code.py index 9f793aa..bbebbc0 100644 --- a/bex/tag_preprocessor/code.py +++ b/bex/tag_preprocessor/code.py @@ -53,6 +53,76 @@ BEHAVIORAL_PREFIXES = ( CALL_PREFIXES = ("function.call", "function.method.call", "reference.call", "reference.class", "constructor", "function") + +def coarsen_token(capname): + """Map a tree-sitter capture name to a language-agnostic category. + + Uses capture names from highlights.scm — same across all languages. + Function calls are kept as raw text (they ARE the behavioral content). + Only structural tokens (keywords, types) are coarsened. + """ + if capname.startswith("function.call") or capname.startswith("reference.call"): + return None # keep raw + if capname.startswith("function.method.call"): + return None # keep raw + if capname.startswith("constructor"): + return None # keep raw + if capname.startswith("function"): + return None # keep raw (bare @function in Kotlin etc.) + if capname.startswith("keyword.return"): + return "RETURN" + if capname.startswith("keyword.raise") or capname.startswith("keyword.throw"): + return "RAISE" + if capname.startswith("keyword.yield"): + return "YIELD" + if capname.startswith("keyword.exception"): + return "EXCEPTION" + if capname.startswith("keyword.conditional"): + return "IF" + if capname.startswith("keyword.repeat"): + return "LOOP" + if capname.startswith("keyword.function"): + return "FUN" + if capname.startswith("keyword"): + return "KW" + if capname.startswith("type.definition"): + return "TYPE" + if capname.startswith("type"): + return "TYPE" + if capname.startswith("variable.builtin"): + return "BUILTIN" + if capname.startswith("variable"): + return "VAR" + if capname.startswith("constant"): + return "CONST" + if capname.startswith("string"): + return "STR" + if capname.startswith("number"): + return "NUM" + if capname.startswith("boolean"): + return "BOOL" + if capname.startswith("attribute"): + return "ATTR" + return "OTHER" + + +def _extract_coarsened_tokens(seq): + """Extract coarsened tokens from a method sequence. + + Function calls are kept as raw text (they are the behavioral content). + Structural tokens (keywords, types) are mapped to language-agnostic + categories via coarsen_token(). + Returns list of strings: either raw text or category label. + """ + result = [] + for capname, text, _ in seq: + for prefix in BEHAVIORAL_PREFIXES: + if capname.startswith(prefix): + cat = coarsen_token(capname) + result.append(text if cat is None else cat) + break + return result + def _extract_call_tokens(seq): """Extract ordered call-like tokens from a method sequence. diff --git a/experiments/EXPERIMENT_LOG.md b/experiments/EXPERIMENT_LOG.md index c8a6322..fe38987 100644 --- a/experiments/EXPERIMENT_LOG.md +++ b/experiments/EXPERIMENT_LOG.md @@ -115,6 +115,43 @@ appears with different method names in different packages. --- +## Round 6: Structural Coarsening (Experiment `coarsen_eval.py`) + +**Hypothesis:** Coarsening structural tokens (keywords, types) while keeping +function names raw reveals cross-package patterns that pure-text misses. + +**Method:** `coarsen_token()` maps tree-sitter capture names to categories. +Function calls kept as raw text (they ARE the content). Only structural tokens +coarsened: RETURN, RAISE, IF, LOOP, EXCEPTION, KW, TYPE, FUN, ATTR. + +**Result:** + +| Codebase | Structural % | Raw coverage | Coarsened coverage | Δ | +|----------|-------------|--------------|-------------------|---| +| RAGSAK | 4.7% | 18.6% | 18.3% | -0.3% | +| Flask | 36.1% | 17.5% | **28.8%** | **+11.3%** | + +Key findings: +- RAGSAK: 95.3% function calls → coarsening has nothing to work with +- Flask: 36.1% structural → coarsening significantly improves grouping +- Flask k=3: cross-package contexts increase 40 → 47 +- Coarsened cross-package shapes: `('IF', 'KW', 'RETURN')` in 4 packages, + `('RETURN', 'render_template', 'render_template')` in 6 packages + +**Verdict:** Coarsening helps codebases with rich structural tokens (Python: +IF, LOOP, EXCEPTION, KW). Doesn't help codebases dominated by function calls +(Kotlin: 95% calls). The approach is sound but language-dependent in practice — +depends on how rich the highlights.scm is. + +**What we learned:** +- The 5% structural tokens DO carry signal when they exist +- Python highlights.scm is much richer than Kotlin's +- Coarsening is not dead — it's a tool for languages with rich captures +- The real question is whether the coarsened patterns are USEFUL, not just + whether they exist + +--- + ## What We Learned (Summary) 1. **Per-package grouping is too sparse.** 1-3 sequences per package isn't diff --git a/experiments/coarsen_eval.py b/experiments/coarsen_eval.py new file mode 100644 index 0000000..a53fedd --- /dev/null +++ b/experiments/coarsen_eval.py @@ -0,0 +1,223 @@ +"""Coarsening Experiment — Compare raw vs coarsened token extraction. + +Tests whether collapsing method names → tree-sitter capture categories +reveals cross-package patterns that raw text misses. +""" + +import json +import time +from collections import defaultdict +from pathlib import Path + +from bex.tag_preprocessor.analyze import ( + _preprocess_files, _file_to_package, scan_directory +) +from bex.tag_preprocessor.code import ( + _extract_call_tokens, _extract_coarsened_tokens +) +from bex.twotinf import build_soa +from bex.rwr0 import rwr0 + + +RESULTS_DIR = Path(__file__).parent / "results" + + +CODEBASES = { + "ragsak": {"name": "RAGSAK", "path": "/home/tobi/Desktop/kesai/RAGSAK", "ext": ".kt"}, + "flask": {"name": "Flask", "path": "/home/tobi/Desktop/dervish/external_refs/flask", "ext": ".py"}, +} + + +def load_data(base, ext): + """Load sequences, extract both raw and coarsened.""" + groups = scan_directory(base) + files = groups.get(ext, []) + sequences, seq_files = _preprocess_files(files) + + raw_seqs = [_extract_call_tokens(seq) for seq in sequences] + coarse_seqs = [_extract_coarsened_tokens(seq) for seq in sequences] + packages = [_file_to_package(fp, base) for fp in seq_files] + + return raw_seqs, coarse_seqs, packages, seq_files + + +def group_by_context(seqs, packages, k): + """Group sequences by first k symbols of their sequence.""" + contexts = defaultdict(list) + for i, seq in enumerate(seqs): + if not seq: + contexts[("_eps",)].append((i, seq)) + else: + ctx = tuple(seq[:k]) + contexts[ctx].append((i, seq)) + return contexts + + +def group_by_package(seqs, packages): + """Group sequences by package.""" + contexts = defaultdict(list) + for i, seq in enumerate(seqs): + contexts[packages[i]].append((i, seq)) + return contexts + + +def infer_sore(seqs): + """Try to infer a SORE from a list of sequences. Returns SORE string or None.""" + if len(seqs) < 2: + return None + clean = [s for s in seqs if s] + if len(clean) < 2: + return None + unique = len(set(tuple(s) for s in clean)) + if unique / len(clean) > 0.9: + return None + alphabet = set() + for s in clean: + alphabet.update(s) + if len(alphabet) > 20: + return None + try: + soa = build_soa(clean) + sore = rwr0(soa) + return sore if sore != "∅" else None + except Exception: + return None + + +def measure_group(ctx, items, label): + """Measure properties of a group.""" + seqs = [seq for _, seq in items] + n = len(seqs) + unique = len(set(tuple(s) for s in seqs)) + unique_ratio = unique / n if n else 1.0 + alphabet = set() + for s in seqs: + alphabet.update(s) + sore = infer_sore(seqs) + return { + "context": str(ctx), + "label": label, + "methods": n, + "unique": unique, + "unique_ratio": round(unique_ratio, 3), + "alphabet_size": len(alphabet), + "sore": sore[:200] if sore else None, + "sore_success": sore is not None, + } + + +def find_cross_package(groups_by_ctx, packages): + """Find contexts that span multiple packages.""" + ctx_to_pkgs = defaultdict(set) + for ctx, items in groups_by_ctx.items(): + for idx, _ in items: + ctx_to_pkgs[ctx].add(packages[idx]) + return {ctx: pkgs for ctx, pkgs in ctx_to_pkgs.items() if len(pkgs) > 1} + + +def run_comparison(name, raw_seqs, coarse_seqs, packages, k_values=(1, 2, 3)): + """Run raw vs coarsened comparison for one codebase.""" + results = {"name": name, "raw": {}, "coarsened": {}} + + for label, seqs in [("raw", raw_seqs), ("coarsened", coarse_seqs)]: + results[label]["seq_count"] = len(seqs) + alphabet = set() + for s in seqs: + alphabet.update(s) + results[label]["alphabet_size"] = len(alphabet) + results[label]["alphabet_sample"] = sorted(alphabet)[:30] + + for k in k_values: + t0 = time.time() + groups = group_by_context(seqs, packages, k) + elapsed = time.time() - t0 + + # Measure all groups + group_results = [] + sore_successes = 0 + methods_in_good = 0 + total_methods = 0 + + for ctx, items in sorted(groups.items(), key=lambda x: -len(x[1])): + m = measure_group(ctx, items, f"{label}_k{k}") + group_results.append(m) + total_methods += m["methods"] + if m["sore_success"]: + sore_successes += 1 + methods_in_good += m["methods"] + + # Cross-package analysis + cross_pkg = find_cross_package(groups, packages) + + results[label][f"k{k}"] = { + "contexts": len(groups), + "sore_successes": sore_successes, + "total_methods": total_methods, + "methods_in_good": methods_in_good, + "coverage": round(methods_in_good / total_methods * 100, 1) if total_methods else 0, + "cross_package_contexts": len(cross_pkg), + "elapsed": round(elapsed, 3), + "top_groups": group_results[:10], + "cross_pkg_examples": [ + {"context": str(ctx), "packages": len(pkgs)} + for ctx, pkgs in sorted(cross_pkg.items(), key=lambda x: -len(x[1]))[:10] + ], + } + + return results + + +def print_results(results): + """Pretty-print comparison results.""" + name = results["name"] + print(f"\n{'=' * 70}") + print(f" {name}") + print(f"{'=' * 70}") + + raw = results["raw"] + coarse = results["coarsened"] + + print(f"\n Alphabet size: raw={raw['alphabet_size']} coarsened={coarse['alphabet_size']} " + f"reduction={raw['alphabet_size'] - coarse['alphabet_size']} ({(1 - coarse['alphabet_size']/raw['alphabet_size'])*100:.0f}%)") + print(f" Coarsened categories: {coarse['alphabet_sample']}") + + for k in [1, 2, 3]: + rk = raw.get(f"k{k}", {}) + ck = coarse.get(f"k{k}", {}) + print(f"\n --- k={k} ---") + print(f" {'':20s} {'Raw':>10s} {'Coarsened':>10s}") + print(f" {'Contexts':20s} {rk.get('contexts',0):10d} {ck.get('contexts',0):10d}") + print(f" {'SORE successes':20s} {rk.get('sore_successes',0):10d} {ck.get('sore_successes',0):10d}") + print(f" {'Coverage':20s} {rk.get('coverage',0):9.1f}% {ck.get('coverage',0):9.1f}%") + print(f" {'Cross-pkg contexts':20s} {rk.get('cross_package_contexts',0):10d} {ck.get('cross_package_contexts',0):10d}") + + # Show cross-package examples from coarsened + cross_examples = ck.get("cross_pkg_examples", []) + if cross_examples: + print(f" Cross-package shapes:") + for ex in cross_examples[:5]: + print(f" {ex['context']} ({ex['packages']} packages)") + + +def main(): + all_results = {} + for key, cfg in CODEBASES.items(): + print(f"\nLoading {cfg['name']}...") + raw_seqs, coarse_seqs, packages, seq_files = load_data(cfg["path"], cfg["ext"]) + print(f" {len(raw_seqs)} methods from {len(set(packages))} packages") + + results = run_comparison(cfg["name"], raw_seqs, coarse_seqs, packages) + all_results[key] = results + print_results(results) + + # Save + out_path = RESULTS_DIR / f"coarsen_{key}.json" + with open(out_path, "w") as f: + json.dump(results, f, indent=2) + print(f"\n Saved to {out_path}") + + return all_results + + +if __name__ == "__main__": + main() diff --git a/experiments/results/coarsen_flask.json b/experiments/results/coarsen_flask.json new file mode 100644 index 0000000..773186c --- /dev/null +++ b/experiments/results/coarsen_flask.json @@ -0,0 +1,993 @@ +{ + "name": "Flask", + "raw": { + "seq_count": 1391, + "alphabet_size": 1055, + "alphabet_sample": [ + "", + "!r", + "\" * 100)", + "\")", + "\", \"index\",", + "\", de", + "\", su", + "\"/\")", + "\"/t", + "\"X-Foo\"", + "\"about\"", + "\"ba", + "\"cus", + "\"fake a", + "\"fo", + "\"foo.vi", + "\"h", + "\"hello\"", + "\"lat", + "\"mer", + "\"nes", + "\"someth", + "\"spam\")", + "\"static", + "# ip addre", + "(\"\"), 40", + "(\"\", hea", + "()", + "(test", + "({\"msg\":" + ], + "k1": { + "contexts": 373, + "sore_successes": 22, + "total_methods": 1391, + "methods_in_good": 64, + "coverage": 4.6, + "cross_package_contexts": 58, + "elapsed": 0.0, + "top_groups": [ + { + "context": "('_eps',)", + "label": "raw_k1", + "methods": 282, + "unique": 1, + "unique_ratio": 0.004, + "alphabet_size": 0, + "sore": null, + "sore_success": false + }, + { + "context": "('isinstance',)", + "label": "raw_k1", + "methods": 66, + "unique": 20, + "unique_ratio": 0.303, + "alphabet_size": 55, + "sore": null, + "sore_success": false + }, + { + "context": "('route',)", + "label": "raw_k1", + "methods": 59, + "unique": 54, + "unique_ratio": 0.915, + "alphabet_size": 148, + "sore": null, + "sore_success": false + }, + { + "context": "('append',)", + "label": "raw_k1", + "methods": 53, + "unique": 3, + "unique_ratio": 0.057, + "alphabet_size": 4, + "sore": null, + "sore_success": false + }, + { + "context": "('get',)", + "label": "raw_k1", + "methods": 50, + "unique": 40, + "unique_ratio": 0.8, + "alphabet_size": 97, + "sore": null, + "sore_success": false + }, + { + "context": "('Blueprint',)", + "label": "raw_k1", + "methods": 41, + "unique": 37, + "unique_ratio": 0.902, + "alphabet_size": 121, + "sore": null, + "sore_success": false + }, + { + "context": "('render_template',)", + "label": "raw_k1", + "methods": 33, + "unique": 2, + "unique_ratio": 0.061, + "alphabet_size": 2, + "sore": null, + "sore_success": false + }, + { + "context": "('Flask',)", + "label": "raw_k1", + "methods": 28, + "unique": 26, + "unique_ratio": 0.929, + "alphabet_size": 63, + "sore": null, + "sore_success": false + }, + { + "context": "('abort',)", + "label": "raw_k1", + "methods": 17, + "unique": 2, + "unique_ratio": 0.118, + "alphabet_size": 3, + "sore": null, + "sore_success": false + }, + { + "context": "('super',)", + "label": "raw_k1", + "methods": 16, + "unique": 12, + "unique_ratio": 0.75, + "alphabet_size": 33, + "sore": null, + "sore_success": false + } + ], + "cross_pkg_examples": [ + { + "context": "('_eps',)", + "packages": 9 + }, + { + "context": "('get',)", + "packages": 7 + }, + { + "context": "('render_template',)", + "packages": 6 + }, + { + "context": "('Flask',)", + "packages": 4 + }, + { + "context": "('pop',)", + "packages": 4 + }, + { + "context": "('isinstance',)", + "packages": 4 + }, + { + "context": "('setdefault',)", + "packages": 4 + }, + { + "context": "('post',)", + "packages": 3 + }, + { + "context": "('app_context',)", + "packages": 3 + }, + { + "context": "('test_client',)", + "packages": 3 + } + ] + }, + "k2": { + "contexts": 389, + "sore_successes": 22, + "total_methods": 1391, + "methods_in_good": 64, + "coverage": 4.6, + "cross_package_contexts": 54, + "elapsed": 0.001, + "top_groups": [ + { + "context": "('_eps',)", + "label": "raw_k2", + "methods": 282, + "unique": 1, + "unique_ratio": 0.004, + "alphabet_size": 0, + "sore": null, + "sore_success": false + }, + { + "context": "('isinstance', 'isinstance')", + "label": "raw_k2", + "methods": 66, + "unique": 20, + "unique_ratio": 0.303, + "alphabet_size": 55, + "sore": null, + "sore_success": false + }, + { + "context": "('route', 'route')", + "label": "raw_k2", + "methods": 59, + "unique": 54, + "unique_ratio": 0.915, + "alphabet_size": 148, + "sore": null, + "sore_success": false + }, + { + "context": "('append', 'append')", + "label": "raw_k2", + "methods": 53, + "unique": 3, + "unique_ratio": 0.057, + "alphabet_size": 4, + "sore": null, + "sore_success": false + }, + { + "context": "('get', 'get')", + "label": "raw_k2", + "methods": 50, + "unique": 40, + "unique_ratio": 0.8, + "alphabet_size": 97, + "sore": null, + "sore_success": false + }, + { + "context": "('Blueprint', 'Blueprint')", + "label": "raw_k2", + "methods": 41, + "unique": 37, + "unique_ratio": 0.902, + "alphabet_size": 121, + "sore": null, + "sore_success": false + }, + { + "context": "('render_template', 'render_template')", + "label": "raw_k2", + "methods": 33, + "unique": 2, + "unique_ratio": 0.061, + "alphabet_size": 2, + "sore": null, + "sore_success": false + }, + { + "context": "('Flask', 'Flask')", + "label": "raw_k2", + "methods": 28, + "unique": 26, + "unique_ratio": 0.929, + "alphabet_size": 63, + "sore": null, + "sore_success": false + }, + { + "context": "('abort', 'abort')", + "label": "raw_k2", + "methods": 17, + "unique": 2, + "unique_ratio": 0.118, + "alphabet_size": 3, + "sore": null, + "sore_success": false + }, + { + "context": "('super', 'super')", + "label": "raw_k2", + "methods": 16, + "unique": 12, + "unique_ratio": 0.75, + "alphabet_size": 33, + "sore": null, + "sore_success": false + } + ], + "cross_pkg_examples": [ + { + "context": "('_eps',)", + "packages": 9 + }, + { + "context": "('get', 'get')", + "packages": 7 + }, + { + "context": "('render_template', 'render_template')", + "packages": 6 + }, + { + "context": "('Flask', 'Flask')", + "packages": 4 + }, + { + "context": "('pop', 'pop')", + "packages": 4 + }, + { + "context": "('isinstance', 'isinstance')", + "packages": 4 + }, + { + "context": "('setdefault', 'setdefault')", + "packages": 4 + }, + { + "context": "('post', 'post')", + "packages": 3 + }, + { + "context": "('app_context', 'app_context')", + "packages": 3 + }, + { + "context": "('test_client', 'test_client')", + "packages": 3 + } + ] + }, + "k3": { + "contexts": 599, + "sore_successes": 43, + "total_methods": 1391, + "methods_in_good": 243, + "coverage": 17.5, + "cross_package_contexts": 40, + "elapsed": 0.001, + "top_groups": [ + { + "context": "('_eps',)", + "label": "raw_k3", + "methods": 282, + "unique": 1, + "unique_ratio": 0.004, + "alphabet_size": 0, + "sore": null, + "sore_success": false + }, + { + "context": "('isinstance', 'isinstance', 'isinstance')", + "label": "raw_k3", + "methods": 66, + "unique": 20, + "unique_ratio": 0.303, + "alphabet_size": 55, + "sore": null, + "sore_success": false + }, + { + "context": "('append', 'append')", + "label": "raw_k3", + "methods": 51, + "unique": 1, + "unique_ratio": 0.02, + "alphabet_size": 1, + "sore": "(append)+", + "sore_success": true + }, + { + "context": "('route', 'route', 'index')", + "label": "raw_k3", + "methods": 42, + "unique": 37, + "unique_ratio": 0.881, + "alphabet_size": 73, + "sore": null, + "sore_success": false + }, + { + "context": "('render_template', 'render_template')", + "label": "raw_k3", + "methods": 31, + "unique": 1, + "unique_ratio": 0.032, + "alphabet_size": 1, + "sore": "(render_template)+", + "sore_success": true + }, + { + "context": "('super', 'super', 'super')", + "label": "raw_k3", + "methods": 16, + "unique": 12, + "unique_ratio": 0.75, + "alphabet_size": 33, + "sore": null, + "sore_success": false + }, + { + "context": "('abort', 'abort')", + "label": "raw_k3", + "methods": 16, + "unique": 1, + "unique_ratio": 0.062, + "alphabet_size": 1, + "sore": "(abort)+", + "sore_success": true + }, + { + "context": "('get', 'get')", + "label": "raw_k3", + "methods": 10, + "unique": 1, + "unique_ratio": 0.1, + "alphabet_size": 1, + "sore": "(get)+", + "sore_success": true + }, + { + "context": "('pop', 'pop')", + "label": "raw_k3", + "methods": 10, + "unique": 1, + "unique_ratio": 0.1, + "alphabet_size": 1, + "sore": "(pop)+", + "sore_success": true + }, + { + "context": "('setdefault', 'setdefault', 'append')", + "label": "raw_k3", + "methods": 9, + "unique": 1, + "unique_ratio": 0.111, + "alphabet_size": 2, + "sore": null, + "sore_success": false + } + ], + "cross_pkg_examples": [ + { + "context": "('_eps',)", + "packages": 9 + }, + { + "context": "('render_template', 'render_template')", + "packages": 6 + }, + { + "context": "('get', 'get', 'get')", + "packages": 4 + }, + { + "context": "('isinstance', 'isinstance', 'isinstance')", + "packages": 4 + }, + { + "context": "('Flask', 'Flask', 'from_mapping')", + "packages": 3 + }, + { + "context": "('test_client', 'test_client')", + "packages": 3 + }, + { + "context": "('get', 'get')", + "packages": 3 + }, + { + "context": "('super', 'super', 'super')", + "packages": 3 + }, + { + "context": "('type', 'type', 'type')", + "packages": 3 + }, + { + "context": "('getattr', 'getattr', 'getattr')", + "packages": 3 + } + ] + } + }, + "coarsened": { + "seq_count": 1391, + "alphabet_size": 1063, + "alphabet_sample": [ + "", + "!r", + "\" * 100)", + "\")", + "\", \"index\",", + "\", de", + "\", su", + "\"/\")", + "\"/t", + "\"X-Foo\"", + "\"about\"", + "\"ba", + "\"cus", + "\"fake a", + "\"fo", + "\"foo.vi", + "\"h", + "\"hello\"", + "\"lat", + "\"mer", + "\"nes", + "\"someth", + "\"spam\")", + "\"static", + "# ip addre", + "(\"\"), 40", + "(\"\", hea", + "()", + "(test", + "({\"msg\":" + ], + "k1": { + "contexts": 166, + "sore_successes": 3, + "total_methods": 1391, + "methods_in_good": 20, + "coverage": 1.4, + "cross_package_contexts": 22, + "elapsed": 0.0, + "top_groups": [ + { + "context": "('RETURN',)", + "label": "coarsened_k1", + "methods": 448, + "unique": 101, + "unique_ratio": 0.225, + "alphabet_size": 97, + "sore": null, + "sore_success": false + }, + { + "context": "('ATTR',)", + "label": "coarsened_k1", + "methods": 120, + "unique": 112, + "unique_ratio": 0.933, + "alphabet_size": 296, + "sore": null, + "sore_success": false + }, + { + "context": "('IF',)", + "label": "coarsened_k1", + "methods": 117, + "unique": 109, + "unique_ratio": 0.932, + "alphabet_size": 209, + "sore": null, + "sore_success": false + }, + { + "context": "('KW',)", + "label": "coarsened_k1", + "methods": 91, + "unique": 80, + "unique_ratio": 0.879, + "alphabet_size": 246, + "sore": null, + "sore_success": false + }, + { + "context": "('EXCEPTION',)", + "label": "coarsened_k1", + "methods": 78, + "unique": 43, + "unique_ratio": 0.551, + "alphabet_size": 83, + "sore": null, + "sore_success": false + }, + { + "context": "('append',)", + "label": "coarsened_k1", + "methods": 51, + "unique": 3, + "unique_ratio": 0.059, + "alphabet_size": 4, + "sore": null, + "sore_success": false + }, + { + "context": "('FUN',)", + "label": "coarsened_k1", + "methods": 48, + "unique": 46, + "unique_ratio": 0.958, + "alphabet_size": 127, + "sore": null, + "sore_success": false + }, + { + "context": "('Blueprint',)", + "label": "coarsened_k1", + "methods": 40, + "unique": 36, + "unique_ratio": 0.9, + "alphabet_size": 119, + "sore": null, + "sore_success": false + }, + { + "context": "('get',)", + "label": "coarsened_k1", + "methods": 25, + "unique": 24, + "unique_ratio": 0.96, + "alphabet_size": 61, + "sore": null, + "sore_success": false + }, + { + "context": "('Flask',)", + "label": "coarsened_k1", + "methods": 23, + "unique": 23, + "unique_ratio": 1.0, + "alphabet_size": 69, + "sore": null, + "sore_success": false + } + ], + "cross_pkg_examples": [ + { + "context": "('RETURN',)", + "packages": 14 + }, + { + "context": "('get',)", + "packages": 7 + }, + { + "context": "('LOOP',)", + "packages": 6 + }, + { + "context": "('IF',)", + "packages": 6 + }, + { + "context": "('FUN',)", + "packages": 5 + }, + { + "context": "('KW',)", + "packages": 5 + }, + { + "context": "('pop',)", + "packages": 4 + }, + { + "context": "('EXCEPTION',)", + "packages": 4 + }, + { + "context": "('setdefault',)", + "packages": 4 + }, + { + "context": "('Flask',)", + "packages": 3 + } + ] + }, + "k2": { + "contexts": 353, + "sore_successes": 23, + "total_methods": 1391, + "methods_in_good": 346, + "coverage": 24.9, + "cross_package_contexts": 49, + "elapsed": 0.0, + "top_groups": [ + { + "context": "('RETURN',)", + "label": "coarsened_k2", + "methods": 234, + "unique": 1, + "unique_ratio": 0.004, + "alphabet_size": 1, + "sore": "RETURN", + "sore_success": true + }, + { + "context": "('ATTR', 'ATTR')", + "label": "coarsened_k2", + "methods": 80, + "unique": 74, + "unique_ratio": 0.925, + "alphabet_size": 171, + "sore": null, + "sore_success": false + }, + { + "context": "('IF', 'KW')", + "label": "coarsened_k2", + "methods": 67, + "unique": 60, + "unique_ratio": 0.896, + "alphabet_size": 134, + "sore": null, + "sore_success": false + }, + { + "context": "('append', 'append')", + "label": "coarsened_k2", + "methods": 51, + "unique": 3, + "unique_ratio": 0.059, + "alphabet_size": 4, + "sore": null, + "sore_success": false + }, + { + "context": "('Blueprint', 'Blueprint')", + "label": "coarsened_k2", + "methods": 40, + "unique": 36, + "unique_ratio": 0.9, + "alphabet_size": 119, + "sore": null, + "sore_success": false + }, + { + "context": "('RETURN', 'render_template')", + "label": "coarsened_k2", + "methods": 33, + "unique": 2, + "unique_ratio": 0.061, + "alphabet_size": 3, + "sore": null, + "sore_success": false + }, + { + "context": "('KW', 'FUN')", + "label": "coarsened_k2", + "methods": 32, + "unique": 30, + "unique_ratio": 0.938, + "alphabet_size": 122, + "sore": null, + "sore_success": false + }, + { + "context": "('RETURN', 'isinstance')", + "label": "coarsened_k2", + "methods": 32, + "unique": 2, + "unique_ratio": 0.062, + "alphabet_size": 6, + "sore": null, + "sore_success": false + }, + { + "context": "('get', 'get')", + "label": "coarsened_k2", + "methods": 25, + "unique": 24, + "unique_ratio": 0.96, + "alphabet_size": 61, + "sore": null, + "sore_success": false + }, + { + "context": "('ATTR', 'route')", + "label": "coarsened_k2", + "methods": 24, + "unique": 22, + "unique_ratio": 0.917, + "alphabet_size": 103, + "sore": null, + "sore_success": false + } + ], + "cross_pkg_examples": [ + { + "context": "('RETURN',)", + "packages": 9 + }, + { + "context": "('get', 'get')", + "packages": 7 + }, + { + "context": "('RETURN', 'render_template')", + "packages": 6 + }, + { + "context": "('LOOP', 'KW')", + "packages": 6 + }, + { + "context": "('IF', 'KW')", + "packages": 5 + }, + { + "context": "('KW', 'FUN')", + "packages": 4 + }, + { + "context": "('pop', 'pop')", + "packages": 4 + }, + { + "context": "('setdefault', 'setdefault')", + "packages": 4 + }, + { + "context": "('IF', 'isinstance')", + "packages": 4 + }, + { + "context": "('Flask', 'Flask')", + "packages": 3 + } + ] + }, + "k3": { + "contexts": 529, + "sore_successes": 30, + "total_methods": 1391, + "methods_in_good": 400, + "coverage": 28.8, + "cross_package_contexts": 47, + "elapsed": 0.0, + "top_groups": [ + { + "context": "('RETURN',)", + "label": "coarsened_k3", + "methods": 234, + "unique": 1, + "unique_ratio": 0.004, + "alphabet_size": 1, + "sore": "RETURN", + "sore_success": true + }, + { + "context": "('IF', 'KW', 'KW')", + "label": "coarsened_k3", + "methods": 34, + "unique": 29, + "unique_ratio": 0.853, + "alphabet_size": 73, + "sore": null, + "sore_success": false + }, + { + "context": "('ATTR', 'ATTR', 'route')", + "label": "coarsened_k3", + "methods": 34, + "unique": 31, + "unique_ratio": 0.912, + "alphabet_size": 74, + "sore": null, + "sore_success": false + }, + { + "context": "('RETURN', 'render_template', 'render_template')", + "label": "coarsened_k3", + "methods": 33, + "unique": 2, + "unique_ratio": 0.061, + "alphabet_size": 3, + "sore": null, + "sore_success": false + }, + { + "context": "('append', 'append')", + "label": "coarsened_k3", + "methods": 33, + "unique": 1, + "unique_ratio": 0.03, + "alphabet_size": 1, + "sore": "(append)+", + "sore_success": true + }, + { + "context": "('RETURN', 'isinstance', 'isinstance')", + "label": "coarsened_k3", + "methods": 32, + "unique": 2, + "unique_ratio": 0.062, + "alphabet_size": 6, + "sore": null, + "sore_success": false + }, + { + "context": "('ATTR', 'ATTR', 'FUN')", + "label": "coarsened_k3", + "methods": 31, + "unique": 28, + "unique_ratio": 0.903, + "alphabet_size": 92, + "sore": null, + "sore_success": false + }, + { + "context": "('ATTR', 'route', 'route')", + "label": "coarsened_k3", + "methods": 24, + "unique": 22, + "unique_ratio": 0.917, + "alphabet_size": 103, + "sore": null, + "sore_success": false + }, + { + "context": "('EXCEPTION',)", + "label": "coarsened_k3", + "methods": 21, + "unique": 1, + "unique_ratio": 0.048, + "alphabet_size": 1, + "sore": "EXCEPTION", + "sore_success": true + }, + { + "context": "('Blueprint', 'Blueprint', 'ATTR')", + "label": "coarsened_k3", + "methods": 21, + "unique": 20, + "unique_ratio": 0.952, + "alphabet_size": 76, + "sore": null, + "sore_success": false + } + ], + "cross_pkg_examples": [ + { + "context": "('RETURN',)", + "packages": 9 + }, + { + "context": "('RETURN', 'render_template', 'render_template')", + "packages": 6 + }, + { + "context": "('get', 'get', 'get')", + "packages": 4 + }, + { + "context": "('IF', 'KW', 'RETURN')", + "packages": 4 + }, + { + "context": "('get', 'get', 'IF')", + "packages": 4 + }, + { + "context": "('IF', 'KW', 'KW')", + "packages": 4 + }, + { + "context": "('IF', 'isinstance', 'isinstance')", + "packages": 4 + }, + { + "context": "('Flask', 'Flask', 'from_mapping')", + "packages": 3 + }, + { + "context": "('RETURN', 'test_client', 'test_client')", + "packages": 3 + }, + { + "context": "('RETURN', 'get', 'get')", + "packages": 3 + } + ] + } + } +} \ No newline at end of file diff --git a/experiments/results/coarsen_ragsak.json b/experiments/results/coarsen_ragsak.json new file mode 100644 index 0000000..69e7421 --- /dev/null +++ b/experiments/results/coarsen_ragsak.json @@ -0,0 +1,993 @@ +{ + "name": "RAGSAK", + "raw": { + "seq_count": 1594, + "alphabet_size": 1724, + "alphabet_sample": [ + "", + "\"Cann", + "\"Fail", + "\"k", + "(", + "(\"", + "()", + "(documen", + "(val scores: List = emptyList())", + ") && s3", + ")) {", + ",", + ".jo", + ".va", + "Acti", + "AgentCapabilityDescriptor", + "AgentDecision", + "AgentExecutionContext", + "AndServerWebExchangeMatcher", + "Any", + "AssertionError", + "AssistantMessage", + "AtomicLong", + "AtomicReference", + "AuthController", + "BCryptPasswordEncoder", + "BatchId", + "BatchJobController", + "BatchJobListener", + "BatchNotFoundException" + ], + "k1": { + "contexts": 510, + "sore_successes": 39, + "total_methods": 1594, + "methods_in_good": 125, + "coverage": 7.8, + "cross_package_contexts": 145, + "elapsed": 0.001, + "top_groups": [ + { + "context": "('every',)", + "label": "raw_k1", + "methods": 102, + "unique": 93, + "unique_ratio": 0.912, + "alphabet_size": 232, + "sore": null, + "sore_success": false + }, + { + "context": "('runTest',)", + "label": "raw_k1", + "methods": 59, + "unique": 54, + "unique_ratio": 0.915, + "alphabet_size": 131, + "sore": null, + "sore_success": false + }, + { + "context": "('listOf',)", + "label": "raw_k1", + "methods": 37, + "unique": 28, + "unique_ratio": 0.757, + "alphabet_size": 99, + "sore": null, + "sore_success": false + }, + { + "context": "('mockk',)", + "label": "raw_k1", + "methods": 31, + "unique": 21, + "unique_ratio": 0.677, + "alphabet_size": 99, + "sore": null, + "sore_success": false + }, + { + "context": "('runBlocking',)", + "label": "raw_k1", + "methods": 29, + "unique": 28, + "unique_ratio": 0.966, + "alphabet_size": 115, + "sore": null, + "sore_success": false + }, + { + "context": "('trim',)", + "label": "raw_k1", + "methods": 27, + "unique": 24, + "unique_ratio": 0.889, + "alphabet_size": 59, + "sore": null, + "sore_success": false + }, + { + "context": "('warn',)", + "label": "raw_k1", + "methods": 26, + "unique": 4, + "unique_ratio": 0.154, + "alphabet_size": 10, + "sore": null, + "sore_success": false + }, + { + "context": "('builder',)", + "label": "raw_k1", + "methods": 25, + "unique": 24, + "unique_ratio": 0.96, + "alphabet_size": 128, + "sore": null, + "sore_success": false + }, + { + "context": "('info',)", + "label": "raw_k1", + "methods": 23, + "unique": 19, + "unique_ratio": 0.826, + "alphabet_size": 73, + "sore": null, + "sore_success": false + }, + { + "context": "('parse',)", + "label": "raw_k1", + "methods": 18, + "unique": 18, + "unique_ratio": 1.0, + "alphabet_size": 49, + "sore": null, + "sore_success": false + } + ], + "cross_pkg_examples": [ + { + "context": "('every',)", + "packages": 18 + }, + { + "context": "('builder',)", + "packages": 16 + }, + { + "context": "('listOf',)", + "packages": 13 + }, + { + "context": "('runTest',)", + "packages": 12 + }, + { + "context": "('info',)", + "packages": 11 + }, + { + "context": "('mockk',)", + "packages": 11 + }, + { + "context": "('trim',)", + "packages": 10 + }, + { + "context": "('isEmpty',)", + "packages": 9 + }, + { + "context": "('_eps',)", + "packages": 9 + }, + { + "context": "('debug',)", + "packages": 9 + } + ] + }, + "k2": { + "contexts": 918, + "sore_successes": 76, + "total_methods": 1594, + "methods_in_good": 249, + "coverage": 15.6, + "cross_package_contexts": 100, + "elapsed": 0.001, + "top_groups": [ + { + "context": "('listOf', 'listOf')", + "label": "raw_k2", + "methods": 37, + "unique": 28, + "unique_ratio": 0.757, + "alphabet_size": 99, + "sore": null, + "sore_success": false + }, + { + "context": "('warn', 'status')", + "label": "raw_k2", + "methods": 22, + "unique": 1, + "unique_ratio": 0.045, + "alphabet_size": 4, + "sore": "warn.status.body.ErrorResponse", + "sore_success": true + }, + { + "context": "('_eps',)", + "label": "raw_k2", + "methods": 16, + "unique": 1, + "unique_ratio": 0.062, + "alphabet_size": 0, + "sore": null, + "sore_success": false + }, + { + "context": "('VectorChunk', 'mapOf')", + "label": "raw_k2", + "methods": 13, + "unique": 11, + "unique_ratio": 0.846, + "alphabet_size": 42, + "sore": null, + "sore_success": false + }, + { + "context": "('ery {', 'stKnowledgeBases()')", + "label": "raw_k2", + "methods": 13, + "unique": 5, + "unique_ratio": 0.385, + "alphabet_size": 13, + "sore": null, + "sore_success": false + }, + { + "context": "('productionClasses', 'filter')", + "label": "raw_k2", + "methods": 13, + "unique": 13, + "unique_ratio": 1.0, + "alphabet_size": 16, + "sore": null, + "sore_success": false + }, + { + "context": "('mockk',)", + "label": "raw_k2", + "methods": 11, + "unique": 1, + "unique_ratio": 0.091, + "alphabet_size": 1, + "sore": "mockk", + "sore_success": true + }, + { + "context": "('filesIn', 'assertTrue')", + "label": "raw_k2", + "methods": 11, + "unique": 5, + "unique_ratio": 0.455, + "alphabet_size": 4, + "sore": null, + "sore_success": false + }, + { + "context": "('runTest', 'ChatResponse')", + "label": "raw_k2", + "methods": 10, + "unique": 10, + "unique_ratio": 1.0, + "alphabet_size": 36, + "sore": null, + "sore_success": false + }, + { + "context": "('runTest', 'mockk')", + "label": "raw_k2", + "methods": 9, + "unique": 7, + "unique_ratio": 0.778, + "alphabet_size": 27, + "sore": null, + "sore_success": false + } + ], + "cross_pkg_examples": [ + { + "context": "('listOf', 'listOf')", + "packages": 13 + }, + { + "context": "('_eps',)", + "packages": 9 + }, + { + "context": "('trim', 'lowercase')", + "packages": 5 + }, + { + "context": "('mutableListOf', 'mutableListOf')", + "packages": 5 + }, + { + "context": "('defaultCapabilityId',)", + "packages": 4 + }, + { + "context": "('every', 'defaultCapabilityId')", + "packages": 4 + }, + { + "context": "('clearAllMocks',)", + "packages": 4 + }, + { + "context": "('VectorChunk', 'mapOf')", + "packages": 4 + }, + { + "context": "('Builder',)", + "packages": 4 + }, + { + "context": "('Any',)", + "packages": 4 + } + ] + }, + "k3": { + "contexts": 1066, + "sore_successes": 89, + "total_methods": 1594, + "methods_in_good": 297, + "coverage": 18.6, + "cross_package_contexts": 69, + "elapsed": 0.0, + "top_groups": [ + { + "context": "('warn', 'status', 'body')", + "label": "raw_k3", + "methods": 22, + "unique": 1, + "unique_ratio": 0.045, + "alphabet_size": 4, + "sore": "warn.status.body.ErrorResponse", + "sore_success": true + }, + { + "context": "('_eps',)", + "label": "raw_k3", + "methods": 16, + "unique": 1, + "unique_ratio": 0.062, + "alphabet_size": 0, + "sore": null, + "sore_success": false + }, + { + "context": "('VectorChunk', 'mapOf', 'mapOf')", + "label": "raw_k3", + "methods": 13, + "unique": 11, + "unique_ratio": 0.846, + "alphabet_size": 42, + "sore": null, + "sore_success": false + }, + { + "context": "('ery {', 'stKnowledgeBases()', 'gRequest(m')", + "label": "raw_k3", + "methods": 12, + "unique": 4, + "unique_ratio": 0.333, + "alphabet_size": 11, + "sore": "ery {.stKnowledgeBases().gRequest(m.(ckKnowledgeBase(re.ertEquals(Kn.\"k|eckKnowledgeBase(r.(sertEquals(K.(\"|(sertEquals(t|sertNull(o)))", + "sore_success": true + }, + { + "context": "('listOf', 'listOf', 'productionFiles')", + "label": "raw_k3", + "methods": 12, + "unique": 6, + "unique_ratio": 0.5, + "alphabet_size": 9, + "sore": null, + "sore_success": false + }, + { + "context": "('mockk',)", + "label": "raw_k3", + "methods": 11, + "unique": 1, + "unique_ratio": 0.091, + "alphabet_size": 1, + "sore": "mockk", + "sore_success": true + }, + { + "context": "('File', 'writeBytes', 'byteArrayOf')", + "label": "raw_k3", + "methods": 8, + "unique": 7, + "unique_ratio": 0.875, + "alphabet_size": 25, + "sore": null, + "sore_success": false + }, + { + "context": "('runTest', 'ChatResponse', 'emptyList')", + "label": "raw_k3", + "methods": 7, + "unique": 7, + "unique_ratio": 1.0, + "alphabet_size": 33, + "sore": null, + "sore_success": false + }, + { + "context": "('filesIn', 'assertTrue', 'hasImport')", + "label": "raw_k3", + "methods": 7, + "unique": 2, + "unique_ratio": 0.286, + "alphabet_size": 3, + "sore": "filesIn.assertTrue.(hasImport)+", + "sore_success": true + }, + { + "context": "('assertEquals', 'of', 'assertFailsWith')", + "label": "raw_k3", + "methods": 7, + "unique": 3, + "unique_ratio": 0.429, + "alphabet_size": 3, + "sore": "assertEquals.(of.(assertFailsWith)?)+", + "sore_success": true + } + ], + "cross_pkg_examples": [ + { + "context": "('_eps',)", + "packages": 9 + }, + { + "context": "('defaultCapabilityId',)", + "packages": 4 + }, + { + "context": "('clearAllMocks',)", + "packages": 4 + }, + { + "context": "('VectorChunk', 'mapOf', 'mapOf')", + "packages": 4 + }, + { + "context": "('Builder',)", + "packages": 4 + }, + { + "context": "('Any',)", + "packages": 4 + }, + { + "context": "('get',)", + "packages": 4 + }, + { + "context": "('registerProperties',)", + "packages": 4 + }, + { + "context": "('assumeTrue', 'isDockerAvailable', 'start')", + "packages": 4 + }, + { + "context": "('listOf', 'listOf', 'forEach')", + "packages": 4 + } + ] + } + }, + "coarsened": { + "seq_count": 1594, + "alphabet_size": 1728, + "alphabet_sample": [ + "", + "\"Cann", + "\"Fail", + "\"k", + "(", + "(\"", + "()", + "(documen", + "(val scores: List = emptyList())", + ") && s3", + ")) {", + ",", + ".jo", + ".va", + "ATTR", + "Acti", + "AgentCapabilityDescriptor", + "AgentDecision", + "AgentExecutionContext", + "AndServerWebExchangeMatcher", + "Any", + "AssertionError", + "AssistantMessage", + "AtomicLong", + "AtomicReference", + "AuthController", + "BCryptPasswordEncoder", + "BatchId", + "BatchJobController", + "BatchJobListener" + ], + "k1": { + "contexts": 474, + "sore_successes": 33, + "total_methods": 1594, + "methods_in_good": 107, + "coverage": 6.7, + "cross_package_contexts": 128, + "elapsed": 0.0, + "top_groups": [ + { + "context": "('every',)", + "label": "coarsened_k1", + "methods": 102, + "unique": 93, + "unique_ratio": 0.912, + "alphabet_size": 232, + "sore": null, + "sore_success": false + }, + { + "context": "('RETURN',)", + "label": "coarsened_k1", + "methods": 91, + "unique": 75, + "unique_ratio": 0.824, + "alphabet_size": 186, + "sore": null, + "sore_success": false + }, + { + "context": "('runTest',)", + "label": "coarsened_k1", + "methods": 59, + "unique": 54, + "unique_ratio": 0.915, + "alphabet_size": 131, + "sore": null, + "sore_success": false + }, + { + "context": "('listOf',)", + "label": "coarsened_k1", + "methods": 36, + "unique": 27, + "unique_ratio": 0.75, + "alphabet_size": 95, + "sore": null, + "sore_success": false + }, + { + "context": "('mockk',)", + "label": "coarsened_k1", + "methods": 31, + "unique": 21, + "unique_ratio": 0.677, + "alphabet_size": 100, + "sore": null, + "sore_success": false + }, + { + "context": "('runBlocking',)", + "label": "coarsened_k1", + "methods": 29, + "unique": 28, + "unique_ratio": 0.966, + "alphabet_size": 117, + "sore": null, + "sore_success": false + }, + { + "context": "('trim',)", + "label": "coarsened_k1", + "methods": 26, + "unique": 23, + "unique_ratio": 0.885, + "alphabet_size": 57, + "sore": null, + "sore_success": false + }, + { + "context": "('warn',)", + "label": "coarsened_k1", + "methods": 26, + "unique": 4, + "unique_ratio": 0.154, + "alphabet_size": 11, + "sore": null, + "sore_success": false + }, + { + "context": "('builder',)", + "label": "coarsened_k1", + "methods": 24, + "unique": 23, + "unique_ratio": 0.958, + "alphabet_size": 126, + "sore": null, + "sore_success": false + }, + { + "context": "('info',)", + "label": "coarsened_k1", + "methods": 23, + "unique": 19, + "unique_ratio": 0.826, + "alphabet_size": 74, + "sore": null, + "sore_success": false + } + ], + "cross_pkg_examples": [ + { + "context": "('RETURN',)", + "packages": 36 + }, + { + "context": "('every',)", + "packages": 18 + }, + { + "context": "('builder',)", + "packages": 15 + }, + { + "context": "('runTest',)", + "packages": 12 + }, + { + "context": "('listOf',)", + "packages": 12 + }, + { + "context": "('info',)", + "packages": 11 + }, + { + "context": "('mockk',)", + "packages": 11 + }, + { + "context": "('trim',)", + "packages": 10 + }, + { + "context": "('isEmpty',)", + "packages": 9 + }, + { + "context": "('debug',)", + "packages": 9 + } + ] + }, + "k2": { + "contexts": 883, + "sore_successes": 69, + "total_methods": 1594, + "methods_in_good": 225, + "coverage": 14.1, + "cross_package_contexts": 108, + "elapsed": 0.001, + "top_groups": [ + { + "context": "('listOf', 'listOf')", + "label": "coarsened_k2", + "methods": 36, + "unique": 27, + "unique_ratio": 0.75, + "alphabet_size": 95, + "sore": null, + "sore_success": false + }, + { + "context": "('warn', 'RETURN')", + "label": "coarsened_k2", + "methods": 23, + "unique": 2, + "unique_ratio": 0.087, + "alphabet_size": 7, + "sore": null, + "sore_success": false + }, + { + "context": "('RETURN',)", + "label": "coarsened_k2", + "methods": 13, + "unique": 1, + "unique_ratio": 0.077, + "alphabet_size": 1, + "sore": "RETURN", + "sore_success": true + }, + { + "context": "('VectorChunk', 'mapOf')", + "label": "coarsened_k2", + "methods": 13, + "unique": 11, + "unique_ratio": 0.846, + "alphabet_size": 42, + "sore": null, + "sore_success": false + }, + { + "context": "('ery {', 'stKnowledgeBases()')", + "label": "coarsened_k2", + "methods": 13, + "unique": 5, + "unique_ratio": 0.385, + "alphabet_size": 13, + "sore": null, + "sore_success": false + }, + { + "context": "('productionClasses', 'filter')", + "label": "coarsened_k2", + "methods": 13, + "unique": 13, + "unique_ratio": 1.0, + "alphabet_size": 16, + "sore": null, + "sore_success": false + }, + { + "context": "('isEmpty', 'RETURN')", + "label": "coarsened_k2", + "methods": 12, + "unique": 12, + "unique_ratio": 1.0, + "alphabet_size": 59, + "sore": null, + "sore_success": false + }, + { + "context": "('mockk',)", + "label": "coarsened_k2", + "methods": 11, + "unique": 1, + "unique_ratio": 0.091, + "alphabet_size": 1, + "sore": "mockk", + "sore_success": true + }, + { + "context": "('filesIn', 'assertTrue')", + "label": "coarsened_k2", + "methods": 11, + "unique": 5, + "unique_ratio": 0.455, + "alphabet_size": 4, + "sore": null, + "sore_success": false + }, + { + "context": "('RETURN', 'RETURN')", + "label": "coarsened_k2", + "methods": 10, + "unique": 10, + "unique_ratio": 1.0, + "alphabet_size": 15, + "sore": null, + "sore_success": false + } + ], + "cross_pkg_examples": [ + { + "context": "('listOf', 'listOf')", + "packages": 12 + }, + { + "context": "('isEmpty', 'RETURN')", + "packages": 9 + }, + { + "context": "('RETURN',)", + "packages": 9 + }, + { + "context": "('RETURN', 'RETURN')", + "packages": 7 + }, + { + "context": "('trim', 'lowercase')", + "packages": 5 + }, + { + "context": "('mutableListOf', 'mutableListOf')", + "packages": 5 + }, + { + "context": "('defaultCapabilityId',)", + "packages": 4 + }, + { + "context": "('every', 'defaultCapabilityId')", + "packages": 4 + }, + { + "context": "('clearAllMocks',)", + "packages": 4 + }, + { + "context": "('VectorChunk', 'mapOf')", + "packages": 4 + } + ] + }, + "k3": { + "contexts": 1062, + "sore_successes": 82, + "total_methods": 1594, + "methods_in_good": 292, + "coverage": 18.3, + "cross_package_contexts": 74, + "elapsed": 0.0, + "top_groups": [ + { + "context": "('warn', 'RETURN', 'status')", + "label": "coarsened_k3", + "methods": 22, + "unique": 1, + "unique_ratio": 0.045, + "alphabet_size": 5, + "sore": "warn.RETURN.status.body.ErrorResponse", + "sore_success": true + }, + { + "context": "('RETURN',)", + "label": "coarsened_k3", + "methods": 13, + "unique": 1, + "unique_ratio": 0.077, + "alphabet_size": 1, + "sore": "RETURN", + "sore_success": true + }, + { + "context": "('VectorChunk', 'mapOf', 'mapOf')", + "label": "coarsened_k3", + "methods": 13, + "unique": 11, + "unique_ratio": 0.846, + "alphabet_size": 42, + "sore": null, + "sore_success": false + }, + { + "context": "('ery {', 'stKnowledgeBases()', 'gRequest(m')", + "label": "coarsened_k3", + "methods": 12, + "unique": 4, + "unique_ratio": 0.333, + "alphabet_size": 11, + "sore": "ery {.stKnowledgeBases().gRequest(m.(ckKnowledgeBase(re.ertEquals(Kn.\"k|eckKnowledgeBase(r.(sertEquals(K.(\"|(sertEquals(t|sertNull(o)))", + "sore_success": true + }, + { + "context": "('listOf', 'listOf', 'productionFiles')", + "label": "coarsened_k3", + "methods": 12, + "unique": 6, + "unique_ratio": 0.5, + "alphabet_size": 9, + "sore": null, + "sore_success": false + }, + { + "context": "('mockk',)", + "label": "coarsened_k3", + "methods": 11, + "unique": 1, + "unique_ratio": 0.091, + "alphabet_size": 1, + "sore": "mockk", + "sore_success": true + }, + { + "context": "('parseStorageUri', 'RETURN', 'RETURN')", + "label": "coarsened_k3", + "methods": 8, + "unique": 5, + "unique_ratio": 0.625, + "alphabet_size": 14, + "sore": null, + "sore_success": false + }, + { + "context": "('File', 'writeBytes', 'byteArrayOf')", + "label": "coarsened_k3", + "methods": 8, + "unique": 7, + "unique_ratio": 0.875, + "alphabet_size": 25, + "sore": null, + "sore_success": false + }, + { + "context": "('runTest', 'ChatResponse', 'emptyList')", + "label": "coarsened_k3", + "methods": 7, + "unique": 7, + "unique_ratio": 1.0, + "alphabet_size": 33, + "sore": null, + "sore_success": false + }, + { + "context": "('filesIn', 'assertTrue', 'hasImport')", + "label": "coarsened_k3", + "methods": 7, + "unique": 2, + "unique_ratio": 0.286, + "alphabet_size": 3, + "sore": "filesIn.assertTrue.(hasImport)+", + "sore_success": true + } + ], + "cross_pkg_examples": [ + { + "context": "('RETURN',)", + "packages": 9 + }, + { + "context": "('defaultCapabilityId',)", + "packages": 4 + }, + { + "context": "('clearAllMocks',)", + "packages": 4 + }, + { + "context": "('VectorChunk', 'mapOf', 'mapOf')", + "packages": 4 + }, + { + "context": "('Builder',)", + "packages": 4 + }, + { + "context": "('Any',)", + "packages": 4 + }, + { + "context": "('get',)", + "packages": 4 + }, + { + "context": "('registerProperties',)", + "packages": 4 + }, + { + "context": "('assumeTrue', 'isDockerAvailable', 'start')", + "packages": 4 + }, + { + "context": "('listOf', 'listOf', 'forEach')", + "packages": 4 + } + ] + } + } +} \ No newline at end of file