tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feat: structural coarsening experiment — keeps function names raw, coarsens keywords

Browse source 
- coarsen_token() maps tree-sitter captures to categories (RETURN, IF, LOOP, etc.)
- Function calls kept as raw text (they ARE the behavioral content)
- Only ~5% structural tokens coarsened
- Results: Flask coverage 17.5% -> 28.8% (+11.3%), RAGSAK unchanged (95% calls)
- Cross-package shapes emerge: ('IF', 'KW', 'RETURN') in 4 Flask packages
This commit is contained in:
tobjend 2026-07-12 00:56:09 +02:00
parent 8e1c7f3767
commit 592974f039
5 changed files with 2316 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
bex/tag_preprocessor/code.py
View file

@ -53,6 +53,76 @@ BEHAVIORAL_PREFIXES = (
CALL_PREFIXES = ("function.call", "function.method.call", "reference.call", "reference.class", "constructor", "function")
def coarsen_token(capname):
"""Map a tree-sitter capture name to a language-agnostic category.
Uses capture names from highlights.scm — same across all languages.
Function calls are kept as raw text (they ARE the behavioral content).
Only structural tokens (keywords, types) are coarsened.
"""
if capname.startswith("function.call") or capname.startswith("reference.call"):
return None # keep raw
if capname.startswith("function.method.call"):
return None # keep raw
if capname.startswith("constructor"):
return None # keep raw
if capname.startswith("function"):
return None # keep raw (bare @function in Kotlin etc.)
if capname.startswith("keyword.return"):
return "RETURN"
if capname.startswith("keyword.raise") or capname.startswith("keyword.throw"):
return "RAISE"
if capname.startswith("keyword.yield"):
return "YIELD"
if capname.startswith("keyword.exception"):
return "EXCEPTION"
if capname.startswith("keyword.conditional"):
return "IF"
if capname.startswith("keyword.repeat"):
return "LOOP"
if capname.startswith("keyword.function"):
return "FUN"
if capname.startswith("keyword"):
return "KW"
if capname.startswith("type.definition"):
return "TYPE"
if capname.startswith("type"):
return "TYPE"
if capname.startswith("variable.builtin"):
return "BUILTIN"
if capname.startswith("variable"):
return "VAR"
if capname.startswith("constant"):
return "CONST"
if capname.startswith("string"):
return "STR"
if capname.startswith("number"):
return "NUM"
if capname.startswith("boolean"):
return "BOOL"
if capname.startswith("attribute"):
return "ATTR"
return "OTHER"
def _extract_coarsened_tokens(seq):
"""Extract coarsened tokens from a method sequence.
Function calls are kept as raw text (they are the behavioral content).
Structural tokens (keywords, types) are mapped to language-agnostic
categories via coarsen_token().
Returns list of strings: either raw text or category label.
"""
result = []
for capname, text, _ in seq:
for prefix in BEHAVIORAL_PREFIXES:
if capname.startswith(prefix):
cat = coarsen_token(capname)
result.append(text if cat is None else cat)
break
return result
def _extract_call_tokens(seq):
"""Extract ordered call-like tokens from a method sequence.

37
experiments/EXPERIMENT_LOG.md
View file

@ -115,6 +115,43 @@ appears with different method names in different packages.
---
## Round 6: Structural Coarsening (Experiment `coarsen_eval.py`)
**Hypothesis:** Coarsening structural tokens (keywords, types) while keeping
function names raw reveals cross-package patterns that pure-text misses.
**Method:** `coarsen_token()` maps tree-sitter capture names to categories.
Function calls kept as raw text (they ARE the content). Only structural tokens
coarsened: RETURN, RAISE, IF, LOOP, EXCEPTION, KW, TYPE, FUN, ATTR.
**Result:**
| Codebase | Structural % | Raw coverage | Coarsened coverage | Δ |
|----------|-------------|--------------|-------------------|---|
| RAGSAK | 4.7% | 18.6% | 18.3% | -0.3% |
| Flask | 36.1% | 17.5% | **28.8%** | **+11.3%** |
Key findings:
- RAGSAK: 95.3% function calls → coarsening has nothing to work with
- Flask: 36.1% structural → coarsening significantly improves grouping
- Flask k=3: cross-package contexts increase 40 → 47
- Coarsened cross-package shapes: `('IF', 'KW', 'RETURN')` in 4 packages,
`('RETURN', 'render_template', 'render_template')` in 6 packages
**Verdict:** Coarsening helps codebases with rich structural tokens (Python:
IF, LOOP, EXCEPTION, KW). Doesn't help codebases dominated by function calls
(Kotlin: 95% calls). The approach is sound but language-dependent in practice —
depends on how rich the highlights.scm is.
**What we learned:**
- The 5% structural tokens DO carry signal when they exist
- Python highlights.scm is much richer than Kotlin's
- Coarsening is not dead — it's a tool for languages with rich captures
- The real question is whether the coarsened patterns are USEFUL, not just
whether they exist
---
## What We Learned (Summary)
1. **Per-package grouping is too sparse.** 1-3 sequences per package isn't

223
experiments/coarsen_eval.py Normal file
View file

@ -0,0 +1,223 @@
"""Coarsening Experiment — Compare raw vs coarsened token extraction.
Tests whether collapsing method names → tree-sitter capture categories
reveals cross-package patterns that raw text misses.
"""
import json
import time
from collections import defaultdict
from pathlib import Path
from bex.tag_preprocessor.analyze import (
_preprocess_files, _file_to_package, scan_directory
)
from bex.tag_preprocessor.code import (
_extract_call_tokens, _extract_coarsened_tokens
)
from bex.twotinf import build_soa
from bex.rwr0 import rwr0
RESULTS_DIR = Path(__file__).parent / "results"
CODEBASES = {
"ragsak": {"name": "RAGSAK", "path": "/home/tobi/Desktop/kesai/RAGSAK", "ext": ".kt"},
"flask": {"name": "Flask", "path": "/home/tobi/Desktop/dervish/external_refs/flask", "ext": ".py"},
}
def load_data(base, ext):
"""Load sequences, extract both raw and coarsened."""
groups = scan_directory(base)
files = groups.get(ext, [])
sequences, seq_files = _preprocess_files(files)
raw_seqs = [_extract_call_tokens(seq) for seq in sequences]
coarse_seqs = [_extract_coarsened_tokens(seq) for seq in sequences]
packages = [_file_to_package(fp, base) for fp in seq_files]
return raw_seqs, coarse_seqs, packages, seq_files
def group_by_context(seqs, packages, k):
"""Group sequences by first k symbols of their sequence."""
contexts = defaultdict(list)
for i, seq in enumerate(seqs):
if not seq:
contexts[("_eps",)].append((i, seq))
else:
ctx = tuple(seq[:k])
contexts[ctx].append((i, seq))
return contexts
def group_by_package(seqs, packages):
"""Group sequences by package."""
contexts = defaultdict(list)
for i, seq in enumerate(seqs):
contexts[packages[i]].append((i, seq))
return contexts
def infer_sore(seqs):
"""Try to infer a SORE from a list of sequences. Returns SORE string or None."""
if len(seqs) < 2:
return None
clean = [s for s in seqs if s]
if len(clean) < 2:
return None
unique = len(set(tuple(s) for s in clean))
if unique / len(clean) > 0.9:
return None
alphabet = set()
for s in clean:
alphabet.update(s)
if len(alphabet) > 20:
return None
try:
soa = build_soa(clean)
sore = rwr0(soa)
return sore if sore != "∅" else None
except Exception:
return None
def measure_group(ctx, items, label):
"""Measure properties of a group."""
seqs = [seq for _, seq in items]
n = len(seqs)
unique = len(set(tuple(s) for s in seqs))
unique_ratio = unique / n if n else 1.0
alphabet = set()
for s in seqs:
alphabet.update(s)
sore = infer_sore(seqs)
return {
"context": str(ctx),
"label": label,
"methods": n,
"unique": unique,
"unique_ratio": round(unique_ratio, 3),
"alphabet_size": len(alphabet),
"sore": sore[:200] if sore else None,
"sore_success": sore is not None,
}
def find_cross_package(groups_by_ctx, packages):
"""Find contexts that span multiple packages."""
ctx_to_pkgs = defaultdict(set)
for ctx, items in groups_by_ctx.items():
for idx, _ in items:
ctx_to_pkgs[ctx].add(packages[idx])
return {ctx: pkgs for ctx, pkgs in ctx_to_pkgs.items() if len(pkgs) > 1}
def run_comparison(name, raw_seqs, coarse_seqs, packages, k_values=(1, 2, 3)):
"""Run raw vs coarsened comparison for one codebase."""
results = {"name": name, "raw": {}, "coarsened": {}}
for label, seqs in [("raw", raw_seqs), ("coarsened", coarse_seqs)]:
results[label]["seq_count"] = len(seqs)
alphabet = set()
for s in seqs:
alphabet.update(s)
results[label]["alphabet_size"] = len(alphabet)
results[label]["alphabet_sample"] = sorted(alphabet)[:30]
for k in k_values:
t0 = time.time()
groups = group_by_context(seqs, packages, k)
elapsed = time.time() - t0
# Measure all groups
group_results = []
sore_successes = 0
methods_in_good = 0
total_methods = 0
for ctx, items in sorted(groups.items(), key=lambda x: -len(x[1])):
m = measure_group(ctx, items, f"{label}_k{k}")
group_results.append(m)
total_methods += m["methods"]
if m["sore_success"]:
sore_successes += 1
methods_in_good += m["methods"]
# Cross-package analysis
cross_pkg = find_cross_package(groups, packages)
results[label][f"k{k}"] = {
"contexts": len(groups),
"sore_successes": sore_successes,
"total_methods": total_methods,
"methods_in_good": methods_in_good,
"coverage": round(methods_in_good / total_methods * 100, 1) if total_methods else 0,
"cross_package_contexts": len(cross_pkg),
"elapsed": round(elapsed, 3),
"top_groups": group_results[:10],
"cross_pkg_examples": [
{"context": str(ctx), "packages": len(pkgs)}
for ctx, pkgs in sorted(cross_pkg.items(), key=lambda x: -len(x[1]))[:10]
],
}
return results
def print_results(results):
"""Pretty-print comparison results."""
name = results["name"]
print(f"\n{'=' * 70}")
print(f" {name}")
print(f"{'=' * 70}")
raw = results["raw"]
coarse = results["coarsened"]
print(f"\n Alphabet size: raw={raw['alphabet_size']} coarsened={coarse['alphabet_size']} "
f"reduction={raw['alphabet_size'] - coarse['alphabet_size']} ({(1 - coarse['alphabet_size']/raw['alphabet_size'])*100:.0f}%)")
print(f" Coarsened categories: {coarse['alphabet_sample']}")
for k in [1, 2, 3]:
rk = raw.get(f"k{k}", {})
ck = coarse.get(f"k{k}", {})
print(f"\n --- k={k} ---")
print(f" {'':20s} {'Raw':>10s} {'Coarsened':>10s}")
print(f" {'Contexts':20s} {rk.get('contexts',0):10d} {ck.get('contexts',0):10d}")
print(f" {'SORE successes':20s} {rk.get('sore_successes',0):10d} {ck.get('sore_successes',0):10d}")
print(f" {'Coverage':20s} {rk.get('coverage',0):9.1f}% {ck.get('coverage',0):9.1f}%")
print(f" {'Cross-pkg contexts':20s} {rk.get('cross_package_contexts',0):10d} {ck.get('cross_package_contexts',0):10d}")
# Show cross-package examples from coarsened
cross_examples = ck.get("cross_pkg_examples", [])
if cross_examples:
print(f" Cross-package shapes:")
for ex in cross_examples[:5]:
print(f" {ex['context']} ({ex['packages']} packages)")
def main():
all_results = {}
for key, cfg in CODEBASES.items():
print(f"\nLoading {cfg['name']}...")
raw_seqs, coarse_seqs, packages, seq_files = load_data(cfg["path"], cfg["ext"])
print(f" {len(raw_seqs)} methods from {len(set(packages))} packages")
results = run_comparison(cfg["name"], raw_seqs, coarse_seqs, packages)
all_results[key] = results
print_results(results)
# Save
out_path = RESULTS_DIR / f"coarsen_{key}.json"
with open(out_path, "w") as f:
json.dump(results, f, indent=2)
print(f"\n Saved to {out_path}")
return all_results
if __name__ == "__main__":
main()

993
experiments/results/coarsen_flask.json Normal file
View file

@ -0,0 +1,993 @@
{
"name": "Flask",
"raw": {
"seq_count": 1391,
"alphabet_size": 1055,
"alphabet_sample": [
"",
"!r",
"\" * 100)",
"\")",
"\", \"index\",",
"\", de",
"\", su",
"\"/\")",
"\"/t",
"\"X-Foo\"",
"\"about\"",
"\"ba",
"\"cus",
"\"fake a",
"\"fo",
"\"foo.vi",
"\"h",
"\"hello\"",
"\"lat",
"\"mer",
"\"nes",
"\"someth",
"\"spam\")",
"\"static",
"# ip addre",
"(\"\"), 40",
"(\"\", hea",
"()",
"(test",
"({\"msg\":"
],
"k1": {
"contexts": 373,
"sore_successes": 22,
"total_methods": 1391,
"methods_in_good": 64,
"coverage": 4.6,
"cross_package_contexts": 58,
"elapsed": 0.0,
"top_groups": [
{
"context": "('_eps',)",
"label": "raw_k1",
"methods": 282,
"unique": 1,
"unique_ratio": 0.004,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('isinstance',)",
"label": "raw_k1",
"methods": 66,
"unique": 20,
"unique_ratio": 0.303,
"alphabet_size": 55,
"sore": null,
"sore_success": false
},
{
"context": "('route',)",
"label": "raw_k1",
"methods": 59,
"unique": 54,
"unique_ratio": 0.915,
"alphabet_size": 148,
"sore": null,
"sore_success": false
},
{
"context": "('append',)",
"label": "raw_k1",
"methods": 53,
"unique": 3,
"unique_ratio": 0.057,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('get',)",
"label": "raw_k1",
"methods": 50,
"unique": 40,
"unique_ratio": 0.8,
"alphabet_size": 97,
"sore": null,
"sore_success": false
},
{
"context": "('Blueprint',)",
"label": "raw_k1",
"methods": 41,
"unique": 37,
"unique_ratio": 0.902,
"alphabet_size": 121,
"sore": null,
"sore_success": false
},
{
"context": "('render_template',)",
"label": "raw_k1",
"methods": 33,
"unique": 2,
"unique_ratio": 0.061,
"alphabet_size": 2,
"sore": null,
"sore_success": false
},
{
"context": "('Flask',)",
"label": "raw_k1",
"methods": 28,
"unique": 26,
"unique_ratio": 0.929,
"alphabet_size": 63,
"sore": null,
"sore_success": false
},
{
"context": "('abort',)",
"label": "raw_k1",
"methods": 17,
"unique": 2,
"unique_ratio": 0.118,
"alphabet_size": 3,
"sore": null,
"sore_success": false
},
{
"context": "('super',)",
"label": "raw_k1",
"methods": 16,
"unique": 12,
"unique_ratio": 0.75,
"alphabet_size": 33,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 9
},
{
"context": "('get',)",
"packages": 7
},
{
"context": "('render_template',)",
"packages": 6
},
{
"context": "('Flask',)",
"packages": 4
},
{
"context": "('pop',)",
"packages": 4
},
{
"context": "('isinstance',)",
"packages": 4
},
{
"context": "('setdefault',)",
"packages": 4
},
{
"context": "('post',)",
"packages": 3
},
{
"context": "('app_context',)",
"packages": 3
},
{
"context": "('test_client',)",
"packages": 3
}
]
},
"k2": {
"contexts": 389,
"sore_successes": 22,
"total_methods": 1391,
"methods_in_good": 64,
"coverage": 4.6,
"cross_package_contexts": 54,
"elapsed": 0.001,
"top_groups": [
{
"context": "('_eps',)",
"label": "raw_k2",
"methods": 282,
"unique": 1,
"unique_ratio": 0.004,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('isinstance', 'isinstance')",
"label": "raw_k2",
"methods": 66,
"unique": 20,
"unique_ratio": 0.303,
"alphabet_size": 55,
"sore": null,
"sore_success": false
},
{
"context": "('route', 'route')",
"label": "raw_k2",
"methods": 59,
"unique": 54,
"unique_ratio": 0.915,
"alphabet_size": 148,
"sore": null,
"sore_success": false
},
{
"context": "('append', 'append')",
"label": "raw_k2",
"methods": 53,
"unique": 3,
"unique_ratio": 0.057,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('get', 'get')",
"label": "raw_k2",
"methods": 50,
"unique": 40,
"unique_ratio": 0.8,
"alphabet_size": 97,
"sore": null,
"sore_success": false
},
{
"context": "('Blueprint', 'Blueprint')",
"label": "raw_k2",
"methods": 41,
"unique": 37,
"unique_ratio": 0.902,
"alphabet_size": 121,
"sore": null,
"sore_success": false
},
{
"context": "('render_template', 'render_template')",
"label": "raw_k2",
"methods": 33,
"unique": 2,
"unique_ratio": 0.061,
"alphabet_size": 2,
"sore": null,
"sore_success": false
},
{
"context": "('Flask', 'Flask')",
"label": "raw_k2",
"methods": 28,
"unique": 26,
"unique_ratio": 0.929,
"alphabet_size": 63,
"sore": null,
"sore_success": false
},
{
"context": "('abort', 'abort')",
"label": "raw_k2",
"methods": 17,
"unique": 2,
"unique_ratio": 0.118,
"alphabet_size": 3,
"sore": null,
"sore_success": false
},
{
"context": "('super', 'super')",
"label": "raw_k2",
"methods": 16,
"unique": 12,
"unique_ratio": 0.75,
"alphabet_size": 33,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 9
},
{
"context": "('get', 'get')",
"packages": 7
},
{
"context": "('render_template', 'render_template')",
"packages": 6
},
{
"context": "('Flask', 'Flask')",
"packages": 4
},
{
"context": "('pop', 'pop')",
"packages": 4
},
{
"context": "('isinstance', 'isinstance')",
"packages": 4
},
{
"context": "('setdefault', 'setdefault')",
"packages": 4
},
{
"context": "('post', 'post')",
"packages": 3
},
{
"context": "('app_context', 'app_context')",
"packages": 3
},
{
"context": "('test_client', 'test_client')",
"packages": 3
}
]
},
"k3": {
"contexts": 599,
"sore_successes": 43,
"total_methods": 1391,
"methods_in_good": 243,
"coverage": 17.5,
"cross_package_contexts": 40,
"elapsed": 0.001,
"top_groups": [
{
"context": "('_eps',)",
"label": "raw_k3",
"methods": 282,
"unique": 1,
"unique_ratio": 0.004,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('isinstance', 'isinstance', 'isinstance')",
"label": "raw_k3",
"methods": 66,
"unique": 20,
"unique_ratio": 0.303,
"alphabet_size": 55,
"sore": null,
"sore_success": false
},
{
"context": "('append', 'append')",
"label": "raw_k3",
"methods": 51,
"unique": 1,
"unique_ratio": 0.02,
"alphabet_size": 1,
"sore": "(append)+",
"sore_success": true
},
{
"context": "('route', 'route', 'index')",
"label": "raw_k3",
"methods": 42,
"unique": 37,
"unique_ratio": 0.881,
"alphabet_size": 73,
"sore": null,
"sore_success": false
},
{
"context": "('render_template', 'render_template')",
"label": "raw_k3",
"methods": 31,
"unique": 1,
"unique_ratio": 0.032,
"alphabet_size": 1,
"sore": "(render_template)+",
"sore_success": true
},
{
"context": "('super', 'super', 'super')",
"label": "raw_k3",
"methods": 16,
"unique": 12,
"unique_ratio": 0.75,
"alphabet_size": 33,
"sore": null,
"sore_success": false
},
{
"context": "('abort', 'abort')",
"label": "raw_k3",
"methods": 16,
"unique": 1,
"unique_ratio": 0.062,
"alphabet_size": 1,
"sore": "(abort)+",
"sore_success": true
},
{
"context": "('get', 'get')",
"label": "raw_k3",
"methods": 10,
"unique": 1,
"unique_ratio": 0.1,
"alphabet_size": 1,
"sore": "(get)+",
"sore_success": true
},
{
"context": "('pop', 'pop')",
"label": "raw_k3",
"methods": 10,
"unique": 1,
"unique_ratio": 0.1,
"alphabet_size": 1,
"sore": "(pop)+",
"sore_success": true
},
{
"context": "('setdefault', 'setdefault', 'append')",
"label": "raw_k3",
"methods": 9,
"unique": 1,
"unique_ratio": 0.111,
"alphabet_size": 2,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 9
},
{
"context": "('render_template', 'render_template')",
"packages": 6
},
{
"context": "('get', 'get', 'get')",
"packages": 4
},
{
"context": "('isinstance', 'isinstance', 'isinstance')",
"packages": 4
},
{
"context": "('Flask', 'Flask', 'from_mapping')",
"packages": 3
},
{
"context": "('test_client', 'test_client')",
"packages": 3
},
{
"context": "('get', 'get')",
"packages": 3
},
{
"context": "('super', 'super', 'super')",
"packages": 3
},
{
"context": "('type', 'type', 'type')",
"packages": 3
},
{
"context": "('getattr', 'getattr', 'getattr')",
"packages": 3
}
]
}
},
"coarsened": {
"seq_count": 1391,
"alphabet_size": 1063,
"alphabet_sample": [
"",
"!r",
"\" * 100)",
"\")",
"\", \"index\",",
"\", de",
"\", su",
"\"/\")",
"\"/t",
"\"X-Foo\"",
"\"about\"",
"\"ba",
"\"cus",
"\"fake a",
"\"fo",
"\"foo.vi",
"\"h",
"\"hello\"",
"\"lat",
"\"mer",
"\"nes",
"\"someth",
"\"spam\")",
"\"static",
"# ip addre",
"(\"\"), 40",
"(\"\", hea",
"()",
"(test",
"({\"msg\":"
],
"k1": {
"contexts": 166,
"sore_successes": 3,
"total_methods": 1391,
"methods_in_good": 20,
"coverage": 1.4,
"cross_package_contexts": 22,
"elapsed": 0.0,
"top_groups": [
{
"context": "('RETURN',)",
"label": "coarsened_k1",
"methods": 448,
"unique": 101,
"unique_ratio": 0.225,
"alphabet_size": 97,
"sore": null,
"sore_success": false
},
{
"context": "('ATTR',)",
"label": "coarsened_k1",
"methods": 120,
"unique": 112,
"unique_ratio": 0.933,
"alphabet_size": 296,
"sore": null,
"sore_success": false
},
{
"context": "('IF',)",
"label": "coarsened_k1",
"methods": 117,
"unique": 109,
"unique_ratio": 0.932,
"alphabet_size": 209,
"sore": null,
"sore_success": false
},
{
"context": "('KW',)",
"label": "coarsened_k1",
"methods": 91,
"unique": 80,
"unique_ratio": 0.879,
"alphabet_size": 246,
"sore": null,
"sore_success": false
},
{
"context": "('EXCEPTION',)",
"label": "coarsened_k1",
"methods": 78,
"unique": 43,
"unique_ratio": 0.551,
"alphabet_size": 83,
"sore": null,
"sore_success": false
},
{
"context": "('append',)",
"label": "coarsened_k1",
"methods": 51,
"unique": 3,
"unique_ratio": 0.059,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('FUN',)",
"label": "coarsened_k1",
"methods": 48,
"unique": 46,
"unique_ratio": 0.958,
"alphabet_size": 127,
"sore": null,
"sore_success": false
},
{
"context": "('Blueprint',)",
"label": "coarsened_k1",
"methods": 40,
"unique": 36,
"unique_ratio": 0.9,
"alphabet_size": 119,
"sore": null,
"sore_success": false
},
{
"context": "('get',)",
"label": "coarsened_k1",
"methods": 25,
"unique": 24,
"unique_ratio": 0.96,
"alphabet_size": 61,
"sore": null,
"sore_success": false
},
{
"context": "('Flask',)",
"label": "coarsened_k1",
"methods": 23,
"unique": 23,
"unique_ratio": 1.0,
"alphabet_size": 69,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('RETURN',)",
"packages": 14
},
{
"context": "('get',)",
"packages": 7
},
{
"context": "('LOOP',)",
"packages": 6
},
{
"context": "('IF',)",
"packages": 6
},
{
"context": "('FUN',)",
"packages": 5
},
{
"context": "('KW',)",
"packages": 5
},
{
"context": "('pop',)",
"packages": 4
},
{
"context": "('EXCEPTION',)",
"packages": 4
},
{
"context": "('setdefault',)",
"packages": 4
},
{
"context": "('Flask',)",
"packages": 3
}
]
},
"k2": {
"contexts": 353,
"sore_successes": 23,
"total_methods": 1391,
"methods_in_good": 346,
"coverage": 24.9,
"cross_package_contexts": 49,
"elapsed": 0.0,
"top_groups": [
{
"context": "('RETURN',)",
"label": "coarsened_k2",
"methods": 234,
"unique": 1,
"unique_ratio": 0.004,
"alphabet_size": 1,
"sore": "RETURN",
"sore_success": true
},
{
"context": "('ATTR', 'ATTR')",
"label": "coarsened_k2",
"methods": 80,
"unique": 74,
"unique_ratio": 0.925,
"alphabet_size": 171,
"sore": null,
"sore_success": false
},
{
"context": "('IF', 'KW')",
"label": "coarsened_k2",
"methods": 67,
"unique": 60,
"unique_ratio": 0.896,
"alphabet_size": 134,
"sore": null,
"sore_success": false
},
{
"context": "('append', 'append')",
"label": "coarsened_k2",
"methods": 51,
"unique": 3,
"unique_ratio": 0.059,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('Blueprint', 'Blueprint')",
"label": "coarsened_k2",
"methods": 40,
"unique": 36,
"unique_ratio": 0.9,
"alphabet_size": 119,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'render_template')",
"label": "coarsened_k2",
"methods": 33,
"unique": 2,
"unique_ratio": 0.061,
"alphabet_size": 3,
"sore": null,
"sore_success": false
},
{
"context": "('KW', 'FUN')",
"label": "coarsened_k2",
"methods": 32,
"unique": 30,
"unique_ratio": 0.938,
"alphabet_size": 122,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'isinstance')",
"label": "coarsened_k2",
"methods": 32,
"unique": 2,
"unique_ratio": 0.062,
"alphabet_size": 6,
"sore": null,
"sore_success": false
},
{
"context": "('get', 'get')",
"label": "coarsened_k2",
"methods": 25,
"unique": 24,
"unique_ratio": 0.96,
"alphabet_size": 61,
"sore": null,
"sore_success": false
},
{
"context": "('ATTR', 'route')",
"label": "coarsened_k2",
"methods": 24,
"unique": 22,
"unique_ratio": 0.917,
"alphabet_size": 103,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('RETURN',)",
"packages": 9
},
{
"context": "('get', 'get')",
"packages": 7
},
{
"context": "('RETURN', 'render_template')",
"packages": 6
},
{
"context": "('LOOP', 'KW')",
"packages": 6
},
{
"context": "('IF', 'KW')",
"packages": 5
},
{
"context": "('KW', 'FUN')",
"packages": 4
},
{
"context": "('pop', 'pop')",
"packages": 4
},
{
"context": "('setdefault', 'setdefault')",
"packages": 4
},
{
"context": "('IF', 'isinstance')",
"packages": 4
},
{
"context": "('Flask', 'Flask')",
"packages": 3
}
]
},
"k3": {
"contexts": 529,
"sore_successes": 30,
"total_methods": 1391,
"methods_in_good": 400,
"coverage": 28.8,
"cross_package_contexts": 47,
"elapsed": 0.0,
"top_groups": [
{
"context": "('RETURN',)",
"label": "coarsened_k3",
"methods": 234,
"unique": 1,
"unique_ratio": 0.004,
"alphabet_size": 1,
"sore": "RETURN",
"sore_success": true
},
{
"context": "('IF', 'KW', 'KW')",
"label": "coarsened_k3",
"methods": 34,
"unique": 29,
"unique_ratio": 0.853,
"alphabet_size": 73,
"sore": null,
"sore_success": false
},
{
"context": "('ATTR', 'ATTR', 'route')",
"label": "coarsened_k3",
"methods": 34,
"unique": 31,
"unique_ratio": 0.912,
"alphabet_size": 74,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'render_template', 'render_template')",
"label": "coarsened_k3",
"methods": 33,
"unique": 2,
"unique_ratio": 0.061,
"alphabet_size": 3,
"sore": null,
"sore_success": false
},
{
"context": "('append', 'append')",
"label": "coarsened_k3",
"methods": 33,
"unique": 1,
"unique_ratio": 0.03,
"alphabet_size": 1,
"sore": "(append)+",
"sore_success": true
},
{
"context": "('RETURN', 'isinstance', 'isinstance')",
"label": "coarsened_k3",
"methods": 32,
"unique": 2,
"unique_ratio": 0.062,
"alphabet_size": 6,
"sore": null,
"sore_success": false
},
{
"context": "('ATTR', 'ATTR', 'FUN')",
"label": "coarsened_k3",
"methods": 31,
"unique": 28,
"unique_ratio": 0.903,
"alphabet_size": 92,
"sore": null,
"sore_success": false
},
{
"context": "('ATTR', 'route', 'route')",
"label": "coarsened_k3",
"methods": 24,
"unique": 22,
"unique_ratio": 0.917,
"alphabet_size": 103,
"sore": null,
"sore_success": false
},
{
"context": "('EXCEPTION',)",
"label": "coarsened_k3",
"methods": 21,
"unique": 1,
"unique_ratio": 0.048,
"alphabet_size": 1,
"sore": "EXCEPTION",
"sore_success": true
},
{
"context": "('Blueprint', 'Blueprint', 'ATTR')",
"label": "coarsened_k3",
"methods": 21,
"unique": 20,
"unique_ratio": 0.952,
"alphabet_size": 76,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('RETURN',)",
"packages": 9
},
{
"context": "('RETURN', 'render_template', 'render_template')",
"packages": 6
},
{
"context": "('get', 'get', 'get')",
"packages": 4
},
{
"context": "('IF', 'KW', 'RETURN')",
"packages": 4
},
{
"context": "('get', 'get', 'IF')",
"packages": 4
},
{
"context": "('IF', 'KW', 'KW')",
"packages": 4
},
{
"context": "('IF', 'isinstance', 'isinstance')",
"packages": 4
},
{
"context": "('Flask', 'Flask', 'from_mapping')",
"packages": 3
},
{
"context": "('RETURN', 'test_client', 'test_client')",
"packages": 3
},
{
"context": "('RETURN', 'get', 'get')",
"packages": 3
}
]
}
}
}

993
experiments/results/coarsen_ragsak.json Normal file
View file

@ -0,0 +1,993 @@
{
"name": "RAGSAK",
"raw": {
"seq_count": 1594,
"alphabet_size": 1724,
"alphabet_sample": [
"",
"\"Cann",
"\"Fail",
"\"k",
"(",
"(\"",
"()",
"(documen",
"(val scores: List<Double> = emptyList())",
") && s3",
")) {",
",",
".jo",
".va",
"Acti",
"AgentCapabilityDescriptor",
"AgentDecision",
"AgentExecutionContext",
"AndServerWebExchangeMatcher",
"Any",
"AssertionError",
"AssistantMessage",
"AtomicLong",
"AtomicReference",
"AuthController",
"BCryptPasswordEncoder",
"BatchId",
"BatchJobController",
"BatchJobListener",
"BatchNotFoundException"
],
"k1": {
"contexts": 510,
"sore_successes": 39,
"total_methods": 1594,
"methods_in_good": 125,
"coverage": 7.8,
"cross_package_contexts": 145,
"elapsed": 0.001,
"top_groups": [
{
"context": "('every',)",
"label": "raw_k1",
"methods": 102,
"unique": 93,
"unique_ratio": 0.912,
"alphabet_size": 232,
"sore": null,
"sore_success": false
},
{
"context": "('runTest',)",
"label": "raw_k1",
"methods": 59,
"unique": 54,
"unique_ratio": 0.915,
"alphabet_size": 131,
"sore": null,
"sore_success": false
},
{
"context": "('listOf',)",
"label": "raw_k1",
"methods": 37,
"unique": 28,
"unique_ratio": 0.757,
"alphabet_size": 99,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "raw_k1",
"methods": 31,
"unique": 21,
"unique_ratio": 0.677,
"alphabet_size": 99,
"sore": null,
"sore_success": false
},
{
"context": "('runBlocking',)",
"label": "raw_k1",
"methods": 29,
"unique": 28,
"unique_ratio": 0.966,
"alphabet_size": 115,
"sore": null,
"sore_success": false
},
{
"context": "('trim',)",
"label": "raw_k1",
"methods": 27,
"unique": 24,
"unique_ratio": 0.889,
"alphabet_size": 59,
"sore": null,
"sore_success": false
},
{
"context": "('warn',)",
"label": "raw_k1",
"methods": 26,
"unique": 4,
"unique_ratio": 0.154,
"alphabet_size": 10,
"sore": null,
"sore_success": false
},
{
"context": "('builder',)",
"label": "raw_k1",
"methods": 25,
"unique": 24,
"unique_ratio": 0.96,
"alphabet_size": 128,
"sore": null,
"sore_success": false
},
{
"context": "('info',)",
"label": "raw_k1",
"methods": 23,
"unique": 19,
"unique_ratio": 0.826,
"alphabet_size": 73,
"sore": null,
"sore_success": false
},
{
"context": "('parse',)",
"label": "raw_k1",
"methods": 18,
"unique": 18,
"unique_ratio": 1.0,
"alphabet_size": 49,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('every',)",
"packages": 18
},
{
"context": "('builder',)",
"packages": 16
},
{
"context": "('listOf',)",
"packages": 13
},
{
"context": "('runTest',)",
"packages": 12
},
{
"context": "('info',)",
"packages": 11
},
{
"context": "('mockk',)",
"packages": 11
},
{
"context": "('trim',)",
"packages": 10
},
{
"context": "('isEmpty',)",
"packages": 9
},
{
"context": "('_eps',)",
"packages": 9
},
{
"context": "('debug',)",
"packages": 9
}
]
},
"k2": {
"contexts": 918,
"sore_successes": 76,
"total_methods": 1594,
"methods_in_good": 249,
"coverage": 15.6,
"cross_package_contexts": 100,
"elapsed": 0.001,
"top_groups": [
{
"context": "('listOf', 'listOf')",
"label": "raw_k2",
"methods": 37,
"unique": 28,
"unique_ratio": 0.757,
"alphabet_size": 99,
"sore": null,
"sore_success": false
},
{
"context": "('warn', 'status')",
"label": "raw_k2",
"methods": 22,
"unique": 1,
"unique_ratio": 0.045,
"alphabet_size": 4,
"sore": "warn.status.body.ErrorResponse",
"sore_success": true
},
{
"context": "('_eps',)",
"label": "raw_k2",
"methods": 16,
"unique": 1,
"unique_ratio": 0.062,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('VectorChunk', 'mapOf')",
"label": "raw_k2",
"methods": 13,
"unique": 11,
"unique_ratio": 0.846,
"alphabet_size": 42,
"sore": null,
"sore_success": false
},
{
"context": "('ery {', 'stKnowledgeBases()')",
"label": "raw_k2",
"methods": 13,
"unique": 5,
"unique_ratio": 0.385,
"alphabet_size": 13,
"sore": null,
"sore_success": false
},
{
"context": "('productionClasses', 'filter')",
"label": "raw_k2",
"methods": 13,
"unique": 13,
"unique_ratio": 1.0,
"alphabet_size": 16,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "raw_k2",
"methods": 11,
"unique": 1,
"unique_ratio": 0.091,
"alphabet_size": 1,
"sore": "mockk",
"sore_success": true
},
{
"context": "('filesIn', 'assertTrue')",
"label": "raw_k2",
"methods": 11,
"unique": 5,
"unique_ratio": 0.455,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'ChatResponse')",
"label": "raw_k2",
"methods": 10,
"unique": 10,
"unique_ratio": 1.0,
"alphabet_size": 36,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'mockk')",
"label": "raw_k2",
"methods": 9,
"unique": 7,
"unique_ratio": 0.778,
"alphabet_size": 27,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('listOf', 'listOf')",
"packages": 13
},
{
"context": "('_eps',)",
"packages": 9
},
{
"context": "('trim', 'lowercase')",
"packages": 5
},
{
"context": "('mutableListOf', 'mutableListOf')",
"packages": 5
},
{
"context": "('defaultCapabilityId',)",
"packages": 4
},
{
"context": "('every', 'defaultCapabilityId')",
"packages": 4
},
{
"context": "('clearAllMocks',)",
"packages": 4
},
{
"context": "('VectorChunk', 'mapOf')",
"packages": 4
},
{
"context": "('Builder',)",
"packages": 4
},
{
"context": "('Any',)",
"packages": 4
}
]
},
"k3": {
"contexts": 1066,
"sore_successes": 89,
"total_methods": 1594,
"methods_in_good": 297,
"coverage": 18.6,
"cross_package_contexts": 69,
"elapsed": 0.0,
"top_groups": [
{
"context": "('warn', 'status', 'body')",
"label": "raw_k3",
"methods": 22,
"unique": 1,
"unique_ratio": 0.045,
"alphabet_size": 4,
"sore": "warn.status.body.ErrorResponse",
"sore_success": true
},
{
"context": "('_eps',)",
"label": "raw_k3",
"methods": 16,
"unique": 1,
"unique_ratio": 0.062,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('VectorChunk', 'mapOf', 'mapOf')",
"label": "raw_k3",
"methods": 13,
"unique": 11,
"unique_ratio": 0.846,
"alphabet_size": 42,
"sore": null,
"sore_success": false
},
{
"context": "('ery {', 'stKnowledgeBases()', 'gRequest(m')",
"label": "raw_k3",
"methods": 12,
"unique": 4,
"unique_ratio": 0.333,
"alphabet_size": 11,
"sore": "ery {.stKnowledgeBases().gRequest(m.(ckKnowledgeBase(re.ertEquals(Kn.\"k|eckKnowledgeBase(r.(sertEquals(K.(\"|(sertEquals(t|sertNull(o)))",
"sore_success": true
},
{
"context": "('listOf', 'listOf', 'productionFiles')",
"label": "raw_k3",
"methods": 12,
"unique": 6,
"unique_ratio": 0.5,
"alphabet_size": 9,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "raw_k3",
"methods": 11,
"unique": 1,
"unique_ratio": 0.091,
"alphabet_size": 1,
"sore": "mockk",
"sore_success": true
},
{
"context": "('File', 'writeBytes', 'byteArrayOf')",
"label": "raw_k3",
"methods": 8,
"unique": 7,
"unique_ratio": 0.875,
"alphabet_size": 25,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'ChatResponse', 'emptyList')",
"label": "raw_k3",
"methods": 7,
"unique": 7,
"unique_ratio": 1.0,
"alphabet_size": 33,
"sore": null,
"sore_success": false
},
{
"context": "('filesIn', 'assertTrue', 'hasImport')",
"label": "raw_k3",
"methods": 7,
"unique": 2,
"unique_ratio": 0.286,
"alphabet_size": 3,
"sore": "filesIn.assertTrue.(hasImport)+",
"sore_success": true
},
{
"context": "('assertEquals', 'of', 'assertFailsWith')",
"label": "raw_k3",
"methods": 7,
"unique": 3,
"unique_ratio": 0.429,
"alphabet_size": 3,
"sore": "assertEquals.(of.(assertFailsWith)?)+",
"sore_success": true
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 9
},
{
"context": "('defaultCapabilityId',)",
"packages": 4
},
{
"context": "('clearAllMocks',)",
"packages": 4
},
{
"context": "('VectorChunk', 'mapOf', 'mapOf')",
"packages": 4
},
{
"context": "('Builder',)",
"packages": 4
},
{
"context": "('Any',)",
"packages": 4
},
{
"context": "('get',)",
"packages": 4
},
{
"context": "('registerProperties',)",
"packages": 4
},
{
"context": "('assumeTrue', 'isDockerAvailable', 'start')",
"packages": 4
},
{
"context": "('listOf', 'listOf', 'forEach')",
"packages": 4
}
]
}
},
"coarsened": {
"seq_count": 1594,
"alphabet_size": 1728,
"alphabet_sample": [
"",
"\"Cann",
"\"Fail",
"\"k",
"(",
"(\"",
"()",
"(documen",
"(val scores: List<Double> = emptyList())",
") && s3",
")) {",
",",
".jo",
".va",
"ATTR",
"Acti",
"AgentCapabilityDescriptor",
"AgentDecision",
"AgentExecutionContext",
"AndServerWebExchangeMatcher",
"Any",
"AssertionError",
"AssistantMessage",
"AtomicLong",
"AtomicReference",
"AuthController",
"BCryptPasswordEncoder",
"BatchId",
"BatchJobController",
"BatchJobListener"
],
"k1": {
"contexts": 474,
"sore_successes": 33,
"total_methods": 1594,
"methods_in_good": 107,
"coverage": 6.7,
"cross_package_contexts": 128,
"elapsed": 0.0,
"top_groups": [
{
"context": "('every',)",
"label": "coarsened_k1",
"methods": 102,
"unique": 93,
"unique_ratio": 0.912,
"alphabet_size": 232,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN',)",
"label": "coarsened_k1",
"methods": 91,
"unique": 75,
"unique_ratio": 0.824,
"alphabet_size": 186,
"sore": null,
"sore_success": false
},
{
"context": "('runTest',)",
"label": "coarsened_k1",
"methods": 59,
"unique": 54,
"unique_ratio": 0.915,
"alphabet_size": 131,
"sore": null,
"sore_success": false
},
{
"context": "('listOf',)",
"label": "coarsened_k1",
"methods": 36,
"unique": 27,
"unique_ratio": 0.75,
"alphabet_size": 95,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "coarsened_k1",
"methods": 31,
"unique": 21,
"unique_ratio": 0.677,
"alphabet_size": 100,
"sore": null,
"sore_success": false
},
{
"context": "('runBlocking',)",
"label": "coarsened_k1",
"methods": 29,
"unique": 28,
"unique_ratio": 0.966,
"alphabet_size": 117,
"sore": null,
"sore_success": false
},
{
"context": "('trim',)",
"label": "coarsened_k1",
"methods": 26,
"unique": 23,
"unique_ratio": 0.885,
"alphabet_size": 57,
"sore": null,
"sore_success": false
},
{
"context": "('warn',)",
"label": "coarsened_k1",
"methods": 26,
"unique": 4,
"unique_ratio": 0.154,
"alphabet_size": 11,
"sore": null,
"sore_success": false
},
{
"context": "('builder',)",
"label": "coarsened_k1",
"methods": 24,
"unique": 23,
"unique_ratio": 0.958,
"alphabet_size": 126,
"sore": null,
"sore_success": false
},
{
"context": "('info',)",
"label": "coarsened_k1",
"methods": 23,
"unique": 19,
"unique_ratio": 0.826,
"alphabet_size": 74,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('RETURN',)",
"packages": 36
},
{
"context": "('every',)",
"packages": 18
},
{
"context": "('builder',)",
"packages": 15
},
{
"context": "('runTest',)",
"packages": 12
},
{
"context": "('listOf',)",
"packages": 12
},
{
"context": "('info',)",
"packages": 11
},
{
"context": "('mockk',)",
"packages": 11
},
{
"context": "('trim',)",
"packages": 10
},
{
"context": "('isEmpty',)",
"packages": 9
},
{
"context": "('debug',)",
"packages": 9
}
]
},
"k2": {
"contexts": 883,
"sore_successes": 69,
"total_methods": 1594,
"methods_in_good": 225,
"coverage": 14.1,
"cross_package_contexts": 108,
"elapsed": 0.001,
"top_groups": [
{
"context": "('listOf', 'listOf')",
"label": "coarsened_k2",
"methods": 36,
"unique": 27,
"unique_ratio": 0.75,
"alphabet_size": 95,
"sore": null,
"sore_success": false
},
{
"context": "('warn', 'RETURN')",
"label": "coarsened_k2",
"methods": 23,
"unique": 2,
"unique_ratio": 0.087,
"alphabet_size": 7,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN',)",
"label": "coarsened_k2",
"methods": 13,
"unique": 1,
"unique_ratio": 0.077,
"alphabet_size": 1,
"sore": "RETURN",
"sore_success": true
},
{
"context": "('VectorChunk', 'mapOf')",
"label": "coarsened_k2",
"methods": 13,
"unique": 11,
"unique_ratio": 0.846,
"alphabet_size": 42,
"sore": null,
"sore_success": false
},
{
"context": "('ery {', 'stKnowledgeBases()')",
"label": "coarsened_k2",
"methods": 13,
"unique": 5,
"unique_ratio": 0.385,
"alphabet_size": 13,
"sore": null,
"sore_success": false
},
{
"context": "('productionClasses', 'filter')",
"label": "coarsened_k2",
"methods": 13,
"unique": 13,
"unique_ratio": 1.0,
"alphabet_size": 16,
"sore": null,
"sore_success": false
},
{
"context": "('isEmpty', 'RETURN')",
"label": "coarsened_k2",
"methods": 12,
"unique": 12,
"unique_ratio": 1.0,
"alphabet_size": 59,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "coarsened_k2",
"methods": 11,
"unique": 1,
"unique_ratio": 0.091,
"alphabet_size": 1,
"sore": "mockk",
"sore_success": true
},
{
"context": "('filesIn', 'assertTrue')",
"label": "coarsened_k2",
"methods": 11,
"unique": 5,
"unique_ratio": 0.455,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'RETURN')",
"label": "coarsened_k2",
"methods": 10,
"unique": 10,
"unique_ratio": 1.0,
"alphabet_size": 15,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('listOf', 'listOf')",
"packages": 12
},
{
"context": "('isEmpty', 'RETURN')",
"packages": 9
},
{
"context": "('RETURN',)",
"packages": 9
},
{
"context": "('RETURN', 'RETURN')",
"packages": 7
},
{
"context": "('trim', 'lowercase')",
"packages": 5
},
{
"context": "('mutableListOf', 'mutableListOf')",
"packages": 5
},
{
"context": "('defaultCapabilityId',)",
"packages": 4
},
{
"context": "('every', 'defaultCapabilityId')",
"packages": 4
},
{
"context": "('clearAllMocks',)",
"packages": 4
},
{
"context": "('VectorChunk', 'mapOf')",
"packages": 4
}
]
},
"k3": {
"contexts": 1062,
"sore_successes": 82,
"total_methods": 1594,
"methods_in_good": 292,
"coverage": 18.3,
"cross_package_contexts": 74,
"elapsed": 0.0,
"top_groups": [
{
"context": "('warn', 'RETURN', 'status')",
"label": "coarsened_k3",
"methods": 22,
"unique": 1,
"unique_ratio": 0.045,
"alphabet_size": 5,
"sore": "warn.RETURN.status.body.ErrorResponse",
"sore_success": true
},
{
"context": "('RETURN',)",
"label": "coarsened_k3",
"methods": 13,
"unique": 1,
"unique_ratio": 0.077,
"alphabet_size": 1,
"sore": "RETURN",
"sore_success": true
},
{
"context": "('VectorChunk', 'mapOf', 'mapOf')",
"label": "coarsened_k3",
"methods": 13,
"unique": 11,
"unique_ratio": 0.846,
"alphabet_size": 42,
"sore": null,
"sore_success": false
},
{
"context": "('ery {', 'stKnowledgeBases()', 'gRequest(m')",
"label": "coarsened_k3",
"methods": 12,
"unique": 4,
"unique_ratio": 0.333,
"alphabet_size": 11,
"sore": "ery {.stKnowledgeBases().gRequest(m.(ckKnowledgeBase(re.ertEquals(Kn.\"k|eckKnowledgeBase(r.(sertEquals(K.(\"|(sertEquals(t|sertNull(o)))",
"sore_success": true
},
{
"context": "('listOf', 'listOf', 'productionFiles')",
"label": "coarsened_k3",
"methods": 12,
"unique": 6,
"unique_ratio": 0.5,
"alphabet_size": 9,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "coarsened_k3",
"methods": 11,
"unique": 1,
"unique_ratio": 0.091,
"alphabet_size": 1,
"sore": "mockk",
"sore_success": true
},
{
"context": "('parseStorageUri', 'RETURN', 'RETURN')",
"label": "coarsened_k3",
"methods": 8,
"unique": 5,
"unique_ratio": 0.625,
"alphabet_size": 14,
"sore": null,
"sore_success": false
},
{
"context": "('File', 'writeBytes', 'byteArrayOf')",
"label": "coarsened_k3",
"methods": 8,
"unique": 7,
"unique_ratio": 0.875,
"alphabet_size": 25,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'ChatResponse', 'emptyList')",
"label": "coarsened_k3",
"methods": 7,
"unique": 7,
"unique_ratio": 1.0,
"alphabet_size": 33,
"sore": null,
"sore_success": false
},
{
"context": "('filesIn', 'assertTrue', 'hasImport')",
"label": "coarsened_k3",
"methods": 7,
"unique": 2,
"unique_ratio": 0.286,
"alphabet_size": 3,
"sore": "filesIn.assertTrue.(hasImport)+",
"sore_success": true
}
],
"cross_pkg_examples": [
{
"context": "('RETURN',)",
"packages": 9
},
{
"context": "('defaultCapabilityId',)",
"packages": 4
},
{
"context": "('clearAllMocks',)",
"packages": 4
},
{
"context": "('VectorChunk', 'mapOf', 'mapOf')",
"packages": 4
},
{
"context": "('Builder',)",
"packages": 4
},
{
"context": "('Any',)",
"packages": 4
},
{
"context": "('get',)",
"packages": 4
},
{
"context": "('registerProperties',)",
"packages": 4
},
{
"context": "('assumeTrue', 'isDockerAvailable', 'start')",
"packages": 4
},
{
"context": "('listOf', 'listOf', 'forEach')",
"packages": 4
}
]
}
}
}