From 9d206f586fe503f57586462a35ef5c00256165cc Mon Sep 17 00:00:00 2001 From: tobjend Date: Sun, 12 Jul 2026 20:18:33 +0200 Subject: [PATCH] docs: Round 19 evaluation plan + golden config - Plan: match rate audit, decomposition A/B, distributional A/B, fragment quality - Golden config: best-known values for all heuristic parameters - Single source of truth in bex/golden_config.py --- bex/golden_config.py | 64 ++++++++++++++++++ experiments/ROUND19_PLAN.md | 125 ++++++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 bex/golden_config.py create mode 100644 experiments/ROUND19_PLAN.md diff --git a/bex/golden_config.py b/bex/golden_config.py new file mode 100644 index 0000000..dd276ac --- /dev/null +++ b/bex/golden_config.py @@ -0,0 +1,64 @@ +"""Golden config — best-known heuristic values for the grammar inference pipeline. + +This file is the single source of truth for optimal parameter values. +Import it when running evaluations or setting up new codebases. + +Last updated: Round 19 (Crucio evaluation) +""" + +# Core pipeline +MIN_COVERAGE = 0.05 # BEX outlier threshold (was 0.8, too aggressive) +MIN_METHODS = 3 # Min methods per group (was 5, lost too many) +SCORING_METHOD = "langsize" # Language Size (Bex et al. arXiv:1004.2372) + +# Grouping +SLICE = "package" # Per-directory (not flat, not reduce) +SPLIT_MIXED = True # Recursive split by first symbol +MAX_DEPTH = 3 # Max recursion depth for split +CLUSTER_METHOD = "first-symbol" # Split method (distributional = no improvement) + +# Quality filter +MIN_STRUCTURE = 0.5 # Drop flat bags (noise) +MAX_MDL = 200.0 # Drop high-score grammars + +# Decomposition (Crucio Phase 2) +DECOMPOSE = True # Break long sequences into fragments +MAX_SEQ_LENGTH = 4 # Max fragment length (5 = too aggressive, 4 = sweet spot) + +# Algorithms +CRX_METHOD = "standard" # Standard CRX (refined = trivial on large groups) +INCLUDE_KORE = False # kORE = slow, no improvement +INCLUDE_IDREGEX = False # iDRegEx = slow, rare benefit +IDREGEX_REFINE = False # iDRegEx refinement = rare benefit + + +def get_golden_config(): + """Return golden config as a dict for easy passing to analyze_directory().""" + return { + "min_coverage": MIN_COVERAGE, + "min_methods": MIN_METHODS, + "method": SCORING_METHOD, + "slice": SLICE, + "split_mixed": SPLIT_MIXED, + "cluster_method": CLUSTER_METHOD, + "min_structure": MIN_STRUCTURE, + "decompose": DECOMPOSE, + "max_seq_length": MAX_SEQ_LENGTH, + "crx_method": CRX_METHOD, + "include_kore": INCLUDE_KORE, + "include_idregex": INCLUDE_IDREGEX, + "idregex_refine": IDREGEX_REFINE, + } + + +def apply_to_directory(dir_path, include=None, main_only=True, **overrides): + """Run analyze_directory with golden config. Override any parameter.""" + from bex.tag_preprocessor.analyze import analyze_directory + config = get_golden_config() + config.update(overrides) + return analyze_directory( + dir_path, + include=include, + main_only=main_only, + **config, + ) diff --git a/experiments/ROUND19_PLAN.md b/experiments/ROUND19_PLAN.md new file mode 100644 index 0000000..b8e4521 --- /dev/null +++ b/experiments/ROUND19_PLAN.md @@ -0,0 +1,125 @@ +# Round 19: Crucio Evaluation & Golden Config + +**Goal:** Quantify whether Crucio-inspired changes (distributional clustering, +decomposition forest) actually improved grammar quality. Define the "golden config" +with best heuristic values. + +**Open questions:** +1. Do the 21 RAGSAK / 22 FastAPI / 11 kotlinx patterns that survive `min_structure=0.5` + actually match their input sequences? +2. Did decomposition improve pattern quality, or just create more patterns? +3. Did distributional clustering help when it differs from first-symbol? +4. What's the compression ratio (grammar size vs input size)? + +--- + +## Experiment 1: Match Rate Audit + +**What:** For each YAML entry, check what fraction of input sequences the grammar +actually accepts via `_matches()`. + +**Method:** +1. Run pipeline with golden config on all 3 codebases +2. For each surviving pattern (min_structure=0.5), reproduce the input sequences +3. Check `_matches(grammar, seq)` for each input sequence +4. Report: match rate, grammar size, compression ratio + +**Metrics:** +- `match_rate` = sequences accepted / total sequences (1.0 = perfect) +- `compression` = len(grammar) / total_input_chars (< 1.0 = good) +- `structure_score` = grammar_structure_score(grammar) + +--- + +## Experiment 2: Decomposition A/B + +**What:** Compare pipeline output WITH decomposition ON vs OFF. + +**Method:** +1. Run with `--decompose --max-seq-length 4` (current config) +2. Run without `--decompose` +3. Compare: pattern count, avg match rate, avg structure score, avg compression + +**Hypothesis:** Decomposition helps diverse codebases (RAGSAK, FastAPI) but +hurts already-structured ones (kotlinx.coroutines). + +--- + +## Experiment 3: Distributional vs First-Symbol A/B + +**What:** Compare `--cluster-method distributional` vs `--cluster-method first-symbol`. + +**Method:** +1. Run with `--cluster-method distributional` +2. Run with `--cluster-method first-symbol` (current default) +3. Compare on the groups where they differ + +**Hypothesis:** Distributional clustering doesn't help because the contexts +are already too specific per-package. + +--- + +## Experiment 4: Decomposition Fragment Quality + +**What:** Are decomposition fragments meaningful sub-patterns or noise? + +**Method:** +1. Take decomposed fragments from a diverse package (e.g., RAGSAK agents/rag/embabel) +2. Check match rate of each fragment's grammar +3. Check if fragments capture real sub-patterns (e.g., "return path" vs "error path") + +--- + +## Golden Config + +The "golden config" captures our best-known heuristic values: + +```python +GOLDEN_CONFIG = { + # Core pipeline + "min_coverage": 0.05, # BEX outlier threshold (was 0.8, too aggressive) + "min_methods": 3, # Min methods per group (was 5, lost too many) + "method": "langsize", # Scoring: Language Size (Bex et al.) + + # Grouping + "slice": "package", # Per-directory (not flat, not reduce) + "split_mixed": True, # Recursive split by first symbol + "max_depth": 3, # Max recursion depth for split + "cluster_method": "first-symbol", # Split method (distributional = no improvement) + + # Quality filter + "min_structure": 0.5, # Drop flat bags (noise) + "max_mdl": 200.0, # Drop high-score grammars + + # Decomposition (Crucio Phase 2) + "decompose": True, # Break long sequences into fragments + "max_seq_length": 4, # Max fragment length (5 = too aggressive, 4 = sweet spot) + + # Algorithms + "crx_method": "standard", # Standard CRX (refined = trivial on large groups) + "include_kore": False, # kORE = slow, no improvement + "include_idregex": False, # iDRegEx = slow, rare benefit + "idregex_refine": False, # iDRegEx refinement = rare benefit +} +``` + +**Rationale for each value:** +- `min_coverage=0.05`: At 0.8, almost all symbols filtered out. 0.05 sees real vocabulary. +- `min_methods=3`: At 5, lost 9 FastAPI grammars. 3 is safe minimum. +- `min_structure=0.5`: Below this, patterns are flat bags (noise). +- `decompose=True`: Helps RAGSAK 7×, FastAPI 1.8×. Hurts kotlinx.coroutines. +- `max_seq_length=4`: At 5, fragments too short. 4 captures meaningful sub-patterns. +- `cluster_method="first-symbol"`: Distributional clustering showed no improvement. +- `crx_method="standard"`: Refined CRX produces trivial output 36% of the time. + +--- + +## Execution Order + +1. Write plan + golden config → commit +2. Experiment 1: Match rate audit (all 3 codebases) +3. Experiment 2: Decomposition A/B +4. Experiment 3: Distributional vs first-symbol A/B +5. Experiment 4: Decomposition fragment quality +6. Update golden config if needed +7. Summarize results and implications