grammar-inference-engine/tests/test_analyze.py

"""Tests for tag-preprocessor orchestrator (analyze.py)."""

from pathlib import Path
import tempfile
import sys

sys.path.insert(0, str(Path(__file__).parent.parent))

from bex.tag_preprocessor.analyze import (
    scan_directory, frequency_filter, infer, analyze_directory, _match_glob,
)


def test_scan_directory_empty():
    with tempfile.TemporaryDirectory() as td:
        result = scan_directory(td)
        assert result == {}
    print("  PASS test_scan_directory_empty")


def test_scan_directory_groups_by_extension():
    with tempfile.TemporaryDirectory() as td:
        d = Path(td)
        (d / "a.py").write_text("x = 1")
        (d / "b.py").write_text("y = 2")
        (d / "c.js").write_text("let x = 1;")
        (d / "d.rs").write_text("fn main() {}")
        result = scan_directory(td)
        assert ".py" in result
        assert ".js" in result
        assert ".rs" in result
        assert len(result[".py"]) == 2
        assert len(result[".js"]) == 1
        assert len(result[".rs"]) == 1
    print("  PASS test_scan_directory_groups_by_extension")


def test_scan_directory_skips_unsupported():
    with tempfile.TemporaryDirectory() as td:
        d = Path(td)
        (d / "f.py").write_text("x = 1")
        (d / "f.txt").write_text("hello")
        (d / "f.md").write_text("# doc")
        result = scan_directory(td)
        assert ".py" in result
        assert ".txt" not in result
        assert ".md" not in result
    print("  PASS test_scan_directory_skips_unsupported")


def test_scan_directory_nested():
    with tempfile.TemporaryDirectory() as td:
        d = Path(td)
        (d / "sub").mkdir()
        (d / "sub" / "a.py").write_text("x = 1")
        (d / "b.py").write_text("y = 2")
        result = scan_directory(td)
        assert len(result[".py"]) == 2
    print("  PASS test_scan_directory_nested")


def test_scan_directory_skips_build_dirs():
    with tempfile.TemporaryDirectory() as td:
        d = Path(td)
        (d / "src" / "main").mkdir(parents=True)
        (d / "build" / "reports").mkdir(parents=True)
        (d / "node_modules" / "pkg").mkdir(parents=True)
        (d / "src" / "main" / "app.py").write_text("x = 1")
        (d / "build" / "reports" / "report.js").write_text("let x = 1;")
        (d / "node_modules" / "pkg" / "index.js").write_text("let y = 2;")
        result = scan_directory(td)
        assert ".py" in result
        assert ".js" not in result
        assert len(result[".py"]) == 1
    print("  PASS test_scan_directory_skips_build_dirs")


def test_frequency_filter_nothing_to_filter():
    sequences = [
        [("function", "foo", 1), ("keyword.return", "return", 2)],
        [("function", "bar", 1), ("keyword.return", "return", 2)],
    ]
    filtered = frequency_filter(sequences, min_coverage=0.5)
    assert len(filtered) == 2
    assert filtered == sequences
    print("  PASS test_frequency_filter_nothing_to_filter")


def test_frequency_filter_removes_infrequent_symbol():
    sequences = [
        [("keyword.return", "return", 1)],
        [("keyword.return", "return", 1)],
        [("function", "rare_fn", 1)],
    ]
    filtered = frequency_filter(sequences, min_coverage=0.67)
    assert len(filtered) == 3
    assert len(filtered[0]) == 1  # "return" kept
    assert len(filtered[1]) == 1  # "return" kept
    assert len(filtered[2]) == 0  # "rare_fn" removed
    print("  PASS test_frequency_filter_removes_infrequent_symbol")


def test_frequency_filter_edge_empty_sequences():
    assert frequency_filter([], min_coverage=0.5) == []
    assert frequency_filter([[], []], min_coverage=0.5) == [[], []]
    print("  PASS test_frequency_filter_edge_empty_sequences")


def test_infer_returns_ensemble_dict():
    with tempfile.TemporaryDirectory() as td:
        d = Path(td)
        (d / "a.py").write_text("def foo():\n    return 1")
        (d / "b.py").write_text("def bar():\n    return 2")
        result = infer([str(d / "a.py"), str(d / "b.py")], ".py", min_coverage=0.5)
        assert isinstance(result, dict)
        assert "best" in result
        assert "all" in result
        assert "why" in result
    print("  PASS test_infer_returns_ensemble_dict")


def test_match_glob():
    assert _match_glob("/repo/src/main/app.kt", "**/src/main/**")
    assert _match_glob("/repo/src/main/org/app.kt", "**/src/main/**")
    assert _match_glob("/repo/src/main/deep/nested/app.kt", "**/src/main/**")
    assert not _match_glob("/repo/src/test/app.kt", "**/src/main/**")
    assert not _match_glob("/repo/build/app.kt", "**/src/main/**")
    print("  PASS test_match_glob")


def test_analyze_directory_include_glob():
    with tempfile.TemporaryDirectory() as td:
        d = Path(td)
        (d / "src" / "main").mkdir(parents=True)
        (d / "src" / "test").mkdir(parents=True)
        (d / "src" / "main" / "prod.py").write_text("x = 1")
        (d / "src" / "test" / "test_prod.py").write_text("y = 2")
        results = analyze_directory(td, include="**/src/main/**")
        assert ".py" in results
        assert len(results[".py"]["all"]) >= 1
    print("  PASS test_analyze_directory_include_glob")


def test_infer_low_coverage_filters_noise():
    with tempfile.TemporaryDirectory() as td:
        d = Path(td)
        (d / "common.py").write_text(
            "def setup():\n    pass\ndef teardown():\n    pass"
        )
        (d / "rare.py").write_text(
            "def setup():\n    pass\ndef one_off():\n    raise Exception('boom')"
        )
        result = infer(
            [str(d / "common.py"), str(d / "rare.py")], ".py", min_coverage=0.6
        )
        assert result["best"] is not None
        assert result["best"]["grammar"] is not None
    print("  PASS test_infer_low_coverage_filters_noise")


def run_all():
    tests = [
        test_scan_directory_empty,
        test_scan_directory_groups_by_extension,
        test_scan_directory_skips_unsupported,
        test_scan_directory_nested,
        test_scan_directory_skips_build_dirs,
        test_match_glob,
        test_analyze_directory_include_glob,
        test_frequency_filter_nothing_to_filter,
        test_frequency_filter_removes_infrequent_symbol,
        test_frequency_filter_edge_empty_sequences,
        test_infer_returns_ensemble_dict,
        test_infer_low_coverage_filters_noise,
    ]
    passed = 0
    failed = 0
    for t in tests:
        try:
            t()
            passed += 1
        except Exception as e:
            print(f"  FAIL {t.__name__}: {e}")
            failed += 1
    print(f"\n{passed} passed, {failed} failed")


if __name__ == "__main__":
    run_all()