tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feature/treesitter-tag-queries #2

Open
tobi wants to merge 78 commits from feature/treesitter-tag-queries into main
pull from: feature/treesitter-tag-queries
merge into: tobi:main
Conversation 0 Commits 78 Files changed 246 +9 -10
3 changed files with 9 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 4a506cd39b - Show all commits

2
bex/tag_preprocessor/analyze.py
View file

@ -51,8 +51,8 @@ def _match_glob(filepath, pattern):
IMPORT_PATTERNS = [ IMPORT_PATTERNS = [
re.compile(r"^\s*import\s+"), re.compile(r"^\s*import\s+"),
re.compile(r"^\s*from\s+"), re.compile(r"^\s*from\s+"),
re.compile(r"^\s*require\s+"),
re.compile(r"^\s*require_relative\s+"), re.compile(r"^\s*require_relative\s+"),
re.compile(r"^\s*require\s+"),
re.compile(r"^\s*#\s*include\s+"), re.compile(r"^\s*#\s*include\s+"),
re.compile(r"^\s*use\s+"), re.compile(r"^\s*use\s+"),
re.compile(r"^\s*include\s+"), re.compile(r"^\s*include\s+"),

4
bex/tag_preprocessor/code.py
View file

@ -51,7 +51,7 @@ BEHAVIORAL_PREFIXES = (
"module", "module",
) )
CALL_PREFIXES = ("function", "reference.call", "reference.class") CALL_PREFIXES = ("function.call", "function.method.call", "reference.call", "reference.class", "constructor", "function")
def _extract_call_tokens(seq): def _extract_call_tokens(seq):
"""Extract ordered call-like tokens from a method sequence. """Extract ordered call-like tokens from a method sequence.
@ -107,8 +107,6 @@ def _find_arglist_node(parent):
for child in parent.children: for child in parent.children:
if child.type in ("argument_list", "arguments"): if child.type in ("argument_list", "arguments"):
return child return child
if child.type == "template_string":
return child
return None return None

13
docs/pipeline-overview.txt
View file

@ -4,7 +4,7 @@
│ (.py .js .ts .kt .rb .go │ │ (.py .js .ts .kt .rb .go │
│ .rs .java .c .cpp .h) │ │ .rs .java .c .cpp .h) │
└──────────┬────────────────────┘ └──────────┬────────────────────┘
│ scan_directory() + .gitignore │ scan_directory()
▼ ▼
┌───────────────────────────────┐ ┌───────────────────────────────┐
│ Files grouped by extension │ │ Files grouped by extension │
@ -68,11 +68,12 @@
│ │ └───────────┬────────────┘ │ │ │ └───────────┬────────────┘ │
│ ▼ ▼ │ │ ▼ ▼ │
│ ┌──────────────────────────────────────────────────┐ │ │ ┌──────────────────────────────────────────────────┐ │
│ │ infer_ensemble(symbol_seqs) │ │ │ │ infer_ensemble(symbol_seqs, min_coverage=0.8) │ │
│ │ ├── CRX (fast, unordered) │ │ │ │ ├── CRX (fast, unordered) │ │
│ │ ├── iDRegEx (ordered regex) │ │ │ │ ├── iDRegEx (ordered regex) │ │
│ │ └── kOREInference (noisy, probabilistic) │ │ │ │ └── kOREInference (noisy, probabilistic) │ │
│ │ └── pick best by MDL score │ │ │ │ └── pick best by MDL score │ │
│ │ └── core/outlier split via _find_core(0.8) │ │
│ └──────────────────────┬───────────────────────────┘ │ │ └──────────────────────┬───────────────────────────┘ │
│ ▼ │ │ ▼ │
│ (label, result, count, meta) │ │ (label, result, count, meta) │