symgrep-codesearch 0.9.1__tar.gz → 0.9.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {symgrep_codesearch-0.9.1/src/symgrep_codesearch.egg-info → symgrep_codesearch-0.9.2}/PKG-INFO +1 -1
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/pyproject.toml +1 -1
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/capability_registry.py +49 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/semantic.py +30 -9
- symgrep_codesearch-0.9.2/src/symgrep/semantic_ivf.py +108 -0
- symgrep_codesearch-0.9.2/src/symgrep/semantic_sqlite.py +295 -0
- symgrep_codesearch-0.9.2/src/symgrep/semantic_sqlite_config.py +19 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2/src/symgrep_codesearch.egg-info}/PKG-INFO +1 -1
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/SOURCES.txt +3 -0
- symgrep_codesearch-0.9.2/tests/test_semantic_census_pruning_pytest.py +42 -0
- symgrep_codesearch-0.9.2/tests/test_sqlite_ivf_retrieval_pytest.py +278 -0
- symgrep_codesearch-0.9.1/src/symgrep/semantic_sqlite.py +0 -156
- symgrep_codesearch-0.9.1/src/symgrep/semantic_sqlite_config.py +0 -7
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/LICENSE +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/README.md +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/setup.cfg +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/__init__.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/absence_adjudication_writer.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/archives.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/blocks.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/capability_registry_data.jsonl +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/capability_surface_checks.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/classfile.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/cli.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/data_symbols.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/docsymbols.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/embedding_process.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/enclosing.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/glob_recall.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/mcp_server.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/mcp_structured.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/report.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/resolve.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/resolve_js.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/resolve_python.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/ripgrep.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/rss_ceiling.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/runtime_paths.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/structural.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/structure.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/dependency_links.txt +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/entry_points.txt +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/requires.txt +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/top_level.txt +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_absence_adjudication_authorship_pytest.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_absence_adjudication_binding_pytest.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_blocks_extraction.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_blocks_retrieval.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_bulk_describe.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_capability_documentation_guard_pytest.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_capability_surface_parity_guard_pytest.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_classfile.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_codex_hook_payload.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_data_symbols.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_describe_lane_confinement.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_doc_symbols.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_embedding_residency_pytest.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_file_structure.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_freshness.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_honest_recall.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_http_ensure.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_http_rss_shutdown_pytest.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_index_reconstruct.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_index_scope.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_index_worker.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_mcp_error_log.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_mcp_registry_behavior_pytest.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_mcp_structured.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_mcp_thread_safety.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_mcp_version.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_name_uniqueness.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_nudge_classify.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_nudge_runaway.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_report_filters.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_resolve.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_resolve_js.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_resolve_python.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_resolve_r2.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_review_regressions.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_rss_ceiling_pytest.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_semantic_freshness_signal.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_semantic_glob_lang.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_sites_and_symbols.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_sqlite_streaming_retrieval_pytest.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_structural.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_symbol_vocabulary.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_ts_fallback_loud.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_ts_symbol_coverage.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_usage.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_veccore_adoption.py +0 -0
- {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_worker_lock_recovery_pytest.py +0 -0
{symgrep_codesearch-0.9.1/src/symgrep_codesearch.egg-info → symgrep_codesearch-0.9.2}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: symgrep-codesearch
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.2
|
|
4
4
|
Summary: Code search with honest recall — structure-aware grep that tells you what it does not know, plus search by meaning
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Project-URL: Repository, https://github.com/ashdamle/symgrep
|
|
@@ -41,7 +41,7 @@ name = "symgrep-codesearch"
|
|
|
41
41
|
# semantic search. Minor bump (dependency surface changed; no breaking import/CLI/MCP change).
|
|
42
42
|
# 0.5.0: added the public `file_structure` library API (+ its MCP tool and `symgrep structure` CLI), import
|
|
43
43
|
# module-name normalization, and the FileStructure `connects` list (structure schema v2).
|
|
44
|
-
version = "0.9.
|
|
44
|
+
version = "0.9.2"
|
|
45
45
|
description = "Code search with honest recall — structure-aware grep that tells you what it does not know, plus search by meaning"
|
|
46
46
|
readme = "README.md"
|
|
47
47
|
# 3.11+ : the maintained grammar pack (tree-sitter-language-pack, a CORE dep below) ships abi3 wheels for
|
|
@@ -7,6 +7,7 @@ import inspect
|
|
|
7
7
|
import hashlib
|
|
8
8
|
from datetime import datetime, timezone
|
|
9
9
|
import json
|
|
10
|
+
from enum import StrEnum
|
|
10
11
|
from typing import Any, Optional, List
|
|
11
12
|
|
|
12
13
|
CAPABILITY_DATA_RESOURCE = 'capability_registry_data.jsonl'
|
|
@@ -53,6 +54,35 @@ RUNTIME_ENVIRONMENT_SETTINGS = (
|
|
|
53
54
|
'SYMGREP_SQLITE_BUSY_MS', 'symgrep.semantic_sqlite_config:SQLITE_READER_BUSY_MS',
|
|
54
55
|
'SQLite semantic reader busy timeout in milliseconds while concurrent indexing '
|
|
55
56
|
'holds a database lock. WAL readers otherwise use committed statement snapshots.'),
|
|
57
|
+
RuntimeEnvironmentSetting(
|
|
58
|
+
'SYMGREP_IVF_ENABLED', 'symgrep.semantic_sqlite_config:IVF_ENABLED',
|
|
59
|
+
'Persistent SQLite inverted-file vector retrieval. Zero disables ANN and logs one '
|
|
60
|
+
'honest warning before exact streaming cosine fallback. Centroids and postings live '
|
|
61
|
+
'in the committed database; approximate candidates are reranked on original vectors. '
|
|
62
|
+
'Reindex prepares live postings. Reader pooling and atomic generation swaps stay bounded.'),
|
|
63
|
+
RuntimeEnvironmentSetting(
|
|
64
|
+
'SYMGREP_IVF_BUCKETS', 'symgrep.semantic_sqlite_config:IVF_BUCKET_COUNT',
|
|
65
|
+
'Centroid count for the inverted-file index. Reindex atomically rebuilds when '
|
|
66
|
+
'training settings differ from the durable model signature. More buckets '
|
|
67
|
+
'narrow candidate scans and require recall measurement.'),
|
|
68
|
+
RuntimeEnvironmentSetting(
|
|
69
|
+
'SYMGREP_IVF_PROBES', 'symgrep.semantic_sqlite_config:IVF_PROBE_COUNT',
|
|
70
|
+
'Initial nearest centroid buckets for symbol queries without a kind filter. Other unit/kind '
|
|
71
|
+
'scopes use every bucket and log exact coverage to preserve recall. More probes improve '
|
|
72
|
+
'approximate recall; sparse results expand probes. Measure complete recall@k against exact streaming search.'),
|
|
73
|
+
RuntimeEnvironmentSetting(
|
|
74
|
+
'SYMGREP_IVF_TRAINING_ROWS', 'symgrep.semantic_sqlite_config:IVF_TRAINING_ROWS',
|
|
75
|
+
'Maximum reservoir sample rows used to train centroids; bounds training memory '
|
|
76
|
+
'independently of corpus size.'),
|
|
77
|
+
RuntimeEnvironmentSetting(
|
|
78
|
+
'SYMGREP_IVF_TRAINING_ITERATIONS', 'symgrep.semantic_sqlite_config:IVF_TRAINING_ITERATIONS',
|
|
79
|
+
'Spherical centroid training iterations for a new index.'),
|
|
80
|
+
RuntimeEnvironmentSetting(
|
|
81
|
+
'SYMGREP_IVF_ASSIGNMENT_ROWS', 'symgrep.semantic_sqlite_config:IVF_ASSIGNMENT_ROWS',
|
|
82
|
+
'Maximum vector rows per centroid assignment batch during training and reindex.'),
|
|
83
|
+
RuntimeEnvironmentSetting(
|
|
84
|
+
'SYMGREP_IVF_RANDOM_SEED', 'symgrep.semantic_sqlite_config:IVF_RANDOM_SEED',
|
|
85
|
+
'Reproducible reservoir sampling and centroid initialization seed for new indexes.'),
|
|
56
86
|
)
|
|
57
87
|
|
|
58
88
|
@dataclass(frozen=True)
|
|
@@ -92,6 +122,25 @@ CAPABILITIES = load_symgrep_capability_declarations()
|
|
|
92
122
|
def capability_by_id(identity):
|
|
93
123
|
return next(cap for cap in CAPABILITIES if cap.id == identity)
|
|
94
124
|
|
|
125
|
+
|
|
126
|
+
SEMANTIC_UNIT_ARGUMENT = next(parameter for parameter in capability_by_id('search').params
|
|
127
|
+
if parameter.name == 'unit')
|
|
128
|
+
SemanticSearchUnit = StrEnum('SemanticSearchUnit',
|
|
129
|
+
{choice.upper(): choice for choice in SEMANTIC_UNIT_ARGUMENT.choices})
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def parse_semantic_search_unit(value):
|
|
133
|
+
"""Parse the declared unit grammar; aliases outside the public enum are errors.
|
|
134
|
+
|
|
135
|
+
In particular, plural spellings are not inferred from their meaning. The CLI,
|
|
136
|
+
MCP schema and direct store callers share the registry's canonical members.
|
|
137
|
+
"""
|
|
138
|
+
try:
|
|
139
|
+
return SemanticSearchUnit(value)
|
|
140
|
+
except (TypeError, ValueError) as error:
|
|
141
|
+
raise ValueError(f'Invalid semantic unit {value!r}; expected one of '
|
|
142
|
+
f'{SEMANTIC_UNIT_ARGUMENT.choices!r}') from error
|
|
143
|
+
|
|
95
144
|
def capability_description(capability):
|
|
96
145
|
"""Render full canonical prose and every parameter's semantics."""
|
|
97
146
|
return '\n\n'.join([capability.summary, capability.description] + [f'{p.name}: {p.help}' for p in capability.params])
|
|
@@ -534,6 +534,24 @@ def _is_generated(p: pathlib.Path, rp: pathlib.Path) -> bool:
|
|
|
534
534
|
return False
|
|
535
535
|
|
|
536
536
|
|
|
537
|
+
def _iter_semantic_census_files(census_root):
|
|
538
|
+
"""Yield exactly the existing indexable census, pruning only rejected subtrees."""
|
|
539
|
+
root_has_mkdocs = (census_root / "mkdocs.yml").exists()
|
|
540
|
+
for directory, directories, filenames in os.walk(census_root, followlinks=False):
|
|
541
|
+
directories[:] = [name for name in directories
|
|
542
|
+
if name not in SKIP_DIRS and not name.startswith(".")
|
|
543
|
+
and not (name == "site" and root_has_mkdocs)]
|
|
544
|
+
for filename in filenames:
|
|
545
|
+
# Match Path.suffix for dotfiles and trailing dots before allocating Paths.
|
|
546
|
+
suffix_separator = filename.rfind(".")
|
|
547
|
+
suffix = filename[suffix_separator:].lower() if 0 < suffix_separator < len(filename) - 1 else ""
|
|
548
|
+
if suffix not in INDEXABLE_EXTS:
|
|
549
|
+
continue
|
|
550
|
+
candidate = pathlib.Path(directory) / filename
|
|
551
|
+
if candidate.is_file() and not _is_generated(candidate, census_root):
|
|
552
|
+
yield candidate
|
|
553
|
+
|
|
554
|
+
|
|
537
555
|
def _looks_minified(text: str) -> bool:
|
|
538
556
|
return any(len(line) > MINIFIED_LINE_CHARS for line in text.splitlines())
|
|
539
557
|
|
|
@@ -672,6 +690,8 @@ def search_by_meaning(query: str, roots: List[str], k: int = 8,
|
|
|
672
690
|
functions and blocks in one list. The returned tuple shape is identical for every unit; block span +
|
|
673
691
|
provenance ride `search_by_meaning.last_hits` (aligned to the return) so the tuple contract never changes.
|
|
674
692
|
"""
|
|
693
|
+
from .capability_registry import parse_semantic_search_unit
|
|
694
|
+
unit = parse_semantic_search_unit(unit).value
|
|
675
695
|
root_abs = [os.path.realpath(r) for r in (roots or [])]
|
|
676
696
|
|
|
677
697
|
def in_scope(p: str) -> bool:
|
|
@@ -797,14 +817,9 @@ def search_by_meaning(query: str, roots: List[str], k: int = 8,
|
|
|
797
817
|
rp = pathlib.Path(_root)
|
|
798
818
|
if not rp.is_dir():
|
|
799
819
|
continue
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
for filename in filenames:
|
|
804
|
-
p = pathlib.Path(directory) / filename
|
|
805
|
-
if p.suffix.lower() in INDEXABLE_EXTS and p.is_file() and not _is_generated(p, rp) \
|
|
806
|
-
and os.path.realpath(p) not in indexed_paths:
|
|
807
|
-
not_indexed += 1
|
|
820
|
+
for p in _iter_semantic_census_files(rp):
|
|
821
|
+
if os.path.realpath(p) not in indexed_paths:
|
|
822
|
+
not_indexed += 1
|
|
808
823
|
search_by_meaning.last_scope = {"in_index": in_index, "out_of_scope": out_of_scope,
|
|
809
824
|
"bad_dim": 0, "roots": root_abs,
|
|
810
825
|
"files_checked": len(file_stamp), "stale_files": stale,
|
|
@@ -1049,9 +1064,10 @@ def _embed_live(con, fns, gone=frozenset()) -> int:
|
|
|
1049
1064
|
con.execute("SELECT key, qual, path, lineno, what, stamp, unit, parent_key, end_lineno, signature, "
|
|
1050
1065
|
"kind FROM fn WHERE what IS NOT NULL AND what != ''")}
|
|
1051
1066
|
live_keys = {f["key"] for f in fns} & set(book)
|
|
1067
|
+
store, core = _core_for(str(index_path()))
|
|
1052
1068
|
if not live_keys and not gone:
|
|
1069
|
+
store.prepare_ann_index(COLLECTION)
|
|
1053
1070
|
return 0
|
|
1054
|
-
store, core = _core_for(str(index_path()))
|
|
1055
1071
|
# EVERY live description goes to veccore; its content-sha re-embeds ONLY the drifted text (a re-described
|
|
1056
1072
|
# function whose sentence CHANGED re-embeds; an unchanged one is skipped, $0). Filtering to 'vector
|
|
1057
1073
|
# missing' here would WRONGLY skip a re-described function whose stored vector still holds its OLD
|
|
@@ -1078,6 +1094,7 @@ def _embed_live(con, fns, gone=frozenset()) -> int:
|
|
|
1078
1094
|
"block_start": lineno if is_spanned else None,
|
|
1079
1095
|
"block_end": end_lineno if is_spanned else None}})
|
|
1080
1096
|
if not items:
|
|
1097
|
+
store.prepare_ann_index(COLLECTION)
|
|
1081
1098
|
return 0
|
|
1082
1099
|
vc = _veccore()
|
|
1083
1100
|
embedded = 0
|
|
@@ -1087,4 +1104,8 @@ def _embed_live(con, fns, gone=frozenset()) -> int:
|
|
|
1087
1104
|
result = core.embed_upsert(COLLECTION, items[start:start + EMBED_UPSERT_CHUNK],
|
|
1088
1105
|
vc.AuditEnvelope(who="symgrep", why="index code by meaning"))
|
|
1089
1106
|
embedded += result.n_embedded
|
|
1107
|
+
# Both CLI reindex and the detached freshness worker reach this seam. Publish
|
|
1108
|
+
# the durable index after vector reconciliation; readers keep their committed
|
|
1109
|
+
# snapshot while a new generation is trained and assigned in bounded chunks.
|
|
1110
|
+
store.prepare_ann_index(COLLECTION)
|
|
1090
1111
|
return embedded
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Portable durable inverted-file cosine index with bounded training workspaces.
|
|
2
|
+
|
|
3
|
+
Only centroids, a bounded training reservoir and one assignment batch enter Python.
|
|
4
|
+
SQLite B-trees hold postings; queries rerank their original float32 vectors exactly.
|
|
5
|
+
"""
|
|
6
|
+
import json
|
|
7
|
+
import numpy as np
|
|
8
|
+
from .semantic_sqlite_config import (IVF_BUCKET_COUNT, IVF_TRAINING_ROWS, IVF_TRAINING_ITERATIONS,
|
|
9
|
+
IVF_ASSIGNMENT_ROWS, IVF_RANDOM_SEED)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def initialize_ivf_schema(connection):
|
|
13
|
+
existing_covering = connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='symgrep_ivf_vectors'").fetchone()
|
|
14
|
+
connection.executescript("""
|
|
15
|
+
CREATE TABLE IF NOT EXISTS symgrep_ivf_centroids(collection TEXT, bucket INTEGER, vector BLOB,
|
|
16
|
+
PRIMARY KEY(collection,bucket));
|
|
17
|
+
CREATE TABLE IF NOT EXISTS symgrep_ivf_vectors(collection TEXT, bucket INTEGER, id TEXT, vector BLOB, payload TEXT,
|
|
18
|
+
PRIMARY KEY(collection,bucket,id)) WITHOUT ROWID;
|
|
19
|
+
CREATE UNIQUE INDEX IF NOT EXISTS symgrep_ivf_covering_members ON symgrep_ivf_vectors(collection,id);
|
|
20
|
+
CREATE TABLE IF NOT EXISTS symgrep_ivf_model_config(collection TEXT PRIMARY KEY, signature TEXT NOT NULL);
|
|
21
|
+
CREATE TABLE IF NOT EXISTS symgrep_ivf_state(collection TEXT PRIMARY KEY, dirty INTEGER NOT NULL);
|
|
22
|
+
CREATE TRIGGER IF NOT EXISTS symgrep_ivf_covering_insert AFTER INSERT ON points BEGIN
|
|
23
|
+
INSERT OR REPLACE INTO symgrep_ivf_state VALUES(new.collection,1);
|
|
24
|
+
DELETE FROM symgrep_ivf_vectors WHERE collection=new.collection AND id=new.id;
|
|
25
|
+
END;
|
|
26
|
+
CREATE TRIGGER IF NOT EXISTS symgrep_ivf_covering_update AFTER UPDATE OF vec,payload,id,collection ON points BEGIN
|
|
27
|
+
INSERT OR REPLACE INTO symgrep_ivf_state VALUES(old.collection,1);
|
|
28
|
+
INSERT OR REPLACE INTO symgrep_ivf_state VALUES(new.collection,1);
|
|
29
|
+
DELETE FROM symgrep_ivf_vectors WHERE collection=old.collection AND id=old.id;
|
|
30
|
+
DELETE FROM symgrep_ivf_vectors WHERE collection=new.collection AND id=new.id;
|
|
31
|
+
END;
|
|
32
|
+
CREATE TRIGGER IF NOT EXISTS symgrep_ivf_covering_delete AFTER DELETE ON points BEGIN
|
|
33
|
+
DELETE FROM symgrep_ivf_vectors WHERE collection=old.collection AND id=old.id;
|
|
34
|
+
END;
|
|
35
|
+
""")
|
|
36
|
+
with connection:
|
|
37
|
+
connection.execute("INSERT OR REPLACE INTO symgrep_ivf_state SELECT DISTINCT collection,1 "
|
|
38
|
+
"FROM symgrep_ivf_centroids WHERE collection NOT IN "
|
|
39
|
+
"(SELECT collection FROM symgrep_ivf_model_config)")
|
|
40
|
+
if not existing_covering:
|
|
41
|
+
with connection:
|
|
42
|
+
connection.execute("INSERT OR REPLACE INTO symgrep_ivf_state SELECT DISTINCT collection,1 FROM symgrep_ivf_centroids")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def ivf_training_configuration_signature():
|
|
46
|
+
"""Persist all model-building choices; query probe changes do not retrain."""
|
|
47
|
+
return json.dumps((IVF_BUCKET_COUNT, IVF_TRAINING_ROWS, IVF_TRAINING_ITERATIONS,
|
|
48
|
+
IVF_ASSIGNMENT_ROWS, IVF_RANDOM_SEED))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def read_ivf_centroids(connection, collection):
|
|
52
|
+
rows = connection.execute('SELECT bucket,vector FROM symgrep_ivf_centroids WHERE collection=? ORDER BY bucket',
|
|
53
|
+
(collection,)).fetchall()
|
|
54
|
+
if any(bucket != ordinal for ordinal, (bucket, _vector) in enumerate(rows)):
|
|
55
|
+
raise ValueError("IVF centroid buckets must be contiguous")
|
|
56
|
+
return np.stack([np.frombuffer(row[1], dtype=np.float32) for row in rows]) if rows else None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def assign_ivf_rows(connection, collection, centroids, rows):
|
|
60
|
+
if not rows:
|
|
61
|
+
return
|
|
62
|
+
vectors = np.stack([np.frombuffer(blob, dtype=np.float32) for _, blob, _payload in rows])
|
|
63
|
+
buckets = np.argmax(vectors @ centroids.T, axis=1)
|
|
64
|
+
connection.executemany('INSERT OR REPLACE INTO symgrep_ivf_vectors VALUES(?,?,?,?,?)',
|
|
65
|
+
[(collection, int(bucket), row[0], row[1], row[2]) for row, bucket in zip(rows, buckets)])
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def rebuild_ivf_collection(connection, collection):
|
|
69
|
+
"""Publish a complete model and postings atomically; reservoir memory has a fixed ceiling."""
|
|
70
|
+
if min(IVF_BUCKET_COUNT, IVF_TRAINING_ROWS, IVF_TRAINING_ITERATIONS, IVF_ASSIGNMENT_ROWS) <= 0:
|
|
71
|
+
raise ValueError("IVF training configuration must be positive")
|
|
72
|
+
generator = np.random.default_rng(IVF_RANDOM_SEED)
|
|
73
|
+
reservoir = []
|
|
74
|
+
for ordinal, (blob,) in enumerate(connection.execute('SELECT vec FROM points WHERE collection=? ORDER BY id',
|
|
75
|
+
(collection,))):
|
|
76
|
+
if ordinal < IVF_TRAINING_ROWS:
|
|
77
|
+
reservoir.append(np.frombuffer(blob, dtype=np.float32).copy())
|
|
78
|
+
else:
|
|
79
|
+
selected = int(generator.integers(ordinal + 1))
|
|
80
|
+
if selected < IVF_TRAINING_ROWS:
|
|
81
|
+
reservoir[selected] = np.frombuffer(blob, dtype=np.float32).copy()
|
|
82
|
+
if not reservoir:
|
|
83
|
+
return False
|
|
84
|
+
training = np.stack(reservoir)
|
|
85
|
+
del reservoir # Release the sampling copies before clustering and index assignment.
|
|
86
|
+
centroids = training[generator.choice(len(training), min(IVF_BUCKET_COUNT, len(training)), replace=False)].copy()
|
|
87
|
+
for _iteration in range(IVF_TRAINING_ITERATIONS):
|
|
88
|
+
sums = np.zeros_like(centroids)
|
|
89
|
+
counts = np.zeros(len(centroids), dtype=np.int64)
|
|
90
|
+
for offset in range(0, len(training), IVF_ASSIGNMENT_ROWS):
|
|
91
|
+
batch = training[offset:offset + IVF_ASSIGNMENT_ROWS]
|
|
92
|
+
labels = np.argmax(batch @ centroids.T, axis=1)
|
|
93
|
+
np.add.at(sums, labels, batch)
|
|
94
|
+
np.add.at(counts, labels, 1)
|
|
95
|
+
occupied = counts > 0
|
|
96
|
+
centroids[occupied] = sums[occupied]
|
|
97
|
+
centroids /= np.maximum(np.linalg.norm(centroids, axis=1, keepdims=True), np.finfo(np.float32).tiny)
|
|
98
|
+
connection.execute('DELETE FROM symgrep_ivf_centroids WHERE collection=?', (collection,))
|
|
99
|
+
connection.executemany('INSERT INTO symgrep_ivf_centroids VALUES(?,?,?)',
|
|
100
|
+
[(collection, bucket, vector.tobytes()) for bucket, vector in enumerate(centroids)])
|
|
101
|
+
connection.execute('DELETE FROM symgrep_ivf_vectors WHERE collection=?', (collection,))
|
|
102
|
+
cursor = connection.execute('SELECT id,vec,payload FROM points WHERE collection=?', (collection,))
|
|
103
|
+
while rows := cursor.fetchmany(IVF_ASSIGNMENT_ROWS):
|
|
104
|
+
assign_ivf_rows(connection, collection, centroids, rows)
|
|
105
|
+
connection.execute('INSERT OR REPLACE INTO symgrep_ivf_model_config VALUES(?,?)',
|
|
106
|
+
(collection, ivf_training_configuration_signature()))
|
|
107
|
+
connection.execute('INSERT OR REPLACE INTO symgrep_ivf_state VALUES(?,0)', (collection,))
|
|
108
|
+
return True
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
"""Bounded SQLite cosine retrieval; corpus vectors never become a Python matrix.
|
|
2
|
+
|
|
3
|
+
The persistent reader pool maps only a bounded prefix per connection. SQLite/OS
|
|
4
|
+
page caching serves the remaining scan; pool size bounds simultaneous mappings and
|
|
5
|
+
page caches. Each SELECT is a committed snapshot, including its liveness join.
|
|
6
|
+
"""
|
|
7
|
+
import logging
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import queue
|
|
11
|
+
import sqlite3
|
|
12
|
+
import threading
|
|
13
|
+
from contextlib import contextmanager
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
from veccore.adapters.sqlite_numpy_store import SqliteNumpyStore
|
|
17
|
+
from veccore.ports import Hit
|
|
18
|
+
from veccore.errors import NotSupported
|
|
19
|
+
from .capability_registry import parse_semantic_search_unit
|
|
20
|
+
|
|
21
|
+
from .semantic_sqlite_config import (
|
|
22
|
+
SQLITE_READER_COUNT, SQLITE_READER_MMAP_BYTES, SQLITE_READER_CACHE_KIB,
|
|
23
|
+
SQLITE_READER_BUSY_MS, IVF_ENABLED, IVF_PROBE_COUNT, IVF_PROBE_GROWTH,
|
|
24
|
+
IVF_RECALL_VALIDATED_UNIT, IVF_BUCKET_COUNT, IVF_TRAINING_ROWS, IVF_TRAINING_ITERATIONS, IVF_ASSIGNMENT_ROWS, IVF_RANDOM_SEED,
|
|
25
|
+
)
|
|
26
|
+
COSINE_MINIMUM_NORM = np.finfo(np.float32).tiny
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class SqliteStreamingStore(SqliteNumpyStore):
|
|
30
|
+
"""Keep veccore's durable write protocol and replace its allocating read path."""
|
|
31
|
+
|
|
32
|
+
def __init__(self, path):
|
|
33
|
+
if min(IVF_PROBE_COUNT, IVF_BUCKET_COUNT, IVF_TRAINING_ROWS,
|
|
34
|
+
IVF_TRAINING_ITERATIONS, IVF_ASSIGNMENT_ROWS) <= 0 or IVF_PROBE_GROWTH <= 1 or IVF_RANDOM_SEED < 0:
|
|
35
|
+
raise ValueError("IVF configuration must have positive workspace/probe sizes, growth > 1 and a nonnegative seed")
|
|
36
|
+
super().__init__(path)
|
|
37
|
+
if SQLITE_READER_COUNT <= 0 or SQLITE_READER_MMAP_BYTES < 0 or SQLITE_READER_CACHE_KIB <= 0:
|
|
38
|
+
raise ValueError("SQLite reader configuration must be bounded and positive")
|
|
39
|
+
self._reader_slots = threading.BoundedSemaphore(SQLITE_READER_COUNT)
|
|
40
|
+
self._idle_readers = queue.LifoQueue()
|
|
41
|
+
self._reader_metrics_lock = threading.Lock()
|
|
42
|
+
self._reader_queries = 0
|
|
43
|
+
self._active_readers = 0
|
|
44
|
+
self._peak_readers = 0
|
|
45
|
+
self._reader_opens = 0
|
|
46
|
+
self._readers_retired = False
|
|
47
|
+
self._ivf_fallback_logged = False
|
|
48
|
+
self._ivf_unvalidated_scope_logged = False
|
|
49
|
+
self._ivf_schema_available = False
|
|
50
|
+
try:
|
|
51
|
+
from .semantic_ivf import initialize_ivf_schema
|
|
52
|
+
with self._con_lock:
|
|
53
|
+
initialize_ivf_schema(self._con)
|
|
54
|
+
self._ivf_schema_available = True
|
|
55
|
+
except sqlite3.Error as error:
|
|
56
|
+
self._log_ivf_fallback(str(error))
|
|
57
|
+
|
|
58
|
+
def _log_ivf_fallback(self, reason):
|
|
59
|
+
with self._reader_metrics_lock:
|
|
60
|
+
if not self._ivf_fallback_logged:
|
|
61
|
+
logging.getLogger(__name__).warning("semantic ANN unavailable (%s); using exact SQLite streaming scan", reason)
|
|
62
|
+
self._ivf_fallback_logged = True
|
|
63
|
+
|
|
64
|
+
def prepare_ann_index(self, collection):
|
|
65
|
+
try:
|
|
66
|
+
return self._prepare_ivf_generation(collection)
|
|
67
|
+
except (sqlite3.Error, OSError, ValueError) as error:
|
|
68
|
+
with self._con_lock:
|
|
69
|
+
self._con.rollback()
|
|
70
|
+
self._log_ivf_fallback(str(error))
|
|
71
|
+
return False
|
|
72
|
+
|
|
73
|
+
def _prepare_ivf_generation(self, collection):
|
|
74
|
+
"""Reindex seam: build once, or reconcile every pending committed point."""
|
|
75
|
+
if not IVF_ENABLED or not self._ivf_schema_available:
|
|
76
|
+
self._log_ivf_fallback("disabled or index schema unavailable")
|
|
77
|
+
return False
|
|
78
|
+
from .semantic_ivf import (read_ivf_centroids, rebuild_ivf_collection, assign_ivf_rows,
|
|
79
|
+
ivf_training_configuration_signature)
|
|
80
|
+
from .semantic_sqlite_config import IVF_ASSIGNMENT_ROWS
|
|
81
|
+
with self._con_lock, self._con:
|
|
82
|
+
self._con.execute("BEGIN IMMEDIATE")
|
|
83
|
+
centroids = read_ivf_centroids(self._con, collection)
|
|
84
|
+
model_config = self._con.execute(
|
|
85
|
+
'SELECT signature FROM symgrep_ivf_model_config WHERE collection=?', (collection,)).fetchone()
|
|
86
|
+
if centroids is None or not model_config or model_config[0] != ivf_training_configuration_signature():
|
|
87
|
+
return rebuild_ivf_collection(self._con, collection)
|
|
88
|
+
cursor = self._con.execute("SELECT p.id,p.vec,p.payload FROM points p LEFT JOIN symgrep_ivf_vectors a "
|
|
89
|
+
"ON a.collection=p.collection AND a.id=p.id WHERE p.collection=? AND a.id IS NULL", (collection,))
|
|
90
|
+
while rows := cursor.fetchmany(IVF_ASSIGNMENT_ROWS):
|
|
91
|
+
assign_ivf_rows(self._con, collection, centroids, rows)
|
|
92
|
+
self._con.execute('INSERT OR REPLACE INTO symgrep_ivf_state VALUES(?,0)', (collection,))
|
|
93
|
+
return True
|
|
94
|
+
|
|
95
|
+
def upsert(self, collection, points):
|
|
96
|
+
from .semantic_ivf import read_ivf_centroids, assign_ivf_rows
|
|
97
|
+
with self._con_lock:
|
|
98
|
+
rows = [(collection, point.id, point.text_sha, int(np.asarray(point.vector).size),
|
|
99
|
+
np.asarray(point.vector, dtype=np.float32).ravel().tobytes(),
|
|
100
|
+
json.dumps(point.payload, ensure_ascii=False)) for point in points]
|
|
101
|
+
with self._con:
|
|
102
|
+
# Acquire the write reservation before reading readiness; another
|
|
103
|
+
# writer cannot invalidate the generation between these steps.
|
|
104
|
+
self._con.execute("BEGIN IMMEDIATE")
|
|
105
|
+
state = self._con.execute('SELECT dirty FROM symgrep_ivf_state WHERE collection=?',
|
|
106
|
+
(collection,)).fetchone() if self._ivf_schema_available else None
|
|
107
|
+
self._con.executemany("INSERT OR REPLACE INTO points(collection,id,text_sha,dim,vec,payload) "
|
|
108
|
+
"VALUES(?,?,?,?,?,?)", rows)
|
|
109
|
+
if IVF_ENABLED and self._ivf_schema_available:
|
|
110
|
+
centroids = read_ivf_centroids(self._con, collection)
|
|
111
|
+
if centroids is not None:
|
|
112
|
+
assign_ivf_rows(self._con, collection, centroids, [(row[1], row[4], row[5]) for row in rows])
|
|
113
|
+
if state and not state[0]:
|
|
114
|
+
self._con.execute('UPDATE symgrep_ivf_state SET dirty=0 WHERE collection=?', (collection,))
|
|
115
|
+
self._matrix_cache.pop(collection, None)
|
|
116
|
+
|
|
117
|
+
def _sqlite_file_identity(self):
|
|
118
|
+
stat = os.stat(self.path)
|
|
119
|
+
return stat.st_dev, stat.st_ino
|
|
120
|
+
|
|
121
|
+
@contextmanager
|
|
122
|
+
def _borrow_sqlite_reader(self):
|
|
123
|
+
with self._reader_slots:
|
|
124
|
+
identity = self._sqlite_file_identity()
|
|
125
|
+
try:
|
|
126
|
+
reader, reader_identity = self._idle_readers.get_nowait()
|
|
127
|
+
except queue.Empty:
|
|
128
|
+
reader, reader_identity = None, None
|
|
129
|
+
if reader is not None and reader_identity != identity:
|
|
130
|
+
reader.close()
|
|
131
|
+
reader = None
|
|
132
|
+
if reader is None:
|
|
133
|
+
# mode=ro rejects an absent/replaced path rather than creating an empty DB.
|
|
134
|
+
from pathlib import Path
|
|
135
|
+
reader = sqlite3.connect(Path(self.path).resolve().as_uri() + "?mode=ro", uri=True,
|
|
136
|
+
check_same_thread=False)
|
|
137
|
+
reader.execute(f"PRAGMA busy_timeout={SQLITE_READER_BUSY_MS}")
|
|
138
|
+
reader.execute(f"PRAGMA mmap_size={SQLITE_READER_MMAP_BYTES}")
|
|
139
|
+
reader.execute(f"PRAGMA cache_size={-SQLITE_READER_CACHE_KIB}")
|
|
140
|
+
reader.execute("PRAGMA temp_store=FILE")
|
|
141
|
+
reader.execute("PRAGMA query_only=ON")
|
|
142
|
+
with self._reader_metrics_lock:
|
|
143
|
+
self._reader_opens += 1
|
|
144
|
+
with self._reader_metrics_lock:
|
|
145
|
+
self._active_readers += 1
|
|
146
|
+
self._peak_readers = max(self._peak_readers, self._active_readers)
|
|
147
|
+
try:
|
|
148
|
+
yield reader
|
|
149
|
+
finally:
|
|
150
|
+
if reader.in_transaction:
|
|
151
|
+
reader.rollback()
|
|
152
|
+
# Remove closures retaining query vectors/scoping sets between requests.
|
|
153
|
+
reader.create_function("symgrep_cosine", 1, None)
|
|
154
|
+
reader.create_function("symgrep_scope", 2, None)
|
|
155
|
+
with self._reader_metrics_lock:
|
|
156
|
+
self._active_readers -= 1
|
|
157
|
+
self._reader_queries += 1
|
|
158
|
+
try:
|
|
159
|
+
reusable = self._sqlite_file_identity() == identity
|
|
160
|
+
except FileNotFoundError:
|
|
161
|
+
reusable = False
|
|
162
|
+
with self._reader_metrics_lock:
|
|
163
|
+
if reusable and not self._readers_retired:
|
|
164
|
+
self._idle_readers.put((reader, identity))
|
|
165
|
+
else:
|
|
166
|
+
reader.close()
|
|
167
|
+
|
|
168
|
+
def retire_readers(self):
|
|
169
|
+
"""Release idle old-inode mappings; in-flight snapshots close when returned."""
|
|
170
|
+
with self._reader_metrics_lock:
|
|
171
|
+
self._readers_retired = True
|
|
172
|
+
while True:
|
|
173
|
+
try:
|
|
174
|
+
reader, _identity = self._idle_readers.get_nowait()
|
|
175
|
+
except queue.Empty:
|
|
176
|
+
break
|
|
177
|
+
reader.close()
|
|
178
|
+
|
|
179
|
+
def close(self):
|
|
180
|
+
self.retire_readers()
|
|
181
|
+
super().close()
|
|
182
|
+
|
|
183
|
+
def has_points(self, collection):
|
|
184
|
+
"""Probe existence without allocating every stored identifier."""
|
|
185
|
+
with self._borrow_sqlite_reader() as reader:
|
|
186
|
+
return reader.execute("SELECT 1 FROM points WHERE collection=? LIMIT 1",
|
|
187
|
+
(collection,)).fetchone() is not None
|
|
188
|
+
|
|
189
|
+
def read_metrics(self):
|
|
190
|
+
"""A bounded-reader diagnostic; matrix loading is absent from this implementation."""
|
|
191
|
+
with self._reader_metrics_lock:
|
|
192
|
+
return dict(queries=self._reader_queries, active_readers=self._active_readers,
|
|
193
|
+
peak_readers=self._peak_readers, reader_opens=self._reader_opens, matrix_loads=0)
|
|
194
|
+
|
|
195
|
+
def search_exact(self, collection, qvec, k, filters=None):
|
|
196
|
+
return self._search_sqlite_vectors(collection, qvec, k, filters, use_ann=False)
|
|
197
|
+
|
|
198
|
+
def search(self, collection, qvec, k, filters=None):
|
|
199
|
+
return self._search_sqlite_vectors(collection, qvec, k, filters, use_ann=True)
|
|
200
|
+
|
|
201
|
+
def _search_sqlite_vectors(self, collection, qvec, k, filters=None, use_ann=True):
|
|
202
|
+
# Validate before SQL or ANN routing: this is the public enum grammar,
|
|
203
|
+
# not an inference about the meaning of arbitrary free text.
|
|
204
|
+
validated_unit = (parse_semantic_search_unit(filters["symgrep_unit"])
|
|
205
|
+
if filters and "symgrep_unit" in filters else None)
|
|
206
|
+
query_vector = np.asarray(qvec, dtype=np.float32).ravel()
|
|
207
|
+
query_vector = query_vector / max(float(np.linalg.norm(query_vector)), COSINE_MINIMUM_NORM)
|
|
208
|
+
scope = (filters or {}).get("symgrep_scope")
|
|
209
|
+
unsupported = set(filters or {}) - {"symgrep_scope", "symgrep_unit", "symgrep_kind"}
|
|
210
|
+
if unsupported:
|
|
211
|
+
raise NotSupported("unsupported semantic filters", self.backend, str(sorted(unsupported)))
|
|
212
|
+
|
|
213
|
+
def sqlite_vector_dot(blob):
|
|
214
|
+
vector = np.frombuffer(blob, dtype=np.float32)
|
|
215
|
+
if vector.shape != query_vector.shape:
|
|
216
|
+
raise ValueError("stored/query embedding dimensions differ")
|
|
217
|
+
return float(np.dot(vector, query_vector))
|
|
218
|
+
|
|
219
|
+
with self._borrow_sqlite_reader() as reader:
|
|
220
|
+
reader.create_function("symgrep_cosine", 1, sqlite_vector_dot)
|
|
221
|
+
reader.create_function("symgrep_scope", 2, scope or (lambda _id, _payload: True))
|
|
222
|
+
clauses = ["p.collection=?", "symgrep_scope(p.id,p.payload)"]
|
|
223
|
+
parameters = [collection]
|
|
224
|
+
join = ""
|
|
225
|
+
if filters and "symgrep_unit" in filters:
|
|
226
|
+
# One statement sees points AND bookkeeping in one committed snapshot.
|
|
227
|
+
join = " JOIN fn f ON f.key=p.id"
|
|
228
|
+
clauses.append("f.what IS NOT NULL AND f.what != ''")
|
|
229
|
+
unit = validated_unit.value
|
|
230
|
+
if unit in ("block", "data"):
|
|
231
|
+
clauses.append("f.unit=?")
|
|
232
|
+
parameters.append(unit)
|
|
233
|
+
elif unit not in ("both", "all"):
|
|
234
|
+
clauses.append("(f.unit='symbol' OR f.unit IS NULL)")
|
|
235
|
+
kind = filters.get("symgrep_kind")
|
|
236
|
+
if kind:
|
|
237
|
+
clauses.append("f.kind=?")
|
|
238
|
+
parameters.append(kind)
|
|
239
|
+
centroid_order = None
|
|
240
|
+
if IVF_PROBE_COUNT <= 0:
|
|
241
|
+
raise ValueError("IVF probe count must be positive")
|
|
242
|
+
if use_ann and IVF_ENABLED and self._ivf_schema_available:
|
|
243
|
+
from .semantic_ivf import read_ivf_centroids
|
|
244
|
+
# Explicit transaction binds index validity, postings, points and fn
|
|
245
|
+
# to the same committed generation, even across concurrent writes.
|
|
246
|
+
reader.execute("BEGIN")
|
|
247
|
+
try:
|
|
248
|
+
state = reader.execute("SELECT dirty FROM symgrep_ivf_state WHERE collection=?", (collection,)).fetchone()
|
|
249
|
+
centroids = read_ivf_centroids(reader, collection) if state and not state[0] else None
|
|
250
|
+
if centroids is not None:
|
|
251
|
+
centroid_order = np.argsort(-(centroids @ query_vector))
|
|
252
|
+
except (sqlite3.Error, ValueError) as error:
|
|
253
|
+
self._log_ivf_fallback(str(error))
|
|
254
|
+
if use_ann and centroid_order is None:
|
|
255
|
+
self._log_ivf_fallback("disabled, missing or pending index generation")
|
|
256
|
+
probe_count = min(IVF_PROBE_COUNT, len(centroid_order)) if centroid_order is not None else 0
|
|
257
|
+
if centroid_order is not None and (
|
|
258
|
+
validated_unit is not IVF_RECALL_VALIDATED_UNIT
|
|
259
|
+
or (filters or {}).get("symgrep_kind")):
|
|
260
|
+
# Only the measured symbol category enables approximate pruning.
|
|
261
|
+
# Other structured filter categories retain exact full coverage.
|
|
262
|
+
probe_count = len(centroid_order)
|
|
263
|
+
with self._reader_metrics_lock:
|
|
264
|
+
if not self._ivf_unvalidated_scope_logged:
|
|
265
|
+
logging.getLogger(__name__).warning(
|
|
266
|
+
"semantic ANN uses exact full-bucket coverage for unvalidated unit/kind filters")
|
|
267
|
+
self._ivf_unvalidated_scope_logged = True
|
|
268
|
+
while True:
|
|
269
|
+
ann_join = ""
|
|
270
|
+
from_clause = "points p"
|
|
271
|
+
ann_clause = ""
|
|
272
|
+
ann_parameters = []
|
|
273
|
+
if centroid_order is not None:
|
|
274
|
+
buckets = [int(bucket) for bucket in centroid_order[:probe_count]]
|
|
275
|
+
from_clause = ("(SELECT collection,id,vector AS vec,payload FROM symgrep_ivf_vectors "
|
|
276
|
+
"WHERE collection=? AND bucket IN (" + ",".join("?" for _ in buckets) + ")) p")
|
|
277
|
+
ann_clause = ""
|
|
278
|
+
ann_parameters = [collection, *buckets]
|
|
279
|
+
try:
|
|
280
|
+
rows = reader.execute(
|
|
281
|
+
"SELECT p.id, symgrep_cosine(p.vec) AS score, p.payload FROM " + from_clause + ann_join + join +
|
|
282
|
+
" WHERE " + " AND ".join(clauses) + ann_clause + " ORDER BY score DESC, p.id LIMIT ?",
|
|
283
|
+
(*ann_parameters, *parameters, max(0, k))).fetchall()
|
|
284
|
+
except sqlite3.Error as error:
|
|
285
|
+
if centroid_order is None:
|
|
286
|
+
raise
|
|
287
|
+
self._log_ivf_fallback(str(error))
|
|
288
|
+
centroid_order = None
|
|
289
|
+
continue
|
|
290
|
+
if centroid_order is None or len(rows) >= k or probe_count == len(centroid_order):
|
|
291
|
+
break
|
|
292
|
+
probe_count = min(len(centroid_order), probe_count * IVF_PROBE_GROWTH)
|
|
293
|
+
if reader.in_transaction:
|
|
294
|
+
reader.rollback()
|
|
295
|
+
return [Hit(id=key, score=score, payload=json.loads(payload)) for key, score, payload in rows]
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Configuration shared by SQLite retrieval and capability docs."""
|
|
2
|
+
import os
|
|
3
|
+
from .capability_registry import parse_semantic_search_unit
|
|
4
|
+
|
|
5
|
+
SQLITE_READER_COUNT = int(os.environ.get("SYMGREP_SQLITE_READERS", "4"))
|
|
6
|
+
SQLITE_READER_MMAP_BYTES = int(os.environ.get("SYMGREP_SQLITE_MMAP_BYTES", str(32 * 1024 * 1024)))
|
|
7
|
+
SQLITE_READER_CACHE_KIB = int(os.environ.get("SYMGREP_SQLITE_CACHE_KIB", "512"))
|
|
8
|
+
SQLITE_READER_BUSY_MS = int(os.environ.get("SYMGREP_SQLITE_BUSY_MS", "10000"))
|
|
9
|
+
|
|
10
|
+
IVF_ENABLED = os.environ.get("SYMGREP_IVF_ENABLED", "1") != "0"
|
|
11
|
+
IVF_BUCKET_COUNT = int(os.environ.get("SYMGREP_IVF_BUCKETS", "256"))
|
|
12
|
+
IVF_PROBE_COUNT = int(os.environ.get("SYMGREP_IVF_PROBES", "128"))
|
|
13
|
+
IVF_TRAINING_ROWS = int(os.environ.get("SYMGREP_IVF_TRAINING_ROWS", "8192"))
|
|
14
|
+
IVF_TRAINING_ITERATIONS = int(os.environ.get("SYMGREP_IVF_TRAINING_ITERATIONS", "12"))
|
|
15
|
+
IVF_ASSIGNMENT_ROWS = int(os.environ.get("SYMGREP_IVF_ASSIGNMENT_ROWS", "256"))
|
|
16
|
+
IVF_RANDOM_SEED = int(os.environ.get("SYMGREP_IVF_RANDOM_SEED", "0"))
|
|
17
|
+
IVF_PROBE_GROWTH = 2
|
|
18
|
+
|
|
19
|
+
IVF_RECALL_VALIDATED_UNIT = parse_semantic_search_unit("symbol")
|
{symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2/src/symgrep_codesearch.egg-info}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: symgrep-codesearch
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.2
|
|
4
4
|
Summary: Code search with honest recall — structure-aware grep that tells you what it does not know, plus search by meaning
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Project-URL: Repository, https://github.com/ashdamle/symgrep
|
{symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/SOURCES.txt
RENAMED
|
@@ -25,6 +25,7 @@ src/symgrep/ripgrep.py
|
|
|
25
25
|
src/symgrep/rss_ceiling.py
|
|
26
26
|
src/symgrep/runtime_paths.py
|
|
27
27
|
src/symgrep/semantic.py
|
|
28
|
+
src/symgrep/semantic_ivf.py
|
|
28
29
|
src/symgrep/semantic_sqlite.py
|
|
29
30
|
src/symgrep/semantic_sqlite_config.py
|
|
30
31
|
src/symgrep/structural.py
|
|
@@ -71,9 +72,11 @@ tests/test_resolve_python.py
|
|
|
71
72
|
tests/test_resolve_r2.py
|
|
72
73
|
tests/test_review_regressions.py
|
|
73
74
|
tests/test_rss_ceiling_pytest.py
|
|
75
|
+
tests/test_semantic_census_pruning_pytest.py
|
|
74
76
|
tests/test_semantic_freshness_signal.py
|
|
75
77
|
tests/test_semantic_glob_lang.py
|
|
76
78
|
tests/test_sites_and_symbols.py
|
|
79
|
+
tests/test_sqlite_ivf_retrieval_pytest.py
|
|
77
80
|
tests/test_sqlite_streaming_retrieval_pytest.py
|
|
78
81
|
tests/test_structural.py
|
|
79
82
|
tests/test_symbol_vocabulary.py
|