symgrep-codesearch 0.9.0__tar.gz → 0.9.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {symgrep_codesearch-0.9.0/src/symgrep_codesearch.egg-info → symgrep_codesearch-0.9.2}/PKG-INFO +1 -1
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/pyproject.toml +3 -1
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/capability_registry.py +66 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/semantic.py +160 -85
- symgrep_codesearch-0.9.2/src/symgrep/semantic_ivf.py +108 -0
- symgrep_codesearch-0.9.2/src/symgrep/semantic_sqlite.py +295 -0
- symgrep_codesearch-0.9.2/src/symgrep/semantic_sqlite_config.py +19 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2/src/symgrep_codesearch.egg-info}/PKG-INFO +1 -1
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/SOURCES.txt +6 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_embedding_residency_pytest.py +20 -42
- symgrep_codesearch-0.9.2/tests/test_semantic_census_pruning_pytest.py +42 -0
- symgrep_codesearch-0.9.2/tests/test_sqlite_ivf_retrieval_pytest.py +278 -0
- symgrep_codesearch-0.9.2/tests/test_sqlite_streaming_retrieval_pytest.py +220 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/LICENSE +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/README.md +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/setup.cfg +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/__init__.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/absence_adjudication_writer.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/archives.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/blocks.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/capability_registry_data.jsonl +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/capability_surface_checks.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/classfile.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/cli.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/data_symbols.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/docsymbols.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/embedding_process.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/enclosing.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/glob_recall.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/mcp_server.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/mcp_structured.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/report.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/resolve.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/resolve_js.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/resolve_python.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/ripgrep.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/rss_ceiling.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/runtime_paths.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/structural.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/structure.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/dependency_links.txt +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/entry_points.txt +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/requires.txt +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/top_level.txt +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_absence_adjudication_authorship_pytest.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_absence_adjudication_binding_pytest.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_blocks_extraction.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_blocks_retrieval.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_bulk_describe.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_capability_documentation_guard_pytest.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_capability_surface_parity_guard_pytest.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_classfile.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_codex_hook_payload.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_data_symbols.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_describe_lane_confinement.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_doc_symbols.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_file_structure.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_freshness.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_honest_recall.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_http_ensure.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_http_rss_shutdown_pytest.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_index_reconstruct.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_index_scope.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_index_worker.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_mcp_error_log.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_mcp_registry_behavior_pytest.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_mcp_structured.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_mcp_thread_safety.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_mcp_version.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_name_uniqueness.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_nudge_classify.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_nudge_runaway.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_report_filters.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_resolve.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_resolve_js.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_resolve_python.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_resolve_r2.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_review_regressions.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_rss_ceiling_pytest.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_semantic_freshness_signal.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_semantic_glob_lang.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_sites_and_symbols.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_structural.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_symbol_vocabulary.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_ts_fallback_loud.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_ts_symbol_coverage.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_usage.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_veccore_adoption.py +0 -0
- {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_worker_lock_recovery_pytest.py +0 -0
{symgrep_codesearch-0.9.0/src/symgrep_codesearch.egg-info → symgrep_codesearch-0.9.2}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: symgrep-codesearch
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.2
|
|
4
4
|
Summary: Code search with honest recall — structure-aware grep that tells you what it does not know, plus search by meaning
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Project-URL: Repository, https://github.com/ashdamle/symgrep
|
|
@@ -7,6 +7,8 @@ build-backend = "setuptools.build_meta"
|
|
|
7
7
|
# Semgrep team. The IMPORT name is unchanged — `import symgrep` — so consumers (honestreview, spendguard,
|
|
8
8
|
# 7thsense, warden) are unaffected by the distribution rename. The package dir stays src/symgrep.
|
|
9
9
|
name = "symgrep-codesearch"
|
|
10
|
+
# 0.9.1: shared, bounded SQLite corpus search; live committed reindex visibility
|
|
11
|
+
# and atomic replacement handling, preserving disposable query embedding children.
|
|
10
12
|
# 0.9.0: fix ONNX-arena and thread-local-matrix retention with disposable serialized
|
|
11
13
|
# embedding processes and per-index thread-safe stores (~1s latency per embedding round).
|
|
12
14
|
# Add the SYMGREP_MAX_RSS_MB self-guard; require psutil>=5.9 and veccore>=0.3.1.
|
|
@@ -39,7 +41,7 @@ name = "symgrep-codesearch"
|
|
|
39
41
|
# semantic search. Minor bump (dependency surface changed; no breaking import/CLI/MCP change).
|
|
40
42
|
# 0.5.0: added the public `file_structure` library API (+ its MCP tool and `symgrep structure` CLI), import
|
|
41
43
|
# module-name normalization, and the FileStructure `connects` list (structure schema v2).
|
|
42
|
-
version = "0.9.
|
|
44
|
+
version = "0.9.2"
|
|
43
45
|
description = "Code search with honest recall — structure-aware grep that tells you what it does not know, plus search by meaning"
|
|
44
46
|
readme = "README.md"
|
|
45
47
|
# 3.11+ : the maintained grammar pack (tree-sitter-language-pack, a CORE dep below) ships abi3 wheels for
|
|
@@ -7,6 +7,7 @@ import inspect
|
|
|
7
7
|
import hashlib
|
|
8
8
|
from datetime import datetime, timezone
|
|
9
9
|
import json
|
|
10
|
+
from enum import StrEnum
|
|
10
11
|
from typing import Any, Optional, List
|
|
11
12
|
|
|
12
13
|
CAPABILITY_DATA_RESOURCE = 'capability_registry_data.jsonl'
|
|
@@ -36,6 +37,52 @@ RUNTIME_ENVIRONMENT_SETTINGS = (
|
|
|
36
37
|
'embedding models; zero/negative/nonfinite values are errors, never a disable switch. '
|
|
37
38
|
'If psutil is missing, RSS containment is Unavailable and disabled; one CRITICAL '
|
|
38
39
|
'entry is durably flushed to the MCP error log explaining that containment is OFF.'),
|
|
40
|
+
RuntimeEnvironmentSetting(
|
|
41
|
+
'SYMGREP_SQLITE_READERS', 'symgrep.semantic_sqlite_config:SQLITE_READER_COUNT',
|
|
42
|
+
'Positive number of reusable SQLite semantic readers per index and admitted complete '
|
|
43
|
+
'semantic requests per process. Admission also bounds bookkeeping/freshness allocations; '
|
|
44
|
+
'additional requests wait before allocating request metadata. Corpus matrices are never loaded.'),
|
|
45
|
+
RuntimeEnvironmentSetting(
|
|
46
|
+
'SYMGREP_SQLITE_MMAP_BYTES', 'symgrep.semantic_sqlite_config:SQLITE_READER_MMAP_BYTES',
|
|
47
|
+
'Nonnegative mmap limit in bytes per SQLite semantic reader. Mapped database pages '
|
|
48
|
+
'are demand-paged by the OS; zero uses SQLite page caching without mmap.'),
|
|
49
|
+
RuntimeEnvironmentSetting(
|
|
50
|
+
'SYMGREP_SQLITE_CACHE_KIB', 'symgrep.semantic_sqlite_config:SQLITE_READER_CACHE_KIB',
|
|
51
|
+
'Positive SQLite page-cache budget in KiB per semantic reader. Corpus vectors '
|
|
52
|
+
'remain in SQLite and only top-k results enter the Python result list.'),
|
|
53
|
+
RuntimeEnvironmentSetting(
|
|
54
|
+
'SYMGREP_SQLITE_BUSY_MS', 'symgrep.semantic_sqlite_config:SQLITE_READER_BUSY_MS',
|
|
55
|
+
'SQLite semantic reader busy timeout in milliseconds while concurrent indexing '
|
|
56
|
+
'holds a database lock. WAL readers otherwise use committed statement snapshots.'),
|
|
57
|
+
RuntimeEnvironmentSetting(
|
|
58
|
+
'SYMGREP_IVF_ENABLED', 'symgrep.semantic_sqlite_config:IVF_ENABLED',
|
|
59
|
+
'Persistent SQLite inverted-file vector retrieval. Zero disables ANN and logs one '
|
|
60
|
+
'honest warning before exact streaming cosine fallback. Centroids and postings live '
|
|
61
|
+
'in the committed database; approximate candidates are reranked on original vectors. '
|
|
62
|
+
'Reindex prepares live postings. Reader pooling and atomic generation swaps stay bounded.'),
|
|
63
|
+
RuntimeEnvironmentSetting(
|
|
64
|
+
'SYMGREP_IVF_BUCKETS', 'symgrep.semantic_sqlite_config:IVF_BUCKET_COUNT',
|
|
65
|
+
'Centroid count for the inverted-file index. Reindex atomically rebuilds when '
|
|
66
|
+
'training settings differ from the durable model signature. More buckets '
|
|
67
|
+
'narrow candidate scans and require recall measurement.'),
|
|
68
|
+
RuntimeEnvironmentSetting(
|
|
69
|
+
'SYMGREP_IVF_PROBES', 'symgrep.semantic_sqlite_config:IVF_PROBE_COUNT',
|
|
70
|
+
'Initial nearest centroid buckets for symbol queries without a kind filter. Other unit/kind '
|
|
71
|
+
'scopes use every bucket and log exact coverage to preserve recall. More probes improve '
|
|
72
|
+
'approximate recall; sparse results expand probes. Measure complete recall@k against exact streaming search.'),
|
|
73
|
+
RuntimeEnvironmentSetting(
|
|
74
|
+
'SYMGREP_IVF_TRAINING_ROWS', 'symgrep.semantic_sqlite_config:IVF_TRAINING_ROWS',
|
|
75
|
+
'Maximum reservoir sample rows used to train centroids; bounds training memory '
|
|
76
|
+
'independently of corpus size.'),
|
|
77
|
+
RuntimeEnvironmentSetting(
|
|
78
|
+
'SYMGREP_IVF_TRAINING_ITERATIONS', 'symgrep.semantic_sqlite_config:IVF_TRAINING_ITERATIONS',
|
|
79
|
+
'Spherical centroid training iterations for a new index.'),
|
|
80
|
+
RuntimeEnvironmentSetting(
|
|
81
|
+
'SYMGREP_IVF_ASSIGNMENT_ROWS', 'symgrep.semantic_sqlite_config:IVF_ASSIGNMENT_ROWS',
|
|
82
|
+
'Maximum vector rows per centroid assignment batch during training and reindex.'),
|
|
83
|
+
RuntimeEnvironmentSetting(
|
|
84
|
+
'SYMGREP_IVF_RANDOM_SEED', 'symgrep.semantic_sqlite_config:IVF_RANDOM_SEED',
|
|
85
|
+
'Reproducible reservoir sampling and centroid initialization seed for new indexes.'),
|
|
39
86
|
)
|
|
40
87
|
|
|
41
88
|
@dataclass(frozen=True)
|
|
@@ -75,6 +122,25 @@ CAPABILITIES = load_symgrep_capability_declarations()
|
|
|
75
122
|
def capability_by_id(identity):
|
|
76
123
|
return next(cap for cap in CAPABILITIES if cap.id == identity)
|
|
77
124
|
|
|
125
|
+
|
|
126
|
+
SEMANTIC_UNIT_ARGUMENT = next(parameter for parameter in capability_by_id('search').params
|
|
127
|
+
if parameter.name == 'unit')
|
|
128
|
+
SemanticSearchUnit = StrEnum('SemanticSearchUnit',
|
|
129
|
+
{choice.upper(): choice for choice in SEMANTIC_UNIT_ARGUMENT.choices})
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def parse_semantic_search_unit(value):
|
|
133
|
+
"""Parse the declared unit grammar; aliases outside the public enum are errors.
|
|
134
|
+
|
|
135
|
+
In particular, plural spellings are not inferred from their meaning. The CLI,
|
|
136
|
+
MCP schema and direct store callers share the registry's canonical members.
|
|
137
|
+
"""
|
|
138
|
+
try:
|
|
139
|
+
return SemanticSearchUnit(value)
|
|
140
|
+
except (TypeError, ValueError) as error:
|
|
141
|
+
raise ValueError(f'Invalid semantic unit {value!r}; expected one of '
|
|
142
|
+
f'{SEMANTIC_UNIT_ARGUMENT.choices!r}') from error
|
|
143
|
+
|
|
78
144
|
def capability_description(capability):
|
|
79
145
|
"""Render full canonical prose and every parameter's semantics."""
|
|
80
146
|
return '\n\n'.join([capability.summary, capability.description] + [f'{p.name}: {p.help}' for p in capability.params])
|
|
@@ -116,20 +116,65 @@ class SemanticStoreRegistry:
|
|
|
116
116
|
def __init__(self):
|
|
117
117
|
self._process_embedder = None
|
|
118
118
|
self._cores_by_path = {}
|
|
119
|
+
self._store_file_identities = {}
|
|
119
120
|
self._registry_lock = threading.Lock()
|
|
121
|
+
from .semantic_sqlite_config import SQLITE_READER_COUNT
|
|
122
|
+
if SQLITE_READER_COUNT <= 0:
|
|
123
|
+
raise ValueError("SQLite reader count must be positive")
|
|
124
|
+
# Bound whole request working sets, not only vector scans. Bookkeeping
|
|
125
|
+
# and filesystem freshness checks also allocate corpus-sized metadata.
|
|
126
|
+
self._semantic_request_slots = threading.BoundedSemaphore(SQLITE_READER_COUNT)
|
|
127
|
+
self._semantic_admission_lock = threading.Lock()
|
|
128
|
+
self._active_semantic_requests = 0
|
|
129
|
+
self._peak_semantic_requests = 0
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def bound_semantic_request(self, operation):
|
|
133
|
+
"""Queue before allocating metadata; admit at most the reader-pool capacity."""
|
|
134
|
+
from functools import wraps
|
|
135
|
+
|
|
136
|
+
@wraps(operation)
|
|
137
|
+
def execute_admitted_semantic_request(*args, **kwargs):
|
|
138
|
+
with self._semantic_request_slots:
|
|
139
|
+
with self._semantic_admission_lock:
|
|
140
|
+
self._active_semantic_requests += 1
|
|
141
|
+
self._peak_semantic_requests = max(self._peak_semantic_requests,
|
|
142
|
+
self._active_semantic_requests)
|
|
143
|
+
try:
|
|
144
|
+
return operation(*args, **kwargs)
|
|
145
|
+
finally:
|
|
146
|
+
with self._semantic_admission_lock:
|
|
147
|
+
self._active_semantic_requests -= 1
|
|
148
|
+
|
|
149
|
+
return execute_admitted_semantic_request
|
|
150
|
+
|
|
151
|
+
def semantic_request_metrics(self):
|
|
152
|
+
"""Expose complete-request admission independently of SQL scan occupancy."""
|
|
153
|
+
with self._semantic_admission_lock:
|
|
154
|
+
return {"active_requests": self._active_semantic_requests,
|
|
155
|
+
"peak_requests": self._peak_semantic_requests}
|
|
156
|
+
|
|
157
|
+
def child_peak_rss_bytes(self):
|
|
158
|
+
"""Read the disposable embedder's latest receipt under the holder lock."""
|
|
159
|
+
with self._registry_lock:
|
|
160
|
+
return (self._process_embedder.last_child_peak_rss_bytes
|
|
161
|
+
if self._process_embedder is not None else None)
|
|
120
162
|
|
|
121
163
|
def core_for_index(self, path_str):
|
|
122
164
|
"""Build once under the registry lock; racing callers reuse the winning pair."""
|
|
123
|
-
hit = self._cores_by_path.get(path_str)
|
|
124
|
-
if hit is not None:
|
|
125
|
-
return hit
|
|
126
165
|
vc = _veccore()
|
|
127
166
|
with self._registry_lock: # one builder wins; a racing caller re-reads the cache
|
|
167
|
+
try:
|
|
168
|
+
current_stat = os.stat(path_str)
|
|
169
|
+
current_identity = (current_stat.st_dev, current_stat.st_ino)
|
|
170
|
+
except FileNotFoundError:
|
|
171
|
+
current_identity = None
|
|
128
172
|
hit = self._cores_by_path.get(path_str)
|
|
129
|
-
if hit is not None:
|
|
173
|
+
if hit is not None and self._store_file_identities[path_str] == current_identity:
|
|
130
174
|
return hit
|
|
131
175
|
try:
|
|
132
|
-
|
|
176
|
+
from .semantic_sqlite import SqliteStreamingStore
|
|
177
|
+
store = SqliteStreamingStore(path=path_str)
|
|
133
178
|
except vc.Unavailable as e: # store_unusable → symgrep's index_unusable
|
|
134
179
|
raise Unavailable(f"the semantic index at {path_str} could not be opened ({type(e).__name__}: "
|
|
135
180
|
f"{str(e)[:80]}) — it may be corrupt. Delete it and re-run `symgrep --index`.",
|
|
@@ -142,6 +187,10 @@ class SemanticStoreRegistry:
|
|
|
142
187
|
core.register(vc.CollectionSpec(name=COLLECTION, backend="sqlite_numpy", embedder=f"fastembed:{MODEL}",
|
|
143
188
|
dim=DIM, payload_schema=("qual", "path", "lineno", "what", "stamp",
|
|
144
189
|
"unit", "parent_key", "block_start", "block_end")))
|
|
190
|
+
opened_stat = os.stat(path_str)
|
|
191
|
+
self._store_file_identities[path_str] = (opened_stat.st_dev, opened_stat.st_ino)
|
|
192
|
+
if hit is not None:
|
|
193
|
+
hit[0].retire_readers()
|
|
145
194
|
self._cores_by_path[path_str] = (store, core)
|
|
146
195
|
return self._cores_by_path[path_str]
|
|
147
196
|
|
|
@@ -152,30 +201,15 @@ _SEMANTIC_STORE_REGISTRY = SemanticStoreRegistry()
|
|
|
152
201
|
def _core_for(path_str):
|
|
153
202
|
"""(store, core) for the index at `path_str`, shared safely across daemon threads.
|
|
154
203
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
never does (it rebuilds in place; a corrupt index is deleted by the USER, then a new process starts).
|
|
160
|
-
Tests point $SYMGREP_INDEX at distinct temp paths, so each gets its own entry. The shared embedder
|
|
161
|
-
port serializes disposable inference processes; native sessions never remain in this cache.
|
|
204
|
+
In-place WAL commits are visible to each SQL search snapshot. A replaced inode
|
|
205
|
+
atomically publishes a fresh store/core pair; old idle readers retire immediately
|
|
206
|
+
and in-flight snapshots release their mappings on return. Query inference stays
|
|
207
|
+
in disposable children and corpus vectors stay in SQLite, never in a matrix.
|
|
162
208
|
Maps veccore's store-open
|
|
163
209
|
failure to symgrep's index_unusable; a missing veccore surfaced earlier as Unavailable(no_embedder)."""
|
|
164
210
|
return _SEMANTIC_STORE_REGISTRY.core_for_index(path_str)
|
|
165
211
|
|
|
166
212
|
|
|
167
|
-
def _search_with_shared_store_lock(store, core, query, limit):
|
|
168
|
-
"""Keep the full corpus allocation and scoring inside the store's reentrant lock.
|
|
169
|
-
|
|
170
|
-
veccore 0.3.1 constructs a fresh matrix even when its saved matrix cache is valid.
|
|
171
|
-
Its method-level lock ends before scoring; holding it across search prevents
|
|
172
|
-
simultaneous requests from retaining a separate transient corpus each.
|
|
173
|
-
"""
|
|
174
|
-
with store._con_lock:
|
|
175
|
-
return core.search(COLLECTION, query, limit,
|
|
176
|
-
_veccore().AuditEnvelope(who="symgrep", why="search code by meaning"))
|
|
177
|
-
|
|
178
|
-
|
|
179
213
|
def _embedder():
|
|
180
214
|
"""The availability PROBE callers/tests use: returns the (unloaded) local embedder, or raises
|
|
181
215
|
Unavailable(no_embedder) if the stack (veccore + fastembed) is not installed. Does not load the model —
|
|
@@ -500,6 +534,24 @@ def _is_generated(p: pathlib.Path, rp: pathlib.Path) -> bool:
|
|
|
500
534
|
return False
|
|
501
535
|
|
|
502
536
|
|
|
537
|
+
def _iter_semantic_census_files(census_root):
|
|
538
|
+
"""Yield exactly the existing indexable census, pruning only rejected subtrees."""
|
|
539
|
+
root_has_mkdocs = (census_root / "mkdocs.yml").exists()
|
|
540
|
+
for directory, directories, filenames in os.walk(census_root, followlinks=False):
|
|
541
|
+
directories[:] = [name for name in directories
|
|
542
|
+
if name not in SKIP_DIRS and not name.startswith(".")
|
|
543
|
+
and not (name == "site" and root_has_mkdocs)]
|
|
544
|
+
for filename in filenames:
|
|
545
|
+
# Match Path.suffix for dotfiles and trailing dots before allocating Paths.
|
|
546
|
+
suffix_separator = filename.rfind(".")
|
|
547
|
+
suffix = filename[suffix_separator:].lower() if 0 < suffix_separator < len(filename) - 1 else ""
|
|
548
|
+
if suffix not in INDEXABLE_EXTS:
|
|
549
|
+
continue
|
|
550
|
+
candidate = pathlib.Path(directory) / filename
|
|
551
|
+
if candidate.is_file() and not _is_generated(candidate, census_root):
|
|
552
|
+
yield candidate
|
|
553
|
+
|
|
554
|
+
|
|
503
555
|
def _looks_minified(text: str) -> bool:
|
|
504
556
|
return any(len(line) > MINIFIED_LINE_CHARS for line in text.splitlines())
|
|
505
557
|
|
|
@@ -594,6 +646,28 @@ def _module_name(f: pathlib.Path) -> str:
|
|
|
594
646
|
return ".".join(reversed(parts))
|
|
595
647
|
|
|
596
648
|
|
|
649
|
+
def _collect_semantic_file_metadata(rows, in_scope):
|
|
650
|
+
"""Consume every bookkeeping row once, retaining only distinct file metadata."""
|
|
651
|
+
in_index = out_of_scope = 0
|
|
652
|
+
file_stamp = {}
|
|
653
|
+
indexed_paths = set()
|
|
654
|
+
path_scope = {}
|
|
655
|
+
for path, stamp in rows:
|
|
656
|
+
in_index += 1
|
|
657
|
+
scoped = path_scope.get(path)
|
|
658
|
+
if scoped is None:
|
|
659
|
+
canonical = os.path.realpath(path)
|
|
660
|
+
indexed_paths.add(canonical)
|
|
661
|
+
scoped = in_scope(canonical)
|
|
662
|
+
path_scope[path] = scoped
|
|
663
|
+
if scoped:
|
|
664
|
+
file_stamp.setdefault(path, stamp)
|
|
665
|
+
else:
|
|
666
|
+
out_of_scope += 1
|
|
667
|
+
return in_index, out_of_scope, file_stamp, indexed_paths
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
@_SEMANTIC_STORE_REGISTRY.bound_semantic_request
|
|
597
671
|
def search_by_meaning(query: str, roots: List[str], k: int = 8,
|
|
598
672
|
unit: str = "symbol", kind: Optional[str] = None,
|
|
599
673
|
globs: Optional[List[str]] = None,
|
|
@@ -616,18 +690,27 @@ def search_by_meaning(query: str, roots: List[str], k: int = 8,
|
|
|
616
690
|
functions and blocks in one list. The returned tuple shape is identical for every unit; block span +
|
|
617
691
|
provenance ride `search_by_meaning.last_hits` (aligned to the return) so the tuple contract never changes.
|
|
618
692
|
"""
|
|
693
|
+
from .capability_registry import parse_semantic_search_unit
|
|
694
|
+
unit = parse_semantic_search_unit(unit).value
|
|
695
|
+
root_abs = [os.path.realpath(r) for r in (roots or [])]
|
|
696
|
+
|
|
697
|
+
def in_scope(p: str) -> bool:
|
|
698
|
+
if not root_abs:
|
|
699
|
+
return True
|
|
700
|
+
return any(p == r or p.startswith(r.rstrip(os.sep) + os.sep) for r in root_abs)
|
|
701
|
+
|
|
619
702
|
con = _db()
|
|
620
703
|
try:
|
|
621
704
|
store, core = _core_for(str(index_path())) # Unavailable(no_embedder) if veccore missing; index_unusable if corrupt
|
|
622
|
-
|
|
705
|
+
has_index = store.has_points(COLLECTION)
|
|
623
706
|
# NO_INDEX is symgrep's own honesty: veccore's search on an empty collection returns [] ("no matches"),
|
|
624
707
|
# which must NEVER read as "nothing does this". An empty collection is "not indexed" — say so, loudly.
|
|
625
|
-
if not
|
|
708
|
+
if not has_index:
|
|
626
709
|
raise Unavailable(
|
|
627
710
|
f"no semantic index at {index_path()} — this is 'not indexed', NOT 'nothing matches'. Build it:\n"
|
|
628
711
|
" symgrep --index <root> [<root> ...]", code="no_index")
|
|
629
712
|
# symgrep's bookkeeping is the authority on which keys are LIVE functions and on the freshness stamp
|
|
630
|
-
#
|
|
713
|
+
# Stream rows before retrieval; retain metadata once per file, never once per symbol.
|
|
631
714
|
# UNIT FILTER — fixed clauses chosen by branch (no value interpolation). Default 'symbol' selects
|
|
632
715
|
# symbol rows + legacy NULLs, so `live` is exactly the pre-v6 set → the symbol search is byte-stable.
|
|
633
716
|
unit_sql = {"block": " AND unit = 'block'", "data": " AND unit = 'data'",
|
|
@@ -638,65 +721,59 @@ def search_by_meaning(query: str, roots: List[str], k: int = 8,
|
|
|
638
721
|
if kind:
|
|
639
722
|
kind_sql = " AND kind = ?"
|
|
640
723
|
params.append(kind)
|
|
641
|
-
|
|
642
|
-
"SELECT
|
|
643
|
-
params)
|
|
724
|
+
metadata_rows = con.execute(
|
|
725
|
+
"SELECT path, stamp FROM fn WHERE what IS NOT NULL AND what != ''" + unit_sql + kind_sql,
|
|
726
|
+
params)
|
|
727
|
+
in_index, out_of_scope, file_stamp, indexed_paths = _collect_semantic_file_metadata(
|
|
728
|
+
metadata_rows, in_scope)
|
|
644
729
|
finally:
|
|
645
730
|
con.close()
|
|
646
731
|
|
|
647
|
-
|
|
648
|
-
|
|
732
|
+
# Mechanical file/liveness predicates run inside SQLite BEFORE its bounded
|
|
733
|
+
# top-k sorter; no corpus-wide Hit/payload list is ever returned to Python.
|
|
734
|
+
allowed = None
|
|
735
|
+
if globs:
|
|
736
|
+
from . import ripgrep
|
|
737
|
+
try:
|
|
738
|
+
ripgrep.rg_binary()
|
|
739
|
+
except ripgrep.RipgrepNotFound as e:
|
|
740
|
+
raise Unavailable("--glob scoping needs ripgrep; install rg or drop --glob.",
|
|
741
|
+
code="glob_needs_rg") from e
|
|
742
|
+
glob_roots = root_abs or [os.path.realpath(r) for r in indexed_roots()]
|
|
743
|
+
allowed = {os.path.realpath(p) for p in ripgrep.list_files(
|
|
744
|
+
glob_roots, include_ignored=True, globs=globs)}
|
|
649
745
|
|
|
650
|
-
|
|
651
|
-
|
|
746
|
+
scope_paths = {}
|
|
747
|
+
|
|
748
|
+
def semantic_sqlite_scope(key, payload_json):
|
|
749
|
+
if not root_abs and allowed is None and not lang:
|
|
652
750
|
return True
|
|
653
|
-
|
|
654
|
-
|
|
751
|
+
payload = json.loads(payload_json)
|
|
752
|
+
raw_path = payload.get("path", "") or ""
|
|
753
|
+
path = scope_paths.get(raw_path)
|
|
754
|
+
if path is None:
|
|
755
|
+
path = os.path.realpath(raw_path)
|
|
756
|
+
scope_paths[raw_path] = path
|
|
757
|
+
if not in_scope(path) or (allowed is not None and path not in allowed):
|
|
758
|
+
return False
|
|
759
|
+
if lang:
|
|
760
|
+
from .enclosing import TS_LANG_BY_EXT
|
|
761
|
+
return TS_LANG_BY_EXT.get(os.path.splitext(path)[1].lower()) == lang
|
|
762
|
+
return True
|
|
655
763
|
|
|
656
|
-
# Fetch ALL nearest (the backend brute-forces the whole matrix regardless), then apply symgrep's roots +
|
|
657
|
-
# liveness filters and take the top k — so the scope filter is EXACT (never an over-fetch that could drop
|
|
658
|
-
# a true in-scope hit ranked past an arbitrary cut).
|
|
659
764
|
try:
|
|
660
|
-
|
|
765
|
+
kept = core.search(COLLECTION, query, k,
|
|
766
|
+
_veccore().AuditEnvelope(who="symgrep", why="search code by meaning"),
|
|
767
|
+
filters={"symgrep_scope": semantic_sqlite_scope,
|
|
768
|
+
"symgrep_unit": unit, "symgrep_kind": kind})
|
|
661
769
|
except Unavailable:
|
|
662
|
-
raise
|
|
770
|
+
raise
|
|
663
771
|
except Exception as e:
|
|
664
772
|
vc = _veccore()
|
|
665
|
-
if isinstance(e, vc.VeccoreError):
|
|
773
|
+
if isinstance(e, vc.VeccoreError):
|
|
666
774
|
code = _VECCORE_CODE_MAP.get(getattr(e, "code", ""), "index_unusable")
|
|
667
775
|
raise Unavailable(str(e), code=code) from e
|
|
668
|
-
raise
|
|
669
|
-
|
|
670
|
-
# LIVENESS: a hit whose key is no longer a live function (deleted/renamed → its bookkeeping row is gone)
|
|
671
|
-
# is an orphan veccore vector — filtered out here, never surfaced. This is the BETWEEN-BUILDS guarantee (a
|
|
672
|
-
# function deleted since the last build that covered it); the next such build reclaims the vector via
|
|
673
|
-
# veccore's delete seam (_embed_live), so orphans do not accumulate.
|
|
674
|
-
kept = [h for h in hits if h.id in live and in_scope((h.payload or {}).get("path", ""))]
|
|
675
|
-
|
|
676
|
-
# GLOB / LANG SCOPING (honest: the literal path/extension decides membership — meaning only RANKS within
|
|
677
|
-
# it, it never SELECTS files). Applied BEFORE the top-k cut below so filtering never under-fills. GLOB
|
|
678
|
-
# reuses ripgrep's OWN glob engine (ignore-neutral via --no-ignore, so an indexed-but-gitignored file
|
|
679
|
-
# still matches its glob and the scope equals what the name search would show); if rg is absent we RAISE
|
|
680
|
-
# rather than return a result that quietly ignores the glob. LANG filters by the same extension→grammar
|
|
681
|
-
# map symbol extraction uses. Both are mechanical set membership (parsing), not a meaning judgement.
|
|
682
|
-
def _hit_path(h):
|
|
683
|
-
return os.path.realpath((h.payload or {}).get("path", "") or "")
|
|
684
|
-
if globs:
|
|
685
|
-
from . import ripgrep
|
|
686
|
-
try:
|
|
687
|
-
ripgrep.rg_binary() # RipgrepNotFound is a deliberate stop — surfaced as Unavailable, not swallowed
|
|
688
|
-
except ripgrep.RipgrepNotFound as e:
|
|
689
|
-
raise Unavailable(
|
|
690
|
-
"--glob scoping needs ripgrep, which is not available — refusing to return a semantic result "
|
|
691
|
-
"that silently ignores the requested glob. Install rg, or drop --glob.",
|
|
692
|
-
code="glob_needs_rg") from e
|
|
693
|
-
glob_roots = root_abs or [os.path.realpath(r) for r in indexed_roots()]
|
|
694
|
-
allowed = {os.path.realpath(p) for p in ripgrep.list_files(glob_roots, include_ignored=True, globs=globs)}
|
|
695
|
-
kept = [h for h in kept if _hit_path(h) in allowed]
|
|
696
|
-
if lang:
|
|
697
|
-
from .enclosing import TS_LANG_BY_EXT
|
|
698
|
-
kept = [h for h in kept
|
|
699
|
-
if TS_LANG_BY_EXT.get(os.path.splitext(_hit_path(h))[1].lower()) == lang]
|
|
776
|
+
raise
|
|
700
777
|
|
|
701
778
|
scored = [(h.score, (h.payload or {}).get("qual"), (h.payload or {}).get("path"),
|
|
702
779
|
(h.payload or {}).get("lineno"), (h.payload or {}).get("what")) for h in kept[:k]]
|
|
@@ -707,12 +784,6 @@ def search_by_meaning(query: str, roots: List[str], k: int = 8,
|
|
|
707
784
|
# STALENESS SIGNAL + scope accounting, from symgrep's bookkeeping. A DISPLAY signal (reported to the
|
|
708
785
|
# reader), never a decision — the re-DESCRIBE decision is keyed on body_hash (the intrinsic), and the
|
|
709
786
|
# stamp only drives the "changed on disk since indexed" line. One stat per distinct in-scope file.
|
|
710
|
-
in_index = len(book)
|
|
711
|
-
out_of_scope = sum(1 for _k, p, _s in book if not in_scope(p))
|
|
712
|
-
file_stamp = {}
|
|
713
|
-
for _k, p, s in book:
|
|
714
|
-
if in_scope(p):
|
|
715
|
-
file_stamp.setdefault(p, s)
|
|
716
787
|
stale, unknown_age, gone_files = 0, 0, 0
|
|
717
788
|
for path, stamp in file_stamp.items():
|
|
718
789
|
try:
|
|
@@ -740,16 +811,14 @@ def search_by_meaning(query: str, roots: List[str], k: int = 8,
|
|
|
740
811
|
# measured count could exceed the claimed one.) The denominator for the freshness rate above is
|
|
741
812
|
# file_stamp (asserted non-empty by `in_index`); this count is a raw count, no rate, so it needs only its
|
|
742
813
|
# named membership constraint, which is `indexed_paths`.
|
|
743
|
-
indexed_paths = {os.path.realpath(p) for _k, p, _s in book}
|
|
744
814
|
scan_roots = root_abs or [os.path.realpath(r) for r in indexed_roots()]
|
|
745
815
|
not_indexed = 0
|
|
746
816
|
for _root in scan_roots:
|
|
747
817
|
rp = pathlib.Path(_root)
|
|
748
818
|
if not rp.is_dir():
|
|
749
819
|
continue
|
|
750
|
-
for p in rp
|
|
751
|
-
if
|
|
752
|
-
and os.path.realpath(p) not in indexed_paths:
|
|
820
|
+
for p in _iter_semantic_census_files(rp):
|
|
821
|
+
if os.path.realpath(p) not in indexed_paths:
|
|
753
822
|
not_indexed += 1
|
|
754
823
|
search_by_meaning.last_scope = {"in_index": in_index, "out_of_scope": out_of_scope,
|
|
755
824
|
"bad_dim": 0, "roots": root_abs,
|
|
@@ -995,9 +1064,10 @@ def _embed_live(con, fns, gone=frozenset()) -> int:
|
|
|
995
1064
|
con.execute("SELECT key, qual, path, lineno, what, stamp, unit, parent_key, end_lineno, signature, "
|
|
996
1065
|
"kind FROM fn WHERE what IS NOT NULL AND what != ''")}
|
|
997
1066
|
live_keys = {f["key"] for f in fns} & set(book)
|
|
1067
|
+
store, core = _core_for(str(index_path()))
|
|
998
1068
|
if not live_keys and not gone:
|
|
1069
|
+
store.prepare_ann_index(COLLECTION)
|
|
999
1070
|
return 0
|
|
1000
|
-
store, core = _core_for(str(index_path()))
|
|
1001
1071
|
# EVERY live description goes to veccore; its content-sha re-embeds ONLY the drifted text (a re-described
|
|
1002
1072
|
# function whose sentence CHANGED re-embeds; an unchanged one is skipped, $0). Filtering to 'vector
|
|
1003
1073
|
# missing' here would WRONGLY skip a re-described function whose stored vector still holds its OLD
|
|
@@ -1024,6 +1094,7 @@ def _embed_live(con, fns, gone=frozenset()) -> int:
|
|
|
1024
1094
|
"block_start": lineno if is_spanned else None,
|
|
1025
1095
|
"block_end": end_lineno if is_spanned else None}})
|
|
1026
1096
|
if not items:
|
|
1097
|
+
store.prepare_ann_index(COLLECTION)
|
|
1027
1098
|
return 0
|
|
1028
1099
|
vc = _veccore()
|
|
1029
1100
|
embedded = 0
|
|
@@ -1033,4 +1104,8 @@ def _embed_live(con, fns, gone=frozenset()) -> int:
|
|
|
1033
1104
|
result = core.embed_upsert(COLLECTION, items[start:start + EMBED_UPSERT_CHUNK],
|
|
1034
1105
|
vc.AuditEnvelope(who="symgrep", why="index code by meaning"))
|
|
1035
1106
|
embedded += result.n_embedded
|
|
1107
|
+
# Both CLI reindex and the detached freshness worker reach this seam. Publish
|
|
1108
|
+
# the durable index after vector reconciliation; readers keep their committed
|
|
1109
|
+
# snapshot while a new generation is trained and assigned in bounded chunks.
|
|
1110
|
+
store.prepare_ann_index(COLLECTION)
|
|
1036
1111
|
return embedded
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Portable durable inverted-file cosine index with bounded training workspaces.
|
|
2
|
+
|
|
3
|
+
Only centroids, a bounded training reservoir and one assignment batch enter Python.
|
|
4
|
+
SQLite B-trees hold postings; queries rerank their original float32 vectors exactly.
|
|
5
|
+
"""
|
|
6
|
+
import json
|
|
7
|
+
import numpy as np
|
|
8
|
+
from .semantic_sqlite_config import (IVF_BUCKET_COUNT, IVF_TRAINING_ROWS, IVF_TRAINING_ITERATIONS,
|
|
9
|
+
IVF_ASSIGNMENT_ROWS, IVF_RANDOM_SEED)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def initialize_ivf_schema(connection):
|
|
13
|
+
existing_covering = connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='symgrep_ivf_vectors'").fetchone()
|
|
14
|
+
connection.executescript("""
|
|
15
|
+
CREATE TABLE IF NOT EXISTS symgrep_ivf_centroids(collection TEXT, bucket INTEGER, vector BLOB,
|
|
16
|
+
PRIMARY KEY(collection,bucket));
|
|
17
|
+
CREATE TABLE IF NOT EXISTS symgrep_ivf_vectors(collection TEXT, bucket INTEGER, id TEXT, vector BLOB, payload TEXT,
|
|
18
|
+
PRIMARY KEY(collection,bucket,id)) WITHOUT ROWID;
|
|
19
|
+
CREATE UNIQUE INDEX IF NOT EXISTS symgrep_ivf_covering_members ON symgrep_ivf_vectors(collection,id);
|
|
20
|
+
CREATE TABLE IF NOT EXISTS symgrep_ivf_model_config(collection TEXT PRIMARY KEY, signature TEXT NOT NULL);
|
|
21
|
+
CREATE TABLE IF NOT EXISTS symgrep_ivf_state(collection TEXT PRIMARY KEY, dirty INTEGER NOT NULL);
|
|
22
|
+
CREATE TRIGGER IF NOT EXISTS symgrep_ivf_covering_insert AFTER INSERT ON points BEGIN
|
|
23
|
+
INSERT OR REPLACE INTO symgrep_ivf_state VALUES(new.collection,1);
|
|
24
|
+
DELETE FROM symgrep_ivf_vectors WHERE collection=new.collection AND id=new.id;
|
|
25
|
+
END;
|
|
26
|
+
CREATE TRIGGER IF NOT EXISTS symgrep_ivf_covering_update AFTER UPDATE OF vec,payload,id,collection ON points BEGIN
|
|
27
|
+
INSERT OR REPLACE INTO symgrep_ivf_state VALUES(old.collection,1);
|
|
28
|
+
INSERT OR REPLACE INTO symgrep_ivf_state VALUES(new.collection,1);
|
|
29
|
+
DELETE FROM symgrep_ivf_vectors WHERE collection=old.collection AND id=old.id;
|
|
30
|
+
DELETE FROM symgrep_ivf_vectors WHERE collection=new.collection AND id=new.id;
|
|
31
|
+
END;
|
|
32
|
+
CREATE TRIGGER IF NOT EXISTS symgrep_ivf_covering_delete AFTER DELETE ON points BEGIN
|
|
33
|
+
DELETE FROM symgrep_ivf_vectors WHERE collection=old.collection AND id=old.id;
|
|
34
|
+
END;
|
|
35
|
+
""")
|
|
36
|
+
with connection:
|
|
37
|
+
connection.execute("INSERT OR REPLACE INTO symgrep_ivf_state SELECT DISTINCT collection,1 "
|
|
38
|
+
"FROM symgrep_ivf_centroids WHERE collection NOT IN "
|
|
39
|
+
"(SELECT collection FROM symgrep_ivf_model_config)")
|
|
40
|
+
if not existing_covering:
|
|
41
|
+
with connection:
|
|
42
|
+
connection.execute("INSERT OR REPLACE INTO symgrep_ivf_state SELECT DISTINCT collection,1 FROM symgrep_ivf_centroids")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def ivf_training_configuration_signature():
|
|
46
|
+
"""Persist all model-building choices; query probe changes do not retrain."""
|
|
47
|
+
return json.dumps((IVF_BUCKET_COUNT, IVF_TRAINING_ROWS, IVF_TRAINING_ITERATIONS,
|
|
48
|
+
IVF_ASSIGNMENT_ROWS, IVF_RANDOM_SEED))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def read_ivf_centroids(connection, collection):
|
|
52
|
+
rows = connection.execute('SELECT bucket,vector FROM symgrep_ivf_centroids WHERE collection=? ORDER BY bucket',
|
|
53
|
+
(collection,)).fetchall()
|
|
54
|
+
if any(bucket != ordinal for ordinal, (bucket, _vector) in enumerate(rows)):
|
|
55
|
+
raise ValueError("IVF centroid buckets must be contiguous")
|
|
56
|
+
return np.stack([np.frombuffer(row[1], dtype=np.float32) for row in rows]) if rows else None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def assign_ivf_rows(connection, collection, centroids, rows):
|
|
60
|
+
if not rows:
|
|
61
|
+
return
|
|
62
|
+
vectors = np.stack([np.frombuffer(blob, dtype=np.float32) for _, blob, _payload in rows])
|
|
63
|
+
buckets = np.argmax(vectors @ centroids.T, axis=1)
|
|
64
|
+
connection.executemany('INSERT OR REPLACE INTO symgrep_ivf_vectors VALUES(?,?,?,?,?)',
|
|
65
|
+
[(collection, int(bucket), row[0], row[1], row[2]) for row, bucket in zip(rows, buckets)])
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def rebuild_ivf_collection(connection, collection):
|
|
69
|
+
"""Publish a complete model and postings atomically; reservoir memory has a fixed ceiling."""
|
|
70
|
+
if min(IVF_BUCKET_COUNT, IVF_TRAINING_ROWS, IVF_TRAINING_ITERATIONS, IVF_ASSIGNMENT_ROWS) <= 0:
|
|
71
|
+
raise ValueError("IVF training configuration must be positive")
|
|
72
|
+
generator = np.random.default_rng(IVF_RANDOM_SEED)
|
|
73
|
+
reservoir = []
|
|
74
|
+
for ordinal, (blob,) in enumerate(connection.execute('SELECT vec FROM points WHERE collection=? ORDER BY id',
|
|
75
|
+
(collection,))):
|
|
76
|
+
if ordinal < IVF_TRAINING_ROWS:
|
|
77
|
+
reservoir.append(np.frombuffer(blob, dtype=np.float32).copy())
|
|
78
|
+
else:
|
|
79
|
+
selected = int(generator.integers(ordinal + 1))
|
|
80
|
+
if selected < IVF_TRAINING_ROWS:
|
|
81
|
+
reservoir[selected] = np.frombuffer(blob, dtype=np.float32).copy()
|
|
82
|
+
if not reservoir:
|
|
83
|
+
return False
|
|
84
|
+
training = np.stack(reservoir)
|
|
85
|
+
del reservoir # Release the sampling copies before clustering and index assignment.
|
|
86
|
+
centroids = training[generator.choice(len(training), min(IVF_BUCKET_COUNT, len(training)), replace=False)].copy()
|
|
87
|
+
for _iteration in range(IVF_TRAINING_ITERATIONS):
|
|
88
|
+
sums = np.zeros_like(centroids)
|
|
89
|
+
counts = np.zeros(len(centroids), dtype=np.int64)
|
|
90
|
+
for offset in range(0, len(training), IVF_ASSIGNMENT_ROWS):
|
|
91
|
+
batch = training[offset:offset + IVF_ASSIGNMENT_ROWS]
|
|
92
|
+
labels = np.argmax(batch @ centroids.T, axis=1)
|
|
93
|
+
np.add.at(sums, labels, batch)
|
|
94
|
+
np.add.at(counts, labels, 1)
|
|
95
|
+
occupied = counts > 0
|
|
96
|
+
centroids[occupied] = sums[occupied]
|
|
97
|
+
centroids /= np.maximum(np.linalg.norm(centroids, axis=1, keepdims=True), np.finfo(np.float32).tiny)
|
|
98
|
+
connection.execute('DELETE FROM symgrep_ivf_centroids WHERE collection=?', (collection,))
|
|
99
|
+
connection.executemany('INSERT INTO symgrep_ivf_centroids VALUES(?,?,?)',
|
|
100
|
+
[(collection, bucket, vector.tobytes()) for bucket, vector in enumerate(centroids)])
|
|
101
|
+
connection.execute('DELETE FROM symgrep_ivf_vectors WHERE collection=?', (collection,))
|
|
102
|
+
cursor = connection.execute('SELECT id,vec,payload FROM points WHERE collection=?', (collection,))
|
|
103
|
+
while rows := cursor.fetchmany(IVF_ASSIGNMENT_ROWS):
|
|
104
|
+
assign_ivf_rows(connection, collection, centroids, rows)
|
|
105
|
+
connection.execute('INSERT OR REPLACE INTO symgrep_ivf_model_config VALUES(?,?)',
|
|
106
|
+
(collection, ivf_training_configuration_signature()))
|
|
107
|
+
connection.execute('INSERT OR REPLACE INTO symgrep_ivf_state VALUES(?,0)', (collection,))
|
|
108
|
+
return True
|