symgrep-codesearch 0.9.0__tar.gz → 0.9.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. {symgrep_codesearch-0.9.0/src/symgrep_codesearch.egg-info → symgrep_codesearch-0.9.2}/PKG-INFO +1 -1
  2. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/pyproject.toml +3 -1
  3. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/capability_registry.py +66 -0
  4. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/semantic.py +160 -85
  5. symgrep_codesearch-0.9.2/src/symgrep/semantic_ivf.py +108 -0
  6. symgrep_codesearch-0.9.2/src/symgrep/semantic_sqlite.py +295 -0
  7. symgrep_codesearch-0.9.2/src/symgrep/semantic_sqlite_config.py +19 -0
  8. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2/src/symgrep_codesearch.egg-info}/PKG-INFO +1 -1
  9. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/SOURCES.txt +6 -0
  10. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_embedding_residency_pytest.py +20 -42
  11. symgrep_codesearch-0.9.2/tests/test_semantic_census_pruning_pytest.py +42 -0
  12. symgrep_codesearch-0.9.2/tests/test_sqlite_ivf_retrieval_pytest.py +278 -0
  13. symgrep_codesearch-0.9.2/tests/test_sqlite_streaming_retrieval_pytest.py +220 -0
  14. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/LICENSE +0 -0
  15. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/README.md +0 -0
  16. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/setup.cfg +0 -0
  17. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/__init__.py +0 -0
  18. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/absence_adjudication_writer.py +0 -0
  19. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/archives.py +0 -0
  20. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/blocks.py +0 -0
  21. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/capability_registry_data.jsonl +0 -0
  22. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/capability_surface_checks.py +0 -0
  23. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/classfile.py +0 -0
  24. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/cli.py +0 -0
  25. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/data_symbols.py +0 -0
  26. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/docsymbols.py +0 -0
  27. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/embedding_process.py +0 -0
  28. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/enclosing.py +0 -0
  29. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/glob_recall.py +0 -0
  30. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/mcp_server.py +0 -0
  31. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/mcp_structured.py +0 -0
  32. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/report.py +0 -0
  33. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/resolve.py +0 -0
  34. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/resolve_js.py +0 -0
  35. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/resolve_python.py +0 -0
  36. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/ripgrep.py +0 -0
  37. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/rss_ceiling.py +0 -0
  38. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/runtime_paths.py +0 -0
  39. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/structural.py +0 -0
  40. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep/structure.py +0 -0
  41. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/dependency_links.txt +0 -0
  42. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/entry_points.txt +0 -0
  43. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/requires.txt +0 -0
  44. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/top_level.txt +0 -0
  45. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_absence_adjudication_authorship_pytest.py +0 -0
  46. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_absence_adjudication_binding_pytest.py +0 -0
  47. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_blocks_extraction.py +0 -0
  48. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_blocks_retrieval.py +0 -0
  49. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_bulk_describe.py +0 -0
  50. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_capability_documentation_guard_pytest.py +0 -0
  51. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_capability_surface_parity_guard_pytest.py +0 -0
  52. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_classfile.py +0 -0
  53. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_codex_hook_payload.py +0 -0
  54. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_data_symbols.py +0 -0
  55. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_describe_lane_confinement.py +0 -0
  56. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_doc_symbols.py +0 -0
  57. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_file_structure.py +0 -0
  58. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_freshness.py +0 -0
  59. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_honest_recall.py +0 -0
  60. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_http_ensure.py +0 -0
  61. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_http_rss_shutdown_pytest.py +0 -0
  62. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_index_reconstruct.py +0 -0
  63. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_index_scope.py +0 -0
  64. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_index_worker.py +0 -0
  65. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_mcp_error_log.py +0 -0
  66. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_mcp_registry_behavior_pytest.py +0 -0
  67. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_mcp_structured.py +0 -0
  68. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_mcp_thread_safety.py +0 -0
  69. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_mcp_version.py +0 -0
  70. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_name_uniqueness.py +0 -0
  71. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_nudge_classify.py +0 -0
  72. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_nudge_runaway.py +0 -0
  73. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_report_filters.py +0 -0
  74. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_resolve.py +0 -0
  75. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_resolve_js.py +0 -0
  76. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_resolve_python.py +0 -0
  77. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_resolve_r2.py +0 -0
  78. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_review_regressions.py +0 -0
  79. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_rss_ceiling_pytest.py +0 -0
  80. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_semantic_freshness_signal.py +0 -0
  81. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_semantic_glob_lang.py +0 -0
  82. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_sites_and_symbols.py +0 -0
  83. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_structural.py +0 -0
  84. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_symbol_vocabulary.py +0 -0
  85. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_ts_fallback_loud.py +0 -0
  86. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_ts_symbol_coverage.py +0 -0
  87. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_usage.py +0 -0
  88. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_veccore_adoption.py +0 -0
  89. {symgrep_codesearch-0.9.0 → symgrep_codesearch-0.9.2}/tests/test_worker_lock_recovery_pytest.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: symgrep-codesearch
3
- Version: 0.9.0
3
+ Version: 0.9.2
4
4
  Summary: Code search with honest recall — structure-aware grep that tells you what it does not know, plus search by meaning
5
5
  License-Expression: MIT
6
6
  Project-URL: Repository, https://github.com/ashdamle/symgrep
@@ -7,6 +7,8 @@ build-backend = "setuptools.build_meta"
7
7
  # Semgrep team. The IMPORT name is unchanged — `import symgrep` — so consumers (honestreview, spendguard,
8
8
  # 7thsense, warden) are unaffected by the distribution rename. The package dir stays src/symgrep.
9
9
  name = "symgrep-codesearch"
10
+ # 0.9.1: shared, bounded SQLite corpus search; live committed reindex visibility
11
+ # and atomic replacement handling, preserving disposable query embedding children.
10
12
  # 0.9.0: fix ONNX-arena and thread-local-matrix retention with disposable serialized
11
13
  # embedding processes and per-index thread-safe stores (~1s latency per embedding round).
12
14
  # Add the SYMGREP_MAX_RSS_MB self-guard; require psutil>=5.9 and veccore>=0.3.1.
@@ -39,7 +41,7 @@ name = "symgrep-codesearch"
39
41
  # semantic search. Minor bump (dependency surface changed; no breaking import/CLI/MCP change).
40
42
  # 0.5.0: added the public `file_structure` library API (+ its MCP tool and `symgrep structure` CLI), import
41
43
  # module-name normalization, and the FileStructure `connects` list (structure schema v2).
42
- version = "0.9.0"
44
+ version = "0.9.2"
43
45
  description = "Code search with honest recall — structure-aware grep that tells you what it does not know, plus search by meaning"
44
46
  readme = "README.md"
45
47
  # 3.11+ : the maintained grammar pack (tree-sitter-language-pack, a CORE dep below) ships abi3 wheels for
@@ -7,6 +7,7 @@ import inspect
7
7
  import hashlib
8
8
  from datetime import datetime, timezone
9
9
  import json
10
+ from enum import StrEnum
10
11
  from typing import Any, Optional, List
11
12
 
12
13
  CAPABILITY_DATA_RESOURCE = 'capability_registry_data.jsonl'
@@ -36,6 +37,52 @@ RUNTIME_ENVIRONMENT_SETTINGS = (
36
37
  'embedding models; zero/negative/nonfinite values are errors, never a disable switch. '
37
38
  'If psutil is missing, RSS containment is Unavailable and disabled; one CRITICAL '
38
39
  'entry is durably flushed to the MCP error log explaining that containment is OFF.'),
40
+ RuntimeEnvironmentSetting(
41
+ 'SYMGREP_SQLITE_READERS', 'symgrep.semantic_sqlite_config:SQLITE_READER_COUNT',
42
+ 'Positive number of reusable SQLite semantic readers per index and admitted complete '
43
+ 'semantic requests per process. Admission also bounds bookkeeping/freshness allocations; '
44
+ 'additional requests wait before allocating request metadata. Corpus matrices are never loaded.'),
45
+ RuntimeEnvironmentSetting(
46
+ 'SYMGREP_SQLITE_MMAP_BYTES', 'symgrep.semantic_sqlite_config:SQLITE_READER_MMAP_BYTES',
47
+ 'Nonnegative mmap limit in bytes per SQLite semantic reader. Mapped database pages '
48
+ 'are demand-paged by the OS; zero uses SQLite page caching without mmap.'),
49
+ RuntimeEnvironmentSetting(
50
+ 'SYMGREP_SQLITE_CACHE_KIB', 'symgrep.semantic_sqlite_config:SQLITE_READER_CACHE_KIB',
51
+ 'Positive SQLite page-cache budget in KiB per semantic reader. Corpus vectors '
52
+ 'remain in SQLite and only top-k results enter the Python result list.'),
53
+ RuntimeEnvironmentSetting(
54
+ 'SYMGREP_SQLITE_BUSY_MS', 'symgrep.semantic_sqlite_config:SQLITE_READER_BUSY_MS',
55
+ 'SQLite semantic reader busy timeout in milliseconds while concurrent indexing '
56
+ 'holds a database lock. WAL readers otherwise use committed statement snapshots.'),
57
+ RuntimeEnvironmentSetting(
58
+ 'SYMGREP_IVF_ENABLED', 'symgrep.semantic_sqlite_config:IVF_ENABLED',
59
+ 'Persistent SQLite inverted-file vector retrieval. Zero disables ANN and logs one '
60
+ 'honest warning before exact streaming cosine fallback. Centroids and postings live '
61
+ 'in the committed database; approximate candidates are reranked on original vectors. '
62
+ 'Reindex prepares live postings. Reader pooling and atomic generation swaps stay bounded.'),
63
+ RuntimeEnvironmentSetting(
64
+ 'SYMGREP_IVF_BUCKETS', 'symgrep.semantic_sqlite_config:IVF_BUCKET_COUNT',
65
+ 'Centroid count for the inverted-file index. Reindex atomically rebuilds when '
66
+ 'training settings differ from the durable model signature. More buckets '
67
+ 'narrow candidate scans and require recall measurement.'),
68
+ RuntimeEnvironmentSetting(
69
+ 'SYMGREP_IVF_PROBES', 'symgrep.semantic_sqlite_config:IVF_PROBE_COUNT',
70
+ 'Initial nearest centroid buckets for symbol queries without a kind filter. Other unit/kind '
71
+ 'scopes use every bucket and log exact coverage to preserve recall. More probes improve '
72
+ 'approximate recall; sparse results expand probes. Measure complete recall@k against exact streaming search.'),
73
+ RuntimeEnvironmentSetting(
74
+ 'SYMGREP_IVF_TRAINING_ROWS', 'symgrep.semantic_sqlite_config:IVF_TRAINING_ROWS',
75
+ 'Maximum reservoir sample rows used to train centroids; bounds training memory '
76
+ 'independently of corpus size.'),
77
+ RuntimeEnvironmentSetting(
78
+ 'SYMGREP_IVF_TRAINING_ITERATIONS', 'symgrep.semantic_sqlite_config:IVF_TRAINING_ITERATIONS',
79
+ 'Spherical centroid training iterations for a new index.'),
80
+ RuntimeEnvironmentSetting(
81
+ 'SYMGREP_IVF_ASSIGNMENT_ROWS', 'symgrep.semantic_sqlite_config:IVF_ASSIGNMENT_ROWS',
82
+ 'Maximum vector rows per centroid assignment batch during training and reindex.'),
83
+ RuntimeEnvironmentSetting(
84
+ 'SYMGREP_IVF_RANDOM_SEED', 'symgrep.semantic_sqlite_config:IVF_RANDOM_SEED',
85
+ 'Reproducible reservoir sampling and centroid initialization seed for new indexes.'),
39
86
  )
40
87
 
41
88
  @dataclass(frozen=True)
@@ -75,6 +122,25 @@ CAPABILITIES = load_symgrep_capability_declarations()
75
122
  def capability_by_id(identity):
76
123
  return next(cap for cap in CAPABILITIES if cap.id == identity)
77
124
 
125
+
126
+ SEMANTIC_UNIT_ARGUMENT = next(parameter for parameter in capability_by_id('search').params
127
+ if parameter.name == 'unit')
128
+ SemanticSearchUnit = StrEnum('SemanticSearchUnit',
129
+ {choice.upper(): choice for choice in SEMANTIC_UNIT_ARGUMENT.choices})
130
+
131
+
132
+ def parse_semantic_search_unit(value):
133
+ """Parse the declared unit grammar; aliases outside the public enum are errors.
134
+
135
+ In particular, plural spellings are not inferred from their meaning. The CLI,
136
+ MCP schema and direct store callers share the registry's canonical members.
137
+ """
138
+ try:
139
+ return SemanticSearchUnit(value)
140
+ except (TypeError, ValueError) as error:
141
+ raise ValueError(f'Invalid semantic unit {value!r}; expected one of '
142
+ f'{SEMANTIC_UNIT_ARGUMENT.choices!r}') from error
143
+
78
144
  def capability_description(capability):
79
145
  """Render full canonical prose and every parameter's semantics."""
80
146
  return '\n\n'.join([capability.summary, capability.description] + [f'{p.name}: {p.help}' for p in capability.params])
@@ -116,20 +116,65 @@ class SemanticStoreRegistry:
116
116
  def __init__(self):
117
117
  self._process_embedder = None
118
118
  self._cores_by_path = {}
119
+ self._store_file_identities = {}
119
120
  self._registry_lock = threading.Lock()
121
+ from .semantic_sqlite_config import SQLITE_READER_COUNT
122
+ if SQLITE_READER_COUNT <= 0:
123
+ raise ValueError("SQLite reader count must be positive")
124
+ # Bound whole request working sets, not only vector scans. Bookkeeping
125
+ # and filesystem freshness checks also allocate corpus-sized metadata.
126
+ self._semantic_request_slots = threading.BoundedSemaphore(SQLITE_READER_COUNT)
127
+ self._semantic_admission_lock = threading.Lock()
128
+ self._active_semantic_requests = 0
129
+ self._peak_semantic_requests = 0
130
+
131
+
132
+ def bound_semantic_request(self, operation):
133
+ """Queue before allocating metadata; admit at most the reader-pool capacity."""
134
+ from functools import wraps
135
+
136
+ @wraps(operation)
137
+ def execute_admitted_semantic_request(*args, **kwargs):
138
+ with self._semantic_request_slots:
139
+ with self._semantic_admission_lock:
140
+ self._active_semantic_requests += 1
141
+ self._peak_semantic_requests = max(self._peak_semantic_requests,
142
+ self._active_semantic_requests)
143
+ try:
144
+ return operation(*args, **kwargs)
145
+ finally:
146
+ with self._semantic_admission_lock:
147
+ self._active_semantic_requests -= 1
148
+
149
+ return execute_admitted_semantic_request
150
+
151
+ def semantic_request_metrics(self):
152
+ """Expose complete-request admission independently of SQL scan occupancy."""
153
+ with self._semantic_admission_lock:
154
+ return {"active_requests": self._active_semantic_requests,
155
+ "peak_requests": self._peak_semantic_requests}
156
+
157
+ def child_peak_rss_bytes(self):
158
+ """Read the disposable embedder's latest receipt under the holder lock."""
159
+ with self._registry_lock:
160
+ return (self._process_embedder.last_child_peak_rss_bytes
161
+ if self._process_embedder is not None else None)
120
162
 
121
163
  def core_for_index(self, path_str):
122
164
  """Build once under the registry lock; racing callers reuse the winning pair."""
123
- hit = self._cores_by_path.get(path_str)
124
- if hit is not None:
125
- return hit
126
165
  vc = _veccore()
127
166
  with self._registry_lock: # one builder wins; a racing caller re-reads the cache
167
+ try:
168
+ current_stat = os.stat(path_str)
169
+ current_identity = (current_stat.st_dev, current_stat.st_ino)
170
+ except FileNotFoundError:
171
+ current_identity = None
128
172
  hit = self._cores_by_path.get(path_str)
129
- if hit is not None:
173
+ if hit is not None and self._store_file_identities[path_str] == current_identity:
130
174
  return hit
131
175
  try:
132
- store = vc.SqliteNumpyStore(path=path_str)
176
+ from .semantic_sqlite import SqliteStreamingStore
177
+ store = SqliteStreamingStore(path=path_str)
133
178
  except vc.Unavailable as e: # store_unusable → symgrep's index_unusable
134
179
  raise Unavailable(f"the semantic index at {path_str} could not be opened ({type(e).__name__}: "
135
180
  f"{str(e)[:80]}) — it may be corrupt. Delete it and re-run `symgrep --index`.",
@@ -142,6 +187,10 @@ class SemanticStoreRegistry:
142
187
  core.register(vc.CollectionSpec(name=COLLECTION, backend="sqlite_numpy", embedder=f"fastembed:{MODEL}",
143
188
  dim=DIM, payload_schema=("qual", "path", "lineno", "what", "stamp",
144
189
  "unit", "parent_key", "block_start", "block_end")))
190
+ opened_stat = os.stat(path_str)
191
+ self._store_file_identities[path_str] = (opened_stat.st_dev, opened_stat.st_ino)
192
+ if hit is not None:
193
+ hit[0].retire_readers()
145
194
  self._cores_by_path[path_str] = (store, core)
146
195
  return self._cores_by_path[path_str]
147
196
 
@@ -152,30 +201,15 @@ _SEMANTIC_STORE_REGISTRY = SemanticStoreRegistry()
152
201
  def _core_for(path_str):
153
202
  """(store, core) for the index at `path_str`, shared safely across daemon threads.
154
203
 
155
- CACHE STALENESS, explicitly: the cache is PROCESS-LIVED and keyed by the index PATH — every symgrep CLI run
156
- is a fresh process, so it starts empty. Within one process a store's sqlite connection SEES other
157
- processes' committed writes (WAL), so a rebuild by a separate `symgrep --index` needs no invalidation. The
158
- only stale case is the SAME path deleted-and-recreated (new inode) inside THIS process — which symgrep
159
- never does (it rebuilds in place; a corrupt index is deleted by the USER, then a new process starts).
160
- Tests point $SYMGREP_INDEX at distinct temp paths, so each gets its own entry. The shared embedder
161
- port serializes disposable inference processes; native sessions never remain in this cache.
204
+ In-place WAL commits are visible to each SQL search snapshot. A replaced inode
205
+ atomically publishes a fresh store/core pair; old idle readers retire immediately
206
+ and in-flight snapshots release their mappings on return. Query inference stays
207
+ in disposable children and corpus vectors stay in SQLite, never in a matrix.
162
208
  Maps veccore's store-open
163
209
  failure to symgrep's index_unusable; a missing veccore surfaced earlier as Unavailable(no_embedder)."""
164
210
  return _SEMANTIC_STORE_REGISTRY.core_for_index(path_str)
165
211
 
166
212
 
167
- def _search_with_shared_store_lock(store, core, query, limit):
168
- """Keep the full corpus allocation and scoring inside the store's reentrant lock.
169
-
170
- veccore 0.3.1 constructs a fresh matrix even when its saved matrix cache is valid.
171
- Its method-level lock ends before scoring; holding it across search prevents
172
- simultaneous requests from retaining a separate transient corpus each.
173
- """
174
- with store._con_lock:
175
- return core.search(COLLECTION, query, limit,
176
- _veccore().AuditEnvelope(who="symgrep", why="search code by meaning"))
177
-
178
-
179
213
  def _embedder():
180
214
  """The availability PROBE callers/tests use: returns the (unloaded) local embedder, or raises
181
215
  Unavailable(no_embedder) if the stack (veccore + fastembed) is not installed. Does not load the model —
@@ -500,6 +534,24 @@ def _is_generated(p: pathlib.Path, rp: pathlib.Path) -> bool:
500
534
  return False
501
535
 
502
536
 
537
+ def _iter_semantic_census_files(census_root):
538
+ """Yield exactly the existing indexable census, pruning only rejected subtrees."""
539
+ root_has_mkdocs = (census_root / "mkdocs.yml").exists()
540
+ for directory, directories, filenames in os.walk(census_root, followlinks=False):
541
+ directories[:] = [name for name in directories
542
+ if name not in SKIP_DIRS and not name.startswith(".")
543
+ and not (name == "site" and root_has_mkdocs)]
544
+ for filename in filenames:
545
+ # Match Path.suffix for dotfiles and trailing dots before allocating Paths.
546
+ suffix_separator = filename.rfind(".")
547
+ suffix = filename[suffix_separator:].lower() if 0 < suffix_separator < len(filename) - 1 else ""
548
+ if suffix not in INDEXABLE_EXTS:
549
+ continue
550
+ candidate = pathlib.Path(directory) / filename
551
+ if candidate.is_file() and not _is_generated(candidate, census_root):
552
+ yield candidate
553
+
554
+
503
555
  def _looks_minified(text: str) -> bool:
504
556
  return any(len(line) > MINIFIED_LINE_CHARS for line in text.splitlines())
505
557
 
@@ -594,6 +646,28 @@ def _module_name(f: pathlib.Path) -> str:
594
646
  return ".".join(reversed(parts))
595
647
 
596
648
 
649
+ def _collect_semantic_file_metadata(rows, in_scope):
650
+ """Consume every bookkeeping row once, retaining only distinct file metadata."""
651
+ in_index = out_of_scope = 0
652
+ file_stamp = {}
653
+ indexed_paths = set()
654
+ path_scope = {}
655
+ for path, stamp in rows:
656
+ in_index += 1
657
+ scoped = path_scope.get(path)
658
+ if scoped is None:
659
+ canonical = os.path.realpath(path)
660
+ indexed_paths.add(canonical)
661
+ scoped = in_scope(canonical)
662
+ path_scope[path] = scoped
663
+ if scoped:
664
+ file_stamp.setdefault(path, stamp)
665
+ else:
666
+ out_of_scope += 1
667
+ return in_index, out_of_scope, file_stamp, indexed_paths
668
+
669
+
670
+ @_SEMANTIC_STORE_REGISTRY.bound_semantic_request
597
671
  def search_by_meaning(query: str, roots: List[str], k: int = 8,
598
672
  unit: str = "symbol", kind: Optional[str] = None,
599
673
  globs: Optional[List[str]] = None,
@@ -616,18 +690,27 @@ def search_by_meaning(query: str, roots: List[str], k: int = 8,
616
690
  functions and blocks in one list. The returned tuple shape is identical for every unit; block span +
617
691
  provenance ride `search_by_meaning.last_hits` (aligned to the return) so the tuple contract never changes.
618
692
  """
693
+ from .capability_registry import parse_semantic_search_unit
694
+ unit = parse_semantic_search_unit(unit).value
695
+ root_abs = [os.path.realpath(r) for r in (roots or [])]
696
+
697
+ def in_scope(p: str) -> bool:
698
+ if not root_abs:
699
+ return True
700
+ return any(p == r or p.startswith(r.rstrip(os.sep) + os.sep) for r in root_abs)
701
+
619
702
  con = _db()
620
703
  try:
621
704
  store, core = _core_for(str(index_path())) # Unavailable(no_embedder) if veccore missing; index_unusable if corrupt
622
- existing = store.existing_ids(COLLECTION)
705
+ has_index = store.has_points(COLLECTION)
623
706
  # NO_INDEX is symgrep's own honesty: veccore's search on an empty collection returns [] ("no matches"),
624
707
  # which must NEVER read as "nothing does this". An empty collection is "not indexed" — say so, loudly.
625
- if not existing:
708
+ if not has_index:
626
709
  raise Unavailable(
627
710
  f"no semantic index at {index_path()} — this is 'not indexed', NOT 'nothing matches'. Build it:\n"
628
711
  " symgrep --index <root> [<root> ...]", code="no_index")
629
712
  # symgrep's bookkeeping is the authority on which keys are LIVE functions and on the freshness stamp
630
- # (materialize before any veccore call so the con is free).
713
+ # Stream rows before retrieval; retain metadata once per file, never once per symbol.
631
714
  # UNIT FILTER — fixed clauses chosen by branch (no value interpolation). Default 'symbol' selects
632
715
  # symbol rows + legacy NULLs, so `live` is exactly the pre-v6 set → the symbol search is byte-stable.
633
716
  unit_sql = {"block": " AND unit = 'block'", "data": " AND unit = 'data'",
@@ -638,65 +721,59 @@ def search_by_meaning(query: str, roots: List[str], k: int = 8,
638
721
  if kind:
639
722
  kind_sql = " AND kind = ?"
640
723
  params.append(kind)
641
- book = con.execute(
642
- "SELECT key, path, stamp FROM fn WHERE what IS NOT NULL AND what != ''" + unit_sql + kind_sql,
643
- params).fetchall()
724
+ metadata_rows = con.execute(
725
+ "SELECT path, stamp FROM fn WHERE what IS NOT NULL AND what != ''" + unit_sql + kind_sql,
726
+ params)
727
+ in_index, out_of_scope, file_stamp, indexed_paths = _collect_semantic_file_metadata(
728
+ metadata_rows, in_scope)
644
729
  finally:
645
730
  con.close()
646
731
 
647
- live = {key for key, _p, _s in book}
648
- root_abs = [os.path.realpath(r) for r in (roots or [])]
732
+ # Mechanical file/liveness predicates run inside SQLite BEFORE its bounded
733
+ # top-k sorter; no corpus-wide Hit/payload list is ever returned to Python.
734
+ allowed = None
735
+ if globs:
736
+ from . import ripgrep
737
+ try:
738
+ ripgrep.rg_binary()
739
+ except ripgrep.RipgrepNotFound as e:
740
+ raise Unavailable("--glob scoping needs ripgrep; install rg or drop --glob.",
741
+ code="glob_needs_rg") from e
742
+ glob_roots = root_abs or [os.path.realpath(r) for r in indexed_roots()]
743
+ allowed = {os.path.realpath(p) for p in ripgrep.list_files(
744
+ glob_roots, include_ignored=True, globs=globs)}
649
745
 
650
- def in_scope(p: str) -> bool:
651
- if not root_abs:
746
+ scope_paths = {}
747
+
748
+ def semantic_sqlite_scope(key, payload_json):
749
+ if not root_abs and allowed is None and not lang:
652
750
  return True
653
- ap = os.path.realpath(p or "")
654
- return any(ap == r or ap.startswith(r.rstrip(os.sep) + os.sep) for r in root_abs)
751
+ payload = json.loads(payload_json)
752
+ raw_path = payload.get("path", "") or ""
753
+ path = scope_paths.get(raw_path)
754
+ if path is None:
755
+ path = os.path.realpath(raw_path)
756
+ scope_paths[raw_path] = path
757
+ if not in_scope(path) or (allowed is not None and path not in allowed):
758
+ return False
759
+ if lang:
760
+ from .enclosing import TS_LANG_BY_EXT
761
+ return TS_LANG_BY_EXT.get(os.path.splitext(path)[1].lower()) == lang
762
+ return True
655
763
 
656
- # Fetch ALL nearest (the backend brute-forces the whole matrix regardless), then apply symgrep's roots +
657
- # liveness filters and take the top k — so the scope filter is EXACT (never an over-fetch that could drop
658
- # a true in-scope hit ranked past an arbitrary cut).
659
764
  try:
660
- hits = _search_with_shared_store_lock(store, core, query, max(k, len(existing)))
765
+ kept = core.search(COLLECTION, query, k,
766
+ _veccore().AuditEnvelope(who="symgrep", why="search code by meaning"),
767
+ filters={"symgrep_scope": semantic_sqlite_scope,
768
+ "symgrep_unit": unit, "symgrep_kind": kind})
661
769
  except Unavailable:
662
- raise # symgrep's own — already a deliberate stop, propagate it
770
+ raise
663
771
  except Exception as e:
664
772
  vc = _veccore()
665
- if isinstance(e, vc.VeccoreError): # re-raise veccore's deliberate stop as symgrep's, same code
773
+ if isinstance(e, vc.VeccoreError):
666
774
  code = _VECCORE_CODE_MAP.get(getattr(e, "code", ""), "index_unusable")
667
775
  raise Unavailable(str(e), code=code) from e
668
- raise # anything else is a real bug — never swallow it
669
-
670
- # LIVENESS: a hit whose key is no longer a live function (deleted/renamed → its bookkeeping row is gone)
671
- # is an orphan veccore vector — filtered out here, never surfaced. This is the BETWEEN-BUILDS guarantee (a
672
- # function deleted since the last build that covered it); the next such build reclaims the vector via
673
- # veccore's delete seam (_embed_live), so orphans do not accumulate.
674
- kept = [h for h in hits if h.id in live and in_scope((h.payload or {}).get("path", ""))]
675
-
676
- # GLOB / LANG SCOPING (honest: the literal path/extension decides membership — meaning only RANKS within
677
- # it, it never SELECTS files). Applied BEFORE the top-k cut below so filtering never under-fills. GLOB
678
- # reuses ripgrep's OWN glob engine (ignore-neutral via --no-ignore, so an indexed-but-gitignored file
679
- # still matches its glob and the scope equals what the name search would show); if rg is absent we RAISE
680
- # rather than return a result that quietly ignores the glob. LANG filters by the same extension→grammar
681
- # map symbol extraction uses. Both are mechanical set membership (parsing), not a meaning judgement.
682
- def _hit_path(h):
683
- return os.path.realpath((h.payload or {}).get("path", "") or "")
684
- if globs:
685
- from . import ripgrep
686
- try:
687
- ripgrep.rg_binary() # RipgrepNotFound is a deliberate stop — surfaced as Unavailable, not swallowed
688
- except ripgrep.RipgrepNotFound as e:
689
- raise Unavailable(
690
- "--glob scoping needs ripgrep, which is not available — refusing to return a semantic result "
691
- "that silently ignores the requested glob. Install rg, or drop --glob.",
692
- code="glob_needs_rg") from e
693
- glob_roots = root_abs or [os.path.realpath(r) for r in indexed_roots()]
694
- allowed = {os.path.realpath(p) for p in ripgrep.list_files(glob_roots, include_ignored=True, globs=globs)}
695
- kept = [h for h in kept if _hit_path(h) in allowed]
696
- if lang:
697
- from .enclosing import TS_LANG_BY_EXT
698
- kept = [h for h in kept
699
- if TS_LANG_BY_EXT.get(os.path.splitext(_hit_path(h))[1].lower()) == lang]
776
+ raise
700
777
 
701
778
  scored = [(h.score, (h.payload or {}).get("qual"), (h.payload or {}).get("path"),
702
779
  (h.payload or {}).get("lineno"), (h.payload or {}).get("what")) for h in kept[:k]]
@@ -707,12 +784,6 @@ def search_by_meaning(query: str, roots: List[str], k: int = 8,
707
784
  # STALENESS SIGNAL + scope accounting, from symgrep's bookkeeping. A DISPLAY signal (reported to the
708
785
  # reader), never a decision — the re-DESCRIBE decision is keyed on body_hash (the intrinsic), and the
709
786
  # stamp only drives the "changed on disk since indexed" line. One stat per distinct in-scope file.
710
- in_index = len(book)
711
- out_of_scope = sum(1 for _k, p, _s in book if not in_scope(p))
712
- file_stamp = {}
713
- for _k, p, s in book:
714
- if in_scope(p):
715
- file_stamp.setdefault(p, s)
716
787
  stale, unknown_age, gone_files = 0, 0, 0
717
788
  for path, stamp in file_stamp.items():
718
789
  try:
@@ -740,16 +811,14 @@ def search_by_meaning(query: str, roots: List[str], k: int = 8,
740
811
  # measured count could exceed the claimed one.) The denominator for the freshness rate above is
741
812
  # file_stamp (asserted non-empty by `in_index`); this count is a raw count, no rate, so it needs only its
742
813
  # named membership constraint, which is `indexed_paths`.
743
- indexed_paths = {os.path.realpath(p) for _k, p, _s in book}
744
814
  scan_roots = root_abs or [os.path.realpath(r) for r in indexed_roots()]
745
815
  not_indexed = 0
746
816
  for _root in scan_roots:
747
817
  rp = pathlib.Path(_root)
748
818
  if not rp.is_dir():
749
819
  continue
750
- for p in rp.rglob("*"):
751
- if p.suffix.lower() in INDEXABLE_EXTS and p.is_file() and not _is_generated(p, rp) \
752
- and os.path.realpath(p) not in indexed_paths:
820
+ for p in _iter_semantic_census_files(rp):
821
+ if os.path.realpath(p) not in indexed_paths:
753
822
  not_indexed += 1
754
823
  search_by_meaning.last_scope = {"in_index": in_index, "out_of_scope": out_of_scope,
755
824
  "bad_dim": 0, "roots": root_abs,
@@ -995,9 +1064,10 @@ def _embed_live(con, fns, gone=frozenset()) -> int:
995
1064
  con.execute("SELECT key, qual, path, lineno, what, stamp, unit, parent_key, end_lineno, signature, "
996
1065
  "kind FROM fn WHERE what IS NOT NULL AND what != ''")}
997
1066
  live_keys = {f["key"] for f in fns} & set(book)
1067
+ store, core = _core_for(str(index_path()))
998
1068
  if not live_keys and not gone:
1069
+ store.prepare_ann_index(COLLECTION)
999
1070
  return 0
1000
- store, core = _core_for(str(index_path()))
1001
1071
  # EVERY live description goes to veccore; its content-sha re-embeds ONLY the drifted text (a re-described
1002
1072
  # function whose sentence CHANGED re-embeds; an unchanged one is skipped, $0). Filtering to 'vector
1003
1073
  # missing' here would WRONGLY skip a re-described function whose stored vector still holds its OLD
@@ -1024,6 +1094,7 @@ def _embed_live(con, fns, gone=frozenset()) -> int:
1024
1094
  "block_start": lineno if is_spanned else None,
1025
1095
  "block_end": end_lineno if is_spanned else None}})
1026
1096
  if not items:
1097
+ store.prepare_ann_index(COLLECTION)
1027
1098
  return 0
1028
1099
  vc = _veccore()
1029
1100
  embedded = 0
@@ -1033,4 +1104,8 @@ def _embed_live(con, fns, gone=frozenset()) -> int:
1033
1104
  result = core.embed_upsert(COLLECTION, items[start:start + EMBED_UPSERT_CHUNK],
1034
1105
  vc.AuditEnvelope(who="symgrep", why="index code by meaning"))
1035
1106
  embedded += result.n_embedded
1107
+ # Both CLI reindex and the detached freshness worker reach this seam. Publish
1108
+ # the durable index after vector reconciliation; readers keep their committed
1109
+ # snapshot while a new generation is trained and assigned in bounded chunks.
1110
+ store.prepare_ann_index(COLLECTION)
1036
1111
  return embedded
@@ -0,0 +1,108 @@
1
+ """Portable durable inverted-file cosine index with bounded training workspaces.
2
+
3
+ Only centroids, a bounded training reservoir and one assignment batch enter Python.
4
+ SQLite B-trees hold postings; queries rerank their original float32 vectors exactly.
5
+ """
6
+ import json
7
+ import numpy as np
8
+ from .semantic_sqlite_config import (IVF_BUCKET_COUNT, IVF_TRAINING_ROWS, IVF_TRAINING_ITERATIONS,
9
+ IVF_ASSIGNMENT_ROWS, IVF_RANDOM_SEED)
10
+
11
+
12
+ def initialize_ivf_schema(connection):
13
+ existing_covering = connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='symgrep_ivf_vectors'").fetchone()
14
+ connection.executescript("""
15
+ CREATE TABLE IF NOT EXISTS symgrep_ivf_centroids(collection TEXT, bucket INTEGER, vector BLOB,
16
+ PRIMARY KEY(collection,bucket));
17
+ CREATE TABLE IF NOT EXISTS symgrep_ivf_vectors(collection TEXT, bucket INTEGER, id TEXT, vector BLOB, payload TEXT,
18
+ PRIMARY KEY(collection,bucket,id)) WITHOUT ROWID;
19
+ CREATE UNIQUE INDEX IF NOT EXISTS symgrep_ivf_covering_members ON symgrep_ivf_vectors(collection,id);
20
+ CREATE TABLE IF NOT EXISTS symgrep_ivf_model_config(collection TEXT PRIMARY KEY, signature TEXT NOT NULL);
21
+ CREATE TABLE IF NOT EXISTS symgrep_ivf_state(collection TEXT PRIMARY KEY, dirty INTEGER NOT NULL);
22
+ CREATE TRIGGER IF NOT EXISTS symgrep_ivf_covering_insert AFTER INSERT ON points BEGIN
23
+ INSERT OR REPLACE INTO symgrep_ivf_state VALUES(new.collection,1);
24
+ DELETE FROM symgrep_ivf_vectors WHERE collection=new.collection AND id=new.id;
25
+ END;
26
+ CREATE TRIGGER IF NOT EXISTS symgrep_ivf_covering_update AFTER UPDATE OF vec,payload,id,collection ON points BEGIN
27
+ INSERT OR REPLACE INTO symgrep_ivf_state VALUES(old.collection,1);
28
+ INSERT OR REPLACE INTO symgrep_ivf_state VALUES(new.collection,1);
29
+ DELETE FROM symgrep_ivf_vectors WHERE collection=old.collection AND id=old.id;
30
+ DELETE FROM symgrep_ivf_vectors WHERE collection=new.collection AND id=new.id;
31
+ END;
32
+ CREATE TRIGGER IF NOT EXISTS symgrep_ivf_covering_delete AFTER DELETE ON points BEGIN
33
+ DELETE FROM symgrep_ivf_vectors WHERE collection=old.collection AND id=old.id;
34
+ END;
35
+ """)
36
+ with connection:
37
+ connection.execute("INSERT OR REPLACE INTO symgrep_ivf_state SELECT DISTINCT collection,1 "
38
+ "FROM symgrep_ivf_centroids WHERE collection NOT IN "
39
+ "(SELECT collection FROM symgrep_ivf_model_config)")
40
+ if not existing_covering:
41
+ with connection:
42
+ connection.execute("INSERT OR REPLACE INTO symgrep_ivf_state SELECT DISTINCT collection,1 FROM symgrep_ivf_centroids")
43
+
44
+
45
+ def ivf_training_configuration_signature():
46
+ """Persist all model-building choices; query probe changes do not retrain."""
47
+ return json.dumps((IVF_BUCKET_COUNT, IVF_TRAINING_ROWS, IVF_TRAINING_ITERATIONS,
48
+ IVF_ASSIGNMENT_ROWS, IVF_RANDOM_SEED))
49
+
50
+
51
+ def read_ivf_centroids(connection, collection):
52
+ rows = connection.execute('SELECT bucket,vector FROM symgrep_ivf_centroids WHERE collection=? ORDER BY bucket',
53
+ (collection,)).fetchall()
54
+ if any(bucket != ordinal for ordinal, (bucket, _vector) in enumerate(rows)):
55
+ raise ValueError("IVF centroid buckets must be contiguous")
56
+ return np.stack([np.frombuffer(row[1], dtype=np.float32) for row in rows]) if rows else None
57
+
58
+
59
+ def assign_ivf_rows(connection, collection, centroids, rows):
60
+ if not rows:
61
+ return
62
+ vectors = np.stack([np.frombuffer(blob, dtype=np.float32) for _, blob, _payload in rows])
63
+ buckets = np.argmax(vectors @ centroids.T, axis=1)
64
+ connection.executemany('INSERT OR REPLACE INTO symgrep_ivf_vectors VALUES(?,?,?,?,?)',
65
+ [(collection, int(bucket), row[0], row[1], row[2]) for row, bucket in zip(rows, buckets)])
66
+
67
+
68
+ def rebuild_ivf_collection(connection, collection):
69
+ """Publish a complete model and postings atomically; reservoir memory has a fixed ceiling."""
70
+ if min(IVF_BUCKET_COUNT, IVF_TRAINING_ROWS, IVF_TRAINING_ITERATIONS, IVF_ASSIGNMENT_ROWS) <= 0:
71
+ raise ValueError("IVF training configuration must be positive")
72
+ generator = np.random.default_rng(IVF_RANDOM_SEED)
73
+ reservoir = []
74
+ for ordinal, (blob,) in enumerate(connection.execute('SELECT vec FROM points WHERE collection=? ORDER BY id',
75
+ (collection,))):
76
+ if ordinal < IVF_TRAINING_ROWS:
77
+ reservoir.append(np.frombuffer(blob, dtype=np.float32).copy())
78
+ else:
79
+ selected = int(generator.integers(ordinal + 1))
80
+ if selected < IVF_TRAINING_ROWS:
81
+ reservoir[selected] = np.frombuffer(blob, dtype=np.float32).copy()
82
+ if not reservoir:
83
+ return False
84
+ training = np.stack(reservoir)
85
+ del reservoir # Release the sampling copies before clustering and index assignment.
86
+ centroids = training[generator.choice(len(training), min(IVF_BUCKET_COUNT, len(training)), replace=False)].copy()
87
+ for _iteration in range(IVF_TRAINING_ITERATIONS):
88
+ sums = np.zeros_like(centroids)
89
+ counts = np.zeros(len(centroids), dtype=np.int64)
90
+ for offset in range(0, len(training), IVF_ASSIGNMENT_ROWS):
91
+ batch = training[offset:offset + IVF_ASSIGNMENT_ROWS]
92
+ labels = np.argmax(batch @ centroids.T, axis=1)
93
+ np.add.at(sums, labels, batch)
94
+ np.add.at(counts, labels, 1)
95
+ occupied = counts > 0
96
+ centroids[occupied] = sums[occupied]
97
+ centroids /= np.maximum(np.linalg.norm(centroids, axis=1, keepdims=True), np.finfo(np.float32).tiny)
98
+ connection.execute('DELETE FROM symgrep_ivf_centroids WHERE collection=?', (collection,))
99
+ connection.executemany('INSERT INTO symgrep_ivf_centroids VALUES(?,?,?)',
100
+ [(collection, bucket, vector.tobytes()) for bucket, vector in enumerate(centroids)])
101
+ connection.execute('DELETE FROM symgrep_ivf_vectors WHERE collection=?', (collection,))
102
+ cursor = connection.execute('SELECT id,vec,payload FROM points WHERE collection=?', (collection,))
103
+ while rows := cursor.fetchmany(IVF_ASSIGNMENT_ROWS):
104
+ assign_ivf_rows(connection, collection, centroids, rows)
105
+ connection.execute('INSERT OR REPLACE INTO symgrep_ivf_model_config VALUES(?,?)',
106
+ (collection, ivf_training_configuration_signature()))
107
+ connection.execute('INSERT OR REPLACE INTO symgrep_ivf_state VALUES(?,0)', (collection,))
108
+ return True