symgrep-codesearch 0.9.1__tar.gz → 0.9.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. {symgrep_codesearch-0.9.1/src/symgrep_codesearch.egg-info → symgrep_codesearch-0.9.2}/PKG-INFO +1 -1
  2. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/pyproject.toml +1 -1
  3. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/capability_registry.py +49 -0
  4. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/semantic.py +30 -9
  5. symgrep_codesearch-0.9.2/src/symgrep/semantic_ivf.py +108 -0
  6. symgrep_codesearch-0.9.2/src/symgrep/semantic_sqlite.py +295 -0
  7. symgrep_codesearch-0.9.2/src/symgrep/semantic_sqlite_config.py +19 -0
  8. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2/src/symgrep_codesearch.egg-info}/PKG-INFO +1 -1
  9. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/SOURCES.txt +3 -0
  10. symgrep_codesearch-0.9.2/tests/test_semantic_census_pruning_pytest.py +42 -0
  11. symgrep_codesearch-0.9.2/tests/test_sqlite_ivf_retrieval_pytest.py +278 -0
  12. symgrep_codesearch-0.9.1/src/symgrep/semantic_sqlite.py +0 -156
  13. symgrep_codesearch-0.9.1/src/symgrep/semantic_sqlite_config.py +0 -7
  14. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/LICENSE +0 -0
  15. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/README.md +0 -0
  16. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/setup.cfg +0 -0
  17. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/__init__.py +0 -0
  18. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/absence_adjudication_writer.py +0 -0
  19. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/archives.py +0 -0
  20. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/blocks.py +0 -0
  21. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/capability_registry_data.jsonl +0 -0
  22. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/capability_surface_checks.py +0 -0
  23. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/classfile.py +0 -0
  24. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/cli.py +0 -0
  25. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/data_symbols.py +0 -0
  26. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/docsymbols.py +0 -0
  27. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/embedding_process.py +0 -0
  28. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/enclosing.py +0 -0
  29. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/glob_recall.py +0 -0
  30. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/mcp_server.py +0 -0
  31. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/mcp_structured.py +0 -0
  32. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/report.py +0 -0
  33. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/resolve.py +0 -0
  34. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/resolve_js.py +0 -0
  35. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/resolve_python.py +0 -0
  36. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/ripgrep.py +0 -0
  37. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/rss_ceiling.py +0 -0
  38. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/runtime_paths.py +0 -0
  39. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/structural.py +0 -0
  40. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep/structure.py +0 -0
  41. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/dependency_links.txt +0 -0
  42. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/entry_points.txt +0 -0
  43. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/requires.txt +0 -0
  44. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/src/symgrep_codesearch.egg-info/top_level.txt +0 -0
  45. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_absence_adjudication_authorship_pytest.py +0 -0
  46. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_absence_adjudication_binding_pytest.py +0 -0
  47. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_blocks_extraction.py +0 -0
  48. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_blocks_retrieval.py +0 -0
  49. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_bulk_describe.py +0 -0
  50. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_capability_documentation_guard_pytest.py +0 -0
  51. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_capability_surface_parity_guard_pytest.py +0 -0
  52. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_classfile.py +0 -0
  53. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_codex_hook_payload.py +0 -0
  54. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_data_symbols.py +0 -0
  55. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_describe_lane_confinement.py +0 -0
  56. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_doc_symbols.py +0 -0
  57. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_embedding_residency_pytest.py +0 -0
  58. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_file_structure.py +0 -0
  59. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_freshness.py +0 -0
  60. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_honest_recall.py +0 -0
  61. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_http_ensure.py +0 -0
  62. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_http_rss_shutdown_pytest.py +0 -0
  63. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_index_reconstruct.py +0 -0
  64. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_index_scope.py +0 -0
  65. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_index_worker.py +0 -0
  66. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_mcp_error_log.py +0 -0
  67. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_mcp_registry_behavior_pytest.py +0 -0
  68. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_mcp_structured.py +0 -0
  69. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_mcp_thread_safety.py +0 -0
  70. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_mcp_version.py +0 -0
  71. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_name_uniqueness.py +0 -0
  72. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_nudge_classify.py +0 -0
  73. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_nudge_runaway.py +0 -0
  74. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_report_filters.py +0 -0
  75. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_resolve.py +0 -0
  76. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_resolve_js.py +0 -0
  77. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_resolve_python.py +0 -0
  78. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_resolve_r2.py +0 -0
  79. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_review_regressions.py +0 -0
  80. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_rss_ceiling_pytest.py +0 -0
  81. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_semantic_freshness_signal.py +0 -0
  82. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_semantic_glob_lang.py +0 -0
  83. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_sites_and_symbols.py +0 -0
  84. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_sqlite_streaming_retrieval_pytest.py +0 -0
  85. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_structural.py +0 -0
  86. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_symbol_vocabulary.py +0 -0
  87. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_ts_fallback_loud.py +0 -0
  88. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_ts_symbol_coverage.py +0 -0
  89. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_usage.py +0 -0
  90. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_veccore_adoption.py +0 -0
  91. {symgrep_codesearch-0.9.1 → symgrep_codesearch-0.9.2}/tests/test_worker_lock_recovery_pytest.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: symgrep-codesearch
3
- Version: 0.9.1
3
+ Version: 0.9.2
4
4
  Summary: Code search with honest recall — structure-aware grep that tells you what it does not know, plus search by meaning
5
5
  License-Expression: MIT
6
6
  Project-URL: Repository, https://github.com/ashdamle/symgrep
@@ -41,7 +41,7 @@ name = "symgrep-codesearch"
41
41
  # semantic search. Minor bump (dependency surface changed; no breaking import/CLI/MCP change).
42
42
  # 0.5.0: added the public `file_structure` library API (+ its MCP tool and `symgrep structure` CLI), import
43
43
  # module-name normalization, and the FileStructure `connects` list (structure schema v2).
44
- version = "0.9.1"
44
+ version = "0.9.2"
45
45
  description = "Code search with honest recall — structure-aware grep that tells you what it does not know, plus search by meaning"
46
46
  readme = "README.md"
47
47
  # 3.11+ : the maintained grammar pack (tree-sitter-language-pack, a CORE dep below) ships abi3 wheels for
@@ -7,6 +7,7 @@ import inspect
7
7
  import hashlib
8
8
  from datetime import datetime, timezone
9
9
  import json
10
+ from enum import StrEnum
10
11
  from typing import Any, Optional, List
11
12
 
12
13
  CAPABILITY_DATA_RESOURCE = 'capability_registry_data.jsonl'
@@ -53,6 +54,35 @@ RUNTIME_ENVIRONMENT_SETTINGS = (
53
54
  'SYMGREP_SQLITE_BUSY_MS', 'symgrep.semantic_sqlite_config:SQLITE_READER_BUSY_MS',
54
55
  'SQLite semantic reader busy timeout in milliseconds while concurrent indexing '
55
56
  'holds a database lock. WAL readers otherwise use committed statement snapshots.'),
57
+ RuntimeEnvironmentSetting(
58
+ 'SYMGREP_IVF_ENABLED', 'symgrep.semantic_sqlite_config:IVF_ENABLED',
59
+ 'Persistent SQLite inverted-file vector retrieval. Zero disables ANN and logs one '
60
+ 'honest warning before exact streaming cosine fallback. Centroids and postings live '
61
+ 'in the committed database; approximate candidates are reranked on original vectors. '
62
+ 'Reindex prepares live postings. Reader pooling and atomic generation swaps stay bounded.'),
63
+ RuntimeEnvironmentSetting(
64
+ 'SYMGREP_IVF_BUCKETS', 'symgrep.semantic_sqlite_config:IVF_BUCKET_COUNT',
65
+ 'Centroid count for the inverted-file index. Reindex atomically rebuilds when '
66
+ 'training settings differ from the durable model signature. More buckets '
67
+ 'narrow candidate scans and require recall measurement.'),
68
+ RuntimeEnvironmentSetting(
69
+ 'SYMGREP_IVF_PROBES', 'symgrep.semantic_sqlite_config:IVF_PROBE_COUNT',
70
+ 'Initial nearest centroid buckets for symbol queries without a kind filter. Other unit/kind '
71
+ 'scopes use every bucket and log exact coverage to preserve recall. More probes improve '
72
+ 'approximate recall; sparse results expand probes. Measure complete recall@k against exact streaming search.'),
73
+ RuntimeEnvironmentSetting(
74
+ 'SYMGREP_IVF_TRAINING_ROWS', 'symgrep.semantic_sqlite_config:IVF_TRAINING_ROWS',
75
+ 'Maximum reservoir sample rows used to train centroids; bounds training memory '
76
+ 'independently of corpus size.'),
77
+ RuntimeEnvironmentSetting(
78
+ 'SYMGREP_IVF_TRAINING_ITERATIONS', 'symgrep.semantic_sqlite_config:IVF_TRAINING_ITERATIONS',
79
+ 'Spherical centroid training iterations for a new index.'),
80
+ RuntimeEnvironmentSetting(
81
+ 'SYMGREP_IVF_ASSIGNMENT_ROWS', 'symgrep.semantic_sqlite_config:IVF_ASSIGNMENT_ROWS',
82
+ 'Maximum vector rows per centroid assignment batch during training and reindex.'),
83
+ RuntimeEnvironmentSetting(
84
+ 'SYMGREP_IVF_RANDOM_SEED', 'symgrep.semantic_sqlite_config:IVF_RANDOM_SEED',
85
+ 'Reproducible reservoir sampling and centroid initialization seed for new indexes.'),
56
86
  )
57
87
 
58
88
  @dataclass(frozen=True)
@@ -92,6 +122,25 @@ CAPABILITIES = load_symgrep_capability_declarations()
92
122
  def capability_by_id(identity):
93
123
  return next(cap for cap in CAPABILITIES if cap.id == identity)
94
124
 
125
+
126
+ SEMANTIC_UNIT_ARGUMENT = next(parameter for parameter in capability_by_id('search').params
127
+ if parameter.name == 'unit')
128
+ SemanticSearchUnit = StrEnum('SemanticSearchUnit',
129
+ {choice.upper(): choice for choice in SEMANTIC_UNIT_ARGUMENT.choices})
130
+
131
+
132
+ def parse_semantic_search_unit(value):
133
+ """Parse the declared unit grammar; aliases outside the public enum are errors.
134
+
135
+ In particular, plural spellings are not inferred from their meaning. The CLI,
136
+ MCP schema and direct store callers share the registry's canonical members.
137
+ """
138
+ try:
139
+ return SemanticSearchUnit(value)
140
+ except (TypeError, ValueError) as error:
141
+ raise ValueError(f'Invalid semantic unit {value!r}; expected one of '
142
+ f'{SEMANTIC_UNIT_ARGUMENT.choices!r}') from error
143
+
95
144
  def capability_description(capability):
96
145
  """Render full canonical prose and every parameter's semantics."""
97
146
  return '\n\n'.join([capability.summary, capability.description] + [f'{p.name}: {p.help}' for p in capability.params])
@@ -534,6 +534,24 @@ def _is_generated(p: pathlib.Path, rp: pathlib.Path) -> bool:
534
534
  return False
535
535
 
536
536
 
537
+ def _iter_semantic_census_files(census_root):
538
+ """Yield exactly the existing indexable census, pruning only rejected subtrees."""
539
+ root_has_mkdocs = (census_root / "mkdocs.yml").exists()
540
+ for directory, directories, filenames in os.walk(census_root, followlinks=False):
541
+ directories[:] = [name for name in directories
542
+ if name not in SKIP_DIRS and not name.startswith(".")
543
+ and not (name == "site" and root_has_mkdocs)]
544
+ for filename in filenames:
545
+ # Match Path.suffix for dotfiles and trailing dots before allocating Paths.
546
+ suffix_separator = filename.rfind(".")
547
+ suffix = filename[suffix_separator:].lower() if 0 < suffix_separator < len(filename) - 1 else ""
548
+ if suffix not in INDEXABLE_EXTS:
549
+ continue
550
+ candidate = pathlib.Path(directory) / filename
551
+ if candidate.is_file() and not _is_generated(candidate, census_root):
552
+ yield candidate
553
+
554
+
537
555
  def _looks_minified(text: str) -> bool:
538
556
  return any(len(line) > MINIFIED_LINE_CHARS for line in text.splitlines())
539
557
 
@@ -672,6 +690,8 @@ def search_by_meaning(query: str, roots: List[str], k: int = 8,
672
690
  functions and blocks in one list. The returned tuple shape is identical for every unit; block span +
673
691
  provenance ride `search_by_meaning.last_hits` (aligned to the return) so the tuple contract never changes.
674
692
  """
693
+ from .capability_registry import parse_semantic_search_unit
694
+ unit = parse_semantic_search_unit(unit).value
675
695
  root_abs = [os.path.realpath(r) for r in (roots or [])]
676
696
 
677
697
  def in_scope(p: str) -> bool:
@@ -797,14 +817,9 @@ def search_by_meaning(query: str, roots: List[str], k: int = 8,
797
817
  rp = pathlib.Path(_root)
798
818
  if not rp.is_dir():
799
819
  continue
800
- # os.walk retains one directory at a time; pathlib.rglob retains a corpus-sized
801
- # set of yielded paths. Do not prune: freshness must inspect every indexable file.
802
- for directory, _directories, filenames in os.walk(rp, followlinks=False):
803
- for filename in filenames:
804
- p = pathlib.Path(directory) / filename
805
- if p.suffix.lower() in INDEXABLE_EXTS and p.is_file() and not _is_generated(p, rp) \
806
- and os.path.realpath(p) not in indexed_paths:
807
- not_indexed += 1
820
+ for p in _iter_semantic_census_files(rp):
821
+ if os.path.realpath(p) not in indexed_paths:
822
+ not_indexed += 1
808
823
  search_by_meaning.last_scope = {"in_index": in_index, "out_of_scope": out_of_scope,
809
824
  "bad_dim": 0, "roots": root_abs,
810
825
  "files_checked": len(file_stamp), "stale_files": stale,
@@ -1049,9 +1064,10 @@ def _embed_live(con, fns, gone=frozenset()) -> int:
1049
1064
  con.execute("SELECT key, qual, path, lineno, what, stamp, unit, parent_key, end_lineno, signature, "
1050
1065
  "kind FROM fn WHERE what IS NOT NULL AND what != ''")}
1051
1066
  live_keys = {f["key"] for f in fns} & set(book)
1067
+ store, core = _core_for(str(index_path()))
1052
1068
  if not live_keys and not gone:
1069
+ store.prepare_ann_index(COLLECTION)
1053
1070
  return 0
1054
- store, core = _core_for(str(index_path()))
1055
1071
  # EVERY live description goes to veccore; its content-sha re-embeds ONLY the drifted text (a re-described
1056
1072
  # function whose sentence CHANGED re-embeds; an unchanged one is skipped, $0). Filtering to 'vector
1057
1073
  # missing' here would WRONGLY skip a re-described function whose stored vector still holds its OLD
@@ -1078,6 +1094,7 @@ def _embed_live(con, fns, gone=frozenset()) -> int:
1078
1094
  "block_start": lineno if is_spanned else None,
1079
1095
  "block_end": end_lineno if is_spanned else None}})
1080
1096
  if not items:
1097
+ store.prepare_ann_index(COLLECTION)
1081
1098
  return 0
1082
1099
  vc = _veccore()
1083
1100
  embedded = 0
@@ -1087,4 +1104,8 @@ def _embed_live(con, fns, gone=frozenset()) -> int:
1087
1104
  result = core.embed_upsert(COLLECTION, items[start:start + EMBED_UPSERT_CHUNK],
1088
1105
  vc.AuditEnvelope(who="symgrep", why="index code by meaning"))
1089
1106
  embedded += result.n_embedded
1107
+ # Both CLI reindex and the detached freshness worker reach this seam. Publish
1108
+ # the durable index after vector reconciliation; readers keep their committed
1109
+ # snapshot while a new generation is trained and assigned in bounded chunks.
1110
+ store.prepare_ann_index(COLLECTION)
1090
1111
  return embedded
@@ -0,0 +1,108 @@
1
+ """Portable durable inverted-file cosine index with bounded training workspaces.
2
+
3
+ Only centroids, a bounded training reservoir and one assignment batch enter Python.
4
+ SQLite B-trees hold postings; queries rerank their original float32 vectors exactly.
5
+ """
6
+ import json
7
+ import numpy as np
8
+ from .semantic_sqlite_config import (IVF_BUCKET_COUNT, IVF_TRAINING_ROWS, IVF_TRAINING_ITERATIONS,
9
+ IVF_ASSIGNMENT_ROWS, IVF_RANDOM_SEED)
10
+
11
+
12
+ def initialize_ivf_schema(connection):
13
+ existing_covering = connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='symgrep_ivf_vectors'").fetchone()
14
+ connection.executescript("""
15
+ CREATE TABLE IF NOT EXISTS symgrep_ivf_centroids(collection TEXT, bucket INTEGER, vector BLOB,
16
+ PRIMARY KEY(collection,bucket));
17
+ CREATE TABLE IF NOT EXISTS symgrep_ivf_vectors(collection TEXT, bucket INTEGER, id TEXT, vector BLOB, payload TEXT,
18
+ PRIMARY KEY(collection,bucket,id)) WITHOUT ROWID;
19
+ CREATE UNIQUE INDEX IF NOT EXISTS symgrep_ivf_covering_members ON symgrep_ivf_vectors(collection,id);
20
+ CREATE TABLE IF NOT EXISTS symgrep_ivf_model_config(collection TEXT PRIMARY KEY, signature TEXT NOT NULL);
21
+ CREATE TABLE IF NOT EXISTS symgrep_ivf_state(collection TEXT PRIMARY KEY, dirty INTEGER NOT NULL);
22
+ CREATE TRIGGER IF NOT EXISTS symgrep_ivf_covering_insert AFTER INSERT ON points BEGIN
23
+ INSERT OR REPLACE INTO symgrep_ivf_state VALUES(new.collection,1);
24
+ DELETE FROM symgrep_ivf_vectors WHERE collection=new.collection AND id=new.id;
25
+ END;
26
+ CREATE TRIGGER IF NOT EXISTS symgrep_ivf_covering_update AFTER UPDATE OF vec,payload,id,collection ON points BEGIN
27
+ INSERT OR REPLACE INTO symgrep_ivf_state VALUES(old.collection,1);
28
+ INSERT OR REPLACE INTO symgrep_ivf_state VALUES(new.collection,1);
29
+ DELETE FROM symgrep_ivf_vectors WHERE collection=old.collection AND id=old.id;
30
+ DELETE FROM symgrep_ivf_vectors WHERE collection=new.collection AND id=new.id;
31
+ END;
32
+ CREATE TRIGGER IF NOT EXISTS symgrep_ivf_covering_delete AFTER DELETE ON points BEGIN
33
+ DELETE FROM symgrep_ivf_vectors WHERE collection=old.collection AND id=old.id;
34
+ END;
35
+ """)
36
+ with connection:
37
+ connection.execute("INSERT OR REPLACE INTO symgrep_ivf_state SELECT DISTINCT collection,1 "
38
+ "FROM symgrep_ivf_centroids WHERE collection NOT IN "
39
+ "(SELECT collection FROM symgrep_ivf_model_config)")
40
+ if not existing_covering:
41
+ with connection:
42
+ connection.execute("INSERT OR REPLACE INTO symgrep_ivf_state SELECT DISTINCT collection,1 FROM symgrep_ivf_centroids")
43
+
44
+
45
+ def ivf_training_configuration_signature():
46
+ """Persist all model-building choices; query probe changes do not retrain."""
47
+ return json.dumps((IVF_BUCKET_COUNT, IVF_TRAINING_ROWS, IVF_TRAINING_ITERATIONS,
48
+ IVF_ASSIGNMENT_ROWS, IVF_RANDOM_SEED))
49
+
50
+
51
+ def read_ivf_centroids(connection, collection):
52
+ rows = connection.execute('SELECT bucket,vector FROM symgrep_ivf_centroids WHERE collection=? ORDER BY bucket',
53
+ (collection,)).fetchall()
54
+ if any(bucket != ordinal for ordinal, (bucket, _vector) in enumerate(rows)):
55
+ raise ValueError("IVF centroid buckets must be contiguous")
56
+ return np.stack([np.frombuffer(row[1], dtype=np.float32) for row in rows]) if rows else None
57
+
58
+
59
+ def assign_ivf_rows(connection, collection, centroids, rows):
60
+ if not rows:
61
+ return
62
+ vectors = np.stack([np.frombuffer(blob, dtype=np.float32) for _, blob, _payload in rows])
63
+ buckets = np.argmax(vectors @ centroids.T, axis=1)
64
+ connection.executemany('INSERT OR REPLACE INTO symgrep_ivf_vectors VALUES(?,?,?,?,?)',
65
+ [(collection, int(bucket), row[0], row[1], row[2]) for row, bucket in zip(rows, buckets)])
66
+
67
+
68
+ def rebuild_ivf_collection(connection, collection):
69
+ """Publish a complete model and postings atomically; reservoir memory has a fixed ceiling."""
70
+ if min(IVF_BUCKET_COUNT, IVF_TRAINING_ROWS, IVF_TRAINING_ITERATIONS, IVF_ASSIGNMENT_ROWS) <= 0:
71
+ raise ValueError("IVF training configuration must be positive")
72
+ generator = np.random.default_rng(IVF_RANDOM_SEED)
73
+ reservoir = []
74
+ for ordinal, (blob,) in enumerate(connection.execute('SELECT vec FROM points WHERE collection=? ORDER BY id',
75
+ (collection,))):
76
+ if ordinal < IVF_TRAINING_ROWS:
77
+ reservoir.append(np.frombuffer(blob, dtype=np.float32).copy())
78
+ else:
79
+ selected = int(generator.integers(ordinal + 1))
80
+ if selected < IVF_TRAINING_ROWS:
81
+ reservoir[selected] = np.frombuffer(blob, dtype=np.float32).copy()
82
+ if not reservoir:
83
+ return False
84
+ training = np.stack(reservoir)
85
+ del reservoir # Release the sampling copies before clustering and index assignment.
86
+ centroids = training[generator.choice(len(training), min(IVF_BUCKET_COUNT, len(training)), replace=False)].copy()
87
+ for _iteration in range(IVF_TRAINING_ITERATIONS):
88
+ sums = np.zeros_like(centroids)
89
+ counts = np.zeros(len(centroids), dtype=np.int64)
90
+ for offset in range(0, len(training), IVF_ASSIGNMENT_ROWS):
91
+ batch = training[offset:offset + IVF_ASSIGNMENT_ROWS]
92
+ labels = np.argmax(batch @ centroids.T, axis=1)
93
+ np.add.at(sums, labels, batch)
94
+ np.add.at(counts, labels, 1)
95
+ occupied = counts > 0
96
+ centroids[occupied] = sums[occupied]
97
+ centroids /= np.maximum(np.linalg.norm(centroids, axis=1, keepdims=True), np.finfo(np.float32).tiny)
98
+ connection.execute('DELETE FROM symgrep_ivf_centroids WHERE collection=?', (collection,))
99
+ connection.executemany('INSERT INTO symgrep_ivf_centroids VALUES(?,?,?)',
100
+ [(collection, bucket, vector.tobytes()) for bucket, vector in enumerate(centroids)])
101
+ connection.execute('DELETE FROM symgrep_ivf_vectors WHERE collection=?', (collection,))
102
+ cursor = connection.execute('SELECT id,vec,payload FROM points WHERE collection=?', (collection,))
103
+ while rows := cursor.fetchmany(IVF_ASSIGNMENT_ROWS):
104
+ assign_ivf_rows(connection, collection, centroids, rows)
105
+ connection.execute('INSERT OR REPLACE INTO symgrep_ivf_model_config VALUES(?,?)',
106
+ (collection, ivf_training_configuration_signature()))
107
+ connection.execute('INSERT OR REPLACE INTO symgrep_ivf_state VALUES(?,0)', (collection,))
108
+ return True
@@ -0,0 +1,295 @@
1
+ """Bounded SQLite cosine retrieval; corpus vectors never become a Python matrix.
2
+
3
+ The persistent reader pool maps only a bounded prefix per connection. SQLite/OS
4
+ page caching serves the remaining scan; pool size bounds simultaneous mappings and
5
+ page caches. Each SELECT is a committed snapshot, including its liveness join.
6
+ """
7
+ import logging
8
+ import json
9
+ import os
10
+ import queue
11
+ import sqlite3
12
+ import threading
13
+ from contextlib import contextmanager
14
+
15
+ import numpy as np
16
+ from veccore.adapters.sqlite_numpy_store import SqliteNumpyStore
17
+ from veccore.ports import Hit
18
+ from veccore.errors import NotSupported
19
+ from .capability_registry import parse_semantic_search_unit
20
+
21
+ from .semantic_sqlite_config import (
22
+ SQLITE_READER_COUNT, SQLITE_READER_MMAP_BYTES, SQLITE_READER_CACHE_KIB,
23
+ SQLITE_READER_BUSY_MS, IVF_ENABLED, IVF_PROBE_COUNT, IVF_PROBE_GROWTH,
24
+ IVF_RECALL_VALIDATED_UNIT, IVF_BUCKET_COUNT, IVF_TRAINING_ROWS, IVF_TRAINING_ITERATIONS, IVF_ASSIGNMENT_ROWS, IVF_RANDOM_SEED,
25
+ )
26
+ COSINE_MINIMUM_NORM = np.finfo(np.float32).tiny
27
+
28
+
29
+ class SqliteStreamingStore(SqliteNumpyStore):
30
+ """Keep veccore's durable write protocol and replace its allocating read path."""
31
+
32
+ def __init__(self, path):
33
+ if min(IVF_PROBE_COUNT, IVF_BUCKET_COUNT, IVF_TRAINING_ROWS,
34
+ IVF_TRAINING_ITERATIONS, IVF_ASSIGNMENT_ROWS) <= 0 or IVF_PROBE_GROWTH <= 1 or IVF_RANDOM_SEED < 0:
35
+ raise ValueError("IVF configuration must have positive workspace/probe sizes, growth > 1 and a nonnegative seed")
36
+ super().__init__(path)
37
+ if SQLITE_READER_COUNT <= 0 or SQLITE_READER_MMAP_BYTES < 0 or SQLITE_READER_CACHE_KIB <= 0:
38
+ raise ValueError("SQLite reader configuration must be bounded and positive")
39
+ self._reader_slots = threading.BoundedSemaphore(SQLITE_READER_COUNT)
40
+ self._idle_readers = queue.LifoQueue()
41
+ self._reader_metrics_lock = threading.Lock()
42
+ self._reader_queries = 0
43
+ self._active_readers = 0
44
+ self._peak_readers = 0
45
+ self._reader_opens = 0
46
+ self._readers_retired = False
47
+ self._ivf_fallback_logged = False
48
+ self._ivf_unvalidated_scope_logged = False
49
+ self._ivf_schema_available = False
50
+ try:
51
+ from .semantic_ivf import initialize_ivf_schema
52
+ with self._con_lock:
53
+ initialize_ivf_schema(self._con)
54
+ self._ivf_schema_available = True
55
+ except sqlite3.Error as error:
56
+ self._log_ivf_fallback(str(error))
57
+
58
+ def _log_ivf_fallback(self, reason):
59
+ with self._reader_metrics_lock:
60
+ if not self._ivf_fallback_logged:
61
+ logging.getLogger(__name__).warning("semantic ANN unavailable (%s); using exact SQLite streaming scan", reason)
62
+ self._ivf_fallback_logged = True
63
+
64
+ def prepare_ann_index(self, collection):
65
+ try:
66
+ return self._prepare_ivf_generation(collection)
67
+ except (sqlite3.Error, OSError, ValueError) as error:
68
+ with self._con_lock:
69
+ self._con.rollback()
70
+ self._log_ivf_fallback(str(error))
71
+ return False
72
+
73
+ def _prepare_ivf_generation(self, collection):
74
+ """Reindex seam: build once, or reconcile every pending committed point."""
75
+ if not IVF_ENABLED or not self._ivf_schema_available:
76
+ self._log_ivf_fallback("disabled or index schema unavailable")
77
+ return False
78
+ from .semantic_ivf import (read_ivf_centroids, rebuild_ivf_collection, assign_ivf_rows,
79
+ ivf_training_configuration_signature)
80
+ from .semantic_sqlite_config import IVF_ASSIGNMENT_ROWS
81
+ with self._con_lock, self._con:
82
+ self._con.execute("BEGIN IMMEDIATE")
83
+ centroids = read_ivf_centroids(self._con, collection)
84
+ model_config = self._con.execute(
85
+ 'SELECT signature FROM symgrep_ivf_model_config WHERE collection=?', (collection,)).fetchone()
86
+ if centroids is None or not model_config or model_config[0] != ivf_training_configuration_signature():
87
+ return rebuild_ivf_collection(self._con, collection)
88
+ cursor = self._con.execute("SELECT p.id,p.vec,p.payload FROM points p LEFT JOIN symgrep_ivf_vectors a "
89
+ "ON a.collection=p.collection AND a.id=p.id WHERE p.collection=? AND a.id IS NULL", (collection,))
90
+ while rows := cursor.fetchmany(IVF_ASSIGNMENT_ROWS):
91
+ assign_ivf_rows(self._con, collection, centroids, rows)
92
+ self._con.execute('INSERT OR REPLACE INTO symgrep_ivf_state VALUES(?,0)', (collection,))
93
+ return True
94
+
95
+ def upsert(self, collection, points):
96
+ from .semantic_ivf import read_ivf_centroids, assign_ivf_rows
97
+ with self._con_lock:
98
+ rows = [(collection, point.id, point.text_sha, int(np.asarray(point.vector).size),
99
+ np.asarray(point.vector, dtype=np.float32).ravel().tobytes(),
100
+ json.dumps(point.payload, ensure_ascii=False)) for point in points]
101
+ with self._con:
102
+ # Acquire the write reservation before reading readiness; another
103
+ # writer cannot invalidate the generation between these steps.
104
+ self._con.execute("BEGIN IMMEDIATE")
105
+ state = self._con.execute('SELECT dirty FROM symgrep_ivf_state WHERE collection=?',
106
+ (collection,)).fetchone() if self._ivf_schema_available else None
107
+ self._con.executemany("INSERT OR REPLACE INTO points(collection,id,text_sha,dim,vec,payload) "
108
+ "VALUES(?,?,?,?,?,?)", rows)
109
+ if IVF_ENABLED and self._ivf_schema_available:
110
+ centroids = read_ivf_centroids(self._con, collection)
111
+ if centroids is not None:
112
+ assign_ivf_rows(self._con, collection, centroids, [(row[1], row[4], row[5]) for row in rows])
113
+ if state and not state[0]:
114
+ self._con.execute('UPDATE symgrep_ivf_state SET dirty=0 WHERE collection=?', (collection,))
115
+ self._matrix_cache.pop(collection, None)
116
+
117
+ def _sqlite_file_identity(self):
118
+ stat = os.stat(self.path)
119
+ return stat.st_dev, stat.st_ino
120
+
121
+ @contextmanager
122
+ def _borrow_sqlite_reader(self):
123
+ with self._reader_slots:
124
+ identity = self._sqlite_file_identity()
125
+ try:
126
+ reader, reader_identity = self._idle_readers.get_nowait()
127
+ except queue.Empty:
128
+ reader, reader_identity = None, None
129
+ if reader is not None and reader_identity != identity:
130
+ reader.close()
131
+ reader = None
132
+ if reader is None:
133
+ # mode=ro rejects an absent/replaced path rather than creating an empty DB.
134
+ from pathlib import Path
135
+ reader = sqlite3.connect(Path(self.path).resolve().as_uri() + "?mode=ro", uri=True,
136
+ check_same_thread=False)
137
+ reader.execute(f"PRAGMA busy_timeout={SQLITE_READER_BUSY_MS}")
138
+ reader.execute(f"PRAGMA mmap_size={SQLITE_READER_MMAP_BYTES}")
139
+ reader.execute(f"PRAGMA cache_size={-SQLITE_READER_CACHE_KIB}")
140
+ reader.execute("PRAGMA temp_store=FILE")
141
+ reader.execute("PRAGMA query_only=ON")
142
+ with self._reader_metrics_lock:
143
+ self._reader_opens += 1
144
+ with self._reader_metrics_lock:
145
+ self._active_readers += 1
146
+ self._peak_readers = max(self._peak_readers, self._active_readers)
147
+ try:
148
+ yield reader
149
+ finally:
150
+ if reader.in_transaction:
151
+ reader.rollback()
152
+ # Remove closures retaining query vectors/scoping sets between requests.
153
+ reader.create_function("symgrep_cosine", 1, None)
154
+ reader.create_function("symgrep_scope", 2, None)
155
+ with self._reader_metrics_lock:
156
+ self._active_readers -= 1
157
+ self._reader_queries += 1
158
+ try:
159
+ reusable = self._sqlite_file_identity() == identity
160
+ except FileNotFoundError:
161
+ reusable = False
162
+ with self._reader_metrics_lock:
163
+ if reusable and not self._readers_retired:
164
+ self._idle_readers.put((reader, identity))
165
+ else:
166
+ reader.close()
167
+
168
+ def retire_readers(self):
169
+ """Release idle old-inode mappings; in-flight snapshots close when returned."""
170
+ with self._reader_metrics_lock:
171
+ self._readers_retired = True
172
+ while True:
173
+ try:
174
+ reader, _identity = self._idle_readers.get_nowait()
175
+ except queue.Empty:
176
+ break
177
+ reader.close()
178
+
179
+ def close(self):
180
+ self.retire_readers()
181
+ super().close()
182
+
183
+ def has_points(self, collection):
184
+ """Probe existence without allocating every stored identifier."""
185
+ with self._borrow_sqlite_reader() as reader:
186
+ return reader.execute("SELECT 1 FROM points WHERE collection=? LIMIT 1",
187
+ (collection,)).fetchone() is not None
188
+
189
+ def read_metrics(self):
190
+ """A bounded-reader diagnostic; matrix loading is absent from this implementation."""
191
+ with self._reader_metrics_lock:
192
+ return dict(queries=self._reader_queries, active_readers=self._active_readers,
193
+ peak_readers=self._peak_readers, reader_opens=self._reader_opens, matrix_loads=0)
194
+
195
+ def search_exact(self, collection, qvec, k, filters=None):
196
+ return self._search_sqlite_vectors(collection, qvec, k, filters, use_ann=False)
197
+
198
+ def search(self, collection, qvec, k, filters=None):
199
+ return self._search_sqlite_vectors(collection, qvec, k, filters, use_ann=True)
200
+
201
+ def _search_sqlite_vectors(self, collection, qvec, k, filters=None, use_ann=True):
202
+ # Validate before SQL or ANN routing: this is the public enum grammar,
203
+ # not an inference about the meaning of arbitrary free text.
204
+ validated_unit = (parse_semantic_search_unit(filters["symgrep_unit"])
205
+ if filters and "symgrep_unit" in filters else None)
206
+ query_vector = np.asarray(qvec, dtype=np.float32).ravel()
207
+ query_vector = query_vector / max(float(np.linalg.norm(query_vector)), COSINE_MINIMUM_NORM)
208
+ scope = (filters or {}).get("symgrep_scope")
209
+ unsupported = set(filters or {}) - {"symgrep_scope", "symgrep_unit", "symgrep_kind"}
210
+ if unsupported:
211
+ raise NotSupported("unsupported semantic filters", self.backend, str(sorted(unsupported)))
212
+
213
+ def sqlite_vector_dot(blob):
214
+ vector = np.frombuffer(blob, dtype=np.float32)
215
+ if vector.shape != query_vector.shape:
216
+ raise ValueError("stored/query embedding dimensions differ")
217
+ return float(np.dot(vector, query_vector))
218
+
219
+ with self._borrow_sqlite_reader() as reader:
220
+ reader.create_function("symgrep_cosine", 1, sqlite_vector_dot)
221
+ reader.create_function("symgrep_scope", 2, scope or (lambda _id, _payload: True))
222
+ clauses = ["p.collection=?", "symgrep_scope(p.id,p.payload)"]
223
+ parameters = [collection]
224
+ join = ""
225
+ if filters and "symgrep_unit" in filters:
226
+ # One statement sees points AND bookkeeping in one committed snapshot.
227
+ join = " JOIN fn f ON f.key=p.id"
228
+ clauses.append("f.what IS NOT NULL AND f.what != ''")
229
+ unit = validated_unit.value
230
+ if unit in ("block", "data"):
231
+ clauses.append("f.unit=?")
232
+ parameters.append(unit)
233
+ elif unit not in ("both", "all"):
234
+ clauses.append("(f.unit='symbol' OR f.unit IS NULL)")
235
+ kind = filters.get("symgrep_kind")
236
+ if kind:
237
+ clauses.append("f.kind=?")
238
+ parameters.append(kind)
239
+ centroid_order = None
240
+ if IVF_PROBE_COUNT <= 0:
241
+ raise ValueError("IVF probe count must be positive")
242
+ if use_ann and IVF_ENABLED and self._ivf_schema_available:
243
+ from .semantic_ivf import read_ivf_centroids
244
+ # Explicit transaction binds index validity, postings, points and fn
245
+ # to the same committed generation, even across concurrent writes.
246
+ reader.execute("BEGIN")
247
+ try:
248
+ state = reader.execute("SELECT dirty FROM symgrep_ivf_state WHERE collection=?", (collection,)).fetchone()
249
+ centroids = read_ivf_centroids(reader, collection) if state and not state[0] else None
250
+ if centroids is not None:
251
+ centroid_order = np.argsort(-(centroids @ query_vector))
252
+ except (sqlite3.Error, ValueError) as error:
253
+ self._log_ivf_fallback(str(error))
254
+ if use_ann and centroid_order is None:
255
+ self._log_ivf_fallback("disabled, missing or pending index generation")
256
+ probe_count = min(IVF_PROBE_COUNT, len(centroid_order)) if centroid_order is not None else 0
257
+ if centroid_order is not None and (
258
+ validated_unit is not IVF_RECALL_VALIDATED_UNIT
259
+ or (filters or {}).get("symgrep_kind")):
260
+ # Only the measured symbol category enables approximate pruning.
261
+ # Other structured filter categories retain exact full coverage.
262
+ probe_count = len(centroid_order)
263
+ with self._reader_metrics_lock:
264
+ if not self._ivf_unvalidated_scope_logged:
265
+ logging.getLogger(__name__).warning(
266
+ "semantic ANN uses exact full-bucket coverage for unvalidated unit/kind filters")
267
+ self._ivf_unvalidated_scope_logged = True
268
+ while True:
269
+ ann_join = ""
270
+ from_clause = "points p"
271
+ ann_clause = ""
272
+ ann_parameters = []
273
+ if centroid_order is not None:
274
+ buckets = [int(bucket) for bucket in centroid_order[:probe_count]]
275
+ from_clause = ("(SELECT collection,id,vector AS vec,payload FROM symgrep_ivf_vectors "
276
+ "WHERE collection=? AND bucket IN (" + ",".join("?" for _ in buckets) + ")) p")
277
+ ann_clause = ""
278
+ ann_parameters = [collection, *buckets]
279
+ try:
280
+ rows = reader.execute(
281
+ "SELECT p.id, symgrep_cosine(p.vec) AS score, p.payload FROM " + from_clause + ann_join + join +
282
+ " WHERE " + " AND ".join(clauses) + ann_clause + " ORDER BY score DESC, p.id LIMIT ?",
283
+ (*ann_parameters, *parameters, max(0, k))).fetchall()
284
+ except sqlite3.Error as error:
285
+ if centroid_order is None:
286
+ raise
287
+ self._log_ivf_fallback(str(error))
288
+ centroid_order = None
289
+ continue
290
+ if centroid_order is None or len(rows) >= k or probe_count == len(centroid_order):
291
+ break
292
+ probe_count = min(len(centroid_order), probe_count * IVF_PROBE_GROWTH)
293
+ if reader.in_transaction:
294
+ reader.rollback()
295
+ return [Hit(id=key, score=score, payload=json.loads(payload)) for key, score, payload in rows]
@@ -0,0 +1,19 @@
1
+ """Configuration shared by SQLite retrieval and capability docs."""
2
+ import os
3
+ from .capability_registry import parse_semantic_search_unit
4
+
5
+ SQLITE_READER_COUNT = int(os.environ.get("SYMGREP_SQLITE_READERS", "4"))
6
+ SQLITE_READER_MMAP_BYTES = int(os.environ.get("SYMGREP_SQLITE_MMAP_BYTES", str(32 * 1024 * 1024)))
7
+ SQLITE_READER_CACHE_KIB = int(os.environ.get("SYMGREP_SQLITE_CACHE_KIB", "512"))
8
+ SQLITE_READER_BUSY_MS = int(os.environ.get("SYMGREP_SQLITE_BUSY_MS", "10000"))
9
+
10
+ IVF_ENABLED = os.environ.get("SYMGREP_IVF_ENABLED", "1") != "0"
11
+ IVF_BUCKET_COUNT = int(os.environ.get("SYMGREP_IVF_BUCKETS", "256"))
12
+ IVF_PROBE_COUNT = int(os.environ.get("SYMGREP_IVF_PROBES", "128"))
13
+ IVF_TRAINING_ROWS = int(os.environ.get("SYMGREP_IVF_TRAINING_ROWS", "8192"))
14
+ IVF_TRAINING_ITERATIONS = int(os.environ.get("SYMGREP_IVF_TRAINING_ITERATIONS", "12"))
15
+ IVF_ASSIGNMENT_ROWS = int(os.environ.get("SYMGREP_IVF_ASSIGNMENT_ROWS", "256"))
16
+ IVF_RANDOM_SEED = int(os.environ.get("SYMGREP_IVF_RANDOM_SEED", "0"))
17
+ IVF_PROBE_GROWTH = 2
18
+
19
+ IVF_RECALL_VALIDATED_UNIT = parse_semantic_search_unit("symbol")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: symgrep-codesearch
3
- Version: 0.9.1
3
+ Version: 0.9.2
4
4
  Summary: Code search with honest recall — structure-aware grep that tells you what it does not know, plus search by meaning
5
5
  License-Expression: MIT
6
6
  Project-URL: Repository, https://github.com/ashdamle/symgrep
@@ -25,6 +25,7 @@ src/symgrep/ripgrep.py
25
25
  src/symgrep/rss_ceiling.py
26
26
  src/symgrep/runtime_paths.py
27
27
  src/symgrep/semantic.py
28
+ src/symgrep/semantic_ivf.py
28
29
  src/symgrep/semantic_sqlite.py
29
30
  src/symgrep/semantic_sqlite_config.py
30
31
  src/symgrep/structural.py
@@ -71,9 +72,11 @@ tests/test_resolve_python.py
71
72
  tests/test_resolve_r2.py
72
73
  tests/test_review_regressions.py
73
74
  tests/test_rss_ceiling_pytest.py
75
+ tests/test_semantic_census_pruning_pytest.py
74
76
  tests/test_semantic_freshness_signal.py
75
77
  tests/test_semantic_glob_lang.py
76
78
  tests/test_sites_and_symbols.py
79
+ tests/test_sqlite_ivf_retrieval_pytest.py
77
80
  tests/test_sqlite_streaming_retrieval_pytest.py
78
81
  tests/test_structural.py
79
82
  tests/test_symbol_vocabulary.py