vectorwave 0.2.9__tar.gz → 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {vectorwave-0.2.9 → vectorwave-1.0.0}/PKG-INFO +25 -3
  2. {vectorwave-0.2.9 → vectorwave-1.0.0}/Readme.md +4 -1
  3. {vectorwave-0.2.9 → vectorwave-1.0.0}/pyproject.toml +35 -4
  4. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/__init__.py +8 -2
  5. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/batch/batch.py +92 -11
  6. vectorwave-1.0.0/src/vectorwave/check/__init__.py +17 -0
  7. vectorwave-1.0.0/src/vectorwave/check/calibrate.py +358 -0
  8. vectorwave-1.0.0/src/vectorwave/check/cli.py +74 -0
  9. vectorwave-1.0.0/src/vectorwave/check/config.py +75 -0
  10. vectorwave-1.0.0/src/vectorwave/check/plugin.py +175 -0
  11. vectorwave-1.0.0/src/vectorwave/check/report.py +48 -0
  12. vectorwave-1.0.0/src/vectorwave/cli/__init__.py +316 -0
  13. vectorwave-1.0.0/src/vectorwave/cli/__main__.py +6 -0
  14. vectorwave-1.0.0/src/vectorwave/cli/dev/compose.yml +33 -0
  15. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/core/auto_injector.py +12 -1
  16. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/core/decorator.py +55 -36
  17. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/core/generator.py +17 -3
  18. vectorwave-1.0.0/src/vectorwave/core/initializer.py +37 -0
  19. vectorwave-1.0.0/src/vectorwave/core/llm/factory.py +10 -0
  20. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/database/archiver.py +35 -34
  21. vectorwave-1.0.0/src/vectorwave/database/dataset.py +154 -0
  22. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/database/db.py +134 -101
  23. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/database/db_search.py +130 -149
  24. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/models/db_config.py +3 -0
  25. vectorwave-1.0.0/src/vectorwave/monitoring/otel.py +167 -0
  26. vectorwave-1.0.0/src/vectorwave/monitoring/tracer.py +524 -0
  27. vectorwave-1.0.0/src/vectorwave/runtime.py +210 -0
  28. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/search/execution_search.py +2 -20
  29. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/search/rag_search.py +2 -2
  30. vectorwave-1.0.0/src/vectorwave/store/__init__.py +19 -0
  31. vectorwave-1.0.0/src/vectorwave/store/base.py +144 -0
  32. vectorwave-1.0.0/src/vectorwave/store/factory.py +43 -0
  33. vectorwave-1.0.0/src/vectorwave/store/lance_store.py +364 -0
  34. vectorwave-1.0.0/src/vectorwave/store/weaviate_store.py +306 -0
  35. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/utils/function_cache.py +16 -8
  36. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/utils/healer.py +19 -3
  37. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/utils/replayer.py +124 -86
  38. vectorwave-1.0.0/src/vectorwave/utils/replayer_semantic.py +130 -0
  39. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/utils/return_caching_utils.py +38 -35
  40. vectorwave-1.0.0/src/vectorwave/utils/serialization.py +17 -0
  41. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/utils/status.py +0 -4
  42. vectorwave-1.0.0/src/vectorwave/vectorizer/__init__.py +0 -0
  43. vectorwave-0.2.9/src/vectorwave/core/llm/factory.py +0 -13
  44. vectorwave-0.2.9/src/vectorwave/database/dataset.py +0 -150
  45. vectorwave-0.2.9/src/vectorwave/monitoring/tracer.py +0 -460
  46. vectorwave-0.2.9/src/vectorwave/utils/replayer_semantic.py +0 -234
  47. {vectorwave-0.2.9 → vectorwave-1.0.0}/crates/Cargo.lock +0 -0
  48. {vectorwave-0.2.9 → vectorwave-1.0.0}/crates/Cargo.toml +0 -0
  49. {vectorwave-0.2.9 → vectorwave-1.0.0}/crates/src/lib.rs +0 -0
  50. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/batch/__init__.py +0 -0
  51. {vectorwave-0.2.9/src/vectorwave/core → vectorwave-1.0.0/src/vectorwave/cli/dev}/__init__.py +0 -0
  52. {vectorwave-0.2.9/src/vectorwave/core/llm → vectorwave-1.0.0/src/vectorwave/core}/__init__.py +0 -0
  53. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/core/core.py +0 -0
  54. {vectorwave-0.2.9/src/vectorwave/database → vectorwave-1.0.0/src/vectorwave/core/llm}/__init__.py +0 -0
  55. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/core/llm/base.py +0 -0
  56. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/core/llm/openai_client.py +0 -0
  57. {vectorwave-0.2.9/src/vectorwave/exception → vectorwave-1.0.0/src/vectorwave/database}/__init__.py +0 -0
  58. {vectorwave-0.2.9/src/vectorwave/models → vectorwave-1.0.0/src/vectorwave/exception}/__init__.py +0 -0
  59. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/exception/exceptions.py +0 -0
  60. {vectorwave-0.2.9/src/vectorwave/monitoring → vectorwave-1.0.0/src/vectorwave/models}/__init__.py +0 -0
  61. {vectorwave-0.2.9/src/vectorwave/monitoring/alert → vectorwave-1.0.0/src/vectorwave/monitoring}/__init__.py +0 -0
  62. {vectorwave-0.2.9/src/vectorwave/prediction → vectorwave-1.0.0/src/vectorwave/monitoring/alert}/__init__.py +0 -0
  63. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/monitoring/alert/base.py +0 -0
  64. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/monitoring/alert/factory.py +0 -0
  65. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/monitoring/alert/null_alerter.py +0 -0
  66. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/monitoring/alert/webhook_alerter.py +0 -0
  67. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/monitoring/monitoring.py +0 -0
  68. {vectorwave-0.2.9/src/vectorwave/search → vectorwave-1.0.0/src/vectorwave/prediction}/__init__.py +0 -0
  69. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/prediction/predictor.py +0 -0
  70. {vectorwave-0.2.9/src/vectorwave/utils → vectorwave-1.0.0/src/vectorwave/search}/__init__.py +0 -0
  71. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/search/extended_search.py +0 -0
  72. {vectorwave-0.2.9/src/vectorwave/vectorizer → vectorwave-1.0.0/src/vectorwave/utils}/__init__.py +0 -0
  73. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/utils/context.py +0 -0
  74. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/utils/github_pr.py +0 -0
  75. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/utils/path_utils.py +0 -0
  76. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/utils/scheduler.py +0 -0
  77. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/vectorizer/base.py +0 -0
  78. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/vectorizer/factory.py +0 -0
  79. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/vectorizer/huggingface_vectorizer.py +0 -0
  80. {vectorwave-0.2.9 → vectorwave-1.0.0}/src/vectorwave/vectorizer/openai_vectorizer.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: vectorwave
3
- Version: 0.2.9
3
+ Version: 1.0.0
4
4
  Classifier: Programming Language :: Python :: 3
5
5
  Classifier: Programming Language :: Python :: 3.10
6
6
  Classifier: Programming Language :: Python :: 3.11
@@ -8,7 +8,7 @@ Classifier: Programming Language :: Python :: 3.12
8
8
  Classifier: Programming Language :: Python :: 3.13
9
9
  Classifier: Programming Language :: Rust
10
10
  Classifier: Operating System :: OS Independent
11
- Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Development Status :: 5 - Production/Stable
12
12
  Classifier: Intended Audience :: Developers
13
13
  Requires-Dist: weaviate-client>=4.0.0
14
14
  Requires-Dist: pydantic-settings>=2.0.0
@@ -17,6 +17,25 @@ Requires-Dist: requests
17
17
  Requires-Dist: openai
18
18
  Requires-Dist: pygithub
19
19
  Requires-Dist: schedule
20
+ Requires-Dist: tomli>=2.0 ; python_full_version < '3.11'
21
+ Requires-Dist: pytest ; extra == 'dev'
22
+ Requires-Dist: pytest-asyncio ; extra == 'dev'
23
+ Requires-Dist: pytest-benchmark ; extra == 'dev'
24
+ Requires-Dist: pytest-recording ; extra == 'dev'
25
+ Requires-Dist: flake8 ; extra == 'dev'
26
+ Requires-Dist: lancedb>=0.30.0 ; extra == 'dev'
27
+ Requires-Dist: opentelemetry-api>=1.20.0 ; extra == 'dev'
28
+ Requires-Dist: opentelemetry-sdk>=1.20.0 ; extra == 'dev'
29
+ Requires-Dist: opentelemetry-exporter-otlp>=1.20.0 ; extra == 'dev'
30
+ Requires-Dist: testcontainers>=4.0.0 ; extra == 'dev'
31
+ Requires-Dist: vcrpy>=6.0.0 ; extra == 'dev'
32
+ Requires-Dist: lancedb>=0.30.0 ; extra == 'lite'
33
+ Requires-Dist: opentelemetry-api>=1.20.0 ; extra == 'otel'
34
+ Requires-Dist: opentelemetry-sdk>=1.20.0 ; extra == 'otel'
35
+ Requires-Dist: opentelemetry-exporter-otlp>=1.20.0 ; extra == 'otel'
36
+ Provides-Extra: dev
37
+ Provides-Extra: lite
38
+ Provides-Extra: otel
20
39
  License-File: LICENSE
21
40
  License-File: NOTICE
22
41
  Summary: VectorWave: Seamless Auto-Vectorization Framework
@@ -26,7 +45,10 @@ Requires-Python: >=3.10
26
45
  Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
27
46
  Project-URL: Repository, https://github.com/cozymori/vectorwave
28
47
 
29
- Need more information? Visit [here](https://cozymori.github.io/vectorwave-docs/)
48
+
49
+ ![VectorWave Logo](./docs_kr/images/vectorwave.png)
50
+
51
+ Need more information? Visit [here](https://www.cozymori.net/vectorwave)
30
52
 
31
53
  # VectorWave
32
54
  ![Star](https://badgen.net/github/stars/cozymori/vectorwave)
@@ -1,4 +1,7 @@
1
- Need more information? Visit [here](https://cozymori.github.io/vectorwave-docs/)
1
+
2
+ ![VectorWave Logo](./docs_kr/images/vectorwave.png)
3
+
4
+ Need more information? Visit [here](https://www.cozymori.net/vectorwave)
2
5
 
3
6
  # VectorWave
4
7
  ![Star](https://badgen.net/github/stars/cozymori/vectorwave)
@@ -5,7 +5,7 @@ build-backend = "maturin"
5
5
 
6
6
  [project]
7
7
  name = "vectorwave"
8
- version = "0.2.09"
8
+ version = "1.0.0"
9
9
  authors = [
10
10
  { name = "junyeonggim", email = "junyeonggim5@gmail.com" },
11
11
  ]
@@ -21,7 +21,7 @@ classifiers = [
21
21
  "Programming Language :: Python :: 3.13",
22
22
  "Programming Language :: Rust",
23
23
  "Operating System :: OS Independent",
24
- "Development Status :: 3 - Alpha",
24
+ "Development Status :: 5 - Production/Stable",
25
25
  "Intended Audience :: Developers",
26
26
  ]
27
27
 
@@ -32,9 +32,39 @@ dependencies = [
32
32
  "requests",
33
33
  "openai",
34
34
  "PyGithub",
35
- "schedule"
35
+ "schedule",
36
+ "tomli>=2.0; python_version < '3.11'",
36
37
  ]
37
38
 
39
+ [project.optional-dependencies]
40
+ dev = [
41
+ "pytest",
42
+ "pytest-asyncio",
43
+ "pytest-benchmark",
44
+ "pytest-recording",
45
+ "flake8",
46
+ "lancedb>=0.30.0",
47
+ "opentelemetry-api>=1.20.0",
48
+ "opentelemetry-sdk>=1.20.0",
49
+ "opentelemetry-exporter-otlp>=1.20.0",
50
+ "testcontainers>=4.0.0",
51
+ "vcrpy>=6.0.0",
52
+ ]
53
+ lite = [
54
+ "lancedb>=0.30.0",
55
+ ]
56
+ otel = [
57
+ "opentelemetry-api>=1.20.0",
58
+ "opentelemetry-sdk>=1.20.0",
59
+ "opentelemetry-exporter-otlp>=1.20.0",
60
+ ]
61
+
62
+ [project.scripts]
63
+ vectorwave = "vectorwave.cli:main"
64
+
65
+ [project.entry-points.pytest11]
66
+ vectorwave_check = "vectorwave.check.plugin"
67
+
38
68
  [project.urls]
39
69
  Repository = "https://github.com/cozymori/vectorwave"
40
70
 
@@ -42,4 +72,5 @@ Repository = "https://github.com/cozymori/vectorwave"
42
72
  python-source = "src"
43
73
  module-name = "vectorwave.vectorwave_core"
44
74
  manifest-path = "crates/Cargo.toml"
45
- features = ["pyo3/extension-module"]
75
+ features = ["pyo3/extension-module"]
76
+ include = ["src/vectorwave/cli/dev/compose.yml"]
@@ -1,5 +1,10 @@
1
+ from . import runtime as _runtime
2
+ # Light up the visible indicator (stderr banner + PID file) on first import.
3
+ # Silence with VECTORWAVE_QUIET=1.
4
+ _runtime.activate()
5
+
1
6
  from .core.decorator import vectorize
2
- from .database.db import initialize_database
7
+ from .database.db import initialize_database, update_database_schema
3
8
  from .database.db_search import search_functions, search_executions, search_errors_by_message, search_functions_hybrid
4
9
  from .monitoring.tracer import trace_span
5
10
  from .search.rag_search import search_and_answer, analyze_trace_log
@@ -25,5 +30,6 @@ __all__ = [
25
30
  'VectorWaveReplayer',
26
31
  'SemanticReplayer',
27
32
  'VectorWaveDatasetManager',
28
- 'VectorWaveAutoInjector'
33
+ 'VectorWaveAutoInjector',
34
+ 'update_database_schema'
29
35
  ]
@@ -1,3 +1,4 @@
1
+ import os
1
2
  import weaviate
2
3
  import atexit
3
4
  import logging
@@ -27,11 +28,24 @@ class WeaviateBatchManager:
27
28
  Uses High-Performance Rust Core if available, otherwise falls back to Python.
28
29
  """
29
30
 
30
- def __init__(self):
31
+ def __init__(self, host: Optional[str] = None, port: Optional[int] = None,
32
+ grpc_port: Optional[int] = None, api_key: Optional[str] = None):
31
33
  self._initialized = False
34
+ self._shutdown_done = False
32
35
  self.settings: WeaviateSettings = get_weaviate_settings()
33
36
  self.client: Optional[weaviate.WeaviateClient] = None
34
37
 
38
+ # Lite mode (LanceDB local file store) skips the Weaviate client entirely.
39
+ # When VECTORWAVE_MODE=lite, _flush_batch_core delegates to the
40
+ # configured VectorStore so no Docker / Weaviate connection is needed.
41
+ self._lite_mode = os.environ.get("VECTORWAVE_MODE", "pro").lower() == "lite"
42
+
43
+ # Store dynamic connection params
44
+ self._host = host
45
+ self._port = port
46
+ self._grpc_port = grpc_port
47
+ self._api_key = api_key
48
+
35
49
  # Batch Configuration
36
50
  self.batch_threshold = self.settings.BATCH_THRESHOLD
37
51
  self.flush_interval = self.settings.FLUSH_INTERVAL_SECONDS
@@ -58,9 +72,27 @@ class WeaviateBatchManager:
58
72
  atexit.register(self.shutdown)
59
73
 
60
74
  def _connect_client(self):
61
- """Attempts to connect to Weaviate."""
75
+ """Attempts to connect to the configured backend (Weaviate or Lite store)."""
76
+ if self._lite_mode:
77
+ try:
78
+ from ..store import get_vector_store
79
+ store = get_vector_store()
80
+ if store.is_ready():
81
+ self._initialized = True
82
+ self.client = None # not used in Lite mode
83
+ except Exception as e:
84
+ logger.warning(f"Lite store init failed: {e}")
85
+ self._initialized = False
86
+ return
87
+
62
88
  try:
63
- self.client = get_weaviate_client(self.settings)
89
+ if self._host is not None:
90
+ self.client = get_weaviate_client(
91
+ host=self._host, port=self._port,
92
+ grpc_port=self._grpc_port, api_key=self._api_key
93
+ )
94
+ else:
95
+ self.client = get_weaviate_client(self.settings)
64
96
  if self.client:
65
97
  self._initialized = True
66
98
  except Exception as e:
@@ -95,17 +127,27 @@ class WeaviateBatchManager:
95
127
  def _flush_batch_core(self, items: List[Dict[str, Any]]):
96
128
  """
97
129
  The actual flush logic called by either Rust or Python worker.
130
+
131
+ In Pro mode (default) this uses Weaviate's bulk batch.dynamic() context.
132
+ In Lite mode it groups items by collection and calls
133
+ VectorStore.insert_many — LanceDB has no equivalent of Weaviate's
134
+ single-context bulk write, but per-collection batching is fine for the
135
+ Lite use case.
98
136
  """
99
137
  if not items:
100
138
  return
101
139
 
102
140
  # 1. Check/Retry Connection
103
- if not self._initialized or not self.client:
141
+ if not self._initialized or (not self._lite_mode and not self.client):
104
142
  self._connect_client()
105
143
  if not self._initialized:
106
144
  return
107
145
 
108
- # 2. Send Batch via Weaviate Client
146
+ if self._lite_mode:
147
+ self._flush_via_store(items)
148
+ return
149
+
150
+ # 2. Send Batch via Weaviate Client (Pro mode)
109
151
  try:
110
152
  # Weaviate v4 batch context
111
153
  with self.client.batch.dynamic() as batch:
@@ -129,6 +171,31 @@ class WeaviateBatchManager:
129
171
  return
130
172
  logger.error(f"❌ Batch Flush Error: {e}")
131
173
 
174
+ def _flush_via_store(self, items: List[Dict[str, Any]]):
175
+ """Lite-mode flush: route items through the VectorStore abstraction."""
176
+ from ..store import get_vector_store
177
+ try:
178
+ store = get_vector_store()
179
+ except Exception as e:
180
+ logger.error(f"❌ Lite store unavailable: {e}")
181
+ return
182
+ by_collection: Dict[str, List[Dict[str, Any]]] = {}
183
+ for item in items:
184
+ by_collection.setdefault(item["collection"], []).append({
185
+ "properties": item["properties"],
186
+ "uuid": item.get("uuid"),
187
+ "vector": item.get("vector"),
188
+ })
189
+ for collection, batch in by_collection.items():
190
+ try:
191
+ # Lite stores create tables lazily, but ensure the schema exists
192
+ # so writes don't fail with "table not found".
193
+ if not store.collection_exists(collection):
194
+ store.ensure_collection(collection, properties=[])
195
+ store.insert_many(collection, batch)
196
+ except Exception as e:
197
+ logger.error(f"❌ Lite batch flush failed for '{collection}': {e}")
198
+
132
199
  # --- Legacy Python Worker Methods (Only used if Rust is missing) ---
133
200
  def _python_worker_loop(self):
134
201
  pending_items = []
@@ -148,9 +215,18 @@ class WeaviateBatchManager:
148
215
  last_flush_time = current_time
149
216
 
150
217
  def shutdown(self):
151
- """Gracefully shuts down."""
218
+ """Gracefully shuts down. Idempotent — repeated calls are no-ops, so the
219
+ atexit handler firing after a test-time cache_clear cannot trigger a
220
+ second shutdown on an already-closed Rust worker or Weaviate client."""
221
+ if self._shutdown_done:
222
+ return
223
+ self._shutdown_done = True
224
+
152
225
  if USE_RUST_CORE:
153
- self._rust_manager.shutdown()
226
+ try:
227
+ self._rust_manager.shutdown()
228
+ except Exception as e:
229
+ logger.debug(f"Rust manager shutdown raised: {e}")
154
230
  else:
155
231
  if not self._stop_event.is_set():
156
232
  self._stop_event.set()
@@ -168,9 +244,14 @@ class WeaviateBatchManager:
168
244
  if self.client:
169
245
  try:
170
246
  self.client.close()
171
- except:
247
+ except Exception:
172
248
  pass
173
249
 
174
- @lru_cache(None)
175
- def get_batch_manager() -> WeaviateBatchManager:
176
- return WeaviateBatchManager()
250
+ @lru_cache()
251
+ def get_batch_manager(
252
+ host: Optional[str] = None,
253
+ port: Optional[int] = None,
254
+ grpc_port: Optional[int] = None,
255
+ api_key: Optional[str] = None
256
+ ) -> WeaviateBatchManager:
257
+ return WeaviateBatchManager(host=host, port=port, grpc_port=grpc_port, api_key=api_key)
@@ -0,0 +1,17 @@
1
+ """VectorWave Check: pytest plugin + CLI for semantic regression testing.
2
+
3
+ Quick usage (declarative marker):
4
+
5
+ @pytest.mark.vectorwave(target="myapp.summarize", strategy="similarity", threshold=0.85)
6
+ def test_summarize_regression():
7
+ pass
8
+
9
+ Quick usage (imperative fixture):
10
+
11
+ def test_summarize_regression(vw_replay):
12
+ result = vw_replay("myapp.summarize", strategy="similarity", threshold=0.85)
13
+ assert result.passed_all, result.report()
14
+ """
15
+ from .plugin import ReplayResult
16
+
17
+ __all__ = ["ReplayResult"]
@@ -0,0 +1,358 @@
1
+ """Threshold calibration for the vectorwave-check pytest plugin.
2
+
3
+ Two measurement modes:
4
+
5
+ * **diversity** (default) — pull existing golden outputs for the target,
6
+ compute pairwise cosine similarity. Reports how diverse the function's
7
+ typical outputs are. No function calls, no side effects.
8
+ * **rerun** — sample a few golden inputs, re-execute the function N times
9
+ per input, compute pairwise similarity within each input group. Reports
10
+ the function's intrinsic noise floor. Hits the function (and any APIs it
11
+ calls); skip for functions with side effects.
12
+
13
+ Both modes emit the same shape so the downstream CLI / pyproject snippet
14
+ generator can stay one path.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import asyncio
19
+ import concurrent.futures
20
+ import importlib
21
+ import inspect
22
+ import logging
23
+ import math
24
+ from dataclasses import dataclass, field
25
+ from typing import Any, Dict, List, Optional, Sequence
26
+
27
+ logger = logging.getLogger(__name__)
28
+
29
+
30
+ PERCENTILES = (5, 10, 25, 50, 75, 95)
31
+
32
+
33
+ @dataclass
34
+ class CalibrationResult:
35
+ function: str
36
+ mode: str
37
+ sample_count: int # how many goldens (diversity) / inputs sampled (rerun)
38
+ pair_count: int # how many similarity pairs computed
39
+ percentiles: Dict[int, float] = field(default_factory=dict)
40
+ recommended_threshold: Optional[float] = None
41
+ recommended_strategy: str = "similarity"
42
+ notes: List[str] = field(default_factory=list)
43
+ vectorizer_name: Optional[str] = None
44
+
45
+
46
+ def _cosine(v1: Sequence[float], v2: Sequence[float]) -> float:
47
+ dot = 0.0
48
+ n1 = 0.0
49
+ n2 = 0.0
50
+ for a, b in zip(v1, v2):
51
+ dot += a * b
52
+ n1 += a * a
53
+ n2 += b * b
54
+ if n1 == 0 or n2 == 0:
55
+ return 0.0
56
+ return dot / (math.sqrt(n1) * math.sqrt(n2))
57
+
58
+
59
+ def _percentile(sorted_values: List[float], p: int) -> float:
60
+ if not sorted_values:
61
+ return 0.0
62
+ if len(sorted_values) == 1:
63
+ return sorted_values[0]
64
+ # linear interpolation between closest ranks
65
+ k = (len(sorted_values) - 1) * p / 100.0
66
+ lo = int(math.floor(k))
67
+ hi = int(math.ceil(k))
68
+ if lo == hi:
69
+ return sorted_values[lo]
70
+ return sorted_values[lo] + (sorted_values[hi] - sorted_values[lo]) * (k - lo)
71
+
72
+
73
+ def _pairwise_similarities(embeddings: List[Sequence[float]]) -> List[float]:
74
+ out: List[float] = []
75
+ for i in range(len(embeddings)):
76
+ for j in range(i + 1, len(embeddings)):
77
+ out.append(_cosine(embeddings[i], embeddings[j]))
78
+ return out
79
+
80
+
81
+ def _embed_all(vectorizer, texts: List[str]) -> List[Sequence[float]]:
82
+ if hasattr(vectorizer, "embed_batch"):
83
+ try:
84
+ return vectorizer.embed_batch(texts)
85
+ except Exception as e: # noqa: BLE001
86
+ logger.warning("embed_batch failed (%s); falling back to per-text embed", e)
87
+ return [vectorizer.embed(t) for t in texts]
88
+
89
+
90
+ def _summarize(
91
+ function: str,
92
+ mode: str,
93
+ sample_count: int,
94
+ similarities: List[float],
95
+ vectorizer_name: Optional[str],
96
+ ) -> CalibrationResult:
97
+ sims_sorted = sorted(similarities)
98
+ percentiles = {p: _percentile(sims_sorted, p) for p in PERCENTILES}
99
+
100
+ notes: List[str] = []
101
+ rec_threshold = percentiles.get(5)
102
+ rec_strategy = "similarity"
103
+
104
+ if not sims_sorted:
105
+ notes.append("No similarity pairs were computed — recommendation not available.")
106
+ rec_threshold = None
107
+ else:
108
+ p5 = percentiles[5]
109
+ p95 = percentiles[95]
110
+ if p5 > 0.99 and p95 > 0.99:
111
+ rec_strategy = "exact"
112
+ rec_threshold = None
113
+ notes.append(
114
+ "Function appears deterministic (p5 and p95 both >0.99). "
115
+ "Use `strategy=\"exact\"` instead of similarity."
116
+ )
117
+ elif p5 < 0.6:
118
+ rec_strategy = "llm"
119
+ notes.append(
120
+ f"Function is highly variable (p5={p5:.3f} < 0.6). "
121
+ "Similarity threshold will be noisy; consider `strategy=\"llm\"` "
122
+ "(LLM-as-a-judge) for more robust regression detection."
123
+ )
124
+
125
+ return CalibrationResult(
126
+ function=function,
127
+ mode=mode,
128
+ sample_count=sample_count,
129
+ pair_count=len(similarities),
130
+ percentiles=percentiles,
131
+ recommended_threshold=rec_threshold,
132
+ recommended_strategy=rec_strategy,
133
+ notes=notes,
134
+ vectorizer_name=vectorizer_name,
135
+ )
136
+
137
+
138
+ def _stringify(value: Any) -> str:
139
+ if isinstance(value, str):
140
+ return value
141
+ try:
142
+ import json
143
+ return json.dumps(value, sort_keys=True, default=str)
144
+ except (TypeError, ValueError):
145
+ return str(value)
146
+
147
+
148
+ def _run_coroutine_safely(coro):
149
+ try:
150
+ asyncio.get_running_loop()
151
+ except RuntimeError:
152
+ return asyncio.run(coro)
153
+ with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
154
+ return executor.submit(asyncio.run, coro).result()
155
+
156
+
157
+ def _vectorizer_name(vectorizer) -> Optional[str]:
158
+ if vectorizer is None:
159
+ return None
160
+ return type(vectorizer).__name__
161
+
162
+
163
+ # ---------------------------------------------------------------------------
164
+ # Mode 1: diversity (cheap, default)
165
+ # ---------------------------------------------------------------------------
166
+
167
+ def _calibrate_diversity(function_full_name: str, samples: int) -> CalibrationResult:
168
+ from ..models.db_config import get_weaviate_settings
169
+ from ..store import get_vector_store
170
+ from ..utils.serialization import deserialize_return_value
171
+ from ..vectorizer.factory import get_vectorizer
172
+
173
+ func_short_name = function_full_name.rsplit(".", 1)[-1]
174
+ settings = get_weaviate_settings()
175
+ store = get_vector_store()
176
+ vectorizer = get_vectorizer()
177
+ if vectorizer is None:
178
+ raise RuntimeError(
179
+ "Calibration requires a local vectorizer (VECTORIZER=huggingface or openai_client). "
180
+ "Server-side vectorization (weaviate_module) cannot embed offline."
181
+ )
182
+
183
+ records = store.query(
184
+ collection=settings.GOLDEN_COLLECTION_NAME,
185
+ filters={"function_name": func_short_name},
186
+ limit=samples,
187
+ )
188
+
189
+ outputs: List[str] = []
190
+ for rec in records:
191
+ raw = rec.properties.get("return_value")
192
+ deserialized = deserialize_return_value(raw)
193
+ outputs.append(_stringify(deserialized))
194
+
195
+ if len(outputs) < 2:
196
+ raise RuntimeError(
197
+ f"Need at least 2 golden samples for '{function_full_name}', found {len(outputs)}. "
198
+ "Run the function a few times in production / replay-capture mode first, "
199
+ "or mark known-good executions as Golden."
200
+ )
201
+
202
+ embeddings = _embed_all(vectorizer, outputs)
203
+ similarities = _pairwise_similarities(embeddings)
204
+
205
+ return _summarize(
206
+ function=function_full_name,
207
+ mode="diversity",
208
+ sample_count=len(outputs),
209
+ similarities=similarities,
210
+ vectorizer_name=_vectorizer_name(vectorizer),
211
+ )
212
+
213
+
214
+ # ---------------------------------------------------------------------------
215
+ # Mode 2: rerun (honest noise floor, opt-in)
216
+ # ---------------------------------------------------------------------------
217
+
218
+ def _calibrate_rerun(
219
+ function_full_name: str,
220
+ samples: int,
221
+ runs: int,
222
+ ) -> CalibrationResult:
223
+ from ..models.db_config import get_weaviate_settings
224
+ from ..utils.replayer import VectorWaveReplayer
225
+ from ..vectorizer.factory import get_vectorizer
226
+
227
+ module_name, func_short_name = function_full_name.rsplit(".", 1)
228
+ module = importlib.import_module(module_name)
229
+ target_func = getattr(module, func_short_name)
230
+ is_async = inspect.iscoroutinefunction(target_func)
231
+
232
+ get_weaviate_settings() # surface config errors early
233
+ vectorizer = get_vectorizer()
234
+ if vectorizer is None:
235
+ raise RuntimeError(
236
+ "Calibration requires a local vectorizer (VECTORIZER=huggingface or openai_client)."
237
+ )
238
+
239
+ helper = VectorWaveReplayer()
240
+ candidates = helper._fetch_test_candidates(func_short_name, limit=samples)
241
+ if not candidates:
242
+ raise RuntimeError(
243
+ f"No golden samples or execution logs found for '{function_full_name}'. "
244
+ "Rerun calibration needs at least one captured input."
245
+ )
246
+
247
+ all_similarities: List[float] = []
248
+ sampled = candidates[:samples]
249
+ for cand in sampled:
250
+ raw_inputs = cand["inputs"]
251
+ inputs = helper._extract_inputs(raw_inputs, target_func)
252
+ outputs: List[str] = []
253
+ for _ in range(runs):
254
+ try:
255
+ if is_async:
256
+ out = _run_coroutine_safely(target_func(**inputs))
257
+ else:
258
+ out = target_func(**inputs)
259
+ except Exception as e: # noqa: BLE001
260
+ logger.warning("Calibration run raised %s; skipping this run", e)
261
+ continue
262
+ outputs.append(_stringify(out))
263
+ if len(outputs) >= 2:
264
+ embeddings = _embed_all(vectorizer, outputs)
265
+ all_similarities.extend(_pairwise_similarities(embeddings))
266
+
267
+ if not all_similarities:
268
+ raise RuntimeError(
269
+ "Rerun calibration produced no usable comparisons "
270
+ "(function raised every time, or only one successful run per input). "
271
+ "Inspect the function's behavior manually."
272
+ )
273
+
274
+ return _summarize(
275
+ function=function_full_name,
276
+ mode="rerun",
277
+ sample_count=len(sampled),
278
+ similarities=all_similarities,
279
+ vectorizer_name=_vectorizer_name(vectorizer),
280
+ )
281
+
282
+
283
+ # ---------------------------------------------------------------------------
284
+ # Public API
285
+ # ---------------------------------------------------------------------------
286
+
287
+ def calibrate(
288
+ function_full_name: str,
289
+ *,
290
+ rerun: bool = False,
291
+ samples: Optional[int] = None,
292
+ runs: int = 10,
293
+ ) -> CalibrationResult:
294
+ """Compute a threshold recommendation for `function_full_name`.
295
+
296
+ Args:
297
+ function_full_name: Fully-qualified target, e.g. `myapp.summarize`.
298
+ rerun: If True, sample inputs and re-execute the function. Hits APIs
299
+ and triggers any side effects. If False (default), only pull
300
+ existing golden outputs and measure diversity — no side effects.
301
+ samples: Number of goldens to pull (diversity) or inputs to sample
302
+ (rerun). Defaults to 30 for diversity, 3 for rerun.
303
+ runs: Re-executions per sampled input. Only used in rerun mode.
304
+ """
305
+ if rerun:
306
+ effective_samples = samples if samples is not None else 3
307
+ return _calibrate_rerun(function_full_name, samples=effective_samples, runs=runs)
308
+ effective_samples = samples if samples is not None else 30
309
+ return _calibrate_diversity(function_full_name, samples=effective_samples)
310
+
311
+
312
+ # ---------------------------------------------------------------------------
313
+ # Rendering for CLI
314
+ # ---------------------------------------------------------------------------
315
+
316
+ def format_pyproject_snippet(result: CalibrationResult) -> str:
317
+ if result.recommended_strategy == "exact":
318
+ return (
319
+ f'[tool.vectorwave.check."{result.function}"]\n'
320
+ f'strategy = "exact"\n'
321
+ )
322
+ threshold = result.recommended_threshold or 0.85
323
+ return (
324
+ f'[tool.vectorwave.check."{result.function}"]\n'
325
+ f'strategy = "{result.recommended_strategy}"\n'
326
+ f"threshold = {threshold:.3f}\n"
327
+ )
328
+
329
+
330
+ def format_report(result: CalibrationResult) -> str:
331
+ lines = [
332
+ f"Calibration for '{result.function}' "
333
+ f"(mode={result.mode}, vectorizer={result.vectorizer_name or 'n/a'})",
334
+ f" samples={result.sample_count}, pairs={result.pair_count}",
335
+ "",
336
+ ]
337
+ for p in PERCENTILES:
338
+ v = result.percentiles.get(p)
339
+ if v is not None:
340
+ lines.append(f" p{p:<3} {v:.4f}")
341
+ lines.append("")
342
+ if result.recommended_threshold is None and result.recommended_strategy == "exact":
343
+ lines.append("Recommended strategy: exact (no threshold needed)")
344
+ else:
345
+ lines.append(
346
+ f"Recommended: strategy={result.recommended_strategy}, "
347
+ f"threshold={result.recommended_threshold:.4f}"
348
+ )
349
+ if result.notes:
350
+ lines.append("")
351
+ for note in result.notes:
352
+ lines.append(f" note: {note}")
353
+ lines.append("")
354
+ lines.append("Add to pyproject.toml:")
355
+ lines.append("")
356
+ for ln in format_pyproject_snippet(result).splitlines():
357
+ lines.append(f" {ln}")
358
+ return "\n".join(lines)