vectorwave 0.3.0__tar.gz → 1.0.0__tar.gz

{vectorwave-0.3.0 → vectorwave-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vectorwave
-Version: 0.3.0
+Version: 1.0.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -8,7 +8,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Rust
 Classifier: Operating System :: OS Independent
-Classifier: Development Status :: 3 - Alpha
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Requires-Dist: weaviate-client>=4.0.0
 Requires-Dist: pydantic-settings>=2.0.0
@@ -17,6 +17,25 @@ Requires-Dist: requests
 Requires-Dist: openai
 Requires-Dist: pygithub
 Requires-Dist: schedule
+Requires-Dist: tomli>=2.0 ; python_full_version < '3.11'
+Requires-Dist: pytest ; extra == 'dev'
+Requires-Dist: pytest-asyncio ; extra == 'dev'
+Requires-Dist: pytest-benchmark ; extra == 'dev'
+Requires-Dist: pytest-recording ; extra == 'dev'
+Requires-Dist: flake8 ; extra == 'dev'
+Requires-Dist: lancedb>=0.30.0 ; extra == 'dev'
+Requires-Dist: opentelemetry-api>=1.20.0 ; extra == 'dev'
+Requires-Dist: opentelemetry-sdk>=1.20.0 ; extra == 'dev'
+Requires-Dist: opentelemetry-exporter-otlp>=1.20.0 ; extra == 'dev'
+Requires-Dist: testcontainers>=4.0.0 ; extra == 'dev'
+Requires-Dist: vcrpy>=6.0.0 ; extra == 'dev'
+Requires-Dist: lancedb>=0.30.0 ; extra == 'lite'
+Requires-Dist: opentelemetry-api>=1.20.0 ; extra == 'otel'
+Requires-Dist: opentelemetry-sdk>=1.20.0 ; extra == 'otel'
+Requires-Dist: opentelemetry-exporter-otlp>=1.20.0 ; extra == 'otel'
+Provides-Extra: dev
+Provides-Extra: lite
+Provides-Extra: otel
 License-File: LICENSE
 License-File: NOTICE
 Summary: VectorWave: Seamless Auto-Vectorization Framework

{vectorwave-0.3.0 → vectorwave-1.0.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "maturin"
 
 [project]
 name = "vectorwave"
-version = "0.3.0"
+version = "1.0.0"
 authors = [
     { name = "junyeonggim", email = "junyeonggim5@gmail.com" },
 ]
@@ -21,7 +21,7 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
     "Programming Language :: Rust",
     "Operating System :: OS Independent",
-    "Development Status :: 3 - Alpha",
+    "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Developers",
 ]
 
@@ -32,9 +32,39 @@ dependencies = [
     "requests",
     "openai",
     "PyGithub",
-    "schedule"
+    "schedule",
+    "tomli>=2.0; python_version < '3.11'",
 ]
 
+[project.optional-dependencies]
+dev = [
+    "pytest",
+    "pytest-asyncio",
+    "pytest-benchmark",
+    "pytest-recording",
+    "flake8",
+    "lancedb>=0.30.0",
+    "opentelemetry-api>=1.20.0",
+    "opentelemetry-sdk>=1.20.0",
+    "opentelemetry-exporter-otlp>=1.20.0",
+    "testcontainers>=4.0.0",
+    "vcrpy>=6.0.0",
+]
+lite = [
+    "lancedb>=0.30.0",
+]
+otel = [
+    "opentelemetry-api>=1.20.0",
+    "opentelemetry-sdk>=1.20.0",
+    "opentelemetry-exporter-otlp>=1.20.0",
+]
+
+[project.scripts]
+vectorwave = "vectorwave.cli:main"
+
+[project.entry-points.pytest11]
+vectorwave_check = "vectorwave.check.plugin"
+
 [project.urls]
 Repository = "https://github.com/cozymori/vectorwave"
 
@@ -42,4 +72,5 @@ Repository = "https://github.com/cozymori/vectorwave"
 python-source = "src"
 module-name = "vectorwave.vectorwave_core"
 manifest-path = "crates/Cargo.toml"
-features = ["pyo3/extension-module"]
+features = ["pyo3/extension-module"]
+include = ["src/vectorwave/cli/dev/compose.yml"]

{vectorwave-0.3.0 → vectorwave-1.0.0}/src/vectorwave/__init__.py

@@ -1,3 +1,8 @@
+from . import runtime as _runtime
+# Light up the visible indicator (stderr banner + PID file) on first import.
+# Silence with VECTORWAVE_QUIET=1.
+_runtime.activate()
+
 from .core.decorator import vectorize
 from .database.db import initialize_database, update_database_schema
 from .database.db_search import search_functions, search_executions, search_errors_by_message, search_functions_hybrid

{vectorwave-0.3.0 → vectorwave-1.0.0}/src/vectorwave/batch/batch.py

@@ -1,3 +1,4 @@
+import os
 import weaviate
 import atexit
 import logging
@@ -30,9 +31,15 @@ class WeaviateBatchManager:
     def __init__(self, host: Optional[str] = None, port: Optional[int] = None,
                  grpc_port: Optional[int] = None, api_key: Optional[str] = None):
         self._initialized = False
+        self._shutdown_done = False
         self.settings: WeaviateSettings = get_weaviate_settings()
         self.client: Optional[weaviate.WeaviateClient] = None
 
+        # Lite mode (LanceDB local file store) skips the Weaviate client entirely.
+        # When VECTORWAVE_MODE=lite, _flush_batch_core delegates to the
+        # configured VectorStore so no Docker / Weaviate connection is needed.
+        self._lite_mode = os.environ.get("VECTORWAVE_MODE", "pro").lower() == "lite"
+
         # Store dynamic connection params
         self._host = host
         self._port = port
@@ -65,7 +72,19 @@ class WeaviateBatchManager:
         atexit.register(self.shutdown)
 
     def _connect_client(self):
-        """Attempts to connect to Weaviate."""
+        """Attempts to connect to the configured backend (Weaviate or Lite store)."""
+        if self._lite_mode:
+            try:
+                from ..store import get_vector_store
+                store = get_vector_store()
+                if store.is_ready():
+                    self._initialized = True
+                    self.client = None  # not used in Lite mode
+            except Exception as e:
+                logger.warning(f"Lite store init failed: {e}")
+                self._initialized = False
+            return
+
         try:
             if self._host is not None:
                 self.client = get_weaviate_client(
@@ -108,17 +127,27 @@ class WeaviateBatchManager:
     def _flush_batch_core(self, items: List[Dict[str, Any]]):
         """
         The actual flush logic called by either Rust or Python worker.
+
+        In Pro mode (default) this uses Weaviate's bulk batch.dynamic() context.
+        In Lite mode it groups items by collection and calls
+        VectorStore.insert_many — LanceDB has no equivalent of Weaviate's
+        single-context bulk write, but per-collection batching is fine for the
+        Lite use case.
         """
         if not items:
             return
 
         # 1. Check/Retry Connection
-        if not self._initialized or not self.client:
+        if not self._initialized or (not self._lite_mode and not self.client):
             self._connect_client()
             if not self._initialized:
                 return
 
-        # 2. Send Batch via Weaviate Client
+        if self._lite_mode:
+            self._flush_via_store(items)
+            return
+
+        # 2. Send Batch via Weaviate Client (Pro mode)
         try:
             # Weaviate v4 batch context
             with self.client.batch.dynamic() as batch:
@@ -142,6 +171,31 @@ class WeaviateBatchManager:
                 return
             logger.error(f"❌ Batch Flush Error: {e}")
 
+    def _flush_via_store(self, items: List[Dict[str, Any]]):
+        """Lite-mode flush: route items through the VectorStore abstraction."""
+        from ..store import get_vector_store
+        try:
+            store = get_vector_store()
+        except Exception as e:
+            logger.error(f"❌ Lite store unavailable: {e}")
+            return
+        by_collection: Dict[str, List[Dict[str, Any]]] = {}
+        for item in items:
+            by_collection.setdefault(item["collection"], []).append({
+                "properties": item["properties"],
+                "uuid": item.get("uuid"),
+                "vector": item.get("vector"),
+            })
+        for collection, batch in by_collection.items():
+            try:
+                # Lite stores create tables lazily, but ensure the schema exists
+                # so writes don't fail with "table not found".
+                if not store.collection_exists(collection):
+                    store.ensure_collection(collection, properties=[])
+                store.insert_many(collection, batch)
+            except Exception as e:
+                logger.error(f"❌ Lite batch flush failed for '{collection}': {e}")
+
     # --- Legacy Python Worker Methods (Only used if Rust is missing) ---
     def _python_worker_loop(self):
         pending_items = []
@@ -161,9 +215,18 @@ class WeaviateBatchManager:
                 last_flush_time = current_time
 
     def shutdown(self):
-        """Gracefully shuts down."""
+        """Gracefully shuts down. Idempotent — repeated calls are no-ops, so the
+        atexit handler firing after a test-time cache_clear cannot trigger a
+        second shutdown on an already-closed Rust worker or Weaviate client."""
+        if self._shutdown_done:
+            return
+        self._shutdown_done = True
+
         if USE_RUST_CORE:
-            self._rust_manager.shutdown()
+            try:
+                self._rust_manager.shutdown()
+            except Exception as e:
+                logger.debug(f"Rust manager shutdown raised: {e}")
         else:
             if not self._stop_event.is_set():
                 self._stop_event.set()
@@ -181,7 +244,7 @@ class WeaviateBatchManager:
         if self.client:
             try:
                 self.client.close()
-            except:
+            except Exception:
                 pass
 
 @lru_cache()

vectorwave-1.0.0/src/vectorwave/check/__init__.py

@@ -0,0 +1,17 @@
+"""VectorWave Check: pytest plugin + CLI for semantic regression testing.
+
+Quick usage (declarative marker):
+
+    @pytest.mark.vectorwave(target="myapp.summarize", strategy="similarity", threshold=0.85)
+    def test_summarize_regression():
+        pass
+
+Quick usage (imperative fixture):
+
+    def test_summarize_regression(vw_replay):
+        result = vw_replay("myapp.summarize", strategy="similarity", threshold=0.85)
+        assert result.passed_all, result.report()
+"""
+from .plugin import ReplayResult
+
+__all__ = ["ReplayResult"]

vectorwave-1.0.0/src/vectorwave/check/calibrate.py

@@ -0,0 +1,358 @@
+"""Threshold calibration for the vectorwave-check pytest plugin.
+
+Two measurement modes:
+
+* **diversity** (default) — pull existing golden outputs for the target,
+  compute pairwise cosine similarity. Reports how diverse the function's
+  typical outputs are. No function calls, no side effects.
+* **rerun** — sample a few golden inputs, re-execute the function N times
+  per input, compute pairwise similarity within each input group. Reports
+  the function's intrinsic noise floor. Hits the function (and any APIs it
+  calls); skip for functions with side effects.
+
+Both modes emit the same shape so the downstream CLI / pyproject snippet
+generator can stay one path.
+"""
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import importlib
+import inspect
+import logging
+import math
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Sequence
+
+logger = logging.getLogger(__name__)
+
+
+PERCENTILES = (5, 10, 25, 50, 75, 95)
+
+
+@dataclass
+class CalibrationResult:
+    function: str
+    mode: str
+    sample_count: int  # how many goldens (diversity) / inputs sampled (rerun)
+    pair_count: int    # how many similarity pairs computed
+    percentiles: Dict[int, float] = field(default_factory=dict)
+    recommended_threshold: Optional[float] = None
+    recommended_strategy: str = "similarity"
+    notes: List[str] = field(default_factory=list)
+    vectorizer_name: Optional[str] = None
+
+
+def _cosine(v1: Sequence[float], v2: Sequence[float]) -> float:
+    dot = 0.0
+    n1 = 0.0
+    n2 = 0.0
+    for a, b in zip(v1, v2):
+        dot += a * b
+        n1 += a * a
+        n2 += b * b
+    if n1 == 0 or n2 == 0:
+        return 0.0
+    return dot / (math.sqrt(n1) * math.sqrt(n2))
+
+
+def _percentile(sorted_values: List[float], p: int) -> float:
+    if not sorted_values:
+        return 0.0
+    if len(sorted_values) == 1:
+        return sorted_values[0]
+    # linear interpolation between closest ranks
+    k = (len(sorted_values) - 1) * p / 100.0
+    lo = int(math.floor(k))
+    hi = int(math.ceil(k))
+    if lo == hi:
+        return sorted_values[lo]
+    return sorted_values[lo] + (sorted_values[hi] - sorted_values[lo]) * (k - lo)
+
+
+def _pairwise_similarities(embeddings: List[Sequence[float]]) -> List[float]:
+    out: List[float] = []
+    for i in range(len(embeddings)):
+        for j in range(i + 1, len(embeddings)):
+            out.append(_cosine(embeddings[i], embeddings[j]))
+    return out
+
+
+def _embed_all(vectorizer, texts: List[str]) -> List[Sequence[float]]:
+    if hasattr(vectorizer, "embed_batch"):
+        try:
+            return vectorizer.embed_batch(texts)
+        except Exception as e:  # noqa: BLE001
+            logger.warning("embed_batch failed (%s); falling back to per-text embed", e)
+    return [vectorizer.embed(t) for t in texts]
+
+
+def _summarize(
+    function: str,
+    mode: str,
+    sample_count: int,
+    similarities: List[float],
+    vectorizer_name: Optional[str],
+) -> CalibrationResult:
+    sims_sorted = sorted(similarities)
+    percentiles = {p: _percentile(sims_sorted, p) for p in PERCENTILES}
+
+    notes: List[str] = []
+    rec_threshold = percentiles.get(5)
+    rec_strategy = "similarity"
+
+    if not sims_sorted:
+        notes.append("No similarity pairs were computed — recommendation not available.")
+        rec_threshold = None
+    else:
+        p5 = percentiles[5]
+        p95 = percentiles[95]
+        if p5 > 0.99 and p95 > 0.99:
+            rec_strategy = "exact"
+            rec_threshold = None
+            notes.append(
+                "Function appears deterministic (p5 and p95 both >0.99). "
+                "Use `strategy=\"exact\"` instead of similarity."
+            )
+        elif p5 < 0.6:
+            rec_strategy = "llm"
+            notes.append(
+                f"Function is highly variable (p5={p5:.3f} < 0.6). "
+                "Similarity threshold will be noisy; consider `strategy=\"llm\"` "
+                "(LLM-as-a-judge) for more robust regression detection."
+            )
+
+    return CalibrationResult(
+        function=function,
+        mode=mode,
+        sample_count=sample_count,
+        pair_count=len(similarities),
+        percentiles=percentiles,
+        recommended_threshold=rec_threshold,
+        recommended_strategy=rec_strategy,
+        notes=notes,
+        vectorizer_name=vectorizer_name,
+    )
+
+
+def _stringify(value: Any) -> str:
+    if isinstance(value, str):
+        return value
+    try:
+        import json
+        return json.dumps(value, sort_keys=True, default=str)
+    except (TypeError, ValueError):
+        return str(value)
+
+
+def _run_coroutine_safely(coro):
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(coro)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+        return executor.submit(asyncio.run, coro).result()
+
+
+def _vectorizer_name(vectorizer) -> Optional[str]:
+    if vectorizer is None:
+        return None
+    return type(vectorizer).__name__
+
+
+# ---------------------------------------------------------------------------
+# Mode 1: diversity (cheap, default)
+# ---------------------------------------------------------------------------
+
+def _calibrate_diversity(function_full_name: str, samples: int) -> CalibrationResult:
+    from ..models.db_config import get_weaviate_settings
+    from ..store import get_vector_store
+    from ..utils.serialization import deserialize_return_value
+    from ..vectorizer.factory import get_vectorizer
+
+    func_short_name = function_full_name.rsplit(".", 1)[-1]
+    settings = get_weaviate_settings()
+    store = get_vector_store()
+    vectorizer = get_vectorizer()
+    if vectorizer is None:
+        raise RuntimeError(
+            "Calibration requires a local vectorizer (VECTORIZER=huggingface or openai_client). "
+            "Server-side vectorization (weaviate_module) cannot embed offline."
+        )
+
+    records = store.query(
+        collection=settings.GOLDEN_COLLECTION_NAME,
+        filters={"function_name": func_short_name},
+        limit=samples,
+    )
+
+    outputs: List[str] = []
+    for rec in records:
+        raw = rec.properties.get("return_value")
+        deserialized = deserialize_return_value(raw)
+        outputs.append(_stringify(deserialized))
+
+    if len(outputs) < 2:
+        raise RuntimeError(
+            f"Need at least 2 golden samples for '{function_full_name}', found {len(outputs)}. "
+            "Run the function a few times in production / replay-capture mode first, "
+            "or mark known-good executions as Golden."
+        )
+
+    embeddings = _embed_all(vectorizer, outputs)
+    similarities = _pairwise_similarities(embeddings)
+
+    return _summarize(
+        function=function_full_name,
+        mode="diversity",
+        sample_count=len(outputs),
+        similarities=similarities,
+        vectorizer_name=_vectorizer_name(vectorizer),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Mode 2: rerun (honest noise floor, opt-in)
+# ---------------------------------------------------------------------------
+
+def _calibrate_rerun(
+    function_full_name: str,
+    samples: int,
+    runs: int,
+) -> CalibrationResult:
+    from ..models.db_config import get_weaviate_settings
+    from ..utils.replayer import VectorWaveReplayer
+    from ..vectorizer.factory import get_vectorizer
+
+    module_name, func_short_name = function_full_name.rsplit(".", 1)
+    module = importlib.import_module(module_name)
+    target_func = getattr(module, func_short_name)
+    is_async = inspect.iscoroutinefunction(target_func)
+
+    get_weaviate_settings()  # surface config errors early
+    vectorizer = get_vectorizer()
+    if vectorizer is None:
+        raise RuntimeError(
+            "Calibration requires a local vectorizer (VECTORIZER=huggingface or openai_client)."
+        )
+
+    helper = VectorWaveReplayer()
+    candidates = helper._fetch_test_candidates(func_short_name, limit=samples)
+    if not candidates:
+        raise RuntimeError(
+            f"No golden samples or execution logs found for '{function_full_name}'. "
+            "Rerun calibration needs at least one captured input."
+        )
+
+    all_similarities: List[float] = []
+    sampled = candidates[:samples]
+    for cand in sampled:
+        raw_inputs = cand["inputs"]
+        inputs = helper._extract_inputs(raw_inputs, target_func)
+        outputs: List[str] = []
+        for _ in range(runs):
+            try:
+                if is_async:
+                    out = _run_coroutine_safely(target_func(**inputs))
+                else:
+                    out = target_func(**inputs)
+            except Exception as e:  # noqa: BLE001
+                logger.warning("Calibration run raised %s; skipping this run", e)
+                continue
+            outputs.append(_stringify(out))
+        if len(outputs) >= 2:
+            embeddings = _embed_all(vectorizer, outputs)
+            all_similarities.extend(_pairwise_similarities(embeddings))
+
+    if not all_similarities:
+        raise RuntimeError(
+            "Rerun calibration produced no usable comparisons "
+            "(function raised every time, or only one successful run per input). "
+            "Inspect the function's behavior manually."
+        )
+
+    return _summarize(
+        function=function_full_name,
+        mode="rerun",
+        sample_count=len(sampled),
+        similarities=all_similarities,
+        vectorizer_name=_vectorizer_name(vectorizer),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def calibrate(
+    function_full_name: str,
+    *,
+    rerun: bool = False,
+    samples: Optional[int] = None,
+    runs: int = 10,
+) -> CalibrationResult:
+    """Compute a threshold recommendation for `function_full_name`.
+
+    Args:
+        function_full_name: Fully-qualified target, e.g. `myapp.summarize`.
+        rerun: If True, sample inputs and re-execute the function. Hits APIs
+            and triggers any side effects. If False (default), only pull
+            existing golden outputs and measure diversity — no side effects.
+        samples: Number of goldens to pull (diversity) or inputs to sample
+            (rerun). Defaults to 30 for diversity, 3 for rerun.
+        runs: Re-executions per sampled input. Only used in rerun mode.
+    """
+    if rerun:
+        effective_samples = samples if samples is not None else 3
+        return _calibrate_rerun(function_full_name, samples=effective_samples, runs=runs)
+    effective_samples = samples if samples is not None else 30
+    return _calibrate_diversity(function_full_name, samples=effective_samples)
+
+
+# ---------------------------------------------------------------------------
+# Rendering for CLI
+# ---------------------------------------------------------------------------
+
+def format_pyproject_snippet(result: CalibrationResult) -> str:
+    if result.recommended_strategy == "exact":
+        return (
+            f'[tool.vectorwave.check."{result.function}"]\n'
+            f'strategy = "exact"\n'
+        )
+    threshold = result.recommended_threshold or 0.85
+    return (
+        f'[tool.vectorwave.check."{result.function}"]\n'
+        f'strategy = "{result.recommended_strategy}"\n'
+        f"threshold = {threshold:.3f}\n"
+    )
+
+
+def format_report(result: CalibrationResult) -> str:
+    lines = [
+        f"Calibration for '{result.function}' "
+        f"(mode={result.mode}, vectorizer={result.vectorizer_name or 'n/a'})",
+        f"  samples={result.sample_count}, pairs={result.pair_count}",
+        "",
+    ]
+    for p in PERCENTILES:
+        v = result.percentiles.get(p)
+        if v is not None:
+            lines.append(f"  p{p:<3} {v:.4f}")
+    lines.append("")
+    if result.recommended_threshold is None and result.recommended_strategy == "exact":
+        lines.append("Recommended strategy: exact (no threshold needed)")
+    else:
+        lines.append(
+            f"Recommended: strategy={result.recommended_strategy}, "
+            f"threshold={result.recommended_threshold:.4f}"
+        )
+    if result.notes:
+        lines.append("")
+        for note in result.notes:
+            lines.append(f"  note: {note}")
+    lines.append("")
+    lines.append("Add to pyproject.toml:")
+    lines.append("")
+    for ln in format_pyproject_snippet(result).splitlines():
+        lines.append(f"  {ln}")
+    return "\n".join(lines)