vectorwave 0.2.6__cp312-cp312-win_amd64.whl

vectorwave/__init__.py

@@ -0,0 +1,29 @@
+from .core.decorator import vectorize
+from .database.db import initialize_database
+from .database.db_search import search_functions, search_executions, search_errors_by_message, search_functions_hybrid
+from .monitoring.tracer import trace_span
+from .search.rag_search import search_and_answer, analyze_trace_log
+from .core.generator import generate_and_register_metadata
+from .utils.healer import VectorWaveHealer
+from .utils.replayer import VectorWaveReplayer
+from .utils.replayer_semantic import SemanticReplayer
+from .database.dataset import VectorWaveDatasetManager
+from .core.auto_injector import VectorWaveAutoInjector
+
+__all__ = [
+    'vectorize',
+    'initialize_database',
+    'search_functions',
+    'search_functions_hybrid',
+    'search_executions',
+    'search_errors_by_message',
+    'trace_span',
+    'search_and_answer',
+    'analyze_trace_log',
+    'generate_and_register_metadata',
+    'VectorWaveHealer',
+    'VectorWaveReplayer',
+    'SemanticReplayer',
+    'VectorWaveDatasetManager',
+    'VectorWaveAutoInjector'
+]

vectorwave/batch/batch.py

@@ -0,0 +1,176 @@
+import weaviate
+import atexit
+import logging
+import threading
+import queue
+import time
+from functools import lru_cache
+from typing import Optional, List, Dict, Any
+
+from ..models.db_config import get_weaviate_settings, WeaviateSettings
+from ..database.db import get_weaviate_client
+
+# Rust Core 모듈 Import 시도
+
+
+logger = logging.getLogger(__name__)
+
+try:
+    from vectorwave.vectorwave_core import RustBatchManager
+    USE_RUST_CORE = True
+except ImportError:
+    USE_RUST_CORE = False
+
+class WeaviateBatchManager:
+    """
+    Manages Weaviate batch imports.
+    Uses High-Performance Rust Core if available, otherwise falls back to Python.
+    """
+
+    def __init__(self):
+        self._initialized = False
+        self.settings: WeaviateSettings = get_weaviate_settings()
+        self.client: Optional[weaviate.WeaviateClient] = None
+
+        # Batch Configuration
+        self.batch_threshold = self.settings.BATCH_THRESHOLD
+        self.flush_interval = self.settings.FLUSH_INTERVAL_SECONDS
+
+        # Connect to DB
+        self._connect_client()
+
+        if USE_RUST_CORE:
+            logger.info(f"🚀 [VectorWave] Rust Core Activated! (Threshold: {self.batch_threshold}, Interval: {self.flush_interval}s)")
+            self._rust_manager = RustBatchManager(
+                self._flush_batch_core,
+                self.batch_threshold,
+                int(self.flush_interval * 1000) # ms 단위 변환
+            )
+            self._worker_thread = None
+        else:
+            logger.warning("⚠️ [VectorWave] Rust Core not found. Using slower Python implementation.")
+            # --- Legacy Python Implementation ---
+            self.queue = queue.Queue(maxsize=10000)
+            self._stop_event = threading.Event()
+            self._start_python_worker()
+
+        # Register shutdown handler
+        atexit.register(self.shutdown)
+
+    def _connect_client(self):
+        """Attempts to connect to Weaviate."""
+        try:
+            self.client = get_weaviate_client(self.settings)
+            if self.client:
+                self._initialized = True
+        except Exception as e:
+            logger.warning(f"Initial DB connection failed: {e}")
+            self._initialized = False
+
+    def _start_python_worker(self):
+        """Starts the legacy Python background thread."""
+        self._worker_thread = threading.Thread(target=self._python_worker_loop, daemon=True)
+        self._worker_thread.start()
+
+    def add_object(self, collection: str, properties: dict, uuid: str = None, vector: Optional[List[float]] = None):
+        """
+        [Public API] Adds an object to the batch queue.
+        """
+        if USE_RUST_CORE:
+
+            self._rust_manager.add_object(collection, properties, uuid, vector)
+        else:
+            # Python Legacy Queue
+            item = {
+                "collection": collection,
+                "properties": properties,
+                "uuid": uuid,
+                "vector": vector
+            }
+            try:
+                self.queue.put_nowait(item)
+            except queue.Full:
+                logger.warning("🚨 VectorWave Log Queue is FULL. Dropping log.")
+
+    def _flush_batch_core(self, items: List[Dict[str, Any]]):
+        """
+        The actual flush logic called by either Rust or Python worker.
+        """
+        if not items:
+            return
+
+        # 1. Check/Retry Connection
+        if not self._initialized or not self.client:
+            self._connect_client()
+            if not self._initialized:
+                return
+
+        # 2. Send Batch via Weaviate Client
+        try:
+            # Weaviate v4 batch context
+            with self.client.batch.dynamic() as batch:
+                for item in items:
+                    batch.add_object(
+                        collection=item['collection'],
+                        properties=item['properties'],
+                        uuid=item.get('uuid'),
+                        vector=item.get('vector')
+                    )
+
+            if len(self.client.batch.failed_objects) > 0:
+                for failed in self.client.batch.failed_objects:
+                    logger.error(f"⚠️ Batch Item Failed: {failed.message}")
+
+        except RuntimeError:
+            return
+        except Exception as e:
+            msg = str(e).lower()
+            if "shutdown" in msg or "closed" in msg:
+                return
+            logger.error(f"❌ Batch Flush Error: {e}")
+
+    # --- Legacy Python Worker Methods (Only used if Rust is missing) ---
+    def _python_worker_loop(self):
+        pending_items = []
+        last_flush_time = time.time()
+
+        while not self._stop_event.is_set():
+            try:
+                item = self.queue.get(timeout=0.5)
+                pending_items.append(item)
+            except queue.Empty:
+                pass
+
+            current_time = time.time()
+            if len(pending_items) >= self.batch_threshold or (pending_items and current_time - last_flush_time >= self.flush_interval):
+                self._flush_batch_core(pending_items)
+                pending_items = []
+                last_flush_time = current_time
+
+    def shutdown(self):
+        """Gracefully shuts down."""
+        if USE_RUST_CORE:
+            self._rust_manager.shutdown()
+        else:
+            if not self._stop_event.is_set():
+                self._stop_event.set()
+                if self._worker_thread and self._worker_thread.is_alive():
+                    self._worker_thread.join(timeout=1.0)
+
+                # Flush remaining items
+                remaining = []
+                while not self.queue.empty():
+                    remaining.append(self.queue.get_nowait())
+                if remaining:
+                    self._flush_batch_core(remaining)
+
+        # Close client
+        if self.client:
+            try:
+                self.client.close()
+            except:
+                pass
+
+@lru_cache(None)
+def get_batch_manager() -> WeaviateBatchManager:
+    return WeaviateBatchManager()

vectorwave/core/auto_injector.py

@@ -0,0 +1,100 @@
+import inspect
+import importlib
+import logging
+from functools import wraps
+from typing import Any, Optional
+
+# VectorWave Core Modules
+from .decorator import vectorize
+from ..monitoring.tracer import trace_span, current_tracer_var
+
+logger = logging.getLogger(__name__)
+
+def create_smart_wrapper(original_func, root_wrapper):
+    """
+    Creates a wrapper that switches behavior based on the Tracing Context.
+
+    Args:
+        original_func: The original function (used when acting as a Child Span).
+        root_wrapper: The wrapper generated by vectorize (used when acting as a Root Span, includes metadata).
+    """
+    @wraps(original_func)
+    def wrapper(*args, **kwargs):
+        # Check for existing tracer at runtime
+        tracer = current_tracer_var.get()
+
+        if tracer:
+            # [Case A] Parent trace exists -> Act as Child Span
+            # (Wrap original function with trace_span and execute)
+            return trace_span(original_func, capture_return_value=True)(*args, **kwargs)
+        else:
+            # [Case B] No parent trace -> Act as Root Span
+            # (Delegate to the already configured root_wrapper)
+            return root_wrapper(*args, **kwargs)
+
+    # Mark to prevent double injection
+    wrapper._is_vectorized = True
+    return wrapper
+
+
+class VectorWaveAutoInjector:
+    """
+    Scans specified modules or packages to automatically inject (Monkey Patch) VectorWave functionality.
+    """
+
+    # Global default configuration
+    _default_config = {}
+
+    @classmethod
+    def configure(cls, **kwargs):
+        """
+        Sets the default configuration to be applied to all inject calls.
+        """
+        cls._default_config.update(kwargs)
+        logger.info(f"⚙️ VectorWave AutoInjector Configured: {cls._default_config}")
+
+    @classmethod
+    def inject(cls, target_module_path: str, recursive: bool = False, **config):
+        """
+        Finds functions within a module and injects VectorWave functionality.
+        **Metadata registration (DB save or Pending) is performed immediately at this point.**
+        """
+        try:
+            module = importlib.import_module(target_module_path)
+        except ImportError as e:
+            logger.error(f"Failed to import module '{target_module_path}': {e}")
+            return None
+
+        # Merge configurations (Defaults < Individual Config)
+        final_config = cls._default_config.copy()
+        final_config.update(config)
+
+        logger.info(f"🌊 [AutoInjector] Injecting VectorWave into: {module.__name__}")
+
+        patched_count = 0
+
+        for name, obj in inspect.getmembers(module):
+            # Target only functions defined within the module
+            if inspect.isfunction(obj) and obj.__module__ == module.__name__:
+
+                # Skip if already processed
+                if getattr(obj, "_is_vectorized", False):
+                    continue
+
+                logger.info(f"  └─ Auto-wiring: {name}()")
+
+                # [Key Change] Call vectorize immediately at injection time to perform metadata registration!
+                # - If auto=False: Immediate DB save
+                # - If auto=True: Add to PENDING list
+                # The returned root_wrapper contains the execution logic (Root Span)
+                root_wrapper = vectorize(**final_config)(obj)
+
+                # Create smart wrapper to judge Context at runtime
+                smart_wrapper = create_smart_wrapper(obj, root_wrapper)
+
+                # Replace (Monkey Patching)
+                setattr(module, name, smart_wrapper)
+                patched_count += 1
+
+        logger.info(f"✨ Injection Complete. {patched_count} functions registered & auto-wired.")
+        return module

vectorwave/core/decorator.py

@@ -0,0 +1,191 @@
+import inspect
+import logging
+from functools import wraps
+from typing import List, Optional, Dict, Any
+
+from weaviate.util import generate_uuid5
+
+from ..batch.batch import get_batch_manager
+from ..models.db_config import get_weaviate_settings
+from ..monitoring.tracer import trace_root, trace_span
+from ..utils.function_cache import function_cache_manager
+from ..utils.return_caching_utils import _check_and_return_cached_result
+from ..vectorizer.factory import get_vectorizer
+from ..utils.context import execution_source_context
+
+logger = logging.getLogger(__name__)
+
+PENDING_FUNCTIONS: List[Dict[str, Any]] = []
+
+
+def vectorize(search_description: Optional[str] = None,
+              sequence_narrative: Optional[str] = None,
+              auto: bool = False,
+              capture_return_value: bool = False,
+              semantic_cache: bool = False,
+              cache_threshold: float = 0.9,
+              replay: bool = False,
+              attributes_to_capture: Optional[List[str]] = None,
+              **execution_tags):
+    """
+    VectorWave Decorator with Auto-Generation support.
+    """
+
+    if semantic_cache:
+        if get_vectorizer() is None:
+            logger.warning(
+                f"Semantic caching requested for '{search_description}' but no Python vectorizer is configured. "
+                f"Disabling semantic_cache."
+            )
+            semantic_cache = False
+
+    if semantic_cache and not capture_return_value:
+        capture_return_value = True
+
+    if replay and not capture_return_value:
+        capture_return_value = True
+
+    def decorator(func):
+        is_async_func = inspect.iscoroutinefunction(func)
+
+        module_name = func.__module__
+        function_name = func.__name__
+        func_identifier = f"{module_name}.{function_name}"
+        func_uuid = generate_uuid5(func_identifier)
+
+        # Prepare attributes to capture
+        final_attributes = ['function_uuid', 'team', 'priority', 'run_id', 'exec_source']
+        if attributes_to_capture:
+            for attr in attributes_to_capture:
+                if attr not in final_attributes:
+                    final_attributes.append(attr)
+
+        if replay:
+            try:
+                sig = inspect.signature(func)
+                for param_name in sig.parameters:
+                    if param_name not in ('self', 'cls') and param_name not in final_attributes:
+                        final_attributes.append(param_name)
+            except Exception as e:
+                logger.warning(f"Failed to inspect signature for replay auto-capture in '{function_name}': {e}")
+
+        # Extract Execution Tags
+        valid_execution_tags = {}
+        settings = get_weaviate_settings()
+        if execution_tags and settings.custom_properties:
+            allowed_keys = set(settings.custom_properties.keys())
+            for key, value in execution_tags.items():
+                if key in allowed_keys:
+                    valid_execution_tags[key] = value
+                else:
+                    logger.warning(
+                        "Function '%s' has undefined execution_tag: '%s'. "
+                        "This tag will be IGNORED. Please add it to your .weaviate_properties file.",
+                        function_name,
+                        key
+                    )
+
+        try:
+            # define static properties
+            docstring = inspect.getdoc(func) or ""
+            source_code = inspect.getsource(func)
+
+            static_properties = {
+                "function_name": function_name,
+                "module_name": module_name,
+                "docstring": docstring,
+                "source_code": source_code,
+                "search_description": search_description,
+                "sequence_narrative": sequence_narrative
+            }
+            static_properties.update(valid_execution_tags)
+
+            if auto:
+                logger.info(f"Function '{function_name}' registered for auto-metadata generation.")
+                PENDING_FUNCTIONS.append({
+                    "func_name": function_name,
+                    "func_uuid": func_uuid,
+                    "func_identifier": func_identifier,
+                    "static_properties": static_properties
+                })
+            else:
+                # [Existing] Immediate Registration Mode
+                current_content_hash = function_cache_manager.calculate_content_hash(func_identifier, static_properties)
+
+                if function_cache_manager.is_cached_and_unchanged(func_uuid, current_content_hash):
+                    logger.info(f"Function '{function_name}' is UNCHANGED. Skipping DB write.")
+                else:
+                    logger.info(f"Function '{function_name}' is NEW or CHANGED. Writing to DB.")
+                    batch = get_batch_manager()
+                    vectorizer = get_vectorizer()
+                    vector_to_add = None
+
+                    if vectorizer and search_description:
+                        try:
+                            vector_to_add = vectorizer.embed(search_description)
+                        except Exception as e:
+                            logger.warning(f"Failed to vectorize '{function_name}': {e}")
+
+                    batch.add_object(
+                        collection=settings.COLLECTION_NAME,
+                        properties=static_properties,
+                        uuid=func_uuid,
+                        vector=vector_to_add
+                    )
+                    function_cache_manager.update_cache(func_uuid, current_content_hash)
+
+        except Exception as e:
+            logger.error("Error in @vectorize setup for '%s': %s", func.__name__, e)
+
+        # --- Wrapper Logic ---
+
+        if is_async_func:
+            @trace_root()
+            @trace_span(attributes_to_capture=final_attributes, capture_return_value=capture_return_value)
+            @wraps(func)
+            async def inner_wrapper(*args, **kwargs):
+                # Remove injected tags from kwargs before calling original func
+                clean_kwargs = {k: v for k, v in kwargs.items() if
+                                k not in valid_execution_tags and k != 'function_uuid' and k != 'exec_source'}
+                return await func(*args, **clean_kwargs)
+
+            @wraps(func)
+            async def outer_wrapper(*args, **kwargs):
+                if semantic_cache:
+                    cached = _check_and_return_cached_result(func, args, kwargs, function_name, cache_threshold, True)
+                    if cached is not None: return cached
+
+                full_kwargs = kwargs.copy()
+                full_kwargs.update(valid_execution_tags)
+                full_kwargs['function_uuid'] = func_uuid
+                full_kwargs['exec_source'] = execution_source_context.get()
+                return await inner_wrapper(*args, **full_kwargs)
+
+            outer_wrapper._is_vectorized = True
+            return outer_wrapper
+
+        else:  # Sync wrapper
+            @trace_root()
+            @trace_span(attributes_to_capture=final_attributes, capture_return_value=capture_return_value)
+            @wraps(func)
+            def inner_wrapper(*args, **kwargs):
+                clean_kwargs = {k: v for k, v in kwargs.items() if
+                                k not in valid_execution_tags and k != 'function_uuid' and k != 'exec_source'}
+                return func(*args, **clean_kwargs)
+
+            @wraps(func)
+            def outer_wrapper(*args, **kwargs):
+                if semantic_cache:
+                    cached = _check_and_return_cached_result(func, args, kwargs, function_name, cache_threshold, False)
+                    if cached is not None: return cached
+
+                full_kwargs = kwargs.copy()
+                full_kwargs.update(valid_execution_tags)
+                full_kwargs['function_uuid'] = func_uuid
+                full_kwargs['exec_source'] = execution_source_context.get()
+                return inner_wrapper(*args, **full_kwargs)
+
+            outer_wrapper._is_vectorized = True
+            return outer_wrapper
+
+    return decorator

vectorwave/core/generator.py

@@ -0,0 +1,140 @@
+import logging
+import json
+from typing import Optional, Dict, Any
+
+from ..utils.function_cache import function_cache_manager
+from ..models.db_config import get_weaviate_settings
+from ..batch.batch import get_batch_manager
+from ..vectorizer.factory import get_vectorizer
+from .decorator import PENDING_FUNCTIONS
+from .llm.factory import get_llm_client
+
+logger = logging.getLogger(__name__)
+
+try:
+    from openai import OpenAI
+except ImportError:
+    OpenAI = None
+
+
+def generate_metadata_via_llm(source_code: str, func_name: str) -> Optional[Dict[str, str]]:
+    """Call LLM to generate description and narrative from source code."""
+    settings = get_weaviate_settings()
+    client = get_llm_client()
+    if not client:
+        return None
+
+    prompt = f"""
+    Analyze the Python function below and generate a JSON object with two keys.
+    Ensure the values are **single strings**, not nested objects.
+
+    1. "search_description": A concise summary of what this function does (for vector search).
+    2. "sequence_narrative": A brief explanation of the context, inputs, and outputs as a narrative text.
+    
+    Function Name: {func_name}
+    Code:
+    ```python
+    {source_code}
+    ```
+    """
+
+    try:
+        # Refactored to use BaseLLMClient interface
+        response_text = client.create_chat_completion(
+            model="gpt-4o-mini",
+            messages=[
+                {"role": "system", "content": "You are a technical documentation assistant. Output only JSON."},
+                {"role": "user", "content": prompt}
+            ],
+            temperature=0.0,
+            response_format={"type": "json_object"},
+            category="auto_doc"
+        )
+
+        if response_text:
+            return json.loads(response_text)
+        return None
+
+    except Exception as e:
+        logger.error(f"LLM generation failed for '{func_name}': {e}")
+        return None
+
+
+def generate_and_register_metadata():
+    """
+    [Entry Point] Processes all functions in PENDING_FUNCTIONS.
+    """
+    if not PENDING_FUNCTIONS:
+        logger.info("No pending functions for auto-generation.")
+        return
+
+    logger.info(f"🚀 Processing {len(PENDING_FUNCTIONS)} functions for auto-documentation...")
+
+    settings = get_weaviate_settings()
+    batch = get_batch_manager()
+    vectorizer = get_vectorizer()
+
+    processed_count = 0
+
+    for item in PENDING_FUNCTIONS:
+        func_name = item["func_name"]
+        func_uuid = item["func_uuid"]
+        func_identifier = item["func_identifier"]
+        static_props = item["static_properties"]
+
+        current_hash = function_cache_manager.calculate_content_hash(func_identifier, static_props)
+
+        cached_meta = function_cache_manager.get_cached_metadata(func_uuid, current_hash)
+
+        final_desc = ""
+        final_narr = ""
+
+        if cached_meta:
+            logger.info(f"✅ [Cache Hit] Loaded metadata for '{func_name}'.")
+            final_desc = cached_meta.get("search_description")
+            final_narr = cached_meta.get("sequence_narrative")
+        else:
+            logger.info(f"🤖 [Auto-Gen] Generating metadata for '{func_name}' via LLM...")
+            generated = generate_metadata_via_llm(static_props["source_code"], func_name)
+
+            if generated:
+                final_desc = generated.get("search_description", "")
+                final_narr = generated.get("sequence_narrative", "")
+
+                if not isinstance(final_desc, str):
+                    final_desc = json.dumps(final_desc, ensure_ascii=False)
+                if not isinstance(final_narr, str):
+                    final_narr = json.dumps(final_narr, ensure_ascii=False)
+
+                # Update Cache with new metadata
+                function_cache_manager.update_cache_with_metadata(
+                    func_uuid, current_hash,
+                    {"search_description": final_desc, "sequence_narrative": final_narr}
+                )
+            else:
+                logger.warning(f"⚠️ Skipping registration for '{func_name}' due to generation failure.")
+                continue
+
+        # 3. Update Properties
+        static_props["search_description"] = final_desc
+        static_props["sequence_narrative"] = final_narr
+
+        # 4. Vectorize the Description
+        vector_to_add = None
+        if vectorizer and final_desc:
+            try:
+                vector_to_add = vectorizer.embed(final_desc)
+            except Exception as e:
+                logger.warning(f"Vectorization failed for '{func_name}': {e}")
+
+        # 5. Register to DB
+        batch.add_object(
+            collection=settings.COLLECTION_NAME,
+            properties=static_props,
+            uuid=func_uuid,
+            vector=vector_to_add
+        )
+        processed_count += 1
+
+    PENDING_FUNCTIONS.clear()
+    logger.info(f"✨ Auto-generation complete. Registered {processed_count} functions.")

vectorwave/core/llm/base.py

@@ -0,0 +1,47 @@
+from abc import ABC, abstractmethod
+from typing import List, Dict, Optional
+
+class BaseLLMClient(ABC):
+    """
+    Abstract interface that all LLM Providers (OpenAI, Anthropic, etc.) must implement.
+    Implementations must handle internal token usage logging.
+    """
+
+    @abstractmethod
+    def create_embedding(self, text: str, model: str, category: str = "default") -> Optional[List[float]]:
+        """
+        Generates text embeddings.
+
+        Args:
+            text: The text to embed.
+            model: The name of the model to use.
+            category: Category for aggregating token usage (e.g., 'execution_log', 'auto_doc').
+
+        Returns:
+            The generated list of embedding vectors (None on failure).
+        """
+        pass
+
+    @abstractmethod
+    def create_chat_completion(
+            self,
+            messages: List[Dict],
+            model: str,
+            temperature: float = 0.1,
+            response_format: Optional[Dict] = None,
+            category: str = "default"
+    ) -> Optional[str]:
+        """
+        Generates a chat completion (response).
+
+        Args:
+            messages: List of conversation messages [{"role": "user", "content": "..."}].
+            model: The name of the model to use.
+            temperature: Parameter for controlling generation diversity.
+            response_format: Response format (e.g., {"type": "json_object"}).
+            category: Category for aggregating token usage (e.g., 'execution_log', 'auto_doc').
+
+        Returns:
+            The generated text response (None on failure).
+        """
+        pass