vectorwave 0.1.3__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vectorwave-0.1.3/src/vectorwave.egg-info → vectorwave-0.1.4}/PKG-INFO +112 -47
- {vectorwave-0.1.3 → vectorwave-0.1.4}/Readme.md +110 -46
- {vectorwave-0.1.3 → vectorwave-0.1.4}/pyproject.toml +3 -2
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/database/test_db.py +2 -2
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/models/test_db_config.py +13 -14
- vectorwave-0.1.4/src/tests/monitoring/test_async_trace.py +259 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/monitoring/test_tracer.py +56 -2
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/core/decorator.py +64 -39
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/database/db.py +7 -3
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/models/db_config.py +20 -1
- vectorwave-0.1.4/src/vectorwave/monitoring/tracer.py +246 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4/src/vectorwave.egg-info}/PKG-INFO +112 -47
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave.egg-info/SOURCES.txt +1 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave.egg-info/requires.txt +1 -0
- vectorwave-0.1.3/src/vectorwave/monitoring/tracer.py +0 -131
- {vectorwave-0.1.3 → vectorwave-0.1.4}/LICENSE +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/MANIFEST.in +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/NOTICE +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/setup.cfg +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/batch/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/batch/test_batch.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/core/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/core/test_decorator.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/database/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/database/test_db_search.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/exception/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/models/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/monitoring/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/prediction/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/tests/vectorizer/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/batch/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/batch/batch.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/core/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/core/core.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/database/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/database/db_search.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/exception/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/exception/exceptions.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/models/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/monitoring/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/monitoring/monitoring.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/prediction/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/prediction/predictor.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/vectorizer/__init__.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/vectorizer/base.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/vectorizer/factory.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/vectorizer/huggingface_vectorizer.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave/vectorizer/openai_vectorizer.py +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave.egg-info/dependency_links.txt +0 -0
- {vectorwave-0.1.3 → vectorwave-0.1.4}/src/vectorwave.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vectorwave
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4
|
|
4
4
|
Summary: VectorWave: Seamless Auto-Vectorization Framework
|
|
5
5
|
Author-email: junyeonggim <junyeonggim5@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -19,34 +19,36 @@ License-File: LICENSE
|
|
|
19
19
|
License-File: NOTICE
|
|
20
20
|
Requires-Dist: weaviate-client>=4.0.0
|
|
21
21
|
Requires-Dist: pydantic-settings>=2.0.0
|
|
22
|
+
Requires-Dist: sentence-transformers
|
|
22
23
|
Dynamic: license-file
|
|
23
24
|
|
|
24
25
|
|
|
26
|
+
|
|
25
27
|
# VectorWave: Seamless Auto-Vectorization Framework
|
|
26
28
|
|
|
27
|
-
[](https://
|
|
29
|
+
[](https://opensource.org/licenses/MIT)
|
|
28
30
|
|
|
29
31
|
## 🌟 Overview
|
|
30
32
|
|
|
31
|
-
**VectorWave** is an innovative framework that uses
|
|
33
|
+
**VectorWave** is an innovative framework that uses **decorators** to automatically save and manage the output of Python functions/methods in a **Vector Database (Vector DB)**. Developers can convert function outputs into intelligent vector data with just a single line of code (`@vectorize`), without worrying about the complex processes of data collection, embedding generation, and Vector DB storage.
|
|
32
34
|
|
|
33
|
-
|
|
35
|
+
-----
|
|
34
36
|
|
|
35
37
|
## ✨ Features
|
|
36
38
|
|
|
37
39
|
* **`@vectorize` Decorator:**
|
|
38
|
-
1. **Static Data Collection:**
|
|
39
|
-
2. **Dynamic Data Logging:**
|
|
40
|
-
* **Distributed Tracing:**
|
|
41
|
-
* **Search Interface:** Provides `search_functions`
|
|
40
|
+
1. **Static Data Collection:** Upon script load, the function's source code, docstring, and metadata are saved once to the `VectorWaveFunctions` collection.
|
|
41
|
+
2. **Dynamic Data Logging:** Each time the function is called, its execution time, success/failure status, error logs, and "dynamic tags" are recorded in the `VectorWaveExecutions` collection.
|
|
42
|
+
* **Distributed Tracing:** Combines `@vectorize` and `@trace_span` decorators to bundle the execution of complex, multi-step workflows under a single **`trace_id`** for analysis.
|
|
43
|
+
* **Search Interface:** Provides `search_functions` and `search_executions` to query the stored vector data (function definitions) and logs (execution history), facilitating the construction of RAG and monitoring systems.
|
|
42
44
|
|
|
43
|
-
|
|
45
|
+
-----
|
|
44
46
|
|
|
45
47
|
## 🚀 Usage
|
|
46
48
|
|
|
47
|
-
VectorWave consists of
|
|
49
|
+
VectorWave consists of "storage" via decorators and "retrieval" via functions, and now includes **execution flow tracing**.
|
|
48
50
|
|
|
49
|
-
### 1
|
|
51
|
+
### 1\. (Required) Database Initialization and Setup
|
|
50
52
|
|
|
51
53
|
```python
|
|
52
54
|
import time
|
|
@@ -56,50 +58,50 @@ from vectorwave import (
|
|
|
56
58
|
search_functions,
|
|
57
59
|
search_executions
|
|
58
60
|
)
|
|
59
|
-
# [
|
|
61
|
+
# [New] Import trace_span separately for distributed tracing.
|
|
60
62
|
from vectorwave.monitoring.tracer import trace_span
|
|
61
63
|
|
|
62
|
-
#
|
|
64
|
+
# Needs to be called only once at script startup.
|
|
63
65
|
try:
|
|
64
66
|
client = initialize_database()
|
|
65
|
-
print("VectorWave DB
|
|
67
|
+
print("VectorWave DB initialization successful.")
|
|
66
68
|
except Exception as e:
|
|
67
69
|
print(f"DB initialization failed: {e}")
|
|
68
70
|
exit()
|
|
69
|
-
|
|
71
|
+
```
|
|
70
72
|
|
|
71
|
-
### 2\. [
|
|
73
|
+
### 2\. [Storage] Using `@vectorize` and Distributed Tracing
|
|
72
74
|
|
|
73
|
-
|
|
75
|
+
`@vectorize` acts as the **Root** of the trace, and applying `@trace_span` to internal functions bundles the workflow execution under a **single `trace_id`**.
|
|
74
76
|
|
|
75
77
|
```python
|
|
76
78
|
# --- Child Span Function: Captures arguments ---
|
|
77
79
|
@trace_span(attributes_to_capture=['user_id', 'amount'])
|
|
78
80
|
def step_1_validate_payment(user_id: str, amount: int):
|
|
79
|
-
"""(Span)
|
|
81
|
+
"""(Span) Validates payment. Logs user_id and amount."""
|
|
80
82
|
print(f" [SPAN 1] Validating payment for {user_id}...")
|
|
81
83
|
time.sleep(0.1)
|
|
82
84
|
return True
|
|
83
85
|
|
|
84
86
|
@trace_span(attributes_to_capture=['user_id', 'receipt_id'])
|
|
85
87
|
def step_2_send_receipt(user_id: str, receipt_id: str):
|
|
86
|
-
"""(Span) Sends
|
|
88
|
+
"""(Span) Sends receipt."""
|
|
87
89
|
print(f" [SPAN 2] Sending receipt {receipt_id}...")
|
|
88
90
|
time.sleep(0.2)
|
|
89
91
|
|
|
90
92
|
|
|
91
|
-
# --- Root Function (@trace_root
|
|
93
|
+
# --- Root Function (acts as @trace_root) ---
|
|
92
94
|
@vectorize(
|
|
93
|
-
search_description="
|
|
94
|
-
sequence_narrative="
|
|
95
|
-
team="billing", #
|
|
96
|
-
priority=1 #
|
|
95
|
+
search_description="Processes a user payment and returns a receipt.",
|
|
96
|
+
sequence_narrative="After payment is complete, a receipt is sent via email.",
|
|
97
|
+
team="billing", # ⬅️ Custom tag (logged on all executions)
|
|
98
|
+
priority=1 # ⬅️ Custom tag (execution importance)
|
|
97
99
|
)
|
|
98
100
|
def process_payment(user_id: str, amount: int):
|
|
99
101
|
"""(Root Span) Executes the user payment workflow."""
|
|
100
102
|
print(f" [ROOT EXEC] process_payment: Starting workflow for {user_id}...")
|
|
101
103
|
|
|
102
|
-
# When
|
|
104
|
+
# When child functions are called, the same trace_id is automatically inherited via ContextVar.
|
|
103
105
|
step_1_validate_payment(user_id=user_id, amount=amount)
|
|
104
106
|
|
|
105
107
|
receipt_id = f"receipt_{user_id}_{amount}"
|
|
@@ -108,20 +110,20 @@ def process_payment(user_id: str, amount: int):
|
|
|
108
110
|
print(f" [ROOT DONE] process_payment")
|
|
109
111
|
return {"status": "success", "receipt_id": receipt_id}
|
|
110
112
|
|
|
111
|
-
# ---
|
|
113
|
+
# --- Function Execution ---
|
|
112
114
|
print("Now calling 'process_payment'...")
|
|
113
|
-
# This single call
|
|
114
|
-
# all
|
|
115
|
+
# This single call will record a total of 3 execution logs (spans) in the DB,
|
|
116
|
+
# and all three logs will be tied to a single 'trace_id'.
|
|
115
117
|
process_payment("user_789", 5000)
|
|
116
118
|
```
|
|
117
119
|
|
|
118
|
-
### 3\. [
|
|
120
|
+
### 3\. [Retrieval ①] Search Function Definitions (for RAG)
|
|
119
121
|
|
|
120
122
|
```python
|
|
121
|
-
# Search for functions related to 'payment' using natural language (vector
|
|
122
|
-
print("\n--- Searching for 'payment' functions ---")
|
|
123
|
+
# Search for functions related to 'payment' using natural language (vector).
|
|
124
|
+
print("\n--- Searching for 'payment' related functions ---")
|
|
123
125
|
payment_funcs = search_functions(
|
|
124
|
-
query="
|
|
126
|
+
query="User payment processing feature",
|
|
125
127
|
limit=3
|
|
126
128
|
)
|
|
127
129
|
for func in payment_funcs:
|
|
@@ -130,9 +132,9 @@ for func in payment_funcs:
|
|
|
130
132
|
print(f" - Similarity (Distance): {func['metadata'].distance:.4f}")
|
|
131
133
|
```
|
|
132
134
|
|
|
133
|
-
### 4\. [
|
|
135
|
+
### 4\. [Retrieval ②] Search Execution Logs (for Monitoring & Tracing)
|
|
134
136
|
|
|
135
|
-
|
|
137
|
+
`search_executions` can now retrieve all related execution logs (spans) based on a `trace_id`.
|
|
136
138
|
|
|
137
139
|
```python
|
|
138
140
|
# 1. Find the Trace ID of a specific workflow (process_payment).
|
|
@@ -144,24 +146,25 @@ latest_payment_span = search_executions(
|
|
|
144
146
|
)
|
|
145
147
|
trace_id = latest_payment_span[0]["trace_id"]
|
|
146
148
|
|
|
147
|
-
# 2.
|
|
149
|
+
# 2. Retrieve all spans belonging to that Trace ID in chronological order.
|
|
148
150
|
print(f"\n--- Full Trace for ID ({trace_id[:8]}...) ---")
|
|
149
151
|
trace_spans = search_executions(
|
|
150
152
|
limit=10,
|
|
151
153
|
filters={"trace_id": trace_id},
|
|
152
154
|
sort_by="timestamp_utc",
|
|
153
|
-
sort_ascending=True #
|
|
155
|
+
sort_ascending=True # Sort ascending to analyze workflow
|
|
154
156
|
)
|
|
155
157
|
|
|
156
158
|
for i, span in enumerate(trace_spans):
|
|
157
159
|
print(f" - [Span {i+1}] {span['function_name']} ({span['duration_ms']:.2f}ms)")
|
|
158
|
-
# Captured arguments (user_id, amount, etc.)
|
|
160
|
+
# Captured arguments (user_id, amount, etc.) from child spans will also be visible.
|
|
159
161
|
|
|
160
|
-
#
|
|
162
|
+
# Expected Output:
|
|
161
163
|
# - [Span 1] step_1_validate_payment (100.81ms)
|
|
162
164
|
# - [Span 2] step_2_send_receipt (202.06ms)
|
|
163
165
|
# - [Span 3] process_payment (333.18ms)
|
|
164
166
|
```
|
|
167
|
+
|
|
165
168
|
-----
|
|
166
169
|
|
|
167
170
|
## ⚙️ Configuration
|
|
@@ -176,10 +179,10 @@ You can select the text vectorization method via the `VECTORIZER` environment va
|
|
|
176
179
|
|
|
177
180
|
| `VECTORIZER` Setting | Description | Required Additional Settings |
|
|
178
181
|
| :--- | :--- | :--- |
|
|
179
|
-
| **`huggingface`** | (Default Recommended) Uses the `sentence-transformers` library to vectorize on your local CPU. No API key is needed
|
|
180
|
-
| **`openai_client`** | (High-Performance) Uses the OpenAI Python client to vectorize with
|
|
181
|
-
| **`weaviate_module`** | (Docker Delegate) Delegates
|
|
182
|
-
| **`none`** | Disables vectorization. Data
|
|
182
|
+
| **`huggingface`** | (Default Recommended) Uses the `sentence-transformers` library to vectorize on your local CPU. No API key is needed. | `HF_MODEL_NAME` (e.g., "sentence-transformers/all-MiniLM-L6-v2") |
|
|
183
|
+
| **`openai_client`** | (High-Performance) Uses the OpenAI Python client to vectorize with models like `text-embedding-3-small`. | `OPENAI_API_KEY` (A valid OpenAI API key) |
|
|
184
|
+
| **`weaviate_module`** | (Docker Delegate) Delegates vectorization to Weaviate's built-in module (e.g., `text2vec-openai`). | `WEAVIATE_VECTORIZER_MODULE`, `OPENAI_API_KEY` |
|
|
185
|
+
| **`none`** | Disables vectorization. Data is stored without vectors. | None |
|
|
183
186
|
|
|
184
187
|
-----
|
|
185
188
|
|
|
@@ -207,6 +210,7 @@ OPENAI_API_KEY=sk-...
|
|
|
207
210
|
|
|
208
211
|
# --- [Advanced] Custom Properties ---
|
|
209
212
|
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
213
|
+
FAILURE_MAPPING_FILE_PATH=.vectorwave_errors.json
|
|
210
214
|
RUN_ID=test-run-001
|
|
211
215
|
```
|
|
212
216
|
|
|
@@ -232,6 +236,7 @@ HF_MODEL_NAME=...
|
|
|
232
236
|
|
|
233
237
|
# --- [Advanced] Custom Properties ---
|
|
234
238
|
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
239
|
+
FAILURE_MAPPING_FILE_PATH=.vectorwave_errors.json
|
|
235
240
|
RUN_ID=test-run-001
|
|
236
241
|
```
|
|
237
242
|
|
|
@@ -256,11 +261,72 @@ OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
|
256
261
|
|
|
257
262
|
# --- [Advanced] Custom Properties ---
|
|
258
263
|
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
264
|
+
FAILURE_MAPPING_FILE_PATH=.vectorwave_errors.json
|
|
259
265
|
RUN_ID=test-run-001
|
|
260
266
|
```
|
|
261
267
|
|
|
262
268
|
-----
|
|
263
269
|
|
|
270
|
+
### 🚀 Advanced Failure Tracing (Error Code)
|
|
271
|
+
|
|
272
|
+
This enhances `VectorWaveExecutions` logs beyond a simple `status: "ERROR"`. An `error_code` property is added to the schema for granular failure analysis.
|
|
273
|
+
|
|
274
|
+
When a function wrapped by `@vectorize` or `@trace_span` fails, the `error_code` is automatically determined based on three priorities:
|
|
275
|
+
|
|
276
|
+
1. **Custom Exception Attribute (Priority 1):**
|
|
277
|
+
The most specific method. If the raised exception object `e` has an `e.error_code` attribute, its value is used.
|
|
278
|
+
|
|
279
|
+
```python
|
|
280
|
+
class PaymentError(Exception):
|
|
281
|
+
def __init__(self, message, error_code):
|
|
282
|
+
super().__init__(message)
|
|
283
|
+
self.error_code = error_code # ⬅️ This attribute is detected.
|
|
284
|
+
|
|
285
|
+
@vectorize(...)
|
|
286
|
+
def process_payment(amount):
|
|
287
|
+
if amount < 0:
|
|
288
|
+
raise PaymentError("Amount < 0", error_code="PAYMENT_NEGATIVE_AMOUNT")
|
|
289
|
+
|
|
290
|
+
# DB Log on execution: { "status": "ERROR", "error_code": "PAYMENT_NEGATIVE_AMOUNT" }
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
2. **Global Mapping File (Priority 2):**
|
|
294
|
+
Centrally manage common exceptions. VectorWave loads a JSON file specified by `FAILURE_MAPPING_FILE_PATH` in your `.env` (default: `.vectorwave_errors.json`) and maps the exception class name to a code.
|
|
295
|
+
|
|
296
|
+
**`.vectorwave_errors.json` Example:**
|
|
297
|
+
|
|
298
|
+
```json
|
|
299
|
+
{
|
|
300
|
+
"ValueError": "INVALID_INPUT",
|
|
301
|
+
"KeyError": "CONFIG_MISSING",
|
|
302
|
+
"TypeError": "INVALID_INPUT"
|
|
303
|
+
}
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
```python
|
|
307
|
+
@vectorize(...)
|
|
308
|
+
def get_config(key):
|
|
309
|
+
return os.environ[key] # ⬅️ Raises KeyError
|
|
310
|
+
|
|
311
|
+
# DB Log on execution: { "status": "ERROR", "error_code": "CONFIG_MISSING" }
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
3. **Default (Priority 3):**
|
|
315
|
+
If neither of the above applies, the exception's class name (e.g., `"ZeroDivisionError"`) is stored as the default `error_code`.
|
|
316
|
+
|
|
317
|
+
**[Usage] Searching for Failures:**
|
|
318
|
+
You can now filter for specific failure types using `search_executions`.
|
|
319
|
+
|
|
320
|
+
```python
|
|
321
|
+
# Find all failure logs categorized as "INVALID_INPUT"
|
|
322
|
+
invalid_logs = search_executions(
|
|
323
|
+
filters={"error_code": "INVALID_INPUT"},
|
|
324
|
+
limit=10
|
|
325
|
+
)
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
-----
|
|
329
|
+
|
|
264
330
|
### Custom Properties and Dynamic Execution Tagging
|
|
265
331
|
|
|
266
332
|
VectorWave can store user-defined metadata in addition to static data (function definitions) and dynamic data (execution logs). This works in two steps.
|
|
@@ -304,7 +370,7 @@ When a function is executed, VectorWave adds tags to the `VectorWaveExecutions`
|
|
|
304
370
|
VectorWave looks for environment variables matching the **UPPERCASE name** of the keys defined in Step 1 (e.g., `RUN_ID`, `EXPERIMENT_ID`). Found values are loaded as `global_custom_values` and added to *all* execution logs. Ideal for run-wide metadata.
|
|
305
371
|
|
|
306
372
|
**2. Function-Specific Tags (Decorator)**
|
|
307
|
-
You can pass tags as keyword arguments (`**execution_tags`) directly to the `@vectorize` decorator.
|
|
373
|
+
You can pass tags as keyword arguments (`**execution_tags`) directly to the `@vectorize` decorator. This is ideal for function-specific metadata.
|
|
308
374
|
|
|
309
375
|
```python
|
|
310
376
|
# --- .env file ---
|
|
@@ -331,7 +397,7 @@ def other_function():
|
|
|
331
397
|
|
|
332
398
|
**Tag Merging and Validation Rules**
|
|
333
399
|
|
|
334
|
-
1. **Validation (Important):** Tags (global or function-specific) will **only** be saved to Weaviate if their key (e.g., `run_id`, `team`, `priority`) was first defined in the `.weaviate_properties` file (Step 1). Tags not defined in the schema are **ignored**, and a warning is
|
|
400
|
+
1. **Validation (Important):** Tags (global or function-specific) will **only** be saved to Weaviate if their key (e.g., `run_id`, `team`, `priority`) was first defined in the `.weaviate_properties` file (Step 1). Tags not defined in the schema are **ignored**, and a warning is logged at startup.
|
|
335
401
|
|
|
336
402
|
2. **Priority (Override):** If a tag key is defined in both places (e.g., global `RUN_ID` in `.env` and `run_id="override-xyz"` in the decorator), the **function-specific tag from the decorator always wins**.
|
|
337
403
|
|
|
@@ -344,9 +410,8 @@ def other_function():
|
|
|
344
410
|
|
|
345
411
|
## 🤝 Contributing
|
|
346
412
|
|
|
347
|
-
|
|
413
|
+
Bug reports, feature requests, and code contributions are all welcome. For details, please see [CONTRIBUTING.md](https://www.google.com/search?q=httpsS://www.google.com/search%3Fq%3DCONTRIBUTING.md).
|
|
348
414
|
|
|
349
415
|
## 📜 License
|
|
350
416
|
|
|
351
|
-
This project is distributed under the MIT License. See the [LICENSE](https://www.google.com/search?q=
|
|
352
|
-
|
|
417
|
+
This project is distributed under the MIT License. See the [LICENSE](https://www.google.com/search?q=LICENSE) file for details.
|
|
@@ -1,29 +1,30 @@
|
|
|
1
1
|
|
|
2
|
+
|
|
2
3
|
# VectorWave: Seamless Auto-Vectorization Framework
|
|
3
4
|
|
|
4
|
-
[](https://
|
|
5
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
6
|
|
|
6
7
|
## 🌟 Overview
|
|
7
8
|
|
|
8
|
-
**VectorWave** is an innovative framework that uses
|
|
9
|
+
**VectorWave** is an innovative framework that uses **decorators** to automatically save and manage the output of Python functions/methods in a **Vector Database (Vector DB)**. Developers can convert function outputs into intelligent vector data with just a single line of code (`@vectorize`), without worrying about the complex processes of data collection, embedding generation, and Vector DB storage.
|
|
9
10
|
|
|
10
|
-
|
|
11
|
+
-----
|
|
11
12
|
|
|
12
13
|
## ✨ Features
|
|
13
14
|
|
|
14
15
|
* **`@vectorize` Decorator:**
|
|
15
|
-
1. **Static Data Collection:**
|
|
16
|
-
2. **Dynamic Data Logging:**
|
|
17
|
-
* **Distributed Tracing:**
|
|
18
|
-
* **Search Interface:** Provides `search_functions`
|
|
16
|
+
1. **Static Data Collection:** Upon script load, the function's source code, docstring, and metadata are saved once to the `VectorWaveFunctions` collection.
|
|
17
|
+
2. **Dynamic Data Logging:** Each time the function is called, its execution time, success/failure status, error logs, and "dynamic tags" are recorded in the `VectorWaveExecutions` collection.
|
|
18
|
+
* **Distributed Tracing:** Combines `@vectorize` and `@trace_span` decorators to bundle the execution of complex, multi-step workflows under a single **`trace_id`** for analysis.
|
|
19
|
+
* **Search Interface:** Provides `search_functions` and `search_executions` to query the stored vector data (function definitions) and logs (execution history), facilitating the construction of RAG and monitoring systems.
|
|
19
20
|
|
|
20
|
-
|
|
21
|
+
-----
|
|
21
22
|
|
|
22
23
|
## 🚀 Usage
|
|
23
24
|
|
|
24
|
-
VectorWave consists of
|
|
25
|
+
VectorWave consists of "storage" via decorators and "retrieval" via functions, and now includes **execution flow tracing**.
|
|
25
26
|
|
|
26
|
-
### 1
|
|
27
|
+
### 1\. (Required) Database Initialization and Setup
|
|
27
28
|
|
|
28
29
|
```python
|
|
29
30
|
import time
|
|
@@ -33,50 +34,50 @@ from vectorwave import (
|
|
|
33
34
|
search_functions,
|
|
34
35
|
search_executions
|
|
35
36
|
)
|
|
36
|
-
# [
|
|
37
|
+
# [New] Import trace_span separately for distributed tracing.
|
|
37
38
|
from vectorwave.monitoring.tracer import trace_span
|
|
38
39
|
|
|
39
|
-
#
|
|
40
|
+
# Needs to be called only once at script startup.
|
|
40
41
|
try:
|
|
41
42
|
client = initialize_database()
|
|
42
|
-
print("VectorWave DB
|
|
43
|
+
print("VectorWave DB initialization successful.")
|
|
43
44
|
except Exception as e:
|
|
44
45
|
print(f"DB initialization failed: {e}")
|
|
45
46
|
exit()
|
|
46
|
-
|
|
47
|
+
```
|
|
47
48
|
|
|
48
|
-
### 2\. [
|
|
49
|
+
### 2\. [Storage] Using `@vectorize` and Distributed Tracing
|
|
49
50
|
|
|
50
|
-
|
|
51
|
+
`@vectorize` acts as the **Root** of the trace, and applying `@trace_span` to internal functions bundles the workflow execution under a **single `trace_id`**.
|
|
51
52
|
|
|
52
53
|
```python
|
|
53
54
|
# --- Child Span Function: Captures arguments ---
|
|
54
55
|
@trace_span(attributes_to_capture=['user_id', 'amount'])
|
|
55
56
|
def step_1_validate_payment(user_id: str, amount: int):
|
|
56
|
-
"""(Span)
|
|
57
|
+
"""(Span) Validates payment. Logs user_id and amount."""
|
|
57
58
|
print(f" [SPAN 1] Validating payment for {user_id}...")
|
|
58
59
|
time.sleep(0.1)
|
|
59
60
|
return True
|
|
60
61
|
|
|
61
62
|
@trace_span(attributes_to_capture=['user_id', 'receipt_id'])
|
|
62
63
|
def step_2_send_receipt(user_id: str, receipt_id: str):
|
|
63
|
-
"""(Span) Sends
|
|
64
|
+
"""(Span) Sends receipt."""
|
|
64
65
|
print(f" [SPAN 2] Sending receipt {receipt_id}...")
|
|
65
66
|
time.sleep(0.2)
|
|
66
67
|
|
|
67
68
|
|
|
68
|
-
# --- Root Function (@trace_root
|
|
69
|
+
# --- Root Function (acts as @trace_root) ---
|
|
69
70
|
@vectorize(
|
|
70
|
-
search_description="
|
|
71
|
-
sequence_narrative="
|
|
72
|
-
team="billing", #
|
|
73
|
-
priority=1 #
|
|
71
|
+
search_description="Processes a user payment and returns a receipt.",
|
|
72
|
+
sequence_narrative="After payment is complete, a receipt is sent via email.",
|
|
73
|
+
team="billing", # ⬅️ Custom tag (logged on all executions)
|
|
74
|
+
priority=1 # ⬅️ Custom tag (execution importance)
|
|
74
75
|
)
|
|
75
76
|
def process_payment(user_id: str, amount: int):
|
|
76
77
|
"""(Root Span) Executes the user payment workflow."""
|
|
77
78
|
print(f" [ROOT EXEC] process_payment: Starting workflow for {user_id}...")
|
|
78
79
|
|
|
79
|
-
# When
|
|
80
|
+
# When child functions are called, the same trace_id is automatically inherited via ContextVar.
|
|
80
81
|
step_1_validate_payment(user_id=user_id, amount=amount)
|
|
81
82
|
|
|
82
83
|
receipt_id = f"receipt_{user_id}_{amount}"
|
|
@@ -85,20 +86,20 @@ def process_payment(user_id: str, amount: int):
|
|
|
85
86
|
print(f" [ROOT DONE] process_payment")
|
|
86
87
|
return {"status": "success", "receipt_id": receipt_id}
|
|
87
88
|
|
|
88
|
-
# ---
|
|
89
|
+
# --- Function Execution ---
|
|
89
90
|
print("Now calling 'process_payment'...")
|
|
90
|
-
# This single call
|
|
91
|
-
# all
|
|
91
|
+
# This single call will record a total of 3 execution logs (spans) in the DB,
|
|
92
|
+
# and all three logs will be tied to a single 'trace_id'.
|
|
92
93
|
process_payment("user_789", 5000)
|
|
93
94
|
```
|
|
94
95
|
|
|
95
|
-
### 3\. [
|
|
96
|
+
### 3\. [Retrieval ①] Search Function Definitions (for RAG)
|
|
96
97
|
|
|
97
98
|
```python
|
|
98
|
-
# Search for functions related to 'payment' using natural language (vector
|
|
99
|
-
print("\n--- Searching for 'payment' functions ---")
|
|
99
|
+
# Search for functions related to 'payment' using natural language (vector).
|
|
100
|
+
print("\n--- Searching for 'payment' related functions ---")
|
|
100
101
|
payment_funcs = search_functions(
|
|
101
|
-
query="
|
|
102
|
+
query="User payment processing feature",
|
|
102
103
|
limit=3
|
|
103
104
|
)
|
|
104
105
|
for func in payment_funcs:
|
|
@@ -107,9 +108,9 @@ for func in payment_funcs:
|
|
|
107
108
|
print(f" - Similarity (Distance): {func['metadata'].distance:.4f}")
|
|
108
109
|
```
|
|
109
110
|
|
|
110
|
-
### 4\. [
|
|
111
|
+
### 4\. [Retrieval ②] Search Execution Logs (for Monitoring & Tracing)
|
|
111
112
|
|
|
112
|
-
|
|
113
|
+
`search_executions` can now retrieve all related execution logs (spans) based on a `trace_id`.
|
|
113
114
|
|
|
114
115
|
```python
|
|
115
116
|
# 1. Find the Trace ID of a specific workflow (process_payment).
|
|
@@ -121,24 +122,25 @@ latest_payment_span = search_executions(
|
|
|
121
122
|
)
|
|
122
123
|
trace_id = latest_payment_span[0]["trace_id"]
|
|
123
124
|
|
|
124
|
-
# 2.
|
|
125
|
+
# 2. Retrieve all spans belonging to that Trace ID in chronological order.
|
|
125
126
|
print(f"\n--- Full Trace for ID ({trace_id[:8]}...) ---")
|
|
126
127
|
trace_spans = search_executions(
|
|
127
128
|
limit=10,
|
|
128
129
|
filters={"trace_id": trace_id},
|
|
129
130
|
sort_by="timestamp_utc",
|
|
130
|
-
sort_ascending=True #
|
|
131
|
+
sort_ascending=True # Sort ascending to analyze workflow
|
|
131
132
|
)
|
|
132
133
|
|
|
133
134
|
for i, span in enumerate(trace_spans):
|
|
134
135
|
print(f" - [Span {i+1}] {span['function_name']} ({span['duration_ms']:.2f}ms)")
|
|
135
|
-
# Captured arguments (user_id, amount, etc.)
|
|
136
|
+
# Captured arguments (user_id, amount, etc.) from child spans will also be visible.
|
|
136
137
|
|
|
137
|
-
#
|
|
138
|
+
# Expected Output:
|
|
138
139
|
# - [Span 1] step_1_validate_payment (100.81ms)
|
|
139
140
|
# - [Span 2] step_2_send_receipt (202.06ms)
|
|
140
141
|
# - [Span 3] process_payment (333.18ms)
|
|
141
142
|
```
|
|
143
|
+
|
|
142
144
|
-----
|
|
143
145
|
|
|
144
146
|
## ⚙️ Configuration
|
|
@@ -153,10 +155,10 @@ You can select the text vectorization method via the `VECTORIZER` environment va
|
|
|
153
155
|
|
|
154
156
|
| `VECTORIZER` Setting | Description | Required Additional Settings |
|
|
155
157
|
| :--- | :--- | :--- |
|
|
156
|
-
| **`huggingface`** | (Default Recommended) Uses the `sentence-transformers` library to vectorize on your local CPU. No API key is needed
|
|
157
|
-
| **`openai_client`** | (High-Performance) Uses the OpenAI Python client to vectorize with
|
|
158
|
-
| **`weaviate_module`** | (Docker Delegate) Delegates
|
|
159
|
-
| **`none`** | Disables vectorization. Data
|
|
158
|
+
| **`huggingface`** | (Default Recommended) Uses the `sentence-transformers` library to vectorize on your local CPU. No API key is needed. | `HF_MODEL_NAME` (e.g., "sentence-transformers/all-MiniLM-L6-v2") |
|
|
159
|
+
| **`openai_client`** | (High-Performance) Uses the OpenAI Python client to vectorize with models like `text-embedding-3-small`. | `OPENAI_API_KEY` (A valid OpenAI API key) |
|
|
160
|
+
| **`weaviate_module`** | (Docker Delegate) Delegates vectorization to Weaviate's built-in module (e.g., `text2vec-openai`). | `WEAVIATE_VECTORIZER_MODULE`, `OPENAI_API_KEY` |
|
|
161
|
+
| **`none`** | Disables vectorization. Data is stored without vectors. | None |
|
|
160
162
|
|
|
161
163
|
-----
|
|
162
164
|
|
|
@@ -184,6 +186,7 @@ OPENAI_API_KEY=sk-...
|
|
|
184
186
|
|
|
185
187
|
# --- [Advanced] Custom Properties ---
|
|
186
188
|
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
189
|
+
FAILURE_MAPPING_FILE_PATH=.vectorwave_errors.json
|
|
187
190
|
RUN_ID=test-run-001
|
|
188
191
|
```
|
|
189
192
|
|
|
@@ -209,6 +212,7 @@ HF_MODEL_NAME=...
|
|
|
209
212
|
|
|
210
213
|
# --- [Advanced] Custom Properties ---
|
|
211
214
|
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
215
|
+
FAILURE_MAPPING_FILE_PATH=.vectorwave_errors.json
|
|
212
216
|
RUN_ID=test-run-001
|
|
213
217
|
```
|
|
214
218
|
|
|
@@ -233,11 +237,72 @@ OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
|
233
237
|
|
|
234
238
|
# --- [Advanced] Custom Properties ---
|
|
235
239
|
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
240
|
+
FAILURE_MAPPING_FILE_PATH=.vectorwave_errors.json
|
|
236
241
|
RUN_ID=test-run-001
|
|
237
242
|
```
|
|
238
243
|
|
|
239
244
|
-----
|
|
240
245
|
|
|
246
|
+
### 🚀 Advanced Failure Tracing (Error Code)
|
|
247
|
+
|
|
248
|
+
This enhances `VectorWaveExecutions` logs beyond a simple `status: "ERROR"`. An `error_code` property is added to the schema for granular failure analysis.
|
|
249
|
+
|
|
250
|
+
When a function wrapped by `@vectorize` or `@trace_span` fails, the `error_code` is automatically determined based on three priorities:
|
|
251
|
+
|
|
252
|
+
1. **Custom Exception Attribute (Priority 1):**
|
|
253
|
+
The most specific method. If the raised exception object `e` has an `e.error_code` attribute, its value is used.
|
|
254
|
+
|
|
255
|
+
```python
|
|
256
|
+
class PaymentError(Exception):
|
|
257
|
+
def __init__(self, message, error_code):
|
|
258
|
+
super().__init__(message)
|
|
259
|
+
self.error_code = error_code # ⬅️ This attribute is detected.
|
|
260
|
+
|
|
261
|
+
@vectorize(...)
|
|
262
|
+
def process_payment(amount):
|
|
263
|
+
if amount < 0:
|
|
264
|
+
raise PaymentError("Amount < 0", error_code="PAYMENT_NEGATIVE_AMOUNT")
|
|
265
|
+
|
|
266
|
+
# DB Log on execution: { "status": "ERROR", "error_code": "PAYMENT_NEGATIVE_AMOUNT" }
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
2. **Global Mapping File (Priority 2):**
|
|
270
|
+
Centrally manage common exceptions. VectorWave loads a JSON file specified by `FAILURE_MAPPING_FILE_PATH` in your `.env` (default: `.vectorwave_errors.json`) and maps the exception class name to a code.
|
|
271
|
+
|
|
272
|
+
**`.vectorwave_errors.json` Example:**
|
|
273
|
+
|
|
274
|
+
```json
|
|
275
|
+
{
|
|
276
|
+
"ValueError": "INVALID_INPUT",
|
|
277
|
+
"KeyError": "CONFIG_MISSING",
|
|
278
|
+
"TypeError": "INVALID_INPUT"
|
|
279
|
+
}
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
```python
|
|
283
|
+
@vectorize(...)
|
|
284
|
+
def get_config(key):
|
|
285
|
+
return os.environ[key] # ⬅️ Raises KeyError
|
|
286
|
+
|
|
287
|
+
# DB Log on execution: { "status": "ERROR", "error_code": "CONFIG_MISSING" }
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
3. **Default (Priority 3):**
|
|
291
|
+
If neither of the above applies, the exception's class name (e.g., `"ZeroDivisionError"`) is stored as the default `error_code`.
|
|
292
|
+
|
|
293
|
+
**[Usage] Searching for Failures:**
|
|
294
|
+
You can now filter for specific failure types using `search_executions`.
|
|
295
|
+
|
|
296
|
+
```python
|
|
297
|
+
# Find all failure logs categorized as "INVALID_INPUT"
|
|
298
|
+
invalid_logs = search_executions(
|
|
299
|
+
filters={"error_code": "INVALID_INPUT"},
|
|
300
|
+
limit=10
|
|
301
|
+
)
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
-----
|
|
305
|
+
|
|
241
306
|
### Custom Properties and Dynamic Execution Tagging
|
|
242
307
|
|
|
243
308
|
VectorWave can store user-defined metadata in addition to static data (function definitions) and dynamic data (execution logs). This works in two steps.
|
|
@@ -281,7 +346,7 @@ When a function is executed, VectorWave adds tags to the `VectorWaveExecutions`
|
|
|
281
346
|
VectorWave looks for environment variables matching the **UPPERCASE name** of the keys defined in Step 1 (e.g., `RUN_ID`, `EXPERIMENT_ID`). Found values are loaded as `global_custom_values` and added to *all* execution logs. Ideal for run-wide metadata.
|
|
282
347
|
|
|
283
348
|
**2. Function-Specific Tags (Decorator)**
|
|
284
|
-
You can pass tags as keyword arguments (`**execution_tags`) directly to the `@vectorize` decorator.
|
|
349
|
+
You can pass tags as keyword arguments (`**execution_tags`) directly to the `@vectorize` decorator. This is ideal for function-specific metadata.
|
|
285
350
|
|
|
286
351
|
```python
|
|
287
352
|
# --- .env file ---
|
|
@@ -308,7 +373,7 @@ def other_function():
|
|
|
308
373
|
|
|
309
374
|
**Tag Merging and Validation Rules**
|
|
310
375
|
|
|
311
|
-
1. **Validation (Important):** Tags (global or function-specific) will **only** be saved to Weaviate if their key (e.g., `run_id`, `team`, `priority`) was first defined in the `.weaviate_properties` file (Step 1). Tags not defined in the schema are **ignored**, and a warning is
|
|
376
|
+
1. **Validation (Important):** Tags (global or function-specific) will **only** be saved to Weaviate if their key (e.g., `run_id`, `team`, `priority`) was first defined in the `.weaviate_properties` file (Step 1). Tags not defined in the schema are **ignored**, and a warning is logged at startup.
|
|
312
377
|
|
|
313
378
|
2. **Priority (Override):** If a tag key is defined in both places (e.g., global `RUN_ID` in `.env` and `run_id="override-xyz"` in the decorator), the **function-specific tag from the decorator always wins**.
|
|
314
379
|
|
|
@@ -321,9 +386,8 @@ def other_function():
|
|
|
321
386
|
|
|
322
387
|
## 🤝 Contributing
|
|
323
388
|
|
|
324
|
-
|
|
389
|
+
Bug reports, feature requests, and code contributions are all welcome. For details, please see [CONTRIBUTING.md](https://www.google.com/search?q=httpsS://www.google.com/search%3Fq%3DCONTRIBUTING.md).
|
|
325
390
|
|
|
326
391
|
## 📜 License
|
|
327
392
|
|
|
328
|
-
This project is distributed under the MIT License. See the [LICENSE](https://www.google.com/search?q=
|
|
329
|
-
|
|
393
|
+
This project is distributed under the MIT License. See the [LICENSE](https://www.google.com/search?q=LICENSE) file for details.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "vectorwave"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.4"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name = "junyeonggim", email = "junyeonggim5@gmail.com" },
|
|
10
10
|
]
|
|
@@ -25,7 +25,8 @@ classifiers = [
|
|
|
25
25
|
|
|
26
26
|
dependencies = [
|
|
27
27
|
"weaviate-client>=4.0.0",
|
|
28
|
-
"pydantic-settings>=2.0.0"
|
|
28
|
+
"pydantic-settings>=2.0.0",
|
|
29
|
+
"sentence-transformers"
|
|
29
30
|
]
|
|
30
31
|
|
|
31
32
|
[project.urls]
|