vectorwave 0.1.2__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vectorwave-0.1.2/src/vectorwave.egg-info → vectorwave-0.1.3}/PKG-INFO +94 -33
- {vectorwave-0.1.2 → vectorwave-0.1.3}/Readme.md +93 -32
- {vectorwave-0.1.2 → vectorwave-0.1.3}/pyproject.toml +5 -5
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/tests/batch/test_batch.py +2 -1
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/tests/database/test_db.py +23 -19
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/tests/models/test_db_config.py +21 -12
- vectorwave-0.1.3/src/tests/monitoring/test_tracer.py +202 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/vectorwave/batch/batch.py +11 -8
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/vectorwave/core/decorator.py +30 -7
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/vectorwave/database/db.py +64 -38
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/vectorwave/database/db_search.py +32 -10
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/vectorwave/models/db_config.py +22 -12
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/vectorwave/monitoring/tracer.py +5 -2
- vectorwave-0.1.3/src/vectorwave/prediction/predictor.py +0 -0
- vectorwave-0.1.3/src/vectorwave/vectorizer/__init__.py +0 -0
- vectorwave-0.1.3/src/vectorwave/vectorizer/base.py +12 -0
- vectorwave-0.1.3/src/vectorwave/vectorizer/factory.py +49 -0
- vectorwave-0.1.3/src/vectorwave/vectorizer/huggingface_vectorizer.py +33 -0
- vectorwave-0.1.3/src/vectorwave/vectorizer/openai_vectorizer.py +35 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3/src/vectorwave.egg-info}/PKG-INFO +94 -33
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/vectorwave.egg-info/SOURCES.txt +8 -1
- {vectorwave-0.1.2 → vectorwave-0.1.3}/LICENSE +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/MANIFEST.in +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/NOTICE +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/setup.cfg +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/tests/__init__.py +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/tests/batch/__init__.py +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/tests/core/__init__.py +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/tests/core/test_decorator.py +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/tests/database/__init__.py +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/tests/database/test_db_search.py +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/tests/exception/__init__.py +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/tests/models/__init__.py +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/tests/monitoring/__init__.py +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/tests/prediction/__init__.py +0 -0
- {vectorwave-0.1.2/src/vectorwave/batch → vectorwave-0.1.3/src/tests/vectorizer}/__init__.py +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/vectorwave/__init__.py +0 -0
- {vectorwave-0.1.2/src/vectorwave/core → vectorwave-0.1.3/src/vectorwave/batch}/__init__.py +0 -0
- {vectorwave-0.1.2/src/vectorwave/database → vectorwave-0.1.3/src/vectorwave/core}/__init__.py +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/vectorwave/core/core.py +0 -0
- {vectorwave-0.1.2/src/vectorwave/exception → vectorwave-0.1.3/src/vectorwave/database}/__init__.py +0 -0
- {vectorwave-0.1.2/src/vectorwave/models → vectorwave-0.1.3/src/vectorwave/exception}/__init__.py +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/vectorwave/exception/exceptions.py +0 -0
- {vectorwave-0.1.2/src/vectorwave/monitoring → vectorwave-0.1.3/src/vectorwave/models}/__init__.py +0 -0
- {vectorwave-0.1.2/src/vectorwave/prediction → vectorwave-0.1.3/src/vectorwave/monitoring}/__init__.py +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/vectorwave/monitoring/monitoring.py +0 -0
- /vectorwave-0.1.2/src/vectorwave/prediction/predictor.py → /vectorwave-0.1.3/src/vectorwave/prediction/__init__.py +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/vectorwave.egg-info/dependency_links.txt +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/vectorwave.egg-info/requires.txt +0 -0
- {vectorwave-0.1.2 → vectorwave-0.1.3}/src/vectorwave.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vectorwave
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: VectorWave: Seamless Auto-Vectorization Framework
|
|
5
5
|
Author-email: junyeonggim <junyeonggim5@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -162,51 +162,112 @@ for i, span in enumerate(trace_spans):
|
|
|
162
162
|
# - [Span 2] step_2_send_receipt (202.06ms)
|
|
163
163
|
# - [Span 3] process_payment (333.18ms)
|
|
164
164
|
```
|
|
165
|
-
|
|
166
165
|
-----
|
|
167
166
|
|
|
168
167
|
## ⚙️ Configuration
|
|
169
168
|
|
|
170
|
-
VectorWave automatically reads Weaviate database connection
|
|
169
|
+
VectorWave automatically reads Weaviate database connection info and **vectorization strategy** from **environment variables** or a `.env` file.
|
|
170
|
+
|
|
171
|
+
Create a `.env` file in your project's root directory (e.g., where `test_ex/example.py` is located) and set the required values.
|
|
172
|
+
|
|
173
|
+
### Vectorizer Strategy (VECTORIZER)
|
|
174
|
+
|
|
175
|
+
You can select the text vectorization method via the `VECTORIZER` environment variable in your `test_ex/.env` file.
|
|
176
|
+
|
|
177
|
+
| `VECTORIZER` Setting | Description | Required Additional Settings |
|
|
178
|
+
| :--- | :--- | :--- |
|
|
179
|
+
| **`huggingface`** | (Default Recommended) Uses the `sentence-transformers` library to vectorize on your local CPU. No API key is needed, making it great for immediate testing. | `HF_MODEL_NAME` (e.g., "sentence-transformers/all-MiniLM-L6-v2") |
|
|
180
|
+
| **`openai_client`** | (High-Performance) Uses the OpenAI Python client to vectorize with modern models like `text-embedding-3-small`. | `OPENAI_API_KEY` (A valid OpenAI API key) |
|
|
181
|
+
| **`weaviate_module`** | (Docker Delegate) Delegates the vectorization task to the Weaviate container's built-in module (e.g., `text2vec-openai`). | `WEAVIATE_VECTORIZER_MODULE`, `OPENAI_API_KEY` |
|
|
182
|
+
| **`none`** | Disables vectorization. Data will be stored without vectors. | None |
|
|
183
|
+
|
|
184
|
+
-----
|
|
185
|
+
|
|
186
|
+
### .env File Examples
|
|
187
|
+
|
|
188
|
+
Configure your `.env` file according to the strategy you want to use.
|
|
189
|
+
|
|
190
|
+
#### Example 1: Using `huggingface` (Local, No API Key)
|
|
191
|
+
|
|
192
|
+
Uses a `sentence-transformers` model on your local machine. Ideal for testing without API keys.
|
|
193
|
+
|
|
194
|
+
```ini
|
|
195
|
+
# .env (Using HuggingFace)
|
|
196
|
+
# --- Basic Weaviate Connection ---
|
|
197
|
+
WEAVIATE_HOST=localhost
|
|
198
|
+
WEAVIATE_PORT=8080
|
|
199
|
+
WEAVIATE_GRPC_PORT=50051
|
|
200
|
+
|
|
201
|
+
# --- [Strategy 1] HuggingFace Config ---
|
|
202
|
+
VECTORIZER="huggingface"
|
|
203
|
+
HF_MODEL_NAME="sentence-transformers/all-MiniLM-L6-v2"
|
|
204
|
+
|
|
205
|
+
# (OPENAI_API_KEY is not required for this mode)
|
|
206
|
+
OPENAI_API_KEY=sk-...
|
|
171
207
|
|
|
172
|
-
|
|
208
|
+
# --- [Advanced] Custom Properties ---
|
|
209
|
+
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
210
|
+
RUN_ID=test-run-001
|
|
211
|
+
```
|
|
173
212
|
|
|
174
|
-
|
|
213
|
+
#### Example 2: Using `openai_client` (Python Client, High-Performance)
|
|
214
|
+
|
|
215
|
+
Directly calls the OpenAI API via the `openai` Python library.
|
|
175
216
|
|
|
176
217
|
```ini
|
|
177
|
-
# .env
|
|
178
|
-
# --- Basic Weaviate Connection
|
|
218
|
+
# .env (Using OpenAI Python Client)
|
|
219
|
+
# --- Basic Weaviate Connection ---
|
|
179
220
|
WEAVIATE_HOST=localhost
|
|
180
221
|
WEAVIATE_PORT=8080
|
|
181
222
|
WEAVIATE_GRPC_PORT=50051
|
|
182
223
|
|
|
183
|
-
# ---
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
#
|
|
187
|
-
|
|
188
|
-
# An OpenAI API key is required if using modules like text2vec-openai.
|
|
189
|
-
# OPENAI_API_KEY=sk-your-key-here
|
|
224
|
+
# --- [Strategy 2] OpenAI Client Config ---
|
|
225
|
+
VECTORIZER="openai_client"
|
|
226
|
+
|
|
227
|
+
# [Required] You must enter a valid OpenAI API key.
|
|
228
|
+
OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
190
229
|
|
|
191
|
-
#
|
|
192
|
-
|
|
230
|
+
# (HF_MODEL_NAME is not used in this mode)
|
|
231
|
+
HF_MODEL_NAME=...
|
|
232
|
+
|
|
233
|
+
# --- [Advanced] Custom Properties ---
|
|
193
234
|
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
235
|
+
RUN_ID=test-run-001
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
#### Example 3: Using `weaviate_module` (Docker Delegate)
|
|
194
239
|
|
|
195
|
-
|
|
196
|
-
|
|
240
|
+
Delegates vectorization to the Weaviate Docker container instead of Python. (See `vw_docker.yml` config).
|
|
241
|
+
|
|
242
|
+
```ini
|
|
243
|
+
# .env (Delegating to Weaviate Module)
|
|
244
|
+
# --- Basic Weaviate Connection ---
|
|
245
|
+
WEAVIATE_HOST=localhost
|
|
246
|
+
WEAVIATE_PORT=8080
|
|
247
|
+
WEAVIATE_GRPC_PORT=50051
|
|
248
|
+
|
|
249
|
+
# --- [Strategy 3] Weaviate Module Config ---
|
|
250
|
+
VECTORIZER="weaviate_module"
|
|
251
|
+
WEAVIATE_VECTORIZER_MODULE=text2vec-openai
|
|
252
|
+
WEAVIATE_GENERATIVE_MODULE=generative-openai
|
|
253
|
+
|
|
254
|
+
# [Required] The Weaviate container will read this API key.
|
|
255
|
+
OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
256
|
+
|
|
257
|
+
# --- [Advanced] Custom Properties ---
|
|
258
|
+
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
197
259
|
RUN_ID=test-run-001
|
|
198
|
-
EXPERIMENT_ID=exp-abc
|
|
199
260
|
```
|
|
200
261
|
|
|
201
262
|
-----
|
|
202
263
|
|
|
203
264
|
### Custom Properties and Dynamic Execution Tagging
|
|
204
265
|
|
|
205
|
-
VectorWave can store user-defined metadata in
|
|
266
|
+
VectorWave can store user-defined metadata in addition to static data (function definitions) and dynamic data (execution logs). This works in two steps.
|
|
206
267
|
|
|
207
|
-
#### Step 1: Define Custom Schema (
|
|
268
|
+
#### Step 1: Define Custom Schema (Tag "Allow-list")
|
|
208
269
|
|
|
209
|
-
Create a JSON file at the path specified by `CUSTOM_PROPERTIES_FILE_PATH` (default: `.weaviate_properties`).
|
|
270
|
+
Create a JSON file at the path specified by `CUSTOM_PROPERTIES_FILE_PATH` in your `.env` file (default: `.weaviate_properties`).
|
|
210
271
|
|
|
211
272
|
This file instructs VectorWave to add **new properties (columns)** to the Weaviate collections. This file acts as an **"allow-list"** for all custom tags.
|
|
212
273
|
|
|
@@ -233,17 +294,17 @@ This file instructs VectorWave to add **new properties (columns)** to the Weavia
|
|
|
233
294
|
}
|
|
234
295
|
```
|
|
235
296
|
|
|
236
|
-
*
|
|
297
|
+
* This definition will add `run_id`, `experiment_id`, `team`, and `priority` properties to both the `VectorWaveFunctions` and `VectorWaveExecutions` collections.
|
|
237
298
|
|
|
238
299
|
#### Step 2: Dynamic Execution Tagging (Adding Values)
|
|
239
300
|
|
|
240
|
-
When a function
|
|
301
|
+
When a function is executed, VectorWave adds tags to the `VectorWaveExecutions` log. These tags are collected and merged from two sources.
|
|
241
302
|
|
|
242
|
-
**1. Global Tags (
|
|
243
|
-
VectorWave
|
|
303
|
+
**1. Global Tags (Environment Variables)**
|
|
304
|
+
VectorWave looks for environment variables matching the **UPPERCASE name** of the keys defined in Step 1 (e.g., `RUN_ID`, `EXPERIMENT_ID`). Found values are loaded as `global_custom_values` and added to *all* execution logs. Ideal for run-wide metadata.
|
|
244
305
|
|
|
245
|
-
**2. Function-Specific Tags (
|
|
246
|
-
You can pass tags
|
|
306
|
+
**2. Function-Specific Tags (Decorator)**
|
|
307
|
+
You can pass tags as keyword arguments (`**execution_tags`) directly to the `@vectorize` decorator. Ideal for function-specific metadata.
|
|
247
308
|
|
|
248
309
|
```python
|
|
249
310
|
# --- .env file ---
|
|
@@ -251,7 +312,7 @@ You can pass tags directly to the `@vectorize` decorator as keyword arguments (`
|
|
|
251
312
|
# TEAM=default-team
|
|
252
313
|
|
|
253
314
|
@vectorize(
|
|
254
|
-
search_description="Process
|
|
315
|
+
search_description="Process payment",
|
|
255
316
|
sequence_narrative="...",
|
|
256
317
|
team="billing", # <-- Function-specific tag
|
|
257
318
|
priority=1 # <-- Function-specific tag
|
|
@@ -270,14 +331,14 @@ def other_function():
|
|
|
270
331
|
|
|
271
332
|
**Tag Merging and Validation Rules**
|
|
272
333
|
|
|
273
|
-
1. **Validation (
|
|
334
|
+
1. **Validation (Important):** Tags (global or function-specific) will **only** be saved to Weaviate if their key (e.g., `run_id`, `team`, `priority`) was first defined in the `.weaviate_properties` file (Step 1). Tags not defined in the schema are **ignored**, and a warning is printed at script startup.
|
|
274
335
|
|
|
275
|
-
2. **Priority (Override):** If a tag key is defined in both places (e.g.,
|
|
336
|
+
2. **Priority (Override):** If a tag key is defined in both places (e.g., global `RUN_ID` in `.env` and `run_id="override-xyz"` in the decorator), the **function-specific tag from the decorator always wins**.
|
|
276
337
|
|
|
277
338
|
**Resulting Logs:**
|
|
278
339
|
|
|
279
|
-
* `process_payment()` log
|
|
280
|
-
* `other_function()` log
|
|
340
|
+
* `process_payment()` execution log: `{"run_id": "global-run-abc", "team": "billing", "priority": 1}`
|
|
341
|
+
* `other_function()` execution log: `{"run_id": "override-run-xyz", "team": "default-team"}`
|
|
281
342
|
|
|
282
343
|
-----
|
|
283
344
|
|
|
@@ -139,51 +139,112 @@ for i, span in enumerate(trace_spans):
|
|
|
139
139
|
# - [Span 2] step_2_send_receipt (202.06ms)
|
|
140
140
|
# - [Span 3] process_payment (333.18ms)
|
|
141
141
|
```
|
|
142
|
-
|
|
143
142
|
-----
|
|
144
143
|
|
|
145
144
|
## ⚙️ Configuration
|
|
146
145
|
|
|
147
|
-
VectorWave automatically reads Weaviate database connection
|
|
146
|
+
VectorWave automatically reads Weaviate database connection info and **vectorization strategy** from **environment variables** or a `.env` file.
|
|
147
|
+
|
|
148
|
+
Create a `.env` file in your project's root directory (e.g., where `test_ex/example.py` is located) and set the required values.
|
|
149
|
+
|
|
150
|
+
### Vectorizer Strategy (VECTORIZER)
|
|
151
|
+
|
|
152
|
+
You can select the text vectorization method via the `VECTORIZER` environment variable in your `test_ex/.env` file.
|
|
153
|
+
|
|
154
|
+
| `VECTORIZER` Setting | Description | Required Additional Settings |
|
|
155
|
+
| :--- | :--- | :--- |
|
|
156
|
+
| **`huggingface`** | (Default Recommended) Uses the `sentence-transformers` library to vectorize on your local CPU. No API key is needed, making it great for immediate testing. | `HF_MODEL_NAME` (e.g., "sentence-transformers/all-MiniLM-L6-v2") |
|
|
157
|
+
| **`openai_client`** | (High-Performance) Uses the OpenAI Python client to vectorize with modern models like `text-embedding-3-small`. | `OPENAI_API_KEY` (A valid OpenAI API key) |
|
|
158
|
+
| **`weaviate_module`** | (Docker Delegate) Delegates the vectorization task to the Weaviate container's built-in module (e.g., `text2vec-openai`). | `WEAVIATE_VECTORIZER_MODULE`, `OPENAI_API_KEY` |
|
|
159
|
+
| **`none`** | Disables vectorization. Data will be stored without vectors. | None |
|
|
160
|
+
|
|
161
|
+
-----
|
|
162
|
+
|
|
163
|
+
### .env File Examples
|
|
164
|
+
|
|
165
|
+
Configure your `.env` file according to the strategy you want to use.
|
|
166
|
+
|
|
167
|
+
#### Example 1: Using `huggingface` (Local, No API Key)
|
|
168
|
+
|
|
169
|
+
Uses a `sentence-transformers` model on your local machine. Ideal for testing without API keys.
|
|
170
|
+
|
|
171
|
+
```ini
|
|
172
|
+
# .env (Using HuggingFace)
|
|
173
|
+
# --- Basic Weaviate Connection ---
|
|
174
|
+
WEAVIATE_HOST=localhost
|
|
175
|
+
WEAVIATE_PORT=8080
|
|
176
|
+
WEAVIATE_GRPC_PORT=50051
|
|
177
|
+
|
|
178
|
+
# --- [Strategy 1] HuggingFace Config ---
|
|
179
|
+
VECTORIZER="huggingface"
|
|
180
|
+
HF_MODEL_NAME="sentence-transformers/all-MiniLM-L6-v2"
|
|
181
|
+
|
|
182
|
+
# (OPENAI_API_KEY is not required for this mode)
|
|
183
|
+
OPENAI_API_KEY=sk-...
|
|
148
184
|
|
|
149
|
-
|
|
185
|
+
# --- [Advanced] Custom Properties ---
|
|
186
|
+
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
187
|
+
RUN_ID=test-run-001
|
|
188
|
+
```
|
|
150
189
|
|
|
151
|
-
|
|
190
|
+
#### Example 2: Using `openai_client` (Python Client, High-Performance)
|
|
191
|
+
|
|
192
|
+
Directly calls the OpenAI API via the `openai` Python library.
|
|
152
193
|
|
|
153
194
|
```ini
|
|
154
|
-
# .env
|
|
155
|
-
# --- Basic Weaviate Connection
|
|
195
|
+
# .env (Using OpenAI Python Client)
|
|
196
|
+
# --- Basic Weaviate Connection ---
|
|
156
197
|
WEAVIATE_HOST=localhost
|
|
157
198
|
WEAVIATE_PORT=8080
|
|
158
199
|
WEAVIATE_GRPC_PORT=50051
|
|
159
200
|
|
|
160
|
-
# ---
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
#
|
|
164
|
-
|
|
165
|
-
# An OpenAI API key is required if using modules like text2vec-openai.
|
|
166
|
-
# OPENAI_API_KEY=sk-your-key-here
|
|
201
|
+
# --- [Strategy 2] OpenAI Client Config ---
|
|
202
|
+
VECTORIZER="openai_client"
|
|
203
|
+
|
|
204
|
+
# [Required] You must enter a valid OpenAI API key.
|
|
205
|
+
OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
167
206
|
|
|
168
|
-
#
|
|
169
|
-
|
|
207
|
+
# (HF_MODEL_NAME is not used in this mode)
|
|
208
|
+
HF_MODEL_NAME=...
|
|
209
|
+
|
|
210
|
+
# --- [Advanced] Custom Properties ---
|
|
170
211
|
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
212
|
+
RUN_ID=test-run-001
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
#### Example 3: Using `weaviate_module` (Docker Delegate)
|
|
171
216
|
|
|
172
|
-
|
|
173
|
-
|
|
217
|
+
Delegates vectorization to the Weaviate Docker container instead of Python. (See `vw_docker.yml` config).
|
|
218
|
+
|
|
219
|
+
```ini
|
|
220
|
+
# .env (Delegating to Weaviate Module)
|
|
221
|
+
# --- Basic Weaviate Connection ---
|
|
222
|
+
WEAVIATE_HOST=localhost
|
|
223
|
+
WEAVIATE_PORT=8080
|
|
224
|
+
WEAVIATE_GRPC_PORT=50051
|
|
225
|
+
|
|
226
|
+
# --- [Strategy 3] Weaviate Module Config ---
|
|
227
|
+
VECTORIZER="weaviate_module"
|
|
228
|
+
WEAVIATE_VECTORIZER_MODULE=text2vec-openai
|
|
229
|
+
WEAVIATE_GENERATIVE_MODULE=generative-openai
|
|
230
|
+
|
|
231
|
+
# [Required] The Weaviate container will read this API key.
|
|
232
|
+
OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
233
|
+
|
|
234
|
+
# --- [Advanced] Custom Properties ---
|
|
235
|
+
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
174
236
|
RUN_ID=test-run-001
|
|
175
|
-
EXPERIMENT_ID=exp-abc
|
|
176
237
|
```
|
|
177
238
|
|
|
178
239
|
-----
|
|
179
240
|
|
|
180
241
|
### Custom Properties and Dynamic Execution Tagging
|
|
181
242
|
|
|
182
|
-
VectorWave can store user-defined metadata in
|
|
243
|
+
VectorWave can store user-defined metadata in addition to static data (function definitions) and dynamic data (execution logs). This works in two steps.
|
|
183
244
|
|
|
184
|
-
#### Step 1: Define Custom Schema (
|
|
245
|
+
#### Step 1: Define Custom Schema (Tag "Allow-list")
|
|
185
246
|
|
|
186
|
-
Create a JSON file at the path specified by `CUSTOM_PROPERTIES_FILE_PATH` (default: `.weaviate_properties`).
|
|
247
|
+
Create a JSON file at the path specified by `CUSTOM_PROPERTIES_FILE_PATH` in your `.env` file (default: `.weaviate_properties`).
|
|
187
248
|
|
|
188
249
|
This file instructs VectorWave to add **new properties (columns)** to the Weaviate collections. This file acts as an **"allow-list"** for all custom tags.
|
|
189
250
|
|
|
@@ -210,17 +271,17 @@ This file instructs VectorWave to add **new properties (columns)** to the Weavia
|
|
|
210
271
|
}
|
|
211
272
|
```
|
|
212
273
|
|
|
213
|
-
*
|
|
274
|
+
* This definition will add `run_id`, `experiment_id`, `team`, and `priority` properties to both the `VectorWaveFunctions` and `VectorWaveExecutions` collections.
|
|
214
275
|
|
|
215
276
|
#### Step 2: Dynamic Execution Tagging (Adding Values)
|
|
216
277
|
|
|
217
|
-
When a function
|
|
278
|
+
When a function is executed, VectorWave adds tags to the `VectorWaveExecutions` log. These tags are collected and merged from two sources.
|
|
218
279
|
|
|
219
|
-
**1. Global Tags (
|
|
220
|
-
VectorWave
|
|
280
|
+
**1. Global Tags (Environment Variables)**
|
|
281
|
+
VectorWave looks for environment variables matching the **UPPERCASE name** of the keys defined in Step 1 (e.g., `RUN_ID`, `EXPERIMENT_ID`). Found values are loaded as `global_custom_values` and added to *all* execution logs. Ideal for run-wide metadata.
|
|
221
282
|
|
|
222
|
-
**2. Function-Specific Tags (
|
|
223
|
-
You can pass tags
|
|
283
|
+
**2. Function-Specific Tags (Decorator)**
|
|
284
|
+
You can pass tags as keyword arguments (`**execution_tags`) directly to the `@vectorize` decorator. Ideal for function-specific metadata.
|
|
224
285
|
|
|
225
286
|
```python
|
|
226
287
|
# --- .env file ---
|
|
@@ -228,7 +289,7 @@ You can pass tags directly to the `@vectorize` decorator as keyword arguments (`
|
|
|
228
289
|
# TEAM=default-team
|
|
229
290
|
|
|
230
291
|
@vectorize(
|
|
231
|
-
search_description="Process
|
|
292
|
+
search_description="Process payment",
|
|
232
293
|
sequence_narrative="...",
|
|
233
294
|
team="billing", # <-- Function-specific tag
|
|
234
295
|
priority=1 # <-- Function-specific tag
|
|
@@ -247,14 +308,14 @@ def other_function():
|
|
|
247
308
|
|
|
248
309
|
**Tag Merging and Validation Rules**
|
|
249
310
|
|
|
250
|
-
1. **Validation (
|
|
311
|
+
1. **Validation (Important):** Tags (global or function-specific) will **only** be saved to Weaviate if their key (e.g., `run_id`, `team`, `priority`) was first defined in the `.weaviate_properties` file (Step 1). Tags not defined in the schema are **ignored**, and a warning is printed at script startup.
|
|
251
312
|
|
|
252
|
-
2. **Priority (Override):** If a tag key is defined in both places (e.g.,
|
|
313
|
+
2. **Priority (Override):** If a tag key is defined in both places (e.g., global `RUN_ID` in `.env` and `run_id="override-xyz"` in the decorator), the **function-specific tag from the decorator always wins**.
|
|
253
314
|
|
|
254
315
|
**Resulting Logs:**
|
|
255
316
|
|
|
256
|
-
* `process_payment()` log
|
|
257
|
-
* `other_function()` log
|
|
317
|
+
* `process_payment()` execution log: `{"run_id": "global-run-abc", "team": "billing", "priority": 1}`
|
|
318
|
+
* `other_function()` execution log: `{"run_id": "override-run-xyz", "team": "default-team"}`
|
|
258
319
|
|
|
259
320
|
-----
|
|
260
321
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "vectorwave"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.3"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name = "junyeonggim", email = "junyeonggim5@gmail.com" },
|
|
10
10
|
]
|
|
@@ -14,10 +14,10 @@ requires-python = ">=3.10"
|
|
|
14
14
|
license = "MIT"
|
|
15
15
|
classifiers = [
|
|
16
16
|
"Programming Language :: Python :: 3",
|
|
17
|
-
"Programming Language :: Python :: 3.10",
|
|
18
|
-
"Programming Language :: Python :: 3.11",
|
|
19
|
-
"Programming Language :: Python :: 3.12",
|
|
20
|
-
"Programming Language :: Python :: 3.13",
|
|
17
|
+
"Programming Language :: Python :: 3.10",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
21
|
"Operating System :: OS Independent",
|
|
22
22
|
"Development Status :: 3 - Alpha",
|
|
23
23
|
"Intended Audience :: Developers",
|
|
@@ -392,16 +392,12 @@ def test_create_schema_vectorizer_openai(test_settings):
|
|
|
392
392
|
mock_collections.exists.return_value = False
|
|
393
393
|
mock_client.collections = mock_collections
|
|
394
394
|
|
|
395
|
-
test_settings.
|
|
396
|
-
test_settings.
|
|
395
|
+
test_settings.VECTORIZER = "weaviate_module"
|
|
396
|
+
test_settings.WEAVIATE_VECTORIZER_MODULE = "text2vec-openai"
|
|
397
|
+
|
|
398
|
+
# [수정] test_settings.GENERATIVE_CONFIG = "generative-openai" ->
|
|
399
|
+
test_settings.WEAVIATE_GENERATIVE_MODULE = "generative-openai"
|
|
397
400
|
|
|
398
|
-
expected_vector_config = {
|
|
399
|
-
"vectorizer": "text2vec-openai",
|
|
400
|
-
"text2vec-openai": {
|
|
401
|
-
"vectorizeClassName": test_settings.IS_VECTORIZE_COLLECTION_NAME,
|
|
402
|
-
}
|
|
403
|
-
}
|
|
404
|
-
expected_generative_config = {"generator": "generative-openai"}
|
|
405
401
|
|
|
406
402
|
# 2. Act
|
|
407
403
|
create_vectorwave_schema(mock_client, test_settings)
|
|
@@ -411,9 +407,16 @@ def test_create_schema_vectorizer_openai(test_settings):
|
|
|
411
407
|
|
|
412
408
|
call_args = mock_collections.create.call_args
|
|
413
409
|
|
|
410
|
+
vector_config_arg = call_args.kwargs.get('vector_config')
|
|
411
|
+
# assert isinstance(vector_config_arg, wvc.Configure.Vectorizer)
|
|
412
|
+
# assert vector_config_arg.name == "text2vec-openai"
|
|
413
|
+
|
|
414
|
+
assert vector_config_arg.vectorizer == "text2vec-openai"
|
|
415
|
+
|
|
416
|
+
# 3b. Generative Config 검증
|
|
417
|
+
generative_config_arg = call_args.kwargs.get('generative_config')
|
|
418
|
+
assert generative_config_arg.generative == "generative-openai"
|
|
414
419
|
|
|
415
|
-
assert call_args.kwargs.get('vector_config') == expected_vector_config
|
|
416
|
-
assert call_args.kwargs.get('generative_config') == expected_generative_config
|
|
417
420
|
|
|
418
421
|
|
|
419
422
|
|
|
@@ -427,9 +430,8 @@ def test_create_schema_vectorizer_none(test_settings):
|
|
|
427
430
|
mock_collections.exists.return_value = False
|
|
428
431
|
mock_client.collections = mock_collections
|
|
429
432
|
|
|
430
|
-
test_settings.
|
|
431
|
-
|
|
432
|
-
expected_vector_config = {"vectorizer": "none"}
|
|
433
|
+
test_settings.VECTORIZER = "none"
|
|
434
|
+
test_settings.WEAVIATE_VECTORIZER_MODULE = ""
|
|
433
435
|
|
|
434
436
|
# 2. Act
|
|
435
437
|
create_vectorwave_schema(mock_client, test_settings)
|
|
@@ -440,7 +442,10 @@ def test_create_schema_vectorizer_none(test_settings):
|
|
|
440
442
|
call_args = mock_collections.create.call_args
|
|
441
443
|
# mock_none.assert_called_once() -> 삭제
|
|
442
444
|
|
|
443
|
-
|
|
445
|
+
vector_config_arg = call_args.kwargs.get('vector_config')
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
assert vector_config_arg.vectorizer == "none"
|
|
444
449
|
|
|
445
450
|
|
|
446
451
|
def test_create_schema_vectorizer_invalid(test_settings):
|
|
@@ -453,12 +458,11 @@ def test_create_schema_vectorizer_invalid(test_settings):
|
|
|
453
458
|
mock_collections.exists.return_value = False
|
|
454
459
|
mock_client.collections = mock_collections
|
|
455
460
|
|
|
456
|
-
test_settings.
|
|
461
|
+
test_settings.VECTORIZER = "unsupported-module"
|
|
457
462
|
|
|
458
463
|
# 2. Act & 3. Assert
|
|
459
464
|
with pytest.raises(SchemaCreationError) as exc_info:
|
|
460
465
|
create_vectorwave_schema(mock_client, test_settings)
|
|
461
466
|
|
|
462
|
-
assert "
|
|
463
|
-
assert "
|
|
464
|
-
assert "text2vec-huggingface" not in str(exc_info.value)
|
|
467
|
+
assert "Invalid VECTORIZER setting" in str(exc_info.value)
|
|
468
|
+
assert "unsupported-module" in str(exc_info.value)
|
|
@@ -60,13 +60,16 @@ def test_get_settings_loads_custom_props_success(mock_open_file, mock_exists):
|
|
|
60
60
|
|
|
61
61
|
|
|
62
62
|
@patch('os.path.exists', return_value=False)
|
|
63
|
-
def test_get_settings_file_not_found(mock_exists,
|
|
63
|
+
def test_get_settings_file_not_found(mock_exists, caplog):
|
|
64
64
|
"""
|
|
65
65
|
Case 2: .weaviate_properties file does not exist
|
|
66
66
|
- settings.custom_properties should be None
|
|
67
|
-
- A 'file not found'
|
|
67
|
+
- A 'file not found' message should be logged at DEBUG level
|
|
68
68
|
"""
|
|
69
|
+
import logging
|
|
70
|
+
|
|
69
71
|
# Arrange
|
|
72
|
+
caplog.set_level(logging.DEBUG) # DEBUG 레벨로 설정 (중요!)
|
|
70
73
|
get_weaviate_settings.cache_clear()
|
|
71
74
|
|
|
72
75
|
# Act
|
|
@@ -76,21 +79,23 @@ def test_get_settings_file_not_found(mock_exists, capsys):
|
|
|
76
79
|
mock_exists.assert_called_with(".weaviate_properties")
|
|
77
80
|
assert settings.custom_properties is None
|
|
78
81
|
|
|
79
|
-
# Check if 'file not found'
|
|
80
|
-
|
|
81
|
-
assert "file not found" in captured.out
|
|
82
|
+
# Check if 'file not found' message was logged
|
|
83
|
+
assert "file not found" in caplog.text.lower() or "not found" in caplog.text
|
|
82
84
|
|
|
83
85
|
|
|
84
86
|
@patch('os.path.exists', return_value=True)
|
|
85
87
|
@patch('builtins.open', new_callable=mock_open, read_data=MOCK_INVALID_JSON)
|
|
86
88
|
@patch('json.load', side_effect=JSONDecodeError("Mock JSON Decode Error", "", 0))
|
|
87
|
-
def test_get_settings_invalid_json(mock_json_load, mock_open_file, mock_exists,
|
|
89
|
+
def test_get_settings_invalid_json(mock_json_load, mock_open_file, mock_exists, caplog):
|
|
88
90
|
"""
|
|
89
91
|
Case 3: File exists but JSON format is invalid
|
|
90
92
|
- settings.custom_properties should be None
|
|
91
|
-
- A 'Could not parse JSON' warning should be
|
|
93
|
+
- A 'Could not parse JSON' warning should be logged
|
|
92
94
|
"""
|
|
95
|
+
import logging
|
|
96
|
+
|
|
93
97
|
# Arrange
|
|
98
|
+
caplog.set_level(logging.WARNING)
|
|
94
99
|
get_weaviate_settings.cache_clear()
|
|
95
100
|
|
|
96
101
|
# Act
|
|
@@ -99,12 +104,16 @@ def test_get_settings_invalid_json(mock_json_load, mock_open_file, mock_exists,
|
|
|
99
104
|
# Assert
|
|
100
105
|
mock_exists.assert_called_once()
|
|
101
106
|
mock_open_file.assert_called_once()
|
|
102
|
-
mock_json_load.assert_called_once()
|
|
103
|
-
assert settings.custom_properties is None
|
|
107
|
+
mock_json_load.assert_called_once() # json.load was called but failed (due to side_effect)
|
|
108
|
+
assert settings.custom_properties is None # Should be None due to parsing failure
|
|
109
|
+
|
|
110
|
+
# Check if 'Could not parse JSON' warning was logged
|
|
111
|
+
assert "Could not parse JSON" in caplog.text
|
|
104
112
|
|
|
105
|
-
#
|
|
106
|
-
|
|
107
|
-
assert
|
|
113
|
+
# Also check the log level
|
|
114
|
+
warning_logs = [r for r in caplog.records if "parse JSON" in r.message]
|
|
115
|
+
assert len(warning_logs) > 0
|
|
116
|
+
assert warning_logs[0].levelname == "WARNING"
|
|
108
117
|
|
|
109
118
|
@patch('os.path.exists', return_value=True)
|
|
110
119
|
@patch('builtins.open', new_callable=mock_open, read_data=MOCK_JSON_DATA)
|