vectorwave 0.1.1__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vectorwave-0.1.1/src/vectorwave.egg-info → vectorwave-0.1.3}/PKG-INFO +96 -36
- {vectorwave-0.1.1 → vectorwave-0.1.3}/Readme.md +93 -32
- {vectorwave-0.1.1 → vectorwave-0.1.3}/pyproject.toml +3 -4
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/tests/batch/test_batch.py +2 -1
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/tests/database/test_db.py +23 -19
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/tests/models/test_db_config.py +21 -12
- vectorwave-0.1.3/src/tests/monitoring/test_tracer.py +202 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/vectorwave/batch/batch.py +11 -8
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/vectorwave/core/decorator.py +30 -7
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/vectorwave/database/db.py +64 -38
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/vectorwave/database/db_search.py +32 -10
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/vectorwave/models/db_config.py +22 -12
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/vectorwave/monitoring/tracer.py +5 -2
- vectorwave-0.1.3/src/vectorwave/prediction/predictor.py +0 -0
- vectorwave-0.1.3/src/vectorwave/vectorizer/__init__.py +0 -0
- vectorwave-0.1.3/src/vectorwave/vectorizer/base.py +12 -0
- vectorwave-0.1.3/src/vectorwave/vectorizer/factory.py +49 -0
- vectorwave-0.1.3/src/vectorwave/vectorizer/huggingface_vectorizer.py +33 -0
- vectorwave-0.1.3/src/vectorwave/vectorizer/openai_vectorizer.py +35 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3/src/vectorwave.egg-info}/PKG-INFO +96 -36
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/vectorwave.egg-info/SOURCES.txt +8 -1
- {vectorwave-0.1.1 → vectorwave-0.1.3}/LICENSE +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/MANIFEST.in +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/NOTICE +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/setup.cfg +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/tests/__init__.py +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/tests/batch/__init__.py +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/tests/core/__init__.py +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/tests/core/test_decorator.py +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/tests/database/__init__.py +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/tests/database/test_db_search.py +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/tests/exception/__init__.py +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/tests/models/__init__.py +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/tests/monitoring/__init__.py +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/tests/prediction/__init__.py +0 -0
- {vectorwave-0.1.1/src/vectorwave/batch → vectorwave-0.1.3/src/tests/vectorizer}/__init__.py +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/vectorwave/__init__.py +0 -0
- {vectorwave-0.1.1/src/vectorwave/core → vectorwave-0.1.3/src/vectorwave/batch}/__init__.py +0 -0
- {vectorwave-0.1.1/src/vectorwave/database → vectorwave-0.1.3/src/vectorwave/core}/__init__.py +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/vectorwave/core/core.py +0 -0
- {vectorwave-0.1.1/src/vectorwave/exception → vectorwave-0.1.3/src/vectorwave/database}/__init__.py +0 -0
- {vectorwave-0.1.1/src/vectorwave/models → vectorwave-0.1.3/src/vectorwave/exception}/__init__.py +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/vectorwave/exception/exceptions.py +0 -0
- {vectorwave-0.1.1/src/vectorwave/monitoring → vectorwave-0.1.3/src/vectorwave/models}/__init__.py +0 -0
- {vectorwave-0.1.1/src/vectorwave/prediction → vectorwave-0.1.3/src/vectorwave/monitoring}/__init__.py +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/vectorwave/monitoring/monitoring.py +0 -0
- /vectorwave-0.1.1/src/vectorwave/prediction/predictor.py → /vectorwave-0.1.3/src/vectorwave/prediction/__init__.py +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/vectorwave.egg-info/dependency_links.txt +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/vectorwave.egg-info/requires.txt +0 -0
- {vectorwave-0.1.1 → vectorwave-0.1.3}/src/vectorwave.egg-info/top_level.txt +0 -0
|
@@ -1,20 +1,19 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vectorwave
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: VectorWave: Seamless Auto-Vectorization Framework
|
|
5
5
|
Author-email: junyeonggim <junyeonggim5@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: Repository, https://github.com/republicofgamja/vtm
|
|
8
8
|
Classifier: Programming Language :: Python :: 3
|
|
9
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
10
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
11
9
|
Classifier: Programming Language :: Python :: 3.10
|
|
12
10
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
11
|
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
13
|
Classifier: Operating System :: OS Independent
|
|
15
14
|
Classifier: Development Status :: 3 - Alpha
|
|
16
15
|
Classifier: Intended Audience :: Developers
|
|
17
|
-
Requires-Python: >=3.
|
|
16
|
+
Requires-Python: >=3.10
|
|
18
17
|
Description-Content-Type: text/markdown
|
|
19
18
|
License-File: LICENSE
|
|
20
19
|
License-File: NOTICE
|
|
@@ -163,51 +162,112 @@ for i, span in enumerate(trace_spans):
|
|
|
163
162
|
# - [Span 2] step_2_send_receipt (202.06ms)
|
|
164
163
|
# - [Span 3] process_payment (333.18ms)
|
|
165
164
|
```
|
|
166
|
-
|
|
167
165
|
-----
|
|
168
166
|
|
|
169
167
|
## ⚙️ Configuration
|
|
170
168
|
|
|
171
|
-
VectorWave automatically reads Weaviate database connection
|
|
169
|
+
VectorWave automatically reads Weaviate database connection info and **vectorization strategy** from **environment variables** or a `.env` file.
|
|
170
|
+
|
|
171
|
+
Create a `.env` file in your project's root directory (e.g., where `test_ex/example.py` is located) and set the required values.
|
|
172
|
+
|
|
173
|
+
### Vectorizer Strategy (VECTORIZER)
|
|
174
|
+
|
|
175
|
+
You can select the text vectorization method via the `VECTORIZER` environment variable in your `test_ex/.env` file.
|
|
176
|
+
|
|
177
|
+
| `VECTORIZER` Setting | Description | Required Additional Settings |
|
|
178
|
+
| :--- | :--- | :--- |
|
|
179
|
+
| **`huggingface`** | (Default Recommended) Uses the `sentence-transformers` library to vectorize on your local CPU. No API key is needed, making it great for immediate testing. | `HF_MODEL_NAME` (e.g., "sentence-transformers/all-MiniLM-L6-v2") |
|
|
180
|
+
| **`openai_client`** | (High-Performance) Uses the OpenAI Python client to vectorize with modern models like `text-embedding-3-small`. | `OPENAI_API_KEY` (A valid OpenAI API key) |
|
|
181
|
+
| **`weaviate_module`** | (Docker Delegate) Delegates the vectorization task to the Weaviate container's built-in module (e.g., `text2vec-openai`). | `WEAVIATE_VECTORIZER_MODULE`, `OPENAI_API_KEY` |
|
|
182
|
+
| **`none`** | Disables vectorization. Data will be stored without vectors. | None |
|
|
183
|
+
|
|
184
|
+
-----
|
|
185
|
+
|
|
186
|
+
### .env File Examples
|
|
187
|
+
|
|
188
|
+
Configure your `.env` file according to the strategy you want to use.
|
|
189
|
+
|
|
190
|
+
#### Example 1: Using `huggingface` (Local, No API Key)
|
|
191
|
+
|
|
192
|
+
Uses a `sentence-transformers` model on your local machine. Ideal for testing without API keys.
|
|
193
|
+
|
|
194
|
+
```ini
|
|
195
|
+
# .env (Using HuggingFace)
|
|
196
|
+
# --- Basic Weaviate Connection ---
|
|
197
|
+
WEAVIATE_HOST=localhost
|
|
198
|
+
WEAVIATE_PORT=8080
|
|
199
|
+
WEAVIATE_GRPC_PORT=50051
|
|
200
|
+
|
|
201
|
+
# --- [Strategy 1] HuggingFace Config ---
|
|
202
|
+
VECTORIZER="huggingface"
|
|
203
|
+
HF_MODEL_NAME="sentence-transformers/all-MiniLM-L6-v2"
|
|
204
|
+
|
|
205
|
+
# (OPENAI_API_KEY is not required for this mode)
|
|
206
|
+
OPENAI_API_KEY=sk-...
|
|
172
207
|
|
|
173
|
-
|
|
208
|
+
# --- [Advanced] Custom Properties ---
|
|
209
|
+
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
210
|
+
RUN_ID=test-run-001
|
|
211
|
+
```
|
|
174
212
|
|
|
175
|
-
|
|
213
|
+
#### Example 2: Using `openai_client` (Python Client, High-Performance)
|
|
214
|
+
|
|
215
|
+
Directly calls the OpenAI API via the `openai` Python library.
|
|
176
216
|
|
|
177
217
|
```ini
|
|
178
|
-
# .env
|
|
179
|
-
# --- Basic Weaviate Connection
|
|
218
|
+
# .env (Using OpenAI Python Client)
|
|
219
|
+
# --- Basic Weaviate Connection ---
|
|
180
220
|
WEAVIATE_HOST=localhost
|
|
181
221
|
WEAVIATE_PORT=8080
|
|
182
222
|
WEAVIATE_GRPC_PORT=50051
|
|
183
223
|
|
|
184
|
-
# ---
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
#
|
|
188
|
-
|
|
189
|
-
# An OpenAI API key is required if using modules like text2vec-openai.
|
|
190
|
-
# OPENAI_API_KEY=sk-your-key-here
|
|
224
|
+
# --- [Strategy 2] OpenAI Client Config ---
|
|
225
|
+
VECTORIZER="openai_client"
|
|
226
|
+
|
|
227
|
+
# [Required] You must enter a valid OpenAI API key.
|
|
228
|
+
OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
191
229
|
|
|
192
|
-
#
|
|
193
|
-
|
|
230
|
+
# (HF_MODEL_NAME is not used in this mode)
|
|
231
|
+
HF_MODEL_NAME=...
|
|
232
|
+
|
|
233
|
+
# --- [Advanced] Custom Properties ---
|
|
194
234
|
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
235
|
+
RUN_ID=test-run-001
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
#### Example 3: Using `weaviate_module` (Docker Delegate)
|
|
195
239
|
|
|
196
|
-
|
|
197
|
-
|
|
240
|
+
Delegates vectorization to the Weaviate Docker container instead of Python. (See `vw_docker.yml` config).
|
|
241
|
+
|
|
242
|
+
```ini
|
|
243
|
+
# .env (Delegating to Weaviate Module)
|
|
244
|
+
# --- Basic Weaviate Connection ---
|
|
245
|
+
WEAVIATE_HOST=localhost
|
|
246
|
+
WEAVIATE_PORT=8080
|
|
247
|
+
WEAVIATE_GRPC_PORT=50051
|
|
248
|
+
|
|
249
|
+
# --- [Strategy 3] Weaviate Module Config ---
|
|
250
|
+
VECTORIZER="weaviate_module"
|
|
251
|
+
WEAVIATE_VECTORIZER_MODULE=text2vec-openai
|
|
252
|
+
WEAVIATE_GENERATIVE_MODULE=generative-openai
|
|
253
|
+
|
|
254
|
+
# [Required] The Weaviate container will read this API key.
|
|
255
|
+
OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
256
|
+
|
|
257
|
+
# --- [Advanced] Custom Properties ---
|
|
258
|
+
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
198
259
|
RUN_ID=test-run-001
|
|
199
|
-
EXPERIMENT_ID=exp-abc
|
|
200
260
|
```
|
|
201
261
|
|
|
202
262
|
-----
|
|
203
263
|
|
|
204
264
|
### Custom Properties and Dynamic Execution Tagging
|
|
205
265
|
|
|
206
|
-
VectorWave can store user-defined metadata in
|
|
266
|
+
VectorWave can store user-defined metadata in addition to static data (function definitions) and dynamic data (execution logs). This works in two steps.
|
|
207
267
|
|
|
208
|
-
#### Step 1: Define Custom Schema (
|
|
268
|
+
#### Step 1: Define Custom Schema (Tag "Allow-list")
|
|
209
269
|
|
|
210
|
-
Create a JSON file at the path specified by `CUSTOM_PROPERTIES_FILE_PATH` (default: `.weaviate_properties`).
|
|
270
|
+
Create a JSON file at the path specified by `CUSTOM_PROPERTIES_FILE_PATH` in your `.env` file (default: `.weaviate_properties`).
|
|
211
271
|
|
|
212
272
|
This file instructs VectorWave to add **new properties (columns)** to the Weaviate collections. This file acts as an **"allow-list"** for all custom tags.
|
|
213
273
|
|
|
@@ -234,17 +294,17 @@ This file instructs VectorWave to add **new properties (columns)** to the Weavia
|
|
|
234
294
|
}
|
|
235
295
|
```
|
|
236
296
|
|
|
237
|
-
*
|
|
297
|
+
* This definition will add `run_id`, `experiment_id`, `team`, and `priority` properties to both the `VectorWaveFunctions` and `VectorWaveExecutions` collections.
|
|
238
298
|
|
|
239
299
|
#### Step 2: Dynamic Execution Tagging (Adding Values)
|
|
240
300
|
|
|
241
|
-
When a function
|
|
301
|
+
When a function is executed, VectorWave adds tags to the `VectorWaveExecutions` log. These tags are collected and merged from two sources.
|
|
242
302
|
|
|
243
|
-
**1. Global Tags (
|
|
244
|
-
VectorWave
|
|
303
|
+
**1. Global Tags (Environment Variables)**
|
|
304
|
+
VectorWave looks for environment variables matching the **UPPERCASE name** of the keys defined in Step 1 (e.g., `RUN_ID`, `EXPERIMENT_ID`). Found values are loaded as `global_custom_values` and added to *all* execution logs. Ideal for run-wide metadata.
|
|
245
305
|
|
|
246
|
-
**2. Function-Specific Tags (
|
|
247
|
-
You can pass tags
|
|
306
|
+
**2. Function-Specific Tags (Decorator)**
|
|
307
|
+
You can pass tags as keyword arguments (`**execution_tags`) directly to the `@vectorize` decorator. Ideal for function-specific metadata.
|
|
248
308
|
|
|
249
309
|
```python
|
|
250
310
|
# --- .env file ---
|
|
@@ -252,7 +312,7 @@ You can pass tags directly to the `@vectorize` decorator as keyword arguments (`
|
|
|
252
312
|
# TEAM=default-team
|
|
253
313
|
|
|
254
314
|
@vectorize(
|
|
255
|
-
search_description="Process
|
|
315
|
+
search_description="Process payment",
|
|
256
316
|
sequence_narrative="...",
|
|
257
317
|
team="billing", # <-- Function-specific tag
|
|
258
318
|
priority=1 # <-- Function-specific tag
|
|
@@ -271,14 +331,14 @@ def other_function():
|
|
|
271
331
|
|
|
272
332
|
**Tag Merging and Validation Rules**
|
|
273
333
|
|
|
274
|
-
1. **Validation (
|
|
334
|
+
1. **Validation (Important):** Tags (global or function-specific) will **only** be saved to Weaviate if their key (e.g., `run_id`, `team`, `priority`) was first defined in the `.weaviate_properties` file (Step 1). Tags not defined in the schema are **ignored**, and a warning is printed at script startup.
|
|
275
335
|
|
|
276
|
-
2. **Priority (Override):** If a tag key is defined in both places (e.g.,
|
|
336
|
+
2. **Priority (Override):** If a tag key is defined in both places (e.g., global `RUN_ID` in `.env` and `run_id="override-xyz"` in the decorator), the **function-specific tag from the decorator always wins**.
|
|
277
337
|
|
|
278
338
|
**Resulting Logs:**
|
|
279
339
|
|
|
280
|
-
* `process_payment()` log
|
|
281
|
-
* `other_function()` log
|
|
340
|
+
* `process_payment()` execution log: `{"run_id": "global-run-abc", "team": "billing", "priority": 1}`
|
|
341
|
+
* `other_function()` execution log: `{"run_id": "override-run-xyz", "team": "default-team"}`
|
|
282
342
|
|
|
283
343
|
-----
|
|
284
344
|
|
|
@@ -139,51 +139,112 @@ for i, span in enumerate(trace_spans):
|
|
|
139
139
|
# - [Span 2] step_2_send_receipt (202.06ms)
|
|
140
140
|
# - [Span 3] process_payment (333.18ms)
|
|
141
141
|
```
|
|
142
|
-
|
|
143
142
|
-----
|
|
144
143
|
|
|
145
144
|
## ⚙️ Configuration
|
|
146
145
|
|
|
147
|
-
VectorWave automatically reads Weaviate database connection
|
|
146
|
+
VectorWave automatically reads Weaviate database connection info and **vectorization strategy** from **environment variables** or a `.env` file.
|
|
147
|
+
|
|
148
|
+
Create a `.env` file in your project's root directory (e.g., where `test_ex/example.py` is located) and set the required values.
|
|
149
|
+
|
|
150
|
+
### Vectorizer Strategy (VECTORIZER)
|
|
151
|
+
|
|
152
|
+
You can select the text vectorization method via the `VECTORIZER` environment variable in your `test_ex/.env` file.
|
|
153
|
+
|
|
154
|
+
| `VECTORIZER` Setting | Description | Required Additional Settings |
|
|
155
|
+
| :--- | :--- | :--- |
|
|
156
|
+
| **`huggingface`** | (Default Recommended) Uses the `sentence-transformers` library to vectorize on your local CPU. No API key is needed, making it great for immediate testing. | `HF_MODEL_NAME` (e.g., "sentence-transformers/all-MiniLM-L6-v2") |
|
|
157
|
+
| **`openai_client`** | (High-Performance) Uses the OpenAI Python client to vectorize with modern models like `text-embedding-3-small`. | `OPENAI_API_KEY` (A valid OpenAI API key) |
|
|
158
|
+
| **`weaviate_module`** | (Docker Delegate) Delegates the vectorization task to the Weaviate container's built-in module (e.g., `text2vec-openai`). | `WEAVIATE_VECTORIZER_MODULE`, `OPENAI_API_KEY` |
|
|
159
|
+
| **`none`** | Disables vectorization. Data will be stored without vectors. | None |
|
|
160
|
+
|
|
161
|
+
-----
|
|
162
|
+
|
|
163
|
+
### .env File Examples
|
|
164
|
+
|
|
165
|
+
Configure your `.env` file according to the strategy you want to use.
|
|
166
|
+
|
|
167
|
+
#### Example 1: Using `huggingface` (Local, No API Key)
|
|
168
|
+
|
|
169
|
+
Uses a `sentence-transformers` model on your local machine. Ideal for testing without API keys.
|
|
170
|
+
|
|
171
|
+
```ini
|
|
172
|
+
# .env (Using HuggingFace)
|
|
173
|
+
# --- Basic Weaviate Connection ---
|
|
174
|
+
WEAVIATE_HOST=localhost
|
|
175
|
+
WEAVIATE_PORT=8080
|
|
176
|
+
WEAVIATE_GRPC_PORT=50051
|
|
177
|
+
|
|
178
|
+
# --- [Strategy 1] HuggingFace Config ---
|
|
179
|
+
VECTORIZER="huggingface"
|
|
180
|
+
HF_MODEL_NAME="sentence-transformers/all-MiniLM-L6-v2"
|
|
181
|
+
|
|
182
|
+
# (OPENAI_API_KEY is not required for this mode)
|
|
183
|
+
OPENAI_API_KEY=sk-...
|
|
148
184
|
|
|
149
|
-
|
|
185
|
+
# --- [Advanced] Custom Properties ---
|
|
186
|
+
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
187
|
+
RUN_ID=test-run-001
|
|
188
|
+
```
|
|
150
189
|
|
|
151
|
-
|
|
190
|
+
#### Example 2: Using `openai_client` (Python Client, High-Performance)
|
|
191
|
+
|
|
192
|
+
Directly calls the OpenAI API via the `openai` Python library.
|
|
152
193
|
|
|
153
194
|
```ini
|
|
154
|
-
# .env
|
|
155
|
-
# --- Basic Weaviate Connection
|
|
195
|
+
# .env (Using OpenAI Python Client)
|
|
196
|
+
# --- Basic Weaviate Connection ---
|
|
156
197
|
WEAVIATE_HOST=localhost
|
|
157
198
|
WEAVIATE_PORT=8080
|
|
158
199
|
WEAVIATE_GRPC_PORT=50051
|
|
159
200
|
|
|
160
|
-
# ---
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
#
|
|
164
|
-
|
|
165
|
-
# An OpenAI API key is required if using modules like text2vec-openai.
|
|
166
|
-
# OPENAI_API_KEY=sk-your-key-here
|
|
201
|
+
# --- [Strategy 2] OpenAI Client Config ---
|
|
202
|
+
VECTORIZER="openai_client"
|
|
203
|
+
|
|
204
|
+
# [Required] You must enter a valid OpenAI API key.
|
|
205
|
+
OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
167
206
|
|
|
168
|
-
#
|
|
169
|
-
|
|
207
|
+
# (HF_MODEL_NAME is not used in this mode)
|
|
208
|
+
HF_MODEL_NAME=...
|
|
209
|
+
|
|
210
|
+
# --- [Advanced] Custom Properties ---
|
|
170
211
|
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
212
|
+
RUN_ID=test-run-001
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
#### Example 3: Using `weaviate_module` (Docker Delegate)
|
|
171
216
|
|
|
172
|
-
|
|
173
|
-
|
|
217
|
+
Delegates vectorization to the Weaviate Docker container instead of Python. (See `vw_docker.yml` config).
|
|
218
|
+
|
|
219
|
+
```ini
|
|
220
|
+
# .env (Delegating to Weaviate Module)
|
|
221
|
+
# --- Basic Weaviate Connection ---
|
|
222
|
+
WEAVIATE_HOST=localhost
|
|
223
|
+
WEAVIATE_PORT=8080
|
|
224
|
+
WEAVIATE_GRPC_PORT=50051
|
|
225
|
+
|
|
226
|
+
# --- [Strategy 3] Weaviate Module Config ---
|
|
227
|
+
VECTORIZER="weaviate_module"
|
|
228
|
+
WEAVIATE_VECTORIZER_MODULE=text2vec-openai
|
|
229
|
+
WEAVIATE_GENERATIVE_MODULE=generative-openai
|
|
230
|
+
|
|
231
|
+
# [Required] The Weaviate container will read this API key.
|
|
232
|
+
OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
233
|
+
|
|
234
|
+
# --- [Advanced] Custom Properties ---
|
|
235
|
+
CUSTOM_PROPERTIES_FILE_PATH=.weaviate_properties
|
|
174
236
|
RUN_ID=test-run-001
|
|
175
|
-
EXPERIMENT_ID=exp-abc
|
|
176
237
|
```
|
|
177
238
|
|
|
178
239
|
-----
|
|
179
240
|
|
|
180
241
|
### Custom Properties and Dynamic Execution Tagging
|
|
181
242
|
|
|
182
|
-
VectorWave can store user-defined metadata in
|
|
243
|
+
VectorWave can store user-defined metadata in addition to static data (function definitions) and dynamic data (execution logs). This works in two steps.
|
|
183
244
|
|
|
184
|
-
#### Step 1: Define Custom Schema (
|
|
245
|
+
#### Step 1: Define Custom Schema (Tag "Allow-list")
|
|
185
246
|
|
|
186
|
-
Create a JSON file at the path specified by `CUSTOM_PROPERTIES_FILE_PATH` (default: `.weaviate_properties`).
|
|
247
|
+
Create a JSON file at the path specified by `CUSTOM_PROPERTIES_FILE_PATH` in your `.env` file (default: `.weaviate_properties`).
|
|
187
248
|
|
|
188
249
|
This file instructs VectorWave to add **new properties (columns)** to the Weaviate collections. This file acts as an **"allow-list"** for all custom tags.
|
|
189
250
|
|
|
@@ -210,17 +271,17 @@ This file instructs VectorWave to add **new properties (columns)** to the Weavia
|
|
|
210
271
|
}
|
|
211
272
|
```
|
|
212
273
|
|
|
213
|
-
*
|
|
274
|
+
* This definition will add `run_id`, `experiment_id`, `team`, and `priority` properties to both the `VectorWaveFunctions` and `VectorWaveExecutions` collections.
|
|
214
275
|
|
|
215
276
|
#### Step 2: Dynamic Execution Tagging (Adding Values)
|
|
216
277
|
|
|
217
|
-
When a function
|
|
278
|
+
When a function is executed, VectorWave adds tags to the `VectorWaveExecutions` log. These tags are collected and merged from two sources.
|
|
218
279
|
|
|
219
|
-
**1. Global Tags (
|
|
220
|
-
VectorWave
|
|
280
|
+
**1. Global Tags (Environment Variables)**
|
|
281
|
+
VectorWave looks for environment variables matching the **UPPERCASE name** of the keys defined in Step 1 (e.g., `RUN_ID`, `EXPERIMENT_ID`). Found values are loaded as `global_custom_values` and added to *all* execution logs. Ideal for run-wide metadata.
|
|
221
282
|
|
|
222
|
-
**2. Function-Specific Tags (
|
|
223
|
-
You can pass tags
|
|
283
|
+
**2. Function-Specific Tags (Decorator)**
|
|
284
|
+
You can pass tags as keyword arguments (`**execution_tags`) directly to the `@vectorize` decorator. Ideal for function-specific metadata.
|
|
224
285
|
|
|
225
286
|
```python
|
|
226
287
|
# --- .env file ---
|
|
@@ -228,7 +289,7 @@ You can pass tags directly to the `@vectorize` decorator as keyword arguments (`
|
|
|
228
289
|
# TEAM=default-team
|
|
229
290
|
|
|
230
291
|
@vectorize(
|
|
231
|
-
search_description="Process
|
|
292
|
+
search_description="Process payment",
|
|
232
293
|
sequence_narrative="...",
|
|
233
294
|
team="billing", # <-- Function-specific tag
|
|
234
295
|
priority=1 # <-- Function-specific tag
|
|
@@ -247,14 +308,14 @@ def other_function():
|
|
|
247
308
|
|
|
248
309
|
**Tag Merging and Validation Rules**
|
|
249
310
|
|
|
250
|
-
1. **Validation (
|
|
311
|
+
1. **Validation (Important):** Tags (global or function-specific) will **only** be saved to Weaviate if their key (e.g., `run_id`, `team`, `priority`) was first defined in the `.weaviate_properties` file (Step 1). Tags not defined in the schema are **ignored**, and a warning is printed at script startup.
|
|
251
312
|
|
|
252
|
-
2. **Priority (Override):** If a tag key is defined in both places (e.g.,
|
|
313
|
+
2. **Priority (Override):** If a tag key is defined in both places (e.g., global `RUN_ID` in `.env` and `run_id="override-xyz"` in the decorator), the **function-specific tag from the decorator always wins**.
|
|
253
314
|
|
|
254
315
|
**Resulting Logs:**
|
|
255
316
|
|
|
256
|
-
* `process_payment()` log
|
|
257
|
-
* `other_function()` log
|
|
317
|
+
* `process_payment()` execution log: `{"run_id": "global-run-abc", "team": "billing", "priority": 1}`
|
|
318
|
+
* `other_function()` execution log: `{"run_id": "override-run-xyz", "team": "default-team"}`
|
|
258
319
|
|
|
259
320
|
-----
|
|
260
321
|
|
|
@@ -4,21 +4,20 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "vectorwave"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.3"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name = "junyeonggim", email = "junyeonggim5@gmail.com" },
|
|
10
10
|
]
|
|
11
11
|
description = "VectorWave: Seamless Auto-Vectorization Framework"
|
|
12
12
|
readme = "Readme.md"
|
|
13
|
-
requires-python = ">=3.
|
|
13
|
+
requires-python = ">=3.10"
|
|
14
14
|
license = "MIT"
|
|
15
15
|
classifiers = [
|
|
16
16
|
"Programming Language :: Python :: 3",
|
|
17
|
-
"Programming Language :: Python :: 3.8",
|
|
18
|
-
"Programming Language :: Python :: 3.9",
|
|
19
17
|
"Programming Language :: Python :: 3.10",
|
|
20
18
|
"Programming Language :: Python :: 3.11",
|
|
21
19
|
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
22
21
|
"Operating System :: OS Independent",
|
|
23
22
|
"Development Status :: 3 - Alpha",
|
|
24
23
|
"Intended Audience :: Developers",
|
|
@@ -392,16 +392,12 @@ def test_create_schema_vectorizer_openai(test_settings):
|
|
|
392
392
|
mock_collections.exists.return_value = False
|
|
393
393
|
mock_client.collections = mock_collections
|
|
394
394
|
|
|
395
|
-
test_settings.
|
|
396
|
-
test_settings.
|
|
395
|
+
test_settings.VECTORIZER = "weaviate_module"
|
|
396
|
+
test_settings.WEAVIATE_VECTORIZER_MODULE = "text2vec-openai"
|
|
397
|
+
|
|
398
|
+
# [수정] test_settings.GENERATIVE_CONFIG = "generative-openai" ->
|
|
399
|
+
test_settings.WEAVIATE_GENERATIVE_MODULE = "generative-openai"
|
|
397
400
|
|
|
398
|
-
expected_vector_config = {
|
|
399
|
-
"vectorizer": "text2vec-openai",
|
|
400
|
-
"text2vec-openai": {
|
|
401
|
-
"vectorizeClassName": test_settings.IS_VECTORIZE_COLLECTION_NAME,
|
|
402
|
-
}
|
|
403
|
-
}
|
|
404
|
-
expected_generative_config = {"generator": "generative-openai"}
|
|
405
401
|
|
|
406
402
|
# 2. Act
|
|
407
403
|
create_vectorwave_schema(mock_client, test_settings)
|
|
@@ -411,9 +407,16 @@ def test_create_schema_vectorizer_openai(test_settings):
|
|
|
411
407
|
|
|
412
408
|
call_args = mock_collections.create.call_args
|
|
413
409
|
|
|
410
|
+
vector_config_arg = call_args.kwargs.get('vector_config')
|
|
411
|
+
# assert isinstance(vector_config_arg, wvc.Configure.Vectorizer)
|
|
412
|
+
# assert vector_config_arg.name == "text2vec-openai"
|
|
413
|
+
|
|
414
|
+
assert vector_config_arg.vectorizer == "text2vec-openai"
|
|
415
|
+
|
|
416
|
+
# 3b. Generative Config 검증
|
|
417
|
+
generative_config_arg = call_args.kwargs.get('generative_config')
|
|
418
|
+
assert generative_config_arg.generative == "generative-openai"
|
|
414
419
|
|
|
415
|
-
assert call_args.kwargs.get('vector_config') == expected_vector_config
|
|
416
|
-
assert call_args.kwargs.get('generative_config') == expected_generative_config
|
|
417
420
|
|
|
418
421
|
|
|
419
422
|
|
|
@@ -427,9 +430,8 @@ def test_create_schema_vectorizer_none(test_settings):
|
|
|
427
430
|
mock_collections.exists.return_value = False
|
|
428
431
|
mock_client.collections = mock_collections
|
|
429
432
|
|
|
430
|
-
test_settings.
|
|
431
|
-
|
|
432
|
-
expected_vector_config = {"vectorizer": "none"}
|
|
433
|
+
test_settings.VECTORIZER = "none"
|
|
434
|
+
test_settings.WEAVIATE_VECTORIZER_MODULE = ""
|
|
433
435
|
|
|
434
436
|
# 2. Act
|
|
435
437
|
create_vectorwave_schema(mock_client, test_settings)
|
|
@@ -440,7 +442,10 @@ def test_create_schema_vectorizer_none(test_settings):
|
|
|
440
442
|
call_args = mock_collections.create.call_args
|
|
441
443
|
# mock_none.assert_called_once() -> 삭제
|
|
442
444
|
|
|
443
|
-
|
|
445
|
+
vector_config_arg = call_args.kwargs.get('vector_config')
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
assert vector_config_arg.vectorizer == "none"
|
|
444
449
|
|
|
445
450
|
|
|
446
451
|
def test_create_schema_vectorizer_invalid(test_settings):
|
|
@@ -453,12 +458,11 @@ def test_create_schema_vectorizer_invalid(test_settings):
|
|
|
453
458
|
mock_collections.exists.return_value = False
|
|
454
459
|
mock_client.collections = mock_collections
|
|
455
460
|
|
|
456
|
-
test_settings.
|
|
461
|
+
test_settings.VECTORIZER = "unsupported-module"
|
|
457
462
|
|
|
458
463
|
# 2. Act & 3. Assert
|
|
459
464
|
with pytest.raises(SchemaCreationError) as exc_info:
|
|
460
465
|
create_vectorwave_schema(mock_client, test_settings)
|
|
461
466
|
|
|
462
|
-
assert "
|
|
463
|
-
assert "
|
|
464
|
-
assert "text2vec-huggingface" not in str(exc_info.value)
|
|
467
|
+
assert "Invalid VECTORIZER setting" in str(exc_info.value)
|
|
468
|
+
assert "unsupported-module" in str(exc_info.value)
|
|
@@ -60,13 +60,16 @@ def test_get_settings_loads_custom_props_success(mock_open_file, mock_exists):
|
|
|
60
60
|
|
|
61
61
|
|
|
62
62
|
@patch('os.path.exists', return_value=False)
|
|
63
|
-
def test_get_settings_file_not_found(mock_exists,
|
|
63
|
+
def test_get_settings_file_not_found(mock_exists, caplog):
|
|
64
64
|
"""
|
|
65
65
|
Case 2: .weaviate_properties file does not exist
|
|
66
66
|
- settings.custom_properties should be None
|
|
67
|
-
- A 'file not found'
|
|
67
|
+
- A 'file not found' message should be logged at DEBUG level
|
|
68
68
|
"""
|
|
69
|
+
import logging
|
|
70
|
+
|
|
69
71
|
# Arrange
|
|
72
|
+
caplog.set_level(logging.DEBUG) # DEBUG 레벨로 설정 (중요!)
|
|
70
73
|
get_weaviate_settings.cache_clear()
|
|
71
74
|
|
|
72
75
|
# Act
|
|
@@ -76,21 +79,23 @@ def test_get_settings_file_not_found(mock_exists, capsys):
|
|
|
76
79
|
mock_exists.assert_called_with(".weaviate_properties")
|
|
77
80
|
assert settings.custom_properties is None
|
|
78
81
|
|
|
79
|
-
# Check if 'file not found'
|
|
80
|
-
|
|
81
|
-
assert "file not found" in captured.out
|
|
82
|
+
# Check if 'file not found' message was logged
|
|
83
|
+
assert "file not found" in caplog.text.lower() or "not found" in caplog.text
|
|
82
84
|
|
|
83
85
|
|
|
84
86
|
@patch('os.path.exists', return_value=True)
|
|
85
87
|
@patch('builtins.open', new_callable=mock_open, read_data=MOCK_INVALID_JSON)
|
|
86
88
|
@patch('json.load', side_effect=JSONDecodeError("Mock JSON Decode Error", "", 0))
|
|
87
|
-
def test_get_settings_invalid_json(mock_json_load, mock_open_file, mock_exists,
|
|
89
|
+
def test_get_settings_invalid_json(mock_json_load, mock_open_file, mock_exists, caplog):
|
|
88
90
|
"""
|
|
89
91
|
Case 3: File exists but JSON format is invalid
|
|
90
92
|
- settings.custom_properties should be None
|
|
91
|
-
- A 'Could not parse JSON' warning should be
|
|
93
|
+
- A 'Could not parse JSON' warning should be logged
|
|
92
94
|
"""
|
|
95
|
+
import logging
|
|
96
|
+
|
|
93
97
|
# Arrange
|
|
98
|
+
caplog.set_level(logging.WARNING)
|
|
94
99
|
get_weaviate_settings.cache_clear()
|
|
95
100
|
|
|
96
101
|
# Act
|
|
@@ -99,12 +104,16 @@ def test_get_settings_invalid_json(mock_json_load, mock_open_file, mock_exists,
|
|
|
99
104
|
# Assert
|
|
100
105
|
mock_exists.assert_called_once()
|
|
101
106
|
mock_open_file.assert_called_once()
|
|
102
|
-
mock_json_load.assert_called_once()
|
|
103
|
-
assert settings.custom_properties is None
|
|
107
|
+
mock_json_load.assert_called_once() # json.load was called but failed (due to side_effect)
|
|
108
|
+
assert settings.custom_properties is None # Should be None due to parsing failure
|
|
109
|
+
|
|
110
|
+
# Check if 'Could not parse JSON' warning was logged
|
|
111
|
+
assert "Could not parse JSON" in caplog.text
|
|
104
112
|
|
|
105
|
-
#
|
|
106
|
-
|
|
107
|
-
assert
|
|
113
|
+
# Also check the log level
|
|
114
|
+
warning_logs = [r for r in caplog.records if "parse JSON" in r.message]
|
|
115
|
+
assert len(warning_logs) > 0
|
|
116
|
+
assert warning_logs[0].levelname == "WARNING"
|
|
108
117
|
|
|
109
118
|
@patch('os.path.exists', return_value=True)
|
|
110
119
|
@patch('builtins.open', new_callable=mock_open, read_data=MOCK_JSON_DATA)
|