vectorwave 0.2.5__tar.gz → 0.2.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vectorwave-0.2.7/PKG-INFO +60 -0
- vectorwave-0.2.7/Readme.md +33 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/crates/Cargo.lock +13 -85
- {vectorwave-0.2.5 → vectorwave-0.2.7}/crates/Cargo.toml +2 -3
- vectorwave-0.2.7/crates/src/lib.rs +171 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/pyproject.toml +1 -1
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/monitoring/tracer.py +11 -34
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/utils/replayer.py +2 -2
- vectorwave-0.2.5/PKG-INFO +0 -680
- vectorwave-0.2.5/Readme.md +0 -653
- vectorwave-0.2.5/crates/src/lib.rs +0 -148
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/__init__.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/batch/__init__.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/batch/batch.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/core/__init__.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/core/auto_injector.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/core/core.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/core/decorator.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/core/generator.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/core/llm/__init__.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/core/llm/base.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/core/llm/factory.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/core/llm/openai_client.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/database/__init__.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/database/archiver.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/database/dataset.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/database/db.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/database/db_search.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/exception/__init__.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/exception/exceptions.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/models/__init__.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/models/db_config.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/monitoring/__init__.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/monitoring/alert/__init__.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/monitoring/alert/base.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/monitoring/alert/factory.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/monitoring/alert/null_alerter.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/monitoring/alert/webhook_alerter.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/monitoring/monitoring.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/prediction/__init__.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/prediction/predictor.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/search/__init__.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/search/execution_search.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/search/extended_search.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/search/rag_search.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/utils/__init__.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/utils/context.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/utils/function_cache.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/utils/healer.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/utils/replayer_semantic.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/utils/return_caching_utils.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/utils/status.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/vectorizer/__init__.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/vectorizer/base.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/vectorizer/factory.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/vectorizer/huggingface_vectorizer.py +0 -0
- {vectorwave-0.2.5 → vectorwave-0.2.7}/src/vectorwave/vectorizer/openai_vectorizer.py +0 -0
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: vectorwave
|
|
3
|
+
Version: 0.2.7
|
|
4
|
+
Classifier: Programming Language :: Python :: 3
|
|
5
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
6
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
9
|
+
Classifier: Programming Language :: Rust
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Requires-Dist: weaviate-client>=4.0.0
|
|
14
|
+
Requires-Dist: pydantic-settings>=2.0.0
|
|
15
|
+
Requires-Dist: sentence-transformers
|
|
16
|
+
Requires-Dist: requests
|
|
17
|
+
Requires-Dist: openai
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
License-File: NOTICE
|
|
20
|
+
Summary: VectorWave: Seamless Auto-Vectorization Framework
|
|
21
|
+
Author-email: junyeonggim <junyeonggim5@gmail.com>
|
|
22
|
+
License-Expression: MIT
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
25
|
+
Project-URL: Repository, https://github.com/cozymori/vectorwave
|
|
26
|
+
|
|
27
|
+
Need more information? Visit [here](https://cozymori.github.io/vectorwave-docs/)
|
|
28
|
+
|
|
29
|
+
# VectorWave
|
|
30
|
+

|
|
31
|
+

|
|
32
|
+

|
|
33
|
+

|
|
34
|
+

|
|
35
|
+

|
|
36
|
+

|
|
37
|
+

|
|
38
|
+

|
|
39
|
+
<br>**Seamless Auto-Vectorization Framework**
|
|
40
|
+
|
|
41
|
+
We transform volatile data that disappears the moment code is executed into a Searchable and Reusable permanent knowledge asset.
|
|
42
|
+
|
|
43
|
+
### Requirements
|
|
44
|
+
|
|
45
|
+
* **Python**: 3.10 ~ 3.13
|
|
46
|
+
* **Docker**: Required to run the Weaviate database.
|
|
47
|
+
* (Optional) **OpenAI API Key**: Required for AI auto-documentation and high-performance embedding.
|
|
48
|
+
|
|
49
|
+
### How to reach us
|
|
50
|
+
|
|
51
|
+
Have questions or found a bug? Please join our community.
|
|
52
|
+
|
|
53
|
+
* **GitHub Issues**: [https://github.com/cozymori/vectorwave/issues](https://github.com/cozymori/vectorwave/issues)
|
|
54
|
+
|
|
55
|
+
### Contributors
|
|
56
|
+
[See the contributors of vectorwave](https://github.com/Cozymori/VectorWave/graphs/contributors)<br>
|
|
57
|
+
VectorWave is an open-source project and we welcome your contributions.
|
|
58
|
+
Please refer to [CONTRIBUTING.md](https://www.google.com/search?q=https://github.com/cozymori/vectorwave/blob/main/CONTRIBUTING.md) in the GitHub repository.
|
|
59
|
+
|
|
60
|
+
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
Need more information? Visit [here](https://cozymori.github.io/vectorwave-docs/)
|
|
2
|
+
|
|
3
|
+
# VectorWave
|
|
4
|
+

|
|
5
|
+

|
|
6
|
+

|
|
7
|
+

|
|
8
|
+

|
|
9
|
+

|
|
10
|
+

|
|
11
|
+

|
|
12
|
+

|
|
13
|
+
<br>**Seamless Auto-Vectorization Framework**
|
|
14
|
+
|
|
15
|
+
We transform volatile data that disappears the moment code is executed into a Searchable and Reusable permanent knowledge asset.
|
|
16
|
+
|
|
17
|
+
### Requirements
|
|
18
|
+
|
|
19
|
+
* **Python**: 3.10 ~ 3.13
|
|
20
|
+
* **Docker**: Required to run the Weaviate database.
|
|
21
|
+
* (Optional) **OpenAI API Key**: Required for AI auto-documentation and high-performance embedding.
|
|
22
|
+
|
|
23
|
+
### How to reach us
|
|
24
|
+
|
|
25
|
+
Have questions or found a bug? Please join our community.
|
|
26
|
+
|
|
27
|
+
* **GitHub Issues**: [https://github.com/cozymori/vectorwave/issues](https://github.com/cozymori/vectorwave/issues)
|
|
28
|
+
|
|
29
|
+
### Contributors
|
|
30
|
+
[See the contributors of vectorwave](https://github.com/Cozymori/VectorWave/graphs/contributors)<br>
|
|
31
|
+
VectorWave is an open-source project and we welcome your contributions.
|
|
32
|
+
Please refer to [CONTRIBUTING.md](https://www.google.com/search?q=https://github.com/cozymori/vectorwave/blob/main/CONTRIBUTING.md) in the GitHub repository.
|
|
33
|
+
|
|
@@ -8,12 +8,6 @@ version = "1.5.0"
|
|
|
8
8
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
9
|
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
|
10
10
|
|
|
11
|
-
[[package]]
|
|
12
|
-
name = "bitflags"
|
|
13
|
-
version = "2.10.0"
|
|
14
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
15
|
-
checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3"
|
|
16
|
-
|
|
17
11
|
[[package]]
|
|
18
12
|
name = "cfg-if"
|
|
19
13
|
version = "1.0.4"
|
|
@@ -37,9 +31,9 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
|
|
|
37
31
|
|
|
38
32
|
[[package]]
|
|
39
33
|
name = "heck"
|
|
40
|
-
version = "0.
|
|
34
|
+
version = "0.5.0"
|
|
41
35
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
42
|
-
checksum = "
|
|
36
|
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
43
37
|
|
|
44
38
|
[[package]]
|
|
45
39
|
name = "indoc"
|
|
@@ -62,21 +56,6 @@ version = "0.2.178"
|
|
|
62
56
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
63
57
|
checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091"
|
|
64
58
|
|
|
65
|
-
[[package]]
|
|
66
|
-
name = "lock_api"
|
|
67
|
-
version = "0.4.14"
|
|
68
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
69
|
-
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
|
|
70
|
-
dependencies = [
|
|
71
|
-
"scopeguard",
|
|
72
|
-
]
|
|
73
|
-
|
|
74
|
-
[[package]]
|
|
75
|
-
name = "log"
|
|
76
|
-
version = "0.4.29"
|
|
77
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
78
|
-
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
|
79
|
-
|
|
80
59
|
[[package]]
|
|
81
60
|
name = "memchr"
|
|
82
61
|
version = "2.7.6"
|
|
@@ -98,29 +77,6 @@ version = "1.21.3"
|
|
|
98
77
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
99
78
|
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
|
|
100
79
|
|
|
101
|
-
[[package]]
|
|
102
|
-
name = "parking_lot"
|
|
103
|
-
version = "0.12.5"
|
|
104
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
105
|
-
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
|
|
106
|
-
dependencies = [
|
|
107
|
-
"lock_api",
|
|
108
|
-
"parking_lot_core",
|
|
109
|
-
]
|
|
110
|
-
|
|
111
|
-
[[package]]
|
|
112
|
-
name = "parking_lot_core"
|
|
113
|
-
version = "0.9.12"
|
|
114
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
115
|
-
checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
|
|
116
|
-
dependencies = [
|
|
117
|
-
"cfg-if",
|
|
118
|
-
"libc",
|
|
119
|
-
"redox_syscall",
|
|
120
|
-
"smallvec",
|
|
121
|
-
"windows-link",
|
|
122
|
-
]
|
|
123
|
-
|
|
124
80
|
[[package]]
|
|
125
81
|
name = "portable-atomic"
|
|
126
82
|
version = "1.12.0"
|
|
@@ -138,15 +94,15 @@ dependencies = [
|
|
|
138
94
|
|
|
139
95
|
[[package]]
|
|
140
96
|
name = "pyo3"
|
|
141
|
-
version = "0.
|
|
97
|
+
version = "0.23.5"
|
|
142
98
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
143
|
-
checksum = "
|
|
99
|
+
checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872"
|
|
144
100
|
dependencies = [
|
|
145
101
|
"cfg-if",
|
|
146
102
|
"indoc",
|
|
147
103
|
"libc",
|
|
148
104
|
"memoffset",
|
|
149
|
-
"
|
|
105
|
+
"once_cell",
|
|
150
106
|
"portable-atomic",
|
|
151
107
|
"pyo3-build-config",
|
|
152
108
|
"pyo3-ffi",
|
|
@@ -156,9 +112,9 @@ dependencies = [
|
|
|
156
112
|
|
|
157
113
|
[[package]]
|
|
158
114
|
name = "pyo3-build-config"
|
|
159
|
-
version = "0.
|
|
115
|
+
version = "0.23.5"
|
|
160
116
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
161
|
-
checksum = "
|
|
117
|
+
checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb"
|
|
162
118
|
dependencies = [
|
|
163
119
|
"once_cell",
|
|
164
120
|
"target-lexicon",
|
|
@@ -166,9 +122,9 @@ dependencies = [
|
|
|
166
122
|
|
|
167
123
|
[[package]]
|
|
168
124
|
name = "pyo3-ffi"
|
|
169
|
-
version = "0.
|
|
125
|
+
version = "0.23.5"
|
|
170
126
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
171
|
-
checksum = "
|
|
127
|
+
checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d"
|
|
172
128
|
dependencies = [
|
|
173
129
|
"libc",
|
|
174
130
|
"pyo3-build-config",
|
|
@@ -176,9 +132,9 @@ dependencies = [
|
|
|
176
132
|
|
|
177
133
|
[[package]]
|
|
178
134
|
name = "pyo3-macros"
|
|
179
|
-
version = "0.
|
|
135
|
+
version = "0.23.5"
|
|
180
136
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
181
|
-
checksum = "
|
|
137
|
+
checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da"
|
|
182
138
|
dependencies = [
|
|
183
139
|
"proc-macro2",
|
|
184
140
|
"pyo3-macros-backend",
|
|
@@ -188,9 +144,9 @@ dependencies = [
|
|
|
188
144
|
|
|
189
145
|
[[package]]
|
|
190
146
|
name = "pyo3-macros-backend"
|
|
191
|
-
version = "0.
|
|
147
|
+
version = "0.23.5"
|
|
192
148
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
193
|
-
checksum = "
|
|
149
|
+
checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028"
|
|
194
150
|
dependencies = [
|
|
195
151
|
"heck",
|
|
196
152
|
"proc-macro2",
|
|
@@ -208,15 +164,6 @@ dependencies = [
|
|
|
208
164
|
"proc-macro2",
|
|
209
165
|
]
|
|
210
166
|
|
|
211
|
-
[[package]]
|
|
212
|
-
name = "redox_syscall"
|
|
213
|
-
version = "0.5.18"
|
|
214
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
215
|
-
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
|
|
216
|
-
dependencies = [
|
|
217
|
-
"bitflags",
|
|
218
|
-
]
|
|
219
|
-
|
|
220
167
|
[[package]]
|
|
221
168
|
name = "rustversion"
|
|
222
169
|
version = "1.0.22"
|
|
@@ -229,12 +176,6 @@ version = "1.0.21"
|
|
|
229
176
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
230
177
|
checksum = "62049b2877bf12821e8f9ad256ee38fdc31db7387ec2d3b3f403024de2034aea"
|
|
231
178
|
|
|
232
|
-
[[package]]
|
|
233
|
-
name = "scopeguard"
|
|
234
|
-
version = "1.2.0"
|
|
235
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
236
|
-
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
|
237
|
-
|
|
238
179
|
[[package]]
|
|
239
180
|
name = "serde"
|
|
240
181
|
version = "1.0.228"
|
|
@@ -278,12 +219,6 @@ dependencies = [
|
|
|
278
219
|
"serde_core",
|
|
279
220
|
]
|
|
280
221
|
|
|
281
|
-
[[package]]
|
|
282
|
-
name = "smallvec"
|
|
283
|
-
version = "1.15.1"
|
|
284
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
285
|
-
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
|
|
286
|
-
|
|
287
222
|
[[package]]
|
|
288
223
|
name = "syn"
|
|
289
224
|
version = "2.0.111"
|
|
@@ -318,14 +253,7 @@ name = "vectorwave_core"
|
|
|
318
253
|
version = "0.1.0"
|
|
319
254
|
dependencies = [
|
|
320
255
|
"crossbeam-channel",
|
|
321
|
-
"log",
|
|
322
256
|
"pyo3",
|
|
323
257
|
"serde",
|
|
324
258
|
"serde_json",
|
|
325
259
|
]
|
|
326
|
-
|
|
327
|
-
[[package]]
|
|
328
|
-
name = "windows-link"
|
|
329
|
-
version = "0.2.1"
|
|
330
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
331
|
-
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
|
@@ -8,8 +8,7 @@ name = "vectorwave_core"
|
|
|
8
8
|
crate-type = ["cdylib"]
|
|
9
9
|
|
|
10
10
|
[dependencies]
|
|
11
|
-
pyo3 = { version = "0.
|
|
12
|
-
crossbeam-channel = "0.5"
|
|
11
|
+
pyo3 = { version = "0.23.3", features = ["extension-module"] }
|
|
13
12
|
serde = { version = "1.0", features = ["derive"] }
|
|
14
13
|
serde_json = "1.0"
|
|
15
|
-
|
|
14
|
+
crossbeam-channel = "0.5"
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
use pyo3::prelude::*;
|
|
2
|
+
use pyo3::types::{PyDict, PyList, PyString, PyBool, PyFloat, PyInt};
|
|
3
|
+
use std::collections::HashSet;
|
|
4
|
+
use std::thread;
|
|
5
|
+
use std::time::{Duration, Instant};
|
|
6
|
+
use std::sync::Mutex;
|
|
7
|
+
use crossbeam_channel::{bounded, Receiver, Sender, TrySendError};
|
|
8
|
+
|
|
9
|
+
struct LogItem {
|
|
10
|
+
collection: PyObject,
|
|
11
|
+
properties: Py<PyDict>,
|
|
12
|
+
uuid: Option<PyObject>,
|
|
13
|
+
vector: Option<Vec<f32>>,
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
#[pyclass]
|
|
17
|
+
struct RustBatchManager {
|
|
18
|
+
sender: Sender<LogItem>,
|
|
19
|
+
#[allow(dead_code)]
|
|
20
|
+
flush_callback: PyObject,
|
|
21
|
+
worker_handle: Mutex<Option<thread::JoinHandle<()>>>,
|
|
22
|
+
stop_signal: Sender<()>,
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
#[pymethods]
|
|
26
|
+
impl RustBatchManager {
|
|
27
|
+
#[new]
|
|
28
|
+
fn new(
|
|
29
|
+
py: Python<'_>,
|
|
30
|
+
callback: PyObject,
|
|
31
|
+
batch_threshold: usize,
|
|
32
|
+
flush_interval_ms: u64,
|
|
33
|
+
) -> Self {
|
|
34
|
+
let (tx, rx) = bounded::<LogItem>(10000);
|
|
35
|
+
let (stop_tx, stop_rx) = bounded(1);
|
|
36
|
+
let worker_callback = callback.clone_ref(py);
|
|
37
|
+
|
|
38
|
+
let handle = thread::spawn(move || {
|
|
39
|
+
Self::worker_loop(rx, stop_rx, worker_callback, batch_threshold, flush_interval_ms);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
RustBatchManager {
|
|
43
|
+
sender: tx,
|
|
44
|
+
flush_callback: callback,
|
|
45
|
+
worker_handle: Mutex::new(Some(handle)),
|
|
46
|
+
stop_signal: stop_tx,
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
#[pyo3(signature = (collection, properties, uuid=None, vector=None))]
|
|
51
|
+
fn add_object(&self, collection: PyObject, properties: Py<PyDict>, uuid: Option<PyObject>, vector: Option<Vec<f32>>) {
|
|
52
|
+
let item = LogItem { collection, properties, uuid, vector };
|
|
53
|
+
let _ = self.sender.try_send(item);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
fn shutdown(&self, py: Python<'_>) {
|
|
57
|
+
let _ = self.stop_signal.send(());
|
|
58
|
+
py.allow_threads(|| {
|
|
59
|
+
if let Ok(mut handle_guard) = self.worker_handle.lock() {
|
|
60
|
+
if let Some(handle) = handle_guard.take() {
|
|
61
|
+
let _ = handle.join();
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
impl RustBatchManager {
|
|
69
|
+
fn worker_loop(rx: Receiver<LogItem>, stop_rx: Receiver<()>, callback: PyObject, threshold: usize, interval_ms: u64) {
|
|
70
|
+
let mut buffer = Vec::with_capacity(threshold);
|
|
71
|
+
let mut last_flush = Instant::now();
|
|
72
|
+
let flush_interval = Duration::from_millis(interval_ms);
|
|
73
|
+
|
|
74
|
+
loop {
|
|
75
|
+
crossbeam_channel::select! {
|
|
76
|
+
recv(stop_rx) -> msg => {
|
|
77
|
+
break;
|
|
78
|
+
}
|
|
79
|
+
recv(rx) -> msg => {
|
|
80
|
+
match msg {
|
|
81
|
+
Ok(item) => buffer.push(item),
|
|
82
|
+
Err(_) => break,
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
default(Duration::from_millis(100)) => {}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
if stop_rx.is_empty() && (buffer.len() >= threshold || (last_flush.elapsed() >= flush_interval && !buffer.is_empty())) {
|
|
90
|
+
|
|
91
|
+
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
|
92
|
+
Self::flush_buffer(&buffer, &callback);
|
|
93
|
+
}));
|
|
94
|
+
buffer.clear();
|
|
95
|
+
last_flush = Instant::now();
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
fn flush_buffer(buffer: &Vec<LogItem>, callback: &PyObject) {
|
|
101
|
+
Python::with_gil(|py| {
|
|
102
|
+
let py_list = PyList::empty(py);
|
|
103
|
+
|
|
104
|
+
for item in buffer {
|
|
105
|
+
let dict = PyDict::new(py);
|
|
106
|
+
let mut set_field = |key: &str, val: PyObject| -> bool {
|
|
107
|
+
if let Err(_) = dict.set_item(key, val) { return false; }
|
|
108
|
+
true
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
if !set_field("collection", item.collection.clone_ref(py)) { continue; }
|
|
112
|
+
if !set_field("properties", item.properties.to_object(py)) { continue; }
|
|
113
|
+
let uuid_val = item.uuid.as_ref().map_or(py.None(), |u| u.clone_ref(py));
|
|
114
|
+
if !set_field("uuid", uuid_val) { continue; }
|
|
115
|
+
let vector_val = item.vector.as_ref().map_or(py.None(), |v| v.to_object(py));
|
|
116
|
+
if !set_field("vector", vector_val) { continue; }
|
|
117
|
+
|
|
118
|
+
if let Err(e) = py_list.append(dict) {
|
|
119
|
+
eprintln!("[RustCore] ⚠️ Failed to append item: {}", e);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if let Err(e) = callback.call1(py, (py_list,)) {
|
|
124
|
+
eprintln!("[RustCore] ⚠️ Flush failed: {}", e);
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
fn process_recursive(py: Python, value: &Bound<'_, PyAny>, sensitive_set: &HashSet<String>) -> PyResult<PyObject> {
|
|
131
|
+
if let Ok(dict_obj) = value.downcast::<PyDict>() {
|
|
132
|
+
let new_dict = PyDict::new(py);
|
|
133
|
+
for (k, v) in dict_obj {
|
|
134
|
+
let k_str = k.to_string().to_lowercase();
|
|
135
|
+
if sensitive_set.contains(&k_str) {
|
|
136
|
+
new_dict.set_item(k, "[MASKED]")?;
|
|
137
|
+
} else {
|
|
138
|
+
new_dict.set_item(k, process_recursive(py, &v, sensitive_set)?)?;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
Ok(new_dict.into())
|
|
142
|
+
} else if let Ok(list_obj) = value.downcast::<PyList>() {
|
|
143
|
+
let new_list = PyList::empty(py);
|
|
144
|
+
for item in list_obj {
|
|
145
|
+
new_list.append(process_recursive(py, &item, sensitive_set)?)?;
|
|
146
|
+
}
|
|
147
|
+
Ok(new_list.into())
|
|
148
|
+
} else {
|
|
149
|
+
if value.is_none() || value.is_instance_of::<PyBool>() || value.is_instance_of::<PyFloat>() || value.is_instance_of::<PyInt>() || value.is_instance_of::<PyString>() {
|
|
150
|
+
Ok(value.clone().unbind())
|
|
151
|
+
} else {
|
|
152
|
+
match value.str() {
|
|
153
|
+
Ok(s) => Ok(s.into()),
|
|
154
|
+
Err(_) => Ok(PyString::new(py, "[SERIALIZATION_ERROR]").into())
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
#[pyfunction]
|
|
161
|
+
fn mask_and_serialize(py: Python, data: &Bound<'_, PyAny>, sensitive_keys: Vec<String>) -> PyResult<PyObject> {
|
|
162
|
+
let sensitive_set: HashSet<String> = sensitive_keys.into_iter().map(|s| s.to_lowercase()).collect();
|
|
163
|
+
process_recursive(py, data, &sensitive_set)
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
#[pymodule]
|
|
167
|
+
fn vectorwave_core(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
|
168
|
+
m.add_class::<RustBatchManager>()?;
|
|
169
|
+
m.add_function(wrap_pyfunction!(mask_and_serialize, m)?)?;
|
|
170
|
+
Ok(())
|
|
171
|
+
}
|
|
@@ -9,6 +9,7 @@ from typing import Optional, List, Dict, Any, Callable
|
|
|
9
9
|
from uuid import uuid4
|
|
10
10
|
from datetime import datetime, timezone
|
|
11
11
|
|
|
12
|
+
import vectorwave.vectorwave_core as vectorwave_core
|
|
12
13
|
from .alert.base import BaseAlerter
|
|
13
14
|
from ..batch.batch import get_batch_manager
|
|
14
15
|
from ..models.db_config import get_weaviate_settings, WeaviateSettings
|
|
@@ -34,31 +35,6 @@ current_tracer_var: ContextVar[Optional[TraceCollector]] = ContextVar('current_t
|
|
|
34
35
|
current_span_id_var: ContextVar[Optional[str]] = ContextVar('current_span_id', default=None)
|
|
35
36
|
|
|
36
37
|
|
|
37
|
-
def _mask_and_serialize(data: Any, sensitive_keys: set) -> Any:
|
|
38
|
-
"""
|
|
39
|
-
Recursively masks sensitive data in dicts/lists and prepares for serialization.
|
|
40
|
-
"""
|
|
41
|
-
if isinstance(data, dict):
|
|
42
|
-
new_dict = {}
|
|
43
|
-
for k, v in data.items():
|
|
44
|
-
if str(k).lower() in sensitive_keys:
|
|
45
|
-
new_dict[k] = "[MASKED]"
|
|
46
|
-
else:
|
|
47
|
-
new_dict[k] = _mask_and_serialize(v, sensitive_keys)
|
|
48
|
-
return new_dict
|
|
49
|
-
|
|
50
|
-
if isinstance(data, list):
|
|
51
|
-
return [_mask_and_serialize(item, sensitive_keys) for item in data]
|
|
52
|
-
|
|
53
|
-
if not isinstance(data, (str, int, float, bool, type(None))):
|
|
54
|
-
try:
|
|
55
|
-
return str(data)
|
|
56
|
-
except Exception:
|
|
57
|
-
return "[SERIALIZATION_ERROR]"
|
|
58
|
-
|
|
59
|
-
return data
|
|
60
|
-
|
|
61
|
-
|
|
62
38
|
def _capture_span_attributes(
|
|
63
39
|
attributes_to_capture: Optional[List[str]],
|
|
64
40
|
kwargs: Dict[str, Any],
|
|
@@ -76,7 +52,8 @@ def _capture_span_attributes(
|
|
|
76
52
|
processed_value = "[MASKED]"
|
|
77
53
|
else:
|
|
78
54
|
raw_value = kwargs[attr_name]
|
|
79
|
-
|
|
55
|
+
# Rust extension expects a List[str] for sensitive_keys, not set
|
|
56
|
+
processed_value = vectorwave_core.mask_and_serialize(raw_value, list(sensitive_keys))
|
|
80
57
|
|
|
81
58
|
captured_attributes[attr_name] = processed_value
|
|
82
59
|
|
|
@@ -160,8 +137,8 @@ def _create_input_vector_data(
|
|
|
160
137
|
the core arguments and vectorize them
|
|
161
138
|
"""
|
|
162
139
|
# 1. Process/Mask positional and keyword arguments
|
|
163
|
-
processed_args =
|
|
164
|
-
processed_kwargs =
|
|
140
|
+
processed_args = vectorwave_core.mask_and_serialize(list(args), list(sensitive_keys))
|
|
141
|
+
processed_kwargs = vectorwave_core.mask_and_serialize(kwargs, list(sensitive_keys))
|
|
165
142
|
|
|
166
143
|
|
|
167
144
|
texts_for_vector = [f"Function Context: {func_name}"]
|
|
@@ -309,7 +286,7 @@ def trace_span(
|
|
|
309
286
|
result = await func(*args, **kwargs)
|
|
310
287
|
|
|
311
288
|
if capture_return_value:
|
|
312
|
-
processed_result =
|
|
289
|
+
processed_result = vectorwave_core.mask_and_serialize(result, list(tracer.settings.sensitive_keys))
|
|
313
290
|
try:
|
|
314
291
|
return_value_log = json.dumps(processed_result)
|
|
315
292
|
except TypeError:
|
|
@@ -355,7 +332,7 @@ def trace_span(
|
|
|
355
332
|
result=return_value_log
|
|
356
333
|
)
|
|
357
334
|
|
|
358
|
-
if tracer.settings.DRIFT_DETECTION_ENABLED and vector_to_add:
|
|
335
|
+
if tracer.settings.DRIFT_DETECTION_ENABLED and vector_to_add and status == "SUCCESS":
|
|
359
336
|
is_drift, dist, nearest_id = check_semantic_drift(
|
|
360
337
|
vector=vector_to_add,
|
|
361
338
|
function_name=func.__name__,
|
|
@@ -442,7 +419,7 @@ def trace_span(
|
|
|
442
419
|
result = func(*args, **kwargs)
|
|
443
420
|
|
|
444
421
|
if capture_return_value:
|
|
445
|
-
processed_result =
|
|
422
|
+
processed_result = vectorwave_core.mask_and_serialize(result, list(tracer.settings.sensitive_keys))
|
|
446
423
|
try:
|
|
447
424
|
return_value_log = json.dumps(processed_result)
|
|
448
425
|
except TypeError:
|
|
@@ -488,7 +465,7 @@ def trace_span(
|
|
|
488
465
|
result=return_value_log
|
|
489
466
|
)
|
|
490
467
|
|
|
491
|
-
if tracer.settings.DRIFT_DETECTION_ENABLED and vector_to_add:
|
|
468
|
+
if tracer.settings.DRIFT_DETECTION_ENABLED and vector_to_add and status == "SUCCESS":
|
|
492
469
|
is_drift, dist, nearest_id = check_semantic_drift(
|
|
493
470
|
vector=vector_to_add,
|
|
494
471
|
function_name=func.__name__,
|
|
@@ -498,7 +475,7 @@ def trace_span(
|
|
|
498
475
|
|
|
499
476
|
if is_drift:
|
|
500
477
|
drift_alert_props = span_properties.copy()
|
|
501
|
-
drift_alert_props["status"] = "WARNING"
|
|
478
|
+
drift_alert_props["status"] = "WARNING"
|
|
502
479
|
drift_alert_props["error_code"] = "SEMANTIC_DRIFT"
|
|
503
480
|
drift_alert_props["error_message"] = (
|
|
504
481
|
f"Anomaly detected in data distribution.\n"
|
|
@@ -535,4 +512,4 @@ def trace_span(
|
|
|
535
512
|
if _func is None:
|
|
536
513
|
return decorator
|
|
537
514
|
else:
|
|
538
|
-
return decorator(_func)
|
|
515
|
+
return decorator(_func)
|
|
@@ -12,7 +12,7 @@ import weaviate.classes.query as wvc_query
|
|
|
12
12
|
|
|
13
13
|
from ..database.db import get_cached_client
|
|
14
14
|
from ..models.db_config import get_weaviate_settings
|
|
15
|
-
|
|
15
|
+
import vectorwave.vectorwave_core as vectorwave_core
|
|
16
16
|
from .context import execution_source_context
|
|
17
17
|
|
|
18
18
|
logger = logging.getLogger(__name__)
|
|
@@ -238,7 +238,7 @@ class VectorWaveReplayer:
|
|
|
238
238
|
collection_name = self.golden_collection_name if is_golden else self.collection_name
|
|
239
239
|
collection = self.client.collections.get(collection_name)
|
|
240
240
|
|
|
241
|
-
processed_val =
|
|
241
|
+
processed_val = vectorwave_core.mask_and_serialize(new_value, [])
|
|
242
242
|
try:
|
|
243
243
|
val_str = json.dumps(processed_val)
|
|
244
244
|
except (TypeError, ValueError):
|