dxmt-python-sdk 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dxmt/__init__.py +166 -0
- dxmt/bisheng_integration.py +226 -0
- dxmt/client.py +2510 -0
- dxmt/crewai_storage.py +524 -0
- dxmt/crewai_tool.py +430 -0
- dxmt/exceptions.py +46 -0
- dxmt/filters.py +360 -0
- dxmt/haystack.py +773 -0
- dxmt/haystack_retrievers.py +372 -0
- dxmt/langchain_vectorstore.py +527 -0
- dxmt/langgraph_store.py +1054 -0
- dxmt/mem0.py +653 -0
- dxmt/metagpt_retriever.py +361 -0
- dxmt/metagpt_vector_store.py +466 -0
- dxmt/migrate/__init__.py +1 -0
- dxmt/migrate/chroma.py +237 -0
- dxmt/py.typed +0 -0
- dxmt/types.py +51 -0
- dxmt_python_sdk-1.6.0.dist-info/METADATA +736 -0
- dxmt_python_sdk-1.6.0.dist-info/RECORD +23 -0
- dxmt_python_sdk-1.6.0.dist-info/WHEEL +5 -0
- dxmt_python_sdk-1.6.0.dist-info/licenses/LICENSE +201 -0
- dxmt_python_sdk-1.6.0.dist-info/top_level.txt +1 -0
dxmt/filters.py
ADDED
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
"""Filters: a typed builder, and a translator from MongoDB-style dicts.
|
|
2
|
+
|
|
3
|
+
The server takes one shape: three flat lists of conditions,
|
|
4
|
+
|
|
5
|
+
{"must": [...], "should": [...], "must_not": [...]}
|
|
6
|
+
|
|
7
|
+
meaning every `must` holds, at least one `should` holds (when there is any), and no `must_not`
|
|
8
|
+
holds. That is not every boolean formula. `(a and b) or c` has no such form, and neither has
|
|
9
|
+
`(a or b) and (c or d)`. Both ways of writing a filter here -- `Field` and the dict translator --
|
|
10
|
+
build that shape exactly or raise `ValueError` saying what cannot be expressed. They never
|
|
11
|
+
approximate: the translator this replaced turned `(a and b) or c` into `a or b or c`, dropped a
|
|
12
|
+
key written beside `$and`, and read a nested `$and` as a field named "$and", each of them a
|
|
13
|
+
query that ran and returned the wrong records.
|
|
14
|
+
|
|
15
|
+
from dxmt import Field
|
|
16
|
+
|
|
17
|
+
f = (Field("lang") == "en") & (Field("year") >= 2020) & ~Field("tags").is_in(["spam"])
|
|
18
|
+
client.search("docs", vector, filter=f)
|
|
19
|
+
|
|
20
|
+
What a condition means is the server's, and is the same on every endpoint:
|
|
21
|
+
|
|
22
|
+
* a field holding an array matches when any element does, for every operation;
|
|
23
|
+
* `!=` / `NotEqual` is true of a record without the field, `not_in` / `NotIn` is not;
|
|
24
|
+
* numbers compare as numbers, so 5 matches 5.0, and ranges compare numbers only;
|
|
25
|
+
* the key `id` is the record id;
|
|
26
|
+
* comparing against an array or object is refused (`tags == ["a", "b"]`).
|
|
27
|
+
|
|
28
|
+
Two conditions on the same array field are tested independently: `year >= 2020` and
|
|
29
|
+
`year <= 2023` holds for `[2019, 2024]`, one element satisfying each.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from typing import Any, Dict, Iterable, List, Optional
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# MongoDB operator to DXMT operation type mapping
|
|
36
|
+
OPERATOR_MAP = {
|
|
37
|
+
"$eq": "Equal",
|
|
38
|
+
"$ne": "NotEqual",
|
|
39
|
+
"$gt": "GreaterThan",
|
|
40
|
+
"$gte": "GreaterThanOrEqual",
|
|
41
|
+
"$lt": "LessThan",
|
|
42
|
+
"$lte": "LessThanOrEqual",
|
|
43
|
+
"$in": "In",
|
|
44
|
+
"$nin": "NotIn",
|
|
45
|
+
"$contains": "Contains",
|
|
46
|
+
"$startswith": "StartsWith",
|
|
47
|
+
"$endswith": "EndsWith",
|
|
48
|
+
"$isempty": "IsEmpty",
|
|
49
|
+
"$isnull": "IsNull",
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _map_operator(mongo_op: str) -> str:
|
|
54
|
+
"""Map MongoDB operator to DXMT operation type.
|
|
55
|
+
|
|
56
|
+
Args:
|
|
57
|
+
mongo_op: MongoDB operator like $gte, $lt, etc.
|
|
58
|
+
|
|
59
|
+
Returns:
|
|
60
|
+
DXMT operation type like GreaterThanOrEqual, LessThan, etc.
|
|
61
|
+
|
|
62
|
+
Raises:
|
|
63
|
+
ValueError: If operator is not supported.
|
|
64
|
+
"""
|
|
65
|
+
if mongo_op not in OPERATOR_MAP:
|
|
66
|
+
raise ValueError(
|
|
67
|
+
f"Unsupported operator: {mongo_op}. "
|
|
68
|
+
f"Supported operators: {list(OPERATOR_MAP.keys())}"
|
|
69
|
+
)
|
|
70
|
+
return OPERATOR_MAP[mongo_op]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
_SCALAR = (str, int, float, bool)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _condition(key: str, op_type: str, value: Any) -> Dict[str, Any]:
|
|
78
|
+
"""One condition in the server's shape, refusing values the server would refuse."""
|
|
79
|
+
if not isinstance(key, str) or not key:
|
|
80
|
+
raise ValueError(f"a filter key must be a non-empty string, not {key!r}")
|
|
81
|
+
if op_type in ("IsEmpty", "IsNull"):
|
|
82
|
+
# No value at all: the condition is about the field, not about something in it. The
|
|
83
|
+
# server refuses an unknown member, so sending `"value": null` would be a 400.
|
|
84
|
+
return {"key": key, "operation": {"type": op_type}}
|
|
85
|
+
if op_type in ("In", "NotIn"):
|
|
86
|
+
if not isinstance(value, (list, tuple)):
|
|
87
|
+
raise ValueError(f"{op_type} on {key!r} takes a list of values, not {value!r}")
|
|
88
|
+
value = list(value)
|
|
89
|
+
bad = [v for v in value if not isinstance(v, _SCALAR)]
|
|
90
|
+
elif op_type in ("Contains", "StartsWith", "EndsWith"):
|
|
91
|
+
bad = [] if isinstance(value, str) else [value]
|
|
92
|
+
elif op_type in ("GreaterThan", "GreaterThanOrEqual", "LessThan", "LessThanOrEqual"):
|
|
93
|
+
if not isinstance(value, (int, float)) or isinstance(value, bool):
|
|
94
|
+
# The server orders numbers only, so `date >= "2024-01-01"` matches nothing.
|
|
95
|
+
raise ValueError(
|
|
96
|
+
f"{op_type} on {key!r} compares against {value!r}; ranges compare numbers "
|
|
97
|
+
"only (store dates as a number, such as a Unix timestamp)"
|
|
98
|
+
)
|
|
99
|
+
bad = []
|
|
100
|
+
else:
|
|
101
|
+
bad = [] if isinstance(value, _SCALAR) else [value]
|
|
102
|
+
if bad:
|
|
103
|
+
raise ValueError(
|
|
104
|
+
f"{op_type} on {key!r} compares against {bad[0]!r}; a filter compares a field "
|
|
105
|
+
"against strings, numbers and booleans only (an array field matches when any "
|
|
106
|
+
"element does, so test the element instead)"
|
|
107
|
+
)
|
|
108
|
+
return {"key": key, "operation": {"type": op_type, "value": value}}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class Filter:
|
|
112
|
+
"""A filter in the server's flat shape. Combine with `&`, `|` and `~`.
|
|
113
|
+
|
|
114
|
+
Each combination either has an exact flat form or raises `ValueError`; see the module
|
|
115
|
+
docstring. Pass one anywhere a filter is taken, or call `to_dict()`.
|
|
116
|
+
"""
|
|
117
|
+
|
|
118
|
+
__slots__ = ("must", "should", "must_not")
|
|
119
|
+
|
|
120
|
+
def __init__(self, must=(), should=(), must_not=()):
|
|
121
|
+
self.must: List[Dict[str, Any]] = list(must)
|
|
122
|
+
self.should: List[Dict[str, Any]] = list(should)
|
|
123
|
+
self.must_not: List[Dict[str, Any]] = list(must_not)
|
|
124
|
+
|
|
125
|
+
def _single(self) -> Optional[Dict[str, Any]]:
|
|
126
|
+
"""The condition this is, when it is exactly one positive condition."""
|
|
127
|
+
if len(self.must) == 1 and not self.should and not self.must_not:
|
|
128
|
+
return self.must[0]
|
|
129
|
+
return None
|
|
130
|
+
|
|
131
|
+
def _positives(self) -> Optional[List[Dict[str, Any]]]:
|
|
132
|
+
"""The conditions of a plain disjunction: one condition, or a `should` alone."""
|
|
133
|
+
single = self._single()
|
|
134
|
+
if single is not None:
|
|
135
|
+
return [single]
|
|
136
|
+
if self.should and not self.must and not self.must_not:
|
|
137
|
+
return list(self.should)
|
|
138
|
+
return None
|
|
139
|
+
|
|
140
|
+
def __and__(self, other: "Filter") -> "Filter":
|
|
141
|
+
if not isinstance(other, Filter):
|
|
142
|
+
return NotImplemented
|
|
143
|
+
if self.should and other.should:
|
|
144
|
+
# (a or b) and (c or d): distributing it is exponential, and one `should` list
|
|
145
|
+
# cannot hold two independent disjunctions.
|
|
146
|
+
raise ValueError(
|
|
147
|
+
"cannot express two OR groups joined by AND; the filter has one `should` list"
|
|
148
|
+
)
|
|
149
|
+
return Filter(
|
|
150
|
+
self.must + other.must,
|
|
151
|
+
self.should or other.should,
|
|
152
|
+
self.must_not + other.must_not,
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
def __or__(self, other: "Filter") -> "Filter":
|
|
156
|
+
if not isinstance(other, Filter):
|
|
157
|
+
return NotImplemented
|
|
158
|
+
left, right = self._positives(), other._positives()
|
|
159
|
+
if left is None or right is None:
|
|
160
|
+
raise ValueError(
|
|
161
|
+
"OR can only join single conditions (or other ORs of them); "
|
|
162
|
+
"(a and b) or c has no form the server takes"
|
|
163
|
+
)
|
|
164
|
+
return Filter(should=left + right)
|
|
165
|
+
|
|
166
|
+
def __invert__(self) -> "Filter":
|
|
167
|
+
positives = self._positives()
|
|
168
|
+
if positives is None:
|
|
169
|
+
# not (a and b) is (not a) or (not b), and there is no negated condition to put
|
|
170
|
+
# in `should`.
|
|
171
|
+
raise ValueError("NOT can only negate a single condition or an OR of them")
|
|
172
|
+
return Filter(must_not=positives)
|
|
173
|
+
|
|
174
|
+
def __bool__(self) -> bool:
|
|
175
|
+
# `a and b`, `not a` and `2020 <= Field("y") <= 2023` all go through bool() and would
|
|
176
|
+
# quietly keep one side. Refusing is the only way to make them loud.
|
|
177
|
+
raise TypeError(
|
|
178
|
+
"a filter has no truth value: combine with &, | and ~ (not and/or/not), and write "
|
|
179
|
+
"a range as Field(k).between(lo, hi) rather than a chained comparison"
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
def __eq__(self, other: object) -> bool:
|
|
183
|
+
return isinstance(other, Filter) and self.to_dict() == other.to_dict()
|
|
184
|
+
|
|
185
|
+
__hash__ = None # type: ignore[assignment]
|
|
186
|
+
|
|
187
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
188
|
+
out: Dict[str, Any] = {}
|
|
189
|
+
for name in ("must", "should", "must_not"):
|
|
190
|
+
conditions = getattr(self, name)
|
|
191
|
+
if conditions:
|
|
192
|
+
out[name] = [
|
|
193
|
+
{"key": c["key"], "operation": dict(c["operation"])} for c in conditions
|
|
194
|
+
]
|
|
195
|
+
return out
|
|
196
|
+
|
|
197
|
+
def __repr__(self) -> str:
|
|
198
|
+
return f"Filter({self.to_dict()!r})"
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
class Field:
|
|
202
|
+
"""A payload field (or `"id"`, the record id), to build conditions on.
|
|
203
|
+
|
|
204
|
+
>>> (Field("year") >= 2020) & (Field("lang") == "en")
|
|
205
|
+
|
|
206
|
+
A dot reads into a nested payload -- `Field("meta.lang")` is the `lang` of the object under
|
|
207
|
+
`meta`, at any depth. A payload that really holds a key with a dot in it keeps working:
|
|
208
|
+
the server tries the whole name as a key before reading it as a path.
|
|
209
|
+
"""
|
|
210
|
+
|
|
211
|
+
__slots__ = ("key",)
|
|
212
|
+
|
|
213
|
+
def __init__(self, key: str):
|
|
214
|
+
if not isinstance(key, str) or not key:
|
|
215
|
+
raise ValueError(f"a field name must be a non-empty string, not {key!r}")
|
|
216
|
+
self.key = key
|
|
217
|
+
|
|
218
|
+
def _op(self, op_type: str, value: Any) -> Filter:
|
|
219
|
+
return Filter(must=[_condition(self.key, op_type, value)])
|
|
220
|
+
|
|
221
|
+
def __eq__(self, value: Any) -> Filter: # type: ignore[override]
|
|
222
|
+
return self._op("Equal", value)
|
|
223
|
+
|
|
224
|
+
def __ne__(self, value: Any) -> Filter: # type: ignore[override]
|
|
225
|
+
return self._op("NotEqual", value)
|
|
226
|
+
|
|
227
|
+
def __gt__(self, value: Any) -> Filter:
|
|
228
|
+
return self._op("GreaterThan", value)
|
|
229
|
+
|
|
230
|
+
def __ge__(self, value: Any) -> Filter:
|
|
231
|
+
return self._op("GreaterThanOrEqual", value)
|
|
232
|
+
|
|
233
|
+
def __lt__(self, value: Any) -> Filter:
|
|
234
|
+
return self._op("LessThan", value)
|
|
235
|
+
|
|
236
|
+
def __le__(self, value: Any) -> Filter:
|
|
237
|
+
return self._op("LessThanOrEqual", value)
|
|
238
|
+
|
|
239
|
+
__hash__ = None # type: ignore[assignment]
|
|
240
|
+
|
|
241
|
+
def is_in(self, values: Iterable[Any]) -> Filter:
|
|
242
|
+
return self._op("In", list(values))
|
|
243
|
+
|
|
244
|
+
def not_in(self, values: Iterable[Any]) -> Filter:
|
|
245
|
+
"""Present and not one of `values`; a record without the field does not match."""
|
|
246
|
+
return self._op("NotIn", list(values))
|
|
247
|
+
|
|
248
|
+
def between(self, low: Any, high: Any) -> Filter:
|
|
249
|
+
"""`low <= field <= high`. On an array field, each bound may be met by a different element."""
|
|
250
|
+
return (self >= low) & (self <= high)
|
|
251
|
+
|
|
252
|
+
def contains(self, text: str) -> Filter:
|
|
253
|
+
return self._op("Contains", text)
|
|
254
|
+
|
|
255
|
+
def starts_with(self, text: str) -> Filter:
|
|
256
|
+
return self._op("StartsWith", text)
|
|
257
|
+
|
|
258
|
+
def ends_with(self, text: str) -> Filter:
|
|
259
|
+
return self._op("EndsWith", text)
|
|
260
|
+
|
|
261
|
+
def is_empty(self) -> Filter:
|
|
262
|
+
"""The field is absent, `null`, or an empty array.
|
|
263
|
+
|
|
264
|
+
The only way to ask for the records a field is *missing* from: `!=` answers a different
|
|
265
|
+
question, since a record holding some other value satisfies that too.
|
|
266
|
+
"""
|
|
267
|
+
return Filter(must=[_condition(self.key, "IsEmpty", None)])
|
|
268
|
+
|
|
269
|
+
def is_null(self) -> Filter:
|
|
270
|
+
"""The field is present and `null` — or holds a `null` element."""
|
|
271
|
+
return Filter(must=[_condition(self.key, "IsNull", None)])
|
|
272
|
+
|
|
273
|
+
def __repr__(self) -> str:
|
|
274
|
+
return f"Field({self.key!r})"
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _field_filter(key: str, value: Any) -> Filter:
|
|
278
|
+
"""`{key: value}` from a dict filter: equality, or `{"$op": v, ...}` all ANDed."""
|
|
279
|
+
if key.startswith("$"):
|
|
280
|
+
raise ValueError(f"unknown operator {key!r} at the top of a filter")
|
|
281
|
+
if isinstance(value, dict):
|
|
282
|
+
if not value:
|
|
283
|
+
raise ValueError(f"empty condition for {key!r}")
|
|
284
|
+
plain = [op for op in value if not op.startswith("$")]
|
|
285
|
+
if plain:
|
|
286
|
+
raise ValueError(
|
|
287
|
+
f"{key!r}: {value!r} mixes operators with plain keys; comparing a field "
|
|
288
|
+
"against an object is not supported"
|
|
289
|
+
)
|
|
290
|
+
result = Filter()
|
|
291
|
+
for op, operand in value.items():
|
|
292
|
+
result = result & Filter(must=[_condition(key, _map_operator(op), operand)])
|
|
293
|
+
return result
|
|
294
|
+
return Filter(must=[_condition(key, "Equal", value)])
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _dict_filter(filter_dict: Dict[str, Any]) -> Filter:
|
|
298
|
+
if not isinstance(filter_dict, dict):
|
|
299
|
+
raise ValueError(f"a filter is a dict, not {filter_dict!r}")
|
|
300
|
+
result = Filter()
|
|
301
|
+
# Every key is a term of an implicit AND -- including a key beside `$and` or `$or`.
|
|
302
|
+
for key, value in filter_dict.items():
|
|
303
|
+
if key in ("$and", "$or", "$nor", "$not"):
|
|
304
|
+
if not isinstance(value, list) or not value:
|
|
305
|
+
raise ValueError(f"{key} takes a non-empty list of filters, not {value!r}")
|
|
306
|
+
parts = [_dict_filter(part) for part in value]
|
|
307
|
+
if key == "$and":
|
|
308
|
+
term = parts[0]
|
|
309
|
+
for part in parts[1:]:
|
|
310
|
+
term = term & part
|
|
311
|
+
else:
|
|
312
|
+
term = parts[0]
|
|
313
|
+
for part in parts[1:]:
|
|
314
|
+
term = term | part
|
|
315
|
+
if key != "$or":
|
|
316
|
+
# $nor, and $not given a list as this SDK has always taken it: none of them.
|
|
317
|
+
term = ~term
|
|
318
|
+
else:
|
|
319
|
+
term = _field_filter(key, value)
|
|
320
|
+
result = result & term
|
|
321
|
+
return result
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def to_filter(value: Any) -> Optional[Dict[str, Any]]:
|
|
325
|
+
"""Any filter this SDK takes, in the server's shape; None for no filter.
|
|
326
|
+
|
|
327
|
+
Takes a `Filter` (from `Field`), a MongoDB-style dict, or the server's own shape, which is
|
|
328
|
+
checked for stray keys and passed through.
|
|
329
|
+
"""
|
|
330
|
+
if value is None:
|
|
331
|
+
return None
|
|
332
|
+
if isinstance(value, Filter):
|
|
333
|
+
return value.to_dict() or None
|
|
334
|
+
if not isinstance(value, dict):
|
|
335
|
+
raise ValueError(f"a filter is a dict or a Filter, not {type(value).__name__}")
|
|
336
|
+
if not value:
|
|
337
|
+
return None
|
|
338
|
+
native = {"must", "should", "must_not"}
|
|
339
|
+
if native & value.keys():
|
|
340
|
+
stray = value.keys() - native
|
|
341
|
+
if stray:
|
|
342
|
+
raise ValueError(
|
|
343
|
+
f"a filter with must/should/must_not cannot also have {sorted(stray)}; "
|
|
344
|
+
"they would be ignored"
|
|
345
|
+
)
|
|
346
|
+
return value
|
|
347
|
+
return _dict_filter(value).to_dict() or None
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def convert_simple_filter_to_dxmt(
|
|
351
|
+
filter_dict: Optional[Dict[str, Any]],
|
|
352
|
+
) -> Optional[Dict[str, Any]]:
|
|
353
|
+
"""Convert a filter to the server's shape. Kept under its old name; see `to_filter`.
|
|
354
|
+
|
|
355
|
+
Accepts simple equality (`{"category": "AI"}`), operators
|
|
356
|
+
(`{"year": {"$gte": 2020, "$lte": 2023}}`), `$and`, `$or`, `$nor` (and `$not` with a list,
|
|
357
|
+
meaning the same), nested wherever the result has an exact form, the server's own shape, and
|
|
358
|
+
a `Filter`. Raises `ValueError` for anything it cannot translate exactly.
|
|
359
|
+
"""
|
|
360
|
+
return to_filter(filter_dict)
|