qsmongo 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qsmongo-0.3.0/.github/workflows/ci.yml +23 -0
- qsmongo-0.3.0/.github/workflows/publish.yml +38 -0
- qsmongo-0.3.0/.gitignore +14 -0
- qsmongo-0.3.0/LICENSE +21 -0
- qsmongo-0.3.0/PKG-INFO +299 -0
- qsmongo-0.3.0/README.md +255 -0
- qsmongo-0.3.0/examples/fastapi_app.py +103 -0
- qsmongo-0.3.0/pyproject.toml +44 -0
- qsmongo-0.3.0/src/qsmongo/__init__.py +45 -0
- qsmongo-0.3.0/src/qsmongo/advisor.py +200 -0
- qsmongo-0.3.0/src/qsmongo/coerce.py +78 -0
- qsmongo-0.3.0/src/qsmongo/cursor.py +171 -0
- qsmongo-0.3.0/src/qsmongo/errors.py +37 -0
- qsmongo-0.3.0/src/qsmongo/parser.py +279 -0
- qsmongo-0.3.0/src/qsmongo/query.py +62 -0
- qsmongo-0.3.0/src/qsmongo/schema.py +116 -0
- qsmongo-0.3.0/tests/test_advisor.py +144 -0
- qsmongo-0.3.0/tests/test_cursor.py +118 -0
- qsmongo-0.3.0/tests/test_edge_cases.py +67 -0
- qsmongo-0.3.0/tests/test_keyset_property.py +86 -0
- qsmongo-0.3.0/tests/test_paging_and_sort.py +73 -0
- qsmongo-0.3.0/tests/test_parser.py +89 -0
- qsmongo-0.3.0/tests/test_projection_and_text.py +94 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
fail-fast: false
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
- uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: ${{ matrix.python-version }}
|
|
20
|
+
- run: pip install -e ".[dev]"
|
|
21
|
+
- run: ruff check .
|
|
22
|
+
- run: ruff format --check .
|
|
23
|
+
- run: pytest -q
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
name: Publish
|
|
2
|
+
|
|
3
|
+
# Publishing is driven by GitHub Releases: cut a release and this ships that tag to PyPI.
|
|
4
|
+
# Authentication is OIDC (PyPI "trusted publishing") — there is no API token to store or rotate.
|
|
5
|
+
on:
|
|
6
|
+
release:
|
|
7
|
+
types: [published]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
build:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
steps:
|
|
13
|
+
- uses: actions/checkout@v4
|
|
14
|
+
- uses: actions/setup-python@v5
|
|
15
|
+
with:
|
|
16
|
+
python-version: "3.12"
|
|
17
|
+
- run: pip install -e ".[dev]" build
|
|
18
|
+
- run: ruff check .
|
|
19
|
+
- run: pytest -q
|
|
20
|
+
- run: python -m build
|
|
21
|
+
- run: pip install twine && twine check dist/*
|
|
22
|
+
- uses: actions/upload-artifact@v4
|
|
23
|
+
with:
|
|
24
|
+
name: dist
|
|
25
|
+
path: dist/
|
|
26
|
+
|
|
27
|
+
publish:
|
|
28
|
+
needs: build
|
|
29
|
+
runs-on: ubuntu-latest
|
|
30
|
+
environment: pypi
|
|
31
|
+
permissions:
|
|
32
|
+
id-token: write # required for trusted publishing; nothing else is granted
|
|
33
|
+
steps:
|
|
34
|
+
- uses: actions/download-artifact@v4
|
|
35
|
+
with:
|
|
36
|
+
name: dist
|
|
37
|
+
path: dist/
|
|
38
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
qsmongo-0.3.0/.gitignore
ADDED
qsmongo-0.3.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Bosko Djokic
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
qsmongo-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: qsmongo
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Turn an HTTP query string into a safe, typed MongoDB filter.
|
|
5
|
+
Project-URL: Homepage, https://github.com/boskodjokic/qsmongo
|
|
6
|
+
Project-URL: Issues, https://github.com/boskodjokic/qsmongo/issues
|
|
7
|
+
Author: Bosko Djokic
|
|
8
|
+
License: MIT License
|
|
9
|
+
|
|
10
|
+
Copyright (c) 2026 Bosko Djokic
|
|
11
|
+
|
|
12
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
13
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
14
|
+
in the Software without restriction, including without limitation the rights
|
|
15
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
16
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
17
|
+
furnished to do so, subject to the following conditions:
|
|
18
|
+
|
|
19
|
+
The above copyright notice and this permission notice shall be included in all
|
|
20
|
+
copies or substantial portions of the Software.
|
|
21
|
+
|
|
22
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
23
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
24
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
25
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
26
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
27
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
28
|
+
SOFTWARE.
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
Keywords: fastapi,filtering,mongodb,pagination,query string
|
|
31
|
+
Classifier: Development Status :: 4 - Beta
|
|
32
|
+
Classifier: Intended Audience :: Developers
|
|
33
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
34
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
35
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
38
|
+
Classifier: Topic :: Database
|
|
39
|
+
Requires-Python: >=3.10
|
|
40
|
+
Provides-Extra: dev
|
|
41
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
42
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
43
|
+
Description-Content-Type: text/markdown
|
|
44
|
+
|
|
45
|
+
# qsmongo
|
|
46
|
+
|
|
47
|
+
[](https://github.com/boskodjokic/qsmongo/actions/workflows/ci.yml)
|
|
48
|
+
[](https://www.python.org/)
|
|
49
|
+
[](LICENSE)
|
|
50
|
+
|
|
51
|
+
Turn an HTTP query string into a MongoDB query that is **safe by construction** — nothing is
|
|
52
|
+
queryable until you declare it — and **fast by construction**: keyset pagination that costs the
|
|
53
|
+
same on page 10,000 as on page 1.
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
GET /users?age__gte=21&status__in=active,trial&fields=name,age&sort=-created_at&per_page=50
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from qsmongo import Field, Schema, parse
|
|
61
|
+
|
|
62
|
+
schema = Schema(
|
|
63
|
+
name=Field(str),
|
|
64
|
+
age=Field(int),
|
|
65
|
+
status=Field(str),
|
|
66
|
+
created_at=Field(datetime, alias="audit.created"),
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
query = parse(request.url.query, schema)
|
|
70
|
+
|
|
71
|
+
query.filter # {'age': {'$gte': 21}, 'status': {'$in': ['active', 'trial']}}
|
|
72
|
+
query.projection # {'name': 1, 'age': 1}
|
|
73
|
+
query.sort # [('audit.created', -1)]
|
|
74
|
+
|
|
75
|
+
collection.find(**query.find_kwargs())
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
No dependencies. Python 3.10+.
|
|
79
|
+
|
|
80
|
+
## Why
|
|
81
|
+
|
|
82
|
+
Every REST API over MongoDB grows the same ad-hoc filter parser, and it goes wrong in the same
|
|
83
|
+
five ways:
|
|
84
|
+
|
|
85
|
+
1. **Type mismatch.** `?age=21` gives you the string `"21"`. Mongo does not match `21` against
|
|
86
|
+
`"21"`, so the endpoint returns an empty list and nobody can tell whether the data is missing or
|
|
87
|
+
the filter is broken.
|
|
88
|
+
2. **Operator injection.** Passing user input into a query document unfiltered means a crafted
|
|
89
|
+
parameter can inject `$where` or `$ne` and walk straight past your access checks.
|
|
90
|
+
3. **Separator collisions.** A product genuinely named `Dolce & Gabbana`, a colour called
|
|
91
|
+
`black and white`, a tag containing a comma — each one quietly truncates the filter.
|
|
92
|
+
4. **Unbounded pages.** `?per_page=100000` is a denial-of-service vector you shipped yourself.
|
|
93
|
+
5. **Offset pagination.** `skip=100000` makes the server walk 100,000 documents it will never
|
|
94
|
+
return, and when the sort key has ties, documents shift between pages and are skipped or served
|
|
95
|
+
twice.
|
|
96
|
+
|
|
97
|
+
## Install
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
pip install qsmongo
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## The grammar
|
|
104
|
+
|
|
105
|
+
| Query string | Result |
|
|
106
|
+
| --- | --- |
|
|
107
|
+
| `name=Ada` | `{"name": "Ada"}` |
|
|
108
|
+
| `age__gte=21` | `{"age": {"$gte": 21}}` |
|
|
109
|
+
| `age__gte=21&age__lt=65` | `{"age": {"$gte": 21, "$lt": 65}}` |
|
|
110
|
+
| `status__in=active,trial` | `{"status": {"$in": ["active", "trial"]}}` |
|
|
111
|
+
| `name__contains=ada` | `{"name": {"$regex": "ada", "$options": "i"}}` |
|
|
112
|
+
| `name__startswith=Ad` | `{"name": {"$regex": "^Ad", "$options": "i"}}` |
|
|
113
|
+
| `deleted_at__exists=false` | `{"deleted_at": {"$exists": False}}` |
|
|
114
|
+
| `fields=name,price` | `{"name": 1, "price": 1}` |
|
|
115
|
+
| `fields=-internal_note` | `{"internal_note": 0}` |
|
|
116
|
+
| `sort=-created_at,name` | `[("created_at", -1), ("name", 1)]` |
|
|
117
|
+
| `page=2&per_page=50` | `skip=50, limit=50` |
|
|
118
|
+
| `after=<cursor>&per_page=50` | keyset range clause, `skip=0` |
|
|
119
|
+
|
|
120
|
+
Operators: `ne` `gt` `gte` `lt` `lte` `in` `nin` `contains` `startswith` `endswith` `exists`
|
|
121
|
+
`regex`. No suffix means equality.
|
|
122
|
+
|
|
123
|
+
## Declaring fields
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
Schema(
|
|
127
|
+
name=Field(str), # eq/ne/in/nin/exists + contains/startswith/endswith
|
|
128
|
+
age=Field(int), # numbers also get gt/gte/lt/lte
|
|
129
|
+
email=Field(str, alias="contact.email"), # public name differs from the document field
|
|
130
|
+
tag=Field(str, multi=True), # ?tag=red&tag=blue -> $in
|
|
131
|
+
sku=Field(str, case_sensitive=True), # index-friendly startswith
|
|
132
|
+
internal_cost=Field(float, projectable=False), # never selectable via ?fields
|
|
133
|
+
description=Field(str, ops=set()), # selectable, never queryable
|
|
134
|
+
pattern=Field(str, ops={"eq", "regex"}), # raw regex is opt-in
|
|
135
|
+
)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
A field that is not declared cannot be queried, sorted on, projected, or smuggled in as an
|
|
139
|
+
operator — `parse` raises `UnknownField` instead. That whitelist *is* the security model.
|
|
140
|
+
|
|
141
|
+
## Keyset pagination
|
|
142
|
+
|
|
143
|
+
Offset paging is fine until it isn't. Pass a `Cursors` codec and clients can page by cursor:
|
|
144
|
+
|
|
145
|
+
```python
|
|
146
|
+
from qsmongo import Cursors, parse
|
|
147
|
+
|
|
148
|
+
cursors = Cursors(secret=settings.CURSOR_SECRET) # HMAC-signed, so clients cannot forge one
|
|
149
|
+
|
|
150
|
+
query = parse(request.url.query, schema, cursors=cursors)
|
|
151
|
+
items = list(collection.find(**query.find_kwargs()))
|
|
152
|
+
|
|
153
|
+
return {"items": items, "next": query.next_cursor(items[-1] if items else None)}
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
The client sends that token back as `?after=<cursor>`, and the skip becomes a range clause:
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
{"$or": [{"score": {"$lt": 42}}, {"score": 42, "_id": {"$gt": "abc"}}]}
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Details that matter:
|
|
163
|
+
|
|
164
|
+
- **An `_id` tiebreaker is appended to every sort** when cursors are enabled, so the ordering is
|
|
165
|
+
total. Without it, two documents sharing a `created_at` can straddle a page boundary and one of
|
|
166
|
+
them is lost.
|
|
167
|
+
- **Cursors are signed** when you supply a secret, and the secret never appears in a `repr`.
|
|
168
|
+
Unsigned, tampered, or truncated tokens raise `InvalidCursor`.
|
|
169
|
+
- **The sort must stay the same between pages.** Changing it raises rather than silently
|
|
170
|
+
paginating nonsense.
|
|
171
|
+
- `after` and `page` are different modes; sending both is an error.
|
|
172
|
+
- Forward-only. Backward paging is not implemented.
|
|
173
|
+
|
|
174
|
+
[`tests/test_keyset_property.py`](tests/test_keyset_property.py) walks a dataset engineered so that
|
|
175
|
+
every page boundary lands in the middle of a tie, and asserts the pages reconstruct the full
|
|
176
|
+
ordering exactly — no document skipped, none repeated.
|
|
177
|
+
|
|
178
|
+
## Index advice
|
|
179
|
+
|
|
180
|
+
A query that parses cleanly can still be a collection scan. `analyze` checks the query against your
|
|
181
|
+
declared indexes using MongoDB's ESR ordering — **E**quality keys first, then **S**ort keys, then
|
|
182
|
+
**R**ange keys — and suggests one when nothing fits:
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
from qsmongo import Index, analyze
|
|
186
|
+
|
|
187
|
+
INDEXES = Index.from_index_information(collection.index_information())
|
|
188
|
+
|
|
189
|
+
advice = analyze(query, INDEXES, extra_equality=["tenant_id"])
|
|
190
|
+
if not advice.ok:
|
|
191
|
+
log.warning("unindexed query\n%s", advice)
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
```
|
|
195
|
+
no index serves this query
|
|
196
|
+
- {status: 1, price: 1, audit.created: -1} filters this query but does not provide the sort
|
|
197
|
+
{audit.created: -1}, so MongoDB sorts in memory (it aborts past its blocking-sort memory
|
|
198
|
+
limit, 100 MB by default)
|
|
199
|
+
suggested index: {status: 1, audit.created: -1, price: 1}
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
That example is the classic mistake: `{status, price, created_at}` looks sensible, but the range on
|
|
203
|
+
`price` sits between the equality key and the sort key, so the index stops providing the ordering.
|
|
204
|
+
Swapping the last two fixes it, and nothing about the query itself changes.
|
|
205
|
+
|
|
206
|
+
It also flags the predicates that cannot use an index at all — case-insensitive or unanchored
|
|
207
|
+
regex, `$ne`/`$nin`, `$exists: false` — reports covered queries, and understands that a sort on a
|
|
208
|
+
field pinned by an equality match is free. `extra_equality` is for the clauses your own code adds
|
|
209
|
+
(a tenant id, a soft-delete flag): they belong at the front of the index, and the advice is wrong
|
|
210
|
+
without them.
|
|
211
|
+
|
|
212
|
+
**This is a lint, not a query planner.** It reasons about the query shape, not your data
|
|
213
|
+
distribution or the real plan cache. `explain()` remains the only ground truth; this is here to
|
|
214
|
+
catch the obvious problems in CI or a dev-mode log, before they reach a database.
|
|
215
|
+
|
|
216
|
+
## Safety
|
|
217
|
+
|
|
218
|
+
- **Keys** are matched against the schema, so `$where=...` or `age__$gt=...` never reach the driver.
|
|
219
|
+
- **Values** are only ever coerced to the declared scalar type. Nothing is `eval`'d or parsed as
|
|
220
|
+
JSON, so a value of `{"$ne": null}` stays the harmless nine-character string it is.
|
|
221
|
+
- **Substring search is escaped.** `contains` / `startswith` / `endswith` run the value through
|
|
222
|
+
`re.escape` and anchor it, so `?name__contains=.*` looks for a literal `.*`. Raw `regex` is
|
|
223
|
+
opt-in per field and length-capped.
|
|
224
|
+
- **`per_page`** is clamped to `max_per_page` (default 100), so a client cannot ask for the
|
|
225
|
+
collection.
|
|
226
|
+
- **Projection is whitelisted**, so a field marked `projectable=False` cannot be selected even
|
|
227
|
+
when it is filterable.
|
|
228
|
+
|
|
229
|
+
Case-insensitive matching cannot use an ordinary index. `case_sensitive=True` drops the `i` option,
|
|
230
|
+
which is what makes `startswith` an index-friendly prefix scan on a large collection.
|
|
231
|
+
|
|
232
|
+
## Errors
|
|
233
|
+
|
|
234
|
+
All inherit `QSMongoError` (a `ValueError`) and carry the offending `.param`:
|
|
235
|
+
|
|
236
|
+
`UnknownField` · `UnsupportedOperator` · `InvalidValue` · `InvalidPagination` · `InvalidCursor` ·
|
|
237
|
+
`InvalidProjection`
|
|
238
|
+
|
|
239
|
+
```python
|
|
240
|
+
try:
|
|
241
|
+
query = parse(request.url.query, schema, cursors=cursors)
|
|
242
|
+
except QSMongoError as exc:
|
|
243
|
+
raise HTTPException(status_code=400, detail={"parameter": exc.param, "error": str(exc)})
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
See [`examples/fastapi_app.py`](examples/fastapi_app.py) for a complete endpoint.
|
|
247
|
+
|
|
248
|
+
## Edge cases, on purpose
|
|
249
|
+
|
|
250
|
+
Covered in [`tests/test_edge_cases.py`](tests/test_edge_cases.py):
|
|
251
|
+
|
|
252
|
+
- `title=black and white` — the word `and` is a value, not a keyword.
|
|
253
|
+
- `title=Dolce%20%26%20Gabbana` — an encoded `&` stays inside the value.
|
|
254
|
+
- `title=Dolce & Gabbana` — an *unencoded* `&` cannot be recovered by any parser, so it raises
|
|
255
|
+
rather than silently searching for `Dolce`.
|
|
256
|
+
- `tag__in=red\,blue,green` — a backslash escapes a comma inside a list.
|
|
257
|
+
- `title=black+white` — `+` decodes to a space, per form encoding.
|
|
258
|
+
- `age=` — a blank value on a typed field is an error, not `0`.
|
|
259
|
+
|
|
260
|
+
## Composing with your own scope
|
|
261
|
+
|
|
262
|
+
`query.filter` is a plain dict, so multi-tenant scoping stays yours — the library never invents
|
|
263
|
+
clauses on your behalf:
|
|
264
|
+
|
|
265
|
+
```python
|
|
266
|
+
collection.find({"$and": [{"tenant_id": user.tenant_id}, query.filter]})
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
Or use the aggregation form when you need `$lookup` afterwards:
|
|
270
|
+
|
|
271
|
+
```python
|
|
272
|
+
collection.aggregate([*query.pipeline(), {"$lookup": {...}}])
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
## Prior art
|
|
276
|
+
|
|
277
|
+
[`mongo-queries-manager`](https://pypi.org/project/mongo-queries-manager/) solves the same problem
|
|
278
|
+
and has more features — projection with `$elemMatch`, `$text`, custom casters. It takes the
|
|
279
|
+
opposite default: every field is queryable unless blacklisted, and types are inferred from the
|
|
280
|
+
value (`5.6` becomes a float, `true` becomes a bool).
|
|
281
|
+
|
|
282
|
+
Use it if you want breadth, or don't want to declare a schema.
|
|
283
|
+
|
|
284
|
+
Use `qsmongo` if you want an undeclared field to be an error rather than a silent leak, a numeric
|
|
285
|
+
SKU like `01234` to stay the string it is in your documents, and cursor pagination.
|
|
286
|
+
[`fastapi-filter`](https://pypi.org/project/fastapi-filter/) is also whitelist-based via pydantic
|
|
287
|
+
models, but is coupled to FastAPI and an ODM; `qsmongo` takes a string and returns a dict.
|
|
288
|
+
|
|
289
|
+
## Development
|
|
290
|
+
|
|
291
|
+
```bash
|
|
292
|
+
pip install -e ".[dev]"
|
|
293
|
+
pytest
|
|
294
|
+
ruff check .
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
## License
|
|
298
|
+
|
|
299
|
+
MIT
|
qsmongo-0.3.0/README.md
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
# qsmongo
|
|
2
|
+
|
|
3
|
+
[](https://github.com/boskodjokic/qsmongo/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.python.org/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
Turn an HTTP query string into a MongoDB query that is **safe by construction** — nothing is
|
|
8
|
+
queryable until you declare it — and **fast by construction**: keyset pagination that costs the
|
|
9
|
+
same on page 10,000 as on page 1.
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
GET /users?age__gte=21&status__in=active,trial&fields=name,age&sort=-created_at&per_page=50
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
from qsmongo import Field, Schema, parse
|
|
17
|
+
|
|
18
|
+
schema = Schema(
|
|
19
|
+
name=Field(str),
|
|
20
|
+
age=Field(int),
|
|
21
|
+
status=Field(str),
|
|
22
|
+
created_at=Field(datetime, alias="audit.created"),
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
query = parse(request.url.query, schema)
|
|
26
|
+
|
|
27
|
+
query.filter # {'age': {'$gte': 21}, 'status': {'$in': ['active', 'trial']}}
|
|
28
|
+
query.projection # {'name': 1, 'age': 1}
|
|
29
|
+
query.sort # [('audit.created', -1)]
|
|
30
|
+
|
|
31
|
+
collection.find(**query.find_kwargs())
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
No dependencies. Python 3.10+.
|
|
35
|
+
|
|
36
|
+
## Why
|
|
37
|
+
|
|
38
|
+
Every REST API over MongoDB grows the same ad-hoc filter parser, and it goes wrong in the same
|
|
39
|
+
five ways:
|
|
40
|
+
|
|
41
|
+
1. **Type mismatch.** `?age=21` gives you the string `"21"`. Mongo does not match `21` against
|
|
42
|
+
`"21"`, so the endpoint returns an empty list and nobody can tell whether the data is missing or
|
|
43
|
+
the filter is broken.
|
|
44
|
+
2. **Operator injection.** Passing user input into a query document unfiltered means a crafted
|
|
45
|
+
parameter can inject `$where` or `$ne` and walk straight past your access checks.
|
|
46
|
+
3. **Separator collisions.** A product genuinely named `Dolce & Gabbana`, a colour called
|
|
47
|
+
`black and white`, a tag containing a comma — each one quietly truncates the filter.
|
|
48
|
+
4. **Unbounded pages.** `?per_page=100000` is a denial-of-service vector you shipped yourself.
|
|
49
|
+
5. **Offset pagination.** `skip=100000` makes the server walk 100,000 documents it will never
|
|
50
|
+
return, and when the sort key has ties, documents shift between pages and are skipped or served
|
|
51
|
+
twice.
|
|
52
|
+
|
|
53
|
+
## Install
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
pip install qsmongo
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## The grammar
|
|
60
|
+
|
|
61
|
+
| Query string | Result |
|
|
62
|
+
| --- | --- |
|
|
63
|
+
| `name=Ada` | `{"name": "Ada"}` |
|
|
64
|
+
| `age__gte=21` | `{"age": {"$gte": 21}}` |
|
|
65
|
+
| `age__gte=21&age__lt=65` | `{"age": {"$gte": 21, "$lt": 65}}` |
|
|
66
|
+
| `status__in=active,trial` | `{"status": {"$in": ["active", "trial"]}}` |
|
|
67
|
+
| `name__contains=ada` | `{"name": {"$regex": "ada", "$options": "i"}}` |
|
|
68
|
+
| `name__startswith=Ad` | `{"name": {"$regex": "^Ad", "$options": "i"}}` |
|
|
69
|
+
| `deleted_at__exists=false` | `{"deleted_at": {"$exists": False}}` |
|
|
70
|
+
| `fields=name,price` | `{"name": 1, "price": 1}` |
|
|
71
|
+
| `fields=-internal_note` | `{"internal_note": 0}` |
|
|
72
|
+
| `sort=-created_at,name` | `[("created_at", -1), ("name", 1)]` |
|
|
73
|
+
| `page=2&per_page=50` | `skip=50, limit=50` |
|
|
74
|
+
| `after=<cursor>&per_page=50` | keyset range clause, `skip=0` |
|
|
75
|
+
|
|
76
|
+
Operators: `ne` `gt` `gte` `lt` `lte` `in` `nin` `contains` `startswith` `endswith` `exists`
|
|
77
|
+
`regex`. No suffix means equality.
|
|
78
|
+
|
|
79
|
+
## Declaring fields
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
Schema(
|
|
83
|
+
name=Field(str), # eq/ne/in/nin/exists + contains/startswith/endswith
|
|
84
|
+
age=Field(int), # numbers also get gt/gte/lt/lte
|
|
85
|
+
email=Field(str, alias="contact.email"), # public name differs from the document field
|
|
86
|
+
tag=Field(str, multi=True), # ?tag=red&tag=blue -> $in
|
|
87
|
+
sku=Field(str, case_sensitive=True), # index-friendly startswith
|
|
88
|
+
internal_cost=Field(float, projectable=False), # never selectable via ?fields
|
|
89
|
+
description=Field(str, ops=set()), # selectable, never queryable
|
|
90
|
+
pattern=Field(str, ops={"eq", "regex"}), # raw regex is opt-in
|
|
91
|
+
)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
A field that is not declared cannot be queried, sorted on, projected, or smuggled in as an
|
|
95
|
+
operator — `parse` raises `UnknownField` instead. That whitelist *is* the security model.
|
|
96
|
+
|
|
97
|
+
## Keyset pagination
|
|
98
|
+
|
|
99
|
+
Offset paging is fine until it isn't. Pass a `Cursors` codec and clients can page by cursor:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from qsmongo import Cursors, parse
|
|
103
|
+
|
|
104
|
+
cursors = Cursors(secret=settings.CURSOR_SECRET) # HMAC-signed, so clients cannot forge one
|
|
105
|
+
|
|
106
|
+
query = parse(request.url.query, schema, cursors=cursors)
|
|
107
|
+
items = list(collection.find(**query.find_kwargs()))
|
|
108
|
+
|
|
109
|
+
return {"items": items, "next": query.next_cursor(items[-1] if items else None)}
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
The client sends that token back as `?after=<cursor>`, and the skip becomes a range clause:
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
{"$or": [{"score": {"$lt": 42}}, {"score": 42, "_id": {"$gt": "abc"}}]}
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Details that matter:
|
|
119
|
+
|
|
120
|
+
- **An `_id` tiebreaker is appended to every sort** when cursors are enabled, so the ordering is
|
|
121
|
+
total. Without it, two documents sharing a `created_at` can straddle a page boundary and one of
|
|
122
|
+
them is lost.
|
|
123
|
+
- **Cursors are signed** when you supply a secret, and the secret never appears in a `repr`.
|
|
124
|
+
Unsigned, tampered, or truncated tokens raise `InvalidCursor`.
|
|
125
|
+
- **The sort must stay the same between pages.** Changing it raises rather than silently
|
|
126
|
+
paginating nonsense.
|
|
127
|
+
- `after` and `page` are different modes; sending both is an error.
|
|
128
|
+
- Forward-only. Backward paging is not implemented.
|
|
129
|
+
|
|
130
|
+
[`tests/test_keyset_property.py`](tests/test_keyset_property.py) walks a dataset engineered so that
|
|
131
|
+
every page boundary lands in the middle of a tie, and asserts the pages reconstruct the full
|
|
132
|
+
ordering exactly — no document skipped, none repeated.
|
|
133
|
+
|
|
134
|
+
## Index advice
|
|
135
|
+
|
|
136
|
+
A query that parses cleanly can still be a collection scan. `analyze` checks the query against your
|
|
137
|
+
declared indexes using MongoDB's ESR ordering — **E**quality keys first, then **S**ort keys, then
|
|
138
|
+
**R**ange keys — and suggests one when nothing fits:
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
from qsmongo import Index, analyze
|
|
142
|
+
|
|
143
|
+
INDEXES = Index.from_index_information(collection.index_information())
|
|
144
|
+
|
|
145
|
+
advice = analyze(query, INDEXES, extra_equality=["tenant_id"])
|
|
146
|
+
if not advice.ok:
|
|
147
|
+
log.warning("unindexed query\n%s", advice)
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
```
|
|
151
|
+
no index serves this query
|
|
152
|
+
- {status: 1, price: 1, audit.created: -1} filters this query but does not provide the sort
|
|
153
|
+
{audit.created: -1}, so MongoDB sorts in memory (it aborts past its blocking-sort memory
|
|
154
|
+
limit, 100 MB by default)
|
|
155
|
+
suggested index: {status: 1, audit.created: -1, price: 1}
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
That example is the classic mistake: `{status, price, created_at}` looks sensible, but the range on
|
|
159
|
+
`price` sits between the equality key and the sort key, so the index stops providing the ordering.
|
|
160
|
+
Swapping the last two fixes it, and nothing about the query itself changes.
|
|
161
|
+
|
|
162
|
+
It also flags the predicates that cannot use an index at all — case-insensitive or unanchored
|
|
163
|
+
regex, `$ne`/`$nin`, `$exists: false` — reports covered queries, and understands that a sort on a
|
|
164
|
+
field pinned by an equality match is free. `extra_equality` is for the clauses your own code adds
|
|
165
|
+
(a tenant id, a soft-delete flag): they belong at the front of the index, and the advice is wrong
|
|
166
|
+
without them.
|
|
167
|
+
|
|
168
|
+
**This is a lint, not a query planner.** It reasons about the query shape, not your data
|
|
169
|
+
distribution or the real plan cache. `explain()` remains the only ground truth; this is here to
|
|
170
|
+
catch the obvious problems in CI or a dev-mode log, before they reach a database.
|
|
171
|
+
|
|
172
|
+
## Safety
|
|
173
|
+
|
|
174
|
+
- **Keys** are matched against the schema, so `$where=...` or `age__$gt=...` never reach the driver.
|
|
175
|
+
- **Values** are only ever coerced to the declared scalar type. Nothing is `eval`'d or parsed as
|
|
176
|
+
JSON, so a value of `{"$ne": null}` stays the harmless nine-character string it is.
|
|
177
|
+
- **Substring search is escaped.** `contains` / `startswith` / `endswith` run the value through
|
|
178
|
+
`re.escape` and anchor it, so `?name__contains=.*` looks for a literal `.*`. Raw `regex` is
|
|
179
|
+
opt-in per field and length-capped.
|
|
180
|
+
- **`per_page`** is clamped to `max_per_page` (default 100), so a client cannot ask for the
|
|
181
|
+
collection.
|
|
182
|
+
- **Projection is whitelisted**, so a field marked `projectable=False` cannot be selected even
|
|
183
|
+
when it is filterable.
|
|
184
|
+
|
|
185
|
+
Case-insensitive matching cannot use an ordinary index. `case_sensitive=True` drops the `i` option,
|
|
186
|
+
which is what makes `startswith` an index-friendly prefix scan on a large collection.
|
|
187
|
+
|
|
188
|
+
## Errors
|
|
189
|
+
|
|
190
|
+
All inherit `QSMongoError` (a `ValueError`) and carry the offending `.param`:
|
|
191
|
+
|
|
192
|
+
`UnknownField` · `UnsupportedOperator` · `InvalidValue` · `InvalidPagination` · `InvalidCursor` ·
|
|
193
|
+
`InvalidProjection`
|
|
194
|
+
|
|
195
|
+
```python
|
|
196
|
+
try:
|
|
197
|
+
query = parse(request.url.query, schema, cursors=cursors)
|
|
198
|
+
except QSMongoError as exc:
|
|
199
|
+
raise HTTPException(status_code=400, detail={"parameter": exc.param, "error": str(exc)})
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
See [`examples/fastapi_app.py`](examples/fastapi_app.py) for a complete endpoint.
|
|
203
|
+
|
|
204
|
+
## Edge cases, on purpose
|
|
205
|
+
|
|
206
|
+
Covered in [`tests/test_edge_cases.py`](tests/test_edge_cases.py):
|
|
207
|
+
|
|
208
|
+
- `title=black and white` — the word `and` is a value, not a keyword.
|
|
209
|
+
- `title=Dolce%20%26%20Gabbana` — an encoded `&` stays inside the value.
|
|
210
|
+
- `title=Dolce & Gabbana` — an *unencoded* `&` cannot be recovered by any parser, so it raises
|
|
211
|
+
rather than silently searching for `Dolce`.
|
|
212
|
+
- `tag__in=red\,blue,green` — a backslash escapes a comma inside a list.
|
|
213
|
+
- `title=black+white` — `+` decodes to a space, per form encoding.
|
|
214
|
+
- `age=` — a blank value on a typed field is an error, not `0`.
|
|
215
|
+
|
|
216
|
+
## Composing with your own scope
|
|
217
|
+
|
|
218
|
+
`query.filter` is a plain dict, so multi-tenant scoping stays yours — the library never invents
|
|
219
|
+
clauses on your behalf:
|
|
220
|
+
|
|
221
|
+
```python
|
|
222
|
+
collection.find({"$and": [{"tenant_id": user.tenant_id}, query.filter]})
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Or use the aggregation form when you need `$lookup` afterwards:
|
|
226
|
+
|
|
227
|
+
```python
|
|
228
|
+
collection.aggregate([*query.pipeline(), {"$lookup": {...}}])
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
## Prior art
|
|
232
|
+
|
|
233
|
+
[`mongo-queries-manager`](https://pypi.org/project/mongo-queries-manager/) solves the same problem
|
|
234
|
+
and has more features — projection with `$elemMatch`, `$text`, custom casters. It takes the
|
|
235
|
+
opposite default: every field is queryable unless blacklisted, and types are inferred from the
|
|
236
|
+
value (`5.6` becomes a float, `true` becomes a bool).
|
|
237
|
+
|
|
238
|
+
Use it if you want breadth, or don't want to declare a schema.
|
|
239
|
+
|
|
240
|
+
Use `qsmongo` if you want an undeclared field to be an error rather than a silent leak, a numeric
|
|
241
|
+
SKU like `01234` to stay the string it is in your documents, and cursor pagination.
|
|
242
|
+
[`fastapi-filter`](https://pypi.org/project/fastapi-filter/) is also whitelist-based via pydantic
|
|
243
|
+
models, but is coupled to FastAPI and an ODM; `qsmongo` takes a string and returns a dict.
|
|
244
|
+
|
|
245
|
+
## Development
|
|
246
|
+
|
|
247
|
+
```bash
|
|
248
|
+
pip install -e ".[dev]"
|
|
249
|
+
pytest
|
|
250
|
+
ruff check .
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
## License
|
|
254
|
+
|
|
255
|
+
MIT
|