fromscratch-pydocdb 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fromscratch_pydocdb-0.3.0.dist-info/METADATA +146 -0
- fromscratch_pydocdb-0.3.0.dist-info/RECORD +17 -0
- fromscratch_pydocdb-0.3.0.dist-info/WHEEL +4 -0
- fromscratch_pydocdb-0.3.0.dist-info/licenses/LICENSE +21 -0
- pydocdb/__init__.py +48 -0
- pydocdb/_version.py +24 -0
- pydocdb/aggregation.py +377 -0
- pydocdb/atomic.py +66 -0
- pydocdb/collection.py +467 -0
- pydocdb/document.py +145 -0
- pydocdb/errors.py +58 -0
- pydocdb/indexing.py +126 -0
- pydocdb/operators.py +187 -0
- pydocdb/persistence.py +125 -0
- pydocdb/projection.py +101 -0
- pydocdb/py.typed +0 -0
- pydocdb/query.py +188 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: fromscratch-pydocdb
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: An educational document store library built from scratch in Python
|
|
5
|
+
Project-URL: Homepage, https://github.com/robcox2910/pydoc-db
|
|
6
|
+
Project-URL: Repository, https://github.com/robcox2910/pydoc-db
|
|
7
|
+
Project-URL: Issues, https://github.com/robcox2910/pydoc-db/issues
|
|
8
|
+
Project-URL: Documentation, https://robcox2910.github.io/pydoc-db/
|
|
9
|
+
Author: Rob Cox
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: database,document,education,nosql,store
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Education
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Topic :: Education
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.14
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: commitizen>=4.4.0; extra == 'dev'
|
|
24
|
+
Requires-Dist: hypothesis>=6.170; extra == 'dev'
|
|
25
|
+
Requires-Dist: pip-audit>=2.10.0; extra == 'dev'
|
|
26
|
+
Requires-Dist: pre-commit>=4.0.1; extra == 'dev'
|
|
27
|
+
Requires-Dist: pyright>=1.1.400; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest-cov>=6.0.0; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest>=8.3.4; extra == 'dev'
|
|
30
|
+
Requires-Dist: ruff>=0.9.3; extra == 'dev'
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# PyDocDB
|
|
34
|
+
|
|
35
|
+
An educational document store (like MongoDB) built from scratch in Python. Part of the **PyLearn** platform for kids aged 12+.
|
|
36
|
+
|
|
37
|
+
PyDocDB stores JSON-like documents without requiring a fixed schema -- each document can have different fields, just like how every index card in a filing cabinet can have different information on it.
|
|
38
|
+
|
|
39
|
+
## Features
|
|
40
|
+
|
|
41
|
+
- **Documents** -- Dict-like objects with auto-generated unique IDs
|
|
42
|
+
- **Collections** -- Named groups of documents with full CRUD operations
|
|
43
|
+
- **Query Language** -- Filter with operators: `$eq`, `$gt`, `$lt`, `$gte`, `$lte`, `$ne`, `$in`, `$exists`, `$and`, `$or`, `$not`
|
|
44
|
+
- **Nested Field Access** -- Query nested fields with dot notation (e.g., `"address.city"`)
|
|
45
|
+
- **Projection** -- Return only specific fields from documents
|
|
46
|
+
- **Indexing** -- Create indexes on fields for faster lookups
|
|
47
|
+
- **Persistence** -- Save/load collections to JSON files on disk
|
|
48
|
+
- **Aggregation** -- Pipeline with `$match`, `$group`, `$sort`, `$limit`, `$count`
|
|
49
|
+
|
|
50
|
+
## Quick Start
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from pydocdb import Collection, save_collection, load_collection, aggregate
|
|
54
|
+
|
|
55
|
+
# Create a collection (like labeling a new folder in a filing cabinet)
|
|
56
|
+
students = Collection("students")
|
|
57
|
+
|
|
58
|
+
# Insert documents (like filing index cards)
|
|
59
|
+
students.insert_one({"name": "Alice", "age": 14, "grade": "9th"})
|
|
60
|
+
students.insert_one({"name": "Bob", "age": 13, "hobby": "chess"})
|
|
61
|
+
students.insert_one({"name": "Charlie", "age": 15, "grade": "10th"})
|
|
62
|
+
|
|
63
|
+
# Find documents with queries
|
|
64
|
+
teenagers = students.find({"age": {"$gte": 13}})
|
|
65
|
+
alice = students.find_one({"name": "Alice"})
|
|
66
|
+
|
|
67
|
+
# Update and delete
|
|
68
|
+
students.update_one({"name": "Alice"}, {"age": 15})
|
|
69
|
+
students.delete_one({"name": "Bob"})
|
|
70
|
+
|
|
71
|
+
# Use projection to get only specific fields
|
|
72
|
+
names = students.find({}, projection={"name": 1, "_id": 0})
|
|
73
|
+
|
|
74
|
+
# Aggregate data
|
|
75
|
+
result = aggregate(students.to_list(), [
|
|
76
|
+
{"$group": {"_id": "$grade", "count": {"$sum": 1}}},
|
|
77
|
+
{"$sort": {"count": -1}},
|
|
78
|
+
])
|
|
79
|
+
|
|
80
|
+
# Save to disk and load later
|
|
81
|
+
save_collection(students, "students.json")
|
|
82
|
+
loaded = load_collection("students.json")
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## Installation
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
uv add pydocdb
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Development
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
git clone https://github.com/robcox2910/pydoc-db.git
|
|
95
|
+
cd pydoc-db
|
|
96
|
+
uv sync --all-extras
|
|
97
|
+
uv run pytest
|
|
98
|
+
uv run ruff check src/ tests/
|
|
99
|
+
uv run pyright src tests
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
## Releasing
|
|
103
|
+
|
|
104
|
+
Other projects in the series ([PyStack](https://github.com/robcox2910/pystack)) use a pinned release of this one,
|
|
105
|
+
not whatever is on `main`. To publish a change for them:
|
|
106
|
+
|
|
107
|
+
1. Merge the change to `main`.
|
|
108
|
+
2. Tag it: `git tag -a vX.Y.Z -m vX.Y.Z && git push origin vX.Y.Z`.
|
|
109
|
+
A change to the public API bumps the minor version (`v0.1.0` → `v0.2.0`);
|
|
110
|
+
anything else bumps the patch (`v0.1.0` → `v0.1.1`).
|
|
111
|
+
3. In each project that uses it, edit the tag in `[tool.uv.sources]`, run
|
|
112
|
+
`uv lock --upgrade-package <name>`, and open a pull request so CI checks
|
|
113
|
+
the pair together.
|
|
114
|
+
|
|
115
|
+
<!-- series:start -->
|
|
116
|
+
## Related Projects
|
|
117
|
+
|
|
118
|
+
This project is one of a series that builds real systems "from scratch in Python" to teach how they work. See the [series home](https://robcox2910.github.io/) for the full story.
|
|
119
|
+
|
|
120
|
+
| Project | What it teaches | Repository | Docs |
|
|
121
|
+
| --- | --- | --- | --- |
|
|
122
|
+
| PyOS | Operating systems | [py-os](https://github.com/robcox2910/py-os) | [Docs](https://robcox2910.github.io/py-os/) |
|
|
123
|
+
| Pebble | Compilers and programming languages | [pebble-lang](https://github.com/robcox2910/pebble-lang) | [Docs](https://robcox2910.github.io/pebble-lang/) |
|
|
124
|
+
| PyDB | Relational databases | [pydb](https://github.com/robcox2910/pydb) | [Docs](https://robcox2910.github.io/pydb/) |
|
|
125
|
+
| PyKV | Key-value stores | [pykv](https://github.com/robcox2910/pykv) | [Docs](https://robcox2910.github.io/pykv/) |
|
|
126
|
+
| **PyDocDB** | **Document databases** | [pydoc-db](https://github.com/robcox2910/pydoc-db) | [Docs](https://robcox2910.github.io/pydoc-db/) |
|
|
127
|
+
| PyGraphDB | Graph databases | [pygraphdb](https://github.com/robcox2910/pygraphdb) | [Docs](https://robcox2910.github.io/pygraphdb/) |
|
|
128
|
+
| PyTSDB | Time-series databases | [pytsdb](https://github.com/robcox2910/pytsdb) | [Docs](https://robcox2910.github.io/pytsdb/) |
|
|
129
|
+
| PyVecDB | Vector databases | [pyvecdb](https://github.com/robcox2910/pyvecdb) | [Docs](https://robcox2910.github.io/pyvecdb/) |
|
|
130
|
+
| PyColDB | Column-family databases | [pycoldb](https://github.com/robcox2910/pycoldb) | [Docs](https://robcox2910.github.io/pycoldb/) |
|
|
131
|
+
| PyCrypt | Cryptography | [pycrypt](https://github.com/robcox2910/pycrypt) | [Docs](https://robcox2910.github.io/pycrypt/) |
|
|
132
|
+
| PyNet | Computer networking | [pynet](https://github.com/robcox2910/pynet) | [Docs](https://robcox2910.github.io/pynet/) |
|
|
133
|
+
| PyWeb | HTTP web servers | [pyweb](https://github.com/robcox2910/pyweb) | [Docs](https://robcox2910.github.io/pyweb/) |
|
|
134
|
+
| PyGit | Version control | [pygit](https://github.com/robcox2910/pygit) | [Docs](https://robcox2910.github.io/pygit/) |
|
|
135
|
+
| PySearch | Search engines | [pysearch](https://github.com/robcox2910/pysearch) | [Docs](https://robcox2910.github.io/pysearch/) |
|
|
136
|
+
| PyMQ | Message queues | [pymq](https://github.com/robcox2910/pymq) | [Docs](https://robcox2910.github.io/pymq/) |
|
|
137
|
+
| PyLLM | Language models (the brain) | [pyllm](https://github.com/robcox2910/pyllm) | [Docs](https://robcox2910.github.io/pyllm/) |
|
|
138
|
+
| PyScheduler | Job scheduling | [pyscheduler](https://github.com/robcox2910/pyscheduler) | [Docs](https://robcox2910.github.io/pyscheduler/) |
|
|
139
|
+
| PyStack | Full-stack integration | [pystack](https://github.com/robcox2910/pystack) | [Docs](https://robcox2910.github.io/pystack/) |
|
|
140
|
+
<!-- series:end -->
|
|
141
|
+
|
|
142
|
+
<!-- docs:start -->
|
|
143
|
+
## Documentation
|
|
144
|
+
|
|
145
|
+
Full docs (with kid-friendly explanations of every idea): https://robcox2910.github.io/pydoc-db/
|
|
146
|
+
<!-- docs:end -->
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
pydocdb/__init__.py,sha256=FS8xCxvqT1uu3lHGcKd9Rh82ajGfHIHFsiRKHa_OUgM,1369
|
|
2
|
+
pydocdb/_version.py,sha256=RCY04NY0kBCyn6ttH6lnVL6Gz2gceOrt45f0yDOOU8A,520
|
|
3
|
+
pydocdb/aggregation.py,sha256=ucovumNWSRoLUhAx8NscXRt2hg0oCnB9yuSfjXtmQ5g,10896
|
|
4
|
+
pydocdb/atomic.py,sha256=QLLPGZqpA-TgVV5lDEKCLB7vVr05bxj97_Fwykz_gi4,2345
|
|
5
|
+
pydocdb/collection.py,sha256=7cem5CLJghwGk5pJE8o_dKkp2mmvIjJsut_YKIhOMnU,15724
|
|
6
|
+
pydocdb/document.py,sha256=ofsx43JfueKFvpZ1pBCk4xh6JBaojkxQzJ6dI9G-tDw,4920
|
|
7
|
+
pydocdb/errors.py,sha256=vi9aKiq28H7sWH8xk1e_X8IR2F08ecDGOWcJCHkJmp8,1792
|
|
8
|
+
pydocdb/indexing.py,sha256=0yvemUgqXgD7AMV8983aiEWiftQ7-BHa_yde_r_dolc,3849
|
|
9
|
+
pydocdb/operators.py,sha256=IA-3gEHiyNvkUn5-iGtdw7Ot_rtPn0LLAYr4e-74o-w,5173
|
|
10
|
+
pydocdb/persistence.py,sha256=LqCEmWYgN8iFrCg_TYrzRkdyvB_0C1VlxAcsFB7XE80,4672
|
|
11
|
+
pydocdb/projection.py,sha256=s2et1ErZjTiMsY1PqvRwD_YL70B3061KN6c9TKljiOs,2908
|
|
12
|
+
pydocdb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
pydocdb/query.py,sha256=tdK5DfvTP8pMeiyWrIqUG-liOxEqoHp7-W4lkZweXEk,6195
|
|
14
|
+
fromscratch_pydocdb-0.3.0.dist-info/METADATA,sha256=5AB3ytxcNtQNGq4Jp4VqGE9y_gTqys4rPLuFotWMVYw,7027
|
|
15
|
+
fromscratch_pydocdb-0.3.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
16
|
+
fromscratch_pydocdb-0.3.0.dist-info/licenses/LICENSE,sha256=NtHM2ak--oE71VCcY50kcCoC1cKS0LRUsxq_xA89RNc,1064
|
|
17
|
+
fromscratch_pydocdb-0.3.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Rob Cox
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pydocdb/__init__.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""PyDocDB -- an educational document store for kids aged 12+.
|
|
2
|
+
|
|
3
|
+
PyDocDB is a document database (like MongoDB) built from scratch in Python.
|
|
4
|
+
It stores JSON-like documents without requiring a fixed schema -- each
|
|
5
|
+
document can have different fields, just like how every index card in a
|
|
6
|
+
filing cabinet can have different information on it.
|
|
7
|
+
|
|
8
|
+
Quick start::
|
|
9
|
+
|
|
10
|
+
from pydocdb import Collection
|
|
11
|
+
|
|
12
|
+
students = Collection("students")
|
|
13
|
+
students.insert_one({"name": "Alice", "age": 14, "grade": "9th"})
|
|
14
|
+
students.insert_one({"name": "Bob", "age": 13, "hobby": "chess"})
|
|
15
|
+
|
|
16
|
+
# Find students older than 13
|
|
17
|
+
results = students.find({"age": {"$gt": 13}})
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from pydocdb.aggregation import aggregate
|
|
21
|
+
from pydocdb.collection import Collection
|
|
22
|
+
from pydocdb.document import Document, DocumentData
|
|
23
|
+
from pydocdb.errors import (
|
|
24
|
+
DocumentNotFoundError,
|
|
25
|
+
DuplicateKeyError,
|
|
26
|
+
InvalidPipelineError,
|
|
27
|
+
InvalidQueryError,
|
|
28
|
+
PersistenceError,
|
|
29
|
+
PyDocDBError,
|
|
30
|
+
)
|
|
31
|
+
from pydocdb.indexing import Index
|
|
32
|
+
from pydocdb.persistence import load_collection, save_collection
|
|
33
|
+
|
|
34
|
+
__all__: list[str] = [
|
|
35
|
+
"Collection",
|
|
36
|
+
"Document",
|
|
37
|
+
"DocumentData",
|
|
38
|
+
"DocumentNotFoundError",
|
|
39
|
+
"DuplicateKeyError",
|
|
40
|
+
"Index",
|
|
41
|
+
"InvalidPipelineError",
|
|
42
|
+
"InvalidQueryError",
|
|
43
|
+
"PersistenceError",
|
|
44
|
+
"PyDocDBError",
|
|
45
|
+
"aggregate",
|
|
46
|
+
"load_collection",
|
|
47
|
+
"save_collection",
|
|
48
|
+
]
|
pydocdb/_version.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.3.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 3, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
pydocdb/aggregation.py
ADDED
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
"""Aggregation pipeline for PyDocDB.
|
|
2
|
+
|
|
3
|
+
Imagine you have a huge collection of trading cards. You might want to:
|
|
4
|
+
1. First, pick out only the rare cards ($match)
|
|
5
|
+
2. Then, group them by type -- Fire, Water, Grass ($group)
|
|
6
|
+
3. Sort the groups by how many cards each has ($sort)
|
|
7
|
+
4. Only look at the top 3 groups ($limit)
|
|
8
|
+
5. Just count how many groups there are ($count)
|
|
9
|
+
|
|
10
|
+
That's an aggregation pipeline! It's a series of steps (called "stages")
|
|
11
|
+
that process your documents one step at a time. Each stage takes the
|
|
12
|
+
output of the previous stage and transforms it.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from pydocdb.document import Document, DocumentData, extract_field
|
|
18
|
+
from pydocdb.errors import InvalidPipelineError
|
|
19
|
+
from pydocdb.indexing import make_hashable
|
|
20
|
+
from pydocdb.query import matches
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def aggregate(
|
|
24
|
+
documents: list[DocumentData],
|
|
25
|
+
pipeline: list[dict[str, Any]],
|
|
26
|
+
) -> list[DocumentData]:
|
|
27
|
+
"""Run an aggregation pipeline on a list of documents.
|
|
28
|
+
|
|
29
|
+
Process documents through a series of stages, where each stage
|
|
30
|
+
transforms the data in some way.
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
documents: The input documents to process.
|
|
34
|
+
pipeline: A list of stage dictionaries (e.g., [{"$match": {...}}, {"$sort": {...}}]).
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
The resulting documents after all stages have been applied.
|
|
38
|
+
|
|
39
|
+
Raises:
|
|
40
|
+
InvalidPipelineError: If a stage is unknown or malformed.
|
|
41
|
+
|
|
42
|
+
"""
|
|
43
|
+
current: list[DocumentData] = list(documents)
|
|
44
|
+
|
|
45
|
+
for stage in pipeline:
|
|
46
|
+
if len(stage) != 1:
|
|
47
|
+
msg = "Each pipeline stage must have exactly one operator"
|
|
48
|
+
raise InvalidPipelineError(msg)
|
|
49
|
+
|
|
50
|
+
stage_name = next(iter(stage))
|
|
51
|
+
stage_value = stage[stage_name]
|
|
52
|
+
|
|
53
|
+
if stage_name == "$match":
|
|
54
|
+
current = _stage_match(current, stage_value)
|
|
55
|
+
elif stage_name == "$group":
|
|
56
|
+
current = _stage_group(current, stage_value)
|
|
57
|
+
elif stage_name == "$sort":
|
|
58
|
+
current = _stage_sort(current, stage_value)
|
|
59
|
+
elif stage_name == "$limit":
|
|
60
|
+
current = _stage_limit(current, stage_value)
|
|
61
|
+
elif stage_name == "$count":
|
|
62
|
+
current = _stage_count(current, stage_value)
|
|
63
|
+
else:
|
|
64
|
+
msg = f"Unknown pipeline stage: {stage_name}"
|
|
65
|
+
raise InvalidPipelineError(msg)
|
|
66
|
+
|
|
67
|
+
return current
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _stage_match(
|
|
71
|
+
documents: list[DocumentData],
|
|
72
|
+
query: dict[str, Any],
|
|
73
|
+
) -> list[DocumentData]:
|
|
74
|
+
"""Filter documents that match a query -- like picking cards from a pile.
|
|
75
|
+
|
|
76
|
+
Args:
|
|
77
|
+
documents: Input documents.
|
|
78
|
+
query: A query dictionary (same format as Collection.find).
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
Only the documents that match the query.
|
|
82
|
+
|
|
83
|
+
"""
|
|
84
|
+
return [doc for doc in documents if matches(Document(doc), query)]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _stage_group(
|
|
88
|
+
documents: list[DocumentData],
|
|
89
|
+
group_spec: dict[str, Any],
|
|
90
|
+
) -> list[DocumentData]:
|
|
91
|
+
"""Group documents by a field and compute aggregations.
|
|
92
|
+
|
|
93
|
+
Like sorting your cards into piles by type, then counting each pile.
|
|
94
|
+
|
|
95
|
+
The ``_id`` field in the spec tells us what to group by.
|
|
96
|
+
Other fields define accumulators:
|
|
97
|
+
- ``$sum``: Add up values (use 1 to count)
|
|
98
|
+
- ``$avg``: Calculate the average
|
|
99
|
+
- ``$min``: Find the smallest value
|
|
100
|
+
- ``$max``: Find the largest value
|
|
101
|
+
|
|
102
|
+
Note:
|
|
103
|
+
Group keys must be hashable to sort documents into piles, so a list
|
|
104
|
+
key comes back as a tuple and a dict key as a tuple of pairs in each
|
|
105
|
+
result's ``_id``. Grouping by simple values (strings, numbers) is the
|
|
106
|
+
common case and is unaffected.
|
|
107
|
+
|
|
108
|
+
Args:
|
|
109
|
+
documents: Input documents.
|
|
110
|
+
group_spec: A dict with ``_id`` (group key) and accumulator fields.
|
|
111
|
+
|
|
112
|
+
Returns:
|
|
113
|
+
One document per group with the computed values.
|
|
114
|
+
|
|
115
|
+
Raises:
|
|
116
|
+
InvalidPipelineError: If ``_id`` is missing from the spec.
|
|
117
|
+
|
|
118
|
+
"""
|
|
119
|
+
if "_id" not in group_spec:
|
|
120
|
+
msg = "$group stage requires an '_id' field"
|
|
121
|
+
raise InvalidPipelineError(msg)
|
|
122
|
+
|
|
123
|
+
group_key = group_spec["_id"]
|
|
124
|
+
|
|
125
|
+
# Build groups
|
|
126
|
+
groups: dict[Any, list[DocumentData]] = {}
|
|
127
|
+
for doc in documents:
|
|
128
|
+
if group_key is None:
|
|
129
|
+
key: Any = None
|
|
130
|
+
elif isinstance(group_key, str) and group_key.startswith("$"):
|
|
131
|
+
key = extract_field(doc, group_key[1:])
|
|
132
|
+
else:
|
|
133
|
+
key = group_key
|
|
134
|
+
|
|
135
|
+
# Make the key hashable
|
|
136
|
+
hashable_key = make_hashable(key)
|
|
137
|
+
if hashable_key not in groups:
|
|
138
|
+
groups[hashable_key] = []
|
|
139
|
+
groups[hashable_key].append(doc)
|
|
140
|
+
|
|
141
|
+
# Apply accumulators to each group
|
|
142
|
+
results: list[DocumentData] = []
|
|
143
|
+
for key_value, group_docs in groups.items():
|
|
144
|
+
result: DocumentData = {"_id": key_value}
|
|
145
|
+
|
|
146
|
+
for field, accumulator in group_spec.items():
|
|
147
|
+
if field == "_id":
|
|
148
|
+
continue
|
|
149
|
+
if isinstance(accumulator, dict):
|
|
150
|
+
acc_dict: dict[str, Any] = dict(accumulator) # pyright: ignore[reportUnknownArgumentType]
|
|
151
|
+
result[field] = _apply_accumulator(acc_dict, group_docs)
|
|
152
|
+
|
|
153
|
+
results.append(result)
|
|
154
|
+
|
|
155
|
+
return results
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _apply_accumulator(
|
|
159
|
+
accumulator: dict[str, Any],
|
|
160
|
+
group_docs: list[DocumentData],
|
|
161
|
+
) -> Any:
|
|
162
|
+
"""Apply a single accumulator to a group of documents.
|
|
163
|
+
|
|
164
|
+
Args:
|
|
165
|
+
accumulator: A dict like ``{"$sum": "$price"}`` or ``{"$sum": 1}``.
|
|
166
|
+
group_docs: The documents in this group.
|
|
167
|
+
|
|
168
|
+
Returns:
|
|
169
|
+
The computed value.
|
|
170
|
+
|
|
171
|
+
Raises:
|
|
172
|
+
InvalidPipelineError: If the accumulator operator is unknown.
|
|
173
|
+
|
|
174
|
+
"""
|
|
175
|
+
op = next(iter(accumulator))
|
|
176
|
+
value_spec = accumulator[op]
|
|
177
|
+
|
|
178
|
+
if op == "$sum":
|
|
179
|
+
return _acc_sum(value_spec, group_docs)
|
|
180
|
+
if op == "$avg":
|
|
181
|
+
return _acc_avg(value_spec, group_docs)
|
|
182
|
+
if op == "$min":
|
|
183
|
+
return _acc_min(value_spec, group_docs)
|
|
184
|
+
if op == "$max":
|
|
185
|
+
return _acc_max(value_spec, group_docs)
|
|
186
|
+
|
|
187
|
+
msg = f"Unknown accumulator: {op}"
|
|
188
|
+
raise InvalidPipelineError(msg)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _acc_sum(value_spec: Any, docs: list[DocumentData]) -> int | float:
|
|
192
|
+
"""Compute sum for a group.
|
|
193
|
+
|
|
194
|
+
Args:
|
|
195
|
+
value_spec: Either a number (like 1 for counting) or a field reference like "$price".
|
|
196
|
+
docs: The documents in the group.
|
|
197
|
+
|
|
198
|
+
Returns:
|
|
199
|
+
The sum.
|
|
200
|
+
|
|
201
|
+
"""
|
|
202
|
+
if isinstance(value_spec, (int, float)):
|
|
203
|
+
return value_spec * len(docs)
|
|
204
|
+
|
|
205
|
+
if isinstance(value_spec, str) and value_spec.startswith("$"):
|
|
206
|
+
field = value_spec[1:]
|
|
207
|
+
total: int | float = 0
|
|
208
|
+
for doc in docs:
|
|
209
|
+
val = extract_field(doc, field)
|
|
210
|
+
if isinstance(val, (int, float)):
|
|
211
|
+
total += val
|
|
212
|
+
return total
|
|
213
|
+
|
|
214
|
+
return 0
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _acc_avg(value_spec: Any, docs: list[DocumentData]) -> float:
|
|
218
|
+
"""Compute average for a group.
|
|
219
|
+
|
|
220
|
+
Args:
|
|
221
|
+
value_spec: A field reference like "$price".
|
|
222
|
+
docs: The documents in the group.
|
|
223
|
+
|
|
224
|
+
Returns:
|
|
225
|
+
The average, or 0.0 if no valid values.
|
|
226
|
+
|
|
227
|
+
"""
|
|
228
|
+
if isinstance(value_spec, str) and value_spec.startswith("$"):
|
|
229
|
+
field = value_spec[1:]
|
|
230
|
+
values: list[float] = []
|
|
231
|
+
for doc in docs:
|
|
232
|
+
val = extract_field(doc, field)
|
|
233
|
+
if isinstance(val, (int, float)):
|
|
234
|
+
values.append(float(val))
|
|
235
|
+
if values:
|
|
236
|
+
return sum(values) / len(values)
|
|
237
|
+
return 0.0
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _acc_min(value_spec: Any, docs: list[DocumentData]) -> Any:
|
|
241
|
+
"""Compute minimum for a group.
|
|
242
|
+
|
|
243
|
+
Args:
|
|
244
|
+
value_spec: A field reference like "$price".
|
|
245
|
+
docs: The documents in the group.
|
|
246
|
+
|
|
247
|
+
Returns:
|
|
248
|
+
The minimum value, or None if no valid values.
|
|
249
|
+
|
|
250
|
+
"""
|
|
251
|
+
if isinstance(value_spec, str) and value_spec.startswith("$"):
|
|
252
|
+
field = value_spec[1:]
|
|
253
|
+
values: list[Any] = []
|
|
254
|
+
for doc in docs:
|
|
255
|
+
val = extract_field(doc, field)
|
|
256
|
+
if val is not None:
|
|
257
|
+
values.append(val)
|
|
258
|
+
if values:
|
|
259
|
+
return min(values)
|
|
260
|
+
return None
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _acc_max(value_spec: Any, docs: list[DocumentData]) -> Any:
|
|
264
|
+
"""Compute maximum for a group.
|
|
265
|
+
|
|
266
|
+
Args:
|
|
267
|
+
value_spec: A field reference like "$price".
|
|
268
|
+
docs: The documents in the group.
|
|
269
|
+
|
|
270
|
+
Returns:
|
|
271
|
+
The maximum value, or None if no valid values.
|
|
272
|
+
|
|
273
|
+
"""
|
|
274
|
+
if isinstance(value_spec, str) and value_spec.startswith("$"):
|
|
275
|
+
field = value_spec[1:]
|
|
276
|
+
values: list[Any] = []
|
|
277
|
+
for doc in docs:
|
|
278
|
+
val = extract_field(doc, field)
|
|
279
|
+
if val is not None:
|
|
280
|
+
values.append(val)
|
|
281
|
+
if values:
|
|
282
|
+
return max(values)
|
|
283
|
+
return None
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _stage_sort(
|
|
287
|
+
documents: list[DocumentData],
|
|
288
|
+
sort_spec: dict[str, int],
|
|
289
|
+
) -> list[DocumentData]:
|
|
290
|
+
"""Sort documents by one or more fields.
|
|
291
|
+
|
|
292
|
+
Like arranging your cards in order -- alphabetically by name,
|
|
293
|
+
or by number from highest to lowest.
|
|
294
|
+
|
|
295
|
+
Args:
|
|
296
|
+
documents: Input documents.
|
|
297
|
+
sort_spec: A dict mapping field names to 1 (ascending) or -1 (descending).
|
|
298
|
+
|
|
299
|
+
Returns:
|
|
300
|
+
The sorted list of documents.
|
|
301
|
+
|
|
302
|
+
"""
|
|
303
|
+
result = list(documents)
|
|
304
|
+
# Apply sorts in reverse order so first key is primary
|
|
305
|
+
for field in reversed(sort_spec):
|
|
306
|
+
direction = sort_spec[field]
|
|
307
|
+
is_reverse = direction == -1
|
|
308
|
+
result.sort(
|
|
309
|
+
key=lambda doc, f=field: _sort_key(doc, f), # type: ignore[misc]
|
|
310
|
+
reverse=is_reverse,
|
|
311
|
+
)
|
|
312
|
+
return result
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _sort_key(doc: DocumentData, field: str) -> Any:
|
|
316
|
+
"""Get a sort-safe key for a document field.
|
|
317
|
+
|
|
318
|
+
Args:
|
|
319
|
+
doc: The document data.
|
|
320
|
+
field: The field to sort by.
|
|
321
|
+
|
|
322
|
+
Returns:
|
|
323
|
+
The value to sort by, with None values sorted last.
|
|
324
|
+
|
|
325
|
+
"""
|
|
326
|
+
val = extract_field(doc, field)
|
|
327
|
+
if val is None:
|
|
328
|
+
return (1, "")
|
|
329
|
+
return (0, val)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _stage_limit(
|
|
333
|
+
documents: list[DocumentData],
|
|
334
|
+
limit: Any,
|
|
335
|
+
) -> list[DocumentData]:
|
|
336
|
+
"""Limit the number of documents -- like looking at only the top N cards.
|
|
337
|
+
|
|
338
|
+
Args:
|
|
339
|
+
documents: Input documents.
|
|
340
|
+
limit: Maximum number of documents to return.
|
|
341
|
+
|
|
342
|
+
Returns:
|
|
343
|
+
At most ``limit`` documents.
|
|
344
|
+
|
|
345
|
+
Raises:
|
|
346
|
+
InvalidPipelineError: If limit is not a positive integer.
|
|
347
|
+
|
|
348
|
+
"""
|
|
349
|
+
if not isinstance(limit, int) or limit < 1:
|
|
350
|
+
msg = "$limit requires a positive integer"
|
|
351
|
+
raise InvalidPipelineError(msg)
|
|
352
|
+
return documents[:limit]
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _stage_count(
|
|
356
|
+
documents: list[DocumentData],
|
|
357
|
+
field_name: Any,
|
|
358
|
+
) -> list[DocumentData]:
|
|
359
|
+
"""Count the number of documents and store the result in a named field.
|
|
360
|
+
|
|
361
|
+
Like counting how many cards are in a pile and writing the number down.
|
|
362
|
+
|
|
363
|
+
Args:
|
|
364
|
+
documents: Input documents.
|
|
365
|
+
field_name: The name of the field to store the count in.
|
|
366
|
+
|
|
367
|
+
Returns:
|
|
368
|
+
A single-element list with the count.
|
|
369
|
+
|
|
370
|
+
Raises:
|
|
371
|
+
InvalidPipelineError: If field_name is not a string.
|
|
372
|
+
|
|
373
|
+
"""
|
|
374
|
+
if not isinstance(field_name, str):
|
|
375
|
+
msg = "$count requires a string field name"
|
|
376
|
+
raise InvalidPipelineError(msg)
|
|
377
|
+
return [{field_name: len(documents)}]
|