dataman-engine 0.1.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataman_engine-0.3.0/PKG-INFO +268 -0
- dataman_engine-0.3.0/README.md +228 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/pyproject.toml +27 -6
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/pyproject.toml.orig +27 -8
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/__init__.py +1 -1
- dataman_engine-0.3.0/src/dataman/asgi.py +3 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/cli.py +14 -4
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/auth.py +1 -1
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/crypto.py +23 -10
- dataman_engine-0.3.0/src/dataman/core/middleware.py +145 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/models.py +2 -0
- dataman_engine-0.3.0/src/dataman/core/urls.py +688 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/views.py +9 -3
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/django_setup.py +12 -0
- dataman_engine-0.3.0/src/dataman/wsgi.py +3 -0
- dataman_engine-0.1.1/PKG-INFO +0 -259
- dataman_engine-0.1.1/README.md +0 -221
- dataman_engine-0.1.1/src/dataman/core/middleware.py +0 -46
- dataman_engine-0.1.1/src/dataman/core/urls.py +0 -495
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/__init__.py +0 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/apps.py +0 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/audit.py +0 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/fields.py +0 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/masking.py +0 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/migrations/0001_initial.py +0 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/migrations/__init__.py +0 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/permissions.py +0 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/router.py +0 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/signals.py +0 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/templates/admin/dashboard.html +0 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/core/templates/admin/login.html +0 -0
- {dataman_engine-0.1.1 → dataman_engine-0.3.0}/src/dataman/py.typed +0 -0
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dataman-engine
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: A dynamic, headless data layer framework built on Django and DRF for instant CRUD APIs and custom business logic.
|
|
5
|
+
Keywords: django,rest,api,crud,headless,generator,scaffold
|
|
6
|
+
Author: Vikash G
|
|
7
|
+
Author-email: Vikash G <vikashgraja@gmail.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Environment :: Web Environment
|
|
12
|
+
Classifier: Framework :: Django
|
|
13
|
+
Classifier: Framework :: Django :: 6.0
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
21
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
23
|
+
Requires-Dist: django>=6.0.2
|
|
24
|
+
Requires-Dist: djangorestframework>=3.15.0
|
|
25
|
+
Requires-Dist: click>=8.1.7
|
|
26
|
+
Requires-Dist: python-dotenv>=1.0.1
|
|
27
|
+
Requires-Dist: django-filter>=25.1
|
|
28
|
+
Requires-Dist: requests>=2.32.3
|
|
29
|
+
Requires-Dist: pydantic>=2.10.5
|
|
30
|
+
Requires-Dist: drf-spectacular>=0.28.0
|
|
31
|
+
Requires-Dist: dj-database-url>=2.3.0
|
|
32
|
+
Requires-Dist: inflection>=0.5.1
|
|
33
|
+
Requires-Dist: uvicorn>=0.34.0
|
|
34
|
+
Requires-Dist: cryptography>=42.0.0
|
|
35
|
+
Requires-Dist: psycopg2-binary>=2.9.13
|
|
36
|
+
Requires-Python: >=3.12
|
|
37
|
+
Project-URL: Homepage, https://github.com/vikashgraja/dataman
|
|
38
|
+
Project-URL: Repository, https://github.com/vikashgraja/dataman.git
|
|
39
|
+
Description-Content-Type: text/markdown
|
|
40
|
+
|
|
41
|
+
# DataMan (Data MiddleMan)
|
|
42
|
+
|
|
43
|
+
[](https://github.com/vikashgraja/DataMan/blob/main/docs/QUICKSTART.md)
|
|
44
|
+
[](#supported-databases)
|
|
45
|
+
[](#asgi-production-mode)
|
|
46
|
+
[](https://github.com/vikashgraja/DataMan/blob/main/LICENSE)
|
|
47
|
+
|
|
48
|
+
**DataMan** is a dynamic, headless backend engine built on top of Django and Django REST Framework. It eliminates the boilerplate of writing standard CRUD APIs, routing, and serializers by allowing you to scaffold endpoints instantly from the command line while preserving your ability to inject custom business logic and strict validation whenever you need it.
|
|
49
|
+
|
|
50
|
+
Looking to build an API in 5 minutes? Check out the [Developer Quickstart Tutorial](https://github.com/vikashgraja/DataMan/blob/main/docs/QUICKSTART.md).
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## 30-Second Demo: Instant REST API
|
|
55
|
+
|
|
56
|
+
Get a fully functional, paginated, and documented REST API running in 4 commands:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
# 1. Install DataMan
|
|
60
|
+
pip install dataman-engine
|
|
61
|
+
|
|
62
|
+
# 2. Initialize project in current directory
|
|
63
|
+
dataman init
|
|
64
|
+
|
|
65
|
+
# 3. Scaffold an Employee table with full CRUD operations
|
|
66
|
+
dataman create table Employee -o crud
|
|
67
|
+
|
|
68
|
+
# 4. Apply migrations and start the server
|
|
69
|
+
dataman makemigration && dataman migrate
|
|
70
|
+
dataman server start
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
### What You Get Instantly:
|
|
74
|
+
* **`GET /api/default/employee/`** — Paginated list with multi-column filtering, search, and ordering.
|
|
75
|
+
* **`POST /api/default/employee/`** — Create records with validation and lifecycle hooks.
|
|
76
|
+
* **`GET /api/default/employee/{id}/`** — Point lookup with automatic relational joins (`?depth=1`).
|
|
77
|
+
* **`PUT /api/default/employee/{id}/`** — Full update.
|
|
78
|
+
* **`PATCH /api/default/employee/{id}/`** — Partial update.
|
|
79
|
+
* **`DELETE /api/default/employee/{id}/`** — Delete record.
|
|
80
|
+
* **Interactive Swagger UI**: [`http://127.0.0.1:8000/api/docs/`](http://127.0.0.1:8000/api/docs/)
|
|
81
|
+
* **OpenAPI 3.0 Schema**: [`http://127.0.0.1:8000/api/schema/`](http://127.0.0.1:8000/api/schema/)
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## Why DataMan?
|
|
86
|
+
|
|
87
|
+
Building standard CRUD backends with existing frameworks requires boilerplate code:
|
|
88
|
+
|
|
89
|
+
```text
|
|
90
|
+
Without DataMan:
|
|
91
|
+
Model ---> Serializer ---> ViewSet ---> Router ---> FilterSet ---> Permissions
|
|
92
|
+
|
|
93
|
+
With DataMan:
|
|
94
|
+
Model ---> Done! (Instant REST Endpoints + Swagger Docs + Auth)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
### Framework Comparison
|
|
98
|
+
|
|
99
|
+
| Framework | CRUD API Setup Requirements |
|
|
100
|
+
| :--- | :--- |
|
|
101
|
+
| **Django REST Framework (DRF)** | Model + Serializer + ViewSet + Router + FilterSet + Permissions |
|
|
102
|
+
| **FastAPI** | SQLAlchemy Model + Pydantic Schema + Router + Endpoints + Dependency Injection |
|
|
103
|
+
| **DataMan** | **Django Model Only (Instant Endpoints, Swagger, Auth & Hooks)** |
|
|
104
|
+
|
|
105
|
+
---
|
|
106
|
+
|
|
107
|
+
## Out-of-the-Box API Features
|
|
108
|
+
|
|
109
|
+
Every scaffolded endpoint automatically delivers production response envelopes:
|
|
110
|
+
|
|
111
|
+
```json
|
|
112
|
+
// GET /api/default/employee/?is_active=true&search=Engineering&ordering=-created_at
|
|
113
|
+
{
|
|
114
|
+
"count": 48,
|
|
115
|
+
"next": "http://127.0.0.1:8000/api/default/employee/?page=2",
|
|
116
|
+
"previous": null,
|
|
117
|
+
"results": [
|
|
118
|
+
{
|
|
119
|
+
"id": 101,
|
|
120
|
+
"name": "Jane Doe",
|
|
121
|
+
"department": "Engineering",
|
|
122
|
+
"email": "jane.doe@example.com",
|
|
123
|
+
"is_active": true,
|
|
124
|
+
"created_at": "2026-09-19T18:00:00Z"
|
|
125
|
+
}
|
|
126
|
+
]
|
|
127
|
+
}
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
* **Filtering**: Filter by query params (e.g., `?is_active=true&salary__gte=80000`).
|
|
131
|
+
* **Fast B-Tree Prefix Search**: Full-text prefix seeks on indexed columns (e.g., `?search=Jane`).
|
|
132
|
+
* **Multi-Column Ordering**: Sort descending or ascending (e.g., `?ordering=-created_at,salary`).
|
|
133
|
+
* **Fast Pagination**: Sliced pagination without expensive full-table `COUNT(*)` overhead on million-row tables.
|
|
134
|
+
* **Relational Expansion**: Expand foreign keys automatically (e.g., `?depth=1`).
|
|
135
|
+
|
|
136
|
+
---
|
|
137
|
+
|
|
138
|
+
## Supported Databases
|
|
139
|
+
|
|
140
|
+
DataMan provides native connection pooling, health probes, and migration routing for:
|
|
141
|
+
|
|
142
|
+
| Database | Support Level | Recommended Use |
|
|
143
|
+
| :--- | :--- | :--- |
|
|
144
|
+
| **PostgreSQL** | Primary / Production | 1M+ scale workloads, multi-database routing, high-concurrency ASGI |
|
|
145
|
+
| **MySQL / MariaDB** | Production | Enterprise relational storage with connection lifecycle pooling |
|
|
146
|
+
| **SQLite** | Prototyping | Zero-configuration local development and rapid test suites |
|
|
147
|
+
|
|
148
|
+
---
|
|
149
|
+
|
|
150
|
+
## Enterprise Adoption & Target Use Cases
|
|
151
|
+
|
|
152
|
+
DataMan is designed for data-intensive enterprise architectures:
|
|
153
|
+
* Continuous Control Monitoring (CCM) & internal audit platforms.
|
|
154
|
+
* SAP & ERP Data Middleware: Expose transactional SQL stores as structured REST APIs.
|
|
155
|
+
* Master Data Management (MDM): Instant CRUD data maintenance portals.
|
|
156
|
+
* Approval Workflows & Vendor Portals: Scoped, token-authenticated transactional APIs.
|
|
157
|
+
|
|
158
|
+
---
|
|
159
|
+
|
|
160
|
+
## Advanced Features
|
|
161
|
+
|
|
162
|
+
Every table generated under `tables/<TableName>/` gives you modular files for fine-grained control:
|
|
163
|
+
|
|
164
|
+
### 1. `config.py` (API Settings)
|
|
165
|
+
```python
|
|
166
|
+
# tables/Customer/config.py
|
|
167
|
+
ALLOWED_OPERATIONS = ["C", "R"] # Only allow Create (POST) and Read (GET)
|
|
168
|
+
REQUIRE_AUTH = True # Lock down this endpoint
|
|
169
|
+
DEPTH = 1 # Automatically serialize nested Foreign Key relationships on read (GET)
|
|
170
|
+
|
|
171
|
+
# Advanced Filtering, Search & Ordering
|
|
172
|
+
FILTER_FIELDS = {
|
|
173
|
+
"price": ["gte", "lte", "exact"],
|
|
174
|
+
"name": ["icontains", "exact"],
|
|
175
|
+
"is_active": ["exact"],
|
|
176
|
+
}
|
|
177
|
+
SEARCH_FIELDS = ["^name", "^email"] # Prefix seek for optimal B-Tree index utilization
|
|
178
|
+
ORDERING_FIELDS = ["created_at", "price"]
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
### 2. Authentication & Scoped Access Tokens
|
|
182
|
+
Generate restricted, fine-grained access tokens directly from the CLI:
|
|
183
|
+
```bash
|
|
184
|
+
dataman token create "Frontend Service" --scopes "customer:read,order:create"
|
|
185
|
+
```
|
|
186
|
+
Use the token with standard `Bearer` or `Token` headers:
|
|
187
|
+
```http
|
|
188
|
+
Authorization: Bearer <your_generated_token_key>
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
### 3. `validation.py` (Data Validation)
|
|
192
|
+
Validate incoming JSON payloads before database execution. Raise `ValidationError` to immediately return `400 Bad Request`.
|
|
193
|
+
```python
|
|
194
|
+
# tables/Customer/validation.py
|
|
195
|
+
from rest_framework.exceptions import ValidationError
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def validate_customer(data):
|
|
199
|
+
if "admin" in data.get("name", "").lower():
|
|
200
|
+
raise ValidationError({"name": "Reserved keyword used."})
|
|
201
|
+
data["name"] = data["name"].strip().title()
|
|
202
|
+
return data
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
### 4. `service.py` (Pre/Post Lifecycle Hooks)
|
|
206
|
+
Run transactional business logic before or after database commits (`before_create`, `after_create`, `before_update`, `after_update`, `before_destroy`, `after_destroy`):
|
|
207
|
+
```python
|
|
208
|
+
# tables/Order/service.py
|
|
209
|
+
from rest_framework.exceptions import ValidationError
|
|
210
|
+
from tables.Product.models import Product
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def before_create(data):
|
|
214
|
+
product = Product.objects.get(id=data["product"])
|
|
215
|
+
quantity = int(data.get("quantity", 1))
|
|
216
|
+
if product.stock_quantity < quantity:
|
|
217
|
+
raise ValidationError({"quantity": "Insufficient inventory available."})
|
|
218
|
+
product.stock_quantity -= quantity
|
|
219
|
+
product.save()
|
|
220
|
+
return data
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
### 5. Multi-Database Architecture
|
|
224
|
+
Organize large projects across isolated physical databases:
|
|
225
|
+
```bash
|
|
226
|
+
# 1. Create a database namespace
|
|
227
|
+
dataman create database analytics_db
|
|
228
|
+
|
|
229
|
+
# 2. Scaffold a table bound to that database
|
|
230
|
+
dataman create table events --database analytics_db
|
|
231
|
+
|
|
232
|
+
# 3. Run migrations across all databases (or target a single database)
|
|
233
|
+
dataman migrate --database analytics_db
|
|
234
|
+
```
|
|
235
|
+
Endpoints are automatically registered at `api/<database_name>/<table_name>/` (e.g. `api/default/employee/` or `api/analytics_db/events/`).
|
|
236
|
+
|
|
237
|
+
---
|
|
238
|
+
|
|
239
|
+
## Production Health & Monitoring
|
|
240
|
+
|
|
241
|
+
DataMan comes with built-in health check endpoints for Kubernetes, Docker, and AWS ECS:
|
|
242
|
+
* `GET /health/live/`: Process liveness probe returning `200 OK`.
|
|
243
|
+
* `GET /health/ready/`: Sanitized readiness probe validating database connectivity and migration synchronization (returns `200 OK` or `503 Service Unavailable`).
|
|
244
|
+
* `GET /health/`: Unified health status with database latency metrics.
|
|
245
|
+
|
|
246
|
+
---
|
|
247
|
+
|
|
248
|
+
## ASGI Production Mode
|
|
249
|
+
|
|
250
|
+
Start high-concurrency production server powered by Uvicorn:
|
|
251
|
+
```bash
|
|
252
|
+
dataman server start --asgi --host 0.0.0.0 --port 8000 --workers 4
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
---
|
|
256
|
+
|
|
257
|
+
## Testing
|
|
258
|
+
|
|
259
|
+
DataMan is tested with **95%+ branch coverage**, verifying authentication edge cases, multi-database routing, and dynamic lifecycle hooks.
|
|
260
|
+
|
|
261
|
+
```bash
|
|
262
|
+
uv run pytest tests/ -v
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
---
|
|
266
|
+
|
|
267
|
+
## License
|
|
268
|
+
MIT License
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
# DataMan (Data MiddleMan)
|
|
2
|
+
|
|
3
|
+
[](https://github.com/vikashgraja/DataMan/blob/main/docs/QUICKSTART.md)
|
|
4
|
+
[](#supported-databases)
|
|
5
|
+
[](#asgi-production-mode)
|
|
6
|
+
[](https://github.com/vikashgraja/DataMan/blob/main/LICENSE)
|
|
7
|
+
|
|
8
|
+
**DataMan** is a dynamic, headless backend engine built on top of Django and Django REST Framework. It eliminates the boilerplate of writing standard CRUD APIs, routing, and serializers by allowing you to scaffold endpoints instantly from the command line while preserving your ability to inject custom business logic and strict validation whenever you need it.
|
|
9
|
+
|
|
10
|
+
Looking to build an API in 5 minutes? Check out the [Developer Quickstart Tutorial](https://github.com/vikashgraja/DataMan/blob/main/docs/QUICKSTART.md).
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
## 30-Second Demo: Instant REST API
|
|
15
|
+
|
|
16
|
+
Get a fully functional, paginated, and documented REST API running in 4 commands:
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
# 1. Install DataMan
|
|
20
|
+
pip install dataman-engine
|
|
21
|
+
|
|
22
|
+
# 2. Initialize project in current directory
|
|
23
|
+
dataman init
|
|
24
|
+
|
|
25
|
+
# 3. Scaffold an Employee table with full CRUD operations
|
|
26
|
+
dataman create table Employee -o crud
|
|
27
|
+
|
|
28
|
+
# 4. Apply migrations and start the server
|
|
29
|
+
dataman makemigration && dataman migrate
|
|
30
|
+
dataman server start
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
### What You Get Instantly:
|
|
34
|
+
* **`GET /api/default/employee/`** — Paginated list with multi-column filtering, search, and ordering.
|
|
35
|
+
* **`POST /api/default/employee/`** — Create records with validation and lifecycle hooks.
|
|
36
|
+
* **`GET /api/default/employee/{id}/`** — Point lookup with automatic relational joins (`?depth=1`).
|
|
37
|
+
* **`PUT /api/default/employee/{id}/`** — Full update.
|
|
38
|
+
* **`PATCH /api/default/employee/{id}/`** — Partial update.
|
|
39
|
+
* **`DELETE /api/default/employee/{id}/`** — Delete record.
|
|
40
|
+
* **Interactive Swagger UI**: [`http://127.0.0.1:8000/api/docs/`](http://127.0.0.1:8000/api/docs/)
|
|
41
|
+
* **OpenAPI 3.0 Schema**: [`http://127.0.0.1:8000/api/schema/`](http://127.0.0.1:8000/api/schema/)
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## Why DataMan?
|
|
46
|
+
|
|
47
|
+
Building standard CRUD backends with existing frameworks requires boilerplate code:
|
|
48
|
+
|
|
49
|
+
```text
|
|
50
|
+
Without DataMan:
|
|
51
|
+
Model ---> Serializer ---> ViewSet ---> Router ---> FilterSet ---> Permissions
|
|
52
|
+
|
|
53
|
+
With DataMan:
|
|
54
|
+
Model ---> Done! (Instant REST Endpoints + Swagger Docs + Auth)
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
### Framework Comparison
|
|
58
|
+
|
|
59
|
+
| Framework | CRUD API Setup Requirements |
|
|
60
|
+
| :--- | :--- |
|
|
61
|
+
| **Django REST Framework (DRF)** | Model + Serializer + ViewSet + Router + FilterSet + Permissions |
|
|
62
|
+
| **FastAPI** | SQLAlchemy Model + Pydantic Schema + Router + Endpoints + Dependency Injection |
|
|
63
|
+
| **DataMan** | **Django Model Only (Instant Endpoints, Swagger, Auth & Hooks)** |
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## Out-of-the-Box API Features
|
|
68
|
+
|
|
69
|
+
Every scaffolded endpoint automatically delivers production response envelopes:
|
|
70
|
+
|
|
71
|
+
```json
|
|
72
|
+
// GET /api/default/employee/?is_active=true&search=Engineering&ordering=-created_at
|
|
73
|
+
{
|
|
74
|
+
"count": 48,
|
|
75
|
+
"next": "http://127.0.0.1:8000/api/default/employee/?page=2",
|
|
76
|
+
"previous": null,
|
|
77
|
+
"results": [
|
|
78
|
+
{
|
|
79
|
+
"id": 101,
|
|
80
|
+
"name": "Jane Doe",
|
|
81
|
+
"department": "Engineering",
|
|
82
|
+
"email": "jane.doe@example.com",
|
|
83
|
+
"is_active": true,
|
|
84
|
+
"created_at": "2026-09-19T18:00:00Z"
|
|
85
|
+
}
|
|
86
|
+
]
|
|
87
|
+
}
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
* **Filtering**: Filter by query params (e.g., `?is_active=true&salary__gte=80000`).
|
|
91
|
+
* **Fast B-Tree Prefix Search**: Full-text prefix seeks on indexed columns (e.g., `?search=Jane`).
|
|
92
|
+
* **Multi-Column Ordering**: Sort descending or ascending (e.g., `?ordering=-created_at,salary`).
|
|
93
|
+
* **Fast Pagination**: Sliced pagination without expensive full-table `COUNT(*)` overhead on million-row tables.
|
|
94
|
+
* **Relational Expansion**: Expand foreign keys automatically (e.g., `?depth=1`).
|
|
95
|
+
|
|
96
|
+
---
|
|
97
|
+
|
|
98
|
+
## Supported Databases
|
|
99
|
+
|
|
100
|
+
DataMan provides native connection pooling, health probes, and migration routing for:
|
|
101
|
+
|
|
102
|
+
| Database | Support Level | Recommended Use |
|
|
103
|
+
| :--- | :--- | :--- |
|
|
104
|
+
| **PostgreSQL** | Primary / Production | 1M+ scale workloads, multi-database routing, high-concurrency ASGI |
|
|
105
|
+
| **MySQL / MariaDB** | Production | Enterprise relational storage with connection lifecycle pooling |
|
|
106
|
+
| **SQLite** | Prototyping | Zero-configuration local development and rapid test suites |
|
|
107
|
+
|
|
108
|
+
---
|
|
109
|
+
|
|
110
|
+
## Enterprise Adoption & Target Use Cases
|
|
111
|
+
|
|
112
|
+
DataMan is designed for data-intensive enterprise architectures:
|
|
113
|
+
* Continuous Control Monitoring (CCM) & internal audit platforms.
|
|
114
|
+
* SAP & ERP Data Middleware: Expose transactional SQL stores as structured REST APIs.
|
|
115
|
+
* Master Data Management (MDM): Instant CRUD data maintenance portals.
|
|
116
|
+
* Approval Workflows & Vendor Portals: Scoped, token-authenticated transactional APIs.
|
|
117
|
+
|
|
118
|
+
---
|
|
119
|
+
|
|
120
|
+
## Advanced Features
|
|
121
|
+
|
|
122
|
+
Every table generated under `tables/<TableName>/` gives you modular files for fine-grained control:
|
|
123
|
+
|
|
124
|
+
### 1. `config.py` (API Settings)
|
|
125
|
+
```python
|
|
126
|
+
# tables/Customer/config.py
|
|
127
|
+
ALLOWED_OPERATIONS = ["C", "R"] # Only allow Create (POST) and Read (GET)
|
|
128
|
+
REQUIRE_AUTH = True # Lock down this endpoint
|
|
129
|
+
DEPTH = 1 # Automatically serialize nested Foreign Key relationships on read (GET)
|
|
130
|
+
|
|
131
|
+
# Advanced Filtering, Search & Ordering
|
|
132
|
+
FILTER_FIELDS = {
|
|
133
|
+
"price": ["gte", "lte", "exact"],
|
|
134
|
+
"name": ["icontains", "exact"],
|
|
135
|
+
"is_active": ["exact"],
|
|
136
|
+
}
|
|
137
|
+
SEARCH_FIELDS = ["^name", "^email"] # Prefix seek for optimal B-Tree index utilization
|
|
138
|
+
ORDERING_FIELDS = ["created_at", "price"]
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
### 2. Authentication & Scoped Access Tokens
|
|
142
|
+
Generate restricted, fine-grained access tokens directly from the CLI:
|
|
143
|
+
```bash
|
|
144
|
+
dataman token create "Frontend Service" --scopes "customer:read,order:create"
|
|
145
|
+
```
|
|
146
|
+
Use the token with standard `Bearer` or `Token` headers:
|
|
147
|
+
```http
|
|
148
|
+
Authorization: Bearer <your_generated_token_key>
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
### 3. `validation.py` (Data Validation)
|
|
152
|
+
Validate incoming JSON payloads before database execution. Raise `ValidationError` to immediately return `400 Bad Request`.
|
|
153
|
+
```python
|
|
154
|
+
# tables/Customer/validation.py
|
|
155
|
+
from rest_framework.exceptions import ValidationError
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def validate_customer(data):
|
|
159
|
+
if "admin" in data.get("name", "").lower():
|
|
160
|
+
raise ValidationError({"name": "Reserved keyword used."})
|
|
161
|
+
data["name"] = data["name"].strip().title()
|
|
162
|
+
return data
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
### 4. `service.py` (Pre/Post Lifecycle Hooks)
|
|
166
|
+
Run transactional business logic before or after database commits (`before_create`, `after_create`, `before_update`, `after_update`, `before_destroy`, `after_destroy`):
|
|
167
|
+
```python
|
|
168
|
+
# tables/Order/service.py
|
|
169
|
+
from rest_framework.exceptions import ValidationError
|
|
170
|
+
from tables.Product.models import Product
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def before_create(data):
|
|
174
|
+
product = Product.objects.get(id=data["product"])
|
|
175
|
+
quantity = int(data.get("quantity", 1))
|
|
176
|
+
if product.stock_quantity < quantity:
|
|
177
|
+
raise ValidationError({"quantity": "Insufficient inventory available."})
|
|
178
|
+
product.stock_quantity -= quantity
|
|
179
|
+
product.save()
|
|
180
|
+
return data
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
### 5. Multi-Database Architecture
|
|
184
|
+
Organize large projects across isolated physical databases:
|
|
185
|
+
```bash
|
|
186
|
+
# 1. Create a database namespace
|
|
187
|
+
dataman create database analytics_db
|
|
188
|
+
|
|
189
|
+
# 2. Scaffold a table bound to that database
|
|
190
|
+
dataman create table events --database analytics_db
|
|
191
|
+
|
|
192
|
+
# 3. Run migrations across all databases (or target a single database)
|
|
193
|
+
dataman migrate --database analytics_db
|
|
194
|
+
```
|
|
195
|
+
Endpoints are automatically registered at `api/<database_name>/<table_name>/` (e.g. `api/default/employee/` or `api/analytics_db/events/`).
|
|
196
|
+
|
|
197
|
+
---
|
|
198
|
+
|
|
199
|
+
## Production Health & Monitoring
|
|
200
|
+
|
|
201
|
+
DataMan comes with built-in health check endpoints for Kubernetes, Docker, and AWS ECS:
|
|
202
|
+
* `GET /health/live/`: Process liveness probe returning `200 OK`.
|
|
203
|
+
* `GET /health/ready/`: Sanitized readiness probe validating database connectivity and migration synchronization (returns `200 OK` or `503 Service Unavailable`).
|
|
204
|
+
* `GET /health/`: Unified health status with database latency metrics.
|
|
205
|
+
|
|
206
|
+
---
|
|
207
|
+
|
|
208
|
+
## ASGI Production Mode
|
|
209
|
+
|
|
210
|
+
Start high-concurrency production server powered by Uvicorn:
|
|
211
|
+
```bash
|
|
212
|
+
dataman server start --asgi --host 0.0.0.0 --port 8000 --workers 4
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
---
|
|
216
|
+
|
|
217
|
+
## Testing
|
|
218
|
+
|
|
219
|
+
DataMan is tested with **95%+ branch coverage**, verifying authentication edge cases, multi-database routing, and dynamic lifecycle hooks.
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
uv run pytest tests/ -v
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
---
|
|
226
|
+
|
|
227
|
+
## License
|
|
228
|
+
MIT License
|
|
@@ -1,20 +1,21 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dataman-engine"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
description = "A dynamic, headless data layer framework built on Django and DRF for instant CRUD APIs and custom business logic."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
7
|
-
requires-python = ">=3.
|
|
7
|
+
requires-python = ">=3.12"
|
|
8
8
|
classifiers = [
|
|
9
9
|
"Development Status :: 4 - Beta",
|
|
10
10
|
"Environment :: Console",
|
|
11
11
|
"Environment :: Web Environment",
|
|
12
12
|
"Framework :: Django",
|
|
13
|
-
"Framework :: Django ::
|
|
13
|
+
"Framework :: Django :: 6.0",
|
|
14
14
|
"Intended Audience :: Developers",
|
|
15
15
|
"Operating System :: OS Independent",
|
|
16
16
|
"Programming Language :: Python",
|
|
17
17
|
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
18
19
|
"Programming Language :: Python :: 3.13",
|
|
19
20
|
"Topic :: Internet :: WWW/HTTP",
|
|
20
21
|
"Topic :: Internet :: WWW/HTTP :: Dynamic Content",
|
|
@@ -42,6 +43,7 @@ dependencies = [
|
|
|
42
43
|
"inflection>=0.5.1",
|
|
43
44
|
"uvicorn>=0.34.0",
|
|
44
45
|
"cryptography>=42.0.0",
|
|
46
|
+
"psycopg2-binary>=2.9.13",
|
|
45
47
|
]
|
|
46
48
|
|
|
47
49
|
[[project.authors]]
|
|
@@ -52,8 +54,8 @@ email = "vikashgraja@gmail.com"
|
|
|
52
54
|
dataman = "dataman.cli:cli"
|
|
53
55
|
|
|
54
56
|
[project.urls]
|
|
55
|
-
Homepage = "https://github.com/
|
|
56
|
-
Repository = "https://github.com/
|
|
57
|
+
Homepage = "https://github.com/vikashgraja/dataman"
|
|
58
|
+
Repository = "https://github.com/vikashgraja/dataman.git"
|
|
57
59
|
|
|
58
60
|
[tool.coverage.run]
|
|
59
61
|
branch = true
|
|
@@ -65,13 +67,14 @@ module-name = "dataman"
|
|
|
65
67
|
|
|
66
68
|
[tool.ruff]
|
|
67
69
|
line-length = 88
|
|
68
|
-
target-version = "
|
|
70
|
+
target-version = "py312"
|
|
69
71
|
exclude = [
|
|
70
72
|
".venv",
|
|
71
73
|
"venv",
|
|
72
74
|
"__pycache__",
|
|
73
75
|
"build",
|
|
74
76
|
"dist",
|
|
77
|
+
"benchmarks",
|
|
75
78
|
]
|
|
76
79
|
|
|
77
80
|
[tool.ruff.lint]
|
|
@@ -97,6 +100,7 @@ line-ending = "auto"
|
|
|
97
100
|
exclude_dirs = [
|
|
98
101
|
"tests",
|
|
99
102
|
".venv",
|
|
103
|
+
"benchmarks",
|
|
100
104
|
]
|
|
101
105
|
skips = ["B101"]
|
|
102
106
|
|
|
@@ -104,6 +108,20 @@ skips = ["B101"]
|
|
|
104
108
|
testpaths = ["tests"]
|
|
105
109
|
addopts = "-v --tb=short"
|
|
106
110
|
|
|
111
|
+
[tool.tox]
|
|
112
|
+
legacy_tox_ini = """
|
|
113
|
+
[tox]
|
|
114
|
+
env_list = py312, py313, py314
|
|
115
|
+
min_version = 4.0
|
|
116
|
+
|
|
117
|
+
[testenv]
|
|
118
|
+
runner = uv-venv-runner
|
|
119
|
+
dependency_groups =
|
|
120
|
+
dev
|
|
121
|
+
commands =
|
|
122
|
+
pytest {posargs}
|
|
123
|
+
"""
|
|
124
|
+
|
|
107
125
|
[build-system]
|
|
108
126
|
requires = ["uv_build>=0.10.6,<0.13.0"]
|
|
109
127
|
build-backend = "uv_build"
|
|
@@ -111,9 +129,12 @@ build-backend = "uv_build"
|
|
|
111
129
|
[dependency-groups]
|
|
112
130
|
dev = [
|
|
113
131
|
"bandit>=1.9.4",
|
|
132
|
+
"locust>=2.32.0",
|
|
114
133
|
"playwright>=1.62.0",
|
|
115
134
|
"pre-commit>=4.6.1",
|
|
116
135
|
"pytest>=9.1.1",
|
|
117
136
|
"pytest-cov>=7.1.0",
|
|
118
137
|
"ruff>=0.16.1",
|
|
138
|
+
"tox>=4.24.1",
|
|
139
|
+
"tox-uv>=1.25.0",
|
|
119
140
|
]
|
|
@@ -1,23 +1,24 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dataman-engine"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
description = "A dynamic, headless data layer framework built on Django and DRF for instant CRUD APIs and custom business logic."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
7
7
|
authors = [
|
|
8
8
|
{ name = "Vikash G", email = "vikashgraja@gmail.com" }
|
|
9
9
|
]
|
|
10
|
-
requires-python = ">=3.
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
11
|
classifiers = [
|
|
12
12
|
"Development Status :: 4 - Beta",
|
|
13
13
|
"Environment :: Console",
|
|
14
14
|
"Environment :: Web Environment",
|
|
15
15
|
"Framework :: Django",
|
|
16
|
-
"Framework :: Django ::
|
|
16
|
+
"Framework :: Django :: 6.0",
|
|
17
17
|
"Intended Audience :: Developers",
|
|
18
18
|
"Operating System :: OS Independent",
|
|
19
19
|
"Programming Language :: Python",
|
|
20
20
|
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
21
22
|
"Programming Language :: Python :: 3.13",
|
|
22
23
|
"Topic :: Internet :: WWW/HTTP",
|
|
23
24
|
"Topic :: Internet :: WWW/HTTP :: Dynamic Content",
|
|
@@ -39,14 +40,15 @@ dependencies = [
|
|
|
39
40
|
"inflection>=0.5.1",
|
|
40
41
|
"uvicorn>=0.34.0",
|
|
41
42
|
"cryptography>=42.0.0",
|
|
43
|
+
"psycopg2-binary>=2.9.13",
|
|
42
44
|
]
|
|
43
45
|
|
|
44
46
|
[project.scripts]
|
|
45
47
|
dataman = "dataman.cli:cli"
|
|
46
48
|
|
|
47
49
|
[project.urls]
|
|
48
|
-
Homepage = "https://github.com/
|
|
49
|
-
Repository = "https://github.com/
|
|
50
|
+
Homepage = "https://github.com/vikashgraja/dataman"
|
|
51
|
+
Repository = "https://github.com/vikashgraja/dataman.git"
|
|
50
52
|
|
|
51
53
|
[tool.coverage.run]
|
|
52
54
|
branch = true
|
|
@@ -62,17 +64,20 @@ module-name = "dataman"
|
|
|
62
64
|
[dependency-groups]
|
|
63
65
|
dev = [
|
|
64
66
|
"bandit>=1.9.4",
|
|
67
|
+
"locust>=2.32.0",
|
|
65
68
|
"playwright>=1.62.0",
|
|
66
69
|
"pre-commit>=4.6.1",
|
|
67
70
|
"pytest>=9.1.1",
|
|
68
71
|
"pytest-cov>=7.1.0",
|
|
69
72
|
"ruff>=0.16.1",
|
|
73
|
+
"tox>=4.24.1",
|
|
74
|
+
"tox-uv>=1.25.0",
|
|
70
75
|
]
|
|
71
76
|
|
|
72
77
|
[tool.ruff]
|
|
73
78
|
line-length = 88
|
|
74
|
-
target-version = "
|
|
75
|
-
exclude = [".venv", "venv", "__pycache__", "build", "dist"]
|
|
79
|
+
target-version = "py312"
|
|
80
|
+
exclude = [".venv", "venv", "__pycache__", "build", "dist", "benchmarks"]
|
|
76
81
|
|
|
77
82
|
[tool.ruff.lint]
|
|
78
83
|
select = [
|
|
@@ -94,9 +99,23 @@ skip-magic-trailing-comma = false
|
|
|
94
99
|
line-ending = "auto"
|
|
95
100
|
|
|
96
101
|
[tool.bandit]
|
|
97
|
-
exclude_dirs = ["tests", ".venv"]
|
|
102
|
+
exclude_dirs = ["tests", ".venv", "benchmarks"]
|
|
98
103
|
skips = ["B101"]
|
|
99
104
|
|
|
100
105
|
[tool.pytest.ini_options]
|
|
101
106
|
testpaths = ["tests"]
|
|
102
107
|
addopts = "-v --tb=short"
|
|
108
|
+
|
|
109
|
+
[tool.tox]
|
|
110
|
+
legacy_tox_ini = """
|
|
111
|
+
[tox]
|
|
112
|
+
env_list = py312, py313, py314
|
|
113
|
+
min_version = 4.0
|
|
114
|
+
|
|
115
|
+
[testenv]
|
|
116
|
+
runner = uv-venv-runner
|
|
117
|
+
dependency_groups =
|
|
118
|
+
dev
|
|
119
|
+
commands =
|
|
120
|
+
pytest {posargs}
|
|
121
|
+
"""
|