dataman-engine 0.1.1__tar.gz → 0.2.0__tar.gz

{dataman_engine-0.1.1 → dataman_engine-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataman-engine
-Version: 0.1.1
+Version: 0.2.0
 Summary: A dynamic, headless data layer framework built on Django and DRF for instant CRUD APIs and custom business logic.
 Keywords: django,rest,api,crud,headless,generator,scaffold
 Author: Vikash G
@@ -10,11 +10,12 @@ Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Environment :: Web Environment
 Classifier: Framework :: Django
-Classifier: Framework :: Django :: 5.0
+Classifier: Framework :: Django :: 6.0
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Internet :: WWW/HTTP
 Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
@@ -31,15 +32,21 @@ Requires-Dist: dj-database-url>=2.3.0
 Requires-Dist: inflection>=0.5.1
 Requires-Dist: uvicorn>=0.34.0
 Requires-Dist: cryptography>=42.0.0
-Requires-Python: >=3.13
-Project-URL: Homepage, https://github.com/vikashg/dataman
-Project-URL: Repository, https://github.com/vikashg/dataman.git
+Requires-Dist: psycopg2-binary>=2.9.13
+Requires-Python: >=3.12
+Project-URL: Homepage, https://github.com/vikashgraja/dataman
+Project-URL: Repository, https://github.com/vikashgraja/dataman.git
 Description-Content-Type: text/markdown
 
 # DataMan (Data MiddleMan)
 
+[![Quickstart Guide](https://img.shields.io/badge/Docs-Quickstart_Guide-blue.svg)](docs/QUICKSTART.md)
+
 **DataMan** is a dynamic, CLI-driven backend framework built on top of Django and Django REST Framework. It eliminates the boilerplate of writing standard CRUD APIs, routing, and serializers by allowing you to scaffold endpoints instantly from the command line while preserving your ability to inject custom business logic and strict validation whenever you need it.
 
+👉 **Looking to build an API in 5 minutes? Check out the [Developer Quickstart Tutorial](docs/QUICKSTART.md)!**
+
+
 ---
 
 ## Features

{dataman_engine-0.1.1 → dataman_engine-0.2.0}/README.md

@@ -1,7 +1,12 @@
 # DataMan (Data MiddleMan)
 
+[![Quickstart Guide](https://img.shields.io/badge/Docs-Quickstart_Guide-blue.svg)](docs/QUICKSTART.md)
+
 **DataMan** is a dynamic, CLI-driven backend framework built on top of Django and Django REST Framework. It eliminates the boilerplate of writing standard CRUD APIs, routing, and serializers by allowing you to scaffold endpoints instantly from the command line while preserving your ability to inject custom business logic and strict validation whenever you need it.
 
+👉 **Looking to build an API in 5 minutes? Check out the [Developer Quickstart Tutorial](docs/QUICKSTART.md)!**
+
+
 ---
 
 ## Features

{dataman_engine-0.1.1 → dataman_engine-0.2.0}/pyproject.toml

@@ -1,20 +1,21 @@
 [project]
 name = "dataman-engine"
-version = "0.1.1"
+version = "0.2.0"
 description = "A dynamic, headless data layer framework built on Django and DRF for instant CRUD APIs and custom business logic."
 readme = "README.md"
 license = "MIT"
-requires-python = ">=3.13"
+requires-python = ">=3.12"
 classifiers = [
     "Development Status :: 4 - Beta",
     "Environment :: Console",
     "Environment :: Web Environment",
     "Framework :: Django",
-    "Framework :: Django :: 5.0",
+    "Framework :: Django :: 6.0",
     "Intended Audience :: Developers",
     "Operating System :: OS Independent",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
     "Topic :: Internet :: WWW/HTTP",
     "Topic :: Internet :: WWW/HTTP :: Dynamic Content",
@@ -42,6 +43,7 @@ dependencies = [
     "inflection>=0.5.1",
     "uvicorn>=0.34.0",
     "cryptography>=42.0.0",
+    "psycopg2-binary>=2.9.13",
 ]
 
 [[project.authors]]
@@ -52,8 +54,8 @@ email = "vikashgraja@gmail.com"
 dataman = "dataman.cli:cli"
 
 [project.urls]
-Homepage = "https://github.com/vikashg/dataman"
-Repository = "https://github.com/vikashg/dataman.git"
+Homepage = "https://github.com/vikashgraja/dataman"
+Repository = "https://github.com/vikashgraja/dataman.git"
 
 [tool.coverage.run]
 branch = true
@@ -65,13 +67,14 @@ module-name = "dataman"
 
 [tool.ruff]
 line-length = 88
-target-version = "py310"
+target-version = "py312"
 exclude = [
     ".venv",
     "venv",
     "__pycache__",
     "build",
     "dist",
+    "benchmarks",
 ]
 
 [tool.ruff.lint]
@@ -97,6 +100,7 @@ line-ending = "auto"
 exclude_dirs = [
     "tests",
     ".venv",
+    "benchmarks",
 ]
 skips = ["B101"]
 
@@ -104,6 +108,20 @@ skips = ["B101"]
 testpaths = ["tests"]
 addopts = "-v --tb=short"
 
+[tool.tox]
+legacy_tox_ini = """
+[tox]
+env_list = py312, py313, py314
+min_version = 4.0
+
+[testenv]
+runner = uv-venv-runner
+dependency_groups =
+    dev
+commands =
+    pytest {posargs}
+"""
+
 [build-system]
 requires = ["uv_build>=0.10.6,<0.13.0"]
 build-backend = "uv_build"
@@ -111,9 +129,12 @@ build-backend = "uv_build"
 [dependency-groups]
 dev = [
     "bandit>=1.9.4",
+    "locust>=2.32.0",
     "playwright>=1.62.0",
     "pre-commit>=4.6.1",
     "pytest>=9.1.1",
     "pytest-cov>=7.1.0",
     "ruff>=0.16.1",
+    "tox>=4.24.1",
+    "tox-uv>=1.25.0",
 ]

{dataman_engine-0.1.1 → dataman_engine-0.2.0}/pyproject.toml.orig

@@ -1,23 +1,24 @@
 [project]
 name = "dataman-engine"
-version = "0.1.1"
+version = "0.2.0"
 description = "A dynamic, headless data layer framework built on Django and DRF for instant CRUD APIs and custom business logic."
 readme = "README.md"
 license = "MIT"
 authors = [
     { name = "Vikash G", email = "vikashgraja@gmail.com" }
 ]
-requires-python = ">=3.13"
+requires-python = ">=3.12"
 classifiers = [
     "Development Status :: 4 - Beta",
     "Environment :: Console",
     "Environment :: Web Environment",
     "Framework :: Django",
-    "Framework :: Django :: 5.0",
+    "Framework :: Django :: 6.0",
     "Intended Audience :: Developers",
     "Operating System :: OS Independent",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
     "Topic :: Internet :: WWW/HTTP",
     "Topic :: Internet :: WWW/HTTP :: Dynamic Content",
@@ -39,14 +40,15 @@ dependencies = [
     "inflection>=0.5.1",
     "uvicorn>=0.34.0",
     "cryptography>=42.0.0",
+    "psycopg2-binary>=2.9.13",
 ]
 
 [project.scripts]
 dataman = "dataman.cli:cli"
 
 [project.urls]
-Homepage = "https://github.com/vikashg/dataman"
-Repository = "https://github.com/vikashg/dataman.git"
+Homepage = "https://github.com/vikashgraja/dataman"
+Repository = "https://github.com/vikashgraja/dataman.git"
 
 [tool.coverage.run]
 branch = true
@@ -62,17 +64,20 @@ module-name = "dataman"
 [dependency-groups]
 dev = [
     "bandit>=1.9.4",
+    "locust>=2.32.0",
     "playwright>=1.62.0",
     "pre-commit>=4.6.1",
     "pytest>=9.1.1",
     "pytest-cov>=7.1.0",
     "ruff>=0.16.1",
+    "tox>=4.24.1",
+    "tox-uv>=1.25.0",
 ]
 
 [tool.ruff]
 line-length = 88
-target-version = "py310"
-exclude = [".venv", "venv", "__pycache__", "build", "dist"]
+target-version = "py312"
+exclude = [".venv", "venv", "__pycache__", "build", "dist", "benchmarks"]
 
 [tool.ruff.lint]
 select = [
@@ -94,9 +99,23 @@ skip-magic-trailing-comma = false
 line-ending = "auto"
 
 [tool.bandit]
-exclude_dirs = ["tests", ".venv"]
+exclude_dirs = ["tests", ".venv", "benchmarks"]
 skips = ["B101"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 addopts = "-v --tb=short"
+
+[tool.tox]
+legacy_tox_ini = """
+[tox]
+env_list = py312, py313, py314
+min_version = 4.0
+
+[testenv]
+runner = uv-venv-runner
+dependency_groups =
+    dev
+commands =
+    pytest {posargs}
+"""

{dataman_engine-0.1.1 → dataman_engine-0.2.0}/src/dataman/__init__.py

@@ -9,7 +9,7 @@ from dataman.core.fields import (
     EncryptedTextField,
 )
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 __all__ = [
     "EncryptedCharField",
     "EncryptedTextField",

dataman_engine-0.2.0/src/dataman/asgi.py

@@ -0,0 +1,3 @@
+from dataman.django_setup import get_asgi_application
+
+application = get_asgi_application()

dataman_engine-0.2.0/src/dataman/core/middleware.py

@@ -0,0 +1,123 @@
+import atexit
+import logging
+import queue
+import threading
+import time
+
+from django.db import close_old_connections
+
+from .models import APILog
+
+logger = logging.getLogger("dataman.telemetry")
+
+_log_queue = queue.Queue(maxsize=50000)
+_shutdown_event = threading.Event()
+
+
+def _telemetry_batch_worker():
+    """
+    High-performance background batch worker. Drains telemetry log queue
+    and persists records using bulk_create to eliminate DB lock contention.
+    """
+    while not _shutdown_event.is_set() or not _log_queue.empty():
+        batch = []
+        try:
+            item = _log_queue.get(timeout=0.05)
+            batch.append(item)
+            while len(batch) < 100:
+                try:
+                    batch.append(_log_queue.get_nowait())
+                except queue.Empty:
+                    break
+        except queue.Empty:
+            continue
+
+        if not batch:
+            continue
+
+        close_old_connections()
+        try:
+            objs = [
+                APILog(
+                    method=m,
+                    path=p,
+                    status_code=s,
+                    duration_ms=d,
+                    ip_address=ip,
+                )
+                for (m, p, s, d, ip) in batch
+            ]
+            APILog.objects.bulk_create(objs, batch_size=100)
+        except Exception as e:
+            logger.error(f"Failed to batch write APILogs: {e}")
+        finally:
+            close_old_connections()
+            for _ in batch:
+                _log_queue.task_done()
+
+
+_worker_thread = threading.Thread(
+    target=_telemetry_batch_worker, name="dataman-telemetry-batch", daemon=True
+)
+_worker_thread.start()
+
+
+def flush_telemetry_logs(timeout=5):
+    """Barrier helper to ensure all queued background logs are flushed to database."""
+    try:
+        _log_queue.join()
+    except Exception as e:
+        logger.warning(f"Timeout flushing telemetry queue: {e}")
+
+
+def _shutdown_telemetry():
+    _shutdown_event.set()
+    flush_telemetry_logs()
+
+
+atexit.register(_shutdown_telemetry)
+
+
+class APILoggingMiddleware:
+    """Middleware to log API request metrics asynchronously for high concurrency."""
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        path = request.path
+        # Only log requests to actual data endpoints,
+        # exclude internal/dashboard/docs/health routes.
+        if (
+            not path.startswith("/api/")
+            or path.startswith("/api/schema")
+            or path.startswith("/api/docs")
+            or path.startswith("/api/redoc")
+            or path.startswith("/api/_internal/")
+            or path.startswith("/api/health")
+            or path.startswith("/health")
+        ):
+            return self.get_response(request)
+
+        start_time = time.perf_counter()
+        response = self.get_response(request)
+        duration_ms = int((time.perf_counter() - start_time) * 1000)
+
+        # Handle IP Address correctly behind proxies
+        x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR")
+        ip = (
+            x_forwarded_for.split(",")[0]
+            if x_forwarded_for
+            else request.META.get("REMOTE_ADDR")
+        )
+
+        try:
+            _log_queue.put_nowait(
+                (request.method, path, response.status_code, duration_ms, ip)
+            )
+        except queue.Full:
+            logger.warning(
+                "Telemetry log queue full, dropping record under extreme load."
+            )
+
+        return response

{dataman_engine-0.1.1 → dataman_engine-0.2.0}/src/dataman/core/models.py

@@ -16,6 +16,8 @@ RESERVED_ROOT_DIRS = {
     ".ruff_cache",
     ".system_generated",
     ".venv",
+    "benchmark",
+    "benchmarks",
     "build",
     "coverage_hack",
     "dist",

{dataman_engine-0.1.1 → dataman_engine-0.2.0}/src/dataman/core/urls.py

@@ -360,6 +360,27 @@ try:
             "perform_destroy": custom_perform_destroy,
         }
 
+        def custom_get_queryset(self, m=model, d=depth):
+            qs = m.objects.all()
+            if d is not None and d > 0:
+                fk_fields = [
+                    f.name
+                    for f in m._meta.fields
+                    if f.is_relation and (f.many_to_one or f.one_to_one)
+                ]
+                if fk_fields:
+                    qs = qs.select_related(*fk_fields)
+                reverse_rel_fields = [
+                    f.get_accessor_name()
+                    for f in m._meta.related_objects
+                    if f.get_accessor_name()
+                ]
+                if reverse_rel_fields:
+                    qs = qs.prefetch_related(*reverse_rel_fields)
+            return qs
+
+        viewset_attrs["get_queryset"] = custom_get_queryset
+
         if depth is not None and depth > 0:
 
             def custom_get_serializer_class(
@@ -386,10 +407,94 @@ try:
         # Pagination
         if page_size:
 
+            class FastPaginator:
+                def __init__(self, object_list, per_page):
+                    self.object_list = object_list
+                    self.per_page = per_page
+                    self._count = None
+
+                @property
+                def count(self):
+                    if self._count is None:
+                        try:
+                            self._count = self.object_list.count()
+                        except Exception:
+                            self._count = 0
+                    return self._count
+
+                @property
+                def num_pages(self):
+                    if self.count == 0:
+                        return 1
+                    return (self.count + self.per_page - 1) // self.per_page
+
+                @property
+                def page_range(self):
+                    return range(1, self.num_pages + 1)
+
+                def page(self, number):
+                    number = int(number)
+                    bottom = (number - 1) * self.per_page
+                    top = bottom + self.per_page
+                    items = list(self.object_list[bottom : top + 1])
+                    has_next = len(items) > self.per_page
+                    if has_next:
+                        items = items[: self.per_page]
+
+                    class FastPage:
+                        def __init__(
+                            self, object_list, number, paginator, has_next, bottom
+                        ):
+                            self.object_list = object_list
+                            self.number = number
+                            self.paginator = paginator
+                            self._has_next = has_next
+                            self._bottom = bottom
+
+                        def __len__(self):
+                            return len(self.object_list)
+
+                        def __iter__(self):
+                            return iter(self.object_list)
+
+                        def __getitem__(self, index):
+                            return self.object_list[index]
+
+                        def has_next(self):
+                            return self._has_next
+
+                        def has_previous(self):
+                            return self.number > 1
+
+                        def next_page_number(self):
+                            return self.number + 1
+
+                        def previous_page_number(self):
+                            return self.number - 1
+
+                        def start_index(self):
+                            return self._bottom + 1
+
+                        def end_index(self):
+                            return self._bottom + len(self.object_list)
+
+                    return FastPage(items, number, self, has_next, bottom)
+
             class CustomPagination(PageNumberPagination):
                 page_size_val = page_size
+                django_paginator_class = FastPaginator
+                page_size_query_param = "page_size"
+                max_page_size = 500
 
                 def get_page_size(self, request):
+                    if self.page_size_query_param:
+                        with contextlib.suppress(Exception):
+                            val = int(
+                                request.query_params.get(
+                                    self.page_size_query_param, self.page_size_val
+                                )
+                            )
+                            return min(max(val, 1), self.max_page_size)
                     return self.page_size_val
 
             viewset_attrs["pagination_class"] = CustomPagination
@@ -406,6 +511,8 @@ try:
             filter_backends.append(OrderingFilter)
             viewset_attrs["ordering_fields"] = ordering_fields
 
+        viewset_attrs["ordering"] = ["-id"]
+
         if filter_backends:
             viewset_attrs["filter_backends"] = filter_backends
 

dataman_engine-0.2.0/src/dataman/wsgi.py

@@ -0,0 +1,3 @@
+from dataman.django_setup import get_wsgi_application
+
+application = get_wsgi_application()

dataman_engine-0.1.1/src/dataman/core/middleware.py

@@ -1,46 +0,0 @@
-import time
-
-from .models import APILog
-
-
-class APILoggingMiddleware:
-    """Middleware to log API request metrics for the dashboard."""
-
-    def __init__(self, get_response):
-        self.get_response = get_response
-
-    def __call__(self, request):
-        path = request.path
-        # Only log requests to actual data endpoints,
-        # exclude internal/dashboard/docs/health routes.
-        if (
-            not path.startswith("/api/")
-            or path.startswith("/api/schema")
-            or path.startswith("/api/docs")
-            or path.startswith("/api/redoc")
-            or path.startswith("/api/_internal/")
-            or path.startswith("/api/health")
-            or path.startswith("/health")
-        ):
-            return self.get_response(request)
-
-        start_time = time.time()
-        response = self.get_response(request)
-        duration_ms = int((time.time() - start_time) * 1000)
-
-        # Handle IP Address correctly behind proxies
-        x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR")
-        if x_forwarded_for:
-            ip = x_forwarded_for.split(",")[0]
-        else:
-            ip = request.META.get("REMOTE_ADDR")
-
-        APILog.objects.create(
-            method=request.method,
-            path=path,
-            status_code=response.status_code,
-            duration_ms=duration_ms,
-            ip_address=ip,
-        )
-
-        return response