django-nativemojo 0.1.15__py3-none-any.whl → 0.1.17__py3-none-any.whl

mojo/helpers/redis/pool.py

@@ -0,0 +1,225 @@
+from typing import Optional, List, Set
+from django.db import models
+from .client import get_connection
+
+
+class RedisBasePool:
+    """Simple Redis pool using atomic Redis operations."""
+
+    def __init__(self, pool_key: str, default_timeout: int = 30):
+        """
+        Initialize the Redis pool.
+
+        Args:
+            pool_key: Unique identifier for this pool
+            default_timeout: Default timeout in seconds for blocking operations
+        """
+        self.pool_key = pool_key
+        self.default_timeout = default_timeout
+        self.redis_client = get_connection()
+
+        self.available_list_key = f"{pool_key}:list"
+        self.all_items_set_key = f"{pool_key}:set"
+
+    def add(self, str_id: str) -> bool:
+        """Add an item to the pool."""
+        if not self.redis_client.sismember(self.all_items_set_key, str_id):
+            self.redis_client.sadd(self.all_items_set_key, str_id)
+            self.redis_client.lpush(self.available_list_key, str_id)
+            return True
+        return False
+
+    def remove(self, str_id: str) -> bool:
+        """Remove an item from the pool entirely."""
+        if not self.redis_client.sismember(self.all_items_set_key, str_id):
+            return False
+
+        self.redis_client.srem(self.all_items_set_key, str_id)
+        self.redis_client.lrem(self.available_list_key, 0, str_id)
+        return True
+
+    def clear(self) -> None:
+        """Clear all items from the pool."""
+        self.redis_client.delete(self.available_list_key)
+        self.redis_client.delete(self.all_items_set_key)
+
+    def checkout(self, str_id: str, timeout: Optional[int] = None) -> bool:
+        """Check out a specific item from the pool."""
+        if not self.redis_client.sismember(self.all_items_set_key, str_id):
+            return False
+
+        if timeout is None:
+            removed = self.redis_client.lrem(self.available_list_key, 1, str_id)
+            return removed > 0
+
+        import time
+        start = time.time()
+        while self.redis_client.lrem(self.available_list_key, 1, str_id) == 0:
+            time.sleep(1.0)
+            elapsed = time.time() - start
+            if elapsed > timeout:
+                return False
+        return True
+
+    def checkin(self, str_id: str) -> bool:
+        """Check in an item back to the pool."""
+        if not self.redis_client.sismember(self.all_items_set_key, str_id):
+            return False
+
+        self.redis_client.lpush(self.available_list_key, str_id)
+        return True
+
+    def list_all(self) -> Set[str]:
+        """List all items in the pool."""
+        return self.redis_client.smembers(self.all_items_set_key)
+
+    def list_available(self) -> List[str]:
+        """List available items in the pool."""
+        return self.redis_client.lrange(self.available_list_key, 0, -1)
+
+    def list_checked_out(self) -> Set[str]:
+        """List checked out items."""
+        all_items = self.list_all()
+        available_items = set(self.list_available())
+        return all_items - available_items
+
+    def destroy_pool(self) -> None:
+        """Completely destroy the pool."""
+        self.clear()
+
+    def get_next_available(self, timeout: Optional[int] = None) -> Optional[str]:
+        """Get the next available item from the pool."""
+        timeout = timeout or self.default_timeout
+
+        result = self.redis_client.brpop(self.available_list_key, timeout=timeout)
+        if result:
+            return result[1]
+        return None
+
+
+class RedisModelPool(RedisBasePool):
+    """Django model-specific Redis pool."""
+
+    def __init__(self, model_cls: models.Model, query_dict: dict,
+                 pool_key: str, default_timeout: int = 30):
+        """
+        Initialize the model pool.
+
+        Args:
+            model_cls: Django model class
+            query_dict: Query parameters to filter model instances
+            pool_key: Unique identifier for this pool
+            default_timeout: Default timeout in seconds
+        """
+        super().__init__(pool_key, default_timeout)
+        self.model_cls = model_cls
+        self.query_dict = query_dict
+
+    def init_pool(self) -> None:
+        """Initialize pool with model instances."""
+        self.destroy_pool()
+
+        queryset = self.model_cls.objects.filter(**self.query_dict)
+        for instance in queryset:
+            item = str(instance.pk)
+            if not self.redis_client.sismember(self.all_items_set_key, item):
+                self.redis_client.sadd(self.all_items_set_key, item)
+            self.redis_client.lpush(self.available_list_key, item)
+
+    def add_to_pool(self, instance: models.Model) -> bool:
+        """Add a model instance to the pool."""
+        item = str(instance.pk)
+        if not self.redis_client.sismember(self.all_items_set_key, item):
+            self.redis_client.sadd(self.all_items_set_key, item)
+        self.redis_client.lpush(self.available_list_key, item)
+        return True
+
+    def remove_from_pool(self, instance: models.Model) -> bool:
+        """Remove instance from pool."""
+        item = str(instance.pk)
+        if not self.redis_client.sismember(self.all_items_set_key, item):
+            return False
+
+        self.redis_client.lrem(self.available_list_key, 0, item)
+        self.redis_client.srem(self.all_items_set_key, item)
+        return True
+
+    def get_next_instance(self, timeout: Optional[int] = None) -> Optional[models.Model]:
+        """Get the next available model instance."""
+        if not self.redis_client.exists(self.all_items_set_key):
+            self.init_pool()
+
+        pk = self.get_next_available(timeout)
+        if pk:
+            try:
+                instance = self.model_cls.objects.get(pk=pk)
+
+                # Verify instance still matches criteria
+                for key, value in self.query_dict.items():
+                    if getattr(instance, key) != value:
+                        self.redis_client.srem(self.all_items_set_key, pk)
+                        return self.get_next_instance(timeout)
+
+                return instance
+            except self.model_cls.DoesNotExist:
+                self.redis_client.srem(self.all_items_set_key, pk)
+                return self.get_next_instance(timeout)
+
+        return None
+
+    def get_specific_instance(self, instance: models.Model) -> bool:
+        """Get a specific instance from pool."""
+        item = str(instance.pk)
+        if not self.redis_client.sismember(self.all_items_set_key, item):
+            return False
+
+        removed = self.redis_client.lrem(self.available_list_key, 1, item)
+        return removed > 0
+
+    def return_instance(self, instance: models.Model) -> bool:
+        """Return a model instance to the pool."""
+        item = str(instance.pk)
+        if not self.redis_client.sismember(self.all_items_set_key, item):
+            return False
+
+        self.redis_client.lpush(self.available_list_key, item)
+        return True
+
+
+# Example usage:
+if __name__ == "__main__":
+    # Basic pool
+    pool = RedisBasePool("test_pool")
+
+    # Add items
+    pool.add("item1")
+    pool.add("item2")
+    pool.add("item3")
+
+    print("All items:", pool.list_all())
+    print("Available:", pool.list_available())
+
+    # Get next available
+    item = pool.get_next_available(timeout=5)
+    print(f"Got item: {item}")
+
+    # Return to pool
+    if item:
+        pool.checkin(item)
+
+    # Django model example:
+    # model_pool = RedisModelPool(
+    #     model_cls=MyModel,
+    #     query_dict={"status": "active"},
+    #     pool_key="active_models"
+    # )
+    #
+    # # Initialize pool
+    # model_pool.init_pool()
+    #
+    # # Get instance
+    # instance = model_pool.get_next_instance(timeout=30)
+    # if instance:
+    #     print(f"Got instance: {instance}")
+    #     # Do work...
+    #     model_pool.return_instance(instance)

mojo/helpers/request.py

@@ -1,5 +1,9 @@
 from objict import objict, nobjict
 from .request_parser import RequestDataParser
+from mojo.helpers.settings import settings
+
+DUID_HEADER = settings.get('DUID_HEADER', 'X-Mojo-UID').replace('-', '_').upper()
+DUID_HEADER = f"HTTP_{DUID_HEADER}"
 
 REQUEST_PARSER = RequestDataParser()
 
@@ -43,6 +47,10 @@ def get_remote_ip(request):
 
 def get_device_id(request):
     # Look for 'buid' or 'duid' in GET parameters
+    duid = request.META.get(DUID_HEADER, None)
+    if duid:
+        return duid
+
     for key in ['__buid__', 'duid', "buid"]:
         if key in request.GET:
             return request.GET[key]

mojo/helpers/response.py

@@ -8,6 +8,7 @@ class JsonResponse(HttpResponse):
             raise TypeError(
                 'In order to allow non-dict objects to be serialized set the '
                 'safe parameter to False.'
+                f'Invalid data type: {type(data)}'
             )
         kwargs.setdefault('content_type', 'application/json')
         if not isinstance(data, objict):
@@ -16,3 +17,10 @@ class JsonResponse(HttpResponse):
             data.code = status
         data = data.to_json(as_string=True)
         super().__init__(content=data, status=status, **kwargs)
+
+
+def error(message, status=400):
+    return JsonResponse(objict(error=message), status=status)
+
+def success(data, status=200):
+    return JsonResponse(objict(status=True, data=data), status=status)

mojo/middleware/auth.py

@@ -23,4 +23,4 @@ class AuthenticationMiddleware(MiddlewareMixin):
         if error is not None:
             return JsonResponse({'error': error}, status=401)
         request.user = user
-        user.touch()
+        user.track(request)

mojo/middleware/cors.py

@@ -0,0 +1,40 @@
+from django.http import HttpResponse
+from mojo.helpers.settings import settings
+
+DUID_HEADER = settings.get('DUID_HEADER', 'X-Mojo-UID')
+
+# middleware/cors.py
+class CORSMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        # Handle preflight requests
+        if request.method == 'OPTIONS':
+            response = HttpResponse()
+        else:
+            response = self.get_response(request)
+
+        # Always allow all origins
+        response['Access-Control-Allow-Origin'] = '*'
+
+        # Allow credentials if needed (note: can't use * origin with credentials)
+        # response['Access-Control-Allow-Credentials'] = 'true'
+
+        # Allow all methods to minimize preflight requests
+        response['Access-Control-Allow-Methods'] = 'GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS'
+
+        # Allow common headers to minimize preflight requests
+        response['Access-Control-Allow-Headers'] = (
+            'Accept, Accept-Encoding, Authorization, Content-Type, '
+            'Origin, User-Agent, X-Requested-With, X-CSRFToken, '
+            f'X-API-Key, {DUID_HEADER}, Cache-Control, Pragma'
+        )
+
+        # Long preflight cache (24 hours)
+        response['Access-Control-Max-Age'] = '86400'
+
+        # Expose headers that frontend might need
+        response['Access-Control-Expose-Headers'] = 'Content-Disposition, X-Total-Count'
+
+        return response

mojo/middleware/logging.py

@@ -1,3 +1,6 @@
+import json
+import threading
+from queue import Queue, Empty
 from mojo.apps.logit.models import Log
 from mojo.helpers.settings import settings
 from mojo.helpers import logit
@@ -7,49 +10,165 @@ API_PREFIX = "/".join([settings.get("MOJO_PREFIX", "api/").rstrip("/"), ""])
 LOGIT_DB_ALL = settings.get("LOGIT_DB_ALL", False)
 LOGIT_FILE_ALL = settings.get("LOGIT_FILE_ALL", False)
 LOGIT_RETURN_REAL_ERROR = settings.get("LOGIT_RETURN_REAL_ERROR", True)
+LOGIT_MAX_RESPONSE_SIZE = settings.get("LOGIT_MAX_RESPONSE_SIZE", 1024)  # 1KB default
 LOGGER = logit.get_logger("requests", "requests.log")
 ERROR_LOGGER = logit.get_logger("error", "error.log")
-LOGIT_NO_LOG_PREFIX = settings.get("LOGIT_NO_LOG_PREFIX", [])
+LOGIT_NO_LOG_PREFIX = settings.get("LOGIT_NO_LOG_PREFIX", ['GET:/api/user'])
+LOGIT_ALWAYS_LOG_PREFIX = settings.get("LOGIT_ALWAYS_LOG_PREFIX", ['POST:/api/user', 'GET:/api/user/'])
+
+# Async logging setup
+log_queue = Queue()
+background_thread = None
+
+def background_logger():
+    """Background thread to process logs without blocking responses."""
+    while True:
+        try:
+            log_item = log_queue.get(timeout=30)  # 30s timeout
+            if log_item is None:  # Shutdown signal
+                break
+
+            log_type, request, content, log_kind = log_item
+
+            if log_type == "db":
+                Log.logit(request, content, log_kind)
+            elif log_type == "file":
+                method = request.method if request else "SYSTEM"
+                ip = getattr(request, 'ip', 'unknown') if request else 'system'
+                path = getattr(request, 'path', 'unknown') if request else 'system'
+                LOGGER.info(f"{log_kind.upper()} - {method} - {ip} - {path}", content)
+
+            log_queue.task_done()
+        except Empty:
+            continue
+        except Exception as e:
+            ERROR_LOGGER.exception(f"Background logging error: {e}")
+
+def start_background_logger():
+    global background_thread
+    if background_thread is None or not background_thread.is_alive():
+        background_thread = threading.Thread(target=background_logger, daemon=True)
+        background_thread.start()
+
+# Start the background logger
+start_background_logger()
 
 class LoggerMiddleware:
     def __init__(self, get_response):
         self.get_response = get_response
 
+    def _request_matches_prefix(self, request, prefix):
+        """
+        Checks if a request matches a given prefix rule.
+        The rule can be a simple path prefix or a "METHOD:/path/prefix".
+        """
+        method = None
+        path_prefix = prefix
+
+        if ":" in prefix:
+            parts = prefix.split(":", 1)
+            # Ensure it's a valid METHOD:/path format
+            if len(parts) == 2 and parts[0].upper() in ('GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS'):
+                method, path_prefix = parts[0].upper(), parts[1]
+
+        # Check if the path matches
+        if not request.path.startswith(path_prefix):
+            return False
+
+        # If a method was specified in the rule, check if it matches
+        if method and request.method != method:
+            return False
+
+        return True
+
     def __call__(self, request):
-        # Log the request before calling get_response
         self.log_request(request)
         try:
             response = self.get_response(request)
         except Exception as e:
-            # Log or store the exception here
             err = ERROR_LOGGER.exception()
-            Log.logit(request, err, "api_error")
+            Log.logit(request, err, "api_error")  # Keep errors synchronous
             error = "system error"
             if LOGIT_RETURN_REAL_ERROR:
                 error = str(e)
             response = JsonResponse(dict(status=False, error=error), status=500)
-        # Log the response after get_response has been called
+
         self.log_response(request, response)
         return response
 
     def can_log(self, request):
-        prefixes = LOGIT_NO_LOG_PREFIX
-        if not isinstance(prefixes, (list, set, tuple)) or not prefixes:
+        """
+        Determines if a request should be logged based on settings.
+        - LOGIT_ALWAYS_LOG_PREFIX overrides LOGIT_NO_LOG_PREFIX.
+        - Rules can be "METHOD:/path/prefix" or "/path/prefix".
+        """
+        # 1. Check LOGIT_ALWAYS_LOG_PREFIX first. If it matches, we must log.
+        for prefix in LOGIT_ALWAYS_LOG_PREFIX:
+            if self._request_matches_prefix(request, prefix):
+                return True
+
+        # 2. Check LOGIT_NO_LOG_PREFIX. If it matches, we must NOT log.
+        for prefix in LOGIT_NO_LOG_PREFIX:
+            if self._request_matches_prefix(request, prefix):
+                return False
+
+        # 3. Default behavior: log if no specific rules apply.
+        return True
+
+    def should_log_full_content(self, request, response):
+        """Fast conditional checks to decide logging strategy."""
+        # Always log errors fully (but still async)
+        if response.status_code >= 400:
             return True
-        return not any(request.path.startswith(prefix) for prefix in prefixes)
+
+        # Quick size check
+        content_length = len(response.content)
+        if content_length > LOGIT_MAX_RESPONSE_SIZE:
+            return False
+
+        # Path-based decisions
+        if request.path.endswith('/list/') or '/list?' in request.path:
+            return False
+
+        return True
+
+    def get_response_log_content(self, request, response):
+        """Extract log content - prioritize log_context if available."""
+
+        # Check for log_context first (fastest path)
+        if hasattr(response, 'log_context') and response.log_context:
+            return json.dumps(response.log_context)
+
+        # Conditional processing based on fast checks
+        if not self.should_log_full_content(request, response):
+            return f"Response: {response.status_code}, Size: {len(response.content)} bytes"
+
+        # For small responses, log full content
+        return response.content
+
+    def queue_log(self, log_type, request, content, log_kind):
+        """Queue log for background processing."""
+        try:
+            log_queue.put((log_type, request, content, log_kind), block=False)
+        except:
+            # If queue is full, just skip this log to avoid blocking
+            pass
 
     def log_request(self, request):
         if not self.can_log(request):
             return
         if LOGIT_DB_ALL:
-            request.request_log = Log.logit(request, request.DATA.to_json(as_string=True), "request")
+            self.queue_log("db", request, request.DATA.to_json(as_string=True), "request")
         if LOGIT_FILE_ALL:
-            LOGGER.info(f"REQUEST - {request.method} - {request.ip} - {request.path}", request._raw_body)
+            self.queue_log("file", request, request._raw_body, "request")
 
     def log_response(self, request, response):
         if not self.can_log(request):
             return
+
+        log_content = self.get_response_log_content(request, response)
+
         if LOGIT_DB_ALL:
-            Log.logit(request, response.content, "response")
+            self.queue_log("db", request, log_content, "response")
         if LOGIT_FILE_ALL:
-            LOGGER.info(f"RESPONSE - {request.method} - {request.ip} - {request.path}", response.content)
+            self.queue_log("file", request, log_content, "response")

mojo/middleware/mojo.py

@@ -2,6 +2,10 @@ from mojo.helpers import request as rhelper
 import time
 from objict import objict
 from mojo.helpers.settings import settings
+from mojo.helpers import dates, logit
+
+
+logger = logit.get_logger("debug", "debug.log")
 
 ANONYMOUS_USER = objict(is_authenticated=False)
 
@@ -18,6 +22,7 @@ class MojoMiddleware:
         request.ip = rhelper.get_remote_ip(request)
         request.user_agent = rhelper.get_user_agent(request)
         request.duid = rhelper.get_device_id(request)
+        # logger.info(f"duid: {request.duid}", request.META)
         if settings.LOGIT_REQUEST_BODY:
             request._raw_body = str(request.body)
         else: