djlogq 1.0.8__py3-none-any.whl → 1.0.9__py3-none-any.whl

{djlogq-1.0.8.dist-info → djlogq-1.0.9.dist-info}/METADATA

@@ -1,10 +1,11 @@
 Metadata-Version: 2.4
 Name: djlogq
-Version: 1.0.8
-Summary: A reusable Django app for asynchronous, thread-safe logging with rich metadata, admin interface, and API support.
+Version: 1.0.9
+Summary: A robust and reusable Django app for asynchronous, thread-safe logging. Features include rich metadata capture, background log cleanup, configurable retention policies, admin interface, and REST API support for seamless log management.
 Author-email: mess <mesnavunawa@gmail.com>
 License: MIT
 Project-URL: Homepage, https://github.com/Mesake94/djlogq
+Project-URL: Downloads, https://pepy.tech/projects/djlogq
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 Requires-Dist: Django
@@ -32,12 +33,7 @@ A reusable Django app that provides asynchronous logging functionality using a s
 - **Extendible**: Easily add your own custom handlers to process logs in different ways.
 
   **Useful built-in and example handlers include:**
-  - **File Handler**: Write logs to a file.
-  - **Email Handler**: Send error logs via email.
-  - **Webhook Handler**: Forward logs to external services (e.g., Slack, Discord, custom endpoints).
-  - **Database Handler**: Store logs in custom database tables.
-  - **Console Handler**: Output logs to the console for development.
-  - **Third-Party Integrations**: Integrate with services like Sentry or Logstash.
+  - **Console Handler**: Output logs to the console
   - You can implement your own handler by subclassing the provided base handler class.
 
 ## Installation
@@ -69,10 +65,20 @@ python manage.py migrate
 ASYNC_LOGGING_CONFIG = {
     'MAX_QUEUE_SIZE': 1000,
     'FLUSH_INTERVAL': 1.0,  # seconds
-    'AUTO_CLEANUP_DAYS': 30,
+    'AUTO_CLEANUP_INTERVAL': 3600,  # seconds
     'ENABLE_REQUEST_LOGGING': True,
     'IGNORE_PATHS': ['/admin/'],  # paths to ignore for request logging
-    'DEFAULT_HANDLERS': [],  # list of handler class paths, e.g. ['logq.handlers.FileHandler']
+    'DEFAULT_HANDLERS': [],  # list of handler class paths, e.g. ['logq.handlers.FileHandler'],
+    # CLEANUP_POLICIES defines how long to keep logs of each level before automatic deletion.
+    # Each policy is a dictionary with:
+    #   - "days": Number of days to retain logs at this level
+    #   - "level": Log level to which this policy applies (e.g., "INFO", "WARNING", "ERROR")
+    #   - "enabled": Whether this cleanup policy is active
+    'CLEANUP_POLICIES': [
+        {"days": 10, "level": "INFO", "enabled": True},     # Keep INFO logs for 10 days
+        {"days": 10, "level": "WARNING", "enabled": True},  # Keep WARNING logs for 10 days
+        {"days": 15, "level": "ERROR", "enabled": True},    # Keep ERROR logs for 15 days
+    ]
 }
 ```
 

{djlogq-1.0.8.dist-info → djlogq-1.0.9.dist-info}/RECORD

@@ -1,10 +1,12 @@
 logq/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 logq/admin.py,sha256=nC8TVXc64G41Mh05ijUSCJmvIDo-kgZHKyyOlmqH-AE,320
-logq/apps.py,sha256=LNBTCEKiwAU2wT3CTjQ-EfXumbBF6Izez5J7mTK3i-U,330
-logq/async_logger.py,sha256=6eQxoUmWHr7nbgwA_Txnp9d91kFZfj_JZUrlIJwl97M,10481
+logq/apps.py,sha256=Nr_uXVq3PhNe5uD3z27sxdK0SlKGJ9S7CniW2pI8_ZM,765
+logq/async_logger.py,sha256=qlUhU08_riJMeuygNPgdPjPVf_vF2dmDkVu_vo8_Etk,10170
+logq/cleanup_service.py,sha256=8rTS_BJdBP8N3QdcLFa7CAV8rXssI3AJriCOBfrty6E,7302
+logq/handlers.py,sha256=C7-znAm1nUuBRf8SKkKtkSrTESeqyGYS9Grfks03r54,1090
 logq/middleware.py,sha256=Y9EAnxrmavyDgVklEW893Wh0eeQPKdCvizcxGzPDLBo,3642
-logq/models.py,sha256=Gu8KLNMn7I4XtEUHQPavSSlcESQ-2Pu5JLKsL97t9Gg,1616
-logq/tests.py,sha256=180_XHLQ3I1am5qlaaf-36n8E-CaRI4ZT3uGZ-OyYJU,10704
+logq/models.py,sha256=YvyLoaYNJVfb1caPkWZYNWcy78Q0CxYgCDv-Ht2DAq4,1618
+logq/tests.py,sha256=-meTXXNWXutVnvLfYV_tom5lV6VCACcsRp6FzD1FCoY,13216
 logq/urls.py,sha256=oGwVM9HXsVY_P86sVPuz5xnFofYfmkL8ZSZDhExhJQk,216
 logq/utils.py,sha256=lNm2TfbpWdKbAeoy4ny8-TFaNFyQGsBL_Wxq0VoxkO4,5235
 logq/views.py,sha256=WJpwiPyfItBbceM1862NcXp_ui4U6WyNBhw2P27mlJ4,3695
@@ -14,7 +16,7 @@ logq/management/commands/clean_logs.py,sha256=Cc33EEqGGwsNcvQjwnLbM6kIb0lBJsyDQl
 logq/migrations/0001_initial.py,sha256=l4f-lUcO7OsABGYiSBp7fdWDt2rLHaIhR78pCKIAAdQ,2007
 logq/migrations/0002_alter_logentry_function_alter_logentry_line_number_and_more.py,sha256=SNBngZmhk9BgcOe8eAojX47V5JKC2V7oW9QtLHWIkFc,750
 logq/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-djlogq-1.0.8.dist-info/METADATA,sha256=FpJJheb91bTyn2ML2zDaRduvmS6iclcnedomCmPMjL0,7426
-djlogq-1.0.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-djlogq-1.0.8.dist-info/top_level.txt,sha256=VWj_EO93x0ix2Y2qm6obWT22L7VPFiZ7lQf0yIaI8do,5
-djlogq-1.0.8.dist-info/RECORD,,
+djlogq-1.0.9.dist-info/METADATA,sha256=sWflz22c7J5FBCMHXtyRV9dNF3n1AbqpD7taVDZ4Wmk,7916
+djlogq-1.0.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+djlogq-1.0.9.dist-info/top_level.txt,sha256=VWj_EO93x0ix2Y2qm6obWT22L7VPFiZ7lQf0yIaI8do,5
+djlogq-1.0.9.dist-info/RECORD,,

logq/apps.py

@@ -1,4 +1,6 @@
 from django.apps import AppConfig
+from django.conf import settings
+import os
 
 
 class LogqConfig(AppConfig):
@@ -8,5 +10,14 @@ class LogqConfig(AppConfig):
 
     def ready(self):
         """Initialize the async logger when the app is ready."""
+        # Prevent multiple initializations during development server reload
+        # RUN_MAIN is set to 'true' only in the child process that actually runs the app
+        if os.environ.get('RUN_MAIN') != 'true':
+            return
+            
         from .async_logger import get_async_logger
-        get_async_logger()
+        from .cleanup_service import start_cleanup_service
+
+        get_async_logger()
+        # dont start the cleanup service in test mode
+        start_cleanup_service()

logq/async_logger.py

@@ -10,18 +10,7 @@ from django.db import transaction
 from django.conf import settings
 from .models import LogEntry, LogLevel
 from typing import List
-
-
-class LogHandler:
-    """Base class for custom log handlers"""
-
-    def handle(self, log_entry:LogEntry) -> None:
-        """Handle a log entry. Overide this method to implement custom logging behavior."""
-        pass
-
-    def flush(self) -> None:
-        """Flush any buffered log entries. Override this method to implement custom flushing behavior."""
-        pass
+from .handlers import LogHandler, ConsoleHandler
 
 
 class AsyncLogger:
@@ -44,7 +33,7 @@ class AsyncLogger:
         self._dropped_lock = threading.Lock()
 
         # initialize custom handlers
-        self.handlers = handlers or []
+        self.handlers = handlers or [ConsoleHandler()]
         self._add_default_handlers()  # add default handlers to the logger
 
     def _add_default_handlers(self):
@@ -76,7 +65,8 @@ class AsyncLogger:
 
     def clear_handlers(self):
         """Remove all custom handlers from the logger."""
-        
+        self.handlers = []
+
     def start(self):
         """Start the logging thread."""
         with self._lock:
@@ -115,7 +105,6 @@ class AsyncLogger:
                         self._flush_batch(batch)
                         batch = []
                         last_flush = current_time
-                        
             except Exception as e:
                 # Log the error to prevent infinite loops
                 print(f"Error in async logger worker: {e}")

logq/cleanup_service.py

@@ -0,0 +1,212 @@
+import threading
+import time
+from datetime import timedelta
+from django.utils import timezone
+from django.conf import settings
+from django.core.management import call_command
+from typing import List
+from .models import LogEntry
+from .async_logger import get_async_logger
+
+
+class CleanupPolicy:
+  """Define cleanup policy for log entries."""
+
+  def __init__(self, days:int, level:str=None, enabled:bool=True):
+    self.days = days
+    self.level = level
+    self.enabled = enabled
+
+  def __str__(self):
+    level_str = f"level={self.level}" if self.level else "all levels"
+    return f"Delete logs older than {self.days} days {level_str}"
+  
+
+class PeriodicCleanupService:
+  """Service that periodically cleans up old log entries.
+  Runs in a separate thread to avoid blocking the main application.
+  """
+
+  def __init__(self, policies:List[CleanupPolicy], check_interval:int=None):
+    """Initialize the cleanup service:
+    Args:
+      policies: List of cleanup policies to apply.
+      check_interval: Interval in seconds between cleanup runs.
+    """
+    self.policies = policies
+    self.check_interval = check_interval or self._get_check_interval()
+
+    self.running = False
+    self.thread = None
+    self._lock = threading.Lock()
+    self.logger = get_async_logger()
+    
+    # Track last cleanup times for each policy
+    self.last_cleanup = {}
+    for policy in policies:
+      self.last_cleanup[policy] = None
+
+  def _get_check_interval(self) -> int:
+    """Get the check interval from settings."""
+    config = getattr(settings, 'ASYNC_LOGGING_CONFIG', {})
+    return config.get('AUTO_CLEANUP_INTERVAL', 3600)
+  
+  def start(self):
+    """Start the cleanup service."""
+    with self._lock:  # Ensure thread safety
+      if not self.running:  # Only start if not already running
+        self.running = True
+        self.thread = threading.Thread(target=self._worker, daemon=True)
+        self.thread.start()
+        self.logger.info(f"Cleanup service started...")
+      else:
+        self.logger.info("Cleanup service already running")
+    
+  
+  def stop(self):
+    """Stop the cleanup service."""
+    with self._lock:
+      if self.running:
+        self.running = False
+        if self.thread:
+          self.thread.join(timeout=10.0)  # Wait for thread to finish
+        self.thread = None
+        self.logger.info("Cleanup service stopped")
+  
+  def _worker(self):
+    """Main worker thread that runs cleanup checks."""
+    while self.running:
+      try:
+        self._check_cleanup()
+        time.sleep(self.check_interval)
+      except Exception as e:
+        self.logger.error(f"Cleanup service has stopped: {e}")
+        time.sleep(60)  # Wait for 1 minute before retrying
+  
+  def _check_cleanup(self):
+    """Check if any cleanup policies should be applied."""
+    with self._lock:
+      now = timezone.now()
+      for policy in self.policies:
+        if not policy.enabled:
+          self.logger.info(f"Cleanup policy {policy} is disabled")
+          continue
+        
+        # Check if it's time to run this policy
+        if self._should_run_policy(policy):
+          self._run_cleanup_policy(policy)
+
+  def _should_run_policy(self, policy:CleanupPolicy) -> bool:
+    """Check if it's time to run this policy."""
+    last_cleanup = self.last_cleanup.get(policy)
+    if last_cleanup is None:
+      return True
+    
+    # Check if the policy has been run in the last check interval
+    return timezone.now() - last_cleanup > timedelta(seconds=self.check_interval)
+  
+  def _run_cleanup_policy(self, policy:CleanupPolicy):
+    """Run the cleanup policy."""
+    try:
+      self.logger.info(f"Running cleanup policy: {policy}")
+      args = [
+        '--days', str(policy.days),
+      ]
+      if policy.level:
+        args.extend(['--level', policy.level])
+      
+      # Run the cleanup command
+      call_command('clean_logs', *args, verbosity=0)
+      # Update the last cleanup time for this policy
+      self.last_cleanup[policy] = timezone.now()
+    except Exception as e:
+      self.logger.error(f"Cleanup policy {policy} has failed: {e}")
+  
+_cleanup_service = None
+_cleanup_service_lock = threading.Lock()
+
+
+def get_cleanup_service() -> PeriodicCleanupService:
+  """
+  Retrieve the singleton instance of the PeriodicCleanupService.
+
+  This function ensures that only one instance of the PeriodicCleanupService exists
+  throughout the application's lifecycle. If the service has not yet been created,
+  it reads the cleanup policies from the Django settings (specifically from
+  ASYNC_LOGGING_CONFIG['CLEANUP_POLICIES']), constructs CleanupPolicy objects for each
+  policy, and initializes the PeriodicCleanupService with these policies and a default
+  check interval of 10 seconds.
+
+  Returns:
+      PeriodicCleanupService: The singleton instance of the cleanup service, which
+      periodically checks and applies log cleanup policies as configured.
+
+  Notes:
+      - The cleanup policies should be defined in Django settings under
+        ASYNC_LOGGING_CONFIG['CLEANUP_POLICIES'] as a list of dictionaries, each
+        representing a policy's parameters.
+      - The service is thread-safe and intended to be started and stopped via
+        start_cleanup_service() and stop_cleanup_service().
+
+  Example:
+      service = get_cleanup_service()
+      service.start()
+  """
+  global _cleanup_service
+  with _cleanup_service_lock:
+    if _cleanup_service is None:
+      # read the policies from the config
+      policies = getattr(settings, 'ASYNC_LOGGING_CONFIG', {}).get('CLEANUP_POLICIES', [])
+      policies = [CleanupPolicy(**policy) for policy in policies]
+      _cleanup_service = PeriodicCleanupService(
+        policies=policies,
+      )
+  return _cleanup_service
+
+
+def start_cleanup_service():
+  """
+  Start the periodic log cleanup service.
+
+  This function retrieves the singleton instance of the PeriodicCleanupService
+  (which is responsible for periodically applying log cleanup policies as defined
+  in the Django settings) and starts its background thread. If the service is
+  already running, calling this function has no effect.
+
+  Usage:
+      start_cleanup_service()
+
+  Notes:
+      - The cleanup service will run in the background, periodically checking and
+        applying the configured cleanup policies.
+      - To stop the service, use stop_cleanup_service().
+      - This function is thread-safe and can be called multiple times safely.
+
+  See Also:
+      - get_cleanup_service(): Retrieves the singleton cleanup service instance.
+      - stop_cleanup_service(): Stops the running cleanup service.
+  """
+  service = get_cleanup_service()
+  service.start()
+
+
+def stop_cleanup_service():
+  """
+  Stop the periodic log cleanup service.
+
+  This function retrieves the singleton instance of the PeriodicCleanupService
+  (which is responsible for periodically applying log cleanup policies as defined
+  in the Django settings) and stops its background thread. If the service is
+  not running, calling this function has no effect.
+
+  Usage:
+      stop_cleanup_service()
+
+  Notes:
+      - This function is thread-safe and can be called multiple times safely.
+      - The service will stop running after the current cleanup cycle completes.
+  """
+  global _cleanup_service
+  with _cleanup_service_lock:
+    if _cleanup_service:
+      _cleanup_service.stop()

logq/handlers.py

@@ -0,0 +1,42 @@
+from .models import LogEntry
+import sys
+
+class LogHandler:
+    """Base class for custom log handlers"""
+
+    def handle(self, log_entry:LogEntry) -> None:
+        """Handle a log entry. Overide this method to implement custom logging behavior."""
+        pass
+
+    def flush(self) -> None:
+        """Flush any buffered log entries. Override this method to implement custom flushing behavior."""
+        pass
+
+
+class ConsoleHandler(LogHandler):
+  """Log handler that prints to the console"""
+  
+  def __init__(self):
+    self.buffer = []
+  
+  def handle(self, log_entry:LogEntry):
+    self.buffer.append({
+      "message": log_entry.message,
+      "level": log_entry.level,
+      "timestamp": log_entry.timestamp.strftime("%Y-%m-%d %H:%M:%S"),
+    })
+
+  def flush(self):
+    if not self.buffer:
+      return
+    lines = []
+    for entry in self.buffer:
+      line = (
+        f"[{entry['timestamp']}] "
+        f"{entry['level']}: "
+        f"{entry['message']} "
+      )
+      lines.append(line)
+    sys.stdout.write('\n'.join(lines) + '\n')
+    sys.stdout.flush()
+    self.buffer.clear()

logq/models.py

@@ -41,4 +41,5 @@ class LogEntry(models.Model):
     def __str__(self):
         """Return a string representation of the log entry."""
         return f"[{self.level}] {self.timestamp.strftime('%Y-%m-%d %H:%M:%S')} - {self.message[:100]}"
-  
+  
+

logq/tests.py

@@ -3,12 +3,14 @@ from django.urls import reverse
 from django.contrib.auth.models import User
 from django.utils import timezone
 from django.db import connection
+from datetime import timedelta
 import json
 import time
 import threading
 from .models import LogEntry, LogLevel
 from .async_logger import AsyncLogger, get_async_logger, stop_async_logger, LogHandler
 from .utils import log_performance, log_function_call
+from .cleanup_service import get_cleanup_service, start_cleanup_service, stop_cleanup_service
 
 
 class AsyncLoggerTestCase(TransactionTestCase):
@@ -106,7 +108,6 @@ class AsyncLoggerTestCase(TransactionTestCase):
         self.assertEqual(dropped_entry.level, LogLevel.WARNING)
         
 
-
 class LogEntryModelTestCase(TransactionTestCase):
     def setUp(self):
         super().setUp()
@@ -282,11 +283,25 @@ class LogHandlerTestCase(TransactionTestCase):
         
         # Create a test handler
         class TestHandler(LogHandler):
+
+            def __init__(self):
+                self.buffer = []
+
             def handle(self, log_entry:LogEntry) -> None:
-                pass
+                self.buffer.append({
+                    "message": log_entry.message,
+                    "level": log_entry.level,
+                    "timestamp": log_entry.timestamp.strftime("%Y-%m-%d %H:%M:%S"),
+                    "module": log_entry.module,
+                    "function": log_entry.function,
+                    "line_number": log_entry.line_number,
+                    "user_id": log_entry.user_id,
+                })
 
             def flush(self) -> None:
-                pass
+                with open("test_log.log", "a") as f:
+                    f.write(json.dumps(self.buffer) + "\n")
+                self.buffer.clear()  # Clear the buffer after writing to file
         
         # Create a logger with the test handler
         logger = get_async_logger()
@@ -306,4 +321,53 @@ class LogHandlerTestCase(TransactionTestCase):
         # Stop the logger
         logger.stop()
         time.sleep(0.2)  # Wait for thread to stop
-        
+
+
+# class CleanupServiceTestCase(TransactionTestCase):
+#     def setUp(self):
+#         super().setUp()
+#         # Stop the global logger to avoid interference
+#         stop_async_logger()
+        
+#         # Clear all existing logs
+#         with connection.cursor() as cursor:
+#             cursor.execute("DELETE FROM logq_logentry")
+        
+#         # Create a properly configured global logger
+#         from .async_logger import _async_logger
+#         from . import async_logger as async_logger_module
+#         from .cleanup_service import get_cleanup_service, start_cleanup_service, stop_cleanup_service
+        
+#         # Create a test logger with fast flush interval
+#         test_logger = AsyncLogger(max_queue_size=100, flush_interval=0.1)
+#         test_logger.start()
+        
+#         # Replace the global logger
+#         async_logger_module._async_logger = test_logger
+#         # create cleanup service
+#         cleanup_service = get_cleanup_service()
+#         cleanup_service.start()
+#         time.sleep(0.2)  # Wait for thre
+    
+#     def tearDown(self):
+#         # Stop the global logger
+#         stop_async_logger()
+#         time.sleep(0.2)  # Wait for thread to stop
+        
+#         # Clear logs after test
+#         with connection.cursor() as cursor:
+#             cursor.execute("DELETE FROM logq_logentry")
+        
+#         # stop cleanup service
+#         stop_cleanup_service()
+#         super().tearDown()
+
+#     def test_cleanup_service(self):
+#         # create a log entry
+#         logger = get_async_logger()
+#         logger.info("Test message")
+#         time.sleep(0.5)
+
+#         # check that the log entry is created
+#         self.assertEqual(LogEntry.objects.count(), 1)
+