django-restit 4.2.83__tar.gz → 4.2.84__tar.gz

{django_restit-4.2.83 → django_restit-4.2.84}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: django-restit
-Version: 4.2.83
+Version: 4.2.84
 Summary: A Rest Framework for DJANGO
 License: MIT
 Author: Ian Starnes

{django_restit-4.2.83 → django_restit-4.2.84}/account/models/notify.py

@@ -228,10 +228,11 @@ class NotificationRecord(models.Model, RestModel):
             body=message)
         try:
             email_record.save()
-            email_record.addAttachments(attachments)
+            if attachments:
+                email_record.addAttachments(attachments)
             email_record.send([nr])
         except Exception as err:
-            rh.log_exception("email send failed", email_to)
+            rh.log_exception("email send failed", member.username, subject)
 
     @classmethod
     def notifyLegacy(cls, notify_users, subject, message=None, 

django_restit-4.2.84/auditlog/cloudwatch.py

@@ -0,0 +1,213 @@
+import boto3
+from datetime import datetime, timedelta
+import re
+import time
+import json
+from objict import objict
+from rest import settings
+from rest import datem
+from rest import helpers as rh
+from concurrent.futures import ThreadPoolExecutor
+
+
+LOG_CACHE = objict()
+
+
+def getClient():
+    if LOG_CACHE.client is None:
+        key = settings.AWS_KEY
+        secret = settings.AWS_SECRET
+        region = settings.AWS_REGION
+        LOG_CACHE.client = boto3.client("logs", aws_access_key_id=key, aws_secret_access_key=secret, region_name=region)
+    return LOG_CACHE.client
+
+
+def log(data, log_group, log_stream):
+    if LOG_CACHE.pool is None:
+        LOG_CACHE.pool = ThreadPoolExecutor(max_workers=1)
+    LOG_CACHE.pool.submit(logToCloudWatch, data, log_group, log_stream)
+    return True
+
+
+def get(log_group, log_stream, size=20):
+    client = getClient()
+    if log_stream is None:
+        log_stream = DEFAULT_LOG_STREAMS
+    if isinstance(log_stream, list):
+        log_events = client.get_log_events(
+            logGroupName=log_group,
+            logStreamNames=log_stream,
+            limit=size,
+            startFromHead=False
+        )
+    else:
+        log_events = client.get_log_events(
+            logGroupName=log_group,
+            logStreamName=log_stream,
+            limit=size,
+            startFromHead=False
+        )
+
+    return log_events
+
+
+
+def logToCloudWatch(message, log_group, log_stream):
+    if isinstance(message, dict):
+        message = json.dumps(message)
+    return logBatchToCloudWatch([
+            dict(
+                timestamp=int(datetime.utcnow().timestamp() * 1000),
+                message=message)
+        ], log_group, log_stream)
+
+
+def logBatchToCloudWatch(batch, log_group, log_stream):
+    return getClient().put_log_events(
+        logGroupName=log_group,
+        logStreamName=log_stream,
+        logEvents=batch
+    )
+
+
+def getLogGroups(names_only=True):
+    response = getClient().describe_log_groups()
+    groups = response.get('logGroups', [])
+    if names_only:
+        return [item["logGroupName"] for item in groups]
+    return groups
+
+
+def getLogStreams(log_group):
+    log_streams = client.describe_log_streams(
+        logGroupName=log_group_name
+    )
+    return log_streams['logStreams']
+
+
+def createLogStream(log_group, log_stream):
+    try:
+        getClient().create_log_stream(logGroupName=log_group, logStreamName=log_stream)
+    except Exception:
+        pass  # Log stream already exists, no need to create it
+
+
+def filter(log_group, log_streams, pattern, start_time="1h", end_time=None, resp_format=None):
+    if log_streams is None:
+        log_streams = DEFAULT_LOG_STREAMS
+
+    start_time, end_time = datem.convert_to_epoch_range(start_time, end_time)
+    rh.debug("filter", start_time, end_time, pattern, log_group, log_streams)
+    client = getClient()
+    response =client.filter_log_events(
+        logGroupName=log_group,
+        logStreamNames=log_streams,
+        filterPattern=f'"{pattern}"',
+        startTime=start_time,
+        endTime=end_time,
+        limit=1000
+    )
+    rh.debug("filter_log_events", response)
+    out = []
+    out.extend(response["events"])
+    prev_next = None
+    while "nextToken" in response and response["nextToken"]:
+        if prev_next == response["nextToken"]:
+            break
+        prev_next = response["nextToken"] 
+        response = client.filter_log_events(
+            logGroupName=log_group,
+            logStreamNames=log_streams,
+            filterPattern=f'"{pattern}"',
+            startTime=start_time,
+            endTime=end_time,
+            nextToken=response["nextToken"]
+        )
+        rh.debug("filter_log_events", response)
+        out.extend(response["events"])
+    if resp_format == "nginx":
+        return dict(events=parseNginxEvents(out))
+    return dict(events=out)
+
+
+QUERY_BY_IP = """fields @message 
+| filter @logStream in {log_streams}
+| filter @message like /^{ip}/"""
+
+QUERY_BY_TEXT = """fields @message 
+| filter @logStream in {log_streams}
+| filter @message like "{text}" """
+
+DEFAULT_LOG_STREAMS = ["access.log", "ui_access.log"]
+
+def startSearch(log_groups, start_time, end_time=None, 
+                text=None, ip=None, query_string=None,
+                log_streams=DEFAULT_LOG_STREAMS):
+    if log_streams is None:
+        log_streams = DEFAULT_LOG_STREAMS
+    # 173\.196\.133\.90
+    if ip is not None and "." in ip:
+        query_string = QUERY_BY_IP.format(ip=ip.replace('.', '\.'), log_streams=log_streams)
+    elif text is not None:
+        query_string = QUERY_BY_TEXT.format(text=text, log_streams=log_streams)
+    return startInsights(log_groups, start_time, end_time, query_string)
+
+
+def startInsights(log_groups, start_time, end_time, query_string):
+    """
+    Executes a CloudWatch Logs Insights query and returns the results.
+
+    :param log_group: The name of the log group to query.
+    :param start_time: The start time of the query (epoch time in seconds).
+    :param end_time: The end time of the query (epoch time in seconds).
+    :param query_string: The query string to use.
+    :param region_name: AWS region name.
+    :return: The query results.
+    """
+    # Create a CloudWatch Logs client
+    client = getClient()
+    start_time, end_time = datem.convert_to_epoch_range(start_time, end_time)
+
+    if log_groups is None:
+        log_groups = getLogGroups()
+    
+    # Start the query
+    start_query_response = client.start_query(
+        logGroupNames=log_groups,
+        startTime=start_time,
+        endTime=end_time,
+        queryString=query_string,
+    )
+    return dict(query_id=start_query_response['queryId'], query=query_string)
+    
+
+def getInsightResults(query_id, resp_format=None):
+    # Wait for the query to complete
+    resp = getClient().get_query_results(queryId=query_id)
+    status = resp["status"].lower()
+    results = insightsToMessage(resp["results"])
+    stats = resp["statistics"]
+    if status == "complete" and resp_format == "nginx":
+        results = parseNginxEvents(results)
+    return dict(status=True, state=status, stats=stats, data=results, count=len(results))
+
+
+def insightsToMessage(events):
+    out = []
+    for fields in events:
+        obj = {}
+        for field in fields:
+            if "field" in field and field["field"][0] == "@":
+                obj[field["field"][1:]] = field["value"]
+        if obj:
+            out.append(obj)
+    return out
+
+def parseNginxEvents(events):
+    pattern = re.compile(
+        r'(?P<ip>\d+\.\d+\.\d+\.\d+) - - \[(?P<time>.+?)\] '
+        r'"(?P<method>\w+) (?P<url>.+?) (?P<protocol>[\w/.]+)" '
+        r'(?P<status>\d+) (?P<bytes>\d+) "(?P<referer>.+?)" '
+        r'"(?P<user_agent>.+?)" (?P<request_time>\S+) (?P<server_port>\d+)'
+    )
+    return [pattern.match(line["message"]).groupdict() for line in events]

django_restit-4.2.84/auditlog/rpc.py

@@ -0,0 +1,156 @@
+from . import models as auditlog
+from . import cloudwatch
+
+from rest import views as rv
+from rest import decorators as rd
+from datetime import datetime
+
+
+@rd.url(r'^plog$')
+@rd.url(r'^plog/(?P<plog_id>\d+)$')
+@rd.perm_required('view_logs')
+def plog_handler(request, plog_id=None):
+    if not plog_id:
+        min_pk = getattr(auditlog.settings, "PLOG_STALE_ID", 0)
+        return auditlog.PersistentLog.on_rest_list(request, qset=auditlog.PersistentLog.objects.filter(pk__gte=min_pk))
+    return auditlog.PersistentLog.on_rest_request(request, plog_id)
+
+
+@rd.urlGET(r'^plog_old$')
+@rd.perm_required('view_logs')
+def plogList(request):
+    auditlog.PersistentLog.on_request_handle()
+
+    graph = request.DATA.get("graph", "default")
+    qset = auditlog.PersistentLog.objects.all()
+    if request.group:
+        qset = qset.filter(group=request.group)
+
+    ip = request.DATA.get("ip")
+    if ip:
+        qset = qset.filter(session__ip=ip)
+
+    path = request.DATA.get("path")
+    if path:
+        qset = qset.filter(remote_path__icontains=path)
+    
+    method = request.DATA.get("method")
+    if method:
+        qset = qset.filter(method=method)
+
+    action = request.DATA.get("action")
+    if action:
+        qset = qset.filter(action=action)
+    
+    component = request.DATA.get("component")
+    if component:
+        qset = qset.filter(component=component)
+    
+    pkey = request.DATA.get("pkey")
+    if pkey:
+        qset = qset.filter(pkey=pkey)
+
+    username = request.DATA.get("username")
+    if username:
+        qset = qset.filter(user__username=username)
+    
+    term = request.DATA.get("term")
+    if term:
+        qset = qset.filter(message__icontains=term)
+    
+    return rv.restList(request, qset.order_by('-when'), **auditlog.PersistentLog.getGraph(graph))
+
+
+@rd.url('server/logs')
+@rd.perm_required('view_logs')
+@rd.requires_params(["log_group", "log_streams"])
+def cloudwatch_logs(request):
+    log_group = request.DATA.get("log_group")
+    log_streams = request.DATA.get("log_streams").split(',')
+    pattern = request.DATA.get("search")
+    if not pattern:
+        data = dict(status=True, data=[], count=0)
+        return rv.restResult(request, data)
+    start = request.DATA.get("dr_start", "1d")
+    end = request.DATA.get("dr_end", field_type=datetime)
+    resp = cloudwatch.filter(log_group, log_streams, pattern,
+                             start_time=start, end_time=end,
+                             resp_format="nginx")
+# else:
+    #     resp = cloudwatch.get(log_group, log_streams, resp_format="nginx")
+    lst = resp["events"]
+    count = len(lst)
+    data = dict(status=True, data=lst, count=len(lst), size=count)
+    if "nextBackwardToken" in resp:
+        data["backward"] = resp["nextBackwardToken"]
+        data["forword"] = resp["nextForwardToken"]
+    return rv.restResult(request, data)
+
+@rd.urlPOST('cloudwatch/log')
+@rd.perm_required('view_logs')
+@rd.requires_params(["log_group", "log_stream", "event"])
+def cloudwatch_log_groups(request):
+    log_group = request.DATA.get("log_group")
+    log_stream = request.DATA.get("log_stream")
+    event = request.DATA.get("event")
+    resp = cloudwatch.logToCloudWatch(event, log_group, log_stream)
+    return rv.restResult(request, dict(status=True, data=resp))
+
+
+@rd.urlGET('cloudwatch/log')
+@rd.perm_required('cloudwatch')
+@rd.requires_params(["log_group", "log_stream"])
+def cloudwatch_log_groups(request):
+    log_group = request.DATA.get("log_group")
+    log_stream = request.DATA.get("log_stream")
+    pattern = request.DATA.get("pattern")
+    if pattern:
+        start = request.DATA.get("dr_start", "6d")
+        end = request.DATA.get("dr_end", field_type=datetime)
+        resp = cloudwatch.filter(log_group, [log_stream], pattern,
+                                 start_time=start, end_time=end,
+                                 resp_format=request.DATA.get("format"))
+    else:
+        resp = cloudwatch.get(log_group, log_stream)
+    lst = resp["events"]
+    data = dict(status=True, data=lst, count=len(lst))
+    if "nextBackwardToken" in resp:
+        data["backward"] = resp["nextBackwardToken"]
+        data["forword"] = resp["nextForwardToken"]
+    data = dict(status=True, data=lst, count=len(lst))
+    return rv.restResult(request, data)
+
+
+@rd.urlGET('cloudwatch/groups')
+@rd.perm_required('cloudwatch')
+def cloudwatch_log_groups(request):
+    return rv.restList(request, cloudwatch.getLogGroups(request.DATA.get("graph", None) is None))
+
+
+@rd.urlPOST('cloudwatch/insights/start')
+@rd.perm_required('cloudwatch')
+@rd.requires_params(["log_groups"])
+def cloudwatch_insights_start(request):
+    log_groups = request.DATA.get("log_groups")
+    log_streams = request.DATA.get("log_streams")
+    start = request.DATA.get("dr_start", "6h")
+    end = request.DATA.get("dr_end", field_type=datetime)
+    ip = request.DATA.get("ip")
+    text = request.DATA.get("text")
+    query = request.DATA.get("query")
+    if ip is None and text is None and query is None:
+        return rv.restPermissionDenied(request)
+    resp = cloudwatch.startSearch(log_groups, start, end, 
+                                      ip=ip, text=text, query_string=query,
+                                      log_streams=log_streams)
+    return rv.restResult(request, dict(status=True, data=resp))
+
+
+@rd.urlGET(r'cloudwatch/insights/results')
+@rd.perm_required('cloudwatch')
+@rd.requires_params(["query_id"])
+def cloudwatch_insights_results(request):
+    query_id = request.DATA.get("query_id")
+    results = cloudwatch.getInsightResults(query_id, request.DATA.get("format"))
+    return rv.restResult(request, results)
+

{django_restit-4.2.83 → django_restit-4.2.84}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "django-restit"
-version = "4.2.83"
+version = "4.2.84"
 description = "A Rest Framework for DJANGO"
 authors = ["Ian Starnes <ians@311labs.com>"]
 license = "MIT"

{django_restit-4.2.83 → django_restit-4.2.84}/rest/datem.py

@@ -312,3 +312,77 @@ def getDateRange(start, end=None, kind=None, zone=None, hour=0, eod=None, end_eo
         if offset:
             end = end + timedelta(hours=offset)
     return start, end
+
+
+def convert_to_epoch_range(start, end=None):
+    """
+    Convert start and end times to epoch timestamps in milliseconds.
+
+    Parameters:
+    start (int, datetime, or str): The start time, which can be:
+        - An int representing the epoch time in milliseconds.
+        - A datetime object representing the start time.
+        - A string representing a timedelta relative to the end time. The string can end with:
+            - 'm' for minutes (e.g., '30m' for 30 minutes ago)
+            - 'h' for hours (e.g., '5h' for 5 hours ago)
+            - 'd' for days (e.g., '2d' for 2 days ago)
+    end (datetime or None): The end time, which can be:
+        - A datetime object representing the end time.
+        - None, in which case the current time (UTC) is used.
+
+    Returns:
+    tuple: A tuple containing two integers:
+        - start_epoch: The epoch time of the start parameter in milliseconds.
+        - end_epoch: The epoch time of the end parameter in milliseconds.
+
+    Raises:
+    ValueError: If the end parameter is not a datetime object or None.
+    ValueError: If the start parameter is not an int, datetime object, or a valid timedelta string.
+    ValueError: If the start string does not end with 'm', 'h', or 'd'.
+
+    Examples:
+    >>> start = "30m"
+    >>> end = datetime(2024, 5, 13, 12, 30, tzinfo=timezone.utc)
+    >>> convert_to_epoch(start, end)
+    (1715676600000, 1715681400000)
+
+    >>> start = "1d"
+    >>> end = None
+    >>> convert_to_epoch(start, end)
+    (1715595000000, 1715681400000)
+
+    >>> start = "5h"
+    >>> end = datetime(2024, 5, 13, 12, 30, tzinfo=timezone.utc)
+    >>> convert_to_epoch(start, end)
+    (1715661000000, 1715681400000)
+    """
+    if end is None:
+        end = datetime.utcnow()
+
+    if isinstance(end, datetime):
+        end_epoch = int(end.timestamp() * 1000)
+    else:
+        raise ValueError("End parameter must be a datetime object or None.")
+
+    if isinstance(start, int):
+        start_epoch = start
+    elif isinstance(start, datetime):
+        start_epoch = int(start.timestamp() * 1000)
+    elif isinstance(start, str):
+        if start.endswith('m'):
+            delta_minutes = int(start[:-1])
+            start_datetime = end - timedelta(minutes=delta_minutes)
+        elif start.endswith('h'):
+            delta_hours = int(start[:-1])
+            start_datetime = end - timedelta(hours=delta_hours)
+        elif start.endswith('d'):
+            delta_days = int(start[:-1])
+            start_datetime = end - timedelta(days=delta_days)
+        else:
+            raise ValueError("Start string must end with 'm' for minutes, 'h' for hours, or 'd' for days.")
+        
+        start_epoch = int(start_datetime.timestamp() * 1000)
+    else:
+        raise ValueError("Start parameter must be an int, datetime object, or a string representing a timedelta.")
+
+    return start_epoch, end_epoch

django_restit-4.2.83/auditlog/cloudwatch.py

@@ -1,93 +0,0 @@
-import boto3
-import datetime
-import time
-import json
-from objict import objict
-from rest import settings
-from concurrent.futures import ThreadPoolExecutor
-
-
-LOG_CACHE = objict()
-
-
-def getClient():
-    if LOG_CACHE.client is None:
-        key = settings.AWS_KEY
-        secret = settings.AWS_SECRET
-        region = settings.AWS_REGION
-        LOG_CACHE.client = boto3.client("logs", aws_access_key_id=key, aws_secret_access_key=secret, region_name=region)
-    return LOG_CACHE.client
-
-
-def log(data, log_group, log_stream):
-    if LOG_CACHE.pool is None:
-        LOG_CACHE.pool = ThreadPoolExecutor(max_workers=1)
-    LOG_CACHE.pool.submit(logToCloudWatch, data, log_group, log_stream)
-    return True
-
-
-def logToCloudWatch(data, log_group, log_stream):
-    message = data
-    if isinstance(message, dict):
-        message = json.dumps(message)
-    return logBatchToCloudWatch([
-            dict(
-                timestamp=int(datetime.datetime.utcnow().timestamp() * 1000),
-                message=message)
-        ], log_group, log_stream)
-
-
-def logBatchToCloudWatch(batch, log_group, log_stream):
-    return getClient().put_log_events(
-        logGroupName=log_group,
-        logStreamName=log_stream,
-        logEvents=batch
-    )
-
-
-def getLogGroups():
-    response = getClient().describe_log_groups()
-    return response.get('logGroups', [])
-
-
-def createLogStream(log_group, log_stream):
-    try:
-        getClient().create_log_stream(logGroupName=log_group, logStreamName=log_stream)
-    except Exception:
-        pass  # Log stream already exists, no need to create it
-
-
-def getInsights(log_group, start_time, end_time, query_string):
-    """
-    Executes a CloudWatch Logs Insights query and returns the results.
-
-    :param log_group: The name of the log group to query.
-    :param start_time: The start time of the query (epoch time in seconds).
-    :param end_time: The end time of the query (epoch time in seconds).
-    :param query_string: The query string to use.
-    :param region_name: AWS region name.
-    :return: The query results.
-    """
-    # Create a CloudWatch Logs client
-    client = getClient()
-    
-    # Start the query
-    start_query_response = client.start_query(
-        logGroupName=log_group,
-        startTime=start_time,
-        endTime=end_time,
-        queryString=query_string,
-    )
-    
-    query_id = start_query_response['queryId']
-    
-    # Wait for the query to complete
-    response = None
-    while response is None or response['status'] == 'Running':
-        time.sleep(1)  # Sleep to rate limit the polling
-        response = client.get_query_results(queryId=query_id)
-    
-    if response['status'] == 'Complete':
-        return response['results']
-    else:
-        raise Exception(f"Query did not complete successfully. Status: {response['status']}")

django_restit-4.2.83/auditlog/rpc.py

@@ -1,59 +0,0 @@
-from . import models as auditlog
-
-from rest import views as rv
-from rest import decorators as rd
-
-
-@rd.url(r'^plog$')
-@rd.url(r'^plog/(?P<plog_id>\d+)$')
-@rd.perm_required('view_logs')
-def plog_handler(request, plog_id=None):
-    if not plog_id:
-        min_pk = getattr(auditlog.settings, "PLOG_STALE_ID", 0)
-        return auditlog.PersistentLog.on_rest_list(request, qset=auditlog.PersistentLog.objects.filter(pk__gte=min_pk))
-    return auditlog.PersistentLog.on_rest_request(request, plog_id)
-
-
-@rd.urlGET(r'^plog_old$')
-@rd.perm_required('view_logs')
-def plogList(request):
-    auditlog.PersistentLog.on_request_handle()
-
-    graph = request.DATA.get("graph", "default")
-    qset = auditlog.PersistentLog.objects.all()
-    if request.group:
-        qset = qset.filter(group=request.group)
-
-    ip = request.DATA.get("ip")
-    if ip:
-        qset = qset.filter(session__ip=ip)
-
-    path = request.DATA.get("path")
-    if path:
-        qset = qset.filter(remote_path__icontains=path)
-    
-    method = request.DATA.get("method")
-    if method:
-        qset = qset.filter(method=method)
-
-    action = request.DATA.get("action")
-    if action:
-        qset = qset.filter(action=action)
-    
-    component = request.DATA.get("component")
-    if component:
-        qset = qset.filter(component=component)
-    
-    pkey = request.DATA.get("pkey")
-    if pkey:
-        qset = qset.filter(pkey=pkey)
-
-    username = request.DATA.get("username")
-    if username:
-        qset = qset.filter(user__username=username)
-    
-    term = request.DATA.get("term")
-    if term:
-        qset = qset.filter(message__icontains=term)
-    
-    return rv.restList(request, qset.order_by('-when'), **auditlog.PersistentLog.getGraph(graph))