django-restit 4.2.83__py3-none-any.whl → 4.2.84__py3-none-any.whl

account/models/notify.py

@@ -228,10 +228,11 @@ class NotificationRecord(models.Model, RestModel):
             body=message)
         try:
             email_record.save()
-            email_record.addAttachments(attachments)
+            if attachments:
+                email_record.addAttachments(attachments)
             email_record.send([nr])
         except Exception as err:
-            rh.log_exception("email send failed", email_to)
+            rh.log_exception("email send failed", member.username, subject)
 
     @classmethod
     def notifyLegacy(cls, notify_users, subject, message=None, 

auditlog/cloudwatch.py

@@ -1,9 +1,12 @@
 import boto3
-import datetime
+from datetime import datetime, timedelta
+import re
 import time
 import json
 from objict import objict
 from rest import settings
+from rest import datem
+from rest import helpers as rh
 from concurrent.futures import ThreadPoolExecutor
 
 
@@ -26,13 +29,35 @@ def log(data, log_group, log_stream):
     return True
 
 
-def logToCloudWatch(data, log_group, log_stream):
-    message = data
+def get(log_group, log_stream, size=20):
+    client = getClient()
+    if log_stream is None:
+        log_stream = DEFAULT_LOG_STREAMS
+    if isinstance(log_stream, list):
+        log_events = client.get_log_events(
+            logGroupName=log_group,
+            logStreamNames=log_stream,
+            limit=size,
+            startFromHead=False
+        )
+    else:
+        log_events = client.get_log_events(
+            logGroupName=log_group,
+            logStreamName=log_stream,
+            limit=size,
+            startFromHead=False
+        )
+
+    return log_events
+
+
+
+def logToCloudWatch(message, log_group, log_stream):
     if isinstance(message, dict):
         message = json.dumps(message)
     return logBatchToCloudWatch([
             dict(
-                timestamp=int(datetime.datetime.utcnow().timestamp() * 1000),
+                timestamp=int(datetime.utcnow().timestamp() * 1000),
                 message=message)
         ], log_group, log_stream)
 
@@ -45,9 +70,19 @@ def logBatchToCloudWatch(batch, log_group, log_stream):
     )
 
 
-def getLogGroups():
+def getLogGroups(names_only=True):
     response = getClient().describe_log_groups()
-    return response.get('logGroups', [])
+    groups = response.get('logGroups', [])
+    if names_only:
+        return [item["logGroupName"] for item in groups]
+    return groups
+
+
+def getLogStreams(log_group):
+    log_streams = client.describe_log_streams(
+        logGroupName=log_group_name
+    )
+    return log_streams['logStreams']
 
 
 def createLogStream(log_group, log_stream):
@@ -57,7 +92,68 @@ def createLogStream(log_group, log_stream):
         pass  # Log stream already exists, no need to create it
 
 
-def getInsights(log_group, start_time, end_time, query_string):
+def filter(log_group, log_streams, pattern, start_time="1h", end_time=None, resp_format=None):
+    if log_streams is None:
+        log_streams = DEFAULT_LOG_STREAMS
+
+    start_time, end_time = datem.convert_to_epoch_range(start_time, end_time)
+    rh.debug("filter", start_time, end_time, pattern, log_group, log_streams)
+    client = getClient()
+    response =client.filter_log_events(
+        logGroupName=log_group,
+        logStreamNames=log_streams,
+        filterPattern=f'"{pattern}"',
+        startTime=start_time,
+        endTime=end_time,
+        limit=1000
+    )
+    rh.debug("filter_log_events", response)
+    out = []
+    out.extend(response["events"])
+    prev_next = None
+    while "nextToken" in response and response["nextToken"]:
+        if prev_next == response["nextToken"]:
+            break
+        prev_next = response["nextToken"] 
+        response = client.filter_log_events(
+            logGroupName=log_group,
+            logStreamNames=log_streams,
+            filterPattern=f'"{pattern}"',
+            startTime=start_time,
+            endTime=end_time,
+            nextToken=response["nextToken"]
+        )
+        rh.debug("filter_log_events", response)
+        out.extend(response["events"])
+    if resp_format == "nginx":
+        return dict(events=parseNginxEvents(out))
+    return dict(events=out)
+
+
+QUERY_BY_IP = """fields @message 
+| filter @logStream in {log_streams}
+| filter @message like /^{ip}/"""
+
+QUERY_BY_TEXT = """fields @message 
+| filter @logStream in {log_streams}
+| filter @message like "{text}" """
+
+DEFAULT_LOG_STREAMS = ["access.log", "ui_access.log"]
+
+def startSearch(log_groups, start_time, end_time=None, 
+                text=None, ip=None, query_string=None,
+                log_streams=DEFAULT_LOG_STREAMS):
+    if log_streams is None:
+        log_streams = DEFAULT_LOG_STREAMS
+    # 173\.196\.133\.90
+    if ip is not None and "." in ip:
+        query_string = QUERY_BY_IP.format(ip=ip.replace('.', '\.'), log_streams=log_streams)
+    elif text is not None:
+        query_string = QUERY_BY_TEXT.format(text=text, log_streams=log_streams)
+    return startInsights(log_groups, start_time, end_time, query_string)
+
+
+def startInsights(log_groups, start_time, end_time, query_string):
     """
     Executes a CloudWatch Logs Insights query and returns the results.
 
@@ -70,24 +166,48 @@ def getInsights(log_group, start_time, end_time, query_string):
     """
     # Create a CloudWatch Logs client
     client = getClient()
+    start_time, end_time = datem.convert_to_epoch_range(start_time, end_time)
+
+    if log_groups is None:
+        log_groups = getLogGroups()
     
     # Start the query
     start_query_response = client.start_query(
-        logGroupName=log_group,
+        logGroupNames=log_groups,
         startTime=start_time,
         endTime=end_time,
         queryString=query_string,
     )
+    return dict(query_id=start_query_response['queryId'], query=query_string)
     
-    query_id = start_query_response['queryId']
-    
+
+def getInsightResults(query_id, resp_format=None):
     # Wait for the query to complete
-    response = None
-    while response is None or response['status'] == 'Running':
-        time.sleep(1)  # Sleep to rate limit the polling
-        response = client.get_query_results(queryId=query_id)
-    
-    if response['status'] == 'Complete':
-        return response['results']
-    else:
-        raise Exception(f"Query did not complete successfully. Status: {response['status']}")
+    resp = getClient().get_query_results(queryId=query_id)
+    status = resp["status"].lower()
+    results = insightsToMessage(resp["results"])
+    stats = resp["statistics"]
+    if status == "complete" and resp_format == "nginx":
+        results = parseNginxEvents(results)
+    return dict(status=True, state=status, stats=stats, data=results, count=len(results))
+
+
+def insightsToMessage(events):
+    out = []
+    for fields in events:
+        obj = {}
+        for field in fields:
+            if "field" in field and field["field"][0] == "@":
+                obj[field["field"][1:]] = field["value"]
+        if obj:
+            out.append(obj)
+    return out
+
+def parseNginxEvents(events):
+    pattern = re.compile(
+        r'(?P<ip>\d+\.\d+\.\d+\.\d+) - - \[(?P<time>.+?)\] '
+        r'"(?P<method>\w+) (?P<url>.+?) (?P<protocol>[\w/.]+)" '
+        r'(?P<status>\d+) (?P<bytes>\d+) "(?P<referer>.+?)" '
+        r'"(?P<user_agent>.+?)" (?P<request_time>\S+) (?P<server_port>\d+)'
+    )
+    return [pattern.match(line["message"]).groupdict() for line in events]

auditlog/rpc.py

@@ -1,7 +1,9 @@
 from . import models as auditlog
+from . import cloudwatch
 
 from rest import views as rv
 from rest import decorators as rd
+from datetime import datetime
 
 
 @rd.url(r'^plog$')
@@ -57,3 +59,98 @@ def plogList(request):
         qset = qset.filter(message__icontains=term)
     
     return rv.restList(request, qset.order_by('-when'), **auditlog.PersistentLog.getGraph(graph))
+
+
+@rd.url('server/logs')
+@rd.perm_required('view_logs')
+@rd.requires_params(["log_group", "log_streams"])
+def cloudwatch_logs(request):
+    log_group = request.DATA.get("log_group")
+    log_streams = request.DATA.get("log_streams").split(',')
+    pattern = request.DATA.get("search")
+    if not pattern:
+        data = dict(status=True, data=[], count=0)
+        return rv.restResult(request, data)
+    start = request.DATA.get("dr_start", "1d")
+    end = request.DATA.get("dr_end", field_type=datetime)
+    resp = cloudwatch.filter(log_group, log_streams, pattern,
+                             start_time=start, end_time=end,
+                             resp_format="nginx")
+# else:
+    #     resp = cloudwatch.get(log_group, log_streams, resp_format="nginx")
+    lst = resp["events"]
+    count = len(lst)
+    data = dict(status=True, data=lst, count=len(lst), size=count)
+    if "nextBackwardToken" in resp:
+        data["backward"] = resp["nextBackwardToken"]
+        data["forword"] = resp["nextForwardToken"]
+    return rv.restResult(request, data)
+
+@rd.urlPOST('cloudwatch/log')
+@rd.perm_required('view_logs')
+@rd.requires_params(["log_group", "log_stream", "event"])
+def cloudwatch_log_groups(request):
+    log_group = request.DATA.get("log_group")
+    log_stream = request.DATA.get("log_stream")
+    event = request.DATA.get("event")
+    resp = cloudwatch.logToCloudWatch(event, log_group, log_stream)
+    return rv.restResult(request, dict(status=True, data=resp))
+
+
+@rd.urlGET('cloudwatch/log')
+@rd.perm_required('cloudwatch')
+@rd.requires_params(["log_group", "log_stream"])
+def cloudwatch_log_groups(request):
+    log_group = request.DATA.get("log_group")
+    log_stream = request.DATA.get("log_stream")
+    pattern = request.DATA.get("pattern")
+    if pattern:
+        start = request.DATA.get("dr_start", "6d")
+        end = request.DATA.get("dr_end", field_type=datetime)
+        resp = cloudwatch.filter(log_group, [log_stream], pattern,
+                                 start_time=start, end_time=end,
+                                 resp_format=request.DATA.get("format"))
+    else:
+        resp = cloudwatch.get(log_group, log_stream)
+    lst = resp["events"]
+    data = dict(status=True, data=lst, count=len(lst))
+    if "nextBackwardToken" in resp:
+        data["backward"] = resp["nextBackwardToken"]
+        data["forword"] = resp["nextForwardToken"]
+    data = dict(status=True, data=lst, count=len(lst))
+    return rv.restResult(request, data)
+
+
+@rd.urlGET('cloudwatch/groups')
+@rd.perm_required('cloudwatch')
+def cloudwatch_log_groups(request):
+    return rv.restList(request, cloudwatch.getLogGroups(request.DATA.get("graph", None) is None))
+
+
+@rd.urlPOST('cloudwatch/insights/start')
+@rd.perm_required('cloudwatch')
+@rd.requires_params(["log_groups"])
+def cloudwatch_insights_start(request):
+    log_groups = request.DATA.get("log_groups")
+    log_streams = request.DATA.get("log_streams")
+    start = request.DATA.get("dr_start", "6h")
+    end = request.DATA.get("dr_end", field_type=datetime)
+    ip = request.DATA.get("ip")
+    text = request.DATA.get("text")
+    query = request.DATA.get("query")
+    if ip is None and text is None and query is None:
+        return rv.restPermissionDenied(request)
+    resp = cloudwatch.startSearch(log_groups, start, end, 
+                                      ip=ip, text=text, query_string=query,
+                                      log_streams=log_streams)
+    return rv.restResult(request, dict(status=True, data=resp))
+
+
+@rd.urlGET(r'cloudwatch/insights/results')
+@rd.perm_required('cloudwatch')
+@rd.requires_params(["query_id"])
+def cloudwatch_insights_results(request):
+    query_id = request.DATA.get("query_id")
+    results = cloudwatch.getInsightResults(query_id, request.DATA.get("format"))
+    return rv.restResult(request, results)
+

{django_restit-4.2.83.dist-info → django_restit-4.2.84.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: django-restit
-Version: 4.2.83
+Version: 4.2.84
 Summary: A Rest Framework for DJANGO
 License: MIT
 Author: Ian Starnes

{django_restit-4.2.83.dist-info → django_restit-4.2.84.dist-info}/RECORD

@@ -30,7 +30,7 @@ account/models/group.py,sha256=iDD_oSgswKV_t_gXZuVK80MvICrZZqdANm2jtGtOFy8,21985
 account/models/legacy.py,sha256=zYdtv4LC0ooxPVqWM-uToPwV-lYWQLorSE6p6yn1xDw,2720
 account/models/member.py,sha256=fzSVVAdbUa1knp1O4JTnYZFYRas7-zDZaOPjZAMCC1Q,52992
 account/models/membership.py,sha256=90EpAhOsGaqphDAkONP6j_qQ0OWSRaQsI8H7E7fgMkE,9249
-account/models/notify.py,sha256=iAq8tjyqouUelYgsMhWlchYmEAuCsKAyNIr5F8_xUeU,15258
+account/models/notify.py,sha256=TOkuVBLAsbzT58FOxII_G3Cj_IDQx16vyehyEsNrDcY,15306
 account/models/passkeys.py,sha256=TJxITUi4DT4_1tW2K7ZlOcRjJuMVl2NtKz7pKQU8-Tw,1516
 account/models/session.py,sha256=ELkWjB_2KXQvPtRPrvuGJpJsqrxCQX_4J53SbqGz_2U,3737
 account/models/settings.py,sha256=gOyRWBVd3BQpjfj_hJPtqX3H46ztyRAFxBrPbv11lQg,2137
@@ -61,7 +61,7 @@ account/templates/unsubscribed.html,sha256=UVp2eM3yqmy6GYzWz1FStO2I7YpWnGuXFJcTt
 auditlog/README,sha256=q4DXhdz5CuMyuxYISHXzhlHnIkRJlojwOMchLzW2qOI,520
 auditlog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 auditlog/admin.py,sha256=-q7fstdFjNeDFfbwdrxVqy0WGKxMpBwrsM7AyG1p80g,1006
-auditlog/cloudwatch.py,sha256=R-B_ByVM3We26YnDoFYIQeWV31CUyS63QTojRAkfWa8,2805
+auditlog/cloudwatch.py,sha256=-u_mbs60bqWMXUNh28S41qqPE5flJZXB_ZdtWc-2YVA,6788
 auditlog/decorators.py,sha256=ZoIv0fhZjxtMEV15NcKijW4xPF5UEScPna60zB3TxZo,6553
 auditlog/middleware.py,sha256=Q4bXg8rnm8y2fMnAsN6ha3Fz6TW8jIzLnvpu4H9SpWE,1537
 auditlog/migrations/0001_initial.py,sha256=X171gKQZIaTO9FGNG1yKTjGSZS0ZjZj5gvimF9-_kks,3309
@@ -69,7 +69,7 @@ auditlog/migrations/0002_alter_persistentlog_session.py,sha256=DkkcIobbHdbniKg5b
 auditlog/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 auditlog/models.py,sha256=skDAiuzR4chC-WNIaH2nm_VVcbnDD6ZtUxBwhk7UY8U,16517
 auditlog/periodic.py,sha256=AUhDeVsZtC47BJ-lklvYEegHoxAzj1RpIvRFSsM7g5E,363
-auditlog/rpc.py,sha256=lhme-ScqwVSKfHo3RlzMVm_C-Yx4xOxPogBEv7b3w1M,1720
+auditlog/rpc.py,sha256=gJgj3Wiar5pVsw8tuhy0jXLkqFkOr3Z-oI2DKelMRAQ,5592
 auditlog/tq.py,sha256=OgzJVspWI6FL92GEhDPtabYoP_Hd3zGNh0E297abz3Y,2415
 auditlog/urls.py,sha256=GNqpN74EpYlMND2UFUdPt5rOkTYYrdbTV0W3fg4zLfQ,163
 inbox/README.md,sha256=jsklDrzD5d94r7cwgaU6Gi1HCjBDfWq7jd92qB0JYPU,2169
@@ -379,7 +379,7 @@ rest/crypto/__init__.py,sha256=Tl0U11rgj1eBYqd6OXJ2_XSdNLumW_JkBZnaJqI6Ldw,72
 rest/crypto/aes.py,sha256=NOVRBRSHCV-om68YpGySWWG-4kako3iEVjq8hxZWPUU,4372
 rest/crypto/privpub.py,sha256=_FioylVcbMmDP80yPYjURmafEiDmEAMkskbc7WF10ac,4082
 rest/crypto/util.py,sha256=agFN2OCPHC70tHNGWrMkkZX4Tt_Ty6imoKEMdTkZpKA,4514
-rest/datem.py,sha256=CHWQXgNltXxtWg34Xio-nNG_2FC6X1RSN3LrCKz2TvU,9469
+rest/datem.py,sha256=JHMvWG8A-n4g915wrZiCtfuhgcLMgNYMXuzXIEtgaPg,12335
 rest/decorators.py,sha256=ig0LATc3-2mhEJPAWHRbIRM-ZOFyjm6e_F9RhpRWidE,15082
 rest/encryption.py,sha256=x6Kiez0tVqfxK26MSsRL3k8OS05ni1gEX2aj3I0S9V0,788
 rest/errors.py,sha256=uKwG9OkLme36etabqK54DMjMQc1fgEoUIAUxXa7WFQw,612
@@ -506,7 +506,7 @@ ws4redis/servers/uwsgi.py,sha256=VyhoCI1DnVFqBiJYHoxqn5Idlf6uJPHvfBKgkjs34mo,172
 ws4redis/settings.py,sha256=K0yBiLUuY81iDM4Yr-k8hbvjn5VVHu5zQhmMK8Dtz0s,1536
 ws4redis/utf8validator.py,sha256=S0OlfjeGRP75aO6CzZsF4oTjRQAgR17OWE9rgZdMBZA,5122
 ws4redis/websocket.py,sha256=R0TUyPsoVRD7Y_oU7w2I6NL4fPwiz5Vl94-fUkZgLHA,14848
-django_restit-4.2.83.dist-info/LICENSE.md,sha256=VHN4hhEeVOoFjtG-5fVv4jesA4SWi0Z-KgOzzN6a1ps,1068
-django_restit-4.2.83.dist-info/METADATA,sha256=rbSXb9-b2DozTJTvyuPOVSB8x-7l1fWIhCqtR013RVg,7645
-django_restit-4.2.83.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-django_restit-4.2.83.dist-info/RECORD,,
+django_restit-4.2.84.dist-info/LICENSE.md,sha256=VHN4hhEeVOoFjtG-5fVv4jesA4SWi0Z-KgOzzN6a1ps,1068
+django_restit-4.2.84.dist-info/METADATA,sha256=nTmKDiYJinMeyK6T2939-Zs5e8U4nFoaG_EV7_D9tkQ,7645
+django_restit-4.2.84.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+django_restit-4.2.84.dist-info/RECORD,,

rest/datem.py

@@ -312,3 +312,77 @@ def getDateRange(start, end=None, kind=None, zone=None, hour=0, eod=None, end_eo
         if offset:
             end = end + timedelta(hours=offset)
     return start, end
+
+
+def convert_to_epoch_range(start, end=None):
+    """
+    Convert start and end times to epoch timestamps in milliseconds.
+
+    Parameters:
+    start (int, datetime, or str): The start time, which can be:
+        - An int representing the epoch time in milliseconds.
+        - A datetime object representing the start time.
+        - A string representing a timedelta relative to the end time. The string can end with:
+            - 'm' for minutes (e.g., '30m' for 30 minutes ago)
+            - 'h' for hours (e.g., '5h' for 5 hours ago)
+            - 'd' for days (e.g., '2d' for 2 days ago)
+    end (datetime or None): The end time, which can be:
+        - A datetime object representing the end time.
+        - None, in which case the current time (UTC) is used.
+
+    Returns:
+    tuple: A tuple containing two integers:
+        - start_epoch: The epoch time of the start parameter in milliseconds.
+        - end_epoch: The epoch time of the end parameter in milliseconds.
+
+    Raises:
+    ValueError: If the end parameter is not a datetime object or None.
+    ValueError: If the start parameter is not an int, datetime object, or a valid timedelta string.
+    ValueError: If the start string does not end with 'm', 'h', or 'd'.
+
+    Examples:
+    >>> start = "30m"
+    >>> end = datetime(2024, 5, 13, 12, 30, tzinfo=timezone.utc)
+    >>> convert_to_epoch(start, end)
+    (1715676600000, 1715681400000)
+
+    >>> start = "1d"
+    >>> end = None
+    >>> convert_to_epoch(start, end)
+    (1715595000000, 1715681400000)
+
+    >>> start = "5h"
+    >>> end = datetime(2024, 5, 13, 12, 30, tzinfo=timezone.utc)
+    >>> convert_to_epoch(start, end)
+    (1715661000000, 1715681400000)
+    """
+    if end is None:
+        end = datetime.utcnow()
+
+    if isinstance(end, datetime):
+        end_epoch = int(end.timestamp() * 1000)
+    else:
+        raise ValueError("End parameter must be a datetime object or None.")
+
+    if isinstance(start, int):
+        start_epoch = start
+    elif isinstance(start, datetime):
+        start_epoch = int(start.timestamp() * 1000)
+    elif isinstance(start, str):
+        if start.endswith('m'):
+            delta_minutes = int(start[:-1])
+            start_datetime = end - timedelta(minutes=delta_minutes)
+        elif start.endswith('h'):
+            delta_hours = int(start[:-1])
+            start_datetime = end - timedelta(hours=delta_hours)
+        elif start.endswith('d'):
+            delta_days = int(start[:-1])
+            start_datetime = end - timedelta(days=delta_days)
+        else:
+            raise ValueError("Start string must end with 'm' for minutes, 'h' for hours, or 'd' for days.")
+        
+        start_epoch = int(start_datetime.timestamp() * 1000)
+    else:
+        raise ValueError("Start parameter must be an int, datetime object, or a string representing a timedelta.")
+
+    return start_epoch, end_epoch