howler-api 2.13.0.dev329__py3-none-any.whl

howler/actions/transition.py

@@ -0,0 +1,200 @@
+import inspect
+from typing import Optional
+
+from howler.common.exceptions import InvalidDataException, NotFoundException
+from howler.common.loader import datastore
+from howler.common.logging import get_logger
+from howler.helper.workflow import Workflow, WorkflowException
+from howler.odm.models.action import VALID_TRIGGERS
+from howler.odm.models.howler_data import (
+    Assessment,
+    HitStatus,
+    HitStatusTransition,
+    Vote,
+)
+from howler.odm.models.user import User
+from howler.services import event_service, hit_service
+from howler.utils.list_utils import flatten_list
+
+OPERATION_ID = "transition"
+
+log = get_logger(__file__)
+
+
+def __parse_workflow_actions(workflow: Workflow) -> dict[str, set[str]]:
+    """Take in a workflow, and parse the steps and transitions of that workflow into a format understood by the UI"""
+    parsed_args: dict[str, set[str]] = {}
+
+    for wf in workflow.transitions.values():
+        if wf["transition"] in [
+            HitStatusTransition.RE_EVALUATE,
+            HitStatusTransition.PROMOTE,
+            HitStatusTransition.DEMOTE,
+        ]:
+            continue
+
+        wf_args = flatten_list(
+            [
+                [var for var in inspect.getfullargspec(m)[0] if var not in ["kwargs", "hit", "user", "transition"]]
+                for m in wf["actions"]
+            ]
+        )
+
+        for key in wf_args:
+            entry = f'transition:{str(wf["transition"])}'
+
+            if key in parsed_args:
+                parsed_args[key].add(entry)
+            else:
+                parsed_args[key] = {entry}
+
+    return parsed_args
+
+
+def execute(
+    query: str,
+    status: str,
+    transition: str,
+    user: User,
+    request_id: Optional[str] = None,
+    **kwargs,
+):
+    """Attempt to excute a transition on a hit.
+
+    The hit must be in the specified status in order for the action to execute - otherwise, the automation will filter
+    out those options.
+
+    Args:
+        query (str): The query on which to apply this automation.
+        request_id (str): The id of this automation run. Used to track the progress via websockets.
+        status (str): The status from which to transition.
+        transition (str): The transition to attempt to execute.
+    """
+    rows = 1000 if "automation_advanced" in user.type else 10
+    hits = datastore().hit.search(f"({query}) AND howler.status:{status}", rows=rows, fl="howler.id")
+
+    ids = [hit.howler.id for hit in hits["items"]]
+
+    if len(ids) < 1:
+        return [
+            {
+                "query": query,
+                "outcome": "skipped",
+                "title": "No matching hits",
+                "message": "No hits matched this query, so the automation skipped.",
+            }
+        ]
+
+    report = []
+
+    if rows < hits["total"]:
+        report.append(
+            {
+                "query": query,
+                "outcome": "skipped",
+                "title": "Too Many Hits",
+                "message": f"A maximum of {rows} hits can be processed at once, but {hits['total']} matched the query.",
+            }
+        )
+
+    num_skipped = datastore().hit.search(f"({query}) AND -howler.status:{status}", rows=1)["total"]
+
+    if num_skipped > 0:
+        report.append(
+            {
+                "query": f"({query}) AND -howler.status:{status}",
+                "outcome": "skipped",
+                "title": f"Skipped {num_skipped} hits",
+                "message": f"These hits did not have the correct status ({status}), and were skipped.",
+            }
+        )
+
+    success_ids = set()
+    total_processed = 0
+    for hit_id in ids:
+        try:
+            hit_service.transition_hit(
+                hit_id,
+                HitStatusTransition[transition],
+                user,
+                **kwargs,
+            )
+            success_ids.add(hit_id)
+        except (InvalidDataException, NotFoundException, WorkflowException) as e:
+            report.append(
+                {
+                    "query": f"howler.id:{hit_id}",
+                    "outcome": "error",
+                    "title": "An error occurred while processing.",
+                    "message": str(e),
+                }
+            )
+
+        total_processed += 1
+        if total_processed % 10 == 0:
+            log.debug("Transition executed on %s hits", total_processed)
+            if request_id is not None:
+                event_service.emit(
+                    "automation",
+                    {
+                        "request_id": request_id,
+                        "processed": total_processed,
+                        "total": len(ids),
+                    },
+                )
+
+    log.info(
+        "Transition %s processed on %s hits (%s successful)",
+        transition,
+        len(ids),
+        len(success_ids),
+    )
+
+    if len(success_ids) > 0:
+        report.append(
+            {
+                "query": f"howler.id:({' OR '.join(success_ids)})",
+                "outcome": "success",
+                "title": "Transition Executed Successfully",
+                "message": f"The transition {transition} successfully executed on {len(success_ids)} hits.",
+            }
+        )
+
+    datastore().hit.commit()
+
+    return report
+
+
+def specification():
+    """Specify various properties of the action, such as title, descriptions, permissions and input steps."""
+    return {
+        "id": OPERATION_ID,
+        "title": "Transition",
+        "priority": 9,
+        "i18nKey": "operations.transition",
+        "description": {
+            "short": "Transition a hit",
+            "long": execute.__doc__,
+        },
+        "roles": ["automation_basic"],
+        "steps": [
+            {
+                "args": {"status": []},
+                "options": {"status": HitStatus.list()},
+                "validation": {"error": {"query": "-howler.status:$status"}},
+            },
+            {
+                "args": {"transition": []},
+                "options": {
+                    "transition": {
+                        f"status:{status}": hit_service.get_transitions(status) for status in HitStatus.list()
+                    },
+                },
+            },
+            {
+                "args": __parse_workflow_actions(hit_service.get_hit_workflow()),
+                "options": {"vote": Vote.list(), "assessment": Assessment.list()},
+            },
+        ],
+        "triggers": [trigger for trigger in VALID_TRIGGERS if trigger != "create"],
+    }

howler/api/__init__.py

@@ -0,0 +1,249 @@
+from sys import exc_info
+from traceback import format_tb
+from typing import Any, Union
+
+from flask import Blueprint, Response, jsonify, make_response, request
+from flask import session as flsk_session
+from prometheus_client import Counter
+
+from howler import odm
+from howler.common.loader import APP_NAME
+from howler.common.logging import get_logger, log_with_traceback
+from howler.config import QUOTA_TRACKER, get_version
+from howler.utils.str_utils import safe_str
+
+API_PREFIX = "/api"
+RAW_API_COUNTER = Counter(
+    f"{APP_NAME.replace('-', '_')}_http_requests_total",
+    "HTTP Requests broken down by method, path, and status",
+    ["method", "path", "status"],
+)
+
+logger = get_logger(__file__)
+
+
+def make_subapi_blueprint(name, api_version=1):
+    """Create a flask Blueprint for a subapi in a standard way."""
+    return Blueprint(name, name, url_prefix="/".join([API_PREFIX, f"v{api_version}", name]))
+
+
+def _make_api_response(
+    data: Any, err: Union[str, Exception] = "", warnings: list[str] = [], status_code: int = 200, cookies: Any = None
+) -> Response:
+    quota_user = flsk_session.pop("quota_user", None)
+    quota_set = flsk_session.pop("quota_set", False)
+    if quota_user and quota_set and not request.path.startswith("/api/v1/borealis"):
+        QUOTA_TRACKER.end(quota_user)
+
+    if type(err) is Exception:  # pragma: no cover
+        trace = exc_info()[2]
+        err = "".join(["\n"] + format_tb(trace) + ["%s: %s\n" % (err.__class__.__name__, str(err))]).rstrip("\n")
+        log_with_traceback(trace, "Exception", is_exception=True)
+
+    if isinstance(data, odm.Model):
+        data = data.as_primitives()
+
+    if isinstance(data, list) and len(data) > 0 and isinstance(data[0], odm.Model):
+        for i in range(len(data)):
+            data[i] = data[i].as_primitives()
+
+    resp = make_response(
+        jsonify(
+            {
+                "api_response": data,
+                "api_error_message": err,
+                "api_warning": warnings,
+                "api_server_version": get_version(),
+                "api_status_code": status_code,
+            }
+        ),
+        status_code,
+    )
+
+    if isinstance(cookies, dict):
+        for k, v in cookies.items():
+            resp.set_cookie(k, v, secure=True, httponly=True, samesite="Lax")
+
+    RAW_API_COUNTER.labels(request.method, str(request.url_rule), status_code).inc()
+    logger.info("%s %s - %s", request.method, request.path, status_code)
+
+    return resp
+
+
+# Some helper functions for make_api_response
+
+DEFAULT_DATA = {True: {"success": True}, False: {"success": False}}
+
+
+def ok(data=DEFAULT_DATA[True], cookies=None):
+    """Returns response with status code 200"""
+    return _make_api_response(data, status_code=200, cookies=cookies)
+
+
+def created(data=DEFAULT_DATA[True], warnings=[], cookies=None):
+    """Returns response with status code 201"""
+    return _make_api_response(data, warnings=warnings, status_code=201, cookies=cookies)
+
+
+def accepted(data=DEFAULT_DATA[True], cookies=None):
+    """Returns response with status code 202"""
+    return _make_api_response(data, status_code=202, cookies=cookies)
+
+
+def no_content(data=None, cookies=None):
+    """Returns response with status code 204"""
+    return _make_api_response(data or DEFAULT_DATA[True], status_code=204, cookies=cookies)
+
+
+def not_modified(data=DEFAULT_DATA[True], cookies=None):
+    """Returns response with status code 304"""
+    return _make_api_response(data, status_code=304, cookies=cookies)
+
+
+def bad_request(data=DEFAULT_DATA[False], err="", cookies=None, warnings=None):
+    """Returns response with status code ies"""
+    return _make_api_response(data, err, status_code=400, cookies=cookies, warnings=warnings)
+
+
+def unauthorized(data=DEFAULT_DATA[False], err="", cookies=None):
+    """Returns response with status code 401"""
+    return _make_api_response(data, err, status_code=401, cookies=cookies)
+
+
+def forbidden(data=DEFAULT_DATA[False], err="", cookies=None):
+    """Returns response with status code 403"""
+    return _make_api_response(data, err, status_code=403, cookies=cookies)
+
+
+def not_found(data=DEFAULT_DATA[False], err="", cookies=None):
+    """Returns response with status code 404"""
+    return _make_api_response(data, err, status_code=404, cookies=cookies)
+
+
+def conflict(data=DEFAULT_DATA[False], err="", cookies=None):
+    """Returns response with status code 409"""
+    return _make_api_response(data, err, status_code=409, cookies=cookies)
+
+
+def precondition_failed(data=DEFAULT_DATA[False], err="", cookies=None):
+    """Returns response with status code 412"""
+    return _make_api_response(data, err, status_code=412, cookies=cookies)
+
+
+def teapot(data={**DEFAULT_DATA[False], "teapot": True}, err="", cookies=None):
+    """Returns response with status code 418"""
+    return _make_api_response(data, err, status_code=418, cookies=cookies)
+
+
+def too_many_requests(data=DEFAULT_DATA[False], err="", cookies=None):
+    """Returns response with status code 429"""
+    return _make_api_response(data, err, status_code=429, cookies=cookies)
+
+
+def internal_error(
+    data={**DEFAULT_DATA[False]},
+    err="Something went wrong. Contact an administrator.",
+    cookies=None,
+):
+    """Returns response with status code 500"""
+    return _make_api_response(data, err, status_code=500, cookies=cookies)
+
+
+def not_implemented(
+    data={**DEFAULT_DATA[False]},
+    err="Something went wrong. Contact an administrator.",
+    cookies=None,
+):
+    """Returns response with status code 501"""
+    return _make_api_response(data, err, status_code=501, cookies=cookies)
+
+
+def bad_gateway(
+    data={**DEFAULT_DATA[False]},
+    err="Something went wrong. Contact an administrator.",
+    cookies=None,
+):
+    """Returns response with status code 502"""
+    return _make_api_response(data, err, status_code=502, cookies=cookies)
+
+
+def service_unavailable(
+    data={**DEFAULT_DATA[False]},
+    err="Something went wrong. Contact an administrator.",
+    cookies=None,
+):
+    """Returns response with status code 503"""
+    return _make_api_response(data, err, status_code=503, cookies=cookies)
+
+
+def make_file_response(data, name, size, status_code=200, content_type="application/octet-stream"):
+    """Returns file response with arbitrary status code"""
+    quota_user = flsk_session.pop("quota_user", None)
+    quota_set = flsk_session.pop("quota_set", False)
+    if quota_user and quota_set:
+        QUOTA_TRACKER.end(quota_user)
+
+    response = make_response(data, status_code)
+    response.headers["Content-Type"] = content_type
+    response.headers["Content-Length"] = size
+    response.headers["Content-Disposition"] = 'attachment; filename="%s"' % safe_str(name)
+    return response
+
+
+def stream_file_response(reader, name, size, status_code=200):
+    """Returns stream response with arbitrary status code"""
+    quota_user = flsk_session.pop("quota_user", None)
+    quota_set = flsk_session.pop("quota_set", False)
+    if quota_user and quota_set:
+        QUOTA_TRACKER.end(quota_user)
+
+    chunk_size = 65535
+
+    def generate():
+        reader.seek(0)
+        while True:
+            data = reader.read(chunk_size)
+            if not data:
+                break
+            yield data
+        reader.close()
+
+    headers = {
+        "Content-Type": "application/octet-stream",
+        "Content-Length": size,
+        "Content-Disposition": 'attachment; filename="%s"' % safe_str(name),
+    }
+    return Response(generate(), status=status_code, headers=headers)
+
+
+def make_binary_response(data, size, status_code=200):
+    """Returns binary response with arbitrary status code"""
+    quota_user = flsk_session.pop("quota_user", None)
+    quota_set = flsk_session.pop("quota_set", False)
+    if quota_user and quota_set:
+        QUOTA_TRACKER.end(quota_user)
+
+    response = make_response(data, status_code)
+    response.headers["Content-Type"] = "application/octet-stream"
+    response.headers["Content-Length"] = size
+    return response
+
+
+def stream_binary_response(reader, status_code=200):
+    """Returns streamed binary response with arbitrary status code"""
+    quota_user = flsk_session.pop("quota_user", None)
+    quota_set = flsk_session.pop("quota_set", False)
+    if quota_user and quota_set:
+        QUOTA_TRACKER.end(quota_user)
+
+    chunk_size = 4096
+
+    def generate():
+        reader.seek(0)
+        while True:
+            data = reader.read(chunk_size)
+            if not data:
+                break
+            yield data
+
+    return Response(generate(), status=status_code, mimetype="application/octet-stream")

howler/api/base.py

@@ -0,0 +1,88 @@
+from flask import Blueprint, current_app, request
+
+from howler.api import API_PREFIX, ok
+from howler.security import api_login
+
+api = Blueprint("api", __name__, url_prefix=API_PREFIX)
+
+
+#####################################
+# API list API (API inception)
+@api.route("/")
+@api_login(audit=False, required_priv=["R", "W"])
+def api_version_list(**_):
+    """List all available API versions.
+
+    Variables:
+    None
+
+    Arguments:
+    None
+
+    Result Example:
+    ["v1", "v2", "v3"]         #List of API versions available
+    """
+    api_list = []
+    for rule in current_app.url_map.iter_rules():
+        if rule.rule.startswith("/api/"):
+            version = rule.rule[5:].split("/", 1)[0]
+            if version not in api_list and version != "":
+                try:
+                    int(version[1:])
+                except ValueError:
+                    continue
+                api_list.append(version)
+
+    return ok(api_list)
+
+
+@api.route("/site_map")
+@api_login(required_type=["admin"], audit=False)
+def site_map(**_):
+    """Check if all pages have been protected by a login decorator
+
+    Variables:
+    None
+
+    Arguments:
+    unsafe_only                    => Only show unsafe pages
+
+    Result Example:
+    [                                #List of pages dictionary containing...
+     {"function": views.default,     #Function name
+      "url": "/",                    #Url to page
+      "protected": true,             #Is function login protected
+      "required_type": false,         #List of user type allowed to view the page
+      "methods": ["GET"]},           #Methods allowed to access the page
+    ]
+    """
+    pages = []
+    for rule in current_app.url_map.iter_rules():
+        func = current_app.view_functions[rule.endpoint]
+        methods = [item for item in (rule.methods or []) if item != "OPTIONS" and item != "HEAD"]
+
+        protected = func.__dict__.get("protected", False)
+        required_type = func.__dict__.get("required_type", ["user"])
+        audit = func.__dict__.get("audit", False)
+        priv = func.__dict__.get("required_priv", "")
+        if "/api/v1/" in rule.rule:
+            prefix = "api.v1."
+        else:
+            prefix = ""
+
+        if "unsafe_only" in request.args and protected:
+            continue
+
+        pages.append(
+            {
+                "function": f"{prefix}{rule.endpoint.replace('apiv1.', '')}",
+                "url": rule.rule,
+                "methods": methods,
+                "protected": protected,
+                "required_type": required_type,
+                "audit": audit,
+                "req_priv": priv,
+            }
+        )
+
+    return ok(sorted(pages, key=lambda i: i["url"]))

howler/api/socket.py

@@ -0,0 +1,114 @@
+import base64
+import json
+import os
+from typing import Any
+
+from flask import Blueprint, request
+
+import howler.services.event_service as event_service
+from howler.api import ok, unauthorized
+from howler.common.logging import get_logger
+from howler.datastore.operations import OdmHelper
+from howler.helper.ws import ConnectionClosed, Server
+from howler.odm.models.hit import Hit
+from howler.security.socket import websocket_auth, ws_response
+from howler.utils.socket_utils import check_action
+
+HWL_INTERPOD_COMMS_SECRET = os.getenv("HWL_INTERPOD_COMMS_SECRET", "secret")
+
+socket_api = Blueprint("socket", "socket", url_prefix="/socket/v1")
+
+socket_api._doc = "Endpoints concerning websocket connectivity between the client and server"  # type: ignore
+
+logger = get_logger(__file__)
+
+hit_helper = OdmHelper(Hit)
+
+
+@socket_api.route("/emit/<event>", methods=["POST"])
+def emit(event: str):
+    """Emit an event to all listening websockets"""
+    if "Authorization" not in request.headers:
+        return unauthorized(err="Missing authorization header")
+
+    auth_data = base64.b64decode(request.headers["Authorization"].split(" ")[1]).decode().split(":")[1]
+
+    if HWL_INTERPOD_COMMS_SECRET == "secret":  # noqa: S105
+        logger.warning("Using default interpod secret! DO NOT allow this on a production instance.")
+
+    if auth_data != HWL_INTERPOD_COMMS_SECRET:
+        logger.warning("Invalid auth secret provided: %s (expected: %s)", auth_data, HWL_INTERPOD_COMMS_SECRET)
+
+        return unauthorized(err="Invalid auth data")
+
+    event_service.emit(event, request.json)
+
+    return ok()
+
+
+@socket_api.route("/connect", websocket=True)
+@websocket_auth(required_priv=["R"])
+def connect(ws: Server, *args: Any, ws_id: str, **kwargs):
+    """Connect to the server to monitor for updates via websocket
+
+    Variables:
+    None
+
+    Optional Arguments:
+    None
+
+    Result Example:
+    A continuous websocket connection
+    """
+    outstanding_actions: list[tuple[str, str, bool]] = []
+
+    def send_hit(data: dict[str, Any]):
+        logger.debug("Sending hit update: %s", data["hit"]["howler"]["id"])
+        ws.send(ws_response("hits", data))
+
+    def send_broadcast(data: dict[str, str]):
+        logger.debug("Sending broadcast: %s", data)
+        ws.send(ws_response("broadcast", {"event": data}))
+
+    def send_action(data: dict[str, str]):
+        logger.debug("Sending action: %s", data)
+        ws.send(ws_response("action", data))
+
+    try:
+        event_service.on("hits", send_hit)
+        event_service.on("broadcast", send_broadcast)
+        event_service.on("action", send_action)
+        while ws.connected:
+            data = ws.receive(10)
+            if data:
+                obj = json.loads(data)
+
+                if "id" not in obj or "action" not in obj or "broadcast" not in obj:
+                    ws.close(
+                        1008,
+                        ws_response(
+                            "error",
+                            error=True,
+                            status=400,
+                            message="Sent data is invalid.",
+                        ),
+                    )
+                    return
+
+                outstanding_actions = check_action(
+                    obj["id"], obj["action"], obj["broadcast"], outstanding_actions=outstanding_actions, **kwargs
+                )
+            else:
+                logger.debug(ws_id + " listening")
+    except Exception as e:
+        if isinstance(e, ConnectionClosed):
+            raise
+        else:
+            logger.exception("Exception on connect.")
+    finally:
+        event_service.off("hits", send_hit)
+        event_service.off("broadcast", send_broadcast)
+        event_service.off("action", send_action)
+
+        for id, action, broadcast in outstanding_actions:
+            outstanding_actions = check_action(id, action, broadcast, outstanding_actions=outstanding_actions, **kwargs)