howler-client 2.4.0.dev37__py3-none-any.whl

howler_client/__init__.py

@@ -0,0 +1,46 @@
+from importlib.metadata import PackageNotFoundError, version
+
+from howler_client.client import Client
+from howler_client.connection import Connection
+
+try:
+    __version__ = version("howler-client")
+except PackageNotFoundError:
+    __version__ = "0.0.0.unknown"
+
+RETRY_FOREVER = 0
+SUPPORTED_APIS = {"v1"}
+
+
+def get_client(
+    server,
+    auth=None,
+    cert=None,
+    debug=lambda x: None,
+    headers=None,
+    retries=RETRY_FOREVER,
+    silence_requests_warnings=True,
+    apikey=None,
+    verify=True,
+    timeout=None,
+    throw_on_bad_request=True,
+    throw_on_max_retries=True,
+    token=None,
+):
+    "Initialize a howler client object"
+    connection = Connection(
+        server,
+        auth,
+        cert,
+        debug,
+        headers,
+        retries,
+        silence_requests_warnings,
+        apikey,
+        verify,
+        timeout,
+        throw_on_bad_request,
+        throw_on_max_retries,
+        token,
+    )
+    return Client(connection)

howler_client/client.py

@@ -0,0 +1,32 @@
+import sys
+
+from howler_client.common.utils import walk_api_path
+from howler_client.connection import Connection
+from howler_client.module.bundle import Bundle
+from howler_client.module.help import Help
+from howler_client.module.hit import Hit
+from howler_client.module.search import Search
+from howler_client.module.user import User
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class Client(object):
+    "Main howler client object, wrapping API calls"
+
+    def __init__(self: Self, connection: Connection):
+        self._connection: Connection = connection
+
+        self.help = Help(self._connection)
+        self.search = Search(self._connection)
+        self.hit = Hit(self._connection, self.search)
+        self.bundle = Bundle(self._connection, self.hit)
+        self.user = User(self._connection)
+
+        paths: list[str] = []
+        walk_api_path(self, [""], paths)
+
+        self.__doc__ = "Client provides the following methods:\n\n" + "\n".join(["\n".join(p + "\n") for p in paths])

howler_client/common/dict_utils.py

@@ -0,0 +1,138 @@
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, AnyStr, Optional, cast
+from typing import Mapping as _Mapping
+
+if TYPE_CHECKING:
+    pass
+
+
+def strip_nulls(d: Any):
+    """Remove null values from a dict"""
+    if isinstance(d, dict):
+        return {k: strip_nulls(v) for k, v in d.items() if v is not None}
+    else:
+        return d
+
+
+def recursive_update(
+    d: Optional[dict[str, Any]],
+    u: Optional[_Mapping[str, Any]],
+    stop_keys: list[AnyStr] = [],
+    allow_recursion: bool = True,
+) -> dict[str, Any]:
+    "Recursively update a dict with another value"
+    if d is None:
+        return cast(dict, u or {})
+
+    if u is None:
+        return d
+
+    for k, v in u.items():
+        if isinstance(v, Mapping) and allow_recursion:
+            d[k] = recursive_update(d.get(k, {}), v, stop_keys=stop_keys, allow_recursion=k not in stop_keys)
+        else:
+            d[k] = v
+
+    return d
+
+
+def get_recursive_delta(
+    d1: Optional[_Mapping[str, Any]],
+    d2: Optional[_Mapping[str, Any]],
+    stop_keys: list[AnyStr] = [],
+    allow_recursion: bool = True,
+) -> Optional[dict[str, Any]]:
+    "Get the recursive difference between two objects"
+    if d1 is None:
+        return cast(dict, d2)
+
+    if d2 is None:
+        return cast(dict, d1)
+
+    out = {}
+    for k1, v1 in d1.items():
+        if isinstance(v1, Mapping) and allow_recursion:
+            internal = get_recursive_delta(
+                v1,
+                d2.get(k1, {}),
+                stop_keys=stop_keys,
+                allow_recursion=k1 not in stop_keys,
+            )
+            if internal:
+                out[k1] = internal
+        else:
+            if k1 in d2:
+                v2 = d2[k1]
+                if v1 != v2:
+                    out[k1] = v2
+
+    for k2, v2 in d2.items():
+        if k2 not in d1:
+            out[k2] = v2
+
+    return out
+
+
+def flatten(data: dict, fields: list[str] = [], parent_key: Optional[str] = None) -> dict:
+    "Flatten a nested dict"
+    items: list[tuple[str, Any]] = []
+
+    # We special case howler.data
+    if parent_key == "howler.data":
+        return {"howler.data": data}
+
+    for k, v in data.items():
+        cur_key = f"{parent_key}.{k}" if parent_key is not None else k
+        if isinstance(v, dict) and (len(fields) == 0 or any(k.startswith(field) for field in fields)):
+            items.extend(flatten(v, fields=fields, parent_key=cur_key).items())
+        else:
+            items.append((cur_key, v))
+
+    return dict(items)
+
+
+def unflatten(data: _Mapping) -> _Mapping:
+    "Unflatted a nested dict"
+    out: dict[str, Any] = dict()
+    for k, v in data.items():
+        parts = k.split(".")
+        d = out
+        for p in parts[:-1]:
+            if p not in d:
+                d[p] = dict()
+            d = d[p]
+        d[parts[-1]] = v
+    return out
+
+
+def prune(data: _Mapping, keys: list[str], parent_key: Optional[str] = None) -> dict[str, Any]:
+    "Remove all keys in the given list from the dict if they exist"
+    pruned_items: list[tuple[str, Any]] = []
+
+    for key, val in data.items():
+        cur_key = f"{parent_key}.{key}" if parent_key else key
+
+        if isinstance(val, dict):
+            child_keys = [_key for _key in keys if _key.startswith(cur_key)]
+
+            if len(child_keys) > 0:
+                pruned_items.append((key, prune(val, child_keys, cur_key)))
+        elif isinstance(val, list):
+            if cur_key not in keys and not any(_key.startswith(cur_key) for _key in keys):
+                continue
+
+            list_result = []
+            for entry in val:
+                if isinstance(val, dict):
+                    child_keys = [_key for _key in keys if _key.startswith(cur_key)]
+
+                    if len(child_keys) > 0:
+                        pruned_items.append((key, prune(val, child_keys, cur_key)))
+                else:
+                    list_result.append(entry)
+
+            pruned_items.append((key, list_result))
+        elif cur_key in keys:
+            pruned_items.append((key, val))
+
+    return {k: v for k, v in pruned_items}

howler_client/common/utils.py

@@ -0,0 +1,113 @@
+import re
+import sys
+from types import FrameType
+from typing import cast
+from urllib.parse import quote
+
+INVALID_STREAM_SEARCH_PARAMS = ("deep_paging_id", "rows", "sort")
+SEARCHABLE = ["hit"]
+API = "v1"
+
+
+class ClientError(Exception):
+    "Generic Exception for the howler client"
+
+    def __init__(self, message, status_code, api_response=None, api_version=None, resp_data=None):
+        super(ClientError, self).__init__(message)
+        self.api_response = api_response
+        self.api_version = api_version
+        self.status_code = status_code
+        self.resp_data = resp_data
+
+
+def _join_param(k, v):
+    val = quote(str(v))
+    if not val:
+        return k
+    return f"{k}={val}"
+
+
+def _join_kw(kw):
+    return "&".join([_join_param(k, v) for k, v in kw.items() if v is not None])
+
+
+def api_path_by_module(obj, *args, **kw):
+    """Calculate the API path using the class and method names as shown below:
+
+    /api/v1/<class_name>/<method_name>/[arg1/[arg2/[...]]][?k1=v1[...]]
+    """
+    c = obj.__class__.__name__.lower()
+    m = cast(FrameType, sys._getframe().f_back).f_code.co_name
+
+    return api_path(f"{c}/{m}", *args, **kw)
+
+
+def _param_ok(k):
+    return k not in ("q", "df", "wt")
+
+
+def api_path(prefix, *args, **kw):
+    """Calculate the API path using the prefix as shown:
+
+    /api/v1/<prefix>/[arg1/[arg2/[...]]][?k1=v1[...]]
+    """
+    path = "/".join(["api", API, prefix] + list(args))
+
+    params_tuples = kw.pop("params_tuples", [])
+    params = "&".join([_join_kw(kw)] + [_join_param(*e) for e in params_tuples if _param_ok(e)])
+    if not params:
+        return path
+
+    return f"{path}?{params}"
+
+
+def stream_output(output):
+    "Stream the output of a response"
+
+    def _do_stream(response):
+        f = output
+        if isinstance(output, str):
+            f = open(output, "wb")
+        for chunk in response.iter_content(chunk_size=1024):
+            if chunk:
+                f.write(chunk)
+        if f != output:
+            f.close()
+        return True
+
+    return _do_stream
+
+
+def walk_api_path(obj, path, paths):
+    "Walk a module and populate a provided list with the paths and their documentation"
+    if isinstance(obj, int):
+        return
+
+    for m in dir(obj):
+        mobj = getattr(obj, m)
+        if m == "__call__":
+            doc = str(mobj.__doc__)
+            if doc in ("x.__call__(...) <==> x(...)", "Call self as a function."):
+                doc = str(obj.__doc__)
+            doc = doc.split("\n\n", 1)[0]
+            doc = re.sub(r"\s+", " ", doc.strip())
+            if doc != "For internal use.":
+                paths.append(f'{".".join(path)}(): {doc}')
+
+            continue
+        elif m.startswith(("_", "im_")):
+            continue
+
+        walk_api_path(mobj, path + [m], paths)
+
+
+def to_pascal_case(snake_str):
+    "Convert a snake_case string to PascalCase"
+    components = snake_str.split("_")
+    return "".join(component.title() for component in components)
+
+
+def to_camel_case(snake_str):
+    "Convert a PascalCase string to snake_case"
+    components = snake_str.split("_")
+    return components[0] + "".join(component.title() for component in components[1:])

howler_client/connection.py

@@ -0,0 +1,204 @@
+import base64
+import json
+import sys
+import time
+import warnings
+from typing import Any, Callable, MutableMapping, Optional, Union
+
+import requests
+
+from howler_client.common.utils import ClientError
+from howler_client.logger import get_logger
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+SUPPORTED_APIS = {"v1"}
+
+logger = get_logger("connection")
+
+
+def convert_api_output(response: requests.Response):
+    "Convert a requests response to a python object based on the returned JSON data"
+    logger.debug("Converting response %s", response.text)
+
+    if response.status_code != 204:
+        return response.json()["api_response"]
+
+    return None
+
+
+class Connection(object):
+    "Abstraction for executing network requests to the Howler API"
+
+    def __init__(  # pylint: disable=R0913
+        self: Self,
+        server: str,
+        auth: Optional[Union[str, tuple[str, str]]],
+        cert: Optional[Union[str, tuple[str, str]]],
+        debug: Callable[[str], None],
+        headers: Optional[MutableMapping[str, Union[str, bytes]]],
+        retries: int,
+        silence_warnings: bool,
+        apikey: Optional[tuple[str, str]],
+        verify: bool,
+        timeout: Optional[int],
+        throw_on_bad_request: bool,
+        throw_on_max_retries: bool,
+        # TODO: Not sure what this argument is for (if used at all)
+        token: Optional[Any],
+    ):
+        self.apikey = apikey
+        self.debug = debug
+        self.max_retries = retries
+        self.server = server
+        self.silence_warnings = silence_warnings
+        self.default_timeout = timeout
+        self.throw_on_bad_request = throw_on_bad_request
+        self.throw_on_max_retries = throw_on_max_retries
+        self.token = token
+
+        session = requests.Session()
+
+        session.headers.update({"Content-Type": "application/json"})
+
+        if auth:
+            if not isinstance(auth, str):
+                auth = base64.b64encode(":".join(auth).encode("utf-8")).decode("utf-8")
+
+            if "." in auth:
+                logger.info("Using JWT Authentication")
+                session.headers.update({"Authorization": f"Bearer {auth}"})
+            else:
+                logger.info("Using Password Authentication")
+                session.headers.update({"Authorization": f"Basic {auth}"})
+        elif apikey:
+            logger.info("Using API Key Authentication")
+            session.headers.update(
+                {"Authorization": f"Basic {base64.b64encode(':'.join(apikey).encode('utf-8')).decode('utf-8')}"}
+            )
+
+        session.verify = verify
+
+        if cert:
+            session.cert = cert
+
+        if headers:
+            logger.debug("Adding additional headers")
+            session.headers.update(headers)
+
+        self.session = session
+
+        if "pytest" in sys.modules:
+            logger.info("Skipping API validation, running in a test environment")
+        else:
+            r = self.request(self.session.get, "api/", convert_api_output)
+            if not isinstance(r, list) or not set(r).intersection(SUPPORTED_APIS):
+                raise ClientError("Supported APIS (%s) are not available" % SUPPORTED_APIS, 400)
+
+    def delete(self, path, **kw):
+        "Execute a DELETE request"
+        return self.request(self.session.delete, path, convert_api_output, **kw)
+
+    def download(self, path, process, **kw):
+        "Download a file from the remote server"
+        return self.request(self.session.get, path, process, **kw)
+
+    def get(self, path, **kw):
+        "Execute a GET request"
+        return self.request(self.session.get, path, convert_api_output, **kw)
+
+    def post(self, path, **kw):
+        "Execute a POST request"
+        return self.request(self.session.post, path, convert_api_output, **kw)
+
+    def put(self, path, **kw):
+        "Execute a PUT request"
+        return self.request(self.session.put, path, convert_api_output, **kw)
+
+    def request(self, func, path, process, **kw):  # noqa: C901
+        "Main request function - prepare and execute a request"
+        self.debug(path)
+
+        # Apply default timeout parameter if not passed elsewhere
+        kw.setdefault("timeout", self.default_timeout)
+
+        retries = 0
+        with warnings.catch_warnings():
+            if self.silence_warnings:
+                warnings.simplefilter("ignore")
+            while self.max_retries < 1 or retries <= self.max_retries:
+                if retries:
+                    time.sleep(min(2, 2 ** (retries - 7)))
+
+                try:
+                    response = func(f"{self.server}/{path}", **kw)
+                    if "XSRF-TOKEN" in response.cookies:
+                        self.session.headers.update({"X-XSRF-TOKEN": response.cookies["XSRF-TOKEN"]})
+
+                    if response.text:
+                        try:
+                            _warnings = json.loads(response.text).get("api_warning", None)
+                        except json.JSONDecodeError:
+                            logger.warning(
+                                "There was an error when decoding the JSON response from the server, no warnings will "
+                                "be shown."
+                            )
+                            _warnings = None
+
+                        if _warnings:
+                            for warning in _warnings:
+                                logger.warning(warning)
+
+                    if response.ok:
+                        return process(response)
+                    elif response.status_code not in (502, 503, 504):
+                        try:
+                            resp_data = response.json()
+
+                            message = "\n".join(
+                                [item["error"] for item in resp_data["api_response"] if "error" in item]
+                            )
+
+                            err_msg = resp_data["api_error_message"]
+
+                            if message:
+                                err_msg = f"{err_msg}\n{message}"
+
+                            logger.error("%s: %s", response.status_code, err_msg)
+
+                            if response.status_code != 400 or self.throw_on_bad_request:
+                                raise ClientError(  # noqa: TRY301
+                                    err_msg,
+                                    response.status_code,
+                                    api_version=resp_data["api_server_version"],
+                                    api_response=resp_data["api_response"],
+                                    resp_data=resp_data,
+                                )
+                            else:
+                                break
+
+                        except Exception as e:
+                            if isinstance(e, ClientError):
+                                raise
+
+                            if response.status_code != 400 or self.throw_on_bad_request:
+                                raise ClientError(response.content, response.status_code) from e
+                            else:
+                                break
+                except (requests.exceptions.SSLError, requests.exceptions.ProxyError):
+                    raise
+                except requests.exceptions.ConnectionError:
+                    pass
+                except OSError as e:
+                    if "Connection aborted" not in str(e):
+                        raise
+
+                retries += 1
+
+            if self.throw_on_max_retries:
+                raise ClientError("Max retry reached, could not perform the request.", None)
+            else:
+                logger.error("Max retry reached, could not perform the request.")

howler_client/logger.py

@@ -0,0 +1,14 @@
+import logging
+
+HWL_LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s | %(message)s"
+HWL_DATE_FORMAT = "%y/%m/%d %H:%M:%S"
+
+BASE_LOGGER = logging.getLogger("howler")
+console = logging.StreamHandler()
+console.setFormatter(logging.Formatter(HWL_LOG_FORMAT, HWL_DATE_FORMAT))
+BASE_LOGGER.addHandler(console)
+
+
+def get_logger(name: str) -> logging.Logger:
+    "Create a logger instance with the given name"
+    return BASE_LOGGER.getChild(name)

howler_client/module/bundle.py

@@ -0,0 +1,132 @@
+import sys
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+from howler_client.common.utils import api_path
+from howler_client.logger import get_logger
+from howler_client.module.hit import Hit
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+if TYPE_CHECKING:
+    from howler_client import Connection
+
+logger = get_logger("bundle")
+
+
+class Bundle(object):
+    """Methods related to hit bundles"""
+
+    def __init__(self: Self, connection: "Connection", hit: Hit):
+        self._connection: Connection = connection
+        self._hit: Hit = hit
+
+    def __call__(self: Self, hit_id: str) -> dict[str, Any]:
+        """Return the bundle for a given ID.
+
+        Args:
+            hit_id (str): ID of the bundle
+
+        Raises:
+            ClientError: The hit does not exist
+            AttributeError: The hit is not a bundle
+
+        Returns:
+            Hit: The bundle in question
+        """
+        result = self._hit(hit_id)
+
+        if result["howler"]["is_bundle"]:
+            return result
+        else:
+            raise AttributeError("This hit is not a bundle! Use client.hit(...) instead.")
+
+    def create_from_map(
+        self: Self,
+        tool_name: str,
+        bundle_hit: dict[str, Any],
+        map: dict[str, list[str]],
+        documents: list[dict[str, Any]],
+        ignore_extra_values: bool = False,
+    ) -> dict[str, Union[str, list[str], None]]:
+        """Create a bundle using a format similar to the hit.create_from_map function
+
+        Args:
+            tool_name (str): Name of the tool the hits will be created for.
+            bundle_hit (Hit): The bundle hit
+            map (dict[str, list[str]]): Dictionary where the keys are the flattened path of the tool's raw document and
+                    the values are a list of flattened path for Howler's fields where the data will be copied into.
+            documents (list[Hit]): A list of hits to create as children of the bundle hit provided
+            ignore_extra_values (bool, optional): Ignore invalid values and return a warning, or throw an error.
+                    Defaults to False.
+
+        Returns:
+            list[dict[str, Optional[str]]]: The list of IDs of the created hits
+        """
+        map = {**map, "bundle": ["howler.is_bundle"]}
+        bundle_hit = {**bundle_hit, "bundle": True}
+        hit = [bundle_hit] + documents
+
+        return self._hit.create_from_map(tool_name, map, hit, ignore_extra_values=ignore_extra_values)
+
+    def create(
+        self: Self,
+        bundle_hit: dict[str, Any],
+        data: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = [],
+        ignore_extra_values: bool = False,
+    ) -> dict[str, Any]:
+        """Create a bundle using a format similar to the hit.create function
+
+        Args:
+            bundle_hit (dict[str, Any]): The bundle hit to create
+            data (Union[dict[str, Any], list[dict[str, Any]]], optional): A Hit or list of Hits to create as
+                    children of the bundle hit
+            ignore_extra_values (bool, optional): Ignore invalid values and return a warning, or throw an error.
+                    Defaults to False.
+
+        Returns:
+            Hit: The created bundle hit
+        """
+        if not data:
+            data = []
+
+        if not isinstance(data, list):
+            data = [data]
+
+        if len(data) > 0:
+            result = self._hit.create(data, ignore_extra_values=ignore_extra_values)
+
+            if not result or len(result["invalid"]) > 0:
+                return result
+
+            hit_ids = [h["howler"]["id"] for h in result["valid"]]
+        else:
+            hit_ids = []
+
+        return self._connection.post(api_path("hit/bundle"), json={"bundle": bundle_hit, "hits": hit_ids})
+
+    def add(self: Self, bundle_id: str, hit_ids: Union[str, list[str]]):
+        """Add a list of hits to a bundle by their IDs
+
+        Args:
+            bundle_id (str): The ID of the bundle we want to add the hits to
+            hit_ids (Union[str, list[str]]): The list of hit IDs to add to the bundle
+        """
+        if not isinstance(hit_ids, list):
+            hit_ids = [hit_ids]
+
+        return self._connection.put(api_path("hit/bundle", bundle_id), json=hit_ids)
+
+    def remove(self: Self, bundle_id: str, hit_ids: Union[str, list[str]]):
+        """Remove a list of hits from a bundle by their IDs
+
+        Args:
+            bundle_id (str): The bundle ID from which to remove the hits
+            hit_ids (Union[str, list[str]]): A list of hit IDs to remove from the bundle
+        """
+        if not isinstance(hit_ids, list):
+            hit_ids = [hit_ids]
+
+        return self._connection.delete(api_path("hit/bundle", bundle_id), json=hit_ids)