howler-client 2.4.0.dev37__py3-none-any.whl

howler_client/module/comment.py

@@ -0,0 +1,59 @@
+import sys
+from typing import TYPE_CHECKING, Any, List
+
+from howler_client.common.utils import api_path
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+if TYPE_CHECKING:
+    from howler_client import Connection
+
+
+class Comment(object):
+    """Help related endpoints"""
+
+    def __init__(self: Self, connection: "Connection"):
+        self._connection = connection
+
+    def add(self: Self, hit_id: str, comment: str) -> dict[str, Any]:
+        """Add a comment to a hit and return it
+
+        Args:
+            hit_id (str): ID of the hit
+            comment: content of the comment
+
+        Returns:
+            dict[str, Any]: The corresponding hit data
+        """
+        return self._connection.post(api_path("hit", hit_id, "comments"), json={"value": comment})
+
+    def edit(self: Self, hit_id: str, comment: str, comment_id: str) -> dict[str, Any]:
+        """Update a comment on a hit and return it
+
+        Args:
+            hit_id (str): ID of the hit
+            comment_id (str): ID of the comment that need to be updated
+            comment: content of the comment
+
+        Returns:
+            dict[str, Any]: The corresponding hit data
+        """
+        return self._connection.put(
+            api_path("hit", hit_id, "comments", comment_id),
+            json={"value": comment},
+        )
+
+    def delete(self: Self, hit_id: str, comment_ids: List[str]) -> dict[str, Any]:
+        """Delete a comment on a hit and return it
+
+        Args:
+            hit_id (str): ID of the hit
+            comment_ids (List[str]): list of all comment ids that need to be removed
+
+        Returns:
+            dict[str, Any]: The corresponding hit data
+        """
+        return self._connection.delete(api_path("hit", hit_id, "comments"), json=comment_ids)

howler_client/module/help.py

@@ -0,0 +1,23 @@
+import sys
+from typing import TYPE_CHECKING
+
+from howler_client.common.utils import api_path
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+if TYPE_CHECKING:
+    from howler_client import Connection
+
+
+class Help(object):
+    """Help related endpoints"""
+
+    def __init__(self: Self, connection: "Connection"):
+        self._connection = connection
+
+    def classification_definition(self):
+        """Return the current system classification definition"""
+        return self._connection.get(api_path("help", "classification_definition"))

howler_client/module/hit.py

@@ -0,0 +1,299 @@
+import json
+import sys
+from hashlib import sha256
+from typing import TYPE_CHECKING, Any, Literal, Optional, Union
+
+from howler_client.common.dict_utils import flatten
+from howler_client.common.utils import ClientError, api_path
+from howler_client.logger import get_logger
+from howler_client.module.comment import Comment
+from howler_client.utils.json_encoders import BytesDatetimeEncoder, DatetimeEncoder
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+if TYPE_CHECKING:
+    from howler_client import Connection
+    from howler_client.module.search import Search
+
+logger = get_logger("hit")
+
+UPDATE_SET = "SET"
+UPDATE_INC = "INC"
+UPDATE_DEC = "DEC"
+UPDATE_MAX = "MAX"
+UPDATE_MIN = "MIN"
+UPDATE_APPEND = "APPEND"
+UPDATE_APPEND_IF_MISSING = "APPEND_IF_MISSING"
+UPDATE_REMOVE = "REMOVE"
+UPDATE_DELETE = "DELETE"
+
+UPDATE_OPERATIONS = [
+    UPDATE_APPEND,
+    UPDATE_APPEND_IF_MISSING,
+    UPDATE_DEC,
+    UPDATE_INC,
+    UPDATE_MAX,
+    UPDATE_MIN,
+    UPDATE_REMOVE,
+    UPDATE_SET,
+    UPDATE_DELETE,
+]
+
+
+class Hit(object):
+    "Operations pertaining to ingesting and interacting with Howler hits."
+
+    def __init__(self: Self, connection: "Connection", search: "Search"):
+        self._connection: "Connection" = connection
+        self._search: "Search" = search
+        self.comment: "Comment" = Comment(connection)
+
+    def __call__(self: Self, hit_id: str) -> dict[str, Any]:
+        """Return the hit for a given ID
+
+        Args:
+            hit_id (str): ID of the hit
+
+        Returns:
+            dict[str, Any]: The corresponding hit data
+        """
+        return self._connection.get(api_path("hit", hit_id))
+
+    def create_from_map(
+        self: Self,
+        tool_name: str,
+        map: dict[str, list[str]],
+        documents: list[dict[str, Any]],
+        ignore_extra_values: bool = False,
+    ) -> dict[str, Union[Optional[str], list[str]]]:
+        """Create hits for a given tool using the raw documents and a map of the document fields to howler's fields.
+
+        Args:
+            tool_name (str): Name of the tool the hits will be created for
+            map (dict[str, list[str]]): Dictionary where the keys are the flattened path of the tool's raw document and
+                    the values are a list of flattened path for Howler's fields where the data will be copied into
+            documents (list[dict[str, Any]]): The data to ingest into howler, in the tool's raw document format
+            ignore_extra_values (bool, optional): Whether to allow extra fields, or raise an error. Defaults to False.
+
+        Returns:
+            dict[str, Union[Optional[str], list[str]]]: A list of IDs/Errors in the same order as the original documents
+        """
+        data = {"map": map, "hits": documents}
+
+        try:
+            result = self._connection.post(
+                api_path("tools", tool_name, "hits", ignore_extra_values=ignore_extra_values),
+                json=data,
+            )
+        except ClientError as e:
+            if e.api_response and isinstance(e.api_response, list):
+                for res in e.api_response:
+                    if "warn" in res and res["warn"]:
+                        logger.warning(res["warn"])
+
+                    if "error" in res and res["error"]:
+                        logger.error(res["error"])  # noqa: TRY400
+            raise
+
+        for res in result:
+            if "warn" in res and res["warn"]:
+                warn = res["warn"]
+                if isinstance(warn, list):
+                    for w in warn:
+                        logger.warn(w)
+                else:
+                    logger.warn(warn)
+
+        return result
+
+    def generate_hash(self: Self, hit: dict[str, Any]) -> str:
+        """Generate hash value for hit using the analytic, detection, and raw_data values from the hit data.
+
+        Args:
+            hit (str): hit data
+
+        Returns:
+            str: A hash value for the hit
+        """
+        howler_data = hit.get("howler.data", [])
+
+        if not isinstance(howler_data, list):
+            howler_data = [howler_data]
+
+        hash_contents = {
+            "analytic": hit["howler.analytic"],
+            "detection": hit.get("howler.detection", "no_detection"),
+            "raw_data": sorted(
+                json.dumps(entry, sort_keys=True, ensure_ascii=True, cls=BytesDatetimeEncoder) for entry in howler_data
+            ),
+        }
+
+        return sha256(json.dumps(hash_contents, sort_keys=True, ensure_ascii=True).encode("utf-8")).hexdigest()
+
+    def create(  # noqa: C901
+        self: Self,
+        data: Union[dict[str, Any], list[dict[str, Any]]],
+        ignore_extra_values: bool = False,
+    ):
+        """Create one or many hits using the howler schema.
+
+        Args:
+            data (Union[dict[str, Any], list[dict[str, Any]]]): The hit or list of hits to create
+            ignore_extra_values (bool, optional): Whtether to ignore extra values, or throw an exception.
+                Defaults to False.
+
+        Returns:
+            dict[str, list[dict[str, Any]]]: A list of valid and invalid hits
+        """
+        if not isinstance(data, list):
+            data = [data]
+
+        final_hit_list = []
+        for hit in data:
+            hit = flatten(hit, fields=["howler"])
+
+            existing_hash = hit.get("howler.hash", None)
+            if existing_hash is None:
+                existing_hash = self.generate_hash(hit)
+
+            hit["howler.hash"] = existing_hash
+
+            if "howler.data" in hit:
+                howler_data = hit["howler.data"]
+                if not isinstance(howler_data, list):
+                    howler_data = [howler_data]
+
+                hit["howler.data"] = [
+                    json.dumps(
+                        entry,
+                        sort_keys=True,
+                        ensure_ascii=True,
+                        cls=BytesDatetimeEncoder,
+                    )
+                    for entry in howler_data
+                ]
+
+            final_hit_list.append(hit)
+
+        search_result = self._search.grouped.hit(
+            "howler.hash",
+            limit=1,
+            filters=[f"howler.hash:{' '.join(list_hit['howler.hash'] for list_hit in final_hit_list)}"],
+        )["items"]
+
+        for hit in final_hit_list:
+            for match in search_result:
+                if hit["howler.hash"] == match["value"]:
+                    matched_hit = match["items"][0]
+
+                    logger.warning(
+                        f"Hit with hash {hit['howler.hash']} already exists in the DB at "
+                        f"id {matched_hit['howler']['id']}, reusing"
+                    )
+                    final_hit_list.remove(hit)
+
+        if len(final_hit_list) < 1:
+            logger.info("No hits to submit.")
+            return None
+
+        result = self._connection.post(
+            api_path("hit", ignore_extra_values=ignore_extra_values),
+            data=json.dumps(final_hit_list, cls=DatetimeEncoder),
+            headers={"Content-Type": "application/json"},
+        )
+
+        if not result:
+            logger.warning("No result was returned.")
+            return result
+
+        for invalid_hit in result["invalid"]:
+            logger.error(invalid_hit["error"])
+
+        for entry in search_result:
+            result["valid"].append(entry["items"][0])
+
+        return result
+
+    def overwrite(self: Self, hit_id: str, new_hit_data: dict[str, Any]):
+        """Overwrite a hit.
+
+        This is different from updating a hit, as you simply provide a partial hit object
+
+        Args:
+            hit_id (str): Id of the hit you would like to overwrite
+            new_hit_data (dict[str, Any]): A partial hit data object to overwrite the specified hit with
+
+        Raises:
+            ClientError: Updates provided were invalid
+        """
+        if not isinstance(new_hit_data, dict):
+            raise TypeError("New hit data must be of type dict.")
+
+        return self._connection.put(api_path(f"hit/{hit_id}/overwrite"), json=new_hit_data)
+
+    def update(self: Self, hit_id: str, updates: list[tuple[str, str, Any]]):
+        """Update a hit.
+
+        Args:
+            hit_id (str): Id of the hit you would like to update
+            updates (list[tuple[str, str, Any]]): A list of updates to run. The first entry in the tuple must be a valid
+                update operation (see UPDATE_OPERATIONS), the second a key for a howler hit, and the third the value
+                to use in the operation.
+
+        Raises:
+            ClientError: Updates provided were invalid
+        """
+        if not isinstance(updates, list):
+            raise TypeError("Updates must be of type list.")
+
+        for update in updates:
+            if not isinstance(update, tuple):
+                raise TypeError("Entries in updates must be of type tuple.")
+
+            if update[0] not in UPDATE_OPERATIONS:
+                raise ClientError(
+                    f"Invalid update - operation must be one of {','.join(UPDATE_OPERATIONS)}!",
+                    400,
+                )
+
+        return self._connection.put(api_path(f"hit/{hit_id}/update"), json=updates)
+
+    def update_by_query(self: Self, query: str, updates: list[tuple[str, str, Any]]):
+        """Update a set of hits by query.
+
+        Args:
+            query (str): Query representing the hits you would like to update
+            updates (list[tuple[str, str, Any]]): A list of updates to run. The first entry in the tuple must be a valid
+                update operation (see UPDATE_OPERATIONS), the second a key for a howler hit, and the third the value
+                to use in the operation.
+
+        Raises:
+            ClientError: Updates provided were invalid
+        """
+        if not isinstance(updates, list):
+            raise TypeError("Updates must be of type list.")
+
+        for update in updates:
+            if not isinstance(update, tuple):
+                raise TypeError("Entries in updates must be of type tuple.")
+            if update[0] not in UPDATE_OPERATIONS:
+                raise ClientError(
+                    f"Invalid update - operation must be one of {','.join(UPDATE_OPERATIONS)}!",
+                    400,
+                )
+
+        return self._connection.put(api_path("hit/update"), json={"query": query, "operations": updates})
+
+    def delete(self: Self, hit_ids: list[str]) -> dict[Literal["success"], bool]:
+        """Delete a list of hits by id
+
+        Returns:
+            dict[Literal["success"], bool]: Whether the delete operation was successful
+        """
+        if not isinstance(hit_ids, list):
+            hit_ids = [hit_ids]
+
+        return self._connection.delete(api_path("hit"), json=hit_ids)

howler_client/module/search/__init__.py

@@ -0,0 +1,84 @@
+import json
+
+from howler_client.common.utils import SEARCHABLE, ClientError, api_path
+from howler_client.module.search.facet import Facet
+from howler_client.module.search.fields import Fields
+from howler_client.module.search.grouped import Grouped
+from howler_client.module.search.histogram import Histogram
+from howler_client.module.search.stats import Stats
+from howler_client.module.search.stream import Stream
+
+
+class Search(object):
+    "Module dedicated to searching collections and performing various other operations like group by or faceting"
+
+    def __init__(self, connection):
+        self._connection = connection
+        self.facet = Facet(connection)
+        self.fields = Fields(connection)
+        self.grouped = Grouped(connection)
+        self.histogram = Histogram(connection)
+        self.stats = Stats(connection)
+        self.stream = Stream(connection, self._do_search)
+
+    def _do_search(self, index, query, use_archive=False, track_total_hits=None, **kwargs):
+        if index not in SEARCHABLE:
+            raise ClientError("Index %s is not searchable" % index, 400)
+
+        filters = kwargs.pop("filters", None)
+        if filters is not None:
+            if isinstance(filters, str):
+                filters = [filters]
+
+            kwargs["filters"] = filters
+
+        kwargs = {k: v for k, v in kwargs.items() if v is not None}
+        kwargs["query"] = query
+        if use_archive:
+            kwargs["use_archive"] = ""
+        if track_total_hits:
+            kwargs["track_total_hits"] = track_total_hits
+        path = api_path("search", index)
+        return self._connection.post(path, data=json.dumps(kwargs))
+
+    def hit(
+        self,
+        query,
+        filters=None,
+        fl=None,
+        offset=0,
+        rows=25,
+        sort=None,
+        timeout=None,
+        use_archive=False,
+        track_total_hits=None,
+    ):
+        """Search hits with a lucene query.
+
+        Required:
+        query   : lucene query (string)
+
+        Optional:
+        filters           : Additional lucene queries used to filter the data (list of strings)
+        fl                : List of fields to return (comma separated string of fields)
+        offset            : Offset at which the query items should start (integer)
+        rows              : Number of records to return (integer)
+        sort              : Field used for sorting with direction (string: ex. 'id desc')
+        timeout           : Max amount of miliseconds the query will run (integer)
+        use_archive       : Also query the archive
+        track_total_hits  : Number of hits to track (default: 10k)
+
+        Returns all results.
+        """
+        return self._do_search(
+            "hit",
+            query,
+            filters=filters,
+            fl=fl,
+            offset=offset,
+            rows=rows,
+            sort=sort,
+            timeout=timeout,
+            use_archive=use_archive,
+            track_total_hits=track_total_hits,
+        )

howler_client/module/search/chunk.py

@@ -0,0 +1,38 @@
+"""Sequence manipulation methods used in parsing raw datastore output."""
+
+from __future__ import annotations
+
+from typing import Generator, Sequence, TypeVar, overload
+
+_T = TypeVar("_T")
+
+
+@overload
+def chunk(items: bytes, n: int) -> Generator[bytes, None, None]: ...
+
+
+@overload
+def chunk(items: str, n: int) -> Generator[str, None, None]: ...
+
+
+@overload
+def chunk(items: Sequence[_T], n: int) -> Generator[Sequence[_T], None, None]: ...
+
+
+def chunk(items, n: int):
+    """Yield n-sized chunks from list.
+
+    >>> list(chunk([1,2,3,4,5,6,7], 2))
+    [[1,2], [3,4], [5,6], [7,]]
+    """
+    for i in range(0, len(items), n):
+        yield items[i : i + n]
+
+
+def chunked_list(items: Sequence[_T], n: int) -> list[Sequence[_T]]:
+    """Create a list of n-sized chunks from list.
+
+    >>> chunked_list([1,2,3,4,5,6,7], 2)
+    [[1,2], [3,4], [5,6], [7,]]
+    """
+    return list(chunk(items, n))

howler_client/module/search/facet.py

@@ -0,0 +1,41 @@
+from howler_client.common.utils import SEARCHABLE, ClientError, api_path
+
+
+class Facet(object):
+    "List most frequent values for a field in the given collection"
+
+    def __init__(self, connection):
+        self._connection = connection
+
+    def _do_facet(self, index, field, **kwargs):
+        if index not in SEARCHABLE:
+            raise ClientError("Index %s is not searchable" % index, 400)
+
+        filters = kwargs.pop("filters", None)
+        if filters is not None:
+            if isinstance(filters, str):
+                filters = [filters]
+
+            filters = [("filters", fq) for fq in filters]
+
+        kwargs = {k: v for k, v in kwargs.items() if v is not None and k != "filters"}
+        if filters is not None:
+            kwargs["params_tuples"] = filters
+        path = api_path("search", "facet", index, field, **kwargs)
+        return self._connection.get(path)
+
+    def hit(self, field, query=None, mincount=None, filters=None, rows=None):
+        """List most frequent value for a field in the hit collection.
+
+        Required:
+        field   : field to extract the facets from
+
+        Optional:
+        query    : Initial query to filter the data (default: 'id:*')
+        filters  : Additional lucene queries used to filter the data (list of strings)
+        mincount : Minimum amount of hits for the value to be returned
+        rows     : The number of different facets to return
+
+        Returns all results.
+        """
+        return self._do_facet("hit", field, query=query, mincount=mincount, filters=filters, rows=rows)

howler_client/module/search/fields.py

@@ -0,0 +1,19 @@
+from howler_client.common.utils import SEARCHABLE, ClientError, api_path
+
+
+class Fields(object):
+    "List the fields of given indexes"
+
+    def __init__(self, connection):
+        self._connection = connection
+
+    def _do_fields(self, index):
+        if index not in SEARCHABLE:
+            raise ClientError("Index %s is not searchable" % index, 400)
+
+        path = api_path("search", "fields", index)
+        return self._connection.get(path)
+
+    def hit(self):
+        """List all fields details for the hit collection."""
+        return self._do_fields("hit")

howler_client/module/search/grouped.py

@@ -0,0 +1,67 @@
+from howler_client.common.utils import SEARCHABLE, ClientError, api_path
+
+
+class Grouped(object):
+    "Module for grouping search results from given indexes"
+
+    def __init__(self, connection):
+        self._connection = connection
+
+    def _do_grouped(self, index, field, **kwargs):
+        if index not in SEARCHABLE:
+            raise ClientError("Index %s is not searchable" % index, 400)
+
+        filters = kwargs.pop("filters", None)
+        if filters is not None:
+            if isinstance(filters, str):
+                filters = [filters]
+
+            filters = [("filters", fq) for fq in filters]
+
+        kwargs = {k: v for k, v in kwargs.items() if v is not None and k != "filters"}
+        if filters is not None:
+            kwargs["params_tuples"] = filters
+        path = api_path("search", "grouped", index, field, **kwargs)
+        return self._connection.get(path)
+
+    def hit(
+        self,
+        field,
+        group_sort=None,
+        limit=None,
+        query=None,
+        filters=None,
+        offset=None,
+        rows=None,
+        sort=None,
+        fl=None,
+    ):
+        """Search hit collection and group result to a given field
+
+        Required:
+        field   : Field used to group the results
+
+        Optional:
+        group_sort : Field used for sorting items in the groups with direction (string: ex. 'id desc')
+        limit      : Maximum number of items returned per group (integer)
+        query      : lucene query (string)
+        filters    : Additional lucene queries used to filter the data (list of strings)
+        offset     : Offset at which the query items should start (integer)
+        rows       : Number of records to return (integer)
+        sort       : Field used for sorting with direction (string: ex. 'id desc')
+        fl         : List of fields to return (comma separated string of fields)
+
+        Returns a generator that transparently and efficiently pages through results.
+        """
+        return self._do_grouped(
+            "hit",
+            field,
+            group_sort=group_sort,
+            limit=limit,
+            query=query,
+            filters=filters,
+            offset=offset,
+            rows=rows,
+            sort=sort,
+            fl=fl,
+        )