howler-client 2.4.0.dev178__tar.gz → 2.4.0.dev209__tar.gz

{howler_client-2.4.0.dev178 → howler_client-2.4.0.dev209}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-client
-Version: 2.4.0.dev178
+Version: 2.4.0.dev209
 Summary: The Howler client library facilitates issuing requests to Howler
 License: MIT
 License-File: LICENSE
@@ -19,7 +19,6 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries
 Requires-Dist: coverage[toml] (>=7.6.1,<8.0.0)
-Requires-Dist: diff-cover (>=9.2.0,<10.0.0)
 Requires-Dist: pycryptodome (>=3.20.0,<4.0.0)
 Requires-Dist: python-baseconv (>=1.2.2,<2.0.0)
 Requires-Dist: requests[security] (>=2.32.0,<3.0.0)

{howler_client-2.4.0.dev178 → howler_client-2.4.0.dev209}/howler_client/__init__.py

@@ -25,7 +25,7 @@ def get_client(
     timeout=None,
     throw_on_bad_request=True,
     throw_on_max_retries=True,
-    token=None,
+    authenticate=None,
 ):
     "Initialize a howler client object"
     connection = Connection(
@@ -41,6 +41,6 @@ def get_client(
         timeout,
         throw_on_bad_request,
         throw_on_max_retries,
-        token,
+        authenticate,
     )
     return Client(connection)

{howler_client-2.4.0.dev178 → howler_client-2.4.0.dev209}/howler_client/common/dict_utils.py

@@ -1,5 +1,5 @@
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, AnyStr, Optional, cast
+from typing import TYPE_CHECKING, Any, AnyStr, cast
 from typing import Mapping as _Mapping
 
 if TYPE_CHECKING:
@@ -15,8 +15,8 @@ def strip_nulls(d: Any):
 
 
 def recursive_update(
-    d: Optional[dict[str, Any]],
-    u: Optional[_Mapping[str, Any]],
+    d: dict[str, Any] | None,
+    u: _Mapping[str, Any] | None,
     stop_keys: list[AnyStr] = [],
     allow_recursion: bool = True,
 ) -> dict[str, Any]:
@@ -37,11 +37,11 @@ def recursive_update(
 
 
 def get_recursive_delta(
-    d1: Optional[_Mapping[str, Any]],
-    d2: Optional[_Mapping[str, Any]],
+    d1: _Mapping[str, Any] | None,
+    d2: _Mapping[str, Any] | None,
     stop_keys: list[AnyStr] = [],
     allow_recursion: bool = True,
-) -> Optional[dict[str, Any]]:
+) -> dict[str, Any] | None:
     "Get the recursive difference between two objects"
     if d1 is None:
         return cast(dict, d2)
@@ -73,7 +73,7 @@ def get_recursive_delta(
     return out
 
 
-def flatten(data: dict, fields: list[str] = [], parent_key: Optional[str] = None) -> dict:
+def flatten(data: dict, fields: list[str] = [], parent_key: str | None = None) -> dict:
     "Flatten a nested dict"
     items: list[tuple[str, Any]] = []
 
@@ -105,7 +105,7 @@ def unflatten(data: _Mapping) -> _Mapping:
     return out
 
 
-def prune(data: _Mapping, keys: list[str], parent_key: Optional[str] = None) -> dict[str, Any]:
+def prune(data: _Mapping, keys: list[str], parent_key: str | None = None) -> dict[str, Any]:
     "Remove all keys in the given list from the dict if they exist"
     pruned_items: list[tuple[str, Any]] = []
 

{howler_client-2.4.0.dev178 → howler_client-2.4.0.dev209}/howler_client/connection.py

@@ -3,7 +3,7 @@ import json
 import sys
 import time
 import warnings
-from typing import Any, Callable, MutableMapping, Optional, Union
+from typing import Callable, MutableMapping
 
 import requests
 
@@ -36,21 +36,21 @@ class Connection(object):
     def __init__(  # pylint: disable=R0913
         self: Self,
         server: str,
-        auth: Optional[Union[str, tuple[str, str]]],
-        cert: Optional[Union[str, tuple[str, str]]],
+        auth: str | tuple[str, str] | None,
+        cert: str | tuple[str, str] | None,
         debug: Callable[[str], None],
-        headers: Optional[MutableMapping[str, Union[str, bytes]]],
+        headers: MutableMapping[str, str | bytes] | None,
         retries: int,
         silence_warnings: bool,
-        apikey: Optional[tuple[str, str]],
+        apikey: tuple[str, str] | None,
         verify: bool,
-        timeout: Optional[int],
+        timeout: int | None,
         throw_on_bad_request: bool,
         throw_on_max_retries: bool,
-        # TODO: Not sure what this argument is for (if used at all)
-        token: Optional[Any],
+        authenticate: Callable[[str | tuple[str, str] | None, tuple[str, str] | None], str] | None = None,
     ):
         self.apikey = apikey
+        self.authenticate = authenticate
         self.debug = debug
         self.max_retries = retries
         self.server = server
@@ -58,13 +58,14 @@ class Connection(object):
         self.default_timeout = timeout
         self.throw_on_bad_request = throw_on_bad_request
         self.throw_on_max_retries = throw_on_max_retries
-        self.token = token
 
         session = requests.Session()
 
         session.headers.update({"Content-Type": "application/json"})
 
-        if auth:
+        if self.authenticate:
+            session.headers.update({"Authorization": f"Bearer {self.authenticate(auth, apikey)}"})
+        elif auth:
             if not isinstance(auth, str):
                 auth = base64.b64encode(":".join(auth).encode("utf-8")).decode("utf-8")
 

{howler_client-2.4.0.dev178 → howler_client-2.4.0.dev209}/howler_client/module/bundle.py

@@ -1,5 +1,5 @@
 import sys
-from typing import TYPE_CHECKING, Any, Optional, Union
+from typing import TYPE_CHECKING, Any
 
 from howler_client.common.utils import api_path
 from howler_client.logger import get_logger
@@ -50,7 +50,7 @@ class Bundle(object):
         map: dict[str, list[str]],
         documents: list[dict[str, Any]],
         ignore_extra_values: bool = False,
-    ) -> dict[str, Union[str, list[str], None]]:
+    ) -> list[dict[str, str | list[str] | None]]:
         """Create a bundle using a format similar to the hit.create_from_map function
 
         Args:
@@ -63,7 +63,7 @@ class Bundle(object):
                     Defaults to False.
 
         Returns:
-            list[dict[str, Optional[str]]]: The list of IDs of the created hits
+            list[dict[str, str | None]]: The list of IDs of the created hits
         """
         map = {**map, "bundle": ["howler.is_bundle"]}
         bundle_hit = {**bundle_hit, "bundle": True}
@@ -74,9 +74,9 @@ class Bundle(object):
     def create(
         self: Self,
         bundle_hit: dict[str, Any],
-        data: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = [],
+        data: dict[str, Any] | list[dict[str, Any]] | None = None,
         ignore_extra_values: bool = False,
-    ) -> dict[str, Any]:
+    ) -> dict[str, Any] | None:
         """Create a bundle using a format similar to the hit.create function
 
         Args:
@@ -107,24 +107,24 @@ class Bundle(object):
 
         return self._connection.post(api_path("hit/bundle"), json={"bundle": bundle_hit, "hits": hit_ids})
 
-    def add(self: Self, bundle_id: str, hit_ids: Union[str, list[str]]):
+    def add(self: Self, bundle_id: str, hit_ids: str | list[str]):
         """Add a list of hits to a bundle by their IDs
 
         Args:
             bundle_id (str): The ID of the bundle we want to add the hits to
-            hit_ids (Union[str, list[str]]): The list of hit IDs to add to the bundle
+            hit_ids (str | list[str]): The list of hit IDs to add to the bundle
         """
         if not isinstance(hit_ids, list):
             hit_ids = [hit_ids]
 
         return self._connection.put(api_path("hit/bundle", bundle_id), json=hit_ids)
 
-    def remove(self: Self, bundle_id: str, hit_ids: Union[str, list[str]]):
+    def remove(self: Self, bundle_id: str, hit_ids: str | list[str]):
         """Remove a list of hits from a bundle by their IDs
 
         Args:
             bundle_id (str): The bundle ID from which to remove the hits
-            hit_ids (Union[str, list[str]]): A list of hit IDs to remove from the bundle
+            hit_ids (str | list[str]): A list of hit IDs to remove from the bundle
         """
         if not isinstance(hit_ids, list):
             hit_ids = [hit_ids]

{howler_client-2.4.0.dev178 → howler_client-2.4.0.dev209}/howler_client/module/hit.py

@@ -1,7 +1,8 @@
 import json
 import sys
+import warnings
 from hashlib import sha256
-from typing import TYPE_CHECKING, Any, Literal, Optional, Union
+from typing import TYPE_CHECKING, Any, Literal
 
 from howler_client.common.dict_utils import flatten
 from howler_client.common.utils import ClientError, api_path
@@ -68,7 +69,7 @@ class Hit(object):
         map: dict[str, list[str]],
         documents: list[dict[str, Any]],
         ignore_extra_values: bool = False,
-    ) -> dict[str, Union[Optional[str], list[str]]]:
+    ) -> list[dict[str, str | list[str] | None]]:
         """Create hits for a given tool using the raw documents and a map of the document fields to howler's fields.
 
         Args:
@@ -79,8 +80,17 @@ class Hit(object):
             ignore_extra_values (bool, optional): Whether to allow extra fields, or raise an error. Defaults to False.
 
         Returns:
-            dict[str, Union[Optional[str], list[str]]]: A list of IDs/Errors in the same order as the original documents
+            list[dict[str, str | list[str] | None]]: One entry per document, each with keys ``id``, ``error``,
+                and ``warn``.
+
+        .. deprecated::
+            Use the regular create() function instead, mapping the record before ingestion.
         """
+        warnings.warn(
+            "create_from_map is deprecated and will be removed in a future version.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         data = {"map": map, "hits": documents}
 
         try:
@@ -103,9 +113,9 @@ class Hit(object):
                 warn = res["warn"]
                 if isinstance(warn, list):
                     for w in warn:
-                        logger.warn(w)
+                        logger.warning(w)
                 else:
-                    logger.warn(warn)
+                    logger.warning(warn)
 
         return result
 
@@ -135,13 +145,13 @@ class Hit(object):
 
     def create(  # noqa: C901
         self: Self,
-        data: Union[dict[str, Any], list[dict[str, Any]]],
+        data: dict[str, Any] | list[dict[str, Any]],
         ignore_extra_values: bool = False,
     ):
         """Create one or many hits using the howler schema.
 
         Args:
-            data (Union[dict[str, Any], list[dict[str, Any]]]): The hit or list of hits to create
+            data (dict[str, Any] | list[dict[str, Any]]): The hit or list of hits to create
             ignore_extra_values (bool, optional): Whtether to ignore extra values, or throw an exception.
                 Defaults to False.
 
@@ -178,22 +188,20 @@ class Hit(object):
 
             final_hit_list.append(hit)
 
-        search_result = self._search.grouped.hit(
+        hashes = [hit["howler.hash"] for hit in final_hit_list]
+        existing_hashes: dict[str, int] = self._search.facet.hit(
             "howler.hash",
-            limit=1,
-            filters=[f"howler.hash:{' '.join(list_hit['howler.hash'] for list_hit in final_hit_list)}"],
-        )["items"]
-
-        for hit in final_hit_list:
-            for match in search_result:
-                if hit["howler.hash"] == match["value"]:
-                    matched_hit = match["items"][0]
-
-                    logger.warning(
-                        f"Hit with hash {hit['howler.hash']} already exists in the DB at "
-                        f"id {matched_hit['howler']['id']}, reusing"
-                    )
-                    final_hit_list.remove(hit)
+            query=f"howler.hash:({' OR '.join(hashes)})",
+            rows=len(hashes),
+        )
+
+        for hit in list(final_hit_list):
+            if hit["howler.hash"] in existing_hashes:
+                logger.warning(
+                    "Hit with hash %s already exists in the DB, reusing",
+                    hit["howler.hash"],
+                )
+                final_hit_list.remove(hit)
 
         if len(final_hit_list) < 1:
             logger.info("No hits to submit.")
@@ -212,9 +220,6 @@ class Hit(object):
         for invalid_hit in result["invalid"]:
             logger.error(invalid_hit["error"])
 
-        for entry in search_result:
-            result["valid"].append(entry["items"][0])
-
         return result
 
     def overwrite(self: Self, hit_id: str, new_hit_data: dict[str, Any]):

{howler_client-2.4.0.dev178 → howler_client-2.4.0.dev209}/howler_client/module/search/__init__.py

@@ -21,7 +21,7 @@ class Search(object):
         self.stats = Stats(connection)
         self.stream = Stream(connection, self._do_search)
 
-    def _do_search(self, index, query, use_archive=False, track_total_hits=None, **kwargs):
+    def _do_search(self, index, query, track_total_hits=None, **kwargs):
         if index not in SEARCHABLE:
             raise ClientError("Index %s is not searchable" % index, 400)
 
@@ -34,8 +34,6 @@ class Search(object):
 
         kwargs = {k: v for k, v in kwargs.items() if v is not None}
         kwargs["query"] = query
-        if use_archive:
-            kwargs["use_archive"] = ""
         if track_total_hits:
             kwargs["track_total_hits"] = track_total_hits
         path = api_path("search", index)
@@ -50,7 +48,6 @@ class Search(object):
         rows=25,
         sort=None,
         timeout=None,
-        use_archive=False,
         track_total_hits=None,
     ):
         """Search hits with a lucene query.
@@ -65,7 +62,6 @@ class Search(object):
         rows              : Number of records to return (integer)
         sort              : Field used for sorting with direction (string: ex. 'id desc')
         timeout           : Max amount of miliseconds the query will run (integer)
-        use_archive       : Also query the archive
         track_total_hits  : Number of hits to track (default: 10k)
 
         Returns all results.
@@ -79,6 +75,5 @@ class Search(object):
             rows=rows,
             sort=sort,
             timeout=timeout,
-            use_archive=use_archive,
             track_total_hits=track_total_hits,
         )

{howler_client-2.4.0.dev178 → howler_client-2.4.0.dev209}/howler_client/module/search/facet.py

@@ -31,11 +31,15 @@ class Facet(object):
         field   : field to extract the facets from
 
         Optional:
-        query    : Initial query to filter the data (default: 'id:*')
+        query    : Initial query to filter the data (default: 'howler.id:*')
         filters  : Additional lucene queries used to filter the data (list of strings)
         mincount : Minimum amount of hits for the value to be returned
         rows     : The number of different facets to return
 
         Returns all results.
         """
+        # Default the query to include all hits if none provided
+        if query is None:
+            query = "howler.id:*"
+
         return self._do_facet("hit", field, query=query, mincount=mincount, filters=filters, rows=rows)

{howler_client-2.4.0.dev178 → howler_client-2.4.0.dev209}/howler_client/module/search/grouped.py

@@ -53,6 +53,10 @@ class Grouped(object):
 
         Returns a generator that transparently and efficiently pages through results.
         """
+        # Default the query to include all hits if none provided
+        if query is None:
+            query = "howler.id:*"
+
         return self._do_grouped(
             "hit",
             field,

{howler_client-2.4.0.dev178 → howler_client-2.4.0.dev209}/howler_client/module/search/histogram.py

@@ -42,7 +42,7 @@ class Histogram(object):
         field   : field to create the histograms with (only work on date or number fields)
 
         Optional:
-        query    : Initial query to filter the data (default: 'id:*')
+        query    : Initial query to filter the data (default: 'howler.id:*')
         filters  : Additional lucene queries used to filter the data (list of strings)
         mincount : Minimum amount of hits for the value to be returned
         start    : Beginning of the histogram range (Default: now-1d or 0)

{howler_client-2.4.0.dev178 → howler_client-2.4.0.dev209}/howler_client/module/search/stats.py

@@ -31,9 +31,13 @@ class Stats(object):
         field   : field to create the stats on (only work on number fields)
 
         Optional:
-        query    : Initial query to filter the data (default: 'id:*')
+        query    : Initial query to filter the data (default: 'howler.id:*')
         filters  : Additional lucene queries used to filter the data (list of strings)
 
         Returns statistics about the field.
         """
+        # Default the query to include all hits if none provided
+        if query is None:
+            query = "howler.id:*"
+
         return self._do_stats("hit", field, query=query, filters=filters)

{howler_client-2.4.0.dev178 → howler_client-2.4.0.dev209}/howler_client/module/search/stream.py

@@ -53,7 +53,8 @@ class Stream(object):
         items: list[Any] = []
         lock = threading.Lock()
         sf_t = threading.Thread(target=self._auto_fill, args=[items, lock, index, query], kwargs=kwargs)
-        sf_t.setDaemon(True)
+        # setDaemon is deprecated; set the daemon attribute instead
+        sf_t.daemon = True
         sf_t.start()
         while not yield_done:
             try:

{howler_client-2.4.0.dev178 → howler_client-2.4.0.dev209}/pyproject.toml

@@ -91,7 +91,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-client"
-version = "2.4.0.dev178"
+version = "2.4.0.dev209"
 description = "The Howler client library facilitates issuing requests to Howler"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",
@@ -128,7 +128,6 @@ pycryptodome = "^3.20.0"
 requests = { extras = ["security"], version = ">=2.32.0,<3.0.0" }
 python-baseconv = "^1.2.2"
 coverage = { extras = ["toml"], version = "^7.6.1" }
-diff-cover = "^9.2.0"
 
 [tool.poetry.group.dev.dependencies]
 ruff = ">=0.6.8,<0.16.0"
@@ -139,7 +138,7 @@ mypy = "^1.11.2"
 pytest = ">=8.3.3,<10.0.0"
 pytest-cov = "^5.0.0"
 passlib = "^1.7.4"
-
+diff-cover = "^10.3.0"
 
 [tool.poetry.group.types.dependencies]
 types-requests = "^2.32.0.20240914"