howler-api 4.0.0.dev847__tar.gz → 4.0.0.dev871__tar.gz

{howler_api-4.0.0.dev847 → howler_api-4.0.0.dev871}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.0.0.dev847
+Version: 4.0.0.dev871
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

howler_api-4.0.0.dev871/howler/api/v1/utils/etag.py

@@ -0,0 +1,106 @@
+"""ETag utility module for handling HTTP ETags in Flask responses.
+
+ETags (Entity Tags) are HTTP headers used for web cache validation and conditional requests.
+They help optimize performance by allowing clients to cache responses and only fetch
+new data when the resource has actually changed.
+"""
+
+import functools
+import re
+
+from flask import Response, request
+
+from howler.api import not_modified
+
+
+def add_etag(getter=None, check_if_match=True):
+    """Decorator to add ETag handling to a Flask response.
+
+    This decorator implements HTTP ETag functionality for API endpoints, enabling:
+    - Conditional requests using If-Match headers
+    - Cache validation to prevent unnecessary data transfers
+    - Version tracking for resources
+
+    When ``getter`` is provided, the decorator pre-fetches the object and its
+    version, injects ``server_version`` into kwargs, caches the object, and
+    supports ``If-Match`` conditional requests.
+
+    When ``getter`` is ``None``, the decorator only handles converting
+    ``(Response, version)`` return tuples into a single Response with the
+    ``ETag`` header set.  This is useful for endpoints that manage their own
+    version retrieval (e.g. v2 endpoints).
+
+    Args:
+        getter: Optional function that retrieves the object and its version.
+                When None, the decorator only handles ETag header setting.
+        check_if_match (bool): Whether to check If-Match headers for conditional requests.
+                               Only used when getter is provided.
+
+    Returns:
+        Decorated function with ETag support
+    """
+
+    def wrapper(f):
+        """Inner wrapper function that applies ETag functionality to the decorated function."""
+
+        @functools.wraps(f)
+        def generate_etag(*args, **kwargs):
+            """Generate and handle ETags for the HTTP response."""
+            if getter is not None:
+                # Retrieve the object and its version using the provided getter function
+                # The getter should return (object, version) tuple
+                obj, version = getter(
+                    kwargs.get("id", kwargs.get("username", None)),
+                    as_odm=True,
+                    version=True,
+                )
+
+                # Handle conditional requests with If-Match header
+                # If the client's version matches the current version and it's a GET request
+                # without metadata parameter, return 304 Not Modified to save bandwidth
+                if (
+                    check_if_match
+                    and "If-Match" in request.headers
+                    and request.headers["If-Match"] == version
+                    and request.method == "GET"
+                    and "metadata" not in request.args
+                ):
+                    return not_modified()
+
+                # Extract the resource type from the API path and create a cache key
+                # e.g., "/api/v1/users/123" becomes "cached_users"
+                key = re.sub(r"^\/api\/v\d+\/(\w+)\/.+$", r"cached_\1", request.path)
+                kwargs[key] = obj
+
+                # Call the original function with the cached object and version
+                values = f(*args, server_version=version, **kwargs)
+
+                # Handle different return value formats from the decorated function
+                # If there is only one return, it's just the response
+                if isinstance(values, Response):
+                    # Only add ETag header for successful responses (not 409 Conflict or 400 Bad Request)
+                    if values.status_code != 409 and values.status_code != 400:
+                        values.headers["ETag"] = version
+                    return values
+
+                # If there are two returns, it's the response and the new version
+                # This happens when the function modifies the resource and returns an updated version
+                else:
+                    if values[0].status_code != 409 and values[0].status_code != 400:
+                        # Add the new ETag version to successful responses
+                        values[0].headers["ETag"] = values[1]
+                    return values[0]
+
+            # No getter: just call the function and handle (Response, version) tuples
+            values = f(*args, **kwargs)
+
+            if isinstance(values, Response):
+                return values
+
+            if values[0].status_code != 409 and values[0].status_code != 400:
+                values[0].headers["ETag"] = values[1]
+            return values[0]
+
+        return generate_etag
+
+    return wrapper

{howler_api-4.0.0.dev847 → howler_api-4.0.0.dev871}/howler/api/v2/ingest.py

@@ -5,6 +5,7 @@ from flask import request
 from mergedeep import Strategy, merge
 
 from howler.api import bad_request, created, forbidden, internal_error, make_subapi_blueprint, no_content, not_found, ok
+from howler.api.v1.utils.etag import add_etag
 from howler.common.exceptions import HowlerException, HowlerValueError
 from howler.common.loader import datastore
 from howler.common.logging import get_logger
@@ -162,8 +163,23 @@ def delete(indexes: str, user: User, **kwargs):
     if non_existing_hit_ids := [id for id in ids if all(not ds[index].exists(id) for index in index_list)]:
         return not_found(err=f"Record ids [{','.join(non_existing_hit_ids)}] do not exist.")
 
-    # TODO: Reimplement in a generic function
-    # hit_service.delete_hits(hit_ids, indexes=index_list)
+    try:
+        remaining = set(ids)
+        for index in index_list:
+            if not remaining:
+                break
+
+            existing = [record_id for record_id in remaining if ds[index].exists(record_id)]
+            if not existing:
+                continue
+
+            for record_id in existing:
+                ds[index].delete(record_id)
+
+            remaining -= set(existing)
+            ds[index].commit()
+    except DataStoreException as e:
+        return internal_error(err=str(e))
 
     return no_content()
 
@@ -239,7 +255,8 @@ def validate(index: str, **kwargs):
 @generate_swagger_docs()
 @ingest_api.route("/<index>/<id>/overwrite", methods=["PATCH"])
 @api_login(audit=False, required_priv=["W"])
-def overwrite(index: str, id: str, server_version: str, **kwargs):
+@add_etag()
+def overwrite(index: str, id: str, **kwargs):
     """Overwrite a record.
 
     Variables:
@@ -286,7 +303,7 @@ def overwrite(index: str, id: str, server_version: str, **kwargs):
             ),
         )
 
-        ds[index].save(id, odm(new_record) if odm else new_record, version=server_version)
+        ds[index].save(id, odm(new_record) if odm else new_record, version=kwargs.get("server_version"))
 
         new_record, new_version = ds[index].get(id, as_obj=False, version=True)
 

{howler_api-4.0.0.dev847 → howler_api-4.0.0.dev871}/howler/api/v2/search.py

@@ -5,7 +5,6 @@ from typing import Any
 from elasticsearch import BadRequestError
 from elasticsearch._sync.client.indices import IndicesClient
 from flask import Request, request
-from werkzeug.exceptions import BadRequest
 
 from howler.api import bad_request, make_subapi_blueprint, ok
 from howler.common.loader import datastore
@@ -30,10 +29,7 @@ def generate_params(request: Request, fields: list[str], multi_fields: list[str]
         params = {}
 
     if request.method == "POST":
-        try:
-            req_data = request.json
-        except BadRequest:
-            req_data = {"query": "*:*"}
+        req_data = request.get_json(silent=True) or {"query": "*:*"}
 
         params = {
             **params,
@@ -131,7 +127,8 @@ def search(indexes: str, **kwargs):
         return bad_request(err="There was no search query.")
 
     metadata = params.pop("metadata", [])
-    result = search_service.search(indexes, query, **params)
+    access_control = params.pop("access_control", None)
+    result = search_service.search(indexes, query, access_control=access_control, **params)
 
     if metadata and any(idx in index_list for idx in ["hit"]):
         hit_service.augment_metadata(result["items"], metadata, user)
@@ -236,7 +233,7 @@ def count(index, **kwargs):
 
     Result Example:
     {
-        "total": 201,                          # Total results found
+        "count": 201,                          # Total results found
     }
     """
     user = kwargs["user"]
@@ -245,7 +242,7 @@ def count(index, **kwargs):
     if collection is None:
         return bad_request(err=f"Not a valid index to search in: {index}")
 
-    params, req_data = generate_params(request, [], [])
+    params, req_data = generate_params(request, ["timeout"], ["filters"])
 
     boolean_fields = ["use_archive"]
     params.update(
@@ -256,15 +253,15 @@ def count(index, **kwargs):
         }
     )
 
-    if has_access_control(index):
-        params.update({"access_control": user["access_control"]})
+    access_control = user["access_control"] if has_access_control(index) else None
 
     query = req_data.get("query", None)
     if not query:
         return bad_request(err="There was no search query.")
 
+    filters = params.pop("filters", [])
     try:
-        return ok(collection().count(query, **params))
+        return ok(collection().count(query, filters, access_control=access_control))
     except (SearchException, BadRequestError) as e:
         return bad_request(err=f"SearchException: {e}")
 

{howler_api-4.0.0.dev847 → howler_api-4.0.0.dev871}/howler/datastore/collection.py

@@ -1922,7 +1922,7 @@ class ESCollection(Generic[ModelType]):
         search result object that consists of the following:
 
             {
-                "total": 123456,  # Total number of documents matching the query
+                "count": 123456,  # Total number of documents matching the query
             }
 
         :param query: lucene query to search for

{howler_api-4.0.0.dev847 → howler_api-4.0.0.dev871}/howler/odm/base.py

@@ -1081,10 +1081,10 @@ class Model:
         Args:
             skip_mappings (bool): Skip over mappings where the real subfield names are unknown.
         """
-        if skip_mappings and hasattr(cls, "_odm_field_cache_skip"):
+        if not no_cache and skip_mappings and "_odm_field_cache_skip" in cls.__dict__:
             return cls._odm_field_cache_skip
 
-        if not skip_mappings and hasattr(cls, "_odm_field_cache"):
+        if not no_cache and not skip_mappings and "_odm_field_cache" in cls.__dict__:
             return cls._odm_field_cache
 
         out = dict()
@@ -1113,10 +1113,10 @@ class Model:
     def add_namespace(cls, namespace: str, field: _Field, index=None, store=None, description=None):
         recursive_set_name(field, namespace)
 
-        if hasattr(cls, "_odm_field_cache_skip"):
+        if "_odm_field_cache_skip" in cls.__dict__:
             cls._odm_field_cache_skip[namespace.rstrip("_")] = field
 
-        if hasattr(cls, "_odm_field_cache"):
+        if "_odm_field_cache" in cls.__dict__:
             cls._odm_field_cache[namespace.rstrip("_")] = field
 
         setattr(cls, namespace, field)
@@ -1131,10 +1131,10 @@ class Model:
 
     @classmethod
     def remove_namespace(cls, namespace: str):
-        if hasattr(cls, "_odm_field_cache_skip"):
+        if "_odm_field_cache_skip" in cls.__dict__:
             del cls._odm_field_cache_skip[namespace.rstrip("_")]
 
-        if hasattr(cls, "_odm_field_cache"):
+        if "_odm_field_cache" in cls.__dict__:
             del cls._odm_field_cache[namespace.rstrip("_")]
 
         delattr(cls, namespace)
@@ -1212,7 +1212,7 @@ class Model:
         include_autogen_note=True,
         defaults=None,
         url_prefix="/howler/odm/class/",
-    ) -> dict | str:
+    ) -> str:
         markdown_content = (
             (
                 '??? success "Auto-Generated Documentation"\n    '
@@ -1524,12 +1524,23 @@ def model(index=None, store=None, description=None, id_field=None):
 
     def _finish_model(cls):
         cls._Model__description = description
-        cls._Model__id_field = id_field
+        fields = cls.fields()
 
-        if cls._Model__id_field is None:
+        if id_field is None:
             cls._Model__id_field = f"{cls.__name__.lower()}_id"
+        else:
+            if not isinstance(id_field, str):
+                raise HowlerTypeError(f"id_field must be a str, got {type(id_field).__name__}")
+
+            if not FLATTENED_OBJECT_SANITIZER.match(id_field) or id_field in BANNED_FIELDS:
+                raise HowlerValueError(f"Illegal id_field name: {id_field}")
+
+            if id_field not in fields and id_field not in cls.flat_fields():
+                raise HowlerValueError(f"id_field must reference a declared field: {id_field}")
+
+            cls._Model__id_field = id_field
 
-        for name, field_data in cls.fields().items():
+        for name, field_data in fields.items():
             if not FIELD_SANITIZER.match(name) or name in BANNED_FIELDS:
                 raise HowlerValueError(f"Illegal variable name: {name}")
 

{howler_api-4.0.0.dev847 → howler_api-4.0.0.dev871}/howler/services/search_service.py

@@ -99,6 +99,7 @@ def search(  # noqa: C901
     timeout: int | None = None,
     track_total_hits: bool = False,
     metadata: list[str] | None = None,
+    access_control: str | None = None,
 ) -> SearchResult[dict[str, Any]]:
     """Search through specified index for a given query. Uses lucene search syntax for query.
 
@@ -136,6 +137,9 @@ def search(  # noqa: C901
     else:
         parsed_filters = filters
 
+    if access_control:
+        parsed_filters.append(access_control)
+
     if query is None:
         query = "id:*"
 

{howler_api-4.0.0.dev847 → howler_api-4.0.0.dev871}/howler/utils/compat.py

@@ -14,6 +14,12 @@ else:
     class StrEnum(str, _Enum):  # type: ignore[no-redef]
         """str + Enum backport for Python < 3.11."""
 
+        def __str__(self) -> str:
+            return str.__str__(self)
+
+        def __format__(self, format_spec: str) -> str:
+            return str.__format__(self, format_spec)
+
     # typing_extensions.TypedDict supports Generic[T] mixing on Python < 3.11;
     # the stdlib version does not gain that until 3.11.
     from typing_extensions import NotRequired, TypedDict  # noqa: F401

{howler_api-4.0.0.dev847 → howler_api-4.0.0.dev871}/pyproject.toml

@@ -152,7 +152,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "4.0.0.dev847"
+version = "4.0.0.dev871"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",

howler_api-4.0.0.dev847/howler/api/v1/utils/etag.py

@@ -1,84 +0,0 @@
-"""ETag utility module for handling HTTP ETags in Flask responses.
-
-ETags (Entity Tags) are HTTP headers used for web cache validation and conditional requests.
-They help optimize performance by allowing clients to cache responses and only fetch
-new data when the resource has actually changed.
-"""
-
-import functools
-import re
-
-from flask import Response, request
-
-from howler.api import not_modified
-
-
-def add_etag(getter, check_if_match=True):
-    """Decorator to add ETag handling to a Flask response.
-
-    This decorator implements HTTP ETag functionality for API endpoints, enabling:
-    - Conditional requests using If-Match headers
-    - Cache validation to prevent unnecessary data transfers
-    - Version tracking for resources
-
-    Args:
-        getter: Function that retrieves the object and its version
-        check_if_match (bool): Whether to check If-Match headers for conditional requests
-
-    Returns:
-        Decorated function with ETag support
-    """
-
-    def wrapper(f):
-        """Inner wrapper function that applies ETag functionality to the decorated function."""
-
-        @functools.wraps(f)
-        def generate_etag(*args, **kwargs):
-            """Generate and handle ETags for the HTTP response."""
-            # Retrieve the object and its version using the provided getter function
-            # The getter should return (object, version) tuple
-            obj, version = getter(
-                kwargs.get("id", kwargs.get("username", None)),
-                as_odm=True,
-                version=True,
-            )
-
-            # Handle conditional requests with If-Match header
-            # If the client's version matches the current version and it's a GET request
-            # without metadata parameter, return 304 Not Modified to save bandwidth
-            if (
-                check_if_match
-                and "If-Match" in request.headers
-                and request.headers["If-Match"] == version
-                and request.method == "GET"
-                and "metadata" not in request.args
-            ):
-                return not_modified()
-
-            # Extract the resource type from the API path and create a cache key
-            # e.g., "/api/v1/users/123" becomes "cached_users"
-            key = re.sub(r"^\/api\/v\d+\/(\w+)\/.+$", r"cached_\1", request.path)
-            kwargs[key] = obj
-
-            # Call the original function with the cached object and version
-            values = f(*args, server_version=version, **kwargs)
-
-            # Handle different return value formats from the decorated function
-            # If there is only one return, it's just the response
-            if isinstance(values, Response):
-                # Only add ETag header for successful responses (not 409 Conflict or 400 Bad Request)
-                if values.status_code != 409 and values.status_code != 400:
-                    values.headers["ETag"] = version
-                return values
-
-            # If there are two returns, it's the response and the new version
-            # This happens when the function modifies the resource and returns an updated version
-            else:
-                if values[0].status_code != 409 and values[0].status_code != 400:
-                    # Add the new ETag version to successful responses
-                    values[0].headers["ETag"] = values[1]
-                return values[0]
-
-        return generate_etag
-
-    return wrapper