howler-api 4.0.0.dev982__tar.gz → 4.0.0.dev1003__tar.gz

{howler_api-4.0.0.dev982 → howler_api-4.0.0.dev1003}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 4.0.0.dev982
+Version: 4.0.0.dev1003
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-4.0.0.dev982 → howler_api-4.0.0.dev1003}/howler/api/v1/search.py

@@ -78,7 +78,6 @@ def search(index, **kwargs):
     sort                =>   How to sort the results (not available in deep paging)
     fl                  =>   List of fields to return
     timeout             =>   Maximum execution time (ms)
-    use_archive         =>   Allow access to the datastore achive (Default: False)
     track_total_hits    =>   Track the total number of query matches, instead of stopping at 10000 (Default: False)
     metadata            =>   A list of additional features to be added to the result alongside the raw results
 
@@ -118,18 +117,9 @@ def search(index, **kwargs):
         "track_total_hits",
     ]
     multi_fields = ["filters", "metadata"]
-    boolean_fields = ["use_archive"]
 
     params, req_data = generate_params(request, fields, multi_fields)
 
-    params.update(
-        {
-            k: str(req_data.get(k, "false")).lower() in ["true", ""]
-            for k in boolean_fields
-            if req_data.get(k, None) is not None
-        }
-    )
-
     if has_access_control(index):
         params.update({"access_control": user["access_control"]})
 
@@ -350,18 +340,9 @@ def sigma_search(index, **kwargs):
         "track_total_hits",
     ]
     multi_fields = ["filters"]
-    boolean_fields = ["use_archive"]
 
     params, req_data = generate_params(request, fields, multi_fields)
 
-    params.update(
-        {
-            k: str(req_data.get(k, "false")).lower() in ["true", ""]
-            for k in boolean_fields
-            if req_data.get(k, None) is not None
-        }
-    )
-
     if has_access_control(index):
         params.update({"access_control": user["access_control"]})
 
@@ -520,7 +501,6 @@ def count(index, **kwargs):
     Optional Arguments:
     filters             =>   List of additional filter queries limit the data
     timeout             =>   Maximum execution time (ms)
-    use_archive         =>   Allow access to the datastore achive (Default: False)
 
     Data Block:
     # Note that the data block is for POST requests only!
@@ -543,15 +523,6 @@ def count(index, **kwargs):
 
     params, req_data = generate_params(request, [], [])
 
-    boolean_fields = ["use_archive"]
-    params.update(
-        {
-            k: str(req_data.get(k, "false")).lower() in ["true", ""]
-            for k in boolean_fields
-            if req_data.get(k, None) is not None
-        }
-    )
-
     if has_access_control(index):
         params.update({"access_control": user["access_control"]})
 

{howler_api-4.0.0.dev982 → howler_api-4.0.0.dev1003}/howler/api/v1/tool.py

@@ -59,6 +59,9 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
             {'id': None, 'error': "Error message"},
         ]
     }
+
+    .. deprecated::
+        Use POST /api/v1/hit/ directly with pre-mapped hit data instead.
     """
     data = request.json
     if not isinstance(data, dict):
@@ -74,7 +77,10 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
 
     if not isinstance(hits, list):
         return bad_request(err="Invalid: 'hits' field is missing or invalid.")
-    warnings = []
+    warnings = [
+        "This endpoint is deprecated and will be removed in a future version. "
+        "Use POST /api/v1/hit/ directly with pre-mapped hit data instead."
+    ]
     # Validate field_map targets
     hit_fields = Hit.flat_fields()
     for targets in field_map.values():
@@ -157,27 +163,48 @@ def create_one_or_many_hits(tool_name: str, user: User, **kwargs):  # noqa: C901
             logger.warning(e)
 
             out.append({"id": None, "error": str(e)})
+
+    # Deduplicate by hash: skip hits whose hash already exists in the datastore
+    if odms:
+        hashes = [odm.howler.hash for odm in odms]
+        existing_hashes: dict[str, int] = datastore().hit.facet(
+            "howler.hash",
+            query=f"howler.hash:({' OR '.join(hashes)})",
+            rows=len(hashes),
+        )
+
+        deduplicated_odms = []
+        for odm in odms:
+            if odm.howler.hash in existing_hashes:
+                logger.warning("Hit with hash %s already exists in the DB, skipping", odm.howler.hash)
+                warnings.append(f"Hit with hash {odm.howler.hash} already exists in the DB and was skipped.")
+                out[:] = [entry for entry in out if entry["id"] != odm.howler.id]
+            else:
+                deduplicated_odms.append(odm)
+
+        odms = deduplicated_odms
+
     # If there are any errors...
     if any([obj["error"] for obj in out]):
         return bad_request(out, warnings=warnings, err="No valid hits were provided")
-    else:
-        for odm in odms:
-            if bundle_hit is not None:
-                bundle_hit.howler.hits.append(odm.howler.id)
-                bundle_hit.howler.bundle_size += 1
-                odm.howler.bundles.append(bundle_hit.howler.id)
 
-            hit_service.create_hit(odm.howler.id, odm, user=user.uname)
+    for odm in odms:
+        if bundle_hit is not None:
+            bundle_hit.howler.hits.append(odm.howler.id)
+            bundle_hit.howler.bundle_size += 1
+            odm.howler.bundles.append(bundle_hit.howler.id)
+
+        hit_service.create_hit(odm.howler.id, odm, user=user.uname)
 
-            analytic_service.save_from_hit(odm, user)
+        analytic_service.save_from_hit(odm, user)
 
-        if bundle_hit:
-            hit_service.create_hit(bundle_hit.howler.id, bundle_hit, user=user.uname)
+    if bundle_hit:
+        hit_service.create_hit(bundle_hit.howler.id, bundle_hit, user=user.uname)
 
-            analytic_service.save_from_hit(bundle_hit, user)
+        analytic_service.save_from_hit(bundle_hit, user)
 
-        datastore().hit.commit()
+    datastore().hit.commit()
 
-        action_service.enqueue_action_execution([entry["id"] for entry in out], trigger="create", user=user)
+    action_service.enqueue_action_execution([entry["id"] for entry in out], trigger="create", user=user)
 
-        return created(out, warnings=warnings)
+    return created(out, warnings=warnings)

howler_api-4.0.0.dev1003/howler/common/net.py

@@ -0,0 +1,128 @@
+import socket
+import uuid
+from ipaddress import IPv4Network, ip_address
+from typing import Union
+
+from howler.common.net_static import TLDS_ALPHA_BY_DOMAIN
+
+# IANA Special-Use Domain Names Registry
+# https://www.iana.org/assignments/special-use-domain-names/special-use-domain-names.xhtml
+IANA_SPECIAL_USE_TLDS = {
+    "LOCALHOST",  # RFC 6761 - loopback address
+    "TEST",  # RFC 6761 - testing
+    "EXAMPLE",  # RFC 6761 - documentation examples
+    "INVALID",  # RFC 6761 - invalid domain names
+    "LOCAL",  # RFC 6762 - mDNS/Bonjour local network names
+    "ONION",  # RFC 7686 - Tor hidden services
+}
+
+# Common private network suffixes used in enterprise environments
+# These are NOT IANA-registered but are widely used organizational conventions
+PRIVATE_NETWORK_SUFFIXES = {
+    "INTERNAL",
+    "LAN",
+    "HOME",
+    "CORP",
+    "LOCALDOMAIN",
+}
+
+
+def is_valid_port(value: Union[int, str, float]) -> bool:
+    "Check if a port is valid"
+    try:
+        if 1 <= int(value) <= 65535:
+            return True
+    except ValueError:
+        pass
+
+    return False
+
+
+def is_valid_domain(
+    domain: str,
+    allow_special_use_tlds: bool = True,
+    allow_private_suffixes: bool = True,
+) -> bool:
+    """Check if a domain is valid.
+
+    Args:
+        domain: The domain to validate.
+        allow_special_use_tlds: If True, accepts IANA special-use TLDs
+            (.localhost, .local, .onion, .test, .example, .invalid).
+            Set to False to reject these domains.
+            Defaults to True for backward compatibility.
+        allow_private_suffixes: If True, accepts common private network suffixes
+            (.internal, .lan, .home, .corp, .localdomain).
+            Set to False to reject these domains.
+            Defaults to True for backward compatibility.
+
+    Note:
+        To validate public internet domains only (e.g., for SSRF protection),
+        set both allow_special_use_tlds=False and allow_private_suffixes=False.
+
+    Returns:
+        True if the domain has a valid TLD, False otherwise.
+    """
+    if "@" in domain:
+        return False
+
+    if "." in domain:
+        tld = domain.split(".")[-1].upper()
+        if tld in TLDS_ALPHA_BY_DOMAIN:
+            return True
+        if allow_special_use_tlds and tld in IANA_SPECIAL_USE_TLDS:
+            return True
+        if allow_private_suffixes and tld in PRIVATE_NETWORK_SUFFIXES:
+            return True
+
+    return False
+
+
+def is_valid_ip(ip: str) -> bool:
+    "Check if an ip is valid"
+    parts = ip.split(".")
+    if len(parts) == 4:
+        for p in parts:
+            try:
+                if not (0 <= int(p) <= 255):
+                    return False
+            except ValueError:
+                return False
+
+        if int(parts[0]) == 0:
+            return False
+
+        if int(parts[3]) == 0:
+            return False
+
+        return True
+
+    return False
+
+
+def is_ip_in_network(ip: str, network: IPv4Network) -> bool:
+    "Check if an ip is in a given network"
+    if not is_valid_ip(ip):
+        return False
+
+    return ip_address(ip) in network
+
+
+def is_valid_email(email: str) -> bool:
+    "Check if an email is valid"
+    parts = email.split("@")
+    if len(parts) == 2:
+        if is_valid_domain(parts[1]):
+            return True
+
+    return False
+
+
+def get_hostname() -> str:
+    "Get the hostname of the computer howler is running on"
+    return socket.gethostname()
+
+
+def get_mac_address() -> str:
+    "Get the mac address of the computer howler is running on"
+    return "".join(["{0:02x}".format((uuid.getnode() >> i) & 0xFF) for i in range(0, 8 * 6, 8)][::-1]).upper()