howler-api 2.13.0.dev345__tar.gz → 3.0.0.dev348__tar.gz

{howler_api-2.13.0.dev345 → howler_api-3.0.0.dev348}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: howler-api
-Version: 2.13.0.dev345
+Version: 3.0.0.dev348
 Summary: Howler - API server
 License: MIT
 Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs

{howler_api-2.13.0.dev345 → howler_api-3.0.0.dev348}/howler/datastore/collection.py

@@ -207,6 +207,7 @@ class ESCollection(Generic[ModelType]):
         "script_fields": [],
     }
     IGNORE_ENSURE_COLLECTION = False
+    ENSURE_COLLECTION_WARNED = False
 
     def __init__(self, datastore: ESStore, name, model_class=None, validate=True, max_attempts=10):
         self.replicas = int(
@@ -227,9 +228,10 @@ class ESCollection(Generic[ModelType]):
 
         if not ESCollection.IGNORE_ENSURE_COLLECTION:
             self._ensure_collection()
-        elif "pytest" not in sys.modules:
+        elif "pytest" not in sys.modules and not ESCollection.ENSURE_COLLECTION_WARNED:
             logger.warning("Skipping ensure collection! This is dangerous. Waiting five seconds before continuing.")
             time.sleep(5)
+            ESCollection.ENSURE_COLLECTION_WARNED = True
 
         self.stored_fields = {}
         if model_class:

{howler_api-2.13.0.dev345 → howler_api-3.0.0.dev348}/howler/datastore/constants.py

@@ -21,6 +21,7 @@ from howler.odm import (
     Integer,
     Json,
     Keyword,
+    Long,
     LowerKeyword,
     PhoneNumber,
     Platform,
@@ -37,6 +38,7 @@ BASE_TYPE_MAPPING = {
     Keyword: "keyword",
     Boolean: "boolean",
     Integer: "integer",
+    Long: "long",
     Float: "float",
     Date: "date",
     Text: "text",

{howler_api-2.13.0.dev345 → howler_api-3.0.0.dev348}/howler/datastore/support/build.py

@@ -15,6 +15,7 @@ from howler.odm import (
     Json,
     Keyword,
     List,
+    Long,
     Mapping,
     Optional,
     Text,
@@ -57,7 +58,7 @@ def build_mapping(field_data, prefix=None, allow_refuse_implicit=True):
                     }
                 )
 
-        elif isinstance(field, (Boolean, Integer, Float, Text)):
+        elif isinstance(field, (Boolean, Integer, Float, Text, Long)):
             mappings[name.strip(".")] = set_mapping(field, {"type": TYPE_MAPPING[field.__class__.__name__]})
 
         elif field.__class__ in ANALYZER_MAPPING:

howler_api-3.0.0.dev348/howler/external/reindex_data.py

@@ -0,0 +1,66 @@
+import sys
+import time
+from typing import Callable
+
+DELAY = 5
+
+
+if __name__ == "__main__":
+    print("This script will allow you to reindex all indexes in elasticsearch.")
+    print("For obvious reasons, be EXTREMELY CAREFUL running this code.")
+
+    for i in range(DELAY):
+        print(f"Continuing in {str(DELAY - i)}...", end="\r")
+        time.sleep(1)
+    print()
+    answer = input("Are you sure you want to reindex data for an index in this cluster? [yes/NO]\n")
+
+    if not answer.startswith("y"):
+        print("Confirmation not provided, stopping.")
+        sys.exit(1)
+
+    from howler.datastore.collection import ESCollection
+
+    ESCollection.IGNORE_ENSURE_COLLECTION = True
+
+    from howler.common import loader
+
+    ds = loader.datastore(archive_access=False)
+
+    indexes: dict[str, tuple[list[str], Callable]] = {
+        "analytic": (ds.analytic.index_list_full, ds.analytic.reindex),
+        "hit": (ds.hit.index_list_full, ds.hit.reindex),
+        "view": (ds.view.index_list_full, ds.view.reindex),
+        "template": (ds.template.index_list_full, ds.template.reindex),
+        "overview": (ds.overview.index_list_full, ds.overview.reindex),
+        "action": (ds.action.index_list_full, ds.action.reindex),
+        "user": (ds.user.index_list_full, ds.user.reindex),
+        "dossier": (ds.dossier.index_list_full, ds.dossier.reindex),
+    }
+
+    print("Which index will you reindex?")
+    index_answer = input(", ".join(indexes.keys()) + "\n> ")
+
+    if index_answer not in indexes:
+        print("Invalid index provided, stopping.")
+        sys.exit(1)
+
+    print("You will be reindexing the following indexes:")
+    print("\n".join(indexes[index_answer][0]))
+
+    answer = input(("\nAre you sure you want to reindex these indexes? [yes/NO]\n"))
+    print()
+
+    if not answer.startswith("y"):
+        print("Confirmation not provided, stopping.")
+        sys.exit(1)
+
+    for i in range(2 * DELAY):
+        print(f"Reindexing in {2 * DELAY - i}...", end="\r")
+        time.sleep(1)
+
+    print()
+
+    result = indexes[index_answer][1]()
+
+    print(f"Reindex complete. Success: {result}.")

{howler_api-2.13.0.dev345 → howler_api-3.0.0.dev348}/howler/odm/base.py

@@ -643,6 +643,27 @@ class Integer(_Field):
             raise HowlerValueError(f"[{'.'.join(context)}]: {str(e)}")
 
 
+class Long(_Field):
+    """
+    A field storing a long value. Equivalent to Integer in python, but sets the ES datatype to long.
+
+    In Elasticsearch, Integer supports values from -2^31 to 2^31-1, while Long supports values from -2^63 to 2^63-1.
+    """
+
+    def check(self, value, context=[], **kwargs):
+        if self.optional and value is None:
+            return None
+
+        if value is None or value == "":
+            if self.default_set:
+                return self.default
+
+        try:
+            return int(value)
+        except ValueError as e:
+            raise HowlerValueError(f"[{'.'.join(context)}]: {str(e)}")
+
+
 class Float(_Field):
     """A field storing a floating point value."""
 

{howler_api-2.13.0.dev345 → howler_api-3.0.0.dev348}/howler/odm/models/analytic.py

@@ -62,7 +62,7 @@ class Analytic(odm.Model):
         default=[],
         description="A list of useful notebooks for the analytic",
     )
-    name: str = odm.Keyword(description="The name of the analytic.")
+    name: str = odm.CaseInsensitiveKeyword(description="The name of the analytic.")
     owner: Optional[str] = odm.Keyword(description="The username of the user who owns this analytic.", optional=True)
     contributors: list[str] = odm.List(
         odm.Keyword(),
@@ -71,7 +71,7 @@ class Analytic(odm.Model):
     )
     description: Optional[str] = odm.Text(description="A markdown description of the analytic", optional=True)
     detections: list[str] = odm.List(
-        odm.Keyword(),
+        odm.CaseInsensitiveKeyword(),
         description="The detections which this analytic contains.",
         default=[],
     )

{howler_api-2.13.0.dev345 → howler_api-3.0.0.dev348}/howler/odm/models/ecs/client.py

@@ -117,7 +117,7 @@ class Client(odm.Model):
             description="Collection of connected Internal Protocol routing prefixes",
         )
     )
-    bytes: Optional[int] = odm.Optional(odm.Integer(description="Bytes sent from the client to the server."))
+    bytes: Optional[int] = odm.Optional(odm.Long(description="Bytes sent from the client to the server."))
     domain: Optional[str] = odm.Optional(odm.Keyword(description="The domain name of the client system."))
     geo: Geo = odm.Optional(
         odm.Compound(

{howler_api-2.13.0.dev345 → howler_api-3.0.0.dev348}/howler/odm/models/howler_data.py

@@ -161,10 +161,10 @@ class Log(odm.Model):
 
 @odm.model(index=True, store=True, description="Hit outline header.")
 class Header(odm.Model):
-    threat: Optional[str] = odm.Optional(odm.Text(description="The IP of the threat."))
-    target: Optional[str] = odm.Optional(odm.Text(description="The target of the hit."))
-    indicators: list[str] = odm.List(odm.Text(description="Indicators of the hit."), default=[])
-    summary: Optional[str] = odm.Optional(odm.Text(description="Summary of the hit."))
+    threat: Optional[str] = odm.Optional(odm.Keyword(description="The IP of the threat."))
+    target: Optional[str] = odm.Optional(odm.Keyword(description="The target of the hit."))
+    indicators: list[str] = odm.List(odm.Keyword(description="Indicators of the hit."), default=[])
+    summary: Optional[str] = odm.Optional(odm.Keyword(description="Summary of the hit."))
 
 
 @odm.model(index=True, store=True, description="Fields describing the location where this alert has been retained.")
@@ -176,15 +176,15 @@ class Incident(odm.Model):
 
 @odm.model(index=True, store=True, description="Labels for the hit")
 class Label(odm.Model):
-    assignments = odm.List(odm.Text(description="List of assignments for the hit."), default=[])
-    generic = odm.List(odm.Text(description="List of generic labels for the hit."), default=[])
-    insight = odm.List(odm.Text(description="List of insight labels for the hit."), default=[])
-    mitigation = odm.List(odm.Text(description="List of mitigation labels for the hit."), default=[])
-    victim = odm.List(odm.Text(description="List of victim labels for the hit."), default=[])
-    campaign = odm.List(odm.Text(description="List of campaign labels for the hit."), default=[])
-    threat = odm.List(odm.Text(description="List of threat labels for the hit."), default=[])
-    tuning = odm.List(odm.Text(description="List of tuning labels for the hit."), default=[])
-    operation = odm.List(odm.Text(description="List of operation labels for the hit."), default=[])
+    assignments = odm.List(odm.Keyword(description="List of assignments for the hit."), default=[])
+    generic = odm.List(odm.Keyword(description="List of generic labels for the hit."), default=[])
+    insight = odm.List(odm.Keyword(description="List of insight labels for the hit."), default=[])
+    mitigation = odm.List(odm.Keyword(description="List of mitigation labels for the hit."), default=[])
+    victim = odm.List(odm.Keyword(description="List of victim labels for the hit."), default=[])
+    campaign = odm.List(odm.Keyword(description="List of campaign labels for the hit."), default=[])
+    threat = odm.List(odm.Keyword(description="List of threat labels for the hit."), default=[])
+    tuning = odm.List(odm.Keyword(description="List of tuning labels for the hit."), default=[])
+    operation = odm.List(odm.Keyword(description="List of operation labels for the hit."), default=[])
 
 
 @odm.model(index=True, store=True, description="Votes for the hit")

{howler_api-2.13.0.dev345 → howler_api-3.0.0.dev348}/howler/odm/models/view.py

@@ -14,7 +14,7 @@ class Settings(odm.Model):
 @odm.model(index=True, store=True, description="Model of views")
 class View(odm.Model):
     view_id: str = odm.UUID(description="A UUID for this view")
-    title: str = odm.Keyword(description="The name of this view.")
+    title: str = odm.CaseInsensitiveKeyword(description="The name of this view.")
     query: str = odm.Keyword(description="The query to run in this view.")
     sort: str = odm.Keyword(description="The sorting to use with this view.", optional=True)
     span: str = odm.Keyword(

{howler_api-2.13.0.dev345 → howler_api-3.0.0.dev348}/howler/odm/randomizer.py

@@ -32,6 +32,7 @@ from howler.odm import (
     Json,
     Keyword,
     List,
+    Long,
     LowerKeyword,
     Mapping,
     Model,
@@ -490,6 +491,8 @@ def random_data_for_field(field: _Field, name: str, minimal: bool = False) -> _A
         return get_random_iso_date()
     elif isinstance(field, Integer):
         return random.randint(128, 4096)
+    elif isinstance(field, Long):
+        return random.randint(1, 223372036854775807)
     elif isinstance(field, Float):
         return random.randint(12800, 409600) / 100.0
     elif isinstance(field, MD5):

{howler_api-2.13.0.dev345 → howler_api-3.0.0.dev348}/howler/utils/lucene.py

@@ -20,7 +20,7 @@ def try_parse_date(date: str) -> Optional[datetime]:
         return None
 
 
-def try_parse_number(number: str | int | float) -> Optional[Union[int, float]]:
+def try_parse_number(number: Union[str, int, float]) -> Optional[Union[int, float]]:
     "Try and parse a number string into an integer or float type, or infinity. Returns None if string is invalid."
     if isinstance(number, (int, float)):
         return number

{howler_api-2.13.0.dev345 → howler_api-3.0.0.dev348}/pyproject.toml

@@ -148,7 +148,7 @@ suppress-none-returning = true
 [tool.poetry]
 package-mode = true
 name = "howler-api"
-version = "2.13.0.dev345"
+version = "3.0.0.dev348"
 description = "Howler - API server"
 authors = [
     "Canadian Centre for Cyber Security <howler@cyber.gc.ca>",

howler_api-2.13.0.dev345/howler/external/reindex_data.py

@@ -1,46 +0,0 @@
-import sys
-import time
-
-DELAY = 5
-
-if __name__ == "__main__":
-    print("This script will allow you to reindex all indexes in elasticsearch.")
-    print("For obvious reasons, be EXTREMELY CAREFUL running this code.")
-
-    for i in range(DELAY):
-        print(f"Continuing in {str(DELAY - i)}...", end="\r")
-        time.sleep(1)
-    print()
-    answer = input("Are you sure you want to reindex all data in this cluster? [yes/NO]\n")
-
-    if not answer.startswith("y"):
-        print("Confirmation not provided, stopping.")
-        sys.exit(1)
-
-    from howler.datastore.collection import ESCollection
-
-    ESCollection.IGNORE_ENSURE_COLLECTION = True
-
-    from howler.common import loader
-
-    ds = loader.datastore(archive_access=False)
-
-    print("You will be reindexing the following indexes:")
-    print("\n".join(ds.hit.index_list_full))
-
-    answer = input(("\nAre you sure you want to reindex all indexes? [yes/NO]\n"))
-    print()
-
-    if not answer.startswith("y"):
-        print("Confirmation not provided, stopping.")
-        sys.exit(1)
-
-    for i in range(2 * DELAY):
-        print(f"Reindexing in {2 * DELAY - i}...", end="\r")
-        time.sleep(1)
-
-    print()
-
-    result = ds.hit.reindex()
-
-    print(f"Reindex complete. Success: {result}.")