howler-api 2.10.0.dev255__py3-none-any.whl → 2.13.0.dev344__py3-none-any.whl

howler/odm/helper.py

@@ -143,8 +143,9 @@ def generate_useful_hit(lookups: dict[str, dict[str, Any]], users: list[User], p
     hit.howler.labels.mitigation = []
     hit.howler.labels.operation = []
     hit.howler.labels.threat = []
+    hit.howler.labels.tuning = []
 
-    label_type = ceil(rand_seed * 6)
+    label_type = ceil(rand_seed * 7)
     if label_type == 1:
         hit.howler.labels.campaign = ["Bad event 2023-07"]
     elif label_type == 2:
@@ -155,6 +156,8 @@ def generate_useful_hit(lookups: dict[str, dict[str, Any]], users: list[User], p
         hit.howler.labels.mitigation = ["Blocked: google.com"]
     elif label_type == 5:
         hit.howler.labels.operation = ["OP_HOWLER"]
+    elif label_type == 6:
+        hit.howler.labels.tuning = ["Tune example"]
     else:
         hit.howler.labels.threat = ["Bad Mojo"]
 
@@ -256,13 +259,13 @@ def generate_useful_hit(lookups: dict[str, dict[str, Any]], users: list[User], p
         ),
     ]
 
-    if config.core.borealis.enabled:
+    if config.core.clue.enabled:
         hit.howler.dossier.append(
             Lead(
                 {
                     "icon": "material-symbols:image",
-                    "label": {"en": "Borealis", "fr": "Borealis"},
-                    "format": "borealis",
+                    "label": {"en": "Clue", "fr": "Clue"},
+                    "format": "clue",
                     "content": "test-plugin.image",
                     "metadata": {"type": "ip", "value": "127.0.01", "classification": "TLP:CLEAR"},
                 }
@@ -273,8 +276,8 @@ def generate_useful_hit(lookups: dict[str, dict[str, Any]], users: list[User], p
             Lead(
                 {
                     "icon": "material-symbols:code-rounded",
-                    "label": {"en": "Borealis", "fr": "Borealis"},
-                    "format": "borealis",
+                    "label": {"en": "Clue", "fr": "Clue"},
+                    "format": "clue",
                     "content": "test-plugin.json",
                     "metadata": {"type": "ip", "value": "127.0.01", "classification": "TLP:CLEAR"},
                 }

howler/odm/models/config.py

@@ -20,11 +20,24 @@ logger.addHandler(console)
 
 
 class RedisServer(BaseModel):
+    """Configuration for a single Redis server instance.
+
+    Defines the connection parameters for a Redis server, including
+    the hostname and port number.
+    """
+
     host: str = Field(description="Hostname of Redis instance")
     port: int = Field(description="Port of Redis instance")
 
 
 class Redis(BaseModel):
+    """Redis configuration for Howler.
+
+    Defines connections to both persistent and non-persistent Redis instances.
+    The non-persistent instance is used for volatile data like caches, while
+    the persistent instance is used for data that needs to survive restarts.
+    """
+
     nonpersistent: RedisServer = Field(
         default=RedisServer(host="127.0.0.1", port=6379), description="A volatile Redis instance"
     )
@@ -35,6 +48,14 @@ class Redis(BaseModel):
 
 
 class Host(BaseModel):
+    """Configuration for a remote host connection.
+
+    Defines connection parameters for external services, including authentication
+    credentials (username/password or API key) and connection details.
+    Environment variables can override username and password using the pattern
+    {NAME}_HOST_USERNAME and {NAME}_HOST_PASSWORD.
+    """
+
     name: str = Field(description="Name of the host")
     username: Optional[str] = Field(description="Username to login with", default=None)
     password: Optional[str] = Field(description="Password to login with", default=None)
@@ -64,6 +85,12 @@ class Host(BaseModel):
 
 
 class Datastore(BaseModel):
+    """Datastore configuration for Howler.
+
+    Defines the backend datastore used by Howler for storing hits and metadata.
+    Currently supports Elasticsearch as the datastore type.
+    """
+
     hosts: list[Host] = Field(
         default=[Host(name="elastic", username="elastic", password="devpass", scheme="http", host="localhost:9200")],  # noqa: S106
         description="List of hosts used for the datastore",
@@ -74,6 +101,13 @@ class Datastore(BaseModel):
 
 
 class Logging(BaseModel):
+    """Logging configuration for Howler.
+
+    Defines how and where Howler logs should be output, including console,
+    file, and syslog destinations. Also controls log level, format, and
+    metric export intervals.
+    """
+
     log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "DISABLED"] = Field(
         default="INFO",
         description="What level of logging should we have?",
@@ -94,6 +128,12 @@ class Logging(BaseModel):
 
 
 class PasswordRequirement(BaseModel):
+    """Password complexity requirements for internal authentication.
+
+    Defines the rules for password creation and validation, including
+    character type requirements and minimum length.
+    """
+
     lower: bool = Field(default=False, description="Password must contain lowercase letters")
     number: bool = Field(default=False, description="Password must contain numbers")
     special: bool = Field(default=False, description="Password must contain special characters")
@@ -102,6 +142,13 @@ class PasswordRequirement(BaseModel):
 
 
 class Internal(BaseModel):
+    """Internal authentication configuration.
+
+    Defines settings for Howler's built-in username/password authentication,
+    including password requirements and brute-force protection via login
+    failure tracking.
+    """
+
     enabled: bool = Field(default=True, description="Internal authentication allowed?")
     failure_ttl: int = Field(
         default=60, description="How long to wait after `max_failures` before re-attempting login?"
@@ -111,6 +158,13 @@ class Internal(BaseModel):
 
 
 class OAuthAutoProperty(BaseModel):
+    """Automatic property assignment based on OAuth attributes.
+
+    Defines rules for automatically assigning user properties (roles,
+    classifications, or access levels) based on pattern matching against
+    OAuth provider data.
+    """
+
     field: str = Field(description="Field to apply `pattern` to")
     pattern: str = Field(description="Regex pattern for auto-prop assignment")
     type: Literal["access", "classification", "role"] = Field(
@@ -120,6 +174,13 @@ class OAuthAutoProperty(BaseModel):
 
 
 class OAuthProvider(BaseModel):
+    """OAuth provider configuration.
+
+    Defines the connection and authentication settings for an OAuth 2.0 provider.
+    Includes user auto-creation, group mapping, JWT validation, and various
+    OAuth endpoints required for the authentication flow.
+    """
+
     auto_create: bool = Field(default=True, description="Auto-create users if they are missing")
     auto_sync: bool = Field(default=False, description="Should we automatically sync with OAuth provider?")
     auto_properties: list[OAuthAutoProperty] = Field(
@@ -191,6 +252,13 @@ class OAuthProvider(BaseModel):
 
 
 class OAuth(BaseModel):
+    """OAuth authentication configuration.
+
+    Top-level OAuth settings including enabling/disabling OAuth authentication,
+    Gravatar integration, and a dictionary of configured OAuth providers.
+    Also controls API key lifetime restrictions for OAuth-authenticated users.
+    """
+
     enabled: bool = Field(default=False, description="Enable use of OAuth?")
     gravatar_enabled: bool = Field(default=True, description="Enable gravatar?")
     providers: dict[str, OAuthProvider] = Field(
@@ -204,6 +272,13 @@ class OAuth(BaseModel):
 
 
 class Auth(BaseModel):
+    """Authentication configuration for Howler.
+
+    Configures all authentication methods supported by Howler, including
+    internal username/password authentication and OAuth providers. Also
+    controls API key settings and restrictions.
+    """
+
     allow_apikeys: bool = Field(default=True, description="Allow API keys?")
     allow_extended_apikeys: bool = Field(default=True, description="Allow extended API keys?")
     max_apikey_duration_amount: Optional[int] = Field(
@@ -218,17 +293,34 @@ class Auth(BaseModel):
 
 
 class APMServer(BaseModel):
-    "APM server configuration"
+    """Application Performance Monitoring (APM) server configuration.
+
+    Defines the connection details for an external APM server used to
+    collect and analyze application performance metrics.
+    """
 
     server_url: Optional[str] = Field(default=None, description="URL to API server")
     token: Optional[str] = Field(default=None, description="Authentication token for server")
 
 
 class Metrics(BaseModel):
+    """Metrics collection configuration.
+
+    Configures how Howler collects and exports application metrics,
+    including integration with external APM servers.
+    """
+
     apm_server: APMServer = APMServer()
 
 
 class Retention(BaseModel):
+    """Hit retention policy configuration.
+
+    Defines the automatic data retention policy for hits, including
+    the maximum age of hits before they are purged and the schedule
+    for running the retention cleanup job.
+    """
+
     enabled: bool = Field(
         default=True,
         description=(
@@ -250,13 +342,49 @@ class Retention(BaseModel):
     )
 
 
+class ViewCleanup(BaseModel):
+    """View cleanup job configuration.
+
+    Defines the schedule and behavior for cleaning up stale dashboard views
+    that reference non-existent backend data.
+    """
+
+    enabled: bool = Field(
+        default=True,
+        description=(
+            "Whether to enable the view cleanup. If enabled, views pinned "
+            "to the dashboard that no longer exist in the backend will be cleared."
+        ),
+    )
+    crontab: str = Field(
+        default="0 0 * * *",
+        description="The crontab that denotes how often to run the view_cleanup job",
+    )
+
+
 class System(BaseModel):
+    """System-level configuration for Howler.
+
+    Defines global system settings including deployment type (production,
+    staging, or development) and configuration for automated maintenance
+    jobs like data retention and view cleanup.
+    """
+
     type: Literal["production", "staging", "development"] = Field(default="development", description="Type of system")
     retention: Retention = Retention()
     "Retention Configuration"
+    view_cleanup: ViewCleanup = ViewCleanup()
+    "View Cleanup Configuration"
 
 
 class UI(BaseModel):
+    """User interface and web server configuration.
+
+    Defines settings for the Howler web UI including Flask configuration,
+    session validation, API auditing, static file locations, and WebSocket
+    integration for real-time updates.
+    """
+
     audit: bool = Field(description="Should API calls be audited and saved to a separate log file?", default=True)
     debug: bool = Field(default=False, description="Enable debugging?")
     static_folder: Optional[str] = Field(
@@ -281,21 +409,35 @@ class UI(BaseModel):
     )
 
 
-class Borealis(BaseModel):
-    enabled: bool = Field(default=False, description="Should borealis integration be enabled?")
+class Clue(BaseModel):
+    """Clue enrichment service integration configuration.
+
+    Defines settings for integrating with Clue, an external enrichment
+    service that can provide additional context and status information for
+    hits displayed in the Howler UI.
+    """
+
+    enabled: bool = Field(default=False, description="Should clue integration be enabled?")
 
     url: str = Field(
         default="http://enrichment-rest.enrichment.svc.cluster.local:5000",
-        description="What url should Howler connect to to interact with Borealis?",
+        description="What url should Howler connect to to interact with Clue?",
     )
 
     status_checks: list[str] = Field(
         default=[],
-        description="A list of borealis fetchers that return status results given a Howler ID to show in the UI.",
+        description="A list of clue fetchers that return status results given a Howler ID to show in the UI.",
     )
 
 
 class Notebook(BaseModel):
+    """Jupyter notebook integration configuration.
+
+    Defines settings for integrating with nbgallery, a collaborative
+    Jupyter notebook platform, allowing users to access and share
+    notebooks related to their Howler analysis work.
+    """
+
     enabled: bool = Field(default=False, description="Should nbgallery notebook integration be enabled?")
 
     scope: Optional[str] = Field(default=None, description="The scope expected by nbgallery for JWTs")
@@ -306,6 +448,13 @@ class Notebook(BaseModel):
 
 
 class Core(BaseModel):
+    """Core application configuration for Howler.
+
+    Aggregates all core service configurations including Redis, metrics,
+    and external integrations like Clue and nbgallery notebooks.
+    Also manages the loading of external plugins.
+    """
+
     plugins: set[str] = Field(description="A list of external plugins to load", default=set())
 
     metrics: Metrics = Metrics()
@@ -314,8 +463,8 @@ class Core(BaseModel):
     redis: Redis = Redis()
     "Configuration for Redis instances"
 
-    borealis: Borealis = Borealis()
-    "Configuration for Borealis Integration"
+    clue: Clue = Clue()
+    "Configuration for Clue Integration"
 
     notebook: Notebook = Notebook()
     "Configuration for Notebook Integration"
@@ -363,13 +512,24 @@ logger.info("Fetching configuration files from %s", ":".join(str(c) for c in con
 
 
 class Config(BaseSettings):
+    """Main Howler configuration model.
+
+    The root configuration object that aggregates all configuration sections
+    including authentication, datastore, logging, system settings, UI, and core
+    services. Configuration can be loaded from YAML files or environment variables
+    with the HWL_ prefix.
+
+    Environment variables use double underscores (__) for nested properties.
+    For example: HWL_DATASTORE__TYPE=elasticsearch
+    """
+
     auth: Auth = Auth()
     core: Core = Core()
     datastore: Datastore = Datastore()
     logging: Logging = Logging()
     system: System = System()
     ui: UI = UI()
-    mapping: dict[str, str] = Field(description="Mapping of alert keys to borealis type", default={})
+    mapping: dict[str, str] = Field(description="Mapping of alert keys to clue types", default={})
 
     model_config = SettingsConfigDict(
         yaml_file=config_locations,

howler/odm/models/howler_data.py

@@ -118,7 +118,8 @@ class Link(odm.Model):
         description=(
             "The icon to show. Either an ID corresponding to an "
             "analytical platform application, or an external link."
-        )
+        ),
+        optional=True,
     )
 
 

howler/odm/models/lead.py

@@ -2,18 +2,9 @@
 from typing import Optional
 
 from howler import odm
-from howler.odm.howler_enum import HowlerEnum
 from howler.odm.models.localized_label import LocalizedLabel
 
 
-class Formats(str, HowlerEnum):
-    BOREALIS = "borealis"
-    MARKDOWN = "markdown"
-
-    def __str__(self) -> str:
-        return self.value
-
-
 @odm.model(
     index=False,
     store=True,
@@ -24,7 +15,7 @@ class Lead(odm.Model):
         description="An optional icon to use in the tab display for this dossier.", optional=True
     )
     label: LocalizedLabel = odm.Compound(LocalizedLabel, description="Labels for the lead in the UI.")
-    format: str = odm.Enum(values=Formats, description="The format of the lead. ")
+    format: str = odm.Keyword(description="The format of the lead.")
     content: str = odm.Text(
         description="The data for the content. Could be a link, raw markdown text, or other valid lead format.",
     )

howler/odm/models/pivot.py

@@ -2,18 +2,9 @@
 from typing import Optional
 
 from howler import odm
-from howler.odm.howler_enum import HowlerEnum
 from howler.odm.models.localized_label import LocalizedLabel
 
 
-class Formats(str, HowlerEnum):
-    BOREALIS = "borealis"
-    LINK = "link"
-
-    def __str__(self) -> str:
-        return self.value
-
-
 @odm.model(
     index=True,
     store=True,
@@ -40,8 +31,8 @@ class Pivot(odm.Model):
         description="An optional icon to use in the tab display for this dossier.", optional=True
     )
     label: LocalizedLabel = odm.Compound(LocalizedLabel, description="Labels for the pivot in the UI.")
-    value: str = odm.Keyword(description="The link/borealis id to pivot on.")
-    format: str = odm.Enum(values=Formats, description="The format of the pivot.")
+    value: str = odm.Keyword(description="The link/plugin information to pivot on.")
+    format: str = odm.Keyword(description="The format of the pivot.")
     mappings: list[Mapping] = odm.List(
         odm.Compound(Mapping),
         default=[],

howler/odm/random_data.py

@@ -820,7 +820,7 @@ def wipe_dossiers(ds: HowlerDatastore):
 
 def setup_hits(ds):
     "Set up hits index"
-    os.environ["ELASTIC_HIT_SHARDS"] = "12"
+    os.environ["ELASTIC_HIT_SHARDS"] = "1"
     os.environ["ELASTIC_HIT_REPLICAS"] = "1"
     ds.hit.fix_shards()
     ds.hit.fix_replicas()

howler/security/__init__.py

@@ -220,8 +220,8 @@ class api_login(object):  # noqa: D101, N801
                     user_id=user.get("uname", None),
                 )
 
-            if request.path.startswith("/api/v1/borealis"):
-                logger.debug("Bypassing quota limits for borealis enrichment")
+            if request.path.startswith("/api/v1/clue"):
+                logger.debug("Bypassing quota limits for clue enrichment")
             elif self.enforce_quota:
                 # Check current user quota
                 flsk_session["quota_user"] = user["uname"]

howler/services/analytic_service.py

@@ -1,5 +1,8 @@
+from typing import Any, Union
+
 from howler.common.loader import datastore
 from howler.common.logging import get_logger
+from howler.datastore.exceptions import SearchException
 from howler.datastore.operations import OdmUpdateOperation
 from howler.odm.models.analytic import Analytic
 from howler.odm.models.hit import Hit
@@ -36,6 +39,34 @@ def update_analytic(
     return result
 
 
+def get_matching_analytics(hits: Union[list[Hit], list[dict[str, Any]]]) -> list[Analytic]:
+    """Get a list of matching analytics for the given list of hits.
+
+    Args:
+        hits (Union[list[Hit], list[dict[str, Any]]]): A list of Hit objects or dictionaries representing hits.
+    Returns:
+        list[Analytic]: A list of Analytic objects that match the analytics referenced in the hits.
+    """
+    if len(hits) < 1:
+        return []
+
+    storage = datastore()
+
+    analytic_names: set[str] = set()
+    for hit in hits:
+        analytic_names.add(f'"{sanitize_lucene_query(hit["howler"]["analytic"])}"')
+
+    try:
+        existing_analytics: list[Analytic] = storage.analytic.search(
+            f'name:({" OR ".join(analytic_names)})', as_obj=True
+        )["items"]
+
+        return existing_analytics
+    except SearchException:
+        logger.exception("Exception on analytic matching")
+        return []
+
+
 def save_from_hit(hit: Hit, user: User):
     """Save updates to an analytic based on a new hit that has been created
 

howler/services/config_service.py

@@ -117,11 +117,11 @@ def get_configuration(user: User, **kwargs):
             },
             "mapping": config.mapping,
             "features": {
-                "borealis": config.core.borealis.enabled,
+                "clue": config.core.clue.enabled,
                 "notebook": config.core.notebook.enabled,
                 **plugin_features,
             },
-            "borealis": {"status_checks": config.core.borealis.status_checks},
+            "clue": {"status_checks": config.core.clue.status_checks},
         },
         "c12nDef": classification_definition,
         "indexes": list_all_fields("admin" in user["type"] if user is not None else False),