pangea-sdk 6.1.1__py3-none-any.whl → 6.2.0b2__py3-none-any.whl

pangea/services/audit/audit.py

@@ -1,16 +1,25 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
+
+# TODO: Modernize.
+# ruff: noqa: UP006, UP035
+
 from __future__ import annotations
 
 import datetime
 import json
-from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
+from collections.abc import Mapping
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union, cast, overload
+
+from pydantic import TypeAdapter
+from typing_extensions import Literal
 
 import pangea.exceptions as pexc
 from pangea.config import PangeaConfig
 from pangea.response import PangeaResponse, PangeaResponseResult
 from pangea.services.audit.exceptions import AuditException, EventCorruption
 from pangea.services.audit.models import (
+    AuditSchema,
     DownloadFormat,
     DownloadRequest,
     DownloadResult,
@@ -18,6 +27,7 @@ from pangea.services.audit.models import (
     EventEnvelope,
     EventVerification,
     ExportRequest,
+    ForwardingConfiguration,
     LogBulkRequest,
     LogBulkResult,
     LogEvent,
@@ -35,6 +45,9 @@ from pangea.services.audit.models import (
     SearchRequest,
     SearchResultOutput,
     SearchResultRequest,
+    ServiceConfig,
+    ServiceConfigFilter,
+    ServiceConfigListResult,
 )
 from pangea.services.audit.signing import Signer, Verifier
 from pangea.services.audit.util import (
@@ -55,7 +68,7 @@ from pangea.utils import canonicalize_nested_json
 
 class AuditBase:
     def __init__(
-        self, private_key_file: str = "", public_key_info: dict[str, str] = {}, tenant_id: str | None = None
+        self, private_key_file: str = "", public_key_info: Mapping[str, str] = {}, tenant_id: str | None = None
     ) -> None:
         self.pub_roots: Dict[int, PublishedRoot] = {}
         self.buffer_data: Optional[str] = None
@@ -90,7 +103,7 @@ class AuditBase:
         return input  # type: ignore[return-value]
 
     def _process_log(self, event: dict, sign_local: bool) -> LogEvent:
-        if event.get("tenant_id", None) is None and self.tenant_id:
+        if event.get("tenant_id") is None and self.tenant_id:
             event["tenant_id"] = self.tenant_id
 
         event = {k: v for k, v in event.items() if v is not None}
@@ -221,10 +234,7 @@ class AuditBase:
         tree_sizes.add(result.root.size)
         tree_sizes.difference_update(self.pub_roots.keys())
 
-        if tree_sizes:
-            arweave_roots = get_arweave_published_roots(result.root.tree_name, tree_sizes)
-        else:
-            arweave_roots = {}
+        arweave_roots = get_arweave_published_roots(result.root.tree_name, tree_sizes) if tree_sizes else {}
 
         return tree_sizes, arweave_roots
 
@@ -385,7 +395,7 @@ class Audit(ServiceBase, AuditBase):
         token: str,
         config: PangeaConfig | None = None,
         private_key_file: str = "",
-        public_key_info: dict[str, str] = {},
+        public_key_info: Mapping[str, str] = {},
         tenant_id: str | None = None,
         logger_name: str = "pangea",
         config_id: str | None = None,
@@ -919,6 +929,298 @@ class Audit(ServiceBase, AuditBase):
         )
         return self.request.post("v1/download_results", DownloadResult, data=input.model_dump(exclude_none=True))
 
+    def get_service_config(self, config_id: str) -> PangeaResponse[ServiceConfig]:
+        """
+        Get a service config.
+
+        OperationId: audit_post_v1beta_config
+
+        Args:
+            id: The config ID
+        """
+
+        response = self.request.post("v1beta/config", PangeaResponseResult, data={"id": config_id})
+        response.result = TypeAdapter(ServiceConfig).validate_python(response.json["result"])
+        return cast(PangeaResponse[ServiceConfig], response)
+
+    @overload
+    def create_service_config(
+        self,
+        version: Literal[1],
+        name: str,
+        *,
+        cold_query_result_retention: str | None = None,
+        hot_storage: str | None = None,
+        query_result_retention: str | None = None,
+        redact_service_config_id: str | None = None,
+        redaction_fields: Sequence[str] | None = None,
+        retention: str | None = None,
+        vault_key_id: str | None = None,
+        vault_service_config_id: str | None = None,
+        vault_sign: bool | None = None,
+    ) -> PangeaResponse[ServiceConfig]:
+        """
+        Create a v1 service config.
+
+        OperationId: audit_post_v1beta_config_create
+
+        Args:
+            name: Configuration name
+            cold_query_result_retention: Retention window for cold query result / state information.
+            hot_storage: Retention window to keep audit logs in hot storage.
+            query_result_retention: Length of time to preserve server-side query result caching.
+            redact_service_config_id: A redact service config that will be used to redact PII from logs.
+            redaction_fields: Fields to perform redaction against.
+            retention: Retention window to store audit logs.
+            vault_key_id: ID of the Vault key used for signing. If missing, use a default Audit key.
+            vault_service_config_id: A vault service config that will be used to sign logs.
+            vault_sign: Enable/disable event signing.
+        """
+
+    @overload
+    def create_service_config(
+        self,
+        version: Literal[2],
+        name: str,
+        *,
+        schema: AuditSchema,
+        cold_query_result_retention: str | None = None,
+        forwarding_configuration: ForwardingConfiguration | None = None,
+        hot_storage: str | None = None,
+        query_result_retention: str | None = None,
+        redact_service_config_id: str | None = None,
+        retention: str | None = None,
+        vault_key_id: str | None = None,
+        vault_service_config_id: str | None = None,
+        vault_sign: bool | None = None,
+    ) -> PangeaResponse[ServiceConfig]:
+        """
+        Create a v2 service config.
+
+        OperationId: audit_post_v1beta_config_create
+
+        Args:
+            name: Configuration name
+            schema: Audit log field configuration. Only settable at create time.
+            cold_query_result_retention: Retention window for cold query result / state information.
+            forwarding_configuration: Configuration for forwarding audit logs to external systems.
+            hot_storage: Retention window to keep audit logs in hot storage.
+            query_result_retention: Length of time to preserve server-side query result caching.
+            redact_service_config_id: A redact service config that will be used to redact PII from logs.
+            retention: Retention window to store audit logs.
+            vault_key_id: ID of the Vault key used for signing. If missing, use a default Audit key.
+            vault_service_config_id: A vault service config that will be used to sign logs.
+            vault_sign: Enable/disable event signing.
+        """
+
+    @overload
+    def create_service_config(
+        self,
+        version: Literal[3],
+        name: str,
+        *,
+        schema: AuditSchema,
+        cold_storage: str | None = None,
+        hot_storage: str | None = None,
+        warm_storage: str | None = None,
+        redact_service_config_id: str | None = None,
+        vault_service_config_id: str | None = None,
+        vault_key_id: str | None = None,
+        vault_sign: bool | None = None,
+        forwarding_configuration: ForwardingConfiguration | None = None,
+    ) -> PangeaResponse[ServiceConfig]:
+        """
+        Create a v3 service config.
+
+        OperationId: audit_post_v1beta_config_create
+
+        Args:
+            name: Configuration name
+            schema: Audit log field configuration. Only settable at create time.
+            cold_storage: Retention window for logs in cold storage. Deleted afterwards.
+            hot_storage: Retention window for logs in hot storage. Migrated to warm, cold, or deleted afterwards.
+            warm_storage: Retention window for logs in warm storage. Migrated to cold or deleted afterwards.
+            redact_service_config_id: A redact service config that will be used to redact PII from logs.
+            vault_service_config_id: A vault service config that will be used to sign logs.
+            vault_key_id: ID of the Vault key used for signing. If missing, use a default Audit key.
+            vault_sign: Enable/disable event signing.
+            forwarding_configuration: Configuration for forwarding audit logs to external systems.
+        """
+
+    def create_service_config(
+        self,
+        version: Literal[1, 2, 3],
+        name: str,
+        *,
+        cold_query_result_retention: str | None = None,
+        cold_storage: str | None = None,
+        forwarding_configuration: ForwardingConfiguration | None = None,
+        hot_storage: str | None = None,
+        query_result_retention: str | None = None,
+        redact_service_config_id: str | None = None,
+        redaction_fields: Sequence[str] | None = None,
+        retention: str | None = None,
+        schema: AuditSchema | None = None,
+        vault_key_id: str | None = None,
+        vault_service_config_id: str | None = None,
+        vault_sign: bool | None = None,
+        warm_storage: str | None = None,
+    ) -> PangeaResponse[ServiceConfig]:
+        """
+        Create a service config.
+
+        OperationId: audit_post_v1beta_config_create
+
+        Args:
+            name: Configuration name
+            cold_query_result_retention: Retention window for cold query result / state information.
+            cold_storage: Retention window for logs in cold storage. Deleted afterwards.
+            forwarding_configuration: Configuration for forwarding audit logs to external systems.
+            hot_storage: Retention window to keep audit logs in hot storage.
+            query_result_retention: Length of time to preserve server-side query result caching.
+            redact_service_config_id: A redact service config that will be used to redact PII from logs.
+            redaction_fields: Fields to perform redaction against.
+            retention: Retention window to store audit logs.
+            schema: Audit log field configuration. Only settable at create time.
+            vault_key_id: ID of the Vault key used for signing. If missing, use a default Audit key.
+            vault_service_config_id: A vault service config that will be used to sign logs.
+            vault_sign: Enable/disable event signing.
+            warm_storage: Retention window for logs in warm storage. Migrated to cold or deleted afterwards.
+        """
+
+        response = self.request.post(
+            "v1beta/config/create",
+            PangeaResponseResult,
+            data={
+                "cold_query_result_retention": cold_query_result_retention,
+                "cold_storage": cold_storage,
+                "forwarding_configuration": forwarding_configuration,
+                "hot_storage": hot_storage,
+                "name": name,
+                "query_result_retention": query_result_retention,
+                "redact_service_config_id": redact_service_config_id,
+                "redaction_fields": redaction_fields,
+                "retention": retention,
+                "schema": schema,
+                "vault_key_id": vault_key_id,
+                "vault_service_config_id": vault_service_config_id,
+                "vault_sign": vault_sign,
+                "warm_storage": warm_storage,
+                "version": version,
+            },
+        )
+        response.result = TypeAdapter(ServiceConfig).validate_python(response.json["result"])
+        return cast(PangeaResponse[ServiceConfig], response)
+
+    def update_service_config(
+        self,
+        config_id: str,
+        *,
+        name: str,
+        updated_at: datetime.datetime,
+        # Optionals.
+        cold_query_result_retention: str | None = None,
+        cold_storage: str | None = None,
+        forwarding_configuration: ForwardingConfiguration | None = None,
+        hot_storage: str | None = None,
+        query_result_retention: str | None = None,
+        redact_service_config_id: str | None = None,
+        retention: str | None = None,
+        schema: AuditSchema | None = None,
+        vault_key_id: str | None = None,
+        vault_service_config_id: str | None = None,
+        vault_sign: bool | None = None,
+        warm_storage: str | None = None,
+    ) -> PangeaResponse[ServiceConfig]:
+        """
+        Update a service config.
+
+        OperationId: audit_post_v1beta_config_update
+
+        Args:
+            id: The config ID
+            name: Configuration name
+            updated_at: The DB timestamp when this config was last updated at
+            cold_query_result_retention: Retention window for cold query result / state information.
+            cold_storage: Retention window for logs in cold storage. Deleted afterwards.
+            forwarding_configuration: Configuration for forwarding audit logs to external systems
+            hot_storage: Retention window to keep audit logs in hot storage
+            query_result_retention: Length of time to preserve server-side query result caching
+            redact_service_config_id: A redact service config that will be used to redact PII from logs
+            retention: Retention window to store audit logs
+            schema: Audit log field configuration
+            vault_key_id: ID of the Vault key used for signing. If missing, use a default Audit key.
+            vault_service_config_id: A vault service config that will be used to sign logs
+            vault_sign: Enable/disable event signing
+            warm_storage: Retention window for logs in warm storage. Migrated to cold or deleted afterwards.
+        """
+
+        response = self.request.post(
+            "v1beta/config/update",
+            PangeaResponseResult,
+            data={
+                "id": config_id,
+                "name": name,
+                "updated_at": updated_at,
+                # Optionals.
+                "cold_query_result_retention": cold_query_result_retention,
+                "cold_storage": cold_storage,
+                "forwarding_configuration": forwarding_configuration,
+                "hot_storage": hot_storage,
+                "query_result_retention": query_result_retention,
+                "redact_service_config_id": redact_service_config_id,
+                "retention": retention,
+                "schema": schema,
+                "vault_key_id": vault_key_id,
+                "vault_service_config_id": vault_service_config_id,
+                "vault_sign": vault_sign,
+                "warm_storage": warm_storage,
+            },
+        )
+        response.result = TypeAdapter(ServiceConfig).validate_python(response.json["result"])
+        return cast(PangeaResponse[ServiceConfig], response)
+
+    def delete_service_config(self, config_id: str) -> PangeaResponse[ServiceConfig]:
+        """
+        Delete a service config.
+
+        OperationId: audit_post_v1beta_config_delete
+
+        Args:
+            id: The config ID
+        """
+
+        response = self.request.post("v1beta/config/delete", PangeaResponseResult, data={"id": config_id})
+        response.result = TypeAdapter(ServiceConfig).validate_python(response.json["result"])
+        return cast(PangeaResponse[ServiceConfig], response)
+
+    def list_service_configs(
+        self,
+        *,
+        filter: ServiceConfigFilter | None = None,
+        last: str | None = None,
+        order: Literal["asc", "desc"] | None = None,
+        order_by: Literal["id", "created_at", "updated_at"] | None = None,
+        size: int | None = None,
+    ) -> PangeaResponse[ServiceConfigListResult]:
+        """
+        List service configs.
+
+        OperationId: audit_post_v1beta_config_list
+
+        Args:
+            last: Reflected value from a previous response to obtain the next page of results.
+            order: Order results asc(ending) or desc(ending).
+            order_by: Which field to order results by.
+            size: Maximum results to include in the response.
+        """
+
+        return self.request.post(
+            "v1beta/config/list",
+            ServiceConfigListResult,
+            data={"filter": filter, "last": last, "order": order, "order_by": order_by, "size": size},
+        )
+
     def update_published_roots(self, result: SearchResultOutput):
         """Fetches series of published root hashes from Arweave
 

pangea/services/audit/models.py

@@ -1,11 +1,18 @@
 # Copyright 2022 Pangea Cyber Corporation
 # Author: Pangea Cyber Corporation
+
+# TODO: Modernize.
+# ruff: noqa: UP006, UP035
+
 from __future__ import annotations
 
 import datetime
 import enum
 from typing import Any, Dict, List, Optional, Sequence, Union
 
+from pydantic import Field
+from typing_extensions import Annotated, Literal
+
 from pangea.response import APIRequestModel, APIResponseModel, PangeaDateTime, PangeaResponseResult
 
 
@@ -495,3 +502,275 @@ class ExportRequest(APIRequestModel):
     Whether or not to include the root hash of the tree and the membership proof
     for each record.
     """
+
+
+class AuditSchemaField(APIResponseModel):
+    """A description of a field in an audit log."""
+
+    id: str
+    """Prefix name / identity for the field."""
+
+    type: Literal["boolean", "datetime", "integer", "string", "string-unindexed", "text"]
+    """The data type for the field."""
+
+    description: Optional[str] = None
+    """Human display description of the field."""
+
+    name: Optional[str] = None
+    """Human display name/title of the field."""
+
+    redact: Optional[bool] = None
+    """If true, redaction is performed against this field (if configured.) Only valid for string type."""
+
+    required: Optional[bool] = None
+    """If true, this field is required to exist in all logged events."""
+
+    size: Optional[int] = None
+    """The maximum size of the field. Only valid for strings, which limits number of UTF-8 characters."""
+
+    ui_default_visible: Optional[bool] = None
+    """If true, this field is visible by default in audit UIs."""
+
+
+class AuditSchema(APIResponseModel):
+    """A description of acceptable fields for an audit log."""
+
+    client_signable: Optional[bool] = None
+    """If true, records contain fields to support client/vault signing."""
+
+    save_malformed: Optional[str] = None
+    """Save (or reject) malformed AuditEvents."""
+
+    tamper_proofing: Optional[bool] = None
+    """If true, records contain fields to support tamper-proofing."""
+
+    fields: Optional[List[AuditSchemaField]] = None
+    """List of field definitions."""
+
+
+class ForwardingConfiguration(APIResponseModel):
+    """Configuration for forwarding audit logs to external systems."""
+
+    type: str
+    """Type of forwarding configuration."""
+
+    forwarding_enabled: Optional[bool] = False
+    """Whether forwarding is enabled."""
+
+    event_url: Optional[str] = None
+    """URL where events will be written to. Must use HTTPS."""
+
+    ack_url: Optional[str] = None
+    """If indexer acknowledgement is required, this must be provided along with a 'channel_id'."""
+
+    channel_id: Optional[str] = None
+    """An optional splunk channel included in each request if indexer acknowledgement is required."""
+
+    public_cert: Optional[str] = None
+    """Public certificate if a self signed TLS cert is being used."""
+
+    index: Optional[str] = None
+    """Optional splunk index passed in the record bodies."""
+
+    vault_config_id: Optional[str] = None
+    """The vault config used to store the HEC token."""
+
+    vault_secret_id: Optional[str] = None
+    """The secret ID where the HEC token is stored in vault."""
+
+
+class ServiceConfigV1(PangeaResponseResult):
+    """Configuration options available for audit service"""
+
+    id: Optional[str] = None
+    """The config ID"""
+
+    version: Literal[1] = 1
+
+    created_at: Optional[str] = None
+    """The DB timestamp when this config was created. Ignored when submitted."""
+
+    updated_at: Optional[str] = None
+    """The DB timestamp when this config was last updated at"""
+
+    name: Optional[str] = None
+    """Configuration name"""
+
+    retention: Optional[str] = None
+    """Retention window to store audit logs."""
+
+    cold_query_result_retention: Optional[str] = None
+    """Retention window for cold query result / state information."""
+
+    hot_storage: Optional[str] = None
+    """Retention window to keep audit logs in hot storage."""
+
+    query_result_retention: Optional[str] = None
+    """Length of time to preserve server-side query result caching."""
+
+    redact_service_config_id: Optional[str] = None
+    """A redact service config that will be used to redact PII from logs."""
+
+    redaction_fields: Optional[List[str]] = None
+    """Fields to perform redaction against."""
+
+    vault_service_config_id: Optional[str] = None
+    """A vault service config that will be used to sign logs."""
+
+    vault_key_id: Optional[str] = None
+    """ID of the Vault key used for signing. If missing, use a default Audit key"""
+
+    vault_sign: Optional[bool] = None
+    """Enable/disable event signing"""
+
+
+class ServiceConfigV2(PangeaResponseResult):
+    """Configuration options available for audit service"""
+
+    audit_schema: AuditSchema = Field(alias="schema")
+    """Audit log field configuration. Only settable at create time."""
+
+    version: Literal[2] = 2
+
+    cold_query_result_retention: Optional[str] = None
+    """Retention window for cold query result / state information."""
+
+    created_at: Optional[str] = None
+    """The DB timestamp when this config was created. Ignored when submitted."""
+
+    hot_storage: Optional[str] = None
+    """Retention window to keep audit logs in hot storage."""
+
+    id: Optional[str] = None
+    """The config ID"""
+
+    name: Optional[str] = None
+    """Configuration name"""
+
+    query_result_retention: Optional[str] = None
+    """Length of time to preserve server-side query result caching."""
+
+    redact_service_config_id: Optional[str] = None
+    """A redact service config that will be used to redact PII from logs."""
+
+    retention: Optional[str] = None
+    """Retention window to store audit logs."""
+
+    updated_at: Optional[str] = None
+    """The DB timestamp when this config was last updated at"""
+
+    vault_key_id: Optional[str] = None
+    """ID of the Vault key used for signing. If missing, use a default Audit key"""
+
+    vault_service_config_id: Optional[str] = None
+    """A vault service config that will be used to sign logs."""
+
+    vault_sign: Optional[bool] = None
+    """Enable/disable event signing"""
+
+    forwarding_configuration: Optional[ForwardingConfiguration] = None
+    """Configuration for forwarding audit logs to external systems."""
+
+
+class ServiceConfigV3(PangeaResponseResult):
+    """Configuration options available for audit service"""
+
+    audit_schema: AuditSchema = Field(alias="schema")
+    """Audit log field configuration. Only settable at create time."""
+
+    version: Literal[3] = 3
+    """Version of the service config."""
+
+    cold_storage: Optional[str] = None
+    """Retention window for logs in cold storage. Deleted afterwards."""
+
+    created_at: Optional[str] = None
+    """The DB timestamp when this config was created. Ignored when submitted."""
+
+    forwarding_configuration: Optional[ForwardingConfiguration] = None
+    """Configuration for forwarding audit logs to external systems."""
+
+    hot_storage: Optional[str] = None
+    """Retention window for logs in hot storage. Migrated to warm, cold, or deleted afterwards."""
+
+    id: Optional[str] = None
+    """The config ID"""
+
+    name: Optional[str] = None
+    """Configuration name"""
+
+    redact_service_config_id: Optional[str] = None
+    """A redact service config that will be used to redact PII from logs."""
+
+    updated_at: Optional[str] = None
+    """The DB timestamp when this config was last updated at"""
+
+    vault_key_id: Optional[str] = None
+    """ID of the Vault key used for signing. If missing, use a default Audit key"""
+
+    vault_service_config_id: Optional[str] = None
+    """A vault service config that will be used to sign logs."""
+
+    vault_sign: Optional[bool] = None
+    """Enable/disable event signing"""
+
+    warm_storage: Optional[str] = None
+    """Retention window for logs in warm storage. Migrated to cold or deleted afterwards."""
+
+
+ServiceConfig = Annotated[
+    Union[ServiceConfigV1, ServiceConfigV2, ServiceConfigV3],
+    Field(discriminator="version"),
+]
+"""Configuration options available for audit service"""
+
+
+class ServiceConfigFilter(APIRequestModel):
+    id: Optional[str] = None
+    """Only records where id equals this value."""
+
+    id__contains: Optional[Sequence[str]] = None
+    """Only records where id includes each substring."""
+
+    id__in: Optional[Sequence[str]] = None
+    """Only records where id equals one of the provided substrings."""
+
+    created_at: Optional[str] = None
+    """Only records where created_at equals this value."""
+
+    created_at__gt: Optional[str] = None
+    """Only records where created_at is greater than this value."""
+
+    created_at__gte: Optional[str] = None
+    """Only records where created_at is greater than or equal to this value."""
+
+    created_at__lt: Optional[str] = None
+    """Only records where created_at is less than this value."""
+
+    created_at__lte: Optional[str] = None
+    """Only records where created_at is less than or equal to this value."""
+
+    updated_at: Optional[str] = None
+    """Only records where updated_at equals this value."""
+
+    updated_at__gt: Optional[str] = None
+    """Only records where updated_at is greater than this value."""
+
+    updated_at__gte: Optional[str] = None
+    """Only records where updated_at is greater than or equal to this value."""
+
+    updated_at__lt: Optional[str] = None
+    """Only records where updated_at is less than this value."""
+
+    updated_at__lte: Optional[str] = None
+    """Only records where updated_at is less than or equal to this value."""
+
+
+class ServiceConfigListResult(PangeaResponseResult):
+    count: int
+    """The total number of service configs matched by the list request."""
+
+    last: str
+    """Used to fetch the next page of the current listing when provided in a repeated request's last parameter."""
+
+    items: Sequence[ServiceConfig]

pangea/services/audit/signing.py

@@ -81,7 +81,7 @@ class Signer:
                 with open(self.private_key_file, "rb") as file:
                     file_bytes = file.read()
             except FileNotFoundError:
-                raise Exception(f"Error: Failed opening private key file {self.private_key_file}")
+                raise Exception(f"Error: Failed opening private key file {self.private_key_file}") from None
 
             privkey = self._decode_private_key(file_bytes)
             for cls, signer in signers.items():