cribl-control-plane 0.0.13__py3-none-any.whl

cribl_control_plane/models/outputcribllake.py

@@ -0,0 +1,396 @@
+"""Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT."""
+
+from __future__ import annotations
+from cribl_control_plane import utils
+from cribl_control_plane.types import BaseModel
+from cribl_control_plane.utils import validate_open_enum
+from enum import Enum
+import pydantic
+from pydantic.functional_validators import PlainValidator
+from typing import List, Optional
+from typing_extensions import Annotated, NotRequired, TypedDict
+
+
+class OutputCriblLakeType(str, Enum, metaclass=utils.OpenEnumMeta):
+    CRIBL_LAKE = "cribl_lake"
+
+
+class OutputCriblLakeSignatureVersion(str, Enum, metaclass=utils.OpenEnumMeta):
+    r"""Signature version to use for signing S3 requests"""
+
+    V2 = "v2"
+    V4 = "v4"
+
+
+class OutputCriblLakeObjectACL(str, Enum, metaclass=utils.OpenEnumMeta):
+    r"""Object ACL to assign to uploaded objects"""
+
+    PRIVATE = "private"
+    PUBLIC_READ = "public-read"
+    PUBLIC_READ_WRITE = "public-read-write"
+    AUTHENTICATED_READ = "authenticated-read"
+    AWS_EXEC_READ = "aws-exec-read"
+    BUCKET_OWNER_READ = "bucket-owner-read"
+    BUCKET_OWNER_FULL_CONTROL = "bucket-owner-full-control"
+
+
+class OutputCriblLakeStorageClass(str, Enum, metaclass=utils.OpenEnumMeta):
+    r"""Storage class to select for uploaded objects"""
+
+    STANDARD = "STANDARD"
+    REDUCED_REDUNDANCY = "REDUCED_REDUNDANCY"
+    STANDARD_IA = "STANDARD_IA"
+    ONEZONE_IA = "ONEZONE_IA"
+    INTELLIGENT_TIERING = "INTELLIGENT_TIERING"
+    GLACIER = "GLACIER"
+    GLACIER_IR = "GLACIER_IR"
+    DEEP_ARCHIVE = "DEEP_ARCHIVE"
+
+
+class OutputCriblLakeServerSideEncryptionForUploadedObjects(
+    str, Enum, metaclass=utils.OpenEnumMeta
+):
+    AES256 = "AES256"
+    AWS_KMS = "aws:kms"
+
+
+class OutputCriblLakeBackpressureBehavior(str, Enum, metaclass=utils.OpenEnumMeta):
+    r"""How to handle events when all receivers are exerting backpressure"""
+
+    BLOCK = "block"
+    DROP = "drop"
+
+
+class OutputCriblLakeDiskSpaceProtection(str, Enum, metaclass=utils.OpenEnumMeta):
+    r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
+
+    BLOCK = "block"
+    DROP = "drop"
+
+
+class AwsAuthenticationMethod(str, Enum, metaclass=utils.OpenEnumMeta):
+    AUTO = "auto"
+    AUTO_RPC = "auto_rpc"
+    MANUAL = "manual"
+
+
+class OutputCriblLakeFormat(str, Enum, metaclass=utils.OpenEnumMeta):
+    JSON = "json"
+    PARQUET = "parquet"
+    DDSS = "ddss"
+
+
+class OutputCriblLakeTypedDict(TypedDict):
+    id: str
+    r"""Unique ID for this output"""
+    type: OutputCriblLakeType
+    pipeline: NotRequired[str]
+    r"""Pipeline to process data before sending out to this output"""
+    system_fields: NotRequired[List[str]]
+    r"""Fields to automatically add to events, such as cribl_pipe. Supports wildcards."""
+    environment: NotRequired[str]
+    r"""Optionally, enable this config only on a specified Git branch. If empty, will be enabled everywhere."""
+    streamtags: NotRequired[List[str]]
+    r"""Tags for filtering and grouping in @{product}"""
+    bucket: NotRequired[str]
+    r"""Name of the destination S3 bucket. Must be a JavaScript expression (which can evaluate to a constant value), enclosed in quotes or backticks. Can be evaluated only at initialization time. Example referencing a Global Variable: `myBucket-${C.vars.myVar}`"""
+    region: NotRequired[str]
+    r"""Region where the S3 bucket is located"""
+    aws_secret_key: NotRequired[str]
+    r"""Secret key. This value can be a constant or a JavaScript expression. Example: `${C.env.SOME_SECRET}`)"""
+    endpoint: NotRequired[str]
+    r"""S3 service endpoint. If empty, defaults to the AWS Region-specific endpoint. Otherwise, it must point to S3-compatible endpoint."""
+    signature_version: NotRequired[OutputCriblLakeSignatureVersion]
+    r"""Signature version to use for signing S3 requests"""
+    reuse_connections: NotRequired[bool]
+    r"""Reuse connections between requests, which can improve performance"""
+    reject_unauthorized: NotRequired[bool]
+    r"""Reject certificates that cannot be verified against a valid CA, such as self-signed certificates"""
+    enable_assume_role: NotRequired[bool]
+    r"""Use Assume Role credentials to access S3"""
+    assume_role_arn: NotRequired[str]
+    r"""Amazon Resource Name (ARN) of the role to assume"""
+    assume_role_external_id: NotRequired[str]
+    r"""External ID to use when assuming role"""
+    duration_seconds: NotRequired[float]
+    r"""Duration of the assumed role's session, in seconds. Minimum is 900 (15 minutes), default is 3600 (1 hour), and maximum is 43200 (12 hours)."""
+    stage_path: NotRequired[str]
+    r"""Filesystem location in which to buffer files, before compressing and moving to final destination. Use performant and stable storage."""
+    add_id_to_stage_path: NotRequired[bool]
+    r"""Add the Output ID value to staging location"""
+    dest_path: NotRequired[str]
+    r"""Lake dataset to send the data to."""
+    object_acl: NotRequired[OutputCriblLakeObjectACL]
+    r"""Object ACL to assign to uploaded objects"""
+    storage_class: NotRequired[OutputCriblLakeStorageClass]
+    r"""Storage class to select for uploaded objects"""
+    server_side_encryption: NotRequired[
+        OutputCriblLakeServerSideEncryptionForUploadedObjects
+    ]
+    kms_key_id: NotRequired[str]
+    r"""ID or ARN of the KMS customer-managed key to use for encryption"""
+    remove_empty_dirs: NotRequired[bool]
+    r"""Remove empty staging directories after moving files"""
+    base_file_name: NotRequired[str]
+    r"""JavaScript expression to define the output filename prefix (can be constant)"""
+    file_name_suffix: NotRequired[str]
+    r"""JavaScript expression to define the output filename suffix (can be constant).  The `__format` variable refers to the value of the `Data format` field (`json` or `raw`).  The `__compression` field refers to the kind of compression being used (`none` or `gzip`)."""
+    max_file_size_mb: NotRequired[float]
+    r"""Maximum uncompressed output file size. Files of this size will be closed and moved to final output location."""
+    max_open_files: NotRequired[float]
+    r"""Maximum number of files to keep open concurrently. When exceeded, @{product} will close the oldest open files and move them to the final output location."""
+    header_line: NotRequired[str]
+    r"""If set, this line will be written to the beginning of each output file"""
+    write_high_water_mark: NotRequired[float]
+    r"""Buffer size used to write to a file"""
+    on_backpressure: NotRequired[OutputCriblLakeBackpressureBehavior]
+    r"""How to handle events when all receivers are exerting backpressure"""
+    deadletter_enabled: NotRequired[bool]
+    r"""If a file fails to move to its final destination after the maximum number of retries, move it to a designated directory to prevent further errors"""
+    on_disk_full_backpressure: NotRequired[OutputCriblLakeDiskSpaceProtection]
+    r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
+    max_file_open_time_sec: NotRequired[float]
+    r"""Maximum amount of time to write to a file. Files open for longer than this will be closed and moved to final output location."""
+    max_file_idle_time_sec: NotRequired[float]
+    r"""Maximum amount of time to keep inactive files open. Files open for longer than this will be closed and moved to final output location."""
+    verify_permissions: NotRequired[bool]
+    r"""Disable if you can access files within the bucket but not the bucket itself"""
+    max_closing_files_to_backpressure: NotRequired[float]
+    r"""Maximum number of files that can be waiting for upload before backpressure is applied"""
+    aws_authentication_method: NotRequired[AwsAuthenticationMethod]
+    format_: NotRequired[OutputCriblLakeFormat]
+    max_concurrent_file_parts: NotRequired[float]
+    r"""Maximum number of parts to upload in parallel per file. Minimum part size is 5MB."""
+    description: NotRequired[str]
+    empty_dir_cleanup_sec: NotRequired[float]
+    r"""How frequently, in seconds, to clean up empty directories"""
+    deadletter_path: NotRequired[str]
+    r"""Storage location for files that fail to reach their final destination after maximum retries are exceeded"""
+    max_retry_num: NotRequired[float]
+    r"""The maximum number of times a file will attempt to move to its final destination before being dead-lettered"""
+
+
+class OutputCriblLake(BaseModel):
+    id: str
+    r"""Unique ID for this output"""
+
+    type: Annotated[OutputCriblLakeType, PlainValidator(validate_open_enum(False))]
+
+    pipeline: Optional[str] = None
+    r"""Pipeline to process data before sending out to this output"""
+
+    system_fields: Annotated[
+        Optional[List[str]], pydantic.Field(alias="systemFields")
+    ] = None
+    r"""Fields to automatically add to events, such as cribl_pipe. Supports wildcards."""
+
+    environment: Optional[str] = None
+    r"""Optionally, enable this config only on a specified Git branch. If empty, will be enabled everywhere."""
+
+    streamtags: Optional[List[str]] = None
+    r"""Tags for filtering and grouping in @{product}"""
+
+    bucket: Optional[str] = None
+    r"""Name of the destination S3 bucket. Must be a JavaScript expression (which can evaluate to a constant value), enclosed in quotes or backticks. Can be evaluated only at initialization time. Example referencing a Global Variable: `myBucket-${C.vars.myVar}`"""
+
+    region: Optional[str] = None
+    r"""Region where the S3 bucket is located"""
+
+    aws_secret_key: Annotated[Optional[str], pydantic.Field(alias="awsSecretKey")] = (
+        None
+    )
+    r"""Secret key. This value can be a constant or a JavaScript expression. Example: `${C.env.SOME_SECRET}`)"""
+
+    endpoint: Optional[str] = None
+    r"""S3 service endpoint. If empty, defaults to the AWS Region-specific endpoint. Otherwise, it must point to S3-compatible endpoint."""
+
+    signature_version: Annotated[
+        Annotated[
+            Optional[OutputCriblLakeSignatureVersion],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="signatureVersion"),
+    ] = OutputCriblLakeSignatureVersion.V4
+    r"""Signature version to use for signing S3 requests"""
+
+    reuse_connections: Annotated[
+        Optional[bool], pydantic.Field(alias="reuseConnections")
+    ] = True
+    r"""Reuse connections between requests, which can improve performance"""
+
+    reject_unauthorized: Annotated[
+        Optional[bool], pydantic.Field(alias="rejectUnauthorized")
+    ] = True
+    r"""Reject certificates that cannot be verified against a valid CA, such as self-signed certificates"""
+
+    enable_assume_role: Annotated[
+        Optional[bool], pydantic.Field(alias="enableAssumeRole")
+    ] = False
+    r"""Use Assume Role credentials to access S3"""
+
+    assume_role_arn: Annotated[Optional[str], pydantic.Field(alias="assumeRoleArn")] = (
+        None
+    )
+    r"""Amazon Resource Name (ARN) of the role to assume"""
+
+    assume_role_external_id: Annotated[
+        Optional[str], pydantic.Field(alias="assumeRoleExternalId")
+    ] = None
+    r"""External ID to use when assuming role"""
+
+    duration_seconds: Annotated[
+        Optional[float], pydantic.Field(alias="durationSeconds")
+    ] = 3600
+    r"""Duration of the assumed role's session, in seconds. Minimum is 900 (15 minutes), default is 3600 (1 hour), and maximum is 43200 (12 hours)."""
+
+    stage_path: Annotated[Optional[str], pydantic.Field(alias="stagePath")] = (
+        "$CRIBL_HOME/state/outputs/staging"
+    )
+    r"""Filesystem location in which to buffer files, before compressing and moving to final destination. Use performant and stable storage."""
+
+    add_id_to_stage_path: Annotated[
+        Optional[bool], pydantic.Field(alias="addIdToStagePath")
+    ] = True
+    r"""Add the Output ID value to staging location"""
+
+    dest_path: Annotated[Optional[str], pydantic.Field(alias="destPath")] = None
+    r"""Lake dataset to send the data to."""
+
+    object_acl: Annotated[
+        Annotated[
+            Optional[OutputCriblLakeObjectACL],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="objectACL"),
+    ] = OutputCriblLakeObjectACL.PRIVATE
+    r"""Object ACL to assign to uploaded objects"""
+
+    storage_class: Annotated[
+        Annotated[
+            Optional[OutputCriblLakeStorageClass],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="storageClass"),
+    ] = None
+    r"""Storage class to select for uploaded objects"""
+
+    server_side_encryption: Annotated[
+        Annotated[
+            Optional[OutputCriblLakeServerSideEncryptionForUploadedObjects],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="serverSideEncryption"),
+    ] = None
+
+    kms_key_id: Annotated[Optional[str], pydantic.Field(alias="kmsKeyId")] = None
+    r"""ID or ARN of the KMS customer-managed key to use for encryption"""
+
+    remove_empty_dirs: Annotated[
+        Optional[bool], pydantic.Field(alias="removeEmptyDirs")
+    ] = True
+    r"""Remove empty staging directories after moving files"""
+
+    base_file_name: Annotated[Optional[str], pydantic.Field(alias="baseFileName")] = (
+        "`CriblOut`"
+    )
+    r"""JavaScript expression to define the output filename prefix (can be constant)"""
+
+    file_name_suffix: Annotated[
+        Optional[str], pydantic.Field(alias="fileNameSuffix")
+    ] = '`.${C.env["CRIBL_WORKER_ID"]}.${__format}${__compression === "gzip" ? ".gz" : ""}`'
+    r"""JavaScript expression to define the output filename suffix (can be constant).  The `__format` variable refers to the value of the `Data format` field (`json` or `raw`).  The `__compression` field refers to the kind of compression being used (`none` or `gzip`)."""
+
+    max_file_size_mb: Annotated[
+        Optional[float], pydantic.Field(alias="maxFileSizeMB")
+    ] = 64
+    r"""Maximum uncompressed output file size. Files of this size will be closed and moved to final output location."""
+
+    max_open_files: Annotated[Optional[float], pydantic.Field(alias="maxOpenFiles")] = (
+        100
+    )
+    r"""Maximum number of files to keep open concurrently. When exceeded, @{product} will close the oldest open files and move them to the final output location."""
+
+    header_line: Annotated[Optional[str], pydantic.Field(alias="headerLine")] = ""
+    r"""If set, this line will be written to the beginning of each output file"""
+
+    write_high_water_mark: Annotated[
+        Optional[float], pydantic.Field(alias="writeHighWaterMark")
+    ] = 64
+    r"""Buffer size used to write to a file"""
+
+    on_backpressure: Annotated[
+        Annotated[
+            Optional[OutputCriblLakeBackpressureBehavior],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="onBackpressure"),
+    ] = OutputCriblLakeBackpressureBehavior.BLOCK
+    r"""How to handle events when all receivers are exerting backpressure"""
+
+    deadletter_enabled: Annotated[
+        Optional[bool], pydantic.Field(alias="deadletterEnabled")
+    ] = False
+    r"""If a file fails to move to its final destination after the maximum number of retries, move it to a designated directory to prevent further errors"""
+
+    on_disk_full_backpressure: Annotated[
+        Annotated[
+            Optional[OutputCriblLakeDiskSpaceProtection],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="onDiskFullBackpressure"),
+    ] = OutputCriblLakeDiskSpaceProtection.BLOCK
+    r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
+
+    max_file_open_time_sec: Annotated[
+        Optional[float], pydantic.Field(alias="maxFileOpenTimeSec")
+    ] = 300
+    r"""Maximum amount of time to write to a file. Files open for longer than this will be closed and moved to final output location."""
+
+    max_file_idle_time_sec: Annotated[
+        Optional[float], pydantic.Field(alias="maxFileIdleTimeSec")
+    ] = 300
+    r"""Maximum amount of time to keep inactive files open. Files open for longer than this will be closed and moved to final output location."""
+
+    verify_permissions: Annotated[
+        Optional[bool], pydantic.Field(alias="verifyPermissions")
+    ] = True
+    r"""Disable if you can access files within the bucket but not the bucket itself"""
+
+    max_closing_files_to_backpressure: Annotated[
+        Optional[float], pydantic.Field(alias="maxClosingFilesToBackpressure")
+    ] = 100
+    r"""Maximum number of files that can be waiting for upload before backpressure is applied"""
+
+    aws_authentication_method: Annotated[
+        Annotated[
+            Optional[AwsAuthenticationMethod], PlainValidator(validate_open_enum(False))
+        ],
+        pydantic.Field(alias="awsAuthenticationMethod"),
+    ] = AwsAuthenticationMethod.AUTO
+
+    format_: Annotated[
+        Annotated[
+            Optional[OutputCriblLakeFormat], PlainValidator(validate_open_enum(False))
+        ],
+        pydantic.Field(alias="format"),
+    ] = None
+
+    max_concurrent_file_parts: Annotated[
+        Optional[float], pydantic.Field(alias="maxConcurrentFileParts")
+    ] = 1
+    r"""Maximum number of parts to upload in parallel per file. Minimum part size is 5MB."""
+
+    description: Optional[str] = None
+
+    empty_dir_cleanup_sec: Annotated[
+        Optional[float], pydantic.Field(alias="emptyDirCleanupSec")
+    ] = 300
+    r"""How frequently, in seconds, to clean up empty directories"""
+
+    deadletter_path: Annotated[
+        Optional[str], pydantic.Field(alias="deadletterPath")
+    ] = "$CRIBL_HOME/state/outputs/dead-letter"
+    r"""Storage location for files that fail to reach their final destination after maximum retries are exceeded"""
+
+    max_retry_num: Annotated[Optional[float], pydantic.Field(alias="maxRetryNum")] = 20
+    r"""The maximum number of times a file will attempt to move to its final destination before being dead-lettered"""