cribl-control-plane 0.0.21__py3-none-any.whl → 0.4.0b23__py3-none-any.whl

cribl_control_plane/models/outputdls3.py

@@ -1,9 +1,13 @@
 """Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT."""
 
 from __future__ import annotations
+from cribl_control_plane import models, utils
 from cribl_control_plane.types import BaseModel
+from cribl_control_plane.utils import validate_open_enum
 from enum import Enum
 import pydantic
+from pydantic import field_serializer
+from pydantic.functional_validators import PlainValidator
 from typing import List, Optional
 from typing_extensions import Annotated, NotRequired, TypedDict
 
@@ -12,100 +16,137 @@ class OutputDlS3Type(str, Enum):
     DL_S3 = "dl_s3"
 
 
-class OutputDlS3AuthenticationMethod(str, Enum):
+class OutputDlS3AuthenticationMethod(str, Enum, metaclass=utils.OpenEnumMeta):
     r"""AWS authentication method. Choose Auto to use IAM roles."""
 
+    # Auto
     AUTO = "auto"
+    # Manual
     MANUAL = "manual"
+    # Secret Key pair
     SECRET = "secret"
 
 
-class OutputDlS3SignatureVersion(str, Enum):
+class OutputDlS3SignatureVersion(str, Enum, metaclass=utils.OpenEnumMeta):
     r"""Signature version to use for signing S3 requests"""
 
     V2 = "v2"
     V4 = "v4"
 
 
-class OutputDlS3ObjectACL(str, Enum):
+class OutputDlS3ObjectACL(str, Enum, metaclass=utils.OpenEnumMeta):
     r"""Object ACL to assign to uploaded objects"""
 
+    # Private
     PRIVATE = "private"
+    # Public Read Only
     PUBLIC_READ = "public-read"
+    # Public Read/Write
     PUBLIC_READ_WRITE = "public-read-write"
+    # Authenticated Read Only
     AUTHENTICATED_READ = "authenticated-read"
+    # AWS EC2 AMI Read Only
     AWS_EXEC_READ = "aws-exec-read"
+    # Bucket Owner Read Only
     BUCKET_OWNER_READ = "bucket-owner-read"
+    # Bucket Owner Full Control
     BUCKET_OWNER_FULL_CONTROL = "bucket-owner-full-control"
 
 
-class OutputDlS3StorageClass(str, Enum):
+class OutputDlS3StorageClass(str, Enum, metaclass=utils.OpenEnumMeta):
     r"""Storage class to select for uploaded objects"""
 
+    # Standard
     STANDARD = "STANDARD"
+    # Reduced Redundancy Storage
     REDUCED_REDUNDANCY = "REDUCED_REDUNDANCY"
+    # Standard, Infrequent Access
     STANDARD_IA = "STANDARD_IA"
+    # One Zone, Infrequent Access
     ONEZONE_IA = "ONEZONE_IA"
+    # Intelligent Tiering
     INTELLIGENT_TIERING = "INTELLIGENT_TIERING"
+    # Glacier Flexible Retrieval
     GLACIER = "GLACIER"
+    # Glacier Instant Retrieval
     GLACIER_IR = "GLACIER_IR"
+    # Glacier Deep Archive
     DEEP_ARCHIVE = "DEEP_ARCHIVE"
 
 
-class OutputDlS3ServerSideEncryptionForUploadedObjects(str, Enum):
+class OutputDlS3ServerSideEncryptionForUploadedObjects(
+    str, Enum, metaclass=utils.OpenEnumMeta
+):
+    # Amazon S3 Managed Key
     AES256 = "AES256"
+    # AWS KMS Managed Key
     AWS_KMS = "aws:kms"
 
 
-class OutputDlS3DataFormat(str, Enum):
+class OutputDlS3DataFormat(str, Enum, metaclass=utils.OpenEnumMeta):
     r"""Format of the output data"""
 
+    # JSON
     JSON = "json"
+    # Raw
     RAW = "raw"
+    # Parquet
     PARQUET = "parquet"
 
 
-class OutputDlS3BackpressureBehavior(str, Enum):
+class OutputDlS3BackpressureBehavior(str, Enum, metaclass=utils.OpenEnumMeta):
     r"""How to handle events when all receivers are exerting backpressure"""
 
+    # Block
     BLOCK = "block"
+    # Drop
     DROP = "drop"
 
 
-class OutputDlS3DiskSpaceProtection(str, Enum):
+class OutputDlS3DiskSpaceProtection(str, Enum, metaclass=utils.OpenEnumMeta):
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
 
+    # Block
     BLOCK = "block"
+    # Drop
     DROP = "drop"
 
 
-class OutputDlS3Compression(str, Enum):
+class OutputDlS3Compression(str, Enum, metaclass=utils.OpenEnumMeta):
     r"""Data compression format to apply to HTTP content before it is delivered"""
 
     NONE = "none"
     GZIP = "gzip"
 
 
-class OutputDlS3CompressionLevel(str, Enum):
+class OutputDlS3CompressionLevel(str, Enum, metaclass=utils.OpenEnumMeta):
     r"""Compression level to apply before moving files to final destination"""
 
+    # Best Speed
     BEST_SPEED = "best_speed"
+    # Normal
     NORMAL = "normal"
+    # Best Compression
     BEST_COMPRESSION = "best_compression"
 
 
-class OutputDlS3ParquetVersion(str, Enum):
+class OutputDlS3ParquetVersion(str, Enum, metaclass=utils.OpenEnumMeta):
     r"""Determines which data types are supported and how they are represented"""
 
+    # 1.0
     PARQUET_1_0 = "PARQUET_1_0"
+    # 2.4
     PARQUET_2_4 = "PARQUET_2_4"
+    # 2.6
     PARQUET_2_6 = "PARQUET_2_6"
 
 
-class OutputDlS3DataPageVersion(str, Enum):
+class OutputDlS3DataPageVersion(str, Enum, metaclass=utils.OpenEnumMeta):
     r"""Serialization format of data pages. Note that some reader implementations use Data page V2's attributes to work more efficiently, while others ignore it."""
 
+    # V1
     DATA_PAGE_V1 = "DATA_PAGE_V1"
+    # V2
     DATA_PAGE_V2 = "DATA_PAGE_V2"
 
 
@@ -121,11 +162,11 @@ class OutputDlS3KeyValueMetadatum(BaseModel):
 
 
 class OutputDlS3TypedDict(TypedDict):
+    type: OutputDlS3Type
     bucket: str
     r"""Name of the destination S3 bucket. Must be a JavaScript expression (which can evaluate to a constant value), enclosed in quotes or backticks. Can be evaluated only at initialization time. Example referencing a Global Variable: `myBucket-${C.vars.myVar}`"""
     id: NotRequired[str]
     r"""Unique ID for this output"""
-    type: NotRequired[OutputDlS3Type]
     pipeline: NotRequired[str]
     r"""Pipeline to process data before sending out to this output"""
     system_fields: NotRequired[List[str]]
@@ -161,7 +202,7 @@ class OutputDlS3TypedDict(TypedDict):
     add_id_to_stage_path: NotRequired[bool]
     r"""Add the Output ID value to staging location"""
     dest_path: NotRequired[str]
-    r"""Prefix to append to files before uploading. Must be a JavaScript expression (which can evaluate to a constant value), enclosed in quotes or backticks. Can be evaluated only at init time. Example referencing a Global Variable: `myKeyPrefix-${C.vars.myVar}`"""
+    r"""Prefix to prepend to files before uploading. Must be a JavaScript expression (which can evaluate to a constant value), enclosed in quotes or backticks. Can be evaluated only at init time. Example referencing a Global Variable: `myKeyPrefix-${C.vars.myVar}`"""
     object_acl: NotRequired[OutputDlS3ObjectACL]
     r"""Object ACL to assign to uploaded objects"""
     storage_class: NotRequired[OutputDlS3StorageClass]
@@ -193,6 +234,8 @@ class OutputDlS3TypedDict(TypedDict):
     r"""If a file fails to move to its final destination after the maximum number of retries, move it to a designated directory to prevent further errors"""
     on_disk_full_backpressure: NotRequired[OutputDlS3DiskSpaceProtection]
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
+    force_close_on_shutdown: NotRequired[bool]
+    r"""Force all staged files to close during an orderly Node shutdown. This triggers immediate upload of in-progress data — regardless of idle time, file age, or size thresholds — to minimize data loss."""
     max_file_open_time_sec: NotRequired[float]
     r"""Maximum amount of time to write to a file. Files open for longer than this will be closed and moved to final output location."""
     max_file_idle_time_sec: NotRequired[float]
@@ -216,6 +259,8 @@ class OutputDlS3TypedDict(TypedDict):
     r"""Compression level to apply before moving files to final destination"""
     automatic_schema: NotRequired[bool]
     r"""Automatically calculate the schema based on the events of each Parquet file generated"""
+    parquet_schema: NotRequired[str]
+    r"""To add a new schema, navigate to Processing > Knowledge > Parquet Schemas"""
     parquet_version: NotRequired[OutputDlS3ParquetVersion]
     r"""Determines which data types are supported and how they are represented"""
     parquet_data_page_version: NotRequired[OutputDlS3DataPageVersion]
@@ -236,6 +281,8 @@ class OutputDlS3TypedDict(TypedDict):
     r"""Parquet tools can use the checksum of a Parquet page to verify data integrity"""
     empty_dir_cleanup_sec: NotRequired[float]
     r"""How frequently, in seconds, to clean up empty directories"""
+    directory_batch_size: NotRequired[float]
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
     deadletter_path: NotRequired[str]
     r"""Storage location for files that fail to reach their final destination after maximum retries are exceeded"""
     max_retry_num: NotRequired[float]
@@ -243,14 +290,14 @@ class OutputDlS3TypedDict(TypedDict):
 
 
 class OutputDlS3(BaseModel):
+    type: OutputDlS3Type
+
     bucket: str
     r"""Name of the destination S3 bucket. Must be a JavaScript expression (which can evaluate to a constant value), enclosed in quotes or backticks. Can be evaluated only at initialization time. Example referencing a Global Variable: `myBucket-${C.vars.myVar}`"""
 
     id: Optional[str] = None
     r"""Unique ID for this output"""
 
-    type: Optional[OutputDlS3Type] = None
-
     pipeline: Optional[str] = None
     r"""Pipeline to process data before sending out to this output"""
 
@@ -274,7 +321,10 @@ class OutputDlS3(BaseModel):
     r"""Secret key. This value can be a constant or a JavaScript expression. Example: `${C.env.SOME_SECRET}`)"""
 
     aws_authentication_method: Annotated[
-        Optional[OutputDlS3AuthenticationMethod],
+        Annotated[
+            Optional[OutputDlS3AuthenticationMethod],
+            PlainValidator(validate_open_enum(False)),
+        ],
         pydantic.Field(alias="awsAuthenticationMethod"),
     ] = OutputDlS3AuthenticationMethod.AUTO
     r"""AWS authentication method. Choose Auto to use IAM roles."""
@@ -283,7 +333,11 @@ class OutputDlS3(BaseModel):
     r"""S3 service endpoint. If empty, defaults to the AWS Region-specific endpoint. Otherwise, it must point to S3-compatible endpoint."""
 
     signature_version: Annotated[
-        Optional[OutputDlS3SignatureVersion], pydantic.Field(alias="signatureVersion")
+        Annotated[
+            Optional[OutputDlS3SignatureVersion],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="signatureVersion"),
     ] = OutputDlS3SignatureVersion.V4
     r"""Signature version to use for signing S3 requests"""
 
@@ -328,20 +382,29 @@ class OutputDlS3(BaseModel):
     r"""Add the Output ID value to staging location"""
 
     dest_path: Annotated[Optional[str], pydantic.Field(alias="destPath")] = ""
-    r"""Prefix to append to files before uploading. Must be a JavaScript expression (which can evaluate to a constant value), enclosed in quotes or backticks. Can be evaluated only at init time. Example referencing a Global Variable: `myKeyPrefix-${C.vars.myVar}`"""
+    r"""Prefix to prepend to files before uploading. Must be a JavaScript expression (which can evaluate to a constant value), enclosed in quotes or backticks. Can be evaluated only at init time. Example referencing a Global Variable: `myKeyPrefix-${C.vars.myVar}`"""
 
     object_acl: Annotated[
-        Optional[OutputDlS3ObjectACL], pydantic.Field(alias="objectACL")
+        Annotated[
+            Optional[OutputDlS3ObjectACL], PlainValidator(validate_open_enum(False))
+        ],
+        pydantic.Field(alias="objectACL"),
     ] = OutputDlS3ObjectACL.PRIVATE
     r"""Object ACL to assign to uploaded objects"""
 
     storage_class: Annotated[
-        Optional[OutputDlS3StorageClass], pydantic.Field(alias="storageClass")
+        Annotated[
+            Optional[OutputDlS3StorageClass], PlainValidator(validate_open_enum(False))
+        ],
+        pydantic.Field(alias="storageClass"),
     ] = None
     r"""Storage class to select for uploaded objects"""
 
     server_side_encryption: Annotated[
-        Optional[OutputDlS3ServerSideEncryptionForUploadedObjects],
+        Annotated[
+            Optional[OutputDlS3ServerSideEncryptionForUploadedObjects],
+            PlainValidator(validate_open_enum(False)),
+        ],
         pydantic.Field(alias="serverSideEncryption"),
     ] = None
 
@@ -354,7 +417,10 @@ class OutputDlS3(BaseModel):
     r"""Remove empty staging directories after moving files"""
 
     format_: Annotated[
-        Optional[OutputDlS3DataFormat], pydantic.Field(alias="format")
+        Annotated[
+            Optional[OutputDlS3DataFormat], PlainValidator(validate_open_enum(False))
+        ],
+        pydantic.Field(alias="format"),
     ] = OutputDlS3DataFormat.JSON
     r"""Format of the output data"""
 
@@ -387,7 +453,11 @@ class OutputDlS3(BaseModel):
     r"""Buffer size used to write to a file"""
 
     on_backpressure: Annotated[
-        Optional[OutputDlS3BackpressureBehavior], pydantic.Field(alias="onBackpressure")
+        Annotated[
+            Optional[OutputDlS3BackpressureBehavior],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="onBackpressure"),
     ] = OutputDlS3BackpressureBehavior.BLOCK
     r"""How to handle events when all receivers are exerting backpressure"""
 
@@ -397,11 +467,19 @@ class OutputDlS3(BaseModel):
     r"""If a file fails to move to its final destination after the maximum number of retries, move it to a designated directory to prevent further errors"""
 
     on_disk_full_backpressure: Annotated[
-        Optional[OutputDlS3DiskSpaceProtection],
+        Annotated[
+            Optional[OutputDlS3DiskSpaceProtection],
+            PlainValidator(validate_open_enum(False)),
+        ],
         pydantic.Field(alias="onDiskFullBackpressure"),
     ] = OutputDlS3DiskSpaceProtection.BLOCK
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
 
+    force_close_on_shutdown: Annotated[
+        Optional[bool], pydantic.Field(alias="forceCloseOnShutdown")
+    ] = False
+    r"""Force all staged files to close during an orderly Node shutdown. This triggers immediate upload of in-progress data — regardless of idle time, file age, or size thresholds — to minimize data loss."""
+
     max_file_open_time_sec: Annotated[
         Optional[float], pydantic.Field(alias="maxFileOpenTimeSec")
     ] = 300
@@ -440,11 +518,17 @@ class OutputDlS3(BaseModel):
     aws_secret: Annotated[Optional[str], pydantic.Field(alias="awsSecret")] = None
     r"""Select or create a stored secret that references your access key and secret key"""
 
-    compress: Optional[OutputDlS3Compression] = OutputDlS3Compression.GZIP
+    compress: Annotated[
+        Optional[OutputDlS3Compression], PlainValidator(validate_open_enum(False))
+    ] = OutputDlS3Compression.GZIP
     r"""Data compression format to apply to HTTP content before it is delivered"""
 
     compression_level: Annotated[
-        Optional[OutputDlS3CompressionLevel], pydantic.Field(alias="compressionLevel")
+        Annotated[
+            Optional[OutputDlS3CompressionLevel],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="compressionLevel"),
     ] = OutputDlS3CompressionLevel.BEST_SPEED
     r"""Compression level to apply before moving files to final destination"""
 
@@ -453,13 +537,25 @@ class OutputDlS3(BaseModel):
     ] = False
     r"""Automatically calculate the schema based on the events of each Parquet file generated"""
 
+    parquet_schema: Annotated[Optional[str], pydantic.Field(alias="parquetSchema")] = (
+        None
+    )
+    r"""To add a new schema, navigate to Processing > Knowledge > Parquet Schemas"""
+
     parquet_version: Annotated[
-        Optional[OutputDlS3ParquetVersion], pydantic.Field(alias="parquetVersion")
+        Annotated[
+            Optional[OutputDlS3ParquetVersion],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="parquetVersion"),
     ] = OutputDlS3ParquetVersion.PARQUET_2_6
     r"""Determines which data types are supported and how they are represented"""
 
     parquet_data_page_version: Annotated[
-        Optional[OutputDlS3DataPageVersion],
+        Annotated[
+            Optional[OutputDlS3DataPageVersion],
+            PlainValidator(validate_open_enum(False)),
+        ],
         pydantic.Field(alias="parquetDataPageVersion"),
     ] = OutputDlS3DataPageVersion.DATA_PAGE_V2
     r"""Serialization format of data pages. Note that some reader implementations use Data page V2's attributes to work more efficiently, while others ignore it."""
@@ -505,6 +601,11 @@ class OutputDlS3(BaseModel):
     ] = 300
     r"""How frequently, in seconds, to clean up empty directories"""
 
+    directory_batch_size: Annotated[
+        Optional[float], pydantic.Field(alias="directoryBatchSize")
+    ] = 1000
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
+
     deadletter_path: Annotated[
         Optional[str], pydantic.Field(alias="deadletterPath")
     ] = "$CRIBL_HOME/state/outputs/dead-letter"
@@ -512,3 +613,111 @@ class OutputDlS3(BaseModel):
 
     max_retry_num: Annotated[Optional[float], pydantic.Field(alias="maxRetryNum")] = 20
     r"""The maximum number of times a file will attempt to move to its final destination before being dead-lettered"""
+
+    @field_serializer("aws_authentication_method")
+    def serialize_aws_authentication_method(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputDlS3AuthenticationMethod(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("signature_version")
+    def serialize_signature_version(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputDlS3SignatureVersion(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("object_acl")
+    def serialize_object_acl(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputDlS3ObjectACL(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("storage_class")
+    def serialize_storage_class(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputDlS3StorageClass(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("server_side_encryption")
+    def serialize_server_side_encryption(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputDlS3ServerSideEncryptionForUploadedObjects(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("format_")
+    def serialize_format_(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputDlS3DataFormat(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("on_backpressure")
+    def serialize_on_backpressure(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputDlS3BackpressureBehavior(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("on_disk_full_backpressure")
+    def serialize_on_disk_full_backpressure(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputDlS3DiskSpaceProtection(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("compress")
+    def serialize_compress(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputDlS3Compression(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("compression_level")
+    def serialize_compression_level(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputDlS3CompressionLevel(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("parquet_version")
+    def serialize_parquet_version(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputDlS3ParquetVersion(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("parquet_data_page_version")
+    def serialize_parquet_data_page_version(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputDlS3DataPageVersion(value)
+            except ValueError:
+                return value
+        return value