cribl-control-plane 0.2.1rc11__py3-none-any.whl → 0.4.0a6__py3-none-any.whl

cribl_control_plane/models/outputmicrosoftfabric.py

@@ -0,0 +1,540 @@
+"""Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT."""
+
+from __future__ import annotations
+from cribl_control_plane import models, utils
+from cribl_control_plane.types import BaseModel
+from cribl_control_plane.utils import validate_open_enum
+from enum import Enum
+import pydantic
+from pydantic import field_serializer
+from pydantic.functional_validators import PlainValidator
+from typing import List, Optional
+from typing_extensions import Annotated, NotRequired, TypedDict
+
+
+class OutputMicrosoftFabricType(str, Enum):
+    MICROSOFT_FABRIC = "microsoft_fabric"
+
+
+class OutputMicrosoftFabricAcknowledgments(int, Enum, metaclass=utils.OpenEnumMeta):
+    r"""Control the number of required acknowledgments"""
+
+    # Leader
+    ONE = 1
+    # None
+    ZERO = 0
+    # All
+    MINUS_1 = -1
+
+
+class OutputMicrosoftFabricRecordDataFormat(str, Enum, metaclass=utils.OpenEnumMeta):
+    r"""Format to use to serialize events before writing to the Event Hubs Kafka brokers"""
+
+    # JSON
+    JSON = "json"
+    # Field _raw
+    RAW = "raw"
+
+
+class OutputMicrosoftFabricSASLMechanism(str, Enum, metaclass=utils.OpenEnumMeta):
+    # PLAIN
+    PLAIN = "plain"
+    # OAUTHBEARER
+    OAUTHBEARER = "oauthbearer"
+
+
+class OutputMicrosoftFabricAuthenticationMethod(
+    str, Enum, metaclass=utils.OpenEnumMeta
+):
+    SECRET = "secret"
+    CERTIFICATE = "certificate"
+
+
+class OutputMicrosoftFabricMicrosoftEntraIDAuthenticationEndpoint(
+    str, Enum, metaclass=utils.OpenEnumMeta
+):
+    r"""Endpoint used to acquire authentication tokens from Azure"""
+
+    HTTPS_LOGIN_MICROSOFTONLINE_COM = "https://login.microsoftonline.com"
+    HTTPS_LOGIN_MICROSOFTONLINE_US = "https://login.microsoftonline.us"
+    HTTPS_LOGIN_PARTNER_MICROSOFTONLINE_CN = "https://login.partner.microsoftonline.cn"
+
+
+class OutputMicrosoftFabricAuthenticationTypedDict(TypedDict):
+    r"""Authentication parameters to use when connecting to bootstrap server. Using TLS is highly recommended."""
+
+    disabled: NotRequired[bool]
+    mechanism: NotRequired[OutputMicrosoftFabricSASLMechanism]
+    username: NotRequired[str]
+    r"""The username for authentication. This should always be $ConnectionString."""
+    text_secret: NotRequired[str]
+    r"""Select or create a stored text secret corresponding to the SASL JASS Password Primary or Password Secondary"""
+    client_secret_auth_type: NotRequired[OutputMicrosoftFabricAuthenticationMethod]
+    client_text_secret: NotRequired[str]
+    r"""Select or create a stored text secret"""
+    certificate_name: NotRequired[str]
+    r"""Select or create a stored certificate"""
+    cert_path: NotRequired[str]
+    priv_key_path: NotRequired[str]
+    passphrase: NotRequired[str]
+    oauth_endpoint: NotRequired[
+        OutputMicrosoftFabricMicrosoftEntraIDAuthenticationEndpoint
+    ]
+    r"""Endpoint used to acquire authentication tokens from Azure"""
+    client_id: NotRequired[str]
+    r"""client_id to pass in the OAuth request parameter"""
+    tenant_id: NotRequired[str]
+    r"""Directory ID (tenant identifier) in Azure Active Directory"""
+    scope: NotRequired[str]
+    r"""Scope to pass in the OAuth request parameter"""
+
+
+class OutputMicrosoftFabricAuthentication(BaseModel):
+    r"""Authentication parameters to use when connecting to bootstrap server. Using TLS is highly recommended."""
+
+    disabled: Optional[bool] = False
+
+    mechanism: Annotated[
+        Optional[OutputMicrosoftFabricSASLMechanism],
+        PlainValidator(validate_open_enum(False)),
+    ] = OutputMicrosoftFabricSASLMechanism.PLAIN
+
+    username: Optional[str] = "$ConnectionString"
+    r"""The username for authentication. This should always be $ConnectionString."""
+
+    text_secret: Annotated[Optional[str], pydantic.Field(alias="textSecret")] = None
+    r"""Select or create a stored text secret corresponding to the SASL JASS Password Primary or Password Secondary"""
+
+    client_secret_auth_type: Annotated[
+        Annotated[
+            Optional[OutputMicrosoftFabricAuthenticationMethod],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="clientSecretAuthType"),
+    ] = OutputMicrosoftFabricAuthenticationMethod.SECRET
+
+    client_text_secret: Annotated[
+        Optional[str], pydantic.Field(alias="clientTextSecret")
+    ] = None
+    r"""Select or create a stored text secret"""
+
+    certificate_name: Annotated[
+        Optional[str], pydantic.Field(alias="certificateName")
+    ] = None
+    r"""Select or create a stored certificate"""
+
+    cert_path: Annotated[Optional[str], pydantic.Field(alias="certPath")] = None
+
+    priv_key_path: Annotated[Optional[str], pydantic.Field(alias="privKeyPath")] = None
+
+    passphrase: Optional[str] = None
+
+    oauth_endpoint: Annotated[
+        Annotated[
+            Optional[OutputMicrosoftFabricMicrosoftEntraIDAuthenticationEndpoint],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="oauthEndpoint"),
+    ] = OutputMicrosoftFabricMicrosoftEntraIDAuthenticationEndpoint.HTTPS_LOGIN_MICROSOFTONLINE_COM
+    r"""Endpoint used to acquire authentication tokens from Azure"""
+
+    client_id: Annotated[Optional[str], pydantic.Field(alias="clientId")] = None
+    r"""client_id to pass in the OAuth request parameter"""
+
+    tenant_id: Annotated[Optional[str], pydantic.Field(alias="tenantId")] = None
+    r"""Directory ID (tenant identifier) in Azure Active Directory"""
+
+    scope: Optional[str] = None
+    r"""Scope to pass in the OAuth request parameter"""
+
+    @field_serializer("mechanism")
+    def serialize_mechanism(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputMicrosoftFabricSASLMechanism(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("client_secret_auth_type")
+    def serialize_client_secret_auth_type(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputMicrosoftFabricAuthenticationMethod(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("oauth_endpoint")
+    def serialize_oauth_endpoint(self, value):
+        if isinstance(value, str):
+            try:
+                return (
+                    models.OutputMicrosoftFabricMicrosoftEntraIDAuthenticationEndpoint(
+                        value
+                    )
+                )
+            except ValueError:
+                return value
+        return value
+
+
+class OutputMicrosoftFabricTLSSettingsClientSideTypedDict(TypedDict):
+    disabled: NotRequired[bool]
+    reject_unauthorized: NotRequired[bool]
+    r"""Reject certificates that are not authorized by a CA in the CA certificate path, or by another trusted CA (such as the system's)"""
+
+
+class OutputMicrosoftFabricTLSSettingsClientSide(BaseModel):
+    disabled: Optional[bool] = False
+
+    reject_unauthorized: Annotated[
+        Optional[bool], pydantic.Field(alias="rejectUnauthorized")
+    ] = True
+    r"""Reject certificates that are not authorized by a CA in the CA certificate path, or by another trusted CA (such as the system's)"""
+
+
+class OutputMicrosoftFabricBackpressureBehavior(
+    str, Enum, metaclass=utils.OpenEnumMeta
+):
+    r"""How to handle events when all receivers are exerting backpressure"""
+
+    # Block
+    BLOCK = "block"
+    # Drop
+    DROP = "drop"
+    # Persistent Queue
+    QUEUE = "queue"
+
+
+class OutputMicrosoftFabricMode(str, Enum, metaclass=utils.OpenEnumMeta):
+    r"""In Error mode, PQ writes events to the filesystem if the Destination is unavailable. In Backpressure mode, PQ writes events to the filesystem when it detects backpressure from the Destination. In Always On mode, PQ always writes events to the filesystem."""
+
+    # Error
+    ERROR = "error"
+    # Backpressure
+    ALWAYS = "always"
+    # Always On
+    BACKPRESSURE = "backpressure"
+
+
+class OutputMicrosoftFabricCompression(str, Enum, metaclass=utils.OpenEnumMeta):
+    r"""Codec to use to compress the persisted data"""
+
+    # None
+    NONE = "none"
+    # Gzip
+    GZIP = "gzip"
+
+
+class OutputMicrosoftFabricQueueFullBehavior(str, Enum, metaclass=utils.OpenEnumMeta):
+    r"""How to handle events when the queue is exerting backpressure (full capacity or low disk). 'Block' is the same behavior as non-PQ blocking. 'Drop new data' throws away incoming data, while leaving the contents of the PQ unchanged."""
+
+    # Block
+    BLOCK = "block"
+    # Drop new data
+    DROP = "drop"
+
+
+class OutputMicrosoftFabricPqControlsTypedDict(TypedDict):
+    pass
+
+
+class OutputMicrosoftFabricPqControls(BaseModel):
+    pass
+
+
+class OutputMicrosoftFabricTypedDict(TypedDict):
+    type: OutputMicrosoftFabricType
+    topic: str
+    r"""Topic name from Fabric Eventstream's endpoint"""
+    bootstrap_server: str
+    r"""Bootstrap server from Fabric Eventstream's endpoint"""
+    id: NotRequired[str]
+    r"""Unique ID for this output"""
+    pipeline: NotRequired[str]
+    r"""Pipeline to process data before sending out to this output"""
+    system_fields: NotRequired[List[str]]
+    r"""Fields to automatically add to events, such as cribl_pipe. Supports wildcards."""
+    environment: NotRequired[str]
+    r"""Optionally, enable this config only on a specified Git branch. If empty, will be enabled everywhere."""
+    streamtags: NotRequired[List[str]]
+    r"""Tags for filtering and grouping in @{product}"""
+    ack: NotRequired[OutputMicrosoftFabricAcknowledgments]
+    r"""Control the number of required acknowledgments"""
+    format_: NotRequired[OutputMicrosoftFabricRecordDataFormat]
+    r"""Format to use to serialize events before writing to the Event Hubs Kafka brokers"""
+    max_record_size_kb: NotRequired[float]
+    r"""Maximum size of each record batch before compression. Setting should be < message.max.bytes settings in Event Hubs brokers."""
+    flush_event_count: NotRequired[float]
+    r"""Maximum number of events in a batch before forcing a flush"""
+    flush_period_sec: NotRequired[float]
+    r"""Maximum time between requests. Small values could cause the payload size to be smaller than the configured Max record size."""
+    connection_timeout: NotRequired[float]
+    r"""Maximum time to wait for a connection to complete successfully"""
+    request_timeout: NotRequired[float]
+    r"""Maximum time to wait for Kafka to respond to a request"""
+    max_retries: NotRequired[float]
+    r"""If messages are failing, you can set the maximum number of retries as high as 100 to prevent loss of data"""
+    max_back_off: NotRequired[float]
+    r"""The maximum wait time for a retry, in milliseconds. Default (and minimum) is 30,000 ms (30 seconds); maximum is 180,000 ms (180 seconds)."""
+    initial_backoff: NotRequired[float]
+    r"""Initial value used to calculate the retry, in milliseconds. Maximum is 600,000 ms (10 minutes)."""
+    backoff_rate: NotRequired[float]
+    r"""Set the backoff multiplier (2-20) to control the retry frequency for failed messages. For faster retries, use a lower multiplier. For slower retries with more delay between attempts, use a higher multiplier. The multiplier is used in an exponential backoff formula; see the Kafka [documentation](https://kafka.js.org/docs/retry-detailed) for details."""
+    authentication_timeout: NotRequired[float]
+    r"""Maximum time to wait for Kafka to respond to an authentication request"""
+    reauthentication_threshold: NotRequired[float]
+    r"""Specifies a time window during which @{product} can reauthenticate if needed. Creates the window measuring backward from the moment when credentials are set to expire."""
+    sasl: NotRequired[OutputMicrosoftFabricAuthenticationTypedDict]
+    r"""Authentication parameters to use when connecting to bootstrap server. Using TLS is highly recommended."""
+    tls: NotRequired[OutputMicrosoftFabricTLSSettingsClientSideTypedDict]
+    on_backpressure: NotRequired[OutputMicrosoftFabricBackpressureBehavior]
+    r"""How to handle events when all receivers are exerting backpressure"""
+    description: NotRequired[str]
+    pq_strict_ordering: NotRequired[bool]
+    r"""Use FIFO (first in, first out) processing. Disable to forward new events to receivers before queue is flushed."""
+    pq_rate_per_sec: NotRequired[float]
+    r"""Throttling rate (in events per second) to impose while writing to Destinations from PQ. Defaults to 0, which disables throttling."""
+    pq_mode: NotRequired[OutputMicrosoftFabricMode]
+    r"""In Error mode, PQ writes events to the filesystem if the Destination is unavailable. In Backpressure mode, PQ writes events to the filesystem when it detects backpressure from the Destination. In Always On mode, PQ always writes events to the filesystem."""
+    pq_max_buffer_size: NotRequired[float]
+    r"""The maximum number of events to hold in memory before writing the events to disk"""
+    pq_max_backpressure_sec: NotRequired[float]
+    r"""How long (in seconds) to wait for backpressure to resolve before engaging the queue"""
+    pq_max_file_size: NotRequired[str]
+    r"""The maximum size to store in each queue file before closing and optionally compressing (KB, MB, etc.)"""
+    pq_max_size: NotRequired[str]
+    r"""The maximum disk space that the queue can consume (as an average per Worker Process) before queueing stops. Enter a numeral with units of KB, MB, etc."""
+    pq_path: NotRequired[str]
+    r"""The location for the persistent queue files. To this field's value, the system will append: /<worker-id>/<output-id>."""
+    pq_compress: NotRequired[OutputMicrosoftFabricCompression]
+    r"""Codec to use to compress the persisted data"""
+    pq_on_backpressure: NotRequired[OutputMicrosoftFabricQueueFullBehavior]
+    r"""How to handle events when the queue is exerting backpressure (full capacity or low disk). 'Block' is the same behavior as non-PQ blocking. 'Drop new data' throws away incoming data, while leaving the contents of the PQ unchanged."""
+    pq_controls: NotRequired[OutputMicrosoftFabricPqControlsTypedDict]
+
+
+class OutputMicrosoftFabric(BaseModel):
+    type: OutputMicrosoftFabricType
+
+    topic: str
+    r"""Topic name from Fabric Eventstream's endpoint"""
+
+    bootstrap_server: str
+    r"""Bootstrap server from Fabric Eventstream's endpoint"""
+
+    id: Optional[str] = None
+    r"""Unique ID for this output"""
+
+    pipeline: Optional[str] = None
+    r"""Pipeline to process data before sending out to this output"""
+
+    system_fields: Annotated[
+        Optional[List[str]], pydantic.Field(alias="systemFields")
+    ] = None
+    r"""Fields to automatically add to events, such as cribl_pipe. Supports wildcards."""
+
+    environment: Optional[str] = None
+    r"""Optionally, enable this config only on a specified Git branch. If empty, will be enabled everywhere."""
+
+    streamtags: Optional[List[str]] = None
+    r"""Tags for filtering and grouping in @{product}"""
+
+    ack: Annotated[
+        Optional[OutputMicrosoftFabricAcknowledgments],
+        PlainValidator(validate_open_enum(True)),
+    ] = OutputMicrosoftFabricAcknowledgments.ONE
+    r"""Control the number of required acknowledgments"""
+
+    format_: Annotated[
+        Annotated[
+            Optional[OutputMicrosoftFabricRecordDataFormat],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="format"),
+    ] = OutputMicrosoftFabricRecordDataFormat.JSON
+    r"""Format to use to serialize events before writing to the Event Hubs Kafka brokers"""
+
+    max_record_size_kb: Annotated[
+        Optional[float], pydantic.Field(alias="maxRecordSizeKB")
+    ] = 768
+    r"""Maximum size of each record batch before compression. Setting should be < message.max.bytes settings in Event Hubs brokers."""
+
+    flush_event_count: Annotated[
+        Optional[float], pydantic.Field(alias="flushEventCount")
+    ] = 1000
+    r"""Maximum number of events in a batch before forcing a flush"""
+
+    flush_period_sec: Annotated[
+        Optional[float], pydantic.Field(alias="flushPeriodSec")
+    ] = 1
+    r"""Maximum time between requests. Small values could cause the payload size to be smaller than the configured Max record size."""
+
+    connection_timeout: Annotated[
+        Optional[float], pydantic.Field(alias="connectionTimeout")
+    ] = 10000
+    r"""Maximum time to wait for a connection to complete successfully"""
+
+    request_timeout: Annotated[
+        Optional[float], pydantic.Field(alias="requestTimeout")
+    ] = 60000
+    r"""Maximum time to wait for Kafka to respond to a request"""
+
+    max_retries: Annotated[Optional[float], pydantic.Field(alias="maxRetries")] = 5
+    r"""If messages are failing, you can set the maximum number of retries as high as 100 to prevent loss of data"""
+
+    max_back_off: Annotated[Optional[float], pydantic.Field(alias="maxBackOff")] = 30000
+    r"""The maximum wait time for a retry, in milliseconds. Default (and minimum) is 30,000 ms (30 seconds); maximum is 180,000 ms (180 seconds)."""
+
+    initial_backoff: Annotated[
+        Optional[float], pydantic.Field(alias="initialBackoff")
+    ] = 300
+    r"""Initial value used to calculate the retry, in milliseconds. Maximum is 600,000 ms (10 minutes)."""
+
+    backoff_rate: Annotated[Optional[float], pydantic.Field(alias="backoffRate")] = 2
+    r"""Set the backoff multiplier (2-20) to control the retry frequency for failed messages. For faster retries, use a lower multiplier. For slower retries with more delay between attempts, use a higher multiplier. The multiplier is used in an exponential backoff formula; see the Kafka [documentation](https://kafka.js.org/docs/retry-detailed) for details."""
+
+    authentication_timeout: Annotated[
+        Optional[float], pydantic.Field(alias="authenticationTimeout")
+    ] = 10000
+    r"""Maximum time to wait for Kafka to respond to an authentication request"""
+
+    reauthentication_threshold: Annotated[
+        Optional[float], pydantic.Field(alias="reauthenticationThreshold")
+    ] = 10000
+    r"""Specifies a time window during which @{product} can reauthenticate if needed. Creates the window measuring backward from the moment when credentials are set to expire."""
+
+    sasl: Optional[OutputMicrosoftFabricAuthentication] = None
+    r"""Authentication parameters to use when connecting to bootstrap server. Using TLS is highly recommended."""
+
+    tls: Optional[OutputMicrosoftFabricTLSSettingsClientSide] = None
+
+    on_backpressure: Annotated[
+        Annotated[
+            Optional[OutputMicrosoftFabricBackpressureBehavior],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="onBackpressure"),
+    ] = OutputMicrosoftFabricBackpressureBehavior.BLOCK
+    r"""How to handle events when all receivers are exerting backpressure"""
+
+    description: Optional[str] = None
+
+    pq_strict_ordering: Annotated[
+        Optional[bool], pydantic.Field(alias="pqStrictOrdering")
+    ] = True
+    r"""Use FIFO (first in, first out) processing. Disable to forward new events to receivers before queue is flushed."""
+
+    pq_rate_per_sec: Annotated[
+        Optional[float], pydantic.Field(alias="pqRatePerSec")
+    ] = 0
+    r"""Throttling rate (in events per second) to impose while writing to Destinations from PQ. Defaults to 0, which disables throttling."""
+
+    pq_mode: Annotated[
+        Annotated[
+            Optional[OutputMicrosoftFabricMode],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="pqMode"),
+    ] = OutputMicrosoftFabricMode.ERROR
+    r"""In Error mode, PQ writes events to the filesystem if the Destination is unavailable. In Backpressure mode, PQ writes events to the filesystem when it detects backpressure from the Destination. In Always On mode, PQ always writes events to the filesystem."""
+
+    pq_max_buffer_size: Annotated[
+        Optional[float], pydantic.Field(alias="pqMaxBufferSize")
+    ] = 42
+    r"""The maximum number of events to hold in memory before writing the events to disk"""
+
+    pq_max_backpressure_sec: Annotated[
+        Optional[float], pydantic.Field(alias="pqMaxBackpressureSec")
+    ] = 30
+    r"""How long (in seconds) to wait for backpressure to resolve before engaging the queue"""
+
+    pq_max_file_size: Annotated[
+        Optional[str], pydantic.Field(alias="pqMaxFileSize")
+    ] = "1 MB"
+    r"""The maximum size to store in each queue file before closing and optionally compressing (KB, MB, etc.)"""
+
+    pq_max_size: Annotated[Optional[str], pydantic.Field(alias="pqMaxSize")] = "5GB"
+    r"""The maximum disk space that the queue can consume (as an average per Worker Process) before queueing stops. Enter a numeral with units of KB, MB, etc."""
+
+    pq_path: Annotated[Optional[str], pydantic.Field(alias="pqPath")] = (
+        "$CRIBL_HOME/state/queues"
+    )
+    r"""The location for the persistent queue files. To this field's value, the system will append: /<worker-id>/<output-id>."""
+
+    pq_compress: Annotated[
+        Annotated[
+            Optional[OutputMicrosoftFabricCompression],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="pqCompress"),
+    ] = OutputMicrosoftFabricCompression.NONE
+    r"""Codec to use to compress the persisted data"""
+
+    pq_on_backpressure: Annotated[
+        Annotated[
+            Optional[OutputMicrosoftFabricQueueFullBehavior],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="pqOnBackpressure"),
+    ] = OutputMicrosoftFabricQueueFullBehavior.BLOCK
+    r"""How to handle events when the queue is exerting backpressure (full capacity or low disk). 'Block' is the same behavior as non-PQ blocking. 'Drop new data' throws away incoming data, while leaving the contents of the PQ unchanged."""
+
+    pq_controls: Annotated[
+        Optional[OutputMicrosoftFabricPqControls], pydantic.Field(alias="pqControls")
+    ] = None
+
+    @field_serializer("ack")
+    def serialize_ack(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputMicrosoftFabricAcknowledgments(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("format_")
+    def serialize_format_(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputMicrosoftFabricRecordDataFormat(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("on_backpressure")
+    def serialize_on_backpressure(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputMicrosoftFabricBackpressureBehavior(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("pq_mode")
+    def serialize_pq_mode(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputMicrosoftFabricMode(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("pq_compress")
+    def serialize_pq_compress(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputMicrosoftFabricCompression(value)
+            except ValueError:
+                return value
+        return value
+
+    @field_serializer("pq_on_backpressure")
+    def serialize_pq_on_backpressure(self, value):
+        if isinstance(value, str):
+            try:
+                return models.OutputMicrosoftFabricQueueFullBehavior(value)
+            except ValueError:
+                return value
+        return value

cribl_control_plane/models/outputminio.py

@@ -62,7 +62,7 @@ class OutputMinioStorageClass(str, Enum, metaclass=utils.OpenEnumMeta):
     REDUCED_REDUNDANCY = "REDUCED_REDUNDANCY"
 
 
-class ServerSideEncryption(str, Enum, metaclass=utils.OpenEnumMeta):
+class OutputMinioServerSideEncryption(str, Enum, metaclass=utils.OpenEnumMeta):
     r"""Server-side encryption for uploaded objects"""
 
     # Amazon S3 Managed Key
@@ -181,7 +181,7 @@ class OutputMinioTypedDict(TypedDict):
     r"""Object ACL to assign to uploaded objects"""
     storage_class: NotRequired[OutputMinioStorageClass]
     r"""Storage class to select for uploaded objects"""
-    server_side_encryption: NotRequired[ServerSideEncryption]
+    server_side_encryption: NotRequired[OutputMinioServerSideEncryption]
     r"""Server-side encryption for uploaded objects"""
     reuse_connections: NotRequired[bool]
     r"""Reuse connections between requests, which can improve performance"""
@@ -213,6 +213,8 @@ class OutputMinioTypedDict(TypedDict):
     r"""If a file fails to move to its final destination after the maximum number of retries, move it to a designated directory to prevent further errors"""
     on_disk_full_backpressure: NotRequired[OutputMinioDiskSpaceProtection]
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
+    force_close_on_shutdown: NotRequired[bool]
+    r"""Force all staged files to close during an orderly Node shutdown. This triggers immediate upload of in-progress data — regardless of idle time, file age, or size thresholds — to minimize data loss."""
     max_file_open_time_sec: NotRequired[float]
     r"""Maximum amount of time to write to a file. Files open for longer than this will be closed and moved to final output location."""
     max_file_idle_time_sec: NotRequired[float]
@@ -252,6 +254,8 @@ class OutputMinioTypedDict(TypedDict):
     r"""Parquet tools can use the checksum of a Parquet page to verify data integrity"""
     empty_dir_cleanup_sec: NotRequired[float]
     r"""How frequently, in seconds, to clean up empty directories"""
+    directory_batch_size: NotRequired[float]
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
     deadletter_path: NotRequired[str]
     r"""Storage location for files that fail to reach their final destination after maximum retries are exceeded"""
     max_retry_num: NotRequired[float]
@@ -341,7 +345,8 @@ class OutputMinio(BaseModel):
 
     server_side_encryption: Annotated[
         Annotated[
-            Optional[ServerSideEncryption], PlainValidator(validate_open_enum(False))
+            Optional[OutputMinioServerSideEncryption],
+            PlainValidator(validate_open_enum(False)),
         ],
         pydantic.Field(alias="serverSideEncryption"),
     ] = None
@@ -431,6 +436,11 @@ class OutputMinio(BaseModel):
     ] = OutputMinioDiskSpaceProtection.BLOCK
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
 
+    force_close_on_shutdown: Annotated[
+        Optional[bool], pydantic.Field(alias="forceCloseOnShutdown")
+    ] = False
+    r"""Force all staged files to close during an orderly Node shutdown. This triggers immediate upload of in-progress data — regardless of idle time, file age, or size thresholds — to minimize data loss."""
+
     max_file_open_time_sec: Annotated[
         Optional[float], pydantic.Field(alias="maxFileOpenTimeSec")
     ] = 300
@@ -537,6 +547,11 @@ class OutputMinio(BaseModel):
     ] = 300
     r"""How frequently, in seconds, to clean up empty directories"""
 
+    directory_batch_size: Annotated[
+        Optional[float], pydantic.Field(alias="directoryBatchSize")
+    ] = 1000
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
+
     deadletter_path: Annotated[
         Optional[str], pydantic.Field(alias="deadletterPath")
     ] = "$CRIBL_HOME/state/outputs/dead-letter"
@@ -585,7 +600,7 @@ class OutputMinio(BaseModel):
     def serialize_server_side_encryption(self, value):
         if isinstance(value, str):
             try:
-                return models.ServerSideEncryption(value)
+                return models.OutputMinioServerSideEncryption(value)
             except ValueError:
                 return value
         return value

cribl_control_plane/models/outputs3.py

@@ -234,6 +234,8 @@ class OutputS3TypedDict(TypedDict):
     r"""If a file fails to move to its final destination after the maximum number of retries, move it to a designated directory to prevent further errors"""
     on_disk_full_backpressure: NotRequired[OutputS3DiskSpaceProtection]
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
+    force_close_on_shutdown: NotRequired[bool]
+    r"""Force all staged files to close during an orderly Node shutdown. This triggers immediate upload of in-progress data — regardless of idle time, file age, or size thresholds — to minimize data loss."""
     max_file_open_time_sec: NotRequired[float]
     r"""Maximum amount of time to write to a file. Files open for longer than this will be closed and moved to final output location."""
     max_file_idle_time_sec: NotRequired[float]
@@ -277,6 +279,8 @@ class OutputS3TypedDict(TypedDict):
     r"""Parquet tools can use the checksum of a Parquet page to verify data integrity"""
     empty_dir_cleanup_sec: NotRequired[float]
     r"""How frequently, in seconds, to clean up empty directories"""
+    directory_batch_size: NotRequired[float]
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
     deadletter_path: NotRequired[str]
     r"""Storage location for files that fail to reach their final destination after maximum retries are exceeded"""
     max_retry_num: NotRequired[float]
@@ -474,6 +478,11 @@ class OutputS3(BaseModel):
     ] = OutputS3DiskSpaceProtection.BLOCK
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
 
+    force_close_on_shutdown: Annotated[
+        Optional[bool], pydantic.Field(alias="forceCloseOnShutdown")
+    ] = False
+    r"""Force all staged files to close during an orderly Node shutdown. This triggers immediate upload of in-progress data — regardless of idle time, file age, or size thresholds — to minimize data loss."""
+
     max_file_open_time_sec: Annotated[
         Optional[float], pydantic.Field(alias="maxFileOpenTimeSec")
     ] = 300
@@ -588,6 +597,11 @@ class OutputS3(BaseModel):
     ] = 300
     r"""How frequently, in seconds, to clean up empty directories"""
 
+    directory_batch_size: Annotated[
+        Optional[float], pydantic.Field(alias="directoryBatchSize")
+    ] = 1000
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
+
     deadletter_path: Annotated[
         Optional[str], pydantic.Field(alias="deadletterPath")
     ] = "$CRIBL_HOME/state/outputs/dead-letter"