cribl-control-plane 0.2.1rc11__py3-none-any.whl → 0.4.0a12__py3-none-any.whl

cribl_control_plane/models/outputdatabricks.py

@@ -146,6 +146,8 @@ class OutputDatabricksTypedDict(TypedDict):
     r"""If a file fails to move to its final destination after the maximum number of retries, move it to a designated directory to prevent further errors"""
     on_disk_full_backpressure: NotRequired[OutputDatabricksDiskSpaceProtection]
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
+    force_close_on_shutdown: NotRequired[bool]
+    r"""Force all staged files to close during an orderly Node shutdown. This triggers immediate upload of in-progress data — regardless of idle time, file age, or size thresholds — to minimize data loss."""
     scope: NotRequired[str]
     r"""OAuth scope for Unity Catalog authentication"""
     catalog: NotRequired[str]
@@ -154,6 +156,8 @@ class OutputDatabricksTypedDict(TypedDict):
     r"""Name of the catalog schema to use for the output"""
     events_volume_name: NotRequired[str]
     r"""Name of the events volume in Databricks"""
+    timeout_sec: NotRequired[float]
+    r"""Amount of time, in seconds, to wait for a request to complete before canceling it"""
     description: NotRequired[str]
     compress: NotRequired[OutputDatabricksCompression]
     r"""Data compression format to apply to HTTP content before it is delivered"""
@@ -183,6 +187,8 @@ class OutputDatabricksTypedDict(TypedDict):
     r"""Parquet tools can use the checksum of a Parquet page to verify data integrity"""
     empty_dir_cleanup_sec: NotRequired[float]
     r"""How frequently, in seconds, to clean up empty directories"""
+    directory_batch_size: NotRequired[float]
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
     deadletter_path: NotRequired[str]
     r"""Storage location for files that fail to reach their final destination after maximum retries are exceeded"""
     max_retry_num: NotRequired[float]
@@ -311,6 +317,11 @@ class OutputDatabricks(BaseModel):
     ] = OutputDatabricksDiskSpaceProtection.BLOCK
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
 
+    force_close_on_shutdown: Annotated[
+        Optional[bool], pydantic.Field(alias="forceCloseOnShutdown")
+    ] = False
+    r"""Force all staged files to close during an orderly Node shutdown. This triggers immediate upload of in-progress data — regardless of idle time, file age, or size thresholds — to minimize data loss."""
+
     scope: Optional[str] = "all-apis"
     r"""OAuth scope for Unity Catalog authentication"""
 
@@ -325,6 +336,9 @@ class OutputDatabricks(BaseModel):
     ] = "events"
     r"""Name of the events volume in Databricks"""
 
+    timeout_sec: Annotated[Optional[float], pydantic.Field(alias="timeoutSec")] = 60
+    r"""Amount of time, in seconds, to wait for a request to complete before canceling it"""
+
     description: Optional[str] = None
 
     compress: Annotated[
@@ -410,6 +424,11 @@ class OutputDatabricks(BaseModel):
     ] = 300
     r"""How frequently, in seconds, to clean up empty directories"""
 
+    directory_batch_size: Annotated[
+        Optional[float], pydantic.Field(alias="directoryBatchSize")
+    ] = 1000
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
+
     deadletter_path: Annotated[
         Optional[str], pydantic.Field(alias="deadletterPath")
     ] = "$CRIBL_HOME/state/outputs/dead-letter"

cribl_control_plane/models/outputdls3.py

@@ -234,6 +234,8 @@ class OutputDlS3TypedDict(TypedDict):
     r"""If a file fails to move to its final destination after the maximum number of retries, move it to a designated directory to prevent further errors"""
     on_disk_full_backpressure: NotRequired[OutputDlS3DiskSpaceProtection]
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
+    force_close_on_shutdown: NotRequired[bool]
+    r"""Force all staged files to close during an orderly Node shutdown. This triggers immediate upload of in-progress data — regardless of idle time, file age, or size thresholds — to minimize data loss."""
     max_file_open_time_sec: NotRequired[float]
     r"""Maximum amount of time to write to a file. Files open for longer than this will be closed and moved to final output location."""
     max_file_idle_time_sec: NotRequired[float]
@@ -279,6 +281,8 @@ class OutputDlS3TypedDict(TypedDict):
     r"""Parquet tools can use the checksum of a Parquet page to verify data integrity"""
     empty_dir_cleanup_sec: NotRequired[float]
     r"""How frequently, in seconds, to clean up empty directories"""
+    directory_batch_size: NotRequired[float]
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
     deadletter_path: NotRequired[str]
     r"""Storage location for files that fail to reach their final destination after maximum retries are exceeded"""
     max_retry_num: NotRequired[float]
@@ -471,6 +475,11 @@ class OutputDlS3(BaseModel):
     ] = OutputDlS3DiskSpaceProtection.BLOCK
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
 
+    force_close_on_shutdown: Annotated[
+        Optional[bool], pydantic.Field(alias="forceCloseOnShutdown")
+    ] = False
+    r"""Force all staged files to close during an orderly Node shutdown. This triggers immediate upload of in-progress data — regardless of idle time, file age, or size thresholds — to minimize data loss."""
+
     max_file_open_time_sec: Annotated[
         Optional[float], pydantic.Field(alias="maxFileOpenTimeSec")
     ] = 300
@@ -592,6 +601,11 @@ class OutputDlS3(BaseModel):
     ] = 300
     r"""How frequently, in seconds, to clean up empty directories"""
 
+    directory_batch_size: Annotated[
+        Optional[float], pydantic.Field(alias="directoryBatchSize")
+    ] = 1000
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
+
     deadletter_path: Annotated[
         Optional[str], pydantic.Field(alias="deadletterPath")
     ] = "$CRIBL_HOME/state/outputs/dead-letter"

cribl_control_plane/models/outputexabeam.py

@@ -137,6 +137,8 @@ class OutputExabeamTypedDict(TypedDict):
     description: NotRequired[str]
     empty_dir_cleanup_sec: NotRequired[float]
     r"""How frequently, in seconds, to clean up empty directories"""
+    directory_batch_size: NotRequired[float]
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
     deadletter_path: NotRequired[str]
     r"""Storage location for files that fail to reach their final destination after maximum retries are exceeded"""
     max_retry_num: NotRequired[float]
@@ -301,6 +303,11 @@ class OutputExabeam(BaseModel):
     ] = 300
     r"""How frequently, in seconds, to clean up empty directories"""
 
+    directory_batch_size: Annotated[
+        Optional[float], pydantic.Field(alias="directoryBatchSize")
+    ] = 1000
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
+
     deadletter_path: Annotated[
         Optional[str], pydantic.Field(alias="deadletterPath")
     ] = "$CRIBL_HOME/state/outputs/dead-letter"

cribl_control_plane/models/outputfilesystem.py

@@ -140,6 +140,8 @@ class OutputFilesystemTypedDict(TypedDict):
     r"""If a file fails to move to its final destination after the maximum number of retries, move it to a designated directory to prevent further errors"""
     on_disk_full_backpressure: NotRequired[OutputFilesystemDiskSpaceProtection]
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
+    force_close_on_shutdown: NotRequired[bool]
+    r"""Force all staged files to close during an orderly Node shutdown. This triggers immediate upload of in-progress data — regardless of idle time, file age, or size thresholds — to minimize data loss."""
     description: NotRequired[str]
     compress: NotRequired[OutputFilesystemCompression]
     r"""Data compression format to apply to HTTP content before it is delivered"""
@@ -169,6 +171,8 @@ class OutputFilesystemTypedDict(TypedDict):
     r"""Parquet tools can use the checksum of a Parquet page to verify data integrity"""
     empty_dir_cleanup_sec: NotRequired[float]
     r"""How frequently, in seconds, to clean up empty directories"""
+    directory_batch_size: NotRequired[float]
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
     deadletter_path: NotRequired[str]
     r"""Storage location for files that fail to reach their final destination after maximum retries are exceeded"""
     max_retry_num: NotRequired[float]
@@ -286,6 +290,11 @@ class OutputFilesystem(BaseModel):
     ] = OutputFilesystemDiskSpaceProtection.BLOCK
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
 
+    force_close_on_shutdown: Annotated[
+        Optional[bool], pydantic.Field(alias="forceCloseOnShutdown")
+    ] = False
+    r"""Force all staged files to close during an orderly Node shutdown. This triggers immediate upload of in-progress data — regardless of idle time, file age, or size thresholds — to minimize data loss."""
+
     description: Optional[str] = None
 
     compress: Annotated[
@@ -371,6 +380,11 @@ class OutputFilesystem(BaseModel):
     ] = 300
     r"""How frequently, in seconds, to clean up empty directories"""
 
+    directory_batch_size: Annotated[
+        Optional[float], pydantic.Field(alias="directoryBatchSize")
+    ] = 1000
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
+
     deadletter_path: Annotated[
         Optional[str], pydantic.Field(alias="deadletterPath")
     ] = "$CRIBL_HOME/state/outputs/dead-letter"

cribl_control_plane/models/outputgooglecloudstorage.py

@@ -211,6 +211,8 @@ class OutputGoogleCloudStorageTypedDict(TypedDict):
     r"""If a file fails to move to its final destination after the maximum number of retries, move it to a designated directory to prevent further errors"""
     on_disk_full_backpressure: NotRequired[OutputGoogleCloudStorageDiskSpaceProtection]
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
+    force_close_on_shutdown: NotRequired[bool]
+    r"""Force all staged files to close during an orderly Node shutdown. This triggers immediate upload of in-progress data — regardless of idle time, file age, or size thresholds — to minimize data loss."""
     description: NotRequired[str]
     compress: NotRequired[OutputGoogleCloudStorageCompression]
     r"""Data compression format to apply to HTTP content before it is delivered"""
@@ -242,6 +244,8 @@ class OutputGoogleCloudStorageTypedDict(TypedDict):
     r"""Parquet tools can use the checksum of a Parquet page to verify data integrity"""
     empty_dir_cleanup_sec: NotRequired[float]
     r"""How frequently, in seconds, to clean up empty directories"""
+    directory_batch_size: NotRequired[float]
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
     deadletter_path: NotRequired[str]
     r"""Storage location for files that fail to reach their final destination after maximum retries are exceeded"""
     max_retry_num: NotRequired[float]
@@ -426,6 +430,11 @@ class OutputGoogleCloudStorage(BaseModel):
     ] = OutputGoogleCloudStorageDiskSpaceProtection.BLOCK
     r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
 
+    force_close_on_shutdown: Annotated[
+        Optional[bool], pydantic.Field(alias="forceCloseOnShutdown")
+    ] = False
+    r"""Force all staged files to close during an orderly Node shutdown. This triggers immediate upload of in-progress data — regardless of idle time, file age, or size thresholds — to minimize data loss."""
+
     description: Optional[str] = None
 
     compress: Annotated[
@@ -512,6 +521,11 @@ class OutputGoogleCloudStorage(BaseModel):
     ] = 300
     r"""How frequently, in seconds, to clean up empty directories"""
 
+    directory_batch_size: Annotated[
+        Optional[float], pydantic.Field(alias="directoryBatchSize")
+    ] = 1000
+    r"""Number of directories to process in each batch during cleanup of empty directories. Minimum is 10, maximum is 10000. Higher values may require more memory."""
+
     deadletter_path: Annotated[
         Optional[str], pydantic.Field(alias="deadletterPath")
     ] = "$CRIBL_HOME/state/outputs/dead-letter"