cribl-control-plane 0.1.0b1__py3-none-any.whl → 0.2.0a1__py3-none-any.whl

cribl_control_plane/models/outputdatabricks.py

@@ -1,439 +0,0 @@
-"""Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT."""
-
-from __future__ import annotations
-from cribl_control_plane import utils
-from cribl_control_plane.types import BaseModel
-from cribl_control_plane.utils import validate_open_enum
-from enum import Enum
-import pydantic
-from pydantic.functional_validators import PlainValidator
-from typing import List, Optional
-from typing_extensions import Annotated, NotRequired, TypedDict
-
-
-class OutputDatabricksType(str, Enum):
-    DATABRICKS = "databricks"
-
-
-class OutputDatabricksDataFormat(str, Enum, metaclass=utils.OpenEnumMeta):
-    r"""Format of the output data"""
-
-    JSON = "json"
-    RAW = "raw"
-    PARQUET = "parquet"
-
-
-class OutputDatabricksBackpressureBehavior(str, Enum, metaclass=utils.OpenEnumMeta):
-    r"""How to handle events when all receivers are exerting backpressure"""
-
-    BLOCK = "block"
-    DROP = "drop"
-
-
-class OutputDatabricksDiskSpaceProtection(str, Enum, metaclass=utils.OpenEnumMeta):
-    r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
-
-    BLOCK = "block"
-    DROP = "drop"
-
-
-class OutputDatabricksAuthenticationMethod(str, Enum, metaclass=utils.OpenEnumMeta):
-    r"""Unity Catalog authentication method. Choose Manual to enter credentials directly, or Secret to use a stored secret."""
-
-    MANUAL = "manual"
-    SECRET = "secret"
-
-
-class OutputDatabricksCompression(str, Enum, metaclass=utils.OpenEnumMeta):
-    r"""Data compression format to apply to HTTP content before it is delivered"""
-
-    NONE = "none"
-    GZIP = "gzip"
-
-
-class OutputDatabricksCompressionLevel(str, Enum, metaclass=utils.OpenEnumMeta):
-    r"""Compression level to apply before moving files to final destination"""
-
-    BEST_SPEED = "best_speed"
-    NORMAL = "normal"
-    BEST_COMPRESSION = "best_compression"
-
-
-class OutputDatabricksParquetVersion(str, Enum, metaclass=utils.OpenEnumMeta):
-    r"""Determines which data types are supported and how they are represented"""
-
-    PARQUET_1_0 = "PARQUET_1_0"
-    PARQUET_2_4 = "PARQUET_2_4"
-    PARQUET_2_6 = "PARQUET_2_6"
-
-
-class OutputDatabricksDataPageVersion(str, Enum, metaclass=utils.OpenEnumMeta):
-    r"""Serialization format of data pages. Note that some reader implementations use Data page V2's attributes to work more efficiently, while others ignore it."""
-
-    DATA_PAGE_V1 = "DATA_PAGE_V1"
-    DATA_PAGE_V2 = "DATA_PAGE_V2"
-
-
-class OutputDatabricksKeyValueMetadatumTypedDict(TypedDict):
-    value: str
-    key: NotRequired[str]
-
-
-class OutputDatabricksKeyValueMetadatum(BaseModel):
-    value: str
-
-    key: Optional[str] = ""
-
-
-class OutputDatabricksTypedDict(TypedDict):
-    type: OutputDatabricksType
-    id: NotRequired[str]
-    r"""Unique ID for this output"""
-    pipeline: NotRequired[str]
-    r"""Pipeline to process data before sending out to this output"""
-    system_fields: NotRequired[List[str]]
-    r"""Fields to automatically add to events, such as cribl_pipe. Supports wildcards."""
-    environment: NotRequired[str]
-    r"""Optionally, enable this config only on a specified Git branch. If empty, will be enabled everywhere."""
-    streamtags: NotRequired[List[str]]
-    r"""Tags for filtering and grouping in @{product}"""
-    dest_path: NotRequired[str]
-    r"""Optional path to prepend to files before uploading. Must be a JavaScript expression (which can evaluate to a constant value), enclosed in quotes or backticks. Can be evaluated only at init time. Example referencing a Global Variable: `myEventsVolumePath-${C.vars.myVar}`"""
-    stage_path: NotRequired[str]
-    r"""Filesystem location in which to buffer files before compressing and moving to final destination. Use performant, stable storage."""
-    add_id_to_stage_path: NotRequired[bool]
-    r"""Add the Output ID value to staging location"""
-    remove_empty_dirs: NotRequired[bool]
-    r"""Remove empty staging directories after moving files"""
-    partition_expr: NotRequired[str]
-    r"""JavaScript expression defining how files are partitioned and organized. Default is date-based. If blank, Stream will fall back to the event's __partition field value – if present – otherwise to each location's root directory."""
-    format_: NotRequired[OutputDatabricksDataFormat]
-    r"""Format of the output data"""
-    base_file_name: NotRequired[str]
-    r"""JavaScript expression to define the output filename prefix (can be constant)"""
-    file_name_suffix: NotRequired[str]
-    r"""JavaScript expression to define the output filename suffix (can be constant).  The `__format` variable refers to the value of the `Data format` field (`json` or `raw`).  The `__compression` field refers to the kind of compression being used (`none` or `gzip`)."""
-    max_file_size_mb: NotRequired[float]
-    r"""Maximum uncompressed output file size. Files of this size will be closed and moved to final output location."""
-    max_file_open_time_sec: NotRequired[float]
-    r"""Maximum amount of time to write to a file. Files open for longer than this will be closed and moved to final output location."""
-    max_file_idle_time_sec: NotRequired[float]
-    r"""Maximum amount of time to keep inactive files open. Files open for longer than this will be closed and moved to final output location."""
-    max_open_files: NotRequired[float]
-    r"""Maximum number of files to keep open concurrently. When exceeded, @{product} will close the oldest open files and move them to the final output location."""
-    header_line: NotRequired[str]
-    r"""If set, this line will be written to the beginning of each output file"""
-    write_high_water_mark: NotRequired[float]
-    r"""Buffer size used to write to a file"""
-    on_backpressure: NotRequired[OutputDatabricksBackpressureBehavior]
-    r"""How to handle events when all receivers are exerting backpressure"""
-    deadletter_enabled: NotRequired[bool]
-    r"""If a file fails to move to its final destination after the maximum number of retries, move it to a designated directory to prevent further errors"""
-    on_disk_full_backpressure: NotRequired[OutputDatabricksDiskSpaceProtection]
-    r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
-    unity_auth_method: NotRequired[OutputDatabricksAuthenticationMethod]
-    r"""Unity Catalog authentication method. Choose Manual to enter credentials directly, or Secret to use a stored secret."""
-    login_url: NotRequired[str]
-    r"""URL for Unity Catalog OAuth token endpoint (example: 'https://your-workspace.cloud.databricks.com/oauth/token')"""
-    client_id: NotRequired[str]
-    r"""JavaScript expression to compute the OAuth client ID for Unity Catalog authentication. Can be a constant."""
-    client_secret: NotRequired[str]
-    r"""JavaScript expression to compute the OAuth client secret for Unity Catalog authentication. Can be a constant."""
-    client_text_secret: NotRequired[str]
-    r"""Select or create a stored secret that references your Client ID and Client Secret"""
-    scope: NotRequired[str]
-    r"""OAuth scope for Unity Catalog authentication"""
-    token_timeout_secs: NotRequired[float]
-    r"""How often the OAuth token should be refreshed"""
-    default_catalog: NotRequired[str]
-    r"""Name of the catalog to use for the output"""
-    default_schema: NotRequired[str]
-    r"""Name of the catalog schema to use for the output"""
-    events_volume_name: NotRequired[str]
-    r"""Name of the events volume in Databricks"""
-    over_write_files: NotRequired[bool]
-    r"""Uploaded files should be overwritten if they already exist. If disabled, upload will fail if a file already exists."""
-    description: NotRequired[str]
-    compress: NotRequired[OutputDatabricksCompression]
-    r"""Data compression format to apply to HTTP content before it is delivered"""
-    compression_level: NotRequired[OutputDatabricksCompressionLevel]
-    r"""Compression level to apply before moving files to final destination"""
-    automatic_schema: NotRequired[bool]
-    r"""Automatically calculate the schema based on the events of each Parquet file generated"""
-    parquet_version: NotRequired[OutputDatabricksParquetVersion]
-    r"""Determines which data types are supported and how they are represented"""
-    parquet_data_page_version: NotRequired[OutputDatabricksDataPageVersion]
-    r"""Serialization format of data pages. Note that some reader implementations use Data page V2's attributes to work more efficiently, while others ignore it."""
-    parquet_row_group_length: NotRequired[float]
-    r"""The number of rows that every group will contain. The final group can contain a smaller number of rows."""
-    parquet_page_size: NotRequired[str]
-    r"""Target memory size for page segments, such as 1MB or 128MB. Generally, lower values improve reading speed, while higher values improve compression."""
-    should_log_invalid_rows: NotRequired[bool]
-    r"""Log up to 3 rows that @{product} skips due to data mismatch"""
-    key_value_metadata: NotRequired[List[OutputDatabricksKeyValueMetadatumTypedDict]]
-    r"""The metadata of files the Destination writes will include the properties you add here as key-value pairs. Useful for tagging. Examples: \"key\":\"OCSF Event Class\", \"value\":\"9001\" """
-    enable_statistics: NotRequired[bool]
-    r"""Statistics profile an entire file in terms of minimum/maximum values within data, numbers of nulls, etc. You can use Parquet tools to view statistics."""
-    enable_write_page_index: NotRequired[bool]
-    r"""One page index contains statistics for one data page. Parquet readers use statistics to enable page skipping."""
-    enable_page_checksum: NotRequired[bool]
-    r"""Parquet tools can use the checksum of a Parquet page to verify data integrity"""
-    empty_dir_cleanup_sec: NotRequired[float]
-    r"""How frequently, in seconds, to clean up empty directories"""
-    deadletter_path: NotRequired[str]
-    r"""Storage location for files that fail to reach their final destination after maximum retries are exceeded"""
-    max_retry_num: NotRequired[float]
-    r"""The maximum number of times a file will attempt to move to its final destination before being dead-lettered"""
-
-
-class OutputDatabricks(BaseModel):
-    type: OutputDatabricksType
-
-    id: Optional[str] = None
-    r"""Unique ID for this output"""
-
-    pipeline: Optional[str] = None
-    r"""Pipeline to process data before sending out to this output"""
-
-    system_fields: Annotated[
-        Optional[List[str]], pydantic.Field(alias="systemFields")
-    ] = None
-    r"""Fields to automatically add to events, such as cribl_pipe. Supports wildcards."""
-
-    environment: Optional[str] = None
-    r"""Optionally, enable this config only on a specified Git branch. If empty, will be enabled everywhere."""
-
-    streamtags: Optional[List[str]] = None
-    r"""Tags for filtering and grouping in @{product}"""
-
-    dest_path: Annotated[Optional[str], pydantic.Field(alias="destPath")] = ""
-    r"""Optional path to prepend to files before uploading. Must be a JavaScript expression (which can evaluate to a constant value), enclosed in quotes or backticks. Can be evaluated only at init time. Example referencing a Global Variable: `myEventsVolumePath-${C.vars.myVar}`"""
-
-    stage_path: Annotated[Optional[str], pydantic.Field(alias="stagePath")] = (
-        "$CRIBL_HOME/state/outputs/staging"
-    )
-    r"""Filesystem location in which to buffer files before compressing and moving to final destination. Use performant, stable storage."""
-
-    add_id_to_stage_path: Annotated[
-        Optional[bool], pydantic.Field(alias="addIdToStagePath")
-    ] = True
-    r"""Add the Output ID value to staging location"""
-
-    remove_empty_dirs: Annotated[
-        Optional[bool], pydantic.Field(alias="removeEmptyDirs")
-    ] = True
-    r"""Remove empty staging directories after moving files"""
-
-    partition_expr: Annotated[Optional[str], pydantic.Field(alias="partitionExpr")] = (
-        "C.Time.strftime(_time ? _time : Date.now()/1000, '%Y/%m/%d')"
-    )
-    r"""JavaScript expression defining how files are partitioned and organized. Default is date-based. If blank, Stream will fall back to the event's __partition field value – if present – otherwise to each location's root directory."""
-
-    format_: Annotated[
-        Annotated[
-            Optional[OutputDatabricksDataFormat],
-            PlainValidator(validate_open_enum(False)),
-        ],
-        pydantic.Field(alias="format"),
-    ] = OutputDatabricksDataFormat.JSON
-    r"""Format of the output data"""
-
-    base_file_name: Annotated[Optional[str], pydantic.Field(alias="baseFileName")] = (
-        "`CriblOut`"
-    )
-    r"""JavaScript expression to define the output filename prefix (can be constant)"""
-
-    file_name_suffix: Annotated[
-        Optional[str], pydantic.Field(alias="fileNameSuffix")
-    ] = '`.${C.env["CRIBL_WORKER_ID"]}.${__format}${__compression === "gzip" ? ".gz" : ""}`'
-    r"""JavaScript expression to define the output filename suffix (can be constant).  The `__format` variable refers to the value of the `Data format` field (`json` or `raw`).  The `__compression` field refers to the kind of compression being used (`none` or `gzip`)."""
-
-    max_file_size_mb: Annotated[
-        Optional[float], pydantic.Field(alias="maxFileSizeMB")
-    ] = 32
-    r"""Maximum uncompressed output file size. Files of this size will be closed and moved to final output location."""
-
-    max_file_open_time_sec: Annotated[
-        Optional[float], pydantic.Field(alias="maxFileOpenTimeSec")
-    ] = 300
-    r"""Maximum amount of time to write to a file. Files open for longer than this will be closed and moved to final output location."""
-
-    max_file_idle_time_sec: Annotated[
-        Optional[float], pydantic.Field(alias="maxFileIdleTimeSec")
-    ] = 30
-    r"""Maximum amount of time to keep inactive files open. Files open for longer than this will be closed and moved to final output location."""
-
-    max_open_files: Annotated[Optional[float], pydantic.Field(alias="maxOpenFiles")] = (
-        100
-    )
-    r"""Maximum number of files to keep open concurrently. When exceeded, @{product} will close the oldest open files and move them to the final output location."""
-
-    header_line: Annotated[Optional[str], pydantic.Field(alias="headerLine")] = ""
-    r"""If set, this line will be written to the beginning of each output file"""
-
-    write_high_water_mark: Annotated[
-        Optional[float], pydantic.Field(alias="writeHighWaterMark")
-    ] = 64
-    r"""Buffer size used to write to a file"""
-
-    on_backpressure: Annotated[
-        Annotated[
-            Optional[OutputDatabricksBackpressureBehavior],
-            PlainValidator(validate_open_enum(False)),
-        ],
-        pydantic.Field(alias="onBackpressure"),
-    ] = OutputDatabricksBackpressureBehavior.BLOCK
-    r"""How to handle events when all receivers are exerting backpressure"""
-
-    deadletter_enabled: Annotated[
-        Optional[bool], pydantic.Field(alias="deadletterEnabled")
-    ] = False
-    r"""If a file fails to move to its final destination after the maximum number of retries, move it to a designated directory to prevent further errors"""
-
-    on_disk_full_backpressure: Annotated[
-        Annotated[
-            Optional[OutputDatabricksDiskSpaceProtection],
-            PlainValidator(validate_open_enum(False)),
-        ],
-        pydantic.Field(alias="onDiskFullBackpressure"),
-    ] = OutputDatabricksDiskSpaceProtection.BLOCK
-    r"""How to handle events when disk space is below the global 'Min free disk space' limit"""
-
-    unity_auth_method: Annotated[
-        Annotated[
-            Optional[OutputDatabricksAuthenticationMethod],
-            PlainValidator(validate_open_enum(False)),
-        ],
-        pydantic.Field(alias="unityAuthMethod"),
-    ] = OutputDatabricksAuthenticationMethod.MANUAL
-    r"""Unity Catalog authentication method. Choose Manual to enter credentials directly, or Secret to use a stored secret."""
-
-    login_url: Annotated[Optional[str], pydantic.Field(alias="loginUrl")] = None
-    r"""URL for Unity Catalog OAuth token endpoint (example: 'https://your-workspace.cloud.databricks.com/oauth/token')"""
-
-    client_id: Annotated[Optional[str], pydantic.Field(alias="clientId")] = None
-    r"""JavaScript expression to compute the OAuth client ID for Unity Catalog authentication. Can be a constant."""
-
-    client_secret: Annotated[Optional[str], pydantic.Field(alias="clientSecret")] = None
-    r"""JavaScript expression to compute the OAuth client secret for Unity Catalog authentication. Can be a constant."""
-
-    client_text_secret: Annotated[
-        Optional[str], pydantic.Field(alias="clientTextSecret")
-    ] = None
-    r"""Select or create a stored secret that references your Client ID and Client Secret"""
-
-    scope: Optional[str] = "all-apis"
-    r"""OAuth scope for Unity Catalog authentication"""
-
-    token_timeout_secs: Annotated[
-        Optional[float], pydantic.Field(alias="tokenTimeoutSecs")
-    ] = 3600
-    r"""How often the OAuth token should be refreshed"""
-
-    default_catalog: Annotated[
-        Optional[str], pydantic.Field(alias="defaultCatalog")
-    ] = "main"
-    r"""Name of the catalog to use for the output"""
-
-    default_schema: Annotated[Optional[str], pydantic.Field(alias="defaultSchema")] = (
-        "external"
-    )
-    r"""Name of the catalog schema to use for the output"""
-
-    events_volume_name: Annotated[
-        Optional[str], pydantic.Field(alias="eventsVolumeName")
-    ] = "events"
-    r"""Name of the events volume in Databricks"""
-
-    over_write_files: Annotated[
-        Optional[bool], pydantic.Field(alias="overWriteFiles")
-    ] = False
-    r"""Uploaded files should be overwritten if they already exist. If disabled, upload will fail if a file already exists."""
-
-    description: Optional[str] = None
-
-    compress: Annotated[
-        Optional[OutputDatabricksCompression], PlainValidator(validate_open_enum(False))
-    ] = OutputDatabricksCompression.GZIP
-    r"""Data compression format to apply to HTTP content before it is delivered"""
-
-    compression_level: Annotated[
-        Annotated[
-            Optional[OutputDatabricksCompressionLevel],
-            PlainValidator(validate_open_enum(False)),
-        ],
-        pydantic.Field(alias="compressionLevel"),
-    ] = OutputDatabricksCompressionLevel.BEST_SPEED
-    r"""Compression level to apply before moving files to final destination"""
-
-    automatic_schema: Annotated[
-        Optional[bool], pydantic.Field(alias="automaticSchema")
-    ] = False
-    r"""Automatically calculate the schema based on the events of each Parquet file generated"""
-
-    parquet_version: Annotated[
-        Annotated[
-            Optional[OutputDatabricksParquetVersion],
-            PlainValidator(validate_open_enum(False)),
-        ],
-        pydantic.Field(alias="parquetVersion"),
-    ] = OutputDatabricksParquetVersion.PARQUET_2_6
-    r"""Determines which data types are supported and how they are represented"""
-
-    parquet_data_page_version: Annotated[
-        Annotated[
-            Optional[OutputDatabricksDataPageVersion],
-            PlainValidator(validate_open_enum(False)),
-        ],
-        pydantic.Field(alias="parquetDataPageVersion"),
-    ] = OutputDatabricksDataPageVersion.DATA_PAGE_V2
-    r"""Serialization format of data pages. Note that some reader implementations use Data page V2's attributes to work more efficiently, while others ignore it."""
-
-    parquet_row_group_length: Annotated[
-        Optional[float], pydantic.Field(alias="parquetRowGroupLength")
-    ] = 10000
-    r"""The number of rows that every group will contain. The final group can contain a smaller number of rows."""
-
-    parquet_page_size: Annotated[
-        Optional[str], pydantic.Field(alias="parquetPageSize")
-    ] = "1MB"
-    r"""Target memory size for page segments, such as 1MB or 128MB. Generally, lower values improve reading speed, while higher values improve compression."""
-
-    should_log_invalid_rows: Annotated[
-        Optional[bool], pydantic.Field(alias="shouldLogInvalidRows")
-    ] = None
-    r"""Log up to 3 rows that @{product} skips due to data mismatch"""
-
-    key_value_metadata: Annotated[
-        Optional[List[OutputDatabricksKeyValueMetadatum]],
-        pydantic.Field(alias="keyValueMetadata"),
-    ] = None
-    r"""The metadata of files the Destination writes will include the properties you add here as key-value pairs. Useful for tagging. Examples: \"key\":\"OCSF Event Class\", \"value\":\"9001\" """
-
-    enable_statistics: Annotated[
-        Optional[bool], pydantic.Field(alias="enableStatistics")
-    ] = True
-    r"""Statistics profile an entire file in terms of minimum/maximum values within data, numbers of nulls, etc. You can use Parquet tools to view statistics."""
-
-    enable_write_page_index: Annotated[
-        Optional[bool], pydantic.Field(alias="enableWritePageIndex")
-    ] = True
-    r"""One page index contains statistics for one data page. Parquet readers use statistics to enable page skipping."""
-
-    enable_page_checksum: Annotated[
-        Optional[bool], pydantic.Field(alias="enablePageChecksum")
-    ] = False
-    r"""Parquet tools can use the checksum of a Parquet page to verify data integrity"""
-
-    empty_dir_cleanup_sec: Annotated[
-        Optional[float], pydantic.Field(alias="emptyDirCleanupSec")
-    ] = 300
-    r"""How frequently, in seconds, to clean up empty directories"""
-
-    deadletter_path: Annotated[
-        Optional[str], pydantic.Field(alias="deadletterPath")
-    ] = "$CRIBL_HOME/state/outputs/dead-letter"
-    r"""Storage location for files that fail to reach their final destination after maximum retries are exceeded"""
-
-    max_retry_num: Annotated[Optional[float], pydantic.Field(alias="maxRetryNum")] = 20
-    r"""The maximum number of times a file will attempt to move to its final destination before being dead-lettered"""