cribl-control-plane 0.0.13__py3-none-any.whl

cribl_control_plane/models/inputs3.py

@@ -0,0 +1,416 @@
+"""Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT."""
+
+from __future__ import annotations
+from cribl_control_plane import utils
+from cribl_control_plane.types import BaseModel
+from cribl_control_plane.utils import validate_open_enum
+from enum import Enum
+import pydantic
+from pydantic.functional_validators import PlainValidator
+from typing import List, Optional
+from typing_extensions import Annotated, NotRequired, TypedDict
+
+
+class InputS3Type(str, Enum, metaclass=utils.OpenEnumMeta):
+    S3 = "s3"
+
+
+class InputS3ConnectionTypedDict(TypedDict):
+    output: str
+    pipeline: NotRequired[str]
+
+
+class InputS3Connection(BaseModel):
+    output: str
+
+    pipeline: Optional[str] = None
+
+
+class InputS3Mode(str, Enum, metaclass=utils.OpenEnumMeta):
+    r"""With Smart mode, PQ will write events to the filesystem only when it detects backpressure from the processing engine. With Always On mode, PQ will always write events directly to the queue before forwarding them to the processing engine."""
+
+    SMART = "smart"
+    ALWAYS = "always"
+
+
+class InputS3Compression(str, Enum, metaclass=utils.OpenEnumMeta):
+    r"""Codec to use to compress the persisted data"""
+
+    NONE = "none"
+    GZIP = "gzip"
+
+
+class InputS3PqTypedDict(TypedDict):
+    mode: NotRequired[InputS3Mode]
+    r"""With Smart mode, PQ will write events to the filesystem only when it detects backpressure from the processing engine. With Always On mode, PQ will always write events directly to the queue before forwarding them to the processing engine."""
+    max_buffer_size: NotRequired[float]
+    r"""The maximum number of events to hold in memory before writing the events to disk"""
+    commit_frequency: NotRequired[float]
+    r"""The number of events to send downstream before committing that Stream has read them"""
+    max_file_size: NotRequired[str]
+    r"""The maximum size to store in each queue file before closing and optionally compressing. Enter a numeral with units of KB, MB, etc."""
+    max_size: NotRequired[str]
+    r"""The maximum disk space that the queue can consume (as an average per Worker Process) before queueing stops. Enter a numeral with units of KB, MB, etc."""
+    path: NotRequired[str]
+    r"""The location for the persistent queue files. To this field's value, the system will append: /<worker-id>/inputs/<input-id>"""
+    compress: NotRequired[InputS3Compression]
+    r"""Codec to use to compress the persisted data"""
+
+
+class InputS3Pq(BaseModel):
+    mode: Annotated[
+        Optional[InputS3Mode], PlainValidator(validate_open_enum(False))
+    ] = InputS3Mode.ALWAYS
+    r"""With Smart mode, PQ will write events to the filesystem only when it detects backpressure from the processing engine. With Always On mode, PQ will always write events directly to the queue before forwarding them to the processing engine."""
+
+    max_buffer_size: Annotated[
+        Optional[float], pydantic.Field(alias="maxBufferSize")
+    ] = 1000
+    r"""The maximum number of events to hold in memory before writing the events to disk"""
+
+    commit_frequency: Annotated[
+        Optional[float], pydantic.Field(alias="commitFrequency")
+    ] = 42
+    r"""The number of events to send downstream before committing that Stream has read them"""
+
+    max_file_size: Annotated[Optional[str], pydantic.Field(alias="maxFileSize")] = (
+        "1 MB"
+    )
+    r"""The maximum size to store in each queue file before closing and optionally compressing. Enter a numeral with units of KB, MB, etc."""
+
+    max_size: Annotated[Optional[str], pydantic.Field(alias="maxSize")] = "5GB"
+    r"""The maximum disk space that the queue can consume (as an average per Worker Process) before queueing stops. Enter a numeral with units of KB, MB, etc."""
+
+    path: Optional[str] = "$CRIBL_HOME/state/queues"
+    r"""The location for the persistent queue files. To this field's value, the system will append: /<worker-id>/inputs/<input-id>"""
+
+    compress: Annotated[
+        Optional[InputS3Compression], PlainValidator(validate_open_enum(False))
+    ] = InputS3Compression.NONE
+    r"""Codec to use to compress the persisted data"""
+
+
+class InputS3AuthenticationMethod(str, Enum, metaclass=utils.OpenEnumMeta):
+    r"""AWS authentication method. Choose Auto to use IAM roles."""
+
+    AUTO = "auto"
+    MANUAL = "manual"
+    SECRET = "secret"
+
+
+class InputS3SignatureVersion(str, Enum, metaclass=utils.OpenEnumMeta):
+    r"""Signature version to use for signing S3 requests"""
+
+    V2 = "v2"
+    V4 = "v4"
+
+
+class InputS3PreprocessTypedDict(TypedDict):
+    disabled: NotRequired[bool]
+    command: NotRequired[str]
+    r"""Command to feed the data through (via stdin) and process its output (stdout)"""
+    args: NotRequired[List[str]]
+    r"""Arguments to be added to the custom command"""
+
+
+class InputS3Preprocess(BaseModel):
+    disabled: Optional[bool] = True
+
+    command: Optional[str] = None
+    r"""Command to feed the data through (via stdin) and process its output (stdout)"""
+
+    args: Optional[List[str]] = None
+    r"""Arguments to be added to the custom command"""
+
+
+class InputS3MetadatumTypedDict(TypedDict):
+    name: str
+    value: str
+    r"""JavaScript expression to compute field's value, enclosed in quotes or backticks. (Can evaluate to a constant.)"""
+
+
+class InputS3Metadatum(BaseModel):
+    name: str
+
+    value: str
+    r"""JavaScript expression to compute field's value, enclosed in quotes or backticks. (Can evaluate to a constant.)"""
+
+
+class InputS3CheckpointingTypedDict(TypedDict):
+    enabled: NotRequired[bool]
+    r"""Resume processing files after an interruption"""
+    retries: NotRequired[float]
+    r"""The number of times to retry processing when a processing error occurs. If Skip file on error is enabled, this setting is ignored."""
+
+
+class InputS3Checkpointing(BaseModel):
+    enabled: Optional[bool] = False
+    r"""Resume processing files after an interruption"""
+
+    retries: Optional[float] = 5
+    r"""The number of times to retry processing when a processing error occurs. If Skip file on error is enabled, this setting is ignored."""
+
+
+class InputS3TypedDict(TypedDict):
+    type: InputS3Type
+    queue_name: str
+    r"""The name, URL, or ARN of the SQS queue to read notifications from. When a non-AWS URL is specified, format must be: '{url}/myQueueName'. Example: 'https://host:port/myQueueName'. Value must be a JavaScript expression (which can evaluate to a constant value), enclosed in quotes or backticks. Can be evaluated only at init time. Example referencing a Global Variable: `https://host:port/myQueue-${C.vars.myVar}`."""
+    id: NotRequired[str]
+    r"""Unique ID for this input"""
+    disabled: NotRequired[bool]
+    pipeline: NotRequired[str]
+    r"""Pipeline to process data from this Source before sending it through the Routes"""
+    send_to_routes: NotRequired[bool]
+    r"""Select whether to send data to Routes, or directly to Destinations."""
+    environment: NotRequired[str]
+    r"""Optionally, enable this config only on a specified Git branch. If empty, will be enabled everywhere."""
+    pq_enabled: NotRequired[bool]
+    r"""Use a disk queue to minimize data loss when connected services block. See [Cribl Docs](https://docs.cribl.io/stream/persistent-queues) for PQ defaults (Cribl-managed Cloud Workers) and configuration options (on-prem and hybrid Workers)."""
+    streamtags: NotRequired[List[str]]
+    r"""Tags for filtering and grouping in @{product}"""
+    connections: NotRequired[List[InputS3ConnectionTypedDict]]
+    r"""Direct connections to Destinations, and optionally via a Pipeline or a Pack"""
+    pq: NotRequired[InputS3PqTypedDict]
+    file_filter: NotRequired[str]
+    r"""Regex matching file names to download and process. Defaults to: .*"""
+    aws_account_id: NotRequired[str]
+    r"""SQS queue owner's AWS account ID. Leave empty if SQS queue is in same AWS account."""
+    aws_authentication_method: NotRequired[InputS3AuthenticationMethod]
+    r"""AWS authentication method. Choose Auto to use IAM roles."""
+    aws_secret_key: NotRequired[str]
+    region: NotRequired[str]
+    r"""AWS Region where the S3 bucket and SQS queue are located. Required, unless the Queue entry is a URL or ARN that includes a Region."""
+    endpoint: NotRequired[str]
+    r"""S3 service endpoint. If empty, defaults to the AWS Region-specific endpoint. Otherwise, it must point to S3-compatible endpoint."""
+    signature_version: NotRequired[InputS3SignatureVersion]
+    r"""Signature version to use for signing S3 requests"""
+    reuse_connections: NotRequired[bool]
+    r"""Reuse connections between requests, which can improve performance"""
+    reject_unauthorized: NotRequired[bool]
+    r"""Reject certificates that cannot be verified against a valid CA, such as self-signed certificates"""
+    breaker_rulesets: NotRequired[List[str]]
+    r"""A list of event-breaking rulesets that will be applied, in order, to the input data stream"""
+    stale_channel_flush_ms: NotRequired[float]
+    r"""How long (in milliseconds) the Event Breaker will wait for new data to be sent to a specific channel before flushing the data stream out, as is, to the Pipelines"""
+    max_messages: NotRequired[float]
+    r"""The maximum number of messages SQS should return in a poll request. Amazon SQS never returns more messages than this value (however, fewer messages might be returned). Valid values: 1 to 10."""
+    visibility_timeout: NotRequired[float]
+    r"""After messages are retrieved by a ReceiveMessage request, @{product} will hide them from subsequent retrieve requests for at least this duration. You can set this as high as 43200 sec. (12 hours)."""
+    num_receivers: NotRequired[float]
+    r"""How many receiver processes to run. The higher the number, the better the throughput - at the expense of CPU overhead."""
+    socket_timeout: NotRequired[float]
+    r"""Socket inactivity timeout (in seconds). Increase this value if timeouts occur due to backpressure."""
+    skip_on_error: NotRequired[bool]
+    r"""Skip files that trigger a processing error. Disabled by default, which allows retries after processing errors."""
+    enable_assume_role: NotRequired[bool]
+    r"""Use Assume Role credentials to access Amazon S3"""
+    assume_role_arn: NotRequired[str]
+    r"""Amazon Resource Name (ARN) of the role to assume"""
+    assume_role_external_id: NotRequired[str]
+    r"""External ID to use when assuming role"""
+    duration_seconds: NotRequired[float]
+    r"""Duration of the assumed role's session, in seconds. Minimum is 900 (15 minutes), default is 3600 (1 hour), and maximum is 43200 (12 hours)."""
+    enable_sqs_assume_role: NotRequired[bool]
+    r"""Use Assume Role credentials when accessing Amazon SQS"""
+    preprocess: NotRequired[InputS3PreprocessTypedDict]
+    metadata: NotRequired[List[InputS3MetadatumTypedDict]]
+    r"""Fields to add to events from this input"""
+    parquet_chunk_size_mb: NotRequired[float]
+    r"""Maximum file size for each Parquet chunk"""
+    parquet_chunk_download_timeout: NotRequired[float]
+    r"""The maximum time allowed for downloading a Parquet chunk. Processing will stop if a chunk cannot be downloaded within the time specified."""
+    checkpointing: NotRequired[InputS3CheckpointingTypedDict]
+    poll_timeout: NotRequired[float]
+    r"""How long to wait for events before trying polling again. The lower the number the higher the AWS bill. The higher the number the longer it will take for the source to react to configuration changes and system restarts."""
+    encoding: NotRequired[str]
+    r"""Character encoding to use when parsing ingested data. When not set, @{product} will default to UTF-8 but may incorrectly interpret multi-byte characters."""
+    tag_after_processing: NotRequired[bool]
+    r"""Add a tag to processed S3 objects. Requires s3:GetObjectTagging and s3:PutObjectTagging AWS permissions."""
+    description: NotRequired[str]
+    aws_api_key: NotRequired[str]
+    aws_secret: NotRequired[str]
+    r"""Select or create a stored secret that references your access key and secret key"""
+    processed_tag_key: NotRequired[str]
+    r"""The key for the S3 object tag applied after processing. This field accepts an expression for dynamic generation."""
+    processed_tag_value: NotRequired[str]
+    r"""The value for the S3 object tag applied after processing. This field accepts an expression for dynamic generation."""
+
+
+class InputS3(BaseModel):
+    type: Annotated[InputS3Type, PlainValidator(validate_open_enum(False))]
+
+    queue_name: Annotated[str, pydantic.Field(alias="queueName")]
+    r"""The name, URL, or ARN of the SQS queue to read notifications from. When a non-AWS URL is specified, format must be: '{url}/myQueueName'. Example: 'https://host:port/myQueueName'. Value must be a JavaScript expression (which can evaluate to a constant value), enclosed in quotes or backticks. Can be evaluated only at init time. Example referencing a Global Variable: `https://host:port/myQueue-${C.vars.myVar}`."""
+
+    id: Optional[str] = None
+    r"""Unique ID for this input"""
+
+    disabled: Optional[bool] = False
+
+    pipeline: Optional[str] = None
+    r"""Pipeline to process data from this Source before sending it through the Routes"""
+
+    send_to_routes: Annotated[Optional[bool], pydantic.Field(alias="sendToRoutes")] = (
+        True
+    )
+    r"""Select whether to send data to Routes, or directly to Destinations."""
+
+    environment: Optional[str] = None
+    r"""Optionally, enable this config only on a specified Git branch. If empty, will be enabled everywhere."""
+
+    pq_enabled: Annotated[Optional[bool], pydantic.Field(alias="pqEnabled")] = False
+    r"""Use a disk queue to minimize data loss when connected services block. See [Cribl Docs](https://docs.cribl.io/stream/persistent-queues) for PQ defaults (Cribl-managed Cloud Workers) and configuration options (on-prem and hybrid Workers)."""
+
+    streamtags: Optional[List[str]] = None
+    r"""Tags for filtering and grouping in @{product}"""
+
+    connections: Optional[List[InputS3Connection]] = None
+    r"""Direct connections to Destinations, and optionally via a Pipeline or a Pack"""
+
+    pq: Optional[InputS3Pq] = None
+
+    file_filter: Annotated[Optional[str], pydantic.Field(alias="fileFilter")] = "/.*/"
+    r"""Regex matching file names to download and process. Defaults to: .*"""
+
+    aws_account_id: Annotated[Optional[str], pydantic.Field(alias="awsAccountId")] = (
+        None
+    )
+    r"""SQS queue owner's AWS account ID. Leave empty if SQS queue is in same AWS account."""
+
+    aws_authentication_method: Annotated[
+        Annotated[
+            Optional[InputS3AuthenticationMethod],
+            PlainValidator(validate_open_enum(False)),
+        ],
+        pydantic.Field(alias="awsAuthenticationMethod"),
+    ] = InputS3AuthenticationMethod.AUTO
+    r"""AWS authentication method. Choose Auto to use IAM roles."""
+
+    aws_secret_key: Annotated[Optional[str], pydantic.Field(alias="awsSecretKey")] = (
+        None
+    )
+
+    region: Optional[str] = None
+    r"""AWS Region where the S3 bucket and SQS queue are located. Required, unless the Queue entry is a URL or ARN that includes a Region."""
+
+    endpoint: Optional[str] = None
+    r"""S3 service endpoint. If empty, defaults to the AWS Region-specific endpoint. Otherwise, it must point to S3-compatible endpoint."""
+
+    signature_version: Annotated[
+        Annotated[
+            Optional[InputS3SignatureVersion], PlainValidator(validate_open_enum(False))
+        ],
+        pydantic.Field(alias="signatureVersion"),
+    ] = InputS3SignatureVersion.V4
+    r"""Signature version to use for signing S3 requests"""
+
+    reuse_connections: Annotated[
+        Optional[bool], pydantic.Field(alias="reuseConnections")
+    ] = True
+    r"""Reuse connections between requests, which can improve performance"""
+
+    reject_unauthorized: Annotated[
+        Optional[bool], pydantic.Field(alias="rejectUnauthorized")
+    ] = True
+    r"""Reject certificates that cannot be verified against a valid CA, such as self-signed certificates"""
+
+    breaker_rulesets: Annotated[
+        Optional[List[str]], pydantic.Field(alias="breakerRulesets")
+    ] = None
+    r"""A list of event-breaking rulesets that will be applied, in order, to the input data stream"""
+
+    stale_channel_flush_ms: Annotated[
+        Optional[float], pydantic.Field(alias="staleChannelFlushMs")
+    ] = 10000
+    r"""How long (in milliseconds) the Event Breaker will wait for new data to be sent to a specific channel before flushing the data stream out, as is, to the Pipelines"""
+
+    max_messages: Annotated[Optional[float], pydantic.Field(alias="maxMessages")] = 1
+    r"""The maximum number of messages SQS should return in a poll request. Amazon SQS never returns more messages than this value (however, fewer messages might be returned). Valid values: 1 to 10."""
+
+    visibility_timeout: Annotated[
+        Optional[float], pydantic.Field(alias="visibilityTimeout")
+    ] = 600
+    r"""After messages are retrieved by a ReceiveMessage request, @{product} will hide them from subsequent retrieve requests for at least this duration. You can set this as high as 43200 sec. (12 hours)."""
+
+    num_receivers: Annotated[Optional[float], pydantic.Field(alias="numReceivers")] = 1
+    r"""How many receiver processes to run. The higher the number, the better the throughput - at the expense of CPU overhead."""
+
+    socket_timeout: Annotated[
+        Optional[float], pydantic.Field(alias="socketTimeout")
+    ] = 300
+    r"""Socket inactivity timeout (in seconds). Increase this value if timeouts occur due to backpressure."""
+
+    skip_on_error: Annotated[Optional[bool], pydantic.Field(alias="skipOnError")] = (
+        False
+    )
+    r"""Skip files that trigger a processing error. Disabled by default, which allows retries after processing errors."""
+
+    enable_assume_role: Annotated[
+        Optional[bool], pydantic.Field(alias="enableAssumeRole")
+    ] = True
+    r"""Use Assume Role credentials to access Amazon S3"""
+
+    assume_role_arn: Annotated[Optional[str], pydantic.Field(alias="assumeRoleArn")] = (
+        None
+    )
+    r"""Amazon Resource Name (ARN) of the role to assume"""
+
+    assume_role_external_id: Annotated[
+        Optional[str], pydantic.Field(alias="assumeRoleExternalId")
+    ] = None
+    r"""External ID to use when assuming role"""
+
+    duration_seconds: Annotated[
+        Optional[float], pydantic.Field(alias="durationSeconds")
+    ] = 3600
+    r"""Duration of the assumed role's session, in seconds. Minimum is 900 (15 minutes), default is 3600 (1 hour), and maximum is 43200 (12 hours)."""
+
+    enable_sqs_assume_role: Annotated[
+        Optional[bool], pydantic.Field(alias="enableSQSAssumeRole")
+    ] = False
+    r"""Use Assume Role credentials when accessing Amazon SQS"""
+
+    preprocess: Optional[InputS3Preprocess] = None
+
+    metadata: Optional[List[InputS3Metadatum]] = None
+    r"""Fields to add to events from this input"""
+
+    parquet_chunk_size_mb: Annotated[
+        Optional[float], pydantic.Field(alias="parquetChunkSizeMB")
+    ] = 5
+    r"""Maximum file size for each Parquet chunk"""
+
+    parquet_chunk_download_timeout: Annotated[
+        Optional[float], pydantic.Field(alias="parquetChunkDownloadTimeout")
+    ] = 600
+    r"""The maximum time allowed for downloading a Parquet chunk. Processing will stop if a chunk cannot be downloaded within the time specified."""
+
+    checkpointing: Optional[InputS3Checkpointing] = None
+
+    poll_timeout: Annotated[Optional[float], pydantic.Field(alias="pollTimeout")] = 10
+    r"""How long to wait for events before trying polling again. The lower the number the higher the AWS bill. The higher the number the longer it will take for the source to react to configuration changes and system restarts."""
+
+    encoding: Optional[str] = None
+    r"""Character encoding to use when parsing ingested data. When not set, @{product} will default to UTF-8 but may incorrectly interpret multi-byte characters."""
+
+    tag_after_processing: Annotated[
+        Optional[bool], pydantic.Field(alias="tagAfterProcessing")
+    ] = False
+    r"""Add a tag to processed S3 objects. Requires s3:GetObjectTagging and s3:PutObjectTagging AWS permissions."""
+
+    description: Optional[str] = None
+
+    aws_api_key: Annotated[Optional[str], pydantic.Field(alias="awsApiKey")] = None
+
+    aws_secret: Annotated[Optional[str], pydantic.Field(alias="awsSecret")] = None
+    r"""Select or create a stored secret that references your access key and secret key"""
+
+    processed_tag_key: Annotated[
+        Optional[str], pydantic.Field(alias="processedTagKey")
+    ] = None
+    r"""The key for the S3 object tag applied after processing. This field accepts an expression for dynamic generation."""
+
+    processed_tag_value: Annotated[
+        Optional[str], pydantic.Field(alias="processedTagValue")
+    ] = None
+    r"""The value for the S3 object tag applied after processing. This field accepts an expression for dynamic generation."""