airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/file_based/config/csv_format.py

@@ -0,0 +1,210 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import codecs
+from enum import Enum
+from typing import Any, Dict, List, Optional, Set, Union
+
+from pydantic.v1 import BaseModel, Field, root_validator, validator
+from pydantic.v1.error_wrappers import ValidationError
+
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
+
+
+class InferenceType(Enum):
+    NONE = "None"
+    PRIMITIVE_TYPES_ONLY = "Primitive Types Only"
+
+
+class CsvHeaderDefinitionType(Enum):
+    FROM_CSV = "From CSV"
+    AUTOGENERATED = "Autogenerated"
+    USER_PROVIDED = "User Provided"
+
+
+class CsvHeaderFromCsv(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "From CSV"
+        discriminator = "header_definition_type"
+
+    header_definition_type: str = Field(
+        CsvHeaderDefinitionType.FROM_CSV.value,
+        const=True,
+    )
+
+    def has_header_row(self) -> bool:
+        return True
+
+
+class CsvHeaderAutogenerated(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Autogenerated"
+        discriminator = "header_definition_type"
+
+    header_definition_type: str = Field(
+        CsvHeaderDefinitionType.AUTOGENERATED.value,
+        const=True,
+    )
+
+    def has_header_row(self) -> bool:
+        return False
+
+
+class CsvHeaderUserProvided(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "User Provided"
+        discriminator = "header_definition_type"
+
+    header_definition_type: str = Field(
+        CsvHeaderDefinitionType.USER_PROVIDED.value,
+        const=True,
+    )
+    column_names: List[str] = Field(
+        title="Column Names",
+        description="The column names that will be used while emitting the CSV records",
+    )
+
+    def has_header_row(self) -> bool:
+        return False
+
+    @validator("column_names")
+    def validate_column_names(cls, v: List[str]) -> List[str]:
+        if not v:
+            raise ValueError(
+                "At least one column name needs to be provided when using user provided headers"
+            )
+        return v
+
+
+DEFAULT_TRUE_VALUES = ["y", "yes", "t", "true", "on", "1"]
+DEFAULT_FALSE_VALUES = ["n", "no", "f", "false", "off", "0"]
+
+
+class CsvFormat(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "CSV Format"
+        discriminator = "filetype"
+
+    filetype: str = Field(
+        "csv",
+        const=True,
+    )
+    delimiter: str = Field(
+        title="Delimiter",
+        description="The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.",
+        default=",",
+    )
+    quote_char: str = Field(
+        title="Quote Character",
+        default='"',
+        description="The character used for quoting CSV values. To disallow quoting, make this field blank.",
+    )
+    escape_char: Optional[str] = Field(
+        title="Escape Character",
+        default=None,
+        description="The character used for escaping special characters. To disallow escaping, leave this field blank.",
+    )
+    encoding: Optional[str] = Field(
+        default="utf8",
+        description='The character encoding of the CSV data. Leave blank to default to <strong>UTF8</strong>. See <a href="https://docs.python.org/3/library/codecs.html#standard-encodings" target="_blank">list of python encodings</a> for allowable options.',
+    )
+    double_quote: bool = Field(
+        title="Double Quote",
+        default=True,
+        description="Whether two quotes in a quoted CSV value denote a single quote in the data.",
+    )
+    null_values: Set[str] = Field(
+        title="Null Values",
+        default=[],
+        description="A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field.",
+    )
+    strings_can_be_null: bool = Field(
+        title="Strings Can Be Null",
+        default=True,
+        description="Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself.",
+    )
+    skip_rows_before_header: int = Field(
+        title="Skip Rows Before Header",
+        default=0,
+        description="The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.",
+    )
+    skip_rows_after_header: int = Field(
+        title="Skip Rows After Header",
+        default=0,
+        description="The number of rows to skip after the header row.",
+    )
+    header_definition: Union[CsvHeaderFromCsv, CsvHeaderAutogenerated, CsvHeaderUserProvided] = (
+        Field(
+            title="CSV Header Definition",
+            default=CsvHeaderFromCsv(header_definition_type=CsvHeaderDefinitionType.FROM_CSV.value),
+            description="How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.",
+        )
+    )
+    true_values: Set[str] = Field(
+        title="True Values",
+        default=DEFAULT_TRUE_VALUES,
+        description="A set of case-sensitive strings that should be interpreted as true values.",
+    )
+    false_values: Set[str] = Field(
+        title="False Values",
+        default=DEFAULT_FALSE_VALUES,
+        description="A set of case-sensitive strings that should be interpreted as false values.",
+    )
+    inference_type: InferenceType = Field(
+        title="Inference Type",
+        default=InferenceType.NONE,
+        description="How to infer the types of the columns. If none, inference default to strings.",
+        airbyte_hidden=True,
+    )
+    ignore_errors_on_fields_mismatch: bool = Field(
+        title="Ignore errors on field mismatch",
+        default=False,
+        description="Whether to ignore errors that occur when the number of fields in the CSV does not match the number of columns in the schema.",
+    )
+
+    @validator("delimiter")
+    def validate_delimiter(cls, v: str) -> str:
+        if v == r"\t":
+            v = "\t"
+        if len(v) != 1:
+            raise ValueError("delimiter should only be one character")
+        if v in {"\r", "\n"}:
+            raise ValueError(f"delimiter cannot be {v}")
+        return v
+
+    @validator("quote_char")
+    def validate_quote_char(cls, v: str) -> str:
+        if len(v) != 1:
+            raise ValueError("quote_char should only be one character")
+        return v
+
+    @validator("escape_char")
+    def validate_escape_char(cls, v: str) -> str:
+        if v is not None and len(v) != 1:
+            raise ValueError("escape_char should only be one character")
+        return v
+
+    @validator("encoding")
+    def validate_encoding(cls, v: str) -> str:
+        try:
+            codecs.lookup(v)
+        except LookupError:
+            raise ValueError(f"invalid encoding format: {v}")
+        return v
+
+    @root_validator
+    def validate_optional_args(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        definition_type = values.get("header_definition_type")
+        column_names = values.get("user_provided_column_names")
+        if definition_type == CsvHeaderDefinitionType.USER_PROVIDED and not column_names:
+            raise ValidationError(
+                "`user_provided_column_names` should be defined if the definition 'User Provided'.",
+                model=CsvFormat,
+            )
+        if definition_type != CsvHeaderDefinitionType.USER_PROVIDED and column_names:
+            raise ValidationError(
+                "`user_provided_column_names` should not be defined if the definition is not 'User Provided'.",
+                model=CsvFormat,
+            )
+        return values

airbyte_cdk/sources/file_based/config/excel_format.py

@@ -0,0 +1,18 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from pydantic.v1 import BaseModel, Field
+
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
+
+
+class ExcelFormat(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Excel Format"
+        discriminator = "filetype"
+
+    filetype: str = Field(
+        "excel",
+        const=True,
+    )

airbyte_cdk/sources/file_based/config/file_based_stream_config.py

@@ -0,0 +1,99 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from enum import Enum
+from typing import Any, List, Mapping, Optional, Union
+
+from pydantic.v1 import BaseModel, Field, validator
+
+from airbyte_cdk.sources.file_based.config.avro_format import AvroFormat
+from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat
+from airbyte_cdk.sources.file_based.config.excel_format import ExcelFormat
+from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat
+from airbyte_cdk.sources.file_based.config.parquet_format import ParquetFormat
+from airbyte_cdk.sources.file_based.config.unstructured_format import UnstructuredFormat
+from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError
+from airbyte_cdk.sources.file_based.schema_helpers import type_mapping_to_jsonschema
+
+PrimaryKeyType = Optional[Union[str, List[str]]]
+
+
+class ValidationPolicy(Enum):
+    emit_record = "Emit Record"
+    skip_record = "Skip Record"
+    wait_for_discover = "Wait for Discover"
+
+
+class FileBasedStreamConfig(BaseModel):
+    name: str = Field(title="Name", description="The name of the stream.")
+    globs: Optional[List[str]] = Field(
+        default=["**"],
+        title="Globs",
+        description='The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look <a href="https://en.wikipedia.org/wiki/Glob_(programming)">here</a>.',
+        order=1,
+    )
+    legacy_prefix: Optional[str] = Field(
+        title="Legacy Prefix",
+        description="The path prefix configured in v3 versions of the S3 connector. This option is deprecated in favor of a single glob.",
+        airbyte_hidden=True,
+    )
+    validation_policy: ValidationPolicy = Field(
+        title="Validation Policy",
+        description="The name of the validation policy that dictates sync behavior when a record does not adhere to the stream schema.",
+        default=ValidationPolicy.emit_record,
+    )
+    input_schema: Optional[str] = Field(
+        title="Input Schema",
+        description="The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.",
+    )
+    primary_key: Optional[str] = Field(
+        title="Primary Key",
+        description="The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.",
+        airbyte_hidden=True,  # Users can create/modify primary keys in the connection configuration so we shouldn't duplicate it here.
+    )
+    days_to_sync_if_history_is_full: int = Field(
+        title="Days To Sync If History Is Full",
+        description="When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.",
+        default=3,
+    )
+    format: Union[
+        AvroFormat, CsvFormat, JsonlFormat, ParquetFormat, UnstructuredFormat, ExcelFormat
+    ] = Field(
+        title="Format",
+        description="The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.",
+    )
+    schemaless: bool = Field(
+        title="Schemaless",
+        description="When enabled, syncs will not validate or structure records against the stream's schema.",
+        default=False,
+    )
+    recent_n_files_to_read_for_schema_discovery: Optional[int] = Field(
+        title="Files To Read For Schema Discover",
+        description="The number of resent files which will be used to discover the schema for this stream.",
+        default=None,
+        gt=0,
+    )
+
+    @validator("input_schema", pre=True)
+    def validate_input_schema(cls, v: Optional[str]) -> Optional[str]:
+        if v:
+            if type_mapping_to_jsonschema(v):
+                return v
+            else:
+                raise ConfigValidationError(FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA)
+        return None
+
+    def get_input_schema(self) -> Optional[Mapping[str, Any]]:
+        """
+        User defined input_schema is defined as a string in the config. This method takes the string representation
+        and converts it into a Mapping[str, Any] which is used by file-based CDK components.
+        """
+        if self.input_schema:
+            schema = type_mapping_to_jsonschema(self.input_schema)
+            if not schema:
+                raise ValueError(
+                    f"Unable to create JSON schema from input schema {self.input_schema}"
+                )
+            return schema
+        return None

airbyte_cdk/sources/file_based/config/jsonl_format.py

@@ -0,0 +1,18 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from pydantic.v1 import BaseModel, Field
+
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
+
+
+class JsonlFormat(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Jsonl Format"
+        discriminator = "filetype"
+
+    filetype: str = Field(
+        "jsonl",
+        const=True,
+    )

airbyte_cdk/sources/file_based/config/parquet_format.py

@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+from pydantic.v1 import BaseModel, Field
+
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
+
+
+class ParquetFormat(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Parquet Format"
+        discriminator = "filetype"
+
+    filetype: str = Field(
+        "parquet",
+        const=True,
+    )
+    # This option is not recommended, but necessary for backwards compatibility
+    decimal_as_float: bool = Field(
+        title="Convert Decimal Fields to Floats",
+        description="Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended.",
+        default=False,
+    )

airbyte_cdk/sources/file_based/config/unstructured_format.py

@@ -0,0 +1,102 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import List, Literal, Optional, Union
+
+from pydantic.v1 import BaseModel, Field
+
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
+
+
+class LocalProcessingConfigModel(BaseModel):
+    mode: Literal["local"] = Field("local", const=True)
+
+    class Config(OneOfOptionConfig):
+        title = "Local"
+        description = (
+            "Process files locally, supporting `fast` and `ocr` modes. This is the default option."
+        )
+        discriminator = "mode"
+
+
+class APIParameterConfigModel(BaseModel):
+    name: str = Field(
+        title="Parameter name",
+        description="The name of the unstructured API parameter to use",
+        examples=["combine_under_n_chars", "languages"],
+    )
+    value: str = Field(
+        title="Value", description="The value of the parameter", examples=["true", "hi_res"]
+    )
+
+
+class APIProcessingConfigModel(BaseModel):
+    mode: Literal["api"] = Field("api", const=True)
+
+    api_key: str = Field(
+        default="",
+        always_show=True,
+        title="API Key",
+        airbyte_secret=True,
+        description="The API key to use matching the environment",
+    )
+
+    api_url: str = Field(
+        default="https://api.unstructured.io",
+        title="API URL",
+        always_show=True,
+        description="The URL of the unstructured API to use",
+        examples=["https://api.unstructured.com"],
+    )
+
+    parameters: Optional[List[APIParameterConfigModel]] = Field(
+        default=[],
+        always_show=True,
+        title="Additional URL Parameters",
+        description="List of parameters send to the API",
+    )
+
+    class Config(OneOfOptionConfig):
+        title = "via API"
+        description = "Process files via an API, using the `hi_res` mode. This option is useful for increased performance and accuracy, but requires an API key and a hosted instance of unstructured."
+        discriminator = "mode"
+
+
+class UnstructuredFormat(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Unstructured Document Format"
+        description = "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file."
+        discriminator = "filetype"
+
+    filetype: str = Field(
+        "unstructured",
+        const=True,
+    )
+
+    skip_unprocessable_files: bool = Field(
+        default=True,
+        title="Skip Unprocessable Files",
+        description="If true, skip files that cannot be parsed and pass the error message along as the _ab_source_file_parse_error field. If false, fail the sync.",
+        always_show=True,
+    )
+
+    strategy: str = Field(
+        always_show=True,
+        order=0,
+        default="auto",
+        title="Parsing Strategy",
+        enum=["auto", "fast", "ocr_only", "hi_res"],
+        description="The strategy used to parse documents. `fast` extracts text directly from the document which doesn't work for all files. `ocr_only` is more reliable, but slower. `hi_res` is the most reliable, but requires an API key and a hosted instance of unstructured and can't be used with local mode. See the unstructured.io documentation for more details: https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf",
+    )
+
+    processing: Union[
+        LocalProcessingConfigModel,
+        APIProcessingConfigModel,
+    ] = Field(
+        default=LocalProcessingConfigModel(mode="local"),
+        title="Processing",
+        description="Processing configuration",
+        discriminator="mode",
+        type="object",
+    )

airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py

@@ -0,0 +1,81 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import (
+    AbstractFileBasedSpec,
+    DeliverRawFiles,
+)
+from airbyte_cdk.sources.specs.transfer_modes import DeliverPermissions
+
+DELIVERY_TYPE_KEY = "delivery_type"
+DELIVERY_TYPE_PERMISSION_TRANSFER_MODE_VALUE = "use_permissions_transfer"
+DELIVERY_TYPE_FILES_TRANSFER_MODE_VALUE = "use_file_transfer"
+PRESERVE_DIRECTORY_STRUCTURE_KEY = "preserve_directory_structure"
+INCLUDE_IDENTITIES_STREAM_KEY = "include_identities_stream"
+
+
+def use_file_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
+    """Returns `True` if the configuration uses file transfer mode."""
+    return (
+        hasattr(parsed_config.delivery_method, DELIVERY_TYPE_KEY)
+        and parsed_config.delivery_method.delivery_type == DELIVERY_TYPE_FILES_TRANSFER_MODE_VALUE
+    )
+
+
+def preserve_directory_structure(parsed_config: AbstractFileBasedSpec) -> bool:
+    """
+    Determines whether to preserve directory structure during file transfer.
+
+    When enabled, files maintain their subdirectory paths in the destination.
+    When disabled, files are flattened to the root of the destination.
+
+    Args:
+        parsed_config: The parsed configuration containing delivery method settings
+
+    Returns:
+        True if directory structure should be preserved (default), False otherwise
+    """
+    if (
+        use_file_transfer(parsed_config)
+        and hasattr(parsed_config.delivery_method, PRESERVE_DIRECTORY_STRUCTURE_KEY)
+        and isinstance(parsed_config.delivery_method, DeliverRawFiles)
+    ):
+        return parsed_config.delivery_method.preserve_directory_structure
+    return True
+
+
+def use_permissions_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
+    """
+    Determines whether to use permissions transfer to sync ACLs and Identities
+
+    Args:
+        parsed_config: The parsed configuration containing delivery method settings
+
+    Returns:
+        True if permissions transfer should be enabled, False otherwise
+    """
+    return (
+        hasattr(parsed_config.delivery_method, DELIVERY_TYPE_KEY)
+        and parsed_config.delivery_method.delivery_type
+        == DELIVERY_TYPE_PERMISSION_TRANSFER_MODE_VALUE
+    )
+
+
+def include_identities_stream(parsed_config: AbstractFileBasedSpec) -> bool:
+    """
+    There are scenarios where user may not have access to identities but still is valuable to get ACLs
+
+    Args:
+        parsed_config: The parsed configuration containing delivery method settings
+
+    Returns:
+        True if we should include Identities stream.
+    """
+    if (
+        use_permissions_transfer(parsed_config)
+        and hasattr(parsed_config.delivery_method, INCLUDE_IDENTITIES_STREAM_KEY)
+        and isinstance(parsed_config.delivery_method, DeliverPermissions)
+    ):
+        return parsed_config.delivery_method.include_identities_stream
+    return False

airbyte_cdk/sources/file_based/discovery_policy/__init__.py

@@ -0,0 +1,8 @@
+from airbyte_cdk.sources.file_based.discovery_policy.abstract_discovery_policy import (
+    AbstractDiscoveryPolicy,
+)
+from airbyte_cdk.sources.file_based.discovery_policy.default_discovery_policy import (
+    DefaultDiscoveryPolicy,
+)
+
+__all__ = ["AbstractDiscoveryPolicy", "DefaultDiscoveryPolicy"]

airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py

@@ -0,0 +1,21 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+
+from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
+
+
+class AbstractDiscoveryPolicy(ABC):
+    """
+    Used during discovery; allows the developer to configure the number of concurrent
+    requests to send to the source, and the number of files to use for schema discovery.
+    """
+
+    @property
+    @abstractmethod
+    def n_concurrent_requests(self) -> int: ...
+
+    @abstractmethod
+    def get_max_n_files_for_schema_inference(self, parser: FileTypeParser) -> int: ...

airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py

@@ -0,0 +1,33 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.file_based.discovery_policy.abstract_discovery_policy import (
+    AbstractDiscoveryPolicy,
+)
+from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
+
+DEFAULT_N_CONCURRENT_REQUESTS = 10
+DEFAULT_MAX_N_FILES_FOR_STREAM_SCHEMA_INFERENCE = 10
+
+
+class DefaultDiscoveryPolicy(AbstractDiscoveryPolicy):
+    """
+    Default number of concurrent requests to send to the source on discover, and number
+    of files to use for schema inference.
+    """
+
+    @property
+    def n_concurrent_requests(self) -> int:
+        return DEFAULT_N_CONCURRENT_REQUESTS
+
+    def get_max_n_files_for_schema_inference(self, parser: FileTypeParser) -> int:
+        return min(
+            filter(
+                None,
+                (
+                    DEFAULT_MAX_N_FILES_FOR_STREAM_SCHEMA_INFERENCE,
+                    parser.parser_max_n_files_for_schema_inference,
+                ),
+            )
+        )