airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py

@@ -0,0 +1,123 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Iterable, Optional
+
+from airbyte_cdk.sources.file_based import AbstractFileBasedSpec
+from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+
+
+class AbstractFileBasedStreamPermissionsReader(ABC):
+    """
+    This class is responsible for reading file permissions and Identities from a source.
+    """
+
+    def __init__(self) -> None:
+        self._config = None
+
+    @property
+    def config(self) -> Optional[AbstractFileBasedSpec]:
+        return self._config
+
+    @config.setter
+    @abstractmethod
+    def config(self, value: AbstractFileBasedSpec) -> None:
+        """
+        FileBasedSource reads the config from disk and parses it, and once parsed, the source sets the config on its StreamReader.
+
+        Note: FileBasedSource only requires the keys defined in the abstract config, whereas concrete implementations of StreamReader
+        will require keys that (for example) allow it to authenticate with the 3rd party.
+
+        Therefore, concrete implementations of AbstractFileBasedStreamPermissionsReader's's config setter should assert that `value` is of the correct
+        config type for that type of StreamReader.
+        """
+        ...
+
+    @abstractmethod
+    def get_file_acl_permissions(self, file: RemoteFile, logger: logging.Logger) -> Dict[str, Any]:
+        """
+        This function should return the allow list for a given file, i.e. the list of all identities and their permission levels associated with it
+
+        e.g.
+        def get_file_acl_permissions(self, file: RemoteFile, logger: logging.Logger):
+            api_conn = some_api.conn(credentials=SOME_CREDENTIALS)
+            result = api_conn.get_file_permissions_info(file.id)
+            return MyPermissionsModel(
+                id=result["id"],
+                access_control_list = result["access_control_list"],
+                is_public = result["is_public"],
+                ).dict()
+        """
+        ...
+
+    @abstractmethod
+    def load_identity_groups(self, logger: logging.Logger) -> Iterable[Dict[str, Any]]:
+        """
+        This function should return the Identities in a determined "space" or "domain" where the file metadata (ACLs) are fetched and ACLs items (Identities) exists.
+
+        e.g.
+        def load_identity_groups(self, logger: logging.Logger) -> Iterable[Dict[str, Any]]:
+            api_conn = some_api.conn(credentials=SOME_CREDENTIALS)
+            users_api = api_conn.users()
+            groups_api = api_conn.groups()
+            members_api = self.google_directory_service.members()
+            for user in users_api.list():
+                yield my_identity_model(id=user.id, name=user.name, email_address=user.email, type="user").dict()
+            for group in groups_api.list():
+                group_obj = my_identity_model(id=group.id, name=groups.name, email_address=user.email, type="group").dict()
+                for member in members_api.list(group=group):
+                    group_obj.member_email_addresses = group_obj.member_email_addresses or []
+                    group_obj.member_email_addresses.append(member.email)
+                yield group_obj.dict()
+        """
+        ...
+
+    @property
+    @abstractmethod
+    def file_permissions_schema(self) -> Dict[str, Any]:
+        """
+        This function should return the permissions schema for file permissions stream.
+
+        e.g.
+        def file_permissions_schema(self) -> Dict[str, Any]:
+            # you can also follow the pattern we have for python connectors and have a json file and read from there e.g. schemas/identities.json
+            return {
+                  "type": "object",
+                  "properties": {
+                    "id": { "type": "string" },
+                    "file_path": { "type": "string" },
+                    "access_control_list": {
+                      "type": "array",
+                      "items": { "type": "string" }
+                    },
+                    "publicly_accessible": { "type": "boolean" }
+                  }
+                }
+        """
+        ...
+
+    @property
+    @abstractmethod
+    def identities_schema(self) -> Dict[str, Any]:
+        """
+        This function should return the identities schema for file identity stream.
+
+        e.g.
+        def identities_schema(self) -> Dict[str, Any]:
+            # you can also follow the pattern we have for python connectors and have a json file and read from there e.g. schemas/identities.json
+            return {
+              "type": "object",
+              "properties": {
+                "id": { "type": "string" },
+                "remote_id": { "type": "string" },
+                "name": { "type": ["null", "string"] },
+                "email_address": { "type": ["null", "string"] },
+                "member_email_addresses": { "type": ["null", "array"] },
+                "type": { "type": "string" },
+              }
+            }
+        """
+        ...

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -0,0 +1,209 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from abc import ABC, abstractmethod
+from datetime import datetime
+from enum import Enum
+from io import IOBase
+from os import makedirs, path
+from typing import Any, Callable, Iterable, List, MutableMapping, Optional, Set, Tuple
+
+from wcmatch.glob import GLOBSTAR, globmatch
+
+from airbyte_cdk.models import AirbyteRecordMessageFileReference
+from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
+from airbyte_cdk.sources.file_based.config.validate_config_transfer_modes import (
+    include_identities_stream,
+    preserve_directory_structure,
+    use_file_transfer,
+)
+from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
+from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+
+
+class FileReadMode(Enum):
+    READ = "r"
+    READ_BINARY = "rb"
+
+
+class AbstractFileBasedStreamReader(ABC):
+    DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+    FILE_RELATIVE_PATH = "file_relative_path"
+    FILE_NAME = "file_name"
+    LOCAL_FILE_PATH = "local_file_path"
+    SOURCE_FILE_URI = "source_file_relative_path"
+    FILE_FOLDER = "file_folder"
+
+    def __init__(self) -> None:
+        self._config = None
+
+    @property
+    def config(self) -> Optional[AbstractFileBasedSpec]:
+        return self._config
+
+    @config.setter
+    @abstractmethod
+    def config(self, value: AbstractFileBasedSpec) -> None:
+        """
+        FileBasedSource reads the config from disk and parses it, and once parsed, the source sets the config on its StreamReader.
+
+        Note: FileBasedSource only requires the keys defined in the abstract config, whereas concrete implementations of StreamReader
+        will require keys that (for example) allow it to authenticate with the 3rd party.
+
+        Therefore, concrete implementations of AbstractFileBasedStreamReader's config setter should assert that `value` is of the correct
+        config type for that type of StreamReader.
+        """
+        ...
+
+    @abstractmethod
+    def open_file(
+        self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger
+    ) -> IOBase:
+        """
+        Return a file handle for reading.
+
+        Many sources will be able to use smart_open to implement this method,
+        for example:
+
+        client = boto3.Session(...)
+        return smart_open.open(remote_file.uri, transport_params={"client": client})
+        """
+        ...
+
+    @abstractmethod
+    def get_matching_files(
+        self,
+        globs: List[str],
+        prefix: Optional[str],
+        logger: logging.Logger,
+    ) -> Iterable[RemoteFile]:
+        """
+        Return all files that match any of the globs.
+
+        Example:
+
+        The source has files "a.json", "foo/a.json", "foo/bar/a.json"
+
+        If globs = ["*.json"] then this method returns ["a.json"].
+
+        If globs = ["foo/*.json"] then this method returns ["foo/a.json"].
+
+        Utility method `self.filter_files_by_globs` and `self.get_prefixes_from_globs`
+        are available, which may be helpful when implementing this method.
+        """
+        ...
+
+    def filter_files_by_globs_and_start_date(
+        self, files: List[RemoteFile], globs: List[str]
+    ) -> Iterable[RemoteFile]:
+        """
+        Utility method for filtering files based on globs.
+        """
+        start_date = (
+            datetime.strptime(self.config.start_date, self.DATE_TIME_FORMAT)
+            if self.config and self.config.start_date
+            else None
+        )
+        seen = set()
+
+        for file in files:
+            if self.file_matches_globs(file, globs):
+                if file.uri not in seen and (not start_date or file.last_modified >= start_date):
+                    seen.add(file.uri)
+                    yield file
+
+    @abstractmethod
+    def file_size(self, file: RemoteFile) -> int:
+        """Utility method to get size of the remote file.
+
+        This is required for connectors that will support writing to
+        files. If the connector does not support writing files, then the
+        subclass can simply `return 0`.
+        """
+        ...
+
+    @staticmethod
+    def file_matches_globs(file: RemoteFile, globs: List[str]) -> bool:
+        # Use the GLOBSTAR flag to enable recursive ** matching
+        # (https://facelessuser.github.io/wcmatch/wcmatch/#globstar)
+        return any(globmatch(file.uri, g, flags=GLOBSTAR) for g in globs)
+
+    @staticmethod
+    def get_prefixes_from_globs(globs: List[str]) -> Set[str]:
+        """
+        Utility method for extracting prefixes from the globs.
+        """
+        prefixes = {glob.split("*")[0] for glob in globs}
+        return set(filter(lambda x: bool(x), prefixes))
+
+    def use_file_transfer(self) -> bool:
+        if self.config:
+            return use_file_transfer(self.config)
+        return False
+
+    def preserve_directory_structure(self) -> bool:
+        # fall back to preserve subdirectories if config is not present or incomplete
+        if self.config:
+            return preserve_directory_structure(self.config)
+        return True
+
+    def include_identities_stream(self) -> bool:
+        if self.config:
+            return include_identities_stream(self.config)
+        return False
+
+    @abstractmethod
+    def upload(
+        self, file: RemoteFile, local_directory: str, logger: logging.Logger
+    ) -> Tuple[FileRecordData, AirbyteRecordMessageFileReference]:
+        """
+        This is required for connectors that will support writing to
+        files. It will handle the logic to download,get,read,acquire or
+        whatever is more efficient to get a file from the source.
+
+        Args:
+               file (RemoteFile): The remote file object containing URI and metadata.
+               local_directory (str): The local directory path where the file will be downloaded.
+               logger (logging.Logger): Logger for logging information and errors.
+
+           Returns:
+               AirbyteRecordMessageFileReference: A file reference object containing:
+                   - staging_file_url (str): The absolute path to the referenced file in the staging area.
+                   - file_size_bytes (int): The size of the referenced file in bytes.
+                   - source_file_relative_path (str): The relative path to the referenced file in source.
+        """
+        ...
+
+    def _get_file_transfer_paths(
+        self,
+        file: RemoteFile,
+        local_directory: str,
+        parse_file_path_from_uri: Optional[Callable[[str], str]] = None,
+    ) -> MutableMapping[str, Any]:
+        preserve_directory_structure = self.preserve_directory_structure()
+        if not parse_file_path_from_uri:
+            file_path = file.uri
+        else:
+            file_path = parse_file_path_from_uri(file.uri)
+
+        file_name = path.basename(file_path)
+        file_folder = path.dirname(file_path)
+        if preserve_directory_structure:
+            # Remove left slashes from source path format to make relative path for writing locally
+            file_relative_path = file_path.lstrip("/")
+        else:
+            file_relative_path = file_name
+        local_file_path = path.join(local_directory, file_relative_path)
+        # Ensure the local directory exists
+        makedirs(path.dirname(local_file_path), exist_ok=True)
+
+        file_paths = {
+            self.FILE_RELATIVE_PATH: file_relative_path,
+            self.LOCAL_FILE_PATH: local_file_path,
+            self.FILE_NAME: file_name,
+            self.FILE_FOLDER: file_folder,
+            self.SOURCE_FILE_URI: file.uri,
+        }
+        return file_paths

airbyte_cdk/sources/file_based/file_record_data.py

@@ -0,0 +1,22 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from datetime import datetime
+from typing import Optional
+
+from pydantic.v1 import BaseModel
+
+
+class FileRecordData(BaseModel):
+    """
+    A record in a file-based stream.
+    """
+
+    folder: str
+    filename: str
+    bytes: int
+
+    id: Optional[str] = None
+    updated_at: Optional[str] = None
+    mime_type: Optional[str] = None

airbyte_cdk/sources/file_based/file_types/__init__.py

@@ -0,0 +1,37 @@
+from typing import Any, Mapping, Type
+
+from airbyte_cdk.sources.file_based.config.avro_format import AvroFormat
+from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat
+from airbyte_cdk.sources.file_based.config.excel_format import ExcelFormat
+from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat
+from airbyte_cdk.sources.file_based.config.parquet_format import ParquetFormat
+from airbyte_cdk.sources.file_based.config.unstructured_format import UnstructuredFormat
+
+from .avro_parser import AvroParser
+from .csv_parser import CsvParser
+from .excel_parser import ExcelParser
+from .file_transfer import FileTransfer
+from .file_type_parser import FileTypeParser
+from .jsonl_parser import JsonlParser
+from .parquet_parser import ParquetParser
+from .unstructured_parser import UnstructuredParser
+
+default_parsers: Mapping[Type[Any], FileTypeParser] = {
+    AvroFormat: AvroParser(),
+    CsvFormat: CsvParser(),
+    ExcelFormat: ExcelParser(),
+    JsonlFormat: JsonlParser(),
+    ParquetFormat: ParquetParser(),
+    UnstructuredFormat: UnstructuredParser(),
+}
+
+__all__ = [
+    "AvroParser",
+    "CsvParser",
+    "ExcelParser",
+    "JsonlParser",
+    "ParquetParser",
+    "UnstructuredParser",
+    "FileTransfer",
+    "default_parsers",
+]

airbyte_cdk/sources/file_based/file_types/avro_parser.py

@@ -0,0 +1,233 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from typing import Any, Dict, Iterable, Mapping, Optional, Tuple, cast
+
+import fastavro
+
+from airbyte_cdk.sources.file_based.config.avro_format import AvroFormat
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
+from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError
+from airbyte_cdk.sources.file_based.file_based_stream_reader import (
+    AbstractFileBasedStreamReader,
+    FileReadMode,
+)
+from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
+from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+from airbyte_cdk.sources.file_based.schema_helpers import SchemaType
+
+AVRO_TYPE_TO_JSON_TYPE = {
+    "null": "null",
+    "boolean": "boolean",
+    "int": "integer",
+    "long": "integer",
+    "float": "number",
+    "double": "string",  # double -> number conversions can lose precision
+    "bytes": "string",
+    "string": "string",
+}
+
+AVRO_LOGICAL_TYPE_TO_JSON = {
+    "decimal": {"type": "string"},
+    "uuid": {"type": "string"},
+    "date": {"type": "string", "format": "date"},
+    "time-millis": {"type": "integer"},
+    "time-micros": {"type": "integer"},
+    "timestamp-millis": {"type": "string", "format": "date-time"},
+    "timestamp-micros": {"type": "string"},
+    "local-timestamp-millis": {"type": "string", "format": "date-time"},
+    "local-timestamp-micros": {"type": "string"},
+    # fastavro does not support duration https://fastavro.readthedocs.io/en/latest/logical_types.html
+}
+
+
+class AvroParser(FileTypeParser):
+    ENCODING = None
+
+    def check_config(self, config: FileBasedStreamConfig) -> Tuple[bool, Optional[str]]:
+        """
+        AvroParser does not require config checks, implicit pydantic validation is enough.
+        """
+        return True, None
+
+    async def infer_schema(
+        self,
+        config: FileBasedStreamConfig,
+        file: RemoteFile,
+        stream_reader: AbstractFileBasedStreamReader,
+        logger: logging.Logger,
+    ) -> SchemaType:
+        avro_format = config.format
+        if not isinstance(avro_format, AvroFormat):
+            raise ValueError(f"Expected ParquetFormat, got {avro_format}")
+
+        with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp:
+            avro_reader = fastavro.reader(fp)  # type: ignore [arg-type]
+            avro_schema = avro_reader.writer_schema
+        if not avro_schema["type"] == "record":  # type: ignore [index, call-overload]
+            unsupported_type = avro_schema["type"]  # type: ignore [index, call-overload]
+            raise ValueError(
+                f"Only record based avro files are supported. Found {unsupported_type}"
+            )
+        json_schema = {
+            field["name"]: AvroParser._convert_avro_type_to_json(  # type: ignore [index]
+                avro_format,
+                field["name"],  # type: ignore [index]
+                field["type"],  # type: ignore [index]
+            )
+            for field in avro_schema["fields"]  # type: ignore [index, call-overload]
+        }
+        return json_schema
+
+    @classmethod
+    def _convert_avro_type_to_json(
+        cls, avro_format: AvroFormat, field_name: str, avro_field: str
+    ) -> Mapping[str, Any]:
+        if isinstance(avro_field, str) and avro_field in AVRO_TYPE_TO_JSON_TYPE:
+            # Legacy behavior to retain backwards compatibility. Long term we should always represent doubles as strings
+            if avro_field == "double" and not avro_format.double_as_string:
+                return {"type": "number"}
+            return {"type": AVRO_TYPE_TO_JSON_TYPE[avro_field]}
+        if isinstance(avro_field, Mapping):
+            if avro_field["type"] == "record":
+                return {
+                    "type": "object",
+                    "properties": {
+                        object_field["name"]: AvroParser._convert_avro_type_to_json(
+                            avro_format, object_field["name"], object_field["type"]
+                        )
+                        for object_field in avro_field["fields"]
+                    },
+                }
+            elif avro_field["type"] == "array":
+                if "items" not in avro_field:
+                    raise ValueError(
+                        f"{field_name} array type does not have a required field items"
+                    )
+                return {
+                    "type": "array",
+                    "items": AvroParser._convert_avro_type_to_json(
+                        avro_format, "", avro_field["items"]
+                    ),
+                }
+            elif avro_field["type"] == "enum":
+                if "symbols" not in avro_field:
+                    raise ValueError(
+                        f"{field_name} enum type does not have a required field symbols"
+                    )
+                if "name" not in avro_field:
+                    raise ValueError(f"{field_name} enum type does not have a required field name")
+                return {"type": "string", "enum": avro_field["symbols"]}
+            elif avro_field["type"] == "map":
+                if "values" not in avro_field:
+                    raise ValueError(f"{field_name} map type does not have a required field values")
+                return {
+                    "type": "object",
+                    "additionalProperties": AvroParser._convert_avro_type_to_json(
+                        avro_format, "", avro_field["values"]
+                    ),
+                }
+            elif avro_field["type"] == "fixed" and avro_field.get("logicalType") != "duration":
+                if "size" not in avro_field:
+                    raise ValueError(f"{field_name} fixed type does not have a required field size")
+                if not isinstance(avro_field["size"], int):
+                    raise ValueError(f"{field_name} fixed type size value is not an integer")
+                return {
+                    "type": "string",
+                    "pattern": f"^[0-9A-Fa-f]{{{avro_field['size'] * 2}}}$",
+                }
+            elif avro_field.get("logicalType") == "decimal":
+                if "precision" not in avro_field:
+                    raise ValueError(
+                        f"{field_name} decimal type does not have a required field precision"
+                    )
+                if "scale" not in avro_field:
+                    raise ValueError(
+                        f"{field_name} decimal type does not have a required field scale"
+                    )
+                max_whole_number_range = avro_field["precision"] - avro_field["scale"]
+                decimal_range = avro_field["scale"]
+
+                # This regex looks like a mess, but it is validation for at least one whole number and optional fractional numbers
+                # For example: ^-?\d{1,5}(?:\.\d{1,3})?$ would accept 12345.123 and 123456.12345 would  be rejected
+                return {
+                    "type": "string",
+                    "pattern": f"^-?\\d{{{1,max_whole_number_range}}}(?:\\.\\d{1,decimal_range})?$",
+                }
+            elif "logicalType" in avro_field:
+                if avro_field["logicalType"] not in AVRO_LOGICAL_TYPE_TO_JSON:
+                    raise ValueError(
+                        f"{avro_field['logicalType']} is not a valid Avro logical type"
+                    )
+                return AVRO_LOGICAL_TYPE_TO_JSON[avro_field["logicalType"]]
+            else:
+                raise ValueError(f"Unsupported avro type: {avro_field}")
+        else:
+            raise ValueError(f"Unsupported avro type: {avro_field}")
+
+    def parse_records(
+        self,
+        config: FileBasedStreamConfig,
+        file: RemoteFile,
+        stream_reader: AbstractFileBasedStreamReader,
+        logger: logging.Logger,
+        discovered_schema: Optional[Mapping[str, SchemaType]],
+    ) -> Iterable[Dict[str, Any]]:
+        avro_format = config.format or AvroFormat(filetype="avro")
+        if not isinstance(avro_format, AvroFormat):
+            raise ValueError(f"Expected ParquetFormat, got {avro_format}")
+
+        line_no = 0
+        try:
+            with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp:
+                avro_reader = fastavro.reader(fp)  # type: ignore [arg-type]
+                schema = avro_reader.writer_schema
+                schema_field_name_to_type = {
+                    field["name"]: cast(dict[str, Any], field["type"])  # type: ignore [index]
+                    for field in schema["fields"]  # type: ignore [index, call-overload]  # If schema is not dict, it is not subscriptable by strings
+                }
+                for record in avro_reader:
+                    line_no += 1
+                    yield {
+                        record_field: self._to_output_value(
+                            avro_format,
+                            schema_field_name_to_type[record_field],  # type: ignore [index] # Any not subscriptable
+                            record[record_field],  # type: ignore [index] # Any not subscriptable
+                        )
+                        for record_field, record_value in schema_field_name_to_type.items()
+                    }
+        except Exception as exc:
+            raise RecordParseError(
+                FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=line_no
+            ) from exc
+
+    @property
+    def file_read_mode(self) -> FileReadMode:
+        return FileReadMode.READ_BINARY
+
+    @staticmethod
+    def _to_output_value(
+        avro_format: AvroFormat, record_type: Mapping[str, Any], record_value: Any
+    ) -> Any:
+        if isinstance(record_value, bytes):
+            return record_value.decode()
+        elif not isinstance(record_type, Mapping):
+            if record_type == "double" and avro_format.double_as_string:
+                return str(record_value)
+            return record_value
+        if record_type.get("logicalType") in ("decimal", "uuid"):
+            return str(record_value)
+        elif record_type.get("logicalType") == "date":
+            return record_value.isoformat()
+        elif record_type.get("logicalType") == "timestamp-millis":
+            return record_value.isoformat(sep="T", timespec="milliseconds")
+        elif record_type.get("logicalType") == "timestamp-micros":
+            return record_value.isoformat(sep="T", timespec="microseconds")
+        elif record_type.get("logicalType") == "local-timestamp-millis":
+            return record_value.isoformat(sep="T", timespec="milliseconds")
+        elif record_type.get("logicalType") == "local-timestamp-micros":
+            return record_value.isoformat(sep="T", timespec="microseconds")
+        else:
+            return record_value