airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/file_based/exceptions.py

@@ -0,0 +1,159 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from enum import Enum
+from typing import Any, List, Union
+
+from airbyte_cdk.models import AirbyteMessage, FailureType
+from airbyte_cdk.utils import AirbyteTracedException
+
+
+class FileBasedSourceError(Enum):
+    EMPTY_STREAM = "No files were identified in the stream. This may be because there are no files in the specified container, or because your glob patterns did not match any files. Please verify that your source contains files last modified after the start_date and that your glob patterns are not overly strict."
+    GLOB_PARSE_ERROR = "Error parsing glob pattern. Please refer to the glob pattern rules at https://facelessuser.github.io/wcmatch/glob/#split."
+    ENCODING_ERROR = "File encoding error. The configured encoding must match file encoding."
+    ERROR_CASTING_VALUE = "Could not cast the value to the expected type."
+    ERROR_CASTING_VALUE_UNRECOGNIZED_TYPE = "Could not cast the value to the expected type because the type is not recognized. Valid types are null, array, boolean, integer, number, object, and string."
+    ERROR_DECODING_VALUE = "Expected a JSON-decodeable value but could not decode record."
+    ERROR_LISTING_FILES = "Error listing files. Please check the credentials provided in the config and verify that they provide permission to list files."
+    ERROR_READING_FILE = "Error opening file. Please check the credentials provided in the config and verify that they provide permission to read files."
+    ERROR_PARSING_RECORD = "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable."
+    ERROR_PARSING_USER_PROVIDED_SCHEMA = (
+        "The provided schema could not be transformed into valid JSON Schema."
+    )
+    ERROR_VALIDATING_RECORD = "One or more records do not pass the schema validation policy. Please modify your input schema, or select a more lenient validation policy."
+    ERROR_PARSING_RECORD_MISMATCHED_COLUMNS = "A header field has resolved to `None`. This indicates that the CSV has more rows than the number of header fields. If you input your schema or headers, please verify that the number of columns corresponds to the number of columns in your CSV's rows."
+    ERROR_PARSING_RECORD_MISMATCHED_ROWS = "A row's value has resolved to `None`. This indicates that the CSV has more columns in the header field than the number of columns in the row(s). If you input your schema or headers, please verify that the number of columns corresponds to the number of columns in your CSV's rows."
+    STOP_SYNC_PER_SCHEMA_VALIDATION_POLICY = "Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema."
+    NULL_VALUE_IN_SCHEMA = "Error during schema inference: no type was detected for key."
+    UNRECOGNIZED_TYPE = "Error during schema inference: unrecognized type."
+    SCHEMA_INFERENCE_ERROR = "Error inferring schema from files. Are the files valid?"
+    INVALID_SCHEMA_ERROR = "No fields were identified for this schema. This may happen if the stream is empty. Please check your configuration to verify that there are files that match the stream's glob patterns."
+    CONFIG_VALIDATION_ERROR = "Error creating stream config object."
+    MISSING_SCHEMA = "Expected `json_schema` in the configured catalog but it is missing."
+    UNDEFINED_PARSER = "No parser is defined for this file type."
+    UNDEFINED_VALIDATION_POLICY = (
+        "The validation policy defined in the config does not exist for the source."
+    )
+
+
+class FileBasedErrorsCollector:
+    """
+    The placeholder for all errors collected.
+    """
+
+    errors: List[AirbyteMessage] = []
+
+    def yield_and_raise_collected(self) -> Any:
+        if self.errors:
+            # emit collected logged messages
+            yield from self.errors
+            # clean the collector
+            self.errors.clear()
+            # raising the single exception
+            raise AirbyteTracedException(
+                internal_message="Please check the logged errors for more information.",
+                message="Some errors occured while reading from the source.",
+                failure_type=FailureType.config_error,
+            )
+
+    def collect(self, logged_error: AirbyteMessage) -> None:
+        self.errors.append(logged_error)
+
+
+class BaseFileBasedSourceError(Exception):
+    def __init__(self, error: Union[FileBasedSourceError, str], **kwargs):  # type: ignore # noqa
+        if isinstance(error, FileBasedSourceError):
+            error = FileBasedSourceError(error).value
+        super().__init__(
+            f"{error} Contact Support if you need assistance.\n{' '.join([f'{k}={v}' for k, v in kwargs.items()])}"
+        )
+
+
+class ConfigValidationError(BaseFileBasedSourceError):
+    pass
+
+
+class InvalidSchemaError(BaseFileBasedSourceError):
+    pass
+
+
+class MissingSchemaError(BaseFileBasedSourceError):
+    pass
+
+
+class NoFilesMatchingError(BaseFileBasedSourceError):
+    pass
+
+
+class RecordParseError(BaseFileBasedSourceError):
+    pass
+
+
+class SchemaInferenceError(BaseFileBasedSourceError):
+    pass
+
+
+class CheckAvailabilityError(BaseFileBasedSourceError):
+    pass
+
+
+class UndefinedParserError(BaseFileBasedSourceError):
+    pass
+
+
+class StopSyncPerValidationPolicy(BaseFileBasedSourceError):
+    pass
+
+
+class ErrorListingFiles(BaseFileBasedSourceError):
+    pass
+
+
+class DuplicatedFilesError(BaseFileBasedSourceError):
+    def __init__(self, duplicated_files_names: List[dict[str, List[str]]], **kwargs: Any):
+        self._duplicated_files_names = duplicated_files_names
+        self._stream_name: str = kwargs["stream"]
+        super().__init__(self._format_duplicate_files_error_message(), **kwargs)
+
+    def _format_duplicate_files_error_message(self) -> str:
+        duplicated_files_messages = []
+        for duplicated_file in self._duplicated_files_names:
+            for duplicated_file_name, file_paths in duplicated_file.items():
+                file_duplicated_message = (
+                    f"{len(file_paths)} duplicates found for file name {duplicated_file_name}:\n\n"
+                    + "".join(f"\n - {file_paths}")
+                )
+                duplicated_files_messages.append(file_duplicated_message)
+
+        error_message = (
+            f"ERROR: Duplicate filenames found for stream {self._stream_name}. "
+            "Duplicate file names are not allowed if the Preserve Sub-Directories in File Paths option is disabled. "
+            "Please remove or rename the duplicate files before attempting to re-run the sync.\n\n"
+            + "\n".join(duplicated_files_messages)
+        )
+
+        return error_message
+
+    def __repr__(self) -> str:
+        """Return a string representation of the exception."""
+        class_name = self.__class__.__name__
+        properties_str = ", ".join(
+            f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_")
+        )
+        return f"{class_name}({properties_str})"
+
+
+class CustomFileBasedException(AirbyteTracedException):
+    """
+    A specialized exception for file-based connectors.
+
+    This exception is designed to bypass the default error handling in the file-based CDK, allowing the use of custom error messages.
+    """
+
+    pass
+
+
+class FileSizeLimitError(CustomFileBasedException):
+    pass

airbyte_cdk/sources/file_based/file_based_source.py

@@ -0,0 +1,466 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+import traceback
+from abc import ABC
+from collections import Counter
+from typing import Any, Iterator, List, Mapping, Optional, Tuple, Type, Union
+
+from pydantic.v1.error_wrappers import ValidationError
+
+from airbyte_cdk.logger import AirbyteLogFormatter, init_logger
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    AirbyteStateMessage,
+    AirbyteStream,
+    ConfiguredAirbyteCatalog,
+    ConnectorSpecification,
+    FailureType,
+    Level,
+    SyncMode,
+)
+from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource
+from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
+from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
+from airbyte_cdk.sources.file_based.availability_strategy import (
+    AbstractFileBasedAvailabilityStrategy,
+    DefaultFileBasedAvailabilityStrategy,
+)
+from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import (
+    FileBasedStreamConfig,
+    ValidationPolicy,
+)
+from airbyte_cdk.sources.file_based.config.validate_config_transfer_modes import (
+    include_identities_stream,
+    preserve_directory_structure,
+    use_file_transfer,
+    use_permissions_transfer,
+)
+from airbyte_cdk.sources.file_based.discovery_policy import (
+    AbstractDiscoveryPolicy,
+    DefaultDiscoveryPolicy,
+)
+from airbyte_cdk.sources.file_based.exceptions import (
+    ConfigValidationError,
+    FileBasedErrorsCollector,
+    FileBasedSourceError,
+)
+from airbyte_cdk.sources.file_based.file_based_stream_permissions_reader import (
+    AbstractFileBasedStreamPermissionsReader,
+)
+from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
+from airbyte_cdk.sources.file_based.file_types import default_parsers
+from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
+from airbyte_cdk.sources.file_based.schema_validation_policies import (
+    DEFAULT_SCHEMA_VALIDATION_POLICIES,
+    AbstractSchemaValidationPolicy,
+)
+from airbyte_cdk.sources.file_based.stream import (
+    AbstractFileBasedStream,
+    DefaultFileBasedStream,
+    FileIdentitiesStream,
+    PermissionsFileBasedStream,
+)
+from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamFacade
+from airbyte_cdk.sources.file_based.stream.concurrent.cursor import (
+    AbstractConcurrentFileBasedCursor,
+    FileBasedConcurrentCursor,
+    FileBasedFinalStateCursor,
+)
+from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
+from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository
+from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
+from airbyte_cdk.utils.analytics_message import create_analytics_message
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
+
+DEFAULT_CONCURRENCY = 100
+MAX_CONCURRENCY = 100
+INITIAL_N_PARTITIONS = MAX_CONCURRENCY // 2
+IDENTITIES_STREAM = "identities"
+
+
+class FileBasedSource(ConcurrentSourceAdapter, ABC):
+    # We make each source override the concurrency level to give control over when they are upgraded.
+    _concurrency_level = None
+
+    def __init__(
+        self,
+        stream_reader: AbstractFileBasedStreamReader,
+        spec_class: Type[AbstractFileBasedSpec],
+        catalog: Optional[ConfiguredAirbyteCatalog],
+        config: Optional[Mapping[str, Any]],
+        state: Optional[List[AirbyteStateMessage]],
+        availability_strategy: Optional[AbstractFileBasedAvailabilityStrategy] = None,
+        discovery_policy: AbstractDiscoveryPolicy = DefaultDiscoveryPolicy(),
+        parsers: Mapping[Type[Any], FileTypeParser] = default_parsers,
+        validation_policies: Mapping[
+            ValidationPolicy, AbstractSchemaValidationPolicy
+        ] = DEFAULT_SCHEMA_VALIDATION_POLICIES,
+        cursor_cls: Type[
+            Union[AbstractConcurrentFileBasedCursor, AbstractFileBasedCursor]
+        ] = FileBasedConcurrentCursor,
+        stream_permissions_reader: Optional[AbstractFileBasedStreamPermissionsReader] = None,
+    ):
+        self.stream_reader = stream_reader
+        self.stream_permissions_reader = stream_permissions_reader
+        self.spec_class = spec_class
+        self.config = config
+        self.catalog = catalog
+        self.state = state
+        self.availability_strategy = availability_strategy or DefaultFileBasedAvailabilityStrategy(
+            stream_reader
+        )
+        self.discovery_policy = discovery_policy
+        self.parsers = parsers
+        self.validation_policies = validation_policies
+        self.stream_schemas = (
+            {s.stream.name: s.stream.json_schema for s in catalog.streams} if catalog else {}
+        )
+        self.cursor_cls = cursor_cls
+        self.logger = init_logger(f"airbyte.{self.name}")
+        self.errors_collector: FileBasedErrorsCollector = FileBasedErrorsCollector()
+        self._message_repository: Optional[MessageRepository] = None
+        concurrent_source = ConcurrentSource.create(
+            MAX_CONCURRENCY,
+            INITIAL_N_PARTITIONS,
+            self.logger,
+            self._slice_logger,
+            self.message_repository,
+        )
+        self._state = None
+        super().__init__(concurrent_source)
+
+    @property
+    def message_repository(self) -> MessageRepository:
+        if self._message_repository is None:
+            self._message_repository = InMemoryMessageRepository(
+                Level(AirbyteLogFormatter.level_mapping[self.logger.level])
+            )
+        return self._message_repository
+
+    def check_connection(
+        self, logger: logging.Logger, config: Mapping[str, Any]
+    ) -> Tuple[bool, Optional[Any]]:
+        """
+        Check that the source can be accessed using the user-provided configuration.
+
+        For each stream, verify that we can list and read files.
+
+        Returns (True, None) if the connection check is successful.
+
+        Otherwise, the "error" object should describe what went wrong.
+        """
+        try:
+            streams = self.streams(config)
+        except Exception as config_exception:
+            raise AirbyteTracedException(
+                internal_message="Please check the logged errors for more information.",
+                message=FileBasedSourceError.CONFIG_VALIDATION_ERROR.value,
+                exception=AirbyteTracedException(exception=config_exception),
+                failure_type=FailureType.config_error,
+            )
+        if len(streams) == 0:
+            return (
+                False,
+                f"No streams are available for source {self.name}. This is probably an issue with the connector. Please verify that your "
+                f"configuration provides permissions to list and read files from the source. Contact support if you are unable to "
+                f"resolve this issue.",
+            )
+
+        errors = []
+        tracebacks = []
+        for stream in streams:
+            if isinstance(stream, FileIdentitiesStream):
+                identity = next(iter(stream.load_identity_groups()))
+                if not identity:
+                    errors.append(
+                        "Unable to get identities for current configuration, please check your credentials"
+                    )
+                continue
+            if not isinstance(stream, AbstractFileBasedStream):
+                raise ValueError(f"Stream {stream} is not a file-based stream.")
+            try:
+                parsed_config = self._get_parsed_config(config)
+                availability_method = (
+                    stream.availability_strategy.check_availability
+                    if use_file_transfer(parsed_config) or use_permissions_transfer(parsed_config)
+                    else stream.availability_strategy.check_availability_and_parsability
+                )
+                (
+                    stream_is_available,
+                    reason,
+                ) = availability_method(stream, logger, self)
+            except AirbyteTracedException as ate:
+                errors.append(f"Unable to connect to stream {stream.name} - {ate.message}")
+                tracebacks.append(traceback.format_exc())
+            except Exception:
+                errors.append(f"Unable to connect to stream {stream.name}")
+                tracebacks.append(traceback.format_exc())
+            else:
+                if not stream_is_available and reason:
+                    errors.append(reason)
+
+        if len(errors) == 1 and len(tracebacks) == 1:
+            raise AirbyteTracedException(
+                internal_message=tracebacks[0],
+                message=f"{errors[0]}",
+                failure_type=FailureType.config_error,
+            )
+        if len(errors) == 1 and len(tracebacks) == 0:
+            raise AirbyteTracedException(
+                message=f"{errors[0]}",
+                failure_type=FailureType.config_error,
+            )
+        elif len(errors) > 1:
+            raise AirbyteTracedException(
+                internal_message="\n".join(tracebacks),
+                message=f"{len(errors)} streams with errors: {', '.join(error for error in errors)}",
+                failure_type=FailureType.config_error,
+            )
+
+        return not bool(errors), (errors or None)
+
+    def streams(self, config: Mapping[str, Any]) -> List[Stream]:
+        """
+        Return a list of this source's streams.
+        """
+
+        if self.catalog:
+            state_manager = ConnectorStateManager(state=self.state)
+        else:
+            # During `check` operations we don't have a catalog so cannot create a state manager.
+            # Since the state manager is only required for incremental syncs, this is fine.
+            state_manager = None
+
+        try:
+            parsed_config = self._get_parsed_config(config)
+            self.stream_reader.config = parsed_config
+            if self.stream_permissions_reader:
+                self.stream_permissions_reader.config = parsed_config
+            streams: List[Stream] = []
+            for stream_config in parsed_config.streams:
+                # Like state_manager, `catalog_stream` may be None during `check`
+                catalog_stream = self._get_stream_from_catalog(stream_config)
+                stream_state = (
+                    state_manager.get_stream_state(catalog_stream.name, catalog_stream.namespace)
+                    if (state_manager and catalog_stream)
+                    else None
+                )
+                self._validate_input_schema(stream_config)
+
+                sync_mode = self._get_sync_mode_from_catalog(stream_config.name)
+
+                if (
+                    sync_mode == SyncMode.full_refresh
+                    and hasattr(self, "_concurrency_level")
+                    and self._concurrency_level is not None
+                ):
+                    cursor = FileBasedFinalStateCursor(
+                        stream_config=stream_config,
+                        stream_namespace=None,
+                        message_repository=self.message_repository,
+                    )
+                    stream = FileBasedStreamFacade.create_from_stream(
+                        stream=self._make_file_based_stream(
+                            stream_config=stream_config,
+                            cursor=cursor,
+                            parsed_config=parsed_config,
+                        ),
+                        source=self,
+                        logger=self.logger,
+                        state=stream_state,
+                        cursor=cursor,
+                    )
+
+                elif (
+                    sync_mode == SyncMode.incremental
+                    and issubclass(self.cursor_cls, AbstractConcurrentFileBasedCursor)
+                    and hasattr(self, "_concurrency_level")
+                    and self._concurrency_level is not None
+                ):
+                    assert (
+                        state_manager is not None
+                    ), "No ConnectorStateManager was created, but it is required for incremental syncs. This is unexpected. Please contact Support."
+
+                    cursor = self.cursor_cls(
+                        stream_config,
+                        stream_config.name,
+                        None,
+                        stream_state,
+                        self.message_repository,
+                        state_manager,
+                        CursorField(DefaultFileBasedStream.ab_last_mod_col),
+                    )
+                    stream = FileBasedStreamFacade.create_from_stream(
+                        stream=self._make_file_based_stream(
+                            stream_config=stream_config,
+                            cursor=cursor,
+                            parsed_config=parsed_config,
+                        ),
+                        source=self,
+                        logger=self.logger,
+                        state=stream_state,
+                        cursor=cursor,
+                    )
+                else:
+                    cursor = self.cursor_cls(stream_config)
+                    stream = self._make_file_based_stream(
+                        stream_config=stream_config,
+                        cursor=cursor,
+                        parsed_config=parsed_config,
+                    )
+
+                streams.append(stream)
+
+            if include_identities_stream(parsed_config):
+                identities_stream = self._make_identities_stream()
+                streams.append(identities_stream)
+            return streams
+
+        except ValidationError as exc:
+            raise ConfigValidationError(FileBasedSourceError.CONFIG_VALIDATION_ERROR) from exc
+
+    def _make_default_stream(
+        self,
+        stream_config: FileBasedStreamConfig,
+        cursor: Optional[AbstractFileBasedCursor],
+        parsed_config: AbstractFileBasedSpec,
+    ) -> AbstractFileBasedStream:
+        return DefaultFileBasedStream(
+            config=stream_config,
+            catalog_schema=self.stream_schemas.get(stream_config.name),
+            stream_reader=self.stream_reader,
+            availability_strategy=self.availability_strategy,
+            discovery_policy=self.discovery_policy,
+            parsers=self.parsers,
+            validation_policy=self._validate_and_get_validation_policy(stream_config),
+            errors_collector=self.errors_collector,
+            cursor=cursor,
+            use_file_transfer=use_file_transfer(parsed_config),
+            preserve_directory_structure=preserve_directory_structure(parsed_config),
+        )
+
+    def _ensure_permissions_reader_available(self) -> None:
+        """
+        Validates that a stream permissions reader is available.
+        Raises a ValueError if the reader is not provided.
+        """
+        if not self.stream_permissions_reader:
+            raise ValueError(
+                "Stream permissions reader is required for streams that use permissions transfer mode."
+            )
+
+    def _make_permissions_stream(
+        self, stream_config: FileBasedStreamConfig, cursor: Optional[AbstractFileBasedCursor]
+    ) -> AbstractFileBasedStream:
+        """
+        Creates a stream that reads permissions from files.
+        """
+        self._ensure_permissions_reader_available()
+        return PermissionsFileBasedStream(
+            config=stream_config,
+            catalog_schema=self.stream_schemas.get(stream_config.name),
+            stream_reader=self.stream_reader,
+            availability_strategy=self.availability_strategy,
+            discovery_policy=self.discovery_policy,
+            parsers=self.parsers,
+            validation_policy=self._validate_and_get_validation_policy(stream_config),
+            errors_collector=self.errors_collector,
+            cursor=cursor,
+            stream_permissions_reader=self.stream_permissions_reader,  # type: ignore
+        )
+
+    def _make_file_based_stream(
+        self,
+        stream_config: FileBasedStreamConfig,
+        cursor: Optional[AbstractFileBasedCursor],
+        parsed_config: AbstractFileBasedSpec,
+    ) -> AbstractFileBasedStream:
+        """
+        Creates different streams depending on the type of the transfer mode selected
+        """
+        if use_permissions_transfer(parsed_config):
+            return self._make_permissions_stream(stream_config, cursor)
+        # we should have a stream for File transfer mode to decouple from DefaultFileBasedStream
+        else:
+            return self._make_default_stream(stream_config, cursor, parsed_config)
+
+    def _make_identities_stream(
+        self,
+    ) -> Stream:
+        self._ensure_permissions_reader_available()
+        return FileIdentitiesStream(
+            catalog_schema=self.stream_schemas.get(FileIdentitiesStream.IDENTITIES_STREAM_NAME),
+            stream_permissions_reader=self.stream_permissions_reader,  # type: ignore
+            discovery_policy=self.discovery_policy,
+            errors_collector=self.errors_collector,
+        )
+
+    def _get_stream_from_catalog(
+        self, stream_config: FileBasedStreamConfig
+    ) -> Optional[AirbyteStream]:
+        if self.catalog:
+            for stream in self.catalog.streams or []:
+                if stream.stream.name == stream_config.name:
+                    return stream.stream
+        return None
+
+    def _get_sync_mode_from_catalog(self, stream_name: str) -> Optional[SyncMode]:
+        if self.catalog:
+            for catalog_stream in self.catalog.streams:
+                if stream_name == catalog_stream.stream.name:
+                    return catalog_stream.sync_mode
+            self.logger.warning(f"No sync mode was found for {stream_name}.")
+        return None
+
+    def read(
+        self,
+        logger: logging.Logger,
+        config: Mapping[str, Any],
+        catalog: ConfiguredAirbyteCatalog,
+        state: Optional[List[AirbyteStateMessage]] = None,
+    ) -> Iterator[AirbyteMessage]:
+        yield from super().read(logger, config, catalog, state)
+        # emit all the errors collected
+        yield from self.errors_collector.yield_and_raise_collected()
+        # count streams using a certain parser
+        parsed_config = self._get_parsed_config(config)
+        for parser, count in Counter(
+            stream.format.filetype for stream in parsed_config.streams
+        ).items():
+            yield create_analytics_message(f"file-cdk-{parser}-stream-count", count)
+
+    def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification:
+        """
+        Returns the specification describing what fields can be configured by a user when setting up a file-based source.
+        """
+
+        return ConnectorSpecification(
+            documentationUrl=self.spec_class.documentation_url(),
+            connectionSpecification=self.spec_class.schema(),
+        )
+
+    def _get_parsed_config(self, config: Mapping[str, Any]) -> AbstractFileBasedSpec:
+        return self.spec_class(**config)
+
+    def _validate_and_get_validation_policy(
+        self, stream_config: FileBasedStreamConfig
+    ) -> AbstractSchemaValidationPolicy:
+        if stream_config.validation_policy not in self.validation_policies:
+            # This should never happen because we validate the config against the schema's validation_policy enum
+            raise ValidationError(
+                f"`validation_policy` must be one of {list(self.validation_policies.keys())}",
+                model=FileBasedStreamConfig,
+            )
+        return self.validation_policies[stream_config.validation_policy]
+
+    def _validate_input_schema(self, stream_config: FileBasedStreamConfig) -> None:
+        if stream_config.schemaless and stream_config.input_schema:
+            raise ValidationError(
+                "`input_schema` and `schemaless` options cannot both be set",
+                model=FileBasedStreamConfig,
+            )