airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev1__py3-none-any.whl

airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py

@@ -0,0 +1,68 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+import re
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+import unidecode
+
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+
+@dataclass
+class KeysToSnakeCaseTransformation(RecordTransformation):
+    token_pattern: re.Pattern[str] = re.compile(
+        r"[A-Z]+[a-z]*|[a-z]+|\d+|(?P<NoToken>[^a-zA-Z\d]+)"
+    )
+
+    def transform(
+        self,
+        record: Dict[str, Any],
+        config: Optional[Config] = None,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> None:
+        transformed_record = self._transform_record(record)
+        record.clear()
+        record.update(transformed_record)
+
+    def _transform_record(self, record: Dict[str, Any]) -> Dict[str, Any]:
+        transformed_record = {}
+        for key, value in record.items():
+            transformed_key = self.process_key(key)
+            transformed_value = value
+
+            if isinstance(value, dict):
+                transformed_value = self._transform_record(value)
+
+            transformed_record[transformed_key] = transformed_value
+        return transformed_record
+
+    def process_key(self, key: str) -> str:
+        key = self.normalize_key(key)
+        tokens = self.tokenize_key(key)
+        tokens = self.filter_tokens(tokens)
+        return self.tokens_to_snake_case(tokens)
+
+    def normalize_key(self, key: str) -> str:
+        return unidecode.unidecode(key)
+
+    def tokenize_key(self, key: str) -> List[str]:
+        tokens = []
+        for match in self.token_pattern.finditer(key):
+            token = match.group(0) if match.group("NoToken") is None else ""
+            tokens.append(token)
+        return tokens
+
+    def filter_tokens(self, tokens: List[str]) -> List[str]:
+        if len(tokens) >= 3:
+            tokens = tokens[:1] + [t for t in tokens[1:-1] if t] + tokens[-1:]
+        if tokens and tokens[0].isdigit():
+            tokens.insert(0, "")
+        return tokens
+
+    def tokens_to_snake_case(self, tokens: List[str]) -> str:
+        return "_".join(token.lower() for token in tokens)

airbyte_cdk/sources/declarative/transformations/remove_fields.py

@@ -3,13 +3,14 @@
 #
 
 from dataclasses import InitVar, dataclass
-from typing import Any, List, Mapping, Optional
+from typing import Any, Dict, List, Mapping, Optional
 
+import dpath
 import dpath.exceptions
-import dpath.util
+
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
-from airbyte_cdk.sources.declarative.types import Config, FieldPointer, StreamSlice, StreamState
+from airbyte_cdk.sources.types import Config, FieldPointer, StreamSlice, StreamState
 
 
 @dataclass
@@ -44,15 +45,17 @@ class RemoveFields(RecordTransformation):
     condition: str = ""
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
-        self._filter_interpolator = InterpolatedBoolean(condition=self.condition, parameters=parameters)
+        self._filter_interpolator = InterpolatedBoolean(
+            condition=self.condition, parameters=parameters
+        )
 
     def transform(
         self,
-        record: Mapping[str, Any],
+        record: Dict[str, Any],
         config: Optional[Config] = None,
         stream_state: Optional[StreamState] = None,
         stream_slice: Optional[StreamSlice] = None,
-    ) -> Mapping[str, Any]:
+    ) -> None:
         """
         :param record: The record to be transformed
         :return: the input record with the requested fields removed
@@ -60,13 +63,13 @@ class RemoveFields(RecordTransformation):
         for pointer in self.field_pointers:
             # the dpath library by default doesn't delete fields from arrays
             try:
-                dpath.util.delete(
+                dpath.delete(
                     record,
                     pointer,
-                    afilter=(lambda x: self._filter_interpolator.eval(config or {}, property=x)) if self.condition else None,
+                    afilter=(lambda x: self._filter_interpolator.eval(config or {}, property=x))
+                    if self.condition
+                    else None,
                 )
             except dpath.exceptions.PathNotFound:
                 # if the (potentially nested) property does not exist, silently skip
                 pass
-
-        return record

airbyte_cdk/sources/declarative/transformations/transformation.py

@@ -4,9 +4,9 @@
 
 from abc import abstractmethod
 from dataclasses import dataclass
-from typing import Any, Mapping, Optional
+from typing import Any, Dict, Optional
 
-from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
 
 
 @dataclass
@@ -18,13 +18,13 @@ class RecordTransformation:
     @abstractmethod
     def transform(
         self,
-        record: Record,
+        record: Dict[str, Any],
         config: Optional[Config] = None,
         stream_state: Optional[StreamState] = None,
         stream_slice: Optional[StreamSlice] = None,
-    ) -> Mapping[str, Any]:
+    ) -> None:
         """
-        Transform a record by adding, deleting, or mutating fields.
+        Transform a record by adding, deleting, or mutating fields directly from the record reference passed in argument.
 
         :param record: The input record to be transformed
         :param config: The user-provided configuration as specified by the source's spec

airbyte_cdk/sources/declarative/types.py

@@ -4,113 +4,22 @@
 
 from __future__ import annotations
 
-from typing import Any, ItemsView, Iterator, KeysView, List, Mapping, Optional, ValuesView
-
-# A FieldPointer designates a path to a field inside a mapping. For example, retrieving ["k1", "k1.2"] in the object {"k1" :{"k1.2":
-# "hello"}] returns "hello"
-FieldPointer = List[str]
-Config = Mapping[str, Any]
-ConnectionDefinition = Mapping[str, Any]
-StreamState = Mapping[str, Any]
-
-
-class Record(Mapping[str, Any]):
-    def __init__(self, data: Mapping[str, Any], associated_slice: Optional[StreamSlice]):
-        self._data = data
-        self._associated_slice = associated_slice
-
-    @property
-    def data(self) -> Mapping[str, Any]:
-        return self._data
-
-    @property
-    def associated_slice(self) -> Optional[StreamSlice]:
-        return self._associated_slice
-
-    def __repr__(self) -> str:
-        return repr(self._data)
-
-    def __getitem__(self, key: str) -> Any:
-        return self._data[key]
-
-    def __len__(self) -> int:
-        return len(self._data)
-
-    def __iter__(self) -> Any:
-        return iter(self._data)
-
-    def __contains__(self, item: object) -> bool:
-        return item in self._data
-
-    def __eq__(self, other: object) -> bool:
-        if isinstance(other, Record):
-            # noinspection PyProtectedMember
-            return self._data == other._data
-        return False
-
-    def __ne__(self, other: object) -> bool:
-        return not self.__eq__(other)
-
-
-class StreamSlice(Mapping[str, Any]):
-    def __init__(self, *, partition: Mapping[str, Any], cursor_slice: Mapping[str, Any]) -> None:
-        self._partition = partition
-        self._cursor_slice = cursor_slice
-        if partition.keys() & cursor_slice.keys():
-            raise ValueError("Keys for partition and incremental sync cursor should not overlap")
-        self._stream_slice = dict(partition) | dict(cursor_slice)
-
-    @property
-    def partition(self) -> Mapping[str, Any]:
-        p = self._partition
-        while isinstance(p, StreamSlice):
-            p = p.partition
-        return p
-
-    @property
-    def cursor_slice(self) -> Mapping[str, Any]:
-        c = self._cursor_slice
-        while isinstance(c, StreamSlice):
-            c = c.cursor_slice
-        return c
-
-    def __repr__(self) -> str:
-        return repr(self._stream_slice)
-
-    def __setitem__(self, key: str, value: Any) -> None:
-        raise ValueError("StreamSlice is immutable")
-
-    def __getitem__(self, key: str) -> Any:
-        return self._stream_slice[key]
-
-    def __len__(self) -> int:
-        return len(self._stream_slice)
-
-    def __iter__(self) -> Iterator[str]:
-        return iter(self._stream_slice)
-
-    def __contains__(self, item: Any) -> bool:
-        return item in self._stream_slice
-
-    def keys(self) -> KeysView[str]:
-        return self._stream_slice.keys()
-
-    def items(self) -> ItemsView[str, Any]:
-        return self._stream_slice.items()
-
-    def values(self) -> ValuesView[Any]:
-        return self._stream_slice.values()
-
-    def get(self, key: str, default: Any = None) -> Optional[Any]:
-        return self._stream_slice.get(key, default)
-
-    def __eq__(self, other: Any) -> bool:
-        if isinstance(other, dict):
-            return self._stream_slice == other
-        if isinstance(other, StreamSlice):
-            # noinspection PyProtectedMember
-            return self._partition == other._partition and self._cursor_slice == other._cursor_slice
-        return False
-
-    def __ne__(self, other: Any) -> bool:
-        return not self.__eq__(other)
+from airbyte_cdk.sources.types import (
+    Config,
+    ConnectionDefinition,
+    FieldPointer,
+    Record,
+    StreamSlice,
+    StreamState,
+)
+
+# Note: This package originally contained class definitions for low-code CDK types, but we promoted them into the Python CDK.
+# We've migrated connectors in the repository to reference the new location, but these assignments are used to retain backwards
+# compatibility for sources created by OSS customers or on forks. This can be removed when we start bumping major versions.
+
+FieldPointer = FieldPointer
+Config = Config
+ConnectionDefinition = ConnectionDefinition
+StreamState = StreamState
+Record = Record
+StreamSlice = StreamSlice

airbyte_cdk/sources/declarative/yaml_declarative_source.py

@@ -3,31 +3,52 @@
 #
 
 import pkgutil
+from typing import Any, List, Mapping, Optional
 
 import yaml
-from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
-from airbyte_cdk.sources.declarative.types import ConnectionDefinition
 
+from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog
+from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
+    ConcurrentDeclarativeSource,
+)
+from airbyte_cdk.sources.types import ConnectionDefinition
 
-class YamlDeclarativeSource(ManifestDeclarativeSource):
+
+class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage]]):
     """Declarative source defined by a yaml file"""
 
-    def __init__(self, path_to_yaml, debug: bool = False):
+    def __init__(
+        self,
+        path_to_yaml: str,
+        debug: bool = False,
+        catalog: Optional[ConfiguredAirbyteCatalog] = None,
+        config: Optional[Mapping[str, Any]] = None,
+        state: Optional[List[AirbyteStateMessage]] = None,
+    ) -> None:
         """
         :param path_to_yaml: Path to the yaml file describing the source
         """
         self._path_to_yaml = path_to_yaml
         source_config = self._read_and_parse_yaml_file(path_to_yaml)
-        super().__init__(source_config, debug)
 
-    def _read_and_parse_yaml_file(self, path_to_yaml_file) -> ConnectionDefinition:
+        super().__init__(
+            catalog=catalog or ConfiguredAirbyteCatalog(streams=[]),
+            config=config or {},
+            state=state or [],
+            source_config=source_config,
+        )
+
+    def _read_and_parse_yaml_file(self, path_to_yaml_file: str) -> ConnectionDefinition:
         package = self.__class__.__module__.split(".")[0]
 
         yaml_config = pkgutil.get_data(package, path_to_yaml_file)
-        decoded_yaml = yaml_config.decode()
-        return self._parse(decoded_yaml)
+        if yaml_config:
+            decoded_yaml = yaml_config.decode()
+            return self._parse(decoded_yaml)
+        else:
+            return {}
 
-    def _emit_manifest_debug_message(self, extra_args: dict):
+    def _emit_manifest_debug_message(self, extra_args: dict[str, Any]) -> None:
         extra_args["path_to_yaml"] = self._path_to_yaml
         self.logger.debug("declarative source created from parsed YAML manifest", extra=extra_args)
 
@@ -39,4 +60,4 @@ class YamlDeclarativeSource(ManifestDeclarativeSource):
         :param connection_definition_str: yaml string to parse
         :return: The ConnectionDefinition parsed from connection_definition_str
         """
-        return yaml.safe_load(connection_definition_str)
+        return yaml.safe_load(connection_definition_str)  # type: ignore # yaml.safe_load doesn't return a type but know it is a Mapping

airbyte_cdk/sources/embedded/base_integration.py

@@ -6,11 +6,15 @@ from abc import ABC, abstractmethod
 from typing import Generic, Iterable, Optional, TypeVar
 
 from airbyte_cdk.connector import TConfig
-from airbyte_cdk.sources.embedded.catalog import create_configured_catalog, get_stream, get_stream_names
+from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStateMessage, SyncMode, Type
+from airbyte_cdk.sources.embedded.catalog import (
+    create_configured_catalog,
+    get_stream,
+    get_stream_names,
+)
 from airbyte_cdk.sources.embedded.runner import SourceRunner
 from airbyte_cdk.sources.embedded.tools import get_defined_id
 from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
-from airbyte_protocol.models import AirbyteRecordMessage, AirbyteStateMessage, SyncMode, Type
 
 TOutput = TypeVar("TOutput")
 
@@ -31,11 +35,15 @@ class BaseEmbeddedIntegration(ABC, Generic[TConfig, TOutput]):
         """
         pass
 
-    def _load_data(self, stream_name: str, state: Optional[AirbyteStateMessage] = None) -> Iterable[TOutput]:
+    def _load_data(
+        self, stream_name: str, state: Optional[AirbyteStateMessage] = None
+    ) -> Iterable[TOutput]:
         catalog = self.source.discover(self.config)
         stream = get_stream(catalog, stream_name)
         if not stream:
-            raise ValueError(f"Stream {stream_name} not found, the following streams are available: {', '.join(get_stream_names(catalog))}")
+            raise ValueError(
+                f"Stream {stream_name} not found, the following streams are available: {', '.join(get_stream_names(catalog))}"
+            )
         if SyncMode.incremental not in stream.supported_sync_modes:
             configured_catalog = create_configured_catalog(stream, sync_mode=SyncMode.full_refresh)
         else:
@@ -43,7 +51,10 @@ class BaseEmbeddedIntegration(ABC, Generic[TConfig, TOutput]):
 
         for message in self.source.read(self.config, configured_catalog, state):
             if message.type == Type.RECORD:
-                output = self._handle_record(message.record, get_defined_id(stream, message.record.data))
+                output = self._handle_record(
+                    message.record,
+                    get_defined_id(stream, message.record.data),  # type: ignore[union-attr, arg-type]
+                )
                 if output:
                     yield output
             elif message.type is Type.STATE and message.state:

airbyte_cdk/sources/embedded/catalog.py

@@ -31,15 +31,27 @@ def to_configured_stream(
     primary_key: Optional[List[List[str]]] = None,
 ) -> ConfiguredAirbyteStream:
     return ConfiguredAirbyteStream(
-        stream=stream, sync_mode=sync_mode, destination_sync_mode=destination_sync_mode, cursor_field=cursor_field, primary_key=primary_key
+        stream=stream,
+        sync_mode=sync_mode,
+        destination_sync_mode=destination_sync_mode,
+        cursor_field=cursor_field,
+        primary_key=primary_key,
     )
 
 
-def to_configured_catalog(configured_streams: List[ConfiguredAirbyteStream]) -> ConfiguredAirbyteCatalog:
+def to_configured_catalog(
+    configured_streams: List[ConfiguredAirbyteStream],
+) -> ConfiguredAirbyteCatalog:
     return ConfiguredAirbyteCatalog(streams=configured_streams)
 
 
-def create_configured_catalog(stream: AirbyteStream, sync_mode: SyncMode = SyncMode.full_refresh) -> ConfiguredAirbyteCatalog:
-    configured_streams = [to_configured_stream(stream, sync_mode=sync_mode, primary_key=stream.source_defined_primary_key)]
+def create_configured_catalog(
+    stream: AirbyteStream, sync_mode: SyncMode = SyncMode.full_refresh
+) -> ConfiguredAirbyteCatalog:
+    configured_streams = [
+        to_configured_stream(
+            stream, sync_mode=sync_mode, primary_key=stream.source_defined_primary_key
+        )
+    ]
 
     return to_configured_catalog(configured_streams)

airbyte_cdk/sources/embedded/runner.py

@@ -8,7 +8,13 @@ from abc import ABC, abstractmethod
 from typing import Generic, Iterable, Optional
 
 from airbyte_cdk.connector import TConfig
-from airbyte_cdk.models import AirbyteCatalog, AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog, ConnectorSpecification
+from airbyte_cdk.models import (
+    AirbyteCatalog,
+    AirbyteMessage,
+    AirbyteStateMessage,
+    ConfiguredAirbyteCatalog,
+    ConnectorSpecification,
+)
 from airbyte_cdk.sources.source import Source
 
 
@@ -22,7 +28,12 @@ class SourceRunner(ABC, Generic[TConfig]):
         pass
 
     @abstractmethod
-    def read(self, config: TConfig, catalog: ConfiguredAirbyteCatalog, state: Optional[AirbyteStateMessage]) -> Iterable[AirbyteMessage]:
+    def read(
+        self,
+        config: TConfig,
+        catalog: ConfiguredAirbyteCatalog,
+        state: Optional[AirbyteStateMessage],
+    ) -> Iterable[AirbyteMessage]:
         pass
 
 
@@ -37,5 +48,10 @@ class CDKRunner(SourceRunner[TConfig]):
     def discover(self, config: TConfig) -> AirbyteCatalog:
         return self._source.discover(self._logger, config)
 
-    def read(self, config: TConfig, catalog: ConfiguredAirbyteCatalog, state: Optional[AirbyteStateMessage]) -> Iterable[AirbyteMessage]:
+    def read(
+        self,
+        config: TConfig,
+        catalog: ConfiguredAirbyteCatalog,
+        state: Optional[AirbyteStateMessage],
+    ) -> Iterable[AirbyteMessage]:
         return self._source.read(self._logger, config, catalog, state=[state] if state else [])

airbyte_cdk/sources/embedded/tools.py

@@ -5,10 +5,13 @@
 from typing import Any, Callable, Dict, Iterable, Optional
 
 import dpath
+
 from airbyte_cdk.models import AirbyteStream
 
 
-def get_first(iterable: Iterable[Any], predicate: Callable[[Any], bool] = lambda m: True) -> Optional[Any]:
+def get_first(
+    iterable: Iterable[Any], predicate: Callable[[Any], bool] = lambda m: True
+) -> Optional[Any]:
     return next(filter(predicate, iterable), None)
 
 
@@ -18,7 +21,7 @@ def get_defined_id(stream: AirbyteStream, data: Dict[str, Any]) -> Optional[str]
     primary_key = []
     for key in stream.source_defined_primary_key:
         try:
-            primary_key.append(str(dpath.util.get(data, key)))
+            primary_key.append(str(dpath.get(data, key)))
         except KeyError:
             primary_key.append("__not_found__")
     return "_".join(primary_key)

airbyte_cdk/sources/file_based/README.md

@@ -0,0 +1,152 @@
+## Behavior
+
+The Airbyte protocol defines the actions `spec`, `discover`, `check` and `read` for a source to be compliant. Here is the high-level description of the flow for a file-based source:
+
+- spec: calls AbstractFileBasedSpec.documentation_url and AbstractFileBasedSpec.schema to return a ConnectorSpecification.
+- discover: calls Source.streams, and subsequently Stream.get_json_schema; this uses Source.open_file to open files during schema discovery.
+- check: Source.check_connection is called from the entrypoint code (in the main CDK).
+- read: Stream.read_records calls Stream.list_files which calls Source.list_matching_files, and then also uses Source.open_file to parse records from the file handle.
+
+## How to Implement Your Own
+
+To create a file-based source a user must extend three classes – AbstractFileBasedSource, AbstractFileBasedSpec, and AbstractStreamReader – to create an implementation for the connector’s specific storage system. They then initialize a FileBasedSource with the instance of AbstractStreamReader specific to their storage system.
+
+The abstract classes house the vast majority of the logic required by file-based sources. For example, when extending AbstractStreamReader, users only have to implement three methods:
+
+- list_matching_files: lists files matching the glob pattern(s) provided in the config.
+- open_file: returns a file handle for reading.
+- config property setter: concrete implementations of AbstractFileBasedStreamReader's config setter should assert that `value` is of the correct config type for that type of StreamReader.
+
+The result is that an implementation of a source might look like this:
+
+```
+class CustomStreamReader(AbstractStreamReader):
+    def open_file(self, remote_file: RemoteFile) -> FileHandler:
+        <...>
+
+    def get_matching_files(
+        self,
+        globs: List[str],
+        logger: logging.Logger,
+    ) -> Iterable[RemoteFile]:
+        <...>
+
+    @config.setter
+    def config(self, value: Config):
+        assert isinstance(value, CustomConfig)
+        self._config = value
+
+
+class CustomConfig(AbstractFileBasedSpec):
+    @classmethod
+    def documentation_url(cls) -> AnyUrl:
+        return AnyUrl("https://docs.airbyte.com/integrations/sources/s3", scheme="https")
+
+    a_spec_field: str = Field(title="A Spec Field", description="This is where you describe the fields of the spec", order=0)
+    <...>
+```
+
+For more information, feel free to check the docstrings of each classes or check specific implementations (like source-s3).
+
+## Supported File Types
+
+### Avro
+
+Avro is a serialization format developed by [Apache](https://avro.apache.org/docs/). Avro configuration options for the file-based CDK:
+
+- `double_as_string`: Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.
+
+### CSV
+
+CSV is a format loosely described by [RFC 4180](https://www.rfc-editor.org/rfc/rfc4180). The format is quite flexible which leads to a ton of options to consider:
+
+- `delimiter`: The character delimiting individual cells in the CSV data. By name, CSV is comma separated so the default value is `,`
+- `quote_char`: When quoted fields are used, it is possible for a field to span multiple lines, even when line breaks appear within such field. The default quote character is `"`.
+- `escape_char`: The character used for escaping special characters.
+- `encoding`: The character encoding of the file. By default, `UTF-8`
+- `double_quote`: Whether two quotes in a quoted CSV value denote a single quote in the data.
+- `quoting_behavior`: The quoting behavior determines when a value in a row should have quote marks added around it.
+- `skip_rows_before_header`: The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.
+- `skip_rows_after_header`: The number of rows to skip after the header row.
+- `autogenerate_column_names`: If your CSV does not have a header row, the file-based CDK will need this enable to generate column names.
+- `null_values`: As CSV does not explicitly define a value for null values, the user can specify a set of case-sensitive strings that should be interpreted as null values.
+- `true_values`: As CSV does not explicitly define a value for positive boolean, the user can specify a set of case-sensitive strings that should be interpreted as true values.
+- `false_values`: As CSV does not explicitly define a value for negative boolean, the user can specify a set of case-sensitive strings that should be interpreted as false values.
+
+### JSONL
+
+[JSONL](https://jsonlines.org/) (or JSON Lines) is a format where each row is a JSON object. There are no configuration option for this format. For backward compatibility reasons, the JSONL parser currently supports multiline objects even though this is not part of the JSONL standard. Following some data gathering, we reserve the right to remove the support for this. Given that files have multiline JSON objects, performances will be slow.
+
+### Parquet
+
+Parquet is a file format defined by [Apache](https://parquet.apache.org/). Configuration options are:
+
+- `decimal_as_float`: Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended.
+
+### Document file types (PDF, DOCX, Markdown)
+
+For file share source connectors, the `unstructured` parser can be used to parse document file types. The textual content of the whole file will be parsed as a single record with a `content` field containing the text encoded as markdown.
+
+To use the unstructured parser, the libraries `poppler` and `tesseract` need to be installed on the system running the connector. For example, on Ubuntu, you can install them with the following command:
+
+```
+apt-get install -y tesseract-ocr poppler-utils
+```
+
+on Mac, you can install these via brew:
+
+```
+brew install poppler
+brew install tesseract
+```
+
+## Schema
+
+Having a schema allows for the file-based CDK to take action when there is a discrepancy between a record and what are the expected types of the record fields.
+
+Schema can be either inferred or user provided.
+
+- If the user defines it a format using JSON types, inference will not apply. Input schemas are a key/value pair of strings describing column name and data type. Supported types are `["string", "number", "integer", "object", "array", "boolean", "null"]`. For example, `{"col1": "string", "col2": "boolean"}`.
+- If the user enables schemaless sync, schema will `{"data": "object"}` and therefore emitted records will look like `{"data": {"col1": val1, …}}`. This is recommended if the contents between files in the stream vary significantly, and/or if data is very nested.
+- Else, the file-based CDK will infer the schema depending on the file type. Some file formats defined the schema as part of their metadata (like Parquet), some do on the record-level (like Avro) and some don't have any explicit typing (like JSON or CSV). Note that all CSV values are inferred as strings except where we are supporting legacy configurations. Any file format that does not define their schema on a metadata level will require the file-based CDK to iterate to a number of records. There is a limit of bytes that will be consumed in order to infer the schema.
+
+### Validation Policies
+
+Users will be required to select one of 3 different options, in the event that records are encountered that don’t conform to the schema.
+
+- Skip nonconforming records: check each record to see if it conforms to the user-input or inferred schema; skip the record if it doesn't conform. We keep a count of the number of records in each file that do and do not conform and emit a log message with these counts once we’re done reading the file.
+- Emit all records: emit all records, even if they do not conform to the user-provided or inferred schema. Columns that don't exist in the configured catalog probably won't be available in the destination's table since that's the current behavior.
+  Only error if there are conflicting field types or malformed rows.
+- Stop the sync and wait for schema re-discovery: if a record is encountered that does not conform to the configured catalog’s schema, we log a message and stop the whole sync. Note: this option is not recommended if the files have very different columns or datatypes, because the inferred schema may vary significantly at discover time.
+
+When the `schemaless` is enabled, validation will be skipped.
+
+## Breaking Changes (compared to previous S3 implementation)
+
+- [CSV] Mapping of type `array` and `object`: before, they were mapped as `large_string` and hence casted as strings. Given the new changes, if `array` or `object` is specified, the value will be casted as `array` and `object` respectively.
+- [CSV] Before, a string value would not be considered as `null_values` if the column type was a string. We will now start to cast string columns with values matching `null_values` to null.
+- [CSV] `decimal_point` option is deprecated: It is not possible anymore to use another character than `.` to separate the integer part from non-integer part. Given that the float is format with another character than this, it will be considered as a string.
+- [Parquet] `columns` option is deprecated: You can use Airbyte column selection in order to have the same behavior. We don't expect it, but this could have impact on the performance as payload could be bigger.
+
+## Incremental syncs
+
+The file-based connectors supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
+
+| Feature                                        | Supported? |
+| :--------------------------------------------- | :--------- |
+| Full Refresh Sync                              | Yes        |
+| Incremental Sync                               | Yes        |
+| Replicate Incremental Deletes                  | No         |
+| Replicate Multiple Files \(pattern matching\)  | Yes        |
+| Replicate Multiple Streams \(distinct tables\) | Yes        |
+| Namespaces                                     | No         |
+
+We recommend you do not manually modify files that are already synced. The connector has file-level granularity, which means adding or modifying a row in a CSV file will trigger a re-sync of the content of that file.
+
+### Incremental sync
+
+After the initial sync, the connector only pulls files that were modified since the last sync.
+
+The connector checkpoints the connection states when it is done syncing all files for a given timestamp. The connection's state only keeps track of the last 10 000 files synced. If more than 10 000 files are synced, the connector won't be able to rely on the connection state to deduplicate files. In this case, the connector will initialize its cursor to the minimum between the earliest file in the history, or 3 days ago.
+
+Both the maximum number of files, and the time buffer can be configured by connector developers.