airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev1__py3-none-any.whl

airbyte_cdk/utils/schema_inferrer.py

@@ -3,18 +3,31 @@
 #
 
 from collections import defaultdict
-from typing import Any, Dict, Mapping, Optional
+from typing import Any, Dict, List, Mapping, Optional
 
-from airbyte_cdk.models import AirbyteRecordMessage
 from genson import SchemaBuilder, SchemaNode
 from genson.schema.strategies.object import Object
 from genson.schema.strategies.scalar import Number
 
+from airbyte_cdk.models import AirbyteRecordMessage
+
+# schema keywords
+_TYPE = "type"
+_NULL_TYPE = "null"
+_OBJECT_TYPE = "object"
+_ANY_OF = "anyOf"
+_ITEMS = "items"
+_PROPERTIES = "properties"
+_REQUIRED = "required"
+
 
 class NoRequiredObj(Object):
     """
     This class has Object behaviour, but it does not generate "required[]" fields
-    every time it parses object. So we dont add unnecessary extra field.
+    every time it parses object. So we don't add unnecessary extra field.
+
+    The logic is that even reading all the data from a source, it does not mean that there can be another record added with those fields as
+    optional. Hence, we make everything nullable.
     """
 
     def to_schema(self) -> Mapping[str, Any]:
@@ -41,6 +54,30 @@ class NoRequiredSchemaBuilder(SchemaBuilder):
 InferredSchema = Dict[str, Any]
 
 
+class SchemaValidationException(Exception):
+    @classmethod
+    def merge_exceptions(
+        cls, exceptions: List["SchemaValidationException"]
+    ) -> "SchemaValidationException":
+        # We assume the schema is the same for all SchemaValidationException
+        return SchemaValidationException(
+            exceptions[0].schema,
+            [x for exception in exceptions for x in exception._validation_errors],
+        )
+
+    def __init__(self, schema: InferredSchema, validation_errors: List[Exception]):
+        self._schema = schema
+        self._validation_errors = validation_errors
+
+    @property
+    def schema(self) -> InferredSchema:
+        return self._schema
+
+    @property
+    def validation_errors(self) -> List[str]:
+        return list(map(lambda error: str(error), self._validation_errors))
+
+
 class SchemaInferrer:
     """
     This class is used to infer a JSON schema which fits all the records passed into it
@@ -53,22 +90,54 @@ class SchemaInferrer:
 
     stream_to_builder: Dict[str, SchemaBuilder]
 
-    def __init__(self) -> None:
+    def __init__(
+        self, pk: Optional[List[List[str]]] = None, cursor_field: Optional[List[List[str]]] = None
+    ) -> None:
         self.stream_to_builder = defaultdict(NoRequiredSchemaBuilder)
+        self._pk = [] if pk is None else pk
+        self._cursor_field = [] if cursor_field is None else cursor_field
 
     def accumulate(self, record: AirbyteRecordMessage) -> None:
         """Uses the input record to add to the inferred schemas maintained by this object"""
         self.stream_to_builder[record.stream].add_object(record.data)
 
-    def get_inferred_schemas(self) -> Dict[str, InferredSchema]:
-        """
-        Returns the JSON schemas for all encountered streams inferred by inspecting all records
-        passed via the accumulate method
-        """
-        schemas = {}
-        for stream_name, builder in self.stream_to_builder.items():
-            schemas[stream_name] = self._clean(builder.to_schema())
-        return schemas
+    def _null_type_in_any_of(self, node: InferredSchema) -> bool:
+        if _ANY_OF in node:
+            return {_TYPE: _NULL_TYPE} in node[_ANY_OF]
+        else:
+            return False
+
+    def _remove_type_from_any_of(self, node: InferredSchema) -> None:
+        if _ANY_OF in node:
+            node.pop(_TYPE, None)
+
+    def _clean_any_of(self, node: InferredSchema) -> None:
+        if len(node[_ANY_OF]) == 2 and self._null_type_in_any_of(node):
+            real_type = (
+                node[_ANY_OF][1] if node[_ANY_OF][0][_TYPE] == _NULL_TYPE else node[_ANY_OF][0]
+            )
+            node.update(real_type)
+            node[_TYPE] = [node[_TYPE], _NULL_TYPE]
+            node.pop(_ANY_OF)
+        # populate `type` for `anyOf` if it's not present to pass all other checks
+        elif len(node[_ANY_OF]) == 2 and not self._null_type_in_any_of(node):
+            node[_TYPE] = [_NULL_TYPE]
+
+    def _clean_properties(self, node: InferredSchema) -> None:
+        for key, value in list(node[_PROPERTIES].items()):
+            if isinstance(value, dict) and value.get(_TYPE, None) == _NULL_TYPE:
+                node[_PROPERTIES].pop(key)
+            else:
+                self._clean(value)
+
+    def _ensure_null_type_on_top(self, node: InferredSchema) -> None:
+        if isinstance(node[_TYPE], list):
+            if _NULL_TYPE in node[_TYPE]:
+                # we want to make sure null is always at the end as it makes schemas more readable
+                node[_TYPE].remove(_NULL_TYPE)
+            node[_TYPE].append(_NULL_TYPE)
+        else:
+            node[_TYPE] = [node[_TYPE], _NULL_TYPE]
 
     def _clean(self, node: InferredSchema) -> InferredSchema:
         """
@@ -76,25 +145,126 @@ class SchemaInferrer:
         - remove anyOf if one of them is just a null value
         - remove properties of type "null"
         """
+
         if isinstance(node, dict):
-            if "anyOf" in node:
-                if len(node["anyOf"]) == 2 and {"type": "null"} in node["anyOf"]:
-                    real_type = node["anyOf"][1] if node["anyOf"][0]["type"] == "null" else node["anyOf"][0]
-                    node.update(real_type)
-                    node["type"] = [node["type"], "null"]
-                    node.pop("anyOf")
-            if "properties" in node and isinstance(node["properties"], dict):
-                for key, value in list(node["properties"].items()):
-                    if isinstance(value, dict) and value.get("type", None) == "null":
-                        node["properties"].pop(key)
-                    else:
-                        self._clean(value)
-            if "items" in node:
-                self._clean(node["items"])
+            if _ANY_OF in node:
+                self._clean_any_of(node)
+
+            if _PROPERTIES in node and isinstance(node[_PROPERTIES], dict):
+                self._clean_properties(node)
+
+            if _ITEMS in node:
+                self._clean(node[_ITEMS])
+
+            # this check needs to follow the "anyOf" cleaning as it might populate `type`
+            self._ensure_null_type_on_top(node)
+
+        # remove added `type: ["null"]` for `anyOf` nested node
+        self._remove_type_from_any_of(node)
+
+        return node
+
+    def _add_required_properties(self, node: InferredSchema) -> InferredSchema:
+        """
+        This method takes properties that should be marked as required (self._pk and self._cursor_field) and travel the schema to mark every
+        node as required.
+        """
+        # Removing nullable for the root as when we call `_clean`, we make everything nullable
+        node[_TYPE] = _OBJECT_TYPE
+
+        exceptions = []
+        for field in [x for x in [self._pk, self._cursor_field] if x]:
+            try:
+                self._add_fields_as_required(node, field)
+            except SchemaValidationException as exception:
+                exceptions.append(exception)
+
+        if exceptions:
+            raise SchemaValidationException.merge_exceptions(exceptions)
+
         return node
 
+    def _add_fields_as_required(self, node: InferredSchema, composite_key: List[List[str]]) -> None:
+        """
+        Take a list of nested keys (this list represents a composite key) and travel the schema to mark every node as required.
+        """
+        errors: List[Exception] = []
+
+        for path in composite_key:
+            try:
+                self._add_field_as_required(node, path)
+            except ValueError as exception:
+                errors.append(exception)
+
+        if errors:
+            raise SchemaValidationException(node, errors)
+
+    def _add_field_as_required(
+        self, node: InferredSchema, path: List[str], traveled_path: Optional[List[str]] = None
+    ) -> None:
+        """
+        Take a nested key and travel the schema to mark every node as required.
+        """
+        self._remove_null_from_type(node)
+        if self._is_leaf(path):
+            return
+
+        if not traveled_path:
+            traveled_path = []
+
+        if _PROPERTIES not in node:
+            # This validation is only relevant when `traveled_path` is empty
+            raise ValueError(
+                f"Path {traveled_path} does not refer to an object but is `{node}` and hence {path} can't be marked as required."
+            )
+
+        next_node = path[0]
+        if next_node not in node[_PROPERTIES]:
+            raise ValueError(
+                f"Path {traveled_path} does not have field `{next_node}` in the schema and hence can't be marked as required."
+            )
+
+        if _TYPE not in node:
+            # We do not expect this case to happen but we added a specific error message just in case
+            raise ValueError(
+                f"Unknown schema error: {traveled_path} is expected to have a type but did not. Schema inferrence is probably broken"
+            )
+
+        if node[_TYPE] not in [
+            _OBJECT_TYPE,
+            [_NULL_TYPE, _OBJECT_TYPE],
+            [_OBJECT_TYPE, _NULL_TYPE],
+        ]:
+            raise ValueError(
+                f"Path {traveled_path} is expected to be an object but was of type `{node['properties'][next_node]['type']}`"
+            )
+
+        if _REQUIRED not in node or not node[_REQUIRED]:
+            node[_REQUIRED] = [next_node]
+        elif next_node not in node[_REQUIRED]:
+            node[_REQUIRED].append(next_node)
+
+        traveled_path.append(next_node)
+        self._add_field_as_required(node[_PROPERTIES][next_node], path[1:], traveled_path)
+
+    def _is_leaf(self, path: List[str]) -> bool:
+        return len(path) == 0
+
+    def _remove_null_from_type(self, node: InferredSchema) -> None:
+        if isinstance(node[_TYPE], list):
+            if _NULL_TYPE in node[_TYPE]:
+                node[_TYPE].remove(_NULL_TYPE)
+            if len(node[_TYPE]) == 1:
+                node[_TYPE] = node[_TYPE][0]
+
     def get_stream_schema(self, stream_name: str) -> Optional[InferredSchema]:
         """
         Returns the inferred JSON schema for the specified stream. Might be `None` if there were no records for the given stream name.
         """
-        return self._clean(self.stream_to_builder[stream_name].to_schema()) if stream_name in self.stream_to_builder else None
+        return (
+            self._add_required_properties(
+                self._clean(self.stream_to_builder[stream_name].to_schema())
+            )
+            if stream_name in self.stream_to_builder
+            else None
+        )

airbyte_cdk/utils/slice_hasher.py

@@ -0,0 +1,30 @@
+import hashlib
+import json
+from typing import Any, Final, Mapping, Optional
+
+
+class SliceEncoder(json.JSONEncoder):
+    def default(self, obj: Any) -> Any:
+        if hasattr(obj, "__json_serializable__"):
+            return obj.__json_serializable__()
+
+        # Let the base class default method raise the TypeError
+        return super().default(obj)
+
+
+class SliceHasher:
+    _ENCODING: Final = "utf-8"
+
+    @classmethod
+    def hash(cls, stream_name: str, stream_slice: Optional[Mapping[str, Any]] = None) -> int:
+        if stream_slice:
+            try:
+                s = json.dumps(stream_slice, sort_keys=True, cls=SliceEncoder)
+                hash_input = f"{stream_name}:{s}".encode(cls._ENCODING)
+            except TypeError as e:
+                raise ValueError(f"Failed to serialize stream slice: {e}")
+        else:
+            hash_input = stream_name.encode(cls._ENCODING)
+
+        # Use last 8 bytes as 64-bit integer for better distribution
+        return int.from_bytes(hashlib.sha256(hash_input).digest()[-8:], "big")

airbyte_cdk/utils/spec_schema_transformations.py

@@ -4,11 +4,12 @@
 
 import json
 import re
+from typing import Any
 
 from jsonschema import RefResolver
 
 
-def resolve_refs(schema: dict) -> dict:
+def resolve_refs(schema: dict[str, Any]) -> dict[str, Any]:
     """
     For spec schemas generated using Pydantic models, the resulting JSON schema can contain refs between object
     relationships.
@@ -17,7 +18,9 @@ def resolve_refs(schema: dict) -> dict:
     str_schema = json.dumps(schema)
     for ref_block in re.findall(r'{"\$ref": "#\/definitions\/.+?(?="})"}', str_schema):
         ref = json.loads(ref_block)["$ref"]
-        str_schema = str_schema.replace(ref_block, json.dumps(json_schema_ref_resolver.resolve(ref)[1]))
-    pyschema: dict = json.loads(str_schema)
+        str_schema = str_schema.replace(
+            ref_block, json.dumps(json_schema_ref_resolver.resolve(ref)[1])
+        )
+    pyschema: dict[str, Any] = json.loads(str_schema)
     del pyschema["definitions"]
     return pyschema

airbyte_cdk/utils/stream_status_utils.py

@@ -4,11 +4,13 @@
 
 
 from datetime import datetime
+from typing import List, Optional, Union
 
 from airbyte_cdk.models import (
     AirbyteMessage,
     AirbyteStream,
     AirbyteStreamStatus,
+    AirbyteStreamStatusReason,
     AirbyteStreamStatusTraceMessage,
     AirbyteTraceMessage,
     StreamDescriptor,
@@ -17,7 +19,11 @@ from airbyte_cdk.models import (
 from airbyte_cdk.models import Type as MessageType
 
 
-def as_airbyte_message(stream: AirbyteStream, current_status: AirbyteStreamStatus) -> AirbyteMessage:
+def as_airbyte_message(
+    stream: Union[AirbyteStream, StreamDescriptor],
+    current_status: AirbyteStreamStatus,
+    reasons: Optional[List[AirbyteStreamStatusReason]] = None,
+) -> AirbyteMessage:
     """
     Builds an AirbyteStreamStatusTraceMessage for the provided stream
     """
@@ -30,6 +36,7 @@ def as_airbyte_message(stream: AirbyteStream, current_status: AirbyteStreamStatu
         stream_status=AirbyteStreamStatusTraceMessage(
             stream_descriptor=StreamDescriptor(name=stream.name, namespace=stream.namespace),
             status=current_status,
+            reasons=reasons,
         ),
     )
 

airbyte_cdk/utils/traced_exception.py

@@ -1,15 +1,17 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
+import time
 import traceback
-from datetime import datetime
-from typing import Optional
+from typing import Any, Optional
+
+import orjson
 
 from airbyte_cdk.models import (
     AirbyteConnectionStatus,
     AirbyteErrorTraceMessage,
     AirbyteMessage,
+    AirbyteMessageSerializer,
     AirbyteTraceMessage,
     FailureType,
     Status,
@@ -31,24 +33,32 @@ class AirbyteTracedException(Exception):
         message: Optional[str] = None,
         failure_type: FailureType = FailureType.system_error,
         exception: Optional[BaseException] = None,
+        stream_descriptor: Optional[StreamDescriptor] = None,
     ):
         """
         :param internal_message: the internal error that caused the failure
         :param message: a user-friendly message that indicates the cause of the error
         :param failure_type: the type of error
         :param exception: the exception that caused the error, from which the stack trace should be retrieved
+        :param stream_descriptor: describe the stream from which the exception comes from
         """
         self.internal_message = internal_message
         self.message = message
         self.failure_type = failure_type
         self._exception = exception
+        self._stream_descriptor = stream_descriptor
         super().__init__(internal_message)
 
-    def as_airbyte_message(self, stream_descriptor: StreamDescriptor = None) -> AirbyteMessage:
+    def as_airbyte_message(
+        self, stream_descriptor: Optional[StreamDescriptor] = None
+    ) -> AirbyteMessage:
         """
         Builds an AirbyteTraceMessage from the exception
+
+        :param stream_descriptor is deprecated, please use the stream_description in `__init__ or `from_exception`. If many
+          stream_descriptors are defined, the one from `as_airbyte_message` will be discarded.
         """
-        now_millis = datetime.now().timestamp() * 1000.0
+        now_millis = time.time_ns() // 1_000_000
 
         trace_exc = self._exception or self
         stack_trace_str = "".join(traceback.TracebackException.from_exception(trace_exc).format())
@@ -57,49 +67,79 @@ class AirbyteTracedException(Exception):
             type=TraceType.ERROR,
             emitted_at=now_millis,
             error=AirbyteErrorTraceMessage(
-                message=self.message or "Something went wrong in the connector. See the logs for more details.",
+                message=self.message
+                or "Something went wrong in the connector. See the logs for more details.",
                 internal_message=self.internal_message,
                 failure_type=self.failure_type,
                 stack_trace=stack_trace_str,
-                stream_descriptor=stream_descriptor,
+                stream_descriptor=self._stream_descriptor
+                if self._stream_descriptor is not None
+                else stream_descriptor,
             ),
         )
 
         return AirbyteMessage(type=MessageType.TRACE, trace=trace_message)
 
-    def as_connection_status_message(self) -> AirbyteMessage:
+    def as_connection_status_message(self) -> Optional[AirbyteMessage]:
         if self.failure_type == FailureType.config_error:
-            output_message = AirbyteMessage(
-                type=MessageType.CONNECTION_STATUS, connectionStatus=AirbyteConnectionStatus(status=Status.FAILED, message=self.message)
+            return AirbyteMessage(
+                type=MessageType.CONNECTION_STATUS,
+                connectionStatus=AirbyteConnectionStatus(
+                    status=Status.FAILED, message=self.message
+                ),
             )
-            return output_message
+        return None
 
     def emit_message(self) -> None:
         """
         Prints the exception as an AirbyteTraceMessage.
         Note that this will be called automatically on uncaught exceptions when using the airbyte_cdk entrypoint.
         """
-        message = self.as_airbyte_message().json(exclude_unset=True)
+        message = orjson.dumps(AirbyteMessageSerializer.dump(self.as_airbyte_message())).decode()
         filtered_message = filter_secrets(message)
         print(filtered_message)
 
     @classmethod
-    def from_exception(cls, exc: BaseException, *args, **kwargs) -> "AirbyteTracedException":  # type: ignore  # ignoring because of args and kwargs
+    def from_exception(
+        cls,
+        exc: BaseException,
+        stream_descriptor: Optional[StreamDescriptor] = None,
+        *args: Any,
+        **kwargs: Any,
+    ) -> "AirbyteTracedException":
         """
         Helper to create an AirbyteTracedException from an existing exception
         :param exc: the exception that caused the error
+        :param stream_descriptor: describe the stream from which the exception comes from
         """
-        return cls(internal_message=str(exc), exception=exc, *args, **kwargs)  # type: ignore  # ignoring because of args and kwargs
+        return cls(
+            internal_message=str(exc),
+            exception=exc,
+            stream_descriptor=stream_descriptor,
+            *args,
+            **kwargs,
+        )  # type: ignore  # ignoring because of args and kwargs
 
-    def as_sanitized_airbyte_message(self, stream_descriptor: StreamDescriptor = None) -> AirbyteMessage:
+    def as_sanitized_airbyte_message(
+        self, stream_descriptor: Optional[StreamDescriptor] = None
+    ) -> AirbyteMessage:
         """
         Builds an AirbyteTraceMessage from the exception and sanitizes any secrets from the message body
+
+        :param stream_descriptor is deprecated, please use the stream_description in `__init__ or `from_exception`. If many
+          stream_descriptors are defined, the one from `as_sanitized_airbyte_message` will be discarded.
         """
         error_message = self.as_airbyte_message(stream_descriptor=stream_descriptor)
-        if error_message.trace.error.message:
-            error_message.trace.error.message = filter_secrets(error_message.trace.error.message)
-        if error_message.trace.error.internal_message:
-            error_message.trace.error.internal_message = filter_secrets(error_message.trace.error.internal_message)
-        if error_message.trace.error.stack_trace:
-            error_message.trace.error.stack_trace = filter_secrets(error_message.trace.error.stack_trace)
+        if error_message.trace.error.message:  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            error_message.trace.error.message = filter_secrets(  # type: ignore[union-attr]
+                error_message.trace.error.message,  # type: ignore[union-attr]
+            )
+        if error_message.trace.error.internal_message:  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            error_message.trace.error.internal_message = filter_secrets(  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+                error_message.trace.error.internal_message  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            )
+        if error_message.trace.error.stack_trace:  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            error_message.trace.error.stack_trace = filter_secrets(  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+                error_message.trace.error.stack_trace  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            )
         return error_message

airbyte_cdk-6.17.1.dev1.dist-info/METADATA

@@ -0,0 +1,109 @@
+Metadata-Version: 2.3
+Name: airbyte-cdk
+Version: 6.17.1.dev1
+Summary: A framework for writing Airbyte Connectors.
+License: MIT
+Keywords: airbyte,connector-development-kit,cdk
+Author: Airbyte
+Author-email: contact@airbyte.io
+Requires-Python: >=3.10,<3.13
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Provides-Extra: file-based
+Provides-Extra: sql
+Provides-Extra: vector-db-based
+Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
+Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
+Requires-Dist: Unidecode (>=1.3,<2.0)
+Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
+Requires-Dist: avro (>=1.11.2,<1.12.0) ; extra == "file-based"
+Requires-Dist: backoff
+Requires-Dist: cachetools
+Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
+Requires-Dist: cryptography (>=42.0.5,<44.0.0)
+Requires-Dist: dpath (>=2.1.6,<3.0.0)
+Requires-Dist: dunamai (>=1.22.0,<2.0.0)
+Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
+Requires-Dist: genson (==1.3.0)
+Requires-Dist: isodate (>=0.6.1,<0.7.0)
+Requires-Dist: jsonref (>=0.2,<0.3)
+Requires-Dist: jsonschema (>=4.17.3,<4.18.0)
+Requires-Dist: langchain (==0.1.16) ; extra == "vector-db-based"
+Requires-Dist: langchain_core (==0.1.42)
+Requires-Dist: markdown ; extra == "file-based"
+Requires-Dist: nltk (==3.9.1)
+Requires-Dist: numpy (<2)
+Requires-Dist: openai[embeddings] (==0.27.9) ; extra == "vector-db-based"
+Requires-Dist: orjson (>=3.10.7,<4.0.0)
+Requires-Dist: pandas (==2.2.2)
+Requires-Dist: pdf2image (==1.16.3) ; extra == "file-based"
+Requires-Dist: pdfminer.six (==20221105) ; extra == "file-based"
+Requires-Dist: pendulum (<3.0.0)
+Requires-Dist: psutil (==6.1.0)
+Requires-Dist: pyarrow (>=15.0.0,<15.1.0) ; extra == "file-based"
+Requires-Dist: pydantic (>=2.7,<3.0)
+Requires-Dist: pyjwt (>=2.8.0,<3.0.0)
+Requires-Dist: pyrate-limiter (>=3.1.0,<3.2.0)
+Requires-Dist: pytesseract (==0.3.10) ; extra == "file-based"
+Requires-Dist: python-calamine (==0.2.3) ; extra == "file-based"
+Requires-Dist: python-dateutil
+Requires-Dist: python-snappy (==0.7.3) ; extra == "file-based"
+Requires-Dist: python-ulid (>=3.0.0,<4.0.0)
+Requires-Dist: pytz (==2024.2)
+Requires-Dist: rapidfuzz (>=3.10.1,<4.0.0)
+Requires-Dist: requests
+Requires-Dist: requests_cache
+Requires-Dist: serpyco-rs (>=1.10.2,<2.0.0)
+Requires-Dist: sqlalchemy (>=2.0,<3.0,!=2.0.36) ; extra == "sql"
+Requires-Dist: tiktoken (==0.8.0) ; extra == "vector-db-based"
+Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"
+Requires-Dist: unstructured[docx,pptx] (==0.10.27) ; extra == "file-based"
+Requires-Dist: wcmatch (==10.0)
+Requires-Dist: xmltodict (>=0.13.0,<0.14.0)
+Project-URL: Documentation, https://docs.airbyte.io/
+Project-URL: Homepage, https://airbyte.com
+Project-URL: Repository, https://github.com/airbytehq/airbyte-python-cdk
+Description-Content-Type: text/markdown
+
+# Airbyte Python CDK and Low-Code CDK
+
+Airbyte Python CDK is a framework for building Airbyte API Source Connectors. It provides a set of
+classes and helpers that make it easy to build a connector against an HTTP API (REST, GraphQL, etc),
+or a generic Python source connector.
+
+## Building Connectors with the CDK
+
+If you're looking to build a connector, we highly recommend that you first
+[start with the Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview).
+It should be enough for 90% connectors out there. For more flexible and complex connectors, use the
+[low-code CDK and `SourceDeclarativeManifest`](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview).
+
+For more information on building connectors, please see the [Connector Development](https://docs.airbyte.com/connector-development/) guide on [docs.airbyte.com](https://docs.airbyte.com).
+
+## Python CDK Overview
+
+Airbyte CDK code is within `airbyte_cdk` directory. Here's a high level overview of what's inside:
+
+- `airbyte_cdk/connector_builder`. Internal wrapper that helps the Connector Builder platform run a declarative manifest (low-code connector). You should not use this code directly. If you need to run a `SourceDeclarativeManifest`, take a look at [`source-declarative-manifest`](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-declarative-manifest) connector implementation instead.
+- `airbyte_cdk/cli/source_declarative_manifest`. This module defines the `source-declarative-manifest` (aka "SDM") connector execution logic and associated CLI.
+- `airbyte_cdk/destinations`. Basic Destination connector support! If you're building a Destination connector in Python, try that. Some of our vector DB destinations like `destination-pinecone` are using that code.
+- `airbyte_cdk/models` expose `airbyte_protocol.models` as a part of `airbyte_cdk` package.
+- `airbyte_cdk/sources/concurrent_source` is the Concurrent CDK implementation. It supports reading data from streams concurrently per slice / partition, useful for connectors with high throughput and high number of records.
+- `airbyte_cdk/sources/declarative` is the low-code CDK. It works on top of Airbyte Python CDK, but provides a declarative manifest language to define streams, operations, etc. This makes it easier to build connectors without writing Python code.
+- `airbyte_cdk/sources/file_based` is the CDK for file-based sources. Examples include S3, Azure, GCS, etc.
+
+## Contributing
+
+For instructions on how to contribute, please see our [Contributing Guide](docs/CONTRIBUTING.md).
+
+## Release Management
+
+Please see the [Release Management](docs/RELEASES.md) guide for information on how to perform releases and pre-releases.
+