airbyte-cdk 6.8.1rc9__py3-none-any.whl → 6.8.2.dev1__py3-none-any.whl

airbyte_cdk/test/entrypoint_wrapper.py

@@ -23,7 +23,7 @@ from io import StringIO
 from pathlib import Path
 from typing import Any, List, Mapping, Optional, Union
 
-from orjson import orjson
+import orjson
 from pydantic import ValidationError as V2ValidationError
 from serpyco_rs import SchemaValidationError
 
@@ -63,7 +63,7 @@ class EntrypointOutput:
     @staticmethod
     def _parse_message(message: str) -> AirbyteMessage:
         try:
-            return AirbyteMessageSerializer.load(orjson.loads(message))  # type: ignore[no-any-return] # Serializer.load() always returns AirbyteMessage
+            return AirbyteMessageSerializer.load(orjson.loads(message))
         except (orjson.JSONDecodeError, SchemaValidationError):
             # The platform assumes that logs that are not of AirbyteMessage format are log messages
             return AirbyteMessage(
@@ -129,14 +129,19 @@ class EntrypointOutput:
         return [
             message
             for message in self._get_message_by_types([Type.TRACE])
-            if message.trace.type == trace_type
-        ]  # type: ignore[union-attr] # trace has `type`
+            if message.trace.type == trace_type  # type: ignore[union-attr] # trace has `type`
+        ]
 
     def is_in_logs(self, pattern: str) -> bool:
         """Check if any log message case-insensitive matches the pattern."""
         return any(
-            re.search(pattern, entry.log.message, flags=re.IGNORECASE) for entry in self.logs
-        )  # type: ignore[union-attr] # log has `message`
+            re.search(
+                pattern,
+                entry.log.message,  # type: ignore[union-attr] # log has `message`
+                flags=re.IGNORECASE,
+            )
+            for entry in self.logs
+        )
 
     def is_not_in_logs(self, pattern: str) -> bool:
         """Check if no log message matches the case-insensitive pattern."""

airbyte_cdk/test/mock_http/response_builder.py

@@ -183,7 +183,7 @@ class HttpResponseBuilder:
 
 def _get_unit_test_folder(execution_folder: str) -> FilePath:
     # FIXME: This function should be removed after the next CDK release to avoid breaking amazon-seller-partner test code.
-    return get_unit_test_folder(execution_folder)  # type: ignore # get_unit_test_folder is known to return a FilePath
+    return get_unit_test_folder(execution_folder)
 
 
 def find_template(resource: str, execution_folder: str) -> Dict[str, Any]:

airbyte_cdk/utils/airbyte_secrets_utils.py

@@ -47,7 +47,7 @@ def get_secrets(
     result = []
     for path in secret_paths:
         try:
-            result.append(dpath.get(config, path))
+            result.append(dpath.get(config, path))  # type: ignore  # dpath expect MutableMapping but doesn't need it
         except KeyError:
             # Since we try to get paths to all known secrets in the spec, in the case of oneOfs, some secret fields may not be present
             # In that case, a KeyError is thrown. This is expected behavior.

airbyte_cdk/utils/event_timing.py

@@ -7,7 +7,7 @@ import logging
 import time
 from contextlib import contextmanager
 from dataclasses import dataclass, field
-from typing import Optional
+from typing import Any, Generator, Literal, Optional
 
 logger = logging.getLogger("airbyte")
 
@@ -18,13 +18,13 @@ class EventTimer:
        Event nesting follows a LIFO pattern, so finish will apply to the last started event.
     """
 
-    def __init__(self, name):
+    def __init__(self, name: str) -> None:
         self.name = name
-        self.events = {}
+        self.events: dict[str, Any] = {}
         self.count = 0
-        self.stack = []
+        self.stack: list[Any] = []
 
-    def start_event(self, name):
+    def start_event(self, name: str) -> None:
         """
         Start a new event and push it to the stack.
         """
@@ -32,7 +32,7 @@ class EventTimer:
         self.count += 1
         self.stack.insert(0, self.events[name])
 
-    def finish_event(self):
+    def finish_event(self) -> None:
         """
         Finish the current event and pop it from the stack.
         """
@@ -43,7 +43,7 @@ class EventTimer:
         else:
             logger.warning(f"{self.name} finish_event called without start_event")
 
-    def report(self, order_by="name"):
+    def report(self, order_by: Literal["name", "duration"] = "name") -> str:
         """
         :param order_by: 'name' or 'duration'
         """
@@ -69,15 +69,15 @@ class Event:
             return (self.end - self.start) / 1e9
         return float("+inf")
 
-    def __str__(self):
+    def __str__(self) -> str:
         return f"{self.name} {datetime.timedelta(seconds=self.duration)}"
 
-    def finish(self):
+    def finish(self) -> None:
         self.end = time.perf_counter_ns()
 
 
 @contextmanager
-def create_timer(name):
+def create_timer(name: str) -> Generator[EventTimer, Any, None]:
     """
     Creates a new EventTimer as a context manager to improve code readability.
     """

airbyte_cdk/utils/message_utils.py

@@ -8,15 +8,16 @@ def get_stream_descriptor(message: AirbyteMessage) -> HashableStreamDescriptor:
     match message.type:
         case Type.RECORD:
             return HashableStreamDescriptor(
-                name=message.record.stream, namespace=message.record.namespace
-            )  # type: ignore[union-attr] # record has `stream` and `namespace`
+                name=message.record.stream,  # type: ignore[union-attr] # record has `stream`
+                namespace=message.record.namespace,  # type: ignore[union-attr] # record has `namespace`
+            )
         case Type.STATE:
             if not message.state.stream or not message.state.stream.stream_descriptor:  # type: ignore[union-attr] # state has `stream`
                 raise ValueError(
                     "State message was not in per-stream state format, which is required for record counts."
                 )
             return HashableStreamDescriptor(
-                name=message.state.stream.stream_descriptor.name,
+                name=message.state.stream.stream_descriptor.name,  # type: ignore[union-attr] # state has `stream`
                 namespace=message.state.stream.stream_descriptor.namespace,  # type: ignore[union-attr] # state has `stream`
             )
         case _:

airbyte_cdk/utils/spec_schema_transformations.py

@@ -4,11 +4,12 @@
 
 import json
 import re
+from typing import Any
 
 from jsonschema import RefResolver
 
 
-def resolve_refs(schema: dict) -> dict:
+def resolve_refs(schema: dict[str, Any]) -> dict[str, Any]:
     """
     For spec schemas generated using Pydantic models, the resulting JSON schema can contain refs between object
     relationships.
@@ -20,6 +21,6 @@ def resolve_refs(schema: dict) -> dict:
         str_schema = str_schema.replace(
             ref_block, json.dumps(json_schema_ref_resolver.resolve(ref)[1])
         )
-    pyschema: dict = json.loads(str_schema)
+    pyschema: dict[str, Any] = json.loads(str_schema)
     del pyschema["definitions"]
     return pyschema

airbyte_cdk/utils/traced_exception.py

@@ -3,9 +3,9 @@
 #
 import time
 import traceback
-from typing import Optional
+from typing import Any, Optional
 
-from orjson import orjson
+import orjson
 
 from airbyte_cdk.models import (
     AirbyteConnectionStatus,
@@ -104,9 +104,9 @@ class AirbyteTracedException(Exception):
         cls,
         exc: BaseException,
         stream_descriptor: Optional[StreamDescriptor] = None,
-        *args,
-        **kwargs,
-    ) -> "AirbyteTracedException":  # type: ignore  # ignoring because of args and kwargs
+        *args: Any,
+        **kwargs: Any,
+    ) -> "AirbyteTracedException":
         """
         Helper to create an AirbyteTracedException from an existing exception
         :param exc: the exception that caused the error
@@ -131,13 +131,15 @@ class AirbyteTracedException(Exception):
         """
         error_message = self.as_airbyte_message(stream_descriptor=stream_descriptor)
         if error_message.trace.error.message:  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
-            error_message.trace.error.message = filter_secrets(error_message.trace.error.message)  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            error_message.trace.error.message = filter_secrets(  # type: ignore[union-attr]
+                error_message.trace.error.message,  # type: ignore[union-attr]
+            )
         if error_message.trace.error.internal_message:  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
-            error_message.trace.error.internal_message = filter_secrets(
-                error_message.trace.error.internal_message
-            )  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            error_message.trace.error.internal_message = filter_secrets(  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+                error_message.trace.error.internal_message  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            )
         if error_message.trace.error.stack_trace:  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
-            error_message.trace.error.stack_trace = filter_secrets(
-                error_message.trace.error.stack_trace
-            )  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            error_message.trace.error.stack_trace = filter_secrets(  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+                error_message.trace.error.stack_trace  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage
+            )
         return error_message

airbyte_cdk-6.8.2.dev1.dist-info/METADATA

@@ -0,0 +1,111 @@
+Metadata-Version: 2.1
+Name: airbyte-cdk
+Version: 6.8.2.dev1
+Summary: A framework for writing Airbyte Connectors.
+Home-page: https://airbyte.com
+License: MIT
+Keywords: airbyte,connector-development-kit,cdk
+Author: Airbyte
+Author-email: contact@airbyte.io
+Requires-Python: >=3.10,<3.13
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Provides-Extra: file-based
+Provides-Extra: sphinx-docs
+Provides-Extra: sql
+Provides-Extra: vector-db-based
+Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
+Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
+Requires-Dist: Sphinx (>=4.2,<4.3) ; extra == "sphinx-docs"
+Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
+Requires-Dist: avro (>=1.11.2,<1.12.0) ; extra == "file-based"
+Requires-Dist: backoff
+Requires-Dist: cachetools
+Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
+Requires-Dist: cryptography (>=42.0.5,<44.0.0)
+Requires-Dist: dpath (>=2.1.6,<3.0.0)
+Requires-Dist: dunamai (>=1.22.0,<2.0.0)
+Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
+Requires-Dist: genson (==1.3.0)
+Requires-Dist: isodate (>=0.6.1,<0.7.0)
+Requires-Dist: jsonref (>=0.2,<0.3)
+Requires-Dist: jsonschema (>=4.17.3,<4.18.0)
+Requires-Dist: langchain (==0.1.16) ; extra == "vector-db-based"
+Requires-Dist: langchain_core (==0.1.42)
+Requires-Dist: markdown ; extra == "file-based"
+Requires-Dist: nltk (==3.9.1)
+Requires-Dist: numpy (<2)
+Requires-Dist: openai[embeddings] (==0.27.9) ; extra == "vector-db-based"
+Requires-Dist: orjson (>=3.10.7,<4.0.0)
+Requires-Dist: pandas (==2.2.2)
+Requires-Dist: pdf2image (==1.16.3) ; extra == "file-based"
+Requires-Dist: pdfminer.six (==20221105) ; extra == "file-based"
+Requires-Dist: pendulum (<3.0.0)
+Requires-Dist: psutil (==6.1.0)
+Requires-Dist: pyarrow (>=15.0.0,<15.1.0) ; extra == "file-based"
+Requires-Dist: pydantic (>=2.7,<3.0)
+Requires-Dist: pyjwt (>=2.8.0,<3.0.0)
+Requires-Dist: pyrate-limiter (>=3.1.0,<3.2.0)
+Requires-Dist: pytesseract (==0.3.10) ; extra == "file-based"
+Requires-Dist: python-calamine (==0.2.3) ; extra == "file-based"
+Requires-Dist: python-dateutil
+Requires-Dist: python-snappy (==0.7.3) ; extra == "file-based"
+Requires-Dist: python-ulid (>=3.0.0,<4.0.0)
+Requires-Dist: pytz (==2024.1)
+Requires-Dist: rapidfuzz (>=3.10.1,<4.0.0)
+Requires-Dist: requests
+Requires-Dist: requests_cache
+Requires-Dist: serpyco-rs (>=1.10.2,<2.0.0)
+Requires-Dist: sphinx-rtd-theme (>=1.0,<1.1) ; extra == "sphinx-docs"
+Requires-Dist: sqlalchemy (>=2.0,<3.0,!=2.0.36) ; extra == "sql"
+Requires-Dist: tiktoken (==0.8.0) ; extra == "vector-db-based"
+Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"
+Requires-Dist: unstructured[docx,pptx] (==0.10.27) ; extra == "file-based"
+Requires-Dist: wcmatch (==10.0)
+Requires-Dist: xmltodict (>=0.13.0,<0.14.0)
+Project-URL: Documentation, https://docs.airbyte.io/
+Project-URL: Repository, https://github.com/airbytehq/airbyte-python-cdk
+Description-Content-Type: text/markdown
+
+# Airbyte Python CDK and Low-Code CDK
+
+Airbyte Python CDK is a framework for building Airbyte API Source Connectors. It provides a set of
+classes and helpers that make it easy to build a connector against an HTTP API (REST, GraphQL, etc),
+or a generic Python source connector.
+
+## Building Connectors with the CDK
+
+If you're looking to build a connector, we highly recommend that you first
+[start with the Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview).
+It should be enough for 90% connectors out there. For more flexible and complex connectors, use the
+[low-code CDK and `SourceDeclarativeManifest`](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview).
+
+For more information on building connectors, please see the [Connector Development](https://docs.airbyte.com/connector-development/) guide on [docs.airbyte.com](https://docs.airbyte.com).
+
+## Python CDK Overview
+
+Airbyte CDK code is within `airbyte_cdk` directory. Here's a high level overview of what's inside:
+
+- `airbyte_cdk/connector_builder`. Internal wrapper that helps the Connector Builder platform run a declarative manifest (low-code connector). You should not use this code directly. If you need to run a `SourceDeclarativeManifest`, take a look at [`source-declarative-manifest`](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-declarative-manifest) connector implementation instead.
+- `airbyte_cdk/cli/source_declarative_manifest`. This module defines the `source-declarative-manifest` (aka "SDM") connector execution logic and associated CLI.
+- `airbyte_cdk/destinations`. Basic Destination connector support! If you're building a Destination connector in Python, try that. Some of our vector DB destinations like `destination-pinecone` are using that code.
+- `airbyte_cdk/models` expose `airbyte_protocol.models` as a part of `airbyte_cdk` package.
+- `airbyte_cdk/sources/concurrent_source` is the Concurrent CDK implementation. It supports reading data from streams concurrently per slice / partition, useful for connectors with high throughput and high number of records.
+- `airbyte_cdk/sources/declarative` is the low-code CDK. It works on top of Airbyte Python CDK, but provides a declarative manifest language to define streams, operations, etc. This makes it easier to build connectors without writing Python code.
+- `airbyte_cdk/sources/file_based` is the CDK for file-based sources. Examples include S3, Azure, GCS, etc.
+
+## Contributing
+
+For instructions on how to contribute, please see our [Contributing Guide](docs/CONTRIBUTING.md).
+
+## Release Management
+
+Please see the [Release Management](docs/RELEASES.md) guide for information on how to perform releases and pre-releases.
+