airbyte-cdk 6.8.1rc6__py3-none-any.whl → 6.8.1rc8__py3-none-any.whl

airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -56,8 +56,9 @@ from airbyte_cdk.sources.types import Config, StreamState
 
 
 class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
-    # By default, we defer to a value of 1 which represents running a connector using the Concurrent CDK engine on only one thread.
-    SINGLE_THREADED_CONCURRENCY_LEVEL = 1
+    # By default, we defer to a value of 2. A value lower than than could cause a PartitionEnqueuer to be stuck in a state of deadlock
+    # because it has hit the limit of futures but not partition reader is consuming them.
+    SINGLE_THREADED_CONCURRENCY_LEVEL = 2
 
     def __init__(
         self,
@@ -78,6 +79,9 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             emit_connector_builder_messages=emit_connector_builder_messages,
             disable_resumable_full_refresh=True,
         )
+        self._config = config
+        self._concurrent_streams: Optional[List[AbstractStream]] = None
+        self._synchronous_streams: Optional[List[Stream]] = None
 
         super().__init__(
             source_config=source_config,
@@ -88,21 +92,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
 
         self._state = state
 
-        self._concurrent_streams: Optional[List[AbstractStream]]
-        self._synchronous_streams: Optional[List[Stream]]
-
-        # If the connector command was SPEC, there is no incoming config, and we cannot instantiate streams because
-        # they might depend on it. Ideally we want to have a static method on this class to get the spec without
-        # any other arguments, but the existing entrypoint.py isn't designed to support this. Just noting this
-        # for our future improvements to the CDK.
-        if config:
-            self._concurrent_streams, self._synchronous_streams = self._group_streams(
-                config=config or {}
-            )
-        else:
-            self._concurrent_streams = None
-            self._synchronous_streams = None
-
         concurrency_level_from_manifest = self._source_config.get("concurrency_level")
         if concurrency_level_from_manifest:
             concurrency_level_component = self._constructor.create_component(
@@ -121,7 +110,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             )  # Partition_generation iterates using range based on this value. If this is floored to zero we end up in a dead lock during start up
         else:
             concurrency_level = self.SINGLE_THREADED_CONCURRENCY_LEVEL
-            initial_number_of_partitions_to_generate = self.SINGLE_THREADED_CONCURRENCY_LEVEL
+            initial_number_of_partitions_to_generate = self.SINGLE_THREADED_CONCURRENCY_LEVEL // 2
 
         self._concurrent_source = ConcurrentSource.create(
             num_workers=concurrency_level,
@@ -131,6 +120,19 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             message_repository=self.message_repository,  # type: ignore  # message_repository is always instantiated with a value by factory
         )
 
+    def _actually_group(self) -> None:
+        # If the connector command was SPEC, there is no incoming config, and we cannot instantiate streams because
+        # they might depend on it. Ideally we want to have a static method on this class to get the spec without
+        # any other arguments, but the existing entrypoint.py isn't designed to support this. Just noting this
+        # for our future improvements to the CDK.
+        if self._config:
+            self._concurrent_streams, self._synchronous_streams = self._group_streams(
+                config=self._config or {}
+            )
+        else:
+            self._concurrent_streams = None
+            self._synchronous_streams = None
+
     def read(
         self,
         logger: logging.Logger,
@@ -140,6 +142,9 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
     ) -> Iterator[AirbyteMessage]:
         # ConcurrentReadProcessor pops streams that are finished being read so before syncing, the names of the concurrent
         # streams must be saved so that they can be removed from the catalog before starting synchronous streams
+        if self._concurrent_streams is None:
+            self._actually_group()
+
         if self._concurrent_streams:
             concurrent_stream_names = set(
                 [concurrent_stream.name for concurrent_stream in self._concurrent_streams]
@@ -165,6 +170,9 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
         yield from super().read(logger, config, filtered_catalog, state)
 
     def discover(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteCatalog:
+        if self._concurrent_streams is None:
+            self._actually_group()
+
         concurrent_streams = self._concurrent_streams or []
         synchronous_streams = self._synchronous_streams or []
         return AirbyteCatalog(

airbyte_cdk/sources/streams/http/http_client.py

@@ -95,6 +95,7 @@ class HttpClient:
     ):
         self._name = name
         self._api_budget: APIBudget = api_budget or APIBudget(policies=[])
+        self._logger = logger
         if session:
             self._session = session
         else:
@@ -108,7 +109,6 @@ class HttpClient:
             )
         if isinstance(authenticator, AuthBase):
             self._session.auth = authenticator
-        self._logger = logger
         self._error_handler = error_handler or HttpStatusErrorHandler(self._logger)
         if backoff_strategy is not None:
             if isinstance(backoff_strategy, list):
@@ -142,8 +142,9 @@ class HttpClient:
             if cache_dir:
                 sqlite_path = str(Path(cache_dir) / self.cache_filename)
             else:
+                self._logger.info("Using memory for cache")  # TODO: remove
                 sqlite_path = "file::memory:?cache=shared"
-            backend = SkipFailureSQLiteCache(sqlite_path)
+            backend = SkipFailureSQLiteCache(self._name, sqlite_path)  # TODO maybe add a busy timeout
             return CachedLimiterSession(
                 sqlite_path, backend=backend, api_budget=self._api_budget, match_headers=True
             )  # type: ignore # there are no typeshed stubs for requests_cache
@@ -541,6 +542,7 @@ class SkipFailureSQLiteDict(requests_cache.backends.sqlite.SQLiteDict):
 class SkipFailureSQLiteCache(requests_cache.backends.sqlite.SQLiteCache):
     def __init__(  # type: ignore  # ignoring as lib is not typed
         self,
+        table_name="response",
         db_path="http_cache",
         serializer=None,
         **kwargs,
@@ -548,11 +550,13 @@ class SkipFailureSQLiteCache(requests_cache.backends.sqlite.SQLiteCache):
         super().__init__(db_path, serializer, **kwargs)
         skwargs = {"serializer": serializer, **kwargs} if serializer else kwargs
         self.responses: requests_cache.backends.sqlite.SQLiteDict = SkipFailureSQLiteDict(
-            db_path, table_name="responses", **skwargs
+            db_path, table_name=table_name, fast_save=True, wal=True, **skwargs
         )
         self.redirects: requests_cache.backends.sqlite.SQLiteDict = SkipFailureSQLiteDict(
             db_path,
-            table_name="redirects",
+            table_name=f"redirects_{table_name}",
+            fast_save=True,
+            wal=True,
             lock=self.responses._lock,
             serializer=None,
             **kwargs,

{airbyte_cdk-6.8.1rc6.dist-info → airbyte_cdk-6.8.1rc8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.8.1rc6
+Version: 6.8.1rc8
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.8.1rc6.dist-info → airbyte_cdk-6.8.1rc8.dist-info}/RECORD

@@ -62,7 +62,7 @@ airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=dAA-UhmMj0WLXCkRQr
 airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
 airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
 airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
-airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=4bpS63fz2K4mLPF33eG1xXgGRzzjjBAvpgDbiDrIs_Q,23549
+airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=rpWo7poxbNYJHPeV2tfiodJ4jeS6-5nfXV31CuIjjw4,23903
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=SX9JjdesN1edN2WVUVMzU_ptqp2QB1OnsnjZ4mwcX7w,2579
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=8VZJP18eJLabSPP1XBSPDaagUBG6q1ynIiPJy3rE2mc,5344
@@ -273,7 +273,7 @@ airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py,sha
 airbyte_cdk/sources/streams/http/error_handlers/response_models.py,sha256=xGIVELBFY0TmH9aUq1ikoqJz8oHLr6di2JLvKWVEO-s,2236
 airbyte_cdk/sources/streams/http/exceptions.py,sha256=njC7MlMJoFYcSGz4mIp6-bqLFTr6vC8ej25X0oSeyjE,1824
 airbyte_cdk/sources/streams/http/http.py,sha256=h0bq4arzMeJsR-5HZNfGYXtZhgVvLbW6myi9fuhMayU,28467
-airbyte_cdk/sources/streams/http/http_client.py,sha256=WxmoFeoISmRnwp2nzfJ0DeD1-05FhL0z8hIni-o7vV8,23067
+airbyte_cdk/sources/streams/http/http_client.py,sha256=mBv9ck2mUv4kvVozPmHN5gHVLIgZ9e11trUsTjskPiM,23308
 airbyte_cdk/sources/streams/http/rate_limiting.py,sha256=IwdjrHKUnU97XO4qONgYRv4YYW51xQ8SJm4WLafXDB8,6351
 airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
 airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=nxI94yJ3bGfpDO8RR3QvOJ-PSW0n9CElSAkgl5ae80Y,10321
@@ -330,8 +330,8 @@ airbyte_cdk/utils/slice_hasher.py,sha256=-pHexlNYoWYPnXNH-M7HEbjmeJe9Zk7SJijdQ7d
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=LVc9KbtMeV_z99jWo0Ou8u4l6eBJ0BWNhxj4zrrGKRs,763
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=a6q51tBS3IdtefuOiL1eBwSmnNAXfjFMlMjSIQ_Tl-o,6165
-airbyte_cdk-6.8.1rc6.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.8.1rc6.dist-info/METADATA,sha256=hIuuXLu8aB31Ru7gvg99UecbsE_uRdvwX8TLSBXVBXg,13522
-airbyte_cdk-6.8.1rc6.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.8.1rc6.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.8.1rc6.dist-info/RECORD,,
+airbyte_cdk-6.8.1rc8.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.8.1rc8.dist-info/METADATA,sha256=cQWW8Fk6YGrxN-sqIW9PKhXR2pKGkURuYFUhsYgP3po,13522
+airbyte_cdk-6.8.1rc8.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.8.1rc8.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.8.1rc8.dist-info/RECORD,,