airbyte-cdk 6.8.1rc5__py3-none-any.whl → 6.8.1rc7__py3-none-any.whl

airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -56,8 +56,9 @@ from airbyte_cdk.sources.types import Config, StreamState
 
 
 class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
-    # By default, we defer to a value of 1 which represents running a connector using the Concurrent CDK engine on only one thread.
-    SINGLE_THREADED_CONCURRENCY_LEVEL = 1
+    # By default, we defer to a value of 2. A value lower than than could cause a PartitionEnqueuer to be stuck in a state of deadlock
+    # because it has hit the limit of futures but not partition reader is consuming them.
+    SINGLE_THREADED_CONCURRENCY_LEVEL = 2
 
     def __init__(
         self,
@@ -121,7 +122,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             )  # Partition_generation iterates using range based on this value. If this is floored to zero we end up in a dead lock during start up
         else:
             concurrency_level = self.SINGLE_THREADED_CONCURRENCY_LEVEL
-            initial_number_of_partitions_to_generate = self.SINGLE_THREADED_CONCURRENCY_LEVEL
+            initial_number_of_partitions_to_generate = self.SINGLE_THREADED_CONCURRENCY_LEVEL // 2
 
         self._concurrent_source = ConcurrentSource.create(
             num_workers=concurrency_level,

airbyte_cdk/sources/declarative/interpolation/jinja.py

@@ -106,10 +106,8 @@ class JinjaInterpolation(Interpolation):
         except UndefinedError:
             pass
 
-        if default is not None:
-            # If result is empty or resulted in an undefined error, evaluate and return the default string
-            return self._literal_eval(self._eval(default, context), valid_types)
-        raise ValueError(f"Cound not interpolate {input_str} and no default provided to fallback")
+        # If result is empty or resulted in an undefined error, evaluate and return the default string
+        return self._literal_eval(self._eval(default, context), valid_types)
 
     def _literal_eval(self, result: Optional[str], valid_types: Optional[Tuple[Type[Any]]]) -> Any:
         try:

airbyte_cdk/sources/streams/http/http_client.py

@@ -95,6 +95,7 @@ class HttpClient:
     ):
         self._name = name
         self._api_budget: APIBudget = api_budget or APIBudget(policies=[])
+        self._logger = logger
         if session:
             self._session = session
         else:
@@ -108,7 +109,6 @@ class HttpClient:
             )
         if isinstance(authenticator, AuthBase):
             self._session.auth = authenticator
-        self._logger = logger
         self._error_handler = error_handler or HttpStatusErrorHandler(self._logger)
         if backoff_strategy is not None:
             if isinstance(backoff_strategy, list):
@@ -140,10 +140,12 @@ class HttpClient:
             # Use in-memory cache if cache_dir is not set
             # This is a non-obvious interface, but it ensures we don't write sql files when running unit tests
             if cache_dir:
+                self._logger.info(f"Using path {cache_dir} for HTTP cache")  # TODO: remove
                 sqlite_path = str(Path(cache_dir) / self.cache_filename)
             else:
+                self._logger.info("Using memory for cache")  # TODO: remove
                 sqlite_path = "file::memory:?cache=shared"
-            backend = SkipFailureSQLiteCache(sqlite_path)
+            backend = SkipFailureSQLiteCache(self._name, sqlite_path)  # TODO maybe add a busy timeout
             return CachedLimiterSession(
                 sqlite_path, backend=backend, api_budget=self._api_budget, match_headers=True
             )  # type: ignore # there are no typeshed stubs for requests_cache
@@ -541,6 +543,7 @@ class SkipFailureSQLiteDict(requests_cache.backends.sqlite.SQLiteDict):
 class SkipFailureSQLiteCache(requests_cache.backends.sqlite.SQLiteCache):
     def __init__(  # type: ignore  # ignoring as lib is not typed
         self,
+        table_name="response",
         db_path="http_cache",
         serializer=None,
         **kwargs,
@@ -548,11 +551,13 @@ class SkipFailureSQLiteCache(requests_cache.backends.sqlite.SQLiteCache):
         super().__init__(db_path, serializer, **kwargs)
         skwargs = {"serializer": serializer, **kwargs} if serializer else kwargs
         self.responses: requests_cache.backends.sqlite.SQLiteDict = SkipFailureSQLiteDict(
-            db_path, table_name="responses", **skwargs
+            db_path, table_name=table_name, fast_save=True, wal=True, **skwargs
         )
         self.redirects: requests_cache.backends.sqlite.SQLiteDict = SkipFailureSQLiteDict(
             db_path,
-            table_name="redirects",
+            table_name=f"redirects_{table_name}",
+            fast_save=True,
+            wal=True,
             lock=self.responses._lock,
             serializer=None,
             **kwargs,

{airbyte_cdk-6.8.1rc5.dist-info → airbyte_cdk-6.8.1rc7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.8.1rc5
+Version: 6.8.1rc7
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.8.1rc5.dist-info → airbyte_cdk-6.8.1rc7.dist-info}/RECORD

@@ -62,7 +62,7 @@ airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=dAA-UhmMj0WLXCkRQr
 airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
 airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
 airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
-airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=4bpS63fz2K4mLPF33eG1xXgGRzzjjBAvpgDbiDrIs_Q,23549
+airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=ra_VipJiMwzycRAA431LfK4EcjN0q_8vucZnT3JDBRs,23647
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=SX9JjdesN1edN2WVUVMzU_ptqp2QB1OnsnjZ4mwcX7w,2579
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=8VZJP18eJLabSPP1XBSPDaagUBG6q1ynIiPJy3rE2mc,5344
@@ -97,7 +97,7 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py,sha256=UrF
 airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py,sha256=i2L0gREX8nHA-pKokdVqwBf4aJgWP71KOxIABj_DHcY,1857
 airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=LYEZnZ_hB7rvBSZxG9s0RSrzsOkDWbBY0_P6qu5lEfc,3212
 airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=-V5UddGm69UKEB6o_O1EIES9kfY8FV_X4Ji8w1yOuSA,981
-airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=KqsN4LNe47OcIMS6wlOXTlLZmbNniHBwrO6tMfnNzYw,6571
+airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=hhms7WeZuPE54BRanmlCvTaiSaU6sIqUk47N87izTN0,6432
 airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=QgIfSVPHx_MMUCgbQdm-NMpUlp_cpk0OQhoRDFtkrxE,4040
 airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=FEOmFo2mwdpmO8pH9jnw-sUAnijjuigZWYqH_0Gq9oQ,12919
 airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -273,7 +273,7 @@ airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py,sha
 airbyte_cdk/sources/streams/http/error_handlers/response_models.py,sha256=xGIVELBFY0TmH9aUq1ikoqJz8oHLr6di2JLvKWVEO-s,2236
 airbyte_cdk/sources/streams/http/exceptions.py,sha256=njC7MlMJoFYcSGz4mIp6-bqLFTr6vC8ej25X0oSeyjE,1824
 airbyte_cdk/sources/streams/http/http.py,sha256=h0bq4arzMeJsR-5HZNfGYXtZhgVvLbW6myi9fuhMayU,28467
-airbyte_cdk/sources/streams/http/http_client.py,sha256=WxmoFeoISmRnwp2nzfJ0DeD1-05FhL0z8hIni-o7vV8,23067
+airbyte_cdk/sources/streams/http/http_client.py,sha256=2WHxqSbqh0cDbybnpQb9v0oNbUhMT4mGQm8jCzs_n5E,23400
 airbyte_cdk/sources/streams/http/rate_limiting.py,sha256=IwdjrHKUnU97XO4qONgYRv4YYW51xQ8SJm4WLafXDB8,6351
 airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
 airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=nxI94yJ3bGfpDO8RR3QvOJ-PSW0n9CElSAkgl5ae80Y,10321
@@ -330,8 +330,8 @@ airbyte_cdk/utils/slice_hasher.py,sha256=-pHexlNYoWYPnXNH-M7HEbjmeJe9Zk7SJijdQ7d
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=LVc9KbtMeV_z99jWo0Ou8u4l6eBJ0BWNhxj4zrrGKRs,763
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=a6q51tBS3IdtefuOiL1eBwSmnNAXfjFMlMjSIQ_Tl-o,6165
-airbyte_cdk-6.8.1rc5.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.8.1rc5.dist-info/METADATA,sha256=ohQnXZ226OXRoI0p7gVCWAgASzO_WrjRRFahcWv3wa8,13522
-airbyte_cdk-6.8.1rc5.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.8.1rc5.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.8.1rc5.dist-info/RECORD,,
+airbyte_cdk-6.8.1rc7.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.8.1rc7.dist-info/METADATA,sha256=2xNK_1ONYB9ECfDZ259cW74ABrJpQk-y-CMTTXslKno,13522
+airbyte_cdk-6.8.1rc7.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.8.1rc7.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.8.1rc7.dist-info/RECORD,,