airbyte-cdk 6.7.0rc2__py3-none-any.whl → 6.7.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. airbyte_cdk/cli/__init__.py +1 -0
  2. airbyte_cdk/cli/source_declarative_manifest/__init__.py +6 -0
  3. airbyte_cdk/cli/source_declarative_manifest/_run.py +224 -0
  4. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  5. airbyte_cdk/config_observation.py +2 -1
  6. airbyte_cdk/connector.py +1 -0
  7. airbyte_cdk/connector_builder/connector_builder_handler.py +1 -1
  8. airbyte_cdk/connector_builder/main.py +2 -1
  9. airbyte_cdk/destinations/destination.py +2 -1
  10. airbyte_cdk/destinations/vector_db_based/config.py +2 -1
  11. airbyte_cdk/destinations/vector_db_based/document_processor.py +4 -3
  12. airbyte_cdk/destinations/vector_db_based/embedder.py +5 -4
  13. airbyte_cdk/entrypoint.py +3 -2
  14. airbyte_cdk/logger.py +2 -1
  15. airbyte_cdk/models/__init__.py +2 -0
  16. airbyte_cdk/models/airbyte_protocol.py +2 -1
  17. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +3 -3
  18. airbyte_cdk/sources/concurrent_source/concurrent_source.py +1 -1
  19. airbyte_cdk/sources/config.py +2 -1
  20. airbyte_cdk/sources/declarative/auth/jwt.py +1 -0
  21. airbyte_cdk/sources/declarative/auth/oauth.py +1 -0
  22. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +1 -0
  23. airbyte_cdk/sources/declarative/auth/token.py +2 -1
  24. airbyte_cdk/sources/declarative/auth/token_provider.py +3 -2
  25. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +24 -54
  26. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +180 -13
  27. airbyte_cdk/sources/declarative/decoders/json_decoder.py +3 -2
  28. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +1 -0
  29. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +1 -0
  30. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +1 -0
  31. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +1 -0
  32. airbyte_cdk/sources/declarative/extractors/http_selector.py +1 -0
  33. airbyte_cdk/sources/declarative/extractors/record_filter.py +6 -48
  34. airbyte_cdk/sources/declarative/extractors/record_selector.py +32 -4
  35. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +2 -1
  36. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +2 -1
  37. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +5 -2
  38. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +5 -2
  39. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +1 -3
  40. airbyte_cdk/sources/declarative/interpolation/jinja.py +5 -4
  41. airbyte_cdk/sources/declarative/manifest_declarative_source.py +4 -3
  42. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +1 -1
  43. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +122 -0
  44. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +26 -17
  45. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +1 -0
  46. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +1 -0
  47. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +1 -0
  48. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +1 -0
  49. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +1 -0
  50. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +1 -0
  51. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +1 -0
  52. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +1 -0
  53. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +1 -0
  54. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +2 -1
  55. airbyte_cdk/sources/declarative/requesters/http_requester.py +1 -0
  56. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +1 -0
  57. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +1 -0
  58. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +1 -0
  59. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +1 -0
  60. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +1 -0
  61. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +1 -0
  62. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +1 -0
  63. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +9 -3
  64. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +2 -1
  65. airbyte_cdk/sources/declarative/requesters/requester.py +1 -0
  66. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +2 -1
  67. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +12 -4
  68. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +7 -4
  69. airbyte_cdk/sources/declarative/transformations/add_fields.py +1 -0
  70. airbyte_cdk/sources/declarative/transformations/remove_fields.py +1 -0
  71. airbyte_cdk/sources/declarative/yaml_declarative_source.py +1 -0
  72. airbyte_cdk/sources/embedded/tools.py +1 -0
  73. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +2 -1
  74. airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
  75. airbyte_cdk/sources/file_based/config/csv_format.py +2 -1
  76. airbyte_cdk/sources/file_based/config/excel_format.py +2 -1
  77. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +2 -1
  78. airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
  79. airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
  80. airbyte_cdk/sources/file_based/config/unstructured_format.py +2 -1
  81. airbyte_cdk/sources/file_based/file_based_source.py +2 -1
  82. airbyte_cdk/sources/file_based/file_based_stream_reader.py +2 -1
  83. airbyte_cdk/sources/file_based/file_types/avro_parser.py +1 -0
  84. airbyte_cdk/sources/file_based/file_types/csv_parser.py +2 -1
  85. airbyte_cdk/sources/file_based/file_types/excel_parser.py +5 -5
  86. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +2 -1
  87. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +2 -1
  88. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +9 -8
  89. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +2 -1
  90. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +5 -4
  91. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +1 -1
  92. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +1 -1
  93. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +1 -1
  94. airbyte_cdk/sources/http_logger.py +1 -0
  95. airbyte_cdk/sources/streams/call_rate.py +1 -2
  96. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +2 -1
  97. airbyte_cdk/sources/streams/concurrent/adapters.py +8 -4
  98. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +2 -1
  99. airbyte_cdk/sources/streams/concurrent/cursor.py +30 -6
  100. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -1
  101. airbyte_cdk/sources/streams/concurrent/partitions/types.py +1 -1
  102. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +1 -1
  103. airbyte_cdk/sources/streams/core.py +2 -1
  104. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +2 -1
  105. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +1 -0
  106. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +1 -0
  107. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +2 -1
  108. airbyte_cdk/sources/streams/http/http.py +3 -2
  109. airbyte_cdk/sources/streams/http/http_client.py +49 -2
  110. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +2 -1
  111. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +1 -0
  112. airbyte_cdk/sources/types.py +14 -1
  113. airbyte_cdk/sources/utils/schema_helpers.py +3 -2
  114. airbyte_cdk/sql/secrets.py +2 -1
  115. airbyte_cdk/sql/shared/sql_processor.py +8 -6
  116. airbyte_cdk/test/entrypoint_wrapper.py +4 -3
  117. airbyte_cdk/test/mock_http/mocker.py +1 -0
  118. airbyte_cdk/utils/schema_inferrer.py +2 -1
  119. airbyte_cdk/utils/slice_hasher.py +1 -1
  120. airbyte_cdk/utils/traced_exception.py +2 -1
  121. {airbyte_cdk-6.7.0rc2.dist-info → airbyte_cdk-6.7.1.dist-info}/METADATA +11 -3
  122. {airbyte_cdk-6.7.0rc2.dist-info → airbyte_cdk-6.7.1.dist-info}/RECORD +125 -121
  123. airbyte_cdk-6.7.1.dist-info/entry_points.txt +3 -0
  124. airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -35
  125. {airbyte_cdk-6.7.0rc2.dist-info → airbyte_cdk-6.7.1.dist-info}/LICENSE.txt +0 -0
  126. {airbyte_cdk-6.7.0rc2.dist-info → airbyte_cdk-6.7.1.dist-info}/WHEEL +0 -0
@@ -7,6 +7,8 @@ from dataclasses import InitVar, dataclass, field
7
7
  from datetime import timedelta
8
8
  from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Union
9
9
 
10
+ from isodate import Duration, duration_isoformat, parse_duration
11
+
10
12
  from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
11
13
  from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
12
14
  from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
@@ -19,7 +21,6 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
19
21
  )
20
22
  from airbyte_cdk.sources.message import MessageRepository
21
23
  from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
22
- from isodate import Duration, duration_isoformat, parse_duration
23
24
 
24
25
 
25
26
  @dataclass
@@ -340,8 +340,11 @@ class GlobalSubstreamCursor(DeclarativeCursor):
340
340
  @staticmethod
341
341
  def _convert_record_to_cursor_record(record: Record) -> Record:
342
342
  return Record(
343
- record.data,
344
- StreamSlice(partition={}, cursor_slice=record.associated_slice.cursor_slice)
343
+ data=record.data,
344
+ stream_name=record.stream_name,
345
+ associated_slice=StreamSlice(
346
+ partition={}, cursor_slice=record.associated_slice.cursor_slice
347
+ )
345
348
  if record.associated_slice
346
349
  else None,
347
350
  )
@@ -325,8 +325,11 @@ class PerPartitionCursor(DeclarativeCursor):
325
325
  @staticmethod
326
326
  def _convert_record_to_cursor_record(record: Record) -> Record:
327
327
  return Record(
328
- record.data,
329
- StreamSlice(partition={}, cursor_slice=record.associated_slice.cursor_slice)
328
+ data=record.data,
329
+ stream_name=record.stream_name,
330
+ associated_slice=StreamSlice(
331
+ partition={}, cursor_slice=record.associated_slice.cursor_slice
332
+ )
330
333
  if record.associated_slice
331
334
  else None,
332
335
  )
@@ -194,9 +194,7 @@ class PerPartitionWithGlobalCursor(DeclarativeCursor):
194
194
  )
195
195
 
196
196
  def should_be_synced(self, record: Record) -> bool:
197
- return self._global_cursor.should_be_synced(
198
- record
199
- ) or self._per_partition_cursor.should_be_synced(record)
197
+ return self._get_active_cursor().should_be_synced(record)
200
198
 
201
199
  def is_greater_than_or_equal(self, first: Record, second: Record) -> bool:
202
200
  return self._global_cursor.is_greater_than_or_equal(first, second)
@@ -6,15 +6,16 @@ import ast
6
6
  from functools import cache
7
7
  from typing import Any, Mapping, Optional, Tuple, Type
8
8
 
9
- from airbyte_cdk.sources.declarative.interpolation.filters import filters
10
- from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
11
- from airbyte_cdk.sources.declarative.interpolation.macros import macros
12
- from airbyte_cdk.sources.types import Config
13
9
  from jinja2 import meta
14
10
  from jinja2.environment import Template
15
11
  from jinja2.exceptions import UndefinedError
16
12
  from jinja2.sandbox import SandboxedEnvironment
17
13
 
14
+ from airbyte_cdk.sources.declarative.interpolation.filters import filters
15
+ from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
16
+ from airbyte_cdk.sources.declarative.interpolation.macros import macros
17
+ from airbyte_cdk.sources.types import Config
18
+
18
19
 
19
20
  class StreamPartitionAccessEnvironment(SandboxedEnvironment):
20
21
  """
@@ -8,9 +8,12 @@ import pkgutil
8
8
  from copy import deepcopy
9
9
  from importlib import metadata
10
10
  from typing import Any, Dict, Iterator, List, Mapping, Optional
11
- from packaging.version import Version, InvalidVersion
12
11
 
13
12
  import yaml
13
+ from jsonschema.exceptions import ValidationError
14
+ from jsonschema.validators import validate
15
+ from packaging.version import InvalidVersion, Version
16
+
14
17
  from airbyte_cdk.models import (
15
18
  AirbyteConnectionStatus,
16
19
  AirbyteMessage,
@@ -44,8 +47,6 @@ from airbyte_cdk.sources.utils.slice_logger import (
44
47
  DebugSliceLogger,
45
48
  SliceLogger,
46
49
  )
47
- from jsonschema.exceptions import ValidationError
48
- from jsonschema.validators import validate
49
50
 
50
51
 
51
52
  class ManifestDeclarativeSource(DeclarativeSource):
@@ -5,9 +5,9 @@ from typing import Any, Mapping
5
5
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
6
6
  from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
7
7
  from airbyte_cdk.sources.declarative.models import (
8
+ CustomIncrementalSync,
8
9
  DatetimeBasedCursor,
9
10
  SubstreamPartitionRouter,
10
- CustomIncrementalSync,
11
11
  )
12
12
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import ParentStreamConfig
13
13
 
@@ -748,6 +748,123 @@ class NoPagination(BaseModel):
748
748
  type: Literal["NoPagination"]
749
749
 
750
750
 
751
+ class State(BaseModel):
752
+ class Config:
753
+ extra = Extra.allow
754
+
755
+ min: int
756
+ max: int
757
+
758
+
759
+ class OauthConnectorInputSpecification(BaseModel):
760
+ class Config:
761
+ extra = Extra.allow
762
+
763
+ consent_url: str = Field(
764
+ ...,
765
+ description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
766
+ examples=[
767
+ {
768
+ "consent_url": "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}"
769
+ },
770
+ {
771
+ "consent_url": "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}"
772
+ },
773
+ ],
774
+ title="DeclarativeOAuth Consent URL",
775
+ )
776
+ scope: Optional[str] = Field(
777
+ None,
778
+ description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.",
779
+ examples=[{"scope": "user:read user:read_orders workspaces:read"}],
780
+ title="(Optional) DeclarativeOAuth Scope",
781
+ )
782
+ access_token_url: str = Field(
783
+ ...,
784
+ description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
785
+ examples=[
786
+ {
787
+ "access_token_url": "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
788
+ }
789
+ ],
790
+ title="DeclarativeOAuth Access Token URL",
791
+ )
792
+ access_token_headers: Optional[Dict[str, Any]] = Field(
793
+ None,
794
+ description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
795
+ examples=[
796
+ {
797
+ "access_token_headers": {
798
+ "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"
799
+ }
800
+ }
801
+ ],
802
+ title="(Optional) DeclarativeOAuth Access Token Headers",
803
+ )
804
+ access_token_params: Optional[Dict[str, Any]] = Field(
805
+ None,
806
+ description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
807
+ examples=[
808
+ {
809
+ "access_token_params": {
810
+ "{auth_code_key}": "{{auth_code_key}}",
811
+ "{client_id_key}": "{{client_id_key}}",
812
+ "{client_secret_key}": "{{client_secret_key}}",
813
+ }
814
+ }
815
+ ],
816
+ title="(Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)",
817
+ )
818
+ extract_output: List[str] = Field(
819
+ ...,
820
+ description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config. ",
821
+ examples=[{"extract_output": ["access_token", "refresh_token", "other_field"]}],
822
+ title="DeclarativeOAuth Extract Output",
823
+ )
824
+ state: Optional[State] = Field(
825
+ None,
826
+ description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity. ",
827
+ examples=[{"state": {"min": 7, "max": 128}}],
828
+ title="(Optional) DeclarativeOAuth Configurable State Query Param",
829
+ )
830
+ client_id_key: Optional[str] = Field(
831
+ None,
832
+ description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.",
833
+ examples=[{"client_id_key": "my_custom_client_id_key_name"}],
834
+ title="(Optional) DeclarativeOAuth Client ID Key Override",
835
+ )
836
+ client_secret_key: Optional[str] = Field(
837
+ None,
838
+ description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.",
839
+ examples=[{"client_secret_key": "my_custom_client_secret_key_name"}],
840
+ title="(Optional) DeclarativeOAuth Client Secret Key Override",
841
+ )
842
+ scope_key: Optional[str] = Field(
843
+ None,
844
+ description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.",
845
+ examples=[{"scope_key": "my_custom_scope_key_key_name"}],
846
+ title="(Optional) DeclarativeOAuth Scope Key Override",
847
+ )
848
+ state_key: Optional[str] = Field(
849
+ None,
850
+ description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider. ",
851
+ examples=[{"state_key": "my_custom_state_key_key_name"}],
852
+ title="(Optional) DeclarativeOAuth State Key Override",
853
+ )
854
+ auth_code_key: Optional[str] = Field(
855
+ None,
856
+ description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider. ",
857
+ examples=[{"auth_code_key": "my_custom_auth_code_key_name"}],
858
+ title="(Optional) DeclarativeOAuth Auth Code Key Override",
859
+ )
860
+ redirect_uri_key: Optional[str] = Field(
861
+ None,
862
+ description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.",
863
+ examples=[{"redirect_uri_key": "my_custom_redirect_uri_key_name"}],
864
+ title="(Optional) DeclarativeOAuth Redirect URI Key Override",
865
+ )
866
+
867
+
751
868
  class OAuthConfigSpecification(BaseModel):
752
869
  class Config:
753
870
  extra = Extra.allow
@@ -766,6 +883,11 @@ class OAuthConfigSpecification(BaseModel):
766
883
  ],
767
884
  title="OAuth user input",
768
885
  )
886
+ oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
887
+ None,
888
+ description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{my_var}`.\n- The nested resolution variables like `{{my_nested_var}}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}\n + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}\n + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}\n + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{auth_code_key}": "{{auth_code_key}}",\n "{client_id_key}": "{{client_id_key}}",\n "{client_secret_key}": "{{client_secret_key}}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }',
889
+ title="DeclarativeOAuth Connector Specification",
890
+ )
769
891
  complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
770
892
  None,
771
893
  description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }",
@@ -17,6 +17,7 @@ from typing import (
17
17
  Mapping,
18
18
  MutableMapping,
19
19
  Optional,
20
+ Tuple,
20
21
  Type,
21
22
  Union,
22
23
  get_args,
@@ -24,6 +25,9 @@ from typing import (
24
25
  get_type_hints,
25
26
  )
26
27
 
28
+ from isodate import parse_duration
29
+ from pydantic.v1 import BaseModel
30
+
27
31
  from airbyte_cdk.models import FailureType, Level
28
32
  from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
29
33
  from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator
@@ -366,8 +370,6 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_sta
366
370
  from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction
367
371
  from airbyte_cdk.sources.types import Config
368
372
  from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
369
- from isodate import parse_duration
370
- from pydantic.v1 import BaseModel
371
373
 
372
374
  ComponentDefinition = Mapping[str, Any]
373
375
 
@@ -752,7 +754,7 @@ class ModelToComponentFactory:
752
754
  config: Config,
753
755
  stream_state: MutableMapping[str, Any],
754
756
  **kwargs: Any,
755
- ) -> ConcurrentCursor:
757
+ ) -> Tuple[ConcurrentCursor, DateTimeStreamStateConverter]:
756
758
  component_type = component_definition.get("type")
757
759
  if component_definition.get("type") != model_type.__name__:
758
760
  raise ValueError(
@@ -883,20 +885,23 @@ class ModelToComponentFactory:
883
885
  if evaluated_step:
884
886
  step_length = parse_duration(evaluated_step)
885
887
 
886
- return ConcurrentCursor(
887
- stream_name=stream_name,
888
- stream_namespace=stream_namespace,
889
- stream_state=stream_state,
890
- message_repository=self._message_repository, # type: ignore # message_repository is always instantiated with a value by factory
891
- connector_state_manager=state_manager,
892
- connector_state_converter=connector_state_converter,
893
- cursor_field=cursor_field,
894
- slice_boundary_fields=slice_boundary_fields,
895
- start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice
896
- end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice
897
- lookback_window=lookback_window,
898
- slice_range=step_length,
899
- cursor_granularity=cursor_granularity,
888
+ return (
889
+ ConcurrentCursor(
890
+ stream_name=stream_name,
891
+ stream_namespace=stream_namespace,
892
+ stream_state=stream_state,
893
+ message_repository=self._message_repository, # type: ignore # message_repository is always instantiated with a value by factory
894
+ connector_state_manager=state_manager,
895
+ connector_state_converter=connector_state_converter,
896
+ cursor_field=cursor_field,
897
+ slice_boundary_fields=slice_boundary_fields,
898
+ start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice
899
+ end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice
900
+ lookback_window=lookback_window,
901
+ slice_range=step_length,
902
+ cursor_granularity=cursor_granularity,
903
+ ),
904
+ connector_state_converter,
900
905
  )
901
906
 
902
907
  @staticmethod
@@ -1776,6 +1781,7 @@ class ModelToComponentFactory:
1776
1781
  self,
1777
1782
  model: RecordSelectorModel,
1778
1783
  config: Config,
1784
+ name: str,
1779
1785
  *,
1780
1786
  transformations: List[RecordTransformation],
1781
1787
  decoder: Optional[Decoder] = None,
@@ -1806,6 +1812,7 @@ class ModelToComponentFactory:
1806
1812
 
1807
1813
  return RecordSelector(
1808
1814
  extractor=extractor,
1815
+ name=name,
1809
1816
  config=config,
1810
1817
  record_filter=record_filter,
1811
1818
  transformations=transformations,
@@ -1876,6 +1883,7 @@ class ModelToComponentFactory:
1876
1883
  )
1877
1884
  record_selector = self._create_component_from_model(
1878
1885
  model=model.record_selector,
1886
+ name=name,
1879
1887
  config=config,
1880
1888
  decoder=decoder,
1881
1889
  transformations=transformations,
@@ -2030,6 +2038,7 @@ class ModelToComponentFactory:
2030
2038
  requester=download_requester,
2031
2039
  record_selector=RecordSelector(
2032
2040
  extractor=ResponseToFileExtractor(),
2041
+ name=name,
2033
2042
  record_filter=None,
2034
2043
  transformations=[],
2035
2044
  schema_normalization=TypeTransformer(TransformConfig.NoTransform),
@@ -7,6 +7,7 @@ from dataclasses import InitVar, dataclass
7
7
  from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
8
8
 
9
9
  import dpath
10
+
10
11
  from airbyte_cdk.models import AirbyteMessage
11
12
  from airbyte_cdk.models import Type as MessageType
12
13
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
+
9
10
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
10
11
  from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
11
12
  from airbyte_cdk.sources.types import Config
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
+
9
10
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
10
11
  from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
11
12
  from airbyte_cdk.sources.types import Config
@@ -7,6 +7,7 @@ from dataclasses import InitVar, dataclass
7
7
  from typing import Any, Mapping, Optional, Union
8
8
 
9
9
  import requests
10
+
10
11
  from airbyte_cdk.models import FailureType
11
12
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
12
13
  from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
@@ -9,6 +9,7 @@ from dataclasses import InitVar, dataclass
9
9
  from typing import Any, Mapping, Optional, Union
10
10
 
11
11
  import requests
12
+
12
13
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
13
14
  from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
14
15
  get_numeric_value_from_header,
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, List, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
+
9
10
  from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
10
11
  from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
11
12
  ErrorResolution,
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, List, Mapping, MutableMapping, Optional, Union
7
7
 
8
8
  import requests
9
+
9
10
  from airbyte_cdk.sources.declarative.requesters.error_handlers.default_http_response_filter import (
10
11
  DefaultHttpResponseFilter,
11
12
  )
@@ -5,6 +5,7 @@
5
5
  from typing import Optional, Union
6
6
 
7
7
  import requests
8
+
8
9
  from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import (
9
10
  HttpResponseFilter,
10
11
  )
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, Optional, Set, Union
7
7
 
8
8
  import requests
9
+
9
10
  from airbyte_cdk.models import FailureType
10
11
  from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
11
12
  from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
@@ -6,6 +6,8 @@ from datetime import timedelta
6
6
  from typing import Any, Dict, Iterable, Mapping, Optional
7
7
 
8
8
  import requests
9
+ from requests import Response
10
+
9
11
  from airbyte_cdk import AirbyteMessage
10
12
  from airbyte_cdk.logger import lazy_log
11
13
  from airbyte_cdk.models import FailureType, Type
@@ -23,7 +25,6 @@ from airbyte_cdk.sources.declarative.requesters.requester import Requester
23
25
  from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
24
26
  from airbyte_cdk.sources.types import Record, StreamSlice
25
27
  from airbyte_cdk.utils import AirbyteTracedException
26
- from requests import Response
27
28
 
28
29
  LOGGER = logging.getLogger("airbyte")
29
30
 
@@ -9,6 +9,7 @@ from typing import Any, Callable, Mapping, MutableMapping, Optional, Union
9
9
  from urllib.parse import urljoin
10
10
 
11
11
  import requests
12
+
12
13
  from airbyte_cdk.sources.declarative.auth.declarative_authenticator import (
13
14
  DeclarativeAuthenticator,
14
15
  NoAuth,
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, Mapping, MutableMapping, Optional, Union
7
7
 
8
8
  import requests
9
+
9
10
  from airbyte_cdk.sources.declarative.decoders import (
10
11
  Decoder,
11
12
  JsonDecoder,
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, MutableMapping, Optional, Union
7
7
 
8
8
  import requests
9
+
9
10
  from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator
10
11
  from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
11
12
 
@@ -7,6 +7,7 @@ from dataclasses import dataclass
7
7
  from typing import Any, Mapping, Optional
8
8
 
9
9
  import requests
10
+
10
11
  from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
11
12
  RequestOptionsProvider,
12
13
  )
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, Dict, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
+
9
10
  from airbyte_cdk.sources.declarative.decoders import (
10
11
  Decoder,
11
12
  JsonDecoder,
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
+
9
10
  from airbyte_cdk.sources.declarative.decoders import (
10
11
  Decoder,
11
12
  JsonDecoder,
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
+
9
10
  from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
10
11
  from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
11
12
  PaginationStrategy,
@@ -7,6 +7,7 @@ from dataclasses import dataclass
7
7
  from typing import Any, Optional
8
8
 
9
9
  import requests
10
+
10
11
  from airbyte_cdk.sources.types import Record
11
12
 
12
13
 
@@ -6,10 +6,12 @@ from abc import ABC, abstractmethod
6
6
  from typing import Any, Optional
7
7
 
8
8
  import requests
9
+
9
10
  from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
10
11
  from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
11
12
  PaginationStrategy,
12
13
  )
14
+ from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor
13
15
  from airbyte_cdk.sources.types import Record
14
16
 
15
17
 
@@ -25,7 +27,11 @@ class PaginationStopCondition(ABC):
25
27
 
26
28
 
27
29
  class CursorStopCondition(PaginationStopCondition):
28
- def __init__(self, cursor: DeclarativeCursor):
30
+ def __init__(
31
+ self,
32
+ cursor: DeclarativeCursor
33
+ | ConcurrentCursor, # migrate to use both old and concurrent versions
34
+ ):
29
35
  self._cursor = cursor
30
36
 
31
37
  def is_met(self, record: Record) -> bool:
@@ -46,8 +52,8 @@ class StopConditionPaginationStrategyDecorator(PaginationStrategy):
46
52
  return None
47
53
  return self._delegate.next_page_token(response, last_page_size, last_record)
48
54
 
49
- def reset(self) -> None:
50
- self._delegate.reset()
55
+ def reset(self, reset_value: Optional[Any] = None) -> None:
56
+ self._delegate.reset(reset_value)
51
57
 
52
58
  def get_page_size(self) -> Optional[int]:
53
59
  return self._delegate.get_page_size()
@@ -5,6 +5,8 @@
5
5
  from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, Mapping, MutableMapping, Optional, Union
7
7
 
8
+ from deprecated import deprecated
9
+
8
10
  from airbyte_cdk.sources.declarative.interpolation.interpolated_nested_mapping import NestedMapping
9
11
  from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_nested_request_input_provider import (
10
12
  InterpolatedNestedRequestInputProvider,
@@ -17,7 +19,6 @@ from airbyte_cdk.sources.declarative.requesters.request_options.request_options_
17
19
  )
18
20
  from airbyte_cdk.sources.source import ExperimentalClassWarning
19
21
  from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
20
- from deprecated import deprecated
21
22
 
22
23
  RequestInput = Union[str, Mapping[str, str]]
23
24
  ValidRequestTypes = (str, list)
@@ -7,6 +7,7 @@ from enum import Enum
7
7
  from typing import Any, Callable, Mapping, MutableMapping, Optional, Union
8
8
 
9
9
  import requests
10
+
10
11
  from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
11
12
  from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
12
13
  RequestOptionsProvider,
@@ -4,6 +4,8 @@
4
4
  from dataclasses import InitVar, dataclass, field
5
5
  from typing import Any, Callable, Iterable, Mapping, Optional
6
6
 
7
+ from deprecated.classic import deprecated
8
+
7
9
  from airbyte_cdk.models import FailureType
8
10
  from airbyte_cdk.sources.declarative.async_job.job_orchestrator import (
9
11
  AsyncJobOrchestrator,
@@ -17,7 +19,6 @@ from airbyte_cdk.sources.source import ExperimentalClassWarning
17
19
  from airbyte_cdk.sources.streams.core import StreamData
18
20
  from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
19
21
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException
20
- from deprecated.classic import deprecated
21
22
 
22
23
 
23
24
  @deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)