airbyte-cdk 6.7.1.dev0__py3-none-any.whl → 6.7.1rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. airbyte_cdk/cli/source_declarative_manifest/_run.py +1 -2
  2. airbyte_cdk/config_observation.py +1 -2
  3. airbyte_cdk/connector.py +0 -1
  4. airbyte_cdk/connector_builder/connector_builder_handler.py +1 -1
  5. airbyte_cdk/connector_builder/main.py +1 -2
  6. airbyte_cdk/destinations/destination.py +1 -2
  7. airbyte_cdk/destinations/vector_db_based/config.py +1 -2
  8. airbyte_cdk/destinations/vector_db_based/document_processor.py +3 -4
  9. airbyte_cdk/destinations/vector_db_based/embedder.py +4 -5
  10. airbyte_cdk/entrypoint.py +2 -3
  11. airbyte_cdk/logger.py +1 -2
  12. airbyte_cdk/models/__init__.py +0 -2
  13. airbyte_cdk/models/airbyte_protocol.py +1 -2
  14. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +3 -3
  15. airbyte_cdk/sources/concurrent_source/concurrent_source.py +1 -1
  16. airbyte_cdk/sources/config.py +1 -2
  17. airbyte_cdk/sources/declarative/auth/jwt.py +0 -1
  18. airbyte_cdk/sources/declarative/auth/oauth.py +0 -1
  19. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +0 -1
  20. airbyte_cdk/sources/declarative/auth/token.py +1 -2
  21. airbyte_cdk/sources/declarative/auth/token_provider.py +2 -3
  22. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +8 -66
  23. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +0 -167
  24. airbyte_cdk/sources/declarative/decoders/json_decoder.py +2 -3
  25. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +0 -1
  26. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +0 -1
  27. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +0 -1
  28. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +0 -1
  29. airbyte_cdk/sources/declarative/extractors/http_selector.py +0 -1
  30. airbyte_cdk/sources/declarative/extractors/record_filter.py +48 -6
  31. airbyte_cdk/sources/declarative/extractors/record_selector.py +4 -32
  32. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +1 -2
  33. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +1 -2
  34. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +2 -5
  35. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +2 -5
  36. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +3 -1
  37. airbyte_cdk/sources/declarative/interpolation/jinja.py +4 -5
  38. airbyte_cdk/sources/declarative/manifest_declarative_source.py +3 -4
  39. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +1 -1
  40. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +0 -122
  41. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +2 -11
  42. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +0 -1
  43. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +0 -1
  44. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +0 -1
  45. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +0 -1
  46. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +0 -1
  47. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +0 -1
  48. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +0 -1
  49. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +0 -1
  50. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +0 -1
  51. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +1 -2
  52. airbyte_cdk/sources/declarative/requesters/http_requester.py +0 -1
  53. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +0 -1
  54. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +0 -1
  55. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +0 -1
  56. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +0 -1
  57. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +0 -1
  58. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +0 -1
  59. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +0 -1
  60. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +3 -9
  61. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +1 -2
  62. airbyte_cdk/sources/declarative/requesters/requester.py +0 -1
  63. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +1 -2
  64. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +7 -12
  65. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +4 -7
  66. airbyte_cdk/sources/declarative/transformations/add_fields.py +0 -1
  67. airbyte_cdk/sources/declarative/transformations/remove_fields.py +0 -1
  68. airbyte_cdk/sources/declarative/yaml_declarative_source.py +0 -1
  69. airbyte_cdk/sources/embedded/tools.py +0 -1
  70. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +1 -2
  71. airbyte_cdk/sources/file_based/config/avro_format.py +1 -2
  72. airbyte_cdk/sources/file_based/config/csv_format.py +1 -2
  73. airbyte_cdk/sources/file_based/config/excel_format.py +1 -2
  74. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +1 -2
  75. airbyte_cdk/sources/file_based/config/jsonl_format.py +1 -2
  76. airbyte_cdk/sources/file_based/config/parquet_format.py +1 -2
  77. airbyte_cdk/sources/file_based/config/unstructured_format.py +1 -2
  78. airbyte_cdk/sources/file_based/file_based_source.py +1 -2
  79. airbyte_cdk/sources/file_based/file_based_stream_reader.py +1 -2
  80. airbyte_cdk/sources/file_based/file_types/avro_parser.py +0 -1
  81. airbyte_cdk/sources/file_based/file_types/csv_parser.py +1 -2
  82. airbyte_cdk/sources/file_based/file_types/excel_parser.py +5 -5
  83. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +1 -2
  84. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +1 -2
  85. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +8 -9
  86. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +1 -2
  87. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +4 -5
  88. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +1 -1
  89. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +1 -1
  90. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +1 -1
  91. airbyte_cdk/sources/http_logger.py +0 -1
  92. airbyte_cdk/sources/streams/call_rate.py +2 -1
  93. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +1 -2
  94. airbyte_cdk/sources/streams/concurrent/adapters.py +4 -8
  95. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +1 -2
  96. airbyte_cdk/sources/streams/concurrent/cursor.py +6 -30
  97. airbyte_cdk/sources/streams/concurrent/default_stream.py +0 -1
  98. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -1
  99. airbyte_cdk/sources/streams/concurrent/partitions/record.py +35 -0
  100. airbyte_cdk/sources/streams/concurrent/partitions/types.py +1 -1
  101. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +1 -1
  102. airbyte_cdk/sources/streams/core.py +1 -2
  103. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +1 -2
  104. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +0 -1
  105. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +0 -1
  106. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +1 -2
  107. airbyte_cdk/sources/streams/http/http.py +2 -3
  108. airbyte_cdk/sources/streams/http/http_client.py +9 -48
  109. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +1 -2
  110. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +0 -1
  111. airbyte_cdk/sources/types.py +1 -14
  112. airbyte_cdk/sources/utils/schema_helpers.py +2 -3
  113. airbyte_cdk/sql/secrets.py +1 -2
  114. airbyte_cdk/sql/shared/sql_processor.py +6 -8
  115. airbyte_cdk/test/entrypoint_wrapper.py +3 -4
  116. airbyte_cdk/test/mock_http/mocker.py +0 -1
  117. airbyte_cdk/utils/schema_inferrer.py +1 -2
  118. airbyte_cdk/utils/slice_hasher.py +1 -1
  119. airbyte_cdk/utils/traced_exception.py +1 -2
  120. {airbyte_cdk-6.7.1.dev0.dist-info → airbyte_cdk-6.7.1rc2.dist-info}/METADATA +2 -9
  121. {airbyte_cdk-6.7.1.dev0.dist-info → airbyte_cdk-6.7.1rc2.dist-info}/RECORD +124 -123
  122. {airbyte_cdk-6.7.1.dev0.dist-info → airbyte_cdk-6.7.1rc2.dist-info}/LICENSE.txt +0 -0
  123. {airbyte_cdk-6.7.1.dev0.dist-info → airbyte_cdk-6.7.1rc2.dist-info}/WHEEL +0 -0
  124. {airbyte_cdk-6.7.1.dev0.dist-info → airbyte_cdk-6.7.1rc2.dist-info}/entry_points.txt +0 -0
@@ -25,8 +25,6 @@ from datetime import datetime
25
25
  from pathlib import Path
26
26
  from typing import Any, cast
27
27
 
28
- from orjson import orjson
29
-
30
28
  from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
31
29
  from airbyte_cdk.models import (
32
30
  AirbyteErrorTraceMessage,
@@ -44,6 +42,7 @@ from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
44
42
  )
45
43
  from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
46
44
  from airbyte_cdk.sources.source import TState
45
+ from orjson import orjson
47
46
 
48
47
 
49
48
  class SourceLocalYaml(YamlDeclarativeSource):
@@ -10,8 +10,6 @@ import time
10
10
  from copy import copy
11
11
  from typing import Any, List, MutableMapping
12
12
 
13
- from orjson import orjson
14
-
15
13
  from airbyte_cdk.models import (
16
14
  AirbyteControlConnectorConfigMessage,
17
15
  AirbyteControlMessage,
@@ -20,6 +18,7 @@ from airbyte_cdk.models import (
20
18
  OrchestratorType,
21
19
  Type,
22
20
  )
21
+ from orjson import orjson
23
22
 
24
23
 
25
24
  class ObservedDict(dict): # type: ignore # disallow_any_generics is set to True, and dict is equivalent to dict[Any]
airbyte_cdk/connector.py CHANGED
@@ -11,7 +11,6 @@ from abc import ABC, abstractmethod
11
11
  from typing import Any, Generic, Mapping, Optional, Protocol, TypeVar
12
12
 
13
13
  import yaml
14
-
15
14
  from airbyte_cdk.models import (
16
15
  AirbyteConnectionStatus,
17
16
  ConnectorSpecification,
@@ -12,8 +12,8 @@ from airbyte_cdk.models import (
12
12
  AirbyteRecordMessage,
13
13
  AirbyteStateMessage,
14
14
  ConfiguredAirbyteCatalog,
15
- Type,
16
15
  )
16
+ from airbyte_cdk.models import Type
17
17
  from airbyte_cdk.models import Type as MessageType
18
18
  from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
19
19
  from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
@@ -6,8 +6,6 @@
6
6
  import sys
7
7
  from typing import Any, List, Mapping, Optional, Tuple
8
8
 
9
- from orjson import orjson
10
-
11
9
  from airbyte_cdk.connector import BaseConnector
12
10
  from airbyte_cdk.connector_builder.connector_builder_handler import (
13
11
  TestReadLimits,
@@ -27,6 +25,7 @@ from airbyte_cdk.models import (
27
25
  from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
28
26
  from airbyte_cdk.sources.source import Source
29
27
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException
28
+ from orjson import orjson
30
29
 
31
30
 
32
31
  def get_config_and_catalog_from_args(
@@ -9,8 +9,6 @@ import sys
9
9
  from abc import ABC, abstractmethod
10
10
  from typing import Any, Iterable, List, Mapping
11
11
 
12
- from orjson import orjson
13
-
14
12
  from airbyte_cdk.connector import Connector
15
13
  from airbyte_cdk.exception_handler import init_uncaught_exception_handler
16
14
  from airbyte_cdk.models import (
@@ -22,6 +20,7 @@ from airbyte_cdk.models import (
22
20
  )
23
21
  from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
24
22
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException
23
+ from orjson import orjson
25
24
 
26
25
  logger = logging.getLogger("airbyte")
27
26
 
@@ -5,10 +5,9 @@
5
5
  from typing import Any, Dict, List, Literal, Optional, Union
6
6
 
7
7
  import dpath
8
- from pydantic.v1 import BaseModel, Field
9
-
10
8
  from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
11
9
  from airbyte_cdk.utils.spec_schema_transformations import resolve_refs
10
+ from pydantic.v1 import BaseModel, Field
12
11
 
13
12
 
14
13
  class SeparatorSplitterConfigModel(BaseModel):
@@ -8,10 +8,6 @@ from dataclasses import dataclass
8
8
  from typing import Any, Dict, List, Mapping, Optional, Tuple
9
9
 
10
10
  import dpath
11
- from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
12
- from langchain.utils import stringify_dict
13
- from langchain_core.documents.base import Document
14
-
15
11
  from airbyte_cdk.destinations.vector_db_based.config import (
16
12
  ProcessingConfigModel,
17
13
  SeparatorSplitterConfigModel,
@@ -25,6 +21,9 @@ from airbyte_cdk.models import (
25
21
  DestinationSyncMode,
26
22
  )
27
23
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType
24
+ from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
25
+ from langchain.utils import stringify_dict
26
+ from langchain_core.documents.base import Document
28
27
 
29
28
  METADATA_STREAM_FIELD = "_ab_stream"
30
29
  METADATA_RECORD_ID_FIELD = "_ab_record_id"
@@ -7,11 +7,6 @@ from abc import ABC, abstractmethod
7
7
  from dataclasses import dataclass
8
8
  from typing import List, Optional, Union, cast
9
9
 
10
- from langchain.embeddings.cohere import CohereEmbeddings
11
- from langchain.embeddings.fake import FakeEmbeddings
12
- from langchain.embeddings.localai import LocalAIEmbeddings
13
- from langchain.embeddings.openai import OpenAIEmbeddings
14
-
15
10
  from airbyte_cdk.destinations.vector_db_based.config import (
16
11
  AzureOpenAIEmbeddingConfigModel,
17
12
  CohereEmbeddingConfigModel,
@@ -24,6 +19,10 @@ from airbyte_cdk.destinations.vector_db_based.config import (
24
19
  from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, format_exception
25
20
  from airbyte_cdk.models import AirbyteRecordMessage
26
21
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType
22
+ from langchain.embeddings.cohere import CohereEmbeddings
23
+ from langchain.embeddings.fake import FakeEmbeddings
24
+ from langchain.embeddings.localai import LocalAIEmbeddings
25
+ from langchain.embeddings.openai import OpenAIEmbeddings
27
26
 
28
27
 
29
28
  @dataclass
airbyte_cdk/entrypoint.py CHANGED
@@ -16,9 +16,6 @@ from typing import Any, DefaultDict, Iterable, List, Mapping, Optional
16
16
  from urllib.parse import urlparse
17
17
 
18
18
  import requests
19
- from orjson import orjson
20
- from requests import PreparedRequest, Response, Session
21
-
22
19
  from airbyte_cdk.connector import TConfig
23
20
  from airbyte_cdk.exception_handler import init_uncaught_exception_handler
24
21
  from airbyte_cdk.logger import init_logger
@@ -41,6 +38,8 @@ from airbyte_cdk.utils import is_cloud_environment, message_utils
41
38
  from airbyte_cdk.utils.airbyte_secrets_utils import get_secrets, update_secrets
42
39
  from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH
43
40
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException
41
+ from orjson import orjson
42
+ from requests import PreparedRequest, Response, Session
44
43
 
45
44
  logger = init_logger("airbyte")
46
45
 
airbyte_cdk/logger.py CHANGED
@@ -7,8 +7,6 @@ import logging
7
7
  import logging.config
8
8
  from typing import Any, Callable, Mapping, Optional, Tuple
9
9
 
10
- from orjson import orjson
11
-
12
10
  from airbyte_cdk.models import (
13
11
  AirbyteLogMessage,
14
12
  AirbyteMessage,
@@ -17,6 +15,7 @@ from airbyte_cdk.models import (
17
15
  Type,
18
16
  )
19
17
  from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
18
+ from orjson import orjson
20
19
 
21
20
  LOGGING_CONFIG = {
22
21
  "version": 1,
@@ -39,9 +39,7 @@ from .airbyte_protocol import (
39
39
  FailureType,
40
40
  Level,
41
41
  OAuthConfigSpecification,
42
- OauthConnectorInputSpecification,
43
42
  OrchestratorType,
44
- State,
45
43
  Status,
46
44
  StreamDescriptor,
47
45
  SyncMode,
@@ -5,11 +5,10 @@
5
5
  from dataclasses import InitVar, dataclass
6
6
  from typing import Annotated, Any, Dict, List, Mapping, Optional, Union
7
7
 
8
+ from airbyte_cdk.models.file_transfer_record_message import AirbyteFileTransferRecordMessage
8
9
  from airbyte_protocol_dataclasses.models import * # noqa: F403 # Allow '*'
9
10
  from serpyco_rs.metadata import Alias
10
11
 
11
- from airbyte_cdk.models.file_transfer_record_message import AirbyteFileTransferRecordMessage
12
-
13
12
  # ruff: noqa: F405 # ignore fuzzy import issues with 'import *'
14
13
 
15
14
 
@@ -17,8 +17,8 @@ from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStrea
17
17
  from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer
18
18
  from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader
19
19
  from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
20
+ from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
20
21
  from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel
21
- from airbyte_cdk.sources.types import Record
22
22
  from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
23
23
  from airbyte_cdk.sources.utils.slice_logger import SliceLogger
24
24
  from airbyte_cdk.utils import AirbyteTracedException
@@ -147,11 +147,11 @@ class ConcurrentReadProcessor:
147
147
  # AbstractStreams are expected to return data as they are expected.
148
148
  # Any transformation on the data should be done before reaching this point
149
149
  message = stream_data_to_airbyte_message(
150
- stream_name=record.stream_name,
150
+ stream_name=record.partition.stream_name(),
151
151
  data_or_message=record.data,
152
152
  is_file_transfer_message=record.is_file_transfer_message,
153
153
  )
154
- stream = self._stream_name_to_instance[record.stream_name]
154
+ stream = self._stream_name_to_instance[record.partition.stream_name()]
155
155
 
156
156
  if message.type == MessageType.RECORD:
157
157
  if self._record_counter[stream.name] == 0:
@@ -18,11 +18,11 @@ from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStrea
18
18
  from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer
19
19
  from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader
20
20
  from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
21
+ from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
21
22
  from airbyte_cdk.sources.streams.concurrent.partitions.types import (
22
23
  PartitionCompleteSentinel,
23
24
  QueueItem,
24
25
  )
25
- from airbyte_cdk.sources.types import Record
26
26
  from airbyte_cdk.sources.utils.slice_logger import DebugSliceLogger, SliceLogger
27
27
 
28
28
 
@@ -4,9 +4,8 @@
4
4
 
5
5
  from typing import Any, Dict
6
6
 
7
- from pydantic.v1 import BaseModel
8
-
9
7
  from airbyte_cdk.sources.utils.schema_helpers import expand_refs, rename_key
8
+ from pydantic.v1 import BaseModel
10
9
 
11
10
 
12
11
  class BaseConfig(BaseModel):
@@ -8,7 +8,6 @@ from datetime import datetime
8
8
  from typing import Any, Mapping, Optional, Union
9
9
 
10
10
  import jwt
11
-
12
11
  from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
13
12
  from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
14
13
  from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, List, Mapping, Optional, Union
7
7
 
8
8
  import pendulum
9
-
10
9
  from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
11
10
  from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
12
11
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
@@ -6,7 +6,6 @@ from dataclasses import dataclass
6
6
  from typing import Any, List, Mapping
7
7
 
8
8
  import dpath
9
-
10
9
  from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
11
10
 
12
11
 
@@ -8,8 +8,6 @@ from dataclasses import InitVar, dataclass
8
8
  from typing import Any, Mapping, Union
9
9
 
10
10
  import requests
11
- from cachetools import TTLCache, cached
12
-
13
11
  from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
14
12
  from airbyte_cdk.sources.declarative.auth.token_provider import TokenProvider
15
13
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
@@ -18,6 +16,7 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
18
16
  RequestOptionType,
19
17
  )
20
18
  from airbyte_cdk.sources.types import Config
19
+ from cachetools import TTLCache, cached
21
20
 
22
21
 
23
22
  @dataclass
@@ -10,9 +10,6 @@ from typing import Any, List, Mapping, Optional, Union
10
10
 
11
11
  import dpath
12
12
  import pendulum
13
- from isodate import Duration
14
- from pendulum import DateTime
15
-
16
13
  from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
17
14
  from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
18
15
  from airbyte_cdk.sources.declarative.exceptions import ReadException
@@ -21,6 +18,8 @@ from airbyte_cdk.sources.declarative.requesters.requester import Requester
21
18
  from airbyte_cdk.sources.http_logger import format_http_message
22
19
  from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
23
20
  from airbyte_cdk.sources.types import Config
21
+ from isodate import Duration
22
+ from pendulum import DateTime
24
23
 
25
24
 
26
25
  class TokenProvider:
@@ -3,7 +3,7 @@
3
3
  #
4
4
 
5
5
  import logging
6
- from typing import Any, Callable, Generic, Iterator, List, Mapping, Optional, Tuple, Union
6
+ from typing import Any, Generic, Iterator, List, Mapping, Optional, Tuple, Union, Callable
7
7
 
8
8
  from airbyte_cdk.models import (
9
9
  AirbyteCatalog,
@@ -27,16 +27,14 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
27
27
  )
28
28
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
29
29
  DatetimeBasedCursor as DatetimeBasedCursorModel,
30
- )
31
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
32
30
  DeclarativeStream as DeclarativeStreamModel,
33
31
  )
34
32
  from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
35
- ComponentDefinition,
36
33
  ModelToComponentFactory,
34
+ ComponentDefinition,
37
35
  )
38
36
  from airbyte_cdk.sources.declarative.requesters import HttpRequester
39
- from airbyte_cdk.sources.declarative.retrievers import Retriever, SimpleRetriever
37
+ from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever, Retriever
40
38
  from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import (
41
39
  DeclarativePartitionFactory,
42
40
  StreamSlicerPartitionGenerator,
@@ -44,15 +42,14 @@ from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_genera
44
42
  from airbyte_cdk.sources.declarative.transformations.add_fields import AddFields
45
43
  from airbyte_cdk.sources.declarative.types import ConnectionDefinition
46
44
  from airbyte_cdk.sources.source import TState
45
+ from airbyte_cdk.sources.types import Config, StreamState
47
46
  from airbyte_cdk.sources.streams import Stream
48
47
  from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
49
48
  from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
50
49
  AlwaysAvailableAvailabilityStrategy,
51
50
  )
52
- from airbyte_cdk.sources.streams.concurrent.cursor import FinalStateCursor
53
51
  from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
54
52
  from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream
55
- from airbyte_cdk.sources.types import Config, StreamState
56
53
 
57
54
 
58
55
  class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
@@ -70,15 +67,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
70
67
  component_factory: Optional[ModelToComponentFactory] = None,
71
68
  **kwargs: Any,
72
69
  ) -> None:
73
- # To reduce the complexity of the concurrent framework, we are not enabling RFR with synthetic
74
- # cursors. We do this by no longer automatically instantiating RFR cursors when converting
75
- # the declarative models into runtime components. Concurrent sources will continue to checkpoint
76
- # incremental streams running in full refresh.
77
- component_factory = component_factory or ModelToComponentFactory(
78
- emit_connector_builder_messages=emit_connector_builder_messages,
79
- disable_resumable_full_refresh=True,
80
- )
81
-
82
70
  super().__init__(
83
71
  source_config=source_config,
84
72
  debug=debug,
@@ -201,24 +189,13 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
201
189
  # these legacy Python streams the way we do low-code streams to determine if they are concurrent compatible,
202
190
  # so we need to treat them as synchronous
203
191
  if isinstance(declarative_stream, DeclarativeStream):
204
- incremental_sync_component_definition = name_to_stream_mapping[
192
+ datetime_based_cursor_component_definition = name_to_stream_mapping[
205
193
  declarative_stream.name
206
194
  ].get("incremental_sync")
207
195
 
208
- partition_router_component_definition = (
209
- name_to_stream_mapping[declarative_stream.name]
210
- .get("retriever")
211
- .get("partition_router")
212
- )
213
-
214
- is_substream_without_incremental = (
215
- partition_router_component_definition
216
- and not incremental_sync_component_definition
217
- )
218
-
219
196
  if (
220
- incremental_sync_component_definition
221
- and incremental_sync_component_definition.get("type", "")
197
+ datetime_based_cursor_component_definition
198
+ and datetime_based_cursor_component_definition.get("type", "")
222
199
  == DatetimeBasedCursorModel.__name__
223
200
  and self._stream_supports_concurrent_partition_processing(
224
201
  declarative_stream=declarative_stream
@@ -234,7 +211,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
234
211
  self._constructor.create_concurrent_cursor_from_datetime_based_cursor(
235
212
  state_manager=state_manager,
236
213
  model_type=DatetimeBasedCursorModel,
237
- component_definition=incremental_sync_component_definition,
214
+ component_definition=datetime_based_cursor_component_definition,
238
215
  stream_name=declarative_stream.name,
239
216
  stream_namespace=declarative_stream.namespace,
240
217
  config=config or {},
@@ -268,41 +245,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
268
245
  cursor=cursor,
269
246
  )
270
247
  )
271
- elif is_substream_without_incremental and hasattr(
272
- declarative_stream.retriever, "stream_slicer"
273
- ):
274
- partition_generator = StreamSlicerPartitionGenerator(
275
- DeclarativePartitionFactory(
276
- declarative_stream.name,
277
- declarative_stream.get_json_schema(),
278
- self._retriever_factory(
279
- name_to_stream_mapping[declarative_stream.name],
280
- config,
281
- {},
282
- ),
283
- self.message_repository,
284
- ),
285
- declarative_stream.retriever.stream_slicer,
286
- )
287
-
288
- final_state_cursor = FinalStateCursor(
289
- stream_name=declarative_stream.name,
290
- stream_namespace=declarative_stream.namespace,
291
- message_repository=self.message_repository,
292
- )
293
-
294
- concurrent_streams.append(
295
- DefaultStream(
296
- partition_generator=partition_generator,
297
- name=declarative_stream.name,
298
- json_schema=declarative_stream.get_json_schema(),
299
- availability_strategy=AlwaysAvailableAvailabilityStrategy(),
300
- primary_key=get_primary_key_from_stream(declarative_stream.primary_key),
301
- cursor_field=None,
302
- logger=self.logger,
303
- cursor=final_state_cursor,
304
- )
305
- )
306
248
  else:
307
249
  synchronous_streams.append(declarative_stream)
308
250
  else:
@@ -1972,173 +1972,6 @@ definitions:
1972
1972
  - app_id:
1973
1973
  type: string
1974
1974
  path_in_connector_config: ["info", "app_id"]
1975
- oauth_connector_input_specification:
1976
- title: DeclarativeOAuth Connector Specification
1977
- description: |-
1978
- The DeclarativeOAuth specific blob.
1979
- Pertains to the fields defined by the connector relating to the OAuth flow.
1980
-
1981
- Interpolation capabilities:
1982
- - The variables placeholders are declared as `{my_var}`.
1983
- - The nested resolution variables like `{{my_nested_var}}` is allowed as well.
1984
-
1985
- - The allowed interpolation context is:
1986
- + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}
1987
- + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}
1988
- + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}
1989
- + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}
1990
- + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}
1991
-
1992
- Examples:
1993
- - The TikTok Marketing DeclarativeOAuth spec:
1994
- {
1995
- "oauth_connector_input_specification": {
1996
- "type": "object",
1997
- "additionalProperties": false,
1998
- "properties": {
1999
- "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
2000
- "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",
2001
- "access_token_params": {
2002
- "{auth_code_key}": "{{auth_code_key}}",
2003
- "{client_id_key}": "{{client_id_key}}",
2004
- "{client_secret_key}": "{{client_secret_key}}"
2005
- },
2006
- "access_token_headers": {
2007
- "Content-Type": "application/json",
2008
- "Accept": "application/json"
2009
- },
2010
- "extract_output": ["data.access_token"],
2011
- "client_id_key": "app_id",
2012
- "client_secret_key": "secret",
2013
- "auth_code_key": "auth_code"
2014
- }
2015
- }
2016
- }
2017
- type: object
2018
- additionalProperties: true
2019
- required:
2020
- - consent_url
2021
- - access_token_url
2022
- - extract_output
2023
- properties:
2024
- consent_url:
2025
- title: DeclarativeOAuth Consent URL
2026
- type: string
2027
- description: |-
2028
- The DeclarativeOAuth Specific string URL string template to initiate the authentication.
2029
- The placeholders are replaced during the processing to provide neccessary values.
2030
- examples:
2031
- - consent_url: https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}
2032
- - consent_url: https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}
2033
- scope:
2034
- title: (Optional) DeclarativeOAuth Scope
2035
- type: string
2036
- description: |-
2037
- The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.
2038
- examples:
2039
- - scope: user:read user:read_orders workspaces:read
2040
- access_token_url:
2041
- title: DeclarativeOAuth Access Token URL
2042
- type: string
2043
- description: |-
2044
- The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.
2045
- The placeholders are replaced during the processing to provide neccessary values.
2046
- examples:
2047
- - access_token_url: https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}
2048
- access_token_headers:
2049
- title: (Optional) DeclarativeOAuth Access Token Headers
2050
- type: object
2051
- additionalProperties: true
2052
- description: |-
2053
- The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
2054
- examples:
2055
- - access_token_headers: {
2056
- "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"
2057
- }
2058
- access_token_params:
2059
- title: (Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)
2060
- type: object
2061
- additionalProperties: true
2062
- description: |-
2063
- The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
2064
- When this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.
2065
- examples:
2066
- - access_token_params: {
2067
- "{auth_code_key}": "{{auth_code_key}}",
2068
- "{client_id_key}": "{{client_id_key}}",
2069
- "{client_secret_key}": "{{client_secret_key}}"
2070
- }
2071
- extract_output:
2072
- title: DeclarativeOAuth Extract Output
2073
- type: array
2074
- items:
2075
- type: string
2076
- description: |-
2077
- The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.
2078
- examples:
2079
- - extract_output: ["access_token", "refresh_token", "other_field"]
2080
- state:
2081
- title: (Optional) DeclarativeOAuth Configurable State Query Param
2082
- type: object
2083
- additionalProperties: true
2084
- required:
2085
- - min
2086
- - max
2087
- description: |-
2088
- The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,
2089
- including length and complexity.
2090
- properties:
2091
- min:
2092
- type: integer
2093
- max:
2094
- type: integer
2095
- examples:
2096
- - state: {
2097
- "min": 7,
2098
- "max": 128,
2099
- }
2100
- client_id_key:
2101
- title: (Optional) DeclarativeOAuth Client ID Key Override
2102
- type: string
2103
- description: |-
2104
- The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.
2105
- examples:
2106
- - client_id_key: "my_custom_client_id_key_name"
2107
- client_secret_key:
2108
- title: (Optional) DeclarativeOAuth Client Secret Key Override
2109
- type: string
2110
- description: |-
2111
- The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.
2112
- examples:
2113
- - client_secret_key: "my_custom_client_secret_key_name"
2114
- scope_key:
2115
- title: (Optional) DeclarativeOAuth Scope Key Override
2116
- type: string
2117
- description: |-
2118
- The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.
2119
- examples:
2120
- - scope_key: "my_custom_scope_key_key_name"
2121
- state_key:
2122
- title: (Optional) DeclarativeOAuth State Key Override
2123
- type: string
2124
- description: |-
2125
- The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.
2126
- examples:
2127
- - state_key: "my_custom_state_key_key_name"
2128
- auth_code_key:
2129
- title: (Optional) DeclarativeOAuth Auth Code Key Override
2130
- type: string
2131
- description: |-
2132
- The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.
2133
- examples:
2134
- - auth_code_key: "my_custom_auth_code_key_name"
2135
- redirect_uri_key:
2136
- title: (Optional) DeclarativeOAuth Redirect URI Key Override
2137
- type: string
2138
- description: |-
2139
- The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.
2140
- examples:
2141
- - redirect_uri_key: "my_custom_redirect_uri_key_name"
2142
1975
  complete_oauth_output_specification:
2143
1976
  title: "OAuth output specification"
2144
1977
  description: |-
@@ -5,12 +5,11 @@ import codecs
5
5
  import logging
6
6
  from dataclasses import InitVar, dataclass
7
7
  from gzip import decompress
8
- from typing import Any, Generator, List, Mapping, MutableMapping, Optional
8
+ from typing import Any, Generator, Mapping, MutableMapping, List, Optional
9
9
 
10
- import orjson
11
10
  import requests
12
-
13
11
  from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
12
+ import orjson
14
13
 
15
14
  logger = logging.getLogger("airbyte")
16
15
 
@@ -4,7 +4,6 @@ import logging
4
4
  from typing import Any, Generator, Mapping
5
5
 
6
6
  import requests
7
-
8
7
  from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
9
8
 
10
9
  logger = logging.getLogger("airbyte")