airbyte-cdk 6.6.8__py3-none-any.whl → 6.6.8rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. airbyte_cdk/cli/source_declarative_manifest/_run.py +1 -2
  2. airbyte_cdk/config_observation.py +1 -2
  3. airbyte_cdk/connector.py +0 -1
  4. airbyte_cdk/connector_builder/connector_builder_handler.py +1 -1
  5. airbyte_cdk/connector_builder/main.py +1 -2
  6. airbyte_cdk/destinations/destination.py +1 -2
  7. airbyte_cdk/destinations/vector_db_based/config.py +1 -2
  8. airbyte_cdk/destinations/vector_db_based/document_processor.py +3 -4
  9. airbyte_cdk/destinations/vector_db_based/embedder.py +4 -5
  10. airbyte_cdk/entrypoint.py +2 -3
  11. airbyte_cdk/logger.py +1 -2
  12. airbyte_cdk/models/airbyte_protocol.py +1 -2
  13. airbyte_cdk/sources/config.py +1 -2
  14. airbyte_cdk/sources/declarative/auth/jwt.py +0 -1
  15. airbyte_cdk/sources/declarative/auth/oauth.py +0 -1
  16. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +0 -1
  17. airbyte_cdk/sources/declarative/auth/token.py +1 -2
  18. airbyte_cdk/sources/declarative/auth/token_provider.py +2 -3
  19. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +60 -8
  20. airbyte_cdk/sources/declarative/decoders/json_decoder.py +2 -3
  21. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +0 -1
  22. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +0 -1
  23. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +0 -1
  24. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +0 -1
  25. airbyte_cdk/sources/declarative/extractors/http_selector.py +0 -1
  26. airbyte_cdk/sources/declarative/extractors/record_selector.py +0 -1
  27. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +1 -2
  28. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +1 -2
  29. airbyte_cdk/sources/declarative/interpolation/jinja.py +4 -5
  30. airbyte_cdk/sources/declarative/manifest_declarative_source.py +3 -4
  31. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +1 -1
  32. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +6 -3
  33. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +0 -1
  34. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +0 -1
  35. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +0 -1
  36. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +0 -1
  37. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +0 -1
  38. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +0 -1
  39. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +0 -1
  40. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +0 -1
  41. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +0 -1
  42. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +1 -2
  43. airbyte_cdk/sources/declarative/requesters/http_requester.py +0 -1
  44. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +0 -1
  45. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +0 -1
  46. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +0 -1
  47. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +0 -1
  48. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +0 -1
  49. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +0 -1
  50. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +0 -1
  51. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +0 -1
  52. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +1 -2
  53. airbyte_cdk/sources/declarative/requesters/requester.py +0 -1
  54. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +1 -2
  55. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +0 -1
  56. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +1 -1
  57. airbyte_cdk/sources/declarative/transformations/add_fields.py +0 -1
  58. airbyte_cdk/sources/declarative/transformations/remove_fields.py +0 -1
  59. airbyte_cdk/sources/declarative/yaml_declarative_source.py +0 -1
  60. airbyte_cdk/sources/embedded/tools.py +0 -1
  61. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +1 -2
  62. airbyte_cdk/sources/file_based/config/avro_format.py +1 -2
  63. airbyte_cdk/sources/file_based/config/csv_format.py +1 -2
  64. airbyte_cdk/sources/file_based/config/excel_format.py +1 -2
  65. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +1 -2
  66. airbyte_cdk/sources/file_based/config/jsonl_format.py +1 -2
  67. airbyte_cdk/sources/file_based/config/parquet_format.py +1 -2
  68. airbyte_cdk/sources/file_based/config/unstructured_format.py +1 -2
  69. airbyte_cdk/sources/file_based/file_based_source.py +1 -2
  70. airbyte_cdk/sources/file_based/file_based_stream_reader.py +1 -2
  71. airbyte_cdk/sources/file_based/file_types/avro_parser.py +0 -1
  72. airbyte_cdk/sources/file_based/file_types/csv_parser.py +1 -2
  73. airbyte_cdk/sources/file_based/file_types/excel_parser.py +5 -5
  74. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +1 -2
  75. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +1 -2
  76. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +8 -9
  77. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +1 -2
  78. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +1 -2
  79. airbyte_cdk/sources/http_logger.py +0 -1
  80. airbyte_cdk/sources/streams/call_rate.py +2 -1
  81. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +1 -2
  82. airbyte_cdk/sources/streams/concurrent/adapters.py +2 -2
  83. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +1 -2
  84. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +1 -1
  85. airbyte_cdk/sources/streams/core.py +1 -2
  86. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +1 -2
  87. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +0 -1
  88. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +0 -1
  89. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +1 -2
  90. airbyte_cdk/sources/streams/http/http.py +2 -3
  91. airbyte_cdk/sources/streams/http/http_client.py +2 -34
  92. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +1 -2
  93. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +0 -1
  94. airbyte_cdk/sources/utils/schema_helpers.py +2 -3
  95. airbyte_cdk/sql/secrets.py +1 -2
  96. airbyte_cdk/sql/shared/sql_processor.py +6 -8
  97. airbyte_cdk/test/entrypoint_wrapper.py +3 -4
  98. airbyte_cdk/test/mock_http/mocker.py +0 -1
  99. airbyte_cdk/utils/schema_inferrer.py +1 -2
  100. airbyte_cdk/utils/slice_hasher.py +1 -1
  101. airbyte_cdk/utils/traced_exception.py +1 -2
  102. {airbyte_cdk-6.6.8.dist-info → airbyte_cdk-6.6.8rc1.dist-info}/METADATA +1 -8
  103. {airbyte_cdk-6.6.8.dist-info → airbyte_cdk-6.6.8rc1.dist-info}/RECORD +106 -106
  104. {airbyte_cdk-6.6.8.dist-info → airbyte_cdk-6.6.8rc1.dist-info}/LICENSE.txt +0 -0
  105. {airbyte_cdk-6.6.8.dist-info → airbyte_cdk-6.6.8rc1.dist-info}/WHEEL +0 -0
  106. {airbyte_cdk-6.6.8.dist-info → airbyte_cdk-6.6.8rc1.dist-info}/entry_points.txt +0 -0
@@ -25,8 +25,6 @@ from datetime import datetime
25
25
  from pathlib import Path
26
26
  from typing import Any, cast
27
27
 
28
- from orjson import orjson
29
-
30
28
  from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
31
29
  from airbyte_cdk.models import (
32
30
  AirbyteErrorTraceMessage,
@@ -44,6 +42,7 @@ from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
44
42
  )
45
43
  from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
46
44
  from airbyte_cdk.sources.source import TState
45
+ from orjson import orjson
47
46
 
48
47
 
49
48
  class SourceLocalYaml(YamlDeclarativeSource):
@@ -10,8 +10,6 @@ import time
10
10
  from copy import copy
11
11
  from typing import Any, List, MutableMapping
12
12
 
13
- from orjson import orjson
14
-
15
13
  from airbyte_cdk.models import (
16
14
  AirbyteControlConnectorConfigMessage,
17
15
  AirbyteControlMessage,
@@ -20,6 +18,7 @@ from airbyte_cdk.models import (
20
18
  OrchestratorType,
21
19
  Type,
22
20
  )
21
+ from orjson import orjson
23
22
 
24
23
 
25
24
  class ObservedDict(dict): # type: ignore # disallow_any_generics is set to True, and dict is equivalent to dict[Any]
airbyte_cdk/connector.py CHANGED
@@ -11,7 +11,6 @@ from abc import ABC, abstractmethod
11
11
  from typing import Any, Generic, Mapping, Optional, Protocol, TypeVar
12
12
 
13
13
  import yaml
14
-
15
14
  from airbyte_cdk.models import (
16
15
  AirbyteConnectionStatus,
17
16
  ConnectorSpecification,
@@ -12,8 +12,8 @@ from airbyte_cdk.models import (
12
12
  AirbyteRecordMessage,
13
13
  AirbyteStateMessage,
14
14
  ConfiguredAirbyteCatalog,
15
- Type,
16
15
  )
16
+ from airbyte_cdk.models import Type
17
17
  from airbyte_cdk.models import Type as MessageType
18
18
  from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
19
19
  from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
@@ -6,8 +6,6 @@
6
6
  import sys
7
7
  from typing import Any, List, Mapping, Optional, Tuple
8
8
 
9
- from orjson import orjson
10
-
11
9
  from airbyte_cdk.connector import BaseConnector
12
10
  from airbyte_cdk.connector_builder.connector_builder_handler import (
13
11
  TestReadLimits,
@@ -27,6 +25,7 @@ from airbyte_cdk.models import (
27
25
  from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
28
26
  from airbyte_cdk.sources.source import Source
29
27
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException
28
+ from orjson import orjson
30
29
 
31
30
 
32
31
  def get_config_and_catalog_from_args(
@@ -9,8 +9,6 @@ import sys
9
9
  from abc import ABC, abstractmethod
10
10
  from typing import Any, Iterable, List, Mapping
11
11
 
12
- from orjson import orjson
13
-
14
12
  from airbyte_cdk.connector import Connector
15
13
  from airbyte_cdk.exception_handler import init_uncaught_exception_handler
16
14
  from airbyte_cdk.models import (
@@ -22,6 +20,7 @@ from airbyte_cdk.models import (
22
20
  )
23
21
  from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
24
22
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException
23
+ from orjson import orjson
25
24
 
26
25
  logger = logging.getLogger("airbyte")
27
26
 
@@ -5,10 +5,9 @@
5
5
  from typing import Any, Dict, List, Literal, Optional, Union
6
6
 
7
7
  import dpath
8
- from pydantic.v1 import BaseModel, Field
9
-
10
8
  from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
11
9
  from airbyte_cdk.utils.spec_schema_transformations import resolve_refs
10
+ from pydantic.v1 import BaseModel, Field
12
11
 
13
12
 
14
13
  class SeparatorSplitterConfigModel(BaseModel):
@@ -8,10 +8,6 @@ from dataclasses import dataclass
8
8
  from typing import Any, Dict, List, Mapping, Optional, Tuple
9
9
 
10
10
  import dpath
11
- from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
12
- from langchain.utils import stringify_dict
13
- from langchain_core.documents.base import Document
14
-
15
11
  from airbyte_cdk.destinations.vector_db_based.config import (
16
12
  ProcessingConfigModel,
17
13
  SeparatorSplitterConfigModel,
@@ -25,6 +21,9 @@ from airbyte_cdk.models import (
25
21
  DestinationSyncMode,
26
22
  )
27
23
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType
24
+ from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
25
+ from langchain.utils import stringify_dict
26
+ from langchain_core.documents.base import Document
28
27
 
29
28
  METADATA_STREAM_FIELD = "_ab_stream"
30
29
  METADATA_RECORD_ID_FIELD = "_ab_record_id"
@@ -7,11 +7,6 @@ from abc import ABC, abstractmethod
7
7
  from dataclasses import dataclass
8
8
  from typing import List, Optional, Union, cast
9
9
 
10
- from langchain.embeddings.cohere import CohereEmbeddings
11
- from langchain.embeddings.fake import FakeEmbeddings
12
- from langchain.embeddings.localai import LocalAIEmbeddings
13
- from langchain.embeddings.openai import OpenAIEmbeddings
14
-
15
10
  from airbyte_cdk.destinations.vector_db_based.config import (
16
11
  AzureOpenAIEmbeddingConfigModel,
17
12
  CohereEmbeddingConfigModel,
@@ -24,6 +19,10 @@ from airbyte_cdk.destinations.vector_db_based.config import (
24
19
  from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, format_exception
25
20
  from airbyte_cdk.models import AirbyteRecordMessage
26
21
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType
22
+ from langchain.embeddings.cohere import CohereEmbeddings
23
+ from langchain.embeddings.fake import FakeEmbeddings
24
+ from langchain.embeddings.localai import LocalAIEmbeddings
25
+ from langchain.embeddings.openai import OpenAIEmbeddings
27
26
 
28
27
 
29
28
  @dataclass
airbyte_cdk/entrypoint.py CHANGED
@@ -16,9 +16,6 @@ from typing import Any, DefaultDict, Iterable, List, Mapping, Optional
16
16
  from urllib.parse import urlparse
17
17
 
18
18
  import requests
19
- from orjson import orjson
20
- from requests import PreparedRequest, Response, Session
21
-
22
19
  from airbyte_cdk.connector import TConfig
23
20
  from airbyte_cdk.exception_handler import init_uncaught_exception_handler
24
21
  from airbyte_cdk.logger import init_logger
@@ -41,6 +38,8 @@ from airbyte_cdk.utils import is_cloud_environment, message_utils
41
38
  from airbyte_cdk.utils.airbyte_secrets_utils import get_secrets, update_secrets
42
39
  from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH
43
40
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException
41
+ from orjson import orjson
42
+ from requests import PreparedRequest, Response, Session
44
43
 
45
44
  logger = init_logger("airbyte")
46
45
 
airbyte_cdk/logger.py CHANGED
@@ -7,8 +7,6 @@ import logging
7
7
  import logging.config
8
8
  from typing import Any, Callable, Mapping, Optional, Tuple
9
9
 
10
- from orjson import orjson
11
-
12
10
  from airbyte_cdk.models import (
13
11
  AirbyteLogMessage,
14
12
  AirbyteMessage,
@@ -17,6 +15,7 @@ from airbyte_cdk.models import (
17
15
  Type,
18
16
  )
19
17
  from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
18
+ from orjson import orjson
20
19
 
21
20
  LOGGING_CONFIG = {
22
21
  "version": 1,
@@ -5,11 +5,10 @@
5
5
  from dataclasses import InitVar, dataclass
6
6
  from typing import Annotated, Any, Dict, List, Mapping, Optional, Union
7
7
 
8
+ from airbyte_cdk.models.file_transfer_record_message import AirbyteFileTransferRecordMessage
8
9
  from airbyte_protocol_dataclasses.models import * # noqa: F403 # Allow '*'
9
10
  from serpyco_rs.metadata import Alias
10
11
 
11
- from airbyte_cdk.models.file_transfer_record_message import AirbyteFileTransferRecordMessage
12
-
13
12
  # ruff: noqa: F405 # ignore fuzzy import issues with 'import *'
14
13
 
15
14
 
@@ -4,9 +4,8 @@
4
4
 
5
5
  from typing import Any, Dict
6
6
 
7
- from pydantic.v1 import BaseModel
8
-
9
7
  from airbyte_cdk.sources.utils.schema_helpers import expand_refs, rename_key
8
+ from pydantic.v1 import BaseModel
10
9
 
11
10
 
12
11
  class BaseConfig(BaseModel):
@@ -8,7 +8,6 @@ from datetime import datetime
8
8
  from typing import Any, Mapping, Optional, Union
9
9
 
10
10
  import jwt
11
-
12
11
  from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
13
12
  from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
14
13
  from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, List, Mapping, Optional, Union
7
7
 
8
8
  import pendulum
9
-
10
9
  from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
11
10
  from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
12
11
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
@@ -6,7 +6,6 @@ from dataclasses import dataclass
6
6
  from typing import Any, List, Mapping
7
7
 
8
8
  import dpath
9
-
10
9
  from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
11
10
 
12
11
 
@@ -8,8 +8,6 @@ from dataclasses import InitVar, dataclass
8
8
  from typing import Any, Mapping, Union
9
9
 
10
10
  import requests
11
- from cachetools import TTLCache, cached
12
-
13
11
  from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
14
12
  from airbyte_cdk.sources.declarative.auth.token_provider import TokenProvider
15
13
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
@@ -18,6 +16,7 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
18
16
  RequestOptionType,
19
17
  )
20
18
  from airbyte_cdk.sources.types import Config
19
+ from cachetools import TTLCache, cached
21
20
 
22
21
 
23
22
  @dataclass
@@ -10,9 +10,6 @@ from typing import Any, List, Mapping, Optional, Union
10
10
 
11
11
  import dpath
12
12
  import pendulum
13
- from isodate import Duration
14
- from pendulum import DateTime
15
-
16
13
  from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
17
14
  from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
18
15
  from airbyte_cdk.sources.declarative.exceptions import ReadException
@@ -21,6 +18,8 @@ from airbyte_cdk.sources.declarative.requesters.requester import Requester
21
18
  from airbyte_cdk.sources.http_logger import format_http_message
22
19
  from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
23
20
  from airbyte_cdk.sources.types import Config
21
+ from isodate import Duration
22
+ from pendulum import DateTime
24
23
 
25
24
 
26
25
  class TokenProvider:
@@ -3,7 +3,7 @@
3
3
  #
4
4
 
5
5
  import logging
6
- from typing import Any, Callable, Generic, Iterator, List, Mapping, Optional, Tuple, Union
6
+ from typing import Any, Generic, Iterator, List, Mapping, Optional, Tuple, Union, Callable
7
7
 
8
8
  from airbyte_cdk.models import (
9
9
  AirbyteCatalog,
@@ -24,19 +24,15 @@ from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
24
24
  from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
25
25
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
26
26
  ConcurrencyLevel as ConcurrencyLevelModel,
27
- )
28
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
29
27
  DatetimeBasedCursor as DatetimeBasedCursorModel,
30
- )
31
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
32
28
  DeclarativeStream as DeclarativeStreamModel,
33
29
  )
34
30
  from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
35
- ComponentDefinition,
36
31
  ModelToComponentFactory,
32
+ ComponentDefinition,
37
33
  )
38
34
  from airbyte_cdk.sources.declarative.requesters import HttpRequester
39
- from airbyte_cdk.sources.declarative.retrievers import Retriever, SimpleRetriever
35
+ from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever, Retriever
40
36
  from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import (
41
37
  DeclarativePartitionFactory,
42
38
  StreamSlicerPartitionGenerator,
@@ -44,14 +40,15 @@ from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_genera
44
40
  from airbyte_cdk.sources.declarative.transformations.add_fields import AddFields
45
41
  from airbyte_cdk.sources.declarative.types import ConnectionDefinition
46
42
  from airbyte_cdk.sources.source import TState
43
+ from airbyte_cdk.sources.types import Config, StreamState
47
44
  from airbyte_cdk.sources.streams import Stream
48
45
  from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
49
46
  from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
50
47
  AlwaysAvailableAvailabilityStrategy,
51
48
  )
49
+ from airbyte_cdk.sources.streams.concurrent.cursor import FinalStateCursor
52
50
  from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
53
51
  from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream
54
- from airbyte_cdk.sources.types import Config, StreamState
55
52
 
56
53
 
57
54
  class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
@@ -69,6 +66,15 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
69
66
  component_factory: Optional[ModelToComponentFactory] = None,
70
67
  **kwargs: Any,
71
68
  ) -> None:
69
+ # To reduce the complexity of the concurrent framework, we are not enabling RFR with synthetic
70
+ # cursors. We do this by no longer automatically instantiating RFR cursors when converting
71
+ # the declarative models into runtime components. Concurrent sources will continue to checkpoint
72
+ # incremental streams running in full refresh.
73
+ component_factory = component_factory or ModelToComponentFactory(
74
+ emit_connector_builder_messages=emit_connector_builder_messages,
75
+ disable_resumable_full_refresh=True,
76
+ )
77
+
72
78
  super().__init__(
73
79
  source_config=source_config,
74
80
  debug=debug,
@@ -195,6 +201,17 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
195
201
  declarative_stream.name
196
202
  ].get("incremental_sync")
197
203
 
204
+ partition_router_component_definition = (
205
+ name_to_stream_mapping[declarative_stream.name]
206
+ .get("retriever")
207
+ .get("partition_router")
208
+ )
209
+
210
+ is_substream_without_incremental = (
211
+ partition_router_component_definition
212
+ and not datetime_based_cursor_component_definition
213
+ )
214
+
198
215
  if (
199
216
  datetime_based_cursor_component_definition
200
217
  and datetime_based_cursor_component_definition.get("type", "")
@@ -247,6 +264,41 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
247
264
  cursor=cursor,
248
265
  )
249
266
  )
267
+ elif is_substream_without_incremental and hasattr(
268
+ declarative_stream.retriever, "stream_slicer"
269
+ ):
270
+ partition_generator = StreamSlicerPartitionGenerator(
271
+ DeclarativePartitionFactory(
272
+ declarative_stream.name,
273
+ declarative_stream.get_json_schema(),
274
+ self._retriever_factory(
275
+ name_to_stream_mapping[declarative_stream.name],
276
+ config,
277
+ {},
278
+ ),
279
+ self.message_repository,
280
+ ),
281
+ declarative_stream.retriever.stream_slicer,
282
+ )
283
+
284
+ cursor = FinalStateCursor(
285
+ stream_name=declarative_stream.name,
286
+ stream_namespace=declarative_stream.namespace,
287
+ message_repository=self.message_repository,
288
+ )
289
+
290
+ concurrent_streams.append(
291
+ DefaultStream(
292
+ partition_generator=partition_generator,
293
+ name=declarative_stream.name,
294
+ json_schema=declarative_stream.get_json_schema(),
295
+ availability_strategy=AlwaysAvailableAvailabilityStrategy(),
296
+ primary_key=get_primary_key_from_stream(declarative_stream.primary_key),
297
+ cursor_field=None,
298
+ logger=self.logger,
299
+ cursor=cursor,
300
+ )
301
+ )
250
302
  else:
251
303
  synchronous_streams.append(declarative_stream)
252
304
  else:
@@ -5,12 +5,11 @@ import codecs
5
5
  import logging
6
6
  from dataclasses import InitVar, dataclass
7
7
  from gzip import decompress
8
- from typing import Any, Generator, List, Mapping, MutableMapping, Optional
8
+ from typing import Any, Generator, Mapping, MutableMapping, List, Optional
9
9
 
10
- import orjson
11
10
  import requests
12
-
13
11
  from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
12
+ import orjson
14
13
 
15
14
  logger = logging.getLogger("airbyte")
16
15
 
@@ -4,7 +4,6 @@ import logging
4
4
  from typing import Any, Generator, Mapping
5
5
 
6
6
  import requests
7
-
8
7
  from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
9
8
 
10
9
  logger = logging.getLogger("airbyte")
@@ -7,7 +7,6 @@ from dataclasses import dataclass
7
7
  from typing import Any, Generator, MutableMapping
8
8
 
9
9
  import requests
10
-
11
10
  from airbyte_cdk.sources.declarative.decoders import Decoder
12
11
 
13
12
  logger = logging.getLogger("airbyte")
@@ -9,7 +9,6 @@ from xml.parsers.expat import ExpatError
9
9
 
10
10
  import requests
11
11
  import xmltodict
12
-
13
12
  from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
14
13
 
15
14
  logger = logging.getLogger("airbyte")
@@ -7,7 +7,6 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Union
7
7
 
8
8
  import dpath
9
9
  import requests
10
-
11
10
  from airbyte_cdk.sources.declarative.decoders import Decoder, JsonDecoder
12
11
  from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
13
12
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
@@ -6,7 +6,6 @@ from abc import abstractmethod
6
6
  from typing import Any, Iterable, Mapping, Optional
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
11
10
 
12
11
 
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, Iterable, List, Mapping, Optional
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
11
10
  from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
12
11
  from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
@@ -10,9 +10,8 @@ from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
10
10
 
11
11
  import pandas as pd
12
12
  import requests
13
- from numpy import nan
14
-
15
13
  from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
14
+ from numpy import nan
16
15
 
17
16
  EMPTY_STR: str = ""
18
17
  DEFAULT_ENCODING: str = "utf-8"
@@ -7,8 +7,6 @@ from dataclasses import InitVar, dataclass, field
7
7
  from datetime import timedelta
8
8
  from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Union
9
9
 
10
- from isodate import Duration, duration_isoformat, parse_duration
11
-
12
10
  from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
13
11
  from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
14
12
  from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
@@ -21,6 +19,7 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
21
19
  )
22
20
  from airbyte_cdk.sources.message import MessageRepository
23
21
  from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
22
+ from isodate import Duration, duration_isoformat, parse_duration
24
23
 
25
24
 
26
25
  @dataclass
@@ -6,15 +6,14 @@ import ast
6
6
  from functools import cache
7
7
  from typing import Any, Mapping, Optional, Tuple, Type
8
8
 
9
- from jinja2 import meta
10
- from jinja2.environment import Template
11
- from jinja2.exceptions import UndefinedError
12
- from jinja2.sandbox import SandboxedEnvironment
13
-
14
9
  from airbyte_cdk.sources.declarative.interpolation.filters import filters
15
10
  from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
16
11
  from airbyte_cdk.sources.declarative.interpolation.macros import macros
17
12
  from airbyte_cdk.sources.types import Config
13
+ from jinja2 import meta
14
+ from jinja2.environment import Template
15
+ from jinja2.exceptions import UndefinedError
16
+ from jinja2.sandbox import SandboxedEnvironment
18
17
 
19
18
 
20
19
  class StreamPartitionAccessEnvironment(SandboxedEnvironment):
@@ -8,12 +8,9 @@ import pkgutil
8
8
  from copy import deepcopy
9
9
  from importlib import metadata
10
10
  from typing import Any, Dict, Iterator, List, Mapping, Optional
11
+ from packaging.version import Version, InvalidVersion
11
12
 
12
13
  import yaml
13
- from jsonschema.exceptions import ValidationError
14
- from jsonschema.validators import validate
15
- from packaging.version import InvalidVersion, Version
16
-
17
14
  from airbyte_cdk.models import (
18
15
  AirbyteConnectionStatus,
19
16
  AirbyteMessage,
@@ -47,6 +44,8 @@ from airbyte_cdk.sources.utils.slice_logger import (
47
44
  DebugSliceLogger,
48
45
  SliceLogger,
49
46
  )
47
+ from jsonschema.exceptions import ValidationError
48
+ from jsonschema.validators import validate
50
49
 
51
50
 
52
51
  class ManifestDeclarativeSource(DeclarativeSource):
@@ -5,9 +5,9 @@ from typing import Any, Mapping
5
5
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
6
6
  from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
7
7
  from airbyte_cdk.sources.declarative.models import (
8
- CustomIncrementalSync,
9
8
  DatetimeBasedCursor,
10
9
  SubstreamPartitionRouter,
10
+ CustomIncrementalSync,
11
11
  )
12
12
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import ParentStreamConfig
13
13
 
@@ -25,9 +25,6 @@ from typing import (
25
25
  get_type_hints,
26
26
  )
27
27
 
28
- from isodate import parse_duration
29
- from pydantic.v1 import BaseModel
30
-
31
28
  from airbyte_cdk.models import FailureType, Level
32
29
  from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
33
30
  from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator
@@ -370,6 +367,8 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_sta
370
367
  from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction
371
368
  from airbyte_cdk.sources.types import Config
372
369
  from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
370
+ from isodate import parse_duration
371
+ from pydantic.v1 import BaseModel
373
372
 
374
373
  ComponentDefinition = Mapping[str, Any]
375
374
 
@@ -384,6 +383,7 @@ class ModelToComponentFactory:
384
383
  emit_connector_builder_messages: bool = False,
385
384
  disable_retries: bool = False,
386
385
  disable_cache: bool = False,
386
+ disable_resumable_full_refresh: bool = False,
387
387
  message_repository: Optional[MessageRepository] = None,
388
388
  ):
389
389
  self._init_mappings()
@@ -392,6 +392,7 @@ class ModelToComponentFactory:
392
392
  self._emit_connector_builder_messages = emit_connector_builder_messages
393
393
  self._disable_retries = disable_retries
394
394
  self._disable_cache = disable_cache
395
+ self._disable_resumable_full_refresh = disable_resumable_full_refresh
395
396
  self._message_repository = message_repository or InMemoryMessageRepository( # type: ignore
396
397
  self._evaluate_log_level(emit_connector_builder_messages)
397
398
  )
@@ -1335,6 +1336,8 @@ class ModelToComponentFactory:
1335
1336
  if model.incremental_sync
1336
1337
  else None
1337
1338
  )
1339
+ elif self._disable_resumable_full_refresh:
1340
+ return stream_slicer
1338
1341
  elif stream_slicer:
1339
1342
  # For the Full-Refresh sub-streams, we use the nested `ChildPartitionResumableFullRefreshCursor`
1340
1343
  return PerPartitionCursor(
@@ -7,7 +7,6 @@ from dataclasses import InitVar, dataclass
7
7
  from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
8
8
 
9
9
  import dpath
10
-
11
10
  from airbyte_cdk.models import AirbyteMessage
12
11
  from airbyte_cdk.models import Type as MessageType
13
12
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
11
10
  from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
12
11
  from airbyte_cdk.sources.types import Config
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
11
10
  from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
12
11
  from airbyte_cdk.sources.types import Config
@@ -7,7 +7,6 @@ from dataclasses import InitVar, dataclass
7
7
  from typing import Any, Mapping, Optional, Union
8
8
 
9
9
  import requests
10
-
11
10
  from airbyte_cdk.models import FailureType
12
11
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
13
12
  from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
@@ -9,7 +9,6 @@ from dataclasses import InitVar, dataclass
9
9
  from typing import Any, Mapping, Optional, Union
10
10
 
11
11
  import requests
12
-
13
12
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
14
13
  from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
15
14
  get_numeric_value_from_header,
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, List, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
11
10
  from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
12
11
  ErrorResolution,