airbyte-cdk 6.7.0__py3-none-any.whl → 6.7.0.dev5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. airbyte_cdk/cli/source_declarative_manifest/_run.py +1 -2
  2. airbyte_cdk/config_observation.py +1 -2
  3. airbyte_cdk/connector.py +0 -1
  4. airbyte_cdk/connector_builder/connector_builder_handler.py +1 -1
  5. airbyte_cdk/connector_builder/main.py +1 -2
  6. airbyte_cdk/destinations/destination.py +1 -2
  7. airbyte_cdk/destinations/vector_db_based/config.py +1 -2
  8. airbyte_cdk/destinations/vector_db_based/document_processor.py +3 -4
  9. airbyte_cdk/destinations/vector_db_based/embedder.py +4 -5
  10. airbyte_cdk/entrypoint.py +2 -3
  11. airbyte_cdk/logger.py +1 -2
  12. airbyte_cdk/models/airbyte_protocol.py +1 -2
  13. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +3 -3
  14. airbyte_cdk/sources/concurrent_source/concurrent_source.py +1 -1
  15. airbyte_cdk/sources/config.py +1 -2
  16. airbyte_cdk/sources/declarative/auth/jwt.py +0 -1
  17. airbyte_cdk/sources/declarative/auth/oauth.py +0 -1
  18. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +0 -1
  19. airbyte_cdk/sources/declarative/auth/token.py +1 -2
  20. airbyte_cdk/sources/declarative/auth/token_provider.py +2 -3
  21. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +4 -6
  22. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +13 -13
  23. airbyte_cdk/sources/declarative/decoders/json_decoder.py +2 -3
  24. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +0 -1
  25. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +0 -1
  26. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +0 -1
  27. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +0 -1
  28. airbyte_cdk/sources/declarative/extractors/http_selector.py +0 -1
  29. airbyte_cdk/sources/declarative/extractors/record_filter.py +48 -6
  30. airbyte_cdk/sources/declarative/extractors/record_selector.py +4 -32
  31. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +1 -2
  32. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +1 -2
  33. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +2 -5
  34. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +2 -5
  35. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +3 -1
  36. airbyte_cdk/sources/declarative/interpolation/jinja.py +4 -5
  37. airbyte_cdk/sources/declarative/manifest_declarative_source.py +3 -4
  38. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +1 -1
  39. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +2 -7
  40. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +0 -1
  41. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +0 -1
  42. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +0 -1
  43. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +0 -1
  44. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +0 -1
  45. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +0 -1
  46. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +0 -1
  47. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +0 -1
  48. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +0 -1
  49. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +1 -2
  50. airbyte_cdk/sources/declarative/requesters/http_requester.py +0 -1
  51. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +0 -1
  52. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +0 -1
  53. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +0 -1
  54. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +0 -1
  55. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +0 -1
  56. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +0 -1
  57. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +0 -1
  58. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +3 -9
  59. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +1 -2
  60. airbyte_cdk/sources/declarative/requesters/requester.py +0 -1
  61. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +1 -2
  62. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +4 -12
  63. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +4 -7
  64. airbyte_cdk/sources/declarative/transformations/add_fields.py +0 -1
  65. airbyte_cdk/sources/declarative/transformations/remove_fields.py +0 -1
  66. airbyte_cdk/sources/declarative/yaml_declarative_source.py +0 -1
  67. airbyte_cdk/sources/embedded/tools.py +0 -1
  68. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +1 -2
  69. airbyte_cdk/sources/file_based/config/avro_format.py +1 -2
  70. airbyte_cdk/sources/file_based/config/csv_format.py +1 -2
  71. airbyte_cdk/sources/file_based/config/excel_format.py +1 -2
  72. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +1 -2
  73. airbyte_cdk/sources/file_based/config/jsonl_format.py +1 -2
  74. airbyte_cdk/sources/file_based/config/parquet_format.py +1 -2
  75. airbyte_cdk/sources/file_based/config/unstructured_format.py +1 -2
  76. airbyte_cdk/sources/file_based/file_based_source.py +1 -2
  77. airbyte_cdk/sources/file_based/file_based_stream_reader.py +1 -2
  78. airbyte_cdk/sources/file_based/file_types/avro_parser.py +0 -1
  79. airbyte_cdk/sources/file_based/file_types/csv_parser.py +1 -2
  80. airbyte_cdk/sources/file_based/file_types/excel_parser.py +5 -5
  81. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +1 -2
  82. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +1 -2
  83. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +8 -9
  84. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +1 -2
  85. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +4 -5
  86. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +1 -1
  87. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +1 -1
  88. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +1 -1
  89. airbyte_cdk/sources/http_logger.py +0 -1
  90. airbyte_cdk/sources/streams/call_rate.py +2 -1
  91. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +1 -2
  92. airbyte_cdk/sources/streams/concurrent/adapters.py +4 -8
  93. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +1 -2
  94. airbyte_cdk/sources/streams/concurrent/cursor.py +6 -30
  95. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -1
  96. airbyte_cdk/sources/streams/concurrent/partitions/record.py +35 -0
  97. airbyte_cdk/sources/streams/concurrent/partitions/types.py +1 -1
  98. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +1 -1
  99. airbyte_cdk/sources/streams/core.py +1 -2
  100. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +1 -2
  101. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +0 -1
  102. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +0 -1
  103. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +1 -2
  104. airbyte_cdk/sources/streams/http/http.py +2 -3
  105. airbyte_cdk/sources/streams/http/http_client.py +2 -49
  106. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +1 -2
  107. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +0 -1
  108. airbyte_cdk/sources/types.py +1 -14
  109. airbyte_cdk/sources/utils/schema_helpers.py +2 -3
  110. airbyte_cdk/sql/secrets.py +1 -2
  111. airbyte_cdk/sql/shared/sql_processor.py +6 -8
  112. airbyte_cdk/test/entrypoint_wrapper.py +3 -4
  113. airbyte_cdk/test/mock_http/mocker.py +0 -1
  114. airbyte_cdk/utils/schema_inferrer.py +1 -2
  115. airbyte_cdk/utils/slice_hasher.py +1 -1
  116. airbyte_cdk/utils/traced_exception.py +1 -2
  117. {airbyte_cdk-6.7.0.dist-info → airbyte_cdk-6.7.0.dev5.dist-info}/METADATA +1 -8
  118. {airbyte_cdk-6.7.0.dist-info → airbyte_cdk-6.7.0.dev5.dist-info}/RECORD +121 -120
  119. {airbyte_cdk-6.7.0.dist-info → airbyte_cdk-6.7.0.dev5.dist-info}/LICENSE.txt +0 -0
  120. {airbyte_cdk-6.7.0.dist-info → airbyte_cdk-6.7.0.dev5.dist-info}/WHEEL +0 -0
  121. {airbyte_cdk-6.7.0.dist-info → airbyte_cdk-6.7.0.dev5.dist-info}/entry_points.txt +0 -0
@@ -3,14 +3,12 @@
3
3
  #
4
4
 
5
5
  from dataclasses import InitVar, dataclass, field
6
- from typing import Any, Iterable, List, Mapping, Optional, Union
6
+ from typing import Any, Iterable, List, Mapping, Optional
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
11
10
  from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
12
11
  from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
13
- from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
14
12
  from airbyte_cdk.sources.declarative.models import SchemaNormalization
15
13
  from airbyte_cdk.sources.declarative.transformations import RecordTransformation
16
14
  from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
@@ -39,34 +37,11 @@ class RecordSelector(HttpSelector):
39
37
  config: Config
40
38
  parameters: InitVar[Mapping[str, Any]]
41
39
  schema_normalization: TypeTransformer
42
- name: str
43
- _name: Union[InterpolatedString, str] = field(init=False, repr=False, default="")
44
40
  record_filter: Optional[RecordFilter] = None
45
41
  transformations: List[RecordTransformation] = field(default_factory=lambda: [])
46
42
 
47
43
  def __post_init__(self, parameters: Mapping[str, Any]) -> None:
48
44
  self._parameters = parameters
49
- self._name = (
50
- InterpolatedString(self._name, parameters=parameters)
51
- if isinstance(self._name, str)
52
- else self._name
53
- )
54
-
55
- @property # type: ignore
56
- def name(self) -> str:
57
- """
58
- :return: Stream name
59
- """
60
- return (
61
- str(self._name.eval(self.config))
62
- if isinstance(self._name, InterpolatedString)
63
- else self._name
64
- )
65
-
66
- @name.setter
67
- def name(self, value: str) -> None:
68
- if not isinstance(value, property):
69
- self._name = value
70
45
 
71
46
  def select_records(
72
47
  self,
@@ -110,7 +85,7 @@ class RecordSelector(HttpSelector):
110
85
  transformed_data = self._transform(filtered_data, stream_state, stream_slice)
111
86
  normalized_data = self._normalize_by_schema(transformed_data, schema=records_schema)
112
87
  for data in normalized_data:
113
- yield Record(data=data, stream_name=self.name, associated_slice=stream_slice)
88
+ yield Record(data, stream_slice)
114
89
 
115
90
  def _normalize_by_schema(
116
91
  self, records: Iterable[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]
@@ -150,9 +125,6 @@ class RecordSelector(HttpSelector):
150
125
  for record in records:
151
126
  for transformation in self.transformations:
152
127
  transformation.transform(
153
- record, # type: ignore # record has type Mapping[str, Any], but Dict[str, Any] expected
154
- config=self.config,
155
- stream_state=stream_state,
156
- stream_slice=stream_slice,
157
- )
128
+ record, config=self.config, stream_state=stream_state, stream_slice=stream_slice
129
+ ) # type: ignore # record has type Mapping[str, Any], but Dict[str, Any] expected
158
130
  yield record
@@ -10,9 +10,8 @@ from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
10
10
 
11
11
  import pandas as pd
12
12
  import requests
13
- from numpy import nan
14
-
15
13
  from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
14
+ from numpy import nan
16
15
 
17
16
  EMPTY_STR: str = ""
18
17
  DEFAULT_ENCODING: str = "utf-8"
@@ -7,8 +7,6 @@ from dataclasses import InitVar, dataclass, field
7
7
  from datetime import timedelta
8
8
  from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Union
9
9
 
10
- from isodate import Duration, duration_isoformat, parse_duration
11
-
12
10
  from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
13
11
  from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
14
12
  from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
@@ -21,6 +19,7 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
21
19
  )
22
20
  from airbyte_cdk.sources.message import MessageRepository
23
21
  from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
22
+ from isodate import Duration, duration_isoformat, parse_duration
24
23
 
25
24
 
26
25
  @dataclass
@@ -340,11 +340,8 @@ class GlobalSubstreamCursor(DeclarativeCursor):
340
340
  @staticmethod
341
341
  def _convert_record_to_cursor_record(record: Record) -> Record:
342
342
  return Record(
343
- data=record.data,
344
- stream_name=record.stream_name,
345
- associated_slice=StreamSlice(
346
- partition={}, cursor_slice=record.associated_slice.cursor_slice
347
- )
343
+ record.data,
344
+ StreamSlice(partition={}, cursor_slice=record.associated_slice.cursor_slice)
348
345
  if record.associated_slice
349
346
  else None,
350
347
  )
@@ -325,11 +325,8 @@ class PerPartitionCursor(DeclarativeCursor):
325
325
  @staticmethod
326
326
  def _convert_record_to_cursor_record(record: Record) -> Record:
327
327
  return Record(
328
- data=record.data,
329
- stream_name=record.stream_name,
330
- associated_slice=StreamSlice(
331
- partition={}, cursor_slice=record.associated_slice.cursor_slice
332
- )
328
+ record.data,
329
+ StreamSlice(partition={}, cursor_slice=record.associated_slice.cursor_slice)
333
330
  if record.associated_slice
334
331
  else None,
335
332
  )
@@ -194,7 +194,9 @@ class PerPartitionWithGlobalCursor(DeclarativeCursor):
194
194
  )
195
195
 
196
196
  def should_be_synced(self, record: Record) -> bool:
197
- return self._get_active_cursor().should_be_synced(record)
197
+ return self._global_cursor.should_be_synced(
198
+ record
199
+ ) or self._per_partition_cursor.should_be_synced(record)
198
200
 
199
201
  def is_greater_than_or_equal(self, first: Record, second: Record) -> bool:
200
202
  return self._global_cursor.is_greater_than_or_equal(first, second)
@@ -6,15 +6,14 @@ import ast
6
6
  from functools import cache
7
7
  from typing import Any, Mapping, Optional, Tuple, Type
8
8
 
9
- from jinja2 import meta
10
- from jinja2.environment import Template
11
- from jinja2.exceptions import UndefinedError
12
- from jinja2.sandbox import SandboxedEnvironment
13
-
14
9
  from airbyte_cdk.sources.declarative.interpolation.filters import filters
15
10
  from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
16
11
  from airbyte_cdk.sources.declarative.interpolation.macros import macros
17
12
  from airbyte_cdk.sources.types import Config
13
+ from jinja2 import meta
14
+ from jinja2.environment import Template
15
+ from jinja2.exceptions import UndefinedError
16
+ from jinja2.sandbox import SandboxedEnvironment
18
17
 
19
18
 
20
19
  class StreamPartitionAccessEnvironment(SandboxedEnvironment):
@@ -8,12 +8,9 @@ import pkgutil
8
8
  from copy import deepcopy
9
9
  from importlib import metadata
10
10
  from typing import Any, Dict, Iterator, List, Mapping, Optional
11
+ from packaging.version import Version, InvalidVersion
11
12
 
12
13
  import yaml
13
- from jsonschema.exceptions import ValidationError
14
- from jsonschema.validators import validate
15
- from packaging.version import InvalidVersion, Version
16
-
17
14
  from airbyte_cdk.models import (
18
15
  AirbyteConnectionStatus,
19
16
  AirbyteMessage,
@@ -47,6 +44,8 @@ from airbyte_cdk.sources.utils.slice_logger import (
47
44
  DebugSliceLogger,
48
45
  SliceLogger,
49
46
  )
47
+ from jsonschema.exceptions import ValidationError
48
+ from jsonschema.validators import validate
50
49
 
51
50
 
52
51
  class ManifestDeclarativeSource(DeclarativeSource):
@@ -5,9 +5,9 @@ from typing import Any, Mapping
5
5
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
6
6
  from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
7
7
  from airbyte_cdk.sources.declarative.models import (
8
- CustomIncrementalSync,
9
8
  DatetimeBasedCursor,
10
9
  SubstreamPartitionRouter,
10
+ CustomIncrementalSync,
11
11
  )
12
12
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import ParentStreamConfig
13
13
 
@@ -25,9 +25,6 @@ from typing import (
25
25
  get_type_hints,
26
26
  )
27
27
 
28
- from isodate import parse_duration
29
- from pydantic.v1 import BaseModel
30
-
31
28
  from airbyte_cdk.models import FailureType, Level
32
29
  from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
33
30
  from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator
@@ -370,6 +367,8 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_sta
370
367
  from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction
371
368
  from airbyte_cdk.sources.types import Config
372
369
  from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
370
+ from isodate import parse_duration
371
+ from pydantic.v1 import BaseModel
373
372
 
374
373
  ComponentDefinition = Mapping[str, Any]
375
374
 
@@ -1781,7 +1780,6 @@ class ModelToComponentFactory:
1781
1780
  self,
1782
1781
  model: RecordSelectorModel,
1783
1782
  config: Config,
1784
- name: str,
1785
1783
  *,
1786
1784
  transformations: List[RecordTransformation],
1787
1785
  decoder: Optional[Decoder] = None,
@@ -1812,7 +1810,6 @@ class ModelToComponentFactory:
1812
1810
 
1813
1811
  return RecordSelector(
1814
1812
  extractor=extractor,
1815
- name=name,
1816
1813
  config=config,
1817
1814
  record_filter=record_filter,
1818
1815
  transformations=transformations,
@@ -1883,7 +1880,6 @@ class ModelToComponentFactory:
1883
1880
  )
1884
1881
  record_selector = self._create_component_from_model(
1885
1882
  model=model.record_selector,
1886
- name=name,
1887
1883
  config=config,
1888
1884
  decoder=decoder,
1889
1885
  transformations=transformations,
@@ -2038,7 +2034,6 @@ class ModelToComponentFactory:
2038
2034
  requester=download_requester,
2039
2035
  record_selector=RecordSelector(
2040
2036
  extractor=ResponseToFileExtractor(),
2041
- name=name,
2042
2037
  record_filter=None,
2043
2038
  transformations=[],
2044
2039
  schema_normalization=TypeTransformer(TransformConfig.NoTransform),
@@ -7,7 +7,6 @@ from dataclasses import InitVar, dataclass
7
7
  from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
8
8
 
9
9
  import dpath
10
-
11
10
  from airbyte_cdk.models import AirbyteMessage
12
11
  from airbyte_cdk.models import Type as MessageType
13
12
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
11
10
  from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
12
11
  from airbyte_cdk.sources.types import Config
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
11
10
  from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
12
11
  from airbyte_cdk.sources.types import Config
@@ -7,7 +7,6 @@ from dataclasses import InitVar, dataclass
7
7
  from typing import Any, Mapping, Optional, Union
8
8
 
9
9
  import requests
10
-
11
10
  from airbyte_cdk.models import FailureType
12
11
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
13
12
  from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
@@ -9,7 +9,6 @@ from dataclasses import InitVar, dataclass
9
9
  from typing import Any, Mapping, Optional, Union
10
10
 
11
11
  import requests
12
-
13
12
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
14
13
  from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
15
14
  get_numeric_value_from_header,
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, List, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
11
10
  from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
12
11
  ErrorResolution,
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, List, Mapping, MutableMapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.requesters.error_handlers.default_http_response_filter import (
11
10
  DefaultHttpResponseFilter,
12
11
  )
@@ -5,7 +5,6 @@
5
5
  from typing import Optional, Union
6
6
 
7
7
  import requests
8
-
9
8
  from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import (
10
9
  HttpResponseFilter,
11
10
  )
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, Optional, Set, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.models import FailureType
11
10
  from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
12
11
  from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
@@ -6,8 +6,6 @@ from datetime import timedelta
6
6
  from typing import Any, Dict, Iterable, Mapping, Optional
7
7
 
8
8
  import requests
9
- from requests import Response
10
-
11
9
  from airbyte_cdk import AirbyteMessage
12
10
  from airbyte_cdk.logger import lazy_log
13
11
  from airbyte_cdk.models import FailureType, Type
@@ -25,6 +23,7 @@ from airbyte_cdk.sources.declarative.requesters.requester import Requester
25
23
  from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
26
24
  from airbyte_cdk.sources.types import Record, StreamSlice
27
25
  from airbyte_cdk.utils import AirbyteTracedException
26
+ from requests import Response
28
27
 
29
28
  LOGGER = logging.getLogger("airbyte")
30
29
 
@@ -9,7 +9,6 @@ from typing import Any, Callable, Mapping, MutableMapping, Optional, Union
9
9
  from urllib.parse import urljoin
10
10
 
11
11
  import requests
12
-
13
12
  from airbyte_cdk.sources.declarative.auth.declarative_authenticator import (
14
13
  DeclarativeAuthenticator,
15
14
  NoAuth,
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, Mapping, MutableMapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.decoders import (
11
10
  Decoder,
12
11
  JsonDecoder,
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, MutableMapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator
11
10
  from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
12
11
 
@@ -7,7 +7,6 @@ from dataclasses import dataclass
7
7
  from typing import Any, Mapping, Optional
8
8
 
9
9
  import requests
10
-
11
10
  from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
12
11
  RequestOptionsProvider,
13
12
  )
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, Dict, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.decoders import (
11
10
  Decoder,
12
11
  JsonDecoder,
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.decoders import (
11
10
  Decoder,
12
11
  JsonDecoder,
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
11
10
  from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
12
11
  PaginationStrategy,
@@ -7,7 +7,6 @@ from dataclasses import dataclass
7
7
  from typing import Any, Optional
8
8
 
9
9
  import requests
10
-
11
10
  from airbyte_cdk.sources.types import Record
12
11
 
13
12
 
@@ -6,12 +6,10 @@ from abc import ABC, abstractmethod
6
6
  from typing import Any, Optional
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
11
10
  from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
12
11
  PaginationStrategy,
13
12
  )
14
- from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor
15
13
  from airbyte_cdk.sources.types import Record
16
14
 
17
15
 
@@ -27,11 +25,7 @@ class PaginationStopCondition(ABC):
27
25
 
28
26
 
29
27
  class CursorStopCondition(PaginationStopCondition):
30
- def __init__(
31
- self,
32
- cursor: DeclarativeCursor
33
- | ConcurrentCursor, # migrate to use both old and concurrent versions
34
- ):
28
+ def __init__(self, cursor: DeclarativeCursor):
35
29
  self._cursor = cursor
36
30
 
37
31
  def is_met(self, record: Record) -> bool:
@@ -52,8 +46,8 @@ class StopConditionPaginationStrategyDecorator(PaginationStrategy):
52
46
  return None
53
47
  return self._delegate.next_page_token(response, last_page_size, last_record)
54
48
 
55
- def reset(self, reset_value: Optional[Any] = None) -> None:
56
- self._delegate.reset(reset_value)
49
+ def reset(self) -> None:
50
+ self._delegate.reset()
57
51
 
58
52
  def get_page_size(self) -> Optional[int]:
59
53
  return self._delegate.get_page_size()
@@ -5,8 +5,6 @@
5
5
  from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, Mapping, MutableMapping, Optional, Union
7
7
 
8
- from deprecated import deprecated
9
-
10
8
  from airbyte_cdk.sources.declarative.interpolation.interpolated_nested_mapping import NestedMapping
11
9
  from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_nested_request_input_provider import (
12
10
  InterpolatedNestedRequestInputProvider,
@@ -19,6 +17,7 @@ from airbyte_cdk.sources.declarative.requesters.request_options.request_options_
19
17
  )
20
18
  from airbyte_cdk.sources.source import ExperimentalClassWarning
21
19
  from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
20
+ from deprecated import deprecated
22
21
 
23
22
  RequestInput = Union[str, Mapping[str, str]]
24
23
  ValidRequestTypes = (str, list)
@@ -7,7 +7,6 @@ from enum import Enum
7
7
  from typing import Any, Callable, Mapping, MutableMapping, Optional, Union
8
8
 
9
9
  import requests
10
-
11
10
  from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
12
11
  from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
13
12
  RequestOptionsProvider,
@@ -4,8 +4,6 @@
4
4
  from dataclasses import InitVar, dataclass, field
5
5
  from typing import Any, Callable, Iterable, Mapping, Optional
6
6
 
7
- from deprecated.classic import deprecated
8
-
9
7
  from airbyte_cdk.models import FailureType
10
8
  from airbyte_cdk.sources.declarative.async_job.job_orchestrator import (
11
9
  AsyncJobOrchestrator,
@@ -19,6 +17,7 @@ from airbyte_cdk.sources.source import ExperimentalClassWarning
19
17
  from airbyte_cdk.sources.streams.core import StreamData
20
18
  from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
21
19
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException
20
+ from deprecated.classic import deprecated
22
21
 
23
22
 
24
23
  @deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
@@ -20,7 +20,6 @@ from typing import (
20
20
  )
21
21
 
22
22
  import requests
23
-
24
23
  from airbyte_cdk.models import AirbyteMessage
25
24
  from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
26
25
  from airbyte_cdk.sources.declarative.incremental import ResumableFullRefreshCursor
@@ -468,9 +467,8 @@ class SimpleRetriever(Retriever):
468
467
  else:
469
468
  return None
470
469
 
471
- def _extract_record(
472
- self, stream_data: StreamData, stream_slice: StreamSlice
473
- ) -> Optional[Record]:
470
+ @staticmethod
471
+ def _extract_record(stream_data: StreamData, stream_slice: StreamSlice) -> Optional[Record]:
474
472
  """
475
473
  As we allow the output of _read_pages to be StreamData, it can be multiple things. Therefore, we need to filter out and normalize
476
474
  to data to streamline the rest of the process.
@@ -479,15 +477,9 @@ class SimpleRetriever(Retriever):
479
477
  # Record is not part of `StreamData` but is the most common implementation of `Mapping[str, Any]` which is part of `StreamData`
480
478
  return stream_data
481
479
  elif isinstance(stream_data, (dict, Mapping)):
482
- return Record(
483
- data=dict(stream_data), associated_slice=stream_slice, stream_name=self.name
484
- )
480
+ return Record(dict(stream_data), stream_slice)
485
481
  elif isinstance(stream_data, AirbyteMessage) and stream_data.record:
486
- return Record(
487
- data=stream_data.record.data, # type:ignore # AirbyteMessage always has record.data
488
- associated_slice=stream_slice,
489
- stream_name=self.name,
490
- )
482
+ return Record(stream_data.record.data, stream_slice)
491
483
  return None
492
484
 
493
485
  # stream_slices is defined with arguments on http stream and fixing this has a long tail of dependencies. Will be resolved by the decoupling of http stream and simple retriever
@@ -1,13 +1,14 @@
1
1
  # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2
2
 
3
- from typing import Any, Callable, Iterable, Mapping, Optional
3
+ from typing import Iterable, Optional, Mapping, Any, Callable
4
4
 
5
5
  from airbyte_cdk.sources.declarative.retrievers import Retriever
6
6
  from airbyte_cdk.sources.message import MessageRepository
7
7
  from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
8
8
  from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator
9
+ from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
9
10
  from airbyte_cdk.sources.streams.concurrent.partitions.stream_slicer import StreamSlicer
10
- from airbyte_cdk.sources.types import Record, StreamSlice
11
+ from airbyte_cdk.sources.types import StreamSlice
11
12
  from airbyte_cdk.utils.slice_hasher import SliceHasher
12
13
 
13
14
 
@@ -58,11 +59,7 @@ class DeclarativePartition(Partition):
58
59
  def read(self) -> Iterable[Record]:
59
60
  for stream_data in self._retriever.read_records(self._json_schema, self._stream_slice):
60
61
  if isinstance(stream_data, Mapping):
61
- yield Record(
62
- data=stream_data,
63
- stream_name=self.stream_name(),
64
- associated_slice=self._stream_slice,
65
- )
62
+ yield Record(stream_data, self)
66
63
  else:
67
64
  self._message_repository.emit_message(stream_data)
68
65
 
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, Dict, List, Mapping, Optional, Type, Union
7
7
 
8
8
  import dpath
9
-
10
9
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
11
10
  from airbyte_cdk.sources.declarative.transformations import RecordTransformation
12
11
  from airbyte_cdk.sources.types import Config, FieldPointer, StreamSlice, StreamState
@@ -7,7 +7,6 @@ from typing import Any, Dict, List, Mapping, Optional
7
7
 
8
8
  import dpath
9
9
  import dpath.exceptions
10
-
11
10
  from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
12
11
  from airbyte_cdk.sources.declarative.transformations import RecordTransformation
13
12
  from airbyte_cdk.sources.types import Config, FieldPointer, StreamSlice, StreamState
@@ -6,7 +6,6 @@ import pkgutil
6
6
  from typing import Any, List, Mapping, Optional
7
7
 
8
8
  import yaml
9
-
10
9
  from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog
11
10
  from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
12
11
  ConcurrentDeclarativeSource,
@@ -5,7 +5,6 @@
5
5
  from typing import Any, Callable, Dict, Iterable, Optional
6
6
 
7
7
  import dpath
8
-
9
8
  from airbyte_cdk.models import AirbyteStream
10
9
 
11
10
 
@@ -7,11 +7,10 @@ from abc import abstractmethod
7
7
  from typing import Any, Dict, List, Literal, Optional, Union
8
8
 
9
9
  import dpath
10
- from pydantic.v1 import AnyUrl, BaseModel, Field
11
-
12
10
  from airbyte_cdk import OneOfOptionConfig
13
11
  from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
14
12
  from airbyte_cdk.sources.utils import schema_helpers
13
+ from pydantic.v1 import AnyUrl, BaseModel, Field
15
14
 
16
15
 
17
16
  class DeliverRecords(BaseModel):
@@ -3,9 +3,8 @@
3
3
  #
4
4
 
5
5
 
6
- from pydantic.v1 import BaseModel, Field
7
-
8
6
  from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
7
+ from pydantic.v1 import BaseModel, Field
9
8
 
10
9
 
11
10
  class AvroFormat(BaseModel):
@@ -6,11 +6,10 @@ import codecs
6
6
  from enum import Enum
7
7
  from typing import Any, Dict, List, Optional, Set, Union
8
8
 
9
+ from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
9
10
  from pydantic.v1 import BaseModel, Field, root_validator, validator
10
11
  from pydantic.v1.error_wrappers import ValidationError
11
12
 
12
- from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
13
-
14
13
 
15
14
  class InferenceType(Enum):
16
15
  NONE = "None"