airbyte-cdk 6.39.3__py3-none-any.whl → 6.40.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -116,6 +116,19 @@ definitions:
116
116
  type: array
117
117
  items:
118
118
  "$ref": "#/definitions/AddedFieldDefinition"
119
+ condition:
120
+ description: Fields will be added if expression is evaluated to True.
121
+ type: string
122
+ default: ""
123
+ interpolation_context:
124
+ - config
125
+ - property
126
+ - parameters
127
+ examples:
128
+ - "{{ property|string == '' }}"
129
+ - "{{ property is integer }}"
130
+ - "{{ property|length > 5 }}"
131
+ - "{{ property == 'some_string_to_match' }}"
119
132
  $parameters:
120
133
  type: object
121
134
  additionalProperties: true
@@ -2265,6 +2278,10 @@ definitions:
2265
2278
  title: Delete Origin Value
2266
2279
  description: Whether to delete the origin value or keep it. Default is False.
2267
2280
  type: boolean
2281
+ replace_record:
2282
+ title: Replace Origin Record
2283
+ description: Whether to replace the origin record or not. Default is False.
2284
+ type: boolean
2268
2285
  $parameters:
2269
2286
  type: object
2270
2287
  additionalProperties: true
@@ -2873,6 +2890,15 @@ definitions:
2873
2890
  type:
2874
2891
  type: string
2875
2892
  enum: [ParentStreamConfig]
2893
+ lazy_read_pointer:
2894
+ title: Lazy Read Pointer
2895
+ description: If set, this will enable lazy reading, using the initial read of parent records to extract child records.
2896
+ type: array
2897
+ default: [ ]
2898
+ items:
2899
+ - type: string
2900
+ interpolation_context:
2901
+ - config
2876
2902
  parent_key:
2877
2903
  title: Parent Key
2878
2904
  description: The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.
@@ -2991,6 +3017,10 @@ definitions:
2991
3017
  - "$ref": "#/definitions/SchemaNormalization"
2992
3018
  - "$ref": "#/definitions/CustomSchemaNormalization"
2993
3019
  default: None
3020
+ transform_before_filtering:
3021
+ description: If true, transformation will be applied before record filtering.
3022
+ type: boolean
3023
+ default: false
2994
3024
  $parameters:
2995
3025
  type: object
2996
3026
  additionalProperties: true
@@ -877,6 +877,11 @@ class DpathFlattenFields(BaseModel):
877
877
  description="Whether to delete the origin value or keep it. Default is False.",
878
878
  title="Delete Origin Value",
879
879
  )
880
+ replace_record: Optional[bool] = Field(
881
+ None,
882
+ description="Whether to replace the origin record or not. Default is False.",
883
+ title="Replace Origin Record",
884
+ )
880
885
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
881
886
 
882
887
 
@@ -1460,6 +1465,16 @@ class AddFields(BaseModel):
1460
1465
  description="List of transformations (path and corresponding value) that will be added to the record.",
1461
1466
  title="Fields",
1462
1467
  )
1468
+ condition: Optional[str] = Field(
1469
+ "",
1470
+ description="Fields will be added if expression is evaluated to True.,",
1471
+ examples=[
1472
+ "{{ property|string == '' }}",
1473
+ "{{ property is integer }}",
1474
+ "{{ property|length > 5 }}",
1475
+ "{{ property == 'some_string_to_match' }}",
1476
+ ],
1477
+ )
1463
1478
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1464
1479
 
1465
1480
 
@@ -1771,6 +1786,10 @@ class RecordSelector(BaseModel):
1771
1786
  description="Responsible for normalization according to the schema.",
1772
1787
  title="Schema Normalization",
1773
1788
  )
1789
+ transform_before_filtering: Optional[bool] = Field(
1790
+ False,
1791
+ description="If true, transformation will be applied before record filtering.",
1792
+ )
1774
1793
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1775
1794
 
1776
1795
 
@@ -2205,6 +2224,11 @@ class DynamicSchemaLoader(BaseModel):
2205
2224
 
2206
2225
  class ParentStreamConfig(BaseModel):
2207
2226
  type: Literal["ParentStreamConfig"]
2227
+ lazy_read_pointer: Optional[List[str]] = Field(
2228
+ [],
2229
+ description="If set, this will enable lazy reading, using the initial read of parent records to extract child records.",
2230
+ title="Lazy Read Pointer",
2231
+ )
2208
2232
  parent_key: str = Field(
2209
2233
  ...,
2210
2234
  description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.",
@@ -438,6 +438,7 @@ from airbyte_cdk.sources.declarative.resolvers import (
438
438
  )
439
439
  from airbyte_cdk.sources.declarative.retrievers import (
440
440
  AsyncRetriever,
441
+ LazySimpleRetriever,
441
442
  SimpleRetriever,
442
443
  SimpleRetrieverTestReadDecorator,
443
444
  )
@@ -712,7 +713,11 @@ class ModelToComponentFactory:
712
713
  )
713
714
  for added_field_definition_model in model.fields
714
715
  ]
715
- return AddFields(fields=added_field_definitions, parameters=model.parameters or {})
716
+ return AddFields(
717
+ fields=added_field_definitions,
718
+ condition=model.condition or "",
719
+ parameters=model.parameters or {},
720
+ )
716
721
 
717
722
  def create_keys_to_lower_transformation(
718
723
  self, model: KeysToLowerModel, config: Config, **kwargs: Any
@@ -748,6 +753,7 @@ class ModelToComponentFactory:
748
753
  delete_origin_value=model.delete_origin_value
749
754
  if model.delete_origin_value is not None
750
755
  else False,
756
+ replace_record=model.replace_record if model.replace_record is not None else False,
751
757
  parameters=model.parameters or {},
752
758
  )
753
759
 
@@ -1745,6 +1751,7 @@ class ModelToComponentFactory:
1745
1751
  transformations.append(
1746
1752
  self._create_component_from_model(model=transformation_model, config=config)
1747
1753
  )
1754
+
1748
1755
  retriever = self._create_component_from_model(
1749
1756
  model=model.retriever,
1750
1757
  config=config,
@@ -1755,6 +1762,7 @@ class ModelToComponentFactory:
1755
1762
  stop_condition_on_cursor=stop_condition_on_cursor,
1756
1763
  client_side_incremental_sync=client_side_incremental_sync,
1757
1764
  transformations=transformations,
1765
+ incremental_sync=model.incremental_sync,
1758
1766
  )
1759
1767
  cursor_field = model.incremental_sync.cursor_field if model.incremental_sync else None
1760
1768
 
@@ -1900,6 +1908,10 @@ class ModelToComponentFactory:
1900
1908
  ) -> Optional[StreamSlicer]:
1901
1909
  retriever_model = model.retriever
1902
1910
 
1911
+ stream_slicer = self._build_stream_slicer_from_partition_router(
1912
+ retriever_model, config, stream_name=model.name
1913
+ )
1914
+
1903
1915
  if retriever_model.type == "AsyncRetriever":
1904
1916
  is_not_datetime_cursor = (
1905
1917
  model.incremental_sync.type != "DatetimeBasedCursor"
@@ -1919,13 +1931,11 @@ class ModelToComponentFactory:
1919
1931
  "AsyncRetriever with cursor other than DatetimeBasedCursor is not supported yet."
1920
1932
  )
1921
1933
 
1922
- if is_partition_router:
1934
+ if is_partition_router and not stream_slicer:
1923
1935
  # Note that this development is also done in parallel to the per partition development which once merged
1924
1936
  # we could support here by calling create_concurrent_cursor_from_perpartition_cursor
1925
1937
  raise ValueError("Per partition state is not supported yet for AsyncRetriever.")
1926
1938
 
1927
- stream_slicer = self._build_stream_slicer_from_partition_router(retriever_model, config)
1928
-
1929
1939
  if model.incremental_sync:
1930
1940
  return self._build_incremental_cursor(model, stream_slicer, config)
1931
1941
 
@@ -2525,6 +2535,16 @@ class ModelToComponentFactory:
2525
2535
  if model.request_option
2526
2536
  else None
2527
2537
  )
2538
+
2539
+ if model.lazy_read_pointer and any("*" in pointer for pointer in model.lazy_read_pointer):
2540
+ raise ValueError(
2541
+ "The '*' wildcard in 'lazy_read_pointer' is not supported — only direct paths are allowed."
2542
+ )
2543
+
2544
+ model_lazy_read_pointer: List[Union[InterpolatedString, str]] = (
2545
+ [x for x in model.lazy_read_pointer] if model.lazy_read_pointer else []
2546
+ )
2547
+
2528
2548
  return ParentStreamConfig(
2529
2549
  parent_key=model.parent_key,
2530
2550
  request_option=request_option,
@@ -2534,6 +2554,7 @@ class ModelToComponentFactory:
2534
2554
  incremental_dependency=model.incremental_dependency or False,
2535
2555
  parameters=model.parameters or {},
2536
2556
  extra_fields=model.extra_fields,
2557
+ lazy_read_pointer=model_lazy_read_pointer,
2537
2558
  )
2538
2559
 
2539
2560
  @staticmethod
@@ -2593,7 +2614,9 @@ class ModelToComponentFactory:
2593
2614
  else None
2594
2615
  )
2595
2616
 
2596
- transform_before_filtering = False
2617
+ assert model.transform_before_filtering is not None # for mypy
2618
+
2619
+ transform_before_filtering = model.transform_before_filtering
2597
2620
  if client_side_incremental_sync:
2598
2621
  record_filter = ClientSideIncrementalRecordFilterDecorator(
2599
2622
  config=config,
@@ -2674,6 +2697,12 @@ class ModelToComponentFactory:
2674
2697
  stop_condition_on_cursor: bool = False,
2675
2698
  client_side_incremental_sync: Optional[Dict[str, Any]] = None,
2676
2699
  transformations: List[RecordTransformation],
2700
+ incremental_sync: Optional[
2701
+ Union[
2702
+ IncrementingCountCursorModel, DatetimeBasedCursorModel, CustomIncrementalSyncModel
2703
+ ]
2704
+ ] = None,
2705
+ **kwargs: Any,
2677
2706
  ) -> SimpleRetriever:
2678
2707
  decoder = (
2679
2708
  self._create_component_from_model(model=model.decoder, config=config)
@@ -2731,6 +2760,45 @@ class ModelToComponentFactory:
2731
2760
  model.ignore_stream_slicer_parameters_on_paginated_requests or False
2732
2761
  )
2733
2762
 
2763
+ if (
2764
+ model.partition_router
2765
+ and isinstance(model.partition_router, SubstreamPartitionRouterModel)
2766
+ and not bool(self._connector_state_manager.get_stream_state(name, None))
2767
+ and any(
2768
+ parent_stream_config.lazy_read_pointer
2769
+ for parent_stream_config in model.partition_router.parent_stream_configs
2770
+ )
2771
+ ):
2772
+ if incremental_sync:
2773
+ if incremental_sync.type != "DatetimeBasedCursor":
2774
+ raise ValueError(
2775
+ f"LazySimpleRetriever only supports DatetimeBasedCursor. Found: {incremental_sync.type}."
2776
+ )
2777
+
2778
+ elif incremental_sync.step or incremental_sync.cursor_granularity:
2779
+ raise ValueError(
2780
+ f"Found more that one slice per parent. LazySimpleRetriever only supports single slice read for stream - {name}."
2781
+ )
2782
+
2783
+ if model.decoder and model.decoder.type != "JsonDecoder":
2784
+ raise ValueError(
2785
+ f"LazySimpleRetriever only supports JsonDecoder. Found: {model.decoder.type}."
2786
+ )
2787
+
2788
+ return LazySimpleRetriever(
2789
+ name=name,
2790
+ paginator=paginator,
2791
+ primary_key=primary_key,
2792
+ requester=requester,
2793
+ record_selector=record_selector,
2794
+ stream_slicer=stream_slicer,
2795
+ request_option_provider=request_options_provider,
2796
+ cursor=cursor,
2797
+ config=config,
2798
+ ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests,
2799
+ parameters=model.parameters or {},
2800
+ )
2801
+
2734
2802
  if self._limit_slices_fetched or self._emit_connector_builder_messages:
2735
2803
  return SimpleRetrieverTestReadDecorator(
2736
2804
  name=name,
@@ -1,12 +1,16 @@
1
1
  #
2
2
  # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
3
  #
4
+
5
+
4
6
  import copy
7
+ import json
5
8
  import logging
6
9
  from dataclasses import InitVar, dataclass
7
10
  from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional, Union
8
11
 
9
12
  import dpath
13
+ import requests
10
14
 
11
15
  from airbyte_cdk.models import AirbyteMessage
12
16
  from airbyte_cdk.models import Type as MessageType
@@ -46,6 +50,7 @@ class ParentStreamConfig:
46
50
  )
47
51
  request_option: Optional[RequestOption] = None
48
52
  incremental_dependency: bool = False
53
+ lazy_read_pointer: Optional[List[Union[InterpolatedString, str]]] = None
49
54
 
50
55
  def __post_init__(self, parameters: Mapping[str, Any]) -> None:
51
56
  self.parent_key = InterpolatedString.create(self.parent_key, parameters=parameters)
@@ -59,6 +64,17 @@ class ParentStreamConfig:
59
64
  for key_path in self.extra_fields
60
65
  ]
61
66
 
67
+ self.lazy_read_pointer = (
68
+ [
69
+ InterpolatedString.create(path, parameters=parameters)
70
+ if isinstance(path, str)
71
+ else path
72
+ for path in self.lazy_read_pointer
73
+ ]
74
+ if self.lazy_read_pointer
75
+ else None
76
+ )
77
+
62
78
 
63
79
  @dataclass
64
80
  class SubstreamPartitionRouter(PartitionRouter):
@@ -196,6 +212,15 @@ class SubstreamPartitionRouter(PartitionRouter):
196
212
  # Add extra fields
197
213
  extracted_extra_fields = self._extract_extra_fields(parent_record, extra_fields)
198
214
 
215
+ if parent_stream_config.lazy_read_pointer:
216
+ extracted_extra_fields = {
217
+ "child_response": self._extract_child_response(
218
+ parent_record,
219
+ parent_stream_config.lazy_read_pointer, # type: ignore[arg-type] # lazy_read_pointer type handeled in __post_init__ of parent_stream_config
220
+ ),
221
+ **extracted_extra_fields,
222
+ }
223
+
199
224
  yield StreamSlice(
200
225
  partition={
201
226
  partition_field: partition_value,
@@ -205,6 +230,21 @@ class SubstreamPartitionRouter(PartitionRouter):
205
230
  extra_fields=extracted_extra_fields,
206
231
  )
207
232
 
233
+ def _extract_child_response(
234
+ self, parent_record: Mapping[str, Any] | AirbyteMessage, pointer: List[InterpolatedString]
235
+ ) -> requests.Response:
236
+ """Extract child records from a parent record based on lazy pointers."""
237
+
238
+ def _create_response(data: MutableMapping[str, Any]) -> SafeResponse:
239
+ """Create a SafeResponse with the given data."""
240
+ response = SafeResponse()
241
+ response.content = json.dumps(data).encode("utf-8")
242
+ response.status_code = 200
243
+ return response
244
+
245
+ path = [path.eval(self.config) for path in pointer]
246
+ return _create_response(dpath.get(parent_record, path, default=[])) # type: ignore # argunet will be a MutableMapping, given input data structure
247
+
208
248
  def _extract_extra_fields(
209
249
  self,
210
250
  parent_record: Mapping[str, Any] | AirbyteMessage,
@@ -280,20 +320,15 @@ class SubstreamPartitionRouter(PartitionRouter):
280
320
 
281
321
  parent_state = stream_state.get("parent_state", {})
282
322
 
283
- # If `parent_state` doesn't exist and at least one parent stream has an incremental dependency,
284
- # copy the child state to parent streams with incremental dependencies.
285
- incremental_dependency = any(
286
- [parent_config.incremental_dependency for parent_config in self.parent_stream_configs]
287
- )
288
- if not parent_state and not incremental_dependency:
289
- return
290
-
291
- if not parent_state and incremental_dependency:
292
- # Migrate child state to parent state format
293
- parent_state = self._migrate_child_state_to_parent_state(stream_state)
294
-
295
323
  # Set state for each parent stream with an incremental dependency
296
324
  for parent_config in self.parent_stream_configs:
325
+ if (
326
+ not parent_state.get(parent_config.stream.name, {})
327
+ and parent_config.incremental_dependency
328
+ ):
329
+ # Migrate child state to parent state format
330
+ parent_state = self._migrate_child_state_to_parent_state(stream_state)
331
+
297
332
  if parent_config.incremental_dependency:
298
333
  parent_config.stream.state = parent_state.get(parent_config.stream.name, {})
299
334
 
@@ -381,3 +416,22 @@ class SubstreamPartitionRouter(PartitionRouter):
381
416
  @property
382
417
  def logger(self) -> logging.Logger:
383
418
  return logging.getLogger("airbyte.SubstreamPartitionRouter")
419
+
420
+
421
+ class SafeResponse(requests.Response):
422
+ """
423
+ A subclass of requests.Response that acts as an interface to migrate parsed child records
424
+ into a response object. This allows seamless interaction with child records as if they
425
+ were original response, ensuring compatibility with methods that expect requests.Response data type.
426
+ """
427
+
428
+ def __getattr__(self, name: str) -> Any:
429
+ return getattr(requests.Response, name, None)
430
+
431
+ @property
432
+ def content(self) -> Optional[bytes]:
433
+ return super().content
434
+
435
+ @content.setter
436
+ def content(self, value: Union[str, bytes]) -> None:
437
+ self._content = value.encode() if isinstance(value, str) else value
@@ -71,7 +71,6 @@ class CursorPaginationStrategy(PaginationStrategy):
71
71
  last_page_token_value: Optional[Any] = None,
72
72
  ) -> Optional[Any]:
73
73
  decoded_response = next(self.decoder.decode(response))
74
-
75
74
  # The default way that link is presented in requests.Response is a string of various links (last, next, etc). This
76
75
  # is not indexable or useful for parsing the cursor, so we replace it with the link dictionary from response.links
77
76
  headers: Dict[str, Any] = dict(response.headers)
@@ -5,6 +5,7 @@
5
5
  from airbyte_cdk.sources.declarative.retrievers.async_retriever import AsyncRetriever
6
6
  from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
7
7
  from airbyte_cdk.sources.declarative.retrievers.simple_retriever import (
8
+ LazySimpleRetriever,
8
9
  SimpleRetriever,
9
10
  SimpleRetrieverTestReadDecorator,
10
11
  )
@@ -14,4 +15,5 @@ __all__ = [
14
15
  "SimpleRetriever",
15
16
  "SimpleRetrieverTestReadDecorator",
16
17
  "AsyncRetriever",
18
+ "LazySimpleRetriever",
17
19
  ]
@@ -6,9 +6,20 @@ import json
6
6
  from dataclasses import InitVar, dataclass, field
7
7
  from functools import partial
8
8
  from itertools import islice
9
- from typing import Any, Callable, Iterable, List, Mapping, Optional, Set, Tuple, Union
9
+ from typing import (
10
+ Any,
11
+ Callable,
12
+ Iterable,
13
+ List,
14
+ Mapping,
15
+ Optional,
16
+ Set,
17
+ Tuple,
18
+ Union,
19
+ )
10
20
 
11
21
  import requests
22
+ from typing_extensions import deprecated
12
23
 
13
24
  from airbyte_cdk.models import AirbyteMessage
14
25
  from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
@@ -28,6 +39,7 @@ from airbyte_cdk.sources.declarative.requesters.requester import Requester
28
39
  from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
29
40
  from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer
30
41
  from airbyte_cdk.sources.http_logger import format_http_message
42
+ from airbyte_cdk.sources.source import ExperimentalClassWarning
31
43
  from airbyte_cdk.sources.streams.core import StreamData
32
44
  from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
33
45
  from airbyte_cdk.utils.mapping_helpers import combine_mappings
@@ -438,8 +450,8 @@ class SimpleRetriever(Retriever):
438
450
  most_recent_record_from_slice = None
439
451
  record_generator = partial(
440
452
  self._parse_records,
453
+ stream_slice=stream_slice,
441
454
  stream_state=self.state or {},
442
- stream_slice=_slice,
443
455
  records_schema=records_schema,
444
456
  )
445
457
 
@@ -618,3 +630,73 @@ class SimpleRetrieverTestReadDecorator(SimpleRetriever):
618
630
  self.name,
619
631
  ),
620
632
  )
633
+
634
+
635
+ @deprecated(
636
+ "This class is experimental. Use at your own risk.",
637
+ category=ExperimentalClassWarning,
638
+ )
639
+ @dataclass
640
+ class LazySimpleRetriever(SimpleRetriever):
641
+ """
642
+ A retriever that supports lazy loading from parent streams.
643
+ """
644
+
645
+ def _read_pages(
646
+ self,
647
+ records_generator_fn: Callable[[Optional[requests.Response]], Iterable[Record]],
648
+ stream_state: Mapping[str, Any],
649
+ stream_slice: StreamSlice,
650
+ ) -> Iterable[Record]:
651
+ response = stream_slice.extra_fields["child_response"]
652
+ if response:
653
+ last_page_size, last_record = 0, None
654
+ for record in records_generator_fn(response): # type: ignore[call-arg] # only _parse_records expected as a func
655
+ last_page_size += 1
656
+ last_record = record
657
+ yield record
658
+
659
+ next_page_token = self._next_page_token(response, last_page_size, last_record, None)
660
+ if next_page_token:
661
+ yield from self._paginate(
662
+ next_page_token,
663
+ records_generator_fn,
664
+ stream_state,
665
+ stream_slice,
666
+ )
667
+
668
+ yield from []
669
+ else:
670
+ yield from self._read_pages(records_generator_fn, stream_state, stream_slice)
671
+
672
+ def _paginate(
673
+ self,
674
+ next_page_token: Any,
675
+ records_generator_fn: Callable[[Optional[requests.Response]], Iterable[Record]],
676
+ stream_state: Mapping[str, Any],
677
+ stream_slice: StreamSlice,
678
+ ) -> Iterable[Record]:
679
+ """Handle pagination by fetching subsequent pages."""
680
+ pagination_complete = False
681
+
682
+ while not pagination_complete:
683
+ response = self._fetch_next_page(stream_state, stream_slice, next_page_token)
684
+ last_page_size, last_record = 0, None
685
+
686
+ for record in records_generator_fn(response): # type: ignore[call-arg] # only _parse_records expected as a func
687
+ last_page_size += 1
688
+ last_record = record
689
+ yield record
690
+
691
+ if not response:
692
+ pagination_complete = True
693
+ else:
694
+ last_page_token_value = (
695
+ next_page_token.get("next_page_token") if next_page_token else None
696
+ )
697
+ next_page_token = self._next_page_token(
698
+ response, last_page_size, last_record, last_page_token_value
699
+ )
700
+
701
+ if not next_page_token:
702
+ pagination_complete = True
@@ -1,5 +1,5 @@
1
1
  #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
2
+ # Copyright (c) 2025 Airbyte, Inc., all rights reserved.
3
3
  #
4
4
 
5
5
  from dataclasses import InitVar, dataclass, field
@@ -7,6 +7,7 @@ from typing import Any, Dict, List, Mapping, Optional, Type, Union
7
7
 
8
8
  import dpath
9
9
 
10
+ from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
10
11
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
11
12
  from airbyte_cdk.sources.declarative.transformations import RecordTransformation
12
13
  from airbyte_cdk.sources.types import Config, FieldPointer, StreamSlice, StreamState
@@ -86,11 +87,16 @@ class AddFields(RecordTransformation):
86
87
 
87
88
  fields: List[AddedFieldDefinition]
88
89
  parameters: InitVar[Mapping[str, Any]]
90
+ condition: str = ""
89
91
  _parsed_fields: List[ParsedAddFieldDefinition] = field(
90
92
  init=False, repr=False, default_factory=list
91
93
  )
92
94
 
93
95
  def __post_init__(self, parameters: Mapping[str, Any]) -> None:
96
+ self._filter_interpolator = InterpolatedBoolean(
97
+ condition=self.condition, parameters=parameters
98
+ )
99
+
94
100
  for add_field in self.fields:
95
101
  if len(add_field.path) < 1:
96
102
  raise ValueError(
@@ -132,7 +138,9 @@ class AddFields(RecordTransformation):
132
138
  for parsed_field in self._parsed_fields:
133
139
  valid_types = (parsed_field.value_type,) if parsed_field.value_type else None
134
140
  value = parsed_field.value.eval(config, valid_types=valid_types, **kwargs)
135
- dpath.new(record, parsed_field.path, value)
141
+ is_empty_condition = not self.condition
142
+ if is_empty_condition or self._filter_interpolator.eval(config, value=value, **kwargs):
143
+ dpath.new(record, parsed_field.path, value)
136
144
 
137
145
  def __eq__(self, other: Any) -> bool:
138
146
  return bool(self.__dict__ == other.__dict__)
@@ -15,6 +15,7 @@ class DpathFlattenFields(RecordTransformation):
15
15
 
16
16
  field_path: List[Union[InterpolatedString, str]] path to the field to flatten.
17
17
  delete_origin_value: bool = False whether to delete origin field or keep it. Default is False.
18
+ replace_record: bool = False whether to replace origin record or not. Default is False.
18
19
 
19
20
  """
20
21
 
@@ -22,6 +23,7 @@ class DpathFlattenFields(RecordTransformation):
22
23
  field_path: List[Union[InterpolatedString, str]]
23
24
  parameters: InitVar[Mapping[str, Any]]
24
25
  delete_origin_value: bool = False
26
+ replace_record: bool = False
25
27
 
26
28
  def __post_init__(self, parameters: Mapping[str, Any]) -> None:
27
29
  self._field_path = [
@@ -48,8 +50,12 @@ class DpathFlattenFields(RecordTransformation):
48
50
  extracted = dpath.get(record, path, default=[])
49
51
 
50
52
  if isinstance(extracted, dict):
51
- conflicts = set(extracted.keys()) & set(record.keys())
52
- if not conflicts:
53
- if self.delete_origin_value:
54
- dpath.delete(record, path)
53
+ if self.replace_record and extracted:
54
+ dpath.delete(record, "**")
55
55
  record.update(extracted)
56
+ else:
57
+ conflicts = set(extracted.keys()) & set(record.keys())
58
+ if not conflicts:
59
+ if self.delete_origin_value:
60
+ dpath.delete(record, path)
61
+ record.update(extracted)
@@ -247,7 +247,7 @@ class AbstractOauth2Authenticator(AuthBase):
247
247
  access_key = self._extract_access_token(response_data)
248
248
  if not access_key:
249
249
  raise Exception(
250
- "Token refresh API response was missing access token {self.get_access_token_name()}"
250
+ f"Token refresh API response was missing access token {self.get_access_token_name()}"
251
251
  )
252
252
  # Add the access token to the list of secrets so it is replaced before logging the response
253
253
  # An argument could be made to remove the prevous access key from the list of secrets, but unmasking values seems like a security incident waiting to happen...
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: airbyte-cdk
3
- Version: 6.39.3
3
+ Version: 6.40.0
4
4
  Summary: A framework for writing Airbyte Connectors.
5
5
  Home-page: https://airbyte.com
6
6
  License: MIT
@@ -71,7 +71,7 @@ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=0I1lOxV7
71
71
  airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
72
72
  airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
73
73
  airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
74
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=TW-7fw3OXmp8hZmbvGJiW_5SCU4f6bzJbTIveQkAPZE,149239
74
+ airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=5lAt9rGpiUElT16jybLALS3DMkyLrz7JNUeyv2nv5c0,150310
75
75
  airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
76
76
  airbyte_cdk/sources/declarative/declarative_stream.py,sha256=dCRlddBUSaJmBNBz1pSO1r2rTw8AP5d2_vlmIeGs2gg,10767
77
77
  airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
@@ -114,20 +114,20 @@ airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW
114
114
  airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
115
115
  airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
116
116
  airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
117
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=FIq1hfhFjMmGB2gnTo7mohxgpwmH-8C4wuiEPKxH_hQ,105591
117
+ airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=uO-NMBY90yb8Kg_SdGTsXgerUKAKBM6rsWovXbvPclI,106527
118
118
  airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
119
119
  airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=jDw_TttD3_hpfevXOH-0Ws0eRuqt6wvED0BqosGPRjI,5938
120
120
  airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
121
121
  airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
122
122
  airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
123
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=Qe28QMKiAWEWeeybPqseSL5xm-_qrgjMY-lFn4VDtJM,143588
123
+ airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=bKo2WsTSNAQkTtcjVSXniwjgLNYaD3Lx_9vM02rakYU,146478
124
124
  airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=HJ-Syp3p7RpyR_OK0X_a2kSyISfu3W-PKrRI16iY0a8,957
125
125
  airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
126
126
  airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
127
127
  airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=tmGGpMoOBmaMfhVZq53AEWxoHm2lmNVi6hA2_IVEnAA,4882
128
128
  airbyte_cdk/sources/declarative/partition_routers/partition_router.py,sha256=YyEIzdmLd1FjbVP3QbQ2VFCLW_P-OGbVh6VpZShp54k,2218
129
129
  airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py,sha256=SKzKjSyfccq4dxGIh-J6ejrgkCHzaiTIazmbmeQiRD4,1942
130
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py,sha256=LlWj-Ofs-xfjlqmDzH8OYpyblP2Pb8bPDdR9g1UZyt0,17693
130
+ airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py,sha256=C15zFH0r4uHZ7otsrm46lHy93uT0vJn1VGs7maFHOHA,19800
131
131
  airbyte_cdk/sources/declarative/requesters/README.md,sha256=DQll2qsIzzTiiP35kJp16ONpr7cFeUQNgPfhl5krB24,2675
132
132
  airbyte_cdk/sources/declarative/requesters/__init__.py,sha256=d7a3OoHbqaJDyyPli3nqqJ2yAW_SLX6XDaBAKOwvpxw,364
133
133
  airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py,sha256=SkEDcJxlT1683rNx93K9whoS0OyUukkuOfToGtgpF58,776
@@ -150,7 +150,7 @@ airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha25
150
150
  airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py,sha256=b1-zKxYOUMHn7ahdWpzKEzfG4A7s_WQWy-vzRqZWzME,2152
151
151
  airbyte_cdk/sources/declarative/requesters/paginators/paginator.py,sha256=TzJF1Q-CFlsHF9lMSfmnGCxRYm9_UQCmBcHYQpc7F30,2376
152
152
  airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py,sha256=2gly8fuZpDNwtu1Qg6oE2jBLGqQRdzSLJdnpk_iDV6I,767
153
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py,sha256=yLzzK5YIRTkXd2Z-BS__AZXuTd6HXjJIxq05K-lQoxI,3898
153
+ airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py,sha256=cOURIXaJLCGQfrDP9A7mtSKIb9rVx7WU1V4dvcEc6sw,3897
154
154
  airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py,sha256=WvGt_DTFcAgTR-NHrlrR7B71yG-L6jmfW-Gwm9iYzjY,3624
155
155
  airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py,sha256=Z2i6a-oKMmOTxHxsTVSnyaShkJ3u8xZw1xIJdx2yxss,2731
156
156
  airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py,sha256=ZBshGQNr5Bb_V8dqnWRISqdXFcjm1CKIXnlfbRhNl8g,1308
@@ -169,10 +169,10 @@ airbyte_cdk/sources/declarative/resolvers/__init__.py,sha256=NiDcz5qi8HPsfX94MUm
169
169
  airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb9artL4ZkeqN8RmEykHH6FJgqXD7fCEnh1X0,1936
170
170
  airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=dz4iJV9liD_LzY_Mn4XmAStoUll60R3MIGWV4aN3pgg,5223
171
171
  airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
172
- airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=U9Hf9OK1bWdVa3cgs2cJm_-O-wOKuvhmRzP3SckL3rg,475
172
+ airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=nQepwG_RfW53sgwvK5dLPqfCx0VjsQ83nYoPjBMAaLM,527
173
173
  airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=Fxwg53i_9R3kMNFtD3gEwZbdW8xlcXYXA5evEhrKunM,5072
174
174
  airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
175
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=fDhc6dMx75UImh1_TfLm4Le59tsHpqIUZnau7uIJyYw,25043
175
+ airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=p6O4FYS7zzPq6uQT2NVnughUjI66tePaXVlyhCAyyv0,27746
176
176
  airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
177
177
  airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=KTACrIE23a83wsm3Rd9Eb4K6-20lrGqYxTHNp9yxsso,1820
178
178
  airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=J8Q_iJYhcSQLWyt0bTZCbDAGpxt9G8FCc6Q9jtGsNzw,10703
@@ -185,8 +185,8 @@ airbyte_cdk/sources/declarative/stream_slicers/__init__.py,sha256=sI9vhc95RwJYOn
185
185
  airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=RW1Q44ml-VWeMl4lNcV6EfyzrzCZkjj-hd0Omx_n_n4,3405
186
186
  airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py,sha256=SOkIPBi2Wu7yxIvA15yFzUAB95a3IzA8LPq5DEqHQQc,725
187
187
  airbyte_cdk/sources/declarative/transformations/__init__.py,sha256=CPJ8TlMpiUmvG3624VYu_NfTzxwKcfBjM2Q2wJ7fkSA,919
188
- airbyte_cdk/sources/declarative/transformations/add_fields.py,sha256=7UHCGc4xOxkYs5iXbPAPrP3-IEY60A-Go8QushsmaqY,4959
189
- airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py,sha256=1A-DWGjMqY4ggzRUZsZ3Sjrt-xsNgwUo5c72sSc5OZ0,2077
188
+ airbyte_cdk/sources/declarative/transformations/add_fields.py,sha256=vxLh0ekB0i_m8GYFpSad9T4S7eRxxtqZaigHLGVoltA,5366
189
+ airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py,sha256=I8oXPAOFhBV1mW_ufMn8Ii7oMbtect0sfLcpBNrKzzw,2374
190
190
  airbyte_cdk/sources/declarative/transformations/flatten_fields.py,sha256=yT3owG6rMKaRX-LJ_T-jSTnh1B5NoAHyH4YZN9yOvE8,1758
191
191
  airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py,sha256=vbIn6ump-Ut6g20yMub7PFoPBhOKVtrHSAUdcOUdLfw,1999
192
192
  airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py,sha256=RTs5KX4V3hM7A6QN1WlGF21YccTIyNH6qQI9IMb__hw,670
@@ -301,7 +301,7 @@ airbyte_cdk/sources/streams/http/http.py,sha256=0uariNq8OFnlX7iqOHwBhecxA-Hfd5hS
301
301
  airbyte_cdk/sources/streams/http/http_client.py,sha256=tDE0ROtxjGMVphvsw8INvGMtZ97hIF-v47pZ3jIyiwc,23011
302
302
  airbyte_cdk/sources/streams/http/rate_limiting.py,sha256=IwdjrHKUnU97XO4qONgYRv4YYW51xQ8SJm4WLafXDB8,6351
303
303
  airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
304
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=P9U8vtcrZ3m0InSG2W0H4gTYTxjQxkIe6mhF9xvO8Ug,18824
304
+ airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=XLVbJNSSnjECQKq7mMxYp1Pi5kbpAp2VmGq0fwMuDRc,18825
305
305
  airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py,sha256=Y3n7J-sk5yGjv_OxtY6Z6k0PEsFZmtIRi-x0KCbaHdA,1010
306
306
  airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=C2j2uVfi9d-3KgHO3NGxIiFdfASjHOtsd6g_LWPYOAs,20311
307
307
  airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=h5PTzcdH-RQLeCg7xZ45w_484OPUDSwNWl_iMJQmZoI,2526
@@ -358,9 +358,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
358
358
  airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
359
359
  airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
360
360
  airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
361
- airbyte_cdk-6.39.3.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
362
- airbyte_cdk-6.39.3.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
363
- airbyte_cdk-6.39.3.dist-info/METADATA,sha256=oToy3NTEXtxefbvn_Nu0CuDIH_EjOpwwZ8hmUrmsZ14,6071
364
- airbyte_cdk-6.39.3.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
365
- airbyte_cdk-6.39.3.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
366
- airbyte_cdk-6.39.3.dist-info/RECORD,,
361
+ airbyte_cdk-6.40.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
362
+ airbyte_cdk-6.40.0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
363
+ airbyte_cdk-6.40.0.dist-info/METADATA,sha256=7nwkCiFCxc6pLHqUlVFT8-02vm15pN2KI2zztPuucyg,6071
364
+ airbyte_cdk-6.40.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
365
+ airbyte_cdk-6.40.0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
366
+ airbyte_cdk-6.40.0.dist-info/RECORD,,