airbyte-cdk 6.33.2.dev1__py3-none-any.whl → 6.33.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,6 @@
3
3
 
4
4
  from __future__ import annotations
5
5
 
6
- from datetime import datetime, timedelta
7
6
  from enum import Enum
8
7
  from typing import Any, Dict, List, Literal, Optional, Union
9
8
 
@@ -652,8 +651,11 @@ class Rate(BaseModel):
652
651
  description="The maximum number of calls allowed within the interval.",
653
652
  title="Limit",
654
653
  )
655
- interval: timedelta = Field(
656
- ..., description="The time interval for the rate limit.", title="Interval"
654
+ interval: str = Field(
655
+ ...,
656
+ description="The time interval for the rate limit.",
657
+ examples=["PT1H", "P1D"],
658
+ title="Interval",
657
659
  )
658
660
 
659
661
 
@@ -927,15 +929,6 @@ class CustomDecoder(BaseModel):
927
929
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
928
930
 
929
931
 
930
- class GzipJsonDecoder(BaseModel):
931
- class Config:
932
- extra = Extra.allow
933
-
934
- type: Literal["GzipJsonDecoder"]
935
- encoding: Optional[str] = "utf-8"
936
- parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
937
-
938
-
939
932
  class MinMaxDatetime(BaseModel):
940
933
  type: Literal["MinMaxDatetime"]
941
934
  datetime: str = Field(
@@ -1314,18 +1307,8 @@ class LegacySessionTokenAuthenticator(BaseModel):
1314
1307
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1315
1308
 
1316
1309
 
1317
- class JsonParser(BaseModel):
1318
- type: Literal["JsonParser"]
1319
- encoding: Optional[str] = "utf-8"
1320
-
1321
-
1322
- class JsonLineParser(BaseModel):
1323
- type: Literal["JsonLineParser"]
1324
- encoding: Optional[str] = "utf-8"
1325
-
1326
-
1327
- class CsvParser(BaseModel):
1328
- type: Literal["CsvParser"]
1310
+ class CsvDecoder(BaseModel):
1311
+ type: Literal["CsvDecoder"]
1329
1312
  encoding: Optional[str] = "utf-8"
1330
1313
  delimiter: Optional[str] = ","
1331
1314
 
@@ -1629,12 +1612,7 @@ class FixedWindowCallRatePolicy(BaseModel):
1629
1612
  extra = Extra.allow
1630
1613
 
1631
1614
  type: Literal["FixedWindowCallRatePolicy"]
1632
- next_reset_ts: datetime = Field(
1633
- ...,
1634
- description="The timestamp when the rate limit will reset.",
1635
- title="Next Reset Timestamp",
1636
- )
1637
- period: timedelta = Field(
1615
+ period: str = Field(
1638
1616
  ..., description="The time interval for the rate limit window.", title="Period"
1639
1617
  )
1640
1618
  call_limit: int = Field(
@@ -1774,9 +1752,9 @@ class RecordSelector(BaseModel):
1774
1752
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1775
1753
 
1776
1754
 
1777
- class GzipParser(BaseModel):
1778
- type: Literal["GzipParser"]
1779
- inner_parser: Union[JsonLineParser, CsvParser, JsonParser]
1755
+ class GzipDecoder(BaseModel):
1756
+ type: Literal["GzipDecoder"]
1757
+ decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
1780
1758
 
1781
1759
 
1782
1760
  class Spec(BaseModel):
@@ -1809,29 +1787,6 @@ class CompositeErrorHandler(BaseModel):
1809
1787
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1810
1788
 
1811
1789
 
1812
- class APIBudget(BaseModel):
1813
- class Config:
1814
- extra = Extra.allow
1815
-
1816
- type: Literal["APIBudget"]
1817
- policies: List[
1818
- Union[
1819
- FixedWindowCallRatePolicy,
1820
- MovingWindowCallRatePolicy,
1821
- UnlimitedCallRatePolicy,
1822
- ]
1823
- ] = Field(
1824
- ...,
1825
- description="List of call rate policies that define how many calls are allowed.",
1826
- title="Policies",
1827
- )
1828
- maximum_attempts_to_acquire: Optional[int] = Field(
1829
- 100000,
1830
- description="The maximum number of attempts to acquire a call before giving up.",
1831
- title="Maximum Attempts to Acquire",
1832
- )
1833
-
1834
-
1835
1790
  class HTTPAPIBudget(BaseModel):
1836
1791
  class Config:
1837
1792
  extra = Extra.allow
@@ -1863,11 +1818,6 @@ class HTTPAPIBudget(BaseModel):
1863
1818
  description="List of HTTP status codes that indicate a rate limit has been hit.",
1864
1819
  title="Status Codes for Rate Limit Hit",
1865
1820
  )
1866
- maximum_attempts_to_acquire: Optional[int] = Field(
1867
- 100000,
1868
- description="The maximum number of attempts to acquire a call before giving up.",
1869
- title="Maximum Attempts to Acquire",
1870
- )
1871
1821
 
1872
1822
 
1873
1823
  class ZipfileDecoder(BaseModel):
@@ -1875,18 +1825,13 @@ class ZipfileDecoder(BaseModel):
1875
1825
  extra = Extra.allow
1876
1826
 
1877
1827
  type: Literal["ZipfileDecoder"]
1878
- parser: Union[GzipParser, JsonParser, JsonLineParser, CsvParser] = Field(
1828
+ decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field(
1879
1829
  ...,
1880
1830
  description="Parser to parse the decompressed data from the zipfile(s).",
1881
1831
  title="Parser",
1882
1832
  )
1883
1833
 
1884
1834
 
1885
- class CompositeRawDecoder(BaseModel):
1886
- type: Literal["CompositeRawDecoder"]
1887
- parser: Union[GzipParser, JsonParser, JsonLineParser, CsvParser]
1888
-
1889
-
1890
1835
  class DeclarativeSource1(BaseModel):
1891
1836
  class Config:
1892
1837
  extra = Extra.forbid
@@ -1903,11 +1848,7 @@ class DeclarativeSource1(BaseModel):
1903
1848
  definitions: Optional[Dict[str, Any]] = None
1904
1849
  spec: Optional[Spec] = None
1905
1850
  concurrency_level: Optional[ConcurrencyLevel] = None
1906
- api_budget: Optional[Union[APIBudget, HTTPAPIBudget]] = Field(
1907
- None,
1908
- description="Defines how many requests can be made to the API in a given time frame. This field accepts either a generic APIBudget or an HTTP-specific configuration (HTTPAPIBudget) to be applied across all streams.",
1909
- title="API Budget",
1910
- )
1851
+ api_budget: Optional[HTTPAPIBudget] = None
1911
1852
  metadata: Optional[Dict[str, Any]] = Field(
1912
1853
  None,
1913
1854
  description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
@@ -1934,11 +1875,7 @@ class DeclarativeSource2(BaseModel):
1934
1875
  definitions: Optional[Dict[str, Any]] = None
1935
1876
  spec: Optional[Spec] = None
1936
1877
  concurrency_level: Optional[ConcurrencyLevel] = None
1937
- api_budget: Optional[Union[APIBudget, HTTPAPIBudget]] = Field(
1938
- None,
1939
- description="Defines how many requests can be made to the API in a given time frame. This field accepts either a generic APIBudget or an HTTP-specific configuration (HTTPAPIBudget) to be applied across all streams.",
1940
- title="API Budget",
1941
- )
1878
+ api_budget: Optional[HTTPAPIBudget] = None
1942
1879
  metadata: Optional[Dict[str, Any]] = Field(
1943
1880
  None,
1944
1881
  description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
@@ -2098,7 +2035,7 @@ class SessionTokenAuthenticator(BaseModel):
2098
2035
  description="Authentication method to use for requests sent to the API, specifying how to inject the session token.",
2099
2036
  title="Data Request Authentication",
2100
2037
  )
2101
- decoder: Optional[Union[JsonDecoder, XmlDecoder, CompositeRawDecoder]] = Field(
2038
+ decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field(
2102
2039
  None, description="Component used to decode the response.", title="Decoder"
2103
2040
  )
2104
2041
  parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
@@ -2298,12 +2235,12 @@ class SimpleRetriever(BaseModel):
2298
2235
  decoder: Optional[
2299
2236
  Union[
2300
2237
  CustomDecoder,
2238
+ CsvDecoder,
2239
+ GzipDecoder,
2301
2240
  JsonDecoder,
2302
2241
  JsonlDecoder,
2303
2242
  IterableDecoder,
2304
2243
  XmlDecoder,
2305
- GzipJsonDecoder,
2306
- CompositeRawDecoder,
2307
2244
  ZipfileDecoder,
2308
2245
  ]
2309
2246
  ] = Field(
@@ -2376,12 +2313,12 @@ class AsyncRetriever(BaseModel):
2376
2313
  decoder: Optional[
2377
2314
  Union[
2378
2315
  CustomDecoder,
2316
+ CsvDecoder,
2317
+ GzipDecoder,
2379
2318
  JsonDecoder,
2380
2319
  JsonlDecoder,
2381
2320
  IterableDecoder,
2382
2321
  XmlDecoder,
2383
- GzipJsonDecoder,
2384
- CompositeRawDecoder,
2385
2322
  ZipfileDecoder,
2386
2323
  ]
2387
2324
  ] = Field(
@@ -2392,12 +2329,12 @@ class AsyncRetriever(BaseModel):
2392
2329
  download_decoder: Optional[
2393
2330
  Union[
2394
2331
  CustomDecoder,
2332
+ CsvDecoder,
2333
+ GzipDecoder,
2395
2334
  JsonDecoder,
2396
2335
  JsonlDecoder,
2397
2336
  IterableDecoder,
2398
2337
  XmlDecoder,
2399
- GzipJsonDecoder,
2400
- CompositeRawDecoder,
2401
2338
  ZipfileDecoder,
2402
2339
  ]
2403
2340
  ] = Field(
@@ -2442,6 +2379,7 @@ class DynamicDeclarativeStream(BaseModel):
2442
2379
 
2443
2380
 
2444
2381
  ComplexFieldType.update_forward_refs()
2382
+ GzipDecoder.update_forward_refs()
2445
2383
  CompositeErrorHandler.update_forward_refs()
2446
2384
  DeclarativeSource1.update_forward_refs()
2447
2385
  DeclarativeSource2.update_forward_refs()
@@ -60,10 +60,8 @@ from airbyte_cdk.sources.declarative.datetime import MinMaxDatetime
60
60
  from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
61
61
  from airbyte_cdk.sources.declarative.decoders import (
62
62
  Decoder,
63
- GzipJsonDecoder,
64
63
  IterableDecoder,
65
64
  JsonDecoder,
66
- JsonlDecoder,
67
65
  PaginationDecoderDecorator,
68
66
  XmlDecoder,
69
67
  ZipfileDecoder,
@@ -103,8 +101,8 @@ from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_mi
103
101
  LegacyToPerPartitionStateMigration,
104
102
  )
105
103
  from airbyte_cdk.sources.declarative.models import (
106
- Clamping,
107
104
  CustomStateMigration,
105
+ GzipDecoder,
108
106
  )
109
107
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
110
108
  AddedFieldDefinition as AddedFieldDefinitionModel,
@@ -112,9 +110,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
112
110
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
113
111
  AddFields as AddFieldsModel,
114
112
  )
115
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
116
- APIBudget as APIBudgetModel,
117
- )
118
113
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
119
114
  ApiKeyAuthenticator as ApiKeyAuthenticatorModel,
120
115
  )
@@ -145,9 +140,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
145
140
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
146
141
  CompositeErrorHandler as CompositeErrorHandlerModel,
147
142
  )
148
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
149
- CompositeRawDecoder as CompositeRawDecoderModel,
150
- )
151
143
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
152
144
  ConcurrencyLevel as ConcurrencyLevelModel,
153
145
  )
@@ -158,7 +150,7 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
158
150
  ConstantBackoffStrategy as ConstantBackoffStrategyModel,
159
151
  )
160
152
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
161
- CsvParser as CsvParserModel,
153
+ CsvDecoder as CsvDecoderModel,
162
154
  )
163
155
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
164
156
  CursorPagination as CursorPaginationModel,
@@ -236,10 +228,7 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
236
228
  FlattenFields as FlattenFieldsModel,
237
229
  )
238
230
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
239
- GzipJsonDecoder as GzipJsonDecoderModel,
240
- )
241
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
242
- GzipParser as GzipParserModel,
231
+ GzipDecoder as GzipDecoderModel,
243
232
  )
244
233
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
245
234
  HTTPAPIBudget as HTTPAPIBudgetModel,
@@ -271,12 +260,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
271
260
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
272
261
  JsonlDecoder as JsonlDecoderModel,
273
262
  )
274
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
275
- JsonLineParser as JsonLineParserModel,
276
- )
277
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
278
- JsonParser as JsonParserModel,
279
- )
280
263
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
281
264
  JwtAuthenticator as JwtAuthenticatorModel,
282
265
  )
@@ -562,9 +545,9 @@ class ModelToComponentFactory:
562
545
  CheckStreamModel: self.create_check_stream,
563
546
  CheckDynamicStreamModel: self.create_check_dynamic_stream,
564
547
  CompositeErrorHandlerModel: self.create_composite_error_handler,
565
- CompositeRawDecoderModel: self.create_composite_raw_decoder,
566
548
  ConcurrencyLevelModel: self.create_concurrency_level,
567
549
  ConstantBackoffStrategyModel: self.create_constant_backoff_strategy,
550
+ CsvDecoderModel: self.create_csv_decoder,
568
551
  CursorPaginationModel: self.create_cursor_pagination,
569
552
  CustomAuthenticatorModel: self.create_custom_component,
570
553
  CustomBackoffStrategyModel: self.create_custom_component,
@@ -594,10 +577,7 @@ class ModelToComponentFactory:
594
577
  InlineSchemaLoaderModel: self.create_inline_schema_loader,
595
578
  JsonDecoderModel: self.create_json_decoder,
596
579
  JsonlDecoderModel: self.create_jsonl_decoder,
597
- JsonLineParserModel: self.create_json_line_parser,
598
- JsonParserModel: self.create_json_parser,
599
- GzipJsonDecoderModel: self.create_gzipjson_decoder,
600
- GzipParserModel: self.create_gzip_parser,
580
+ GzipDecoderModel: self.create_gzip_decoder,
601
581
  KeysToLowerModel: self.create_keys_to_lower_transformation,
602
582
  KeysToSnakeCaseModel: self.create_keys_to_snake_transformation,
603
583
  KeysReplaceModel: self.create_keys_replace_transformation,
@@ -638,7 +618,6 @@ class ModelToComponentFactory:
638
618
  StreamConfigModel: self.create_stream_config,
639
619
  ComponentMappingDefinitionModel: self.create_components_mapping_definition,
640
620
  ZipfileDecoderModel: self.create_zipfile_decoder,
641
- APIBudgetModel: self.create_api_budget,
642
621
  HTTPAPIBudgetModel: self.create_http_api_budget,
643
622
  FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy,
644
623
  MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy,
@@ -972,6 +951,17 @@ class ModelToComponentFactory:
972
951
  parameters={},
973
952
  )
974
953
 
954
+ @staticmethod
955
+ def apply_stream_state_migrations(
956
+ stream_state_migrations: List[Any] | None, stream_state: MutableMapping[str, Any]
957
+ ) -> MutableMapping[str, Any]:
958
+ if stream_state_migrations:
959
+ for state_migration in stream_state_migrations:
960
+ if state_migration.should_migrate(stream_state):
961
+ # The state variable is expected to be mutable but the migrate method returns an immutable mapping.
962
+ stream_state = dict(state_migration.migrate(stream_state))
963
+ return stream_state
964
+
975
965
  def create_concurrent_cursor_from_datetime_based_cursor(
976
966
  self,
977
967
  model_type: Type[BaseModel],
@@ -981,6 +971,7 @@ class ModelToComponentFactory:
981
971
  config: Config,
982
972
  message_repository: Optional[MessageRepository] = None,
983
973
  runtime_lookback_window: Optional[datetime.timedelta] = None,
974
+ stream_state_migrations: Optional[List[Any]] = None,
984
975
  **kwargs: Any,
985
976
  ) -> ConcurrentCursor:
986
977
  # Per-partition incremental streams can dynamically create child cursors which will pass their current
@@ -991,6 +982,7 @@ class ModelToComponentFactory:
991
982
  if "stream_state" not in kwargs
992
983
  else kwargs["stream_state"]
993
984
  )
985
+ stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
994
986
 
995
987
  component_type = component_definition.get("type")
996
988
  if component_definition.get("type") != model_type.__name__:
@@ -1226,6 +1218,7 @@ class ModelToComponentFactory:
1226
1218
  config: Config,
1227
1219
  stream_state: MutableMapping[str, Any],
1228
1220
  partition_router: PartitionRouter,
1221
+ stream_state_migrations: Optional[List[Any]] = None,
1229
1222
  **kwargs: Any,
1230
1223
  ) -> ConcurrentPerPartitionCursor:
1231
1224
  component_type = component_definition.get("type")
@@ -1274,8 +1267,10 @@ class ModelToComponentFactory:
1274
1267
  stream_namespace=stream_namespace,
1275
1268
  config=config,
1276
1269
  message_repository=NoopMessageRepository(),
1270
+ stream_state_migrations=stream_state_migrations,
1277
1271
  )
1278
1272
  )
1273
+ stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
1279
1274
 
1280
1275
  # Return the concurrent cursor and state converter
1281
1276
  return ConcurrentPerPartitionCursor(
@@ -1784,6 +1779,7 @@ class ModelToComponentFactory:
1784
1779
  stream_name=model.name or "",
1785
1780
  stream_namespace=None,
1786
1781
  config=config or {},
1782
+ stream_state_migrations=model.state_migrations,
1787
1783
  )
1788
1784
  return (
1789
1785
  self._create_component_from_model(model=model.incremental_sync, config=config)
@@ -2092,25 +2088,26 @@ class ModelToComponentFactory:
2092
2088
  )
2093
2089
 
2094
2090
  @staticmethod
2095
- def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> JsonDecoder:
2091
+ def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> Decoder:
2096
2092
  return JsonDecoder(parameters={})
2097
2093
 
2098
2094
  @staticmethod
2099
- def create_json_parser(model: JsonParserModel, config: Config, **kwargs: Any) -> JsonParser:
2100
- encoding = model.encoding if model.encoding else "utf-8"
2101
- return JsonParser(encoding=encoding)
2095
+ def create_csv_decoder(model: CsvDecoderModel, config: Config, **kwargs: Any) -> Decoder:
2096
+ return CompositeRawDecoder(
2097
+ parser=ModelToComponentFactory._get_parser(model, config), stream_response=True
2098
+ )
2102
2099
 
2103
2100
  @staticmethod
2104
- def create_jsonl_decoder(
2105
- model: JsonlDecoderModel, config: Config, **kwargs: Any
2106
- ) -> JsonlDecoder:
2107
- return JsonlDecoder(parameters={})
2101
+ def create_jsonl_decoder(model: JsonlDecoderModel, config: Config, **kwargs: Any) -> Decoder:
2102
+ return CompositeRawDecoder(
2103
+ parser=ModelToComponentFactory._get_parser(model, config), stream_response=True
2104
+ )
2108
2105
 
2109
2106
  @staticmethod
2110
- def create_json_line_parser(
2111
- model: JsonLineParserModel, config: Config, **kwargs: Any
2112
- ) -> JsonLineParser:
2113
- return JsonLineParser(encoding=model.encoding)
2107
+ def create_gzip_decoder(model: GzipDecoderModel, config: Config, **kwargs: Any) -> Decoder:
2108
+ return CompositeRawDecoder(
2109
+ parser=ModelToComponentFactory._get_parser(model, config), stream_response=True
2110
+ )
2114
2111
 
2115
2112
  @staticmethod
2116
2113
  def create_iterable_decoder(
@@ -2122,33 +2119,30 @@ class ModelToComponentFactory:
2122
2119
  def create_xml_decoder(model: XmlDecoderModel, config: Config, **kwargs: Any) -> XmlDecoder:
2123
2120
  return XmlDecoder(parameters={})
2124
2121
 
2125
- @staticmethod
2126
- def create_gzipjson_decoder(
2127
- model: GzipJsonDecoderModel, config: Config, **kwargs: Any
2128
- ) -> GzipJsonDecoder:
2129
- return GzipJsonDecoder(parameters={}, encoding=model.encoding)
2130
-
2131
2122
  def create_zipfile_decoder(
2132
2123
  self, model: ZipfileDecoderModel, config: Config, **kwargs: Any
2133
2124
  ) -> ZipfileDecoder:
2134
- parser = self._create_component_from_model(model=model.parser, config=config)
2135
- return ZipfileDecoder(parser=parser)
2136
-
2137
- def create_gzip_parser(
2138
- self, model: GzipParserModel, config: Config, **kwargs: Any
2139
- ) -> GzipParser:
2140
- inner_parser = self._create_component_from_model(model=model.inner_parser, config=config)
2141
- return GzipParser(inner_parser=inner_parser)
2125
+ return ZipfileDecoder(parser=ModelToComponentFactory._get_parser(model.decoder, config))
2142
2126
 
2143
2127
  @staticmethod
2144
- def create_csv_parser(model: CsvParserModel, config: Config, **kwargs: Any) -> CsvParser:
2145
- return CsvParser(encoding=model.encoding, delimiter=model.delimiter)
2128
+ def _get_parser(model: BaseModel, config: Config) -> Parser:
2129
+ if isinstance(model, JsonDecoderModel):
2130
+ # Note that the logic is a bit different from the JsonDecoder as there is some legacy that is maintained to return {} on error cases
2131
+ return JsonParser()
2132
+ elif isinstance(model, JsonlDecoderModel):
2133
+ return JsonLineParser()
2134
+ elif isinstance(model, CsvDecoderModel):
2135
+ return CsvParser(encoding=model.encoding, delimiter=model.delimiter)
2136
+ elif isinstance(model, GzipDecoderModel):
2137
+ return GzipParser(
2138
+ inner_parser=ModelToComponentFactory._get_parser(model.decoder, config)
2139
+ )
2140
+ elif isinstance(
2141
+ model, (CustomDecoderModel, IterableDecoderModel, XmlDecoderModel, ZipfileDecoderModel)
2142
+ ):
2143
+ raise ValueError(f"Decoder type {model} does not have parser associated to it")
2146
2144
 
2147
- def create_composite_raw_decoder(
2148
- self, model: CompositeRawDecoderModel, config: Config, **kwargs: Any
2149
- ) -> CompositeRawDecoder:
2150
- parser = self._create_component_from_model(model=model.parser, config=config)
2151
- return CompositeRawDecoder(parser=parser)
2145
+ raise ValueError(f"Unknown decoder type {model}")
2152
2146
 
2153
2147
  @staticmethod
2154
2148
  def create_json_file_schema_loader(
@@ -2421,6 +2415,8 @@ class ModelToComponentFactory:
2421
2415
  if model.record_filter
2422
2416
  else None
2423
2417
  )
2418
+
2419
+ transform_before_filtering = False
2424
2420
  if client_side_incremental_sync:
2425
2421
  record_filter = ClientSideIncrementalRecordFilterDecorator(
2426
2422
  config=config,
@@ -2430,6 +2426,8 @@ class ModelToComponentFactory:
2430
2426
  else None,
2431
2427
  **client_side_incremental_sync,
2432
2428
  )
2429
+ transform_before_filtering = True
2430
+
2433
2431
  schema_normalization = (
2434
2432
  TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization])
2435
2433
  if isinstance(model.schema_normalization, SchemaNormalizationModel)
@@ -2444,6 +2442,7 @@ class ModelToComponentFactory:
2444
2442
  transformations=transformations or [],
2445
2443
  schema_normalization=schema_normalization,
2446
2444
  parameters=model.parameters or {},
2445
+ transform_before_filtering=transform_before_filtering,
2447
2446
  )
2448
2447
 
2449
2448
  @staticmethod
@@ -2965,17 +2964,6 @@ class ModelToComponentFactory:
2965
2964
  else:
2966
2965
  return False
2967
2966
 
2968
- def create_api_budget(self, model: APIBudgetModel, config: Config, **kwargs: Any) -> APIBudget:
2969
- policies = [
2970
- self._create_component_from_model(model=policy, config=config)
2971
- for policy in model.policies
2972
- ]
2973
-
2974
- return APIBudget(
2975
- policies=policies,
2976
- maximum_attempts_to_acquire=model.maximum_attempts_to_acquire or 100000,
2977
- )
2978
-
2979
2967
  def create_http_api_budget(
2980
2968
  self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any
2981
2969
  ) -> HttpAPIBudget:
@@ -2986,10 +2974,9 @@ class ModelToComponentFactory:
2986
2974
 
2987
2975
  return HttpAPIBudget(
2988
2976
  policies=policies,
2989
- maximum_attempts_to_acquire=model.maximum_attempts_to_acquire or 100000,
2990
2977
  ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset",
2991
2978
  ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining",
2992
- status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or (429,),
2979
+ status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429],
2993
2980
  )
2994
2981
 
2995
2982
  def create_fixed_window_call_rate_policy(
@@ -2999,9 +2986,12 @@ class ModelToComponentFactory:
2999
2986
  self._create_component_from_model(model=matcher, config=config)
3000
2987
  for matcher in model.matchers
3001
2988
  ]
2989
+
2990
+ # Set the initial reset timestamp to 10 days from now.
2991
+ # This value will be updated by the first request.
3002
2992
  return FixedWindowCallRatePolicy(
3003
- next_reset_ts=model.next_reset_ts,
3004
- period=model.period,
2993
+ next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10),
2994
+ period=parse_duration(model.period),
3005
2995
  call_limit=model.call_limit,
3006
2996
  matchers=matchers,
3007
2997
  )
@@ -3036,7 +3026,7 @@ class ModelToComponentFactory:
3036
3026
  def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate:
3037
3027
  return Rate(
3038
3028
  limit=model.limit,
3039
- interval=model.interval,
3029
+ interval=parse_duration(model.interval),
3040
3030
  )
3041
3031
 
3042
3032
  def create_http_request_matcher(
@@ -3051,16 +3041,6 @@ class ModelToComponentFactory:
3051
3041
  )
3052
3042
 
3053
3043
  def set_api_budget(self, component_definition: ComponentDefinition, config: Config) -> None:
3054
- model_str = component_definition.get("type")
3055
- if model_str == "APIBudget":
3056
- # Annotate model_type as a type that is a subclass of BaseModel
3057
- model_type: Union[Type[APIBudgetModel], Type[HTTPAPIBudgetModel]] = APIBudgetModel
3058
- elif model_str == "HTTPAPIBudget":
3059
- model_type = HTTPAPIBudgetModel
3060
- else:
3061
- raise ValueError(f"Unknown API Budget type: {model_str}")
3062
-
3063
- # create_component expects a type[BaseModel] and returns an instance of that model.
3064
3044
  self._api_budget = self.create_component(
3065
- model_type=model_type, component_definition=component_definition, config=config
3045
+ model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config
3066
3046
  )