airbyte-cdk 6.33.2.dev1__py3-none-any.whl → 6.33.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/declarative/auth/oauth.py +6 -1
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +15 -1
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +30 -129
- airbyte_cdk/sources/declarative/decoders/__init__.py +0 -4
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +7 -2
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +12 -58
- airbyte_cdk/sources/declarative/extractors/record_selector.py +12 -3
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +6 -11
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +22 -84
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +66 -86
- airbyte_cdk/sources/streams/call_rate.py +148 -86
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +3 -0
- {airbyte_cdk-6.33.2.dev1.dist-info → airbyte_cdk-6.33.4.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.33.2.dev1.dist-info → airbyte_cdk-6.33.4.dist-info}/RECORD +18 -18
- {airbyte_cdk-6.33.2.dev1.dist-info → airbyte_cdk-6.33.4.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.33.2.dev1.dist-info → airbyte_cdk-6.33.4.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.33.2.dev1.dist-info → airbyte_cdk-6.33.4.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.33.2.dev1.dist-info → airbyte_cdk-6.33.4.dist-info}/entry_points.txt +0 -0
@@ -3,7 +3,6 @@
|
|
3
3
|
|
4
4
|
from __future__ import annotations
|
5
5
|
|
6
|
-
from datetime import datetime, timedelta
|
7
6
|
from enum import Enum
|
8
7
|
from typing import Any, Dict, List, Literal, Optional, Union
|
9
8
|
|
@@ -652,8 +651,11 @@ class Rate(BaseModel):
|
|
652
651
|
description="The maximum number of calls allowed within the interval.",
|
653
652
|
title="Limit",
|
654
653
|
)
|
655
|
-
interval:
|
656
|
-
...,
|
654
|
+
interval: str = Field(
|
655
|
+
...,
|
656
|
+
description="The time interval for the rate limit.",
|
657
|
+
examples=["PT1H", "P1D"],
|
658
|
+
title="Interval",
|
657
659
|
)
|
658
660
|
|
659
661
|
|
@@ -927,15 +929,6 @@ class CustomDecoder(BaseModel):
|
|
927
929
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
928
930
|
|
929
931
|
|
930
|
-
class GzipJsonDecoder(BaseModel):
|
931
|
-
class Config:
|
932
|
-
extra = Extra.allow
|
933
|
-
|
934
|
-
type: Literal["GzipJsonDecoder"]
|
935
|
-
encoding: Optional[str] = "utf-8"
|
936
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
937
|
-
|
938
|
-
|
939
932
|
class MinMaxDatetime(BaseModel):
|
940
933
|
type: Literal["MinMaxDatetime"]
|
941
934
|
datetime: str = Field(
|
@@ -1314,18 +1307,8 @@ class LegacySessionTokenAuthenticator(BaseModel):
|
|
1314
1307
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1315
1308
|
|
1316
1309
|
|
1317
|
-
class
|
1318
|
-
type: Literal["
|
1319
|
-
encoding: Optional[str] = "utf-8"
|
1320
|
-
|
1321
|
-
|
1322
|
-
class JsonLineParser(BaseModel):
|
1323
|
-
type: Literal["JsonLineParser"]
|
1324
|
-
encoding: Optional[str] = "utf-8"
|
1325
|
-
|
1326
|
-
|
1327
|
-
class CsvParser(BaseModel):
|
1328
|
-
type: Literal["CsvParser"]
|
1310
|
+
class CsvDecoder(BaseModel):
|
1311
|
+
type: Literal["CsvDecoder"]
|
1329
1312
|
encoding: Optional[str] = "utf-8"
|
1330
1313
|
delimiter: Optional[str] = ","
|
1331
1314
|
|
@@ -1629,12 +1612,7 @@ class FixedWindowCallRatePolicy(BaseModel):
|
|
1629
1612
|
extra = Extra.allow
|
1630
1613
|
|
1631
1614
|
type: Literal["FixedWindowCallRatePolicy"]
|
1632
|
-
|
1633
|
-
...,
|
1634
|
-
description="The timestamp when the rate limit will reset.",
|
1635
|
-
title="Next Reset Timestamp",
|
1636
|
-
)
|
1637
|
-
period: timedelta = Field(
|
1615
|
+
period: str = Field(
|
1638
1616
|
..., description="The time interval for the rate limit window.", title="Period"
|
1639
1617
|
)
|
1640
1618
|
call_limit: int = Field(
|
@@ -1774,9 +1752,9 @@ class RecordSelector(BaseModel):
|
|
1774
1752
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1775
1753
|
|
1776
1754
|
|
1777
|
-
class
|
1778
|
-
type: Literal["
|
1779
|
-
|
1755
|
+
class GzipDecoder(BaseModel):
|
1756
|
+
type: Literal["GzipDecoder"]
|
1757
|
+
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
|
1780
1758
|
|
1781
1759
|
|
1782
1760
|
class Spec(BaseModel):
|
@@ -1809,29 +1787,6 @@ class CompositeErrorHandler(BaseModel):
|
|
1809
1787
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1810
1788
|
|
1811
1789
|
|
1812
|
-
class APIBudget(BaseModel):
|
1813
|
-
class Config:
|
1814
|
-
extra = Extra.allow
|
1815
|
-
|
1816
|
-
type: Literal["APIBudget"]
|
1817
|
-
policies: List[
|
1818
|
-
Union[
|
1819
|
-
FixedWindowCallRatePolicy,
|
1820
|
-
MovingWindowCallRatePolicy,
|
1821
|
-
UnlimitedCallRatePolicy,
|
1822
|
-
]
|
1823
|
-
] = Field(
|
1824
|
-
...,
|
1825
|
-
description="List of call rate policies that define how many calls are allowed.",
|
1826
|
-
title="Policies",
|
1827
|
-
)
|
1828
|
-
maximum_attempts_to_acquire: Optional[int] = Field(
|
1829
|
-
100000,
|
1830
|
-
description="The maximum number of attempts to acquire a call before giving up.",
|
1831
|
-
title="Maximum Attempts to Acquire",
|
1832
|
-
)
|
1833
|
-
|
1834
|
-
|
1835
1790
|
class HTTPAPIBudget(BaseModel):
|
1836
1791
|
class Config:
|
1837
1792
|
extra = Extra.allow
|
@@ -1863,11 +1818,6 @@ class HTTPAPIBudget(BaseModel):
|
|
1863
1818
|
description="List of HTTP status codes that indicate a rate limit has been hit.",
|
1864
1819
|
title="Status Codes for Rate Limit Hit",
|
1865
1820
|
)
|
1866
|
-
maximum_attempts_to_acquire: Optional[int] = Field(
|
1867
|
-
100000,
|
1868
|
-
description="The maximum number of attempts to acquire a call before giving up.",
|
1869
|
-
title="Maximum Attempts to Acquire",
|
1870
|
-
)
|
1871
1821
|
|
1872
1822
|
|
1873
1823
|
class ZipfileDecoder(BaseModel):
|
@@ -1875,18 +1825,13 @@ class ZipfileDecoder(BaseModel):
|
|
1875
1825
|
extra = Extra.allow
|
1876
1826
|
|
1877
1827
|
type: Literal["ZipfileDecoder"]
|
1878
|
-
|
1828
|
+
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder] = Field(
|
1879
1829
|
...,
|
1880
1830
|
description="Parser to parse the decompressed data from the zipfile(s).",
|
1881
1831
|
title="Parser",
|
1882
1832
|
)
|
1883
1833
|
|
1884
1834
|
|
1885
|
-
class CompositeRawDecoder(BaseModel):
|
1886
|
-
type: Literal["CompositeRawDecoder"]
|
1887
|
-
parser: Union[GzipParser, JsonParser, JsonLineParser, CsvParser]
|
1888
|
-
|
1889
|
-
|
1890
1835
|
class DeclarativeSource1(BaseModel):
|
1891
1836
|
class Config:
|
1892
1837
|
extra = Extra.forbid
|
@@ -1903,11 +1848,7 @@ class DeclarativeSource1(BaseModel):
|
|
1903
1848
|
definitions: Optional[Dict[str, Any]] = None
|
1904
1849
|
spec: Optional[Spec] = None
|
1905
1850
|
concurrency_level: Optional[ConcurrencyLevel] = None
|
1906
|
-
api_budget: Optional[
|
1907
|
-
None,
|
1908
|
-
description="Defines how many requests can be made to the API in a given time frame. This field accepts either a generic APIBudget or an HTTP-specific configuration (HTTPAPIBudget) to be applied across all streams.",
|
1909
|
-
title="API Budget",
|
1910
|
-
)
|
1851
|
+
api_budget: Optional[HTTPAPIBudget] = None
|
1911
1852
|
metadata: Optional[Dict[str, Any]] = Field(
|
1912
1853
|
None,
|
1913
1854
|
description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
|
@@ -1934,11 +1875,7 @@ class DeclarativeSource2(BaseModel):
|
|
1934
1875
|
definitions: Optional[Dict[str, Any]] = None
|
1935
1876
|
spec: Optional[Spec] = None
|
1936
1877
|
concurrency_level: Optional[ConcurrencyLevel] = None
|
1937
|
-
api_budget: Optional[
|
1938
|
-
None,
|
1939
|
-
description="Defines how many requests can be made to the API in a given time frame. This field accepts either a generic APIBudget or an HTTP-specific configuration (HTTPAPIBudget) to be applied across all streams.",
|
1940
|
-
title="API Budget",
|
1941
|
-
)
|
1878
|
+
api_budget: Optional[HTTPAPIBudget] = None
|
1942
1879
|
metadata: Optional[Dict[str, Any]] = Field(
|
1943
1880
|
None,
|
1944
1881
|
description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
|
@@ -2098,7 +2035,7 @@ class SessionTokenAuthenticator(BaseModel):
|
|
2098
2035
|
description="Authentication method to use for requests sent to the API, specifying how to inject the session token.",
|
2099
2036
|
title="Data Request Authentication",
|
2100
2037
|
)
|
2101
|
-
decoder: Optional[Union[JsonDecoder, XmlDecoder
|
2038
|
+
decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field(
|
2102
2039
|
None, description="Component used to decode the response.", title="Decoder"
|
2103
2040
|
)
|
2104
2041
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
@@ -2298,12 +2235,12 @@ class SimpleRetriever(BaseModel):
|
|
2298
2235
|
decoder: Optional[
|
2299
2236
|
Union[
|
2300
2237
|
CustomDecoder,
|
2238
|
+
CsvDecoder,
|
2239
|
+
GzipDecoder,
|
2301
2240
|
JsonDecoder,
|
2302
2241
|
JsonlDecoder,
|
2303
2242
|
IterableDecoder,
|
2304
2243
|
XmlDecoder,
|
2305
|
-
GzipJsonDecoder,
|
2306
|
-
CompositeRawDecoder,
|
2307
2244
|
ZipfileDecoder,
|
2308
2245
|
]
|
2309
2246
|
] = Field(
|
@@ -2376,12 +2313,12 @@ class AsyncRetriever(BaseModel):
|
|
2376
2313
|
decoder: Optional[
|
2377
2314
|
Union[
|
2378
2315
|
CustomDecoder,
|
2316
|
+
CsvDecoder,
|
2317
|
+
GzipDecoder,
|
2379
2318
|
JsonDecoder,
|
2380
2319
|
JsonlDecoder,
|
2381
2320
|
IterableDecoder,
|
2382
2321
|
XmlDecoder,
|
2383
|
-
GzipJsonDecoder,
|
2384
|
-
CompositeRawDecoder,
|
2385
2322
|
ZipfileDecoder,
|
2386
2323
|
]
|
2387
2324
|
] = Field(
|
@@ -2392,12 +2329,12 @@ class AsyncRetriever(BaseModel):
|
|
2392
2329
|
download_decoder: Optional[
|
2393
2330
|
Union[
|
2394
2331
|
CustomDecoder,
|
2332
|
+
CsvDecoder,
|
2333
|
+
GzipDecoder,
|
2395
2334
|
JsonDecoder,
|
2396
2335
|
JsonlDecoder,
|
2397
2336
|
IterableDecoder,
|
2398
2337
|
XmlDecoder,
|
2399
|
-
GzipJsonDecoder,
|
2400
|
-
CompositeRawDecoder,
|
2401
2338
|
ZipfileDecoder,
|
2402
2339
|
]
|
2403
2340
|
] = Field(
|
@@ -2442,6 +2379,7 @@ class DynamicDeclarativeStream(BaseModel):
|
|
2442
2379
|
|
2443
2380
|
|
2444
2381
|
ComplexFieldType.update_forward_refs()
|
2382
|
+
GzipDecoder.update_forward_refs()
|
2445
2383
|
CompositeErrorHandler.update_forward_refs()
|
2446
2384
|
DeclarativeSource1.update_forward_refs()
|
2447
2385
|
DeclarativeSource2.update_forward_refs()
|
@@ -60,10 +60,8 @@ from airbyte_cdk.sources.declarative.datetime import MinMaxDatetime
|
|
60
60
|
from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
|
61
61
|
from airbyte_cdk.sources.declarative.decoders import (
|
62
62
|
Decoder,
|
63
|
-
GzipJsonDecoder,
|
64
63
|
IterableDecoder,
|
65
64
|
JsonDecoder,
|
66
|
-
JsonlDecoder,
|
67
65
|
PaginationDecoderDecorator,
|
68
66
|
XmlDecoder,
|
69
67
|
ZipfileDecoder,
|
@@ -103,8 +101,8 @@ from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_mi
|
|
103
101
|
LegacyToPerPartitionStateMigration,
|
104
102
|
)
|
105
103
|
from airbyte_cdk.sources.declarative.models import (
|
106
|
-
Clamping,
|
107
104
|
CustomStateMigration,
|
105
|
+
GzipDecoder,
|
108
106
|
)
|
109
107
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
110
108
|
AddedFieldDefinition as AddedFieldDefinitionModel,
|
@@ -112,9 +110,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
112
110
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
113
111
|
AddFields as AddFieldsModel,
|
114
112
|
)
|
115
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
116
|
-
APIBudget as APIBudgetModel,
|
117
|
-
)
|
118
113
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
119
114
|
ApiKeyAuthenticator as ApiKeyAuthenticatorModel,
|
120
115
|
)
|
@@ -145,9 +140,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
145
140
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
146
141
|
CompositeErrorHandler as CompositeErrorHandlerModel,
|
147
142
|
)
|
148
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
149
|
-
CompositeRawDecoder as CompositeRawDecoderModel,
|
150
|
-
)
|
151
143
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
152
144
|
ConcurrencyLevel as ConcurrencyLevelModel,
|
153
145
|
)
|
@@ -158,7 +150,7 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
158
150
|
ConstantBackoffStrategy as ConstantBackoffStrategyModel,
|
159
151
|
)
|
160
152
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
161
|
-
|
153
|
+
CsvDecoder as CsvDecoderModel,
|
162
154
|
)
|
163
155
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
164
156
|
CursorPagination as CursorPaginationModel,
|
@@ -236,10 +228,7 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
236
228
|
FlattenFields as FlattenFieldsModel,
|
237
229
|
)
|
238
230
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
239
|
-
|
240
|
-
)
|
241
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
242
|
-
GzipParser as GzipParserModel,
|
231
|
+
GzipDecoder as GzipDecoderModel,
|
243
232
|
)
|
244
233
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
245
234
|
HTTPAPIBudget as HTTPAPIBudgetModel,
|
@@ -271,12 +260,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
271
260
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
272
261
|
JsonlDecoder as JsonlDecoderModel,
|
273
262
|
)
|
274
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
275
|
-
JsonLineParser as JsonLineParserModel,
|
276
|
-
)
|
277
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
278
|
-
JsonParser as JsonParserModel,
|
279
|
-
)
|
280
263
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
281
264
|
JwtAuthenticator as JwtAuthenticatorModel,
|
282
265
|
)
|
@@ -562,9 +545,9 @@ class ModelToComponentFactory:
|
|
562
545
|
CheckStreamModel: self.create_check_stream,
|
563
546
|
CheckDynamicStreamModel: self.create_check_dynamic_stream,
|
564
547
|
CompositeErrorHandlerModel: self.create_composite_error_handler,
|
565
|
-
CompositeRawDecoderModel: self.create_composite_raw_decoder,
|
566
548
|
ConcurrencyLevelModel: self.create_concurrency_level,
|
567
549
|
ConstantBackoffStrategyModel: self.create_constant_backoff_strategy,
|
550
|
+
CsvDecoderModel: self.create_csv_decoder,
|
568
551
|
CursorPaginationModel: self.create_cursor_pagination,
|
569
552
|
CustomAuthenticatorModel: self.create_custom_component,
|
570
553
|
CustomBackoffStrategyModel: self.create_custom_component,
|
@@ -594,10 +577,7 @@ class ModelToComponentFactory:
|
|
594
577
|
InlineSchemaLoaderModel: self.create_inline_schema_loader,
|
595
578
|
JsonDecoderModel: self.create_json_decoder,
|
596
579
|
JsonlDecoderModel: self.create_jsonl_decoder,
|
597
|
-
|
598
|
-
JsonParserModel: self.create_json_parser,
|
599
|
-
GzipJsonDecoderModel: self.create_gzipjson_decoder,
|
600
|
-
GzipParserModel: self.create_gzip_parser,
|
580
|
+
GzipDecoderModel: self.create_gzip_decoder,
|
601
581
|
KeysToLowerModel: self.create_keys_to_lower_transformation,
|
602
582
|
KeysToSnakeCaseModel: self.create_keys_to_snake_transformation,
|
603
583
|
KeysReplaceModel: self.create_keys_replace_transformation,
|
@@ -638,7 +618,6 @@ class ModelToComponentFactory:
|
|
638
618
|
StreamConfigModel: self.create_stream_config,
|
639
619
|
ComponentMappingDefinitionModel: self.create_components_mapping_definition,
|
640
620
|
ZipfileDecoderModel: self.create_zipfile_decoder,
|
641
|
-
APIBudgetModel: self.create_api_budget,
|
642
621
|
HTTPAPIBudgetModel: self.create_http_api_budget,
|
643
622
|
FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy,
|
644
623
|
MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy,
|
@@ -972,6 +951,17 @@ class ModelToComponentFactory:
|
|
972
951
|
parameters={},
|
973
952
|
)
|
974
953
|
|
954
|
+
@staticmethod
|
955
|
+
def apply_stream_state_migrations(
|
956
|
+
stream_state_migrations: List[Any] | None, stream_state: MutableMapping[str, Any]
|
957
|
+
) -> MutableMapping[str, Any]:
|
958
|
+
if stream_state_migrations:
|
959
|
+
for state_migration in stream_state_migrations:
|
960
|
+
if state_migration.should_migrate(stream_state):
|
961
|
+
# The state variable is expected to be mutable but the migrate method returns an immutable mapping.
|
962
|
+
stream_state = dict(state_migration.migrate(stream_state))
|
963
|
+
return stream_state
|
964
|
+
|
975
965
|
def create_concurrent_cursor_from_datetime_based_cursor(
|
976
966
|
self,
|
977
967
|
model_type: Type[BaseModel],
|
@@ -981,6 +971,7 @@ class ModelToComponentFactory:
|
|
981
971
|
config: Config,
|
982
972
|
message_repository: Optional[MessageRepository] = None,
|
983
973
|
runtime_lookback_window: Optional[datetime.timedelta] = None,
|
974
|
+
stream_state_migrations: Optional[List[Any]] = None,
|
984
975
|
**kwargs: Any,
|
985
976
|
) -> ConcurrentCursor:
|
986
977
|
# Per-partition incremental streams can dynamically create child cursors which will pass their current
|
@@ -991,6 +982,7 @@ class ModelToComponentFactory:
|
|
991
982
|
if "stream_state" not in kwargs
|
992
983
|
else kwargs["stream_state"]
|
993
984
|
)
|
985
|
+
stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
|
994
986
|
|
995
987
|
component_type = component_definition.get("type")
|
996
988
|
if component_definition.get("type") != model_type.__name__:
|
@@ -1226,6 +1218,7 @@ class ModelToComponentFactory:
|
|
1226
1218
|
config: Config,
|
1227
1219
|
stream_state: MutableMapping[str, Any],
|
1228
1220
|
partition_router: PartitionRouter,
|
1221
|
+
stream_state_migrations: Optional[List[Any]] = None,
|
1229
1222
|
**kwargs: Any,
|
1230
1223
|
) -> ConcurrentPerPartitionCursor:
|
1231
1224
|
component_type = component_definition.get("type")
|
@@ -1274,8 +1267,10 @@ class ModelToComponentFactory:
|
|
1274
1267
|
stream_namespace=stream_namespace,
|
1275
1268
|
config=config,
|
1276
1269
|
message_repository=NoopMessageRepository(),
|
1270
|
+
stream_state_migrations=stream_state_migrations,
|
1277
1271
|
)
|
1278
1272
|
)
|
1273
|
+
stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
|
1279
1274
|
|
1280
1275
|
# Return the concurrent cursor and state converter
|
1281
1276
|
return ConcurrentPerPartitionCursor(
|
@@ -1784,6 +1779,7 @@ class ModelToComponentFactory:
|
|
1784
1779
|
stream_name=model.name or "",
|
1785
1780
|
stream_namespace=None,
|
1786
1781
|
config=config or {},
|
1782
|
+
stream_state_migrations=model.state_migrations,
|
1787
1783
|
)
|
1788
1784
|
return (
|
1789
1785
|
self._create_component_from_model(model=model.incremental_sync, config=config)
|
@@ -2092,25 +2088,26 @@ class ModelToComponentFactory:
|
|
2092
2088
|
)
|
2093
2089
|
|
2094
2090
|
@staticmethod
|
2095
|
-
def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) ->
|
2091
|
+
def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> Decoder:
|
2096
2092
|
return JsonDecoder(parameters={})
|
2097
2093
|
|
2098
2094
|
@staticmethod
|
2099
|
-
def
|
2100
|
-
|
2101
|
-
|
2095
|
+
def create_csv_decoder(model: CsvDecoderModel, config: Config, **kwargs: Any) -> Decoder:
|
2096
|
+
return CompositeRawDecoder(
|
2097
|
+
parser=ModelToComponentFactory._get_parser(model, config), stream_response=True
|
2098
|
+
)
|
2102
2099
|
|
2103
2100
|
@staticmethod
|
2104
|
-
def create_jsonl_decoder(
|
2105
|
-
|
2106
|
-
|
2107
|
-
|
2101
|
+
def create_jsonl_decoder(model: JsonlDecoderModel, config: Config, **kwargs: Any) -> Decoder:
|
2102
|
+
return CompositeRawDecoder(
|
2103
|
+
parser=ModelToComponentFactory._get_parser(model, config), stream_response=True
|
2104
|
+
)
|
2108
2105
|
|
2109
2106
|
@staticmethod
|
2110
|
-
def
|
2111
|
-
|
2112
|
-
|
2113
|
-
|
2107
|
+
def create_gzip_decoder(model: GzipDecoderModel, config: Config, **kwargs: Any) -> Decoder:
|
2108
|
+
return CompositeRawDecoder(
|
2109
|
+
parser=ModelToComponentFactory._get_parser(model, config), stream_response=True
|
2110
|
+
)
|
2114
2111
|
|
2115
2112
|
@staticmethod
|
2116
2113
|
def create_iterable_decoder(
|
@@ -2122,33 +2119,30 @@ class ModelToComponentFactory:
|
|
2122
2119
|
def create_xml_decoder(model: XmlDecoderModel, config: Config, **kwargs: Any) -> XmlDecoder:
|
2123
2120
|
return XmlDecoder(parameters={})
|
2124
2121
|
|
2125
|
-
@staticmethod
|
2126
|
-
def create_gzipjson_decoder(
|
2127
|
-
model: GzipJsonDecoderModel, config: Config, **kwargs: Any
|
2128
|
-
) -> GzipJsonDecoder:
|
2129
|
-
return GzipJsonDecoder(parameters={}, encoding=model.encoding)
|
2130
|
-
|
2131
2122
|
def create_zipfile_decoder(
|
2132
2123
|
self, model: ZipfileDecoderModel, config: Config, **kwargs: Any
|
2133
2124
|
) -> ZipfileDecoder:
|
2134
|
-
parser
|
2135
|
-
return ZipfileDecoder(parser=parser)
|
2136
|
-
|
2137
|
-
def create_gzip_parser(
|
2138
|
-
self, model: GzipParserModel, config: Config, **kwargs: Any
|
2139
|
-
) -> GzipParser:
|
2140
|
-
inner_parser = self._create_component_from_model(model=model.inner_parser, config=config)
|
2141
|
-
return GzipParser(inner_parser=inner_parser)
|
2125
|
+
return ZipfileDecoder(parser=ModelToComponentFactory._get_parser(model.decoder, config))
|
2142
2126
|
|
2143
2127
|
@staticmethod
|
2144
|
-
def
|
2145
|
-
|
2128
|
+
def _get_parser(model: BaseModel, config: Config) -> Parser:
|
2129
|
+
if isinstance(model, JsonDecoderModel):
|
2130
|
+
# Note that the logic is a bit different from the JsonDecoder as there is some legacy that is maintained to return {} on error cases
|
2131
|
+
return JsonParser()
|
2132
|
+
elif isinstance(model, JsonlDecoderModel):
|
2133
|
+
return JsonLineParser()
|
2134
|
+
elif isinstance(model, CsvDecoderModel):
|
2135
|
+
return CsvParser(encoding=model.encoding, delimiter=model.delimiter)
|
2136
|
+
elif isinstance(model, GzipDecoderModel):
|
2137
|
+
return GzipParser(
|
2138
|
+
inner_parser=ModelToComponentFactory._get_parser(model.decoder, config)
|
2139
|
+
)
|
2140
|
+
elif isinstance(
|
2141
|
+
model, (CustomDecoderModel, IterableDecoderModel, XmlDecoderModel, ZipfileDecoderModel)
|
2142
|
+
):
|
2143
|
+
raise ValueError(f"Decoder type {model} does not have parser associated to it")
|
2146
2144
|
|
2147
|
-
|
2148
|
-
self, model: CompositeRawDecoderModel, config: Config, **kwargs: Any
|
2149
|
-
) -> CompositeRawDecoder:
|
2150
|
-
parser = self._create_component_from_model(model=model.parser, config=config)
|
2151
|
-
return CompositeRawDecoder(parser=parser)
|
2145
|
+
raise ValueError(f"Unknown decoder type {model}")
|
2152
2146
|
|
2153
2147
|
@staticmethod
|
2154
2148
|
def create_json_file_schema_loader(
|
@@ -2421,6 +2415,8 @@ class ModelToComponentFactory:
|
|
2421
2415
|
if model.record_filter
|
2422
2416
|
else None
|
2423
2417
|
)
|
2418
|
+
|
2419
|
+
transform_before_filtering = False
|
2424
2420
|
if client_side_incremental_sync:
|
2425
2421
|
record_filter = ClientSideIncrementalRecordFilterDecorator(
|
2426
2422
|
config=config,
|
@@ -2430,6 +2426,8 @@ class ModelToComponentFactory:
|
|
2430
2426
|
else None,
|
2431
2427
|
**client_side_incremental_sync,
|
2432
2428
|
)
|
2429
|
+
transform_before_filtering = True
|
2430
|
+
|
2433
2431
|
schema_normalization = (
|
2434
2432
|
TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization])
|
2435
2433
|
if isinstance(model.schema_normalization, SchemaNormalizationModel)
|
@@ -2444,6 +2442,7 @@ class ModelToComponentFactory:
|
|
2444
2442
|
transformations=transformations or [],
|
2445
2443
|
schema_normalization=schema_normalization,
|
2446
2444
|
parameters=model.parameters or {},
|
2445
|
+
transform_before_filtering=transform_before_filtering,
|
2447
2446
|
)
|
2448
2447
|
|
2449
2448
|
@staticmethod
|
@@ -2965,17 +2964,6 @@ class ModelToComponentFactory:
|
|
2965
2964
|
else:
|
2966
2965
|
return False
|
2967
2966
|
|
2968
|
-
def create_api_budget(self, model: APIBudgetModel, config: Config, **kwargs: Any) -> APIBudget:
|
2969
|
-
policies = [
|
2970
|
-
self._create_component_from_model(model=policy, config=config)
|
2971
|
-
for policy in model.policies
|
2972
|
-
]
|
2973
|
-
|
2974
|
-
return APIBudget(
|
2975
|
-
policies=policies,
|
2976
|
-
maximum_attempts_to_acquire=model.maximum_attempts_to_acquire or 100000,
|
2977
|
-
)
|
2978
|
-
|
2979
2967
|
def create_http_api_budget(
|
2980
2968
|
self, model: HTTPAPIBudgetModel, config: Config, **kwargs: Any
|
2981
2969
|
) -> HttpAPIBudget:
|
@@ -2986,10 +2974,9 @@ class ModelToComponentFactory:
|
|
2986
2974
|
|
2987
2975
|
return HttpAPIBudget(
|
2988
2976
|
policies=policies,
|
2989
|
-
maximum_attempts_to_acquire=model.maximum_attempts_to_acquire or 100000,
|
2990
2977
|
ratelimit_reset_header=model.ratelimit_reset_header or "ratelimit-reset",
|
2991
2978
|
ratelimit_remaining_header=model.ratelimit_remaining_header or "ratelimit-remaining",
|
2992
|
-
status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or
|
2979
|
+
status_codes_for_ratelimit_hit=model.status_codes_for_ratelimit_hit or [429],
|
2993
2980
|
)
|
2994
2981
|
|
2995
2982
|
def create_fixed_window_call_rate_policy(
|
@@ -2999,9 +2986,12 @@ class ModelToComponentFactory:
|
|
2999
2986
|
self._create_component_from_model(model=matcher, config=config)
|
3000
2987
|
for matcher in model.matchers
|
3001
2988
|
]
|
2989
|
+
|
2990
|
+
# Set the initial reset timestamp to 10 days from now.
|
2991
|
+
# This value will be updated by the first request.
|
3002
2992
|
return FixedWindowCallRatePolicy(
|
3003
|
-
next_reset_ts=
|
3004
|
-
period=model.period,
|
2993
|
+
next_reset_ts=datetime.datetime.now() + datetime.timedelta(days=10),
|
2994
|
+
period=parse_duration(model.period),
|
3005
2995
|
call_limit=model.call_limit,
|
3006
2996
|
matchers=matchers,
|
3007
2997
|
)
|
@@ -3036,7 +3026,7 @@ class ModelToComponentFactory:
|
|
3036
3026
|
def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate:
|
3037
3027
|
return Rate(
|
3038
3028
|
limit=model.limit,
|
3039
|
-
interval=model.interval,
|
3029
|
+
interval=parse_duration(model.interval),
|
3040
3030
|
)
|
3041
3031
|
|
3042
3032
|
def create_http_request_matcher(
|
@@ -3051,16 +3041,6 @@ class ModelToComponentFactory:
|
|
3051
3041
|
)
|
3052
3042
|
|
3053
3043
|
def set_api_budget(self, component_definition: ComponentDefinition, config: Config) -> None:
|
3054
|
-
model_str = component_definition.get("type")
|
3055
|
-
if model_str == "APIBudget":
|
3056
|
-
# Annotate model_type as a type that is a subclass of BaseModel
|
3057
|
-
model_type: Union[Type[APIBudgetModel], Type[HTTPAPIBudgetModel]] = APIBudgetModel
|
3058
|
-
elif model_str == "HTTPAPIBudget":
|
3059
|
-
model_type = HTTPAPIBudgetModel
|
3060
|
-
else:
|
3061
|
-
raise ValueError(f"Unknown API Budget type: {model_str}")
|
3062
|
-
|
3063
|
-
# create_component expects a type[BaseModel] and returns an instance of that model.
|
3064
3044
|
self._api_budget = self.create_component(
|
3065
|
-
model_type=
|
3045
|
+
model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config
|
3066
3046
|
)
|