airbyte-cdk 6.45.10__py3-none-any.whl → 6.46.0.post4.dev14712712756__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. airbyte_cdk/cli/__init__.py +9 -1
  2. airbyte_cdk/cli/airbyte_cdk/__init__.py +86 -0
  3. airbyte_cdk/cli/airbyte_cdk/_connector.py +179 -0
  4. airbyte_cdk/cli/airbyte_cdk/_image.py +95 -0
  5. airbyte_cdk/cli/airbyte_cdk/_manifest.py +24 -0
  6. airbyte_cdk/cli/airbyte_cdk/_secrets.py +150 -0
  7. airbyte_cdk/cli/airbyte_cdk/_util.py +43 -0
  8. airbyte_cdk/cli/airbyte_cdk/_version.py +13 -0
  9. airbyte_cdk/connector_builder/connector_builder_handler.py +10 -0
  10. airbyte_cdk/models/connector_metadata.py +97 -0
  11. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +114 -81
  12. airbyte_cdk/sources/declarative/manifest_declarative_source.py +122 -45
  13. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +90 -78
  14. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +9 -0
  15. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +2 -2
  16. airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py +462 -0
  17. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +2 -2
  18. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +64 -33
  19. airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py +0 -10
  20. airbyte_cdk/test/standard_tests/connector_base.py +51 -25
  21. airbyte_cdk/test/standard_tests/declarative_sources.py +3 -1
  22. airbyte_cdk/test/standard_tests/test_resources.py +69 -0
  23. airbyte_cdk/test/standard_tests/util.py +79 -0
  24. airbyte_cdk/utils/docker.py +337 -0
  25. airbyte_cdk/utils/docker_image_templates.py +101 -0
  26. {airbyte_cdk-6.45.10.dist-info → airbyte_cdk-6.46.0.post4.dev14712712756.dist-info}/METADATA +6 -1
  27. {airbyte_cdk-6.45.10.dist-info → airbyte_cdk-6.46.0.post4.dev14712712756.dist-info}/RECORD +31 -18
  28. {airbyte_cdk-6.45.10.dist-info → airbyte_cdk-6.46.0.post4.dev14712712756.dist-info}/entry_points.txt +1 -0
  29. {airbyte_cdk-6.45.10.dist-info → airbyte_cdk-6.46.0.post4.dev14712712756.dist-info}/LICENSE.txt +0 -0
  30. {airbyte_cdk-6.45.10.dist-info → airbyte_cdk-6.46.0.post4.dev14712712756.dist-info}/LICENSE_SHORT +0 -0
  31. {airbyte_cdk-6.45.10.dist-info → airbyte_cdk-6.46.0.post4.dev14712712756.dist-info}/WHEEL +0 -0
@@ -0,0 +1,97 @@
1
+ """Models to represent the structure of a `metadata.yaml` file."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from enum import Enum
6
+ from pathlib import Path
7
+
8
+ import yaml
9
+ from pydantic import BaseModel, Field
10
+
11
+
12
+ class ConnectorLanguage(str, Enum):
13
+ """Connector implementation language."""
14
+
15
+ PYTHON = "python"
16
+ JAVA = "java"
17
+ LOW_CODE = "low-code"
18
+ MANIFEST_ONLY = "manifest-only"
19
+ UNKNOWN = "unknown"
20
+
21
+
22
+ class ConnectorBuildOptions(BaseModel):
23
+ """Connector build options from metadata.yaml."""
24
+
25
+ model_config = {"extra": "allow"}
26
+
27
+ baseImage: str | None = Field(
28
+ None,
29
+ description="Base image to use for building the connector",
30
+ )
31
+ path: str | None = Field(
32
+ None,
33
+ description="Path to the connector code within the repository",
34
+ )
35
+
36
+
37
+ class ConnectorMetadata(BaseModel):
38
+ """Connector metadata from metadata.yaml."""
39
+
40
+ model_config = {"extra": "allow"}
41
+
42
+ dockerRepository: str = Field(..., description="Docker repository for the connector image")
43
+ dockerImageTag: str = Field(..., description="Docker image tag for the connector")
44
+
45
+ tags: list[str] = Field(
46
+ default=[],
47
+ description="List of tags for the connector",
48
+ )
49
+
50
+ @property
51
+ def language(self) -> ConnectorLanguage:
52
+ """Get the connector language."""
53
+ for tag in self.tags:
54
+ if tag.startswith("language:"):
55
+ language = tag.split(":", 1)[1]
56
+ if language == "python":
57
+ return ConnectorLanguage.PYTHON
58
+ elif language == "java":
59
+ return ConnectorLanguage.JAVA
60
+ elif language == "low-code":
61
+ return ConnectorLanguage.LOW_CODE
62
+ elif language == "manifest-only":
63
+ return ConnectorLanguage.MANIFEST_ONLY
64
+
65
+ return ConnectorLanguage.UNKNOWN
66
+
67
+ connectorBuildOptions: ConnectorBuildOptions | None = Field(
68
+ None, description="Options for building the connector"
69
+ )
70
+
71
+
72
+ class MetadataFile(BaseModel):
73
+ """Represents the structure of a metadata.yaml file."""
74
+
75
+ model_config = {"extra": "allow"}
76
+
77
+ data: ConnectorMetadata = Field(..., description="Connector metadata")
78
+
79
+ @classmethod
80
+ def from_file(
81
+ cls,
82
+ file_path: Path,
83
+ ) -> MetadataFile:
84
+ """Load metadata from a YAML file."""
85
+ if not file_path.exists():
86
+ raise FileNotFoundError(f"Metadata file not found: {file_path!s}")
87
+
88
+ metadata_content = file_path.read_text()
89
+ metadata_dict = yaml.safe_load(metadata_content)
90
+
91
+ if not metadata_dict or "data" not in metadata_dict:
92
+ raise ValueError(
93
+ "Invalid metadata format: missing 'data' field in YAML file '{file_path!s}'"
94
+ )
95
+
96
+ metadata_file = MetadataFile.model_validate(metadata_dict)
97
+ return metadata_file
@@ -440,7 +440,9 @@ definitions:
440
440
  description: Backoff time in seconds.
441
441
  anyOf:
442
442
  - type: number
443
+ title: Number of seconds
443
444
  - type: string
445
+ title: Interpolated Value
444
446
  interpolation_context:
445
447
  - config
446
448
  examples:
@@ -1057,15 +1059,18 @@ definitions:
1057
1059
  type: string
1058
1060
  enum: [JwtAuthenticator]
1059
1061
  secret_key:
1062
+ title: Secret Key
1060
1063
  type: string
1061
1064
  description: Secret used to sign the JSON web token.
1062
1065
  examples:
1063
1066
  - "{{ config['secret_key'] }}"
1064
1067
  base64_encode_secret_key:
1068
+ title: Base64-encode Secret Key
1065
1069
  type: boolean
1066
1070
  description: When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.
1067
1071
  default: False
1068
1072
  algorithm:
1073
+ title: Algorithm
1069
1074
  type: string
1070
1075
  description: Algorithm used to sign the JSON web token.
1071
1076
  enum:
@@ -1389,27 +1394,27 @@ definitions:
1389
1394
  type:
1390
1395
  type: string
1391
1396
  enum: [DeclarativeStream]
1397
+ name:
1398
+ title: Name
1399
+ description: The stream name.
1400
+ type: string
1401
+ default: ""
1402
+ example:
1403
+ - "Users"
1392
1404
  retriever:
1393
1405
  title: Retriever
1394
1406
  description: Component used to coordinate how records are extracted across stream slices and request pages.
1395
1407
  anyOf:
1408
+ - "$ref": "#/definitions/SimpleRetriever"
1396
1409
  - "$ref": "#/definitions/AsyncRetriever"
1397
1410
  - "$ref": "#/definitions/CustomRetriever"
1398
- - "$ref": "#/definitions/SimpleRetriever"
1399
1411
  incremental_sync:
1400
1412
  title: Incremental Sync
1401
1413
  description: Component used to fetch data incrementally based on a time field in the data.
1402
1414
  anyOf:
1403
- - "$ref": "#/definitions/CustomIncrementalSync"
1404
1415
  - "$ref": "#/definitions/DatetimeBasedCursor"
1405
1416
  - "$ref": "#/definitions/IncrementingCountCursor"
1406
- name:
1407
- title: Name
1408
- description: The stream name.
1409
- type: string
1410
- default: ""
1411
- example:
1412
- - "Users"
1417
+ - "$ref": "#/definitions/CustomIncrementalSync"
1413
1418
  primary_key:
1414
1419
  title: Primary Key
1415
1420
  description: The primary key of the stream.
@@ -1419,8 +1424,8 @@ definitions:
1419
1424
  title: Schema Loader
1420
1425
  description: Component used to retrieve the schema for the current stream.
1421
1426
  anyOf:
1422
- - "$ref": "#/definitions/DynamicSchemaLoader"
1423
1427
  - "$ref": "#/definitions/InlineSchemaLoader"
1428
+ - "$ref": "#/definitions/DynamicSchemaLoader"
1424
1429
  - "$ref": "#/definitions/JsonFileSchemaLoader"
1425
1430
  - "$ref": "#/definitions/CustomSchemaLoader"
1426
1431
  # TODO we have move the transformation to the RecordSelector level in the code but kept this here for
@@ -1459,7 +1464,7 @@ definitions:
1459
1464
  properties:
1460
1465
  type:
1461
1466
  type: string
1462
- enum: [ FileUploader ]
1467
+ enum: [FileUploader]
1463
1468
  requester:
1464
1469
  description: Requester component that describes how to prepare HTTP requests to send to the source API.
1465
1470
  anyOf:
@@ -1484,6 +1489,9 @@ definitions:
1484
1489
  examples:
1485
1490
  - "{{ record.id }}/{{ record.file_name }}/"
1486
1491
  - "{{ record.id }}_{{ record.file_name }}/"
1492
+ $parameters:
1493
+ type: object
1494
+ additionalProperties: true
1487
1495
  $parameters:
1488
1496
  type: object
1489
1497
  additional_properties: true
@@ -1709,13 +1717,15 @@ definitions:
1709
1717
  title: Pagination Strategy
1710
1718
  description: Strategy defining how records are paginated.
1711
1719
  anyOf:
1720
+ - "$ref": "#/definitions/PageIncrement"
1721
+ - "$ref": "#/definitions/OffsetIncrement"
1712
1722
  - "$ref": "#/definitions/CursorPagination"
1713
1723
  - "$ref": "#/definitions/CustomPaginationStrategy"
1714
- - "$ref": "#/definitions/OffsetIncrement"
1715
- - "$ref": "#/definitions/PageIncrement"
1716
1724
  page_size_option:
1725
+ title: Inject Page Size Into Outgoing HTTP Request
1717
1726
  "$ref": "#/definitions/RequestOption"
1718
1727
  page_token_option:
1728
+ title: Inject Page Token Into Outgoing HTTP Request
1719
1729
  anyOf:
1720
1730
  - "$ref": "#/definitions/RequestOption"
1721
1731
  - "$ref": "#/definitions/RequestPath"
@@ -1813,6 +1823,8 @@ definitions:
1813
1823
  type: object
1814
1824
  additionalProperties: true
1815
1825
  SessionTokenAuthenticator:
1826
+ title: Session Token Authenticator
1827
+ description: Authenticator for requests using the session token as an API key that's injected into the request.
1816
1828
  type: object
1817
1829
  required:
1818
1830
  - type
@@ -1905,8 +1917,9 @@ definitions:
1905
1917
  type: string
1906
1918
  enum: [HttpRequester]
1907
1919
  url_base:
1920
+ linkable: true
1908
1921
  title: API Base URL
1909
- description: Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
1922
+ description: The Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
1910
1923
  type: string
1911
1924
  interpolation_context:
1912
1925
  - config
@@ -1924,7 +1937,7 @@ definitions:
1924
1937
  - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}"
1925
1938
  path:
1926
1939
  title: URL Path
1927
- description: Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
1940
+ description: The Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
1928
1941
  type: string
1929
1942
  interpolation_context:
1930
1943
  - config
@@ -1939,27 +1952,6 @@ definitions:
1939
1952
  - "/products"
1940
1953
  - "/quotes/{{ stream_partition['id'] }}/quote_line_groups"
1941
1954
  - "/trades/{{ config['symbol_id'] }}/history"
1942
- authenticator:
1943
- title: Authenticator
1944
- description: Authentication method to use for requests sent to the API.
1945
- anyOf:
1946
- - "$ref": "#/definitions/ApiKeyAuthenticator"
1947
- - "$ref": "#/definitions/BasicHttpAuthenticator"
1948
- - "$ref": "#/definitions/BearerAuthenticator"
1949
- - "$ref": "#/definitions/CustomAuthenticator"
1950
- - "$ref": "#/definitions/OAuthAuthenticator"
1951
- - "$ref": "#/definitions/JwtAuthenticator"
1952
- - "$ref": "#/definitions/NoAuth"
1953
- - "$ref": "#/definitions/SessionTokenAuthenticator"
1954
- - "$ref": "#/definitions/LegacySessionTokenAuthenticator"
1955
- - "$ref": "#/definitions/SelectiveAuthenticator"
1956
- error_handler:
1957
- title: Error Handler
1958
- description: Error handler component that defines how to handle errors.
1959
- anyOf:
1960
- - "$ref": "#/definitions/DefaultErrorHandler"
1961
- - "$ref": "#/definitions/CustomErrorHandler"
1962
- - "$ref": "#/definitions/CompositeErrorHandler"
1963
1955
  http_method:
1964
1956
  title: HTTP Method
1965
1957
  description: The HTTP method used to fetch data from the source (can be GET or POST).
@@ -1971,14 +1963,33 @@ definitions:
1971
1963
  examples:
1972
1964
  - GET
1973
1965
  - POST
1966
+ authenticator:
1967
+ title: Authenticator
1968
+ linkable: true
1969
+ description: Authentication method to use for requests sent to the API.
1970
+ anyOf:
1971
+ - "$ref": "#/definitions/NoAuth"
1972
+ - "$ref": "#/definitions/ApiKeyAuthenticator"
1973
+ - "$ref": "#/definitions/BasicHttpAuthenticator"
1974
+ - "$ref": "#/definitions/BearerAuthenticator"
1975
+ - "$ref": "#/definitions/OAuthAuthenticator"
1976
+ - "$ref": "#/definitions/JwtAuthenticator"
1977
+ - "$ref": "#/definitions/SessionTokenAuthenticator"
1978
+ - "$ref": "#/definitions/SelectiveAuthenticator"
1979
+ - "$ref": "#/definitions/CustomAuthenticator"
1980
+ - "$ref": "#/definitions/LegacySessionTokenAuthenticator"
1981
+ fetch_properties_from_endpoint:
1982
+ title: Fetch Properties from Endpoint
1983
+ description: Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.
1984
+ "$ref": "#/definitions/PropertiesFromEndpoint"
1974
1985
  request_body_data:
1975
1986
  title: Request Body Payload (Non-JSON)
1976
1987
  description: Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.
1977
1988
  anyOf:
1978
- - type: string
1979
1989
  - type: object
1980
1990
  additionalProperties:
1981
1991
  type: string
1992
+ - type: string
1982
1993
  interpolation_context:
1983
1994
  - next_page_token
1984
1995
  - stream_interval
@@ -1993,9 +2004,9 @@ definitions:
1993
2004
  title: Request Body JSON Payload
1994
2005
  description: Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.
1995
2006
  anyOf:
1996
- - type: string
1997
2007
  - type: object
1998
2008
  additionalProperties: true
2009
+ - type: string
1999
2010
  interpolation_context:
2000
2011
  - next_page_token
2001
2012
  - stream_interval
@@ -2012,10 +2023,12 @@ definitions:
2012
2023
  title: Request Headers
2013
2024
  description: Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.
2014
2025
  anyOf:
2015
- - type: string
2016
2026
  - type: object
2027
+ title: Key/Value Pairs
2017
2028
  additionalProperties:
2018
2029
  type: string
2030
+ - type: string
2031
+ title: Interpolated Value
2019
2032
  interpolation_context:
2020
2033
  - next_page_token
2021
2034
  - stream_interval
@@ -2028,12 +2041,14 @@ definitions:
2028
2041
  title: Query Parameters
2029
2042
  description: Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.
2030
2043
  anyOf:
2031
- - type: string
2032
2044
  - type: object
2045
+ title: Key/Value Pairs
2033
2046
  additionalProperties:
2034
2047
  anyOf:
2035
2048
  - type: string
2036
- - $ref": "#/definitions/QueryProperties"
2049
+ - $ref: "#/definitions/QueryProperties"
2050
+ - type: string
2051
+ title: Interpolated Value
2037
2052
  interpolation_context:
2038
2053
  - next_page_token
2039
2054
  - stream_interval
@@ -2044,6 +2059,13 @@ definitions:
2044
2059
  - query: 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
2045
2060
  - searchIn: "{{ ','.join(config.get('search_in', [])) }}"
2046
2061
  - sort_by[asc]: updated_at
2062
+ error_handler:
2063
+ title: Error Handler
2064
+ description: Error handler component that defines how to handle errors.
2065
+ anyOf:
2066
+ - "$ref": "#/definitions/DefaultErrorHandler"
2067
+ - "$ref": "#/definitions/CompositeErrorHandler"
2068
+ - "$ref": "#/definitions/CustomErrorHandler"
2047
2069
  use_cache:
2048
2070
  title: Use Cache
2049
2071
  description: Enables stream requests caching. This field is automatically set by the CDK.
@@ -2258,6 +2280,7 @@ definitions:
2258
2280
  title: Schema
2259
2281
  description: Describes a streams' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.
2260
2282
  type: object
2283
+ additionalProperties: true
2261
2284
  JsonFileSchemaLoader:
2262
2285
  title: Json File Schema Loader
2263
2286
  description: Loads the schema from a json file.
@@ -2351,7 +2374,7 @@ definitions:
2351
2374
  properties:
2352
2375
  type:
2353
2376
  type: string
2354
- enum: [ KeyTransformation ]
2377
+ enum: [KeyTransformation]
2355
2378
  prefix:
2356
2379
  title: Key Prefix
2357
2380
  description: Prefix to add for object keys. If not provided original keys remain unchanged.
@@ -2939,7 +2962,9 @@ definitions:
2939
2962
  description: The number of records to include in each pages.
2940
2963
  anyOf:
2941
2964
  - type: integer
2965
+ title: Number of Records
2942
2966
  - type: string
2967
+ title: Interpolated Value
2943
2968
  interpolation_context:
2944
2969
  - config
2945
2970
  - response
@@ -2971,7 +2996,9 @@ definitions:
2971
2996
  - config
2972
2997
  anyOf:
2973
2998
  - type: integer
2999
+ title: Number of Records
2974
3000
  - type: string
3001
+ title: Interpolated Value
2975
3002
  examples:
2976
3003
  - 100
2977
3004
  - "100"
@@ -3065,10 +3092,13 @@ definitions:
3065
3092
  description: The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.
3066
3093
  anyOf:
3067
3094
  - type: string
3095
+ title: Single Key
3068
3096
  - type: array
3097
+ title: Composite Key
3069
3098
  items:
3070
3099
  type: string
3071
3100
  - type: array
3101
+ title: Composite Key of Nested Fields
3072
3102
  items:
3073
3103
  type: array
3074
3104
  items:
@@ -3178,6 +3208,7 @@ definitions:
3178
3208
  type: string
3179
3209
  enum: [RecordFilter]
3180
3210
  condition:
3211
+ title: Condition
3181
3212
  description: The predicate to filter a record. Records will be removed if evaluated to False.
3182
3213
  type: string
3183
3214
  default: ""
@@ -3207,25 +3238,24 @@ definitions:
3207
3238
  enum: [RecordSelector]
3208
3239
  extractor:
3209
3240
  anyOf:
3210
- - "$ref": "#/definitions/CustomRecordExtractor"
3211
3241
  - "$ref": "#/definitions/DpathExtractor"
3242
+ - "$ref": "#/definitions/CustomRecordExtractor"
3212
3243
  record_filter:
3213
3244
  title: Record Filter
3214
3245
  description: Responsible for filtering records to be emitted by the Source.
3215
3246
  anyOf:
3216
- - "$ref": "#/definitions/CustomRecordFilter"
3217
3247
  - "$ref": "#/definitions/RecordFilter"
3248
+ - "$ref": "#/definitions/CustomRecordFilter"
3218
3249
  schema_normalization:
3219
3250
  title: Schema Normalization
3220
3251
  description: Responsible for normalization according to the schema.
3221
3252
  anyOf:
3222
3253
  - "$ref": "#/definitions/SchemaNormalization"
3223
3254
  - "$ref": "#/definitions/CustomSchemaNormalization"
3224
- default: None
3225
3255
  transform_before_filtering:
3256
+ title: Transform Before Filtering
3226
3257
  description: If true, transformation will be applied before record filtering.
3227
3258
  type: boolean
3228
- default: false
3229
3259
  $parameters:
3230
3260
  type: object
3231
3261
  additionalProperties: true
@@ -3234,11 +3264,11 @@ definitions:
3234
3264
  description: Responsible for normalization according to the schema.
3235
3265
  type: string
3236
3266
  enum:
3237
- - None
3238
3267
  - Default
3239
- examples:
3240
3268
  - None
3269
+ examples:
3241
3270
  - Default
3271
+ - None
3242
3272
  RemoveFields:
3243
3273
  title: Remove Fields
3244
3274
  description: A transformation which removes fields from a record. The fields removed are designated using FieldPointers. During transformation, if a field or any of its parents does not exist in the record, no error is thrown.
@@ -3394,6 +3424,7 @@ definitions:
3394
3424
  type: object
3395
3425
  additionalProperties: true
3396
3426
  StateDelegatingStream:
3427
+ title: State Delegating Stream
3397
3428
  description: (This component is experimental. Use at your own risk.) Orchestrate the retriever's usage based on the state value.
3398
3429
  type: object
3399
3430
  required:
@@ -3413,17 +3444,18 @@ definitions:
3413
3444
  example:
3414
3445
  - "Users"
3415
3446
  full_refresh_stream:
3416
- title: Retriever
3447
+ title: Full Refresh Stream
3417
3448
  description: Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.
3418
3449
  "$ref": "#/definitions/DeclarativeStream"
3419
3450
  incremental_stream:
3420
- title: Retriever
3451
+ title: Incremental Stream
3421
3452
  description: Component used to coordinate how records are extracted across stream slices and request pages when the state provided.
3422
3453
  "$ref": "#/definitions/DeclarativeStream"
3423
3454
  $parameters:
3424
3455
  type: object
3425
3456
  additionalProperties: true
3426
3457
  SimpleRetriever:
3458
+ title: Synchronous Retriever
3427
3459
  description: Retrieves records by synchronously sending requests to fetch records. The retriever acts as an orchestrator between the requester, the record selector, the paginator, and the partition router.
3428
3460
  type: object
3429
3461
  required:
@@ -3434,14 +3466,26 @@ definitions:
3434
3466
  type:
3435
3467
  type: string
3436
3468
  enum: [SimpleRetriever]
3437
- record_selector:
3438
- description: Component that describes how to extract records from a HTTP response.
3439
- "$ref": "#/definitions/RecordSelector"
3440
3469
  requester:
3441
3470
  description: Requester component that describes how to prepare HTTP requests to send to the source API.
3442
3471
  anyOf:
3443
- - "$ref": "#/definitions/CustomRequester"
3444
3472
  - "$ref": "#/definitions/HttpRequester"
3473
+ - "$ref": "#/definitions/CustomRequester"
3474
+ decoder:
3475
+ title: Decoder
3476
+ description: Component decoding the response so records can be extracted.
3477
+ anyOf:
3478
+ - "$ref": "#/definitions/CsvDecoder"
3479
+ - "$ref": "#/definitions/GzipDecoder"
3480
+ - "$ref": "#/definitions/JsonDecoder"
3481
+ - "$ref": "#/definitions/JsonlDecoder"
3482
+ - "$ref": "#/definitions/IterableDecoder"
3483
+ - "$ref": "#/definitions/XmlDecoder"
3484
+ - "$ref": "#/definitions/ZipfileDecoder"
3485
+ - "$ref": "#/definitions/CustomDecoder"
3486
+ record_selector:
3487
+ description: Component that describes how to extract records from a HTTP response.
3488
+ "$ref": "#/definitions/RecordSelector"
3445
3489
  paginator:
3446
3490
  description: Paginator component that describes how to navigate through the API's pages.
3447
3491
  anyOf:
@@ -3456,29 +3500,17 @@ definitions:
3456
3500
  description: PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.
3457
3501
  default: []
3458
3502
  anyOf:
3459
- - "$ref": "#/definitions/CustomPartitionRouter"
3460
3503
  - "$ref": "#/definitions/ListPartitionRouter"
3461
3504
  - "$ref": "#/definitions/SubstreamPartitionRouter"
3462
3505
  - "$ref": "#/definitions/GroupingPartitionRouter"
3506
+ - "$ref": "#/definitions/CustomPartitionRouter"
3463
3507
  - type: array
3464
3508
  items:
3465
3509
  anyOf:
3466
- - "$ref": "#/definitions/CustomPartitionRouter"
3467
3510
  - "$ref": "#/definitions/ListPartitionRouter"
3468
3511
  - "$ref": "#/definitions/SubstreamPartitionRouter"
3469
3512
  - "$ref": "#/definitions/GroupingPartitionRouter"
3470
- decoder:
3471
- title: Decoder
3472
- description: Component decoding the response so records can be extracted.
3473
- anyOf:
3474
- - "$ref": "#/definitions/CustomDecoder"
3475
- - "$ref": "#/definitions/CsvDecoder"
3476
- - "$ref": "#/definitions/GzipDecoder"
3477
- - "$ref": "#/definitions/JsonDecoder"
3478
- - "$ref": "#/definitions/JsonlDecoder"
3479
- - "$ref": "#/definitions/IterableDecoder"
3480
- - "$ref": "#/definitions/XmlDecoder"
3481
- - "$ref": "#/definitions/ZipfileDecoder"
3513
+ - "$ref": "#/definitions/CustomPartitionRouter"
3482
3514
  $parameters:
3483
3515
  type: object
3484
3516
  additionalProperties: true
@@ -3544,6 +3576,7 @@ definitions:
3544
3576
  items:
3545
3577
  type: string
3546
3578
  AsyncRetriever:
3579
+ title: Asynchronous Retriever
3547
3580
  description: "Retrieves records by Asynchronously sending requests to fetch records. The retriever acts as an orchestrator between the requester, the record selector, the paginator, and the partition router."
3548
3581
  type: object
3549
3582
  required:
@@ -3569,29 +3602,29 @@ definitions:
3569
3602
  status_extractor:
3570
3603
  description: Responsible for fetching the actual status of the async job.
3571
3604
  anyOf:
3572
- - "$ref": "#/definitions/CustomRecordExtractor"
3573
3605
  - "$ref": "#/definitions/DpathExtractor"
3606
+ - "$ref": "#/definitions/CustomRecordExtractor"
3574
3607
  download_target_extractor:
3575
3608
  description: Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.
3576
3609
  anyOf:
3577
- - "$ref": "#/definitions/CustomRecordExtractor"
3578
3610
  - "$ref": "#/definitions/DpathExtractor"
3611
+ - "$ref": "#/definitions/CustomRecordExtractor"
3579
3612
  download_extractor:
3580
3613
  description: Responsible for fetching the records from provided urls.
3581
3614
  anyOf:
3582
- - "$ref": "#/definitions/CustomRecordExtractor"
3583
3615
  - "$ref": "#/definitions/DpathExtractor"
3616
+ - "$ref": "#/definitions/CustomRecordExtractor"
3584
3617
  - "$ref": "#/definitions/ResponseToFileExtractor"
3585
3618
  creation_requester:
3586
3619
  description: Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.
3587
3620
  anyOf:
3588
- - "$ref": "#/definitions/CustomRequester"
3589
3621
  - "$ref": "#/definitions/HttpRequester"
3622
+ - "$ref": "#/definitions/CustomRequester"
3590
3623
  polling_requester:
3591
3624
  description: Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.
3592
3625
  anyOf:
3593
- - "$ref": "#/definitions/CustomRequester"
3594
3626
  - "$ref": "#/definitions/HttpRequester"
3627
+ - "$ref": "#/definitions/CustomRequester"
3595
3628
  polling_job_timeout:
3596
3629
  description: The time in minutes after which the single Async Job should be considered as Timed Out.
3597
3630
  anyOf:
@@ -3602,13 +3635,13 @@ definitions:
3602
3635
  download_target_requester:
3603
3636
  description: Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.
3604
3637
  anyOf:
3605
- - "$ref": "#/definitions/CustomRequester"
3606
3638
  - "$ref": "#/definitions/HttpRequester"
3639
+ - "$ref": "#/definitions/CustomRequester"
3607
3640
  download_requester:
3608
3641
  description: Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.
3609
3642
  anyOf:
3610
- - "$ref": "#/definitions/CustomRequester"
3611
3643
  - "$ref": "#/definitions/HttpRequester"
3644
+ - "$ref": "#/definitions/CustomRequester"
3612
3645
  download_paginator:
3613
3646
  description: Paginator component that describes how to navigate through the API's pages during download.
3614
3647
  anyOf:
@@ -3617,34 +3650,33 @@ definitions:
3617
3650
  abort_requester:
3618
3651
  description: Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.
3619
3652
  anyOf:
3620
- - "$ref": "#/definitions/CustomRequester"
3621
3653
  - "$ref": "#/definitions/HttpRequester"
3654
+ - "$ref": "#/definitions/CustomRequester"
3622
3655
  delete_requester:
3623
3656
  description: Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.
3624
3657
  anyOf:
3625
- - "$ref": "#/definitions/CustomRequester"
3626
3658
  - "$ref": "#/definitions/HttpRequester"
3659
+ - "$ref": "#/definitions/CustomRequester"
3627
3660
  partition_router:
3628
3661
  title: Partition Router
3629
3662
  description: PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.
3630
3663
  default: []
3631
3664
  anyOf:
3632
- - "$ref": "#/definitions/CustomPartitionRouter"
3633
3665
  - "$ref": "#/definitions/ListPartitionRouter"
3634
3666
  - "$ref": "#/definitions/SubstreamPartitionRouter"
3635
3667
  - "$ref": "#/definitions/GroupingPartitionRouter"
3668
+ - "$ref": "#/definitions/CustomPartitionRouter"
3636
3669
  - type: array
3637
3670
  items:
3638
3671
  anyOf:
3639
- - "$ref": "#/definitions/CustomPartitionRouter"
3640
3672
  - "$ref": "#/definitions/ListPartitionRouter"
3641
3673
  - "$ref": "#/definitions/SubstreamPartitionRouter"
3642
3674
  - "$ref": "#/definitions/GroupingPartitionRouter"
3675
+ - "$ref": "#/definitions/CustomPartitionRouter"
3643
3676
  decoder:
3644
3677
  title: Decoder
3645
3678
  description: Component decoding the response so records can be extracted.
3646
3679
  anyOf:
3647
- - "$ref": "#/definitions/CustomDecoder"
3648
3680
  - "$ref": "#/definitions/CsvDecoder"
3649
3681
  - "$ref": "#/definitions/GzipDecoder"
3650
3682
  - "$ref": "#/definitions/JsonDecoder"
@@ -3652,11 +3684,11 @@ definitions:
3652
3684
  - "$ref": "#/definitions/IterableDecoder"
3653
3685
  - "$ref": "#/definitions/XmlDecoder"
3654
3686
  - "$ref": "#/definitions/ZipfileDecoder"
3687
+ - "$ref": "#/definitions/CustomDecoder"
3655
3688
  download_decoder:
3656
3689
  title: Download Decoder
3657
3690
  description: Component decoding the download response so records can be extracted.
3658
3691
  anyOf:
3659
- - "$ref": "#/definitions/CustomDecoder"
3660
3692
  - "$ref": "#/definitions/CsvDecoder"
3661
3693
  - "$ref": "#/definitions/GzipDecoder"
3662
3694
  - "$ref": "#/definitions/JsonDecoder"
@@ -3664,6 +3696,7 @@ definitions:
3664
3696
  - "$ref": "#/definitions/IterableDecoder"
3665
3697
  - "$ref": "#/definitions/XmlDecoder"
3666
3698
  - "$ref": "#/definitions/ZipfileDecoder"
3699
+ - "$ref": "#/definitions/CustomDecoder"
3667
3700
  $parameters:
3668
3701
  type: object
3669
3702
  additionalProperties: true