airbyte-cdk 6.6.8rc12__py3-none-any.whl → 6.6.8rc13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/source_declarative_manifest/_run.py +2 -1
- airbyte_cdk/config_observation.py +2 -1
- airbyte_cdk/connector.py +1 -0
- airbyte_cdk/connector_builder/connector_builder_handler.py +1 -1
- airbyte_cdk/connector_builder/main.py +2 -1
- airbyte_cdk/destinations/destination.py +2 -1
- airbyte_cdk/destinations/vector_db_based/config.py +2 -1
- airbyte_cdk/destinations/vector_db_based/document_processor.py +4 -3
- airbyte_cdk/destinations/vector_db_based/embedder.py +5 -4
- airbyte_cdk/entrypoint.py +3 -2
- airbyte_cdk/logger.py +2 -1
- airbyte_cdk/models/airbyte_protocol.py +2 -1
- airbyte_cdk/sources/config.py +2 -1
- airbyte_cdk/sources/declarative/auth/jwt.py +1 -0
- airbyte_cdk/sources/declarative/auth/oauth.py +1 -0
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +1 -0
- airbyte_cdk/sources/declarative/auth/token.py +2 -1
- airbyte_cdk/sources/declarative/auth/token_provider.py +3 -2
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +6 -4
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +3 -2
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +1 -0
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +1 -0
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +1 -0
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +1 -0
- airbyte_cdk/sources/declarative/extractors/http_selector.py +1 -0
- airbyte_cdk/sources/declarative/extractors/record_selector.py +1 -0
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +2 -1
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +2 -1
- airbyte_cdk/sources/declarative/interpolation/jinja.py +5 -4
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +4 -3
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +1 -1
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +3 -2
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +1 -0
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +2 -1
- airbyte_cdk/sources/declarative/requesters/http_requester.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +1 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +2 -1
- airbyte_cdk/sources/declarative/requesters/requester.py +1 -0
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +2 -1
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +1 -0
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +1 -1
- airbyte_cdk/sources/declarative/transformations/add_fields.py +1 -0
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +1 -0
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +1 -0
- airbyte_cdk/sources/embedded/tools.py +1 -0
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +2 -1
- airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
- airbyte_cdk/sources/file_based/config/csv_format.py +2 -1
- airbyte_cdk/sources/file_based/config/excel_format.py +2 -1
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +2 -1
- airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
- airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
- airbyte_cdk/sources/file_based/config/unstructured_format.py +2 -1
- airbyte_cdk/sources/file_based/file_based_source.py +2 -1
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +2 -1
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +1 -0
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +2 -1
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +5 -5
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +2 -1
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +2 -1
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +9 -8
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +2 -1
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +2 -1
- airbyte_cdk/sources/http_logger.py +1 -0
- airbyte_cdk/sources/streams/call_rate.py +1 -2
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +2 -1
- airbyte_cdk/sources/streams/concurrent/adapters.py +2 -2
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +2 -1
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +1 -1
- airbyte_cdk/sources/streams/core.py +2 -1
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +2 -1
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +1 -0
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +1 -0
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +2 -1
- airbyte_cdk/sources/streams/http/http.py +3 -2
- airbyte_cdk/sources/streams/http/http_client.py +34 -3
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +2 -1
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +1 -0
- airbyte_cdk/sources/utils/schema_helpers.py +3 -2
- airbyte_cdk/sql/secrets.py +2 -1
- airbyte_cdk/sql/shared/sql_processor.py +8 -6
- airbyte_cdk/test/entrypoint_wrapper.py +4 -3
- airbyte_cdk/test/mock_http/mocker.py +1 -0
- airbyte_cdk/utils/schema_inferrer.py +2 -1
- airbyte_cdk/utils/slice_hasher.py +1 -1
- airbyte_cdk/utils/traced_exception.py +2 -1
- {airbyte_cdk-6.6.8rc12.dist-info → airbyte_cdk-6.6.8rc13.dist-info}/METADATA +8 -1
- {airbyte_cdk-6.6.8rc12.dist-info → airbyte_cdk-6.6.8rc13.dist-info}/RECORD +106 -106
- {airbyte_cdk-6.6.8rc12.dist-info → airbyte_cdk-6.6.8rc13.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.6.8rc12.dist-info → airbyte_cdk-6.6.8rc13.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.6.8rc12.dist-info → airbyte_cdk-6.6.8rc13.dist-info}/entry_points.txt +0 -0
@@ -25,6 +25,8 @@ from datetime import datetime
|
|
25
25
|
from pathlib import Path
|
26
26
|
from typing import Any, cast
|
27
27
|
|
28
|
+
from orjson import orjson
|
29
|
+
|
28
30
|
from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
|
29
31
|
from airbyte_cdk.models import (
|
30
32
|
AirbyteErrorTraceMessage,
|
@@ -42,7 +44,6 @@ from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
|
|
42
44
|
)
|
43
45
|
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
|
44
46
|
from airbyte_cdk.sources.source import TState
|
45
|
-
from orjson import orjson
|
46
47
|
|
47
48
|
|
48
49
|
class SourceLocalYaml(YamlDeclarativeSource):
|
@@ -10,6 +10,8 @@ import time
|
|
10
10
|
from copy import copy
|
11
11
|
from typing import Any, List, MutableMapping
|
12
12
|
|
13
|
+
from orjson import orjson
|
14
|
+
|
13
15
|
from airbyte_cdk.models import (
|
14
16
|
AirbyteControlConnectorConfigMessage,
|
15
17
|
AirbyteControlMessage,
|
@@ -18,7 +20,6 @@ from airbyte_cdk.models import (
|
|
18
20
|
OrchestratorType,
|
19
21
|
Type,
|
20
22
|
)
|
21
|
-
from orjson import orjson
|
22
23
|
|
23
24
|
|
24
25
|
class ObservedDict(dict): # type: ignore # disallow_any_generics is set to True, and dict is equivalent to dict[Any]
|
airbyte_cdk/connector.py
CHANGED
@@ -12,8 +12,8 @@ from airbyte_cdk.models import (
|
|
12
12
|
AirbyteRecordMessage,
|
13
13
|
AirbyteStateMessage,
|
14
14
|
ConfiguredAirbyteCatalog,
|
15
|
+
Type,
|
15
16
|
)
|
16
|
-
from airbyte_cdk.models import Type
|
17
17
|
from airbyte_cdk.models import Type as MessageType
|
18
18
|
from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
|
19
19
|
from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
|
@@ -6,6 +6,8 @@
|
|
6
6
|
import sys
|
7
7
|
from typing import Any, List, Mapping, Optional, Tuple
|
8
8
|
|
9
|
+
from orjson import orjson
|
10
|
+
|
9
11
|
from airbyte_cdk.connector import BaseConnector
|
10
12
|
from airbyte_cdk.connector_builder.connector_builder_handler import (
|
11
13
|
TestReadLimits,
|
@@ -25,7 +27,6 @@ from airbyte_cdk.models import (
|
|
25
27
|
from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
|
26
28
|
from airbyte_cdk.sources.source import Source
|
27
29
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
28
|
-
from orjson import orjson
|
29
30
|
|
30
31
|
|
31
32
|
def get_config_and_catalog_from_args(
|
@@ -9,6 +9,8 @@ import sys
|
|
9
9
|
from abc import ABC, abstractmethod
|
10
10
|
from typing import Any, Iterable, List, Mapping
|
11
11
|
|
12
|
+
from orjson import orjson
|
13
|
+
|
12
14
|
from airbyte_cdk.connector import Connector
|
13
15
|
from airbyte_cdk.exception_handler import init_uncaught_exception_handler
|
14
16
|
from airbyte_cdk.models import (
|
@@ -20,7 +22,6 @@ from airbyte_cdk.models import (
|
|
20
22
|
)
|
21
23
|
from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
|
22
24
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
23
|
-
from orjson import orjson
|
24
25
|
|
25
26
|
logger = logging.getLogger("airbyte")
|
26
27
|
|
@@ -5,9 +5,10 @@
|
|
5
5
|
from typing import Any, Dict, List, Literal, Optional, Union
|
6
6
|
|
7
7
|
import dpath
|
8
|
+
from pydantic.v1 import BaseModel, Field
|
9
|
+
|
8
10
|
from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
|
9
11
|
from airbyte_cdk.utils.spec_schema_transformations import resolve_refs
|
10
|
-
from pydantic.v1 import BaseModel, Field
|
11
12
|
|
12
13
|
|
13
14
|
class SeparatorSplitterConfigModel(BaseModel):
|
@@ -8,6 +8,10 @@ from dataclasses import dataclass
|
|
8
8
|
from typing import Any, Dict, List, Mapping, Optional, Tuple
|
9
9
|
|
10
10
|
import dpath
|
11
|
+
from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
|
12
|
+
from langchain.utils import stringify_dict
|
13
|
+
from langchain_core.documents.base import Document
|
14
|
+
|
11
15
|
from airbyte_cdk.destinations.vector_db_based.config import (
|
12
16
|
ProcessingConfigModel,
|
13
17
|
SeparatorSplitterConfigModel,
|
@@ -21,9 +25,6 @@ from airbyte_cdk.models import (
|
|
21
25
|
DestinationSyncMode,
|
22
26
|
)
|
23
27
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType
|
24
|
-
from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
|
25
|
-
from langchain.utils import stringify_dict
|
26
|
-
from langchain_core.documents.base import Document
|
27
28
|
|
28
29
|
METADATA_STREAM_FIELD = "_ab_stream"
|
29
30
|
METADATA_RECORD_ID_FIELD = "_ab_record_id"
|
@@ -7,6 +7,11 @@ from abc import ABC, abstractmethod
|
|
7
7
|
from dataclasses import dataclass
|
8
8
|
from typing import List, Optional, Union, cast
|
9
9
|
|
10
|
+
from langchain.embeddings.cohere import CohereEmbeddings
|
11
|
+
from langchain.embeddings.fake import FakeEmbeddings
|
12
|
+
from langchain.embeddings.localai import LocalAIEmbeddings
|
13
|
+
from langchain.embeddings.openai import OpenAIEmbeddings
|
14
|
+
|
10
15
|
from airbyte_cdk.destinations.vector_db_based.config import (
|
11
16
|
AzureOpenAIEmbeddingConfigModel,
|
12
17
|
CohereEmbeddingConfigModel,
|
@@ -19,10 +24,6 @@ from airbyte_cdk.destinations.vector_db_based.config import (
|
|
19
24
|
from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, format_exception
|
20
25
|
from airbyte_cdk.models import AirbyteRecordMessage
|
21
26
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType
|
22
|
-
from langchain.embeddings.cohere import CohereEmbeddings
|
23
|
-
from langchain.embeddings.fake import FakeEmbeddings
|
24
|
-
from langchain.embeddings.localai import LocalAIEmbeddings
|
25
|
-
from langchain.embeddings.openai import OpenAIEmbeddings
|
26
27
|
|
27
28
|
|
28
29
|
@dataclass
|
airbyte_cdk/entrypoint.py
CHANGED
@@ -16,6 +16,9 @@ from typing import Any, DefaultDict, Iterable, List, Mapping, Optional
|
|
16
16
|
from urllib.parse import urlparse
|
17
17
|
|
18
18
|
import requests
|
19
|
+
from orjson import orjson
|
20
|
+
from requests import PreparedRequest, Response, Session
|
21
|
+
|
19
22
|
from airbyte_cdk.connector import TConfig
|
20
23
|
from airbyte_cdk.exception_handler import init_uncaught_exception_handler
|
21
24
|
from airbyte_cdk.logger import init_logger
|
@@ -38,8 +41,6 @@ from airbyte_cdk.utils import is_cloud_environment, message_utils
|
|
38
41
|
from airbyte_cdk.utils.airbyte_secrets_utils import get_secrets, update_secrets
|
39
42
|
from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH
|
40
43
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
41
|
-
from orjson import orjson
|
42
|
-
from requests import PreparedRequest, Response, Session
|
43
44
|
|
44
45
|
logger = init_logger("airbyte")
|
45
46
|
|
airbyte_cdk/logger.py
CHANGED
@@ -7,6 +7,8 @@ import logging
|
|
7
7
|
import logging.config
|
8
8
|
from typing import Any, Callable, Mapping, Optional, Tuple
|
9
9
|
|
10
|
+
from orjson import orjson
|
11
|
+
|
10
12
|
from airbyte_cdk.models import (
|
11
13
|
AirbyteLogMessage,
|
12
14
|
AirbyteMessage,
|
@@ -15,7 +17,6 @@ from airbyte_cdk.models import (
|
|
15
17
|
Type,
|
16
18
|
)
|
17
19
|
from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
|
18
|
-
from orjson import orjson
|
19
20
|
|
20
21
|
LOGGING_CONFIG = {
|
21
22
|
"version": 1,
|
@@ -5,10 +5,11 @@
|
|
5
5
|
from dataclasses import InitVar, dataclass
|
6
6
|
from typing import Annotated, Any, Dict, List, Mapping, Optional, Union
|
7
7
|
|
8
|
-
from airbyte_cdk.models.file_transfer_record_message import AirbyteFileTransferRecordMessage
|
9
8
|
from airbyte_protocol_dataclasses.models import * # noqa: F403 # Allow '*'
|
10
9
|
from serpyco_rs.metadata import Alias
|
11
10
|
|
11
|
+
from airbyte_cdk.models.file_transfer_record_message import AirbyteFileTransferRecordMessage
|
12
|
+
|
12
13
|
# ruff: noqa: F405 # ignore fuzzy import issues with 'import *'
|
13
14
|
|
14
15
|
|
airbyte_cdk/sources/config.py
CHANGED
@@ -4,9 +4,10 @@
|
|
4
4
|
|
5
5
|
from typing import Any, Dict
|
6
6
|
|
7
|
-
from airbyte_cdk.sources.utils.schema_helpers import expand_refs, rename_key
|
8
7
|
from pydantic.v1 import BaseModel
|
9
8
|
|
9
|
+
from airbyte_cdk.sources.utils.schema_helpers import expand_refs, rename_key
|
10
|
+
|
10
11
|
|
11
12
|
class BaseConfig(BaseModel):
|
12
13
|
"""Base class for connector spec, adds the following behaviour:
|
@@ -8,6 +8,7 @@ from datetime import datetime
|
|
8
8
|
from typing import Any, Mapping, Optional, Union
|
9
9
|
|
10
10
|
import jwt
|
11
|
+
|
11
12
|
from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
|
12
13
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
13
14
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
|
|
6
6
|
from typing import Any, List, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import pendulum
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
|
10
11
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
|
11
12
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
@@ -8,6 +8,8 @@ from dataclasses import InitVar, dataclass
|
|
8
8
|
from typing import Any, Mapping, Union
|
9
9
|
|
10
10
|
import requests
|
11
|
+
from cachetools import TTLCache, cached
|
12
|
+
|
11
13
|
from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
|
12
14
|
from airbyte_cdk.sources.declarative.auth.token_provider import TokenProvider
|
13
15
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
@@ -16,7 +18,6 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
|
|
16
18
|
RequestOptionType,
|
17
19
|
)
|
18
20
|
from airbyte_cdk.sources.types import Config
|
19
|
-
from cachetools import TTLCache, cached
|
20
21
|
|
21
22
|
|
22
23
|
@dataclass
|
@@ -10,6 +10,9 @@ from typing import Any, List, Mapping, Optional, Union
|
|
10
10
|
|
11
11
|
import dpath
|
12
12
|
import pendulum
|
13
|
+
from isodate import Duration
|
14
|
+
from pendulum import DateTime
|
15
|
+
|
13
16
|
from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
|
14
17
|
from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
|
15
18
|
from airbyte_cdk.sources.declarative.exceptions import ReadException
|
@@ -18,8 +21,6 @@ from airbyte_cdk.sources.declarative.requesters.requester import Requester
|
|
18
21
|
from airbyte_cdk.sources.http_logger import format_http_message
|
19
22
|
from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
|
20
23
|
from airbyte_cdk.sources.types import Config
|
21
|
-
from isodate import Duration
|
22
|
-
from pendulum import DateTime
|
23
24
|
|
24
25
|
|
25
26
|
class TokenProvider:
|
@@ -3,7 +3,7 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
import logging
|
6
|
-
from typing import Any, Generic, Iterator, List, Mapping, Optional, Tuple, Union
|
6
|
+
from typing import Any, Callable, Generic, Iterator, List, Mapping, Optional, Tuple, Union
|
7
7
|
|
8
8
|
from airbyte_cdk.models import (
|
9
9
|
AirbyteCatalog,
|
@@ -27,14 +27,16 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
27
27
|
)
|
28
28
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
29
29
|
DatetimeBasedCursor as DatetimeBasedCursorModel,
|
30
|
+
)
|
31
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
30
32
|
DeclarativeStream as DeclarativeStreamModel,
|
31
33
|
)
|
32
34
|
from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
|
33
|
-
ModelToComponentFactory,
|
34
35
|
ComponentDefinition,
|
36
|
+
ModelToComponentFactory,
|
35
37
|
)
|
36
38
|
from airbyte_cdk.sources.declarative.requesters import HttpRequester
|
37
|
-
from airbyte_cdk.sources.declarative.retrievers import
|
39
|
+
from airbyte_cdk.sources.declarative.retrievers import Retriever, SimpleRetriever
|
38
40
|
from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import (
|
39
41
|
DeclarativePartitionFactory,
|
40
42
|
StreamSlicerPartitionGenerator,
|
@@ -42,7 +44,6 @@ from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_genera
|
|
42
44
|
from airbyte_cdk.sources.declarative.transformations.add_fields import AddFields
|
43
45
|
from airbyte_cdk.sources.declarative.types import ConnectionDefinition
|
44
46
|
from airbyte_cdk.sources.source import TState
|
45
|
-
from airbyte_cdk.sources.types import Config, StreamState
|
46
47
|
from airbyte_cdk.sources.streams import Stream
|
47
48
|
from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
|
48
49
|
from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
|
@@ -50,6 +51,7 @@ from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
|
|
50
51
|
)
|
51
52
|
from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
|
52
53
|
from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream
|
54
|
+
from airbyte_cdk.sources.types import Config, StreamState
|
53
55
|
|
54
56
|
|
55
57
|
class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
@@ -5,11 +5,12 @@ import codecs
|
|
5
5
|
import logging
|
6
6
|
from dataclasses import InitVar, dataclass
|
7
7
|
from gzip import decompress
|
8
|
-
from typing import Any, Generator, Mapping, MutableMapping,
|
8
|
+
from typing import Any, Generator, List, Mapping, MutableMapping, Optional
|
9
9
|
|
10
|
+
import orjson
|
10
11
|
import requests
|
12
|
+
|
11
13
|
from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
|
12
|
-
import orjson
|
13
14
|
|
14
15
|
logger = logging.getLogger("airbyte")
|
15
16
|
|
@@ -7,6 +7,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Union
|
|
7
7
|
|
8
8
|
import dpath
|
9
9
|
import requests
|
10
|
+
|
10
11
|
from airbyte_cdk.sources.declarative.decoders import Decoder, JsonDecoder
|
11
12
|
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
|
12
13
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
|
|
6
6
|
from typing import Any, Iterable, List, Mapping, Optional
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
|
10
11
|
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
|
11
12
|
from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
|
@@ -10,9 +10,10 @@ from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
|
|
10
10
|
|
11
11
|
import pandas as pd
|
12
12
|
import requests
|
13
|
-
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
|
14
13
|
from numpy import nan
|
15
14
|
|
15
|
+
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
|
16
|
+
|
16
17
|
EMPTY_STR: str = ""
|
17
18
|
DEFAULT_ENCODING: str = "utf-8"
|
18
19
|
DOWNLOAD_CHUNK_SIZE: int = 1024 * 10
|
@@ -7,6 +7,8 @@ from dataclasses import InitVar, dataclass, field
|
|
7
7
|
from datetime import timedelta
|
8
8
|
from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Union
|
9
9
|
|
10
|
+
from isodate import Duration, duration_isoformat, parse_duration
|
11
|
+
|
10
12
|
from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
|
11
13
|
from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
|
12
14
|
from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
|
@@ -19,7 +21,6 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
|
|
19
21
|
)
|
20
22
|
from airbyte_cdk.sources.message import MessageRepository
|
21
23
|
from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
|
22
|
-
from isodate import Duration, duration_isoformat, parse_duration
|
23
24
|
|
24
25
|
|
25
26
|
@dataclass
|
@@ -6,15 +6,16 @@ import ast
|
|
6
6
|
from functools import cache
|
7
7
|
from typing import Any, Mapping, Optional, Tuple, Type
|
8
8
|
|
9
|
-
from airbyte_cdk.sources.declarative.interpolation.filters import filters
|
10
|
-
from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
|
11
|
-
from airbyte_cdk.sources.declarative.interpolation.macros import macros
|
12
|
-
from airbyte_cdk.sources.types import Config
|
13
9
|
from jinja2 import meta
|
14
10
|
from jinja2.environment import Template
|
15
11
|
from jinja2.exceptions import UndefinedError
|
16
12
|
from jinja2.sandbox import SandboxedEnvironment
|
17
13
|
|
14
|
+
from airbyte_cdk.sources.declarative.interpolation.filters import filters
|
15
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
|
16
|
+
from airbyte_cdk.sources.declarative.interpolation.macros import macros
|
17
|
+
from airbyte_cdk.sources.types import Config
|
18
|
+
|
18
19
|
|
19
20
|
class StreamPartitionAccessEnvironment(SandboxedEnvironment):
|
20
21
|
"""
|
@@ -8,9 +8,12 @@ import pkgutil
|
|
8
8
|
from copy import deepcopy
|
9
9
|
from importlib import metadata
|
10
10
|
from typing import Any, Dict, Iterator, List, Mapping, Optional
|
11
|
-
from packaging.version import Version, InvalidVersion
|
12
11
|
|
13
12
|
import yaml
|
13
|
+
from jsonschema.exceptions import ValidationError
|
14
|
+
from jsonschema.validators import validate
|
15
|
+
from packaging.version import InvalidVersion, Version
|
16
|
+
|
14
17
|
from airbyte_cdk.models import (
|
15
18
|
AirbyteConnectionStatus,
|
16
19
|
AirbyteMessage,
|
@@ -44,8 +47,6 @@ from airbyte_cdk.sources.utils.slice_logger import (
|
|
44
47
|
DebugSliceLogger,
|
45
48
|
SliceLogger,
|
46
49
|
)
|
47
|
-
from jsonschema.exceptions import ValidationError
|
48
|
-
from jsonschema.validators import validate
|
49
50
|
|
50
51
|
|
51
52
|
class ManifestDeclarativeSource(DeclarativeSource):
|
@@ -5,9 +5,9 @@ from typing import Any, Mapping
|
|
5
5
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
6
6
|
from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
|
7
7
|
from airbyte_cdk.sources.declarative.models import (
|
8
|
+
CustomIncrementalSync,
|
8
9
|
DatetimeBasedCursor,
|
9
10
|
SubstreamPartitionRouter,
|
10
|
-
CustomIncrementalSync,
|
11
11
|
)
|
12
12
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import ParentStreamConfig
|
13
13
|
|
@@ -25,6 +25,9 @@ from typing import (
|
|
25
25
|
get_type_hints,
|
26
26
|
)
|
27
27
|
|
28
|
+
from isodate import parse_duration
|
29
|
+
from pydantic.v1 import BaseModel
|
30
|
+
|
28
31
|
from airbyte_cdk.models import FailureType, Level
|
29
32
|
from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
|
30
33
|
from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator
|
@@ -367,8 +370,6 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_sta
|
|
367
370
|
from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction
|
368
371
|
from airbyte_cdk.sources.types import Config
|
369
372
|
from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
|
370
|
-
from isodate import parse_duration
|
371
|
-
from pydantic.v1 import BaseModel
|
372
373
|
|
373
374
|
ComponentDefinition = Mapping[str, Any]
|
374
375
|
|
@@ -7,6 +7,7 @@ from dataclasses import InitVar, dataclass
|
|
7
7
|
from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
|
8
8
|
|
9
9
|
import dpath
|
10
|
+
|
10
11
|
from airbyte_cdk.models import AirbyteMessage
|
11
12
|
from airbyte_cdk.models import Type as MessageType
|
12
13
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
10
11
|
from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
|
11
12
|
from airbyte_cdk.sources.types import Config
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
10
11
|
from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
|
11
12
|
from airbyte_cdk.sources.types import Config
|
@@ -7,6 +7,7 @@ from dataclasses import InitVar, dataclass
|
|
7
7
|
from typing import Any, Mapping, Optional, Union
|
8
8
|
|
9
9
|
import requests
|
10
|
+
|
10
11
|
from airbyte_cdk.models import FailureType
|
11
12
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
12
13
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
|
@@ -9,6 +9,7 @@ from dataclasses import InitVar, dataclass
|
|
9
9
|
from typing import Any, Mapping, Optional, Union
|
10
10
|
|
11
11
|
import requests
|
12
|
+
|
12
13
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
13
14
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
|
14
15
|
get_numeric_value_from_header,
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, List, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
|
10
11
|
from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
|
11
12
|
ErrorResolution,
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
|
|
6
6
|
from typing import Any, List, Mapping, MutableMapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.default_http_response_filter import (
|
10
11
|
DefaultHttpResponseFilter,
|
11
12
|
)
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Set, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.models import FailureType
|
10
11
|
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
11
12
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
@@ -6,6 +6,8 @@ from datetime import timedelta
|
|
6
6
|
from typing import Any, Dict, Iterable, Mapping, Optional
|
7
7
|
|
8
8
|
import requests
|
9
|
+
from requests import Response
|
10
|
+
|
9
11
|
from airbyte_cdk import AirbyteMessage
|
10
12
|
from airbyte_cdk.logger import lazy_log
|
11
13
|
from airbyte_cdk.models import FailureType, Type
|
@@ -23,7 +25,6 @@ from airbyte_cdk.sources.declarative.requesters.requester import Requester
|
|
23
25
|
from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
|
24
26
|
from airbyte_cdk.sources.types import Record, StreamSlice
|
25
27
|
from airbyte_cdk.utils import AirbyteTracedException
|
26
|
-
from requests import Response
|
27
28
|
|
28
29
|
LOGGER = logging.getLogger("airbyte")
|
29
30
|
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, MutableMapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator
|
10
11
|
from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
|
11
12
|
|