dagster-airbyte 0.26.18__py3-none-any.whl → 0.28.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dagster_airbyte/__init__.py +12 -3
- dagster_airbyte/asset_decorator.py +5 -4
- dagster_airbyte/asset_defs.py +78 -31
- dagster_airbyte/components/__init__.py +0 -0
- dagster_airbyte/components/workspace_component/__init__.py +0 -0
- dagster_airbyte/components/workspace_component/component.py +433 -0
- dagster_airbyte/components/workspace_component/scaffolder.py +30 -0
- dagster_airbyte/legacy_resources.py +826 -0
- dagster_airbyte/managed/reconciliation.py +9 -7
- dagster_airbyte/ops.py +2 -1
- dagster_airbyte/resources.py +591 -937
- dagster_airbyte/translator.py +20 -1
- dagster_airbyte/utils.py +1 -1
- dagster_airbyte/version.py +1 -1
- {dagster_airbyte-0.26.18.dist-info → dagster_airbyte-0.28.3.dist-info}/METADATA +5 -5
- dagster_airbyte-0.28.3.dist-info/RECORD +28 -0
- {dagster_airbyte-0.26.18.dist-info → dagster_airbyte-0.28.3.dist-info}/entry_points.txt +3 -0
- dagster_airbyte-0.26.18.dist-info/RECORD +0 -23
- {dagster_airbyte-0.26.18.dist-info → dagster_airbyte-0.28.3.dist-info}/WHEEL +0 -0
- {dagster_airbyte-0.26.18.dist-info → dagster_airbyte-0.28.3.dist-info}/licenses/LICENSE +0 -0
- {dagster_airbyte-0.26.18.dist-info → dagster_airbyte-0.28.3.dist-info}/top_level.txt +0 -0
|
@@ -1,12 +1,14 @@
|
|
|
1
|
-
from collections.abc import Iterable, Mapping, Sequence
|
|
2
|
-
from typing import Any,
|
|
1
|
+
from collections.abc import Callable, Iterable, Mapping, Sequence
|
|
2
|
+
from typing import Any, Optional, Union, cast
|
|
3
3
|
|
|
4
4
|
import dagster._check as check
|
|
5
5
|
from dagster import AssetKey
|
|
6
6
|
from dagster._annotations import beta, deprecated, public
|
|
7
|
-
from dagster._core.definitions.
|
|
7
|
+
from dagster._core.definitions.assets.definition.cacheable_assets_definition import (
|
|
8
|
+
CacheableAssetsDefinition,
|
|
9
|
+
)
|
|
8
10
|
from dagster._core.definitions.events import CoercibleToAssetKeyPrefix
|
|
9
|
-
from dagster._core.definitions.freshness_policy import
|
|
11
|
+
from dagster._core.definitions.freshness_policy import LegacyFreshnessPolicy
|
|
10
12
|
from dagster._core.definitions.resource_definition import ResourceDefinition
|
|
11
13
|
from dagster._core.execution.context.init import build_init_resource_context
|
|
12
14
|
from dagster._utils.merger import deep_merge_dicts
|
|
@@ -26,6 +28,7 @@ from dagster_airbyte.asset_defs import (
|
|
|
26
28
|
AirbyteConnectionMetadata,
|
|
27
29
|
AirbyteInstanceCacheableAssetsDefinition,
|
|
28
30
|
)
|
|
31
|
+
from dagster_airbyte.legacy_resources import AirbyteResource
|
|
29
32
|
from dagster_airbyte.managed.types import (
|
|
30
33
|
MANAGED_ELEMENTS_DEPRECATION_MSG,
|
|
31
34
|
AirbyteConnection,
|
|
@@ -37,7 +40,6 @@ from dagster_airbyte.managed.types import (
|
|
|
37
40
|
InitializedAirbyteDestination,
|
|
38
41
|
InitializedAirbyteSource,
|
|
39
42
|
)
|
|
40
|
-
from dagster_airbyte.resources import AirbyteResource
|
|
41
43
|
from dagster_airbyte.utils import clean_name, is_basic_normalization_operation
|
|
42
44
|
|
|
43
45
|
|
|
@@ -698,7 +700,7 @@ class AirbyteManagedElementCacheableAssetsDefinition(AirbyteInstanceCacheableAss
|
|
|
698
700
|
connection_to_io_manager_key_fn: Optional[Callable[[str], Optional[str]]],
|
|
699
701
|
connection_to_asset_key_fn: Optional[Callable[[AirbyteConnectionMetadata, str], AssetKey]],
|
|
700
702
|
connection_to_freshness_policy_fn: Optional[
|
|
701
|
-
Callable[[AirbyteConnectionMetadata], Optional[
|
|
703
|
+
Callable[[AirbyteConnectionMetadata], Optional[LegacyFreshnessPolicy]]
|
|
702
704
|
],
|
|
703
705
|
):
|
|
704
706
|
defined_conn_names = {conn.name for conn in connections}
|
|
@@ -744,7 +746,7 @@ def load_assets_from_connections(
|
|
|
744
746
|
Callable[[AirbyteConnectionMetadata, str], AssetKey]
|
|
745
747
|
] = None,
|
|
746
748
|
connection_to_freshness_policy_fn: Optional[
|
|
747
|
-
Callable[[AirbyteConnectionMetadata], Optional[
|
|
749
|
+
Callable[[AirbyteConnectionMetadata], Optional[LegacyFreshnessPolicy]]
|
|
748
750
|
] = None,
|
|
749
751
|
) -> CacheableAssetsDefinition:
|
|
750
752
|
"""Loads Airbyte connection assets from a configured AirbyteResource instance, checking against a list of AirbyteConnection objects.
|
dagster_airbyte/ops.py
CHANGED
|
@@ -5,7 +5,8 @@ from dagster import Config, In, Nothing, Out, Output, op
|
|
|
5
5
|
from dagster._core.storage.tags import COMPUTE_KIND_TAG
|
|
6
6
|
from pydantic import Field
|
|
7
7
|
|
|
8
|
-
from dagster_airbyte.
|
|
8
|
+
from dagster_airbyte.legacy_resources import BaseAirbyteResource
|
|
9
|
+
from dagster_airbyte.resources import DEFAULT_POLL_INTERVAL_SECONDS
|
|
9
10
|
from dagster_airbyte.types import AirbyteOutput
|
|
10
11
|
from dagster_airbyte.utils import _get_attempt, generate_materializations
|
|
11
12
|
|