dagster-airbyte 0.26.18__py3-none-any.whl → 0.28.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dagster_airbyte/__init__.py +12 -3
- dagster_airbyte/asset_decorator.py +5 -4
- dagster_airbyte/asset_defs.py +78 -31
- dagster_airbyte/components/__init__.py +0 -0
- dagster_airbyte/components/workspace_component/__init__.py +0 -0
- dagster_airbyte/components/workspace_component/component.py +433 -0
- dagster_airbyte/components/workspace_component/scaffolder.py +30 -0
- dagster_airbyte/legacy_resources.py +826 -0
- dagster_airbyte/managed/reconciliation.py +9 -7
- dagster_airbyte/ops.py +2 -1
- dagster_airbyte/resources.py +591 -937
- dagster_airbyte/translator.py +20 -1
- dagster_airbyte/utils.py +1 -1
- dagster_airbyte/version.py +1 -1
- {dagster_airbyte-0.26.18.dist-info → dagster_airbyte-0.28.3.dist-info}/METADATA +5 -5
- dagster_airbyte-0.28.3.dist-info/RECORD +28 -0
- {dagster_airbyte-0.26.18.dist-info → dagster_airbyte-0.28.3.dist-info}/entry_points.txt +3 -0
- dagster_airbyte-0.26.18.dist-info/RECORD +0 -23
- {dagster_airbyte-0.26.18.dist-info → dagster_airbyte-0.28.3.dist-info}/WHEEL +0 -0
- {dagster_airbyte-0.26.18.dist-info → dagster_airbyte-0.28.3.dist-info}/licenses/LICENSE +0 -0
- {dagster_airbyte-0.26.18.dist-info → dagster_airbyte-0.28.3.dist-info}/top_level.txt +0 -0
dagster_airbyte/translator.py
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
from collections.abc import Mapping, Sequence
|
|
2
|
+
from datetime import datetime
|
|
2
3
|
from enum import Enum
|
|
3
4
|
from typing import Any, Optional
|
|
4
5
|
|
|
5
6
|
from dagster._annotations import beta, deprecated
|
|
6
7
|
from dagster._core.definitions.asset_key import AssetKey
|
|
7
|
-
from dagster._core.definitions.asset_spec import AssetSpec
|
|
8
|
+
from dagster._core.definitions.assets.definition.asset_spec import AssetSpec
|
|
8
9
|
from dagster._core.definitions.metadata.metadata_set import NamespacedMetadataSet, TableMetadataSet
|
|
9
10
|
from dagster._record import record
|
|
10
11
|
from dagster._utils.cached_method import cached_method
|
|
@@ -138,6 +139,13 @@ class AirbyteJob:
|
|
|
138
139
|
|
|
139
140
|
id: int
|
|
140
141
|
status: str
|
|
142
|
+
type: str
|
|
143
|
+
connection_id: str | None = None
|
|
144
|
+
start_time: datetime | None = None
|
|
145
|
+
last_updated_at: datetime | None = None
|
|
146
|
+
duration: str | None = None
|
|
147
|
+
bytes_synced: int | None = None
|
|
148
|
+
rows_synced: int | None = None
|
|
141
149
|
|
|
142
150
|
@classmethod
|
|
143
151
|
def from_job_details(
|
|
@@ -147,6 +155,17 @@ class AirbyteJob:
|
|
|
147
155
|
return cls(
|
|
148
156
|
id=job_details["jobId"],
|
|
149
157
|
status=job_details["status"],
|
|
158
|
+
type=job_details["jobType"],
|
|
159
|
+
connection_id=job_details.get("connectionId"),
|
|
160
|
+
start_time=datetime.fromisoformat(job_details["startTime"])
|
|
161
|
+
if "startTime" in job_details
|
|
162
|
+
else None,
|
|
163
|
+
last_updated_at=datetime.fromisoformat(job_details["lastUpdatedAt"])
|
|
164
|
+
if "lastUpdatedAt" in job_details
|
|
165
|
+
else None,
|
|
166
|
+
duration=job_details.get("duration"),
|
|
167
|
+
bytes_synced=job_details.get("bytesScanned"),
|
|
168
|
+
rows_synced=job_details.get("rowsSynced"),
|
|
150
169
|
)
|
|
151
170
|
|
|
152
171
|
|
dagster_airbyte/utils.py
CHANGED
|
@@ -13,7 +13,7 @@ from dagster._utils.names import clean_name_lower
|
|
|
13
13
|
from dagster_airbyte.types import AirbyteOutput
|
|
14
14
|
|
|
15
15
|
if TYPE_CHECKING:
|
|
16
|
-
from dagster_airbyte import DagsterAirbyteTranslator
|
|
16
|
+
from dagster_airbyte.translator import DagsterAirbyteTranslator
|
|
17
17
|
|
|
18
18
|
DAGSTER_AIRBYTE_TRANSLATOR_METADATA_KEY = "dagster-airbyte/dagster_airbyte_translator"
|
|
19
19
|
|
dagster_airbyte/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.
|
|
1
|
+
__version__ = "0.28.3"
|
|
@@ -1,26 +1,26 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-airbyte
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.28.3
|
|
4
4
|
Summary: Package for integrating Airbyte with Dagster.
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
|
|
6
6
|
Author: Dagster Labs
|
|
7
7
|
Author-email: hello@dagsterlabs.com
|
|
8
8
|
License: Apache-2.0
|
|
9
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
10
9
|
Classifier: Programming Language :: Python :: 3.10
|
|
11
10
|
Classifier: Programming Language :: Python :: 3.11
|
|
12
11
|
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
13
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
14
|
Classifier: Operating System :: OS Independent
|
|
15
|
-
Requires-Python: >=3.
|
|
15
|
+
Requires-Python: >=3.10,<3.14
|
|
16
16
|
License-File: LICENSE
|
|
17
|
-
Requires-Dist: dagster==1.
|
|
17
|
+
Requires-Dist: dagster==1.12.3
|
|
18
18
|
Requires-Dist: requests
|
|
19
19
|
Provides-Extra: test
|
|
20
20
|
Requires-Dist: requests-mock; extra == "test"
|
|
21
21
|
Requires-Dist: flaky; extra == "test"
|
|
22
22
|
Provides-Extra: managed
|
|
23
|
-
Requires-Dist: dagster-managed-elements==0.
|
|
23
|
+
Requires-Dist: dagster-managed-elements==0.28.3; extra == "managed"
|
|
24
24
|
Dynamic: author
|
|
25
25
|
Dynamic: author-email
|
|
26
26
|
Dynamic: classifier
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
dagster_airbyte/__init__.py,sha256=UO5Gf6M6q1c7bbL5uW6NH9HuNkaH2y_xVfqMY9tN7xk,2073
|
|
2
|
+
dagster_airbyte/asset_decorator.py,sha256=bCA22GG8ch5eGmm6BkztIfTY6ROXWJq1AInzO7fwJ2M,4667
|
|
3
|
+
dagster_airbyte/asset_defs.py,sha256=8yDuE2r1_4IAl9bY7BkbJpINWgJjtPY1t_WZeedGZho,51909
|
|
4
|
+
dagster_airbyte/cli.py,sha256=HErteP1MjfHozKKSrznh0yAreKETbXp5NDHzXGsdvvE,425
|
|
5
|
+
dagster_airbyte/legacy_resources.py,sha256=ks5doF0YZr7nXeU6ibQh1qiIMPAd_c2jusOkqdZKysc,31125
|
|
6
|
+
dagster_airbyte/ops.py,sha256=zDuROSIpqMhiioycTpLRgGnrp71lZNd4ts6-yybPg_s,4273
|
|
7
|
+
dagster_airbyte/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
|
|
8
|
+
dagster_airbyte/resources.py,sha256=Nb-cPHYtkzYjmIb7c0onA9hYLpe5JLcEjbXQf9_3Cf4,41654
|
|
9
|
+
dagster_airbyte/translator.py,sha256=-aT-Au1VqaqHnRvivOaHslK2C6db5eNKB3mAJbQwvw8,8364
|
|
10
|
+
dagster_airbyte/types.py,sha256=TYUjI3skjLYeANjesgJ-IAJNu8bAnL1ymsUfz5LsRTE,1565
|
|
11
|
+
dagster_airbyte/utils.py,sha256=wG9119kXi87JgcOjK7iNozr-svZocJBQYoHBMmnXZcE,4092
|
|
12
|
+
dagster_airbyte/version.py,sha256=PFj1PlFYtLhN7k4TLEHMneuQ4Ep8o7DCrl69TL8I0SY,23
|
|
13
|
+
dagster_airbyte/components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
+
dagster_airbyte/components/workspace_component/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
|
+
dagster_airbyte/components/workspace_component/component.py,sha256=yTc_uzU5--kbjfDzE0TbVHWqqrRu3kh1C-YzN-xWg34,16208
|
|
16
|
+
dagster_airbyte/components/workspace_component/scaffolder.py,sha256=zk5HDJ6C8zcfUWKslJVyqyBB0LV3JzNAocY2ptT517s,1046
|
|
17
|
+
dagster_airbyte/managed/__init__.py,sha256=6SBtyNOMJ9Cu2UIwFExJHpL_ZVFo3rPMvyIxVOsKvWE,469
|
|
18
|
+
dagster_airbyte/managed/reconciliation.py,sha256=V9bNMwNRvyIXF0O3cSaPlw3CmpbVaWQVEQarhrXf7n0,34866
|
|
19
|
+
dagster_airbyte/managed/types.py,sha256=isPfX8L9YwtZAf9Vk4hhxBePLR00AEldsdK2TsM1H2o,14611
|
|
20
|
+
dagster_airbyte/managed/generated/__init__.py,sha256=eYq-yfXEeffuKAVFXY8plD0se1wHjFNVqklpbu9gljw,108
|
|
21
|
+
dagster_airbyte/managed/generated/destinations.py,sha256=x1wmWlXvOJHtfaZva3ErdKuVS--sDvfidSXR5ji9G5w,119692
|
|
22
|
+
dagster_airbyte/managed/generated/sources.py,sha256=y0TPNvcRd8c9mhje-NoXsHeKRPt1nXcpww8mNAtqCps,282685
|
|
23
|
+
dagster_airbyte-0.28.3.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
|
|
24
|
+
dagster_airbyte-0.28.3.dist-info/METADATA,sha256=92qHM2fBXVeMMibGM-YiIvuDqe43_ZwxmMrAuKJByUI,1117
|
|
25
|
+
dagster_airbyte-0.28.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
26
|
+
dagster_airbyte-0.28.3.dist-info/entry_points.txt,sha256=096yvfMP-gNsCgDg9vDQtinis5QGpD-e_kHEhcHaML8,120
|
|
27
|
+
dagster_airbyte-0.28.3.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
|
|
28
|
+
dagster_airbyte-0.28.3.dist-info/RECORD,,
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
dagster_airbyte/__init__.py,sha256=Cdn8CiAL5HSFeFjmKIZgXJqccj11xqQFzRfUQlD15b0,1720
|
|
2
|
-
dagster_airbyte/asset_decorator.py,sha256=U6pbeyjE0meeXHed7ZVhaRfAsvLwTvDB14BSyDUYHqY,4565
|
|
3
|
-
dagster_airbyte/asset_defs.py,sha256=X-ong0InnxXe61_H9IZc6ATCACylKb-JQrPbtH-XTHo,50019
|
|
4
|
-
dagster_airbyte/cli.py,sha256=HErteP1MjfHozKKSrznh0yAreKETbXp5NDHzXGsdvvE,425
|
|
5
|
-
dagster_airbyte/ops.py,sha256=oOEczVYpqVRTc1-0osfpR5FZbPjNJDYihgRjk_qtOQA,4229
|
|
6
|
-
dagster_airbyte/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
|
|
7
|
-
dagster_airbyte/resources.py,sha256=ynOI_MbPnKwcW9OhqrbDchmP6RY4eXQ1018PFJokg4g,54090
|
|
8
|
-
dagster_airbyte/translator.py,sha256=CL2EKeZIahgoFvtZpfEwVa2KmCuvUENXByM6F7369LQ,7529
|
|
9
|
-
dagster_airbyte/types.py,sha256=TYUjI3skjLYeANjesgJ-IAJNu8bAnL1ymsUfz5LsRTE,1565
|
|
10
|
-
dagster_airbyte/utils.py,sha256=SupW5PLwzyZHWTHKe17vh2PlGdcRT7xbYbMCT-ubGDo,4081
|
|
11
|
-
dagster_airbyte/version.py,sha256=rX7I8qgJ5-T_gu8HTMqYW-dXs0krqtSxfKQANv9bF0M,24
|
|
12
|
-
dagster_airbyte/managed/__init__.py,sha256=6SBtyNOMJ9Cu2UIwFExJHpL_ZVFo3rPMvyIxVOsKvWE,469
|
|
13
|
-
dagster_airbyte/managed/reconciliation.py,sha256=LHtJc2zRwk1gAc86pKVSZ0rnfLNTT9r3usRFQilYCwY,34803
|
|
14
|
-
dagster_airbyte/managed/types.py,sha256=isPfX8L9YwtZAf9Vk4hhxBePLR00AEldsdK2TsM1H2o,14611
|
|
15
|
-
dagster_airbyte/managed/generated/__init__.py,sha256=eYq-yfXEeffuKAVFXY8plD0se1wHjFNVqklpbu9gljw,108
|
|
16
|
-
dagster_airbyte/managed/generated/destinations.py,sha256=x1wmWlXvOJHtfaZva3ErdKuVS--sDvfidSXR5ji9G5w,119692
|
|
17
|
-
dagster_airbyte/managed/generated/sources.py,sha256=y0TPNvcRd8c9mhje-NoXsHeKRPt1nXcpww8mNAtqCps,282685
|
|
18
|
-
dagster_airbyte-0.26.18.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
|
|
19
|
-
dagster_airbyte-0.26.18.dist-info/METADATA,sha256=kJwOhe4l6P9_tIt5-15L54zyXQbEVHRH1J-PlvG_A4o,1118
|
|
20
|
-
dagster_airbyte-0.26.18.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
21
|
-
dagster_airbyte-0.26.18.dist-info/entry_points.txt,sha256=XrbLOz3LpgPV5fdwMmgdP6Rp1AfSG07KeWIddLqh7Lw,61
|
|
22
|
-
dagster_airbyte-0.26.18.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
|
|
23
|
-
dagster_airbyte-0.26.18.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|