dagster-airbyte 0.24.13__py3-none-any.whl → 0.25.0__py3-none-any.whl

dagster_airbyte/__init__.py

@@ -17,7 +17,6 @@ except ImportError:
 from dagster_airbyte.asset_defs import (
     build_airbyte_assets as build_airbyte_assets,
     load_assets_from_airbyte_instance as load_assets_from_airbyte_instance,
-    load_assets_from_airbyte_project as load_assets_from_airbyte_project,
 )
 from dagster_airbyte.ops import airbyte_sync_op as airbyte_sync_op
 from dagster_airbyte.resources import (

dagster_airbyte/asset_defs.py

@@ -33,7 +33,6 @@ from dagster import (
     SourceAsset,
     _check as check,
 )
-from dagster._annotations import deprecated
 from dagster._core.definitions import AssetsDefinition, multi_asset
 from dagster._core.definitions.cacheable_assets import (
     AssetsDefinitionCacheableData,
@@ -116,11 +115,11 @@ def _build_airbyte_asset_defn_metadata(
     for table in destination_tables:
         internal_deps[table] = set(upstream_assets or [])
 
-    relation_identifiers: Dict[str, str] = {}
+    table_names: Dict[str, str] = {}
     for table in destination_tables:
         if destination_database and destination_schema and table:
-            # Use the destination raw table name to create the relation identifier
-            relation_identifiers[table] = ".".join(
+            # Use the destination raw table name to create the table name
+            table_names[table] = ".".join(
                 [
                     destination_database,
                     destination_schema,
@@ -129,7 +128,7 @@ def _build_airbyte_asset_defn_metadata(
             )
             if normalization_tables and normalization_raw_table_names_by_table:
                 for normalization_table in normalization_tables.get(table, set()):
-                    relation_identifiers[normalization_table] = ".".join(
+                    table_names[normalization_table] = ".".join(
                         [
                             destination_database,
                             destination_schema,
@@ -156,7 +155,7 @@ def _build_airbyte_asset_defn_metadata(
                 table: {
                     **TableMetadataSet(
                         column_schema=schema_by_table_name.get(table),
-                        relation_identifier=relation_identifiers.get(table),
+                        table_name=table_names.get(table),
                     ),
                 }
                 for table in tables
@@ -302,15 +301,13 @@ def build_airbyte_assets(
         chain([destination_tables], normalization_tables.values() if normalization_tables else [])
     )
 
-    relation_identifiers: Dict[str, str] = {}
+    table_names: Dict[str, str] = {}
     for table in destination_tables:
         if destination_database and destination_schema and table:
-            relation_identifiers[table] = ".".join(
-                [destination_database, destination_schema, table]
-            )
+            table_names[table] = ".".join([destination_database, destination_schema, table])
             if normalization_tables:
                 for normalization_table in normalization_tables.get(table, set()):
-                    relation_identifiers[normalization_table] = ".".join(
+                    table_names[normalization_table] = ".".join(
                         [
                             destination_database,
                             destination_schema,
@@ -328,7 +325,7 @@ def build_airbyte_assets(
                 {
                     **TableMetadataSet(
                         column_schema=schema_by_table_name.get(table),
-                        relation_identifier=relation_identifiers.get(table),
+                        table_name=table_names.get(table),
                     ),
                 }
             ),
@@ -1035,129 +1032,3 @@ def load_assets_from_airbyte_instance(
         connection_to_freshness_policy_fn=connection_to_freshness_policy_fn,
         connection_to_auto_materialize_policy_fn=connection_to_auto_materialize_policy_fn,
     )
-
-
-@deprecated(
-    breaking_version="1.9",
-    additional_warn_text="The Airbyte Octavia CLI has been deprecated. Consider using load_assets_from_airbyte_instance instead.",
-)
-def load_assets_from_airbyte_project(
-    project_dir: str,
-    workspace_id: Optional[str] = None,
-    key_prefix: Optional[CoercibleToAssetKeyPrefix] = None,
-    create_assets_for_normalization_tables: bool = True,
-    connection_to_group_fn: Optional[Callable[[str], Optional[str]]] = _clean_name,
-    connection_meta_to_group_fn: Optional[
-        Callable[[AirbyteConnectionMetadata], Optional[str]]
-    ] = None,
-    io_manager_key: Optional[str] = None,
-    connection_to_io_manager_key_fn: Optional[Callable[[str], Optional[str]]] = None,
-    connection_filter: Optional[Callable[[AirbyteConnectionMetadata], bool]] = None,
-    connection_directories: Optional[Sequence[str]] = None,
-    connection_to_asset_key_fn: Optional[
-        Callable[[AirbyteConnectionMetadata, str], AssetKey]
-    ] = None,
-    connection_to_freshness_policy_fn: Optional[
-        Callable[[AirbyteConnectionMetadata], Optional[FreshnessPolicy]]
-    ] = None,
-    connection_to_auto_materialize_policy_fn: Optional[
-        Callable[[AirbyteConnectionMetadata], Optional[AutoMaterializePolicy]]
-    ] = None,
-) -> CacheableAssetsDefinition:
-    """Loads an Airbyte project into a set of Dagster assets.
-
-    Point to the root folder of an Airbyte project synced using the Octavia CLI. For
-    more information, see https://airbyte.com/tutorials/version-control-airbyte-configurations.
-
-    Args:
-        project_dir (str): The path to the root of your Airbyte project, containing sources, destinations,
-            and connections folders.
-        workspace_id (Optional[str]): The ID of the Airbyte workspace to load connections from. Only
-            required if multiple workspace state YAMLfiles exist in the project.
-        key_prefix (Optional[CoercibleToAssetKeyPrefix]): A prefix for the asset keys created.
-        create_assets_for_normalization_tables (bool): If True, assets will be created for tables
-            created by Airbyte's normalization feature. If False, only the destination tables
-            will be created. Defaults to True.
-        connection_to_group_fn (Optional[Callable[[str], Optional[str]]]): Function which returns an asset
-            group name for a given Airbyte connection name. If None, no groups will be created. Defaults
-            to a basic sanitization function.
-        connection_meta_to_group_fn (Optional[Callable[[AirbyteConnectionMetadata], Optional[str]]]): Function
-            which returns an asset group name for a given Airbyte connection metadata. If None and connection_to_group_fn
-            is None, no groups will be created. Defaults to None.
-        io_manager_key (Optional[str]): The I/O manager key to use for all assets. Defaults to "io_manager".
-            Use this if all assets should be loaded from the same source, otherwise use connection_to_io_manager_key_fn.
-        connection_to_io_manager_key_fn (Optional[Callable[[str], Optional[str]]]): Function which returns an
-            I/O manager key for a given Airbyte connection name. When other ops are downstream of the loaded assets,
-            the IOManager specified determines how the inputs to those ops are loaded. Defaults to "io_manager".
-        connection_filter (Optional[Callable[[AirbyteConnectionMetadata], bool]]): Optional function which
-            takes in connection metadata and returns False if the connection should be excluded from the output assets.
-        connection_directories (Optional[List[str]]): Optional list of connection directories to load assets from.
-            If omitted, all connections in the Airbyte project are loaded. May be faster than connection_filter
-            if the project has many connections or if the connection yaml files are large.
-        connection_to_asset_key_fn (Optional[Callable[[AirbyteConnectionMetadata, str], AssetKey]]): Optional function which
-            takes in connection metadata and table name and returns an asset key for the table. If None, the default asset
-            key is based on the table name. Any asset key prefix will be applied to the output of this function.
-        connection_to_freshness_policy_fn (Optional[Callable[[AirbyteConnectionMetadata], Optional[FreshnessPolicy]]]):
-            Optional function which takes in connection metadata and returns a freshness policy for the connection's assets.
-            If None, no freshness policies will be applied to the assets.
-        connection_to_auto_materialize_policy_fn (Optional[Callable[[AirbyteConnectionMetadata], Optional[AutoMaterializePolicy]]]):
-            Optional function which takes in connection metadata and returns an auto materialization policy for the connection's assets.
-            If None, no auto materialization policies will be applied to the assets.
-
-    **Examples:**
-
-    Loading all Airbyte connections as assets:
-
-    .. code-block:: python
-
-        from dagster_airbyte import load_assets_from_airbyte_project
-
-        airbyte_assets = load_assets_from_airbyte_project(
-            project_dir="path/to/airbyte/project",
-        )
-
-    Filtering the set of loaded connections:
-
-    .. code-block:: python
-
-        from dagster_airbyte import load_assets_from_airbyte_project
-
-        airbyte_assets = load_assets_from_airbyte_project(
-            project_dir="path/to/airbyte/project",
-            connection_filter=lambda meta: "snowflake" in meta.name,
-        )
-    """
-    if isinstance(key_prefix, str):
-        key_prefix = [key_prefix]
-    key_prefix = check.list_param(key_prefix or [], "key_prefix", of_type=str)
-
-    check.invariant(
-        not io_manager_key or not connection_to_io_manager_key_fn,
-        "Cannot specify both io_manager_key and connection_to_io_manager_key_fn",
-    )
-    if not connection_to_io_manager_key_fn:
-        connection_to_io_manager_key_fn = lambda _: io_manager_key
-
-    check.invariant(
-        not connection_meta_to_group_fn
-        or not connection_to_group_fn
-        or connection_to_group_fn == _clean_name,
-        "Cannot specify both connection_meta_to_group_fn and connection_to_group_fn",
-    )
-
-    if not connection_meta_to_group_fn and connection_to_group_fn:
-        connection_meta_to_group_fn = lambda meta: connection_to_group_fn(meta.name)
-
-    return AirbyteYAMLCacheableAssetsDefinition(
-        project_dir=project_dir,
-        workspace_id=workspace_id,
-        key_prefix=key_prefix,
-        create_assets_for_normalization_tables=create_assets_for_normalization_tables,
-        connection_meta_to_group_fn=connection_meta_to_group_fn,
-        connection_to_io_manager_key_fn=connection_to_io_manager_key_fn,
-        connection_filter=connection_filter,
-        connection_directories=connection_directories,
-        connection_to_asset_key_fn=connection_to_asset_key_fn,
-        connection_to_freshness_policy_fn=connection_to_freshness_policy_fn,
-        connection_to_auto_materialize_policy_fn=connection_to_auto_materialize_policy_fn,
-    )

dagster_airbyte/version.py

@@ -1 +1 @@
-__version__ = "0.24.13"
+__version__ = "0.25.0"

{dagster_airbyte-0.24.13.dist-info → dagster_airbyte-0.25.0.dist-info}/METADATA

@@ -1,24 +1,23 @@
 Metadata-Version: 2.1
 Name: dagster-airbyte
-Version: 0.24.13
+Version: 0.25.0
 Summary: Package for integrating Airbyte with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-airbyte
 Author: Dagster Labs
 Author-email: hello@dagsterlabs.com
 License: Apache-2.0
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8,<3.13
+Requires-Python: >=3.9,<3.13
 License-File: LICENSE
-Requires-Dist: dagster ==1.8.13
+Requires-Dist: dagster ==1.9.0
 Requires-Dist: requests
 Provides-Extra: managed
-Requires-Dist: dagster-managed-elements ==0.24.13 ; extra == 'managed'
+Requires-Dist: dagster-managed-elements ==0.25.0 ; extra == 'managed'
 Provides-Extra: test
 Requires-Dist: requests-mock ; extra == 'test'
 

{dagster_airbyte-0.24.13.dist-info → dagster_airbyte-0.25.0.dist-info}/RECORD

@@ -1,21 +1,21 @@
-dagster_airbyte/__init__.py,sha256=eXDY4rzDz0PKzPAaO0zWw6uP8RQY-SgU0EXa7GMdM2k,1305
-dagster_airbyte/asset_defs.py,sha256=_jG8PpusrnkIKxow8VsQte8sEZa_IbCDTvBpQoBwuww,53307
+dagster_airbyte/__init__.py,sha256=DpJXhv73w1oKpWnXRRfItCTb61oEZR3FJzyMZzBgRfM,1231
+dagster_airbyte/asset_defs.py,sha256=UFzHdNRgFPNU8xqDGcD-ce-J9I82HIj3N2GN5tg8G_Y,46039
 dagster_airbyte/cli.py,sha256=HErteP1MjfHozKKSrznh0yAreKETbXp5NDHzXGsdvvE,425
 dagster_airbyte/ops.py,sha256=pq6mp7vN2wXgo3gJMuWaAcxTmfkZ7d1zWzPyL_auSEY,4208
 dagster_airbyte/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
 dagster_airbyte/resources.py,sha256=FoHI_Eiw2F6tnI1KZMfkgXhThdU6DplHp6INsbjUFsg,29802
 dagster_airbyte/types.py,sha256=w1DyTcXyuzrG3wfkOPYFtwj7snHcgqf-dC7_pRjiE1Q,1544
 dagster_airbyte/utils.py,sha256=cFKCkGFAvwr17KFTeqpVtQRDsNo4zpqw9yr2-1YSJeI,2823
-dagster_airbyte/version.py,sha256=pK8aUmHXhMGTRJ9g6PF60nsyauWvAxl-kjvZZup3fB8,24
+dagster_airbyte/version.py,sha256=Mu4JbSLl5nr-J2figk5hmW2mrw4skf_oeIzxbnpcgwY,23
 dagster_airbyte/managed/__init__.py,sha256=6SBtyNOMJ9Cu2UIwFExJHpL_ZVFo3rPMvyIxVOsKvWE,469
 dagster_airbyte/managed/reconciliation.py,sha256=HgrLT-Xs8vWY9SfbdBXuorMf60KCn5Qz7bPITW5MxJo,34862
 dagster_airbyte/managed/types.py,sha256=ja056Wm7_ZFw1XGSNmdxmBy2TcOxbnylJCpRA2ng2TE,14596
 dagster_airbyte/managed/generated/__init__.py,sha256=eYq-yfXEeffuKAVFXY8plD0se1wHjFNVqklpbu9gljw,108
 dagster_airbyte/managed/generated/destinations.py,sha256=x1wmWlXvOJHtfaZva3ErdKuVS--sDvfidSXR5ji9G5w,119692
 dagster_airbyte/managed/generated/sources.py,sha256=wyNoGJiNvW8mjRRs6b-_lWFs0Fgy-MZlRaxiN6bP-4s,282691
-dagster_airbyte-0.24.13.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
-dagster_airbyte-0.24.13.dist-info/METADATA,sha256=Q_SExnCtz5nDuHoEVacCzzFYilmjSeNEBz1u_KUZeY8,929
-dagster_airbyte-0.24.13.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-dagster_airbyte-0.24.13.dist-info/entry_points.txt,sha256=XrbLOz3LpgPV5fdwMmgdP6Rp1AfSG07KeWIddLqh7Lw,61
-dagster_airbyte-0.24.13.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
-dagster_airbyte-0.24.13.dist-info/RECORD,,
+dagster_airbyte-0.25.0.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
+dagster_airbyte-0.25.0.dist-info/METADATA,sha256=et8s1-ISVtc1doheso3hjYjhvfFi0jDIf0TuxSXzAGc,876
+dagster_airbyte-0.25.0.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+dagster_airbyte-0.25.0.dist-info/entry_points.txt,sha256=XrbLOz3LpgPV5fdwMmgdP6Rp1AfSG07KeWIddLqh7Lw,61
+dagster_airbyte-0.25.0.dist-info/top_level.txt,sha256=HLwIRQCzqItn88_KbPP8DNTKKQEBUVKk6NCn4PrCtqY,16
+dagster_airbyte-0.25.0.dist-info/RECORD,,