dagster-airbyte 0.25.1__py3-none-any.whl → 0.25.10__py3-none-any.whl

dagster_airbyte/managed/generated/sources.py

@@ -1,5 +1,5 @@
 # ruff: noqa: A001, A002
-from typing import List, Optional, Union
+from typing import Optional, Union
 
 import dagster._check as check
 from dagster._annotations import public
@@ -154,7 +154,7 @@ class LinkedinAdsSource(GeneratedAirbyteSource):
         name: str,
         credentials: Union["LinkedinAdsSource.OAuth20", "LinkedinAdsSource.AccessToken"],
         start_date: str,
-        account_ids: Optional[List[int]] = None,
+        account_ids: Optional[list[int]] = None,
     ):
         """Airbyte Source for Linkedin Ads.
 
@@ -826,9 +826,9 @@ class JiraSource(GeneratedAirbyteSource):
         api_token: str,
         domain: str,
         email: str,
-        projects: Optional[List[str]] = None,
+        projects: Optional[list[str]] = None,
         start_date: Optional[str] = None,
-        additional_fields: Optional[List[str]] = None,
+        additional_fields: Optional[list[str]] = None,
         expand_issue_changelog: Optional[bool] = None,
         render_fields: Optional[bool] = None,
         enable_experimental_streams: Optional[bool] = None,
@@ -1078,7 +1078,7 @@ class QualarooSource(GeneratedAirbyteSource):
         token: str,
         key: str,
         start_date: str,
-        survey_ids: Optional[List[str]] = None,
+        survey_ids: Optional[list[str]] = None,
     ):
         """Airbyte Source for Qualaroo.
 
@@ -1405,7 +1405,7 @@ class OracleSource(GeneratedAirbyteSource):
             "OracleSource.TLSEncryptedVerifyCertificate",
         ],
         password: Optional[str] = None,
-        schemas: Optional[List[str]] = None,
+        schemas: Optional[list[str]] = None,
         jdbc_url_params: Optional[str] = None,
     ):
         """Airbyte Source for Oracle.
@@ -1694,7 +1694,7 @@ class LookerSource(GeneratedAirbyteSource):
         domain: str,
         client_id: str,
         client_secret: str,
-        run_look_ids: Optional[List[str]] = None,
+        run_look_ids: Optional[list[str]] = None,
     ):
         """Airbyte Source for Looker.
 
@@ -1786,8 +1786,8 @@ class AmazonAdsSource(GeneratedAirbyteSource):
         report_wait_timeout: Optional[int] = None,
         report_generation_max_retries: Optional[int] = None,
         start_date: Optional[str] = None,
-        profiles: Optional[List[int]] = None,
-        state_filter: Optional[List[str]] = None,
+        profiles: Optional[list[int]] = None,
+        state_filter: Optional[list[str]] = None,
     ):
         """Airbyte Source for Amazon Ads.
 
@@ -2192,7 +2192,7 @@ class SearchMetricsSource(GeneratedAirbyteSource):
 class TypeformSource(GeneratedAirbyteSource):
     @public
     def __init__(
-        self, name: str, start_date: str, token: str, form_ids: Optional[List[str]] = None
+        self, name: str, start_date: str, token: str, form_ids: Optional[list[str]] = None
     ):
         """Airbyte Source for Typeform.
 
@@ -2355,10 +2355,10 @@ class AdjustSource(GeneratedAirbyteSource):
         self,
         name: str,
         api_token: str,
-        dimensions: List[str],
+        dimensions: list[str],
         ingest_start: str,
-        metrics: List[str],
-        additional_metrics: Optional[List[str]] = None,
+        metrics: list[str],
+        additional_metrics: Optional[list[str]] = None,
         until_today: Optional[bool] = None,
     ):
         """Airbyte Source for Adjust.
@@ -2448,7 +2448,7 @@ class GoogleAdsSource(GeneratedAirbyteSource):
         customer_id: str,
         start_date: str,
         end_date: Optional[str] = None,
-        custom_queries: Optional[List[CustomGAQLQueriesEntry]] = None,
+        custom_queries: Optional[list[CustomGAQLQueriesEntry]] = None,
         login_customer_id: Optional[str] = None,
         conversion_window_days: Optional[int] = None,
     ):
@@ -2590,7 +2590,7 @@ class SalesforceSource(GeneratedAirbyteSource):
         is_sandbox: Optional[bool] = None,
         auth_type: Optional[str] = None,
         start_date: Optional[str] = None,
-        streams_criteria: Optional[List[FilterSalesforceObjectsEntry]] = None,
+        streams_criteria: Optional[list[FilterSalesforceObjectsEntry]] = None,
     ):
         """Airbyte Source for Salesforce.
 
@@ -2836,8 +2836,8 @@ class OrbSource(GeneratedAirbyteSource):
         api_key: str,
         start_date: Optional[str] = None,
         lookback_window_days: Optional[int] = None,
-        string_event_properties_keys: Optional[List[str]] = None,
-        numeric_event_properties_keys: Optional[List[str]] = None,
+        string_event_properties_keys: Optional[list[str]] = None,
+        numeric_event_properties_keys: Optional[list[str]] = None,
     ):
         """Airbyte Source for Orb.
 
@@ -3230,7 +3230,7 @@ class SlackSource(GeneratedAirbyteSource):
         credentials: Union[
             "SlackSource.DefaultOAuth20Authorization", "SlackSource.APITokenCredentials"
         ],
-        channel_filter: Optional[List[str]] = None,
+        channel_filter: Optional[list[str]] = None,
     ):
         """Airbyte Source for Slack.
 
@@ -3575,7 +3575,7 @@ class PostgresSource(GeneratedAirbyteSource):
             "PostgresSource.SSHKeyAuthentication",
             "PostgresSource.PasswordAuthentication",
         ],
-        schemas: Optional[List[str]] = None,
+        schemas: Optional[list[str]] = None,
         password: Optional[str] = None,
         jdbc_url_params: Optional[str] = None,
         ssl: Optional[bool] = None,
@@ -3643,7 +3643,7 @@ class TrelloSource(GeneratedAirbyteSource):
         token: str,
         key: str,
         start_date: str,
-        board_ids: Optional[List[str]] = None,
+        board_ids: Optional[list[str]] = None,
     ):
         """Airbyte Source for Trello.
 
@@ -3741,7 +3741,7 @@ class S3Source(GeneratedAirbyteSource):
         def __init__(
             self,
             filetype: Optional[str] = None,
-            columns: Optional[List[str]] = None,
+            columns: Optional[list[str]] = None,
             batch_size: Optional[int] = None,
             buffer_size: Optional[int] = None,
         ):
@@ -4005,7 +4005,7 @@ class MssqlSource(GeneratedAirbyteSource):
             "MssqlSource.EncryptedVerifyCertificate",
         ],
         replication_method: Union["MssqlSource.Standard", "MssqlSource.LogicalReplicationCDC"],
-        schemas: Optional[List[str]] = None,
+        schemas: Optional[list[str]] = None,
         password: Optional[str] = None,
         jdbc_url_params: Optional[str] = None,
     ):
@@ -4096,7 +4096,7 @@ class RedshiftSource(GeneratedAirbyteSource):
         database: str,
         username: str,
         password: str,
-        schemas: Optional[List[str]] = None,
+        schemas: Optional[list[str]] = None,
         jdbc_url_params: Optional[str] = None,
     ):
         """Airbyte Source for Redshift.
@@ -4228,7 +4228,7 @@ class SentrySource(GeneratedAirbyteSource):
         organization: str,
         project: str,
         hostname: Optional[str] = None,
-        discover_fields: Optional[List[str]] = None,
+        discover_fields: Optional[list[str]] = None,
     ):
         """Airbyte Source for Sentry.
 
@@ -4317,7 +4317,7 @@ class PythonHttpTutorialSource(GeneratedAirbyteSource):
 
 class AirtableSource(GeneratedAirbyteSource):
     @public
-    def __init__(self, name: str, api_key: str, base_id: str, tables: List[str]):
+    def __init__(self, name: str, api_key: str, base_id: str, tables: list[str]):
         """Airbyte Source for Airtable.
 
         Documentation can be found at https://docs.airbyte.com/integrations/sources/airtable
@@ -5123,7 +5123,7 @@ class NetsuiteSource(GeneratedAirbyteSource):
         token_key: str,
         token_secret: str,
         start_datetime: str,
-        object_types: Optional[List[str]] = None,
+        object_types: Optional[list[str]] = None,
         window_in_days: Optional[int] = None,
     ):
         """Airbyte Source for Netsuite.
@@ -5190,7 +5190,7 @@ class Dv360Source(GeneratedAirbyteSource):
         partner_id: int,
         start_date: str,
         end_date: Optional[str] = None,
-        filters: Optional[List[str]] = None,
+        filters: Optional[list[str]] = None,
     ):
         """Airbyte Source for Dv 360.
 
@@ -5775,7 +5775,7 @@ class GoogleSearchConsoleSource(GeneratedAirbyteSource):
     def __init__(
         self,
         name: str,
-        site_urls: List[str],
+        site_urls: list[str],
         start_date: str,
         authorization: Union[
             "GoogleSearchConsoleSource.OAuth",
@@ -5816,9 +5816,9 @@ class FacebookMarketingSource(GeneratedAirbyteSource):
         def __init__(
             self,
             name: str,
-            fields: Optional[List[str]] = None,
-            breakdowns: Optional[List[str]] = None,
-            action_breakdowns: Optional[List[str]] = None,
+            fields: Optional[list[str]] = None,
+            breakdowns: Optional[list[str]] = None,
+            action_breakdowns: Optional[list[str]] = None,
             time_increment: Optional[int] = None,
             start_date: Optional[str] = None,
             end_date: Optional[str] = None,
@@ -5847,7 +5847,7 @@ class FacebookMarketingSource(GeneratedAirbyteSource):
         end_date: Optional[str] = None,
         include_deleted: Optional[bool] = None,
         fetch_thumbnail_images: Optional[bool] = None,
-        custom_insights: Optional[List[InsightConfig]] = None,
+        custom_insights: Optional[list[InsightConfig]] = None,
         page_size: Optional[int] = None,
         insights_lookback_window: Optional[int] = None,
         max_batch_size: Optional[int] = None,
@@ -5891,7 +5891,7 @@ class FacebookMarketingSource(GeneratedAirbyteSource):
 class SurveymonkeySource(GeneratedAirbyteSource):
     @public
     def __init__(
-        self, name: str, access_token: str, start_date: str, survey_ids: Optional[List[str]] = None
+        self, name: str, access_token: str, start_date: str, survey_ids: Optional[list[str]] = None
     ):
         """Airbyte Source for Surveymonkey.
 

dagster_airbyte/managed/reconciliation.py

@@ -1,16 +1,5 @@
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    Iterable,
-    List,
-    Mapping,
-    Optional,
-    Sequence,
-    Tuple,
-    Union,
-    cast,
-)
+from collections.abc import Iterable, Mapping, Sequence
+from typing import Any, Callable, Optional, Union, cast
 
 import dagster._check as check
 from dagster import AssetKey
@@ -36,7 +25,6 @@ from dagster_managed_elements.utils import UNSET, diff_dicts
 from dagster_airbyte.asset_defs import (
     AirbyteConnectionMetadata,
     AirbyteInstanceCacheableAssetsDefinition,
-    _clean_name,
 )
 from dagster_airbyte.managed.types import (
     MANAGED_ELEMENTS_DEPRECATION_MSG,
@@ -50,7 +38,7 @@ from dagster_airbyte.managed.types import (
     InitializedAirbyteSource,
 )
 from dagster_airbyte.resources import AirbyteResource
-from dagster_airbyte.utils import is_basic_normalization_operation
+from dagster_airbyte.utils import clean_name, is_basic_normalization_operation
 
 
 def gen_configured_stream_json(
@@ -170,13 +158,13 @@ def reconcile_sources(
     dry_run: bool,
     should_delete: bool,
     ignore_secrets: bool,
-) -> Tuple[Mapping[str, InitializedAirbyteSource], ManagedElementCheckResult]:
+) -> tuple[Mapping[str, InitializedAirbyteSource], ManagedElementCheckResult]:
     """Generates a diff of the configured and existing sources and reconciles them to match the
     configured state if dry_run is False.
     """
     diff = ManagedElementDiff()
 
-    initialized_sources: Dict[str, InitializedAirbyteSource] = {}
+    initialized_sources: dict[str, InitializedAirbyteSource] = {}
     for source_name in set(config_sources.keys()).union(existing_sources.keys()):
         configured_source = config_sources.get(source_name)
         existing_source = existing_sources.get(source_name)
@@ -224,7 +212,7 @@ def reconcile_sources(
             else:
                 if not dry_run:
                     create_result = cast(
-                        Dict[str, str],
+                        dict[str, str],
                         check.not_none(
                             res.make_request(
                                 endpoint="/sources/create",
@@ -257,13 +245,13 @@ def reconcile_destinations(
     dry_run: bool,
     should_delete: bool,
     ignore_secrets: bool,
-) -> Tuple[Mapping[str, InitializedAirbyteDestination], ManagedElementCheckResult]:
+) -> tuple[Mapping[str, InitializedAirbyteDestination], ManagedElementCheckResult]:
     """Generates a diff of the configured and existing destinations and reconciles them to match the
     configured state if dry_run is False.
     """
     diff = ManagedElementDiff()
 
-    initialized_destinations: Dict[str, InitializedAirbyteDestination] = {}
+    initialized_destinations: dict[str, InitializedAirbyteDestination] = {}
     for destination_name in set(config_destinations.keys()).union(existing_destinations.keys()):
         configured_destination = config_destinations.get(destination_name)
         existing_destination = existing_destinations.get(destination_name)
@@ -312,7 +300,7 @@ def reconcile_destinations(
             else:
                 if not dry_run:
                     create_result = cast(
-                        Dict[str, str],
+                        dict[str, str],
                         check.not_none(
                             res.make_request(
                                 endpoint="/destinations/create",
@@ -358,23 +346,23 @@ def reconcile_config(
         workspace_id = res.get_default_workspace()
 
         existing_sources_raw = cast(
-            Dict[str, List[Dict[str, Any]]],
+            dict[str, list[dict[str, Any]]],
             check.not_none(
                 res.make_request(endpoint="/sources/list", data={"workspaceId": workspace_id})
             ),
         )
         existing_dests_raw = cast(
-            Dict[str, List[Dict[str, Any]]],
+            dict[str, list[dict[str, Any]]],
             check.not_none(
                 res.make_request(endpoint="/destinations/list", data={"workspaceId": workspace_id})
             ),
         )
 
-        existing_sources: Dict[str, InitializedAirbyteSource] = {
+        existing_sources: dict[str, InitializedAirbyteSource] = {
             source_json["name"]: InitializedAirbyteSource.from_api_json(source_json)
             for source_json in existing_sources_raw.get("sources", [])
         }
-        existing_dests: Dict[str, InitializedAirbyteDestination] = {
+        existing_dests: dict[str, InitializedAirbyteDestination] = {
             destination_json["name"]: InitializedAirbyteDestination.from_api_json(destination_json)
             for destination_json in existing_dests_raw.get("destinations", [])
         }
@@ -434,7 +422,7 @@ def reconcile_normalization(
     existing_basic_norm_op_id = None
     if existing_connection_id:
         operations = cast(
-            Dict[str, List[Dict[str, str]]],
+            dict[str, list[dict[str, str]]],
             check.not_none(
                 res.make_request(
                     endpoint="/operations/list",
@@ -462,7 +450,7 @@ def reconcile_normalization(
                 return existing_basic_norm_op_id
             else:
                 return cast(
-                    Dict[str, str],
+                    dict[str, str],
                     check.not_none(
                         res.make_request(
                             endpoint="/operations/create",
@@ -504,12 +492,12 @@ def reconcile_connections_pre(
     diff = ManagedElementDiff()
 
     existing_connections_raw = cast(
-        Dict[str, List[Dict[str, Any]]],
+        dict[str, list[dict[str, Any]]],
         check.not_none(
             res.make_request(endpoint="/connections/list", data={"workspaceId": workspace_id})
         ),
     )
-    existing_connections: Dict[str, InitializedAirbyteConnection] = {
+    existing_connections: dict[str, InitializedAirbyteConnection] = {
         connection_json["name"]: InitializedAirbyteConnection.from_api_json(
             connection_json, existing_sources, existing_destinations
         )
@@ -549,7 +537,7 @@ def reconcile_connections_post(
 ) -> None:
     """Creates new and modifies existing connections based on the config if dry_run is False."""
     existing_connections_raw = cast(
-        Dict[str, List[Dict[str, Any]]],
+        dict[str, list[dict[str, Any]]],
         check.not_none(
             res.make_request(endpoint="/connections/list", data={"workspaceId": workspace_id})
         ),
@@ -725,9 +713,9 @@ class AirbyteManagedElementCacheableAssetsDefinition(AirbyteInstanceCacheableAss
             connection_to_asset_key_fn=connection_to_asset_key_fn,
             connection_to_freshness_policy_fn=connection_to_freshness_policy_fn,
         )
-        self._connections: List[AirbyteConnection] = list(connections)
+        self._connections: list[AirbyteConnection] = list(connections)
 
-    def _get_connections(self) -> Sequence[Tuple[str, AirbyteConnectionMetadata]]:
+    def _get_connections(self) -> Sequence[tuple[str, AirbyteConnectionMetadata]]:
         diff = reconcile_config(self._airbyte_instance, self._connections, dry_run=True)
         if isinstance(diff, ManagedElementDiff) and not diff.is_empty():
             raise ValueError(
@@ -746,7 +734,7 @@ def load_assets_from_connections(
     connections: Iterable[AirbyteConnection],
     key_prefix: Optional[CoercibleToAssetKeyPrefix] = None,
     create_assets_for_normalization_tables: bool = True,
-    connection_to_group_fn: Optional[Callable[[str], Optional[str]]] = _clean_name,
+    connection_to_group_fn: Optional[Callable[[str], Optional[str]]] = clean_name,
     connection_meta_to_group_fn: Optional[
         Callable[[AirbyteConnectionMetadata], Optional[str]]
     ] = None,
@@ -821,7 +809,7 @@ def load_assets_from_connections(
     check.invariant(
         not connection_meta_to_group_fn
         or not connection_to_group_fn
-        or connection_to_group_fn == _clean_name,
+        or connection_to_group_fn == clean_name,
         "Cannot specify both connection_meta_to_group_fn and connection_to_group_fn",
     )
 

dagster_airbyte/managed/types.py

@@ -1,7 +1,8 @@
 import json
 from abc import ABC
+from collections.abc import Mapping
 from enum import Enum
-from typing import Any, Dict, List, Mapping, Optional, Union
+from typing import Any, Optional, Union
 
 import dagster._check as check
 from dagster._annotations import deprecated, public
@@ -24,14 +25,14 @@ class AirbyteSyncMode(ABC):
     def __eq__(self, other: Any) -> bool:
         return isinstance(other, AirbyteSyncMode) and self.to_json() == other.to_json()
 
-    def __init__(self, json_repr: Dict[str, Any]):
+    def __init__(self, json_repr: dict[str, Any]):
         self.json_repr = json_repr
 
-    def to_json(self) -> Dict[str, Any]:
+    def to_json(self) -> dict[str, Any]:
         return self.json_repr
 
     @classmethod
-    def from_json(cls, json_repr: Dict[str, Any]) -> "AirbyteSyncMode":
+    def from_json(cls, json_repr: dict[str, Any]) -> "AirbyteSyncMode":
         return cls(
             {
                 k: v
@@ -86,7 +87,7 @@ class AirbyteSyncMode(ABC):
     def incremental_append_dedup(
         cls,
         cursor_field: Optional[str] = None,
-        primary_key: Optional[Union[str, List[str]]] = None,
+        primary_key: Optional[Union[str, list[str]]] = None,
     ) -> "AirbyteSyncMode":
         """Syncs new records from the source, appending to an append-only history
         table in the destination. Also generates a deduplicated view mirroring the
@@ -349,11 +350,11 @@ class InitializedAirbyteConnection:
         )
 
 
-def _remove_none_values(obj: Dict[str, Any]) -> Dict[str, Any]:
+def _remove_none_values(obj: dict[str, Any]) -> dict[str, Any]:
     return {k: v for k, v in obj.items() if v is not None}
 
 
-def _dump_class(obj: Any) -> Dict[str, Any]:
+def _dump_class(obj: Any) -> dict[str, Any]:
     return json.loads(json.dumps(obj, default=lambda o: _remove_none_values(o.__dict__)))
 
 

dagster_airbyte/ops.py

@@ -1,4 +1,5 @@
-from typing import Any, Iterable, List, Optional
+from collections.abc import Iterable
+from typing import Any, Optional
 
 from dagster import Config, In, Nothing, Out, Output, op
 from dagster._core.storage.tags import COMPUTE_KIND_TAG
@@ -40,7 +41,7 @@ class AirbyteSyncConfig(Config):
             "be yielded when the op executes."
         ),
     )
-    asset_key_prefix: List[str] = Field(
+    asset_key_prefix: list[str] = Field(
         ["airbyte"],
         description=(
             "If provided and yield_materializations is True, these components will be used to "