dagstermill 0.23.3__py3-none-any.whl → 0.27.6__py3-none-any.whl

dagstermill/__init__.py

@@ -1,15 +1,16 @@
-from dagster._core.libraries import DagsterLibraryRegistry
-
-from .asset_factory import define_dagstermill_asset as define_dagstermill_asset
-from .context import DagstermillExecutionContext as DagstermillExecutionContext
-from .errors import DagstermillError as DagstermillError
-from .factory import define_dagstermill_op as define_dagstermill_op
-from .io_managers import (
+from dagster_shared.libraries import DagsterLibraryRegistry
+
+import dagstermill.factory as factory  # noqa: F401
+from dagstermill.asset_factory import define_dagstermill_asset as define_dagstermill_asset
+from dagstermill.context import DagstermillExecutionContext as DagstermillExecutionContext
+from dagstermill.errors import DagstermillError as DagstermillError
+from dagstermill.factory import define_dagstermill_op as define_dagstermill_op
+from dagstermill.io_managers import (
     ConfigurableLocalOutputNotebookIOManager as ConfigurableLocalOutputNotebookIOManager,
     local_output_notebook_io_manager as local_output_notebook_io_manager,
 )
-from .manager import MANAGER_FOR_NOTEBOOK_INSTANCE as _MANAGER_FOR_NOTEBOOK_INSTANCE
-from .version import __version__ as __version__
+from dagstermill.manager import MANAGER_FOR_NOTEBOOK_INSTANCE as _MANAGER_FOR_NOTEBOOK_INSTANCE
+from dagstermill.version import __version__ as __version__
 
 DagsterLibraryRegistry.register("dagstermill", __version__)
 

dagstermill/__main__.py

@@ -1,3 +1,3 @@
-from .cli import main
+from dagstermill.cli import main
 
 main()

dagstermill/asset_factory.py

@@ -1,6 +1,7 @@
 import pickle
 import tempfile
-from typing import Any, Callable, Iterable, Mapping, Optional, Set, Type, Union, cast
+from collections.abc import Iterable, Mapping
+from typing import Any, Callable, Optional, Union, cast
 
 import dagster._check as check
 from dagster import (
@@ -16,11 +17,13 @@ from dagster import (
     SourceAsset,
     asset,
 )
+from dagster._annotations import beta, beta_param
 from dagster._config.pythonic_config import Config, infer_schema_from_config_class
 from dagster._config.pythonic_config.type_check_utils import safe_is_subclass
 from dagster._core.definitions.events import CoercibleToAssetKey, CoercibleToAssetKeyPrefix
-from dagster._core.definitions.utils import normalize_tags
 from dagster._core.execution.context.compute import OpExecutionContext
+from dagster._core.storage.tags import COMPUTE_KIND_TAG
+from dagster._utils.tags import normalize_tags
 
 from dagstermill.factory import _clean_path_for_windows, execute_notebook
 
@@ -67,6 +70,8 @@ def _make_dagstermill_asset_compute_fn(
     return _t_fn
 
 
+@beta
+@beta_param(param="resource_defs")
 def define_dagstermill_asset(
     name: str,
     notebook_path: str,
@@ -75,7 +80,7 @@ def define_dagstermill_asset(
     deps: Optional[Iterable[Union[CoercibleToAssetKey, AssetsDefinition, SourceAsset]]] = None,
     metadata: Optional[Mapping[str, Any]] = None,
     config_schema: Optional[Union[Any, Mapping[str, Any]]] = None,
-    required_resource_keys: Optional[Set[str]] = None,
+    required_resource_keys: Optional[set[str]] = None,
     resource_defs: Optional[Mapping[str, ResourceDefinition]] = None,
     description: Optional[str] = None,
     partitions_def: Optional[PartitionsDefinition] = None,
@@ -84,7 +89,8 @@ def define_dagstermill_asset(
     io_manager_key: Optional[str] = None,
     retry_policy: Optional[RetryPolicy] = None,
     save_notebook_on_failure: bool = False,
-    non_argument_deps: Optional[Union[Set[AssetKey], Set[str]]] = None,
+    non_argument_deps: Optional[Union[set[AssetKey], set[str]]] = None,
+    asset_tags: Optional[Mapping[str, Any]] = None,
 ) -> AssetsDefinition:
     """Creates a Dagster asset for a Jupyter notebook.
 
@@ -114,7 +120,7 @@ def define_dagstermill_asset(
         group_name (Optional[str]): A string name used to organize multiple assets into groups. If not provided,
             the name "default" is used.
         resource_defs (Optional[Mapping[str, ResourceDefinition]]):
-            (Experimental) A mapping of resource keys to resource definitions. These resources
+            (Beta) A mapping of resource keys to resource definitions. These resources
             will be initialized during execution, and can be accessed from the
             context within the notebook.
         io_manager_key (Optional[str]): A string key for the IO manager used to store the output notebook.
@@ -123,6 +129,7 @@ def define_dagstermill_asset(
         save_notebook_on_failure (bool): If True and the notebook fails during execution, the failed notebook will be
             written to the Dagster storage directory. The location of the file will be printed in the Dagster logs.
             Defaults to False.
+        asset_tags (Optional[Dict[str, Any]]): A dictionary of tags to apply to the asset.
         non_argument_deps (Optional[Union[Set[AssetKey], Set[str]]]): Deprecated, use deps instead. Set of asset keys that are
             upstream dependencies, but do not pass an input to the asset.
 
@@ -172,7 +179,7 @@ def define_dagstermill_asset(
         io_manager_key, "io_manager_key", default="output_notebook_io_manager"
     )
 
-    user_tags = normalize_tags(op_tags).tags
+    user_tags = normalize_tags(op_tags)
     if op_tags is not None:
         check.invariant(
             "notebook_path" not in op_tags,
@@ -180,15 +187,18 @@ def define_dagstermill_asset(
             " is reserved for use by Dagster",
         )
         check.invariant(
-            "kind" not in op_tags,
-            "user-defined op tags contains the `kind` key, but the `kind` key is reserved for"
+            COMPUTE_KIND_TAG not in op_tags,
+            f"user-defined op tags contains the `{COMPUTE_KIND_TAG}` key, but the `{COMPUTE_KIND_TAG}` key is reserved for"
             " use by Dagster",
         )
 
-    default_tags = {"notebook_path": _clean_path_for_windows(notebook_path), "kind": "ipynb"}
+    default_tags = {
+        "notebook_path": _clean_path_for_windows(notebook_path),
+        COMPUTE_KIND_TAG: "ipynb",
+    }
 
     if safe_is_subclass(config_schema, Config):
-        config_schema = infer_schema_from_config_class(cast(Type[Config], config_schema))
+        config_schema = infer_schema_from_config_class(cast("type[Config]", config_schema))
 
     return asset(
         name=name,
@@ -207,6 +217,7 @@ def define_dagstermill_asset(
         io_manager_key=io_mgr_key,
         retry_policy=retry_policy,
         non_argument_deps=non_argument_deps,
+        tags=asset_tags,
     )(
         _make_dagstermill_asset_compute_fn(
             name=name,

dagstermill/cli.py

@@ -1,13 +1,14 @@
 import copy
 import os
 import subprocess
-from typing import Mapping, Optional
+from collections.abc import Mapping
+from typing import Optional
 
 import click
 import dagster._check as check
 import nbformat
-from dagster._seven.json import loads
 from dagster._utils import mkdir_p, safe_isfile
+from dagster_shared.seven.json import loads
 from papermill.iorw import load_notebook_node, write_ipynb
 
 

dagstermill/context.py

@@ -1,4 +1,5 @@
-from typing import AbstractSet, Any, Mapping, Optional, cast
+from collections.abc import Mapping
+from typing import AbstractSet, Any, Optional, cast  # noqa: UP035
 
 from dagster import (
     DagsterRun,
@@ -6,17 +7,18 @@ from dagster import (
     OpDefinition,
     _check as check,
 )
-from dagster._annotations import public
+from dagster._annotations import beta, public
 from dagster._core.definitions.dependency import Node, NodeHandle
 from dagster._core.definitions.repository_definition.repository_definition import (
     RepositoryDefinition,
 )
-from dagster._core.execution.context.compute import AbstractComputeExecutionContext
+from dagster._core.execution.context.op_execution_context import AbstractComputeExecutionContext
 from dagster._core.execution.context.system import PlanExecutionContext, StepExecutionContext
 from dagster._core.log_manager import DagsterLogManager
 from dagster._core.system_config.objects import ResolvedRunConfig
 
 
+@beta
 class DagstermillExecutionContext(AbstractComputeExecutionContext):
     """Dagstermill-specific execution context.
 
@@ -121,7 +123,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
     @property
     def run(self) -> DagsterRun:
         """:class:`dagster.DagsterRun`: The job run for the context."""
-        return cast(DagsterRun, self._job_context.dagster_run)
+        return cast("DagsterRun", self._job_context.dagster_run)
 
     @property
     def log(self) -> DagsterLogManager:
@@ -139,7 +141,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
         In interactive contexts, this may be a dagstermill-specific shim, depending whether an
         op definition was passed to ``dagstermill.get_context``.
         """
-        return cast(OpDefinition, self._job_def.node_def_named(self.op_name))
+        return cast("OpDefinition", self._job_def.node_def_named(self.op_name))
 
     @property
     def node(self) -> Node:

dagstermill/engine.py

@@ -1,7 +1,7 @@
 import nbformat
 from papermill.log import logger
 
-from .compat import ExecutionError, is_papermill_2
+from dagstermill.compat import ExecutionError, is_papermill_2
 
 if is_papermill_2():
     from papermill.clientwrap import PapermillNotebookClient
@@ -86,9 +86,7 @@ else:
         # the kernel down. Note that atexit doesn't seem to work at all in ipython, and hooking into
         # the ipython post_execute event doesn't work in papermill.
         def papermill_process(self, nb_man, resources):
-            _, resources = super(DagstermillExecutePreprocessor, self).papermill_process(
-                nb_man, resources
-            )
+            _, resources = super().papermill_process(nb_man, resources)
 
             new_cell = nbformat.v4.new_code_cell(
                 source="import dagstermill as __dm_dagstermill\n__dm_dagstermill._teardown()\n"

dagstermill/errors.py

@@ -1,4 +1,4 @@
-from dagster._core.errors import DagsterError
+from dagster_shared.error import DagsterError
 
 
 class DagstermillError(DagsterError):

dagstermill/examples/__init__.py

@@ -1 +1 @@
-from .repository import notebook_repo as notebook_repo
+from dagstermill.examples.repository import notebook_repo as notebook_repo

dagstermill/examples/repository.py

@@ -23,7 +23,7 @@ from dagster import (
     with_resources,
 )
 from dagster._config.pythonic_config import Config
-from dagster._core.definitions.asset_graph import AssetGraph
+from dagster._core.definitions.assets.graph.asset_graph import AssetGraph
 from dagster._core.definitions.utils import DEFAULT_OUTPUT
 from dagster._utils import PICKLE_PROTOCOL, file_relative_path
 
@@ -303,12 +303,12 @@ def error_job():
 
 if DAGSTER_PANDAS_PRESENT and SKLEARN_PRESENT and MATPLOTLIB_PRESENT:
     # We need type-ignores here because type checkers don't understand the `*_PRESENT` kwargs.
-    clean_data = test_nb_op("clean_data", outs={DEFAULT_OUTPUT: Out(DataFrame)})
+    clean_data = test_nb_op("clean_data", outs={DEFAULT_OUTPUT: Out(DataFrame)})  # pyright: ignore[reportPossiblyUnboundVariable]
 
     # FIXME add an output to this
-    tutorial_LR = test_nb_op("tutorial_LR", ins={"df": In(DataFrame)})
+    tutorial_LR = test_nb_op("tutorial_LR", ins={"df": In(DataFrame)})  # pyright: ignore[reportPossiblyUnboundVariable]
 
-    tutorial_RF = test_nb_op("tutorial_RF", ins={"df": In(DataFrame)})
+    tutorial_RF = test_nb_op("tutorial_RF", ins={"df": In(DataFrame)})  # pyright: ignore[reportPossiblyUnboundVariable]
 
     @job(resource_defs=common_resource_defs)
     def tutorial_job():
@@ -570,6 +570,7 @@ custom_io_mgr_key_asset = dagstermill.define_dagstermill_asset(
     name="custom_io_mgr_key",
     notebook_path=nb_test_path("hello_world"),
     io_manager_key="my_custom_io_manager",
+    asset_tags={"foo": "bar"},
 )
 
 yield_event_asset = dagstermill.define_dagstermill_asset(

dagstermill/factory.py

@@ -4,7 +4,8 @@ import pickle
 import sys
 import tempfile
 import uuid
-from typing import Any, Callable, Iterable, Mapping, Optional, Sequence, Set, Type, Union, cast
+from collections.abc import Iterable, Mapping, Sequence
+from typing import Any, Callable, Optional, Union, cast
 
 import nbformat
 import papermill
@@ -14,29 +15,31 @@ from dagster import (
     Out,
     Output,
     _check as check,
-    _seven,
 )
+from dagster._annotations import beta
 from dagster._config.pythonic_config import Config, infer_schema_from_config_class
 from dagster._config.pythonic_config.type_check_utils import safe_is_subclass
 from dagster._core.definitions.events import AssetMaterialization, Failure, RetryRequested
 from dagster._core.definitions.metadata import MetadataValue
 from dagster._core.definitions.reconstruct import ReconstructableJob
-from dagster._core.definitions.utils import normalize_tags
 from dagster._core.execution.context.compute import OpExecutionContext
 from dagster._core.execution.context.input import build_input_context
 from dagster._core.execution.context.system import StepExecutionContext
 from dagster._core.execution.plan.outputs import StepOutputHandle
+from dagster._core.storage.tags import COMPUTE_KIND_TAG
 from dagster._serdes import pack_value
-from dagster._seven import get_system_temp_directory
 from dagster._utils import mkdir_p, safe_tempfile_path
 from dagster._utils.error import serializable_error_info_from_exc_info
+from dagster._utils.tags import normalize_tags
+from dagster_shared import seven
+from dagster_shared.seven import get_system_temp_directory
 from papermill.engines import papermill_engines
 from papermill.iorw import load_notebook_node, write_ipynb
 
-from .compat import ExecutionError
-from .engine import DagstermillEngine
-from .errors import DagstermillError
-from .translator import DagsterTranslator
+from dagstermill.compat import ExecutionError
+from dagstermill.engine import DagstermillEngine
+from dagstermill.errors import DagstermillError
+from dagstermill.translator import DagsterTranslator
 
 
 def _clean_path_for_windows(notebook_path: str) -> str:
@@ -60,6 +63,7 @@ def _find_first_tagged_cell_index(nb, tag):
 # This is based on papermill.parameterize.parameterize_notebook
 # Typically, papermill injects the injected-parameters cell *below* the parameters cell
 # but we want to *replace* the parameters cell, which is what this function does.
+@beta
 def replace_parameters(context, nb, parameters):
     """Assigned parameters into the appropriate place in the input notebook.
 
@@ -101,11 +105,12 @@ def replace_parameters(context, nb, parameters):
         after = nb.cells
 
     nb.cells = before + [newcell] + after
-    nb.metadata.papermill["parameters"] = _seven.json.dumps(parameters)
+    nb.metadata.papermill["parameters"] = seven.json.dumps(parameters)
 
     return nb
 
 
+@beta
 def get_papermill_parameters(
     step_context: StepExecutionContext,
     inputs: Mapping[str, object],
@@ -159,6 +164,7 @@ def get_papermill_parameters(
     return parameters
 
 
+@beta
 def execute_notebook(
     step_context: StepExecutionContext,
     name: str,
@@ -340,13 +346,14 @@ def _make_dagstermill_compute_fn(
     return _t_fn
 
 
+@beta
 def define_dagstermill_op(
     name: str,
     notebook_path: str,
     ins: Optional[Mapping[str, In]] = None,
     outs: Optional[Mapping[str, Out]] = None,
     config_schema: Optional[Union[Any, Mapping[str, Any]]] = None,
-    required_resource_keys: Optional[Set[str]] = None,
+    required_resource_keys: Optional[set[str]] = None,
     output_notebook_name: Optional[str] = None,
     asset_key_prefix: Optional[Union[Sequence[str], str]] = None,
     description: Optional[str] = None,
@@ -401,7 +408,7 @@ def define_dagstermill_op(
         required_resource_keys.add(io_mgr_key)
         outs = {
             **outs,
-            cast(str, output_notebook_name): Out(io_manager_key=io_mgr_key),
+            cast("str", output_notebook_name): Out(io_manager_key=io_mgr_key),
         }
 
     if isinstance(asset_key_prefix, str):
@@ -412,7 +419,7 @@ def define_dagstermill_op(
     default_description = f"This op is backed by the notebook at {notebook_path}"
     description = check.opt_str_param(description, "description", default=default_description)
 
-    user_tags = normalize_tags(tags).tags
+    user_tags = normalize_tags(tags)
     if tags is not None:
         check.invariant(
             "notebook_path" not in tags,
@@ -420,14 +427,17 @@ def define_dagstermill_op(
             " is reserved for use by Dagster",
         )
         check.invariant(
-            "kind" not in tags,
+            COMPUTE_KIND_TAG not in tags,
             "user-defined op tags contains the `kind` key, but the `kind` key is reserved for"
             " use by Dagster",
         )
-    default_tags = {"notebook_path": _clean_path_for_windows(notebook_path), "kind": "ipynb"}
+    default_tags = {
+        "notebook_path": _clean_path_for_windows(notebook_path),
+        COMPUTE_KIND_TAG: "ipynb",
+    }
 
     if safe_is_subclass(config_schema, Config):
-        config_schema = infer_schema_from_config_class(cast(Type[Config], config_schema))
+        config_schema = infer_schema_from_config_class(cast("type[Config]", config_schema))
 
     return OpDefinition(
         name=name,

dagstermill/io_managers.py

@@ -1,6 +1,7 @@
 import os
+from collections.abc import Sequence
 from pathlib import Path
-from typing import Any, List, Optional, Sequence
+from typing import Any, Optional
 
 import dagster._check as check
 from dagster import (
@@ -10,6 +11,7 @@ from dagster import (
     InitResourceContext,
     IOManager,
 )
+from dagster._annotations import beta
 from dagster._core.definitions.metadata import MetadataValue
 from dagster._core.execution.context.input import InputContext
 from dagster._core.execution.context.output import OutputContext
@@ -33,7 +35,7 @@ class OutputNotebookIOManager(IOManager):
 
 class LocalOutputNotebookIOManager(OutputNotebookIOManager):
     def __init__(self, base_dir: str, asset_key_prefix: Optional[Sequence[str]] = None):
-        super(LocalOutputNotebookIOManager, self).__init__(asset_key_prefix=asset_key_prefix)
+        super().__init__(asset_key_prefix=asset_key_prefix)
         self.base_dir = base_dir
         self.write_mode = "wb"
         self.read_mode = "rb"
@@ -82,6 +84,7 @@ class LocalOutputNotebookIOManager(OutputNotebookIOManager):
             return file_obj.read()
 
 
+@beta
 class ConfigurableLocalOutputNotebookIOManager(ConfigurableIOManagerFactory):
     """Built-in IO Manager for handling output notebook."""
 
@@ -92,7 +95,7 @@ class ConfigurableLocalOutputNotebookIOManager(ConfigurableIOManagerFactory):
             " directory if not provided."
         ),
     )
-    asset_key_prefix: List[str] = Field(
+    asset_key_prefix: list[str] = Field(
         default=[],
         description=(
             "Asset key prefix to apply to assets materialized for output notebooks. Defaults to no"
@@ -111,6 +114,7 @@ class ConfigurableLocalOutputNotebookIOManager(ConfigurableIOManagerFactory):
         )
 
 
+@beta
 @dagster_maintained_io_manager
 @io_manager(config_schema=ConfigurableLocalOutputNotebookIOManager.to_config_schema())
 def local_output_notebook_io_manager(init_context) -> LocalOutputNotebookIOManager:

dagstermill/manager.py

@@ -1,7 +1,9 @@
 import os
 import pickle
 import uuid
-from typing import TYPE_CHECKING, AbstractSet, Any, Mapping, Optional, cast
+from asyncio import AbstractEventLoop
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, AbstractSet, Any, Optional, cast  # noqa: UP035
 
 from dagster import (
     AssetMaterialization,
@@ -14,6 +16,7 @@ from dagster import (
     TypeCheck,
     _check as check,
 )
+from dagster._annotations import beta
 from dagster._core.definitions.dependency import NodeHandle
 from dagster._core.definitions.events import RetryRequested
 from dagster._core.definitions.graph_definition import GraphDefinition
@@ -27,7 +30,6 @@ from dagster._core.execution.api import create_execution_plan, scoped_job_contex
 from dagster._core.execution.plan.outputs import StepOutputHandle
 from dagster._core.execution.plan.plan import ExecutionPlan
 from dagster._core.execution.plan.state import KnownExecutionState
-from dagster._core.execution.plan.step import ExecutionStep
 from dagster._core.execution.resources_init import (
     get_required_resource_keys_to_init,
     resource_initialization_event_generator,
@@ -42,12 +44,13 @@ from dagster._loggers import colored_console_logger
 from dagster._serdes import unpack_value
 from dagster._utils import EventGenerationManager
 
-from .context import DagstermillExecutionContext, DagstermillRuntimeExecutionContext
-from .errors import DagstermillError
-from .serialize import PICKLE_PROTOCOL
+from dagstermill.context import DagstermillExecutionContext, DagstermillRuntimeExecutionContext
+from dagstermill.errors import DagstermillError
+from dagstermill.serialize import PICKLE_PROTOCOL
 
 if TYPE_CHECKING:
     from dagster._core.definitions.node_definition import NodeDefinition
+    from dagster._core.execution.plan.step import ExecutionStep
 
 
 class DagstermillResourceEventGenerationManager(EventGenerationManager):
@@ -60,14 +63,10 @@ class DagstermillResourceEventGenerationManager(EventGenerationManager):
         return iter(())
 
     def teardown(self):
-        return [
-            teardown_event
-            for teardown_event in super(
-                DagstermillResourceEventGenerationManager, self
-            ).generate_teardown_events()
-        ]
+        return [teardown_event for teardown_event in super().generate_teardown_events()]
 
 
+@beta
 class Manager:
     def __init__(self):
         self.job = None
@@ -87,6 +86,7 @@ class Manager:
         resource_keys_to_init: Optional[AbstractSet[str]],
         instance: Optional[DagsterInstance],
         emit_persistent_events: Optional[bool],
+        event_loop: Optional[AbstractEventLoop],
     ):
         """Drop-in replacement for
         `dagster._core.execution.resources_init.resource_initialization_manager`.  It uses a
@@ -101,6 +101,7 @@ class Manager:
             resource_keys_to_init=resource_keys_to_init,
             instance=instance,
             emit_persistent_events=emit_persistent_events,
+            event_loop=event_loop,
         )
         self.resource_manager = DagstermillResourceEventGenerationManager(
             generator, ScopedResourcesBuilder
@@ -195,9 +196,9 @@ class Manager:
                 op_name=op.name,
                 node_handle=node_handle,
                 step_context=cast(
-                    StepExecutionContext,
+                    "StepExecutionContext",
                     job_context.for_step(
-                        cast(ExecutionStep, execution_plan.get_step_by_key(step_key)),
+                        cast("ExecutionStep", execution_plan.get_step_by_key(step_key)),
                         known_state=known_state,
                     ),
                 ),
@@ -311,18 +312,18 @@ class Manager:
         # deferred import for perf
         import scrapbook
 
-        if not self.op_def.has_output(output_name):
+        if not self.op_def.has_output(output_name):  # pyright: ignore[reportOptionalMemberAccess]
             raise DagstermillError(
-                f"Op {self.op_def.name} does not have output named {output_name}.Expected one of"
-                f" {[str(output_def.name) for output_def in self.op_def.output_defs]}"
+                f"Op {self.op_def.name} does not have output named {output_name}.Expected one of"  # pyright: ignore[reportOptionalMemberAccess]
+                f" {[str(output_def.name) for output_def in self.op_def.output_defs]}"  # pyright: ignore[reportOptionalMemberAccess]
             )
 
         # pass output value cross process boundary using io manager
-        step_context = self.context._step_context  # noqa: SLF001
+        step_context = self.context._step_context  # noqa: SLF001  # pyright: ignore[reportAttributeAccessIssue,reportOptionalMemberAccess]
         # Note: yield_result currently does not support DynamicOutput
 
         # dagstermill assets do not support yielding additional results within the notebook:
-        if len(step_context.job_def.asset_layer.executable_asset_keys) > 0:
+        if len(step_context.job_def.asset_layer.executable_asset_keys) > 0:  # pyright: ignore[reportArgumentType]
             raise DagstermillError(
                 "dagstermill assets do not currently support dagstermill.yield_result"
             )
@@ -369,7 +370,7 @@ class Manager:
         import scrapbook
 
         event_id = f"event-{uuid.uuid4()}"
-        out_file_path = os.path.join(self.marshal_dir, event_id)
+        out_file_path = os.path.join(self.marshal_dir, event_id)  # pyright: ignore[reportCallIssue,reportArgumentType]
         with open(out_file_path, "wb") as fd:
             fd.write(pickle.dumps(dagster_event, PICKLE_PROTOCOL))
 

dagstermill/py.typed

@@ -0,0 +1 @@
+partial

dagstermill/serialize.py

@@ -1,11 +1,11 @@
-from dagster import _seven
+from dagster_shared import seven
 
 PICKLE_PROTOCOL = 2
 
 
 def is_json_serializable(value):
     try:
-        _seven.json.dumps(value)
+        seven.json.dumps(value)
         return True
     except TypeError:
         return False

dagstermill/test_utils.py

@@ -0,0 +1,45 @@
+import os
+from contextlib import contextmanager
+
+from dagster import execute_job
+from dagster._core.definitions.metadata import NotebookMetadataValue, PathMetadataValue
+from dagster._core.definitions.reconstruct import ReconstructableJob
+from dagster._core.test_utils import instance_for_test
+
+
+def get_path(materialization_event):
+    for value in materialization_event.event_specific_data.materialization.metadata.values():
+        if isinstance(value, (PathMetadataValue, NotebookMetadataValue)):
+            return value.value
+
+
+def cleanup_result_notebook(result):
+    if not result:
+        return
+    materialization_events = [
+        x for x in result.all_events if x.event_type_value == "ASSET_MATERIALIZATION"
+    ]
+    for materialization_event in materialization_events:
+        result_path = get_path(materialization_event)
+        if os.path.exists(result_path):  # pyright: ignore[reportArgumentType]
+            os.unlink(result_path)  # pyright: ignore[reportArgumentType]
+
+
+@contextmanager
+def exec_for_test(fn_name, env=None, raise_on_error=True, **kwargs):
+    result = None
+    recon_job = ReconstructableJob.for_module("dagstermill.examples.repository", fn_name)
+
+    with instance_for_test() as instance:
+        try:
+            with execute_job(
+                job=recon_job,
+                run_config=env,
+                instance=instance,
+                raise_on_error=raise_on_error,
+                **kwargs,
+            ) as result:
+                yield result
+        finally:
+            if result:
+                cleanup_result_notebook(result)

dagstermill/translator.py

@@ -1,6 +1,6 @@
 import papermill
 import papermill.translators
-from dagster import _seven
+from dagster_shared import seven
 
 RESERVED_INPUT_NAMES = [
     "__dm_context",
@@ -16,7 +16,7 @@ RESERVED_INPUT_NAMES = [
 
 INJECTED_BOILERPLATE = """
 # Injected parameters
-from dagster import seven as __dm_seven
+from dagster_shared import seven as __dm_seven
 import dagstermill as __dm_dagstermill
 context = __dm_dagstermill._reconstitute_job_context(
     **{{
@@ -51,7 +51,7 @@ class DagsterTranslator(papermill.translators.PythonTranslator):
         )
 
         for key in job_context_args:
-            job_context_args[key] = _seven.json.dumps(job_context_args[key])
+            job_context_args[key] = seven.json.dumps(job_context_args[key])
 
         content = INJECTED_BOILERPLATE.format(job_context_args=job_context_args)
 

dagstermill/version.py

@@ -1 +1 @@
-__version__ = "0.23.3"
+__version__ = "0.27.6"

dagstermill-0.27.6.dist-info/METADATA

@@ -0,0 +1,35 @@
+Metadata-Version: 2.4
+Name: dagstermill
+Version: 0.27.6
+Summary: run notebooks using the Dagster tools
+Author: Dagster Labs
+Author-email: hello@dagsterlabs.com
+License: Apache-2.0
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9,<3.14
+License-File: LICENSE
+Requires-Dist: dagster==1.11.6
+Requires-Dist: ipykernel!=5.4.0,!=5.4.1,>=4.9.0
+Requires-Dist: ipython_genutils>=0.2.0
+Requires-Dist: packaging>=20.9
+Requires-Dist: papermill>=1.0.0
+Requires-Dist: scrapbook>=0.5.0
+Requires-Dist: nbconvert
+Requires-Dist: jupyter-client<8
+Provides-Extra: test
+Requires-Dist: matplotlib; extra == "test"
+Requires-Dist: scikit-learn>=0.19.0; extra == "test"
+Requires-Dist: tqdm<=4.48; extra == "test"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

dagstermill-0.27.6.dist-info/RECORD

@@ -0,0 +1,24 @@
+dagstermill/__init__.py,sha256=kTAwcogNwhvjcMeWhYJKRP-uF6MazOZhiSoCiABbm2Y,1263
+dagstermill/__main__.py,sha256=GqkyStw1mh9zw2gMbQBQpIqW0hpr3I8unVWrw_VqdW8,41
+dagstermill/asset_factory.py,sha256=SfnLlMNYxfB3KgIDpHlZ4vGLsDzS40Pe33n6lPwXQvs,9843
+dagstermill/cli.py,sha256=J0vLeYGwcytvmyyL6ar29QDQ0G0Cdcmk_yM5JgnoqNw,4527
+dagstermill/compat.py,sha256=GCfUaGC3eIEhlZP_VFtua8hFKNtwXocOAfYxuKZ5X3I,526
+dagstermill/context.py,sha256=U-Os8kqiD2SBhF7im7bhn8xddmOPFt7-mZYu8fg7C0o,6175
+dagstermill/engine.py,sha256=exxp3VUOKcTWjEOlc0NQZkxeRPmR3JYTFp7BBH5xM7I,5788
+dagstermill/errors.py,sha256=4w6hTVfK_oX64Eusi66CvfdWZNVpCb96VEYMdCvTaD0,141
+dagstermill/factory.py,sha256=AlY-v8fhHjrncRvED3qQZQSYTUpAw0xtN_5iLRT6kjE,18872
+dagstermill/io_managers.py,sha256=zRIMpZ_j61942pTttrXdwwkS8Qv20x0r3V0JWf1xnSg,4440
+dagstermill/manager.py,sha256=_VHYoAZ7NPaYbLaA1-2Ph5_mmb-LSQN-OQRadD6soKg,16235
+dagstermill/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
+dagstermill/serialize.py,sha256=nvNv29bnxrRA5hXYcDJm8HV7Xsj0C4sRB2oUpHXCres,193
+dagstermill/test_utils.py,sha256=_OX3bXMpqCZDRF-Cq7UsBWOMaWuqgDPpIBuddk-ll-8,1585
+dagstermill/translator.py,sha256=6rKVaN3lcx8d5M1Dlir578s-fsATrTNq-ZJDmUDn9wk,2182
+dagstermill/version.py,sha256=BwKhBzWMdVser1JHOUEX0Aa2nBqgua67wsNi17fRle0,23
+dagstermill/examples/__init__.py,sha256=_2u28VKkHmd16vz0Q2N7Fw5QKsJ65HEnIfJL26gNUIw,75
+dagstermill/examples/repository.py,sha256=P_5rkdjsdUZ-fUci_ImZAx9JZMv7iCd1eZGPzQ2LUZc,16021
+dagstermill-0.27.6.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
+dagstermill-0.27.6.dist-info/METADATA,sha256=DYI0SibfUIU1DLhWdNfeYv6vF2fg8-TcD0qnM08yF_w,1109
+dagstermill-0.27.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dagstermill-0.27.6.dist-info/entry_points.txt,sha256=885a7vvhABYWEj7W28elkzSmIcKO3REkdd5h4Z4DEJs,53
+dagstermill-0.27.6.dist-info/top_level.txt,sha256=YDelJKdA5YIIrjsObdd8U4E9YhuXJLRe9NKfUzud9Uc,12
+dagstermill-0.27.6.dist-info/RECORD,,

{dagstermill-0.23.3.dist-info → dagstermill-0.27.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{dagstermill-0.23.3.dist-info → dagstermill-0.27.6.dist-info/licenses}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2023 Dagster Labs, Inc".
+   Copyright 2025 Dagster Labs, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

dagstermill-0.23.3.dist-info/METADATA

@@ -1,28 +0,0 @@
-Metadata-Version: 2.1
-Name: dagstermill
-Version: 0.23.3
-Summary: run notebooks using the Dagster tools
-Author: Dagster Labs
-Author-email: hello@dagsterlabs.com
-License: Apache-2.0
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8,<3.13
-License-File: LICENSE
-Requires-Dist: dagster ==1.7.3
-Requires-Dist: ipykernel !=5.4.0,!=5.4.1,>=4.9.0
-Requires-Dist: ipython-genutils >=0.2.0
-Requires-Dist: packaging >=20.9
-Requires-Dist: papermill >=1.0.0
-Requires-Dist: scrapbook >=0.5.0
-Requires-Dist: nbconvert
-Requires-Dist: jupyter-client <8
-Provides-Extra: test
-Requires-Dist: matplotlib ; extra == 'test'
-Requires-Dist: scikit-learn >=0.19.0 ; extra == 'test'
-Requires-Dist: tqdm <=4.48 ; extra == 'test'
-

dagstermill-0.23.3.dist-info/RECORD

@@ -1,22 +0,0 @@
-dagstermill/__init__.py,sha256=WZRYSjL1yAM1w5cqmC1t2T7hvuW-RGGY5tV9vZGHZwU,1133
-dagstermill/__main__.py,sha256=bYt9eEaoRQWdejEHFD8REx9jxVEdZptECFsV7F49Ink,30
-dagstermill/asset_factory.py,sha256=3moIwVr3dzFuDRVAdRZlQlO00nfGFAiA8cvmOgJi1zw,9463
-dagstermill/cli.py,sha256=lffeaQgenSR7s8MhNUBNA_BeVehlMUrdg7gnQl1WTdY,4494
-dagstermill/compat.py,sha256=GCfUaGC3eIEhlZP_VFtua8hFKNtwXocOAfYxuKZ5X3I,526
-dagstermill/context.py,sha256=3_SkRBM9wDB6T64Lc4yJZGp8mpA_PMXkHlF8U1I19A4,6104
-dagstermill/engine.py,sha256=4CZW-eni4TnG8C8VhPkZZgj6djItdOUKhHPzvaVhaYo,5843
-dagstermill/errors.py,sha256=WOmpAGp-J1XhyGK_LT3ZZKsBwF5dvrWbqSaSldtoh6Y,141
-dagstermill/factory.py,sha256=R9IiuAFi35nHneuWsegcXEH1iTp4qUuO0zmtizmBZb0,18638
-dagstermill/io_managers.py,sha256=D9UA4KufzrOeCan3TJyPogD0uIJBjK80ExikdMHqweI,4403
-dagstermill/manager.py,sha256=HPk1PtVOc7MDFb3F3t-XECvhPYRPRAXNBSapXYAIUBA,15788
-dagstermill/serialize.py,sha256=eXW3c26CiILT_uebyFcAKBnsiNxnjyGT_ch3PfGyjek,188
-dagstermill/translator.py,sha256=h1VPAOWtdjLKzFjRmyN9hO_R6qJuAETNkJydfdgwWGM,2170
-dagstermill/version.py,sha256=HTz96uK4_QHPX5i3NrEcr4eXcxlyepsy_nU-3niBQe8,23
-dagstermill/examples/__init__.py,sha256=kzan-9zFjxaJ8o9bqUso44gcGiOmJrlq4JYO-yIBQao,55
-dagstermill/examples/repository.py,sha256=hKQ1_GzHFiSf4oZmr1HYQmyRSMG0HDq25m0Y0oz0eXQ,15827
-dagstermill-0.23.3.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
-dagstermill-0.23.3.dist-info/METADATA,sha256=_vNUVd86D0at1bx1O2eZFPBYN-SCX_UISSsjVqCg2Wg,985
-dagstermill-0.23.3.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-dagstermill-0.23.3.dist-info/entry_points.txt,sha256=885a7vvhABYWEj7W28elkzSmIcKO3REkdd5h4Z4DEJs,53
-dagstermill-0.23.3.dist-info/top_level.txt,sha256=YDelJKdA5YIIrjsObdd8U4E9YhuXJLRe9NKfUzud9Uc,12
-dagstermill-0.23.3.dist-info/RECORD,,