dagstermill 0.20.2__tar.gz → 0.27.9__tar.gz

{dagstermill-0.20.2 → dagstermill-0.27.9}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2023 Elementl, Inc.
+   Copyright 2025 Dagster Labs, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

dagstermill-0.27.9/PKG-INFO

@@ -0,0 +1,35 @@
+Metadata-Version: 2.4
+Name: dagstermill
+Version: 0.27.9
+Summary: run notebooks using the Dagster tools
+Author: Dagster Labs
+Author-email: hello@dagsterlabs.com
+License: Apache-2.0
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9,<3.14
+License-File: LICENSE
+Requires-Dist: dagster==1.11.9
+Requires-Dist: ipykernel!=5.4.0,!=5.4.1,>=4.9.0
+Requires-Dist: ipython_genutils>=0.2.0
+Requires-Dist: packaging>=20.9
+Requires-Dist: papermill>=1.0.0
+Requires-Dist: scrapbook>=0.5.0
+Requires-Dist: nbconvert
+Requires-Dist: jupyter-client<8
+Provides-Extra: test
+Requires-Dist: matplotlib; extra == "test"
+Requires-Dist: scikit-learn>=0.19.0; extra == "test"
+Requires-Dist: tqdm<=4.48; extra == "test"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

dagstermill-0.27.9/README.md

@@ -0,0 +1,4 @@
+# dagstermill
+
+The docs for `dagstermill` can be found
+[here](https://docs.dagster.io/api/python-api/libraries/dagstermill).

dagstermill-0.27.9/dagstermill/__init__.py

@@ -0,0 +1,27 @@
+from dagster_shared.libraries import DagsterLibraryRegistry
+
+import dagstermill.factory as factory  # noqa: F401
+from dagstermill.asset_factory import define_dagstermill_asset as define_dagstermill_asset
+from dagstermill.context import DagstermillExecutionContext as DagstermillExecutionContext
+from dagstermill.errors import DagstermillError as DagstermillError
+from dagstermill.factory import define_dagstermill_op as define_dagstermill_op
+from dagstermill.io_managers import (
+    ConfigurableLocalOutputNotebookIOManager as ConfigurableLocalOutputNotebookIOManager,
+    local_output_notebook_io_manager as local_output_notebook_io_manager,
+)
+from dagstermill.manager import MANAGER_FOR_NOTEBOOK_INSTANCE as _MANAGER_FOR_NOTEBOOK_INSTANCE
+from dagstermill.version import __version__ as __version__
+
+DagsterLibraryRegistry.register("dagstermill", __version__)
+
+get_context = _MANAGER_FOR_NOTEBOOK_INSTANCE.get_context
+
+yield_result = _MANAGER_FOR_NOTEBOOK_INSTANCE.yield_result
+
+yield_event = _MANAGER_FOR_NOTEBOOK_INSTANCE.yield_event
+
+_reconstitute_job_context = _MANAGER_FOR_NOTEBOOK_INSTANCE.reconstitute_job_context
+
+_teardown = _MANAGER_FOR_NOTEBOOK_INSTANCE.teardown_resources
+
+_load_input_parameter = _MANAGER_FOR_NOTEBOOK_INSTANCE.load_input_parameter

dagstermill-0.27.9/dagstermill/__main__.py

@@ -0,0 +1,3 @@
+from dagstermill.cli import main
+
+main()

{dagstermill-0.20.2 → dagstermill-0.27.9}/dagstermill/asset_factory.py

@@ -1,6 +1,7 @@
 import pickle
 import tempfile
-from typing import Any, Callable, Iterable, Mapping, Optional, Sequence, Set, Type, Union, cast
+from collections.abc import Iterable, Mapping
+from typing import Any, Callable, Optional, Union, cast
 
 import dagster._check as check
 from dagster import (
@@ -16,11 +17,13 @@ from dagster import (
     SourceAsset,
     asset,
 )
+from dagster._annotations import beta, beta_param
 from dagster._config.pythonic_config import Config, infer_schema_from_config_class
-from dagster._config.pythonic_config.utils import safe_is_subclass
+from dagster._config.pythonic_config.type_check_utils import safe_is_subclass
 from dagster._core.definitions.events import CoercibleToAssetKey, CoercibleToAssetKeyPrefix
-from dagster._core.definitions.utils import validate_tags
 from dagster._core.execution.context.compute import OpExecutionContext
+from dagster._core.storage.tags import COMPUTE_KIND_TAG
+from dagster._utils.tags import normalize_tags
 
 from dagstermill.factory import _clean_path_for_windows, execute_notebook
 
@@ -67,15 +70,17 @@ def _make_dagstermill_asset_compute_fn(
     return _t_fn
 
 
+@beta
+@beta_param(param="resource_defs")
 def define_dagstermill_asset(
     name: str,
     notebook_path: str,
     key_prefix: Optional[CoercibleToAssetKeyPrefix] = None,
     ins: Optional[Mapping[str, AssetIn]] = None,
-    deps: Optional[Sequence[Union[CoercibleToAssetKey, AssetsDefinition, SourceAsset]]] = None,
+    deps: Optional[Iterable[Union[CoercibleToAssetKey, AssetsDefinition, SourceAsset]]] = None,
     metadata: Optional[Mapping[str, Any]] = None,
     config_schema: Optional[Union[Any, Mapping[str, Any]]] = None,
-    required_resource_keys: Optional[Set[str]] = None,
+    required_resource_keys: Optional[set[str]] = None,
     resource_defs: Optional[Mapping[str, ResourceDefinition]] = None,
     description: Optional[str] = None,
     partitions_def: Optional[PartitionsDefinition] = None,
@@ -84,7 +89,8 @@ def define_dagstermill_asset(
     io_manager_key: Optional[str] = None,
     retry_policy: Optional[RetryPolicy] = None,
     save_notebook_on_failure: bool = False,
-    non_argument_deps: Optional[Union[Set[AssetKey], Set[str]]] = None,
+    non_argument_deps: Optional[Union[set[AssetKey], set[str]]] = None,
+    asset_tags: Optional[Mapping[str, Any]] = None,
 ) -> AssetsDefinition:
     """Creates a Dagster asset for a Jupyter notebook.
 
@@ -104,7 +110,7 @@ def define_dagstermill_asset(
             if it does not. If not set, Dagster will accept any config provided for the op.
         metadata (Optional[Dict[str, Any]]): A dict of metadata entries for the asset.
         required_resource_keys (Optional[Set[str]]): Set of resource handles required by the notebook.
-        description (Optional[str]): Description of the asset to display in Dagit.
+        description (Optional[str]): Description of the asset to display in the Dagster UI.
         partitions_def (Optional[PartitionsDefinition]): Defines the set of partition keys that
             compose the asset.
         op_tags (Optional[Dict[str, Any]]): A dictionary of tags for the op that computes the asset.
@@ -114,7 +120,7 @@ def define_dagstermill_asset(
         group_name (Optional[str]): A string name used to organize multiple assets into groups. If not provided,
             the name "default" is used.
         resource_defs (Optional[Mapping[str, ResourceDefinition]]):
-            (Experimental) A mapping of resource keys to resource definitions. These resources
+            (Beta) A mapping of resource keys to resource definitions. These resources
             will be initialized during execution, and can be accessed from the
             context within the notebook.
         io_manager_key (Optional[str]): A string key for the IO manager used to store the output notebook.
@@ -123,6 +129,7 @@ def define_dagstermill_asset(
         save_notebook_on_failure (bool): If True and the notebook fails during execution, the failed notebook will be
             written to the Dagster storage directory. The location of the file will be printed in the Dagster logs.
             Defaults to False.
+        asset_tags (Optional[Dict[str, Any]]): A dictionary of tags to apply to the asset.
         non_argument_deps (Optional[Union[Set[AssetKey], Set[str]]]): Deprecated, use deps instead. Set of asset keys that are
             upstream dependencies, but do not pass an input to the asset.
 
@@ -172,27 +179,26 @@ def define_dagstermill_asset(
         io_manager_key, "io_manager_key", default="output_notebook_io_manager"
     )
 
-    user_tags = validate_tags(op_tags)
+    user_tags = normalize_tags(op_tags)
     if op_tags is not None:
         check.invariant(
             "notebook_path" not in op_tags,
-            (
-                "user-defined op tags contains the `notebook_path` key, but the `notebook_path` key"
-                " is reserved for use by Dagster"
-            ),
+            "user-defined op tags contains the `notebook_path` key, but the `notebook_path` key"
+            " is reserved for use by Dagster",
         )
         check.invariant(
-            "kind" not in op_tags,
-            (
-                "user-defined op tags contains the `kind` key, but the `kind` key is reserved for"
-                " use by Dagster"
-            ),
+            COMPUTE_KIND_TAG not in op_tags,
+            f"user-defined op tags contains the `{COMPUTE_KIND_TAG}` key, but the `{COMPUTE_KIND_TAG}` key is reserved for"
+            " use by Dagster",
         )
 
-    default_tags = {"notebook_path": _clean_path_for_windows(notebook_path), "kind": "ipynb"}
+    default_tags = {
+        "notebook_path": _clean_path_for_windows(notebook_path),
+        COMPUTE_KIND_TAG: "ipynb",
+    }
 
     if safe_is_subclass(config_schema, Config):
-        config_schema = infer_schema_from_config_class(cast(Type[Config], config_schema))
+        config_schema = infer_schema_from_config_class(cast("type[Config]", config_schema))
 
     return asset(
         name=name,
@@ -211,6 +217,7 @@ def define_dagstermill_asset(
         io_manager_key=io_mgr_key,
         retry_policy=retry_policy,
         non_argument_deps=non_argument_deps,
+        tags=asset_tags,
     )(
         _make_dagstermill_asset_compute_fn(
             name=name,

{dagstermill-0.20.2 → dagstermill-0.27.9}/dagstermill/cli.py

@@ -1,13 +1,14 @@
 import copy
 import os
 import subprocess
-from typing import Mapping, Optional
+from collections.abc import Mapping
+from typing import Optional
 
 import click
 import dagster._check as check
 import nbformat
-from dagster._seven.json import loads
 from dagster._utils import mkdir_p, safe_isfile
+from dagster_shared.seven.json import loads
 from papermill.iorw import load_notebook_node, write_ipynb
 
 
@@ -110,11 +111,9 @@ def execute_create_notebook(notebook: str, force_overwrite: bool, kernel: str):
 
     if not force_overwrite and safe_isfile(notebook_path):
         click.confirm(
-            (
-                "Warning, {notebook_path} already exists and continuing "
-                "will overwrite the existing notebook. "
-                "Are you sure you want to continue?"
-            ).format(notebook_path=notebook_path),
+            f"Warning, {notebook_path} already exists and continuing "
+            "will overwrite the existing notebook. "
+            "Are you sure you want to continue?",
             abort=True,
         )
 

{dagstermill-0.20.2 → dagstermill-0.27.9}/dagstermill/context.py

@@ -1,4 +1,5 @@
-from typing import AbstractSet, Any, Mapping, Optional, cast
+from collections.abc import Mapping
+from typing import AbstractSet, Any, Optional, cast  # noqa: UP035
 
 from dagster import (
     DagsterRun,
@@ -6,15 +7,18 @@ from dagster import (
     OpDefinition,
     _check as check,
 )
-from dagster._annotations import public
+from dagster._annotations import beta, public
 from dagster._core.definitions.dependency import Node, NodeHandle
-from dagster._core.execution.context.compute import AbstractComputeExecutionContext
+from dagster._core.definitions.repository_definition.repository_definition import (
+    RepositoryDefinition,
+)
+from dagster._core.execution.context.op_execution_context import AbstractComputeExecutionContext
 from dagster._core.execution.context.system import PlanExecutionContext, StepExecutionContext
 from dagster._core.log_manager import DagsterLogManager
 from dagster._core.system_config.objects import ResolvedRunConfig
-from dagster._utils.backcompat import deprecation_warning
 
 
+@beta
 class DagstermillExecutionContext(AbstractComputeExecutionContext):
     """Dagstermill-specific execution context.
 
@@ -101,6 +105,11 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
         """
         return self._job_def
 
+    @property
+    def repository_def(self) -> RepositoryDefinition:
+        """:class:`dagster.RepositoryDefinition`: The repository definition for the context."""
+        raise NotImplementedError
+
     @property
     def resources(self) -> Any:
         """collections.namedtuple: A dynamically-created type whose properties allow access to
@@ -114,7 +123,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
     @property
     def run(self) -> DagsterRun:
         """:class:`dagster.DagsterRun`: The job run for the context."""
-        return cast(DagsterRun, self._job_context.dagster_run)
+        return cast("DagsterRun", self._job_context.dagster_run)
 
     @property
     def log(self) -> DagsterLogManager:
@@ -132,7 +141,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
         In interactive contexts, this may be a dagstermill-specific shim, depending whether an
         op definition was passed to ``dagstermill.get_context``.
         """
-        return cast(OpDefinition, self._job_def.node_def_named(self.op_name))
+        return cast("OpDefinition", self._job_def.node_def_named(self.op_name))
 
     @property
     def node(self) -> Node:
@@ -141,11 +150,6 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
         In interactive contexts, this may be a dagstermill-specific shim, depending whether an
         op definition was passed to ``dagstermill.get_context``.
         """
-        deprecation_warning(
-            "DagstermillExecutionContext.solid_def",
-            "0.17.0",
-            "use the 'op_def' property instead.",
-        )
         return self.job_def.get_node(self.node_handle)
 
     @public

{dagstermill-0.20.2 → dagstermill-0.27.9}/dagstermill/engine.py

@@ -1,7 +1,7 @@
 import nbformat
 from papermill.log import logger
 
-from .compat import ExecutionError, is_papermill_2
+from dagstermill.compat import ExecutionError, is_papermill_2
 
 if is_papermill_2():
     from papermill.clientwrap import PapermillNotebookClient
@@ -86,9 +86,7 @@ else:
         # the kernel down. Note that atexit doesn't seem to work at all in ipython, and hooking into
         # the ipython post_execute event doesn't work in papermill.
         def papermill_process(self, nb_man, resources):
-            _, resources = super(DagstermillExecutePreprocessor, self).papermill_process(
-                nb_man, resources
-            )
+            _, resources = super().papermill_process(nb_man, resources)
 
             new_cell = nbformat.v4.new_code_cell(
                 source="import dagstermill as __dm_dagstermill\n__dm_dagstermill._teardown()\n"

{dagstermill-0.20.2 → dagstermill-0.27.9}/dagstermill/errors.py

@@ -1,4 +1,4 @@
-from dagster._core.errors import DagsterError
+from dagster_shared.error import DagsterError
 
 
 class DagstermillError(DagsterError):

dagstermill-0.27.9/dagstermill/examples/__init__.py

@@ -0,0 +1 @@
+from dagstermill.examples.repository import notebook_repo as notebook_repo

{dagstermill-0.20.2 → dagstermill-0.27.9}/dagstermill/examples/repository.py

@@ -23,7 +23,7 @@ from dagster import (
     with_resources,
 )
 from dagster._config.pythonic_config import Config
-from dagster._core.definitions.asset_graph import AssetGraph
+from dagster._core.definitions.assets.graph.asset_graph import AssetGraph
 from dagster._core.definitions.utils import DEFAULT_OUTPUT
 from dagster._utils import PICKLE_PROTOCOL, file_relative_path
 
@@ -96,7 +96,9 @@ def hello_world_job():
 
 @job(
     resource_defs={
-        "output_notebook_io_manager": ConfigurableLocalOutputNotebookIOManager.configure_at_launch(),
+        "output_notebook_io_manager": (
+            ConfigurableLocalOutputNotebookIOManager.configure_at_launch()
+        ),
         "io_manager": FilesystemIOManager(),
     }
 )
@@ -301,12 +303,12 @@ def error_job():
 
 if DAGSTER_PANDAS_PRESENT and SKLEARN_PRESENT and MATPLOTLIB_PRESENT:
     # We need type-ignores here because type checkers don't understand the `*_PRESENT` kwargs.
-    clean_data = test_nb_op("clean_data", outs={DEFAULT_OUTPUT: Out(DataFrame)})  # type: ignore
+    clean_data = test_nb_op("clean_data", outs={DEFAULT_OUTPUT: Out(DataFrame)})  # pyright: ignore[reportPossiblyUnboundVariable]
 
     # FIXME add an output to this
-    tutorial_LR = test_nb_op("tutorial_LR", ins={"df": In(DataFrame)})  # type: ignore
+    tutorial_LR = test_nb_op("tutorial_LR", ins={"df": In(DataFrame)})  # pyright: ignore[reportPossiblyUnboundVariable]
 
-    tutorial_RF = test_nb_op("tutorial_RF", ins={"df": In(DataFrame)})  # type: ignore
+    tutorial_RF = test_nb_op("tutorial_RF", ins={"df": In(DataFrame)})  # pyright: ignore[reportPossiblyUnboundVariable]
 
     @job(resource_defs=common_resource_defs)
     def tutorial_job():
@@ -568,6 +570,7 @@ custom_io_mgr_key_asset = dagstermill.define_dagstermill_asset(
     name="custom_io_mgr_key",
     notebook_path=nb_test_path("hello_world"),
     io_manager_key="my_custom_io_manager",
+    asset_tags={"foo": "bar"},
 )
 
 yield_event_asset = dagstermill.define_dagstermill_asset(

{dagstermill-0.20.2 → dagstermill-0.27.9}/dagstermill/factory.py

@@ -4,7 +4,8 @@ import pickle
 import sys
 import tempfile
 import uuid
-from typing import Any, Callable, Iterable, Mapping, Optional, Sequence, Set, Type, Union, cast
+from collections.abc import Iterable, Mapping, Sequence
+from typing import Any, Callable, Optional, Union, cast
 
 import nbformat
 import papermill
@@ -14,33 +15,35 @@ from dagster import (
     Out,
     Output,
     _check as check,
-    _seven,
 )
+from dagster._annotations import beta
 from dagster._config.pythonic_config import Config, infer_schema_from_config_class
-from dagster._config.pythonic_config.utils import safe_is_subclass
+from dagster._config.pythonic_config.type_check_utils import safe_is_subclass
 from dagster._core.definitions.events import AssetMaterialization, Failure, RetryRequested
 from dagster._core.definitions.metadata import MetadataValue
 from dagster._core.definitions.reconstruct import ReconstructableJob
-from dagster._core.definitions.utils import validate_tags
 from dagster._core.execution.context.compute import OpExecutionContext
 from dagster._core.execution.context.input import build_input_context
 from dagster._core.execution.context.system import StepExecutionContext
 from dagster._core.execution.plan.outputs import StepOutputHandle
+from dagster._core.storage.tags import COMPUTE_KIND_TAG
 from dagster._serdes import pack_value
-from dagster._seven import get_system_temp_directory
 from dagster._utils import mkdir_p, safe_tempfile_path
 from dagster._utils.error import serializable_error_info_from_exc_info
+from dagster._utils.tags import normalize_tags
+from dagster_shared import seven
+from dagster_shared.seven import get_system_temp_directory
 from papermill.engines import papermill_engines
 from papermill.iorw import load_notebook_node, write_ipynb
 
-from .compat import ExecutionError
-from .engine import DagstermillEngine
-from .errors import DagstermillError
-from .translator import DagsterTranslator
+from dagstermill.compat import ExecutionError
+from dagstermill.engine import DagstermillEngine
+from dagstermill.errors import DagstermillError
+from dagstermill.translator import DagsterTranslator
 
 
 def _clean_path_for_windows(notebook_path: str) -> str:
-    """In windows, the notebook cant render in dagit unless the C: prefix is removed.
+    """In windows, the notebook can't render in the Dagster UI unless the C: prefix is removed.
     os.path.splitdrive will split the path into (drive, tail), so just return the tail.
     """
     return os.path.splitdrive(notebook_path)[1]
@@ -60,6 +63,7 @@ def _find_first_tagged_cell_index(nb, tag):
 # This is based on papermill.parameterize.parameterize_notebook
 # Typically, papermill injects the injected-parameters cell *below* the parameters cell
 # but we want to *replace* the parameters cell, which is what this function does.
+@beta
 def replace_parameters(context, nb, parameters):
     """Assigned parameters into the appropriate place in the input notebook.
 
@@ -101,11 +105,12 @@ def replace_parameters(context, nb, parameters):
         after = nb.cells
 
     nb.cells = before + [newcell] + after
-    nb.metadata.papermill["parameters"] = _seven.json.dumps(parameters)
+    nb.metadata.papermill["parameters"] = seven.json.dumps(parameters)
 
     return nb
 
 
+@beta
 def get_papermill_parameters(
     step_context: StepExecutionContext,
     inputs: Mapping[str, object],
@@ -159,6 +164,7 @@ def get_papermill_parameters(
     return parameters
 
 
+@beta
 def execute_notebook(
     step_context: StepExecutionContext,
     name: str,
@@ -199,9 +205,7 @@ def execute_notebook(
 
         except Exception as ex:
             step_context.log.warn(
-                "Error when attempting to materialize executed notebook: {exc}".format(
-                    exc=str(serializable_error_info_from_exc_info(sys.exc_info()))
-                )
+                f"Error when attempting to materialize executed notebook: {serializable_error_info_from_exc_info(sys.exc_info())!s}"
             )
 
             if isinstance(ex, ExecutionError):
@@ -342,13 +346,14 @@ def _make_dagstermill_compute_fn(
     return _t_fn
 
 
+@beta
 def define_dagstermill_op(
     name: str,
     notebook_path: str,
     ins: Optional[Mapping[str, In]] = None,
     outs: Optional[Mapping[str, Out]] = None,
     config_schema: Optional[Union[Any, Mapping[str, Any]]] = None,
-    required_resource_keys: Optional[Set[str]] = None,
+    required_resource_keys: Optional[set[str]] = None,
     output_notebook_name: Optional[str] = None,
     asset_key_prefix: Optional[Union[Sequence[str], str]] = None,
     description: Optional[str] = None,
@@ -403,7 +408,7 @@ def define_dagstermill_op(
         required_resource_keys.add(io_mgr_key)
         outs = {
             **outs,
-            cast(str, output_notebook_name): Out(io_manager_key=io_mgr_key),
+            cast("str", output_notebook_name): Out(io_manager_key=io_mgr_key),
         }
 
     if isinstance(asset_key_prefix, str):
@@ -414,26 +419,25 @@ def define_dagstermill_op(
     default_description = f"This op is backed by the notebook at {notebook_path}"
     description = check.opt_str_param(description, "description", default=default_description)
 
-    user_tags = validate_tags(tags)
+    user_tags = normalize_tags(tags)
     if tags is not None:
         check.invariant(
             "notebook_path" not in tags,
-            (
-                "user-defined op tags contains the `notebook_path` key, but the `notebook_path` key"
-                " is reserved for use by Dagster"
-            ),
+            "user-defined op tags contains the `notebook_path` key, but the `notebook_path` key"
+            " is reserved for use by Dagster",
         )
         check.invariant(
-            "kind" not in tags,
-            (
-                "user-defined op tags contains the `kind` key, but the `kind` key is reserved for"
-                " use by Dagster"
-            ),
+            COMPUTE_KIND_TAG not in tags,
+            "user-defined op tags contains the `kind` key, but the `kind` key is reserved for"
+            " use by Dagster",
         )
-    default_tags = {"notebook_path": _clean_path_for_windows(notebook_path), "kind": "ipynb"}
+    default_tags = {
+        "notebook_path": _clean_path_for_windows(notebook_path),
+        COMPUTE_KIND_TAG: "ipynb",
+    }
 
     if safe_is_subclass(config_schema, Config):
-        config_schema = infer_schema_from_config_class(cast(Type[Config], config_schema))
+        config_schema = infer_schema_from_config_class(cast("type[Config]", config_schema))
 
     return OpDefinition(
         name=name,

{dagstermill-0.20.2 → dagstermill-0.27.9}/dagstermill/io_managers.py

@@ -1,6 +1,7 @@
 import os
+from collections.abc import Sequence
 from pathlib import Path
-from typing import Any, List, Optional, Sequence
+from typing import Any, Optional
 
 import dagster._check as check
 from dagster import (
@@ -10,6 +11,7 @@ from dagster import (
     InitResourceContext,
     IOManager,
 )
+from dagster._annotations import beta
 from dagster._core.definitions.metadata import MetadataValue
 from dagster._core.execution.context.input import InputContext
 from dagster._core.execution.context.output import OutputContext
@@ -33,7 +35,7 @@ class OutputNotebookIOManager(IOManager):
 
 class LocalOutputNotebookIOManager(OutputNotebookIOManager):
     def __init__(self, base_dir: str, asset_key_prefix: Optional[Sequence[str]] = None):
-        super(LocalOutputNotebookIOManager, self).__init__(asset_key_prefix=asset_key_prefix)
+        super().__init__(asset_key_prefix=asset_key_prefix)
         self.base_dir = base_dir
         self.write_mode = "wb"
         self.read_mode = "rb"
@@ -82,6 +84,7 @@ class LocalOutputNotebookIOManager(OutputNotebookIOManager):
             return file_obj.read()
 
 
+@beta
 class ConfigurableLocalOutputNotebookIOManager(ConfigurableIOManagerFactory):
     """Built-in IO Manager for handling output notebook."""
 
@@ -92,7 +95,7 @@ class ConfigurableLocalOutputNotebookIOManager(ConfigurableIOManagerFactory):
             " directory if not provided."
         ),
     )
-    asset_key_prefix: List[str] = Field(
+    asset_key_prefix: list[str] = Field(
         default=[],
         description=(
             "Asset key prefix to apply to assets materialized for output notebooks. Defaults to no"
@@ -111,6 +114,7 @@ class ConfigurableLocalOutputNotebookIOManager(ConfigurableIOManagerFactory):
         )
 
 
+@beta
 @dagster_maintained_io_manager
 @io_manager(config_schema=ConfigurableLocalOutputNotebookIOManager.to_config_schema())
 def local_output_notebook_io_manager(init_context) -> LocalOutputNotebookIOManager:

{dagstermill-0.20.2 → dagstermill-0.27.9}/dagstermill/manager.py

@@ -1,7 +1,9 @@
 import os
 import pickle
 import uuid
-from typing import AbstractSet, Any, Mapping, Optional, cast
+from asyncio import AbstractEventLoop
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, AbstractSet, Any, Optional, cast  # noqa: UP035
 
 from dagster import (
     AssetMaterialization,
@@ -14,12 +16,12 @@ from dagster import (
     TypeCheck,
     _check as check,
 )
+from dagster._annotations import beta
 from dagster._core.definitions.dependency import NodeHandle
 from dagster._core.definitions.events import RetryRequested
 from dagster._core.definitions.graph_definition import GraphDefinition
 from dagster._core.definitions.job_base import InMemoryJob
 from dagster._core.definitions.job_definition import JobDefinition
-from dagster._core.definitions.node_definition import NodeDefinition
 from dagster._core.definitions.op_definition import OpDefinition
 from dagster._core.definitions.reconstruct import ReconstructableJob
 from dagster._core.definitions.resource_definition import ScopedResourcesBuilder
@@ -27,7 +29,7 @@ from dagster._core.events import DagsterEvent
 from dagster._core.execution.api import create_execution_plan, scoped_job_context
 from dagster._core.execution.plan.outputs import StepOutputHandle
 from dagster._core.execution.plan.plan import ExecutionPlan
-from dagster._core.execution.plan.step import ExecutionStep
+from dagster._core.execution.plan.state import KnownExecutionState
 from dagster._core.execution.resources_init import (
     get_required_resource_keys_to_init,
     resource_initialization_event_generator,
@@ -42,9 +44,13 @@ from dagster._loggers import colored_console_logger
 from dagster._serdes import unpack_value
 from dagster._utils import EventGenerationManager
 
-from .context import DagstermillExecutionContext, DagstermillRuntimeExecutionContext
-from .errors import DagstermillError
-from .serialize import PICKLE_PROTOCOL
+from dagstermill.context import DagstermillExecutionContext, DagstermillRuntimeExecutionContext
+from dagstermill.errors import DagstermillError
+from dagstermill.serialize import PICKLE_PROTOCOL
+
+if TYPE_CHECKING:
+    from dagster._core.definitions.node_definition import NodeDefinition
+    from dagster._core.execution.plan.step import ExecutionStep
 
 
 class DagstermillResourceEventGenerationManager(EventGenerationManager):
@@ -57,14 +63,10 @@ class DagstermillResourceEventGenerationManager(EventGenerationManager):
         return iter(())
 
     def teardown(self):
-        return [
-            teardown_event
-            for teardown_event in super(
-                DagstermillResourceEventGenerationManager, self
-            ).generate_teardown_events()
-        ]
+        return [teardown_event for teardown_event in super().generate_teardown_events()]
 
 
+@beta
 class Manager:
     def __init__(self):
         self.job = None
@@ -84,6 +86,7 @@ class Manager:
         resource_keys_to_init: Optional[AbstractSet[str]],
         instance: Optional[DagsterInstance],
         emit_persistent_events: Optional[bool],
+        event_loop: Optional[AbstractEventLoop],
     ):
         """Drop-in replacement for
         `dagster._core.execution.resources_init.resource_initialization_manager`.  It uses a
@@ -98,6 +101,7 @@ class Manager:
             resource_keys_to_init=resource_keys_to_init,
             instance=instance,
             emit_persistent_events=emit_persistent_events,
+            event_loop=event_loop,
         )
         self.resource_manager = DagstermillResourceEventGenerationManager(
             generator, ScopedResourcesBuilder
@@ -175,6 +179,12 @@ class Manager:
             # Set this flag even though we're not in test for clearer error reporting
             raise_on_error=True,
         ) as job_context:
+            known_state = None
+            if dagster_run.parent_run_id:
+                known_state = KnownExecutionState.build_for_reexecution(
+                    instance=instance,
+                    parent_run=check.not_none(instance.get_run_by_id(dagster_run.parent_run_id)),
+                )
             self.context = DagstermillRuntimeExecutionContext(
                 job_context=job_context,
                 job_def=job_def,
@@ -186,9 +196,10 @@ class Manager:
                 op_name=op.name,
                 node_handle=node_handle,
                 step_context=cast(
-                    StepExecutionContext,
+                    "StepExecutionContext",
                     job_context.for_step(
-                        cast(ExecutionStep, execution_plan.get_step_by_key(step_key))
+                        cast("ExecutionStep", execution_plan.get_step_by_key(step_key)),
+                        known_state=known_state,
                     ),
                 ),
             )
@@ -301,18 +312,18 @@ class Manager:
         # deferred import for perf
         import scrapbook
 
-        if not self.op_def.has_output(output_name):
+        if not self.op_def.has_output(output_name):  # pyright: ignore[reportOptionalMemberAccess]
             raise DagstermillError(
-                f"Op {self.op_def.name} does not have output named {output_name}.Expected one of"
-                f" {[str(output_def.name) for output_def in self.op_def.output_defs]}"
+                f"Op {self.op_def.name} does not have output named {output_name}.Expected one of"  # pyright: ignore[reportOptionalMemberAccess]
+                f" {[str(output_def.name) for output_def in self.op_def.output_defs]}"  # pyright: ignore[reportOptionalMemberAccess]
             )
 
         # pass output value cross process boundary using io manager
-        step_context = self.context._step_context  # noqa: SLF001
+        step_context = self.context._step_context  # noqa: SLF001  # pyright: ignore[reportAttributeAccessIssue,reportOptionalMemberAccess]
         # Note: yield_result currently does not support DynamicOutput
 
         # dagstermill assets do not support yielding additional results within the notebook:
-        if len(step_context.job_def.asset_layer.asset_keys) > 0:
+        if len(step_context.job_def.asset_layer.executable_asset_keys) > 0:  # pyright: ignore[reportArgumentType]
             raise DagstermillError(
                 "dagstermill assets do not currently support dagstermill.yield_result"
             )
@@ -359,7 +370,7 @@ class Manager:
         import scrapbook
 
         event_id = f"event-{uuid.uuid4()}"
-        out_file_path = os.path.join(self.marshal_dir, event_id)
+        out_file_path = os.path.join(self.marshal_dir, event_id)  # pyright: ignore[reportCallIssue,reportArgumentType]
         with open(out_file_path, "wb") as fd:
             fd.write(pickle.dumps(dagster_event, PICKLE_PROTOCOL))
 

{dagstermill-0.20.2 → dagstermill-0.27.9}/dagstermill/serialize.py

@@ -1,11 +1,11 @@
-from dagster import _seven
+from dagster_shared import seven
 
 PICKLE_PROTOCOL = 2
 
 
 def is_json_serializable(value):
     try:
-        _seven.json.dumps(value)
+        seven.json.dumps(value)
         return True
     except TypeError:
         return False

dagstermill-0.27.9/dagstermill/test_utils.py

@@ -0,0 +1,45 @@
+import os
+from contextlib import contextmanager
+
+from dagster import execute_job
+from dagster._core.definitions.metadata import NotebookMetadataValue, PathMetadataValue
+from dagster._core.definitions.reconstruct import ReconstructableJob
+from dagster._core.test_utils import instance_for_test
+
+
+def get_path(materialization_event):
+    for value in materialization_event.event_specific_data.materialization.metadata.values():
+        if isinstance(value, (PathMetadataValue, NotebookMetadataValue)):
+            return value.value
+
+
+def cleanup_result_notebook(result):
+    if not result:
+        return
+    materialization_events = [
+        x for x in result.all_events if x.event_type_value == "ASSET_MATERIALIZATION"
+    ]
+    for materialization_event in materialization_events:
+        result_path = get_path(materialization_event)
+        if os.path.exists(result_path):  # pyright: ignore[reportArgumentType]
+            os.unlink(result_path)  # pyright: ignore[reportArgumentType]
+
+
+@contextmanager
+def exec_for_test(fn_name, env=None, raise_on_error=True, **kwargs):
+    result = None
+    recon_job = ReconstructableJob.for_module("dagstermill.examples.repository", fn_name)
+
+    with instance_for_test() as instance:
+        try:
+            with execute_job(
+                job=recon_job,
+                run_config=env,
+                instance=instance,
+                raise_on_error=raise_on_error,
+                **kwargs,
+            ) as result:
+                yield result
+        finally:
+            if result:
+                cleanup_result_notebook(result)

{dagstermill-0.20.2 → dagstermill-0.27.9}/dagstermill/translator.py

@@ -1,6 +1,6 @@
 import papermill
 import papermill.translators
-from dagster import _seven
+from dagster_shared import seven
 
 RESERVED_INPUT_NAMES = [
     "__dm_context",
@@ -16,7 +16,7 @@ RESERVED_INPUT_NAMES = [
 
 INJECTED_BOILERPLATE = """
 # Injected parameters
-from dagster import seven as __dm_seven
+from dagster_shared import seven as __dm_seven
 import dagstermill as __dm_dagstermill
 context = __dm_dagstermill._reconstitute_job_context(
     **{{
@@ -51,7 +51,7 @@ class DagsterTranslator(papermill.translators.PythonTranslator):
         )
 
         for key in job_context_args:
-            job_context_args[key] = _seven.json.dumps(job_context_args[key])
+            job_context_args[key] = seven.json.dumps(job_context_args[key])
 
         content = INJECTED_BOILERPLATE.format(job_context_args=job_context_args)
 

dagstermill-0.27.9/dagstermill/version.py

@@ -0,0 +1 @@
+__version__ = "0.27.9"

dagstermill-0.27.9/dagstermill.egg-info/PKG-INFO

@@ -0,0 +1,35 @@
+Metadata-Version: 2.4
+Name: dagstermill
+Version: 0.27.9
+Summary: run notebooks using the Dagster tools
+Author: Dagster Labs
+Author-email: hello@dagsterlabs.com
+License: Apache-2.0
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9,<3.14
+License-File: LICENSE
+Requires-Dist: dagster==1.11.9
+Requires-Dist: ipykernel!=5.4.0,!=5.4.1,>=4.9.0
+Requires-Dist: ipython_genutils>=0.2.0
+Requires-Dist: packaging>=20.9
+Requires-Dist: papermill>=1.0.0
+Requires-Dist: scrapbook>=0.5.0
+Requires-Dist: nbconvert
+Requires-Dist: jupyter-client<8
+Provides-Extra: test
+Requires-Dist: matplotlib; extra == "test"
+Requires-Dist: scikit-learn>=0.19.0; extra == "test"
+Requires-Dist: tqdm<=4.48; extra == "test"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

{dagstermill-0.20.2 → dagstermill-0.27.9}/dagstermill.egg-info/SOURCES.txt

@@ -16,6 +16,7 @@ dagstermill/io_managers.py
 dagstermill/manager.py
 dagstermill/py.typed
 dagstermill/serialize.py
+dagstermill/test_utils.py
 dagstermill/translator.py
 dagstermill/version.py
 dagstermill.egg-info/PKG-INFO

{dagstermill-0.20.2 → dagstermill-0.27.9}/dagstermill.egg-info/requires.txt

@@ -1,4 +1,4 @@
-dagster==1.4.2
+dagster==1.11.9
 ipykernel!=5.4.0,!=5.4.1,>=4.9.0
 ipython_genutils>=0.2.0
 packaging>=20.9

{dagstermill-0.20.2 → dagstermill-0.27.9}/setup.py

@@ -1,11 +1,10 @@
 from pathlib import Path
-from typing import Dict
 
 from setuptools import find_packages, setup
 
 
 def get_version() -> str:
-    version: Dict[str, str] = {}
+    version: dict[str, str] = {}
     with open(Path(__file__).parent / "dagstermill/version.py", encoding="utf8") as fp:
         exec(fp.read(), version)
 
@@ -19,20 +18,20 @@ setup(
     name="dagstermill",
     version=ver,
     description="run notebooks using the Dagster tools",
-    author="Elementl",
-    author_email="hello@elementl.com",
+    author="Dagster Labs",
+    author_email="hello@dagsterlabs.com",
     license="Apache-2.0",
     packages=find_packages(exclude=["dagstermill_tests*"]),
     classifiers=[
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: OS Independent",
     ],
+    python_requires=">=3.9,<3.14",
     install_requires=[
-        "dagster==1.4.2",
+        "dagster==1.11.9",
         # ipykernel 5.4.0 and 5.4.1 broke papermill
         # see https://github.com/dagster-io/dagster/issues/3401,
         # https://github.com/nteract/papermill/issues/519,

dagstermill-0.20.2/PKG-INFO

@@ -1,15 +0,0 @@
-Metadata-Version: 2.1
-Name: dagstermill
-Version: 0.20.2
-Summary: run notebooks using the Dagster tools
-Author: Elementl
-Author-email: hello@elementl.com
-License: Apache-2.0
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Operating System :: OS Independent
-Provides-Extra: test
-License-File: LICENSE

dagstermill-0.20.2/README.md

@@ -1,4 +0,0 @@
-# dagstermill
-
-The docs for `dagstermill` can be found
-[here](https://docs.dagster.io/_apidocs/libraries/dagstermill).

dagstermill-0.20.2/dagstermill/__init__.py

@@ -1,26 +0,0 @@
-from dagster._core.libraries import DagsterLibraryRegistry
-
-from .asset_factory import define_dagstermill_asset as define_dagstermill_asset
-from .context import DagstermillExecutionContext as DagstermillExecutionContext
-from .errors import DagstermillError as DagstermillError
-from .factory import define_dagstermill_op as define_dagstermill_op
-from .io_managers import (
-    ConfigurableLocalOutputNotebookIOManager as ConfigurableLocalOutputNotebookIOManager,
-    local_output_notebook_io_manager as local_output_notebook_io_manager,
-)
-from .manager import MANAGER_FOR_NOTEBOOK_INSTANCE as _MANAGER_FOR_NOTEBOOK_INSTANCE
-from .version import __version__ as __version__
-
-DagsterLibraryRegistry.register("dagstermill", __version__)
-
-get_context = _MANAGER_FOR_NOTEBOOK_INSTANCE.get_context
-
-yield_result = _MANAGER_FOR_NOTEBOOK_INSTANCE.yield_result
-
-yield_event = _MANAGER_FOR_NOTEBOOK_INSTANCE.yield_event
-
-_reconstitute_job_context = _MANAGER_FOR_NOTEBOOK_INSTANCE.reconstitute_job_context
-
-_teardown = _MANAGER_FOR_NOTEBOOK_INSTANCE.teardown_resources
-
-_load_input_parameter = _MANAGER_FOR_NOTEBOOK_INSTANCE.load_input_parameter

dagstermill-0.20.2/dagstermill/__main__.py

@@ -1,3 +0,0 @@
-from .cli import main
-
-main()

dagstermill-0.20.2/dagstermill/examples/__init__.py

@@ -1 +0,0 @@
-from .repository import notebook_repo as notebook_repo

dagstermill-0.20.2/dagstermill/version.py

@@ -1 +0,0 @@
-__version__ = "0.20.2"

dagstermill-0.20.2/dagstermill.egg-info/PKG-INFO

@@ -1,15 +0,0 @@
-Metadata-Version: 2.1
-Name: dagstermill
-Version: 0.20.2
-Summary: run notebooks using the Dagster tools
-Author: Elementl
-Author-email: hello@elementl.com
-License: Apache-2.0
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Operating System :: OS Independent
-Provides-Extra: test
-License-File: LICENSE