dagstermill 0.19.1__py3-none-any.whl → 0.19.3__py3-none-any.whl

dagstermill/__init__.py

@@ -4,7 +4,10 @@ from .asset_factory import define_dagstermill_asset as define_dagstermill_asset
 from .context import DagstermillExecutionContext as DagstermillExecutionContext
 from .errors import DagstermillError as DagstermillError
 from .factory import define_dagstermill_op as define_dagstermill_op
-from .io_managers import local_output_notebook_io_manager as local_output_notebook_io_manager
+from .io_managers import (
+    ConfigurableLocalOutputNotebookIOManager as ConfigurableLocalOutputNotebookIOManager,
+    local_output_notebook_io_manager as local_output_notebook_io_manager,
+)
 from .manager import MANAGER_FOR_NOTEBOOK_INSTANCE as _MANAGER_FOR_NOTEBOOK_INSTANCE
 from .version import __version__ as __version__
 
@@ -16,7 +19,7 @@ yield_result = _MANAGER_FOR_NOTEBOOK_INSTANCE.yield_result
 
 yield_event = _MANAGER_FOR_NOTEBOOK_INSTANCE.yield_event
 
-_reconstitute_pipeline_context = _MANAGER_FOR_NOTEBOOK_INSTANCE.reconstitute_pipeline_context
+_reconstitute_job_context = _MANAGER_FOR_NOTEBOOK_INSTANCE.reconstitute_job_context
 
 _teardown = _MANAGER_FOR_NOTEBOOK_INSTANCE.teardown_resources
 

dagstermill/asset_factory.py

@@ -1,6 +1,6 @@
 import pickle
 import tempfile
-from typing import Any, Callable, Iterable, Mapping, Optional, Set, Union
+from typing import Any, Callable, Iterable, Mapping, Optional, Set, Type, Union, cast
 
 import dagster._check as check
 from dagster import (
@@ -15,6 +15,8 @@ from dagster import (
     RetryRequested,
     asset,
 )
+from dagster._config.pythonic_config import Config, infer_schema_from_config_class
+from dagster._config.pythonic_config.utils import safe_is_subclass
 from dagster._core.definitions.events import CoercibleToAssetKeyPrefix
 from dagster._core.definitions.utils import validate_tags
 from dagster._core.execution.context.compute import OpExecutionContext
@@ -185,6 +187,9 @@ def define_dagstermill_asset(
 
     default_tags = {"notebook_path": _clean_path_for_windows(notebook_path), "kind": "ipynb"}
 
+    if safe_is_subclass(config_schema, Config):
+        config_schema = infer_schema_from_config_class(cast(Type[Config], config_schema))
+
     return asset(
         name=name,
         key_prefix=key_prefix,

dagstermill/context.py

@@ -12,7 +12,6 @@ from dagster._core.execution.context.compute import AbstractComputeExecutionCont
 from dagster._core.execution.context.system import PlanExecutionContext, StepExecutionContext
 from dagster._core.log_manager import DagsterLogManager
 from dagster._core.system_config.objects import ResolvedRunConfig
-from dagster._legacy import PipelineDefinition
 from dagster._utils.backcompat import deprecation_warning
 
 
@@ -24,17 +23,15 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
 
     def __init__(
         self,
-        pipeline_context: PlanExecutionContext,
-        pipeline_def: PipelineDefinition,
+        job_context: PlanExecutionContext,
+        job_def: JobDefinition,
         resource_keys_to_init: AbstractSet[str],
         op_name: str,
         node_handle: NodeHandle,
         op_config: Any = None,
     ):
-        self._pipeline_context = check.inst_param(
-            pipeline_context, "pipeline_context", PlanExecutionContext
-        )
-        self._pipeline_def = check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition)
+        self._job_context = check.inst_param(job_context, "job_context", PlanExecutionContext)
+        self._job_def = check.inst_param(job_def, "job_def", JobDefinition)
         self._resource_keys_to_init = check.set_param(
             resource_keys_to_init, "resource_keys_to_init", of_type=str
         )
@@ -52,7 +49,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
             bool
         """
         check.str_param(key, "key")
-        return self._pipeline_context.has_tag(key)
+        return self._job_context.has_tag(key)
 
     def get_tag(self, key: str) -> Optional[str]:
         """Get a logging tag defined on the context.
@@ -64,44 +61,35 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
             str
         """
         check.str_param(key, "key")
-        return self._pipeline_context.get_tag(key)
+        return self._job_context.get_tag(key)
 
     @public
     @property
     def run_id(self) -> str:
         """str: The run_id for the context."""
-        return self._pipeline_context.run_id
+        return self._job_context.run_id
 
     @public
     @property
     def run_config(self) -> Mapping[str, Any]:
         """dict: The run_config for the context."""
-        return self._pipeline_context.run_config
+        return self._job_context.run_config
 
     @property
     def resolved_run_config(self) -> ResolvedRunConfig:
         """:class:`dagster.ResolvedRunConfig`: The resolved_run_config for the context."""
-        return self._pipeline_context.resolved_run_config
+        return self._job_context.resolved_run_config
 
     @public
     @property
     def logging_tags(self) -> Mapping[str, str]:
         """dict: The logging tags for the context."""
-        return self._pipeline_context.logging_tags
+        return self._job_context.logging_tags
 
     @public
     @property
     def job_name(self) -> str:
-        return self._pipeline_context.job_name
-
-    @property
-    def pipeline_name(self) -> str:
-        deprecation_warning(
-            "DagstermillExecutionContext.pipeline_name",
-            "0.17.0",
-            "use the 'job_name' property instead.",
-        )
-        return self.job_name
+        return self._job_context.job_name
 
     @public
     @property
@@ -110,34 +98,14 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
 
         This will be a dagstermill-specific shim.
         """
-        return cast(
-            JobDefinition,
-            check.inst(
-                self._pipeline_def,
-                JobDefinition,
-                "Accessing job_def inside a legacy pipeline. Use pipeline_def instead.",
-            ),
-        )
-
-    @property
-    def pipeline_def(self) -> PipelineDefinition:
-        """:class:`dagster.PipelineDefinition`: The pipeline definition for the context.
-
-        This will be a dagstermill-specific shim.
-        """
-        deprecation_warning(
-            "DagstermillExecutionContext.pipeline_def",
-            "0.17.0",
-            "use the 'job_def' property instead.",
-        )
-        return self._pipeline_def
+        return self._job_def
 
     @property
     def resources(self) -> Any:
         """collections.namedtuple: A dynamically-created type whose properties allow access to
         resources.
         """
-        return self._pipeline_context.scoped_resources_builder.build(
+        return self._job_context.scoped_resources_builder.build(
             required_resource_keys=self._resource_keys_to_init,
         )
 
@@ -145,16 +113,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
     @property
     def run(self) -> DagsterRun:
         """:class:`dagster.DagsterRun`: The job run for the context."""
-        return cast(DagsterRun, self._pipeline_context.dagster_run)
-
-    @property
-    def pipeline_run(self) -> DagsterRun:
-        deprecation_warning(
-            "DagstermillExecutionContext.pipeline_run",
-            "0.17.0",
-            "use the 'run' property instead.",
-        )
-        return self.run
+        return cast(DagsterRun, self._job_context.dagster_run)
 
     @property
     def log(self) -> DagsterLogManager:
@@ -162,7 +121,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
 
         Call, e.g., ``log.info()`` to log messages through the Dagster machinery.
         """
-        return self._pipeline_context.log
+        return self._job_context.log
 
     @public
     @property
@@ -172,7 +131,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
         In interactive contexts, this may be a dagstermill-specific shim, depending whether an
         op definition was passed to ``dagstermill.get_context``.
         """
-        return cast(OpDefinition, self._pipeline_def.node_def_named(self.op_name))
+        return cast(OpDefinition, self._job_def.node_def_named(self.op_name))
 
     @property
     def node(self) -> Node:
@@ -186,7 +145,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
             "0.17.0",
             "use the 'op_def' property instead.",
         )
-        return self.pipeline_def.get_node(self.node_handle)
+        return self.job_def.get_node(self.node_handle)
 
     @public
     @property
@@ -204,8 +163,8 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
 class DagstermillRuntimeExecutionContext(DagstermillExecutionContext):
     def __init__(
         self,
-        pipeline_context: PlanExecutionContext,
-        pipeline_def: PipelineDefinition,
+        job_context: PlanExecutionContext,
+        job_def: JobDefinition,
         resource_keys_to_init: AbstractSet[str],
         op_name: str,
         step_context: StepExecutionContext,
@@ -214,8 +173,8 @@ class DagstermillRuntimeExecutionContext(DagstermillExecutionContext):
     ):
         self._step_context = check.inst_param(step_context, "step_context", StepExecutionContext)
         super().__init__(
-            pipeline_context,
-            pipeline_def,
+            job_context,
+            job_def,
             resource_keys_to_init,
             op_name,
             node_handle,

dagstermill/examples/repository.py

@@ -7,7 +7,6 @@ from dagster import (
     AssetSelection,
     Field,
     In,
-    Int,
     List,
     Out,
     ResourceDefinition,
@@ -23,11 +22,15 @@ from dagster import (
     resource,
     with_resources,
 )
+from dagster._config.pythonic_config import Config
 from dagster._core.definitions.utils import DEFAULT_OUTPUT
 from dagster._utils import PICKLE_PROTOCOL, file_relative_path
 
 import dagstermill
-from dagstermill.io_managers import local_output_notebook_io_manager
+from dagstermill.io_managers import (
+    ConfigurableLocalOutputNotebookIOManager,
+    local_output_notebook_io_manager,
+)
 
 try:
     from dagster_pandas import DataFrame
@@ -90,6 +93,16 @@ def hello_world_job():
     hello_world()
 
 
+@job(
+    resource_defs={
+        "output_notebook_io_manager": ConfigurableLocalOutputNotebookIOManager.configure_at_launch(),
+        "io_manager": fs_io_manager,
+    }
+)
+def hello_world_job_struct_resource() -> None:
+    hello_world()
+
+
 def build_hello_world_job():
     @job(
         resource_defs={
@@ -136,6 +149,31 @@ def hello_world_config_job():
     goodbye_config()
 
 
+class HelloWorldConfig(Config):
+    greeting: str = "hello"
+
+
+hello_world_config_struct = test_nb_op("hello_world_config_struct", config_schema=HelloWorldConfig)
+
+
+class GoodbyeConfig(Config):
+    farewell: str = "goodbye"
+
+
+goodbye_config_struct = test_nb_op(
+    name="goodbye_config_struct",
+    path=nb_test_path("print_dagstermill_context_op_config"),
+    output_notebook_name="notebook",
+    config_schema=GoodbyeConfig,
+)
+
+
+@job(resource_defs=common_resource_defs)
+def hello_world_config_job_struct() -> None:
+    hello_world_config_struct()
+    goodbye_config_struct()
+
+
 @job(resource_defs=common_resource_defs)
 def alias_config_job():
     hello_world_config.alias("aliased_greeting")()
@@ -239,9 +277,9 @@ def double_add_job():
     add_two_numbers.alias("add_two_numbers_2")(return_three(), return_four())
 
 
-@op(config_schema=Int)
-def load_constant(context):
-    return context.op_config
+@op
+def load_constant(config: int) -> int:
+    return config
 
 
 @job(resource_defs=common_resource_defs)
@@ -536,8 +574,8 @@ yield_event_asset = dagstermill.define_dagstermill_asset(
 )
 
 
-# this is hacky. We need a ReconstructablePipeline to run dagstermill, and
-# ReconstructablePipeline.for_module() find the jobs defined in this file. So we need to resolve all
+# this is hacky. We need a ReconstructableJob to run dagstermill, and
+# ReconstructableJob.for_module() find the jobs defined in this file. So we need to resolve all
 # of the asset jobs outside of the repository function.
 assets = with_resources(
     [
@@ -585,6 +623,7 @@ def notebook_repo():
         bad_kernel_job,
         error_job,
         hello_world_job,
+        hello_world_job_struct_resource,
         hello_world_with_custom_tags_and_description_job,
         hello_world_config_job,
         hello_world_explicit_yield_job,

dagstermill/factory.py

@@ -4,7 +4,7 @@ import pickle
 import sys
 import tempfile
 import uuid
-from typing import Any, Callable, Iterable, Mapping, Optional, Sequence, Set, Union, cast
+from typing import Any, Callable, Iterable, Mapping, Optional, Sequence, Set, Type, Union, cast
 
 import nbformat
 import papermill
@@ -16,9 +16,11 @@ from dagster import (
     _check as check,
     _seven,
 )
+from dagster._config.pythonic_config import Config, infer_schema_from_config_class
+from dagster._config.pythonic_config.utils import safe_is_subclass
 from dagster._core.definitions.events import AssetMaterialization, Failure, RetryRequested
 from dagster._core.definitions.metadata import MetadataValue
-from dagster._core.definitions.reconstruct import ReconstructablePipeline
+from dagster._core.definitions.reconstruct import ReconstructableJob
 from dagster._core.definitions.utils import validate_tags
 from dagster._core.execution.context.compute import OpExecutionContext
 from dagster._core.execution.context.input import build_input_context
@@ -121,7 +123,7 @@ def get_papermill_parameters(
     marshal_dir = os.path.normpath(os.path.join(temp_dir, "dagstermill", str(run_id), "marshal"))
     mkdir_p(marshal_dir)
 
-    if not isinstance(step_context.pipeline, ReconstructablePipeline):
+    if not isinstance(step_context.job, ReconstructableJob):
         if compute_descriptor == "asset":
             raise DagstermillError(
                 "Can't execute a dagstermill asset that is not reconstructable. "
@@ -133,7 +135,7 @@ def get_papermill_parameters(
                 "Use the reconstructable() function if executing from python"
             )
 
-    dm_executable_dict = step_context.pipeline.to_dict()
+    dm_executable_dict = step_context.job.to_dict()
 
     dm_context_dict = {
         "output_log_path": output_log_path,
@@ -430,6 +432,9 @@ def define_dagstermill_op(
         )
     default_tags = {"notebook_path": _clean_path_for_windows(notebook_path), "kind": "ipynb"}
 
+    if safe_is_subclass(config_schema, Config):
+        config_schema = infer_schema_from_config_class(cast(Type[Config], config_schema))
+
     return OpDefinition(
         name=name,
         compute_fn=_make_dagstermill_compute_fn(

dagstermill/io_managers.py

@@ -1,15 +1,21 @@
 import os
 from pathlib import Path
-from typing import Any, Optional, Sequence
+from typing import Any, List, Optional, Sequence
 
 import dagster._check as check
-from dagster import AssetKey, AssetMaterialization
-from dagster._config import Field
+from dagster import (
+    AssetKey,
+    AssetMaterialization,
+    ConfigurableIOManagerFactory,
+    InitResourceContext,
+    IOManager,
+)
 from dagster._core.definitions.metadata import MetadataValue
 from dagster._core.execution.context.input import InputContext
 from dagster._core.execution.context.output import OutputContext
-from dagster._core.storage.io_manager import IOManager, io_manager
+from dagster._core.storage.io_manager import io_manager
 from dagster._utils import mkdir_p
+from pydantic import Field
 
 from dagstermill.factory import _clean_path_for_windows
 
@@ -26,8 +32,6 @@ class OutputNotebookIOManager(IOManager):
 
 
 class LocalOutputNotebookIOManager(OutputNotebookIOManager):
-    """Built-in IO Manager for handling output notebook."""
-
     def __init__(self, base_dir: str, asset_key_prefix: Optional[Sequence[str]] = None):
         super(LocalOutputNotebookIOManager, self).__init__(asset_key_prefix=asset_key_prefix)
         self.base_dir = base_dir
@@ -70,7 +74,7 @@ class LocalOutputNotebookIOManager(OutputNotebookIOManager):
                 )
             )
 
-    def load_input(self, context) -> bytes:
+    def load_input(self, context: InputContext) -> bytes:
         check.inst_param(context, "context", InputContext)
         # pass output notebook to downstream ops as File Object
         output_context = check.not_none(context.upstream_output)
@@ -78,17 +82,32 @@ class LocalOutputNotebookIOManager(OutputNotebookIOManager):
             return file_obj.read()
 
 
-@io_manager(
-    config_schema={
-        "asset_key_prefix": Field(str, is_required=False),
-        "base_dir": Field(str, is_required=False),
-    },
-)
-def local_output_notebook_io_manager(init_context):
-    """Built-in IO Manager that handles output notebooks."""
-    return LocalOutputNotebookIOManager(
-        base_dir=init_context.resource_config.get(
-            "base_dir", init_context.instance.storage_directory()
+class ConfigurableLocalOutputNotebookIOManager(ConfigurableIOManagerFactory):
+    """Built-in IO Manager for handling output notebook."""
+
+    base_dir: Optional[str] = Field(
+        default=None,
+        description=(
+            "Base directory to use for output notebooks. Defaults to the Dagster instance storage"
+            " directory if not provided."
         ),
-        asset_key_prefix=init_context.resource_config.get("asset_key_prefix", []),
     )
+    asset_key_prefix: List[str] = Field(
+        default=[],
+        description=(
+            "Asset key prefix to apply to assets materialized for output notebooks. Defaults to no"
+            " prefix."
+        ),
+    )
+
+    def create_io_manager(self, context: InitResourceContext) -> "LocalOutputNotebookIOManager":
+        return LocalOutputNotebookIOManager(
+            base_dir=self.base_dir or check.not_none(context.instance).storage_directory(),
+            asset_key_prefix=self.asset_key_prefix,
+        )
+
+
+@io_manager(config_schema=ConfigurableLocalOutputNotebookIOManager.to_config_schema())
+def local_output_notebook_io_manager(init_context) -> LocalOutputNotebookIOManager:
+    """Built-in IO Manager that handles output notebooks."""
+    return ConfigurableLocalOutputNotebookIOManager.from_resource_context(init_context)

dagstermill/manager.py

@@ -16,14 +16,15 @@ from dagster import (
 )
 from dagster._core.definitions.dependency import NodeHandle
 from dagster._core.definitions.events import RetryRequested
+from dagster._core.definitions.graph_definition import GraphDefinition
+from dagster._core.definitions.job_base import InMemoryJob
+from dagster._core.definitions.job_definition import JobDefinition
 from dagster._core.definitions.node_definition import NodeDefinition
 from dagster._core.definitions.op_definition import OpDefinition
-from dagster._core.definitions.pipeline_base import InMemoryPipeline
-from dagster._core.definitions.reconstruct import ReconstructablePipeline
+from dagster._core.definitions.reconstruct import ReconstructableJob
 from dagster._core.definitions.resource_definition import ScopedResourcesBuilder
-from dagster._core.errors import DagsterInvariantViolationError
 from dagster._core.events import DagsterEvent
-from dagster._core.execution.api import scoped_pipeline_context
+from dagster._core.execution.api import scoped_job_context
 from dagster._core.execution.plan.outputs import StepOutputHandle
 from dagster._core.execution.plan.plan import ExecutionPlan
 from dagster._core.execution.plan.step import ExecutionStep
@@ -34,14 +35,12 @@ from dagster._core.execution.resources_init import (
 from dagster._core.instance import DagsterInstance
 from dagster._core.instance.ref import InstanceRef
 from dagster._core.log_manager import DagsterLogManager
-from dagster._core.storage.pipeline_run import DagsterRun, DagsterRunStatus
+from dagster._core.storage.dagster_run import DagsterRun, DagsterRunStatus
 from dagster._core.system_config.objects import ResolvedRunConfig, ResourceConfig
 from dagster._core.utils import make_new_run_id
-from dagster._legacy import ModeDefinition, PipelineDefinition
 from dagster._loggers import colored_console_logger
 from dagster._serdes import unpack_value
 from dagster._utils import EventGenerationManager
-from dagster._utils.backcompat import deprecation_warning
 
 from .context import DagstermillExecutionContext, DagstermillRuntimeExecutionContext
 from .errors import DagstermillError
@@ -68,9 +67,9 @@ class DagstermillResourceEventGenerationManager(EventGenerationManager):
 
 class Manager:
     def __init__(self):
-        self.pipeline = None
+        self.job = None
         self.op_def: Optional[NodeDefinition] = None
-        self.in_pipeline: bool = False
+        self.in_job: bool = False
         self.marshal_dir: Optional[str] = None
         self.context = None
         self.resource_manager = None
@@ -105,10 +104,10 @@ class Manager:
         )
         return self.resource_manager
 
-    def reconstitute_pipeline_context(
+    def reconstitute_job_context(
         self,
         executable_dict: Mapping[str, Any],
-        pipeline_run_dict: Mapping[str, Any],
+        job_run_dict: Mapping[str, Any],
         node_handle_kwargs: Mapping[str, Any],
         instance_ref_dict: Mapping[str, Any],
         step_key: str,
@@ -118,26 +117,26 @@ class Manager:
     ):
         """Reconstitutes a context for dagstermill-managed execution.
 
-        You'll see this function called to reconstruct a pipeline context within the ``injected
+        You'll see this function called to reconstruct a job context within the ``injected
         parameters`` cell of a dagstermill output notebook. Users should not call this function
         interactively except when debugging output notebooks.
 
         Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a
         context for interactive exploration and development. This call will be replaced by one to
-        :func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by
+        :func:`dagstermill.reconstitute_job_context` when the notebook is executed by
         dagstermill.
         """
         check.opt_str_param(output_log_path, "output_log_path")
         check.opt_str_param(marshal_dir, "marshal_dir")
         run_config = check.opt_mapping_param(run_config, "run_config", key_type=str)
-        check.mapping_param(pipeline_run_dict, "pipeline_run_dict")
+        check.mapping_param(job_run_dict, "job_run_dict")
         check.mapping_param(executable_dict, "executable_dict")
         check.mapping_param(node_handle_kwargs, "node_handle_kwargs")
         check.mapping_param(instance_ref_dict, "instance_ref_dict")
         check.str_param(step_key, "step_key")
 
-        pipeline = ReconstructablePipeline.from_dict(executable_dict)
-        pipeline_def = pipeline.get_definition()
+        job = ReconstructableJob.from_dict(executable_dict)
+        job_def = job.get_definition()
 
         try:
             instance_ref = unpack_value(instance_ref_dict, InstanceRef)
@@ -147,51 +146,49 @@ class Manager:
                 "Error when attempting to resolve DagsterInstance from serialized InstanceRef"
             ) from err
 
-        dagster_run = unpack_value(pipeline_run_dict, DagsterRun)
+        dagster_run = unpack_value(job_run_dict, DagsterRun)
 
         node_handle = NodeHandle.from_dict(node_handle_kwargs)
-        op = pipeline_def.get_node(node_handle)
+        op = job_def.get_node(node_handle)
         op_def = op.definition
 
         self.marshal_dir = marshal_dir
-        self.in_pipeline = True
+        self.in_job = True
         self.op_def = op_def
-        self.pipeline = pipeline
+        self.job = job
 
-        resolved_run_config = ResolvedRunConfig.build(
-            pipeline_def, run_config, mode=dagster_run.mode
-        )
+        resolved_run_config = ResolvedRunConfig.build(job_def, run_config)
 
         execution_plan = ExecutionPlan.build(
-            self.pipeline,
+            self.job,
             resolved_run_config,
             step_keys_to_execute=dagster_run.step_keys_to_execute,
         )
 
-        with scoped_pipeline_context(
+        with scoped_job_context(
             execution_plan,
-            pipeline,
+            job,
             run_config,
             dagster_run,
             instance,
             scoped_resources_builder_cm=self._setup_resources,
             # Set this flag even though we're not in test for clearer error reporting
             raise_on_error=True,
-        ) as pipeline_context:
+        ) as job_context:
             self.context = DagstermillRuntimeExecutionContext(
-                pipeline_context=pipeline_context,
-                pipeline_def=pipeline_def,
+                job_context=job_context,
+                job_def=job_def,
                 op_config=run_config.get("ops", {}).get(op.name, {}).get("config"),
                 resource_keys_to_init=get_required_resource_keys_to_init(
                     execution_plan,
-                    pipeline_def,
+                    job_def,
                     resolved_run_config,
                 ),
                 op_name=op.name,
                 node_handle=node_handle,
                 step_context=cast(
                     StepExecutionContext,
-                    pipeline_context.for_step(
+                    job_context.for_step(
                         cast(ExecutionStep, execution_plan.get_step_by_key(step_key))
                     ),
                 ),
@@ -204,7 +201,6 @@ class Manager:
         op_config: Any = None,
         resource_defs: Optional[Mapping[str, ResourceDefinition]] = None,
         logger_defs: Optional[Mapping[str, LoggerDefinition]] = None,
-        mode_def: Optional[ModeDefinition] = None,
         run_config: Optional[dict] = None,
     ) -> DagstermillExecutionContext:
         """Get a dagstermill execution context for interactive exploration and development.
@@ -220,31 +216,8 @@ class Manager:
         Returns:
             :py:class:`~dagstermill.DagstermillExecutionContext`
         """
-        check.opt_inst_param(mode_def, "mode_def", ModeDefinition)
         run_config = check.opt_dict_param(run_config, "run_config", key_type=str)
 
-        if resource_defs and mode_def:
-            raise DagsterInvariantViolationError(
-                "Attempted to provide both resource_defs and mode_def arguments to"
-                " `dagstermill.get_context`. Please provide one or the other."
-            )
-
-        if logger_defs and mode_def:
-            raise DagsterInvariantViolationError(
-                "Attempted to provide both logger_defs and mode_def arguments to"
-                " `dagstermill.get_context`. Please provide one or the other."
-            )
-
-        if mode_def:
-            deprecation_warning(
-                "mode_def argument to dagstermill.get_context",
-                "0.17.0",
-                (
-                    "Use the resource_defs argument to provide resources, and the logger_defs"
-                    " argument to provide loggers."
-                ),
-            )
-
         # If we are running non-interactively, and there is already a context reconstituted, return
         # that context rather than overwriting it.
         if self.context is not None and isinstance(
@@ -252,64 +225,63 @@ class Manager:
         ):
             return self.context
 
-        if not mode_def:
-            if not logger_defs:
-                logger_defs = {"dagstermill": colored_console_logger}
-                run_config["loggers"] = {"dagstermill": {}}
-            logger_defs = check.opt_mapping_param(logger_defs, "logger_defs")
-            resource_defs = check.opt_mapping_param(resource_defs, "resource_defs")
-            mode_def = ModeDefinition(logger_defs=logger_defs, resource_defs=resource_defs)
+        if not logger_defs:
+            logger_defs = {"dagstermill": colored_console_logger}
+            run_config["loggers"] = {"dagstermill": {}}
+        logger_defs = check.opt_mapping_param(logger_defs, "logger_defs")
+        resource_defs = check.opt_mapping_param(resource_defs, "resource_defs")
 
         op_def = OpDefinition(
             name="this_op",
             compute_fn=lambda *args, **kwargs: None,
             description="Ephemeral op constructed by dagstermill.get_context()",
-            required_resource_keys=mode_def.resource_key_set,
+            required_resource_keys=set(resource_defs.keys()),
         )
 
-        pipeline_def = PipelineDefinition(
-            [op_def], mode_defs=[mode_def], name="ephemeral_dagstermill_pipeline"
+        job_def = JobDefinition(
+            graph_def=GraphDefinition(name="ephemeral_dagstermill_pipeline", node_defs=[op_def]),
+            logger_defs=logger_defs,
+            resource_defs=resource_defs,
         )
 
         run_id = make_new_run_id()
 
-        # construct stubbed PipelineRun for notebook exploration...
-        # The actual pipeline run during pipeline execution will be serialized and reconstituted
-        # in the `reconstitute_pipeline_context` call
-        pipeline_run = DagsterRun(
-            pipeline_name=pipeline_def.name,
+        # construct stubbed DagsterRun for notebook exploration...
+        # The actual dagster run during job execution will be serialized and reconstituted
+        # in the `reconstitute_job_context` call
+        dagster_run = DagsterRun(
+            job_name=job_def.name,
             run_id=run_id,
             run_config=run_config,
-            mode=mode_def.name,
             step_keys_to_execute=None,
             status=DagsterRunStatus.NOT_STARTED,
             tags=None,
         )
 
-        self.in_pipeline = False
+        self.in_job = False
         self.op_def = op_def
-        self.pipeline = pipeline_def
+        self.job = job_def
 
-        resolved_run_config = ResolvedRunConfig.build(pipeline_def, run_config, mode=mode_def.name)
+        resolved_run_config = ResolvedRunConfig.build(job_def, run_config)
 
-        pipeline = InMemoryPipeline(pipeline_def)
-        execution_plan = ExecutionPlan.build(pipeline, resolved_run_config)
+        job = InMemoryJob(job_def)
+        execution_plan = ExecutionPlan.build(job, resolved_run_config)
 
-        with scoped_pipeline_context(
+        with scoped_job_context(
             execution_plan,
-            pipeline,
+            job,
             run_config,
-            pipeline_run,
+            dagster_run,
             DagsterInstance.ephemeral(),
             scoped_resources_builder_cm=self._setup_resources,
-        ) as pipeline_context:
+        ) as job_context:
             self.context = DagstermillExecutionContext(
-                pipeline_context=pipeline_context,
-                pipeline_def=pipeline_def,
+                job_context=job_context,
+                job_def=job_def,
                 op_config=op_config,
                 resource_keys_to_init=get_required_resource_keys_to_init(
                     execution_plan,
-                    pipeline_def,
+                    job_def,
                     resolved_run_config,
                 ),
                 op_name=op_def.name,
@@ -327,7 +299,7 @@ class Manager:
             value (Any): The value to yield.
             output_name (Optional[str]): The name of the result to yield (default: ``'result'``).
         """
-        if not self.in_pipeline:
+        if not self.in_job:
             return value
 
         # deferred import for perf
@@ -384,7 +356,7 @@ class Manager:
                 f" type, one of {valid_types}."
             )
 
-        if not self.in_pipeline:
+        if not self.in_job:
             return dagster_event
 
         # deferred import for perf

dagstermill/translator.py

@@ -18,11 +18,11 @@ INJECTED_BOILERPLATE = """
 # Injected parameters
 from dagster import seven as __dm_seven
 import dagstermill as __dm_dagstermill
-context = __dm_dagstermill._reconstitute_pipeline_context(
+context = __dm_dagstermill._reconstitute_job_context(
     **{{
         key: __dm_seven.json.loads(value)
         for key, value
-        in {pipeline_context_args}.items()
+        in {job_context_args}.items()
     }}
 )
 """
@@ -41,19 +41,19 @@ class DagsterTranslator(papermill.translators.PythonTranslator):
         assert "__dm_input_names" in parameters
 
         context_args = parameters["__dm_context"]
-        pipeline_context_args = dict(
+        job_context_args = dict(
             executable_dict=parameters["__dm_executable_dict"],
-            pipeline_run_dict=parameters["__dm_pipeline_run_dict"],
+            job_run_dict=parameters["__dm_pipeline_run_dict"],
             node_handle_kwargs=parameters["__dm_node_handle_kwargs"],
             instance_ref_dict=parameters["__dm_instance_ref_dict"],
             step_key=parameters["__dm_step_key"],
             **context_args,
         )
 
-        for key in pipeline_context_args:
-            pipeline_context_args[key] = _seven.json.dumps(pipeline_context_args[key])
+        for key in job_context_args:
+            job_context_args[key] = _seven.json.dumps(job_context_args[key])
 
-        content = INJECTED_BOILERPLATE.format(pipeline_context_args=pipeline_context_args)
+        content = INJECTED_BOILERPLATE.format(job_context_args=job_context_args)
 
         for input_name in parameters["__dm_input_names"]:
             dm_load_input_call = f"__dm_dagstermill._load_input_parameter('{input_name}')"

dagstermill/version.py

@@ -1 +1 @@
-__version__ = "0.19.1"
+__version__ = "0.19.3"

{dagstermill-0.19.1.dist-info → dagstermill-0.19.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagstermill
-Version: 0.19.1
+Version: 0.19.3
 Summary: run notebooks using the Dagster tools
 Author: Elementl
 Author-email: hello@elementl.com
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 License-File: LICENSE
-Requires-Dist: dagster (==1.3.1)
+Requires-Dist: dagster (==1.3.3)
 Requires-Dist: ipykernel (!=5.4.0,!=5.4.1,>=4.9.0)
 Requires-Dist: ipython-genutils (>=0.2.0)
 Requires-Dist: packaging (>=20.9)

dagstermill-0.19.3.dist-info/RECORD

@@ -0,0 +1,22 @@
+dagstermill/__init__.py,sha256=WZRYSjL1yAM1w5cqmC1t2T7hvuW-RGGY5tV9vZGHZwU,1133
+dagstermill/__main__.py,sha256=bYt9eEaoRQWdejEHFD8REx9jxVEdZptECFsV7F49Ink,30
+dagstermill/asset_factory.py,sha256=Qz2vcCjGLUE0IyUn9ceGSTV9f_eTojWhm89jMjxYg8U,9134
+dagstermill/cli.py,sha256=NluBLUhAcf47DaQ7o1W9IhF9Ps8HCUc6v5xMYqO_BBk,4569
+dagstermill/compat.py,sha256=GCfUaGC3eIEhlZP_VFtua8hFKNtwXocOAfYxuKZ5X3I,526
+dagstermill/context.py,sha256=uyyfwMxxuataxLlwBS3SV1AtC5Y8_-TwC45GQ17oxok,5965
+dagstermill/engine.py,sha256=4CZW-eni4TnG8C8VhPkZZgj6djItdOUKhHPzvaVhaYo,5843
+dagstermill/errors.py,sha256=WOmpAGp-J1XhyGK_LT3ZZKsBwF5dvrWbqSaSldtoh6Y,141
+dagstermill/factory.py,sha256=Hl-X6AiQ3RXRF7piW5atWxq7AwM2zD3MUmIz5EohTiE,18747
+dagstermill/io_managers.py,sha256=kk36zAvxnnDot6he9CBTm8-VTDsbx9v1MUY6BaZeeKQ,4258
+dagstermill/manager.py,sha256=_XrERGVBDfgKEOT7GeOqRm4kdSYkFe3by6pPTVarNr4,15497
+dagstermill/serialize.py,sha256=eXW3c26CiILT_uebyFcAKBnsiNxnjyGT_ch3PfGyjek,188
+dagstermill/translator.py,sha256=h1VPAOWtdjLKzFjRmyN9hO_R6qJuAETNkJydfdgwWGM,2170
+dagstermill/version.py,sha256=LpBqSbK1KuDbSUey7PHJzdDwxD-diUSlq4z4kUt_6cU,23
+dagstermill/examples/__init__.py,sha256=kzan-9zFjxaJ8o9bqUso44gcGiOmJrlq4JYO-yIBQao,55
+dagstermill/examples/repository.py,sha256=OY0x-4nJRBKxOsS1hrFFhx9cRX0NQ0YdYvXgg5n9Dco,15725
+dagstermill-0.19.3.dist-info/LICENSE,sha256=-gtoVIAZYUHYmNHISZg982FI4Oh19mV1nxgTVW8eCB8,11344
+dagstermill-0.19.3.dist-info/METADATA,sha256=uS9-dMCqtniDBVR8BbcwMCTImZQP_1ST1M7rD3z-F54,1017
+dagstermill-0.19.3.dist-info/WHEEL,sha256=p46_5Uhzqz6AzeSosiOnxK-zmFja1i22CrQCjmYe8ec,92
+dagstermill-0.19.3.dist-info/entry_points.txt,sha256=885a7vvhABYWEj7W28elkzSmIcKO3REkdd5h4Z4DEJs,53
+dagstermill-0.19.3.dist-info/top_level.txt,sha256=YDelJKdA5YIIrjsObdd8U4E9YhuXJLRe9NKfUzud9Uc,12
+dagstermill-0.19.3.dist-info/RECORD,,

dagstermill-0.19.1.dist-info/RECORD

@@ -1,22 +0,0 @@
-dagstermill/__init__.py,sha256=3xNVyJruilLSc6JYrf0W5v5KF_tzUCDQBpYf1nel378,1044
-dagstermill/__main__.py,sha256=bYt9eEaoRQWdejEHFD8REx9jxVEdZptECFsV7F49Ink,30
-dagstermill/asset_factory.py,sha256=4cVAD1DybnMiRi-O6VgRvgP7JYcJnJo-Gs7zmA8JyH8,8833
-dagstermill/cli.py,sha256=NluBLUhAcf47DaQ7o1W9IhF9Ps8HCUc6v5xMYqO_BBk,4569
-dagstermill/compat.py,sha256=GCfUaGC3eIEhlZP_VFtua8hFKNtwXocOAfYxuKZ5X3I,526
-dagstermill/context.py,sha256=By48t6GoPkK2_bVoBCjfhoCR2WVtqSRGYazCoqLdgn8,7307
-dagstermill/engine.py,sha256=4CZW-eni4TnG8C8VhPkZZgj6djItdOUKhHPzvaVhaYo,5843
-dagstermill/errors.py,sha256=WOmpAGp-J1XhyGK_LT3ZZKsBwF5dvrWbqSaSldtoh6Y,141
-dagstermill/factory.py,sha256=tF5bE097lkPINa9OqHEAA92JPiOQULWKM4UVsA2_JKA,18472
-dagstermill/io_managers.py,sha256=OGUjSH--pN8qsTDBgGPLWdeamXwEZOjfNX9rQfNsQEs,3533
-dagstermill/manager.py,sha256=A7lN22c_Yx8HQOFqobL4BbdaXn2EqpiIaCGMHEhnGZE,16960
-dagstermill/serialize.py,sha256=eXW3c26CiILT_uebyFcAKBnsiNxnjyGT_ch3PfGyjek,188
-dagstermill/translator.py,sha256=LpnD_4-4SgRAsHPh7REywkiXRr_yvoszcECk_d_JuUQ,2215
-dagstermill/version.py,sha256=ACwqBctK6p_9Oj2rXWiSmVOPJURrCCh7l_Ak1Mk2h-o,23
-dagstermill/examples/__init__.py,sha256=kzan-9zFjxaJ8o9bqUso44gcGiOmJrlq4JYO-yIBQao,55
-dagstermill/examples/repository.py,sha256=V88N5HjmFSLOWU_jN70hvrYfoMAhVZAS36z17za4Ugk,14797
-dagstermill-0.19.1.dist-info/LICENSE,sha256=-gtoVIAZYUHYmNHISZg982FI4Oh19mV1nxgTVW8eCB8,11344
-dagstermill-0.19.1.dist-info/METADATA,sha256=U22NWvMuNmhkK0A4J1EvGYcZ5A4TYWWgbMkZNsMZeqI,1017
-dagstermill-0.19.1.dist-info/WHEEL,sha256=p46_5Uhzqz6AzeSosiOnxK-zmFja1i22CrQCjmYe8ec,92
-dagstermill-0.19.1.dist-info/entry_points.txt,sha256=885a7vvhABYWEj7W28elkzSmIcKO3REkdd5h4Z4DEJs,53
-dagstermill-0.19.1.dist-info/top_level.txt,sha256=YDelJKdA5YIIrjsObdd8U4E9YhuXJLRe9NKfUzud9Uc,12
-dagstermill-0.19.1.dist-info/RECORD,,