dagstermill 0.17.19__py3-none-any.whl → 0.17.21__py3-none-any.whl

dagstermill/__init__.py

@@ -1,4 +1,4 @@
-from dagster._core.utils import check_dagster_package_version
+from dagster._core.libraries import DagsterLibraryRegistry
 
 from .asset_factory import define_dagstermill_asset as define_dagstermill_asset
 from .context import DagstermillExecutionContext as DagstermillExecutionContext
@@ -8,7 +8,7 @@ from .io_managers import local_output_notebook_io_manager as local_output_notebo
 from .manager import MANAGER_FOR_NOTEBOOK_INSTANCE as _MANAGER_FOR_NOTEBOOK_INSTANCE
 from .version import __version__ as __version__
 
-check_dagster_package_version("dagstermill", __version__)
+DagsterLibraryRegistry.register("dagstermill", __version__)
 
 get_context = _MANAGER_FOR_NOTEBOOK_INSTANCE.get_context
 

dagstermill/context.py

@@ -27,9 +27,9 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
         pipeline_context: PlanExecutionContext,
         pipeline_def: PipelineDefinition,
         resource_keys_to_init: AbstractSet[str],
-        solid_name: str,
-        solid_handle: NodeHandle,
-        solid_config: Any = None,
+        op_name: str,
+        node_handle: NodeHandle,
+        op_config: Any = None,
     ):
         self._pipeline_context = check.inst_param(
             pipeline_context, "pipeline_context", PlanExecutionContext
@@ -38,9 +38,9 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
         self._resource_keys_to_init = check.set_param(
             resource_keys_to_init, "resource_keys_to_init", of_type=str
         )
-        self.solid_name = check.str_param(solid_name, "solid_name")
-        self.solid_handle = check.inst_param(solid_handle, "solid_handle", NodeHandle)
-        self._solid_config = solid_config
+        self.op_name = check.str_param(op_name, "op_name")
+        self.node_handle = check.inst_param(node_handle, "node_handle", NodeHandle)
+        self._op_config = op_config
 
     def has_tag(self, key: str) -> bool:
         """Check if a logging tag is defined on the context.
@@ -145,7 +145,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
     @property
     def run(self) -> DagsterRun:
         """:class:`dagster.DagsterRun`: The job run for the context."""
-        return cast(DagsterRun, self._pipeline_context.pipeline_run)
+        return cast(DagsterRun, self._pipeline_context.dagster_run)
 
     @property
     def pipeline_run(self) -> DagsterRun:
@@ -172,35 +172,21 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
         In interactive contexts, this may be a dagstermill-specific shim, depending whether an
         op definition was passed to ``dagstermill.get_context``.
         """
-        return cast(OpDefinition, self._pipeline_def.solid_def_named(self.solid_name))
+        return cast(OpDefinition, self._pipeline_def.solid_def_named(self.op_name))
 
     @property
-    def solid_def(self) -> OpDefinition:
-        """:class:`dagster.SolidDefinition`: The solid definition for the context.
+    def node(self) -> Node:
+        """:class:`dagster.Node`: The node for the context.
 
-        In interactive contexts, this may be a dagstermill-specific shim, depending whether a
-        solid definition was passed to ``dagstermill.get_context``.
-        """
-        deprecation_warning(
-            "DagstermillExecutionContext.solid_def",
-            "0.17.0",
-            "use the 'op_def' property instead.",
-        )
-        return cast(OpDefinition, self._pipeline_def.solid_def_named(self.solid_name))
-
-    @property
-    def solid(self) -> Node:
-        """:class:`dagster.Node`: The solid for the context.
-
-        In interactive contexts, this may be a dagstermill-specific shim, depending whether a
-        solid definition was passed to ``dagstermill.get_context``.
+        In interactive contexts, this may be a dagstermill-specific shim, depending whether an
+        op definition was passed to ``dagstermill.get_context``.
         """
         deprecation_warning(
             "DagstermillExecutionContext.solid_def",
             "0.17.0",
             "use the 'op_def' property instead.",
         )
-        return self.pipeline_def.get_solid(self.solid_handle)
+        return self.pipeline_def.get_solid(self.node_handle)
 
     @public
     @property
@@ -208,23 +194,11 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
         """collections.namedtuple: A dynamically-created type whose properties allow access to
         op-specific config.
         """
-        if self._solid_config:
-            return self._solid_config
-
-        solid_config = self.resolved_run_config.solids.get(self.solid_name)
-        return solid_config.config if solid_config else None
+        if self._op_config:
+            return self._op_config
 
-    @property
-    def solid_config(self) -> Any:
-        """collections.namedtuple: A dynamically-created type whose properties allow access to
-        solid-specific config.
-        """
-        deprecation_warning(
-            "DagstermillExecutionContext.solid_config",
-            "0.17.0",
-            "use the 'op_config' property instead.",
-        )
-        return self.op_config
+        op_config = self.resolved_run_config.ops.get(self.op_name)
+        return op_config.config if op_config else None
 
 
 class DagstermillRuntimeExecutionContext(DagstermillExecutionContext):
@@ -233,19 +207,19 @@ class DagstermillRuntimeExecutionContext(DagstermillExecutionContext):
         pipeline_context: PlanExecutionContext,
         pipeline_def: PipelineDefinition,
         resource_keys_to_init: AbstractSet[str],
-        solid_name: str,
+        op_name: str,
         step_context: StepExecutionContext,
-        solid_handle: NodeHandle,
-        solid_config: Any = None,
+        node_handle: NodeHandle,
+        op_config: Any = None,
     ):
         self._step_context = check.inst_param(step_context, "step_context", StepExecutionContext)
         super().__init__(
             pipeline_context,
             pipeline_def,
             resource_keys_to_init,
-            solid_name,
-            solid_handle,
-            solid_config,
+            op_name,
+            node_handle,
+            op_config,
         )
 
     @property

dagstermill/factory.py

@@ -141,15 +141,15 @@ def get_papermill_parameters(
         "run_config": step_context.run_config,
     }
 
-    dm_solid_handle_kwargs = step_context.solid_handle._asdict()
+    dm_node_handle_kwargs = step_context.node_handle._asdict()
     dm_step_key = step_context.step.key
 
     parameters = {}
 
     parameters["__dm_context"] = dm_context_dict
     parameters["__dm_executable_dict"] = dm_executable_dict
-    parameters["__dm_pipeline_run_dict"] = pack_value(step_context.pipeline_run)
-    parameters["__dm_solid_handle_kwargs"] = dm_solid_handle_kwargs
+    parameters["__dm_pipeline_run_dict"] = pack_value(step_context.dagster_run)
+    parameters["__dm_node_handle_kwargs"] = dm_node_handle_kwargs
     parameters["__dm_instance_ref_dict"] = pack_value(step_context.instance.get_ref())
     parameters["__dm_step_key"] = dm_step_key
     parameters["__dm_input_names"] = list(inputs.keys())
@@ -235,7 +235,7 @@ def _handle_events_from_notebook(
 
     output_nb = scrapbook.read_notebook(executed_notebook_path)
 
-    for output_name, _ in step_context.solid_def.output_dict.items():
+    for output_name, _ in step_context.op_def.output_dict.items():
         data_dict = output_nb.scraps.data_dict
         if output_name in data_dict:
             # read outputs that were passed out of process via io manager from `yield_result`

dagstermill/manager.py

@@ -32,6 +32,7 @@ from dagster._core.execution.resources_init import (
     resource_initialization_event_generator,
 )
 from dagster._core.instance import DagsterInstance
+from dagster._core.instance.ref import InstanceRef
 from dagster._core.log_manager import DagsterLogManager
 from dagster._core.storage.pipeline_run import DagsterRun, DagsterRunStatus
 from dagster._core.system_config.objects import ResolvedRunConfig, ResourceConfig
@@ -39,8 +40,8 @@ from dagster._core.utils import make_new_run_id
 from dagster._legacy import Materialization, ModeDefinition, PipelineDefinition
 from dagster._loggers import colored_console_logger
 from dagster._serdes import unpack_value
-from dagster._utils import EventGenerationManager, ensure_gen
-from dagster._utils.backcompat import canonicalize_backcompat_args, deprecation_warning
+from dagster._utils import EventGenerationManager
+from dagster._utils.backcompat import deprecation_warning
 
 from .context import DagstermillExecutionContext, DagstermillRuntimeExecutionContext
 from .errors import DagstermillError
@@ -68,7 +69,7 @@ class DagstermillResourceEventGenerationManager(EventGenerationManager):
 class Manager:
     def __init__(self):
         self.pipeline = None
-        self.solid_def: Optional[NodeDefinition] = None
+        self.op_def: Optional[NodeDefinition] = None
         self.in_pipeline: bool = False
         self.marshal_dir: Optional[str] = None
         self.context = None
@@ -80,7 +81,7 @@ class Manager:
         resource_configs: Mapping[str, ResourceConfig],
         log_manager: DagsterLogManager,
         execution_plan: Optional[ExecutionPlan],
-        pipeline_run: Optional[DagsterRun],
+        dagster_run: Optional[DagsterRun],
         resource_keys_to_init: Optional[AbstractSet[str]],
         instance: Optional[DagsterInstance],
         emit_persistent_events: Optional[bool],
@@ -95,7 +96,7 @@ class Manager:
             resource_configs=resource_configs,
             log_manager=log_manager,
             execution_plan=execution_plan,
-            pipeline_run=pipeline_run,
+            dagster_run=dagster_run,
             resource_keys_to_init=resource_keys_to_init,
             instance=instance,
             emit_persistent_events=emit_persistent_events,
@@ -108,8 +109,8 @@ class Manager:
     def reconstitute_pipeline_context(
         self,
         executable_dict: Mapping[str, Any],
-        pipeline_run_dict: Mapping[str, DagsterRun],
-        solid_handle_kwargs: Mapping[str, Any],
+        pipeline_run_dict: Mapping[str, Any],
+        node_handle_kwargs: Mapping[str, Any],
         instance_ref_dict: Mapping[str, Any],
         step_key: str,
         output_log_path: Optional[str] = None,
@@ -132,7 +133,7 @@ class Manager:
         run_config = check.opt_mapping_param(run_config, "run_config", key_type=str)
         check.mapping_param(pipeline_run_dict, "pipeline_run_dict")
         check.mapping_param(executable_dict, "executable_dict")
-        check.mapping_param(solid_handle_kwargs, "solid_handle_kwargs")
+        check.mapping_param(node_handle_kwargs, "node_handle_kwargs")
         check.mapping_param(instance_ref_dict, "instance_ref_dict")
         check.str_param(step_key, "step_key")
 
@@ -140,22 +141,22 @@ class Manager:
         pipeline_def = pipeline.get_definition()
 
         try:
-            instance_ref = unpack_value(instance_ref_dict)
+            instance_ref = cast(InstanceRef, unpack_value(instance_ref_dict))
             instance = DagsterInstance.from_ref(instance_ref)
         except Exception as err:
             raise DagstermillError(
                 "Error when attempting to resolve DagsterInstance from serialized InstanceRef"
             ) from err
 
-        pipeline_run = unpack_value(pipeline_run_dict)
+        pipeline_run = cast(DagsterRun, unpack_value(pipeline_run_dict))
 
-        solid_handle = NodeHandle.from_dict(solid_handle_kwargs)
-        solid = pipeline_def.get_solid(solid_handle)
-        solid_def = solid.definition
+        node_handle = NodeHandle.from_dict(node_handle_kwargs)
+        op = pipeline_def.get_solid(node_handle)
+        op_def = op.definition
 
         self.marshal_dir = marshal_dir
         self.in_pipeline = True
-        self.solid_def = solid_def
+        self.op_def = op_def
         self.pipeline = pipeline
 
         resolved_run_config = ResolvedRunConfig.build(
@@ -181,14 +182,14 @@ class Manager:
             self.context = DagstermillRuntimeExecutionContext(
                 pipeline_context=pipeline_context,
                 pipeline_def=pipeline_def,
-                solid_config=run_config.get("ops", {}).get(solid.name, {}).get("config"),
+                op_config=run_config.get("ops", {}).get(op.name, {}).get("config"),
                 resource_keys_to_init=get_required_resource_keys_to_init(
                     execution_plan,
                     pipeline_def,
                     resolved_run_config,
                 ),
-                solid_name=solid.name,
-                solid_handle=solid_handle,
+                op_name=op.name,
+                node_handle=node_handle,
                 step_context=cast(
                     StepExecutionContext,
                     pipeline_context.for_step(
@@ -204,7 +205,6 @@ class Manager:
         op_config: Any = None,
         resource_defs: Optional[Mapping[str, ResourceDefinition]] = None,
         logger_defs: Optional[Mapping[str, LoggerDefinition]] = None,
-        solid_config: Any = None,
         mode_def: Optional[ModeDefinition] = None,
         run_config: Optional[dict] = None,
     ) -> DagstermillExecutionContext:
@@ -236,10 +236,6 @@ class Manager:
                 " `dagstermill.get_context`. Please provide one or the other."
             )
 
-        solid_config = canonicalize_backcompat_args(
-            op_config, "op_config", solid_config, "solid_config", "0.17.0"
-        )
-
         if mode_def:
             deprecation_warning(
                 "mode_def argument to dagstermill.get_context",
@@ -265,15 +261,15 @@ class Manager:
             resource_defs = check.opt_mapping_param(resource_defs, "resource_defs")
             mode_def = ModeDefinition(logger_defs=logger_defs, resource_defs=resource_defs)
 
-        solid_def = OpDefinition(
-            name="this_solid",
+        op_def = OpDefinition(
+            name="this_op",
             compute_fn=lambda *args, **kwargs: None,
-            description="Ephemeral solid constructed by dagstermill.get_context()",
+            description="Ephemeral op constructed by dagstermill.get_context()",
             required_resource_keys=mode_def.resource_key_set,
         )
 
         pipeline_def = PipelineDefinition(
-            [solid_def], mode_defs=[mode_def], name="ephemeral_dagstermill_pipeline"
+            [op_def], mode_defs=[mode_def], name="ephemeral_dagstermill_pipeline"
         )
 
         run_id = make_new_run_id()
@@ -292,7 +288,7 @@ class Manager:
         )
 
         self.in_pipeline = False
-        self.solid_def = solid_def
+        self.op_def = op_def
         self.pipeline = pipeline_def
 
         resolved_run_config = ResolvedRunConfig.build(pipeline_def, run_config, mode=mode_def.name)
@@ -311,14 +307,14 @@ class Manager:
             self.context = DagstermillExecutionContext(
                 pipeline_context=pipeline_context,
                 pipeline_def=pipeline_def,
-                solid_config=solid_config,
+                op_config=op_config,
                 resource_keys_to_init=get_required_resource_keys_to_init(
                     execution_plan,
                     pipeline_def,
                     resolved_run_config,
                 ),
-                solid_name=solid_def.name,
-                solid_handle=NodeHandle(solid_def.name, parent=None),
+                op_name=op_def.name,
+                node_handle=NodeHandle(op_def.name, parent=None),
             )
 
         return self.context
@@ -338,10 +334,10 @@ class Manager:
         # deferred import for perf
         import scrapbook
 
-        if not self.solid_def.has_output(output_name):
+        if not self.op_def.has_output(output_name):
             raise DagstermillError(
-                f"Op {self.solid_def.name} does not have output named {output_name}.Expected one of"
-                f" {[str(output_def.name) for output_def in self.solid_def.output_defs]}"
+                f"Op {self.op_def.name} does not have output named {output_name}.Expected one of"
+                f" {[str(output_def.name) for output_def in self.op_def.output_defs]}"
             )
 
         # pass output value cross process boundary using io manager
@@ -414,10 +410,8 @@ class Manager:
             check.failed("Expected DagstermillRuntimeExecutionContext")
         step_context = dm_context.step_context  # pylint: disable=protected-access
         step_input = step_context.step.step_input_named(input_name)
-        input_def = step_context.solid_def.input_def_named(input_name)
-        for event_or_input_value in ensure_gen(
-            step_input.source.load_input_object(step_context, input_def)
-        ):
+        input_def = step_context.op_def.input_def_named(input_name)
+        for event_or_input_value in step_input.source.load_input_object(step_context, input_def):
             if isinstance(event_or_input_value, DagsterEvent):
                 continue
             else:

dagstermill/translator.py

@@ -8,7 +8,7 @@ RESERVED_INPUT_NAMES = [
     "__dm_executable_dict",
     "__dm_json",
     "__dm_pipeline_run_dict",
-    "__dm_solid_handle_kwargs",
+    "__dm_node_handle_kwargs",
     "__dm_instance_ref_dict",
     "__dm_step_key",
     "__dm_input_names",
@@ -35,7 +35,7 @@ class DagsterTranslator(papermill.translators.PythonTranslator):
         assert "__dm_context" in parameters
         assert "__dm_executable_dict" in parameters
         assert "__dm_pipeline_run_dict" in parameters
-        assert "__dm_solid_handle_kwargs" in parameters
+        assert "__dm_node_handle_kwargs" in parameters
         assert "__dm_instance_ref_dict" in parameters
         assert "__dm_step_key" in parameters
         assert "__dm_input_names" in parameters
@@ -44,7 +44,7 @@ class DagsterTranslator(papermill.translators.PythonTranslator):
         pipeline_context_args = dict(
             executable_dict=parameters["__dm_executable_dict"],
             pipeline_run_dict=parameters["__dm_pipeline_run_dict"],
-            solid_handle_kwargs=parameters["__dm_solid_handle_kwargs"],
+            node_handle_kwargs=parameters["__dm_node_handle_kwargs"],
             instance_ref_dict=parameters["__dm_instance_ref_dict"],
             step_key=parameters["__dm_step_key"],
             **context_args,

dagstermill/version.py

@@ -1 +1 @@
-__version__ = "0.17.19"
+__version__ = "0.17.21"

{dagstermill-0.17.19.dist-info → dagstermill-0.17.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagstermill
-Version: 0.17.19
+Version: 0.17.21
 Summary: run notebooks using the Dagster tools
 Home-page: UNKNOWN
 Author: Elementl
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 License-File: LICENSE
-Requires-Dist: dagster (==1.1.19)
+Requires-Dist: dagster (==1.1.21)
 Requires-Dist: ipykernel (!=5.4.0,!=5.4.1,>=4.9.0)
 Requires-Dist: ipython-genutils (>=0.2.0)
 Requires-Dist: packaging (>=20.9)

{dagstermill-0.17.19.dist-info → dagstermill-0.17.21.dist-info}/RECORD

@@ -1,22 +1,22 @@
-dagstermill/__init__.py,sha256=h-l8uokGSDjw1gG6LGX5bngZXtT7mdm_cOaOgPPXwIo,1045
+dagstermill/__init__.py,sha256=3xNVyJruilLSc6JYrf0W5v5KF_tzUCDQBpYf1nel378,1044
 dagstermill/__main__.py,sha256=bYt9eEaoRQWdejEHFD8REx9jxVEdZptECFsV7F49Ink,30
 dagstermill/asset_factory.py,sha256=4cVAD1DybnMiRi-O6VgRvgP7JYcJnJo-Gs7zmA8JyH8,8833
 dagstermill/cli.py,sha256=AcrZS9NAeZ1OScmjrFBvXNdrTHXhEfjMjLPU9pO442I,4656
 dagstermill/compat.py,sha256=m2Sbq1kr0yfr1WJuKFF5YMAcZTIk_aD5DJivwZjJQvU,558
-dagstermill/context.py,sha256=sLvwY3NYsWRf-OwdLN9a9FLgfdUSaJM_ddTtebLVR_k,8331
+dagstermill/context.py,sha256=lc01TEufiSv9yQFXlXMSuhiR1KvsiMe3xUgLxIV2ZuM,7309
 dagstermill/engine.py,sha256=0VJfMpi8tXmt8oljvOXAzHYP2xpPTbOxxFuvAS_AwLo,5928
 dagstermill/errors.py,sha256=WOmpAGp-J1XhyGK_LT3ZZKsBwF5dvrWbqSaSldtoh6Y,141
-dagstermill/factory.py,sha256=Qi7W-bau5T42pe25pZ-32WBdF82HIUkFyTcpyZCKQ0Q,18480
+dagstermill/factory.py,sha256=tF5bE097lkPINa9OqHEAA92JPiOQULWKM4UVsA2_JKA,18472
 dagstermill/io_managers.py,sha256=OGUjSH--pN8qsTDBgGPLWdeamXwEZOjfNX9rQfNsQEs,3533
-dagstermill/manager.py,sha256=o4wLxOW_be-LfgbedT_wKItXEGrPAvxmBlBVl6_sKWI,17372
+dagstermill/manager.py,sha256=A7wPb_75ayExpsN8HV396Zxce5tyiinSqU7CprsXFMQ,17117
 dagstermill/serialize.py,sha256=eXW3c26CiILT_uebyFcAKBnsiNxnjyGT_ch3PfGyjek,188
-dagstermill/translator.py,sha256=ej2an-UwRBzVCN4aKTQ3-b3FEvNMsUTc4OHIytrIloA,2227
-dagstermill/version.py,sha256=LKtJnPhS5GRuhOh9obHGnDlncpF9oAEamdbIKdiL3Tw,24
+dagstermill/translator.py,sha256=go_5ocu575epGPmGJcHqAqH7N7-aSdtiaMHw5T9CXvs,2223
+dagstermill/version.py,sha256=02pERK8eGbKCmhhaP6R4kszX1AZhworPgk9xCzHc0Ak,24
 dagstermill/examples/__init__.py,sha256=kzan-9zFjxaJ8o9bqUso44gcGiOmJrlq4JYO-yIBQao,55
 dagstermill/examples/repository.py,sha256=LoQSv5TaOYwlz9RAWtfChm6ELQKhMIrvb3owvG-xRyY,14808
-dagstermill-0.17.19.dist-info/LICENSE,sha256=-gtoVIAZYUHYmNHISZg982FI4Oh19mV1nxgTVW8eCB8,11344
-dagstermill-0.17.19.dist-info/METADATA,sha256=PLYLGCkC4YkNPiJUIzsaP17HgF1kRfKrdP6sBch1jiE,1014
-dagstermill-0.17.19.dist-info/WHEEL,sha256=p46_5Uhzqz6AzeSosiOnxK-zmFja1i22CrQCjmYe8ec,92
-dagstermill-0.17.19.dist-info/entry_points.txt,sha256=BAnkw0uZ7RK7YSz8L0Vz_2lO26XSs2sdtIXsL6wjmSk,54
-dagstermill-0.17.19.dist-info/top_level.txt,sha256=YDelJKdA5YIIrjsObdd8U4E9YhuXJLRe9NKfUzud9Uc,12
-dagstermill-0.17.19.dist-info/RECORD,,
+dagstermill-0.17.21.dist-info/LICENSE,sha256=-gtoVIAZYUHYmNHISZg982FI4Oh19mV1nxgTVW8eCB8,11344
+dagstermill-0.17.21.dist-info/METADATA,sha256=PAYnhVGoR85NNtxMv5kSVNXBGqywzp3OM0ZHwnYyoMY,1014
+dagstermill-0.17.21.dist-info/WHEEL,sha256=p46_5Uhzqz6AzeSosiOnxK-zmFja1i22CrQCjmYe8ec,92
+dagstermill-0.17.21.dist-info/entry_points.txt,sha256=BAnkw0uZ7RK7YSz8L0Vz_2lO26XSs2sdtIXsL6wjmSk,54
+dagstermill-0.17.21.dist-info/top_level.txt,sha256=YDelJKdA5YIIrjsObdd8U4E9YhuXJLRe9NKfUzud9Uc,12
+dagstermill-0.17.21.dist-info/RECORD,,