dagstermill 0.19.2__tar.gz → 0.19.3__tar.gz

{dagstermill-0.19.2/dagstermill.egg-info → dagstermill-0.19.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagstermill
-Version: 0.19.2
+Version: 0.19.3
 Summary: run notebooks using the Dagster tools
 Author: Elementl
 Author-email: hello@elementl.com

{dagstermill-0.19.2 → dagstermill-0.19.3}/dagstermill/__init__.py

@@ -19,7 +19,7 @@ yield_result = _MANAGER_FOR_NOTEBOOK_INSTANCE.yield_result
 
 yield_event = _MANAGER_FOR_NOTEBOOK_INSTANCE.yield_event
 
-_reconstitute_pipeline_context = _MANAGER_FOR_NOTEBOOK_INSTANCE.reconstitute_pipeline_context
+_reconstitute_job_context = _MANAGER_FOR_NOTEBOOK_INSTANCE.reconstitute_job_context
 
 _teardown = _MANAGER_FOR_NOTEBOOK_INSTANCE.teardown_resources
 

{dagstermill-0.19.2 → dagstermill-0.19.3}/dagstermill/context.py

@@ -23,17 +23,15 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
 
     def __init__(
         self,
-        pipeline_context: PlanExecutionContext,
-        pipeline_def: JobDefinition,
+        job_context: PlanExecutionContext,
+        job_def: JobDefinition,
         resource_keys_to_init: AbstractSet[str],
         op_name: str,
         node_handle: NodeHandle,
         op_config: Any = None,
     ):
-        self._pipeline_context = check.inst_param(
-            pipeline_context, "pipeline_context", PlanExecutionContext
-        )
-        self._pipeline_def = check.inst_param(pipeline_def, "pipeline_def", JobDefinition)
+        self._job_context = check.inst_param(job_context, "job_context", PlanExecutionContext)
+        self._job_def = check.inst_param(job_def, "job_def", JobDefinition)
         self._resource_keys_to_init = check.set_param(
             resource_keys_to_init, "resource_keys_to_init", of_type=str
         )
@@ -51,7 +49,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
             bool
         """
         check.str_param(key, "key")
-        return self._pipeline_context.has_tag(key)
+        return self._job_context.has_tag(key)
 
     def get_tag(self, key: str) -> Optional[str]:
         """Get a logging tag defined on the context.
@@ -63,44 +61,35 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
             str
         """
         check.str_param(key, "key")
-        return self._pipeline_context.get_tag(key)
+        return self._job_context.get_tag(key)
 
     @public
     @property
     def run_id(self) -> str:
         """str: The run_id for the context."""
-        return self._pipeline_context.run_id
+        return self._job_context.run_id
 
     @public
     @property
     def run_config(self) -> Mapping[str, Any]:
         """dict: The run_config for the context."""
-        return self._pipeline_context.run_config
+        return self._job_context.run_config
 
     @property
     def resolved_run_config(self) -> ResolvedRunConfig:
         """:class:`dagster.ResolvedRunConfig`: The resolved_run_config for the context."""
-        return self._pipeline_context.resolved_run_config
+        return self._job_context.resolved_run_config
 
     @public
     @property
     def logging_tags(self) -> Mapping[str, str]:
         """dict: The logging tags for the context."""
-        return self._pipeline_context.logging_tags
+        return self._job_context.logging_tags
 
     @public
     @property
     def job_name(self) -> str:
-        return self._pipeline_context.job_name
-
-    @property
-    def pipeline_name(self) -> str:
-        deprecation_warning(
-            "DagstermillExecutionContext.pipeline_name",
-            "0.17.0",
-            "use the 'job_name' property instead.",
-        )
-        return self.job_name
+        return self._job_context.job_name
 
     @public
     @property
@@ -109,34 +98,14 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
 
         This will be a dagstermill-specific shim.
         """
-        return cast(
-            JobDefinition,
-            check.inst(
-                self._pipeline_def,
-                JobDefinition,
-                "Accessing job_def inside a legacy pipeline. Use pipeline_def instead.",
-            ),
-        )
-
-    @property
-    def pipeline_def(self) -> JobDefinition:
-        """:class:`dagster.PipelineDefinition`: The pipeline definition for the context.
-
-        This will be a dagstermill-specific shim.
-        """
-        deprecation_warning(
-            "DagstermillExecutionContext.pipeline_def",
-            "0.17.0",
-            "use the 'job_def' property instead.",
-        )
-        return self._pipeline_def
+        return self._job_def
 
     @property
     def resources(self) -> Any:
         """collections.namedtuple: A dynamically-created type whose properties allow access to
         resources.
         """
-        return self._pipeline_context.scoped_resources_builder.build(
+        return self._job_context.scoped_resources_builder.build(
             required_resource_keys=self._resource_keys_to_init,
         )
 
@@ -144,16 +113,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
     @property
     def run(self) -> DagsterRun:
         """:class:`dagster.DagsterRun`: The job run for the context."""
-        return cast(DagsterRun, self._pipeline_context.dagster_run)
-
-    @property
-    def pipeline_run(self) -> DagsterRun:
-        deprecation_warning(
-            "DagstermillExecutionContext.pipeline_run",
-            "0.17.0",
-            "use the 'run' property instead.",
-        )
-        return self.run
+        return cast(DagsterRun, self._job_context.dagster_run)
 
     @property
     def log(self) -> DagsterLogManager:
@@ -161,7 +121,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
 
         Call, e.g., ``log.info()`` to log messages through the Dagster machinery.
         """
-        return self._pipeline_context.log
+        return self._job_context.log
 
     @public
     @property
@@ -171,7 +131,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
         In interactive contexts, this may be a dagstermill-specific shim, depending whether an
         op definition was passed to ``dagstermill.get_context``.
         """
-        return cast(OpDefinition, self._pipeline_def.node_def_named(self.op_name))
+        return cast(OpDefinition, self._job_def.node_def_named(self.op_name))
 
     @property
     def node(self) -> Node:
@@ -185,7 +145,7 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
             "0.17.0",
             "use the 'op_def' property instead.",
         )
-        return self.pipeline_def.get_node(self.node_handle)
+        return self.job_def.get_node(self.node_handle)
 
     @public
     @property
@@ -203,8 +163,8 @@ class DagstermillExecutionContext(AbstractComputeExecutionContext):
 class DagstermillRuntimeExecutionContext(DagstermillExecutionContext):
     def __init__(
         self,
-        pipeline_context: PlanExecutionContext,
-        pipeline_def: JobDefinition,
+        job_context: PlanExecutionContext,
+        job_def: JobDefinition,
         resource_keys_to_init: AbstractSet[str],
         op_name: str,
         step_context: StepExecutionContext,
@@ -213,8 +173,8 @@ class DagstermillRuntimeExecutionContext(DagstermillExecutionContext):
     ):
         self._step_context = check.inst_param(step_context, "step_context", StepExecutionContext)
         super().__init__(
-            pipeline_context,
-            pipeline_def,
+            job_context,
+            job_def,
             resource_keys_to_init,
             op_name,
             node_handle,

{dagstermill-0.19.2 → dagstermill-0.19.3}/dagstermill/examples/repository.py

@@ -574,8 +574,8 @@ yield_event_asset = dagstermill.define_dagstermill_asset(
 )
 
 
-# this is hacky. We need a ReconstructablePipeline to run dagstermill, and
-# ReconstructablePipeline.for_module() find the jobs defined in this file. So we need to resolve all
+# this is hacky. We need a ReconstructableJob to run dagstermill, and
+# ReconstructableJob.for_module() find the jobs defined in this file. So we need to resolve all
 # of the asset jobs outside of the repository function.
 assets = with_resources(
     [

{dagstermill-0.19.2 → dagstermill-0.19.3}/dagstermill/factory.py

@@ -20,7 +20,7 @@ from dagster._config.pythonic_config import Config, infer_schema_from_config_cla
 from dagster._config.pythonic_config.utils import safe_is_subclass
 from dagster._core.definitions.events import AssetMaterialization, Failure, RetryRequested
 from dagster._core.definitions.metadata import MetadataValue
-from dagster._core.definitions.reconstruct import ReconstructablePipeline
+from dagster._core.definitions.reconstruct import ReconstructableJob
 from dagster._core.definitions.utils import validate_tags
 from dagster._core.execution.context.compute import OpExecutionContext
 from dagster._core.execution.context.input import build_input_context
@@ -123,7 +123,7 @@ def get_papermill_parameters(
     marshal_dir = os.path.normpath(os.path.join(temp_dir, "dagstermill", str(run_id), "marshal"))
     mkdir_p(marshal_dir)
 
-    if not isinstance(step_context.pipeline, ReconstructablePipeline):
+    if not isinstance(step_context.job, ReconstructableJob):
         if compute_descriptor == "asset":
             raise DagstermillError(
                 "Can't execute a dagstermill asset that is not reconstructable. "
@@ -135,7 +135,7 @@ def get_papermill_parameters(
                 "Use the reconstructable() function if executing from python"
             )
 
-    dm_executable_dict = step_context.pipeline.to_dict()
+    dm_executable_dict = step_context.job.to_dict()
 
     dm_context_dict = {
         "output_log_path": output_log_path,

{dagstermill-0.19.2 → dagstermill-0.19.3}/dagstermill/io_managers.py

@@ -93,7 +93,7 @@ class ConfigurableLocalOutputNotebookIOManager(ConfigurableIOManagerFactory):
         ),
     )
     asset_key_prefix: List[str] = Field(
-        [],
+        default=[],
         description=(
             "Asset key prefix to apply to assets materialized for output notebooks. Defaults to no"
             " prefix."

{dagstermill-0.19.2 → dagstermill-0.19.3}/dagstermill/manager.py

@@ -17,14 +17,14 @@ from dagster import (
 from dagster._core.definitions.dependency import NodeHandle
 from dagster._core.definitions.events import RetryRequested
 from dagster._core.definitions.graph_definition import GraphDefinition
+from dagster._core.definitions.job_base import InMemoryJob
 from dagster._core.definitions.job_definition import JobDefinition
 from dagster._core.definitions.node_definition import NodeDefinition
 from dagster._core.definitions.op_definition import OpDefinition
-from dagster._core.definitions.pipeline_base import InMemoryPipeline
-from dagster._core.definitions.reconstruct import ReconstructablePipeline
+from dagster._core.definitions.reconstruct import ReconstructableJob
 from dagster._core.definitions.resource_definition import ScopedResourcesBuilder
 from dagster._core.events import DagsterEvent
-from dagster._core.execution.api import scoped_pipeline_context
+from dagster._core.execution.api import scoped_job_context
 from dagster._core.execution.plan.outputs import StepOutputHandle
 from dagster._core.execution.plan.plan import ExecutionPlan
 from dagster._core.execution.plan.step import ExecutionStep
@@ -35,7 +35,7 @@ from dagster._core.execution.resources_init import (
 from dagster._core.instance import DagsterInstance
 from dagster._core.instance.ref import InstanceRef
 from dagster._core.log_manager import DagsterLogManager
-from dagster._core.storage.pipeline_run import DagsterRun, DagsterRunStatus
+from dagster._core.storage.dagster_run import DagsterRun, DagsterRunStatus
 from dagster._core.system_config.objects import ResolvedRunConfig, ResourceConfig
 from dagster._core.utils import make_new_run_id
 from dagster._loggers import colored_console_logger
@@ -67,9 +67,9 @@ class DagstermillResourceEventGenerationManager(EventGenerationManager):
 
 class Manager:
     def __init__(self):
-        self.pipeline = None
+        self.job = None
         self.op_def: Optional[NodeDefinition] = None
-        self.in_pipeline: bool = False
+        self.in_job: bool = False
         self.marshal_dir: Optional[str] = None
         self.context = None
         self.resource_manager = None
@@ -104,10 +104,10 @@ class Manager:
         )
         return self.resource_manager
 
-    def reconstitute_pipeline_context(
+    def reconstitute_job_context(
         self,
         executable_dict: Mapping[str, Any],
-        pipeline_run_dict: Mapping[str, Any],
+        job_run_dict: Mapping[str, Any],
         node_handle_kwargs: Mapping[str, Any],
         instance_ref_dict: Mapping[str, Any],
         step_key: str,
@@ -117,26 +117,26 @@ class Manager:
     ):
         """Reconstitutes a context for dagstermill-managed execution.
 
-        You'll see this function called to reconstruct a pipeline context within the ``injected
+        You'll see this function called to reconstruct a job context within the ``injected
         parameters`` cell of a dagstermill output notebook. Users should not call this function
         interactively except when debugging output notebooks.
 
         Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a
         context for interactive exploration and development. This call will be replaced by one to
-        :func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by
+        :func:`dagstermill.reconstitute_job_context` when the notebook is executed by
         dagstermill.
         """
         check.opt_str_param(output_log_path, "output_log_path")
         check.opt_str_param(marshal_dir, "marshal_dir")
         run_config = check.opt_mapping_param(run_config, "run_config", key_type=str)
-        check.mapping_param(pipeline_run_dict, "pipeline_run_dict")
+        check.mapping_param(job_run_dict, "job_run_dict")
         check.mapping_param(executable_dict, "executable_dict")
         check.mapping_param(node_handle_kwargs, "node_handle_kwargs")
         check.mapping_param(instance_ref_dict, "instance_ref_dict")
         check.str_param(step_key, "step_key")
 
-        pipeline = ReconstructablePipeline.from_dict(executable_dict)
-        pipeline_def = pipeline.get_definition()
+        job = ReconstructableJob.from_dict(executable_dict)
+        job_def = job.get_definition()
 
         try:
             instance_ref = unpack_value(instance_ref_dict, InstanceRef)
@@ -146,49 +146,49 @@ class Manager:
                 "Error when attempting to resolve DagsterInstance from serialized InstanceRef"
             ) from err
 
-        dagster_run = unpack_value(pipeline_run_dict, DagsterRun)
+        dagster_run = unpack_value(job_run_dict, DagsterRun)
 
         node_handle = NodeHandle.from_dict(node_handle_kwargs)
-        op = pipeline_def.get_node(node_handle)
+        op = job_def.get_node(node_handle)
         op_def = op.definition
 
         self.marshal_dir = marshal_dir
-        self.in_pipeline = True
+        self.in_job = True
         self.op_def = op_def
-        self.pipeline = pipeline
+        self.job = job
 
-        resolved_run_config = ResolvedRunConfig.build(pipeline_def, run_config)
+        resolved_run_config = ResolvedRunConfig.build(job_def, run_config)
 
         execution_plan = ExecutionPlan.build(
-            self.pipeline,
+            self.job,
             resolved_run_config,
             step_keys_to_execute=dagster_run.step_keys_to_execute,
         )
 
-        with scoped_pipeline_context(
+        with scoped_job_context(
             execution_plan,
-            pipeline,
+            job,
             run_config,
             dagster_run,
             instance,
             scoped_resources_builder_cm=self._setup_resources,
             # Set this flag even though we're not in test for clearer error reporting
             raise_on_error=True,
-        ) as pipeline_context:
+        ) as job_context:
             self.context = DagstermillRuntimeExecutionContext(
-                pipeline_context=pipeline_context,
-                pipeline_def=pipeline_def,
+                job_context=job_context,
+                job_def=job_def,
                 op_config=run_config.get("ops", {}).get(op.name, {}).get("config"),
                 resource_keys_to_init=get_required_resource_keys_to_init(
                     execution_plan,
-                    pipeline_def,
+                    job_def,
                     resolved_run_config,
                 ),
                 op_name=op.name,
                 node_handle=node_handle,
                 step_context=cast(
                     StepExecutionContext,
-                    pipeline_context.for_step(
+                    job_context.for_step(
                         cast(ExecutionStep, execution_plan.get_step_by_key(step_key))
                     ),
                 ),
@@ -238,7 +238,7 @@ class Manager:
             required_resource_keys=set(resource_defs.keys()),
         )
 
-        pipeline_def = JobDefinition(
+        job_def = JobDefinition(
             graph_def=GraphDefinition(name="ephemeral_dagstermill_pipeline", node_defs=[op_def]),
             logger_defs=logger_defs,
             resource_defs=resource_defs,
@@ -246,11 +246,11 @@ class Manager:
 
         run_id = make_new_run_id()
 
-        # construct stubbed PipelineRun for notebook exploration...
-        # The actual pipeline run during pipeline execution will be serialized and reconstituted
-        # in the `reconstitute_pipeline_context` call
-        pipeline_run = DagsterRun(
-            pipeline_name=pipeline_def.name,
+        # construct stubbed DagsterRun for notebook exploration...
+        # The actual dagster run during job execution will be serialized and reconstituted
+        # in the `reconstitute_job_context` call
+        dagster_run = DagsterRun(
+            job_name=job_def.name,
             run_id=run_id,
             run_config=run_config,
             step_keys_to_execute=None,
@@ -258,30 +258,30 @@ class Manager:
             tags=None,
         )
 
-        self.in_pipeline = False
+        self.in_job = False
         self.op_def = op_def
-        self.pipeline = pipeline_def
+        self.job = job_def
 
-        resolved_run_config = ResolvedRunConfig.build(pipeline_def, run_config)
+        resolved_run_config = ResolvedRunConfig.build(job_def, run_config)
 
-        pipeline = InMemoryPipeline(pipeline_def)
-        execution_plan = ExecutionPlan.build(pipeline, resolved_run_config)
+        job = InMemoryJob(job_def)
+        execution_plan = ExecutionPlan.build(job, resolved_run_config)
 
-        with scoped_pipeline_context(
+        with scoped_job_context(
             execution_plan,
-            pipeline,
+            job,
             run_config,
-            pipeline_run,
+            dagster_run,
             DagsterInstance.ephemeral(),
             scoped_resources_builder_cm=self._setup_resources,
-        ) as pipeline_context:
+        ) as job_context:
             self.context = DagstermillExecutionContext(
-                pipeline_context=pipeline_context,
-                pipeline_def=pipeline_def,
+                job_context=job_context,
+                job_def=job_def,
                 op_config=op_config,
                 resource_keys_to_init=get_required_resource_keys_to_init(
                     execution_plan,
-                    pipeline_def,
+                    job_def,
                     resolved_run_config,
                 ),
                 op_name=op_def.name,
@@ -299,7 +299,7 @@ class Manager:
             value (Any): The value to yield.
             output_name (Optional[str]): The name of the result to yield (default: ``'result'``).
         """
-        if not self.in_pipeline:
+        if not self.in_job:
             return value
 
         # deferred import for perf
@@ -356,7 +356,7 @@ class Manager:
                 f" type, one of {valid_types}."
             )
 
-        if not self.in_pipeline:
+        if not self.in_job:
             return dagster_event
 
         # deferred import for perf

{dagstermill-0.19.2 → dagstermill-0.19.3}/dagstermill/translator.py

@@ -18,11 +18,11 @@ INJECTED_BOILERPLATE = """
 # Injected parameters
 from dagster import seven as __dm_seven
 import dagstermill as __dm_dagstermill
-context = __dm_dagstermill._reconstitute_pipeline_context(
+context = __dm_dagstermill._reconstitute_job_context(
     **{{
         key: __dm_seven.json.loads(value)
         for key, value
-        in {pipeline_context_args}.items()
+        in {job_context_args}.items()
     }}
 )
 """
@@ -41,19 +41,19 @@ class DagsterTranslator(papermill.translators.PythonTranslator):
         assert "__dm_input_names" in parameters
 
         context_args = parameters["__dm_context"]
-        pipeline_context_args = dict(
+        job_context_args = dict(
             executable_dict=parameters["__dm_executable_dict"],
-            pipeline_run_dict=parameters["__dm_pipeline_run_dict"],
+            job_run_dict=parameters["__dm_pipeline_run_dict"],
             node_handle_kwargs=parameters["__dm_node_handle_kwargs"],
             instance_ref_dict=parameters["__dm_instance_ref_dict"],
             step_key=parameters["__dm_step_key"],
             **context_args,
         )
 
-        for key in pipeline_context_args:
-            pipeline_context_args[key] = _seven.json.dumps(pipeline_context_args[key])
+        for key in job_context_args:
+            job_context_args[key] = _seven.json.dumps(job_context_args[key])
 
-        content = INJECTED_BOILERPLATE.format(pipeline_context_args=pipeline_context_args)
+        content = INJECTED_BOILERPLATE.format(job_context_args=job_context_args)
 
         for input_name in parameters["__dm_input_names"]:
             dm_load_input_call = f"__dm_dagstermill._load_input_parameter('{input_name}')"

dagstermill-0.19.3/dagstermill/version.py

@@ -0,0 +1 @@
+__version__ = "0.19.3"

{dagstermill-0.19.2 → dagstermill-0.19.3/dagstermill.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagstermill
-Version: 0.19.2
+Version: 0.19.3
 Summary: run notebooks using the Dagster tools
 Author: Elementl
 Author-email: hello@elementl.com

{dagstermill-0.19.2 → dagstermill-0.19.3}/dagstermill.egg-info/requires.txt

@@ -1,4 +1,4 @@
-dagster==1.3.2
+dagster==1.3.3
 ipykernel!=5.4.0,!=5.4.1,>=4.9.0
 ipython_genutils>=0.2.0
 packaging>=20.9

{dagstermill-0.19.2 → dagstermill-0.19.3}/setup.py

@@ -33,7 +33,7 @@ setup(
         "Operating System :: OS Independent",
     ],
     install_requires=[
-        "dagster==1.3.2",
+        "dagster==1.3.3",
         # ipykernel 5.4.0 and 5.4.1 broke papermill
         # see https://github.com/dagster-io/dagster/issues/3401,
         # https://github.com/nteract/papermill/issues/519,

dagstermill-0.19.2/dagstermill/version.py

@@ -1 +0,0 @@
-__version__ = "0.19.2"