dagster-ray 0.0.0__tar.gz

dagster_ray-0.0.0/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023 Elementl, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

dagster_ray-0.0.0/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.1
+Name: dagster-ray
+Version: 0.0.0
+Summary: Dagster integration library for Ray
+Home-page: https://github.com/danielgafni/dagster-ray
+License: Apache-2.0
+Keywords: dagster,ray,ETL,distributed
+Author: Daniel Gafni
+Author-email: danielgafni16@gmail.com
+Requires-Python: >=3.8,<4.0
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Dist: dagster (>=1.3.5,<2.0.0)
+Requires-Dist: dagster-k8s (>=0.20.10,<0.21.0)
+Requires-Dist: pyarrow (>=8.0.0)
+Requires-Dist: ray (>=2.6.3,<3.0.0)
+Project-URL: Repository, https://github.com/danielgafni/dagster-ray
+Description-Content-Type: text/markdown
+
+# `dagster-ray`
+
+[Ray](https://github.com/ray-project/ray) integration library for [Dagster](https://github.com/dagster-io/dagster).
+
+> [!WARNING]
+> WIP, perhaps not usable.
+
+# Features
+
+## Resource
+
+`RayResource` provides access to a Ray cluster from within a Dagster pipeline.
+
+## IOManagers
+WIP
+
+## Executor
+WIP
+

dagster_ray-0.0.0/README.md

@@ -0,0 +1,18 @@
+# `dagster-ray`
+
+[Ray](https://github.com/ray-project/ray) integration library for [Dagster](https://github.com/dagster-io/dagster).
+
+> [!WARNING]
+> WIP, perhaps not usable.
+
+# Features
+
+## Resource
+
+`RayResource` provides access to a Ray cluster from within a Dagster pipeline.
+
+## IOManagers
+WIP
+
+## Executor
+WIP

dagster_ray-0.0.0/dagster_ray/__init__.py

@@ -0,0 +1,5 @@
+from dagster_ray.resource import RayResource
+
+__all__ = [
+    "RayResource",
+]

dagster_ray-0.0.0/dagster_ray/_version.py

@@ -0,0 +1 @@
+__version__ = "0.0.0"  # this field is managed by poetry-dynamic-versioning

dagster_ray-0.0.0/dagster_ray/configs.py

@@ -0,0 +1,13 @@
+from typing import List, Optional
+
+from dagster import Config
+
+
+class ContainerConfig(Config):
+    image: Optional[str] = None
+
+
+class RuntimeEnvConfig(Config):
+    working_dir: Optional[str] = None
+    pip: Optional[List[str]] = None
+    container: Optional[ContainerConfig] = None

dagster_ray-0.0.0/dagster_ray/executor.py

@@ -0,0 +1,384 @@
+# import multiprocessing
+# import os
+# import socket
+# from pathlib import Path
+# from typing import Optional, Dict, Any
+#
+# import ray
+# from dagster import Field, MetadataEntry, Noneable
+# from dagster import _check as check
+# from dagster import executor, InitExecutorContext
+# from dagster._core.errors import (
+#     DagsterExecutionInterruptedError,
+#     DagsterSubprocessError,
+# )
+# from dagster._core.events import DagsterEvent, EngineEventData
+# from dagster._core.execution.api import create_execution_plan, execute_plan_iterator
+# from dagster._core.execution.context.system import PlanOrchestrationContext
+# from dagster._core.execution.context_creation_job import create_context_free_log_manager
+# from dagster._core.execution.plan.plan import ExecutionPlan
+# from dagster._core.execution.retries import RetryMode
+# from dagster._core.executor.base import Executor
+# from dagster._core.instance import DagsterInstance
+# from dagster._utils.timing import format_duration, time_execution_scope
+# from ray import remote
+#
+#
+# RAY_TIMEOUT = 1
+#
+#
+# @remote(num_cpus=1)  # TODO configurable per step
+# class RayProcessCommand:
+#     def __init__(
+#         self,
+#         run_config,
+#         pipeline_run,
+#         instance_ref,
+#         step_key,
+#         recon_pipeline,
+#         retry_mode,
+#         known_state,
+#     ):
+#         self.run_config = run_config
+#         self.pipeline_run = pipeline_run
+#         self.instance_ref = instance_ref
+#
+#         self.step_key = step_key
+#         self.recon_pipeline = recon_pipeline
+#         self.retry_mode = retry_mode
+#         self.known_state = known_state
+#
+#         self.events = []
+#         self.execution_complete = False
+#
+#     def heartbeat(self):
+#         return True
+#
+#     def get_events(self):
+#         return self.events
+#
+#     def start(self):
+#         for event in self.execute():
+#             self.events.append(event)
+#
+#         return True
+#
+#     def next(self):
+#         if self.events:
+#             return self.events.pop(0)
+#         return None
+#
+#     def execute(self):
+#         pipeline = self.recon_pipeline
+#         with DagsterInstance.from_ref(self.instance_ref) as instance:
+#             execution_plan = create_execution_plan(
+#                 pipeline=pipeline,
+#                 run_config=self.run_config,
+#                 mode=self.pipeline_run.mode,
+#                 step_keys_to_execute=[self.step_key],
+#                 known_state=self.known_state,
+#             )
+#
+#             log_manager = create_context_free_log_manager(instance, self.pipeline_run)
+#
+#             yield DagsterEvent.step_worker_started(
+#                 log_manager,
+#                 self.pipeline_run.pipeline_name,
+#                 # TODO modify to make sense in ray context
+#                 message='Executing step "{}" in subprocess.'.format(self.step_key),
+#                 metadata_entries=[
+#                     MetadataEntry("pid", value=str(os.getpid())),
+#                 ],
+#                 step_key=self.step_key,
+#             )
+#
+#             try:
+#                 yield from execute_plan_iterator(
+#                     execution_plan,
+#                     pipeline,
+#                     self.pipeline_run,
+#                     run_config=self.run_config,
+#                     retry_mode=self.retry_mode.for_inner_plan(),
+#                     instance=instance,
+#                 )
+#             except:
+#                 raise ValueError()
+#         yield (1, 1, 1)
+#
+#
+# class RayExecutor(Executor):
+#     def __init__(
+#         self,
+#         retries: RetryMode,
+#         max_concurrent: int,
+#         resources: Optional[Dict[str, Any]] = None
+#     ):
+#         self._retries = check.inst_param(retries, "retries", RetryMode)
+#         max_concurrent = max_concurrent if max_concurrent else multiprocessing.cpu_count()  # TODO whats smart?
+#         self._max_concurrent = check.int_param(max_concurrent, "max_concurrent")
+#
+#         resources = resources if resources is not None else {}
+#         self._resources = resources.copy()
+#
+#     @property
+#     def retries(self):
+#         return self._retries
+#
+#     def execute(self, plan_context, execution_plan):
+#         check.inst_param(plan_context, "plan_context", PlanOrchestrationContext)
+#         check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
+#
+#         pipeline = plan_context.reconstructable_pipeline
+#         limit = self._max_concurrent
+#         host = socket.gethostname()
+#         yield DagsterEvent.engine_event(
+#             plan_context,
+#             f"Executing steps using ray executor: driver host ({host})",
+#             ## TODO: change?
+#             event_specific_data=EngineEventData.multiprocess(
+#                 host, step_keys_to_execute=execution_plan.step_keys_to_execute
+#             ),
+#         )
+#
+#         # It would be good to implement a reference tracking algorithm here so we could
+#         # garbage collect results that are no longer needed by any steps
+#         # https://github.com/dagster-io/dagster/issues/811
+#         with time_execution_scope() as timer_result:
+#             with execution_plan.start(retry_mode=self.retries) as active_execution:
+#                 active_iters = {}
+#                 actors = {}
+#                 stopping = False
+#
+#                 while (not stopping and not active_execution.is_complete) or active_iters:
+#                     if active_execution.check_for_interrupts():
+#                         step_keys, active_actors = zip(*actors.items())
+#                         yield DagsterEvent.engine_event(
+#                             plan_context,
+#                             "Ray executor: received termination signal - " "killing all actors",
+#                             EngineEventData.interrupted(list(step_keys)),
+#                         )
+#                         stopping = True
+#                         active_execution.mark_interrupted()
+#
+#                         # cancels all active actors
+#                         def kill_actor(actor):
+#                             try:
+#                                 ray.kill(actor)
+#                             except ray.exceptions.RayActorError:
+#                                 # actor already dead
+#                                 pass
+#
+#                         list(map(kill_actor, active_actors))
+#
+#                     # start iterators
+#                     while len(active_iters) < limit and not stopping:
+#                         steps = active_execution.get_steps_to_execute(limit=(limit - len(active_iters)))
+#
+#                         if not steps:
+#                             break
+#
+#                         for step in steps:
+#                             step_context = plan_context.for_step(step)
+#                             # need max_concurrency otherwise get will block when
+#                             # when trying to fetch any events
+#                             # thread concurrency is considered experimental
+#                             # consider async
+#
+#                             actor = RayProcessCommand.options(
+#                                 max_concurrency=2,
+#                                 # num_cpus=step_context.tags
+#                                 **self._resources,
+#                             ).remote(
+#                                 run_config=step_context.run_config,
+#                                 pipeline_run=step_context.pipeline_run,
+#                                 instance_ref=step_context.instance.get_ref(),
+#                                 step_key=step.key,
+#                                 recon_pipeline=pipeline,
+#                                 retry_mode=self.retries,
+#                                 known_state=active_execution.get_known_state(),
+#                             )
+#                             # term_events[step.key] = multiproc_ctx.Event()
+#                             active_iters[step.key] = execute_step_out_of_process(actor, step_context)
+#                             actors[step.key] = actor
+#
+#                     # process active iterators
+#                     empty_iters = []
+#                     for key, step_iter in active_iters.items():
+#                         try:
+#                             event_or_none = next(step_iter)
+#                             if event_or_none is None:
+#                                 continue
+#                             else:
+#                                 yield event_or_none
+#                                 active_execution.handle_event(event_or_none)
+#
+#                         # except ChildProcessCrashException as crash:
+#                         #     serializable_error = serializable_error_info_from_exc_info(
+#                         #         sys.exc_info()
+#                         #     )
+#                         #     step_context = plan_context.for_step(
+#                         #         active_execution.get_step_by_key(key)
+#                         #     )
+#                         #     yield DagsterEvent.engine_event(
+#                         #         step_context,
+#                         #         (
+#                         #             "Multiprocess executor: child process for step {step_key} "
+#                         #             "unexpectedly exited with code {exit_code}"
+#                         #         ).format(step_key=key, exit_code=crash.exit_code),
+#                         #         EngineEventData.engine_error(serializable_error),
+#                         #     )
+#                         #     step_failure_event = DagsterEvent.step_failure_event(
+#                         #         step_context=plan_context.for_step(
+#                         #             active_execution.get_step_by_key(key)
+#                         #         ),
+#                         #         step_failure_data=StepFailureData(
+#                         #             error=serializable_error, user_failure_data=None
+#                         #         ),
+#                         #     )
+#                         #     active_execution.handle_event(step_failure_event)
+#                         #     yield step_failure_event
+#                         #     empty_iters.append(key)
+#                         except StopIteration:
+#                             # raise Exception()
+#                             empty_iters.append(key)
+#                     # clear and mark complete finished iterators
+#                     # plan_context.log.info(list(actors.keys()))
+#                     for key in empty_iters:
+#                         del active_iters[key]
+#                         del actors[key]
+#                         active_execution.verify_complete(plan_context, key)
+#
+#                     # process skipped and abandoned steps
+#                     yield from active_execution.plan_events_iterator(plan_context)
+#
+#                 # TODO: handle errors properly
+#                 errs = {actor: None for step_key, actor in active_iters.items() if not actor.heartbeat()}
+#
+#                 # After termination starts, raise an interrupted exception once all subprocesses
+#                 # have finished cleaning up (and the only errors were from being interrupted)
+#                 if (
+#                     stopping
+#                     and (not active_iters)
+#                     and all([err_info.cls_name == "DagsterExecutionInterruptedError" for err_info in errs.values()])
+#                 ):
+#                     yield DagsterEvent.engine_event(
+#                         plan_context,
+#                         "Ray executor: killed all active child processes",
+#                         event_specific_data=EngineEventData(),
+#                     )
+#                     raise DagsterExecutionInterruptedError()
+#                 elif errs:
+#                     raise DagsterSubprocessError(
+#                         "During ray execution errors occurred in actors:\n{error_list}".format(
+#                             error_list="\n".join(
+#                                 [
+#                                     "In process {pid}: {err}".format(pid=pid, err=err.to_string())
+#                                     for pid, err in errs.items()
+#                                 ]
+#                             )
+#                         ),
+#                         subprocess_error_infos=list(errs.values()),
+#                     )
+#
+#         yield DagsterEvent.engine_event(
+#             plan_context,
+#             "Ray executor: driver exiting after {duration}".format(
+#                 duration=format_duration(timer_result.millis),
+#             ),
+#             event_specific_data=EngineEventData.multiprocess(os.getpid()),
+#         )
+#
+#
+# def wait_fetch(obj):
+#     complete, _ = ray.wait([obj], timeout=0, fetch_local=True)
+#
+#     if complete:
+#         return ray.get(obj, timeout=RAY_TIMEOUT)
+#
+#     return False
+#
+#
+# def execute_step_out_of_process(actor, step_context):
+#     # engine_event
+#     # yield DagsterEvent.step_worker_starting(
+#     #     step_context,
+#     #     'Launching ray actor for "{}".'.format(step.key),
+#     #     metadata_entries=[],
+#     # )
+#     # raise Exception(command.__dir__())
+#     finished = actor.start.remote()
+#     next_ref = actor.next.remote()
+#     # ray.get(finished)
+#     while True:
+#         # check finished before getting any events
+#         # is_finished = wait_fetch(finished)
+#
+#         # will block here, not ideal
+#         # this required in case is_finished is but the next return is in flight
+#         ret = wait_fetch(next_ref)
+#
+#         # step_context.log.info(f"{ret}, {is_finished}")
+#
+#         if ret == (1, 1, 1):
+#             # with capture_interrupts():
+#             try:
+#                 # will except if the call did
+#                 _ = ray.get(finished, timeout=RAY_TIMEOUT)
+#             except (
+#                 Exception,
+#                 KeyboardInterrupt,
+#                 DagsterExecutionInterruptedError,
+#             ):
+#                 yield DagsterEvent.engine_event(
+#                     step_context,
+#                     message="interrupted",
+#                     event_specific_data=EngineEventData(),
+#                 )
+#             return
+#         if ret is None or isinstance(ret, DagsterEvent):
+#             next_ref = actor.next.remote()
+#             yield ret
+#         elif not ret:
+#             # ret is false (wait_fetch not ready)
+#             yield None
+#         else:
+#             check.failed("Unexpected return value from actor {}".format(type(ret)))
+#         # continue
+#         # try:
+#         #     # throws exception when actor is dead
+#         #     ray.get(actor.heartbeat.remote(), timeout=RAY_TIMEOUT)
+#         # except ray.exceptions.RayActorError:
+#         #     return
+#         # except ray.exceptions.GetTimeoutError:
+#         #     yield DagsterEvent.engine_event(
+#         #             step_context,
+#         #             message=f"get timed out",
+#         #             event_specific_data=EngineEventData(),
+#         #         )
+#         # raise
+#
+#
+# @executor(
+#     name="ray",
+#     config_schema=
+# )
+# def ray_executor(init_context: InitExecutorContext):
+#     """Ray-based executor."""
+#     cwd = Path(os.getcwd())
+#
+#     exc_cfg = init_context.executor_config
+#     ray_host = exc_cfg.get("host", "localhost")
+#     ray_port = exc_cfg.get("port", 10001)
+#     code_dir = exc_cfg.get("code_dir", cwd.as_posix())
+#     requirements_path = exc_cfg.get("requirements_path")
+#     max_concurrency = exc_cfg.get("max_concurrency", 10)
+#
+#     reqs = []
+#     if requirements_path is not None:
+#         reqs = Path(requirements_path).read_text().split("\n")
+#
+#     ray.init(
+#         f"ray://{ray_host}:{ray_port}", runtime_env={"working_dir": code_dir, "pip": reqs},
+#         storage=exc_cfg.get("storage")
+#     )
+#     return RayExecutor(retries=RetryMode.DISABLED, max_concurrent=max_concurrency)

dagster_ray-0.0.0/dagster_ray/executor_2.py

@@ -0,0 +1,284 @@
+# import os
+# from pathlib import Path
+# from typing import Iterator, List, Optional, cast, Dict, Any, Sequence
+#
+# import kubernetes.config
+# import ray
+# from dagster import (
+#     Field,
+#     IntSource,
+#     Noneable,
+#     StringSource,
+#     _check as check,
+#     executor, DagsterInstance, JobExecutionResult,
+# )
+# from dagster._core.definitions import ReconstructableJob
+# from dagster._core.definitions.executor_definition import multiple_process_executor_requirements
+# from dagster._core.definitions.metadata import MetadataValue
+# from dagster._core.events import DagsterEvent, EngineEventData
+# from dagster._core.execution.api import execute_plan_iterator, create_execution_plan, execute_plan, execute_job, \
+#     ReexecutionOptions
+# from dagster._core.execution.plan.state import KnownExecutionState
+# from dagster._core.execution.retries import RetryMode, get_retries_config
+# from dagster._core.execution.tags import get_tag_concurrency_limits_config
+# from dagster._core.executor.base import Executor
+# from dagster._core.executor.init import InitExecutorContext
+# from dagster._core.executor.step_delegating import (
+#     CheckStepHealthResult,
+#     StepDelegatingExecutor,
+#     StepHandler,
+#     StepHandlerContext,
+# )
+# from ray import ObjectRef
+# from ray.actor import ActorHandle
+# from dagster._utils.merger import merge_dicts
+#
+# USER_DEFINED_RAY_CONFIG_SCHEMA = {}
+#
+# RAY_TIMEOUT = 5.
+#
+# RAY_EXECUTOR_CONFIG_SCHEMA = {
+#         "host": Field(str, is_required=False, description="Ray Cluster host"),
+#         "port": Field(int, is_required=False, description="Ray Cluster port"),
+#         "code_dir": Field(str, is_required=False),
+#         "requirements_path": Field(str, is_required=False),
+#         "image": Field(str, default_value=os.getenv("DAGSTER_CURRENT_IMAGE"), is_required=False),
+#         "max_concurrency": Field(int, is_required=False),
+#         "storage": Field(Noneable(str), is_required=False, description="Shared storage path for Ray workers"),
+#         "resources": Field(Noneable(Dict[str, Any]),
+#         is_required=False, description="Resources to pass to Ray workers"),
+#         "retries": get_retries_config(),
+#         "max_concurrent": Field(
+#             IntSource,
+#             is_required=False,
+#             description=(
+#                 "Limit on the number of pods that will run concurrently within the scope "
+#                 "of a Dagster run. Note that this limit is per run, not global."
+#             ),
+#         ),
+#         "tag_concurrency_limits": get_tag_concurrency_limits_config(),
+#         "step_k8s_config": Field(
+#             dict,
+#             is_required=False,
+#             description="Raw Ray configuration for each step launched by the executor.",
+#         )
+# }
+#
+#
+#
+#
+# class RayStepHandler(StepHandler):
+#     @property
+#     def name(self):
+#         return "RayStepHandler"
+#
+#     def __init__(
+#         self,
+#         host: str,
+#         port: int = 6379,
+#         image: Optional[str] = None,
+#         code_dir: Optional[str] = None,
+#         requirements_path: Optional[str] = None,
+#         storage: Optional[str] = None,
+#         resources: Optional[Dict[str, Any]] = None,
+#     ):
+#         super().__init__()
+#
+#         self._image = check.str_param(image, "image")
+#         self._host = check.opt_str_param(host, "host", default="localhost")
+#         self._port = check.int_param(port, "port")
+#         self._code_dir = check.opt_str_param(code_dir, "code_dir")
+#         self._requirements_path = check.opt_str_param(requirements_path, "requirements_path")
+#         self._storage = check.opt_str_param(storage, "storage")
+#         self._resources = check.opt_dict_param(resources, "resources")
+#
+#         runtime_env = {}
+#
+#         if self._code_dir is not None:
+#             runtime_env["working_dir"] = self._code_dir
+#
+#         if self._requirements_path is not None:
+#             reqs = Path(self._requirements_path).read_text().split("\n")
+#             runtime_env["pip"] = reqs
+#
+#         if self._image is not None:
+#             runtime_env["container"] = {
+#                 "image": self._image,
+#             }
+#
+#         ray.init(
+#             address=f"ray://{host}:{port}",
+#             runtime_env=runtime_env,
+#             storage=self._storage,
+#         )
+#
+#         self.steps_refs: Dict[str, ObjectRef] = {}
+#
+#     def _get_step_key(self, step_handler_context: StepHandlerContext) -> str:
+#
+#         step_keys_to_execute = cast(
+#             List[str], step_handler_context.execute_step_args.step_keys_to_execute
+#         )
+#         assert len(step_keys_to_execute) == 1, "Launching multiple steps is not currently supported"
+#         return step_keys_to_execute[0]
+#
+#     @ray.remote
+#     def execute_on_ray(self, step_handler_context: StepHandlerContext) -> JobExecutionResult:
+#         step_key = self._get_step_key(step_handler_context)
+#         instance = step_handler_context.instance
+#         dagster_run = step_handler_context.dagster_run
+#         run_config = step_handler_context.dagster_run.run_config
+#
+#         return execute_job(
+#             job=step_handler_context._plan_context.reconstructable_job,  # noqa
+#             instance=instance,
+#             run_config=run_config,
+#             op_selection=step_handler_context.dagster_run.resolved_op_selection,
+#             reexecution_options=ReexecutionOptions(
+#                 parent_run_id=dagster_run.get_parent_run_id(),
+#                 step_selection=[step_key]
+#             ),
+#             tags=dagster_run.tags,
+#         )
+#
+#     def launch_step(self, step_handler_context: StepHandlerContext) -> Iterator[DagsterEvent]:
+#         step_key = self._get_step_key(step_handler_context)
+#         run = step_handler_context.dagster_run
+#         labels = {
+#             "dagster/job": run.job_name,
+#             "dagster/op": step_key,
+#             "dagster/run-id": step_handler_context.execute_step_args.run_id,
+#         }
+#         if run.external_job_origin:
+#             labels["dagster/code-location"] = (
+#                 run.external_job_origin.external_repository_origin.code_location_origin.location_name
+#             )
+#
+#         yield DagsterEvent.step_worker_starting(
+#             step_handler_context.get_step_context(step_key),
+#             message=f'Executing step "{step_key}" in Ray Cluster',
+#             metadata={
+#                 "uri": MetadataValue.path(f"ray://{self._host}:{self._port}"),
+#             },
+#         )
+#
+#         self.steps_refs[step_key] = self.execute_on_ray.options(num_cpus=..., num_gpus=...)
+#         .remote(step_handler_context)
+#
+#     def check_step_health(self, step_handler_context: StepHandlerContext) -> CheckStepHealthResult:
+#         step_key = self._get_step_key(step_handler_context)
+#
+#         complete, _ = ray.wait([self.steps_refs[step_key]], timeout=0, fetch_local=False)
+#
+#         if status.failed:
+#             return CheckStepHealthResult.unhealthy(
+#                 reason=f"Discovered failed Kubernetes job {job_name} for step {step_key}.",
+#             )
+#
+#         return CheckStepHealthResult.healthy()
+#
+#     def terminate_step(self, step_handler_context: StepHandlerContext) -> Iterator[DagsterEvent]:
+#         step_key = self._get_step_key(step_handler_context)
+#
+#         job_name = self._get_k8s_step_job_name(step_handler_context)
+#         container_context = self._get_container_context(step_handler_context)
+#
+#         yield DagsterEvent.engine_event(
+#             step_handler_context.get_step_context(step_key),
+#             message=f"Deleting Kubernetes job {job_name} for step",
+#             event_specific_data=EngineEventData(),
+#         )
+#
+#         self._api_client.delete_job(job_name=job_name, namespace=container_context.namespace)
+#
+#     def _wait_fetch(self, ref: ObjectRef) -> Optional[Any]:
+#         complete, _ = ray.wait([ref], timeout=0, fetch_local=True)
+#
+#         if complete:
+#             return ray.get(ref, timeout=RAY_TIMEOUT)
+#         else:
+#             return None
+#
+#     def _execute_in_ray(self, actor: ActorHandle, step_handler_context: StepHandlerContext):
+#         finished = actor.start.remote()
+#         next_ref = actor.next.remote()
+#         while True:
+#             # check finished before getting any events
+#             # is_finished = wait_fetch(finished)
+#
+#             # will block here, not ideal
+#             # this required in case is_finished is but the next return is in flight
+#             ret = self._wait_fetch(next_ref)
+#
+#             if ret == (1, 1, 1):
+#                 # with capture_interrupts():
+#                 try:
+#                     # will except if the call did
+#                     _ = ray.get(finished, timeout=RAY_TIMEOUT)
+#                 except (
+#                         Exception,
+#                         KeyboardInterrupt,
+#                         DagsterExecutionInterruptedError,
+#                 ):
+#                     yield DagsterEvent.engine_event(
+#                         step_context,
+#                         message="interrupted",
+#                         event_specific_data=EngineEventData(),
+#                     )
+#                 return
+#             if ret is None or isinstance(ret, DagsterEvent):
+#                 next_ref = actor.next.remote()
+#                 yield ret
+#             elif not ret:
+#                 # ret is false (wait_fetch not ready)
+#                 yield None
+#             else:
+#                 check.failed("Unexpected return value from actor {}".format(type(ret)))
+#             # continue
+#             # try:
+#             #     # throws exception when actor is dead
+#             #     ray.get(actor.heartbeat.remote(), timeout=RAY_TIMEOUT)
+#             # except ray.exceptions.RayActorError:
+#             #     return
+#             # except ray.exceptions.GetTimeoutError:
+#             #     yield DagsterEvent.engine_event(
+#             #             step_context,
+#             #             message=f"get timed out",
+#             #             event_specific_data=EngineEventData(),
+#             #         )
+#             # raise
+#
+#
+# @executor(
+#     name="ray",
+#     config_schema=RAY_EXECUTOR_CONFIG_SCHEMA,
+#     requirements=multiple_process_executor_requirements(),
+# )
+# def ray_executor(init_context: InitExecutorContext) -> Executor:
+#     """Executor which launches steps with Ray
+#
+#     `max_concurrent` limits the number of steps that will execute concurrently for one run. By default
+#     there is no limit- it will maximally parallel as allowed by the DAG. Note that this is not a
+#     global limit.
+#
+#     Configuration set using `tags` on a `@job` will only apply to the `run` level. For configuration
+#     to apply at each `step` it must be set using `tags` for each `@op`.
+#     """
+#     # run_launcher = (
+#     #     init_context.instance.run_launcher
+#     #     if isinstance(init_context.instance.run_launcher, K8sRunLauncher)
+#     #     else None
+#     # )
+#
+#     exc_cfg = init_context.executor_config
+#
+#     return StepDelegatingExecutor(
+#         RayStepHandler(
+#             ...
+#         ),
+#         retries=RetryMode.from_config(exc_cfg["retries"]),  # type: ignore
+#         max_concurrent=check.opt_int_elem(exc_cfg, "max_concurrent"),
+#         tag_concurrency_limits=check.opt_list_elem(exc_cfg, "tag_concurrency_limits"),
+#         should_verify_step=True,
+#     )
+#

dagster_ray-0.0.0/dagster_ray/io_managers.py

@@ -0,0 +1,38 @@
+# import pickle
+# from typing import Any
+#
+# import ray
+#
+# from dagster import UPathIOManager, InputContext, OutputContext, InitResourceContext, ConfigurableIOManager
+# from pydantic import PrivateAttr
+# from upath import UPath
+#
+# from dagster_ray.resource import RayResource
+#
+#
+# class BaseRayIOManager(ConfigurableIOManager, UPathIOManager):
+#     base_dir: str
+#     extension: str = ".ray"
+#     ray: RayResource
+#
+#     _base_path: UPath = PrivateAttr()
+#
+#     def setup_for_execution(self, context: InitResourceContext) -> None:
+#         self._base_path = UPath(self.base_dir)
+#
+#     def dump_to_path(self, context: OutputContext, obj: Any, path: "UPath"):
+#         ref = ray.put(obj)
+#         path.write_bytes(pickle.dumps(ref))
+#
+#
+# class RayRefIOManager(BaseRayIOManager):
+#     def load_from_path(self, context: InputContext, path: "UPath") -> ray.ObjectRef:
+#         ref = pickle.loads(path.read_bytes())
+#         assert isinstance(ref, ray.ObjectRef)
+#         return ref
+#
+# #
+# # class RayIOManager(BaseRayIOManager):
+# #     def load_from_path(self, context: InputContext, path: "UPath") -> Any:
+# #         ref = pickle.loads(path.read_bytes())
+# #         return ray.get(ref)

dagster_ray-0.0.0/dagster_ray/resource.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Generator, Optional
+
+import ray
+from dagster import ConfigurableResource, InitResourceContext
+
+from dagster_ray.configs import RuntimeEnvConfig
+
+
+class RayResource(ConfigurableResource):
+    """
+    A resource that provides access to the Ray cluster.
+    """
+
+    address: Optional[str]
+    local: bool = False
+    runtime_env: Optional[RuntimeEnvConfig] = None
+    storage: Optional[str] = None
+    # todo: add requirements_path
+
+    @contextmanager
+    def yield_for_execution(self, context: InitResourceContext) -> Generator[RayResource, None, None]:
+        """
+        Setups Ray and yields the resource.
+        Warning: this will shutdown the Ray process after execution.
+        """
+
+        # raise RuntimeError()
+        assert context.log is not None
+
+        if self.runtime_env is not None:
+            runtime_env = self.runtime_env.dict()
+        else:
+            runtime_env = {}
+
+        if runtime_env.get("container") is not None and runtime_env["container"].get("image") is None:
+            assert context.dagster_run is not None
+            runtime_env["container"]["image"] = context.dagster_run.tags.get("dagster/image")
+
+        ray.init(
+            self.address,
+            runtime_env=runtime_env,
+            storage=self.storage,
+        )
+        if self.address is not None:
+            context.log.info(f"Connected to Ray Cluster at {self.address}")
+
+        yield self
+
+        ray.shutdown()
+
+
+RAY_RESOURCE_CONFIG_SCHEMA = RayResource.to_fields_dict()

dagster_ray-0.0.0/pyproject.toml

@@ -0,0 +1,155 @@
+[tool.poetry]
+name = "dagster-ray"
+version = "0.0.0"
+description = "Dagster integration library for Ray"
+authors = [
+    "Daniel Gafni <danielgafni16@gmail.com>"
+]
+readme = "README.md"
+packages = [{include = "dagster_ray"}]
+repository = "https://github.com/danielgafni/dagster-ray"
+keywords = [
+    "dagster",
+    "ray",
+    "ETL",
+    "distributed",
+]
+classifiers = [
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+license = "Apache-2.0"
+
+[tool.poetry.dependencies]
+python = "^3.8"
+dagster = "^1.3.5"
+pyarrow = ">=8.0.0"
+ray = "^2.6.3"
+dagster-k8s = "^0.20.10"
+
+
+[tool.poetry.group.dev.dependencies]
+hypothesis = "^6.77.0"
+pytest = "^7.3.1"
+deepdiff = "^6.3.0"
+isort = "^5.12.0"
+ruff = "^0.0.272"
+pyright = "^1.1.313"
+tox = "^4.6.0"
+tox-gh = "^1.0.0"
+pre-commit = "^3.3.2"
+dagit = "^1.3.9"
+black = "^23.3.0"
+pytest-cases = "^3.6.14"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry-dynamic-versioning]
+enable = true
+strict = false
+vcs = "git"
+style = "pep440"
+dirty = true
+
+[tool.poetry-dynamic-versioning.substitution]
+files = [
+    "pyproject.toml",
+    "dagster_ray/_version.py"
+]
+
+[tool.pytest.ini_options]
+log_cli = true
+log_level = "INFO"
+
+[tool.isort]
+profile = "black"
+line_length = 120
+src_paths = ["dagster_ray", "tests"]
+
+[tool.black]
+line-length = 120
+target-version = ['py39']
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.pytest_cache
+  | \.ruff_cache
+  | \.venv
+  | build
+  | dist
+  )/
+'''
+
+[tool.ruff]
+line-length = 120
+src = [
+    "dagster_ray",
+    "tests"
+]
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+]
+[tool.ruff.isort]
+known-first-party = ["dagster_ray", "tests"]
+
+[tool.pyright]
+reportPropertyTypeMismatch = true
+reportImportCycles = true
+reportWildcardImportFromLibrary = true
+reportUntypedFunctionDecorator = true
+reportUntypedClassDecorator = true
+
+include = [
+    "dagster_ray",
+    "tests"
+]
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+]