torchx-nightly 2025.9.17__py3-none-any.whl → 2025.9.19__py3-none-any.whl

torchx/components/__init__.py

@@ -298,13 +298,6 @@ imagine the component is defined as:
    * ``*args=["--help"]``: ``torchx run comp.py:f -- --help``
    * ``*args=["--i", "2"]``: ``torchx run comp.py:f --i 1 -- --i 2``
 
-Run in a Pipeline
---------------------------------
-
-The :ref:`torchx.pipelines<pipelines:torchx.pipelines>` define adapters that
-convert a torchx component into the object that represents a pipeline "stage" in the
-target pipeline platform (see :ref:`Pipelines` for a list of supported pipeline orchestrators).
-
 Additional Resources
 -----------------------
 

torchx/schedulers/__init__.py

@@ -22,7 +22,6 @@ DEFAULT_SCHEDULER_MODULES: Mapping[str, str] = {
     "aws_batch": "torchx.schedulers.aws_batch_scheduler",
     "aws_sagemaker": "torchx.schedulers.aws_sagemaker_scheduler",
     "gcp_batch": "torchx.schedulers.gcp_batch_scheduler",
-    "ray": "torchx.schedulers.ray_scheduler",
     "lsf": "torchx.schedulers.lsf_scheduler",
 }
 

torchx/specs/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
@@ -52,14 +51,19 @@ from torchx.util.entrypoints import load_group
 
 from torchx.util.modules import import_attr
 
-AWS_NAMED_RESOURCES: Mapping[str, Callable[[], Resource]] = import_attr(
+GiB: int = 1024
+
+ResourceFactory = Callable[[], Resource]
+
+AWS_NAMED_RESOURCES: Mapping[str, ResourceFactory] = import_attr(
     "torchx.specs.named_resources_aws", "NAMED_RESOURCES", default={}
 )
-GENERIC_NAMED_RESOURCES: Mapping[str, Callable[[], Resource]] = import_attr(
+GENERIC_NAMED_RESOURCES: Mapping[str, ResourceFactory] = import_attr(
     "torchx.specs.named_resources_generic", "NAMED_RESOURCES", default={}
 )
-
-GiB: int = 1024
+FB_NAMED_RESOURCES: Mapping[str, ResourceFactory] = import_attr(
+    "torchx.specs.fb.named_resources", "NAMED_RESOURCES", default={}
+)
 
 
 def _load_named_resources() -> Dict[str, Callable[[], Resource]]:
@@ -69,6 +73,7 @@ def _load_named_resources() -> Dict[str, Callable[[], Resource]]:
     for name, resource in {
         **GENERIC_NAMED_RESOURCES,
         **AWS_NAMED_RESOURCES,
+        **FB_NAMED_RESOURCES,
         **resource_methods,
     }.items():
         materialized_resources[name] = resource

torchx/specs/api.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
@@ -83,6 +82,8 @@ class Resource:
         memMB: MB of ram
         capabilities: additional hardware specs (interpreted by scheduler)
         devices: a list of named devices with their quantities
+        tags: metadata tags for the resource (not interpreted by schedulers)
+          used to add non-functional information about resources (e.g. whether it is an alias of another resource)
 
     Note: you should prefer to use named_resources instead of specifying the raw
     resource requirement directly.
@@ -93,6 +94,7 @@ class Resource:
     memMB: int
     capabilities: Dict[str, Any] = field(default_factory=dict)
     devices: Dict[str, int] = field(default_factory=dict)
+    tags: Dict[str, object] = field(default_factory=dict)
 
     @staticmethod
     def copy(original: "Resource", **capabilities: Any) -> "Resource":
@@ -101,6 +103,7 @@ class Resource:
         are present in the original resource and as parameter, the one from parameter
         will be used.
         """
+
         res_capabilities = dict(original.capabilities)
         res_capabilities.update(capabilities)
         return Resource(

{torchx_nightly-2025.9.17.dist-info → torchx_nightly-2025.9.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: torchx-nightly
-Version: 2025.9.17
+Version: 2025.9.19
 Summary: TorchX SDK and Components
 Home-page: https://github.com/pytorch/torchx
 Author: TorchX Devs
@@ -41,8 +41,6 @@ Requires-Dist: google-cloud-logging==3.10.0; extra == "dev"
 Requires-Dist: google-cloud-runtimeconfig==0.34.0; extra == "dev"
 Requires-Dist: hydra-core; extra == "dev"
 Requires-Dist: ipython; extra == "dev"
-Requires-Dist: kfp==1.8.22; extra == "dev"
-Requires-Dist: protobuf==3.20.3; extra == "dev"
 Requires-Dist: mlflow-skinny; extra == "dev"
 Requires-Dist: moto~=5.0.8; extra == "dev"
 Requires-Dist: pyre-extensions; extra == "dev"
@@ -60,7 +58,6 @@ Requires-Dist: torchtext==0.18.0; extra == "dev"
 Requires-Dist: torchvision==0.23.0; extra == "dev"
 Requires-Dist: typing-extensions; extra == "dev"
 Requires-Dist: ts==0.5.1; extra == "dev"
-Requires-Dist: ray[default]; extra == "dev"
 Requires-Dist: wheel; extra == "dev"
 Requires-Dist: lintrunner; extra == "dev"
 Requires-Dist: lintrunner-adapters; extra == "dev"
@@ -72,12 +69,8 @@ Provides-Extra: gcp_batch
 Requires-Dist: google-cloud-batch>=0.5.0; extra == "gcp-batch"
 Requires-Dist: google-cloud-logging>=3.0.0; extra == "gcp-batch"
 Requires-Dist: google-cloud-runtimeconfig>=0.33.2; extra == "gcp-batch"
-Provides-Extra: kfp
-Requires-Dist: kfp==1.6.2; extra == "kfp"
 Provides-Extra: kubernetes
 Requires-Dist: kubernetes>=11; extra == "kubernetes"
-Provides-Extra: ray
-Requires-Dist: ray>=1.12.1; extra == "ray"
 
 [![PyPI](https://img.shields.io/pypi/v/torchx)](https://pypi.org/project/torchx/)
 [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://github.com/pytorch/torchx/blob/main/LICENSE)
@@ -100,7 +93,6 @@ TorchX currently supports:
 * AWS Batch
 * Docker
 * Local
-* Ray (prototype)
 * GCP Batch (prototype)
 
 Need a scheduler not listed? [Let us know!](https://github.com/pytorch/torchx/issues?q=is%3Aopen+is%3Aissue+label%3Ascheduler-request)
@@ -136,15 +128,9 @@ pip install torchx
 # install torchx sdk and CLI -- all dependencies
 pip install "torchx[dev]"
 
-# install torchx kubeflow pipelines (kfp) support
-pip install "torchx[kfp]"
-
 # install torchx Kubernetes / Volcano support
 pip install "torchx[kubernetes]"
 
-# install torchx Ray support
-pip install "torchx[ray]"
-
 # install torchx GCP Batch support
 pip install "torchx[gcp_batch]"
 ```

{torchx_nightly-2025.9.17.dist-info → torchx_nightly-2025.9.19.dist-info}/RECORD

@@ -22,7 +22,7 @@ torchx/cli/cmd_status.py,sha256=22IAEmKs0qkG6kJi83u9dRX2Q-ntT7yehVx7FxtY-vQ,2114
 torchx/cli/cmd_tracker.py,sha256=RfLxE4Cq1wfk7k051RtZ8RPJp0pEKSCa3KmTeRs3LF8,5218
 torchx/cli/colors.py,sha256=yLMes7e_UoLAfhxE0W6edhc58t83UHAlnCN2ANPeuXw,568
 torchx/cli/main.py,sha256=1Jf2cnO6Y2W69Adt88avmNPVrL6ZR4Hkff6GVB4293k,3484
-torchx/components/__init__.py,sha256=6Sb8RWRGObajkH7eFSKv5bHaN5bzTqJiSEmrIIo3OIc,12121
+torchx/components/__init__.py,sha256=YT2D_w4df1TN_Soek6vhk6fv8WFxT-_mgs6SINrP0BI,11810
 torchx/components/component_test_base.py,sha256=22iNSdVa_qTW3SMM30Pw5UEWlK4DZVw0C03EqYiaLOI,4150
 torchx/components/dist.py,sha256=9jECk3jjQ4Yh4oWDK8vnQ7kcI0-OWCbbwj8uvBdI9FU,14588
 torchx/components/interpret.py,sha256=g8gkKdDJvsBfX1ZrpVT7n2bMEtmwRV_1AqDyAnnQ_aA,697
@@ -46,15 +46,7 @@ torchx/examples/apps/lightning/interpret.py,sha256=Hd3kE5a6FyhxCmJBfTzb4Tlj518zh
 torchx/examples/apps/lightning/model.py,sha256=4CgObWfANqDN9emYSdmCpbRe_V_Lef_Hd3M-yayDbZE,4045
 torchx/examples/apps/lightning/profiler.py,sha256=SSSihnwjeUTkBoz0E3qn1b-wbkfUIowscx2ND_37zyw,1915
 torchx/examples/apps/lightning/train.py,sha256=0wvvshGHvZowePB4LfclXwn40X7i9euM0ReETWBcPSo,6253
-torchx/examples/pipelines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-torchx/examples/pipelines/kfp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-torchx/examples/pipelines/kfp/advanced_pipeline.py,sha256=U5N_XmpxbuEIh-hDayjJ5Lnk2lYvmgr7oznFnsKUk5g,8431
-torchx/examples/pipelines/kfp/dist_pipeline.py,sha256=xFn59P1S22o2zOJ2LhlIkhjYH3le0zp2sLPNj5idTnE,2203
-torchx/examples/pipelines/kfp/intro_pipeline.py,sha256=oWdMHPLWf5nKRm0hS7psF0yUp8Tf7tfR-Sm3YuUCmWk,2776
 torchx/pipelines/__init__.py,sha256=2MbRVk5xwRjg-d2qPemeXpEhDsocMQumPQ53lsesZAI,606
-torchx/pipelines/kfp/__init__.py,sha256=8iJ8lql_fxwuk9VCYSxXnX6tPL228fB5mDZpOs-kpn4,736
-torchx/pipelines/kfp/adapter.py,sha256=5GeHULjb1kxG6wJtYVLpNkgdzUi4iYEaR42VFOwT6fY,9045
-torchx/pipelines/kfp/version.py,sha256=mYBxd6bm4MeR34D--xo-JLQ9wHeAl_ZQLwbItCf9tr0,539
 torchx/runner/__init__.py,sha256=x8Sz7s_tLxPgJgvWIhK4ju9BNZU61uBFywGwDY6CqJs,315
 torchx/runner/api.py,sha256=0kDyOgmAcq0X-bTWiBIqX3BJOCYSa5-TZ7o2Hrqxzdw,30053
 torchx/runner/config.py,sha256=20X-vveAJVjb1AjjDSC6x_BVcdrTj9_ZLt_CHTykiFo,18266
@@ -64,7 +56,7 @@ torchx/runner/events/handlers.py,sha256=ThHCIJW21BfBgB7b6ftyjASJmD1KdizpjuTtsyqn
 torchx/runtime/__init__.py,sha256=Wxje2BryzeQneFu5r6P9JJiEKG-_C9W1CcZ_JNrKT6g,593
 torchx/runtime/tracking/__init__.py,sha256=dYnAPnrXYREfPXkpHhdOFkcYIODWEbA13PdD-wLQYBo,3055
 torchx/runtime/tracking/api.py,sha256=SmUQyUKZqG3KlAhT7CJOGqRz1O274E4m63wQeOVq3CU,5472
-torchx/schedulers/__init__.py,sha256=igIBdxGhkuzH7oYVFXIA9xwjkSn3QzWZ_9dhfdl_M0I,2299
+torchx/schedulers/__init__.py,sha256=hliMsZHZNOKue0uTHUWxvO0V7xsKApBxN4Wb_9L0Mz4,2253
 torchx/schedulers/api.py,sha256=lfxNhrEO6eYYqVuQzzj9sTXrZShuZkyYxJ1jPE-Lvpo,14561
 torchx/schedulers/aws_batch_scheduler.py,sha256=hFxYzSZEK2SVS5sEyQC5YvNI0JJUJUQsWORlYpj_h3M,28105
 torchx/schedulers/aws_sagemaker_scheduler.py,sha256=flN8GumKE2Dz4X_foAt6Jnvt-ZVojWs6pcyrHwB0hz0,20921
@@ -76,14 +68,10 @@ torchx/schedulers/kubernetes_mcad_scheduler.py,sha256=1tuzq3OutCMdSPqg_dNmCHt_wy
 torchx/schedulers/kubernetes_scheduler.py,sha256=0_loGJ7WnxEr9dhgFt3Gw-7nVLirMDVN-MAFTCq7erE,28217
 torchx/schedulers/local_scheduler.py,sha256=ttnxFDy48_DSYDEW-no27OirFZOyfrjwJ2S1MwBUi74,41929
 torchx/schedulers/lsf_scheduler.py,sha256=YS6Yel8tXJqLPxbcGz95lZG2nCi36AQXdNDyuBJePKg,17661
-torchx/schedulers/ray_scheduler.py,sha256=T-jsGSOa8O-h1kTUU7Q7Fk1RILL1Yzvuos_WFSQF8Fo,15795
 torchx/schedulers/slurm_scheduler.py,sha256=vZt102OxuTGj0ZE-V9dWbldtOyL2VbHcxADm_osL7Y4,31568
 torchx/schedulers/streams.py,sha256=8_SLezgnWgfv_zXUsJCUM34-h2dtv25NmZuxEwkzmxw,2007
-torchx/schedulers/ray/__init__.py,sha256=fE0IHi1JJpxsNVBNzWNee2thrNXFFRhY94c80RxNSIE,231
-torchx/schedulers/ray/ray_common.py,sha256=pyNYFvTKVwdjDAeCBNbPwAWwVNmlLOJWExfn90XY8u8,610
-torchx/schedulers/ray/ray_driver.py,sha256=RdaCLfth16ky-5PDVOWRe_RuheWJu9xufWux2F9T7iw,12302
-torchx/specs/__init__.py,sha256=Gw_2actqR_oWFtxEkGXCxGk_yrWK5JDZzwysyyqmXao,6438
-torchx/specs/api.py,sha256=wkhHOxeWH_tFO3npKqPhNg4VX2NH5gPIFEylkPBo3AU,41315
+torchx/specs/__init__.py,sha256=HU7OoXs7aBBi0IenB49QIONRzoG1Ovs1Qlm9KnsvqfE,6609
+torchx/specs/api.py,sha256=Rzv_Yx8yyseARbC928ZvqAZbvaXhJRDAbcyPsxiblF4,41543
 torchx/specs/builders.py,sha256=aozVl4q3h0mY5DDJCY1M1CyLC9SW66KJy8JIih8bZJo,13810
 torchx/specs/file_linter.py,sha256=6_aoeuS5d9UwXseKKfPgWNTwxj-f7G1i3uO9mQepti4,14402
 torchx/specs/finder.py,sha256=FcB6jQTNwnpc4OMV0F349kk0leu6O7JYjH_GW2d6GXE,17503
@@ -115,9 +103,9 @@ torchx/workspace/__init__.py,sha256=cZsKVvUWwDYcGhe6SCXQGBQfbk_yTnKEImOkI6xmu30,
 torchx/workspace/api.py,sha256=Ct_75VU94fsH9Rf1WRe-wJGpVgl5O05S_Dq_t2ArJWA,11348
 torchx/workspace/dir_workspace.py,sha256=npNW_IjUZm_yS5r-8hrRkH46ndDd9a_eApT64m1S1T4,2268
 torchx/workspace/docker_workspace.py,sha256=PFu2KQNVC-0p2aKJ-W_BKA9ZOmXdCY2ABEkCExp3udQ,10269
-torchx_nightly-2025.9.17.dist-info/LICENSE,sha256=WVHfXhFC0Ia8LTKt_nJVYobdqTJVg_4J3Crrfm2A8KQ,1721
-torchx_nightly-2025.9.17.dist-info/METADATA,sha256=yrw7Nu_XLhDmm0HDXN59bbVIADlRARi0yOUl6VnJnyU,6104
-torchx_nightly-2025.9.17.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-torchx_nightly-2025.9.17.dist-info/entry_points.txt,sha256=T328AMXeKI3JZnnxfkEew2ZcMN1oQDtkXjMz7lkV-P4,169
-torchx_nightly-2025.9.17.dist-info/top_level.txt,sha256=pxew3bc2gsiViS0zADs0jb6kC5v8o_Yy_85fhHj_J1A,7
-torchx_nightly-2025.9.17.dist-info/RECORD,,
+torchx_nightly-2025.9.19.dist-info/LICENSE,sha256=WVHfXhFC0Ia8LTKt_nJVYobdqTJVg_4J3Crrfm2A8KQ,1721
+torchx_nightly-2025.9.19.dist-info/METADATA,sha256=4S3QpZtzHIUqC2VdZK_czPBhOWnQHdbTvjhFyAdE94s,5693
+torchx_nightly-2025.9.19.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+torchx_nightly-2025.9.19.dist-info/entry_points.txt,sha256=T328AMXeKI3JZnnxfkEew2ZcMN1oQDtkXjMz7lkV-P4,169
+torchx_nightly-2025.9.19.dist-info/top_level.txt,sha256=pxew3bc2gsiViS0zADs0jb6kC5v8o_Yy_85fhHj_J1A,7
+torchx_nightly-2025.9.19.dist-info/RECORD,,

torchx/examples/pipelines/kfp/advanced_pipeline.py

@@ -1,289 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree.
-
-# pyre-strict
-
-"""
-Advanced KubeFlow Pipelines Example
-===================================
-
-This is an example pipeline using KubeFlow Pipelines built with only TorchX
-components.
-
-KFP adapters can be used transform the TorchX components directly into
-something that can be used within KFP.
-"""
-
-# %%
-# Input Arguments
-# ###############
-# Lets first define some arguments for the pipeline.
-
-import argparse
-import os.path
-import sys
-from typing import Dict
-
-import kfp
-import torchx
-from torchx import specs
-from torchx.components.dist import ddp as dist_ddp
-from torchx.components.serve import torchserve
-from torchx.components.utils import copy as utils_copy, python as utils_python
-from torchx.pipelines.kfp.adapter import container_from_app
-
-
-parser = argparse.ArgumentParser(description="example kfp pipeline")
-
-# %%
-# TorchX components are built around images. Depending on what scheduler
-# you're using this can vary but for KFP these images are specified as
-# docker containers. We have one container for the example apps and one for
-# the standard built in apps. If you modify the torchx example code you'll
-# need to rebuild the container before launching it on KFP
-
-
-parser.add_argument(
-    "--image",
-    type=str,
-    help="docker image to use for the examples apps",
-    default=torchx.IMAGE,
-)
-
-# %%
-# Most TorchX components use
-# `fsspec <https://filesystem-spec.readthedocs.io/en/latest/>`_ to abstract
-# away dealing with remote filesystems. This allows the components to take
-# paths like ``s3://`` to make it easy to use cloud storage providers.
-parser.add_argument(
-    "--output_path",
-    type=str,
-    help="path to place the data",
-    required=True,
-)
-parser.add_argument("--load_path", type=str, help="checkpoint path to load from")
-
-# %%
-# This example uses the torchserve for inference so we need to specify some
-# options. This assumes you have a TorchServe instance running in the same
-# Kubernetes cluster with with the service name ``torchserve`` in the default
-# namespace.
-#
-# See https://github.com/pytorch/serve/blob/master/kubernetes/README.md for info
-# on how to setup TorchServe.
-parser.add_argument(
-    "--management_api",
-    type=str,
-    help="path to the torchserve management API",
-    default="http://torchserve.default.svc.cluster.local:8081",
-)
-parser.add_argument(
-    "--model_name",
-    type=str,
-    help="the name of the inference model",
-    default="tiny_image_net",
-)
-
-# %% Parse the arguments, you'll need to set these accordingly if running from a
-# notebook.
-
-
-if "NOTEBOOK" in globals():
-    argv = [
-        "--output_path",
-        "/tmp/output",
-    ]
-else:
-    argv = sys.argv[1:]
-
-args: argparse.Namespace = parser.parse_args(argv)
-
-# %%
-# Creating the Components
-# #######################
-# The first step is downloading the data to somewhere we can work on it. For
-# this we can just the builtin copy component. This component takes two valid
-# fsspec paths and copies them from one to another. In this case we're using
-# http as the source and a file under the output_path as the output.
-
-
-data_path: str = os.path.join(args.output_path, "tiny-imagenet-200.zip")
-copy_app: specs.AppDef = utils_copy(
-    "http://cs231n.stanford.edu/tiny-imagenet-200.zip",
-    data_path,
-    image=args.image,
-)
-
-# %%
-# The next component is for data preprocessing. This takes in the raw data from
-# the previous operator and runs some transforms on it for use with the trainer.
-#
-# datapreproc outputs the data to a specified fsspec path. These paths are all
-# specified ahead of time so we have a fully static pipeline.
-
-
-processed_data_path: str = os.path.join(args.output_path, "processed")
-datapreproc_app: specs.AppDef = utils_python(
-    "--output_path",
-    processed_data_path,
-    "--input_path",
-    data_path,
-    "--limit",
-    "100",
-    image=args.image,
-    m="torchx.examples.apps.datapreproc.datapreproc",
-    cpu=1,
-    memMB=1024,
-)
-
-# %%
-# Next we'll create the trainer component that takes in the training data from the
-# previous datapreproc component. We've defined this in a separate component
-# file as you normally would.
-#
-# Having a separate component file allows you to launch your trainer from the
-# TorchX CLI via ``torchx run`` for fast iteration as well as run it from a
-# pipeline in an automated fashion.
-
-# make sure examples is on the path
-if "__file__" in globals():
-    sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
-
-
-logs_path: str = os.path.join(args.output_path, "logs")
-models_path: str = os.path.join(args.output_path, "models")
-
-trainer_app: specs.AppDef = dist_ddp(
-    *(
-        "--output_path",
-        models_path,
-        "--load_path",
-        args.load_path or "",
-        "--log_path",
-        logs_path,
-        "--data_path",
-        processed_data_path,
-        "--epochs",
-        str(1),
-    ),
-    image=args.image,
-    m="torchx.examples.apps.lightning.train",
-    j="1x1",
-    # per node resource settings
-    cpu=1,
-    memMB=3000,
-)
-
-# %%
-# To have the tensorboard path show up in KFPs UI we need to some metadata so
-# KFP knows where to consume the metrics from.
-#
-# This will get used when we create the KFP container.
-
-
-ui_metadata: Dict[str, object] = {
-    "outputs": [
-        {
-            "type": "tensorboard",
-            "source": os.path.join(logs_path, "lightning_logs"),
-        }
-    ]
-}
-
-# %%
-# For the inference, we're leveraging one of the builtin TorchX components. This
-# component takes in a model and uploads it to the TorchServe management API
-# endpoints.
-
-
-serve_app: specs.AppDef = torchserve(
-    model_path=os.path.join(models_path, "model.mar"),
-    management_api=args.management_api,
-    image=args.image,
-    params={
-        "model_name": args.model_name,
-        # set this to allocate a worker
-        # "initial_workers": 1,
-    },
-)
-
-# %%
-# For model interpretability we're leveraging a custom component stored in it's
-# own component file. This component takes in the output from datapreproc and
-# train components and produces images with integrated gradient results.
-
-interpret_path: str = os.path.join(args.output_path, "interpret")
-interpret_app: specs.AppDef = utils_python(
-    *(
-        "--load_path",
-        os.path.join(models_path, "last.ckpt"),
-        "--data_path",
-        processed_data_path,
-        "--output_path",
-        interpret_path,
-    ),
-    image=args.image,
-    m="torchx.examples.apps.lightning.interpret",
-)
-
-# %%
-# Pipeline Definition
-# ###################
-# The last step is to define the actual pipeline using the torchx components via
-# the KFP adapter and export the pipeline package that can be uploaded to a KFP
-# cluster.
-#
-# The KFP adapter currently doesn't track the input and outputs so the
-# containers need to have their dependencies specified via `.after()`.
-#
-# We call `.set_tty()` to make the logs from the components more responsive for
-# example purposes.
-
-
-def pipeline() -> None:
-    # container_from_app creates a KFP container from the TorchX app
-    # definition.
-    copy = container_from_app(copy_app)
-    copy.container.set_tty()
-
-    datapreproc = container_from_app(datapreproc_app)
-    datapreproc.container.set_tty()
-    datapreproc.after(copy)
-
-    # For the trainer we want to log that UI metadata so you can access
-    # tensorboard from the UI.
-    trainer = container_from_app(trainer_app, ui_metadata=ui_metadata)
-    trainer.container.set_tty()
-    trainer.after(datapreproc)
-
-    if False:
-        serve = container_from_app(serve_app)
-        serve.container.set_tty()
-        serve.after(trainer)
-
-    if False:
-        # Serve and interpret only require the trained model so we can run them
-        # in parallel to each other.
-        interpret = container_from_app(interpret_app)
-        interpret.container.set_tty()
-        interpret.after(trainer)
-
-
-kfp.compiler.Compiler().compile(
-    pipeline_func=pipeline,
-    package_path="pipeline.yaml",
-)
-
-with open("pipeline.yaml", "rt") as f:
-    print(f.read())
-
-# %%
-# Once this has all run you should have a pipeline file (typically
-# pipeline.yaml) that you can upload to your KFP cluster via the UI or
-# a kfp.Client.
-
-# sphinx_gallery_thumbnail_path = '_static/img/gallery-kfp.png'

torchx/examples/pipelines/kfp/dist_pipeline.py

@@ -1,71 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree.
-
-# pyre-strict
-
-"""
-Distributed KubeFlow Pipelines Example
-======================================
-
-This is an example KFP pipeline that uses resource_from_app to launch a
-distributed operator using the kubernetes/volcano job scheduler. This only works
-in Kubernetes KFP clusters with https://volcano.sh/en/docs/ installed on them.
-"""
-
-import kfp
-from torchx import specs
-from torchx.pipelines.kfp.adapter import resource_from_app
-
-
-def pipeline() -> None:
-    # First we define our AppDef for the component, we set
-    echo_app = specs.AppDef(
-        name="test-dist",
-        roles=[
-            specs.Role(
-                name="dist-echo",
-                image="alpine",
-                entrypoint="/bin/echo",
-                args=["hello dist!"],
-                num_replicas=3,
-            ),
-        ],
-    )
-
-    # To convert the TorchX AppDef into a KFP container we use
-    # the resource_from_app adapter. This takes generates a KFP Kubernetes
-    # resource operator definition from the TorchX app def and instantiates it.
-    echo_container: kfp.dsl.BaseOp = resource_from_app(echo_app, queue="default")
-
-
-# %%
-# To generate the pipeline definition file we need to call into the KFP compiler
-# with our pipeline function.
-
-kfp.compiler.Compiler().compile(
-    pipeline_func=pipeline,
-    package_path="pipeline.yaml",
-)
-
-with open("pipeline.yaml", "rt") as f:
-    print(f.read())
-
-# %%
-# Once this has all run you should have a pipeline file (typically
-# pipeline.yaml) that you can upload to your KFP cluster via the UI or
-# a kfp.Client.
-#
-# See the
-# `KFP SDK Examples <https://www.kubeflow.org/docs/components/pipelines/legacy-v1/tutorials/sdk-examples/#examples>`_
-# for more info on launching KFP pipelines.
-
-# %%
-# See the :ref:`examples_pipelines/kfp/advanced_pipeline:Advanced KubeFlow Pipelines Example` for how to chain multiple
-# components together and use builtin components.
-
-
-# sphinx_gallery_thumbnail_path = '_static/img/gallery-kfp.png'

torchx/examples/pipelines/kfp/intro_pipeline.py

@@ -1,83 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree.
-
-# pyre-strict
-
-"""
-Intro KubeFlow Pipelines Example
-================================
-
-This an introductory pipeline using KubeFlow Pipelines built with only TorchX
-components.
-
-TorchX is intended to allow making cross platform components. As such, we have
-a standard definition that uses adapters to convert it to the specific
-pipeline platform. This is an example of using the KFP adapter to run a TorchX
-component as part of a KubeFlow Pipeline.
-
-TorchX tries to leverage standard mechanisms wherever possible. For KFP we use
-the existing KFP pipeline definition syntax and add a single
-`component_from_app` conversion step to convert a TorchX component into one
-KFP can understand.
-
-Typically you have a separate component file but for this example we define the
-AppDef inline.
-"""
-
-import kfp
-from torchx import specs
-from torchx.pipelines.kfp.adapter import container_from_app
-
-
-def pipeline() -> None:
-    # First we define our AppDef for the component. AppDef is a core part of TorchX
-    # and can be used to describe complex distributed multi container apps or
-    # just a single node component like here.
-    echo_app: specs.AppDef = specs.AppDef(
-        name="examples-intro",
-        roles=[
-            specs.Role(
-                name="worker",
-                entrypoint="/bin/echo",
-                args=["Hello TorchX!"],
-                image="alpine",
-            )
-        ],
-    )
-
-    # To convert the TorchX AppDef into a KFP container we use
-    # the container_from_app adapter. This takes generates a KFP component
-    # definition from the TorchX app def and instantiates it into a container.
-    echo_container: kfp.dsl.ContainerOp = container_from_app(echo_app)
-
-
-# %%
-# To generate the pipeline definition file we need to call into the KFP compiler
-# with our pipeline function.
-
-kfp.compiler.Compiler().compile(
-    pipeline_func=pipeline,
-    package_path="pipeline.yaml",
-)
-
-with open("pipeline.yaml", "rt") as f:
-    print(f.read())
-
-# %%
-# Once this has all run you should have a pipeline file (typically
-# pipeline.yaml) that you can upload to your KFP cluster via the UI or
-# a kfp.Client.
-#
-# See the
-# `KFP SDK Examples <https://www.kubeflow.org/docs/components/pipelines/legacy-v1/tutorials/sdk-examples/#examples>`_
-# for more info on launching KFP pipelines.
-
-# %%
-# See the :ref:`examples_pipelines/kfp/advanced_pipeline:Advanced KubeFlow Pipelines Example` for how to chain multiple
-# components together and use builtin components.
-
-# sphinx_gallery_thumbnail_path = '_static/img/gallery-kfp.png'