orient_express 0.3.4__tar.gz → 0.4.2__tar.gz

{orient_express-0.3.4 → orient_express-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: orient_express
-Version: 0.3.4
+Version: 0.4.2
 Summary: A library to simplify model deployment to Vertex AI
 Author: Alexey Zankevich
 Author-email: alex.zankevich@shiftsmart.com
@@ -12,7 +12,6 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: google-cloud-aiplatform
 Requires-Dist: google-cloud-storage
-Requires-Dist: jupyter (>=1.1.1,<2.0.0)
 Requires-Dist: pandas
 Description-Content-Type: text/markdown
 
@@ -172,3 +171,81 @@ df = pd.DataFrame(titanic_data)
 model_wrapper.remote_predict(df)
 ```
 
+## Pipeline Deployment Function
+
+Orient express library also have a helper function to simplify Vertex AI pipeline deployment.
+
+Create `deploy.py` script
+```python
+
+from orient_express.deployment import deploy_pipeline
+
+import argparse
+import conf
+
+from pipeline import pipeline
+from orient_express.deployment import deploy_pipeline
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--run-type", required=True)
+
+    args = parser.parse_args()
+    deploy_pipeline(run_type=args.run_type,
+                    pipeline_dsl=pipeline,
+                    pipeline_root=conf.PIPELINE_ROOT,
+                    pipeline_name=conf.PIPELINE_NAME,
+                    pipeline_display_name=conf.PIPELINE_DISPLAY_NAME,
+                    pipeline_schedule_name=conf.SCHEDULE_NAME,
+                    gcp_project=conf.PROJECT_ID,
+                    gcp_location='us-central1',
+                    gcp_service_account=conf.SERVICE_ACCOUNT,
+                    gcp_network=conf.NETWORK_NAME,
+                    gcp_labels={"team": "ml"})
+```
+
+And conf.py, make sure to replace the sample values with yours.
+```python
+
+import os
+
+BASE_PATH = "gs://pipelines-bucket/vertex-ai/pipelines"
+
+PIPELINE_NAME = "my-pipeline"
+PIPELINE_ROOT = f"{BASE_PATH}/{PIPELINE_NAME}"
+PIPELINE_TEMP_ROOT = f"{BASE_PATH}/{PIPELINE_NAME}-temp"
+
+PIPELINE_DISPLAY_NAME = "My Pipeline"
+PIPELINE_DESCRIPTION = "My example pipeline"
+
+NETWORK_NAME = "project network id"
+
+DOCKER_IMAGE = "us-docker.pkg.dev/my-project/my-artifactory/my-pipeline:latest
+BASE_IMAGE = "python:3.11"
+PROJECT_ID = "my-project"
+PROJECT_REGION = "us-central1"
+
+SERVICE_ACCOUNT = "my-service-account@my-project.iam.gserviceaccount.com"
+SCHEDULE_NAME = "My Pipeline"
+```
+
+For testing it on a local machine, make sure to authorize to GCP first
+```shell
+
+gcloud auth application-default login
+
+```
+
+Finally, run the pipeline (it will execute once)
+```shell
+
+python deploy.py --run-type single-run
+```
+
+Or, create a scheduler to run continuously
+```shell
+
+python deploy.py --run-type scheduled
+```
+

{orient_express-0.3.4 → orient_express-0.4.2}/README.md

@@ -153,3 +153,81 @@ df = pd.DataFrame(titanic_data)
 
 model_wrapper.remote_predict(df)
 ```
+
+## Pipeline Deployment Function
+
+Orient express library also have a helper function to simplify Vertex AI pipeline deployment.
+
+Create `deploy.py` script
+```python
+
+from orient_express.deployment import deploy_pipeline
+
+import argparse
+import conf
+
+from pipeline import pipeline
+from orient_express.deployment import deploy_pipeline
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--run-type", required=True)
+
+    args = parser.parse_args()
+    deploy_pipeline(run_type=args.run_type,
+                    pipeline_dsl=pipeline,
+                    pipeline_root=conf.PIPELINE_ROOT,
+                    pipeline_name=conf.PIPELINE_NAME,
+                    pipeline_display_name=conf.PIPELINE_DISPLAY_NAME,
+                    pipeline_schedule_name=conf.SCHEDULE_NAME,
+                    gcp_project=conf.PROJECT_ID,
+                    gcp_location='us-central1',
+                    gcp_service_account=conf.SERVICE_ACCOUNT,
+                    gcp_network=conf.NETWORK_NAME,
+                    gcp_labels={"team": "ml"})
+```
+
+And conf.py, make sure to replace the sample values with yours.
+```python
+
+import os
+
+BASE_PATH = "gs://pipelines-bucket/vertex-ai/pipelines"
+
+PIPELINE_NAME = "my-pipeline"
+PIPELINE_ROOT = f"{BASE_PATH}/{PIPELINE_NAME}"
+PIPELINE_TEMP_ROOT = f"{BASE_PATH}/{PIPELINE_NAME}-temp"
+
+PIPELINE_DISPLAY_NAME = "My Pipeline"
+PIPELINE_DESCRIPTION = "My example pipeline"
+
+NETWORK_NAME = "project network id"
+
+DOCKER_IMAGE = "us-docker.pkg.dev/my-project/my-artifactory/my-pipeline:latest
+BASE_IMAGE = "python:3.11"
+PROJECT_ID = "my-project"
+PROJECT_REGION = "us-central1"
+
+SERVICE_ACCOUNT = "my-service-account@my-project.iam.gserviceaccount.com"
+SCHEDULE_NAME = "My Pipeline"
+```
+
+For testing it on a local machine, make sure to authorize to GCP first
+```shell
+
+gcloud auth application-default login
+
+```
+
+Finally, run the pipeline (it will execute once)
+```shell
+
+python deploy.py --run-type single-run
+```
+
+Or, create a scheduler to run continuously
+```shell
+
+python deploy.py --run-type scheduled
+```

orient_express-0.4.2/orient_express/deployment.py

@@ -0,0 +1,99 @@
+import logging
+import argparse
+from typing import Optional
+
+from kfp import compiler, dsl
+
+import conf
+import google.cloud.aiplatform as aip
+
+
+def delete_old_schedules(schedule_name):
+    for schedule in aip.PipelineJobSchedule.list(
+        filter=f'display_name="{schedule_name}"', order_by="create_time desc"
+    ):
+        logging.info(f"Deleting the old schedule {schedule}")
+        schedule.delete()
+
+
+def deploy_pipeline(
+    run_type: str,
+    pipeline_dsl: str,
+    pipeline_root: str,
+    pipeline_name: str,
+    pipeline_display_name: str,
+    pipeline_schedule_name: str,
+    gcp_project: str,
+    gcp_location: str,
+    gcp_service_account: str,
+    gcp_network: str,
+    gcp_labels: Optional[dict[str, str]] = {},
+    cron_string: Optional[str] = "*/15 * * * *",
+    template_path: Optional[str] = "pipeline.yaml",
+    pipeline_job_parameters: Optional[dict] = {},
+    max_concurrent_run_count: Optional[int] = 1,
+):
+    """
+    :param run_type: "single-run" or "scheduled"
+    :param pipeline_dsl: pipeline DSL object
+    :param pipeline_root: a root folder for the pipelines to keep their intermediate data
+    :param pipeline_name: pipeline name
+    :param pipeline_display_name: human readable name of the pipeline
+    :param cron_string: a string that defines how often the pipeline should run
+    :param template_path: a path to a template yaml file (pipeline.yaml by default)
+    :param gcp_labels: additional labels to add to the pipeline
+    :param pipeline_job_parameters: additional parameters to pass to the pipeline.
+        Make sure to set the correct signatures of the pipeline function
+    :param max_concurrent_run_count: maximum number of concurrent runs
+    """
+    if run_type == "scheduled":
+        is_scheduled_run = True
+    elif run_type == "single-run":
+        is_scheduled_run = False
+    else:
+        raise Exception(f"Unknown run type {run_type}")
+
+    print("### Deployment Summary")
+    compiler.Compiler().compile(pipeline_dsl, package_path=template_path)
+
+    aip.init(
+        project=gcp_project,
+        location=gcp_location,
+        service_account=gcp_service_account,
+    )
+
+    labels = {"run_type": run_type, "service": pipeline_name}
+    labels.update(gcp_labels)
+
+    # Prepare the pipeline job
+    job = aip.PipelineJob(
+        enable_caching=False,
+        display_name=pipeline_display_name,
+        template_path=template_path,
+        pipeline_root=pipeline_root,
+        location=gcp_location,
+        labels=labels,
+        parameter_values=pipeline_job_parameters,
+    )
+    if is_scheduled_run:
+        delete_old_schedules(pipeline_schedule_name)
+
+        pipeline_job_schedule = aip.PipelineJobSchedule(
+            pipeline_job=job, display_name=pipeline_schedule_name
+        )
+
+        pipeline_job_schedule.create(
+            cron=cron_string,
+            max_concurrent_run_count=max_concurrent_run_count,
+            service_account=gcp_service_account,
+            network=gcp_network,
+        )
+    else:
+        job.submit(
+            service_account=gcp_service_account,
+            network=gcp_network,
+        )
+
+
+def validate_pipeline(pipeline_dsl, package_path="pipeline.yaml"):
+    compiler.Compiler().compile(pipeline_dsl, package_path=package_path)

{orient_express-0.3.4 → orient_express-0.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "orient_express"
-version = "0.3.4"
+version = "0.4.2"
 description = "A library to simplify model deployment to Vertex AI"
 authors = ["Alexey Zankevich <alex.zankevich@shiftsmart.com>"]
 readme = "README.md"
@@ -10,7 +10,6 @@ python = ">=3.9,<3.13"
 google-cloud-aiplatform = "*"
 google-cloud-storage = "*"
 pandas = "*"
-jupyter = "^1.1.1"
 
 [tool.poetry.dev-dependencies]
 black = "24.10.0"