cloud-submit 0.1__tar.gz

cloud_submit-0.1/.gitignore

@@ -0,0 +1,8 @@
+*.swp
+__pycache__
+dist
+
+docs/examples/*/temp
+docs/examples/*/images
+docs/examples/*/artifacts
+docs/examples/*/userconfig/default.yaml

cloud_submit-0.1/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2018 The Python Packaging Authority
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cloud_submit-0.1/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: cloud-submit
+Version: 0.1
+Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
+Author: Martin Wiebusch
+License-File: LICENSE
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.7
+Requires-Dist: click
+Requires-Dist: pyyaml

cloud_submit-0.1/README.md

@@ -0,0 +1,29 @@
+# cloud-submit
+
+A tool for packaging Python code with Docker and executing it in local and
+cloud environments.
+
+## Installation
+
+```bash
+$ pip install cloud-submit
+```
+
+## Documentation
+
+This package provides a Python module called `cloud_submit` as well as a
+command-line tool called `csub`.
+
+Have a look at the
+[examples](https://github.com/mwiebusch78/cloud-submit/tree/main/docs/examples)
+to see how to set up a project directory. To run any of the examples you will
+have to copy `userconfig/template.yaml` to `userconfig/default.yaml` and fill
+in the blanks.
+
+To run the AWS example you need to have the AWS CLI (v2) installed locally
+and you need an AWS account. The comments in `userconfig/template.yaml` tell
+you how you need to configure your AWS account so that the example can run.
+
+Run `csub --help` to explore the functionality of `csub` and its subcommands.
+
+Proper documentation will be added as soon as I have the time. (Promise!)

cloud_submit-0.1/docs/examples/aws/src/csub/build_config.py

@@ -0,0 +1,153 @@
+import datetime as dt
+import cloud_submit as cs
+
+# import AWS-specific environments that ship with cloud_submit.
+import cloud_submit.envs.aws
+from cloud_submit.envs.aws import LocalAWSEnv, RemoteAWSEnv
+
+# csub automatically adds your project's src directory to sys.path, so
+# you can import any modules defined there.
+from parameters import PARAMS
+
+
+PROJECT_NAME = 'basic-aws'
+
+
+def build_config(project_root, userconfig):
+    """Create an instance of cloud_submit.Config describing your project.
+
+    Note:
+      The csub command expects a function with this name defined in
+      src/csub/build_config.py. It calls this function to obtain a
+      cloud_submit.Config object which describes the environments, images,
+      artifacts and pipelines defined in your project.
+
+    Args:
+      project_root (str): The root directory of your project.
+      userconfig (dict): A dict holding user-specific information. This
+        corresponds to the content of the file userconfigs/default.yaml
+        in your project directory (or some other yaml file in that directory
+        if csub was called with the --user option).
+
+    Returns:
+      config (cloud_submit.Config): The config object describing the project.
+    """
+
+    # Images
+
+    images = [
+        cs.BaseImage(
+            name='base',
+            instructions=r"""
+            FROM python
+            RUN \
+              cd /opt && \
+              curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" \
+                -o "awscliv2.zip" && \
+              unzip awscliv2.zip && \
+              ./aws/install && \
+              rm -rf awscliv2.zip aws
+            RUN pip install --root-user-action=ignore polars numpy
+            """,
+        ),
+        cs.ExecutionImage(
+            name='exec',
+            parent='base',
+        ),
+    ]
+
+    # Artifacts
+
+    artifacts = [
+        cs.Artifact('train_data.parquet', kind='file', scope='run'),
+        cs.Artifact('coefficients.json', kind='file', scope='run'),
+        cs.Artifact('predictions.parquet', kind='file', scope='run'),
+    ]
+
+    # Pipelines
+
+    train_pipeline = cs.Pipeline(
+        name='train',
+        steps=[
+            cs.Step(
+                name='generate',
+                spec=cs.Spec(
+                    cpu=2,
+                    memory=6.5,
+                    disk=20,
+                ),
+                function='generate:generate_step',
+                image='exec',
+                params={
+                    'random_seed': PARAMS['random_seed'],
+                    'num_samples': PARAMS['num_samples'],
+                    'alpha': PARAMS['alpha'],
+                    'beta': PARAMS['beta'],
+                    'sigma': PARAMS['sigma'],
+                },
+                outputs={
+                    'train_data_path': cs.local('train_data.parquet'),
+                },
+            ),
+            cs.Step(
+                name='fit',
+                spec=cs.Spec(
+                    cpu=2,
+                    memory=6.5,
+                    disk=20,
+                ),
+                function='fit:fit_step',
+                image='exec',
+                inputs={
+                    'train_data_path': cs.local('train_data.parquet'),
+                },
+                outputs={
+                    'coefficients_path': cs.local('coefficients.json'),
+                    'predictions_path': cs.local('predictions.parquet'),
+                },
+            ),
+        ],
+    )
+
+    config = cs.Config(
+        project_name=PROJECT_NAME,
+        user_name=userconfig['username'],
+        project_root=project_root,
+        images=images,
+        artifacts=artifacts,
+        pipelines=[train_pipeline],
+        environments=[
+            LocalAWSEnv(
+                name='local',
+                project=PROJECT_NAME,
+                user=userconfig['username'],
+                aws_account_id=userconfig['aws_account_id'],
+                aws_region=userconfig['aws_region'],
+                aws_profile=userconfig['aws_profile'],
+                docker_namespace=userconfig['docker_namespace'],
+                s3_bucket=userconfig['s3_bucket'],
+                s3_prefix=userconfig['s3_prefix'],
+            ),
+            RemoteAWSEnv(
+                name='remote',
+                project=PROJECT_NAME,
+                user=userconfig['username'],
+                aws_account_id=userconfig['aws_account_id'],
+                aws_region=userconfig['aws_region'],
+                aws_profile=userconfig['aws_profile'],
+                docker_namespace=userconfig['docker_namespace'],
+                s3_bucket=userconfig['s3_bucket'],
+                s3_prefix=userconfig['s3_prefix'],
+                ecs_cluster_arn=userconfig['ecs_cluster_arn'],
+                ecs_capacity_provider=userconfig['ecs_capacity_provider'],
+                ecs_infrastructure_role_arn=
+                    userconfig['ecs_infrastructure_role_arn'],
+                ecs_execution_role_arn=userconfig['ecs_execution_role_arn'],
+                ecs_task_role_arn=userconfig['ecs_task_role_arn'],
+                stepfunctions_role_arn=userconfig['stepfunctions_role_arn'],
+            ),
+        ],
+    )
+
+    return config
+

cloud_submit-0.1/docs/examples/aws/src/fit.py

@@ -0,0 +1,42 @@
+import os
+import json
+
+import numpy as np
+import polars as pl
+
+
+def fit_step(
+    train_data_path,
+    coefficients_path,
+    predictions_path,
+):
+    print(f'Fitting model.')
+
+    train_data = pl.read_parquet(os.path.join(train_data_path, '*.parquet'))
+    x = train_data.get_column('x').to_numpy()
+    y = train_data.get_column('y').to_numpy()
+
+    # add intercept term
+    intercept = np.ones(len(x))
+    x = np.stack([intercept, x], axis=1)
+
+    # fit linear regression model
+    coeffs = np.linalg.solve(np.dot(x.T, x), np.dot(x.T, y))
+
+    # compute in-sample predictions
+    preds = np.dot(x, coeffs)
+    preds = train_data.with_columns(pl.Series('pred', preds))
+
+    # save coefficients
+    with open(coefficients_path, 'w') as stream:
+        json.dump(
+            {
+                'alpha': float(coeffs[0]),
+                'beta': float(coeffs[1]),
+            },
+            stream,
+        )
+
+    # save in-sample predictions
+    preds.write_parquet(predictions_path)
+

cloud_submit-0.1/docs/examples/aws/src/generate.py

@@ -0,0 +1,32 @@
+import os
+
+import numpy as np
+import polars as pl
+
+
+def generate_step(
+    random_seed,
+    num_samples,
+    alpha,
+    beta,
+    sigma,
+    train_data_path,
+):
+    print(f'Generating training data.')
+
+    rng = np.random.default_rng(random_seed)
+    x = rng.normal(size=num_samples)
+    y = alpha + beta*x + sigma*rng.normal(size=num_samples)
+
+    train_data = pl.DataFrame([
+        pl.Series('x', x),
+        pl.Series('y', y),
+    ])
+
+    os.mkdir(train_data_path)
+    split_index = len(train_data)//2
+    train_data[:split_index].write_parquet(
+        os.path.join(train_data_path, 'part0.parquet'))
+    train_data[split_index:].write_parquet(
+        os.path.join(train_data_path, 'part1.parquet'))
+

cloud_submit-0.1/docs/examples/aws/src/parameters.py

@@ -0,0 +1,8 @@
+PARAMS = {
+    # Parameters for data generation.
+    'random_seed': 42,   # The random seed.
+    'num_samples': 100,  # The number of training samples to generate.
+    'alpha': 1.0,        # The true value of the intercept coefficient.
+    'beta': 2.0,         # The true value of the linear coefficient.
+    'sigma': 0.1,        # The standard deviation of the error term.
+}

cloud_submit-0.1/docs/examples/aws/userconfig/template.yaml

@@ -0,0 +1,112 @@
+# Your username is used for keeping your artifacts, images and cloud resources
+# separate from those of other users.
+username: # TODO: choose a user name.
+
+# This is the ID of your AWS account. Make sure you specify it as a string,
+# e.g. "123456789012".
+aws_account_id: # TODO: add you AWS account ID.
+
+# This is the region where your AWS resources were/are created
+# (e.g. eu-north-1).
+aws_region: # TODO: add your AWS region.
+
+# This is the name of the profile that you want to use to authenticate with
+# AWS. Look at ~/.aws/config to see what profiles you have configured.
+# Normally there should be one called "default" which is the one that's used
+# when you run aws commands without a --profile option.
+aws_profile: default
+
+# This is the docker namespace that will be used to store docker images
+# needed in your project. You have to set up an Elastic Container Registy
+# in AWS in the region you specified under `aws_region`. Images under this
+# namespace must be mutable and you must allow repo creation on push for
+# this namespace. It is also recommended to enable blob mounting on the
+# registry.
+docker_namespace: csub
+
+# This is the name of the S3 bucket where your artifacts will be stored.
+s3_bucket: # TODO: add the name of your S3 bucket.
+
+# This is the S3 prefix under which your artifacts will be stored.
+s3_prefix: csub
+
+# If you want to run pipelines in AWS you have to set up a cluster in
+# Elastic Container Service (ECS). The cluster must allow the use
+# of managed EC2 instances (second option in the cluster creation form
+# in the AWS console).
+#
+# If you only want to do local runs you can use an string value for this
+# and all following parameters.
+ecs_cluster_arn: n/a # TODO: add the ARN of your ECS cluster.
+
+# The capacity provider should have been created automatically when you
+# created the cluster. To find its name, view your cluster in the AWS console
+# and look at the "Infrastructure" tab. Note that we need the *name* of the
+# capacity provider, not its ARN.
+ecs_capacity_provider: n/a # TODO: add the name of your capacity provider.
+
+# The infrastructure role is the role used to manage the managed EC2 instances
+# in your cluster. You should have created this role when you created the
+# cluster.
+ecs_infrastructure_role_arn: n/a # TODO: add the ARN of your infrastructure role
+
+# To create the next two roles you can open the form for creating a task
+# definition in ECS. In the "Task roles" sections it asks you to enter a
+# "Task role" and a "Task execution role". Next to the dropdowns there are
+# "Create new role" buttons which let you create those roles on the fly.
+# You do not need to complete the task definition after you have created
+# them.
+ecs_task_role_arn: n/a # TODO: add the ARN of your task role
+ecs_execution_role_arn: n/a # TODO: add the ARN of your task execution role
+
+# The stepfunction role is used to orchestrate your ECS tasks via AWS
+# Stepfunctions. This role should have the AWSLambdaRole policy and the
+# following custom inline policy:
+#
+# {
+#   "Version": "2012-10-17",
+#   "Statement": [
+#     {
+#       "Sid": "ECSOrchestrationPermissions",
+#       "Effect": "Allow",
+#       "Action": [
+#         "ecs:RegisterTaskDefinition",
+#         "ecs:DeregisterTaskDefinition",
+#         "ecs:RunTask",
+#         "ecs:DescribeTasks",
+#         "ecs:StopTask"
+#       ],
+#       "Resource": "*"
+#     },
+#     {
+#       "Sid": "PassRolesToECSInstances",
+#       "Effect": "Allow",
+#       "Action": "iam:PassRole",
+#       "Resource": [
+#         "{INSTANCE_ROLE_ARN}",
+#         "{INFRASTRUCTURE_ROLE_ARN}",
+#         "{TASK_EXECUTION_ROLE_ARN}",
+#         "{TASK_ROLE_ARN}"
+#       ]
+#     },
+#     {
+#       "Sid": "EventBridgeSyncTracking",
+#       "Effect": "Allow",
+#       "Action": [
+#         "events:PutTargets",
+#         "events:PutRule",
+#         "events:DescribeRule"
+#       ],
+#       "Resource": "arn:aws:events:*:*:rule/StepFunctionsGetEventsForECSTaskRule"
+#     }
+#   ]
+# }
+#
+# Here {INFRASTUCTURE_ROLE_ARN}, {TASK_EXECUTION_ROLE_ARN} and {TASK_ROLE_ARN}
+# are the role ARNs you specified earlier and {INSTANCE_ROLE_ARN} is the ARN
+# of another role that you set up when you created your cluster.
+#
+# Also note that the user you authenticate as when you submit a job (i.e. the
+# user of the profile you specified under `aws_profile`) must have permission
+# to assume the stepfunction role you specify here.
+stepfunctions_role_arn: n/a # TODO: add your stepfunctions role ARN.

cloud_submit-0.1/docs/examples/basic/src/csub/build_config.py

@@ -0,0 +1,102 @@
+import datetime as dt
+import cloud_submit as cs
+
+# csub automatically adds your project's src directory to sys.path, so
+# you can import any modules defined there.
+from parameters import PARAMS
+
+
+PROJECT_NAME = 'basic'
+
+
+def build_config(project_root, userconfig):
+    """Create an instance of cloud_submit.Config describing your project.
+
+    Note:
+      The csub command expects a function with this name defined in
+      src/csub/build_config.py. It calls this function to obtain a
+      cloud_submit.Config object which describes the environments, images,
+      artifacts and pipelines defined in your project.
+
+    Args:
+      project_root (str): The root directory of your project.
+      userconfig (dict): A dict holding user-specific information. This
+        corresponds to the content of the file userconfigs/default.yaml
+        in your project directory (or some other yaml file in that directory
+        if csub was called with the --user option).
+
+    Returns:
+      config (cloud_submit.Config): The config object describing the project.
+    """
+
+    # Images
+
+    images = [
+        cs.BaseImage(
+            name='base',
+            instructions="""
+            FROM python
+            RUN pip install --root-user-action=ignore polars numpy
+            """,
+        ),
+        cs.ExecutionImage(
+            name='exec',
+            parent='base',
+        ),
+    ]
+
+    # Artifacts
+
+    artifacts = [
+        cs.Artifact('train_data.parquet', kind='file', scope='run'),
+        cs.Artifact('coefficients.json', kind='file', scope='run'),
+        cs.Artifact('predictions.parquet', kind='file', scope='run'),
+    ]
+
+    # Pipelines
+
+    train_pipeline = cs.Pipeline(
+        name='train',
+        steps=[
+            cs.Step(
+                name='generate',
+                function='generate:generate_step',
+                image='exec',
+                params={
+                    'random_seed': PARAMS['random_seed'],
+                    'num_samples': PARAMS['num_samples'],
+                    'alpha': PARAMS['alpha'],
+                    'beta': PARAMS['beta'],
+                    'sigma': PARAMS['sigma'],
+                },
+                outputs={
+                    'train_data_path': cs.local('train_data.parquet'),
+                },
+            ),
+            cs.Step(
+                name='fit',
+                function='fit:fit_step',
+                image='exec',
+                inputs={
+                    'train_data_path': cs.local('train_data.parquet'),
+                },
+                outputs={
+                    'coefficients_path': cs.local('coefficients.json'),
+                    'predictions_path': cs.local('predictions.parquet'),
+                },
+            ),
+        ],
+    )
+
+
+    config = cs.Config(
+        project_name=PROJECT_NAME,
+        user_name=userconfig['username'],
+        project_root=project_root,
+        images=images,
+        artifacts=artifacts,
+        pipelines=[train_pipeline],
+    )
+
+    return config
+

cloud_submit-0.1/docs/examples/basic/src/fit.py

@@ -0,0 +1,41 @@
+import json
+
+import numpy as np
+import polars as pl
+
+
+def fit_step(
+    train_data_path,
+    coefficients_path,
+    predictions_path,
+):
+    print(f'Fitting model.')
+
+    train_data = pl.read_parquet(train_data_path)
+    x = train_data.get_column('x').to_numpy()
+    y = train_data.get_column('y').to_numpy()
+
+    # add intercept term
+    intercept = np.ones(len(x))
+    x = np.stack([intercept, x], axis=1)
+
+    # fit linear regression model
+    coeffs = np.linalg.solve(np.dot(x.T, x), np.dot(x.T, y))
+
+    # compute in-sample predictions
+    preds = np.dot(x, coeffs)
+    preds = train_data.with_columns(pl.Series('pred', preds))
+
+    # save coefficients
+    with open(coefficients_path, 'w') as stream:
+        json.dump(
+            {
+                'alpha': float(coeffs[0]),
+                'beta': float(coeffs[1]),
+            },
+            stream,
+        )
+
+    # save in-sample predictions
+    preds.write_parquet(predictions_path)
+

cloud_submit-0.1/docs/examples/basic/src/generate.py

@@ -0,0 +1,24 @@
+import numpy as np
+import polars as pl
+
+
+def generate_step(
+    random_seed,
+    num_samples,
+    alpha,
+    beta,
+    sigma,
+    train_data_path,
+):
+    print(f'Generating training data.')
+
+    rng = np.random.default_rng(random_seed)
+    x = rng.normal(size=num_samples)
+    y = alpha + beta*x + sigma*rng.normal(size=num_samples)
+
+    train_data = pl.DataFrame([
+        pl.Series('x', x),
+        pl.Series('y', y),
+    ])
+    train_data.write_parquet(train_data_path)
+

cloud_submit-0.1/docs/examples/basic/src/parameters.py

@@ -0,0 +1,8 @@
+PARAMS = {
+    # Parameters for data generation.
+    'random_seed': 42,   # The random seed.
+    'num_samples': 100,  # The number of training samples to generate.
+    'alpha': 1.0,        # The true value of the intercept coefficient.
+    'beta': 2.0,         # The true value of the linear coefficient.
+    'sigma': 0.1,        # The standard deviation of the error term.
+}

cloud_submit-0.1/docs/examples/basic/userconfig/template.yaml

@@ -0,0 +1,7 @@
+# Your username is used for keeping your artifacts, images and cloud resources
+# separate from those of other users.
+#
+# (In this example we only use a single local environment, so there are no
+# other users here. But the username becomes very important when you're running
+# jobs in the cloud on an account that you share with other people.)
+userconfig: # TODO: choose a user name.