aigear 0.0.1__tar.gz

aigear-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 RetailAI Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

aigear-0.0.1/PKG-INFO

@@ -0,0 +1,195 @@
+Metadata-Version: 2.4
+Name: aigear
+Version: 0.0.1
+Summary: Machine learning microservices based on gRPC
+Author: Retail AI Groups Inc.
+License: MIT License
+        
+        Copyright (c) 2024 RetailAI Inc.
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/retail-ai-inc/gear
+Project-URL: Repository, https://github.com/retail-ai-inc/aigear.git
+Project-URL: Issues, https://github.com/retail-ai-inc/aigear/issues
+Keywords: MLOps,AI,ML,Model Serving,Model Deployment,Model Training
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: Unix
+Classifier: Operating System :: MacOS
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aigear[common]
+Requires-Dist: aigear[docker]
+Requires-Dist: aigear[msgrpc]
+Requires-Dist: aigear[gcp]
+Provides-Extra: common
+Requires-Dist: tabulate>=0.9; extra == "common"
+Requires-Dist: cloudpickle>=2.0.0; extra == "common"
+Requires-Dist: pydantic>=2.11.9; extra == "common"
+Requires-Dist: datamodel_code_generator>=0.34.0; extra == "common"
+Provides-Extra: docker
+Requires-Dist: docker>=6.13; extra == "docker"
+Provides-Extra: msgrpc
+Requires-Dist: grpcio>=1.54.2; extra == "msgrpc"
+Requires-Dist: protobuf>=4.23.3; extra == "msgrpc"
+Requires-Dist: grpcio-health-checking>=1.56.0; extra == "msgrpc"
+Requires-Dist: sentry-sdk>=1.29.2; extra == "msgrpc"
+Provides-Extra: gcp
+Requires-Dist: google-cloud-logging>=3.0.0; extra == "gcp"
+Requires-Dist: google-cloud-secret-manager>=2.24.0; extra == "gcp"
+Requires-Dist: google-cloud-storage>=3.2.0; extra == "gcp"
+Requires-Dist: google-cloud-compute>=1.38.0; extra == "gcp"
+Dynamic: license-file
+
+# Aigear
+Aigear is an MLOps work orchestration framework that is serverless and aims to help you build pipelines more quickly.
+It will not change the programming style of Python and provides an optimized out-of-the-box toolbox.
+
+Aigear believes that everything is a task, and it allows you to make any extensions according to the task.
+Within Aigear, it performs topology analysis on tasks to achieve optimal parallel execution.
+
+You need to note that aigear only executes functions created by yourself in parallel.
+
+## Getting started
+
+Aigear requires Python 3.8 or later. To install the latest or upgrade to the latest version of Aigear, run the following command:
+
+```bash
+pip install -U aigear
+```
+
+With just two decorators(`@task` and `@workflow`), you can create a pipeline.
+Please refer to `/aigear/example/pipeline`.
+
+Here is an example of iris classification:
+```python
+from sklearn.datasets import load_iris
+from sklearn.model_selection import train_test_split
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import accuracy_score
+from aigear.pipeline import workflow, task
+import pickle
+import json
+import os
+
+
+@task
+def load_data():
+    iris = load_iris()
+    return iris
+
+
+@task
+def split_dataset(iris):
+    X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2, random_state=42)
+    return X_train, X_test, y_train, y_test
+
+
+@task
+def fit_model(X_train, y_train):
+    clf = LogisticRegression(random_state=0, max_iter=1000, solver="lbfgs")
+    model = clf.fit(X_train, y_train)
+    return model
+
+
+@task
+def evaluate(clf, X_test, y_test):
+    y_pred = clf.predict(X_test)
+    accuracy = accuracy_score(y_test, y_pred)
+    return y_pred, accuracy
+
+
+@task
+def get_env_variables():
+    with open("env.json", "r") as f:
+        env = json.load(f)
+    model_path = env.get("grpc", {}).get("servers", {}).get("demo", {}).get("modelPath")
+    return model_path
+
+
+@task
+def save_model(model, model_path):
+    with open(model_path, "wb") as md:
+        pickle.dump(model, md)
+
+
+@workflow
+def my_pipeline():
+    iris = load_data()
+    X_train, X_test, y_train, y_test = split_dataset(iris)
+    model = fit_model(X_train, y_train)
+    y_pred, accuracy = evaluate(model, X_test, y_test)
+    print("accuracy：", accuracy)
+
+    model_path = get_env_variables()
+    save_model(model, model_path)
+
+
+if __name__ == '__main__':
+    # # Run the pipeline directly
+    # my_pipeline()
+    
+    # Run pipeline in parallel
+    my_pipeline.run_in_executor()
+```
+
+Similarly, the task can also be executed in two ways.
+```python
+if __name__ == "__main__":
+    # # Run the pipeline directly
+    # load_data()
+    
+    # Run pipeline in parallel
+    load_data.run_in_executor()
+```
+
+If you want to deploy a pipeline or create a model service in local docker, it's quite simple.
+
+```python
+if __name__ == '__main__':
+    current_directory = os.getcwd()
+    volumes = {
+        current_directory: {'bind': "/pipeline", 'mode': 'rw'}
+    }
+    hostname = "demo-host"
+    ports = {'50051/tcp': 50051}
+    service_dir = "demo"
+
+    my_pipeline.deploy(
+        volumes=volumes,
+        skip_build_image=False
+    ).to_service(
+        hostname=hostname,
+        ports=ports,
+        volumes=volumes,
+        tag=service_dir,
+    )
+```
+
+

aigear-0.0.1/README.md

@@ -0,0 +1,126 @@
+# Aigear
+Aigear is an MLOps work orchestration framework that is serverless and aims to help you build pipelines more quickly.
+It will not change the programming style of Python and provides an optimized out-of-the-box toolbox.
+
+Aigear believes that everything is a task, and it allows you to make any extensions according to the task.
+Within Aigear, it performs topology analysis on tasks to achieve optimal parallel execution.
+
+You need to note that aigear only executes functions created by yourself in parallel.
+
+## Getting started
+
+Aigear requires Python 3.8 or later. To install the latest or upgrade to the latest version of Aigear, run the following command:
+
+```bash
+pip install -U aigear
+```
+
+With just two decorators(`@task` and `@workflow`), you can create a pipeline.
+Please refer to `/aigear/example/pipeline`.
+
+Here is an example of iris classification:
+```python
+from sklearn.datasets import load_iris
+from sklearn.model_selection import train_test_split
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import accuracy_score
+from aigear.pipeline import workflow, task
+import pickle
+import json
+import os
+
+
+@task
+def load_data():
+    iris = load_iris()
+    return iris
+
+
+@task
+def split_dataset(iris):
+    X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2, random_state=42)
+    return X_train, X_test, y_train, y_test
+
+
+@task
+def fit_model(X_train, y_train):
+    clf = LogisticRegression(random_state=0, max_iter=1000, solver="lbfgs")
+    model = clf.fit(X_train, y_train)
+    return model
+
+
+@task
+def evaluate(clf, X_test, y_test):
+    y_pred = clf.predict(X_test)
+    accuracy = accuracy_score(y_test, y_pred)
+    return y_pred, accuracy
+
+
+@task
+def get_env_variables():
+    with open("env.json", "r") as f:
+        env = json.load(f)
+    model_path = env.get("grpc", {}).get("servers", {}).get("demo", {}).get("modelPath")
+    return model_path
+
+
+@task
+def save_model(model, model_path):
+    with open(model_path, "wb") as md:
+        pickle.dump(model, md)
+
+
+@workflow
+def my_pipeline():
+    iris = load_data()
+    X_train, X_test, y_train, y_test = split_dataset(iris)
+    model = fit_model(X_train, y_train)
+    y_pred, accuracy = evaluate(model, X_test, y_test)
+    print("accuracy：", accuracy)
+
+    model_path = get_env_variables()
+    save_model(model, model_path)
+
+
+if __name__ == '__main__':
+    # # Run the pipeline directly
+    # my_pipeline()
+    
+    # Run pipeline in parallel
+    my_pipeline.run_in_executor()
+```
+
+Similarly, the task can also be executed in two ways.
+```python
+if __name__ == "__main__":
+    # # Run the pipeline directly
+    # load_data()
+    
+    # Run pipeline in parallel
+    load_data.run_in_executor()
+```
+
+If you want to deploy a pipeline or create a model service in local docker, it's quite simple.
+
+```python
+if __name__ == '__main__':
+    current_directory = os.getcwd()
+    volumes = {
+        current_directory: {'bind': "/pipeline", 'mode': 'rw'}
+    }
+    hostname = "demo-host"
+    ports = {'50051/tcp': 50051}
+    service_dir = "demo"
+
+    my_pipeline.deploy(
+        volumes=volumes,
+        skip_build_image=False
+    ).to_service(
+        hostname=hostname,
+        ports=ports,
+        volumes=volumes,
+        tag=service_dir,
+    )
+```
+
+

aigear-0.0.1/pyproject.toml

@@ -0,0 +1,91 @@
+[build-system]
+requires = ["setuptools>=46.4.0", "wheel>=0.36.2"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "aigear"
+version = "0.0.1"
+#dynamic = ["version"]
+description = "Machine learning microservices based on gRPC"
+keywords = ["MLOps", "AI", "ML", "Model Serving", "Model Deployment", "Model Training"]
+authors = [{ name = "Retail AI Groups Inc." }]
+readme = "README.md"
+license = { file = 'LICENSE' }
+requires-python = ">=3.8"
+classifiers = [
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: Unix",
+    "Operating System :: MacOS",
+]
+dependencies = [
+    "aigear[common]",
+    "aigear[docker]",
+    "aigear[msgrpc]",
+    "aigear[gcp]",
+]
+
+[project.urls]
+Homepage = "https://github.com/retail-ai-inc/gear"
+Repository = "https://github.com/retail-ai-inc/aigear.git"
+Issues = "https://github.com/retail-ai-inc/aigear/issues"
+
+[project.optional-dependencies]
+common = [
+    "tabulate>=0.9",
+    "cloudpickle>=2.0.0",
+    "pydantic>=2.11.9",
+    "datamodel_code_generator>=0.34.0",
+]
+docker = [
+    "docker>=6.13",
+]
+msgrpc = [
+    "grpcio>=1.54.2",
+    "protobuf>=4.23.3",
+    "grpcio-health-checking>=1.56.0",
+    "sentry-sdk>=1.29.2",
+]
+gcp = [
+    "google-cloud-logging>=3.0.0",
+    "google-cloud-secret-manager>=2.24.0",
+    "google-cloud-storage>=3.2.0",
+    "google-cloud-compute>=1.38.0"
+]
+
+[project.scripts]
+aigear-init = "aigear.cli.project_cli:project_init"
+aigear-gcp-infra = "aigear.cli.gcp_cli:gcp_infra_init"
+aigear-workflow = "aigear.cli.task_running:task_run"
+aigear-scheduler = "aigear.cli.scheduler:scheduler_init"
+artifacts-image-create = "aigear.cli.artifacts_image:artifacts_image_init"
+aigear-grpc = "aigear.cli.grpc_service:grpc_run"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = [
+    "aigear*",
+]
+
+[tool.setuptools.package-data]
+"aigear" = [
+    "template/*",
+    "template/**/*",
+    "common/*",
+    "common/**/*",
+    "**/*.yaml",
+    "**/*.yml",
+    "**/*.json",
+    "**/*.toml",
+    "**/*.js",
+    "**/*.md",
+    "**/Dockerfile",
+]

aigear-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

aigear-0.0.1/src/aigear/__init__.py

@@ -0,0 +1,27 @@
+from .project import Project
+from ._version import __version__
+from .common import (
+    Logging,
+    generate_schema,
+    generate_schema_for_json,
+    SecretManager,
+)
+from aigear.infrastructure import Infra
+from aigear.deploy.gcp.scheduler import create_scheduler
+from aigear.common import AigearConfig
+
+
+__all__ = [
+    "Logging",
+    "generate_schema",
+    "generate_schema_for_json",
+    "SecretManager",
+    "Infra",
+]
+
+__all__.extend(
+    [
+        Project,
+        __version__,
+    ]
+)

aigear-0.0.1/src/aigear/_version.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

aigear-0.0.1/src/aigear/cli/artifacts_image.py

@@ -0,0 +1,24 @@
+import argparse
+from aigear.deploy.gcp.artifacts_image import create_artifacts_image
+
+
+def get_argument():
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('--dockerfile_path', default=".",
+                        help='path of dockerfile')
+    parser.add_argument('--image_name',
+                        help='The name of the Docker image')
+    parser.add_argument('--image_version', default="latest",
+                        help='The version of the Docker image')
+    args = parser.parse_args()
+    return args
+
+def artifacts_image_init():
+    args = get_argument()
+    create_artifacts_image(
+        dockerfile_path=args.dockerfile_path,
+        image_name=args.image_name,
+        image_version = args.image_version
+    )

aigear-0.0.1/src/aigear/cli/gcp_cli.py

@@ -0,0 +1,6 @@
+from aigear.infrastructure.gcp import Infra
+
+
+def gcp_infra_init():
+    gcp_infra = Infra()
+    gcp_infra.create()

aigear-0.0.1/src/aigear/cli/grpc_service.py

@@ -0,0 +1,23 @@
+import os
+import sys
+import argparse
+from aigear.service.grpc.grpc_service import grpc_service
+
+
+def get_argument():
+    # Arg
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('--model_class_path',
+                        help='Deploy gRPC services by company code')
+    args = parser.parse_args()
+    return args
+
+def grpc_run():
+    project_root = os.getcwd()
+    if project_root not in sys.path:
+        sys.path.insert(0, project_root)
+
+    args = get_argument()
+    grpc_service(args.model_class_path)

aigear-0.0.1/src/aigear/cli/project_cli.py

@@ -0,0 +1,33 @@
+import argparse
+from ..project import Project
+
+
+def get_argument():
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument(
+        "--name",
+        default="template_project",
+        help="Project name."
+    )
+    parser.add_argument(
+        "--pipeline_versions",
+        default="pipeline_version_1",
+        type=str,
+        help="Comma-separated list of pipeline names (e.g., pipeline_v1,pipeline_v2). If not provided, will use pipeline_version_1"
+    )
+    args = parser.parse_args()
+    return args
+
+
+def project_init():
+    args = get_argument()
+    # Parse pipelines
+    pipeline_versions = [p.strip() for p in args.pipeline_versions.split(',')]
+
+    project = Project(
+        name=args.name,
+        pipeline_versions=pipeline_versions
+    )
+    project.init()

aigear-0.0.1/src/aigear/cli/scheduler.py

@@ -0,0 +1,19 @@
+import argparse
+from aigear.deploy.gcp.scheduler import create_scheduler
+
+
+def get_argument():
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('--version', default="",
+                        help='Version of the pipeline')
+    parser.add_argument('--step_names', default="",
+                        help='Name of the pipeline step')
+    args = parser.parse_args()
+    return args
+
+def scheduler_init():
+    args = get_argument()
+    step_names = args.step_names.split(",")
+    create_scheduler(args.version, step_names)

aigear-0.0.1/src/aigear/cli/task_running.py

@@ -0,0 +1,33 @@
+import importlib
+import logging
+import argparse
+import sys
+import os
+
+
+def run_step(pipeline_version, step):
+    project_root = os.getcwd()
+    if project_root not in sys.path:
+        sys.path.insert(0, project_root)
+
+    module_name = f"src.pipelines.{step}"
+    try:
+        module = importlib.import_module(module_name)
+        module.main(pipeline_version)
+    except Exception as e:
+        logging.error(f"Error while executing {module_name}: {e}")
+
+def get_argument():
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('--version', default="",
+                        help='Version of the pipeline')
+    parser.add_argument('--step', default="",
+                        help='Name of the pipeline step')
+    args = parser.parse_args()
+    return args
+
+def task_run():
+    args = get_argument()
+    run_step(args.version, args.step)

aigear-0.0.1/src/aigear/common/__init__.py

@@ -0,0 +1,16 @@
+from aigear.common.sh import run_sh, run_sh_stream
+from aigear.common.logger import Logging
+from aigear.common.config import AigearConfig
+from aigear.common.dynamic_type import generate_schema, generate_schema_for_json
+from aigear.common.secretmanager import SecretManager
+
+
+__all__ = [
+    "run_sh",
+    "run_sh_stream",
+    "Logging",
+    "AigearConfig",
+    "generate_schema",
+    "generate_schema_for_json",
+    "SecretManager",
+]

aigear-0.0.1/src/aigear/common/config.py

@@ -0,0 +1,59 @@
+import json
+import os
+from typing import Optional
+from aigear.common.schema.config_schema import Config
+from aigear.common.logger import Logging
+
+
+logger = Logging(log_name=__name__).console_logging()
+
+def read_config(env_path) -> Optional[Config]:
+    if not os.path.exists(env_path):
+        logger.error(f"Configuration file {env_path} not found.")
+        return None
+
+    with open(env_path, "r", encoding="utf-8") as f:
+        cfg = json.load(f)
+
+    return cfg
+
+class AigearConfig:
+    _config = None
+
+    @classmethod
+    def load(cls):
+        if cls._config:
+            return cls._config
+        
+        cfg = read_config(env_path=os.path.join(os.getcwd(), "env.json"))
+        cls._config = Config(**cfg["aigear"])
+        return cls._config
+
+    @classmethod
+    def get_config(cls) -> Config:
+        if not cls._config:
+            cls.load()
+        return cls._config
+
+class PipelinesConfig:
+    _config = None
+
+    @classmethod
+    def load(cls):
+        if cls._config:
+            return cls._config
+
+        cfg = read_config(env_path=os.path.join(os.getcwd(), "env.json"))
+        cls._config = cfg.get("pipelines", {})
+        return cls._config
+
+    @classmethod
+    def get_config(cls) -> Config:
+        if not cls._config:
+            cls.load()
+        return cls._config
+
+def get_project_name() -> str:
+    cfg = read_config(env_path=os.path.join(os.getcwd(), "env.json"))
+    project_name = cfg.get("project_name")
+    return project_name