wanna-ml 0.5.24__tar.gz

wanna_ml-0.5.24/CHANGELOG.md

@@ -0,0 +1,11 @@
+# Changelog #
+
+## Version 0.2.3 ##
+* pipeline_network bug fix
+* Introduction of notebook build command for CI/CD environments
+
+## Version 0.2.2 ##
+* allow for building and pushing containers in an environment without access to GCP  
+
+## Version 0.2.1 ##
+* Initial OSS release

wanna_ml-0.5.24/LICENCE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Avast Software s.r.o.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

wanna_ml-0.5.24/PKG-INFO

@@ -0,0 +1,137 @@
+Metadata-Version: 2.1
+Name: wanna-ml
+Version: 0.5.24
+Summary: CLI tool for managing ML projects on Vertex AI
+Home-page: https://avast.github.io/wanna-ml
+License: MIT
+Keywords: mlops,vertex-ai,GCP,google-cloud-platform,machine-learning
+Author: Joao Da Silva
+Author-email: joao.dasilvacunha@gendigital.com
+Requires-Python: >=3.10.0,<4.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Requires-Dist: Jinja2 (>=3.1.6,<4.0.0)
+Requires-Dist: PyYAML (>=6.0.3,<7.0.0)
+Requires-Dist: case-converter (>=1.2.0,<2.0.0)
+Requires-Dist: cookiecutter (>=2.6.0,<3.0.0)
+Requires-Dist: cron-validator (>=1.0.8,<2.0.0)
+Requires-Dist: dirhash (>=0.5.0,<0.6.0)
+Requires-Dist: email-validator (>=2.3.0,<3.0.0)
+Requires-Dist: emoji (>=2.15.0,<3.0.0)
+Requires-Dist: gcloud-config-helper (>=0.3.1,<0.4.0)
+Requires-Dist: gitpython (>=3.1.45,<4.0.0)
+Requires-Dist: google-cloud-aiplatform (>=1.121.0,<2.0.0)
+Requires-Dist: google-cloud-build (>=3.33.0,<4.0.0)
+Requires-Dist: google-cloud-compute (>=1.40.0,<2.0.0)
+Requires-Dist: google-cloud-functions (>=1.20.4,<2.0.0)
+Requires-Dist: google-cloud-logging (>=3.12.1,<4.0.0)
+Requires-Dist: google-cloud-monitoring (>=2.28.0,<3.0.0)
+Requires-Dist: google-cloud-notebooks (>=1.13.3,<2.0.0)
+Requires-Dist: google-cloud-pubsub (>=2.31.1,<3.0.0)
+Requires-Dist: google-cloud-scheduler (>=2.16.1,<3.0.0)
+Requires-Dist: google-cloud-storage (>=2.19.0,<4.0.0)
+Requires-Dist: halo (>=0.0.31,<0.0.32)
+Requires-Dist: igittigitt (>=2.1.5,<3.0.0)
+Requires-Dist: kfp (>=2.13.0,<3.0.0)
+Requires-Dist: pathvalidate (>=3.3.1,<4.0.0)
+Requires-Dist: pendulum (>=3.1.0,<4.0.0)
+Requires-Dist: pydantic (>=2.12.3,<3.0.0)
+Requires-Dist: python-on-whales (>=0.78.0,<0.79.0)
+Requires-Dist: pyyaml-include (>=2.2,<3.0)
+Requires-Dist: rich (>=14.2.0,<15.0.0)
+Requires-Dist: smart-open[gcs] (>=7.3.1,<8.0.0)
+Requires-Dist: treelib (>=1.8.0,<2.0.0)
+Requires-Dist: typed-lazyimport (>=0.3.1,<0.4.0)
+Requires-Dist: typer (>=0.20.0,<0.21.0)
+Requires-Dist: waiting (>=1.5.0,<2.0.0)
+Project-URL: Documentation, https://avast.github.io/wanna-ml
+Description-Content-Type: text/markdown
+
+# WANNA-ML
+
+---
+
+<p align="center" font-style="italic"> 
+<em> Complete MLOps framework for Vertex-AI  </em>
+</p>
+
+---
+
+<p align="center">
+<a href="https://github.com/avast/wanna-ml/actions/workflows/build.yml" target="_blank">
+    <img src="https://github.com/avast/wanna-ml/actions/workflows/build.yml/badge.svg" alt="Build">
+</a>
+<a href="https://github.com/avast/wanna-ml/actions/workflows/release.yml" target="_blank">
+    <img src="https://github.com/avast/wanna-ml/actions/workflows/release.yml/badge.svg" alt="Release">
+</a>
+<a href="https://codecov.io/gh/avast/wanna-ml" target="_blank">
+    <img src="https://codecov.io/gh/avast/wanna-ml/branch/master/graph/badge.svg?token=TAFWK4GJPR" alt="Coverage">
+</a>
+<a href="https://pypi.org/project/wanna-ml/" target="_blank">
+    <img src="https://img.shields.io/pypi/v/wanna-ml?color=%2334D058&label=pypi%20package" alt="Package version">
+</a>
+</p>
+
+# About WANNA-ML
+
+WANNA-ML is a CLI tool that helps researchers, data scientists, and ML Engineers quickly adapt to Google Cloud Platform (GCP) and get started on the cloud in almost no time.
+
+It makes it easy to start a Jupyter notebook, run training jobs and pipelines, build a Docker container, export logs to Tensorboards, and much more.
+
+We build on top of Vertex-AI managed services and integrate with other GCP services like Cloud Build and Artifact Registry to provide you with a standardized structure for managing ML assets on GCP.
+
+
+## Help
+
+See the [documentation](https://avast.github.io/wanna-ml/) for more details.
+
+
+## Get started
+
+### Installation
+Install using `pip install -U wanna-ml`.
+
+For more information on the installation process and requirements, visit out [installation page in documentation](https://avast.github.io/wanna-ml/installation)
+
+### Authentication
+WANNA-ML relies on `gcloud` for user authentication. 
+
+1. Install the `gcloud` CLI - follow [official guide](https://cloud.google.com/sdk/docs/install)
+2. Authenticate with the `gcloud init`
+3. Set you Google Application Credentials `gcloud auth application-default login`
+
+### Docker Build
+You can use a local Docker daemon to build Docker images, but it is not required. 
+You are free to choose between local building on GCP Cloud Build. 
+If you prefer local Docker image building, install  [Docker Desktop](https://www.docker.com/products/docker-desktop/).
+
+### GCP IAM Roles and Permissions
+Different WANNA-ML calls require different GCP permissions to create given resources on GCP. Our [documentation page](https://avast.github.io/wanna-ml/)
+lists recommended GCP IAM roles for each `wanna` command.
+
+## Examples
+Jump to [the samples](https://github.com/avast/wanna-ml/tree/master/samples) to see a complete solution 
+for various use cases.
+
+## Issues
+Please report issues to [GitHub](https://github.com/avast/wanna-ml/issues).
+
+## Contributing
+Your contributions are always welcome, see [CONTRIBUTING.md](https://github.com/avast/wanna-ml/blob/master/CONTRIBUTING.md) for more information.
+If you like WANNA-ML, don't forget to give our project a star! 
+
+## Licence
+Distributed under the MIT License - see [LICENSE](https://github.com/avast/wanna-ml/blob/master/LICENCE).
+

wanna_ml-0.5.24/README.md

@@ -0,0 +1,75 @@
+# WANNA-ML
+
+---
+
+<p align="center" font-style="italic"> 
+<em> Complete MLOps framework for Vertex-AI  </em>
+</p>
+
+---
+
+<p align="center">
+<a href="https://github.com/avast/wanna-ml/actions/workflows/build.yml" target="_blank">
+    <img src="https://github.com/avast/wanna-ml/actions/workflows/build.yml/badge.svg" alt="Build">
+</a>
+<a href="https://github.com/avast/wanna-ml/actions/workflows/release.yml" target="_blank">
+    <img src="https://github.com/avast/wanna-ml/actions/workflows/release.yml/badge.svg" alt="Release">
+</a>
+<a href="https://codecov.io/gh/avast/wanna-ml" target="_blank">
+    <img src="https://codecov.io/gh/avast/wanna-ml/branch/master/graph/badge.svg?token=TAFWK4GJPR" alt="Coverage">
+</a>
+<a href="https://pypi.org/project/wanna-ml/" target="_blank">
+    <img src="https://img.shields.io/pypi/v/wanna-ml?color=%2334D058&label=pypi%20package" alt="Package version">
+</a>
+</p>
+
+# About WANNA-ML
+
+WANNA-ML is a CLI tool that helps researchers, data scientists, and ML Engineers quickly adapt to Google Cloud Platform (GCP) and get started on the cloud in almost no time.
+
+It makes it easy to start a Jupyter notebook, run training jobs and pipelines, build a Docker container, export logs to Tensorboards, and much more.
+
+We build on top of Vertex-AI managed services and integrate with other GCP services like Cloud Build and Artifact Registry to provide you with a standardized structure for managing ML assets on GCP.
+
+
+## Help
+
+See the [documentation](https://avast.github.io/wanna-ml/) for more details.
+
+
+## Get started
+
+### Installation
+Install using `pip install -U wanna-ml`.
+
+For more information on the installation process and requirements, visit out [installation page in documentation](https://avast.github.io/wanna-ml/installation)
+
+### Authentication
+WANNA-ML relies on `gcloud` for user authentication. 
+
+1. Install the `gcloud` CLI - follow [official guide](https://cloud.google.com/sdk/docs/install)
+2. Authenticate with the `gcloud init`
+3. Set you Google Application Credentials `gcloud auth application-default login`
+
+### Docker Build
+You can use a local Docker daemon to build Docker images, but it is not required. 
+You are free to choose between local building on GCP Cloud Build. 
+If you prefer local Docker image building, install  [Docker Desktop](https://www.docker.com/products/docker-desktop/).
+
+### GCP IAM Roles and Permissions
+Different WANNA-ML calls require different GCP permissions to create given resources on GCP. Our [documentation page](https://avast.github.io/wanna-ml/)
+lists recommended GCP IAM roles for each `wanna` command.
+
+## Examples
+Jump to [the samples](https://github.com/avast/wanna-ml/tree/master/samples) to see a complete solution 
+for various use cases.
+
+## Issues
+Please report issues to [GitHub](https://github.com/avast/wanna-ml/issues).
+
+## Contributing
+Your contributions are always welcome, see [CONTRIBUTING.md](https://github.com/avast/wanna-ml/blob/master/CONTRIBUTING.md) for more information.
+If you like WANNA-ML, don't forget to give our project a star! 
+
+## Licence
+Distributed under the MIT License - see [LICENSE](https://github.com/avast/wanna-ml/blob/master/LICENCE).

wanna_ml-0.5.24/pyproject.toml

@@ -0,0 +1,186 @@
+[tool.poetry]
+authors = ["Joao Da Silva <joao.dasilvacunha@gendigital.com>"]
+description = "CLI tool for managing ML projects on Vertex AI"
+name = "wanna-ml"
+version = "v0.5.24"
+packages = [
+    { include = "wanna", from = "src" },
+]
+include = ["CHANGELOG.md"]
+readme = "README.md"
+license = "MIT"
+keywords = ["mlops", "vertex-ai", "GCP", "google-cloud-platform", "machine-learning"]
+homepage = "https://avast.github.io/wanna-ml"
+documentation = "https://avast.github.io/wanna-ml"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Typing :: Typed",
+]
+
+[tool.poetry.dependencies]
+python = ">=3.10.0, <4.0"
+typed-lazyimport = ">=0.3.1,<0.4.0"
+kfp = "^2.13.0"
+google-cloud-aiplatform = "^1.121.0"
+google-cloud-build = "^3.33.0"
+google-cloud-compute = "^1.40.0"
+google-cloud-functions = "^1.20.4"
+google-cloud-logging = "^3.12.1"
+google-cloud-monitoring = "^2.28.0"
+google-cloud-notebooks = "^1.13.3"
+google-cloud-scheduler = "^2.16.1"
+google-cloud-storage = ">=2.19.0,<4.0.0"
+gcloud-config-helper = "^0.3.1"
+google-cloud-pubsub = "^2.31.1"
+case-converter = "^1.2.0"
+cookiecutter = "^2.6.0"
+cron-validator = "^1.0.8"
+dirhash = "^0.5.0"
+email-validator = "^2.3.0"
+emoji = "^2.15.0"
+gitpython = "^3.1.45"
+halo = "^0.0.31"
+Jinja2 = "^3.1.6"
+pathvalidate = "^3.3.1"
+pydantic = "^2.12.3"
+python-on-whales = "^0.78.0"
+pyyaml-include = "^2.2"
+PyYAML = "^6.0.3"
+smart-open = {extras = ["gcs"], version = "^7.3.1"}
+treelib = "^1.8.0"
+typer = "^0.20.0"
+waiting = "^1.5.0"
+rich = "^14.2.0"
+pendulum = "^3.1.0"
+igittigitt = "^2.1.5"
+
+[tool.poetry.group.samples.dependencies]
+google-cloud-pipeline-components = "^2.21.0"
+xgboost = "^3.1.1"
+scikit-learn = "^1.7.2"
+
+[tool.poetry.group.dev.dependencies]
+mkdocs-click = "^0.8.1"
+mkdocs-material = "^9.4.6"
+mkdocs-typer = "^0.0.3"
+mkdocstrings = "^0.23.0"
+mkdocstrings-python = "^1.7.3"
+mkautodoc = "^0.2.0"
+markdown = "^3.5"
+mock = "^5.1.0"
+mypy = "^1.8.0"
+pytest = "^8.0.1"
+pytest-cov = "^5.0.0"
+pytest-ruff = "^0.3.1"
+pytest-mypy = "^0.10.3"
+pytest-mock = "^3.12.0"
+poethepoet = "^0.22.0"
+types-mock = "^5.1.0.3"
+types-pyyaml = "^6.0.12"
+types-requests = "^2.31.0"
+types-setuptools = "^68.2.0"
+pre-commit = "^2.19.0"
+pytest-env = "^1.1.5"
+
+[tool.pytest.ini_options]
+addopts = """
+    --verbose
+    --ruff
+    --ruff-format
+    --cov-report term
+    --cov-report xml:coverage.xml
+    --cov=src
+    --mypy
+    --durations=0
+    --ignore=templates
+    --ignore=samples
+    --ignore=dev_tests
+    """
+#addopts = """
+#    --ignore=templates
+#    --ignore=samples
+#    --ignore=dev_tests
+#    -s
+#    """
+filterwarnings = [
+    "error",
+    "ignore::UserWarning", # note the use of single quote below to denote "raw" strings in TOML
+    'ignore::DeprecationWarning',
+    'ignore::SyntaxWarning',
+    'ignore::ResourceWarning',
+]
+env = [
+    "LABEL = test"
+]
+
+[tool.poe.tasks]
+# Separate checks until https://github.com/astral-sh/ruff/issues/8232 is merged
+_ruff_lint = "ruff check --fix ."
+_ruff_format = "ruff format ."
+format-code = ["_ruff_lint", "_ruff_format"]
+docs-deploy = "mkdocs gh-deploy --force"
+docs-serve = "mkdocs serve"
+
+[tool.ruff]
+line-length = 99
+output-format = "full"
+extend-exclude = [
+    "**pb2.py",
+    "**pb2.pyi",
+    "samples/**",
+    "templates/blank/**",
+    "templates/sklearn/**",
+    "dev_tests/**",
+]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "PERF", "ARG"]
+ignore = [
+    "E1", # handled by formatter
+    "E4", # handled by formatter
+    "E5", # handled by formatter
+    "E722",
+    "PERF401"
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**.py" = [
+    "ARG001",  # needed for mocks
+    "ARG002"  # needed for mocks
+]
+
+[tool.mypy]
+exclude = [
+    "samples/pipelines/*",
+    "dev_tests/*",
+]
+follow_imports = "silent"
+mypy_path = ["src"]
+namespace_packages = true
+explicit_package_bases = true
+strict = true
+strict_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+disallow_any_generics = true
+implicit_reexport = true
+ignore_missing_imports = true
+
+[tool.poetry.scripts]
+wanna = "wanna.cli.__main__:wanna"
+
+[build-system]
+build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core>=1.0.0"]

wanna_ml-0.5.24/src/wanna/__init__.py

@@ -0,0 +1,7 @@
+# mypy: allow-untyped-calls
+from importlib import metadata as importlib_metadata
+
+try:
+    __version__ = importlib_metadata.version(__name__)
+except importlib_metadata.PackageNotFoundError:
+    __version__ = "snapshot"

wanna_ml-0.5.24/src/wanna/cli/__init__.py

@@ -0,0 +1,7 @@
+# mypy: allow-untyped-calls
+from importlib import metadata as importlib_metadata
+
+try:
+    __version__ = importlib_metadata.version(__name__)
+except importlib_metadata.PackageNotFoundError:
+    __version__ = "snapshot"

wanna_ml-0.5.24/src/wanna/cli/__main__.py

@@ -0,0 +1,83 @@
+import logging
+from pathlib import Path
+
+import typer
+from cookiecutter.main import cookiecutter
+
+from wanna.core.loggers.wanna_logger import get_logger
+
+from .plugins.runner import PluginRunner
+from .version import perform_check
+
+logger = get_logger(__name__)
+
+logging.getLogger("smart_open").setLevel(logging.ERROR)
+logging.getLogger("google").setLevel(logging.ERROR)
+logging.getLogger("kfp").setLevel(logging.ERROR)
+
+runner = PluginRunner()
+app = runner.app
+
+
+@app.command(name="version", help="Print your current and latest available version")
+def version():
+    perform_check()
+
+
+@app.command(name="init", help="Initiate a new wanna-ml project from template.")
+def init(
+    output_dir: Path = typer.Option(
+        ...,
+        "--output-dir",
+        prompt="Where do you want to initiate your wanna-ml repository? (input '.' to use the current directory)",
+        help="The output directory where wanna-ml repository will be created",
+    ),
+    template: str = typer.Option(
+        ...,
+        "--template",
+        "-t",
+        help="The git repository of the template you want to use",
+    ),
+    checkout: str | None = typer.Option(
+        None,
+        "--checkout",
+        "-c",
+        help="The branch, tag or commit to checkout after cloning the repository",
+    ),
+    directory: str | None = typer.Option(
+        None,
+        "--directory",
+        "-d",
+        help="The directory within the repository to use as the template",
+    ),
+    overwrite_if_exists: bool = typer.Option(
+        False,
+        "--overwrite-if-exists",
+        help="Overwrite the contents of the output directory if it exists",
+    ),
+    no_input: bool = typer.Option(
+        False,
+        "--no-input",
+        help="Do not prompt for parameters and only use cookiecutter.json file content",
+    ),
+):
+    result_dir = cookiecutter(
+        template=template,
+        checkout=checkout,
+        directory=directory,
+        output_dir=output_dir,
+        overwrite_if_exists=overwrite_if_exists,
+        no_input=no_input,
+    )
+    logger.user_success(f"Repo initiated at {result_dir}")
+
+
+def wanna():
+    """
+    Main entrypoint for wanna cli
+    """
+    app()
+
+
+if __name__ == "__main__":
+    wanna()

wanna_ml-0.5.24/src/wanna/cli/plugins/base_plugin.py

@@ -0,0 +1,23 @@
+import abc
+from typing import Any, Callable, Union
+
+import typer
+
+CommandRegistration = Union[Callable[..., None], tuple[Callable[..., None], dict[Any, Any]]]
+
+
+class BasePlugin(abc.ABC):
+    """Some doc"""
+
+    def __init__(self) -> None:
+        self.app = typer.Typer()
+
+    def register_command(self, func: CommandRegistration) -> None:
+        if isinstance(func, tuple):
+            self.app.command(context_settings=func[1])(func[0])
+        else:
+            self.app.command()(func)
+
+    def register_many(self, funcs: list[CommandRegistration]) -> None:
+        for func in funcs:
+            self.register_command(func)

wanna_ml-0.5.24/src/wanna/cli/plugins/common_options.py

@@ -0,0 +1,45 @@
+import typer
+
+from wanna.core.deployment.models import PushMode
+
+profile_name_option = typer.Option(
+    "default",
+    "--profile",
+    "-p",
+    envvar="WANNA_GCP_PROFILE_NAME",
+    help="Name of the GCP profile you want to use. "
+    "Profiles are loaded from wanna-ml yaml config and "
+    "(optionally) from this file too.",
+)
+
+wanna_file_option = typer.Option(
+    "wanna.yaml",
+    "--file",
+    "-f",
+    envvar="WANNA_FILE",
+    help="Path to the wanna-ml yaml configuration. Use -f - to pass the configuration through stdin.",
+)
+
+push_mode_option: PushMode = typer.Option(
+    PushMode.all,
+    "--mode",
+    "-m",
+    help="Push mode, this is useful if you want to "
+    "push containers in one step and deploy instances in other."
+    "Use all for dev",
+)
+
+
+def instance_name_option(instance_type: str, operation: str, help: str | None = None):
+    return typer.Option(
+        "all",
+        "--name",
+        "-n",
+        help=help
+        or f"Specify only one {instance_type} from your wanna-ml yaml configuration to {operation}. "
+        f"Choose 'all' to {operation} all {instance_type}s.",
+    )
+
+
+def version_option(instance_type: str):
+    return typer.Option("dev", "--version", "-v", help=f"{instance_type} version")