clarifai 10.8.4__py3-none-any.whl → 10.8.5__py3-none-any.whl

clarifai/models/model_serving/cli/_utils.py

@@ -1,53 +0,0 @@
-from __future__ import annotations  # isort: skip
-import os
-import shutil
-import subprocess
-from typing import Dict, Union
-
-from ..constants import (CLARIFAI_EXAMPLES_REPO, CLARIFAI_EXAMPLES_REPO_PATH,
-                         MODEL_UPLOAD_EXAMPLE_FOLDER)
-
-
-def download_examples_repo(forced_download: bool = False):
-
-  def _pull():
-    subprocess.run(f"git clone {CLARIFAI_EXAMPLES_REPO} {CLARIFAI_EXAMPLES_REPO_PATH}", shell=True)
-
-  if not os.path.isdir(CLARIFAI_EXAMPLES_REPO_PATH):
-    print(f"Download examples to {CLARIFAI_EXAMPLES_REPO_PATH}")
-    _pull()
-  else:
-    if forced_download:
-
-      def _rm_dir_readonly(func, path, _):
-        import stat
-        os.chmod(path, stat.S_IWRITE)
-        func(path)
-
-      print("Removing old examples...")
-      shutil.rmtree(CLARIFAI_EXAMPLES_REPO_PATH, onerror=_rm_dir_readonly)
-      _pull()
-
-
-def list_model_upload_examples(
-    forced_download: bool = False) -> Dict[str, tuple[str, Union[str, None]]]:
-  download_examples_repo(forced_download)
-  model_upload_folder = MODEL_UPLOAD_EXAMPLE_FOLDER
-  model_upload_path = os.path.join(CLARIFAI_EXAMPLES_REPO_PATH, model_upload_folder)
-  examples = {}
-  for model_type_ex in os.listdir(model_upload_path):
-    _folder = os.path.join(model_upload_path, model_type_ex)
-    if os.path.isdir(_folder):
-      _walk = list(os.walk(_folder))
-      if len(_walk) > 0:
-        _, model_names, _files = _walk[0]
-        readme = [item for item in _files if "readme" in item.lower()]
-        for name in model_names:
-          examples.update({
-              f"{model_type_ex}/{name}": [
-                  os.path.join(_folder, name),
-                  os.path.join(_folder, readme[0]) or None
-              ]
-          })
-
-  return examples

clarifai/models/model_serving/cli/base.py

@@ -1,14 +0,0 @@
-from abc import ABC, abstractmethod
-from argparse import _SubParsersAction
-
-
-class BaseClarifaiCli(ABC):
-
-  @staticmethod
-  @abstractmethod
-  def register(parser: _SubParsersAction):
-    raise NotImplementedError()
-
-  @abstractmethod
-  def run(self):
-    raise NotImplementedError()

clarifai/models/model_serving/cli/build.py

@@ -1,79 +0,0 @@
-# Copyright 2023 Clarifai, Inc.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Commandline interface for model upload utils."""
-import argparse
-import os
-import subprocess
-
-from ..repo_build import RepositoryBuilder
-from .base import BaseClarifaiCli
-
-
-class BuildCli(BaseClarifaiCli):
-
-  @staticmethod
-  def register(parser: argparse._SubParsersAction):
-    parser = parser.add_parser("build", help="Build clarifai model for uploading")
-    sub_parser = parser.add_subparsers()
-
-    BuildModelSubCli.register(sub_parser)
-
-    parser.set_defaults(func=BuildCli)
-
-
-class BuildModelSubCli(BaseClarifaiCli):
-
-  @staticmethod
-  def register(parser: argparse._SubParsersAction):
-    sub_parser = parser.add_parser("model", help="Build Clarifai model")
-    sub_parser.add_argument(
-        "path",
-        type=str,
-        nargs='?',
-        help="Path to working directory, default is current directory",
-        default=".")
-    sub_parser.add_argument(
-        "--out-path", type=str, required=False, help="Output path of built model", default=None)
-    sub_parser.add_argument(
-        "--name",
-        type=str,
-        required=False,
-        help="Name of built file, default is `clarifai_model_id` in config if set or `model`",
-        default=None)
-    sub_parser.add_argument(
-        "--no-test",
-        action="store_true",
-        help="Trigger this flag to skip testing before uploading")
-    sub_parser.set_defaults(func=BuildModelSubCli)
-
-  def __init__(self, args: argparse.Namespace) -> None:
-    self.path = args.path
-    self.no_test = args.no_test
-    self.test_path = os.path.join(self.path, "test.py")
-    self.output_path = args.out_path or self.path
-    self.serving_backend = "triton"
-    self.name = args.name
-
-  def run(self):
-
-    # Run test before uploading
-    if not self.no_test:
-      assert os.path.exists(
-          self.test_path), FileNotFoundError(f"Could not find `test.py` in {self.path}")
-      result = subprocess.run(f"pytest -s --log-level=INFO {self.test_path}", shell=True)
-      assert result.returncode == 0, "Test has failed. Please make sure no error exists in your code."
-
-    # build
-    print("Start building...")
-    RepositoryBuilder.build(
-        self.path, backend=self.serving_backend, output_dir=self.output_path, name=self.name)

clarifai/models/model_serving/cli/clarifai_clis.py

@@ -1,33 +0,0 @@
-from argparse import ArgumentParser
-
-from .build import BuildCli
-from .create import CreateCli
-from .example_cli import ExampleCli
-from .login import LoginCli
-from .upload import UploadCli
-
-
-def main():
-
-  parser = ArgumentParser("clarifai")
-  cmd_parser = parser.add_subparsers(help="Clarifai cli helpers")
-
-  UploadCli.register(cmd_parser)
-  CreateCli.register(cmd_parser)
-  LoginCli.register(cmd_parser)
-  ExampleCli.register(cmd_parser)
-  BuildCli.register(cmd_parser)
-
-  args = parser.parse_args()
-
-  if not hasattr(args, "func"):
-    parser.print_help()
-    exit(1)
-
-  # Run
-  service = args.func(args)
-  service.run()
-
-
-if __name__ == "__main__":
-  main()

clarifai/models/model_serving/cli/create.py

@@ -1,171 +0,0 @@
-import os
-import shutil
-from argparse import Namespace, _SubParsersAction
-from typing import List
-
-from InquirerPy import prompt
-
-from ..model_config import MODEL_TYPES
-from ..repo_build import RepositoryBuilder
-from ._utils import list_model_upload_examples
-from .base import BaseClarifaiCli
-
-
-class CreateCli(BaseClarifaiCli):
-
-  @staticmethod
-  def register(parser: _SubParsersAction):
-    creator_parser = parser.add_parser("create", help="Create component of Clarifai platform")
-    sub_creator_parser = creator_parser.add_subparsers()
-
-    SubCreateModelCli.register(sub_creator_parser)
-
-    creator_parser.set_defaults(func=CreateCli)
-
-
-class SubCreateModelCli(BaseClarifaiCli):
-
-  @staticmethod
-  def register(parser: _SubParsersAction):
-    model_parser = parser.add_parser("model")
-    model_parser.add_argument(
-        "--working-dir",
-        type=str,
-        required=True,
-        help="Path to your working dir. Create new dir if it does not exist")
-    model_parser.add_argument(
-        "--from-example",
-        required=False,
-        action="store_true",
-        help="Create repository from example")
-    model_parser.add_argument(
-        "--example-id",
-        required=False,
-        type=str,
-        help="Example id, run `clarifai example list` to list of examples")
-
-    model_parser.add_argument(
-        "--type",
-        type=str,
-        choices=MODEL_TYPES,
-        required=False,
-        help="Clarifai supported model types.")
-    model_parser.add_argument(
-        "--image-shape",
-        nargs='+',
-        type=int,
-        required=False,
-        help="H W dims for models with an image input type. H and W each have a max value of 1024",
-        default=[-1, -1])
-    model_parser.add_argument(
-        "--max-bs", type=int, default=1, required=False, help="Max batch size")
-
-    model_parser.add_argument(
-        "--overwrite", action="store_true", help="Overwrite working-dir if exists")
-
-    model_parser.set_defaults(func=SubCreateModelCli)
-
-  def __init__(self, args: Namespace) -> None:
-    self.working_dir: str = args.working_dir
-    self.from_example = args.from_example
-    self.example_id = args.example_id
-    self.overwrite = args.overwrite
-
-    if os.path.exists(self.working_dir):
-      if self.overwrite:
-        print(f"Overwrite {self.working_dir}")
-      else:
-        raise FileExistsError(
-            f"{self.working_dir} exists. If you want to overwrite it, please set `--overwrite` flag"
-        )
-
-    # prevent wrong args when creating from example
-    if not self.from_example:
-      self.image_shape: List[int] = args.image_shape
-
-      self.type: str = args.type
-      self.max_bs: int = args.max_bs
-
-    else:
-      if not self.example_id:
-        questions = [
-            {
-                "type": "list",
-                "message": "Select an example:",
-                "choices": list_model_upload_examples(),
-            },
-        ]
-        result = prompt(questions)
-        self.example_id = result[0]
-
-      else:
-        available_examples = list(list_model_upload_examples().keys())
-        assert self.example_id in available_examples, f"Available examples are: {available_examples}, got {self.example_id}."
-
-  def run(self):
-    if self.from_example:
-      os.makedirs(self.working_dir, exist_ok=True)
-      model_repo, readme = list_model_upload_examples()[self.example_id]
-      shutil.copytree(model_repo, self.working_dir, dirs_exist_ok=True)
-      if readme:
-        shutil.copy(readme, os.path.join(self.working_dir, "readme.md"))
-
-    else:
-      RepositoryBuilder.init_repository(
-          self.type,
-          self.working_dir,
-          backend="triton",
-          max_batch_size=self.max_bs,
-          image_shape=self.image_shape)
-
-    from itertools import islice
-    from pathlib import Path
-
-    def tree(dir_path: Path,
-             level: int = -1,
-             limit_to_directories: bool = False,
-             length_limit: int = 1000):
-      # prefix components:
-      space = '    '
-      branch = '│   '
-      # pointers:
-      tee = '├── '
-      last = '└── '
-      """Given a directory Path object print a visual tree structure"""
-      dir_path = Path(dir_path)  # accept string coerceable to Path
-      files = 0
-      directories = 0
-
-      def inner(dir_path: Path, prefix: str = '', level=-1):
-        nonlocal files, directories
-        if not level:
-          return  # 0, stop iterating
-        if limit_to_directories:
-          contents = [d for d in dir_path.iterdir() if d.is_dir()]
-        else:
-          contents = list(dir_path.iterdir())
-        pointers = [tee] * (len(contents) - 1) + [last]
-        for pointer, path in zip(pointers, contents):
-          if path.is_dir():
-            yield prefix + pointer + path.name
-            directories += 1
-            extension = branch if pointer == tee else space
-            yield from inner(path, prefix=prefix + extension, level=level - 1)
-          elif not limit_to_directories:
-            yield prefix + pointer + path.name
-            files += 1
-
-      print(dir_path.name)
-      iterator = inner(dir_path, level=level)
-      for line in islice(iterator, length_limit):
-        print(line)
-      if next(iterator, None):
-        print(f'... length_limit, {length_limit}, reached, counted:')
-      print(f'\n{directories} directories' + (f', {files} files' if files else ''))
-
-    print("-" * 75)
-    print(f"* Created repository at: {self.working_dir}")
-    tree(self.working_dir)
-    print()
-    print("* Please make sure your code is tested using `test.py` before uploading")
-    print("-" * 75)

clarifai/models/model_serving/cli/example_cli.py

@@ -1,34 +0,0 @@
-from argparse import Namespace, _SubParsersAction
-
-from ._utils import list_model_upload_examples
-from .base import BaseClarifaiCli
-
-
-class ExampleCli(BaseClarifaiCli):
-
-  @staticmethod
-  def register(parser: _SubParsersAction):
-    creator_parser = parser.add_parser("example", help="Download/List examples of model upload")
-    sub_creator_parser = creator_parser.add_subparsers()
-
-    SubListExampleCli.register(sub_creator_parser)
-
-    creator_parser.set_defaults(func=ExampleCli)
-
-
-class SubListExampleCli(BaseClarifaiCli):
-
-  @staticmethod
-  def register(parser: _SubParsersAction):
-    _parser = parser.add_parser("list")
-    _parser.add_argument("--force-download", action="store_true", help="Force download examples")
-    _parser.set_defaults(func=SubListExampleCli)
-
-  def __init__(self, args: Namespace) -> None:
-    self.force_download = args.force_download
-
-  def run(self):
-    _list = list_model_upload_examples(self.force_download)
-    print(f"Found {len(_list)} examples")
-    for each in _list:
-      print(f" * {each}")

clarifai/models/model_serving/cli/login.py

@@ -1,26 +0,0 @@
-import argparse
-
-from clarifai.models.model_serving.constants import CLARIFAI_PAT_PATH
-from ..utils import _persist_pat
-from .base import BaseClarifaiCli
-
-
-class LoginCli(BaseClarifaiCli):
-
-  @staticmethod
-  def register(parser: argparse._SubParsersAction):
-    upload_parser = parser.add_parser("login", help="Login to Clarifai and save PAT locally")
-    upload_parser.set_defaults(func=LoginCli)
-
-  def __init__(self, args: argparse.Namespace) -> None:
-    pass
-
-  def _parse_config(self):
-    # do something with self.config_path
-    raise NotImplementedError()
-
-  def run(self):
-    msg = "Get your PAT from https://clarifai.com/settings/security and pass it here: "
-    _pat = input(msg)
-    _persist_pat(_pat)
-    print(f"Your PAT is saved at {CLARIFAI_PAT_PATH}")

clarifai/models/model_serving/cli/upload.py

@@ -1,183 +0,0 @@
-# Copyright 2023 Clarifai, Inc.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Commandline interface for model upload utils."""
-import argparse
-import os
-import subprocess
-
-from clarifai.models.model_serving.model_config import get_model_config, load_user_config
-from clarifai.models.model_serving.model_config.inference_parameter import InferParamManager
-
-from ..constants import BUILT_MODEL_EXT
-from ..utils import login
-from .base import BaseClarifaiCli
-
-
-class UploadCli(BaseClarifaiCli):
-
-  @staticmethod
-  def register(parser: argparse._SubParsersAction):
-    creator_parser = parser.add_parser("upload", help="Upload component to Clarifai platform")
-    sub_creator_parser = creator_parser.add_subparsers()
-
-    UploadModelSubCli.register(sub_creator_parser)
-
-    creator_parser.set_defaults(func=UploadCli)
-
-
-class UploadModelSubCli(BaseClarifaiCli):
-
-  @staticmethod
-  def register(parser: argparse._SubParsersAction):
-    upload_parser = parser.add_parser("model", help="Upload Clarifai model")
-    upload_parser.add_argument(
-        "path",
-        type=str,
-        nargs='?',
-        help=
-        "Path to working dir to get clarifai_config.yaml or path to yaml. Default is current directory",
-        default=".")
-    upload_parser.add_argument(
-        "--url", type=str, required=False, help="Direct download url of zip file", default=None)
-    upload_parser.add_argument(
-        "--file", type=str, required=False, help="Local built file", default=None)
-    upload_parser.add_argument("--id", type=str, required=False, help="Model ID")
-    upload_parser.add_argument(
-        "--user-app",
-        type=str,
-        required=False,
-        help="User ID and App ID separated by '/', e.g., <user_id>/<app_id>")
-    upload_parser.add_argument(
-        "--no-test",
-        action="store_true",
-        help="Trigger this flag to skip testing before uploading")
-    upload_parser.add_argument(
-        "--no-resume",
-        action="store_true",
-        help="Trigger this flag to not resume uploading local file")
-    upload_parser.add_argument(
-        "--update-version",
-        action="store_true",
-        required=False,
-        help="Update exist model with new version")
-
-    upload_parser.set_defaults(func=UploadModelSubCli)
-
-  def __init__(self, args: argparse.Namespace) -> None:
-    self.no_test = args.no_test
-    self.no_resume = args.no_resume
-
-    working_dir_or_config = args.path
-    # if input a config file, then not running test
-    if working_dir_or_config.endswith(".yaml"):
-      # to folder
-      working_dir_or_config = os.path.split(working_dir_or_config)[0]
-      config_yaml_path = working_dir_or_config
-      self.test_path = None
-      self.no_test = True
-    # if it is a directory - working dir then it must contain config and test
-    else:
-      config_yaml_path = os.path.join(working_dir_or_config, "clarifai_config.yaml")
-      self.test_path = os.path.join(working_dir_or_config, "test.py")
-
-    assert os.path.exists(config_yaml_path), FileNotFoundError(
-        f"`{config_yaml_path}` does not exist")
-    self.config = load_user_config(cfg_path=config_yaml_path)
-
-    self.file = args.file
-    self.url = args.url
-    if self.file:
-      assert not self.url, ValueError("Expected either file or url, not both.")
-      assert os.path.exists(self.file), FileNotFoundError
-    elif self.url:
-      if len(self.url.split(":")) == 1:
-        # if URL has no scheme, default to https
-        self.url = f"https://{self.url}"
-      assert self.url.startswith("http") or self.url.startswith("https") or self.url.startswith(
-          "s3"
-      ), f"Invalid URL scheme, supported schemes are 'http', 'https', or 's3'. Got {self.url}"
-      self.file = None
-    else:
-      for _fname in os.listdir(working_dir_or_config):
-        if _fname.endswith(BUILT_MODEL_EXT):
-          self.file = os.path.join(working_dir_or_config, _fname)
-          break
-      assert self.file, ValueError(
-          f"Not using url/file but also not found built file with extension {BUILT_MODEL_EXT}")
-
-    self.user_id, self.app_id = "", ""
-    user_app = args.user_app
-    self.url: str = args.url
-    self.update_version = args.update_version
-
-    clarifai_cfg = self.config.clarifai_model
-    self.url: str = args.url
-    self.id = args.id or clarifai_cfg.clarifai_model_id
-    self.type = clarifai_cfg.type
-    self.desc = clarifai_cfg.description
-    self.infer_param = clarifai_cfg.inference_parameters
-    user_app = user_app or clarifai_cfg.clarifai_user_app_id
-
-    if user_app:
-      user_app = user_app.split('/')
-      assert len(
-          user_app
-      ) == 2, f"id must be combination of user_id and app_id separated by `/`, e.g. <user_id>/<app_id>. Got {args.id}"
-      self.user_id, self.app_id = user_app
-
-    login()
-
-  def run(self):
-    from clarifai.client import App, Model
-
-    # Run test before uploading
-    if not self.no_test:
-      assert os.path.exists(self.test_path), FileNotFoundError(f"Not found {self.test_path}")
-      result = subprocess.run(f"pytest -s --log-level=INFO {self.test_path}", shell=True)
-      assert result.returncode == 0, "Test has failed. Please make sure no error exists in your code."
-
-    clarifai_key_map = get_model_config(model_type=self.type).clarifai_model.field_maps
-    # inference parameters
-    inference_parameters = None
-    if isinstance(self.infer_param, str) and os.path.isfile(self.infer_param):
-      inference_parameters = InferParamManager(json_path=self.infer_param).get_list_params()
-    inputs = clarifai_key_map.input_fields_map
-    outputs = clarifai_key_map.output_fields_map
-
-    # if updating new version of existing model
-    def update_version():
-      model = Model(model_id=self.id, app_id=self.app_id)
-      if self.url:
-        model.create_version_by_url(
-            url=self.url,
-            input_field_maps=inputs,
-            output_field_maps=outputs,
-            inference_parameter_configs=inference_parameters,
-            description=self.desc)
-      elif self.file:
-        model.create_version_by_file(
-            file_path=self.file,
-            input_field_maps=inputs,
-            output_field_maps=outputs,
-            inference_parameter_configs=inference_parameters,
-            no_resume=self.no_resume,
-            description=self.desc)
-      else:
-        raise ValueError
-
-    if self.update_version:
-      update_version()
-    else:
-      # creating new model
-      _ = App(app_id=self.app_id).create_model(self.id, model_type_id=self.type)
-      update_version()

clarifai/models/model_serving/constants.py

@@ -1,21 +0,0 @@
-import os
-
-MAX_HW_DIM = 1024
-IMAGE_TENSOR_NAME = "image"
-TEXT_TENSOR_NAME = "text"
-
-BUILT_MODEL_EXT = ".clarifai"
-
-USER_CACHE_DIR = os.path.join(os.path.expanduser("~"), ".cache")
-CLARIFAI_HOME = os.path.expanduser(
-    os.getenv(
-        "CLARIFAI_HOME",
-        os.path.join(os.getenv("XDG_CACHE_HOME", USER_CACHE_DIR), "clarifai"),
-    ))
-os.makedirs(CLARIFAI_HOME, exist_ok=True)
-CLARIFAI_PAT_PATH = os.path.join(CLARIFAI_HOME, "pat")
-
-CLARIFAI_EXAMPLES_REPO = "https://github.com/Clarifai/examples.git"
-repo_name = CLARIFAI_EXAMPLES_REPO.split("/")[-1].replace(".git", "")
-CLARIFAI_EXAMPLES_REPO_PATH = os.path.join(CLARIFAI_HOME, repo_name)
-MODEL_UPLOAD_EXAMPLE_FOLDER = "model_upload"