clarifai 10.8.5__tar.gz → 10.8.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {clarifai-10.8.5/clarifai.egg-info → clarifai-10.8.7}/PKG-INFO +1 -1
- clarifai-10.8.7/clarifai/__init__.py +1 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/app.py +2 -3
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/auth/helper.py +9 -11
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/base.py +4 -3
- clarifai-10.8.7/clarifai/client/compute_cluster.py +196 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/dataset.py +2 -2
- clarifai-10.8.7/clarifai/client/deployment.py +51 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/input.py +2 -2
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/model.py +2 -2
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/module.py +2 -2
- clarifai-10.8.7/clarifai/client/nodepool.py +207 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/user.py +133 -2
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/workflow.py +2 -2
- clarifai-10.8.7/clarifai/constants/base.py +1 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/export/inputs_annotations.py +1 -3
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/rag/rag.py +2 -2
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/models/model_upload.py +89 -47
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/server.py +1 -1
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/utils/loader.py +13 -7
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/utils/url_fetcher.py +1 -1
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/utils/evaluation/helpers.py +1 -2
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/utils/evaluation/main.py +1 -2
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/utils/logging.py +6 -0
- {clarifai-10.8.5 → clarifai-10.8.7/clarifai.egg-info}/PKG-INFO +1 -1
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai.egg-info/SOURCES.txt +4 -1
- clarifai-10.8.5/clarifai/__init__.py +0 -1
- clarifai-10.8.5/clarifai/runners/utils/logging.py +0 -6
- {clarifai-10.8.5 → clarifai-10.8.7}/LICENSE +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/MANIFEST.in +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/README.md +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/cli.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/auth/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/auth/register.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/auth/stub.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/lister.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/client/search.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/constants/dataset.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/constants/input.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/constants/model.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/constants/rag.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/constants/search.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/constants/workflow.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/export/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/upload/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/upload/base.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/upload/features.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/upload/image.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/upload/loaders/README.md +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/upload/loaders/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/upload/loaders/coco_captions.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/upload/loaders/coco_detection.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/upload/loaders/imagenet_classification.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/upload/loaders/xview_detection.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/upload/multimodal.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/upload/text.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/datasets/upload/utils.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/errors.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/models/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/models/api.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/modules/README.md +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/modules/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/modules/css.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/modules/pages.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/modules/style.css +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/rag/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/rag/utils.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/dockerfile_template/Dockerfile.cpu.template +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/dockerfile_template/Dockerfile.cuda.template +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/models/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/models/base_typed_model.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/models/model_class.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/models/model_runner.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/models/model_servicer.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/utils/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/utils/data_handler.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/runners/utils/data_utils.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/schema/search.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/urls/helper.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/utils/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/utils/constants.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/utils/evaluation/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/utils/evaluation/testset_annotation_parser.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/utils/misc.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/utils/model_train.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/versions.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/workflows/__init__.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/workflows/export.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/workflows/utils.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai/workflows/validate.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai.egg-info/dependency_links.txt +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai.egg-info/entry_points.txt +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai.egg-info/requires.txt +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/clarifai.egg-info/top_level.txt +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/pyproject.toml +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/requirements.txt +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/setup.cfg +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/setup.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/tests/test_app.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/tests/test_auth.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/tests/test_data_upload.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/tests/test_eval.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/tests/test_misc.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/tests/test_model_predict.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/tests/test_model_train.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/tests/test_modules.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/tests/test_rag.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/tests/test_search.py +0 -0
- {clarifai-10.8.5 → clarifai-10.8.7}/tests/test_stub.py +0 -0
@@ -0,0 +1 @@
|
|
1
|
+
__version__ = "10.8.7"
|
@@ -19,8 +19,7 @@ from clarifai.client.workflow import Workflow
|
|
19
19
|
from clarifai.constants.model import TRAINABLE_MODEL_TYPES
|
20
20
|
from clarifai.errors import UserError
|
21
21
|
from clarifai.urls.helper import ClarifaiUrlHelper
|
22
|
-
from clarifai.utils.logging import
|
23
|
-
get_logger)
|
22
|
+
from clarifai.utils.logging import display_concept_relations_tree, display_workflow_tree, logger
|
24
23
|
from clarifai.utils.misc import concept_relations_accumulation
|
25
24
|
from clarifai.workflows.utils import get_yaml_output_info_proto, is_same_yaml_model
|
26
25
|
from clarifai.workflows.validate import validate
|
@@ -61,7 +60,7 @@ class App(Lister, BaseClient):
|
|
61
60
|
|
62
61
|
self.kwargs = {**kwargs, 'id': app_id}
|
63
62
|
self.app_info = resources_pb2.App(**self.kwargs)
|
64
|
-
self.logger =
|
63
|
+
self.logger = logger
|
65
64
|
BaseClient.__init__(
|
66
65
|
self,
|
67
66
|
user_id=self.user_id,
|
@@ -58,17 +58,15 @@ def https_cache(cache: dict, url: str) -> str:
|
|
58
58
|
|
59
59
|
class ClarifaiAuthHelper:
|
60
60
|
|
61
|
-
def __init__(
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
validate: bool = True,
|
71
|
-
):
|
61
|
+
def __init__(self,
|
62
|
+
user_id: str = "",
|
63
|
+
app_id: str = "",
|
64
|
+
pat: str = "",
|
65
|
+
token: str = "",
|
66
|
+
base: str = DEFAULT_BASE,
|
67
|
+
ui: str = DEFAULT_UI,
|
68
|
+
root_certificates_path: str = None,
|
69
|
+
validate: bool = True):
|
72
70
|
"""
|
73
71
|
A helper to get the authorization information needed to make API calls with the grpc
|
74
72
|
client to a specific app using a personal access token.
|
@@ -8,6 +8,7 @@ from google.protobuf.wrappers_pb2 import BoolValue
|
|
8
8
|
|
9
9
|
from clarifai.client.auth import create_stub
|
10
10
|
from clarifai.client.auth.helper import ClarifaiAuthHelper
|
11
|
+
from clarifai.constants.base import COMPUTE_ORCHESTRATION_RESOURCES
|
11
12
|
from clarifai.errors import ApiError, UserError
|
12
13
|
from clarifai.utils.constants import CLARIFAI_PAT_ENV_VAR, CLARIFAI_SESSION_TOKEN_ENV_VAR
|
13
14
|
from clarifai.utils.misc import get_from_dict_or_env
|
@@ -88,8 +89,8 @@ class BaseClient:
|
|
88
89
|
if kwargs.get("url"):
|
89
90
|
default_kwargs.pop("user_id", "")
|
90
91
|
default_kwargs.pop("app_id", "")
|
91
|
-
# Remove app_id if the class name
|
92
|
-
if
|
92
|
+
# Remove app_id if the class name is a compute orchestration resource
|
93
|
+
if any(co_resource in _clss.__name__ for co_resource in COMPUTE_ORCHESTRATION_RESOURCES):
|
93
94
|
default_kwargs.pop("app_id", "")
|
94
95
|
kwargs.update({**default_kwargs, "base_url": _base})
|
95
96
|
|
@@ -181,7 +182,7 @@ class BaseClient:
|
|
181
182
|
value = resources_pb2.ImageInfo(**value)
|
182
183
|
elif key == 'hosted_image_info':
|
183
184
|
continue
|
184
|
-
elif key in ['metadata']:
|
185
|
+
elif key in ['metadata', 'presets']:
|
185
186
|
if isinstance(value, dict) and value != {}:
|
186
187
|
value_s = struct_pb2.Struct()
|
187
188
|
value_s.update(value)
|
@@ -0,0 +1,196 @@
|
|
1
|
+
import os
|
2
|
+
from typing import Any, Dict, Generator, List
|
3
|
+
|
4
|
+
import yaml
|
5
|
+
from clarifai_grpc.grpc.api import resources_pb2, service_pb2
|
6
|
+
from clarifai_grpc.grpc.api.status import status_code_pb2
|
7
|
+
from google.protobuf.json_format import MessageToDict
|
8
|
+
|
9
|
+
from clarifai.client.base import BaseClient
|
10
|
+
from clarifai.client.lister import Lister
|
11
|
+
from clarifai.client.nodepool import Nodepool
|
12
|
+
from clarifai.errors import UserError
|
13
|
+
from clarifai.utils.logging import logger
|
14
|
+
|
15
|
+
|
16
|
+
class ComputeCluster(Lister, BaseClient):
|
17
|
+
"""ComputeCluster is a class that provides access to Clarifai API endpoints related to Compute Cluster information."""
|
18
|
+
|
19
|
+
def __init__(self,
|
20
|
+
compute_cluster_id: str = None,
|
21
|
+
user_id: str = None,
|
22
|
+
base_url: str = "https://api.clarifai.com",
|
23
|
+
pat: str = None,
|
24
|
+
token: str = None,
|
25
|
+
root_certificates_path: str = None,
|
26
|
+
**kwargs):
|
27
|
+
"""Initializes an ComputeCluster object.
|
28
|
+
|
29
|
+
Args:
|
30
|
+
compute_cluster_id (str): The ComputeCluster ID for the ComputeCluster to interact with.
|
31
|
+
user_id (str): The user ID of the user.
|
32
|
+
base_url (str): Base API url. Default "https://api.clarifai.com"
|
33
|
+
pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
|
34
|
+
token (str): A session token for authentication. Accepts either a session token or a pat. Can be set as env var CLARIFAI_SESSION_TOKEN
|
35
|
+
root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
|
36
|
+
**kwargs: Additional keyword arguments to be passed to the compute cluster.
|
37
|
+
"""
|
38
|
+
self.kwargs = {**kwargs, 'id': compute_cluster_id, 'user_id': user_id}
|
39
|
+
self.compute_cluster_info = resources_pb2.ComputeCluster(**self.kwargs)
|
40
|
+
self.logger = logger
|
41
|
+
BaseClient.__init__(
|
42
|
+
self,
|
43
|
+
user_id=self.user_id,
|
44
|
+
base=base_url,
|
45
|
+
pat=pat,
|
46
|
+
token=token,
|
47
|
+
root_certificates_path=root_certificates_path)
|
48
|
+
Lister.__init__(self)
|
49
|
+
|
50
|
+
def list_nodepools(self, page_no: int = None,
|
51
|
+
per_page: int = None) -> Generator[Nodepool, None, None]:
|
52
|
+
"""Lists all the available nodepools of the compute cluster.
|
53
|
+
|
54
|
+
Args:
|
55
|
+
compute_cluster_id (str): The compute cluster ID to list the nodepools.
|
56
|
+
page_no (int): The page number to list.
|
57
|
+
per_page (int): The number of items per page.
|
58
|
+
|
59
|
+
Yields:
|
60
|
+
Nodepool: Nodepool objects for the nodepools in the compute cluster.
|
61
|
+
|
62
|
+
Example:
|
63
|
+
>>> from clarifai.client.compute_cluster import ComputeCluster
|
64
|
+
>>> compute_cluster = ComputeCluster(compute_cluster_id="compute_cluster_id", user_id="user_id")
|
65
|
+
>>> all_nodepools = list(compute_cluster.list_nodepools())
|
66
|
+
|
67
|
+
Note:
|
68
|
+
Defaults to 16 per page if page_no is specified and per_page is not specified.
|
69
|
+
If both page_no and per_page are None, then lists all the resources.
|
70
|
+
"""
|
71
|
+
request_data = dict(user_app_id=self.user_app_id, compute_cluster_id=self.id)
|
72
|
+
all_nodepools_info = self.list_pages_generator(
|
73
|
+
self.STUB.ListNodepools,
|
74
|
+
service_pb2.ListNodepoolsRequest,
|
75
|
+
request_data,
|
76
|
+
per_page=per_page,
|
77
|
+
page_no=page_no)
|
78
|
+
|
79
|
+
for nodepool_info in all_nodepools_info:
|
80
|
+
yield Nodepool.from_auth_helper(auth=self.auth_helper, **nodepool_info)
|
81
|
+
|
82
|
+
def _process_nodepool_config(self, config_filepath: str) -> Dict[str, Any]:
|
83
|
+
with open(config_filepath, "r") as file:
|
84
|
+
nodepool_config = yaml.safe_load(file)
|
85
|
+
|
86
|
+
assert "nodepool" in nodepool_config, "nodepool info not found in the config file"
|
87
|
+
nodepool = nodepool_config['nodepool']
|
88
|
+
assert "instance_types" in nodepool, "region not found in the config file"
|
89
|
+
assert "node_capacity_type" in nodepool, "managed_by not found in the config file"
|
90
|
+
assert "max_instances" in nodepool, "cluster_type not found in the config file"
|
91
|
+
nodepool['compute_cluster'] = resources_pb2.ComputeCluster(id=self.id, user_id=self.user_id)
|
92
|
+
nodepool['node_capacity_type'] = resources_pb2.NodeCapacityType(capacity_types=[
|
93
|
+
capacity_type for capacity_type in nodepool['node_capacity_type']['capacity_types']
|
94
|
+
])
|
95
|
+
instance_types = []
|
96
|
+
for instance_type in nodepool['instance_types']:
|
97
|
+
if 'compute_info' in instance_type:
|
98
|
+
instance_type['compute_info'] = resources_pb2.ComputeInfo(**instance_type['compute_info'])
|
99
|
+
instance_types.append(resources_pb2.InstanceType(**instance_type))
|
100
|
+
nodepool['instance_types'] = instance_types
|
101
|
+
if "visibility" in nodepool:
|
102
|
+
nodepool["visibility"] = resources_pb2.Visibility(**nodepool["visibility"])
|
103
|
+
return nodepool
|
104
|
+
|
105
|
+
def create_nodepool(self, nodepool_id: str, config_filepath: str) -> Nodepool:
|
106
|
+
"""Creates a nodepool for the compute cluster.
|
107
|
+
|
108
|
+
Args:
|
109
|
+
nodepool_id (str): The nodepool ID for the nodepool to create.
|
110
|
+
config_filepath (str): The path to the nodepool config file.
|
111
|
+
|
112
|
+
Returns:
|
113
|
+
Nodepool: A Nodepool object for the specified nodepool ID.
|
114
|
+
|
115
|
+
Example:
|
116
|
+
>>> from clarifai.client.compute_cluster import ComputeCluster
|
117
|
+
>>> compute_cluster = ComputeCluster(compute_cluster_id="compute_cluster_id", user_id="user_id")
|
118
|
+
>>> nodepool = compute_cluster.create_nodepool(nodepool_id="nodepool_id", config_filepath="config.yml")
|
119
|
+
"""
|
120
|
+
if not os.path.exists(config_filepath):
|
121
|
+
raise UserError(f"Nodepool config file not found at {config_filepath}")
|
122
|
+
|
123
|
+
nodepool_config = self._process_nodepool_config(config_filepath)
|
124
|
+
|
125
|
+
if 'id' in nodepool_config:
|
126
|
+
nodepool_id = nodepool_config['id']
|
127
|
+
nodepool_config.pop('id')
|
128
|
+
|
129
|
+
request = service_pb2.PostNodepoolsRequest(
|
130
|
+
user_app_id=self.user_app_id,
|
131
|
+
compute_cluster_id=self.id,
|
132
|
+
nodepools=[resources_pb2.Nodepool(id=nodepool_id, **nodepool_config)])
|
133
|
+
response = self._grpc_request(self.STUB.PostNodepools, request)
|
134
|
+
if response.status.code != status_code_pb2.SUCCESS:
|
135
|
+
raise Exception(response.status)
|
136
|
+
self.logger.info("\nNodepool created\n%s", response.status)
|
137
|
+
|
138
|
+
return Nodepool.from_auth_helper(self.auth_helper, nodepool_id=nodepool_id)
|
139
|
+
|
140
|
+
def nodepool(self, nodepool_id: str) -> Nodepool:
|
141
|
+
"""Returns a Nodepool object for the existing nodepool ID.
|
142
|
+
|
143
|
+
Args:
|
144
|
+
nodepool_id (str): The nodepool ID for the nodepool to interact with.
|
145
|
+
|
146
|
+
Returns:
|
147
|
+
Nodepool: A Nodepool object for the existing nodepool ID.
|
148
|
+
|
149
|
+
Example:
|
150
|
+
>>> from clarifai.client.compute_cluster import ComputeCluster
|
151
|
+
>>> compute_cluster = ComputeCluster(compute_cluster_id="compute_cluster_id", user_id="user_id")
|
152
|
+
>>> nodepool = compute_cluster.nodepool(nodepool_id="nodepool_id")
|
153
|
+
"""
|
154
|
+
request = service_pb2.GetNodepoolRequest(
|
155
|
+
user_app_id=self.user_app_id, compute_cluster_id=self.id, nodepool_id=nodepool_id)
|
156
|
+
response = self._grpc_request(self.STUB.GetNodepool, request)
|
157
|
+
|
158
|
+
if response.status.code != status_code_pb2.SUCCESS:
|
159
|
+
raise Exception(response.status)
|
160
|
+
dict_response = MessageToDict(response, preserving_proto_field_name=True)
|
161
|
+
kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]],
|
162
|
+
list(dict_response.keys())[1])
|
163
|
+
|
164
|
+
return Nodepool.from_auth_helper(auth=self.auth_helper, **kwargs)
|
165
|
+
|
166
|
+
def delete_nodepools(self, nodepool_ids: List[str]) -> None:
|
167
|
+
"""Deletes list of nodepools for the compute cluster.
|
168
|
+
|
169
|
+
Args:
|
170
|
+
nodepool_ids (List[str]): The nodepool IDs of the compute cluster to delete.
|
171
|
+
|
172
|
+
Example:
|
173
|
+
>>> from clarifai.client.compute_cluster import ComputeCluster
|
174
|
+
>>> compute_cluster = ComputeCluster(compute_cluster_id="compute_cluster_id", user_id="user_id")
|
175
|
+
>>> compute_cluster.delete_nodepools(nodepool_ids=["nodepool_id1", "nodepool_id2"])
|
176
|
+
"""
|
177
|
+
assert isinstance(nodepool_ids, list), "nodepool_ids param should be a list"
|
178
|
+
|
179
|
+
request = service_pb2.DeleteNodepoolsRequest(
|
180
|
+
user_app_id=self.user_app_id, compute_cluster_id=self.id, ids=nodepool_ids)
|
181
|
+
response = self._grpc_request(self.STUB.DeleteNodepools, request)
|
182
|
+
|
183
|
+
if response.status.code != status_code_pb2.SUCCESS:
|
184
|
+
raise Exception(response.status)
|
185
|
+
self.logger.info("\nNodepools Deleted\n%s", response.status)
|
186
|
+
|
187
|
+
def __getattr__(self, name):
|
188
|
+
return getattr(self.compute_cluster_info, name)
|
189
|
+
|
190
|
+
def __str__(self):
|
191
|
+
init_params = [param for param in self.kwargs.keys()]
|
192
|
+
attribute_strings = [
|
193
|
+
f"{param}={getattr(self.compute_cluster_info, param)}" for param in init_params
|
194
|
+
if hasattr(self.compute_cluster_info, param)
|
195
|
+
]
|
196
|
+
return f"Clarifai Compute Cluster Details: \n{', '.join(attribute_strings)}\n"
|
@@ -30,7 +30,7 @@ from clarifai.datasets.upload.text import TextClassificationDataset
|
|
30
30
|
from clarifai.datasets.upload.utils import DisplayUploadStatus
|
31
31
|
from clarifai.errors import UserError
|
32
32
|
from clarifai.urls.helper import ClarifaiUrlHelper
|
33
|
-
from clarifai.utils.logging import add_file_handler,
|
33
|
+
from clarifai.utils.logging import add_file_handler, logger, process_log_files
|
34
34
|
from clarifai.utils.misc import BackoffIterator, Chunker
|
35
35
|
|
36
36
|
ClarifaiDatasetType = TypeVar('ClarifaiDatasetType', VisualClassificationDataset,
|
@@ -86,7 +86,7 @@ class Dataset(Lister, BaseClient):
|
|
86
86
|
token=token,
|
87
87
|
base_url=base_url,
|
88
88
|
root_certificates_path=root_certificates_path)
|
89
|
-
self.logger =
|
89
|
+
self.logger = logger
|
90
90
|
BaseClient.__init__(
|
91
91
|
self,
|
92
92
|
user_id=self.user_id,
|
@@ -0,0 +1,51 @@
|
|
1
|
+
from clarifai_grpc.grpc.api import resources_pb2
|
2
|
+
|
3
|
+
from clarifai.client.base import BaseClient
|
4
|
+
from clarifai.client.lister import Lister
|
5
|
+
from clarifai.utils.logging import logger
|
6
|
+
|
7
|
+
|
8
|
+
class Deployment(Lister, BaseClient):
|
9
|
+
"""Deployment is a class that provides access to Clarifai API endpoints related to Deployment information."""
|
10
|
+
|
11
|
+
def __init__(self,
|
12
|
+
deployment_id: str = None,
|
13
|
+
user_id: str = None,
|
14
|
+
base_url: str = "https://api.clarifai.com",
|
15
|
+
pat: str = None,
|
16
|
+
token: str = None,
|
17
|
+
root_certificates_path: str = None,
|
18
|
+
**kwargs):
|
19
|
+
"""Initializes a Deployment object.
|
20
|
+
|
21
|
+
Args:
|
22
|
+
deployment_id (str): The Deployment ID for the Deployment to interact with.
|
23
|
+
user_id (str): The user ID of the user.
|
24
|
+
base_url (str): Base API url. Default "https://api.clarifai.com"
|
25
|
+
pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
|
26
|
+
token (str): A session token for authentication. Accepts either a session token or a pat. Can be set as env var CLARIFAI_SESSION_TOKEN
|
27
|
+
root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
|
28
|
+
**kwargs: Additional keyword arguments to be passed to the deployment.
|
29
|
+
"""
|
30
|
+
self.kwargs = {**kwargs, 'id': deployment_id, 'user_id': user_id}
|
31
|
+
self.deployment_info = resources_pb2.Deployment(**self.kwargs)
|
32
|
+
self.logger = logger
|
33
|
+
BaseClient.__init__(
|
34
|
+
self,
|
35
|
+
user_id=user_id,
|
36
|
+
base=base_url,
|
37
|
+
pat=pat,
|
38
|
+
token=token,
|
39
|
+
root_certificates_path=root_certificates_path)
|
40
|
+
Lister.__init__(self)
|
41
|
+
|
42
|
+
def __getattr__(self, name):
|
43
|
+
return getattr(self.deployment_info, name)
|
44
|
+
|
45
|
+
def __str__(self):
|
46
|
+
init_params = [param for param in self.kwargs.keys()]
|
47
|
+
attribute_strings = [
|
48
|
+
f"{param}={getattr(self.deployment_info, param)}" for param in init_params
|
49
|
+
if hasattr(self.deployment_info, param)
|
50
|
+
]
|
51
|
+
return f"Deployment Details: \n{', '.join(attribute_strings)}\n"
|
@@ -21,7 +21,7 @@ from clarifai.client.lister import Lister
|
|
21
21
|
from clarifai.constants.dataset import MAX_RETRIES
|
22
22
|
from clarifai.constants.input import MAX_UPLOAD_BATCH_SIZE
|
23
23
|
from clarifai.errors import UserError
|
24
|
-
from clarifai.utils.logging import
|
24
|
+
from clarifai.utils.logging import logger
|
25
25
|
from clarifai.utils.misc import BackoffIterator, Chunker
|
26
26
|
|
27
27
|
|
@@ -52,7 +52,7 @@ class Inputs(Lister, BaseClient):
|
|
52
52
|
self.app_id = app_id
|
53
53
|
self.kwargs = {**kwargs}
|
54
54
|
self.input_info = resources_pb2.Input(**self.kwargs)
|
55
|
-
self.logger =
|
55
|
+
self.logger = logger
|
56
56
|
BaseClient.__init__(
|
57
57
|
self,
|
58
58
|
user_id=self.user_id,
|
@@ -23,7 +23,7 @@ from clarifai.constants.model import (CHUNK_SIZE, MAX_CHUNK_SIZE, MAX_MODEL_PRED
|
|
23
23
|
MODEL_EXPORT_TIMEOUT, RANGE_SIZE, TRAINABLE_MODEL_TYPES)
|
24
24
|
from clarifai.errors import UserError
|
25
25
|
from clarifai.urls.helper import ClarifaiUrlHelper
|
26
|
-
from clarifai.utils.logging import
|
26
|
+
from clarifai.utils.logging import logger
|
27
27
|
from clarifai.utils.misc import BackoffIterator
|
28
28
|
from clarifai.utils.model_train import (find_and_replace_key, params_parser,
|
29
29
|
response_to_model_params, response_to_param_info,
|
@@ -68,7 +68,7 @@ class Model(Lister, BaseClient):
|
|
68
68
|
kwargs = {'user_id': user_id, 'app_id': app_id}
|
69
69
|
self.kwargs = {**kwargs, 'id': model_id, 'model_version': model_version, }
|
70
70
|
self.model_info = resources_pb2.Model(**self.kwargs)
|
71
|
-
self.logger =
|
71
|
+
self.logger = logger
|
72
72
|
self.training_params = {}
|
73
73
|
BaseClient.__init__(
|
74
74
|
self,
|
@@ -6,7 +6,7 @@ from clarifai.client.base import BaseClient
|
|
6
6
|
from clarifai.client.lister import Lister
|
7
7
|
from clarifai.errors import UserError
|
8
8
|
from clarifai.urls.helper import ClarifaiUrlHelper
|
9
|
-
from clarifai.utils.logging import
|
9
|
+
from clarifai.utils.logging import logger
|
10
10
|
|
11
11
|
|
12
12
|
class Module(Lister, BaseClient):
|
@@ -44,7 +44,7 @@ class Module(Lister, BaseClient):
|
|
44
44
|
|
45
45
|
self.kwargs = {**kwargs, 'id': module_id, 'module_version': module_version}
|
46
46
|
self.module_info = resources_pb2.Module(**self.kwargs)
|
47
|
-
self.logger =
|
47
|
+
self.logger = logger
|
48
48
|
BaseClient.__init__(
|
49
49
|
self,
|
50
50
|
user_id=self.user_id,
|
@@ -0,0 +1,207 @@
|
|
1
|
+
import os
|
2
|
+
from typing import Any, Dict, Generator, List
|
3
|
+
|
4
|
+
import yaml
|
5
|
+
from clarifai_grpc.grpc.api import resources_pb2, service_pb2
|
6
|
+
from clarifai_grpc.grpc.api.status import status_code_pb2
|
7
|
+
from google.protobuf.json_format import MessageToDict
|
8
|
+
|
9
|
+
from clarifai.client.base import BaseClient
|
10
|
+
from clarifai.client.deployment import Deployment
|
11
|
+
from clarifai.client.lister import Lister
|
12
|
+
from clarifai.errors import UserError
|
13
|
+
from clarifai.utils.logging import logger
|
14
|
+
|
15
|
+
|
16
|
+
class Nodepool(Lister, BaseClient):
|
17
|
+
"""Nodepool is a class that provides access to Clarifai API endpoints related to Nodepool information."""
|
18
|
+
|
19
|
+
def __init__(self,
|
20
|
+
nodepool_id: str = None,
|
21
|
+
user_id: str = None,
|
22
|
+
base_url: str = "https://api.clarifai.com",
|
23
|
+
pat: str = None,
|
24
|
+
token: str = None,
|
25
|
+
root_certificates_path: str = None,
|
26
|
+
**kwargs):
|
27
|
+
"""Initializes a Nodepool object.
|
28
|
+
|
29
|
+
Args:
|
30
|
+
nodepool_id (str): The Nodepool ID for the Nodepool to interact with.
|
31
|
+
user_id (str): The user ID of the user.
|
32
|
+
base_url (str): Base API url. Default "https://api.clarifai.com"
|
33
|
+
pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
|
34
|
+
token (str): A session token for authentication. Accepts either a session token or a pat. Can be set as env var CLARIFAI_SESSION_TOKEN
|
35
|
+
root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
|
36
|
+
**kwargs: Additional keyword arguments to be passed to the nodepool.
|
37
|
+
"""
|
38
|
+
self.kwargs = {**kwargs, 'id': nodepool_id}
|
39
|
+
self.nodepool_info = resources_pb2.Nodepool(**self.kwargs)
|
40
|
+
self.logger = logger
|
41
|
+
BaseClient.__init__(
|
42
|
+
self,
|
43
|
+
user_id=user_id,
|
44
|
+
base=base_url,
|
45
|
+
pat=pat,
|
46
|
+
token=token,
|
47
|
+
root_certificates_path=root_certificates_path)
|
48
|
+
Lister.__init__(self)
|
49
|
+
|
50
|
+
def list_deployments(self,
|
51
|
+
filter_by: Dict[str, Any] = {},
|
52
|
+
page_no: int = None,
|
53
|
+
per_page: int = None) -> Generator[Deployment, None, None]:
|
54
|
+
"""Lists all the available deployments of compute cluster.
|
55
|
+
|
56
|
+
Args:
|
57
|
+
filter_by (Dict[str, Any]): The filter to apply to the list of deployments.
|
58
|
+
page_no (int): The page number to list.
|
59
|
+
per_page (int): The number of items per page.
|
60
|
+
|
61
|
+
Yields:
|
62
|
+
Deployment: Deployment objects for the nodepools in the compute cluster.
|
63
|
+
|
64
|
+
Example:
|
65
|
+
>>> from clarifai.client.nodepool import Nodepool
|
66
|
+
>>> nodepool = Nodepool(nodepool_id="nodepool_id", user_id="user_id")
|
67
|
+
>>> all_deployments = list(nodepool.list_deployments())
|
68
|
+
|
69
|
+
Note:
|
70
|
+
Defaults to 16 per page if page_no is specified and per_page is not specified.
|
71
|
+
If both page_no and per_page are None, then lists all the resources.
|
72
|
+
"""
|
73
|
+
request_data = dict(user_app_id=self.user_app_id, nodepool_id=self.id, **filter_by)
|
74
|
+
all_deployments_info = self.list_pages_generator(
|
75
|
+
self.STUB.ListDeployments,
|
76
|
+
service_pb2.ListDeploymentsRequest,
|
77
|
+
request_data,
|
78
|
+
per_page=per_page,
|
79
|
+
page_no=page_no)
|
80
|
+
|
81
|
+
for deployment_info in all_deployments_info:
|
82
|
+
yield Deployment.from_auth_helper(auth=self.auth_helper, **deployment_info)
|
83
|
+
|
84
|
+
def _process_deployment_config(self, config_filepath: str) -> Dict[str, Any]:
|
85
|
+
with open(config_filepath, "r") as file:
|
86
|
+
deployment_config = yaml.safe_load(file)
|
87
|
+
|
88
|
+
assert "deployment" in deployment_config, "deployment info not found in the config file"
|
89
|
+
deployment = deployment_config['deployment']
|
90
|
+
assert "autoscale_config" in deployment, "autoscale_config not found in the config file"
|
91
|
+
assert ("worker" in deployment) and (
|
92
|
+
("model" in deployment["worker"]) or
|
93
|
+
("workflow" in deployment["worker"])), "worker info not found in the config file"
|
94
|
+
assert "scheduling_choice" in deployment, "scheduling_choice not found in the config file"
|
95
|
+
assert "nodepools" in deployment, "nodepools not found in the config file"
|
96
|
+
deployment['user_id'] = self.user_app_id.user_id
|
97
|
+
deployment['autoscale_config'] = resources_pb2.AutoscaleConfig(
|
98
|
+
**deployment['autoscale_config'])
|
99
|
+
deployment['nodepools'] = [
|
100
|
+
resources_pb2.Nodepool(
|
101
|
+
id=nodepool['id'],
|
102
|
+
compute_cluster=resources_pb2.ComputeCluster(
|
103
|
+
id=nodepool['compute_cluster']['id'], user_id=self.user_app_id.user_id))
|
104
|
+
for nodepool in deployment['nodepools']
|
105
|
+
]
|
106
|
+
if 'user' in deployment['worker']:
|
107
|
+
deployment['worker']['user'] = resources_pb2.User(**deployment['worker']['user'])
|
108
|
+
elif 'model' in deployment['worker']:
|
109
|
+
deployment['worker']['model'] = resources_pb2.Model(**deployment['worker']['model'])
|
110
|
+
elif 'workflow' in deployment['worker']:
|
111
|
+
deployment['worker']['workflow'] = resources_pb2.Workflow(**deployment['worker']['workflow'])
|
112
|
+
deployment['worker'] = resources_pb2.Worker(**deployment['worker'])
|
113
|
+
if "visibility" in deployment:
|
114
|
+
deployment["visibility"] = resources_pb2.Visibility(**deployment["visibility"])
|
115
|
+
return deployment
|
116
|
+
|
117
|
+
def create_deployment(self, deployment_id: str, config_filepath: str) -> Deployment:
|
118
|
+
"""Creates a deployment for the nodepool.
|
119
|
+
|
120
|
+
Args:
|
121
|
+
deployment_id (str): The deployment ID for the deployment to create.
|
122
|
+
config_filepath (str): The path to the deployment config file.
|
123
|
+
|
124
|
+
Returns:
|
125
|
+
Deployment: A Deployment object for the specified deployment ID.
|
126
|
+
|
127
|
+
Example:
|
128
|
+
>>> from clarifai.client.nodepool import Nodepool
|
129
|
+
>>> nodepool = Nodepool(nodepool_id="nodepool_id", user_id="user_id")
|
130
|
+
>>> deployment = nodepool.create_deployment(deployment_id="deployment_id", config_filepath="config.yml")
|
131
|
+
"""
|
132
|
+
if not os.path.exists(config_filepath):
|
133
|
+
raise UserError(f"Deployment config file not found at {config_filepath}")
|
134
|
+
|
135
|
+
deployment_config = self._process_deployment_config(config_filepath)
|
136
|
+
|
137
|
+
if 'id' in deployment_config:
|
138
|
+
deployment_id = deployment_config['id']
|
139
|
+
deployment_config.pop('id')
|
140
|
+
|
141
|
+
request = service_pb2.PostDeploymentsRequest(
|
142
|
+
user_app_id=self.user_app_id,
|
143
|
+
deployments=[resources_pb2.Deployment(id=deployment_id, **deployment_config)])
|
144
|
+
response = self._grpc_request(self.STUB.PostDeployments, request)
|
145
|
+
if response.status.code != status_code_pb2.SUCCESS:
|
146
|
+
raise Exception(response.status)
|
147
|
+
self.logger.info("\nDeployment created\n%s", response.status)
|
148
|
+
|
149
|
+
return Deployment.from_auth_helper(self.auth_helper, deployment_id=deployment_id)
|
150
|
+
|
151
|
+
def deployment(self, deployment_id: str) -> Deployment:
|
152
|
+
"""Returns a Deployment object for the existing deployment ID.
|
153
|
+
|
154
|
+
Args:
|
155
|
+
deployment_id (str): The deployment ID for the deployment to interact with.
|
156
|
+
|
157
|
+
Returns:
|
158
|
+
Deployment: A Deployment object for the existing deployment ID.
|
159
|
+
|
160
|
+
Example:
|
161
|
+
>>> from clarifai.client.nodepool import Nodepool
|
162
|
+
>>> nodepool = Nodepool(nodepool_id="nodepool_id", user_id="user_id")
|
163
|
+
>>> deployment = nodepool.deployment(deployment_id="deployment_id")
|
164
|
+
"""
|
165
|
+
request = service_pb2.GetDeploymentRequest(
|
166
|
+
user_app_id=self.user_app_id, deployment_id=deployment_id)
|
167
|
+
response = self._grpc_request(self.STUB.GetDeployment, request)
|
168
|
+
|
169
|
+
if response.status.code != status_code_pb2.SUCCESS:
|
170
|
+
raise Exception(response.status)
|
171
|
+
dict_response = MessageToDict(response, preserving_proto_field_name=True)
|
172
|
+
kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]],
|
173
|
+
list(dict_response.keys())[1])
|
174
|
+
|
175
|
+
return Deployment.from_auth_helper(auth=self.auth_helper, **kwargs)
|
176
|
+
|
177
|
+
def delete_deployments(self, deployment_ids: List[str]) -> None:
|
178
|
+
"""Deletes list of deployments for the nodepool.
|
179
|
+
|
180
|
+
Args:
|
181
|
+
deployment_ids (List[str]): The list of deployment IDs to delete.
|
182
|
+
|
183
|
+
Example:
|
184
|
+
>>> from clarifai.client.nodepool import Nodepool
|
185
|
+
>>> nodepool = Nodepool(nodepool_id="nodepool_id", user_id="user_id")
|
186
|
+
>>> nodepool.delete_deployments(deployment_ids=["deployment_id1", "deployment_id2"])
|
187
|
+
"""
|
188
|
+
assert isinstance(deployment_ids, list), "deployment_ids param should be a list"
|
189
|
+
|
190
|
+
request = service_pb2.DeleteDeploymentsRequest(
|
191
|
+
user_app_id=self.user_app_id, ids=deployment_ids)
|
192
|
+
response = self._grpc_request(self.STUB.DeleteDeployments, request)
|
193
|
+
|
194
|
+
if response.status.code != status_code_pb2.SUCCESS:
|
195
|
+
raise Exception(response.status)
|
196
|
+
self.logger.info("\nDeployments Deleted\n%s", response.status)
|
197
|
+
|
198
|
+
def __getattr__(self, name):
|
199
|
+
return getattr(self.nodepool_info, name)
|
200
|
+
|
201
|
+
def __str__(self):
|
202
|
+
init_params = [param for param in self.kwargs.keys()]
|
203
|
+
attribute_strings = [
|
204
|
+
f"{param}={getattr(self.nodepool_info, param)}" for param in init_params
|
205
|
+
if hasattr(self.nodepool_info, param)
|
206
|
+
]
|
207
|
+
return f"Nodepool Details: \n{', '.join(attribute_strings)}\n"
|