clarifai 11.2.1__tar.gz → 11.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {clarifai-11.2.1/clarifai.egg-info → clarifai-11.2.3}/PKG-INFO +2 -2
- clarifai-11.2.3/clarifai/__init__.py +1 -0
- clarifai-11.2.3/clarifai/cli/base.py +260 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/cli/compute_cluster.py +24 -21
- clarifai-11.2.3/clarifai/cli/deployment.py +112 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/cli/model.py +1 -1
- clarifai-11.2.3/clarifai/cli/nodepool.py +112 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/app.py +1 -1
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/auth/stub.py +4 -5
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/dataset.py +3 -4
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/models/model_builder.py +38 -15
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/models/model_run_locally.py +2 -1
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/utils/const.py +9 -8
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/utils/loader.py +36 -6
- clarifai-11.2.3/clarifai/utils/cli.py +168 -0
- clarifai-11.2.3/clarifai/utils/config.py +105 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/utils/constants.py +4 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/utils/logging.py +64 -21
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/utils/misc.py +2 -0
- {clarifai-11.2.1 → clarifai-11.2.3/clarifai.egg-info}/PKG-INFO +2 -2
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai.egg-info/SOURCES.txt +1 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai.egg-info/requires.txt +1 -1
- {clarifai-11.2.1 → clarifai-11.2.3}/requirements.txt +1 -1
- {clarifai-11.2.1 → clarifai-11.2.3}/tests/test_misc.py +17 -5
- clarifai-11.2.1/clarifai/__init__.py +0 -1
- clarifai-11.2.1/clarifai/cli/base.py +0 -124
- clarifai-11.2.1/clarifai/cli/deployment.py +0 -88
- clarifai-11.2.1/clarifai/cli/nodepool.py +0 -94
- clarifai-11.2.1/clarifai/utils/cli.py +0 -79
- {clarifai-11.2.1 → clarifai-11.2.3}/LICENSE +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/MANIFEST.in +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/README.md +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/cli/README.md +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/cli/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/cli/__main__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/cli.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/auth/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/auth/helper.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/auth/register.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/base.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/compute_cluster.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/deployment.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/input.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/lister.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/model.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/module.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/nodepool.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/search.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/user.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/client/workflow.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/constants/base.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/constants/dataset.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/constants/input.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/constants/model.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/constants/rag.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/constants/search.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/constants/workflow.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/export/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/export/inputs_annotations.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/upload/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/upload/base.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/upload/features.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/upload/image.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/upload/loaders/README.md +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/upload/loaders/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/upload/loaders/coco_captions.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/upload/loaders/coco_detection.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/upload/loaders/imagenet_classification.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/upload/loaders/xview_detection.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/upload/multimodal.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/upload/text.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/datasets/upload/utils.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/errors.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/models/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/models/api.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/modules/README.md +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/modules/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/modules/css.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/modules/pages.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/modules/style.css +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/rag/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/rag/rag.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/rag/utils.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/dockerfile_template/Dockerfile.template +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/models/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/models/base_typed_model.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/models/model_class.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/models/model_runner.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/models/model_servicer.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/server.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/utils/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/utils/data_handler.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/utils/data_utils.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/runners/utils/url_fetcher.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/schema/search.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/urls/helper.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/utils/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/utils/evaluation/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/utils/evaluation/helpers.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/utils/evaluation/main.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/utils/evaluation/testset_annotation_parser.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/utils/model_train.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/versions.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/workflows/__init__.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/workflows/export.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/workflows/utils.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai/workflows/validate.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai.egg-info/dependency_links.txt +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai.egg-info/entry_points.txt +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/clarifai.egg-info/top_level.txt +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/pyproject.toml +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/setup.cfg +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/setup.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/tests/test_app.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/tests/test_auth.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/tests/test_data_upload.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/tests/test_eval.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/tests/test_model_predict.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/tests/test_model_train.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/tests/test_modules.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/tests/test_rag.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/tests/test_search.py +0 -0
- {clarifai-11.2.1 → clarifai-11.2.3}/tests/test_stub.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: clarifai
|
3
|
-
Version: 11.2.
|
3
|
+
Version: 11.2.3
|
4
4
|
Summary: Clarifai Python SDK
|
5
5
|
Home-page: https://github.com/Clarifai/clarifai-python
|
6
6
|
Author: Clarifai
|
@@ -21,7 +21,7 @@ Requires-Python: >=3.8
|
|
21
21
|
Description-Content-Type: text/markdown
|
22
22
|
License-File: LICENSE
|
23
23
|
Requires-Dist: clarifai-grpc>=11.2.6
|
24
|
-
Requires-Dist: clarifai-protocol>=0.0.
|
24
|
+
Requires-Dist: clarifai-protocol>=0.0.21
|
25
25
|
Requires-Dist: numpy>=1.22.0
|
26
26
|
Requires-Dist: tqdm>=4.65.0
|
27
27
|
Requires-Dist: rich>=13.4.2
|
@@ -0,0 +1 @@
|
|
1
|
+
__version__ = "11.2.3"
|
@@ -0,0 +1,260 @@
|
|
1
|
+
import json
|
2
|
+
import os
|
3
|
+
import sys
|
4
|
+
|
5
|
+
import click
|
6
|
+
import yaml
|
7
|
+
|
8
|
+
from clarifai.utils.cli import AliasedGroup, TableFormatter, load_command_modules
|
9
|
+
from clarifai.utils.config import Config, Context
|
10
|
+
from clarifai.utils.constants import DEFAULT_CONFIG
|
11
|
+
from clarifai.utils.logging import logger
|
12
|
+
|
13
|
+
|
14
|
+
#@click.group(cls=CustomMultiGroup)
|
15
|
+
@click.group(cls=AliasedGroup)
|
16
|
+
@click.option('--config', default=DEFAULT_CONFIG)
|
17
|
+
@click.pass_context
|
18
|
+
def cli(ctx, config):
|
19
|
+
"""Clarifai CLI"""
|
20
|
+
ctx.ensure_object(dict)
|
21
|
+
if os.path.exists(config):
|
22
|
+
cfg = Config.from_yaml(filename=config)
|
23
|
+
ctx.obj = cfg
|
24
|
+
else:
|
25
|
+
cfg = Config(
|
26
|
+
filename=config,
|
27
|
+
current_context='default',
|
28
|
+
contexts={
|
29
|
+
'default':
|
30
|
+
Context(
|
31
|
+
'default',
|
32
|
+
CLARIFAI_PAT=os.environ.get('CLARIFAI_PAT', ''),
|
33
|
+
CLARIFAI_USER_ID=os.environ.get('CLARIFAI_USER_ID', ''),
|
34
|
+
CLARIFAI_API_BASE=os.environ.get('CLARIFAI_API_BASE', 'api.clarifai.com'),
|
35
|
+
)
|
36
|
+
})
|
37
|
+
try:
|
38
|
+
cfg.to_yaml(config)
|
39
|
+
except Exception:
|
40
|
+
logger.warning("Could not write configuration to disk. Could be a read only file system.")
|
41
|
+
ctx.obj = cfg # still have the default config even if couldn't write.
|
42
|
+
|
43
|
+
|
44
|
+
@cli.command()
|
45
|
+
@click.argument('shell', type=click.Choice(['bash', 'zsh']))
|
46
|
+
def shell_completion(shell):
|
47
|
+
"""Shell completion script"""
|
48
|
+
os.system(f"_CLARIFAI_COMPLETE={shell}_source clarifai")
|
49
|
+
|
50
|
+
|
51
|
+
@cli.group(['cfg'], cls=AliasedGroup)
|
52
|
+
def config():
|
53
|
+
"""Manage CLI configuration"""
|
54
|
+
|
55
|
+
|
56
|
+
@config.command(['e'])
|
57
|
+
@click.pass_context
|
58
|
+
def edit(ctx):
|
59
|
+
"""Edit the configuration file"""
|
60
|
+
os.system(f'{os.environ.get("EDITOR", "vi")} {ctx.obj.filename}')
|
61
|
+
|
62
|
+
|
63
|
+
@config.command(['current'])
|
64
|
+
@click.option('-o', '--output-format', default='name', type=click.Choice(['name', 'json', 'yaml']))
|
65
|
+
@click.pass_context
|
66
|
+
def current_context(ctx, output_format):
|
67
|
+
"""Get the current context"""
|
68
|
+
if output_format == 'name':
|
69
|
+
print(ctx.obj.current_context)
|
70
|
+
else:
|
71
|
+
if output_format == 'json':
|
72
|
+
print(json.dumps(ctx.obj.contexts[ctx.obj.current_context].to_serializable_dict()))
|
73
|
+
else:
|
74
|
+
print(yaml.safe_dump(ctx.obj.contexts[ctx.obj.current_context].to_serializable_dict()))
|
75
|
+
|
76
|
+
|
77
|
+
@config.command(['list', 'ls'])
|
78
|
+
@click.option(
|
79
|
+
'-o', '--output-format', default='wide', type=click.Choice(['wide', 'name', 'json', 'yaml']))
|
80
|
+
@click.pass_context
|
81
|
+
def get_contexts(ctx, output_format):
|
82
|
+
"""Get all contexts"""
|
83
|
+
if output_format == 'wide':
|
84
|
+
formatter = TableFormatter(custom_columns={
|
85
|
+
'': lambda c: '*' if c.name == ctx.obj.current_context else '',
|
86
|
+
'NAME': lambda c: c.name,
|
87
|
+
'USER_ID': lambda c: c.user_id,
|
88
|
+
'API_BASE': lambda c: c.api_base,
|
89
|
+
'PAT': lambda c: pat_display(c.pat),
|
90
|
+
})
|
91
|
+
print(formatter.format(ctx.obj.contexts.values(), fmt="plain"))
|
92
|
+
elif output_format == 'name':
|
93
|
+
print('\n'.join(ctx.obj.contexts))
|
94
|
+
elif output_format in ('json', 'yaml'):
|
95
|
+
dicts = [v.__dict__ for c, v in ctx.obj.contexts.items()]
|
96
|
+
for d in dicts:
|
97
|
+
d.pop('pat')
|
98
|
+
if output_format == 'json':
|
99
|
+
print(json.dumps(dicts))
|
100
|
+
elif output_format == 'yaml':
|
101
|
+
print(yaml.safe_dump(dicts))
|
102
|
+
|
103
|
+
|
104
|
+
@config.command(['use'])
|
105
|
+
@click.argument('context-name', type=str)
|
106
|
+
@click.pass_context
|
107
|
+
def use_context(ctx, context_name):
|
108
|
+
"""Set the current context"""
|
109
|
+
if context_name not in ctx.obj.contexts:
|
110
|
+
raise click.UsageError('Context not found')
|
111
|
+
ctx.obj.current_context = context_name
|
112
|
+
ctx.obj.to_yaml()
|
113
|
+
print(f'Set {context_name} as the current context')
|
114
|
+
|
115
|
+
|
116
|
+
@config.command(['cat'])
|
117
|
+
@click.option('-o', '--output-format', default='yaml', type=click.Choice(['yaml', 'json']))
|
118
|
+
@click.pass_obj
|
119
|
+
def dump(ctx_obj, output_format):
|
120
|
+
"""Dump the configuration to stdout"""
|
121
|
+
if output_format == 'yaml':
|
122
|
+
yaml.safe_dump(ctx_obj.to_dict(), sys.stdout)
|
123
|
+
else:
|
124
|
+
json.dump(ctx_obj.to_dict(), sys.stdout, indent=2)
|
125
|
+
|
126
|
+
|
127
|
+
@cli.command()
|
128
|
+
@click.argument('api_url', default="https://api.clarifai.com")
|
129
|
+
@click.option('--user_id', required=False, help='User ID')
|
130
|
+
@click.pass_context
|
131
|
+
def login(ctx, api_url, user_id):
|
132
|
+
"""Login command to set PAT and other configurations."""
|
133
|
+
|
134
|
+
name = input('context name (default: "default"): ')
|
135
|
+
user_id = user_id if user_id is not None else input('user id: ')
|
136
|
+
pat = input_or_default(
|
137
|
+
'personal access token value (default: "ENVVAR" to get our of env var rather than config): ',
|
138
|
+
'ENVVAR')
|
139
|
+
|
140
|
+
context = Context(
|
141
|
+
name,
|
142
|
+
CLARIFAI_API_BASE=api_url,
|
143
|
+
CLARIFAI_USER_ID=user_id,
|
144
|
+
CLARIFAI_PAT=pat,
|
145
|
+
)
|
146
|
+
|
147
|
+
if context.name == '':
|
148
|
+
context.name = 'default'
|
149
|
+
|
150
|
+
ctx.obj.contexts[context.name] = context
|
151
|
+
ctx.obj.current_context = context.name
|
152
|
+
|
153
|
+
ctx.obj.to_yaml()
|
154
|
+
|
155
|
+
|
156
|
+
@cli.group(cls=AliasedGroup)
|
157
|
+
def context():
|
158
|
+
"""Manage contexts"""
|
159
|
+
|
160
|
+
|
161
|
+
def pat_display(pat):
|
162
|
+
return pat[:5] + "****"
|
163
|
+
|
164
|
+
|
165
|
+
@context.command(['ls'])
|
166
|
+
@click.pass_context
|
167
|
+
def list(ctx):
|
168
|
+
"""List available contexts"""
|
169
|
+
formatter = TableFormatter(custom_columns={
|
170
|
+
'': lambda c: '*' if c.name == ctx.obj.current_context else '',
|
171
|
+
'NAME': lambda c: c.name,
|
172
|
+
'USER_ID': lambda c: c.user_id,
|
173
|
+
'API_BASE': lambda c: c.api_base,
|
174
|
+
'PAT': lambda c: pat_display(c.pat)
|
175
|
+
})
|
176
|
+
print(formatter.format(ctx.obj.contexts.values(), fmt="plain"))
|
177
|
+
|
178
|
+
|
179
|
+
def input_or_default(prompt, default):
|
180
|
+
value = input(prompt)
|
181
|
+
return value if value else default
|
182
|
+
|
183
|
+
|
184
|
+
@context.command()
|
185
|
+
@click.argument('name')
|
186
|
+
@click.option('--user-id', required=False, help='User ID')
|
187
|
+
@click.option('--base-url', required=False, help='Base URL')
|
188
|
+
@click.option('--pat', required=False, help='Personal access token')
|
189
|
+
@click.pass_context
|
190
|
+
def create(
|
191
|
+
ctx,
|
192
|
+
name,
|
193
|
+
user_id=None,
|
194
|
+
base_url=None,
|
195
|
+
pat=None,
|
196
|
+
):
|
197
|
+
"""Create a new context"""
|
198
|
+
if name in ctx.obj.contexts:
|
199
|
+
print(f'{name} already exists')
|
200
|
+
exit(1)
|
201
|
+
if not user_id:
|
202
|
+
user_id = input('user id: ')
|
203
|
+
if not base_url:
|
204
|
+
base_url = input_or_default('base url (default: https://api.clarifai.com): ',
|
205
|
+
'https://api.clarifai.com')
|
206
|
+
if not pat:
|
207
|
+
pat = input_or_default(
|
208
|
+
'personal access token value (default: "ENVVAR" to get our of env var rather than config): ',
|
209
|
+
'ENVVAR')
|
210
|
+
|
211
|
+
context = Context(name, CLARIFAI_USER_ID=user_id, CLARIFAI_API_BASE=base_url, CLARIFAI_PAT=pat)
|
212
|
+
ctx.obj.contexts[context.name] = context
|
213
|
+
ctx.obj.to_yaml()
|
214
|
+
|
215
|
+
|
216
|
+
# write a click command to delete a context
|
217
|
+
@context.command(['rm'])
|
218
|
+
@click.argument('name')
|
219
|
+
@click.pass_context
|
220
|
+
def delete(ctx, name):
|
221
|
+
"""Delete a context"""
|
222
|
+
if name not in ctx.obj.contexts:
|
223
|
+
print(f'{name} is not a valid context')
|
224
|
+
exit(1)
|
225
|
+
ctx.obj.contexts.pop(name)
|
226
|
+
ctx.obj.to_yaml()
|
227
|
+
print(f'{name} deleted')
|
228
|
+
|
229
|
+
|
230
|
+
@context.command()
|
231
|
+
@click.argument('name', type=str)
|
232
|
+
@click.pass_context
|
233
|
+
def use(ctx, name):
|
234
|
+
"""Set the current context"""
|
235
|
+
if name not in ctx.obj.contexts:
|
236
|
+
raise click.UsageError('Context not found')
|
237
|
+
ctx.obj.current_context = name
|
238
|
+
ctx.obj.to_yaml()
|
239
|
+
print(f'Set {name} as the current context')
|
240
|
+
|
241
|
+
|
242
|
+
@cli.command()
|
243
|
+
@click.argument('script', type=str)
|
244
|
+
@click.option('--context', type=str, help='Context to use')
|
245
|
+
@click.pass_context
|
246
|
+
def run(ctx, script, context=None):
|
247
|
+
"""Execute a script with the current context's environment"""
|
248
|
+
context = ctx.obj.current if not context else context
|
249
|
+
cmd = f'CLARIFAI_USER_ID={context.user_id} CLARIFAI_API_BASE={context.base_url} CLARIFAI_PAT={context.pat} '
|
250
|
+
cmd += ' '.join([f'{k}={v}' for k, v in context.env.items()])
|
251
|
+
cmd += f' {script}'
|
252
|
+
os.system(cmd)
|
253
|
+
|
254
|
+
|
255
|
+
# Import the CLI commands to register them
|
256
|
+
load_command_modules()
|
257
|
+
|
258
|
+
|
259
|
+
def main():
|
260
|
+
cli()
|
@@ -1,58 +1,61 @@
|
|
1
1
|
import click
|
2
2
|
|
3
3
|
from clarifai.cli.base import cli
|
4
|
-
from clarifai.
|
5
|
-
from clarifai.utils.cli import display_co_resources, validate_context
|
4
|
+
from clarifai.utils.cli import AliasedGroup, display_co_resources, validate_context
|
6
5
|
|
7
6
|
|
8
|
-
@cli.group(['computecluster', 'cc'])
|
7
|
+
@cli.group(['computecluster', 'cc'], cls=AliasedGroup)
|
9
8
|
def computecluster():
|
10
9
|
"""Manage Compute Clusters: create, delete, list"""
|
11
|
-
pass
|
12
10
|
|
13
11
|
|
14
|
-
@computecluster.command()
|
12
|
+
@computecluster.command(['c'])
|
13
|
+
@click.argument('compute_cluster_id')
|
15
14
|
@click.option(
|
16
15
|
'--config',
|
17
16
|
type=click.Path(exists=True),
|
18
17
|
required=True,
|
19
18
|
help='Path to the compute cluster config file.')
|
20
|
-
@click.option(
|
21
|
-
'-cc_id',
|
22
|
-
'--compute_cluster_id',
|
23
|
-
required=False,
|
24
|
-
help='New Compute Cluster ID for the compute cluster to create.')
|
25
19
|
@click.pass_context
|
26
|
-
def create(ctx,
|
20
|
+
def create(ctx, compute_cluster_id, config):
|
27
21
|
"""Create a new Compute Cluster with the given config file."""
|
28
|
-
|
22
|
+
from clarifai.client.user import User
|
29
23
|
validate_context(ctx)
|
30
|
-
user = User(
|
24
|
+
user = User(
|
25
|
+
user_id=ctx.obj.current.user_id, pat=ctx.obj.current.pat, base_url=ctx.obj.current.api_base)
|
31
26
|
if compute_cluster_id:
|
32
27
|
user.create_compute_cluster(config, compute_cluster_id=compute_cluster_id)
|
33
28
|
else:
|
34
29
|
user.create_compute_cluster(config)
|
35
30
|
|
36
31
|
|
37
|
-
@computecluster.command()
|
32
|
+
@computecluster.command(['ls'])
|
38
33
|
@click.option('--page_no', required=False, help='Page number to list.', default=1)
|
39
34
|
@click.option('--per_page', required=False, help='Number of items per page.', default=16)
|
40
35
|
@click.pass_context
|
41
36
|
def list(ctx, page_no, per_page):
|
42
37
|
"""List all compute clusters for the user."""
|
43
|
-
|
38
|
+
from clarifai.client.user import User
|
44
39
|
validate_context(ctx)
|
45
|
-
user = User(
|
40
|
+
user = User(
|
41
|
+
user_id=ctx.obj.current.user_id, pat=ctx.obj.current.pat, base_url=ctx.obj.current.api_base)
|
46
42
|
response = user.list_compute_clusters(page_no, per_page)
|
47
|
-
display_co_resources(
|
43
|
+
display_co_resources(
|
44
|
+
response,
|
45
|
+
custom_columns={
|
46
|
+
'ID': lambda c: c.id,
|
47
|
+
'USER_ID': lambda c: c.user_id,
|
48
|
+
'DESCRIPTION': lambda c: c.description,
|
49
|
+
})
|
48
50
|
|
49
51
|
|
50
|
-
@computecluster.command()
|
51
|
-
@click.
|
52
|
+
@computecluster.command(['rm'])
|
53
|
+
@click.argument('compute_cluster_id')
|
52
54
|
@click.pass_context
|
53
55
|
def delete(ctx, compute_cluster_id):
|
54
56
|
"""Deletes a compute cluster for the user."""
|
55
|
-
|
57
|
+
from clarifai.client.user import User
|
56
58
|
validate_context(ctx)
|
57
|
-
user = User(
|
59
|
+
user = User(
|
60
|
+
user_id=ctx.obj.current.user_id, pat=ctx.obj.current.pat, base_url=ctx.obj.current.api_base)
|
58
61
|
user.delete_compute_clusters([compute_cluster_id])
|
@@ -0,0 +1,112 @@
|
|
1
|
+
import click
|
2
|
+
|
3
|
+
from clarifai.cli.base import cli
|
4
|
+
from clarifai.utils.cli import AliasedGroup, display_co_resources, from_yaml, validate_context
|
5
|
+
|
6
|
+
|
7
|
+
@cli.group(['deployment', 'dp'], cls=AliasedGroup)
|
8
|
+
def deployment():
|
9
|
+
"""Manage Deployments: create, delete, list"""
|
10
|
+
|
11
|
+
|
12
|
+
@deployment.command(['c'])
|
13
|
+
@click.argument('nodepool_id')
|
14
|
+
@click.argument('deployment_id')
|
15
|
+
@click.option(
|
16
|
+
'--config',
|
17
|
+
type=click.Path(exists=True),
|
18
|
+
required=True,
|
19
|
+
help='Path to the deployment config file.')
|
20
|
+
@click.pass_context
|
21
|
+
def create(ctx, nodepool_id, deployment_id, config):
|
22
|
+
"""Create a new Deployment with the given config file."""
|
23
|
+
|
24
|
+
from clarifai.client.nodepool import Nodepool
|
25
|
+
validate_context(ctx)
|
26
|
+
if not nodepool_id:
|
27
|
+
deployment_config = from_yaml(config)
|
28
|
+
nodepool_id = deployment_config['deployment']['nodepools'][0]['id']
|
29
|
+
|
30
|
+
nodepool = Nodepool(
|
31
|
+
nodepool_id=nodepool_id,
|
32
|
+
user_id=ctx.obj.current.user_id,
|
33
|
+
pat=ctx.obj.current.pat,
|
34
|
+
base_url=ctx.obj.current.api_base)
|
35
|
+
if deployment_id:
|
36
|
+
nodepool.create_deployment(config, deployment_id=deployment_id)
|
37
|
+
else:
|
38
|
+
nodepool.create_deployment(config)
|
39
|
+
|
40
|
+
|
41
|
+
@deployment.command(['ls'])
|
42
|
+
@click.argument('nodepool_id', default="")
|
43
|
+
@click.option('--page_no', required=False, help='Page number to list.', default=1)
|
44
|
+
@click.option('--per_page', required=False, help='Number of items per page.', default=16)
|
45
|
+
@click.pass_context
|
46
|
+
def list(ctx, nodepool_id, page_no, per_page):
|
47
|
+
"""List all deployments for the nodepool."""
|
48
|
+
from clarifai.client.compute_cluster import ComputeCluster
|
49
|
+
from clarifai.client.nodepool import Nodepool
|
50
|
+
from clarifai.client.user import User
|
51
|
+
|
52
|
+
validate_context(ctx)
|
53
|
+
if nodepool_id:
|
54
|
+
nodepool = Nodepool(
|
55
|
+
nodepool_id=nodepool_id,
|
56
|
+
user_id=ctx.obj.current.user_id,
|
57
|
+
pat=ctx.obj.current.pat,
|
58
|
+
base_url=ctx.obj.current.api_base)
|
59
|
+
response = nodepool.list_deployments(page_no=page_no, per_page=per_page)
|
60
|
+
else:
|
61
|
+
user = User(
|
62
|
+
user_id=ctx.obj.current.user_id,
|
63
|
+
pat=ctx.obj.current.pat,
|
64
|
+
base_url=ctx.obj.current.api_base)
|
65
|
+
ccs = user.list_compute_clusters(page_no, per_page)
|
66
|
+
nps = []
|
67
|
+
for cc in ccs:
|
68
|
+
compute_cluster = ComputeCluster(
|
69
|
+
compute_cluster_id=cc.id,
|
70
|
+
user_id=ctx.obj.current.user_id,
|
71
|
+
pat=ctx.obj.current.pat,
|
72
|
+
base_url=ctx.obj.current.api_base)
|
73
|
+
nps.extend([i for i in compute_cluster.list_nodepools(page_no, per_page)])
|
74
|
+
response = []
|
75
|
+
for np in nps:
|
76
|
+
nodepool = Nodepool(
|
77
|
+
nodepool_id=np.id,
|
78
|
+
user_id=ctx.obj.current.user_id,
|
79
|
+
pat=ctx.obj.current.pat,
|
80
|
+
base_url=ctx.obj.current.api_base)
|
81
|
+
response.extend([i for i in nodepool.list_deployments(page_no=page_no, per_page=per_page)])
|
82
|
+
|
83
|
+
display_co_resources(
|
84
|
+
response,
|
85
|
+
custom_columns={
|
86
|
+
'ID': lambda c: c.id,
|
87
|
+
'USER_ID': lambda c: c.user_id,
|
88
|
+
'COMPUTE_CLUSTER_ID': lambda c: c.nodepools[0].compute_cluster.id,
|
89
|
+
'NODEPOOL_ID': lambda c: c.nodepools[0].id,
|
90
|
+
'MODEL_USER_ID': lambda c: c.worker.model.user_id,
|
91
|
+
'MODEL_APP_ID': lambda c: c.worker.model.app_id,
|
92
|
+
'MODEL_ID': lambda c: c.worker.model.id,
|
93
|
+
'MODEL_VERSION_ID': lambda c: c.worker.model.model_version.id,
|
94
|
+
'DESCRIPTION': lambda c: c.description,
|
95
|
+
})
|
96
|
+
|
97
|
+
|
98
|
+
@deployment.command(['rm'])
|
99
|
+
@click.argument('nodepool_id')
|
100
|
+
@click.argument('deployment_id')
|
101
|
+
@click.pass_context
|
102
|
+
def delete(ctx, nodepool_id, deployment_id):
|
103
|
+
"""Deletes a deployment for the nodepool."""
|
104
|
+
from clarifai.client.nodepool import Nodepool
|
105
|
+
|
106
|
+
validate_context(ctx)
|
107
|
+
nodepool = Nodepool(
|
108
|
+
nodepool_id=nodepool_id,
|
109
|
+
user_id=ctx.obj.current.user_id,
|
110
|
+
pat=ctx.obj.current.pat,
|
111
|
+
base_url=ctx.obj.current.api_base)
|
112
|
+
nodepool.delete_deployments([deployment_id])
|
@@ -0,0 +1,112 @@
|
|
1
|
+
import click
|
2
|
+
|
3
|
+
from clarifai.cli.base import cli
|
4
|
+
from clarifai.utils.cli import (AliasedGroup, display_co_resources, dump_yaml, from_yaml,
|
5
|
+
validate_context)
|
6
|
+
|
7
|
+
|
8
|
+
@cli.group(['nodepool', 'np'], cls=AliasedGroup)
|
9
|
+
def nodepool():
|
10
|
+
"""Manage Nodepools: create, delete, list"""
|
11
|
+
|
12
|
+
|
13
|
+
@nodepool.command(['c'])
|
14
|
+
@click.argument('compute_cluster_id')
|
15
|
+
@click.argument('nodepool_id')
|
16
|
+
@click.option(
|
17
|
+
'--config',
|
18
|
+
type=click.Path(exists=True),
|
19
|
+
required=True,
|
20
|
+
help='Path to the nodepool config file.')
|
21
|
+
@click.pass_context
|
22
|
+
def create(ctx, compute_cluster_id, nodepool_id, config):
|
23
|
+
"""Create a new Nodepool with the given config file."""
|
24
|
+
from clarifai.client.compute_cluster import ComputeCluster
|
25
|
+
|
26
|
+
validate_context(ctx)
|
27
|
+
nodepool_config = from_yaml(config)
|
28
|
+
if not compute_cluster_id:
|
29
|
+
if 'compute_cluster' not in nodepool_config['nodepool']:
|
30
|
+
click.echo(
|
31
|
+
"Please provide a compute cluster ID either in the config file or using --compute_cluster_id flag",
|
32
|
+
err=True)
|
33
|
+
return
|
34
|
+
compute_cluster_id = nodepool_config['nodepool']['compute_cluster']['id']
|
35
|
+
else:
|
36
|
+
if 'compute_cluster' not in nodepool_config['nodepool']:
|
37
|
+
nodepool_config['nodepool']['compute_cluster']['id'] = compute_cluster_id
|
38
|
+
dump_yaml(config, nodepool_config)
|
39
|
+
|
40
|
+
compute_cluster = ComputeCluster(
|
41
|
+
compute_cluster_id=compute_cluster_id,
|
42
|
+
user_id=ctx.obj.current.user_id,
|
43
|
+
pat=ctx.obj.current.pat,
|
44
|
+
base_url=ctx.obj.current.api_base)
|
45
|
+
if nodepool_id:
|
46
|
+
compute_cluster.create_nodepool(config, nodepool_id=nodepool_id)
|
47
|
+
else:
|
48
|
+
compute_cluster.create_nodepool(config)
|
49
|
+
|
50
|
+
|
51
|
+
@nodepool.command(['ls'])
|
52
|
+
@click.argument('compute_cluster_id', default="")
|
53
|
+
@click.option('--page_no', required=False, help='Page number to list.', default=1)
|
54
|
+
@click.option('--per_page', required=False, help='Number of items per page.', default=128)
|
55
|
+
@click.pass_context
|
56
|
+
def list(ctx, compute_cluster_id, page_no, per_page):
|
57
|
+
"""List all nodepools for the user across all compute clusters. If compute_cluster_id is provided
|
58
|
+
it will list only within that compute cluster. """
|
59
|
+
from clarifai.client.compute_cluster import ComputeCluster
|
60
|
+
from clarifai.client.user import User
|
61
|
+
|
62
|
+
validate_context(ctx)
|
63
|
+
|
64
|
+
cc_id = compute_cluster_id
|
65
|
+
|
66
|
+
if cc_id:
|
67
|
+
compute_cluster = ComputeCluster(
|
68
|
+
compute_cluster_id=cc_id,
|
69
|
+
user_id=ctx.obj.current.user_id,
|
70
|
+
pat=ctx.obj.current.pat,
|
71
|
+
base_url=ctx.obj.current.api_base)
|
72
|
+
response = compute_cluster.list_nodepools(page_no, per_page)
|
73
|
+
else:
|
74
|
+
user = User(
|
75
|
+
user_id=ctx.obj.current.user_id,
|
76
|
+
pat=ctx.obj.current.pat,
|
77
|
+
base_url=ctx.obj.current.api_base)
|
78
|
+
ccs = user.list_compute_clusters(page_no, per_page)
|
79
|
+
response = []
|
80
|
+
for cc in ccs:
|
81
|
+
compute_cluster = ComputeCluster(
|
82
|
+
compute_cluster_id=cc.id,
|
83
|
+
user_id=ctx.obj.current.user_id,
|
84
|
+
pat=ctx.obj.current.pat,
|
85
|
+
base_url=ctx.obj.current.api_base)
|
86
|
+
response.extend([i for i in compute_cluster.list_nodepools(page_no, per_page)])
|
87
|
+
|
88
|
+
display_co_resources(
|
89
|
+
response,
|
90
|
+
custom_columns={
|
91
|
+
'ID': lambda c: c.id,
|
92
|
+
'USER_ID': lambda c: c.compute_cluster.user_id,
|
93
|
+
'COMPUTE_CLUSTER_ID': lambda c: c.compute_cluster.id,
|
94
|
+
'DESCRIPTION': lambda c: c.description,
|
95
|
+
})
|
96
|
+
|
97
|
+
|
98
|
+
@nodepool.command(['rm'])
|
99
|
+
@click.argument('compute_cluster_id')
|
100
|
+
@click.argument('nodepool_id')
|
101
|
+
@click.pass_context
|
102
|
+
def delete(ctx, compute_cluster_id, nodepool_id):
|
103
|
+
"""Deletes a nodepool for the user."""
|
104
|
+
from clarifai.client.compute_cluster import ComputeCluster
|
105
|
+
|
106
|
+
validate_context(ctx)
|
107
|
+
compute_cluster = ComputeCluster(
|
108
|
+
compute_cluster_id=compute_cluster_id,
|
109
|
+
user_id=ctx.obj.current.user_id,
|
110
|
+
pat=ctx.obj.current.pat,
|
111
|
+
base_url=ctx.obj.current.api_base)
|
112
|
+
compute_cluster.delete_nodepools([nodepool_id])
|
@@ -629,7 +629,7 @@ class App(Lister, BaseClient):
|
|
629
629
|
|
630
630
|
Args:
|
631
631
|
model_id (str): The model ID for the model to interact with.
|
632
|
-
|
632
|
+
model_version (Dict): The model version ID for the model version to interact with.
|
633
633
|
|
634
634
|
Returns:
|
635
635
|
Model: A Model object for the existing model ID.
|
@@ -1,5 +1,4 @@
|
|
1
1
|
import itertools
|
2
|
-
import logging
|
3
2
|
import time
|
4
3
|
from concurrent.futures import ThreadPoolExecutor
|
5
4
|
|
@@ -8,7 +7,7 @@ from clarifai_grpc.grpc.api.status import status_code_pb2
|
|
8
7
|
|
9
8
|
from clarifai.client.auth.helper import ClarifaiAuthHelper
|
10
9
|
from clarifai.client.auth.register import RpcCallable, V2Stub
|
11
|
-
|
10
|
+
from clarifai.utils.logging import logger
|
12
11
|
throttle_status_codes = {
|
13
12
|
status_code_pb2.CONN_THROTTLED,
|
14
13
|
status_code_pb2.CONN_EXCEED_HOURLY_LIMIT,
|
@@ -26,7 +25,7 @@ def validate_response(response, attempt, max_attempts):
|
|
26
25
|
def handle_simple_response(response):
|
27
26
|
if hasattr(response, 'status') and hasattr(response.status, 'code'):
|
28
27
|
if (response.status.code in throttle_status_codes) and attempt < max_attempts:
|
29
|
-
|
28
|
+
logger.debug('Retrying with status %s' % str(response.status))
|
30
29
|
return None # Indicates a retry is needed
|
31
30
|
else:
|
32
31
|
return response
|
@@ -42,7 +41,7 @@ def validate_response(response, attempt, max_attempts):
|
|
42
41
|
return itertools.chain([validated_response], response)
|
43
42
|
return None # Indicates a retry is needed
|
44
43
|
except grpc.RpcError as e:
|
45
|
-
|
44
|
+
logger.error('Error processing streaming response: %s' % str(e))
|
46
45
|
return None # Indicates an error
|
47
46
|
else:
|
48
47
|
# Handle simple response validation
|
@@ -143,7 +142,7 @@ class _RetryRpcCallable(RpcCallable):
|
|
143
142
|
return v
|
144
143
|
except grpc.RpcError as e:
|
145
144
|
if (e.code() in retry_codes_grpc) and attempt < self.max_attempts:
|
146
|
-
|
145
|
+
logger.debug('Retrying with status %s' % e.code())
|
147
146
|
else:
|
148
147
|
raise
|
149
148
|
|
@@ -1,4 +1,3 @@
|
|
1
|
-
import logging
|
2
1
|
import os
|
3
2
|
import time
|
4
3
|
import uuid
|
@@ -354,7 +353,7 @@ class Dataset(Lister, BaseClient):
|
|
354
353
|
break
|
355
354
|
if failed_input_ids:
|
356
355
|
retry_input_ids = [dataset_obj.all_input_ids[id] for id in failed_input_ids]
|
357
|
-
|
356
|
+
logger.warning(
|
358
357
|
f"Retrying upload for {len(failed_input_ids)} inputs in current batch: {retry_input_ids}\n"
|
359
358
|
)
|
360
359
|
failed_retrying_inputs, _, retry_response = self._upload_inputs_annotations(
|
@@ -494,7 +493,7 @@ class Dataset(Lister, BaseClient):
|
|
494
493
|
add_file_handler(self.logger, f"Dataset_Upload{str(int(datetime.now().timestamp()))}.log")
|
495
494
|
|
496
495
|
if retry_duplicates and duplicate_input_ids:
|
497
|
-
|
496
|
+
logger.warning(f"Retrying upload for {len(duplicate_input_ids)} duplicate inputs...\n")
|
498
497
|
duplicate_inputs_indexes = [input["Index"] for input in duplicate_input_ids]
|
499
498
|
self.upload_dataset(
|
500
499
|
dataloader=dataloader,
|
@@ -505,7 +504,7 @@ class Dataset(Lister, BaseClient):
|
|
505
504
|
|
506
505
|
if failed_input_ids:
|
507
506
|
#failed_inputs= ([input["Input_ID"] for input in failed_input_ids])
|
508
|
-
|
507
|
+
logger.warning(f"Retrying upload for {len(failed_input_ids)} failed inputs...\n")
|
509
508
|
failed_input_indexes = [input["Index"] for input in failed_input_ids]
|
510
509
|
self.upload_dataset(
|
511
510
|
dataloader=dataloader, log_retry_ids=failed_input_indexes, is_log_retry=True, **kwargs)
|