clarifai 11.2.0__py3-none-any.whl → 11.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clarifai/__init__.py +1 -1
- clarifai/cli/base.py +11 -13
- clarifai/cli/compute_cluster.py +8 -1
- clarifai/cli/deployment.py +5 -1
- clarifai/cli/model.py +25 -13
- clarifai/cli/nodepool.py +4 -1
- clarifai/client/app.py +1 -1
- clarifai/runners/models/model_builder.py +21 -8
- clarifai/runners/models/model_run_locally.py +9 -5
- clarifai/runners/utils/loader.py +36 -6
- clarifai/utils/cli.py +9 -0
- {clarifai-11.2.0.dist-info → clarifai-11.2.2.dist-info}/METADATA +2 -2
- {clarifai-11.2.0.dist-info → clarifai-11.2.2.dist-info}/RECORD +17 -17
- {clarifai-11.2.0.dist-info → clarifai-11.2.2.dist-info}/WHEEL +1 -1
- {clarifai-11.2.0.dist-info → clarifai-11.2.2.dist-info}/entry_points.txt +0 -0
- {clarifai-11.2.0.dist-info → clarifai-11.2.2.dist-info}/licenses/LICENSE +0 -0
- {clarifai-11.2.0.dist-info → clarifai-11.2.2.dist-info}/top_level.txt +0 -0
clarifai/__init__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "11.2.
|
1
|
+
__version__ = "11.2.2"
|
clarifai/cli/base.py
CHANGED
@@ -43,17 +43,10 @@ def cli(ctx):
|
|
43
43
|
"""Clarifai CLI"""
|
44
44
|
ctx.ensure_object(dict)
|
45
45
|
config_path = f"{os.getenv('HOME')}/.clarifai/config.yaml"
|
46
|
-
config_dir = os.path.dirname(config_path)
|
47
46
|
if os.path.exists(config_path):
|
48
47
|
ctx.obj = from_yaml(config_path)
|
49
48
|
else:
|
50
49
|
ctx.obj = {}
|
51
|
-
if ctx.invoked_subcommand != 'login':
|
52
|
-
logger.error("Error getting config: CLI config file missing")
|
53
|
-
logger.info("Running `login` to set up the CLI configuration.")
|
54
|
-
if not os.path.exists(config_dir):
|
55
|
-
os.makedirs(config_dir)
|
56
|
-
ctx.invoke(login)
|
57
50
|
|
58
51
|
|
59
52
|
@cli.command()
|
@@ -74,16 +67,16 @@ def login(ctx, config, env, user_id):
|
|
74
67
|
|
75
68
|
if 'pat' in ctx.obj:
|
76
69
|
os.environ["CLARIFAI_PAT"] = ctx.obj['pat']
|
77
|
-
|
70
|
+
logger.info("Loaded PAT from config file.")
|
78
71
|
elif 'CLARIFAI_PAT' in os.environ:
|
79
72
|
ctx.obj['pat'] = os.environ["CLARIFAI_PAT"]
|
80
|
-
|
73
|
+
logger.info("Loaded PAT from environment variable.")
|
81
74
|
else:
|
82
75
|
_pat = click.prompt(
|
83
76
|
"Get your PAT from https://clarifai.com/settings/security and pass it here", type=str)
|
84
77
|
os.environ["CLARIFAI_PAT"] = _pat
|
85
78
|
ctx.obj['pat'] = _pat
|
86
|
-
|
79
|
+
logger.info("PAT saved successfully.")
|
87
80
|
|
88
81
|
if user_id:
|
89
82
|
ctx.obj['user_id'] = user_id
|
@@ -97,7 +90,7 @@ def login(ctx, config, env, user_id):
|
|
97
90
|
user_id = click.prompt("Pass the User ID here", type=str)
|
98
91
|
os.environ["CLARIFAI_USER_ID"] = user_id
|
99
92
|
ctx.obj['user_id'] = user_id
|
100
|
-
|
93
|
+
logger.info("User ID saved successfully.")
|
101
94
|
|
102
95
|
if env:
|
103
96
|
ctx.obj['env'] = env
|
@@ -113,9 +106,14 @@ def login(ctx, config, env, user_id):
|
|
113
106
|
ctx.obj['env'] = 'prod'
|
114
107
|
ctx.obj['base_url'] = set_base_url(ctx.obj['env'])
|
115
108
|
os.environ["CLARIFAI_API_BASE"] = ctx.obj['base_url']
|
116
|
-
|
109
|
+
logger.info("Base URL saved successfully.")
|
110
|
+
|
111
|
+
config_path = f"{os.getenv('HOME')}/.clarifai/config.yaml"
|
112
|
+
config_dir = os.path.dirname(config_path)
|
113
|
+
if not os.path.exists(config_dir):
|
114
|
+
os.makedirs(config_dir)
|
117
115
|
|
118
|
-
dump_yaml(ctx.obj,
|
116
|
+
dump_yaml(ctx.obj, config_path)
|
119
117
|
|
120
118
|
|
121
119
|
# Import the CLI commands to register them
|
clarifai/cli/compute_cluster.py
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
import click
|
2
|
+
|
2
3
|
from clarifai.cli.base import cli
|
3
4
|
from clarifai.client.user import User
|
4
|
-
from clarifai.utils.cli import display_co_resources
|
5
|
+
from clarifai.utils.cli import display_co_resources, validate_context
|
5
6
|
|
6
7
|
|
7
8
|
@cli.group(['computecluster', 'cc'])
|
@@ -24,6 +25,8 @@ def computecluster():
|
|
24
25
|
@click.pass_context
|
25
26
|
def create(ctx, config, compute_cluster_id):
|
26
27
|
"""Create a new Compute Cluster with the given config file."""
|
28
|
+
|
29
|
+
validate_context(ctx)
|
27
30
|
user = User(user_id=ctx.obj['user_id'], pat=ctx.obj['pat'], base_url=ctx.obj['base_url'])
|
28
31
|
if compute_cluster_id:
|
29
32
|
user.create_compute_cluster(config, compute_cluster_id=compute_cluster_id)
|
@@ -37,6 +40,8 @@ def create(ctx, config, compute_cluster_id):
|
|
37
40
|
@click.pass_context
|
38
41
|
def list(ctx, page_no, per_page):
|
39
42
|
"""List all compute clusters for the user."""
|
43
|
+
|
44
|
+
validate_context(ctx)
|
40
45
|
user = User(user_id=ctx.obj['user_id'], pat=ctx.obj['pat'], base_url=ctx.obj['base_url'])
|
41
46
|
response = user.list_compute_clusters(page_no, per_page)
|
42
47
|
display_co_resources(response, "Compute Cluster")
|
@@ -47,5 +52,7 @@ def list(ctx, page_no, per_page):
|
|
47
52
|
@click.pass_context
|
48
53
|
def delete(ctx, compute_cluster_id):
|
49
54
|
"""Deletes a compute cluster for the user."""
|
55
|
+
|
56
|
+
validate_context(ctx)
|
50
57
|
user = User(user_id=ctx.obj['user_id'], pat=ctx.obj['pat'], base_url=ctx.obj['base_url'])
|
51
58
|
user.delete_compute_clusters([compute_cluster_id])
|
clarifai/cli/deployment.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
import click
|
2
2
|
from clarifai.cli.base import cli
|
3
3
|
from clarifai.client.nodepool import Nodepool
|
4
|
-
from clarifai.utils.cli import display_co_resources, from_yaml
|
4
|
+
from clarifai.utils.cli import display_co_resources, from_yaml, validate_context
|
5
5
|
|
6
6
|
|
7
7
|
@cli.group(['deployment', 'dpl'])
|
@@ -29,6 +29,8 @@ def deployment():
|
|
29
29
|
@click.pass_context
|
30
30
|
def create(ctx, nodepool_id, config, deployment_id):
|
31
31
|
"""Create a new Deployment with the given config file."""
|
32
|
+
|
33
|
+
validate_context(ctx)
|
32
34
|
if not nodepool_id:
|
33
35
|
deployment_config = from_yaml(config)
|
34
36
|
nodepool_id = deployment_config['deployment']['nodepools'][0]['id']
|
@@ -56,6 +58,7 @@ def create(ctx, nodepool_id, config, deployment_id):
|
|
56
58
|
def list(ctx, nodepool_id, page_no, per_page):
|
57
59
|
"""List all deployments for the nodepool."""
|
58
60
|
|
61
|
+
validate_context(ctx)
|
59
62
|
nodepool = Nodepool(
|
60
63
|
nodepool_id=nodepool_id,
|
61
64
|
user_id=ctx.obj['user_id'],
|
@@ -76,6 +79,7 @@ def list(ctx, nodepool_id, page_no, per_page):
|
|
76
79
|
def delete(ctx, nodepool_id, deployment_id):
|
77
80
|
"""Deletes a deployment for the nodepool."""
|
78
81
|
|
82
|
+
validate_context(ctx)
|
79
83
|
nodepool = Nodepool(
|
80
84
|
nodepool_id=nodepool_id,
|
81
85
|
user_id=ctx.obj['user_id'],
|
clarifai/cli/model.py
CHANGED
@@ -96,11 +96,14 @@ def download_checkpoints(model_path, out_path, stage):
|
|
96
96
|
help=
|
97
97
|
'Keep the Docker image after testing the model locally (applicable for container mode). Defaults to False.'
|
98
98
|
)
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
99
|
+
@click.option(
|
100
|
+
'--skip_dockerfile',
|
101
|
+
is_flag=True,
|
102
|
+
help=
|
103
|
+
'Flag to skip generating a dockerfile so that you can manually edit an already created dockerfile. Apply for `--mode conatainer`.',
|
104
|
+
)
|
105
|
+
def test_locally(model_path, keep_env=False, keep_image=False, mode='env', skip_dockerfile=False):
|
106
|
+
"""Test model locally."""
|
104
107
|
try:
|
105
108
|
from clarifai.runners.models import model_run_locally
|
106
109
|
if mode == 'env' and keep_image:
|
@@ -114,7 +117,11 @@ def test_locally(model_path, keep_env=False, keep_image=False, mode='env'):
|
|
114
117
|
elif mode == "container":
|
115
118
|
click.echo("Testing model locally inside a container...")
|
116
119
|
model_run_locally.main(
|
117
|
-
model_path,
|
120
|
+
model_path,
|
121
|
+
inside_container=True,
|
122
|
+
run_model_server=False,
|
123
|
+
keep_image=keep_image,
|
124
|
+
skip_dockerfile=skip_dockerfile)
|
118
125
|
click.echo("Model tested successfully.")
|
119
126
|
except Exception as e:
|
120
127
|
click.echo(f"Failed to test model locally: {e}", err=True)
|
@@ -154,11 +161,14 @@ def test_locally(model_path, keep_env=False, keep_image=False, mode='env'):
|
|
154
161
|
help=
|
155
162
|
'Keep the Docker image after testing the model locally (applicable for container mode). Defaults to False.'
|
156
163
|
)
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
164
|
+
@click.option(
|
165
|
+
'--skip_dockerfile',
|
166
|
+
is_flag=True,
|
167
|
+
help=
|
168
|
+
'Flag to skip generating a dockerfile so that you can manually edit an already created dockerfile. Apply for `--mode conatainer`.',
|
169
|
+
)
|
170
|
+
def run_locally(model_path, port, mode, keep_env, keep_image, skip_dockerfile=False):
|
171
|
+
"""Run the model locally and start a gRPC server to serve the model."""
|
162
172
|
try:
|
163
173
|
from clarifai.runners.models import model_run_locally
|
164
174
|
if mode == 'env' and keep_image:
|
@@ -176,7 +186,8 @@ def run_locally(model_path, port, mode, keep_env, keep_image):
|
|
176
186
|
inside_container=True,
|
177
187
|
run_model_server=True,
|
178
188
|
port=port,
|
179
|
-
keep_image=keep_image
|
189
|
+
keep_image=keep_image,
|
190
|
+
skip_dockerfile=skip_dockerfile)
|
180
191
|
click.echo(f"Model server started locally from {model_path} in {mode} mode.")
|
181
192
|
except Exception as e:
|
182
193
|
click.echo(f"Failed to starts model server locally: {e}", err=True)
|
@@ -230,7 +241,8 @@ def predict(ctx, config, model_id, user_id, app_id, model_url, file_path, url, b
|
|
230
241
|
import json
|
231
242
|
|
232
243
|
from clarifai.client.model import Model
|
233
|
-
from clarifai.utils.cli import from_yaml
|
244
|
+
from clarifai.utils.cli import from_yaml, validate_context
|
245
|
+
validate_context(ctx)
|
234
246
|
if config:
|
235
247
|
config = from_yaml(config)
|
236
248
|
model_id, user_id, app_id, model_url, file_path, url, bytes, input_type, compute_cluster_id, nodepool_id, deployment_id, inference_params, output_config = (
|
clarifai/cli/nodepool.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
import click
|
2
2
|
from clarifai.cli.base import cli
|
3
3
|
from clarifai.client.compute_cluster import ComputeCluster
|
4
|
-
from clarifai.utils.cli import display_co_resources, dump_yaml, from_yaml
|
4
|
+
from clarifai.utils.cli import display_co_resources, dump_yaml, from_yaml, validate_context
|
5
5
|
|
6
6
|
|
7
7
|
@cli.group(['nodepool', 'np'])
|
@@ -27,6 +27,7 @@ def nodepool():
|
|
27
27
|
def create(ctx, compute_cluster_id, config, nodepool_id):
|
28
28
|
"""Create a new Nodepool with the given config file."""
|
29
29
|
|
30
|
+
validate_context(ctx)
|
30
31
|
nodepool_config = from_yaml(config)
|
31
32
|
if not compute_cluster_id:
|
32
33
|
if 'compute_cluster' not in nodepool_config['nodepool']:
|
@@ -63,6 +64,7 @@ def create(ctx, compute_cluster_id, config, nodepool_id):
|
|
63
64
|
def list(ctx, compute_cluster_id, page_no, per_page):
|
64
65
|
"""List all nodepools for the user."""
|
65
66
|
|
67
|
+
validate_context(ctx)
|
66
68
|
compute_cluster = ComputeCluster(
|
67
69
|
compute_cluster_id=compute_cluster_id,
|
68
70
|
user_id=ctx.obj['user_id'],
|
@@ -83,6 +85,7 @@ def list(ctx, compute_cluster_id, page_no, per_page):
|
|
83
85
|
def delete(ctx, compute_cluster_id, nodepool_id):
|
84
86
|
"""Deletes a nodepool for the user."""
|
85
87
|
|
88
|
+
validate_context(ctx)
|
86
89
|
compute_cluster = ComputeCluster(
|
87
90
|
compute_cluster_id=compute_cluster_id,
|
88
91
|
user_id=ctx.obj['user_id'],
|
clarifai/client/app.py
CHANGED
@@ -629,7 +629,7 @@ class App(Lister, BaseClient):
|
|
629
629
|
|
630
630
|
Args:
|
631
631
|
model_id (str): The model ID for the model to interact with.
|
632
|
-
|
632
|
+
model_version (Dict): The model version ID for the model version to interact with.
|
633
633
|
|
634
634
|
Returns:
|
635
635
|
Model: A Model object for the existing model ID.
|
@@ -171,7 +171,14 @@ class ModelBuilder:
|
|
171
171
|
|
172
172
|
# get from config.yaml otherwise fall back to HF_TOKEN env var.
|
173
173
|
hf_token = self.config.get("checkpoints").get("hf_token", os.environ.get("HF_TOKEN", None))
|
174
|
-
|
174
|
+
|
175
|
+
allowed_file_patterns = self.config.get("checkpoints").get('allowed_file_patterns', None)
|
176
|
+
if isinstance(allowed_file_patterns, str):
|
177
|
+
allowed_file_patterns = [allowed_file_patterns]
|
178
|
+
ignore_file_patterns = self.config.get("checkpoints").get('ignore_file_patterns', None)
|
179
|
+
if isinstance(ignore_file_patterns, str):
|
180
|
+
ignore_file_patterns = [ignore_file_patterns]
|
181
|
+
return loader_type, repo_id, hf_token, when, allowed_file_patterns, ignore_file_patterns
|
175
182
|
|
176
183
|
def _check_app_exists(self):
|
177
184
|
resp = self.client.STUB.GetApp(service_pb2.GetAppRequest(user_app_id=self.client.user_app_id))
|
@@ -216,7 +223,7 @@ class ModelBuilder:
|
|
216
223
|
assert model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE, f"Model type {model_type_id} not supported for concepts"
|
217
224
|
|
218
225
|
if self.config.get("checkpoints"):
|
219
|
-
loader_type, _, hf_token, _ = self._validate_config_checkpoints()
|
226
|
+
loader_type, _, hf_token, _, _, _ = self._validate_config_checkpoints()
|
220
227
|
|
221
228
|
if loader_type == "huggingface" and hf_token:
|
222
229
|
is_valid_token = HuggingFaceLoader.validate_hftoken(hf_token)
|
@@ -480,8 +487,10 @@ class ModelBuilder:
|
|
480
487
|
if not self.config.get("checkpoints"):
|
481
488
|
logger.info("No checkpoints specified in the config file")
|
482
489
|
return path
|
490
|
+
clarifai_model_type_id = self.config.get('model').get('model_type_id')
|
483
491
|
|
484
|
-
loader_type, repo_id, hf_token, when = self._validate_config_checkpoints(
|
492
|
+
loader_type, repo_id, hf_token, when, allowed_file_patterns, ignore_file_patterns = self._validate_config_checkpoints(
|
493
|
+
)
|
485
494
|
if stage not in ["build", "upload", "runtime"]:
|
486
495
|
raise Exception("Invalid stage provided, must be one of ['build', 'upload', 'runtime']")
|
487
496
|
if when != stage:
|
@@ -490,14 +499,18 @@ class ModelBuilder:
|
|
490
499
|
)
|
491
500
|
return path
|
492
501
|
|
493
|
-
success =
|
502
|
+
success = False
|
494
503
|
if loader_type == "huggingface":
|
495
|
-
loader = HuggingFaceLoader(
|
504
|
+
loader = HuggingFaceLoader(
|
505
|
+
repo_id=repo_id, token=hf_token, model_type_id=clarifai_model_type_id)
|
496
506
|
# for runtime default to /tmp path
|
497
507
|
if stage == "runtime" and checkpoint_path_override is None:
|
498
508
|
checkpoint_path_override = self.default_runtime_checkpoint_path()
|
499
509
|
path = checkpoint_path_override if checkpoint_path_override else self.checkpoint_path
|
500
|
-
success = loader.download_checkpoints(
|
510
|
+
success = loader.download_checkpoints(
|
511
|
+
path,
|
512
|
+
allowed_file_patterns=allowed_file_patterns,
|
513
|
+
ignore_file_patterns=ignore_file_patterns)
|
501
514
|
|
502
515
|
if loader_type:
|
503
516
|
if not success:
|
@@ -569,7 +582,7 @@ class ModelBuilder:
|
|
569
582
|
logger.debug(f"Will tar it into file: {file_path}")
|
570
583
|
|
571
584
|
model_type_id = self.config.get('model').get('model_type_id')
|
572
|
-
loader_type, repo_id, hf_token, when = self._validate_config_checkpoints()
|
585
|
+
loader_type, repo_id, hf_token, when, _, _ = self._validate_config_checkpoints()
|
573
586
|
|
574
587
|
if (model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE) and 'concepts' not in self.config:
|
575
588
|
logger.info(
|
@@ -617,7 +630,7 @@ class ModelBuilder:
|
|
617
630
|
# First check for the env variable, then try querying huggingface. If all else fails, use the default.
|
618
631
|
checkpoint_size = os.environ.get('CHECKPOINT_SIZE_BYTES', 0)
|
619
632
|
if not checkpoint_size:
|
620
|
-
_, repo_id, _, _ = self._validate_config_checkpoints()
|
633
|
+
_, repo_id, _, _, _, _ = self._validate_config_checkpoints()
|
621
634
|
checkpoint_size = HuggingFaceLoader.get_huggingface_checkpoint_total_size(repo_id)
|
622
635
|
if not checkpoint_size:
|
623
636
|
checkpoint_size = self.DEFAULT_CHECKPOINT_SIZE
|
@@ -473,21 +473,25 @@ def main(model_path,
|
|
473
473
|
inside_container=False,
|
474
474
|
port=8080,
|
475
475
|
keep_env=False,
|
476
|
-
keep_image=False
|
476
|
+
keep_image=False,
|
477
|
+
skip_dockerfile: bool = False):
|
477
478
|
|
478
479
|
manager = ModelRunLocally(model_path)
|
479
480
|
# get whatever stage is in config.yaml to force download now
|
480
481
|
# also always write to where upload/build wants to, not the /tmp folder that runtime stage uses
|
481
|
-
_, _, _, when = manager.builder._validate_config_checkpoints()
|
482
|
+
_, _, _, when, _, _ = manager.builder._validate_config_checkpoints()
|
482
483
|
manager.builder.download_checkpoints(
|
483
484
|
stage=when, checkpoint_path_override=manager.builder.checkpoint_path)
|
484
485
|
if inside_container:
|
485
486
|
if not manager.is_docker_installed():
|
486
487
|
sys.exit(1)
|
487
|
-
|
488
|
+
if not skip_dockerfile:
|
489
|
+
manager.builder.create_dockerfile()
|
488
490
|
image_tag = manager._docker_hash()
|
489
|
-
|
490
|
-
|
491
|
+
model_id = manager.config['model']['id'].lower()
|
492
|
+
# must be in lowercase
|
493
|
+
image_name = f"{model_id}:{image_tag}"
|
494
|
+
container_name = model_id
|
491
495
|
if not manager.docker_image_exists(image_name):
|
492
496
|
manager.build_docker_image(image_name=image_name)
|
493
497
|
try:
|
clarifai/runners/utils/loader.py
CHANGED
@@ -6,6 +6,7 @@ import shutil
|
|
6
6
|
|
7
7
|
import requests
|
8
8
|
|
9
|
+
from clarifai.runners.utils.const import CONCEPTS_REQUIRED_MODEL_TYPE
|
9
10
|
from clarifai.utils.logging import logger
|
10
11
|
|
11
12
|
|
@@ -13,9 +14,10 @@ class HuggingFaceLoader:
|
|
13
14
|
|
14
15
|
HF_DOWNLOAD_TEXT = "The 'huggingface_hub' package is not installed. Please install it using 'pip install huggingface_hub'."
|
15
16
|
|
16
|
-
def __init__(self, repo_id=None, token=None):
|
17
|
+
def __init__(self, repo_id=None, token=None, model_type_id=None):
|
17
18
|
self.repo_id = repo_id
|
18
19
|
self.token = token
|
20
|
+
self.clarifai_model_type_id = model_type_id
|
19
21
|
if token:
|
20
22
|
if self.validate_hftoken(token):
|
21
23
|
try:
|
@@ -43,13 +45,17 @@ class HuggingFaceLoader:
|
|
43
45
|
f"Error setting up Hugging Face token, please make sure you have the correct token: {e}")
|
44
46
|
return False
|
45
47
|
|
46
|
-
def download_checkpoints(self,
|
48
|
+
def download_checkpoints(self,
|
49
|
+
checkpoint_path: str,
|
50
|
+
allowed_file_patterns=None,
|
51
|
+
ignore_file_patterns=None):
|
47
52
|
# throw error if huggingface_hub wasn't installed
|
48
53
|
try:
|
49
54
|
from huggingface_hub import snapshot_download
|
50
55
|
except ImportError:
|
51
56
|
raise ImportError(self.HF_DOWNLOAD_TEXT)
|
52
|
-
if os.path.exists(checkpoint_path) and self.validate_download(
|
57
|
+
if os.path.exists(checkpoint_path) and self.validate_download(
|
58
|
+
checkpoint_path, allowed_file_patterns, ignore_file_patterns):
|
53
59
|
logger.info("Checkpoints already exist")
|
54
60
|
return True
|
55
61
|
else:
|
@@ -61,10 +67,16 @@ class HuggingFaceLoader:
|
|
61
67
|
return False
|
62
68
|
|
63
69
|
self.ignore_patterns = self._get_ignore_patterns()
|
70
|
+
if ignore_file_patterns:
|
71
|
+
if self.ignore_patterns:
|
72
|
+
self.ignore_patterns.extend(ignore_file_patterns)
|
73
|
+
else:
|
74
|
+
self.ignore_patterns = ignore_file_patterns
|
64
75
|
snapshot_download(
|
65
76
|
repo_id=self.repo_id,
|
66
77
|
local_dir=checkpoint_path,
|
67
78
|
local_dir_use_symlinks=False,
|
79
|
+
allow_patterns=allowed_file_patterns,
|
68
80
|
ignore_patterns=self.ignore_patterns)
|
69
81
|
# Remove the `.cache` folder if it exists
|
70
82
|
cache_path = os.path.join(checkpoint_path, ".cache")
|
@@ -75,7 +87,8 @@ class HuggingFaceLoader:
|
|
75
87
|
logger.error(f"Error downloading model checkpoints {e}")
|
76
88
|
return False
|
77
89
|
finally:
|
78
|
-
is_downloaded = self.validate_download(checkpoint_path
|
90
|
+
is_downloaded = self.validate_download(checkpoint_path, allowed_file_patterns,
|
91
|
+
ignore_file_patterns)
|
79
92
|
if not is_downloaded:
|
80
93
|
logger.error("Error validating downloaded model checkpoints")
|
81
94
|
return False
|
@@ -109,9 +122,13 @@ class HuggingFaceLoader:
|
|
109
122
|
from huggingface_hub import file_exists, repo_exists
|
110
123
|
except ImportError:
|
111
124
|
raise ImportError(self.HF_DOWNLOAD_TEXT)
|
112
|
-
|
125
|
+
if self.clarifai_model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE:
|
126
|
+
return repo_exists(self.repo_id) and file_exists(self.repo_id, 'config.json')
|
127
|
+
else:
|
128
|
+
return repo_exists(self.repo_id)
|
113
129
|
|
114
|
-
def validate_download(self, checkpoint_path: str
|
130
|
+
def validate_download(self, checkpoint_path: str, allowed_file_patterns: list,
|
131
|
+
ignore_file_patterns: list):
|
115
132
|
# check if model exists on HF
|
116
133
|
try:
|
117
134
|
from huggingface_hub import list_repo_files
|
@@ -120,7 +137,20 @@ class HuggingFaceLoader:
|
|
120
137
|
# Get the list of files on the repo
|
121
138
|
repo_files = list_repo_files(self.repo_id, token=self.token)
|
122
139
|
|
140
|
+
# Get the list of files on the repo that are allowed
|
141
|
+
if allowed_file_patterns:
|
142
|
+
|
143
|
+
def should_allow(file_path):
|
144
|
+
return any(fnmatch.fnmatch(file_path, pattern) for pattern in allowed_file_patterns)
|
145
|
+
|
146
|
+
repo_files = [f for f in repo_files if should_allow(f)]
|
147
|
+
|
123
148
|
self.ignore_patterns = self._get_ignore_patterns()
|
149
|
+
if ignore_file_patterns:
|
150
|
+
if self.ignore_patterns:
|
151
|
+
self.ignore_patterns.extend(ignore_file_patterns)
|
152
|
+
else:
|
153
|
+
self.ignore_patterns = ignore_file_patterns
|
124
154
|
# Get the list of files on the repo that are not ignored
|
125
155
|
if getattr(self, "ignore_patterns", None):
|
126
156
|
patterns = self.ignore_patterns
|
clarifai/utils/cli.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
import importlib
|
2
2
|
import os
|
3
3
|
import pkgutil
|
4
|
+
import sys
|
4
5
|
|
5
6
|
import click
|
6
7
|
import yaml
|
@@ -10,6 +11,8 @@ from rich.panel import Panel
|
|
10
11
|
from rich.style import Style
|
11
12
|
from rich.text import Text
|
12
13
|
|
14
|
+
from clarifai.utils.logging import logger
|
15
|
+
|
13
16
|
|
14
17
|
def from_yaml(filename: str):
|
15
18
|
try:
|
@@ -68,3 +71,9 @@ def display_co_resources(response, resource_type):
|
|
68
71
|
border_style="green",
|
69
72
|
width=60)
|
70
73
|
console.print(panel)
|
74
|
+
|
75
|
+
|
76
|
+
def validate_context(ctx):
|
77
|
+
if ctx.obj == {}:
|
78
|
+
logger.error("CLI config file missing. Run `clarifai login` to set up the CLI config.")
|
79
|
+
sys.exit(1)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: clarifai
|
3
|
-
Version: 11.2.
|
3
|
+
Version: 11.2.2
|
4
4
|
Summary: Clarifai Python SDK
|
5
5
|
Home-page: https://github.com/Clarifai/clarifai-python
|
6
6
|
Author: Clarifai
|
@@ -20,7 +20,7 @@ Classifier: Operating System :: OS Independent
|
|
20
20
|
Requires-Python: >=3.8
|
21
21
|
Description-Content-Type: text/markdown
|
22
22
|
License-File: LICENSE
|
23
|
-
Requires-Dist: clarifai-grpc>=11.2.
|
23
|
+
Requires-Dist: clarifai-grpc>=11.2.6
|
24
24
|
Requires-Dist: clarifai-protocol>=0.0.20
|
25
25
|
Requires-Dist: numpy>=1.22.0
|
26
26
|
Requires-Dist: tqdm>=4.65.0
|
@@ -1,17 +1,17 @@
|
|
1
|
-
clarifai/__init__.py,sha256=
|
1
|
+
clarifai/__init__.py,sha256=_cDM_DxpY5KiS6333RgrXKdnvdwjkQID_vppBsYvGPM,23
|
2
2
|
clarifai/cli.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
3
3
|
clarifai/errors.py,sha256=RwzTajwds51wLD0MVlMC5kcpBnzRpreDLlazPSBZxrg,2605
|
4
4
|
clarifai/versions.py,sha256=jctnczzfGk_S3EnVqb2FjRKfSREkNmvNEwAAa_VoKiQ,222
|
5
5
|
clarifai/cli/README.md,sha256=YGApHfeUyu5P0Pdth-mqQCQftWHDxz6bugDlvDXDhOE,1942
|
6
6
|
clarifai/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
7
7
|
clarifai/cli/__main__.py,sha256=utJ2F40cl0jPHcYdTlGZRqpPfZ0CtVYB-8Ft0b2fWD4,72
|
8
|
-
clarifai/cli/base.py,sha256=
|
9
|
-
clarifai/cli/compute_cluster.py,sha256=
|
10
|
-
clarifai/cli/deployment.py,sha256=
|
11
|
-
clarifai/cli/model.py,sha256=
|
12
|
-
clarifai/cli/nodepool.py,sha256=
|
8
|
+
clarifai/cli/base.py,sha256=SOAic4T6Hg7VJJ-UDdjTuTNI1Hu_YANmzW_giUjK24o,3853
|
9
|
+
clarifai/cli/compute_cluster.py,sha256=v77YUbZjmXbTq0wCmcUJ8TzYSfFEtB1na87-iZuG21Q,1921
|
10
|
+
clarifai/cli/deployment.py,sha256=Vv92Qo7Q3UmMiCdrGQ4qNVNVzfcDyl-GB6B7kx8gyFw,2597
|
11
|
+
clarifai/cli/model.py,sha256=PR-A7V3D78iAjNx2hNeputiWuIyINrx2SJmjKtfMoQ4,12079
|
12
|
+
clarifai/cli/nodepool.py,sha256=eYo3l9NMSwjeFzsuwKSYVkR1FdZ7_NcCdNvzGsBBMzY,3139
|
13
13
|
clarifai/client/__init__.py,sha256=xI1U0l5AZdRThvQAXCLsd9axxyFzXXJ22m8LHqVjQRU,662
|
14
|
-
clarifai/client/app.py,sha256=
|
14
|
+
clarifai/client/app.py,sha256=FnKvKksYZwdry0e4Obh-trdSO1mv6QcVAa3kzKiQMpU,38340
|
15
15
|
clarifai/client/base.py,sha256=hSHOqkXbSKyaRDeylMMnkhUHCAHhEqno4KI0CXGziBA,7536
|
16
16
|
clarifai/client/compute_cluster.py,sha256=EvW9TJjPvInUlggfg1A98sxoWH8_PY5rCVXZhsj6ac0,8705
|
17
17
|
clarifai/client/dataset.py,sha256=y3zKT_VhP1gyN3OO-b3cPeW21ZXyKbQ7ZJkEG06bsTU,32096
|
@@ -66,21 +66,21 @@ clarifai/runners/server.py,sha256=xHDLdhQApCgYG19QOKXqJNCGNyw1Vsvobq3UdryDrVc,41
|
|
66
66
|
clarifai/runners/dockerfile_template/Dockerfile.template,sha256=5cjv7U8PmWa3DB_5B1CqSYh_6GE0E0np52TIAa7EIDE,2312
|
67
67
|
clarifai/runners/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
68
68
|
clarifai/runners/models/base_typed_model.py,sha256=0QCWxch8CcyJSKvE1D4PILd2RSnQZHTmx4DXlQQ6dpo,7856
|
69
|
-
clarifai/runners/models/model_builder.py,sha256=
|
69
|
+
clarifai/runners/models/model_builder.py,sha256=NAZiOm2ok-ThQ-jcAyBTRRCFufmyxGY76vlkLMpYHq4,33320
|
70
70
|
clarifai/runners/models/model_class.py,sha256=9JSPAr4U4K7xI0kSl-q0mHB06zknm2OR-8XIgBCto94,1611
|
71
|
-
clarifai/runners/models/model_run_locally.py,sha256=
|
71
|
+
clarifai/runners/models/model_run_locally.py,sha256=jz4ZvRj0eMCEziV3X37wH2AEe3Gl9Icx7Quy1ymaDe0,20556
|
72
72
|
clarifai/runners/models/model_runner.py,sha256=PyxwK-33hLlhkD07tTXkjWZ_iNlZHl9_8AZ2W7WfExI,6097
|
73
73
|
clarifai/runners/models/model_servicer.py,sha256=jtQmtGeQlvQ5ttMvVw7CMnNzq-rLkTaxR2IWF9SnHwk,2808
|
74
74
|
clarifai/runners/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
75
75
|
clarifai/runners/utils/const.py,sha256=bwj-Pcw558-pasdIFbNhnkn-9oiCdojYH1fNTTUG2gU,1048
|
76
76
|
clarifai/runners/utils/data_handler.py,sha256=sxy9zlAgI6ETuxCQhUgEXAn2GCsaW1GxpK6GTaMne0g,6966
|
77
77
|
clarifai/runners/utils/data_utils.py,sha256=R1iQ82TuQ9JwxCJk8yEB1Lyb0BYVhVbWJI9YDi1zGOs,318
|
78
|
-
clarifai/runners/utils/loader.py,sha256=
|
78
|
+
clarifai/runners/utils/loader.py,sha256=Sl0m29RDtMPx2cIiSbbDFtKHQj2ktXQ5CnkvaHi-zDc,8804
|
79
79
|
clarifai/runners/utils/url_fetcher.py,sha256=v_8JOWmkyFAzsBulsieKX7Nfjy1Yg7wGSZeqfEvw2cg,1640
|
80
80
|
clarifai/schema/search.py,sha256=JjTi8ammJgZZ2OGl4K6tIA4zEJ1Fr2ASZARXavI1j5c,2448
|
81
81
|
clarifai/urls/helper.py,sha256=tjoMGGHuWX68DUB0pk4MEjrmFsClUAQj2jmVEM_Sy78,4751
|
82
82
|
clarifai/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
83
|
-
clarifai/utils/cli.py,sha256=
|
83
|
+
clarifai/utils/cli.py,sha256=P29JalmKQLQBWqbW2_bumCLNSUUo8APtLXD8zS0dcIk,2160
|
84
84
|
clarifai/utils/constants.py,sha256=MG_iHnSwNEyUZOpvsrTicNwaT4CIjmlK_Ixk_qqEX8g,142
|
85
85
|
clarifai/utils/logging.py,sha256=CVy8OsLrlbg-b8qe88kb1yO_9wi9wRYfF-QkIaN9xE8,11936
|
86
86
|
clarifai/utils/misc.py,sha256=4aNOHPTNdX-WGxuNab8qKePoCMUnscd-aN_dQEIVSWk,2933
|
@@ -93,9 +93,9 @@ clarifai/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
|
|
93
93
|
clarifai/workflows/export.py,sha256=vICRhIreqDSShxLKjHNM2JwzKsf1B4fdXB0ciMcA70k,1945
|
94
94
|
clarifai/workflows/utils.py,sha256=nGeB_yjVgUO9kOeKTg4OBBaBz-AwXI3m-huSVj-9W18,1924
|
95
95
|
clarifai/workflows/validate.py,sha256=yJq03MaJqi5AK3alKGJJBR89xmmjAQ31sVufJUiOqY8,2556
|
96
|
-
clarifai-11.2.
|
97
|
-
clarifai-11.2.
|
98
|
-
clarifai-11.2.
|
99
|
-
clarifai-11.2.
|
100
|
-
clarifai-11.2.
|
101
|
-
clarifai-11.2.
|
96
|
+
clarifai-11.2.2.dist-info/licenses/LICENSE,sha256=mUqF_d12-qE2n41g7C5_sq-BMLOcj6CNN-jevr15YHU,555
|
97
|
+
clarifai-11.2.2.dist-info/METADATA,sha256=mWqPwNuC1LdGkPWUu5ugnqADyHzcwR18PA66roRCPj4,22472
|
98
|
+
clarifai-11.2.2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
99
|
+
clarifai-11.2.2.dist-info/entry_points.txt,sha256=X9FZ4Z-i_r2Ud1RpZ9sNIFYuu_-9fogzCMCRUD9hyX0,51
|
100
|
+
clarifai-11.2.2.dist-info/top_level.txt,sha256=wUMdCQGjkxaynZ6nZ9FAnvBUCgp5RJUVFSy2j-KYo0s,9
|
101
|
+
clarifai-11.2.2.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|