clarifai 9.8.1__py3-none-any.whl → 9.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clarifai/client/app.py +115 -14
- clarifai/client/base.py +11 -4
- clarifai/client/dataset.py +8 -3
- clarifai/client/input.py +34 -28
- clarifai/client/model.py +71 -2
- clarifai/client/module.py +4 -2
- clarifai/client/runner.py +161 -0
- clarifai/client/search.py +173 -0
- clarifai/client/user.py +110 -4
- clarifai/client/workflow.py +27 -2
- clarifai/constants/search.py +2 -0
- clarifai/datasets/upload/loaders/xview_detection.py +1 -1
- clarifai/models/model_serving/README.md +3 -3
- clarifai/models/model_serving/cli/deploy_cli.py +2 -3
- clarifai/models/model_serving/cli/repository.py +3 -5
- clarifai/models/model_serving/constants.py +1 -5
- clarifai/models/model_serving/docs/custom_config.md +5 -6
- clarifai/models/model_serving/docs/dependencies.md +5 -10
- clarifai/models/model_serving/examples/image_classification/age_vit/requirements.txt +1 -0
- clarifai/models/model_serving/examples/text_classification/xlm-roberta/requirements.txt +1 -0
- clarifai/models/model_serving/examples/text_to_image/sd-v1.5/requirements.txt +1 -0
- clarifai/models/model_serving/examples/text_to_text/bart-summarize/requirements.txt +1 -0
- clarifai/models/model_serving/examples/visual_detection/yolov5x/requirements.txt +1 -1
- clarifai/models/model_serving/examples/visual_embedding/vit-base/requirements.txt +1 -0
- clarifai/models/model_serving/examples/visual_segmentation/segformer-b2/requirements.txt +1 -0
- clarifai/models/model_serving/model_config/__init__.py +2 -0
- clarifai/models/model_serving/model_config/config.py +298 -0
- clarifai/models/model_serving/model_config/model_types_config/text-classifier.yaml +18 -0
- clarifai/models/model_serving/model_config/model_types_config/text-embedder.yaml +18 -0
- clarifai/models/model_serving/model_config/model_types_config/text-to-image.yaml +18 -0
- clarifai/models/model_serving/model_config/model_types_config/text-to-text.yaml +18 -0
- clarifai/models/model_serving/model_config/model_types_config/visual-classifier.yaml +18 -0
- clarifai/models/model_serving/model_config/model_types_config/visual-detector.yaml +28 -0
- clarifai/models/model_serving/model_config/model_types_config/visual-embedder.yaml +18 -0
- clarifai/models/model_serving/model_config/model_types_config/visual-segmenter.yaml +18 -0
- clarifai/models/model_serving/model_config/serializer.py +1 -1
- clarifai/models/model_serving/models/default_test.py +22 -21
- clarifai/models/model_serving/models/output.py +2 -2
- clarifai/models/model_serving/pb_model_repository.py +2 -5
- clarifai/runners/__init__.py +0 -0
- clarifai/runners/example.py +33 -0
- clarifai/schema/search.py +60 -0
- clarifai/utils/logging.py +53 -3
- clarifai/versions.py +1 -1
- clarifai/workflows/__init__.py +0 -0
- clarifai/workflows/export.py +68 -0
- clarifai/workflows/utils.py +59 -0
- clarifai/workflows/validate.py +67 -0
- {clarifai-9.8.1.dist-info → clarifai-9.9.0.dist-info}/METADATA +20 -2
- {clarifai-9.8.1.dist-info → clarifai-9.9.0.dist-info}/RECORD +102 -86
- clarifai_utils/client/app.py +115 -14
- clarifai_utils/client/base.py +11 -4
- clarifai_utils/client/dataset.py +8 -3
- clarifai_utils/client/input.py +34 -28
- clarifai_utils/client/model.py +71 -2
- clarifai_utils/client/module.py +4 -2
- clarifai_utils/client/runner.py +161 -0
- clarifai_utils/client/search.py +173 -0
- clarifai_utils/client/user.py +110 -4
- clarifai_utils/client/workflow.py +27 -2
- clarifai_utils/constants/search.py +2 -0
- clarifai_utils/datasets/upload/loaders/xview_detection.py +1 -1
- clarifai_utils/models/model_serving/README.md +3 -3
- clarifai_utils/models/model_serving/cli/deploy_cli.py +2 -3
- clarifai_utils/models/model_serving/cli/repository.py +3 -5
- clarifai_utils/models/model_serving/constants.py +1 -5
- clarifai_utils/models/model_serving/docs/custom_config.md +5 -6
- clarifai_utils/models/model_serving/docs/dependencies.md +5 -10
- clarifai_utils/models/model_serving/examples/image_classification/age_vit/requirements.txt +1 -0
- clarifai_utils/models/model_serving/examples/text_classification/xlm-roberta/requirements.txt +1 -0
- clarifai_utils/models/model_serving/examples/text_to_image/sd-v1.5/requirements.txt +1 -0
- clarifai_utils/models/model_serving/examples/text_to_text/bart-summarize/requirements.txt +1 -0
- clarifai_utils/models/model_serving/examples/visual_detection/yolov5x/requirements.txt +1 -1
- clarifai_utils/models/model_serving/examples/visual_embedding/vit-base/requirements.txt +1 -0
- clarifai_utils/models/model_serving/examples/visual_segmentation/segformer-b2/requirements.txt +1 -0
- clarifai_utils/models/model_serving/model_config/__init__.py +2 -0
- clarifai_utils/models/model_serving/model_config/config.py +298 -0
- clarifai_utils/models/model_serving/model_config/model_types_config/text-classifier.yaml +18 -0
- clarifai_utils/models/model_serving/model_config/model_types_config/text-embedder.yaml +18 -0
- clarifai_utils/models/model_serving/model_config/model_types_config/text-to-image.yaml +18 -0
- clarifai_utils/models/model_serving/model_config/model_types_config/text-to-text.yaml +18 -0
- clarifai_utils/models/model_serving/model_config/model_types_config/visual-classifier.yaml +18 -0
- clarifai_utils/models/model_serving/model_config/model_types_config/visual-detector.yaml +28 -0
- clarifai_utils/models/model_serving/model_config/model_types_config/visual-embedder.yaml +18 -0
- clarifai_utils/models/model_serving/model_config/model_types_config/visual-segmenter.yaml +18 -0
- clarifai_utils/models/model_serving/model_config/serializer.py +1 -1
- clarifai_utils/models/model_serving/models/default_test.py +22 -21
- clarifai_utils/models/model_serving/models/output.py +2 -2
- clarifai_utils/models/model_serving/pb_model_repository.py +2 -5
- clarifai_utils/runners/__init__.py +0 -0
- clarifai_utils/runners/example.py +33 -0
- clarifai_utils/schema/search.py +60 -0
- clarifai_utils/utils/logging.py +53 -3
- clarifai_utils/versions.py +1 -1
- clarifai_utils/workflows/__init__.py +0 -0
- clarifai_utils/workflows/export.py +68 -0
- clarifai_utils/workflows/utils.py +59 -0
- clarifai_utils/workflows/validate.py +67 -0
- clarifai/models/model_serving/envs/triton_conda-cp3.8-torch1.13.1-19f97078.yaml +0 -35
- clarifai/models/model_serving/envs/triton_conda-cp3.8-torch2.0.0-ce980f28.yaml +0 -51
- clarifai/models/model_serving/examples/image_classification/age_vit/triton_conda.yaml +0 -1
- clarifai/models/model_serving/examples/text_classification/xlm-roberta/triton_conda.yaml +0 -1
- clarifai/models/model_serving/examples/text_to_image/sd-v1.5/triton_conda.yaml +0 -1
- clarifai/models/model_serving/examples/text_to_text/bart-summarize/triton_conda.yaml +0 -1
- clarifai/models/model_serving/examples/visual_detection/yolov5x/triton_conda.yaml +0 -1
- clarifai/models/model_serving/examples/visual_embedding/vit-base/triton_conda.yaml +0 -1
- clarifai/models/model_serving/examples/visual_segmentation/segformer-b2/triton_conda.yaml +0 -1
- clarifai/models/model_serving/model_config/deploy.py +0 -75
- clarifai/models/model_serving/model_config/triton_config.py +0 -226
- clarifai_utils/models/model_serving/envs/triton_conda-cp3.8-torch1.13.1-19f97078.yaml +0 -35
- clarifai_utils/models/model_serving/envs/triton_conda-cp3.8-torch2.0.0-ce980f28.yaml +0 -51
- clarifai_utils/models/model_serving/examples/image_classification/age_vit/triton_conda.yaml +0 -1
- clarifai_utils/models/model_serving/examples/text_classification/xlm-roberta/triton_conda.yaml +0 -1
- clarifai_utils/models/model_serving/examples/text_to_image/sd-v1.5/triton_conda.yaml +0 -1
- clarifai_utils/models/model_serving/examples/text_to_text/bart-summarize/triton_conda.yaml +0 -1
- clarifai_utils/models/model_serving/examples/visual_detection/yolov5x/triton_conda.yaml +0 -1
- clarifai_utils/models/model_serving/examples/visual_embedding/vit-base/triton_conda.yaml +0 -1
- clarifai_utils/models/model_serving/examples/visual_segmentation/segformer-b2/triton_conda.yaml +0 -1
- clarifai_utils/models/model_serving/model_config/deploy.py +0 -75
- clarifai_utils/models/model_serving/model_config/triton_config.py +0 -226
- {clarifai-9.8.1.dist-info → clarifai-9.9.0.dist-info}/LICENSE +0 -0
- {clarifai-9.8.1.dist-info → clarifai-9.9.0.dist-info}/WHEEL +0 -0
- {clarifai-9.8.1.dist-info → clarifai-9.9.0.dist-info}/entry_points.txt +0 -0
- {clarifai-9.8.1.dist-info → clarifai-9.9.0.dist-info}/top_level.txt +0 -0
clarifai/client/app.py
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import uuid
|
|
1
3
|
from typing import Any, Dict, List
|
|
2
4
|
|
|
5
|
+
import yaml
|
|
3
6
|
from clarifai_grpc.grpc.api import resources_pb2, service_pb2
|
|
4
7
|
from clarifai_grpc.grpc.api.status import status_code_pb2
|
|
5
8
|
from google.protobuf.json_format import MessageToDict
|
|
@@ -10,22 +13,30 @@ from clarifai.client.input import Inputs
|
|
|
10
13
|
from clarifai.client.lister import Lister
|
|
11
14
|
from clarifai.client.model import Model
|
|
12
15
|
from clarifai.client.module import Module
|
|
16
|
+
from clarifai.client.search import Search
|
|
13
17
|
from clarifai.client.workflow import Workflow
|
|
14
18
|
from clarifai.errors import UserError
|
|
15
19
|
from clarifai.urls.helper import ClarifaiUrlHelper
|
|
16
|
-
from clarifai.utils.logging import get_logger
|
|
20
|
+
from clarifai.utils.logging import display_workflow_tree, get_logger
|
|
21
|
+
from clarifai.workflows.utils import get_yaml_output_info_proto, is_same_yaml_model
|
|
22
|
+
from clarifai.workflows.validate import validate
|
|
17
23
|
|
|
18
24
|
|
|
19
25
|
class App(Lister, BaseClient):
|
|
20
26
|
"""App is a class that provides access to Clarifai API endpoints related to App information."""
|
|
21
27
|
|
|
22
|
-
def __init__(self,
|
|
28
|
+
def __init__(self,
|
|
29
|
+
url_init: str = "",
|
|
30
|
+
app_id: str = "",
|
|
31
|
+
base_url: str = "https://api.clarifai.com",
|
|
32
|
+
**kwargs):
|
|
23
33
|
"""Initializes an App object.
|
|
24
34
|
|
|
25
35
|
Args:
|
|
26
36
|
url_init (str): The URL to initialize the app object.
|
|
27
37
|
app_id (str): The App ID for the App to interact with.
|
|
28
|
-
|
|
38
|
+
base_url (str): Base API url. Default "https://api.clarifai.com"
|
|
39
|
+
**kwargs: Additional keyword arguments to be passed to the App.
|
|
29
40
|
- name (str): The name of the app.
|
|
30
41
|
- description (str): The description of the app.
|
|
31
42
|
"""
|
|
@@ -37,7 +48,7 @@ class App(Lister, BaseClient):
|
|
|
37
48
|
self.kwargs = {**kwargs, 'id': app_id}
|
|
38
49
|
self.app_info = resources_pb2.App(**self.kwargs)
|
|
39
50
|
self.logger = get_logger(logger_level="INFO", name=__name__)
|
|
40
|
-
BaseClient.__init__(self, user_id=self.user_id, app_id=self.id)
|
|
51
|
+
BaseClient.__init__(self, user_id=self.user_id, app_id=self.id, base=base_url)
|
|
41
52
|
Lister.__init__(self)
|
|
42
53
|
|
|
43
54
|
def list_datasets(self) -> List[Dataset]:
|
|
@@ -236,30 +247,96 @@ class App(Lister, BaseClient):
|
|
|
236
247
|
|
|
237
248
|
return Model(model_id=model_id, **kwargs)
|
|
238
249
|
|
|
239
|
-
def create_workflow(self,
|
|
250
|
+
def create_workflow(self,
|
|
251
|
+
config_filepath: str,
|
|
252
|
+
generate_new_id: bool = False,
|
|
253
|
+
display: bool = True) -> Workflow:
|
|
240
254
|
"""Creates a workflow for the app.
|
|
241
255
|
|
|
242
256
|
Args:
|
|
243
|
-
|
|
244
|
-
|
|
257
|
+
config_filepath (str): The path to the yaml workflow config file.
|
|
258
|
+
generate_new_id (bool): If True, generate a new workflow ID.
|
|
259
|
+
display (bool): If True, display the workflow nodes tree.
|
|
245
260
|
|
|
246
261
|
Returns:
|
|
247
|
-
Workflow: A Workflow object for the specified workflow
|
|
262
|
+
Workflow: A Workflow object for the specified workflow config.
|
|
248
263
|
|
|
249
264
|
Example:
|
|
250
265
|
>>> from clarifai.client.app import App
|
|
251
266
|
>>> app = App(app_id="app_id", user_id="user_id")
|
|
252
|
-
>>> workflow = app.create_workflow(
|
|
267
|
+
>>> workflow = app.create_workflow(config_filepath="config.yml")
|
|
253
268
|
"""
|
|
269
|
+
if not os.path.exists(config_filepath):
|
|
270
|
+
raise UserError(f"Workflow config file not found at {config_filepath}")
|
|
271
|
+
|
|
272
|
+
with open(config_filepath, 'r') as file:
|
|
273
|
+
data = yaml.safe_load(file)
|
|
274
|
+
|
|
275
|
+
data = validate(data)
|
|
276
|
+
workflow = data['workflow']
|
|
277
|
+
|
|
278
|
+
# Get all model objects from the workflow nodes.
|
|
279
|
+
all_models = []
|
|
280
|
+
for node in workflow['nodes']:
|
|
281
|
+
output_info = get_yaml_output_info_proto(node['model'].get('output_info', None))
|
|
282
|
+
try:
|
|
283
|
+
model = self.model(
|
|
284
|
+
node['model']['model_id'],
|
|
285
|
+
node['model'].get('model_version_id', ""),
|
|
286
|
+
user_id=node['model'].get('user_id', ""),
|
|
287
|
+
app_id=node['model'].get('app_id', ""))
|
|
288
|
+
except Exception as e:
|
|
289
|
+
if "Model does not exist" in str(e):
|
|
290
|
+
model = self.create_model(
|
|
291
|
+
**{k: v
|
|
292
|
+
for k, v in node['model'].items() if k != 'output_info'})
|
|
293
|
+
model_version = model.create_model_version(output_info=output_info)
|
|
294
|
+
all_models.append(model_version.model_info)
|
|
295
|
+
continue
|
|
296
|
+
|
|
297
|
+
# If the model version ID is specified, or if the yaml model is the same as the one in the api
|
|
298
|
+
if node["model"].get("model_version_id", "") or is_same_yaml_model(
|
|
299
|
+
model.model_info, node["model"]):
|
|
300
|
+
all_models.append(model.model_info)
|
|
301
|
+
else: # Create a new model version
|
|
302
|
+
model = model.create_model_version(output_info=output_info)
|
|
303
|
+
all_models.append(model.model_info)
|
|
304
|
+
|
|
305
|
+
# Convert nodes to resources_pb2.WorkflowNodes.
|
|
306
|
+
nodes = []
|
|
307
|
+
for i, yml_node in enumerate(workflow['nodes']):
|
|
308
|
+
node = resources_pb2.WorkflowNode(
|
|
309
|
+
id=yml_node['id'],
|
|
310
|
+
model=all_models[i],
|
|
311
|
+
)
|
|
312
|
+
# Add node inputs if they exist, i.e. if these nodes do not connect directly to the input.
|
|
313
|
+
if yml_node.get("node_inputs"):
|
|
314
|
+
for ni in yml_node.get("node_inputs"):
|
|
315
|
+
node.node_inputs.append(resources_pb2.NodeInput(node_id=ni['node_id']))
|
|
316
|
+
nodes.append(node)
|
|
317
|
+
|
|
318
|
+
workflow_id = workflow['id']
|
|
319
|
+
if generate_new_id:
|
|
320
|
+
workflow_id = str(uuid.uuid4())
|
|
321
|
+
|
|
322
|
+
# Create the workflow.
|
|
254
323
|
request = service_pb2.PostWorkflowsRequest(
|
|
255
|
-
user_app_id=self.user_app_id,
|
|
324
|
+
user_app_id=self.user_app_id,
|
|
325
|
+
workflows=[resources_pb2.Workflow(id=workflow_id, nodes=nodes)])
|
|
326
|
+
|
|
256
327
|
response = self._grpc_request(self.STUB.PostWorkflows, request)
|
|
257
328
|
if response.status.code != status_code_pb2.SUCCESS:
|
|
258
329
|
raise Exception(response.status)
|
|
259
330
|
self.logger.info("\nWorkflow created\n%s", response.status)
|
|
260
|
-
kwargs.update({'app_id': self.id, 'user_id': self.user_id})
|
|
261
331
|
|
|
262
|
-
|
|
332
|
+
dict_response = MessageToDict(response, preserving_proto_field_name=True)
|
|
333
|
+
# Display the workflow nodes tree.
|
|
334
|
+
if display:
|
|
335
|
+
display_workflow_tree(dict_response["workflows"][0]["nodes"])
|
|
336
|
+
kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]][0],
|
|
337
|
+
"workflow")
|
|
338
|
+
|
|
339
|
+
return Workflow(**kwargs)
|
|
263
340
|
|
|
264
341
|
def create_module(self, module_id: str, description: str, **kwargs) -> Module:
|
|
265
342
|
"""Creates a module for the app.
|
|
@@ -329,8 +406,16 @@ class App(Lister, BaseClient):
|
|
|
329
406
|
>>> app = App(app_id="app_id", user_id="user_id")
|
|
330
407
|
>>> model_v1 = app.model(model_id="model_id", model_version_id="model_version_id")
|
|
331
408
|
"""
|
|
332
|
-
|
|
333
|
-
|
|
409
|
+
# Change user_app_id based on whether user_id or app_id is specified.
|
|
410
|
+
if kwargs.get("user_id") or kwargs.get("app_id"):
|
|
411
|
+
request = service_pb2.GetModelRequest(
|
|
412
|
+
user_app_id=self.auth_helper.get_user_app_id_proto(
|
|
413
|
+
kwargs.get("user_id"), kwargs.get("app_id")),
|
|
414
|
+
model_id=model_id,
|
|
415
|
+
version_id=model_version_id)
|
|
416
|
+
else:
|
|
417
|
+
request = service_pb2.GetModelRequest(
|
|
418
|
+
user_app_id=self.user_app_id, model_id=model_id, version_id=model_version_id)
|
|
334
419
|
response = self._grpc_request(self.STUB.GetModel, request)
|
|
335
420
|
|
|
336
421
|
if response.status.code != status_code_pb2.SUCCESS:
|
|
@@ -471,6 +556,22 @@ class App(Lister, BaseClient):
|
|
|
471
556
|
raise Exception(response.status)
|
|
472
557
|
self.logger.info("\nModule Deleted\n%s", response.status)
|
|
473
558
|
|
|
559
|
+
def search(self, **kwargs) -> Model:
|
|
560
|
+
"""Returns a Search object for the user and app ID.
|
|
561
|
+
|
|
562
|
+
Args:
|
|
563
|
+
see the Search class in clarifai.client.search for kwargs.
|
|
564
|
+
|
|
565
|
+
Returns:
|
|
566
|
+
Search: A Search object for the user and app ID.
|
|
567
|
+
|
|
568
|
+
Example:
|
|
569
|
+
>>> from clarifai.client.app import App
|
|
570
|
+
>>> app = App(app_id="app_id", user_id="user_id")
|
|
571
|
+
>>> search_client = app.search(top_k=12, metric="euclidean")
|
|
572
|
+
"""
|
|
573
|
+
return Search(**kwargs)
|
|
574
|
+
|
|
474
575
|
def __getattr__(self, name):
|
|
475
576
|
return getattr(self.app_info, name)
|
|
476
577
|
|
clarifai/client/base.py
CHANGED
|
@@ -2,7 +2,7 @@ import os
|
|
|
2
2
|
from datetime import datetime
|
|
3
3
|
from typing import Any, Callable
|
|
4
4
|
|
|
5
|
-
from google.protobuf
|
|
5
|
+
from google.protobuf import struct_pb2
|
|
6
6
|
from google.protobuf.timestamp_pb2 import Timestamp
|
|
7
7
|
from google.protobuf.wrappers_pb2 import BoolValue
|
|
8
8
|
|
|
@@ -71,7 +71,10 @@ class BaseClient:
|
|
|
71
71
|
try:
|
|
72
72
|
datetime_obj = datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S.%fZ')
|
|
73
73
|
except ValueError:
|
|
74
|
-
|
|
74
|
+
try:
|
|
75
|
+
datetime_obj = datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%SZ')
|
|
76
|
+
except ValueError:
|
|
77
|
+
return Timestamp()
|
|
75
78
|
|
|
76
79
|
# Convert the datetime object to a Timestamp object
|
|
77
80
|
timestamp_obj = Timestamp()
|
|
@@ -99,8 +102,12 @@ class BaseClient:
|
|
|
99
102
|
value = self.convert_string_to_timestamp(value)
|
|
100
103
|
elif key in ['workflow_recommended']:
|
|
101
104
|
value = BoolValue(value=True)
|
|
102
|
-
elif key in ['
|
|
103
|
-
|
|
105
|
+
elif key in ['fields_map', 'params']:
|
|
106
|
+
value_s = struct_pb2.Struct()
|
|
107
|
+
value_s.update(value)
|
|
108
|
+
value = value_s
|
|
109
|
+
elif key in ['metadata']:
|
|
110
|
+
continue # TODO Fix "app_duplication"
|
|
104
111
|
new_item[key] = convert_recursive(value)
|
|
105
112
|
return new_item
|
|
106
113
|
elif isinstance(item, list):
|
clarifai/client/dataset.py
CHANGED
|
@@ -29,13 +29,18 @@ ClarifaiDatasetType = TypeVar('ClarifaiDatasetType', VisualClassificationDataset
|
|
|
29
29
|
class Dataset(Lister, BaseClient):
|
|
30
30
|
"""Dataset is a class that provides access to Clarifai API endpoints related to Dataset information."""
|
|
31
31
|
|
|
32
|
-
def __init__(self,
|
|
32
|
+
def __init__(self,
|
|
33
|
+
url_init: str = "",
|
|
34
|
+
dataset_id: str = "",
|
|
35
|
+
base_url: str = "https://api.clarifai.com",
|
|
36
|
+
**kwargs):
|
|
33
37
|
"""Initializes a Dataset object.
|
|
34
38
|
|
|
35
39
|
Args:
|
|
36
40
|
url_init (str): The URL to initialize the dataset object.
|
|
37
41
|
dataset_id (str): The Dataset ID within the App to interact with.
|
|
38
|
-
|
|
42
|
+
base_url (str): Base API url. Default "https://api.clarifai.com"
|
|
43
|
+
**kwargs: Additional keyword arguments to be passed to the Dataset.
|
|
39
44
|
"""
|
|
40
45
|
if url_init != "" and dataset_id != "":
|
|
41
46
|
raise UserError("You can only specify one of url_init or dataset_id.")
|
|
@@ -51,7 +56,7 @@ class Dataset(Lister, BaseClient):
|
|
|
51
56
|
self.chunk_size = 128 # limit max protos in a req
|
|
52
57
|
self.task = None # Upload dataset type
|
|
53
58
|
self.input_object = Inputs(user_id=self.user_id, app_id=self.app_id)
|
|
54
|
-
BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id)
|
|
59
|
+
BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url)
|
|
55
60
|
Lister.__init__(self)
|
|
56
61
|
|
|
57
62
|
def _concurrent_annot_upload(self, annots: List[List[resources_pb2.Annotation]]
|
clarifai/client/input.py
CHANGED
|
@@ -23,12 +23,18 @@ from clarifai.utils.misc import BackoffIterator, Chunker
|
|
|
23
23
|
class Inputs(Lister, BaseClient):
|
|
24
24
|
"""Inputs is a class that provides access to Clarifai API endpoints related to Input information."""
|
|
25
25
|
|
|
26
|
-
def __init__(self,
|
|
26
|
+
def __init__(self,
|
|
27
|
+
user_id: str = "",
|
|
28
|
+
app_id: str = "",
|
|
29
|
+
logger_level: str = "INFO",
|
|
30
|
+
base_url: str = "https://api.clarifai.com",
|
|
31
|
+
**kwargs):
|
|
27
32
|
"""Initializes an Input object.
|
|
28
33
|
|
|
29
34
|
Args:
|
|
30
35
|
user_id (str): A user ID for authentication.
|
|
31
36
|
app_id (str): An app ID for the application to interact with.
|
|
37
|
+
base_url (str): Base API url. Default "https://api.clarifai.com"
|
|
32
38
|
**kwargs: Additional keyword arguments to be passed to the Input
|
|
33
39
|
"""
|
|
34
40
|
self.user_id = user_id
|
|
@@ -36,7 +42,7 @@ class Inputs(Lister, BaseClient):
|
|
|
36
42
|
self.kwargs = {**kwargs}
|
|
37
43
|
self.input_info = resources_pb2.Input(**self.kwargs)
|
|
38
44
|
self.logger = get_logger(logger_level=logger_level, name=__name__)
|
|
39
|
-
BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id)
|
|
45
|
+
BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url)
|
|
40
46
|
Lister.__init__(self)
|
|
41
47
|
|
|
42
48
|
def _get_proto(self,
|
|
@@ -121,8 +127,8 @@ class Inputs(Lister, BaseClient):
|
|
|
121
127
|
Input: An Input object for the specified input ID.
|
|
122
128
|
|
|
123
129
|
Example:
|
|
124
|
-
>>> from clarifai.client.input import
|
|
125
|
-
>>> input_obj =
|
|
130
|
+
>>> from clarifai.client.input import Inputs
|
|
131
|
+
>>> input_obj = Inputs()
|
|
126
132
|
>>> input_proto = input_obj.get_input_from_url(input_id = 'demo', image_url='https://samples.clarifai.com/metro-north.jpg')
|
|
127
133
|
"""
|
|
128
134
|
if not any((image_url, video_url, audio_url, text_url)):
|
|
@@ -163,8 +169,8 @@ class Inputs(Lister, BaseClient):
|
|
|
163
169
|
Input: An Input object for the specified input ID.
|
|
164
170
|
|
|
165
171
|
Example:
|
|
166
|
-
>>> from clarifai.client.input import
|
|
167
|
-
>>> input_obj =
|
|
172
|
+
>>> from clarifai.client.input import Inputs
|
|
173
|
+
>>> input_obj = Inputs()
|
|
168
174
|
>>> input_proto = input_obj.get_input_from_file(input_id = 'demo', video_file='file_path')
|
|
169
175
|
"""
|
|
170
176
|
if not any((image_file, video_file, audio_file, text_file)):
|
|
@@ -205,8 +211,8 @@ class Inputs(Lister, BaseClient):
|
|
|
205
211
|
Input: An Input object for the specified input ID.
|
|
206
212
|
|
|
207
213
|
Example:
|
|
208
|
-
>>> from clarifai.client.input import
|
|
209
|
-
>>> input_obj =
|
|
214
|
+
>>> from clarifai.client.input import Inputs
|
|
215
|
+
>>> input_obj = Inputs()
|
|
210
216
|
>>> image = open('demo.jpg', 'rb').read()
|
|
211
217
|
>>> video = open('demo.mp4', 'rb').read()
|
|
212
218
|
>>> input_proto = input_obj.get_input_from_bytes(input_id = 'demo',image_bytes =image, video_bytes=video)
|
|
@@ -240,8 +246,8 @@ class Inputs(Lister, BaseClient):
|
|
|
240
246
|
list of Input: A list of Input objects for the specified folder.
|
|
241
247
|
|
|
242
248
|
Example:
|
|
243
|
-
>>> from clarifai.client.input import
|
|
244
|
-
>>> input_obj =
|
|
249
|
+
>>> from clarifai.client.input import Inputs
|
|
250
|
+
>>> input_obj = Inputs()
|
|
245
251
|
>>> input_protos = input_obj.get_image_inputs_from_folder(folder_path='demo_folder')
|
|
246
252
|
"""
|
|
247
253
|
input_protos = []
|
|
@@ -270,8 +276,8 @@ class Inputs(Lister, BaseClient):
|
|
|
270
276
|
Text: An Input object for the specified input ID.
|
|
271
277
|
|
|
272
278
|
Example:
|
|
273
|
-
>>> from clarifai.client.input import
|
|
274
|
-
>>> input_obj =
|
|
279
|
+
>>> from clarifai.client.input import Inputs
|
|
280
|
+
>>> input_obj = Inputs()
|
|
275
281
|
>>> input_protos = input_obj.get_text_input(input_id = 'demo', raw_text = 'This is a test')
|
|
276
282
|
"""
|
|
277
283
|
text_pb = resources_pb2.Text(raw=raw_text)
|
|
@@ -296,8 +302,8 @@ class Inputs(Lister, BaseClient):
|
|
|
296
302
|
inputs: List of inputs
|
|
297
303
|
|
|
298
304
|
Example:
|
|
299
|
-
>>> from clarifai.client.input import
|
|
300
|
-
>>> input_obj =
|
|
305
|
+
>>> from clarifai.client.input import Inputs
|
|
306
|
+
>>> input_obj = Inputs()
|
|
301
307
|
>>> input_protos = input_obj.get_inputs_from_csv(csv_path='filepath', input_type='text', csv_type='raw')
|
|
302
308
|
"""
|
|
303
309
|
input_protos = []
|
|
@@ -357,8 +363,8 @@ class Inputs(Lister, BaseClient):
|
|
|
357
363
|
list of Input: A list of Input objects for the specified folder.
|
|
358
364
|
|
|
359
365
|
Example:
|
|
360
|
-
>>> from clarifai.client.input import
|
|
361
|
-
>>> input_obj =
|
|
366
|
+
>>> from clarifai.client.input import Inputs
|
|
367
|
+
>>> input_obj = Inputs()
|
|
362
368
|
>>> input_protos = input_obj.get_text_inputs_from_folder(folder_path='demo_folder')
|
|
363
369
|
"""
|
|
364
370
|
input_protos = []
|
|
@@ -385,8 +391,8 @@ class Inputs(Lister, BaseClient):
|
|
|
385
391
|
An annotation object for the specified input ID.
|
|
386
392
|
|
|
387
393
|
Example:
|
|
388
|
-
>>> from clarifai.client.input import
|
|
389
|
-
>>> input_obj =
|
|
394
|
+
>>> from clarifai.client.input import Inputs
|
|
395
|
+
>>> input_obj = Inputs()
|
|
390
396
|
>>> input_obj.get_annotation_proto(input_id='demo', label='demo', annotations=[x_min, y_min, x_max, y_max])
|
|
391
397
|
"""
|
|
392
398
|
if not isinstance(annotations, list):
|
|
@@ -424,8 +430,8 @@ class Inputs(Lister, BaseClient):
|
|
|
424
430
|
An annotation object for the specified input ID.
|
|
425
431
|
|
|
426
432
|
Example:
|
|
427
|
-
>>> from clarifai.client.input import
|
|
428
|
-
>>> input_obj =
|
|
433
|
+
>>> from clarifai.client.input import Inputs
|
|
434
|
+
>>> input_obj = Inputs()
|
|
429
435
|
>>> input_obj.get_mask_proto(input_id='demo', label='demo', polygons=[[[x,y],...,[x,y]],...])
|
|
430
436
|
"""
|
|
431
437
|
if not isinstance(polygons, list):
|
|
@@ -471,8 +477,8 @@ class Inputs(Lister, BaseClient):
|
|
|
471
477
|
input_job_id: job id for the upload request.
|
|
472
478
|
|
|
473
479
|
Example:
|
|
474
|
-
>>> from clarifai.client.input import
|
|
475
|
-
>>> input_obj =
|
|
480
|
+
>>> from clarifai.client.input import Inputs
|
|
481
|
+
>>> input_obj = Inputs(user_id = 'user_id', app_id = 'demo_app')
|
|
476
482
|
>>> input_obj.upload_from_url(input_id='demo', image_url='https://samples.clarifai.com/metro-north.jpg')
|
|
477
483
|
"""
|
|
478
484
|
input_pb = self.get_input_from_url(input_id, image_url, video_url, audio_url, text_url,
|
|
@@ -501,8 +507,8 @@ class Inputs(Lister, BaseClient):
|
|
|
501
507
|
input_job_id: job id for the upload request.
|
|
502
508
|
|
|
503
509
|
Example:
|
|
504
|
-
>>> from clarifai.client.input import
|
|
505
|
-
>>> input_obj =
|
|
510
|
+
>>> from clarifai.client.input import Inputs
|
|
511
|
+
>>> input_obj = Inputs(user_id = 'user_id', app_id = 'demo_app')
|
|
506
512
|
>>> input_obj.upload_from_file(input_id='demo', audio_file='demo.mp3')
|
|
507
513
|
"""
|
|
508
514
|
input_pb = self.get_input_from_file(input_id, image_file, video_file, audio_file, text_file,
|
|
@@ -531,8 +537,8 @@ class Inputs(Lister, BaseClient):
|
|
|
531
537
|
input_job_id: job id for the upload request.
|
|
532
538
|
|
|
533
539
|
Example:
|
|
534
|
-
>>> from clarifai.client.input import
|
|
535
|
-
>>> input_obj =
|
|
540
|
+
>>> from clarifai.client.input import Inputs
|
|
541
|
+
>>> input_obj = Inputs(user_id = 'user_id', app_id = 'demo_app')
|
|
536
542
|
>>> image = open('demo.jpg', 'rb').read()
|
|
537
543
|
>>> input_obj.upload_from_bytes(input_id='demo', image_bytes=image)
|
|
538
544
|
"""
|
|
@@ -553,8 +559,8 @@ class Inputs(Lister, BaseClient):
|
|
|
553
559
|
input_job_id (str): job id for the upload request.
|
|
554
560
|
|
|
555
561
|
Example:
|
|
556
|
-
>>> from clarifai.client.input import
|
|
557
|
-
>>> input_obj =
|
|
562
|
+
>>> from clarifai.client.input import Inputs
|
|
563
|
+
>>> input_obj = Inputs(user_id = 'user_id', app_id = 'demo_app')
|
|
558
564
|
>>> input_obj.upload_text(input_id = 'demo', raw_text = 'This is a test')
|
|
559
565
|
"""
|
|
560
566
|
input_pb = self._get_proto(
|
clarifai/client/model.py
CHANGED
|
@@ -5,6 +5,7 @@ from typing import Dict, List
|
|
|
5
5
|
from clarifai_grpc.grpc.api import resources_pb2, service_pb2
|
|
6
6
|
from clarifai_grpc.grpc.api.resources_pb2 import Input
|
|
7
7
|
from clarifai_grpc.grpc.api.status import status_code_pb2
|
|
8
|
+
from google.protobuf.json_format import MessageToDict
|
|
8
9
|
|
|
9
10
|
from clarifai.client.base import BaseClient
|
|
10
11
|
from clarifai.client.lister import Lister
|
|
@@ -22,6 +23,7 @@ class Model(Lister, BaseClient):
|
|
|
22
23
|
model_id: str = "",
|
|
23
24
|
model_version: Dict = {'id': ""},
|
|
24
25
|
output_config: Dict = {'min_value': 0},
|
|
26
|
+
base_url: str = "https://api.clarifai.com",
|
|
25
27
|
**kwargs):
|
|
26
28
|
"""Initializes a Model object.
|
|
27
29
|
|
|
@@ -34,7 +36,8 @@ class Model(Lister, BaseClient):
|
|
|
34
36
|
max_concepts (int): The maximum number of concepts to return.
|
|
35
37
|
select_concepts (list[Concept]): The concepts to select.
|
|
36
38
|
sample_ms (int): The number of milliseconds to sample.
|
|
37
|
-
|
|
39
|
+
base_url (str): Base API url. Default "https://api.clarifai.com"
|
|
40
|
+
**kwargs: Additional keyword arguments to be passed to the Model.
|
|
38
41
|
"""
|
|
39
42
|
if url_init != "" and model_id != "":
|
|
40
43
|
raise UserError("You can only specify one of url_init or model_id.")
|
|
@@ -49,9 +52,75 @@ class Model(Lister, BaseClient):
|
|
|
49
52
|
'output_info': {'output_config': output_config}}
|
|
50
53
|
self.model_info = resources_pb2.Model(**self.kwargs)
|
|
51
54
|
self.logger = get_logger(logger_level="INFO")
|
|
52
|
-
BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id)
|
|
55
|
+
BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url)
|
|
53
56
|
Lister.__init__(self)
|
|
54
57
|
|
|
58
|
+
def create_model_version(self, **kwargs) -> 'Model':
|
|
59
|
+
"""Creates a model version for the Model.
|
|
60
|
+
|
|
61
|
+
Args:
|
|
62
|
+
**kwargs: Additional keyword arguments to be passed to Model Version.
|
|
63
|
+
- description (str): The description of the model version.
|
|
64
|
+
- concepts (list[Concept]): The concepts to associate with the model version.
|
|
65
|
+
- output_info (resources_pb2.OutputInfo(): The output info to associate with the model version.
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
Model: A Model object for the specified model ID.
|
|
69
|
+
|
|
70
|
+
Example:
|
|
71
|
+
>>> from clarifai.client.model import Model
|
|
72
|
+
>>> model = Model("model_url")
|
|
73
|
+
or
|
|
74
|
+
>>> model = Model(model_id='model_id', user_id='user_id', app_id='app_id')
|
|
75
|
+
>>> model_version = model.create_model_version(description='model_version_description')
|
|
76
|
+
"""
|
|
77
|
+
request = service_pb2.PostModelVersionsRequest(
|
|
78
|
+
user_app_id=self.user_app_id,
|
|
79
|
+
model_id=self.id,
|
|
80
|
+
model_versions=[resources_pb2.ModelVersion(**kwargs)])
|
|
81
|
+
|
|
82
|
+
response = self._grpc_request(self.STUB.PostModelVersions, request)
|
|
83
|
+
if response.status.code != status_code_pb2.SUCCESS:
|
|
84
|
+
raise Exception(response.status)
|
|
85
|
+
self.logger.info("\nModel Version created\n%s", response.status)
|
|
86
|
+
|
|
87
|
+
kwargs.update({'app_id': self.app_id, 'user_id': self.user_id})
|
|
88
|
+
dict_response = MessageToDict(response, preserving_proto_field_name=True)
|
|
89
|
+
kwargs = self.process_response_keys(dict_response['model'], 'model')
|
|
90
|
+
|
|
91
|
+
return Model(**kwargs)
|
|
92
|
+
|
|
93
|
+
def list_versions(self) -> List['Model']:
|
|
94
|
+
"""Lists all the versions for the model.
|
|
95
|
+
|
|
96
|
+
Returns:
|
|
97
|
+
List[Model]: A list of Model objects for the versions of the model.
|
|
98
|
+
|
|
99
|
+
Example:
|
|
100
|
+
>>> from clarifai.client.model import Model
|
|
101
|
+
>>> model = Model("model_url") # Example URL: https://clarifai.com/clarifai/main/models/general-image-recognition
|
|
102
|
+
or
|
|
103
|
+
>>> model = Model(model_id='model_id', user_id='user_id', app_id='app_id')
|
|
104
|
+
>>> all_model_versions = model.list_versions()
|
|
105
|
+
"""
|
|
106
|
+
request_data = dict(
|
|
107
|
+
user_app_id=self.user_app_id,
|
|
108
|
+
model_id=self.id,
|
|
109
|
+
per_page=self.default_page_size,
|
|
110
|
+
)
|
|
111
|
+
all_model_versions_info = list(
|
|
112
|
+
self.list_all_pages_generator(self.STUB.ListModelVersions,
|
|
113
|
+
service_pb2.ListModelVersionsRequest, request_data))
|
|
114
|
+
|
|
115
|
+
for model_version_info in all_model_versions_info:
|
|
116
|
+
model_version_info['id'] = model_version_info['model_version_id']
|
|
117
|
+
del model_version_info['model_version_id']
|
|
118
|
+
|
|
119
|
+
return [
|
|
120
|
+
Model(model_id=self.id, **dict(self.kwargs, model_version=model_version_info))
|
|
121
|
+
for model_version_info in all_model_versions_info
|
|
122
|
+
]
|
|
123
|
+
|
|
55
124
|
def predict(self, inputs: List[Input]):
|
|
56
125
|
"""Predicts the model based on the given inputs.
|
|
57
126
|
|
clarifai/client/module.py
CHANGED
|
@@ -16,6 +16,7 @@ class Module(Lister, BaseClient):
|
|
|
16
16
|
url_init: str = "",
|
|
17
17
|
module_id: str = "",
|
|
18
18
|
module_version: Dict = {'id': ""},
|
|
19
|
+
base_url: str = "https://api.clarifai.com",
|
|
19
20
|
**kwargs):
|
|
20
21
|
"""Initializes a Module object.
|
|
21
22
|
|
|
@@ -23,7 +24,8 @@ class Module(Lister, BaseClient):
|
|
|
23
24
|
url_init (str): The URL to initialize the module object.
|
|
24
25
|
module_id (str): The Module ID to interact with.
|
|
25
26
|
module_version (dict): The Module Version to interact with.
|
|
26
|
-
|
|
27
|
+
base_url (str): Base API url. Default "https://api.clarifai.com"
|
|
28
|
+
**kwargs: Additional keyword arguments to be passed to the Module.
|
|
27
29
|
"""
|
|
28
30
|
if url_init != "" and module_id != "":
|
|
29
31
|
raise UserError("You can only specify one of url_init or module_id.")
|
|
@@ -38,7 +40,7 @@ class Module(Lister, BaseClient):
|
|
|
38
40
|
self.kwargs = {**kwargs, 'id': module_id, 'module_version': module_version}
|
|
39
41
|
self.module_info = resources_pb2.Module(**self.kwargs)
|
|
40
42
|
self.logger = get_logger(logger_level="INFO")
|
|
41
|
-
BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id)
|
|
43
|
+
BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url)
|
|
42
44
|
Lister.__init__(self)
|
|
43
45
|
|
|
44
46
|
def list_versions(self) -> List['Module']:
|