clarifai 11.6.6__py3-none-any.whl → 11.6.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clarifai/__init__.py +1 -1
- clarifai/cli/README.md +51 -0
- clarifai/cli/base.py +10 -10
- clarifai/cli/pipeline.py +70 -1
- clarifai/cli/pipeline_step.py +67 -1
- clarifai/cli/templates/pipeline_step_templates.py +3 -3
- clarifai/cli/templates/pipeline_templates.py +7 -7
- clarifai/client/__init__.py +2 -0
- clarifai/client/app.py +147 -0
- clarifai/client/nodepool.py +7 -2
- clarifai/client/pipeline.py +31 -27
- clarifai/client/pipeline_step.py +72 -0
- clarifai/client/user.py +74 -0
- clarifai/runners/pipelines/pipeline_builder.py +29 -16
- clarifai/runners/server.py +29 -22
- clarifai/runners/utils/code_script.py +3 -1
- clarifai/runners/utils/pipeline_validation.py +4 -4
- clarifai/utils/cli.py +35 -1
- clarifai/utils/config.py +13 -0
- {clarifai-11.6.6.dist-info → clarifai-11.6.7.dist-info}/METADATA +1 -1
- {clarifai-11.6.6.dist-info → clarifai-11.6.7.dist-info}/RECORD +25 -24
- {clarifai-11.6.6.dist-info → clarifai-11.6.7.dist-info}/WHEEL +0 -0
- {clarifai-11.6.6.dist-info → clarifai-11.6.7.dist-info}/entry_points.txt +0 -0
- {clarifai-11.6.6.dist-info → clarifai-11.6.7.dist-info}/licenses/LICENSE +0 -0
- {clarifai-11.6.6.dist-info → clarifai-11.6.7.dist-info}/top_level.txt +0 -0
clarifai/__init__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "11.6.
|
1
|
+
__version__ = "11.6.7"
|
clarifai/cli/README.md
CHANGED
@@ -121,6 +121,57 @@ Delete existing Compute Clusters:
|
|
121
121
|
$ clarifai computecluster delete --compute_cluster_id <compute-cluster-id>
|
122
122
|
```
|
123
123
|
|
124
|
+
## Pipelines
|
125
|
+
|
126
|
+
### List Pipelines
|
127
|
+
|
128
|
+
List all pipelines for the user across all apps:
|
129
|
+
|
130
|
+
```bash
|
131
|
+
$ clarifai pipeline list
|
132
|
+
```
|
133
|
+
|
134
|
+
List pipelines within a specific app:
|
135
|
+
|
136
|
+
```bash
|
137
|
+
$ clarifai pipeline list --app_id <app-id>
|
138
|
+
```
|
139
|
+
|
140
|
+
List with pagination:
|
141
|
+
|
142
|
+
```bash
|
143
|
+
$ clarifai pipeline list --page_no 1 --per_page 10
|
144
|
+
```
|
145
|
+
|
146
|
+
### List Pipeline Steps
|
147
|
+
|
148
|
+
List all pipeline steps for the user across all apps:
|
149
|
+
|
150
|
+
```bash
|
151
|
+
$ clarifai pipelinestep list
|
152
|
+
```
|
153
|
+
|
154
|
+
List pipeline steps within a specific app:
|
155
|
+
|
156
|
+
```bash
|
157
|
+
$ clarifai pipelinestep list --app_id <app-id>
|
158
|
+
```
|
159
|
+
|
160
|
+
List pipeline steps for a specific pipeline:
|
161
|
+
|
162
|
+
```bash
|
163
|
+
$ clarifai pipelinestep list --app_id <app-id> --pipeline_id <pipeline-id>
|
164
|
+
```
|
165
|
+
|
166
|
+
### Aliases
|
167
|
+
|
168
|
+
Both commands support the `ls` alias for convenience:
|
169
|
+
|
170
|
+
```bash
|
171
|
+
$ clarifai pipeline ls
|
172
|
+
$ clarifai pipelinestep ls
|
173
|
+
```
|
174
|
+
|
124
175
|
## Learn More
|
125
176
|
|
126
177
|
* [Example Configs](https://github.com/Clarifai/examples/tree/main/ComputeOrchestration/configs)
|
clarifai/cli/base.py
CHANGED
@@ -72,18 +72,18 @@ def login(ctx, api_url, user_id):
|
|
72
72
|
"""Login command to set PAT and other configurations."""
|
73
73
|
from clarifai.utils.cli import validate_context_auth
|
74
74
|
|
75
|
+
# Input user_id if not supplied
|
76
|
+
if not user_id:
|
77
|
+
user_id = click.prompt('Enter your Clarifai user ID', type=str)
|
78
|
+
|
75
79
|
click.echo('> To authenticate, you\'ll need a Personal Access Token (PAT).')
|
76
80
|
click.echo(
|
77
|
-
'> You can create one from your account settings: https://clarifai.com/settings/security\n'
|
81
|
+
f'> You can create one from your account settings: https://clarifai.com/{user_id}/settings/security\n'
|
78
82
|
)
|
79
83
|
|
80
84
|
# Securely input PAT
|
81
85
|
pat = getpass.getpass('Enter your Personal Access Token: ')
|
82
86
|
|
83
|
-
# Input user_id if not supplied
|
84
|
-
if not user_id:
|
85
|
-
user_id = click.prompt('Enter your Clarifai user ID', type=str)
|
86
|
-
|
87
87
|
# Progress indicator
|
88
88
|
click.echo('\n> Verifying token...')
|
89
89
|
validate_context_auth(pat, user_id, api_url)
|
@@ -122,7 +122,7 @@ def input_or_default(prompt, default):
|
|
122
122
|
|
123
123
|
|
124
124
|
# Context management commands under config group
|
125
|
-
@config.command(aliases=['get-contexts', 'list-contexts'])
|
125
|
+
@config.command(aliases=['get-contexts', 'list-contexts', 'ls'])
|
126
126
|
@click.option(
|
127
127
|
'-o', '--output-format', default='wide', type=click.Choice(['wide', 'name', 'json', 'yaml'])
|
128
128
|
)
|
@@ -161,7 +161,7 @@ def get_contexts(ctx, output_format):
|
|
161
161
|
print(yaml.safe_dump(dicts))
|
162
162
|
|
163
163
|
|
164
|
-
@config.command(aliases=['use-context'])
|
164
|
+
@config.command(aliases=['use-context', 'use'])
|
165
165
|
@click.argument('name', type=str)
|
166
166
|
@click.pass_context
|
167
167
|
def use_context(ctx, name):
|
@@ -173,7 +173,7 @@ def use_context(ctx, name):
|
|
173
173
|
print(f'Set {name} as the current context')
|
174
174
|
|
175
175
|
|
176
|
-
@config.command(aliases=['current-context'])
|
176
|
+
@config.command(aliases=['current-context', 'current'])
|
177
177
|
@click.option('-o', '--output-format', default='name', type=click.Choice(['name', 'json', 'yaml']))
|
178
178
|
@click.pass_context
|
179
179
|
def current_context(ctx, output_format):
|
@@ -186,7 +186,7 @@ def current_context(ctx, output_format):
|
|
186
186
|
print(yaml.safe_dump(ctx.obj.contexts[ctx.obj.current_context].to_serializable_dict()))
|
187
187
|
|
188
188
|
|
189
|
-
@config.command(aliases=['create-context', '
|
189
|
+
@config.command(aliases=['create-context', 'create'])
|
190
190
|
@click.argument('name')
|
191
191
|
@click.option('--user-id', required=False, help='User ID')
|
192
192
|
@click.option('--base-url', required=False, help='Base URL')
|
@@ -233,7 +233,7 @@ def edit(
|
|
233
233
|
os.system(f'{os.environ.get("EDITOR", "vi")} {ctx.obj.filename}')
|
234
234
|
|
235
235
|
|
236
|
-
@config.command(aliases=['delete-context'])
|
236
|
+
@config.command(aliases=['delete-context', 'delete'])
|
237
237
|
@click.argument('name')
|
238
238
|
@click.pass_context
|
239
239
|
def delete_context(ctx, name):
|
clarifai/cli/pipeline.py
CHANGED
@@ -4,15 +4,24 @@ import shutil
|
|
4
4
|
import click
|
5
5
|
|
6
6
|
from clarifai.cli.base import cli
|
7
|
+
from clarifai.client.app import App
|
8
|
+
from clarifai.client.user import User
|
9
|
+
from clarifai.utils.cli import (
|
10
|
+
AliasedGroup,
|
11
|
+
convert_timestamp_to_string,
|
12
|
+
display_co_resources,
|
13
|
+
validate_context,
|
14
|
+
)
|
7
15
|
from clarifai.utils.logging import logger
|
8
16
|
|
9
17
|
|
10
18
|
@cli.group(
|
11
19
|
['pipeline', 'pl'],
|
20
|
+
cls=AliasedGroup,
|
12
21
|
context_settings={'max_content_width': shutil.get_terminal_size().columns - 10},
|
13
22
|
)
|
14
23
|
def pipeline():
|
15
|
-
"""Manage pipelines: upload, init, etc"""
|
24
|
+
"""Manage pipelines: upload, init, list, etc"""
|
16
25
|
|
17
26
|
|
18
27
|
@pipeline.command()
|
@@ -113,6 +122,19 @@ def run(
|
|
113
122
|
monitor_interval = config_data.get('monitor_interval', monitor_interval)
|
114
123
|
log_file = config_data.get('log_file', log_file)
|
115
124
|
monitor = config_data.get('monitor', monitor)
|
125
|
+
elif ctx.obj.current:
|
126
|
+
if not user_id:
|
127
|
+
user_id = ctx.obj.current.get('user_id', '')
|
128
|
+
if not app_id:
|
129
|
+
app_id = ctx.obj.current.get('app_id', '')
|
130
|
+
if not pipeline_id:
|
131
|
+
pipeline_id = ctx.obj.current.get('pipeline_id', '')
|
132
|
+
if not pipeline_version_id:
|
133
|
+
pipeline_version_id = ctx.obj.current.get('pipeline_version_id', '')
|
134
|
+
if not nodepool_id:
|
135
|
+
nodepool_id = ctx.obj.current.get('nodepool_id', '')
|
136
|
+
if not compute_cluster_id:
|
137
|
+
compute_cluster_id = ctx.obj.current.get('compute_cluster_id', '')
|
116
138
|
|
117
139
|
# compute_cluster_id and nodepool_id are mandatory regardless of whether pipeline_url is provided
|
118
140
|
if not compute_cluster_id or not nodepool_id:
|
@@ -275,3 +297,50 @@ def init(pipeline_path):
|
|
275
297
|
)
|
276
298
|
logger.info("4. Add dependencies to requirements.txt files as needed")
|
277
299
|
logger.info("5. Run 'clarifai pipeline upload config.yaml' to upload your pipeline")
|
300
|
+
|
301
|
+
|
302
|
+
@pipeline.command(['ls'])
|
303
|
+
@click.option('--page_no', required=False, help='Page number to list.', default=1)
|
304
|
+
@click.option('--per_page', required=False, help='Number of items per page.', default=16)
|
305
|
+
@click.option(
|
306
|
+
'--app_id',
|
307
|
+
required=False,
|
308
|
+
help='App ID to list pipelines from. If not provided, lists across all apps.',
|
309
|
+
)
|
310
|
+
@click.pass_context
|
311
|
+
def list(ctx, page_no, per_page, app_id):
|
312
|
+
"""List all pipelines for the user."""
|
313
|
+
validate_context(ctx)
|
314
|
+
|
315
|
+
if app_id:
|
316
|
+
app = App(
|
317
|
+
app_id=app_id,
|
318
|
+
user_id=ctx.obj.current.user_id,
|
319
|
+
pat=ctx.obj.current.pat,
|
320
|
+
base_url=ctx.obj.current.api_base,
|
321
|
+
)
|
322
|
+
response = app.list_pipelines(page_no=page_no, per_page=per_page)
|
323
|
+
else:
|
324
|
+
user = User(
|
325
|
+
user_id=ctx.obj.current.user_id,
|
326
|
+
pat=ctx.obj.current.pat,
|
327
|
+
base_url=ctx.obj.current.api_base,
|
328
|
+
)
|
329
|
+
response = user.list_pipelines(page_no=page_no, per_page=per_page)
|
330
|
+
|
331
|
+
display_co_resources(
|
332
|
+
response,
|
333
|
+
custom_columns={
|
334
|
+
'ID': lambda p: getattr(p, 'pipeline_id', ''),
|
335
|
+
'USER_ID': lambda p: getattr(p, 'user_id', ''),
|
336
|
+
'APP_ID': lambda p: getattr(p, 'app_id', ''),
|
337
|
+
'VERSION_ID': lambda p: getattr(p, 'pipeline_version_id', ''),
|
338
|
+
'DESCRIPTION': lambda p: getattr(p, 'description', ''),
|
339
|
+
'CREATED_AT': lambda ps: convert_timestamp_to_string(getattr(ps, 'created_at', '')),
|
340
|
+
'MODIFIED_AT': lambda ps: convert_timestamp_to_string(getattr(ps, 'modified_at', '')),
|
341
|
+
},
|
342
|
+
sort_by_columns=[
|
343
|
+
('CREATED_AT', 'desc'),
|
344
|
+
('ID', 'asc'),
|
345
|
+
],
|
346
|
+
)
|
clarifai/cli/pipeline_step.py
CHANGED
@@ -4,15 +4,24 @@ import shutil
|
|
4
4
|
import click
|
5
5
|
|
6
6
|
from clarifai.cli.base import cli
|
7
|
+
from clarifai.client.app import App
|
8
|
+
from clarifai.client.user import User
|
9
|
+
from clarifai.utils.cli import (
|
10
|
+
AliasedGroup,
|
11
|
+
convert_timestamp_to_string,
|
12
|
+
display_co_resources,
|
13
|
+
validate_context,
|
14
|
+
)
|
7
15
|
from clarifai.utils.logging import logger
|
8
16
|
|
9
17
|
|
10
18
|
@cli.group(
|
11
19
|
['pipelinestep', 'ps'],
|
20
|
+
cls=AliasedGroup,
|
12
21
|
context_settings={'max_content_width': shutil.get_terminal_size().columns - 10},
|
13
22
|
)
|
14
23
|
def pipeline_step():
|
15
|
-
"""Manage pipeline steps: upload, test, etc"""
|
24
|
+
"""Manage pipeline steps: upload, test, list, etc"""
|
16
25
|
|
17
26
|
|
18
27
|
@pipeline_step.command()
|
@@ -102,3 +111,60 @@ def init(pipeline_step_path):
|
|
102
111
|
logger.info("2. Update the pipeline step configuration in config.yaml")
|
103
112
|
logger.info("3. Add your pipeline step dependencies to requirements.txt")
|
104
113
|
logger.info("4. Implement your pipeline step logic in 1/pipeline_step.py")
|
114
|
+
|
115
|
+
|
116
|
+
@pipeline_step.command(['ls'])
|
117
|
+
@click.option('--page_no', required=False, help='Page number to list.', default=1)
|
118
|
+
@click.option('--per_page', required=False, help='Number of items per page.', default=16)
|
119
|
+
@click.option(
|
120
|
+
'--app_id',
|
121
|
+
required=False,
|
122
|
+
help='App ID to list pipeline steps from. If not provided, lists across all apps.',
|
123
|
+
)
|
124
|
+
@click.option(
|
125
|
+
'--pipeline_id',
|
126
|
+
required=False,
|
127
|
+
help='Pipeline ID to list pipeline steps from. Must be used with --app_id.',
|
128
|
+
)
|
129
|
+
@click.pass_context
|
130
|
+
def list(ctx, page_no, per_page, app_id, pipeline_id):
|
131
|
+
"""List all pipeline steps for the user."""
|
132
|
+
validate_context(ctx)
|
133
|
+
|
134
|
+
if pipeline_id and not app_id:
|
135
|
+
raise click.UsageError("--pipeline_id must be used together with --app_id")
|
136
|
+
|
137
|
+
if app_id:
|
138
|
+
app = App(
|
139
|
+
app_id=app_id,
|
140
|
+
user_id=ctx.obj.current.user_id,
|
141
|
+
pat=ctx.obj.current.pat,
|
142
|
+
base_url=ctx.obj.current.api_base,
|
143
|
+
)
|
144
|
+
response = app.list_pipeline_steps(
|
145
|
+
pipeline_id=pipeline_id, page_no=page_no, per_page=per_page
|
146
|
+
)
|
147
|
+
else:
|
148
|
+
user = User(
|
149
|
+
user_id=ctx.obj.current.user_id,
|
150
|
+
pat=ctx.obj.current.pat,
|
151
|
+
base_url=ctx.obj.current.api_base,
|
152
|
+
)
|
153
|
+
response = user.list_pipeline_steps(page_no=page_no, per_page=per_page)
|
154
|
+
|
155
|
+
display_co_resources(
|
156
|
+
response,
|
157
|
+
custom_columns={
|
158
|
+
'PIPELINE_STEP_ID': lambda ps: getattr(ps, 'pipeline_step_id', ''),
|
159
|
+
'USER_ID': lambda ps: getattr(ps, 'user_id', ''),
|
160
|
+
'APP_ID': lambda ps: getattr(ps, 'app_id', ''),
|
161
|
+
'VERSION_ID': lambda ps: getattr(ps, 'pipeline_step_version_id', ''),
|
162
|
+
'DESCRIPTION': lambda ps: getattr(ps, 'description', ''),
|
163
|
+
'CREATED_AT': lambda ps: convert_timestamp_to_string(getattr(ps, 'created_at', '')),
|
164
|
+
'MODIFIED_AT': lambda ps: convert_timestamp_to_string(getattr(ps, 'modified_at', '')),
|
165
|
+
},
|
166
|
+
sort_by_columns=[
|
167
|
+
('CREATED_AT', 'desc'),
|
168
|
+
('PIPELINE_STEP_ID', 'asc'),
|
169
|
+
],
|
170
|
+
)
|
@@ -35,7 +35,7 @@ def get_pipeline_step_template():
|
|
35
35
|
return '''import argparse
|
36
36
|
|
37
37
|
import clarifai
|
38
|
-
|
38
|
+
from clarifai.utils.logging import logger
|
39
39
|
|
40
40
|
def main():
|
41
41
|
parser = argparse.ArgumentParser(description='Concatenate two strings.')
|
@@ -44,9 +44,9 @@ def main():
|
|
44
44
|
|
45
45
|
args = parser.parse_args()
|
46
46
|
|
47
|
-
|
47
|
+
logger.info(clarifai.__version__)
|
48
48
|
|
49
|
-
|
49
|
+
logger.info(f"Concatenation Output: {args.param_a + args.param_b}")
|
50
50
|
|
51
51
|
|
52
52
|
if __name__ == "__main__":
|
@@ -25,12 +25,12 @@ def get_pipeline_config_template():
|
|
25
25
|
steps:
|
26
26
|
- - name: step-a
|
27
27
|
templateRef:
|
28
|
-
name: users/your_user_id/apps/your_app_id/
|
29
|
-
template: users/your_user_id/apps/your_app_id/
|
28
|
+
name: users/your_user_id/apps/your_app_id/pipeline_steps/stepA # TODO: please fill in
|
29
|
+
template: users/your_user_id/apps/your_app_id/pipeline_steps/stepA # TODO: please fill in
|
30
30
|
- - name: step-b
|
31
31
|
templateRef:
|
32
|
-
name: users/your_user_id/apps/your_app_id/
|
33
|
-
template: users/your_user_id/apps/your_app_id/
|
32
|
+
name: users/your_user_id/apps/your_app_id/pipeline_steps/stepB # TODO: please fill in
|
33
|
+
template: users/your_user_id/apps/your_app_id/pipeline_steps/stepB # TODO: please fill in
|
34
34
|
"""
|
35
35
|
|
36
36
|
|
@@ -60,7 +60,7 @@ def get_pipeline_step_template(step_id: str):
|
|
60
60
|
return f'''import argparse
|
61
61
|
|
62
62
|
import clarifai
|
63
|
-
|
63
|
+
from clarifai.utils.logging import logger
|
64
64
|
|
65
65
|
def main():
|
66
66
|
parser = argparse.ArgumentParser(description='{step_id} processing step.')
|
@@ -68,10 +68,10 @@ def main():
|
|
68
68
|
|
69
69
|
args = parser.parse_args()
|
70
70
|
|
71
|
-
|
71
|
+
logger.info(clarifai.__version__)
|
72
72
|
|
73
73
|
# TODO: Implement your pipeline step logic here
|
74
|
-
|
74
|
+
logger.info(f"{step_id} processed: {{args.input_text}}")
|
75
75
|
|
76
76
|
|
77
77
|
if __name__ == "__main__":
|
clarifai/client/__init__.py
CHANGED
@@ -8,6 +8,7 @@ from clarifai.client.lister import Lister
|
|
8
8
|
from clarifai.client.model import Model
|
9
9
|
from clarifai.client.module import Module
|
10
10
|
from clarifai.client.pipeline import Pipeline
|
11
|
+
from clarifai.client.pipeline_step import PipelineStep
|
11
12
|
from clarifai.client.search import Search
|
12
13
|
from clarifai.client.user import User
|
13
14
|
from clarifai.client.workflow import Workflow
|
@@ -20,6 +21,7 @@ __all__ = [
|
|
20
21
|
'Model',
|
21
22
|
'Workflow',
|
22
23
|
'Pipeline',
|
24
|
+
'PipelineStep',
|
23
25
|
'Module',
|
24
26
|
'Lister',
|
25
27
|
'Dataset',
|
clarifai/client/app.py
CHANGED
@@ -14,6 +14,8 @@ from clarifai.client.input import Inputs
|
|
14
14
|
from clarifai.client.lister import Lister
|
15
15
|
from clarifai.client.model import Model
|
16
16
|
from clarifai.client.module import Module
|
17
|
+
from clarifai.client.pipeline import Pipeline
|
18
|
+
from clarifai.client.pipeline_step import PipelineStep
|
17
19
|
from clarifai.client.search import Search
|
18
20
|
from clarifai.client.workflow import Workflow
|
19
21
|
from clarifai.constants.model import TRAINABLE_MODEL_TYPES
|
@@ -198,6 +200,151 @@ class App(Lister, BaseClient):
|
|
198
200
|
continue
|
199
201
|
yield Workflow.from_auth_helper(auth=self.auth_helper, **workflow_info)
|
200
202
|
|
203
|
+
def list_pipelines(
|
204
|
+
self,
|
205
|
+
filter_by: Dict[str, Any] = {},
|
206
|
+
only_in_app: bool = True,
|
207
|
+
page_no: int = None,
|
208
|
+
per_page: int = None,
|
209
|
+
) -> Generator[dict, None, None]:
|
210
|
+
"""Lists all the pipelines for the user.
|
211
|
+
|
212
|
+
Args:
|
213
|
+
filter_by (dict): A dictionary of filters to apply to the list of pipelines.
|
214
|
+
only_in_app (bool): If True, only return pipelines that are in the app.
|
215
|
+
page_no (int): The page number to list.
|
216
|
+
per_page (int): The number of items per page.
|
217
|
+
|
218
|
+
Yields:
|
219
|
+
Dict: Dictionaries containing information about the pipelines.
|
220
|
+
|
221
|
+
Example:
|
222
|
+
>>> from clarifai.client.app import App
|
223
|
+
>>> app = App(app_id="app_id", user_id="user_id")
|
224
|
+
>>> all_pipelines = list(app.list_pipelines())
|
225
|
+
|
226
|
+
Note:
|
227
|
+
Defaults to 16 per page if page_no is specified and per_page is not specified.
|
228
|
+
If both page_no and per_page are None, then lists all the resources.
|
229
|
+
"""
|
230
|
+
request_data = dict(user_app_id=self.user_app_id, **filter_by)
|
231
|
+
all_pipelines_info = self.list_pages_generator(
|
232
|
+
self.STUB.ListPipelines,
|
233
|
+
service_pb2.ListPipelinesRequest,
|
234
|
+
request_data,
|
235
|
+
per_page=per_page,
|
236
|
+
page_no=page_no,
|
237
|
+
)
|
238
|
+
|
239
|
+
for pipeline_info in all_pipelines_info:
|
240
|
+
pipeline = self._process_pipeline_info(
|
241
|
+
pipeline_info, self.auth_helper, self.id, only_in_app
|
242
|
+
)
|
243
|
+
if pipeline is not None:
|
244
|
+
yield pipeline
|
245
|
+
|
246
|
+
@staticmethod
|
247
|
+
def _process_pipeline_info(pipeline_info, auth_helper, app_id=None, only_in_app=False):
|
248
|
+
"""Helper method to process pipeline info and create Pipeline objects.
|
249
|
+
|
250
|
+
Args:
|
251
|
+
pipeline_info: Raw pipeline info from API
|
252
|
+
auth_helper: Auth helper instance
|
253
|
+
app_id: App ID to filter by (if only_in_app is True)
|
254
|
+
only_in_app: Whether to filter by app_id
|
255
|
+
|
256
|
+
Returns:
|
257
|
+
Pipeline object or None if filtered out
|
258
|
+
"""
|
259
|
+
if only_in_app and app_id:
|
260
|
+
if pipeline_info.get('app_id') != app_id:
|
261
|
+
return None
|
262
|
+
|
263
|
+
# Map API field names to constructor parameter names
|
264
|
+
pipeline_kwargs = pipeline_info.copy()
|
265
|
+
if 'id' in pipeline_kwargs:
|
266
|
+
pipeline_kwargs['pipeline_id'] = pipeline_kwargs.pop('id')
|
267
|
+
if 'pipeline_version' in pipeline_kwargs:
|
268
|
+
pipeline_version = pipeline_kwargs.pop('pipeline_version')
|
269
|
+
pipeline_kwargs['pipeline_version_id'] = pipeline_version.get('id', '')
|
270
|
+
|
271
|
+
return Pipeline.from_auth_helper(auth=auth_helper, **pipeline_kwargs)
|
272
|
+
|
273
|
+
@staticmethod
|
274
|
+
def _process_pipeline_step_info(
|
275
|
+
pipeline_step_info, auth_helper, app_id=None, only_in_app=False
|
276
|
+
):
|
277
|
+
"""Helper method to process pipeline step info and create PipelineStep objects.
|
278
|
+
|
279
|
+
Args:
|
280
|
+
pipeline_step_info: Raw pipeline step info from API
|
281
|
+
auth_helper: Auth helper instance
|
282
|
+
app_id: App ID to filter by (if only_in_app is True)
|
283
|
+
only_in_app: Whether to filter by app_id
|
284
|
+
|
285
|
+
Returns:
|
286
|
+
PipelineStep object or None if filtered out
|
287
|
+
"""
|
288
|
+
if only_in_app and app_id:
|
289
|
+
if pipeline_step_info.get('app_id') != app_id:
|
290
|
+
return None
|
291
|
+
|
292
|
+
# Map API field names to constructor parameter names
|
293
|
+
step_kwargs = pipeline_step_info.copy()
|
294
|
+
if 'pipeline_step' in step_kwargs:
|
295
|
+
pipeline_step = step_kwargs.pop('pipeline_step')
|
296
|
+
step_kwargs['pipeline_step_id'] = pipeline_step.get('id', '')
|
297
|
+
|
298
|
+
return PipelineStep.from_auth_helper(auth=auth_helper, **step_kwargs)
|
299
|
+
|
300
|
+
def list_pipeline_steps(
|
301
|
+
self,
|
302
|
+
pipeline_id: str = None,
|
303
|
+
filter_by: Dict[str, Any] = {},
|
304
|
+
only_in_app: bool = True,
|
305
|
+
page_no: int = None,
|
306
|
+
per_page: int = None,
|
307
|
+
) -> Generator[dict, None, None]:
|
308
|
+
"""Lists all the pipeline steps for the user.
|
309
|
+
|
310
|
+
Args:
|
311
|
+
pipeline_id (str): If provided, only list pipeline steps from this pipeline.
|
312
|
+
filter_by (dict): A dictionary of filters to apply to the list of pipeline steps.
|
313
|
+
only_in_app (bool): If True, only return pipeline steps that are in the app.
|
314
|
+
page_no (int): The page number to list.
|
315
|
+
per_page (int): The number of items per page.
|
316
|
+
|
317
|
+
Yields:
|
318
|
+
Dict: Dictionaries containing information about the pipeline steps.
|
319
|
+
|
320
|
+
Example:
|
321
|
+
>>> from clarifai.client.app import App
|
322
|
+
>>> app = App(app_id="app_id", user_id="user_id")
|
323
|
+
>>> all_pipeline_steps = list(app.list_pipeline_steps())
|
324
|
+
|
325
|
+
Note:
|
326
|
+
Defaults to 16 per page if page_no is specified and per_page is not specified.
|
327
|
+
If both page_no and per_page are None, then lists all the resources.
|
328
|
+
"""
|
329
|
+
request_data = dict(user_app_id=self.user_app_id, **filter_by)
|
330
|
+
if pipeline_id:
|
331
|
+
request_data['pipeline_id'] = pipeline_id
|
332
|
+
|
333
|
+
all_pipeline_steps_info = self.list_pages_generator(
|
334
|
+
self.STUB.ListPipelineStepVersions,
|
335
|
+
service_pb2.ListPipelineStepVersionsRequest,
|
336
|
+
request_data,
|
337
|
+
per_page=per_page,
|
338
|
+
page_no=page_no,
|
339
|
+
)
|
340
|
+
|
341
|
+
for pipeline_step_info in all_pipeline_steps_info:
|
342
|
+
pipeline_step = self._process_pipeline_step_info(
|
343
|
+
pipeline_step_info, self.auth_helper, self.id, only_in_app
|
344
|
+
)
|
345
|
+
if pipeline_step is not None:
|
346
|
+
yield pipeline_step
|
347
|
+
|
201
348
|
def list_modules(
|
202
349
|
self,
|
203
350
|
filter_by: Dict[str, Any] = {},
|
clarifai/client/nodepool.py
CHANGED
@@ -94,7 +94,9 @@ class Nodepool(Lister, BaseClient):
|
|
94
94
|
), "worker info not found in the config file"
|
95
95
|
assert "scheduling_choice" in deployment, "scheduling_choice not found in the config file"
|
96
96
|
assert "nodepools" in deployment, "nodepools not found in the config file"
|
97
|
-
deployment['user_id'] =
|
97
|
+
deployment['user_id'] = (
|
98
|
+
deployment['user_id'] if 'user_id' in deployment else self.user_app_id.user_id
|
99
|
+
)
|
98
100
|
if "autoscale_config" in deployment:
|
99
101
|
deployment['autoscale_config'] = resources_pb2.AutoscaleConfig(
|
100
102
|
**deployment['autoscale_config']
|
@@ -103,7 +105,10 @@ class Nodepool(Lister, BaseClient):
|
|
103
105
|
resources_pb2.Nodepool(
|
104
106
|
id=nodepool['id'],
|
105
107
|
compute_cluster=resources_pb2.ComputeCluster(
|
106
|
-
id=nodepool['compute_cluster']['id'],
|
108
|
+
id=nodepool['compute_cluster']['id'],
|
109
|
+
user_id=nodepool['compute_cluster']['user_id']
|
110
|
+
if 'user_id' in nodepool['compute_cluster']
|
111
|
+
else self.user_app_id.user_id,
|
107
112
|
),
|
108
113
|
)
|
109
114
|
for nodepool in deployment['nodepools']
|
clarifai/client/pipeline.py
CHANGED
@@ -4,6 +4,7 @@ from typing import Dict, List
|
|
4
4
|
|
5
5
|
from clarifai_grpc.grpc.api import resources_pb2, service_pb2
|
6
6
|
from clarifai_grpc.grpc.api.status import status_code_pb2
|
7
|
+
from google.protobuf import json_format
|
7
8
|
|
8
9
|
from clarifai.client.base import BaseClient
|
9
10
|
from clarifai.client.lister import Lister
|
@@ -13,22 +14,6 @@ from clarifai.utils.constants import DEFAULT_BASE
|
|
13
14
|
from clarifai.utils.logging import logger
|
14
15
|
|
15
16
|
|
16
|
-
def _get_status_name(status_code: int) -> str:
|
17
|
-
"""Get the human-readable name for a status code."""
|
18
|
-
status_mapping = {
|
19
|
-
# Job status codes (these are the actual values based on the error message showing 64001)
|
20
|
-
64001: "JOB_QUEUED",
|
21
|
-
64002: "JOB_RUNNING",
|
22
|
-
64003: "JOB_COMPLETED",
|
23
|
-
64004: "JOB_FAILED",
|
24
|
-
64005: "JOB_UNEXPECTED_ERROR",
|
25
|
-
# Standard status codes
|
26
|
-
10000: "SUCCESS",
|
27
|
-
10010: "MIXED_STATUS",
|
28
|
-
}
|
29
|
-
return status_mapping.get(status_code, f"UNKNOWN_STATUS_{status_code}")
|
30
|
-
|
31
|
-
|
32
17
|
class Pipeline(Lister, BaseClient):
|
33
18
|
"""Pipeline is a class that provides access to Clarifai API endpoints related to Pipeline information."""
|
34
19
|
|
@@ -82,13 +67,17 @@ class Pipeline(Lister, BaseClient):
|
|
82
67
|
|
83
68
|
self.pipeline_id = pipeline_id
|
84
69
|
self.pipeline_version_id = pipeline_version_id
|
85
|
-
self.pipeline_version_run_id = pipeline_version_run_id or str(uuid.uuid4())
|
70
|
+
self.pipeline_version_run_id = pipeline_version_run_id or str(uuid.uuid4().hex)
|
86
71
|
self.user_id = user_id
|
87
72
|
self.app_id = app_id
|
88
73
|
self.nodepool_id = nodepool_id
|
89
74
|
self.compute_cluster_id = compute_cluster_id
|
90
75
|
self.log_file = log_file
|
91
76
|
|
77
|
+
# Store all kwargs as attributes for API data
|
78
|
+
for key, value in kwargs.items():
|
79
|
+
setattr(self, key, value)
|
80
|
+
|
92
81
|
BaseClient.__init__(
|
93
82
|
self,
|
94
83
|
user_id=user_id,
|
@@ -152,9 +141,15 @@ class Pipeline(Lister, BaseClient):
|
|
152
141
|
)
|
153
142
|
|
154
143
|
if response.status.code != status_code_pb2.StatusCode.SUCCESS:
|
155
|
-
|
156
|
-
|
157
|
-
|
144
|
+
if response.status.code == status_code_pb2.StatusCode.CONN_DOES_NOT_EXIST:
|
145
|
+
logger.error(
|
146
|
+
f"Pipeline {self.pipeline_id} does not exist, did you call 'clarifai pipeline upload' first? "
|
147
|
+
)
|
148
|
+
return json_format.MessageToDict(response, preserving_proto_field_name=True)
|
149
|
+
else:
|
150
|
+
raise UserError(
|
151
|
+
f"Failed to start pipeline run: {response.status.description}. Details: {response.status.details}. Code: {status_code_pb2.StatusCode.Name(response.status.code)}."
|
152
|
+
)
|
158
153
|
|
159
154
|
if not response.pipeline_version_runs:
|
160
155
|
raise UserError("No pipeline version run was created")
|
@@ -218,6 +213,9 @@ class Pipeline(Lister, BaseClient):
|
|
218
213
|
continue
|
219
214
|
|
220
215
|
pipeline_run = run_response.pipeline_version_run
|
216
|
+
pipeline_run_dict = json_format.MessageToDict(
|
217
|
+
pipeline_run, preserving_proto_field_name=True
|
218
|
+
)
|
221
219
|
|
222
220
|
# Display new log entries
|
223
221
|
self._display_new_logs(run_id, seen_logs)
|
@@ -233,7 +231,7 @@ class Pipeline(Lister, BaseClient):
|
|
233
231
|
orch_status = pipeline_run.orchestration_status
|
234
232
|
if hasattr(orch_status, 'status') and orch_status.status:
|
235
233
|
status_code = orch_status.status.code
|
236
|
-
status_name =
|
234
|
+
status_name = status_code_pb2.StatusCode.Name(status_code)
|
237
235
|
logger.info(f"Pipeline run status: {status_code} ({status_name})")
|
238
236
|
|
239
237
|
# Display orchestration status details if available
|
@@ -241,23 +239,29 @@ class Pipeline(Lister, BaseClient):
|
|
241
239
|
logger.info(f"Orchestration status: {orch_status.description}")
|
242
240
|
|
243
241
|
# Success codes that allow continuation: JOB_RUNNING, JOB_QUEUED
|
244
|
-
if status_code in [
|
242
|
+
if status_code in [
|
243
|
+
status_code_pb2.JOB_QUEUED,
|
244
|
+
status_code_pb2.JOB_RUNNING,
|
245
|
+
]: # JOB_QUEUED, JOB_RUNNING
|
245
246
|
logger.info(f"Pipeline run in progress: {status_code} ({status_name})")
|
246
247
|
# Continue monitoring
|
247
248
|
# Successful terminal state: JOB_COMPLETED
|
248
|
-
elif status_code ==
|
249
|
+
elif status_code == status_code_pb2.JOB_COMPLETED: # JOB_COMPLETED
|
249
250
|
logger.info("Pipeline run completed successfully!")
|
250
|
-
return {"status": "success", "pipeline_version_run":
|
251
|
+
return {"status": "success", "pipeline_version_run": pipeline_run_dict}
|
251
252
|
# Failure terminal states: JOB_UNEXPECTED_ERROR, JOB_FAILED
|
252
|
-
elif status_code in [
|
253
|
+
elif status_code in [
|
254
|
+
status_code_pb2.JOB_FAILED,
|
255
|
+
status_code_pb2.JOB_UNEXPECTED_ERROR,
|
256
|
+
]: # JOB_FAILED, JOB_UNEXPECTED_ERROR
|
253
257
|
logger.error(
|
254
258
|
f"Pipeline run failed with status: {status_code} ({status_name})"
|
255
259
|
)
|
256
|
-
return {"status": "failed", "pipeline_version_run":
|
260
|
+
return {"status": "failed", "pipeline_version_run": pipeline_run_dict}
|
257
261
|
# Handle legacy SUCCESS status for backward compatibility
|
258
262
|
elif status_code == status_code_pb2.StatusCode.SUCCESS:
|
259
263
|
logger.info("Pipeline run completed successfully!")
|
260
|
-
return {"status": "success", "pipeline_version_run":
|
264
|
+
return {"status": "success", "pipeline_version_run": pipeline_run_dict}
|
261
265
|
elif status_code != status_code_pb2.StatusCode.MIXED_STATUS:
|
262
266
|
# Log other unexpected statuses but continue monitoring
|
263
267
|
logger.warning(
|
@@ -0,0 +1,72 @@
|
|
1
|
+
from clarifai.client.base import BaseClient
|
2
|
+
from clarifai.client.lister import Lister
|
3
|
+
from clarifai.urls.helper import ClarifaiUrlHelper
|
4
|
+
from clarifai.utils.constants import DEFAULT_BASE
|
5
|
+
|
6
|
+
|
7
|
+
class PipelineStep(Lister, BaseClient):
|
8
|
+
"""PipelineStep is a class that provides access to Clarifai API endpoints related to PipelineStep information."""
|
9
|
+
|
10
|
+
def __init__(
|
11
|
+
self,
|
12
|
+
url: str = None,
|
13
|
+
pipeline_step_id: str = None,
|
14
|
+
pipeline_step_version_id: str = None,
|
15
|
+
user_id: str = None,
|
16
|
+
app_id: str = None,
|
17
|
+
pipeline_id: str = None,
|
18
|
+
base_url: str = DEFAULT_BASE,
|
19
|
+
pat: str = None,
|
20
|
+
token: str = None,
|
21
|
+
root_certificates_path: str = None,
|
22
|
+
**kwargs,
|
23
|
+
):
|
24
|
+
"""Initializes a PipelineStep object.
|
25
|
+
|
26
|
+
Args:
|
27
|
+
url (str): The URL to initialize the pipeline step object.
|
28
|
+
pipeline_step_id (str): The PipelineStep ID for the PipelineStep to interact with.
|
29
|
+
pipeline_step_version_id (str): The PipelineStep version ID for the PipelineStep to interact with.
|
30
|
+
user_id (str): The User ID for the PipelineStep to interact with.
|
31
|
+
app_id (str): The App ID for the PipelineStep to interact with.
|
32
|
+
pipeline_id (str): The Pipeline ID for the PipelineStep to interact with.
|
33
|
+
base_url (str): Base API url. Default "https://api.clarifai.com"
|
34
|
+
pat (str): A personal access token for authentication.
|
35
|
+
token (str): A session token for authentication.
|
36
|
+
root_certificates_path (str): Path to the SSL root certificates file.
|
37
|
+
**kwargs: Additional keyword arguments to be passed to the BaseClient.
|
38
|
+
"""
|
39
|
+
if url:
|
40
|
+
user_id, app_id, _, pipeline_step_id, pipeline_step_version_id = (
|
41
|
+
ClarifaiUrlHelper.split_clarifai_url(url)
|
42
|
+
)
|
43
|
+
|
44
|
+
# Store all kwargs as attributes for API data
|
45
|
+
for key, value in kwargs.items():
|
46
|
+
setattr(self, key, value)
|
47
|
+
|
48
|
+
self.kwargs = {
|
49
|
+
"pipeline_step_id": pipeline_step_id,
|
50
|
+
"pipeline_step_version_id": pipeline_step_version_id,
|
51
|
+
"user_id": user_id,
|
52
|
+
"app_id": app_id,
|
53
|
+
"pipeline_id": pipeline_id,
|
54
|
+
**kwargs,
|
55
|
+
}
|
56
|
+
|
57
|
+
BaseClient.__init__(
|
58
|
+
self,
|
59
|
+
user_id=user_id,
|
60
|
+
app_id=app_id,
|
61
|
+
base=base_url,
|
62
|
+
pat=pat,
|
63
|
+
token=token,
|
64
|
+
root_certificates_path=root_certificates_path,
|
65
|
+
)
|
66
|
+
Lister.__init__(self)
|
67
|
+
|
68
|
+
self.pipeline_step_id = pipeline_step_id
|
69
|
+
self.pipeline_step_version_id = pipeline_step_version_id
|
70
|
+
self.pipeline_id = pipeline_id
|
71
|
+
self.user_id = user_id
|
72
|
+
self.app_id = app_id
|
clarifai/client/user.py
CHANGED
@@ -153,6 +153,80 @@ class User(Lister, BaseClient):
|
|
153
153
|
for compute_cluster_info in all_compute_clusters_info:
|
154
154
|
yield ComputeCluster.from_auth_helper(self.auth_helper, **compute_cluster_info)
|
155
155
|
|
156
|
+
def list_pipelines(
|
157
|
+
self, page_no: int = None, per_page: int = None
|
158
|
+
) -> Generator[dict, None, None]:
|
159
|
+
"""List all pipelines for the user across all apps
|
160
|
+
|
161
|
+
Args:
|
162
|
+
page_no (int): The page number to list.
|
163
|
+
per_page (int): The number of items per page.
|
164
|
+
|
165
|
+
Yields:
|
166
|
+
Dict: Dictionaries containing information about the pipelines.
|
167
|
+
|
168
|
+
Example:
|
169
|
+
>>> from clarifai.client.user import User
|
170
|
+
>>> client = User(user_id="user_id")
|
171
|
+
>>> all_pipelines = list(client.list_pipelines())
|
172
|
+
|
173
|
+
Note:
|
174
|
+
Defaults to 16 per page if page_no is specified and per_page is not specified.
|
175
|
+
If both page_no and per_page are None, then lists all the resources.
|
176
|
+
"""
|
177
|
+
request_data = dict(user_app_id=self.user_app_id)
|
178
|
+
all_pipelines_info = self.list_pages_generator(
|
179
|
+
self.STUB.ListPipelines,
|
180
|
+
service_pb2.ListPipelinesRequest,
|
181
|
+
request_data,
|
182
|
+
per_page=per_page,
|
183
|
+
page_no=page_no,
|
184
|
+
)
|
185
|
+
|
186
|
+
for pipeline_info in all_pipelines_info:
|
187
|
+
pipeline = App._process_pipeline_info(
|
188
|
+
pipeline_info, self.auth_helper, only_in_app=False
|
189
|
+
)
|
190
|
+
if pipeline is not None:
|
191
|
+
yield pipeline
|
192
|
+
|
193
|
+
def list_pipeline_steps(
|
194
|
+
self, page_no: int = None, per_page: int = None
|
195
|
+
) -> Generator[dict, None, None]:
|
196
|
+
"""List all pipeline steps for the user across all apps
|
197
|
+
|
198
|
+
Args:
|
199
|
+
page_no (int): The page number to list.
|
200
|
+
per_page (int): The number of items per page.
|
201
|
+
|
202
|
+
Yields:
|
203
|
+
Dict: Dictionaries containing information about the pipeline steps.
|
204
|
+
|
205
|
+
Example:
|
206
|
+
>>> from clarifai.client.user import User
|
207
|
+
>>> client = User(user_id="user_id")
|
208
|
+
>>> all_pipeline_steps = list(client.list_pipeline_steps())
|
209
|
+
|
210
|
+
Note:
|
211
|
+
Defaults to 16 per page if page_no is specified and per_page is not specified.
|
212
|
+
If both page_no and per_page are None, then lists all the resources.
|
213
|
+
"""
|
214
|
+
request_data = dict(user_app_id=self.user_app_id)
|
215
|
+
all_pipeline_steps_info = self.list_pages_generator(
|
216
|
+
self.STUB.ListPipelineStepVersions,
|
217
|
+
service_pb2.ListPipelineStepVersionsRequest,
|
218
|
+
request_data,
|
219
|
+
per_page=per_page,
|
220
|
+
page_no=page_no,
|
221
|
+
)
|
222
|
+
|
223
|
+
for pipeline_step_info in all_pipeline_steps_info:
|
224
|
+
pipeline_step = App._process_pipeline_step_info(
|
225
|
+
pipeline_step_info, self.auth_helper, only_in_app=False
|
226
|
+
)
|
227
|
+
if pipeline_step is not None:
|
228
|
+
yield pipeline_step
|
229
|
+
|
156
230
|
def create_app(self, app_id: str, base_workflow: str = 'Empty', **kwargs) -> App:
|
157
231
|
"""Creates an app for the user.
|
158
232
|
|
@@ -86,7 +86,7 @@ class PipelineBuilder:
|
|
86
86
|
|
87
87
|
if not step_directories:
|
88
88
|
logger.info("No pipeline steps to upload (step_directories is empty)")
|
89
|
-
return
|
89
|
+
return False # treat this as an error.
|
90
90
|
|
91
91
|
logger.info(f"Uploading {len(step_directories)} pipeline steps...")
|
92
92
|
|
@@ -191,25 +191,38 @@ class PipelineBuilder:
|
|
191
191
|
if "templateRef" in step:
|
192
192
|
template_ref = step["templateRef"]
|
193
193
|
name = template_ref["name"]
|
194
|
+
# Extract step name
|
195
|
+
parts = name.split('/')
|
194
196
|
|
195
197
|
# Check if this is a templateRef without version that we uploaded
|
196
198
|
if self.validator.TEMPLATE_REF_WITHOUT_VERSION_PATTERN.match(name):
|
197
|
-
# Extract step name
|
198
|
-
parts = name.split('/')
|
199
199
|
step_name = parts[-1]
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
#
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
200
|
+
# The step name should match the directory name or be derivable from it
|
201
|
+
version_id = self.uploaded_step_versions.get(step_name, None)
|
202
|
+
if version_id is not None:
|
203
|
+
# Update the templateRef to include version
|
204
|
+
new_name = f"{name}/versions/{version_id}"
|
205
|
+
template_ref["name"] = new_name
|
206
|
+
template_ref["template"] = new_name
|
207
|
+
logger.info(f"Updated templateRef from {name} to {new_name}")
|
208
|
+
elif self.validator.TEMPLATE_REF_WITH_VERSION_PATTERN.match(name):
|
209
|
+
# strip the /versions/{version_id} from the end of name
|
210
|
+
# to get the name like above
|
211
|
+
orig_name = name
|
212
|
+
name = orig_name.rsplit('/versions/', 1)[0]
|
213
|
+
step_name = parts[-3] # Get the step name from the path
|
214
|
+
|
215
|
+
# if it already has a version, make sure it matches the uploaded
|
216
|
+
# version
|
217
|
+
version_id = self.uploaded_step_versions.get(step_name, None)
|
218
|
+
if version_id is not None:
|
219
|
+
# Update the templateRef to include version
|
220
|
+
new_name = f"{name}/versions/{version_id}"
|
221
|
+
template_ref["name"] = new_name
|
222
|
+
template_ref["template"] = new_name
|
223
|
+
logger.info(
|
224
|
+
f"Updated templateRef from {orig_name} to {new_name}"
|
225
|
+
)
|
213
226
|
|
214
227
|
def create_pipeline(self) -> bool:
|
215
228
|
"""Create the pipeline using PostPipelines RPC."""
|
clarifai/runners/server.py
CHANGED
@@ -134,28 +134,35 @@ def serve(
|
|
134
134
|
pat=pat,
|
135
135
|
num_parallel_polls=num_threads,
|
136
136
|
)
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
method_signatures
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
137
|
+
|
138
|
+
if context is None:
|
139
|
+
logger.debug("Context is None. Skipping code snippet generation.")
|
140
|
+
else:
|
141
|
+
method_signatures = builder.get_method_signatures(mocking=False)
|
142
|
+
from clarifai.runners.utils import code_script
|
143
|
+
|
144
|
+
snippet = code_script.generate_client_script(
|
145
|
+
method_signatures,
|
146
|
+
user_id=context.user_id,
|
147
|
+
app_id=context.app_id,
|
148
|
+
model_id=context.model_id,
|
149
|
+
deployment_id=context.deployment_id,
|
150
|
+
base_url=context.api_base,
|
151
|
+
)
|
152
|
+
logger.info(
|
153
|
+
"✅ Your model is running locally and is ready for requests from the API...\n"
|
154
|
+
)
|
155
|
+
logger.info(
|
156
|
+
f"> Code Snippet: To call your model via the API, use this code snippet:\n{snippet}"
|
157
|
+
)
|
158
|
+
logger.info(
|
159
|
+
f"> Playground: To chat with your model, visit: {context.ui}/playground?model={context.model_id}__{context.model_version_id}&user_id={context.user_id}&app_id={context.app_id}\n"
|
160
|
+
)
|
161
|
+
logger.info(
|
162
|
+
f"> API URL: To call your model via the API, use this model URL: {context.ui}/users/{context.user_id}/apps/{context.app_id}/models/{context.model_id}\n"
|
163
|
+
)
|
164
|
+
logger.info("Press CTRL+C to stop the runner.\n")
|
165
|
+
|
159
166
|
runner.start() # start the runner to fetch work from the API.
|
160
167
|
|
161
168
|
|
@@ -112,7 +112,9 @@ print(response)
|
|
112
112
|
deployment_id = None
|
113
113
|
else:
|
114
114
|
deployment_id = (
|
115
|
-
'os.environ
|
115
|
+
'os.environ.get("CLARIFAI_DEPLOYMENT_ID", None)'
|
116
|
+
if not deployment_id
|
117
|
+
else repr(deployment_id)
|
116
118
|
)
|
117
119
|
|
118
120
|
deployment_line = (
|
@@ -11,10 +11,10 @@ class PipelineConfigValidator:
|
|
11
11
|
|
12
12
|
# Regex patterns for templateRef validation
|
13
13
|
TEMPLATE_REF_WITH_VERSION_PATTERN = re.compile(
|
14
|
-
r'^users/([^/]+)/apps/([^/]+)/
|
14
|
+
r'^users/([^/]+)/apps/([^/]+)/pipeline_steps/([^/]+)/versions/([^/]+)$'
|
15
15
|
)
|
16
16
|
TEMPLATE_REF_WITHOUT_VERSION_PATTERN = re.compile(
|
17
|
-
r'^users/([^/]+)/apps/([^/]+)/
|
17
|
+
r'^users/([^/]+)/apps/([^/]+)/pipeline_steps/([^/]+)$'
|
18
18
|
)
|
19
19
|
|
20
20
|
@classmethod
|
@@ -120,8 +120,8 @@ class PipelineConfigValidator:
|
|
120
120
|
):
|
121
121
|
raise ValueError(
|
122
122
|
f"templateRef name '{name}' must match either pattern:\n"
|
123
|
-
f" - users/{{user_id}}/apps/{{app_id}}/
|
124
|
-
f" - users/{{user_id}}/apps/{{app_id}}/
|
123
|
+
f" - users/{{user_id}}/apps/{{app_id}}/pipeline_steps/{{step_id}}\n"
|
124
|
+
f" - users/{{user_id}}/apps/{{app_id}}/pipeline_steps/{{step_id}}/versions/{{version_id}}"
|
125
125
|
)
|
126
126
|
|
127
127
|
@classmethod
|
clarifai/utils/cli.py
CHANGED
@@ -9,6 +9,7 @@ from typing import OrderedDict
|
|
9
9
|
|
10
10
|
import click
|
11
11
|
import yaml
|
12
|
+
from google.protobuf.timestamp_pb2 import Timestamp
|
12
13
|
from tabulate import tabulate
|
13
14
|
|
14
15
|
from clarifai.utils.logging import logger
|
@@ -47,8 +48,24 @@ def display_co_resources(
|
|
47
48
|
'USER_ID': lambda c: c.user_id,
|
48
49
|
'DESCRIPTION': lambda c: c.description,
|
49
50
|
},
|
51
|
+
sort_by_columns=None,
|
50
52
|
):
|
51
|
-
"""
|
53
|
+
"""
|
54
|
+
Display compute orchestration resources listing results using rich.
|
55
|
+
|
56
|
+
:param response: Iterable of resource objects to display.
|
57
|
+
:param custom_columns: A dictionary mapping column names to extractor functions.
|
58
|
+
Defaults to ID, USER_ID, and DESCRIPTION.
|
59
|
+
:param sort_by_columns: Optional list of (column_name, order) tuples specifying sort order.
|
60
|
+
Only column names present in `custom_columns` are considered.
|
61
|
+
Order should be 'asc' or 'desc'.
|
62
|
+
:return: None. Prints the formatted table.
|
63
|
+
"""
|
64
|
+
if sort_by_columns:
|
65
|
+
for column, order in reversed(sort_by_columns): # reversed for stable multi-level sort
|
66
|
+
if column in custom_columns:
|
67
|
+
reverse = order.lower() == 'desc'
|
68
|
+
response = sorted(response, key=custom_columns[column], reverse=reverse)
|
52
69
|
|
53
70
|
formatter = TableFormatter(custom_columns)
|
54
71
|
print(formatter.format(list(response), fmt="plain"))
|
@@ -355,3 +372,20 @@ def check_requirements_installed(model_path):
|
|
355
372
|
except Exception as e:
|
356
373
|
logger.error(f"Failed to check requirements: {e}")
|
357
374
|
return False
|
375
|
+
|
376
|
+
|
377
|
+
def convert_timestamp_to_string(timestamp: Timestamp) -> str:
|
378
|
+
"""Converts a Timestamp object to a string.
|
379
|
+
|
380
|
+
Args:
|
381
|
+
timestamp (Timestamp): The Timestamp object to convert.
|
382
|
+
|
383
|
+
Returns:
|
384
|
+
str: The converted string in ISO 8601 format.
|
385
|
+
"""
|
386
|
+
if not timestamp:
|
387
|
+
return ""
|
388
|
+
|
389
|
+
datetime_obj = timestamp.ToDatetime()
|
390
|
+
|
391
|
+
return datetime_obj.strftime('%Y-%m-%dT%H:%M:%SZ')
|
clarifai/utils/config.py
CHANGED
@@ -68,6 +68,19 @@ class Context(OrderedDict):
|
|
68
68
|
except KeyError as e:
|
69
69
|
raise AttributeError(f"'{type(self).__name__}' object has no attribute '{key}'") from e
|
70
70
|
|
71
|
+
def get(self, key, default=None):
|
72
|
+
"""Get the key from the config. You can pass a lowercase
|
73
|
+
key like "pat" and it will check if the environment variable CLARIFAI_PAT set and use that
|
74
|
+
first.
|
75
|
+
If no env var, then it checks if that env var name is in the config and use that.
|
76
|
+
If not then checks if "pat" is in the config, if not then it falls back to CLARIFAI_PAT in
|
77
|
+
the environment variables, else returns the default value.
|
78
|
+
"""
|
79
|
+
try:
|
80
|
+
return self.__getattr__(key)
|
81
|
+
except AttributeError:
|
82
|
+
return default
|
83
|
+
|
71
84
|
def __hasattr__(self, key):
|
72
85
|
if key == "name":
|
73
86
|
return True
|
@@ -1,23 +1,23 @@
|
|
1
|
-
clarifai/__init__.py,sha256=
|
1
|
+
clarifai/__init__.py,sha256=vHcuCYQBbVvJPdk__Lp8UGY_w9oIIXRPc0xgnIK6beI,23
|
2
2
|
clarifai/cli.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
3
3
|
clarifai/errors.py,sha256=GXa6D4v_L404J83jnRNFPH7s-1V9lk7w6Ws99f1g-AY,2772
|
4
4
|
clarifai/versions.py,sha256=ecSuEB_nOL2XSoYHDw2n23XUbm_KPOGjudMXmQrGdS8,224
|
5
|
-
clarifai/cli/README.md,sha256=
|
5
|
+
clarifai/cli/README.md,sha256=Rm0Sk61u7PrWQBgA6idv0R6ldJGhnVHSlIF2rVzRk8g,3428
|
6
6
|
clarifai/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
7
7
|
clarifai/cli/__main__.py,sha256=7nPbLW7Jr2shkgMPvnxpn4xYGMvIcnqluJ69t9w4H_k,74
|
8
|
-
clarifai/cli/base.py,sha256=
|
8
|
+
clarifai/cli/base.py,sha256=_nXkRu3F8s1SL1LspxUWo5MCbdq7e6gI-iYb3B0-2Oc,9605
|
9
9
|
clarifai/cli/compute_cluster.py,sha256=8Xss0Obrp6l1XuxJe0luOqU_pf8vXGDRi6jyIe8qR6k,2282
|
10
10
|
clarifai/cli/deployment.py,sha256=9C4I6_kyMxRkWl6h681wc79-3mAtDHtTUaxRv05OZMs,4262
|
11
11
|
clarifai/cli/model.py,sha256=r_-loAszX8vOYlW1YI7TgNA4CFPZy4dicqtywoZ39R8,43546
|
12
12
|
clarifai/cli/nodepool.py,sha256=H6OIdUW_EiyDUwZogzEDoYmVwEjLMsgoDlPyE7gjIuU,4245
|
13
|
-
clarifai/cli/pipeline.py,sha256=
|
14
|
-
clarifai/cli/pipeline_step.py,sha256=
|
13
|
+
clarifai/cli/pipeline.py,sha256=9ZWVLokXHxqXy0g1_1ACuRX20ckrzyKwnsFfuJZm6Nw,13130
|
14
|
+
clarifai/cli/pipeline_step.py,sha256=dvoC2vAsDcxOCy88VV0X42PG22_7JSu9sfBVsk-Cix4,6133
|
15
15
|
clarifai/cli/templates/__init__.py,sha256=HbMlZuYOMyVJde73ijNAevmSRUpIttGlHdwyO4W-JOs,44
|
16
16
|
clarifai/cli/templates/model_templates.py,sha256=q1dKF5jHhX_1g4qtiYcEbJdWQaxCv2BSEAXI4AR2JOo,9709
|
17
|
-
clarifai/cli/templates/pipeline_step_templates.py,sha256=
|
18
|
-
clarifai/cli/templates/pipeline_templates.py,sha256=
|
19
|
-
clarifai/client/__init__.py,sha256=
|
20
|
-
clarifai/client/app.py,sha256=
|
17
|
+
clarifai/cli/templates/pipeline_step_templates.py,sha256=w1IJghF_4wWyEmHR1925N0hpGKocy3G7ezhxTH-0XCc,1716
|
18
|
+
clarifai/cli/templates/pipeline_templates.py,sha256=l7-0yj1aw4jVZb0xjuhc9sW9LJfG3qaGqgwt7bLURCc,4462
|
19
|
+
clarifai/client/__init__.py,sha256=MWEG_jTGn6UWbGCznsZxObJ5h65k2igD1462qID2pgI,840
|
20
|
+
clarifai/client/app.py,sha256=8_Y5esptlFSGW2WS0uly9IBop37hJVpxnhs-vBIZrtc,47205
|
21
21
|
clarifai/client/base.py,sha256=aduMVVSPUSQ-MsDpj4yuHmCTUNbCTPt76dhRZ14euVg,10284
|
22
22
|
clarifai/client/compute_cluster.py,sha256=q-JuBai4pKpSeDAn-P61IKyGjizX-IV-O5huWAYx5DU,10279
|
23
23
|
clarifai/client/dataset.py,sha256=OgdpZkQ_vYmRxL8-qphcNozpvPV1bWTlte9Jv6UkKb8,35299
|
@@ -27,11 +27,12 @@ clarifai/client/lister.py,sha256=1YEm2suNxPaJO4x9V5szgD_YX6N_00vgSO-7m0HagY8,220
|
|
27
27
|
clarifai/client/model.py,sha256=DeB-es6rp0FJQVY46oO2eR_-Ux5tTmkM_qYsN1X6WcE,90793
|
28
28
|
clarifai/client/model_client.py,sha256=pyPg1C3gb5Q3_DXmMmOB5AeORR1IDaaM7n0LGc3ciLk,38217
|
29
29
|
clarifai/client/module.py,sha256=jLViQYvVV3FmRN_ivvbk83uwsx7CgYGeEx4dYAr6yD4,4537
|
30
|
-
clarifai/client/nodepool.py,sha256=
|
31
|
-
clarifai/client/pipeline.py,sha256=
|
30
|
+
clarifai/client/nodepool.py,sha256=0E8-VgNMpdx2sAGotS3ZwfeUc9k3DBa3QJEJ1wKroVA,16700
|
31
|
+
clarifai/client/pipeline.py,sha256=Kawnp16iWwty4TZGDbZvU_G5PJzCXQqf50Dc-AYLaAI,14880
|
32
|
+
clarifai/client/pipeline_step.py,sha256=_R0PhVCspAQatCss-0O0O9p9CpvU_y9cSSEeUp_7cV0,2765
|
32
33
|
clarifai/client/runner.py,sha256=5xCiqByGGscfNm0IjHelhDTx8-9l8G0C3HL-3YZogK8,2253
|
33
34
|
clarifai/client/search.py,sha256=3LLfATrdU43a0mRNITmJV-E53bhfafZkYsbwkTtlnyU,15661
|
34
|
-
clarifai/client/user.py,sha256
|
35
|
+
clarifai/client/user.py,sha256=-D7itPC_XCMtFnsWUOMCPU5PQd8s-ER8k-RIG2V673Y,24798
|
35
36
|
clarifai/client/workflow.py,sha256=Bqh8lAmJcSbviJebckchTxucYlU11UQEhFSov7elpsk,13417
|
36
37
|
clarifai/client/auth/__init__.py,sha256=7EwR0NrozkAUwpUnCsqXvE_p0wqx_SelXlSpKShKJK0,136
|
37
38
|
clarifai/client/auth/helper.py,sha256=eitq1uQBoJ1TPjsTtbJ9g7eqT3mFm9vgc4HQR1s-zNg,16283
|
@@ -71,7 +72,7 @@ clarifai/rag/__init__.py,sha256=wu3PzAzo7uqgrEzuaC9lY_3gj1HFiR3GU3elZIKTT5g,40
|
|
71
72
|
clarifai/rag/rag.py,sha256=EG3GoFrHFCmA70Tz49_0Jo1-3WIaHSgWGHecPeErcdc,14170
|
72
73
|
clarifai/rag/utils.py,sha256=_gVZdABuMnraCKViLruV75x0F3IpgFXN6amYSGE5_xc,4462
|
73
74
|
clarifai/runners/__init__.py,sha256=wXLaSljH7qLeJCrZdKEnlQh2tNqTQAIZKWOu2rZ6wGs,279
|
74
|
-
clarifai/runners/server.py,sha256=
|
75
|
+
clarifai/runners/server.py,sha256=5xX0Pj_XCKRyEfF4v8DQqvdj94DMWY75sxTetdM9dKg,6253
|
75
76
|
clarifai/runners/dockerfile_template/Dockerfile.template,sha256=nEnIMqzhAXDbd0Ht7QQm0U7AVPIHWQD6kYnFZ0TKJUM,2428
|
76
77
|
clarifai/runners/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
77
78
|
clarifai/runners/models/dummy_openai_model.py,sha256=pcmAVbqTTGG4J3BLVjKfvM_SQ-GET_XexIUdLcr9Zvo,8373
|
@@ -87,16 +88,16 @@ clarifai/runners/models/visual_detector_class.py,sha256=ky4oFAkGCKPpGPdgaOso-n6D
|
|
87
88
|
clarifai/runners/pipeline_steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
88
89
|
clarifai/runners/pipeline_steps/pipeline_step_builder.py,sha256=X2xWg4QM4kX3cr8JkG1RmAiVE8-XH8XbomjmXa16zmY,21333
|
89
90
|
clarifai/runners/pipelines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
90
|
-
clarifai/runners/pipelines/pipeline_builder.py,sha256=
|
91
|
+
clarifai/runners/pipelines/pipeline_builder.py,sha256=0FBjb8l7mWlCwBsBLkHM3znNQB9HPLEOYrrE53ntjCE,13810
|
91
92
|
clarifai/runners/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
92
|
-
clarifai/runners/utils/code_script.py,sha256=
|
93
|
+
clarifai/runners/utils/code_script.py,sha256=zZ5xjd13z2oV4CUWgMU0w6J6dRr01Kl2K-8pPyw0MiY,14346
|
93
94
|
clarifai/runners/utils/const.py,sha256=MK7lTzzJKbOiyiUtG_jlJXfz_xNKMn5LjkQ9vjbttXE,1538
|
94
95
|
clarifai/runners/utils/data_utils.py,sha256=HRpMYR2O0OiDpXXhOManLHTeomC4bFnXMHVAiT_12yE,20856
|
95
96
|
clarifai/runners/utils/loader.py,sha256=K5Y8MPbIe5STw2gDnrL8KqFgKNxEo7bz-RV0ip1T4PM,10900
|
96
97
|
clarifai/runners/utils/method_signatures.py,sha256=qdHaO8ZIgP6BBXXMhMPhcQ46dse-XMP2t4VJCNG7O3Q,18335
|
97
98
|
clarifai/runners/utils/model_utils.py,sha256=MCYhV_00anptIO1Qtxgzn3DlGI2UHV0ESFT3wLNcAMM,6413
|
98
99
|
clarifai/runners/utils/openai_convertor.py,sha256=ZlIrvvfHttD_DavLvmKZdL8gNq_TQvQtZVnYamwdWz4,8248
|
99
|
-
clarifai/runners/utils/pipeline_validation.py,sha256=
|
100
|
+
clarifai/runners/utils/pipeline_validation.py,sha256=GFgFbrlS0UvGXcYTS84CyX7hIVU2CUNVQiMnCAjNfhw,6444
|
100
101
|
clarifai/runners/utils/serializers.py,sha256=pI7GqMTC0T3Lu_X8v8TO4RiplO-gC_49Ns37jYwsPtg,7908
|
101
102
|
clarifai/runners/utils/url_fetcher.py,sha256=pGkrpj692FEvMsV6pNbmidfYKzQV7APzjja_fwd1Aig,4792
|
102
103
|
clarifai/runners/utils/data_types/__init__.py,sha256=iBtmPkIoLK9ew6ZiXElGt2YBBTDLEA0fmxE_eOYzvhk,478
|
@@ -104,8 +105,8 @@ clarifai/runners/utils/data_types/data_types.py,sha256=UBHTNPshr94qUs2KqkYis0VlA
|
|
104
105
|
clarifai/schema/search.py,sha256=o9-ct8ulLZByB3RCVwZWPgaDwdcW7cM5s-g8oyAz89s,2421
|
105
106
|
clarifai/urls/helper.py,sha256=z6LnLGgLHxD8scFtyRdxqYIRJNhxqPkfLe53UtTLUBY,11727
|
106
107
|
clarifai/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
107
|
-
clarifai/utils/cli.py,sha256=
|
108
|
-
clarifai/utils/config.py,sha256=
|
108
|
+
clarifai/utils/cli.py,sha256=xTenEXjhWV-lpTqlfeRO4WNakT5QszNi8GO4yH4qcMs,13184
|
109
|
+
clarifai/utils/config.py,sha256=dENYtcWW7Il5MInvIaYe0MZn0hW1fbIb0Lzk8rQ_geQ,7671
|
109
110
|
clarifai/utils/constants.py,sha256=klcdSiVccrdJVRXL9qq08p2lWL3KerXcQOTIG6y9298,2414
|
110
111
|
clarifai/utils/logging.py,sha256=0we53uTqUvzrulC86whu-oeWNxn1JjJL0OQ98Bwf9vo,15198
|
111
112
|
clarifai/utils/misc.py,sha256=ld7_lls4CrjHzKchsSgJs4T-rVpUVEzRy4MDWBEQlyA,22812
|
@@ -119,9 +120,9 @@ clarifai/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
|
|
119
120
|
clarifai/workflows/export.py,sha256=HvUYG9N_-UZoRR0-_tdGbZ950_AeBqawSppgUxQebR0,1913
|
120
121
|
clarifai/workflows/utils.py,sha256=ESL3INcouNcLKCh-nMpfXX-YbtCzX7tz7hT57_RGQ3M,2079
|
121
122
|
clarifai/workflows/validate.py,sha256=UhmukyHkfxiMFrPPeBdUTiCOHQT5-shqivlBYEyKTlU,2931
|
122
|
-
clarifai-11.6.
|
123
|
-
clarifai-11.6.
|
124
|
-
clarifai-11.6.
|
125
|
-
clarifai-11.6.
|
126
|
-
clarifai-11.6.
|
127
|
-
clarifai-11.6.
|
123
|
+
clarifai-11.6.7.dist-info/licenses/LICENSE,sha256=mUqF_d12-qE2n41g7C5_sq-BMLOcj6CNN-jevr15YHU,555
|
124
|
+
clarifai-11.6.7.dist-info/METADATA,sha256=uwUjxG1M12d67EI6GLAU9jly-li-CQr0SJcyBNoK0hc,22737
|
125
|
+
clarifai-11.6.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
126
|
+
clarifai-11.6.7.dist-info/entry_points.txt,sha256=X9FZ4Z-i_r2Ud1RpZ9sNIFYuu_-9fogzCMCRUD9hyX0,51
|
127
|
+
clarifai-11.6.7.dist-info/top_level.txt,sha256=wUMdCQGjkxaynZ6nZ9FAnvBUCgp5RJUVFSy2j-KYo0s,9
|
128
|
+
clarifai-11.6.7.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|