clarifai 11.6.6__py3-none-any.whl → 11.6.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clarifai/__init__.py +1 -1
- clarifai/cli/README.md +51 -0
- clarifai/cli/base.py +42 -16
- clarifai/cli/model.py +5 -25
- clarifai/cli/pipeline.py +70 -1
- clarifai/cli/pipeline_step.py +67 -1
- clarifai/cli/templates/pipeline_step_templates.py +3 -3
- clarifai/cli/templates/pipeline_templates.py +7 -7
- clarifai/client/__init__.py +2 -0
- clarifai/client/app.py +147 -0
- clarifai/client/nodepool.py +7 -2
- clarifai/client/pipeline.py +31 -27
- clarifai/client/pipeline_step.py +72 -0
- clarifai/client/user.py +74 -0
- clarifai/runners/pipelines/pipeline_builder.py +29 -16
- clarifai/runners/server.py +28 -22
- clarifai/runners/utils/code_script.py +3 -1
- clarifai/runners/utils/pipeline_validation.py +4 -4
- clarifai/utils/cli.py +35 -1
- clarifai/utils/config.py +13 -0
- {clarifai-11.6.6.dist-info → clarifai-11.6.8.dist-info}/METADATA +1 -1
- {clarifai-11.6.6.dist-info → clarifai-11.6.8.dist-info}/RECORD +26 -25
- {clarifai-11.6.6.dist-info → clarifai-11.6.8.dist-info}/WHEEL +0 -0
- {clarifai-11.6.6.dist-info → clarifai-11.6.8.dist-info}/entry_points.txt +0 -0
- {clarifai-11.6.6.dist-info → clarifai-11.6.8.dist-info}/licenses/LICENSE +0 -0
- {clarifai-11.6.6.dist-info → clarifai-11.6.8.dist-info}/top_level.txt +0 -0
clarifai/__init__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "11.6.
|
1
|
+
__version__ = "11.6.8"
|
clarifai/cli/README.md
CHANGED
@@ -121,6 +121,57 @@ Delete existing Compute Clusters:
|
|
121
121
|
$ clarifai computecluster delete --compute_cluster_id <compute-cluster-id>
|
122
122
|
```
|
123
123
|
|
124
|
+
## Pipelines
|
125
|
+
|
126
|
+
### List Pipelines
|
127
|
+
|
128
|
+
List all pipelines for the user across all apps:
|
129
|
+
|
130
|
+
```bash
|
131
|
+
$ clarifai pipeline list
|
132
|
+
```
|
133
|
+
|
134
|
+
List pipelines within a specific app:
|
135
|
+
|
136
|
+
```bash
|
137
|
+
$ clarifai pipeline list --app_id <app-id>
|
138
|
+
```
|
139
|
+
|
140
|
+
List with pagination:
|
141
|
+
|
142
|
+
```bash
|
143
|
+
$ clarifai pipeline list --page_no 1 --per_page 10
|
144
|
+
```
|
145
|
+
|
146
|
+
### List Pipeline Steps
|
147
|
+
|
148
|
+
List all pipeline steps for the user across all apps:
|
149
|
+
|
150
|
+
```bash
|
151
|
+
$ clarifai pipelinestep list
|
152
|
+
```
|
153
|
+
|
154
|
+
List pipeline steps within a specific app:
|
155
|
+
|
156
|
+
```bash
|
157
|
+
$ clarifai pipelinestep list --app_id <app-id>
|
158
|
+
```
|
159
|
+
|
160
|
+
List pipeline steps for a specific pipeline:
|
161
|
+
|
162
|
+
```bash
|
163
|
+
$ clarifai pipelinestep list --app_id <app-id> --pipeline_id <pipeline-id>
|
164
|
+
```
|
165
|
+
|
166
|
+
### Aliases
|
167
|
+
|
168
|
+
Both commands support the `ls` alias for convenience:
|
169
|
+
|
170
|
+
```bash
|
171
|
+
$ clarifai pipeline ls
|
172
|
+
$ clarifai pipelinestep ls
|
173
|
+
```
|
174
|
+
|
124
175
|
## Learn More
|
125
176
|
|
126
177
|
* [Example Configs](https://github.com/Clarifai/examples/tree/main/ComputeOrchestration/configs)
|
clarifai/cli/base.py
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
import getpass
|
2
1
|
import json
|
3
2
|
import os
|
4
3
|
import sys
|
@@ -72,18 +71,30 @@ def login(ctx, api_url, user_id):
|
|
72
71
|
"""Login command to set PAT and other configurations."""
|
73
72
|
from clarifai.utils.cli import validate_context_auth
|
74
73
|
|
74
|
+
# Input user_id if not supplied
|
75
|
+
if not user_id:
|
76
|
+
user_id = click.prompt('Enter your Clarifai user ID', type=str)
|
77
|
+
|
75
78
|
click.echo('> To authenticate, you\'ll need a Personal Access Token (PAT).')
|
76
79
|
click.echo(
|
77
|
-
'> You can create one from your account settings: https://clarifai.com/settings/security\n'
|
80
|
+
f'> You can create one from your account settings: https://clarifai.com/{user_id}/settings/security\n'
|
78
81
|
)
|
79
82
|
|
80
83
|
# Securely input PAT
|
81
|
-
pat =
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
84
|
+
pat = input_or_default(
|
85
|
+
'Enter your Personal Access Token (PAT) value (or type "ENVVAR" to use an environment variable): ',
|
86
|
+
'ENVVAR',
|
87
|
+
)
|
88
|
+
if pat.lower() == 'envvar':
|
89
|
+
pat = os.environ.get('CLARIFAI_PAT')
|
90
|
+
if not pat:
|
91
|
+
logger.error(
|
92
|
+
'Environment variable "CLARIFAI_PAT" not set. Please set it in your terminal.'
|
93
|
+
)
|
94
|
+
click.echo(
|
95
|
+
'Aborting login. Please set the environment variable or provide a PAT value and try again.'
|
96
|
+
)
|
97
|
+
click.abort()
|
87
98
|
# Progress indicator
|
88
99
|
click.echo('\n> Verifying token...')
|
89
100
|
validate_context_auth(pat, user_id, api_url)
|
@@ -122,7 +133,7 @@ def input_or_default(prompt, default):
|
|
122
133
|
|
123
134
|
|
124
135
|
# Context management commands under config group
|
125
|
-
@config.command(aliases=['get-contexts', 'list-contexts'])
|
136
|
+
@config.command(aliases=['get-contexts', 'list-contexts', 'ls'])
|
126
137
|
@click.option(
|
127
138
|
'-o', '--output-format', default='wide', type=click.Choice(['wide', 'name', 'json', 'yaml'])
|
128
139
|
)
|
@@ -152,16 +163,21 @@ def get_contexts(ctx, output_format):
|
|
152
163
|
elif output_format == 'name':
|
153
164
|
print('\n'.join(ctx.obj.contexts))
|
154
165
|
elif output_format in ('json', 'yaml'):
|
155
|
-
dicts = [
|
156
|
-
for
|
157
|
-
|
166
|
+
dicts = []
|
167
|
+
for c, v in ctx.obj.contexts.items():
|
168
|
+
context_dict = {}
|
169
|
+
d = v.to_serializable_dict()
|
170
|
+
d.pop('CLARIFAI_PAT', None)
|
171
|
+
context_dict['name'] = c
|
172
|
+
context_dict['env'] = d
|
173
|
+
dicts.append(context_dict)
|
158
174
|
if output_format == 'json':
|
159
175
|
print(json.dumps(dicts))
|
160
176
|
elif output_format == 'yaml':
|
161
177
|
print(yaml.safe_dump(dicts))
|
162
178
|
|
163
179
|
|
164
|
-
@config.command(aliases=['use-context'])
|
180
|
+
@config.command(aliases=['use-context', 'use'])
|
165
181
|
@click.argument('name', type=str)
|
166
182
|
@click.pass_context
|
167
183
|
def use_context(ctx, name):
|
@@ -173,7 +189,7 @@ def use_context(ctx, name):
|
|
173
189
|
print(f'Set {name} as the current context')
|
174
190
|
|
175
191
|
|
176
|
-
@config.command(aliases=['current-context'])
|
192
|
+
@config.command(aliases=['current-context', 'current'])
|
177
193
|
@click.option('-o', '--output-format', default='name', type=click.Choice(['name', 'json', 'yaml']))
|
178
194
|
@click.pass_context
|
179
195
|
def current_context(ctx, output_format):
|
@@ -186,7 +202,7 @@ def current_context(ctx, output_format):
|
|
186
202
|
print(yaml.safe_dump(ctx.obj.contexts[ctx.obj.current_context].to_serializable_dict()))
|
187
203
|
|
188
204
|
|
189
|
-
@config.command(aliases=['create-context', '
|
205
|
+
@config.command(aliases=['create-context', 'create'])
|
190
206
|
@click.argument('name')
|
191
207
|
@click.option('--user-id', required=False, help='User ID')
|
192
208
|
@click.option('--base-url', required=False, help='Base URL')
|
@@ -216,6 +232,16 @@ def create_context(
|
|
216
232
|
'personal access token value (default: "ENVVAR" to get our of env var rather than config): ',
|
217
233
|
'ENVVAR',
|
218
234
|
)
|
235
|
+
if pat.lower() == 'envvar':
|
236
|
+
pat = os.environ.get('CLARIFAI_PAT')
|
237
|
+
if not pat:
|
238
|
+
logger.error(
|
239
|
+
'Environment variable "CLARIFAI_PAT" not set. Please set it in your terminal.'
|
240
|
+
)
|
241
|
+
click.echo(
|
242
|
+
'Aborting context creation. Please set the environment variable or provide a PAT value and try again.'
|
243
|
+
)
|
244
|
+
click.abort()
|
219
245
|
validate_context_auth(pat, user_id, base_url)
|
220
246
|
context = Context(name, CLARIFAI_USER_ID=user_id, CLARIFAI_API_BASE=base_url, CLARIFAI_PAT=pat)
|
221
247
|
ctx.obj.contexts[context.name] = context
|
@@ -233,7 +259,7 @@ def edit(
|
|
233
259
|
os.system(f'{os.environ.get("EDITOR", "vi")} {ctx.obj.filename}')
|
234
260
|
|
235
261
|
|
236
|
-
@config.command(aliases=['delete-context'])
|
262
|
+
@config.command(aliases=['delete-context', 'delete'])
|
237
263
|
@click.argument('name')
|
238
264
|
@click.pass_context
|
239
265
|
def delete_context(ctx, name):
|
clarifai/cli/model.py
CHANGED
@@ -728,7 +728,11 @@ def local_runner(ctx, model_path, pool_size, verbose):
|
|
728
728
|
model_versions = [v for v in model.list_versions()]
|
729
729
|
if len(model_versions) == 0:
|
730
730
|
logger.warning("No model versions found. Creating a new version for local runner.")
|
731
|
-
|
731
|
+
# add the signatures for local runner on how to call it.
|
732
|
+
signatures = builder.get_method_signatures(mocking=True)
|
733
|
+
version = model.create_version(
|
734
|
+
pretrained_model_config={"local_dev": True}, method_signatures=signatures
|
735
|
+
).model_version
|
732
736
|
ctx.obj.current.CLARIFAI_MODEL_VERSION_ID = version.id
|
733
737
|
ctx.obj.to_yaml()
|
734
738
|
else:
|
@@ -883,30 +887,6 @@ def local_runner(ctx, model_path, pool_size, verbose):
|
|
883
887
|
logger.error(f"Failed to customize Ollama model: {e}")
|
884
888
|
raise click.Abort()
|
885
889
|
|
886
|
-
# don't mock for local runner since you need the dependencies to run the code anyways.
|
887
|
-
method_signatures = builder.get_method_signatures(mocking=False)
|
888
|
-
|
889
|
-
from clarifai.runners.utils import code_script
|
890
|
-
|
891
|
-
snippet = code_script.generate_client_script(
|
892
|
-
method_signatures,
|
893
|
-
user_id=user_id,
|
894
|
-
app_id=app_id,
|
895
|
-
model_id=model_id,
|
896
|
-
deployment_id=deployment_id,
|
897
|
-
base_url=ctx.obj.current.api_base,
|
898
|
-
)
|
899
|
-
|
900
|
-
logger.info(f"""\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
901
|
-
# About to start up the local runner in this terminal...
|
902
|
-
# Here is a code snippet to call this model once it start from another terminal:{snippet}
|
903
|
-
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
904
|
-
""")
|
905
|
-
|
906
|
-
logger.info(
|
907
|
-
f"Playground: To chat with your model, visit:\n{ctx.obj.current.ui}/playground?model={model.id}__{version.id}&user_id={user_id}&app_id={app_id}"
|
908
|
-
)
|
909
|
-
|
910
890
|
logger.info("✅ Starting local runner...")
|
911
891
|
|
912
892
|
# This reads the config.yaml from the model_path so we alter it above first.
|
clarifai/cli/pipeline.py
CHANGED
@@ -4,15 +4,24 @@ import shutil
|
|
4
4
|
import click
|
5
5
|
|
6
6
|
from clarifai.cli.base import cli
|
7
|
+
from clarifai.client.app import App
|
8
|
+
from clarifai.client.user import User
|
9
|
+
from clarifai.utils.cli import (
|
10
|
+
AliasedGroup,
|
11
|
+
convert_timestamp_to_string,
|
12
|
+
display_co_resources,
|
13
|
+
validate_context,
|
14
|
+
)
|
7
15
|
from clarifai.utils.logging import logger
|
8
16
|
|
9
17
|
|
10
18
|
@cli.group(
|
11
19
|
['pipeline', 'pl'],
|
20
|
+
cls=AliasedGroup,
|
12
21
|
context_settings={'max_content_width': shutil.get_terminal_size().columns - 10},
|
13
22
|
)
|
14
23
|
def pipeline():
|
15
|
-
"""Manage pipelines: upload, init, etc"""
|
24
|
+
"""Manage pipelines: upload, init, list, etc"""
|
16
25
|
|
17
26
|
|
18
27
|
@pipeline.command()
|
@@ -113,6 +122,19 @@ def run(
|
|
113
122
|
monitor_interval = config_data.get('monitor_interval', monitor_interval)
|
114
123
|
log_file = config_data.get('log_file', log_file)
|
115
124
|
monitor = config_data.get('monitor', monitor)
|
125
|
+
elif ctx.obj.current:
|
126
|
+
if not user_id:
|
127
|
+
user_id = ctx.obj.current.get('user_id', '')
|
128
|
+
if not app_id:
|
129
|
+
app_id = ctx.obj.current.get('app_id', '')
|
130
|
+
if not pipeline_id:
|
131
|
+
pipeline_id = ctx.obj.current.get('pipeline_id', '')
|
132
|
+
if not pipeline_version_id:
|
133
|
+
pipeline_version_id = ctx.obj.current.get('pipeline_version_id', '')
|
134
|
+
if not nodepool_id:
|
135
|
+
nodepool_id = ctx.obj.current.get('nodepool_id', '')
|
136
|
+
if not compute_cluster_id:
|
137
|
+
compute_cluster_id = ctx.obj.current.get('compute_cluster_id', '')
|
116
138
|
|
117
139
|
# compute_cluster_id and nodepool_id are mandatory regardless of whether pipeline_url is provided
|
118
140
|
if not compute_cluster_id or not nodepool_id:
|
@@ -275,3 +297,50 @@ def init(pipeline_path):
|
|
275
297
|
)
|
276
298
|
logger.info("4. Add dependencies to requirements.txt files as needed")
|
277
299
|
logger.info("5. Run 'clarifai pipeline upload config.yaml' to upload your pipeline")
|
300
|
+
|
301
|
+
|
302
|
+
@pipeline.command(['ls'])
|
303
|
+
@click.option('--page_no', required=False, help='Page number to list.', default=1)
|
304
|
+
@click.option('--per_page', required=False, help='Number of items per page.', default=16)
|
305
|
+
@click.option(
|
306
|
+
'--app_id',
|
307
|
+
required=False,
|
308
|
+
help='App ID to list pipelines from. If not provided, lists across all apps.',
|
309
|
+
)
|
310
|
+
@click.pass_context
|
311
|
+
def list(ctx, page_no, per_page, app_id):
|
312
|
+
"""List all pipelines for the user."""
|
313
|
+
validate_context(ctx)
|
314
|
+
|
315
|
+
if app_id:
|
316
|
+
app = App(
|
317
|
+
app_id=app_id,
|
318
|
+
user_id=ctx.obj.current.user_id,
|
319
|
+
pat=ctx.obj.current.pat,
|
320
|
+
base_url=ctx.obj.current.api_base,
|
321
|
+
)
|
322
|
+
response = app.list_pipelines(page_no=page_no, per_page=per_page)
|
323
|
+
else:
|
324
|
+
user = User(
|
325
|
+
user_id=ctx.obj.current.user_id,
|
326
|
+
pat=ctx.obj.current.pat,
|
327
|
+
base_url=ctx.obj.current.api_base,
|
328
|
+
)
|
329
|
+
response = user.list_pipelines(page_no=page_no, per_page=per_page)
|
330
|
+
|
331
|
+
display_co_resources(
|
332
|
+
response,
|
333
|
+
custom_columns={
|
334
|
+
'ID': lambda p: getattr(p, 'pipeline_id', ''),
|
335
|
+
'USER_ID': lambda p: getattr(p, 'user_id', ''),
|
336
|
+
'APP_ID': lambda p: getattr(p, 'app_id', ''),
|
337
|
+
'VERSION_ID': lambda p: getattr(p, 'pipeline_version_id', ''),
|
338
|
+
'DESCRIPTION': lambda p: getattr(p, 'description', ''),
|
339
|
+
'CREATED_AT': lambda ps: convert_timestamp_to_string(getattr(ps, 'created_at', '')),
|
340
|
+
'MODIFIED_AT': lambda ps: convert_timestamp_to_string(getattr(ps, 'modified_at', '')),
|
341
|
+
},
|
342
|
+
sort_by_columns=[
|
343
|
+
('CREATED_AT', 'desc'),
|
344
|
+
('ID', 'asc'),
|
345
|
+
],
|
346
|
+
)
|
clarifai/cli/pipeline_step.py
CHANGED
@@ -4,15 +4,24 @@ import shutil
|
|
4
4
|
import click
|
5
5
|
|
6
6
|
from clarifai.cli.base import cli
|
7
|
+
from clarifai.client.app import App
|
8
|
+
from clarifai.client.user import User
|
9
|
+
from clarifai.utils.cli import (
|
10
|
+
AliasedGroup,
|
11
|
+
convert_timestamp_to_string,
|
12
|
+
display_co_resources,
|
13
|
+
validate_context,
|
14
|
+
)
|
7
15
|
from clarifai.utils.logging import logger
|
8
16
|
|
9
17
|
|
10
18
|
@cli.group(
|
11
19
|
['pipelinestep', 'ps'],
|
20
|
+
cls=AliasedGroup,
|
12
21
|
context_settings={'max_content_width': shutil.get_terminal_size().columns - 10},
|
13
22
|
)
|
14
23
|
def pipeline_step():
|
15
|
-
"""Manage pipeline steps: upload, test, etc"""
|
24
|
+
"""Manage pipeline steps: upload, test, list, etc"""
|
16
25
|
|
17
26
|
|
18
27
|
@pipeline_step.command()
|
@@ -102,3 +111,60 @@ def init(pipeline_step_path):
|
|
102
111
|
logger.info("2. Update the pipeline step configuration in config.yaml")
|
103
112
|
logger.info("3. Add your pipeline step dependencies to requirements.txt")
|
104
113
|
logger.info("4. Implement your pipeline step logic in 1/pipeline_step.py")
|
114
|
+
|
115
|
+
|
116
|
+
@pipeline_step.command(['ls'])
|
117
|
+
@click.option('--page_no', required=False, help='Page number to list.', default=1)
|
118
|
+
@click.option('--per_page', required=False, help='Number of items per page.', default=16)
|
119
|
+
@click.option(
|
120
|
+
'--app_id',
|
121
|
+
required=False,
|
122
|
+
help='App ID to list pipeline steps from. If not provided, lists across all apps.',
|
123
|
+
)
|
124
|
+
@click.option(
|
125
|
+
'--pipeline_id',
|
126
|
+
required=False,
|
127
|
+
help='Pipeline ID to list pipeline steps from. Must be used with --app_id.',
|
128
|
+
)
|
129
|
+
@click.pass_context
|
130
|
+
def list(ctx, page_no, per_page, app_id, pipeline_id):
|
131
|
+
"""List all pipeline steps for the user."""
|
132
|
+
validate_context(ctx)
|
133
|
+
|
134
|
+
if pipeline_id and not app_id:
|
135
|
+
raise click.UsageError("--pipeline_id must be used together with --app_id")
|
136
|
+
|
137
|
+
if app_id:
|
138
|
+
app = App(
|
139
|
+
app_id=app_id,
|
140
|
+
user_id=ctx.obj.current.user_id,
|
141
|
+
pat=ctx.obj.current.pat,
|
142
|
+
base_url=ctx.obj.current.api_base,
|
143
|
+
)
|
144
|
+
response = app.list_pipeline_steps(
|
145
|
+
pipeline_id=pipeline_id, page_no=page_no, per_page=per_page
|
146
|
+
)
|
147
|
+
else:
|
148
|
+
user = User(
|
149
|
+
user_id=ctx.obj.current.user_id,
|
150
|
+
pat=ctx.obj.current.pat,
|
151
|
+
base_url=ctx.obj.current.api_base,
|
152
|
+
)
|
153
|
+
response = user.list_pipeline_steps(page_no=page_no, per_page=per_page)
|
154
|
+
|
155
|
+
display_co_resources(
|
156
|
+
response,
|
157
|
+
custom_columns={
|
158
|
+
'PIPELINE_STEP_ID': lambda ps: getattr(ps, 'pipeline_step_id', ''),
|
159
|
+
'USER_ID': lambda ps: getattr(ps, 'user_id', ''),
|
160
|
+
'APP_ID': lambda ps: getattr(ps, 'app_id', ''),
|
161
|
+
'VERSION_ID': lambda ps: getattr(ps, 'pipeline_step_version_id', ''),
|
162
|
+
'DESCRIPTION': lambda ps: getattr(ps, 'description', ''),
|
163
|
+
'CREATED_AT': lambda ps: convert_timestamp_to_string(getattr(ps, 'created_at', '')),
|
164
|
+
'MODIFIED_AT': lambda ps: convert_timestamp_to_string(getattr(ps, 'modified_at', '')),
|
165
|
+
},
|
166
|
+
sort_by_columns=[
|
167
|
+
('CREATED_AT', 'desc'),
|
168
|
+
('PIPELINE_STEP_ID', 'asc'),
|
169
|
+
],
|
170
|
+
)
|
@@ -35,7 +35,7 @@ def get_pipeline_step_template():
|
|
35
35
|
return '''import argparse
|
36
36
|
|
37
37
|
import clarifai
|
38
|
-
|
38
|
+
from clarifai.utils.logging import logger
|
39
39
|
|
40
40
|
def main():
|
41
41
|
parser = argparse.ArgumentParser(description='Concatenate two strings.')
|
@@ -44,9 +44,9 @@ def main():
|
|
44
44
|
|
45
45
|
args = parser.parse_args()
|
46
46
|
|
47
|
-
|
47
|
+
logger.info(clarifai.__version__)
|
48
48
|
|
49
|
-
|
49
|
+
logger.info(f"Concatenation Output: {args.param_a + args.param_b}")
|
50
50
|
|
51
51
|
|
52
52
|
if __name__ == "__main__":
|
@@ -25,12 +25,12 @@ def get_pipeline_config_template():
|
|
25
25
|
steps:
|
26
26
|
- - name: step-a
|
27
27
|
templateRef:
|
28
|
-
name: users/your_user_id/apps/your_app_id/
|
29
|
-
template: users/your_user_id/apps/your_app_id/
|
28
|
+
name: users/your_user_id/apps/your_app_id/pipeline_steps/stepA # TODO: please fill in
|
29
|
+
template: users/your_user_id/apps/your_app_id/pipeline_steps/stepA # TODO: please fill in
|
30
30
|
- - name: step-b
|
31
31
|
templateRef:
|
32
|
-
name: users/your_user_id/apps/your_app_id/
|
33
|
-
template: users/your_user_id/apps/your_app_id/
|
32
|
+
name: users/your_user_id/apps/your_app_id/pipeline_steps/stepB # TODO: please fill in
|
33
|
+
template: users/your_user_id/apps/your_app_id/pipeline_steps/stepB # TODO: please fill in
|
34
34
|
"""
|
35
35
|
|
36
36
|
|
@@ -60,7 +60,7 @@ def get_pipeline_step_template(step_id: str):
|
|
60
60
|
return f'''import argparse
|
61
61
|
|
62
62
|
import clarifai
|
63
|
-
|
63
|
+
from clarifai.utils.logging import logger
|
64
64
|
|
65
65
|
def main():
|
66
66
|
parser = argparse.ArgumentParser(description='{step_id} processing step.')
|
@@ -68,10 +68,10 @@ def main():
|
|
68
68
|
|
69
69
|
args = parser.parse_args()
|
70
70
|
|
71
|
-
|
71
|
+
logger.info(clarifai.__version__)
|
72
72
|
|
73
73
|
# TODO: Implement your pipeline step logic here
|
74
|
-
|
74
|
+
logger.info(f"{step_id} processed: {{args.input_text}}")
|
75
75
|
|
76
76
|
|
77
77
|
if __name__ == "__main__":
|
clarifai/client/__init__.py
CHANGED
@@ -8,6 +8,7 @@ from clarifai.client.lister import Lister
|
|
8
8
|
from clarifai.client.model import Model
|
9
9
|
from clarifai.client.module import Module
|
10
10
|
from clarifai.client.pipeline import Pipeline
|
11
|
+
from clarifai.client.pipeline_step import PipelineStep
|
11
12
|
from clarifai.client.search import Search
|
12
13
|
from clarifai.client.user import User
|
13
14
|
from clarifai.client.workflow import Workflow
|
@@ -20,6 +21,7 @@ __all__ = [
|
|
20
21
|
'Model',
|
21
22
|
'Workflow',
|
22
23
|
'Pipeline',
|
24
|
+
'PipelineStep',
|
23
25
|
'Module',
|
24
26
|
'Lister',
|
25
27
|
'Dataset',
|
clarifai/client/app.py
CHANGED
@@ -14,6 +14,8 @@ from clarifai.client.input import Inputs
|
|
14
14
|
from clarifai.client.lister import Lister
|
15
15
|
from clarifai.client.model import Model
|
16
16
|
from clarifai.client.module import Module
|
17
|
+
from clarifai.client.pipeline import Pipeline
|
18
|
+
from clarifai.client.pipeline_step import PipelineStep
|
17
19
|
from clarifai.client.search import Search
|
18
20
|
from clarifai.client.workflow import Workflow
|
19
21
|
from clarifai.constants.model import TRAINABLE_MODEL_TYPES
|
@@ -198,6 +200,151 @@ class App(Lister, BaseClient):
|
|
198
200
|
continue
|
199
201
|
yield Workflow.from_auth_helper(auth=self.auth_helper, **workflow_info)
|
200
202
|
|
203
|
+
def list_pipelines(
|
204
|
+
self,
|
205
|
+
filter_by: Dict[str, Any] = {},
|
206
|
+
only_in_app: bool = True,
|
207
|
+
page_no: int = None,
|
208
|
+
per_page: int = None,
|
209
|
+
) -> Generator[dict, None, None]:
|
210
|
+
"""Lists all the pipelines for the user.
|
211
|
+
|
212
|
+
Args:
|
213
|
+
filter_by (dict): A dictionary of filters to apply to the list of pipelines.
|
214
|
+
only_in_app (bool): If True, only return pipelines that are in the app.
|
215
|
+
page_no (int): The page number to list.
|
216
|
+
per_page (int): The number of items per page.
|
217
|
+
|
218
|
+
Yields:
|
219
|
+
Dict: Dictionaries containing information about the pipelines.
|
220
|
+
|
221
|
+
Example:
|
222
|
+
>>> from clarifai.client.app import App
|
223
|
+
>>> app = App(app_id="app_id", user_id="user_id")
|
224
|
+
>>> all_pipelines = list(app.list_pipelines())
|
225
|
+
|
226
|
+
Note:
|
227
|
+
Defaults to 16 per page if page_no is specified and per_page is not specified.
|
228
|
+
If both page_no and per_page are None, then lists all the resources.
|
229
|
+
"""
|
230
|
+
request_data = dict(user_app_id=self.user_app_id, **filter_by)
|
231
|
+
all_pipelines_info = self.list_pages_generator(
|
232
|
+
self.STUB.ListPipelines,
|
233
|
+
service_pb2.ListPipelinesRequest,
|
234
|
+
request_data,
|
235
|
+
per_page=per_page,
|
236
|
+
page_no=page_no,
|
237
|
+
)
|
238
|
+
|
239
|
+
for pipeline_info in all_pipelines_info:
|
240
|
+
pipeline = self._process_pipeline_info(
|
241
|
+
pipeline_info, self.auth_helper, self.id, only_in_app
|
242
|
+
)
|
243
|
+
if pipeline is not None:
|
244
|
+
yield pipeline
|
245
|
+
|
246
|
+
@staticmethod
|
247
|
+
def _process_pipeline_info(pipeline_info, auth_helper, app_id=None, only_in_app=False):
|
248
|
+
"""Helper method to process pipeline info and create Pipeline objects.
|
249
|
+
|
250
|
+
Args:
|
251
|
+
pipeline_info: Raw pipeline info from API
|
252
|
+
auth_helper: Auth helper instance
|
253
|
+
app_id: App ID to filter by (if only_in_app is True)
|
254
|
+
only_in_app: Whether to filter by app_id
|
255
|
+
|
256
|
+
Returns:
|
257
|
+
Pipeline object or None if filtered out
|
258
|
+
"""
|
259
|
+
if only_in_app and app_id:
|
260
|
+
if pipeline_info.get('app_id') != app_id:
|
261
|
+
return None
|
262
|
+
|
263
|
+
# Map API field names to constructor parameter names
|
264
|
+
pipeline_kwargs = pipeline_info.copy()
|
265
|
+
if 'id' in pipeline_kwargs:
|
266
|
+
pipeline_kwargs['pipeline_id'] = pipeline_kwargs.pop('id')
|
267
|
+
if 'pipeline_version' in pipeline_kwargs:
|
268
|
+
pipeline_version = pipeline_kwargs.pop('pipeline_version')
|
269
|
+
pipeline_kwargs['pipeline_version_id'] = pipeline_version.get('id', '')
|
270
|
+
|
271
|
+
return Pipeline.from_auth_helper(auth=auth_helper, **pipeline_kwargs)
|
272
|
+
|
273
|
+
@staticmethod
|
274
|
+
def _process_pipeline_step_info(
|
275
|
+
pipeline_step_info, auth_helper, app_id=None, only_in_app=False
|
276
|
+
):
|
277
|
+
"""Helper method to process pipeline step info and create PipelineStep objects.
|
278
|
+
|
279
|
+
Args:
|
280
|
+
pipeline_step_info: Raw pipeline step info from API
|
281
|
+
auth_helper: Auth helper instance
|
282
|
+
app_id: App ID to filter by (if only_in_app is True)
|
283
|
+
only_in_app: Whether to filter by app_id
|
284
|
+
|
285
|
+
Returns:
|
286
|
+
PipelineStep object or None if filtered out
|
287
|
+
"""
|
288
|
+
if only_in_app and app_id:
|
289
|
+
if pipeline_step_info.get('app_id') != app_id:
|
290
|
+
return None
|
291
|
+
|
292
|
+
# Map API field names to constructor parameter names
|
293
|
+
step_kwargs = pipeline_step_info.copy()
|
294
|
+
if 'pipeline_step' in step_kwargs:
|
295
|
+
pipeline_step = step_kwargs.pop('pipeline_step')
|
296
|
+
step_kwargs['pipeline_step_id'] = pipeline_step.get('id', '')
|
297
|
+
|
298
|
+
return PipelineStep.from_auth_helper(auth=auth_helper, **step_kwargs)
|
299
|
+
|
300
|
+
def list_pipeline_steps(
|
301
|
+
self,
|
302
|
+
pipeline_id: str = None,
|
303
|
+
filter_by: Dict[str, Any] = {},
|
304
|
+
only_in_app: bool = True,
|
305
|
+
page_no: int = None,
|
306
|
+
per_page: int = None,
|
307
|
+
) -> Generator[dict, None, None]:
|
308
|
+
"""Lists all the pipeline steps for the user.
|
309
|
+
|
310
|
+
Args:
|
311
|
+
pipeline_id (str): If provided, only list pipeline steps from this pipeline.
|
312
|
+
filter_by (dict): A dictionary of filters to apply to the list of pipeline steps.
|
313
|
+
only_in_app (bool): If True, only return pipeline steps that are in the app.
|
314
|
+
page_no (int): The page number to list.
|
315
|
+
per_page (int): The number of items per page.
|
316
|
+
|
317
|
+
Yields:
|
318
|
+
Dict: Dictionaries containing information about the pipeline steps.
|
319
|
+
|
320
|
+
Example:
|
321
|
+
>>> from clarifai.client.app import App
|
322
|
+
>>> app = App(app_id="app_id", user_id="user_id")
|
323
|
+
>>> all_pipeline_steps = list(app.list_pipeline_steps())
|
324
|
+
|
325
|
+
Note:
|
326
|
+
Defaults to 16 per page if page_no is specified and per_page is not specified.
|
327
|
+
If both page_no and per_page are None, then lists all the resources.
|
328
|
+
"""
|
329
|
+
request_data = dict(user_app_id=self.user_app_id, **filter_by)
|
330
|
+
if pipeline_id:
|
331
|
+
request_data['pipeline_id'] = pipeline_id
|
332
|
+
|
333
|
+
all_pipeline_steps_info = self.list_pages_generator(
|
334
|
+
self.STUB.ListPipelineStepVersions,
|
335
|
+
service_pb2.ListPipelineStepVersionsRequest,
|
336
|
+
request_data,
|
337
|
+
per_page=per_page,
|
338
|
+
page_no=page_no,
|
339
|
+
)
|
340
|
+
|
341
|
+
for pipeline_step_info in all_pipeline_steps_info:
|
342
|
+
pipeline_step = self._process_pipeline_step_info(
|
343
|
+
pipeline_step_info, self.auth_helper, self.id, only_in_app
|
344
|
+
)
|
345
|
+
if pipeline_step is not None:
|
346
|
+
yield pipeline_step
|
347
|
+
|
201
348
|
def list_modules(
|
202
349
|
self,
|
203
350
|
filter_by: Dict[str, Any] = {},
|
clarifai/client/nodepool.py
CHANGED
@@ -94,7 +94,9 @@ class Nodepool(Lister, BaseClient):
|
|
94
94
|
), "worker info not found in the config file"
|
95
95
|
assert "scheduling_choice" in deployment, "scheduling_choice not found in the config file"
|
96
96
|
assert "nodepools" in deployment, "nodepools not found in the config file"
|
97
|
-
deployment['user_id'] =
|
97
|
+
deployment['user_id'] = (
|
98
|
+
deployment['user_id'] if 'user_id' in deployment else self.user_app_id.user_id
|
99
|
+
)
|
98
100
|
if "autoscale_config" in deployment:
|
99
101
|
deployment['autoscale_config'] = resources_pb2.AutoscaleConfig(
|
100
102
|
**deployment['autoscale_config']
|
@@ -103,7 +105,10 @@ class Nodepool(Lister, BaseClient):
|
|
103
105
|
resources_pb2.Nodepool(
|
104
106
|
id=nodepool['id'],
|
105
107
|
compute_cluster=resources_pb2.ComputeCluster(
|
106
|
-
id=nodepool['compute_cluster']['id'],
|
108
|
+
id=nodepool['compute_cluster']['id'],
|
109
|
+
user_id=nodepool['compute_cluster']['user_id']
|
110
|
+
if 'user_id' in nodepool['compute_cluster']
|
111
|
+
else self.user_app_id.user_id,
|
107
112
|
),
|
108
113
|
)
|
109
114
|
for nodepool in deployment['nodepools']
|