clams-python 1.2.1__tar.gz → 1.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {clams_python-1.2.1/clams_python.egg-info → clams_python-1.2.3}/PKG-INFO +2 -2
- clams_python-1.2.3/VERSION +1 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/app/__init__.py +34 -6
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/appmetadata/__init__.py +6 -3
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/__init__.py +64 -14
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/app/Containerfile.template +14 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/app/app.py.template +13 -1
- clams_python-1.2.3/clams/develop/templates/app/cli.py.template +92 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/mmif_utils/rewind.py +10 -21
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/restify/__init__.py +4 -1
- clams_python-1.2.3/clams/ver/__init__.py +1 -0
- {clams_python-1.2.1 → clams_python-1.2.3/clams_python.egg-info}/PKG-INFO +2 -2
- {clams_python-1.2.1 → clams_python-1.2.3}/clams_python.egg-info/SOURCES.txt +1 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams_python.egg-info/requires.txt +1 -1
- {clams_python-1.2.1 → clams_python-1.2.3}/requirements.txt +1 -1
- {clams_python-1.2.1 → clams_python-1.2.3}/tests/test_clamsapp.py +22 -4
- clams_python-1.2.1/VERSION +0 -1
- clams_python-1.2.1/clams/ver/__init__.py +0 -1
- {clams_python-1.2.1 → clams_python-1.2.3}/LICENSE +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/MANIFEST.in +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/README.md +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/__init__.py +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/app/.dockerignore.template +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/app/.gitignore.template +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/app/LICENSE.template +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/app/README.md.template +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/app/metadata.py.template +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/app/requirements.txt.template +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/gha/for-clams-team.md.template +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/gha/workflows/issue-apps-project.yml.template +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/gha/workflows/issue-assign.yml.template +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/gha/workflows/issue-close.yml.template +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/gha/workflows/publish.yml.template +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/mmif_utils/__init__.py +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/mmif_utils/source.py +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams/serve/__init__.py +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams_python.egg-info/dependency_links.txt +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams_python.egg-info/entry_points.txt +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/clams_python.egg-info/top_level.txt +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/setup.cfg +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/setup.py +0 -0
- {clams_python-1.2.1 → clams_python-1.2.3}/tests/test_clamscli.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: clams-python
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.3
|
|
4
4
|
Summary: A collection of APIs to develop CLAMS app for python
|
|
5
5
|
Home-page: https://clams.ai
|
|
6
6
|
Author: Brandeis Lab for Linguistics and Computation
|
|
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
|
|
|
14
14
|
Requires-Python: >=3.8
|
|
15
15
|
Description-Content-Type: text/markdown
|
|
16
16
|
License-File: LICENSE
|
|
17
|
-
Requires-Dist: mmif-python==1.0.
|
|
17
|
+
Requires-Dist: mmif-python==1.0.15
|
|
18
18
|
Requires-Dist: Flask>=2
|
|
19
19
|
Requires-Dist: Flask-RESTful>=0.3.9
|
|
20
20
|
Requires-Dist: gunicorn>=20
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
1.2.3
|
|
@@ -20,6 +20,16 @@ logging.basicConfig(
|
|
|
20
20
|
datefmt="%Y-%m-%d %H:%M:%S")
|
|
21
21
|
|
|
22
22
|
|
|
23
|
+
falsy_values = [
|
|
24
|
+
'False',
|
|
25
|
+
'false',
|
|
26
|
+
'F',
|
|
27
|
+
'f',
|
|
28
|
+
'0',
|
|
29
|
+
0,
|
|
30
|
+
False
|
|
31
|
+
]
|
|
32
|
+
|
|
23
33
|
class ClamsApp(ABC):
|
|
24
34
|
"""
|
|
25
35
|
An abstract class to define API's for ClamsApps. A CLAMS app should inherit
|
|
@@ -178,7 +188,14 @@ class ClamsApp(ABC):
|
|
|
178
188
|
raise ValueError(f"Value for parameter \"{parameter.name}\" must be one of {parameter.choices}.")
|
|
179
189
|
refined[parameter.name] = casted[parameter.name]
|
|
180
190
|
elif parameter.default is not None:
|
|
181
|
-
|
|
191
|
+
# have to cast the default values as well, since
|
|
192
|
+
# 1. `map` type default values are not actually expected as a map (dict)
|
|
193
|
+
# 2. developers can use data type not matching the spec
|
|
194
|
+
if isinstance(parameter.default, list):
|
|
195
|
+
casted_default = self.annotate_param_caster.cast({parameter.name: list(map(str, parameter.default))})
|
|
196
|
+
else:
|
|
197
|
+
casted_default = self.annotate_param_caster.cast({parameter.name: [str(parameter.default)]})
|
|
198
|
+
refined.update(casted_default)
|
|
182
199
|
else:
|
|
183
200
|
raise ValueError(f"Cannot find configuration for a required parameter \"{parameter.name}\".")
|
|
184
201
|
# raw input params are hidden under a special key
|
|
@@ -338,8 +355,8 @@ class ParameterCaster(object):
|
|
|
338
355
|
assert isinstance(vs, list), f"Expected a list of values for key {k}, but got {vs} of type {type(vs)}"
|
|
339
356
|
assert all(isinstance(v, str) for v in vs), f"Expected a list of strings for key {k}, but got {vs} of types {[type(v) for v in vs]}"
|
|
340
357
|
if k in self.param_spec:
|
|
358
|
+
valuetype, multivalued = self.param_spec[k]
|
|
341
359
|
for v in vs:
|
|
342
|
-
valuetype, multivalued = self.param_spec[k]
|
|
343
360
|
if multivalued or k not in casted: # effectively only keeps the first value for non-multi params
|
|
344
361
|
if valuetype == bool:
|
|
345
362
|
v = self.bool_param(v)
|
|
@@ -358,11 +375,22 @@ class ParameterCaster(object):
|
|
|
358
375
|
casted.setdefault(k, []).append(v)
|
|
359
376
|
else:
|
|
360
377
|
casted[k] = v
|
|
378
|
+
# when an empty value is passed (usually as a default value)
|
|
379
|
+
# just add an empty list or dict as a placeholder
|
|
380
|
+
# explicit check for `len == 0` is required to prevent
|
|
381
|
+
# empty values are set for params that don't have default values
|
|
382
|
+
if multivalued and len(vs) == 0:
|
|
383
|
+
if valuetype == dict:
|
|
384
|
+
casted.setdefault(k, {})
|
|
385
|
+
else:
|
|
386
|
+
casted.setdefault(k, [])
|
|
361
387
|
else:
|
|
362
|
-
if
|
|
363
|
-
|
|
364
|
-
|
|
388
|
+
# if the parameter is not defined in the spec, there's no easy way to figure out
|
|
389
|
+
# the user's intention whether it should be a list (multivalued) or a single value.
|
|
390
|
+
if len(vs) == 1:
|
|
365
391
|
casted[k] = vs[0]
|
|
392
|
+
else:
|
|
393
|
+
casted[k] = vs
|
|
366
394
|
return casted # pytype: disable=bad-return-type
|
|
367
395
|
|
|
368
396
|
@staticmethod
|
|
@@ -370,7 +398,7 @@ class ParameterCaster(object):
|
|
|
370
398
|
"""
|
|
371
399
|
Helper function to convert string values to bool type.
|
|
372
400
|
"""
|
|
373
|
-
return False if value in
|
|
401
|
+
return False if value in falsy_values else True
|
|
374
402
|
|
|
375
403
|
@staticmethod
|
|
376
404
|
def float_param(value) -> float:
|
|
@@ -162,7 +162,8 @@ class RuntimeParameter(_BaseModel):
|
|
|
162
162
|
type: param_value_types = pydantic.Field(
|
|
163
163
|
...,
|
|
164
164
|
description=f"Type of the parameter value the app expects. Must be one of {param_value_types_values}. When "
|
|
165
|
-
"type is ``map``, ``multivalued=true`` is
|
|
165
|
+
"type is ``map``, ``multivalued=true`` is forced, and when ``boolean``, ``multivalued=false`` is "
|
|
166
|
+
"forced. \n\n"
|
|
166
167
|
"Notes for developers: \n\n"
|
|
167
168
|
"When the type is ``map``, the parameter value (still a single string from the users' perspective) "
|
|
168
169
|
"must be formatted as a ``KEY:VALUE`` pair, namely a colon-separated string. To pass multiple "
|
|
@@ -186,7 +187,7 @@ class RuntimeParameter(_BaseModel):
|
|
|
186
187
|
description="(optional) Default value for the parameter.\n\n"
|
|
187
188
|
"Notes for developers: \n\n"
|
|
188
189
|
"Setting a default value makes a parameter `optional`. \n\n"
|
|
189
|
-
"When ``multivalued=
|
|
190
|
+
"When ``multivalued=true``, the default value should be a list of values. \n\n"
|
|
190
191
|
"When ``type=map``, the default value should be a list of colon-separated strings. \n\n"
|
|
191
192
|
)
|
|
192
193
|
multivalued: bool = pydantic.Field(
|
|
@@ -420,8 +421,10 @@ class AppMetadata(pydantic.BaseModel):
|
|
|
420
421
|
# see https://docs.pydantic.dev/1.10/usage/types/#unions
|
|
421
422
|
# e.g. casting 0.1 using the `primitives` dict will result in 0 (int)
|
|
422
423
|
# while casting "0.1" using the `primitives` dict will result in 0.1 (float)
|
|
423
|
-
if type == 'map'
|
|
424
|
+
if type == 'map':
|
|
424
425
|
multivalued = True
|
|
426
|
+
if type == 'boolean':
|
|
427
|
+
multivalued = False
|
|
425
428
|
if default is not None:
|
|
426
429
|
if isinstance(default, list):
|
|
427
430
|
default = [str(d) for d in default]
|
|
@@ -7,6 +7,7 @@ from typing import List
|
|
|
7
7
|
|
|
8
8
|
import clams
|
|
9
9
|
|
|
10
|
+
update_tmp_suffix = '.tmp'
|
|
10
11
|
available_recipes = {
|
|
11
12
|
'app': {
|
|
12
13
|
'description': 'Skeleton code for a CLAMS app',
|
|
@@ -26,7 +27,7 @@ class CookieCutter(object):
|
|
|
26
27
|
def __init__(self, name: str, outdir: str, recipes: List[str]):
|
|
27
28
|
self.rawname = name
|
|
28
29
|
self.name_tokens = self.tokenize_rawname()
|
|
29
|
-
self.
|
|
30
|
+
self.outdir = pathlib.Path(outdir)
|
|
30
31
|
if recipes:
|
|
31
32
|
self.recipes = recipes
|
|
32
33
|
else:
|
|
@@ -42,24 +43,30 @@ class CookieCutter(object):
|
|
|
42
43
|
words.pop()
|
|
43
44
|
return words
|
|
44
45
|
|
|
45
|
-
def bake(self):
|
|
46
|
+
def bake(self, update_level=0):
|
|
46
47
|
print(f"Baking {self.recipes}")
|
|
47
48
|
for recipe in self.recipes:
|
|
48
49
|
src_dir = pathlib.Path(__file__).parent / 'templates' / available_recipes[recipe]['sourcedir']
|
|
49
|
-
dst_dir = self.
|
|
50
|
+
dst_dir = self.outdir / self.rawname / available_recipes[recipe]['targetdir']
|
|
50
51
|
if recipe == 'app':
|
|
51
|
-
|
|
52
|
+
caps = [t.capitalize() for t in self.name_tokens]
|
|
53
|
+
app_vars = {
|
|
54
|
+
'CLAMS_VERSION': clams.__version__,
|
|
55
|
+
'APP_CLASS_NAME': "".join(caps),
|
|
56
|
+
'APP_NAME': " ".join(caps),
|
|
57
|
+
'APP_IDENTIFIER': '-'.join(self.name_tokens)
|
|
58
|
+
}
|
|
59
|
+
if update_level > 0:
|
|
60
|
+
self.reheat_app(src_dir, dst_dir, app_vars, reheat_level=update_level)
|
|
61
|
+
else:
|
|
62
|
+
if dst_dir.exists():
|
|
63
|
+
raise FileExistsError(f" {dst_dir} already exists. Did you mean `--update`? ")
|
|
64
|
+
self.bake_app(src_dir, dst_dir, app_vars)
|
|
52
65
|
if recipe == 'gha':
|
|
66
|
+
# There's nothing for devs to tweak GHA template, so first generation and updating are the same.
|
|
53
67
|
self.bake_gha(src_dir, dst_dir)
|
|
54
68
|
|
|
55
|
-
def bake_app(self, src_dir, dst_dir):
|
|
56
|
-
caps = [t.capitalize() for t in self.name_tokens]
|
|
57
|
-
templating_vars = {
|
|
58
|
-
'CLAMS_VERSION': clams.__version__,
|
|
59
|
-
'APP_CLASS_NAME': "".join(caps),
|
|
60
|
-
'APP_NAME': " ".join(caps),
|
|
61
|
-
'APP_IDENTIFIER': '-'.join(self.name_tokens)
|
|
62
|
-
}
|
|
69
|
+
def bake_app(self, src_dir, dst_dir, templating_vars):
|
|
63
70
|
for g in src_dir.glob("**/*.template"):
|
|
64
71
|
r = g.relative_to(src_dir).parent
|
|
65
72
|
f = g.with_suffix('').name
|
|
@@ -71,7 +78,40 @@ class CookieCutter(object):
|
|
|
71
78
|
out_f.write(compiled)
|
|
72
79
|
print(f"App skeleton code is copied to {self.rawname}")
|
|
73
80
|
print(f" Checkout {self.rawname}/README.md for the next steps!")
|
|
74
|
-
|
|
81
|
+
|
|
82
|
+
def reheat_app(self, src_dir, dst_dir, templating_vars, reheat_level=1):
|
|
83
|
+
essentials = ['app.py', 'metadata.py', 'cli.py', 'Containerfile', 'requirements.txt']
|
|
84
|
+
for template in src_dir.glob("**/*.template"):
|
|
85
|
+
dirname = template.relative_to(src_dir).parent
|
|
86
|
+
basename = template.with_suffix('').name
|
|
87
|
+
if basename not in essentials:
|
|
88
|
+
# if non-essential, just skip when updating
|
|
89
|
+
continue
|
|
90
|
+
in_f = open(template, 'r')
|
|
91
|
+
tmpl_to_compile = Template(in_f.read())
|
|
92
|
+
compiled = tmpl_to_compile.safe_substitute(templating_vars)
|
|
93
|
+
in_f.close()
|
|
94
|
+
ori_fpath = dst_dir / dirname / basename
|
|
95
|
+
if not ori_fpath.exists():
|
|
96
|
+
# this file is new in this version of cookiecutter
|
|
97
|
+
with open(ori_fpath, 'w') as out_f:
|
|
98
|
+
out_f.write(compiled)
|
|
99
|
+
else:
|
|
100
|
+
ori_f = open(ori_fpath, 'r')
|
|
101
|
+
ori_content = ori_f.read()
|
|
102
|
+
if ori_content != compiled:
|
|
103
|
+
# when the target file already exists, we need to do diff & patch
|
|
104
|
+
# TODO (krim @ 5/5/24): add update level 2 and 3 code here
|
|
105
|
+
out_fpath = f'{ori_fpath}{update_tmp_suffix}'
|
|
106
|
+
print(f' {dst_dir / dirname / basename} already exists, generating a tmp file: {out_fpath}')
|
|
107
|
+
with open(out_fpath, 'w') as out_f:
|
|
108
|
+
out_f.write(compiled)
|
|
109
|
+
else:
|
|
110
|
+
print(f' {dst_dir / dirname / basename} already exists, but the content is unchanged from the '
|
|
111
|
+
f'template, skipping re-generating')
|
|
112
|
+
print(f"App skeleton code is updated in {self.rawname}")
|
|
113
|
+
print(f" Checkout {self.rawname}/README.md for the next steps!")
|
|
114
|
+
|
|
75
115
|
def bake_gha(self, src_dir, dst_dir):
|
|
76
116
|
self.simple_recursive_copy_minus_template_suffix(src_dir, dst_dir)
|
|
77
117
|
print(f"GitHub Actions workflow files are copied to {self.rawname}/.github")
|
|
@@ -128,12 +168,22 @@ def prep_argparser(**kwargs):
|
|
|
128
168
|
nargs='?',
|
|
129
169
|
help='The name of the parent directory where the app skeleton directory is placed. (default: current directory)'
|
|
130
170
|
)
|
|
171
|
+
parser.add_argument(
|
|
172
|
+
'-u', '--update',
|
|
173
|
+
action='count',
|
|
174
|
+
help=f'Set update level by passing this flag multiple times. This is EXPERIMENTAL, and developers MUST NOT'
|
|
175
|
+
f'rely on the update results, and should conduct manual checks afterward. LEVEL 0: does not update and '
|
|
176
|
+
f'raise an error when existing directory found. LEVEL 1: generate non-existing files and generate '
|
|
177
|
+
f'`{update_tmp_suffix}`-suffixed files for existing one. LEVEL 2 (WIP): generate non-existing files and '
|
|
178
|
+
f'automatically generate patch files for existing files. LEVEL 3 (WIP): generate non-existing files and '
|
|
179
|
+
f'apply patches to existing files. (default: 0)'
|
|
180
|
+
)
|
|
131
181
|
return parser
|
|
132
182
|
|
|
133
183
|
|
|
134
184
|
def main(args):
|
|
135
185
|
cutter = CookieCutter(name=args.name, outdir=args.parent_dir, recipes=args.recipes)
|
|
136
|
-
cutter.bake()
|
|
186
|
+
cutter.bake(args.update)
|
|
137
187
|
|
|
138
188
|
if __name__ == '__main__':
|
|
139
189
|
parser = prep_argparser()
|
{clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/app/Containerfile.template
RENAMED
|
@@ -14,6 +14,20 @@ ARG CLAMS_APP_VERSION
|
|
|
14
14
|
ENV CLAMS_APP_VERSION ${CLAMS_APP_VERSION}
|
|
15
15
|
################################################################################
|
|
16
16
|
|
|
17
|
+
################################################################################
|
|
18
|
+
# This is duplicate from the base image Containerfile
|
|
19
|
+
# but makes sure the cache directories are consistent across all CLAMS apps
|
|
20
|
+
|
|
21
|
+
# https://github.com/openai/whisper/blob/ba3f3cd54b0e5b8ce1ab3de13e32122d0d5f98ab/whisper/__init__.py#L130
|
|
22
|
+
ENV XDG_CACHE_HOME='/cache'
|
|
23
|
+
# https://huggingface.co/docs/huggingface_hub/main/en/package_reference/environment_variables#hfhome
|
|
24
|
+
ENV HF_HOME="/cache/huggingface"
|
|
25
|
+
# https://pytorch.org/docs/stable/hub.html#where-are-my-downloaded-models-saved
|
|
26
|
+
ENV TORCH_HOME="/cache/torch"
|
|
27
|
+
|
|
28
|
+
RUN mkdir /cache && rm -rf /root/.cache && ln -s /cache /root/.cache
|
|
29
|
+
################################################################################
|
|
30
|
+
|
|
17
31
|
################################################################################
|
|
18
32
|
# clams-python base images are based on debian distro
|
|
19
33
|
# install more system packages as needed using the apt manager
|
|
@@ -39,6 +39,16 @@ class $APP_CLASS_NAME(ClamsApp):
|
|
|
39
39
|
# see https://sdk.clams.ai/autodoc/clams.app.html#clams.app.ClamsApp._annotate
|
|
40
40
|
raise NotImplementedError
|
|
41
41
|
|
|
42
|
+
def get_app():
|
|
43
|
+
"""
|
|
44
|
+
This function effectively creates an instance of the app class, without any arguments passed in, meaning, any
|
|
45
|
+
external information such as initial app configuration should be set without using function arguments. The easiest
|
|
46
|
+
way to do this is to set global variables before calling this.
|
|
47
|
+
"""
|
|
48
|
+
# for example:
|
|
49
|
+
# return $APP_CLASS_NAME(create, from, global, params)
|
|
50
|
+
raise NotImplementedError
|
|
51
|
+
|
|
42
52
|
|
|
43
53
|
if __name__ == "__main__":
|
|
44
54
|
parser = argparse.ArgumentParser()
|
|
@@ -50,7 +60,9 @@ if __name__ == "__main__":
|
|
|
50
60
|
parsed_args = parser.parse_args()
|
|
51
61
|
|
|
52
62
|
# create the app instance
|
|
53
|
-
|
|
63
|
+
# if get_app() call requires any "configurations", they should be set now as global variables
|
|
64
|
+
# and referenced in the get_app() function. NOTE THAT you should not change the signature of get_app()
|
|
65
|
+
app = get_app()
|
|
54
66
|
|
|
55
67
|
http_app = Restifier(app, port=int(parsed_args.port))
|
|
56
68
|
# for running the application in production mode
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
The purpose of this file is to define a thin CLI interface for your app
|
|
4
|
+
|
|
5
|
+
DO NOT CHANGE the name of the file
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import sys
|
|
10
|
+
from contextlib import redirect_stdout
|
|
11
|
+
|
|
12
|
+
import app
|
|
13
|
+
|
|
14
|
+
import clams.app
|
|
15
|
+
from clams import AppMetadata
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def metadata_to_argparser(app_metadata: AppMetadata) -> argparse.ArgumentParser:
|
|
19
|
+
"""
|
|
20
|
+
Automatically generate an argparse.ArgumentParser from parameters specified in the app metadata (metadata.py).
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
parser = argparse.ArgumentParser(
|
|
24
|
+
description=f"{app_metadata.name}: {app_metadata.description} (visit {app_metadata.url} for more info)",
|
|
25
|
+
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
26
|
+
|
|
27
|
+
# parse cli args from app parameters
|
|
28
|
+
for parameter in app_metadata.parameters:
|
|
29
|
+
if parameter.multivalued:
|
|
30
|
+
a = parser.add_argument(
|
|
31
|
+
f"--{parameter.name}",
|
|
32
|
+
help=parameter.description,
|
|
33
|
+
nargs='+',
|
|
34
|
+
action='extend',
|
|
35
|
+
type=str
|
|
36
|
+
)
|
|
37
|
+
else:
|
|
38
|
+
a = parser.add_argument(
|
|
39
|
+
f"--{parameter.name}",
|
|
40
|
+
help=parameter.description,
|
|
41
|
+
nargs=1,
|
|
42
|
+
action="store",
|
|
43
|
+
type=str)
|
|
44
|
+
if parameter.choices is not None:
|
|
45
|
+
a.choices = parameter.choices
|
|
46
|
+
if parameter.default is not None:
|
|
47
|
+
a.help += f" (default: {parameter.default}"
|
|
48
|
+
if parameter.type == "boolean":
|
|
49
|
+
a.help += (f", any value except for {[v for v in clams.app.falsy_values if isinstance(v, str)]} "
|
|
50
|
+
f"will be interpreted as True")
|
|
51
|
+
a.help += ')'
|
|
52
|
+
# then we don't have to add default values to the arg_parser
|
|
53
|
+
# since that's handled by the app._refined_params() method.
|
|
54
|
+
parser.add_argument('IN_MMIF_FILE', nargs='?', type=argparse.FileType('r'),
|
|
55
|
+
help='input MMIF file path, or STDIN if `-` or not provided. NOTE: When running this cli.py in '
|
|
56
|
+
'a containerized environment, make sure the container is run with `-i` flag to keep stdin '
|
|
57
|
+
'open.',
|
|
58
|
+
# will check if stdin is a keyboard, and return None if it is
|
|
59
|
+
default=None if sys.stdin.isatty() else sys.stdin)
|
|
60
|
+
parser.add_argument('OUT_MMIF_FILE', nargs='?', type=argparse.FileType('w'),
|
|
61
|
+
help='output MMIF file path, or STDOUT if `-` or not provided. NOTE: When this is set to '
|
|
62
|
+
'STDOUT, any print statements in the app code will be redirected to stderr.',
|
|
63
|
+
default=sys.stdout)
|
|
64
|
+
return parser
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
if __name__ == "__main__":
|
|
68
|
+
clamsapp = app.get_app()
|
|
69
|
+
arg_parser = metadata_to_argparser(app_metadata=clamsapp.metadata)
|
|
70
|
+
args = arg_parser.parse_args()
|
|
71
|
+
if args.IN_MMIF_FILE:
|
|
72
|
+
in_data = args.IN_MMIF_FILE.read()
|
|
73
|
+
# since flask webapp interface will pass parameters as "unflattened" dict to handle multivalued parameters
|
|
74
|
+
# (https://werkzeug.palletsprojects.com/en/latest/datastructures/#werkzeug.datastructures.MultiDict.to_dict)
|
|
75
|
+
# we need to convert arg_parsers results into a similar structure, which is the dict values are wrapped in lists
|
|
76
|
+
params = {}
|
|
77
|
+
for pname, pvalue in vars(args).items():
|
|
78
|
+
if pvalue is None or pname in ['IN_MMIF_FILE', 'OUT_MMIF_FILE']:
|
|
79
|
+
continue
|
|
80
|
+
elif isinstance(pvalue, list):
|
|
81
|
+
params[pname] = pvalue
|
|
82
|
+
else:
|
|
83
|
+
params[pname] = [pvalue]
|
|
84
|
+
if args.OUT_MMIF_FILE.name == '<stdout>':
|
|
85
|
+
with redirect_stdout(sys.stderr):
|
|
86
|
+
out_mmif = clamsapp.annotate(in_data, **params)
|
|
87
|
+
else:
|
|
88
|
+
out_mmif = clamsapp.annotate(in_data, **params)
|
|
89
|
+
args.OUT_MMIF_FILE.write(out_mmif)
|
|
90
|
+
else:
|
|
91
|
+
arg_parser.print_help()
|
|
92
|
+
sys.exit(1)
|
|
@@ -77,10 +77,14 @@ def describe_argparser():
|
|
|
77
77
|
def prep_argparser(**kwargs):
|
|
78
78
|
parser = argparse.ArgumentParser(description=describe_argparser()[1],
|
|
79
79
|
formatter_class=argparse.RawDescriptionHelpFormatter, **kwargs)
|
|
80
|
-
parser.add_argument("
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
help=
|
|
80
|
+
parser.add_argument("IN_MMIF_FILE",
|
|
81
|
+
nargs="?", type=argparse.FileType("r"),
|
|
82
|
+
default=None if sys.stdin.isatty() else sys.stdin,
|
|
83
|
+
help='input MMIF file path, or STDIN if `-` or not provided.')
|
|
84
|
+
parser.add_argument("OUT_MMIF_FILE",
|
|
85
|
+
nargs="?", type=argparse.FileType("w"),
|
|
86
|
+
default=sys.stdout,
|
|
87
|
+
help='output MMIF file path, or STDOUT if `-` or not provided.')
|
|
84
88
|
parser.add_argument("-p", '--pretty', action='store_true',
|
|
85
89
|
help="Pretty-print rewound MMIF")
|
|
86
90
|
parser.add_argument("-n", '--number', default="0", type=int,
|
|
@@ -92,7 +96,7 @@ def prep_argparser(**kwargs):
|
|
|
92
96
|
|
|
93
97
|
|
|
94
98
|
def main(args):
|
|
95
|
-
mmif_obj = mmif.Mmif(
|
|
99
|
+
mmif_obj = mmif.Mmif(args.IN_MMIF_FILE.read())
|
|
96
100
|
|
|
97
101
|
if args.number == 0: # If user doesn't know how many views to rewind, give them choices.
|
|
98
102
|
choice = prompt_user(mmif_obj)
|
|
@@ -101,22 +105,7 @@ def main(args):
|
|
|
101
105
|
if not isinstance(choice, int) or choice <= 0:
|
|
102
106
|
raise ValueError(f"Only can rewind by a positive number of views. Got {choice}.")
|
|
103
107
|
|
|
104
|
-
|
|
105
|
-
# Check if the same file name exist in the path and avoid overwriting.
|
|
106
|
-
output_fp = P(args.output)
|
|
107
|
-
if output_fp.is_file():
|
|
108
|
-
parent = output_fp.parent
|
|
109
|
-
stem = output_fp.stem
|
|
110
|
-
suffix = output_fp.suffix
|
|
111
|
-
count = 1
|
|
112
|
-
while (parent / f"{stem}_{count}{suffix}").is_file():
|
|
113
|
-
count += 1
|
|
114
|
-
output_fp = parent / f"{stem}_{count}{suffix}"
|
|
115
|
-
|
|
116
|
-
out_f = open(output_fp, 'w')
|
|
117
|
-
else:
|
|
118
|
-
out_f = sys.stdout
|
|
119
|
-
out_f.write(rewind_mmif(mmif_obj, choice, args.mode == 'view').serialize(pretty=args.pretty))
|
|
108
|
+
args.OUT_MMIF_FILE.write(rewind_mmif(mmif_obj, choice, args.mode == 'view').serialize(pretty=args.pretty))
|
|
120
109
|
|
|
121
110
|
|
|
122
111
|
if __name__ == "__main__":
|
|
@@ -124,7 +124,9 @@ class ClamsHTTPApi(Resource):
|
|
|
124
124
|
|
|
125
125
|
:return: Returns app metadata in a HTTP response.
|
|
126
126
|
"""
|
|
127
|
-
|
|
127
|
+
# this will catch duplicate arguments with different values into a list under the key
|
|
128
|
+
raw_params = request.args.to_dict(flat=False)
|
|
129
|
+
return self.json_to_response(self.cla.appmetadata(**raw_params))
|
|
128
130
|
|
|
129
131
|
def post(self) -> Response:
|
|
130
132
|
"""
|
|
@@ -143,6 +145,7 @@ class ClamsHTTPApi(Resource):
|
|
|
143
145
|
try:
|
|
144
146
|
return self.json_to_response(self.cla.annotate(raw_data, **raw_params))
|
|
145
147
|
except Exception:
|
|
148
|
+
self.cla.logger.exception("Error in annotation")
|
|
146
149
|
return self.json_to_response(self.cla.record_error(raw_data, **raw_params).serialize(pretty=True), status=500)
|
|
147
150
|
|
|
148
151
|
put = post
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.2.3"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: clams-python
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.3
|
|
4
4
|
Summary: A collection of APIs to develop CLAMS app for python
|
|
5
5
|
Home-page: https://clams.ai
|
|
6
6
|
Author: Brandeis Lab for Linguistics and Computation
|
|
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
|
|
|
14
14
|
Requires-Python: >=3.8
|
|
15
15
|
Description-Content-Type: text/markdown
|
|
16
16
|
License-File: LICENSE
|
|
17
|
-
Requires-Dist: mmif-python==1.0.
|
|
17
|
+
Requires-Dist: mmif-python==1.0.15
|
|
18
18
|
Requires-Dist: Flask>=2
|
|
19
19
|
Requires-Dist: Flask-RESTful>=0.3.9
|
|
20
20
|
Requires-Dist: gunicorn>=20
|
|
@@ -14,6 +14,7 @@ clams/develop/templates/app/Containerfile.template
|
|
|
14
14
|
clams/develop/templates/app/LICENSE.template
|
|
15
15
|
clams/develop/templates/app/README.md.template
|
|
16
16
|
clams/develop/templates/app/app.py.template
|
|
17
|
+
clams/develop/templates/app/cli.py.template
|
|
17
18
|
clams/develop/templates/app/metadata.py.template
|
|
18
19
|
clams/develop/templates/app/requirements.txt.template
|
|
19
20
|
clams/develop/templates/gha/for-clams-team.md.template
|
|
@@ -236,8 +236,8 @@ class TestClamsApp(unittest.TestCase):
|
|
|
236
236
|
self.assertEqual(v1.metadata.app, self.app.metadata.identifier)
|
|
237
237
|
self.assertEqual(len(v1.metadata.parameters), 0)
|
|
238
238
|
v2 = m.new_view()
|
|
239
|
-
|
|
240
|
-
self.app.sign_view(v2, self.app._refine_params(**
|
|
239
|
+
args2 = {'undefined_param1': ['value1']} # values are lists as our restifier uses `to_dict(flat=False)`
|
|
240
|
+
self.app.sign_view(v2, self.app._refine_params(**args2))
|
|
241
241
|
self.assertEqual(v2.metadata.app, self.app.metadata.identifier)
|
|
242
242
|
self.assertEqual(len(v2.metadata.parameters), 1)
|
|
243
243
|
self.assertFalse(clams.ClamsApp._RAW_PARAMS_KEY in v2.metadata.appConfiguration)
|
|
@@ -245,8 +245,22 @@ class TestClamsApp(unittest.TestCase):
|
|
|
245
245
|
# any parameter with defaults should be recorded
|
|
246
246
|
if param.default is not None:
|
|
247
247
|
self.assertTrue(param.name in v2.metadata.appConfiguration)
|
|
248
|
-
elif param.name not in
|
|
248
|
+
elif param.name not in args2:
|
|
249
249
|
self.assertFalse(param.name in v2.metadata.appConfiguration)
|
|
250
|
+
v3 = m.new_view()
|
|
251
|
+
self.app.metadata.add_parameter('multivalued_param', 'dummy parameter with multivalues', type='string', multivalued=True, default=[])
|
|
252
|
+
args3 = {'undefined_param1': ['value1']}
|
|
253
|
+
self.app.sign_view(v3, self.app._refine_params(**args3))
|
|
254
|
+
self.assertEqual(len(v3.metadata.parameters), 1)
|
|
255
|
+
self.assertEqual(len(v3.metadata.appConfiguration), 3) # universal (pretty) + `raise_error` in metadata.py + `multivalued_param`
|
|
256
|
+
self.assertEqual(v3.metadata.appConfiguration['multivalued_param'], [])
|
|
257
|
+
v4 = m.new_view()
|
|
258
|
+
multiple_values = ['value1', 'value2']
|
|
259
|
+
args4 = {'multivalued_param': multiple_values}
|
|
260
|
+
self.app.sign_view(v4, self.app._refine_params(**args4))
|
|
261
|
+
self.assertEqual(len(v4.metadata.parameters), 1)
|
|
262
|
+
self.assertEqual(len(v4.metadata.appConfiguration), 3)
|
|
263
|
+
self.assertEqual(len(v4.metadata.parameters['multivalued_param']), len(str(multiple_values)))
|
|
250
264
|
|
|
251
265
|
def test_annotate(self):
|
|
252
266
|
# The example app is hard-coded to **always** emit version mismatch warning
|
|
@@ -307,6 +321,10 @@ class TestClamsApp(unittest.TestCase):
|
|
|
307
321
|
description='float +def +mv')
|
|
308
322
|
self.app.metadata.add_parameter('param7', type='number', default=[], multivalued=True,
|
|
309
323
|
description='float +def +empty +mv')
|
|
324
|
+
# need to manually rebuild caster object, because the metadata has updated after the app is initialized
|
|
325
|
+
for param_spec in self.app.metadata.parameters:
|
|
326
|
+
self.app.annotate_param_spec[param_spec.name] = (param_spec.type, param_spec.multivalued)
|
|
327
|
+
self.app.annotate_param_caster = ParameterCaster(self.app.annotate_param_spec)
|
|
310
328
|
with self.assertRaises(ValueError):
|
|
311
329
|
self.app._refine_params(**{}) # param1 is required, but not passed
|
|
312
330
|
dummy_params = {'param1': ['okay'], 'non_parameter': ['should be ignored']}
|
|
@@ -375,7 +393,7 @@ class TestRestifier(unittest.TestCase):
|
|
|
375
393
|
def test_can_put_as_json(self):
|
|
376
394
|
put = self.app.put('/', data=ExampleInputMMIF.get_mmif(), headers={"Content-Type": "Application/json"})
|
|
377
395
|
self.assertIsNotNone(put)
|
|
378
|
-
self.assertEqual(put.status_code
|
|
396
|
+
self.assertEqual(200, put.status_code)
|
|
379
397
|
self.assertIsNotNone(Mmif(put.get_data(as_text=True)))
|
|
380
398
|
|
|
381
399
|
def test_can_pass_params(self):
|
clams_python-1.2.1/VERSION
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
1.2.1
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "1.2.1"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/app/.dockerignore.template
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/app/requirements.txt.template
RENAMED
|
File without changes
|
{clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/gha/for-clams-team.md.template
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{clams_python-1.2.1 → clams_python-1.2.3}/clams/develop/templates/gha/workflows/publish.yml.template
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|