paralex 2.2.3__tar.gz → 2.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {paralex-2.2.3 → paralex-2.2.4}/PKG-INFO +1 -1
- {paralex-2.2.3 → paralex-2.2.4}/paralex/__init__.py +1 -1
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/datasets.md +2 -1
- {paralex-2.2.3 → paralex-2.2.4}/paralex/explorer.py +14 -5
- {paralex-2.2.3 → paralex-2.2.4}/paralex/main.py +2 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex.egg-info/PKG-INFO +1 -1
- {paralex-2.2.3 → paralex-2.2.4}/README.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/advanced-metadata.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/changelog.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/css/extra.css +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/examples.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/faq.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/graph-paralex.png +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/index.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/known-datasets.csv +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/language_stats.csv +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/long-form.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/mapframe.html +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/metadata.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/owl2html.xsl +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/paralex.png +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/paralex_blk.png +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/paralex_ontology.xml +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/principles.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/result.geojson +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/specs.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/specs.md.orig +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/specs.pdf +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/specs.tex +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/standard.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/docs/tutorial.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/gendoc.py +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/markdown.py +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/meta.py +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/paths.py +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/standard/columns_spec.json +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/standard/data_sheet.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/standard/files_spec.json +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/standard/package_spec.json +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/standard/paralex.package.json +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/standard/paralex.package.json.orig +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/strategies.py +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/templates/__init__.py +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/templates/field.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/templates/package.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/templates/resource.md +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/utils.py +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex/validate.py +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex.egg-info/SOURCES.txt +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex.egg-info/dependency_links.txt +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex.egg-info/entry_points.txt +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex.egg-info/requires.txt +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/paralex.egg-info/top_level.txt +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/setup.cfg +0 -0
- {paralex-2.2.3 → paralex-2.2.4}/setup.py +0 -0
|
@@ -16,8 +16,9 @@ paralex get <ZENODO_ID> --output <PATH> # Downloads the dataset with id ZENODO_
|
|
|
16
16
|
??? advice "Optional arguments"
|
|
17
17
|
`list` accepts the following arguments:
|
|
18
18
|
|
|
19
|
-
- `--iso <ISO>` followed by iso codes
|
|
19
|
+
- `--iso <ISO>` (followed by iso codes) filters the list of datasets and displays only matching datasets.
|
|
20
20
|
- `-o/--output <PATH>` saves the dataset list as a CSV table to `PATH`.
|
|
21
|
+
- `-u/--update` forces update of all the metadata. The command takes longer to complete.
|
|
21
22
|
|
|
22
23
|
`get` accepts the following arguments:
|
|
23
24
|
|
|
@@ -91,21 +91,27 @@ def _get_previous_data():
|
|
|
91
91
|
return None
|
|
92
92
|
|
|
93
93
|
|
|
94
|
-
def _gather_dataset_info(glottolog_data, save=True):
|
|
94
|
+
def _gather_dataset_info(glottolog_data, save=True, update=False):
|
|
95
95
|
"""
|
|
96
96
|
Gets information on all datasets and stores in a DataFrame.
|
|
97
97
|
|
|
98
98
|
Arguments:
|
|
99
99
|
save (bool): Whether to save the dataframe as CSV
|
|
100
100
|
or to return the object.
|
|
101
|
+
update (bool): Whether to force update of the data.
|
|
101
102
|
"""
|
|
102
103
|
old = _get_previous_data()
|
|
103
104
|
records = _get_zenodo_datasets()
|
|
104
105
|
data = []
|
|
105
|
-
fields = ['id', 'name', 'lang', 'paralex_version', 'version', 'contributors', 'title', 'doi', 'comment']
|
|
106
|
+
fields = ['id', 'permanent_id', 'name', 'lang', 'paralex_version', 'version', 'contributors', 'title', 'doi', 'comment']
|
|
106
107
|
for record in records:
|
|
107
108
|
rec_id = record['id']
|
|
108
|
-
if (
|
|
109
|
+
if (
|
|
110
|
+
update
|
|
111
|
+
or (old is None)
|
|
112
|
+
or not (rec_id in old.index)
|
|
113
|
+
or not ('permanent_id' in old.columns)):
|
|
114
|
+
|
|
109
115
|
package_infos = _get_package_infos(record)
|
|
110
116
|
|
|
111
117
|
######### TEMPORARY HACK TO FAKE UPDATED DATASETS
|
|
@@ -176,6 +182,7 @@ def _get_package_file(record):
|
|
|
176
182
|
|
|
177
183
|
def _get_package_infos(record):
|
|
178
184
|
rec_id = record['id']
|
|
185
|
+
pid = record['conceptrecid']
|
|
179
186
|
doi = record["links"]["doi"]
|
|
180
187
|
title = record["title"]
|
|
181
188
|
log.info(f"Updating data for package '{title}'.")
|
|
@@ -183,6 +190,7 @@ def _get_package_infos(record):
|
|
|
183
190
|
if json_data is not None:
|
|
184
191
|
return {"title": title,
|
|
185
192
|
"id": rec_id,
|
|
193
|
+
'permanent_id': pid,
|
|
186
194
|
"doi": doi,
|
|
187
195
|
"name": json_data.get("name", json_data.get("title", unknown)),
|
|
188
196
|
"lang": json_data.get("languages_iso639", unknown),
|
|
@@ -193,6 +201,7 @@ def _get_package_infos(record):
|
|
|
193
201
|
|
|
194
202
|
return {"title": title,
|
|
195
203
|
"id": rec_id,
|
|
204
|
+
'permanent_id': pid,
|
|
196
205
|
"doi": doi,
|
|
197
206
|
"comment": "no valid metadata found"
|
|
198
207
|
}
|
|
@@ -203,11 +212,11 @@ def list_datasets(args):
|
|
|
203
212
|
Provides a list of all Paralex datasets that are available
|
|
204
213
|
"""
|
|
205
214
|
glottocode_data = _get_glottocode()
|
|
206
|
-
df = _gather_dataset_info(glottocode_data, save=False)
|
|
215
|
+
df = _gather_dataset_info(glottocode_data, save=False, update = args.update)
|
|
207
216
|
if args.iso:
|
|
208
217
|
log.info('Returning datasets for ISO code %s only', args.iso)
|
|
209
218
|
df = df[df.lang.isin(args.iso)]
|
|
210
|
-
df = df[['doi', 'lang', 'lang_name', 'family', 'version', 'paralex_version']]
|
|
219
|
+
df = df[['permanent_id', 'doi', 'lang', 'lang_name', 'family', 'version', 'paralex_version']]
|
|
211
220
|
if args.output:
|
|
212
221
|
df.to_csv(args.output)
|
|
213
222
|
print(df.to_markdown())
|
|
@@ -46,6 +46,8 @@ def parse_args():
|
|
|
46
46
|
help='Lists all available Paralex datasets.')
|
|
47
47
|
gather.add_argument("--iso", action="extend", nargs="+", type=str, help="Language codes to catch.")
|
|
48
48
|
gather.add_argument("-o", '--output', type=str, help="Output file. Saves the data to a CSV file.")
|
|
49
|
+
gather.add_argument("-u", '--update', help="Force update of all the dataset metadata.",
|
|
50
|
+
action="store_true", default=False)
|
|
49
51
|
gather.set_defaults(func=list_datasets)
|
|
50
52
|
|
|
51
53
|
# Subparser to download a dataset
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|