paralex 2.2.3__tar.gz → 2.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {paralex-2.2.3 → paralex-2.2.5}/PKG-INFO +1 -1
- {paralex-2.2.3 → paralex-2.2.5}/paralex/__init__.py +1 -1
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/datasets.md +2 -1
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/tutorial.md +94 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/explorer.py +16 -6
- {paralex-2.2.3 → paralex-2.2.5}/paralex/main.py +2 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/meta.py +6 -5
- {paralex-2.2.3 → paralex-2.2.5}/paralex/utils.py +1 -1
- {paralex-2.2.3 → paralex-2.2.5}/paralex.egg-info/PKG-INFO +1 -1
- {paralex-2.2.3 → paralex-2.2.5}/README.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/advanced-metadata.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/changelog.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/css/extra.css +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/examples.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/faq.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/graph-paralex.png +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/index.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/known-datasets.csv +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/language_stats.csv +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/long-form.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/mapframe.html +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/metadata.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/owl2html.xsl +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/paralex.png +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/paralex_blk.png +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/paralex_ontology.xml +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/principles.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/result.geojson +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/specs.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/specs.md.orig +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/specs.pdf +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/specs.tex +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/standard.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/gendoc.py +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/markdown.py +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/paths.py +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/standard/columns_spec.json +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/standard/data_sheet.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/standard/files_spec.json +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/standard/package_spec.json +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/standard/paralex.package.json +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/standard/paralex.package.json.orig +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/strategies.py +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/templates/__init__.py +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/templates/field.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/templates/package.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/templates/resource.md +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex/validate.py +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex.egg-info/SOURCES.txt +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex.egg-info/dependency_links.txt +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex.egg-info/entry_points.txt +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex.egg-info/requires.txt +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/paralex.egg-info/top_level.txt +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/setup.cfg +0 -0
- {paralex-2.2.3 → paralex-2.2.5}/setup.py +0 -0
|
@@ -16,8 +16,9 @@ paralex get <ZENODO_ID> --output <PATH> # Downloads the dataset with id ZENODO_
|
|
|
16
16
|
??? advice "Optional arguments"
|
|
17
17
|
`list` accepts the following arguments:
|
|
18
18
|
|
|
19
|
-
- `--iso <ISO>` followed by iso codes
|
|
19
|
+
- `--iso <ISO>` (followed by iso codes) filters the list of datasets and displays only matching datasets.
|
|
20
20
|
- `-o/--output <PATH>` saves the dataset list as a CSV table to `PATH`.
|
|
21
|
+
- `-u/--update` forces update of all the metadata. The command takes longer to complete.
|
|
21
22
|
|
|
22
23
|
`get` accepts the following arguments:
|
|
23
24
|
|
|
@@ -121,6 +121,41 @@ Validation and testing can be setup to run each time the data changes, if you tr
|
|
|
121
121
|
- paralex validate *.package.json
|
|
122
122
|
```
|
|
123
123
|
|
|
124
|
+
=== "Pipelines with github"
|
|
125
|
+
|
|
126
|
+
With github, create a plain text file (yaml formatted) in a
|
|
127
|
+
folder `.github/workflows/`, called for example `validation.yml`, with the following content:
|
|
128
|
+
|
|
129
|
+
``` yaml title=".github/workflows/validation.yml"
|
|
130
|
+
name: Validate
|
|
131
|
+
|
|
132
|
+
on:
|
|
133
|
+
push:
|
|
134
|
+
branches: [ "main" ]
|
|
135
|
+
|
|
136
|
+
jobs:
|
|
137
|
+
validate:
|
|
138
|
+
name: Validation
|
|
139
|
+
runs-on: ubuntu-latest
|
|
140
|
+
strategy:
|
|
141
|
+
fail-fast: false
|
|
142
|
+
|
|
143
|
+
steps:
|
|
144
|
+
- uses: actions/checkout@v3
|
|
145
|
+
- name: Set up Python 3.11
|
|
146
|
+
uses: actions/setup-python@v3
|
|
147
|
+
with:
|
|
148
|
+
python-version: 3.11
|
|
149
|
+
- name: Install dependencies
|
|
150
|
+
run: |
|
|
151
|
+
python -m pip install --upgrade pip
|
|
152
|
+
pip install frictionless paralex
|
|
153
|
+
- name: validate
|
|
154
|
+
run: |
|
|
155
|
+
frictionless validate *.package.json
|
|
156
|
+
paralex validate *.package.json
|
|
157
|
+
```
|
|
158
|
+
|
|
124
159
|
## Publishing
|
|
125
160
|
|
|
126
161
|
### The raw data files
|
|
@@ -219,12 +254,71 @@ To generate the site, add a pipeline file:
|
|
|
219
254
|
- master
|
|
220
255
|
```
|
|
221
256
|
|
|
257
|
+
=== "github pages"
|
|
258
|
+
|
|
259
|
+
With github, create a plain text file (yaml formatted) in a folder `.github/workflows/`, called for example `pages.yml`.
|
|
260
|
+
You need to activate github pages through actions in the repository setting on the github site.
|
|
261
|
+
The site will then be served at `https://<username>.github.io/<repository-name>`.
|
|
262
|
+
For more on github pages, see [the github pages docs](https://docs.github.com/en/pages).
|
|
263
|
+
|
|
264
|
+
``` yaml title=".github/workflows/pages.yml"
|
|
265
|
+
name: Pages
|
|
266
|
+
|
|
267
|
+
on:
|
|
268
|
+
push:
|
|
269
|
+
branches: [ "main" ]
|
|
270
|
+
|
|
271
|
+
permissions:
|
|
272
|
+
contents: read
|
|
273
|
+
pages: write
|
|
274
|
+
id-token: write
|
|
275
|
+
|
|
276
|
+
jobs:
|
|
277
|
+
build:
|
|
278
|
+
name: Deploy site
|
|
279
|
+
runs-on: ubuntu-latest
|
|
280
|
+
steps:
|
|
281
|
+
- name: Checkout main
|
|
282
|
+
uses: actions/checkout@v3
|
|
283
|
+
- name: Set up Python 3.11
|
|
284
|
+
uses: actions/setup-python@v3
|
|
285
|
+
with:
|
|
286
|
+
python-version: 3.11
|
|
287
|
+
- name: Install dependencies
|
|
288
|
+
run: |
|
|
289
|
+
python -m pip install --upgrade pip
|
|
290
|
+
pip install pandas mkdocs>=1.1.2 mkdocs-material mkdocs_paralex_plugin
|
|
291
|
+
- name: Setup Pages
|
|
292
|
+
id: pages
|
|
293
|
+
uses: actions/configure-pages@v5
|
|
294
|
+
- name: Build with mkdir
|
|
295
|
+
run: |
|
|
296
|
+
mkdir -p docs/
|
|
297
|
+
mkdocs build -d public/ --strict --verbose
|
|
298
|
+
- name: Upload artifact
|
|
299
|
+
uses: actions/upload-pages-artifact@v3
|
|
300
|
+
with:
|
|
301
|
+
path: ./public
|
|
302
|
+
|
|
303
|
+
deploy:
|
|
304
|
+
environment:
|
|
305
|
+
name: github-pages
|
|
306
|
+
url: ${{ steps.deployment.outputs.page_url }}
|
|
307
|
+
runs-on: ubuntu-latest
|
|
308
|
+
needs: build
|
|
309
|
+
steps:
|
|
310
|
+
- name: Deploy to GitHub Pages
|
|
311
|
+
id: deployment
|
|
312
|
+
uses: actions/deploy-pages@v4
|
|
313
|
+
```
|
|
314
|
+
|
|
222
315
|
Here are some examples of such generated sites:
|
|
223
316
|
|
|
224
317
|
- [Eesthetic: Estonian N and V](https://sbeniamine.gitlab.io/estonianparadigms/)
|
|
225
318
|
- [Aravelex: Arabic V](https://sbeniamine.gitlab.io/aravelex)
|
|
226
319
|
- [VeLePo: European Portuguese V](https://sbeniamine.gitlab.io/europeanportugueseverbs/)
|
|
227
320
|
- [Vlexique: French V](https://sbeniamine.gitlab.io/vlexique)
|
|
321
|
+
- [Ngkolmpu](https://maecarroll.github.io/Ngkolmpu-Paralex/) (github pages)
|
|
228
322
|
|
|
229
323
|
|
|
230
324
|
## Archiving
|
|
@@ -91,21 +91,27 @@ def _get_previous_data():
|
|
|
91
91
|
return None
|
|
92
92
|
|
|
93
93
|
|
|
94
|
-
def _gather_dataset_info(glottolog_data, save=True):
|
|
94
|
+
def _gather_dataset_info(glottolog_data, save=True, update=False):
|
|
95
95
|
"""
|
|
96
96
|
Gets information on all datasets and stores in a DataFrame.
|
|
97
97
|
|
|
98
98
|
Arguments:
|
|
99
99
|
save (bool): Whether to save the dataframe as CSV
|
|
100
100
|
or to return the object.
|
|
101
|
+
update (bool): Whether to force update of the data.
|
|
101
102
|
"""
|
|
102
103
|
old = _get_previous_data()
|
|
103
104
|
records = _get_zenodo_datasets()
|
|
104
105
|
data = []
|
|
105
|
-
fields = ['id', 'name', 'lang', 'paralex_version', 'version', 'contributors', 'title', 'doi', 'comment']
|
|
106
|
+
fields = ['id', 'permanent_id', 'name', 'lang', 'paralex_version', 'version', 'contributors', 'title', 'doi', 'comment']
|
|
106
107
|
for record in records:
|
|
107
108
|
rec_id = record['id']
|
|
108
|
-
if (
|
|
109
|
+
if (
|
|
110
|
+
update
|
|
111
|
+
or (old is None)
|
|
112
|
+
or not (rec_id in old.index)
|
|
113
|
+
or not ('permanent_id' in old.columns)):
|
|
114
|
+
|
|
109
115
|
package_infos = _get_package_infos(record)
|
|
110
116
|
|
|
111
117
|
######### TEMPORARY HACK TO FAKE UPDATED DATASETS
|
|
@@ -123,8 +129,9 @@ def _gather_dataset_info(glottolog_data, save=True):
|
|
|
123
129
|
|
|
124
130
|
def get_language(lg):
|
|
125
131
|
try:
|
|
126
|
-
return Language.match(lg)
|
|
132
|
+
return Language.match(str(lg))
|
|
127
133
|
except LanguageNotFoundError:
|
|
134
|
+
log.warning(f"Language not found for key: {lg}")
|
|
128
135
|
return None
|
|
129
136
|
|
|
130
137
|
log.info('Matching language names.')
|
|
@@ -176,6 +183,7 @@ def _get_package_file(record):
|
|
|
176
183
|
|
|
177
184
|
def _get_package_infos(record):
|
|
178
185
|
rec_id = record['id']
|
|
186
|
+
pid = record['conceptrecid']
|
|
179
187
|
doi = record["links"]["doi"]
|
|
180
188
|
title = record["title"]
|
|
181
189
|
log.info(f"Updating data for package '{title}'.")
|
|
@@ -183,6 +191,7 @@ def _get_package_infos(record):
|
|
|
183
191
|
if json_data is not None:
|
|
184
192
|
return {"title": title,
|
|
185
193
|
"id": rec_id,
|
|
194
|
+
'permanent_id': pid,
|
|
186
195
|
"doi": doi,
|
|
187
196
|
"name": json_data.get("name", json_data.get("title", unknown)),
|
|
188
197
|
"lang": json_data.get("languages_iso639", unknown),
|
|
@@ -193,6 +202,7 @@ def _get_package_infos(record):
|
|
|
193
202
|
|
|
194
203
|
return {"title": title,
|
|
195
204
|
"id": rec_id,
|
|
205
|
+
'permanent_id': pid,
|
|
196
206
|
"doi": doi,
|
|
197
207
|
"comment": "no valid metadata found"
|
|
198
208
|
}
|
|
@@ -203,11 +213,11 @@ def list_datasets(args):
|
|
|
203
213
|
Provides a list of all Paralex datasets that are available
|
|
204
214
|
"""
|
|
205
215
|
glottocode_data = _get_glottocode()
|
|
206
|
-
df = _gather_dataset_info(glottocode_data, save=False)
|
|
216
|
+
df = _gather_dataset_info(glottocode_data, save=False, update = args.update)
|
|
207
217
|
if args.iso:
|
|
208
218
|
log.info('Returning datasets for ISO code %s only', args.iso)
|
|
209
219
|
df = df[df.lang.isin(args.iso)]
|
|
210
|
-
df = df[['doi', 'lang', 'lang_name', 'family', 'version', 'paralex_version']]
|
|
220
|
+
df = df[['permanent_id', 'doi', 'lang', 'lang_name', 'family', 'version', 'paralex_version']]
|
|
211
221
|
if args.output:
|
|
212
222
|
df.to_csv(args.output)
|
|
213
223
|
print(df.to_markdown())
|
|
@@ -46,6 +46,8 @@ def parse_args():
|
|
|
46
46
|
help='Lists all available Paralex datasets.')
|
|
47
47
|
gather.add_argument("--iso", action="extend", nargs="+", type=str, help="Language codes to catch.")
|
|
48
48
|
gather.add_argument("-o", '--output', type=str, help="Output file. Saves the data to a CSV file.")
|
|
49
|
+
gather.add_argument("-u", '--update', help="Force update of all the dataset metadata.",
|
|
50
|
+
action="store_true", default=False)
|
|
49
51
|
gather.set_defaults(func=list_datasets)
|
|
50
52
|
|
|
51
53
|
# Subparser to download a dataset
|
|
@@ -209,11 +209,12 @@ def paralex_factory(title, files=None, tables=None, **kwargs):
|
|
|
209
209
|
del res_metadata["schema"]["foreignKeys"]
|
|
210
210
|
|
|
211
211
|
# Remove standard columns which are not present
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
212
|
+
if "schema" in res_metadata:
|
|
213
|
+
new_table_fields = list(res_metadata["schema"]["fields"])
|
|
214
|
+
for field in res_metadata["schema"]["fields"]:
|
|
215
|
+
if field["name"] not in cols:
|
|
216
|
+
new_table_fields.remove(field)
|
|
217
|
+
res_metadata["schema"]["fields"] = new_table_fields
|
|
217
218
|
|
|
218
219
|
r = fl.Resource(res_metadata, detector=fl.Detector(schema_sync=True))
|
|
219
220
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|