paralex 2.2.3__tar.gz → 2.2.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {paralex-2.2.3 → paralex-2.2.5}/PKG-INFO +1 -1
  2. {paralex-2.2.3 → paralex-2.2.5}/paralex/__init__.py +1 -1
  3. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/datasets.md +2 -1
  4. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/tutorial.md +94 -0
  5. {paralex-2.2.3 → paralex-2.2.5}/paralex/explorer.py +16 -6
  6. {paralex-2.2.3 → paralex-2.2.5}/paralex/main.py +2 -0
  7. {paralex-2.2.3 → paralex-2.2.5}/paralex/meta.py +6 -5
  8. {paralex-2.2.3 → paralex-2.2.5}/paralex/utils.py +1 -1
  9. {paralex-2.2.3 → paralex-2.2.5}/paralex.egg-info/PKG-INFO +1 -1
  10. {paralex-2.2.3 → paralex-2.2.5}/README.md +0 -0
  11. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/advanced-metadata.md +0 -0
  12. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/changelog.md +0 -0
  13. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/css/extra.css +0 -0
  14. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/examples.md +0 -0
  15. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/faq.md +0 -0
  16. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/graph-paralex.png +0 -0
  17. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/index.md +0 -0
  18. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/known-datasets.csv +0 -0
  19. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/language_stats.csv +0 -0
  20. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/long-form.md +0 -0
  21. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/mapframe.html +0 -0
  22. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/metadata.md +0 -0
  23. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/owl2html.xsl +0 -0
  24. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/paralex.png +0 -0
  25. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/paralex_blk.png +0 -0
  26. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/paralex_ontology.xml +0 -0
  27. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/principles.md +0 -0
  28. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/result.geojson +0 -0
  29. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/specs.md +0 -0
  30. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/specs.md.orig +0 -0
  31. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/specs.pdf +0 -0
  32. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/specs.tex +0 -0
  33. {paralex-2.2.3 → paralex-2.2.5}/paralex/docs/standard.md +0 -0
  34. {paralex-2.2.3 → paralex-2.2.5}/paralex/gendoc.py +0 -0
  35. {paralex-2.2.3 → paralex-2.2.5}/paralex/markdown.py +0 -0
  36. {paralex-2.2.3 → paralex-2.2.5}/paralex/paths.py +0 -0
  37. {paralex-2.2.3 → paralex-2.2.5}/paralex/standard/columns_spec.json +0 -0
  38. {paralex-2.2.3 → paralex-2.2.5}/paralex/standard/data_sheet.md +0 -0
  39. {paralex-2.2.3 → paralex-2.2.5}/paralex/standard/files_spec.json +0 -0
  40. {paralex-2.2.3 → paralex-2.2.5}/paralex/standard/package_spec.json +0 -0
  41. {paralex-2.2.3 → paralex-2.2.5}/paralex/standard/paralex.package.json +0 -0
  42. {paralex-2.2.3 → paralex-2.2.5}/paralex/standard/paralex.package.json.orig +0 -0
  43. {paralex-2.2.3 → paralex-2.2.5}/paralex/strategies.py +0 -0
  44. {paralex-2.2.3 → paralex-2.2.5}/paralex/templates/__init__.py +0 -0
  45. {paralex-2.2.3 → paralex-2.2.5}/paralex/templates/field.md +0 -0
  46. {paralex-2.2.3 → paralex-2.2.5}/paralex/templates/package.md +0 -0
  47. {paralex-2.2.3 → paralex-2.2.5}/paralex/templates/resource.md +0 -0
  48. {paralex-2.2.3 → paralex-2.2.5}/paralex/validate.py +0 -0
  49. {paralex-2.2.3 → paralex-2.2.5}/paralex.egg-info/SOURCES.txt +0 -0
  50. {paralex-2.2.3 → paralex-2.2.5}/paralex.egg-info/dependency_links.txt +0 -0
  51. {paralex-2.2.3 → paralex-2.2.5}/paralex.egg-info/entry_points.txt +0 -0
  52. {paralex-2.2.3 → paralex-2.2.5}/paralex.egg-info/requires.txt +0 -0
  53. {paralex-2.2.3 → paralex-2.2.5}/paralex.egg-info/top_level.txt +0 -0
  54. {paralex-2.2.3 → paralex-2.2.5}/setup.cfg +0 -0
  55. {paralex-2.2.3 → paralex-2.2.5}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: paralex
3
- Version: 2.2.3
3
+ Version: 2.2.5
4
4
  Summary: Standard for lexicons of inflected paradigms
5
5
  Author: Sacha Beniamine
6
6
  Classifier: Operating System :: OS Independent
@@ -1,4 +1,4 @@
1
- VERSION = "2.2.3"
1
+ VERSION = "2.2.5"
2
2
 
3
3
  from .paths import templates_path, standard_path, docs_path
4
4
  from .meta import paralex_factory
@@ -16,8 +16,9 @@ paralex get <ZENODO_ID> --output <PATH> # Downloads the dataset with id ZENODO_
16
16
  ??? advice "Optional arguments"
17
17
  `list` accepts the following arguments:
18
18
 
19
- - `--iso <ISO>` followed by iso codes filter the list of datasets and displays only matching datasets.
19
+ - `--iso <ISO>` (followed by iso codes) filters the list of datasets and displays only matching datasets.
20
20
  - `-o/--output <PATH>` saves the dataset list as a CSV table to `PATH`.
21
+ - `-u/--update` forces update of all the metadata. The command takes longer to complete.
21
22
 
22
23
  `get` accepts the following arguments:
23
24
 
@@ -121,6 +121,41 @@ Validation and testing can be setup to run each time the data changes, if you tr
121
121
  - paralex validate *.package.json
122
122
  ```
123
123
 
124
+ === "Pipelines with github"
125
+
126
+ With github, create a plain text file (yaml formatted) in a
127
+ folder `.github/workflows/`, called for example `validation.yml`, with the following content:
128
+
129
+ ``` yaml title=".github/workflows/validation.yml"
130
+ name: Validate
131
+
132
+ on:
133
+ push:
134
+ branches: [ "main" ]
135
+
136
+ jobs:
137
+ validate:
138
+ name: Validation
139
+ runs-on: ubuntu-latest
140
+ strategy:
141
+ fail-fast: false
142
+
143
+ steps:
144
+ - uses: actions/checkout@v3
145
+ - name: Set up Python 3.11
146
+ uses: actions/setup-python@v3
147
+ with:
148
+ python-version: 3.11
149
+ - name: Install dependencies
150
+ run: |
151
+ python -m pip install --upgrade pip
152
+ pip install frictionless paralex
153
+ - name: validate
154
+ run: |
155
+ frictionless validate *.package.json
156
+ paralex validate *.package.json
157
+ ```
158
+
124
159
  ## Publishing
125
160
 
126
161
  ### The raw data files
@@ -219,12 +254,71 @@ To generate the site, add a pipeline file:
219
254
  - master
220
255
  ```
221
256
 
257
+ === "github pages"
258
+
259
+ With github, create a plain text file (yaml formatted) in a folder `.github/workflows/`, called for example `pages.yml`.
260
+ You need to activate github pages through actions in the repository setting on the github site.
261
+ The site will then be served at `https://<username>.github.io/<repository-name>`.
262
+ For more on github pages, see [the github pages docs](https://docs.github.com/en/pages).
263
+
264
+ ``` yaml title=".github/workflows/pages.yml"
265
+ name: Pages
266
+
267
+ on:
268
+ push:
269
+ branches: [ "main" ]
270
+
271
+ permissions:
272
+ contents: read
273
+ pages: write
274
+ id-token: write
275
+
276
+ jobs:
277
+ build:
278
+ name: Deploy site
279
+ runs-on: ubuntu-latest
280
+ steps:
281
+ - name: Checkout main
282
+ uses: actions/checkout@v3
283
+ - name: Set up Python 3.11
284
+ uses: actions/setup-python@v3
285
+ with:
286
+ python-version: 3.11
287
+ - name: Install dependencies
288
+ run: |
289
+ python -m pip install --upgrade pip
290
+ pip install pandas mkdocs>=1.1.2 mkdocs-material mkdocs_paralex_plugin
291
+ - name: Setup Pages
292
+ id: pages
293
+ uses: actions/configure-pages@v5
294
+ - name: Build with mkdir
295
+ run: |
296
+ mkdir -p docs/
297
+ mkdocs build -d public/ --strict --verbose
298
+ - name: Upload artifact
299
+ uses: actions/upload-pages-artifact@v3
300
+ with:
301
+ path: ./public
302
+
303
+ deploy:
304
+ environment:
305
+ name: github-pages
306
+ url: ${{ steps.deployment.outputs.page_url }}
307
+ runs-on: ubuntu-latest
308
+ needs: build
309
+ steps:
310
+ - name: Deploy to GitHub Pages
311
+ id: deployment
312
+ uses: actions/deploy-pages@v4
313
+ ```
314
+
222
315
  Here are some examples of such generated sites:
223
316
 
224
317
  - [Eesthetic: Estonian N and V](https://sbeniamine.gitlab.io/estonianparadigms/)
225
318
  - [Aravelex: Arabic V](https://sbeniamine.gitlab.io/aravelex)
226
319
  - [VeLePo: European Portuguese V](https://sbeniamine.gitlab.io/europeanportugueseverbs/)
227
320
  - [Vlexique: French V](https://sbeniamine.gitlab.io/vlexique)
321
+ - [Ngkolmpu](https://maecarroll.github.io/Ngkolmpu-Paralex/) (github pages)
228
322
 
229
323
 
230
324
  ## Archiving
@@ -91,21 +91,27 @@ def _get_previous_data():
91
91
  return None
92
92
 
93
93
 
94
- def _gather_dataset_info(glottolog_data, save=True):
94
+ def _gather_dataset_info(glottolog_data, save=True, update=False):
95
95
  """
96
96
  Gets information on all datasets and stores in a DataFrame.
97
97
 
98
98
  Arguments:
99
99
  save (bool): Whether to save the dataframe as CSV
100
100
  or to return the object.
101
+ update (bool): Whether to force update of the data.
101
102
  """
102
103
  old = _get_previous_data()
103
104
  records = _get_zenodo_datasets()
104
105
  data = []
105
- fields = ['id', 'name', 'lang', 'paralex_version', 'version', 'contributors', 'title', 'doi', 'comment']
106
+ fields = ['id', 'permanent_id', 'name', 'lang', 'paralex_version', 'version', 'contributors', 'title', 'doi', 'comment']
106
107
  for record in records:
107
108
  rec_id = record['id']
108
- if (old is None) or not (rec_id in old.index):
109
+ if (
110
+ update
111
+ or (old is None)
112
+ or not (rec_id in old.index)
113
+ or not ('permanent_id' in old.columns)):
114
+
109
115
  package_infos = _get_package_infos(record)
110
116
 
111
117
  ######### TEMPORARY HACK TO FAKE UPDATED DATASETS
@@ -123,8 +129,9 @@ def _gather_dataset_info(glottolog_data, save=True):
123
129
 
124
130
  def get_language(lg):
125
131
  try:
126
- return Language.match(lg)
132
+ return Language.match(str(lg))
127
133
  except LanguageNotFoundError:
134
+ log.warning(f"Language not found for key: {lg}")
128
135
  return None
129
136
 
130
137
  log.info('Matching language names.')
@@ -176,6 +183,7 @@ def _get_package_file(record):
176
183
 
177
184
  def _get_package_infos(record):
178
185
  rec_id = record['id']
186
+ pid = record['conceptrecid']
179
187
  doi = record["links"]["doi"]
180
188
  title = record["title"]
181
189
  log.info(f"Updating data for package '{title}'.")
@@ -183,6 +191,7 @@ def _get_package_infos(record):
183
191
  if json_data is not None:
184
192
  return {"title": title,
185
193
  "id": rec_id,
194
+ 'permanent_id': pid,
186
195
  "doi": doi,
187
196
  "name": json_data.get("name", json_data.get("title", unknown)),
188
197
  "lang": json_data.get("languages_iso639", unknown),
@@ -193,6 +202,7 @@ def _get_package_infos(record):
193
202
 
194
203
  return {"title": title,
195
204
  "id": rec_id,
205
+ 'permanent_id': pid,
196
206
  "doi": doi,
197
207
  "comment": "no valid metadata found"
198
208
  }
@@ -203,11 +213,11 @@ def list_datasets(args):
203
213
  Provides a list of all Paralex datasets that are available
204
214
  """
205
215
  glottocode_data = _get_glottocode()
206
- df = _gather_dataset_info(glottocode_data, save=False)
216
+ df = _gather_dataset_info(glottocode_data, save=False, update = args.update)
207
217
  if args.iso:
208
218
  log.info('Returning datasets for ISO code %s only', args.iso)
209
219
  df = df[df.lang.isin(args.iso)]
210
- df = df[['doi', 'lang', 'lang_name', 'family', 'version', 'paralex_version']]
220
+ df = df[['permanent_id', 'doi', 'lang', 'lang_name', 'family', 'version', 'paralex_version']]
211
221
  if args.output:
212
222
  df.to_csv(args.output)
213
223
  print(df.to_markdown())
@@ -46,6 +46,8 @@ def parse_args():
46
46
  help='Lists all available Paralex datasets.')
47
47
  gather.add_argument("--iso", action="extend", nargs="+", type=str, help="Language codes to catch.")
48
48
  gather.add_argument("-o", '--output', type=str, help="Output file. Saves the data to a CSV file.")
49
+ gather.add_argument("-u", '--update', help="Force update of all the dataset metadata.",
50
+ action="store_true", default=False)
49
51
  gather.set_defaults(func=list_datasets)
50
52
 
51
53
  # Subparser to download a dataset
@@ -209,11 +209,12 @@ def paralex_factory(title, files=None, tables=None, **kwargs):
209
209
  del res_metadata["schema"]["foreignKeys"]
210
210
 
211
211
  # Remove standard columns which are not present
212
- new_table_fields = list(res_metadata["schema"]["fields"])
213
- for field in res_metadata["schema"]["fields"]:
214
- if field["name"] not in cols:
215
- new_table_fields.remove(field)
216
- res_metadata["schema"]["fields"] = new_table_fields
212
+ if "schema" in res_metadata:
213
+ new_table_fields = list(res_metadata["schema"]["fields"])
214
+ for field in res_metadata["schema"]["fields"]:
215
+ if field["name"] not in cols:
216
+ new_table_fields.remove(field)
217
+ res_metadata["schema"]["fields"] = new_table_fields
217
218
 
218
219
  r = fl.Resource(res_metadata, detector=fl.Detector(schema_sync=True))
219
220
 
@@ -52,4 +52,4 @@ def read_table_from_paths(paths, package, **kwargs):
52
52
  , axis=0)
53
53
 
54
54
  def absolute_path(package, path):
55
- return Path(package.basepath) / path
55
+ return Path(package.basepath or "./") / path
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: paralex
3
- Version: 2.2.3
3
+ Version: 2.2.5
4
4
  Summary: Standard for lexicons of inflected paradigms
5
5
  Author: Sacha Beniamine
6
6
  Classifier: Operating System :: OS Independent
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes