pyglottolog 5.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyglottolog/__init__.py +9 -0
- pyglottolog/__main__.py +88 -0
- pyglottolog/admin_commands/__init__.py +0 -0
- pyglottolog/admin_commands/bib.py +8 -0
- pyglottolog/admin_commands/check.py +197 -0
- pyglottolog/admin_commands/elpubbib.py +8 -0
- pyglottolog/admin_commands/evobib.py +17 -0
- pyglottolog/admin_commands/isobib.py +8 -0
- pyglottolog/admin_commands/isoretirements.py +8 -0
- pyglottolog/admin_commands/langindex.py +33 -0
- pyglottolog/admin_commands/languoidstats.py +28 -0
- pyglottolog/admin_commands/lff2tree.py +50 -0
- pyglottolog/admin_commands/release.py +28 -0
- pyglottolog/admin_commands/tree2lff.py +8 -0
- pyglottolog/admin_commands/update.py +15 -0
- pyglottolog/admin_commands/updatebib.py +31 -0
- pyglottolog/admin_commands/updatelinks.py +31 -0
- pyglottolog/admin_commands/updatemacroareas.py +26 -0
- pyglottolog/admin_commands/updatemetadata.py +19 -0
- pyglottolog/admin_commands/updatesources.py +36 -0
- pyglottolog/api.py +430 -0
- pyglottolog/cli_util.py +97 -0
- pyglottolog/commands/__init__.py +0 -0
- pyglottolog/commands/create.py +48 -0
- pyglottolog/commands/edit.py +28 -0
- pyglottolog/commands/htmlmap.py +160 -0
- pyglottolog/commands/iso2codes.py +37 -0
- pyglottolog/commands/langdatastats.py +26 -0
- pyglottolog/commands/show.py +51 -0
- pyglottolog/commands/tree.py +59 -0
- pyglottolog/config/__init__.py +298 -0
- pyglottolog/config/aes_sources.ini +29 -0
- pyglottolog/config/aes_status.ini +52 -0
- pyglottolog/config/document_types.ini +317 -0
- pyglottolog/config/editors.ini +33 -0
- pyglottolog/config/language_types.ini +45 -0
- pyglottolog/config/languoid_levels.ini +24 -0
- pyglottolog/config/macroareas.ini +32 -0
- pyglottolog/config/med_types.ini +30 -0
- pyglottolog/config/publication.ini +21 -0
- pyglottolog/data/africa.geojson.gz +0 -0
- pyglottolog/data/asia.geojson.gz +0 -0
- pyglottolog/data/europe.geojson.gz +0 -0
- pyglottolog/data/northamerica.geojson.gz +0 -0
- pyglottolog/data/oceania.geojson.gz +0 -0
- pyglottolog/data/southamerica.geojson.gz +0 -0
- pyglottolog/homelands.py +242 -0
- pyglottolog/iso.py +455 -0
- pyglottolog/languoids/__init__.py +23 -0
- pyglottolog/languoids/languoid.py +696 -0
- pyglottolog/languoids/models.py +412 -0
- pyglottolog/latex.py +538 -0
- pyglottolog/lff.py +396 -0
- pyglottolog/links/__init__.py +14 -0
- pyglottolog/links/util.py +100 -0
- pyglottolog/links/wikidata.py +68 -0
- pyglottolog/metadata.py +101 -0
- pyglottolog/monster.py +145 -0
- pyglottolog/references/__init__.py +11 -0
- pyglottolog/references/bibfiles.py +449 -0
- pyglottolog/references/bibfiles_db.py +972 -0
- pyglottolog/references/bibtex.py +189 -0
- pyglottolog/references/bibtex_undiacritic.py +26 -0
- pyglottolog/references/elpubbib.py +143 -0
- pyglottolog/references/evobib.py +71 -0
- pyglottolog/references/hhtypes.py +73 -0
- pyglottolog/references/isbns.py +115 -0
- pyglottolog/references/libmonster.py +500 -0
- pyglottolog/references/roman.py +42 -0
- pyglottolog/references/util.py +130 -0
- pyglottolog/templates/htmlmap/htmlmap.html +24 -0
- pyglottolog/templates/htmlmap/htmlmap.js +53 -0
- pyglottolog/util.py +146 -0
- pyglottolog-5.0.0.dist-info/METADATA +160 -0
- pyglottolog-5.0.0.dist-info/RECORD +79 -0
- pyglottolog-5.0.0.dist-info/WHEEL +5 -0
- pyglottolog-5.0.0.dist-info/entry_points.txt +3 -0
- pyglottolog-5.0.0.dist-info/licenses/LICENSE +201 -0
- pyglottolog-5.0.0.dist-info/top_level.txt +1 -0
pyglottolog/__init__.py
ADDED
pyglottolog/__main__.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Main command line interface of the pyglottolog package.
|
|
3
|
+
"""
|
|
4
|
+
import sys
|
|
5
|
+
import pathlib
|
|
6
|
+
import argparse
|
|
7
|
+
import contextlib
|
|
8
|
+
|
|
9
|
+
from cldfcatalog import Config, Catalog
|
|
10
|
+
from clldutils.clilib import register_subcommands, get_parser_and_subparsers, ParserError
|
|
11
|
+
from clldutils.loglib import Logging
|
|
12
|
+
|
|
13
|
+
import pyglottolog
|
|
14
|
+
from pyglottolog import Glottolog
|
|
15
|
+
import pyglottolog.commands
|
|
16
|
+
import pyglottolog.admin_commands
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def main(**kw):
|
|
20
|
+
"""Read-only commands."""
|
|
21
|
+
return _main(pyglottolog.commands, **kw)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def admin_main(**kw):
|
|
25
|
+
"""Admin commands, i.e. command that change the data."""
|
|
26
|
+
return _main(pyglottolog.admin_commands, **kw)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _main( # noqa: PLR0913,PLR0917
|
|
30
|
+
commands, args=None, catch_all=False, parsed_args=None, log=None, test=False):
|
|
31
|
+
try:
|
|
32
|
+
repos = Config.from_file().get_clone('glottolog')
|
|
33
|
+
except KeyError: # pragma: no cover
|
|
34
|
+
repos = pathlib.Path('.')
|
|
35
|
+
parser, subparsers = get_parser_and_subparsers('glottolog')
|
|
36
|
+
parser.add_argument(
|
|
37
|
+
'--repos',
|
|
38
|
+
help="clone of glottolog/glottolog",
|
|
39
|
+
default=repos,
|
|
40
|
+
type=pathlib.Path)
|
|
41
|
+
parser.add_argument(
|
|
42
|
+
'--repos-version',
|
|
43
|
+
help="version of repository data. Requires a git clone!",
|
|
44
|
+
default=None)
|
|
45
|
+
parser.add_argument(
|
|
46
|
+
'--pkg-dir',
|
|
47
|
+
help=argparse.SUPPRESS,
|
|
48
|
+
default=pathlib.Path(__file__).parent)
|
|
49
|
+
register_subcommands(subparsers, commands)
|
|
50
|
+
|
|
51
|
+
args = parsed_args or parser.parse_args(args=args)
|
|
52
|
+
args.test = test
|
|
53
|
+
|
|
54
|
+
if not hasattr(args, "main"):
|
|
55
|
+
parser.print_help()
|
|
56
|
+
return 1
|
|
57
|
+
|
|
58
|
+
with contextlib.ExitStack() as stack:
|
|
59
|
+
if not log: # pragma: no cover
|
|
60
|
+
stack.enter_context(Logging(args.log, level=args.log_level))
|
|
61
|
+
else:
|
|
62
|
+
args.log = log
|
|
63
|
+
if args.repos_version: # pragma: no cover
|
|
64
|
+
# If a specific version of the data is to be used, we make
|
|
65
|
+
# use of a Catalog as context manager:
|
|
66
|
+
stack.enter_context(Catalog(args.repos, tag=args.repos_version))
|
|
67
|
+
try:
|
|
68
|
+
args.repos = Glottolog(args.repos)
|
|
69
|
+
except Exception as e: # pylint: disable=W0718
|
|
70
|
+
print(e)
|
|
71
|
+
return _main(commands, args=[args._command, '-h']) # pylint: disable=W0212
|
|
72
|
+
args.log.info('glottolog/glottolog at %s', args.repos.repos)
|
|
73
|
+
try:
|
|
74
|
+
return args.main(args) or 0
|
|
75
|
+
except KeyboardInterrupt: # pragma: no cover
|
|
76
|
+
return 0
|
|
77
|
+
except ParserError as e:
|
|
78
|
+
print(e)
|
|
79
|
+
return _main(commands, args=[args._command, '-h']) # pylint: disable=W0212
|
|
80
|
+
except Exception as e: # pragma: no cover
|
|
81
|
+
if catch_all:
|
|
82
|
+
print(e)
|
|
83
|
+
return 1
|
|
84
|
+
raise
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
if __name__ == '__main__': # pragma: no cover
|
|
88
|
+
sys.exit(main() or 0)
|
|
File without changes
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Check the glottolog data for consistency.
|
|
3
|
+
"""
|
|
4
|
+
import logging
|
|
5
|
+
import pathlib
|
|
6
|
+
import collections
|
|
7
|
+
import dataclasses
|
|
8
|
+
|
|
9
|
+
import pyglottolog
|
|
10
|
+
from pyglottolog.languoids import Reference, LanguoidMapType, Languoid
|
|
11
|
+
from pyglottolog.util import message
|
|
12
|
+
from pyglottolog.cli_util import LanguoidStats
|
|
13
|
+
import pyglottolog.iso
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def register(parser): # pylint: disable=C0116
|
|
17
|
+
parser.add_argument(
|
|
18
|
+
'--bib-only',
|
|
19
|
+
action='store_true',
|
|
20
|
+
default=False,
|
|
21
|
+
)
|
|
22
|
+
parser.add_argument(
|
|
23
|
+
'--tree-only',
|
|
24
|
+
action='store_true',
|
|
25
|
+
default=False,
|
|
26
|
+
)
|
|
27
|
+
parser.add_argument(
|
|
28
|
+
'--old-languoids',
|
|
29
|
+
action='store_true',
|
|
30
|
+
default=False,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclasses.dataclass
|
|
35
|
+
class LanguoidLookup:
|
|
36
|
+
"""Easy languoid lookup by various attributes."""
|
|
37
|
+
by_id: LanguoidMapType = dataclasses.field(default_factory=dict)
|
|
38
|
+
by_hid: LanguoidMapType = dataclasses.field(default_factory=dict)
|
|
39
|
+
by_name: dict[str, set[Languoid]] = dataclasses.field(
|
|
40
|
+
default_factory=lambda: collections.defaultdict(set))
|
|
41
|
+
by_level: collections.Counter[str] = dataclasses.field(default_factory=collections.Counter)
|
|
42
|
+
by_category: collections.Counter[str] = dataclasses.field(default_factory=collections.Counter)
|
|
43
|
+
|
|
44
|
+
def update(self, lang: Languoid, levels, log: logging.Logger):
|
|
45
|
+
"""Update the lookup tables and counts, adding data about a new languoid."""
|
|
46
|
+
if lang.id in self.by_id:
|
|
47
|
+
log.error(message(
|
|
48
|
+
lang.id, f'duplicate glottocode\n{self.by_id[lang.id].dir}\n{lang.dir}'))
|
|
49
|
+
self.by_id[lang.id] = lang
|
|
50
|
+
|
|
51
|
+
self.by_level.update([lang.level.name])
|
|
52
|
+
if lang.level == levels.language:
|
|
53
|
+
self.by_category.update([lang.category])
|
|
54
|
+
|
|
55
|
+
self.by_name[lang.name].add(lang)
|
|
56
|
+
if lang.hid is not None:
|
|
57
|
+
if lang.hid in self.by_hid:
|
|
58
|
+
log.error(message(
|
|
59
|
+
lang.hid, f'duplicate hid\n{self.by_hid[lang.hid].dir}\n{lang.dir}'))
|
|
60
|
+
else:
|
|
61
|
+
self.by_hid[lang.hid] = lang
|
|
62
|
+
|
|
63
|
+
@staticmethod
|
|
64
|
+
def _log_counter(counter, name):
|
|
65
|
+
msg = [name + ':']
|
|
66
|
+
maxl = max(len(k) for k in counter.keys()) + 1
|
|
67
|
+
for k, lg in counter.most_common():
|
|
68
|
+
msg.append(('{0:<%s} {1:>8,}' % maxl).format(k + ':', lg))
|
|
69
|
+
msg.append(('{0:<%s} {1:>8,}' % maxl).format('', sum(list(counter.values()))))
|
|
70
|
+
print('\n'.join(msg))
|
|
71
|
+
|
|
72
|
+
def log(self):
|
|
73
|
+
"""Print summary stats about languoids in the repos."""
|
|
74
|
+
self._log_counter(self.by_level, 'Languoids by level')
|
|
75
|
+
self._log_counter(self.by_category, 'Languages by category')
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def run(args): # noqa: C901
|
|
79
|
+
def error(obj, msg):
|
|
80
|
+
args.log.error(message(obj, msg))
|
|
81
|
+
|
|
82
|
+
def warn(obj, msg): # pragma: no cover
|
|
83
|
+
args.log.warning(message(obj, msg))
|
|
84
|
+
|
|
85
|
+
def info(obj, msg):
|
|
86
|
+
args.log.info(message(obj, msg))
|
|
87
|
+
|
|
88
|
+
# Store all available BibTeX keys to check references in languoid info files.
|
|
89
|
+
refkeys = set()
|
|
90
|
+
for bibfile in args.repos.bibfiles:
|
|
91
|
+
if not args.tree_only:
|
|
92
|
+
# Check the BibTeX formatting.
|
|
93
|
+
bibfile.check(args.log)
|
|
94
|
+
refkeys = refkeys.union(bibfile.keys())
|
|
95
|
+
|
|
96
|
+
if args.bib_only:
|
|
97
|
+
return
|
|
98
|
+
|
|
99
|
+
iso = args.repos.iso
|
|
100
|
+
iso.log = args.log
|
|
101
|
+
info(iso, 'checking ISO codes')
|
|
102
|
+
info(args.repos, 'checking tree')
|
|
103
|
+
|
|
104
|
+
_check_config(args.repos, refkeys, args.log)
|
|
105
|
+
|
|
106
|
+
stats = LanguoidLookup()
|
|
107
|
+
for lang in args.repos.languoids():
|
|
108
|
+
stats.update(lang, args.repos.languoid_levels, args.log)
|
|
109
|
+
if iso and lang.iso:
|
|
110
|
+
iso.check_lang(args.repos, lang)
|
|
111
|
+
|
|
112
|
+
for lang in stats.by_id.values():
|
|
113
|
+
_check_lang_attrs(lang, args.repos, refkeys, args.log)
|
|
114
|
+
_check_level_consistency(lang, args.repos, stats.by_id, args.log)
|
|
115
|
+
|
|
116
|
+
if args.old_languoids:
|
|
117
|
+
if not args.repos.build_path(LanguoidStats.__fname__).exists():
|
|
118
|
+
raise ValueError() # pragma: no cover
|
|
119
|
+
old_languoid_stats = LanguoidStats.from_json(args.repos)
|
|
120
|
+
old_languoid_stats.check(stats.by_id, args.log)
|
|
121
|
+
|
|
122
|
+
iso.check_coverage()
|
|
123
|
+
|
|
124
|
+
bookkeeping_gc = args.repos.language_types.bookkeeping.pseudo_family_id
|
|
125
|
+
for name, gcs in sorted(stats.by_name.items()):
|
|
126
|
+
if len(gcs) > 1:
|
|
127
|
+
# duplicate names:
|
|
128
|
+
method = error
|
|
129
|
+
if len([1 for n in gcs if n.level != args.repos.languoid_levels.dialect]) <= 1:
|
|
130
|
+
# at most one of the languoids is not a dialect, just warn
|
|
131
|
+
method = warn # pragma: no cover
|
|
132
|
+
if len([1 for n in gcs if (not n.lineage) or (n.lineage[0][1] != bookkeeping_gc)]) <= 1:
|
|
133
|
+
# at most one of the languoids is not in bookkeping, just warn
|
|
134
|
+
method = warn # pragma: no cover
|
|
135
|
+
lgs = ', '.join(sorted([f'{n.id} <{n.level.name[0]}>' for n in gcs]))
|
|
136
|
+
method(name, f'duplicate name: {lgs}')
|
|
137
|
+
|
|
138
|
+
stats.log()
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _check_config(api, refkeys, log):
|
|
142
|
+
for p in pathlib.Path(pyglottolog.__file__).parent.joinpath('config').glob('*.ini'):
|
|
143
|
+
for obj in getattr(api, p.stem).values():
|
|
144
|
+
ref_id = getattr(obj, 'reference_id', None)
|
|
145
|
+
if ref_id and ref_id not in refkeys:
|
|
146
|
+
log.error(message(obj, f'missing reference: {ref_id}'))
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _check_lang_attrs(lang: Languoid, api, refkeys, log): # noqa: C901
|
|
150
|
+
if lang.latitude and not -90 <= lang.latitude <= 90:
|
|
151
|
+
log.error(message(lang, f'invalid latitude: {lang.latitude}'))
|
|
152
|
+
if lang.longitude and not -180 <= lang.longitude <= 180:
|
|
153
|
+
log.error(message(lang, f'invalid longitude: {lang.longitude}'))
|
|
154
|
+
|
|
155
|
+
assert isinstance(lang.countries, list)
|
|
156
|
+
assert isinstance(lang.macroareas, list)
|
|
157
|
+
assert (lang.timespan is None) or isinstance(lang.timespan, tuple)
|
|
158
|
+
|
|
159
|
+
if 'sources' in lang.cfg:
|
|
160
|
+
for ref in Reference.from_list(lang.cfg.getlist('sources', 'glottolog')):
|
|
161
|
+
if ref.key not in refkeys:
|
|
162
|
+
log.error(message(lang, f'missing source: {ref}'))
|
|
163
|
+
|
|
164
|
+
for attr in ['classification_comment', 'ethnologue_comment', 'endangerment']:
|
|
165
|
+
obj = getattr(lang, attr)
|
|
166
|
+
if obj:
|
|
167
|
+
obj.check(lang, refkeys, log)
|
|
168
|
+
|
|
169
|
+
if not lang.id.startswith('unun9') and lang.id not in api.glottocodes:
|
|
170
|
+
log.error(message(lang, 'unregistered glottocode'))
|
|
171
|
+
|
|
172
|
+
for attr in ['level', 'name']:
|
|
173
|
+
if not getattr(lang, attr):
|
|
174
|
+
log.error(message(lang, f'missing {attr}')) # pragma: no cover
|
|
175
|
+
|
|
176
|
+
timespan = lang.timespan
|
|
177
|
+
if timespan and not (
|
|
178
|
+
lang.endangerment and lang.endangerment.status == api.aes_status.extinct):
|
|
179
|
+
log.error(message(lang, 'timespan specified for non-extinct languoid')) # pragma: no cover
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _check_level_consistency(lang, api, languoids, log):
|
|
183
|
+
ancestors = lang.ancestors_from_nodemap(languoids)
|
|
184
|
+
children = lang.children_from_nodemap(languoids)
|
|
185
|
+
if lang.level == api.languoid_levels.language:
|
|
186
|
+
parent = ancestors[-1] if ancestors else None
|
|
187
|
+
if parent and parent.level != api.languoid_levels.family: # pragma: no cover
|
|
188
|
+
log.error(message(lang, f'invalid nesting of language under {parent.level}'))
|
|
189
|
+
for child in children:
|
|
190
|
+
if child.level != api.languoid_levels.dialect: # pragma: no cover
|
|
191
|
+
log.error(message(child, f'invalid nesting of {child.level} under language'))
|
|
192
|
+
elif lang.level == api.languoid_levels.family:
|
|
193
|
+
for d in lang.dir.iterdir():
|
|
194
|
+
if d.is_dir():
|
|
195
|
+
break
|
|
196
|
+
else:
|
|
197
|
+
log.error(message(lang, 'family without children')) # pragma: no cover
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command to update the evobib ref provider.
|
|
3
|
+
"""
|
|
4
|
+
from clldutils.clilib import PathType
|
|
5
|
+
|
|
6
|
+
from pyglottolog.references import evobib
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def register(parser): # pylint: disable=C0116
|
|
10
|
+
parser.add_argument(
|
|
11
|
+
'bib',
|
|
12
|
+
type=PathType(type='file'),
|
|
13
|
+
help='path to downloaded evobib-converted.bib')
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def run(args): # pragma: no cover # pylint: disable=C0116
|
|
17
|
+
evobib.update(args.bib, args.repos.bibfiles['evobib.bib'], args.log)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Create an index pages listing and linking to all languoids of a specified level.
|
|
3
|
+
"""
|
|
4
|
+
import collections
|
|
5
|
+
|
|
6
|
+
from clldutils.misc import slug
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def run(args): # pylint: disable=C0116
|
|
10
|
+
def make_index(level, languoids, repos):
|
|
11
|
+
fname = dict( # pylint: disable=R1735
|
|
12
|
+
language='languages', family='families', dialect='dialects')[level.name]
|
|
13
|
+
links = collections.defaultdict(dict)
|
|
14
|
+
for lang in languoids:
|
|
15
|
+
label = f'{lang.name} [{lang.id}]'
|
|
16
|
+
if lang.iso:
|
|
17
|
+
label += f'[{lang.iso}]'
|
|
18
|
+
links[slug(lang.name)[0]][label] = \
|
|
19
|
+
lang.fname.relative_to(repos.languoids_path())
|
|
20
|
+
|
|
21
|
+
with repos.languoids_path(fname + '.md').open('w', encoding='utf8') as fp:
|
|
22
|
+
fp.write(f'## {fname.capitalize()}\n\n')
|
|
23
|
+
fp.write(' '.join(f'[-{i.upper()}-]({fname}_{i}.md)' for i in sorted(links.keys())))
|
|
24
|
+
fp.write('\n')
|
|
25
|
+
|
|
26
|
+
for i, langs in links.items():
|
|
27
|
+
with repos.languoids_path(f'{fname}_{i}.md').open('w', encoding='utf8') as fp:
|
|
28
|
+
for label in sorted(langs.keys()):
|
|
29
|
+
fp.write(f'- [{label}]({langs[label]})\n')
|
|
30
|
+
|
|
31
|
+
langs = list(args.repos.languoids())
|
|
32
|
+
for level in args.repos.languoid_levels.values():
|
|
33
|
+
make_index(level, [lang for lang in langs if lang.level == level], args.repos)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Write simple languoid stats to build/languoids.json.
|
|
3
|
+
|
|
4
|
+
This is to allow comparison between two branches of the repos.
|
|
5
|
+
|
|
6
|
+
Intended usage:
|
|
7
|
+
```
|
|
8
|
+
git checkout <tag>
|
|
9
|
+
glottolog-admin languoidstats write
|
|
10
|
+
git checkout <other tag>
|
|
11
|
+
glottolog-admin languoidstats check
|
|
12
|
+
```
|
|
13
|
+
"""
|
|
14
|
+
from pyglottolog.cli_util import LanguoidStats
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def register(parser): # pylint: disable=C0116
|
|
18
|
+
parser.add_argument('cmd', choices=['write', 'check'])
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def run(args): # pylint: disable=C0116
|
|
22
|
+
if args.cmd == 'write':
|
|
23
|
+
stats = LanguoidStats.from_tree(args.repos)
|
|
24
|
+
stats.to_json(args.repos)
|
|
25
|
+
return
|
|
26
|
+
|
|
27
|
+
stats = LanguoidStats.from_json(args.repos)
|
|
28
|
+
stats.check({lg.id: lg for lg in args.repos.languoids()}, args.log)
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Recreate tree from lff.txt and dff.txt
|
|
3
|
+
"""
|
|
4
|
+
from pyglottolog import lff
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def run(args): # pylint: disable=C0116
|
|
8
|
+
try:
|
|
9
|
+
lff.lff2tree(args.repos, args.log)
|
|
10
|
+
except ValueError: # pragma: no cover
|
|
11
|
+
print("""
|
|
12
|
+
Something went wrong! Roll back inconsistent state running
|
|
13
|
+
|
|
14
|
+
rm -rf languoids
|
|
15
|
+
git checkout languoids
|
|
16
|
+
""")
|
|
17
|
+
raise
|
|
18
|
+
|
|
19
|
+
print("""
|
|
20
|
+
Run
|
|
21
|
+
|
|
22
|
+
git status
|
|
23
|
+
|
|
24
|
+
to inspect changes in the directory tree.
|
|
25
|
+
You can run
|
|
26
|
+
|
|
27
|
+
diff -rbB build/tree/ languoids/tree/
|
|
28
|
+
|
|
29
|
+
to inspect the changes in detail.
|
|
30
|
+
|
|
31
|
+
- To discard changes run
|
|
32
|
+
|
|
33
|
+
git checkout languoids/tree
|
|
34
|
+
|
|
35
|
+
- To commit and push changes, run
|
|
36
|
+
|
|
37
|
+
git add -A languoids/tree/...
|
|
38
|
+
|
|
39
|
+
for any newly created nodes listed under
|
|
40
|
+
|
|
41
|
+
# Untracked files:
|
|
42
|
+
# (use "git add <file>..." to include in what will be committed)
|
|
43
|
+
#
|
|
44
|
+
# languoids/tree/...
|
|
45
|
+
|
|
46
|
+
followed by
|
|
47
|
+
|
|
48
|
+
git commit -a -m"reason for change of classification"
|
|
49
|
+
git push origin
|
|
50
|
+
""")
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Write release info to .zenodo.json, CITATION.md and CONTRIBUTORS.md
|
|
3
|
+
"""
|
|
4
|
+
import shlex
|
|
5
|
+
import subprocess
|
|
6
|
+
|
|
7
|
+
import packaging.version
|
|
8
|
+
|
|
9
|
+
from pyglottolog.metadata import prepare_release
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def register(parser): # pylint: disable=C0116
|
|
13
|
+
parser.add_argument('--version', help="version number without leading 'v'", default=None)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def run(args): # pylint: disable=C0116
|
|
17
|
+
try: # pragma: no cover
|
|
18
|
+
branch = ''
|
|
19
|
+
out = subprocess.check_output(shlex.split(f"git -C {args.repos.repos} branch"))
|
|
20
|
+
for line in out.decode('utf8').split('\n'):
|
|
21
|
+
if line.startswith('*'):
|
|
22
|
+
branch = line[1:].strip()
|
|
23
|
+
assert branch.startswith('release')
|
|
24
|
+
except subprocess.CalledProcessError:
|
|
25
|
+
pass
|
|
26
|
+
version = getattr(args.repos.publication.zenodo, 'version', args.version)
|
|
27
|
+
assert not packaging.version.parse(version).is_prerelease, 'invalid release version number'
|
|
28
|
+
print(prepare_release(args.repos, version=version))
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Run all update commands in sequence.
|
|
3
|
+
"""
|
|
4
|
+
from pyglottolog.admin_commands import (
|
|
5
|
+
isoretirements, langindex, updatelinks, updatemetadata, updatesources, updatemacroareas,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def run(args): # pragma: no cover # pylint: disable=C0116
|
|
10
|
+
isoretirements.run(args)
|
|
11
|
+
langindex.run(args)
|
|
12
|
+
updatelinks.run(args)
|
|
13
|
+
updatemetadata.run(args)
|
|
14
|
+
updatesources.run(args)
|
|
15
|
+
updatemacroareas.run(args)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Update a refprovider bib from an updated source bib.
|
|
3
|
+
"""
|
|
4
|
+
from clldutils.clilib import PathType
|
|
5
|
+
from clldutils.path import TemporaryDirectory
|
|
6
|
+
|
|
7
|
+
from linglit.bibtex import merge
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def register(parser): # pylint: disable=C0116
|
|
11
|
+
parser.add_argument(
|
|
12
|
+
'bibkey',
|
|
13
|
+
choices=['cldf', 'benjamins', 'glossa', 'langsci', 'dplace', 'ldh', 'jocp'],
|
|
14
|
+
)
|
|
15
|
+
parser.add_argument(
|
|
16
|
+
'bib',
|
|
17
|
+
type=PathType(type='file'),
|
|
18
|
+
help='path to updated source bib')
|
|
19
|
+
parser.add_argument(
|
|
20
|
+
'--merge',
|
|
21
|
+
action='store_true',
|
|
22
|
+
default=False,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def run(args): # pragma: no cover # pylint: disable=C0116
|
|
27
|
+
with TemporaryDirectory() as tmp:
|
|
28
|
+
if args.merge:
|
|
29
|
+
merge(args.bib, tmp / args.bib.name, delatex=True)
|
|
30
|
+
args.bib = tmp / args.bib.name
|
|
31
|
+
args.repos.bibfiles[args.bibkey + '.bib'].update(args.bib, log=args.log)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Update links.
|
|
3
|
+
"""
|
|
4
|
+
# Make sure we import all link providers:
|
|
5
|
+
from pyglottolog.links import * # noqa: F401, F403 # pylint: disable=W0614,W0401
|
|
6
|
+
from pyglottolog.links.util import LinkProvider
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def providers(): # pylint: disable=C0116
|
|
10
|
+
return {cls.__name__.lower(): cls for cls in LinkProvider.__subclasses__()}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def register(parser): # pylint: disable=C0116
|
|
14
|
+
parser.add_argument('provider', nargs='*', help='|'.join(providers()))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def run(args): # pylint: disable=C0116
|
|
18
|
+
langs = list(args.repos.languoids())
|
|
19
|
+
updated = set()
|
|
20
|
+
for cls in LinkProvider.__subclasses__():
|
|
21
|
+
if getattr(cls, '__inactive__', False):
|
|
22
|
+
continue
|
|
23
|
+
name = cls.__name__.lower()
|
|
24
|
+
if (not getattr(args, 'provider', None)) or (name in args.provider):
|
|
25
|
+
args.log.info(f'updating {name} links ...')
|
|
26
|
+
i = 0
|
|
27
|
+
for i, lg in enumerate(cls(args.repos).iterupdated(langs), start=1):
|
|
28
|
+
lg.write_info()
|
|
29
|
+
updated.add(lg.id)
|
|
30
|
+
args.log.info(f'... {i} done')
|
|
31
|
+
print(f'{len(updated)} languoids updated')
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Add missing macroareas for dialects.
|
|
3
|
+
|
|
4
|
+
By default, dialects are assumed to share the macroarea of their language.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def run(args): # pylint: disable=C0116
|
|
9
|
+
langs = {lang.id: lang for lang in args.repos.languoids()}
|
|
10
|
+
updated = []
|
|
11
|
+
|
|
12
|
+
for dialect in langs.values():
|
|
13
|
+
if dialect.level == args.repos.languoid_levels['dialect'] and not dialect.macroareas:
|
|
14
|
+
for _, lid, _ in reversed(dialect.lineage):
|
|
15
|
+
language = langs[lid]
|
|
16
|
+
if language.level == args.repos.languoid_levels['language']:
|
|
17
|
+
break
|
|
18
|
+
else: # pragma: no cover
|
|
19
|
+
raise ValueError()
|
|
20
|
+
dialect.macroareas = list(language.macroareas)
|
|
21
|
+
updated.append(dialect)
|
|
22
|
+
|
|
23
|
+
for lang in updated:
|
|
24
|
+
lang.write_info()
|
|
25
|
+
|
|
26
|
+
print(f'{len(updated)} dialects updated')
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Update metadata.json from config/publication.ini
|
|
3
|
+
"""
|
|
4
|
+
import json
|
|
5
|
+
import configparser
|
|
6
|
+
|
|
7
|
+
from clldutils.metadata import Metadata
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def run(args): # pragma: no cover # pylint: disable=C0116
|
|
11
|
+
cfg = configparser.ConfigParser()
|
|
12
|
+
cfg.read(str(args.repos.path('config', 'publication.ini')))
|
|
13
|
+
md = {'title' if k == 'name' else k: v for k, v in cfg['web'].items()}
|
|
14
|
+
md['publisher.contact'] = cfg['web']['contact']
|
|
15
|
+
for k in ['publisher', 'license']:
|
|
16
|
+
md.update(**{k + '.' + kk: v for kk, v in cfg[k].items()})
|
|
17
|
+
|
|
18
|
+
args.repos.path('metadata.json').write_text(
|
|
19
|
+
json.dumps(Metadata.from_jsonld({}, defaults=md).to_jsonld(), indent=2))
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Update the [sources] section in languoid info files according to `lgcode` fields in bibfiles.
|
|
3
|
+
"""
|
|
4
|
+
import collections
|
|
5
|
+
|
|
6
|
+
from pyglottolog.languoids import Reference
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def run(args): # noqa: C901
|
|
10
|
+
langs = args.repos.languoids_by_code()
|
|
11
|
+
updated = []
|
|
12
|
+
sources = collections.defaultdict(set)
|
|
13
|
+
glrefs = set()
|
|
14
|
+
|
|
15
|
+
for entry in args.repos.bibfiles['hh.bib'].iterentries():
|
|
16
|
+
for lang in entry.languoids(langs)[0]:
|
|
17
|
+
sources[lang.id].add(f'hh:{entry.key}')
|
|
18
|
+
if entry.fields.get('glottolog_ref_id'):
|
|
19
|
+
glrefs.add(entry.fields['glottolog_ref_id'])
|
|
20
|
+
|
|
21
|
+
for bib in args.repos.bibfiles:
|
|
22
|
+
if bib.id == 'hh':
|
|
23
|
+
continue
|
|
24
|
+
for entry in bib.iterentries():
|
|
25
|
+
# If language associations have already been read from an equivalent record in hh.bib,
|
|
26
|
+
# we disregard the entry.
|
|
27
|
+
if entry.fields.get('glottolog_ref_id') not in glrefs:
|
|
28
|
+
for lang in entry.languoids(langs)[0]:
|
|
29
|
+
sources[lang.id].add(f'{bib.id}:{entry.key}')
|
|
30
|
+
|
|
31
|
+
for gc, refs in sources.items():
|
|
32
|
+
if refs != set(r.key for r in langs[gc].sources):
|
|
33
|
+
langs[gc].sources = [Reference(key=ref) for ref in sorted(refs)]
|
|
34
|
+
langs[gc].write_info()
|
|
35
|
+
updated.append(gc)
|
|
36
|
+
print(f'{len(updated)} languoids updated')
|