refdatakit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- refdatakit/__init__.py +8 -0
- refdatakit/canonical_json.py +34 -0
- refdatakit/cli.py +32 -0
- refdatakit/country_codes.py +11 -0
- refdatakit/data/__init__.py +0 -0
- refdatakit/data/datasets/cities/by-country/BD.json +1 -0
- refdatakit/data/datasets/cities/by-country/BT.json +1 -0
- refdatakit/data/datasets/cities/by-country/LK.json +1 -0
- refdatakit/data/datasets/cities/by-country/MV.json +1 -0
- refdatakit/data/datasets/cities/by-country/NP.json +1 -0
- refdatakit/data/datasets/cities/by-country/SG.json +1 -0
- refdatakit/data/datasets/countries/data.json +1 -0
- refdatakit/data/datasets/institutions/by-country/BD.json +1 -0
- refdatakit/data/datasets/institutions/by-country/BT.json +1 -0
- refdatakit/data/datasets/institutions/by-country/LK.json +1 -0
- refdatakit/data/datasets/institutions/by-country/MV.json +1 -0
- refdatakit/data/datasets/institutions/by-country/NP.json +1 -0
- refdatakit/data/datasets/institutions/by-country/SG.json +1 -0
- refdatakit/data/datasets/manifest.json +1 -0
- refdatakit/data/datasets/subdivisions/by-country/BD.json +1 -0
- refdatakit/data/schemas/city.schema.json +84 -0
- refdatakit/data/schemas/country.schema.json +98 -0
- refdatakit/data/schemas/entity.schema.json +174 -0
- refdatakit/data/schemas/manifest.schema.json +94 -0
- refdatakit/data/schemas/subdivision.schema.json +65 -0
- refdatakit/data.py +48 -0
- refdatakit/validator.py +512 -0
- refdatakit-0.1.0.dist-info/METADATA +39 -0
- refdatakit-0.1.0.dist-info/RECORD +33 -0
- refdatakit-0.1.0.dist-info/WHEEL +5 -0
- refdatakit-0.1.0.dist-info/entry_points.txt +2 -0
- refdatakit-0.1.0.dist-info/licenses/LICENSE +21 -0
- refdatakit-0.1.0.dist-info/top_level.txt +1 -0
refdatakit/validator.py
ADDED
|
@@ -0,0 +1,512 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import re
|
|
6
|
+
import unicodedata
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path, PurePosixPath
|
|
9
|
+
from typing import Literal, NotRequired, TypedDict, cast
|
|
10
|
+
|
|
11
|
+
from jsonschema import Draft202012Validator, FormatChecker
|
|
12
|
+
from jsonschema.exceptions import SchemaError
|
|
13
|
+
|
|
14
|
+
from .canonical_json import canonical_json
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class DatasetValidationError(ValueError):
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class DatasetSource(TypedDict):
|
|
22
|
+
source_url: str
|
|
23
|
+
license: str
|
|
24
|
+
attribution: str
|
|
25
|
+
fields: list[str]
|
|
26
|
+
data_as_of: NotRequired[str]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class DatasetEntry(TypedDict):
|
|
30
|
+
path: str
|
|
31
|
+
source_url: str
|
|
32
|
+
license: str
|
|
33
|
+
attribution: str
|
|
34
|
+
sha256: str
|
|
35
|
+
data_as_of: NotRequired[str]
|
|
36
|
+
additional_sources: NotRequired[list[DatasetSource]]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class DatasetVocabularies(TypedDict):
|
|
40
|
+
entity_tags: list[str]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class DatasetManifest(TypedDict):
|
|
44
|
+
datasets: list[DatasetEntry]
|
|
45
|
+
vocabularies: DatasetVocabularies
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class DatasetDefinition:
|
|
50
|
+
domain: Literal["country", "subdivision", "city", "entity"]
|
|
51
|
+
schema: str
|
|
52
|
+
country_code: str | None = None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class DatasetRecords:
|
|
57
|
+
definition: DatasetDefinition
|
|
58
|
+
entry: DatasetEntry
|
|
59
|
+
records: list[object]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
DATASET_PATH_PATTERNS: tuple[
|
|
63
|
+
tuple[re.Pattern[str], Literal["country", "subdivision", "city", "entity"], str],
|
|
64
|
+
...,
|
|
65
|
+
] = (
|
|
66
|
+
(
|
|
67
|
+
re.compile(r"datasets/countries/data\.json"),
|
|
68
|
+
"country",
|
|
69
|
+
"schemas/country.schema.json",
|
|
70
|
+
),
|
|
71
|
+
(
|
|
72
|
+
re.compile(r"datasets/subdivisions/by-country/([A-Z]{2})\.json"),
|
|
73
|
+
"subdivision",
|
|
74
|
+
"schemas/subdivision.schema.json",
|
|
75
|
+
),
|
|
76
|
+
(
|
|
77
|
+
re.compile(r"datasets/cities/by-country/([A-Z]{2})\.json"),
|
|
78
|
+
"city",
|
|
79
|
+
"schemas/city.schema.json",
|
|
80
|
+
),
|
|
81
|
+
(
|
|
82
|
+
re.compile(r"datasets/institutions/by-country/([A-Z]{2})\.json"),
|
|
83
|
+
"entity",
|
|
84
|
+
"schemas/entity.schema.json",
|
|
85
|
+
),
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
IDENTITY_FIELDS: dict[str, str] = {
|
|
90
|
+
"country": "code",
|
|
91
|
+
"subdivision": "code",
|
|
92
|
+
"city": "id",
|
|
93
|
+
"entity": "id",
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _repository_file(root: Path, relative_path: str) -> Path:
|
|
98
|
+
path = PurePosixPath(relative_path)
|
|
99
|
+
if path.is_absolute() or ".." in path.parts or "\\" in relative_path:
|
|
100
|
+
raise DatasetValidationError(f"Expected a repository-relative path: {relative_path}")
|
|
101
|
+
|
|
102
|
+
try:
|
|
103
|
+
resolved_path = (root / Path(*path.parts)).resolve(strict=True)
|
|
104
|
+
except FileNotFoundError as error:
|
|
105
|
+
raise DatasetValidationError(f"Missing repository file: {relative_path}") from error
|
|
106
|
+
try:
|
|
107
|
+
resolved_path.relative_to(root)
|
|
108
|
+
except ValueError as error:
|
|
109
|
+
raise DatasetValidationError(f"Path resolves outside the repository: {relative_path}") from error
|
|
110
|
+
return resolved_path
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _read_json(path: Path) -> object:
|
|
114
|
+
try:
|
|
115
|
+
return json.loads(path.read_bytes())
|
|
116
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
|
117
|
+
raise DatasetValidationError(f"Invalid JSON in {path}: {error}") from error
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _validate_instance(validator: Draft202012Validator, instance: object, description: str) -> None:
|
|
121
|
+
errors = sorted(
|
|
122
|
+
validator.iter_errors(instance),
|
|
123
|
+
key=lambda error: tuple(str(part) for part in error.absolute_path),
|
|
124
|
+
)
|
|
125
|
+
if errors:
|
|
126
|
+
details = "; ".join(
|
|
127
|
+
f"/{'/'.join(str(part) for part in error.absolute_path)} {error.message}"
|
|
128
|
+
for error in errors
|
|
129
|
+
)
|
|
130
|
+
raise DatasetValidationError(f"{description} is invalid: {details}")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _schema_validator(schema: object, description: str) -> Draft202012Validator:
|
|
134
|
+
try:
|
|
135
|
+
Draft202012Validator.check_schema(schema)
|
|
136
|
+
except SchemaError as error:
|
|
137
|
+
raise DatasetValidationError(
|
|
138
|
+
f"{description} is not a valid JSON Schema: {error.message}"
|
|
139
|
+
) from error
|
|
140
|
+
return Draft202012Validator(schema, format_checker=FormatChecker())
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _schema_with_entity_tags(schema: object, entity_tags: list[str]) -> object:
|
|
144
|
+
if (
|
|
145
|
+
not isinstance(schema, dict)
|
|
146
|
+
or not isinstance(schema.get("$defs"), dict)
|
|
147
|
+
or not isinstance(schema["$defs"].get("tag"), dict)
|
|
148
|
+
):
|
|
149
|
+
raise DatasetValidationError("schemas/entity.schema.json must define $defs.tag")
|
|
150
|
+
|
|
151
|
+
return {
|
|
152
|
+
**schema,
|
|
153
|
+
"$defs": {
|
|
154
|
+
**schema["$defs"],
|
|
155
|
+
"tag": {
|
|
156
|
+
**schema["$defs"]["tag"],
|
|
157
|
+
"enum": entity_tags,
|
|
158
|
+
},
|
|
159
|
+
},
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _required_string(record: object, field: str, description: str) -> str:
|
|
164
|
+
if not isinstance(record, dict) or not isinstance(record.get(field), str):
|
|
165
|
+
raise DatasetValidationError(f"{description} must have a string {field}")
|
|
166
|
+
return cast(str, record[field])
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _optional_string(record: object, field: str, description: str) -> str | None:
|
|
170
|
+
if not isinstance(record, dict):
|
|
171
|
+
raise DatasetValidationError(f"{description} must be an object")
|
|
172
|
+
|
|
173
|
+
value = record.get(field)
|
|
174
|
+
if value is None:
|
|
175
|
+
return None
|
|
176
|
+
if not isinstance(value, str):
|
|
177
|
+
raise DatasetValidationError(f"{description} {field} must be a string")
|
|
178
|
+
return value
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _official_non_english_languages(record: object, description: str) -> set[str]:
|
|
182
|
+
if not isinstance(record, dict) or not isinstance(record.get("official_languages"), list):
|
|
183
|
+
raise DatasetValidationError(f"{description} must have an official_languages array")
|
|
184
|
+
|
|
185
|
+
languages: set[str] = set()
|
|
186
|
+
for language in record["official_languages"]:
|
|
187
|
+
if not isinstance(language, str):
|
|
188
|
+
raise DatasetValidationError(f"{description} official_languages must contain strings")
|
|
189
|
+
code = language.split("-", maxsplit=1)[0].lower()
|
|
190
|
+
if not re.fullmatch(r"[a-z]{2,3}", code):
|
|
191
|
+
raise DatasetValidationError(
|
|
192
|
+
f"{description} official language {language} has an invalid code"
|
|
193
|
+
)
|
|
194
|
+
if code != "en":
|
|
195
|
+
languages.add(code)
|
|
196
|
+
return languages
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _validate_entity_language_names(
|
|
200
|
+
record: object,
|
|
201
|
+
official_languages: set[str],
|
|
202
|
+
description: str,
|
|
203
|
+
) -> None:
|
|
204
|
+
if (
|
|
205
|
+
not isinstance(record, dict)
|
|
206
|
+
or not isinstance(record.get("names"), dict)
|
|
207
|
+
or not isinstance(record["names"].get("languages"), dict)
|
|
208
|
+
):
|
|
209
|
+
raise DatasetValidationError(f"{description} names.languages must be an object")
|
|
210
|
+
|
|
211
|
+
name_languages = {language for language in official_languages if len(language) == 2}
|
|
212
|
+
actual_languages = set(record["names"]["languages"])
|
|
213
|
+
unexpected = sorted(actual_languages - name_languages)
|
|
214
|
+
if unexpected:
|
|
215
|
+
raise DatasetValidationError(
|
|
216
|
+
f"{description} names.languages contains non-official languages: {', '.join(unexpected)}"
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
aliases = record.get("localized_aliases", [])
|
|
220
|
+
if not isinstance(aliases, list):
|
|
221
|
+
raise DatasetValidationError(f"{description} localized_aliases must be an array")
|
|
222
|
+
for index, alias in enumerate(aliases):
|
|
223
|
+
language = _required_string(
|
|
224
|
+
alias,
|
|
225
|
+
"language",
|
|
226
|
+
f"{description} localized_aliases[{index}]",
|
|
227
|
+
).split("-", maxsplit=1)[0].lower()
|
|
228
|
+
if language != "en" and language not in official_languages:
|
|
229
|
+
raise DatasetValidationError(
|
|
230
|
+
f"{description} localized_aliases[{index}] language {language} is not official"
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _definition_for_path(path: str) -> DatasetDefinition:
|
|
235
|
+
for pattern, domain, schema in DATASET_PATH_PATTERNS:
|
|
236
|
+
match = pattern.fullmatch(path)
|
|
237
|
+
if match:
|
|
238
|
+
country_code = match.group(1) if domain != "country" else None
|
|
239
|
+
return DatasetDefinition(domain=domain, schema=schema, country_code=country_code)
|
|
240
|
+
raise DatasetValidationError(f"Unsupported dataset path: {path}")
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _list_dataset_files(root: Path) -> list[str]:
|
|
244
|
+
datasets_directory = _repository_file(root, "datasets")
|
|
245
|
+
files = []
|
|
246
|
+
for path in datasets_directory.rglob("*.json"):
|
|
247
|
+
relative_path = path.relative_to(root).as_posix()
|
|
248
|
+
if relative_path != "datasets/manifest.json":
|
|
249
|
+
files.append(relative_path)
|
|
250
|
+
return sorted(files)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _validate_record_partition(
|
|
254
|
+
record: object,
|
|
255
|
+
definition: DatasetDefinition,
|
|
256
|
+
description: str,
|
|
257
|
+
) -> None:
|
|
258
|
+
if definition.country_code is None:
|
|
259
|
+
return
|
|
260
|
+
if _required_string(record, "country_code", description) != definition.country_code:
|
|
261
|
+
raise DatasetValidationError(
|
|
262
|
+
f"{description} country_code does not match {definition.country_code}"
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _english_sort_key(
|
|
267
|
+
record: object,
|
|
268
|
+
definition: DatasetDefinition,
|
|
269
|
+
description: str,
|
|
270
|
+
) -> tuple[str, str]:
|
|
271
|
+
if definition.domain == "entity":
|
|
272
|
+
if not isinstance(record, dict) or not isinstance(record.get("names"), dict):
|
|
273
|
+
raise DatasetValidationError(f"{description} must have an object names")
|
|
274
|
+
english_name = _required_string(record["names"], "en", f"{description} names")
|
|
275
|
+
else:
|
|
276
|
+
english_name = _required_string(record, "name", description)
|
|
277
|
+
identity = _required_string(record, IDENTITY_FIELDS[definition.domain], description)
|
|
278
|
+
return (unicodedata.normalize("NFC", english_name).lower(), identity)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _english_sort_warning(dataset: DatasetRecords) -> str | None:
|
|
282
|
+
previous: tuple[str, str] | None = None
|
|
283
|
+
for index, record in enumerate(dataset.records):
|
|
284
|
+
description = f"{dataset.entry['path']}[{index}]"
|
|
285
|
+
english_name, identity = _english_sort_key(record, dataset.definition, description)
|
|
286
|
+
current = (english_name, identity)
|
|
287
|
+
if previous is not None and previous > current:
|
|
288
|
+
return f"{dataset.entry['path']} is not sorted by English name at record {index}"
|
|
289
|
+
previous = current
|
|
290
|
+
return None
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _validate_cross_dataset_references(datasets: list[DatasetRecords]) -> None:
|
|
294
|
+
countries: set[str] = set()
|
|
295
|
+
country_languages: dict[str, set[str]] = {}
|
|
296
|
+
subdivisions: dict[str, set[str]] = {}
|
|
297
|
+
cities: dict[str, set[str]] = {}
|
|
298
|
+
record_identities: set[tuple[str, str, str]] = set()
|
|
299
|
+
|
|
300
|
+
for dataset in datasets:
|
|
301
|
+
for index, record in enumerate(dataset.records):
|
|
302
|
+
description = f"{dataset.entry['path']}[{index}]"
|
|
303
|
+
domain = dataset.definition.domain
|
|
304
|
+
identity_field = IDENTITY_FIELDS[domain]
|
|
305
|
+
identity = _required_string(record, identity_field, description)
|
|
306
|
+
country_code = (
|
|
307
|
+
""
|
|
308
|
+
if domain == "country"
|
|
309
|
+
else _required_string(record, "country_code", description)
|
|
310
|
+
)
|
|
311
|
+
scoped_identity = (domain, country_code, identity)
|
|
312
|
+
if scoped_identity in record_identities:
|
|
313
|
+
raise DatasetValidationError(
|
|
314
|
+
f"{description} duplicates {identity_field} {identity}"
|
|
315
|
+
)
|
|
316
|
+
record_identities.add(scoped_identity)
|
|
317
|
+
|
|
318
|
+
if domain == "country":
|
|
319
|
+
countries.add(identity)
|
|
320
|
+
country_languages[identity] = _official_non_english_languages(record, description)
|
|
321
|
+
elif domain == "subdivision":
|
|
322
|
+
subdivisions.setdefault(country_code, set()).add(identity)
|
|
323
|
+
elif domain == "city":
|
|
324
|
+
cities.setdefault(country_code, set()).add(identity)
|
|
325
|
+
|
|
326
|
+
for dataset in datasets:
|
|
327
|
+
for index, record in enumerate(dataset.records):
|
|
328
|
+
description = f"{dataset.entry['path']}[{index}]"
|
|
329
|
+
domain = dataset.definition.domain
|
|
330
|
+
if domain == "country":
|
|
331
|
+
continue
|
|
332
|
+
|
|
333
|
+
country_code = _required_string(record, "country_code", description)
|
|
334
|
+
if country_code not in countries:
|
|
335
|
+
raise DatasetValidationError(
|
|
336
|
+
f"{description} references unknown country {country_code}"
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
subdivision_code = _optional_string(record, "subdivision_code", description)
|
|
340
|
+
if (
|
|
341
|
+
subdivision_code is not None
|
|
342
|
+
and subdivision_code not in subdivisions.get(country_code, set())
|
|
343
|
+
):
|
|
344
|
+
raise DatasetValidationError(
|
|
345
|
+
f"{description} references unknown subdivision {subdivision_code}"
|
|
346
|
+
)
|
|
347
|
+
|
|
348
|
+
if domain == "subdivision":
|
|
349
|
+
code = _required_string(record, "code", description)
|
|
350
|
+
parent_code = _optional_string(record, "parent_code", description)
|
|
351
|
+
if parent_code == code:
|
|
352
|
+
raise DatasetValidationError(f"{description} cannot be its own parent")
|
|
353
|
+
if (
|
|
354
|
+
parent_code is not None
|
|
355
|
+
and parent_code not in subdivisions.get(country_code, set())
|
|
356
|
+
):
|
|
357
|
+
raise DatasetValidationError(
|
|
358
|
+
f"{description} references unknown parent subdivision {parent_code}"
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
if domain == "entity":
|
|
362
|
+
city_id = _optional_string(record, "city_id", description)
|
|
363
|
+
if city_id is not None and city_id not in cities.get(country_code, set()):
|
|
364
|
+
raise DatasetValidationError(f"{description} references unknown city {city_id}")
|
|
365
|
+
_validate_entity_language_names(
|
|
366
|
+
record,
|
|
367
|
+
country_languages.get(country_code, set()),
|
|
368
|
+
description,
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def sort_repository(root_directory: Path) -> None:
|
|
373
|
+
root = root_directory.resolve(strict=True)
|
|
374
|
+
manifest_path = _repository_file(root, "datasets/manifest.json")
|
|
375
|
+
manifest_schema_path = _repository_file(root, "schemas/manifest.schema.json")
|
|
376
|
+
manifest_value = _read_json(manifest_path)
|
|
377
|
+
manifest_validator = _schema_validator(
|
|
378
|
+
_read_json(manifest_schema_path),
|
|
379
|
+
"schemas/manifest.schema.json",
|
|
380
|
+
)
|
|
381
|
+
_validate_instance(manifest_validator, manifest_value, "datasets/manifest.json")
|
|
382
|
+
manifest = cast(DatasetManifest, manifest_value)
|
|
383
|
+
|
|
384
|
+
dataset_paths: set[str] = set()
|
|
385
|
+
datasets: list[DatasetRecords] = []
|
|
386
|
+
for entry in manifest["datasets"]:
|
|
387
|
+
if entry["path"] in dataset_paths:
|
|
388
|
+
raise DatasetValidationError(f"Duplicate dataset path: {entry['path']}")
|
|
389
|
+
dataset_paths.add(entry["path"])
|
|
390
|
+
|
|
391
|
+
definition = _definition_for_path(entry["path"])
|
|
392
|
+
data_path = _repository_file(root, entry["path"])
|
|
393
|
+
schema_path = _repository_file(root, definition.schema)
|
|
394
|
+
records_value = _read_json(data_path)
|
|
395
|
+
if not isinstance(records_value, list):
|
|
396
|
+
raise DatasetValidationError(f"{entry['path']} must contain a JSON array")
|
|
397
|
+
|
|
398
|
+
schema = _read_json(schema_path)
|
|
399
|
+
if definition.domain == "entity":
|
|
400
|
+
schema = _schema_with_entity_tags(schema, manifest["vocabularies"]["entity_tags"])
|
|
401
|
+
record_validator = _schema_validator(schema, definition.schema)
|
|
402
|
+
for index, record in enumerate(records_value):
|
|
403
|
+
description = f"{entry['path']}[{index}]"
|
|
404
|
+
_validate_instance(record_validator, record, description)
|
|
405
|
+
_validate_record_partition(record, definition, description)
|
|
406
|
+
datasets.append(DatasetRecords(definition, entry, records_value))
|
|
407
|
+
|
|
408
|
+
existing_files = _list_dataset_files(root)
|
|
409
|
+
missing_manifest_entries = sorted(set(existing_files) - dataset_paths)
|
|
410
|
+
missing_dataset_files = sorted(dataset_paths - set(existing_files))
|
|
411
|
+
if missing_manifest_entries:
|
|
412
|
+
raise DatasetValidationError(
|
|
413
|
+
f"Dataset files missing from manifest: {', '.join(missing_manifest_entries)}"
|
|
414
|
+
)
|
|
415
|
+
if missing_dataset_files:
|
|
416
|
+
raise DatasetValidationError(
|
|
417
|
+
f"Manifest paths do not exist as dataset files: {', '.join(missing_dataset_files)}"
|
|
418
|
+
)
|
|
419
|
+
|
|
420
|
+
_validate_cross_dataset_references(datasets)
|
|
421
|
+
for dataset in datasets:
|
|
422
|
+
dataset.records.sort(key=lambda record: _english_sort_key(
|
|
423
|
+
record,
|
|
424
|
+
dataset.definition,
|
|
425
|
+
dataset.entry["path"],
|
|
426
|
+
))
|
|
427
|
+
contents = canonical_json(dataset.records)
|
|
428
|
+
data_path = _repository_file(root, dataset.entry["path"])
|
|
429
|
+
data_path.write_bytes(contents)
|
|
430
|
+
dataset.entry["sha256"] = hashlib.sha256(contents).hexdigest()
|
|
431
|
+
|
|
432
|
+
manifest_path.write_bytes(canonical_json(manifest))
|
|
433
|
+
validate_repository(root)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def validate_repository(root_directory: Path) -> list[str]:
|
|
437
|
+
root = root_directory.resolve(strict=True)
|
|
438
|
+
manifest_path = _repository_file(root, "datasets/manifest.json")
|
|
439
|
+
manifest_schema_path = _repository_file(root, "schemas/manifest.schema.json")
|
|
440
|
+
manifest_contents = manifest_path.read_bytes()
|
|
441
|
+
manifest_value = _read_json(manifest_path)
|
|
442
|
+
try:
|
|
443
|
+
is_canonical_manifest = canonical_json(manifest_value) == manifest_contents
|
|
444
|
+
except (TypeError, ValueError) as error:
|
|
445
|
+
raise DatasetValidationError(f"Manifest cannot be serialized canonically: {error}") from error
|
|
446
|
+
if not is_canonical_manifest:
|
|
447
|
+
raise DatasetValidationError("datasets/manifest.json is not canonically serialized")
|
|
448
|
+
|
|
449
|
+
manifest_schema = _read_json(manifest_schema_path)
|
|
450
|
+
manifest_validator = _schema_validator(manifest_schema, "schemas/manifest.schema.json")
|
|
451
|
+
_validate_instance(manifest_validator, manifest_value, "datasets/manifest.json")
|
|
452
|
+
manifest = cast(DatasetManifest, manifest_value)
|
|
453
|
+
|
|
454
|
+
dataset_paths: set[str] = set()
|
|
455
|
+
validated_datasets: list[DatasetRecords] = []
|
|
456
|
+
for entry in manifest["datasets"]:
|
|
457
|
+
if entry["path"] in dataset_paths:
|
|
458
|
+
raise DatasetValidationError(f"Duplicate dataset path: {entry['path']}")
|
|
459
|
+
dataset_paths.add(entry["path"])
|
|
460
|
+
|
|
461
|
+
definition = _definition_for_path(entry["path"])
|
|
462
|
+
data_path = _repository_file(root, entry["path"])
|
|
463
|
+
schema_path = _repository_file(root, definition.schema)
|
|
464
|
+
data_bytes = data_path.read_bytes()
|
|
465
|
+
records_value = _read_json(data_path)
|
|
466
|
+
try:
|
|
467
|
+
is_canonical_dataset = canonical_json(records_value) == data_bytes
|
|
468
|
+
except (TypeError, ValueError) as error:
|
|
469
|
+
raise DatasetValidationError(
|
|
470
|
+
f"{entry['path']} cannot be serialized canonically: {error}"
|
|
471
|
+
) from error
|
|
472
|
+
if not is_canonical_dataset:
|
|
473
|
+
raise DatasetValidationError(f"{entry['path']} is not canonically serialized")
|
|
474
|
+
|
|
475
|
+
checksum = hashlib.sha256(data_bytes).hexdigest()
|
|
476
|
+
if checksum != entry["sha256"]:
|
|
477
|
+
raise DatasetValidationError(f"Checksum mismatch for {entry['path']}")
|
|
478
|
+
|
|
479
|
+
if not isinstance(records_value, list):
|
|
480
|
+
raise DatasetValidationError(f"{entry['path']} must contain a JSON array")
|
|
481
|
+
|
|
482
|
+
schema = _read_json(schema_path)
|
|
483
|
+
if definition.domain == "entity":
|
|
484
|
+
schema = _schema_with_entity_tags(schema, manifest["vocabularies"]["entity_tags"])
|
|
485
|
+
record_validator = _schema_validator(schema, definition.schema)
|
|
486
|
+
for index, record in enumerate(records_value):
|
|
487
|
+
description = f"{entry['path']}[{index}]"
|
|
488
|
+
_validate_instance(record_validator, record, description)
|
|
489
|
+
_validate_record_partition(record, definition, description)
|
|
490
|
+
validated_datasets.append(
|
|
491
|
+
DatasetRecords(definition=definition, entry=entry, records=records_value)
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
existing_files = _list_dataset_files(root)
|
|
495
|
+
missing_manifest_entries = sorted(set(existing_files) - dataset_paths)
|
|
496
|
+
missing_dataset_files = sorted(dataset_paths - set(existing_files))
|
|
497
|
+
if missing_manifest_entries:
|
|
498
|
+
raise DatasetValidationError(
|
|
499
|
+
f"Dataset files missing from manifest: {', '.join(missing_manifest_entries)}"
|
|
500
|
+
)
|
|
501
|
+
if missing_dataset_files:
|
|
502
|
+
raise DatasetValidationError(
|
|
503
|
+
f"Manifest paths do not exist as dataset files: {', '.join(missing_dataset_files)}"
|
|
504
|
+
)
|
|
505
|
+
|
|
506
|
+
warnings = [
|
|
507
|
+
warning
|
|
508
|
+
for dataset in validated_datasets
|
|
509
|
+
if (warning := _english_sort_warning(dataset)) is not None
|
|
510
|
+
]
|
|
511
|
+
_validate_cross_dataset_references(validated_datasets)
|
|
512
|
+
return warnings
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: refdatakit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Source-aware reference data and validation tools
|
|
5
|
+
Author: RefDataKit contributors
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/rezwan4029/refdatakit#readme
|
|
8
|
+
Project-URL: Documentation, https://github.com/rezwan4029/refdatakit/blob/main/packages/python/README.md
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Requires-Python: >=3.11
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: jsonschema<5,>=4.23
|
|
15
|
+
Requires-Dist: rfc8785<0.2,>=0.1.4
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest>=8.3; extra == "dev"
|
|
18
|
+
Requires-Dist: ruff>=0.11; extra == "dev"
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# refdatakit
|
|
22
|
+
|
|
23
|
+
Source-aware regional reference data and validation tools for Python.
|
|
24
|
+
|
|
25
|
+
Documentation: [Python package guide](https://github.com/rezwan4029/refdatakit/blob/main/packages/python/README.md) · [Project overview](https://github.com/rezwan4029/refdatakit#readme) · [JavaScript/TypeScript package guide](https://github.com/rezwan4029/refdatakit/blob/main/packages/js/README.md).
|
|
26
|
+
|
|
27
|
+
Install from PyPI with `pip install refdatakit`. The package includes the versioned JSON datasets. Data access is local and makes no network requests.
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from refdatakit import CountryCode, get_cities, get_country, get_institutions
|
|
31
|
+
|
|
32
|
+
country = get_country(CountryCode.BD)
|
|
33
|
+
cities = get_cities(CountryCode.BD)
|
|
34
|
+
institutions = get_institutions(CountryCode.BD)
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Available accessors are `get_countries()`, `get_country(code)`, `get_cities(code)`, `get_subdivisions(code)`, and `get_institutions(code)`. Country arguments require a generated `CountryCode` enum member; passing a raw string raises `TypeError`. They return dictionaries/lists matching the shared JSON schemas. Source provenance and dataset licenses are recorded in the bundled manifest; dataset terms may differ from the MIT-licensed SDK code.
|
|
38
|
+
|
|
39
|
+
The `refdatakit validate` and `refdatakit sort` commands are repository maintenance tools. The Python package exposes `validate_repository(path)` and `sort_repository(path)` for contributor workflows.
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
refdatakit/__init__.py,sha256=kI5GZQxnqpm99Ce3rs7AcIGZdyBEmDIcgUJM7xMF0P4,408
|
|
2
|
+
refdatakit/canonical_json.py,sha256=5fi1CnHUrto798yF64ARRo8FP483Oi2JPbbj2GOfL_E,1284
|
|
3
|
+
refdatakit/cli.py,sha256=gg3NkXLMsy6CdRML_Lb7MOhaQqcmQ6pKo407LG4pvYU,1316
|
|
4
|
+
refdatakit/country_codes.py,sha256=__9WxOkOVRhlEfKtV6SEU0bWag0w6ztarji0Yl58lF8,199
|
|
5
|
+
refdatakit/data.py,sha256=1IpmuxjcDiQ5vU3ZEnaZHs9jbEHUkzVSrTA8IXCcB7w,1687
|
|
6
|
+
refdatakit/validator.py,sha256=acFDprPICW8NjzidhHmMLKpjuwo2rrC9ZIFISn3EQjE,19882
|
|
7
|
+
refdatakit/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
refdatakit/data/datasets/manifest.json,sha256=CMb1uQ-cELzNzmSYfxxK5m_IV1NvaRdo_vwCFydiVL4,12146
|
|
9
|
+
refdatakit/data/datasets/cities/by-country/BD.json,sha256=jR2o7_5myELufMG_Q4_0P8_6sVNivN7FQ5ajIoYJlmw,9579
|
|
10
|
+
refdatakit/data/datasets/cities/by-country/BT.json,sha256=aJeoLrKKhL24p5F38KFFcRUV6-8zj_7xdBcK0yOxads,137
|
|
11
|
+
refdatakit/data/datasets/cities/by-country/LK.json,sha256=SEuFX7mhNHh9nd5QobXaWo-Iat7p6skVFlLRVSlOgKM,1035
|
|
12
|
+
refdatakit/data/datasets/cities/by-country/MV.json,sha256=UPA3CFrP0NVk3qXEu9ofp0QxVzXJjVpwsBNiqAELO3I,298
|
|
13
|
+
refdatakit/data/datasets/cities/by-country/NP.json,sha256=970Kcea0vrAa0GRHCFBdGmc8rfpqYlwefRUplYjdtZA,820
|
|
14
|
+
refdatakit/data/datasets/cities/by-country/SG.json,sha256=LlScLCZx-y0Hc9q12aldXEx8eoAfU00KHWBD5N3dK7M,268
|
|
15
|
+
refdatakit/data/datasets/countries/data.json,sha256=L1z9LrnGBFR-cWl13Qp0kCCTX0lg7uVB9MnB30TPkU4,898
|
|
16
|
+
refdatakit/data/datasets/institutions/by-country/BD.json,sha256=i0IhTm2RxyUoI0Jy-SDAGhmS9tB6f-Tqj-Hj2cgsJYU,91614
|
|
17
|
+
refdatakit/data/datasets/institutions/by-country/BT.json,sha256=I9GP2H0SKpa3VuADOCftMfncvTXrB7cfqO3kSDq2SmQ,1940
|
|
18
|
+
refdatakit/data/datasets/institutions/by-country/LK.json,sha256=IdJlHOkkXQbAlG3xLMyvQ_A6-ioaweTygL1JZRZgKNo,6024
|
|
19
|
+
refdatakit/data/datasets/institutions/by-country/MV.json,sha256=jsf6FK5I90pgb2ICfSzwMk3LogcwPb6-4mpKIbrnKnA,1854
|
|
20
|
+
refdatakit/data/datasets/institutions/by-country/NP.json,sha256=qNQBLYGiAyH2ljyyT1J-xHYcd9_8kiR0r6cjRmtr8D4,3851
|
|
21
|
+
refdatakit/data/datasets/institutions/by-country/SG.json,sha256=yQI_KbunUli1H8G51KKXn7Gx-_RjgybN5zrdGhJvqHw,6014
|
|
22
|
+
refdatakit/data/datasets/subdivisions/by-country/BD.json,sha256=nm3TZ2gE4ZQcz7kOJGLB6oBv6hruLsik4PEO1ad5yzg,1315
|
|
23
|
+
refdatakit/data/schemas/city.schema.json,sha256=lU2Gd0Q5amw2N5UoF_1r57bWTUkboOvEeePAIK-EABA,1880
|
|
24
|
+
refdatakit/data/schemas/country.schema.json,sha256=MAumIg--QmDw2vzBgf4LvwZmTQXbrdS3F8EgldjMG2I,2507
|
|
25
|
+
refdatakit/data/schemas/entity.schema.json,sha256=z4XqIkBAn3uwog03dTY92OUpkSTLoAJkgu9rVDTV-bs,3968
|
|
26
|
+
refdatakit/data/schemas/manifest.schema.json,sha256=3eQHV6AVX9s2kJ_xyafkjiwBUvtoiVsKthg4m6a64q8,2524
|
|
27
|
+
refdatakit/data/schemas/subdivision.schema.json,sha256=kkFwOjr4nF3nt9bJ2_bLiCpBw0IVafqsUfKS9zATb0Y,1500
|
|
28
|
+
refdatakit-0.1.0.dist-info/licenses/LICENSE,sha256=AceJ6-88pwOg59nr84IL_4dpAhVEDS6z6WFs-desMSc,1080
|
|
29
|
+
refdatakit-0.1.0.dist-info/METADATA,sha256=zWDItqvNs1IABJvzUy0GiHxEJxQpGEIH6Jx6KJQbHe8,2120
|
|
30
|
+
refdatakit-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
31
|
+
refdatakit-0.1.0.dist-info/entry_points.txt,sha256=A-SRkK17ZyVgzJeSzamHVEvyk39GMtS6O68cRC-KyVk,51
|
|
32
|
+
refdatakit-0.1.0.dist-info/top_level.txt,sha256=VBQQ0UDfxMi-_R31XZhviStJjkPoQ1mEGEJDEKI2M2k,11
|
|
33
|
+
refdatakit-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 RefDataKit contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
refdatakit
|