microdata-tools 2.0.1__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/.gitignore +5 -9
- microdata_tools-2.0.1/README.md → microdata_tools-2.2.0/PKG-INFO +25 -1
- microdata_tools-2.0.1/PKG-INFO → microdata_tools-2.2.0/README.md +9 -15
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/__init__.py +11 -6
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/adapter/local_storage.py +1 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/steps/data_reader.py +105 -58
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/steps/dataset_validator.py +144 -125
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/steps/metadata_enricher.py +9 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/pyproject.toml +12 -10
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/LICENSE.md +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/__init__.py +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/__init__.py +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/_crypto_config.py +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/_decrypt.py +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/_encrypt.py +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/_utils.py +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/exceptions/__init__.py +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/keys.py +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/temporal_attributes.py +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_id_types.py +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/BEDRIFT.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/BK_HELSESTASJONSKONSULTASJON.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/BRUKSENHET.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/FAMILIE.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/FENGSLINGER_TILGANG.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/FORETAK.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/HKDIR_HYU_OPPMELDING.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/HKDIR_STUDIESOKNAD.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/HUSHOLDNING.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/JOBB.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KJORETOY.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KOMMUNE.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KRG_KREFTTILFELLE.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KURS.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/MALEPUNKT.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/NPR_EPISODE.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/NPR_PROSEDYRE.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/NPR_SYKEHUSOPPHOLD.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/PERSON.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/SOESKEN.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/SOSTIL.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/TRAFIKKULYKKE.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/TRAFIKKULYKKE_PERSON.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/VIRKSOMHET.json +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/__init__.py +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/exceptions/__init__.py +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/model/__init__.py +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/model/metadata.py +0 -0
- {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/steps/metadata_reader.py +0 -0
|
@@ -60,17 +60,13 @@ coverage.xml
|
|
|
60
60
|
.pytest_cache/
|
|
61
61
|
cover/
|
|
62
62
|
|
|
63
|
-
tests/resources/validation/validate_dataset/big_datasets
|
|
64
|
-
tests/resources/validation/validate_dataset/big_datasets
|
|
63
|
+
tests/resources/validation/validate_dataset/big_datasets/*/*.csv
|
|
64
|
+
tests/resources/validation/validate_dataset/big_datasets/*/*.json
|
|
65
|
+
tests/resources/validation/validate_dataset/big_datasets/*/*.csv.rowcount
|
|
66
|
+
tests/resources/validation/steps/dataset_validator/parquet/*.parquet
|
|
65
67
|
|
|
66
|
-
tests/resources/validation/validate_dataset/big_datasets/EVENT_DS/EVENT_DS.csv
|
|
67
|
-
tests/resources/validation/validate_dataset/big_datasets/EVENT_DS/EVENT_DS.csv.rowcount
|
|
68
68
|
|
|
69
|
-
|
|
70
|
-
tests/resources/validation/validate_dataset/big_datasets/FIXED_DS/FIXED_DS.csv.rowcount
|
|
71
|
-
|
|
72
|
-
tests/resources/validation/validate_dataset/big_datasets/STATUS_DS/STATUS_DS.csv
|
|
73
|
-
tests/resources/validation/validate_dataset/big_datasets/STATUS_DS/STATUS_DS.csv.rowcount
|
|
69
|
+
*.parquet
|
|
74
70
|
|
|
75
71
|
# Translations
|
|
76
72
|
*.mo
|
|
@@ -1,3 +1,19 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: microdata-tools
|
|
3
|
+
Version: 2.2.0
|
|
4
|
+
Summary: Tools for the microdata.no platform
|
|
5
|
+
Author: microdata-developers
|
|
6
|
+
License: MIT
|
|
7
|
+
License-File: LICENSE.md
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Requires-Dist: adbc-driver-manager==1.10.0
|
|
10
|
+
Requires-Dist: adbc-driver-sqlite==1.10.0
|
|
11
|
+
Requires-Dist: cryptography>=50.0.0
|
|
12
|
+
Requires-Dist: numpy>=2.2.0
|
|
13
|
+
Requires-Dist: pyarrow==23.0.1
|
|
14
|
+
Requires-Dist: pydantic>=2.11.7
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
1
17
|
# microdata-tools
|
|
2
18
|
Tools for the [microdata.no](https://www.microdata.no/) platform
|
|
3
19
|
|
|
@@ -104,6 +120,15 @@ The data file is a csv file seperated by semicolons. A valid example would be:
|
|
|
104
120
|
000000000000003;123;2020-01-01;2020-12-31;
|
|
105
121
|
000000000000004;123;2020-01-01;2020-12-31;
|
|
106
122
|
```
|
|
123
|
+
|
|
124
|
+
For datasets with temporalityType FIXED the value does not change over time, so there is no time period to describe — both the start and stop columns should be left empty. On microdata.no such variables are shown with an infinite validity period ("∞"), see [BEFOLKNING_KJOENN](https://microdata.no/discovery/variable/no.ssb.fdb/56/BEFOLKNING_KJOENN) for an example.
|
|
125
|
+
```csv
|
|
126
|
+
000000000000001;123;;;
|
|
127
|
+
000000000000002;123;;;
|
|
128
|
+
000000000000003;123;;;
|
|
129
|
+
000000000000004;123;;;
|
|
130
|
+
```
|
|
131
|
+
|
|
107
132
|
Read more about the data format and columns in [the documentation](https://statisticsnorway.github.io/microdata-tools/).
|
|
108
133
|
|
|
109
134
|
The metadata files should be in json format. The requirements for the metadata is best described through the [Pydantic model](/microdata_tools/validation/model/metadata.py), [the examples](/docs/examples), and [the metadata model](https://statisticsnorway.github.io/microdata-tools/metadata-model/).
|
|
@@ -156,4 +181,3 @@ if you dont have it installed on your system you can use:
|
|
|
156
181
|
uv run pre-commit
|
|
157
182
|
```
|
|
158
183
|
Read more about [pre-commit](https://pre-commit.com/#intro)
|
|
159
|
-
|
|
@@ -1,17 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: microdata-tools
|
|
3
|
-
Version: 2.0.1
|
|
4
|
-
Summary: Tools for the microdata.no platform
|
|
5
|
-
Author: microdata-developers
|
|
6
|
-
License: MIT
|
|
7
|
-
License-File: LICENSE.md
|
|
8
|
-
Requires-Python: >=3.10
|
|
9
|
-
Requires-Dist: cryptography>=50.0.0
|
|
10
|
-
Requires-Dist: numpy>=2.2.0
|
|
11
|
-
Requires-Dist: pyarrow==23.0.1
|
|
12
|
-
Requires-Dist: pydantic>=2.11.7
|
|
13
|
-
Description-Content-Type: text/markdown
|
|
14
|
-
|
|
15
1
|
# microdata-tools
|
|
16
2
|
Tools for the [microdata.no](https://www.microdata.no/) platform
|
|
17
3
|
|
|
@@ -118,6 +104,15 @@ The data file is a csv file seperated by semicolons. A valid example would be:
|
|
|
118
104
|
000000000000003;123;2020-01-01;2020-12-31;
|
|
119
105
|
000000000000004;123;2020-01-01;2020-12-31;
|
|
120
106
|
```
|
|
107
|
+
|
|
108
|
+
For datasets with temporalityType FIXED the value does not change over time, so there is no time period to describe — both the start and stop columns should be left empty. On microdata.no such variables are shown with an infinite validity period ("∞"), see [BEFOLKNING_KJOENN](https://microdata.no/discovery/variable/no.ssb.fdb/56/BEFOLKNING_KJOENN) for an example.
|
|
109
|
+
```csv
|
|
110
|
+
000000000000001;123;;;
|
|
111
|
+
000000000000002;123;;;
|
|
112
|
+
000000000000003;123;;;
|
|
113
|
+
000000000000004;123;;;
|
|
114
|
+
```
|
|
115
|
+
|
|
121
116
|
Read more about the data format and columns in [the documentation](https://statisticsnorway.github.io/microdata-tools/).
|
|
122
117
|
|
|
123
118
|
The metadata files should be in json format. The requirements for the metadata is best described through the [Pydantic model](/microdata_tools/validation/model/metadata.py), [the examples](/docs/examples), and [the metadata model](https://statisticsnorway.github.io/microdata-tools/metadata-model/).
|
|
@@ -170,4 +165,3 @@ if you dont have it installed on your system you can use:
|
|
|
170
165
|
uv run pre-commit
|
|
171
166
|
```
|
|
172
167
|
Read more about [pre-commit](https://pre-commit.com/#intro)
|
|
173
|
-
|
|
@@ -88,12 +88,16 @@ def validate_dataset(
|
|
|
88
88
|
|
|
89
89
|
# Read data
|
|
90
90
|
parquet_path = working_directory_path / f"{dataset_name}.parquet"
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
91
|
+
sqlite_path = working_directory_path / f"{dataset_name}.sqlite3.db"
|
|
92
|
+
filesystem_dataset = (
|
|
93
|
+
data_reader.read_and_sanitize_csv_write_sqlite_and_parquet(
|
|
94
|
+
input_data_path,
|
|
95
|
+
parquet_path,
|
|
96
|
+
sqlite_path,
|
|
97
|
+
identifier_data_type,
|
|
98
|
+
measure_data_type,
|
|
99
|
+
temporality_type,
|
|
100
|
+
)
|
|
97
101
|
)
|
|
98
102
|
|
|
99
103
|
# Enrich metadata with temporal data
|
|
@@ -112,6 +116,7 @@ def validate_dataset(
|
|
|
112
116
|
# Validate data
|
|
113
117
|
dataset_validator.validate_dataset(
|
|
114
118
|
pyarrow.dataset.dataset(parquet_path),
|
|
119
|
+
sqlite_path,
|
|
115
120
|
measure_data_type,
|
|
116
121
|
code_list,
|
|
117
122
|
sentinel_list,
|
{microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/steps/data_reader.py
RENAMED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
# pyright: reportAttributeAccessIssue=false
|
|
2
2
|
import logging
|
|
3
|
+
import os.path
|
|
4
|
+
import sqlite3
|
|
3
5
|
from datetime import datetime, timedelta
|
|
4
6
|
from pathlib import Path
|
|
5
|
-
from typing import Dict
|
|
7
|
+
from typing import Dict
|
|
6
8
|
|
|
9
|
+
import adbc_driver_sqlite.dbapi
|
|
7
10
|
import pyarrow
|
|
8
11
|
import pyarrow.dataset
|
|
9
12
|
import pyarrow.dataset as ds
|
|
@@ -74,40 +77,69 @@ def _get_csv_convert_options(
|
|
|
74
77
|
)
|
|
75
78
|
|
|
76
79
|
|
|
77
|
-
def
|
|
80
|
+
def _csv_stream_to_sqlite_and_parquet(
|
|
78
81
|
identifier_data_type: str,
|
|
79
82
|
measure_data_type: str,
|
|
80
83
|
temporality_type: str,
|
|
81
84
|
reader: pyarrow.csv.CSVStreamingReader,
|
|
82
85
|
writer: pyarrow.parquet.ParquetWriter,
|
|
86
|
+
conn: sqlite3.Connection,
|
|
83
87
|
) -> None:
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
"
|
|
97
|
-
"
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
88
|
+
with conn.cursor() as cursor:
|
|
89
|
+
found_deprecated_stop_dates = False
|
|
90
|
+
while True:
|
|
91
|
+
try:
|
|
92
|
+
batch = reader.read_next_batch()
|
|
93
|
+
except StopIteration:
|
|
94
|
+
logger.debug("End of file reached for CSV file")
|
|
95
|
+
break
|
|
96
|
+
table = pyarrow.Table.from_batches([batch])
|
|
97
|
+
unit_id = _sanitize_unit_id(table, identifier_data_type)
|
|
98
|
+
value = _sanitize_value(table, measure_data_type)
|
|
99
|
+
epoch_start = _cast_to_epoch_date(table, "start")
|
|
100
|
+
epoch_stop = _cast_to_epoch_date(table, "stop")
|
|
101
|
+
if temporality_type == "FIXED":
|
|
102
|
+
if epoch_stop.null_count < len(epoch_stop):
|
|
103
|
+
found_deprecated_stop_dates = True
|
|
104
|
+
epoch_stop = _empty_column(epoch_stop)
|
|
105
|
+
columns = [unit_id, value, epoch_start, epoch_stop]
|
|
106
|
+
column_names = [
|
|
107
|
+
"unit_id",
|
|
108
|
+
"value",
|
|
109
|
+
"start_epoch_days",
|
|
110
|
+
"stop_epoch_days",
|
|
111
|
+
]
|
|
112
|
+
if temporality_type in ["STATUS", "ACCUMULATED"]:
|
|
113
|
+
columns.append(_generate_start_year(table))
|
|
114
|
+
column_names.append("start_year")
|
|
115
|
+
table = pyarrow.Table.from_arrays(columns, column_names)
|
|
116
|
+
writer.write_table(table)
|
|
117
|
+
|
|
118
|
+
column_names2 = [
|
|
119
|
+
"unit_id",
|
|
120
|
+
"start_epoch_days",
|
|
121
|
+
"stop_epoch_days",
|
|
122
|
+
]
|
|
123
|
+
columns2 = [unit_id, epoch_start, epoch_stop]
|
|
124
|
+
cursor.adbc_ingest(
|
|
125
|
+
"dataset",
|
|
126
|
+
pyarrow.Table.from_arrays(columns2, column_names2),
|
|
127
|
+
mode="append",
|
|
128
|
+
)
|
|
129
|
+
conn.commit()
|
|
130
|
+
if found_deprecated_stop_dates:
|
|
131
|
+
logger.warning(
|
|
132
|
+
"DEPRECATED: found stop dates in the #4 column of a dataset "
|
|
133
|
+
"with temporalityType FIXED. These dates are ignored, and "
|
|
134
|
+
"support for them will be removed in a future version. Leave "
|
|
135
|
+
"both date columns empty for FIXED datasets."
|
|
136
|
+
)
|
|
106
137
|
|
|
107
138
|
|
|
108
|
-
def
|
|
139
|
+
def _csv_to_sqlite_and_parquet(
|
|
109
140
|
input_csv_path: Path,
|
|
110
|
-
output_parquet_path:
|
|
141
|
+
output_parquet_path: Path,
|
|
142
|
+
conn: sqlite3.Connection,
|
|
111
143
|
identifier_data_type: str,
|
|
112
144
|
measure_data_type: str,
|
|
113
145
|
temporality_type: str,
|
|
@@ -141,13 +173,20 @@ def _csv_to_parquet(
|
|
|
141
173
|
schema_list.append(("start_year", pyarrow.string()))
|
|
142
174
|
schema = pyarrow.schema(schema_list)
|
|
143
175
|
with parquet.ParquetWriter(output_parquet_path, schema) as writer:
|
|
144
|
-
|
|
176
|
+
_csv_stream_to_sqlite_and_parquet(
|
|
145
177
|
identifier_data_type,
|
|
146
178
|
measure_data_type,
|
|
147
179
|
temporality_type,
|
|
148
180
|
reader,
|
|
149
181
|
writer,
|
|
182
|
+
conn,
|
|
150
183
|
)
|
|
184
|
+
logger.debug("Done streaming, writing parquet metadata")
|
|
185
|
+
logger.debug("Creating sqlite index ...")
|
|
186
|
+
conn.execute(
|
|
187
|
+
"CREATE INDEX IF NOT EXISTS index_unit_id ON dataset(unit_id)"
|
|
188
|
+
)
|
|
189
|
+
conn.commit()
|
|
151
190
|
return pyarrow.dataset.dataset(output_parquet_path)
|
|
152
191
|
except ArrowInvalid as e:
|
|
153
192
|
raise ValidationError(
|
|
@@ -181,6 +220,14 @@ def _sanitize_value(
|
|
|
181
220
|
return table["value"]
|
|
182
221
|
|
|
183
222
|
|
|
223
|
+
def _empty_column(column: pyarrow.ChunkedArray) -> pyarrow.Array:
|
|
224
|
+
"""
|
|
225
|
+
Returns an all null column of the same length and type. Used to
|
|
226
|
+
ignore the deprecated stop dates of a FIXED dataset.
|
|
227
|
+
"""
|
|
228
|
+
return pyarrow.nulls(len(column), type=column.type)
|
|
229
|
+
|
|
230
|
+
|
|
184
231
|
def _cast_to_epoch_date(
|
|
185
232
|
table: pyarrow.Table, column_name: str
|
|
186
233
|
) -> pyarrow.Array:
|
|
@@ -200,9 +247,10 @@ def _generate_start_year(table: pyarrow.Table) -> pyarrow.Array:
|
|
|
200
247
|
)
|
|
201
248
|
|
|
202
249
|
|
|
203
|
-
def
|
|
250
|
+
def read_and_sanitize_csv_write_sqlite_and_parquet(
|
|
204
251
|
input_data_path: Path,
|
|
205
252
|
output_parquet_path: Path,
|
|
253
|
+
sqlite_path: Path,
|
|
206
254
|
identifier_data_type: str,
|
|
207
255
|
measure_data_type: str,
|
|
208
256
|
temporality_type: str,
|
|
@@ -212,13 +260,32 @@ def read_and_sanitize_csv_write_parquet(
|
|
|
212
260
|
ensures the input csv data follows the requirements for the
|
|
213
261
|
microdata data model.
|
|
214
262
|
"""
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
263
|
+
if os.path.exists(sqlite_path):
|
|
264
|
+
os.remove(sqlite_path)
|
|
265
|
+
with adbc_driver_sqlite.dbapi.connect(
|
|
266
|
+
sqlite_path, autocommit=False
|
|
267
|
+
) as conn:
|
|
268
|
+
conn.execute(
|
|
269
|
+
"""CREATE TABLE dataset
|
|
270
|
+
(
|
|
271
|
+
unit_id VARCHAR,
|
|
272
|
+
start_epoch_days INTEGER,
|
|
273
|
+
stop_epoch_days INTEGER
|
|
274
|
+
)"""
|
|
275
|
+
)
|
|
276
|
+
conn.commit()
|
|
277
|
+
# row_count = get_row_count(input_data_path, show_progress)
|
|
278
|
+
logger.debug("Streaming to sqlite and parquet ...")
|
|
279
|
+
filesystem_dataset = _csv_to_sqlite_and_parquet(
|
|
280
|
+
input_data_path,
|
|
281
|
+
output_parquet_path,
|
|
282
|
+
conn,
|
|
283
|
+
identifier_data_type,
|
|
284
|
+
measure_data_type,
|
|
285
|
+
temporality_type,
|
|
286
|
+
)
|
|
287
|
+
logger.debug("Returning FileSystemDataset")
|
|
288
|
+
return filesystem_dataset
|
|
222
289
|
|
|
223
290
|
|
|
224
291
|
def _min_max(
|
|
@@ -235,18 +302,6 @@ def _min_max(
|
|
|
235
302
|
return min_v, max_v
|
|
236
303
|
|
|
237
304
|
|
|
238
|
-
def _max(
|
|
239
|
-
filesystem_dataset: pyarrow.dataset.FileSystemDataset, column: str
|
|
240
|
-
) -> Tuple[int | None, int | None]:
|
|
241
|
-
return _min_max(filesystem_dataset, column)[1]
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
def _min(
|
|
245
|
-
filesystem_dataset: pyarrow.dataset.FileSystemDataset, column: str
|
|
246
|
-
) -> Tuple[int | None, int | None]:
|
|
247
|
-
return _min_max(filesystem_dataset, column)[0]
|
|
248
|
-
|
|
249
|
-
|
|
250
305
|
def get_temporal_data(
|
|
251
306
|
dataset: pyarrow.dataset.FileSystemDataset, temporality_type: str
|
|
252
307
|
) -> Dict[str, int]:
|
|
@@ -257,17 +312,9 @@ def get_temporal_data(
|
|
|
257
312
|
"""
|
|
258
313
|
temporal_data = {}
|
|
259
314
|
if temporality_type == "FIXED":
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
"Could not read data in fourth column (Stop date)."
|
|
264
|
-
" Is this column empty?"
|
|
265
|
-
)
|
|
266
|
-
raise ValidationError(error_string, errors=[error_string])
|
|
267
|
-
temporal_data["start"] = "1900-01-01"
|
|
268
|
-
temporal_data["latest"] = (
|
|
269
|
-
datetime(1970, 1, 1) + timedelta(days=stop_max)
|
|
270
|
-
).strftime("%Y-%m-%d")
|
|
315
|
+
# A FIXED dataset has no dates in its data file. Its temporal
|
|
316
|
+
# coverage is derived from the metadata by the metadata_enricher.
|
|
317
|
+
return temporal_data
|
|
271
318
|
else:
|
|
272
319
|
start_min, start_max = _min_max(dataset, "start_epoch_days")
|
|
273
320
|
stop_min, stop_max = _min_max(dataset, "stop_epoch_days")
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
# pyright: reportAttributeAccessIssue=false
|
|
2
|
+
import os.path
|
|
3
|
+
import sqlite3
|
|
2
4
|
from datetime import datetime
|
|
3
|
-
from
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import List, Union
|
|
4
7
|
|
|
5
|
-
from pyarrow import Table,
|
|
8
|
+
from pyarrow import Table, dataset
|
|
6
9
|
from pyarrow.dataset import FileSystemDataset
|
|
7
10
|
|
|
8
11
|
from microdata_tools.validation.exceptions import ValidationError
|
|
@@ -105,20 +108,20 @@ def _fixed_temporal_variables_check(data: FileSystemDataset) -> None:
|
|
|
105
108
|
"""
|
|
106
109
|
Any given row in a table with temporalityType=FIXED is valid only if:
|
|
107
110
|
* The start_epoch_days column contains null (empty)
|
|
108
|
-
|
|
111
|
+
|
|
112
|
+
The stop_epoch_days column is always empty for a FIXED dataset. The
|
|
113
|
+
value of a FIXED variable does not change over time, so there is no
|
|
114
|
+
time period to describe. Any stop dates in the data file are ignored
|
|
115
|
+
by the data_reader.
|
|
109
116
|
"""
|
|
110
|
-
start_is_valid_filter = dataset.field("start_epoch_days").is_valid()
|
|
111
|
-
stop_is_null_filter = dataset.field("stop_epoch_days").is_null()
|
|
112
117
|
invalid_rows = data.to_table(
|
|
113
|
-
filter=
|
|
118
|
+
filter=dataset.field("start_epoch_days").is_valid(),
|
|
114
119
|
columns=["unit_id"],
|
|
115
120
|
)
|
|
116
121
|
if len(invalid_rows) > 0:
|
|
117
122
|
raise ValidationError(
|
|
118
|
-
"#3
|
|
119
|
-
errors=_get_error_list(
|
|
120
|
-
invalid_rows, "Invalid #3 and/or #4 columns"
|
|
121
|
-
),
|
|
123
|
+
"#3 column",
|
|
124
|
+
errors=_get_error_list(invalid_rows, "Expected empty #3 column"),
|
|
122
125
|
)
|
|
123
126
|
|
|
124
127
|
|
|
@@ -208,49 +211,35 @@ def _accumulated_temporal_variables_check(data: FileSystemDataset) -> None:
|
|
|
208
211
|
)
|
|
209
212
|
|
|
210
213
|
|
|
211
|
-
def _only_unique_identifiers_check(
|
|
214
|
+
def _only_unique_identifiers_check(conn: sqlite3.Connection) -> None:
|
|
212
215
|
"""
|
|
213
216
|
A table with temporalityType=FIXED is only valid if all
|
|
214
217
|
cells in the unit_id column are unique.
|
|
215
218
|
"""
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
),
|
|
222
|
-
identifiers["unit_id"],
|
|
223
|
-
],
|
|
224
|
-
names=["bucket", "unit_id"],
|
|
225
|
-
)
|
|
226
|
-
unique_buckets = compute.unique(identifiers["bucket"])
|
|
227
|
-
for unique_bucket in unique_buckets:
|
|
228
|
-
bucket_table = identifiers.filter(
|
|
229
|
-
dataset.field("bucket") == unique_bucket
|
|
230
|
-
)
|
|
231
|
-
bucket_row_count = len(bucket_table)
|
|
232
|
-
unique_identifiers_count = len(compute.unique(bucket_table["unit_id"]))
|
|
233
|
-
if unique_identifiers_count != bucket_row_count:
|
|
219
|
+
prev_unit_id = None
|
|
220
|
+
for (unit_id,) in conn.execute(
|
|
221
|
+
"SELECT unit_id FROM dataset ORDER BY unit_id ASC"
|
|
222
|
+
):
|
|
223
|
+
if unit_id == prev_unit_id:
|
|
234
224
|
raise ValidationError(
|
|
235
225
|
"#1 column",
|
|
236
226
|
errors=["Duplicate identifiers in #1 column"],
|
|
237
227
|
)
|
|
228
|
+
else:
|
|
229
|
+
prev_unit_id = unit_id
|
|
238
230
|
|
|
239
231
|
|
|
240
|
-
def _status_uniquesness_check(
|
|
232
|
+
def _status_uniquesness_check(conn: sqlite3.Connection) -> None:
|
|
241
233
|
"""
|
|
242
234
|
A table with temporalityType=STATUS is valid only if all
|
|
243
235
|
cells in the unit_id column are unique per status date.
|
|
244
236
|
"""
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
)
|
|
252
|
-
unique_identifiers = compute.unique(status_table["unit_id"])
|
|
253
|
-
if len(unique_identifiers) != len(status_table):
|
|
237
|
+
prev = None, None
|
|
238
|
+
for curr in conn.execute(
|
|
239
|
+
"SELECT unit_id, start_epoch_days FROM dataset "
|
|
240
|
+
+ "ORDER BY unit_id ASC, start_epoch_days ASC"
|
|
241
|
+
):
|
|
242
|
+
if curr == prev:
|
|
254
243
|
raise ValidationError(
|
|
255
244
|
"#1, #3 and #4 columns",
|
|
256
245
|
errors=[
|
|
@@ -258,110 +247,140 @@ def _status_uniquesness_check(data: FileSystemDataset) -> None:
|
|
|
258
247
|
"(#3 and #4 column)"
|
|
259
248
|
],
|
|
260
249
|
)
|
|
250
|
+
else:
|
|
251
|
+
prev = curr
|
|
261
252
|
|
|
262
253
|
|
|
263
|
-
def
|
|
254
|
+
def _from_epoch_days_to_date(epoch_days: Union[int, None]) -> str:
|
|
255
|
+
return (
|
|
256
|
+
""
|
|
257
|
+
if epoch_days is None
|
|
258
|
+
else datetime.fromtimestamp(epoch_days * 24 * 60 * 60).strftime(
|
|
259
|
+
"%Y-%m-%d"
|
|
260
|
+
)
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _find_overlap(start_list: list, stop_list: list) -> Union[str, None]:
|
|
264
265
|
"""
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
266
|
+
Looks for overlapping timespans where each timespan
|
|
267
|
+
is defined by a start_date at an index from the start_list,
|
|
268
|
+
and a stop_date at the same index from the stop_list.
|
|
268
269
|
"""
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
"
|
|
270
|
+
for i in range(len(start_list) - 1):
|
|
271
|
+
if stop_list[i] is None:
|
|
272
|
+
return (
|
|
273
|
+
f"timespan: ({_from_epoch_days_to_date(start_list[i])} - "
|
|
274
|
+
") overlaps with "
|
|
275
|
+
f"timespan: "
|
|
276
|
+
f"({_from_epoch_days_to_date(start_list[i + 1])} - "
|
|
277
|
+
f"{_from_epoch_days_to_date(stop_list[i + 1])})"
|
|
276
278
|
)
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
if stop_list[i] is None:
|
|
287
|
-
return (
|
|
288
|
-
f"timespan: ({from_epoch_days_to_date(start_list[i])} - "
|
|
289
|
-
") overlaps with "
|
|
290
|
-
f"timespan: "
|
|
291
|
-
f"({from_epoch_days_to_date(start_list[i + 1])} - "
|
|
292
|
-
f"{from_epoch_days_to_date(stop_list[i + 1])})"
|
|
293
|
-
)
|
|
294
|
-
if stop_list[i] >= start_list[i + 1]:
|
|
295
|
-
return (
|
|
296
|
-
f"timespan: ({from_epoch_days_to_date(start_list[i])} - "
|
|
297
|
-
f"{from_epoch_days_to_date(stop_list[i])}) "
|
|
298
|
-
f"overlaps with timespan: "
|
|
299
|
-
f"({from_epoch_days_to_date(start_list[i + 1])} - "
|
|
300
|
-
f"{from_epoch_days_to_date(stop_list[i + 1])})"
|
|
301
|
-
)
|
|
302
|
-
return None
|
|
279
|
+
if stop_list[i] >= start_list[i + 1]:
|
|
280
|
+
return (
|
|
281
|
+
f"timespan: ({_from_epoch_days_to_date(start_list[i])} - "
|
|
282
|
+
f"{_from_epoch_days_to_date(stop_list[i])}) "
|
|
283
|
+
f"overlaps with timespan: "
|
|
284
|
+
f"({_from_epoch_days_to_date(start_list[i + 1])} - "
|
|
285
|
+
f"{_from_epoch_days_to_date(stop_list[i + 1])})"
|
|
286
|
+
)
|
|
287
|
+
return None
|
|
303
288
|
|
|
304
|
-
def batch(iterable: Sequence, batch_size: int) -> Iterator:
|
|
305
|
-
for index in range(0, len(iterable), batch_size):
|
|
306
|
-
yield iterable[index : index + batch_size]
|
|
307
289
|
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
)
|
|
319
|
-
identifier_time_spans = identifier_time_spans.group_by(
|
|
320
|
-
"unit_id", use_threads=False
|
|
321
|
-
).aggregate([("start_epoch_days", "list"), ("stop_epoch_days", "list")])
|
|
322
|
-
for i in range(len(identifier_time_spans)):
|
|
323
|
-
overlap_message = find_overlap(
|
|
324
|
-
identifier_time_spans["start_epoch_days_list"][i].as_py(),
|
|
325
|
-
identifier_time_spans["stop_epoch_days_list"][i].as_py(),
|
|
290
|
+
def _check_overlap(
|
|
291
|
+
error_list: list[str], unit_id: str, start_list: list, stop_list: list
|
|
292
|
+
) -> None:
|
|
293
|
+
overlap_message = _find_overlap(start_list, stop_list)
|
|
294
|
+
if overlap_message is not None:
|
|
295
|
+
error_list.append(
|
|
296
|
+
(
|
|
297
|
+
"Invalid overlapping timespans for identifier"
|
|
298
|
+
f' "{unit_id}":'
|
|
299
|
+
f" {overlap_message}"
|
|
326
300
|
)
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
(
|
|
330
|
-
"Invalid overlapping timespans for identifier"
|
|
331
|
-
f' "{identifier_time_spans["unit_id"][i]}":'
|
|
332
|
-
f" {overlap_message}"
|
|
333
|
-
)
|
|
334
|
-
)
|
|
335
|
-
if len(error_list) > 49:
|
|
336
|
-
raise ValidationError(
|
|
337
|
-
"#1, #3 and #4 columns",
|
|
338
|
-
errors=error_list,
|
|
339
|
-
)
|
|
340
|
-
if error_list:
|
|
301
|
+
)
|
|
302
|
+
if len(error_list) > 49:
|
|
341
303
|
raise ValidationError(
|
|
342
304
|
"#1, #3 and #4 columns",
|
|
343
305
|
errors=error_list,
|
|
344
306
|
)
|
|
345
307
|
|
|
346
308
|
|
|
309
|
+
def _no_overlapping_timespans_check(
|
|
310
|
+
conn: sqlite3.Connection,
|
|
311
|
+
) -> None:
|
|
312
|
+
"""
|
|
313
|
+
A table with temporalityType=(EVENT|ACCUMULATED) is valid
|
|
314
|
+
only if all rows for a given identifier contains no overlapping
|
|
315
|
+
timespans in the start_epoch_days and stop_epoch_days columns.
|
|
316
|
+
"""
|
|
317
|
+
error_list = []
|
|
318
|
+
cursor = conn.cursor()
|
|
319
|
+
cursor.execute(
|
|
320
|
+
"SELECT unit_id, start_epoch_days, stop_epoch_days "
|
|
321
|
+
+ "FROM dataset "
|
|
322
|
+
+ "ORDER BY unit_id, start_epoch_days"
|
|
323
|
+
)
|
|
324
|
+
curr_unit_id = None
|
|
325
|
+
start_list = []
|
|
326
|
+
stop_list = []
|
|
327
|
+
while True:
|
|
328
|
+
res = cursor.fetchone()
|
|
329
|
+
if res is None:
|
|
330
|
+
if curr_unit_id is not None:
|
|
331
|
+
_check_overlap(error_list, curr_unit_id, start_list, stop_list)
|
|
332
|
+
break
|
|
333
|
+
else:
|
|
334
|
+
unit_id, start_epoch_days, stop_epoch_days = res
|
|
335
|
+
if curr_unit_id is None:
|
|
336
|
+
curr_unit_id = unit_id
|
|
337
|
+
start_list.append(start_epoch_days)
|
|
338
|
+
stop_list.append(stop_epoch_days)
|
|
339
|
+
elif curr_unit_id == unit_id:
|
|
340
|
+
start_list.append(start_epoch_days)
|
|
341
|
+
stop_list.append(stop_epoch_days)
|
|
342
|
+
elif curr_unit_id != unit_id:
|
|
343
|
+
_check_overlap(error_list, curr_unit_id, start_list, stop_list)
|
|
344
|
+
curr_unit_id = unit_id
|
|
345
|
+
start_list = [start_epoch_days]
|
|
346
|
+
stop_list = [stop_epoch_days]
|
|
347
|
+
else:
|
|
348
|
+
raise RuntimeError("Unhandled state!")
|
|
349
|
+
if error_list:
|
|
350
|
+
raise ValidationError(
|
|
351
|
+
"#1, #3 and #4 columns",
|
|
352
|
+
errors=error_list,
|
|
353
|
+
)
|
|
354
|
+
|
|
355
|
+
|
|
347
356
|
def validate_dataset(
|
|
348
357
|
data: FileSystemDataset,
|
|
358
|
+
sqlite_path: Path,
|
|
349
359
|
measure_data_type: str,
|
|
350
360
|
code_list: Union[List, None],
|
|
351
361
|
sentinel_list: Union[List, None],
|
|
352
362
|
temporality_type: str,
|
|
353
363
|
) -> None:
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
364
|
+
assert os.path.exists(sqlite_path)
|
|
365
|
+
conn = sqlite3.connect(sqlite_path)
|
|
366
|
+
try:
|
|
367
|
+
_valid_unit_id_check(data)
|
|
368
|
+
_valid_value_column_check(
|
|
369
|
+
data, measure_data_type, code_list, sentinel_list
|
|
370
|
+
)
|
|
371
|
+
if temporality_type == "FIXED":
|
|
372
|
+
_fixed_temporal_variables_check(data)
|
|
373
|
+
_only_unique_identifiers_check(conn)
|
|
374
|
+
elif temporality_type == "STATUS":
|
|
375
|
+
_status_temporal_variables_check(data)
|
|
376
|
+
_status_uniquesness_check(conn)
|
|
377
|
+
elif temporality_type == "ACCUMULATED":
|
|
378
|
+
_accumulated_temporal_variables_check(data)
|
|
379
|
+
_no_overlapping_timespans_check(conn)
|
|
380
|
+
elif temporality_type == "EVENT":
|
|
381
|
+
_event_temporal_variables_check(data)
|
|
382
|
+
_no_overlapping_timespans_check(conn)
|
|
383
|
+
else:
|
|
384
|
+
raise RuntimeError(f"Unknown temporality type '{temporality_type}'")
|
|
385
|
+
finally:
|
|
386
|
+
conn.close()
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
from datetime import datetime
|
|
2
3
|
|
|
3
4
|
logger = logging.getLogger()
|
|
4
5
|
|
|
@@ -9,6 +10,14 @@ def enrich_with_temporal_coverage(metadata: dict, temporal_data: dict) -> None:
|
|
|
9
10
|
)
|
|
10
11
|
data_revision = metadata["dataRevision"]
|
|
11
12
|
temporality_type = metadata["temporalityType"]
|
|
13
|
+
if temporality_type == "FIXED":
|
|
14
|
+
# A FIXED dataset has no dates in its data file, so there is no
|
|
15
|
+
# temporal coverage to derive from the data.
|
|
16
|
+
data_revision["temporalCoverageStart"] = "1900-01-01"
|
|
17
|
+
data_revision["temporalCoverageLatest"] = datetime.now().strftime(
|
|
18
|
+
"%Y-%m-%d"
|
|
19
|
+
)
|
|
20
|
+
return
|
|
12
21
|
data_revision["temporalCoverageStart"] = temporal_data["start"]
|
|
13
22
|
data_revision["temporalCoverageLatest"] = temporal_data["latest"]
|
|
14
23
|
if temporality_type == "STATUS":
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "microdata-tools"
|
|
3
|
-
version = "2.0
|
|
3
|
+
version = "2.2.0"
|
|
4
4
|
description = "Tools for the microdata.no platform"
|
|
5
5
|
authors = [{ name = "microdata-developers" }]
|
|
6
6
|
license = { text = "MIT" }
|
|
@@ -11,6 +11,8 @@ dependencies = [
|
|
|
11
11
|
"numpy>=2.2.0",
|
|
12
12
|
"pyarrow==23.0.1",
|
|
13
13
|
"pydantic>=2.11.7",
|
|
14
|
+
"adbc-driver-manager==1.10.0",
|
|
15
|
+
"adbc-driver-sqlite==1.10.0"
|
|
14
16
|
]
|
|
15
17
|
|
|
16
18
|
[dependency-groups]
|
|
@@ -27,30 +29,30 @@ exclude-newer = "7 days"
|
|
|
27
29
|
required-version = ">=0.7.0"
|
|
28
30
|
|
|
29
31
|
[build-system]
|
|
30
|
-
build-backend
|
|
31
|
-
requires
|
|
32
|
+
build-backend = 'hatchling.build'
|
|
33
|
+
requires = ['hatchling']
|
|
32
34
|
|
|
33
35
|
[tool.hatch.build.targets.sdist]
|
|
34
36
|
packages = ["microdata_tools"]
|
|
35
37
|
include = [
|
|
36
|
-
|
|
38
|
+
"microdata_tools/validation/components/unit_type_variables/*.json"
|
|
37
39
|
]
|
|
38
40
|
|
|
39
41
|
[tool.hatch.build.targets.wheel]
|
|
40
42
|
packages = ["microdata_tools"]
|
|
41
43
|
include = [
|
|
42
|
-
|
|
44
|
+
"microdata_tools/validation/components/unit_type_variables/*.json"
|
|
43
45
|
]
|
|
44
46
|
|
|
45
47
|
[tool.ruff]
|
|
46
48
|
line-length = 80
|
|
47
49
|
target-version = "py313"
|
|
48
50
|
lint.select = [
|
|
49
|
-
"F",
|
|
50
|
-
"T",
|
|
51
|
-
"E",
|
|
52
|
-
"W",
|
|
53
|
-
"I",
|
|
51
|
+
"F", # Pyflakes errors (including unused imports, undefined names)
|
|
52
|
+
"T", # flake8-type-checking (imports used only for typing)
|
|
53
|
+
"E", # pycodestyle errors (style errors)
|
|
54
|
+
"W", # pycodestyle warnings
|
|
55
|
+
"I", # isort import ordering
|
|
54
56
|
"ANN", # type annotation related rules
|
|
55
57
|
]
|
|
56
58
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/exceptions/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/exceptions/__init__.py
RENAMED
|
File without changes
|
{microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/model/__init__.py
RENAMED
|
File without changes
|
{microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/model/metadata.py
RENAMED
|
File without changes
|
{microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/steps/metadata_reader.py
RENAMED
|
File without changes
|