microdata-tools 2.1.0__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/.gitignore +2 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/PKG-INFO +10 -2
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/README.md +9 -1
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/steps/data_reader.py +24 -24
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/steps/dataset_validator.py +8 -8
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/steps/metadata_enricher.py +9 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/pyproject.toml +1 -1
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/LICENSE.md +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/__init__.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/__init__.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/_crypto_config.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/_decrypt.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/_encrypt.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/_utils.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/exceptions/__init__.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/keys.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/__init__.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/adapter/local_storage.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/temporal_attributes.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_id_types.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/BEDRIFT.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/BK_HELSESTASJONSKONSULTASJON.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/BRUKSENHET.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/FAMILIE.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/FENGSLINGER_TILGANG.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/FORETAK.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/HKDIR_HYU_OPPMELDING.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/HKDIR_STUDIESOKNAD.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/HUSHOLDNING.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/JOBB.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KJORETOY.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KOMMUNE.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KRG_KREFTTILFELLE.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KURS.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/MALEPUNKT.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/NPR_EPISODE.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/NPR_PROSEDYRE.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/NPR_SYKEHUSOPPHOLD.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/PERSON.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/SOESKEN.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/SOSTIL.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/TRAFIKKULYKKE.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/TRAFIKKULYKKE_PERSON.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/VIRKSOMHET.json +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/__init__.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/exceptions/__init__.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/model/__init__.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/model/metadata.py +0 -0
- {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/steps/metadata_reader.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: microdata-tools
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.0
|
|
4
4
|
Summary: Tools for the microdata.no platform
|
|
5
5
|
Author: microdata-developers
|
|
6
6
|
License: MIT
|
|
@@ -120,6 +120,15 @@ The data file is a csv file seperated by semicolons. A valid example would be:
|
|
|
120
120
|
000000000000003;123;2020-01-01;2020-12-31;
|
|
121
121
|
000000000000004;123;2020-01-01;2020-12-31;
|
|
122
122
|
```
|
|
123
|
+
|
|
124
|
+
For datasets with temporalityType FIXED the value does not change over time, so there is no time period to describe — both the start and stop columns should be left empty. On microdata.no such variables are shown with an infinite validity period ("∞"), see [BEFOLKNING_KJOENN](https://microdata.no/discovery/variable/no.ssb.fdb/56/BEFOLKNING_KJOENN) for an example.
|
|
125
|
+
```csv
|
|
126
|
+
000000000000001;123;;;
|
|
127
|
+
000000000000002;123;;;
|
|
128
|
+
000000000000003;123;;;
|
|
129
|
+
000000000000004;123;;;
|
|
130
|
+
```
|
|
131
|
+
|
|
123
132
|
Read more about the data format and columns in [the documentation](https://statisticsnorway.github.io/microdata-tools/).
|
|
124
133
|
|
|
125
134
|
The metadata files should be in json format. The requirements for the metadata is best described through the [Pydantic model](/microdata_tools/validation/model/metadata.py), [the examples](/docs/examples), and [the metadata model](https://statisticsnorway.github.io/microdata-tools/metadata-model/).
|
|
@@ -172,4 +181,3 @@ if you dont have it installed on your system you can use:
|
|
|
172
181
|
uv run pre-commit
|
|
173
182
|
```
|
|
174
183
|
Read more about [pre-commit](https://pre-commit.com/#intro)
|
|
175
|
-
|
|
@@ -104,6 +104,15 @@ The data file is a csv file seperated by semicolons. A valid example would be:
|
|
|
104
104
|
000000000000003;123;2020-01-01;2020-12-31;
|
|
105
105
|
000000000000004;123;2020-01-01;2020-12-31;
|
|
106
106
|
```
|
|
107
|
+
|
|
108
|
+
For datasets with temporalityType FIXED the value does not change over time, so there is no time period to describe — both the start and stop columns should be left empty. On microdata.no such variables are shown with an infinite validity period ("∞"), see [BEFOLKNING_KJOENN](https://microdata.no/discovery/variable/no.ssb.fdb/56/BEFOLKNING_KJOENN) for an example.
|
|
109
|
+
```csv
|
|
110
|
+
000000000000001;123;;;
|
|
111
|
+
000000000000002;123;;;
|
|
112
|
+
000000000000003;123;;;
|
|
113
|
+
000000000000004;123;;;
|
|
114
|
+
```
|
|
115
|
+
|
|
107
116
|
Read more about the data format and columns in [the documentation](https://statisticsnorway.github.io/microdata-tools/).
|
|
108
117
|
|
|
109
118
|
The metadata files should be in json format. The requirements for the metadata is best described through the [Pydantic model](/microdata_tools/validation/model/metadata.py), [the examples](/docs/examples), and [the metadata model](https://statisticsnorway.github.io/microdata-tools/metadata-model/).
|
|
@@ -156,4 +165,3 @@ if you dont have it installed on your system you can use:
|
|
|
156
165
|
uv run pre-commit
|
|
157
166
|
```
|
|
158
167
|
Read more about [pre-commit](https://pre-commit.com/#intro)
|
|
159
|
-
|
{microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/steps/data_reader.py
RENAMED
|
@@ -4,7 +4,7 @@ import os.path
|
|
|
4
4
|
import sqlite3
|
|
5
5
|
from datetime import datetime, timedelta
|
|
6
6
|
from pathlib import Path
|
|
7
|
-
from typing import Dict
|
|
7
|
+
from typing import Dict
|
|
8
8
|
|
|
9
9
|
import adbc_driver_sqlite.dbapi
|
|
10
10
|
import pyarrow
|
|
@@ -86,6 +86,7 @@ def _csv_stream_to_sqlite_and_parquet(
|
|
|
86
86
|
conn: sqlite3.Connection,
|
|
87
87
|
) -> None:
|
|
88
88
|
with conn.cursor() as cursor:
|
|
89
|
+
found_deprecated_stop_dates = False
|
|
89
90
|
while True:
|
|
90
91
|
try:
|
|
91
92
|
batch = reader.read_next_batch()
|
|
@@ -97,6 +98,10 @@ def _csv_stream_to_sqlite_and_parquet(
|
|
|
97
98
|
value = _sanitize_value(table, measure_data_type)
|
|
98
99
|
epoch_start = _cast_to_epoch_date(table, "start")
|
|
99
100
|
epoch_stop = _cast_to_epoch_date(table, "stop")
|
|
101
|
+
if temporality_type == "FIXED":
|
|
102
|
+
if epoch_stop.null_count < len(epoch_stop):
|
|
103
|
+
found_deprecated_stop_dates = True
|
|
104
|
+
epoch_stop = _empty_column(epoch_stop)
|
|
100
105
|
columns = [unit_id, value, epoch_start, epoch_stop]
|
|
101
106
|
column_names = [
|
|
102
107
|
"unit_id",
|
|
@@ -122,6 +127,13 @@ def _csv_stream_to_sqlite_and_parquet(
|
|
|
122
127
|
mode="append",
|
|
123
128
|
)
|
|
124
129
|
conn.commit()
|
|
130
|
+
if found_deprecated_stop_dates:
|
|
131
|
+
logger.warning(
|
|
132
|
+
"DEPRECATED: found stop dates in the #4 column of a dataset "
|
|
133
|
+
"with temporalityType FIXED. These dates are ignored, and "
|
|
134
|
+
"support for them will be removed in a future version. Leave "
|
|
135
|
+
"both date columns empty for FIXED datasets."
|
|
136
|
+
)
|
|
125
137
|
|
|
126
138
|
|
|
127
139
|
def _csv_to_sqlite_and_parquet(
|
|
@@ -208,6 +220,14 @@ def _sanitize_value(
|
|
|
208
220
|
return table["value"]
|
|
209
221
|
|
|
210
222
|
|
|
223
|
+
def _empty_column(column: pyarrow.ChunkedArray) -> pyarrow.Array:
|
|
224
|
+
"""
|
|
225
|
+
Returns an all null column of the same length and type. Used to
|
|
226
|
+
ignore the deprecated stop dates of a FIXED dataset.
|
|
227
|
+
"""
|
|
228
|
+
return pyarrow.nulls(len(column), type=column.type)
|
|
229
|
+
|
|
230
|
+
|
|
211
231
|
def _cast_to_epoch_date(
|
|
212
232
|
table: pyarrow.Table, column_name: str
|
|
213
233
|
) -> pyarrow.Array:
|
|
@@ -282,18 +302,6 @@ def _min_max(
|
|
|
282
302
|
return min_v, max_v
|
|
283
303
|
|
|
284
304
|
|
|
285
|
-
def _max(
|
|
286
|
-
filesystem_dataset: pyarrow.dataset.FileSystemDataset, column: str
|
|
287
|
-
) -> Tuple[int | None, int | None]:
|
|
288
|
-
return _min_max(filesystem_dataset, column)[1]
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
def _min(
|
|
292
|
-
filesystem_dataset: pyarrow.dataset.FileSystemDataset, column: str
|
|
293
|
-
) -> Tuple[int | None, int | None]:
|
|
294
|
-
return _min_max(filesystem_dataset, column)[0]
|
|
295
|
-
|
|
296
|
-
|
|
297
305
|
def get_temporal_data(
|
|
298
306
|
dataset: pyarrow.dataset.FileSystemDataset, temporality_type: str
|
|
299
307
|
) -> Dict[str, int]:
|
|
@@ -304,17 +312,9 @@ def get_temporal_data(
|
|
|
304
312
|
"""
|
|
305
313
|
temporal_data = {}
|
|
306
314
|
if temporality_type == "FIXED":
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
"Could not read data in fourth column (Stop date)."
|
|
311
|
-
" Is this column empty?"
|
|
312
|
-
)
|
|
313
|
-
raise ValidationError(error_string, errors=[error_string])
|
|
314
|
-
temporal_data["start"] = "1900-01-01"
|
|
315
|
-
temporal_data["latest"] = (
|
|
316
|
-
datetime(1970, 1, 1) + timedelta(days=stop_max)
|
|
317
|
-
).strftime("%Y-%m-%d")
|
|
315
|
+
# A FIXED dataset has no dates in its data file. Its temporal
|
|
316
|
+
# coverage is derived from the metadata by the metadata_enricher.
|
|
317
|
+
return temporal_data
|
|
318
318
|
else:
|
|
319
319
|
start_min, start_max = _min_max(dataset, "start_epoch_days")
|
|
320
320
|
stop_min, stop_max = _min_max(dataset, "stop_epoch_days")
|
|
@@ -108,20 +108,20 @@ def _fixed_temporal_variables_check(data: FileSystemDataset) -> None:
|
|
|
108
108
|
"""
|
|
109
109
|
Any given row in a table with temporalityType=FIXED is valid only if:
|
|
110
110
|
* The start_epoch_days column contains null (empty)
|
|
111
|
-
|
|
111
|
+
|
|
112
|
+
The stop_epoch_days column is always empty for a FIXED dataset. The
|
|
113
|
+
value of a FIXED variable does not change over time, so there is no
|
|
114
|
+
time period to describe. Any stop dates in the data file are ignored
|
|
115
|
+
by the data_reader.
|
|
112
116
|
"""
|
|
113
|
-
start_is_valid_filter = dataset.field("start_epoch_days").is_valid()
|
|
114
|
-
stop_is_null_filter = dataset.field("stop_epoch_days").is_null()
|
|
115
117
|
invalid_rows = data.to_table(
|
|
116
|
-
filter=
|
|
118
|
+
filter=dataset.field("start_epoch_days").is_valid(),
|
|
117
119
|
columns=["unit_id"],
|
|
118
120
|
)
|
|
119
121
|
if len(invalid_rows) > 0:
|
|
120
122
|
raise ValidationError(
|
|
121
|
-
"#3
|
|
122
|
-
errors=_get_error_list(
|
|
123
|
-
invalid_rows, "Invalid #3 and/or #4 columns"
|
|
124
|
-
),
|
|
123
|
+
"#3 column",
|
|
124
|
+
errors=_get_error_list(invalid_rows, "Expected empty #3 column"),
|
|
125
125
|
)
|
|
126
126
|
|
|
127
127
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
from datetime import datetime
|
|
2
3
|
|
|
3
4
|
logger = logging.getLogger()
|
|
4
5
|
|
|
@@ -9,6 +10,14 @@ def enrich_with_temporal_coverage(metadata: dict, temporal_data: dict) -> None:
|
|
|
9
10
|
)
|
|
10
11
|
data_revision = metadata["dataRevision"]
|
|
11
12
|
temporality_type = metadata["temporalityType"]
|
|
13
|
+
if temporality_type == "FIXED":
|
|
14
|
+
# A FIXED dataset has no dates in its data file, so there is no
|
|
15
|
+
# temporal coverage to derive from the data.
|
|
16
|
+
data_revision["temporalCoverageStart"] = "1900-01-01"
|
|
17
|
+
data_revision["temporalCoverageLatest"] = datetime.now().strftime(
|
|
18
|
+
"%Y-%m-%d"
|
|
19
|
+
)
|
|
20
|
+
return
|
|
12
21
|
data_revision["temporalCoverageStart"] = temporal_data["start"]
|
|
13
22
|
data_revision["temporalCoverageLatest"] = temporal_data["latest"]
|
|
14
23
|
if temporality_type == "STATUS":
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/exceptions/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/adapter/local_storage.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/exceptions/__init__.py
RENAMED
|
File without changes
|
{microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/model/__init__.py
RENAMED
|
File without changes
|
{microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/model/metadata.py
RENAMED
|
File without changes
|
{microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/steps/metadata_reader.py
RENAMED
|
File without changes
|