microdata-tools 2.1.0__tar.gz → 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/.gitignore +2 -0
  2. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/PKG-INFO +10 -2
  3. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/README.md +9 -1
  4. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/steps/data_reader.py +24 -24
  5. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/steps/dataset_validator.py +8 -8
  6. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/steps/metadata_enricher.py +9 -0
  7. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/pyproject.toml +1 -1
  8. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/LICENSE.md +0 -0
  9. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/__init__.py +0 -0
  10. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/__init__.py +0 -0
  11. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/_crypto_config.py +0 -0
  12. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/_decrypt.py +0 -0
  13. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/_encrypt.py +0 -0
  14. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/_utils.py +0 -0
  15. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/exceptions/__init__.py +0 -0
  16. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/packaging/keys.py +0 -0
  17. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/__init__.py +0 -0
  18. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/adapter/local_storage.py +0 -0
  19. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/temporal_attributes.py +0 -0
  20. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_id_types.py +0 -0
  21. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/BEDRIFT.json +0 -0
  22. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/BK_HELSESTASJONSKONSULTASJON.json +0 -0
  23. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/BRUKSENHET.json +0 -0
  24. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/FAMILIE.json +0 -0
  25. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/FENGSLINGER_TILGANG.json +0 -0
  26. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/FORETAK.json +0 -0
  27. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/HKDIR_HYU_OPPMELDING.json +0 -0
  28. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/HKDIR_STUDIESOKNAD.json +0 -0
  29. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/HUSHOLDNING.json +0 -0
  30. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/JOBB.json +0 -0
  31. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KJORETOY.json +0 -0
  32. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KOMMUNE.json +0 -0
  33. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KRG_KREFTTILFELLE.json +0 -0
  34. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KURS.json +0 -0
  35. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/MALEPUNKT.json +0 -0
  36. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/NPR_EPISODE.json +0 -0
  37. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/NPR_PROSEDYRE.json +0 -0
  38. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/NPR_SYKEHUSOPPHOLD.json +0 -0
  39. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/PERSON.json +0 -0
  40. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/SOESKEN.json +0 -0
  41. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/SOSTIL.json +0 -0
  42. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/TRAFIKKULYKKE.json +0 -0
  43. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/TRAFIKKULYKKE_PERSON.json +0 -0
  44. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/VIRKSOMHET.json +0 -0
  45. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/__init__.py +0 -0
  46. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/exceptions/__init__.py +0 -0
  47. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/model/__init__.py +0 -0
  48. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/model/metadata.py +0 -0
  49. {microdata_tools-2.1.0 → microdata_tools-2.2.0}/microdata_tools/validation/steps/metadata_reader.py +0 -0
@@ -66,6 +66,8 @@ tests/resources/validation/validate_dataset/big_datasets/*/*.csv.rowcount
66
66
  tests/resources/validation/steps/dataset_validator/parquet/*.parquet
67
67
 
68
68
 
69
+ *.parquet
70
+
69
71
  # Translations
70
72
  *.mo
71
73
  *.pot
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: microdata-tools
3
- Version: 2.1.0
3
+ Version: 2.2.0
4
4
  Summary: Tools for the microdata.no platform
5
5
  Author: microdata-developers
6
6
  License: MIT
@@ -120,6 +120,15 @@ The data file is a csv file seperated by semicolons. A valid example would be:
120
120
  000000000000003;123;2020-01-01;2020-12-31;
121
121
  000000000000004;123;2020-01-01;2020-12-31;
122
122
  ```
123
+
124
+ For datasets with temporalityType FIXED the value does not change over time, so there is no time period to describe — both the start and stop columns should be left empty. On microdata.no such variables are shown with an infinite validity period ("∞"), see [BEFOLKNING_KJOENN](https://microdata.no/discovery/variable/no.ssb.fdb/56/BEFOLKNING_KJOENN) for an example.
125
+ ```csv
126
+ 000000000000001;123;;;
127
+ 000000000000002;123;;;
128
+ 000000000000003;123;;;
129
+ 000000000000004;123;;;
130
+ ```
131
+
123
132
  Read more about the data format and columns in [the documentation](https://statisticsnorway.github.io/microdata-tools/).
124
133
 
125
134
  The metadata files should be in json format. The requirements for the metadata is best described through the [Pydantic model](/microdata_tools/validation/model/metadata.py), [the examples](/docs/examples), and [the metadata model](https://statisticsnorway.github.io/microdata-tools/metadata-model/).
@@ -172,4 +181,3 @@ if you dont have it installed on your system you can use:
172
181
  uv run pre-commit
173
182
  ```
174
183
  Read more about [pre-commit](https://pre-commit.com/#intro)
175
-
@@ -104,6 +104,15 @@ The data file is a csv file seperated by semicolons. A valid example would be:
104
104
  000000000000003;123;2020-01-01;2020-12-31;
105
105
  000000000000004;123;2020-01-01;2020-12-31;
106
106
  ```
107
+
108
+ For datasets with temporalityType FIXED the value does not change over time, so there is no time period to describe — both the start and stop columns should be left empty. On microdata.no such variables are shown with an infinite validity period ("∞"), see [BEFOLKNING_KJOENN](https://microdata.no/discovery/variable/no.ssb.fdb/56/BEFOLKNING_KJOENN) for an example.
109
+ ```csv
110
+ 000000000000001;123;;;
111
+ 000000000000002;123;;;
112
+ 000000000000003;123;;;
113
+ 000000000000004;123;;;
114
+ ```
115
+
107
116
  Read more about the data format and columns in [the documentation](https://statisticsnorway.github.io/microdata-tools/).
108
117
 
109
118
  The metadata files should be in json format. The requirements for the metadata is best described through the [Pydantic model](/microdata_tools/validation/model/metadata.py), [the examples](/docs/examples), and [the metadata model](https://statisticsnorway.github.io/microdata-tools/metadata-model/).
@@ -156,4 +165,3 @@ if you dont have it installed on your system you can use:
156
165
  uv run pre-commit
157
166
  ```
158
167
  Read more about [pre-commit](https://pre-commit.com/#intro)
159
-
@@ -4,7 +4,7 @@ import os.path
4
4
  import sqlite3
5
5
  from datetime import datetime, timedelta
6
6
  from pathlib import Path
7
- from typing import Dict, Tuple
7
+ from typing import Dict
8
8
 
9
9
  import adbc_driver_sqlite.dbapi
10
10
  import pyarrow
@@ -86,6 +86,7 @@ def _csv_stream_to_sqlite_and_parquet(
86
86
  conn: sqlite3.Connection,
87
87
  ) -> None:
88
88
  with conn.cursor() as cursor:
89
+ found_deprecated_stop_dates = False
89
90
  while True:
90
91
  try:
91
92
  batch = reader.read_next_batch()
@@ -97,6 +98,10 @@ def _csv_stream_to_sqlite_and_parquet(
97
98
  value = _sanitize_value(table, measure_data_type)
98
99
  epoch_start = _cast_to_epoch_date(table, "start")
99
100
  epoch_stop = _cast_to_epoch_date(table, "stop")
101
+ if temporality_type == "FIXED":
102
+ if epoch_stop.null_count < len(epoch_stop):
103
+ found_deprecated_stop_dates = True
104
+ epoch_stop = _empty_column(epoch_stop)
100
105
  columns = [unit_id, value, epoch_start, epoch_stop]
101
106
  column_names = [
102
107
  "unit_id",
@@ -122,6 +127,13 @@ def _csv_stream_to_sqlite_and_parquet(
122
127
  mode="append",
123
128
  )
124
129
  conn.commit()
130
+ if found_deprecated_stop_dates:
131
+ logger.warning(
132
+ "DEPRECATED: found stop dates in the #4 column of a dataset "
133
+ "with temporalityType FIXED. These dates are ignored, and "
134
+ "support for them will be removed in a future version. Leave "
135
+ "both date columns empty for FIXED datasets."
136
+ )
125
137
 
126
138
 
127
139
  def _csv_to_sqlite_and_parquet(
@@ -208,6 +220,14 @@ def _sanitize_value(
208
220
  return table["value"]
209
221
 
210
222
 
223
+ def _empty_column(column: pyarrow.ChunkedArray) -> pyarrow.Array:
224
+ """
225
+ Returns an all null column of the same length and type. Used to
226
+ ignore the deprecated stop dates of a FIXED dataset.
227
+ """
228
+ return pyarrow.nulls(len(column), type=column.type)
229
+
230
+
211
231
  def _cast_to_epoch_date(
212
232
  table: pyarrow.Table, column_name: str
213
233
  ) -> pyarrow.Array:
@@ -282,18 +302,6 @@ def _min_max(
282
302
  return min_v, max_v
283
303
 
284
304
 
285
- def _max(
286
- filesystem_dataset: pyarrow.dataset.FileSystemDataset, column: str
287
- ) -> Tuple[int | None, int | None]:
288
- return _min_max(filesystem_dataset, column)[1]
289
-
290
-
291
- def _min(
292
- filesystem_dataset: pyarrow.dataset.FileSystemDataset, column: str
293
- ) -> Tuple[int | None, int | None]:
294
- return _min_max(filesystem_dataset, column)[0]
295
-
296
-
297
305
  def get_temporal_data(
298
306
  dataset: pyarrow.dataset.FileSystemDataset, temporality_type: str
299
307
  ) -> Dict[str, int]:
@@ -304,17 +312,9 @@ def get_temporal_data(
304
312
  """
305
313
  temporal_data = {}
306
314
  if temporality_type == "FIXED":
307
- stop_max = _max(dataset, "stop_epoch_days")
308
- if stop_max is None:
309
- error_string = (
310
- "Could not read data in fourth column (Stop date)."
311
- " Is this column empty?"
312
- )
313
- raise ValidationError(error_string, errors=[error_string])
314
- temporal_data["start"] = "1900-01-01"
315
- temporal_data["latest"] = (
316
- datetime(1970, 1, 1) + timedelta(days=stop_max)
317
- ).strftime("%Y-%m-%d")
315
+ # A FIXED dataset has no dates in its data file. Its temporal
316
+ # coverage is derived from the metadata by the metadata_enricher.
317
+ return temporal_data
318
318
  else:
319
319
  start_min, start_max = _min_max(dataset, "start_epoch_days")
320
320
  stop_min, stop_max = _min_max(dataset, "stop_epoch_days")
@@ -108,20 +108,20 @@ def _fixed_temporal_variables_check(data: FileSystemDataset) -> None:
108
108
  """
109
109
  Any given row in a table with temporalityType=FIXED is valid only if:
110
110
  * The start_epoch_days column contains null (empty)
111
- * The stop_epoch_days column contains a non-null value (int32)
111
+
112
+ The stop_epoch_days column is always empty for a FIXED dataset. The
113
+ value of a FIXED variable does not change over time, so there is no
114
+ time period to describe. Any stop dates in the data file are ignored
115
+ by the data_reader.
112
116
  """
113
- start_is_valid_filter = dataset.field("start_epoch_days").is_valid()
114
- stop_is_null_filter = dataset.field("stop_epoch_days").is_null()
115
117
  invalid_rows = data.to_table(
116
- filter=start_is_valid_filter | stop_is_null_filter,
118
+ filter=dataset.field("start_epoch_days").is_valid(),
117
119
  columns=["unit_id"],
118
120
  )
119
121
  if len(invalid_rows) > 0:
120
122
  raise ValidationError(
121
- "#3 and #4 columns",
122
- errors=_get_error_list(
123
- invalid_rows, "Invalid #3 and/or #4 columns"
124
- ),
123
+ "#3 column",
124
+ errors=_get_error_list(invalid_rows, "Expected empty #3 column"),
125
125
  )
126
126
 
127
127
 
@@ -1,4 +1,5 @@
1
1
  import logging
2
+ from datetime import datetime
2
3
 
3
4
  logger = logging.getLogger()
4
5
 
@@ -9,6 +10,14 @@ def enrich_with_temporal_coverage(metadata: dict, temporal_data: dict) -> None:
9
10
  )
10
11
  data_revision = metadata["dataRevision"]
11
12
  temporality_type = metadata["temporalityType"]
13
+ if temporality_type == "FIXED":
14
+ # A FIXED dataset has no dates in its data file, so there is no
15
+ # temporal coverage to derive from the data.
16
+ data_revision["temporalCoverageStart"] = "1900-01-01"
17
+ data_revision["temporalCoverageLatest"] = datetime.now().strftime(
18
+ "%Y-%m-%d"
19
+ )
20
+ return
12
21
  data_revision["temporalCoverageStart"] = temporal_data["start"]
13
22
  data_revision["temporalCoverageLatest"] = temporal_data["latest"]
14
23
  if temporality_type == "STATUS":
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "microdata-tools"
3
- version = "2.1.0"
3
+ version = "2.2.0"
4
4
  description = "Tools for the microdata.no platform"
5
5
  authors = [{ name = "microdata-developers" }]
6
6
  license = { text = "MIT" }