microdata-tools 2.0.1__tar.gz → 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/.gitignore +5 -9
  2. microdata_tools-2.0.1/README.md → microdata_tools-2.2.0/PKG-INFO +25 -1
  3. microdata_tools-2.0.1/PKG-INFO → microdata_tools-2.2.0/README.md +9 -15
  4. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/__init__.py +11 -6
  5. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/adapter/local_storage.py +1 -0
  6. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/steps/data_reader.py +105 -58
  7. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/steps/dataset_validator.py +144 -125
  8. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/steps/metadata_enricher.py +9 -0
  9. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/pyproject.toml +12 -10
  10. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/LICENSE.md +0 -0
  11. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/__init__.py +0 -0
  12. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/__init__.py +0 -0
  13. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/_crypto_config.py +0 -0
  14. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/_decrypt.py +0 -0
  15. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/_encrypt.py +0 -0
  16. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/_utils.py +0 -0
  17. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/exceptions/__init__.py +0 -0
  18. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/packaging/keys.py +0 -0
  19. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/temporal_attributes.py +0 -0
  20. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_id_types.py +0 -0
  21. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/BEDRIFT.json +0 -0
  22. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/BK_HELSESTASJONSKONSULTASJON.json +0 -0
  23. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/BRUKSENHET.json +0 -0
  24. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/FAMILIE.json +0 -0
  25. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/FENGSLINGER_TILGANG.json +0 -0
  26. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/FORETAK.json +0 -0
  27. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/HKDIR_HYU_OPPMELDING.json +0 -0
  28. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/HKDIR_STUDIESOKNAD.json +0 -0
  29. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/HUSHOLDNING.json +0 -0
  30. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/JOBB.json +0 -0
  31. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KJORETOY.json +0 -0
  32. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KOMMUNE.json +0 -0
  33. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KRG_KREFTTILFELLE.json +0 -0
  34. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/KURS.json +0 -0
  35. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/MALEPUNKT.json +0 -0
  36. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/NPR_EPISODE.json +0 -0
  37. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/NPR_PROSEDYRE.json +0 -0
  38. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/NPR_SYKEHUSOPPHOLD.json +0 -0
  39. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/PERSON.json +0 -0
  40. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/SOESKEN.json +0 -0
  41. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/SOSTIL.json +0 -0
  42. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/TRAFIKKULYKKE.json +0 -0
  43. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/TRAFIKKULYKKE_PERSON.json +0 -0
  44. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/VIRKSOMHET.json +0 -0
  45. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/components/unit_type_variables/__init__.py +0 -0
  46. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/exceptions/__init__.py +0 -0
  47. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/model/__init__.py +0 -0
  48. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/model/metadata.py +0 -0
  49. {microdata_tools-2.0.1 → microdata_tools-2.2.0}/microdata_tools/validation/steps/metadata_reader.py +0 -0
@@ -60,17 +60,13 @@ coverage.xml
60
60
  .pytest_cache/
61
61
  cover/
62
62
 
63
- tests/resources/validation/validate_dataset/big_datasets/ACCUMULATED_DS/ACCUMULATED_DS.csv
64
- tests/resources/validation/validate_dataset/big_datasets/ACCUMULATED_DS/ACCUMULATED_DS.csv.rowcount
63
+ tests/resources/validation/validate_dataset/big_datasets/*/*.csv
64
+ tests/resources/validation/validate_dataset/big_datasets/*/*.json
65
+ tests/resources/validation/validate_dataset/big_datasets/*/*.csv.rowcount
66
+ tests/resources/validation/steps/dataset_validator/parquet/*.parquet
65
67
 
66
- tests/resources/validation/validate_dataset/big_datasets/EVENT_DS/EVENT_DS.csv
67
- tests/resources/validation/validate_dataset/big_datasets/EVENT_DS/EVENT_DS.csv.rowcount
68
68
 
69
- tests/resources/validation/validate_dataset/big_datasets/FIXED_DS/FIXED_DS.csv
70
- tests/resources/validation/validate_dataset/big_datasets/FIXED_DS/FIXED_DS.csv.rowcount
71
-
72
- tests/resources/validation/validate_dataset/big_datasets/STATUS_DS/STATUS_DS.csv
73
- tests/resources/validation/validate_dataset/big_datasets/STATUS_DS/STATUS_DS.csv.rowcount
69
+ *.parquet
74
70
 
75
71
  # Translations
76
72
  *.mo
@@ -1,3 +1,19 @@
1
+ Metadata-Version: 2.5
2
+ Name: microdata-tools
3
+ Version: 2.2.0
4
+ Summary: Tools for the microdata.no platform
5
+ Author: microdata-developers
6
+ License: MIT
7
+ License-File: LICENSE.md
8
+ Requires-Python: >=3.10
9
+ Requires-Dist: adbc-driver-manager==1.10.0
10
+ Requires-Dist: adbc-driver-sqlite==1.10.0
11
+ Requires-Dist: cryptography>=50.0.0
12
+ Requires-Dist: numpy>=2.2.0
13
+ Requires-Dist: pyarrow==23.0.1
14
+ Requires-Dist: pydantic>=2.11.7
15
+ Description-Content-Type: text/markdown
16
+
1
17
  # microdata-tools
2
18
  Tools for the [microdata.no](https://www.microdata.no/) platform
3
19
 
@@ -104,6 +120,15 @@ The data file is a csv file seperated by semicolons. A valid example would be:
104
120
  000000000000003;123;2020-01-01;2020-12-31;
105
121
  000000000000004;123;2020-01-01;2020-12-31;
106
122
  ```
123
+
124
+ For datasets with temporalityType FIXED the value does not change over time, so there is no time period to describe — both the start and stop columns should be left empty. On microdata.no such variables are shown with an infinite validity period ("∞"), see [BEFOLKNING_KJOENN](https://microdata.no/discovery/variable/no.ssb.fdb/56/BEFOLKNING_KJOENN) for an example.
125
+ ```csv
126
+ 000000000000001;123;;;
127
+ 000000000000002;123;;;
128
+ 000000000000003;123;;;
129
+ 000000000000004;123;;;
130
+ ```
131
+
107
132
  Read more about the data format and columns in [the documentation](https://statisticsnorway.github.io/microdata-tools/).
108
133
 
109
134
  The metadata files should be in json format. The requirements for the metadata is best described through the [Pydantic model](/microdata_tools/validation/model/metadata.py), [the examples](/docs/examples), and [the metadata model](https://statisticsnorway.github.io/microdata-tools/metadata-model/).
@@ -156,4 +181,3 @@ if you dont have it installed on your system you can use:
156
181
  uv run pre-commit
157
182
  ```
158
183
  Read more about [pre-commit](https://pre-commit.com/#intro)
159
-
@@ -1,17 +1,3 @@
1
- Metadata-Version: 2.4
2
- Name: microdata-tools
3
- Version: 2.0.1
4
- Summary: Tools for the microdata.no platform
5
- Author: microdata-developers
6
- License: MIT
7
- License-File: LICENSE.md
8
- Requires-Python: >=3.10
9
- Requires-Dist: cryptography>=50.0.0
10
- Requires-Dist: numpy>=2.2.0
11
- Requires-Dist: pyarrow==23.0.1
12
- Requires-Dist: pydantic>=2.11.7
13
- Description-Content-Type: text/markdown
14
-
15
1
  # microdata-tools
16
2
  Tools for the [microdata.no](https://www.microdata.no/) platform
17
3
 
@@ -118,6 +104,15 @@ The data file is a csv file seperated by semicolons. A valid example would be:
118
104
  000000000000003;123;2020-01-01;2020-12-31;
119
105
  000000000000004;123;2020-01-01;2020-12-31;
120
106
  ```
107
+
108
+ For datasets with temporalityType FIXED the value does not change over time, so there is no time period to describe — both the start and stop columns should be left empty. On microdata.no such variables are shown with an infinite validity period ("∞"), see [BEFOLKNING_KJOENN](https://microdata.no/discovery/variable/no.ssb.fdb/56/BEFOLKNING_KJOENN) for an example.
109
+ ```csv
110
+ 000000000000001;123;;;
111
+ 000000000000002;123;;;
112
+ 000000000000003;123;;;
113
+ 000000000000004;123;;;
114
+ ```
115
+
121
116
  Read more about the data format and columns in [the documentation](https://statisticsnorway.github.io/microdata-tools/).
122
117
 
123
118
  The metadata files should be in json format. The requirements for the metadata is best described through the [Pydantic model](/microdata_tools/validation/model/metadata.py), [the examples](/docs/examples), and [the metadata model](https://statisticsnorway.github.io/microdata-tools/metadata-model/).
@@ -170,4 +165,3 @@ if you dont have it installed on your system you can use:
170
165
  uv run pre-commit
171
166
  ```
172
167
  Read more about [pre-commit](https://pre-commit.com/#intro)
173
-
@@ -88,12 +88,16 @@ def validate_dataset(
88
88
 
89
89
  # Read data
90
90
  parquet_path = working_directory_path / f"{dataset_name}.parquet"
91
- filesystem_dataset = data_reader.read_and_sanitize_csv_write_parquet(
92
- input_data_path,
93
- parquet_path,
94
- identifier_data_type,
95
- measure_data_type,
96
- temporality_type,
91
+ sqlite_path = working_directory_path / f"{dataset_name}.sqlite3.db"
92
+ filesystem_dataset = (
93
+ data_reader.read_and_sanitize_csv_write_sqlite_and_parquet(
94
+ input_data_path,
95
+ parquet_path,
96
+ sqlite_path,
97
+ identifier_data_type,
98
+ measure_data_type,
99
+ temporality_type,
100
+ )
97
101
  )
98
102
 
99
103
  # Enrich metadata with temporal data
@@ -112,6 +116,7 @@ def validate_dataset(
112
116
  # Validate data
113
117
  dataset_validator.validate_dataset(
114
118
  pyarrow.dataset.dataset(parquet_path),
119
+ sqlite_path,
115
120
  measure_data_type,
116
121
  code_list,
117
122
  sentinel_list,
@@ -73,6 +73,7 @@ def clean_up_temporary_files(
73
73
  ) -> None:
74
74
  generated_files = [
75
75
  f"{dataset_name}.parquet",
76
+ f"{dataset_name}.sqlite3.db",
76
77
  f"{dataset_name}.json",
77
78
  ]
78
79
  if delete_working_directory:
@@ -1,9 +1,12 @@
1
1
  # pyright: reportAttributeAccessIssue=false
2
2
  import logging
3
+ import os.path
4
+ import sqlite3
3
5
  from datetime import datetime, timedelta
4
6
  from pathlib import Path
5
- from typing import Dict, Tuple
7
+ from typing import Dict
6
8
 
9
+ import adbc_driver_sqlite.dbapi
7
10
  import pyarrow
8
11
  import pyarrow.dataset
9
12
  import pyarrow.dataset as ds
@@ -74,40 +77,69 @@ def _get_csv_convert_options(
74
77
  )
75
78
 
76
79
 
77
- def _csv_stream_to_parquet(
80
+ def _csv_stream_to_sqlite_and_parquet(
78
81
  identifier_data_type: str,
79
82
  measure_data_type: str,
80
83
  temporality_type: str,
81
84
  reader: pyarrow.csv.CSVStreamingReader,
82
85
  writer: pyarrow.parquet.ParquetWriter,
86
+ conn: sqlite3.Connection,
83
87
  ) -> None:
84
- while True:
85
- try:
86
- batch = reader.read_next_batch()
87
- except StopIteration:
88
- break
89
- table = pyarrow.Table.from_batches([batch])
90
- unit_id = _sanitize_unit_id(table, identifier_data_type)
91
- value = _sanitize_value(table, measure_data_type)
92
- epoch_start = _cast_to_epoch_date(table, "start")
93
- epoch_stop = _cast_to_epoch_date(table, "stop")
94
- columns = [unit_id, value, epoch_start, epoch_stop]
95
- column_names = [
96
- "unit_id",
97
- "value",
98
- "start_epoch_days",
99
- "stop_epoch_days",
100
- ]
101
- if temporality_type in ["STATUS", "ACCUMULATED"]:
102
- columns.append(_generate_start_year(table))
103
- column_names.append("start_year")
104
- table = pyarrow.Table.from_arrays(columns, column_names)
105
- writer.write_table(table)
88
+ with conn.cursor() as cursor:
89
+ found_deprecated_stop_dates = False
90
+ while True:
91
+ try:
92
+ batch = reader.read_next_batch()
93
+ except StopIteration:
94
+ logger.debug("End of file reached for CSV file")
95
+ break
96
+ table = pyarrow.Table.from_batches([batch])
97
+ unit_id = _sanitize_unit_id(table, identifier_data_type)
98
+ value = _sanitize_value(table, measure_data_type)
99
+ epoch_start = _cast_to_epoch_date(table, "start")
100
+ epoch_stop = _cast_to_epoch_date(table, "stop")
101
+ if temporality_type == "FIXED":
102
+ if epoch_stop.null_count < len(epoch_stop):
103
+ found_deprecated_stop_dates = True
104
+ epoch_stop = _empty_column(epoch_stop)
105
+ columns = [unit_id, value, epoch_start, epoch_stop]
106
+ column_names = [
107
+ "unit_id",
108
+ "value",
109
+ "start_epoch_days",
110
+ "stop_epoch_days",
111
+ ]
112
+ if temporality_type in ["STATUS", "ACCUMULATED"]:
113
+ columns.append(_generate_start_year(table))
114
+ column_names.append("start_year")
115
+ table = pyarrow.Table.from_arrays(columns, column_names)
116
+ writer.write_table(table)
117
+
118
+ column_names2 = [
119
+ "unit_id",
120
+ "start_epoch_days",
121
+ "stop_epoch_days",
122
+ ]
123
+ columns2 = [unit_id, epoch_start, epoch_stop]
124
+ cursor.adbc_ingest(
125
+ "dataset",
126
+ pyarrow.Table.from_arrays(columns2, column_names2),
127
+ mode="append",
128
+ )
129
+ conn.commit()
130
+ if found_deprecated_stop_dates:
131
+ logger.warning(
132
+ "DEPRECATED: found stop dates in the #4 column of a dataset "
133
+ "with temporalityType FIXED. These dates are ignored, and "
134
+ "support for them will be removed in a future version. Leave "
135
+ "both date columns empty for FIXED datasets."
136
+ )
106
137
 
107
138
 
108
- def _csv_to_parquet(
139
+ def _csv_to_sqlite_and_parquet(
109
140
  input_csv_path: Path,
110
- output_parquet_path: str,
141
+ output_parquet_path: Path,
142
+ conn: sqlite3.Connection,
111
143
  identifier_data_type: str,
112
144
  measure_data_type: str,
113
145
  temporality_type: str,
@@ -141,13 +173,20 @@ def _csv_to_parquet(
141
173
  schema_list.append(("start_year", pyarrow.string()))
142
174
  schema = pyarrow.schema(schema_list)
143
175
  with parquet.ParquetWriter(output_parquet_path, schema) as writer:
144
- _csv_stream_to_parquet(
176
+ _csv_stream_to_sqlite_and_parquet(
145
177
  identifier_data_type,
146
178
  measure_data_type,
147
179
  temporality_type,
148
180
  reader,
149
181
  writer,
182
+ conn,
150
183
  )
184
+ logger.debug("Done streaming, writing parquet metadata")
185
+ logger.debug("Creating sqlite index ...")
186
+ conn.execute(
187
+ "CREATE INDEX IF NOT EXISTS index_unit_id ON dataset(unit_id)"
188
+ )
189
+ conn.commit()
151
190
  return pyarrow.dataset.dataset(output_parquet_path)
152
191
  except ArrowInvalid as e:
153
192
  raise ValidationError(
@@ -181,6 +220,14 @@ def _sanitize_value(
181
220
  return table["value"]
182
221
 
183
222
 
223
+ def _empty_column(column: pyarrow.ChunkedArray) -> pyarrow.Array:
224
+ """
225
+ Returns an all null column of the same length and type. Used to
226
+ ignore the deprecated stop dates of a FIXED dataset.
227
+ """
228
+ return pyarrow.nulls(len(column), type=column.type)
229
+
230
+
184
231
  def _cast_to_epoch_date(
185
232
  table: pyarrow.Table, column_name: str
186
233
  ) -> pyarrow.Array:
@@ -200,9 +247,10 @@ def _generate_start_year(table: pyarrow.Table) -> pyarrow.Array:
200
247
  )
201
248
 
202
249
 
203
- def read_and_sanitize_csv_write_parquet(
250
+ def read_and_sanitize_csv_write_sqlite_and_parquet(
204
251
  input_data_path: Path,
205
252
  output_parquet_path: Path,
253
+ sqlite_path: Path,
206
254
  identifier_data_type: str,
207
255
  measure_data_type: str,
208
256
  temporality_type: str,
@@ -212,13 +260,32 @@ def read_and_sanitize_csv_write_parquet(
212
260
  ensures the input csv data follows the requirements for the
213
261
  microdata data model.
214
262
  """
215
- return _csv_to_parquet(
216
- input_data_path,
217
- output_parquet_path,
218
- identifier_data_type,
219
- measure_data_type,
220
- temporality_type,
221
- )
263
+ if os.path.exists(sqlite_path):
264
+ os.remove(sqlite_path)
265
+ with adbc_driver_sqlite.dbapi.connect(
266
+ sqlite_path, autocommit=False
267
+ ) as conn:
268
+ conn.execute(
269
+ """CREATE TABLE dataset
270
+ (
271
+ unit_id VARCHAR,
272
+ start_epoch_days INTEGER,
273
+ stop_epoch_days INTEGER
274
+ )"""
275
+ )
276
+ conn.commit()
277
+ # row_count = get_row_count(input_data_path, show_progress)
278
+ logger.debug("Streaming to sqlite and parquet ...")
279
+ filesystem_dataset = _csv_to_sqlite_and_parquet(
280
+ input_data_path,
281
+ output_parquet_path,
282
+ conn,
283
+ identifier_data_type,
284
+ measure_data_type,
285
+ temporality_type,
286
+ )
287
+ logger.debug("Returning FileSystemDataset")
288
+ return filesystem_dataset
222
289
 
223
290
 
224
291
  def _min_max(
@@ -235,18 +302,6 @@ def _min_max(
235
302
  return min_v, max_v
236
303
 
237
304
 
238
- def _max(
239
- filesystem_dataset: pyarrow.dataset.FileSystemDataset, column: str
240
- ) -> Tuple[int | None, int | None]:
241
- return _min_max(filesystem_dataset, column)[1]
242
-
243
-
244
- def _min(
245
- filesystem_dataset: pyarrow.dataset.FileSystemDataset, column: str
246
- ) -> Tuple[int | None, int | None]:
247
- return _min_max(filesystem_dataset, column)[0]
248
-
249
-
250
305
  def get_temporal_data(
251
306
  dataset: pyarrow.dataset.FileSystemDataset, temporality_type: str
252
307
  ) -> Dict[str, int]:
@@ -257,17 +312,9 @@ def get_temporal_data(
257
312
  """
258
313
  temporal_data = {}
259
314
  if temporality_type == "FIXED":
260
- stop_max = _max(dataset, "stop_epoch_days")
261
- if stop_max is None:
262
- error_string = (
263
- "Could not read data in fourth column (Stop date)."
264
- " Is this column empty?"
265
- )
266
- raise ValidationError(error_string, errors=[error_string])
267
- temporal_data["start"] = "1900-01-01"
268
- temporal_data["latest"] = (
269
- datetime(1970, 1, 1) + timedelta(days=stop_max)
270
- ).strftime("%Y-%m-%d")
315
+ # A FIXED dataset has no dates in its data file. Its temporal
316
+ # coverage is derived from the metadata by the metadata_enricher.
317
+ return temporal_data
271
318
  else:
272
319
  start_min, start_max = _min_max(dataset, "start_epoch_days")
273
320
  stop_min, stop_max = _min_max(dataset, "stop_epoch_days")
@@ -1,8 +1,11 @@
1
1
  # pyright: reportAttributeAccessIssue=false
2
+ import os.path
3
+ import sqlite3
2
4
  from datetime import datetime
3
- from typing import Iterator, List, Sequence, Union
5
+ from pathlib import Path
6
+ from typing import List, Union
4
7
 
5
- from pyarrow import Table, compute, dataset
8
+ from pyarrow import Table, dataset
6
9
  from pyarrow.dataset import FileSystemDataset
7
10
 
8
11
  from microdata_tools.validation.exceptions import ValidationError
@@ -105,20 +108,20 @@ def _fixed_temporal_variables_check(data: FileSystemDataset) -> None:
105
108
  """
106
109
  Any given row in a table with temporalityType=FIXED is valid only if:
107
110
  * The start_epoch_days column contains null (empty)
108
- * The stop_epoch_days column contains a non-null value (int32)
111
+
112
+ The stop_epoch_days column is always empty for a FIXED dataset. The
113
+ value of a FIXED variable does not change over time, so there is no
114
+ time period to describe. Any stop dates in the data file are ignored
115
+ by the data_reader.
109
116
  """
110
- start_is_valid_filter = dataset.field("start_epoch_days").is_valid()
111
- stop_is_null_filter = dataset.field("stop_epoch_days").is_null()
112
117
  invalid_rows = data.to_table(
113
- filter=start_is_valid_filter | stop_is_null_filter,
118
+ filter=dataset.field("start_epoch_days").is_valid(),
114
119
  columns=["unit_id"],
115
120
  )
116
121
  if len(invalid_rows) > 0:
117
122
  raise ValidationError(
118
- "#3 and #4 columns",
119
- errors=_get_error_list(
120
- invalid_rows, "Invalid #3 and/or #4 columns"
121
- ),
123
+ "#3 column",
124
+ errors=_get_error_list(invalid_rows, "Expected empty #3 column"),
122
125
  )
123
126
 
124
127
 
@@ -208,49 +211,35 @@ def _accumulated_temporal_variables_check(data: FileSystemDataset) -> None:
208
211
  )
209
212
 
210
213
 
211
- def _only_unique_identifiers_check(data: FileSystemDataset) -> None:
214
+ def _only_unique_identifiers_check(conn: sqlite3.Connection) -> None:
212
215
  """
213
216
  A table with temporalityType=FIXED is only valid if all
214
217
  cells in the unit_id column are unique.
215
218
  """
216
- identifiers = data.to_table(columns=["unit_id"])
217
- identifiers = Table.from_arrays(
218
- [
219
- compute.utf8_slice_codeunits(
220
- identifiers["unit_id"], start=0, stop=1
221
- ),
222
- identifiers["unit_id"],
223
- ],
224
- names=["bucket", "unit_id"],
225
- )
226
- unique_buckets = compute.unique(identifiers["bucket"])
227
- for unique_bucket in unique_buckets:
228
- bucket_table = identifiers.filter(
229
- dataset.field("bucket") == unique_bucket
230
- )
231
- bucket_row_count = len(bucket_table)
232
- unique_identifiers_count = len(compute.unique(bucket_table["unit_id"]))
233
- if unique_identifiers_count != bucket_row_count:
219
+ prev_unit_id = None
220
+ for (unit_id,) in conn.execute(
221
+ "SELECT unit_id FROM dataset ORDER BY unit_id ASC"
222
+ ):
223
+ if unit_id == prev_unit_id:
234
224
  raise ValidationError(
235
225
  "#1 column",
236
226
  errors=["Duplicate identifiers in #1 column"],
237
227
  )
228
+ else:
229
+ prev_unit_id = unit_id
238
230
 
239
231
 
240
- def _status_uniquesness_check(data: FileSystemDataset) -> None:
232
+ def _status_uniquesness_check(conn: sqlite3.Connection) -> None:
241
233
  """
242
234
  A table with temporalityType=STATUS is valid only if all
243
235
  cells in the unit_id column are unique per status date.
244
236
  """
245
- all_status_dates = data.to_table(columns=["start_epoch_days"])
246
- unique_status_dates = compute.unique(all_status_dates["start_epoch_days"])
247
- for status_date in unique_status_dates:
248
- status_table = data.to_table(
249
- columns=["unit_id"],
250
- filter=dataset.field("start_epoch_days") == status_date,
251
- )
252
- unique_identifiers = compute.unique(status_table["unit_id"])
253
- if len(unique_identifiers) != len(status_table):
237
+ prev = None, None
238
+ for curr in conn.execute(
239
+ "SELECT unit_id, start_epoch_days FROM dataset "
240
+ + "ORDER BY unit_id ASC, start_epoch_days ASC"
241
+ ):
242
+ if curr == prev:
254
243
  raise ValidationError(
255
244
  "#1, #3 and #4 columns",
256
245
  errors=[
@@ -258,110 +247,140 @@ def _status_uniquesness_check(data: FileSystemDataset) -> None:
258
247
  "(#3 and #4 column)"
259
248
  ],
260
249
  )
250
+ else:
251
+ prev = curr
261
252
 
262
253
 
263
- def _no_overlapping_timespans_check(data: FileSystemDataset) -> None:
254
+ def _from_epoch_days_to_date(epoch_days: Union[int, None]) -> str:
255
+ return (
256
+ ""
257
+ if epoch_days is None
258
+ else datetime.fromtimestamp(epoch_days * 24 * 60 * 60).strftime(
259
+ "%Y-%m-%d"
260
+ )
261
+ )
262
+
263
+
264
+ def _find_overlap(start_list: list, stop_list: list) -> Union[str, None]:
264
265
  """
265
- A table with temporalityType=(EVENT|ACCUMULATED) is valid
266
- only if all rows for a given identifier contains no overlapping
267
- timespans in the start_epoch_days and stop_epoch_days columns.
266
+ Looks for overlapping timespans where each timespan
267
+ is defined by a start_date at an index from the start_list,
268
+ and a stop_date at the same index from the stop_list.
268
269
  """
269
-
270
- def from_epoch_days_to_date(epoch_days: Union[int, None]) -> str:
271
- return (
272
- ""
273
- if epoch_days is None
274
- else datetime.fromtimestamp(epoch_days * 24 * 60 * 60).strftime(
275
- "%Y-%m-%d"
270
+ for i in range(len(start_list) - 1):
271
+ if stop_list[i] is None:
272
+ return (
273
+ f"timespan: ({_from_epoch_days_to_date(start_list[i])} - "
274
+ ") overlaps with "
275
+ f"timespan: "
276
+ f"({_from_epoch_days_to_date(start_list[i + 1])} - "
277
+ f"{_from_epoch_days_to_date(stop_list[i + 1])})"
276
278
  )
277
- )
278
-
279
- def find_overlap(start_list: list, stop_list: list) -> Union[str, None]:
280
- """
281
- Looks for overlapping timespans where each timespan
282
- is defined by a start_date at an index from the start_list,
283
- and a stop_date at the same index from the stop_list.
284
- """
285
- for i in range(len(start_list) - 1):
286
- if stop_list[i] is None:
287
- return (
288
- f"timespan: ({from_epoch_days_to_date(start_list[i])} - "
289
- ") overlaps with "
290
- f"timespan: "
291
- f"({from_epoch_days_to_date(start_list[i + 1])} - "
292
- f"{from_epoch_days_to_date(stop_list[i + 1])})"
293
- )
294
- if stop_list[i] >= start_list[i + 1]:
295
- return (
296
- f"timespan: ({from_epoch_days_to_date(start_list[i])} - "
297
- f"{from_epoch_days_to_date(stop_list[i])}) "
298
- f"overlaps with timespan: "
299
- f"({from_epoch_days_to_date(start_list[i + 1])} - "
300
- f"{from_epoch_days_to_date(stop_list[i + 1])})"
301
- )
302
- return None
279
+ if stop_list[i] >= start_list[i + 1]:
280
+ return (
281
+ f"timespan: ({_from_epoch_days_to_date(start_list[i])} - "
282
+ f"{_from_epoch_days_to_date(stop_list[i])}) "
283
+ f"overlaps with timespan: "
284
+ f"({_from_epoch_days_to_date(start_list[i + 1])} - "
285
+ f"{_from_epoch_days_to_date(stop_list[i + 1])})"
286
+ )
287
+ return None
303
288
 
304
- def batch(iterable: Sequence, batch_size: int) -> Iterator:
305
- for index in range(0, len(iterable), batch_size):
306
- yield iterable[index : index + batch_size]
307
289
 
308
- identifiers = data.to_table(columns=["unit_id"])
309
- unique_identifiers = compute.unique(identifiers["unit_id"])
310
- error_list = []
311
- for identifier_batch in batch(unique_identifiers, 500_000):
312
- identifier_time_spans = data.to_table(
313
- filter=dataset.field("unit_id").isin(identifier_batch),
314
- columns=["unit_id", "start_epoch_days", "stop_epoch_days"],
315
- )
316
- identifier_time_spans = identifier_time_spans.sort_by(
317
- [("start_epoch_days", "ascending")]
318
- )
319
- identifier_time_spans = identifier_time_spans.group_by(
320
- "unit_id", use_threads=False
321
- ).aggregate([("start_epoch_days", "list"), ("stop_epoch_days", "list")])
322
- for i in range(len(identifier_time_spans)):
323
- overlap_message = find_overlap(
324
- identifier_time_spans["start_epoch_days_list"][i].as_py(),
325
- identifier_time_spans["stop_epoch_days_list"][i].as_py(),
290
+ def _check_overlap(
291
+ error_list: list[str], unit_id: str, start_list: list, stop_list: list
292
+ ) -> None:
293
+ overlap_message = _find_overlap(start_list, stop_list)
294
+ if overlap_message is not None:
295
+ error_list.append(
296
+ (
297
+ "Invalid overlapping timespans for identifier"
298
+ f' "{unit_id}":'
299
+ f" {overlap_message}"
326
300
  )
327
- if overlap_message is not None:
328
- error_list.append(
329
- (
330
- "Invalid overlapping timespans for identifier"
331
- f' "{identifier_time_spans["unit_id"][i]}":'
332
- f" {overlap_message}"
333
- )
334
- )
335
- if len(error_list) > 49:
336
- raise ValidationError(
337
- "#1, #3 and #4 columns",
338
- errors=error_list,
339
- )
340
- if error_list:
301
+ )
302
+ if len(error_list) > 49:
341
303
  raise ValidationError(
342
304
  "#1, #3 and #4 columns",
343
305
  errors=error_list,
344
306
  )
345
307
 
346
308
 
309
+ def _no_overlapping_timespans_check(
310
+ conn: sqlite3.Connection,
311
+ ) -> None:
312
+ """
313
+ A table with temporalityType=(EVENT|ACCUMULATED) is valid
314
+ only if all rows for a given identifier contains no overlapping
315
+ timespans in the start_epoch_days and stop_epoch_days columns.
316
+ """
317
+ error_list = []
318
+ cursor = conn.cursor()
319
+ cursor.execute(
320
+ "SELECT unit_id, start_epoch_days, stop_epoch_days "
321
+ + "FROM dataset "
322
+ + "ORDER BY unit_id, start_epoch_days"
323
+ )
324
+ curr_unit_id = None
325
+ start_list = []
326
+ stop_list = []
327
+ while True:
328
+ res = cursor.fetchone()
329
+ if res is None:
330
+ if curr_unit_id is not None:
331
+ _check_overlap(error_list, curr_unit_id, start_list, stop_list)
332
+ break
333
+ else:
334
+ unit_id, start_epoch_days, stop_epoch_days = res
335
+ if curr_unit_id is None:
336
+ curr_unit_id = unit_id
337
+ start_list.append(start_epoch_days)
338
+ stop_list.append(stop_epoch_days)
339
+ elif curr_unit_id == unit_id:
340
+ start_list.append(start_epoch_days)
341
+ stop_list.append(stop_epoch_days)
342
+ elif curr_unit_id != unit_id:
343
+ _check_overlap(error_list, curr_unit_id, start_list, stop_list)
344
+ curr_unit_id = unit_id
345
+ start_list = [start_epoch_days]
346
+ stop_list = [stop_epoch_days]
347
+ else:
348
+ raise RuntimeError("Unhandled state!")
349
+ if error_list:
350
+ raise ValidationError(
351
+ "#1, #3 and #4 columns",
352
+ errors=error_list,
353
+ )
354
+
355
+
347
356
  def validate_dataset(
348
357
  data: FileSystemDataset,
358
+ sqlite_path: Path,
349
359
  measure_data_type: str,
350
360
  code_list: Union[List, None],
351
361
  sentinel_list: Union[List, None],
352
362
  temporality_type: str,
353
363
  ) -> None:
354
- _valid_unit_id_check(data)
355
- _valid_value_column_check(data, measure_data_type, code_list, sentinel_list)
356
- if temporality_type == "FIXED":
357
- _fixed_temporal_variables_check(data)
358
- _only_unique_identifiers_check(data)
359
- elif temporality_type == "STATUS":
360
- _status_temporal_variables_check(data)
361
- _status_uniquesness_check(data)
362
- elif temporality_type == "ACCUMULATED":
363
- _accumulated_temporal_variables_check(data)
364
- _no_overlapping_timespans_check(data)
365
- elif temporality_type == "EVENT":
366
- _event_temporal_variables_check(data)
367
- _no_overlapping_timespans_check(data)
364
+ assert os.path.exists(sqlite_path)
365
+ conn = sqlite3.connect(sqlite_path)
366
+ try:
367
+ _valid_unit_id_check(data)
368
+ _valid_value_column_check(
369
+ data, measure_data_type, code_list, sentinel_list
370
+ )
371
+ if temporality_type == "FIXED":
372
+ _fixed_temporal_variables_check(data)
373
+ _only_unique_identifiers_check(conn)
374
+ elif temporality_type == "STATUS":
375
+ _status_temporal_variables_check(data)
376
+ _status_uniquesness_check(conn)
377
+ elif temporality_type == "ACCUMULATED":
378
+ _accumulated_temporal_variables_check(data)
379
+ _no_overlapping_timespans_check(conn)
380
+ elif temporality_type == "EVENT":
381
+ _event_temporal_variables_check(data)
382
+ _no_overlapping_timespans_check(conn)
383
+ else:
384
+ raise RuntimeError(f"Unknown temporality type '{temporality_type}'")
385
+ finally:
386
+ conn.close()
@@ -1,4 +1,5 @@
1
1
  import logging
2
+ from datetime import datetime
2
3
 
3
4
  logger = logging.getLogger()
4
5
 
@@ -9,6 +10,14 @@ def enrich_with_temporal_coverage(metadata: dict, temporal_data: dict) -> None:
9
10
  )
10
11
  data_revision = metadata["dataRevision"]
11
12
  temporality_type = metadata["temporalityType"]
13
+ if temporality_type == "FIXED":
14
+ # A FIXED dataset has no dates in its data file, so there is no
15
+ # temporal coverage to derive from the data.
16
+ data_revision["temporalCoverageStart"] = "1900-01-01"
17
+ data_revision["temporalCoverageLatest"] = datetime.now().strftime(
18
+ "%Y-%m-%d"
19
+ )
20
+ return
12
21
  data_revision["temporalCoverageStart"] = temporal_data["start"]
13
22
  data_revision["temporalCoverageLatest"] = temporal_data["latest"]
14
23
  if temporality_type == "STATUS":
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "microdata-tools"
3
- version = "2.0.1"
3
+ version = "2.2.0"
4
4
  description = "Tools for the microdata.no platform"
5
5
  authors = [{ name = "microdata-developers" }]
6
6
  license = { text = "MIT" }
@@ -11,6 +11,8 @@ dependencies = [
11
11
  "numpy>=2.2.0",
12
12
  "pyarrow==23.0.1",
13
13
  "pydantic>=2.11.7",
14
+ "adbc-driver-manager==1.10.0",
15
+ "adbc-driver-sqlite==1.10.0"
14
16
  ]
15
17
 
16
18
  [dependency-groups]
@@ -27,30 +29,30 @@ exclude-newer = "7 days"
27
29
  required-version = ">=0.7.0"
28
30
 
29
31
  [build-system]
30
- build-backend = 'hatchling.build'
31
- requires = ['hatchling']
32
+ build-backend = 'hatchling.build'
33
+ requires = ['hatchling']
32
34
 
33
35
  [tool.hatch.build.targets.sdist]
34
36
  packages = ["microdata_tools"]
35
37
  include = [
36
- "microdata_tools/validation/components/unit_type_variables/*.json"
38
+ "microdata_tools/validation/components/unit_type_variables/*.json"
37
39
  ]
38
40
 
39
41
  [tool.hatch.build.targets.wheel]
40
42
  packages = ["microdata_tools"]
41
43
  include = [
42
- "microdata_tools/validation/components/unit_type_variables/*.json"
44
+ "microdata_tools/validation/components/unit_type_variables/*.json"
43
45
  ]
44
46
 
45
47
  [tool.ruff]
46
48
  line-length = 80
47
49
  target-version = "py313"
48
50
  lint.select = [
49
- "F", # Pyflakes errors (including unused imports, undefined names)
50
- "T", # flake8-type-checking (imports used only for typing)
51
- "E", # pycodestyle errors (style errors)
52
- "W", # pycodestyle warnings
53
- "I", # isort import ordering
51
+ "F", # Pyflakes errors (including unused imports, undefined names)
52
+ "T", # flake8-type-checking (imports used only for typing)
53
+ "E", # pycodestyle errors (style errors)
54
+ "W", # pycodestyle warnings
55
+ "I", # isort import ordering
54
56
  "ANN", # type annotation related rules
55
57
  ]
56
58