microdata-tools 2.0.1__tar.gz → 2.1.0__tar.gz

{microdata_tools-2.0.1 → microdata_tools-2.1.0}/.gitignore

@@ -60,17 +60,11 @@ coverage.xml
 .pytest_cache/
 cover/
 
-tests/resources/validation/validate_dataset/big_datasets/ACCUMULATED_DS/ACCUMULATED_DS.csv
-tests/resources/validation/validate_dataset/big_datasets/ACCUMULATED_DS/ACCUMULATED_DS.csv.rowcount
+tests/resources/validation/validate_dataset/big_datasets/*/*.csv
+tests/resources/validation/validate_dataset/big_datasets/*/*.json
+tests/resources/validation/validate_dataset/big_datasets/*/*.csv.rowcount
+tests/resources/validation/steps/dataset_validator/parquet/*.parquet
 
-tests/resources/validation/validate_dataset/big_datasets/EVENT_DS/EVENT_DS.csv
-tests/resources/validation/validate_dataset/big_datasets/EVENT_DS/EVENT_DS.csv.rowcount
-
-tests/resources/validation/validate_dataset/big_datasets/FIXED_DS/FIXED_DS.csv
-tests/resources/validation/validate_dataset/big_datasets/FIXED_DS/FIXED_DS.csv.rowcount
-
-tests/resources/validation/validate_dataset/big_datasets/STATUS_DS/STATUS_DS.csv
-tests/resources/validation/validate_dataset/big_datasets/STATUS_DS/STATUS_DS.csv.rowcount
 
 # Translations
 *.mo

{microdata_tools-2.0.1 → microdata_tools-2.1.0}/PKG-INFO

@@ -1,11 +1,13 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: microdata-tools
-Version: 2.0.1
+Version: 2.1.0
 Summary: Tools for the microdata.no platform
 Author: microdata-developers
 License: MIT
 License-File: LICENSE.md
 Requires-Python: >=3.10
+Requires-Dist: adbc-driver-manager==1.10.0
+Requires-Dist: adbc-driver-sqlite==1.10.0
 Requires-Dist: cryptography>=50.0.0
 Requires-Dist: numpy>=2.2.0
 Requires-Dist: pyarrow==23.0.1

{microdata_tools-2.0.1 → microdata_tools-2.1.0}/microdata_tools/validation/__init__.py

@@ -88,12 +88,16 @@ def validate_dataset(
 
         # Read data
         parquet_path = working_directory_path / f"{dataset_name}.parquet"
-        filesystem_dataset = data_reader.read_and_sanitize_csv_write_parquet(
-            input_data_path,
-            parquet_path,
-            identifier_data_type,
-            measure_data_type,
-            temporality_type,
+        sqlite_path = working_directory_path / f"{dataset_name}.sqlite3.db"
+        filesystem_dataset = (
+            data_reader.read_and_sanitize_csv_write_sqlite_and_parquet(
+                input_data_path,
+                parquet_path,
+                sqlite_path,
+                identifier_data_type,
+                measure_data_type,
+                temporality_type,
+            )
         )
 
         # Enrich metadata with temporal data
@@ -112,6 +116,7 @@ def validate_dataset(
         # Validate data
         dataset_validator.validate_dataset(
             pyarrow.dataset.dataset(parquet_path),
+            sqlite_path,
             measure_data_type,
             code_list,
             sentinel_list,

{microdata_tools-2.0.1 → microdata_tools-2.1.0}/microdata_tools/validation/adapter/local_storage.py

@@ -73,6 +73,7 @@ def clean_up_temporary_files(
 ) -> None:
     generated_files = [
         f"{dataset_name}.parquet",
+        f"{dataset_name}.sqlite3.db",
         f"{dataset_name}.json",
     ]
     if delete_working_directory:

{microdata_tools-2.0.1 → microdata_tools-2.1.0}/microdata_tools/validation/steps/data_reader.py

@@ -1,9 +1,12 @@
 # pyright: reportAttributeAccessIssue=false
 import logging
+import os.path
+import sqlite3
 from datetime import datetime, timedelta
 from pathlib import Path
 from typing import Dict, Tuple
 
+import adbc_driver_sqlite.dbapi
 import pyarrow
 import pyarrow.dataset
 import pyarrow.dataset as ds
@@ -74,40 +77,57 @@ def _get_csv_convert_options(
     )
 
 
-def _csv_stream_to_parquet(
+def _csv_stream_to_sqlite_and_parquet(
     identifier_data_type: str,
     measure_data_type: str,
     temporality_type: str,
     reader: pyarrow.csv.CSVStreamingReader,
     writer: pyarrow.parquet.ParquetWriter,
+    conn: sqlite3.Connection,
 ) -> None:
-    while True:
-        try:
-            batch = reader.read_next_batch()
-        except StopIteration:
-            break
-        table = pyarrow.Table.from_batches([batch])
-        unit_id = _sanitize_unit_id(table, identifier_data_type)
-        value = _sanitize_value(table, measure_data_type)
-        epoch_start = _cast_to_epoch_date(table, "start")
-        epoch_stop = _cast_to_epoch_date(table, "stop")
-        columns = [unit_id, value, epoch_start, epoch_stop]
-        column_names = [
-            "unit_id",
-            "value",
-            "start_epoch_days",
-            "stop_epoch_days",
-        ]
-        if temporality_type in ["STATUS", "ACCUMULATED"]:
-            columns.append(_generate_start_year(table))
-            column_names.append("start_year")
-        table = pyarrow.Table.from_arrays(columns, column_names)
-        writer.write_table(table)
+    with conn.cursor() as cursor:
+        while True:
+            try:
+                batch = reader.read_next_batch()
+            except StopIteration:
+                logger.debug("End of file reached for CSV file")
+                break
+            table = pyarrow.Table.from_batches([batch])
+            unit_id = _sanitize_unit_id(table, identifier_data_type)
+            value = _sanitize_value(table, measure_data_type)
+            epoch_start = _cast_to_epoch_date(table, "start")
+            epoch_stop = _cast_to_epoch_date(table, "stop")
+            columns = [unit_id, value, epoch_start, epoch_stop]
+            column_names = [
+                "unit_id",
+                "value",
+                "start_epoch_days",
+                "stop_epoch_days",
+            ]
+            if temporality_type in ["STATUS", "ACCUMULATED"]:
+                columns.append(_generate_start_year(table))
+                column_names.append("start_year")
+            table = pyarrow.Table.from_arrays(columns, column_names)
+            writer.write_table(table)
+
+            column_names2 = [
+                "unit_id",
+                "start_epoch_days",
+                "stop_epoch_days",
+            ]
+            columns2 = [unit_id, epoch_start, epoch_stop]
+            cursor.adbc_ingest(
+                "dataset",
+                pyarrow.Table.from_arrays(columns2, column_names2),
+                mode="append",
+            )
+            conn.commit()
 
 
-def _csv_to_parquet(
+def _csv_to_sqlite_and_parquet(
     input_csv_path: Path,
-    output_parquet_path: str,
+    output_parquet_path: Path,
+    conn: sqlite3.Connection,
     identifier_data_type: str,
     measure_data_type: str,
     temporality_type: str,
@@ -141,13 +161,20 @@ def _csv_to_parquet(
                 schema_list.append(("start_year", pyarrow.string()))
             schema = pyarrow.schema(schema_list)
             with parquet.ParquetWriter(output_parquet_path, schema) as writer:
-                _csv_stream_to_parquet(
+                _csv_stream_to_sqlite_and_parquet(
                     identifier_data_type,
                     measure_data_type,
                     temporality_type,
                     reader,
                     writer,
+                    conn,
                 )
+                logger.debug("Done streaming, writing parquet metadata")
+        logger.debug("Creating sqlite index ...")
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS index_unit_id ON dataset(unit_id)"
+        )
+        conn.commit()
         return pyarrow.dataset.dataset(output_parquet_path)
     except ArrowInvalid as e:
         raise ValidationError(
@@ -200,9 +227,10 @@ def _generate_start_year(table: pyarrow.Table) -> pyarrow.Array:
     )
 
 
-def read_and_sanitize_csv_write_parquet(
+def read_and_sanitize_csv_write_sqlite_and_parquet(
     input_data_path: Path,
     output_parquet_path: Path,
+    sqlite_path: Path,
     identifier_data_type: str,
     measure_data_type: str,
     temporality_type: str,
@@ -212,13 +240,32 @@ def read_and_sanitize_csv_write_parquet(
     ensures the input csv data follows the requirements for the
     microdata data model.
     """
-    return _csv_to_parquet(
-        input_data_path,
-        output_parquet_path,
-        identifier_data_type,
-        measure_data_type,
-        temporality_type,
-    )
+    if os.path.exists(sqlite_path):
+        os.remove(sqlite_path)
+    with adbc_driver_sqlite.dbapi.connect(
+        sqlite_path, autocommit=False
+    ) as conn:
+        conn.execute(
+            """CREATE TABLE dataset
+               (
+                   unit_id          VARCHAR,
+                   start_epoch_days INTEGER,
+                   stop_epoch_days  INTEGER
+               )"""
+        )
+        conn.commit()
+        # row_count = get_row_count(input_data_path, show_progress)
+        logger.debug("Streaming to sqlite and parquet ...")
+        filesystem_dataset = _csv_to_sqlite_and_parquet(
+            input_data_path,
+            output_parquet_path,
+            conn,
+            identifier_data_type,
+            measure_data_type,
+            temporality_type,
+        )
+        logger.debug("Returning FileSystemDataset")
+        return filesystem_dataset
 
 
 def _min_max(

{microdata_tools-2.0.1 → microdata_tools-2.1.0}/microdata_tools/validation/steps/dataset_validator.py

@@ -1,8 +1,11 @@
 # pyright: reportAttributeAccessIssue=false
+import os.path
+import sqlite3
 from datetime import datetime
-from typing import Iterator, List, Sequence, Union
+from pathlib import Path
+from typing import List, Union
 
-from pyarrow import Table, compute, dataset
+from pyarrow import Table, dataset
 from pyarrow.dataset import FileSystemDataset
 
 from microdata_tools.validation.exceptions import ValidationError
@@ -208,49 +211,35 @@ def _accumulated_temporal_variables_check(data: FileSystemDataset) -> None:
         )
 
 
-def _only_unique_identifiers_check(data: FileSystemDataset) -> None:
+def _only_unique_identifiers_check(conn: sqlite3.Connection) -> None:
     """
     A table with temporalityType=FIXED is only valid if all
     cells in the unit_id column are unique.
     """
-    identifiers = data.to_table(columns=["unit_id"])
-    identifiers = Table.from_arrays(
-        [
-            compute.utf8_slice_codeunits(
-                identifiers["unit_id"], start=0, stop=1
-            ),
-            identifiers["unit_id"],
-        ],
-        names=["bucket", "unit_id"],
-    )
-    unique_buckets = compute.unique(identifiers["bucket"])
-    for unique_bucket in unique_buckets:
-        bucket_table = identifiers.filter(
-            dataset.field("bucket") == unique_bucket
-        )
-        bucket_row_count = len(bucket_table)
-        unique_identifiers_count = len(compute.unique(bucket_table["unit_id"]))
-        if unique_identifiers_count != bucket_row_count:
+    prev_unit_id = None
+    for (unit_id,) in conn.execute(
+        "SELECT unit_id FROM dataset ORDER BY unit_id ASC"
+    ):
+        if unit_id == prev_unit_id:
             raise ValidationError(
                 "#1 column",
                 errors=["Duplicate identifiers in #1 column"],
             )
+        else:
+            prev_unit_id = unit_id
 
 
-def _status_uniquesness_check(data: FileSystemDataset) -> None:
+def _status_uniquesness_check(conn: sqlite3.Connection) -> None:
     """
     A table with temporalityType=STATUS is valid only if all
     cells in the unit_id column are unique per status date.
     """
-    all_status_dates = data.to_table(columns=["start_epoch_days"])
-    unique_status_dates = compute.unique(all_status_dates["start_epoch_days"])
-    for status_date in unique_status_dates:
-        status_table = data.to_table(
-            columns=["unit_id"],
-            filter=dataset.field("start_epoch_days") == status_date,
-        )
-        unique_identifiers = compute.unique(status_table["unit_id"])
-        if len(unique_identifiers) != len(status_table):
+    prev = None, None
+    for curr in conn.execute(
+        "SELECT unit_id, start_epoch_days FROM dataset "
+        + "ORDER BY unit_id ASC, start_epoch_days ASC"
+    ):
+        if curr == prev:
             raise ValidationError(
                 "#1, #3 and #4 columns",
                 errors=[
@@ -258,110 +247,140 @@ def _status_uniquesness_check(data: FileSystemDataset) -> None:
                     "(#3 and #4 column)"
                 ],
             )
+        else:
+            prev = curr
+
+
+def _from_epoch_days_to_date(epoch_days: Union[int, None]) -> str:
+    return (
+        ""
+        if epoch_days is None
+        else datetime.fromtimestamp(epoch_days * 24 * 60 * 60).strftime(
+            "%Y-%m-%d"
+        )
+    )
 
 
-def _no_overlapping_timespans_check(data: FileSystemDataset) -> None:
+def _find_overlap(start_list: list, stop_list: list) -> Union[str, None]:
     """
-    A table with temporalityType=(EVENT|ACCUMULATED) is valid
-    only if all rows for a given identifier contains no overlapping
-    timespans in the start_epoch_days and stop_epoch_days columns.
+    Looks for overlapping timespans where each timespan
+    is defined by a start_date at an index from the start_list,
+    and a stop_date at the same index from the stop_list.
     """
-
-    def from_epoch_days_to_date(epoch_days: Union[int, None]) -> str:
-        return (
-            ""
-            if epoch_days is None
-            else datetime.fromtimestamp(epoch_days * 24 * 60 * 60).strftime(
-                "%Y-%m-%d"
+    for i in range(len(start_list) - 1):
+        if stop_list[i] is None:
+            return (
+                f"timespan: ({_from_epoch_days_to_date(start_list[i])} - "
+                ") overlaps with "
+                f"timespan: "
+                f"({_from_epoch_days_to_date(start_list[i + 1])} - "
+                f"{_from_epoch_days_to_date(stop_list[i + 1])})"
             )
-        )
-
-    def find_overlap(start_list: list, stop_list: list) -> Union[str, None]:
-        """
-        Looks for overlapping timespans where each timespan
-        is defined by a start_date at an index from the start_list,
-        and a stop_date at the same index from the stop_list.
-        """
-        for i in range(len(start_list) - 1):
-            if stop_list[i] is None:
-                return (
-                    f"timespan: ({from_epoch_days_to_date(start_list[i])} - "
-                    ") overlaps with "
-                    f"timespan: "
-                    f"({from_epoch_days_to_date(start_list[i + 1])} - "
-                    f"{from_epoch_days_to_date(stop_list[i + 1])})"
-                )
-            if stop_list[i] >= start_list[i + 1]:
-                return (
-                    f"timespan: ({from_epoch_days_to_date(start_list[i])} - "
-                    f"{from_epoch_days_to_date(stop_list[i])}) "
-                    f"overlaps with timespan: "
-                    f"({from_epoch_days_to_date(start_list[i + 1])} - "
-                    f"{from_epoch_days_to_date(stop_list[i + 1])})"
-                )
-        return None
+        if stop_list[i] >= start_list[i + 1]:
+            return (
+                f"timespan: ({_from_epoch_days_to_date(start_list[i])} - "
+                f"{_from_epoch_days_to_date(stop_list[i])}) "
+                f"overlaps with timespan: "
+                f"({_from_epoch_days_to_date(start_list[i + 1])} - "
+                f"{_from_epoch_days_to_date(stop_list[i + 1])})"
+            )
+    return None
 
-    def batch(iterable: Sequence, batch_size: int) -> Iterator:
-        for index in range(0, len(iterable), batch_size):
-            yield iterable[index : index + batch_size]
 
-    identifiers = data.to_table(columns=["unit_id"])
-    unique_identifiers = compute.unique(identifiers["unit_id"])
-    error_list = []
-    for identifier_batch in batch(unique_identifiers, 500_000):
-        identifier_time_spans = data.to_table(
-            filter=dataset.field("unit_id").isin(identifier_batch),
-            columns=["unit_id", "start_epoch_days", "stop_epoch_days"],
-        )
-        identifier_time_spans = identifier_time_spans.sort_by(
-            [("start_epoch_days", "ascending")]
-        )
-        identifier_time_spans = identifier_time_spans.group_by(
-            "unit_id", use_threads=False
-        ).aggregate([("start_epoch_days", "list"), ("stop_epoch_days", "list")])
-        for i in range(len(identifier_time_spans)):
-            overlap_message = find_overlap(
-                identifier_time_spans["start_epoch_days_list"][i].as_py(),
-                identifier_time_spans["stop_epoch_days_list"][i].as_py(),
+def _check_overlap(
+    error_list: list[str], unit_id: str, start_list: list, stop_list: list
+) -> None:
+    overlap_message = _find_overlap(start_list, stop_list)
+    if overlap_message is not None:
+        error_list.append(
+            (
+                "Invalid overlapping timespans for identifier"
+                f' "{unit_id}":'
+                f" {overlap_message}"
             )
-            if overlap_message is not None:
-                error_list.append(
-                    (
-                        "Invalid overlapping timespans for identifier"
-                        f' "{identifier_time_spans["unit_id"][i]}":'
-                        f" {overlap_message}"
-                    )
-                )
-            if len(error_list) > 49:
-                raise ValidationError(
-                    "#1, #3 and #4 columns",
-                    errors=error_list,
-                )
-        if error_list:
+        )
+        if len(error_list) > 49:
             raise ValidationError(
                 "#1, #3 and #4 columns",
                 errors=error_list,
             )
 
 
+def _no_overlapping_timespans_check(
+    conn: sqlite3.Connection,
+) -> None:
+    """
+    A table with temporalityType=(EVENT|ACCUMULATED) is valid
+    only if all rows for a given identifier contains no overlapping
+    timespans in the start_epoch_days and stop_epoch_days columns.
+    """
+    error_list = []
+    cursor = conn.cursor()
+    cursor.execute(
+        "SELECT unit_id, start_epoch_days, stop_epoch_days "
+        + "FROM dataset "
+        + "ORDER BY unit_id, start_epoch_days"
+    )
+    curr_unit_id = None
+    start_list = []
+    stop_list = []
+    while True:
+        res = cursor.fetchone()
+        if res is None:
+            if curr_unit_id is not None:
+                _check_overlap(error_list, curr_unit_id, start_list, stop_list)
+            break
+        else:
+            unit_id, start_epoch_days, stop_epoch_days = res
+            if curr_unit_id is None:
+                curr_unit_id = unit_id
+                start_list.append(start_epoch_days)
+                stop_list.append(stop_epoch_days)
+            elif curr_unit_id == unit_id:
+                start_list.append(start_epoch_days)
+                stop_list.append(stop_epoch_days)
+            elif curr_unit_id != unit_id:
+                _check_overlap(error_list, curr_unit_id, start_list, stop_list)
+                curr_unit_id = unit_id
+                start_list = [start_epoch_days]
+                stop_list = [stop_epoch_days]
+            else:
+                raise RuntimeError("Unhandled state!")
+    if error_list:
+        raise ValidationError(
+            "#1, #3 and #4 columns",
+            errors=error_list,
+        )
+
+
 def validate_dataset(
     data: FileSystemDataset,
+    sqlite_path: Path,
     measure_data_type: str,
     code_list: Union[List, None],
     sentinel_list: Union[List, None],
     temporality_type: str,
 ) -> None:
-    _valid_unit_id_check(data)
-    _valid_value_column_check(data, measure_data_type, code_list, sentinel_list)
-    if temporality_type == "FIXED":
-        _fixed_temporal_variables_check(data)
-        _only_unique_identifiers_check(data)
-    elif temporality_type == "STATUS":
-        _status_temporal_variables_check(data)
-        _status_uniquesness_check(data)
-    elif temporality_type == "ACCUMULATED":
-        _accumulated_temporal_variables_check(data)
-        _no_overlapping_timespans_check(data)
-    elif temporality_type == "EVENT":
-        _event_temporal_variables_check(data)
-        _no_overlapping_timespans_check(data)
+    assert os.path.exists(sqlite_path)
+    conn = sqlite3.connect(sqlite_path)
+    try:
+        _valid_unit_id_check(data)
+        _valid_value_column_check(
+            data, measure_data_type, code_list, sentinel_list
+        )
+        if temporality_type == "FIXED":
+            _fixed_temporal_variables_check(data)
+            _only_unique_identifiers_check(conn)
+        elif temporality_type == "STATUS":
+            _status_temporal_variables_check(data)
+            _status_uniquesness_check(conn)
+        elif temporality_type == "ACCUMULATED":
+            _accumulated_temporal_variables_check(data)
+            _no_overlapping_timespans_check(conn)
+        elif temporality_type == "EVENT":
+            _event_temporal_variables_check(data)
+            _no_overlapping_timespans_check(conn)
+        else:
+            raise RuntimeError(f"Unknown temporality type '{temporality_type}'")
+    finally:
+        conn.close()

{microdata_tools-2.0.1 → microdata_tools-2.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "microdata-tools"
-version = "2.0.1"
+version = "2.1.0"
 description = "Tools for the microdata.no platform"
 authors = [{ name = "microdata-developers" }]
 license = { text = "MIT" }
@@ -11,6 +11,8 @@ dependencies = [
     "numpy>=2.2.0",
     "pyarrow==23.0.1",
     "pydantic>=2.11.7",
+    "adbc-driver-manager==1.10.0",
+    "adbc-driver-sqlite==1.10.0"
 ]
 
 [dependency-groups]
@@ -27,30 +29,30 @@ exclude-newer = "7 days"
 required-version = ">=0.7.0"
 
 [build-system]
-build-backend  = 'hatchling.build'
-requires       = ['hatchling']
+build-backend = 'hatchling.build'
+requires = ['hatchling']
 
 [tool.hatch.build.targets.sdist]
 packages = ["microdata_tools"]
 include = [
-  "microdata_tools/validation/components/unit_type_variables/*.json"
+    "microdata_tools/validation/components/unit_type_variables/*.json"
 ]
 
 [tool.hatch.build.targets.wheel]
 packages = ["microdata_tools"]
 include = [
-  "microdata_tools/validation/components/unit_type_variables/*.json"
+    "microdata_tools/validation/components/unit_type_variables/*.json"
 ]
 
 [tool.ruff]
 line-length = 80
 target-version = "py313"
 lint.select = [
-    "F",   # Pyflakes errors (including unused imports, undefined names)
-    "T",   # flake8-type-checking (imports used only for typing)
-    "E",   # pycodestyle errors (style errors)
-    "W",   # pycodestyle warnings
-    "I",   # isort import ordering
+    "F", # Pyflakes errors (including unused imports, undefined names)
+    "T", # flake8-type-checking (imports used only for typing)
+    "E", # pycodestyle errors (style errors)
+    "W", # pycodestyle warnings
+    "I", # isort import ordering
     "ANN", # type annotation related rules
 ]