microdata-tools 2.2.2__tar.gz → 2.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- microdata_tools-2.2.4/PKG-INFO +105 -0
- microdata_tools-2.2.4/README.md +89 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/pyproject.toml +6 -6
- microdata_tools-2.2.2/PKG-INFO +0 -183
- microdata_tools-2.2.2/README.md +0 -167
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/.gitignore +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/LICENSE.md +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/__init__.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/packaging/__init__.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/packaging/_crypto_config.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/packaging/_decrypt.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/packaging/_encrypt.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/packaging/_utils.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/packaging/exceptions/__init__.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/packaging/keys.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/__init__.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/adapter/local_storage.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/temporal_attributes.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_id_types.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/BEDRIFT.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/BK_HELSESTASJONSKONSULTASJON.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/BRUKSENHET.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/FAMILIE.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/FENGSLINGER_TILGANG.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/FORETAK.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/HKDIR_HYU_OPPMELDING.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/HKDIR_STUDIESOKNAD.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/HUSHOLDNING.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/JOBB.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/KJORETOY.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/KOMMUNE.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/KRG_KREFTTILFELLE.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/KURS.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/MALEPUNKT.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/NPR_EPISODE.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/NPR_PROSEDYRE.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/NPR_SYKEHUSOPPHOLD.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/PERSON.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/SOESKEN.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/SOSTIL.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/TRAFIKKULYKKE.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/TRAFIKKULYKKE_PERSON.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/VIRKSOMHET.json +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/components/unit_type_variables/__init__.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/exceptions/__init__.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/model/__init__.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/model/metadata.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/steps/data_reader.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/steps/dataset_validator.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/steps/metadata_enricher.py +0 -0
- {microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/steps/metadata_reader.py +0 -0
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: microdata-tools
|
|
3
|
+
Version: 2.2.4
|
|
4
|
+
Summary: Tools for the microdata.no platform
|
|
5
|
+
Author: microdata-developers
|
|
6
|
+
License: MIT
|
|
7
|
+
License-File: LICENSE.md
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Requires-Dist: adbc-driver-manager==1.12.0
|
|
10
|
+
Requires-Dist: adbc-driver-sqlite==1.12.0
|
|
11
|
+
Requires-Dist: cryptography>=50.0.1
|
|
12
|
+
Requires-Dist: numpy>=2.2.0
|
|
13
|
+
Requires-Dist: pyarrow==25.0.1
|
|
14
|
+
Requires-Dist: pydantic>=2.13.5
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# microdata-tools
|
|
18
|
+
Tools for the [microdata.no](https://www.microdata.no/) platform
|
|
19
|
+
|
|
20
|
+
`microdata-tools` is a Python library with two audiences:
|
|
21
|
+
- **Data owners** preparing datasets for microdata.no: validate your data and metadata, then encrypt and package the files into a `.tar` archive ready for upload.
|
|
22
|
+
- **microdata.no services** (such as job-executor) that receive packaged datasets: unpackage and decrypt them, and validate them on the receiving end.
|
|
23
|
+
|
|
24
|
+
A dataset normally contains:
|
|
25
|
+
- A CSV file with one identifier, one value and the dates that describe when the value applies.
|
|
26
|
+
- A JSON file describing the dataset, its variable and its values.
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
29
|
+
`microdata-tools` can be installed from PyPI using pip:
|
|
30
|
+
```
|
|
31
|
+
pip install microdata-tools
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Documentation
|
|
35
|
+
|
|
36
|
+
If you're a data owner preparing a dataset, the full guide lives on the documentation site:
|
|
37
|
+
|
|
38
|
+
- **[Getting started](https://statisticsnorway.github.io/microdata-tools/)** — install, prepare, validate and package your dataset, step by step.
|
|
39
|
+
- **[The metadata model](https://statisticsnorway.github.io/microdata-tools/metadata-model/)** — a field-by-field reference for the metadata JSON file.
|
|
40
|
+
- **[The data file format](https://statisticsnorway.github.io/microdata-tools/data-file-format/)** — a reference for the CSV data file, including validation rules.
|
|
41
|
+
- **[The packaging format](https://statisticsnorway.github.io/microdata-tools/packaging-format/)** — how your data is encrypted and packaged.
|
|
42
|
+
|
|
43
|
+
The sections below are aimed at developers integrating `microdata-tools` into another service (such as job-executor), and at contributors to this repository.
|
|
44
|
+
|
|
45
|
+
## Unpackaging a dataset
|
|
46
|
+
|
|
47
|
+
The `unpackage_dataset()` function will reverse the packaging step: it untars the archive and use the combined ML-KEM-768/X25519 private key from `microdata_private_key.pem` to recover the symmetric key, which is then used to decrypt the data file.
|
|
48
|
+
|
|
49
|
+
The packaged file must have the `<DATASET_NAME>.tar` extension. Its contents should be:
|
|
50
|
+
|
|
51
|
+
- `<DATASET_NAME>.json`: Required metadata file.
|
|
52
|
+
- `<DATASET_NAME>.csv.encr`: Optional encrypted data file.
|
|
53
|
+
- `<DATASET_NAME>.kem.encr`: Optional HPKE ciphertext file containing the encrypted symmetric key. Required if `.csv.encr` is present.
|
|
54
|
+
|
|
55
|
+
Decryption uses the combined ML-KEM-768/X25519 private key located at `PRIVATE_KEY_DIR` to recover the symmetric decryption key. The dataset is then stored in `output_dir/archive/unpackaged` after a successful run, or `output_dir/archive/failed` after an unsuccessful one.
|
|
56
|
+
For details on how packaging encrypts a dataset in the first place, see [the packaging format](https://statisticsnorway.github.io/microdata-tools/packaging-format/).
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
## Contribute
|
|
60
|
+
|
|
61
|
+
### Set up
|
|
62
|
+
To work on this repository you need to install [uv](https://docs.astral.sh/uv/):
|
|
63
|
+
```
|
|
64
|
+
# macOS / linux / BashOnWindows
|
|
65
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
66
|
+
|
|
67
|
+
# Windows powershell
|
|
68
|
+
powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
|
|
69
|
+
```
|
|
70
|
+
Then install the virtual environment from the root directory:
|
|
71
|
+
```
|
|
72
|
+
uv sync
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### Running unit tests
|
|
76
|
+
Open terminal and go to root directory of the project and run:
|
|
77
|
+
````
|
|
78
|
+
uv run pytest
|
|
79
|
+
````
|
|
80
|
+
|
|
81
|
+
### Pre-commit
|
|
82
|
+
There are currently 3 active rules: Ruff-format, Ruff-lint and sync lock file.
|
|
83
|
+
Install pre-commit
|
|
84
|
+
```sh
|
|
85
|
+
pip install pre-commit
|
|
86
|
+
```
|
|
87
|
+
If you've made changes to the pre-commit-config.yaml or its a new project install the hooks with:
|
|
88
|
+
```sh
|
|
89
|
+
pre-commit install
|
|
90
|
+
```
|
|
91
|
+
Now it should run when you do:
|
|
92
|
+
```sh
|
|
93
|
+
git commit
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
By default it only runs against changed files. To force the hooks to run against all files:
|
|
97
|
+
```sh
|
|
98
|
+
pre-commit run --all-files
|
|
99
|
+
```
|
|
100
|
+
if you dont have it installed on your system you can use:
|
|
101
|
+
(but then it won't run when you use the git-cli)
|
|
102
|
+
```sh
|
|
103
|
+
uv run pre-commit
|
|
104
|
+
```
|
|
105
|
+
Read more about [pre-commit](https://pre-commit.com/#intro)
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# microdata-tools
|
|
2
|
+
Tools for the [microdata.no](https://www.microdata.no/) platform
|
|
3
|
+
|
|
4
|
+
`microdata-tools` is a Python library with two audiences:
|
|
5
|
+
- **Data owners** preparing datasets for microdata.no: validate your data and metadata, then encrypt and package the files into a `.tar` archive ready for upload.
|
|
6
|
+
- **microdata.no services** (such as job-executor) that receive packaged datasets: unpackage and decrypt them, and validate them on the receiving end.
|
|
7
|
+
|
|
8
|
+
A dataset normally contains:
|
|
9
|
+
- A CSV file with one identifier, one value and the dates that describe when the value applies.
|
|
10
|
+
- A JSON file describing the dataset, its variable and its values.
|
|
11
|
+
|
|
12
|
+
## Installation
|
|
13
|
+
`microdata-tools` can be installed from PyPI using pip:
|
|
14
|
+
```
|
|
15
|
+
pip install microdata-tools
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Documentation
|
|
19
|
+
|
|
20
|
+
If you're a data owner preparing a dataset, the full guide lives on the documentation site:
|
|
21
|
+
|
|
22
|
+
- **[Getting started](https://statisticsnorway.github.io/microdata-tools/)** — install, prepare, validate and package your dataset, step by step.
|
|
23
|
+
- **[The metadata model](https://statisticsnorway.github.io/microdata-tools/metadata-model/)** — a field-by-field reference for the metadata JSON file.
|
|
24
|
+
- **[The data file format](https://statisticsnorway.github.io/microdata-tools/data-file-format/)** — a reference for the CSV data file, including validation rules.
|
|
25
|
+
- **[The packaging format](https://statisticsnorway.github.io/microdata-tools/packaging-format/)** — how your data is encrypted and packaged.
|
|
26
|
+
|
|
27
|
+
The sections below are aimed at developers integrating `microdata-tools` into another service (such as job-executor), and at contributors to this repository.
|
|
28
|
+
|
|
29
|
+
## Unpackaging a dataset
|
|
30
|
+
|
|
31
|
+
The `unpackage_dataset()` function will reverse the packaging step: it untars the archive and use the combined ML-KEM-768/X25519 private key from `microdata_private_key.pem` to recover the symmetric key, which is then used to decrypt the data file.
|
|
32
|
+
|
|
33
|
+
The packaged file must have the `<DATASET_NAME>.tar` extension. Its contents should be:
|
|
34
|
+
|
|
35
|
+
- `<DATASET_NAME>.json`: Required metadata file.
|
|
36
|
+
- `<DATASET_NAME>.csv.encr`: Optional encrypted data file.
|
|
37
|
+
- `<DATASET_NAME>.kem.encr`: Optional HPKE ciphertext file containing the encrypted symmetric key. Required if `.csv.encr` is present.
|
|
38
|
+
|
|
39
|
+
Decryption uses the combined ML-KEM-768/X25519 private key located at `PRIVATE_KEY_DIR` to recover the symmetric decryption key. The dataset is then stored in `output_dir/archive/unpackaged` after a successful run, or `output_dir/archive/failed` after an unsuccessful one.
|
|
40
|
+
For details on how packaging encrypts a dataset in the first place, see [the packaging format](https://statisticsnorway.github.io/microdata-tools/packaging-format/).
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
## Contribute
|
|
44
|
+
|
|
45
|
+
### Set up
|
|
46
|
+
To work on this repository you need to install [uv](https://docs.astral.sh/uv/):
|
|
47
|
+
```
|
|
48
|
+
# macOS / linux / BashOnWindows
|
|
49
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
50
|
+
|
|
51
|
+
# Windows powershell
|
|
52
|
+
powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
|
|
53
|
+
```
|
|
54
|
+
Then install the virtual environment from the root directory:
|
|
55
|
+
```
|
|
56
|
+
uv sync
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### Running unit tests
|
|
60
|
+
Open terminal and go to root directory of the project and run:
|
|
61
|
+
````
|
|
62
|
+
uv run pytest
|
|
63
|
+
````
|
|
64
|
+
|
|
65
|
+
### Pre-commit
|
|
66
|
+
There are currently 3 active rules: Ruff-format, Ruff-lint and sync lock file.
|
|
67
|
+
Install pre-commit
|
|
68
|
+
```sh
|
|
69
|
+
pip install pre-commit
|
|
70
|
+
```
|
|
71
|
+
If you've made changes to the pre-commit-config.yaml or its a new project install the hooks with:
|
|
72
|
+
```sh
|
|
73
|
+
pre-commit install
|
|
74
|
+
```
|
|
75
|
+
Now it should run when you do:
|
|
76
|
+
```sh
|
|
77
|
+
git commit
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
By default it only runs against changed files. To force the hooks to run against all files:
|
|
81
|
+
```sh
|
|
82
|
+
pre-commit run --all-files
|
|
83
|
+
```
|
|
84
|
+
if you dont have it installed on your system you can use:
|
|
85
|
+
(but then it won't run when you use the git-cli)
|
|
86
|
+
```sh
|
|
87
|
+
uv run pre-commit
|
|
88
|
+
```
|
|
89
|
+
Read more about [pre-commit](https://pre-commit.com/#intro)
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "microdata-tools"
|
|
3
|
-
version = "2.2.
|
|
3
|
+
version = "2.2.4"
|
|
4
4
|
description = "Tools for the microdata.no platform"
|
|
5
5
|
authors = [{ name = "microdata-developers" }]
|
|
6
6
|
license = { text = "MIT" }
|
|
7
7
|
readme = "README.md"
|
|
8
8
|
requires-python = ">=3.10"
|
|
9
9
|
dependencies = [
|
|
10
|
-
"cryptography>=50.0.
|
|
10
|
+
"cryptography>=50.0.1",
|
|
11
11
|
"numpy>=2.2.0",
|
|
12
12
|
"pyarrow==25.0.1",
|
|
13
|
-
"pydantic>=2.
|
|
14
|
-
"adbc-driver-manager==1.
|
|
15
|
-
"adbc-driver-sqlite==1.
|
|
13
|
+
"pydantic>=2.13.5",
|
|
14
|
+
"adbc-driver-manager==1.12.0",
|
|
15
|
+
"adbc-driver-sqlite==1.12.0",
|
|
16
16
|
]
|
|
17
17
|
|
|
18
18
|
[dependency-groups]
|
|
@@ -20,7 +20,7 @@ dev = [
|
|
|
20
20
|
"pre-commit>=4.3.0",
|
|
21
21
|
"pytest-cov>=6.2.1",
|
|
22
22
|
"pytest>=8.4.1",
|
|
23
|
-
"ruff>=0.
|
|
23
|
+
"ruff>=0.16.8",
|
|
24
24
|
]
|
|
25
25
|
|
|
26
26
|
[tool.uv]
|
microdata_tools-2.2.2/PKG-INFO
DELETED
|
@@ -1,183 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.5
|
|
2
|
-
Name: microdata-tools
|
|
3
|
-
Version: 2.2.2
|
|
4
|
-
Summary: Tools for the microdata.no platform
|
|
5
|
-
Author: microdata-developers
|
|
6
|
-
License: MIT
|
|
7
|
-
License-File: LICENSE.md
|
|
8
|
-
Requires-Python: >=3.10
|
|
9
|
-
Requires-Dist: adbc-driver-manager==1.10.0
|
|
10
|
-
Requires-Dist: adbc-driver-sqlite==1.10.0
|
|
11
|
-
Requires-Dist: cryptography>=50.0.0
|
|
12
|
-
Requires-Dist: numpy>=2.2.0
|
|
13
|
-
Requires-Dist: pyarrow==25.0.1
|
|
14
|
-
Requires-Dist: pydantic>=2.11.7
|
|
15
|
-
Description-Content-Type: text/markdown
|
|
16
|
-
|
|
17
|
-
# microdata-tools
|
|
18
|
-
Tools for the [microdata.no](https://www.microdata.no/) platform
|
|
19
|
-
|
|
20
|
-
## Installation
|
|
21
|
-
`microdata-tools` can be installed from PyPI using pip:
|
|
22
|
-
```
|
|
23
|
-
pip install microdata-tools
|
|
24
|
-
```
|
|
25
|
-
|
|
26
|
-
## Usage
|
|
27
|
-
Once you have your metadata and data files ready to go, they should be named and stored like this:
|
|
28
|
-
```
|
|
29
|
-
my-input-directory/
|
|
30
|
-
MY_DATASET_NAME/
|
|
31
|
-
MY_DATASET_NAME.csv
|
|
32
|
-
MY_DATASET_NAME.json
|
|
33
|
-
```
|
|
34
|
-
The CSV file is optional in some cases.
|
|
35
|
-
|
|
36
|
-
### Package dataset
|
|
37
|
-
The `package_dataset()` function will encrypt and package your dataset as a tar archive. The process is as follows:
|
|
38
|
-
|
|
39
|
-
1. Generate the symmetric key for a dataset.
|
|
40
|
-
2. Encrypt the dataset data (CSV) using an AES-256-GCM symmetric key and store the encrypted file as `<DATASET_NAME>.csv.encr`
|
|
41
|
-
3. Encrypt the symmetric key using HPKE with the combined ML-KEM-768/X25519 public key from `microdata_public_key.pem` and store the resulting HPKE ciphertext as <DATASET_NAME>.kem.encr
|
|
42
|
-
4. Gather the encrypted CSV, ciphertext file and metadata (JSON) file in one tar file.
|
|
43
|
-
|
|
44
|
-
### Unpackage dataset
|
|
45
|
-
The `unpackage_dataset()` function will untar and your dataset and use the combined ML-KEM-768/X25519 private key from `microdata_private_key.pem` to recover the symmetric key, which is then used to decrypt the dataset.
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
The packaged file has to have the `<DATASET_NAME>.tar` extension. Its contents should be as follows:
|
|
49
|
-
|
|
50
|
-
```<DATASET_NAME>.json``` : Required medata file.
|
|
51
|
-
|
|
52
|
-
```<DATASET_NAME>.csv.encr``` : Optional encrypted dataset file.
|
|
53
|
-
|
|
54
|
-
```<DATASET_NAME>.kem.encr``` : Optional HPKE ciphertext file containing the encrypted symmetric key required to decrypt the dataset file. Required if the `.csv.encr` file is present.
|
|
55
|
-
|
|
56
|
-
Decryption uses the combined ML-KEM-768/X25519 private key located at ```PRIVATE_KEY_DIR``` to recover the symmetric decryption key.
|
|
57
|
-
|
|
58
|
-
The packaged file is then stored in `output_dir/archive/unpackaged` after a successful run or `output_dir/archive/failed` after an unsuccessful run.
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
## Example
|
|
62
|
-
|
|
63
|
-
Store your metadata and data files according to the structure described above, and put
|
|
64
|
-
the provided public key in a directory of your choice.
|
|
65
|
-
Then:
|
|
66
|
-
|
|
67
|
-
```py
|
|
68
|
-
from pathlib import Path
|
|
69
|
-
from microdata_tools import package_dataset
|
|
70
|
-
|
|
71
|
-
package_dataset(
|
|
72
|
-
public_key_dir=Path("path/to/key_directory"),
|
|
73
|
-
dataset_dir=Path("path/to/MY_DATASET_NAME"),
|
|
74
|
-
output_dir=Path("path/to/output"),
|
|
75
|
-
)
|
|
76
|
-
```
|
|
77
|
-
|
|
78
|
-
This produces `path/to/output/MY_DATASET_NAME.tar`, which can be uploaded to microdata.
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
### Validation
|
|
82
|
-
|
|
83
|
-
Once you have your metadata and data files ready to go, they should be named and stored like this:
|
|
84
|
-
```
|
|
85
|
-
my-input-directory/
|
|
86
|
-
MY_DATASET_NAME/
|
|
87
|
-
MY_DATASET_NAME.csv
|
|
88
|
-
MY_DATASET_NAME.json
|
|
89
|
-
```
|
|
90
|
-
Note that the filename only allows upper case letters A-Z, number 0-9 and underscores.
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
Import microdata-tools in your script and validate your files:
|
|
94
|
-
```py
|
|
95
|
-
from microdata_tools import validate_dataset
|
|
96
|
-
|
|
97
|
-
validation_errors = validate_dataset(
|
|
98
|
-
"MY_DATASET_NAME",
|
|
99
|
-
input_directory="path/to/my-input-directory"
|
|
100
|
-
)
|
|
101
|
-
|
|
102
|
-
if not validation_errors:
|
|
103
|
-
print("My dataset is valid")
|
|
104
|
-
else:
|
|
105
|
-
print("Dataset is invalid :(")
|
|
106
|
-
# You can print your errors like this:
|
|
107
|
-
for error in validation_errors:
|
|
108
|
-
print(error)
|
|
109
|
-
```
|
|
110
|
-
|
|
111
|
-
For a more in-depth explanation of usage visit [the usage documentation](https://statisticsnorway.github.io/microdata-tools/USAGE).
|
|
112
|
-
|
|
113
|
-
### Data format description
|
|
114
|
-
A dataset as defined in microdata consists of one data file, and one metadata file.
|
|
115
|
-
|
|
116
|
-
The data file is a csv file seperated by semicolons. A valid example would be:
|
|
117
|
-
```csv
|
|
118
|
-
000000000000001;123;2020-01-01;2020-12-31;
|
|
119
|
-
000000000000002;123;2020-01-01;2020-12-31;
|
|
120
|
-
000000000000003;123;2020-01-01;2020-12-31;
|
|
121
|
-
000000000000004;123;2020-01-01;2020-12-31;
|
|
122
|
-
```
|
|
123
|
-
|
|
124
|
-
For datasets with temporalityType FIXED the value does not change over time, so there is no time period to describe — both the start and stop columns should be left empty. On microdata.no such variables are shown with an infinite validity period ("∞"), see [BEFOLKNING_KJOENN](https://microdata.no/discovery/variable/no.ssb.fdb/56/BEFOLKNING_KJOENN) for an example.
|
|
125
|
-
```csv
|
|
126
|
-
000000000000001;123;;;
|
|
127
|
-
000000000000002;123;;;
|
|
128
|
-
000000000000003;123;;;
|
|
129
|
-
000000000000004;123;;;
|
|
130
|
-
```
|
|
131
|
-
|
|
132
|
-
Read more about the data format and columns in [the documentation](https://statisticsnorway.github.io/microdata-tools/).
|
|
133
|
-
|
|
134
|
-
The metadata files should be in json format. The requirements for the metadata is best described through the [Pydantic model](/microdata_tools/validation/model/metadata.py), [the examples](/docs/examples), and [the metadata model](https://statisticsnorway.github.io/microdata-tools/metadata-model/).
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
## Contribute
|
|
138
|
-
|
|
139
|
-
### Set up
|
|
140
|
-
To work on this repository you need to install [uv](https://docs.astral.sh/uv/):
|
|
141
|
-
```
|
|
142
|
-
# macOS / linux / BashOnWindows
|
|
143
|
-
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
144
|
-
|
|
145
|
-
# Windows powershell
|
|
146
|
-
powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
|
|
147
|
-
```
|
|
148
|
-
Then install the virtual environment from the root directory:
|
|
149
|
-
```
|
|
150
|
-
uv sync
|
|
151
|
-
```
|
|
152
|
-
|
|
153
|
-
### Running unit tests
|
|
154
|
-
Open terminal and go to root directory of the project and run:
|
|
155
|
-
````
|
|
156
|
-
uv run pytest
|
|
157
|
-
````
|
|
158
|
-
|
|
159
|
-
### Pre-commit
|
|
160
|
-
There are currently 3 active rules: Ruff-format, Ruff-lint and sync lock file.
|
|
161
|
-
Install pre-commit
|
|
162
|
-
```sh
|
|
163
|
-
pip install pre-commit
|
|
164
|
-
```
|
|
165
|
-
If you've made changes to the pre-commit-config.yaml or its a new project install the hooks with:
|
|
166
|
-
```sh
|
|
167
|
-
pre-commit install
|
|
168
|
-
```
|
|
169
|
-
Now it should run when you do:
|
|
170
|
-
```sh
|
|
171
|
-
git commit
|
|
172
|
-
```
|
|
173
|
-
|
|
174
|
-
By default it only runs against changed files. To force the hooks to run against all files:
|
|
175
|
-
```sh
|
|
176
|
-
pre-commit run --all-files
|
|
177
|
-
```
|
|
178
|
-
if you dont have it installed on your system you can use:
|
|
179
|
-
(but then it won't run when you use the git-cli)
|
|
180
|
-
```sh
|
|
181
|
-
uv run pre-commit
|
|
182
|
-
```
|
|
183
|
-
Read more about [pre-commit](https://pre-commit.com/#intro)
|
microdata_tools-2.2.2/README.md
DELETED
|
@@ -1,167 +0,0 @@
|
|
|
1
|
-
# microdata-tools
|
|
2
|
-
Tools for the [microdata.no](https://www.microdata.no/) platform
|
|
3
|
-
|
|
4
|
-
## Installation
|
|
5
|
-
`microdata-tools` can be installed from PyPI using pip:
|
|
6
|
-
```
|
|
7
|
-
pip install microdata-tools
|
|
8
|
-
```
|
|
9
|
-
|
|
10
|
-
## Usage
|
|
11
|
-
Once you have your metadata and data files ready to go, they should be named and stored like this:
|
|
12
|
-
```
|
|
13
|
-
my-input-directory/
|
|
14
|
-
MY_DATASET_NAME/
|
|
15
|
-
MY_DATASET_NAME.csv
|
|
16
|
-
MY_DATASET_NAME.json
|
|
17
|
-
```
|
|
18
|
-
The CSV file is optional in some cases.
|
|
19
|
-
|
|
20
|
-
### Package dataset
|
|
21
|
-
The `package_dataset()` function will encrypt and package your dataset as a tar archive. The process is as follows:
|
|
22
|
-
|
|
23
|
-
1. Generate the symmetric key for a dataset.
|
|
24
|
-
2. Encrypt the dataset data (CSV) using an AES-256-GCM symmetric key and store the encrypted file as `<DATASET_NAME>.csv.encr`
|
|
25
|
-
3. Encrypt the symmetric key using HPKE with the combined ML-KEM-768/X25519 public key from `microdata_public_key.pem` and store the resulting HPKE ciphertext as <DATASET_NAME>.kem.encr
|
|
26
|
-
4. Gather the encrypted CSV, ciphertext file and metadata (JSON) file in one tar file.
|
|
27
|
-
|
|
28
|
-
### Unpackage dataset
|
|
29
|
-
The `unpackage_dataset()` function will untar and your dataset and use the combined ML-KEM-768/X25519 private key from `microdata_private_key.pem` to recover the symmetric key, which is then used to decrypt the dataset.
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
The packaged file has to have the `<DATASET_NAME>.tar` extension. Its contents should be as follows:
|
|
33
|
-
|
|
34
|
-
```<DATASET_NAME>.json``` : Required medata file.
|
|
35
|
-
|
|
36
|
-
```<DATASET_NAME>.csv.encr``` : Optional encrypted dataset file.
|
|
37
|
-
|
|
38
|
-
```<DATASET_NAME>.kem.encr``` : Optional HPKE ciphertext file containing the encrypted symmetric key required to decrypt the dataset file. Required if the `.csv.encr` file is present.
|
|
39
|
-
|
|
40
|
-
Decryption uses the combined ML-KEM-768/X25519 private key located at ```PRIVATE_KEY_DIR``` to recover the symmetric decryption key.
|
|
41
|
-
|
|
42
|
-
The packaged file is then stored in `output_dir/archive/unpackaged` after a successful run or `output_dir/archive/failed` after an unsuccessful run.
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
## Example
|
|
46
|
-
|
|
47
|
-
Store your metadata and data files according to the structure described above, and put
|
|
48
|
-
the provided public key in a directory of your choice.
|
|
49
|
-
Then:
|
|
50
|
-
|
|
51
|
-
```py
|
|
52
|
-
from pathlib import Path
|
|
53
|
-
from microdata_tools import package_dataset
|
|
54
|
-
|
|
55
|
-
package_dataset(
|
|
56
|
-
public_key_dir=Path("path/to/key_directory"),
|
|
57
|
-
dataset_dir=Path("path/to/MY_DATASET_NAME"),
|
|
58
|
-
output_dir=Path("path/to/output"),
|
|
59
|
-
)
|
|
60
|
-
```
|
|
61
|
-
|
|
62
|
-
This produces `path/to/output/MY_DATASET_NAME.tar`, which can be uploaded to microdata.
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
### Validation
|
|
66
|
-
|
|
67
|
-
Once you have your metadata and data files ready to go, they should be named and stored like this:
|
|
68
|
-
```
|
|
69
|
-
my-input-directory/
|
|
70
|
-
MY_DATASET_NAME/
|
|
71
|
-
MY_DATASET_NAME.csv
|
|
72
|
-
MY_DATASET_NAME.json
|
|
73
|
-
```
|
|
74
|
-
Note that the filename only allows upper case letters A-Z, number 0-9 and underscores.
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
Import microdata-tools in your script and validate your files:
|
|
78
|
-
```py
|
|
79
|
-
from microdata_tools import validate_dataset
|
|
80
|
-
|
|
81
|
-
validation_errors = validate_dataset(
|
|
82
|
-
"MY_DATASET_NAME",
|
|
83
|
-
input_directory="path/to/my-input-directory"
|
|
84
|
-
)
|
|
85
|
-
|
|
86
|
-
if not validation_errors:
|
|
87
|
-
print("My dataset is valid")
|
|
88
|
-
else:
|
|
89
|
-
print("Dataset is invalid :(")
|
|
90
|
-
# You can print your errors like this:
|
|
91
|
-
for error in validation_errors:
|
|
92
|
-
print(error)
|
|
93
|
-
```
|
|
94
|
-
|
|
95
|
-
For a more in-depth explanation of usage visit [the usage documentation](https://statisticsnorway.github.io/microdata-tools/USAGE).
|
|
96
|
-
|
|
97
|
-
### Data format description
|
|
98
|
-
A dataset as defined in microdata consists of one data file, and one metadata file.
|
|
99
|
-
|
|
100
|
-
The data file is a csv file seperated by semicolons. A valid example would be:
|
|
101
|
-
```csv
|
|
102
|
-
000000000000001;123;2020-01-01;2020-12-31;
|
|
103
|
-
000000000000002;123;2020-01-01;2020-12-31;
|
|
104
|
-
000000000000003;123;2020-01-01;2020-12-31;
|
|
105
|
-
000000000000004;123;2020-01-01;2020-12-31;
|
|
106
|
-
```
|
|
107
|
-
|
|
108
|
-
For datasets with temporalityType FIXED the value does not change over time, so there is no time period to describe — both the start and stop columns should be left empty. On microdata.no such variables are shown with an infinite validity period ("∞"), see [BEFOLKNING_KJOENN](https://microdata.no/discovery/variable/no.ssb.fdb/56/BEFOLKNING_KJOENN) for an example.
|
|
109
|
-
```csv
|
|
110
|
-
000000000000001;123;;;
|
|
111
|
-
000000000000002;123;;;
|
|
112
|
-
000000000000003;123;;;
|
|
113
|
-
000000000000004;123;;;
|
|
114
|
-
```
|
|
115
|
-
|
|
116
|
-
Read more about the data format and columns in [the documentation](https://statisticsnorway.github.io/microdata-tools/).
|
|
117
|
-
|
|
118
|
-
The metadata files should be in json format. The requirements for the metadata is best described through the [Pydantic model](/microdata_tools/validation/model/metadata.py), [the examples](/docs/examples), and [the metadata model](https://statisticsnorway.github.io/microdata-tools/metadata-model/).
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
## Contribute
|
|
122
|
-
|
|
123
|
-
### Set up
|
|
124
|
-
To work on this repository you need to install [uv](https://docs.astral.sh/uv/):
|
|
125
|
-
```
|
|
126
|
-
# macOS / linux / BashOnWindows
|
|
127
|
-
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
128
|
-
|
|
129
|
-
# Windows powershell
|
|
130
|
-
powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
|
|
131
|
-
```
|
|
132
|
-
Then install the virtual environment from the root directory:
|
|
133
|
-
```
|
|
134
|
-
uv sync
|
|
135
|
-
```
|
|
136
|
-
|
|
137
|
-
### Running unit tests
|
|
138
|
-
Open terminal and go to root directory of the project and run:
|
|
139
|
-
````
|
|
140
|
-
uv run pytest
|
|
141
|
-
````
|
|
142
|
-
|
|
143
|
-
### Pre-commit
|
|
144
|
-
There are currently 3 active rules: Ruff-format, Ruff-lint and sync lock file.
|
|
145
|
-
Install pre-commit
|
|
146
|
-
```sh
|
|
147
|
-
pip install pre-commit
|
|
148
|
-
```
|
|
149
|
-
If you've made changes to the pre-commit-config.yaml or its a new project install the hooks with:
|
|
150
|
-
```sh
|
|
151
|
-
pre-commit install
|
|
152
|
-
```
|
|
153
|
-
Now it should run when you do:
|
|
154
|
-
```sh
|
|
155
|
-
git commit
|
|
156
|
-
```
|
|
157
|
-
|
|
158
|
-
By default it only runs against changed files. To force the hooks to run against all files:
|
|
159
|
-
```sh
|
|
160
|
-
pre-commit run --all-files
|
|
161
|
-
```
|
|
162
|
-
if you dont have it installed on your system you can use:
|
|
163
|
-
(but then it won't run when you use the git-cli)
|
|
164
|
-
```sh
|
|
165
|
-
uv run pre-commit
|
|
166
|
-
```
|
|
167
|
-
Read more about [pre-commit](https://pre-commit.com/#intro)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/packaging/exceptions/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/adapter/local_storage.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/exceptions/__init__.py
RENAMED
|
File without changes
|
{microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/model/__init__.py
RENAMED
|
File without changes
|
{microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/model/metadata.py
RENAMED
|
File without changes
|
{microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/steps/data_reader.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{microdata_tools-2.2.2 → microdata_tools-2.2.4}/microdata_tools/validation/steps/metadata_reader.py
RENAMED
|
File without changes
|