dapla-toolbelt-metadata 0.4.2__py3-none-any.whl → 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dapla-toolbelt-metadata might be problematic. Click here for more details.
- dapla_metadata/__init__.py +9 -0
- dapla_metadata/dapla/__init__.py +4 -0
- dapla_metadata/{_shared → dapla}/user_info.py +66 -20
- dapla_metadata/datasets/code_list.py +1 -1
- dapla_metadata/datasets/core.py +1 -1
- dapla_metadata/datasets/dapla_dataset_path_info.py +128 -14
- dapla_metadata/datasets/dataset_parser.py +21 -15
- dapla_metadata/datasets/model_backwards_compatibility.py +6 -6
- dapla_metadata/datasets/model_validation.py +2 -2
- dapla_metadata/datasets/utility/constants.py +1 -0
- dapla_metadata/datasets/utility/enums.py +1 -1
- dapla_metadata/datasets/utility/utils.py +8 -12
- dapla_metadata/standards/__init__.py +4 -0
- dapla_metadata/standards/name_validator.py +250 -0
- dapla_metadata/standards/standard_validators.py +98 -0
- dapla_metadata/standards/utils/__init__.py +1 -0
- dapla_metadata/standards/utils/constants.py +49 -0
- dapla_metadata/variable_definitions/__init__.py +5 -3
- dapla_metadata/variable_definitions/{generated → _generated}/.openapi-generator/FILES +0 -5
- dapla_metadata/variable_definitions/_generated/.openapi-generator/VERSION +1 -0
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/__init__.py +0 -5
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/api/__init__.py +0 -1
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/api/data_migration_api.py +2 -2
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/api/draft_variable_definitions_api.py +14 -14
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/api/patches_api.py +15 -15
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/api/validity_periods_api.py +8 -281
- dapla_metadata/variable_definitions/{generated/vardef_client/api/public_api.py → _generated/vardef_client/api/variable_definitions_api.py} +73 -358
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/models/__init__.py +2 -6
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/models/complete_response.py +8 -32
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/models/contact.py +2 -2
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/models/draft.py +8 -23
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/models/language_string_type.py +7 -6
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/models/owner.py +2 -2
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/models/patch.py +16 -61
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/models/problem.py +2 -2
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/models/update_draft.py +22 -55
- dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/models/validity_period.py +14 -48
- dapla_metadata/variable_definitions/_generated/vardef_client/models/variable_status.py +33 -0
- dapla_metadata/variable_definitions/_utils/__init__.py +1 -0
- dapla_metadata/variable_definitions/{_client.py → _utils/_client.py} +5 -3
- dapla_metadata/variable_definitions/{config.py → _utils/config.py} +25 -1
- dapla_metadata/variable_definitions/_utils/constants.py +41 -0
- dapla_metadata/variable_definitions/_utils/descriptions.py +89 -0
- dapla_metadata/variable_definitions/_utils/files.py +273 -0
- dapla_metadata/variable_definitions/_utils/template_files.py +112 -0
- dapla_metadata/variable_definitions/_utils/variable_definition_files.py +93 -0
- dapla_metadata/variable_definitions/exceptions.py +141 -11
- dapla_metadata/variable_definitions/resources/vardef_model_descriptions_nb.yaml +63 -0
- dapla_metadata/variable_definitions/vardef.py +131 -10
- dapla_metadata/variable_definitions/variable_definition.py +251 -43
- {dapla_toolbelt_metadata-0.4.2.dist-info → dapla_toolbelt_metadata-0.6.0.dist-info}/METADATA +8 -10
- dapla_toolbelt_metadata-0.6.0.dist-info/RECORD +89 -0
- {dapla_toolbelt_metadata-0.4.2.dist-info → dapla_toolbelt_metadata-0.6.0.dist-info}/WHEEL +1 -1
- dapla_metadata/variable_definitions/generated/.openapi-generator/VERSION +0 -1
- dapla_metadata/variable_definitions/generated/vardef_client/api/variable_definitions_api.py +0 -1205
- dapla_metadata/variable_definitions/generated/vardef_client/models/klass_reference.py +0 -99
- dapla_metadata/variable_definitions/generated/vardef_client/models/rendered_contact.py +0 -92
- dapla_metadata/variable_definitions/generated/vardef_client/models/rendered_variable_definition.py +0 -235
- dapla_metadata/variable_definitions/generated/vardef_client/models/supported_languages.py +0 -33
- dapla_metadata/variable_definitions/generated/vardef_client/models/variable_status.py +0 -33
- dapla_toolbelt_metadata-0.4.2.dist-info/RECORD +0 -81
- /dapla_metadata/variable_definitions/{generated → _generated}/.openapi-generator-ignore +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/README.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/__init__.py +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/api_client.py +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/api_response.py +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/configuration.py +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/CompleteResponse.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/Contact.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/DataMigrationApi.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/Draft.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/DraftVariableDefinitionsApi.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/LanguageStringType.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/Owner.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/Patch.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/PatchesApi.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/PublicApi.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/SupportedLanguages.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/UpdateDraft.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/ValidityPeriod.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/ValidityPeriodsApi.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/VariableDefinitionsApi.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/docs/VariableStatus.md +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/exceptions.py +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/py.typed +0 -0
- /dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/rest.py +0 -0
- {dapla_toolbelt_metadata-0.4.2.dist-info → dapla_toolbelt_metadata-0.6.0.dist-info}/LICENSE +0 -0
dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/models/update_draft.py
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
"""Variable Definitions
|
|
1
|
+
"""Internal Variable Definitions Administration API
|
|
2
2
|
|
|
3
|
-
## Introduction Variable Definitions are centralized definitions of concrete variables which are typically present in multiple datasets. Variable Definitions support standardization of data and metadata and facilitate sharing and joining of data by clarifying when variables have an identical definition. ## Maintenance of Variable Definitions This API allows for creation, maintenance and access of Variable Definitions. ### Ownership Creation and maintenance of variables may only be performed by Statistics Norway employees representing a specific Dapla team, who are defined as the owners of a given Variable Definition. The team an owner represents must be specified when making a request through the `active_group` query parameter. All maintenance is to be performed by the owners, with no intervention from administrators. ### Status All Variable Definitions have an associated status. The possible values for status are `DRAFT`, `PUBLISHED_INTERNAL` and `PUBLISHED_EXTERNAL`.
|
|
3
|
+
## Introduction Variable Definitions are centralized definitions of concrete variables which are typically present in multiple datasets. Variable Definitions support standardization of data and metadata and facilitate sharing and joining of data by clarifying when variables have an identical definition. ## Maintenance of Variable Definitions This API allows for creation, maintenance and access of Variable Definitions. ### Ownership Creation and maintenance of variables may only be performed by Statistics Norway employees representing a specific Dapla team, who are defined as the owners of a given Variable Definition. The team an owner represents must be specified when making a request through the `active_group` query parameter. All maintenance is to be performed by the owners, with no intervention from administrators. ### Status All Variable Definitions have an associated status. The possible values for status are `DRAFT`, `PUBLISHED_INTERNAL` and `PUBLISHED_EXTERNAL`. #### Draft When a Variable Definition is created it is assigned the status `DRAFT`. Under this status the Variable Definition is: - Only visible to Statistics Norway employees. - Mutable (it may be changed directly without need for versioning). - Not suitable to refer to from other systems. This status may be changed to `PUBLISHED_INTERNAL` or `PUBLISHED_EXTERNAL` with a direct update. #### Published Internal Under this status the Variable Definition is: - Only visible to Statistics Norway employees. - Immutable (all changes are versioned). - Suitable to refer to in internal systems for statistics production. - Not suitable to refer to for external use (for example in Statistikkbanken). This status may be changed to `PUBLISHED_EXTERNAL` by creating a Patch version. #### Published External Under this status the Variable Definition is: - Visible to the general public. - Immutable (all changes are versioned). - Suitable to refer to from any system. This status may not be changed as it would break immutability. If a Variable Definition is no longer relevant then its period of validity should be ended by specifying a `valid_until` date in a Patch version. ### Immutability Variable Definitions are immutable. This means that any changes must be performed in a strict versioning system. Consumers can avoid being exposed to breaking changes by specifying a `date_of_validity` when they request a Variable Definition. #### Patches Patches are for changes which do not affect the fundamental meaning of the Variable Definition. #### Validity Periods Validity Periods are versions with a period defined by a `valid_from` date and optionally a `valid_until` date. If the fundamental meaning of a Variable Definition is to be changed, it should be done by creating a new Validity Period.
|
|
4
4
|
|
|
5
5
|
The version of the OpenAPI document: 0.1
|
|
6
6
|
Contact: metadata@ssb.no
|
|
@@ -36,36 +36,31 @@ from ..models.variable_status import VariableStatus
|
|
|
36
36
|
class UpdateDraft(BaseModel):
|
|
37
37
|
"""Update variable definition Data structure with all fields optional for updating a Draft Variable Definition."""
|
|
38
38
|
|
|
39
|
-
name: LanguageStringType | None =
|
|
40
|
-
default=None,
|
|
41
|
-
description="Name of the variable. Must be unique for a given Unit Type and Owner combination.",
|
|
42
|
-
)
|
|
39
|
+
name: LanguageStringType | None = None
|
|
43
40
|
short_name: Annotated[str, Field(strict=True)] | None = Field(
|
|
44
41
|
default=None,
|
|
45
42
|
description="Recommended short name. Must be unique within an organization.",
|
|
46
43
|
)
|
|
47
|
-
definition: LanguageStringType | None =
|
|
48
|
-
default=None, description="Definition of the variable."
|
|
49
|
-
)
|
|
44
|
+
definition: LanguageStringType | None = None
|
|
50
45
|
classification_reference: StrictStr | None = Field(
|
|
51
46
|
default=None,
|
|
52
47
|
description="ID of a classification or code list from Klass. The given classification defines all possible values for the defined variable.",
|
|
53
48
|
)
|
|
54
|
-
unit_types: list[
|
|
49
|
+
unit_types: list[Annotated[str, Field(min_length=1, strict=True)]] | None = Field(
|
|
55
50
|
default=None,
|
|
56
51
|
description="A list of one or more unit types, e.g. person, vehicle, household. Must be defined as codes from https://www.ssb.no/klass/klassifikasjoner/702.",
|
|
57
52
|
)
|
|
58
|
-
subject_fields: list[
|
|
59
|
-
|
|
60
|
-
|
|
53
|
+
subject_fields: list[Annotated[str, Field(min_length=1, strict=True)]] | None = (
|
|
54
|
+
Field(
|
|
55
|
+
default=None,
|
|
56
|
+
description="A list of subject fields that the variable is used in. Must be defined as codes from https://www.ssb.no/klass/klassifikasjoner/618.",
|
|
57
|
+
)
|
|
61
58
|
)
|
|
62
59
|
contains_special_categories_of_personal_data: StrictBool | None = Field(
|
|
63
60
|
default=None,
|
|
64
61
|
description="True if variable instances contain particularly sensitive information. Applies even if the information or identifiers are pseudonymized. Information within the following categories are regarded as particularly sensitive: Ethnicity, Political alignment, Religion, Philosophical beliefs, Union membership, Genetics, Biometrics, Health, Sexual relations, Sexual orientation",
|
|
65
62
|
)
|
|
66
|
-
variable_status: VariableStatus | None =
|
|
67
|
-
default=None, description="Status of the life cycle of the variable"
|
|
68
|
-
)
|
|
63
|
+
variable_status: VariableStatus | None = None
|
|
69
64
|
measurement_type: StrictStr | None = Field(
|
|
70
65
|
default=None,
|
|
71
66
|
description="Type of measurement for the variable, e.g. length, volume, currency. Must be defined as codes from https://www.ssb.no/klass/klassifikasjoner/303",
|
|
@@ -74,22 +69,17 @@ class UpdateDraft(BaseModel):
|
|
|
74
69
|
default=None,
|
|
75
70
|
description="The variable definition is valid from this date inclusive",
|
|
76
71
|
)
|
|
72
|
+
valid_until: date | None = None
|
|
77
73
|
external_reference_uri: StrictStr | None = Field(
|
|
78
74
|
default=None, description="A link (URI) to an external definition/documentation"
|
|
79
75
|
)
|
|
80
|
-
comment: LanguageStringType | None =
|
|
81
|
-
default=None,
|
|
82
|
-
description="Optional comment to explain the definition or communicate potential changes.",
|
|
83
|
-
)
|
|
76
|
+
comment: LanguageStringType | None = None
|
|
84
77
|
related_variable_definition_uris: list[StrictStr] | None = Field(
|
|
85
78
|
default=None,
|
|
86
79
|
description="Link(s) to related definitions of variables - a list of one or more definitions. For example for a variable after-tax income it could be relevant to link to definitions of income from work, property income etc.",
|
|
87
80
|
)
|
|
88
|
-
owner: Owner | None =
|
|
89
|
-
|
|
90
|
-
description="Owner of the definition, i.e. responsible Dapla team (statistics team) and information about access management groups.",
|
|
91
|
-
)
|
|
92
|
-
contact: Contact | None = Field(default=None, description="Contact details")
|
|
81
|
+
owner: Owner | None = None
|
|
82
|
+
contact: Contact | None = None
|
|
93
83
|
__properties: ClassVar[list[str]] = [
|
|
94
84
|
"name",
|
|
95
85
|
"short_name",
|
|
@@ -101,6 +91,7 @@ class UpdateDraft(BaseModel):
|
|
|
101
91
|
"variable_status",
|
|
102
92
|
"measurement_type",
|
|
103
93
|
"valid_from",
|
|
94
|
+
"valid_until",
|
|
104
95
|
"external_reference_uri",
|
|
105
96
|
"comment",
|
|
106
97
|
"related_variable_definition_uris",
|
|
@@ -170,21 +161,11 @@ class UpdateDraft(BaseModel):
|
|
|
170
161
|
# override the default output from pydantic by calling `to_dict()` of contact
|
|
171
162
|
if self.contact:
|
|
172
163
|
_dict["contact"] = self.contact.to_dict()
|
|
173
|
-
# set to None if name (nullable) is None
|
|
174
|
-
# and model_fields_set contains the field
|
|
175
|
-
if self.name is None and "name" in self.model_fields_set:
|
|
176
|
-
_dict["name"] = None
|
|
177
|
-
|
|
178
164
|
# set to None if short_name (nullable) is None
|
|
179
165
|
# and model_fields_set contains the field
|
|
180
166
|
if self.short_name is None and "short_name" in self.model_fields_set:
|
|
181
167
|
_dict["short_name"] = None
|
|
182
168
|
|
|
183
|
-
# set to None if definition (nullable) is None
|
|
184
|
-
# and model_fields_set contains the field
|
|
185
|
-
if self.definition is None and "definition" in self.model_fields_set:
|
|
186
|
-
_dict["definition"] = None
|
|
187
|
-
|
|
188
169
|
# set to None if classification_reference (nullable) is None
|
|
189
170
|
# and model_fields_set contains the field
|
|
190
171
|
if (
|
|
@@ -211,11 +192,6 @@ class UpdateDraft(BaseModel):
|
|
|
211
192
|
):
|
|
212
193
|
_dict["contains_special_categories_of_personal_data"] = None
|
|
213
194
|
|
|
214
|
-
# set to None if variable_status (nullable) is None
|
|
215
|
-
# and model_fields_set contains the field
|
|
216
|
-
if self.variable_status is None and "variable_status" in self.model_fields_set:
|
|
217
|
-
_dict["variable_status"] = None
|
|
218
|
-
|
|
219
195
|
# set to None if measurement_type (nullable) is None
|
|
220
196
|
# and model_fields_set contains the field
|
|
221
197
|
if (
|
|
@@ -229,6 +205,11 @@ class UpdateDraft(BaseModel):
|
|
|
229
205
|
if self.valid_from is None and "valid_from" in self.model_fields_set:
|
|
230
206
|
_dict["valid_from"] = None
|
|
231
207
|
|
|
208
|
+
# set to None if valid_until (nullable) is None
|
|
209
|
+
# and model_fields_set contains the field
|
|
210
|
+
if self.valid_until is None and "valid_until" in self.model_fields_set:
|
|
211
|
+
_dict["valid_until"] = None
|
|
212
|
+
|
|
232
213
|
# set to None if external_reference_uri (nullable) is None
|
|
233
214
|
# and model_fields_set contains the field
|
|
234
215
|
if (
|
|
@@ -237,11 +218,6 @@ class UpdateDraft(BaseModel):
|
|
|
237
218
|
):
|
|
238
219
|
_dict["external_reference_uri"] = None
|
|
239
220
|
|
|
240
|
-
# set to None if comment (nullable) is None
|
|
241
|
-
# and model_fields_set contains the field
|
|
242
|
-
if self.comment is None and "comment" in self.model_fields_set:
|
|
243
|
-
_dict["comment"] = None
|
|
244
|
-
|
|
245
221
|
# set to None if related_variable_definition_uris (nullable) is None
|
|
246
222
|
# and model_fields_set contains the field
|
|
247
223
|
if (
|
|
@@ -250,16 +226,6 @@ class UpdateDraft(BaseModel):
|
|
|
250
226
|
):
|
|
251
227
|
_dict["related_variable_definition_uris"] = None
|
|
252
228
|
|
|
253
|
-
# set to None if owner (nullable) is None
|
|
254
|
-
# and model_fields_set contains the field
|
|
255
|
-
if self.owner is None and "owner" in self.model_fields_set:
|
|
256
|
-
_dict["owner"] = None
|
|
257
|
-
|
|
258
|
-
# set to None if contact (nullable) is None
|
|
259
|
-
# and model_fields_set contains the field
|
|
260
|
-
if self.contact is None and "contact" in self.model_fields_set:
|
|
261
|
-
_dict["contact"] = None
|
|
262
|
-
|
|
263
229
|
return _dict
|
|
264
230
|
|
|
265
231
|
@classmethod
|
|
@@ -289,6 +255,7 @@ class UpdateDraft(BaseModel):
|
|
|
289
255
|
"variable_status": obj.get("variable_status"),
|
|
290
256
|
"measurement_type": obj.get("measurement_type"),
|
|
291
257
|
"valid_from": obj.get("valid_from"),
|
|
258
|
+
"valid_until": obj.get("valid_until"),
|
|
292
259
|
"external_reference_uri": obj.get("external_reference_uri"),
|
|
293
260
|
"comment": LanguageStringType.from_dict(obj["comment"])
|
|
294
261
|
if obj.get("comment") is not None
|
dapla_metadata/variable_definitions/{generated → _generated}/vardef_client/models/validity_period.py
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
"""Variable Definitions
|
|
1
|
+
"""Internal Variable Definitions Administration API
|
|
2
2
|
|
|
3
|
-
## Introduction Variable Definitions are centralized definitions of concrete variables which are typically present in multiple datasets. Variable Definitions support standardization of data and metadata and facilitate sharing and joining of data by clarifying when variables have an identical definition. ## Maintenance of Variable Definitions This API allows for creation, maintenance and access of Variable Definitions. ### Ownership Creation and maintenance of variables may only be performed by Statistics Norway employees representing a specific Dapla team, who are defined as the owners of a given Variable Definition. The team an owner represents must be specified when making a request through the `active_group` query parameter. All maintenance is to be performed by the owners, with no intervention from administrators. ### Status All Variable Definitions have an associated status. The possible values for status are `DRAFT`, `PUBLISHED_INTERNAL` and `PUBLISHED_EXTERNAL`.
|
|
3
|
+
## Introduction Variable Definitions are centralized definitions of concrete variables which are typically present in multiple datasets. Variable Definitions support standardization of data and metadata and facilitate sharing and joining of data by clarifying when variables have an identical definition. ## Maintenance of Variable Definitions This API allows for creation, maintenance and access of Variable Definitions. ### Ownership Creation and maintenance of variables may only be performed by Statistics Norway employees representing a specific Dapla team, who are defined as the owners of a given Variable Definition. The team an owner represents must be specified when making a request through the `active_group` query parameter. All maintenance is to be performed by the owners, with no intervention from administrators. ### Status All Variable Definitions have an associated status. The possible values for status are `DRAFT`, `PUBLISHED_INTERNAL` and `PUBLISHED_EXTERNAL`. #### Draft When a Variable Definition is created it is assigned the status `DRAFT`. Under this status the Variable Definition is: - Only visible to Statistics Norway employees. - Mutable (it may be changed directly without need for versioning). - Not suitable to refer to from other systems. This status may be changed to `PUBLISHED_INTERNAL` or `PUBLISHED_EXTERNAL` with a direct update. #### Published Internal Under this status the Variable Definition is: - Only visible to Statistics Norway employees. - Immutable (all changes are versioned). - Suitable to refer to in internal systems for statistics production. - Not suitable to refer to for external use (for example in Statistikkbanken). This status may be changed to `PUBLISHED_EXTERNAL` by creating a Patch version. #### Published External Under this status the Variable Definition is: - Visible to the general public. - Immutable (all changes are versioned). - Suitable to refer to from any system. This status may not be changed as it would break immutability. If a Variable Definition is no longer relevant then its period of validity should be ended by specifying a `valid_until` date in a Patch version. ### Immutability Variable Definitions are immutable. This means that any changes must be performed in a strict versioning system. Consumers can avoid being exposed to breaking changes by specifying a `date_of_validity` when they request a Variable Definition. #### Patches Patches are for changes which do not affect the fundamental meaning of the Variable Definition. #### Validity Periods Validity Periods are versions with a period defined by a `valid_from` date and optionally a `valid_until` date. If the fundamental meaning of a Variable Definition is to be changed, it should be done by creating a new Validity Period.
|
|
4
4
|
|
|
5
5
|
The version of the OpenAPI document: 0.1
|
|
6
6
|
Contact: metadata@ssb.no
|
|
@@ -15,6 +15,7 @@ import json
|
|
|
15
15
|
import pprint
|
|
16
16
|
import re # noqa: F401
|
|
17
17
|
from datetime import date
|
|
18
|
+
from typing import Annotated
|
|
18
19
|
from typing import Any
|
|
19
20
|
from typing import ClassVar
|
|
20
21
|
|
|
@@ -27,36 +28,31 @@ from typing_extensions import Self
|
|
|
27
28
|
|
|
28
29
|
from ..models.contact import Contact
|
|
29
30
|
from ..models.language_string_type import LanguageStringType
|
|
30
|
-
from ..models.variable_status import VariableStatus
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
class ValidityPeriod(BaseModel):
|
|
34
34
|
"""Create a new Validity Period on a Published Variable Definition."""
|
|
35
35
|
|
|
36
|
-
name: LanguageStringType | None =
|
|
37
|
-
|
|
38
|
-
description="Name of the variable. Must be unique for a given Unit Type and Owner combination.",
|
|
39
|
-
)
|
|
40
|
-
definition: LanguageStringType = Field(description="Definition of the variable.")
|
|
36
|
+
name: LanguageStringType | None = None
|
|
37
|
+
definition: LanguageStringType
|
|
41
38
|
classification_reference: StrictStr | None = Field(
|
|
42
39
|
default=None,
|
|
43
40
|
description="ID of a classification or code list from Klass. The given classification defines all possible values for the defined variable.",
|
|
44
41
|
)
|
|
45
|
-
unit_types: list[
|
|
42
|
+
unit_types: list[Annotated[str, Field(min_length=1, strict=True)]] | None = Field(
|
|
46
43
|
default=None,
|
|
47
44
|
description="A list of one or more unit types, e.g. person, vehicle, household. Must be defined as codes from https://www.ssb.no/klass/klassifikasjoner/702.",
|
|
48
45
|
)
|
|
49
|
-
subject_fields: list[
|
|
50
|
-
|
|
51
|
-
|
|
46
|
+
subject_fields: list[Annotated[str, Field(min_length=1, strict=True)]] | None = (
|
|
47
|
+
Field(
|
|
48
|
+
default=None,
|
|
49
|
+
description="A list of subject fields that the variable is used in. Must be defined as codes from https://www.ssb.no/klass/klassifikasjoner/618.",
|
|
50
|
+
)
|
|
52
51
|
)
|
|
53
52
|
contains_special_categories_of_personal_data: StrictBool | None = Field(
|
|
54
53
|
default=None,
|
|
55
54
|
description="True if variable instances contain particularly sensitive information. Applies even if the information or identifiers are pseudonymized. Information within the following categories are regarded as particularly sensitive: Ethnicity, Political alignment, Religion, Philosophical beliefs, Union membership, Genetics, Biometrics, Health, Sexual relations, Sexual orientation",
|
|
56
55
|
)
|
|
57
|
-
variable_status: VariableStatus | None = Field(
|
|
58
|
-
default=None, description="Status of the life cycle of the variable"
|
|
59
|
-
)
|
|
60
56
|
measurement_type: StrictStr | None = Field(
|
|
61
57
|
default=None,
|
|
62
58
|
description="Type of measurement for the variable, e.g. length, volume, currency. Must be defined as codes from https://www.ssb.no/klass/klassifikasjoner/303",
|
|
@@ -67,15 +63,12 @@ class ValidityPeriod(BaseModel):
|
|
|
67
63
|
external_reference_uri: StrictStr | None = Field(
|
|
68
64
|
default=None, description="A link (URI) to an external definition/documentation"
|
|
69
65
|
)
|
|
70
|
-
comment: LanguageStringType | None =
|
|
71
|
-
default=None,
|
|
72
|
-
description="Optional comment to explain the definition or communicate potential changes.",
|
|
73
|
-
)
|
|
66
|
+
comment: LanguageStringType | None = None
|
|
74
67
|
related_variable_definition_uris: list[StrictStr] | None = Field(
|
|
75
68
|
default=None,
|
|
76
69
|
description="Link(s) to related definitions of variables - a list of one or more definitions. For example for a variable after-tax income it could be relevant to link to definitions of income from work, property income etc.",
|
|
77
70
|
)
|
|
78
|
-
contact: Contact | None =
|
|
71
|
+
contact: Contact | None = None
|
|
79
72
|
__properties: ClassVar[list[str]] = [
|
|
80
73
|
"name",
|
|
81
74
|
"definition",
|
|
@@ -83,7 +76,6 @@ class ValidityPeriod(BaseModel):
|
|
|
83
76
|
"unit_types",
|
|
84
77
|
"subject_fields",
|
|
85
78
|
"contains_special_categories_of_personal_data",
|
|
86
|
-
"variable_status",
|
|
87
79
|
"measurement_type",
|
|
88
80
|
"valid_from",
|
|
89
81
|
"external_reference_uri",
|
|
@@ -121,13 +113,8 @@ class ValidityPeriod(BaseModel):
|
|
|
121
113
|
* `None` is only added to the output dict for nullable fields that
|
|
122
114
|
were set at model initialization. Other fields with value `None`
|
|
123
115
|
are ignored.
|
|
124
|
-
* OpenAPI `readOnly` fields are excluded.
|
|
125
116
|
"""
|
|
126
|
-
excluded_fields: set[str] = set(
|
|
127
|
-
[
|
|
128
|
-
"variable_status",
|
|
129
|
-
]
|
|
130
|
-
)
|
|
117
|
+
excluded_fields: set[str] = set([])
|
|
131
118
|
|
|
132
119
|
_dict = self.model_dump(
|
|
133
120
|
by_alias=True,
|
|
@@ -146,11 +133,6 @@ class ValidityPeriod(BaseModel):
|
|
|
146
133
|
# override the default output from pydantic by calling `to_dict()` of contact
|
|
147
134
|
if self.contact:
|
|
148
135
|
_dict["contact"] = self.contact.to_dict()
|
|
149
|
-
# set to None if name (nullable) is None
|
|
150
|
-
# and model_fields_set contains the field
|
|
151
|
-
if self.name is None and "name" in self.model_fields_set:
|
|
152
|
-
_dict["name"] = None
|
|
153
|
-
|
|
154
136
|
# set to None if classification_reference (nullable) is None
|
|
155
137
|
# and model_fields_set contains the field
|
|
156
138
|
if (
|
|
@@ -177,11 +159,6 @@ class ValidityPeriod(BaseModel):
|
|
|
177
159
|
):
|
|
178
160
|
_dict["contains_special_categories_of_personal_data"] = None
|
|
179
161
|
|
|
180
|
-
# set to None if variable_status (nullable) is None
|
|
181
|
-
# and model_fields_set contains the field
|
|
182
|
-
if self.variable_status is None and "variable_status" in self.model_fields_set:
|
|
183
|
-
_dict["variable_status"] = None
|
|
184
|
-
|
|
185
162
|
# set to None if measurement_type (nullable) is None
|
|
186
163
|
# and model_fields_set contains the field
|
|
187
164
|
if (
|
|
@@ -198,11 +175,6 @@ class ValidityPeriod(BaseModel):
|
|
|
198
175
|
):
|
|
199
176
|
_dict["external_reference_uri"] = None
|
|
200
177
|
|
|
201
|
-
# set to None if comment (nullable) is None
|
|
202
|
-
# and model_fields_set contains the field
|
|
203
|
-
if self.comment is None and "comment" in self.model_fields_set:
|
|
204
|
-
_dict["comment"] = None
|
|
205
|
-
|
|
206
178
|
# set to None if related_variable_definition_uris (nullable) is None
|
|
207
179
|
# and model_fields_set contains the field
|
|
208
180
|
if (
|
|
@@ -211,11 +183,6 @@ class ValidityPeriod(BaseModel):
|
|
|
211
183
|
):
|
|
212
184
|
_dict["related_variable_definition_uris"] = None
|
|
213
185
|
|
|
214
|
-
# set to None if contact (nullable) is None
|
|
215
|
-
# and model_fields_set contains the field
|
|
216
|
-
if self.contact is None and "contact" in self.model_fields_set:
|
|
217
|
-
_dict["contact"] = None
|
|
218
|
-
|
|
219
186
|
return _dict
|
|
220
187
|
|
|
221
188
|
@classmethod
|
|
@@ -241,7 +208,6 @@ class ValidityPeriod(BaseModel):
|
|
|
241
208
|
"contains_special_categories_of_personal_data": obj.get(
|
|
242
209
|
"contains_special_categories_of_personal_data"
|
|
243
210
|
),
|
|
244
|
-
"variable_status": obj.get("variable_status"),
|
|
245
211
|
"measurement_type": obj.get("measurement_type"),
|
|
246
212
|
"valid_from": obj.get("valid_from"),
|
|
247
213
|
"external_reference_uri": obj.get("external_reference_uri"),
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""Internal Variable Definitions Administration API
|
|
2
|
+
|
|
3
|
+
## Introduction Variable Definitions are centralized definitions of concrete variables which are typically present in multiple datasets. Variable Definitions support standardization of data and metadata and facilitate sharing and joining of data by clarifying when variables have an identical definition. ## Maintenance of Variable Definitions This API allows for creation, maintenance and access of Variable Definitions. ### Ownership Creation and maintenance of variables may only be performed by Statistics Norway employees representing a specific Dapla team, who are defined as the owners of a given Variable Definition. The team an owner represents must be specified when making a request through the `active_group` query parameter. All maintenance is to be performed by the owners, with no intervention from administrators. ### Status All Variable Definitions have an associated status. The possible values for status are `DRAFT`, `PUBLISHED_INTERNAL` and `PUBLISHED_EXTERNAL`. #### Draft When a Variable Definition is created it is assigned the status `DRAFT`. Under this status the Variable Definition is: - Only visible to Statistics Norway employees. - Mutable (it may be changed directly without need for versioning). - Not suitable to refer to from other systems. This status may be changed to `PUBLISHED_INTERNAL` or `PUBLISHED_EXTERNAL` with a direct update. #### Published Internal Under this status the Variable Definition is: - Only visible to Statistics Norway employees. - Immutable (all changes are versioned). - Suitable to refer to in internal systems for statistics production. - Not suitable to refer to for external use (for example in Statistikkbanken). This status may be changed to `PUBLISHED_EXTERNAL` by creating a Patch version. #### Published External Under this status the Variable Definition is: - Visible to the general public. - Immutable (all changes are versioned). - Suitable to refer to from any system. This status may not be changed as it would break immutability. If a Variable Definition is no longer relevant then its period of validity should be ended by specifying a `valid_until` date in a Patch version. ### Immutability Variable Definitions are immutable. This means that any changes must be performed in a strict versioning system. Consumers can avoid being exposed to breaking changes by specifying a `date_of_validity` when they request a Variable Definition. #### Patches Patches are for changes which do not affect the fundamental meaning of the Variable Definition. #### Validity Periods Validity Periods are versions with a period defined by a `valid_from` date and optionally a `valid_until` date. If the fundamental meaning of a Variable Definition is to be changed, it should be done by creating a new Validity Period.
|
|
4
|
+
|
|
5
|
+
The version of the OpenAPI document: 0.1
|
|
6
|
+
Contact: metadata@ssb.no
|
|
7
|
+
Generated by OpenAPI Generator (https://openapi-generator.tech)
|
|
8
|
+
|
|
9
|
+
Do not edit the class manually.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
from enum import Enum
|
|
16
|
+
|
|
17
|
+
from typing_extensions import Self
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class VariableStatus(str, Enum):
|
|
21
|
+
"""Life cycle status of a variable definition."""
|
|
22
|
+
|
|
23
|
+
"""
|
|
24
|
+
allowed enum values
|
|
25
|
+
"""
|
|
26
|
+
DRAFT = "DRAFT"
|
|
27
|
+
PUBLISHED_INTERNAL = "PUBLISHED_INTERNAL"
|
|
28
|
+
PUBLISHED_EXTERNAL = "PUBLISHED_EXTERNAL"
|
|
29
|
+
|
|
30
|
+
@classmethod
|
|
31
|
+
def from_json(cls, json_str: str) -> Self:
|
|
32
|
+
"""Create an instance of VariableStatus from a JSON string"""
|
|
33
|
+
return cls(json.loads(json_str))
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Util methods for variable definitions."""
|
|
@@ -1,10 +1,12 @@
|
|
|
1
|
-
from dapla_metadata.variable_definitions.
|
|
2
|
-
from dapla_metadata.variable_definitions.generated.vardef_client.api_client import (
|
|
1
|
+
from dapla_metadata.variable_definitions._generated.vardef_client.api_client import (
|
|
3
2
|
ApiClient,
|
|
4
3
|
)
|
|
5
|
-
from dapla_metadata.variable_definitions.
|
|
4
|
+
from dapla_metadata.variable_definitions._generated.vardef_client.configuration import (
|
|
6
5
|
Configuration,
|
|
7
6
|
)
|
|
7
|
+
from dapla_metadata.variable_definitions._utils.config import (
|
|
8
|
+
get_vardef_client_configuration,
|
|
9
|
+
)
|
|
8
10
|
|
|
9
11
|
|
|
10
12
|
class VardefClient:
|
|
@@ -3,11 +3,35 @@ from dapla_metadata._shared.config import get_dapla_environment
|
|
|
3
3
|
from dapla_metadata._shared.config import get_dapla_group_context
|
|
4
4
|
from dapla_metadata._shared.config import get_oidc_token
|
|
5
5
|
from dapla_metadata._shared.enums import DaplaEnvironment
|
|
6
|
-
from dapla_metadata.variable_definitions.
|
|
6
|
+
from dapla_metadata.variable_definitions._generated.vardef_client.configuration import (
|
|
7
7
|
Configuration,
|
|
8
8
|
)
|
|
9
9
|
|
|
10
10
|
VARDEF_HOST_TEST = "https://metadata.intern.test.ssb.no"
|
|
11
|
+
WORKSPACE_DIR = "WORKSPACE_DIR"
|
|
12
|
+
VARDEF_DESCRIPTIONS_FILE_PATH = "VARDEF_DESCRIPTIONS_FILE_PATH"
|
|
13
|
+
VARDEF_DEFAULT_DESCRIPTION_PATH = (
|
|
14
|
+
"variable_definitions/resources/vardef_model_descriptions_nb.yaml"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get_descriptions_path() -> str:
|
|
19
|
+
"""Get the relative file path from the repo root to the Norwegian descriptions.
|
|
20
|
+
|
|
21
|
+
First checks the `VARDEF_DESCRIPTIONS_FILE_PATH` environment variable; if not set, returns a default path.
|
|
22
|
+
|
|
23
|
+
Returns:
|
|
24
|
+
str: The file path to the descriptions.
|
|
25
|
+
"""
|
|
26
|
+
return (
|
|
27
|
+
get_config_item(VARDEF_DESCRIPTIONS_FILE_PATH)
|
|
28
|
+
or VARDEF_DEFAULT_DESCRIPTION_PATH
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def get_workspace_dir() -> str | None:
|
|
33
|
+
"""Get the path to work directory from workspace environment variable."""
|
|
34
|
+
return get_config_item(WORKSPACE_DIR)
|
|
11
35
|
|
|
12
36
|
|
|
13
37
|
def get_active_group() -> str:
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Constants for variable definitions."""
|
|
2
|
+
|
|
3
|
+
from datetime import date
|
|
4
|
+
|
|
5
|
+
VARIABLE_DEFINITIONS_DIR = "variable_definitions"
|
|
6
|
+
|
|
7
|
+
VARIABLE_STATUS_FIELD_NAME = "variable_status"
|
|
8
|
+
OWNER_FIELD_NAME = "owner"
|
|
9
|
+
|
|
10
|
+
TEMPLATE_HEADER = "--- Variabeldefinisjon mal ---\n"
|
|
11
|
+
HEADER = "--- Variabeldefinisjon ---\n"
|
|
12
|
+
|
|
13
|
+
TEMPLATE_SECTION_HEADER_STATUS = "\n--- Statusfelt. Verdi 'DRAFT' før publisering. Ikke rediger hvis du oppretter en ny variabeldefinisjon. ---\n"
|
|
14
|
+
TEMPLATE_SECTION_HEADER_OWNER = "\n--- Eierteam og grupper. Ikke rediger hvis du oppretter en ny variabeldefinisjon, verdien genereres ---\n"
|
|
15
|
+
TEMPLATE_SECTION_HEADER_MACHINE_GENERATED = (
|
|
16
|
+
"\n--- Maskin-genererte felt. Ikke rediger. ---\n"
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
TEMPLATE_HEADER_EN = "--- Variable definition template ---\n"
|
|
20
|
+
HEADER_EN = "--- Variable definition ---\n"
|
|
21
|
+
TEMPLATE_SECTION_HEADER_STATUS_EN = "\n--- Status field. Value 'DRAFT' before publishing. Do not edit if creating new variable defintion ---\n"
|
|
22
|
+
TEMPLATE_SECTION_HEADER_OWNER_EN = "\n--- Owner team and groups. Do not edit if creating new variable defintion, value is generated ---\n"
|
|
23
|
+
TEMPLATE_SECTION_HEADER_MACHINE_GENERATED_EN = (
|
|
24
|
+
"\n--- Machine generated fields. Do not edit ---\n"
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
NORWEGIAN_DESCRIPTIONS = "norwegian_description"
|
|
28
|
+
|
|
29
|
+
DEFAULT_DATE = date(1000, 1, 1)
|
|
30
|
+
|
|
31
|
+
MACHINE_GENERATED_FIELDS = [
|
|
32
|
+
"id",
|
|
33
|
+
"patch_id",
|
|
34
|
+
"created_at",
|
|
35
|
+
"created_by",
|
|
36
|
+
"last_updated_at",
|
|
37
|
+
"last_updated_by",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
OPTIONAL_FIELD = "~ Valgfritt felt ~"
|
|
41
|
+
REQUIRED_FIELD = "! Obligatorisk felt !"
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Utilities for dynamically adding extra fields to Pydantic models, specifically Norwegian descriptions."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
from typing import cast
|
|
7
|
+
|
|
8
|
+
import ruamel.yaml
|
|
9
|
+
from pydantic import BaseModel
|
|
10
|
+
from pydantic import Field
|
|
11
|
+
|
|
12
|
+
from dapla_metadata.variable_definitions._utils.config import get_descriptions_path
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from pydantic.config import JsonDict
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def get_package_root() -> Path:
|
|
21
|
+
"""Get an absolute Path to the root of the package (dapla_metadata)."""
|
|
22
|
+
number_of_directories_up_from_descriptions_file = 2
|
|
23
|
+
return (
|
|
24
|
+
Path(__file__)
|
|
25
|
+
.resolve()
|
|
26
|
+
.parents[number_of_directories_up_from_descriptions_file]
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def load_descriptions(file_path: Path) -> dict:
|
|
31
|
+
"""Load and return the contents of a YAML file as a dictionary.
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
file_path (Path): Path to the YAML file.
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
dict: Parsed contents of the YAML file.
|
|
38
|
+
"""
|
|
39
|
+
with Path.open(file_path, encoding="utf-8") as f:
|
|
40
|
+
return ruamel.yaml.YAML().load(f)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def apply_norwegian_descriptions_to_model(
|
|
44
|
+
model: type[BaseModel],
|
|
45
|
+
) -> None:
|
|
46
|
+
"""Add Norwegian descriptions to the fields of a Pydantic model.
|
|
47
|
+
|
|
48
|
+
This function globally modifies the model fields by inserting a Norwegian description
|
|
49
|
+
from a predefined dictionary. If a field does not have a corresponding
|
|
50
|
+
Norwegian description, a default message is used.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
model (BaseModel): A Pydantic model instance to be updated.
|
|
54
|
+
|
|
55
|
+
Returns:
|
|
56
|
+
None: The function modifies the model in place.
|
|
57
|
+
"""
|
|
58
|
+
new_fields = {}
|
|
59
|
+
|
|
60
|
+
descriptions = load_descriptions(
|
|
61
|
+
get_package_root() / get_descriptions_path(),
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
for field_name, field_info in model.model_fields.items():
|
|
65
|
+
new_description: str = descriptions.get(
|
|
66
|
+
field_name,
|
|
67
|
+
f"No description in norwegian found for {field_name}",
|
|
68
|
+
)
|
|
69
|
+
if "No description in norwegian found" in new_description:
|
|
70
|
+
logger.warning("Missing description for %s", field_name)
|
|
71
|
+
else:
|
|
72
|
+
logger.debug("Field %s: %s", field_name, new_description)
|
|
73
|
+
|
|
74
|
+
new_fields[field_name] = Field( # type: ignore[call-overload]
|
|
75
|
+
default=field_info.default,
|
|
76
|
+
alias=field_info.alias,
|
|
77
|
+
title=field_info.title,
|
|
78
|
+
description=field_info.description,
|
|
79
|
+
json_schema_extra=cast(
|
|
80
|
+
"JsonDict",
|
|
81
|
+
{
|
|
82
|
+
"norwegian_description": new_description,
|
|
83
|
+
"annotation": field_info.annotation,
|
|
84
|
+
},
|
|
85
|
+
),
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
model.model_fields.update(new_fields) # Apply changes
|
|
89
|
+
model.model_rebuild()
|