ssb-klass-python 1.0.6__tar.gz → 1.0.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/PKG-INFO +2 -1
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/pyproject.toml +9 -3
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/classes/classification.py +1 -25
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/classes/codes.py +12 -19
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/classes/correspondence.py +8 -28
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/classes/family.py +0 -7
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/classes/search.py +2 -11
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/classes/variant.py +6 -47
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/classes/version.py +2 -18
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/requests/klass_requests.py +2 -2
- ssb_klass_python-1.0.7/src/klass/utility/filters.py +82 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/widgets/search_ipywidget.py +3 -3
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/LICENSE +0 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/README.md +0 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/__init__.py +0 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/classes/__init__.py +0 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/config.py +0 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/py.typed +0 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/requests/__init__.py +0 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/requests/klass_types.py +0 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/requests/sections.py +0 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/requests/validate.py +0 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/utility/classification.py +0 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/utility/cleanup.py +0 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/utility/codes.py +0 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/utility/naming.py +0 -0
- {ssb_klass_python-1.0.6 → ssb_klass_python-1.0.7}/src/klass/widgets/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ssb-klass-python
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.7
|
|
4
4
|
Summary: A Python package built on top of KLASS's API for retrieving classifications, codes, correspondences etc.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.14
|
|
|
17
17
|
Requires-Dist: ipython (<9)
|
|
18
18
|
Requires-Dist: ipywidgets (>=8.0.6)
|
|
19
19
|
Requires-Dist: pandas (>=1.5.3)
|
|
20
|
+
Requires-Dist: pyarrow (>=10.0.1)
|
|
20
21
|
Requires-Dist: python-dateutil (>=2.8.2)
|
|
21
22
|
Requires-Dist: requests (>=2.31.0)
|
|
22
23
|
Requires-Dist: toml (>=0.10.2)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ssb-klass-python"
|
|
3
|
-
version = "1.0.
|
|
3
|
+
version = "1.0.7"
|
|
4
4
|
description = "A Python package built on top of KLASS's API for retrieving classifications, codes, correspondences etc."
|
|
5
5
|
authors = [{name="Carl Corneil", email="ssb-pythonistas <ssb-pythonistas@ssb.no>"}, {name = "Statistics Norway" }]
|
|
6
6
|
license = "MIT"
|
|
@@ -12,10 +12,10 @@ dependencies = [
|
|
|
12
12
|
"toml >=0.10.2",
|
|
13
13
|
"ipywidgets >=8.0.6",
|
|
14
14
|
"pandas >=1.5.3",
|
|
15
|
+
"pyarrow >=10.0.1",
|
|
15
16
|
"requests >=2.31.0",
|
|
16
17
|
"ipython <9",
|
|
17
18
|
"typing-extensions >=4.12.2",
|
|
18
|
-
|
|
19
19
|
]
|
|
20
20
|
|
|
21
21
|
[project.urls]
|
|
@@ -38,7 +38,7 @@ Changelog = "https://github.com/statisticsnorway/ssb-klass-python/releases"
|
|
|
38
38
|
pygments = ">=2.10.0"
|
|
39
39
|
black = { extras = ["jupyter"], version = ">=24.3.0" }
|
|
40
40
|
coverage = { extras = ["toml"], version = ">=6.2" }
|
|
41
|
-
|
|
41
|
+
pydoclint = ">=0.8.3"
|
|
42
42
|
furo = ">=2021.11.12"
|
|
43
43
|
mypy = ">=0.930"
|
|
44
44
|
ipykernel = ">=6.15.3"
|
|
@@ -127,6 +127,7 @@ ignore = [
|
|
|
127
127
|
"ANN202", # Don't requiere return type annotation for private functions.
|
|
128
128
|
"ANN401", # Allow type annotation with type Any.
|
|
129
129
|
"D100", # Supress undocumented-public-module. Only doc of public api required.
|
|
130
|
+
"D107", # We are of the opinion that inits dont need a docstring, when the class has one...
|
|
130
131
|
"E402", # Supress module-import-not-at-top-of-file, needed in jupyter notebooks.
|
|
131
132
|
"E501", # Supress line-too-long warnings: trust black's judgement on this one.
|
|
132
133
|
]
|
|
@@ -167,6 +168,11 @@ classmethod-decorators = [
|
|
|
167
168
|
]
|
|
168
169
|
"demo/*.py" = ["B018"] # Ignore the useless expressions in the notebooks
|
|
169
170
|
|
|
171
|
+
[tool.pydoclint]
|
|
172
|
+
style = "google"
|
|
173
|
+
arg-type-hints-in-docstring = false
|
|
174
|
+
native-mode-noqa-location = "definition"
|
|
175
|
+
|
|
170
176
|
[build-system]
|
|
171
177
|
requires = ["poetry-core>=1.0.0"]
|
|
172
178
|
build-backend = "poetry.core.masonry.api"
|
|
@@ -33,33 +33,10 @@ class KlassClassification:
|
|
|
33
33
|
you first need to get the classification at a specific time (a KlassVersion)
|
|
34
34
|
by using get_version() for example.
|
|
35
35
|
|
|
36
|
-
Attributes:
|
|
37
|
-
versions (list): A list of the data the Classifications has on its versions.
|
|
38
|
-
Versions represent the changes to the classifications codelists placed in time.
|
|
39
|
-
name (str): The name of the classification.
|
|
40
|
-
classification_id (str): The ID of the classification.
|
|
41
|
-
classificationType (str): The type of the classification.
|
|
42
|
-
lastModified (str): The last time the classification was modified. ISO-stringified datetime(ISO-datetime)
|
|
43
|
-
description (str): A longer description of the classification.
|
|
44
|
-
primaryLanguage (str): The primary language of the classification. "nb", "nn" or "en".
|
|
45
|
-
language (str): The language chosen at initialization of the classification. "nb", "nn" or "en".
|
|
46
|
-
copyrighted (bool): Whether the classification is copyrighted.
|
|
47
|
-
includeShortName (bool): If true, indicates that classificationItems may have shortnames.
|
|
48
|
-
includeNotes (bool): If true, indicates that classificationItems may have notes.
|
|
49
|
-
contactPerson (dict): A dictionary containing the contact person of the classification.
|
|
50
|
-
owningSection (str): The section (part of Statistics Norway)that owns the classification.
|
|
51
|
-
statisticalUnits (list): Statistical units assigned to classification
|
|
52
|
-
include_future (bool): Whether to include future versions of the classification.
|
|
53
|
-
_links (dict): A dictionary containing the links to different possible endpoints using the classification.
|
|
54
|
-
|
|
55
36
|
Args:
|
|
56
37
|
classification_id: The classification_id of the classification. For example: '36'
|
|
57
38
|
language: The language of the classification. "nb", "nn" or "en".
|
|
58
39
|
include_future: Whether to include future versions of the classification.
|
|
59
|
-
|
|
60
|
-
Raises:
|
|
61
|
-
ValueError: If the language is not "no", "nb" or "en".
|
|
62
|
-
If the include_future is not a bool.
|
|
63
40
|
"""
|
|
64
41
|
|
|
65
42
|
def __init__(
|
|
@@ -68,7 +45,6 @@ class KlassClassification:
|
|
|
68
45
|
language: Language = "nb",
|
|
69
46
|
include_future: bool = False,
|
|
70
47
|
) -> None:
|
|
71
|
-
"""Get the data for the classification from the API."""
|
|
72
48
|
self.classification_id = classification_id
|
|
73
49
|
self.language: Language = language
|
|
74
50
|
self.include_future = include_future
|
|
@@ -163,7 +139,7 @@ class KlassClassification:
|
|
|
163
139
|
"""Reformats the versions into a simple dict with just the IDs as keys and names as values.
|
|
164
140
|
|
|
165
141
|
Returns:
|
|
166
|
-
dict: Version IDs as keys, and version names as values.
|
|
142
|
+
dict[int, str]: Version IDs as keys, and version names as values.
|
|
167
143
|
"""
|
|
168
144
|
return {v["version_id"]: v["name"] for v in self.versions[::-1]}
|
|
169
145
|
|
|
@@ -7,6 +7,8 @@ from typing_extensions import Self
|
|
|
7
7
|
from ..requests.klass_requests import codes
|
|
8
8
|
from ..requests.klass_requests import codes_at
|
|
9
9
|
from ..requests.klass_types import Language
|
|
10
|
+
from ..utility.filters import apply_presentation_name_fallback
|
|
11
|
+
from ..utility.filters import limit_na_level
|
|
10
12
|
|
|
11
13
|
|
|
12
14
|
class KlassCodes:
|
|
@@ -14,12 +16,6 @@ class KlassCodes:
|
|
|
14
16
|
|
|
15
17
|
The codelist is owned by the Classification through a Version, and will be valid for a time period.
|
|
16
18
|
|
|
17
|
-
Attributes:
|
|
18
|
-
data (pd.DataFrame): The pandas DataFrame of the codes.
|
|
19
|
-
classification_id (str): The classification ID.
|
|
20
|
-
from_date (str): The start date of the time period. "YYYY-MM-DD".
|
|
21
|
-
to_date (str): The end date of the time period. "YYYY-MM-DD".
|
|
22
|
-
|
|
23
19
|
Args:
|
|
24
20
|
classification_id: The classification ID.
|
|
25
21
|
from_date: The start date of the time period. "YYYY-MM-DD".
|
|
@@ -29,14 +25,6 @@ class KlassCodes:
|
|
|
29
25
|
presentation_name_pattern: A pattern for filtering the code names.
|
|
30
26
|
language: The language of the code names. Defaults to "nb".
|
|
31
27
|
include_future: Whether to include future codes. Defaults to False.
|
|
32
|
-
|
|
33
|
-
Raises:
|
|
34
|
-
ValueError: If from_date or to_date is not a valid date or date-string YYYY-MM-DD.
|
|
35
|
-
ValueError: If select_codes contains anything except numbers and the special characters "*" (star) or "-" (dash).
|
|
36
|
-
ValueError: If select_level is anything except a whole number.
|
|
37
|
-
ValueError: If presentation_name_pattern is not a valid pattern.
|
|
38
|
-
ValueError: If language is not "nb", "nn" or "en".
|
|
39
|
-
ValueError: If include_future is not a bool.
|
|
40
28
|
"""
|
|
41
29
|
|
|
42
30
|
def __init__(
|
|
@@ -50,7 +38,6 @@ class KlassCodes:
|
|
|
50
38
|
language: Language = "nb",
|
|
51
39
|
include_future: bool = False,
|
|
52
40
|
) -> None:
|
|
53
|
-
"""Get the data from the KLASS-api belonging to the code-list."""
|
|
54
41
|
self.classification_id = classification_id
|
|
55
42
|
if not from_date:
|
|
56
43
|
from_date = datetime.now().strftime("%Y-%m-%d")
|
|
@@ -110,7 +97,7 @@ class KlassCodes:
|
|
|
110
97
|
include_future: Whether to include future codes.
|
|
111
98
|
|
|
112
99
|
Returns:
|
|
113
|
-
|
|
100
|
+
Self: Returns self to make the method more easily chainable.
|
|
114
101
|
"""
|
|
115
102
|
if not from_date:
|
|
116
103
|
from_date = datetime.now().strftime("%Y-%m-%d")
|
|
@@ -132,7 +119,7 @@ class KlassCodes:
|
|
|
132
119
|
raise_on_empty_data: Whether to raise an error if the returned dataframe is empty. Defaults to True.
|
|
133
120
|
|
|
134
121
|
Returns:
|
|
135
|
-
|
|
122
|
+
Self: Returns self to make the method more easily chainable.
|
|
136
123
|
|
|
137
124
|
Raises:
|
|
138
125
|
ValueError: If the returned dataframe is empty, there is probably something too narrow in the parameters.
|
|
@@ -169,6 +156,8 @@ class KlassCodes:
|
|
|
169
156
|
key: str = "code",
|
|
170
157
|
value: str | None = None, # default is "name" if not set
|
|
171
158
|
other: str | None = None,
|
|
159
|
+
remove_na: bool = True,
|
|
160
|
+
select_level: int | None = None,
|
|
172
161
|
) -> dict[str, str] | defaultdict[str, str]:
|
|
173
162
|
"""Extract two columns from the data, turning them into a dict.
|
|
174
163
|
|
|
@@ -178,9 +167,11 @@ class KlassCodes:
|
|
|
178
167
|
key: The name of the column with the values you want as keys.
|
|
179
168
|
value: The name of the column with the values you want as values in your dict.
|
|
180
169
|
other: If key is missing from dict, return this value instead, if you specify an OTHER-value.
|
|
170
|
+
remove_na: Set to False if you want to keep empty mappings over the key and value columns. Empty is defined as empty strings or NA-types.
|
|
171
|
+
select_level: Keep only a specific level defines the variants codes / groups.
|
|
181
172
|
|
|
182
173
|
Returns:
|
|
183
|
-
dict | defaultdict: The extracted columns as a dict or defaultdict.
|
|
174
|
+
dict[str, str] | defaultdict[str, str]: The extracted columns as a dict or defaultdict.
|
|
184
175
|
"""
|
|
185
176
|
if not value:
|
|
186
177
|
# If you bothered specifying a pattern, we assume you want it
|
|
@@ -188,7 +179,9 @@ class KlassCodes:
|
|
|
188
179
|
value = "presentationName"
|
|
189
180
|
else:
|
|
190
181
|
value = "name"
|
|
191
|
-
|
|
182
|
+
data, value_col = apply_presentation_name_fallback(self.data, value)
|
|
183
|
+
limit_data = limit_na_level(data, key, value_col, remove_na, select_level)
|
|
184
|
+
mapping = dict(zip(limit_data[key], limit_data[value_col], strict=False))
|
|
192
185
|
if other:
|
|
193
186
|
mapping = defaultdict(lambda: other, mapping)
|
|
194
187
|
return mapping
|
|
@@ -12,6 +12,8 @@ from ..requests.klass_requests import corresponds
|
|
|
12
12
|
from ..requests.klass_types import CorrespondsType
|
|
13
13
|
from ..requests.klass_types import Language
|
|
14
14
|
from ..requests.klass_types import T_correspondanceMaps
|
|
15
|
+
from ..utility.filters import apply_presentation_name_fallback
|
|
16
|
+
from ..utility.filters import limit_na_level
|
|
15
17
|
|
|
16
18
|
|
|
17
19
|
class KlassCorrespondence:
|
|
@@ -23,19 +25,6 @@ class KlassCorrespondence:
|
|
|
23
25
|
You can identify the correspondence by their individual ids,
|
|
24
26
|
or by the source classification ID + the target classification ID + a specific time.
|
|
25
27
|
|
|
26
|
-
Attributes:
|
|
27
|
-
data (pd.DataFrame): The pandas DataFrame of the correspondences.
|
|
28
|
-
correspondence (list): The list of the correspondences returned by the API.
|
|
29
|
-
correspondence_id (str): The ID of the correspondence.
|
|
30
|
-
source_classification_id (str): The ID of the source classification.
|
|
31
|
-
target_classification_id (str): The ID of the target classification.
|
|
32
|
-
from_date (str): The start date of the correspondence.
|
|
33
|
-
to_date (str, optional): The end date of the correspondence.
|
|
34
|
-
contain_quarter (int): The number of quarters the correspondence should contain,
|
|
35
|
-
this replaces the to_date during initialization.
|
|
36
|
-
language (str): The language of the correspondence. "nb", "nn" or "en".
|
|
37
|
-
include_future (bool): If the correspondence should include future correspondences.
|
|
38
|
-
|
|
39
28
|
Args:
|
|
40
29
|
correspondence_id: The ID of the correspondence.
|
|
41
30
|
source_classification_id: The ID of the source classification.
|
|
@@ -85,7 +74,6 @@ class KlassCorrespondence:
|
|
|
85
74
|
language: Language = "nb",
|
|
86
75
|
include_future: bool = False,
|
|
87
76
|
) -> None:
|
|
88
|
-
"""Get the correspondence-data from the API."""
|
|
89
77
|
self.correspondence_id = correspondence_id
|
|
90
78
|
self.source_classification_id = source_classification_id
|
|
91
79
|
self.target_classification_id = target_classification_id
|
|
@@ -136,7 +124,7 @@ class KlassCorrespondence:
|
|
|
136
124
|
Gets and reshapes correspondences based on attributes on the class.
|
|
137
125
|
|
|
138
126
|
Returns:
|
|
139
|
-
|
|
127
|
+
Self: Returns self to make the method more easily chainable.
|
|
140
128
|
|
|
141
129
|
Raises:
|
|
142
130
|
ValueError: If you are filling out the wrong combination of correspondence_id, source_classification_id,
|
|
@@ -194,7 +182,7 @@ class KlassCorrespondence:
|
|
|
194
182
|
Uses the attribute "contain_quarter" to determine which quarter to use.
|
|
195
183
|
|
|
196
184
|
Returns:
|
|
197
|
-
The last date of the quarter.
|
|
185
|
+
str: The last date of the quarter.
|
|
198
186
|
|
|
199
187
|
Raises:
|
|
200
188
|
ValueError: if from date is missing
|
|
@@ -234,19 +222,11 @@ class KlassCorrespondence:
|
|
|
234
222
|
select_level: Keep only a specific level defines the variants codes / groups.
|
|
235
223
|
|
|
236
224
|
Returns:
|
|
237
|
-
dict | defaultdict: The dictionary of the correspondence.
|
|
225
|
+
dict[str, str | None] | defaultdict[str, str | None]: The dictionary of the correspondence.
|
|
238
226
|
"""
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
(limit_data[[key, value]].notna().all(axis=1))
|
|
243
|
-
| (limit_data[[key, value]].astype("string") == "")
|
|
244
|
-
]
|
|
245
|
-
if select_level:
|
|
246
|
-
limit_data = limit_data[
|
|
247
|
-
limit_data["level"].astype("string[pyarrow]") == str(select_level)
|
|
248
|
-
]
|
|
249
|
-
mapping = dict(zip(limit_data[key], limit_data[value], strict=False))
|
|
227
|
+
data, value_col = apply_presentation_name_fallback(self.data, value)
|
|
228
|
+
limit_data = limit_na_level(data, key, value_col, remove_na, select_level)
|
|
229
|
+
mapping = dict(zip(limit_data[key], limit_data[value_col], strict=False))
|
|
250
230
|
if other:
|
|
251
231
|
mapping = defaultdict(lambda: other, mapping)
|
|
252
232
|
return mapping
|
|
@@ -10,18 +10,11 @@ class KlassFamily:
|
|
|
10
10
|
Families in Klass "own" / "have" several classifications.
|
|
11
11
|
Families are owned by sections (a part of Statistics Norway who is responsible for the family).
|
|
12
12
|
|
|
13
|
-
Attributes:
|
|
14
|
-
classifications (list): A list of classifications in the family.
|
|
15
|
-
family_id (str): The ID of the family.
|
|
16
|
-
name (str): The name of the family.
|
|
17
|
-
_links (dict): A dictionary of API links referencing itself.
|
|
18
|
-
|
|
19
13
|
Args:
|
|
20
14
|
family_id: The ID of the family.
|
|
21
15
|
"""
|
|
22
16
|
|
|
23
17
|
def __init__(self, family_id: str | int) -> None:
|
|
24
|
-
"""Get the family data from the klass-api, setting it as attributes on the object."""
|
|
25
18
|
self.family_id = family_id
|
|
26
19
|
# Setting for mypy
|
|
27
20
|
result: ClassificationFamiliesByIdType = classificationfamilies_by_id(
|
|
@@ -12,13 +12,6 @@ from .family import KlassFamily
|
|
|
12
12
|
class KlassSearchClassifications:
|
|
13
13
|
"""Use to search for classifications.
|
|
14
14
|
|
|
15
|
-
Attributes:
|
|
16
|
-
classifications (list): A list of KlassClassification objects.
|
|
17
|
-
query (str): The search query.
|
|
18
|
-
include_codelists (bool): Whether to include codelists in the search results.
|
|
19
|
-
ssbsection (str): The SSB section who owns the classification you are searching for.
|
|
20
|
-
no_dupes (bool): Whether to remove duplicates from the search results.
|
|
21
|
-
|
|
22
15
|
Args:
|
|
23
16
|
query: The search query.
|
|
24
17
|
include_codelists: Whether to include codelists in the search results.
|
|
@@ -34,7 +27,6 @@ class KlassSearchClassifications:
|
|
|
34
27
|
ssbsection: str = "",
|
|
35
28
|
no_dupes: bool = False,
|
|
36
29
|
) -> None:
|
|
37
|
-
"""Get data from the KLASS-api, setting it as attributes on this object."""
|
|
38
30
|
self.query = query
|
|
39
31
|
self.include_codelists = include_codelists
|
|
40
32
|
self.ssbsection = ssbsection
|
|
@@ -79,7 +71,7 @@ class KlassSearchClassifications:
|
|
|
79
71
|
no_dupes: Set to True if you want equal results to be filtered out.
|
|
80
72
|
|
|
81
73
|
Returns:
|
|
82
|
-
list: The cleaned classifications.
|
|
74
|
+
list[ClassificationSearchResultsPartType]: The cleaned classifications.
|
|
83
75
|
"""
|
|
84
76
|
classification_replace: list[ClassificationSearchResultsPartType] = []
|
|
85
77
|
seen = []
|
|
@@ -172,7 +164,6 @@ class KlassSearchFamilies:
|
|
|
172
164
|
include_codelists: bool = False,
|
|
173
165
|
language: Language = "nb",
|
|
174
166
|
) -> None:
|
|
175
|
-
"""Get data from the KLASS-api, setting it as attributes on this object."""
|
|
176
167
|
self.ssbsection = ssbsection
|
|
177
168
|
self.include_codelists = include_codelists
|
|
178
169
|
self.language = language
|
|
@@ -205,7 +196,7 @@ class KlassSearchFamilies:
|
|
|
205
196
|
attributes.
|
|
206
197
|
|
|
207
198
|
Returns:
|
|
208
|
-
|
|
199
|
+
Self: Returns self to make the method more easily chainable.
|
|
209
200
|
"""
|
|
210
201
|
result = classificationfamilies(
|
|
211
202
|
ssbsection=self.ssbsection,
|
|
@@ -8,6 +8,8 @@ from ..requests.klass_requests import variants_by_id
|
|
|
8
8
|
from ..requests.klass_types import CorrespondenceTablesType
|
|
9
9
|
from ..requests.klass_types import Language
|
|
10
10
|
from ..requests.klass_types import VariantsByIdType
|
|
11
|
+
from ..utility.filters import apply_presentation_name_fallback
|
|
12
|
+
from ..utility.filters import limit_na_level
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
class KlassVariant:
|
|
@@ -22,25 +24,6 @@ class KlassVariant:
|
|
|
22
24
|
It sets a new upper level of codes (amount of study points),
|
|
23
25
|
for a set of lower-level existing codes (NUS codes, level 5).
|
|
24
26
|
|
|
25
|
-
Attributes:
|
|
26
|
-
data (pd.DataFrame): The classificationItems as a pandas dataframe. Usually what you're after?
|
|
27
|
-
variant_id (str): The variant_id of the variant. For example: '36'.
|
|
28
|
-
name (str): The name of the variant.
|
|
29
|
-
contactPerson (dict): The contact person of the variant.
|
|
30
|
-
owningSection (str): The owning section of the variant.
|
|
31
|
-
lastModified (str): Stringified iso-datetime for last modification.
|
|
32
|
-
published (list[str]): Languages that the variant is published in.
|
|
33
|
-
validFrom (str): Date-string from when the version is valid.
|
|
34
|
-
validTo (str, optional): Date-string to when the version is valid.
|
|
35
|
-
introduction (str): A longer description of the variant.
|
|
36
|
-
correspondenceTables (list): The correspondence tables of the variant.
|
|
37
|
-
changelogs (list): The changelogs of the variant.
|
|
38
|
-
levels (list[dict]): The levels contained in the codelist (items).
|
|
39
|
-
classificationItems (list[dict]): The codelist-elements of the variant.
|
|
40
|
-
select_level (int): The level of the dataset to keep. For example: 0.
|
|
41
|
-
language (str): The language of the variant to select. For example: 'nb'.
|
|
42
|
-
_links (dict): The links returned from the API.
|
|
43
|
-
|
|
44
27
|
Args:
|
|
45
28
|
variant_id: The variant_id of the variant. For example: '1959'.
|
|
46
29
|
select_level: The level of the dataset to keep. For example: 5.
|
|
@@ -53,7 +36,6 @@ class KlassVariant:
|
|
|
53
36
|
select_level: int | None = None,
|
|
54
37
|
language: Language = "nb",
|
|
55
38
|
) -> None:
|
|
56
|
-
"""Get the data from the KLASS-api to populate this objects attributes."""
|
|
57
39
|
self.variant_id = variant_id
|
|
58
40
|
self.select_level = select_level
|
|
59
41
|
self.language: Language = language
|
|
@@ -139,19 +121,11 @@ class KlassVariant:
|
|
|
139
121
|
select_level: Usually you want level 2, not level 1, as level 1 just defines the variants codes / groups.
|
|
140
122
|
|
|
141
123
|
Returns:
|
|
142
|
-
dict | defaultdict: The extracted columns as a dict or defaultdict.
|
|
124
|
+
dict[str, str] | defaultdict[str, str]: The extracted columns as a dict or defaultdict.
|
|
143
125
|
"""
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
(limit_data[[key, value]].notna().all(axis=1))
|
|
148
|
-
| (limit_data[[key, value]].astype("string") == "")
|
|
149
|
-
]
|
|
150
|
-
if select_level:
|
|
151
|
-
limit_data = limit_data[
|
|
152
|
-
limit_data["level"].astype("string[pyarrow]") == str(select_level)
|
|
153
|
-
]
|
|
154
|
-
mapping = dict(zip(limit_data[key], limit_data[value], strict=False))
|
|
126
|
+
data, value_col = apply_presentation_name_fallback(self.data, value)
|
|
127
|
+
limit_data = limit_na_level(data, key, value_col, remove_na, select_level)
|
|
128
|
+
mapping = dict(zip(limit_data[key], limit_data[value_col], strict=False))
|
|
155
129
|
if other:
|
|
156
130
|
mapping = defaultdict(lambda: other, mapping)
|
|
157
131
|
return mapping
|
|
@@ -170,20 +144,6 @@ class KlassVariantSearchByName(KlassVariant):
|
|
|
170
144
|
It does not have to be extensive (all filled out), but can, for example,
|
|
171
145
|
redefine upper levels, for some lower-level codes.
|
|
172
146
|
|
|
173
|
-
Attributes:
|
|
174
|
-
data (pd.DataFrame): The codelists from the Variant as a pandas dataframe. Usually what you're after?
|
|
175
|
-
classification_id (str): The classification ID.
|
|
176
|
-
variant_name (str): The start of the variant name.
|
|
177
|
-
from_date (str): The start of the date range.
|
|
178
|
-
to_date (str): The end of the date range.
|
|
179
|
-
select_codes (str): Limit the result to codes matching this pattern.
|
|
180
|
-
See rules: https://data.ssb.no/api/klass/v1/api-guide.html#_selectcodes
|
|
181
|
-
select_level (str): The level of codes to keep in the dataset.
|
|
182
|
-
presentation_name_pattern (str): Used to build an alternative presentation name for the codes.
|
|
183
|
-
See rules: https://data.ssb.no/api/klass/v1/api-guide.html#_presentationnamepattern
|
|
184
|
-
language (str): Language of the names, select "en", "nb" or "nn".
|
|
185
|
-
include_future (bool): Whether to include future codes. Defaults to False.
|
|
186
|
-
|
|
187
147
|
Args:
|
|
188
148
|
classification_id: The classification ID.
|
|
189
149
|
variant_name: The start of the variant name.
|
|
@@ -210,7 +170,6 @@ class KlassVariantSearchByName(KlassVariant):
|
|
|
210
170
|
language: Language = "nb",
|
|
211
171
|
include_future: bool = False,
|
|
212
172
|
) -> None:
|
|
213
|
-
"""Get the data from the KLASS-api, setting it as attributes on the object."""
|
|
214
173
|
self.classification_id = classification_id
|
|
215
174
|
self.variant_name = variant_name
|
|
216
175
|
self.from_date = from_date
|
|
@@ -16,21 +16,6 @@ class KlassVersion:
|
|
|
16
16
|
|
|
17
17
|
For example, the ID of NUS valid in 2023 is 1954, while the ID of NUS without being time-specific is 36.
|
|
18
18
|
|
|
19
|
-
Attributes:
|
|
20
|
-
data (pd.DataFrame): The codelist of the classification-version as a pandas dataframe.
|
|
21
|
-
name (str): The name of the version.
|
|
22
|
-
validFrom (str): The date the version is valid from.
|
|
23
|
-
validTo (str): The date the version is valid to (if any).
|
|
24
|
-
lastModified (str): The date the version was last modified.
|
|
25
|
-
published (list): A list of languages that the version is published in.
|
|
26
|
-
introduction (str): A longer description of the version.
|
|
27
|
-
contactPerson (dict): A dictionary of the contact person of the version.
|
|
28
|
-
owningSection (str): The name of the section that owns the version.
|
|
29
|
-
legalBase (str): The basis in law for the classification.
|
|
30
|
-
publications (str): Where the classification is published (URL).
|
|
31
|
-
derivedFrom (str): Notes on where the classification was derived from.
|
|
32
|
-
correspondenceTables (list): A list of correspondence-tables of the version.
|
|
33
|
-
|
|
34
19
|
Args:
|
|
35
20
|
version_id: The ID of the version.
|
|
36
21
|
select_level: The level in the codelist-data to keep. Defaults to 0 (keep all).
|
|
@@ -45,7 +30,6 @@ class KlassVersion:
|
|
|
45
30
|
language: Language = "nb",
|
|
46
31
|
include_future: bool = False,
|
|
47
32
|
) -> None:
|
|
48
|
-
"""Set up the object with data from the KLASS-API."""
|
|
49
33
|
self.version_id = version_id
|
|
50
34
|
self.select_level = select_level
|
|
51
35
|
self.language = language.lower()
|
|
@@ -139,7 +123,7 @@ class KlassVersion:
|
|
|
139
123
|
select_level: The level of the version to keep in the data. Setting to 0 keeps all levels.
|
|
140
124
|
|
|
141
125
|
Returns:
|
|
142
|
-
|
|
126
|
+
Self: Returns self to make the method more easily chainable.
|
|
143
127
|
"""
|
|
144
128
|
select_level = select_level if select_level else self.select_level
|
|
145
129
|
data = pd.json_normalize(self.classificationItems)
|
|
@@ -272,7 +256,7 @@ class KlassVersion:
|
|
|
272
256
|
With the IDs as keys.
|
|
273
257
|
|
|
274
258
|
Returns:
|
|
275
|
-
dict: A nested dictionary of the available correspondences.
|
|
259
|
+
dict[str, dict[str, str]]: A nested dictionary of the available correspondences.
|
|
276
260
|
"""
|
|
277
261
|
tables = {}
|
|
278
262
|
for tab in self.correspondenceTables:
|
|
@@ -38,8 +38,8 @@ def get_json(url: str, params: ParamsAfterType) -> Any:
|
|
|
38
38
|
Used in most of the following functions.
|
|
39
39
|
|
|
40
40
|
Args:
|
|
41
|
-
url
|
|
42
|
-
params
|
|
41
|
+
url: The URL to the endpoint.
|
|
42
|
+
params: The parameters to send to the endpoint.
|
|
43
43
|
|
|
44
44
|
Returns:
|
|
45
45
|
Any: The JSON response from the endpoint, hard to type because all endpoints have differently structured responses.
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from typing import Final
|
|
2
|
+
from typing import Literal
|
|
3
|
+
from typing import cast
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
STRING_DTYPE: Final[Literal["string[pyarrow]"]] = "string[pyarrow]"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def limit_na_level(
|
|
11
|
+
df: pd.DataFrame,
|
|
12
|
+
key: str,
|
|
13
|
+
value: str,
|
|
14
|
+
remove_na: bool = True,
|
|
15
|
+
select_level: int | None = None,
|
|
16
|
+
) -> pd.DataFrame:
|
|
17
|
+
"""Filter a correspondence DataFrame by NA/empty values and optional level.
|
|
18
|
+
|
|
19
|
+
Creates a filtered copy of the input DataFrame where rows may be removed
|
|
20
|
+
based on missing or empty values in the specified key/value columns and/or
|
|
21
|
+
restricted to a specific classification level.
|
|
22
|
+
|
|
23
|
+
If ``remove_na`` is True, rows are kept only if both ``key`` and ``value``:
|
|
24
|
+
- are not NA
|
|
25
|
+
- are not empty strings (after string conversion)
|
|
26
|
+
|
|
27
|
+
If ``select_level`` is provided, only rows where the ``level`` column
|
|
28
|
+
matches the given value are retained.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
df: The input DataFrame containing correspondence data.
|
|
32
|
+
key: Column name used as dictionary keys in downstream mapping.
|
|
33
|
+
value: Column name used as dictionary values in downstream mapping.
|
|
34
|
+
remove_na: Whether to remove rows where key/value are NA or empty strings.
|
|
35
|
+
select_level: Optional classification level to filter on.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
pd.DataFrame: A filtered copy of the input DataFrame.
|
|
39
|
+
"""
|
|
40
|
+
limit_data = df.copy()
|
|
41
|
+
if remove_na:
|
|
42
|
+
print(key, value)
|
|
43
|
+
non_empty = (
|
|
44
|
+
limit_data[[key, value]].astype(STRING_DTYPE).fillna("") != ""
|
|
45
|
+
).all(axis=1)
|
|
46
|
+
mask = cast(pd.Series, limit_data[[key, value]].notna().all(axis=1) & non_empty)
|
|
47
|
+
limit_data = limit_data.loc[mask]
|
|
48
|
+
print(limit_data)
|
|
49
|
+
if select_level:
|
|
50
|
+
level_mask = cast(
|
|
51
|
+
pd.Series,
|
|
52
|
+
limit_data["level"].astype(STRING_DTYPE) == str(select_level),
|
|
53
|
+
)
|
|
54
|
+
limit_data = limit_data.loc[level_mask]
|
|
55
|
+
return limit_data
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def apply_presentation_name_fallback(
|
|
59
|
+
df: pd.DataFrame,
|
|
60
|
+
value: str,
|
|
61
|
+
fallback: str = "name",
|
|
62
|
+
fallback_col: str = "_value_fallback",
|
|
63
|
+
) -> tuple[pd.DataFrame, str]:
|
|
64
|
+
"""Create a fallback value column when presentation names are empty.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
df: Input DataFrame.
|
|
68
|
+
value: The value column name requested.
|
|
69
|
+
fallback: The fallback column name to use when presentation names are empty.
|
|
70
|
+
fallback_col: The name of the temporary column to store the fallback result.
|
|
71
|
+
|
|
72
|
+
Returns:
|
|
73
|
+
tuple[pd.DataFrame, str]: The DataFrame (possibly augmented) and the effective value column name.
|
|
74
|
+
"""
|
|
75
|
+
if value != "presentationName" or fallback not in df.columns:
|
|
76
|
+
return df, value
|
|
77
|
+
data = df.copy()
|
|
78
|
+
data[fallback_col] = data["presentationName"].astype(STRING_DTYPE).fillna("")
|
|
79
|
+
empty_mask = cast(pd.Series, data[fallback_col] == "")
|
|
80
|
+
if empty_mask.any():
|
|
81
|
+
data.loc[empty_mask, fallback_col] = data[fallback].astype(STRING_DTYPE)
|
|
82
|
+
return data, fallback_col
|
|
@@ -17,8 +17,8 @@ def search_classification(no_dupes: bool = True) -> widgets.VBox:
|
|
|
17
17
|
that'll let you get data from the classification.
|
|
18
18
|
|
|
19
19
|
Args:
|
|
20
|
-
no_dupes
|
|
21
|
-
|
|
20
|
+
no_dupes: To include duplicate results or not in the result.
|
|
21
|
+
Dupes are caused by multiple languages being returned.
|
|
22
22
|
|
|
23
23
|
Returns:
|
|
24
24
|
widgets.VBox: Containing the nested ipywidgets-GUI. Jupyter will automatically display it.
|
|
@@ -86,7 +86,7 @@ def format_classification_text(search_class: KlassSearchClassifications) -> str:
|
|
|
86
86
|
"""Format the classifications from a search into a html-string that can be used in the widget results.
|
|
87
87
|
|
|
88
88
|
Args:
|
|
89
|
-
search_class
|
|
89
|
+
search_class: The search-results.
|
|
90
90
|
|
|
91
91
|
Returns:
|
|
92
92
|
str: The formatted html-string.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|