dapla-toolbelt-metadata 0.6.0__py3-none-any.whl → 0.6.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dapla-toolbelt-metadata might be problematic. Click here for more details.
- dapla_metadata/datasets/dataset_parser.py +14 -7
- {dapla_toolbelt_metadata-0.6.0.dist-info → dapla_toolbelt_metadata-0.6.1.dist-info}/METADATA +1 -1
- {dapla_toolbelt_metadata-0.6.0.dist-info → dapla_toolbelt_metadata-0.6.1.dist-info}/RECORD +5 -5
- {dapla_toolbelt_metadata-0.6.0.dist-info → dapla_toolbelt_metadata-0.6.1.dist-info}/WHEEL +1 -1
- {dapla_toolbelt_metadata-0.6.0.dist-info → dapla_toolbelt_metadata-0.6.1.dist-info}/LICENSE +0 -0
|
@@ -7,7 +7,6 @@ from __future__ import annotations
|
|
|
7
7
|
|
|
8
8
|
import pathlib # noqa: TC003 import is needed for docs build
|
|
9
9
|
import re
|
|
10
|
-
import typing as t
|
|
11
10
|
from abc import ABC
|
|
12
11
|
from abc import abstractmethod
|
|
13
12
|
from typing import TYPE_CHECKING
|
|
@@ -56,6 +55,8 @@ KNOWN_FLOAT_TYPES = (
|
|
|
56
55
|
|
|
57
56
|
KNOWN_STRING_TYPES = (
|
|
58
57
|
"string",
|
|
58
|
+
"string[pyarrow]",
|
|
59
|
+
"large_string",
|
|
59
60
|
"str",
|
|
60
61
|
"char",
|
|
61
62
|
"varchar",
|
|
@@ -67,13 +68,18 @@ KNOWN_STRING_TYPES = (
|
|
|
67
68
|
|
|
68
69
|
KNOWN_DATETIME_TYPES = (
|
|
69
70
|
"timestamp",
|
|
71
|
+
"timestamp[s]",
|
|
72
|
+
"timestamp[ms]",
|
|
70
73
|
"timestamp[us]",
|
|
71
74
|
"timestamp[ns]",
|
|
75
|
+
"datetime",
|
|
72
76
|
"datetime64",
|
|
73
|
-
"
|
|
74
|
-
"
|
|
77
|
+
"datetime64[s]",
|
|
78
|
+
"datetime64[ms]",
|
|
79
|
+
"datetime64[us]",
|
|
80
|
+
"datetime64[ns]",
|
|
75
81
|
"date",
|
|
76
|
-
"
|
|
82
|
+
"date32[day]",
|
|
77
83
|
"time",
|
|
78
84
|
)
|
|
79
85
|
|
|
@@ -91,8 +97,6 @@ TYPE_MAP: dict[str, DataType] = {}
|
|
|
91
97
|
for concrete_type, abstract_type in TYPE_CORRESPONDENCE:
|
|
92
98
|
TYPE_MAP.update(dict.fromkeys(concrete_type, abstract_type))
|
|
93
99
|
|
|
94
|
-
TDatasetParser = t.TypeVar("TDatasetParser", bound="DatasetParser")
|
|
95
|
-
|
|
96
100
|
|
|
97
101
|
class DatasetParser(ABC):
|
|
98
102
|
"""Abstract Base Class for all Dataset parsers.
|
|
@@ -118,7 +122,7 @@ class DatasetParser(ABC):
|
|
|
118
122
|
# Gzipped parquet files can be read with DatasetParserParquet
|
|
119
123
|
match = re.search(PARQUET_GZIP_FILE_SUFFIX, str(dataset).lower())
|
|
120
124
|
file_type = PARQUET_GZIP_FILE_SUFFIX if match else file_type
|
|
121
|
-
# Extract the appropriate reader class from the SUPPORTED_FILE_TYPES dict
|
|
125
|
+
# Extract the appropriate reader class from the SUPPORTED_FILE_TYPES dict
|
|
122
126
|
reader = SUPPORTED_DATASET_FILE_SUFFIXES[file_type](dataset)
|
|
123
127
|
except IndexError as e:
|
|
124
128
|
# Thrown when just one element is returned from split, meaning there is no file extension supplied
|
|
@@ -149,6 +153,9 @@ class DatasetParser(ABC):
|
|
|
149
153
|
|
|
150
154
|
Arguments:
|
|
151
155
|
data_type: The concrete data type to map.
|
|
156
|
+
|
|
157
|
+
Returns:
|
|
158
|
+
The abstract data type or None
|
|
152
159
|
"""
|
|
153
160
|
return TYPE_MAP.get(data_type.lower(), None)
|
|
154
161
|
|
|
@@ -9,7 +9,7 @@ dapla_metadata/datasets/__init__.py,sha256=TvzskpdFC6hGcC9_55URT5jr5wNAPzXuISd2U
|
|
|
9
9
|
dapla_metadata/datasets/code_list.py,sha256=gNSJsDoQsOgwr-nivNoV51cf19ar_I_Qd9S8LXhdJVE,9065
|
|
10
10
|
dapla_metadata/datasets/core.py,sha256=gp-zTJyn5Z_9bbmHF749OUx0kpYviiDAP9MDwSZiRW8,21768
|
|
11
11
|
dapla_metadata/datasets/dapla_dataset_path_info.py,sha256=zdkVjxlqXMBe7eTAneUrTDP0_fx7JsEQ_0JrKjREhfU,26854
|
|
12
|
-
dapla_metadata/datasets/dataset_parser.py,sha256=
|
|
12
|
+
dapla_metadata/datasets/dataset_parser.py,sha256=bc3KOIDQGgdZMPh3XVHhiKMsY6FxIY9glvGlwTM4g7I,8233
|
|
13
13
|
dapla_metadata/datasets/external_sources/__init__.py,sha256=qvIdXwqyEmXNUCB94ZtZXRzifdW4hiXASFFPtC70f6E,83
|
|
14
14
|
dapla_metadata/datasets/external_sources/external_sources.py,sha256=9eIcOIUbaodNX1w9Tj2wl4U4wUmr5kF1R0i01fKUzGs,2974
|
|
15
15
|
dapla_metadata/datasets/model_backwards_compatibility.py,sha256=0NZM0j3P3ngmssH69t24BKagAxEL3veb_n5TkWK2jaA,19167
|
|
@@ -83,7 +83,7 @@ dapla_metadata/variable_definitions/exceptions.py,sha256=z6Gtd84FboDu7vWjC3wathI
|
|
|
83
83
|
dapla_metadata/variable_definitions/resources/vardef_model_descriptions_nb.yaml,sha256=cA1jh6JBTPWluSc2fhC__lMj8wmDrlk3CJODzSY8nKY,4771
|
|
84
84
|
dapla_metadata/variable_definitions/vardef.py,sha256=ZBV8ezGRJ8rsmb2H-ZyVX3-DNnMDPIZsHa7MzlcEkNM,11340
|
|
85
85
|
dapla_metadata/variable_definitions/variable_definition.py,sha256=Q_XjMKaJhPwoM0vyfgYVWKYNAYFX0caNWWthXmQxeNs,14784
|
|
86
|
-
dapla_toolbelt_metadata-0.6.
|
|
87
|
-
dapla_toolbelt_metadata-0.6.
|
|
88
|
-
dapla_toolbelt_metadata-0.6.
|
|
89
|
-
dapla_toolbelt_metadata-0.6.
|
|
86
|
+
dapla_toolbelt_metadata-0.6.1.dist-info/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
|
|
87
|
+
dapla_toolbelt_metadata-0.6.1.dist-info/METADATA,sha256=f4K7r9ABjeJAuZ-inim82NKKsBMj5u6Qvl_DynYi_EM,4866
|
|
88
|
+
dapla_toolbelt_metadata-0.6.1.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
|
|
89
|
+
dapla_toolbelt_metadata-0.6.1.dist-info/RECORD,,
|
|
File without changes
|