hydroserverpy 0.4.0__py3-none-any.whl → 0.5.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of hydroserverpy might be problematic. Click here for more details.
- hydroserverpy/__init__.py +2 -3
- hydroserverpy/api/http.py +22 -0
- hydroserverpy/api/main.py +173 -0
- hydroserverpy/api/models/__init__.py +21 -0
- hydroserverpy/api/models/base.py +74 -0
- hydroserverpy/api/models/etl/__init__.py +0 -0
- hydroserverpy/api/models/etl/data_archive.py +105 -0
- hydroserverpy/api/models/etl/data_source.py +150 -0
- hydroserverpy/api/models/etl/orchestration_configuration.py +35 -0
- hydroserverpy/api/models/etl/orchestration_system.py +78 -0
- hydroserverpy/api/models/iam/__init__.py +0 -0
- hydroserverpy/api/models/iam/account.py +12 -0
- hydroserverpy/api/models/iam/collaborator.py +34 -0
- hydroserverpy/api/models/iam/role.py +10 -0
- hydroserverpy/api/models/iam/workspace.py +238 -0
- hydroserverpy/api/models/sta/__init__.py +0 -0
- hydroserverpy/api/models/sta/datastream.py +338 -0
- hydroserverpy/api/models/sta/observed_property.py +72 -0
- hydroserverpy/api/models/sta/processing_level.py +50 -0
- hydroserverpy/api/models/sta/result_qualifier.py +49 -0
- hydroserverpy/api/models/sta/sensor.py +105 -0
- hydroserverpy/api/models/sta/thing.py +217 -0
- hydroserverpy/api/models/sta/unit.py +49 -0
- hydroserverpy/api/services/__init__.py +11 -0
- hydroserverpy/api/services/base.py +102 -0
- hydroserverpy/api/services/etl/__init__.py +0 -0
- hydroserverpy/api/services/etl/data_archive.py +196 -0
- hydroserverpy/api/services/etl/data_source.py +196 -0
- hydroserverpy/api/services/etl/orchestration_system.py +74 -0
- hydroserverpy/api/services/iam/__init__.py +0 -0
- hydroserverpy/api/services/iam/workspace.py +126 -0
- hydroserverpy/api/services/sta/__init__.py +0 -0
- hydroserverpy/api/services/sta/datastream.py +354 -0
- hydroserverpy/api/services/sta/observed_property.py +100 -0
- hydroserverpy/api/services/sta/processing_level.py +78 -0
- hydroserverpy/api/services/sta/result_qualifier.py +74 -0
- hydroserverpy/api/services/sta/sensor.py +116 -0
- hydroserverpy/api/services/sta/thing.py +188 -0
- hydroserverpy/api/services/sta/unit.py +82 -0
- hydroserverpy/etl/loaders/hydroserver_loader.py +1 -1
- hydroserverpy/etl_csv/hydroserver_etl_csv.py +49 -34
- {hydroserverpy-0.4.0.dist-info → hydroserverpy-0.5.0b2.dist-info}/METADATA +4 -3
- hydroserverpy-0.5.0b2.dist-info/RECORD +66 -0
- {hydroserverpy-0.4.0.dist-info → hydroserverpy-0.5.0b2.dist-info}/WHEEL +1 -1
- hydroserverpy/core/endpoints/__init__.py +0 -9
- hydroserverpy/core/endpoints/base.py +0 -146
- hydroserverpy/core/endpoints/data_loaders.py +0 -93
- hydroserverpy/core/endpoints/data_sources.py +0 -93
- hydroserverpy/core/endpoints/datastreams.py +0 -225
- hydroserverpy/core/endpoints/observed_properties.py +0 -111
- hydroserverpy/core/endpoints/processing_levels.py +0 -111
- hydroserverpy/core/endpoints/result_qualifiers.py +0 -111
- hydroserverpy/core/endpoints/sensors.py +0 -111
- hydroserverpy/core/endpoints/things.py +0 -261
- hydroserverpy/core/endpoints/units.py +0 -111
- hydroserverpy/core/schemas/__init__.py +0 -9
- hydroserverpy/core/schemas/base.py +0 -124
- hydroserverpy/core/schemas/data_loaders.py +0 -73
- hydroserverpy/core/schemas/data_sources.py +0 -223
- hydroserverpy/core/schemas/datastreams.py +0 -330
- hydroserverpy/core/schemas/observed_properties.py +0 -43
- hydroserverpy/core/schemas/processing_levels.py +0 -31
- hydroserverpy/core/schemas/result_qualifiers.py +0 -26
- hydroserverpy/core/schemas/sensors.py +0 -68
- hydroserverpy/core/schemas/things.py +0 -346
- hydroserverpy/core/schemas/units.py +0 -29
- hydroserverpy/core/service.py +0 -200
- hydroserverpy-0.4.0.dist-info/RECORD +0 -51
- /hydroserverpy/{core → api}/__init__.py +0 -0
- {hydroserverpy-0.4.0.dist-info → hydroserverpy-0.5.0b2.dist-info/licenses}/LICENSE +0 -0
- {hydroserverpy-0.4.0.dist-info → hydroserverpy-0.5.0b2.dist-info}/top_level.txt +0 -0
- {hydroserverpy-0.4.0.dist-info → hydroserverpy-0.5.0b2.dist-info}/zip-safe +0 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
from typing import Optional, Union, List, TYPE_CHECKING
|
|
2
|
+
from uuid import UUID
|
|
3
|
+
from ..base import SensorThingsService
|
|
4
|
+
from hydroserverpy.api.models import Sensor
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from hydroserverpy import HydroServer
|
|
9
|
+
from hydroserverpy.api.models import Workspace
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class SensorService(SensorThingsService):
|
|
13
|
+
def __init__(self, connection: "HydroServer"):
|
|
14
|
+
self._model = Sensor
|
|
15
|
+
self._api_route = "api/data"
|
|
16
|
+
self._endpoint_route = "sensors"
|
|
17
|
+
self._sta_route = "api/sensorthings/v1.1/Sensors"
|
|
18
|
+
|
|
19
|
+
super().__init__(connection)
|
|
20
|
+
|
|
21
|
+
def list(
|
|
22
|
+
self,
|
|
23
|
+
workspace: Optional[Union["Workspace", UUID, str]] = None,
|
|
24
|
+
page: int = 1,
|
|
25
|
+
page_size: int = 100,
|
|
26
|
+
) -> List["Sensor"]:
|
|
27
|
+
"""Fetch a collection of sensors."""
|
|
28
|
+
|
|
29
|
+
params = {"$top": page_size, "$skip": page_size * (page - 1)}
|
|
30
|
+
|
|
31
|
+
if workspace:
|
|
32
|
+
params["$filter"] = (
|
|
33
|
+
f"properties/workspace/id eq '{str(getattr(workspace, 'uid', workspace))}'"
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
return super()._list(params=params)
|
|
37
|
+
|
|
38
|
+
def get(
|
|
39
|
+
self, uid: Union[UUID, str], fetch_by_datastream_uid: bool = False
|
|
40
|
+
) -> "Sensor":
|
|
41
|
+
"""Get a sensor by ID."""
|
|
42
|
+
|
|
43
|
+
return self._get(
|
|
44
|
+
uid=str(uid),
|
|
45
|
+
path=(
|
|
46
|
+
f"api/sensorthings/v1.1/Datastreams('{str(uid)}')/Sensor"
|
|
47
|
+
if fetch_by_datastream_uid
|
|
48
|
+
else None
|
|
49
|
+
),
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
def create(
|
|
53
|
+
self,
|
|
54
|
+
workspace: Union["Workspace", UUID, str],
|
|
55
|
+
name: str,
|
|
56
|
+
description: str,
|
|
57
|
+
encoding_type: str,
|
|
58
|
+
method_type: str,
|
|
59
|
+
manufacturer: Optional[str] = None,
|
|
60
|
+
sensor_model: Optional[str] = None,
|
|
61
|
+
sensor_model_link: Optional[str] = None,
|
|
62
|
+
method_link: Optional[str] = None,
|
|
63
|
+
method_code: Optional[str] = None,
|
|
64
|
+
) -> "Sensor":
|
|
65
|
+
"""Create a new sensor."""
|
|
66
|
+
|
|
67
|
+
kwargs = {
|
|
68
|
+
"name": name,
|
|
69
|
+
"description": description,
|
|
70
|
+
"encodingType": encoding_type,
|
|
71
|
+
"methodType": method_type,
|
|
72
|
+
"manufacturer": manufacturer,
|
|
73
|
+
"model": sensor_model,
|
|
74
|
+
"modelLink": sensor_model_link,
|
|
75
|
+
"methodLink": method_link,
|
|
76
|
+
"methodCode": method_code,
|
|
77
|
+
"workspaceId": str(getattr(workspace, "uid", workspace)),
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return super()._create(**kwargs)
|
|
81
|
+
|
|
82
|
+
def update(
|
|
83
|
+
self,
|
|
84
|
+
uid: Union[UUID, str],
|
|
85
|
+
name: str = ...,
|
|
86
|
+
description: str = ...,
|
|
87
|
+
encoding_type: str = ...,
|
|
88
|
+
method_type: str = ...,
|
|
89
|
+
manufacturer: Optional[str] = ...,
|
|
90
|
+
sensor_model: Optional[str] = ...,
|
|
91
|
+
sensor_model_link: Optional[str] = ...,
|
|
92
|
+
method_link: Optional[str] = ...,
|
|
93
|
+
method_code: Optional[str] = ...,
|
|
94
|
+
) -> "Sensor":
|
|
95
|
+
"""Update a sensor."""
|
|
96
|
+
|
|
97
|
+
kwargs = {
|
|
98
|
+
"name": name,
|
|
99
|
+
"description": description,
|
|
100
|
+
"encodingType": encoding_type,
|
|
101
|
+
"methodType": method_type,
|
|
102
|
+
"manufacturer": manufacturer,
|
|
103
|
+
"model": sensor_model,
|
|
104
|
+
"modelLink": sensor_model_link,
|
|
105
|
+
"methodLink": method_link,
|
|
106
|
+
"methodCode": method_code,
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
return super()._update(
|
|
110
|
+
uid=str(uid), **{k: v for k, v in kwargs.items() if v is not ...}
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
def delete(self, uid: Union[UUID, str]) -> None:
|
|
114
|
+
"""Delete a sensor."""
|
|
115
|
+
|
|
116
|
+
super()._delete(uid=str(uid))
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from typing import TYPE_CHECKING, Union, IO, List, Dict, Optional
|
|
3
|
+
from uuid import UUID
|
|
4
|
+
from ..base import SensorThingsService
|
|
5
|
+
from hydroserverpy.api.models import Thing
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from hydroserverpy import HydroServer
|
|
10
|
+
from hydroserverpy.api.models import Workspace
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ThingService(SensorThingsService):
|
|
14
|
+
def __init__(self, connection: "HydroServer"):
|
|
15
|
+
self._model = Thing
|
|
16
|
+
self._api_route = "api/data"
|
|
17
|
+
self._endpoint_route = "things"
|
|
18
|
+
self._sta_route = "api/sensorthings/v1.1/Things"
|
|
19
|
+
|
|
20
|
+
super().__init__(connection)
|
|
21
|
+
|
|
22
|
+
def list(
|
|
23
|
+
self,
|
|
24
|
+
workspace: Optional[Union["Workspace", UUID, str]] = None,
|
|
25
|
+
page: int = 1,
|
|
26
|
+
page_size: int = 100,
|
|
27
|
+
) -> List["Thing"]:
|
|
28
|
+
"""Fetch a collection of things."""
|
|
29
|
+
|
|
30
|
+
params = {
|
|
31
|
+
"$top": page_size,
|
|
32
|
+
"$skip": page_size * (page - 1),
|
|
33
|
+
"$expand": "Locations",
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
if workspace:
|
|
37
|
+
params["$filter"] = (
|
|
38
|
+
f"properties/workspace/id eq '{str(getattr(workspace, 'uid', workspace))}'"
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
return super()._list(params=params)
|
|
42
|
+
|
|
43
|
+
def get(
|
|
44
|
+
self, uid: Union[UUID, str], fetch_by_datastream_uid: bool = False
|
|
45
|
+
) -> "Thing":
|
|
46
|
+
"""Get a thing by ID."""
|
|
47
|
+
|
|
48
|
+
params = {"$expand": "Locations"}
|
|
49
|
+
return self._get(
|
|
50
|
+
uid=str(uid),
|
|
51
|
+
path=(
|
|
52
|
+
f"api/sensorthings/v1.1/Datastreams('{str(uid)}')/Thing"
|
|
53
|
+
if fetch_by_datastream_uid
|
|
54
|
+
else None
|
|
55
|
+
),
|
|
56
|
+
params=params,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
def create(
|
|
60
|
+
self,
|
|
61
|
+
workspace: Union["Workspace", UUID, str],
|
|
62
|
+
name: str,
|
|
63
|
+
description: str,
|
|
64
|
+
sampling_feature_type: str,
|
|
65
|
+
sampling_feature_code: str,
|
|
66
|
+
site_type: str,
|
|
67
|
+
is_private: False,
|
|
68
|
+
latitude: float,
|
|
69
|
+
longitude: float,
|
|
70
|
+
elevation_m: Optional[float] = None,
|
|
71
|
+
elevation_datum: Optional[str] = None,
|
|
72
|
+
state: Optional[str] = None,
|
|
73
|
+
county: Optional[str] = None,
|
|
74
|
+
country: Optional[str] = None,
|
|
75
|
+
data_disclaimer: Optional[str] = None,
|
|
76
|
+
) -> "Thing":
|
|
77
|
+
"""Create a new thing."""
|
|
78
|
+
|
|
79
|
+
kwargs = {
|
|
80
|
+
"name": name,
|
|
81
|
+
"description": description,
|
|
82
|
+
"samplingFeatureType": sampling_feature_type,
|
|
83
|
+
"samplingFeatureCode": sampling_feature_code,
|
|
84
|
+
"siteType": site_type,
|
|
85
|
+
"isPrivate": is_private,
|
|
86
|
+
"latitude": latitude,
|
|
87
|
+
"longitude": longitude,
|
|
88
|
+
"elevation_m": elevation_m,
|
|
89
|
+
"elevationDatum": elevation_datum,
|
|
90
|
+
"state": state,
|
|
91
|
+
"county": county,
|
|
92
|
+
"country": country,
|
|
93
|
+
"dataDisclaimer": data_disclaimer,
|
|
94
|
+
"workspaceId": str(getattr(workspace, "uid", workspace)),
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
return super()._create(**kwargs)
|
|
98
|
+
|
|
99
|
+
def update(
|
|
100
|
+
self,
|
|
101
|
+
uid: Union[UUID, str],
|
|
102
|
+
name: str = ...,
|
|
103
|
+
description: str = ...,
|
|
104
|
+
sampling_feature_type: str = ...,
|
|
105
|
+
sampling_feature_code: str = ...,
|
|
106
|
+
site_type: str = ...,
|
|
107
|
+
is_private: False = ...,
|
|
108
|
+
latitude: float = ...,
|
|
109
|
+
longitude: float = ...,
|
|
110
|
+
elevation_m: Optional[float] = ...,
|
|
111
|
+
elevation_datum: Optional[str] = ...,
|
|
112
|
+
state: Optional[str] = ...,
|
|
113
|
+
county: Optional[str] = ...,
|
|
114
|
+
country: Optional[str] = ...,
|
|
115
|
+
data_disclaimer: Optional[str] = ...,
|
|
116
|
+
) -> "Thing":
|
|
117
|
+
"""Update a thing."""
|
|
118
|
+
|
|
119
|
+
kwargs = {
|
|
120
|
+
"name": name,
|
|
121
|
+
"description": description,
|
|
122
|
+
"samplingFeatureType": sampling_feature_type,
|
|
123
|
+
"samplingFeatureCode": sampling_feature_code,
|
|
124
|
+
"siteType": site_type,
|
|
125
|
+
"isPrivate": is_private,
|
|
126
|
+
"latitude": latitude,
|
|
127
|
+
"longitude": longitude,
|
|
128
|
+
"elevation_m": elevation_m,
|
|
129
|
+
"elevationDatum": elevation_datum,
|
|
130
|
+
"state": state,
|
|
131
|
+
"county": county,
|
|
132
|
+
"country": country,
|
|
133
|
+
"dataDisclaimer": data_disclaimer,
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
return super()._update(
|
|
137
|
+
uid=str(uid), **{k: v for k, v in kwargs.items() if v is not ...}
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
def delete(self, uid: Union[UUID, str]) -> None:
|
|
141
|
+
"""Delete a thing."""
|
|
142
|
+
|
|
143
|
+
super()._delete(uid=str(uid))
|
|
144
|
+
|
|
145
|
+
def add_tag(self, uid: Union[UUID, str], key: str, value: str) -> Dict[str, str]:
|
|
146
|
+
"""Tag a HydroServer thing."""
|
|
147
|
+
|
|
148
|
+
return self._connection.request(
|
|
149
|
+
"post",
|
|
150
|
+
f"{self._api_route}/{self._endpoint_route}/{str(uid)}/tags",
|
|
151
|
+
data=json.dumps({"key": key, "value": value}),
|
|
152
|
+
).json()
|
|
153
|
+
|
|
154
|
+
def update_tag(self, uid: Union[UUID, str], key: str, value: str) -> Dict[str, str]:
|
|
155
|
+
"""Update the tag of a HydroServer thing."""
|
|
156
|
+
|
|
157
|
+
return self._connection.request(
|
|
158
|
+
"put",
|
|
159
|
+
f"{self._api_route}/{self._endpoint_route}/{str(uid)}/tags",
|
|
160
|
+
data=json.dumps({"key": key, "value": value}),
|
|
161
|
+
).json()
|
|
162
|
+
|
|
163
|
+
def delete_tag(self, uid: Union[UUID, str], key: str) -> None:
|
|
164
|
+
"""Remove a tag from a HydroServer thing."""
|
|
165
|
+
|
|
166
|
+
self._connection.request(
|
|
167
|
+
"delete",
|
|
168
|
+
f"{self._api_route}/{self._endpoint_route}/{str(uid)}/tags",
|
|
169
|
+
data=json.dumps({"key": key}),
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
def add_photo(self, uid: Union[UUID, str], file: IO[bytes]) -> Dict[str, str]:
|
|
173
|
+
"""Add a photo of a HydroServer thing."""
|
|
174
|
+
|
|
175
|
+
return self._connection.request(
|
|
176
|
+
"post",
|
|
177
|
+
f"{self._api_route}/{self._endpoint_route}/{str(uid)}/photos",
|
|
178
|
+
files={"file": file},
|
|
179
|
+
).json()
|
|
180
|
+
|
|
181
|
+
def delete_photo(self, uid: Union[UUID, str], name: str) -> None:
|
|
182
|
+
"""Delete a photo of a HydroServer thing."""
|
|
183
|
+
|
|
184
|
+
self._connection.request(
|
|
185
|
+
"delete",
|
|
186
|
+
f"{self._api_route}/{self._endpoint_route}/{str(uid)}/photos",
|
|
187
|
+
data=json.dumps({"name": name}),
|
|
188
|
+
)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from typing import Optional, Union, List, TYPE_CHECKING
|
|
2
|
+
from uuid import UUID
|
|
3
|
+
from ..base import EndpointService
|
|
4
|
+
from hydroserverpy.api.models import Unit
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from hydroserverpy import HydroServer
|
|
9
|
+
from hydroserverpy.api.models import Workspace
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class UnitService(EndpointService):
|
|
13
|
+
def __init__(self, connection: "HydroServer"):
|
|
14
|
+
self._model = Unit
|
|
15
|
+
self._api_route = "api/data"
|
|
16
|
+
self._endpoint_route = "units"
|
|
17
|
+
|
|
18
|
+
super().__init__(connection)
|
|
19
|
+
|
|
20
|
+
def list(
|
|
21
|
+
self,
|
|
22
|
+
workspace: Optional[Union["Workspace", UUID, str]] = None,
|
|
23
|
+
) -> List["Unit"]:
|
|
24
|
+
"""Fetch a collection of units."""
|
|
25
|
+
|
|
26
|
+
workspace_id = getattr(workspace, "uid", workspace)
|
|
27
|
+
workspace_id = str(workspace_id) if workspace_id else None
|
|
28
|
+
|
|
29
|
+
return super()._list(
|
|
30
|
+
params={"workspace_id": workspace_id} if workspace_id else {},
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
def get(self, uid: Union[UUID, str]) -> "Unit":
|
|
34
|
+
"""Get a unit by ID."""
|
|
35
|
+
|
|
36
|
+
return super()._get(uid=str(uid))
|
|
37
|
+
|
|
38
|
+
def create(
|
|
39
|
+
self,
|
|
40
|
+
workspace: Union["Workspace", UUID, str],
|
|
41
|
+
name: str,
|
|
42
|
+
symbol: str,
|
|
43
|
+
definition: str,
|
|
44
|
+
unit_type: str,
|
|
45
|
+
) -> "Unit":
|
|
46
|
+
"""Create a new unit."""
|
|
47
|
+
|
|
48
|
+
kwargs = {
|
|
49
|
+
"name": name,
|
|
50
|
+
"symbol": symbol,
|
|
51
|
+
"definition": definition,
|
|
52
|
+
"type": unit_type,
|
|
53
|
+
"workspaceId": str(getattr(workspace, "uid", workspace)),
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
return super()._create(**kwargs)
|
|
57
|
+
|
|
58
|
+
def update(
|
|
59
|
+
self,
|
|
60
|
+
uid: Union[UUID, str],
|
|
61
|
+
name: str = ...,
|
|
62
|
+
symbol: str = ...,
|
|
63
|
+
definition: str = ...,
|
|
64
|
+
unit_type: str = ...,
|
|
65
|
+
) -> "Unit":
|
|
66
|
+
"""Update a unit."""
|
|
67
|
+
|
|
68
|
+
kwargs = {
|
|
69
|
+
"name": name,
|
|
70
|
+
"symbol": symbol,
|
|
71
|
+
"definition": definition,
|
|
72
|
+
"type": unit_type,
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return super()._update(
|
|
76
|
+
uid=str(uid), **{k: v for k, v in kwargs.items() if v is not ...}
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
def delete(self, uid: Union[UUID, str]) -> None:
|
|
80
|
+
"""Delete a unit."""
|
|
81
|
+
|
|
82
|
+
super()._delete(uid=str(uid))
|
|
@@ -10,7 +10,7 @@ from .exceptions import HeaderParsingError, TimestampParsingError
|
|
|
10
10
|
import warnings
|
|
11
11
|
|
|
12
12
|
if TYPE_CHECKING:
|
|
13
|
-
from
|
|
13
|
+
from hydroserverpy.api.models import DataSource
|
|
14
14
|
|
|
15
15
|
logger = logging.getLogger("hydroserver_etl")
|
|
16
16
|
logger.addHandler(logging.NullHandler())
|
|
@@ -36,10 +36,15 @@ class HydroServerETLCSV:
|
|
|
36
36
|
datastream.uid: datastream for datastream in data_source.datastreams
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
self._datastream_mapping = {
|
|
40
|
+
mapping["targetIdentifier"]: mapping["sourceIdentifier"]
|
|
41
|
+
for payload in self._data_source.settings["payloads"]
|
|
42
|
+
for mapping in payload.get("mappings", [])
|
|
43
|
+
}
|
|
44
|
+
|
|
39
45
|
self._timestamp_column_index = None
|
|
40
46
|
self._datastream_column_indexes = None
|
|
41
47
|
self._datastream_start_row_indexes = {}
|
|
42
|
-
self._last_loaded_timestamp = self._data_source.data_source_thru
|
|
43
48
|
|
|
44
49
|
self._message = None
|
|
45
50
|
self._failed_datastreams = []
|
|
@@ -59,7 +64,10 @@ class HydroServerETLCSV:
|
|
|
59
64
|
:return: None
|
|
60
65
|
"""
|
|
61
66
|
|
|
62
|
-
data_reader = csv.reader(
|
|
67
|
+
data_reader = csv.reader(
|
|
68
|
+
self._data_file,
|
|
69
|
+
delimiter=self._data_source.settings["transformer"]["delimiter"],
|
|
70
|
+
)
|
|
63
71
|
|
|
64
72
|
try:
|
|
65
73
|
for i, row in enumerate(data_reader):
|
|
@@ -104,13 +112,13 @@ class HydroServerETLCSV:
|
|
|
104
112
|
:return: A list of datetime and value pairs for each datastream
|
|
105
113
|
"""
|
|
106
114
|
|
|
107
|
-
if index == self._data_source.
|
|
108
|
-
index == self._data_source.
|
|
115
|
+
if index == self._data_source.settings["transformer"]["headerRow"] or (
|
|
116
|
+
index == self._data_source.settings["transformer"]["dataStartRow"]
|
|
109
117
|
and self._timestamp_column_index is None
|
|
110
118
|
):
|
|
111
119
|
self._parse_file_header(row)
|
|
112
120
|
|
|
113
|
-
if index < self._data_source.
|
|
121
|
+
if index < self._data_source.settings["transformer"]["dataStartRow"]:
|
|
114
122
|
return
|
|
115
123
|
|
|
116
124
|
timestamp = self._parse_row_timestamp(row)
|
|
@@ -135,7 +143,7 @@ class HydroServerETLCSV:
|
|
|
135
143
|
"phenomenon_time": timestamp,
|
|
136
144
|
"result": row[
|
|
137
145
|
self._datastream_column_indexes[
|
|
138
|
-
datastream.
|
|
146
|
+
self._datastream_mapping[str(datastream.uid)]
|
|
139
147
|
]
|
|
140
148
|
],
|
|
141
149
|
}
|
|
@@ -155,17 +163,19 @@ class HydroServerETLCSV:
|
|
|
155
163
|
|
|
156
164
|
try:
|
|
157
165
|
self._timestamp_column_index = (
|
|
158
|
-
row.index(self._data_source.
|
|
159
|
-
if isinstance(
|
|
160
|
-
|
|
166
|
+
row.index(self._data_source.settings["transformer"]["timestampKey"])
|
|
167
|
+
if isinstance(
|
|
168
|
+
self._data_source.settings["transformer"]["timestampKey"], str
|
|
169
|
+
)
|
|
170
|
+
else int(self._data_source.settings["transformer"]["timestampKey"]) - 1
|
|
161
171
|
)
|
|
162
172
|
if self._timestamp_column_index > len(row):
|
|
163
173
|
raise ValueError
|
|
164
174
|
self._datastream_column_indexes = {
|
|
165
|
-
datastream.
|
|
166
|
-
row.index(datastream.
|
|
167
|
-
if not datastream.
|
|
168
|
-
else int(datastream.
|
|
175
|
+
self._datastream_mapping[str(datastream.uid)]: (
|
|
176
|
+
row.index(self._datastream_mapping[str(datastream.uid)])
|
|
177
|
+
if not self._datastream_mapping[str(datastream.uid)].isdigit()
|
|
178
|
+
else int(self._datastream_mapping[str(datastream.uid)]) - 1
|
|
169
179
|
)
|
|
170
180
|
for datastream in self._datastreams.values()
|
|
171
181
|
}
|
|
@@ -190,28 +200,40 @@ class HydroServerETLCSV:
|
|
|
190
200
|
|
|
191
201
|
try:
|
|
192
202
|
if (
|
|
193
|
-
self._data_source.
|
|
194
|
-
|
|
203
|
+
self._data_source.settings["transformer"].get("timestampFormat")
|
|
204
|
+
== "ISO8601"
|
|
205
|
+
or self._data_source.settings["transformer"].get("timestampFormat")
|
|
206
|
+
is None
|
|
195
207
|
):
|
|
196
208
|
timestamp = isoparse(row[self._timestamp_column_index])
|
|
197
209
|
else:
|
|
198
210
|
timestamp = datetime.strptime(
|
|
199
211
|
row[self._timestamp_column_index],
|
|
200
|
-
self._data_source.
|
|
212
|
+
self._data_source.settings["transformer"].get("timestampFormat"),
|
|
201
213
|
)
|
|
202
214
|
except ValueError as e:
|
|
203
215
|
raise TimestampParsingError(str(e)) from e
|
|
204
216
|
|
|
205
217
|
if timestamp.tzinfo is None:
|
|
206
|
-
if not self._data_source.
|
|
218
|
+
if not self._data_source.settings["transformer"].get(
|
|
219
|
+
"timestampOffset"
|
|
220
|
+
) or self._data_source.settings["transformer"].get(
|
|
221
|
+
"timestampOffset"
|
|
222
|
+
).endswith(
|
|
223
|
+
"0000"
|
|
224
|
+
):
|
|
207
225
|
timestamp = timestamp.replace(tzinfo=timezone.utc)
|
|
208
226
|
else:
|
|
209
227
|
try:
|
|
210
228
|
timestamp = timestamp.replace(
|
|
211
229
|
tzinfo=datetime.strptime(
|
|
212
|
-
self._data_source.
|
|
230
|
+
self._data_source.settings["transformer"].get(
|
|
231
|
+
"timestampFormat"
|
|
232
|
+
)[:-2]
|
|
213
233
|
+ ":"
|
|
214
|
-
+ self._data_source.
|
|
234
|
+
+ self._data_source.settings["transformer"].get(
|
|
235
|
+
"timestampFormat"
|
|
236
|
+
)[3:],
|
|
215
237
|
"%z",
|
|
216
238
|
).tzinfo
|
|
217
239
|
)
|
|
@@ -264,12 +286,6 @@ class HydroServerETLCSV:
|
|
|
264
286
|
except HTTPError:
|
|
265
287
|
failed_datastreams.append(datastream_id)
|
|
266
288
|
|
|
267
|
-
if not self._last_loaded_timestamp or (
|
|
268
|
-
observations[-1]["phenomenon_time"]
|
|
269
|
-
and observations[-1]["phenomenon_time"]
|
|
270
|
-
> self._last_loaded_timestamp
|
|
271
|
-
):
|
|
272
|
-
self._last_loaded_timestamp = observations[-1]["phenomenon_time"]
|
|
273
289
|
elif datastream_id in self._failed_datastreams:
|
|
274
290
|
logger.info(
|
|
275
291
|
f"Skipping observations POST request from "
|
|
@@ -292,29 +308,28 @@ class HydroServerETLCSV:
|
|
|
292
308
|
"""
|
|
293
309
|
|
|
294
310
|
if self._data_source.crontab is not None:
|
|
295
|
-
|
|
311
|
+
next_run = croniter.croniter(
|
|
296
312
|
self._data_source.crontab, datetime.now()
|
|
297
313
|
).get_next(datetime)
|
|
298
314
|
elif (
|
|
299
315
|
self._data_source.interval is not None
|
|
300
316
|
and self._data_source.interval_units is not None
|
|
301
317
|
):
|
|
302
|
-
|
|
318
|
+
next_run = datetime.now() + timedelta(
|
|
303
319
|
**{self._data_source.interval_units: self._data_source.interval}
|
|
304
320
|
)
|
|
305
321
|
else:
|
|
306
|
-
|
|
322
|
+
next_run = None
|
|
307
323
|
|
|
308
|
-
self._data_source.
|
|
309
|
-
self._data_source.last_sync_successful = (
|
|
324
|
+
self._data_source.last_run_successful = (
|
|
310
325
|
True
|
|
311
326
|
if not self._file_timestamp_error
|
|
312
327
|
and not self._file_header_error
|
|
313
328
|
and len(self._failed_datastreams) == 0
|
|
314
329
|
else False
|
|
315
330
|
)
|
|
316
|
-
self._data_source.
|
|
317
|
-
self._data_source.
|
|
318
|
-
self._data_source.
|
|
331
|
+
self._data_source.last_run_message = self._message
|
|
332
|
+
self._data_source.last_run = datetime.now(timezone.utc)
|
|
333
|
+
self._data_source.next_run = next_run
|
|
319
334
|
|
|
320
335
|
self._data_source.save()
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: hydroserverpy
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0b2
|
|
4
4
|
Requires-Python: <4,>=3.9
|
|
5
5
|
License-File: LICENSE
|
|
6
6
|
Requires-Dist: requests>=2
|
|
7
7
|
Requires-Dist: pydantic>=2.6
|
|
8
|
+
Requires-Dist: pydantic[email]>=2.6
|
|
8
9
|
Requires-Dist: pandas>=2.2
|
|
9
10
|
Requires-Dist: numpy>=2.0
|
|
10
11
|
Requires-Dist: pyyaml>=5
|
|
@@ -12,7 +13,7 @@ Requires-Dist: simplejson>=3
|
|
|
12
13
|
Requires-Dist: crontab>=1
|
|
13
14
|
Requires-Dist: python-dateutil>=2.8.2
|
|
14
15
|
Requires-Dist: croniter>=2.0.1
|
|
15
|
-
Requires-Dist: country-list>=1.1.0
|
|
16
16
|
Requires-Dist: jmespath>=1.0.1
|
|
17
17
|
Provides-Extra: docs
|
|
18
18
|
Requires-Dist: sphinx_autodoc_typehints; extra == "docs"
|
|
19
|
+
Dynamic: license-file
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
hydroserverpy/__init__.py,sha256=FgaGFyhCjwmpJYEKNzOZxvfRx2neWMaOybj1z02_VSE,218
|
|
2
|
+
hydroserverpy/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
hydroserverpy/api/http.py,sha256=C5DgvEeiu54RaL-9oouFPSKosC8Uy5Qdwm5hYh2Ps-s,620
|
|
4
|
+
hydroserverpy/api/main.py,sha256=OWJpCIuheBgWQA7R33BiG89-upMS-a3K_AScbVeVjxQ,4760
|
|
5
|
+
hydroserverpy/api/models/__init__.py,sha256=ELrf3b7Aix7YcVF__Q_8e_G_FF8GYlX0J5l7fkGcHnY,690
|
|
6
|
+
hydroserverpy/api/models/base.py,sha256=dc2tfMSgizymxAAOVURfy7Jzeh6xIiiq7hfWZI7l1_Q,2280
|
|
7
|
+
hydroserverpy/api/models/etl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
hydroserverpy/api/models/etl/data_archive.py,sha256=u-gpvUsaWaw0kyF3bPMm2e55Jx2yhvSV9ufXXaNtrTc,3429
|
|
9
|
+
hydroserverpy/api/models/etl/data_source.py,sha256=ca-9KKVhkLNaUn3vOIk-JgdWk58fTRme8YKIesk8WIw,5455
|
|
10
|
+
hydroserverpy/api/models/etl/orchestration_configuration.py,sha256=ElSrgi7ioFZJFJg6aGogW5ZZk7fA17y4p--yWwiOhZ0,1367
|
|
11
|
+
hydroserverpy/api/models/etl/orchestration_system.py,sha256=25En2G0z1gQzN-RW3UlrEGgkC952QDW21oYnawCX8hY,2357
|
|
12
|
+
hydroserverpy/api/models/iam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
hydroserverpy/api/models/iam/account.py,sha256=7COk_CPYFlthg1uFWTBlJESfnuqMW90TSjZoIcBb-_8,439
|
|
14
|
+
hydroserverpy/api/models/iam/collaborator.py,sha256=jp661DKDCwk8c8HFPAV-YVhEc80F5eGDKaSHmH62Q8Q,1007
|
|
15
|
+
hydroserverpy/api/models/iam/role.py,sha256=8FVTj_1QwtPF9tk7baliMVg000kjc5N8oP6eYo8vTDY,275
|
|
16
|
+
hydroserverpy/api/models/iam/workspace.py,sha256=s9u1oZyOdxM7txjJARFcIBrWMHQSDxODdreiatFsXJs,7331
|
|
17
|
+
hydroserverpy/api/models/sta/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
18
|
+
hydroserverpy/api/models/sta/datastream.py,sha256=sB-KifvegbyDUnyPE_NCHFrab1ZSVVb6g-Gs7kUgMiE,10774
|
|
19
|
+
hydroserverpy/api/models/sta/observed_property.py,sha256=ThTg8aPMHPxbk9Hzpxw3AwM16gE1xvYpRK8UkiOdGeA,2180
|
|
20
|
+
hydroserverpy/api/models/sta/processing_level.py,sha256=y5_0wX7QGXgswvukXJtbpOiTCZ9pI8E08DXaTSUHakg,1470
|
|
21
|
+
hydroserverpy/api/models/sta/result_qualifier.py,sha256=IJcY04KjP9e2D-jPzUJjH2PC-JvDNCjbi5LKkTVSwgw,1416
|
|
22
|
+
hydroserverpy/api/models/sta/sensor.py,sha256=TD9R1Uwcu1t9tRQBfk0crsSJmV5UN_9kH9Ye9b7lDJc,3055
|
|
23
|
+
hydroserverpy/api/models/sta/thing.py,sha256=o4Xn_Luy2IEOCBjXTbek7GvPoXZyKA0dhfzoFM6nfTs,6357
|
|
24
|
+
hydroserverpy/api/models/sta/unit.py,sha256=Pbxxp9hZErsrYImIb8-1HVnZAsJopE3US_AplSQWOJQ,1398
|
|
25
|
+
hydroserverpy/api/services/__init__.py,sha256=B0kGyn8_HlIBf9deU6qLocQTbxe59qestUZtLU9CeXk,532
|
|
26
|
+
hydroserverpy/api/services/base.py,sha256=zLhRDlf4h4zBXHpicKMI7xgQrXN_wHvvthvQzYNyj2E,3415
|
|
27
|
+
hydroserverpy/api/services/etl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
28
|
+
hydroserverpy/api/services/etl/data_archive.py,sha256=hlNJOHJSZ1kV2n2xivWIBtT1Eovj65PDbwpAyXnlZsM,6506
|
|
29
|
+
hydroserverpy/api/services/etl/data_source.py,sha256=DCgTyh8lF2iwh4uszePFg9UupXxJCN7Ww9Ut1MQKHis,6491
|
|
30
|
+
hydroserverpy/api/services/etl/orchestration_system.py,sha256=JFuSJJUq4JJUt8KlZ-Ga0ktyQIe2U0Sa7ogd4oLjex4,2166
|
|
31
|
+
hydroserverpy/api/services/iam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
32
|
+
hydroserverpy/api/services/iam/workspace.py,sha256=jJiqkMxFEp9REjR4LXyVp2o45CGBrrEadGelPPCuRJs,4547
|
|
33
|
+
hydroserverpy/api/services/sta/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
34
|
+
hydroserverpy/api/services/sta/datastream.py,sha256=_m-xFom3z8wo5-1_q8NjWUpcw36wYv1brIG7xeGGadk,12402
|
|
35
|
+
hydroserverpy/api/services/sta/observed_property.py,sha256=nRlqBldJpXlj8VOZ4EwNOs4ZgmBw5w-EqAChfM3Z0Z0,2908
|
|
36
|
+
hydroserverpy/api/services/sta/processing_level.py,sha256=Oupfeww2XgT83AwR5Spt91VjZK6MG0XIl11Et9fRjA0,2255
|
|
37
|
+
hydroserverpy/api/services/sta/result_qualifier.py,sha256=XG5Ng3xdFT-l3Ktkuq23Cty1RfmepBO7EQ9gPzidZuA,2069
|
|
38
|
+
hydroserverpy/api/services/sta/sensor.py,sha256=SbDhLjlOaM2ypLDfXmQVinj7eHHJ_fHxjTD68dM2pQI,3473
|
|
39
|
+
hydroserverpy/api/services/sta/thing.py,sha256=QL7IBwHHIgDFBpXnQF-LOUpxiRlm_HFWB1qqe7Iqq9s,5972
|
|
40
|
+
hydroserverpy/api/services/sta/unit.py,sha256=ksO-3g___9pPNBNbgM0jyDf1NeBqX79fjeJjCshrftY,2138
|
|
41
|
+
hydroserverpy/etl/__init__.py,sha256=qK2m4LZl8czR3VE8SxrlipSy5tLGLNB60lxD7dD0GjU,659
|
|
42
|
+
hydroserverpy/etl/hydroserver_etl.py,sha256=FSdvM3T7QHEWWulWRT8t-FMHSxAGB4GvleUXtSk5IWc,1507
|
|
43
|
+
hydroserverpy/etl/types.py,sha256=4PY3CM-uoXIsf2lhcqtLC6HaRGXe7HKGDU22R8-H35c,135
|
|
44
|
+
hydroserverpy/etl/extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
45
|
+
hydroserverpy/etl/extractors/base.py,sha256=GZKJfAhfJedRcNagnoqUiDZn286r-JzM7dW_F1dWsfY,275
|
|
46
|
+
hydroserverpy/etl/extractors/ftp_extractor.py,sha256=5LwvHuvLk6LwRSVyE9EkV3DPgVlAvRrOBpl1a8B7dLg,1387
|
|
47
|
+
hydroserverpy/etl/extractors/http_extractor.py,sha256=-duQwnsFBk4NQS2qhO55evcCUOnrBe3JX_LU9RyysX4,2709
|
|
48
|
+
hydroserverpy/etl/extractors/local_file_extractor.py,sha256=T_Y9NTO0cC5L9mDPbIG6wYlXDQoatg8MobP97liFl4U,692
|
|
49
|
+
hydroserverpy/etl/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
50
|
+
hydroserverpy/etl/loaders/base.py,sha256=DrA9u7SNBxkPKqaszlP368yNbxihdqIGzP8rA6NAp6U,295
|
|
51
|
+
hydroserverpy/etl/loaders/hydroserver_loader.py,sha256=bl4Z5TkXgJyKEHuPdY7LSrKMKOoZW_EDFl8dIRwDkv4,2549
|
|
52
|
+
hydroserverpy/etl/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
53
|
+
hydroserverpy/etl/transformers/base.py,sha256=237oVBhS3HQ3fcE4bZT5U1437WzV2x5kOFC229DY53M,1741
|
|
54
|
+
hydroserverpy/etl/transformers/csv_transformer.py,sha256=9DKSO4NfUUDlr_c6UnH4AU3-7LxwSSeuQdou0iiCjdM,3238
|
|
55
|
+
hydroserverpy/etl/transformers/json_transformer.py,sha256=ity0MXcYjEnlun4Y6cVSrnjrglKrK4JOXXHxWHIHN2A,2323
|
|
56
|
+
hydroserverpy/etl_csv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
57
|
+
hydroserverpy/etl_csv/exceptions.py,sha256=0UY8YUlNepG0y6FfH36hJyR1bOhwYHSZIdUSSMTg7GA,314
|
|
58
|
+
hydroserverpy/etl_csv/hydroserver_etl_csv.py,sha256=1RRqZWXb8pMhkeg6Tn1IbQ8SdKNgOl78eqeBguEXnog,13849
|
|
59
|
+
hydroserverpy/quality/__init__.py,sha256=GGBMkFSXciJLYrbV-NraFrj_mXWCy_GTcy9KKrKXU4c,84
|
|
60
|
+
hydroserverpy/quality/service.py,sha256=U02UfLKVmFvr5ySiH0n0JYzUIabq5uprrHIiwcqBlqY,13879
|
|
61
|
+
hydroserverpy-0.5.0b2.dist-info/licenses/LICENSE,sha256=xVqFxDw3QOEJukakL7gQCqIMTQ1dlSCTo6Oc1otNW80,1508
|
|
62
|
+
hydroserverpy-0.5.0b2.dist-info/METADATA,sha256=IBrwK9SbCsLvo0JQyIEQmBCxSMpdkZ4i0uNTTVU96cY,532
|
|
63
|
+
hydroserverpy-0.5.0b2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
|
64
|
+
hydroserverpy-0.5.0b2.dist-info/top_level.txt,sha256=Zf37hrncXLOYvXhgCrf5mZdeq81G9fShdE2LfYbtb7w,14
|
|
65
|
+
hydroserverpy-0.5.0b2.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
66
|
+
hydroserverpy-0.5.0b2.dist-info/RECORD,,
|