hydroserverpy 0.4.0__py3-none-any.whl → 0.5.0b2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of hydroserverpy might be problematic. Click here for more details.

Files changed (72) hide show
  1. hydroserverpy/__init__.py +2 -3
  2. hydroserverpy/api/http.py +22 -0
  3. hydroserverpy/api/main.py +173 -0
  4. hydroserverpy/api/models/__init__.py +21 -0
  5. hydroserverpy/api/models/base.py +74 -0
  6. hydroserverpy/api/models/etl/__init__.py +0 -0
  7. hydroserverpy/api/models/etl/data_archive.py +105 -0
  8. hydroserverpy/api/models/etl/data_source.py +150 -0
  9. hydroserverpy/api/models/etl/orchestration_configuration.py +35 -0
  10. hydroserverpy/api/models/etl/orchestration_system.py +78 -0
  11. hydroserverpy/api/models/iam/__init__.py +0 -0
  12. hydroserverpy/api/models/iam/account.py +12 -0
  13. hydroserverpy/api/models/iam/collaborator.py +34 -0
  14. hydroserverpy/api/models/iam/role.py +10 -0
  15. hydroserverpy/api/models/iam/workspace.py +238 -0
  16. hydroserverpy/api/models/sta/__init__.py +0 -0
  17. hydroserverpy/api/models/sta/datastream.py +338 -0
  18. hydroserverpy/api/models/sta/observed_property.py +72 -0
  19. hydroserverpy/api/models/sta/processing_level.py +50 -0
  20. hydroserverpy/api/models/sta/result_qualifier.py +49 -0
  21. hydroserverpy/api/models/sta/sensor.py +105 -0
  22. hydroserverpy/api/models/sta/thing.py +217 -0
  23. hydroserverpy/api/models/sta/unit.py +49 -0
  24. hydroserverpy/api/services/__init__.py +11 -0
  25. hydroserverpy/api/services/base.py +102 -0
  26. hydroserverpy/api/services/etl/__init__.py +0 -0
  27. hydroserverpy/api/services/etl/data_archive.py +196 -0
  28. hydroserverpy/api/services/etl/data_source.py +196 -0
  29. hydroserverpy/api/services/etl/orchestration_system.py +74 -0
  30. hydroserverpy/api/services/iam/__init__.py +0 -0
  31. hydroserverpy/api/services/iam/workspace.py +126 -0
  32. hydroserverpy/api/services/sta/__init__.py +0 -0
  33. hydroserverpy/api/services/sta/datastream.py +354 -0
  34. hydroserverpy/api/services/sta/observed_property.py +100 -0
  35. hydroserverpy/api/services/sta/processing_level.py +78 -0
  36. hydroserverpy/api/services/sta/result_qualifier.py +74 -0
  37. hydroserverpy/api/services/sta/sensor.py +116 -0
  38. hydroserverpy/api/services/sta/thing.py +188 -0
  39. hydroserverpy/api/services/sta/unit.py +82 -0
  40. hydroserverpy/etl/loaders/hydroserver_loader.py +1 -1
  41. hydroserverpy/etl_csv/hydroserver_etl_csv.py +49 -34
  42. {hydroserverpy-0.4.0.dist-info → hydroserverpy-0.5.0b2.dist-info}/METADATA +4 -3
  43. hydroserverpy-0.5.0b2.dist-info/RECORD +66 -0
  44. {hydroserverpy-0.4.0.dist-info → hydroserverpy-0.5.0b2.dist-info}/WHEEL +1 -1
  45. hydroserverpy/core/endpoints/__init__.py +0 -9
  46. hydroserverpy/core/endpoints/base.py +0 -146
  47. hydroserverpy/core/endpoints/data_loaders.py +0 -93
  48. hydroserverpy/core/endpoints/data_sources.py +0 -93
  49. hydroserverpy/core/endpoints/datastreams.py +0 -225
  50. hydroserverpy/core/endpoints/observed_properties.py +0 -111
  51. hydroserverpy/core/endpoints/processing_levels.py +0 -111
  52. hydroserverpy/core/endpoints/result_qualifiers.py +0 -111
  53. hydroserverpy/core/endpoints/sensors.py +0 -111
  54. hydroserverpy/core/endpoints/things.py +0 -261
  55. hydroserverpy/core/endpoints/units.py +0 -111
  56. hydroserverpy/core/schemas/__init__.py +0 -9
  57. hydroserverpy/core/schemas/base.py +0 -124
  58. hydroserverpy/core/schemas/data_loaders.py +0 -73
  59. hydroserverpy/core/schemas/data_sources.py +0 -223
  60. hydroserverpy/core/schemas/datastreams.py +0 -330
  61. hydroserverpy/core/schemas/observed_properties.py +0 -43
  62. hydroserverpy/core/schemas/processing_levels.py +0 -31
  63. hydroserverpy/core/schemas/result_qualifiers.py +0 -26
  64. hydroserverpy/core/schemas/sensors.py +0 -68
  65. hydroserverpy/core/schemas/things.py +0 -346
  66. hydroserverpy/core/schemas/units.py +0 -29
  67. hydroserverpy/core/service.py +0 -200
  68. hydroserverpy-0.4.0.dist-info/RECORD +0 -51
  69. /hydroserverpy/{core → api}/__init__.py +0 -0
  70. {hydroserverpy-0.4.0.dist-info → hydroserverpy-0.5.0b2.dist-info/licenses}/LICENSE +0 -0
  71. {hydroserverpy-0.4.0.dist-info → hydroserverpy-0.5.0b2.dist-info}/top_level.txt +0 -0
  72. {hydroserverpy-0.4.0.dist-info → hydroserverpy-0.5.0b2.dist-info}/zip-safe +0 -0
@@ -0,0 +1,116 @@
1
+ from typing import Optional, Union, List, TYPE_CHECKING
2
+ from uuid import UUID
3
+ from ..base import SensorThingsService
4
+ from hydroserverpy.api.models import Sensor
5
+
6
+
7
+ if TYPE_CHECKING:
8
+ from hydroserverpy import HydroServer
9
+ from hydroserverpy.api.models import Workspace
10
+
11
+
12
+ class SensorService(SensorThingsService):
13
+ def __init__(self, connection: "HydroServer"):
14
+ self._model = Sensor
15
+ self._api_route = "api/data"
16
+ self._endpoint_route = "sensors"
17
+ self._sta_route = "api/sensorthings/v1.1/Sensors"
18
+
19
+ super().__init__(connection)
20
+
21
+ def list(
22
+ self,
23
+ workspace: Optional[Union["Workspace", UUID, str]] = None,
24
+ page: int = 1,
25
+ page_size: int = 100,
26
+ ) -> List["Sensor"]:
27
+ """Fetch a collection of sensors."""
28
+
29
+ params = {"$top": page_size, "$skip": page_size * (page - 1)}
30
+
31
+ if workspace:
32
+ params["$filter"] = (
33
+ f"properties/workspace/id eq '{str(getattr(workspace, 'uid', workspace))}'"
34
+ )
35
+
36
+ return super()._list(params=params)
37
+
38
+ def get(
39
+ self, uid: Union[UUID, str], fetch_by_datastream_uid: bool = False
40
+ ) -> "Sensor":
41
+ """Get a sensor by ID."""
42
+
43
+ return self._get(
44
+ uid=str(uid),
45
+ path=(
46
+ f"api/sensorthings/v1.1/Datastreams('{str(uid)}')/Sensor"
47
+ if fetch_by_datastream_uid
48
+ else None
49
+ ),
50
+ )
51
+
52
+ def create(
53
+ self,
54
+ workspace: Union["Workspace", UUID, str],
55
+ name: str,
56
+ description: str,
57
+ encoding_type: str,
58
+ method_type: str,
59
+ manufacturer: Optional[str] = None,
60
+ sensor_model: Optional[str] = None,
61
+ sensor_model_link: Optional[str] = None,
62
+ method_link: Optional[str] = None,
63
+ method_code: Optional[str] = None,
64
+ ) -> "Sensor":
65
+ """Create a new sensor."""
66
+
67
+ kwargs = {
68
+ "name": name,
69
+ "description": description,
70
+ "encodingType": encoding_type,
71
+ "methodType": method_type,
72
+ "manufacturer": manufacturer,
73
+ "model": sensor_model,
74
+ "modelLink": sensor_model_link,
75
+ "methodLink": method_link,
76
+ "methodCode": method_code,
77
+ "workspaceId": str(getattr(workspace, "uid", workspace)),
78
+ }
79
+
80
+ return super()._create(**kwargs)
81
+
82
+ def update(
83
+ self,
84
+ uid: Union[UUID, str],
85
+ name: str = ...,
86
+ description: str = ...,
87
+ encoding_type: str = ...,
88
+ method_type: str = ...,
89
+ manufacturer: Optional[str] = ...,
90
+ sensor_model: Optional[str] = ...,
91
+ sensor_model_link: Optional[str] = ...,
92
+ method_link: Optional[str] = ...,
93
+ method_code: Optional[str] = ...,
94
+ ) -> "Sensor":
95
+ """Update a sensor."""
96
+
97
+ kwargs = {
98
+ "name": name,
99
+ "description": description,
100
+ "encodingType": encoding_type,
101
+ "methodType": method_type,
102
+ "manufacturer": manufacturer,
103
+ "model": sensor_model,
104
+ "modelLink": sensor_model_link,
105
+ "methodLink": method_link,
106
+ "methodCode": method_code,
107
+ }
108
+
109
+ return super()._update(
110
+ uid=str(uid), **{k: v for k, v in kwargs.items() if v is not ...}
111
+ )
112
+
113
+ def delete(self, uid: Union[UUID, str]) -> None:
114
+ """Delete a sensor."""
115
+
116
+ super()._delete(uid=str(uid))
@@ -0,0 +1,188 @@
1
+ import json
2
+ from typing import TYPE_CHECKING, Union, IO, List, Dict, Optional
3
+ from uuid import UUID
4
+ from ..base import SensorThingsService
5
+ from hydroserverpy.api.models import Thing
6
+
7
+
8
+ if TYPE_CHECKING:
9
+ from hydroserverpy import HydroServer
10
+ from hydroserverpy.api.models import Workspace
11
+
12
+
13
+ class ThingService(SensorThingsService):
14
+ def __init__(self, connection: "HydroServer"):
15
+ self._model = Thing
16
+ self._api_route = "api/data"
17
+ self._endpoint_route = "things"
18
+ self._sta_route = "api/sensorthings/v1.1/Things"
19
+
20
+ super().__init__(connection)
21
+
22
+ def list(
23
+ self,
24
+ workspace: Optional[Union["Workspace", UUID, str]] = None,
25
+ page: int = 1,
26
+ page_size: int = 100,
27
+ ) -> List["Thing"]:
28
+ """Fetch a collection of things."""
29
+
30
+ params = {
31
+ "$top": page_size,
32
+ "$skip": page_size * (page - 1),
33
+ "$expand": "Locations",
34
+ }
35
+
36
+ if workspace:
37
+ params["$filter"] = (
38
+ f"properties/workspace/id eq '{str(getattr(workspace, 'uid', workspace))}'"
39
+ )
40
+
41
+ return super()._list(params=params)
42
+
43
+ def get(
44
+ self, uid: Union[UUID, str], fetch_by_datastream_uid: bool = False
45
+ ) -> "Thing":
46
+ """Get a thing by ID."""
47
+
48
+ params = {"$expand": "Locations"}
49
+ return self._get(
50
+ uid=str(uid),
51
+ path=(
52
+ f"api/sensorthings/v1.1/Datastreams('{str(uid)}')/Thing"
53
+ if fetch_by_datastream_uid
54
+ else None
55
+ ),
56
+ params=params,
57
+ )
58
+
59
+ def create(
60
+ self,
61
+ workspace: Union["Workspace", UUID, str],
62
+ name: str,
63
+ description: str,
64
+ sampling_feature_type: str,
65
+ sampling_feature_code: str,
66
+ site_type: str,
67
+ is_private: False,
68
+ latitude: float,
69
+ longitude: float,
70
+ elevation_m: Optional[float] = None,
71
+ elevation_datum: Optional[str] = None,
72
+ state: Optional[str] = None,
73
+ county: Optional[str] = None,
74
+ country: Optional[str] = None,
75
+ data_disclaimer: Optional[str] = None,
76
+ ) -> "Thing":
77
+ """Create a new thing."""
78
+
79
+ kwargs = {
80
+ "name": name,
81
+ "description": description,
82
+ "samplingFeatureType": sampling_feature_type,
83
+ "samplingFeatureCode": sampling_feature_code,
84
+ "siteType": site_type,
85
+ "isPrivate": is_private,
86
+ "latitude": latitude,
87
+ "longitude": longitude,
88
+ "elevation_m": elevation_m,
89
+ "elevationDatum": elevation_datum,
90
+ "state": state,
91
+ "county": county,
92
+ "country": country,
93
+ "dataDisclaimer": data_disclaimer,
94
+ "workspaceId": str(getattr(workspace, "uid", workspace)),
95
+ }
96
+
97
+ return super()._create(**kwargs)
98
+
99
+ def update(
100
+ self,
101
+ uid: Union[UUID, str],
102
+ name: str = ...,
103
+ description: str = ...,
104
+ sampling_feature_type: str = ...,
105
+ sampling_feature_code: str = ...,
106
+ site_type: str = ...,
107
+ is_private: False = ...,
108
+ latitude: float = ...,
109
+ longitude: float = ...,
110
+ elevation_m: Optional[float] = ...,
111
+ elevation_datum: Optional[str] = ...,
112
+ state: Optional[str] = ...,
113
+ county: Optional[str] = ...,
114
+ country: Optional[str] = ...,
115
+ data_disclaimer: Optional[str] = ...,
116
+ ) -> "Thing":
117
+ """Update a thing."""
118
+
119
+ kwargs = {
120
+ "name": name,
121
+ "description": description,
122
+ "samplingFeatureType": sampling_feature_type,
123
+ "samplingFeatureCode": sampling_feature_code,
124
+ "siteType": site_type,
125
+ "isPrivate": is_private,
126
+ "latitude": latitude,
127
+ "longitude": longitude,
128
+ "elevation_m": elevation_m,
129
+ "elevationDatum": elevation_datum,
130
+ "state": state,
131
+ "county": county,
132
+ "country": country,
133
+ "dataDisclaimer": data_disclaimer,
134
+ }
135
+
136
+ return super()._update(
137
+ uid=str(uid), **{k: v for k, v in kwargs.items() if v is not ...}
138
+ )
139
+
140
+ def delete(self, uid: Union[UUID, str]) -> None:
141
+ """Delete a thing."""
142
+
143
+ super()._delete(uid=str(uid))
144
+
145
+ def add_tag(self, uid: Union[UUID, str], key: str, value: str) -> Dict[str, str]:
146
+ """Tag a HydroServer thing."""
147
+
148
+ return self._connection.request(
149
+ "post",
150
+ f"{self._api_route}/{self._endpoint_route}/{str(uid)}/tags",
151
+ data=json.dumps({"key": key, "value": value}),
152
+ ).json()
153
+
154
+ def update_tag(self, uid: Union[UUID, str], key: str, value: str) -> Dict[str, str]:
155
+ """Update the tag of a HydroServer thing."""
156
+
157
+ return self._connection.request(
158
+ "put",
159
+ f"{self._api_route}/{self._endpoint_route}/{str(uid)}/tags",
160
+ data=json.dumps({"key": key, "value": value}),
161
+ ).json()
162
+
163
+ def delete_tag(self, uid: Union[UUID, str], key: str) -> None:
164
+ """Remove a tag from a HydroServer thing."""
165
+
166
+ self._connection.request(
167
+ "delete",
168
+ f"{self._api_route}/{self._endpoint_route}/{str(uid)}/tags",
169
+ data=json.dumps({"key": key}),
170
+ )
171
+
172
+ def add_photo(self, uid: Union[UUID, str], file: IO[bytes]) -> Dict[str, str]:
173
+ """Add a photo of a HydroServer thing."""
174
+
175
+ return self._connection.request(
176
+ "post",
177
+ f"{self._api_route}/{self._endpoint_route}/{str(uid)}/photos",
178
+ files={"file": file},
179
+ ).json()
180
+
181
+ def delete_photo(self, uid: Union[UUID, str], name: str) -> None:
182
+ """Delete a photo of a HydroServer thing."""
183
+
184
+ self._connection.request(
185
+ "delete",
186
+ f"{self._api_route}/{self._endpoint_route}/{str(uid)}/photos",
187
+ data=json.dumps({"name": name}),
188
+ )
@@ -0,0 +1,82 @@
1
+ from typing import Optional, Union, List, TYPE_CHECKING
2
+ from uuid import UUID
3
+ from ..base import EndpointService
4
+ from hydroserverpy.api.models import Unit
5
+
6
+
7
+ if TYPE_CHECKING:
8
+ from hydroserverpy import HydroServer
9
+ from hydroserverpy.api.models import Workspace
10
+
11
+
12
+ class UnitService(EndpointService):
13
+ def __init__(self, connection: "HydroServer"):
14
+ self._model = Unit
15
+ self._api_route = "api/data"
16
+ self._endpoint_route = "units"
17
+
18
+ super().__init__(connection)
19
+
20
+ def list(
21
+ self,
22
+ workspace: Optional[Union["Workspace", UUID, str]] = None,
23
+ ) -> List["Unit"]:
24
+ """Fetch a collection of units."""
25
+
26
+ workspace_id = getattr(workspace, "uid", workspace)
27
+ workspace_id = str(workspace_id) if workspace_id else None
28
+
29
+ return super()._list(
30
+ params={"workspace_id": workspace_id} if workspace_id else {},
31
+ )
32
+
33
+ def get(self, uid: Union[UUID, str]) -> "Unit":
34
+ """Get a unit by ID."""
35
+
36
+ return super()._get(uid=str(uid))
37
+
38
+ def create(
39
+ self,
40
+ workspace: Union["Workspace", UUID, str],
41
+ name: str,
42
+ symbol: str,
43
+ definition: str,
44
+ unit_type: str,
45
+ ) -> "Unit":
46
+ """Create a new unit."""
47
+
48
+ kwargs = {
49
+ "name": name,
50
+ "symbol": symbol,
51
+ "definition": definition,
52
+ "type": unit_type,
53
+ "workspaceId": str(getattr(workspace, "uid", workspace)),
54
+ }
55
+
56
+ return super()._create(**kwargs)
57
+
58
+ def update(
59
+ self,
60
+ uid: Union[UUID, str],
61
+ name: str = ...,
62
+ symbol: str = ...,
63
+ definition: str = ...,
64
+ unit_type: str = ...,
65
+ ) -> "Unit":
66
+ """Update a unit."""
67
+
68
+ kwargs = {
69
+ "name": name,
70
+ "symbol": symbol,
71
+ "definition": definition,
72
+ "type": unit_type,
73
+ }
74
+
75
+ return super()._update(
76
+ uid=str(uid), **{k: v for k, v in kwargs.items() if v is not ...}
77
+ )
78
+
79
+ def delete(self, uid: Union[UUID, str]) -> None:
80
+ """Delete a unit."""
81
+
82
+ super()._delete(uid=str(uid))
@@ -1,4 +1,4 @@
1
- from hydroserverpy.core.service import HydroServer
1
+ from hydroserverpy import HydroServer
2
2
  from typing import Dict, Optional
3
3
  from .base import Loader
4
4
  import logging
@@ -10,7 +10,7 @@ from .exceptions import HeaderParsingError, TimestampParsingError
10
10
  import warnings
11
11
 
12
12
  if TYPE_CHECKING:
13
- from ..core.schemas import DataSource
13
+ from hydroserverpy.api.models import DataSource
14
14
 
15
15
  logger = logging.getLogger("hydroserver_etl")
16
16
  logger.addHandler(logging.NullHandler())
@@ -36,10 +36,15 @@ class HydroServerETLCSV:
36
36
  datastream.uid: datastream for datastream in data_source.datastreams
37
37
  }
38
38
 
39
+ self._datastream_mapping = {
40
+ mapping["targetIdentifier"]: mapping["sourceIdentifier"]
41
+ for payload in self._data_source.settings["payloads"]
42
+ for mapping in payload.get("mappings", [])
43
+ }
44
+
39
45
  self._timestamp_column_index = None
40
46
  self._datastream_column_indexes = None
41
47
  self._datastream_start_row_indexes = {}
42
- self._last_loaded_timestamp = self._data_source.data_source_thru
43
48
 
44
49
  self._message = None
45
50
  self._failed_datastreams = []
@@ -59,7 +64,10 @@ class HydroServerETLCSV:
59
64
  :return: None
60
65
  """
61
66
 
62
- data_reader = csv.reader(self._data_file, delimiter=self._data_source.delimiter)
67
+ data_reader = csv.reader(
68
+ self._data_file,
69
+ delimiter=self._data_source.settings["transformer"]["delimiter"],
70
+ )
63
71
 
64
72
  try:
65
73
  for i, row in enumerate(data_reader):
@@ -104,13 +112,13 @@ class HydroServerETLCSV:
104
112
  :return: A list of datetime and value pairs for each datastream
105
113
  """
106
114
 
107
- if index == self._data_source.header_row or (
108
- index == self._data_source.data_start_row
115
+ if index == self._data_source.settings["transformer"]["headerRow"] or (
116
+ index == self._data_source.settings["transformer"]["dataStartRow"]
109
117
  and self._timestamp_column_index is None
110
118
  ):
111
119
  self._parse_file_header(row)
112
120
 
113
- if index < self._data_source.data_start_row:
121
+ if index < self._data_source.settings["transformer"]["dataStartRow"]:
114
122
  return
115
123
 
116
124
  timestamp = self._parse_row_timestamp(row)
@@ -135,7 +143,7 @@ class HydroServerETLCSV:
135
143
  "phenomenon_time": timestamp,
136
144
  "result": row[
137
145
  self._datastream_column_indexes[
138
- datastream.data_source_column
146
+ self._datastream_mapping[str(datastream.uid)]
139
147
  ]
140
148
  ],
141
149
  }
@@ -155,17 +163,19 @@ class HydroServerETLCSV:
155
163
 
156
164
  try:
157
165
  self._timestamp_column_index = (
158
- row.index(self._data_source.timestamp_column)
159
- if isinstance(self._data_source.timestamp_column, str)
160
- else int(self._data_source.timestamp_column) - 1
166
+ row.index(self._data_source.settings["transformer"]["timestampKey"])
167
+ if isinstance(
168
+ self._data_source.settings["transformer"]["timestampKey"], str
169
+ )
170
+ else int(self._data_source.settings["transformer"]["timestampKey"]) - 1
161
171
  )
162
172
  if self._timestamp_column_index > len(row):
163
173
  raise ValueError
164
174
  self._datastream_column_indexes = {
165
- datastream.data_source_column: (
166
- row.index(datastream.data_source_column)
167
- if not datastream.data_source_column.isdigit()
168
- else int(datastream.data_source_column) - 1
175
+ self._datastream_mapping[str(datastream.uid)]: (
176
+ row.index(self._datastream_mapping[str(datastream.uid)])
177
+ if not self._datastream_mapping[str(datastream.uid)].isdigit()
178
+ else int(self._datastream_mapping[str(datastream.uid)]) - 1
169
179
  )
170
180
  for datastream in self._datastreams.values()
171
181
  }
@@ -190,28 +200,40 @@ class HydroServerETLCSV:
190
200
 
191
201
  try:
192
202
  if (
193
- self._data_source.timestamp_format == "iso"
194
- or self._data_source.timestamp_format is None
203
+ self._data_source.settings["transformer"].get("timestampFormat")
204
+ == "ISO8601"
205
+ or self._data_source.settings["transformer"].get("timestampFormat")
206
+ is None
195
207
  ):
196
208
  timestamp = isoparse(row[self._timestamp_column_index])
197
209
  else:
198
210
  timestamp = datetime.strptime(
199
211
  row[self._timestamp_column_index],
200
- self._data_source.timestamp_format,
212
+ self._data_source.settings["transformer"].get("timestampFormat"),
201
213
  )
202
214
  except ValueError as e:
203
215
  raise TimestampParsingError(str(e)) from e
204
216
 
205
217
  if timestamp.tzinfo is None:
206
- if not self._data_source.timestamp_offset:
218
+ if not self._data_source.settings["transformer"].get(
219
+ "timestampOffset"
220
+ ) or self._data_source.settings["transformer"].get(
221
+ "timestampOffset"
222
+ ).endswith(
223
+ "0000"
224
+ ):
207
225
  timestamp = timestamp.replace(tzinfo=timezone.utc)
208
226
  else:
209
227
  try:
210
228
  timestamp = timestamp.replace(
211
229
  tzinfo=datetime.strptime(
212
- self._data_source.timestamp_offset[:-2]
230
+ self._data_source.settings["transformer"].get(
231
+ "timestampFormat"
232
+ )[:-2]
213
233
  + ":"
214
- + self._data_source.timestamp_offset[3:],
234
+ + self._data_source.settings["transformer"].get(
235
+ "timestampFormat"
236
+ )[3:],
215
237
  "%z",
216
238
  ).tzinfo
217
239
  )
@@ -264,12 +286,6 @@ class HydroServerETLCSV:
264
286
  except HTTPError:
265
287
  failed_datastreams.append(datastream_id)
266
288
 
267
- if not self._last_loaded_timestamp or (
268
- observations[-1]["phenomenon_time"]
269
- and observations[-1]["phenomenon_time"]
270
- > self._last_loaded_timestamp
271
- ):
272
- self._last_loaded_timestamp = observations[-1]["phenomenon_time"]
273
289
  elif datastream_id in self._failed_datastreams:
274
290
  logger.info(
275
291
  f"Skipping observations POST request from "
@@ -292,29 +308,28 @@ class HydroServerETLCSV:
292
308
  """
293
309
 
294
310
  if self._data_source.crontab is not None:
295
- next_sync = croniter.croniter(
311
+ next_run = croniter.croniter(
296
312
  self._data_source.crontab, datetime.now()
297
313
  ).get_next(datetime)
298
314
  elif (
299
315
  self._data_source.interval is not None
300
316
  and self._data_source.interval_units is not None
301
317
  ):
302
- next_sync = datetime.now() + timedelta(
318
+ next_run = datetime.now() + timedelta(
303
319
  **{self._data_source.interval_units: self._data_source.interval}
304
320
  )
305
321
  else:
306
- next_sync = None
322
+ next_run = None
307
323
 
308
- self._data_source.data_source_thru = self._last_loaded_timestamp
309
- self._data_source.last_sync_successful = (
324
+ self._data_source.last_run_successful = (
310
325
  True
311
326
  if not self._file_timestamp_error
312
327
  and not self._file_header_error
313
328
  and len(self._failed_datastreams) == 0
314
329
  else False
315
330
  )
316
- self._data_source.last_sync_message = self._message
317
- self._data_source.last_synced = datetime.now(timezone.utc)
318
- self._data_source.next_sync = next_sync
331
+ self._data_source.last_run_message = self._message
332
+ self._data_source.last_run = datetime.now(timezone.utc)
333
+ self._data_source.next_run = next_run
319
334
 
320
335
  self._data_source.save()
@@ -1,10 +1,11 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: hydroserverpy
3
- Version: 0.4.0
3
+ Version: 0.5.0b2
4
4
  Requires-Python: <4,>=3.9
5
5
  License-File: LICENSE
6
6
  Requires-Dist: requests>=2
7
7
  Requires-Dist: pydantic>=2.6
8
+ Requires-Dist: pydantic[email]>=2.6
8
9
  Requires-Dist: pandas>=2.2
9
10
  Requires-Dist: numpy>=2.0
10
11
  Requires-Dist: pyyaml>=5
@@ -12,7 +13,7 @@ Requires-Dist: simplejson>=3
12
13
  Requires-Dist: crontab>=1
13
14
  Requires-Dist: python-dateutil>=2.8.2
14
15
  Requires-Dist: croniter>=2.0.1
15
- Requires-Dist: country-list>=1.1.0
16
16
  Requires-Dist: jmespath>=1.0.1
17
17
  Provides-Extra: docs
18
18
  Requires-Dist: sphinx_autodoc_typehints; extra == "docs"
19
+ Dynamic: license-file
@@ -0,0 +1,66 @@
1
+ hydroserverpy/__init__.py,sha256=FgaGFyhCjwmpJYEKNzOZxvfRx2neWMaOybj1z02_VSE,218
2
+ hydroserverpy/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
+ hydroserverpy/api/http.py,sha256=C5DgvEeiu54RaL-9oouFPSKosC8Uy5Qdwm5hYh2Ps-s,620
4
+ hydroserverpy/api/main.py,sha256=OWJpCIuheBgWQA7R33BiG89-upMS-a3K_AScbVeVjxQ,4760
5
+ hydroserverpy/api/models/__init__.py,sha256=ELrf3b7Aix7YcVF__Q_8e_G_FF8GYlX0J5l7fkGcHnY,690
6
+ hydroserverpy/api/models/base.py,sha256=dc2tfMSgizymxAAOVURfy7Jzeh6xIiiq7hfWZI7l1_Q,2280
7
+ hydroserverpy/api/models/etl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
+ hydroserverpy/api/models/etl/data_archive.py,sha256=u-gpvUsaWaw0kyF3bPMm2e55Jx2yhvSV9ufXXaNtrTc,3429
9
+ hydroserverpy/api/models/etl/data_source.py,sha256=ca-9KKVhkLNaUn3vOIk-JgdWk58fTRme8YKIesk8WIw,5455
10
+ hydroserverpy/api/models/etl/orchestration_configuration.py,sha256=ElSrgi7ioFZJFJg6aGogW5ZZk7fA17y4p--yWwiOhZ0,1367
11
+ hydroserverpy/api/models/etl/orchestration_system.py,sha256=25En2G0z1gQzN-RW3UlrEGgkC952QDW21oYnawCX8hY,2357
12
+ hydroserverpy/api/models/iam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
13
+ hydroserverpy/api/models/iam/account.py,sha256=7COk_CPYFlthg1uFWTBlJESfnuqMW90TSjZoIcBb-_8,439
14
+ hydroserverpy/api/models/iam/collaborator.py,sha256=jp661DKDCwk8c8HFPAV-YVhEc80F5eGDKaSHmH62Q8Q,1007
15
+ hydroserverpy/api/models/iam/role.py,sha256=8FVTj_1QwtPF9tk7baliMVg000kjc5N8oP6eYo8vTDY,275
16
+ hydroserverpy/api/models/iam/workspace.py,sha256=s9u1oZyOdxM7txjJARFcIBrWMHQSDxODdreiatFsXJs,7331
17
+ hydroserverpy/api/models/sta/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
18
+ hydroserverpy/api/models/sta/datastream.py,sha256=sB-KifvegbyDUnyPE_NCHFrab1ZSVVb6g-Gs7kUgMiE,10774
19
+ hydroserverpy/api/models/sta/observed_property.py,sha256=ThTg8aPMHPxbk9Hzpxw3AwM16gE1xvYpRK8UkiOdGeA,2180
20
+ hydroserverpy/api/models/sta/processing_level.py,sha256=y5_0wX7QGXgswvukXJtbpOiTCZ9pI8E08DXaTSUHakg,1470
21
+ hydroserverpy/api/models/sta/result_qualifier.py,sha256=IJcY04KjP9e2D-jPzUJjH2PC-JvDNCjbi5LKkTVSwgw,1416
22
+ hydroserverpy/api/models/sta/sensor.py,sha256=TD9R1Uwcu1t9tRQBfk0crsSJmV5UN_9kH9Ye9b7lDJc,3055
23
+ hydroserverpy/api/models/sta/thing.py,sha256=o4Xn_Luy2IEOCBjXTbek7GvPoXZyKA0dhfzoFM6nfTs,6357
24
+ hydroserverpy/api/models/sta/unit.py,sha256=Pbxxp9hZErsrYImIb8-1HVnZAsJopE3US_AplSQWOJQ,1398
25
+ hydroserverpy/api/services/__init__.py,sha256=B0kGyn8_HlIBf9deU6qLocQTbxe59qestUZtLU9CeXk,532
26
+ hydroserverpy/api/services/base.py,sha256=zLhRDlf4h4zBXHpicKMI7xgQrXN_wHvvthvQzYNyj2E,3415
27
+ hydroserverpy/api/services/etl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
28
+ hydroserverpy/api/services/etl/data_archive.py,sha256=hlNJOHJSZ1kV2n2xivWIBtT1Eovj65PDbwpAyXnlZsM,6506
29
+ hydroserverpy/api/services/etl/data_source.py,sha256=DCgTyh8lF2iwh4uszePFg9UupXxJCN7Ww9Ut1MQKHis,6491
30
+ hydroserverpy/api/services/etl/orchestration_system.py,sha256=JFuSJJUq4JJUt8KlZ-Ga0ktyQIe2U0Sa7ogd4oLjex4,2166
31
+ hydroserverpy/api/services/iam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
32
+ hydroserverpy/api/services/iam/workspace.py,sha256=jJiqkMxFEp9REjR4LXyVp2o45CGBrrEadGelPPCuRJs,4547
33
+ hydroserverpy/api/services/sta/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
34
+ hydroserverpy/api/services/sta/datastream.py,sha256=_m-xFom3z8wo5-1_q8NjWUpcw36wYv1brIG7xeGGadk,12402
35
+ hydroserverpy/api/services/sta/observed_property.py,sha256=nRlqBldJpXlj8VOZ4EwNOs4ZgmBw5w-EqAChfM3Z0Z0,2908
36
+ hydroserverpy/api/services/sta/processing_level.py,sha256=Oupfeww2XgT83AwR5Spt91VjZK6MG0XIl11Et9fRjA0,2255
37
+ hydroserverpy/api/services/sta/result_qualifier.py,sha256=XG5Ng3xdFT-l3Ktkuq23Cty1RfmepBO7EQ9gPzidZuA,2069
38
+ hydroserverpy/api/services/sta/sensor.py,sha256=SbDhLjlOaM2ypLDfXmQVinj7eHHJ_fHxjTD68dM2pQI,3473
39
+ hydroserverpy/api/services/sta/thing.py,sha256=QL7IBwHHIgDFBpXnQF-LOUpxiRlm_HFWB1qqe7Iqq9s,5972
40
+ hydroserverpy/api/services/sta/unit.py,sha256=ksO-3g___9pPNBNbgM0jyDf1NeBqX79fjeJjCshrftY,2138
41
+ hydroserverpy/etl/__init__.py,sha256=qK2m4LZl8czR3VE8SxrlipSy5tLGLNB60lxD7dD0GjU,659
42
+ hydroserverpy/etl/hydroserver_etl.py,sha256=FSdvM3T7QHEWWulWRT8t-FMHSxAGB4GvleUXtSk5IWc,1507
43
+ hydroserverpy/etl/types.py,sha256=4PY3CM-uoXIsf2lhcqtLC6HaRGXe7HKGDU22R8-H35c,135
44
+ hydroserverpy/etl/extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
45
+ hydroserverpy/etl/extractors/base.py,sha256=GZKJfAhfJedRcNagnoqUiDZn286r-JzM7dW_F1dWsfY,275
46
+ hydroserverpy/etl/extractors/ftp_extractor.py,sha256=5LwvHuvLk6LwRSVyE9EkV3DPgVlAvRrOBpl1a8B7dLg,1387
47
+ hydroserverpy/etl/extractors/http_extractor.py,sha256=-duQwnsFBk4NQS2qhO55evcCUOnrBe3JX_LU9RyysX4,2709
48
+ hydroserverpy/etl/extractors/local_file_extractor.py,sha256=T_Y9NTO0cC5L9mDPbIG6wYlXDQoatg8MobP97liFl4U,692
49
+ hydroserverpy/etl/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
50
+ hydroserverpy/etl/loaders/base.py,sha256=DrA9u7SNBxkPKqaszlP368yNbxihdqIGzP8rA6NAp6U,295
51
+ hydroserverpy/etl/loaders/hydroserver_loader.py,sha256=bl4Z5TkXgJyKEHuPdY7LSrKMKOoZW_EDFl8dIRwDkv4,2549
52
+ hydroserverpy/etl/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
53
+ hydroserverpy/etl/transformers/base.py,sha256=237oVBhS3HQ3fcE4bZT5U1437WzV2x5kOFC229DY53M,1741
54
+ hydroserverpy/etl/transformers/csv_transformer.py,sha256=9DKSO4NfUUDlr_c6UnH4AU3-7LxwSSeuQdou0iiCjdM,3238
55
+ hydroserverpy/etl/transformers/json_transformer.py,sha256=ity0MXcYjEnlun4Y6cVSrnjrglKrK4JOXXHxWHIHN2A,2323
56
+ hydroserverpy/etl_csv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
57
+ hydroserverpy/etl_csv/exceptions.py,sha256=0UY8YUlNepG0y6FfH36hJyR1bOhwYHSZIdUSSMTg7GA,314
58
+ hydroserverpy/etl_csv/hydroserver_etl_csv.py,sha256=1RRqZWXb8pMhkeg6Tn1IbQ8SdKNgOl78eqeBguEXnog,13849
59
+ hydroserverpy/quality/__init__.py,sha256=GGBMkFSXciJLYrbV-NraFrj_mXWCy_GTcy9KKrKXU4c,84
60
+ hydroserverpy/quality/service.py,sha256=U02UfLKVmFvr5ySiH0n0JYzUIabq5uprrHIiwcqBlqY,13879
61
+ hydroserverpy-0.5.0b2.dist-info/licenses/LICENSE,sha256=xVqFxDw3QOEJukakL7gQCqIMTQ1dlSCTo6Oc1otNW80,1508
62
+ hydroserverpy-0.5.0b2.dist-info/METADATA,sha256=IBrwK9SbCsLvo0JQyIEQmBCxSMpdkZ4i0uNTTVU96cY,532
63
+ hydroserverpy-0.5.0b2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
64
+ hydroserverpy-0.5.0b2.dist-info/top_level.txt,sha256=Zf37hrncXLOYvXhgCrf5mZdeq81G9fShdE2LfYbtb7w,14
65
+ hydroserverpy-0.5.0b2.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
66
+ hydroserverpy-0.5.0b2.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.6.0)
2
+ Generator: setuptools (78.1.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5