hydroserverpy 0.3.0__py3-none-any.whl → 0.5.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of hydroserverpy might be problematic. Click here for more details.
- hydroserverpy/__init__.py +3 -4
- hydroserverpy/api/http.py +24 -0
- hydroserverpy/api/main.py +152 -0
- hydroserverpy/api/models/__init__.py +18 -0
- hydroserverpy/api/models/base.py +74 -0
- hydroserverpy/api/models/etl/__init__.py +0 -0
- hydroserverpy/api/models/iam/__init__.py +0 -0
- hydroserverpy/api/models/iam/account.py +12 -0
- hydroserverpy/api/models/iam/collaborator.py +34 -0
- hydroserverpy/api/models/iam/role.py +10 -0
- hydroserverpy/api/models/iam/workspace.py +203 -0
- hydroserverpy/api/models/sta/__init__.py +0 -0
- hydroserverpy/api/models/sta/datastream.py +336 -0
- hydroserverpy/api/models/sta/observed_property.py +72 -0
- hydroserverpy/api/models/sta/processing_level.py +50 -0
- hydroserverpy/api/models/sta/result_qualifier.py +49 -0
- hydroserverpy/api/models/sta/sensor.py +105 -0
- hydroserverpy/api/models/sta/thing.py +217 -0
- hydroserverpy/api/models/sta/unit.py +49 -0
- hydroserverpy/api/services/__init__.py +8 -0
- hydroserverpy/api/services/base.py +92 -0
- hydroserverpy/api/services/etl/__init__.py +0 -0
- hydroserverpy/api/services/iam/__init__.py +0 -0
- hydroserverpy/api/services/iam/workspace.py +126 -0
- hydroserverpy/api/services/sta/__init__.py +0 -0
- hydroserverpy/api/services/sta/datastream.py +354 -0
- hydroserverpy/api/services/sta/observed_property.py +98 -0
- hydroserverpy/api/services/sta/processing_level.py +78 -0
- hydroserverpy/api/services/sta/result_qualifier.py +74 -0
- hydroserverpy/api/services/sta/sensor.py +116 -0
- hydroserverpy/api/services/sta/thing.py +188 -0
- hydroserverpy/api/services/sta/unit.py +82 -0
- hydroserverpy/etl/__init__.py +21 -0
- hydroserverpy/etl/extractors/__init__.py +0 -0
- hydroserverpy/etl/extractors/base.py +13 -0
- hydroserverpy/etl/extractors/ftp_extractor.py +50 -0
- hydroserverpy/etl/extractors/http_extractor.py +84 -0
- hydroserverpy/etl/extractors/local_file_extractor.py +25 -0
- hydroserverpy/etl/hydroserver_etl.py +40 -0
- hydroserverpy/etl/loaders/__init__.py +0 -0
- hydroserverpy/etl/loaders/base.py +13 -0
- hydroserverpy/etl/loaders/hydroserver_loader.py +68 -0
- hydroserverpy/etl/transformers/__init__.py +0 -0
- hydroserverpy/etl/transformers/base.py +52 -0
- hydroserverpy/etl/transformers/csv_transformer.py +88 -0
- hydroserverpy/etl/transformers/json_transformer.py +62 -0
- hydroserverpy/etl/types.py +7 -0
- hydroserverpy/etl_csv/__init__.py +0 -0
- hydroserverpy/{etl/service.py → etl_csv/hydroserver_etl_csv.py} +93 -55
- hydroserverpy/quality/service.py +84 -70
- hydroserverpy-0.5.0b1.dist-info/METADATA +19 -0
- hydroserverpy-0.5.0b1.dist-info/RECORD +59 -0
- {hydroserverpy-0.3.0.dist-info → hydroserverpy-0.5.0b1.dist-info}/WHEEL +1 -1
- hydroserverpy/core/endpoints/__init__.py +0 -9
- hydroserverpy/core/endpoints/base.py +0 -133
- hydroserverpy/core/endpoints/data_loaders.py +0 -92
- hydroserverpy/core/endpoints/data_sources.py +0 -92
- hydroserverpy/core/endpoints/datastreams.py +0 -188
- hydroserverpy/core/endpoints/observed_properties.py +0 -93
- hydroserverpy/core/endpoints/processing_levels.py +0 -93
- hydroserverpy/core/endpoints/result_qualifiers.py +0 -93
- hydroserverpy/core/endpoints/sensors.py +0 -93
- hydroserverpy/core/endpoints/things.py +0 -240
- hydroserverpy/core/endpoints/units.py +0 -93
- hydroserverpy/core/schemas/__init__.py +0 -9
- hydroserverpy/core/schemas/base.py +0 -117
- hydroserverpy/core/schemas/data_loaders.py +0 -71
- hydroserverpy/core/schemas/data_sources.py +0 -206
- hydroserverpy/core/schemas/datastreams.py +0 -299
- hydroserverpy/core/schemas/observed_properties.py +0 -35
- hydroserverpy/core/schemas/processing_levels.py +0 -27
- hydroserverpy/core/schemas/result_qualifiers.py +0 -23
- hydroserverpy/core/schemas/sensors.py +0 -53
- hydroserverpy/core/schemas/things.py +0 -309
- hydroserverpy/core/schemas/units.py +0 -30
- hydroserverpy/core/service.py +0 -186
- hydroserverpy-0.3.0.dist-info/METADATA +0 -18
- hydroserverpy-0.3.0.dist-info/RECORD +0 -36
- /hydroserverpy/{core → api}/__init__.py +0 -0
- /hydroserverpy/{etl → etl_csv}/exceptions.py +0 -0
- {hydroserverpy-0.3.0.dist-info → hydroserverpy-0.5.0b1.dist-info/licenses}/LICENSE +0 -0
- {hydroserverpy-0.3.0.dist-info → hydroserverpy-0.5.0b1.dist-info}/top_level.txt +0 -0
- {hydroserverpy-0.3.0.dist-info → hydroserverpy-0.5.0b1.dist-info}/zip-safe +0 -0
hydroserverpy/quality/service.py
CHANGED
|
@@ -20,33 +20,33 @@ from enum import Enum
|
|
|
20
20
|
class TimeUnit(Enum):
|
|
21
21
|
"""Enumeration for time units."""
|
|
22
22
|
|
|
23
|
-
SECOND =
|
|
24
|
-
MINUTE =
|
|
25
|
-
HOUR =
|
|
26
|
-
DAY =
|
|
27
|
-
WEEK =
|
|
28
|
-
MONTH =
|
|
29
|
-
YEAR =
|
|
23
|
+
SECOND = "s"
|
|
24
|
+
MINUTE = "m"
|
|
25
|
+
HOUR = "h"
|
|
26
|
+
DAY = "D"
|
|
27
|
+
WEEK = "W"
|
|
28
|
+
MONTH = "M"
|
|
29
|
+
YEAR = "Y"
|
|
30
30
|
|
|
31
31
|
|
|
32
32
|
class FilterOperation(Enum):
|
|
33
33
|
"""Enumeration for filter operations."""
|
|
34
34
|
|
|
35
|
-
LT =
|
|
36
|
-
LTE =
|
|
37
|
-
GT =
|
|
38
|
-
GTE =
|
|
39
|
-
E =
|
|
35
|
+
LT = "LT"
|
|
36
|
+
LTE = "LTE"
|
|
37
|
+
GT = "GT"
|
|
38
|
+
GTE = "GTE"
|
|
39
|
+
E = "E"
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
class Operator(Enum):
|
|
43
43
|
"""Enumeration for mathematical operations."""
|
|
44
44
|
|
|
45
|
-
MULT =
|
|
46
|
-
DIV =
|
|
47
|
-
ADD =
|
|
48
|
-
SUB =
|
|
49
|
-
ASSIGN =
|
|
45
|
+
MULT = "MULT"
|
|
46
|
+
DIV = "DIV"
|
|
47
|
+
ADD = "ADD"
|
|
48
|
+
SUB = "SUB"
|
|
49
|
+
ASSIGN = "ASSIGN"
|
|
50
50
|
|
|
51
51
|
|
|
52
52
|
class HydroServerQualityControl:
|
|
@@ -61,15 +61,23 @@ class HydroServerQualityControl:
|
|
|
61
61
|
|
|
62
62
|
datastream_id: Union[UUID, str]
|
|
63
63
|
|
|
64
|
-
def __init__(
|
|
64
|
+
def __init__(
|
|
65
|
+
self, datastream_id: Union[UUID, str], observations: pd.DataFrame
|
|
66
|
+
) -> None:
|
|
65
67
|
|
|
66
|
-
assert
|
|
67
|
-
|
|
68
|
-
|
|
68
|
+
assert (
|
|
69
|
+
"timestamp" in observations.columns
|
|
70
|
+
), "Observations must have a 'timestamp' column"
|
|
71
|
+
assert pd.api.types.is_datetime64_any_dtype(
|
|
72
|
+
observations["timestamp"]
|
|
73
|
+
), "Observations 'timestamp' column must be of datetime type"
|
|
69
74
|
|
|
70
|
-
assert
|
|
71
|
-
|
|
72
|
-
|
|
75
|
+
assert (
|
|
76
|
+
"value" in observations.columns
|
|
77
|
+
), "Observations must have a 'value' column"
|
|
78
|
+
assert pd.api.types.is_float_dtype(
|
|
79
|
+
observations["value"]
|
|
80
|
+
), "Observations 'value' column must be of float type"
|
|
73
81
|
|
|
74
82
|
self.datastream_id = str(datastream_id)
|
|
75
83
|
self._df = observations
|
|
@@ -94,7 +102,9 @@ class HydroServerQualityControl:
|
|
|
94
102
|
###################
|
|
95
103
|
|
|
96
104
|
@staticmethod
|
|
97
|
-
def _has_filter(
|
|
105
|
+
def _has_filter(
|
|
106
|
+
data_filter: Dict[str, Union[float, int]], key: FilterOperation
|
|
107
|
+
) -> bool:
|
|
98
108
|
"""
|
|
99
109
|
Checks if a given filter operation exists in the filter dictionary.
|
|
100
110
|
|
|
@@ -106,9 +116,9 @@ class HydroServerQualityControl:
|
|
|
106
116
|
:rtype: bool
|
|
107
117
|
"""
|
|
108
118
|
|
|
109
|
-
return (
|
|
110
|
-
key.value
|
|
111
|
-
|
|
119
|
+
return key.value in data_filter and (
|
|
120
|
+
isinstance(data_filter[key.value], float)
|
|
121
|
+
or isinstance(data_filter[key.value], int)
|
|
112
122
|
)
|
|
113
123
|
|
|
114
124
|
def filter(self, data_filter: Dict[str, Union[float, int]]) -> None:
|
|
@@ -122,24 +132,19 @@ class HydroServerQualityControl:
|
|
|
122
132
|
query = []
|
|
123
133
|
|
|
124
134
|
if self._has_filter(data_filter, FilterOperation.LT):
|
|
125
|
-
query.append(
|
|
126
|
-
f'`value` < {data_filter[FilterOperation.LT.value]}')
|
|
135
|
+
query.append(f"`value` < {data_filter[FilterOperation.LT.value]}")
|
|
127
136
|
|
|
128
137
|
if self._has_filter(data_filter, FilterOperation.LTE):
|
|
129
|
-
query.append(
|
|
130
|
-
f'`value` <= {data_filter[FilterOperation.LTE.value]}')
|
|
138
|
+
query.append(f"`value` <= {data_filter[FilterOperation.LTE.value]}")
|
|
131
139
|
|
|
132
140
|
if self._has_filter(data_filter, FilterOperation.GT):
|
|
133
|
-
query.append(
|
|
134
|
-
f'`value` > {data_filter[FilterOperation.GT.value]}')
|
|
141
|
+
query.append(f"`value` > {data_filter[FilterOperation.GT.value]}")
|
|
135
142
|
|
|
136
143
|
if self._has_filter(data_filter, FilterOperation.GTE):
|
|
137
|
-
query.append(
|
|
138
|
-
f'`value` >= {data_filter[FilterOperation.GTE.value]}')
|
|
144
|
+
query.append(f"`value` >= {data_filter[FilterOperation.GTE.value]}")
|
|
139
145
|
|
|
140
146
|
if self._has_filter(data_filter, FilterOperation.E):
|
|
141
|
-
query.append(
|
|
142
|
-
f'`value` == {data_filter[FilterOperation.E.value]}')
|
|
147
|
+
query.append(f"`value` == {data_filter[FilterOperation.E.value]}")
|
|
143
148
|
|
|
144
149
|
if len(query):
|
|
145
150
|
self._filtered_observations = self._df.query(" | ".join(query))
|
|
@@ -162,9 +167,13 @@ class HydroServerQualityControl:
|
|
|
162
167
|
:rtype: pd.DataFrame
|
|
163
168
|
"""
|
|
164
169
|
|
|
165
|
-
return self.observations[
|
|
170
|
+
return self.observations[
|
|
171
|
+
self._df["timestamp"].diff() > np.timedelta64(time_value, time_unit)
|
|
172
|
+
]
|
|
166
173
|
|
|
167
|
-
def fill_gap(
|
|
174
|
+
def fill_gap(
|
|
175
|
+
self, gap: Tuple[int, str], fill: Tuple[int, str], interpolate_values: bool
|
|
176
|
+
) -> pd.DataFrame:
|
|
168
177
|
"""
|
|
169
178
|
Fills identified gaps in the observations with placeholder values and optionally interpolates the values.
|
|
170
179
|
|
|
@@ -188,8 +197,8 @@ class HydroServerQualityControl:
|
|
|
188
197
|
gap_end_index = gap_row[0]
|
|
189
198
|
gap_start_index = gap_end_index - 1
|
|
190
199
|
|
|
191
|
-
gap_start_date = self._df.iloc[gap_start_index][
|
|
192
|
-
gap_end_date = self._df.iloc[gap_end_index][
|
|
200
|
+
gap_start_date = self._df.iloc[gap_start_index]["timestamp"]
|
|
201
|
+
gap_end_date = self._df.iloc[gap_end_index]["timestamp"]
|
|
193
202
|
|
|
194
203
|
start = gap_start_date + time_gap
|
|
195
204
|
end = gap_end_date
|
|
@@ -210,15 +219,15 @@ class HydroServerQualityControl:
|
|
|
210
219
|
self.interpolate(added_index)
|
|
211
220
|
|
|
212
221
|
# Return the list of points that filled the gaps
|
|
213
|
-
return pd.DataFrame(
|
|
214
|
-
points, columns=['timestamp', 'value']
|
|
215
|
-
)
|
|
222
|
+
return pd.DataFrame(points, columns=["timestamp", "value"])
|
|
216
223
|
|
|
217
224
|
######################################
|
|
218
225
|
# Data point operations
|
|
219
226
|
######################################
|
|
220
227
|
|
|
221
|
-
def add_points(
|
|
228
|
+
def add_points(
|
|
229
|
+
self, points: List[List[Union[str, float]]], index: Optional[List[int]] = None
|
|
230
|
+
) -> None:
|
|
222
231
|
"""
|
|
223
232
|
Adds new points to the observations, optionally at specified indices.
|
|
224
233
|
|
|
@@ -251,8 +260,7 @@ class HydroServerQualityControl:
|
|
|
251
260
|
df1 = self._df.iloc[:idx, :]
|
|
252
261
|
df2 = self._df.iloc[idx:, :]
|
|
253
262
|
|
|
254
|
-
points_df = pd.DataFrame(
|
|
255
|
-
val, columns=['timestamp', 'value'])
|
|
263
|
+
points_df = pd.DataFrame(val, columns=["timestamp", "value"])
|
|
256
264
|
self._df = pd.concat([df1, points_df, df2]).reset_index(drop=True)
|
|
257
265
|
|
|
258
266
|
else:
|
|
@@ -260,17 +268,18 @@ class HydroServerQualityControl:
|
|
|
260
268
|
# data in the DataFrame is pre-sorted.
|
|
261
269
|
|
|
262
270
|
# Create a new dataframe with the points
|
|
263
|
-
points_df = pd.DataFrame(
|
|
264
|
-
points, columns=['timestamp', 'value'])
|
|
271
|
+
points_df = pd.DataFrame(points, columns=["timestamp", "value"])
|
|
265
272
|
|
|
266
273
|
# Concatenate both dataframes. New rows will be at the end.
|
|
267
274
|
self._df = pd.concat([self._df, points_df])
|
|
268
275
|
|
|
269
276
|
# Sort and reset index
|
|
270
|
-
self._df = self._df.sort_values(
|
|
277
|
+
self._df = self._df.sort_values("timestamp")
|
|
271
278
|
self._df.reset_index(drop=True, inplace=True)
|
|
272
279
|
|
|
273
|
-
def change_values(
|
|
280
|
+
def change_values(
|
|
281
|
+
self, index_list: List[int], operator: str, value: Union[int, float]
|
|
282
|
+
) -> None:
|
|
274
283
|
"""
|
|
275
284
|
Changes the values of observations based on the specified operator and value.
|
|
276
285
|
|
|
@@ -299,7 +308,9 @@ class HydroServerQualityControl:
|
|
|
299
308
|
else:
|
|
300
309
|
return x
|
|
301
310
|
|
|
302
|
-
self._df.loc[index_list,
|
|
311
|
+
self._df.loc[index_list, "value"] = self._df.loc[index_list, "value"].apply(
|
|
312
|
+
operation
|
|
313
|
+
)
|
|
303
314
|
|
|
304
315
|
def delete_points(self, index_list: List[int]) -> None:
|
|
305
316
|
"""
|
|
@@ -312,7 +323,9 @@ class HydroServerQualityControl:
|
|
|
312
323
|
self._df.drop(index=index_list, inplace=True)
|
|
313
324
|
self._df.reset_index(drop=True, inplace=True)
|
|
314
325
|
|
|
315
|
-
def shift_points(
|
|
326
|
+
def shift_points(
|
|
327
|
+
self, index_list: List[int], time_value: int, time_unit: str
|
|
328
|
+
) -> None:
|
|
316
329
|
"""
|
|
317
330
|
Shifts the timestamps of the observations at the specified indices by a given time value and unit.
|
|
318
331
|
|
|
@@ -328,8 +341,10 @@ class HydroServerQualityControl:
|
|
|
328
341
|
condition = self._df.index.isin(index_list)
|
|
329
342
|
|
|
330
343
|
# Apply the shift
|
|
331
|
-
self._df.loc[condition,
|
|
332
|
-
|
|
344
|
+
self._df.loc[condition, "timestamp"] = (
|
|
345
|
+
self._df.loc[condition, "timestamp"] + shift_value
|
|
346
|
+
)
|
|
347
|
+
self._df = self._df.sort_values("timestamp")
|
|
333
348
|
self._df.reset_index(drop=True, inplace=True)
|
|
334
349
|
|
|
335
350
|
def interpolate(self, index_list: List[int]) -> None:
|
|
@@ -341,8 +356,8 @@ class HydroServerQualityControl:
|
|
|
341
356
|
"""
|
|
342
357
|
|
|
343
358
|
condition = self._df.index.isin(index_list)
|
|
344
|
-
self._df[
|
|
345
|
-
self._df[
|
|
359
|
+
self._df["value"].mask(condition, inplace=True)
|
|
360
|
+
self._df["value"].interpolate(method="linear", inplace=True)
|
|
346
361
|
|
|
347
362
|
def drift_correction(self, start: int, end: int, gap_width: float) -> pd.DataFrame:
|
|
348
363
|
"""
|
|
@@ -360,32 +375,31 @@ class HydroServerQualityControl:
|
|
|
360
375
|
|
|
361
376
|
# validate range
|
|
362
377
|
if start >= end:
|
|
363
|
-
print(
|
|
378
|
+
print("Start and end index cannot overlap")
|
|
364
379
|
return self._df
|
|
365
380
|
elif end > len(self._df) - 1:
|
|
366
|
-
print(
|
|
381
|
+
print("End index out of range")
|
|
367
382
|
return self._df
|
|
368
383
|
elif start < 0:
|
|
369
|
-
print(
|
|
384
|
+
print("Start index must be greater than or equal to 0")
|
|
370
385
|
return self._df
|
|
371
386
|
|
|
372
|
-
points = self._df.iloc[start:end + 1]
|
|
373
|
-
start_date = points.iloc[0][
|
|
374
|
-
end_date = points.iloc[-1][
|
|
387
|
+
points = self._df.iloc[start : end + 1]
|
|
388
|
+
start_date = points.iloc[0]["timestamp"]
|
|
389
|
+
end_date = points.iloc[-1]["timestamp"]
|
|
375
390
|
|
|
376
391
|
x_l = (end_date - start_date).total_seconds()
|
|
377
392
|
ndv = -9999
|
|
378
393
|
# y_n = y_0 + G(x_i / x_l)
|
|
379
394
|
|
|
380
395
|
def f(row):
|
|
381
|
-
if row[
|
|
382
|
-
return (
|
|
383
|
-
row[
|
|
384
|
-
(gap_width * ((row['timestamp'] - start_date).total_seconds() / x_l))
|
|
396
|
+
if row["value"] != ndv:
|
|
397
|
+
return row["value"] + (
|
|
398
|
+
gap_width * ((row["timestamp"] - start_date).total_seconds() / x_l)
|
|
385
399
|
)
|
|
386
400
|
else:
|
|
387
|
-
return row[
|
|
401
|
+
return row["value"]
|
|
388
402
|
|
|
389
|
-
self._df.loc[points.index,
|
|
403
|
+
self._df.loc[points.index, "value"] = points.apply(f, axis=1)
|
|
390
404
|
|
|
391
405
|
return self._df
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hydroserverpy
|
|
3
|
+
Version: 0.5.0b1
|
|
4
|
+
Requires-Python: <4,>=3.9
|
|
5
|
+
License-File: LICENSE
|
|
6
|
+
Requires-Dist: requests>=2
|
|
7
|
+
Requires-Dist: pydantic>=2.6
|
|
8
|
+
Requires-Dist: pydantic[email]>=2.6
|
|
9
|
+
Requires-Dist: pandas>=2.2
|
|
10
|
+
Requires-Dist: numpy>=2.0
|
|
11
|
+
Requires-Dist: pyyaml>=5
|
|
12
|
+
Requires-Dist: simplejson>=3
|
|
13
|
+
Requires-Dist: crontab>=1
|
|
14
|
+
Requires-Dist: python-dateutil>=2.8.2
|
|
15
|
+
Requires-Dist: croniter>=2.0.1
|
|
16
|
+
Requires-Dist: jmespath>=1.0.1
|
|
17
|
+
Provides-Extra: docs
|
|
18
|
+
Requires-Dist: sphinx_autodoc_typehints; extra == "docs"
|
|
19
|
+
Dynamic: license-file
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
hydroserverpy/__init__.py,sha256=FgaGFyhCjwmpJYEKNzOZxvfRx2neWMaOybj1z02_VSE,218
|
|
2
|
+
hydroserverpy/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
hydroserverpy/api/http.py,sha256=be2L_0t9RtpjRlS4YYlJ0T09wyvUYw2-fg5eXBQwNXA,632
|
|
4
|
+
hydroserverpy/api/main.py,sha256=2IVNMsf5h1UWfDZSxJn1wewOmN2D2kIMq7iqwiBqCNY,4216
|
|
5
|
+
hydroserverpy/api/models/__init__.py,sha256=YwO9ixElkQp3Nwi6gb0XPj4H32aNmUOFm0oXc20E_F8,550
|
|
6
|
+
hydroserverpy/api/models/base.py,sha256=dc2tfMSgizymxAAOVURfy7Jzeh6xIiiq7hfWZI7l1_Q,2280
|
|
7
|
+
hydroserverpy/api/models/etl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
hydroserverpy/api/models/iam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
hydroserverpy/api/models/iam/account.py,sha256=7COk_CPYFlthg1uFWTBlJESfnuqMW90TSjZoIcBb-_8,439
|
|
10
|
+
hydroserverpy/api/models/iam/collaborator.py,sha256=jp661DKDCwk8c8HFPAV-YVhEc80F5eGDKaSHmH62Q8Q,1007
|
|
11
|
+
hydroserverpy/api/models/iam/role.py,sha256=8FVTj_1QwtPF9tk7baliMVg000kjc5N8oP6eYo8vTDY,275
|
|
12
|
+
hydroserverpy/api/models/iam/workspace.py,sha256=IonbIWMGAuxPVum0i0kEy0EJq7l79eH40-2P_SZAJHs,6194
|
|
13
|
+
hydroserverpy/api/models/sta/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
+
hydroserverpy/api/models/sta/datastream.py,sha256=SfBixRPsR0j6g1hNnjg0fzlCK4AfQxo8voXVyU0tS-o,10744
|
|
15
|
+
hydroserverpy/api/models/sta/observed_property.py,sha256=ThTg8aPMHPxbk9Hzpxw3AwM16gE1xvYpRK8UkiOdGeA,2180
|
|
16
|
+
hydroserverpy/api/models/sta/processing_level.py,sha256=y5_0wX7QGXgswvukXJtbpOiTCZ9pI8E08DXaTSUHakg,1470
|
|
17
|
+
hydroserverpy/api/models/sta/result_qualifier.py,sha256=IJcY04KjP9e2D-jPzUJjH2PC-JvDNCjbi5LKkTVSwgw,1416
|
|
18
|
+
hydroserverpy/api/models/sta/sensor.py,sha256=TD9R1Uwcu1t9tRQBfk0crsSJmV5UN_9kH9Ye9b7lDJc,3055
|
|
19
|
+
hydroserverpy/api/models/sta/thing.py,sha256=o4Xn_Luy2IEOCBjXTbek7GvPoXZyKA0dhfzoFM6nfTs,6357
|
|
20
|
+
hydroserverpy/api/models/sta/unit.py,sha256=Pbxxp9hZErsrYImIb8-1HVnZAsJopE3US_AplSQWOJQ,1398
|
|
21
|
+
hydroserverpy/api/services/__init__.py,sha256=PhqbFlKfZ8a3WJidJUwa0gJ3OADcivepEEqfbQXb1dc,371
|
|
22
|
+
hydroserverpy/api/services/base.py,sha256=Y2aJDMcVR7qxTgZQh1s8UYyRsoogjDY6TkCxvV9jfxA,3025
|
|
23
|
+
hydroserverpy/api/services/etl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
|
+
hydroserverpy/api/services/iam/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
25
|
+
hydroserverpy/api/services/iam/workspace.py,sha256=jJiqkMxFEp9REjR4LXyVp2o45CGBrrEadGelPPCuRJs,4547
|
|
26
|
+
hydroserverpy/api/services/sta/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
27
|
+
hydroserverpy/api/services/sta/datastream.py,sha256=_m-xFom3z8wo5-1_q8NjWUpcw36wYv1brIG7xeGGadk,12402
|
|
28
|
+
hydroserverpy/api/services/sta/observed_property.py,sha256=7V2PjsQqEGPEWk1n_o3MS0UM_tKQdTCEjDYM2XKMk0I,2894
|
|
29
|
+
hydroserverpy/api/services/sta/processing_level.py,sha256=Oupfeww2XgT83AwR5Spt91VjZK6MG0XIl11Et9fRjA0,2255
|
|
30
|
+
hydroserverpy/api/services/sta/result_qualifier.py,sha256=XG5Ng3xdFT-l3Ktkuq23Cty1RfmepBO7EQ9gPzidZuA,2069
|
|
31
|
+
hydroserverpy/api/services/sta/sensor.py,sha256=SbDhLjlOaM2ypLDfXmQVinj7eHHJ_fHxjTD68dM2pQI,3473
|
|
32
|
+
hydroserverpy/api/services/sta/thing.py,sha256=QL7IBwHHIgDFBpXnQF-LOUpxiRlm_HFWB1qqe7Iqq9s,5972
|
|
33
|
+
hydroserverpy/api/services/sta/unit.py,sha256=ksO-3g___9pPNBNbgM0jyDf1NeBqX79fjeJjCshrftY,2138
|
|
34
|
+
hydroserverpy/etl/__init__.py,sha256=qK2m4LZl8czR3VE8SxrlipSy5tLGLNB60lxD7dD0GjU,659
|
|
35
|
+
hydroserverpy/etl/hydroserver_etl.py,sha256=FSdvM3T7QHEWWulWRT8t-FMHSxAGB4GvleUXtSk5IWc,1507
|
|
36
|
+
hydroserverpy/etl/types.py,sha256=4PY3CM-uoXIsf2lhcqtLC6HaRGXe7HKGDU22R8-H35c,135
|
|
37
|
+
hydroserverpy/etl/extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
38
|
+
hydroserverpy/etl/extractors/base.py,sha256=GZKJfAhfJedRcNagnoqUiDZn286r-JzM7dW_F1dWsfY,275
|
|
39
|
+
hydroserverpy/etl/extractors/ftp_extractor.py,sha256=5LwvHuvLk6LwRSVyE9EkV3DPgVlAvRrOBpl1a8B7dLg,1387
|
|
40
|
+
hydroserverpy/etl/extractors/http_extractor.py,sha256=-duQwnsFBk4NQS2qhO55evcCUOnrBe3JX_LU9RyysX4,2709
|
|
41
|
+
hydroserverpy/etl/extractors/local_file_extractor.py,sha256=T_Y9NTO0cC5L9mDPbIG6wYlXDQoatg8MobP97liFl4U,692
|
|
42
|
+
hydroserverpy/etl/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
43
|
+
hydroserverpy/etl/loaders/base.py,sha256=DrA9u7SNBxkPKqaszlP368yNbxihdqIGzP8rA6NAp6U,295
|
|
44
|
+
hydroserverpy/etl/loaders/hydroserver_loader.py,sha256=bl4Z5TkXgJyKEHuPdY7LSrKMKOoZW_EDFl8dIRwDkv4,2549
|
|
45
|
+
hydroserverpy/etl/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
46
|
+
hydroserverpy/etl/transformers/base.py,sha256=237oVBhS3HQ3fcE4bZT5U1437WzV2x5kOFC229DY53M,1741
|
|
47
|
+
hydroserverpy/etl/transformers/csv_transformer.py,sha256=9DKSO4NfUUDlr_c6UnH4AU3-7LxwSSeuQdou0iiCjdM,3238
|
|
48
|
+
hydroserverpy/etl/transformers/json_transformer.py,sha256=ity0MXcYjEnlun4Y6cVSrnjrglKrK4JOXXHxWHIHN2A,2323
|
|
49
|
+
hydroserverpy/etl_csv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
50
|
+
hydroserverpy/etl_csv/exceptions.py,sha256=0UY8YUlNepG0y6FfH36hJyR1bOhwYHSZIdUSSMTg7GA,314
|
|
51
|
+
hydroserverpy/etl_csv/hydroserver_etl_csv.py,sha256=lpZmlh4BAsljLLCEj4UiDtty3m28MJbnSiOE71gDrCQ,13260
|
|
52
|
+
hydroserverpy/quality/__init__.py,sha256=GGBMkFSXciJLYrbV-NraFrj_mXWCy_GTcy9KKrKXU4c,84
|
|
53
|
+
hydroserverpy/quality/service.py,sha256=U02UfLKVmFvr5ySiH0n0JYzUIabq5uprrHIiwcqBlqY,13879
|
|
54
|
+
hydroserverpy-0.5.0b1.dist-info/licenses/LICENSE,sha256=xVqFxDw3QOEJukakL7gQCqIMTQ1dlSCTo6Oc1otNW80,1508
|
|
55
|
+
hydroserverpy-0.5.0b1.dist-info/METADATA,sha256=ZhKOorHNQG_J18X3L2HnUqBp5gFZBx8QuwwlmPpc0FM,532
|
|
56
|
+
hydroserverpy-0.5.0b1.dist-info/WHEEL,sha256=L0N565qmK-3nM2eBoMNFszYJ_MTx03_tQ0CQu1bHLYo,91
|
|
57
|
+
hydroserverpy-0.5.0b1.dist-info/top_level.txt,sha256=Zf37hrncXLOYvXhgCrf5mZdeq81G9fShdE2LfYbtb7w,14
|
|
58
|
+
hydroserverpy-0.5.0b1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
59
|
+
hydroserverpy-0.5.0b1.dist-info/RECORD,,
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
from .data_loaders import DataLoaderEndpoint
|
|
2
|
-
from .data_sources import DataSourceEndpoint
|
|
3
|
-
from .datastreams import DatastreamEndpoint
|
|
4
|
-
from .observed_properties import ObservedPropertyEndpoint
|
|
5
|
-
from .processing_levels import ProcessingLevelEndpoint
|
|
6
|
-
from .result_qualifiers import ResultQualifierEndpoint
|
|
7
|
-
from .sensors import SensorEndpoint
|
|
8
|
-
from .things import ThingEndpoint
|
|
9
|
-
from .units import UnitEndpoint
|
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
from uuid import UUID
|
|
3
|
-
from typing import TYPE_CHECKING, Type, Union, List, TypeVar, Optional
|
|
4
|
-
|
|
5
|
-
if TYPE_CHECKING:
|
|
6
|
-
from hydroserverpy import HydroServer
|
|
7
|
-
from hydroserverpy.core.schemas.base import HydroServerCoreModel
|
|
8
|
-
|
|
9
|
-
HydroServerModelType = TypeVar('HydroServerModelType', bound=HydroServerCoreModel)
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
def expand_docstring(model: Optional[Type['HydroServerCoreModel']] = None, include_uid: bool = False):
|
|
13
|
-
def decorator(func):
|
|
14
|
-
docstring = func.__doc__
|
|
15
|
-
if model is not None or include_uid is True:
|
|
16
|
-
docstring += '\n'
|
|
17
|
-
if include_uid is True:
|
|
18
|
-
docstring += f':param uid: The entity ID.\n'
|
|
19
|
-
docstring += f':type uid: Union[UUID, str]\n'
|
|
20
|
-
if model is not None:
|
|
21
|
-
for field_name, field in model.model_fields.items():
|
|
22
|
-
docstring += f':param {field_name}: {field.description}\n'
|
|
23
|
-
docstring += f':type {field_name}: {getattr(field.annotation, "__name__", field.annotation)}\n'
|
|
24
|
-
func.__doc__ = docstring
|
|
25
|
-
return func
|
|
26
|
-
return decorator
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
class HydroServerEndpoint:
|
|
30
|
-
"""
|
|
31
|
-
A base class for interacting with specific API endpoints within a HydroServer service.
|
|
32
|
-
|
|
33
|
-
:ivar _model: The model class associated with this endpoint.
|
|
34
|
-
:ivar _api_route: The base route of the API.
|
|
35
|
-
:ivar _endpoint_route: The specific route of the endpoint.
|
|
36
|
-
"""
|
|
37
|
-
|
|
38
|
-
_model: Type['HydroServerCoreModel']
|
|
39
|
-
_api_route: str
|
|
40
|
-
_endpoint_route: str
|
|
41
|
-
|
|
42
|
-
def __init__(self, service: 'HydroServer') -> None:
|
|
43
|
-
"""
|
|
44
|
-
Initialize the HydroServerEndpoint.
|
|
45
|
-
|
|
46
|
-
:param service: The HydroServer service instance to use for requests.
|
|
47
|
-
:type service: HydroServer
|
|
48
|
-
"""
|
|
49
|
-
|
|
50
|
-
self._service = service
|
|
51
|
-
|
|
52
|
-
def _get(self, uid: Optional[Union[UUID, str]] = None, params: dict = None) -> Union[
|
|
53
|
-
List['HydroServerModelType'], 'HydroServerModelType'
|
|
54
|
-
]:
|
|
55
|
-
"""
|
|
56
|
-
Fetch an entity collection or single entity from a HydroServer endpoint.
|
|
57
|
-
|
|
58
|
-
:param uid: The unique identifier of the entity to retrieve.
|
|
59
|
-
:type uid: Optional[Union[UUID, str]]
|
|
60
|
-
:returns: A model instance representing the entity.
|
|
61
|
-
:rtype: HydroServerCoreModel
|
|
62
|
-
"""
|
|
63
|
-
|
|
64
|
-
if params is None:
|
|
65
|
-
params = {}
|
|
66
|
-
|
|
67
|
-
path = f'{self._api_route}/data/{self._endpoint_route}{"/" + str(uid) if uid else ""}'
|
|
68
|
-
response = getattr(self._service, '_request')('get', path, params=params)
|
|
69
|
-
|
|
70
|
-
if uid:
|
|
71
|
-
entity = json.loads(response.content)
|
|
72
|
-
result = self._model(_endpoint=self, _uid=UUID(str(entity.pop('id'))), **entity)
|
|
73
|
-
else:
|
|
74
|
-
result = [
|
|
75
|
-
self._model(_endpoint=self, _uid=UUID(str(entity.pop('id'))), **entity)
|
|
76
|
-
for entity in json.loads(response.content)
|
|
77
|
-
]
|
|
78
|
-
|
|
79
|
-
return result
|
|
80
|
-
|
|
81
|
-
def _post(self, **kwargs) -> 'HydroServerModelType':
|
|
82
|
-
"""
|
|
83
|
-
Create a new entity using the endpoint.
|
|
84
|
-
|
|
85
|
-
:param kwargs: The attributes to set on the new entity.
|
|
86
|
-
:returns: A model instance representing the newly created entity.
|
|
87
|
-
:rtype: HydroServerModelType
|
|
88
|
-
"""
|
|
89
|
-
|
|
90
|
-
response = getattr(self._service, '_request')(
|
|
91
|
-
'post', f'{self._api_route}/data/{self._endpoint_route}',
|
|
92
|
-
headers={'Content-type': 'application/json'},
|
|
93
|
-
data=self._model(_endpoint=self, **kwargs).json(exclude_unset=True, by_alias=True),
|
|
94
|
-
)
|
|
95
|
-
entity = json.loads(response.content)
|
|
96
|
-
|
|
97
|
-
return self._model(_endpoint=self, _uid=UUID(str(entity.pop('id'))), **entity)
|
|
98
|
-
|
|
99
|
-
def _patch(self, uid: Union[UUID, str], **kwargs) -> 'HydroServerModelType':
|
|
100
|
-
"""
|
|
101
|
-
Update an existing entity in the endpoint.
|
|
102
|
-
|
|
103
|
-
:param uid: The unique identifier of the entity to update.
|
|
104
|
-
:type uid: Union[UUID, str]
|
|
105
|
-
:param kwargs: The attributes to update on the entity.
|
|
106
|
-
:returns: A model instance representing the updated entity.
|
|
107
|
-
:rtype: HydroServerModelType
|
|
108
|
-
"""
|
|
109
|
-
|
|
110
|
-
response = getattr(self._service, '_request')(
|
|
111
|
-
'patch', f'{self._api_route}/data/{self._endpoint_route}/{str(uid)}',
|
|
112
|
-
headers={'Content-type': 'application/json'},
|
|
113
|
-
data=json.dumps({
|
|
114
|
-
self._model.model_fields[key].serialization_alias: value
|
|
115
|
-
for key, value in kwargs.items()
|
|
116
|
-
}, default=str)
|
|
117
|
-
)
|
|
118
|
-
entity = json.loads(response.content)
|
|
119
|
-
|
|
120
|
-
return self._model(_endpoint=self, _uid=UUID(str(entity.pop('id'))), **entity)
|
|
121
|
-
|
|
122
|
-
def _delete(self, uid: Union[UUID, str]) -> None:
|
|
123
|
-
"""
|
|
124
|
-
Delete an entity from the endpoint by its unique identifier.
|
|
125
|
-
|
|
126
|
-
:param uid: The unique identifier of the entity to delete.
|
|
127
|
-
:type uid: Union[UUID, str]
|
|
128
|
-
:returns: None
|
|
129
|
-
"""
|
|
130
|
-
|
|
131
|
-
getattr(self._service, '_request')(
|
|
132
|
-
'delete', f'{self._api_route}/data/{self._endpoint_route}/{str(uid)}',
|
|
133
|
-
)
|
|
@@ -1,92 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
from typing import Union, List, TYPE_CHECKING
|
|
3
|
-
from uuid import UUID
|
|
4
|
-
from hydroserverpy.core.endpoints.base import HydroServerEndpoint, expand_docstring
|
|
5
|
-
from hydroserverpy.core.endpoints.data_sources import DataSourceEndpoint
|
|
6
|
-
from hydroserverpy.core.schemas import DataLoader, DataSource
|
|
7
|
-
|
|
8
|
-
if TYPE_CHECKING:
|
|
9
|
-
from hydroserverpy.core.service import HydroServer
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
class DataLoaderEndpoint(HydroServerEndpoint):
|
|
13
|
-
"""
|
|
14
|
-
An endpoint for interacting with DataLoader entities in the HydroServer service.
|
|
15
|
-
|
|
16
|
-
:ivar _model: The model class associated with this endpoint, set to `DataLoader`.
|
|
17
|
-
:ivar _api_route: The base route of the API, derived from the service.
|
|
18
|
-
:ivar _endpoint_route: The specific route of the endpoint, set to `'data-loaders'`.
|
|
19
|
-
"""
|
|
20
|
-
|
|
21
|
-
def __init__(self, service: 'HydroServer') -> None:
|
|
22
|
-
"""
|
|
23
|
-
Initialize the DataLoaderEndpoint.
|
|
24
|
-
|
|
25
|
-
:param service: The HydroServer service instance to use for requests.
|
|
26
|
-
:type service: HydroServer
|
|
27
|
-
"""
|
|
28
|
-
|
|
29
|
-
super().__init__(service)
|
|
30
|
-
self._model = DataLoader
|
|
31
|
-
self._api_route = self._service.api_route
|
|
32
|
-
self._endpoint_route = 'data-loaders'
|
|
33
|
-
|
|
34
|
-
def list(self) -> List[DataLoader]:
|
|
35
|
-
"""
|
|
36
|
-
Retrieve a collection of data loaders owned by the logged-in user.
|
|
37
|
-
"""
|
|
38
|
-
|
|
39
|
-
return super()._get()
|
|
40
|
-
|
|
41
|
-
@expand_docstring(include_uid=True)
|
|
42
|
-
def get(self, uid: Union[UUID, str]) -> DataLoader:
|
|
43
|
-
"""
|
|
44
|
-
Retrieve a data loader owned by the logged-in user.
|
|
45
|
-
"""
|
|
46
|
-
|
|
47
|
-
return super()._get(uid)
|
|
48
|
-
|
|
49
|
-
@expand_docstring(model=DataLoader)
|
|
50
|
-
def create(self, **kwargs) -> DataLoader:
|
|
51
|
-
"""
|
|
52
|
-
Create a new data loader in HydroServer.
|
|
53
|
-
"""
|
|
54
|
-
|
|
55
|
-
return super()._post(**kwargs)
|
|
56
|
-
|
|
57
|
-
@expand_docstring(model=DataLoader, include_uid=True)
|
|
58
|
-
def update(self, uid: Union[UUID, str], **kwargs) -> DataLoader:
|
|
59
|
-
"""
|
|
60
|
-
Update an existing data loader in HydroServer.
|
|
61
|
-
"""
|
|
62
|
-
|
|
63
|
-
return super()._patch(uid=uid, **kwargs)
|
|
64
|
-
|
|
65
|
-
@expand_docstring(include_uid=True)
|
|
66
|
-
def delete(self, uid: Union[UUID, str]) -> None:
|
|
67
|
-
"""
|
|
68
|
-
Delete an existing data loader in HydroServer.
|
|
69
|
-
"""
|
|
70
|
-
|
|
71
|
-
super()._delete(uid=uid)
|
|
72
|
-
|
|
73
|
-
def list_data_sources(self, uid: Union[UUID, str]) -> List[DataSource]:
|
|
74
|
-
"""
|
|
75
|
-
Retrieve a list of data source entities associated with a specific data loader.
|
|
76
|
-
|
|
77
|
-
:param uid: The unique identifier of the data loader.
|
|
78
|
-
:type uid: Union[UUID, str]
|
|
79
|
-
:returns: A list of data sour instances associated with the data loader.
|
|
80
|
-
:rtype: List[DataSource]
|
|
81
|
-
"""
|
|
82
|
-
|
|
83
|
-
response = getattr(self._service, '_request')(
|
|
84
|
-
'get', f'{self._api_route}/data/{self._endpoint_route}/{str(uid)}/data-sources'
|
|
85
|
-
)
|
|
86
|
-
|
|
87
|
-
endpoint = DataSourceEndpoint(self._service)
|
|
88
|
-
|
|
89
|
-
return [
|
|
90
|
-
DataSource(_endpoint=endpoint, _uid=UUID(str(entity.pop('id'))), **entity)
|
|
91
|
-
for entity in json.loads(response.content)
|
|
92
|
-
]
|