api-24sea 2.2.0__tar.gz → 2.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {api_24sea-2.2.0 → api_24sea-2.2.1}/PKG-INFO +8 -2
- {api_24sea-2.2.0 → api_24sea-2.2.1}/README.md +6 -0
- {api_24sea-2.2.0 → api_24sea-2.2.1}/api_24sea/datasignals/__init__.py +4 -0
- {api_24sea-2.2.0 → api_24sea-2.2.1}/api_24sea/datasignals/core.py +109 -35
- {api_24sea-2.2.0 → api_24sea-2.2.1}/api_24sea/datasignals/schemas.py +17 -2
- {api_24sea-2.2.0 → api_24sea-2.2.1}/api_24sea/utils.py +235 -34
- {api_24sea-2.2.0 → api_24sea-2.2.1}/api_24sea/version.py +1 -1
- {api_24sea-2.2.0 → api_24sea-2.2.1}/pyproject.toml +1 -1
- {api_24sea-2.2.0 → api_24sea-2.2.1}/LICENSE +0 -0
- {api_24sea-2.2.0 → api_24sea-2.2.1}/api_24sea/__init__.py +0 -0
- {api_24sea-2.2.0 → api_24sea-2.2.1}/api_24sea/abc.py +0 -0
- {api_24sea-2.2.0 → api_24sea-2.2.1}/api_24sea/datasignals/fatigue.py +0 -0
- {api_24sea-2.2.0 → api_24sea-2.2.1}/api_24sea/exceptions.py +0 -0
- {api_24sea-2.2.0 → api_24sea-2.2.1}/api_24sea/singleton.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: api-24sea
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.1
|
|
4
4
|
Summary: Facilitate the users' interaction with the 24SEA API (https://api/24sea/eu) by providing pandas interfaces to the API endpoints.
|
|
5
5
|
Author-email: Pietro D'Antuono <pietro.dantuono@24sea.eu>
|
|
6
6
|
Maintainer-email: Pietro D'Antuono <pietro.dantuono@24sea.eu>
|
|
@@ -27,7 +27,7 @@ Requires-Dist: pydantic
|
|
|
27
27
|
Requires-Dist: shorthand_datetime>=0.2.1,<1.0.0
|
|
28
28
|
Requires-Dist: httpx>=0.28.0,<1.0.0
|
|
29
29
|
Requires-Dist: tqdm>=4.65.0,<5.0.0
|
|
30
|
-
Requires-Dist: api-24sea-ai>=0.
|
|
30
|
+
Requires-Dist: api-24sea-ai>=0.2.0 ; extra == "ai"
|
|
31
31
|
Requires-Dist: py-fatigue>=2.0.0 ; extra == "fatigue" and ( python_version>='3.10')
|
|
32
32
|
Requires-Dist: py-fatigue>=1.0.0,<2.0.0 ; extra == "fatigue" and ( python_version<'3.10')
|
|
33
33
|
Requires-Dist: swifter>=1.4.0 ; extra == "fatigue"
|
|
@@ -219,6 +219,9 @@ both, the metrics, and timestamps.
|
|
|
219
219
|
- Timestamps: Timezone-aware datetime, strings in ISO 8601 format, or
|
|
220
220
|
shorthand strings compatible with the [shorthand_datetime
|
|
221
221
|
package](https://pypi.org/project/shorthand-datetime/).
|
|
222
|
+
- `method`: Optional HTTP method for the backend request. Default is
|
|
223
|
+
`GET`. Supported values are `GET`, `POST`, `PUT`, `PATCH`, and
|
|
224
|
+
`DELETE`.
|
|
222
225
|
|
|
223
226
|
``` python
|
|
224
227
|
# %%
|
|
@@ -395,6 +398,9 @@ Therefore, the following combinations are possible:
|
|
|
395
398
|
- Star schema as `dict[str, dict[str, Any]]`:
|
|
396
399
|
`outer_join_on_timestamp` is irrelevant `as_star_schema=True`, and
|
|
397
400
|
`as_dict=True`.
|
|
401
|
+
- `method`: Optional HTTP method for the backend request. Default is
|
|
402
|
+
`GET`. Supported values are `GET`, `POST`, `PUT`, `PATCH`, and
|
|
403
|
+
`DELETE`.
|
|
398
404
|
|
|
399
405
|
``` python
|
|
400
406
|
# %%
|
|
@@ -179,6 +179,9 @@ both, the metrics, and timestamps.
|
|
|
179
179
|
- Timestamps: Timezone-aware datetime, strings in ISO 8601 format, or
|
|
180
180
|
shorthand strings compatible with the [shorthand_datetime
|
|
181
181
|
package](https://pypi.org/project/shorthand-datetime/).
|
|
182
|
+
- `method`: Optional HTTP method for the backend request. Default is
|
|
183
|
+
`GET`. Supported values are `GET`, `POST`, `PUT`, `PATCH`, and
|
|
184
|
+
`DELETE`.
|
|
182
185
|
|
|
183
186
|
``` python
|
|
184
187
|
# %%
|
|
@@ -355,6 +358,9 @@ Therefore, the following combinations are possible:
|
|
|
355
358
|
- Star schema as `dict[str, dict[str, Any]]`:
|
|
356
359
|
`outer_join_on_timestamp` is irrelevant `as_star_schema=True`, and
|
|
357
360
|
`as_dict=True`.
|
|
361
|
+
- `method`: Optional HTTP method for the backend request. Default is
|
|
362
|
+
`GET`. Supported values are `GET`, `POST`, `PUT`, `PATCH`, and
|
|
363
|
+
`DELETE`.
|
|
358
364
|
|
|
359
365
|
``` python
|
|
360
366
|
# %%
|
|
@@ -144,6 +144,7 @@ class DataSignals:
|
|
|
144
144
|
end_timestamp: Union[str, datetime.datetime],
|
|
145
145
|
location: Optional[Union[List[str], str]] = None,
|
|
146
146
|
force_cache_miss: bool = False,
|
|
147
|
+
method: str = "GET",
|
|
147
148
|
):
|
|
148
149
|
"""Get the data signals from the 24SEA API.
|
|
149
150
|
|
|
@@ -184,6 +185,7 @@ class DataSignals:
|
|
|
184
185
|
as_dict=False,
|
|
185
186
|
location=location,
|
|
186
187
|
force_cache_miss=force_cache_miss,
|
|
188
|
+
method=method,
|
|
187
189
|
)
|
|
188
190
|
|
|
189
191
|
def as_dict(
|
|
@@ -310,6 +312,7 @@ class DataSignals:
|
|
|
310
312
|
end_timestamp: Union[str, datetime.datetime],
|
|
311
313
|
location: Optional[Union[List, str]] = None,
|
|
312
314
|
force_cache_miss: bool = False,
|
|
315
|
+
method: str = "GET",
|
|
313
316
|
) -> Optional[
|
|
314
317
|
Union[
|
|
315
318
|
pd.DataFrame,
|
|
@@ -354,4 +357,5 @@ class DataSignals:
|
|
|
354
357
|
end_timestamp=end_timestamp,
|
|
355
358
|
location=location,
|
|
356
359
|
force_cache_miss=force_cache_miss,
|
|
360
|
+
method=method,
|
|
357
361
|
)
|
|
@@ -7,7 +7,6 @@ import logging
|
|
|
7
7
|
from typing import Any, Dict, List, Optional, Union
|
|
8
8
|
from warnings import simplefilter
|
|
9
9
|
|
|
10
|
-
import httpx
|
|
11
10
|
import pandas as pd
|
|
12
11
|
from pandas import __version__ as pd_version
|
|
13
12
|
|
|
@@ -86,14 +85,21 @@ class API(AuthABC):
|
|
|
86
85
|
permissions_overview: Optional[pd.DataFrame] = None,
|
|
87
86
|
):
|
|
88
87
|
"""Authenticate with username/password"""
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
self._metrics_overview = (
|
|
92
|
-
permissions_overview
|
|
93
|
-
if permissions_overview is not None
|
|
94
|
-
else self._metrics_overview
|
|
88
|
+
had_credentials = (
|
|
89
|
+
self._username is not None or self._password is not None
|
|
95
90
|
)
|
|
96
|
-
|
|
91
|
+
credentials_changed = (username, password) != (
|
|
92
|
+
self._username,
|
|
93
|
+
self._password,
|
|
94
|
+
)
|
|
95
|
+
self._set_auth(username, password)
|
|
96
|
+
if permissions_overview is not None:
|
|
97
|
+
self._permissions_overview = permissions_overview
|
|
98
|
+
self._metrics_overview = permissions_overview
|
|
99
|
+
elif had_credentials and credentials_changed:
|
|
100
|
+
self._permissions_overview = None
|
|
101
|
+
self._metrics_overview = None
|
|
102
|
+
self._selected_metrics = None
|
|
97
103
|
|
|
98
104
|
r_profile = U.handle_request(
|
|
99
105
|
f"{self.base_url}profile/",
|
|
@@ -104,7 +110,7 @@ class API(AuthABC):
|
|
|
104
110
|
if (
|
|
105
111
|
r_profile.status_code == 200 or self._metrics_overview is not None
|
|
106
112
|
): # noqa: E501
|
|
107
|
-
self._authenticated = True
|
|
113
|
+
self._authenticated: bool = True
|
|
108
114
|
# fmt: off
|
|
109
115
|
logging.info(f"\033[32;1m{username} has access to "
|
|
110
116
|
f"\033[4m{U.BASE_URL}.\033[0m")
|
|
@@ -249,6 +255,7 @@ class API(AuthABC):
|
|
|
249
255
|
threads: Optional[int] = None,
|
|
250
256
|
location: Optional[Union[List, str]] = None,
|
|
251
257
|
force_cache_miss: bool = False,
|
|
258
|
+
method: str = "POST",
|
|
252
259
|
) -> Optional[
|
|
253
260
|
Union[
|
|
254
261
|
pd.DataFrame,
|
|
@@ -310,6 +317,8 @@ class API(AuthABC):
|
|
|
310
317
|
force_cache_miss : bool, optional
|
|
311
318
|
Whether to force a cache miss on the backend data endpoint.
|
|
312
319
|
Default is False.
|
|
320
|
+
method : str, optional
|
|
321
|
+
HTTP method to use for the backend request. Default is ``"GET"``.
|
|
313
322
|
|
|
314
323
|
Returns
|
|
315
324
|
-------
|
|
@@ -336,6 +345,7 @@ class API(AuthABC):
|
|
|
336
345
|
as_dict=as_dict,
|
|
337
346
|
as_star_schema=as_star_schema,
|
|
338
347
|
force_cache_miss=force_cache_miss,
|
|
348
|
+
method=method,
|
|
339
349
|
)
|
|
340
350
|
logging.info(
|
|
341
351
|
f"\n\033[32;1mRequested time range:\033[0;34m "
|
|
@@ -373,6 +383,7 @@ class API(AuthABC):
|
|
|
373
383
|
self._auth,
|
|
374
384
|
timeout,
|
|
375
385
|
force_cache_miss=query.force_cache_miss,
|
|
386
|
+
method=query.method,
|
|
376
387
|
): site
|
|
377
388
|
for site, group in grouped_metrics
|
|
378
389
|
}
|
|
@@ -385,7 +396,8 @@ class API(AuthABC):
|
|
|
385
396
|
_get_group_locations(group),
|
|
386
397
|
query.start_timestamp, query.end_timestamp, query.headers,
|
|
387
398
|
group, self._auth, timeout,
|
|
388
|
-
force_cache_miss=query.force_cache_miss
|
|
399
|
+
force_cache_miss=query.force_cache_miss,
|
|
400
|
+
method=query.method))
|
|
389
401
|
)
|
|
390
402
|
# fmt: on
|
|
391
403
|
# data_frames.append(pd.DataFrame(r_.json()))
|
|
@@ -436,6 +448,7 @@ class API(AuthABC):
|
|
|
436
448
|
timeout: int = 3600,
|
|
437
449
|
threads: Optional[int] = None,
|
|
438
450
|
location: Optional[Union[List, str]] = None,
|
|
451
|
+
method: str = "GET",
|
|
439
452
|
) -> Union[
|
|
440
453
|
pd.DataFrame, Dict[str, Union[Dict[str, pd.DataFrame], Dict[str, Any]]]
|
|
441
454
|
]:
|
|
@@ -466,6 +479,8 @@ class API(AuthABC):
|
|
|
466
479
|
The location name or List of location names. This is a legacy
|
|
467
480
|
parameter, and it is deprecated. Please use the `locations`
|
|
468
481
|
parameter instead.
|
|
482
|
+
method : str, optional
|
|
483
|
+
HTTP method to use for the backend request. Default is ``"GET"``.
|
|
469
484
|
|
|
470
485
|
Returns
|
|
471
486
|
-------
|
|
@@ -483,6 +498,7 @@ class API(AuthABC):
|
|
|
483
498
|
location=location,
|
|
484
499
|
metrics=metrics,
|
|
485
500
|
headers=headers,
|
|
501
|
+
method=method,
|
|
486
502
|
)
|
|
487
503
|
|
|
488
504
|
logging.info(
|
|
@@ -516,6 +532,7 @@ class API(AuthABC):
|
|
|
516
532
|
group,
|
|
517
533
|
self._auth,
|
|
518
534
|
timeout,
|
|
535
|
+
method=query.method,
|
|
519
536
|
): site
|
|
520
537
|
for site, group in grouped_metrics
|
|
521
538
|
}
|
|
@@ -533,7 +550,8 @@ class API(AuthABC):
|
|
|
533
550
|
query.start_timestamp,
|
|
534
551
|
query.end_timestamp,
|
|
535
552
|
query.headers, group,
|
|
536
|
-
self._auth, timeout
|
|
553
|
+
self._auth, timeout,
|
|
554
|
+
method=query.method)
|
|
537
555
|
if isinstance(result, list):
|
|
538
556
|
stats_list.extend(result)
|
|
539
557
|
else:
|
|
@@ -564,6 +582,7 @@ class API(AuthABC):
|
|
|
564
582
|
timeout: int = 3600,
|
|
565
583
|
threads: Optional[int] = None,
|
|
566
584
|
location: Optional[Union[List, str]] = None,
|
|
585
|
+
method: str = "GET",
|
|
567
586
|
) -> Union[
|
|
568
587
|
pd.DataFrame, Dict[str, Union[Dict[str, pd.DataFrame], Dict[str, Any]]]
|
|
569
588
|
]:
|
|
@@ -594,6 +613,8 @@ class API(AuthABC):
|
|
|
594
613
|
The location name or List of location names. This is a legacy
|
|
595
614
|
parameter, and it is deprecated. Please use the `locations`
|
|
596
615
|
parameter instead.
|
|
616
|
+
method : str, optional
|
|
617
|
+
HTTP method to use for the backend request. Default is ``"GET"``.
|
|
597
618
|
|
|
598
619
|
Returns
|
|
599
620
|
-------
|
|
@@ -611,6 +632,7 @@ class API(AuthABC):
|
|
|
611
632
|
location=location,
|
|
612
633
|
metrics=metrics,
|
|
613
634
|
headers=headers,
|
|
635
|
+
method=method,
|
|
614
636
|
)
|
|
615
637
|
|
|
616
638
|
logging.info(
|
|
@@ -644,6 +666,7 @@ class API(AuthABC):
|
|
|
644
666
|
group,
|
|
645
667
|
self._auth,
|
|
646
668
|
timeout,
|
|
669
|
+
method=query.method,
|
|
647
670
|
): site
|
|
648
671
|
for site, group in grouped_metrics
|
|
649
672
|
}
|
|
@@ -661,7 +684,8 @@ class API(AuthABC):
|
|
|
661
684
|
query.start_timestamp,
|
|
662
685
|
query.end_timestamp,
|
|
663
686
|
query.headers, group,
|
|
664
|
-
self._auth, timeout
|
|
687
|
+
self._auth, timeout,
|
|
688
|
+
method=query.method)
|
|
665
689
|
if isinstance(result, list):
|
|
666
690
|
stats_list.extend(result)
|
|
667
691
|
else:
|
|
@@ -694,6 +718,7 @@ class API(AuthABC):
|
|
|
694
718
|
timeout: int = 3600,
|
|
695
719
|
threads: Optional[int] = None,
|
|
696
720
|
location: Optional[Union[List, str]] = None,
|
|
721
|
+
method: str = "GET",
|
|
697
722
|
) -> Union[
|
|
698
723
|
pd.DataFrame, Dict[str, Union[Dict[str, pd.DataFrame], Dict[str, Any]]]
|
|
699
724
|
]:
|
|
@@ -734,6 +759,8 @@ class API(AuthABC):
|
|
|
734
759
|
The location name or List of location names. This is a legacy
|
|
735
760
|
parameter, and it is deprecated. Please use the `locations`
|
|
736
761
|
parameter instead.
|
|
762
|
+
method : str, optional
|
|
763
|
+
HTTP method to use for the backend request. Default is ``"GET"``.
|
|
737
764
|
|
|
738
765
|
Returns
|
|
739
766
|
-------
|
|
@@ -754,6 +781,7 @@ class API(AuthABC):
|
|
|
754
781
|
sampling_interval_seconds=sampling_interval_seconds,
|
|
755
782
|
as_dict=as_dict,
|
|
756
783
|
headers=headers,
|
|
784
|
+
method=method,
|
|
757
785
|
)
|
|
758
786
|
|
|
759
787
|
logging.info(
|
|
@@ -789,6 +817,7 @@ class API(AuthABC):
|
|
|
789
817
|
group,
|
|
790
818
|
self._auth,
|
|
791
819
|
timeout,
|
|
820
|
+
method=query.method,
|
|
792
821
|
): site
|
|
793
822
|
for site, group in grouped_metrics
|
|
794
823
|
}
|
|
@@ -808,6 +837,7 @@ class API(AuthABC):
|
|
|
808
837
|
group,
|
|
809
838
|
self._auth,
|
|
810
839
|
timeout,
|
|
840
|
+
method=query.method,
|
|
811
841
|
))
|
|
812
842
|
# fmt: on
|
|
813
843
|
# av_list is a list of lists, I need to expand it to a single list
|
|
@@ -829,6 +859,7 @@ class API(AuthABC):
|
|
|
829
859
|
self,
|
|
830
860
|
sites: Union[str, List[str]],
|
|
831
861
|
locations: Optional[Union[List[str], str]],
|
|
862
|
+
method: str = "GET",
|
|
832
863
|
) -> pd.DataFrame:
|
|
833
864
|
"""Get oldest timestamp for one or multiple sites (sync)."""
|
|
834
865
|
all_dfs = []
|
|
@@ -836,38 +867,61 @@ class API(AuthABC):
|
|
|
836
867
|
sites = [sites]
|
|
837
868
|
import concurrent.futures
|
|
838
869
|
|
|
839
|
-
|
|
840
|
-
|
|
870
|
+
queries = []
|
|
871
|
+
for site in sites:
|
|
872
|
+
query = S.GetOldestTimestampSchema(
|
|
873
|
+
site=site, locations=locations, method=method
|
|
874
|
+
)
|
|
841
875
|
query_df = query.get_selected_locations(self._metrics_overview)
|
|
842
876
|
query_locs = query_df.location.to_list()
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
query.headers or {"accept": "application/json"}) # pylint: disable=C0301 # noqa: E501
|
|
850
|
-
# fmt: on
|
|
851
|
-
js = r_.json()
|
|
852
|
-
if js == []:
|
|
853
|
-
logging.warning(
|
|
854
|
-
f"\033[33;1m⚠️ No data found for {query.site}.\033[0m"
|
|
877
|
+
queries.append(
|
|
878
|
+
(
|
|
879
|
+
str(query_df.site.iloc[0]).lower(),
|
|
880
|
+
",".join(query_locs),
|
|
881
|
+
query.headers or {"accept": "application/json"},
|
|
882
|
+
query.method,
|
|
855
883
|
)
|
|
856
|
-
|
|
884
|
+
)
|
|
857
885
|
|
|
858
886
|
try:
|
|
859
887
|
with concurrent.futures.ThreadPoolExecutor(
|
|
860
888
|
max_workers=len(sites), thread_name_prefix="24SEA"
|
|
861
889
|
) as executor:
|
|
862
890
|
future_to_site = {
|
|
863
|
-
executor.submit(
|
|
864
|
-
|
|
891
|
+
executor.submit(
|
|
892
|
+
U.fetch_oldest_timestamp_sync,
|
|
893
|
+
f"{self.base_url}oldest_timestamp/",
|
|
894
|
+
query_site,
|
|
895
|
+
query_locations,
|
|
896
|
+
query_headers,
|
|
897
|
+
self._auth,
|
|
898
|
+
3600,
|
|
899
|
+
False,
|
|
900
|
+
query_method,
|
|
901
|
+
): query_site
|
|
902
|
+
for query_site, query_locations, query_headers, query_method in queries
|
|
865
903
|
}
|
|
866
904
|
for future in concurrent.futures.as_completed(future_to_site):
|
|
867
905
|
all_dfs.append(future.result())
|
|
868
906
|
except RuntimeError:
|
|
869
|
-
for
|
|
870
|
-
|
|
907
|
+
for (
|
|
908
|
+
query_site,
|
|
909
|
+
query_locations,
|
|
910
|
+
query_headers,
|
|
911
|
+
query_method,
|
|
912
|
+
) in queries:
|
|
913
|
+
all_dfs.append(
|
|
914
|
+
U.fetch_oldest_timestamp_sync(
|
|
915
|
+
f"{self.base_url}oldest_timestamp/",
|
|
916
|
+
query_site,
|
|
917
|
+
query_locations,
|
|
918
|
+
query_headers,
|
|
919
|
+
self._auth,
|
|
920
|
+
3600,
|
|
921
|
+
False,
|
|
922
|
+
query_method,
|
|
923
|
+
)
|
|
924
|
+
)
|
|
871
925
|
return (
|
|
872
926
|
pd.concat(all_dfs, ignore_index=True) if all_dfs else pd.DataFrame()
|
|
873
927
|
)
|
|
@@ -883,6 +937,7 @@ class API(AuthABC):
|
|
|
883
937
|
headers: Optional[Dict[str, str]] = None,
|
|
884
938
|
timeout: int = 3600,
|
|
885
939
|
threads: Optional[int] = None,
|
|
940
|
+
method: str = "GET",
|
|
886
941
|
) -> Dict[str, Any]:
|
|
887
942
|
"""
|
|
888
943
|
Run get_stats for predefined intervals:
|
|
@@ -915,6 +970,7 @@ class API(AuthABC):
|
|
|
915
970
|
headers=headers,
|
|
916
971
|
timeout=timeout,
|
|
917
972
|
threads=threads,
|
|
973
|
+
method=method,
|
|
918
974
|
)
|
|
919
975
|
return results
|
|
920
976
|
|
|
@@ -973,6 +1029,7 @@ class AsyncAPI(API):
|
|
|
973
1029
|
timeout: int = 1800,
|
|
974
1030
|
location: Optional[Union[List, str]] = None,
|
|
975
1031
|
force_cache_miss: bool = False,
|
|
1032
|
+
method: str = "POST",
|
|
976
1033
|
) -> Optional[
|
|
977
1034
|
Union[
|
|
978
1035
|
pd.DataFrame,
|
|
@@ -1042,6 +1099,8 @@ class AsyncAPI(API):
|
|
|
1042
1099
|
force_cache_miss : bool, optional
|
|
1043
1100
|
Whether to force a cache miss on the backend data endpoint.
|
|
1044
1101
|
Default is False.
|
|
1102
|
+
method : str, optional
|
|
1103
|
+
HTTP method to use for the backend request. Default is ``"GET"``.
|
|
1045
1104
|
|
|
1046
1105
|
Returns
|
|
1047
1106
|
-------
|
|
@@ -1066,6 +1125,7 @@ class AsyncAPI(API):
|
|
|
1066
1125
|
as_dict=as_dict,
|
|
1067
1126
|
as_star_schema=as_star_schema,
|
|
1068
1127
|
force_cache_miss=force_cache_miss,
|
|
1128
|
+
method=method,
|
|
1069
1129
|
)
|
|
1070
1130
|
|
|
1071
1131
|
metrics_overview = await self.get_metrics_overview()
|
|
@@ -1086,7 +1146,8 @@ class AsyncAPI(API):
|
|
|
1086
1146
|
query.start_timestamp, query.end_timestamp,
|
|
1087
1147
|
query.headers, group, self._auth, timeout,
|
|
1088
1148
|
max_retries,
|
|
1089
|
-
force_cache_miss=query.force_cache_miss
|
|
1149
|
+
force_cache_miss=query.force_cache_miss,
|
|
1150
|
+
method=query.method)
|
|
1090
1151
|
for site, group in grouped_metrics]
|
|
1091
1152
|
chunk_size_dict = U.estimate_chunk_size(
|
|
1092
1153
|
tasks,
|
|
@@ -1154,6 +1215,7 @@ class AsyncAPI(API):
|
|
|
1154
1215
|
self,
|
|
1155
1216
|
sites: Union[str, List[str]],
|
|
1156
1217
|
locations: Optional[Union[List[str], str]],
|
|
1218
|
+
method: str = "GET",
|
|
1157
1219
|
) -> pd.DataFrame:
|
|
1158
1220
|
"""Get oldest timestamp for one or multiple sites (async).
|
|
1159
1221
|
|
|
@@ -1175,7 +1237,9 @@ class AsyncAPI(API):
|
|
|
1175
1237
|
if isinstance(sites, str):
|
|
1176
1238
|
sites = [sites]
|
|
1177
1239
|
for site in sites:
|
|
1178
|
-
query = S.GetOldestTimestampSchema(
|
|
1240
|
+
query = S.GetOldestTimestampSchema(
|
|
1241
|
+
site=site, locations=locations, method=method
|
|
1242
|
+
)
|
|
1179
1243
|
query_df = query.get_selected_locations(self._metrics_overview)
|
|
1180
1244
|
query_locs = query_df.location.to_list()
|
|
1181
1245
|
query_site = str(query_df.site.iloc[0]).lower()
|
|
@@ -1189,6 +1253,7 @@ class AsyncAPI(API):
|
|
|
1189
1253
|
3600,
|
|
1190
1254
|
5,
|
|
1191
1255
|
False,
|
|
1256
|
+
query.method,
|
|
1192
1257
|
)
|
|
1193
1258
|
)
|
|
1194
1259
|
results = await asyncio.gather(*tasks)
|
|
@@ -1209,6 +1274,7 @@ class AsyncAPI(API):
|
|
|
1209
1274
|
headers: Optional[Dict[str, str]] = None,
|
|
1210
1275
|
timeout: int = 3600,
|
|
1211
1276
|
location: Optional[Union[List, str]] = None,
|
|
1277
|
+
method: str = "GET",
|
|
1212
1278
|
) -> Union[
|
|
1213
1279
|
pd.DataFrame, Dict[str, Union[Dict[str, pd.DataFrame], Dict[str, Any]]]
|
|
1214
1280
|
]:
|
|
@@ -1254,6 +1320,7 @@ class AsyncAPI(API):
|
|
|
1254
1320
|
location=location,
|
|
1255
1321
|
metrics=metrics,
|
|
1256
1322
|
headers=headers,
|
|
1323
|
+
method=method,
|
|
1257
1324
|
)
|
|
1258
1325
|
|
|
1259
1326
|
logging.info(
|
|
@@ -1278,7 +1345,7 @@ class AsyncAPI(API):
|
|
|
1278
1345
|
_get_group_locations(group),
|
|
1279
1346
|
query.start_timestamp, query.end_timestamp,
|
|
1280
1347
|
query.headers, group, self._auth, timeout,
|
|
1281
|
-
5, True)
|
|
1348
|
+
5, True, method=query.method)
|
|
1282
1349
|
for site, group in grouped_metrics]
|
|
1283
1350
|
# fmt: on
|
|
1284
1351
|
stats_list = await U.gather_in_chunks(
|
|
@@ -1310,6 +1377,7 @@ class AsyncAPI(API):
|
|
|
1310
1377
|
headers: Optional[Dict[str, str]] = None,
|
|
1311
1378
|
timeout: int = 3600,
|
|
1312
1379
|
location: Optional[Union[List, str]] = None,
|
|
1380
|
+
method: str = "GET",
|
|
1313
1381
|
) -> Union[
|
|
1314
1382
|
pd.DataFrame, Dict[str, Union[Dict[str, pd.DataFrame], Dict[str, Any]]]
|
|
1315
1383
|
]:
|
|
@@ -1355,6 +1423,7 @@ class AsyncAPI(API):
|
|
|
1355
1423
|
location=location,
|
|
1356
1424
|
metrics=metrics,
|
|
1357
1425
|
headers=headers,
|
|
1426
|
+
method=method,
|
|
1358
1427
|
)
|
|
1359
1428
|
|
|
1360
1429
|
logging.info(
|
|
@@ -1380,7 +1449,7 @@ class AsyncAPI(API):
|
|
|
1380
1449
|
_get_group_locations(group),
|
|
1381
1450
|
query.start_timestamp, query.end_timestamp,
|
|
1382
1451
|
query.headers, group, self._auth, timeout,
|
|
1383
|
-
5, True)
|
|
1452
|
+
5, True, method=query.method)
|
|
1384
1453
|
for site, group in grouped_metrics]
|
|
1385
1454
|
# fmt: on
|
|
1386
1455
|
stats_list = await U.gather_in_chunks(
|
|
@@ -1414,6 +1483,7 @@ class AsyncAPI(API):
|
|
|
1414
1483
|
headers: Optional[Dict[str, str]] = None,
|
|
1415
1484
|
timeout: int = 3600,
|
|
1416
1485
|
location: Optional[Union[List, str]] = None,
|
|
1486
|
+
method: str = "GET",
|
|
1417
1487
|
) -> Union[
|
|
1418
1488
|
pd.DataFrame, Dict[str, Union[Dict[str, pd.DataFrame], Dict[str, Any]]]
|
|
1419
1489
|
]:
|
|
@@ -1472,6 +1542,7 @@ class AsyncAPI(API):
|
|
|
1472
1542
|
sampling_interval_seconds=sampling_interval_seconds,
|
|
1473
1543
|
as_dict=as_dict,
|
|
1474
1544
|
headers=headers,
|
|
1545
|
+
method=method,
|
|
1475
1546
|
)
|
|
1476
1547
|
|
|
1477
1548
|
logging.info(
|
|
@@ -1505,6 +1576,7 @@ class AsyncAPI(API):
|
|
|
1505
1576
|
self._auth,
|
|
1506
1577
|
timeout,
|
|
1507
1578
|
5, # max_retries
|
|
1579
|
+
query.method,
|
|
1508
1580
|
)
|
|
1509
1581
|
for site, group in grouped_metrics
|
|
1510
1582
|
]
|
|
@@ -1554,6 +1626,7 @@ class AsyncAPI(API):
|
|
|
1554
1626
|
as_dict: bool = False,
|
|
1555
1627
|
headers: Optional[Dict[str, str]] = None,
|
|
1556
1628
|
timeout: int = 3600,
|
|
1629
|
+
method: str = "GET",
|
|
1557
1630
|
) -> Dict[str, Any]:
|
|
1558
1631
|
"""
|
|
1559
1632
|
Async version of get_stats_predefined_intervals.
|
|
@@ -1579,6 +1652,7 @@ class AsyncAPI(API):
|
|
|
1579
1652
|
as_dict=as_dict,
|
|
1580
1653
|
headers=headers,
|
|
1581
1654
|
timeout=timeout,
|
|
1655
|
+
method=method,
|
|
1582
1656
|
)
|
|
1583
1657
|
|
|
1584
1658
|
tasks = {name: _run(s, e) for name, (s, e) in intervals.items()}
|
|
@@ -31,6 +31,7 @@ class BaseDataSignalSchema(U.BaseModel):
|
|
|
31
31
|
as_dict: Optional[bool] = False
|
|
32
32
|
as_star_schema: Optional[bool] = False
|
|
33
33
|
headers: Optional[Dict[str, str]] = {"accept": "application/json"}
|
|
34
|
+
method: str = "GET"
|
|
34
35
|
|
|
35
36
|
def __init__(self, **data):
|
|
36
37
|
data = self._normalize_deprecated_location_input(data)
|
|
@@ -116,6 +117,11 @@ class BaseDataSignalSchema(U.BaseModel):
|
|
|
116
117
|
return {"accept": "application/json"}
|
|
117
118
|
return v
|
|
118
119
|
|
|
120
|
+
@U.field_validator("method", mode="before")
|
|
121
|
+
def validate_method(cls, v: str) -> str:
|
|
122
|
+
"""Validate and normalize request method input."""
|
|
123
|
+
return U.normalize_http_method(v)
|
|
124
|
+
|
|
119
125
|
@property
|
|
120
126
|
def query_str(self) -> str:
|
|
121
127
|
"""Build a query string based on self."""
|
|
@@ -144,10 +150,13 @@ class BaseDataSignalSchema(U.BaseModel):
|
|
|
144
150
|
site=lambda x: x["site"].str.lower(),
|
|
145
151
|
location=lambda x: x["location"].str.upper(),
|
|
146
152
|
)
|
|
153
|
+
groupby_key: Union[str, List[str]] = (
|
|
154
|
+
groupby_columns[0] if len(groupby_columns) == 1 else groupby_columns
|
|
155
|
+
)
|
|
147
156
|
if self.metrics in (["all"], "all"):
|
|
148
157
|
group_base = normalized[["site", "location"]].drop_duplicates()
|
|
149
|
-
return group_base.assign(metric="all").groupby(
|
|
150
|
-
return normalized.groupby(
|
|
158
|
+
return group_base.assign(metric="all").groupby(groupby_key)
|
|
159
|
+
return normalized.groupby(groupby_key)
|
|
151
160
|
|
|
152
161
|
# @lru_cache
|
|
153
162
|
def group_metrics(self, df: Optional[pd.DataFrame]) -> DataFrameGroupBy:
|
|
@@ -175,6 +184,7 @@ class GetOldestTimestampSchema(U.BaseModel):
|
|
|
175
184
|
site: str # renamed from 'sites' to match caller
|
|
176
185
|
locations: Optional[Union[str, List[str]]] = None
|
|
177
186
|
headers: Optional[Dict[str, str]] = {"accept": "application/json"}
|
|
187
|
+
method: str = "GET"
|
|
178
188
|
|
|
179
189
|
@U.field_validator("locations", mode="before")
|
|
180
190
|
def validate_locations(cls, v) -> Optional[str]:
|
|
@@ -202,6 +212,11 @@ class GetOldestTimestampSchema(U.BaseModel):
|
|
|
202
212
|
return v
|
|
203
213
|
raise TypeError("headers must be a dictionary")
|
|
204
214
|
|
|
215
|
+
@U.field_validator("method", mode="before")
|
|
216
|
+
def validate_method(cls, v: str) -> str:
|
|
217
|
+
"""Validate and normalize request method input."""
|
|
218
|
+
return U.normalize_http_method(v)
|
|
219
|
+
|
|
205
220
|
@property
|
|
206
221
|
def query_str(self) -> str:
|
|
207
222
|
# fmt: off
|
|
@@ -9,7 +9,16 @@ import time
|
|
|
9
9
|
import warnings
|
|
10
10
|
from collections import defaultdict
|
|
11
11
|
from types import CoroutineType
|
|
12
|
-
from typing import
|
|
12
|
+
from typing import (
|
|
13
|
+
Any,
|
|
14
|
+
DefaultDict,
|
|
15
|
+
Dict,
|
|
16
|
+
Iterable,
|
|
17
|
+
List,
|
|
18
|
+
Optional,
|
|
19
|
+
Tuple,
|
|
20
|
+
Union,
|
|
21
|
+
)
|
|
13
22
|
|
|
14
23
|
import httpx
|
|
15
24
|
import pandas as pd
|
|
@@ -94,6 +103,8 @@ def run_tqdm(
|
|
|
94
103
|
|
|
95
104
|
|
|
96
105
|
BASE_URL = "https://api.24sea.eu/routes/v1/"
|
|
106
|
+
SUPPORTED_HTTP_METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE")
|
|
107
|
+
SPECIAL_QUERY_PARAM_KEYS = ("include_cyclecount", "on_conflict")
|
|
97
108
|
PYDANTIC_V2 = version.parse_version(pydantic_version.VERSION).major >= 2
|
|
98
109
|
|
|
99
110
|
if PYDANTIC_V2:
|
|
@@ -131,6 +142,121 @@ else:
|
|
|
131
142
|
return decorator
|
|
132
143
|
|
|
133
144
|
|
|
145
|
+
def normalize_http_method(method: str) -> str:
|
|
146
|
+
"""Normalize and validate an HTTP method.
|
|
147
|
+
|
|
148
|
+
Parameters
|
|
149
|
+
----------
|
|
150
|
+
method : str
|
|
151
|
+
HTTP verb to normalize.
|
|
152
|
+
|
|
153
|
+
Returns
|
|
154
|
+
-------
|
|
155
|
+
str
|
|
156
|
+
Upper-case HTTP method.
|
|
157
|
+
|
|
158
|
+
Raises
|
|
159
|
+
------
|
|
160
|
+
ValueError
|
|
161
|
+
If the provided method is not supported.
|
|
162
|
+
"""
|
|
163
|
+
normalized_method = method.upper()
|
|
164
|
+
if normalized_method not in SUPPORTED_HTTP_METHODS:
|
|
165
|
+
raise ValueError(
|
|
166
|
+
"Unsupported HTTP method: "
|
|
167
|
+
f"{method}. Supported methods are: "
|
|
168
|
+
f"{', '.join(SUPPORTED_HTTP_METHODS)}"
|
|
169
|
+
)
|
|
170
|
+
return normalized_method
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def build_json_request_payload(
|
|
174
|
+
params: Optional[Dict],
|
|
175
|
+
json: Optional[Dict] = None,
|
|
176
|
+
) -> Optional[Dict]:
|
|
177
|
+
"""Normalize a non-GET payload to the backend JSON contract.
|
|
178
|
+
|
|
179
|
+
Parameters
|
|
180
|
+
----------
|
|
181
|
+
params : dict, optional
|
|
182
|
+
Request parameters produced by the caller.
|
|
183
|
+
json : dict, optional
|
|
184
|
+
Explicit JSON payload fallback.
|
|
185
|
+
|
|
186
|
+
Returns
|
|
187
|
+
-------
|
|
188
|
+
dict or None
|
|
189
|
+
Normalized JSON payload.
|
|
190
|
+
"""
|
|
191
|
+
source_payload = params if params else json
|
|
192
|
+
if source_payload is None:
|
|
193
|
+
return None
|
|
194
|
+
|
|
195
|
+
payload = dict(source_payload)
|
|
196
|
+
for key in SPECIAL_QUERY_PARAM_KEYS:
|
|
197
|
+
payload.pop(key, None)
|
|
198
|
+
|
|
199
|
+
project = payload.get("project")
|
|
200
|
+
if isinstance(project, list) and len(project) == 1:
|
|
201
|
+
payload["project"] = project[0]
|
|
202
|
+
|
|
203
|
+
locations = payload.get("locations")
|
|
204
|
+
location = payload.pop("location", None)
|
|
205
|
+
if locations is None and location is not None:
|
|
206
|
+
payload["locations"] = (
|
|
207
|
+
location if isinstance(location, list) else [location]
|
|
208
|
+
)
|
|
209
|
+
elif isinstance(locations, str):
|
|
210
|
+
payload["locations"] = [item for item in locations.split(",") if item]
|
|
211
|
+
|
|
212
|
+
for key in ("metrics", "predictions"):
|
|
213
|
+
value = payload.get(key)
|
|
214
|
+
if isinstance(value, str):
|
|
215
|
+
payload[key] = [item for item in value.split(",") if item]
|
|
216
|
+
|
|
217
|
+
return {key: value for key, value in payload.items() if value is not None}
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def build_httpx_request_kwargs(
|
|
221
|
+
method: str,
|
|
222
|
+
params: Dict,
|
|
223
|
+
json: Optional[Dict] = None,
|
|
224
|
+
) -> Tuple[Optional[Dict], Optional[Dict]]:
|
|
225
|
+
"""Build ``httpx`` request kwargs based on the HTTP method.
|
|
226
|
+
|
|
227
|
+
GET requests keep using query-string parameters. Non-GET requests build
|
|
228
|
+
the request payload from ``params`` and only fall back to ``json`` when
|
|
229
|
+
``params`` is empty.
|
|
230
|
+
|
|
231
|
+
Parameters
|
|
232
|
+
----------
|
|
233
|
+
method : str
|
|
234
|
+
Normalized HTTP method.
|
|
235
|
+
params : dict
|
|
236
|
+
Parameters provided by the caller.
|
|
237
|
+
json : dict, optional
|
|
238
|
+
Explicit JSON payload fallback.
|
|
239
|
+
|
|
240
|
+
Returns
|
|
241
|
+
-------
|
|
242
|
+
Tuple[Optional[Dict], Optional[Dict]]
|
|
243
|
+
``params`` and ``json`` values to forward to ``httpx.request``.
|
|
244
|
+
"""
|
|
245
|
+
if method == "GET":
|
|
246
|
+
return params, json
|
|
247
|
+
|
|
248
|
+
payload = build_json_request_payload(params, json)
|
|
249
|
+
if method not in {"PATCH", "PUT"}:
|
|
250
|
+
return None, payload
|
|
251
|
+
|
|
252
|
+
out_params = {
|
|
253
|
+
key: params[key]
|
|
254
|
+
for key in SPECIAL_QUERY_PARAM_KEYS
|
|
255
|
+
if key in params and params[key] is not None
|
|
256
|
+
}
|
|
257
|
+
return out_params or None, payload
|
|
258
|
+
|
|
259
|
+
|
|
134
260
|
def handle_request(
|
|
135
261
|
url: str,
|
|
136
262
|
params: Dict,
|
|
@@ -138,6 +264,8 @@ def handle_request(
|
|
|
138
264
|
headers: Optional[Dict[str, str]] = {"accept": "application/json"},
|
|
139
265
|
max_retries: int = 10,
|
|
140
266
|
timeout: int = 3600,
|
|
267
|
+
method: str = "GET",
|
|
268
|
+
json: Optional[Dict] = None,
|
|
141
269
|
) -> httpx.Response:
|
|
142
270
|
"""Handle the request to the 24SEA API and manage errors using httpx.
|
|
143
271
|
|
|
@@ -161,6 +289,10 @@ def handle_request(
|
|
|
161
289
|
httpx.Response
|
|
162
290
|
The response object if the request was successful, otherwise error.
|
|
163
291
|
"""
|
|
292
|
+
method = normalize_http_method(method)
|
|
293
|
+
request_params, request_json = build_httpx_request_kwargs(
|
|
294
|
+
method, params, json
|
|
295
|
+
)
|
|
164
296
|
if auth is None:
|
|
165
297
|
auth = httpx.BasicAuth("", "")
|
|
166
298
|
retry_count = 0
|
|
@@ -168,8 +300,9 @@ def handle_request(
|
|
|
168
300
|
while True:
|
|
169
301
|
try:
|
|
170
302
|
# fmt: off
|
|
171
|
-
r_ = httpx.
|
|
172
|
-
|
|
303
|
+
r_ = httpx.request(method, url, auth=auth, headers=headers,
|
|
304
|
+
timeout=timeout, params=request_params,
|
|
305
|
+
json=request_json)
|
|
173
306
|
# fmt: on
|
|
174
307
|
if r_.status_code != 502 or retry_count >= max_retries:
|
|
175
308
|
break
|
|
@@ -196,23 +329,22 @@ async def handle_request_async(
|
|
|
196
329
|
json: Optional[Dict] = None,
|
|
197
330
|
) -> httpx.Response:
|
|
198
331
|
"""Asynchronously handle the request to the 24SEA API using httpx's
|
|
199
|
-
AsyncClient. Supports GET, POST, PUT, DELETE methods."""
|
|
332
|
+
AsyncClient. Supports GET, POST, PUT, PATCH, DELETE methods."""
|
|
333
|
+
method = normalize_http_method(method)
|
|
334
|
+
request_params, request_json = build_httpx_request_kwargs(
|
|
335
|
+
method, params, json
|
|
336
|
+
)
|
|
337
|
+
if auth is None:
|
|
338
|
+
auth = httpx.BasicAuth("", "")
|
|
200
339
|
retry_count = 0
|
|
201
340
|
async with httpx.AsyncClient(
|
|
202
341
|
auth=auth, headers=headers, timeout=timeout
|
|
203
342
|
) as client:
|
|
204
343
|
while True:
|
|
205
344
|
try:
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
r_ = await client.post(url, params=params, json=json)
|
|
210
|
-
elif method.upper() == "PUT":
|
|
211
|
-
r_ = await client.put(url, params=params, json=json)
|
|
212
|
-
elif method.upper() == "DELETE":
|
|
213
|
-
r_ = await client.delete(url, params=params)
|
|
214
|
-
else:
|
|
215
|
-
raise ValueError(f"Unsupported HTTP method: {method}")
|
|
345
|
+
r_ = await client.request(
|
|
346
|
+
method, url, params=request_params, json=request_json
|
|
347
|
+
)
|
|
216
348
|
if r_.status_code != 502 or retry_count >= max_retries:
|
|
217
349
|
break
|
|
218
350
|
retry_count += 1
|
|
@@ -360,6 +492,7 @@ def parse_timestamp(
|
|
|
360
492
|
series = df["timestamp"]
|
|
361
493
|
if series is None:
|
|
362
494
|
raise E.DataSignalsError(d_e)
|
|
495
|
+
|
|
363
496
|
try:
|
|
364
497
|
# Try parsing with different formats
|
|
365
498
|
for fmt in formats:
|
|
@@ -551,7 +684,7 @@ async def gather_in_chunks(
|
|
|
551
684
|
def fetch_data_sync(
|
|
552
685
|
url,
|
|
553
686
|
site: str,
|
|
554
|
-
locations: Union[str, List[str]
|
|
687
|
+
locations: Union[str, List[str]],
|
|
555
688
|
start_timestamp: Union[datetime.datetime, str],
|
|
556
689
|
end_timestamp: Union[datetime.datetime, str],
|
|
557
690
|
headers: Optional[Dict[str, str]],
|
|
@@ -560,6 +693,7 @@ def fetch_data_sync(
|
|
|
560
693
|
timeout: int,
|
|
561
694
|
target: str = "metric",
|
|
562
695
|
force_cache_miss: bool = False,
|
|
696
|
+
method: str = "GET",
|
|
563
697
|
) -> Any:
|
|
564
698
|
"""Syncronously fetch metrics data for the datasignals API app."""
|
|
565
699
|
group_values = normalize_group_values(group[target].tolist())
|
|
@@ -567,11 +701,15 @@ def fetch_data_sync(
|
|
|
567
701
|
s_ = "• " + ",".join(group_values).replace(
|
|
568
702
|
",", f"\n {len(target) + 1} • "
|
|
569
703
|
)
|
|
570
|
-
locations_list :
|
|
704
|
+
locations_list : List[str] = locations.split(",") if isinstance(locations, str) else locations or []
|
|
571
705
|
logging.info(f"\033[32;1m⏳ Getting data for {site} - "
|
|
572
706
|
f"{locations_list}...\n📊 \033[35;1m{target.capitalize()}s: "
|
|
573
707
|
f"\033[0;34m{s_}\n\033[0m")
|
|
574
708
|
# fmt: on
|
|
709
|
+
if not locations_list:
|
|
710
|
+
raise E.DataSignalsError(
|
|
711
|
+
"Locations must be provided as a string or list of strings."
|
|
712
|
+
)
|
|
575
713
|
r_ = handle_request(
|
|
576
714
|
url,
|
|
577
715
|
{
|
|
@@ -579,12 +717,14 @@ def fetch_data_sync(
|
|
|
579
717
|
"end_timestamp": end_timestamp,
|
|
580
718
|
"project": [site],
|
|
581
719
|
"locations": locations_list,
|
|
720
|
+
"location": locations_list[0],
|
|
582
721
|
f"{target}s": ",".join(group_values),
|
|
583
722
|
"force_cache_miss": force_cache_miss,
|
|
584
723
|
},
|
|
585
724
|
auth,
|
|
586
725
|
headers,
|
|
587
726
|
timeout=timeout,
|
|
727
|
+
method=method,
|
|
588
728
|
)
|
|
589
729
|
# Warn if empty
|
|
590
730
|
result_json = flatten_api_response(r_.json())
|
|
@@ -599,7 +739,7 @@ def fetch_data_sync(
|
|
|
599
739
|
def fetch_availability_sync(
|
|
600
740
|
url,
|
|
601
741
|
site: str,
|
|
602
|
-
locations: Union[str, List[str]
|
|
742
|
+
locations: Union[str, List[str]],
|
|
603
743
|
start_timestamp: Union[datetime.datetime, str],
|
|
604
744
|
end_timestamp: Union[datetime.datetime, str],
|
|
605
745
|
granularity: Union[str, int],
|
|
@@ -608,6 +748,7 @@ def fetch_availability_sync(
|
|
|
608
748
|
group: pd.DataFrame,
|
|
609
749
|
auth: Optional[httpx.BasicAuth],
|
|
610
750
|
timeout: int,
|
|
751
|
+
method: str = "GET",
|
|
611
752
|
) -> Any:
|
|
612
753
|
"""Syncronously fetch metrics data for the datasignals API app."""
|
|
613
754
|
metrics_values = normalize_group_values(group["metric"].tolist())
|
|
@@ -637,7 +778,9 @@ def fetch_availability_sync(
|
|
|
637
778
|
params["bucket_seconds"] = granularity
|
|
638
779
|
params["sampling_interval_seconds"] = sampling_interval_seconds
|
|
639
780
|
|
|
640
|
-
r_ = handle_request(
|
|
781
|
+
r_ = handle_request(
|
|
782
|
+
url, params, auth, headers, timeout=timeout, method=method
|
|
783
|
+
)
|
|
641
784
|
# Warn if empty
|
|
642
785
|
result_json = flatten_api_response(r_.json())
|
|
643
786
|
if result_json == []:
|
|
@@ -661,6 +804,7 @@ async def fetch_availability_async(
|
|
|
661
804
|
auth: Optional[httpx.BasicAuth],
|
|
662
805
|
timeout: int,
|
|
663
806
|
max_retries: int,
|
|
807
|
+
method: str = "GET",
|
|
664
808
|
) -> Any:
|
|
665
809
|
"""Asynchronously fetch availability data for a site/location."""
|
|
666
810
|
metrics_values = normalize_group_values(group["metric"].tolist())
|
|
@@ -689,7 +833,13 @@ async def fetch_availability_async(
|
|
|
689
833
|
params["bucket_seconds"] = granularity
|
|
690
834
|
params["sampling_interval_seconds"] = sampling_interval_seconds
|
|
691
835
|
r_ = await handle_request_async(
|
|
692
|
-
url,
|
|
836
|
+
url,
|
|
837
|
+
params,
|
|
838
|
+
auth,
|
|
839
|
+
headers,
|
|
840
|
+
max_retries=max_retries,
|
|
841
|
+
timeout=timeout,
|
|
842
|
+
method=method,
|
|
693
843
|
)
|
|
694
844
|
result_json = flatten_api_response(r_.json())
|
|
695
845
|
if result_json == []:
|
|
@@ -702,7 +852,7 @@ async def fetch_availability_async(
|
|
|
702
852
|
async def fetch_data_async(
|
|
703
853
|
url,
|
|
704
854
|
site: str,
|
|
705
|
-
locations: Union[str, List[str]
|
|
855
|
+
locations: Union[str, List[str]],
|
|
706
856
|
start_timestamp: Union[datetime.datetime, str],
|
|
707
857
|
end_timestamp: Union[datetime.datetime, str],
|
|
708
858
|
headers: Optional[Dict[str, str]],
|
|
@@ -713,34 +863,39 @@ async def fetch_data_async(
|
|
|
713
863
|
as_dict: bool = False,
|
|
714
864
|
target: str = "metric",
|
|
715
865
|
force_cache_miss: bool = False,
|
|
866
|
+
method: str = "GET",
|
|
716
867
|
) -> Union[pd.DataFrame, Dict[str, Any]]:
|
|
717
868
|
"""Asyncronously fetch metrics data for the datasignals API app."""
|
|
718
869
|
tgt_str = "data" if target == "metric" else "predictions"
|
|
719
870
|
group_values = normalize_group_values(group[target].tolist())
|
|
720
|
-
locations_list:
|
|
721
|
-
locations.split(",")
|
|
722
|
-
if isinstance(locations, str)
|
|
723
|
-
else locations or None
|
|
871
|
+
locations_list: List[str] = (
|
|
872
|
+
locations.split(",") if isinstance(locations, str) else locations or []
|
|
724
873
|
)
|
|
725
874
|
s_ = "• " + ",".join(group_values).replace(",", "\n • ")
|
|
726
875
|
logging.info(
|
|
727
876
|
f"\033[32;1m⏳ Getting {tgt_str} for {site} - {locations_list}..."
|
|
728
877
|
f"\n📊 \033[35;1m{target.capitalize()}s: \033[0;34m{s_}\n\033[0m"
|
|
729
878
|
)
|
|
879
|
+
payload = {
|
|
880
|
+
"start_timestamp": start_timestamp,
|
|
881
|
+
"end_timestamp": end_timestamp,
|
|
882
|
+
"project": [site],
|
|
883
|
+
f"{target}s": ",".join(group_values),
|
|
884
|
+
"force_cache_miss": force_cache_miss,
|
|
885
|
+
}
|
|
886
|
+
if len(locations_list) == 1:
|
|
887
|
+
payload["location"] = locations_list[0]
|
|
888
|
+
elif locations_list:
|
|
889
|
+
payload["locations"] = locations_list
|
|
890
|
+
|
|
730
891
|
r_ = await handle_request_async(
|
|
731
892
|
url,
|
|
732
|
-
|
|
733
|
-
"start_timestamp": start_timestamp,
|
|
734
|
-
"end_timestamp": end_timestamp,
|
|
735
|
-
"project": [site],
|
|
736
|
-
"locations": locations_list,
|
|
737
|
-
f"{target}s": ",".join(group_values),
|
|
738
|
-
"force_cache_miss": force_cache_miss,
|
|
739
|
-
},
|
|
893
|
+
payload,
|
|
740
894
|
auth,
|
|
741
895
|
headers,
|
|
742
896
|
max_retries=max_retries,
|
|
743
897
|
timeout=timeout,
|
|
898
|
+
method=method,
|
|
744
899
|
)
|
|
745
900
|
result_json = flatten_api_response(r_.json())
|
|
746
901
|
if result_json == []:
|
|
@@ -752,6 +907,47 @@ async def fetch_data_async(
|
|
|
752
907
|
return pd.DataFrame(result_json)
|
|
753
908
|
|
|
754
909
|
|
|
910
|
+
def fetch_oldest_timestamp_sync(
|
|
911
|
+
url: str,
|
|
912
|
+
site: str,
|
|
913
|
+
locations: Optional[str],
|
|
914
|
+
headers: Dict[str, str],
|
|
915
|
+
auth: Optional[httpx.BasicAuth],
|
|
916
|
+
timeout: int,
|
|
917
|
+
as_dict: bool = False,
|
|
918
|
+
method: str = "GET",
|
|
919
|
+
) -> Union[pd.DataFrame, Dict[str, Any]]:
|
|
920
|
+
"""Synchronously fetch oldest timestamp data for a site/location."""
|
|
921
|
+
if locations:
|
|
922
|
+
formatted_locations = " • " + "\n • ".join(locations.split(","))
|
|
923
|
+
logging.info(
|
|
924
|
+
f"\033[32;1m⏳ Getting oldest timestamps for {site} at the "
|
|
925
|
+
f"following locations:\n{formatted_locations}\n\033[0m"
|
|
926
|
+
)
|
|
927
|
+
else:
|
|
928
|
+
logging.info(
|
|
929
|
+
f"\033[32;1m⏳ Getting oldest timestamps for {site}\n\033[0m"
|
|
930
|
+
)
|
|
931
|
+
r_ = handle_request(
|
|
932
|
+
url,
|
|
933
|
+
(
|
|
934
|
+
{"project": site, "locations": locations}
|
|
935
|
+
if locations
|
|
936
|
+
else {"project": site}
|
|
937
|
+
),
|
|
938
|
+
auth,
|
|
939
|
+
headers,
|
|
940
|
+
timeout=timeout,
|
|
941
|
+
method=method,
|
|
942
|
+
)
|
|
943
|
+
result_json = r_.json()
|
|
944
|
+
if result_json == []:
|
|
945
|
+
logging.warning(f"\033[33;1m⚠️ No data found for {site}.\033[0m")
|
|
946
|
+
if as_dict:
|
|
947
|
+
return result_json
|
|
948
|
+
return pd.DataFrame(result_json)
|
|
949
|
+
|
|
950
|
+
|
|
755
951
|
async def fetch_oldest_timestamp_async(
|
|
756
952
|
url,
|
|
757
953
|
site: str,
|
|
@@ -761,6 +957,7 @@ async def fetch_oldest_timestamp_async(
|
|
|
761
957
|
timeout: int,
|
|
762
958
|
max_retries: int,
|
|
763
959
|
as_dict: bool = False,
|
|
960
|
+
method: str = "GET",
|
|
764
961
|
) -> Union[pd.DataFrame, Dict[str, Any]]:
|
|
765
962
|
# Format locations with bullets and newlines
|
|
766
963
|
if locations:
|
|
@@ -784,6 +981,7 @@ async def fetch_oldest_timestamp_async(
|
|
|
784
981
|
headers,
|
|
785
982
|
max_retries=max_retries,
|
|
786
983
|
timeout=timeout,
|
|
984
|
+
method=method,
|
|
787
985
|
)
|
|
788
986
|
result_json = r_.json()
|
|
789
987
|
if result_json == []:
|
|
@@ -1056,7 +1254,7 @@ def series_to_type(
|
|
|
1056
1254
|
if dtype in ("float", float, "int", int):
|
|
1057
1255
|
return pd.to_numeric(series, errors="coerce").astype(dtype) # type: ignore # pylint: disable=C301 # noqa: E501
|
|
1058
1256
|
if dtype in ("str", "string", str):
|
|
1059
|
-
return series.astype(str)
|
|
1257
|
+
return series.astype(str).astype(object)
|
|
1060
1258
|
return series
|
|
1061
1259
|
|
|
1062
1260
|
|
|
@@ -1199,7 +1397,10 @@ def calendar_monthly_availability(
|
|
|
1199
1397
|
if isinstance(idx_out, pd.DatetimeIndex):
|
|
1200
1398
|
if idx_out.tz is not None:
|
|
1201
1399
|
idx_out = idx_out.tz_convert(None)
|
|
1202
|
-
idx_out = pd.DatetimeIndex(
|
|
1400
|
+
idx_out = pd.DatetimeIndex(
|
|
1401
|
+
pd.to_datetime(idx_out.astype(str)),
|
|
1402
|
+
name=idx_out.name,
|
|
1403
|
+
)
|
|
1203
1404
|
df_monthly.index = idx_out
|
|
1204
1405
|
|
|
1205
1406
|
return df_monthly
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|