hsclient 0.3.4__tar.gz → 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hsclient-0.3.4 → hsclient-1.0.0}/PKG-INFO +15 -2
- {hsclient-0.3.4 → hsclient-1.0.0}/hsclient/hydroshare.py +76 -16
- {hsclient-0.3.4 → hsclient-1.0.0}/hsclient/json_models.py +5 -5
- {hsclient-0.3.4 → hsclient-1.0.0}/hsclient.egg-info/PKG-INFO +15 -2
- {hsclient-0.3.4 → hsclient-1.0.0}/hsclient.egg-info/SOURCES.txt +4 -1
- {hsclient-0.3.4 → hsclient-1.0.0}/hsclient.egg-info/requires.txt +1 -1
- {hsclient-0.3.4 → hsclient-1.0.0}/setup.py +2 -2
- hsclient-1.0.0/tests/test_data_objects.py +320 -0
- hsclient-1.0.0/tests/test_functional.py +566 -0
- hsclient-1.0.0/tests/test_json_models.py +106 -0
- {hsclient-0.3.4 → hsclient-1.0.0}/LICENSE.txt +0 -0
- {hsclient-0.3.4 → hsclient-1.0.0}/README.md +0 -0
- {hsclient-0.3.4 → hsclient-1.0.0}/hsclient/__init__.py +0 -0
- {hsclient-0.3.4 → hsclient-1.0.0}/hsclient/oauth2_model.py +0 -0
- {hsclient-0.3.4 → hsclient-1.0.0}/hsclient/utils.py +0 -0
- {hsclient-0.3.4 → hsclient-1.0.0}/hsclient.egg-info/dependency_links.txt +0 -0
- {hsclient-0.3.4 → hsclient-1.0.0}/hsclient.egg-info/top_level.txt +0 -0
- {hsclient-0.3.4 → hsclient-1.0.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: hsclient
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 1.0.0
|
|
4
4
|
Summary: A python client for managing HydroShare resources
|
|
5
5
|
Home-page: https://github.com/hydroshare/hsclient
|
|
6
6
|
Author: Scott Black
|
|
@@ -13,12 +13,25 @@ Classifier: Programming Language :: Python :: 3.8
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.9
|
|
14
14
|
Requires-Python: >=3.9
|
|
15
15
|
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE.txt
|
|
17
|
+
Requires-Dist: hsmodels>=1.0.0
|
|
18
|
+
Requires-Dist: requests
|
|
19
|
+
Requires-Dist: requests_oauthlib
|
|
16
20
|
Provides-Extra: pandas
|
|
21
|
+
Requires-Dist: pandas; extra == "pandas"
|
|
17
22
|
Provides-Extra: xarray
|
|
23
|
+
Requires-Dist: netCDF4; extra == "xarray"
|
|
24
|
+
Requires-Dist: xarray; extra == "xarray"
|
|
18
25
|
Provides-Extra: rasterio
|
|
26
|
+
Requires-Dist: rasterio; extra == "rasterio"
|
|
19
27
|
Provides-Extra: fiona
|
|
28
|
+
Requires-Dist: fiona; extra == "fiona"
|
|
20
29
|
Provides-Extra: all
|
|
21
|
-
|
|
30
|
+
Requires-Dist: pandas; extra == "all"
|
|
31
|
+
Requires-Dist: netCDF4; extra == "all"
|
|
32
|
+
Requires-Dist: xarray; extra == "all"
|
|
33
|
+
Requires-Dist: rasterio; extra == "all"
|
|
34
|
+
Requires-Dist: fiona; extra == "all"
|
|
22
35
|
|
|
23
36
|
# hsclient
|
|
24
37
|
A python client for interacting with HydroShare in an object oriented way.
|
|
@@ -393,7 +393,7 @@ class DataObjectSupportingAggregation(Aggregation):
|
|
|
393
393
|
|
|
394
394
|
@staticmethod
|
|
395
395
|
def create(aggr_cls, base_aggr):
|
|
396
|
-
"""
|
|
396
|
+
"""Creates a type specific aggregation object from an instance of Aggregation"""
|
|
397
397
|
aggr = aggr_cls(base_aggr._map_path, base_aggr._hs_session, base_aggr._parsed_checksums)
|
|
398
398
|
aggr._retrieved_map = base_aggr._retrieved_map
|
|
399
399
|
aggr._retrieved_metadata = base_aggr._retrieved_metadata
|
|
@@ -410,6 +410,8 @@ class DataObjectSupportingAggregation(Aggregation):
|
|
|
410
410
|
@property
|
|
411
411
|
def data_object(self) -> \
|
|
412
412
|
Union['pandas.DataFrame', 'fiona.Collection', 'rasterio.DatasetReader', 'xarray.Dataset', None]:
|
|
413
|
+
"""Returns the data object for the aggregation if the aggregation has been loaded as
|
|
414
|
+
a data object, otherwise None"""
|
|
413
415
|
return self._data_object
|
|
414
416
|
|
|
415
417
|
def _get_file_path(self, agg_path):
|
|
@@ -478,18 +480,32 @@ class DataObjectSupportingAggregation(Aggregation):
|
|
|
478
480
|
|
|
479
481
|
|
|
480
482
|
class NetCDFAggregation(DataObjectSupportingAggregation):
|
|
483
|
+
"""Represents a Multidimensional Aggregation in HydroShare"""
|
|
481
484
|
|
|
482
485
|
@classmethod
|
|
483
486
|
def create(cls, base_aggr):
|
|
484
487
|
return super().create(aggr_cls=cls, base_aggr=base_aggr)
|
|
485
488
|
|
|
486
489
|
def as_data_object(self, agg_path: str) -> 'xarray.Dataset':
|
|
490
|
+
"""
|
|
491
|
+
Loads the Multidimensional aggregation to a xarray Dataset object
|
|
492
|
+
:param agg_path: the path to the Multidimensional aggregation
|
|
493
|
+
:return: the Multidimensional aggregation as a xarray Dataset object
|
|
494
|
+
"""
|
|
487
495
|
if xarray is None:
|
|
488
496
|
raise Exception("xarray package was not found")
|
|
489
497
|
return self._get_data_object(agg_path=agg_path, func=xarray.open_dataset)
|
|
490
498
|
|
|
491
499
|
def save_data_object(self, resource: 'Resource', agg_path: str, as_new_aggr: bool = False,
|
|
492
500
|
destination_path: str = "") -> 'Aggregation':
|
|
501
|
+
"""
|
|
502
|
+
Saves the xarray Dataset object to the Multidimensional aggregation
|
|
503
|
+
:param resource: the resource containing the aggregation
|
|
504
|
+
:param agg_path: the path to the Multidimensional aggregation
|
|
505
|
+
:param as_new_aggr: Defaults False, set to True to create a new Multidimensional aggregation
|
|
506
|
+
:param destination_path: the destination path in Hydroshare to save the new aggregation
|
|
507
|
+
:return: the updated or new Multidimensional aggregation
|
|
508
|
+
"""
|
|
493
509
|
|
|
494
510
|
self._validate_aggregation_for_update(resource, AggregationType.MultidimensionalAggregation)
|
|
495
511
|
file_path = self._validate_aggregation_path(agg_path, for_save_data=True)
|
|
@@ -526,12 +542,18 @@ class NetCDFAggregation(DataObjectSupportingAggregation):
|
|
|
526
542
|
|
|
527
543
|
|
|
528
544
|
class TimeseriesAggregation(DataObjectSupportingAggregation):
|
|
529
|
-
|
|
545
|
+
"""Represents a Time Series Aggregation in HydroShare"""
|
|
530
546
|
@classmethod
|
|
531
547
|
def create(cls, base_aggr):
|
|
532
548
|
return super().create(aggr_cls=cls, base_aggr=base_aggr)
|
|
533
549
|
|
|
534
550
|
def as_data_object(self, agg_path: str, series_id: str = "") -> 'pandas.DataFrame':
|
|
551
|
+
"""
|
|
552
|
+
Loads the Time Series aggregation to a pandas DataFrame object
|
|
553
|
+
:param agg_path: the path to the Time Series aggregation
|
|
554
|
+
:param series_id: the series id of the time series to retrieve
|
|
555
|
+
:return: the Time Series aggregation as a pandas DataFrame object
|
|
556
|
+
"""
|
|
535
557
|
if pandas is None:
|
|
536
558
|
raise Exception("pandas package not found")
|
|
537
559
|
|
|
@@ -548,6 +570,14 @@ class TimeseriesAggregation(DataObjectSupportingAggregation):
|
|
|
548
570
|
def save_data_object(self, resource: 'Resource', agg_path: str, as_new_aggr: bool = False,
|
|
549
571
|
destination_path: str = "") -> 'Aggregation':
|
|
550
572
|
|
|
573
|
+
"""
|
|
574
|
+
Saves the pandas DataFrame object to the Time Series aggregation
|
|
575
|
+
:param resource: the resource containing the aggregation
|
|
576
|
+
:param agg_path: the path to the Time Series aggregation
|
|
577
|
+
:param as_new_aggr: Defaults False, set to True to create a new Time Series aggregation
|
|
578
|
+
:param destination_path: the destination path in Hydroshare to save the new aggregation
|
|
579
|
+
:return: the updated or new Time Series aggregation
|
|
580
|
+
"""
|
|
551
581
|
self._validate_aggregation_for_update(resource, AggregationType.TimeSeriesAggregation)
|
|
552
582
|
file_path = self._validate_aggregation_path(agg_path, for_save_data=True)
|
|
553
583
|
with closing(sqlite3.connect(file_path)) as conn:
|
|
@@ -597,7 +627,7 @@ class TimeseriesAggregation(DataObjectSupportingAggregation):
|
|
|
597
627
|
|
|
598
628
|
|
|
599
629
|
class GeoFeatureAggregation(DataObjectSupportingAggregation):
|
|
600
|
-
|
|
630
|
+
"""Represents a Geo Feature Aggregation in HydroShare"""
|
|
601
631
|
@classmethod
|
|
602
632
|
def create(cls, base_aggr):
|
|
603
633
|
return super().create(aggr_cls=cls, base_aggr=base_aggr)
|
|
@@ -616,12 +646,25 @@ class GeoFeatureAggregation(DataObjectSupportingAggregation):
|
|
|
616
646
|
return file_path
|
|
617
647
|
|
|
618
648
|
def as_data_object(self, agg_path: str) -> 'fiona.Collection':
|
|
649
|
+
"""
|
|
650
|
+
Loads the Geo Feature aggregation to a fiona Collection object
|
|
651
|
+
:param agg_path: the path to the Geo Feature aggregation
|
|
652
|
+
:return: the Geo Feature aggregation as a fiona Collection object
|
|
653
|
+
"""
|
|
619
654
|
if fiona is None:
|
|
620
655
|
raise Exception("fiona package was not found")
|
|
621
656
|
return self._get_data_object(agg_path=agg_path, func=fiona.open)
|
|
622
657
|
|
|
623
658
|
def save_data_object(self, resource: 'Resource', agg_path: str, as_new_aggr: bool = False,
|
|
624
659
|
destination_path: str = "") -> 'Aggregation':
|
|
660
|
+
"""
|
|
661
|
+
Saves the fiona Collection object to the Geo Feature aggregation
|
|
662
|
+
:param resource: the resource containing the aggregation
|
|
663
|
+
:param agg_path: the path to the Geo Feature aggregation
|
|
664
|
+
:param as_new_aggr: Defaults False, set to True to create a new Geo Feature aggregation
|
|
665
|
+
:param destination_path: the destination path in Hydroshare to save the new aggregation
|
|
666
|
+
:return: the updated or new Geo Feature aggregation
|
|
667
|
+
"""
|
|
625
668
|
def upload_shape_files(main_file_path, dst_path=""):
|
|
626
669
|
shp_file_dir_path = os.path.dirname(main_file_path)
|
|
627
670
|
filename_starts_with = f"{pathlib.Path(main_file_path).stem}."
|
|
@@ -685,7 +728,7 @@ class GeoFeatureAggregation(DataObjectSupportingAggregation):
|
|
|
685
728
|
|
|
686
729
|
|
|
687
730
|
class GeoRasterAggregation(DataObjectSupportingAggregation):
|
|
688
|
-
|
|
731
|
+
"""Represents a Geo Raster Aggregation in HydroShare"""
|
|
689
732
|
@classmethod
|
|
690
733
|
def create(cls, base_aggr):
|
|
691
734
|
return super().create(aggr_cls=cls, base_aggr=base_aggr)
|
|
@@ -741,12 +784,25 @@ class GeoRasterAggregation(DataObjectSupportingAggregation):
|
|
|
741
784
|
return file_path
|
|
742
785
|
|
|
743
786
|
def as_data_object(self, agg_path: str) -> 'rasterio.DatasetReader':
|
|
787
|
+
"""
|
|
788
|
+
Loads the Geo Raster aggregation to a rasterio DatasetReader object
|
|
789
|
+
:param agg_path: the path to the Geo Raster aggregation
|
|
790
|
+
:return: the Geo Raster aggregation as a rasterio DatasetReader object
|
|
791
|
+
"""
|
|
744
792
|
if rasterio is None:
|
|
745
793
|
raise Exception("rasterio package was not found")
|
|
746
794
|
return self._get_data_object(agg_path=agg_path, func=rasterio.open)
|
|
747
795
|
|
|
748
796
|
def save_data_object(self, resource: 'Resource', agg_path: str, as_new_aggr: bool = False,
|
|
749
797
|
destination_path: str = "") -> 'Aggregation':
|
|
798
|
+
"""
|
|
799
|
+
Saves the rasterio DatasetReader object to the Geo Raster aggregation
|
|
800
|
+
:param resource: the resource containing the aggregation
|
|
801
|
+
:param agg_path: the path to the Geo Raster aggregation
|
|
802
|
+
:param as_new_aggr: Defaults False, set to True to create a new Geo Raster aggregation
|
|
803
|
+
:param destination_path: the destination path in Hydroshare to save the new aggregation
|
|
804
|
+
:return: the updated or new Geo Raster aggregation
|
|
805
|
+
"""
|
|
750
806
|
def upload_raster_files(dst_path=""):
|
|
751
807
|
raster_files = []
|
|
752
808
|
for item in os.listdir(agg_path):
|
|
@@ -808,7 +864,7 @@ class Resource(Aggregation):
|
|
|
808
864
|
|
|
809
865
|
@property
|
|
810
866
|
def _hsapi_path(self):
|
|
811
|
-
path = urlparse(self.metadata.identifier).path
|
|
867
|
+
path = urlparse(str(self.metadata.identifier)).path
|
|
812
868
|
return '/hsapi' + path
|
|
813
869
|
|
|
814
870
|
def _upload(self, file, destination_path):
|
|
@@ -1308,9 +1364,9 @@ class HydroShareSession:
|
|
|
1308
1364
|
of OAuth2 token dropping optional fields that are None."""
|
|
1309
1365
|
if isinstance(token, dict) or isinstance(token, Token):
|
|
1310
1366
|
# try to coerce into Token model
|
|
1311
|
-
o = Token.
|
|
1367
|
+
o = Token.model_validate(token)
|
|
1312
1368
|
# drop None fields from output
|
|
1313
|
-
return o.
|
|
1369
|
+
return o.model_dump(exclude_none=True)
|
|
1314
1370
|
else:
|
|
1315
1371
|
error_message = "token must be hsclient.Token or dictionary following schema:\n" "{}".format(
|
|
1316
1372
|
pformat(Token.__annotations__, sort_dicts=False)
|
|
@@ -1446,15 +1502,19 @@ class HydroShare:
|
|
|
1446
1502
|
if to_date:
|
|
1447
1503
|
params["to_date"] = to_date.strftime('%Y-%m-%d')
|
|
1448
1504
|
if spatial_coverage:
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1505
|
+
yield Exception(
|
|
1506
|
+
"Bad Request, status_code 400, spatial_coverage queries are disabled."
|
|
1507
|
+
)
|
|
1508
|
+
# TODO: re-enable after resolution of https://github.com/hydroshare/hydroshare/issues/5240
|
|
1509
|
+
# params["coverage_type"] = spatial_coverage.type
|
|
1510
|
+
# if spatial_coverage.type == "point":
|
|
1511
|
+
# params["north"] = spatial_coverage.north
|
|
1512
|
+
# params["east"] = spatial_coverage.east
|
|
1513
|
+
# else:
|
|
1514
|
+
# params["north"] = spatial_coverage.northlimit
|
|
1515
|
+
# params["east"] = spatial_coverage.eastlimit
|
|
1516
|
+
# params["south"] = spatial_coverage.southlimit
|
|
1517
|
+
# params["west"] = spatial_coverage.westlimit
|
|
1458
1518
|
response = self._hs_session.get("/hsapi/resource/", 200, params=params)
|
|
1459
1519
|
|
|
1460
1520
|
res = response.json()
|
|
@@ -2,7 +2,7 @@ from datetime import datetime
|
|
|
2
2
|
from typing import Dict, List, Tuple
|
|
3
3
|
|
|
4
4
|
from hsmodels.schemas.enums import UserIdentifierType
|
|
5
|
-
from pydantic import AnyUrl, BaseModel,
|
|
5
|
+
from pydantic import AnyUrl, BaseModel, HttpUrl, field_validator
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
class User(BaseModel):
|
|
@@ -12,13 +12,13 @@ class User(BaseModel):
|
|
|
12
12
|
phone: str = None
|
|
13
13
|
address: str = None
|
|
14
14
|
organization: str = None
|
|
15
|
-
website:
|
|
16
|
-
identifiers: Dict[UserIdentifierType,
|
|
15
|
+
website: HttpUrl = None
|
|
16
|
+
identifiers: Dict[UserIdentifierType, AnyUrl] = {}
|
|
17
17
|
type: str = None
|
|
18
18
|
subject_areas: List[str] = []
|
|
19
19
|
date_joined: datetime = None
|
|
20
20
|
|
|
21
|
-
@
|
|
21
|
+
@field_validator("subject_areas", mode='before')
|
|
22
22
|
def split_subject_areas(cls, value):
|
|
23
23
|
return value.split(", ") if value else []
|
|
24
24
|
|
|
@@ -43,7 +43,7 @@ class ResourcePreview(BaseModel):
|
|
|
43
43
|
resource_map_url: str = None
|
|
44
44
|
resource_metadata_url: str = None
|
|
45
45
|
|
|
46
|
-
@
|
|
46
|
+
@field_validator("authors", mode='before')
|
|
47
47
|
def handle_null_author(cls, v):
|
|
48
48
|
# return empty list when supplied authors field is None.
|
|
49
49
|
if v is None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: hsclient
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 1.0.0
|
|
4
4
|
Summary: A python client for managing HydroShare resources
|
|
5
5
|
Home-page: https://github.com/hydroshare/hsclient
|
|
6
6
|
Author: Scott Black
|
|
@@ -13,12 +13,25 @@ Classifier: Programming Language :: Python :: 3.8
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.9
|
|
14
14
|
Requires-Python: >=3.9
|
|
15
15
|
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE.txt
|
|
17
|
+
Requires-Dist: hsmodels>=1.0.0
|
|
18
|
+
Requires-Dist: requests
|
|
19
|
+
Requires-Dist: requests_oauthlib
|
|
16
20
|
Provides-Extra: pandas
|
|
21
|
+
Requires-Dist: pandas; extra == "pandas"
|
|
17
22
|
Provides-Extra: xarray
|
|
23
|
+
Requires-Dist: netCDF4; extra == "xarray"
|
|
24
|
+
Requires-Dist: xarray; extra == "xarray"
|
|
18
25
|
Provides-Extra: rasterio
|
|
26
|
+
Requires-Dist: rasterio; extra == "rasterio"
|
|
19
27
|
Provides-Extra: fiona
|
|
28
|
+
Requires-Dist: fiona; extra == "fiona"
|
|
20
29
|
Provides-Extra: all
|
|
21
|
-
|
|
30
|
+
Requires-Dist: pandas; extra == "all"
|
|
31
|
+
Requires-Dist: netCDF4; extra == "all"
|
|
32
|
+
Requires-Dist: xarray; extra == "all"
|
|
33
|
+
Requires-Dist: rasterio; extra == "all"
|
|
34
|
+
Requires-Dist: fiona; extra == "all"
|
|
22
35
|
|
|
23
36
|
# hsclient
|
|
24
37
|
A python client for interacting with HydroShare in an object oriented way.
|
|
@@ -11,4 +11,7 @@ hsclient.egg-info/PKG-INFO
|
|
|
11
11
|
hsclient.egg-info/SOURCES.txt
|
|
12
12
|
hsclient.egg-info/dependency_links.txt
|
|
13
13
|
hsclient.egg-info/requires.txt
|
|
14
|
-
hsclient.egg-info/top_level.txt
|
|
14
|
+
hsclient.egg-info/top_level.txt
|
|
15
|
+
tests/test_data_objects.py
|
|
16
|
+
tests/test_functional.py
|
|
17
|
+
tests/test_json_models.py
|
|
@@ -5,11 +5,11 @@ README = (pathlib.Path(__file__).parent / "README.md").read_text()
|
|
|
5
5
|
|
|
6
6
|
setup(
|
|
7
7
|
name='hsclient',
|
|
8
|
-
version='0.
|
|
8
|
+
version='1.0.0',
|
|
9
9
|
packages=find_packages(include=['hsclient', 'hsclient.*'],
|
|
10
10
|
exclude=("tests",)),
|
|
11
11
|
install_requires=[
|
|
12
|
-
'hsmodels>=0.
|
|
12
|
+
'hsmodels>=1.0.0',
|
|
13
13
|
'requests',
|
|
14
14
|
'requests_oauthlib',
|
|
15
15
|
],
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import pathlib
|
|
3
|
+
import tempfile
|
|
4
|
+
|
|
5
|
+
import fiona
|
|
6
|
+
import pytest
|
|
7
|
+
import rasterio
|
|
8
|
+
from fiona.model import to_dict
|
|
9
|
+
from hsmodels.schemas.enums import AggregationType
|
|
10
|
+
from rasterio.windows import Window
|
|
11
|
+
|
|
12
|
+
from hsclient import GeoFeatureAggregation, GeoRasterAggregation, NetCDFAggregation, TimeseriesAggregation
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@pytest.mark.parametrize("search_by", ["type", "file_path"])
|
|
16
|
+
def test_timeseries_as_data_object(timeseries_resource, search_by):
|
|
17
|
+
timeseries_resource.refresh()
|
|
18
|
+
if search_by == 'type':
|
|
19
|
+
aggr = timeseries_resource.aggregation(type=AggregationType.TimeSeriesAggregation)
|
|
20
|
+
else:
|
|
21
|
+
file_path = "ODM2_Multi_Site_One_Variable.sqlite"
|
|
22
|
+
aggr = timeseries_resource.aggregation(file__path=file_path)
|
|
23
|
+
|
|
24
|
+
assert type(aggr) is TimeseriesAggregation
|
|
25
|
+
series_result = next(
|
|
26
|
+
r for r in aggr.metadata.time_series_results if r.series_id == "2837b7d9-1ebc-11e6-a16e-f45c8999816f"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
30
|
+
# download aggregation
|
|
31
|
+
unzip_to = os.path.join(tmp, "unzipped_aggr")
|
|
32
|
+
os.makedirs(unzip_to)
|
|
33
|
+
agg_path = timeseries_resource.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
|
|
34
|
+
# load timseries data for the specified series to pandas DataFrame
|
|
35
|
+
pd_dataframe = aggr.as_data_object(agg_path=agg_path, series_id=series_result.series_id)
|
|
36
|
+
assert len(pd_dataframe) == 1333
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@pytest.mark.parametrize("as_new_aggr", [False, True])
|
|
40
|
+
def test_timeseries_save_data_object(timeseries_resource, as_new_aggr):
|
|
41
|
+
timeseries_resource.refresh()
|
|
42
|
+
file_path = "ODM2_Multi_Site_One_Variable.sqlite"
|
|
43
|
+
aggr = timeseries_resource.aggregation(file__path=file_path)
|
|
44
|
+
assert aggr is not None
|
|
45
|
+
assert type(aggr) is TimeseriesAggregation
|
|
46
|
+
series_id = '4a6f095c-1ebc-11e6-8a10-f45c8999816f'
|
|
47
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
48
|
+
# download aggregation
|
|
49
|
+
unzip_to = os.path.join(tmp, "unzipped_aggr")
|
|
50
|
+
os.makedirs(unzip_to)
|
|
51
|
+
agg_path = timeseries_resource.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
|
|
52
|
+
pd_dataframe = aggr.as_data_object(agg_path=agg_path, series_id=series_id)
|
|
53
|
+
assert pd_dataframe.__class__.__name__ == "DataFrame"
|
|
54
|
+
rows, columns = pd_dataframe.shape
|
|
55
|
+
# delete 10 rows
|
|
56
|
+
pd_dataframe.drop(pd_dataframe.index[0:10], axis=0, inplace=True)
|
|
57
|
+
dst_path = ""
|
|
58
|
+
if as_new_aggr:
|
|
59
|
+
dst_path = "raster_aggr_folder"
|
|
60
|
+
timeseries_resource.folder_create(dst_path)
|
|
61
|
+
|
|
62
|
+
aggr = aggr.save_data_object(resource=timeseries_resource, agg_path=agg_path, as_new_aggr=as_new_aggr,
|
|
63
|
+
destination_path=dst_path)
|
|
64
|
+
assert type(aggr) is TimeseriesAggregation
|
|
65
|
+
|
|
66
|
+
# check the updated/new timeseries aggregation
|
|
67
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
68
|
+
# download aggregation
|
|
69
|
+
unzip_to = os.path.join(tmp, "unzipped_aggr")
|
|
70
|
+
os.makedirs(unzip_to)
|
|
71
|
+
if as_new_aggr:
|
|
72
|
+
file_path = f"{dst_path}/{file_path}"
|
|
73
|
+
|
|
74
|
+
aggr = timeseries_resource.aggregation(file__path=file_path)
|
|
75
|
+
assert aggr is not None
|
|
76
|
+
assert type(aggr) is TimeseriesAggregation
|
|
77
|
+
|
|
78
|
+
agg_path = timeseries_resource.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
|
|
79
|
+
pd_dataframe = aggr.as_data_object(agg_path=agg_path, series_id=series_id)
|
|
80
|
+
updated_rows, update_columns = pd_dataframe.shape
|
|
81
|
+
assert rows == updated_rows + 10
|
|
82
|
+
assert columns == update_columns
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@pytest.mark.parametrize("search_by", ["type", "file_path"])
|
|
86
|
+
def test_raster_as_data_object(resource_with_raster_aggr, search_by):
|
|
87
|
+
resource_with_raster_aggr.refresh()
|
|
88
|
+
if search_by == "type":
|
|
89
|
+
aggr = resource_with_raster_aggr.aggregation(type=AggregationType.GeographicRasterAggregation)
|
|
90
|
+
else:
|
|
91
|
+
file_path = "logan.vrt"
|
|
92
|
+
aggr = resource_with_raster_aggr.aggregation(file__path=file_path)
|
|
93
|
+
|
|
94
|
+
assert type(aggr) is GeoRasterAggregation
|
|
95
|
+
|
|
96
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
97
|
+
# download aggregation
|
|
98
|
+
unzip_to = os.path.join(tmp, "unzipped_aggr")
|
|
99
|
+
os.makedirs(unzip_to)
|
|
100
|
+
agg_path = resource_with_raster_aggr.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
|
|
101
|
+
|
|
102
|
+
dataset = aggr.as_data_object(agg_path=agg_path)
|
|
103
|
+
assert dataset.__class__.__name__ == "DatasetReader"
|
|
104
|
+
# raster should have 1 band
|
|
105
|
+
assert dataset.count == 1
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@pytest.mark.parametrize("as_new_aggr", [False, True])
|
|
109
|
+
def test_raster_save_data_object(resource_with_raster_aggr, as_new_aggr):
|
|
110
|
+
resource_with_raster_aggr.refresh()
|
|
111
|
+
file_path = "logan.vrt"
|
|
112
|
+
aggr = resource_with_raster_aggr.aggregation(file__path=file_path)
|
|
113
|
+
assert aggr is not None
|
|
114
|
+
assert type(aggr) is GeoRasterAggregation
|
|
115
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
116
|
+
# download aggregation
|
|
117
|
+
unzip_to = os.path.join(tmp, "unzipped_aggr")
|
|
118
|
+
os.makedirs(unzip_to)
|
|
119
|
+
agg_path = resource_with_raster_aggr.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
|
|
120
|
+
rasterio_reader = aggr.as_data_object(agg_path=agg_path)
|
|
121
|
+
assert rasterio_reader.__class__.__name__ == "DatasetReader"
|
|
122
|
+
# edit raster data - sub-setting the dataset
|
|
123
|
+
new_width = rasterio_reader.width - 9
|
|
124
|
+
new_height = rasterio_reader.height - 10
|
|
125
|
+
updated_width = new_width
|
|
126
|
+
updated_height = new_height
|
|
127
|
+
subset_window = Window(0, 0, new_width, new_height)
|
|
128
|
+
subset_band = rasterio_reader.read(1, window=subset_window)
|
|
129
|
+
output_raster_dir_path = os.path.join(tmp, "updated_aggr")
|
|
130
|
+
os.makedirs(output_raster_dir_path)
|
|
131
|
+
update_raster_filename = "updated_logan.tif"
|
|
132
|
+
output_raster_file_path = os.path.join(output_raster_dir_path, update_raster_filename)
|
|
133
|
+
profile = rasterio_reader.profile
|
|
134
|
+
rasterio_reader.close()
|
|
135
|
+
profile['driver'] = "GTiff"
|
|
136
|
+
profile['width'] = new_width
|
|
137
|
+
profile['height'] = new_height
|
|
138
|
+
|
|
139
|
+
with rasterio.open(output_raster_file_path, "w", **profile) as dst:
|
|
140
|
+
dst.write(subset_band, 1)
|
|
141
|
+
|
|
142
|
+
dst_path = ""
|
|
143
|
+
if as_new_aggr:
|
|
144
|
+
dst_path = "raster_aggr_folder"
|
|
145
|
+
resource_with_raster_aggr.folder_create(dst_path)
|
|
146
|
+
|
|
147
|
+
# save the new tif file to update the aggregation or create a new aggregation
|
|
148
|
+
aggr = aggr.save_data_object(resource=resource_with_raster_aggr, agg_path=output_raster_dir_path,
|
|
149
|
+
as_new_aggr=as_new_aggr, destination_path=dst_path)
|
|
150
|
+
assert aggr is not None
|
|
151
|
+
assert type(aggr) is GeoRasterAggregation
|
|
152
|
+
|
|
153
|
+
# check the updated raster aggregation
|
|
154
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
155
|
+
# download aggregation
|
|
156
|
+
unzip_to = os.path.join(tmp, "unzipped_aggr")
|
|
157
|
+
os.makedirs(unzip_to)
|
|
158
|
+
file_path = pathlib.Path(update_raster_filename).stem + ".vrt"
|
|
159
|
+
if as_new_aggr:
|
|
160
|
+
file_path = f"{dst_path}/{file_path}"
|
|
161
|
+
|
|
162
|
+
aggr = resource_with_raster_aggr.aggregation(file__path=file_path)
|
|
163
|
+
assert aggr is not None
|
|
164
|
+
assert type(aggr) is GeoRasterAggregation
|
|
165
|
+
|
|
166
|
+
agg_path = resource_with_raster_aggr.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
|
|
167
|
+
rasterio_reader = aggr.as_data_object(agg_path=agg_path)
|
|
168
|
+
assert updated_height == rasterio_reader.height
|
|
169
|
+
assert updated_width == rasterio_reader.width
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@pytest.mark.parametrize("search_by", ["type", "file_path"])
|
|
173
|
+
def test_netcdf_as_data_object(resource_with_netcdf_aggr, search_by):
|
|
174
|
+
resource_with_netcdf_aggr.refresh()
|
|
175
|
+
if search_by == 'type':
|
|
176
|
+
aggr = resource_with_netcdf_aggr.aggregation(type=AggregationType.MultidimensionalAggregation)
|
|
177
|
+
else:
|
|
178
|
+
file_path = "SWE_time.nc"
|
|
179
|
+
aggr = resource_with_netcdf_aggr.aggregation(file__path=file_path)
|
|
180
|
+
|
|
181
|
+
assert type(aggr) is NetCDFAggregation
|
|
182
|
+
|
|
183
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
184
|
+
# download aggregation
|
|
185
|
+
unzip_to = os.path.join(tmp, "unzipped_aggr")
|
|
186
|
+
os.makedirs(unzip_to)
|
|
187
|
+
agg_path = resource_with_netcdf_aggr.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
|
|
188
|
+
dataset = aggr.as_data_object(agg_path=agg_path)
|
|
189
|
+
assert dataset.__class__.__name__ == "Dataset"
|
|
190
|
+
# netcdf dimensions
|
|
191
|
+
assert dataset.dims['time'] == 2184
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
@pytest.mark.parametrize("as_new_aggr", [False, True])
|
|
195
|
+
def test_netcdf_save_data_object(resource_with_netcdf_aggr, as_new_aggr):
|
|
196
|
+
resource_with_netcdf_aggr.refresh()
|
|
197
|
+
file_path = "SWE_time.nc"
|
|
198
|
+
aggr = resource_with_netcdf_aggr.aggregation(file__path=file_path)
|
|
199
|
+
assert aggr is not None
|
|
200
|
+
assert type(aggr) is NetCDFAggregation
|
|
201
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
202
|
+
# download aggregation
|
|
203
|
+
unzip_to = os.path.join(tmp, "unzipped_aggr")
|
|
204
|
+
os.makedirs(unzip_to)
|
|
205
|
+
agg_path = resource_with_netcdf_aggr.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
|
|
206
|
+
xr_dataset = aggr.as_data_object(agg_path=agg_path)
|
|
207
|
+
assert xr_dataset.__class__.__name__ == "Dataset"
|
|
208
|
+
agg_title = "This is a modified title for this aggregation by hsclient"
|
|
209
|
+
xr_dataset.attrs["title"] = agg_title
|
|
210
|
+
dst_path = ""
|
|
211
|
+
if as_new_aggr:
|
|
212
|
+
dst_path = "netcdf_aggr_folder"
|
|
213
|
+
resource_with_netcdf_aggr.folder_create(dst_path)
|
|
214
|
+
|
|
215
|
+
aggr = aggr.save_data_object(resource=resource_with_netcdf_aggr, agg_path=agg_path, as_new_aggr=as_new_aggr,
|
|
216
|
+
destination_path=dst_path)
|
|
217
|
+
|
|
218
|
+
assert type(aggr) is NetCDFAggregation
|
|
219
|
+
xr_dataset = aggr.as_data_object(agg_path=agg_path)
|
|
220
|
+
assert xr_dataset.attrs["title"] == agg_title
|
|
221
|
+
|
|
222
|
+
# check the updated/new netcdf aggregation
|
|
223
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
224
|
+
# download aggregation
|
|
225
|
+
unzip_to = os.path.join(tmp, "unzipped_aggr")
|
|
226
|
+
os.makedirs(unzip_to)
|
|
227
|
+
if as_new_aggr:
|
|
228
|
+
file_path = f"{dst_path}/{file_path}"
|
|
229
|
+
|
|
230
|
+
aggr = resource_with_netcdf_aggr.aggregation(file__path=file_path)
|
|
231
|
+
assert aggr is not None
|
|
232
|
+
assert type(aggr) is NetCDFAggregation
|
|
233
|
+
|
|
234
|
+
agg_path = resource_with_netcdf_aggr.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
|
|
235
|
+
xr_dataset = aggr.as_data_object(agg_path=agg_path)
|
|
236
|
+
assert xr_dataset.attrs["title"] == agg_title
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
@pytest.mark.parametrize("search_by", ["type", "file_path"])
|
|
240
|
+
def test_geofeature_as_data_object(resource_with_geofeature_aggr, search_by):
|
|
241
|
+
resource_with_geofeature_aggr.refresh()
|
|
242
|
+
if search_by == "type":
|
|
243
|
+
aggr = resource_with_geofeature_aggr.aggregation(type=AggregationType.GeographicFeatureAggregation)
|
|
244
|
+
else:
|
|
245
|
+
file_path = "watersheds.shp"
|
|
246
|
+
aggr = resource_with_geofeature_aggr.aggregation(file__path=file_path)
|
|
247
|
+
|
|
248
|
+
assert aggr is not None
|
|
249
|
+
assert type(aggr) is GeoFeatureAggregation
|
|
250
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
251
|
+
# download aggregation
|
|
252
|
+
unzip_to = os.path.join(tmp, "unzipped_aggr")
|
|
253
|
+
os.makedirs(unzip_to)
|
|
254
|
+
agg_path = resource_with_geofeature_aggr.aggregation_download(aggregation=aggr, save_path=tmp,
|
|
255
|
+
unzip_to=unzip_to)
|
|
256
|
+
fn_collection = aggr.as_data_object(agg_path=agg_path)
|
|
257
|
+
assert fn_collection.__class__.__name__ == "Collection"
|
|
258
|
+
# check projection type
|
|
259
|
+
assert str(fn_collection.crs) == "EPSG:26912"
|
|
260
|
+
# close the fiona collection object so that the temp dir can be cleaned up.
|
|
261
|
+
fn_collection.close()
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
@pytest.mark.parametrize("as_new_aggr", [False, True])
|
|
265
|
+
def test_geofeature_save_data_object(resource_with_geofeature_aggr, as_new_aggr):
|
|
266
|
+
resource_with_geofeature_aggr.refresh()
|
|
267
|
+
file_path = "watersheds.shp"
|
|
268
|
+
aggr = resource_with_geofeature_aggr.aggregation(file__path=file_path)
|
|
269
|
+
assert aggr is not None
|
|
270
|
+
assert type(aggr) is GeoFeatureAggregation
|
|
271
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
272
|
+
# download aggregation
|
|
273
|
+
unzip_to = os.path.join(tmp, "unzipped_aggr")
|
|
274
|
+
os.makedirs(unzip_to)
|
|
275
|
+
agg_path = resource_with_geofeature_aggr.aggregation_download(aggregation=aggr, save_path=tmp,
|
|
276
|
+
unzip_to=unzip_to)
|
|
277
|
+
fn_collection = aggr.as_data_object(agg_path=agg_path)
|
|
278
|
+
assert fn_collection.__class__.__name__ == "Collection"
|
|
279
|
+
original_shp_filename = os.path.basename(fn_collection.path)
|
|
280
|
+
updated_shp_file_dir = os.path.join(tmp, "updated_aggr")
|
|
281
|
+
os.makedirs(updated_shp_file_dir)
|
|
282
|
+
output_shp_file_path = os.path.join(updated_shp_file_dir, original_shp_filename)
|
|
283
|
+
with fiona.open(output_shp_file_path, 'w', schema=fn_collection.schema, driver=fn_collection.driver,
|
|
284
|
+
crs=fn_collection.crs) as out_shp_file:
|
|
285
|
+
for feature in fn_collection:
|
|
286
|
+
ft_dict = to_dict(feature)
|
|
287
|
+
if ft_dict['properties']['Id'] < 5:
|
|
288
|
+
out_shp_file.write(feature)
|
|
289
|
+
|
|
290
|
+
dst_path = ""
|
|
291
|
+
if as_new_aggr:
|
|
292
|
+
dst_path = "geo_aggr_folder"
|
|
293
|
+
resource_with_geofeature_aggr.folder_create(dst_path)
|
|
294
|
+
|
|
295
|
+
aggr = aggr.save_data_object(resource=resource_with_geofeature_aggr, agg_path=updated_shp_file_dir,
|
|
296
|
+
as_new_aggr=as_new_aggr, destination_path=dst_path)
|
|
297
|
+
assert aggr is not None
|
|
298
|
+
assert type(aggr) is GeoFeatureAggregation
|
|
299
|
+
assert aggr.data_object is None
|
|
300
|
+
|
|
301
|
+
# check the updated geo-feature aggregation
|
|
302
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
303
|
+
# download aggregation
|
|
304
|
+
unzip_to = os.path.join(tmp, "unzipped_aggr")
|
|
305
|
+
os.makedirs(unzip_to)
|
|
306
|
+
if as_new_aggr:
|
|
307
|
+
file_path = f"{dst_path}/{file_path}"
|
|
308
|
+
|
|
309
|
+
aggr = resource_with_geofeature_aggr.aggregation(file__path=file_path)
|
|
310
|
+
assert aggr is not None
|
|
311
|
+
assert type(aggr) is GeoFeatureAggregation
|
|
312
|
+
|
|
313
|
+
agg_path = resource_with_geofeature_aggr.aggregation_download(aggregation=aggr, save_path=tmp,
|
|
314
|
+
unzip_to=unzip_to)
|
|
315
|
+
fn_collection = aggr.as_data_object(agg_path=agg_path)
|
|
316
|
+
for feature in fn_collection:
|
|
317
|
+
ft_dict = to_dict(feature)
|
|
318
|
+
assert ft_dict['properties']['Id'] < 5
|
|
319
|
+
# need to close the data object so that the tmp directory can be cleaned up
|
|
320
|
+
fn_collection.close()
|