hsclient 0.3.4__tar.gz → 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: hsclient
3
- Version: 0.3.4
3
+ Version: 1.0.0
4
4
  Summary: A python client for managing HydroShare resources
5
5
  Home-page: https://github.com/hydroshare/hsclient
6
6
  Author: Scott Black
@@ -13,12 +13,25 @@ Classifier: Programming Language :: Python :: 3.8
13
13
  Classifier: Programming Language :: Python :: 3.9
14
14
  Requires-Python: >=3.9
15
15
  Description-Content-Type: text/markdown
16
+ License-File: LICENSE.txt
17
+ Requires-Dist: hsmodels>=1.0.0
18
+ Requires-Dist: requests
19
+ Requires-Dist: requests_oauthlib
16
20
  Provides-Extra: pandas
21
+ Requires-Dist: pandas; extra == "pandas"
17
22
  Provides-Extra: xarray
23
+ Requires-Dist: netCDF4; extra == "xarray"
24
+ Requires-Dist: xarray; extra == "xarray"
18
25
  Provides-Extra: rasterio
26
+ Requires-Dist: rasterio; extra == "rasterio"
19
27
  Provides-Extra: fiona
28
+ Requires-Dist: fiona; extra == "fiona"
20
29
  Provides-Extra: all
21
- License-File: LICENSE.txt
30
+ Requires-Dist: pandas; extra == "all"
31
+ Requires-Dist: netCDF4; extra == "all"
32
+ Requires-Dist: xarray; extra == "all"
33
+ Requires-Dist: rasterio; extra == "all"
34
+ Requires-Dist: fiona; extra == "all"
22
35
 
23
36
  # hsclient
24
37
  A python client for interacting with HydroShare in an object oriented way.
@@ -393,7 +393,7 @@ class DataObjectSupportingAggregation(Aggregation):
393
393
 
394
394
  @staticmethod
395
395
  def create(aggr_cls, base_aggr):
396
- """creates a type specific aggregation object from an instance of Aggregation"""
396
+ """Creates a type specific aggregation object from an instance of Aggregation"""
397
397
  aggr = aggr_cls(base_aggr._map_path, base_aggr._hs_session, base_aggr._parsed_checksums)
398
398
  aggr._retrieved_map = base_aggr._retrieved_map
399
399
  aggr._retrieved_metadata = base_aggr._retrieved_metadata
@@ -410,6 +410,8 @@ class DataObjectSupportingAggregation(Aggregation):
410
410
  @property
411
411
  def data_object(self) -> \
412
412
  Union['pandas.DataFrame', 'fiona.Collection', 'rasterio.DatasetReader', 'xarray.Dataset', None]:
413
+ """Returns the data object for the aggregation if the aggregation has been loaded as
414
+ a data object, otherwise None"""
413
415
  return self._data_object
414
416
 
415
417
  def _get_file_path(self, agg_path):
@@ -478,18 +480,32 @@ class DataObjectSupportingAggregation(Aggregation):
478
480
 
479
481
 
480
482
  class NetCDFAggregation(DataObjectSupportingAggregation):
483
+ """Represents a Multidimensional Aggregation in HydroShare"""
481
484
 
482
485
  @classmethod
483
486
  def create(cls, base_aggr):
484
487
  return super().create(aggr_cls=cls, base_aggr=base_aggr)
485
488
 
486
489
  def as_data_object(self, agg_path: str) -> 'xarray.Dataset':
490
+ """
491
+ Loads the Multidimensional aggregation to a xarray Dataset object
492
+ :param agg_path: the path to the Multidimensional aggregation
493
+ :return: the Multidimensional aggregation as a xarray Dataset object
494
+ """
487
495
  if xarray is None:
488
496
  raise Exception("xarray package was not found")
489
497
  return self._get_data_object(agg_path=agg_path, func=xarray.open_dataset)
490
498
 
491
499
  def save_data_object(self, resource: 'Resource', agg_path: str, as_new_aggr: bool = False,
492
500
  destination_path: str = "") -> 'Aggregation':
501
+ """
502
+ Saves the xarray Dataset object to the Multidimensional aggregation
503
+ :param resource: the resource containing the aggregation
504
+ :param agg_path: the path to the Multidimensional aggregation
505
+ :param as_new_aggr: Defaults False, set to True to create a new Multidimensional aggregation
506
+ :param destination_path: the destination path in Hydroshare to save the new aggregation
507
+ :return: the updated or new Multidimensional aggregation
508
+ """
493
509
 
494
510
  self._validate_aggregation_for_update(resource, AggregationType.MultidimensionalAggregation)
495
511
  file_path = self._validate_aggregation_path(agg_path, for_save_data=True)
@@ -526,12 +542,18 @@ class NetCDFAggregation(DataObjectSupportingAggregation):
526
542
 
527
543
 
528
544
  class TimeseriesAggregation(DataObjectSupportingAggregation):
529
-
545
+ """Represents a Time Series Aggregation in HydroShare"""
530
546
  @classmethod
531
547
  def create(cls, base_aggr):
532
548
  return super().create(aggr_cls=cls, base_aggr=base_aggr)
533
549
 
534
550
  def as_data_object(self, agg_path: str, series_id: str = "") -> 'pandas.DataFrame':
551
+ """
552
+ Loads the Time Series aggregation to a pandas DataFrame object
553
+ :param agg_path: the path to the Time Series aggregation
554
+ :param series_id: the series id of the time series to retrieve
555
+ :return: the Time Series aggregation as a pandas DataFrame object
556
+ """
535
557
  if pandas is None:
536
558
  raise Exception("pandas package not found")
537
559
 
@@ -548,6 +570,14 @@ class TimeseriesAggregation(DataObjectSupportingAggregation):
548
570
  def save_data_object(self, resource: 'Resource', agg_path: str, as_new_aggr: bool = False,
549
571
  destination_path: str = "") -> 'Aggregation':
550
572
 
573
+ """
574
+ Saves the pandas DataFrame object to the Time Series aggregation
575
+ :param resource: the resource containing the aggregation
576
+ :param agg_path: the path to the Time Series aggregation
577
+ :param as_new_aggr: Defaults False, set to True to create a new Time Series aggregation
578
+ :param destination_path: the destination path in Hydroshare to save the new aggregation
579
+ :return: the updated or new Time Series aggregation
580
+ """
551
581
  self._validate_aggregation_for_update(resource, AggregationType.TimeSeriesAggregation)
552
582
  file_path = self._validate_aggregation_path(agg_path, for_save_data=True)
553
583
  with closing(sqlite3.connect(file_path)) as conn:
@@ -597,7 +627,7 @@ class TimeseriesAggregation(DataObjectSupportingAggregation):
597
627
 
598
628
 
599
629
  class GeoFeatureAggregation(DataObjectSupportingAggregation):
600
-
630
+ """Represents a Geo Feature Aggregation in HydroShare"""
601
631
  @classmethod
602
632
  def create(cls, base_aggr):
603
633
  return super().create(aggr_cls=cls, base_aggr=base_aggr)
@@ -616,12 +646,25 @@ class GeoFeatureAggregation(DataObjectSupportingAggregation):
616
646
  return file_path
617
647
 
618
648
  def as_data_object(self, agg_path: str) -> 'fiona.Collection':
649
+ """
650
+ Loads the Geo Feature aggregation to a fiona Collection object
651
+ :param agg_path: the path to the Geo Feature aggregation
652
+ :return: the Geo Feature aggregation as a fiona Collection object
653
+ """
619
654
  if fiona is None:
620
655
  raise Exception("fiona package was not found")
621
656
  return self._get_data_object(agg_path=agg_path, func=fiona.open)
622
657
 
623
658
  def save_data_object(self, resource: 'Resource', agg_path: str, as_new_aggr: bool = False,
624
659
  destination_path: str = "") -> 'Aggregation':
660
+ """
661
+ Saves the fiona Collection object to the Geo Feature aggregation
662
+ :param resource: the resource containing the aggregation
663
+ :param agg_path: the path to the Geo Feature aggregation
664
+ :param as_new_aggr: Defaults False, set to True to create a new Geo Feature aggregation
665
+ :param destination_path: the destination path in Hydroshare to save the new aggregation
666
+ :return: the updated or new Geo Feature aggregation
667
+ """
625
668
  def upload_shape_files(main_file_path, dst_path=""):
626
669
  shp_file_dir_path = os.path.dirname(main_file_path)
627
670
  filename_starts_with = f"{pathlib.Path(main_file_path).stem}."
@@ -685,7 +728,7 @@ class GeoFeatureAggregation(DataObjectSupportingAggregation):
685
728
 
686
729
 
687
730
  class GeoRasterAggregation(DataObjectSupportingAggregation):
688
-
731
+ """Represents a Geo Raster Aggregation in HydroShare"""
689
732
  @classmethod
690
733
  def create(cls, base_aggr):
691
734
  return super().create(aggr_cls=cls, base_aggr=base_aggr)
@@ -741,12 +784,25 @@ class GeoRasterAggregation(DataObjectSupportingAggregation):
741
784
  return file_path
742
785
 
743
786
  def as_data_object(self, agg_path: str) -> 'rasterio.DatasetReader':
787
+ """
788
+ Loads the Geo Raster aggregation to a rasterio DatasetReader object
789
+ :param agg_path: the path to the Geo Raster aggregation
790
+ :return: the Geo Raster aggregation as a rasterio DatasetReader object
791
+ """
744
792
  if rasterio is None:
745
793
  raise Exception("rasterio package was not found")
746
794
  return self._get_data_object(agg_path=agg_path, func=rasterio.open)
747
795
 
748
796
  def save_data_object(self, resource: 'Resource', agg_path: str, as_new_aggr: bool = False,
749
797
  destination_path: str = "") -> 'Aggregation':
798
+ """
799
+ Saves the rasterio DatasetReader object to the Geo Raster aggregation
800
+ :param resource: the resource containing the aggregation
801
+ :param agg_path: the path to the Geo Raster aggregation
802
+ :param as_new_aggr: Defaults False, set to True to create a new Geo Raster aggregation
803
+ :param destination_path: the destination path in Hydroshare to save the new aggregation
804
+ :return: the updated or new Geo Raster aggregation
805
+ """
750
806
  def upload_raster_files(dst_path=""):
751
807
  raster_files = []
752
808
  for item in os.listdir(agg_path):
@@ -808,7 +864,7 @@ class Resource(Aggregation):
808
864
 
809
865
  @property
810
866
  def _hsapi_path(self):
811
- path = urlparse(self.metadata.identifier).path
867
+ path = urlparse(str(self.metadata.identifier)).path
812
868
  return '/hsapi' + path
813
869
 
814
870
  def _upload(self, file, destination_path):
@@ -1308,9 +1364,9 @@ class HydroShareSession:
1308
1364
  of OAuth2 token dropping optional fields that are None."""
1309
1365
  if isinstance(token, dict) or isinstance(token, Token):
1310
1366
  # try to coerce into Token model
1311
- o = Token.parse_obj(token)
1367
+ o = Token.model_validate(token)
1312
1368
  # drop None fields from output
1313
- return o.dict(exclude_none=True)
1369
+ return o.model_dump(exclude_none=True)
1314
1370
  else:
1315
1371
  error_message = "token must be hsclient.Token or dictionary following schema:\n" "{}".format(
1316
1372
  pformat(Token.__annotations__, sort_dicts=False)
@@ -1446,15 +1502,19 @@ class HydroShare:
1446
1502
  if to_date:
1447
1503
  params["to_date"] = to_date.strftime('%Y-%m-%d')
1448
1504
  if spatial_coverage:
1449
- params["coverage_type"] = spatial_coverage.type
1450
- if spatial_coverage.type == "point":
1451
- params["north"] = spatial_coverage.north
1452
- params["east"] = spatial_coverage.east
1453
- else:
1454
- params["north"] = spatial_coverage.northlimit
1455
- params["east"] = spatial_coverage.eastlimit
1456
- params["south"] = spatial_coverage.southlimit
1457
- params["west"] = spatial_coverage.westlimit
1505
+ yield Exception(
1506
+ "Bad Request, status_code 400, spatial_coverage queries are disabled."
1507
+ )
1508
+ # TODO: re-enable after resolution of https://github.com/hydroshare/hydroshare/issues/5240
1509
+ # params["coverage_type"] = spatial_coverage.type
1510
+ # if spatial_coverage.type == "point":
1511
+ # params["north"] = spatial_coverage.north
1512
+ # params["east"] = spatial_coverage.east
1513
+ # else:
1514
+ # params["north"] = spatial_coverage.northlimit
1515
+ # params["east"] = spatial_coverage.eastlimit
1516
+ # params["south"] = spatial_coverage.southlimit
1517
+ # params["west"] = spatial_coverage.westlimit
1458
1518
  response = self._hs_session.get("/hsapi/resource/", 200, params=params)
1459
1519
 
1460
1520
  res = response.json()
@@ -2,7 +2,7 @@ from datetime import datetime
2
2
  from typing import Dict, List, Tuple
3
3
 
4
4
  from hsmodels.schemas.enums import UserIdentifierType
5
- from pydantic import AnyUrl, BaseModel, validator
5
+ from pydantic import AnyUrl, BaseModel, HttpUrl, field_validator
6
6
 
7
7
 
8
8
  class User(BaseModel):
@@ -12,13 +12,13 @@ class User(BaseModel):
12
12
  phone: str = None
13
13
  address: str = None
14
14
  organization: str = None
15
- website: str = None
16
- identifiers: Dict[UserIdentifierType, str] = {}
15
+ website: HttpUrl = None
16
+ identifiers: Dict[UserIdentifierType, AnyUrl] = {}
17
17
  type: str = None
18
18
  subject_areas: List[str] = []
19
19
  date_joined: datetime = None
20
20
 
21
- @validator("subject_areas", pre=True)
21
+ @field_validator("subject_areas", mode='before')
22
22
  def split_subject_areas(cls, value):
23
23
  return value.split(", ") if value else []
24
24
 
@@ -43,7 +43,7 @@ class ResourcePreview(BaseModel):
43
43
  resource_map_url: str = None
44
44
  resource_metadata_url: str = None
45
45
 
46
- @validator("authors", pre=True)
46
+ @field_validator("authors", mode='before')
47
47
  def handle_null_author(cls, v):
48
48
  # return empty list when supplied authors field is None.
49
49
  if v is None:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: hsclient
3
- Version: 0.3.4
3
+ Version: 1.0.0
4
4
  Summary: A python client for managing HydroShare resources
5
5
  Home-page: https://github.com/hydroshare/hsclient
6
6
  Author: Scott Black
@@ -13,12 +13,25 @@ Classifier: Programming Language :: Python :: 3.8
13
13
  Classifier: Programming Language :: Python :: 3.9
14
14
  Requires-Python: >=3.9
15
15
  Description-Content-Type: text/markdown
16
+ License-File: LICENSE.txt
17
+ Requires-Dist: hsmodels>=1.0.0
18
+ Requires-Dist: requests
19
+ Requires-Dist: requests_oauthlib
16
20
  Provides-Extra: pandas
21
+ Requires-Dist: pandas; extra == "pandas"
17
22
  Provides-Extra: xarray
23
+ Requires-Dist: netCDF4; extra == "xarray"
24
+ Requires-Dist: xarray; extra == "xarray"
18
25
  Provides-Extra: rasterio
26
+ Requires-Dist: rasterio; extra == "rasterio"
19
27
  Provides-Extra: fiona
28
+ Requires-Dist: fiona; extra == "fiona"
20
29
  Provides-Extra: all
21
- License-File: LICENSE.txt
30
+ Requires-Dist: pandas; extra == "all"
31
+ Requires-Dist: netCDF4; extra == "all"
32
+ Requires-Dist: xarray; extra == "all"
33
+ Requires-Dist: rasterio; extra == "all"
34
+ Requires-Dist: fiona; extra == "all"
22
35
 
23
36
  # hsclient
24
37
  A python client for interacting with HydroShare in an object oriented way.
@@ -11,4 +11,7 @@ hsclient.egg-info/PKG-INFO
11
11
  hsclient.egg-info/SOURCES.txt
12
12
  hsclient.egg-info/dependency_links.txt
13
13
  hsclient.egg-info/requires.txt
14
- hsclient.egg-info/top_level.txt
14
+ hsclient.egg-info/top_level.txt
15
+ tests/test_data_objects.py
16
+ tests/test_functional.py
17
+ tests/test_json_models.py
@@ -1,4 +1,4 @@
1
- hsmodels>=0.5.5
1
+ hsmodels>=1.0.0
2
2
  requests
3
3
  requests_oauthlib
4
4
 
@@ -5,11 +5,11 @@ README = (pathlib.Path(__file__).parent / "README.md").read_text()
5
5
 
6
6
  setup(
7
7
  name='hsclient',
8
- version='0.3.4',
8
+ version='1.0.0',
9
9
  packages=find_packages(include=['hsclient', 'hsclient.*'],
10
10
  exclude=("tests",)),
11
11
  install_requires=[
12
- 'hsmodels>=0.5.5',
12
+ 'hsmodels>=1.0.0',
13
13
  'requests',
14
14
  'requests_oauthlib',
15
15
  ],
@@ -0,0 +1,320 @@
1
+ import os
2
+ import pathlib
3
+ import tempfile
4
+
5
+ import fiona
6
+ import pytest
7
+ import rasterio
8
+ from fiona.model import to_dict
9
+ from hsmodels.schemas.enums import AggregationType
10
+ from rasterio.windows import Window
11
+
12
+ from hsclient import GeoFeatureAggregation, GeoRasterAggregation, NetCDFAggregation, TimeseriesAggregation
13
+
14
+
15
+ @pytest.mark.parametrize("search_by", ["type", "file_path"])
16
+ def test_timeseries_as_data_object(timeseries_resource, search_by):
17
+ timeseries_resource.refresh()
18
+ if search_by == 'type':
19
+ aggr = timeseries_resource.aggregation(type=AggregationType.TimeSeriesAggregation)
20
+ else:
21
+ file_path = "ODM2_Multi_Site_One_Variable.sqlite"
22
+ aggr = timeseries_resource.aggregation(file__path=file_path)
23
+
24
+ assert type(aggr) is TimeseriesAggregation
25
+ series_result = next(
26
+ r for r in aggr.metadata.time_series_results if r.series_id == "2837b7d9-1ebc-11e6-a16e-f45c8999816f"
27
+ )
28
+
29
+ with tempfile.TemporaryDirectory() as tmp:
30
+ # download aggregation
31
+ unzip_to = os.path.join(tmp, "unzipped_aggr")
32
+ os.makedirs(unzip_to)
33
+ agg_path = timeseries_resource.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
34
+ # load timseries data for the specified series to pandas DataFrame
35
+ pd_dataframe = aggr.as_data_object(agg_path=agg_path, series_id=series_result.series_id)
36
+ assert len(pd_dataframe) == 1333
37
+
38
+
39
+ @pytest.mark.parametrize("as_new_aggr", [False, True])
40
+ def test_timeseries_save_data_object(timeseries_resource, as_new_aggr):
41
+ timeseries_resource.refresh()
42
+ file_path = "ODM2_Multi_Site_One_Variable.sqlite"
43
+ aggr = timeseries_resource.aggregation(file__path=file_path)
44
+ assert aggr is not None
45
+ assert type(aggr) is TimeseriesAggregation
46
+ series_id = '4a6f095c-1ebc-11e6-8a10-f45c8999816f'
47
+ with tempfile.TemporaryDirectory() as tmp:
48
+ # download aggregation
49
+ unzip_to = os.path.join(tmp, "unzipped_aggr")
50
+ os.makedirs(unzip_to)
51
+ agg_path = timeseries_resource.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
52
+ pd_dataframe = aggr.as_data_object(agg_path=agg_path, series_id=series_id)
53
+ assert pd_dataframe.__class__.__name__ == "DataFrame"
54
+ rows, columns = pd_dataframe.shape
55
+ # delete 10 rows
56
+ pd_dataframe.drop(pd_dataframe.index[0:10], axis=0, inplace=True)
57
+ dst_path = ""
58
+ if as_new_aggr:
59
+ dst_path = "raster_aggr_folder"
60
+ timeseries_resource.folder_create(dst_path)
61
+
62
+ aggr = aggr.save_data_object(resource=timeseries_resource, agg_path=agg_path, as_new_aggr=as_new_aggr,
63
+ destination_path=dst_path)
64
+ assert type(aggr) is TimeseriesAggregation
65
+
66
+ # check the updated/new timeseries aggregation
67
+ with tempfile.TemporaryDirectory() as tmp:
68
+ # download aggregation
69
+ unzip_to = os.path.join(tmp, "unzipped_aggr")
70
+ os.makedirs(unzip_to)
71
+ if as_new_aggr:
72
+ file_path = f"{dst_path}/{file_path}"
73
+
74
+ aggr = timeseries_resource.aggregation(file__path=file_path)
75
+ assert aggr is not None
76
+ assert type(aggr) is TimeseriesAggregation
77
+
78
+ agg_path = timeseries_resource.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
79
+ pd_dataframe = aggr.as_data_object(agg_path=agg_path, series_id=series_id)
80
+ updated_rows, update_columns = pd_dataframe.shape
81
+ assert rows == updated_rows + 10
82
+ assert columns == update_columns
83
+
84
+
85
+ @pytest.mark.parametrize("search_by", ["type", "file_path"])
86
+ def test_raster_as_data_object(resource_with_raster_aggr, search_by):
87
+ resource_with_raster_aggr.refresh()
88
+ if search_by == "type":
89
+ aggr = resource_with_raster_aggr.aggregation(type=AggregationType.GeographicRasterAggregation)
90
+ else:
91
+ file_path = "logan.vrt"
92
+ aggr = resource_with_raster_aggr.aggregation(file__path=file_path)
93
+
94
+ assert type(aggr) is GeoRasterAggregation
95
+
96
+ with tempfile.TemporaryDirectory() as tmp:
97
+ # download aggregation
98
+ unzip_to = os.path.join(tmp, "unzipped_aggr")
99
+ os.makedirs(unzip_to)
100
+ agg_path = resource_with_raster_aggr.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
101
+
102
+ dataset = aggr.as_data_object(agg_path=agg_path)
103
+ assert dataset.__class__.__name__ == "DatasetReader"
104
+ # raster should have 1 band
105
+ assert dataset.count == 1
106
+
107
+
108
+ @pytest.mark.parametrize("as_new_aggr", [False, True])
109
+ def test_raster_save_data_object(resource_with_raster_aggr, as_new_aggr):
110
+ resource_with_raster_aggr.refresh()
111
+ file_path = "logan.vrt"
112
+ aggr = resource_with_raster_aggr.aggregation(file__path=file_path)
113
+ assert aggr is not None
114
+ assert type(aggr) is GeoRasterAggregation
115
+ with tempfile.TemporaryDirectory() as tmp:
116
+ # download aggregation
117
+ unzip_to = os.path.join(tmp, "unzipped_aggr")
118
+ os.makedirs(unzip_to)
119
+ agg_path = resource_with_raster_aggr.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
120
+ rasterio_reader = aggr.as_data_object(agg_path=agg_path)
121
+ assert rasterio_reader.__class__.__name__ == "DatasetReader"
122
+ # edit raster data - sub-setting the dataset
123
+ new_width = rasterio_reader.width - 9
124
+ new_height = rasterio_reader.height - 10
125
+ updated_width = new_width
126
+ updated_height = new_height
127
+ subset_window = Window(0, 0, new_width, new_height)
128
+ subset_band = rasterio_reader.read(1, window=subset_window)
129
+ output_raster_dir_path = os.path.join(tmp, "updated_aggr")
130
+ os.makedirs(output_raster_dir_path)
131
+ update_raster_filename = "updated_logan.tif"
132
+ output_raster_file_path = os.path.join(output_raster_dir_path, update_raster_filename)
133
+ profile = rasterio_reader.profile
134
+ rasterio_reader.close()
135
+ profile['driver'] = "GTiff"
136
+ profile['width'] = new_width
137
+ profile['height'] = new_height
138
+
139
+ with rasterio.open(output_raster_file_path, "w", **profile) as dst:
140
+ dst.write(subset_band, 1)
141
+
142
+ dst_path = ""
143
+ if as_new_aggr:
144
+ dst_path = "raster_aggr_folder"
145
+ resource_with_raster_aggr.folder_create(dst_path)
146
+
147
+ # save the new tif file to update the aggregation or create a new aggregation
148
+ aggr = aggr.save_data_object(resource=resource_with_raster_aggr, agg_path=output_raster_dir_path,
149
+ as_new_aggr=as_new_aggr, destination_path=dst_path)
150
+ assert aggr is not None
151
+ assert type(aggr) is GeoRasterAggregation
152
+
153
+ # check the updated raster aggregation
154
+ with tempfile.TemporaryDirectory() as tmp:
155
+ # download aggregation
156
+ unzip_to = os.path.join(tmp, "unzipped_aggr")
157
+ os.makedirs(unzip_to)
158
+ file_path = pathlib.Path(update_raster_filename).stem + ".vrt"
159
+ if as_new_aggr:
160
+ file_path = f"{dst_path}/{file_path}"
161
+
162
+ aggr = resource_with_raster_aggr.aggregation(file__path=file_path)
163
+ assert aggr is not None
164
+ assert type(aggr) is GeoRasterAggregation
165
+
166
+ agg_path = resource_with_raster_aggr.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
167
+ rasterio_reader = aggr.as_data_object(agg_path=agg_path)
168
+ assert updated_height == rasterio_reader.height
169
+ assert updated_width == rasterio_reader.width
170
+
171
+
172
+ @pytest.mark.parametrize("search_by", ["type", "file_path"])
173
+ def test_netcdf_as_data_object(resource_with_netcdf_aggr, search_by):
174
+ resource_with_netcdf_aggr.refresh()
175
+ if search_by == 'type':
176
+ aggr = resource_with_netcdf_aggr.aggregation(type=AggregationType.MultidimensionalAggregation)
177
+ else:
178
+ file_path = "SWE_time.nc"
179
+ aggr = resource_with_netcdf_aggr.aggregation(file__path=file_path)
180
+
181
+ assert type(aggr) is NetCDFAggregation
182
+
183
+ with tempfile.TemporaryDirectory() as tmp:
184
+ # download aggregation
185
+ unzip_to = os.path.join(tmp, "unzipped_aggr")
186
+ os.makedirs(unzip_to)
187
+ agg_path = resource_with_netcdf_aggr.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
188
+ dataset = aggr.as_data_object(agg_path=agg_path)
189
+ assert dataset.__class__.__name__ == "Dataset"
190
+ # netcdf dimensions
191
+ assert dataset.dims['time'] == 2184
192
+
193
+
194
+ @pytest.mark.parametrize("as_new_aggr", [False, True])
195
+ def test_netcdf_save_data_object(resource_with_netcdf_aggr, as_new_aggr):
196
+ resource_with_netcdf_aggr.refresh()
197
+ file_path = "SWE_time.nc"
198
+ aggr = resource_with_netcdf_aggr.aggregation(file__path=file_path)
199
+ assert aggr is not None
200
+ assert type(aggr) is NetCDFAggregation
201
+ with tempfile.TemporaryDirectory() as tmp:
202
+ # download aggregation
203
+ unzip_to = os.path.join(tmp, "unzipped_aggr")
204
+ os.makedirs(unzip_to)
205
+ agg_path = resource_with_netcdf_aggr.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
206
+ xr_dataset = aggr.as_data_object(agg_path=agg_path)
207
+ assert xr_dataset.__class__.__name__ == "Dataset"
208
+ agg_title = "This is a modified title for this aggregation by hsclient"
209
+ xr_dataset.attrs["title"] = agg_title
210
+ dst_path = ""
211
+ if as_new_aggr:
212
+ dst_path = "netcdf_aggr_folder"
213
+ resource_with_netcdf_aggr.folder_create(dst_path)
214
+
215
+ aggr = aggr.save_data_object(resource=resource_with_netcdf_aggr, agg_path=agg_path, as_new_aggr=as_new_aggr,
216
+ destination_path=dst_path)
217
+
218
+ assert type(aggr) is NetCDFAggregation
219
+ xr_dataset = aggr.as_data_object(agg_path=agg_path)
220
+ assert xr_dataset.attrs["title"] == agg_title
221
+
222
+ # check the updated/new netcdf aggregation
223
+ with tempfile.TemporaryDirectory() as tmp:
224
+ # download aggregation
225
+ unzip_to = os.path.join(tmp, "unzipped_aggr")
226
+ os.makedirs(unzip_to)
227
+ if as_new_aggr:
228
+ file_path = f"{dst_path}/{file_path}"
229
+
230
+ aggr = resource_with_netcdf_aggr.aggregation(file__path=file_path)
231
+ assert aggr is not None
232
+ assert type(aggr) is NetCDFAggregation
233
+
234
+ agg_path = resource_with_netcdf_aggr.aggregation_download(aggregation=aggr, save_path=tmp, unzip_to=unzip_to)
235
+ xr_dataset = aggr.as_data_object(agg_path=agg_path)
236
+ assert xr_dataset.attrs["title"] == agg_title
237
+
238
+
239
+ @pytest.mark.parametrize("search_by", ["type", "file_path"])
240
+ def test_geofeature_as_data_object(resource_with_geofeature_aggr, search_by):
241
+ resource_with_geofeature_aggr.refresh()
242
+ if search_by == "type":
243
+ aggr = resource_with_geofeature_aggr.aggregation(type=AggregationType.GeographicFeatureAggregation)
244
+ else:
245
+ file_path = "watersheds.shp"
246
+ aggr = resource_with_geofeature_aggr.aggregation(file__path=file_path)
247
+
248
+ assert aggr is not None
249
+ assert type(aggr) is GeoFeatureAggregation
250
+ with tempfile.TemporaryDirectory() as tmp:
251
+ # download aggregation
252
+ unzip_to = os.path.join(tmp, "unzipped_aggr")
253
+ os.makedirs(unzip_to)
254
+ agg_path = resource_with_geofeature_aggr.aggregation_download(aggregation=aggr, save_path=tmp,
255
+ unzip_to=unzip_to)
256
+ fn_collection = aggr.as_data_object(agg_path=agg_path)
257
+ assert fn_collection.__class__.__name__ == "Collection"
258
+ # check projection type
259
+ assert str(fn_collection.crs) == "EPSG:26912"
260
+ # close the fiona collection object so that the temp dir can be cleaned up.
261
+ fn_collection.close()
262
+
263
+
264
+ @pytest.mark.parametrize("as_new_aggr", [False, True])
265
+ def test_geofeature_save_data_object(resource_with_geofeature_aggr, as_new_aggr):
266
+ resource_with_geofeature_aggr.refresh()
267
+ file_path = "watersheds.shp"
268
+ aggr = resource_with_geofeature_aggr.aggregation(file__path=file_path)
269
+ assert aggr is not None
270
+ assert type(aggr) is GeoFeatureAggregation
271
+ with tempfile.TemporaryDirectory() as tmp:
272
+ # download aggregation
273
+ unzip_to = os.path.join(tmp, "unzipped_aggr")
274
+ os.makedirs(unzip_to)
275
+ agg_path = resource_with_geofeature_aggr.aggregation_download(aggregation=aggr, save_path=tmp,
276
+ unzip_to=unzip_to)
277
+ fn_collection = aggr.as_data_object(agg_path=agg_path)
278
+ assert fn_collection.__class__.__name__ == "Collection"
279
+ original_shp_filename = os.path.basename(fn_collection.path)
280
+ updated_shp_file_dir = os.path.join(tmp, "updated_aggr")
281
+ os.makedirs(updated_shp_file_dir)
282
+ output_shp_file_path = os.path.join(updated_shp_file_dir, original_shp_filename)
283
+ with fiona.open(output_shp_file_path, 'w', schema=fn_collection.schema, driver=fn_collection.driver,
284
+ crs=fn_collection.crs) as out_shp_file:
285
+ for feature in fn_collection:
286
+ ft_dict = to_dict(feature)
287
+ if ft_dict['properties']['Id'] < 5:
288
+ out_shp_file.write(feature)
289
+
290
+ dst_path = ""
291
+ if as_new_aggr:
292
+ dst_path = "geo_aggr_folder"
293
+ resource_with_geofeature_aggr.folder_create(dst_path)
294
+
295
+ aggr = aggr.save_data_object(resource=resource_with_geofeature_aggr, agg_path=updated_shp_file_dir,
296
+ as_new_aggr=as_new_aggr, destination_path=dst_path)
297
+ assert aggr is not None
298
+ assert type(aggr) is GeoFeatureAggregation
299
+ assert aggr.data_object is None
300
+
301
+ # check the updated geo-feature aggregation
302
+ with tempfile.TemporaryDirectory() as tmp:
303
+ # download aggregation
304
+ unzip_to = os.path.join(tmp, "unzipped_aggr")
305
+ os.makedirs(unzip_to)
306
+ if as_new_aggr:
307
+ file_path = f"{dst_path}/{file_path}"
308
+
309
+ aggr = resource_with_geofeature_aggr.aggregation(file__path=file_path)
310
+ assert aggr is not None
311
+ assert type(aggr) is GeoFeatureAggregation
312
+
313
+ agg_path = resource_with_geofeature_aggr.aggregation_download(aggregation=aggr, save_path=tmp,
314
+ unzip_to=unzip_to)
315
+ fn_collection = aggr.as_data_object(agg_path=agg_path)
316
+ for feature in fn_collection:
317
+ ft_dict = to_dict(feature)
318
+ assert ft_dict['properties']['Id'] < 5
319
+ # need to close the data object so that the tmp directory can be cleaned up
320
+ fn_collection.close()