envrihub 0.0.1__tar.gz

envrihub-0.0.1/LICENSE

@@ -0,0 +1,18 @@
+MIT No Attribution
+
+Copyright 2024 Dario De Nart
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

envrihub-0.0.1/PKG-INFO

@@ -0,0 +1,86 @@
+Metadata-Version: 2.1
+Name: envrihub
+Version: 0.0.1
+Summary: Python library to streamline interaction with the ENVRI-Hub APIs, providing a pythonic facade to data and service access.
+Home-page: https://gitlab.a.incd.pt/envri-hub-next/vre-lib
+Author: ENVRI Community
+Author-email: envri-hub-next-wp13-14@mailman.egi.eu
+License: MIT
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# ENVRI Hub's VRE Library
+This is the official ENVRI-Hub Python library, its purpose is to streamline interaction with the ENVRI-Hub APIs, providing a pythonic facade to data and service access.
+
+# Quickstart
+After installing the package with a quick
+```
+pip install envrihub
+```
+You can start using the ENVRI-HUB resorces right away through the *Hub* object:
+```
+from envrihub import Hub
+
+hub = Hub()
+```
+You can query it to retrieve resources that match your needs:
+
+```
+for res in hub.search_catalogue('bacon'):
+    print(res.title)
+```
+You can specify free text queries, time boundaries, geographic boundaries, dara providers and/or variables you expect in your data.
+Here is an example of geographical search with a [WKT](https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry) polygon:
+
+```
+geography = 'POLYGON((10.70 48.34,
+                28.98 48.34
+                28.98 36.17,
+                10.70 36.17,
+                10.70 48.34))'
+
+for i in hub.search_catalogue(geography = geography):
+    print(i.title)
+```
+
+Just type `help(hub.seach_catalogue)` for the full details.
+
+You can also access a resurce directly if you know its unique *identifier* in the Catalogue of Services:
+```
+res = hub.fetch_from_catalogue('b646c445-57b8-4553-bf2f-12448ee16b55')
+```
+Retrieved resources have the following properties:
++ `title`: a human readable title for the resource;
++ `id`: the resource's identifier in the Catalogue of Services;
++ `description`: a human readable description of the resource;
++ `metadata`: the whole EPOS-DCAT-AP metadata of the resource;
++ `dao`: the *data access object* that allows you to get the actual data. All DAOs have an `access` method.
+
+
+DAO objects are auto-generated according to the resource's metadata and can have additional methods to access data, when in doubt check them out with the `help` function:
+```
+help(res.dao)
+```
+If the resource is a Web Service, the DAO object allows to query such a service with all due parameters:
+```
+res = hub.fetch_from_catalogue('16c3ae2f-ba39-4239-964d-12e67c378fef')
+res.dao.access(id='138228')
+```
+Or if the resource is a static file it lets you download it either as a bytes object or directly to your file system.
+```
+res = hub.fetch_from_catalogue('b646c445-57b8-4553-bf2f-12448ee16b55')
+byte_stream = res.dao.access() # bytes object
+res.dao.download('path-to.file') # local file download
+```
+Digital kleptomaniacs rejoice! This means that with a handful of lines you can now scrape the whole ENVRI-HUB!
+```
+for res in hub.search_catalogue():
+    if res.is_downloadable():
+        res.dao.download(res.id)
+```
+
+# Contributing
+To contribute, you have to attend *ENVRI-Hub Next*'s WP13 monthly meetings. For now, if we never saw you, your pull requests will be rejected.
+
+# Acknowledgements
+This project is funded by the [ENVRI-Hub Next project](https://envri.eu/envri-hub-next/). The project received funding from the European Union’s Horizon Europe research and innovation programme under grant agreement No 101131141.

envrihub-0.0.1/README.md

@@ -0,0 +1,75 @@
+# ENVRI Hub's VRE Library
+This is the official ENVRI-Hub Python library, its purpose is to streamline interaction with the ENVRI-Hub APIs, providing a pythonic facade to data and service access.
+
+# Quickstart
+After installing the package with a quick
+```
+pip install envrihub
+```
+You can start using the ENVRI-HUB resorces right away through the *Hub* object:
+```
+from envrihub import Hub
+
+hub = Hub()
+```
+You can query it to retrieve resources that match your needs:
+
+```
+for res in hub.search_catalogue('bacon'):
+    print(res.title)
+```
+You can specify free text queries, time boundaries, geographic boundaries, dara providers and/or variables you expect in your data.
+Here is an example of geographical search with a [WKT](https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry) polygon:
+
+```
+geography = 'POLYGON((10.70 48.34,
+                28.98 48.34
+                28.98 36.17,
+                10.70 36.17,
+                10.70 48.34))'
+
+for i in hub.search_catalogue(geography = geography):
+    print(i.title)
+```
+
+Just type `help(hub.seach_catalogue)` for the full details.
+
+You can also access a resurce directly if you know its unique *identifier* in the Catalogue of Services:
+```
+res = hub.fetch_from_catalogue('b646c445-57b8-4553-bf2f-12448ee16b55')
+```
+Retrieved resources have the following properties:
++ `title`: a human readable title for the resource;
++ `id`: the resource's identifier in the Catalogue of Services;
++ `description`: a human readable description of the resource;
++ `metadata`: the whole EPOS-DCAT-AP metadata of the resource;
++ `dao`: the *data access object* that allows you to get the actual data. All DAOs have an `access` method.
+
+
+DAO objects are auto-generated according to the resource's metadata and can have additional methods to access data, when in doubt check them out with the `help` function:
+```
+help(res.dao)
+```
+If the resource is a Web Service, the DAO object allows to query such a service with all due parameters:
+```
+res = hub.fetch_from_catalogue('16c3ae2f-ba39-4239-964d-12e67c378fef')
+res.dao.access(id='138228')
+```
+Or if the resource is a static file it lets you download it either as a bytes object or directly to your file system.
+```
+res = hub.fetch_from_catalogue('b646c445-57b8-4553-bf2f-12448ee16b55')
+byte_stream = res.dao.access() # bytes object
+res.dao.download('path-to.file') # local file download
+```
+Digital kleptomaniacs rejoice! This means that with a handful of lines you can now scrape the whole ENVRI-HUB!
+```
+for res in hub.search_catalogue():
+    if res.is_downloadable():
+        res.dao.download(res.id)
+```
+
+# Contributing
+To contribute, you have to attend *ENVRI-Hub Next*'s WP13 monthly meetings. For now, if we never saw you, your pull requests will be rejected.
+
+# Acknowledgements
+This project is funded by the [ENVRI-Hub Next project](https://envri.eu/envri-hub-next/). The project received funding from the European Union’s Horizon Europe research and innovation programme under grant agreement No 101131141.

envrihub-0.0.1/envrihub/__init__.py

@@ -0,0 +1,5 @@
+'''the official ENVRI-HUB python package, allowing you to automate resource search and data access'''
+__all__ =['Hub']
+
+from envrihub.hub import Hub
+

envrihub-0.0.1/envrihub/dataset.py

@@ -0,0 +1,15 @@
+class Resource():
+    def __init__(self, id):
+        self.id = id
+    
+class Dataset(Resource):
+    '''This class encapsulates datasets in the ENVRI-HUB.
+    it is made up of two attributes:
+    + Data: the actual data, either a datastructure or a data access object
+    + Metadata: information about the data, expressed as a dictionary-like structure
+    '''
+    def __init__(self, id):
+        self.id = id
+
+class Service(Resource):
+    pass

envrihub-0.0.1/envrihub/decorators.py

@@ -0,0 +1,19 @@
+import inspect
+
+
+def delegates(to=None, keep=False):
+    "Decorator: replace `**kwargs` in signature with params from `to`"
+    def _f(f):
+        if to is None: to_f,from_f = f.__base__.__init__,f.__init__
+        else:          to_f,from_f = to,f
+        sig = inspect.signature(from_f)
+        sigd = dict(sig.parameters)
+        k = sigd.pop('kwargs')
+        s2 = {k:v for k,v in inspect.signature(to_f).parameters.items()
+              if v.default != inspect.Parameter.empty and k not in sigd}
+        sigd.update(s2)
+        if keep: sigd['kwargs'] = k
+        from_f.__signature__ = sig.replace(parameters=sigd.values())
+        return f
+    return _f
+

envrihub-0.0.1/envrihub/hub.py

@@ -0,0 +1,54 @@
+from datetime import datetime
+from collections.abc import Iterator
+from envrihub.cos.catalogue import EposCatalogue
+import shapely
+import logging
+
+from envrihub.cos.models import Distribution
+
+
+class Hub():
+
+    def __init__(self, **kwargs):
+        self.catalogue = EposCatalogue(**kwargs)
+    
+    def search_catalogue(self, query:str = '', start_date:datetime=None, end_date:datetime=None, geography = None, exv = None, provider=None)->Iterator[Distribution]:
+        '''
+        Parameters
+        ----------
+        query:str
+
+        start_date:datetime 
+
+        end_date: datetime
+
+        geography: str | shapely.Geometry | tuple
+            geographcial bounds in WKT format, with WGS84 coordinates. Altenratively
+            also Shapely Geometry objects and tuples can be accepted
+
+        exv:str|list[str]
+            essential variables to lookup
+
+        provider: str
+
+        '''
+        # parse wkt geogrpahy into a Shape
+        if isinstance(geography, str):
+            try:
+                polygon=shapely.from_wkt(geography )
+                bbox = polygon.bounds
+            except:
+                logging.error('Invalid WKT geography')
+        elif isinstance(geography, shapely.Geometry):
+            bbox = geography.bounds
+        elif isinstance(geography, tuple):
+            bbox = geography
+        else:bbox = None 
+        # here we are not using yield from becasue we may want to add some further
+        # processing/reshaping here in the near future...
+        for i in self.catalogue.search(query=query, bbox=bbox, startDate=start_date, endDate=end_date, exvs=exv, organisations=provider):
+            yield i
+
+    def fetch_from_catalogue(self, resource_id)->Distribution:
+        '''Retrieves a specific resource from the catalogue'''
+        return self.catalogue.retrieve_resource(resource_id)

envrihub-0.0.1/envrihub.egg-info/PKG-INFO

@@ -0,0 +1,86 @@
+Metadata-Version: 2.1
+Name: envrihub
+Version: 0.0.1
+Summary: Python library to streamline interaction with the ENVRI-Hub APIs, providing a pythonic facade to data and service access.
+Home-page: https://gitlab.a.incd.pt/envri-hub-next/vre-lib
+Author: ENVRI Community
+Author-email: envri-hub-next-wp13-14@mailman.egi.eu
+License: MIT
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# ENVRI Hub's VRE Library
+This is the official ENVRI-Hub Python library, its purpose is to streamline interaction with the ENVRI-Hub APIs, providing a pythonic facade to data and service access.
+
+# Quickstart
+After installing the package with a quick
+```
+pip install envrihub
+```
+You can start using the ENVRI-HUB resorces right away through the *Hub* object:
+```
+from envrihub import Hub
+
+hub = Hub()
+```
+You can query it to retrieve resources that match your needs:
+
+```
+for res in hub.search_catalogue('bacon'):
+    print(res.title)
+```
+You can specify free text queries, time boundaries, geographic boundaries, dara providers and/or variables you expect in your data.
+Here is an example of geographical search with a [WKT](https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry) polygon:
+
+```
+geography = 'POLYGON((10.70 48.34,
+                28.98 48.34
+                28.98 36.17,
+                10.70 36.17,
+                10.70 48.34))'
+
+for i in hub.search_catalogue(geography = geography):
+    print(i.title)
+```
+
+Just type `help(hub.seach_catalogue)` for the full details.
+
+You can also access a resurce directly if you know its unique *identifier* in the Catalogue of Services:
+```
+res = hub.fetch_from_catalogue('b646c445-57b8-4553-bf2f-12448ee16b55')
+```
+Retrieved resources have the following properties:
++ `title`: a human readable title for the resource;
++ `id`: the resource's identifier in the Catalogue of Services;
++ `description`: a human readable description of the resource;
++ `metadata`: the whole EPOS-DCAT-AP metadata of the resource;
++ `dao`: the *data access object* that allows you to get the actual data. All DAOs have an `access` method.
+
+
+DAO objects are auto-generated according to the resource's metadata and can have additional methods to access data, when in doubt check them out with the `help` function:
+```
+help(res.dao)
+```
+If the resource is a Web Service, the DAO object allows to query such a service with all due parameters:
+```
+res = hub.fetch_from_catalogue('16c3ae2f-ba39-4239-964d-12e67c378fef')
+res.dao.access(id='138228')
+```
+Or if the resource is a static file it lets you download it either as a bytes object or directly to your file system.
+```
+res = hub.fetch_from_catalogue('b646c445-57b8-4553-bf2f-12448ee16b55')
+byte_stream = res.dao.access() # bytes object
+res.dao.download('path-to.file') # local file download
+```
+Digital kleptomaniacs rejoice! This means that with a handful of lines you can now scrape the whole ENVRI-HUB!
+```
+for res in hub.search_catalogue():
+    if res.is_downloadable():
+        res.dao.download(res.id)
+```
+
+# Contributing
+To contribute, you have to attend *ENVRI-Hub Next*'s WP13 monthly meetings. For now, if we never saw you, your pull requests will be rejected.
+
+# Acknowledgements
+This project is funded by the [ENVRI-Hub Next project](https://envri.eu/envri-hub-next/). The project received funding from the European Union’s Horizon Europe research and innovation programme under grant agreement No 101131141.

envrihub-0.0.1/envrihub.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+LICENSE
+README.md
+setup.py
+envrihub/__init__.py
+envrihub/dataset.py
+envrihub/decorators.py
+envrihub/hub.py
+envrihub.egg-info/PKG-INFO
+envrihub.egg-info/SOURCES.txt
+envrihub.egg-info/dependency_links.txt
+envrihub.egg-info/requires.txt
+envrihub.egg-info/top_level.txt
+tests/test_basics.py
+tests/test_cos.py
+tests/test_hub.py
+tests/test_openapi.py

envrihub-0.0.1/envrihub.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

envrihub-0.0.1/envrihub.egg-info/requires.txt

@@ -0,0 +1,7 @@
+shapely
+retry
+openapi3-parser
+prance
+pandas
+argopy
+SPARQLWrapper

envrihub-0.0.1/envrihub.egg-info/top_level.txt

@@ -0,0 +1 @@
+envrihub

envrihub-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

envrihub-0.0.1/setup.py

@@ -0,0 +1,18 @@
+from setuptools import setup
+
+with open("README.md", 'r') as f:
+    long_description = f.read()
+
+setup(
+   name='envrihub',
+   version='0.0.1',
+   description='Python library to streamline interaction with the ENVRI-Hub APIs, providing a pythonic facade to data and service access.',
+   license="MIT",
+   long_description=long_description,
+   long_description_content_type="text/markdown",
+   author='ENVRI Community',
+   author_email='envri-hub-next-wp13-14@mailman.egi.eu',
+   url="https://gitlab.a.incd.pt/envri-hub-next/vre-lib",
+   packages=['envrihub'],
+   install_requires=['shapely','retry','openapi3-parser','prance','pandas','argopy','SPARQLWrapper'], #external packages as dependencies
+)

envrihub-0.0.1/tests/test_basics.py

@@ -0,0 +1,20 @@
+## Datetime parsing
+from datetime import datetime
+import pytz
+from envrihub.data_access.argo import to_argo_datetime
+from envrihub.data_access.open_api_client import _to_parameter_name
+from envrihub.data_access.seadatanet import to_seadatanet_datetime
+
+
+def test_datetime_parsing_argo_outformat():
+    t = to_argo_datetime(datetime(2024, 1, 23, 9, 21, 54, tzinfo=pytz.utc))
+    target = '2024-01-23T09:21:54+00:00'
+    assert t==target, f'Incorrectly parsed datetime, expected {target}, but got {t}'
+
+def test_datetime_parsing_seadatanet_outformat():
+    t = to_seadatanet_datetime(datetime(2024, 1, 23, 9, 21, 54, tzinfo=pytz.utc))
+    target = '2024-01-23'
+    assert t==target, f'Incorrectly parsed datetime, expected {target}, but got {t}'
+
+def test_to_parameter_name():
+    assert _to_parameter_name('Content-Type') == 'Content_Type', 'Parameter names are not not sanitized enough'

envrihub-0.0.1/tests/test_cos.py

@@ -0,0 +1,40 @@
+from collections.abc import Iterable
+from envrihub.cos.catalogue import EposCatalogue
+from envrihub.cos.models import Distribution
+from envrihub.data_access.models import DataAccessObject
+TEST_CATALOGUE_ADDRESS ='https://catalogue.staging.envri.eu/api/v1'
+
+def test_distribution():
+    cos =  EposCatalogue(TEST_CATALOGUE_ADDRESS )
+    results = list(cos.search('beacon'))
+    dist = cos.retrieve_resource(results[0].id)
+    assert isinstance(dist, Distribution), f'{str(dist)} is not a Distribution object'
+
+def test_distribution_dao():
+    cos =  EposCatalogue(TEST_CATALOGUE_ADDRESS )
+    results = list(cos.search('beacon'))
+    dist = cos.retrieve_resource(results[0].id)
+    assert dist.id == results[0].id, f'Catalogue returned id {str(dist.id)} which is different from the requested one'
+
+def test_dao_access():
+    cos =  EposCatalogue(TEST_CATALOGUE_ADDRESS )
+    results = list(cos.search('marine species'))
+    dist = cos.retrieve_resource(results[0].id)
+    data = dist.dao.access()
+    assert isinstance(data, Iterable), 'Unable to parse JSON. Probably connection does not work'
+
+def test_catalogue_search_free_text():
+    cos =  EposCatalogue(TEST_CATALOGUE_ADDRESS )
+    results = list(cos.search('marine species'))
+    assert len(results)>1, 'Catalogue does not find sh*t with free text search'
+
+def test_catalogue_search_coords():
+    cos =  EposCatalogue(TEST_CATALOGUE_ADDRESS )
+    results = list(cos.search(bbox = (10.70, 36.17, 28.98, 48.34)))
+    assert len(results)>1, 'Catalogue does not find sh*t with geo search'
+
+def test_file_access_from_cos():
+    cos =  EposCatalogue(TEST_CATALOGUE_ADDRESS )
+    results = list(cos.search('beacon'))
+    dist = cos.retrieve_resource(results[0].id)
+    assert isinstance(dist.dao.access(), bytes), 'File access does not download bytes object'

envrihub-0.0.1/tests/test_hub.py

@@ -0,0 +1,8 @@
+'''Testing the Hub class'''
+
+from envrihub import Hub
+
+def test_geo_search():
+    geography = 'POLYGON((10.703125000000004 48.345191092562935,28.984375000000004 48.345191092562935,28.984375000000004 36.17766212248528,10.703125000000004 36.17766212248528,10.703125000000004 48.345191092562935))'
+    hub = Hub()
+    assert len(list(hub.search_catalogue(geography = geography)))>0, 'Geographic search from the Hub does not work'

envrihub-0.0.1/tests/test_openapi.py

@@ -0,0 +1,27 @@
+import envrihub
+from envrihub.data_access.models import DataAccessObject, OpenAPI3DataAccess
+from envrihub.data_access.open_api_client import build_access_class, recursion_limit_handler_void
+from prance import ResolvingParser
+
+
+def test_recursive_spec_parsing():
+    parser = ResolvingParser('https://catalogue.staging.envri.eu/api/v1/openapi.json', recursion_limit_handler=recursion_limit_handler_void)
+    parser.parse()
+    specification = parser.specification
+    assert isinstance(specification, dict), f'Parsed specfication is {type(specification)} instead of good ol dict'
+
+def test_openapi_client():
+    parser = ResolvingParser('https://catalogue.staging.envri.eu/api/v1/openapi.json', recursion_limit_handler=recursion_limit_handler_void)
+    parser.parse()
+    specification = parser.specification
+    dao = build_access_class(specification, base_address='https://catalogue.staging.envri.eu/')()
+    assert isinstance(dao, OpenAPI3DataAccess), 'Wrong DAO type found'
+
+def test_openapi_client_call():
+    parser = ResolvingParser('https://catalogue.staging.envri.eu/api/v1/openapi.json', recursion_limit_handler=recursion_limit_handler_void)
+    parser.parse()
+    specification = parser.specification
+    dao = build_access_class(specification, base_address='https://catalogue.staging.envri.eu/')
+    dao_instance = dao()
+    result = dao_instance.resourcessearch(q='minchia')
+    assert isinstance(result, dict), 'OpenAPI client does not manage to make a proper GET request'