xcube-cci 0.12.dev0__tar.gz → 0.12.1.dev0__tar.gz

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/CHANGES.md

@@ -1,4 +1,7 @@
-## Changes in 0.12.0 (in development)
+## Changes in 0.12.1 (in development)
+
+## Changes in 0.12
+* Updated list of kerchunk datasets
 * Added xarray Data Trees as fourth data model. Data Trees provide a tree-like 
   hierarchical structure of datasets, where single nodes can be accessed by an
   identifier. This is a preferable approach to splitting datasets into sub-datasets, 
@@ -15,7 +18,10 @@
   "esacci.RD.satellite-orbit-frequency.L3S.WL.multi-sensor.multi-platform.MERGED.v1-1.r1~AMAZON_NEGRO_SAO-FELIPE".
   To open the dataset with a subset of places, pass `place_names` as parameter 
   to the store method `open_params()`. You can get a list of available place names 
-  from `get_open_data_params_schema()`.  
+  from `get_open_data_params_schema()`.
+* We fixed an issue with geodataframes which could cause that not all data were read in.
+* Fixed issues that data could not be accessed from ICESHEETS datasets with multiple time steps in base files.
+  Solves [#69](https://github.com/dcs4cop/xcube-cci/issues/69)
 
 ## Changes in 0.11.10
 * Revised warnings

xcube_cci-0.12.1.dev0/MANIFEST.in

@@ -0,0 +1,7 @@
+include xcube_cci/data/excluded_data_sources
+include xcube_cci/data/dataset_states.json
+include xcube_cci/data/datatree_states.json
+include xcube_cci/data/geodataframe_states.json
+include xcube_cci/data/vectordatacube_states.json
+include xcube_cci/data/stores.yml
+include xcube_cci/data/kc_refs.json

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xcube_cci
-Version: 0.12.dev0
+Version: 0.12.1.dev0
 Summary: An xcube plugin that allows generating data cubes from the ESA CCI Open Data Portal.
 Author: ESA CCI Toolbox Development Team
 License: MIT License

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "xcube_cci"
-version = "0.12.dev0"
+version = "0.12.1.dev0"
 authors = [
   { name="ESA CCI Toolbox Development Team" },
 ]

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/setup.py

@@ -22,7 +22,7 @@
 # SOFTWARE.
 
 
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
 requirements = [
     # Sync with ./environment.yml.
@@ -47,7 +47,11 @@ setup(
         'xcube_cci/data/excluded_data_sources',
         'xcube_cci/data/stores.yml',
         'xcube_cci/data/kc_refs.json',
-        'xcube_cci/data/dataset_states.json']
+        'xcube_cci/data/dataset_states.json',
+        'xcube_cci/data/datatree_states.json',
+        'xcube_cci/data/geodataframe_states.json',
+        'xcube_cci/data/vectordatacube_states.json',
+    ]
                  )],
     install_requires=requirements,
 )

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/test/open_all.py

@@ -32,7 +32,6 @@ import click
 import xarray as xr
 import zarr.storage
 
-from xcube.core.normalize import cubify_dataset
 from xcube_cci.cciodp import CciOdp
 from xcube_cci.chunkstore import CciChunkStore
 
@@ -46,14 +45,14 @@ DEFAULT_OUTPUT = 'open-report'
               help=f'Output directory. Defaults to "{DEFAULT_OUTPUT}".')
 @click.option('--force',
               is_flag=True,
-              help=f'Force deletion of an existing OUTPUT_DIR.')
+              help='Force deletion of an existing OUTPUT_DIR.')
 @click.option('--cache', 'cache_size',
               metavar='CACHE_SIZE',
-              help=f'Cache size, e.g. "100M" or "2G". If given, an in-memory LRU cache will be used.')
+              help='Cache size, e.g. "100M" or "2G". If given, an in-memory LRU cache will be used.')
 @click.option('--observe',
               is_flag=True,
-              help=f'Dump data requests. '
-                   f'Note, turning this flag on will slightly increase measured durations.')
+              help='Dump data requests. '
+                   'Note, turning this flag on will slightly increase measured durations.')
 @click.argument('dataset_id', nargs=-1, required=False)
 def gen_report(output_dir: str,
                force: bool,
@@ -136,7 +135,7 @@ def gen_report(output_dir: str,
 
     report_type_specifiers(output_dir, ts_dict=id_to_ts)
 
-    logging.info(f'Tests completed.')
+    logging.info('Tests completed.')
 
 
 def get_selected_dataset_ids(odp, ds_id_parts):
@@ -193,7 +192,7 @@ def report_error(output_dir: str, ds_id: str, t0: float, stage: str, e: Exceptio
 
 
 def report_type_specifiers(output_dir: str, ts_dict: dict):
-    json_path = os.path.join(output_dir, f'type_specifiers.json')
+    json_path = os.path.join(output_dir, 'type_specifiers.json')
     with open(json_path, 'w') as fp:
         json.dump(ts_dict, fp, indent=2)
 

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/test/test_cciodp.py

@@ -1,12 +1,11 @@
-import numpy as np
 import os
-import pandas as pd
 import unittest
 from unittest import skip, skipIf
 
-from xcube_cci.cciodp import get_res
-from xcube_cci.cciodp import CciOdp
-from xcube_cci.cciodp import find_datetime_format
+import numpy as np
+import pandas as pd
+
+from xcube_cci.cciodp import CciOdp, find_datetime_format, get_res
 from xcube_cci.constants import OPENSEARCH_CEDA_URL
 
 AEROSOL_ID = 'esacci.AEROSOL.day.L3C.AER_PRODUCTS.AATSR.Envisat.ORAC.04-01-.r1'

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/test/test_chunkstore.py

@@ -1,11 +1,11 @@
-import numpy
 import os
-import pandas as pd
 import unittest
-import xarray as xr
-
 from unittest import skipIf
 
+import numpy
+import pandas as pd
+import xarray as xr
+
 from xcube_cci.cciodp import CciOdp
 from xcube_cci.chunkstore import CciChunkStore
 
@@ -133,7 +133,7 @@ class CciChunkStoreTest(unittest.TestCase):
             'O3_vmr', (5, 0, 0, 0)
         )
         self.assertIsNotNone(dim_indexes)
-        self.assertEqual(slice(None, None, None), dim_indexes[0])
+        self.assertEqual(slice(0, 1), dim_indexes[0])
         self.assertEqual(slice(0, 17), dim_indexes[1])
         self.assertEqual(slice(0, 180), dim_indexes[2])
         self.assertEqual(slice(0, 360), dim_indexes[3])

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/test/test_dataaccess.py

@@ -1,28 +1,22 @@
 import datetime as dt
 import os
 import unittest
-from unittest import skip
-from unittest import skipIf
+from unittest import skip, skipIf
 
 from xcube.core.gridmapping import GridMapping
 from xcube.core.normalize import normalize_dataset
-from xcube.core.store import DATASET_TYPE
-from xcube.core.store import DataStoreError
-from xcube.core.store import GEO_DATA_FRAME_TYPE
-from xcube.core.store.descriptor import DatasetDescriptor
-from xcube.core.store.descriptor import GeoDataFrameDescriptor
+from xcube.core.store import DATASET_TYPE, GEO_DATA_FRAME_TYPE, DataStoreError, new_data_store
+from xcube.core.store.descriptor import (DatasetDescriptor,
+                                         GeoDataFrameDescriptor)
 from xcube.core.verify import assert_cube
 
-from xcube_cci.dataaccess import CciOdpDataStore
-from xcube_cci.dataaccess import CciOdpDataFrameOpener
-from xcube_cci.dataaccess import CciOdpDatasetOpener
-from xcube_cci.dataaccess import CciOdpDataTreeOpener
-from xcube_cci.dataaccess import get_temporal_resolution_from_id
-from xcube_cci.dataaccess import CciOdpVectorDataCubeOpener
-from xcube_cci.dataaccess import VectorDataCubeDescriptor
-from xcube_cci.dataaccess import VECTOR_DATA_CUBE_TYPE
-from xcube_cci.dtaccess import DataTreeDescriptor
-from xcube_cci.dtaccess import DATATREE_TYPE
+from xcube_cci.dataaccess import (VECTOR_DATA_CUBE_TYPE, CciOdpDataFrameOpener,
+                                  CciOdpDatasetOpener, CciOdpDataStore,
+                                  CciOdpDataTreeOpener,
+                                  CciOdpVectorDataCubeOpener,
+                                  VectorDataCubeDescriptor,
+                                  get_temporal_resolution_from_id)
+from xcube_cci.dtaccess import DATATREE_TYPE, DataTreeDescriptor
 
 AEROSOL_DAY_ID = 'esacci.AEROSOL.day.L3.AAI.multi-sensor.multi-platform.' \
                  'MSAAI.1-7.r1'
@@ -34,7 +28,8 @@ BIOMASS_ID = "esacci.BIOMASS.yr.L4.AGB.multi-sensor.multi-platform.MERGED.5-0.10
 CLOUD_ID = "esacci.CLOUD.mon.L3C.CLD_PRODUCTS.MODIS.Terra.MODIS_TERRA.2-0.r1"
 ICESHEETS_ID = 'esacci.ICESHEETS.yr.Unspecified.IV.PALSAR.ALOS.UNSPECIFIED.' \
                '1-1.greenland_margin_2006_2011'
-HR_LC_ID = "esacci.FIRE.mon.L3S.BA.MODIS.Terra.MODIS_TERRA.v5-1.pixel"
+FIRE_ID = "esacci.FIRE.mon.L3S.BA.MODIS.Terra.MODIS_TERRA.v5-1.pixel"
+HR_LC_ID = "esacci.LC.yr.L4.Map.multi-sensor.multi-platform.HRLC10-A02.v1-2.Amazonia"
 LAKES_ID = 'esacci.LAKES.day.L3S.LK_PRODUCTS.multi-sensor.multi-platform.' \
            'MERGED.v1-1.r1'
 OZONE_MON_ID = 'esacci.OZONE.mon.L3.NP.multi-sensor.multi-platform.MERGED.' \
@@ -88,6 +83,39 @@ class DataAccessTest(unittest.TestCase):
         ))
 
 
+class NewDataStoreTest(unittest.TestCase):
+
+    def test_new_data_store_esa_cci(self):
+        self.assertIsNotNone(new_data_store("esa-cci"))
+
+    def test_new_data_store_esa_cdc(self):
+        self.assertIsNotNone(new_data_store("esa-cdc"))
+
+    def test_new_data_store_esa_climate_data_centre(self):
+        self.assertIsNotNone(new_data_store("esa-climate-data-centre"))
+
+    def test_new_data_store_esa_cci_kc(self):
+        self.assertIsNotNone(new_data_store("esa-cci-kc"))
+
+    def test_new_data_store_esa_cdc_kc(self):
+        self.assertIsNotNone(new_data_store("esa-cdc-kc"))
+
+    def test_new_data_store_esa_cci_zarr(self):
+        self.assertIsNotNone(new_data_store("esa-cci-zarr"))
+
+    def test_new_data_store_esa_ccikc(self):
+        self.assertIsNotNone(new_data_store("ccikc"))
+
+    def test_new_data_store_cciodp(self):
+        self.assertIsNotNone(new_data_store("cciodp"))
+
+    def test_new_data_store_ccikc(self):
+        self.assertIsNotNone(new_data_store("ccikc"))
+
+    def test_new_data_store_ccizarr(self):
+        self.assertIsNotNone(new_data_store("ccizarr"))
+
+
 class CciOdpDatasetOpenerTest(unittest.TestCase):
 
     def setUp(self) -> None:
@@ -675,9 +703,10 @@ class CciOdpDatasetOpenerNormalizeTest(unittest.TestCase):
 
 
 def user_agent(ext: str = "") -> str:
-    from xcube_cci.version import version
     from platform import machine, python_version, system
 
+    from xcube_cci.version import version
+
     return "xcube-cci-testing/{version} (Python {python};" \
            " {system} {arch}) {ext}".format(
         version=version,
@@ -770,7 +799,7 @@ class CciOdpDataFrameOpenerTest(unittest.TestCase):
              'platform', 'orbit_track_number', 'mission_cycle_number'},
             set(gdf.columns)
         )
-        self.assertEqual((1747, 7), gdf.shape)
+        self.assertEqual((1747, 7), gdf[0:2000].shape)
 
 
 class CciOdpVectorDataCubeOpenerTest(unittest.TestCase):
@@ -855,15 +884,15 @@ class CciOdpDataTreeOpenerTest(unittest.TestCase):
     @skipIf(os.environ.get('XCUBE_CCI_DISABLE_WEB_TESTS', '1') == '1',
             'XCUBE_CCI_DISABLE_WEB_TESTS = 1')
     def test_has_data(self):
-        self.assertTrue(self._opener.has_data(HR_LC_ID))
+        self.assertTrue(self._opener.has_data(FIRE_ID))
 
     @skipIf(os.environ.get('XCUBE_CCI_DISABLE_WEB_TESTS', '1') == '1',
             'XCUBE_CCI_DISABLE_WEB_TESTS = 1')
     def test_describe_data_fire(self):
-        descriptor = self._opener.describe_data([HR_LC_ID])[0]
+        descriptor = self._opener.describe_data([FIRE_ID])[0]
         self.assertIsNotNone(descriptor)
         self.assertIsInstance(descriptor, DataTreeDescriptor)
-        self.assertEqual(HR_LC_ID, descriptor.data_id)
+        self.assertEqual(FIRE_ID, descriptor.data_id)
         self.assertEqual('datatree', str(descriptor.data_type))
         self.assertEqual(["y", "x", "time", "bnds"], list(descriptor.dims.keys()))
         self.assertEqual(2, descriptor.dims['bnds'])
@@ -887,7 +916,7 @@ class CciOdpDataTreeOpenerTest(unittest.TestCase):
     @skipIf(os.environ.get('XCUBE_CCI_DISABLE_WEB_TESTS', '1') == '1',
             'XCUBE_CCI_DISABLE_WEB_TESTS = 1')
     def test_get_open_data_params_schema(self):
-        schema = self._opener.get_open_data_params_schema(HR_LC_ID).to_dict()
+        schema = self._opener.get_open_data_params_schema(FIRE_ID).to_dict()
         self.assertIsNotNone(schema)
         self.assertTrue('variable_names' in schema['properties'])
         self.assertTrue('place_names' in schema['properties'])
@@ -898,7 +927,7 @@ class CciOdpDataTreeOpenerTest(unittest.TestCase):
             'XCUBE_CCI_DISABLE_WEB_TESTS = 1')
     def test_open_data(self):
         data = self._opener.open_data(
-            HR_LC_ID, place_names=["AREA_1", "AREA_2"], variable_names=["JD", "CL"]
+            FIRE_ID, place_names=["AREA_1", "AREA_2"], variable_names=["JD", "CL"]
         )
         self.assertIsNotNone(data)
         self.assertTrue(data.is_root)
@@ -913,6 +942,24 @@ class CciOdpDataTreeOpenerTest(unittest.TestCase):
         self.assertEqual({264, 28499, 57888}, set(area_1_dt.dataset.JD.shape))
         self.assertIsNotNone(area_1_dt.dataset.zarr_store.get())
 
+    @skipIf(os.environ.get('XCUBE_CCI_DISABLE_WEB_TESTS', '1') == '1',
+            'XCUBE_CCI_DISABLE_WEB_TESTS = 1')
+    def test_open_data_lc(self):
+        place_name = "A02T20KPA"
+        variable_name = "CL01"
+        data = self._opener.open_data(HR_LC_ID, place_names=[place_name], variable_names=[variable_name])
+        ds = data.get(place_name)
+        self.assertIsNotNone(ds)
+        self.assertTrue(data.is_root)
+        self.assertTrue(data.is_empty)
+        self.assertEqual({"A02T20KPA"}, set(data.keys()))
+        sub_dt = data.get("A02T20KPA")
+        self.assertIsNotNone(sub_dt.dataset)
+        self.assertEqual(1, len(sub_dt.dataset.data_vars))
+        self.assertIn("CL01", sub_dt.dataset.data_vars)
+        self.assertEqual({"time", "x", "y", "bnds"}, set(sub_dt.dataset.dims.keys()))
+        self.assertEqual((1, 10980, 10980), sub_dt.dataset.CL01.shape)
+        self.assertIsNotNone(sub_dt.dataset.zarr_store.get())
 
 class CciOdpDataStoreTest(unittest.TestCase):
 
@@ -994,7 +1041,7 @@ class CciDataNormalizationTest(unittest.TestCase):
         good_datasets = []
         for data in all_data:
             if 'climatology' in data.data_id:
-                print(f'Cannot read data due to unsupported frequency')
+                print('Cannot read data due to unsupported frequency')
                 datasets_with_unsupported_frequencies.append(data.data_id)
                 continue
             if not data.data_vars or len(data.data_vars) < 1:

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/test/test_dataframeaccess.py

@@ -1,21 +1,28 @@
+import os
+from unittest import skipIf
+from unittest import TestCase
+
 import pandas as pd
-import unittest
 
 from xcube_cci.cciodp import CciOdp
 from xcube_cci.dataframeaccess import DataFrameAccessor
 
 GHG_DS_ID = "esacci.GHG.satellite-orbit-frequency.L2.CH4.SCIAMACHY.Envisat.IMAP.v7-2.r1"
 
-class DataFrameAccessTest(unittest.TestCase):
+class DataFrameAccessTest(TestCase):
 
     def setUp(self) -> None:
         ccicdc = CciOdp(data_type='geodataframe')
         self._dfa = DataFrameAccessor(ccicdc, GHG_DS_ID, {})
 
+    @skipIf(os.environ.get('XCUBE_CCI_DISABLE_WEB_TESTS', '1') == '1',
+            'XCUBE_CCI_DISABLE_WEB_TESTS = 1')
     def test_get_geodataframe(self):
         gdf = self._dfa.get_geodataframe()
         self.assertIsNotNone(gdf)
 
+    @skipIf(os.environ.get('XCUBE_CCI_DISABLE_WEB_TESTS', '1') == '1',
+            'XCUBE_CCI_DISABLE_WEB_TESTS = 1')
     def test_get_geodataframe_for_dataset(self):
         gdf = self._dfa._get_features_from_cci_cdc(
             GHG_DS_ID,

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/test/test_dtaccess.py

@@ -1,11 +1,12 @@
-import jsonschema
 import unittest
 
+import jsonschema
 from xcube.core.store import DatasetDescriptor
 from xcube.util.jsonschema import JsonObjectSchema
 
 from xcube_cci.dtaccess import DataTreeDescriptor
 
+
 class DataTreeDescriptorTest(unittest.TestCase):
     def test_get_schema(self):
         schema = DataTreeDescriptor.get_schema()
@@ -122,9 +123,9 @@ class DataTreeDescriptorTest(unittest.TestCase):
         self.assertEqual("EPSG:4326", descriptor.crs)
         self.assertEqual(1, len(descriptor.data_nodes))
         self.assertTrue("first_inner_tree" in descriptor.data_nodes)
-        self.assertIsInstance(descriptor.data_nodes.get("firstInnerTree"), DataTreeDescriptor)
+        self.assertIsInstance(descriptor.data_nodes.get("first_inner_tree"), DataTreeDescriptor)
 
-    def test_from_dict_datatree_descriptors_as_dict(self):
+    def test_from_dict_datatree_descriptors_with_inner_trees_as_dict(self):
         descriptor_dict = dict(
             data_id="xyz",
             data_type="datatree",

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/test/test_kcaccess.py

@@ -23,11 +23,9 @@ import os
 import unittest
 
 import xarray as xr
+from xcube.core.store import (DatasetDescriptor, DataStore, DataType,
+                              MutableDataStore)
 
-from xcube.core.store import DataStore
-from xcube.core.store import MutableDataStore
-from xcube.core.store import DatasetDescriptor
-from xcube.core.store import DataType
 from xcube_cci.kcaccess import CciKerchunkDataStore
 
 
@@ -56,7 +54,7 @@ class CciKerchunkDataStoreTest(unittest.TestCase):
     def test_list_data_ids(self):
         data_ids = self.store.list_data_ids()
         self.assertIn(
-            "ESACCI-BIOMASS-L4-AGB-CHANGE-100m-2018-2017-fv4.0-kr1.1",
+            "ESACCI-BIOMASS-L4-AGB-CHANGE-100m-2020-2010-fv4.0-kr1.1",
             data_ids
         )
         self.assertIn(
@@ -78,19 +76,19 @@ class CciKerchunkDataStoreTest(unittest.TestCase):
 
     def test_has_data(self):
         store = self.store
-        data_id = 'ESACCI-L4_FIRE-BA-MODIS-20010101-20200120-fv5.1-kr1.2'
+        data_id = 'ESACCI-BIOMASS-L4-AGB-CHANGE-100m-2020-2010-fv4.0-kr1.1'
         self.assertEqual(True, store.has_data(data_id))
         self.assertEqual(False, store.has_data("lst-cube"))
 
     def test_describe_data(self):
-        data_id = 'ESACCI-L4_FIRE-BA-MODIS-20010101-20200120-fv5.1-kr1.2'
+        data_id = 'ESACCI-BIOMASS-L4-AGB-CHANGE-100m-2020-2010-fv4.0-kr1.1'
         descriptor = self.store.describe_data(data_id)
         self.assertIsInstance(descriptor, DatasetDescriptor)
         self.assertEqual(data_id, descriptor.data_id)
         self.assertIsInstance(descriptor.data_type, DataType)
         self.assertIs(xr.Dataset, descriptor.data_type.dtype)
         self.assertIsInstance(descriptor.bbox, tuple)
-        self.assertIsNone(descriptor.spatial_res)  # ?
+        self.assertIsInstance(descriptor.spatial_res, float)
         self.assertIsInstance(descriptor.dims, dict)
         self.assertIsInstance(descriptor.coords, dict)
         self.assertIsInstance(descriptor.data_vars, dict)
@@ -108,7 +106,7 @@ class CciKerchunkDataStoreTest(unittest.TestCase):
 
     def test_search_data(self):
         search_results = list(self.store.search_data())
-        self.assertEqual(164, len(search_results))
+        self.assertEqual(209, len(search_results))
         for descriptor, data_id in zip(search_results, self.store.get_data_ids()):
             self.assertIsInstance(descriptor, DatasetDescriptor)
             self.assertEqual(data_id, descriptor.data_id)

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/test/test_normalize.py

@@ -1,7 +1,7 @@
 from unittest import TestCase
 
-from xcube_cci.normalize import normalize_dims_description
-from xcube_cci.normalize import normalize_variable_dims_description
+from xcube_cci.normalize import (normalize_dims_description,
+                                 normalize_variable_dims_description)
 
 
 class TestNormalize(TestCase):

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/test/test_timeutil.py

@@ -1,4 +1,5 @@
 from unittest import TestCase
+
 from xcube_cci.timeutil import get_timestrings_from_string
 
 

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/test/test_zarraccess.py

@@ -24,6 +24,7 @@ import unittest
 
 from xcube.core.store import DataStoreError
 from xcube.util.jsonschema import JsonObjectSchema
+
 from xcube_cci.zarraccess import CciZarrDataStore
 
 

{xcube_cci-0.12.dev0 → xcube_cci-0.12.1.dev0}/xcube_cci/__init__.py

@@ -20,4 +20,5 @@
 # SOFTWARE.
 
 from .version import version
+
 __version__ = version