climate-ref 0.5.0__py3-none-any.whl

climate_ref/dataset_registry/sample_data.txt

@@ -0,0 +1,60 @@
+CMIP6/C4MIP/CSIRO/ACCESS-ESM1-5/esm-1pct-brch-1000PgC/r1i1p1f1/Amon/tas/gn/v20191206/tas_Amon_ACCESS-ESM1-5_esm-1pct-brch-1000PgC_r1i1p1f1_gn_016801-026812.nc 32f8e310d1da60516972d509419a7f81709e6f3e60fd3923c2a212ec06a62af5
+CMIP6/C4MIP/CSIRO/ACCESS-ESM1-5/esm-1pct-brch-1000PgC/r1i1p1f1/fx/areacella/gn/v20191206/areacella_fx_ACCESS-ESM1-5_esm-1pct-brch-1000PgC_r1i1p1f1_gn.nc 826b3f5bcf9cd320ad302780eeb9f704e1090188be1090cdfa91778e2352202c
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/fco2antt/gn/v20190815/fco2antt_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_185001-186912.nc c18f42666e91852cb82d03e4d8d764dd4e12dab1776f70a5527621d3d6e05d8c
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/fco2antt/gn/v20190815/fco2antt_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_187001-188912.nc a3fbd25e8069b6ac612ef79e07551b79dee6a96640294be3f5521223bf372add
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/fco2antt/gn/v20190815/fco2antt_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_189001-190912.nc e5cede6894466d3229f7d4abd135065f930f73803f17cbb23ddc673b3aeead6d
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/fco2antt/gn/v20190815/fco2antt_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_191001-191412.nc 3739b234b0733ae812aee17d0ffeb2fa03b2293bb2f13e789e3c266e2e7b9c9f
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_185001-186912.nc 14ab558cd768d22d6f78990327d24d4369647a73fe508723946a00532896a43f
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_187001-188912.nc d7086d4bd6d7934f8f194da01ad668840e8be1e5c1fa5056e81be410d3571856
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_189001-190912.nc b0556ce505d39d62d01d4a62933476dbc001715e2a331edcba3a6170db075d89
+CMIP6/C4MIP/MPI-M/MPI-ESM1-2-LR/esm-1pctCO2/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-1pctCO2_r1i1p1f1_gn_191001-191412.nc 46e8f4e5b5f2ff9cc018d6713a0e8d4666925e803a0e17886ff7137006791c04
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/1pctCO2/r1i1p1f1/Amon/tas/gn/v20191115/tas_Amon_ACCESS-ESM1-5_1pctCO2_r1i1p1f1_gn_010101-025012.nc d29c1e0651d6c179ad5dc8ac8961ade5cd9456f506d9743255664e44360bac62
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/1pctCO2/r1i1p1f1/fx/areacella/gn/v20191115/areacella_fx_ACCESS-ESM1-5_1pctCO2_r1i1p1f1_gn.nc 07ae2f59188889030a7c453bca5f8c6a19f22f1b544b3987ba50a7a4f306c82d
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/abrupt-4xCO2/r1i1p1f1/Amon/rlut/gn/v20191115/rlut_Amon_ACCESS-ESM1-5_abrupt-4xCO2_r1i1p1f1_gn_010101-012512.nc 0da7114197033589e61a7ed6f53412e0727b540e5da9d1b7ed6a51ee2a4629c6
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/abrupt-4xCO2/r1i1p1f1/Amon/rsdt/gn/v20191115/rsdt_Amon_ACCESS-ESM1-5_abrupt-4xCO2_r1i1p1f1_gn_010101-012512.nc 3b00f242368a30fabe0db6a8789cf06cacaa0a3ff3726ade731f2ee488a751c5
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/abrupt-4xCO2/r1i1p1f1/Amon/rsut/gn/v20191115/rsut_Amon_ACCESS-ESM1-5_abrupt-4xCO2_r1i1p1f1_gn_010101-012512.nc 4303d70390ef0e1dac94cbe4cf6354e452cda9a7892eb06dd6c6b834ff09bd86
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/abrupt-4xCO2/r1i1p1f1/Amon/tas/gn/v20191115/tas_Amon_ACCESS-ESM1-5_abrupt-4xCO2_r1i1p1f1_gn_010101-012512.nc 0c71cbeb2667a00a452cd7acb2380a162b90abbc5413d31c103941b3bde1882a
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/abrupt-4xCO2/r1i1p1f1/fx/areacella/gn/v20191115/areacella_fx_ACCESS-ESM1-5_abrupt-4xCO2_r1i1p1f1_gn.nc 3a1846b06105c44c93d4612518fc7f068e67a115f69b21b6cd81225fe82e4f60
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/pr/gn/v20191115/pr_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_185001-201412.nc acc821dc400f53166379d2e23095bc2690d7ca7db6c7a6f88ae29a8771b3c65a
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/psl/gn/v20191115/psl_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc b63a3d4051cf17568df808836b189826da580ca8e1db949b1e93a71c80756c8d
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/rlut/gn/v20191115/rlut_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc 44a3c90a41744101afb00344f50947fe46444fe5d6bd3623c0c19aa02a378c86
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/rsdt/gn/v20191115/rsdt_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc a4e1fc3a4a4d00c2fa18ec616338426eb3d91165db3bc57e565ffdc8d6bd9d34
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/rsut/gn/v20191115/rsut_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc 8d492ef1f2bb654220fe64977d9942a33af0962ee9afa4017dcc75b6f0103015
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/tas/gn/v20191115/tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_185001-201412.nc 38e055e57aea5a9ae76ed3fc5325be6783b5694a9edc28aafd24dd462b32e5ce
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Amon/ts/gn/v20191115/ts_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc e02530449c92e0ffc72e9edeba57f5d38ab8652a28486c1c2b9ddada1f38fbd9
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Lmon/gpp/gn/v20191115/gpp_Lmon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc da36ed1653f7aafe40a4fc9b99004a46cb45231697ce6b3413dfc171980c37df
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Ofx/areacello/gn/v20191115/areacello_Ofx_ACCESS-ESM1-5_historical_r1i1p1f1_gn.nc 6808b64c7328bd118537bfb7cfd35748b4e84cae3f6a5586403aa9d8040e4d0b
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/Omon/tos/gn/v20191115/tos_Omon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-201412.nc 94121a2233aff78ef8799c5d59b6b6f3e7d3f2fb7ceb3a4a1645943ef3e88040
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/SImon/siconc/gn/v20200817/siconc_SImon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_197901-201412.nc 886c62a24797e893fe11b8de4d16c8a277bdee931b692d533f2fb3fa39820aa1
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/fx/areacella/gn/v20191115/areacella_fx_ACCESS-ESM1-5_historical_r1i1p1f1_gn.nc 4587a7b0110a226a805f681ee9fe456d20ec310302b2c120334c21595e4e96cb
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r1i1p1f1/fx/sftlf/gn/v20191115/sftlf_fx_ACCESS-ESM1-5_historical_r1i1p1f1_gn.nc bf9b256dc97cf72956ed14880e5e85a10e57f10956fb66e21192ec4cfffbdca7
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r2i1p1f1/Amon/psl/gn/v20191128/psl_Amon_ACCESS-ESM1-5_historical_r2i1p1f1_gn_200001-201412.nc 0abba30a4b21f1ddd4273eb8684e42a23a217d8be2360f6cfc348f89ebe0237e
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r2i1p1f1/Amon/ts/gn/v20191128/ts_Amon_ACCESS-ESM1-5_historical_r2i1p1f1_gn_200001-201412.nc 81c8e83d405ef92cb149a272386b89d2f135d2b620c64924ff77ed55f217fb5a
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/historical/r2i1p1f1/fx/areacella/gn/v20191128/areacella_fx_ACCESS-ESM1-5_historical_r2i1p1f1_gn.nc dc449fdd4a7677e681b79ce185a95f92be3620036eca0a6b6e8a2ab320d199c4
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/piControl/r1i1p1f1/Amon/rlut/gn/v20210316/rlut_Amon_ACCESS-ESM1-5_piControl_r1i1p1f1_gn_010101-012512.nc 375990b89a38ab390826d3c3efeef4e9295299164eba119e4545165079b86942
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/piControl/r1i1p1f1/Amon/rsdt/gn/v20210316/rsdt_Amon_ACCESS-ESM1-5_piControl_r1i1p1f1_gn_010101-012512.nc e647cd4f0cb0ff9e2727f1a5f8a636ddad6c62bded06c415d28f6d1c0632c471
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/piControl/r1i1p1f1/Amon/rsut/gn/v20210316/rsut_Amon_ACCESS-ESM1-5_piControl_r1i1p1f1_gn_010101-012512.nc 63cc0aa1927ded178e79f836ac9f2a058ca96b4cf901339754440bf9a0c55d04
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/piControl/r1i1p1f1/Amon/tas/gn/v20210316/tas_Amon_ACCESS-ESM1-5_piControl_r1i1p1f1_gn_010101-018012.nc 28267d35d304d3f3d4bb222eb2a0631a951ed3aaa626e3d0364f83e9ad6e0554
+CMIP6/CMIP/CSIRO/ACCESS-ESM1-5/piControl/r1i1p1f1/fx/areacella/gn/v20210316/areacella_fx_ACCESS-ESM1-5_piControl_r1i1p1f1_gn.nc 0eeabbcf35b548cb943e3f45befadf8c4c605e1ad097996cd04cf95ea073b706
+CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/esm-piControl/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-piControl_r1i1p1f1_gn_185001-186912.nc 98dc5c8453e98e008b63b73a3004d984644d45ceaad9776534693f209e96deed
+CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/esm-piControl/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-piControl_r1i1p1f1_gn_187001-188912.nc 7d6b9aeaeb3b77dfee3a9239dfd49db5e6525582d7080251072f07208c560caf
+CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/esm-piControl/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-piControl_r1i1p1f1_gn_189001-190912.nc b712699e038f83bf9ea72acb111857ebd3de01a5c6c2f34f5573d998a08f9d17
+CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/esm-piControl/r1i1p1f1/Amon/tas/gn/v20190815/tas_Amon_MPI-ESM1-2-LR_esm-piControl_r1i1p1f1_gn_191001-191512.nc 55b2f09622e999f172b26cc4fbe35ab75cb4a52677f8f4795777604ace566f79
+CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/esm-piControl/r1i1p1f1/fx/areacella/gn/v20190815/areacella_fx_MPI-ESM1-2-LR_esm-piControl_r1i1p1f1_gn.nc 048a705763d6fb08376539095e1092291278d9ad8ada7d3ccc431ae682000f03
+CMIP6/DAMIP/CSIRO/ACCESS-ESM1-5/hist-GHG/r1i1p1f1/Amon/psl/gn/v20200615/psl_Amon_ACCESS-ESM1-5_hist-GHG_r1i1p1f1_gn_200001-202012.nc f5474029b4fc0822382b8024b0d074fc9f706b5f8b9fc1b86c8516b020bfcd1c
+CMIP6/DAMIP/CSIRO/ACCESS-ESM1-5/hist-GHG/r1i1p1f1/Amon/ts/gn/v20200615/ts_Amon_ACCESS-ESM1-5_hist-GHG_r1i1p1f1_gn_200001-202012.nc 90cfaa8bbb4d1207e35a1a8ae2c535d756b3efa6996b53446104e7c6d920260f
+CMIP6/DAMIP/CSIRO/ACCESS-ESM1-5/hist-GHG/r1i1p1f1/fx/areacella/gn/v20200615/areacella_fx_ACCESS-ESM1-5_hist-GHG_r1i1p1f1_gn.nc 10bb46e5bf3da739389f17239a5d128aa9fd419fdef7e96fc1e9ba50a49e0efb
+CMIP6/DAMIP/CSIRO/ACCESS-ESM1-5/hist-GHG/r2i1p1f1/Amon/psl/gn/v20200615/psl_Amon_ACCESS-ESM1-5_hist-GHG_r2i1p1f1_gn_200001-202012.nc 85a58bb08630b87bc0aecf65f064a6b5d8355b07aa01e5f6517e8f356eb55c0c
+CMIP6/DAMIP/CSIRO/ACCESS-ESM1-5/hist-GHG/r2i1p1f1/Amon/ts/gn/v20200615/ts_Amon_ACCESS-ESM1-5_hist-GHG_r2i1p1f1_gn_200001-202012.nc 5bcaa3c3f887baf1cf3edc9c891fa5c600489ce23983136cbd98226eeab46822
+CMIP6/DAMIP/CSIRO/ACCESS-ESM1-5/hist-GHG/r2i1p1f1/fx/areacella/gn/v20200615/areacella_fx_ACCESS-ESM1-5_hist-GHG_r2i1p1f1_gn.nc f5d513c3ebdbe88320794a36cbd229a68aab4ff0a13e240072e8b5ca09410ea7
+CMIP6/ScenarioMIP/CSIRO/ACCESS-ESM1-5/ssp126/r1i1p1f1/Amon/pr/gn/v20210318/pr_Amon_ACCESS-ESM1-5_ssp126_r1i1p1f1_gn_201501-210012.nc 0fe1c4b7c49ce1d7e7213c5bb5ea7b2597f68aef50f1795deefe07a5bafbc67c
+CMIP6/ScenarioMIP/CSIRO/ACCESS-ESM1-5/ssp126/r1i1p1f1/Amon/rsdt/gn/v20210318/rsdt_Amon_ACCESS-ESM1-5_ssp126_r1i1p1f1_gn_201501-202512.nc 3afba9008a6b334d2bc44b4038b012ae1eca95ab1c886936a7d07bbb2070a9c8
+CMIP6/ScenarioMIP/CSIRO/ACCESS-ESM1-5/ssp126/r1i1p1f1/Amon/rsut/gn/v20210318/rsut_Amon_ACCESS-ESM1-5_ssp126_r1i1p1f1_gn_201501-202512.nc b6f624150e1bfe987d10ef750b9ae72e2486927496285defc2a686ffaa5387bc
+CMIP6/ScenarioMIP/CSIRO/ACCESS-ESM1-5/ssp126/r1i1p1f1/Amon/tas/gn/v20210318/tas_Amon_ACCESS-ESM1-5_ssp126_r1i1p1f1_gn_201501-210012.nc 640678c83d60c562651fa409f09df8bb7ce560576938fdfd7c932ea10e585db6
+CMIP6/ScenarioMIP/CSIRO/ACCESS-ESM1-5/ssp126/r1i1p1f1/Omon/tos/gn/v20210318/tos_Omon_ACCESS-ESM1-5_ssp126_r1i1p1f1_gn_201501-202512.nc 31a85fade7f921d2650fbcd43f3886f7111d64e65d9c9b32d61e184efdd042bc
+CMIP6/ScenarioMIP/CSIRO/ACCESS-ESM1-5/ssp126/r1i1p1f1/fx/areacella/gn/v20210318/areacella_fx_ACCESS-ESM1-5_ssp126_r1i1p1f1_gn.nc b5ed05309c4a3000b551b1548d88cf1b910ad23347bc39f0094a935e26d3afe6
+obs4MIPs/NASA-JPL/AIRS-2-1/ta/gn/v20201110/taNobs_AIRS-2-1_gn_200209-201609.nc 3489895fc6cdd936ae64fa64fa221474e50f6b6bf347458c82d9a61f945f2d9d
+obs4MIPs/NASA-JPL/AIRS-2-1/ta/gn/v20201110/taStderr_AIRS-2-1_gn_200209-201609.nc 81e12ba5c6b058ace93737a3b69b317d2beb17e07fd6aa9f709b3e528ebfb4a2
+obs4MIPs/NASA-JPL/AIRS-2-1/ta/gn/v20201110/ta_AIRS-2-1_gn_200209-201609.nc a72d7172cd0c9df9eb0199082b196655490e5628fbb6a61ed1e7f8f83c610c0b
+obs4REF/obs4MIPs_PCMDI_monthly/MOHC/HadISST-1-1/mon/ts/gn/v20210727/ts_mon_HadISST-1-1_PCMDI_gn_187001-201907.nc 4f9a9270d001fc30488b49cdafe28e77db88e78e981ab580f0fae209f849a2da
+obs4REF/obs4MIPs_PCMDI_monthly/NOAA-ESRL-PSD/20CR/mon/psl/gn/v20210727/psl_mon_20CR_PCMDI_gn_187101-201212.nc 357e8915cc2ad30af1dd02cbecfb55f3083c13f54a11912e2f28396ccc84bd9c

climate_ref/datasets/__init__.py

@@ -0,0 +1,40 @@
+"""
+Dataset handling utilities
+"""
+
+from typing import TYPE_CHECKING, Any
+
+from climate_ref_core.datasets import SourceDatasetType
+
+if TYPE_CHECKING:
+    from climate_ref.datasets.base import DatasetAdapter
+
+
+def get_dataset_adapter(source_type: str, **kwargs: Any) -> "DatasetAdapter":
+    """
+    Get the appropriate adapter for the specified source type
+
+    Parameters
+    ----------
+    source_type
+        Type of source dataset
+
+    Returns
+    -------
+    :
+        DatasetAdapter instance
+    """
+    if source_type.lower() == SourceDatasetType.CMIP6.value:
+        from climate_ref.datasets.cmip6 import CMIP6DatasetAdapter
+
+        return CMIP6DatasetAdapter(**kwargs)
+    elif source_type.lower() == SourceDatasetType.obs4MIPs.value.lower():
+        from climate_ref.datasets.obs4mips import Obs4MIPsDatasetAdapter
+
+        return Obs4MIPsDatasetAdapter(**kwargs)
+    elif source_type.lower() == SourceDatasetType.PMPClimatology.value.lower():
+        from climate_ref.datasets.pmp_climatology import PMPClimatologyDatasetAdapter
+
+        return PMPClimatologyDatasetAdapter(**kwargs)
+    else:
+        raise ValueError(f"Unknown source type: {source_type}")

climate_ref/datasets/base.py

@@ -0,0 +1,214 @@
+from pathlib import Path
+from typing import Protocol
+
+import pandas as pd
+from loguru import logger
+from sqlalchemy.orm import joinedload
+
+from climate_ref.config import Config
+from climate_ref.database import Database
+from climate_ref.datasets.utils import validate_path
+from climate_ref.models.dataset import Dataset, DatasetFile
+from climate_ref_core.exceptions import RefException
+
+
+def _log_duplicate_metadata(
+    data_catalog: pd.DataFrame, unique_metadata: pd.DataFrame, slug_column: str
+) -> None:
+    # Drop out the rows where the values are the same
+    invalid_datasets = unique_metadata[unique_metadata.gt(1).any(axis=1)]
+    # Drop out the columns where the values are the same
+    invalid_datasets = invalid_datasets[invalid_datasets.columns[invalid_datasets.gt(1).any(axis=0)]]
+
+    for instance_id in invalid_datasets.index:
+        # Get the columns where the values are different
+        invalid_dataset_nunique = invalid_datasets.loc[instance_id]
+        invalid_dataset_columns = invalid_dataset_nunique[invalid_dataset_nunique.gt(1)].index.tolist()
+
+        # Include time_range in the list of invalid columns to make debugging easier
+        invalid_dataset_columns.append("time_range")
+
+        data_catalog_subset = data_catalog[data_catalog[slug_column] == instance_id]
+
+        logger.error(
+            f"Dataset {instance_id} has varying metadata:\n{data_catalog_subset[invalid_dataset_columns]}"
+        )
+
+
+class DatasetAdapter(Protocol):
+    """
+    An adapter to provide a common interface for different dataset types
+
+    This allows the same code to work with different dataset types.
+    """
+
+    dataset_cls: type[Dataset]
+    slug_column: str
+    dataset_specific_metadata: tuple[str, ...]
+    file_specific_metadata: tuple[str, ...] = ()
+
+    def pretty_subset(self, data_catalog: pd.DataFrame) -> pd.DataFrame:
+        """
+        Get a subset of the data_catalog to pretty print
+        """
+        ...
+
+    def find_local_datasets(self, file_or_directory: Path) -> pd.DataFrame:
+        """
+        Generate a data catalog from the specified file or directory
+
+        This data catalog should contain all the metadata needed by the database.
+        The index of the data catalog should be the dataset slug.
+        """
+        ...
+
+    def validate_data_catalog(self, data_catalog: pd.DataFrame, skip_invalid: bool = False) -> pd.DataFrame:
+        """
+        Validate a data catalog
+
+        Parameters
+        ----------
+        data_catalog
+            Data catalog to validate
+        skip_invalid
+            If True, ignore datasets with invalid metadata and remove them from the resulting data catalog.
+
+        Raises
+        ------
+        ValueError
+            If `skip_invalid` is False (default) and the data catalog contains validation errors.
+
+        Returns
+        -------
+        :
+            Validated data catalog
+        """
+        # Check if the data catalog contains the required columns
+        missing_columns = set(self.dataset_specific_metadata + self.file_specific_metadata) - set(
+            data_catalog.columns
+        )
+        if missing_columns:
+            raise ValueError(f"Data catalog is missing required columns: {missing_columns}")
+
+        # Verify that the dataset specific columns don't vary by dataset by counting the unique values
+        # for each dataset and checking if there are any that have more than one unique value.
+        unique_metadata = (
+            data_catalog[list(self.dataset_specific_metadata)].groupby(self.slug_column).nunique()
+        )
+        if unique_metadata.gt(1).any(axis=1).any():
+            _log_duplicate_metadata(data_catalog, unique_metadata, self.slug_column)
+
+            if skip_invalid:
+                data_catalog = data_catalog[
+                    ~data_catalog[self.slug_column].isin(
+                        unique_metadata[unique_metadata.gt(1).any(axis=1)].index
+                    )
+                ]
+            else:
+                raise ValueError("Dataset specific metadata varies by dataset")
+
+        return data_catalog
+
+    def register_dataset(
+        self, config: Config, db: Database, data_catalog_dataset: pd.DataFrame
+    ) -> Dataset | None:
+        """
+        Register a dataset in the database using the data catalog
+
+        Parameters
+        ----------
+        config
+            Configuration object
+        db
+            Database instance
+        data_catalog_dataset
+            A subset of the data catalog containing the metadata for a single dataset
+
+        Returns
+        -------
+        :
+            Registered dataset if successful, else None
+        """
+        DatasetModel = self.dataset_cls
+
+        self.validate_data_catalog(data_catalog_dataset)
+        unique_slugs = data_catalog_dataset[self.slug_column].unique()
+        if len(unique_slugs) != 1:
+            raise RefException(f"Found multiple datasets in the same directory: {unique_slugs}")
+        slug = unique_slugs[0]
+
+        dataset_metadata = data_catalog_dataset[list(self.dataset_specific_metadata)].iloc[0].to_dict()
+        dataset, created = db.get_or_create(DatasetModel, slug=slug, **dataset_metadata)
+        if not created:
+            logger.warning(f"{dataset} already exists in the database. Skipping")
+            return None
+        db.session.flush()
+        for dataset_file in data_catalog_dataset.to_dict(orient="records"):
+            path = validate_path(dataset_file.pop("path"))
+
+            db.session.add(
+                DatasetFile(
+                    path=str(path),
+                    dataset_id=dataset.id,
+                    start_time=dataset_file.pop("start_time"),
+                    end_time=dataset_file.pop("end_time"),
+                )
+            )
+        return dataset
+
+    def load_catalog(
+        self, db: Database, include_files: bool = True, limit: int | None = None
+    ) -> pd.DataFrame:
+        """
+        Load the data catalog containing the currently tracked datasets/files from the database
+
+        Iterating over different datasets within the data catalog can be done using a `groupby`
+        operation for the `instance_id` column.
+
+        The index of the data catalog is the primary key of the dataset.
+        This should be maintained during any processing.
+
+        Returns
+        -------
+        :
+            Data catalog containing the metadata for the currently ingested datasets
+        """
+        DatasetModel = self.dataset_cls
+        dataset_type = DatasetModel.__mapper_args__["polymorphic_identity"]
+        # TODO: Paginate this query to avoid loading all the data at once
+        if include_files:
+            result = (
+                db.session.query(DatasetFile)
+                # The join is necessary to be able to order by the dataset columns
+                .join(DatasetFile.dataset)
+                .where(Dataset.dataset_type == dataset_type)
+                # The joinedload is necessary to avoid N+1 queries (one for each dataset)
+                # https://docs.sqlalchemy.org/en/14/orm/loading_relationships.html#the-zen-of-joined-eager-loading
+                .options(joinedload(DatasetFile.dataset.of_type(DatasetModel)))
+                .order_by(Dataset.updated_at.desc())
+                .limit(limit)
+                .all()
+            )
+
+            return pd.DataFrame(
+                [
+                    {
+                        **{k: getattr(file, k) for k in self.file_specific_metadata},
+                        **{k: getattr(file.dataset, k) for k in self.dataset_specific_metadata},
+                    }
+                    for file in result
+                ],
+                index=[file.dataset.id for file in result],
+            )
+        else:
+            result_datasets = (
+                db.session.query(DatasetModel).order_by(Dataset.updated_at.desc()).limit(limit).all()
+            )
+
+            return pd.DataFrame(
+                [
+                    {k: getattr(dataset, k) for k in self.dataset_specific_metadata}
+                    for dataset in result_datasets
+                ],
+                index=[file.id for file in result_datasets],
+            )

climate_ref/datasets/cmip6.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import warnings
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import ecgtools.parsers
+import pandas as pd
+from ecgtools import Builder
+
+from climate_ref.datasets.base import DatasetAdapter
+from climate_ref.models.dataset import CMIP6Dataset
+
+
+def _parse_datetime(dt_str: pd.Series[str]) -> pd.Series[datetime | Any]:
+    """
+    Pandas tries to coerce everything to their own datetime format, which is not what we want here.
+    """
+
+    def _inner(date_string: str | None) -> datetime | None:
+        if not date_string:
+            return None
+
+        # Try to parse the date string with and without milliseconds
+        try:
+            dt = datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S")
+        except ValueError:
+            dt = datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S.%f")
+
+        return dt
+
+    return pd.Series(
+        [_inner(dt) for dt in dt_str],
+        index=dt_str.index,
+        dtype="object",
+    )
+
+
+def _apply_fixes(data_catalog: pd.DataFrame) -> pd.DataFrame:
+    def _fix_parent_variant_label(group: pd.DataFrame) -> pd.DataFrame:
+        if group["parent_variant_label"].nunique() == 1:
+            return group
+        group["parent_variant_label"] = group["variant_label"].iloc[0]
+
+        return group
+
+    data_catalog = (
+        data_catalog.groupby("instance_id")
+        .apply(_fix_parent_variant_label, include_groups=False)
+        .reset_index(level="instance_id")
+    )
+
+    if "branch_time_in_child" in data_catalog:
+        data_catalog["branch_time_in_child"] = _clean_branch_time(data_catalog["branch_time_in_child"])
+    if "branch_time_in_parent" in data_catalog:
+        data_catalog["branch_time_in_parent"] = _clean_branch_time(data_catalog["branch_time_in_parent"])
+
+    return data_catalog
+
+
+def _clean_branch_time(branch_time: pd.Series[str]) -> pd.Series[float]:
+    # EC-Earth3 uses "D" as a suffix for the branch_time_in_child and branch_time_in_parent columns
+    # Handle missing values (these result in nan values)
+    return pd.to_numeric(branch_time.astype(str).str.replace("D", ""), errors="coerce")
+
+
+class CMIP6DatasetAdapter(DatasetAdapter):
+    """
+    Adapter for CMIP6 datasets
+    """
+
+    dataset_cls = CMIP6Dataset
+    slug_column = "instance_id"
+
+    dataset_specific_metadata = (
+        "activity_id",
+        "branch_method",
+        "branch_time_in_child",
+        "branch_time_in_parent",
+        "experiment",
+        "experiment_id",
+        "frequency",
+        "grid",
+        "grid_label",
+        "institution_id",
+        "nominal_resolution",
+        "parent_activity_id",
+        "parent_experiment_id",
+        "parent_source_id",
+        "parent_time_units",
+        "parent_variant_label",
+        "product",
+        "realm",
+        "source_id",
+        "source_type",
+        "sub_experiment",
+        "sub_experiment_id",
+        "table_id",
+        "variable_id",
+        "variant_label",
+        "member_id",
+        "vertical_levels",
+        "version",
+        # Variable identifiers
+        "standard_name",
+        "long_name",
+        "units",
+        slug_column,
+    )
+
+    file_specific_metadata = ("start_time", "end_time", "path")
+
+    def __init__(self, n_jobs: int = 1):
+        self.n_jobs = n_jobs
+
+    def pretty_subset(self, data_catalog: pd.DataFrame) -> pd.DataFrame:
+        """
+        Get a subset of the data_catalog to pretty print
+
+        This is particularly useful for CMIP6 datasets, which have a lot of metadata columns.
+
+        Parameters
+        ----------
+        data_catalog
+            Data catalog to subset
+
+        Returns
+        -------
+        :
+            Subset of the data catalog to pretty print
+
+        """
+        return data_catalog[
+            [
+                "activity_id",
+                "institution_id",
+                "source_id",
+                "experiment_id",
+                "member_id",
+                "table_id",
+                "variable_id",
+                "grid_label",
+                "version",
+            ]
+        ]
+
+    def find_local_datasets(self, file_or_directory: Path) -> pd.DataFrame:
+        """
+        Generate a data catalog from the specified file or directory
+
+        Each dataset may contain multiple files, which are represented as rows in the data catalog.
+        Each dataset has a unique identifier, which is in `slug_column`.
+
+        Parameters
+        ----------
+        file_or_directory
+            File or directory containing the datasets
+
+        Returns
+        -------
+        :
+            Data catalog containing the metadata for the dataset
+        """
+        with warnings.catch_warnings():
+            # Ignore the DeprecationWarning from xarray
+            warnings.simplefilter("ignore", DeprecationWarning)
+
+            builder = Builder(
+                paths=[str(file_or_directory)],
+                depth=10,
+                include_patterns=["*.nc"],
+                joblib_parallel_kwargs={"n_jobs": self.n_jobs},
+            ).build(parsing_func=ecgtools.parsers.parse_cmip6)
+
+        datasets: pd.DataFrame = builder.df.drop(["init_year"], axis=1)
+
+        # Convert the start_time and end_time columns to datetime objects
+        # We don't know the calendar used in the dataset (TODO: Check what ecgtools does)
+        datasets["start_time"] = _parse_datetime(datasets["start_time"])
+        datasets["end_time"] = _parse_datetime(datasets["end_time"])
+
+        drs_items = [
+            "activity_id",
+            "institution_id",
+            "source_id",
+            "experiment_id",
+            "member_id",
+            "table_id",
+            "variable_id",
+            "grid_label",
+            "version",
+        ]
+        datasets["instance_id"] = datasets.apply(
+            lambda row: "CMIP6." + ".".join([row[item] for item in drs_items]), axis=1
+        )
+
+        # Temporary fix for some datasets
+        # TODO: Replace with a standalone package that contains metadata fixes for CMIP6 datasets
+        datasets = _apply_fixes(datasets)
+
+        return datasets