climate-ref 0.6.4__py3-none-any.whl → 0.6.6__py3-none-any.whl

climate_ref/models/dataset.py

@@ -30,7 +30,7 @@ class Dataset(Base):
 
     In the case of CMIP6 datasets, this is the instance_id.
     """
-    dataset_type: Mapped[SourceDatasetType] = mapped_column(nullable=False)
+    dataset_type: Mapped[SourceDatasetType] = mapped_column(nullable=False, index=True)
     """
     Type of dataset
     """
@@ -45,6 +45,16 @@ class Dataset(Base):
     Updating a dataset will trigger a new diagnostic calculation.
     """
 
+    # Universal finalisation flag for all dataset types
+    # Only CMIP6 currently uses unfinalised datasets in practice; other types should be finalised on creation.
+    finalised: Mapped[bool] = mapped_column(default=True, nullable=False)
+    """
+    Whether the complete set of metadata for the dataset has been finalised.
+
+    For CMIP6, ingestion may initially create unfinalised datasets (False) until all metadata is extracted.
+    For other dataset types (e.g., obs4MIPs, PMP climatology), this should be True upon creation.
+    """
+
     def __repr__(self) -> str:
         return f"<Dataset slug={self.slug} dataset_type={self.dataset_type} >"
 
@@ -63,7 +73,9 @@ class DatasetFile(Base):
     __tablename__ = "dataset_file"
 
     id: Mapped[int] = mapped_column(primary_key=True)
-    dataset_id: Mapped[int] = mapped_column(ForeignKey("dataset.id", ondelete="CASCADE"), nullable=False)
+    dataset_id: Mapped[int] = mapped_column(
+        ForeignKey("dataset.id", ondelete="CASCADE"), nullable=False, index=True
+    )
     """
     Foreign key to the dataset table
     """
@@ -90,9 +102,7 @@ class CMIP6Dataset(Dataset):
     """
     Represents a CMIP6 dataset
 
-    Fields that are not marked as required in
-    https://wcrp-cmip.github.io/WGCM_Infrastructure_Panel/Papers/CMIP6_global_attributes_filenames_CVs_v6.2.7.pdf
-    are optional.
+    Fields that are not in the DRS are marked optional.
     """
 
     __tablename__ = "cmip6_dataset"
@@ -102,37 +112,37 @@ class CMIP6Dataset(Dataset):
     branch_method: Mapped[str] = mapped_column(nullable=True)
     branch_time_in_child: Mapped[float] = mapped_column(nullable=True)
     branch_time_in_parent: Mapped[float] = mapped_column(nullable=True)
-    experiment: Mapped[str] = mapped_column()
-    experiment_id: Mapped[str] = mapped_column()
-    frequency: Mapped[str] = mapped_column()
-    grid: Mapped[str] = mapped_column()
+    experiment: Mapped[str] = mapped_column(nullable=True)
+    experiment_id: Mapped[str] = mapped_column(index=True)
+    frequency: Mapped[str] = mapped_column(nullable=True)
+    grid: Mapped[str] = mapped_column(nullable=True)
     grid_label: Mapped[str] = mapped_column()
     institution_id: Mapped[str] = mapped_column()
     long_name: Mapped[str] = mapped_column(nullable=True)
-    member_id: Mapped[str] = mapped_column()
-    nominal_resolution: Mapped[str] = mapped_column()
+    member_id: Mapped[str] = mapped_column(index=True)
+    nominal_resolution: Mapped[str] = mapped_column(nullable=True)
     parent_activity_id: Mapped[str] = mapped_column(nullable=True)
     parent_experiment_id: Mapped[str] = mapped_column(nullable=True)
     parent_source_id: Mapped[str] = mapped_column(nullable=True)
     parent_time_units: Mapped[str] = mapped_column(nullable=True)
     parent_variant_label: Mapped[str] = mapped_column(nullable=True)
-    realm: Mapped[str] = mapped_column()
-    product: Mapped[str] = mapped_column()
-    source_id: Mapped[str] = mapped_column()
-    standard_name: Mapped[str] = mapped_column()
-    source_type: Mapped[str] = mapped_column()
-    sub_experiment: Mapped[str] = mapped_column()
-    sub_experiment_id: Mapped[str] = mapped_column()
+    realm: Mapped[str] = mapped_column(nullable=True)
+    product: Mapped[str] = mapped_column(nullable=True)
+    source_id: Mapped[str] = mapped_column(index=True)
+    standard_name: Mapped[str] = mapped_column(nullable=True)
+    source_type: Mapped[str] = mapped_column(nullable=True)
+    sub_experiment: Mapped[str] = mapped_column(nullable=True)
+    sub_experiment_id: Mapped[str] = mapped_column(nullable=True)
     table_id: Mapped[str] = mapped_column()
-    units: Mapped[str] = mapped_column()
+    units: Mapped[str] = mapped_column(nullable=True)
     variable_id: Mapped[str] = mapped_column()
     variant_label: Mapped[str] = mapped_column()
     vertical_levels: Mapped[int] = mapped_column(nullable=True)
     version: Mapped[str] = mapped_column()
 
-    instance_id: Mapped[str] = mapped_column()
+    instance_id: Mapped[str] = mapped_column(index=True)
     """
-    Unique identifier for the dataset.
+    Unique identifier for the dataset (including the version).
     """
 
     __mapper_args__: ClassVar[Any] = {"polymorphic_identity": SourceDatasetType.CMIP6}  # type: ignore

climate_ref/models/execution.py

@@ -40,7 +40,7 @@ class ExecutionGroup(CreatedUpdatedMixin, Base):
 
     id: Mapped[int] = mapped_column(primary_key=True)
 
-    diagnostic_id: Mapped[int] = mapped_column(ForeignKey("diagnostic.id"))
+    diagnostic_id: Mapped[int] = mapped_column(ForeignKey("diagnostic.id"), index=True)
     """
     The diagnostic that this execution group belongs to
     """
@@ -103,8 +103,8 @@ class ExecutionGroup(CreatedUpdatedMixin, Base):
 execution_datasets = Table(
     "execution_dataset",
     Base.metadata,
-    Column("execution_id", ForeignKey("execution.id")),
-    Column("dataset_id", ForeignKey("dataset.id")),
+    Column("execution_id", ForeignKey("execution.id"), index=True),
+    Column("dataset_id", ForeignKey("dataset.id"), index=True),
 )
 
 
@@ -136,7 +136,8 @@ class Execution(CreatedUpdatedMixin, Base):
         ForeignKey(
             "execution_group.id",
             name="fk_execution_id",
-        )
+        ),
+        index=True,
     )
     """
     The execution group that this execution belongs to
@@ -149,7 +150,7 @@ class Execution(CreatedUpdatedMixin, Base):
     This is used to verify if an existing diagnostic execution has been run with the same datasets.
     """
 
-    successful: Mapped[bool] = mapped_column(nullable=True)
+    successful: Mapped[bool] = mapped_column(nullable=True, index=True)
     """
     Was the run successful
     """

climate_ref/models/metric_value.py

@@ -47,13 +47,13 @@ class MetricValue(CreatedUpdatedMixin, Base):
     }
 
     id: Mapped[int] = mapped_column(primary_key=True)
-    execution_id: Mapped[int] = mapped_column(ForeignKey("execution.id"))
+    execution_id: Mapped[int] = mapped_column(ForeignKey("execution.id"), index=True)
 
     attributes: Mapped[dict[str, Any]] = mapped_column()
 
     execution: Mapped["Execution"] = relationship(back_populates="values")
 
-    type: Mapped[MetricValueType] = mapped_column()
+    type: Mapped[MetricValueType] = mapped_column(index=True)
     """
     Type of metric value
 

climate_ref/testing.py

@@ -27,7 +27,7 @@ def _determine_test_directory() -> Path | None:
 
 
 TEST_DATA_DIR = _determine_test_directory()
-SAMPLE_DATA_VERSION = "v0.6.3"
+SAMPLE_DATA_VERSION = "v0.7.2"
 
 
 def fetch_sample_data(force_cleanup: bool = False, symlink: bool = False) -> None:

{climate_ref-0.6.4.dist-info → climate_ref-0.6.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: climate-ref
-Version: 0.6.4
+Version: 0.6.6
 Summary: Application which runs the CMIP Rapid Evaluation Framework
 Author-email: Jared Lewis <jared.lewis@climate-resource.com>, Mika Pflueger <mika.pflueger@climate-resource.com>, Bouwe Andela <b.andela@esciencecenter.nl>, Jiwoo Lee <lee1043@llnl.gov>, Min Xu <xum1@ornl.gov>, Nathan Collier <collierno@ornl.gov>, Dora Hegedus <dora.hegedus@stfc.ac.uk>
 License-Expression: Apache-2.0
@@ -70,7 +70,7 @@ pip install climate-ref
 If you want to use the diagnostic providers for the Assessment Fast Track, you can install them with:
 
 ```bash
-pip install climate-ref[aft-providers]
+pip install "climate-ref[aft-providers]"
 ```
 
 ## Quick Start

{climate_ref-0.6.4.dist-info → climate_ref-0.6.6.dist-info}/RECORD

@@ -1,33 +1,35 @@
 climate_ref/__init__.py,sha256=M45QGfl0KCPK48A8MjI08weNvZHMYH__GblraQMxsoM,808
 climate_ref/_config_helpers.py,sha256=-atI5FX7SukhLE_jz_rL-EHQ7s0YYqKu3dSFYWxSyMU,6632
 climate_ref/alembic.ini,sha256=WRvbwSIFuZ7hWNMnR2-yHPJAwYUnwhvRYBzkJhtpGdg,3535
-climate_ref/config.py,sha256=4jlqYnR5es9fmKI3sfT9snnutH342o9XAMf6ipNS2lI,17083
+climate_ref/config.py,sha256=WW6R7RLwEDuI11XYLYO57FwvmQz1psq9bNM3WVL3e_s,17481
 climate_ref/constants.py,sha256=9RaNLgUSuQva7ki4eRW3TjOKeVP6T81QNiu0veB1zVk,111
 climate_ref/database.py,sha256=b_6XHdr78Mo7KeLqQJ5DjLsySHPdQE83P8dRpdMfzfM,8661
 climate_ref/provider_registry.py,sha256=dyfj4vU6unKHNXtT03HafQtAi3LilL37uvu3paCnmNY,4159
 climate_ref/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 climate_ref/slurm.py,sha256=XWXVPdXP-4BDib3bxYW9uPcAJdPpo1ixYZAI_y1cZuw,7305
 climate_ref/solver.py,sha256=T5sQjweSvpUMG4q8MfbGjljxa5kBgKxNotT78PwyxqU,16804
-climate_ref/testing.py,sha256=1b9lVCJlKxjJ7JGq6zDD2gK3BEM9ZVv1dbA-j6yb4Yk,4256
+climate_ref/testing.py,sha256=nSB8levCt0Ol6odumBJgy5HMKbusRCr19SuIbkMN3MQ,4256
 climate_ref/cli/__init__.py,sha256=fvENOeL4j7CMPANVxrDlWfaB0QUvvYgrWcm5ptbL0P8,4350
 climate_ref/cli/_utils.py,sha256=6bIb8zEVvzXyKpv8MG58T-T2L2jH-G8WNrOOGpz3uCw,1918
 climate_ref/cli/config.py,sha256=ak4Rn9S6fH23PkHHlI-pXuPiZYOvUB4r26eu3p525-M,532
 climate_ref/cli/datasets.py,sha256=4iYQZ0ceoF-Cd8eSpS4Q8b9aLt_vEDUw5slzGw02DsY,8277
-climate_ref/cli/executions.py,sha256=2MjwxCdRB-uVJUg7RluDIf_IsoclRn22ibJjk_nhfPo,7215
+climate_ref/cli/executions.py,sha256=fKQ-3IvYaTEOCIhqNNrFWgwGyHPeVmKWlDQ8VH-K9rM,7721
 climate_ref/cli/providers.py,sha256=-5hQkJc01jON1bc1dk--tSWTesyiHOzZuYMb9Vxge9k,2613
 climate_ref/cli/solve.py,sha256=ZTXrwDFDXNrX5GLMJTN9tFnpV3zlcZbEu2aF3JDJVxI,2367
 climate_ref/dataset_registry/obs4ref_reference.txt,sha256=2zJMbsAsQ49KaWziX3CqrlILq9yN7S2ygmfV3V5rsnw,8395
-climate_ref/dataset_registry/sample_data.txt,sha256=3JAHy14pRbLlo9-oNxUXLgZ_QOFJXUieEftBbapSY8E,20124
+climate_ref/dataset_registry/sample_data.txt,sha256=MK-tvVRLOeNtEnmIQVnZrz27dSmsoSVsxmgTC6R-3v0,45237
 climate_ref/datasets/__init__.py,sha256=PV3u5ZmhyfcHbKqySgwVA8m4-naZgxzydLXSBqdTGLM,1171
-climate_ref/datasets/base.py,sha256=yoip8UCcTCUPn2xVlsJ1If9zXw_476dDYViH5iMgcIE,10352
-climate_ref/datasets/cmip6.py,sha256=3MVJ1kPdw6f6V3G4gdHIiqDGUyMqPs-_wttkw2YKAH0,8425
-climate_ref/datasets/obs4mips.py,sha256=CmMm4kopfb0yFsMSgUlHUm8clGJImBaodSkh6lAv_Ug,5926
+climate_ref/datasets/base.py,sha256=uZ55u625ckRNjsn-AqJg4_xO5uvHchqYvwBZIt4iHtY,11017
+climate_ref/datasets/cmip6.py,sha256=KO761ConHvX40n9X0xLrxjhzN7wmighNWL2JyYygRAA,7049
+climate_ref/datasets/cmip6_parsers.py,sha256=wH4WKQAR2_aniXwsW7nch6nIpXk2pSpPxkT4unjV4hQ,6041
+climate_ref/datasets/obs4mips.py,sha256=q0_erQb4k5KBaGMvEGgUtVSDvXQjuftqDmvW4QZpWZI,6138
 climate_ref/datasets/pmp_climatology.py,sha256=goHDc_3B2Wdiy_hmpERNvWDdDYZACPOyFDt3Du6nGc0,534
 climate_ref/datasets/utils.py,sha256=iLJO7h4G3DWsRe9hIC4qkIyi5_zIW1ZMw-FDASLujtM,359
-climate_ref/executor/__init__.py,sha256=PYtJs3oBS_GiUHbt8BF-6wJibpF6_vREm1Cg9TxVbLI,648
-climate_ref/executor/hpc.py,sha256=4o90sCyoC4jlkem3BXNo4uwFZpIvOUGfrqYucB6EtU8,12251
+climate_ref/executor/__init__.py,sha256=tXuXxoFQKH9-te4O3bJg9dVrBKtjRS2si0yzRsHVfGk,902
+climate_ref/executor/hpc.py,sha256=DEp_Q5kDfBTPF_4mgyedjKbXtNZ-OmIkS9bbYo2B1FQ,14153
 climate_ref/executor/local.py,sha256=65LUl41YtURFb87YTWZQHjDpIRlIKJ5Ny51c9DZjy0s,8582
-climate_ref/executor/result_handling.py,sha256=i7ZMX5vvyPY5gW-WWd-JHLi1BLviB9FXhn4FE8C9d4w,7787
+climate_ref/executor/pbs_scheduler.py,sha256=WoH1sTmDl7bdmYodpcxZjkUSvInYUcWR4x7buIgBxqk,5807
+climate_ref/executor/result_handling.py,sha256=2VQHbLKdv109pQwQO_cTqxsAXAq9S-s4w8aYg10VWms,10796
 climate_ref/executor/synchronous.py,sha256=o4TndsoKMu9AzJYLkusU9lRkgHCy6HcCP46tEs6o86U,1895
 climate_ref/migrations/README,sha256=xM5osYbyEbEFA2eh5kwary_oh-5VFWtDubA-vgWwvlE,935
 climate_ref/migrations/env.py,sha256=8GvBLhGTuQy6MKYMj7QszJEQ2LNewf1Z9kB9dBHQs9I,4375
@@ -35,16 +37,19 @@ climate_ref/migrations/script.py.mako,sha256=MEqL-2qATlST9TAOeYgscMn1uy6HUS9NFvD
 climate_ref/migrations/versions/2025-05-02T1418_341a4aa2551e_regenerate.py,sha256=S8Q4THCI4TPnlaQHgQJUCiNW5LAyQClaiTB-0dwhtXU,14050
 climate_ref/migrations/versions/2025-05-09T2032_03dbb4998e49_series_metric_value.py,sha256=s9nZ_l64pSF7sWN53rRPCQlqW_xHqR8tlWhU-ovmsME,2043
 climate_ref/migrations/versions/2025-07-03T1505_795c1e6cf496_drop_unique_requirement_on_slug.py,sha256=TfBHJkm3oPlz0P5Z1tiY6LBp2B1oDvdyL_OOYoV-OiI,984
+climate_ref/migrations/versions/2025-07-20T1521_94beace57a9c_cmip6_finalised.py,sha256=pzxhT9aFaEqyyUOWf0AExsr6sMW9syyr1RUhy0nj3AM,2907
+climate_ref/migrations/versions/2025-08-05T0327_a1b2c3d4e5f6_finalised_on_base_dataset.py,sha256=Uen3ryUfmIpASeIQYuUJd9xDxnEl5ItTx3jKuHoDSko,2033
+climate_ref/migrations/versions/2025-09-05T2019_8d28e5e0f9c3_add_indexes.py,sha256=r8xKMIykdQbnrtBgmfTtfs5uHlmiQJNp7g5IdWi4zrs,5456
 climate_ref/models/__init__.py,sha256=rUDKRANeAEAHVOrzJVIZoZ99dDG5O4AGzHmOpC876Nc,801
-climate_ref/models/base.py,sha256=YMyovT2Z_tRv59zz6qC9YCCDodhO3x6OLnFdBtPJkho,1271
-climate_ref/models/dataset.py,sha256=Rpwrx0HqOJBHs4Sb4n6B0In__Uo0PqXSZKvZR-juGCg,7491
+climate_ref/models/base.py,sha256=jY7aMH1DAekH2Zs5v-r1O4mpAAfZq2Fn4EGXrO1vFvk,1297
+climate_ref/models/dataset.py,sha256=eW0FV--ImpeZH08cGlC5PPKD088F4D_zv5qEVet8iNo,8156
 climate_ref/models/diagnostic.py,sha256=0mKVvASEWNxx41R2Y-5VxplarZ4JAP6q0oaO14FKZuk,1751
-climate_ref/models/execution.py,sha256=lRCpaKLSR7rZbuoL94GW76tm9wLMsSDoIOA7bIa6xgY,9848
-climate_ref/models/metric_value.py,sha256=44OLcZz-qLx-p_9w7YWDKpD5S7Y9HyTKKsvSb77RBro,10190
+climate_ref/models/execution.py,sha256=iqT8jrbP_RTyDtIO2nAczmp-rtouuaHzkb7g1IOhJ-0,9917
+climate_ref/models/metric_value.py,sha256=XsmZvp21WMykBjDePDqsZsT2UefEFFskoZCd4uOXK-4,10212
 climate_ref/models/provider.py,sha256=RAE2qAAxwObu-72CdK4kt5ACMmKYEn07WJm7DU9hF28,990
-climate_ref-0.6.4.dist-info/METADATA,sha256=2SVSEweQ-ZaXnnpymVG9iBc8ImUBwQwnVF9Y2-0tJz8,4505
-climate_ref-0.6.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-climate_ref-0.6.4.dist-info/entry_points.txt,sha256=IaggEJlDIhoYWXdXJafacWbWtCcoEqUKceP1qD7_7vU,44
-climate_ref-0.6.4.dist-info/licenses/LICENCE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-climate_ref-0.6.4.dist-info/licenses/NOTICE,sha256=4qTlax9aX2-mswYJuVrLqJ9jK1IkN5kSBqfVvYLF3Ws,128
-climate_ref-0.6.4.dist-info/RECORD,,
+climate_ref-0.6.6.dist-info/METADATA,sha256=1Y-tl8sg-djWxY2UeLsN_A5paYJAFPvZGVf1rfJPtuE,4507
+climate_ref-0.6.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+climate_ref-0.6.6.dist-info/entry_points.txt,sha256=IaggEJlDIhoYWXdXJafacWbWtCcoEqUKceP1qD7_7vU,44
+climate_ref-0.6.6.dist-info/licenses/LICENCE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+climate_ref-0.6.6.dist-info/licenses/NOTICE,sha256=4qTlax9aX2-mswYJuVrLqJ9jK1IkN5kSBqfVvYLF3Ws,128
+climate_ref-0.6.6.dist-info/RECORD,,