esgvoc 2.0.2__py3-none-any.whl

esgvoc/api/data_descriptors/source.py

@@ -0,0 +1,120 @@
+"""
+Model (i.e. schema/definition) of the source descriptor
+"""
+
+from pydantic import Field
+
+from esgvoc.api.data_descriptors.organisation import Organisation
+from esgvoc.api.data_descriptors.data_descriptor import PlainTermDataDescriptor
+from esgvoc.api.data_descriptors.EMD_models.model_component import EMDModelComponent
+from esgvoc.api.pydantic_handler import create_union
+
+
+class SourceCMIP7(PlainTermDataDescriptor):
+    """
+    Source of the dataset (CMIP7 format with contributors and model_components)
+
+    Examples: "CanESM6-MR", "CR-CMIP-1-0-0"
+
+    The more precise meaning of source depends on the kind of dataset this is.
+    For model output, 'source' refers to a numerical representations of the Earth's climate system.
+    This source is the model which was used to generate the dataset.
+    Such models simulate the interactions between the atmosphere, oceans, land surface, and ice.
+    They are based on fundamental physical, chemical, and biological processes
+    and are used to understand past, present, and future climate conditions.
+    Each source or model is typically associated with a specific research institution, center, or group.
+    For instance, models like 'EC-Earth' are developed by a consortium of European institutes,
+    while 'GFDL-CM4' is developed by the Geophysical Fluid Dynamics Laboratory (GFDL) in the United States.
+
+    For model inputs i.e. forcings, the 'source' is a unique identifier
+    for the group that produced the data and its version.
+    This is a different convention from almost all other cases
+    (which really muddies the meaning of the term).
+    """
+
+    label: str
+    """
+    Label to use for this source
+
+    Unlike the `drs_name`, this can contain any characters
+    """
+
+    label_extended: str
+    """
+    Extended label to use for this source
+
+    Unlike the `drs_name`, this can contain any characters.
+    If desired, it can include lots of verbose information
+    (unlike `label`, which should be more terse).
+    It can also just be the same as `label`
+    if the person registering the source wishes.
+    """
+
+    # Note: Allowing str is under discussion.
+    # Using this to get things working.
+    # Long-term, we might do something different.
+    contributors: list[Organisation | str]
+    """
+    Organisation(s) using this source
+
+    Using is a bit vaguely defined, but in practice it is the organisation(s)
+    that submit data using this source.
+    """
+
+    # Note: Allowing str is under discussion.
+    # Using this to get things working.
+    # Long-term, we might do something different.
+    model_components: list[EMDModelComponent | str]
+    """
+    Model components
+
+    If this source is not a model, this can/will just be an empty list.
+    """
+
+    @property
+    def source(self) -> str:
+        """
+        Source label as used by CMOR
+        """
+        raise NotImplementedError
+        # Something like:
+        # label (release year from EMD if known):
+        # (for each model component)\n component: component name (description)
+
+
+class SourceLegacy(PlainTermDataDescriptor):
+    """
+    Legacy source model for CMIP6 and earlier versions.
+
+    This version uses different field names and structure compared to the CMIP7 format.
+    """
+
+    activity_participation: list[str] | None = None
+    """Activities this source participates in."""
+
+    cohort: list[str] = Field(default_factory=list)
+    """Cohort grouping for this source."""
+
+    organisation_id: list[str] = Field(default_factory=list)
+    """Organisation IDs associated with this source."""
+
+    label: str
+    """Label to use for this source."""
+
+    label_extended: str | None = None
+    """Extended label to use for this source."""
+
+    license: dict = Field(default_factory=dict)
+    """License information for this source."""
+
+    model_component: dict | None = Field(
+        default=None,
+        description="Dictionary containing the model components that make up this climate source"
+    )
+    """Model component information (legacy format)."""
+
+    release_year: int | None = None
+    """Year this source was released."""
+
+
+Source = create_union(SourceCMIP7, SourceLegacy)

esgvoc/api/data_descriptors/source_type.py

@@ -0,0 +1,5 @@
+from esgvoc.api.data_descriptors.data_descriptor import PlainTermDataDescriptor
+
+
+class SourceType(PlainTermDataDescriptor):
+    pass

esgvoc/api/data_descriptors/sub_experiment.py

@@ -0,0 +1,5 @@
+from esgvoc.api.data_descriptors.data_descriptor import PlainTermDataDescriptor
+
+
+class SubExperiment(PlainTermDataDescriptor):
+    pass

esgvoc/api/data_descriptors/table.py

@@ -0,0 +1,28 @@
+from pydantic import Field, field_validator
+
+from esgvoc.api.data_descriptors.data_descriptor import PlainTermDataDescriptor
+
+
+class Table(PlainTermDataDescriptor):
+    product: str | None
+    table_date: str | None
+    variable_entry: list[str] = Field(default_factory=list)
+
+    @field_validator("variable_entry", mode="before")
+    @classmethod
+    def normalize_variable_entry(cls, v):
+        """
+        Normalize variable_entry to ensure all items are strings.
+        If items are dicts (resolved references), extract the 'id' field.
+        """
+        if not isinstance(v, list):
+            return v
+
+        result = []
+        for item in v:
+            if isinstance(item, dict):
+                # Extract the id from the resolved object
+                result.append(item.get("id", str(item)))
+            else:
+                result.append(item)
+        return result

esgvoc/api/data_descriptors/temporal_label.py

@@ -0,0 +1,20 @@
+"""
+Model (i.e. schema/definition) of the temporal label data descriptor
+"""
+
+from esgvoc.api.data_descriptors.data_descriptor import PlainTermDataDescriptor
+
+
+class TemporalLabel(PlainTermDataDescriptor):
+    """
+    Label that describes a specific temporal sampling approach
+
+    Examples: "tavg", "tpt", "tclm"
+
+    This is set to "ti" ("time-independent") when the data has no time axis.
+    For underlying details and logic, please see
+    [Taylor et al., 2025](https://docs.google.com/document/d/19jzecgymgiiEsTDzaaqeLP6pTvLT-NzCMaq-wu-QoOc/edit?pli=1&tab=t.0).
+
+    This label is used as the area component of a branded variable's suffix
+    (see :py:class:`BrandedSuffix`).
+    """  # noqa: E501

esgvoc/api/data_descriptors/time_range.py

@@ -0,0 +1,17 @@
+"""
+Model (i.e. schema/definition) of the time range data descriptor
+"""
+
+from esgvoc.api.data_descriptors.data_descriptor import PatternTermDataDescriptor
+
+
+class TimeRange(PatternTermDataDescriptor):
+    """
+    Time range spanned by the data
+
+    Examples: "185001-202112", "18500101-20211231", "203101010130-203112312230", "185001-186412-clim"
+
+    The right choice of time range is tightly coupled to the frequency of the data.
+    This coupling is not captured within the CVs.
+    (It is hopefully enforced elsewhere e.g. in QAQC workflows.)
+    """

esgvoc/api/data_descriptors/title.py

@@ -0,0 +1,5 @@
+from esgvoc.api.data_descriptors.data_descriptor import PatternTermDataDescriptor
+
+
+class Title(PatternTermDataDescriptor):
+    pass

esgvoc/api/data_descriptors/tracking_id.py

@@ -0,0 +1,67 @@
+"""
+Model (i.e. schema/definition) of the tracking ID data descriptor
+"""
+
+from esgvoc.api.data_descriptors.data_descriptor import PatternTermDataDescriptor
+
+
+class TrackingId(PatternTermDataDescriptor):
+    """
+    Tracking ID, i.e. unique ID, of a file
+
+    Examples: "hdl:21.14107/f6635404-8a1a-4aa9-918d-3792e8321f04",
+              "hdl:21.14100/718ee427-4efb-46a8-9f89-8192593b15fe"
+
+    This data descriptor applies only at the file level,
+    not to datasets as a whole (each file in a dataset gets a unique ID).
+
+    The regex is of the form `hdl:<prefix>/<uuid>`,
+    where `<prefix>` is a prefix which is the same for all files in the same 'group'
+    and `<uuid>` is a universally unique ID (UUID).
+
+    The 'group' is a bit loosely defined and can be defined in different ways
+    for different purposes.
+    For CMIP phases, the 'group' is usually the CMIP phase
+    i.e. all files that are part of the same CMIP phase use the same tracking ID prefix
+    (e.g. all CMIP6 files have the same prefix, all CMIP7 files have the same prefix).
+    (Also note that some projects haven't really got this right,
+    e.g. the input4MIPs project has re-used the CMIP6 prefix, rather than using its own.)
+
+    The prefixes come from [handle.net](https://www.handle.net/index.html).
+    [handle.net](https://www.handle.net/index.html) prefixes
+    are allotted to different CMIP (or other project) exercises.
+    These prefixes are used by the
+    [handle.net](https://www.handle.net/index.html)
+    service to group all the entries for the given exercise together.
+    These prefixes allow a) each file to have a unique ID
+    and b) users to look up entries for all files using their unique ID
+    via the [handle.net](https://www.handle.net/index.html) service.
+    The prefixes are currently managed and registered by DKRZ
+    on behalf of the ESGF team (we think, it's not 100% clear).
+
+    The last part of the tracking ID is a UUID.
+    The specification of a UUID is defined elsewhere
+    (apparently in ISO/IEC 9834-8:2014).
+    A new UUID must be generated for every single file
+    such that every file has a unique tracking id
+    (this uniqueness is both within a project thanks to the differing UUIDs
+    and across projects thanks to the differing prefixes).
+    Most programming languages have native support for UUID generation
+    (e.g. the `uuid` library is part of Python's standard library).
+    For a standalone solution, the OSSP utility is available.
+    It can be accessed from http://www.ossp.org/pkg/lib/uuid/.
+    Since CMIP6, version 4 UUIDs (random number based) have been required.
+
+    The tracking IDs are
+    used by a PID service
+    so that users can find further information about the file
+    by going to `hdl.handle.net/<tracking_id_after_the_hdl_colon_prefix_is_removed>`
+    e.g. `hdl.handle.net/21.14107/f6635404-8a1a-4aa9-918d-3792e8321f04`
+    (a working link from CMIP6 for those who would like to see a live demonstration is
+    [hdl.handle.net/21.14100/f2f502c9-9626-31c6-b016-3f7c0534803b](),
+    which was inferred from a file in which the tracking ID is
+    `hdl:21.14100/f2f502c9-9626-31c6-b016-3f7c0534803b`).
+    (Or at least this link with handle.net is the intention.
+    It hasn't always happened
+    e.g this is not the case for all CMIP7 input4MIPs files.)
+    """

esgvoc/api/data_descriptors/variable.py

@@ -0,0 +1,56 @@
+"""
+Model (i.e. schema/definition) of the variable data descriptor
+"""
+
+from esgvoc.api.data_descriptors.data_descriptor import PlainTermDataDescriptor
+
+
+class Variable(PlainTermDataDescriptor):
+    """
+    A climate-related quantity or measurement.
+
+    Examples: "tas", "pr", "psl", "rlut"
+
+    These quantities represent key physical, chemical or biological properties of the Earth system
+    and can be the result of direct observation of the climate system or simulations.
+    Variables cover a range of aspects of the climate system,
+    such as temperature, precipitation, sea level, radiation, or atmospheric composition.
+    Some more information for variables that have been used in CMIP:
+
+    - *tas*: Near-surface air temperature (measured at 2 meters above the surface)
+    - *pr*: Precipitation
+    - *psl*: Sea-level pressure
+    - *zg*: Geopotential height
+    - *rlut*: Top-of-atmosphere longwave radiation
+    - *siconc*: Sea-ice concentration
+    - *co2*: Atmospheric CO2 concentration
+
+    Since CMIP7, the concept of a variable has been augmented with the idea of 'branding',
+    leading to the idea of a 'branded variable'.
+    For details, see :py:class:`BrandedVariable`.
+
+    Sometimes 'variable' is also referred to as 'root name' or 'out name'.
+    There is mostly a one to one mapping between CF standard names and variables.
+    However, this is not always possible so please don't assume this is always the case.
+    """
+
+    long_name: str | None
+    """
+    Long name of the variable
+
+    This is free text and can take any value
+    """
+
+    standard_name: str | None
+    """
+    Standard name of the variable
+
+    The standard names are defined by the CF-conventions.
+
+    If `None`, this variable has no standard name according to the CF-conventions.
+    """
+
+    units: str
+    """
+    Units of the variable
+    """

esgvoc/api/data_descriptors/variant_label.py

@@ -0,0 +1,25 @@
+"""
+Model (i.e. schema/definition) of the forcing index data descriptor
+"""
+
+from esgvoc.api.data_descriptors.data_descriptor import PatternTermDataDescriptor
+
+
+class VariantLabel(PatternTermDataDescriptor):
+    """
+    The variant which provides information about how a dataset was created
+
+    Examples: "r1i1p1f1", "r2i2p2f1", "r1i198001p1f1", "r1i198001ap1f1", "r1i199001bp1f1"
+
+    Really, this should be a composite term.
+    However, as there is no separator between the parts, it has to be a pattern term
+    (at least at the moment).
+
+    As of the latest round of CMIP,
+    the variant label is usually composed of the following components:
+
+    #. :py:class:`RealizationIndex`
+    #. :py:class:`InitializationIndex`
+    #. :py:class:`PhysicsIndex`
+    #. :py:class:`ForcingIndex`
+    """

esgvoc/api/data_descriptors/vertical_label.py

@@ -0,0 +1,20 @@
+"""
+Model (i.e. schema/definition) of the vertical label data descriptor
+"""
+
+from esgvoc.api.data_descriptors.data_descriptor import PlainTermDataDescriptor
+
+
+class VerticalLabel(PlainTermDataDescriptor):
+    """
+    Label that describes a specific vertical sampling approach
+
+    Examples: "h2m", "200hPa", "p19"
+
+    This is set to "u" ("unspecified") when the data has no vertical dimension.
+    For underlying details and logic, please see
+    [Taylor et al., 2025](https://docs.google.com/document/d/19jzecgymgiiEsTDzaaqeLP6pTvLT-NzCMaq-wu-QoOc/edit?pli=1&tab=t.0).
+
+    This label is used as the area component of a branded variable's suffix
+    (see :py:class:`BrandedSuffix`).
+    """  # noqa: E501

esgvoc/api/project_specs.py

@@ -0,0 +1,143 @@
+from enum import Enum
+
+from pydantic import BaseModel, ConfigDict
+
+
+class DrsType(str, Enum):
+    """
+    The types of DRS specification (directory, file name and dataset id).
+    """
+
+    DIRECTORY = "directory"
+    """The DRS directory specification type."""
+    FILE_NAME = "file_name"
+    """The DRS file name specification type."""
+    DATASET_ID = "dataset_id"
+    """The DRS dataset id specification type."""
+
+
+class DrsPart(BaseModel):
+    """A fragment of a DRS specification"""
+
+    source_collection: str
+    """The collection id."""
+    source_collection_term: str | None = None
+    "Specifies a specific term in the collection."
+    is_required: bool
+    """Whether the collection is required for the DRS specification or not."""
+
+    def __str__(self) -> str:
+        return self.source_collection
+
+
+class DrsSpecification(BaseModel):
+    """
+    A DRS specification.
+    """
+
+    type: DrsType
+    """The type of the specification."""
+    regex: str
+    """General pattern for simples checks"""
+    separator: str
+    """The textual separator string or character."""
+    properties: dict | None = None
+    """The other specifications (e.g., file name extension for file name DRS specification)."""
+    parts: list[DrsPart]
+    """The parts of the DRS specification."""
+
+
+class AttributeProperty(BaseModel):
+    """
+    A NetCDF global attribute property specification.
+    """
+
+    source_collection: str
+    "The project collection that originated the property."
+    is_required: bool
+    "Specifies if the attribute must be present in the NetCDF file."
+    value_type: str
+    "The type of the attribute value."
+    specific_key: str | None = None
+    "Specifies a specific key in the collection."
+    field_name: str | None = None
+    "The name of the attribute field."
+    default_value: str | None = None
+    "The default value for the attribute."
+
+
+class CatalogProperty(BaseModel):
+    """
+    A dataset property described in a catalog.
+    """
+
+    source_collection: str | None
+    "The project collection that originated the property. `None` value means that the property "
+    "is not related to any collection of the project. So the property has limited specifications."
+    catalog_field_value_type: str
+    "The type of the field value."
+    is_required: bool
+    "Specifies if the property must be present in the dataset properties."
+    source_collection_term: str | None = None
+    "Specifies a specific term in the collection."
+    catalog_field_name: str | None = None
+    "The name of the collection referenced in the catalog."
+    source_collection_key: str | None = None
+    "Specifies a key other than drs_name in the collection."
+
+
+class CatalogExtension(BaseModel):
+    name: str
+    """The name of the extension"""
+    version: str
+    """The version of the extension"""
+
+
+class CatalogProperties(BaseModel):
+    name: str
+    """The name of the catalog system."""
+    url_template: str
+    """The URI template of the catalog system."""
+    extensions: list[CatalogExtension]
+    """The extensions of the catalog."""
+
+
+AttributeSpecification = list[AttributeProperty]
+
+
+class CatalogSpecification(BaseModel):
+    """
+    A catalog specifications.
+    """
+
+    version: str
+    """The version of the catalog."""
+
+    catalog_properties: CatalogProperties
+    """The properties of the catalog."""
+
+    dataset_properties: list[CatalogProperty]
+    "The properties of the dataset described in a catalog."
+    file_properties: list[CatalogProperty]
+    "The properties of the files described in a catalog."
+
+
+class ProjectSpecs(BaseModel):
+    """
+    A project specifications.
+    """
+
+    project_id: str
+    """The project id."""
+    description: str
+    """The description of the project."""
+    version: str
+    """The git_hash used as the version"""
+    drs_specs: dict[DrsType, DrsSpecification] | None = None
+    """The DRS specifications of the project (directory, file name and dataset id)."""
+    # TODO: release = None when all projects have catalog_specs.yaml.
+    catalog_specs: CatalogSpecification | None = None
+    """The catalog specifications of the project."""
+    attr_specs: AttributeSpecification | None = None
+    """The NetCDF global attribute specifications of the project."""
+    model_config = ConfigDict(extra="allow")