earthcode 0.1.0__py3-none-any.whl

earthcode/metadata_input_definitions.py

@@ -0,0 +1,338 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class ProjectCollectionMetadata(BaseModel):
+    """Defines validated inputs used to construct an OSC project collection."""
+
+    project_id: str = Field(
+        ...,
+        description="Custom project identifier using dashes to separate words, e.g. '4datlantic-ohc'.",
+    )
+    project_title: str = Field(
+        ...,
+        description="Official project title as used in the ESA contract.",
+    )
+    project_description: str = Field(
+        ...,
+        description="Short textual description of the project.",
+    )
+    project_status: str = Field(
+        ...,
+        description="Project status, expected as 'ongoing' or 'completed'.",
+    )
+    project_license: str = Field(
+        ...,
+        description="Overall license used for project outputs; use an allowed SPDX-style value or 'various'.",
+    )
+    project_bbox: list[list[float]] = Field(..., description="One or more [west, south, east, north] bounding boxes.")
+    project_start_datetime: datetime = Field(
+        ...,
+        description="Project start date and time.",
+    )
+    project_end_datetime: datetime = Field(
+        ...,
+        description="Project end date and time.",
+    )
+    project_themes: list[str] = Field(
+        ...,
+        description="One or more OSC themes, e.g. atmosphere, cryosphere, land, magnetosphere-ionosphere, oceans, solid-earth.",
+    )
+    to_name: str = Field(
+        ...,
+        description="Full name of the ESA Technical Officer supporting the project.",
+    )
+    to_email: str = Field(
+        ...,
+        description="Email address of the ESA Technical Officer.",
+    )
+    consortium_members: list[tuple[str, str]] = Field(
+        ...,
+        description="Project consortium members as (name, contact_email) tuples.",
+    )
+    website_link: str = Field(
+        ...,
+        description="Project website URL.",
+    )
+    eo4society_link: Optional[str] = Field(
+        None,
+        description="EO4Society project page URL when available.",
+    )
+
+
+class ProductCollectionMetadata(BaseModel):
+    """Defines validated inputs used to construct an OSC product collection."""
+
+    product_id: str = Field(
+        ...,
+        description="Custom product identifier distinct from the project ID, using dashes between words.",
+    )
+    product_title: Optional[str] = Field(
+        None,
+        description="Human-readable product title.",
+    )
+    product_description: str = Field(
+        ...,
+        description="General metadata description of the product or dataset.",
+    )
+    product_bbox: list[list[float]] = Field(..., description="One or more [west, south, east, north] bounding boxes.")
+    product_start_datetime: datetime = Field(
+        ...,
+        description="Product or dataset start date and time.",
+    )
+    product_end_datetime: datetime = Field(
+        ...,
+        description="Product or dataset end date and time.",
+    )
+    product_license: str = Field(
+        ...,
+        description="License for this product; should be a valid allowed OSC license value.",
+    )
+    product_keywords: Optional[list[str]] = Field(
+        None,
+        description="Up to five short keywords to improve product discovery.",
+    )
+    product_status: str = Field(
+        ...,
+        description="Product status, expected as 'ongoing' or 'completed'.",
+    )
+    product_region: Optional[str] = Field(
+        None,
+        description="Semantic region covered by the product, e.g. Belgium or Global.",
+    )
+    product_themes: list[str] = Field(
+        ...,
+        description="One or more OSC themes applicable to the product.",
+    )
+    product_missions: list[str] = Field(
+        ...,
+        description="One or more EO mission identifiers used to generate the product.",
+    )
+    product_variables: list[str] = Field(
+        ...,
+        description="One or more OSC variable identifiers describing the product.",
+    )
+    project_id: str = Field(
+        ...,
+        description="Identifier of the related project; must match an existing or newly created project.",
+    )
+    project_title: str = Field(
+        ...,
+        description="Title of the related project corresponding to project_id.",
+    )
+    product_parameters: Optional[list[str]] = Field(
+        None,
+        description="CF-convention style parameter names associated with the product, e.g. 'leaf_area_index'.",
+    )
+    product_doi: Optional[str] = Field(
+        None,
+        description="DOI URL assigned to the product, if available.",
+    )
+    access_link: Optional[str] = Field(
+        None,
+        description="Direct URL for accessing data; this should be a valid URL.",
+    )
+    documentation_link: Optional[str] = Field(
+        None,
+        description="URL to supporting documentation such as handbook or validation report.",
+    )
+    license_link: Optional[str] = Field(
+        None,
+        description="URL to license, if its not a standard SPDX one.",
+    ) 
+    item_link: Optional[str] = Field(
+        None,
+        description="URL to an external STAC item or collection representing the dataset, if available.",
+    )
+    item_title: Optional[str] = Field(
+        None,
+        description="Title to the item.",
+    )
+    
+
+
+class WorkflowMetadata(BaseModel):
+    """Defines validated inputs used to construct an OSC workflow record."""
+
+    workflow_id: str = Field(
+        ...,
+        description="Custom workflow identifier, distinct from project and product IDs.",
+    )
+    workflow_title: str = Field(
+        ...,
+        description="Workflow title.",
+    )
+    workflow_description: str = Field(
+        ...,
+        description="Short description of the workflow purpose and content.",
+    )
+    workflow_license: str = Field(
+        ...,
+        description="Workflow license value, typically SPDX-style or 'various' when needed.",
+    )
+    workflow_keywords: list[str] = Field(
+        ...,
+        description="Up to five short keywords to improve workflow discovery.",
+    )
+    workflow_formats: list[str] = Field(
+        ...,
+        description="Input/output data formats used by the workflow, e.g. GeoTIFF or NetCDF.",
+    )
+    workflow_themes: list[str] = Field(
+        ...,
+        description="One or more OSC themes associated with the workflow.",
+    )
+    codeurl: str = Field(
+        ...,
+        description="Active URL to the repository where the workflow/code can be discovered.",
+    )
+    project_id: str = Field(
+        ...,
+        description="Identifier of the associated project; must match an existing or newly created project.",
+    )
+    project_title: str = Field(
+        ...,
+        description="Title of the associated project corresponding to project_id.",
+    )
+    workflow_doi: Optional[str] = Field(
+        None,
+        description="DOI for the workflow record when available.",
+    )
+    workflow_bbox: Optional[list[list[float]]] = Field(
+        None,
+        description="Optional workflow spatial coverage as [west, south, east, north] bounding boxes.",
+    )
+    workflow_start_datetime: Optional[datetime] = Field(
+        None,
+        description="Optional workflow validity/execution start datetime.",
+    )
+    workflow_end_datetime: Optional[datetime] = Field(
+        None,
+        description="Optional workflow validity/execution end datetime.",
+    )
+
+
+class ExperimentMetadata(BaseModel):
+    """Defines validated inputs used to construct an OSC experiment record."""
+
+    experiment_id: str = Field(
+        ...,
+        description="Custom identifier of the experiment record.",
+    )
+    experiment_title: str = Field(
+        ...,
+        description="Experiment title.",
+    )
+    experiment_description: str = Field(
+        ...,
+        description="Description of the workflow execution that generated the product.",
+    )
+    experiment_license: str = Field(
+        ...,
+        description="License value for the experiment record.",
+    )
+    experiment_keywords: list[str] = Field(
+        ...,
+        description="Keywords supporting experiment discovery.",
+    )
+    experiment_formats: list[str] = Field(
+        ...,
+        description="Input/output formats used in the experiment, e.g. GeoTIFF, Zarr, netCDF.",
+    )
+    experiment_themes: list[str] = Field(
+        ...,
+        description="One or more OSC themes associated with the experiment.",
+    )
+    experiment_input_parameters_link: str = Field(
+        ...,
+        description="URL to the specification of experiment input parameters.",
+    )
+    experiment_enviroment_link: str = Field(
+        ...,
+        description="URL to the runtime environment description used to execute the experiment.",
+    )
+    workflow_id: str = Field(
+        ...,
+        description="Identifier of the associated workflow.",
+    )
+    workflow_title: str = Field(
+        ...,
+        description="Title of the associated workflow.",
+    )
+    product_id: str = Field(
+        ...,
+        description="Identifier of the associated product.",
+    )
+    product_title: str = Field(
+        ...,
+        description="Title of the associated product.",
+    )
+    contacts: Optional[list[dict]] = Field(
+        None,
+        description="Optional contact objects for the experiment record.",
+    )
+    experiment_bbox: Optional[list[list[float]]] = Field(
+        None,
+        description="Optional experiment spatial coverage as [west, south, east, north] bounding boxes.",
+    )
+    experiment_start_datetime: Optional[datetime] = Field(
+        None,
+        description="Optional experiment start datetime.",
+    )
+    experiment_end_datetime: Optional[datetime] = Field(
+        None,
+        description="Optional experiment end datetime.",
+    )
+
+
+class ItemMetadata(BaseModel):
+    """Defines validated inputs used to construct an OSC STAC Item."""
+
+    itemid: str = Field(
+        ...,
+        description="Unique STAC item identifier.",
+    )
+    geometry: dict = Field(
+        ...,
+        description="GeoJSON geometry describing the item footprint.",
+    )
+    data_time: datetime = Field(
+        ...,
+        description="Primary timestamp of the item.",
+    )
+    bbox: list[float] = Field(
+        ...,
+        description="Item bounding box as [west, south, east, north].",
+    )
+    product_id: str = Field(
+        ...,
+        description="Identifier of the parent product/collection for this item.",
+    )
+    license: str = Field(
+        ...,
+        description="License applicable to this item.",
+    )
+    description: str = Field(
+        ...,
+        description="Short item description.",
+    )
+    data_url: str = Field(
+        ...,
+        description="URL to the primary data asset.",
+    )
+    data_mime_type: str = Field(
+        ...,
+        description="Media type of the primary data asset.",
+    )
+    data_title: str = Field(
+        ...,
+        description="Human-readable title of the primary data asset.",
+    )
+    extra_fields: Optional[dict] = Field(
+        None,
+        description="Additional custom STAC fields to add to the item.",
+    )

earthcode/search.py

@@ -0,0 +1,209 @@
+"""Search interface for Lance vector store of Open Science Catalog items.
+
+Provides semantic search across products, variables, missions, and projects using FastEmbed embeddings.
+
+Returns:
+    list[pystac.Collection | pystac.Catalog]: Search results as PySTAC objects.
+"""
+
+import json
+from pathlib import Path
+import shutil
+import tempfile
+import lance
+import numpy as np
+import pystac
+from fastembed import TextEmbedding
+
+LANCE_URI = "s3://pangeo-test-fires/vector_store_v5/"
+MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
+LANCE_BASE_STORAGE_OPTIONS = {
+    "region": "eu-west-2",
+    "aws_skip_signature": "true",
+}
+OPEN_SCIENCE_CATALOG_LINK = "https://opensciencedata.esa.int/stac-browser/#"
+URL_TO_INJECT = {
+    "products": OPEN_SCIENCE_CATALOG_LINK + "/products/{id}/collection.json",
+    "variables": OPEN_SCIENCE_CATALOG_LINK + "/variables/{id}/catalog.json",
+    "projects": OPEN_SCIENCE_CATALOG_LINK + "/projects/{id}/collection.json",
+    "eo-missions": OPEN_SCIENCE_CATALOG_LINK + "/eo-missions/{id}/catalog.json",
+    "themes": OPEN_SCIENCE_CATALOG_LINK + "/themes/{id}/catalog.json",
+    "experiments": OPEN_SCIENCE_CATALOG_LINK + "/experiments/{id}/record.json",
+    "workflows": OPEN_SCIENCE_CATALOG_LINK + "/workflows/{id}/record.json",
+}
+
+_ds = None
+_model = None
+
+
+def search(
+    query=None,
+    *,
+    limit=10,
+    bbox=None,
+    intersects=True,
+    collection_ids=None,
+    theme=None,
+    variable=None,
+    mission=None,
+    keyword=None,
+    type="products",
+):
+    # check valid inputs for type
+    if type not in ("products", "variables", "eo-missions", "projects"):
+        raise ValueError(
+            f"Invalid type '{type}'. Must be one of 'products', 'variables', 'eo-missions', or 'projects'."
+        )
+
+    # check valid inputs for themes:
+    valid_themes = {
+        "land",
+        "oceans",
+        "atmosphere",
+        "cryosphere",
+        "magnetosphere-ionosphere",
+        "solid-earth",
+    }
+    if theme:
+        themes = (
+            theme if isinstance(theme, (list, tuple, set)) else [theme]
+        )  # handle if list or str
+        for t in themes:
+            if t not in valid_themes:
+                raise ValueError(
+                    f"Invalid theme '{t}'. Must be one of {sorted(valid_themes)}."
+                )
+
+    # dataset / model caches
+    global _ds, _model
+    if _ds is None or getattr(_ds, "uri", None) != LANCE_URI.rstrip("/") + "/":
+        _ds = lance.dataset(
+            LANCE_URI.rstrip("/") + "/", storage_options=LANCE_BASE_STORAGE_OPTIONS
+        )
+    if _model is None:
+        try:
+            _model = TextEmbedding(model_name=MODEL_NAME)
+        except Exception as exc:
+            if "NO_SUCHFILE" not in str(exc) and "NoSuchFile" not in str(exc):
+                raise
+            shutil.rmtree(Path(tempfile.gettempdir()) / "fastembed_cache", ignore_errors=True)
+            _model = TextEmbedding(model_name=MODEL_NAME)
+
+    # build filter string
+    parts = []
+    parts.append(f"`group` = '{type}'")
+    if collection_ids:
+        if isinstance(collection_ids, str):
+            collection_ids = [collection_ids]
+        parts.append("id IN (" + ",".join(f"'{c}'" for c in collection_ids) + ")")
+
+    if theme and type in ("products", "variables"):
+        themes = (
+            theme if isinstance(theme, (list, tuple, set)) else [theme]
+        )  # handle if list or str
+        theme_filters = [
+            f"LOWER(theme_ids) LIKE '%|{str(t).lower()}|%'" for t in themes if t
+        ]
+        if theme_filters:
+            parts.append("(" + " OR ".join(theme_filters) + ")")
+
+    if variable and type == "products":
+        variables = variable if isinstance(variable, (list, tuple, set)) else [variable]
+        variable_filters = [
+            f"LOWER(variable_ids) LIKE '%|{str(v).lower()}|%'" for v in variables if v
+        ]
+        if variable_filters:
+            parts.append("(" + " OR ".join(variable_filters) + ")")
+
+    if mission and type == "products":
+        missions = mission if isinstance(mission, (list, tuple, set)) else [mission]
+        mission_filters = [
+            f"LOWER(mission_ids) LIKE '%|{str(m).lower()}|%'" for m in missions if m
+        ]
+        if mission_filters:
+            parts.append("(" + " OR ".join(mission_filters) + ")")
+
+    if keyword:
+        keywords = keyword if isinstance(keyword, (list, tuple, set)) else [keyword]
+        kw_filters = [
+            "("
+            + " OR ".join(
+                [
+                    f"LOWER(title) LIKE '%{str(kw).lower()}%'",
+                    f"LOWER(description) LIKE '%{str(kw).lower()}%'",
+                    f"LOWER(keywords) LIKE '%|{str(kw).lower()}|%'",
+                ]
+            )
+            + ")"
+            for kw in keywords
+            if kw
+        ]
+        if kw_filters:
+            parts.append("(" + " OR ".join(kw_filters) + ")")
+
+    if bbox and len(bbox) >= 4:
+        minx, miny, maxx, maxy = bbox[:4]
+        if intersects:
+            parts.append(
+                f"bbox_minx <= {maxx} AND bbox_maxx >= {minx} AND bbox_miny <= {maxy} AND bbox_maxy >= {miny}"
+            )
+        else:
+            parts.append(
+                f"bbox_minx >= {minx} AND bbox_maxx <= {maxx} AND bbox_miny >= {miny} AND bbox_maxy <= {maxy}"
+            )
+
+    filt = " AND ".join(parts) if parts else None
+
+    cols = [
+        "id",
+        "group",
+        "title",
+        "description",
+        "keywords",
+        "bbox_minx",
+        "bbox_miny",
+        "bbox_maxx",
+        "bbox_maxy",
+        "item_json",
+    ]
+
+    if query and query.strip():
+        vec = next(_model.embed([query]))
+
+        tbl = _ds.scanner(
+            columns=cols,
+            filter=filt,
+            nearest={
+                "column": "embedding",
+                "q": np.asarray(vec, dtype=np.float32),
+                "k": limit,
+            },
+            prefilter=True,
+            limit=limit,
+        ).to_table()
+
+    else:
+        tbl = _ds.to_table(columns=cols, filter=filt, limit=limit)
+
+    results = []
+
+    for row in tbl.to_pylist():
+        item_j = json.loads(row["item_json"])
+        item = (
+            pystac.Collection.from_dict(item_j)
+            if item_j.get("type") == "Collection"
+            else pystac.Catalog.from_dict(item_j)
+        )
+        item.extra_fields["osc_url"] = URL_TO_INJECT.get(type, "").format(id=row["id"])
+
+        results.append(item)
+    return results
+
+
+# if __name__ == "__main__":
+# for grp in ["products", "variables", "eo-missions", "projects"]:
+#     print(grp, [c.title for c in search("forest fires", type=grp, limit=2)])
+# print(len(search("forest fires", theme="land", limit=2))) # one or more results expected - with theme = land
+# print(len(search("forest fires", theme="ocean", limit=2))) # no results expected
+# print(search(variable="burned-area")[0].title) # expect something that has a variable of fire
+# print(search(keyword="Seasonal Fire Modeling")[0].title)