earthcode 0.1.0__tar.gz

earthcode-0.1.0/.gitattributes

@@ -0,0 +1,2 @@
+# SCM syntax highlighting & preventing 3-way merges
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true

earthcode-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,49 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+    working-directory: earthcode-library
+
+jobs:
+  tests:
+    name: Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+
+    steps:
+      - name: Checkout earthcode-library
+        uses: actions/checkout@v6
+        with:
+          path: earthcode-library
+
+      - name: Checkout open-science-catalog-metadata
+        uses: actions/checkout@v6
+        with:
+          repository: ESA-EarthCODE/open-science-catalog-metadata
+          path: open-science-catalog-metadata
+
+      - name: Set up Pixi
+        uses: prefix-dev/setup-pixi@v0.9.4
+        with:
+          pixi-version: v0.59.0
+          manifest-path: earthcode-library/pixi.toml
+          environments: default
+          locked: true
+          cache: true
+          cache-write: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+
+      - name: Run test suite
+        run: pixi run -e default pytest

earthcode-0.1.0/.github/workflows/pages.yml

@@ -0,0 +1,77 @@
+name: Docs
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - README.md
+      - guide/**
+      - examples/**
+      - _config.yml
+      - _toc.yml
+      - pixi.toml
+      - pixi.lock
+      - pyproject.toml
+      - .github/workflows/pages.yml
+  pull_request:
+    paths:
+      - README.md
+      - guide/**
+      - examples/**
+      - _config.yml
+      - _toc.yml
+      - pixi.toml
+      - pixi.lock
+      - pyproject.toml
+      - .github/workflows/pages.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build Docs
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Install Jupyter Book
+        run: python -m pip install --upgrade "jupyter-book>=1.0.4,<2"
+
+      - name: Build site
+        run: jupyter-book build .
+
+      - name: Upload Pages artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: _build/html
+
+  deploy:
+    name: Deploy Docs
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

earthcode-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,128 @@
+# name: Publish
+
+# on:
+#   push:
+#     tags:
+#       - "v*"
+#   workflow_dispatch:
+
+# permissions:
+#   contents: read
+
+# concurrency:
+#   group: publish-${{ github.ref }}
+#   cancel-in-progress: false
+
+# jobs:
+#   build:
+#     name: Build Distribution
+#     runs-on: ubuntu-latest
+
+#     steps:
+#       - name: Checkout repository
+#         uses: actions/checkout@v6
+#         with:
+#           persist-credentials: false
+
+#       - name: Set up Python
+#         uses: actions/setup-python@v6
+#         with:
+#           python-version: "3.12"
+
+#       - name: Install build tools
+#         run: python -m pip install --upgrade build twine
+
+#       - name: Build distribution artifacts
+#         run: python -m build
+
+#       - name: Check distribution metadata
+#         run: python -m twine check dist/*
+
+#       - name: Upload distribution artifacts
+#         uses: actions/upload-artifact@v5
+#         with:
+#           name: python-package-distributions
+#           path: dist/
+
+#   publish-to-testpypi:
+#     name: Publish to TestPyPI
+#     if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main'
+#     needs: build
+#     runs-on: ubuntu-latest
+#     environment:
+#       name: testpypi
+#       url: https://test.pypi.org/p/earthcode
+#     permissions:
+#       id-token: write
+
+#     steps:
+#       - name: Download distribution artifacts
+#         uses: actions/download-artifact@v6
+#         with:
+#           name: python-package-distributions
+#           path: dist/
+
+#       - name: Publish to TestPyPI
+#         uses: pypa/gh-action-pypi-publish@release/v1
+#         with:
+#           repository-url: https://test.pypi.org/legacy/
+
+#   publish-to-pypi:
+#     name: Publish to PyPI
+#     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+#     needs: build
+#     runs-on: ubuntu-latest
+#     environment:
+#       name: pypi
+#       url: https://pypi.org/p/earthcode
+#     permissions:
+#       id-token: write
+
+#     steps:
+#       - name: Download distribution artifacts
+#         uses: actions/download-artifact@v6
+#         with:
+#           name: python-package-distributions
+#           path: dist/
+
+#       - name: Publish to PyPI
+#         uses: pypa/gh-action-pypi-publish@release/v1
+
+
+
+# Best Practices
+
+# Use Trusted Publishing, not long-lived PyPI API tokens.
+# Publish real releases only from version tags.
+# Build once and publish the exact same dist/ artifacts.
+# Use TestPyPI first.
+# Require manual approval on the GitHub pypi environment.
+# Keep the trusted workflow file path and environment names stable.
+# PyPI matches on exact repo, workflow filename, and optional environment name.
+# Don’t make this a reusable workflow.
+# PyPI currently does not support reusable workflows as trusted publishers.
+# Treat versions as immutable.
+# If you publish 0.1.0 to TestPyPI and need another try, bump to something like 0.1.0rc1, 0.1.0rc2, etc.
+# What You Need To Do
+
+# Commit and push publish.yml (line 1).
+# In GitHub, create two environments in the repo settings:
+# testpypi
+# pypi
+# On the pypi environment, require manual approval.
+# On PyPI, add a pending Trusted Publisher for:
+# package name: earthcode
+# owner: ESA-EarthCODE
+# repository: earthcode-library
+# workflow file: .github/workflows/publish.yml
+# environment: pypi
+# On TestPyPI, add the same pending publisher, but with environment testpypi.
+# If you don’t already have one, create a separate TestPyPI account.
+# Run the workflow manually from main once to publish to TestPyPI.
+# Verify install from TestPyPI with:
+# pip install --index-url https://test.pypi.org/simple --extra-index-url https://pypi.org/simple earthcode==0.1.0
+# For a real release:
+# bump version in pyproject.toml (line 3)
+# commit it
+# tag it
+# push the tag

earthcode-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+
+# pixi environments
+.pixi
+*.egg-info
+*__pycache__*
+*.json
+test*.ipynb
+open-science-catalog-metadata
+htmlcov/
+*.coverage
+_build/

earthcode-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 EarthCODE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

earthcode-0.1.0/PKG-INFO

@@ -0,0 +1,70 @@
+Metadata-Version: 2.4
+Name: earthcode
+Version: 0.1.0
+Summary: Tools for creating, validating, and searching EarthCODE Open Science Catalog metadata.
+Project-URL: Homepage, https://github.com/ESA-EarthCODE/earthcode-library
+Project-URL: Repository, https://github.com/ESA-EarthCODE/earthcode-library
+Project-URL: Issues, https://github.com/ESA-EarthCODE/earthcode-library/issues
+Project-URL: Documentation, https://esa-earthcode.github.io/earthcode-library/
+Author-email: Krasen Samardzhiev <krasensam@gmail.com>, Deyan Samardzhiev <dean@lampata.co.uk>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: earth observation,earthcode,geospatial
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.12
+Requires-Dist: bottleneck<2,>=1.6.0
+Requires-Dist: dask<2026,>=2025.12.0
+Requires-Dist: fastembed<0.8,>=0.7.4
+Requires-Dist: fsspec<2026,>=2025.10.0
+Requires-Dist: geopandas<2,>=1.1.2
+Requires-Dist: jsonschema<5,>=4.25.1
+Requires-Dist: netcdf4<2,>=1.7.3
+Requires-Dist: numpy<3,>=2.4.1
+Requires-Dist: pandas<4,>=3.0.0
+Requires-Dist: pillow<12,>=10.3
+Requires-Dist: pyarrow<24,>=23.0.0
+Requires-Dist: pydantic<3,>=2.12.5
+Requires-Dist: pylance<0.25,>=0.24.0
+Requires-Dist: pystac<2,>=1.14.1
+Requires-Dist: requests<3,>=2.32.5
+Requires-Dist: rioxarray<0.21,>=0.20.0
+Requires-Dist: shapely<3,>=2.1.2
+Requires-Dist: xarray<2026,>=2025.12.0
+Requires-Dist: xstac<2,>=1.2.0
+Requires-Dist: zarr<4,>=3.1.5
+Provides-Extra: dev
+Requires-Dist: jupyter-book<2,>=1.0.4; extra == 'dev'
+Requires-Dist: jupyterlab<5,>=4.5.0; extra == 'dev'
+Requires-Dist: papermill<3,>=2.7.0; extra == 'dev'
+Requires-Dist: pytest-cov<8,>=7.0.0; extra == 'dev'
+Requires-Dist: pytest<10,>=9.0.2; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# earthcode
+
+Python tools for creating, validating, and searching EarthCODE Open Science Catalog metadata.
+
+```bash
+pip install earthcode
+```
+
+## Development
+
+To run:
+
+1. `git clone https://github.com/ESA-EarthCODE/earthcode-library.git`
+2. Install pixi - https://pixi.sh/dev/installation/
+3. `cd earthcode-library`
+4. `pixi install`
+5. `pixi run jupyter lab`
+
+We have examples for:
+- `./examples/example_create_osc_entries.ipynb` - shows how to create OSC entries
+- `./examples/contribute_via_pr_osc.ipynb` - shows how to add newly created entries to the OSC, using a GitHub pull request
+- `./examples/contribute_via_osc_editor.ipynb` - shows how to add entries to the OSC, using a combination of this library and the OSC Editor (a GUI tool)
+- `./examples/earthcode_publishing_guide.ipynb` - is a simplified introduction to the OSC and the necessary steps to publish data

earthcode-0.1.0/README.md

@@ -0,0 +1,23 @@
+# earthcode
+
+Python tools for creating, validating, and searching EarthCODE Open Science Catalog metadata.
+
+```bash
+pip install earthcode
+```
+
+## Development
+
+To run:
+
+1. `git clone https://github.com/ESA-EarthCODE/earthcode-library.git`
+2. Install pixi - https://pixi.sh/dev/installation/
+3. `cd earthcode-library`
+4. `pixi install`
+5. `pixi run jupyter lab`
+
+We have examples for:
+- `./examples/example_create_osc_entries.ipynb` - shows how to create OSC entries
+- `./examples/contribute_via_pr_osc.ipynb` - shows how to add newly created entries to the OSC, using a GitHub pull request
+- `./examples/contribute_via_osc_editor.ipynb` - shows how to add entries to the OSC, using a combination of this library and the OSC Editor (a GUI tool)
+- `./examples/earthcode_publishing_guide.ipynb` - is a simplified introduction to the OSC and the necessary steps to publish data

earthcode-0.1.0/_config.yml

@@ -0,0 +1,20 @@
+title: earthcode
+author: EarthCODE
+copyright: "2026"
+only_build_toc_files: true
+exclude_patterns:
+  - _build
+  - .github/*
+  - .pixi/*
+  - .pytest_cache/*
+  - "**.ipynb_checkpoints"
+execute:
+  execute_notebooks: "off"
+repository:
+  url: https://github.com/ESA-EarthCODE/earthcode-library
+  path_to_book: .
+  branch: main
+html:
+  use_repository_button: true
+  use_issues_button: true
+  home_page_in_navbar: true

earthcode-0.1.0/_toc.yml

@@ -0,0 +1,20 @@
+format: jb-book
+root: README
+parts:
+  - caption: Guide
+    chapters:
+      - file: guide/0.Prerequisites
+      - file: guide/1.Project
+      - file: guide/2.0.Product
+      - file: guide/2.1.Product_files_PRR
+      - file: guide/2.1.Product_files_self_hosted
+      - file: guide/3.Workflow
+      - file: guide/4.Experiment
+  - caption: Examples
+    chapters:
+      - file: examples/earthcode_publishing_guide
+      - file: examples/earthcode_data_discovery
+      - file: examples/example_create_osc_entries
+      - file: examples/contribute_via_pr_osc
+      - file: examples/contribute_via_osc_editor
+      - file: examples/glambie_notebook_osc

earthcode-0.1.0/cli/generate_embeddings.py

@@ -0,0 +1,159 @@
+"""
+Load documents from the specified STAC open science catalog and upload to s3 storage as lance table.
+
+Currently only handles OSC collections/catalogs for 'products', 'variables', 'eo-missions', and 'projects'.
+It does NOT handle the stac items within collections. In future this can be handled with multiple indexes and tables.
+
+- Build (defaults baked in): `pixi run python generate_embeddings.py`
+- Explicit build: `pixi run python cli/generate_embeddings.py`
+
+
+Returns:
+    None
+"""
+
+import argparse
+import json
+from pathlib import Path
+import lance
+import numpy as np
+import pyarrow as pa
+from fastembed import TextEmbedding
+
+DEFAULT_ROOT_DIR = "../open-science-catalog-metadata"
+DEFAULT_GROUPS = ["products", "variables", "eo-missions", "projects"]
+DEFAULT_LANCE_URI = "s3://pangeo-test-fires/vector_store_v5/"
+LANCE_BASE_STORAGE_OPTIONS = {
+    "region": "eu-west-2",
+    "aws_skip_signature": "true",
+}  # to be implemented
+MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
+
+
+# ---------------------------- helpers ---------------------------- #
+
+
+def extract_theme_ids(data):
+    theme_ids = []
+    for theme in data.get("themes", []):
+        for concept in theme.get("concepts", []):
+            cid = concept.get("id")
+            if cid:
+                theme_ids.append(str(cid))
+    return theme_ids
+
+
+def flatten_metadata(data):
+    parts = []
+    parts.append(data.get("id", ""))
+    parts.append(data.get("title", ""))
+    parts.append(data.get("description", ""))
+    parts.extend(data.get("keywords", []))
+    parts.extend(data.get("osc:variables", []))
+    parts.extend(data.get("osc:missions", []))
+    parts.extend(extract_theme_ids(data))
+    return "\n".join(p for p in parts if p)
+
+
+def create_row_from_stac_file(path, group):
+    data = json.loads(path.read_text())
+    theme_ids = extract_theme_ids(data)
+    variable_ids = [str(v) for v in data.get("osc:variables", []) if v]
+    mission_ids = [str(m) for m in data.get("osc:missions", []) if m]
+    keywords = [str(k) for k in data.get("keywords", []) if k]
+    bboxes = data.get("extent", {}).get("spatial", {}).get("bbox") or [
+        [-180, -90, 180, 90]
+    ]
+    bminx = min([b[0] for b in bboxes if len(b) >= 4], default=None)
+    bminy = min([b[1] for b in bboxes if len(b) >= 4], default=None)
+    bmaxx = max([b[2] for b in bboxes if len(b) >= 4], default=None)
+    bmaxy = max([b[3] for b in bboxes if len(b) >= 4], default=None)
+    return {
+        "id": data.get("id", path.parent.name),
+        "group": group,
+        "title": data.get("title", ""),
+        "description": data.get("description", ""),
+        "path": str(path),
+        "bbox_minx": bminx,
+        "bbox_miny": bminy,
+        "bbox_maxx": bmaxx,
+        "bbox_maxy": bmaxy,
+        "item_json": json.dumps(data),
+        "theme_ids": f"|{'|'.join(theme_ids)}|" if theme_ids else "",
+        "variable_ids": f"|{'|'.join(variable_ids)}|" if variable_ids else "",
+        "mission_ids": f"|{'|'.join(mission_ids)}|" if mission_ids else "",
+        "keywords": f"|{'|'.join(keywords)}|" if keywords else "",
+        "text": flatten_metadata(data),
+    }
+
+
+def load_documents(stac_dir, group):
+    if not stac_dir.exists():
+        raise FileNotFoundError(f"Group dir not found: {stac_dir}")
+
+    targets = list(stac_dir.glob("**/collection.json")) + list(
+        stac_dir.glob("**/catalog.json")
+    )
+    rows = [create_row_from_stac_file(p, group) for p in sorted(targets)]
+
+    if not rows:
+        raise RuntimeError(f"No STAC collections/catalogs found under {stac_dir}")
+    return rows
+
+
+def build_embeddings(texts, model_name):
+    model = TextEmbedding(model_name=model_name)
+    return np.asarray(list(model.embed(texts)), dtype=np.float32)
+
+
+# ----------------------------- main ------------------------------ #
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Build Lance dataset.")
+    parser.add_argument(
+        "--root-dir",
+        default=DEFAULT_ROOT_DIR,
+        help="Base OSC metadata dir containing group folders.",
+    )
+    parser.add_argument(
+        "--groups",
+        nargs="+",
+        default=DEFAULT_GROUPS,
+        help="Group folder names under root-dir to ingest (e.g., products variables eo-missions projects).",
+    )
+    parser.add_argument(
+        "--lance-uri",
+        default=DEFAULT_LANCE_URI,
+        help="Where to write the Lance dataset.",
+    )
+    parser.add_argument(
+        "--model", default=MODEL_NAME, help="FastEmbed model name."
+    )
+    args = parser.parse_args()
+
+    # get documents in pyarrow table
+    root = Path(args.root_dir)
+    rows = [row for grp in args.groups for row in load_documents(root / grp, grp)]
+
+    # build embeddings
+    texts = [r["text"] for r in rows]
+    embeddings = build_embeddings(texts, args.model)
+    embed_array = pa.FixedSizeListArray.from_arrays(
+        pa.array(embeddings.astype(np.float32).ravel(), type=pa.float32()),
+        embeddings.shape[1],
+    )
+
+    # build and write lance dataset
+    table = pa.Table.from_pylist(rows)
+    table = table.append_column("embedding", embed_array)
+    table = table.drop(["text"])
+    lance.write_dataset(table, args.lance_uri, mode="overwrite")
+
+    print(
+        f"Wrote {table.num_rows} rows to {args.lance_uri} with dim={embeddings.shape[1]}"
+    )
+
+
+if __name__ == "__main__":
+    main()