everycure-datasets 0.2.33__tar.gz

everycure_datasets-0.2.33/.gitignore

@@ -0,0 +1,28 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyc
+
+# Virtual environments
+.venv/
+venv/
+
+# uv
+.uv/
+
+# Build artifacts
+dist/
+build/
+*.egg-info/
+
+# IDE (keep .vscode/settings.json for schema validation)
+.vscode/*
+!.vscode/settings.json
+.idea/
+
+# Environment variables
+.env
+.env.local
+
+# OS
+.DS_Store

everycure_datasets-0.2.33/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: everycure-datasets
+Version: 0.2.33
+Requires-Python: >=3.11
+Requires-Dist: gitpython>=3.1.45
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pyyaml>=6.0.0
+Provides-Extra: kedro
+Requires-Dist: kedro-datasets[pandas,spark]>=6.0.0; extra == 'kedro'
+Requires-Dist: kedro>=0.19.1; extra == 'kedro'
+Requires-Dist: semantic-version>=2.10.0; extra == 'kedro'

everycure_datasets-0.2.33/pyproject.toml

@@ -0,0 +1,52 @@
+[project]
+name = "everycure-datasets"
+version = "0.2.33"
+requires-python = ">=3.11"
+scripts = {datasets = "everycure.datasets.cli:app"}
+dependencies = [
+    "gitpython>=3.1.45",
+    "pydantic>=2.0.0",
+    "pyyaml>=6.0.0",
+]
+[project.optional-dependencies]
+kedro = [
+    "kedro>=0.19.1",
+    "kedro-datasets[pandas,spark]>=6.0.0",
+    "semantic-version>=2.10.0",
+]
+
+[tool.hatch.build]
+include = ["src/**"]
+
+[tool.hatch.build.targets.sdist]
+exclude = [
+  "datasets/**",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/everycure"]
+
+# [tool.uv]
+# keyring-provider = "subprocess"
+#
+# [[tool.uv.index]]
+# name = "pypi"
+# url = "https://pypi.org/simple"
+#
+# [[tool.uv.index]]
+# name = "everycure"
+# url = "https://oauth2accesstoken@us-central1-python.pkg.dev/core-422020/everycure/simple/"
+# publish-url = "https://us-central1-python.pkg.dev/core-422020/everycure/"
+# default = false
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[dependency-groups]
+dev = [
+    "keyrings-google-artifactregistry-auth>=1.1.2",
+    "pytest>=9.0.1",
+    "pre-commit>=4.3.0",
+    "typer>=0.12.0",
+]

everycure_datasets-0.2.33/src/everycure/__init__.py

@@ -0,0 +1,3 @@
+"""Every Cure namespace package."""
+
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

everycure_datasets-0.2.33/src/everycure/datasets/__init__.py

@@ -0,0 +1,6 @@
+"""Dataset Registry - Centralized dataset metadata management."""
+
+from everycure.datasets.models.v1 import DatasetMetadataV1
+from everycure.datasets.models.v2 import DatasetMetadataV2
+
+__all__ = ["DatasetMetadataV1", "DatasetMetadataV2"]

everycure_datasets-0.2.33/src/everycure/datasets/cli.py

@@ -0,0 +1,64 @@
+"""CLI for datasets registry operations."""
+
+from pathlib import Path
+
+import typer
+
+from everycure.datasets.generate_schema import generate_all_schemas
+from everycure.datasets.validate import validate_datasets
+
+app = typer.Typer(
+    name="datasets",
+    help="Datasets registry management CLI",
+    add_completion=False,
+    no_args_is_help=True,
+)
+
+schema_app = typer.Typer(
+    name="schema",
+    help="Schema management commands",
+    add_completion=False,
+)
+
+app.add_typer(schema_app)
+
+
+@app.callback(invoke_without_command=True)
+def main(ctx: typer.Context) -> None:
+    """Datasets registry management CLI."""
+    if ctx.invoked_subcommand is None:
+        typer.echo(ctx.get_help())
+
+
+@app.command()
+def validate(
+    datasets_dir: Path | None = typer.Option(
+        None,
+        "--datasets-dir",
+        "-d",
+        help="Path to the datasets directory (default: auto-detect)",
+    ),
+) -> None:
+    """
+    Validate dataset YAML files and directory structure.
+
+    Checks:
+    - Dataset names are snake_case
+    - Version directories follow semantic versioning (MAJOR.MINOR.PATCH)
+    """
+    return validate_datasets(datasets_dir)
+
+
+@schema_app.command()
+def generate() -> None:
+    """
+    Generate JSON schema(s) from the Pydantic models.
+    """
+    repo_root = Path.cwd()
+    schema_dir = repo_root / ".schema"
+    generate_all_schemas(schema_dir)
+    typer.echo(f"✓ Generated all JSON schemas in {schema_dir}")
+
+
+if __name__ == "__main__":
+    app()

everycure_datasets-0.2.33/src/everycure/datasets/generate_schema.py

@@ -0,0 +1,62 @@
+"""Generate JSON schema from Pydantic models."""
+
+import json
+from pathlib import Path
+
+from everycure.datasets.models.v1 import DatasetMetadataV1
+from everycure.datasets.models.v2 import DatasetMetadataV2
+
+
+def generate_schema(model_class, output_path: Path, schema_id: str) -> None:
+    """
+    Generate JSON schema from a Pydantic model.
+
+    Args:
+        model_class: The Pydantic model class to generate schema from
+        output_path: Path where the JSON schema should be written
+        schema_id: The $id for the schema (FQDN)
+    """
+    # Get the JSON schema from the Pydantic model
+    schema = model_class.model_json_schema(
+        mode="serialization",
+        by_alias=True,
+    )
+
+    # Update the $id to use the provided FQDN
+    schema["$id"] = schema_id
+
+    # Remove fields with defaults from required list
+    # but keep them in properties so they're documented
+    fields_with_defaults = ["schema_version", "status", "created_at"]
+    if "required" in schema:
+        for field in fields_with_defaults:
+            if field in schema["required"]:
+                schema["required"].remove(field)
+
+    # Write to file with proper formatting
+    with open(output_path, "w") as f:
+        json.dump(schema, f, indent=2)
+        f.write("\n")  # Add trailing newline
+
+    print(f"Generated JSON schema at {output_path}")
+
+
+def generate_all_schemas(schema_dir: Path) -> None:
+    """Generate all schema versions."""
+    schema_dir.mkdir(parents=True, exist_ok=True)
+
+    # Generate v1 schema
+    v1_path = schema_dir / "dataset.v1.schema.json"
+    generate_schema(
+        DatasetMetadataV1,
+        v1_path,
+        "https://everycure.org/schemas/dataset.v1.schema.json",
+    )
+
+    # Generate v2 schema
+    v2_path = schema_dir / "dataset.v2.schema.json"
+    generate_schema(
+        DatasetMetadataV2,
+        v2_path,
+        "https://everycure.org/schemas/dataset.v2.schema.json",
+    )

everycure_datasets-0.2.33/src/everycure/datasets/kedro/__init__.py

@@ -0,0 +1,5 @@
+from .catalog_dataset import DataCatalogDataset
+
+__all__ = [
+    "DataCatalogDataset",
+]

everycure_datasets-0.2.33/src/everycure/datasets/kedro/catalog_dataset.py

@@ -0,0 +1,385 @@
+import logging
+from pathlib import Path
+from typing import Any
+
+import typer
+import yaml
+from kedro.io.core import AbstractDataset, DatasetError
+from kedro_datasets import pandas, spark
+from semantic_version import NpmSpec, Version
+
+from everycure.datasets.kedro.storage import GitStorageService, is_uri
+from everycure.datasets.models.v1 import (
+    ColumnSchema,
+    DatasetMetadataV1,
+    DatasetSchema,
+)
+
+logger = logging.getLogger(__name__)
+
+app = typer.Typer()
+
+
+class DataCatalogDataset(AbstractDataset):
+    """Custom dataset to load and read resources
+
+    Examples:
+
+        ```yaml
+            catalog_diseases:
+                type: everycure.datasets.kedro.catalog_dataset.DataCatalogDataset
+                dataset: disease_list
+                engine: spark
+
+                load_args:
+                    version: ~0.2.0
+
+                # Arguments for the underlying Kedro engine (e.g., spark.SparkDataset)
+                save_args:
+                    mode: overwrite
+
+                    # Arguments for the DatasetMetadataV1 model
+                    catalog_args:
+                        description: "Dataset description"
+                        message: "Optional message"
+                        owner:
+                            name: "John Doe"
+                            email: "john@example.com"
+                        location:
+                            uri: "gs://path/to/{version}/data.parquet"
+                            format: "parquet"
+                        # ...etc
+
+        ```
+    """
+
+    def __init__(
+        self,
+        *,
+        dataset: str | dict[str, Any],
+        engine: str,
+        save_args: dict[str, Any] = None,
+        load_args: dict[str, Any] = None,
+        **kwargs,
+    ):
+        self._dataset = dataset
+        self._engine = engine
+        self._storage_service = GitStorageService.get_instance()
+        self._save_args = save_args or {}
+        self._catalog_args = self._save_args.pop("catalog_args", None)
+        self._load_args = load_args or {}
+
+    @property
+    def filepath(self) -> str:
+        semver_pattern = self._load_args.get("version")
+        version, _ = self.best_match(self.versions, semver_pattern)
+        content = self._storage_service.get(
+            Path(f"datasets/{self._dataset}/{version}/dataset.yaml")
+        )
+        if content is None:
+            raise DatasetError(
+                f"Dataset metadata file not found for '{self._dataset}' version '{version}'"
+            )
+        dataset = DatasetMetadataV1.model_validate(yaml.safe_load(content))
+
+        return str(dataset.location.uri)
+
+    def load(self) -> Any:
+        """Dataset loading
+
+        Dataset loads the best matching version of the requested
+        dataset using the pattern.
+        """
+        # Make a copy to avoid modifying the original dict
+        engine_load_args = self._load_args.copy()
+        semver_pattern = engine_load_args.pop("version", None)
+        assert_latest = engine_load_args.pop("assert_latest", False)
+
+        version, is_latest = self.best_match(self.versions, semver_pattern)
+
+        if version is None:
+            raise DatasetError(
+                f"No version matched for dataset '{self._dataset}', available versions: {','.join(self.versions)}"
+            )
+
+        if assert_latest and not is_latest:
+            raise DatasetError(
+                f"Newer version for dataset '{self._dataset}' available!"
+            )
+
+        logger.info(f"Using version {version} for dataset '{self._dataset}'")
+        try:
+            content = self._storage_service.get(
+                Path(f"datasets/{self._dataset}/{version}/dataset.yaml")
+            )
+            if content is None:
+                raise DatasetError(
+                    f"Dataset metadata file not found for '{self._dataset}' version '{version}'"
+                )
+            dataset = DatasetMetadataV1.model_validate(yaml.safe_load(content))
+
+            return self.get_dataset(
+                dataset.location.format.value,
+                str(dataset.location.uri),
+                engine_load_args,
+                {},  # save_args are not used in load
+            ).load()
+        except Exception as e:
+            raise DatasetError(
+                f"Failed to load version for dataset '{self._dataset}': {e}"
+            ) from e
+
+    @staticmethod
+    def _uri_to_path(uri: str) -> str:
+        """Convert file:// URLs to file paths for local file access.
+
+        kedro_datasets expects file paths, not file:// URLs.
+        Other URI schemes (http, https, gs, s3, etc.) are passed through unchanged.
+        """
+        if uri.startswith("file://"):
+            from urllib.parse import unquote, urlparse
+
+            parsed = urlparse(uri)
+            return unquote(parsed.path)
+        return uri
+
+    def get_dataset(
+        self,
+        format_: str,
+        file_path: str,
+        load_args: dict[str, Any],
+        save_args: dict[str, Any],
+    ) -> AbstractDataset:
+        # Convert file:// URLs to paths for local file access
+        file_path = self._uri_to_path(file_path)
+
+        if self._engine == "spark":
+            if format_ == "tsv":
+                return spark.SparkDataset(
+                    filepath=file_path,
+                    file_format="csv",
+                    load_args={
+                        **load_args,
+                        "sep": "\t",
+                        "header": True,
+                        "index": False,
+                    },
+                    save_args=save_args,
+                )
+
+            return spark.SparkDataset(
+                filepath=file_path,
+                file_format=format_,
+                load_args={**load_args, "header": True, "index": False},
+                save_args=save_args,
+            )
+
+        if self._engine == "pandas":
+            if format_ == "csv":
+                return pandas.CSVDataset(
+                    filepath=file_path,
+                    load_args=load_args,
+                    save_args=save_args,
+                )
+
+            if format_ == "parquet":
+                return pandas.ParquetDataset(
+                    filepath=file_path, load_args=load_args, save_args=save_args
+                )
+
+        raise ValueError(f"Unsupported engine: {self._engine} and format {format}")
+
+    def get_schema(self, data) -> DatasetSchema:
+        """Get dataset schema as DatasetSchema model."""
+        columns = None
+        row_count = None
+
+        if self._engine == "pandas":
+            type_map = {
+                "int64": "int",
+                "Int64": "int",
+                "float64": "float",
+                "object": "string",
+                "bool": "bool",
+                "datetime64[ns]": "datetime",
+            }
+            columns = [
+                ColumnSchema(name=col, type=type_map.get(str(dtype), "unknown"))
+                for col, dtype in data.dtypes.items()
+            ]
+            row_count = len(data)
+
+        elif self._engine == "spark":
+            spark_map = {
+                "IntegerType()": "int",
+                "LongType()": "int",
+                "DoubleType()": "float",
+                "FloatType()": "float",
+                "StringType()": "string",
+                "BooleanType()": "bool",
+                "TimestampType()": "datetime",
+                "DateType()": "date",
+            }
+            columns = [
+                ColumnSchema(
+                    name=field.name,
+                    type=spark_map.get(str(field.dataType), "unknown"),
+                )
+                for field in data.schema.fields
+            ]
+            row_count = data.count()
+
+        else:
+            raise ValueError(f"Unsupported engine: {self._engine}")
+
+        return DatasetSchema(columns=columns, row_count=row_count)
+
+    def save(self, data: Any) -> None:
+        """Dataset saving
+
+        Dataset is saved using the next relevant semversion based
+        on the catalog arguments.
+        """
+        if not self._catalog_args:
+            raise DatasetError("Required 'catalog_args' missing in save_args.")
+
+        # 1. Calculate dynamic properties from _catalog_args
+        save_version = self._catalog_args.get("version")
+        message = self._catalog_args.get("message")
+        if not save_version:
+            save_version = self.prompt_version_bump()
+            if not save_version:  # User cancelled prompt
+                logger.warning("Save cancelled by user.")
+                return
+
+        if not message:
+            message = (
+                typer.prompt("Optional message", default="", show_default=False) or None
+            )
+
+        # 2. Prepare the dictionary of dynamic/runtime arguments
+        dynamic_args = {
+            "name": self._dataset,
+            "version": save_version,
+            # is aliased in original, hence writing to schema, not dataset_schema
+            "schema": self.get_schema(data).model_dump(exclude_none=True),
+        }
+        if message:
+            dynamic_args["message"] = message
+
+        metadata_dict = {**self._catalog_args, **dynamic_args}
+
+        # 4. Post-merge processing: Handle the version placeholder in the URI
+        location = metadata_dict.get("location", {})
+        if "uri" in location:
+            # Format the template string first, then convert to URI if needed
+            formatted_filesystem_path_str = location["uri"].format(
+                version=metadata_dict["version"]
+            )
+            if not is_uri(formatted_filesystem_path_str):
+                path_obj = Path(formatted_filesystem_path_str)
+                if not path_obj.is_absolute():
+                    path_obj = path_obj.resolve()
+                location["uri"] = path_obj.as_uri()
+            else:
+                location["uri"] = formatted_filesystem_path_str
+        else:
+            raise DatasetError("Required 'location.uri' missing in catalog_args.")
+
+        # 5. Validate the final dictionary and create the Pydantic object
+        try:
+            dataset_metadata = DatasetMetadataV1.model_validate(metadata_dict)
+        except Exception as e:  # Catches Pydantic's ValidationError
+            raise DatasetError(f"Invalid dataset metadata configuration: {e}") from e
+
+        # 6. Save the dataset file using the correct engine and self._save_args
+        self.get_dataset(
+            dataset_metadata.location.format.value,
+            str(dataset_metadata.location.uri),
+            {},  # load_args not used in save path
+            self._save_args,  # Pass engine-specific save_args directly
+        ).save(data)
+
+        # 7. Save the metadata YAML file
+        self._storage_service.save(
+            f"datasets/{dataset_metadata.name}/{dataset_metadata.version}/dataset.yaml",
+            yaml.dump(dataset_metadata.model_dump(mode="json", by_alias=True)),
+            commit_msg=f"🤖 Create version {dataset_metadata.version} for '{dataset_metadata.name}'",
+        )
+
+    @staticmethod
+    def best_match(versions: list[str], pattern: str) -> tuple[str | None, bool]:
+        """Function to find the best semver match.
+
+        Args:
+            versions: List of available versions
+            pattern: semver pattern to match
+        Returns:
+            Best match, and boolean indicating whether this is the last version.
+        """
+        if not pattern:
+            spec = NpmSpec("*")
+        else:
+            spec = NpmSpec(pattern)
+        parsed_versions = [Version(v) for v in versions]
+
+        # Find versions that satisfy the pattern
+        matching = [v for v in parsed_versions if v in spec]
+        if not matching:
+            return None, False
+
+        best_version = max(matching)
+        latest_version = max(parsed_versions)
+        is_latest = best_version == latest_version
+
+        return str(best_version), is_latest
+
+    def prompt_version_bump(self) -> tuple[str, str | None]:
+        """Prompt user for bumping information."""
+        parsed = [Version(v) for v in self.versions]
+        current_version = max([*parsed, Version("0.0.0")])
+        typer.echo(f"Saving dataset: '{self._dataset}'")
+        typer.echo(f"Current version: '{current_version}'")
+
+        allowed = ["major", "minor", "patch"]
+        bump_type = typer.prompt("Which part to bump? (major/minor/patch)").lower()
+        while bump_type not in allowed:
+            bump_type = typer.prompt(
+                "Invalid choice. Please choose major, minor, or patch"
+            ).lower()
+
+        new_version = {
+            "major": Version(major=current_version.major + 1, minor=0, patch=0),
+            "minor": Version(
+                major=current_version.major, minor=current_version.minor + 1, patch=0
+            ),
+            "patch": Version(
+                major=current_version.major,
+                minor=current_version.minor,
+                patch=current_version.patch + 1,
+            ),
+        }[bump_type]
+
+        if not typer.confirm(
+            f"Do you want to save dataset '{self._dataset}' with version '{new_version}'?"
+        ):
+            typer.echo("Save cancelled.")
+            return None, None
+
+        return str(new_version)
+
+    @property
+    def versions(self) -> list[str]:
+        """Function to get versions for dataset."""
+        paths = self._storage_service.ls(f"datasets/{self._dataset}/*")
+        return [
+            str(path.relative_to(Path(f"datasets/{self._dataset}"))) for path in paths
+        ]
+
+    def _describe(self) -> dict[str, Any]:
+        """Describe the dataset by returning its metadata."""
+        return {
+            "dataset": self._dataset,
+            "engine": self._engine,
+            "versions": self.versions,
+        }

everycure_datasets-0.2.33/src/everycure/datasets/kedro/hooks.py

@@ -0,0 +1,69 @@
+import logging
+import shutil
+from pathlib import Path
+
+from git import InvalidGitRepositoryError, Repo
+from kedro.framework.hooks import hook_impl
+
+from everycure.datasets.kedro.storage import GitStorageService
+
+logger = logging.getLogger(__name__)
+
+
+class GitStorageHook:
+    """Kedro hook to clone or update a Git repository before running a pipeline."""
+
+    def __init__(
+        self,
+        repo_url: str,
+        target_dir: str = "data/external/data-catalog",
+        branch: str = "main",
+        force: bool = False,
+        pull: bool = True,
+    ):
+        self.repo_url = repo_url
+        self.target_dir = Path(target_dir)
+        self.branch = branch
+        self.force = force
+        self.pull = pull
+
+    @hook_impl
+    def after_context_created(self, context):
+        """Clone or update repo before the pipeline runs."""
+
+        if self.force and self.target_dir.exists():
+            shutil.rmtree(self.target_dir)
+
+        if not self.target_dir.exists():
+            repo = Repo.clone_from(self.repo_url, self.target_dir, branch=self.branch)
+
+        else:
+            try:
+                repo = Repo(self.target_dir)
+                logger.info(f"📁 Existing repo found: {repo.working_dir}")
+
+                if self.pull:
+                    logger.info(f"⬇️ Pulling latest changes from {self.branch}")
+                    origin = repo.remotes.origin
+                    origin.fetch()
+                    origin.pull(self.branch)
+                    logger.info("✅ Repository updated.")
+                else:
+                    logger.info("🔸 Skipping pull, using existing contents.")
+
+                # Ensure branch consistency
+                repo.git.checkout(self.branch)
+            except InvalidGitRepositoryError:
+                logger.info(
+                    f"⚠️ {self.target_dir} exists but is not a Git repo. Re-cloning."
+                )
+                shutil.rmtree(self.target_dir)
+                Repo.clone_from(self.repo_url, self.target_dir, branch=self.branch)
+
+        self.git_service = GitStorageService(
+            root_path=str(self.target_dir),
+            user="Kedro",
+            email="kedro@everycure.org",
+        )
+
+        logger.info(f"🔁 GitStorageService ready at {self.target_dir}")

everycure_datasets-0.2.33/src/everycure/datasets/kedro/storage.py

@@ -0,0 +1,233 @@
+import abc
+import glob
+import logging
+import os
+from pathlib import Path
+from urllib.parse import urlparse
+
+from git import InvalidGitRepositoryError, NoSuchPathError, Repo
+
+logger = logging.getLogger(__name__)
+
+
+def is_uri(path: str) -> bool:
+    """Check if a string is a valid URI (has a scheme)."""
+    parsed = urlparse(path)
+    return bool(parsed.scheme)
+
+
+class StorageService(abc.ABC):
+    """
+    Abstract base class defining a storage service.
+    """
+
+    def __init__(self, root_path: str):
+        self._root_path = root_path
+
+    @abc.abstractmethod
+    def exists(self, file_path: Path) -> bool:
+        """
+        Function to verify whether given path exists.
+
+        Args:
+            file_path: Path to the file
+        Returns:
+            boolean representing existence
+        """
+
+    @abc.abstractmethod
+    def ls(self, glob_path: str) -> list[str]:
+        """
+        Function to list files in the given path.
+
+        Args:
+            glob_path: Path to the directory
+        Returns:
+            list of paths to files in the directory
+        """
+
+    @abc.abstractmethod
+    def get(self, file_path: Path) -> str | None:
+        """
+        Function to retrieve the contents of the given path.
+
+        Args:
+            file_path: Path to the file
+        Returns:
+            string representing file contents
+        """
+
+    @abc.abstractmethod
+    def save(
+        self,
+        file_path: Path | list[Path],
+        contents: str,
+        overwrite: bool = False,
+        **kwargs,
+    ) -> Path:
+        """
+        Function to save data in the given location.
+
+        Args:
+            file_path:  path or paths to files
+            contents: file contents
+            overwrite: boolean indicating file can be overwritten
+        Returns:
+            path to the materialized file
+        """
+
+
+class LocalStorageService(StorageService):
+    """
+    Specific StorageService that materializes files locally.
+    """
+
+    def exists(self, file_path: Path) -> bool:
+        """
+        Function to verify whether given path exists.
+
+        Args:
+            file_path: Path to the file
+        Returns:
+            boolean representing existence
+        """
+        full_path = Path(self._root_path) / file_path
+        return full_path.exists()
+
+    def ls(self, glob_path: str) -> list[Path]:
+        """
+        Function to list files in the given path.
+
+        Args:
+            glob_path: Path to the directory
+        Returns:
+            list of paths to files in the directory
+        """
+        globs = glob.glob(
+            f"{self._root_path}/{glob_path}",
+            recursive=True,
+        )
+
+        return [Path(glob).relative_to(self._root_path) for glob in globs]
+
+    def get(self, file_path: Path) -> str | None:
+        """
+        Function to retrieve the contents of the given path.
+
+        Args:
+            file_path: Path to the file
+        Returns:
+            string representing file contents
+        """
+        full_path = Path(self._root_path) / file_path
+
+        if full_path.exists():
+            return full_path.open(encoding="utf-8").read()
+
+        return None
+
+    def save(
+        self, file_path: Path, contents: str, overwrite: bool = False, **kwargs
+    ) -> Path:
+        """
+        Function to save data in the given location.
+
+        Args:
+            file_path: file destination path
+            contents: file contents
+            overwrite: boolean indicating file can be overwritten
+        Returns:
+            path to the materialized file
+        """
+
+        full_path = Path(self._root_path) / file_path
+
+        if overwrite is False and full_path.exists():
+            raise FileExistsError()
+
+        os.makedirs(os.path.dirname(full_path), exist_ok=True)
+        with full_path.open("w+", encoding="utf-8") as file:
+            file.write(contents)
+
+        return full_path
+
+
+class GitStorageService(LocalStorageService):
+    _instance = None
+
+    def __new__(cls, root_path: str, user: str, email: str, remote: str = "origin"):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self, root_path: str, user: str, email: str, remote: str = "origin"):
+        # prevent reinitialization
+        if getattr(self, "_initialized", False):
+            return
+
+        try:
+            self._root_path = root_path
+            self._remote = remote
+            self._repo = Repo(str(root_path))
+
+            with self._repo.config_writer() as git_config:
+                git_config.set_value("user", "email", email)
+                git_config.set_value("user", "name", user)
+
+            logging.info(f"✅ Initialized GitStorageService at {root_path}")
+        except (InvalidGitRepositoryError, NoSuchPathError) as err:
+            logging.error(f"❌ Git repo error: {err}")
+            raise err
+
+        self._initialized = True
+
+    def save(
+        self,
+        file_path: Path,
+        contents: str,
+        overwrite: bool = False,
+        fetch_latest: bool = True,
+        auto_commit: bool = True,
+        commit_msg: str = None,
+        **kwargs,
+    ) -> Path:
+        """
+        Function to save data in the given location.
+
+        Args:
+            file_path:  path or paths to files
+            contents: file contents
+            fetch_latest: boolean indicating to fetch latest state before saving
+            overwrite: boolean indicating file can be overwritten
+            auto_commit: boolean indicating whether to auto-commit changes
+            commit_msg: commit message
+        Returns:
+            path to the materialized file
+        """
+        if fetch_latest:
+            self._repo.remote(self._remote).pull()
+
+        full_path = super().save(file_path, contents, overwrite)
+
+        if auto_commit and full_path is not None:
+            commit_msg = (
+                f"add {os.path.basename(full_path)}" if not commit_msg else commit_msg
+            )
+            self.commit_and_push([file_path], commit_msg)
+
+        return full_path
+
+    def commit_and_push(self, file_paths: list[Path], msg: str):
+        """
+        Helper function to commit and push the given file.
+        """
+        self._repo.git.add(file_paths)
+        self._repo.index.commit(msg)
+        push = self._repo.remote(self._remote).push()
+        push.raise_if_error()
+
+    @staticmethod
+    def get_instance() -> "GitStorageService":
+        if GitStorageService._instance is None:
+            raise RuntimeError("GitStorageService has not been initialized yet.")
+        return GitStorageService._instance

everycure_datasets-0.2.33/src/everycure/datasets/models/__init__.py

@@ -0,0 +1,6 @@
+"""Dataset metadata models."""
+
+from everycure.datasets.models.v1 import DatasetMetadataV1
+from everycure.datasets.models.v2 import DatasetMetadataV2
+
+__all__ = ["DatasetMetadataV1", "DatasetMetadataV2"]

everycure_datasets-0.2.33/src/everycure/datasets/models/v1.py

@@ -0,0 +1,194 @@
+"""Dataset Metadata v1 - Initial version matching the JSON schema."""
+
+from datetime import UTC, datetime
+from enum import Enum
+from typing import Any, Optional
+
+from pydantic import AnyUrl, BaseModel, Field, HttpUrl, field_validator
+
+
+class StorageType(str, Enum):
+    """Storage type for dataset location."""
+
+    GCS = "gcs"
+    S3 = "s3"
+    LOCAL = "local"
+    BIGQUERY = "bigquery"
+    POSTGRES = "postgres"
+
+
+class FileFormat(str, Enum):
+    """File format for dataset."""
+
+    TSV = "tsv"
+    CSV = "csv"
+    PARQUET = "parquet"
+    JSON = "json"
+    JSONL = "jsonl"
+    AVRO = "avro"
+    ORC = "orc"
+
+
+class DatasetStatus(str, Enum):
+    """Dataset status."""
+
+    ACTIVE = "active"
+    DEPRECATED = "deprecated"
+    ARCHIVED = "archived"
+
+
+class Location(BaseModel):
+    """Dataset location information."""
+
+    type: StorageType = Field(..., description="Storage type")
+    uri: AnyUrl = Field(..., description="Full URI to the dataset")
+    format: FileFormat = Field(..., description="File format")
+
+    model_config = {"extra": "forbid"}
+
+
+class Owner(BaseModel):
+    """Dataset owner information."""
+
+    name: str = Field(..., min_length=1, description="Owner name")
+    email: Optional[str] = Field(None, description="Owner email address")
+
+    model_config = {"extra": "forbid"}
+
+
+class Origin(BaseModel):
+    """Dataset origin information."""
+
+    system: str = Field(..., description="Pipeline or system name")
+    url: HttpUrl = Field(..., description="GitHub URL to source code")
+    commit: Optional[str] = Field(
+        None,
+        pattern=r"^[a-f0-9]{7,40}$",
+        description="Git commit hash (7-40 hex characters)",
+    )
+    tag: Optional[str] = Field(None, description="Git tag")
+
+    model_config = {"extra": "forbid"}
+
+
+class ColumnSchema(BaseModel):
+    """Schema definition for a single column."""
+
+    name: str = Field(..., description="Column name")
+    type: str = Field(..., description="Column data type")
+    description: Optional[str] = Field(None, description="Column description")
+
+    model_config = {"extra": "forbid"}
+
+
+class DatasetSchema(BaseModel):
+    """Dataset schema information."""
+
+    row_count: Optional[int] = Field(None, ge=0, description="Number of rows")
+    columns: Optional[list[ColumnSchema]] = Field(
+        None, description="List of column definitions"
+    )
+
+    model_config = {"extra": "forbid"}
+
+
+class DatasetMetadataV1(BaseModel):
+    """
+    Dataset Metadata v1.
+
+    This model represents the metadata for a dataset in the registry.
+    It matches the structure defined in dataset.schema.json.
+    """
+
+    # Schema version - tracks the version of this metadata definition itself
+    schema_version: str = Field(
+        default="1.0.0",
+        description="Version of the dataset metadata schema definition",
+    )
+    name: str = Field(
+        ...,
+        pattern=r"^[a-z][a-z0-9_]*$",
+        description="Dataset name in snake_case",
+    )
+    version: str = Field(
+        ...,
+        pattern=r"^\d+\.\d+\.\d+$",
+        description="Semantic version (e.g., 0.2.0)",
+    )
+    description: Optional[str] = Field(
+        None, min_length=10, description="Brief description of the dataset"
+    )
+    message: Optional[str] = Field(
+        None, description="Optional message about this dataset version"
+    )
+    location: Location = Field(..., description="Dataset location")
+    created_at: datetime = Field(
+        default_factory=lambda: datetime.now(UTC),
+        description="ISO 8601 timestamp",
+    )
+    owner: Owner = Field(..., description="Dataset owner")
+    origin: Origin = Field(..., description="Dataset origin")
+    status: DatasetStatus = Field(
+        default=DatasetStatus.ACTIVE, description="Dataset status"
+    )
+    lineage: Optional[dict[str, Any]] = Field(
+        default=None, description="Placeholder for future lineage tracking"
+    )
+    # Use model_field to avoid shadowing BaseModel.schema
+    dataset_schema: Optional[DatasetSchema] = Field(
+        default=None,
+        alias="schema",
+        description="Dataset schema information",
+    )
+    metadata: Optional[dict[str, Any]] = Field(
+        default=None, description="Additional metadata dictionary"
+    )
+    tags: Optional[list[str]] = Field(
+        default=None,
+        description="Tags for discoverability (lowercase with hyphens)",
+    )
+    related_docs: Optional[HttpUrl] = Field(
+        default=None, description="Link to documentation"
+    )
+    deprecated_by: Optional[str] = Field(
+        default=None, description="Version that replaces this dataset"
+    )
+    deprecation_date: Optional[datetime] = Field(
+        default=None, description="Date when dataset was deprecated"
+    )
+
+    @field_validator("description")
+    @classmethod
+    def validate_description(cls, v: Optional[str]) -> Optional[str]:
+        """Validate description length if provided."""
+        if v is not None and len(v) < 10:
+            raise ValueError("Description must be at least 10 characters long")
+        return v
+
+    @field_validator("tags")
+    @classmethod
+    def validate_tags(cls, v: Optional[list[str]]) -> Optional[list[str]]:
+        """Validate tag format."""
+        if v is None:
+            return v
+        for tag in v:
+            if not tag or not tag.replace("-", "").replace("_", "").isalnum():
+                raise ValueError(
+                    f"Tag '{tag}' must contain only lowercase alphanumeric characters and hyphens"
+                )
+            if tag != tag.lower():
+                raise ValueError(f"Tag '{tag}' must be lowercase")
+        # Ensure unique tags
+        if len(v) != len(set(v)):
+            raise ValueError("Tags must be unique")
+        return v
+
+    model_config = {
+        "extra": "forbid",
+        "json_schema_extra": {
+            "$schema": "http://json-schema.org/2020-12/schema#",
+            "$id": "https://everycure.org/schemas/dataset.v1.schema.json",
+            "title": "Dataset Metadata v1",
+            "description": "Schema for dataset registry metadata files (v1)",
+        },
+    }

everycure_datasets-0.2.33/src/everycure/datasets/models/v2.py

@@ -0,0 +1,31 @@
+"""Dataset Metadata v2 - Refined version (placeholder for future improvements)."""
+
+from everycure.datasets.models.v1 import DatasetMetadataV1
+
+
+class DatasetMetadataV2(DatasetMetadataV1):
+    """
+    Dataset Metadata v2.
+
+    This is a placeholder for a future refined version of the dataset metadata model.
+    When v2 is implemented, it will include improvements and refinements over v1.
+
+    For now, this class inherits from v1 to maintain compatibility.
+    """
+
+    # TODO: Add v2-specific fields and improvements here
+    # Examples of potential improvements:
+    # - Better validation
+    # - Additional metadata fields
+    # - Improved lineage tracking
+    # - Enhanced schema definitions
+
+    model_config = {
+        "extra": "forbid",
+        "json_schema_extra": {
+            "$schema": "http://json-schema.org/2020-12/schema#",
+            "$id": "https://everycure.org/schemas/dataset.v2.schema.json",
+            "title": "Dataset Metadata v2",
+            "description": "Schema for dataset registry metadata files (v2)",
+        },
+    }

everycure_datasets-0.2.33/src/everycure/datasets/validate.py

@@ -0,0 +1,161 @@
+"""
+Pre-commit validation script for dataset registry.
+
+Checks:
+1. All version directories follow semantic versioning (MAJOR.MINOR.PATCH)
+2. Dataset folder names are snake_case
+3. No files are edited in datasets/ on main branch (immutability check)
+"""
+
+import yaml
+import re
+import sys
+from pathlib import Path
+from pydantic import ValidationError
+
+from logging import getLogger
+
+from everycure.datasets.models.v1 import DatasetMetadataV1
+
+logger = getLogger(__name__)
+
+# Patterns
+SNAKE_CASE_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$")
+SEMVER_PATTERN = re.compile(r"^\d+\.\d+\.\d+$")
+
+
+def get_dataset_directories(datasets_dir: Path) -> list[Path]:
+    """Get all dataset directories that don't start with '.' or '_'."""
+    return [
+        item
+        for item in datasets_dir.iterdir()
+        if item.is_dir()
+        and not item.name.startswith(".")
+        and not item.name.startswith("_")
+    ]
+
+
+def get_version_directories(dataset_dir: Path) -> list[Path]:
+    """Get all valid version directories in a dataset directory."""
+    versions = []
+    for item in dataset_dir.iterdir():
+        if item.is_dir() and SEMVER_PATTERN.match(item.name):
+            versions.append(item)
+    return versions
+
+
+def get_dataset_specification_files(datasets_dir: Path) -> list[Path]:
+    """Get all dataset specification files in a dataset directory."""
+    files = []
+
+    for dataset_dir in datasets_dir.iterdir():
+        for versioned_dataset in get_version_directories(dataset_dir):
+            if (versioned_dataset / "dataset.yaml").is_file():
+                files.append(versioned_dataset / "dataset.yaml")
+    return files
+
+
+def check_snake_case_names(datasets_dir: Path) -> list[str]:
+    """Check that all dataset names are snake_case."""
+    errors = []
+
+    for item in get_dataset_directories(datasets_dir):
+        if not SNAKE_CASE_PATTERN.match(item.name):
+            errors.append(
+                f"Dataset name '{item.name}' is not snake_case. "
+                f"Use lowercase letters, numbers, and underscores only."
+            )
+
+    return errors
+
+
+def check_semver_directories(datasets_dir: Path) -> list[str]:
+    """Check that all version directories follow semantic versioning."""
+    errors = []
+
+    for dataset in get_dataset_directories(datasets_dir):
+        for item in dataset.iterdir():
+            # Skip hidden files
+            if item.name.startswith("."):
+                continue
+
+            if item.is_dir() and not SEMVER_PATTERN.match(item.name):
+                errors.append(
+                    f"Version directory '{dataset.name}/{item.name}' does not follow "
+                    f"semantic versioning (MAJOR.MINOR.PATCH). Example: 0.1.0"
+                )
+
+    return errors
+
+
+def check_schema_validity(datasets_dir: Path) -> list[str]:
+    """Check that all schema files are valid."""
+    errors = []
+
+    for dataset_file in get_dataset_specification_files(datasets_dir):
+        dataset_yaml = yaml.safe_load(dataset_file.read_text())
+        try:
+            DatasetMetadataV1.model_validate(dataset_yaml)
+        except ValidationError as e:
+            print(f"Schema file '{dataset_file.relative_to(datasets_dir)}' is not valid: {e}")
+            errors.append(f"Schema file '{dataset_file.relative_to(datasets_dir)}' is not valid.")
+        
+    return errors
+
+def _find_repo_root() -> Path:
+    """Find the repository root by walking up from current directory or file location."""
+    # Start from current working directory
+    current = Path.cwd()
+
+    # Walk up looking for pyproject.toml (repo marker)
+    for path in [current, *current.parents]:
+        if (path / "pyproject.toml").exists() and (path / "datasets").exists():
+            return path
+
+    # Fallback: use file location (we're in src/everycure/datasets/validate.py)
+    return Path(__file__).parent.parent.parent.parent
+
+
+def validate_datasets(datasets_dir: Path | None = None) -> int:
+    """
+    Run all validation checks.
+
+    Args:
+        datasets_dir: Path to the datasets directory. If None, will try to find it
+                     relative to the current working directory or repository root.
+
+    Returns:
+        0 if validation passes, 1 if it fails.
+    """
+    if datasets_dir is None:
+        repo_root = _find_repo_root()
+        datasets_dir = repo_root / "datasets"
+
+    if not datasets_dir.exists():
+        print(
+            f"Error: datasets/ directory not found at {datasets_dir}", file=sys.stderr
+        )
+        return 1
+
+    all_errors = []
+
+    # Run all checks
+    print("Checking dataset naming conventions...")
+    all_errors.extend(check_snake_case_names(datasets_dir))
+
+    print("Checking semantic versioning...")
+    all_errors.extend(check_semver_directories(datasets_dir))
+
+    print("Checking schema validity...")
+    all_errors.extend(check_schema_validity(datasets_dir))
+
+    # Report results
+    if all_errors:
+        print("\n❌ Validation failed with the following errors:\n", file=sys.stderr)
+        for error in all_errors:
+            print(f"  {error}", file=sys.stderr)
+        print(f"\nTotal errors: {len(all_errors)}", file=sys.stderr)
+        return 1
+    else:
+        print("\n✅ All validation checks passed!")
+        return 0