everycure-datasets 0.1.1__tar.gz

everycure_datasets-0.1.1/.gitignore

@@ -0,0 +1,28 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyc
+
+# Virtual environments
+.venv/
+venv/
+
+# uv
+.uv/
+
+# Build artifacts
+dist/
+build/
+*.egg-info/
+
+# IDE (keep .vscode/settings.json for schema validation)
+.vscode/*
+!.vscode/settings.json
+.idea/
+
+# Environment variables
+.env
+.env.local
+
+# OS
+.DS_Store

everycure_datasets-0.1.1/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: everycure-datasets
+Version: 0.1.1
+Requires-Python: >=3.12
+Requires-Dist: pre-commit>=4.3.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: typer>=0.12.0

everycure_datasets-0.1.1/pyproject.toml

@@ -0,0 +1,38 @@
+[project]
+name = "everycure-datasets"
+version = "0.1.1"
+requires-python = ">=3.12"
+dependencies = [
+    "pre-commit>=4.3.0",
+    "pydantic>=2.0.0",
+    "typer>=0.12.0",
+]
+scripts = {datasets = "everycure.datasets.cli:app"}
+
+[tool.hatch.build]
+include = ["src/**"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/everycure"]
+
+[tool.uv]
+keyring-provider = "subprocess"
+
+[[tool.uv.index]]
+name = "pypi"
+url = "https://pypi.org/simple"
+
+[[tool.uv.index]]
+name = "everycure"
+url = "https://oauth2accesstoken@us-central1-python.pkg.dev/core-422020/everycure/simple/"
+publish-url = "https://us-central1-python.pkg.dev/core-422020/everycure/"
+default = false
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[dependency-groups]
+dev = [
+    "keyrings-google-artifactregistry-auth>=1.1.2",
+]

everycure_datasets-0.1.1/src/everycure/__init__.py

@@ -0,0 +1,3 @@
+"""Every Cure namespace package."""
+
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

everycure_datasets-0.1.1/src/everycure/datasets/__init__.py

@@ -0,0 +1,6 @@
+"""Dataset Registry - Centralized dataset metadata management."""
+
+from everycure.datasets.models.v1 import DatasetMetadataV1
+from everycure.datasets.models.v2 import DatasetMetadataV2
+
+__all__ = ["DatasetMetadataV1", "DatasetMetadataV2"]

everycure_datasets-0.1.1/src/everycure/datasets/cli.py

@@ -0,0 +1,64 @@
+"""CLI for datasets registry operations."""
+
+from pathlib import Path
+
+import typer
+
+from everycure.datasets.generate_schema import generate_all_schemas
+from everycure.datasets.validate import validate_datasets
+
+app = typer.Typer(
+    name="datasets",
+    help="Datasets registry management CLI",
+    add_completion=False,
+    no_args_is_help=True,
+)
+
+schema_app = typer.Typer(
+    name="schema",
+    help="Schema management commands",
+    add_completion=False,
+)
+
+app.add_typer(schema_app)
+
+
+@app.callback(invoke_without_command=True)
+def main(ctx: typer.Context) -> None:
+    """Datasets registry management CLI."""
+    if ctx.invoked_subcommand is None:
+        typer.echo(ctx.get_help())
+
+
+@app.command()
+def validate(
+    datasets_dir: Path | None = typer.Option(
+        None,
+        "--datasets-dir",
+        "-d",
+        help="Path to the datasets directory (default: auto-detect)",
+    ),
+) -> None:
+    """
+    Validate dataset YAML files and directory structure.
+
+    Checks:
+    - Dataset names are snake_case
+    - Version directories follow semantic versioning (MAJOR.MINOR.PATCH)
+    """
+    return validate_datasets(datasets_dir)
+
+
+@schema_app.command()
+def generate() -> None:
+    """
+    Generate JSON schema(s) from the Pydantic models.
+    """
+    repo_root = Path.cwd()
+    schema_dir = repo_root / ".schema"
+    generate_all_schemas(schema_dir)
+    typer.echo(f"✓ Generated all JSON schemas in {schema_dir}")
+
+
+if __name__ == "__main__":
+    app()

everycure_datasets-0.1.1/src/everycure/datasets/generate_schema.py

@@ -0,0 +1,62 @@
+"""Generate JSON schema from Pydantic models."""
+
+import json
+from pathlib import Path
+
+from everycure.datasets.models.v1 import DatasetMetadataV1
+from everycure.datasets.models.v2 import DatasetMetadataV2
+
+
+def generate_schema(model_class, output_path: Path, schema_id: str) -> None:
+    """
+    Generate JSON schema from a Pydantic model.
+
+    Args:
+        model_class: The Pydantic model class to generate schema from
+        output_path: Path where the JSON schema should be written
+        schema_id: The $id for the schema (FQDN)
+    """
+    # Get the JSON schema from the Pydantic model
+    schema = model_class.model_json_schema(
+        mode="serialization",
+        by_alias=True,
+    )
+
+    # Update the $id to use the provided FQDN
+    schema["$id"] = schema_id
+
+    # Remove fields with defaults from required list
+    # but keep them in properties so they're documented
+    fields_with_defaults = ["schema_version", "status", "created_at"]
+    if "required" in schema:
+        for field in fields_with_defaults:
+            if field in schema["required"]:
+                schema["required"].remove(field)
+
+    # Write to file with proper formatting
+    with open(output_path, "w") as f:
+        json.dump(schema, f, indent=2)
+        f.write("\n")  # Add trailing newline
+
+    print(f"Generated JSON schema at {output_path}")
+
+
+def generate_all_schemas(schema_dir: Path) -> None:
+    """Generate all schema versions."""
+    schema_dir.mkdir(parents=True, exist_ok=True)
+
+    # Generate v1 schema
+    v1_path = schema_dir / "dataset.v1.schema.json"
+    generate_schema(
+        DatasetMetadataV1,
+        v1_path,
+        "https://everycure.org/schemas/dataset.v1.schema.json",
+    )
+
+    # Generate v2 schema
+    v2_path = schema_dir / "dataset.v2.schema.json"
+    generate_schema(
+        DatasetMetadataV2,
+        v2_path,
+        "https://everycure.org/schemas/dataset.v2.schema.json",
+    )

everycure_datasets-0.1.1/src/everycure/datasets/models/__init__.py

@@ -0,0 +1,6 @@
+"""Dataset metadata models."""
+
+from everycure.datasets.models.v1 import DatasetMetadataV1
+from everycure.datasets.models.v2 import DatasetMetadataV2
+
+__all__ = ["DatasetMetadataV1", "DatasetMetadataV2"]

everycure_datasets-0.1.1/src/everycure/datasets/models/v1.py

@@ -0,0 +1,193 @@
+"""Dataset Metadata v1 - Initial version matching the JSON schema."""
+
+from datetime import UTC, datetime
+from enum import Enum
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field, field_validator, HttpUrl
+
+
+class StorageType(str, Enum):
+    """Storage type for dataset location."""
+
+    GCS = "gcs"
+    S3 = "s3"
+    LOCAL = "local"
+    BIGQUERY = "bigquery"
+    POSTGRES = "postgres"
+
+
+class FileFormat(str, Enum):
+    """File format for dataset."""
+
+    TSV = "tsv"
+    CSV = "csv"
+    PARQUET = "parquet"
+    JSON = "json"
+    JSONL = "jsonl"
+    AVRO = "avro"
+    ORC = "orc"
+
+
+class DatasetStatus(str, Enum):
+    """Dataset status."""
+
+    ACTIVE = "active"
+    DEPRECATED = "deprecated"
+    ARCHIVED = "archived"
+
+
+class Location(BaseModel):
+    """Dataset location information."""
+
+    type: StorageType = Field(..., description="Storage type")
+    uri: HttpUrl = Field(..., description="Full URI to the dataset")
+    format: FileFormat = Field(..., description="File format")
+
+    model_config = {"extra": "forbid"}
+
+
+class Owner(BaseModel):
+    """Dataset owner information."""
+
+    name: str = Field(..., min_length=1, description="Owner name")
+    email: Optional[str] = Field(None, description="Owner email address")
+
+    model_config = {"extra": "forbid"}
+
+
+class Origin(BaseModel):
+    """Dataset origin information."""
+
+    system: str = Field(..., description="Pipeline or system name")
+    url: HttpUrl = Field(..., description="GitHub URL to source code")
+    commit: str = Field(
+        ...,
+        pattern=r"^[a-f0-9]{7,40}$",
+        description="Git commit hash (7-40 hex characters)",
+    )
+
+    model_config = {"extra": "forbid"}
+
+
+class ColumnSchema(BaseModel):
+    """Schema definition for a single column."""
+
+    name: str = Field(..., description="Column name")
+    type: str = Field(..., description="Column data type")
+    description: Optional[str] = Field(None, description="Column description")
+
+    model_config = {"extra": "forbid"}
+
+
+class DatasetSchema(BaseModel):
+    """Dataset schema information."""
+
+    row_count: Optional[int] = Field(None, ge=0, description="Number of rows")
+    columns: Optional[list[ColumnSchema]] = Field(
+        None, description="List of column definitions"
+    )
+
+    model_config = {"extra": "forbid"}
+
+
+class DatasetMetadataV1(BaseModel):
+    """
+    Dataset Metadata v1.
+
+    This model represents the metadata for a dataset in the registry.
+    It matches the structure defined in dataset.schema.json.
+    """
+
+    # Schema version - tracks the version of this metadata definition itself
+    schema_version: str = Field(
+        default="1.0.0",
+        description="Version of the dataset metadata schema definition",
+    )
+    name: str = Field(
+        ...,
+        pattern=r"^[a-z][a-z0-9_]*$",
+        description="Dataset name in snake_case",
+    )
+    version: str = Field(
+        ...,
+        pattern=r"^\d+\.\d+\.\d+$",
+        description="Semantic version (e.g., 0.2.0)",
+    )
+    description: Optional[str] = Field(
+        None, min_length=10, description="Brief description of the dataset"
+    )
+    message: Optional[str] = Field(
+        None, description="Optional message about this dataset version"
+    )
+    location: Location = Field(..., description="Dataset location")
+    created_at: datetime = Field(
+        default_factory=lambda: datetime.now(UTC),
+        description="ISO 8601 timestamp",
+    )
+    owner: Owner = Field(..., description="Dataset owner")
+    origin: Origin = Field(..., description="Dataset origin")
+    status: DatasetStatus = Field(
+        default=DatasetStatus.ACTIVE, description="Dataset status"
+    )
+    lineage: Optional[dict[str, Any]] = Field(
+        default=None, description="Placeholder for future lineage tracking"
+    )
+    # Use model_field to avoid shadowing BaseModel.schema
+    dataset_schema: Optional[DatasetSchema] = Field(
+        default=None,
+        alias="schema",
+        description="Dataset schema information",
+    )
+    metadata: Optional[dict[str, Any]] = Field(
+        default=None, description="Additional metadata dictionary"
+    )
+    tags: Optional[list[str]] = Field(
+        default=None,
+        description="Tags for discoverability (lowercase with hyphens)",
+    )
+    related_docs: Optional[HttpUrl] = Field(
+        default=None, description="Link to documentation"
+    )
+    deprecated_by: Optional[str] = Field(
+        default=None, description="Version that replaces this dataset"
+    )
+    deprecation_date: Optional[datetime] = Field(
+        default=None, description="Date when dataset was deprecated"
+    )
+
+    @field_validator("description")
+    @classmethod
+    def validate_description(cls, v: Optional[str]) -> Optional[str]:
+        """Validate description length if provided."""
+        if v is not None and len(v) < 10:
+            raise ValueError("Description must be at least 10 characters long")
+        return v
+
+    @field_validator("tags")
+    @classmethod
+    def validate_tags(cls, v: Optional[list[str]]) -> Optional[list[str]]:
+        """Validate tag format."""
+        if v is None:
+            return v
+        for tag in v:
+            if not tag or not tag.replace("-", "").replace("_", "").isalnum():
+                raise ValueError(
+                    f"Tag '{tag}' must contain only lowercase alphanumeric characters and hyphens"
+                )
+            if tag != tag.lower():
+                raise ValueError(f"Tag '{tag}' must be lowercase")
+        # Ensure unique tags
+        if len(v) != len(set(v)):
+            raise ValueError("Tags must be unique")
+        return v
+
+    model_config = {
+        "extra": "forbid",
+        "json_schema_extra": {
+            "$schema": "http://json-schema.org/2020-12/schema#",
+            "$id": "https://everycure.org/schemas/dataset.v1.schema.json",
+            "title": "Dataset Metadata v1",
+            "description": "Schema for dataset registry metadata files (v1)",
+        },
+    }

everycure_datasets-0.1.1/src/everycure/datasets/models/v2.py

@@ -0,0 +1,31 @@
+"""Dataset Metadata v2 - Refined version (placeholder for future improvements)."""
+
+from everycure.datasets.models.v1 import DatasetMetadataV1
+
+
+class DatasetMetadataV2(DatasetMetadataV1):
+    """
+    Dataset Metadata v2.
+
+    This is a placeholder for a future refined version of the dataset metadata model.
+    When v2 is implemented, it will include improvements and refinements over v1.
+
+    For now, this class inherits from v1 to maintain compatibility.
+    """
+
+    # TODO: Add v2-specific fields and improvements here
+    # Examples of potential improvements:
+    # - Better validation
+    # - Additional metadata fields
+    # - Improved lineage tracking
+    # - Enhanced schema definitions
+
+    model_config = {
+        "extra": "forbid",
+        "json_schema_extra": {
+            "$schema": "http://json-schema.org/2020-12/schema#",
+            "$id": "https://everycure.org/schemas/dataset.v2.schema.json",
+            "title": "Dataset Metadata v2",
+            "description": "Schema for dataset registry metadata files (v2)",
+        },
+    }

everycure_datasets-0.1.1/src/everycure/datasets/validate.py

@@ -0,0 +1,125 @@
+"""
+Pre-commit validation script for dataset registry.
+
+Checks:
+1. All version directories follow semantic versioning (MAJOR.MINOR.PATCH)
+2. Dataset folder names are snake_case
+3. No files are edited in datasets/ on main branch (immutability check)
+"""
+
+import re
+import sys
+from pathlib import Path
+
+# Patterns
+SNAKE_CASE_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$")
+SEMVER_PATTERN = re.compile(r"^\d+\.\d+\.\d+$")
+
+
+def get_dataset_directories(datasets_dir: Path) -> list[Path]:
+    """Get all dataset directories that don't start with '.' or '_'."""
+    return [
+        item
+        for item in datasets_dir.iterdir()
+        if item.is_dir()
+        and not item.name.startswith(".")
+        and not item.name.startswith("_")
+    ]
+
+
+def get_version_directories(dataset_dir: Path) -> list[Path]:
+    """Get all valid version directories in a dataset directory."""
+    versions = []
+    for item in dataset_dir.iterdir():
+        if item.is_dir() and SEMVER_PATTERN.match(item.name):
+            versions.append(item)
+    return versions
+
+
+def check_snake_case_names(datasets_dir: Path) -> list[str]:
+    """Check that all dataset names are snake_case."""
+    errors = []
+
+    for item in get_dataset_directories(datasets_dir):
+        if not SNAKE_CASE_PATTERN.match(item.name):
+            errors.append(
+                f"Dataset name '{item.name}' is not snake_case. "
+                f"Use lowercase letters, numbers, and underscores only."
+            )
+
+    return errors
+
+
+def check_semver_directories(datasets_dir: Path) -> list[str]:
+    """Check that all version directories follow semantic versioning."""
+    errors = []
+
+    for dataset in get_dataset_directories(datasets_dir):
+        for item in dataset.iterdir():
+            # Skip hidden files
+            if item.name.startswith("."):
+                continue
+
+            if item.is_dir() and not SEMVER_PATTERN.match(item.name):
+                errors.append(
+                    f"Version directory '{dataset.name}/{item.name}' does not follow "
+                    f"semantic versioning (MAJOR.MINOR.PATCH). Example: 0.1.0"
+                )
+
+    return errors
+
+
+def _find_repo_root() -> Path:
+    """Find the repository root by walking up from current directory or file location."""
+    # Start from current working directory
+    current = Path.cwd()
+
+    # Walk up looking for pyproject.toml (repo marker)
+    for path in [current, *current.parents]:
+        if (path / "pyproject.toml").exists() and (path / "datasets").exists():
+            return path
+
+    # Fallback: use file location (we're in src/everycure/datasets/validate.py)
+    return Path(__file__).parent.parent.parent.parent
+
+
+def validate_datasets(datasets_dir: Path | None = None) -> int:
+    """
+    Run all validation checks.
+
+    Args:
+        datasets_dir: Path to the datasets directory. If None, will try to find it
+                     relative to the current working directory or repository root.
+
+    Returns:
+        0 if validation passes, 1 if it fails.
+    """
+    if datasets_dir is None:
+        repo_root = _find_repo_root()
+        datasets_dir = repo_root / "datasets"
+
+    if not datasets_dir.exists():
+        print(
+            f"Error: datasets/ directory not found at {datasets_dir}", file=sys.stderr
+        )
+        return 1
+
+    all_errors = []
+
+    # Run all checks
+    print("Checking dataset naming conventions...")
+    all_errors.extend(check_snake_case_names(datasets_dir))
+
+    print("Checking semantic versioning...")
+    all_errors.extend(check_semver_directories(datasets_dir))
+
+    # Report results
+    if all_errors:
+        print("\n❌ Validation failed with the following errors:\n", file=sys.stderr)
+        for error in all_errors:
+            print(f"  {error}", file=sys.stderr)
+        print(f"\nTotal errors: {len(all_errors)}", file=sys.stderr)
+        return 1
+    else:
+        print("\n✅ All validation checks passed!")
+        return 0