drc-vantage 0.1.0__tar.gz

drc_vantage-0.1.0/.env.example

@@ -0,0 +1,17 @@
+# PostgreSQL (data warehouse)
+POSTGRES_HOST=localhost
+POSTGRES_PORT=5432
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=pOstgres123!
+POSTGRES_DATABASE=demo
+POSTGRES_DEFAULT_SCHEMA=public
+
+# Vantage configuration
+VANTAGE_SCHEMA=vantage
+GENERATE_AI_VALIDATIONS=false
+
+# MinIO / S3
+S3_ENDPOINT_URL=http://localhost:9000
+S3_ACCESS_KEY=minioadmin
+S3_SECRET_KEY=minioadmin
+S3_BUCKET_NAME=demo

drc_vantage-0.1.0/.gitignore

@@ -0,0 +1,79 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# Dependencies
+**/node_modules/
+**/.pnp
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/versions
+
+# Testing
+**/coverage/
+
+# Next.js
+**/.next/
+**/out/
+
+# Production
+**/build/
+
+# Python
+**/__pycache__/
+**/*.py[cod]
+**/.venv/
+**/dist/
+**/wheels/
+**/*.egg-info/
+**/.ruff_cache/
+**/.pytest_cache/
+
+# TypeScript
+**/*.tsbuildinfo
+**/next-env.d.ts
+
+# Prisma
+**/prisma/generated/
+
+# Environment variables
+.env
+.env.*
+!.env.example
+!.env.*.example
+
+# Debug logs
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# Deployment
+**/.vercel/
+
+# OS
+**/.DS_Store
+**/Thumbs.db
+
+# IDE — ignore personal settings, keep shared workspace config
+**/.vscode/*
+!**/.vscode/settings.json
+!**/.vscode/extensions.json
+!**/.vscode/tasks.json
+!**/.vscode/launch.json
+**/.idea/
+
+# Cursor
+**/.cursor/
+
+# Local themes & branding
+**/src/config/themes/
+**/public/branding/
+
+# Notebooks
+*.ipynb_checkpoints/
+notebook.ipynb
+
+# Secrets
+*.pem

drc_vantage-0.1.0/.python-version

@@ -0,0 +1 @@
+3.14

drc_vantage-0.1.0/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.4
+Name: drc-vantage
+Version: 0.1.0
+Summary: DRC Vantage SDK
+Requires-Python: >=3.14
+Requires-Dist: boto3>=1.43.46
+Requires-Dist: langchain-core>=0.3.0
+Requires-Dist: langchain-openai>=0.3.0
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: numpy>=2.5.1
+Requires-Dist: pillow>=11.0.0
+Requires-Dist: psycopg[binary]>=3.3.4
+Requires-Dist: pydantic-settings>=2.14.2
+Requires-Dist: pydantic>=2.13.4
+Requires-Dist: resend>=2.32.2
+Requires-Dist: scikit-learn>=1.9.0
+Requires-Dist: sqlalchemy>=2.0.51

drc_vantage-0.1.0/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "drc-vantage"
+version = "0.1.0"
+description = "DRC Vantage SDK"
+readme = "README.md"
+requires-python = ">=3.14"
+dependencies = [
+    "boto3>=1.43.46",
+    "langchain-core>=0.3.0",
+    "langchain-openai>=0.3.0",
+    "loguru>=0.7.3",
+    "numpy>=2.5.1",
+    "pillow>=11.0.0",
+    "psycopg[binary]>=3.3.4",
+    "pydantic>=2.13.4",
+    "pydantic-settings>=2.14.2",
+    "resend>=2.32.2",
+    "scikit-learn>=1.9.0",
+    "sqlalchemy>=2.0.51",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/vantage"]

drc_vantage-0.1.0/src/vantage/__init__.py

@@ -0,0 +1,21 @@
+"""DRC Vantage SDK — platform data pipelines."""
+
+from vantage.client import Base, Client, configure, ensure_database_exists, get_client, get_session, get_settings
+from vantage.data_sources.data_source import DataSource
+from vantage.data_sources.data_source_category import DataSourceCategory
+from vantage.data_storage_system.data_storage_system import DataStorageSystem
+from vantage.datasets.dataset import Dataset
+
+__all__ = [
+    "Base",
+    "Client",
+    "DataSource",
+    "DataSourceCategory",
+    "DataStorageSystem",
+    "Dataset",
+    "configure",
+    "ensure_database_exists",
+    "get_client",
+    "get_session",
+    "get_settings",
+]

drc_vantage-0.1.0/src/vantage/client.py

@@ -0,0 +1,162 @@
+"""Vantage SDK client: database access and runtime configuration."""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from functools import lru_cache
+from typing import Any, Generator, Protocol, runtime_checkable
+from urllib.parse import quote_plus
+
+from loguru import logger
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+@runtime_checkable
+class VantageSettings(Protocol):
+    """Settings contract expected by the SDK (implemented by each consumer repo)."""
+
+    vantage_schema: str
+    generate_ai_validations: bool
+
+    @property
+    def is_postgres_configured(self) -> bool: ...
+
+    postgres_database: str
+    postgres_user: str
+    postgres_host: str
+    postgres_port: int
+    postgres_password: Any
+
+    openai_api_key: str | None
+    resend_api_key: str | None
+
+    s3_endpoint_url: str | None
+    s3_access_key: str | None
+    s3_secret_key: str | None
+    s3_bucket_name: str | None
+
+    def is_s3_config_valid(self) -> bool: ...
+
+    @property
+    def is_s3_configured(self) -> bool: ...
+
+
+_client: Client | None = None
+
+
+def _get_secret(value: Any) -> str:
+    if hasattr(value, "get_secret_value"):
+        return value.get_secret_value()
+    return str(value)
+
+
+class Client:
+    """Entry point for SDK runtime configuration and database access."""
+
+    def __init__(self, settings: VantageSettings) -> None:
+        self.settings = settings
+        self._engine: Engine | None = None
+        self._session_maker: sessionmaker[Session] | None = None
+
+    def _build_postgres_url(self, database: str | None = None) -> str:
+        settings = self.settings
+        if not settings.is_postgres_configured:
+            raise ValueError("PostgreSQL is not configured. Set POSTGRES_* variables in your .env file.")
+
+        db = database or settings.postgres_database
+        password = quote_plus(_get_secret(settings.postgres_password))
+        user = quote_plus(settings.postgres_user)
+        return f"postgresql+psycopg://{user}:{password}@{settings.postgres_host}:{settings.postgres_port}/{db}"
+
+    def get_engine(self) -> Engine:
+        if self._engine is None:
+            self._engine = create_engine(self._build_postgres_url(), pool_pre_ping=True)
+        return self._engine
+
+    def get_session_maker(self) -> sessionmaker[Session]:
+        if self._session_maker is None:
+            self._session_maker = sessionmaker(
+                bind=self.get_engine(),
+                class_=Session,
+                expire_on_commit=False,
+            )
+        return self._session_maker
+
+    @contextmanager
+    def get_session(self) -> Generator[Session, None, None]:
+        session = self.get_session_maker()()
+        try:
+            yield session
+            session.commit()
+        except Exception:
+            session.rollback()
+            raise
+        finally:
+            session.close()
+
+    def ensure_database_exists(self) -> None:
+        settings = self.settings
+
+        if not settings.is_postgres_configured:
+            logger.warning("Database configuration is not valid, skipping database creation check")
+            return
+
+        engine = create_engine(
+            self._build_postgres_url("postgres"),
+            pool_pre_ping=True,
+            isolation_level="AUTOCOMMIT",
+        )
+
+        try:
+            with engine.connect() as conn:
+                exists = conn.execute(
+                    text("SELECT 1 FROM pg_database WHERE datname = :dbname"),
+                    {"dbname": settings.postgres_database},
+                ).scalar()
+
+                if exists:
+                    logger.info(f"Database '{settings.postgres_database}' already exists")
+                    return
+
+                logger.info(f"Database '{settings.postgres_database}' does not exist, creating it...")
+                conn.execute(text(f'CREATE DATABASE "{settings.postgres_database}"'))
+                logger.info(f"Database '{settings.postgres_database}' created successfully")
+        finally:
+            engine.dispose()
+
+
+def configure(client: Client) -> None:
+    """Set the active SDK client for the current process."""
+    global _client
+    _client = client
+    get_settings.cache_clear()
+
+
+def get_client() -> Client:
+    if _client is None:
+        raise RuntimeError(
+            "Vantage client is not configured. Call vantage.configure(Client(settings)) before using the SDK."
+        )
+    return _client
+
+
+@lru_cache
+def get_settings() -> VantageSettings:
+    return get_client().settings
+
+
+@contextmanager
+def get_session() -> Generator[Session, None, None]:
+    with get_client().get_session() as session:
+        yield session
+
+
+def ensure_database_exists() -> None:
+    """Ensure the configured PostgreSQL database exists."""
+    get_client().ensure_database_exists()

drc_vantage-0.1.0/src/vantage/core/logo.py

@@ -0,0 +1,156 @@
+"""Prepare and upload data source logos to S3 (mirrors frontend logo pipeline)."""
+
+import io
+import mimetypes
+from pathlib import Path
+from urllib.parse import urlparse
+from urllib.request import Request, urlopen
+from uuid import uuid4
+
+from loguru import logger
+from PIL import Image
+from vantage.client import get_settings
+from vantage.core.s3 import upload_object
+
+SVG_MIME = "image/svg+xml"
+MAX_INPUT_BYTES = 8 * 1024 * 1024
+LOGO_MAX_DIMENSION = 200
+WEBP_QUALITY = 85
+LOGO_FETCH_USER_AGENT = "demo-flows/0.1 (Foresyn; data source logo sync)"
+
+
+def get_extension_for_logo_content_type(content_type: str) -> str | None:
+    normalized = content_type.split(";")[0].strip().lower()
+    mapping = {
+        "image/webp": "webp",
+        "image/svg+xml": "svg",
+        "image/png": "png",
+        "image/jpeg": "jpg",
+        "image/jpg": "jpg",
+    }
+    return mapping.get(normalized)
+
+
+def prepare_data_source_logo(
+    image_bytes: bytes,
+    *,
+    filename: str | None = None,
+    content_type: str | None = None,
+) -> tuple[bytes, str]:
+    """
+    Prepare a logo for upload.
+
+    Raster images: resize/compress to WebP, max 200×200px.
+    SVG: kept as vector text (no raster compression), basic structure check.
+    """
+    if len(image_bytes) > MAX_INPUT_BYTES:
+        raise ValueError("FILE_TOO_LARGE")
+
+    guessed_type = content_type or (
+        mimetypes.guess_type(filename or "")[0] if filename else None
+    )
+    looks_svg = guessed_type == SVG_MIME or (
+        filename and filename.lower().endswith(".svg")
+    )
+
+    if looks_svg:
+        return _prepare_svg_logo(image_bytes)
+
+    try:
+        return _prepare_raster_logo(image_bytes)
+    except ValueError:
+        raise
+    except Exception as exc:
+        raise ValueError("COMPRESS_FAILED") from exc
+
+
+def upload_data_source_logo(image_buffer: bytes, content_type: str) -> str:
+    """
+    Upload a data source logo to S3. Uses a random object name (never the client filename).
+
+    Returns:
+        S3 object key to store in ``DataSource.logoUrl`` (e.g. ``logos/uuid.webp``)
+    """
+    ext = get_extension_for_logo_content_type(content_type)
+    if not ext:
+        raise ValueError(f"Unsupported logo content type: {content_type}")
+
+    normalized_type = content_type.split(";")[0].strip().lower()
+    object_key = f"logos/{uuid4()}.{ext}"
+
+    try:
+        upload_object(
+            object_key=object_key,
+            body=image_buffer,
+            content_type=normalized_type or f"image/{'jpeg' if ext == 'jpg' else ext}",
+        )
+        logger.info("Uploaded data source logo to %s", object_key)
+        return object_key
+    except Exception as exc:
+        logger.exception("Error uploading data source logo to S3")
+        raise RuntimeError(
+            f"Failed to upload data source logo: {exc if isinstance(exc, Exception) else 'Unknown error'}"
+        ) from exc
+
+
+def upload_data_source_logo_from_path(path: str | Path) -> str:
+    """Read a local file, prepare it, upload to S3, and return the object key."""
+    file_path = Path(path)
+    if not file_path.is_file():
+        raise FileNotFoundError(f"Logo file not found: {file_path}")
+
+    raw = file_path.read_bytes()
+    prepared, content_type = prepare_data_source_logo(raw, filename=file_path.name)
+    return upload_data_source_logo(prepared, content_type)
+
+
+def upload_data_source_logo_from_url(url: str) -> str:
+    """Download a remote image, prepare it, upload to S3, and return the object key."""
+    if not get_settings().is_s3_configured:
+        return url
+
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https"):
+        raise ValueError(f"Invalid logo URL: {url}")
+
+    request = Request(url, headers={"User-Agent": LOGO_FETCH_USER_AGENT})
+    with urlopen(request, timeout=30) as response:
+        raw = response.read()
+        content_type = response.headers.get("Content-Type")
+        filename = Path(parsed.path).name or None
+
+    prepared, content_type = prepare_data_source_logo(
+        raw, filename=filename, content_type=content_type
+    )
+    return upload_data_source_logo(prepared, content_type)
+
+
+def _prepare_svg_logo(image_bytes: bytes) -> tuple[bytes, str]:
+    try:
+        text = image_bytes.decode("utf-8")
+    except UnicodeDecodeError as exc:
+        raise ValueError("INVALID_SVG") from exc
+
+    if not text.strip().lstrip().lower().startswith("<svg"):
+        raise ValueError("INVALID_SVG")
+
+    return text.encode("utf-8"), SVG_MIME
+
+
+def _prepare_raster_logo(image_bytes: bytes) -> tuple[bytes, str]:
+    with Image.open(io.BytesIO(image_bytes)) as img:
+        img.thumbnail(
+            (LOGO_MAX_DIMENSION, LOGO_MAX_DIMENSION), Image.Resampling.LANCZOS
+        )
+
+        if img.mode in ("RGBA", "LA") or (
+            img.mode == "P" and "transparency" in img.info
+        ):
+            img = img.convert("RGBA")
+        else:
+            img = img.convert("RGB")
+
+        out = io.BytesIO()
+        img.save(out, format="WEBP", quality=WEBP_QUALITY, method=6)
+
+    return out.getvalue(), "image/webp"

drc_vantage-0.1.0/src/vantage/core/organization.py

@@ -0,0 +1,52 @@
+"""Shared helpers for resolving platform organizations."""
+
+from typing import Optional
+
+from sqlalchemy import text
+
+from vantage.client import get_session
+from vantage.client import get_settings
+
+
+def get_organization_by_slug(organization_slug: str) -> Optional[str]:
+    with get_session() as session:
+        organization = session.execute(
+            text(f"SELECT id FROM {get_settings().vantage_schema}.organization WHERE slug = :organization_slug"),
+            {"organization_slug": organization_slug},
+        ).fetchone()
+        return organization[0] if organization else None
+
+
+def get_organization_count() -> int:
+    with get_session() as session:
+        row = session.execute(text(f"SELECT COUNT(*) FROM {get_settings().vantage_schema}.organization")).fetchone()
+        return int(row[0]) if row else 0
+
+
+def get_single_organization_id_if_only_one_exists() -> Optional[str]:
+    with get_session() as session:
+        rows = session.execute(text(f"SELECT id FROM {get_settings().vantage_schema}.organization LIMIT 2")).fetchall()
+        if len(rows) == 1:
+            return rows[0][0]
+        return None
+
+
+def resolve_organization_id(
+    organization_id: Optional[str] = None,
+    organization_slug: Optional[str] = None,
+) -> str:
+    if organization_slug:
+        org_id = get_organization_by_slug(organization_slug=organization_slug)
+        if not org_id:
+            raise ValueError(f"Organization with slug '{organization_slug}' not found")
+        return org_id
+    if organization_id:
+        return organization_id
+    org_id = get_single_organization_id_if_only_one_exists()
+    if org_id:
+        return org_id
+    if get_organization_count() == 0:
+        raise ValueError("No organization found; provide organization_id or organization_slug")
+    raise ValueError(
+        "Multiple organizations found; provide organization_id or organization_slug"
+    )

drc_vantage-0.1.0/src/vantage/core/s3.py

@@ -0,0 +1,63 @@
+"""S3-compatible object storage client (MinIO, AWS S3, etc.)."""
+
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Any
+
+import boto3
+from botocore.config import Config
+
+from vantage.client import get_settings
+
+
+@dataclass(frozen=True)
+class S3Config:
+    endpoint_url: str
+    access_key: str
+    secret_key: str
+    bucket_name: str
+
+
+def get_s3_config() -> S3Config:
+    settings = get_settings()
+    if not settings.is_s3_config_valid():
+        raise ValueError(
+            "S3 configuration is incomplete. Set S3_ENDPOINT_URL, S3_ACCESS_KEY, S3_SECRET_KEY, and S3_BUCKET_NAME."
+        )
+    return S3Config(
+        endpoint_url=settings.s3_endpoint_url,  # type: ignore[arg-type]
+        access_key=settings.s3_access_key,  # type: ignore[arg-type]
+        secret_key=settings.s3_secret_key,  # type: ignore[arg-type]
+        bucket_name=settings.s3_bucket_name,  # type: ignore[arg-type]
+    )
+
+
+@lru_cache
+def get_s3_client() -> Any:
+    config = get_s3_config()
+    return boto3.client(
+        "s3",
+        endpoint_url=config.endpoint_url,
+        aws_access_key_id=config.access_key,
+        aws_secret_access_key=config.secret_key,
+        region_name="us-east-1",
+        config=Config(signature_version="s3v4"),
+    )
+
+
+def upload_object(
+    object_key: str,
+    body: bytes,
+    content_type: str,
+) -> str:
+    """Upload bytes to the configured bucket. Returns the object key."""
+    config = get_s3_config()
+    client = get_s3_client()
+    client.put_object(
+        Bucket=config.bucket_name,
+        Key=object_key,
+        Body=body,
+        ContentLength=len(body),
+        ContentType=content_type,
+    )
+    return object_key

drc_vantage-0.1.0/src/vantage/data_sources/data_source.py

@@ -0,0 +1,68 @@
+"""Platform data source definition."""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional, Union
+
+from vantage.client import get_session
+from vantage.data_sources.data_source_category import DataSourceCategory
+from vantage.data_sources.sync.data_source_helpers import get_data_source_id_by_slug
+from vantage.core.logo import (
+    upload_data_source_logo_from_path,
+    upload_data_source_logo_from_url,
+)
+from vantage.data_sources.sync.sync_data_source import sync_data_source
+from vantage.core.organization import resolve_organization_id
+
+
+@dataclass
+class DataSource:
+    """Platform data source."""
+
+    slug: str  # URL-safe unique identifier for the data source
+    name: str  # Display name shown in the platform
+    category: DataSourceCategory  # Category this data source belongs to (synced first)
+    description: Optional[str] = None  # Optional summary shown in the platform UI
+    logo_path: Optional[Union[str, Path]] = None  # Local logo file uploaded on first sync
+    logo_url: Optional[str] = None  # Remote logo URL or existing platform logo URL
+    organization_id: Optional[str] = None  # Platform organization UUID (falls back to category value)
+    organization_slug: Optional[str] = None  # Organization slug (falls back to category value)
+    data_source_id: Optional[str] = field(default=None, init=False)  # Set after sync(); platform data source UUID
+
+    def _resolve_organization_id(self) -> str:
+        return resolve_organization_id(
+            organization_id=self.organization_id or self.category.organization_id,
+            organization_slug=self.organization_slug or self.category.organization_slug,
+        )
+
+    def _resolve_logo_url(self) -> str:
+        if self.logo_path and self.logo_url:
+            raise ValueError("Provide only one of logo_path or logo_url")
+        if self.logo_path:
+            return upload_data_source_logo_from_path(self.logo_path)
+        if self.logo_url:
+            if self.logo_url.startswith(("http://", "https://")):
+                return upload_data_source_logo_from_url(self.logo_url)
+            return self.logo_url
+        raise ValueError("Either logo_path or logo_url is required")
+
+    def sync(self) -> str:
+        """Sync this data source (and its category) to the vantage. Returns the data source ID."""
+        org_id = self._resolve_organization_id()
+        self.category.organization_id = self.category.organization_id or org_id
+        category_id = self.category.sync()
+
+        with get_session() as session:
+            existing = get_data_source_id_by_slug(session, org_id, self.slug) is not None
+
+        logo_url = "" if existing else self._resolve_logo_url()
+
+        self.data_source_id = sync_data_source(
+            slug=self.slug,
+            name=self.name,
+            logo_url=logo_url,
+            category_id=category_id,
+            description=self.description,
+            organization_id=org_id,
+        )
+        return self.data_source_id