drc-vantage 1.0.7__tar.gz

drc_vantage-1.0.7/.env.example

@@ -0,0 +1,17 @@
+# PostgreSQL (data warehouse)
+POSTGRES_HOST=localhost
+POSTGRES_PORT=5432
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=pOstgres123!
+POSTGRES_DATABASE=demo
+POSTGRES_DEFAULT_SCHEMA=public
+
+# Vantage configuration
+VANTAGE_SCHEMA=vantage
+GENERATE_AI_VALIDATIONS=false
+
+# MinIO / S3
+S3_ENDPOINT_URL=http://localhost:9000
+S3_ACCESS_KEY=minioadmin
+S3_SECRET_KEY=minioadmin
+S3_BUCKET_NAME=demo

drc_vantage-1.0.7/.gitignore

@@ -0,0 +1,79 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# Dependencies
+**/node_modules/
+**/.pnp
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/versions
+
+# Testing
+**/coverage/
+
+# Next.js
+**/.next/
+**/out/
+
+# Production
+**/build/
+
+# Python
+**/__pycache__/
+**/*.py[cod]
+**/.venv/
+**/dist/
+**/wheels/
+**/*.egg-info/
+**/.ruff_cache/
+**/.pytest_cache/
+
+# TypeScript
+**/*.tsbuildinfo
+**/next-env.d.ts
+
+# Prisma
+**/prisma/generated/
+
+# Environment variables
+.env
+.env.*
+!.env.example
+!.env.*.example
+
+# Debug logs
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# Deployment
+**/.vercel/
+
+# OS
+**/.DS_Store
+**/Thumbs.db
+
+# IDE — ignore personal settings, keep shared workspace config
+**/.vscode/*
+!**/.vscode/settings.json
+!**/.vscode/extensions.json
+!**/.vscode/tasks.json
+!**/.vscode/launch.json
+**/.idea/
+
+# Cursor
+**/.cursor/
+
+# Local themes & branding
+**/src/config/themes/
+**/public/branding/
+
+# Notebooks
+*.ipynb_checkpoints/
+notebook.ipynb
+
+# Secrets
+*.pem

drc_vantage-1.0.7/.python-version

@@ -0,0 +1 @@
+3.14

drc_vantage-1.0.7/PKG-INFO

@@ -0,0 +1,70 @@
+Metadata-Version: 2.4
+Name: drc-vantage
+Version: 1.0.7
+Summary: Python SDK for the DRC Vantage data platform
+Project-URL: Homepage, https://github.com/nathan294/vantage
+Project-URL: Repository, https://github.com/nathan294/vantage
+Project-URL: Issues, https://github.com/nathan294/vantage/issues
+Author: DRC
+License-Expression: LicenseRef-Proprietary
+Keywords: analytics,data,governance,postgresql,sdk,vantage
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: Other/Proprietary License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.13
+Requires-Dist: boto3>=1.43.46
+Requires-Dist: langchain-core>=0.3.0
+Requires-Dist: langchain-openai>=0.3.0
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: numpy>=2.5.1
+Requires-Dist: pillow>=11.0.0
+Requires-Dist: psycopg[binary]>=3.3.4
+Requires-Dist: pydantic>=2.13.4
+Requires-Dist: resend>=2.32.2
+Requires-Dist: scikit-learn>=1.9.0
+Requires-Dist: sqlalchemy>=2.0.51
+Description-Content-Type: text/markdown
+
+# DRC Vantage SDK
+
+Python SDK for the [Vantage](https://github.com/nathan294/vantage) data platform: dataset sync, validation, analysis, KPI computation, and smart cockpit pipelines.
+
+## Requirements
+
+- Python 3.13+
+
+## Installation
+
+```bash
+pip install drc-vantage
+```
+
+## Quick start
+
+```python
+from vantage import Client, Dataset
+
+client = Client(
+    postgres_host="localhost",
+    postgres_user="postgres",
+    postgres_password="secret",
+    postgres_database="vantage",
+)
+
+dataset = Dataset(
+    client=client,
+    location={"schema": "marts", "table": "customers"},
+    dataset_name="Customers",
+    dataset_type="TABLE",
+)
+dataset.sync()
+```
+
+## License
+
+Proprietary. All rights reserved.

drc_vantage-1.0.7/README.md

@@ -0,0 +1,38 @@
+# DRC Vantage SDK
+
+Python SDK for the [Vantage](https://github.com/nathan294/vantage) data platform: dataset sync, validation, analysis, KPI computation, and smart cockpit pipelines.
+
+## Requirements
+
+- Python 3.13+
+
+## Installation
+
+```bash
+pip install drc-vantage
+```
+
+## Quick start
+
+```python
+from vantage import Client, Dataset
+
+client = Client(
+    postgres_host="localhost",
+    postgres_user="postgres",
+    postgres_password="secret",
+    postgres_database="vantage",
+)
+
+dataset = Dataset(
+    client=client,
+    location={"schema": "marts", "table": "customers"},
+    dataset_name="Customers",
+    dataset_type="TABLE",
+)
+dataset.sync()
+```
+
+## License
+
+Proprietary. All rights reserved.

drc_vantage-1.0.7/pyproject.toml

@@ -0,0 +1,44 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "drc-vantage"
+version = "1.0.7"
+description = "Python SDK for the DRC Vantage data platform"
+readme = "README.md"
+requires-python = ">=3.13"
+license = "LicenseRef-Proprietary"
+authors = [{ name = "DRC" }]
+keywords = ["vantage", "data", "sdk", "postgresql", "analytics", "governance"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: Other/Proprietary License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Database",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "boto3>=1.43.46",
+    "langchain-core>=0.3.0",
+    "langchain-openai>=0.3.0",
+    "loguru>=0.7.3",
+    "numpy>=2.5.1",
+    "pillow>=11.0.0",
+    "psycopg[binary]>=3.3.4",
+    "pydantic>=2.13.4",
+    "resend>=2.32.2",
+    "scikit-learn>=1.9.0",
+    "sqlalchemy>=2.0.51",
+]
+
+[project.urls]
+Homepage = "https://github.com/nathan294/vantage"
+Repository = "https://github.com/nathan294/vantage"
+Issues = "https://github.com/nathan294/vantage/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/vantage"]

drc_vantage-1.0.7/src/vantage/__init__.py

@@ -0,0 +1,16 @@
+"""DRC Vantage SDK — platform data pipelines."""
+
+from vantage.client import Base, Client
+from vantage.data_sources.data_source import DataSource
+from vantage.data_sources.data_source_category import DataSourceCategory
+from vantage.data_storage_system.data_storage_system import DataStorageSystem
+from vantage.datasets.dataset import Dataset
+
+__all__ = [
+    "Base",
+    "Client",
+    "DataSource",
+    "DataSourceCategory",
+    "DataStorageSystem",
+    "Dataset",
+]

drc_vantage-1.0.7/src/vantage/client.py

@@ -0,0 +1,126 @@
+"""Vantage SDK client: database access and runtime configuration."""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Any, Generator
+from urllib.parse import quote_plus
+
+from loguru import logger
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+def _get_secret(value: Any) -> str:
+    if hasattr(value, "get_secret_value"):
+        return value.get_secret_value()
+    return str(value)
+
+
+class Client:
+    """Entry point for SDK runtime configuration and database access."""
+
+    def __init__(
+        self,
+        *,
+        vantage_schema: str = "vantage",
+        generate_ai_validations: bool = False,
+        postgres_host: str | None = None,
+        postgres_port: int = 5432,
+        postgres_user: str | None = None,
+        postgres_password: Any | None = None,
+        postgres_database: str = "vantage",
+        openai_api_key: str | None = None,
+        resend_api_key: str | None = None,
+        s3_client: Any | None = None,
+        s3_bucket_name: str | None = None,
+    ) -> None:
+        self.vantage_schema = vantage_schema
+        self.generate_ai_validations = generate_ai_validations
+        self.postgres_host = postgres_host
+        self.postgres_port = postgres_port
+        self.postgres_user = postgres_user
+        self.postgres_password = postgres_password
+        self.postgres_database = postgres_database
+        self.openai_api_key = openai_api_key
+        self.resend_api_key = resend_api_key
+        self.s3_client = s3_client
+        self.s3_bucket_name = s3_bucket_name
+        self._engine: Engine | None = None
+        self._session_maker: sessionmaker[Session] | None = None
+
+    @property
+    def is_postgres_configured(self) -> bool:
+        return bool(self.postgres_host and self.postgres_user and self.postgres_password)
+
+    @property
+    def is_s3_configured(self) -> bool:
+        return self.s3_client is not None and self.s3_bucket_name is not None
+
+    def _build_postgres_url(self, database: str | None = None) -> str:
+        if not self.is_postgres_configured:
+            raise ValueError("PostgreSQL is not configured. Set POSTGRES_* variables in your .env file.")
+
+        db = database or self.postgres_database
+        password = quote_plus(_get_secret(self.postgres_password))
+        user = quote_plus(self.postgres_user)  # type: ignore[arg-type]
+        return f"postgresql+psycopg://{user}:{password}@{self.postgres_host}:{self.postgres_port}/{db}"
+
+    def get_engine(self) -> Engine:
+        if self._engine is None:
+            self._engine = create_engine(self._build_postgres_url(), pool_pre_ping=True)
+        return self._engine
+
+    def get_session_maker(self) -> sessionmaker[Session]:
+        if self._session_maker is None:
+            self._session_maker = sessionmaker(
+                bind=self.get_engine(),
+                class_=Session,
+                expire_on_commit=False,
+            )
+        return self._session_maker
+
+    @contextmanager
+    def get_session(self) -> Generator[Session, None, None]:
+        session = self.get_session_maker()()
+        try:
+            yield session
+            session.commit()
+        except Exception:
+            session.rollback()
+            raise
+        finally:
+            session.close()
+
+    def ensure_database_exists(self) -> None:
+        if not self.is_postgres_configured:
+            logger.warning("Database configuration is not valid, skipping database creation check")
+            return
+
+        engine = create_engine(
+            self._build_postgres_url("postgres"),
+            pool_pre_ping=True,
+            isolation_level="AUTOCOMMIT",
+        )
+
+        try:
+            with engine.connect() as conn:
+                exists = conn.execute(
+                    text("SELECT 1 FROM pg_database WHERE datname = :dbname"),
+                    {"dbname": self.postgres_database},
+                ).scalar()
+
+                if exists:
+                    logger.info(f"Database '{self.postgres_database}' already exists")
+                    return
+
+                logger.info(f"Database '{self.postgres_database}' does not exist, creating it...")
+                conn.execute(text(f'CREATE DATABASE "{self.postgres_database}"'))
+                logger.info(f"Database '{self.postgres_database}' created successfully")
+        finally:
+            engine.dispose()

drc_vantage-1.0.7/src/vantage/core/logo.py

@@ -0,0 +1,157 @@
+"""Prepare and upload data source logos to S3 (mirrors frontend logo pipeline)."""
+
+import io
+import mimetypes
+from pathlib import Path
+from urllib.parse import urlparse
+from urllib.request import Request, urlopen
+from uuid import uuid4
+
+from loguru import logger
+from PIL import Image
+from vantage.client import Client
+from vantage.core.s3 import upload_object
+
+SVG_MIME = "image/svg+xml"
+MAX_INPUT_BYTES = 8 * 1024 * 1024
+LOGO_MAX_DIMENSION = 200
+WEBP_QUALITY = 85
+LOGO_FETCH_USER_AGENT = "demo-flows/0.1 (Foresyn; data source logo sync)"
+
+
+def get_extension_for_logo_content_type(content_type: str) -> str | None:
+    normalized = content_type.split(";")[0].strip().lower()
+    mapping = {
+        "image/webp": "webp",
+        "image/svg+xml": "svg",
+        "image/png": "png",
+        "image/jpeg": "jpg",
+        "image/jpg": "jpg",
+    }
+    return mapping.get(normalized)
+
+
+def prepare_data_source_logo(
+    image_bytes: bytes,
+    *,
+    filename: str | None = None,
+    content_type: str | None = None,
+) -> tuple[bytes, str]:
+    """
+    Prepare a logo for upload.
+
+    Raster images: resize/compress to WebP, max 200×200px.
+    SVG: kept as vector text (no raster compression), basic structure check.
+    """
+    if len(image_bytes) > MAX_INPUT_BYTES:
+        raise ValueError("FILE_TOO_LARGE")
+
+    guessed_type = content_type or (
+        mimetypes.guess_type(filename or "")[0] if filename else None
+    )
+    looks_svg = guessed_type == SVG_MIME or (
+        filename and filename.lower().endswith(".svg")
+    )
+
+    if looks_svg:
+        return _prepare_svg_logo(image_bytes)
+
+    try:
+        return _prepare_raster_logo(image_bytes)
+    except ValueError:
+        raise
+    except Exception as exc:
+        raise ValueError("COMPRESS_FAILED") from exc
+
+
+def upload_data_source_logo(client: Client, image_buffer: bytes, content_type: str) -> str:
+    """
+    Upload a data source logo to S3. Uses a random object name (never the client filename).
+
+    Returns:
+        S3 object key to store in ``DataSource.logoUrl`` (e.g. ``logos/uuid.webp``)
+    """
+    ext = get_extension_for_logo_content_type(content_type)
+    if not ext:
+        raise ValueError(f"Unsupported logo content type: {content_type}")
+
+    normalized_type = content_type.split(";")[0].strip().lower()
+    object_key = f"logos/{uuid4()}.{ext}"
+
+    try:
+        upload_object(
+            client=client,
+            object_key=object_key,
+            body=image_buffer,
+            content_type=normalized_type or f"image/{'jpeg' if ext == 'jpg' else ext}",
+        )
+        logger.info("Uploaded data source logo to %s", object_key)
+        return object_key
+    except Exception as exc:
+        logger.exception("Error uploading data source logo to S3")
+        raise RuntimeError(
+            f"Failed to upload data source logo: {exc if isinstance(exc, Exception) else 'Unknown error'}"
+        ) from exc
+
+
+def upload_data_source_logo_from_path(client: Client, path: str | Path) -> str:
+    """Read a local file, prepare it, upload to S3, and return the object key."""
+    file_path = Path(path)
+    if not file_path.is_file():
+        raise FileNotFoundError(f"Logo file not found: {file_path}")
+
+    raw = file_path.read_bytes()
+    prepared, content_type = prepare_data_source_logo(raw, filename=file_path.name)
+    return upload_data_source_logo(client, prepared, content_type)
+
+
+def upload_data_source_logo_from_url(client: Client, url: str) -> str:
+    """Download a remote image, prepare it, upload to S3, and return the object key."""
+    if not client.is_s3_configured:
+        return url
+
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https"):
+        raise ValueError(f"Invalid logo URL: {url}")
+
+    request = Request(url, headers={"User-Agent": LOGO_FETCH_USER_AGENT})
+    with urlopen(request, timeout=30) as response:
+        raw = response.read()
+        content_type = response.headers.get("Content-Type")
+        filename = Path(parsed.path).name or None
+
+    prepared, content_type = prepare_data_source_logo(
+        raw, filename=filename, content_type=content_type
+    )
+    return upload_data_source_logo(client, prepared, content_type)
+
+
+def _prepare_svg_logo(image_bytes: bytes) -> tuple[bytes, str]:
+    try:
+        text = image_bytes.decode("utf-8")
+    except UnicodeDecodeError as exc:
+        raise ValueError("INVALID_SVG") from exc
+
+    if not text.strip().lstrip().lower().startswith("<svg"):
+        raise ValueError("INVALID_SVG")
+
+    return text.encode("utf-8"), SVG_MIME
+
+
+def _prepare_raster_logo(image_bytes: bytes) -> tuple[bytes, str]:
+    with Image.open(io.BytesIO(image_bytes)) as img:
+        img.thumbnail(
+            (LOGO_MAX_DIMENSION, LOGO_MAX_DIMENSION), Image.Resampling.LANCZOS
+        )
+
+        if img.mode in ("RGBA", "LA") or (
+            img.mode == "P" and "transparency" in img.info
+        ):
+            img = img.convert("RGBA")
+        else:
+            img = img.convert("RGB")
+
+        out = io.BytesIO()
+        img.save(out, format="WEBP", quality=WEBP_QUALITY, method=6)
+
+    return out.getvalue(), "image/webp"

drc_vantage-1.0.7/src/vantage/core/organization.py

@@ -0,0 +1,51 @@
+"""Shared helpers for resolving platform organizations."""
+
+from typing import Optional
+
+from sqlalchemy import text
+
+from vantage.client import Client
+
+
+def get_organization_by_slug(client: Client, organization_slug: str) -> Optional[str]:
+    with client.get_session() as session:
+        organization = session.execute(
+            text(f"SELECT id FROM {client.vantage_schema}.organization WHERE slug = :organization_slug"),
+            {"organization_slug": organization_slug},
+        ).fetchone()
+        return organization[0] if organization else None
+
+
+def get_organization_count(client: Client) -> int:
+    with client.get_session() as session:
+        row = session.execute(text(f"SELECT COUNT(*) FROM {client.vantage_schema}.organization")).fetchone()
+        return int(row[0]) if row else 0
+
+
+def get_single_organization_id_if_only_one_exists(client: Client) -> Optional[str]:
+    with client.get_session() as session:
+        rows = session.execute(text(f"SELECT id FROM {client.vantage_schema}.organization LIMIT 2")).fetchall()
+        if len(rows) == 1:
+            return rows[0][0]
+        return None
+
+
+def resolve_organization_id(client: Client,
+    organization_id: Optional[str] = None,
+    organization_slug: Optional[str] = None,
+) -> str:
+    if organization_slug:
+        org_id = get_organization_by_slug(client, organization_slug=organization_slug)
+        if not org_id:
+            raise ValueError(f"Organization with slug '{organization_slug}' not found")
+        return org_id
+    if organization_id:
+        return organization_id
+    org_id = get_single_organization_id_if_only_one_exists(client)
+    if org_id:
+        return org_id
+    if get_organization_count(client) == 0:
+        raise ValueError("No organization found; provide organization_id or organization_slug")
+    raise ValueError(
+        "Multiple organizations found; provide organization_id or organization_slug"
+    )

drc_vantage-1.0.7/src/vantage/core/s3.py

@@ -0,0 +1,31 @@
+"""S3-compatible object storage client (MinIO, AWS S3, etc.)."""
+
+from typing import Any
+
+from vantage.client import Client
+
+
+def get_s3_client(client: Client) -> Any:
+    if not client.is_s3_configured:
+        raise ValueError("S3 is not configured. Pass s3_client and s3_bucket_name to Client.")
+    return client.s3_client
+
+
+def upload_object(
+    client: Client,
+    object_key: str,
+    body: bytes,
+    content_type: str,
+) -> str:
+    """Upload bytes to the configured bucket. Returns the object key."""
+    if not client.is_s3_configured:
+        raise ValueError("S3 is not configured. Pass s3_client and s3_bucket_name to Client.")
+
+    client.s3_client.put_object(
+        Bucket=client.s3_bucket_name,
+        Key=object_key,
+        Body=body,
+        ContentLength=len(body),
+        ContentType=content_type,
+    )
+    return object_key