phlo-postgres 0.1.0__tar.gz

phlo_postgres-0.1.0/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: phlo-postgres
+Version: 0.1.0
+Summary: Postgres service plugin for Phlo
+Author-email: Phlo Team <team@phlo.dev>
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/plain
+Requires-Dist: phlo>=0.1.0
+Requires-Dist: pyyaml>=6.0.1
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+
+Postgres service plugin for Phlo.

phlo_postgres-0.1.0/README.md

@@ -0,0 +1,65 @@
+# phlo-postgres
+
+PostgreSQL database service for Phlo.
+
+## Description
+
+Core PostgreSQL database for metadata storage, lineage tracking, and operational data. Includes optional Prometheus exporter for database metrics.
+
+## Installation
+
+```bash
+pip install phlo-postgres
+# or
+phlo plugin install postgres
+```
+
+## Configuration
+
+| Variable                 | Default  | Description            |
+| ------------------------ | -------- | ---------------------- |
+| `POSTGRES_PORT`          | `5432`   | PostgreSQL port        |
+| `POSTGRES_USER`          | `phlo`   | Database username      |
+| `POSTGRES_PASSWORD`      | `phlo`   | Database password      |
+| `POSTGRES_DB`            | `phlo`   | Database name          |
+| `POSTGRES_SSL_MODE`      | `prefer` | SSL mode               |
+| `POSTGRES_EXPORTER_PORT` | `9187`   | Postgres exporter port |
+
+## Auto-Configuration
+
+This package is **fully auto-configured**:
+
+| Feature                | How It Works                                               |
+| ---------------------- | ---------------------------------------------------------- |
+| **Grafana Datasource** | Auto-registers as Grafana datasource via labels            |
+| **postgres-exporter**  | Optional Prometheus exporter for native PostgreSQL metrics |
+| **Service Discovery**  | Exporter auto-scraped by Prometheus                        |
+
+### Grafana Labels
+
+```yaml
+compose:
+  labels:
+    phlo.grafana.datasource: "true"
+    phlo.grafana.datasource.type: "postgres"
+    phlo.grafana.datasource.name: "PostgreSQL"
+```
+
+## Usage
+
+```bash
+# Start PostgreSQL
+phlo services start --service postgres
+
+# Start with exporter (for observability)
+phlo services start --service postgres,postgres-exporter
+```
+
+## Endpoints
+
+- **PostgreSQL**: `localhost:5432`
+- **Exporter Metrics**: `http://localhost:9187/metrics`
+
+## Entry Points
+
+- `phlo.plugins.services` - Provides `PostgresServicePlugin` and `PostgresExporterServicePlugin`

phlo_postgres-0.1.0/pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["setuptools>=45", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "phlo-postgres"
+version = "0.1.0"
+description = "Postgres service plugin for Phlo"
+readme = {text = "Postgres service plugin for Phlo.", content-type = "text/plain"}
+requires-python = ">=3.11"
+authors = [
+    {name = "Phlo Team", email = "team@phlo.dev"},
+]
+license = {text = "MIT"}
+dependencies = [
+    "phlo>=0.1.0",
+    "pyyaml>=6.0.1",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "ruff>=0.1.0",
+]
+
+[project.entry-points."phlo.plugins.services"]
+postgres = "phlo_postgres.plugin:PostgresServicePlugin"
+postgres-exporter = "phlo_postgres.plugin:PostgresExporterServicePlugin"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+phlo_postgres = ["service.yaml", "exporter_service.yaml"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"

phlo_postgres-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

phlo_postgres-0.1.0/src/phlo_postgres/__init__.py

@@ -0,0 +1,7 @@
+"""Postgres service plugin package."""
+
+from phlo_postgres.plugin import PostgresServicePlugin
+from phlo_postgres.settings import PostgresSettings, get_settings
+
+__all__ = ["PostgresServicePlugin", "PostgresSettings", "get_settings"]
+__version__ = "0.1.0"

phlo_postgres-0.1.0/src/phlo_postgres/exporter_service.yaml

@@ -0,0 +1,39 @@
+name: postgres-exporter
+description: Prometheus exporter for PostgreSQL metrics
+category: observability
+default: false
+profile: observability
+
+image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0
+
+depends_on:
+  - postgres
+
+compose:
+  restart: unless-stopped
+  labels:
+    phlo.metrics.enabled: "true"
+    phlo.metrics.port: "postgres-exporter:9187"
+    phlo.metrics.path: "/metrics"
+  environment:
+    DATA_SOURCE_NAME: postgresql://${POSTGRES_USER:-phlo}:${POSTGRES_PASSWORD:-phlo}@postgres:5432/${POSTGRES_DB:-phlo}?sslmode=disable
+  ports:
+    - "${POSTGRES_EXPORTER_PORT:-9187}:9187"
+  healthcheck:
+    test:
+      [
+        "CMD",
+        "wget",
+        "--quiet",
+        "--tries=1",
+        "--spider",
+        "http://localhost:9187/metrics",
+      ]
+    interval: 10s
+    timeout: 5s
+    retries: 5
+
+env_vars:
+  POSTGRES_EXPORTER_PORT:
+    default: 9187
+    description: Postgres exporter metrics port

phlo_postgres-0.1.0/src/phlo_postgres/plugin.py

@@ -0,0 +1,47 @@
+"""Postgres service plugin."""
+
+from __future__ import annotations
+
+from importlib import resources
+from typing import Any
+
+import yaml
+from phlo.plugins import PluginMetadata, ServicePlugin
+
+
+class PostgresServicePlugin(ServicePlugin):
+    """Service plugin for Postgres."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        return PluginMetadata(
+            name="postgres",
+            version="0.1.0",
+            description="PostgreSQL database for metadata and operational storage",
+            author="Phlo Team",
+            tags=["core", "database", "postgres"],
+        )
+
+    @property
+    def service_definition(self) -> dict[str, Any]:
+        service_path = resources.files("phlo_postgres").joinpath("service.yaml")
+        return yaml.safe_load(service_path.read_text(encoding="utf-8"))
+
+
+class PostgresExporterServicePlugin(ServicePlugin):
+    """Service plugin for Postgres Prometheus exporter."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        return PluginMetadata(
+            name="postgres-exporter",
+            version="0.1.0",
+            description="Prometheus exporter for PostgreSQL metrics",
+            author="Phlo Team",
+            tags=["observability", "metrics", "postgres"],
+        )
+
+    @property
+    def service_definition(self) -> dict[str, Any]:
+        service_path = resources.files("phlo_postgres").joinpath("exporter_service.yaml")
+        return yaml.safe_load(service_path.read_text(encoding="utf-8"))

phlo_postgres-0.1.0/src/phlo_postgres/service.yaml

@@ -0,0 +1,60 @@
+name: postgres
+description: PostgreSQL database for metadata and operational storage
+category: core
+default: true
+
+image: postgres:16-alpine
+
+compose:
+  restart: unless-stopped
+  labels:
+    phlo.metrics.enabled: "false"
+    phlo.grafana.datasource: "true"
+    phlo.grafana.datasource.type: "postgres"
+    phlo.grafana.datasource.name: "PostgreSQL"
+    phlo.grafana.datasource.url: "postgres:5432"
+    phlo.grafana.datasource.database: "${POSTGRES_DB:-phlo}"
+  environment:
+    POSTGRES_USER: ${POSTGRES_USER:-phlo}
+    POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-phlo}
+    POSTGRES_DB: ${POSTGRES_DB:-phlo}
+    # SSL/TLS
+    POSTGRES_SSL_MODE: ${POSTGRES_SSL_MODE:-prefer}
+  ports:
+    - "${POSTGRES_PORT:-5432}:5432"
+  volumes:
+    - ./volumes/postgres:/var/lib/postgresql/data
+    - ./volumes/postgres-certs:/var/lib/postgresql/certs:ro
+  healthcheck:
+    test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-phlo}"]
+    interval: 10s
+    timeout: 5s
+    retries: 10
+
+env_vars:
+  POSTGRES_USER:
+    default: phlo
+    description: PostgreSQL username
+  POSTGRES_PASSWORD:
+    default: phlo
+    description: PostgreSQL password
+    secret: true
+  POSTGRES_DB:
+    default: phlo
+    description: PostgreSQL database name
+  POSTGRES_PORT:
+    default: 5432
+    description: PostgreSQL host port
+  # SSL/TLS
+  POSTGRES_SSL_MODE:
+    default: "prefer"
+    description: "SSL mode: disable, allow, prefer, require, verify-ca, verify-full"
+  POSTGRES_SSL_CERT_FILE:
+    default: ""
+    description: "Path to SSL certificate file"
+  POSTGRES_SSL_KEY_FILE:
+    default: ""
+    description: "Path to SSL private key file"
+  POSTGRES_SSL_CA_FILE:
+    default: ""
+    description: "Path to SSL CA certificate file"

phlo_postgres-0.1.0/src/phlo_postgres/settings.py

@@ -0,0 +1,34 @@
+"""Postgres settings."""
+
+from __future__ import annotations
+
+from functools import lru_cache
+from urllib.parse import quote_plus
+
+from pydantic import Field
+
+from phlo.config.base import BaseConfig
+
+
+class PostgresSettings(BaseConfig):
+    """PostgreSQL database connection and schema configuration."""
+
+    postgres_host: str = Field(default="postgres", description="PostgreSQL host")
+    postgres_port: int = Field(default=10000, description="PostgreSQL port")
+    postgres_user: str = Field(default="lake", description="PostgreSQL username")
+    postgres_password: str = Field(default="phlo", description="PostgreSQL password")
+    postgres_db: str = Field(default="lakehouse", description="PostgreSQL database name")
+    postgres_mart_schema: str = Field(
+        default="marts", description="Schema for published mart tables"
+    )
+
+    def get_postgres_connection_string(self, include_db: bool = True) -> str:
+        db_part = f"/{self.postgres_db}" if include_db else ""
+        user = quote_plus(self.postgres_user)
+        password = quote_plus(self.postgres_password)
+        return f"postgresql://{user}:{password}@{self.postgres_host}:{self.postgres_port}{db_part}"
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> PostgresSettings:
+    return PostgresSettings()

phlo_postgres-0.1.0/src/phlo_postgres.egg-info/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: phlo-postgres
+Version: 0.1.0
+Summary: Postgres service plugin for Phlo
+Author-email: Phlo Team <team@phlo.dev>
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/plain
+Requires-Dist: phlo>=0.1.0
+Requires-Dist: pyyaml>=6.0.1
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+
+Postgres service plugin for Phlo.

phlo_postgres-0.1.0/src/phlo_postgres.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+README.md
+pyproject.toml
+src/phlo_postgres/__init__.py
+src/phlo_postgres/exporter_service.yaml
+src/phlo_postgres/plugin.py
+src/phlo_postgres/service.yaml
+src/phlo_postgres/settings.py
+src/phlo_postgres.egg-info/PKG-INFO
+src/phlo_postgres.egg-info/SOURCES.txt
+src/phlo_postgres.egg-info/dependency_links.txt
+src/phlo_postgres.egg-info/entry_points.txt
+src/phlo_postgres.egg-info/requires.txt
+src/phlo_postgres.egg-info/top_level.txt
+tests/test_integration_postgres.py
+tests/test_postgres_plugin.py

phlo_postgres-0.1.0/src/phlo_postgres.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

phlo_postgres-0.1.0/src/phlo_postgres.egg-info/entry_points.txt

@@ -0,0 +1,3 @@
+[phlo.plugins.services]
+postgres = phlo_postgres.plugin:PostgresServicePlugin
+postgres-exporter = phlo_postgres.plugin:PostgresExporterServicePlugin

phlo_postgres-0.1.0/src/phlo_postgres.egg-info/requires.txt

@@ -0,0 +1,6 @@
+phlo>=0.1.0
+pyyaml>=6.0.1
+
+[dev]
+pytest>=7.0
+ruff>=0.1.0

phlo_postgres-0.1.0/src/phlo_postgres.egg-info/top_level.txt

@@ -0,0 +1 @@
+phlo_postgres

phlo_postgres-0.1.0/tests/test_integration_postgres.py

@@ -0,0 +1,295 @@
+"""Comprehensive integration tests for phlo-postgres.
+
+Per TEST_STRATEGY.md Level 2 (Functional):
+- Connection String Building: Test connection string construction
+- DB Ops: Create user, create DB, verify connectivity
+- Service Plugin: Verify plugin registration
+"""
+
+from unittest.mock import patch, MagicMock
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+# =============================================================================
+# Connection Configuration Tests
+# =============================================================================
+
+
+class TestPostgresConfiguration:
+    """Test Postgres configuration and connection building."""
+
+    def test_postgres_config_accessible(self):
+        """Test Postgres configuration is accessible from phlo_postgres.settings."""
+        from phlo_postgres.settings import get_settings
+
+        settings = get_settings()
+
+        assert hasattr(settings, "postgres_host")
+        assert hasattr(settings, "postgres_port")
+        assert hasattr(settings, "postgres_db")
+        assert hasattr(settings, "postgres_user")
+
+    def test_postgres_config_has_defaults(self):
+        """Test Postgres config has sensible defaults."""
+        from phlo_postgres.settings import get_settings
+
+        settings = get_settings()
+
+        assert settings.postgres_host is not None
+        assert settings.postgres_port > 0
+        assert settings.postgres_db is not None
+
+    def test_connection_string_building(self):
+        """Test building a connection string from config."""
+        from phlo_postgres.settings import get_settings
+
+        settings = get_settings()
+
+        # Build connection string
+        conn_str = (
+            f"postgresql://{settings.postgres_user}:{settings.postgres_password}"
+            f"@{settings.postgres_host}:{settings.postgres_port}/{settings.postgres_db}"
+        )
+
+        assert "postgresql://" in conn_str
+        assert str(settings.postgres_port) in conn_str
+
+
+# =============================================================================
+# Service Plugin Tests
+# =============================================================================
+
+
+class TestPostgresServicePlugin:
+    """Test Postgres service plugin."""
+
+    def test_plugin_initializes(self):
+        """Test PostgresServicePlugin can be instantiated."""
+        from phlo_postgres.plugin import PostgresServicePlugin
+
+        plugin = PostgresServicePlugin()
+        assert plugin is not None
+
+    def test_plugin_metadata(self):
+        """Test plugin metadata is correctly defined."""
+        from phlo_postgres.plugin import PostgresServicePlugin
+
+        plugin = PostgresServicePlugin()
+        metadata = plugin.metadata
+
+        assert metadata.name == "postgres"
+        assert metadata.version is not None
+
+    def test_service_definition_loads(self):
+        """Test service definition YAML can be loaded."""
+        from phlo_postgres.plugin import PostgresServicePlugin
+
+        plugin = PostgresServicePlugin()
+        service_def = plugin.service_definition
+
+        assert isinstance(service_def, dict)
+        # Service definitions have flat structure with 'name' and 'compose' keys
+        assert "name" in service_def or "compose" in service_def
+
+    def test_service_definition_has_image(self):
+        """Test service definition has container image specified."""
+        from phlo_postgres.plugin import PostgresServicePlugin
+
+        plugin = PostgresServicePlugin()
+        service_def = plugin.service_definition
+
+        # Navigate to find image
+        services = service_def.get("services", {})
+        if services:
+            # Should have postgres service with image
+            for svc_name, svc_config in services.items():
+                if "postgres" in svc_name.lower():
+                    assert "image" in svc_config or "build" in svc_config
+
+
+# =============================================================================
+# Psycopg2 Connection Tests (with mocks)
+# =============================================================================
+
+
+class TestPostgresConnectionMocked:
+    """Test Postgres connections with mocks."""
+
+    def test_psycopg2_connect_with_config(self):
+        """Test psycopg2 connection using config values."""
+        import psycopg2
+        from phlo_postgres.settings import get_settings
+
+        settings = get_settings()
+
+        mock_conn = MagicMock()
+
+        with patch("psycopg2.connect", return_value=mock_conn) as mock_connect:
+            # Simulate what most code does
+            psycopg2.connect(
+                host=settings.postgres_host,
+                port=settings.postgres_port,
+                database=settings.postgres_db,
+                user=settings.postgres_user,
+                password=settings.postgres_password,
+            )
+
+            mock_connect.assert_called_once()
+            # Verify kwargs
+            call_kwargs = mock_connect.call_args.kwargs
+            assert call_kwargs["host"] == settings.postgres_host
+            assert call_kwargs["database"] == settings.postgres_db
+
+    def test_connection_with_cursor_context(self):
+        """Test connection with cursor context manager."""
+        mock_cursor = MagicMock()
+        mock_cursor.fetchall.return_value = [(1,)]
+
+        mock_conn = MagicMock()
+        mock_conn.cursor.return_value.__enter__ = lambda _: mock_cursor
+        mock_conn.cursor.return_value.__exit__ = MagicMock(return_value=False)
+
+        with patch("psycopg2.connect", return_value=mock_conn):
+            import psycopg2
+
+            conn = psycopg2.connect(host="localhost", database="test")
+            with conn.cursor() as cur:
+                cur.execute("SELECT 1")
+                result = cur.fetchall()
+
+            assert result == [(1,)]
+
+
+# =============================================================================
+# Functional Integration Tests (Real Postgres if available)
+# =============================================================================
+
+
+@pytest.fixture
+def postgres_connection():
+    """Fixture providing a real Postgres connection if available."""
+    from phlo_postgres.settings import get_settings
+    import psycopg2
+
+    settings = get_settings()
+
+    try:
+        conn = psycopg2.connect(
+            host=settings.postgres_host,
+            port=settings.postgres_port,
+            database=settings.postgres_db,
+            user=settings.postgres_user,
+            password=settings.postgres_password,
+            connect_timeout=5,
+        )
+        yield conn
+        conn.close()
+    except Exception as e:
+        pytest.skip(f"Postgres not available: {e}")
+
+
+class TestPostgresIntegrationReal:
+    """Real integration tests against a running Postgres instance."""
+
+    def test_simple_query(self, postgres_connection):
+        """Test simple query execution against real Postgres."""
+        with postgres_connection.cursor() as cur:
+            cur.execute("SELECT 1 AS one")
+            result = cur.fetchall()
+
+        assert result == [(1,)]
+
+    def test_version_query(self, postgres_connection):
+        """Test querying Postgres version."""
+        with postgres_connection.cursor() as cur:
+            cur.execute("SELECT version()")
+            result = cur.fetchone()
+
+        assert result is not None
+        assert "PostgreSQL" in result[0]
+
+    def test_create_temp_table(self, postgres_connection):
+        """Test creating and querying a temporary table."""
+        with postgres_connection.cursor() as cur:
+            cur.execute("""
+                CREATE TEMP TABLE test_table (
+                    id SERIAL PRIMARY KEY,
+                    name VARCHAR(100)
+                )
+            """)
+            cur.execute("INSERT INTO test_table (name) VALUES ('test1'), ('test2')")
+            cur.execute("SELECT COUNT(*) FROM test_table")
+            result = cur.fetchone()
+
+        assert result[0] == 2
+
+    def test_list_databases(self, postgres_connection):
+        """Test listing databases."""
+        with postgres_connection.cursor() as cur:
+            cur.execute("""
+                SELECT datname FROM pg_database
+                WHERE datistemplate = false
+            """)
+            databases = [row[0] for row in cur.fetchall()]
+
+        # Should have at least the connected database
+        assert len(databases) >= 1
+
+    def test_list_tables(self, postgres_connection):
+        """Test listing tables in public schema."""
+        with postgres_connection.cursor() as cur:
+            cur.execute("""
+                SELECT table_name
+                FROM information_schema.tables
+                WHERE table_schema = 'public'
+            """)
+            tables = [row[0] for row in cur.fetchall()]
+
+        # May be empty if no tables
+        assert isinstance(tables, list)
+
+    def test_transaction_rollback(self, postgres_connection):
+        """Test transaction rollback."""
+        postgres_connection.autocommit = False
+
+        with postgres_connection.cursor() as cur:
+            cur.execute("CREATE TEMP TABLE rollback_test (id INT)")
+            cur.execute("INSERT INTO rollback_test VALUES (1)")
+
+        # Rollback
+        postgres_connection.rollback()
+
+        # Table should not exist after rollback
+        with postgres_connection.cursor() as cur:
+            cur.execute("""
+                SELECT COUNT(*) FROM information_schema.tables
+                WHERE table_name = 'rollback_test'
+            """)
+            result = cur.fetchone()
+
+        # This may vary based on temp table behavior
+        assert result is not None
+
+
+# =============================================================================
+# Version and Export Tests
+# =============================================================================
+
+
+class TestPostgresExports:
+    """Test module exports and version."""
+
+    def test_plugin_importable(self):
+        """Test PostgresServicePlugin is importable."""
+        from phlo_postgres.plugin import PostgresServicePlugin
+
+        assert PostgresServicePlugin is not None
+
+    def test_module_importable(self):
+        """Test phlo_postgres module is importable."""
+        import phlo_postgres
+
+        assert phlo_postgres is not None

phlo_postgres-0.1.0/tests/test_postgres_plugin.py

@@ -0,0 +1,11 @@
+"""Tests for Postgres service plugin."""
+
+from phlo_postgres.plugin import PostgresServicePlugin
+
+
+def test_postgres_service_definition():
+    plugin = PostgresServicePlugin()
+    service_definition = plugin.service_definition
+
+    assert service_definition["name"] == "postgres"
+    assert service_definition["category"] == "core"