phlo-clickhouse 0.1.0__tar.gz

phlo_clickhouse-0.1.0/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.4
+Name: phlo-clickhouse
+Version: 0.1.0
+Summary: ClickHouse service and resource plugin for Phlo
+Author-email: Phlo Team <team@phlo.dev>
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/plain
+Requires-Dist: phlo>=0.1.0
+Requires-Dist: clickhouse-connect>=0.8.0
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pyarrow>=12.0.0
+Provides-Extra: dbt
+Requires-Dist: dbt-core>=1.8; extra == "dbt"
+Requires-Dist: dbt-clickhouse>=1.8; extra == "dbt"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+
+ClickHouse service and resource plugin for Phlo.

phlo_clickhouse-0.1.0/README.md

@@ -0,0 +1,72 @@
+# phlo-clickhouse
+
+ClickHouse service and resource plugin for Phlo.
+
+## Overview
+
+`phlo-clickhouse` provides ClickHouse as a combined `table_store`, `query_engine`, and `publish_target` capability in Phlo. Unlike the existing bundled stack (DLT -> Iceberg -> Trino/dbt -> Postgres), ClickHouse can serve all three data plane roles in a single service.
+
+## Installation
+
+```bash
+pip install phlo-clickhouse
+```
+
+## Usage
+
+### Starting ClickHouse
+
+```bash
+phlo services start --service clickhouse
+```
+
+This starts both the ClickHouse server and the setup container that creates the default databases (`raw`, `staging`, `curated`, `marts`).
+
+### Running Queries
+
+```bash
+phlo clickhouse query "SELECT version()"
+phlo clickhouse query --file query.sql
+```
+
+### Checking Status
+
+```bash
+phlo clickhouse status
+```
+
+## Configuration
+
+The following environment variables can be used to configure ClickHouse:
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `CLICKHOUSE_VERSION` | `latest` | ClickHouse server version tag |
+| `CLICKHOUSE_HTTP_PORT` | `8123` | ClickHouse HTTP interface port |
+| `CLICKHOUSE_NATIVE_PORT` | `19000` | ClickHouse native protocol port |
+| `CLICKHOUSE_METRICS_PORT` | `9363` | ClickHouse Prometheus metrics port |
+| `CLICKHOUSE_USER` | `default` | ClickHouse default username |
+| `CLICKHOUSE_PASSWORD` | | ClickHouse default user password |
+| `CLICKHOUSE_DB` | `default` | Default ClickHouse database |
+
+## Capabilities
+
+This plugin registers the following capabilities:
+
+- **Table Store**: ClickHouse MergeTree engine
+- **Query Engine**: ClickHouse SQL
+- **Publish Target**: ClickHouse marts database
+
+## dbt Integration
+
+Install with dbt support:
+
+```bash
+pip install phlo-clickhouse[dbt]
+```
+
+This provides the `dbt-clickhouse` adapter for running dbt transforms against ClickHouse.
+
+## License
+
+MIT

phlo_clickhouse-0.1.0/pyproject.toml

@@ -0,0 +1,55 @@
+[build-system]
+requires = ["setuptools>=45", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "phlo-clickhouse"
+version = "0.1.0"
+description = "ClickHouse service and resource plugin for Phlo"
+readme = {text = "ClickHouse service and resource plugin for Phlo.", content-type = "text/plain"}
+requires-python = ">=3.11"
+authors = [
+    {name = "Phlo Team", email = "team@phlo.dev"},
+]
+license = {text = "MIT"}
+dependencies = [
+    "phlo>=0.1.0",
+    "clickhouse-connect>=0.8.0",
+    "pyyaml>=6.0.1",
+    "pandas>=2.0.0",
+    "pyarrow>=12.0.0",
+]
+
+[project.optional-dependencies]
+dbt = [
+    "dbt-core>=1.8",
+    "dbt-clickhouse>=1.8",
+]
+dev = [
+    "pytest>=7.0",
+    "ruff>=0.1.0",
+]
+
+[project.entry-points."phlo.plugins.services"]
+clickhouse = "phlo_clickhouse.plugin:ClickHouseServicePlugin"
+clickhouse-setup = "phlo_clickhouse.plugin:ClickHouseSetupServicePlugin"
+
+[project.entry-points."phlo.plugins.resources"]
+clickhouse = "phlo_clickhouse.plugin:ClickHouseResourceProvider"
+
+[project.entry-points."phlo.plugins.cli"]
+clickhouse = "phlo_clickhouse.cli_plugin:ClickHouseCliPlugin"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+phlo_clickhouse = ["service.yaml", "clickhouse-setup.yaml"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"

phlo_clickhouse-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

phlo_clickhouse-0.1.0/src/phlo_clickhouse/__init__.py

@@ -0,0 +1,19 @@
+"""ClickHouse service and resource plugin package."""
+
+from phlo_clickhouse.plugin import (
+    ClickHouseResourceProvider,
+    ClickHouseServicePlugin,
+    ClickHouseSetupServicePlugin,
+)
+from phlo_clickhouse.resource import ClickHouseResource
+from phlo_clickhouse.settings import ClickHouseSettings, get_settings
+
+__all__ = [
+    "ClickHouseResource",
+    "ClickHouseResourceProvider",
+    "ClickHouseServicePlugin",
+    "ClickHouseSetupServicePlugin",
+    "ClickHouseSettings",
+    "get_settings",
+]
+__version__ = "0.1.0"

phlo_clickhouse-0.1.0/src/phlo_clickhouse/cli.py

@@ -0,0 +1,154 @@
+"""CLI commands for the ClickHouse data plane service."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from shutil import which
+from subprocess import TimeoutExpired
+
+import click
+
+from phlo.cli.infrastructure.command import CommandError, run_command
+from phlo.cli.infrastructure.compose import compose_base_cmd
+from phlo.cli.infrastructure.utils import get_project_name
+from phlo.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def _read_query(*, query: str | None, file: Path | None) -> str:
+    """Return SQL text from inline query or file input."""
+    if query and file:
+        raise click.ClickException("Use either an inline query or --file, not both.")
+    if file is not None:
+        try:
+            sql = file.read_text(encoding="utf-8")
+        except OSError as exc:
+            raise click.ClickException(f"Failed to read SQL file: {file}") from exc
+        if sql.strip():
+            return sql
+        raise click.ClickException(f"SQL file is empty: {file}")
+    if query and query.strip():
+        return query
+    raise click.ClickException("Provide a SQL query argument or --file.")
+
+
+def _ensure_phlo_dir() -> Path:
+    """Return the local .phlo directory or exit with a clear error."""
+    phlo_dir = Path.cwd() / ".phlo"
+    if phlo_dir.exists():
+        return phlo_dir
+    raise click.ClickException(".phlo directory not found. Run 'phlo services init' first.")
+
+
+def _require_docker() -> None:
+    """Validate that Docker is installed and responsive."""
+    if which("docker") is None:
+        raise click.ClickException("docker command not found.")
+    try:
+        result = run_command(
+            ["docker", "info"],
+            timeout_seconds=10,
+            capture_output=True,
+            check=False,
+        )
+    except TimeoutExpired as exc:
+        raise click.ClickException("docker info timed out.") from exc
+    if result.returncode == 0:
+        return
+    raise click.ClickException("Docker is not running.")
+
+
+@click.group(name="clickhouse")
+def clickhouse_group() -> None:
+    """Query and inspect the ClickHouse data plane service."""
+
+
+@clickhouse_group.command(name="query")
+@click.argument("query", required=False)
+@click.option(
+    "--file",
+    "query_file",
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+)
+@click.option("--format", "output_format", default="TabSeparatedRaw", show_default=True)
+@click.option("--timeout", "timeout_seconds", default=30, show_default=True, type=int)
+def clickhouse_query(
+    query: str | None,
+    query_file: Path | None,
+    output_format: str,
+    timeout_seconds: int,
+) -> None:
+    """Execute a SQL query against the running ClickHouse service."""
+    _require_docker()
+    phlo_dir = _ensure_phlo_dir()
+    project_name = get_project_name()
+    sql = _read_query(query=query, file=query_file)
+
+    cmd = compose_base_cmd(phlo_dir=phlo_dir, project_name=project_name)
+    cmd.extend(
+        [
+            "exec",
+            "-T",
+            "clickhouse",
+            "clickhouse-client",
+            "--multiquery",
+            "--format",
+            output_format,
+            "--query",
+            sql,
+        ]
+    )
+
+    try:
+        result = run_command(
+            cmd,
+            timeout_seconds=timeout_seconds,
+            capture_output=True,
+            check=True,
+        )
+    except CommandError as exc:
+        stderr = exc.stderr.strip()
+        raise click.ClickException(stderr or str(exc)) from exc
+    except TimeoutExpired as exc:
+        raise click.ClickException(f"Query timed out after {timeout_seconds} seconds.") from exc
+
+    if result.stdout:
+        click.echo(result.stdout, nl=False)
+
+
+@clickhouse_group.command(name="status")
+def clickhouse_status() -> None:
+    """Show ClickHouse service status and basic server info."""
+    _require_docker()
+    phlo_dir = _ensure_phlo_dir()
+    project_name = get_project_name()
+
+    cmd = compose_base_cmd(phlo_dir=phlo_dir, project_name=project_name)
+    cmd.extend(
+        [
+            "exec",
+            "-T",
+            "clickhouse",
+            "clickhouse-client",
+            "--query",
+            "SELECT version() AS version, uptime() AS uptime_seconds, "
+            "currentDatabase() AS current_database",
+        ]
+    )
+
+    try:
+        result = run_command(
+            cmd,
+            timeout_seconds=10,
+            capture_output=True,
+            check=True,
+        )
+    except CommandError as exc:
+        stderr = exc.stderr.strip()
+        raise click.ClickException(stderr or str(exc)) from exc
+    except TimeoutExpired as exc:
+        raise click.ClickException("Status check timed out.") from exc
+
+    if result.stdout:
+        click.echo(result.stdout, nl=False)

phlo_clickhouse-0.1.0/src/phlo_clickhouse/cli_plugin.py

@@ -0,0 +1,24 @@
+"""CLI plugin for ClickHouse commands."""
+
+from __future__ import annotations
+
+import click
+
+from phlo.plugins.base import CliCommandPlugin, PluginMetadata
+
+from phlo_clickhouse.cli import clickhouse_group
+
+
+class ClickHouseCliPlugin(CliCommandPlugin):
+    """Register ClickHouse CLI commands."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        return PluginMetadata(
+            name="clickhouse",
+            version="0.1.0",
+            description="CLI commands for ClickHouse data plane access",
+        )
+
+    def get_cli_commands(self) -> list[click.Command]:
+        return [clickhouse_group]

phlo_clickhouse-0.1.0/src/phlo_clickhouse/clickhouse-setup.yaml

@@ -0,0 +1,30 @@
+# Companion service for clickhouse - creates required databases
+name: clickhouse-setup
+description: Initialize ClickHouse databases for data plane
+category: data
+default: false
+
+image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-latest}
+
+depends_on:
+  - clickhouse
+
+compose:
+  restart: "no"
+  entrypoint: >
+    /bin/sh -c "
+    until clickhouse-client --host clickhouse --user $${CLICKHOUSE_USER:-default}
+    --password $${CLICKHOUSE_PASSWORD:-}
+    --query 'SELECT 1' 2>/dev/null; do
+    echo 'Waiting for ClickHouse...' && sleep 2;
+    done &&
+    clickhouse-client --host clickhouse --user $${CLICKHOUSE_USER:-default}
+    --password $${CLICKHOUSE_PASSWORD:-}
+    --multiquery --query '
+    CREATE DATABASE IF NOT EXISTS raw;
+    CREATE DATABASE IF NOT EXISTS staging;
+    CREATE DATABASE IF NOT EXISTS curated;
+    CREATE DATABASE IF NOT EXISTS marts;
+    ' &&
+    echo 'ClickHouse databases created successfully'
+    "

phlo_clickhouse-0.1.0/src/phlo_clickhouse/plugin.py

@@ -0,0 +1,145 @@
+"""ClickHouse service and resource provider plugins."""
+
+from __future__ import annotations
+
+from importlib import resources
+from time import perf_counter
+from typing import Any
+
+import yaml
+
+from phlo.capabilities import (
+    CapabilitySupport,
+    PublishTargetSpec,
+    ResourceSpec,
+    TableStoreSpec,
+)
+from phlo.capabilities.specs import QueryEngineSpec
+from phlo.logging import get_logger
+from phlo.plugins import PluginMetadata, ResourceProviderPlugin, ServicePlugin
+from phlo_clickhouse.publish_target import ClickHousePublishTarget
+from phlo_clickhouse.resource import CLICKHOUSE_QUERY_ENGINE_SUPPORT, ClickHouseResource
+from phlo_clickhouse.settings import get_settings as get_clickhouse_settings
+
+logger = get_logger(__name__)
+
+
+def _load_service_definition(resource_name: str, service_name: str) -> dict[str, Any]:
+    start = perf_counter()
+    logger.info(
+        "clickhouse_service_definition_load_started",
+        service_name=service_name,
+        resource_name=resource_name,
+    )
+    service_path = resources.files("phlo_clickhouse").joinpath(resource_name)
+    try:
+        data = yaml.safe_load(service_path.read_text(encoding="utf-8"))
+    except Exception:
+        logger.error(
+            "clickhouse_service_definition_load_failed",
+            service_name=service_name,
+            resource_name=resource_name,
+            elapsed_ms=round((perf_counter() - start) * 1000, 2),
+            exc_info=True,
+        )
+        raise
+
+    service_count = len(data.get("services", {})) if isinstance(data, dict) else None
+    logger.info(
+        "clickhouse_service_definition_load_completed",
+        service_name=service_name,
+        resource_name=resource_name,
+        elapsed_ms=round((perf_counter() - start) * 1000, 2),
+        service_count=service_count,
+    )
+    return data
+
+
+class ClickHouseServicePlugin(ServicePlugin):
+    """Service plugin for ClickHouse."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        return PluginMetadata(
+            name="clickhouse",
+            version="0.1.0",
+            description="ClickHouse analytical database for data plane",
+            author="Phlo Team",
+            tags=["data", "query", "storage"],
+        )
+
+    @property
+    def service_definition(self) -> dict[str, Any]:
+        return _load_service_definition("service.yaml", "clickhouse")
+
+
+class ClickHouseSetupServicePlugin(ServicePlugin):
+    """Service plugin for ClickHouse database initialization."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        return PluginMetadata(
+            name="clickhouse-setup",
+            version="0.1.0",
+            description="Initialize ClickHouse databases for data plane",
+            author="Phlo Team",
+            tags=["data", "bootstrap"],
+        )
+
+    @property
+    def service_definition(self) -> dict[str, Any]:
+        return _load_service_definition("clickhouse-setup.yaml", "clickhouse-setup")
+
+
+class ClickHouseResourceProvider(ResourceProviderPlugin):
+    """Resource provider plugin for ClickHouse."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        return PluginMetadata(
+            name="clickhouse",
+            version="0.1.0",
+            description="ClickHouse resource for Phlo",
+            support=CapabilitySupport(),
+        )
+
+    def get_resources(self) -> list[ResourceSpec]:
+        return [ResourceSpec(name="clickhouse", resource=ClickHouseResource())]
+
+    def get_table_stores(self) -> list[TableStoreSpec]:
+        return [
+            TableStoreSpec(
+                name="clickhouse",
+                provider=ClickHouseResource(),
+                support=CapabilitySupport(
+                    supports_snapshots=False,
+                    supports_schema_evolution=True,
+                ),
+            )
+        ]
+
+    def get_query_engines(self) -> list[QueryEngineSpec]:
+        settings = get_clickhouse_settings()
+        return [
+            QueryEngineSpec(
+                name="clickhouse",
+                provider=ClickHouseResource(),
+                metadata={
+                    "host": settings.clickhouse_host,
+                    "port": settings.clickhouse_http_port,
+                    "native_port": settings.clickhouse_native_port,
+                    "default_database": settings.clickhouse_db,
+                    "service_type": "ClickHouse",
+                },
+                support=CLICKHOUSE_QUERY_ENGINE_SUPPORT,
+            )
+        ]
+
+    def get_publish_targets(self) -> list[PublishTargetSpec]:
+        return [
+            PublishTargetSpec(
+                name="clickhouse",
+                provider=ClickHousePublishTarget(),
+                metadata={"target_system": "clickhouse", "role": "serving"},
+            )
+        ]

phlo_clickhouse-0.1.0/src/phlo_clickhouse/publish_target.py

@@ -0,0 +1,16 @@
+"""ClickHouse publish target for mart publishing."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from phlo_clickhouse.resource import ClickHouseResource
+
+
+@dataclass
+class ClickHousePublishTarget:
+    """Publish target backed by ClickHouse."""
+
+    resource: ClickHouseResource = field(default_factory=ClickHouseResource)
+    target_system: str = "clickhouse"
+    default_schema: str = "marts"

phlo_clickhouse-0.1.0/src/phlo_clickhouse/resource.py

@@ -0,0 +1,239 @@
+"""ClickHouse resource for executing queries."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+import time
+from typing import TYPE_CHECKING, Any, Iterable
+
+import clickhouse_connect
+import pandas as pd
+
+from phlo.capabilities import CapabilitySupport
+from phlo.logging import get_logger
+from phlo_clickhouse.settings import get_settings as get_clickhouse_settings
+
+if TYPE_CHECKING:
+    from clickhouse_connect.driver import Client
+
+logger = get_logger(__name__)
+
+CLICKHOUSE_QUERY_ENGINE_SUPPORT = CapabilitySupport(
+    supports_snapshots=False,
+    supports_time_travel=False,
+)
+
+
+@dataclass
+class ClickHouseResource:
+    """Resource wrapper for ClickHouse connections and query execution."""
+
+    host: str | None = None
+    port: int | None = None
+    user: str | None = None
+    password: str | None = None
+    database: str | None = None
+    secure: bool | None = None
+
+    def _settings(self):
+        return get_clickhouse_settings()
+
+    def get_client(self) -> "Client":
+        """Create and return a ClickHouse client."""
+        settings = self._settings()
+        return clickhouse_connect.get_client(
+            host=self.host or settings.clickhouse_host,
+            port=self.port or settings.clickhouse_http_port,
+            username=self.user or settings.clickhouse_user,
+            password=self.password or settings.clickhouse_password,
+            database=self.database or settings.clickhouse_db,
+            secure=self.secure if self.secure is not None else settings.clickhouse_secure,
+        )
+
+    def execute(self, sql: str, params: Iterable[object] | None = None) -> list[list[Any]]:
+        """Execute SQL and return query results."""
+        client = self.get_client()
+        try:
+            result = client.query(sql, parameters=list(params or []))
+            return result.result_rows
+        finally:
+            client.close()
+
+    def command(self, sql: str) -> Any:
+        """Execute a command (DDL/DML) that returns a single value or None."""
+        client = self.get_client()
+        try:
+            return client.command(sql)
+        finally:
+            client.close()
+
+    def wait_ready(
+        self,
+        *,
+        timeout: float = 60.0,
+        interval: float = 1.0,
+    ) -> None:
+        """Wait for ClickHouse to accept queries."""
+        deadline = time.monotonic() + timeout
+        last_error: Exception | None = None
+        interval = max(interval, 0.0)
+        settings = self._settings()
+        while time.monotonic() < deadline:
+            try:
+                self.command("SELECT 1")
+                logger.info(
+                    "clickhouse_wait_ready_succeeded",
+                    host=self.host or settings.clickhouse_host,
+                    port=self.port or settings.clickhouse_http_port,
+                )
+                return
+            except Exception as exc:  # noqa: BLE001
+                last_error = exc
+                logger.debug(
+                    "clickhouse_wait_ready_retry",
+                    host=self.host or settings.clickhouse_host,
+                    port=self.port or settings.clickhouse_http_port,
+                    retry_interval_seconds=interval,
+                )
+                time.sleep(interval)
+        logger.error(
+            "clickhouse_wait_ready_timeout",
+            host=self.host or settings.clickhouse_host,
+            port=self.port or settings.clickhouse_http_port,
+            timeout_seconds=timeout,
+        )
+        raise TimeoutError(f"ClickHouse not ready after {timeout:.1f}s") from last_error
+
+    def _escape_identifier(self, name: str) -> str:
+        """Escape a ClickHouse identifier (database, table, column) with backticks."""
+        return f"`{name.replace('`', '``')}`"
+
+    def ensure_table(
+        self,
+        *,
+        table_name: str,
+        schema: Any,
+        partition_spec: Any = None,
+        override_ref: str | None = None,
+    ) -> Any:
+        """Ensure a destination table exists."""
+        settings = self._settings()
+        database = self._escape_identifier(self.database or settings.clickhouse_db)
+        table = self._escape_identifier(table_name)
+
+        columns_def = self._schema_to_columns(schema)
+
+        partition_by = ""
+        if partition_spec:
+            partition_cols = [self._escape_identifier(p[0]) for p in partition_spec]
+            partition_by = f"PARTITION BY ({', '.join(partition_cols)})"
+
+        sql = f"CREATE TABLE IF NOT EXISTS {database}.{table} ({columns_def}) ENGINE = MergeTree() {partition_by} ORDER BY tuple()"
+
+        return self.command(sql)
+
+    def append_parquet(
+        self,
+        *,
+        table_name: str,
+        data_path: str | Path,
+        override_ref: str | None = None,
+    ) -> dict[str, int]:
+        """Append staged parquet data to a destination table."""
+        settings = self._settings()
+        database = self._escape_identifier(self.database or settings.clickhouse_db)
+        table = self._escape_identifier(table_name)
+
+        data_path_str = str(data_path)
+        df = pd.read_parquet(data_path_str)
+        row_count = len(df)
+
+        client = self.get_client()
+        try:
+            client.insert_df(f"{database}.{table}", df)
+        finally:
+            client.close()
+
+        return {"rows_inserted": row_count}
+
+    def merge_parquet(
+        self,
+        *,
+        table_name: str,
+        data_path: str | Path,
+        unique_key: str,
+        override_ref: str | None = None,
+    ) -> dict[str, int]:
+        """Merge staged parquet data into a destination table."""
+        settings = self._settings()
+        database = self._escape_identifier(self.database or settings.clickhouse_db)
+        table = self._escape_identifier(table_name)
+        key = self._escape_identifier(unique_key)
+
+        data_path_str = str(data_path)
+        df = pd.read_parquet(data_path_str)
+        row_count = len(df)
+
+        unique_keys = df[unique_key].tolist()
+        if unique_keys:
+            keys_str = ", ".join(f"'{k}'" for k in unique_keys)
+            delete_sql = f"ALTER TABLE {database}.{table} DELETE WHERE {key} IN ({keys_str})"
+            self.command(delete_sql)
+
+        client = self.get_client()
+        try:
+            client.insert_df(f"{database}.{table}", df)
+        finally:
+            client.close()
+
+        return {"rows_inserted": row_count, "rows_deleted": len(unique_keys)}
+
+    def _schema_to_columns(self, schema: Any) -> str:
+        """Convert a schema to ClickHouse column definitions."""
+        if hasattr(schema, "to_schema"):
+            schema = schema.to_schema()
+
+        if hasattr(schema, "columns"):
+            columns = []
+            for name, col in schema.columns.items():
+                ch_type = self._pandas_type_to_clickhouse(col.dtype)
+                columns.append(f"{name} {ch_type}")
+            return ", ".join(columns)
+
+        if hasattr(schema, "fields"):
+            columns = []
+            for field in schema.fields:
+                ch_type = self._python_type_to_clickhouse(field.type)
+                columns.append(f"{field.name} {ch_type}")
+            return ", ".join(columns)
+
+        raise TypeError(
+            f"Unsupported schema type: {type(schema).__name__}. Expected a schema with 'columns' or 'fields' attribute."
+        )
+
+    def _pandas_type_to_clickhouse(self, dtype: Any) -> str:
+        """Convert pandas dtype to ClickHouse type."""
+        import pandas as pd
+
+        if pd.api.types.is_integer_dtype(dtype):
+            return "Int64"
+        if pd.api.types.is_float_dtype(dtype):
+            return "Float64"
+        if pd.api.types.is_bool_dtype(dtype):
+            return "UInt8"
+        if pd.api.types.is_datetime64_any_dtype(dtype):
+            return "DateTime64"
+        if pd.api.types.is_string_dtype(dtype):
+            return "String"
+        return "String"
+
+    def _python_type_to_clickhouse(self, py_type: Any) -> str:
+        """Convert Python type to ClickHouse type."""
+        type_map = {
+            int: "Int64",
+            float: "Float64",
+            str: "String",
+            bool: "UInt8",
+        }
+        return type_map.get(py_type, "String")

phlo_clickhouse-0.1.0/src/phlo_clickhouse/service.yaml

@@ -0,0 +1,85 @@
+name: clickhouse
+description: ClickHouse analytical database for data plane
+category: data
+default: false
+
+image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-latest}
+
+compose:
+  restart: unless-stopped
+  labels:
+    phlo.metrics.enabled: "true"
+    phlo.metrics.port: "clickhouse:9363"
+    phlo.metrics.path: "/metrics"
+    phlo.grafana.datasource: "true"
+    phlo.grafana.datasource.type: "grafana-clickhouse-datasource"
+    phlo.grafana.datasource.name: "ClickHouse"
+    phlo.grafana.datasource.url: "clickhouse:9000"
+    traefik.enable: "true"
+    traefik.http.routers.clickhouse.rule: "Host(`clickhouse.${TRAEFIK_DOMAIN:-phlo.localhost}`)"
+    traefik.http.routers.clickhouse.entrypoints: "web"
+    traefik.http.services.clickhouse.loadbalancer.server.port: "8123"
+  environment:
+    CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default}
+    CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-}
+    CLICKHOUSE_DB: ${CLICKHOUSE_DB:-default}
+    CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: ${CLICKHOUSE_ACCESS_MANAGEMENT:-1}
+  ports:
+    - "${CLICKHOUSE_HTTP_PORT:-8123}:8123"
+    - "${CLICKHOUSE_NATIVE_PORT:-19000}:9000"
+    - "${CLICKHOUSE_METRICS_PORT:-9363}:9363"
+  volumes:
+    - ./volumes/clickhouse/data:/var/lib/clickhouse
+    - ./volumes/clickhouse/logs:/var/log/clickhouse-server
+  healthcheck:
+    test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8123/ping"]
+    interval: 10s
+    timeout: 5s
+    retries: 10
+    start_period: 30s
+  ulimits:
+    nofile:
+      soft: 262144
+      hard: 262144
+
+env_vars:
+  CLICKHOUSE_VERSION:
+    default: "latest"
+    description: ClickHouse server version tag
+  CLICKHOUSE_HTTP_PORT:
+    default: 8123
+    description: ClickHouse HTTP interface port
+  CLICKHOUSE_NATIVE_PORT:
+    default: 19000
+    description: ClickHouse native protocol port
+  CLICKHOUSE_METRICS_PORT:
+    default: 9363
+    description: ClickHouse Prometheus metrics port
+  CLICKHOUSE_USER:
+    default: "default"
+    description: ClickHouse default username
+  CLICKHOUSE_PASSWORD:
+    default: ""
+    description: ClickHouse default user password
+    secret: true
+  CLICKHOUSE_DB:
+    default: "default"
+    description: Default ClickHouse database
+  CLICKHOUSE_ACCESS_MANAGEMENT:
+    default: "1"
+    description: "Enable SQL-driven access management (1=enabled, 0=disabled)"
+  CLICKHOUSE_HTTPS_PORT:
+    default: ""
+    description: "HTTPS port (empty = disabled)"
+  CLICKHOUSE_TLS_CERT_FILE:
+    default: ""
+    description: "Path to TLS certificate file"
+  CLICKHOUSE_TLS_KEY_FILE:
+    default: ""
+    description: "Path to TLS private key file"
+  CLICKHOUSE_MAX_MEMORY_USAGE:
+    default: ""
+    description: "Maximum memory usage per query (e.g., 10000000000 for ~10GB)"
+  CLICKHOUSE_MAX_SERVER_MEMORY_USAGE_RATIO:
+    default: ""
+    description: "Max fraction of total RAM the server can use (e.g., 0.9)"

phlo_clickhouse-0.1.0/src/phlo_clickhouse/settings.py

@@ -0,0 +1,37 @@
+"""ClickHouse settings."""
+
+from __future__ import annotations
+
+from functools import lru_cache
+
+from pydantic import Field
+
+from phlo.config.base import BaseConfig
+
+
+class ClickHouseSettings(BaseConfig):
+    """ClickHouse data plane configuration."""
+
+    clickhouse_host: str = Field(default="clickhouse", description="ClickHouse service hostname")
+    clickhouse_http_port: int = Field(default=8123, description="ClickHouse HTTP interface port")
+    clickhouse_native_port: int = Field(
+        default=19000, description="ClickHouse native protocol port"
+    )
+    clickhouse_user: str = Field(default="default", description="ClickHouse username")
+    clickhouse_password: str = Field(default="", description="ClickHouse password")
+    clickhouse_db: str = Field(default="default", description="Default ClickHouse database")
+    clickhouse_secure: bool = Field(default=False, description="Use TLS for ClickHouse connections")
+
+    def clickhouse_http_endpoint(self) -> str:
+        """Return host:port endpoint for ClickHouse HTTP interface."""
+        return f"{self.clickhouse_host}:{self.clickhouse_http_port}"
+
+    def clickhouse_native_endpoint(self) -> str:
+        """Return host:port endpoint for ClickHouse native interface."""
+        return f"{self.clickhouse_host}:{self.clickhouse_native_port}"
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> ClickHouseSettings:
+    """Return cached ClickHouse settings."""
+    return ClickHouseSettings()

phlo_clickhouse-0.1.0/src/phlo_clickhouse.egg-info/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.4
+Name: phlo-clickhouse
+Version: 0.1.0
+Summary: ClickHouse service and resource plugin for Phlo
+Author-email: Phlo Team <team@phlo.dev>
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/plain
+Requires-Dist: phlo>=0.1.0
+Requires-Dist: clickhouse-connect>=0.8.0
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pyarrow>=12.0.0
+Provides-Extra: dbt
+Requires-Dist: dbt-core>=1.8; extra == "dbt"
+Requires-Dist: dbt-clickhouse>=1.8; extra == "dbt"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+
+ClickHouse service and resource plugin for Phlo.

phlo_clickhouse-0.1.0/src/phlo_clickhouse.egg-info/SOURCES.txt

@@ -0,0 +1,21 @@
+README.md
+pyproject.toml
+src/phlo_clickhouse/__init__.py
+src/phlo_clickhouse/cli.py
+src/phlo_clickhouse/cli_plugin.py
+src/phlo_clickhouse/clickhouse-setup.yaml
+src/phlo_clickhouse/plugin.py
+src/phlo_clickhouse/publish_target.py
+src/phlo_clickhouse/resource.py
+src/phlo_clickhouse/service.yaml
+src/phlo_clickhouse/settings.py
+src/phlo_clickhouse.egg-info/PKG-INFO
+src/phlo_clickhouse.egg-info/SOURCES.txt
+src/phlo_clickhouse.egg-info/dependency_links.txt
+src/phlo_clickhouse.egg-info/entry_points.txt
+src/phlo_clickhouse.egg-info/requires.txt
+src/phlo_clickhouse.egg-info/top_level.txt
+tests/test_clickhouse_capabilities.py
+tests/test_clickhouse_plugin.py
+tests/test_clickhouse_resource.py
+tests/test_clickhouse_settings.py

phlo_clickhouse-0.1.0/src/phlo_clickhouse.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

phlo_clickhouse-0.1.0/src/phlo_clickhouse.egg-info/entry_points.txt

@@ -0,0 +1,9 @@
+[phlo.plugins.cli]
+clickhouse = phlo_clickhouse.cli_plugin:ClickHouseCliPlugin
+
+[phlo.plugins.resources]
+clickhouse = phlo_clickhouse.plugin:ClickHouseResourceProvider
+
+[phlo.plugins.services]
+clickhouse = phlo_clickhouse.plugin:ClickHouseServicePlugin
+clickhouse-setup = phlo_clickhouse.plugin:ClickHouseSetupServicePlugin

phlo_clickhouse-0.1.0/src/phlo_clickhouse.egg-info/requires.txt

@@ -0,0 +1,13 @@
+phlo>=0.1.0
+clickhouse-connect>=0.8.0
+pyyaml>=6.0.1
+pandas>=2.0.0
+pyarrow>=12.0.0
+
+[dbt]
+dbt-core>=1.8
+dbt-clickhouse>=1.8
+
+[dev]
+pytest>=7.0
+ruff>=0.1.0

phlo_clickhouse-0.1.0/src/phlo_clickhouse.egg-info/top_level.txt

@@ -0,0 +1 @@
+phlo_clickhouse

phlo_clickhouse-0.1.0/tests/test_clickhouse_capabilities.py

@@ -0,0 +1,59 @@
+"""Tests for ClickHouse resource provider capabilities."""
+
+from phlo_clickhouse.plugin import ClickHouseResourceProvider
+
+
+def test_clickhouse_resource_provider_metadata():
+    """Validate ClickHouse resource provider metadata."""
+
+    provider = ClickHouseResourceProvider()
+    metadata = provider.metadata
+
+    assert metadata.name == "clickhouse"
+    assert metadata.version == "0.1.0"
+
+
+def test_clickhouse_resource_provider_get_resources():
+    """Validate ClickHouse resource provider returns resources."""
+
+    provider = ClickHouseResourceProvider()
+    resources = provider.get_resources()
+
+    assert len(resources) == 1
+    assert resources[0].name == "clickhouse"
+
+
+def test_clickhouse_resource_provider_get_table_stores():
+    """Validate ClickHouse resource provider returns table store specs."""
+
+    provider = ClickHouseResourceProvider()
+    table_stores = provider.get_table_stores()
+
+    assert len(table_stores) == 1
+    assert table_stores[0].name == "clickhouse"
+    assert table_stores[0].support.supports_snapshots is False
+    assert table_stores[0].support.supports_schema_evolution is True
+
+
+def test_clickhouse_resource_provider_get_query_engines():
+    """Validate ClickHouse resource provider returns query engine specs."""
+
+    provider = ClickHouseResourceProvider()
+    query_engines = provider.get_query_engines()
+
+    assert len(query_engines) == 1
+    assert query_engines[0].name == "clickhouse"
+    assert query_engines[0].metadata["service_type"] == "ClickHouse"
+    assert query_engines[0].support.supports_snapshots is False
+    assert query_engines[0].support.supports_time_travel is False
+
+
+def test_clickhouse_resource_provider_get_publish_targets():
+    """Validate ClickHouse resource provider returns publish target specs."""
+
+    provider = ClickHouseResourceProvider()
+    publish_targets = provider.get_publish_targets()
+
+    assert len(publish_targets) == 1
+    assert publish_targets[0].name == "clickhouse"
+    assert publish_targets[0].metadata["target_system"] == "clickhouse"

phlo_clickhouse-0.1.0/tests/test_clickhouse_plugin.py

@@ -0,0 +1,26 @@
+"""Tests for ClickHouse service plugin."""
+
+from phlo_clickhouse.plugin import ClickHouseServicePlugin
+
+
+def test_clickhouse_service_definition():
+    """Validate ClickHouse service definition fields."""
+
+    plugin = ClickHouseServicePlugin()
+    service_definition = plugin.service_definition
+
+    assert service_definition["name"] == "clickhouse"
+    assert service_definition["category"] == "data"
+
+
+def test_clickhouse_service_metadata():
+    """Validate ClickHouse service plugin metadata."""
+
+    plugin = ClickHouseServicePlugin()
+    metadata = plugin.metadata
+
+    assert metadata.name == "clickhouse"
+    assert metadata.version == "0.1.0"
+    assert "data" in metadata.tags
+    assert "query" in metadata.tags
+    assert "storage" in metadata.tags

phlo_clickhouse-0.1.0/tests/test_clickhouse_resource.py

@@ -0,0 +1,43 @@
+"""Tests for ClickHouse resource."""
+
+from phlo_clickhouse.resource import CLICKHOUSE_QUERY_ENGINE_SUPPORT, ClickHouseResource
+
+
+def test_clickhouse_resource_defaults():
+    """Validate ClickHouse resource default values."""
+
+    resource = ClickHouseResource()
+
+    assert resource.host is None
+    assert resource.port is None
+    assert resource.user is None
+    assert resource.password is None
+    assert resource.database is None
+    assert resource.secure is None
+
+
+def test_clickhouse_resource_with_overrides():
+    """Validate ClickHouse resource with override values."""
+
+    resource = ClickHouseResource(
+        host="my-clickhouse",
+        port=9000,
+        user="admin",
+        password="secret",
+        database="mydb",
+        secure=True,
+    )
+
+    assert resource.host == "my-clickhouse"
+    assert resource.port == 9000
+    assert resource.user == "admin"
+    assert resource.password == "secret"
+    assert resource.database == "mydb"
+    assert resource.secure is True
+
+
+def test_clickhouse_query_engine_support():
+    """Validate ClickHouse query engine support flags."""
+
+    assert CLICKHOUSE_QUERY_ENGINE_SUPPORT.supports_snapshots is False
+    assert CLICKHOUSE_QUERY_ENGINE_SUPPORT.supports_time_travel is False

phlo_clickhouse-0.1.0/tests/test_clickhouse_settings.py

@@ -0,0 +1,64 @@
+"""Tests for ClickHouse settings."""
+
+from phlo_clickhouse.settings import ClickHouseSettings, get_settings
+
+
+def test_clickhouse_settings_defaults():
+    """Validate ClickHouse settings default values."""
+
+    settings = ClickHouseSettings()
+
+    assert settings.clickhouse_host == "clickhouse"
+    assert settings.clickhouse_http_port == 8123
+    assert settings.clickhouse_native_port == 19000
+    assert settings.clickhouse_user == "default"
+    assert settings.clickhouse_password == ""
+    assert settings.clickhouse_db == "default"
+    assert settings.clickhouse_secure is False
+
+
+def test_clickhouse_settings_http_endpoint():
+    """Validate ClickHouse HTTP endpoint generation."""
+
+    settings = ClickHouseSettings()
+
+    assert settings.clickhouse_http_endpoint() == "clickhouse:8123"
+
+
+def test_clickhouse_settings_native_endpoint():
+    """Validate ClickHouse native endpoint generation."""
+
+    settings = ClickHouseSettings()
+
+    assert settings.clickhouse_native_endpoint() == "clickhouse:19000"
+
+
+def test_clickhouse_settings_with_overrides():
+    """Validate ClickHouse settings with override values."""
+
+    settings = ClickHouseSettings(
+        clickhouse_host="my-host",
+        clickhouse_http_port=9000,
+        clickhouse_native_port=9001,
+        clickhouse_user="admin",
+        clickhouse_password="secret",
+        clickhouse_db="mydb",
+        clickhouse_secure=True,
+    )
+
+    assert settings.clickhouse_host == "my-host"
+    assert settings.clickhouse_http_port == 9000
+    assert settings.clickhouse_native_port == 9001
+    assert settings.clickhouse_user == "admin"
+    assert settings.clickhouse_password == "secret"
+    assert settings.clickhouse_db == "mydb"
+    assert settings.clickhouse_secure is True
+
+
+def test_get_settings_returns_cached():
+    """Validate that get_settings returns cached instance."""
+
+    settings1 = get_settings()
+    settings2 = get_settings()
+
+    assert settings1 is settings2