phlo-rustfs 0.1.0__tar.gz

phlo_rustfs-0.1.0/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: phlo-rustfs
+Version: 0.1.0
+Summary: RustFS service plugin for Phlo
+Author-email: Phlo Team <team@phlo.dev>
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/plain
+Requires-Dist: phlo>=0.1.0
+Requires-Dist: pyyaml>=6.0.1
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+
+RustFS S3-compatible object storage service plugin for Phlo.

phlo_rustfs-0.1.0/README.md

@@ -0,0 +1,62 @@
+# phlo-rustfs
+
+RustFS S3-compatible object storage plugin for Phlo.
+
+## Description
+
+Provides S3-compatible object storage for the data lake using [RustFS](https://github.com/rustfs/rustfs), an Apache 2.0 licensed, Rust-built, 100% S3-compatible object storage server. Stores Iceberg table data, staging files, and backups.
+
+## Installation
+
+```bash
+pip install phlo-rustfs
+# or
+phlo plugin install rustfs
+```
+
+## Configuration
+
+| Variable                         | Default       | Description                  |
+| -------------------------------- | ------------- | ---------------------------- |
+| `RUSTFS_ACCESS_KEY`              | `rustfsadmin` | Access key (username)        |
+| `RUSTFS_SECRET_KEY`              | `rustfsadmin` | Secret key (password)        |
+| `RUSTFS_API_PORT`                | `9000`        | S3 API port                  |
+| `RUSTFS_CONSOLE_PORT`            | `9001`        | Web console port             |
+| `RUSTFS_CORS_ALLOWED_ORIGINS`    | `*`           | CORS allowed origins (S3)   |
+| `RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS` | `*`    | CORS allowed origins (console) |
+
+## Usage
+
+```bash
+phlo services start --service rustfs
+```
+
+This targeted start also prepares `./volumes/rustfs` for the non-root RustFS container and creates
+the default `lake`, `warehouse/`, and `stage/` S3 layout automatically.
+
+## Endpoints
+
+- **S3 API**: `http://localhost:9000`
+- **Console**: `http://localhost:9001`
+
+## Migration from MinIO
+
+If you're switching from MinIO to RustFS, update your environment variables:
+
+```bash
+# Before (MinIO)
+AWS_S3_ENDPOINT=http://minio:9000
+AWS_ACCESS_KEY_ID=minio
+AWS_SECRET_ACCESS_KEY=minio123
+
+# After (RustFS)
+AWS_S3_ENDPOINT=http://rustfs:9000
+AWS_ACCESS_KEY_ID=rustfsadmin
+AWS_SECRET_ACCESS_KEY=rustfsadmin
+```
+
+For data migration, see the [spec documentation](../../docs/architecture/specs/0015-phlo-rustfs.md).
+
+## Entry Points
+
+- `phlo.plugins.services` - Provides `RustfsServicePlugin`, `RustfsSetupServicePlugin`

phlo_rustfs-0.1.0/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools>=45", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "phlo-rustfs"
+version = "0.1.0"
+description = "RustFS service plugin for Phlo"
+readme = {text = "RustFS S3-compatible object storage service plugin for Phlo.", content-type = "text/plain"}
+requires-python = ">=3.11"
+authors = [
+    {name = "Phlo Team", email = "team@phlo.dev"},
+]
+license = {text = "MIT"}
+dependencies = [
+    "phlo>=0.1.0",
+    "pyyaml>=6.0.1",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "ruff>=0.1.0",
+]
+
+[project.entry-points."phlo.plugins.services"]
+rustfs = "phlo_rustfs.plugin:RustfsServicePlugin"
+rustfs-setup = "phlo_rustfs.plugin:RustfsSetupServicePlugin"
+
+[project.entry-points."phlo.plugins.resources"]
+rustfs = "phlo_rustfs.plugin:RustfsResourceProvider"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+phlo_rustfs = ["service.yaml", "rustfs-setup.yaml", "rustfs-volume-setup.yaml"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"

phlo_rustfs-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

phlo_rustfs-0.1.0/src/phlo_rustfs/__init__.py

@@ -0,0 +1,7 @@
+"""RustFS service plugin package."""
+
+from phlo_rustfs.plugin import RustfsServicePlugin
+from phlo_rustfs.settings import RustfsSettings, get_settings
+
+__all__ = ["RustfsServicePlugin", "RustfsSettings", "get_settings"]
+__version__ = "0.1.0"

phlo_rustfs-0.1.0/src/phlo_rustfs/plugin.py

@@ -0,0 +1,136 @@
+"""RustFS service plugin."""
+
+from __future__ import annotations
+
+from importlib import resources
+from time import perf_counter
+from typing import Any
+
+import yaml
+
+from phlo.capabilities import ObjectStoreSpec, ResourceSpec
+from phlo.logging import get_logger
+from phlo.plugins import PluginMetadata, ResourceProviderPlugin, ServicePlugin
+
+logger = get_logger(__name__)
+
+
+def _load_service_definition(resource_name: str, service_name: str) -> dict[str, Any]:
+    start = perf_counter()
+    logger.info(
+        "rustfs_service_definition_load_started",
+        service_name=service_name,
+        resource_name=resource_name,
+    )
+    service_path = resources.files("phlo_rustfs").joinpath(resource_name)
+    try:
+        data = yaml.safe_load(service_path.read_text(encoding="utf-8"))
+    except Exception:
+        logger.error(
+            "rustfs_service_definition_load_failed",
+            service_name=service_name,
+            resource_name=resource_name,
+            elapsed_ms=round((perf_counter() - start) * 1000, 2),
+            exc_info=True,
+        )
+        raise
+
+    service_count = len(data.get("services", {})) if isinstance(data, dict) else None
+    logger.info(
+        "rustfs_service_definition_load_completed",
+        service_name=service_name,
+        resource_name=resource_name,
+        elapsed_ms=round((perf_counter() - start) * 1000, 2),
+        service_count=service_count,
+    )
+    return data
+
+
+class RustfsServicePlugin(ServicePlugin):
+    """Service plugin for RustFS."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        """Return metadata describing the RustFS service plugin."""
+        return PluginMetadata(
+            name="rustfs",
+            version="0.1.0",
+            description="S3-compatible object storage for data lake (RustFS)",
+            author="Phlo Team",
+            tags=["core", "storage", "s3"],
+        )
+
+    @property
+    def service_definition(self) -> dict[str, Any]:
+        """Load the RustFS service definition."""
+        return _load_service_definition("service.yaml", "rustfs")
+
+
+class RustfsSetupServicePlugin(ServicePlugin):
+    """Service plugin for RustFS bucket initialization."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        """Return metadata describing the RustFS setup plugin."""
+        return PluginMetadata(
+            name="rustfs-setup",
+            version="0.1.0",
+            description="Initialize RustFS buckets for data lake",
+            author="Phlo Team",
+            tags=["core", "storage", "bootstrap"],
+        )
+
+    @property
+    def service_definition(self) -> dict[str, Any]:
+        """Load the RustFS setup service definition."""
+        return _load_service_definition("rustfs-setup.yaml", "rustfs-setup")
+
+
+class RustfsObjectStoreProvider:
+    """Capability provider for RustFS-backed object storage."""
+
+    def to_sling_connection(self) -> dict[str, Any]:
+        """Return a Sling-compatible S3 connection definition."""
+        from phlo_rustfs.settings import get_settings
+
+        settings = get_settings()
+        return {
+            "type": "s3",
+            "endpoint": f"http://{settings.rustfs_endpoint()}",
+            "access_key_id": settings.rustfs_access_key,
+            "secret_access_key": settings.rustfs_secret_key,
+            "region": settings.s3_region,
+        }
+
+
+class RustfsResourceProvider(ResourceProviderPlugin):
+    """Resource provider plugin for RustFS capabilities."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        """Return plugin metadata for the RustFS resource provider."""
+        return PluginMetadata(
+            name="rustfs",
+            version="0.1.0",
+            description="RustFS object-store capability for Phlo",
+            tags=["core", "storage", "s3"],
+        )
+
+    def get_resources(self) -> list[ResourceSpec]:
+        """Return resource specs exposed by this provider."""
+        return []
+
+    def get_object_stores(self) -> list[ObjectStoreSpec]:
+        """Return object-store capability specs exposed by this provider."""
+        provider = RustfsObjectStoreProvider()
+        return [
+            ObjectStoreSpec(
+                name="rustfs",
+                provider=provider,
+                metadata={
+                    "storage_system": "s3",
+                    "type": "s3",
+                    "endpoint": provider.to_sling_connection()["endpoint"],
+                },
+            )
+        ]

phlo_rustfs-0.1.0/src/phlo_rustfs/rustfs-setup.yaml

@@ -0,0 +1,28 @@
+# Companion service for rustfs - creates required buckets
+name: rustfs-setup
+description: Initialize RustFS buckets for data lake
+category: core
+default: false
+
+image: amazon/aws-cli:2.15.40
+
+depends_on:
+  - rustfs
+
+compose:
+  restart: "no"
+  entrypoint: /bin/sh
+  command: >
+    -c "
+    until aws --endpoint-url http://rustfs:9000 s3api list-buckets --region us-east-1 >/dev/null 2>&1; do
+    echo 'Waiting for RustFS...' && sleep 2;
+    done &&
+    aws --endpoint-url http://rustfs:9000 s3 mb s3://lake --region us-east-1 || true &&
+    aws --endpoint-url http://rustfs:9000 s3api put-object --bucket lake --key warehouse/ --region us-east-1 >/dev/null &&
+    aws --endpoint-url http://rustfs:9000 s3api put-object --bucket lake --key stage/ --region us-east-1 >/dev/null &&
+    echo 'RustFS bucket prefixes created successfully'
+    "
+  environment:
+    AWS_ACCESS_KEY_ID: ${RUSTFS_ACCESS_KEY:-rustfsadmin}
+    AWS_SECRET_ACCESS_KEY: ${RUSTFS_SECRET_KEY:-rustfsadmin}
+    AWS_DEFAULT_REGION: us-east-1

phlo_rustfs-0.1.0/src/phlo_rustfs/rustfs-volume-setup.yaml

@@ -0,0 +1,20 @@
+# Companion service for rustfs - prepares writable bind mount ownership
+name: rustfs-volume-setup
+description: Initialize RustFS data volume permissions
+category: core
+default: false
+
+image: alpine:3.20
+
+compose:
+  restart: "no"
+  user: "0:0"
+  entrypoint: /bin/sh
+  command: >
+    -c "
+    mkdir -p /data &&
+    chown -R 10001:10001 /data &&
+    echo 'RustFS data volume ownership initialized'
+    "
+  volumes:
+    - ./volumes/rustfs:/data

phlo_rustfs-0.1.0/src/phlo_rustfs/service.yaml

@@ -0,0 +1,53 @@
+name: rustfs
+description: S3-compatible object storage for data lake (RustFS)
+category: core
+default: false
+
+depends_on:
+  - rustfs-volume-setup
+
+image: rustfs/rustfs:latest
+
+compose:
+  restart: unless-stopped
+  user: "10001:10001"
+  environment:
+    RUSTFS_VOLUMES: /data
+    RUSTFS_ADDRESS: "0.0.0.0:9000"
+    RUSTFS_CONSOLE_ADDRESS: "0.0.0.0:9001"
+    RUSTFS_CONSOLE_ENABLE: "true"
+    RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:-rustfsadmin}
+    RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:-rustfsadmin}
+    RUSTFS_CORS_ALLOWED_ORIGINS: ${RUSTFS_CORS_ALLOWED_ORIGINS:-*}
+    RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS: ${RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS:-*}
+  ports:
+    - "${RUSTFS_API_PORT:-9000}:9000"
+    - "${RUSTFS_CONSOLE_PORT:-9001}:9001"
+  volumes:
+    - ./volumes/rustfs:/data
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:9000/health"]
+    interval: 10s
+    timeout: 5s
+    retries: 10
+
+env_vars:
+  RUSTFS_ACCESS_KEY:
+    default: rustfsadmin
+    description: RustFS access key (username)
+  RUSTFS_SECRET_KEY:
+    default: rustfsadmin
+    description: RustFS secret key (password)
+    secret: true
+  RUSTFS_API_PORT:
+    default: 9000
+    description: RustFS S3 API host port
+  RUSTFS_CONSOLE_PORT:
+    default: 9001
+    description: RustFS web console host port
+  RUSTFS_CORS_ALLOWED_ORIGINS:
+    default: "*"
+    description: CORS allowed origins for S3 API
+  RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS:
+    default: "*"
+    description: CORS allowed origins for web console

phlo_rustfs-0.1.0/src/phlo_rustfs/settings.py

@@ -0,0 +1,30 @@
+"""RustFS settings."""
+
+from __future__ import annotations
+
+from functools import lru_cache
+
+from pydantic import Field
+
+from phlo.config.base import BaseConfig
+
+
+class RustfsSettings(BaseConfig):
+    """RustFS S3-compatible storage configuration."""
+
+    rustfs_host: str = Field(default="rustfs", description="RustFS service hostname")
+    rustfs_access_key: str = Field(default="rustfsadmin", description="RustFS access key")
+    rustfs_secret_key: str = Field(default="rustfsadmin", description="RustFS secret key")
+    rustfs_api_port: int = Field(default=9000, description="RustFS S3 API port")
+    rustfs_console_port: int = Field(default=9001, description="RustFS console port")
+    s3_region: str = Field(default="us-east-1", description="S3 region")
+
+    def rustfs_endpoint(self) -> str:
+        """Return host:port endpoint for RustFS S3 API."""
+        return f"{self.rustfs_host}:{self.rustfs_api_port}"
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> RustfsSettings:
+    """Return cached RustFS settings."""
+    return RustfsSettings()

phlo_rustfs-0.1.0/src/phlo_rustfs.egg-info/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: phlo-rustfs
+Version: 0.1.0
+Summary: RustFS service plugin for Phlo
+Author-email: Phlo Team <team@phlo.dev>
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/plain
+Requires-Dist: phlo>=0.1.0
+Requires-Dist: pyyaml>=6.0.1
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+
+RustFS S3-compatible object storage service plugin for Phlo.

phlo_rustfs-0.1.0/src/phlo_rustfs.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+README.md
+pyproject.toml
+src/phlo_rustfs/__init__.py
+src/phlo_rustfs/plugin.py
+src/phlo_rustfs/rustfs-setup.yaml
+src/phlo_rustfs/rustfs-volume-setup.yaml
+src/phlo_rustfs/service.yaml
+src/phlo_rustfs/settings.py
+src/phlo_rustfs.egg-info/PKG-INFO
+src/phlo_rustfs.egg-info/SOURCES.txt
+src/phlo_rustfs.egg-info/dependency_links.txt
+src/phlo_rustfs.egg-info/entry_points.txt
+src/phlo_rustfs.egg-info/requires.txt
+src/phlo_rustfs.egg-info/top_level.txt
+tests/test_rustfs_plugin.py
+tests/test_rustfs_settings.py

phlo_rustfs-0.1.0/src/phlo_rustfs.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

phlo_rustfs-0.1.0/src/phlo_rustfs.egg-info/entry_points.txt

@@ -0,0 +1,6 @@
+[phlo.plugins.resources]
+rustfs = phlo_rustfs.plugin:RustfsResourceProvider
+
+[phlo.plugins.services]
+rustfs = phlo_rustfs.plugin:RustfsServicePlugin
+rustfs-setup = phlo_rustfs.plugin:RustfsSetupServicePlugin

phlo_rustfs-0.1.0/src/phlo_rustfs.egg-info/requires.txt

@@ -0,0 +1,6 @@
+phlo>=0.1.0
+pyyaml>=6.0.1
+
+[dev]
+pytest>=7.0
+ruff>=0.1.0

phlo_rustfs-0.1.0/src/phlo_rustfs.egg-info/top_level.txt

@@ -0,0 +1 @@
+phlo_rustfs

phlo_rustfs-0.1.0/tests/test_rustfs_plugin.py

@@ -0,0 +1,67 @@
+"""Tests for RustFS service plugin."""
+
+from phlo_rustfs.plugin import RustfsResourceProvider, RustfsServicePlugin, RustfsSetupServicePlugin
+
+
+def test_rustfs_service_definition():
+    """Validate RustFS service definition fields."""
+    plugin = RustfsServicePlugin()
+    service_definition = plugin.service_definition
+
+    assert service_definition["name"] == "rustfs"
+    assert service_definition["category"] == "core"
+    assert service_definition["default"] is False
+    assert "rustfs/rustfs" in service_definition["image"]
+    assert "rustfs-volume-setup" in service_definition["depends_on"]
+
+
+def test_rustfs_plugin_metadata():
+    """Validate RustFS plugin metadata."""
+    plugin = RustfsServicePlugin()
+    metadata = plugin.metadata
+
+    assert metadata.name == "rustfs"
+    assert metadata.version == "0.1.0"
+    assert "storage" in metadata.tags
+    assert "s3" in metadata.tags
+
+
+def test_rustfs_setup_service_definition():
+    """Validate RustFS setup service definition fields."""
+    plugin = RustfsSetupServicePlugin()
+    service_definition = plugin.service_definition
+
+    assert service_definition["name"] == "rustfs-setup"
+    assert service_definition["category"] == "core"
+    assert "rustfs" in service_definition["depends_on"]
+
+
+def test_rustfs_setup_plugin_metadata():
+    """Validate RustFS setup plugin metadata."""
+    plugin = RustfsSetupServicePlugin()
+    metadata = plugin.metadata
+
+    assert metadata.name == "rustfs-setup"
+    assert "bootstrap" in metadata.tags
+
+
+def test_rustfs_resource_provider_exposes_object_store(monkeypatch) -> None:
+    """RustFS should expose an object_store capability."""
+    monkeypatch.setattr(
+        "phlo_rustfs.plugin.RustfsObjectStoreProvider.to_sling_connection",
+        lambda _self: {
+            "type": "s3",
+            "endpoint": "http://rustfs:9000",
+            "access_key_id": "rustfs",
+            "secret_access_key": "secret",
+            "region": "us-east-1",
+        },
+    )
+
+    provider = RustfsResourceProvider()
+
+    object_stores = provider.get_object_stores()
+
+    assert len(object_stores) == 1
+    assert object_stores[0].name == "rustfs"
+    assert object_stores[0].metadata["endpoint"] == "http://rustfs:9000"

phlo_rustfs-0.1.0/tests/test_rustfs_settings.py

@@ -0,0 +1,39 @@
+"""Tests for RustFS settings."""
+
+from phlo_rustfs.settings import RustfsSettings
+
+
+def test_rustfs_settings_defaults():
+    """Validate default settings values."""
+    settings = RustfsSettings()
+
+    assert settings.rustfs_host == "rustfs"
+    assert settings.rustfs_access_key == "rustfsadmin"
+    assert settings.rustfs_secret_key == "rustfsadmin"
+    assert settings.rustfs_api_port == 9000
+    assert settings.rustfs_console_port == 9001
+    assert settings.s3_region == "us-east-1"
+
+
+def test_rustfs_endpoint():
+    """Validate endpoint format."""
+    settings = RustfsSettings()
+    endpoint = settings.rustfs_endpoint()
+
+    assert endpoint == "rustfs:9000"
+
+
+def test_rustfs_endpoint_custom_port():
+    """Validate endpoint with custom port."""
+    settings = RustfsSettings(rustfs_api_port=19000)
+    endpoint = settings.rustfs_endpoint()
+
+    assert endpoint == "rustfs:19000"
+
+
+def test_rustfs_endpoint_custom_host():
+    """Validate endpoint with custom host."""
+    settings = RustfsSettings(rustfs_host="storage.local")
+    endpoint = settings.rustfs_endpoint()
+
+    assert endpoint == "storage.local:9000"