planar 0.9.3__py3-none-any.whl → 0.11.0__py3-none-any.whl

planar/routers/dataset_router.py

@@ -0,0 +1,213 @@
+import io
+from typing import AsyncGenerator
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+from fastapi import APIRouter, HTTPException, Query
+from fastapi.responses import StreamingResponse
+from ibis.common.exceptions import TableNotFound
+from pydantic import BaseModel
+
+from planar.data.exceptions import DatasetNotFoundError
+from planar.data.utils import (
+    get_dataset,
+    get_dataset_metadata,
+    list_datasets,
+    list_schemas,
+)
+from planar.logging import get_logger
+from planar.security.authorization import (
+    DatasetAction,
+    DatasetResource,
+    validate_authorization_for,
+)
+
+logger = get_logger(__name__)
+
+
+class DatasetMetadata(BaseModel):
+    name: str
+    table_schema: dict
+    row_count: int
+
+
+def create_dataset_router() -> APIRouter:
+    router = APIRouter(tags=["Planar Datasets"])
+
+    @router.get("/schemas", response_model=list[str])
+    async def get_schemas():
+        validate_authorization_for(
+            DatasetResource(), DatasetAction.DATASET_LIST_SCHEMAS
+        )
+        schemas = await list_schemas()
+        return schemas
+
+    @router.get("/metadata", response_model=list[DatasetMetadata])
+    async def list_planar_datasets(
+        limit: int = Query(100, ge=1, le=1000),
+        offset: int = Query(0, ge=0),
+        schema_name: str = Query("main"),
+    ):
+        validate_authorization_for(DatasetResource(), DatasetAction.DATASET_LIST)
+        datasets = await list_datasets(limit, offset)
+
+        response = []
+        for dataset in datasets:
+            metadata = await get_dataset_metadata(dataset.name, schema_name)
+
+            if not metadata:
+                continue
+
+            schema = metadata["schema"]
+            row_count = metadata["row_count"]
+
+            response.append(
+                DatasetMetadata(
+                    name=dataset.name,
+                    row_count=row_count,
+                    table_schema={
+                        field_name: str(field_type)
+                        for field_name, field_type in schema.items()
+                    },
+                )
+            )
+
+        return response
+
+    @router.get("/metadata/{dataset_name}", response_model=DatasetMetadata)
+    async def get_planar_dataset(dataset_name: str, schema_name: str = "main"):
+        validate_authorization_for(
+            DatasetResource(dataset_name=dataset_name),
+            DatasetAction.DATASET_VIEW_DETAILS,
+        )
+        try:
+            metadata = await get_dataset_metadata(dataset_name, schema_name)
+
+            if not metadata:
+                raise HTTPException(
+                    status_code=404, detail=f"Dataset {dataset_name} not found"
+                )
+
+            schema = metadata["schema"]
+            row_count = metadata["row_count"]
+
+            return DatasetMetadata(
+                name=dataset_name,
+                row_count=row_count,
+                table_schema={
+                    field_name: str(field_type)
+                    for field_name, field_type in schema.items()
+                },
+            )
+        except (DatasetNotFoundError, TableNotFound):
+            raise HTTPException(
+                status_code=404, detail=f"Dataset {dataset_name} not found"
+            )
+
+    @router.get(
+        "/content/{dataset_name}/arrow-stream", response_class=StreamingResponse
+    )
+    async def stream_dataset_content(
+        dataset_name: str,
+        batch_size: int = Query(100, ge=1, le=1000),
+        limit: int | None = Query(None, ge=1),
+    ):
+        validate_authorization_for(
+            DatasetResource(dataset_name=dataset_name),
+            DatasetAction.DATASET_STREAM_CONTENT,
+        )
+        try:
+            dataset = await get_dataset(dataset_name)
+
+            # Apply limit parameter if specified
+            table = await dataset.read(limit=limit)
+
+            schema = table.schema().to_pyarrow()
+
+            async def stream_content() -> AsyncGenerator[bytes, None]:
+                sink = io.BytesIO()
+
+                try:
+                    with pa.ipc.new_stream(sink, schema) as writer:
+                        yield sink.getvalue()  # yield the schema
+
+                        batch_count = 0
+                        for batch in table.to_pyarrow_batches(chunk_size=batch_size):
+                            # reset the sink to only stream
+                            # the current batch
+                            # we don't want to stream the schema or previous
+                            # batches again
+                            sink.seek(0)
+                            sink.truncate(0)
+
+                            writer.write_batch(batch)
+                            yield sink.getvalue()
+                            batch_count += 1
+
+                        # For empty datasets, ensure we have a complete stream
+                        if batch_count == 0:
+                            # Write an empty batch to ensure valid Arrow stream format
+                            empty_batch = pa.RecordBatch.from_arrays(
+                                [pa.array([], type=field.type) for field in schema],
+                                schema=schema,
+                            )
+                            sink.seek(0)
+                            sink.truncate(0)
+                            writer.write_batch(empty_batch)
+                            yield sink.getvalue()
+                finally:
+                    # Explicit BytesIO cleanup for memory safety
+                    sink.close()
+
+            return StreamingResponse(
+                stream_content(),
+                media_type="application/vnd.apache.arrow.stream",
+                headers={
+                    "Content-Disposition": f"attachment; filename={dataset_name}.arrow",
+                    "X-Batch-Size": str(batch_size),
+                    "X-Row-Limit": str(limit) if limit else "unlimited",
+                },
+            )
+        except (DatasetNotFoundError, TableNotFound):
+            raise HTTPException(
+                status_code=404, detail=f"Dataset {dataset_name} not found"
+            )
+
+    @router.get("/content/{dataset_name}/download")
+    async def download_dataset(dataset_name: str, schema_name: str = "main"):
+        validate_authorization_for(
+            DatasetResource(dataset_name=dataset_name),
+            DatasetAction.DATASET_DOWNLOAD,
+        )
+        try:
+            arrow_buffer = pa.BufferOutputStream()
+            dataset = await get_dataset(dataset_name, schema_name)
+
+            pyarrow_table = await dataset.to_pyarrow()
+
+            pq.write_table(pyarrow_table, arrow_buffer)
+
+            if arrow_buffer.tell() == 0:
+                logger.warning(
+                    "Dataset is empty",
+                    dataset_name=dataset_name,
+                    schema_name=schema_name,
+                )
+
+            buffer = arrow_buffer.getvalue()
+            parquet_bytes = buffer.to_pybytes()
+            bytes_io = io.BytesIO(parquet_bytes)
+
+            return StreamingResponse(
+                bytes_io,
+                media_type="application/x-parquet",
+                headers={
+                    "Content-Disposition": f"attachment; filename={dataset_name}.parquet"
+                },
+            )
+        except (DatasetNotFoundError, TableNotFound):
+            raise HTTPException(
+                status_code=404, detail=f"Dataset {dataset_name} not found"
+            )
+
+    return router

planar/routers/info.py

@@ -1,16 +1,34 @@
+import importlib.metadata
+from typing import Literal, TypedDict
+
 from fastapi import APIRouter, Depends
 from pydantic import BaseModel
-from sqlalchemy.ext.asyncio import AsyncSession
-from sqlmodel import col, distinct, func, select
+from sqlmodel import col, func, select
+from sqlmodel.ext.asyncio.session import AsyncSession
 
+from planar.config import PlanarConfig, get_environment
+from planar.data.config import DataConfig
+from planar.files.storage.config import StorageConfig
 from planar.human.models import HumanTask, HumanTaskStatus
 from planar.logging import get_logger
-from planar.object_config import ConfigurableObjectType, ObjectConfiguration
+from planar.object_registry import ObjectRegistry
 from planar.session import get_session
 from planar.workflows.models import Workflow, WorkflowStatus
 
 logger = get_logger(__name__)
 
+StorageInfo = Literal["s3", "localdir", "azure_blob"]
+
+
+class DatasetsInfo(BaseModel):
+    catalog: Literal["duckdb", "postgres", "sqlite"]
+    storage: StorageInfo
+
+
+class SystemFeatures(BaseModel):
+    storage: StorageInfo | None = None
+    datasets: DatasetsInfo | None = None
+
 
 class SystemInfo(BaseModel):
     """Combined application information and system statistics"""
@@ -19,6 +37,11 @@ class SystemInfo(BaseModel):
     title: str
     description: str
 
+    version: str
+    environment: str
+
+    features: SystemFeatures
+
     # System stats
     total_workflow_runs: int = 0
     completed_runs: int = 0
@@ -27,7 +50,18 @@ class SystemInfo(BaseModel):
     active_agents: int = 0
 
 
-async def get_system_stats(session: AsyncSession = Depends(get_session)) -> dict:
+class SystemStats(TypedDict):
+    total_workflow_runs: int
+    completed_runs: int
+    in_progress_runs: int
+    pending_human_tasks: int
+    active_agents: int
+
+
+async def get_system_stats(
+    registry: ObjectRegistry,
+    session: AsyncSession = Depends(get_session),
+) -> SystemStats:
     """
     Get system-wide statistics directly from the database.
 
@@ -35,8 +69,10 @@ async def get_system_stats(session: AsyncSession = Depends(get_session)) -> dict
     rather than fetching all records and calculating in the application.
     """
     try:
+        agent_count = len(registry.get_agents())
+
         # Get workflow run counts
-        workflow_stats = await session.execute(
+        workflow_stats = await session.exec(
             select(
                 func.count().label("total_runs"),
                 func.count(col(Workflow.id))
@@ -47,42 +83,21 @@ async def get_system_stats(session: AsyncSession = Depends(get_session)) -> dict
                 .label("in_progress_runs"),
             ).select_from(Workflow)
         )
-        workflow_row = workflow_stats.one()
+        total_runs, completed_runs, in_progress_runs = workflow_stats.one()
 
         # Get pending human task count
-        human_task_query = await session.execute(
+        human_task_query = await session.exec(
             select(func.count())
             .select_from(HumanTask)
             .where(HumanTask.status == HumanTaskStatus.PENDING)
         )
-        pending_tasks = human_task_query.scalar() or 0
-
-        # Get agent count from the registry or count distinct agent configs
-        agent_count = 0
-        try:
-            # Count distinct agent names in the AgentConfig table
-            agent_query = await session.execute(
-                select(
-                    func.count(distinct(ObjectConfiguration.object_name))
-                ).select_from(
-                    select(ObjectConfiguration)
-                    .where(
-                        ObjectConfiguration.object_type == ConfigurableObjectType.AGENT
-                    )
-                    .subquery()
-                )
-            )
-            agent_count = agent_query.scalar() or 0
-        except Exception:
-            logger.exception("error counting agents")
-            # Fallback to 0
-            agent_count = 0
+        pending_tasks = human_task_query.one()
 
         # Return stats dict
         return {
-            "total_workflow_runs": workflow_row.total_runs or 0,
-            "completed_runs": workflow_row.completed_runs or 0,
-            "in_progress_runs": workflow_row.in_progress_runs or 0,
+            "total_workflow_runs": total_runs,
+            "completed_runs": completed_runs,
+            "in_progress_runs": in_progress_runs,
             "pending_human_tasks": pending_tasks,
             "active_agents": agent_count,
         }
@@ -98,13 +113,31 @@ async def get_system_stats(session: AsyncSession = Depends(get_session)) -> dict
         }
 
 
-def create_info_router(title: str, description: str) -> APIRouter:
+def get_app_version() -> str:
+    try:
+        return importlib.metadata.version("planar")
+    except importlib.metadata.PackageNotFoundError:
+        logger.warning("Planar package not found, returning development version")
+        return "development"
+
+
+def get_storage_info(cfg: StorageConfig) -> StorageInfo:
+    return cfg.backend
+
+
+def get_datasets_info(cfg: DataConfig) -> DatasetsInfo | None:
+    return DatasetsInfo(catalog=cfg.catalog.type, storage=get_storage_info(cfg.storage))
+
+
+def create_info_router(
+    title: str, description: str, config: PlanarConfig, registry: ObjectRegistry
+) -> APIRouter:
     """
     Create a router for serving combined application information and system statistics.
 
     This router provides a single endpoint to retrieve the application's title,
     description, and system-wide statistics on workflow runs, human tasks,
-    and registered agents.
+    and registered agents, as well as the application's features and configuration.
 
     Args:
         title: The application title
@@ -125,7 +158,17 @@ def create_info_router(title: str, description: str) -> APIRouter:
         Returns:
             SystemInfo object containing app details and system stats
         """
-        stats = await get_system_stats(session)
-        return SystemInfo(title=title, description=description, **stats)
+        stats = await get_system_stats(registry, session)
+        return SystemInfo(
+            title=title,
+            description=description,
+            version=get_app_version(),
+            environment=get_environment(),
+            features=SystemFeatures(
+                storage=get_storage_info(config.storage) if config.storage else None,
+                datasets=get_datasets_info(config.data) if config.data else None,
+            ),
+            **stats,
+        )
 
     return router

planar/routers/models.py

@@ -79,6 +79,7 @@ class WorkflowDefinition(BaseModel):
     total_runs: int
     run_statuses: WorkflowRunStatusCounts
     durations: DurationStats | None = None
+    is_interactive: bool
 
 
 class StepStats(BaseModel):

planar/routers/workflow.py

@@ -217,6 +217,7 @@ def create_workflow_router(
                         }
                     ),
                     durations=duration_stats,
+                    is_interactive=workflow.is_interactive,
                 )
             )
 
@@ -257,6 +258,7 @@ def create_workflow_router(
                 **{status.value: count for status, count in run_statuses.items()}
             ),
             durations=duration_stats,
+            is_interactive=wf.is_interactive,
         )
 
     @router.get("/{workflow_name}/runs", response_model=WorkflowRunList)

planar/scaffold_templates/pyproject.toml.j2

@@ -3,7 +3,7 @@ name = "{{ name }}"
 version = "0.1.0"
 requires-python = ">=3.12"
 dependencies = [
-    "planar>=0.9.0",
+    "planar[data]>=0.10.0",
 ]
 
 [[tool.uv.index]]

planar/security/authorization.py

@@ -87,12 +87,23 @@ class RuleAction(str, Enum):
     RULE_SIMULATE = "Rule::Simulate"
 
 
+class DatasetAction(str, Enum):
+    """Actions that can be performed on datasets."""
+
+    DATASET_LIST_SCHEMAS = "Dataset::ListSchemas"
+    DATASET_LIST = "Dataset::List"
+    DATASET_VIEW_DETAILS = "Dataset::ViewDetails"
+    DATASET_STREAM_CONTENT = "Dataset::StreamContent"
+    DATASET_DOWNLOAD = "Dataset::Download"
+
+
 class ResourceType(str, Enum):
     PRINCIPAL = "Principal"
     WORKFLOW = "Workflow"
     ENTITY = "Entity"
     AGENT = "Agent"
     Rule = "Rule"
+    DATASET = "Dataset"
 
 
 class EntityIdentifier(TypedDict):
@@ -129,7 +140,12 @@ class RuleResource:
     rule_name: str | None = None
 
 
-ResourceDescriptor = AgentResource | WorkflowResource | RuleResource
+@dataclass(frozen=True, slots=True)
+class DatasetResource:
+    dataset_name: str | None = None
+
+
+ResourceDescriptor = AgentResource | WorkflowResource | RuleResource | DatasetResource
 
 
 class CedarEntity(BaseModel):
@@ -209,6 +225,15 @@ class CedarEntity(BaseModel):
             resource_attributes={"rule_name": rule_name},
         )
 
+    @staticmethod
+    def from_dataset(dataset_name: str | None) -> "CedarEntity":
+        """Create a CedarEntity instance from dataset data"""
+        return CedarEntity(
+            resource_type=ResourceType.DATASET,
+            resource_key="dataset_name",
+            resource_attributes={"dataset_name": dataset_name},
+        )
+
 
 class PolicyService:
     """Service for managing and evaluating Authorization policies."""
@@ -272,7 +297,7 @@ class PolicyService:
     def is_allowed(
         self,
         principal: CedarEntity,
-        action: str | WorkflowAction | AgentAction | RuleAction,
+        action: str | WorkflowAction | AgentAction | RuleAction | DatasetAction,
         resource: CedarEntity,
     ) -> bool:
         """Check if the principal is permitted to perform the action on the resource.
@@ -294,6 +319,7 @@ class PolicyService:
             isinstance(action, WorkflowAction)
             or isinstance(action, AgentAction)
             or isinstance(action, RuleAction)
+            or isinstance(action, DatasetAction)
         ):
             action = f'Action::"{action.value}"'
         else:
@@ -346,7 +372,7 @@ class PolicyService:
 
 def validate_authorization_for(
     resource_descriptor: ResourceDescriptor,
-    action: WorkflowAction | AgentAction | RuleAction,
+    action: WorkflowAction | AgentAction | RuleAction | DatasetAction,
 ):
     authz_service = get_policy_service()
 
@@ -363,6 +389,8 @@ def validate_authorization_for(
             entity = CedarEntity.from_agent(resource_descriptor.id)
         case RuleAction() if isinstance(resource_descriptor, RuleResource):
             entity = CedarEntity.from_rule(resource_descriptor.rule_name)
+        case DatasetAction() if isinstance(resource_descriptor, DatasetResource):
+            entity = CedarEntity.from_dataset(resource_descriptor.dataset_name)
         case _:
             raise ValueError(
                 f"Invalid resource descriptor {type(resource_descriptor).__name__} for action {action}"

planar/security/default_policies.cedar

@@ -74,4 +74,29 @@ permit (
     principal,
     action == Action::"Rule::Simulate",
     resource
+);
+permit (
+    principal,
+    action == Action::"Dataset::ListSchemas",
+    resource
+);
+permit (
+    principal,
+    action == Action::"Dataset::List",
+    resource
+);
+permit (
+    principal,
+    action == Action::"Dataset::ViewDetails",
+    resource
+);
+permit (
+    principal,
+    action == Action::"Dataset::StreamContent",
+    resource
+);
+permit (
+    principal,
+    action == Action::"Dataset::Download",
+    resource
 );

planar/testing/fixtures.py

@@ -16,6 +16,8 @@ Usage in external projects:
 
 Available fixtures:
 - storage: In-memory file storage for tests
+- data_config: Test data configuration with SQLite catalog and local storage
+- app_with_data: PlanarApp instance with data configuration
 - tmp_db_url: Parametrized database URL (SQLite/PostgreSQL)
 - session: Database session
 - client: Planar test client
@@ -33,8 +35,11 @@ from pathlib import Path
 
 import pytest
 
-from planar.config import load_config
+from planar.app import PlanarApp
+from planar.config import load_config, load_environment_aware_config
+from planar.data.config import DataConfig, SQLiteCatalogConfig
 from planar.db import DatabaseManager, new_session
+from planar.files.storage.config import LocalDirectoryConfig
 from planar.files.storage.context import set_storage
 from planar.logging import set_context_metadata
 from planar.object_registry import ObjectRegistry
@@ -114,6 +119,34 @@ async def storage():
     yield storage
 
 
+@pytest.fixture()
+def data_config(tmp_path):
+    """Create a test data configuration."""
+    data_dir = tmp_path / "data"
+    data_dir.mkdir(exist_ok=True)
+
+    catalog_path = data_dir / "test.sqlite"
+    storage_path = data_dir / "ducklake_files"
+    storage_path.mkdir(exist_ok=True)
+
+    return DataConfig(
+        catalog=SQLiteCatalogConfig(type="sqlite", path=str(catalog_path)),
+        storage=LocalDirectoryConfig(backend="localdir", directory=str(storage_path)),
+    )
+
+
+@pytest.fixture(name="app_with_data")
+def app_with_data_fixture(data_config):
+    """Create a PlanarApp with data configuration."""
+    config = load_environment_aware_config()
+
+    config.data = data_config
+
+    app = PlanarApp(config=config)
+
+    return app
+
+
 @pytest.fixture()
 def tmp_sqlite_url(tmp_db_path: str):
     return f"sqlite+aiosqlite:///{tmp_db_path}"

planar/testing/planar_test_client.py

@@ -53,5 +53,5 @@ async def wait_all_event_loop_tasks():
             break
         try:
             await asyncio.gather(*other_tasks)
-        except asyncio.CancelledError:
+        except (asyncio.CancelledError, Exception):
             pass

planar/workflows/decorators.py

@@ -78,7 +78,7 @@ def step(
     return decorator
 
 
-def workflow(*, name: str | None = None):
+def workflow(*, name: str | None = None, is_interactive: bool = False):
     """
     Decorator to define a workflow.
 
@@ -177,6 +177,7 @@ def workflow(*, name: str | None = None):
             start_step=start_workflow_step,
             wait_for_completion=wait_for_completion,
             wrapped_fn=run_workflow,
+            is_interactive=is_interactive,
         )
 
         return wf_wrapper

planar/workflows/wrappers.py

@@ -33,6 +33,7 @@ class WorkflowWrapper(Wrapper[P, T, U, R]):
     start: Callable[P, Coroutine[T, U, Workflow]]
     start_step: Callable[P, Coroutine[T, U, UUID]]
     wait_for_completion: Callable[[UUID], Coroutine[T, U, R]]
+    is_interactive: bool
 
 
 @dataclass(kw_only=True)

{planar-0.9.3.dist-info → planar-0.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: planar
-Version: 0.9.3
+Version: 0.11.0
 Summary: Add your description here
 License-Expression: LicenseRef-Proprietary
 Requires-Dist: aiofiles>=24.1.0
@@ -296,6 +296,14 @@ To disable SQLite testing:
 PLANAR_TEST_SQLITE=0 uv run pytest
 ```
 
+### Test coverage
+
+We use [pytest-cov](https://pypi.org/project/pytest-cov/) to measure test coverage. To generate a simple coverage report use the following command:
+
+```bash
+uv run pytest --cov=planar
+```
+
 ### Pre-commit hooks
 
 We use [pre-commit](https://pre-commit.com/) to manage pre-commit hooks. To install the pre-commit hooks, run the following command: