forktex-core 0.1.0__tar.gz

forktex_core-0.1.0/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: forktex-core
+Version: 0.1.0
+Summary: Shared database (PostgreSQL) and cache (Redis) primitives for the FORKTEX ecosystem
+Requires-Python: >=3.11
+Requires-Dist: asyncpg>=0.29
+Requires-Dist: pydantic>=2.0
+Requires-Dist: sqlalchemy[asyncio]>=2.0
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.7.0; extra == 'dev'
+Provides-Extra: redis
+Requires-Dist: redis[hiredis]>=5.0; extra == 'redis'

forktex_core-0.1.0/README.md

@@ -0,0 +1,183 @@
+# forktex-core
+
+Shared async database (PostgreSQL) and cache (Redis) primitives for the [FORKTEX](https://forktex.com) ecosystem.
+
+Extracted from battle-tested patterns across 4 production APIs (Network, Cloud, Intelligence, Workflow) into a single reusable library.
+
+## Install
+
+```bash
+pip install forktex-core            # PostgreSQL only
+pip install forktex-core[redis]     # PostgreSQL + Redis
+```
+
+## Architecture
+
+```
+forktex-core
+  psql/                             # PostgreSQL async primitives
+    connection.py                   # Engine init, session management, transactional decorator
+    models.py                       # BaseDBModel, TimestampMixin, AuditMixin, JsonModelColumn
+    crud.py                         # PageResponse, ScrollResponse, paginate, get, create, find_one_by
+  redis/                            # Redis async caching
+    connection.py                   # Init, close, health check
+    ops.py                          # get/set/delete, fetch-or-set, stale-while-revalidate
+    decorators.py                   # @cached decorator with TTL + SWR support
+    serialization.py                # Pydantic-aware JSON serialization
+    namespaces.py                   # Key prefix management
+```
+
+## PostgreSQL (`forktex_core.psql`)
+
+### Connection
+
+```python
+from forktex_core.psql import init_engine, close_engine, get_session, with_transactional_session
+
+# Initialize — accepts any SQLAlchemy engine kwargs
+init_engine("postgresql+asyncpg://user:pass@localhost/db")
+
+# With pool tuning (as used by Intelligence API)
+init_engine("postgresql+asyncpg://...", pool_size=20, max_overflow=10, pool_pre_ping=True)
+
+# FastAPI lifespan
+@asynccontextmanager
+async def lifespan(app):
+    init_engine(settings.db_url)
+    yield
+    await close_engine()
+
+# Session — auto-commits on success, rolls back on error
+async with get_session() as session:
+    user = await session.get(User, user_id)
+
+# Decorator — auto-injects session if not provided
+@with_transactional_session
+async def create_user(session: AsyncSession, email: str) -> User:
+    ...
+```
+
+### Models
+
+```python
+from forktex_core.psql import BaseDBModel, TimestampMixin, AuditMixin, JsonModelColumn
+
+# Basic model with timestamps
+class Project(BaseDBModel, TimestampMixin):
+    __tablename__ = "project"
+    id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
+    name: Mapped[str] = mapped_column(String(255))
+
+# Full audit trail with soft delete
+class Task(BaseDBModel, AuditMixin):
+    __tablename__ = "task"
+    unique_fields = ("project_id", "name")  # partial unique index on active rows
+
+    id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
+    project_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("project.id"))
+    name: Mapped[str] = mapped_column(String(255))
+```
+
+**AuditMixin** provides:
+- `created_at` / `updated_at` (server defaults)
+- `created_by_id` / `updated_by_id` (plain UUID, no FK — consumers add their own)
+- `archived_at` / `is_active` (soft delete with consistency constraint)
+- Partial unique index on `unique_fields` (active records only)
+
+### CRUD
+
+```python
+from forktex_core.psql import paginate, paginate_scroll, get, find_one_by, create, list_all
+
+# Paginated query with total count
+page = await paginate(session, Task, page=1, page_size=20,
+                      conditions=[Task.project_id == project_id],
+                      order_by=[Task.created_at.desc()])
+# page.data, page.has_more, page.total_count, page.total_pages
+
+# Cursor-based scroll (no COUNT query)
+scroll = await paginate_scroll(session, Task, limit=50)
+# scroll.data, scroll.has_more
+
+# Single record
+task = await get(session, Task, task_id)
+task = await find_one_by(session, Task, name="my-task", project_id=pid)
+
+# Create with conflict detection
+try:
+    task = await create(session, Task, name="new", project_id=pid)
+except ConflictError:
+    ...  # unique constraint violated
+```
+
+## Redis (`forktex_core.redis`)
+
+### Connection
+
+```python
+from forktex_core.redis import init, close, available, get_client
+
+await init("redis://localhost:6379/0")
+assert available()
+await close()
+```
+
+### Caching
+
+```python
+from forktex_core.redis import cached, ops
+
+# Simple TTL cache
+@cached(ttl=300)
+async def get_profile(user_id: str) -> dict:
+    ...
+
+# Stale-while-revalidate (fast reads, background refresh)
+@cached(ttl=60, stale_ttl=300)
+async def get_feed(org_id: str) -> dict:
+    ...
+
+# Manual ops
+await ops.set("key", "value", ex=60)
+value = await ops.get("key")
+await ops.invalidate_prefix("user:")
+```
+
+### Namespaces
+
+```python
+from enum import StrEnum
+from forktex_core.redis.namespaces import key_for
+
+class CachePrefix(StrEnum):
+    USER = "user"
+    FEED = "feed"
+
+key = key_for(CachePrefix.USER, user_id)  # "user:abc-123"
+```
+
+## Consumers
+
+| Project | Uses |
+|---------|------|
+| [Network API](https://network.forktex.com) | psql (connection, models, AuditMixin, crud) + redis (all) |
+| [Cloud API](https://cloud.forktex.com) | psql (connection, models, crud) |
+| [Intelligence API](https://intelligence.forktex.com) | psql (connection with pool tuning, models) |
+| [Workflow API](https://workflow.forktex.com) | psql (connection, models, crud) |
+| [forktex-workflow SDK](../workflow/sdk) | psql (models as base for ORM) |
+
+## Testing
+
+Tests use real PostgreSQL and Redis via [testcontainers](https://testcontainers-python.readthedocs.io/):
+
+```bash
+make test-core   # 23 tests — real PG + Redis containers
+```
+
+## Part of forktex-py
+
+This package is maintained in the [forktex-py monorepo](https://github.com/forktex/forktex-python) alongside:
+- `forktex` — CLI agent framework
+- `forktex-documents` — Jinja2 + WeasyPrint rendering
+- `forktex-intelligence` — Intelligence API SDK
+- `forktex-cloud` — Cloud platform SDK

forktex_core-0.1.0/pyproject.toml

@@ -0,0 +1,34 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "forktex-core"
+version = "0.1.0"
+description = "Shared database (PostgreSQL) and cache (Redis) primitives for the FORKTEX ecosystem"
+requires-python = ">=3.11"
+dependencies = [
+    "sqlalchemy[asyncio]>=2.0",
+    "asyncpg>=0.29",
+    "pydantic>=2.0",
+]
+
+[project.optional-dependencies]
+redis = ["redis[hiredis]>=5.0"]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.24",
+    "ruff>=0.7.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/forktex_core"]
+
+[tool.ruff]
+target-version = "py311"
+line-length = 120
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "session"
+asyncio_default_test_loop_scope = "session"

forktex_core-0.1.0/src/forktex_core/__init__.py

@@ -0,0 +1,3 @@
+"""FORKTEX Core — shared database and cache primitives."""
+
+__version__ = "0.1.0"

forktex_core-0.1.0/src/forktex_core/psql/__init__.py

@@ -0,0 +1,47 @@
+"""PostgreSQL async primitives: connection, base models, CRUD utilities."""
+
+from forktex_core.psql.connection import (
+    init_engine,
+    close_engine,
+    get_session,
+    with_transactional_session,
+)
+from forktex_core.psql.models import (
+    BaseDBModel,
+    ReprMixin,
+    TimestampMixin,
+    AuditMixin,
+    JsonModelColumn,
+)
+from forktex_core.psql.crud import (
+    PageResponse,
+    ScrollResponse,
+    ConflictError,
+    get,
+    list_all,
+    paginate,
+    paginate_scroll,
+    find_one_by,
+    create,
+)
+
+__all__ = [
+    "init_engine",
+    "close_engine",
+    "get_session",
+    "with_transactional_session",
+    "BaseDBModel",
+    "ReprMixin",
+    "TimestampMixin",
+    "AuditMixin",
+    "JsonModelColumn",
+    "PageResponse",
+    "ScrollResponse",
+    "ConflictError",
+    "get",
+    "list_all",
+    "paginate",
+    "paginate_scroll",
+    "find_one_by",
+    "create",
+]

forktex_core-0.1.0/src/forktex_core/psql/connection.py

@@ -0,0 +1,112 @@
+"""Async SQLAlchemy engine and session management for PostgreSQL.
+
+Unified connection module ForkTex Ecosystem api Archtypes
+Supports configurable engine kwargs for pool tuning.
+
+Usage:
+    # Basic (network/cloud style — default pool settings)
+    init_engine("postgresql+asyncpg://user:pass@host/db")
+
+    # With pool tuning (intelligence style)
+    init_engine(
+        "postgresql+asyncpg://user:pass@host/db",
+        pool_size=20,
+        max_overflow=10,
+        pool_pre_ping=True,
+    )
+
+    # FastAPI lifespan
+    @asynccontextmanager
+    async def lifespan(app):
+        init_engine(settings.db_url)
+        yield
+        await close_engine()
+
+    # Route handler
+    async def my_route(session: AsyncSession = Depends(get_session)):
+        ...
+
+    # Service layer
+    @with_transactional_session
+    async def my_service(session: AsyncSession, ...):
+        ...
+"""
+
+from contextlib import asynccontextmanager
+import functools
+from typing import AsyncGenerator, Optional
+
+from sqlalchemy.ext.asyncio import (
+    async_sessionmaker,
+    create_async_engine,
+    AsyncSession,
+    AsyncEngine,
+)
+
+
+# Module-level references (set by init_engine).
+engine: Optional[AsyncEngine] = None
+_async_sessionmaker: Optional[async_sessionmaker] = None
+
+
+def init_engine(db_url: str, *, echo: bool = False, **engine_kwargs) -> async_sessionmaker:
+    """Initialize the async engine and session factory.
+
+    Args:
+        db_url: SQLAlchemy async database URL (e.g. postgresql+asyncpg://...).
+        echo: Echo SQL statements to stdout.
+        **engine_kwargs: Forwarded to create_async_engine (pool_size, max_overflow,
+            pool_pre_ping, pool_recycle, etc.).
+
+    Returns:
+        The configured async_sessionmaker.
+    """
+    global engine, _async_sessionmaker
+    engine = create_async_engine(db_url, echo=echo, **engine_kwargs)
+    _async_sessionmaker = async_sessionmaker(bind=engine, expire_on_commit=False, class_=AsyncSession)
+    return _async_sessionmaker
+
+
+async def close_engine() -> None:
+    """Dispose the engine on app shutdown."""
+    global engine
+    if engine is not None:
+        await engine.dispose()
+        engine = None
+
+
+@asynccontextmanager
+async def get_session() -> AsyncGenerator[AsyncSession, None]:
+    """Yield a transactional async session. Commits on success, rolls back on error.
+
+    Can be used as a FastAPI dependency via ``Depends(get_session)`` or as a
+    plain async context manager in service code.
+    """
+    assert _async_sessionmaker is not None, "Engine/sessionmaker not initialized — call init_engine() first"
+    async with _async_sessionmaker() as session:
+        try:
+            yield session
+            await session.commit()
+        except Exception:
+            await session.rollback()
+            raise
+
+
+def with_transactional_session(func):
+    """Decorator that auto-injects a session as the first arg if not already provided.
+
+    If the first positional arg is already an ``AsyncSession``, or ``session``
+    is in kwargs, the function is called as-is. Otherwise a new session is
+    created via ``get_session()``.
+    """
+
+    @functools.wraps(func)
+    async def wrapper(*args, **kwargs):
+        if args and isinstance(args[0], AsyncSession):
+            return await func(*args, **kwargs)
+        if "session" in kwargs:
+            return await func(*args, **kwargs)
+        async with get_session() as session:
+            return await func(session, *args, **kwargs)
+
+    return wrapper

forktex_core-0.1.0/src/forktex_core/psql/crud.py

@@ -0,0 +1,253 @@
+"""Generic async CRUD utilities for SQLAlchemy.
+
+Provides paginated queries (page-based and cursor-based), single-record
+lookups, and creation helpers with conflict detection.
+"""
+
+from math import ceil
+from typing import Any, Callable, Generic, List, Optional, Type, TypeVar
+
+from pydantic import BaseModel
+from sqlalchemy import ColumnElement, Select, func, select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import InstrumentedAttribute
+
+
+T = TypeVar("T")
+U = TypeVar("U")
+
+
+# ---------------------------------------------------------------------------
+# Response containers
+# ---------------------------------------------------------------------------
+
+
+class PageResponse(BaseModel, Generic[T]):
+    """Page-based pagination response."""
+
+    data: list[T]
+    has_more: bool
+    limit: int
+    total_count: Optional[int] = None
+    current_page: Optional[int] = None
+    total_pages: Optional[int] = None
+
+    def apply_to_page_data(self, map_func: Callable[[T], U]) -> "PageResponse[U]":
+        return PageResponse[U](
+            data=[map_func(item) for item in self.data],
+            has_more=self.has_more,
+            limit=self.limit,
+            total_count=self.total_count,
+            current_page=self.current_page,
+            total_pages=self.total_pages,
+        )
+
+
+class ScrollResponse(BaseModel, Generic[T]):
+    """Cursor-based (scroll) pagination response."""
+
+    data: list[T]
+    limit: int
+    has_more: bool
+    next_cursor: str | None = None
+
+    def apply_to_scroll_data(self, map_func: Callable[[T], U]) -> "ScrollResponse[U]":
+        return ScrollResponse[U](
+            data=[map_func(item) for item in self.data],
+            limit=self.limit,
+            has_more=self.has_more,
+            next_cursor=self.next_cursor,
+        )
+
+
+# ---------------------------------------------------------------------------
+# Exceptions
+# ---------------------------------------------------------------------------
+
+
+class ConflictError(Exception):
+    """Raised when an INSERT/UPDATE violates a uniqueness constraint."""
+
+    pass
+
+
+# ---------------------------------------------------------------------------
+# Single-record operations
+# ---------------------------------------------------------------------------
+
+
+async def get(
+    session: AsyncSession,
+    model: Type[T],
+    value: Any,
+    *,
+    key: str = "id",
+    options: list | None = None,
+) -> T | None:
+    """Retrieve a single record by column value (default: primary key)."""
+    if not hasattr(model, key):
+        raise AttributeError(f"{model.__name__} has no attribute '{key}'")
+
+    stmt = select(model).where(getattr(model, key) == value)
+    if options:
+        stmt = stmt.options(*options)
+    result = await session.execute(stmt)
+    return result.scalar_one_or_none()
+
+
+async def find_one_by(
+    session: AsyncSession,
+    model: Type[T],
+    **filters,
+) -> Optional[T]:
+    """Find a single record matching all keyword filters."""
+    stmt = select(model).filter_by(**filters)
+    res = await session.execute(stmt)
+    return res.scalar_one_or_none()
+
+
+async def list_all(
+    session: AsyncSession,
+    model: Type[T],
+    *,
+    options: list | None = None,
+) -> List[T]:
+    """Return all records of a model (use sparingly on large tables)."""
+    stmt = select(model)
+    if options:
+        stmt = stmt.options(*options)
+    res = await session.execute(stmt)
+    return list(res.scalars().all())
+
+
+async def create(
+    session: AsyncSession,
+    model: Type[T],
+    **values,
+) -> T:
+    """Create a new record. Raises ``ConflictError`` on integrity violation."""
+    obj = model(**values)
+    session.add(obj)
+    try:
+        await session.flush()
+    except IntegrityError as exc:
+        raise ConflictError(str(exc)) from exc
+    await session.refresh(obj)
+    return obj
+
+
+# ---------------------------------------------------------------------------
+# Paginated queries
+# ---------------------------------------------------------------------------
+
+
+async def paginate(
+    session: AsyncSession,
+    model: Type[T],
+    page: int = 1,
+    page_size: int = 100,
+    conditions: Optional[List[ColumnElement]] = None,
+    order_by: Optional[list[ColumnElement]] = None,
+    joins: Optional[list[InstrumentedAttribute]] = None,
+    options: list | None = None,
+) -> "PageResponse[T]":
+    """Page-based pagination with total count."""
+    stmt = select(model)
+
+    if options:
+        stmt = stmt.options(*options)
+    if joins:
+        for join in joins:
+            stmt = stmt.outerjoin(join)
+    if conditions:
+        stmt = stmt.where(*conditions)
+    if order_by:
+        stmt = stmt.order_by(*order_by)
+
+    return await _paginate_query(session, stmt, page, page_size)
+
+
+async def paginate_scroll(
+    session: AsyncSession,
+    model: Type[T],
+    limit: int = 20,
+    conditions: Optional[List[ColumnElement]] = None,
+    order_by: Optional[list[ColumnElement]] = None,
+    joins: Optional[list[InstrumentedAttribute]] = None,
+    options: list | None = None,
+) -> "ScrollResponse[T]":
+    """Cursor-based (scroll) pagination. Fetches limit+1 to detect has_more."""
+    stmt = select(model)
+
+    if options:
+        stmt = stmt.options(*options)
+    if joins:
+        for join in joins:
+            stmt = stmt.outerjoin(join)
+    if conditions:
+        stmt = stmt.where(*conditions)
+    if order_by:
+        stmt = stmt.order_by(*order_by)
+
+    return await _paginate_scroll_query(session, stmt, limit)
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+async def _paginate_query(
+    session: AsyncSession,
+    query: Select[Any],
+    page: int = 1,
+    page_size: int = 100,
+) -> "PageResponse[T]":
+    if page < 1:
+        page = 1
+    if page_size < 1:
+        page_size = 10
+
+    total_count_stmt = select(func.count()).select_from(query.subquery())
+    total_count = (await session.execute(total_count_stmt)).scalar_one()
+
+    offset = (page - 1) * page_size
+    paginated_stmt = query.offset(offset).limit(page_size)
+    result = await session.execute(paginated_stmt)
+    data = list(result.scalars().all())
+
+    has_more = (page * page_size) < total_count
+    total_pages = ceil(total_count / page_size) if page_size else 1
+
+    return PageResponse(
+        data=data,
+        has_more=has_more,
+        limit=page_size,
+        total_count=total_count,
+        current_page=page,
+        total_pages=total_pages,
+    )
+
+
+async def _paginate_scroll_query(
+    session: AsyncSession,
+    query: Select[Any],
+    limit: int = 20,
+) -> "ScrollResponse[T]":
+    if limit < 1:
+        limit = 20
+
+    scroll_stmt = query.limit(limit + 1)
+    result = await session.execute(scroll_stmt)
+    data = list(result.scalars().all())
+
+    has_more = len(data) > limit
+    if has_more:
+        data = data[:limit]
+
+    return ScrollResponse(
+        data=data,
+        has_more=has_more,
+        limit=limit,
+    )