swiftpy-framework 0.1.0__py3-none-any.whl

swiftpy/cache/drivers/redis.py

@@ -0,0 +1,421 @@
+from __future__ import annotations
+
+import asyncio
+import inspect
+import json
+import logging
+import math
+import time
+import uuid
+from abc import ABC, abstractmethod
+from collections.abc import Awaitable, Callable
+from datetime import date, datetime
+from datetime import time as dt_time
+from decimal import Decimal
+from enum import Enum
+from typing import Any
+from uuid import UUID
+
+import redis.asyncio as aioredis
+from redis.exceptions import RedisError, ResponseError
+
+from swiftpy.cache.driver import _SENTINEL, CacheDriver
+from swiftpy.support.redis_config import resolve_redis_params
+
+logger = logging.getLogger("swiftpy.cache")
+
+_INCR_WITH_TTL_LUA = """
+local existed = redis.call('exists', KEYS[1])
+local current = redis.call('incrby', KEYS[1], ARGV[1])
+local ttl = tonumber(ARGV[2])
+if existed == 0 and ttl and ttl > 0 then
+    redis.call('pexpire', KEYS[1], math.max(1, math.floor(ttl * 1000)))
+end
+return current
+"""
+
+
+class CacheSerializer(ABC):
+    """Abstract interface for pluggable cache serializers."""
+
+    @abstractmethod
+    def serialize(self, value: Any) -> str:
+        """Convert a Python value into a string representation for caching."""
+        ...
+
+    @abstractmethod
+    def deserialize(self, raw: str) -> Any:
+        """Reconstruct a Python value from its cached string representation."""
+        ...
+
+
+def _cache_json_default(obj: Any) -> Any:
+    if isinstance(obj, (bytes, bytearray, memoryview)):
+        raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable")
+    if isinstance(obj, datetime):
+        return {"__swiftpy_type__": "datetime", "val": obj.isoformat()}
+    if isinstance(obj, date):
+        return {"__swiftpy_type__": "date", "val": obj.isoformat()}
+    if isinstance(obj, dt_time):
+        return {"__swiftpy_type__": "time", "val": obj.isoformat()}
+    if isinstance(obj, UUID):
+        return {"__swiftpy_type__": "uuid", "val": str(obj)}
+    if isinstance(obj, Decimal):
+        return {"__swiftpy_type__": "decimal", "val": str(obj)}
+    if isinstance(obj, Enum):
+        return obj.value
+    if hasattr(obj, "to_dict") and callable(obj.to_dict):
+        raw_dict = obj.to_dict()
+        if isinstance(raw_dict, dict):
+            hidden = set(getattr(obj, "__hidden__", ()) or ())
+            if hidden:
+                return {k: v for k, v in raw_dict.items() if k not in hidden}
+            return raw_dict
+        return raw_dict
+    raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable")
+
+
+def _cache_json_object_hook(dct: dict[str, Any]) -> Any:
+    if "__swiftpy_type__" in dct and "val" in dct and len(dct) == 2:
+        t = dct["__swiftpy_type__"]
+        v = dct["val"]
+        if t == "datetime":
+            return datetime.fromisoformat(v)
+        if t == "date":
+            return date.fromisoformat(v)
+        if t == "time":
+            return dt_time.fromisoformat(v)
+        if t == "uuid":
+            return UUID(v)
+        if t == "decimal":
+            return Decimal(v)
+    return dct
+
+
+class JsonCacheSerializer(CacheSerializer):
+    """Default JSON serializer supporting datetime, UUID, Decimal, and models."""
+
+    def serialize(self, value: Any) -> str:
+        return json.dumps(
+            value,
+            default=_cache_json_default,
+            ensure_ascii=False,
+            separators=(",", ":"),
+        )
+
+    def deserialize(self, raw: str) -> Any:
+        return json.loads(raw, object_hook=_cache_json_object_hook)
+
+
+class RedisDriver(CacheDriver):
+    """
+    Redis cache driver utilizing redis.asyncio connection pooling.
+    """
+
+    def __init__(
+        self,
+        redis_client: aioredis.Redis | None = None,
+        connection_pool: aioredis.ConnectionPool | None = None,
+        redis_url: str | None = None,
+        host: str | None = None,
+        port: int | None = None,
+        password: str | None = None,
+        database: int = 0,
+        prefix: str = "swiftpy_cache:",
+        default_ttl: float | None = None,
+        max_connections: int | None = None,
+        timeout: float = 30.0,
+        serializer: CacheSerializer | None = None,
+    ) -> None:
+        super().__init__()
+        self._client = redis_client
+        self._pool = connection_pool
+        self._redis_url = redis_url
+        self.host = host
+        self.port = port
+        self.password = password
+        self.database = database
+        self.prefix = prefix
+        self.default_ttl = default_ttl
+        self.serializer = (
+            serializer if serializer is not None else JsonCacheSerializer()
+        )
+        self.max_connections = max_connections
+        self.timeout = timeout
+        self._owns_client = redis_client is None
+        self._owns_pool = connection_pool is None and redis_client is None
+        self._client_lock = asyncio.Lock()
+
+    async def get_client(self) -> aioredis.Redis:
+        """Lazily initialize and return the async Redis client."""
+        if self._client is not None:
+            return self._client
+        async with self._client_lock:
+            if self._client is not None:
+                return self._client
+            if self._redis_url:
+                self._client = aioredis.from_url(
+                    self._redis_url,
+                    decode_responses=True,
+                )
+            else:
+                if self._pool is None:
+                    params = resolve_redis_params(
+                        section="cache",
+                        host=self.host,
+                        port=self.port,
+                        password=self.password,
+                        database=self.database,
+                    )
+                    max_conns = self.max_connections or 50
+
+                    self._pool = aioredis.BlockingConnectionPool(
+                        host=params["host"],
+                        port=params["port"],
+                        db=params["database"],
+                        password=params["password"] or None,
+                        decode_responses=True,
+                        max_connections=max_conns,
+                        timeout=self.timeout,
+                    )
+                self._client = aioredis.Redis(
+                    connection_pool=self._pool,
+                    decode_responses=True,
+                )
+            self._owns_client = True
+            return self._client
+
+    def _prefixed_key(self, key: str) -> str:
+        return f"{self.prefix}{key}"
+
+    def _serialize(self, value: Any) -> str:
+        return self.serializer.serialize(value)
+
+    def _deserialize(self, raw: str | bytes | None, default: Any) -> Any:
+        if raw is None:
+            return default
+        if isinstance(raw, bytes):
+            raw = raw.decode("utf-8")
+        try:
+            return self.serializer.deserialize(raw)
+        except (json.JSONDecodeError, TypeError, ValueError, UnicodeDecodeError):
+            return raw
+
+    async def get(self, key: str, default: Any = None) -> Any:
+        client = await self.get_client()
+        raw = await client.get(self._prefixed_key(key))
+        return self._deserialize(raw, default)
+
+    async def set(
+        self,
+        key: str,
+        value: Any,
+        ttl: float | None = None,
+    ) -> bool:
+        client = await self.get_client()
+        full_key = self._prefixed_key(key)
+        effective_ttl = ttl if ttl is not None else self.default_ttl
+
+        if effective_ttl is not None and effective_ttl <= 0:
+            await client.delete(full_key)
+            return True
+
+        serialized = self._serialize(value)
+        if effective_ttl is not None:
+            if isinstance(effective_ttl, float) and not effective_ttl.is_integer():
+                await client.set(full_key, serialized, px=int(effective_ttl * 1000))
+            else:
+                await client.set(full_key, serialized, ex=int(effective_ttl))
+        else:
+            await client.set(full_key, serialized)
+
+        return True
+
+    async def forget(self, key: str) -> bool:
+        client = await self.get_client()
+        count = await client.delete(self._prefixed_key(key))
+        return bool(count > 0)
+
+    async def has(self, key: str) -> bool:
+        client = await self.get_client()
+        count = await client.exists(self._prefixed_key(key))
+        return bool(count > 0)
+
+    async def flush(self) -> bool:
+        client = await self.get_client()
+        pattern = f"{self.prefix}*"
+        batch: list[str] = []
+        async for key in client.scan_iter(match=pattern, count=500):
+            batch.append(key.decode("utf-8") if isinstance(key, bytes) else str(key))
+            if len(batch) >= 500:
+                await client.unlink(*batch)
+                batch.clear()
+
+        if batch:
+            await client.unlink(*batch)
+        return True
+
+    async def add(
+        self,
+        key: str,
+        value: Any,
+        ttl: float | None = None,
+    ) -> bool:
+        client = await self.get_client()
+        full_key = self._prefixed_key(key)
+        effective_ttl = ttl if ttl is not None else self.default_ttl
+
+        if effective_ttl is not None and effective_ttl <= 0:
+            exists = await client.exists(full_key)
+            return not bool(exists)
+
+        serialized = self._serialize(value)
+        if effective_ttl is not None:
+            if isinstance(effective_ttl, float) and not effective_ttl.is_integer():
+                res = await client.set(
+                    full_key,
+                    serialized,
+                    px=int(effective_ttl * 1000),
+                    nx=True,
+                )
+            else:
+                res = await client.set(
+                    full_key,
+                    serialized,
+                    ex=int(effective_ttl),
+                    nx=True,
+                )
+        else:
+            res = await client.set(full_key, serialized, nx=True)
+
+        return bool(res)
+
+    async def increment(
+        self, key: str, value: int = 1, ttl: float | None = None
+    ) -> int:
+        client = await self.get_client()
+        full_key = self._prefixed_key(key)
+        effective_ttl = ttl if ttl is not None else self.default_ttl
+        try:
+            if effective_ttl is not None and effective_ttl > 0:
+                res = await client.eval(
+                    _INCR_WITH_TTL_LUA, 1, full_key, value, effective_ttl
+                )
+                return int(res)
+            new_val = await client.incrby(full_key, value)
+            return int(new_val)
+        except ResponseError as exc:
+            raise TypeError(
+                f"Cannot increment non-numeric cached value: {exc}"
+            ) from exc
+
+    async def decrement(
+        self, key: str, value: int = 1, ttl: float | None = None
+    ) -> int:
+        return await self.increment(key, -value, ttl)
+
+    async def remember(
+        self,
+        key: str,
+        ttl: float | None,
+        callback: Callable[[], Any] | Callable[[], Awaitable[Any]],
+        lock_timeout: float = 10.0,
+    ) -> Any:
+        """
+        Get an item from Redis, or execute callback with in-process coalescing and distributed lock protection.
+
+        Uses both per-key asyncio lock coalescing and Redis SET NX (millisecond PX expiration)
+        so concurrent requests across coroutines or processes invoke callback at most once.
+        Even if ``lock_timeout <= 0``, attempts a single non-blocking SET NX acquisition before falling back.
+        """
+        # Fast path
+        cached = await self.get(key, _SENTINEL)
+        if cached is not _SENTINEL:
+            return cached
+
+        async with self._acquire_key_lock(key, lock_timeout=lock_timeout) as local_acquired:
+            if not local_acquired:
+                logger.warning(
+                    "Cache lock timed out waiting for key '%s'; computing value directly without caching",
+                    key,
+                )
+                if inspect.iscoroutinefunction(callback):
+                    return await callback()
+                res = callback()
+                return await res if inspect.isawaitable(res) else res
+
+            # Re-check after winning the in-process key lock
+            cached = await self.get(key, _SENTINEL)
+            if cached is not _SENTINEL:
+                return cached
+
+            client = await self.get_client()
+            lock_key = f"{self.prefix}lock:{key}"
+            token = uuid.uuid4().hex
+            lock_ttl_ms = (
+                max(1, math.ceil(float(lock_timeout) * 1000))
+                if lock_timeout > 0
+                else 1000
+            )
+            deadline = time.monotonic() + max(0.0, float(lock_timeout))
+
+            while True:
+                # Try to acquire distributed lock with millisecond precision
+                acquired = await client.set(
+                    lock_key,
+                    token,
+                    px=lock_ttl_ms,
+                    nx=True,
+                )
+                if acquired:
+                    try:
+                        # Double-checked locking
+                        cached = await self.get(key, _SENTINEL)
+                        if cached is not _SENTINEL:
+                            return cached
+
+                        if inspect.iscoroutinefunction(callback):
+                            result = await callback()
+                        else:
+                            res = callback()
+                            result = await res if inspect.isawaitable(res) else res
+
+                        await self.set(key, result, ttl)
+                        return result
+                    finally:
+                        lua_release = (
+                            "if redis.call('get', KEYS[1]) == ARGV[1] then "
+                            "return redis.call('del', KEYS[1]) "
+                            "else return 0 end"
+                        )
+                        try:
+                            await client.eval(lua_release, 1, lock_key, token)
+                        except RedisError as exc:
+                            logger.debug("Failed to release lock %s: %s", lock_key, exc)
+
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    break
+
+                await asyncio.sleep(min(0.02, remaining))
+                cached = await self.get(key, _SENTINEL)
+                if cached is not _SENTINEL:
+                    return cached
+
+            # Fallback if timeout elapsed: compute directly without caching and log warning
+            logger.warning(
+                "Cache lock timed out waiting for key '%s'; computing value directly without caching",
+                key,
+            )
+            if inspect.iscoroutinefunction(callback):
+                return await callback()
+            res = callback()
+            return await res if inspect.isawaitable(res) else res
+
+    async def close(self) -> None:
+        if self._client is not None and self._owns_client:
+            await self._client.aclose()
+            self._client = None
+        if self._owns_pool and self._pool is not None:
+            await self._pool.disconnect()
+            self._pool = None

swiftpy/console/cli.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import argparse
+import sys
+
+from swiftpy.console.commands.cache import CacheCommand
+from swiftpy.console.commands.make import MakeCommand
+from swiftpy.console.commands.migrate import MigrateCommand
+from swiftpy.console.commands.new import NewCommand
+from swiftpy.console.commands.queue import QueueCommand
+from swiftpy.console.commands.seed import SeedCommand
+from swiftpy.console.commands.start import StartCommand
+from swiftpy.console.commands.storage import StorageCommand
+
+
+class CLI:
+    """
+    SwiftPY command line entrypoint.
+
+    Responsible for:
+    - Parsing arguments
+    - Dispatching commands
+    - Delegating execution
+    """
+
+    def __init__(self) -> None:
+        self.parser = argparse.ArgumentParser(
+            prog="swiftpy",
+            description="SwiftPY Framework CLI",
+        )
+
+        self.subparsers = self.parser.add_subparsers(
+            dest="command",
+            required=True,
+        )
+
+        self._register_commands()
+
+    def _register_commands(self) -> None:
+        """Register all framework commands."""
+        NewCommand.register(self.subparsers)
+        StartCommand.register(self.subparsers)
+        MigrateCommand.register(self.subparsers)
+        SeedCommand.register(self.subparsers)
+        MakeCommand.register(self.subparsers)
+        QueueCommand.register(self.subparsers)
+        CacheCommand.register(self.subparsers)
+        StorageCommand.register(self.subparsers)
+
+    def run(self, args: list[str] | None = None) -> None:
+        """Parse arguments and dispatch."""
+        parsed_args = self.parser.parse_args(args)
+
+        handler = getattr(parsed_args, "_handler", None)
+
+        if handler is None:
+            self.parser.print_help()
+            return
+
+        handler(parsed_args)
+
+
+def main() -> None:
+    CLI().run(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()

swiftpy/console/commands/cache.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+from swiftpy.ai.prompts.loader import PromptLoader
+from swiftpy.console.loader import load_application
+from swiftpy.core.config import Config, ConfigurationError
+
+
+def _paint(text: str, code: str) -> str:
+    """Format text with ANSI escape codes only if stdout is a terminal and NO_COLOR is unset."""
+    if sys.stdout.isatty() and "NO_COLOR" not in os.environ:
+        return f"\033[{code}m{text}\033[0m"
+    return text
+
+
+class CacheCommand:
+    """Console commands for building and clearing precompiled application caches."""
+
+    @staticmethod
+    def register(
+        subparsers: argparse._SubParsersAction[argparse.ArgumentParser],
+    ) -> None:
+        # 1. `swiftpy cache:build`
+        build_parser = subparsers.add_parser(
+            "cache:build",
+            help="Precompile routes, configuration, and prompt templates into serialized cache files",
+        )
+        CacheCommand._add_build_arguments(build_parser)
+        build_parser.set_defaults(_handler=CacheCommand.handle_build)
+
+        # 2. `swiftpy cache:clear`
+        clear_parser = subparsers.add_parser(
+            "cache:clear",
+            help="Clear precompiled route, configuration, and prompt caches",
+        )
+        CacheCommand._add_clear_arguments(clear_parser)
+        clear_parser.set_defaults(_handler=CacheCommand.handle_clear)
+
+        # 3. `swiftpy cache` group with `build` and `clear` subcommands
+        cache_group_parser = subparsers.add_parser(
+            "cache",
+            help="Manage application caches",
+        )
+        cache_subparsers = cache_group_parser.add_subparsers(
+            dest="cache_subcommand",
+            required=True,
+        )
+
+        sub_build = cache_subparsers.add_parser(
+            "build",
+            help="Precompile routes, configuration, and prompt templates",
+        )
+        CacheCommand._add_build_arguments(sub_build)
+        sub_build.set_defaults(_handler=CacheCommand.handle_build)
+
+        sub_clear = cache_subparsers.add_parser(
+            "clear",
+            help="Clear precompiled caches",
+        )
+        CacheCommand._add_clear_arguments(sub_clear)
+        sub_clear.set_defaults(_handler=CacheCommand.handle_clear)
+
+    @staticmethod
+    def _add_build_arguments(parser: argparse.ArgumentParser) -> None:
+        parser.add_argument(
+            "--app",
+            default="main:app",
+            help="Application ASGI target (default: 'main:app')",
+        )
+        parser.add_argument(
+            "--cache-dir",
+            default="bootstrap/cache",
+            help="Cache directory path (default: 'bootstrap/cache')",
+        )
+
+    @staticmethod
+    def _add_clear_arguments(parser: argparse.ArgumentParser) -> None:
+        parser.add_argument(
+            "--cache-dir",
+            default="bootstrap/cache",
+            help="Cache directory path (default: 'bootstrap/cache')",
+        )
+
+    @staticmethod
+    def handle_build(args: argparse.Namespace) -> None:
+        cache_dir = Path(getattr(args, "cache_dir", "bootstrap/cache"))
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        target_app = getattr(args, "app", "main:app")
+
+        # Clean prior cache to ensure clean compilation
+        for f in (
+            cache_dir / "config.json",
+            cache_dir / "prompts.json",
+        ):
+            if f.is_file():
+                f.unlink()
+
+        sys.stdout.write(
+            _paint(f"Building application cache into '{cache_dir}'...\n", "1;34")
+        )
+
+        # 1. Load application instance
+        try:
+            app = load_application(target_app)
+        except (
+            ImportError,
+            AttributeError,
+            TypeError,
+            ConfigurationError,
+            OSError,
+            ValueError,
+            RuntimeError,
+            SyntaxError,
+        ) as exc:
+            sys.stderr.write(
+                _paint(f"Error loading application '{target_app}': {exc}\n", "1;31")
+            )
+            raise SystemExit(1) from exc
+
+        # 2. Precompile Config
+        config_cache_file = cache_dir / "config.json"
+        app.config.dump_cache(cache_path=config_cache_file)
+        sys.stdout.write(
+            f"  {_paint('✓', '1;32')} Configuration precompiled -> {config_cache_file}\n"
+        )
+
+        # 3. Precompile Prompts
+        prompts_cache_file = cache_dir / "prompts.json"
+        prompt_loader = PromptLoader.get_instance()
+        prompt_loader.dump_cache(cache_path=prompts_cache_file)
+        sys.stdout.write(
+            f"  {_paint('✓', '1;32')} Prompt templates precompiled -> {prompts_cache_file}\n"
+        )
+
+        sys.stdout.write(
+            _paint(
+                "✨ Application cache built successfully for production deployment.\n",
+                "1;32",
+            )
+        )
+
+    @staticmethod
+    def handle_clear(args: argparse.Namespace) -> None:
+        cache_dir = Path(getattr(args, "cache_dir", "bootstrap/cache"))
+
+        config_file = cache_dir / "config.json"
+        prompts_file = cache_dir / "prompts.json"
+
+        Config.clear_cache(config_file)
+        PromptLoader.clear_compiled_cache(prompts_file)
+
+        sys.stdout.write(
+            _paint(
+                f"✨ Application cache cleared successfully from '{cache_dir}'.\n",
+                "1;32",
+            )
+        )
+
+
+__all__ = ["CacheCommand"]