tokenome 0.1.0__py3-none-any.whl

tokenome/__init__.py

@@ -0,0 +1,17 @@
+from .client import TokenLens
+from .context import (
+    clear_context,
+    clear_default_context,
+    get_context,
+    set_context,
+    set_default_context,
+)
+
+__all__ = [
+    "TokenLens",
+    "set_context",
+    "set_default_context",
+    "get_context",
+    "clear_context",
+    "clear_default_context",
+]

tokenome/client.py

@@ -0,0 +1,350 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+from dataclasses import dataclass
+from importlib import import_module
+from importlib.metadata import PackageNotFoundError, version
+from typing import Any
+
+from .config import (
+    DEFAULT_DROP_POLICY,
+    DEFAULT_DURABILITY,
+    DEFAULT_ENDPOINT,
+    DEFAULT_MAX_SPOOL_AGE_DAYS,
+    DEFAULT_MAX_SPOOL_BYTES,
+    DEFAULT_QUEUE_MAXSIZE,
+    DEFAULT_RETRY_MAX_DELAY_SECONDS,
+    DEFAULT_TIMEOUT_SECONDS,
+    TokenomeConfig,
+)
+from .context import clear_context, clear_default_context
+from .delivery.sender import HttpSender
+from .delivery.worker import DeliveryWorker
+from .errors import MissingOptionalDependencyError, UnsupportedProviderVersionError
+from .limits import DEFAULT_MAX_PAYLOAD_BYTES
+from .models import TelemetryEvent, WrapperDefaults
+from .providers.openai import OPENAI_SUPPORTED_VERSION_RANGE
+from .serialization import sanitize_tags
+from .spool import SQLiteEventSpool
+from .spool.memory import MemorySpool
+from .wrapper import wrap_provider_client
+
+logger = logging.getLogger("tokenome")
+
+_ENV_API_KEY = "TOKENLENS_API_KEY"
+_ENV_ENDPOINT = "TOKENLENS_ENDPOINT"
+_ENV_DEBUG = "TOKENLENS_DEBUG"
+_ENV_ENABLED = "TOKENLENS_ENABLED"
+_ENV_TAGS = "TOKENLENS_TAGS"
+_ENV_TIMEOUT = "TOKENLENS_TIMEOUT_SECONDS"
+_ENV_QUEUE_MAXSIZE = "TOKENLENS_QUEUE_MAXSIZE"
+_ENV_PROJECT_ID = "TOKENLENS_PROJECT_ID"
+_ENV_ENVIRONMENT = "TOKENLENS_ENVIRONMENT"
+_ENV_DURABILITY = "TOKENLENS_DURABILITY"
+_ENV_SPOOL_PATH = "TOKENLENS_SPOOL_PATH"
+_ENV_MAX_SPOOL_BYTES = "TOKENLENS_MAX_SPOOL_BYTES"
+_ENV_MAX_SPOOL_AGE_DAYS = "TOKENLENS_MAX_SPOOL_AGE_DAYS"
+_ENV_RETRY_MAX_DELAY_SECONDS = "TOKENLENS_RETRY_MAX_DELAY_SECONDS"
+_ENV_DROP_POLICY = "TOKENLENS_DROP_POLICY"
+
+_TRUTHY = {"1", "true", "t", "yes", "y", "on"}
+_FALSY = {"0", "false", "f", "no", "n", "off"}
+_ALLOWED_DURABILITY = {"durable_local", "best_effort", "agent"}
+_ALLOWED_DROP_POLICY = {"drop_oldest", "drop_newest", "block"}
+
+
+@dataclass(slots=True)
+class _TokenLensState:
+    config: TokenomeConfig
+    spool: SQLiteEventSpool | MemorySpool
+    sender: HttpSender
+    worker: DeliveryWorker
+
+    def enqueue(self, event: TelemetryEvent) -> None:
+        if not self.config.enabled:
+            return
+        try:
+            stored = self.spool.append(event)
+        except Exception as exc:
+            # Spool corruption or unexpected error — log at debug and drop.
+            logger.debug("spool append failed for event %s: %s", event.event_id, exc)
+            return
+        if stored:
+            self.worker.notify_event_available()
+        else:
+            # Capacity drop — spool is full or event oversized.
+            logger.debug("spool dropped event %s (capacity or size limit)", event.event_id)
+
+    def flush(self) -> bool:
+        return self.worker.flush(timeout=10.0)
+
+    def close(self) -> None:
+        self.worker.close(timeout=2.0)
+        self.sender.close()
+        self.spool.close()
+
+
+_default_client: TokenLens | None = None
+
+
+class TokenLens:
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        project_id: str | None = None,
+        environment: str | None = None,
+        endpoint: str = DEFAULT_ENDPOINT,
+        enabled: bool = True,
+        debug: bool = False,
+        timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
+        queue_maxsize: int = DEFAULT_QUEUE_MAXSIZE,
+        durability: str = DEFAULT_DURABILITY,
+        spool_path: str | None = None,
+        max_spool_bytes: int = DEFAULT_MAX_SPOOL_BYTES,
+        max_spool_age_days: int = DEFAULT_MAX_SPOOL_AGE_DAYS,
+        retry_max_delay_seconds: float = DEFAULT_RETRY_MAX_DELAY_SECONDS,
+        drop_policy: str = DEFAULT_DROP_POLICY,
+        default_tags: dict[str, object] | None = None,
+    ) -> None:
+        self.project_id = project_id
+        self.environment = environment
+        normalized_durability = _parse_choice(
+            durability,
+            env_name="durability",
+            allowed=_ALLOWED_DURABILITY,
+        )
+        normalized_drop_policy = _parse_choice(
+            drop_policy,
+            env_name="drop_policy",
+            allowed=_ALLOWED_DROP_POLICY,
+        )
+        if normalized_durability == "agent":
+            raise ValueError("durability='agent' is documented future mode and not implemented yet")
+
+        sanitized_default_tags = sanitize_tags(default_tags) or {}
+        config = TokenomeConfig(
+            api_key=api_key,
+            endpoint=endpoint,
+            enabled=enabled,
+            debug=debug,
+            timeout_seconds=timeout_seconds,
+            queue_maxsize=queue_maxsize,
+            durability=normalized_durability,  # type: ignore[arg-type]
+            spool_path=spool_path,
+            max_spool_bytes=max_spool_bytes,
+            max_spool_age_days=max_spool_age_days,
+            retry_max_delay_seconds=retry_max_delay_seconds,
+            drop_policy=normalized_drop_policy,  # type: ignore[arg-type]
+            default_tags=sanitized_default_tags,
+        )
+
+        if normalized_durability == "durable_local":
+            spool: SQLiteEventSpool | MemorySpool = SQLiteEventSpool(
+                path=spool_path,
+                max_bytes=max_spool_bytes,
+                max_age_days=max_spool_age_days,
+                drop_policy=normalized_drop_policy,  # type: ignore[arg-type]
+            )
+        else:
+            spool = MemorySpool(max_size=queue_maxsize)
+
+        sender = HttpSender(
+            endpoint=endpoint,
+            api_key=api_key,
+            timeout_seconds=timeout_seconds,
+            debug=debug,
+        )
+        worker = DeliveryWorker(
+            spool=spool,
+            sender=sender,
+            max_payload_bytes=DEFAULT_MAX_PAYLOAD_BYTES,
+            shutdown_timeout_seconds=2.0,
+        )
+        worker.start()
+
+        self._state = _TokenLensState(
+            config=config,
+            spool=spool,
+            sender=sender,
+            worker=worker,
+        )
+
+    @property
+    def config(self) -> TokenomeConfig:
+        return self._state.config
+
+    @classmethod
+    def init(cls, **kwargs: Any) -> TokenLens:
+        global _default_client
+        if _default_client is not None:
+            _default_client.close()
+        _default_client = cls(**kwargs)
+        return _default_client
+
+    @classmethod
+    def init_from_env(cls) -> TokenLens:
+        api_key = os.getenv(_ENV_API_KEY)
+        if not api_key:
+            raise ValueError(f"Missing required environment variable: {_ENV_API_KEY}")
+        return cls.init(
+            api_key=api_key,
+            project_id=os.getenv(_ENV_PROJECT_ID),
+            environment=os.getenv(_ENV_ENVIRONMENT),
+            endpoint=os.getenv(_ENV_ENDPOINT, DEFAULT_ENDPOINT),
+            enabled=_parse_bool(os.getenv(_ENV_ENABLED), default=True),
+            debug=_parse_bool(os.getenv(_ENV_DEBUG), default=False),
+            timeout_seconds=float(os.getenv(_ENV_TIMEOUT, str(DEFAULT_TIMEOUT_SECONDS))),
+            queue_maxsize=int(os.getenv(_ENV_QUEUE_MAXSIZE, "10000")),
+            durability=os.getenv(_ENV_DURABILITY, DEFAULT_DURABILITY),
+            spool_path=os.getenv(_ENV_SPOOL_PATH),
+            max_spool_bytes=int(os.getenv(_ENV_MAX_SPOOL_BYTES, str(DEFAULT_MAX_SPOOL_BYTES))),
+            max_spool_age_days=int(
+                os.getenv(_ENV_MAX_SPOOL_AGE_DAYS, str(DEFAULT_MAX_SPOOL_AGE_DAYS))
+            ),
+            retry_max_delay_seconds=float(
+                os.getenv(_ENV_RETRY_MAX_DELAY_SECONDS, str(DEFAULT_RETRY_MAX_DELAY_SECONDS))
+            ),
+            drop_policy=os.getenv(_ENV_DROP_POLICY, DEFAULT_DROP_POLICY),
+            default_tags=_parse_tags(os.getenv(_ENV_TAGS)),
+        )
+
+    @classmethod
+    def is_initialized(cls) -> bool:
+        return _default_client is not None
+
+    def wrap(
+        self,
+        client: Any,
+        *,
+        provider: str | None = None,
+        route: str | None = None,
+        feature: str | None = None,
+        user_id: str | None = None,
+        user_id_hash: str | None = None,
+        session_id: str | None = None,
+        tags: dict[str, object] | None = None,
+    ) -> Any:
+        defaults = WrapperDefaults(
+            provider=provider or _guess_provider(client),
+            project_id=self.project_id,
+            environment=self.environment,
+            route=route,
+            feature=feature,
+            user_id=user_id,
+            user_id_hash=user_id_hash,
+            session_id=session_id,
+            tags={**self.config.default_tags, **(sanitize_tags(tags) or {})},
+        )
+        return wrap_provider_client(client, self._state.enqueue, defaults)
+
+    def wrap_openai(self, client: Any, **context_overrides: Any) -> Any:
+        return self.wrap(client, provider="openai", **context_overrides)
+
+    def OpenAI(self, *args: Any, **kwargs: Any) -> Any:
+        openai = _import_openai_module()
+        _ensure_supported_openai_version()
+        client = openai.OpenAI(*args, **kwargs)
+        return self.wrap_openai(client)
+
+    def AsyncOpenAI(self, *args: Any, **kwargs: Any) -> Any:
+        openai = _import_openai_module()
+        _ensure_supported_openai_version()
+        client = openai.AsyncOpenAI(*args, **kwargs)
+        return self.wrap_openai(client)
+
+    def flush(self) -> bool:
+        return self._state.flush()
+
+    def close(self) -> None:
+        global _default_client
+        self._state.close()
+        if _default_client is self:
+            _default_client = None
+        clear_context()
+        clear_default_context()
+
+
+def get_state() -> _TokenLensState | None:
+    if _default_client is None:
+        return None
+    return _default_client._state
+
+
+def _parse_bool(raw: str | None, *, default: bool) -> bool:
+    if raw is None:
+        return default
+    normalized = raw.strip().lower()
+    if normalized in _TRUTHY:
+        return True
+    if normalized in _FALSY:
+        return False
+    raise ValueError(f"Invalid boolean value: {raw}")
+
+
+def _parse_tags(raw: str | None) -> dict[str, object] | None:
+    if not raw:
+        return None
+    data = json.loads(raw)
+    if not isinstance(data, dict):
+        raise ValueError(f"{_ENV_TAGS} must be a JSON object")
+    return dict(data)
+
+
+def _parse_choice(raw: str, *, env_name: str, allowed: set[str]) -> str:
+    normalized = raw.strip().lower()
+    if normalized not in allowed:
+        options = ", ".join(sorted(allowed))
+        raise ValueError(f"Invalid {env_name} value: {raw}. Expected one of: {options}")
+    return normalized
+
+
+def _guess_provider(client: Any) -> str:
+    module = type(client).__module__.lower()
+    if "anthropic" in module:
+        return "anthropic"
+    if "openai" in module:
+        return "openai"
+    if hasattr(client, "responses") or (
+        hasattr(client, "chat") and hasattr(client.chat, "completions")
+    ):
+        return "openai-compatible"
+    return type(client).__name__.lower()
+
+
+def _import_openai_module() -> Any:
+    try:
+        return import_module("openai")
+    except ModuleNotFoundError as exc:
+        raise MissingOptionalDependencyError(
+            'OpenAI integration requires `pip install "tokenome[openai]"`.'
+        ) from exc
+
+
+def _ensure_supported_openai_version() -> str:
+    try:
+        installed = version("openai")
+    except PackageNotFoundError as exc:
+        raise MissingOptionalDependencyError(
+            'OpenAI integration requires `pip install "tokenome[openai]"`.'
+        ) from exc
+
+    major = _major_version(installed)
+    if major < 2:
+        raise UnsupportedProviderVersionError(
+            "Unsupported openai version "
+            f"`{installed}`. Supported range: {OPENAI_SUPPORTED_VERSION_RANGE}."
+        )
+    if major >= 3:
+        raise UnsupportedProviderVersionError(
+            "Untested openai version "
+            f"`{installed}`. Supported range: {OPENAI_SUPPORTED_VERSION_RANGE}."
+        )
+    return installed
+
+
+def _major_version(raw: str) -> int:
+    head = raw.split(".", maxsplit=1)[0]
+    return int(head)

tokenome/config.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Literal
+
+DEFAULT_ENDPOINT = "https://api.tokenome.ai/v1/events/batch"
+DEFAULT_QUEUE_MAXSIZE = 10_000
+DEFAULT_TIMEOUT_SECONDS = 1.5
+DEFAULT_DURABILITY: Literal["durable_local", "best_effort", "agent"] = "durable_local"
+DEFAULT_MAX_SPOOL_BYTES = 100 * 1024 * 1024
+DEFAULT_MAX_SPOOL_AGE_DAYS = 7
+DEFAULT_RETRY_MAX_DELAY_SECONDS = 900.0
+DEFAULT_DROP_POLICY: Literal["drop_oldest", "drop_newest", "block"] = "drop_oldest"
+
+
+@dataclass(slots=True, frozen=True)
+class TokenomeConfig:
+    """Immutable configuration for Tokenome SDK.
+
+    Batch sizing and flush interval are not exposed as constructor parameters
+    because they are part of the SDK/server contract. Changing them risks
+    413 Payload Too Large, rate limits, or server-side rejection.
+    """
+
+    api_key: str
+    endpoint: str = DEFAULT_ENDPOINT
+    enabled: bool = True
+    debug: bool = False
+    timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS
+    queue_maxsize: int = DEFAULT_QUEUE_MAXSIZE
+    durability: Literal["durable_local", "best_effort", "agent"] = DEFAULT_DURABILITY
+    spool_path: str | None = None
+    max_spool_bytes: int = DEFAULT_MAX_SPOOL_BYTES
+    max_spool_age_days: int = DEFAULT_MAX_SPOOL_AGE_DAYS
+    retry_max_delay_seconds: float = DEFAULT_RETRY_MAX_DELAY_SECONDS
+    drop_policy: Literal["drop_oldest", "drop_newest", "block"] = DEFAULT_DROP_POLICY
+    default_tags: dict[str, object] = field(default_factory=dict)

tokenome/context.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+from contextvars import ContextVar
+from dataclasses import dataclass, field
+
+
+@dataclass(slots=True)
+class TrackingContext:
+    route: str | None = None
+    feature: str | None = None
+    user_id: str | None = None
+    user_id_hash: str | None = None
+    session_id: str | None = None
+    tags: dict[str, object] = field(default_factory=dict)
+
+
+_ctx: ContextVar[TrackingContext | None] = ContextVar("tokenome_context", default=None)
+_default_context = TrackingContext()
+
+
+def _merge(base: TrackingContext, override: TrackingContext | None) -> TrackingContext:
+    if override is None:
+        override = TrackingContext()
+    return TrackingContext(
+        route=override.route if override.route is not None else base.route,
+        feature=override.feature if override.feature is not None else base.feature,
+        user_id=override.user_id if override.user_id is not None else base.user_id,
+        user_id_hash=(
+            override.user_id_hash if override.user_id_hash is not None else base.user_id_hash
+        ),
+        session_id=override.session_id if override.session_id is not None else base.session_id,
+        tags={**base.tags, **override.tags},
+    )
+
+
+def set_context(
+    *,
+    route: str | None = None,
+    feature: str | None = None,
+    user_id: str | None = None,
+    user_id_hash: str | None = None,
+    session_id: str | None = None,
+    tags: dict[str, object] | None = None,
+) -> None:
+    _ctx.set(
+        TrackingContext(
+            route=route,
+            feature=feature,
+            user_id=user_id,
+            user_id_hash=user_id_hash,
+            session_id=session_id,
+            tags=dict(tags or {}),
+        )
+    )
+
+
+def set_default_context(
+    *,
+    route: str | None = None,
+    feature: str | None = None,
+    user_id: str | None = None,
+    user_id_hash: str | None = None,
+    session_id: str | None = None,
+    tags: dict[str, object] | None = None,
+) -> None:
+    global _default_context
+    _default_context = TrackingContext(
+        route=route,
+        feature=feature,
+        user_id=user_id,
+        user_id_hash=user_id_hash,
+        session_id=session_id,
+        tags=dict(tags or {}),
+    )
+
+
+def get_context() -> TrackingContext:
+    return _merge(_default_context, _ctx.get())
+
+
+def clear_context() -> None:
+    _ctx.set(None)
+
+
+def clear_default_context() -> None:
+    global _default_context
+    _default_context = TrackingContext()

tokenome/delivery/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from .result import SendResult
+from .sender import HttpSender, SenderProtocol
+from .worker import DeliveryWorker
+
+__all__ = ["DeliveryWorker", "HttpSender", "SendResult", "SenderProtocol"]

tokenome/delivery/result.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(slots=True)
+class SendResult:
+    """Result of a batch send attempt."""
+
+    ok: bool = False
+    retryable: bool = False
+    status_code: int | None = None
+    error: str | None = None
+    retry_after_seconds: int | None = None
+    extra: dict[str, Any] | None = None
+    acked: list[str] | None = None
+    dropped: list[str] | None = None
+    retryable_ids: list[str] | None = None
+
+    @classmethod
+    def success(
+        cls, *, acked: list[str] | None = None, status_code: int | None = None
+    ) -> SendResult:
+        return cls(ok=True, retryable=False, status_code=status_code, error=None, acked=acked or [])
+
+    @classmethod
+    def retryable_error(
+        cls,
+        *,
+        status_code: int | None = None,
+        error: str | None = None,
+        retry_after_seconds: int | None = None,
+        retryable_ids: list[str] | None = None,
+    ) -> SendResult:
+        return cls(
+            ok=False,
+            retryable=True,
+            status_code=status_code,
+            error=error,
+            retry_after_seconds=retry_after_seconds,
+            retryable_ids=retryable_ids or [],
+        )
+
+    @classmethod
+    def fatal_error(
+        cls,
+        *,
+        status_code: int | None = None,
+        error: str | None = None,
+        dropped: list[str] | None = None,
+    ) -> SendResult:
+        return cls(
+            ok=False, retryable=False, status_code=status_code, error=error, dropped=dropped or []
+        )