bot-knows 0.1.0__py3-none-any.whl

bot_knows/services/recall_service.py

@@ -0,0 +1,296 @@
+"""Recall service for bot_knows.
+
+This module provides the evidence-weighted recall service with
+spaced repetition-style decay and reinforcement.
+"""
+
+import math
+import time
+from typing import Literal
+
+from bot_knows.interfaces.graph import GraphServiceInterface
+from bot_knows.interfaces.storage import StorageInterface
+from bot_knows.logging import get_logger
+from bot_knows.models.recall import RecallItemDTO, TopicRecallStateDTO
+
+__all__ = [
+    "CONTEXT_WEIGHTS",
+    "RecallService",
+]
+
+logger = get_logger(__name__)
+
+# Context weights for reinforcement
+CONTEXT_WEIGHTS: dict[str, float] = {
+    "passive": 0.2,  # Passive read (background access)
+    "active": 0.6,  # Active query (user explicitly asked)
+    "recall": 1.0,  # Recall prompt (spaced repetition review)
+}
+
+
+class RecallService:
+    """Evidence-weighted recall service.
+
+    Implements a spaced repetition-inspired recall system with:
+    - Time-based decay: strength *= exp(-Δt / (stability * 86400))
+    - Reinforcement: strength += confidence * novelty * context_weight
+    - Stability growth: stability += k * confidence
+    - Semantic reinforcement: boost related topics
+
+    Example:
+        service = RecallService(storage, graph)
+
+        # Reinforce when topic is accessed
+        state = await service.reinforce(topic_id, confidence=0.9, context="active")
+
+        # Get topics due for review
+        due_topics = await service.get_due_topics(threshold=0.3)
+    """
+
+    def __init__(
+        self,
+        storage: StorageInterface,
+        graph: GraphServiceInterface,
+        stability_k: float = 0.1,
+        semantic_boost: float = 0.1,
+    ) -> None:
+        """Initialize service with dependencies.
+
+        Args:
+            storage: Storage interface for recall states
+            graph: Graph interface for related topics
+            stability_k: Factor for stability growth on reinforcement
+            semantic_boost: Factor for boosting related topics
+        """
+        self._storage = storage
+        self._graph = graph
+        self._stability_k = stability_k
+        self._semantic_boost = semantic_boost
+
+    async def reinforce(
+        self,
+        topic_id: str,
+        confidence: float,
+        novelty_factor: float = 1.0,
+        context: Literal["passive", "active", "recall"] = "passive",
+    ) -> TopicRecallStateDTO:
+        """Reinforce a topic's recall strength.
+
+        Formulas:
+            delta = confidence * novelty_factor * context_weight
+            strength = min(1.0, strength + delta)
+            stability += k * confidence
+
+        Also boosts semantically related topics.
+
+        Args:
+            topic_id: Topic to reinforce
+            confidence: Evidence confidence (0.0-1.0)
+            novelty_factor: How novel this reinforcement is
+            context: Interaction context (passive/active/recall)
+
+        Returns:
+            Updated TopicRecallStateDTO
+        """
+        now = int(time.time())
+
+        # Get or create state
+        state = await self._storage.get_recall_state(topic_id)
+        if not state:
+            state = TopicRecallStateDTO(
+                topic_id=topic_id,
+                strength=0.0,
+                last_seen=now,
+                last_updated=now,
+                stability=1.0,
+            )
+
+        # Apply decay first
+        state = self._apply_decay(state, now)
+
+        # Calculate reinforcement
+        context_weight = CONTEXT_WEIGHTS.get(context, 0.2)
+        delta = confidence * novelty_factor * context_weight
+        new_strength = min(1.0, state.strength + delta)
+        new_stability = state.stability + self._stability_k * confidence
+
+        # Create updated state
+        new_state = TopicRecallStateDTO(
+            topic_id=topic_id,
+            strength=new_strength,
+            last_seen=now,
+            last_updated=now,
+            stability=new_stability,
+        )
+
+        # Save state
+        await self._storage.save_recall_state(new_state)
+
+        # Boost related topics
+        await self._boost_related_topics(topic_id, state.strength)
+
+        logger.debug(
+            "topic_reinforced",
+            topic_id=topic_id,
+            old_strength=state.strength,
+            new_strength=new_strength,
+            context=context,
+        )
+
+        return new_state
+
+    async def apply_decay(
+        self,
+        topic_id: str,
+        current_time: int | None = None,
+    ) -> TopicRecallStateDTO | None:
+        """Apply time-based decay to a topic.
+
+        Formula: strength *= exp(-Δt / (stability * 86400))
+
+        Args:
+            topic_id: Topic to decay
+            current_time: Current time (epoch seconds), default: now
+
+        Returns:
+            Updated state or None if topic has no state
+        """
+        state = await self._storage.get_recall_state(topic_id)
+        if not state:
+            return None
+
+        now = current_time or int(time.time())
+        new_state = self._apply_decay(state, now)
+
+        await self._storage.save_recall_state(new_state)
+        return new_state
+
+    async def batch_decay_update(self) -> int:
+        """Apply decay to all topics (scheduled task).
+
+        Returns:
+            Number of topics updated
+        """
+        now = int(time.time())
+        states = await self._storage.get_all_recall_states()
+
+        updated = 0
+        for state in states:
+            new_state = self._apply_decay(state, now)
+            if new_state.strength != state.strength:
+                await self._storage.save_recall_state(new_state)
+                updated += 1
+
+        logger.info("batch_decay_completed", updated_count=updated)
+        return updated
+
+    async def get_due_topics(
+        self,
+        threshold: float = 0.3,
+        limit: int = 10,
+    ) -> list[RecallItemDTO]:
+        """Get topics due for recall review.
+
+        Topics with strength below threshold are considered due.
+        Results are sorted by due_score (higher = more urgent).
+
+        Args:
+            threshold: Strength threshold for being "due"
+            limit: Maximum number of topics
+
+        Returns:
+            List of RecallItemDTO sorted by priority
+        """
+        states = await self._storage.get_due_topics(threshold)
+
+        items: list[RecallItemDTO] = []
+        for state in states[:limit]:
+            topic = await self._storage.get_topic(state.topic_id)
+            if not topic:
+                continue
+
+            # Get related topics
+            related = await self._graph.get_related_topics(state.topic_id, limit=5)
+
+            # Calculate due score
+            due_score = self._calculate_due_score(state)
+
+            items.append(
+                RecallItemDTO(
+                    topic=topic,
+                    recall_state=state,
+                    due_score=due_score,
+                    related_topics=[r[0] for r in related],
+                )
+            )
+
+        # Sort by due_score descending
+        items.sort(key=lambda x: x.due_score, reverse=True)
+        return items
+
+    def _apply_decay(
+        self,
+        state: TopicRecallStateDTO,
+        current_time: int,
+    ) -> TopicRecallStateDTO:
+        """Apply time-based decay to state.
+
+        Formula: strength *= exp(-Δt / (stability * 86400))
+        """
+        delta_t = current_time - state.last_updated
+        if delta_t <= 0:
+            return state
+
+        # stability is in days, so multiply by seconds per day
+        decay_factor = math.exp(-delta_t / (state.stability * 86400))
+        new_strength = state.strength * decay_factor
+
+        return TopicRecallStateDTO(
+            topic_id=state.topic_id,
+            strength=new_strength,
+            last_seen=state.last_seen,
+            last_updated=current_time,
+            stability=state.stability,
+        )
+
+    async def _boost_related_topics(
+        self,
+        topic_id: str,
+        source_strength: float,
+    ) -> None:
+        """Boost strength of related topics.
+
+        Formula: strength += source_strength * edge_weight * semantic_boost
+        """
+        related = await self._graph.get_related_topics(topic_id)
+
+        for related_id, edge_weight in related:
+            state = await self._storage.get_recall_state(related_id)
+            if not state:
+                continue
+
+            boost = source_strength * edge_weight * self._semantic_boost
+            new_strength = min(1.0, state.strength + boost)
+
+            if new_strength > state.strength:
+                new_state = TopicRecallStateDTO(
+                    topic_id=related_id,
+                    strength=new_strength,
+                    last_seen=state.last_seen,
+                    last_updated=state.last_updated,
+                    stability=state.stability,
+                )
+                await self._storage.save_recall_state(new_state)
+
+    def _calculate_due_score(self, state: TopicRecallStateDTO) -> float:
+        """Calculate priority score for recall review.
+
+        Higher score = more urgent for review.
+        Factors: lower strength, older last_seen
+        """
+        now = time.time()
+        age_days = (now - state.last_seen) / 86400
+
+        # Lower strength = higher priority
+        # Older = higher priority
+        return (1.0 - state.strength) * (1.0 + age_days * 0.1)

bot_knows/services/tasks.py

@@ -0,0 +1,128 @@
+"""Task orchestration for bot_knows.
+
+This module provides Taskiq-based background task definitions
+for async processing of imports, extractions, and scheduled jobs.
+"""
+
+from typing import Any
+
+from bot_knows.logging import get_logger
+
+__all__ = [
+    "create_broker",
+    "create_scheduler",
+]
+
+logger = get_logger(__name__)
+
+
+def create_broker(redis_url: str) -> Any:
+    """Create Taskiq broker for background tasks.
+
+    Args:
+        redis_url: Redis URL for task queue
+
+    Returns:
+        Configured TaskiqRedisStreamBroker
+
+    Example:
+        broker = create_broker("redis://localhost:6379")
+
+        @broker.task
+        async def my_task():
+            ...
+    """
+    try:
+        from taskiq_redis import RedisStreamBroker
+
+        broker = RedisStreamBroker(url=redis_url)
+        logger.info("taskiq_broker_created", url=redis_url)
+        return broker
+    except ImportError:
+        logger.warning("taskiq_redis_not_installed")
+        return None
+
+
+def create_scheduler(broker: Any) -> Any:
+    """Create Taskiq scheduler for periodic tasks.
+
+    Args:
+        broker: Taskiq broker instance
+
+    Returns:
+        Configured TaskiqScheduler
+    """
+    if broker is None:
+        return None
+
+    try:
+        from taskiq import TaskiqScheduler
+
+        scheduler = TaskiqScheduler(broker)
+        logger.info("taskiq_scheduler_created")
+        return scheduler
+    except ImportError:
+        logger.warning("taskiq_not_installed")
+        return None
+
+
+# Task definitions (to be registered with broker)
+# These are placeholder implementations - actual implementation
+# requires a running broker instance.
+
+
+async def process_chat_import_task(
+    source: str,
+    raw_export: dict[str, Any],
+) -> dict[str, Any]:
+    """Background task to process chat import.
+
+    Args:
+        source: Import source identifier
+        raw_export: Raw export data
+
+    Returns:
+        Import result summary
+    """
+    # Import orchestration would go here
+    # This requires dependency injection of services
+    logger.info("process_chat_import_task", source=source)
+    return {"status": "completed", "source": source}
+
+
+async def extract_topics_task(
+    message_id: str,
+) -> dict[str, Any]:
+    """Background task to extract topics from message.
+
+    Args:
+        message_id: Message ID to process
+
+    Returns:
+        Extraction result summary
+    """
+    logger.info("extract_topics_task", message_id=message_id)
+    return {"status": "completed", "message_id": message_id}
+
+
+async def batch_decay_task() -> dict[str, Any]:
+    """Scheduled task to update decay for all topics.
+
+    This should be scheduled to run periodically (e.g., daily).
+
+    Returns:
+        Decay update summary
+    """
+    logger.info("batch_decay_task_started")
+    # Would call RecallService.batch_decay_update()
+    return {"status": "completed"}
+
+
+# Example of how to set up scheduled tasks with a broker:
+#
+# broker = create_broker("redis://localhost:6379")
+# scheduler = create_scheduler(broker)
+#
+# if scheduler:
+#     # Run decay update every 24 hours at midnight
+#     scheduler.schedule(batch_decay_task, cron="0 0 * * *")

bot_knows/services/topic_extraction.py

@@ -0,0 +1,199 @@
+"""Topic extraction service for bot_knows.
+
+This module provides the service for extracting topics from messages.
+"""
+
+import time
+
+from bot_knows.interfaces.embedding import EmbeddingServiceInterface
+from bot_knows.interfaces.llm import LLMInterface
+from bot_knows.logging import get_logger
+from bot_knows.models.message import MessageDTO
+from bot_knows.models.topic import TopicDTO, TopicEvidenceDTO
+from bot_knows.utils.hashing import generate_evidence_id, generate_topic_id
+
+__all__ = [
+    "TopicCandidate",
+    "TopicExtractionService",
+]
+
+logger = get_logger(__name__)
+
+
+class TopicCandidate:
+    """Represents a candidate topic extracted from a message.
+
+    This is an intermediate representation before deduplication.
+    """
+
+    def __init__(
+        self,
+        extracted_name: str,
+        confidence: float,
+        embedding: list[float],
+        source_message_id: str,
+    ) -> None:
+        self.extracted_name = extracted_name
+        self.confidence = confidence
+        self.embedding = embedding
+        self.source_message_id = source_message_id
+
+
+class TopicExtractionService:
+    """Service for extracting topics from messages.
+
+    Uses LLM for topic extraction and embedding service for
+    generating embeddings for semantic matching.
+
+    Example:
+        service = TopicExtractionService(llm, embedding_service)
+        candidates = await service.extract(message)
+    """
+
+    def __init__(
+        self,
+        llm: LLMInterface,
+        embedding_service: EmbeddingServiceInterface,
+    ) -> None:
+        """Initialize service with dependencies.
+
+        Args:
+            llm: LLM interface for topic extraction
+            embedding_service: Embedding service for vector generation
+        """
+        self._llm = llm
+        self._embedding = embedding_service
+
+    async def extract(self, message: MessageDTO) -> list[TopicCandidate]:
+        """Extract topic candidates from a message.
+
+        Args:
+            message: Message to extract topics from
+
+        Returns:
+            List of TopicCandidate objects
+        """
+        if message.is_empty:
+            return []
+
+        # Extract raw topics using LLM
+        try:
+            raw_topics = await self._llm.extract_topics(
+                message.user_content,
+                message.assistant_content,
+            )
+        except Exception as e:
+            logger.warning("topic_extraction_failed", error=str(e))
+            return []
+
+        if not raw_topics:
+            return []
+
+        # Generate embeddings for all topics
+        topic_names = [name for name, _ in raw_topics]
+        try:
+            embeddings = await self._embedding.embed_batch(topic_names)
+        except Exception as e:
+            logger.warning("embedding_generation_failed", error=str(e))
+            return []
+
+        # Create candidates
+        candidates = []
+        for (name, confidence), embedding in zip(raw_topics, embeddings, strict=False):
+            candidate = TopicCandidate(
+                extracted_name=name,
+                confidence=confidence,
+                embedding=embedding,
+                source_message_id=message.message_id,
+            )
+            candidates.append(candidate)
+
+        logger.debug(
+            "topics_extracted",
+            message_id=message.message_id,
+            count=len(candidates),
+        )
+
+        return candidates
+
+    async def create_topic_and_evidence(
+        self,
+        candidate: TopicCandidate,
+        canonical_name: str | None = None,
+    ) -> tuple[TopicDTO, TopicEvidenceDTO]:
+        """Create new TopicDTO and TopicEvidenceDTO from candidate.
+
+        Args:
+            candidate: Topic candidate
+            canonical_name: Normalized name (if None, uses extracted_name)
+
+        Returns:
+            Tuple of (TopicDTO, TopicEvidenceDTO)
+        """
+        name = canonical_name or await self._llm.normalize_topic_name(candidate.extracted_name)
+        now = int(time.time())
+
+        topic_id = generate_topic_id(name, candidate.source_message_id)
+        evidence_id = generate_evidence_id(
+            topic_id=topic_id,
+            extracted_name=candidate.extracted_name,
+            source_message_id=candidate.source_message_id,
+            timestamp=now,
+        )
+
+        topic = TopicDTO(
+            topic_id=topic_id,
+            canonical_name=name,
+            centroid_embedding=candidate.embedding,
+            evidence_count=1,
+            importance=candidate.confidence * 0.1,  # Initial importance
+            recall_strength=0.0,
+        )
+
+        evidence = TopicEvidenceDTO(
+            evidence_id=evidence_id,
+            topic_id=topic_id,
+            extracted_name=candidate.extracted_name,
+            source_message_id=candidate.source_message_id,
+            confidence=candidate.confidence,
+            timestamp=now,
+        )
+
+        return topic, evidence
+
+    async def create_evidence_for_existing(
+        self,
+        candidate: TopicCandidate,
+        existing_topic: TopicDTO,
+    ) -> tuple[TopicDTO, TopicEvidenceDTO]:
+        """Create evidence for existing topic and update centroid.
+
+        Args:
+            candidate: Topic candidate
+            existing_topic: Existing topic to link to
+
+        Returns:
+            Tuple of (updated TopicDTO, new TopicEvidenceDTO)
+        """
+        now = int(time.time())
+
+        evidence_id = generate_evidence_id(
+            topic_id=existing_topic.topic_id,
+            extracted_name=candidate.extracted_name,
+            source_message_id=candidate.source_message_id,
+            timestamp=now,
+        )
+
+        evidence = TopicEvidenceDTO(
+            evidence_id=evidence_id,
+            topic_id=existing_topic.topic_id,
+            extracted_name=candidate.extracted_name,
+            source_message_id=candidate.source_message_id,
+            confidence=candidate.confidence,
+            timestamp=now,
+        )
+
+        # Update topic centroid
+        updated_topic = existing_topic.with_updated_centroid(candidate.embedding)
+
+        return updated_topic, evidence

bot_knows/utils/__init__.py

@@ -0,0 +1,22 @@
+"""Utility functions for bot_knows.
+
+This module contains internal utility functions.
+"""
+
+from bot_knows.utils.hashing import (
+    generate_chat_id,
+    generate_evidence_id,
+    generate_message_id,
+    generate_topic_id,
+    hash_text,
+    stable_hash,
+)
+
+__all__ = [
+    "generate_chat_id",
+    "generate_evidence_id",
+    "generate_message_id",
+    "generate_topic_id",
+    "hash_text",
+    "stable_hash",
+]