reme-ai 0.1.0__py3-none-any.whl

reme_ai/summary/personal/load_today_memory_op.py

@@ -0,0 +1,117 @@
+from typing import List
+
+from flowllm import C, BaseLLMOp
+from flowllm.schema.vector_node import VectorNode
+from loguru import logger
+
+from reme_ai.schema.memory import BaseMemory, vector_node_to_memory
+from reme_ai.utils.datetime_handler import DatetimeHandler
+
+
+@C.register_op()
+class LoadTodayMemoryOp(BaseLLMOp):
+    """
+    Operation to load today's memories from vector store for deduplication.
+    Focuses specifically on retrieving and deduplicating memories from the current date.
+    """
+    file_path: str = __file__
+
+    def execute(self):
+        """
+        Load today's memories from vector store and perform deduplication.
+        
+        This operation:
+        1. Retrieves memories from today using vector store search
+        2. Converts vector nodes to memory objects
+        3. Performs deduplication based on content similarity
+        4. Stores deduplicated memories in context
+        """
+        # Get operation parameters
+        top_k = self.op_params.get("top_k", 50)
+
+        # Get required context values
+        workspace_id = self.context.workspace_id
+        user_name = self.context.get("user_name", "user")
+
+        logger.info(f"Loading today's memories for user: {user_name} (top_k: {top_k})")
+
+        # Get today's memories from vector store
+        today_memories = self._retrieve_today_memories(workspace_id, user_name, top_k)
+
+        if not today_memories:
+            logger.info("No memories found for today")
+            self.context.today_memories = []
+            return
+
+        logger.info(f"Retrieved {len(today_memories)} memories from today")
+        self.context.today_memories = today_memories
+        logger.info(f"Final today's memory list size: {len(today_memories)}")
+
+    def _retrieve_today_memories(self, workspace_id: str, user_name: str, top_k: int) -> List[BaseMemory]:
+        """
+        Retrieve memories from today using vector store with date filtering.
+        
+        Args:
+            workspace_id: Workspace identifier
+            user_name: Target username
+            top_k: Maximum number of memories to retrieve
+            
+        Returns:
+            List of today's memories
+        """
+        try:
+            # Get today's date for filtering
+            dt_handler = DatetimeHandler()
+            today_date = dt_handler.datetime_format().split()[0]  # Extract date part (YYYY-MM-DD)
+
+            logger.info(f"Searching for memories from date: {today_date}")
+
+            # Create filter criteria for today's memories
+            filter_dict = {
+                "memory_type": "personal",
+                "target": user_name,
+                "created_date": today_date
+            }
+
+            # Search vector store with date filter
+            nodes: List[VectorNode] = self.vector_store.search(
+                query="",  # Empty query to get all results for today
+                workspace_id=workspace_id,
+                top_k=top_k,
+                filter_dict=filter_dict
+            )
+
+            logger.info(f"Vector store returned {len(nodes)} nodes for today")
+
+            # Convert vector nodes to memory objects
+            memories = self._convert_nodes_to_memories(nodes)
+            logger.info(f"Successfully converted {len(memories)} nodes to memories")
+
+            return memories
+
+        except Exception as e:
+            logger.error(f"Error retrieving today's memories: {e}")
+            return []
+
+    @staticmethod
+    def _convert_nodes_to_memories(nodes: List[VectorNode]) -> List[BaseMemory]:
+        """
+        Convert vector nodes to memory objects.
+        
+        Args:
+            nodes: List of vector nodes from vector store
+            
+        Returns:
+            List of converted memory objects
+        """
+        memories = []
+
+        for i, node in enumerate(nodes):
+            try:
+                memory = vector_node_to_memory(node)
+                memories.append(memory)
+            except Exception as e:
+                logger.warning(f"Failed to convert node {i} to memory: {e}")
+                continue
+
+        return memories

reme_ai/summary/personal/long_contra_repeat_op.py

@@ -0,0 +1,210 @@
+import re
+from typing import List
+
+from flowllm import C, BaseLLMOp
+from flowllm.enumeration.role import Role
+from flowllm.schema.message import Message
+from loguru import logger
+
+from reme_ai.schema.memory import BaseMemory, PersonalMemory
+
+
+@C.register_op()
+class LongContraRepeatOp(BaseLLMOp):
+    """
+    Manages and updates memory entries within a conversation scope by identifying
+    and handling contradictions or redundancies. It extends BaseLLMOp to provide
+    specialized functionality for long conversations with potential contradictory
+    or repetitive statements.
+    """
+    file_path: str = __file__
+
+    def execute(self):
+        """
+        Analyze memories for contradictions and redundancies, resolving conflicts.
+        
+        Process:
+        1. Get updated insight memories from previous operation
+        2. Check for contradictions and redundancies among memories
+        3. Resolve conflicts by keeping most recent/accurate information
+        4. Filter out redundant memories
+        5. Store cleaned memory list in context
+        """
+        # Get memories from previous operation
+        updated_insights = self.context.response.metadata.get("updated_insight_memories", [])
+
+        if not updated_insights:
+            logger.info("No updated insight memories to process for contradictions")
+            self.context.response.metadata["memory_list"] = []
+            return
+
+        # Get operation parameters
+        max_memories_to_process = self.op_params.get("long_contra_repeat_max_count", 50)
+        enable_processing = self.op_params.get("enable_long_contra_repeat", True)
+
+        if not enable_processing:
+            logger.info("Long contradiction/repeat processing is disabled")
+            self.context.response.metadata["memory_list"] = updated_insights
+            return
+
+        # Sort memories by creation time (most recent first) and limit count
+        sorted_memories = sorted(
+            updated_insights,
+            key=lambda x: x.time_created,
+            reverse=True
+        )[:max_memories_to_process]
+
+        if len(sorted_memories) <= 1:
+            logger.info("Only one memory to process, skipping contradiction analysis")
+            self.context.response.metadata["memory_list"] = sorted_memories
+            return
+
+        logger.info(f"Processing {len(sorted_memories)} memories for contradictions and redundancies")
+
+        # Analyze and resolve contradictions
+        filtered_memories = self._analyze_and_resolve_conflicts(sorted_memories)
+
+        # Store results in context
+        self.context.response.metadata["memory_list"] = filtered_memories
+        logger.info(f"Conflict resolution: {len(sorted_memories)} -> {len(filtered_memories)} memories")
+
+    def _analyze_and_resolve_conflicts(self, memories: List[BaseMemory]) -> List[BaseMemory]:
+        """
+        Analyze memories for contradictions and redundancies using LLM.
+        
+        Args:
+            memories: List of memories to analyze
+            
+        Returns:
+            List of filtered memories with conflicts resolved
+        """
+        user_name = self.context.get("user_name", "user")
+
+        # Prepare memory content for LLM analysis
+        memory_texts = []
+        for i, memory in enumerate(memories):
+            memory_texts.append(f"{i + 1} {memory.content}")
+
+        # Build LLM prompt
+        system_prompt = self.prompt_format(
+            prompt_name="long_contra_repeat_system",
+            num_obs=len(memory_texts),
+            user_name=user_name
+        )
+        few_shot = self.prompt_format(
+            prompt_name="long_contra_repeat_few_shot",
+            user_name=user_name
+        )
+        user_query = self.prompt_format(
+            prompt_name="long_contra_repeat_user_query",
+            user_query="\n".join(memory_texts)
+        )
+
+        full_prompt = f"{system_prompt}\n\n{few_shot}\n\n{user_query}"
+        logger.info(f"Contradiction analysis prompt length: {len(full_prompt)} chars")
+
+        # Get LLM analysis
+        response = self.llm.chat([Message(role=Role.USER, content=full_prompt)])
+
+        if not response or not response.content:
+            logger.warning("Empty response from LLM, keeping all memories")
+            return memories
+
+        # Parse response and filter memories
+        return self._parse_and_filter_memories(response.content, memories, user_name)
+
+    @staticmethod
+    def _parse_and_filter_memories(response_text: str, memories: List[BaseMemory], user_name: str) -> List[BaseMemory]:
+        """Parse LLM response and filter memories based on contradiction/containment analysis"""
+
+        # Use class method to parse the response
+        judgments = LongContraRepeatOp.parse_long_contra_repeat_response(response_text)
+
+        if not judgments:
+            logger.warning("No valid judgments found in response")
+            return memories
+
+        # Process each judgment
+        filtered_memories = []
+        processed_indices = set()
+
+        for idx, judgment, modified_content in judgments:
+            try:
+                memory_idx = idx - 1  # Convert to 0-based index
+                if memory_idx >= len(memories):
+                    logger.warning(f"Invalid index {memory_idx} for memories list of length {len(memories)}")
+                    continue
+
+                processed_indices.add(memory_idx)
+                memory = memories[memory_idx]
+                judgment_lower = judgment.lower()
+
+                if judgment_lower in ['矛盾', 'contradiction']:
+                    # For contradictory memories, either modify content or mark for removal
+                    if modified_content.strip():
+                        # Create new memory with modified content
+                        modified_memory = PersonalMemory(
+                            workspace_id=memory.workspace_id,
+                            memory_id=memory.memory_id,
+                            content=modified_content.strip(),
+                            target=memory.target if hasattr(memory, 'target') else user_name,
+                            author=memory.author,
+                            metadata={**memory.metadata, 'modified_by': 'long_contra_repeat'}
+                        )
+                        modified_memory.update_time_modified()
+                        filtered_memories.append(modified_memory)
+                        logger.info(f"Modified contradictory memory {idx}: {modified_content.strip()[:50]}...")
+                    else:
+                        # Remove contradictory memory without modification
+                        logger.info(f"Removing contradictory memory {idx}: {memory.content[:50]}...")
+
+                elif judgment_lower in ['被包含', 'contained']:
+                    # Remove contained/redundant memories
+                    logger.info(f"Removing contained memory {idx}: {memory.content[:50]}...")
+
+                else:  # 'none' case
+                    # Keep the memory as is
+                    filtered_memories.append(memory)
+
+            except Exception as e:
+                logger.warning(f"Error processing judgment for index {idx}: {e}")
+                continue
+
+        # Add any memories that weren't processed (shouldn't happen with correct LLM response)
+        for i, memory in enumerate(memories):
+            if i not in processed_indices:
+                filtered_memories.append(memory)
+                logger.warning(f"Memory {i + 1} was not processed by LLM, keeping as is")
+
+        return filtered_memories
+
+    def get_language_value(self, value_dict: dict):
+        """Get language-specific value from dictionary"""
+        return value_dict.get(self.language, value_dict.get("en"))
+
+    @staticmethod
+    def parse_long_contra_repeat_response(response_text: str) -> List[tuple]:
+        """Parse long contra repeat response to extract judgments"""
+        # Pattern to match both Chinese and English judgment formats
+        # Chinese: 判断：<序号> <矛盾|被包含|无> <修改后的内容>
+        # English: Judgment: <Index> <Contradiction|Contained|None> <Modified content>
+        pattern = r"判断：<(\d+)>\s*<(矛盾|被包含|无)>\s*<([^<>]*)>|Judgment:\s*<(\d+)>\s*<(Contradiction|Contained|None)>\s*<([^<>]*)>"
+        matches = re.findall(pattern, response_text, re.IGNORECASE | re.MULTILINE)
+
+        judgments = []
+        for match in matches:
+            # Handle both Chinese and English patterns
+            if match[0]:  # Chinese pattern
+                idx_str, judgment, modified_content = match[0], match[1], match[2]
+            else:  # English pattern
+                idx_str, judgment, modified_content = match[3], match[4], match[5]
+
+            try:
+                idx = int(idx_str)
+                judgments.append((idx, judgment, modified_content))
+            except ValueError:
+                logger.warning(f"Invalid index format: {idx_str}")
+                continue
+
+        logger.info(f"Parsed {len(judgments)} long contra repeat judgments from response")
+        return judgments

reme_ai/summary/personal/update_insight_op.py

@@ -0,0 +1,244 @@
+import re
+from typing import List
+
+from flowllm import C, BaseLLMOp
+from flowllm.schema.message import Message
+from loguru import logger
+
+from reme_ai.schema.memory import PersonalMemory
+
+
+@C.register_op()
+class UpdateInsightOp(BaseLLMOp):
+    """
+    This class is responsible for updating insight value in a memory system. It filters insight nodes
+    based on their association with observed nodes, utilizes a ranking model to prioritize them,
+    generates refreshed insights via an LLM, and manages node statuses and content updates.
+    """
+    file_path: str = __file__
+
+    def execute(self):
+        """
+        Update insight values based on new observation memories.
+        
+        Process:
+        1. Get insight subjects and personal memories from context
+        2. Find relevant observations for each insight subject
+        3. Update insight values using LLM integration
+        4. Store updated insights in context
+        """
+        # Get memories from previous operations
+        insight_memories = self.context.response.metadata.get("insight_memories", [])
+        personal_memories = self.context.response.metadata.get("personal_memories", [])
+
+        if not insight_memories:
+            logger.info("No insight memories to update")
+            self.context.response.metadata["updated_insight_memories"] = []
+            return
+
+        if not personal_memories:
+            logger.info("No observation memories available for insight updates")
+            self.context.response.metadata["updated_insight_memories"] = []
+            return
+
+        # Get operation parameters
+        update_insight_threshold = self.op_params.get("update_insight_threshold", 0.3)
+        update_insight_max_count = self.op_params.get("update_insight_max_count", 5)
+        user_name = self.context.get("user_name", "user")
+
+        logger.info(f"Updating {len(insight_memories)} insights with {len(personal_memories)} observations")
+
+        # Score and filter insights based on relevance to observations
+        scored_insights = self._score_insights_by_relevance(
+            insight_memories, personal_memories, update_insight_threshold
+        )
+
+        if not scored_insights:
+            logger.info("No insights meet relevance threshold for updating")
+            self.context.response.metadata["updated_insight_memories"] = []
+            return
+
+        # Select top insights for updating
+        top_insights = sorted(scored_insights, key=lambda x: x[1], reverse=True)[:update_insight_max_count]
+        logger.info(f"Selected {len(top_insights)} insights for updating")
+
+        # Update each selected insight
+        updated_insights = []
+        for insight_memory, relevance_score, relevant_observations in top_insights:
+            updated_insight = self._update_insight_with_observations(
+                insight_memory, relevant_observations, user_name
+            )
+            if updated_insight:
+                updated_insights.append(updated_insight)
+
+        # Store results in context
+        self.context.response.metadata["updated_insight_memories"] = updated_insights
+        logger.info(f"Successfully updated {len(updated_insights)} insight memories")
+
+    def _score_insights_by_relevance(self, insight_memories: List[PersonalMemory],
+                                     observation_memories: List[PersonalMemory],
+                                     threshold: float) -> List[tuple]:
+        """
+        Score insight memories based on relevance to observation memories.
+        
+        Args:
+            insight_memories: List of insight memories to score
+            observation_memories: List of observation memories for comparison
+            threshold: Minimum relevance score threshold
+            
+        Returns:
+            List[tuple]: List of (insight_memory, relevance_score, relevant_observations)
+        """
+        scored_insights = []
+
+        for insight_memory in insight_memories:
+            relevant_observations = []
+            max_relevance = 0.0
+
+            insight_subject = getattr(insight_memory, 'reflection_subject', '') or insight_memory.content
+            insight_keywords = set(insight_memory.content.lower().split())
+
+            # Find observations relevant to this insight
+            for obs_memory in observation_memories:
+                relevance_score = self._calculate_relevance_score(
+                    insight_memory, obs_memory, insight_keywords
+                )
+
+                if relevance_score >= threshold:
+                    relevant_observations.append(obs_memory)
+                    max_relevance = max(max_relevance, relevance_score)
+
+            # Include insight if it has relevant observations
+            if relevant_observations:
+                scored_insights.append((insight_memory, max_relevance, relevant_observations))
+                logger.info(
+                    f"Insight '{insight_subject[:40]}...' scored {max_relevance:.3f} with {len(relevant_observations)} observations"
+                )
+
+        return scored_insights
+
+    @staticmethod
+    def _calculate_relevance_score(insight_memory: PersonalMemory,
+                                   obs_memory: PersonalMemory, insight_keywords: set) -> float:
+        """Calculate relevance score between insight and observation memory"""
+        # High relevance for same reflection subject
+        insight_subject = getattr(insight_memory, 'reflection_subject', '')
+        obs_subject = getattr(obs_memory, 'reflection_subject', '')
+
+        if insight_subject and obs_subject and insight_subject == obs_subject:
+            return 0.9
+
+        # Medium relevance for keyword overlap
+        obs_keywords = set(obs_memory.content.lower().split())
+        intersection = len(insight_keywords.intersection(obs_keywords))
+        union = len(insight_keywords.union(obs_keywords))
+
+        return intersection / union if union > 0 else 0.0
+
+    def _update_insight_with_observations(self, insight_memory: PersonalMemory,
+                                          relevant_observations: List[PersonalMemory],
+                                          user_name: str) -> PersonalMemory:
+        """
+        Update a single insight memory based on relevant observations using LLM.
+        
+        Args:
+            insight_memory: The insight memory to update
+            relevant_observations: List of relevant observation memories
+            user_name: The target username
+            
+        Returns:
+            PersonalMemory: Updated insight memory or None if update failed
+        """
+        logger.info(
+            f"Updating insight: {insight_memory.content[:50]}... with {len(relevant_observations)} observations")
+
+        # Build observation context
+        observation_texts = [obs.content for obs in relevant_observations]
+
+        # Create prompt using the prompt format method
+        insight_key = insight_memory.reflection_subject or "personal_info"
+        insight_key_value = f"{insight_key}: {insight_memory.content}"
+
+        system_prompt = self.prompt_format(prompt_name="update_insight_system", user_name=user_name)
+        few_shot = self.prompt_format(prompt_name="update_insight_few_shot", user_name=user_name)
+        user_query = self.prompt_format(prompt_name="update_insight_user_query",
+                                        user_query="\n".join(observation_texts),
+                                        insight_key=insight_key,
+                                        insight_key_value=insight_key_value)
+
+        full_prompt = f"{system_prompt}\n\n{few_shot}\n\n{user_query}"
+        logger.info(f"update_insight_prompt={full_prompt}")
+
+        def parse_update_response(message: Message) -> PersonalMemory:
+            """Parse LLM response and create updated insight memory"""
+            response_text = message.content
+            logger.info(f"update_insight_response={response_text}")
+
+            # Parse the response to extract updated insight
+            updated_content = UpdateInsightOp.parse_update_insight_response(response_text, self.language)
+
+            if not updated_content or updated_content.lower() in ['无', 'none', '']:
+                logger.info(f"No update needed for insight: {insight_memory.content[:50]}...")
+                return insight_memory
+
+            if updated_content == insight_memory.content:
+                logger.info(f"Insight content unchanged: {insight_memory.content[:50]}...")
+                return insight_memory
+
+            # Create updated insight memory
+            updated_insight = PersonalMemory(
+                workspace_id=insight_memory.workspace_id,
+                memory_id=insight_memory.memory_id,
+                memory_type="personal_insight",
+                content=updated_content,
+                target=insight_memory.target,
+                reflection_subject=insight_memory.reflection_subject,
+                author=getattr(self.llm, "model_name", "system"),
+                metadata={
+                    **insight_memory.metadata,
+                    "updated_by": "update_insight_op",
+                    "original_content": insight_memory.content,
+                    "update_reason": "integrated_new_observations"
+                }
+            )
+            updated_insight.update_time_modified()
+
+            logger.info(f"Updated insight: {updated_content[:50]}...")
+            return updated_insight
+
+        # Use LLM chat with callback function
+        try:
+            return self.llm.chat(messages=[Message(content=full_prompt)], callback_fn=parse_update_response)
+        except Exception as e:
+            logger.error(f"Error updating insight: {e}")
+            return insight_memory
+
+    @staticmethod
+    def parse_update_insight_response(response_text: str, language: str = "en") -> str:
+        """Parse update insight response to extract updated insight content"""
+        # Pattern to match both Chinese and English insight formats
+        # Chinese: {user_name}的资料: <信息>
+        # English: {user_name}'s profile: <Information>
+        if language in ["zh", "cn"]:
+            pattern = r"的资料[：:]\s*<([^<>]+)>"
+        else:
+            pattern = r"profile[：:]\s*<([^<>]+)>"
+
+        matches = re.findall(pattern, response_text, re.IGNORECASE | re.MULTILINE)
+
+        if matches:
+            insight_content = matches[0].strip()
+            logger.info(f"Parsed insight content: {insight_content}")
+            return insight_content
+
+        # Fallback: try to find content between angle brackets
+        fallback_pattern = r"<([^<>]+)>"
+        fallback_matches = re.findall(fallback_pattern, response_text)
+        if fallback_matches:
+            # Get the last match as it's likely the final answer
+            insight_content = fallback_matches[-1].strip()
+            logger.info(f"Parsed insight content (fallback): {insight_content}")
+            return insight_content
+
+        logger.warning("No insight content found in response")
+        return ""

reme_ai/summary/task/__init__.py

@@ -0,0 +1,10 @@
+from .comparative_extraction_op import ComparativeExtractionOp
+from .failure_extraction_op import FailureExtractionOp
+from .memory_deduplication_op import MemoryDeduplicationOp
+from .memory_validation_op import MemoryValidationOp
+from .pdf_preprocess_op_wrapper import PDFPreprocessOp
+from .simple_comparative_summary_op import SimpleComparativeSummaryOp
+from .simple_summary_op import SimpleSummaryOp
+from .success_extraction_op import SuccessExtractionOp
+from .trajectory_preprocess_op import TrajectoryPreprocessOp
+from .trajectory_segmentation_op import TrajectorySegmentationOp