empathy-framework 4.4.0__py3-none-any.whl → 4.5.1__py3-none-any.whl

empathy_os/memory/short_term.py

@@ -33,6 +33,10 @@ from typing import Any
 
 import structlog
 
+from .security.pii_scrubber import PIIScrubber
+from .security.secrets_detector import SecretsDetector
+from .security.secrets_detector import Severity as SecretSeverity
+
 logger = structlog.get_logger(__name__)
 
 try:
@@ -100,6 +104,10 @@ class RedisConfig:
     password: str | None = None
     use_mock: bool = False
 
+    # Security settings
+    pii_scrub_enabled: bool = True  # Scrub PII before storing (HIPAA/GDPR compliance)
+    secrets_detection_enabled: bool = True  # Block storage of detected secrets
+
     # SSL/TLS settings
     ssl: bool = False
     ssl_cert_reqs: str | None = None  # "required", "optional", "none"
@@ -166,6 +174,11 @@ class RedisMetrics:
     publish_count: int = 0
     stream_append_count: int = 0
 
+    # Security metrics
+    pii_scrubbed_total: int = 0  # Total PII instances scrubbed
+    pii_scrub_operations: int = 0  # Operations that had PII scrubbed
+    secrets_blocked_total: int = 0  # Total secrets blocked from storage
+
     def record_operation(self, operation: str, latency_ms: float, success: bool = True) -> None:
         """Record an operation metric."""
         self.operations_total += 1
@@ -217,6 +230,11 @@ class RedisMetrics:
                 "publish": self.publish_count,
                 "stream_append": self.stream_append_count,
             },
+            "security": {
+                "pii_scrubbed_total": self.pii_scrubbed_total,
+                "pii_scrub_operations": self.pii_scrub_operations,
+                "secrets_blocked_total": self.secrets_blocked_total,
+            },
         }
 
 
@@ -387,6 +405,10 @@ class ConflictContext:
         )
 
 
+class SecurityError(Exception):
+    """Raised when a security policy is violated (e.g., secrets detected in data)."""
+
+
 class RedisShortTermMemory:
     """Redis-backed short-term memory for agent coordination
 
@@ -488,6 +510,18 @@ class RedisShortTermMemory:
         self._mock_streams: dict[str, list[tuple[str, dict]]] = {}
         self._mock_pubsub_handlers: dict[str, list[Callable[[dict], None]]] = {}
 
+        # Security: Initialize PII scrubber and secrets detector
+        self._pii_scrubber: PIIScrubber | None = None
+        self._secrets_detector: SecretsDetector | None = None
+
+        if self._config.pii_scrub_enabled:
+            self._pii_scrubber = PIIScrubber(enable_name_detection=False)
+            logger.debug("pii_scrubber_enabled", message="PII scrubbing active for short-term memory")
+
+        if self._config.secrets_detection_enabled:
+            self._secrets_detector = SecretsDetector()
+            logger.debug("secrets_detector_enabled", message="Secrets detection active for short-term memory")
+
         if self.use_mock:
             self._client = None
         else:
@@ -621,6 +655,82 @@ class RedisShortTermMemory:
         # Convert bytes to strings - needed for API return type
         return [k.decode() if isinstance(k, bytes) else str(k) for k in keys]
 
+    # === Security Methods ===
+
+    def _sanitize_data(self, data: Any) -> tuple[Any, int]:
+        """Sanitize data by scrubbing PII and checking for secrets.
+
+        Args:
+            data: Data to sanitize (dict, list, or str)
+
+        Returns:
+            Tuple of (sanitized_data, pii_count)
+
+        Raises:
+            SecurityError: If secrets are detected and blocking is enabled
+
+        """
+        pii_count = 0
+
+        if data is None:
+            return data, 0
+
+        # Convert data to string for scanning
+        if isinstance(data, dict):
+            data_str = json.dumps(data)
+        elif isinstance(data, list):
+            data_str = json.dumps(data)
+        elif isinstance(data, str):
+            data_str = data
+        else:
+            # For other types, convert to string
+            data_str = str(data)
+
+        # Check for secrets first (before modifying data)
+        if self._secrets_detector is not None:
+            detections = self._secrets_detector.detect(data_str)
+            # Block critical and high severity secrets
+            critical_secrets = [
+                d for d in detections
+                if d.severity in (SecretSeverity.CRITICAL, SecretSeverity.HIGH)
+            ]
+            if critical_secrets:
+                self._metrics.secrets_blocked_total += len(critical_secrets)
+                secret_types = [d.secret_type.value for d in critical_secrets]
+                logger.warning(
+                    "secrets_detected_blocked",
+                    secret_types=secret_types,
+                    count=len(critical_secrets),
+                )
+                raise SecurityError(
+                    f"Cannot store data containing secrets: {secret_types}. "
+                    "Remove sensitive credentials before storing."
+                )
+
+        # Scrub PII
+        if self._pii_scrubber is not None:
+            sanitized_str, pii_detections = self._pii_scrubber.scrub(data_str)
+            pii_count = len(pii_detections)
+
+            if pii_count > 0:
+                self._metrics.pii_scrubbed_total += pii_count
+                self._metrics.pii_scrub_operations += 1
+                logger.debug(
+                    "pii_scrubbed",
+                    pii_count=pii_count,
+                    pii_types=[d.pii_type for d in pii_detections],
+                )
+
+                # Convert back to original type
+                if isinstance(data, dict):
+                    return json.loads(sanitized_str), pii_count
+                elif isinstance(data, list):
+                    return json.loads(sanitized_str), pii_count
+                else:
+                    return sanitized_str, pii_count
+
+        return data, pii_count
+
     # === Working Memory (Stash/Retrieve) ===
 
     def stash(
@@ -629,6 +739,7 @@ class RedisShortTermMemory:
         data: Any,
         credentials: AgentCredentials,
         ttl: TTLStrategy = TTLStrategy.WORKING_RESULTS,
+        skip_sanitization: bool = False,
     ) -> bool:
         """Stash data in short-term memory
 
@@ -637,6 +748,7 @@ class RedisShortTermMemory:
             data: Data to store (will be JSON serialized)
             credentials: Agent credentials
             ttl: Time-to-live strategy
+            skip_sanitization: Skip PII scrubbing and secrets detection (use with caution)
 
         Returns:
             True if successful
@@ -644,6 +756,12 @@ class RedisShortTermMemory:
         Raises:
             ValueError: If key is empty or invalid
             PermissionError: If credentials lack write access
+            SecurityError: If secrets are detected in data (when secrets_detection_enabled)
+
+        Note:
+            PII (emails, SSNs, phone numbers, etc.) is automatically scrubbed
+            before storage unless skip_sanitization=True or pii_scrub_enabled=False.
+            Secrets (API keys, passwords, etc.) will block storage by default.
 
         Example:
             >>> memory.stash("analysis_v1", {"findings": [...]}, creds)
@@ -659,6 +777,17 @@ class RedisShortTermMemory:
                 "cannot write to memory. Requires CONTRIBUTOR or higher.",
             )
 
+        # Sanitize data (PII scrubbing + secrets detection)
+        if not skip_sanitization:
+            data, pii_count = self._sanitize_data(data)
+            if pii_count > 0:
+                logger.info(
+                    "stash_pii_scrubbed",
+                    key=key,
+                    agent_id=credentials.agent_id,
+                    pii_count=pii_count,
+                )
+
         full_key = f"{self.PREFIX_WORKING}{credentials.agent_id}:{key}"
         payload = {
             "data": data,
@@ -2218,6 +2347,63 @@ class RedisShortTermMemory:
             except Exception:
                 pass
 
+    # =========================================================================
+    # CROSS-SESSION COMMUNICATION
+    # =========================================================================
+
+    def enable_cross_session(
+        self,
+        access_tier: AccessTier = AccessTier.CONTRIBUTOR,
+        auto_announce: bool = True,
+    ):
+        """Enable cross-session communication for this memory instance.
+
+        This allows agents in different Claude Code sessions to communicate
+        and coordinate via Redis.
+
+        Args:
+            access_tier: Access tier for this session
+            auto_announce: Whether to announce presence automatically
+
+        Returns:
+            CrossSessionCoordinator instance
+
+        Raises:
+            ValueError: If in mock mode (Redis required for cross-session)
+
+        Example:
+            >>> memory = RedisShortTermMemory()
+            >>> coordinator = memory.enable_cross_session(AccessTier.CONTRIBUTOR)
+            >>> print(f"Session ID: {coordinator.agent_id}")
+            >>> sessions = coordinator.get_active_sessions()
+
+        """
+        if self.use_mock:
+            raise ValueError(
+                "Cross-session communication requires Redis. "
+                "Set REDIS_HOST/REDIS_PORT or disable mock mode."
+            )
+
+        from .cross_session import CrossSessionCoordinator, SessionType
+
+        coordinator = CrossSessionCoordinator(
+            memory=self,
+            session_type=SessionType.CLAUDE,
+            access_tier=access_tier,
+            auto_announce=auto_announce,
+        )
+
+        return coordinator
+
+    def cross_session_available(self) -> bool:
+        """Check if cross-session communication is available.
+
+        Returns:
+            True if Redis is connected (not mock mode)
+
+        """
+        return not self.use_mock and self._client is not None
+
     # =========================================================================
     # CLEANUP AND LIFECYCLE
     # =========================================================================

empathy_os/meta_workflows/builtin_templates.py

@@ -80,7 +80,7 @@ RELEASE_PREP_TEMPLATE = MetaWorkflowTemplate(
     agent_composition_rules=[
         AgentCompositionRule(
             role="Security Auditor",
-            base_template="security-analyst",
+            base_template="security_auditor",
             tier_strategy=TierStrategy.CAPABLE_FIRST,
             tools=["grep", "bandit", "safety"],
             required_responses={"security_scan": "Yes"},
@@ -92,7 +92,7 @@ RELEASE_PREP_TEMPLATE = MetaWorkflowTemplate(
         ),
         AgentCompositionRule(
             role="Test Coverage Analyst",
-            base_template="test-analyst",
+            base_template="test_coverage_analyzer",
             tier_strategy=TierStrategy.PROGRESSIVE,
             tools=["pytest", "coverage"],
             required_responses={"test_coverage_check": "Yes"},
@@ -104,7 +104,7 @@ RELEASE_PREP_TEMPLATE = MetaWorkflowTemplate(
         ),
         AgentCompositionRule(
             role="Code Quality Reviewer",
-            base_template="code-reviewer",
+            base_template="code_reviewer",
             tier_strategy=TierStrategy.PROGRESSIVE,
             tools=["ruff", "mypy"],
             required_responses={"quality_review": "Yes"},
@@ -116,7 +116,7 @@ RELEASE_PREP_TEMPLATE = MetaWorkflowTemplate(
         ),
         AgentCompositionRule(
             role="Documentation Specialist",
-            base_template="doc-reviewer",
+            base_template="documentation_writer",
             tier_strategy=TierStrategy.CHEAP_ONLY,
             tools=["pydocstyle"],
             required_responses={"doc_verification": "Yes"},
@@ -181,7 +181,7 @@ TEST_COVERAGE_BOOST_TEMPLATE = MetaWorkflowTemplate(
     agent_composition_rules=[
         AgentCompositionRule(
             role="Gap Analyzer",
-            base_template="coverage-analyst",
+            base_template="test_coverage_analyzer",
             tier_strategy=TierStrategy.PROGRESSIVE,
             tools=["pytest-cov", "coverage"],
             required_responses={},
@@ -193,7 +193,7 @@ TEST_COVERAGE_BOOST_TEMPLATE = MetaWorkflowTemplate(
         ),
         AgentCompositionRule(
             role="Test Generator",
-            base_template="test-generator",
+            base_template="test_generator",
             tier_strategy=TierStrategy.CAPABLE_FIRST,
             tools=["ast", "pytest"],
             required_responses={},
@@ -208,7 +208,7 @@ TEST_COVERAGE_BOOST_TEMPLATE = MetaWorkflowTemplate(
         ),
         AgentCompositionRule(
             role="Test Validator",
-            base_template="test-validator",
+            base_template="test_validator",
             tier_strategy=TierStrategy.CHEAP_ONLY,
             tools=["pytest"],
             required_responses={},
@@ -250,8 +250,8 @@ TEST_MAINTENANCE_TEMPLATE = MetaWorkflowTemplate(
                 id="max_files",
                 text="Maximum files to process",
                 type=QuestionType.SINGLE_SELECT,
-                options=["5", "10", "20", "50"],
-                default="10",
+                options=["5", "10", "20", "30", "50"],
+                default="30",
                 help_text="Limit number of files to process per run",
             ),
             FormQuestion(
@@ -274,7 +274,7 @@ TEST_MAINTENANCE_TEMPLATE = MetaWorkflowTemplate(
     agent_composition_rules=[
         AgentCompositionRule(
             role="Test Analyst",
-            base_template="test-analyst",
+            base_template="test_coverage_analyzer",
             tier_strategy=TierStrategy.PROGRESSIVE,
             tools=["pytest-cov", "coverage"],
             required_responses={"mode": ["full", "analyze"]},
@@ -289,7 +289,7 @@ TEST_MAINTENANCE_TEMPLATE = MetaWorkflowTemplate(
         ),
         AgentCompositionRule(
             role="Test Generator",
-            base_template="test-generator",
+            base_template="test_generator",
             tier_strategy=TierStrategy.CAPABLE_FIRST,
             tools=["ast", "pytest"],
             required_responses={"mode": ["full", "generate"]},
@@ -300,7 +300,7 @@ TEST_MAINTENANCE_TEMPLATE = MetaWorkflowTemplate(
         ),
         AgentCompositionRule(
             role="Test Validator",
-            base_template="test-validator",
+            base_template="test_validator",
             tier_strategy=TierStrategy.CHEAP_ONLY,
             tools=["pytest"],
             required_responses={"mode": ["full", "validate"], "auto_validation": "Yes"},
@@ -311,7 +311,7 @@ TEST_MAINTENANCE_TEMPLATE = MetaWorkflowTemplate(
         ),
         AgentCompositionRule(
             role="Test Reporter",
-            base_template="reporter",
+            base_template="report_generator",
             tier_strategy=TierStrategy.CHEAP_ONLY,
             tools=[],
             required_responses={},
@@ -373,7 +373,7 @@ MANAGE_DOCS_TEMPLATE = MetaWorkflowTemplate(
     agent_composition_rules=[
         AgentCompositionRule(
             role="Documentation Analyst",
-            base_template="doc-analyst",
+            base_template="documentation_analyst",
             tier_strategy=TierStrategy.PROGRESSIVE,
             tools=["ast", "pydocstyle"],
             required_responses={},
@@ -388,7 +388,7 @@ MANAGE_DOCS_TEMPLATE = MetaWorkflowTemplate(
         ),
         AgentCompositionRule(
             role="Documentation Reviewer",
-            base_template="doc-reviewer",
+            base_template="documentation_writer",
             tier_strategy=TierStrategy.PROGRESSIVE,
             tools=[],
             required_responses={},
@@ -463,7 +463,7 @@ FEATURE_OVERVIEW_TEMPLATE = MetaWorkflowTemplate(
     agent_composition_rules=[
         AgentCompositionRule(
             role="Code Scanner",
-            base_template="generic",
+            base_template="generic_agent",
             tier_strategy=TierStrategy.CAPABLE_FIRST,
             tools=["read", "grep", "glob"],
             required_responses={},
@@ -475,7 +475,7 @@ FEATURE_OVERVIEW_TEMPLATE = MetaWorkflowTemplate(
         ),
         AgentCompositionRule(
             role="Insights Reporter",
-            base_template="generic",
+            base_template="generic_agent",
             tier_strategy=TierStrategy.CAPABLE_FIRST,
             tools=["read"],
             required_responses={},
@@ -487,7 +487,7 @@ FEATURE_OVERVIEW_TEMPLATE = MetaWorkflowTemplate(
         ),
         AgentCompositionRule(
             role="Architecture Analyst",
-            base_template="generic",
+            base_template="architecture_analyst",
             tier_strategy=TierStrategy.CAPABLE_FIRST,
             tools=["read"],
             required_responses={"include_diagrams": "Yes"},
@@ -499,7 +499,7 @@ FEATURE_OVERVIEW_TEMPLATE = MetaWorkflowTemplate(
         ),
         AgentCompositionRule(
             role="Quality Reviewer",
-            base_template="generic",
+            base_template="code_reviewer",
             tier_strategy=TierStrategy.PREMIUM_ONLY,
             tools=["read"],
             required_responses={},
@@ -511,7 +511,7 @@ FEATURE_OVERVIEW_TEMPLATE = MetaWorkflowTemplate(
         ),
         AgentCompositionRule(
             role="Blog Content Creator",
-            base_template="generic",
+            base_template="generic_agent",
             tier_strategy=TierStrategy.PREMIUM_ONLY,
             tools=["write"],
             required_responses={"include_blog_summary": "Yes"},

empathy_os/meta_workflows/cli_meta_workflows.py

@@ -246,6 +246,12 @@ def run_workflow(
         "-u",
         help="User ID for memory integration",
     ),
+    json_output: bool = typer.Option(
+        False,
+        "--json",
+        "-j",
+        help="Output result as JSON (for programmatic use)",
+    ),
 ):
     """Execute a meta-workflow from template.
 
@@ -261,32 +267,43 @@ def run_workflow(
         empathy meta-workflow run release-prep
         empathy meta-workflow run test-coverage-boost --real
         empathy meta-workflow run manage-docs --use-defaults
+        empathy meta-workflow run release-prep --json --use-defaults
     """
+    import json
+
     try:
         # Load template
-        console.print(f"\n[bold]Loading template:[/bold] {template_id}")
+        if not json_output:
+            console.print(f"\n[bold]Loading template:[/bold] {template_id}")
         registry = TemplateRegistry()
         template = registry.load_template(template_id)
 
         if not template:
-            console.print(f"[red]Template not found:[/red] {template_id}")
+            if json_output:
+                print(json.dumps({"success": False, "error": f"Template not found: {template_id}"}))
+            else:
+                console.print(f"[red]Template not found:[/red] {template_id}")
             raise typer.Exit(code=1)
 
-        console.print(f"[green]✓[/green] {template.name}")
+        if not json_output:
+            console.print(f"[green]✓[/green] {template.name}")
 
         # Setup memory if requested
         pattern_learner = None
         if use_memory:
-            console.print("\n[bold]Initializing memory integration...[/bold]")
+            if not json_output:
+                console.print("\n[bold]Initializing memory integration...[/bold]")
             from empathy_os.memory.unified import UnifiedMemory
 
             try:
                 memory = UnifiedMemory(user_id=user_id)
                 pattern_learner = PatternLearner(memory=memory)
-                console.print("[green]✓[/green] Memory enabled")
+                if not json_output:
+                    console.print("[green]✓[/green] Memory enabled")
             except Exception as e:
-                console.print(f"[yellow]Warning:[/yellow] Memory initialization failed: {e}")
-                console.print("[yellow]Continuing without memory integration[/yellow]")
+                if not json_output:
+                    console.print(f"[yellow]Warning:[/yellow] Memory initialization failed: {e}")
+                    console.print("[yellow]Continuing without memory integration[/yellow]")
 
         # Create workflow
         workflow = MetaWorkflow(
@@ -295,14 +312,49 @@ def run_workflow(
         )
 
         # Execute (will ask questions via AskUserQuestion unless --use-defaults)
-        console.print("\n[bold]Executing workflow...[/bold]")
-        console.print(f"Mode: {'Mock' if mock else 'Real'}")
-        if use_defaults:
-            console.print("[cyan]Using default values (non-interactive)[/cyan]")
+        if not json_output:
+            console.print("\n[bold]Executing workflow...[/bold]")
+            console.print(f"Mode: {'Mock' if mock else 'Real'}")
+            if use_defaults:
+                console.print("[cyan]Using default values (non-interactive)[/cyan]")
 
         result = workflow.execute(mock_execution=mock, use_defaults=use_defaults)
 
-        # Display summary
+        # JSON output mode - print result as JSON and exit
+        if json_output:
+            output = {
+                "run_id": result.run_id,
+                "template_id": template_id,
+                "timestamp": result.timestamp,
+                "success": result.success,
+                "error": result.error,
+                "total_cost": result.total_cost,
+                "total_duration": result.total_duration,
+                "agents_created": len(result.agents_created),
+                "form_responses": {
+                    "template_id": result.form_responses.template_id,
+                    "responses": result.form_responses.responses,
+                    "timestamp": result.form_responses.timestamp,
+                    "response_id": result.form_responses.response_id,
+                },
+                "agent_results": [
+                    {
+                        "agent_id": ar.agent_id,
+                        "role": ar.role,
+                        "success": ar.success,
+                        "cost": ar.cost,
+                        "duration": ar.duration,
+                        "tier_used": ar.tier_used,
+                        "output": ar.output,
+                        "error": ar.error,
+                    }
+                    for ar in result.agent_results
+                ],
+            }
+            print(json.dumps(output))
+            return
+
+        # Display summary (normal mode)
         console.print("\n[bold green]Execution Complete![/bold green]\n")
 
         summary_lines = [
@@ -340,9 +392,12 @@ def run_workflow(
         console.print()
 
     except Exception as e:
-        console.print(f"\n[red]Error:[/red] {e}")
-        import traceback
-        traceback.print_exc()
+        if json_output:
+            print(json.dumps({"success": False, "error": str(e)}))
+        else:
+            console.print(f"\n[red]Error:[/red] {e}")
+            import traceback
+            traceback.print_exc()
         raise typer.Exit(code=1)
 
 

empathy_os/meta_workflows/workflow.py

@@ -12,6 +12,28 @@ Updated: 2026-01-18 (v4.3.0 - Real LLM execution with Anthropic client)
 Purpose: Core orchestration for meta-workflows
 """
 
+# Load environment variables from .env file
+# Try multiple locations: project root, home directory, empathy config
+try:
+    from pathlib import Path
+
+    from dotenv import load_dotenv
+
+    # Try common .env locations
+    _env_paths = [
+        Path.cwd() / ".env",  # Current working directory
+        Path(__file__).parent.parent.parent.parent / ".env",  # Project root
+        Path.home() / ".env",  # Home directory
+        Path.home() / ".empathy" / ".env",  # Empathy config directory
+    ]
+
+    for _env_path in _env_paths:
+        if _env_path.exists():
+            load_dotenv(_env_path)
+            break
+except ImportError:
+    pass  # dotenv not installed, use environment variables directly
+
 import json
 import logging
 import time
@@ -616,7 +638,14 @@ class MetaWorkflow:
         try:
             from anthropic import Anthropic
 
-            client = Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY"))
+            api_key = os.environ.get("ANTHROPIC_API_KEY")
+            if not api_key:
+                raise ValueError(
+                    "ANTHROPIC_API_KEY not found. Set it in environment or .env file "
+                    "(checked: ./env, ~/.env, ~/.empathy/.env)"
+                )
+
+            client = Anthropic(api_key=api_key)
 
             # Execute the LLM call
             response = client.messages.create(