continuum-toolkit 1.0.0__py3-none-any.whl

extractors/config/secret_sanitizer.py

@@ -0,0 +1,91 @@
+"""
+Project Continuum - Secret Redaction and Sanitization Engine
+============================================================
+Ensures that credentials, API keys, private tokens, passwords, and secrets
+are never collected, stored, or exposed in Continuum state or evidence.
+"""
+
+import re
+from typing import Any, Dict, List, Set, Union
+
+
+class SecretSanitizer:
+    """
+    Detects and redacts sensitive information such as API keys, tokens,
+    passwords, and private keys.
+    """
+
+    # Known sensitive key substrings (case-insensitive)
+    SENSITIVE_KEY_NAMES: Set[str] = {
+        "password",
+        "secret",
+        "token",
+        "api_key",
+        "apikey",
+        "access_key",
+        "auth_token",
+        "jwt",
+        "private_key",
+        "client_secret",
+        "ssh_key",
+        "database_url",
+        "db_pass",
+        "encryption_key",
+        "signature",
+        "credential"
+    }
+
+    # Regex patterns for common secret formats
+    SECRET_VALUE_PATTERNS = [
+        re.compile(r"-----BEGIN (?:[A-Z0-9\s]+ )?PRIVATE KEY-----.*?-----END (?:[A-Z0-9\s]+ )?PRIVATE KEY-----", re.DOTALL),
+        re.compile(r"AKIA[0-9A-Z]{16}"),  # AWS Access Key ID
+        re.compile(r"ghp_[a-zA-Z0-9]{36}"),  # GitHub Personal Access Token
+        re.compile(r"gho_[a-zA-Z0-9]{36}"),  # GitHub OAuth Access Token
+        re.compile(r"glpat-[a-zA-Z0-9\-_]{20,32}"),  # GitLab Personal Access Token
+        re.compile(r"sk-[a-zA-Z0-9]{20,64}"),  # OpenAI Secret Key
+        re.compile(r"AIza[0-9A-Za-z\-_]{25,45}"),  # Google API Key
+        re.compile(r"ey[A-Za-z0-9-_=]+\.[A-Za-z0-9-_=]+\.?[A-Za-z0-9-_.+/=]*"),  # JWT Token
+        re.compile(r"[a-f0-9]{32,64}"),  # Raw Hex Key / Token
+    ]
+
+    @classmethod
+    def is_sensitive_key(cls, key_name: str) -> bool:
+        normalized = key_name.lower().replace("-", "_").replace(".", "_")
+        return any(sensitive in normalized for sensitive in cls.SENSITIVE_KEY_NAMES)
+
+    @classmethod
+    def contains_secret_pattern(cls, value: str) -> bool:
+        if not isinstance(value, str) or len(value) < 8:
+            return False
+        return any(pattern.search(value) for pattern in cls.SECRET_VALUE_PATTERNS)
+
+    @classmethod
+    def sanitize_value(cls, key: str, value: Any) -> Any:
+        if cls.is_sensitive_key(key):
+            return "<REDACTED_SECRET>"
+        if isinstance(value, str):
+            for pattern in cls.SECRET_VALUE_PATTERNS:
+                if pattern.search(value):
+                    return "<REDACTED_SECRET>"
+            return value
+        elif isinstance(value, dict):
+            return cls.sanitize_dict(value)
+        elif isinstance(value, list):
+            return [cls.sanitize_value(key, item) for item in value]
+        return value
+
+    @classmethod
+    def sanitize_dict(cls, data: Dict[str, Any]) -> Dict[str, Any]:
+        sanitized = {}
+        for k, v in data.items():
+            if cls.is_sensitive_key(k):
+                sanitized[k] = "<REDACTED_SECRET>"
+            elif isinstance(v, dict):
+                sanitized[k] = cls.sanitize_dict(v)
+            elif isinstance(v, list):
+                sanitized[k] = [cls.sanitize_value(k, item) for item in v]
+            elif isinstance(v, str):
+                sanitized[k] = cls.sanitize_value(k, v)
+            else:
+                sanitized[k] = v
+        return sanitized

extractors/config_extractor.py

@@ -0,0 +1,172 @@
+"""
+Project Continuum - Configuration & Environment Evidence Extractor
+===================================================================
+Milestone 2 - Phase 2: Configuration, Environment & Metadata Extraction.
+Scans for project manifests (package.json, pyproject.toml, Cargo.toml, go.mod,
+pubspec.yaml, Dockerfile, docker-compose.yml), detects lockfiles, extracts
+dependencies, scripts, and build environments while redacting all secrets.
+"""
+
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set
+
+from core.enums import EvidenceType, EvidenceLevel
+from core.evidence import Evidence
+from core.state_models import (
+    CanonicalProjectState,
+    ManifestInfo,
+    ProjectState,
+)
+from extractors.base import BaseEvidenceExtractor, DEFAULT_IGNORE_DIRS
+from extractors.config.parsers import ManifestParser
+from extractors.config.secret_sanitizer import SecretSanitizer
+
+
+LOCKFILE_NAMES: Set[str] = {
+    "package-lock.json",
+    "yarn.lock",
+    "pnpm-lock.yaml",
+    "Cargo.lock",
+    "poetry.lock",
+    "Pipfile.lock",
+    "pubspec.lock",
+    "go.sum",
+    "composer.lock"
+}
+
+
+class ConfigEvidenceExtractor(BaseEvidenceExtractor):
+    """
+    Extractor responsible for Phase 2: Manifest, config, and environment discovery.
+    """
+
+    @property
+    def extractor_name(self) -> str:
+        return "ConfigEvidenceExtractor"
+
+    @property
+    def supported_evidence_types(self) -> List[EvidenceType]:
+        return [
+            EvidenceType.CONFIG_FILE
+        ]
+
+    def can_extract(self, target_path_or_input: str) -> bool:
+        path = Path(target_path_or_input)
+        return path.exists() and path.is_dir()
+
+    def extract(self, target_path_or_input: str, context: Optional[Dict[str, Any]] = None) -> List[Evidence]:
+        root_path = Path(target_path_or_input).resolve()
+        evidence_list: List[Evidence] = []
+
+        manifests: List[ManifestInfo] = []
+        lockfiles_detected: List[str] = []
+        container_configs: List[Dict[str, Any]] = []
+        env_templates: List[Dict[str, Any]] = []
+
+        for path in sorted(root_path.rglob("*")):
+            if not path.is_file():
+                continue
+
+            parts = set(path.relative_to(root_path).parts)
+            if any(ignored in parts for ignored in DEFAULT_IGNORE_DIRS):
+                continue
+
+            filename = path.name
+            rel_path = path.relative_to(root_path).as_posix()
+
+            # 1. Lockfiles
+            if filename in LOCKFILE_NAMES:
+                lockfiles_detected.append(rel_path)
+                continue
+
+            # Read content safely
+            try:
+                content = path.read_text(encoding="utf-8", errors="replace")
+            except Exception:
+                continue
+
+            # 2. Manifests
+            if filename == "package.json":
+                m = ManifestParser.parse_package_json(rel_path, content)
+                manifests.append(m)
+            elif filename == "pyproject.toml":
+                m = ManifestParser.parse_pyproject_toml(rel_path, content)
+                manifests.append(m)
+            elif filename == "Cargo.toml":
+                m = ManifestParser.parse_cargo_toml(rel_path, content)
+                manifests.append(m)
+            elif filename == "go.mod":
+                m = ManifestParser.parse_go_mod(rel_path, content)
+                manifests.append(m)
+            elif filename == "pubspec.yaml":
+                m = ManifestParser.parse_pubspec_yaml(rel_path, content)
+                manifests.append(m)
+            elif filename == "Dockerfile" or filename.startswith("Dockerfile."):
+                d = ManifestParser.parse_dockerfile(rel_path, content)
+                container_configs.append(d)
+            elif filename in {".env.example", ".env.template", ".env.sample"}:
+                e = ManifestParser.parse_env_template(rel_path, content)
+                env_templates.append(e)
+
+        # 3. Create Manifest Evidence Records
+        for m in manifests:
+            ev = self.create_evidence(
+                evidence_type=EvidenceType.CONFIG_FILE,
+                summary=f"Discovered manifest {m.manifest_type} ({m.project_name or 'unnamed'} v{m.version or '0.0.0'}) with {len(m.dependencies)} dependencies",
+                raw_payload={
+                    "manifest_type": m.manifest_type,
+                    "file_path": m.file_path,
+                    "project_name": m.project_name,
+                    "version": m.version,
+                    "dependencies": m.dependencies,
+                    "dev_dependencies": m.dev_dependencies,
+                    "scripts": m.scripts
+                },
+                source_uri=m.file_path,
+                locator=f"manifest:{m.file_path}"
+            )
+            m.evidence_id = ev.id
+            evidence_list.append(ev)
+
+        # 4. Create Project Environment & Container Evidence
+        if container_configs or env_templates or lockfiles_detected:
+            evidence_list.append(self.create_evidence(
+                evidence_type=EvidenceType.CONFIG_FILE,
+                summary=f"Discovered project configuration: {len(lockfiles_detected)} lockfiles, {len(container_configs)} containers, {len(env_templates)} env templates",
+                raw_payload={
+                    "lockfiles": lockfiles_detected,
+                    "container_configs": container_configs,
+                    "env_templates": env_templates
+                },
+                source_uri=str(root_path),
+                locator="config:environment"
+            ))
+
+        return evidence_list
+
+    def populate_project_state(self, root_dir: str, canonical_state: CanonicalProjectState) -> ProjectState:
+        """Populates ProjectState.manifests and registers config evidence."""
+        root_path = Path(root_dir).resolve()
+        evidence_list = self.extract(str(root_path))
+
+        manifest_records: List[ManifestInfo] = []
+
+        for ev in evidence_list:
+            canonical_state.add_evidence(ev)
+            if "manifest_type" in ev.raw_payload:
+                m = ManifestInfo(
+                    manifest_type=ev.raw_payload["manifest_type"],
+                    file_path=ev.raw_payload["file_path"],
+                    project_name=ev.raw_payload.get("project_name"),
+                    version=ev.raw_payload.get("version"),
+                    dependencies=ev.raw_payload.get("dependencies", {}),
+                    dev_dependencies=ev.raw_payload.get("dev_dependencies", {}),
+                    scripts=ev.raw_payload.get("scripts", {}),
+                    evidence_id=ev.id
+                )
+                manifest_records.append(m)
+
+        ps = canonical_state.project_state
+        ps.manifests.extend(manifest_records)
+        ps.evidence_ids.extend([ev.id for ev in evidence_list])
+        return ps

extractors/conversation/analyzers.py

@@ -0,0 +1,193 @@
+"""
+Project Continuum - Conversational Intent, Claim & Execution Analyzer
+=====================================================================
+Analyzes conversation transcripts to extract requirements, unverified
+agent claims, architectural decisions, assumptions, unresolved questions,
+file references, and tactical execution state.
+"""
+
+from dataclasses import dataclass, field
+import re
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+from core.enums import Status
+from core.state_models import (
+    AgentClaim,
+    ArchitecturalDecision,
+    NextActionRecommendation,
+    Requirement,
+    TacticalTask,
+    UnresolvedQuestion,
+)
+from extractors.conversation.models import ConversationTranscript, ConversationTurn, TurnRole
+
+
+@dataclass
+class TranscriptAnalysisResult:
+    """Structured insights extracted from a conversation transcript."""
+    requirements: List[Requirement] = field(default_factory=list)
+    agent_claims: List[AgentClaim] = field(default_factory=list)
+    decisions: List[ArchitecturalDecision] = field(default_factory=list)
+    assumptions: List[str] = field(default_factory=list)
+    unresolved_questions: List[UnresolvedQuestion] = field(default_factory=list)
+    tactical_tasks: List[TacticalTask] = field(default_factory=list)
+    referenced_files: List[str] = field(default_factory=list)
+    next_action: Optional[NextActionRecommendation] = None
+
+
+class TranscriptAnalyzer:
+    """
+    Parses conversational dialogue turns and extracts semantic intent, claims,
+    and execution state using pattern matching and ontological heuristics.
+    """
+
+    # Requirements keywords (user intent)
+    REQUIREMENT_PATTERNS = [
+        re.compile(r"(?:we need to|please|i want to|implement|create|build|add support for|must have|requirement:?)\s+([^.\n]+)", re.IGNORECASE),
+        re.compile(r"(?:feature request:?|goal:?)\s+([^.\n]+)", re.IGNORECASE)
+    ]
+
+    # Completion / Progress claims (assistant statements)
+    CLAIM_PATTERNS = [
+        re.compile(r"(?:i have|i've|we have|we've)\s+(?:completed|implemented|finished|created|fixed|verified|tested|added)\s+([^.\n]+)", re.IGNORECASE),
+        re.compile(r"(?:is now|are now)\s+(?:complete|working|passing all tests|implemented|ready)\b([^.\n]*)", re.IGNORECASE),
+        re.compile(r"(?:tests are passing|all tests pass|build succeeded)", re.IGNORECASE)
+    ]
+
+    # Architectural decisions
+    DECISION_PATTERNS = [
+        re.compile(r"(?:decided to|let's use|we will use|architectural decision:?|decision:?|chosen approach:?)\s+([^.\n]+)", re.IGNORECASE),
+        re.compile(r"(?:instead of\s+[\w\s]+,\s*we(?:'ll| will)\s+use\s+([^.\n]+))", re.IGNORECASE)
+    ]
+
+    # Questions & dilemmas
+    QUESTION_PATTERNS = [
+        re.compile(r"^(?:.*?\s+)?([^.!\n]+\?)", re.MULTILINE),
+    ]
+
+    # File path detector
+    FILE_PATH_PATTERN = re.compile(
+        r"""(?:[\w\-_\/.]+\.(?:py|js|ts|tsx|jsx|json|toml|yaml|yml|rs|go|md|txt|sql|html|css|dart|c|cpp|h))""",
+        re.IGNORECASE
+    )
+
+    # Next action patterns
+    NEXT_ACTION_PATTERNS = [
+        re.compile(r"(?:next steps?:?|next action:?|next,?\s+(?:we should|let's|i will))\s+([^.\n]+)", re.IGNORECASE),
+        re.compile(r"(?:now we can proceed to|moving on to)\s+([^.\n]+)", re.IGNORECASE)
+    ]
+
+    @classmethod
+    def analyze(cls, transcript: ConversationTranscript) -> TranscriptAnalysisResult:
+        result = TranscriptAnalysisResult()
+        req_id_counter = 1
+        claim_id_counter = 1
+        dec_id_counter = 1
+        q_id_counter = 1
+        task_id_counter = 1
+
+        all_file_refs: Set[str] = set()
+
+        for turn in transcript.turns:
+            content = turn.content.strip()
+            if not content:
+                continue
+
+            # Detect file references across all turns
+            for match in cls.FILE_PATH_PATTERN.finditer(content):
+                path_match = match.group(0).strip("`'\"(),")
+                if "/" in path_match or "." in path_match:
+                    all_file_refs.add(path_match)
+
+            # 1. Analyze User Turns (Requirements & Assumptions)
+            if turn.role == TurnRole.USER:
+                for pat in cls.REQUIREMENT_PATTERNS:
+                    for match in pat.finditer(content):
+                        title = match.group(1).strip()
+                        if len(title) > 5 and not any(r.title.lower() == title.lower() for r in result.requirements):
+                            result.requirements.append(Requirement(
+                                id=f"req_{req_id_counter:03d}",
+                                title=title,
+                                description=f"Extracted from User turn #{turn.turn_index}: '{match.group(0).strip()}'",
+                                source_turn=turn.turn_index,
+                                status=Status.PENDING
+                            ))
+                            req_id_counter += 1
+
+                # Check for explicit assumptions
+                if "assuming" in content.lower() or "assumption" in content.lower():
+                    result.assumptions.append(f"User turn #{turn.turn_index}: {content[:120]}")
+
+            # 2. Analyze Assistant Turns (Claims, Decisions, Questions, Tasks)
+            elif turn.role == TurnRole.ASSISTANT:
+                # Claims
+                for pat in cls.CLAIM_PATTERNS:
+                    for match in pat.finditer(content):
+                        claim_text = match.group(0).strip()
+                        if len(claim_text) > 8:
+                            result.agent_claims.append(AgentClaim(
+                                id=f"claim_{claim_id_counter:03d}",
+                                claim_text=claim_text,
+                                target_component=cls._extract_target_component(claim_text),
+                                claimed_status=Status.VERIFIED,
+                                source_agent=turn.author_model or "assistant",
+                                turn_id=turn.turn_index
+                            ))
+                            claim_id_counter += 1
+
+                # Architectural Decisions
+                for pat in cls.DECISION_PATTERNS:
+                    for match in pat.finditer(content):
+                        dec_title = match.group(1).strip()
+                        if len(dec_title) > 5:
+                            result.decisions.append(ArchitecturalDecision(
+                                id=f"arch_{dec_id_counter:03d}",
+                                title=dec_title,
+                                rationale=f"Agreed in Assistant turn #{turn.turn_index}",
+                                recorded_at=turn.timestamp
+                            ))
+                            dec_id_counter += 1
+
+                # Unresolved Questions (Assistant asking user)
+                if "?" in content:
+                    for match in cls.QUESTION_PATTERNS:
+                        for q_match in match.finditer(content):
+                            q_text = q_match.group(0).strip()
+                            if len(q_text) > 10 and not any(q.question == q_text for q in result.unresolved_questions):
+                                result.unresolved_questions.append(UnresolvedQuestion(
+                                    id=f"q_{q_id_counter:03d}",
+                                    question=q_text,
+                                    context=f"Asked by assistant at turn #{turn.turn_index}",
+                                    blocking=True
+                                ))
+                                q_id_counter += 1
+
+                # Tactical Tasks (In-flight work)
+                if "task" in content.lower() or "working on" in content.lower() or "currently" in content.lower():
+                    result.tactical_tasks.append(TacticalTask(
+                        id=f"task_{task_id_counter:03d}",
+                        title=f"Turn #{turn.turn_index} action item",
+                        notes=content[:200],
+                        status=Status.IN_PROGRESS
+                    ))
+                    task_id_counter += 1
+
+                # Next Action
+                for pat in cls.NEXT_ACTION_PATTERNS:
+                    match = pat.search(content)
+                    if match:
+                        action_desc = match.group(1).strip()
+                        result.next_action = NextActionRecommendation(
+                            action_type="CONTINUE_EXECUTION",
+                            target_uri="workspace",
+                            description=action_desc
+                        )
+
+        result.referenced_files = sorted(list(all_file_refs))
+        return result
+
+    @staticmethod
+    def _extract_target_component(claim_text: str) -> Optional[str]:
+        """Heuristically extracts symbol or component name from claim text."""
+        words = re.findall(r"[A-Z][a-zA-Z0-9]+|\b[\w\-_\/.]+\.\w+\b", claim_text)
+        return words[0] if words else None

extractors/conversation/models.py

@@ -0,0 +1,148 @@
+"""
+Project Continuum - Vendor-Neutral Conversation Models
+======================================================
+Defines universal representations for conversational transcripts,
+turns, roles, and tool interactions across AI models/platforms.
+"""
+
+from dataclasses import dataclass, field, asdict
+from datetime import datetime, timezone
+from enum import Enum
+import json
+from typing import Any, Dict, List, Optional
+import uuid
+
+
+class TurnRole(str, Enum):
+    USER = "user"
+    ASSISTANT = "assistant"
+    SYSTEM = "system"
+    TOOL = "tool"
+
+
+@dataclass
+class ConversationTurn:
+    """Represents an individual message turn in a conversation."""
+    turn_index: int
+    role: TurnRole
+    content: str
+    timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    author_model: Optional[str] = None
+    tool_calls: List[Dict[str, Any]] = field(default_factory=list)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        d = asdict(self)
+        d["role"] = self.role.value
+        return d
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ConversationTurn":
+        d = dict(data)
+        d["role"] = TurnRole(d.get("role", "user"))
+        if "turn_index" not in d:
+            d["turn_index"] = int(d.pop("turn_id", "1").replace("t", "")) if isinstance(d.get("turn_id"), str) and d.get("turn_id", "").replace("t", "").isdigit() else 1
+        d.pop("turn_id", None)
+        return cls(**d)
+
+
+@dataclass
+class ConversationTranscript:
+    """Vendor-neutral aggregated conversation transcript."""
+    session_id: str = field(default_factory=lambda: f"sess_{uuid.uuid4().hex[:8]}")
+    platform: str = "generic"  # claude, openai, gemini, cursor, custom
+    model_name: Optional[str] = None
+    turns: List[ConversationTurn] = field(default_factory=list)
+    created_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def add_turn(self, role: TurnRole, content: str, author_model: Optional[str] = None, **kwargs) -> ConversationTurn:
+        turn = ConversationTurn(
+            turn_index=len(self.turns) + 1,
+            role=role,
+            content=content,
+            author_model=author_model or self.model_name,
+            **kwargs
+        )
+        self.turns.append(turn)
+        return turn
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "session_id": self.session_id,
+            "platform": self.platform,
+            "model_name": self.model_name,
+            "created_at": self.created_at,
+            "turns": [t.to_dict() for t in self.turns],
+            "metadata": self.metadata
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ConversationTranscript":
+        return cls(
+            session_id=data.get("session_id", f"sess_{uuid.uuid4().hex[:8]}"),
+            platform=data.get("platform", "generic"),
+            model_name=data.get("model_name"),
+            created_at=data.get("created_at", datetime.now(timezone.utc).isoformat()),
+            turns=[ConversationTurn.from_dict(t) for t in data.get("turns", [])],
+            metadata=data.get("metadata", {})
+        )
+
+    @classmethod
+    def from_json(cls, json_str: str) -> "ConversationTranscript":
+        data = json.loads(json_str)
+        if isinstance(data, list):  # Array of messages format
+            transcript = cls()
+            for idx, msg in enumerate(data, start=1):
+                role_str = msg.get("role", "user").lower()
+                role = TurnRole(role_str) if role_str in [r.value for r in TurnRole] else TurnRole.USER
+                transcript.turns.append(ConversationTurn(
+                    turn_index=idx,
+                    role=role,
+                    content=msg.get("content", ""),
+                    author_model=msg.get("model")
+                ))
+            return transcript
+        return cls.from_dict(data)
+
+    @classmethod
+    def from_plain_text(cls, text: str, default_role: TurnRole = TurnRole.USER) -> "ConversationTranscript":
+        """Parses plain text transcripts (e.g., 'User: ...\nAssistant: ...')."""
+        transcript = cls()
+        lines = text.splitlines()
+        current_role = default_role
+        current_buffer: List[str] = []
+        turn_counter = 1
+
+        def flush_turn():
+            nonlocal turn_counter, current_buffer
+            if current_buffer:
+                content = "\n".join(current_buffer).strip()
+                if content:
+                    transcript.turns.append(ConversationTurn(
+                        turn_index=turn_counter,
+                        role=current_role,
+                        content=content
+                    ))
+                    turn_counter += 1
+                current_buffer = []
+
+        for line in lines:
+            lower = line.lower().strip()
+            if lower.startswith("user:") or lower.startswith("human:"):
+                flush_turn()
+                current_role = TurnRole.USER
+                current_buffer.append(line.split(":", 1)[1].strip())
+            elif lower.startswith("assistant:") or lower.startswith("ai:") or lower.startswith("bot:"):
+                flush_turn()
+                current_role = TurnRole.ASSISTANT
+                current_buffer.append(line.split(":", 1)[1].strip())
+            elif lower.startswith("system:"):
+                flush_turn()
+                current_role = TurnRole.SYSTEM
+                current_buffer.append(line.split(":", 1)[1].strip())
+            else:
+                current_buffer.append(line)
+
+        flush_turn()
+        return transcript