alysis-code 0.13.0__py3-none-any.whl

alysis_code/compaction/importance.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from typing import Any
+
+from alysis_code.token_budget import estimate_tokens
+
+_WS_RE = re.compile(r"\s+")
+_REQUIREMENT_RE = re.compile(
+    r"\b(must|do not|never|should|acceptance criteria|requirement|constraint|scope)\b",
+    re.IGNORECASE,
+)
+_ERROR_RE = re.compile(
+    r"\b(traceback|exception|error|failed|failure|stack trace|non-zero exit|exit code)\b",
+    re.IGNORECASE,
+)
+_VERIFY_RE = re.compile(
+    r"\b(pytest|ruff|npm test|pnpm test|cargo test|go test|mvn test)\b",
+    re.IGNORECASE,
+)
+_PATH_RE = re.compile(
+    r"\b([A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)+|[A-Za-z0-9_.-]+\.(py|md|toml|json|yaml|yml|sh|ts|tsx|js|jsx|rs|go))\b"
+)
+_DIFF_RE = re.compile(r"diff --git|@@\s+-\d", re.IGNORECASE)
+_CODE_FENCE_RE = re.compile(r"```")
+_COMMAND_RE = re.compile(
+    r"\b(python3?\s+-m|pip\s+install|git\s+(status|diff|apply|commit|merge)|docker\s+run|bwrap\b)\b",
+    re.IGNORECASE,
+)
+
+
+@dataclass(frozen=True)
+class ScoredTurn:
+    start: int
+    end: int
+    token_estimate: int
+    score: float
+    density: float
+    reasons: list[str]
+    user_preview: str
+
+
+def _normalize_preview(text: str, *, limit: int = 200) -> str:
+    compact = _WS_RE.sub(" ", text).strip()
+    if len(compact) <= limit:
+        return compact
+    return compact[:limit].rstrip() + "..."
+
+
+def extract_text(msg: dict[str, Any]) -> str:
+    content = msg.get("content")
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts: list[str] = []
+        for part in content:
+            if isinstance(part, dict):
+                part_type = str(part.get("type") or "")
+                if part_type == "text":
+                    parts.append(str(part.get("text") or ""))
+                elif part_type == "image_url":
+                    parts.append("<image>")
+                else:
+                    parts.append(str(part))
+            else:
+                parts.append(str(part))
+        return "\n".join(p for p in parts if p)
+    if content is None:
+        return ""
+    return str(content)
+
+
+def score_text(text: str) -> tuple[float, list[str]]:
+    if not text.strip():
+        return 0.0, []
+
+    score = 0.0
+    reasons: list[str] = []
+    lowered = text.lower()
+
+    req_hits = len(_REQUIREMENT_RE.findall(text))
+    if req_hits > 0:
+        score += min(4.0, 1.2 * req_hits)
+        reasons.append("requirements_or_constraints")
+
+    err_hits = len(_ERROR_RE.findall(text))
+    if err_hits > 0:
+        score += min(4.5, 1.5 * err_hits)
+        reasons.append("errors_or_failures")
+
+    verify_hits = len(_VERIFY_RE.findall(text))
+    if verify_hits > 0:
+        score += min(2.0, 0.8 * verify_hits)
+        reasons.append("verification_commands")
+
+    if _DIFF_RE.search(text):
+        score += 1.8
+        reasons.append("diff_or_patch")
+
+    path_hits = len(_PATH_RE.findall(text))
+    if path_hits > 0:
+        score += min(2.5, 0.4 * path_hits)
+        reasons.append("file_paths")
+
+    if _CODE_FENCE_RE.search(text):
+        score += 1.0
+        reasons.append("code_block")
+
+    command_hits = len(_COMMAND_RE.findall(text))
+    if command_hits > 0:
+        score += min(2.5, 0.6 * command_hits)
+        reasons.append("shell_or_git_commands")
+
+    if "acceptance criteria" in lowered:
+        score += 1.5
+        reasons.append("acceptance_criteria")
+
+    deduped: list[str] = []
+    seen: set[str] = set()
+    for reason in reasons:
+        key = reason.casefold()
+        if key in seen:
+            continue
+        seen.add(key)
+        deduped.append(reason)
+    return score, deduped
+
+
+def estimate_turn_tokens(turn_messages: list[dict[str, Any]]) -> int:
+    from alysis_code.compaction.conversation_compactor import (
+        sanitize_messages_for_estimation,
+    )
+
+    payload = json.dumps(
+        sanitize_messages_for_estimation(turn_messages),
+        ensure_ascii=False,
+        sort_keys=True,
+    )
+    return max(1, estimate_tokens(payload))
+
+
+def score_turn(turn_messages: list[dict[str, Any]]) -> tuple[float, list[str], str]:
+    score = 0.0
+    reasons: list[str] = []
+    user_preview = ""
+
+    for msg in turn_messages:
+        if not isinstance(msg, dict):
+            continue
+        role = str(msg.get("role") or "").strip().lower()
+        if role == "user":
+            score += 2.0
+        elif role == "assistant":
+            score += 1.0
+        elif role == "tool":
+            score += 0.5
+
+        text = extract_text(msg)
+        text_score, text_reasons = score_text(text)
+        score += text_score
+        reasons.extend(text_reasons)
+
+        if not user_preview and role == "user":
+            user_preview = _normalize_preview(text)
+
+    deduped: list[str] = []
+    seen: set[str] = set()
+    for reason in reasons:
+        key = reason.casefold()
+        if key in seen:
+            continue
+        seen.add(key)
+        deduped.append(reason)
+
+    return score, deduped, user_preview

alysis_code/compaction/settings.py

@@ -0,0 +1,297 @@
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import Any
+
+from alysis_code.branding import env_get
+from alysis_code.config import AppConfig
+from alysis_code.model_metadata_utils import (
+    parse_bool,
+    parse_non_negative_float,
+    parse_positive_int,
+)
+
+
+@dataclass(frozen=True)
+class CompactionSettings:
+    enabled: bool = True
+    offload_tool_outputs: bool = True
+    tool_output_offload_threshold_chars: int = 6000
+    tool_output_preview_chars: int = 2000
+    summarize_conversation: bool = True
+    recent_user_turns_to_keep: int = 8
+    trigger_ratio: float = 0.90
+    target_ratio: float = 0.70
+    max_chunk_messages: int = 60
+    safety_margin_tokens: int = 512
+    execution_min_removable_messages: int = 2
+    execution_min_removable_tokens: int = 1024
+    importance_enabled: bool = True
+    importance_strategy: str = "lowest_density"
+    pin_score_threshold: float = 6.0
+    max_pins: int = 20
+    max_pins_chars: int = 8000
+    pin_snippet_chars: int = 600
+    importance_use_llm: bool = False
+    importance_llm_max_turns: int = 12
+    cache_aware_compaction: bool = True
+    cache_aware_min_trigger_ratio: float = 0.72
+
+
+def _resolve_bool(*, raw: Any, fallback: bool) -> bool:
+    parsed = parse_bool(raw)
+    return fallback if parsed is None else parsed
+
+
+def _resolve_positive_int(*, raw: Any, fallback: int) -> int:
+    parsed = parse_positive_int(raw)
+    if parsed is None:
+        return fallback
+    return parsed
+
+
+def _resolve_ratio(*, raw: Any, fallback: float) -> float:
+    parsed = parse_non_negative_float(raw)
+    if parsed is None:
+        return fallback
+    if parsed <= 0.05 or parsed >= 0.98:
+        return fallback
+    return parsed
+
+
+def _resolve_non_negative_float(*, raw: Any, fallback: float) -> float:
+    parsed = parse_non_negative_float(raw)
+    if parsed is None:
+        return fallback
+    if not math.isfinite(parsed):
+        return fallback
+    return parsed
+
+
+def _resolve_importance_strategy(*, raw: Any, fallback: str) -> str:
+    value = str(raw).strip().lower()
+    if value in {"oldest", "lowest_density"}:
+        return value
+    return fallback
+
+
+def resolve_compaction_settings(cfg: AppConfig) -> CompactionSettings:
+    """
+    Resolve compaction settings from environment and cfg.extra_fields.
+
+    Precedence:
+    1) defaults
+    2) env vars
+    3) cfg.extra_fields["compaction"] (highest priority)
+    """
+
+    settings = CompactionSettings()
+
+    enabled = _resolve_bool(
+        raw=env_get("ALYSIS_ENABLE_COMPACTION"),
+        fallback=settings.enabled,
+    )
+    offload_tool_outputs = _resolve_bool(
+        raw=env_get("ALYSIS_OFFLOAD_TOOL_OUTPUTS"),
+        fallback=settings.offload_tool_outputs,
+    )
+    threshold_chars = _resolve_positive_int(
+        raw=env_get("ALYSIS_TOOL_OUTPUT_OFFLOAD_THRESHOLD_CHARS"),
+        fallback=settings.tool_output_offload_threshold_chars,
+    )
+    preview_chars = _resolve_positive_int(
+        raw=env_get("ALYSIS_TOOL_OUTPUT_PREVIEW_CHARS"),
+        fallback=settings.tool_output_preview_chars,
+    )
+    summarize_conversation = _resolve_bool(
+        raw=env_get("ALYSIS_SUMMARIZE_CONVERSATION"),
+        fallback=settings.summarize_conversation,
+    )
+    recent_user_turns_to_keep = _resolve_positive_int(
+        raw=env_get("ALYSIS_COMPACTION_RECENT_TURNS"),
+        fallback=settings.recent_user_turns_to_keep,
+    )
+    trigger_ratio = _resolve_ratio(
+        raw=env_get("ALYSIS_COMPACTION_TRIGGER_RATIO"),
+        fallback=settings.trigger_ratio,
+    )
+    target_ratio = _resolve_ratio(
+        raw=env_get("ALYSIS_COMPACTION_TARGET_RATIO"),
+        fallback=settings.target_ratio,
+    )
+    max_chunk_messages = _resolve_positive_int(
+        raw=env_get("ALYSIS_COMPACTION_MAX_CHUNK_MESSAGES"),
+        fallback=settings.max_chunk_messages,
+    )
+    safety_margin_tokens = _resolve_positive_int(
+        raw=env_get("ALYSIS_COMPACTION_SAFETY_MARGIN_TOKENS"),
+        fallback=settings.safety_margin_tokens,
+    )
+    execution_min_removable_messages = _resolve_positive_int(
+        raw=env_get("ALYSIS_EXECUTION_COMPACTION_MIN_REMOVABLE_MESSAGES"),
+        fallback=settings.execution_min_removable_messages,
+    )
+    execution_min_removable_tokens = _resolve_positive_int(
+        raw=env_get("ALYSIS_EXECUTION_COMPACTION_MIN_REMOVABLE_TOKENS"),
+        fallback=settings.execution_min_removable_tokens,
+    )
+    importance_enabled = _resolve_bool(
+        raw=env_get("ALYSIS_IMPORTANCE_ENABLED"),
+        fallback=settings.importance_enabled,
+    )
+    importance_strategy = _resolve_importance_strategy(
+        raw=env_get("ALYSIS_IMPORTANCE_STRATEGY"),
+        fallback=settings.importance_strategy,
+    )
+    pin_score_threshold = _resolve_non_negative_float(
+        raw=env_get("ALYSIS_PIN_SCORE_THRESHOLD"),
+        fallback=settings.pin_score_threshold,
+    )
+    max_pins = _resolve_positive_int(
+        raw=env_get("ALYSIS_MAX_PINS"),
+        fallback=settings.max_pins,
+    )
+    max_pins_chars = _resolve_positive_int(
+        raw=env_get("ALYSIS_MAX_PINS_CHARS"),
+        fallback=settings.max_pins_chars,
+    )
+    pin_snippet_chars = _resolve_positive_int(
+        raw=env_get("ALYSIS_PIN_SNIPPET_CHARS"),
+        fallback=settings.pin_snippet_chars,
+    )
+    importance_use_llm = _resolve_bool(
+        raw=env_get("ALYSIS_IMPORTANCE_USE_LLM"),
+        fallback=settings.importance_use_llm,
+    )
+    importance_llm_max_turns = _resolve_positive_int(
+        raw=env_get("ALYSIS_IMPORTANCE_LLM_MAX_TURNS"),
+        fallback=settings.importance_llm_max_turns,
+    )
+    cache_aware_compaction = _resolve_bool(
+        raw=env_get("ALYSIS_CACHE_AWARE_COMPACTION"),
+        fallback=settings.cache_aware_compaction,
+    )
+    cache_aware_min_trigger_ratio = _resolve_ratio(
+        raw=env_get("ALYSIS_CACHE_AWARE_MIN_TRIGGER_RATIO"),
+        fallback=settings.cache_aware_min_trigger_ratio,
+    )
+
+    raw_compaction = (
+        cfg.extra_fields.get("compaction") if isinstance(cfg.extra_fields, dict) else None
+    )
+    if isinstance(raw_compaction, dict):
+        enabled = _resolve_bool(raw=raw_compaction.get("enabled"), fallback=enabled)
+        offload_tool_outputs = _resolve_bool(
+            raw=raw_compaction.get("offload_tool_outputs"),
+            fallback=offload_tool_outputs,
+        )
+        threshold_chars = _resolve_positive_int(
+            raw=raw_compaction.get("tool_output_offload_threshold_chars"),
+            fallback=threshold_chars,
+        )
+        preview_chars = _resolve_positive_int(
+            raw=raw_compaction.get("tool_output_preview_chars"),
+            fallback=preview_chars,
+        )
+        summarize_conversation = _resolve_bool(
+            raw=raw_compaction.get("summarize_conversation"),
+            fallback=summarize_conversation,
+        )
+        recent_user_turns_to_keep = _resolve_positive_int(
+            raw=raw_compaction.get("recent_user_turns_to_keep"),
+            fallback=recent_user_turns_to_keep,
+        )
+        trigger_ratio = _resolve_ratio(
+            raw=raw_compaction.get("trigger_ratio"),
+            fallback=trigger_ratio,
+        )
+        target_ratio = _resolve_ratio(
+            raw=raw_compaction.get("target_ratio"),
+            fallback=target_ratio,
+        )
+        max_chunk_messages = _resolve_positive_int(
+            raw=raw_compaction.get("max_chunk_messages"),
+            fallback=max_chunk_messages,
+        )
+        safety_margin_tokens = _resolve_positive_int(
+            raw=raw_compaction.get("safety_margin_tokens"),
+            fallback=safety_margin_tokens,
+        )
+        execution_min_removable_messages = _resolve_positive_int(
+            raw=raw_compaction.get("execution_min_removable_messages"),
+            fallback=execution_min_removable_messages,
+        )
+        execution_min_removable_tokens = _resolve_positive_int(
+            raw=raw_compaction.get("execution_min_removable_tokens"),
+            fallback=execution_min_removable_tokens,
+        )
+        importance_enabled = _resolve_bool(
+            raw=raw_compaction.get("importance_enabled"),
+            fallback=importance_enabled,
+        )
+        importance_strategy = _resolve_importance_strategy(
+            raw=raw_compaction.get("importance_strategy"),
+            fallback=importance_strategy,
+        )
+        pin_score_threshold = _resolve_non_negative_float(
+            raw=raw_compaction.get("pin_score_threshold"),
+            fallback=pin_score_threshold,
+        )
+        max_pins = _resolve_positive_int(
+            raw=raw_compaction.get("max_pins"),
+            fallback=max_pins,
+        )
+        max_pins_chars = _resolve_positive_int(
+            raw=raw_compaction.get("max_pins_chars"),
+            fallback=max_pins_chars,
+        )
+        pin_snippet_chars = _resolve_positive_int(
+            raw=raw_compaction.get("pin_snippet_chars"),
+            fallback=pin_snippet_chars,
+        )
+        importance_use_llm = _resolve_bool(
+            raw=raw_compaction.get("importance_use_llm"),
+            fallback=importance_use_llm,
+        )
+        importance_llm_max_turns = _resolve_positive_int(
+            raw=raw_compaction.get("importance_llm_max_turns"),
+            fallback=importance_llm_max_turns,
+        )
+        cache_aware_compaction = _resolve_bool(
+            raw=raw_compaction.get("cache_aware_compaction"),
+            fallback=cache_aware_compaction,
+        )
+        cache_aware_min_trigger_ratio = _resolve_ratio(
+            raw=raw_compaction.get("cache_aware_min_trigger_ratio"),
+            fallback=cache_aware_min_trigger_ratio,
+        )
+
+    if target_ratio >= trigger_ratio:
+        trigger_ratio = settings.trigger_ratio
+        target_ratio = settings.target_ratio
+
+    return CompactionSettings(
+        enabled=enabled,
+        offload_tool_outputs=offload_tool_outputs,
+        tool_output_offload_threshold_chars=threshold_chars,
+        tool_output_preview_chars=preview_chars,
+        summarize_conversation=summarize_conversation,
+        recent_user_turns_to_keep=recent_user_turns_to_keep,
+        trigger_ratio=trigger_ratio,
+        target_ratio=target_ratio,
+        max_chunk_messages=max_chunk_messages,
+        safety_margin_tokens=safety_margin_tokens,
+        execution_min_removable_messages=execution_min_removable_messages,
+        execution_min_removable_tokens=execution_min_removable_tokens,
+        importance_enabled=importance_enabled,
+        importance_strategy=importance_strategy,
+        pin_score_threshold=pin_score_threshold,
+        max_pins=max_pins,
+        max_pins_chars=max_pins_chars,
+        pin_snippet_chars=pin_snippet_chars,
+        importance_use_llm=importance_use_llm,
+        importance_llm_max_turns=importance_llm_max_turns,
+        cache_aware_compaction=cache_aware_compaction,
+        cache_aware_min_trigger_ratio=cache_aware_min_trigger_ratio,
+    )