utim-cli 1.0.0__py3-none-any.whl

utim_cli/reflection.py

@@ -0,0 +1,200 @@
+"""
+Automated Task Reflection & Experience System — Powered by Hugging Face Vector Embeddings.
+
+This module captures learnings, architecture rules, user preferences, and failure corrections
+at the end of agent tasks, indexing them into ChromaDB using Hugging Face model embeddings.
+"""
+
+import os
+import json
+import time
+import uuid
+import sqlite3
+from typing import Dict, List, Optional
+from datetime import datetime
+import requests
+
+
+MEMORY_FILE = ".utim_tmp/task_reflections.json"
+CONVENTIONS_FILE = ".utim_conventions.md"
+
+
+def extract_learnings(user_message: str, assistant_content: str, 
+                      tool_results: List[Dict], llm_key: str, elapsed_seconds: int = 0, iterations: int = 0) -> Dict:
+    """
+    Use LLM to analyze completed work and extract learnings, preferences, conventions, corrections, and time management insights.
+    
+    Returns dictionary with preferences, conventions, rules, corrections, time_reflection, and sub-agent specific learnings.
+    """
+    if not llm_key:
+        return {}
+    
+    # Prepare context for reflection
+    tool_summary = []
+    for r in tool_results[:10]:  # Limit for context
+        name = r.get("func_name", "") or r.get("name", "")
+        result = str(r.get("result", ""))[:300]
+        tool_summary.append(f"- {name}: {result[:100]}...")
+    
+    tool_text = "\n".join(tool_summary)
+    
+    prompt = f"""Analyze this completed interaction and extract ULTRA-CONCISE actionable rules (each strictly under 12 words, no long prose):
+1. User preferences (e.g., "always use pytest")
+2. Project conventions discovered (naming/structure rules)
+3. Architectural rules learned
+4. Failure corrections or reasoning lessons learned
+5. Sub-agent specific rules (for `project_res`, `plan_project`, `web_search`)
+6. Time-management reflection (single rule under 80 chars)
+
+Task:
+User: {user_message}
+
+Assistant: {assistant_content[:600]}
+
+Tools used:
+{tool_text}
+
+Return ONLY a JSON object with the following structure:
+{{
+  "preferences": ["..."],
+  "conventions": ["..."],
+  "rules": ["..."],
+  "corrections": ["..."],
+  "time_reflection": "...",
+  "subagent_learnings": {{
+    "project_res": {{ "rules": ["..."], "experiences": ["..."] }},
+    "plan_project": {{ "rules": ["..."], "experiences": ["..."] }},
+    "web_search": {{ "rules": ["..."], "experiences": ["..."] }}
+  }}
+}}"""
+    
+    try:
+        # Prefer direct API endpoint or router
+        apiUrl = os.environ.get("ROUTER_API_URL", "https://openrouter.ai/api/v1/chat/completions")
+        resp = requests.post(
+            apiUrl,
+            json={
+                "model": "cohere/north-mini-code:free",
+                "messages": [
+                    {"role": "system", "content": "You are a reflection engine. Extract technical learnings, logical corrections, and time adaptations from completed work."},
+                    {"role": "user", "content": prompt}
+                ]
+            },
+            headers={"Authorization": f"Bearer {llm_key}"},
+            timeout=15
+        )
+        if resp.status_code == 200:
+            content = resp.json()["choices"][0]["message"]["content"]
+            import re
+            content = re.sub(r"<think(?:ing)?>.*?</think(?:ing)?>", "", content, flags=re.DOTALL).strip()
+            if content.startswith("```"):
+                lines = content.splitlines()
+                if lines[0].startswith("```"):
+                    lines = lines[1:]
+                if lines[-1].startswith("```"):
+                    lines = lines[:-1]
+                content = "\n".join(lines).strip()
+                if content.startswith("json"):
+                    content = content[4:].strip()
+            return json.loads(content)
+    except Exception:
+        pass
+    
+    return {}
+
+
+def save_learnings(learnings: Dict, project_dir: str = ".", user_message: str = "", assistant_content: str = "", elapsed_seconds: int = 0, iterations: int = 0):
+    """
+    Save learnings to memory.json, .utim_conventions.md, and the Hugging Face Reflection Vector DB.
+    """
+    os.makedirs(".utim_tmp", exist_ok=True)
+    
+    # 1. Update task reflections history
+    if learnings and any(learnings.get(k) for k in ["conventions", "rules", "preferences", "corrections", "subagent_learnings", "time_reflection"]):
+        reflections = []
+        if os.path.exists(MEMORY_FILE):
+            try:
+                with open(MEMORY_FILE, "r", encoding="utf-8") as f:
+                    reflections = json.load(f)
+            except Exception:
+                reflections = []
+        
+        entry = {
+            "timestamp": datetime.now().isoformat(),
+            "user_task": user_message[:200],
+            "learnings": learnings
+        }
+        reflections.append(entry)
+        reflections = reflections[-100:]
+        
+        with open(MEMORY_FILE, "w", encoding="utf-8") as f:
+            json.dump(reflections, f, indent=2)
+    
+    # 2. Update project conventions file
+    conventions_path = os.path.join(project_dir, CONVENTIONS_FILE)
+    existing = ""
+    if os.path.exists(conventions_path):
+        try:
+            with open(conventions_path, "r", encoding="utf-8") as f:
+                existing = f.read()
+        except Exception:
+            pass
+    
+    new_section = ""
+    if learnings.get("conventions"):
+        new_section += f"\n\n## Conventions (added {datetime.now().strftime('%Y-%m-%d %H:%M')})\n"
+        for c in learnings["conventions"]:
+            new_section += f"- {c}\n"
+    if learnings.get("rules"):
+        new_section += f"\n### Architectural Rules\n"
+        for r in learnings["rules"]:
+            new_section += f"- {r}\n"
+    if learnings.get("preferences"):
+        new_section += f"\n### Preferences\n"
+        for p in learnings["preferences"]:
+            new_section += f"- {p}\n"
+            
+    if new_section:
+        try:
+            with open(conventions_path, "w", encoding="utf-8") as f:
+                f.write(existing + new_section)
+        except Exception:
+            pass
+            
+    # 3. Save to Vector Memory DB using Hugging Face model
+    try:
+        from utim_cli.vector_memory import get_reflections_memory, store_reflection
+        
+        for pref in learnings.get("preferences", []):
+            store_reflection(content=pref, category="user_preference", task_prompt=user_message)
+            
+        for rule in learnings.get("rules", []):
+            store_reflection(content=rule, category="architectural_rule", task_prompt=user_message)
+            
+        for conv in learnings.get("conventions", []):
+            store_reflection(content=conv, category="project_convention", task_prompt=user_message)
+            
+        for corr in learnings.get("corrections", []):
+            store_reflection(content=corr, category="failure_correction", task_prompt=user_message)
+            
+        if user_message and assistant_content:
+            summary = f"Task: {user_message[:300]}\nResolution: {assistant_content[:400]}"
+            store_reflection(content=summary, category="task_experience", task_prompt=user_message)
+    except Exception as e:
+        pass
+
+
+def run_reflection_phase(user_message: str, assistant_content: str,
+                         tool_results: List[Dict], elapsed_seconds: int = 0, iterations: int = 0) -> Dict:
+    """
+    Main entry point for the reflection phase.
+    Analyzes completed work and stores learnings for future tasks.
+    """
+    llm_key = os.getenv("OPENROUTER_API_KEY") or os.getenv("UTIM_API_KEY") or "mock_key"
+    learnings = {}
+    
+    if llm_key and llm_key != "mock_key":
+        learnings = extract_learnings(user_message, assistant_content, tool_results, llm_key, elapsed_seconds, iterations)
+    
+    save_learnings(learnings, user_message=user_message, assistant_content=assistant_content, elapsed_seconds=elapsed_seconds, iterations=iterations)
+    return learnings

utim_cli/report.py

@@ -0,0 +1,100 @@
+import os
+import re
+import sys
+import shutil
+import zipfile
+import io
+from utim_cli.logger import redact_text, LOG_FILE
+from utim_cli.doctor import run_diagnostics
+from rich.console import Console
+
+# ── Unicode → ASCII symbol map ────────────────────────────────────────────────
+_UNICODE_TO_ASCII: list = [
+    # Rich / doctor symbols
+    ("\u2713", "[OK]"),       # ✓
+    ("\u2717", "[FAIL]"),     # ✗
+    ("\u2022", "-"),          # •
+    ("\u2b21", "#"),          # ⬡
+    ("\u2026", "..."),        # …
+    ("\u2192", "->"),         # →
+    ("\u2714", "[OK]"),       # ✔
+    ("\u2718", "[FAIL]"),     # ✘
+    ("\u25b6", ">"),          # ▶
+    ("\u25cf", "*"),          # ●
+    # Emoji used in doctor / report
+    ("\U0001f4c4", "[file]"),
+    ("\u270f",     "[edit]"),
+    ("\U0001f5d1", "[del]"),
+    ("\U0001f4e6", "[pkg]"),
+    ("\u26a1",     "[run]"),
+    ("\U0001f4c1", "[dir]"),
+    ("\U0001f50d", "[search]"),
+    ("\U0001f9e0", "[ai]"),
+]
+
+_ANSI_RE = re.compile(r"\x1b\[[0-9;]*[mKHFABCDJsu]")
+
+
+def _to_ascii(text: str) -> str:
+    """Strip ANSI escape codes and replace Unicode symbols with ASCII equivalents.
+
+    Any remaining non-ASCII character (e.g. user-supplied filenames or log
+    entries) is replaced with '?' so the output is always 7-bit clean and safe
+    to print on any Windows code page.
+    """
+    # 1. Remove ANSI colour/cursor escape sequences
+    text = _ANSI_RE.sub("", text)
+    # 2. Map known symbols to ASCII stand-ins
+    for uni, asc in _UNICODE_TO_ASCII:
+        text = text.replace(uni, asc)
+    # 3. Encode to ASCII, replacing anything still non-ASCII
+    return text.encode("ascii", errors="replace").decode("ascii")
+
+
+def create_report_bundle() -> str:
+    """Create a support report zip bundle with sensitive data redacted.
+
+    The file written inside the zip is ASCII-only so it can be opened and
+    printed on any Windows console regardless of the active code page.
+    """
+    report_dir = ".utim_tmp"
+    os.makedirs(report_dir, exist_ok=True)
+
+    report_txt_path = os.path.join(report_dir, "support_report.txt")
+    bundle_zip_path = os.path.join(report_dir, "report_bundle.zip")
+
+    try:
+        # Capture Rich diagnostics into a StringIO buffer (no terminal needed)
+        buf = io.StringIO()
+        buf_console = Console(file=buf, force_terminal=False, width=100)
+        run_diagnostics(buf_console)
+        diagnostics_text = buf.getvalue()
+
+        with open(report_txt_path, "w", encoding="utf-8") as f:
+            f.write("=== UTIM SUPPORT REPORT ===\n")
+            ts = os.path.getmtime(LOG_FILE) if os.path.exists(LOG_FILE) else "unknown"
+            f.write(f"Timestamp: {ts}\n\n")
+
+            f.write("=== DIAGNOSTICS ===\n")
+            # Redact first, then ASCII-ify so redaction markers stay readable
+            f.write(_to_ascii(redact_text(diagnostics_text)) + "\n\n")
+
+            f.write("=== REDACTED DEBUG LOG ===\n")
+            if os.path.exists(LOG_FILE):
+                with open(LOG_FILE, "r", encoding="utf-8") as lf:
+                    log_content = lf.read()
+                f.write(_to_ascii(redact_text(log_content)))
+            else:
+                f.write("(no debug log found)\n")
+
+        with zipfile.ZipFile(bundle_zip_path, "w", zipfile.ZIP_DEFLATED) as z:
+            z.write(report_txt_path, "support_report.txt")
+
+        try:
+            os.remove(report_txt_path)
+        except Exception:
+            pass
+
+        return bundle_zip_path
+    except Exception as e:
+        raise RuntimeError(f"Failed to create support bundle: {e}")

utim_cli/scrapy_search.py

@@ -0,0 +1,229 @@
+"""
+Scrapy-based web scraping enhancement for the web_search tool.
+Provides robust, async-capable scraping with proper HTTP semantics.
+"""
+
+import asyncio
+import os
+from typing import Dict, List, Optional
+from dataclasses import dataclass
+
+
+@dataclass
+class ScrapedContent:
+    """Container for scraped page content."""
+    url: str
+    title: str
+    text: str
+    error: Optional[str] = None
+
+
+async def scrape_urls_with_playwright(urls: List[str], use_js: bool = False, timeout: int = 10) -> Dict[str, str]:
+    """
+    Scrape URLs using Scrapy with optional Playwright for JavaScript rendering.
+    
+    Args:
+        urls: List of URLs to scrape
+        use_js: Whether to use Playwright for JavaScript-heavy sites
+        timeout: Request timeout in seconds
+    
+    Returns:
+        Dictionary mapping URL to scraped text content
+    """
+    from scrapy import Spider, Request
+    from scrapy.crawler import CrawlerProcess
+    from scrapy.utils.project import get_project_settings
+    from scrapy.utils.log import configure_logging
+    
+    results = {}
+    
+    if use_js and len(urls) > 0:
+        # Use Playwright for JS rendering
+        return await _scrape_with_playwright(urls, timeout)
+    else:
+        # Use regular Scrapy for static content
+        return await _scrape_static(urls, timeout)
+
+
+async def _scrape_with_playwright(urls: List[str], timeout: int) -> Dict[str, str]:
+    """Scrape URLs using Playwright for JavaScript rendering."""
+    try:
+        from playwright.async_api import async_playwright
+    except ImportError:
+        # Fallback to static scraping if Playwright not available
+        return await _scrape_static(urls, timeout)
+    
+    results = {}
+    
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True)
+        context = await browser.new_context(
+            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36"
+        )
+        
+        for url in urls:
+            try:
+                page = await context.new_page()
+                await page.goto(url, wait_until="networkidle", timeout=timeout * 1000)
+                content = await page.content()
+                
+                # Extract text content
+                from bs4 import BeautifulSoup
+                soup = BeautifulSoup(content, 'html.parser')
+                
+                # Remove script and style elements
+                for script in soup(["script", "style", "nav", "header", "footer", "aside"]):
+                    script.decompose()
+                
+                text = soup.get_text(separator='\n', strip=True)
+                # Clean up whitespace
+                lines = [line.strip() for line in text.splitlines() if line.strip()]
+                results[url] = '\n'.join(lines)[:6000]
+                
+                await page.close()
+            except Exception as e:
+                results[url] = f"Error: {str(e)}"
+        
+        await context.close()
+        await browser.close()
+    
+    return results
+
+
+async def _scrape_static(urls: List[str], timeout: int) -> Dict[str, str]:
+    """Scrape URLs using standard Scrapy (no JavaScript)."""
+    import shutil
+    
+    # Check if scrapy-playwright is available
+    use_playwright = shutil.which('playwright') is not None
+    
+    # Create a temporary Scrapy project settings
+    from scrapy.settings import Settings
+    
+    settings = Settings()
+    settings.set('USER_AGENT', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
+    settings.set('ROBOTSTXT_OBEY', True)
+    settings.set('DOWNLOAD_TIMEOUT', timeout)
+    settings.set('CONCURRENT_REQUESTS', min(len(urls), 8))
+    settings.set('CONCURRENT_REQUESTS_PER_DOMAIN', 2)
+    settings.set('RETRY_TIMES', 2)
+    settings.set('LOG_LEVEL', 'ERROR')
+    
+    # Configure Playwright if available
+    if use_playwright:
+        settings.set('PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT', timeout * 1000)
+    
+    # Import here to avoid issues if not installed
+    try:
+        from scrapy.crawler import CrawlerRunner
+        from twisted.internet import asyncioreactor
+        asyncioreactor.install()
+    except ImportError:
+        # Fall back to simple requests if Scrapy has issues
+        return _fallback_requests_scraping(urls, timeout)
+    
+    # Create inline spider
+    from scrapy import Spider
+    from itemadapter import ItemAdapter
+    
+    class InlineSpider(Spider):
+        name = 'inline_spider'
+        
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            self.results = {}
+            self.urls_to_scrape = urls
+        
+        def start_requests(self):
+            for url in self.urls_to_scrape:
+                yield Request(url=url, callback=self.parse, errback=self.errback)
+        
+        def parse(self, response):
+            # Extract text with BeautifulSoup
+            try:
+                from bs4 import BeautifulSoup
+                soup = BeautifulSoup(response.text, 'html.parser')
+                
+                # Remove unwanted elements
+                for elem in soup(["script", "style", "nav", "header", "footer", "aside"]):
+                    elem.decompose()
+                
+                text = soup.get_text(separator='\n', strip=True)
+                lines = [line.strip() for line in text.splitlines() if line.strip()]
+                self.results[response.url] = '\n'.join(lines)[:6000]
+            except Exception as e:
+                self.results[response.url] = f"Parse error: {str(e)}"
+            
+            yield {'url': response.url}
+        
+        def errback(self, failure):
+            self.results[failure.request.url] = f"Error: {str(failure.value)}"
+    
+    # Run the spider
+    from twisted.internet import reactor
+    from scrapy.crawler import CrawlerRunner
+    
+    runner = CrawlerRunner(settings)
+    spider = InlineSpider()
+    
+    try:
+        d = runner.crawl(spider)
+        d.addBoth(lambda _: reactor.stop())
+        reactor.run()
+    except Exception:
+        # Fallback if reactor issues
+        return _fallback_requests_scraping(urls, timeout)
+    
+    return spider.results or _fallback_requests_scraping(urls, timeout)
+
+
+def _fallback_requests_scraping(urls: List[str], timeout: int) -> Dict[str, str]:
+    """Fallback to requests-based scraping if Scrapy fails."""
+    import requests
+    import re
+    import html as html_lib
+    
+    results = {}
+    
+    for url in urls:
+        try:
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+            }
+            r = requests.get(url, headers=headers, timeout=timeout)
+            if r.status_code == 200:
+                html_content = r.text
+                # Remove scripts, styles
+                html_content = re.sub(r'<(script|style).*?>.*?</\1>', '', html_content, flags=re.DOTALL | re.IGNORECASE)
+                # Remove html tags
+                text = re.sub(r'<.*?>', ' ', html_content)
+                text = html_lib.unescape(text)
+                # Format whitespace
+                lines = [l.strip() for l in text.splitlines() if l.strip()]
+                results[url] = '\n'.join(lines)[:6000]
+        except Exception as e:
+            results[url] = f"Error: {str(e)}"
+    
+    return results
+
+
+def enhanced_scrape_urls(urls: List[str], use_js: bool = False, timeout: int = 10) -> Dict[str, str]:
+    """
+    Synchronous wrapper for scrape_urls_with_playwright.
+    
+    This is the main entry point used by web_search.
+    """
+    try:
+        # Try async approach first
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            # If event loop is already running, use a new one
+            return asyncio.run(scrape_urls_with_playwright(urls, use_js, timeout))
+        else:
+            return loop.run_until_complete(scrape_urls_with_playwright(urls, use_js, timeout))
+    except RuntimeError:
+        # No event loop, create one
+        return asyncio.run(scrape_urls_with_playwright(urls, use_js, timeout))
+    except Exception:
+        # Fallback to sync requests
+        return _fallback_requests_scraping(urls, timeout)