vishap-analyzer 5.8.0__py3-none-any.whl

src/api/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+"""API helpers: auth, rate limit."""

src/api/auth.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+"""API-key authentication (dogfood P0 fix).
+
+Dogfood finding: POST /analyze, /history/clear, /mode were open
+to any client on 0.0.0.0:5001.
+
+Behavior:
+- If CONVEYOR_API_KEY is set in .env -> require X-API-Key header.
+- If CONVEYOR_API_KEY is NOT set -> API works ONLY on 127.0.0.1
+  (see main.py uvicorn.run binding).
+"""
+import os
+from fastapi import Header, HTTPException, status
+
+API_KEY = os.getenv("CONVEYOR_API_KEY", "").strip()
+
+
+async def verify_api_key(x_api_key: str = Header(default="")) -> None:
+    """FastAPI dependency: verify X-API-Key header.
+
+    If API_KEY is empty (dev mode), allow all (main.py binds to 127.0.0.1).
+    If API_KEY is set, require exact match.
+    """
+    if not API_KEY:
+        return
+
+    if not x_api_key:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="X-API-Key header required",
+        )
+    if x_api_key != API_KEY:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Invalid API key",
+        )
+
+
+def is_auth_enabled() -> bool:
+    """True if CONVEYOR_API_KEY is set."""
+    return bool(API_KEY)

src/api/rate_limit.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+"""Simple in-memory rate limiter (dogfood P0 fix).
+
+Dogfood finding: no rate limiting on /analyze -> any client could
+exhaust DeepSeek API budget.
+
+Limits (per IP):
+- 10 requests / minute for /analyze, /analyze/batch
+- 30 requests / minute for /history, /mode
+"""
+import time
+from collections import defaultdict, deque
+from fastapi import Request, HTTPException, status
+
+
+_BUCKETS = defaultdict(deque)
+
+LIMIT_ANALYZE = 10
+LIMIT_READ = 30
+WINDOW_SECONDS = 60
+
+
+def _check(key: str, limit: int) -> None:
+    now = time.time()
+    bucket = _BUCKETS[key]
+
+    while bucket and bucket[0] < now - WINDOW_SECONDS:
+        bucket.popleft()
+
+    if len(bucket) >= limit:
+        raise HTTPException(
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            detail="Rate limit exceeded: {} req / {}s".format(limit, WINDOW_SECONDS),
+        )
+
+    bucket.append(now)
+
+
+async def rate_limit_analyze(request: Request) -> None:
+    ip = request.client.host if request.client else "unknown"
+    _check("analyze:" + ip, LIMIT_ANALYZE)
+
+
+async def rate_limit_read(request: Request) -> None:
+    ip = request.client.host if request.client else "unknown"
+    _check("read:" + ip, LIMIT_READ)

src/cli.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+"""Vishap CLI entry point.
+
+Usage:
+    vishap analyze <file>
+    vishap info
+    vishap --version
+"""
+import argparse
+import sys
+from pathlib import Path
+
+from src.config import APP_NAME
+from src.pipeline.orchestrator import run_pipeline
+
+
+def cmd_analyze(args):
+    """Analyze a file."""
+    path = Path(args.input)
+    if not path.exists():
+        print(f"Error: {path} not found", file=sys.stderr)
+        sys.exit(1)
+
+    material = path.read_text(encoding="utf-8", errors="replace")
+    result = run_pipeline(material, path.name, verbose=not args.quiet)
+
+    md = result.get("final_markdown", "")
+    audit = result.get("audit", {})
+
+    if args.output:
+        Path(args.output).write_text(md, encoding="utf-8")
+        print(f"Written: {args.output}")
+    else:
+        print(md)
+        print()
+        print(f"audit_id:      {audit.get('audit_id')}")
+        print(f"material_hash: {audit.get('material_hash')}")
+        print(f"verdict:       {audit.get('critique_verdict')} / {audit.get('risk_verdict')}")
+
+
+def cmd_info(args):
+    """Show info."""
+    print(f"{APP_NAME} V5.8 - 6-stage LLM pipeline")
+    print("Pipeline: Stage 0-5 (Orchestrator > Scanner > Analyst > Researcher > Synthesizer > Transmitter)")
+    print("Features: audit trail, PII redaction, compliance flags, SAST/SCA, SARIF, retry, fallback")
+    print()
+    print("Commands:")
+    print("  vishap analyze <file>   - analyze a file")
+    print("  vishap info             - this help")
+    print()
+    print("API: python -m src.main   - FastAPI on :5001")
+    print("UI:  streamlit run src/ui/streamlit_app.py  - Streamlit on :8501")
+
+
+def main():
+    ap = argparse.ArgumentParser(prog="vishap", description=f"{APP_NAME} V5.8")
+    ap.add_argument("--version", action="version", version=f"{APP_NAME} V5.8")
+
+    sub = ap.add_subparsers(dest="command")
+
+    p_analyze = sub.add_parser("analyze", help="Analyze a file")
+    p_analyze.add_argument("input", help="Path to file")
+    p_analyze.add_argument("--quiet", action="store_true")
+    p_analyze.add_argument("--output", "-o", help="Output markdown path")
+    p_analyze.set_defaults(func=cmd_analyze)
+
+    p_info = sub.add_parser("info", help="Show info")
+    p_info.set_defaults(func=cmd_info)
+
+    args = ap.parse_args()
+    if not args.command:
+        ap.print_help()
+        sys.exit(1)
+
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

src/config.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+"""
+Конфигурация Vishap.
+"""
+import os
+from pathlib import Path
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# === Пути ===
+ROOT_DIR = Path(__file__).parent.parent
+DATA_DIR = ROOT_DIR / "data"
+PROMPTS_DIR = ROOT_DIR / "vishap"
+TOOLS_DIR = ROOT_DIR / "src" / "tools"
+
+# === DeepSeek API ===
+DEEPSEEK_API_KEY = os.getenv("DEEPSEEK_API_KEY", "")
+DEEPSEEK_URL = "https://api.deepseek.com/v1/chat/completions"
+DEEPSEEK_MODEL = "deepseek-chat"
+
+# === Ollama (локально) ===
+OLLAMA_MODEL = "qwen2.5:7b"
+OLLAMA_URL = "http://localhost:11434"
+
+# === Режим LLM ===
+MODE = "auto"
+
+# === Генерация ===
+TEMPERATURE = 0.0
+TEMPERATURE_CREATIVE = 0.3
+MAX_TOKENS = 8000
+
+# === Сервер ===
+PORT = 5001
+APP_NAME = "Vishap"
+
+# === Данные ===
+DATA_DIR.mkdir(exist_ok=True)

src/llm/async_client.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+"""
+Асинхронный клиент LLM с fallback:
+  cloud → DeepSeek API
+  local → Ollama
+  auto  → облако, при ошибке → локально
+"""
+from src.storage.mode import get_mode
+from src.llm import async_deepseek_client, async_ollama_client
+
+
+async def async_chat(system: str, user: str,
+                     temperature: float = 0.0,
+                     max_tokens: int = 2000) -> dict:
+    """Асинхронный вызов LLM. Аналог chat() с fallback."""
+    mode = get_mode()
+
+    if mode == "local":
+        result = await async_ollama_client.chat(system, user, temperature)
+        result["mode"] = "local"
+        return result
+
+    if mode == "cloud":
+        result = await async_deepseek_client.chat(system, user, temperature, max_tokens)
+        result["mode"] = "cloud"
+        return result
+
+    # auto
+    try:
+        result = await async_deepseek_client.chat(system, user, temperature, max_tokens)
+        result["mode"] = "cloud"
+        return result
+    except Exception as e:
+        print(f"[Vishap] DeepSeek недоступен: {e}. Fallback → локально.")
+        try:
+            result = await async_ollama_client.chat(system, user, temperature)
+            result["mode"] = "local"
+            return result
+        except Exception as e2:
+            return {
+                "text": f"[Ошибка LLM] cloud: {e}; local: {e2}",
+                "tokens_in": 0, "tokens_out": 0,
+                "model": "none", "mode": "error",
+            }

src/llm/async_deepseek_client.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+"""Асинхронный клиент DeepSeek API (через openai SDK)."""
+import logging
+
+from openai import AsyncOpenAI, APIConnectionError, RateLimitError, APIStatusError
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_exponential,
+    retry_if_exception_type,
+    before_sleep_log,
+)
+
+from src.config import DEEPSEEK_API_KEY, DEEPSEEK_MODEL
+
+logger = logging.getLogger(__name__)
+
+
+def get_async_client() -> AsyncOpenAI:
+    if not DEEPSEEK_API_KEY:
+        raise ValueError("DEEPSEEK_API_KEY не задан в .env")
+    return AsyncOpenAI(
+        api_key=DEEPSEEK_API_KEY,
+        base_url="https://api.deepseek.com/v1",
+    )
+
+
+@retry(
+    stop=stop_after_attempt(3),
+    wait=wait_exponential(multiplier=1, min=2, max=30),
+    retry=retry_if_exception_type((APIConnectionError, RateLimitError, APIStatusError)),
+    before_sleep=before_sleep_log(logger, logging.WARNING),
+    reraise=True,
+)
+async def chat(system: str, user: str,
+               temperature: float = 0.0,
+               max_tokens: int = 2000) -> dict:
+    """Асинхронный вызов DeepSeek. Возвращает {text, tokens_in, tokens_out, model}."""
+    client = get_async_client()
+    response = await client.chat.completions.create(
+        model=DEEPSEEK_MODEL,
+        messages=[
+            {"role": "system", "content": system},
+            {"role": "user", "content": user},
+        ],
+        temperature=temperature,
+        max_tokens=max_tokens,
+    )
+    return {
+        "text": response.choices[0].message.content,
+        "tokens_in": response.usage.prompt_tokens,
+        "tokens_out": response.usage.completion_tokens,
+        "model": DEEPSEEK_MODEL,
+    }

src/llm/async_ollama_client.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+"""Асинхронный клиент Ollama (локальный LLM)."""
+import ollama
+from src.config import OLLAMA_MODEL
+
+
+async def chat(system: str, user: str,
+               temperature: float = 0.0) -> dict:
+    """Асинхронный вызов Ollama. Возвращает {text, tokens_in, tokens_out, model}."""
+    client = ollama.AsyncClient()
+    response = await client.chat(
+        model=OLLAMA_MODEL,
+        messages=[
+            {"role": "system", "content": system},
+            {"role": "user", "content": user},
+        ],
+        options={"temperature": temperature},
+    )
+    return {
+        "text": response["message"]["content"],
+        "tokens_in": 0,
+        "tokens_out": 0,
+        "model": OLLAMA_MODEL,
+    }

src/llm/client.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+"""
+Единый клиент LLM с fallback:
+  cloud → DeepSeek API
+  local → Ollama
+  auto  → облако, при ошибке → локально
+"""
+from src.storage.mode import get_mode
+from src.llm import deepseek_client, ollama_client
+
+
+def chat(system: str, user: str,
+         temperature: float = 0.0,
+         max_tokens: int = 2000) -> dict:
+    """Возвращает {text, tokens_in, tokens_out, model, mode}."""
+    mode = get_mode()
+
+    if mode == "local":
+        result = ollama_client.chat(system, user, temperature)
+        result["mode"] = "local"
+        return result
+
+    if mode == "cloud":
+        result = deepseek_client.chat(system, user, temperature, max_tokens)
+        result["mode"] = "cloud"
+        return result
+
+    # auto
+    try:
+        result = deepseek_client.chat(system, user, temperature, max_tokens)
+        result["mode"] = "cloud"
+        return result
+    except Exception as e:
+        print(f"[Vishap] DeepSeek недоступен: {e}. Fallback → локально.")
+        try:
+            result = ollama_client.chat(system, user, temperature)
+            result["mode"] = "local"
+            return result
+        except Exception as e2:
+            return {
+                "text": f"[Ошибка LLM] cloud: {e}; local: {e2}",
+                "tokens_in": 0, "tokens_out": 0,
+                "model": "none", "mode": "error",
+            }

src/llm/deepseek_client.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+"""
+Клиент DeepSeek API (через openai SDK).
+"""
+import logging
+
+from openai import OpenAI, APIConnectionError, RateLimitError, APIStatusError
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_exponential,
+    retry_if_exception_type,
+    before_sleep_log,
+)
+
+from src.config import DEEPSEEK_API_KEY, DEEPSEEK_URL, DEEPSEEK_MODEL
+
+logger = logging.getLogger(__name__)
+
+
+def get_client() -> OpenAI:
+    if not DEEPSEEK_API_KEY:
+        raise ValueError("DEEPSEEK_API_KEY не задан в .env")
+    return OpenAI(
+        api_key=DEEPSEEK_API_KEY,
+        base_url="https://api.deepseek.com/v1",
+        timeout=60.0,        # dogfood P0: HTTP timeout (was default 600s)
+        max_retries=0,       # retries handled by tenacity, not openai
+    )
+
+
+@retry(
+    stop=stop_after_attempt(3),
+    wait=wait_exponential(multiplier=1, min=2, max=30),
+    retry=retry_if_exception_type((APIConnectionError, RateLimitError, APIStatusError, TimeoutError)),
+    before_sleep=before_sleep_log(logger, logging.WARNING),
+    reraise=True,
+)
+def chat(system: str, user: str,
+         temperature: float = 0.0,
+         max_tokens: int = 2000) -> dict:
+    """Возвращает {text, tokens_in, tokens_out, model}."""
+    client = get_client()
+    response = client.chat.completions.create(
+        model=DEEPSEEK_MODEL,
+        messages=[
+            {"role": "system", "content": system},
+            {"role": "user", "content": user},
+        ],
+        temperature=temperature,
+        max_tokens=max_tokens,
+    )
+    return {
+        "text": response.choices[0].message.content,
+        "tokens_in": response.usage.prompt_tokens,
+        "tokens_out": response.usage.completion_tokens,
+        "model": DEEPSEEK_MODEL,
+    }

src/llm/ollama_client.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+"""
+Клиент Ollama (локальный LLM).
+"""
+import ollama
+from src.config import OLLAMA_MODEL
+
+
+def chat(system: str, user: str,
+         temperature: float = 0.0) -> dict:
+    """Возвращает {text, tokens_in, tokens_out, model}."""
+    response = ollama.chat(
+        model=OLLAMA_MODEL,
+        messages=[
+            {"role": "system", "content": system},
+            {"role": "user", "content": user},
+        ],
+        options={"temperature": temperature},
+    )
+    return {
+        "text": response["message"]["content"],
+        "tokens_in": 0,
+        "tokens_out": 0,
+        "model": OLLAMA_MODEL,
+    }