shadow-debugger 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shadow_debugger/__init__.py +293 -0
- shadow_debugger/_version.py +1 -0
- shadow_debugger/analyzer.py +108 -0
- shadow_debugger/config.py +56 -0
- shadow_debugger/core.py +244 -0
- shadow_debugger/inspector.py +176 -0
- shadow_debugger/integrations/__init__.py +224 -0
- shadow_debugger/patch_check.py +57 -0
- shadow_debugger/providers/__init__.py +23 -0
- shadow_debugger/providers/anthropic.py +54 -0
- shadow_debugger/providers/base.py +92 -0
- shadow_debugger/providers/gemini.py +66 -0
- shadow_debugger/providers/ollama.py +65 -0
- shadow_debugger/providers/openai.py +63 -0
- shadow_debugger/py.typed +1 -0
- shadow_debugger/redaction.py +190 -0
- shadow_debugger/reporter.py +128 -0
- shadow_debugger/throttler.py +64 -0
- shadow_debugger-0.1.0.dist-info/METADATA +292 -0
- shadow_debugger-0.1.0.dist-info/RECORD +23 -0
- shadow_debugger-0.1.0.dist-info/WHEEL +5 -0
- shadow_debugger-0.1.0.dist-info/licenses/LICENSE +21 -0
- shadow_debugger-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Built-in LLM providers. SDKs are imported lazily, only when a provider is used."""
|
|
2
|
+
|
|
3
|
+
from .base import (
|
|
4
|
+
Provider,
|
|
5
|
+
ProviderError,
|
|
6
|
+
available_providers,
|
|
7
|
+
create_provider,
|
|
8
|
+
register_provider,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
# Importing the modules registers the built-in providers.
|
|
12
|
+
from . import anthropic as _anthropic # noqa: F401,E402
|
|
13
|
+
from . import gemini as _gemini # noqa: F401,E402
|
|
14
|
+
from . import ollama as _ollama # noqa: F401,E402
|
|
15
|
+
from . import openai as _openai # noqa: F401,E402
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"Provider",
|
|
19
|
+
"ProviderError",
|
|
20
|
+
"available_providers",
|
|
21
|
+
"create_provider",
|
|
22
|
+
"register_provider",
|
|
23
|
+
]
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Anthropic Claude provider (``pip install shadow-debugger[anthropic]``)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import threading
|
|
6
|
+
from typing import Any, Optional
|
|
7
|
+
|
|
8
|
+
from .base import Provider, ProviderError, register_provider
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@register_provider("anthropic", "claude")
|
|
12
|
+
class AnthropicProvider(Provider):
|
|
13
|
+
default_model = "claude-haiku-5-5"
|
|
14
|
+
env_key = "ANTHROPIC_API_KEY"
|
|
15
|
+
max_tokens = 2048
|
|
16
|
+
|
|
17
|
+
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
18
|
+
super().__init__(*args, **kwargs)
|
|
19
|
+
self._client: Optional[Any] = None
|
|
20
|
+
self._client_lock = threading.Lock()
|
|
21
|
+
|
|
22
|
+
def _get_client(self) -> Any:
|
|
23
|
+
with self._client_lock:
|
|
24
|
+
if self._client is None:
|
|
25
|
+
try:
|
|
26
|
+
import anthropic
|
|
27
|
+
except ImportError as exc:
|
|
28
|
+
raise ProviderError(
|
|
29
|
+
"anthropic: package not installed. Run: pip install 'shadow-debugger[anthropic]'"
|
|
30
|
+
) from exc
|
|
31
|
+
kwargs: dict = {"api_key": self.api_key, "timeout": self.timeout, "max_retries": 2}
|
|
32
|
+
if self.base_url:
|
|
33
|
+
kwargs["base_url"] = self.base_url
|
|
34
|
+
self._client = anthropic.Anthropic(**kwargs)
|
|
35
|
+
return self._client
|
|
36
|
+
|
|
37
|
+
def complete(self, system: str, prompt: str) -> str:
|
|
38
|
+
self.check_configuration()
|
|
39
|
+
client = self._get_client()
|
|
40
|
+
try:
|
|
41
|
+
response = client.messages.create(
|
|
42
|
+
model=self.model,
|
|
43
|
+
max_tokens=self.max_tokens,
|
|
44
|
+
system=system,
|
|
45
|
+
messages=[{"role": "user", "content": prompt}],
|
|
46
|
+
)
|
|
47
|
+
except Exception as exc:
|
|
48
|
+
raise ProviderError(f"anthropic: request to {self.model} failed: {exc}") from exc
|
|
49
|
+
text = "".join(
|
|
50
|
+
getattr(block, "text", "") for block in (response.content or []) if getattr(block, "type", "") == "text"
|
|
51
|
+
)
|
|
52
|
+
if not text:
|
|
53
|
+
raise ProviderError(f"anthropic: empty response from {self.model}")
|
|
54
|
+
return text
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""Provider abstraction: one small class per LLM backend, discovered through a registry."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import abc
|
|
6
|
+
import os
|
|
7
|
+
from typing import Callable, ClassVar, Dict, List, Optional, Sequence, Type, TypeVar
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ProviderError(Exception):
|
|
11
|
+
"""Raised when an LLM backend cannot produce an analysis (config, network, API error)."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Provider(abc.ABC):
|
|
15
|
+
"""Base class for LLM backends.
|
|
16
|
+
|
|
17
|
+
Subclasses implement :meth:`complete`. To add a backend, subclass, decorate with
|
|
18
|
+
:func:`register_provider` and import the module in ``providers/__init__.py``::
|
|
19
|
+
|
|
20
|
+
@register_provider("mistral")
|
|
21
|
+
class MistralProvider(Provider):
|
|
22
|
+
default_model = "mistral-small-latest"
|
|
23
|
+
env_key = "MISTRAL_API_KEY"
|
|
24
|
+
|
|
25
|
+
def complete(self, system: str, prompt: str) -> str:
|
|
26
|
+
...
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
name: ClassVar[str] = ""
|
|
30
|
+
default_model: ClassVar[str] = ""
|
|
31
|
+
env_key: ClassVar[Optional[str]] = None
|
|
32
|
+
requires_api_key: ClassVar[bool] = True
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
model: Optional[str] = None,
|
|
37
|
+
api_key: Optional[str] = None,
|
|
38
|
+
base_url: Optional[str] = None,
|
|
39
|
+
timeout: float = 60.0,
|
|
40
|
+
fallback_models: Sequence[str] = (),
|
|
41
|
+
) -> None:
|
|
42
|
+
self.model = model or self.default_model
|
|
43
|
+
self.api_key = api_key or (os.getenv(self.env_key) if self.env_key else None)
|
|
44
|
+
self.base_url = base_url
|
|
45
|
+
self.timeout = timeout
|
|
46
|
+
self.fallback_models: List[str] = [m for m in fallback_models if m and m != self.model]
|
|
47
|
+
|
|
48
|
+
def check_configuration(self) -> None:
|
|
49
|
+
"""Raise ProviderError early if the provider is obviously misconfigured."""
|
|
50
|
+
if self.requires_api_key and not self.api_key:
|
|
51
|
+
hint = f" or set the {self.env_key} environment variable" if self.env_key else ""
|
|
52
|
+
raise ProviderError(f"{self.name}: missing API key. Pass api_key to init(){hint}.")
|
|
53
|
+
|
|
54
|
+
@abc.abstractmethod
|
|
55
|
+
def complete(self, system: str, prompt: str) -> str:
|
|
56
|
+
"""Return the model's text answer or raise :class:`ProviderError`."""
|
|
57
|
+
|
|
58
|
+
def __repr__(self) -> str:
|
|
59
|
+
return f"{type(self).__name__}(model={self.model!r})"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
_REGISTRY: Dict[str, Type[Provider]] = {}
|
|
63
|
+
|
|
64
|
+
P = TypeVar("P", bound=Type[Provider])
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def register_provider(*names: str) -> Callable[[P], P]:
|
|
68
|
+
"""Class decorator registering a provider under one or more names."""
|
|
69
|
+
|
|
70
|
+
def decorator(cls: P) -> P:
|
|
71
|
+
if not names:
|
|
72
|
+
raise ValueError("register_provider() needs at least one name")
|
|
73
|
+
cls.name = cls.name or names[0]
|
|
74
|
+
for n in names:
|
|
75
|
+
_REGISTRY[n.lower()] = cls
|
|
76
|
+
return cls
|
|
77
|
+
|
|
78
|
+
return decorator
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def available_providers() -> List[str]:
|
|
82
|
+
return sorted(_REGISTRY)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def create_provider(name: str, **kwargs) -> Provider:
|
|
86
|
+
key = name.lower().strip()
|
|
87
|
+
if key not in _REGISTRY:
|
|
88
|
+
raise ValueError(
|
|
89
|
+
f"Unknown provider {name!r}. Available: {', '.join(available_providers())}. "
|
|
90
|
+
"You can also pass a Provider instance."
|
|
91
|
+
)
|
|
92
|
+
return _REGISTRY[key](**kwargs)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Google Gemini provider (``pip install shadow-debugger[gemini]``)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import threading
|
|
6
|
+
from typing import Any, Optional
|
|
7
|
+
|
|
8
|
+
from .base import Provider, ProviderError, register_provider
|
|
9
|
+
|
|
10
|
+
# Status codes worth trying the next fallback model for. The SDK already retries
|
|
11
|
+
# transient errors with exponential backoff before raising.
|
|
12
|
+
_FALLBACK_CODES = {404, 429, 500, 502, 503, 504}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@register_provider("gemini", "google")
|
|
16
|
+
class GeminiProvider(Provider):
|
|
17
|
+
default_model = "gemini-3.8-flash"
|
|
18
|
+
env_key = "GEMINI_API_KEY"
|
|
19
|
+
|
|
20
|
+
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
21
|
+
super().__init__(*args, **kwargs)
|
|
22
|
+
self._client: Optional[Any] = None
|
|
23
|
+
self._client_lock = threading.Lock()
|
|
24
|
+
|
|
25
|
+
def _get_client(self) -> Any:
|
|
26
|
+
with self._client_lock:
|
|
27
|
+
if self._client is None:
|
|
28
|
+
try:
|
|
29
|
+
from google import genai
|
|
30
|
+
except ImportError as exc:
|
|
31
|
+
raise ProviderError(
|
|
32
|
+
"gemini: google-genai is not installed. Run: pip install 'shadow-debugger[gemini]'"
|
|
33
|
+
) from exc
|
|
34
|
+
self._client = genai.Client(
|
|
35
|
+
api_key=self.api_key,
|
|
36
|
+
http_options={"timeout": int(self.timeout * 1000)}, # milliseconds
|
|
37
|
+
)
|
|
38
|
+
return self._client
|
|
39
|
+
|
|
40
|
+
def complete(self, system: str, prompt: str) -> str:
|
|
41
|
+
self.check_configuration()
|
|
42
|
+
client = self._get_client()
|
|
43
|
+
last_error: Optional[BaseException] = None
|
|
44
|
+
|
|
45
|
+
for model in [self.model, *self.fallback_models]:
|
|
46
|
+
try:
|
|
47
|
+
interaction = client.interactions.create(
|
|
48
|
+
model=model,
|
|
49
|
+
system_instruction=system,
|
|
50
|
+
input=prompt,
|
|
51
|
+
store=False, # crash data must not be retained server-side
|
|
52
|
+
)
|
|
53
|
+
text = getattr(interaction, "output_text", None)
|
|
54
|
+
if not text:
|
|
55
|
+
raise ProviderError(f"gemini: empty response from {model}")
|
|
56
|
+
return str(text)
|
|
57
|
+
except ProviderError:
|
|
58
|
+
raise
|
|
59
|
+
except Exception as exc:
|
|
60
|
+
last_error = exc
|
|
61
|
+
code = getattr(exc, "code", None) or getattr(exc, "status_code", None)
|
|
62
|
+
if code in _FALLBACK_CODES:
|
|
63
|
+
continue
|
|
64
|
+
raise ProviderError(f"gemini: request to {model} failed: {exc}") from exc
|
|
65
|
+
|
|
66
|
+
raise ProviderError(f"gemini: all models failed; last error: {last_error}") from last_error
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Ollama provider: fully local, no API key, no extra dependency."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import urllib.error
|
|
8
|
+
import urllib.request
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from .base import Provider, ProviderError, register_provider
|
|
12
|
+
|
|
13
|
+
DEFAULT_OLLAMA_HOST = "http://localhost:11434"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def normalize_host(host: str) -> str:
|
|
17
|
+
host = host.strip().rstrip("/")
|
|
18
|
+
if "://" not in host:
|
|
19
|
+
# OLLAMA_HOST is often set as "0.0.0.0:11434" or "myhost:11434"
|
|
20
|
+
host = "http://" + host
|
|
21
|
+
return host.replace("://0.0.0.0", "://localhost")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@register_provider("ollama", "local")
|
|
25
|
+
class OllamaProvider(Provider):
|
|
26
|
+
default_model = "llama3.2"
|
|
27
|
+
requires_api_key = False
|
|
28
|
+
|
|
29
|
+
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
30
|
+
super().__init__(*args, **kwargs)
|
|
31
|
+
self.base_url = normalize_host(self.base_url or os.getenv("OLLAMA_HOST") or DEFAULT_OLLAMA_HOST)
|
|
32
|
+
|
|
33
|
+
def complete(self, system: str, prompt: str) -> str:
|
|
34
|
+
payload = {
|
|
35
|
+
"model": self.model,
|
|
36
|
+
"messages": [
|
|
37
|
+
{"role": "system", "content": system},
|
|
38
|
+
{"role": "user", "content": prompt},
|
|
39
|
+
],
|
|
40
|
+
"stream": False,
|
|
41
|
+
"options": {"temperature": 0.2},
|
|
42
|
+
}
|
|
43
|
+
request = urllib.request.Request(
|
|
44
|
+
f"{self.base_url}/api/chat",
|
|
45
|
+
data=json.dumps(payload).encode("utf-8"),
|
|
46
|
+
headers={"Content-Type": "application/json"},
|
|
47
|
+
method="POST",
|
|
48
|
+
)
|
|
49
|
+
try:
|
|
50
|
+
with urllib.request.urlopen(request, timeout=self.timeout) as resp:
|
|
51
|
+
data = json.loads(resp.read().decode("utf-8"))
|
|
52
|
+
except urllib.error.HTTPError as exc:
|
|
53
|
+
detail = exc.read().decode("utf-8", errors="replace")[:300]
|
|
54
|
+
raise ProviderError(f"ollama: HTTP {exc.code} from {self.base_url}: {detail}") from exc
|
|
55
|
+
except urllib.error.URLError as exc:
|
|
56
|
+
raise ProviderError(
|
|
57
|
+
f"ollama: cannot reach {self.base_url} (is `ollama serve` running?): {exc.reason}"
|
|
58
|
+
) from exc
|
|
59
|
+
except (TimeoutError, OSError, ValueError) as exc:
|
|
60
|
+
raise ProviderError(f"ollama: request failed: {exc}") from exc
|
|
61
|
+
|
|
62
|
+
content = (data.get("message") or {}).get("content")
|
|
63
|
+
if not content:
|
|
64
|
+
raise ProviderError(f"ollama: empty response from model {self.model}")
|
|
65
|
+
return content
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""OpenAI provider (``pip install shadow-debugger[openai]``).
|
|
2
|
+
|
|
3
|
+
Also works with any OpenAI-compatible server (Groq, Together, LM Studio, vLLM...) via ``base_url``.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import threading
|
|
9
|
+
from typing import Any, Optional
|
|
10
|
+
|
|
11
|
+
from .base import Provider, ProviderError, register_provider
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@register_provider("openai")
|
|
15
|
+
class OpenAIProvider(Provider):
|
|
16
|
+
default_model = "gpt-5.4-nano"
|
|
17
|
+
env_key = "OPENAI_API_KEY"
|
|
18
|
+
|
|
19
|
+
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
20
|
+
super().__init__(*args, **kwargs)
|
|
21
|
+
self._client: Optional[Any] = None
|
|
22
|
+
self._client_lock = threading.Lock()
|
|
23
|
+
|
|
24
|
+
def check_configuration(self) -> None:
|
|
25
|
+
# Local OpenAI-compatible servers usually don't need a key.
|
|
26
|
+
if self.base_url:
|
|
27
|
+
return
|
|
28
|
+
super().check_configuration()
|
|
29
|
+
|
|
30
|
+
def _get_client(self) -> Any:
|
|
31
|
+
with self._client_lock:
|
|
32
|
+
if self._client is None:
|
|
33
|
+
try:
|
|
34
|
+
from openai import OpenAI
|
|
35
|
+
except ImportError as exc:
|
|
36
|
+
raise ProviderError(
|
|
37
|
+
"openai: package not installed. Run: pip install 'shadow-debugger[openai]'"
|
|
38
|
+
) from exc
|
|
39
|
+
self._client = OpenAI(
|
|
40
|
+
api_key=self.api_key or "not-needed",
|
|
41
|
+
base_url=self.base_url,
|
|
42
|
+
timeout=self.timeout,
|
|
43
|
+
max_retries=2,
|
|
44
|
+
)
|
|
45
|
+
return self._client
|
|
46
|
+
|
|
47
|
+
def complete(self, system: str, prompt: str) -> str:
|
|
48
|
+
self.check_configuration()
|
|
49
|
+
client = self._get_client()
|
|
50
|
+
try:
|
|
51
|
+
response = client.chat.completions.create(
|
|
52
|
+
model=self.model,
|
|
53
|
+
messages=[
|
|
54
|
+
{"role": "system", "content": system},
|
|
55
|
+
{"role": "user", "content": prompt},
|
|
56
|
+
],
|
|
57
|
+
)
|
|
58
|
+
except Exception as exc:
|
|
59
|
+
raise ProviderError(f"openai: request to {self.model} failed: {exc}") from exc
|
|
60
|
+
content = response.choices[0].message.content if response.choices else None
|
|
61
|
+
if not content:
|
|
62
|
+
raise ProviderError(f"openai: empty response from {self.model}")
|
|
63
|
+
return content
|
shadow_debugger/py.typed
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# PEP 561 marker file
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"""Secret scrubbing and safe serialization for local variables and request metadata."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime as _dt
|
|
6
|
+
import decimal
|
|
7
|
+
import enum
|
|
8
|
+
import fractions
|
|
9
|
+
import itertools
|
|
10
|
+
import pathlib
|
|
11
|
+
import re
|
|
12
|
+
import types
|
|
13
|
+
import uuid
|
|
14
|
+
from typing import Any, Iterable, List, Optional, Pattern
|
|
15
|
+
|
|
16
|
+
REDACTED = "[REDACTED]"
|
|
17
|
+
|
|
18
|
+
_DEFAULT_KEY_PATTERN = (
|
|
19
|
+
r"passw|passwd|pwd|secret|token|api[_-]?key|apikey|access[_-]?key|"
|
|
20
|
+
r"priv(?:ate)?[_-]?key|credential|bearer|cookie|session[_-]?id|"
|
|
21
|
+
r"credit[_-]?card|card[_-]?number|cvv|ssn|signature|"
|
|
22
|
+
# Match 'auth' as a word or 'authorization', so keys like 'author' are left alone
|
|
23
|
+
r"(?:^|[^a-z])auth(?:$|[^a-z]|orization)"
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
_DEFAULT_VALUE_PATTERNS: List[str] = [
|
|
27
|
+
r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----",
|
|
28
|
+
r"\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}",
|
|
29
|
+
r"\bsk-[A-Za-z0-9_-]{16,}",
|
|
30
|
+
r"\bAIza[0-9A-Za-z_-]{30,}",
|
|
31
|
+
r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b",
|
|
32
|
+
r"\bgh[pousr]_[A-Za-z0-9]{30,}",
|
|
33
|
+
r"\bgithub_pat_[A-Za-z0-9_]{30,}",
|
|
34
|
+
r"\bxox[abprs]-[A-Za-z0-9-]{10,}",
|
|
35
|
+
r"\b(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{10,}",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
_PREFIXED_PATTERNS: List[Pattern[str]] = [
|
|
39
|
+
re.compile(r"(?i)\b((?:bearer|basic)\s+)[A-Za-z0-9\-._~+/]{16,}=*"),
|
|
40
|
+
re.compile(
|
|
41
|
+
r"(?i)([?&;](?:[^=&\s#]*?(?:token|key|secret|passw|pwd|auth|signature|sig|code|session)"
|
|
42
|
+
r"[^=&\s#]*)=)[^&\s#'\"]+"
|
|
43
|
+
),
|
|
44
|
+
re.compile(r"(?i)(\b[a-z][a-z0-9+.-]*://[^:/\s@]+:)[^@\s/]+(?=@)"),
|
|
45
|
+
re.compile(
|
|
46
|
+
r"(?i)((?:passw(?:or)?d|passwd|pwd|secret|token|api[_-]?key|apikey|access[_-]?key)"
|
|
47
|
+
r"[\"']?\s*[=:]\s*)([\"'])[^\"'\n]{3,}\2"
|
|
48
|
+
),
|
|
49
|
+
re.compile(
|
|
50
|
+
r"(?i)((?:passw(?:or)?d|passwd|pwd|secret|token|api[_-]?key|apikey|access[_-]?key)"
|
|
51
|
+
r"\s*[=:]\s*)(?![\"'])[^\s'\",;&()]{8,}(?![(\w])"
|
|
52
|
+
),
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
_SAFE_REPR_TYPES = (
|
|
56
|
+
_dt.datetime,
|
|
57
|
+
_dt.date,
|
|
58
|
+
_dt.time,
|
|
59
|
+
_dt.timedelta,
|
|
60
|
+
decimal.Decimal,
|
|
61
|
+
fractions.Fraction,
|
|
62
|
+
uuid.UUID,
|
|
63
|
+
complex,
|
|
64
|
+
pathlib.PurePath,
|
|
65
|
+
enum.Enum,
|
|
66
|
+
range,
|
|
67
|
+
type,
|
|
68
|
+
types.ModuleType,
|
|
69
|
+
types.FunctionType,
|
|
70
|
+
types.BuiltinFunctionType,
|
|
71
|
+
types.MethodType,
|
|
72
|
+
BaseException,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class Redactor:
|
|
77
|
+
"""Turns runtime values into JSON-safe structures while stripping secrets."""
|
|
78
|
+
|
|
79
|
+
def __init__(
|
|
80
|
+
self,
|
|
81
|
+
extra_key_patterns: Iterable[str] = (),
|
|
82
|
+
extra_value_patterns: Iterable[str] = (),
|
|
83
|
+
max_depth: int = 4,
|
|
84
|
+
max_items: int = 30,
|
|
85
|
+
max_string: int = 300,
|
|
86
|
+
) -> None:
|
|
87
|
+
key_sources = [_DEFAULT_KEY_PATTERN, *extra_key_patterns]
|
|
88
|
+
self._key_re = re.compile("|".join(f"(?:{p})" for p in key_sources), re.IGNORECASE)
|
|
89
|
+
self._value_res = [re.compile(p) for p in (*_DEFAULT_VALUE_PATTERNS, *extra_value_patterns)]
|
|
90
|
+
self.max_depth = max_depth
|
|
91
|
+
self.max_items = max_items
|
|
92
|
+
self.max_string = max_string
|
|
93
|
+
|
|
94
|
+
def is_sensitive_key(self, key: str) -> bool:
|
|
95
|
+
return bool(key) and bool(self._key_re.search(key))
|
|
96
|
+
|
|
97
|
+
def redact_text(self, text: Optional[str], limit: Optional[int] = None) -> str:
|
|
98
|
+
if not text:
|
|
99
|
+
return text or ""
|
|
100
|
+
for pattern in _PREFIXED_PATTERNS:
|
|
101
|
+
text = pattern.sub(lambda m: m.group(1) + REDACTED, text)
|
|
102
|
+
for pattern in self._value_res:
|
|
103
|
+
text = pattern.sub(REDACTED, text)
|
|
104
|
+
if limit is not None and len(text) > limit:
|
|
105
|
+
text = text[:limit] + f"... (truncated, {len(text) - limit} more chars)"
|
|
106
|
+
return text
|
|
107
|
+
|
|
108
|
+
def sanitize(self, key: str, value: Any, _depth: int = 0, _seen: Optional[set] = None) -> Any:
|
|
109
|
+
if self.is_sensitive_key(str(key)):
|
|
110
|
+
return REDACTED
|
|
111
|
+
|
|
112
|
+
if value is None or isinstance(value, bool):
|
|
113
|
+
return value
|
|
114
|
+
if isinstance(value, (int, float)):
|
|
115
|
+
return value
|
|
116
|
+
if isinstance(value, str):
|
|
117
|
+
return self.redact_text(value[: self.max_string * 4], limit=self.max_string)
|
|
118
|
+
if isinstance(value, (bytes, bytearray, memoryview)):
|
|
119
|
+
return f"<{type(value).__name__} len={len(value)}>"
|
|
120
|
+
|
|
121
|
+
if _depth >= self.max_depth:
|
|
122
|
+
return f"<{_type_name(value)} ... (max depth)>"
|
|
123
|
+
|
|
124
|
+
seen = _seen if _seen is not None else set()
|
|
125
|
+
if id(value) in seen:
|
|
126
|
+
return f"<{_type_name(value)} (circular reference)>"
|
|
127
|
+
|
|
128
|
+
if isinstance(value, dict):
|
|
129
|
+
seen.add(id(value))
|
|
130
|
+
try:
|
|
131
|
+
result = {}
|
|
132
|
+
for i, (k, v) in enumerate(value.items()):
|
|
133
|
+
if i >= self.max_items:
|
|
134
|
+
result["..."] = f"{len(value) - self.max_items} more keys"
|
|
135
|
+
break
|
|
136
|
+
result[str(k)] = self.sanitize(str(k), v, _depth + 1, seen)
|
|
137
|
+
return result
|
|
138
|
+
finally:
|
|
139
|
+
seen.discard(id(value))
|
|
140
|
+
|
|
141
|
+
if isinstance(value, (list, tuple, set, frozenset)):
|
|
142
|
+
seen.add(id(value))
|
|
143
|
+
try:
|
|
144
|
+
out = [
|
|
145
|
+
self.sanitize("", item, _depth + 1, seen)
|
|
146
|
+
for item in itertools.islice(value, self.max_items)
|
|
147
|
+
]
|
|
148
|
+
if len(value) > self.max_items:
|
|
149
|
+
out.append(f"... ({len(value) - self.max_items} more items)")
|
|
150
|
+
return out
|
|
151
|
+
finally:
|
|
152
|
+
seen.discard(id(value))
|
|
153
|
+
|
|
154
|
+
if isinstance(value, _SAFE_REPR_TYPES):
|
|
155
|
+
return self.redact_text(_safe_repr(value), limit=self.max_string)
|
|
156
|
+
|
|
157
|
+
# For arbitrary objects (like Django QuerySets), read __dict__ directly
|
|
158
|
+
# instead of calling repr()/str(), which could trigger lazy DB queries.
|
|
159
|
+
attrs = _instance_dict(value)
|
|
160
|
+
if attrs is None:
|
|
161
|
+
return f"<{_type_name(value)}>"
|
|
162
|
+
seen.add(id(value))
|
|
163
|
+
try:
|
|
164
|
+
public = {k: v for k, v in attrs.items() if not k.startswith("__")}
|
|
165
|
+
return {"__type__": _type_name(value), **self.sanitize("", public, _depth + 1, seen)}
|
|
166
|
+
finally:
|
|
167
|
+
seen.discard(id(value))
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _type_name(value: Any) -> str:
|
|
171
|
+
cls = type(value)
|
|
172
|
+
module = getattr(cls, "__module__", "")
|
|
173
|
+
if module in ("builtins", "", None):
|
|
174
|
+
return cls.__qualname__
|
|
175
|
+
return f"{module}.{cls.__qualname__}"
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _safe_repr(value: Any) -> str:
|
|
179
|
+
try:
|
|
180
|
+
return repr(value)
|
|
181
|
+
except Exception:
|
|
182
|
+
return f"<{_type_name(value)} (repr failed)>"
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _instance_dict(value: Any) -> Optional[dict]:
|
|
186
|
+
try:
|
|
187
|
+
d = object.__getattribute__(value, "__dict__")
|
|
188
|
+
except Exception:
|
|
189
|
+
return None
|
|
190
|
+
return d if isinstance(d, dict) else None
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""Writes diagnoses to Markdown files, the ``shadow_debugger`` logger and (optionally) stderr."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import sys
|
|
10
|
+
import threading
|
|
11
|
+
from datetime import datetime, timezone
|
|
12
|
+
from typing import Any, Dict, Optional
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger("shadow_debugger")
|
|
15
|
+
|
|
16
|
+
_UNSAFE_FILENAME_CHARS = re.compile(r"[^A-Za-z0-9_.-]+")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class ReportManager:
|
|
20
|
+
def __init__(self, output_dir: Optional[str] = "./shadow_reports", print_to_console: bool = False) -> None:
|
|
21
|
+
self.output_dir = output_dir
|
|
22
|
+
self.print_to_console = print_to_console
|
|
23
|
+
self._dir_lock = threading.Lock()
|
|
24
|
+
self._dir_ready = False
|
|
25
|
+
|
|
26
|
+
def _ensure_dir(self) -> str:
|
|
27
|
+
assert self.output_dir is not None
|
|
28
|
+
# Created lazily: importing/initializing the library must not touch the filesystem.
|
|
29
|
+
with self._dir_lock:
|
|
30
|
+
if not self._dir_ready:
|
|
31
|
+
os.makedirs(self.output_dir, exist_ok=True)
|
|
32
|
+
self._dir_ready = True
|
|
33
|
+
return self.output_dir
|
|
34
|
+
|
|
35
|
+
def _new_report_path(self, error_context: Dict[str, Any], now: datetime) -> str:
|
|
36
|
+
directory = self._ensure_dir()
|
|
37
|
+
exc_type = _UNSAFE_FILENAME_CHARS.sub("_", str(error_context.get("exception_type", "Error")))[:60]
|
|
38
|
+
fingerprint = str(error_context.get("fingerprint", ""))[:8] or "nofp"
|
|
39
|
+
base = f"incident_{now.strftime('%Y%m%d_%H%M%S')}_{exc_type}_{fingerprint}"
|
|
40
|
+
for suffix in range(1000):
|
|
41
|
+
path = os.path.join(directory, f"{base}{'' if suffix == 0 else f'_{suffix}'}.md")
|
|
42
|
+
try:
|
|
43
|
+
# 'x' mode fails if the file exists: no silent overwrite between workers.
|
|
44
|
+
with open(path, "x", encoding="utf-8"):
|
|
45
|
+
return path
|
|
46
|
+
except FileExistsError:
|
|
47
|
+
continue
|
|
48
|
+
raise OSError(f"Could not allocate a unique report filename for {base}")
|
|
49
|
+
|
|
50
|
+
def render(self, error_context: Dict[str, Any], ai_analysis: str, timestamp: datetime) -> str:
|
|
51
|
+
exc_type = error_context.get("exception_type", "Error")
|
|
52
|
+
parts = [
|
|
53
|
+
"# Shadow Debugger Incident Report",
|
|
54
|
+
"",
|
|
55
|
+
f"**Timestamp:** {timestamp.isoformat()} ",
|
|
56
|
+
f"**Exception:** `{exc_type}`: {error_context.get('exception_message')} ",
|
|
57
|
+
f"**Location:** `{error_context.get('file')}:{error_context.get('line')}`"
|
|
58
|
+
f" in `{error_context.get('function') or '?'}` ",
|
|
59
|
+
f"**Fingerprint:** `{error_context.get('fingerprint')}`"
|
|
60
|
+
f" (occurrences so far: {error_context.get('occurrence_count', 1)}) ",
|
|
61
|
+
]
|
|
62
|
+
if error_context.get("model"):
|
|
63
|
+
parts.append(f"**Model:** `{error_context['model']}` ")
|
|
64
|
+
parts += ["", "---", "", ai_analysis.strip(), ""]
|
|
65
|
+
|
|
66
|
+
check = error_context.get("patch_check")
|
|
67
|
+
if check and check.get("attempted"):
|
|
68
|
+
status = "Passed" if check.get("success") else "Failed"
|
|
69
|
+
parts += ["### Proposed Code Syntax Check", f"**Status:** {status}. {check.get('message')}", ""]
|
|
70
|
+
|
|
71
|
+
if error_context.get("extra_metadata"):
|
|
72
|
+
parts += [
|
|
73
|
+
"### Request / Application Metadata",
|
|
74
|
+
"```json",
|
|
75
|
+
json.dumps(error_context["extra_metadata"], indent=2, ensure_ascii=False, default=str),
|
|
76
|
+
"```",
|
|
77
|
+
"",
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
parts += ["---", "### Raw Traceback (redacted)", "```text", str(error_context.get("traceback", "")).rstrip(), "```", ""]
|
|
81
|
+
return "\n".join(parts)
|
|
82
|
+
|
|
83
|
+
def report(self, error_context: Dict[str, Any], ai_analysis: str) -> Optional[str]:
|
|
84
|
+
"""Persist and announce a diagnosis. Returns the report path (or None)."""
|
|
85
|
+
now = datetime.now(timezone.utc)
|
|
86
|
+
exc_type = error_context.get("exception_type", "Error")
|
|
87
|
+
saved_path: Optional[str] = None
|
|
88
|
+
|
|
89
|
+
if self.output_dir:
|
|
90
|
+
saved_path = self._new_report_path(error_context, now)
|
|
91
|
+
with open(saved_path, "w", encoding="utf-8") as fh:
|
|
92
|
+
fh.write(self.render(error_context, ai_analysis, now))
|
|
93
|
+
|
|
94
|
+
logger.info(
|
|
95
|
+
"Incident diagnosed: %s: %s (report: %s)",
|
|
96
|
+
exc_type,
|
|
97
|
+
error_context.get("exception_message"),
|
|
98
|
+
saved_path or "not saved",
|
|
99
|
+
extra={"shadow_incident": error_context, "shadow_analysis": ai_analysis},
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
if self.print_to_console:
|
|
103
|
+
self._print(error_context, ai_analysis, saved_path)
|
|
104
|
+
return saved_path
|
|
105
|
+
|
|
106
|
+
@staticmethod
|
|
107
|
+
def _print(error_context: Dict[str, Any], ai_analysis: str, saved_path: Optional[str]) -> None:
|
|
108
|
+
bar = "=" * 60
|
|
109
|
+
lines = [
|
|
110
|
+
"",
|
|
111
|
+
bar,
|
|
112
|
+
"[Shadow Debugger] Incident diagnosed in background",
|
|
113
|
+
f"Exception: {error_context.get('exception_type')}: {error_context.get('exception_message')}",
|
|
114
|
+
]
|
|
115
|
+
check = error_context.get("patch_check")
|
|
116
|
+
if check and check.get("attempted"):
|
|
117
|
+
lines.append(f"Syntax check: {check.get('message')}")
|
|
118
|
+
if saved_path:
|
|
119
|
+
lines.append(f"Report written to: {saved_path}")
|
|
120
|
+
lines += [bar, ai_analysis, bar, ""]
|
|
121
|
+
text = "\n".join(lines)
|
|
122
|
+
stream = sys.stderr
|
|
123
|
+
try:
|
|
124
|
+
stream.write(text + "\n")
|
|
125
|
+
except UnicodeEncodeError:
|
|
126
|
+
encoding = getattr(stream, "encoding", None) or "ascii"
|
|
127
|
+
stream.write(text.encode(encoding, errors="replace").decode(encoding) + "\n")
|
|
128
|
+
stream.flush()
|