klyrek-core 0.1.0__tar.gz

klyrek_core-0.1.0/.gitignore

@@ -0,0 +1,20 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+.qodo/
+build/
+dist/
+.venv/
+venv/
+.env
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+*.log
+.idea/
+.vscode/
+!.vscode/extensions.json
+klyrek_output/

klyrek_core-0.1.0/PKG-INFO

@@ -0,0 +1,37 @@
+Metadata-Version: 2.4
+Name: klyrek-core
+Version: 0.1.0
+Summary: Shared utilities, configuration, logging, plugins, and common models for the Klyrek ecosystem
+Author: Klyrek Contributors
+License: MIT
+Keywords: appsec,pentesting,reconnaissance,security
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Security
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# klyrek-core
+
+Shared foundation for the Klyrek ecosystem: configuration, logging, the plugin registry, common
+domain models, and the authorization-scope guard that every other Klyrek package builds on.
+
+## Scope enforcement
+
+Klyrek modules are expected to refuse to act against a target that hasn't been explicitly
+declared in scope. `klyrek_core.scope` provides `AuthorizationScope`, which downstream packages
+call before making any network request:
+
+```python
+from klyrek_core.scope import AuthorizationScope
+
+scope = AuthorizationScope(authorized_hosts=["target.com", "*.target.com"])
+scope.check("https://target.com/login")  # OK
+scope.check("https://evil.com")          # raises ScopeViolationError
+```

klyrek_core-0.1.0/README.md

@@ -0,0 +1,18 @@
+# klyrek-core
+
+Shared foundation for the Klyrek ecosystem: configuration, logging, the plugin registry, common
+domain models, and the authorization-scope guard that every other Klyrek package builds on.
+
+## Scope enforcement
+
+Klyrek modules are expected to refuse to act against a target that hasn't been explicitly
+declared in scope. `klyrek_core.scope` provides `AuthorizationScope`, which downstream packages
+call before making any network request:
+
+```python
+from klyrek_core.scope import AuthorizationScope
+
+scope = AuthorizationScope(authorized_hosts=["target.com", "*.target.com"])
+scope.check("https://target.com/login")  # OK
+scope.check("https://evil.com")          # raises ScopeViolationError
+```

klyrek_core-0.1.0/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "klyrek-core"
+version = "0.1.0"
+description = "Shared utilities, configuration, logging, plugins, and common models for the Klyrek ecosystem"
+readme = "README.md"
+license = { text = "MIT" }
+requires-python = ">=3.10"
+authors = [{ name = "Klyrek Contributors" }]
+keywords = ["security", "reconnaissance", "appsec", "pentesting"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Information Technology",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Security",
+]
+dependencies = []
+
+[project.optional-dependencies]
+dev = ["pytest>=8.0", "ruff>=0.6", "mypy>=1.10"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/klyrek_core"]

klyrek_core-0.1.0/src/klyrek_core/__init__.py

@@ -0,0 +1,31 @@
+"""klyrek-core: shared config, logging, plugins, and models for the Klyrek ecosystem."""
+
+from klyrek_core.config import KlyrekConfig
+from klyrek_core.exceptions import ConfigError, KlyrekError, PluginError, ScopeViolationError
+from klyrek_core.logging import get_logger
+from klyrek_core.models import Endpoint, Finding, ScanResult, Severity, Target, Technology
+from klyrek_core.plugins import Plugin, PluginRegistry, register_plugin, registry
+from klyrek_core.scope import AuthorizationScope
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "AuthorizationScope",
+    "ConfigError",
+    "Endpoint",
+    "Finding",
+    "KlyrekConfig",
+    "KlyrekError",
+    "Plugin",
+    "PluginError",
+    "PluginRegistry",
+    "ScanResult",
+    "ScopeViolationError",
+    "Severity",
+    "Target",
+    "Technology",
+    "__version__",
+    "get_logger",
+    "register_plugin",
+    "registry",
+]

klyrek_core-0.1.0/src/klyrek_core/config.py

@@ -0,0 +1,54 @@
+"""Runtime configuration shared across Klyrek packages."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, fields
+from pathlib import Path
+from typing import Any, get_type_hints
+
+from klyrek_core.exceptions import ConfigError
+
+ENV_PREFIX = "KLYREK_"
+
+
+@dataclass(slots=True)
+class KlyrekConfig:
+    """Shared, non-secret defaults consumed by every Klyrek package.
+
+    Individual packages (klyrek-http, klyrek-crawler, ...) may layer their own
+    config on top of this, but request pacing, identification, and output
+    location live here so behavior is consistent across the whole ecosystem.
+    """
+
+    output_dir: Path = Path("klyrek_output")
+    user_agent: str = "Klyrek/0.1 (+https://github.com/klyrek/klyrek)"
+    timeout_seconds: float = 15.0
+    rate_limit_per_host: float = 5.0
+    max_concurrency: int = 10
+    respect_robots_txt: bool = True
+    verify_tls: bool = True
+
+    @classmethod
+    def from_env(cls, prefix: str = ENV_PREFIX) -> KlyrekConfig:
+        """Build a config from ``KLYREK_*`` environment variables, falling back to defaults."""
+        hints = get_type_hints(cls)
+        overrides: dict[str, Any] = {}
+        for f in fields(cls):
+            env_name = f"{prefix}{f.name.upper()}"
+            raw = os.environ.get(env_name)
+            if raw is None:
+                continue
+            overrides[f.name] = _coerce(raw, hints[f.name], env_name)
+        return cls(**overrides)
+
+
+def _coerce(raw: str, type_: type, env_name: str) -> Any:
+    try:
+        if type_ is bool:
+            return raw.strip().lower() in {"1", "true", "yes", "on"}
+        if type_ is Path:
+            return Path(raw)
+        return type_(raw)
+    except ValueError as exc:
+        raise ConfigError(f"Invalid value for {env_name}: {raw!r}") from exc

klyrek_core-0.1.0/src/klyrek_core/exceptions.py

@@ -0,0 +1,26 @@
+"""Exception hierarchy shared across all Klyrek packages."""
+
+from __future__ import annotations
+
+
+class KlyrekError(Exception):
+    """Base class for all Klyrek exceptions."""
+
+
+class ConfigError(KlyrekError):
+    """Raised when configuration is missing or invalid."""
+
+
+class ScopeViolationError(KlyrekError):
+    """Raised when an action targets a host that is not declared in scope."""
+
+    def __init__(self, host: str) -> None:
+        self.host = host
+        super().__init__(
+            f"'{host}' is not an authorized target. Add it to the scan's "
+            "AuthorizationScope before targeting it."
+        )
+
+
+class PluginError(KlyrekError):
+    """Raised for plugin registration or lookup failures."""

klyrek_core-0.1.0/src/klyrek_core/logging.py

@@ -0,0 +1,39 @@
+"""Logging setup shared across Klyrek packages.
+
+Every package should call ``get_logger(__name__)`` rather than configuring its own
+handlers, so a scan that pulls in crawler + api + tech + report modules produces one
+consistent, readable log stream instead of each package fighting over root config.
+"""
+
+from __future__ import annotations
+
+import logging
+
+_CONFIGURED = False
+_BASE_LOGGER_NAME = "klyrek"
+
+
+def configure_logging(level: int | str = logging.INFO) -> None:
+    """Configure the shared 'klyrek' logger tree. Safe to call multiple times."""
+    global _CONFIGURED
+    logger = logging.getLogger(_BASE_LOGGER_NAME)
+    logger.setLevel(level)
+    if not _CONFIGURED:
+        handler = logging.StreamHandler()
+        handler.setFormatter(
+            logging.Formatter(
+                fmt="%(asctime)s %(levelname)-8s %(name)s: %(message)s",
+                datefmt="%Y-%m-%d %H:%M:%S",
+            )
+        )
+        logger.addHandler(handler)
+        logger.propagate = False
+        _CONFIGURED = True
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Return a logger under the shared 'klyrek' namespace, e.g. 'klyrek.crawler.pages'."""
+    if not _CONFIGURED:
+        configure_logging()
+    qualified = name if name.startswith(_BASE_LOGGER_NAME) else f"{_BASE_LOGGER_NAME}.{name}"
+    return logging.getLogger(qualified)

klyrek_core-0.1.0/src/klyrek_core/models.py

@@ -0,0 +1,105 @@
+"""Common domain models shared across Klyrek packages.
+
+These are the shapes every module (crawler, api, tech, auth, js, assets, headers,
+report, ...) reads and writes, so a scan built from multiple packages composes into
+one coherent application map instead of a pile of unrelated outputs.
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _new_id() -> str:
+    return uuid.uuid4().hex
+
+
+class Severity(str, Enum):
+    INFO = "info"
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+    CRITICAL = "critical"
+
+
+@dataclass(slots=True)
+class Target:
+    """The authorized root of a scan."""
+
+    base_url: str
+    id: str = field(default_factory=_new_id)
+    hosts: list[str] = field(default_factory=list)
+    notes: str | None = None
+    created_at: datetime = field(default_factory=_now)
+
+
+@dataclass(slots=True)
+class Technology:
+    """A fingerprinted technology observed on the target (framework, server, CMS, ...)."""
+
+    name: str
+    category: str
+    version: str | None = None
+    confidence: float = 1.0
+    evidence: list[str] = field(default_factory=list)
+    discovered_by: str | None = None
+
+
+@dataclass(slots=True)
+class Endpoint:
+    """A discovered page, form action, or API route."""
+
+    url: str
+    method: str = "GET"
+    id: str = field(default_factory=_new_id)
+    source: str = "crawler"
+    status_code: int | None = None
+    content_type: str | None = None
+    params: list[str] = field(default_factory=list)
+    requires_auth: bool | None = None
+    discovered_by: str | None = None
+    discovered_at: datetime = field(default_factory=_now)
+
+
+@dataclass(slots=True)
+class Finding:
+    """An observation surfaced for human review — not a confirmed vulnerability."""
+
+    title: str
+    severity: Severity
+    id: str = field(default_factory=_new_id)
+    description: str = ""
+    endpoint: Endpoint | None = None
+    evidence: list[str] = field(default_factory=list)
+    module: str | None = None
+    created_at: datetime = field(default_factory=_now)
+
+
+@dataclass(slots=True)
+class ScanResult:
+    """The aggregate output of a scan: everything Klyrek learned about a target."""
+
+    target: Target
+    id: str = field(default_factory=_new_id)
+    endpoints: list[Endpoint] = field(default_factory=list)
+    technologies: list[Technology] = field(default_factory=list)
+    findings: list[Finding] = field(default_factory=list)
+    started_at: datetime = field(default_factory=_now)
+    finished_at: datetime | None = None
+    metadata: dict[str, str] = field(default_factory=dict)
+
+    def add_endpoint(self, endpoint: Endpoint) -> None:
+        self.endpoints.append(endpoint)
+
+    def add_technology(self, technology: Technology) -> None:
+        self.technologies.append(technology)
+
+    def add_finding(self, finding: Finding) -> None:
+        self.findings.append(finding)

klyrek_core-0.1.0/src/klyrek_core/plugins.py

@@ -0,0 +1,77 @@
+"""Plugin architecture shared across Klyrek packages.
+
+Downstream packages (klyrek-tech, klyrek-osint, ...) register capabilities as
+plugins under a category (e.g. "tech-fingerprint", "osint-source") rather than
+being hard-wired into the core, so third parties can extend Klyrek without
+forking it.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from typing import Any, ClassVar
+
+from klyrek_core.exceptions import PluginError
+
+
+class Plugin(ABC):
+    """Base class for anything pluggable into a Klyrek scan."""
+
+    name: ClassVar[str]
+    category: ClassVar[str]
+
+    @abstractmethod
+    def run(self, *args: Any, **kwargs: Any) -> Any:
+        """Execute the plugin's capability."""
+
+
+class PluginRegistry:
+    """Holds registered plugins, keyed by category then name."""
+
+    def __init__(self) -> None:
+        self._plugins: dict[str, dict[str, type[Plugin]]] = defaultdict(dict)
+
+    def register(self, plugin_cls: type[Plugin]) -> type[Plugin]:
+        if not getattr(plugin_cls, "name", None) or not getattr(plugin_cls, "category", None):
+            raise PluginError(
+                f"{plugin_cls.__name__} must define both 'name' and 'category' class attributes"
+            )
+        bucket = self._plugins[plugin_cls.category]
+        if plugin_cls.name in bucket:
+            raise PluginError(
+                f"Plugin '{plugin_cls.name}' is already registered under category "
+                f"'{plugin_cls.category}'"
+            )
+        bucket[plugin_cls.name] = plugin_cls
+        return plugin_cls
+
+    def get(self, category: str, name: str) -> type[Plugin]:
+        try:
+            return self._plugins[category][name]
+        except KeyError as exc:
+            raise PluginError(f"No plugin '{name}' registered under category '{category}'") from exc
+
+    def list(self, category: str | None = None) -> list[type[Plugin]]:
+        if category is not None:
+            return list(self._plugins.get(category, {}).values())
+        return [plugin for bucket in self._plugins.values() for plugin in bucket.values()]
+
+
+#: Process-wide default registry. Packages may instantiate their own
+#: ``PluginRegistry`` for isolated testing instead of using this one.
+registry = PluginRegistry()
+
+
+def register_plugin(plugin_cls: type[Plugin]) -> type[Plugin]:
+    """Class decorator: register a plugin on the default registry.
+
+    Example:
+        @register_plugin
+        class WordPressFingerprint(Plugin):
+            name = "wordpress"
+            category = "tech-fingerprint"
+
+            def run(self, response): ...
+    """
+    return registry.register(plugin_cls)

klyrek_core-0.1.0/src/klyrek_core/scope.py

@@ -0,0 +1,50 @@
+"""Authorization scope enforcement.
+
+Every Klyrek package that makes a network request is expected to call
+``AuthorizationScope.check()`` (or ``is_authorized()``) first. This is the mechanism
+behind Klyrek's "authorized assessments only" principle: a scan can only ever touch
+hosts the operator has explicitly declared, so a typo'd target, a redirect to a
+third-party domain, or a misconfigured crawl can't silently reach out-of-scope
+infrastructure.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from fnmatch import fnmatch
+from urllib.parse import urlsplit
+
+from klyrek_core.exceptions import ScopeViolationError
+
+
+def _hostname(url_or_host: str) -> str:
+    """Extract a bare hostname from a URL or a already-bare host string."""
+    if "//" in url_or_host:
+        host = urlsplit(url_or_host).hostname
+    else:
+        host = urlsplit(f"//{url_or_host}").hostname
+    if not host:
+        raise ValueError(f"Could not determine a hostname from '{url_or_host}'")
+    return host.lower()
+
+
+@dataclass(slots=True)
+class AuthorizationScope:
+    """The set of hosts a scan is authorized to touch.
+
+    Patterns support ``fnmatch``-style wildcards, e.g. ``*.target.com``.
+    """
+
+    authorized_hosts: list[str] = field(default_factory=list)
+
+    def add(self, host_pattern: str) -> None:
+        self.authorized_hosts.append(host_pattern.lower())
+
+    def is_authorized(self, url_or_host: str) -> bool:
+        host = _hostname(url_or_host)
+        return any(fnmatch(host, pattern) for pattern in self.authorized_hosts)
+
+    def check(self, url_or_host: str) -> None:
+        """Raise ScopeViolationError if the given URL/host is not in scope."""
+        if not self.is_authorized(url_or_host):
+            raise ScopeViolationError(_hostname(url_or_host))

klyrek_core-0.1.0/tests/test_config.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+
+from klyrek_core.config import KlyrekConfig
+
+
+def test_defaults():
+    config = KlyrekConfig()
+    assert config.output_dir == Path("klyrek_output")
+    assert config.timeout_seconds == 15.0
+    assert config.verify_tls is True
+
+
+def test_from_env_overrides(monkeypatch):
+    monkeypatch.setenv("KLYREK_TIMEOUT_SECONDS", "30")
+    monkeypatch.setenv("KLYREK_VERIFY_TLS", "false")
+    monkeypatch.setenv("KLYREK_OUTPUT_DIR", "/tmp/scan")
+
+    config = KlyrekConfig.from_env()
+
+    assert config.timeout_seconds == 30.0
+    assert config.verify_tls is False
+    assert config.output_dir == Path("/tmp/scan")
+
+
+def test_from_env_ignores_unset_vars(monkeypatch):
+    monkeypatch.delenv("KLYREK_RATE_LIMIT_PER_HOST", raising=False)
+    config = KlyrekConfig.from_env()
+    assert config.rate_limit_per_host == 5.0

klyrek_core-0.1.0/tests/test_logging.py

@@ -0,0 +1,17 @@
+import logging
+
+from klyrek_core.logging import get_logger
+
+
+def test_get_logger_namespaced():
+    logger = get_logger("crawler.pages")
+    assert logger.name == "klyrek.crawler.pages"
+
+
+def test_get_logger_idempotent_handlers():
+    get_logger("a")
+    base = logging.getLogger("klyrek")
+    count_after_first = len(base.handlers)
+
+    get_logger("b")
+    assert len(base.handlers) == count_after_first

klyrek_core-0.1.0/tests/test_models.py

@@ -0,0 +1,37 @@
+from klyrek_core.models import Endpoint, Finding, ScanResult, Severity, Target, Technology
+
+
+def test_scan_result_aggregates():
+    target = Target(base_url="https://target.com", hosts=["target.com"])
+    result = ScanResult(target=target)
+
+    endpoint = Endpoint(url="https://target.com/api/users", method="GET", source="api")
+    result.add_endpoint(endpoint)
+
+    tech = Technology(name="nginx", category="web-server", version="1.25")
+    result.add_technology(tech)
+
+    finding = Finding(
+        title="Missing security headers",
+        severity=Severity.LOW,
+        endpoint=endpoint,
+        module="klyrek-headers",
+    )
+    result.add_finding(finding)
+
+    assert result.target is target
+    assert result.endpoints == [endpoint]
+    assert result.technologies == [tech]
+    assert result.findings == [finding]
+    assert result.findings[0].endpoint.url == "https://target.com/api/users"
+
+
+def test_ids_are_unique():
+    e1 = Endpoint(url="https://target.com/a")
+    e2 = Endpoint(url="https://target.com/b")
+    assert e1.id != e2.id
+
+
+def test_severity_is_str_enum():
+    assert Severity.CRITICAL == "critical"
+    assert Severity("high") is Severity.HIGH

klyrek_core-0.1.0/tests/test_plugins.py

@@ -0,0 +1,55 @@
+import pytest
+
+from klyrek_core.exceptions import PluginError
+from klyrek_core.plugins import Plugin, PluginRegistry
+
+
+class DummyFingerprint(Plugin):
+    name = "dummy"
+    category = "tech-fingerprint"
+
+    def run(self, response):
+        return "matched"
+
+
+def test_register_and_get():
+    registry = PluginRegistry()
+    registry.register(DummyFingerprint)
+
+    plugin_cls = registry.get("tech-fingerprint", "dummy")
+    assert plugin_cls is DummyFingerprint
+    assert plugin_cls().run(response=None) == "matched"
+
+
+def test_list_by_category():
+    registry = PluginRegistry()
+    registry.register(DummyFingerprint)
+    assert registry.list("tech-fingerprint") == [DummyFingerprint]
+    assert registry.list("other-category") == []
+    assert registry.list() == [DummyFingerprint]
+
+
+def test_get_missing_plugin_raises():
+    registry = PluginRegistry()
+    with pytest.raises(PluginError):
+        registry.get("tech-fingerprint", "nonexistent")
+
+
+def test_duplicate_registration_raises():
+    registry = PluginRegistry()
+    registry.register(DummyFingerprint)
+    with pytest.raises(PluginError):
+        registry.register(DummyFingerprint)
+
+
+def test_missing_name_or_category_raises():
+    class Broken(Plugin):
+        name = "broken"
+        category = ""
+
+        def run(self):
+            return None
+
+    registry = PluginRegistry()
+    with pytest.raises(PluginError):
+        registry.register(Broken)

klyrek_core-0.1.0/tests/test_scope.py

@@ -0,0 +1,42 @@
+import pytest
+
+from klyrek_core.exceptions import ScopeViolationError
+from klyrek_core.scope import AuthorizationScope
+
+
+def test_exact_host_authorized():
+    scope = AuthorizationScope(authorized_hosts=["target.com"])
+    assert scope.is_authorized("https://target.com/login")
+
+
+def test_unauthorized_host_rejected():
+    scope = AuthorizationScope(authorized_hosts=["target.com"])
+    assert not scope.is_authorized("https://evil.com")
+
+
+def test_wildcard_subdomain():
+    scope = AuthorizationScope(authorized_hosts=["*.target.com"])
+    assert scope.is_authorized("https://api.target.com/v1")
+    assert not scope.is_authorized("https://target.com")
+
+
+def test_check_raises_on_violation():
+    scope = AuthorizationScope(authorized_hosts=["target.com"])
+    with pytest.raises(ScopeViolationError):
+        scope.check("https://evil.com/steal")
+
+
+def test_check_passes_silently_when_authorized():
+    scope = AuthorizationScope(authorized_hosts=["target.com"])
+    scope.check("https://target.com")
+
+
+def test_bare_host_input():
+    scope = AuthorizationScope(authorized_hosts=["target.com"])
+    assert scope.is_authorized("target.com")
+
+
+def test_add_pattern():
+    scope = AuthorizationScope()
+    scope.add("Target.com")
+    assert scope.is_authorized("https://target.com")