docsentinel 1.0.0__py3-none-any.whl

docsentinel/__init__.py

@@ -0,0 +1,20 @@
+"""Public interface for the M0 documentation inventory."""
+
+from docsentinel.baseline import filter_baselined, fingerprint, load_baseline, save_baseline
+from docsentinel.config import Config, ConfigError, load_config
+from docsentinel.engine import scan
+from docsentinel.models import Document, Finding, ScanResult
+
+__all__ = [
+    "Config",
+    "ConfigError",
+    "Document",
+    "Finding",
+    "ScanResult",
+    "filter_baselined",
+    "fingerprint",
+    "load_baseline",
+    "load_config",
+    "save_baseline",
+    "scan",
+]

docsentinel/baseline.py

@@ -0,0 +1,56 @@
+"""Finding fingerprints and baseline suppression mechanism."""
+
+import hashlib
+import json
+from pathlib import Path
+
+from docsentinel.config import ConfigError
+from docsentinel.models import Finding
+
+__all__ = ["filter_baselined", "fingerprint", "load_baseline", "save_baseline"]
+
+
+def fingerprint(finding: Finding) -> str:
+    """Compute a stable fingerprint string for a finding."""
+    message_hash = hashlib.sha256(finding.message.encode("utf-8")).hexdigest()[:12]
+    return f"{finding.rule}:{finding.path}:{message_hash}"
+
+
+def load_baseline(path: Path) -> frozenset[str]:
+    """Load suppressed finding fingerprints from a JSON baseline file.
+
+    Returns an empty frozenset if the file does not exist.
+    Raises ConfigError if the file cannot be read, contains invalid JSON,
+    or does not adhere strictly to the `{"fingerprints": [...]}` schema
+    with string list items.
+    """
+    if not path.exists():
+        return frozenset()
+    try:
+        with path.open("r", encoding="utf-8") as stream:
+            data = json.load(stream)
+    except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise ConfigError(f"Cannot read {path}: {exc}") from exc
+
+    if not isinstance(data, dict):
+        raise ConfigError(f"Invalid baseline shape in {path}: expected a JSON object")
+    if set(data.keys()) != {"fingerprints"}:
+        raise ConfigError(f"Invalid baseline in {path}: expected exactly 'fingerprints' key")
+    fingerprints = data["fingerprints"]
+    if not isinstance(fingerprints, list) or not all(isinstance(item, str) for item in fingerprints):
+        raise ConfigError(f"Invalid baseline in {path}: 'fingerprints' must be a list of strings")
+    return frozenset(fingerprints)
+
+
+def save_baseline(path: Path, findings: tuple[Finding, ...]) :
+    """Save finding fingerprints to a JSON baseline file in deterministic sorted order."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    payload = {"fingerprints": sorted({fingerprint(f) for f in findings})}
+    content = json.dumps(payload, indent=2) + "\n"
+    with path.open("w", encoding="utf-8") as stream:
+        stream.write(content)
+
+
+def filter_baselined(findings: tuple[Finding, ...], known: frozenset[str]) -> tuple[Finding, ...]:
+    """Filter out findings whose fingerprints are present in the known baseline set."""
+    return tuple(finding for finding in findings if fingerprint(finding) not in known)

docsentinel/cli.py

@@ -0,0 +1,237 @@
+"""Command-line access to the shared scan engine."""
+
+import argparse
+import sys
+from dataclasses import replace
+from pathlib import Path
+from typing import Optional
+
+from docsentinel import deep_support
+from docsentinel.baseline import save_baseline
+from docsentinel.config import ConfigError, load_config
+from docsentinel.detectors.loader import ensure_standard_loaded
+from docsentinel.engine import scan
+from docsentinel.errors import ExtraNotInstalledError
+from docsentinel.models import ScanResult
+from docsentinel.reporting import neutralize_text, render
+from docsentinel.rules import Rule, rules_for_profile
+
+TEMPLATE = (
+    'profile = "fast"\n'
+    'include = ["*.md", "**/*.md"]\n'
+    'exclude = [".git", ".venv", "node_modules"]\n'
+    "select = []\n"
+    "ignore = []\n"
+    "\n"
+    '# Standard profile thresholds (profile = "standard"); defaults shown.\n'
+    "# max_grade_level = 14.0          # DS201: flag prose above this readability grade\n"
+    "# duplicate_similarity = 0.85     # DS202: flag paragraph pairs at/above this cosine similarity\n"
+    "# min_information_score = 0.15    # DS203: flag paragraphs scoring below this information score\n"
+    '# docsig_paths = ["src"]          # DS204: Python source roots checked against docstrings\n'
+    "#\n"
+    "# Statistical size limits (standard profile); defaults shown.\n"
+    "# min_document_words = 50         # DS201: skip documents with fewer prose words\n"
+    "# min_paragraph_words = 5         # DS202/DS203: skip paragraphs with fewer words\n"
+    "# min_paragraph_chars = 30        # DS202/DS203: skip paragraphs shorter than this\n"
+)
+
+
+def _opt_in_suffix(rule: Rule) -> str:
+    """Return the ``rules`` listing suffix marking an opt-in policy rule, else empty.
+
+    :param rule: A registered rule.
+    :returns: ``" [opt-in: set <option>]"`` for an opt-in policy rule, or ``""``
+        for an ordinary always-active rule.
+    """
+    return f" [opt-in: set {rule.opt_in_option}]" if rule.opt_in_option else ""
+
+
+def _standard_rule_lines() -> list[str]:
+    """Return formatted rule lines for the standard profile, or install guidance if missing."""
+    try:
+        ensure_standard_loaded()
+    except ExtraNotInstalledError:
+        missing_msg = ["standard profile: install docsentinel[standard] to enable (DS2xx)"]
+        return missing_msg
+    registered = rules_for_profile("standard")
+    if registered:
+        return [f"  {rule.code}: {rule.title}{_opt_in_suffix(rule)}" for rule in registered]
+    return ["  (none registered)"]
+
+
+def _deep_doctor_lines() -> list[str]:
+    """Return formatted lines for doctor describing deep profile readiness."""
+    libs = deep_support.libraries_available()
+    model = deep_support.model_available()
+    ready = deep_support.deep_ready()
+
+    lines = []
+    if ready:
+        lines.append("deep profile: ready")
+        lines.append("  ✓ deep libraries installed (spacy, textacy)")
+        lines.append(f"  ✓ deep model installed ({deep_support.MODEL_NAME})")
+    else:
+        lines.append("deep profile: not ready")
+        if libs:
+            lines.append("  ✓ deep libraries installed (spacy, textacy)")
+        else:
+            lines.append(f"  ✗ deep libraries missing: {deep_support.DEEP_EXTRA_HINT}")
+        if model:
+            lines.append(f"  ✓ deep model installed ({deep_support.MODEL_NAME})")
+        else:
+            lines.append(
+                f"  ✗ deep model missing ({deep_support.MODEL_NAME}): python -m spacy download {deep_support.MODEL_NAME}"
+            )
+    return lines
+
+
+def _exit_code(result: ScanResult, *, strict: bool = False) -> int:
+    """Return the exit code for a scan result.
+
+    Deep profile findings are heuristic candidates and advisory-only: they never fail
+    the exit code (returning 0), even when strict is True. This profile-keyed exemption
+    ensures candidate contradictions inform review without breaking builds.
+    A future --fail-on-deep flag / deep_fail setting is deferred.
+
+    For fast and standard profiles, returns 1 if strict mode is enabled and there are any
+    findings, or if any finding has severity == "error". Otherwise, returns 0.
+    """
+    if result.profile == "deep":
+        return 0
+    if strict:
+        return 1 if result.findings else 0
+    return 1 if any(finding.severity == "error" for finding in result.findings) else 0
+
+
+def main(argv: Optional[list[str]] = None) -> int:
+    """Run the DocSentinel command line interface.
+
+    Parameters
+    ----------
+    argv : Optional[list[str]]
+        Command-line arguments; defaults to ``sys.argv[1:]`` when omitted.
+
+    Returns
+    -------
+    int
+        Process exit code: 0 on success, 1 for blocking findings, 2 on failure.
+    """
+    parser = argparse.ArgumentParser(prog="docsentinel")
+    commands = parser.add_subparsers(dest="command", required=True)
+    scan_parser = commands.add_parser("scan", help="Inventory Markdown documents")
+    scan_parser.add_argument("root", nargs="?", default=".", help="Directory or file to check")
+    scan_parser.add_argument("--config", type=Path)
+    scan_parser.add_argument("--profile", choices=("fast", "standard", "deep"))
+    scan_parser.add_argument("--select", type=str, default=None)
+    scan_parser.add_argument("--ignore", type=str, default=None)
+    scan_parser.add_argument("--baseline", type=Path, default=None)
+    scan_parser.add_argument("--format", choices=("text", "json", "github"), default="text")
+    scan_parser.add_argument("--strict", action="store_true")
+    scan_parser.add_argument(
+        "--i-understand-deep-is-heuristic",
+        action="store_true",
+        dest="deep_confirm",
+        help="Explicitly confirm execution of heuristic deep profile detectors",
+    )
+    baseline_parser = commands.add_parser("baseline", help="Record current findings into a baseline file")
+    baseline_parser.add_argument("root", nargs="?", default=".", help="Directory or file to check")
+    baseline_parser.add_argument("--config", type=Path)
+    baseline_parser.add_argument("--output", type=Path)
+    baseline_parser.add_argument("--profile", choices=("fast", "standard", "deep"))
+    init_parser = commands.add_parser("init", help="Create a starter config")
+    init_parser.add_argument("root", nargs="?", default=".")
+    commands.add_parser("rules", help="List active detection rules")
+    commands.add_parser("doctor", help="Show M0 capabilities")
+    args = parser.parse_args(argv)
+    if args.command == "init":
+        target = Path(args.root) / "doc_sentinel.toml"
+        try:
+            with target.open("x", encoding="utf-8") as stream:
+                stream.write(TEMPLATE)
+        except OSError as exc:
+            print(f"Cannot create {target}: {exc}", file=sys.stderr)
+            return 2
+        print(f"Created {target}")
+        return 0
+    if args.command == "rules":
+        for profile in ("fast", "standard", "deep"):
+            print(f"{profile}:")
+            if profile == "standard":
+                for line in _standard_rule_lines():
+                    print(line)
+            else:
+                registered = rules_for_profile(profile)
+                if registered:
+                    for rule in registered:
+                        print(f"  {rule.code}: {rule.title}{_opt_in_suffix(rule)}")
+                else:
+                    print("  (none registered)")
+        return 0
+    if args.command == "doctor":
+        print("Inventory: available")
+        standard_lines = _standard_rule_lines()
+        if standard_lines == ["standard profile: install docsentinel[standard] to enable (DS2xx)"]:
+            print("standard profile: not installed (pip install docsentinel[standard])")
+        else:
+            print("standard profile: available")
+        for line in _deep_doctor_lines():
+            print(line)
+        for profile in ("fast", "standard", "deep"):
+            registered = rules_for_profile(profile)
+            opt_in = tuple(rule for rule in registered if rule.opt_in_option)
+            summary = f"{profile} rules: {len(registered)} registered"
+            if opt_in:
+                codes = ", ".join(rule.code for rule in opt_in)
+                summary += f" ({len(opt_in)} opt-in, off unless configured: {codes})"
+            print(summary)
+        return 0
+    if args.command == "baseline":
+        try:
+            target = Path(args.root).resolve()
+            directory = target.parent if target.is_file() else target
+            settings = load_config(directory, args.config)
+            if args.profile is not None:
+                settings = replace(settings, profile=args.profile)
+            if args.output is not None:
+                output_path = Path(args.output) if Path(args.output).is_absolute() else Path.cwd() / args.output
+            elif settings.baseline is not None:
+                output_path = (
+                    Path(settings.baseline) if Path(settings.baseline).is_absolute() else directory / settings.baseline
+                )
+            else:
+                output_path = directory / ".docsentinel-baseline.json"
+
+            unbaselined_config = replace(settings, baseline=None)
+            result = scan(args.root, config=unbaselined_config)
+            save_baseline(output_path, result.findings)
+        except (ConfigError, ValueError, OSError) as exc:
+            print(f"Baseline generation failed: {neutralize_text(str(exc))}", file=sys.stderr)
+            return 2
+        print(f"Saved baseline: {output_path}")
+        return 0
+    try:
+        target = Path(args.root).resolve()
+        settings = load_config(target.parent if target.is_file() else target, args.config)
+        changes = {}
+        if args.profile is not None:
+            changes["profile"] = args.profile
+        if args.select is not None:
+            changes["select"] = tuple(item.strip() for item in args.select.split(",") if item.strip())
+        if args.ignore is not None:
+            changes["ignore"] = tuple(item.strip() for item in args.ignore.split(",") if item.strip())
+        if args.baseline is not None:
+            changes["baseline"] = str(args.baseline)
+        if getattr(args, "deep_confirm", False):
+            changes["deep_confirm"] = True
+        if changes:
+            settings = replace(settings, **changes)
+        result = scan(args.root, config=settings)
+    except (ConfigError, ValueError, OSError) as exc:
+        print(f"Scan failed: {neutralize_text(str(exc))}", file=sys.stderr)
+        return 2
+    print(render(result, args.format))
+    return _exit_code(result, strict=args.strict)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docsentinel/code_discovery.py

@@ -0,0 +1,84 @@
+"""Discover Python source files for code-vs-doc signature checks."""
+
+import os
+from fnmatch import fnmatchcase
+from pathlib import Path
+
+from docsentinel.config import Config
+
+
+def discover_python(root: Path, config: Config) -> tuple[Path, ...]:
+    """Return a reproducible inventory of matching Python source files."""
+    if not root.is_dir():
+        raise ValueError(f"Not a directory: {root}")
+    root = root.resolve()
+
+    def raise_walk_error(error: OSError):
+        """Re-raise directory walk errors encountered during os.walk."""
+        raise error
+
+    discovered: list[Path] = []
+
+    for entry in config.docsig_paths:
+        configured_target = root / entry
+        if configured_target.is_symlink():
+            continue
+        target = configured_target.resolve()
+        if not target.is_relative_to(root):
+            raise ValueError(f"docsig path must remain under scan root: {entry}")
+        if not target.exists():
+            continue
+
+        if target.is_file():
+            if not target.name.endswith(".py"):
+                continue
+            try:
+                relative = target.relative_to(root).as_posix()
+                rel_parts = target.relative_to(root).parts[:-1]
+            except ValueError:
+                relative = target.as_posix()
+                rel_parts = target.parts[:-1]
+            if any(fnmatchcase(relative, pattern) for pattern in config.exclude):
+                continue
+            if any(fnmatchcase(part, pattern) for part in rel_parts for pattern in config.exclude):
+                continue
+            discovered.append(target)
+        elif target.is_dir():
+            try:
+                target_rel = target.relative_to(root).as_posix()
+                target_parts = target.relative_to(root).parts
+            except ValueError:
+                target_rel = target.as_posix()
+                target_parts = target.parts
+            if any(fnmatchcase(target_rel, pattern) for pattern in config.exclude):
+                continue
+            if any(fnmatchcase(part, pattern) for part in target_parts for pattern in config.exclude):
+                continue
+
+            for current, directories, filenames in os.walk(target, topdown=True, onerror=raise_walk_error):
+                current_path = Path(current)
+                directories[:] = [
+                    name
+                    for name in directories
+                    if not (current_path / name).is_symlink()
+                    and not any(fnmatchcase(name, pattern) for pattern in config.exclude)
+                ]
+                for filename in filenames:
+                    path = current_path / filename
+                    if not path.is_file() or path.is_symlink():
+                        continue
+                    if not filename.endswith(".py"):
+                        continue
+                    try:
+                        relative = path.relative_to(root).as_posix()
+                        rel_parts = path.relative_to(root).parts[:-1]
+                    except ValueError:
+                        relative = path.as_posix()
+                        rel_parts = path.parts[:-1]
+                    if any(fnmatchcase(relative, pattern) for pattern in config.exclude):
+                        continue
+                    if any(fnmatchcase(part, pattern) for part in rel_parts for pattern in config.exclude):
+                        continue
+                    discovered.append(path)
+
+    return tuple(sorted(set(discovered)))