truesignal-cli 0.1.0__py3-none-any.whl

truesignal/__init__.py

@@ -0,0 +1,44 @@
+"""
+Library entry point -- import truesignal's connectors and provenance layer programmatically.
+
+    from truesignal import ALL_CONNECTORS, get_connector
+
+    for connector in ALL_CONNECTORS:
+        for item in connector.fetch_items():
+            print(item.status, item.url)
+
+This is the Python port of the truesignal-cli npm package
+(https://www.npmjs.com/package/truesignal-cli). Both distributions ship the same five source
+connectors (CISA-KEV, Cloudflare Radar, Reddit, Telegram, GDELT) and the same no-fabrication
+provenance guarantee; see https://github.com/RudrenduPaul/truesignal for the canonical
+documentation and the original TypeScript source.
+"""
+from .connectors import ALL_CONNECTORS, get_connector
+from .provenance import (
+    fetch_with_fallback,
+    get_cache_dir,
+    read_cache,
+    stamp_fallback,
+    stamp_live,
+    write_cache,
+)
+from .types import Connector, ConnectorNotConfiguredError, FeedItem, ItemStatus, UnstampedItem
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Connector",
+    "FeedItem",
+    "ItemStatus",
+    "UnstampedItem",
+    "ConnectorNotConfiguredError",
+    "ALL_CONNECTORS",
+    "get_connector",
+    "fetch_with_fallback",
+    "stamp_live",
+    "stamp_fallback",
+    "read_cache",
+    "write_cache",
+    "get_cache_dir",
+    "__version__",
+]

truesignal/_util.py

@@ -0,0 +1,28 @@
+"""Small internal helpers shared across connectors and the provenance layer. Not part of the
+public API (hence the leading underscore) -- see truesignal/__init__.py for the public surface."""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+
+def format_iso8601(dt: datetime) -> str:
+    """
+    Formats a datetime as a JavaScript-``Date.prototype.toISOString()``-equivalent string:
+    always UTC, always exactly 3 millisecond digits, always a trailing "Z". Naive datetimes are
+    assumed to already be UTC (every caller in this codebase constructs them that way).
+    """
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    dt = dt.astimezone(timezone.utc)
+    milliseconds = dt.microsecond // 1000
+    return dt.strftime("%Y-%m-%dT%H:%M:%S") + f".{milliseconds:03d}Z"
+
+
+def parse_iso8601(value: str) -> datetime:
+    """Parses a real upstream ISO-8601 timestamp (with or without a trailing 'Z') into an aware
+    UTC datetime. Raises ValueError on anything that isn't a real, parseable instant."""
+    normalized = value[:-1] + "+00:00" if value.endswith("Z") else value
+    dt = datetime.fromisoformat(normalized)
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt.astimezone(timezone.utc)

truesignal/cli.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""
+truesignal CLI entry point. This module is deliberately thin -- it wires argparse's argument
+parsing to the testable logic in cli_helpers.py and returns process exit codes. See ExitCode in
+cli_helpers.py for what each code means.
+
+Ported from src/truesignal/cli.ts (which uses `commander`); this port uses the stdlib `argparse`
+to avoid a CLI-framework dependency. Console entry point: `truesignal <command> [options]`,
+installed via the `truesignal` console-script defined in python/pyproject.toml.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import List, Optional
+
+from .cli_helpers import (
+    ExitCode,
+    format_feed_item_human,
+    format_init_report,
+    get_connector_statuses,
+    run_feed,
+    run_verify,
+)
+from .connectors import ALL_CONNECTORS
+
+_VERSION = "0.1.0"
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="truesignal",
+        description=(
+            "A provenance-first OSINT/security intelligence feed. Every item carries a real "
+            "source URL, a real timestamp, and an explicit live/fallback flag -- never a "
+            "fabricated or silently-replayed data point."
+        ),
+    )
+    parser.add_argument("--version", "-V", action="version", version=f"truesignal-cli {_VERSION}")
+
+    subparsers = parser.add_subparsers(dest="command")
+
+    init_parser = subparsers.add_parser(
+        "init",
+        help=(
+            "Check which connectors are ready to use right now and which environment variables "
+            "are still needed for the rest. CISA-KEV and GDELT need no configuration -- "
+            "truesignal works with zero setup for those two sources."
+        ),
+    )
+    init_parser.add_argument(
+        "--json", action="store_true", help="print machine-readable JSON instead of a human-readable report"
+    )
+
+    feed_parser = subparsers.add_parser(
+        "feed",
+        help=(
+            "Pull the current feed from every configured connector, or one connector with "
+            "--source. Prints human-readable output by default; use --json for a stable, "
+            "agent-parseable schema."
+        ),
+    )
+    feed_parser.add_argument("--source", default=None, help="only pull from this connector, e.g. cisa-kev, gdelt")
+    feed_parser.add_argument("--json", action="store_true", help="print machine-readable JSON instead of human-readable text")
+
+    verify_parser = subparsers.add_parser(
+        "verify",
+        help=(
+            "Re-fetch the source connector named in <item-id> and confirm whether that item "
+            "still resolves to real, live provenance, has fallen back to cached data, or can no "
+            "longer be found."
+        ),
+    )
+    verify_parser.add_argument("item_id", help="a feed item id from `truesignal feed`, e.g. cisa-kev:CVE-2026-12345")
+    verify_parser.add_argument("--json", action="store_true", help="print machine-readable JSON instead of human-readable text")
+
+    return parser
+
+
+def run_cli(argv: List[str]) -> int:
+    """
+    `argv` follows the sys.argv convention: argv[0] is the program name, the real arguments start
+    at argv[1]. Returns the process exit code.
+    """
+    parser = build_parser()
+    args = parser.parse_args(argv[1:])
+
+    if args.command == "init":
+        return _run_init(json_output=args.json)
+    if args.command == "feed":
+        return _run_feed(source=args.source, json_output=args.json)
+    if args.command == "verify":
+        return _run_verify(item_id=args.item_id, json_output=args.json)
+
+    parser.print_help()
+    return ExitCode.SUCCESS
+
+
+def _run_init(json_output: bool) -> int:
+    statuses = get_connector_statuses(ALL_CONNECTORS)
+    if json_output:
+        print(
+            json.dumps(
+                {
+                    "connectors": [
+                        {
+                            "name": s.name,
+                            "label": s.label,
+                            "requires_config": s.requires_config,
+                            "configured": s.configured,
+                            "missing_env_vars": s.missing_env_vars,
+                        }
+                        for s in statuses
+                    ]
+                },
+                indent=2,
+            )
+        )
+    else:
+        print(format_init_report(statuses))
+    ready_count = sum(1 for s in statuses if not s.requires_config or s.configured)
+    return ExitCode.SUCCESS if ready_count > 0 else ExitCode.NO_CONNECTORS_CONFIGURED
+
+
+def _run_feed(source: Optional[str], json_output: bool) -> int:
+    result = run_feed(ALL_CONNECTORS, source)
+
+    if result.unknown_source:
+        known = ", ".join(c.name for c in ALL_CONNECTORS)
+        sys.stderr.write(f'Unknown source "{result.unknown_source}". Known sources: {known}\n')
+        return ExitCode.GENERAL_ERROR
+
+    if not result.items and result.skipped and not result.failures:
+        if json_output:
+            print(json.dumps(result.to_dict(), indent=2))
+        else:
+            sys.stderr.write(
+                f"No connectors are configured to run. Missing: {', '.join(result.skipped)}. "
+                f'Run "truesignal init" to see what\'s needed.\n'
+            )
+        return ExitCode.NO_CONNECTORS_CONFIGURED
+
+    if json_output:
+        print(json.dumps(result.to_dict(), indent=2))
+    else:
+        if result.skipped:
+            print(f"Skipped (not configured): {', '.join(result.skipped)}")
+        if not result.items:
+            print("No items returned.")
+        for item in result.items:
+            print(format_feed_item_human(item))
+        for failure in result.failures:
+            sys.stderr.write(f"{failure.source}: {failure.error}\n")
+
+    if not result.items and result.failures:
+        return ExitCode.NETWORK_ERROR
+    return ExitCode.SUCCESS
+
+
+def _run_verify(item_id: str, json_output: bool) -> int:
+    result = run_verify(ALL_CONNECTORS, item_id)
+
+    if json_output:
+        print(json.dumps(result.to_dict(), indent=2))
+    elif result.error_message:
+        sys.stderr.write(f"{result.error_message}\n")
+    elif result.found:
+        if result.status == "live":
+            provenance = "LIVE"
+        else:
+            provenance = f"FALLBACK ({result.fallback_age_seconds or 0}s old cached data)"
+        print(f"{item_id}: {provenance} -- {result.url} -- {result.timestamp}")
+    else:
+        print(f"{item_id}: not found in the current feed (it may have rolled off, or never existed).")
+
+    if result.error_kind in ("invalid-id", "unknown-source"):
+        return ExitCode.INVALID_ITEM_ID
+    if result.error_kind == "not-configured":
+        return ExitCode.NO_CONNECTORS_CONFIGURED
+    if result.error_kind == "network-error":
+        return ExitCode.NETWORK_ERROR
+    if not result.found:
+        return ExitCode.GENERAL_ERROR
+    return ExitCode.SUCCESS
+
+
+def main() -> None:
+    try:
+        code = run_cli(sys.argv)
+    except SystemExit:
+        raise
+    except Exception as error:  # noqa: BLE001 -- top-level crash guard, mirrors src/cli.ts's catch-all
+        sys.stderr.write(f"{error}\n")
+        sys.exit(ExitCode.GENERAL_ERROR)
+    else:
+        sys.exit(code)
+
+
+if __name__ == "__main__":
+    main()

truesignal/cli_helpers.py

@@ -0,0 +1,314 @@
+"""
+Testable logic behind the CLI. cli.py wires this up to argparse and sys.exit(); every function
+here is pure enough to unit test without spawning a real process.
+
+Direct port of src/truesignal/cli-helpers.ts. Field/function names are snake_case here (Python
+convention) where the TypeScript source uses camelCase.
+"""
+from __future__ import annotations
+
+import concurrent.futures
+import re
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import IntEnum
+from typing import Dict, List, Optional, Sequence, Tuple
+
+from ._util import parse_iso8601
+from .types import Connector, FeedItem, ItemStatus
+
+
+class ExitCode(IntEnum):
+    """Exit codes this CLI uses. Documented here and in the per-subcommand --help text."""
+
+    #: Command completed successfully.
+    SUCCESS = 0
+    #: Unexpected/uncategorized error.
+    GENERAL_ERROR = 1
+    #: The requested connector(s) exist but none are configured -- nothing could run.
+    NO_CONNECTORS_CONFIGURED = 2
+    #: A configured connector's upstream fetch failed and produced no fallback data either.
+    NETWORK_ERROR = 3
+    #: `verify <item-id>` was given an id that doesn't parse or names an unknown source.
+    INVALID_ITEM_ID = 4
+
+
+@dataclass(frozen=True)
+class ConnectorStatus:
+    name: str
+    label: str
+    requires_config: bool
+    configured: bool
+    missing_env_vars: List[str] = field(default_factory=list)
+
+
+def get_connector_statuses(connectors: Sequence[Connector]) -> List[ConnectorStatus]:
+    """Computes the configuration status of every connector, for `truesignal init`."""
+    statuses = []
+    for connector in connectors:
+        configured = connector.is_configured()
+        missing = list(connector.config_env_vars) if connector.requires_config and not configured else []
+        statuses.append(
+            ConnectorStatus(
+                name=connector.name,
+                label=connector.label,
+                requires_config=connector.requires_config,
+                configured=configured,
+                missing_env_vars=missing,
+            )
+        )
+    return statuses
+
+
+def format_init_report(statuses: Sequence[ConnectorStatus]) -> str:
+    """Human-readable report for `truesignal init`."""
+    lines = ["truesignal connector status:", ""]
+    for status in statuses:
+        if not status.requires_config:
+            lines.append(f"  [ready]        {status.label} ({status.name}) -- no configuration needed")
+        elif status.configured:
+            lines.append(f"  [ready]        {status.label} ({status.name}) -- configured")
+        else:
+            lines.append(
+                f"  [not configured] {status.label} ({status.name}) -- set {', '.join(status.missing_env_vars)}"
+            )
+    ready_count = sum(1 for s in statuses if not s.requires_config or s.configured)
+    lines.append("")
+    lines.append(f"{ready_count}/{len(statuses)} connectors ready.")
+    if ready_count == 0:
+        lines.append(
+            "No connectors are usable. This should not happen -- CISA-KEV and GDELT need no configuration."
+        )
+    else:
+        if ready_count < len(statuses):
+            lines.append("Set the missing environment variables above to enable the rest. See .env.example.")
+        lines.append('Next: run "truesignal feed" to see your feed now.')
+    return "\n".join(lines)
+
+
+@dataclass(frozen=True)
+class ConnectorSelection:
+    #: Connectors to actually query.
+    to_query: List[Connector]
+    #: Connectors that exist but are not configured, skipped rather than failed on.
+    skipped: List[Connector]
+    #: Set if --source named a connector that does not exist.
+    unknown_source: Optional[str] = None
+
+
+def select_connectors(
+    all_connectors: Sequence[Connector],
+    source_filter: Optional[str] = None,
+) -> ConnectorSelection:
+    """
+    Resolves which connectors `truesignal feed` should query, given an optional --source filter.
+    Unconfigured connectors are skipped (reported, never crashed on); an unknown --source name is
+    reported back as an error rather than silently ignored.
+    """
+    candidates = list(all_connectors)
+    if source_filter:
+        match = next((c for c in all_connectors if c.name == source_filter), None)
+        if not match:
+            return ConnectorSelection(to_query=[], skipped=[], unknown_source=source_filter)
+        candidates = [match]
+    to_query = [c for c in candidates if not c.requires_config or c.is_configured()]
+    skipped = [c for c in candidates if c.requires_config and not c.is_configured()]
+    return ConnectorSelection(to_query=to_query, skipped=skipped)
+
+
+def _relative_age(timestamp: str, now: datetime) -> str:
+    ms = (now - parse_iso8601(timestamp)).total_seconds() * 1000
+    seconds = max(0, round(ms / 1000))
+    if seconds < 60:
+        return f"{seconds}s ago"
+    minutes = round(seconds / 60)
+    if minutes < 60:
+        return f"{minutes}m ago"
+    hours = round(minutes / 60)
+    if hours < 24:
+        return f"{hours}h ago"
+    days = round(hours / 24)
+    return f"{days}d ago"
+
+
+def _format_duration(seconds: int) -> str:
+    if seconds < 60:
+        return f"{seconds}s"
+    minutes = round(seconds / 60)
+    if minutes < 60:
+        return f"{minutes}m"
+    hours = round(minutes / 60)
+    if hours < 24:
+        return f"{hours}h"
+    days = round(hours / 24)
+    return f"{days}d"
+
+
+_CONTROL_CHAR_PATTERN = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]")
+
+
+def _sanitize_for_terminal(text: str) -> str:
+    """
+    Strips control characters (C0/C1, DEL) from untrusted upstream text before it reaches a
+    terminal. Reddit and Telegram content is attacker-controlled -- a crafted post title or
+    message could otherwise carry ANSI/OSC escape sequences (cursor manipulation, terminal-title
+    spoofing, an OSC-8 hyperlink escape that visually disguises the printed URL) straight into
+    the user's terminal via `print`.
+    """
+    return _CONTROL_CHAR_PATTERN.sub("", text)
+
+
+def format_feed_item_human(item: FeedItem, now: Optional[datetime] = None) -> str:
+    """Formats one feed item as a single human-readable line."""
+    now = now or datetime.now(timezone.utc)
+    if item.status == "live":
+        tag = "[live]"
+    else:
+        tag = f"[fallback, {_format_duration(item.fallback_age_seconds or 0)} old]"
+    title = _sanitize_for_terminal(item.title)
+    url = _sanitize_for_terminal(item.url)
+    return f"{tag} {item.source}: {title} -- {url} -- {_relative_age(item.timestamp, now)}"
+
+
+def parse_item_id(item_id: str) -> Optional[Tuple[str, str]]:
+    """Splits a feed item id (`{source}:{native_id}`) into its source and native-id parts."""
+    separator_index = item_id.find(":")
+    if separator_index <= 0 or separator_index == len(item_id) - 1:
+        return None
+    return item_id[:separator_index], item_id[separator_index + 1 :]
+
+
+@dataclass(frozen=True)
+class ConnectorFailure:
+    source: str
+    error: str
+
+
+@dataclass(frozen=True)
+class FeedRunResult:
+    items: List[FeedItem]
+    skipped: List[str]
+    failures: List[ConnectorFailure]
+    unknown_source: Optional[str] = None
+
+    def to_dict(self) -> Dict:
+        data: Dict = {
+            "items": [item.to_dict() for item in self.items],
+            "skipped": self.skipped,
+            "failures": [{"source": f.source, "error": f.error} for f in self.failures],
+        }
+        if self.unknown_source is not None:
+            data["unknown_source"] = self.unknown_source
+        return data
+
+
+def run_feed(connectors: Sequence[Connector], source_filter: Optional[str] = None) -> FeedRunResult:
+    """
+    Runs `truesignal feed` end to end against real connectors: resolves which to query, fetches
+    them concurrently, and never lets one connector's failure take down the others.
+    """
+    selection = select_connectors(connectors, source_filter)
+    if selection.unknown_source:
+        return FeedRunResult(items=[], skipped=[], failures=[], unknown_source=selection.unknown_source)
+
+    items: List[FeedItem] = []
+    failures: List[ConnectorFailure] = []
+
+    if selection.to_query:
+        # Preserve connector order in the aggregated output, same as Promise.allSettled's
+        # index-ordered results in the TypeScript source.
+        with concurrent.futures.ThreadPoolExecutor(max_workers=len(selection.to_query)) as executor:
+            futures = [executor.submit(connector.fetch_items) for connector in selection.to_query]
+            for connector, future in zip(selection.to_query, futures):
+                try:
+                    items.extend(future.result())
+                except Exception as error:  # noqa: BLE001 -- one connector's failure must not sink the others
+                    failures.append(ConnectorFailure(source=connector.name, error=str(error)))
+
+    return FeedRunResult(items=items, skipped=[c.name for c in selection.skipped], failures=failures)
+
+
+VerifyErrorKind = str  # "invalid-id" | "unknown-source" | "not-configured" | "network-error"
+
+
+@dataclass(frozen=True)
+class VerifyResult:
+    item_id: str
+    found: bool
+    status: Optional[ItemStatus] = None
+    url: Optional[str] = None
+    timestamp: Optional[str] = None
+    fallback_age_seconds: Optional[int] = None
+    error_kind: Optional[VerifyErrorKind] = None
+    error_message: Optional[str] = None
+
+    def to_dict(self) -> Dict:
+        data: Dict = {"item_id": self.item_id, "found": self.found}
+        for key, value in (
+            ("status", self.status),
+            ("url", self.url),
+            ("timestamp", self.timestamp),
+            ("fallback_age_seconds", self.fallback_age_seconds),
+            ("error_kind", self.error_kind),
+            ("error_message", self.error_message),
+        ):
+            if value is not None:
+                data[key] = value
+        return data
+
+
+def run_verify(connectors: Sequence[Connector], item_id: str) -> VerifyResult:
+    """Re-fetches the named connector and confirms whether `item_id` still resolves to real data."""
+    parsed = parse_item_id(item_id)
+    if not parsed:
+        return VerifyResult(
+            item_id=item_id,
+            found=False,
+            error_kind="invalid-id",
+            error_message=(
+                f'Could not parse item id "{item_id}". Expected format: <source>:<native-id>, '
+                f"e.g. cisa-kev:CVE-2026-12345"
+            ),
+        )
+    source, _native_id = parsed
+
+    connector = next((c for c in connectors if c.name == source), None)
+    if not connector:
+        known = ", ".join(c.name for c in connectors)
+        return VerifyResult(
+            item_id=item_id,
+            found=False,
+            error_kind="unknown-source",
+            error_message=f'Unknown source "{source}" in item id "{item_id}". Known sources: {known}',
+        )
+
+    if connector.requires_config and not connector.is_configured():
+        return VerifyResult(
+            item_id=item_id,
+            found=False,
+            error_kind="not-configured",
+            error_message=f'Connector "{connector.name}" is not configured. Run "truesignal init" to see what is needed.',
+        )
+
+    try:
+        items = connector.fetch_items()
+    except Exception as error:  # noqa: BLE001 -- mirrors src/cli-helpers.ts's catch-all around fetchItems()
+        return VerifyResult(
+            item_id=item_id,
+            found=False,
+            error_kind="network-error",
+            error_message=f"Failed to verify: {error}",
+        )
+
+    found_item = next((item for item in items if item.id == item_id), None)
+    if not found_item:
+        return VerifyResult(item_id=item_id, found=False)
+
+    return VerifyResult(
+        item_id=item_id,
+        found=True,
+        status=found_item.status,
+        url=found_item.url,
+        timestamp=found_item.timestamp,
+        fallback_age_seconds=found_item.fallback_age_seconds,
+    )

truesignal/connectors/__init__.py

@@ -0,0 +1,48 @@
+"""
+Every connector this build ships, in a fixed order. Adding a new source is: implement Connector
+(truesignal/types.py) in a new module in this package, then add it to ALL_CONNECTORS below --
+nothing else in the CLI or provenance layer needs to change. See CONTRIBUTING.md.
+
+Direct port of src/truesignal/connectors/index.ts.
+"""
+from __future__ import annotations
+
+from typing import List, Optional, Sequence
+
+from ..types import Connector
+from .cisa_kev import CisaKevConnector, cisa_kev_connector
+from .cloudflare_radar import CloudflareRadarConnector, cloudflare_radar_connector
+from .gdelt import GdeltConnector, gdelt_connector
+from .reddit import RedditConnector, reddit_connector
+from .telegram import TelegramConnector, telegram_connector
+
+ALL_CONNECTORS: Sequence[Connector] = (
+    cisa_kev_connector,
+    cloudflare_radar_connector,
+    reddit_connector,
+    telegram_connector,
+    gdelt_connector,
+)
+
+
+def get_connector(name: str) -> Optional[Connector]:
+    for connector in ALL_CONNECTORS:
+        if connector.name == name:
+            return connector
+    return None
+
+
+__all__ = [
+    "ALL_CONNECTORS",
+    "get_connector",
+    "CisaKevConnector",
+    "cisa_kev_connector",
+    "CloudflareRadarConnector",
+    "cloudflare_radar_connector",
+    "RedditConnector",
+    "reddit_connector",
+    "TelegramConnector",
+    "telegram_connector",
+    "GdeltConnector",
+    "gdelt_connector",
+]