signals-engine 0.1.0__tar.gz

signals_engine-0.1.0/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.4
+Name: signals-engine
+Version: 0.1.0
+Summary: Python collect CLI for signal-oriented collection lanes
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: PyYAML>=6.0
+Requires-Dist: httpx>=0.27.0
+
+# Signal Engine
+
+Python collect CLI for signal-oriented collection lanes.
+
+## v1 scope
+- collect-only runtime
+- signal markdown / index / state outputs
+- thin run.json manifest
+- first migration target: x-feed

signals_engine-0.1.0/README.md

@@ -0,0 +1,9 @@
+# Signal Engine
+
+Python collect CLI for signal-oriented collection lanes.
+
+## v1 scope
+- collect-only runtime
+- signal markdown / index / state outputs
+- thin run.json manifest
+- first migration target: x-feed

signals_engine-0.1.0/pyproject.toml

@@ -0,0 +1,20 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "signals-engine"
+version = "0.1.0"
+description = "Python collect CLI for signal-oriented collection lanes"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = ["PyYAML>=6.0", "httpx>=0.27.0"]
+
+[project.scripts]
+signal-engine = "signal_engine.cli:main"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]

signals_engine-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

signals_engine-0.1.0/src/signal_engine/__init__.py

@@ -0,0 +1 @@
+__all__ = []

signals_engine-0.1.0/src/signal_engine/cli.py

@@ -0,0 +1,37 @@
+"""Signal Engine CLI entry point."""
+import argparse
+import sys
+
+from .commands import collect, diagnose, status, lanes, config
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(prog="signal-engine")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    collect.add_parser(sub)
+    diagnose.add_parser(sub)
+    status.add_parser(sub)
+    lanes.add_parser(sub)
+    config.add_parser(sub)
+
+    args = parser.parse_args()
+
+    if args.command is None:
+        parser.print_help()
+        return 1
+
+    return COMMANDS[args.command](args)
+
+
+COMMANDS: dict[str, callable] = {
+    "collect": collect.run,
+    "diagnose": diagnose.run,
+    "status": status.run,
+    "lanes": lanes.run,
+    "config": config.run,
+}
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

signals_engine-0.1.0/src/signal_engine/commands/collect.py

@@ -0,0 +1,110 @@
+"""collect command."""
+import argparse
+import os
+from pathlib import Path
+
+from ..core import RunContext, ConfigError, RunStatus
+
+
+def add_parser(sub: argparse._SubParsersAction) -> argparse.ArgumentParser:
+    p = sub.add_parser("collect", help="Collect signals for a lane")
+    p.add_argument("--lane", required=True, help="Lane name (e.g. x-feed)")
+    p.add_argument("--date", default=None, help="Date in YYYY-MM-DD format (default: today)")
+    p.add_argument("--data-dir", default=None, help="Data directory path")
+    p.add_argument("--config", default=None, help="Config file path")
+    p.add_argument(
+        "--debug-log",
+        default=None,
+        help="Path to debug log file (default: <data-dir>/debug.log)",
+    )
+    return p
+
+
+def load_config(config_path: str | None) -> dict:
+    """Load lanes config from yaml."""
+    if config_path:
+        path = Path(config_path).expanduser()
+    else:
+        default_config = os.environ.get(
+            "DAILY_LANE_CONFIG",
+            str(Path.home() / ".daily-lane" / "config" / "lanes.yaml")
+        )
+        path = Path(default_config)
+
+    if not path.exists():
+        raise ConfigError(f"Config file not found: {path}")
+
+    with open(path) as f:
+        import yaml
+        return yaml.safe_load(f)
+
+
+def run(args: argparse.Namespace) -> int:
+    """Execute the collect command.
+
+    Exit codes:
+        0 = SUCCESS or EMPTY (normal completion)
+        1 = FAILED or exception
+    """
+    import sys
+    from datetime import date
+    from ..core.debuglog import debug_log
+    from ..runtime.collect import collect_lane
+
+    lane = args.lane
+    run_date = args.date or date.today().isoformat()
+
+    if args.data_dir:
+        data_dir = Path(args.data_dir).expanduser()
+    else:
+        data_dir = Path(os.environ.get(
+            "DAILY_LANE_DATA_DIR",
+            str(Path.home() / ".daily-lane-data")
+        ))
+
+    # Debug log path
+    if args.debug_log:
+        debug_log_path = Path(args.debug_log)
+    else:
+        debug_log_path = data_dir / "debug.log"
+
+    config = load_config(args.config)
+    ctx = RunContext(
+        lane=lane,
+        date=run_date,
+        data_dir=data_dir,
+        config=config,
+        debug_log_path=debug_log_path,
+    )
+    ctx.ensure_dirs()
+
+    debug_log(f"[collect] START lane={lane} date={run_date} data_dir={data_dir}", log_file=debug_log_path)
+
+    try:
+        result = collect_lane(ctx)
+        debug_log(
+            f"[collect] END status={result.status.value} signals={result.signals_written} "
+            f"session={result.session_id or 'n/a'}",
+            log_file=debug_log_path,
+        )
+        if result.errors:
+            for err in result.errors:
+                debug_log(f"[collect] ERROR: {err}", log_file=debug_log_path)
+
+        print(
+            f"[{result.status.value}] {lane}/{run_date}: "
+            f"{result.signals_written} signals, "
+            f"session={result.session_id or 'n/a'}",
+            file=sys.stderr,
+        )
+        if result.errors:
+            for err in result.errors:
+                print(f"  ERROR: {err}", file=sys.stderr)
+
+        # Exit code: 0 for SUCCESS/EMPTY, 1 for FAILED
+        return 0 if result.status in (RunStatus.SUCCESS, RunStatus.EMPTY) else 1
+
+    except Exception as e:
+        debug_log(f"[collect] EXCEPTION: {e}", log_file=debug_log_path)
+        print(f"ERROR: {e}", file=sys.stderr)
+        return 1

signals_engine-0.1.0/src/signal_engine/commands/config.py

@@ -0,0 +1,38 @@
+"""config command."""
+import argparse
+
+
+def add_parser(sub: argparse._SubParsersAction) -> argparse.ArgumentParser:
+    p = sub.add_parser("config", help="Config operations")
+    sub2 = p.add_subparsers(dest="subcommand")
+    sub2.add_parser("check", help="Check config file")
+    return p
+
+
+def run(args: argparse.Namespace) -> int:
+    """Execute the config command."""
+    import sys
+    import yaml
+    import os
+    from pathlib import Path
+
+    if args.subcommand == "check" or args.subcommand is None:
+        default_config = os.environ.get(
+            "DAILY_LANE_CONFIG",
+            str(Path.home() / ".daily-lane" / "config" / "lanes.yaml")
+        )
+        path = Path(default_config)
+        if not path.exists():
+            print(f"ERROR: config not found: {path}", file=sys.stderr)
+            return 1
+        try:
+            with open(path) as f:
+                data = yaml.safe_load(f)
+            lanes = list(data.get("lanes", {}).keys())
+            print(f"OK: {path}", file=sys.stdout)
+            print(f"Lanes: {', '.join(lanes)}", file=sys.stdout)
+            return 0
+        except Exception as e:
+            print(f"ERROR: {e}", file=sys.stderr)
+            return 1
+    return 0

signals_engine-0.1.0/src/signal_engine/commands/diagnose.py

@@ -0,0 +1,57 @@
+"""diagnose command."""
+import argparse
+import os
+from pathlib import Path
+
+import yaml
+
+
+def add_parser(sub: argparse._SubParsersAction) -> argparse.ArgumentParser:
+    p = sub.add_parser("diagnose", help="Run diagnostics for a lane")
+    p.add_argument("--lane", required=True, help="Lane name")
+    p.add_argument("--data-dir", default=None, help="Data directory path")
+    p.add_argument("--config", default=None, help="Config file path")
+    p.add_argument(
+        "--debug-log",
+        default=None,
+        help="Path to debug log file (default: <data-dir>/debug.log)",
+    )
+    return p
+
+
+def run(args: argparse.Namespace) -> int:
+    """Execute the diagnose command."""
+    import sys
+    from ..core.debuglog import debug_log
+    from ..runtime.diagnose import diagnose_lane
+
+    data_dir = Path(args.data_dir) if args.data_dir else None
+    debug_log_path = None
+    if args.debug_log:
+        debug_log_path = Path(args.debug_log)
+    elif data_dir:
+        debug_log_path = data_dir / "debug.log"
+
+    config = None
+    config_path = args.config or os.environ.get(
+        "DAILY_LANE_CONFIG",
+        str(Path.home() / ".daily-lane" / "config" / "lanes.yaml")
+    )
+    if Path(config_path).exists():
+        try:
+            with open(config_path) as f:
+                config = yaml.safe_load(f)
+        except Exception:
+            pass
+
+    debug_log(f"[diagnose] START lane={args.lane} data_dir={data_dir}", log_file=debug_log_path)
+
+    try:
+        result = diagnose_lane(args.lane, data_dir=data_dir, config=config)
+        debug_log(f"[diagnose] END exit_code={result.exit_code}", log_file=debug_log_path)
+        print(result.output, file=sys.stdout)
+        return 0 if result.exit_code == 0 else 1
+    except Exception as e:
+        debug_log(f"[diagnose] EXCEPTION: {e}", log_file=debug_log_path)
+        print(f"ERROR: {e}", file=sys.stderr)
+        return 1

signals_engine-0.1.0/src/signal_engine/commands/lanes.py

@@ -0,0 +1,20 @@
+"""lanes command."""
+import argparse
+
+
+def add_parser(sub: argparse._SubParsersAction) -> argparse.ArgumentParser:
+    p = sub.add_parser("lanes", help="List available lanes")
+    sub2 = p.add_subparsers(dest="subcommand")
+    sub2.add_parser("list", help="List lanes")
+    return p
+
+
+def run(args: argparse.Namespace) -> int:
+    """Execute the lanes command."""
+    import sys
+    from ..lanes.registry import LANE_REGISTRY
+    if args.subcommand == "list" or args.subcommand is None:
+        for name in LANE_REGISTRY:
+            print(name, file=sys.stdout)
+        return 0
+    return 0

signals_engine-0.1.0/src/signal_engine/commands/status.py

@@ -0,0 +1,26 @@
+"""status command."""
+import argparse
+from pathlib import Path
+
+
+def add_parser(sub: argparse._SubParsersAction) -> argparse.ArgumentParser:
+    p = sub.add_parser("status", help="Show status for a lane run")
+    p.add_argument("--lane", required=True, help="Lane name")
+    p.add_argument("--date", required=True, help="Date in YYYY-MM-DD format")
+    p.add_argument("--data-dir", default=None, help="Data directory path")
+    return p
+
+
+def run(args: argparse.Namespace) -> int:
+    """Execute the status command."""
+    import sys
+    import json
+    from ..runtime.status import get_run_status
+    data_dir = Path(args.data_dir) if args.data_dir else None
+    try:
+        result = get_run_status(args.lane, args.date, data_dir=data_dir)
+        print(json.dumps(result, indent=2), file=sys.stdout)
+        return 0
+    except Exception as e:
+        print(f"ERROR: {e}", file=sys.stderr)
+        return 1

signals_engine-0.1.0/src/signal_engine/core/__init__.py

@@ -0,0 +1,27 @@
+"""Core module: models, context, paths, errors."""
+from .models import RunStatus, SignalRecord, RunResult
+from .context import RunContext
+from .paths import signal_file_path, index_file_path, run_json_path, state_file_path
+from .errors import (
+    SignalEngineError,
+    SourceError,
+    ConfigError,
+    RenderError,
+    WriteError,
+)
+
+__all__ = [
+    "RunStatus",
+    "SignalRecord",
+    "RunResult",
+    "RunContext",
+    "signal_file_path",
+    "index_file_path",
+    "run_json_path",
+    "state_file_path",
+    "SignalEngineError",
+    "SourceError",
+    "ConfigError",
+    "RenderError",
+    "WriteError",
+]

signals_engine-0.1.0/src/signal_engine/core/context.py

@@ -0,0 +1,33 @@
+"""Run context for Signal Engine."""
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class RunContext:
+    """Immutable run context passed through the collect pipeline."""
+    lane: str
+    date: str
+    data_dir: Path
+    config: dict
+    debug_log_path: Path | None = field(default=None)
+
+    @property
+    def signals_dir(self) -> Path:
+        return self.data_dir / "signals" / self.lane / self.date / "signals"
+
+    @property
+    def state_dir(self) -> Path:
+        return self.data_dir / "signals" / self.lane / "state"
+
+    @property
+    def index_path(self) -> Path:
+        return self.data_dir / "signals" / self.lane / self.date / "index.md"
+
+    @property
+    def run_json_path(self) -> Path:
+        return self.data_dir / "signals" / self.lane / self.date / "run.json"
+
+    def ensure_dirs(self) -> None:
+        self.signals_dir.mkdir(parents=True, exist_ok=True)
+        self.state_dir.mkdir(parents=True, exist_ok=True)

signals_engine-0.1.0/src/signal_engine/core/debuglog.py

@@ -0,0 +1,23 @@
+"""Minimal debug logging helper.
+
+Writes to stderr and optionally appends to a log file.
+No log rotation, no structured JSON — just plain text lines.
+"""
+from pathlib import Path
+import sys
+
+
+def debug_log(message: str, log_file: Path | str | None = None) -> None:
+    """Write a debug line to stderr and optionally append to a log file.
+
+    Args:
+        message: The debug message to log.
+        log_file: Optional path to a file to append to. If None, only stderr.
+    """
+    line = f"[debug] {message}"
+    print(line, file=sys.stderr)
+    if log_file is not None:
+        p = Path(log_file).expanduser()
+        p.parent.mkdir(parents=True, exist_ok=True)
+        with open(p, "a") as f:
+            f.write(line + "\n")

signals_engine-0.1.0/src/signal_engine/core/errors.py

@@ -0,0 +1,26 @@
+"""Custom errors for Signal Engine."""
+
+
+class SignalEngineError(Exception):
+    """Base exception for all Signal Engine errors."""
+    pass
+
+
+class SourceError(SignalEngineError):
+    """Raised when a data source fails to return usable data."""
+    pass
+
+
+class ConfigError(SignalEngineError):
+    """Raised when configuration is invalid or missing."""
+    pass
+
+
+class RenderError(SignalEngineError):
+    """Raised when rendering a derived artifact fails."""
+    pass
+
+
+class WriteError(SignalEngineError):
+    """Raised when writing an output file fails."""
+    pass

signals_engine-0.1.0/src/signal_engine/core/models.py

@@ -0,0 +1,52 @@
+"""Core data models for Signal Engine."""
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class RunStatus(str, Enum):
+    SUCCESS = "success"
+    FAILED = "failed"
+    EMPTY = "empty"
+
+
+@dataclass
+class SignalRecord:
+    # Core fixed fields (align with v1 spec)
+    lane: str
+    signal_type: str
+    source: str
+    entity_type: str
+    entity_id: str
+    title: str
+    source_url: str
+    fetched_at: str
+    file_path: str | None = None
+
+    # x-feed explicit internal fields
+    session_id: str = ""  # set at collect time for frontmatter compatibility
+    handle: str = ""
+    post_id: str = ""
+    created_at: str = ""
+    position: int = 0
+    text_preview: str = ""
+    likes: int = 0
+    retweets: int = 0
+    replies: int = 0
+    views: int = 0
+
+
+@dataclass
+class RunResult:
+    lane: str
+    date: str
+    status: RunStatus
+    started_at: str
+    session_id: str | None = None
+    finished_at: str | None = None
+    warnings: list[str] = field(default_factory=list)
+    errors: list[str] = field(default_factory=list)
+    signal_records: list[SignalRecord] = field(default_factory=list)
+    repos_checked: int = 0
+    signals_written: int = 0
+    signal_types_count: dict[str, int] = field(default_factory=dict)
+    index_file: str | None = None

signals_engine-0.1.0/src/signal_engine/core/paths.py

@@ -0,0 +1,24 @@
+"""Path construction utilities for Signal Engine."""
+from pathlib import Path
+
+
+def signal_file_path(lane: str, date: str, filename: str) -> Path:
+    """Return the full path to a signal file under a lane/date/signals/ directory."""
+    return Path("signals") / lane / date / "signals" / filename
+
+
+def index_file_path(lane: str, date: str) -> Path:
+    """Return the full path to an index.md under a lane/date/ directory."""
+    return Path("signals") / lane / date / "index.md"
+
+
+def run_json_path(lane: str, date: str) -> Path:
+    """Return the full path to a run.json under a lane/date/ directory."""
+    return Path("signals") / lane / date / "run.json"
+
+
+def state_file_path(lane: str, owner: str, repo: str, file_type: str) -> Path:
+    """Return the full path to a state file."""
+    safe_owner = owner.replace("/", "__")
+    safe_repo = repo.replace("/", "__")
+    return Path("signals") / lane / "state" / f"{safe_owner}__{safe_repo}__{file_type}.md"

signals_engine-0.1.0/src/signal_engine/lanes/__init__.py

@@ -0,0 +1,4 @@
+"""Lanes module: registry and lane implementations."""
+from .registry import LANE_REGISTRY, get_lane_collector
+
+__all__ = ["LANE_REGISTRY", "get_lane_collector"]

signals_engine-0.1.0/src/signal_engine/lanes/registry.py

@@ -0,0 +1,21 @@
+"""Lane registry — maps lane name to collector callable."""
+from typing import Callable
+from ..core import RunResult, RunContext
+
+
+CollectorFn = Callable[[RunContext], RunResult]
+
+
+LANE_REGISTRY: dict[str, CollectorFn | None] = {
+    "x-feed": None,  # filled by lanes.x_feed
+}
+
+
+def get_lane_collector(lane: str) -> CollectorFn | None:
+    """Return the collector for a given lane, or None if not registered."""
+    return LANE_REGISTRY.get(lane)
+
+
+def register_lane(lane: str, collector: CollectorFn) -> None:
+    """Register a lane collector at runtime."""
+    LANE_REGISTRY[lane] = collector