agent-data-cli 0.1.0__py3-none-any.whl

agent_data_cli/store/db.py

@@ -0,0 +1,298 @@
+from __future__ import annotations
+
+import re
+import sqlite3
+
+from agent_data_cli.core.models import (
+    ActionAuditRecord,
+    ChannelRecord,
+    ContentBatchWriteResult,
+    ContentQueryRow,
+    ContentSyncBatch,
+    ContentNode,
+    ContentRecord,
+    HealthRecord,
+    SourceDescriptor,
+    SourceStorageSpec,
+    SubscriptionRecord,
+)
+
+from . import audit as audit_store
+from . import channels as channel_store
+from . import configs as config_store
+from . import content as content_store
+from . import groups as group_store
+from . import health as health_store
+from .migrations import SCHEMA
+from . import subscriptions as subscription_store
+
+
+class Store:
+    def __init__(self, path: str) -> None:
+        self.path = path
+        self._connection = sqlite3.connect(self.path)
+        self._connection.row_factory = sqlite3.Row
+        self._storage_specs: dict[str, SourceStorageSpec] = {}
+
+    def init_schema(self, storage_specs: list[SourceStorageSpec] | None = None) -> None:
+        if storage_specs is not None:
+            self.set_storage_specs(storage_specs)
+        with self._connect() as connection:
+            connection.executescript(SCHEMA)
+
+    def set_storage_specs(self, storage_specs: list[SourceStorageSpec]) -> None:
+        specs_by_source: dict[str, SourceStorageSpec] = {}
+        for spec in storage_specs:
+            if spec.source in specs_by_source:
+                raise RuntimeError(f"duplicate storage spec for source: {spec.source}")
+            specs_by_source[spec.source] = spec
+        self._storage_specs = specs_by_source
+
+    def upsert_source(self, source: SourceDescriptor) -> None:
+        with self._connect() as connection:
+            channel_store.upsert_source(connection, source)
+
+    def upsert_channel(self, channel: ChannelRecord) -> None:
+        with self._connect() as connection:
+            channel_store.upsert_channel(connection, channel)
+
+    def get_channel(self, source: str, channel_key: str) -> ChannelRecord | None:
+        with self._connect() as connection:
+            return channel_store.get_channel(connection, source, channel_key)
+
+    def add_subscription(
+        self,
+        source: str,
+        channel_key: str,
+        display_name: str,
+        metadata: dict[str, str],
+    ) -> SubscriptionRecord:
+        with self._connect() as connection:
+            return subscription_store.add_subscription(
+                connection,
+                source=source,
+                channel_key=channel_key,
+                display_name=display_name,
+                metadata=metadata,
+            )
+
+    def set_source_config(
+        self,
+        source: str,
+        key: str,
+        value: str,
+        value_type: str,
+        is_secret: bool,
+    ) -> None:
+        with self._connect() as connection:
+            config_store.set_source_config(connection, source, key, value, value_type, is_secret)
+
+    def unset_source_config(self, source: str, key: str) -> None:
+        with self._connect() as connection:
+            config_store.unset_source_config(connection, source, key)
+
+    def set_cli_config(
+        self,
+        key: str,
+        value: str,
+        value_type: str,
+        is_secret: bool,
+    ) -> None:
+        with self._connect() as connection:
+            config_store.set_cli_config(connection, key, value, value_type, is_secret)
+
+    def unset_cli_config(self, key: str) -> None:
+        with self._connect() as connection:
+            config_store.unset_cli_config(connection, key)
+
+    def list_cli_configs(self):
+        with self._connect() as connection:
+            return config_store.list_cli_configs(connection)
+
+    def get_cli_config_map(self):
+        return {entry.key: entry for entry in self.list_cli_configs()}
+
+    def list_source_configs(self, source: str | None = None):
+        with self._connect() as connection:
+            return config_store.list_source_configs(connection, source)
+
+    def get_source_config_map(self, source: str):
+        return {
+            entry.key: entry
+            for entry in self.list_source_configs(source)
+        }
+
+    def prune_source_configs(self, allowed_keys_by_source: dict[str, set[str]]) -> None:
+        with self._connect() as connection:
+            config_store.prune_source_configs(connection, allowed_keys_by_source)
+
+    def prune_cli_configs(self, allowed_keys: set[str]) -> None:
+        with self._connect() as connection:
+            config_store.prune_cli_configs(connection, allowed_keys)
+
+    def purge_source_state(self, source: str) -> None:
+        with self._connect() as connection:
+            group_store.delete_source_group_members(connection, source)
+            subscription_store.delete_source_subscriptions(connection, source)
+            config_store.delete_source_configs(connection, source)
+            content_store.delete_source_sync_state(connection, source)
+            health_store.delete_health(connection, source)
+            audit_store.delete_action_audits(connection, source)
+            content_store.delete_source_content(connection, source)
+            channel_store.delete_channels(connection, source)
+            channel_store.delete_source(connection, source)
+
+    def create_group(self, group_name: str) -> None:
+        with self._connect() as connection:
+            group_store.create_group(connection, group_name)
+
+    def delete_group(self, group_name: str) -> None:
+        with self._connect() as connection:
+            group_store.delete_group(connection, group_name)
+
+    def list_groups(self):
+        with self._connect() as connection:
+            return group_store.list_groups(connection)
+
+    def add_group_source(self, group_name: str, source: str) -> None:
+        with self._connect() as connection:
+            group_store.add_group_source(connection, group_name, source)
+
+    def add_group_channel(self, group_name: str, source: str, channel_key: str) -> None:
+        with self._connect() as connection:
+            group_store.add_group_channel(connection, group_name, source, channel_key)
+
+    def remove_group_source(self, group_name: str, source: str) -> None:
+        with self._connect() as connection:
+            group_store.remove_group_source(connection, group_name, source)
+
+    def remove_group_channel(self, group_name: str, source: str, channel_key: str) -> None:
+        with self._connect() as connection:
+            group_store.remove_group_channel(connection, group_name, source, channel_key)
+
+    def list_group_members(self, group_name: str):
+        with self._connect() as connection:
+            return group_store.list_group_members(connection, group_name)
+
+    def expand_group_update_targets(self, group_name: str) -> list[tuple[str, str]]:
+        with self._connect() as connection:
+            return group_store.expand_group_update_targets(connection, group_name)
+
+    def remove_subscription(self, source: str, channel_key: str) -> None:
+        with self._connect() as connection:
+            subscription_store.remove_subscription(connection, source, channel_key)
+
+    def list_subscriptions(self, source: str | None = None) -> list[SubscriptionRecord]:
+        with self._connect() as connection:
+            return subscription_store.list_subscriptions(connection, source)
+
+    def is_subscribed(self, source: str, channel_key: str) -> bool:
+        with self._connect() as connection:
+            return subscription_store.is_subscribed(connection, source, channel_key)
+
+    def upsert_content(self, record: ContentRecord) -> bool:
+        with self._connect() as connection:
+            return content_store.upsert_content(connection, "", record)
+
+    def write_content_batch(
+        self,
+        source: str,
+        channel_key: str,
+        batch: ContentSyncBatch,
+    ) -> ContentBatchWriteResult:
+        with self._connect() as connection:
+            return content_store.write_content_batch(
+                connection,
+                source=source,
+                channel_key=channel_key,
+                batch=batch,
+            )
+
+    def insert_action_audit(self, record: ActionAuditRecord) -> None:
+        with self._connect() as connection:
+            audit_store.insert_action_audit(connection, record)
+
+    def set_sync_state(self, source: str, channel_key: str, cursor: str) -> None:
+        with self._connect() as connection:
+            content_store.set_sync_state(connection, source, channel_key, cursor)
+
+    def save_health(self, record: HealthRecord) -> None:
+        with self._connect() as connection:
+            health_store.save_health(connection, record)
+
+    def get_latest_health(self, source: str) -> HealthRecord | None:
+        with self._connect() as connection:
+            return health_store.get_latest_health(connection, source)
+
+    def list_content(
+        self,
+        source: str,
+        channel_key: str,
+        *,
+        record_type: str | None = None,
+        limit: int = 10,
+        since: str | None = None,
+        fetch_all: bool = False,
+    ) -> list[ContentRecord]:
+        with self._connect() as connection:
+            return content_store.list_content(
+                connection,
+                table_name="",
+                source=source,
+                channel_key=channel_key,
+                record_type=record_type,
+                limit=limit,
+                since=since,
+                fetch_all=fetch_all,
+            )
+
+    def query_content(
+        self,
+        *,
+        source: str | None = None,
+        channel_key: str | None = None,
+        group_name: str | None = None,
+        record_type: str | None = None,
+        parent_ref: str | None = None,
+        children_ref: str | None = None,
+        depth: int = 1,
+        since: str | None = None,
+        keywords: str | None = None,
+        limit: int = 10,
+        fetch_all: bool = False,
+    ) -> list[ContentQueryRow]:
+        with self._connect() as connection:
+            return content_store.query_content(
+                connection,
+                storage_specs=self._storage_specs,
+                source=source,
+                channel_key=channel_key,
+                group_name=group_name,
+                record_type=record_type,
+                parent_ref=parent_ref,
+                children_ref=children_ref,
+                depth=depth,
+                since=since,
+                keywords=keywords,
+                limit=limit,
+                fetch_all=fetch_all,
+            )
+
+    def list_content_channels(self, source: str, content_key: str) -> tuple[str, ...]:
+        with self._connect() as connection:
+            return content_store.list_content_channels(connection, source, content_key)
+
+    def _connect(self) -> sqlite3.Connection:
+        return self._connection
+
+    def _table_for_source(self, source: str) -> str:
+        return self._require_storage_spec(source).table_name
+
+    def _require_storage_spec(self, source: str) -> SourceStorageSpec:
+        try:
+            return self._storage_specs[source]
+        except KeyError as exc:
+            raise RuntimeError(f"no storage spec registered for source: {source}") from exc
+
+    def _is_valid_identifier(self, value: str) -> bool:
+        return re.fullmatch(r"[a-z][a-z0-9_]*", value) is not None

agent_data_cli/store/groups.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import sqlite3
+
+from agent_data_cli.utils.time import utc_now_iso
+
+from .repositories import row_to_group, row_to_group_member, row_to_subscription
+
+
+def create_group(connection: sqlite3.Connection, group_name: str) -> None:
+    connection.execute(
+        """
+        INSERT INTO groups (group_name, created_at)
+        VALUES (?, ?)
+        ON CONFLICT(group_name) DO NOTHING
+        """,
+        (group_name, utc_now_iso()),
+    )
+
+
+def delete_group(connection: sqlite3.Connection, group_name: str) -> None:
+    connection.execute("DELETE FROM group_members WHERE group_name = ?", (group_name,))
+    connection.execute("DELETE FROM groups WHERE group_name = ?", (group_name,))
+
+
+def list_groups(connection: sqlite3.Connection):
+    rows = connection.execute(
+        "SELECT group_name, created_at FROM groups ORDER BY group_name"
+    ).fetchall()
+    return [row_to_group(row) for row in rows]
+
+
+def add_group_source(connection: sqlite3.Connection, group_name: str, source: str) -> None:
+    create_group(connection, group_name)
+    connection.execute(
+        """
+        INSERT INTO group_members (group_name, member_type, source, channel_key)
+        VALUES (?, 'source', ?, '')
+        ON CONFLICT(group_name, member_type, source, channel_key) DO NOTHING
+        """,
+        (group_name, source),
+    )
+
+
+def add_group_channel(connection: sqlite3.Connection, group_name: str, source: str, channel_key: str) -> None:
+    create_group(connection, group_name)
+    connection.execute(
+        """
+        INSERT INTO group_members (group_name, member_type, source, channel_key)
+        VALUES (?, 'channel', ?, ?)
+        ON CONFLICT(group_name, member_type, source, channel_key) DO NOTHING
+        """,
+        (group_name, source, channel_key),
+    )
+
+
+def remove_group_source(connection: sqlite3.Connection, group_name: str, source: str) -> None:
+    connection.execute(
+        """
+        DELETE FROM group_members
+        WHERE group_name = ? AND member_type = 'source' AND source = ? AND channel_key = ''
+        """,
+        (group_name, source),
+    )
+
+
+def remove_group_channel(connection: sqlite3.Connection, group_name: str, source: str, channel_key: str) -> None:
+    connection.execute(
+        """
+        DELETE FROM group_members
+        WHERE group_name = ? AND member_type = 'channel' AND source = ? AND channel_key = ?
+        """,
+        (group_name, source, channel_key),
+    )
+
+
+def delete_source_group_members(connection: sqlite3.Connection, source: str) -> None:
+    connection.execute("DELETE FROM group_members WHERE source = ?", (source,))
+
+
+def list_group_members(connection: sqlite3.Connection, group_name: str):
+    rows = connection.execute(
+        """
+        SELECT group_name, member_type, source, channel_key
+        FROM group_members
+        WHERE group_name = ?
+        ORDER BY member_type, source, channel_key
+        """,
+        (group_name,),
+    ).fetchall()
+    return [row_to_group_member(row) for row in rows]
+
+
+def expand_group_update_targets(connection: sqlite3.Connection, group_name: str) -> list[tuple[str, str]]:
+    targets: set[tuple[str, str]] = set()
+    for member in list_group_members(connection, group_name):
+        if member.member_type == "channel" and member.channel_key:
+            targets.add((member.source, member.channel_key))
+            continue
+        if member.member_type == "source":
+            rows = connection.execute(
+                """
+                SELECT
+                    subscription_id,
+                    source,
+                    channel_key,
+                    display_name,
+                    created_at,
+                    last_updated_at,
+                    enabled,
+                    metadata_json
+                FROM subscriptions
+                WHERE source = ?
+                ORDER BY channel_key
+                """,
+                (member.source,),
+            ).fetchall()
+            for subscription in (row_to_subscription(row) for row in rows):
+                targets.add((subscription.source, subscription.channel_key))
+    return sorted(targets)

agent_data_cli/store/health.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import sqlite3
+
+from agent_data_cli.core.models import HealthRecord
+
+from .repositories import row_to_health
+
+
+def save_health(connection: sqlite3.Connection, record: HealthRecord) -> None:
+    connection.execute(
+        """
+        INSERT INTO health_checks (
+            source,
+            status,
+            checked_at,
+            latency_ms,
+            error,
+            details
+        )
+        VALUES (?, ?, ?, ?, ?, ?)
+        ON CONFLICT(source) DO UPDATE SET
+            status = excluded.status,
+            checked_at = excluded.checked_at,
+            latency_ms = excluded.latency_ms,
+            error = excluded.error,
+            details = excluded.details
+        """,
+        (
+            record.source,
+            record.status,
+            record.checked_at,
+            record.latency_ms,
+            record.error,
+            record.details,
+        ),
+    )
+
+
+def get_latest_health(connection: sqlite3.Connection, source: str) -> HealthRecord | None:
+    row = connection.execute(
+        """
+        SELECT source, status, checked_at, latency_ms, error, details
+        FROM health_checks
+        WHERE source = ?
+        """,
+        (source,),
+    ).fetchone()
+    return row_to_health(row)
+
+
+def delete_health(connection: sqlite3.Connection, source: str) -> None:
+    connection.execute("DELETE FROM health_checks WHERE source = ?", (source,))

agent_data_cli/store/migrations.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS sources (
+    name TEXT PRIMARY KEY,
+    display_name TEXT NOT NULL,
+    summary TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS channels (
+    source TEXT NOT NULL,
+    channel_id TEXT NOT NULL,
+    channel_key TEXT NOT NULL,
+    display_name TEXT NOT NULL,
+    url TEXT NOT NULL,
+    metadata_json TEXT NOT NULL,
+    PRIMARY KEY (source, channel_key)
+);
+
+CREATE TABLE IF NOT EXISTS subscriptions (
+    subscription_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    source TEXT NOT NULL,
+    channel_key TEXT NOT NULL,
+    display_name TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    last_updated_at TEXT,
+    enabled INTEGER NOT NULL,
+    metadata_json TEXT NOT NULL,
+    UNIQUE (source, channel_key)
+);
+
+CREATE TABLE IF NOT EXISTS source_configs (
+    source TEXT NOT NULL,
+    key TEXT NOT NULL,
+    value TEXT NOT NULL,
+    value_type TEXT NOT NULL,
+    is_secret INTEGER NOT NULL,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (source, key)
+);
+
+CREATE TABLE IF NOT EXISTS cli_configs (
+    key TEXT PRIMARY KEY,
+    value TEXT NOT NULL,
+    value_type TEXT NOT NULL,
+    is_secret INTEGER NOT NULL,
+    updated_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS groups (
+    group_name TEXT PRIMARY KEY,
+    created_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS group_members (
+    group_name TEXT NOT NULL,
+    member_type TEXT NOT NULL,
+    source TEXT NOT NULL,
+    channel_key TEXT NOT NULL,
+    PRIMARY KEY (group_name, member_type, source, channel_key)
+);
+
+CREATE TABLE IF NOT EXISTS sync_state (
+    source TEXT NOT NULL,
+    channel_key TEXT NOT NULL,
+    cursor TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (source, channel_key)
+);
+
+CREATE TABLE IF NOT EXISTS health_checks (
+    source TEXT PRIMARY KEY,
+    status TEXT NOT NULL,
+    checked_at TEXT NOT NULL,
+    latency_ms INTEGER NOT NULL,
+    error TEXT,
+    details TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS action_audits (
+    audit_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    executed_at TEXT NOT NULL,
+    action TEXT NOT NULL,
+    source TEXT NOT NULL,
+    mode TEXT,
+    target_kind TEXT NOT NULL,
+    targets_json TEXT NOT NULL,
+    params_summary TEXT NOT NULL,
+    status TEXT NOT NULL,
+    error TEXT,
+    dry_run INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS content_nodes (
+    node_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    source TEXT NOT NULL,
+    content_key TEXT NOT NULL,
+    content_type TEXT NOT NULL,
+    external_id TEXT NOT NULL,
+    title TEXT NOT NULL,
+    url TEXT NOT NULL,
+    snippet TEXT NOT NULL,
+    author TEXT,
+    published_at TEXT,
+    fetched_at TEXT NOT NULL,
+    raw_payload TEXT NOT NULL,
+    content_ref TEXT,
+    UNIQUE (source, content_key)
+);
+
+CREATE INDEX IF NOT EXISTS idx_content_nodes_source_time
+ON content_nodes(source, published_at DESC, node_id DESC);
+
+CREATE INDEX IF NOT EXISTS idx_content_nodes_source_type_time
+ON content_nodes(source, content_type, published_at DESC, node_id DESC);
+
+CREATE TABLE IF NOT EXISTS content_channel_links (
+    source TEXT NOT NULL,
+    channel_key TEXT NOT NULL,
+    content_key TEXT NOT NULL,
+    membership_kind TEXT NOT NULL,
+    linked_at TEXT NOT NULL,
+    PRIMARY KEY (source, channel_key, content_key)
+);
+
+CREATE INDEX IF NOT EXISTS idx_content_channel_links_source_channel
+ON content_channel_links(source, channel_key, membership_kind);
+
+CREATE INDEX IF NOT EXISTS idx_content_channel_links_source_content
+ON content_channel_links(source, content_key);
+
+CREATE TABLE IF NOT EXISTS content_relations (
+    source TEXT NOT NULL,
+    from_content_key TEXT NOT NULL,
+    relation_type TEXT NOT NULL,
+    to_content_key TEXT NOT NULL,
+    relation_semantic TEXT,
+    position INTEGER,
+    metadata_json TEXT NOT NULL,
+    PRIMARY KEY (source, from_content_key, relation_type, to_content_key)
+);
+
+CREATE INDEX IF NOT EXISTS idx_content_relations_source_parent
+ON content_relations(source, to_content_key, relation_type, position);
+
+CREATE INDEX IF NOT EXISTS idx_content_relations_source_child
+ON content_relations(source, from_content_key, relation_type, position);
+"""
+
+
+def build_content_table_schema(table_name: str) -> str:
+    return f"""
+CREATE TABLE IF NOT EXISTS {table_name} (
+    record_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    source TEXT NOT NULL,
+    channel_key TEXT NOT NULL,
+    record_type TEXT NOT NULL,
+    external_id TEXT NOT NULL,
+    title TEXT NOT NULL,
+    url TEXT NOT NULL,
+    snippet TEXT NOT NULL,
+    author TEXT,
+    published_at TEXT,
+    fetched_at TEXT NOT NULL,
+    raw_payload TEXT NOT NULL,
+    dedup_key TEXT NOT NULL UNIQUE,
+    content_ref TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_{table_name}_channel_type_time
+ON {table_name}(channel_key, record_type, published_at DESC, record_id DESC);
+
+CREATE INDEX IF NOT EXISTS idx_{table_name}_time
+ON {table_name}(published_at DESC, record_id DESC);
+"""