contextbase-plugin-codex-local 0.2.9__py3-none-any.whl

contextbase_plugin_codex_local-0.2.9.dist-info/METADATA

@@ -0,0 +1,12 @@
+Metadata-Version: 2.3
+Name: contextbase-plugin-codex-local
+Version: 0.2.9
+Summary: Codex local plugin for ContextBase
+Author: Alizain Feerasta
+Author-email: Alizain Feerasta <alizain.feerasta@gmail.com>
+Requires-Dist: contextbase-shared-plugins==0.2.9
+Requires-Dist: dagster==1.12.14
+Requires-Dist: dagster-dlt==0.28.14
+Requires-Dist: dlt>=1.26.0
+Requires-Dist: pydantic>=2.12.0
+Requires-Python: >=3.14, <3.15

contextbase_plugin_codex_local-0.2.9.dist-info/RECORD

@@ -0,0 +1,16 @@
+plugin_codex_local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+plugin_codex_local/binding_config.py,sha256=O0XCVcuYAgPL0ETBUkFHbWgWn-NPH1cMupLemxnzjGw,319
+plugin_codex_local/component.py,sha256=DfrjukSu9wu062gk5pAFRatFfmUT20T29JvIkadTyQs,3730
+plugin_codex_local/defs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+plugin_codex_local/defs/defs.yaml,sha256=v4A5kPSPPZEEI5QfIaavAo6_1CnKGm7io9mlI8ZcpSU,59
+plugin_codex_local/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+plugin_codex_local/models/ctx.py,sha256=UWab_pHMFEyTBDmyqekphE-UQP87b7Vi8O5p0sMtjQs,483
+plugin_codex_local/models/translators.py,sha256=2Ejj_WrPhoFAZI6uQ_PUkzUg0j9ITIZVyORdoMeTZoA,2108
+plugin_codex_local/plugin.json,sha256=ghXsEmoqR3tdHjTxlPBqUpQ4MhaMahxrTXo960-HHaI,84
+plugin_codex_local/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+plugin_codex_local/sources/snapshot.py,sha256=VTHP20xkVQYAtsRtmK1PHruFIOeHvh9eI7DclQUMHIY,2111
+plugin_codex_local/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+plugin_codex_local/utils/parse.py,sha256=h8TxTH_hJmE1vSgGJbu5aWjqeHT1LgrhNHAFj53NufU,5309
+contextbase_plugin_codex_local-0.2.9.dist-info/WHEEL,sha256=i9aSRDivn5iP9LaR1BLQX2GNAuriQWPsFwbbWygTX2k,81
+contextbase_plugin_codex_local-0.2.9.dist-info/METADATA,sha256=xUFnrzSCJads87VUEdPTar4hwMJAf4kadpmtDOLnPoE,410
+contextbase_plugin_codex_local-0.2.9.dist-info/RECORD,,

contextbase_plugin_codex_local-0.2.9.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.11.15
+Root-Is-Purelib: true
+Tag: py3-none-any

plugin_codex_local/binding_config.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import Field
+
+from shared_plugins.bindings import BaseBindingConfigModel, ResolvedPath
+
+
+class CodexLocalBindingConfig(BaseBindingConfigModel):
+    codex_dir: ResolvedPath = Field(
+        default_factory=lambda: Path.home() / ".codex",
+    )

plugin_codex_local/component.py

@@ -0,0 +1,110 @@
+import dagster as dg
+from dagster import AssetExecutionContext
+from dagster_dlt import DagsterDltResource
+from shared_plugins.automation import non_overlapping_automation_condition
+from shared_plugins.bindings import parse_binding_config
+from shared_plugins.control_plane import ControlPlaneClient
+from shared_plugins.dlt import resolve_partition_binding, run_dlt_pipeline
+from shared_plugins.naming import (
+    dagster_asset_group_name,
+    dagster_asset_tags,
+    dagster_dlt_asset_key,
+    dagster_partition_def_name,
+    dagster_pool_name,
+    dlt_source_name,
+    plugin_id_from_module,
+)
+from shared_plugins.resources import DLT_RESOURCE
+
+from .binding_config import CodexLocalBindingConfig
+from .sources.snapshot import codex_local_snapshot_source
+
+PLUGIN_ID = plugin_id_from_module(__file__)
+SNAPSHOT_JOB = "snapshot"
+SNAPSHOT_SOURCE_NAME = dlt_source_name(PLUGIN_ID, SNAPSHOT_JOB)
+
+
+def _build_snapshot_specs(
+    partitions_def: dg.PartitionsDefinition,
+    automation_condition: dg.AutomationCondition,
+) -> list[dg.AssetSpec]:
+    shared = dict(
+        group_name=dagster_asset_group_name(PLUGIN_ID),
+        tags=dagster_asset_tags(PLUGIN_ID),
+        automation_condition=automation_condition,
+        partitions_def=partitions_def,
+    )
+
+    session_key = dagster_dlt_asset_key(SNAPSHOT_SOURCE_NAME, "session")
+    record_key = dagster_dlt_asset_key(SNAPSHOT_SOURCE_NAME, "record")
+
+    return [
+        dg.AssetSpec(
+            key=session_key,
+            **shared,
+        ),
+        dg.AssetSpec(
+            key=record_key,
+            deps=[session_key],
+            **shared,
+        ),
+    ]
+
+
+class CodexLocalSyncComponent(dg.Component):
+    def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
+        partitions_def = dg.DynamicPartitionsDefinition(
+            name=dagster_partition_def_name(PLUGIN_ID)
+        )
+
+        snapshot_specs = _build_snapshot_specs(
+            partitions_def=partitions_def,
+            automation_condition=non_overlapping_automation_condition(
+                dg.AutomationCondition.on_missing()
+                | dg.AutomationCondition.on_cron("*/15 * * * *")
+            ),
+        )
+
+        @dg.multi_asset(
+            specs=snapshot_specs,
+            can_subset=True,
+            name="codex_local_snapshot",
+            pool=dagster_pool_name(PLUGIN_ID),
+        )
+        def codex_local_snapshot_assets(
+            context: AssetExecutionContext,
+            dlt_resource: DagsterDltResource,
+            control_plane: dg.ResourceParam[ControlPlaneClient],
+        ):
+            binding = resolve_partition_binding(
+                context=context,
+                control_plane=control_plane,
+                plugin_id=PLUGIN_ID,
+            )
+            binding_id = str(binding.binding_id)
+            cfg = parse_binding_config(binding, CodexLocalBindingConfig)
+
+            source = codex_local_snapshot_source(binding_id, cfg)
+            yield from run_dlt_pipeline(
+                context=context,
+                dlt_resource=dlt_resource,
+                source=source,
+                plugin_id=PLUGIN_ID,
+                binding_id=binding_id,
+                job_name=SNAPSHOT_JOB,
+            )
+
+        automation_sensor = dg.AutomationConditionSensorDefinition(
+            name="codex_local_automation_sensor",
+            target=dg.AssetSelection.assets(codex_local_snapshot_assets),
+            default_status=dg.DefaultSensorStatus.RUNNING,
+            minimum_interval_seconds=30,
+        )
+
+        return dg.Definitions(
+            assets=[codex_local_snapshot_assets],
+            sensors=[automation_sensor],
+            resources={
+                "dlt_resource": DLT_RESOURCE,
+            },
+        )

plugin_codex_local/defs/defs.yaml

@@ -0,0 +1 @@
+type: plugin_codex_local.component.CodexLocalSyncComponent

plugin_codex_local/models/ctx.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import AwareDatetime
+from shared_plugins.models import CtxModel, IdStr, NonNegativeInt
+
+
+class SessionRow(CtxModel):
+    file_path: IdStr
+    is_archived: bool
+    file_mtime: AwareDatetime | None = None
+
+
+class RecordRow(CtxModel):
+    file_path: IdStr
+    line_index: NonNegativeInt
+    record_type: str
+    payload_type: str | None = None
+    timestamp: AwareDatetime | None = None
+    payload: Any = None

plugin_codex_local/models/translators.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from collections.abc import Iterable, Iterator
+from typing import Any
+
+from shared_plugins.values import parse_utc_datetime_from_str
+
+from ..utils.parse import ParsedSession
+from .ctx import RecordRow, SessionRow
+
+
+def _strip_null_bytes(value: Any) -> Any:
+    """Recursively strip \\x00 null bytes from strings in a structure.
+
+    Postgres rejects \\u0000 in both text and jsonb columns.
+    """
+    if isinstance(value, str):
+        return value.replace("\x00", "")
+    if isinstance(value, dict):
+        return {k: _strip_null_bytes(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_strip_null_bytes(item) for item in value]
+    return value
+
+
+def sessions_to_ctx_models(
+    *,
+    binding_id: str,
+    snapshots: Iterable[ParsedSession],
+) -> Iterator[SessionRow]:
+    """Yield one SessionRow per parsed session file."""
+    for snapshot in snapshots:
+        yield SessionRow(
+            ctx_binding_id=binding_id,
+            ctx_source_updated_at=snapshot.file_mtime,
+            file_path=snapshot.file_path,
+            is_archived=snapshot.is_archived,
+            file_mtime=snapshot.file_mtime,
+        )
+
+
+def records_to_ctx_models(
+    *,
+    binding_id: str,
+    snapshots: Iterable[ParsedSession],
+) -> Iterator[RecordRow]:
+    """Yield one RecordRow per valid JSONL line across all sessions."""
+    for snapshot in snapshots:
+        for line in snapshot.lines:
+            if line.record_type is None:
+                continue
+
+            ts = (
+                parse_utc_datetime_from_str(line.timestamp_raw)
+                if line.timestamp_raw
+                else None
+            )
+
+            payload = _strip_null_bytes(line.payload)
+
+            yield RecordRow(
+                ctx_binding_id=binding_id,
+                ctx_source_updated_at=ts,
+                file_path=snapshot.file_path,
+                line_index=line.line_index,
+                record_type=line.record_type,
+                payload_type=line.payload_type,
+                timestamp=ts,
+                payload=payload,
+            )

plugin_codex_local/plugin.json

@@ -0,0 +1,7 @@
+{
+	"auth": {
+		"type": "none"
+	},
+	"mode": "dagster",
+	"plugin_id": "codex_local"
+}

plugin_codex_local/sources/snapshot.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any
+
+import dlt
+from shared_plugins.naming import (
+    dlt_resource_name,
+    dlt_source_name,
+    plugin_id_from_module,
+)
+from shared_plugins.resources import ctx_dlt_resource
+
+from ..binding_config import CodexLocalBindingConfig
+from ..models.ctx import RecordRow, SessionRow
+from ..models.translators import (
+    records_to_ctx_models,
+    sessions_to_ctx_models,
+)
+from ..utils.parse import parse_session_snapshots
+
+PLUGIN_ID = plugin_id_from_module(__file__)
+JOB = "snapshot"
+MERGE_WRITE_DISPOSITION = {"disposition": "merge", "strategy": "delete-insert"}
+MERGE_KEY = ("_ctx_binding_id",)
+
+
+@dlt.source(name=dlt_source_name(PLUGIN_ID, JOB))
+def codex_local_snapshot_source(
+    binding_id: str,
+    cfg: CodexLocalBindingConfig,
+) -> tuple[Any, ...]:
+    sessions_dir = cfg.codex_dir / "sessions"
+    archived_dir = cfg.codex_dir / "archived_sessions"
+
+    @ctx_dlt_resource(
+        name=dlt_resource_name("session"),
+        write_disposition=MERGE_WRITE_DISPOSITION,
+        merge_key=MERGE_KEY,
+        primary_key=("_ctx_binding_id", "file_path"),
+    )
+    def session_resource() -> Iterator[SessionRow]:
+        snapshots = parse_session_snapshots(
+            sessions_dir=sessions_dir,
+            archived_dir=archived_dir,
+        )
+        yield from sessions_to_ctx_models(
+            binding_id=binding_id,
+            snapshots=snapshots,
+        )
+
+    @ctx_dlt_resource(
+        name=dlt_resource_name("record"),
+        write_disposition=MERGE_WRITE_DISPOSITION,
+        merge_key=MERGE_KEY,
+        primary_key=("_ctx_binding_id", "file_path", "line_index"),
+        columns={"payload": {"data_type": "json"}},
+    )
+    def record_resource() -> Iterator[RecordRow]:
+        snapshots = parse_session_snapshots(
+            sessions_dir=sessions_dir,
+            archived_dir=archived_dir,
+        )
+        yield from records_to_ctx_models(
+            binding_id=binding_id,
+            snapshots=snapshots,
+        )
+
+    return (
+        session_resource,
+        record_resource,
+    )

plugin_codex_local/utils/parse.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from shared_plugins.values import as_mapping, as_string
+
+
+@dataclass(frozen=True)
+class ParsedLine:
+    """A single parsed JSONL line."""
+
+    line_index: int
+    record_type: str | None
+    payload_type: str | None
+    timestamp_raw: str | None
+    payload: Any
+
+
+@dataclass(frozen=True)
+class SessionFileRef:
+    """Filesystem reference to a JSONL file before parsing."""
+
+    file_path: str
+    is_archived: bool
+
+
+@dataclass(frozen=True)
+class ParsedSession:
+    """A fully parsed session file."""
+
+    file_path: str
+    is_archived: bool
+    file_mtime: datetime | None
+    lines: list[ParsedLine]
+
+
+def _parse_json_line(
+    line: str,
+) -> tuple[str | None, str | None, str | None, Any]:
+    """Parse a single JSON line into (record_type, payload_type, timestamp_raw, payload).
+
+    Returns (None, None, None, None) if the line cannot be parsed or lacks a type field.
+    """
+    try:
+        parsed = json.loads(line)
+    except (json.JSONDecodeError, ValueError):
+        return None, None, None, None
+
+    obj = as_mapping(parsed)
+    if obj is None:
+        return None, None, None, None
+
+    record_type = as_string(obj.get("type"))
+    if record_type is None:
+        return None, None, None, None
+
+    payload = obj.get("payload")
+
+    payload_type: str | None = None
+    payload_mapping = as_mapping(payload)
+    if payload_mapping is not None:
+        payload_type = as_string(payload_mapping.get("type"))
+
+    timestamp_raw = as_string(obj.get("timestamp"))
+
+    return record_type, payload_type, timestamp_raw, payload
+
+
+def _scan_jsonl_files_recursive(root_dir: Path) -> list[str]:
+    """Recursively list all .jsonl files under a directory."""
+    if not root_dir.exists():
+        return []
+    if not root_dir.is_dir():
+        raise RuntimeError(f"Codex session path is not a directory: '{root_dir}'.")
+
+    files: list[str] = []
+    stack: list[Path] = [root_dir]
+    while stack:
+        current = stack.pop()
+        try:
+            entries = list(os.scandir(current))
+        except OSError as exc:
+            raise RuntimeError(
+                f"Failed to scan Codex session directory '{current}'."
+            ) from exc
+        for entry in entries:
+            try:
+                is_dir = entry.is_dir(follow_symlinks=False)
+                is_file = entry.is_file(follow_symlinks=False)
+            except OSError as exc:
+                raise RuntimeError(
+                    f"Failed to inspect Codex session path '{entry.path}'."
+                ) from exc
+
+            if is_dir:
+                stack.append(Path(entry.path))
+            elif is_file and entry.name.lower().endswith(".jsonl"):
+                files.append(entry.path)
+    return files
+
+
+def _list_jsonl_files(
+    sessions_dir: Path,
+    archived_dir: Path,
+) -> list[SessionFileRef]:
+    """List all JSONL files from sessions and archived directories, deduped by path."""
+    unique: dict[str, SessionFileRef] = {}
+    for fp in _scan_jsonl_files_recursive(sessions_dir):
+        unique.setdefault(fp, SessionFileRef(file_path=fp, is_archived=False))
+    for fp in _scan_jsonl_files_recursive(archived_dir):
+        unique.setdefault(fp, SessionFileRef(file_path=fp, is_archived=True))
+    return [unique[k] for k in sorted(unique)]
+
+
+def _get_file_mtime(file_path: str) -> datetime | None:
+    """Return the file's modification time as UTC datetime."""
+    try:
+        stat_result = Path(file_path).stat()
+    except OSError as exc:
+        raise RuntimeError(f"Failed to stat Codex session file '{file_path}'.") from exc
+    return datetime.fromtimestamp(stat_result.st_mtime, tz=timezone.utc)
+
+
+def _parse_session_file(file_ref: SessionFileRef) -> ParsedSession:
+    """Parse a single JSONL session file.
+
+    Every discovered file is a session snapshot; unreadable files fail loudly.
+    """
+    try:
+        with open(file_ref.file_path, "r", encoding="utf-8") as handle:
+            raw_lines = handle.readlines()
+    except (OSError, UnicodeError) as exc:
+        raise RuntimeError(
+            f"Failed to read Codex session file '{file_ref.file_path}'."
+        ) from exc
+
+    parsed_lines: list[ParsedLine] = []
+
+    for idx, raw_line in enumerate(raw_lines):
+        stripped = raw_line.strip()
+        if not stripped:
+            continue
+
+        record_type, payload_type, timestamp_raw, payload = _parse_json_line(stripped)
+
+        parsed_lines.append(
+            ParsedLine(
+                line_index=idx,
+                record_type=record_type,
+                payload_type=payload_type,
+                timestamp_raw=timestamp_raw,
+                payload=payload,
+            )
+        )
+
+    return ParsedSession(
+        file_path=file_ref.file_path,
+        is_archived=file_ref.is_archived,
+        file_mtime=_get_file_mtime(file_ref.file_path),
+        lines=parsed_lines,
+    )
+
+
+def parse_session_snapshots(
+    sessions_dir: Path,
+    archived_dir: Path,
+) -> list[ParsedSession]:
+    """Scan and parse all Codex session files from both directories."""
+    return [
+        _parse_session_file(file_ref)
+        for file_ref in _list_jsonl_files(sessions_dir, archived_dir)
+    ]