kc-cli 0.4.0__py3-none-any.whl

kc/commands/task.py

@@ -0,0 +1,260 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Annotated
+
+import typer
+
+from kc.atomic_write import atomic_write_text
+from kc.commands.common import json_dumps, parse_input_json, run, validate_payload_schema
+from kc.config import load_config
+from kc.errors import KcError
+from kc.ids import new_id
+from kc.models.task import TaskRecord
+from kc.output import emit_success
+from kc.paths import current_paths
+from kc.search.fts import ensure_index, search_ranges
+from kc.store.sqlite import load_task, save_task
+from kc.store.transaction import mutation_transaction
+
+app = typer.Typer(help="Manage durable task records for external-agent workflows.")
+
+STATE_EVENTS = {
+    "awaiting_agent": "artifact_created",
+    "awaiting_validation": "artifact_validated",
+    "awaiting_apply": "artifact_applied",
+}
+
+
+def _event_schema(event: str | None) -> dict | None:
+    if event in {"artifact_created", "artifact_validated", "artifact_apply_dry_run", "artifact_applied"}:
+        return {
+            "type": "object",
+            "required": ["path"],
+            "properties": {"path": {"type": "string"}, "valid": {"type": "boolean"}},
+        }
+    if event and event.startswith("blocked_"):
+        return {
+            "type": "object",
+            "required": ["reason"],
+            "properties": {"reason": {"type": "string"}, "path": {"type": "string"}},
+        }
+    return None
+
+
+def _next_commands_for_status(task: TaskRecord) -> list[str]:
+    target = task.target_artifacts[0] if task.target_artifacts else "<artifact>"
+    if task.status == "awaiting_agent":
+        return [f"kc task resume --task-id {task.task_id} --event artifact_created --input @event.json"]
+    if task.status == "awaiting_validation":
+        return [
+            f"kc artifact validate --file {target}",
+            f"kc task resume --task-id {task.task_id} --event artifact_validated --input @event.json",
+        ]
+    if task.status == "awaiting_apply":
+        return [
+            f"kc artifact diff --file {target}",
+            f"kc artifact apply --file {target} --dry-run",
+            f"kc artifact apply --file {target} --yes",
+            f"kc task resume --task-id {task.task_id} --event artifact_applied --input @event.json",
+        ]
+    return []
+
+
+def _set_expected_event(task: TaskRecord) -> None:
+    task.expected_event_name = STATE_EVENTS.get(task.status)
+    task.expected_event_schema = _event_schema(task.expected_event_name)
+    task.next_commands = _next_commands_for_status(task)
+
+
+def _now() -> str:
+    return datetime.now(UTC).isoformat()
+
+
+@app.command("start", help="Create a task, gather candidate ranges, and optionally enter awaiting-agent state.")
+def start(
+    goal: Annotated[str, typer.Option("--goal", help="Knowledge-work goal.")],
+    shape: Annotated[
+        str, typer.Option("--shape", help="Expected artifact/answer shape.")
+    ] = "knowledge_page",
+    domain: Annotated[list[str] | None, typer.Option("--domain", help="Domain tag.")] = None,
+    target: Annotated[
+        list[str] | None, typer.Option("--target", help="Target artifact path.")
+    ] = None,
+    await_agent: Annotated[bool, typer.Option("--await-agent/--no-await-agent")] = True,
+) -> None:
+    def _run() -> None:
+        paths = current_paths()
+        timestamp = _now()
+        ensure_index(paths.sqlite_path, paths.sources_jsonl, paths.ranges_jsonl)
+        ranges = search_ranges(
+            paths.sqlite_path,
+            goal,
+            domain=(domain or [None])[0],
+            limit=20,
+        )
+        task = TaskRecord(
+            task_id=new_id("task"),
+            goal=goal,
+            status="awaiting_agent" if await_agent else "completed",
+            created_at=timestamp,
+            updated_at=timestamp,
+            shape=shape,
+            domain=list(domain or []),
+            candidate_sources=sorted({item["source_id"] for item in ranges}),
+            candidate_ranges=[item["range_id"] for item in ranges],
+            target_artifacts=list(target or []),
+            agent_instructions=[
+                "Read the candidate source ranges.",
+                "Create or update the target artifact yourself.",
+                "Do not add material claims without kc citation tokens.",
+                "Run kc artifact validate before apply.",
+            ],
+            next_commands=[
+                f"kc artifact validate --file {target[0] if target else '<artifact>'}",
+                f"kc artifact diff --file {target[0] if target else '<artifact>'}",
+                f"kc artifact apply --file {target[0] if target else '<artifact>'} --dry-run",
+            ],
+            expected_event_name="artifact_created" if await_agent else None,
+            expected_event_schema={
+                "type": "object",
+                "required": ["path"],
+                "properties": {"path": {"type": "string"}},
+            }
+            if await_agent
+            else None,
+        )
+        _set_expected_event(task)
+        with mutation_transaction(paths, "task.start", [paths.tasks_dir / f"{task.task_id}.json"]) as tx:
+            save_task(paths.sqlite_path, task)
+            paths.tasks_dir.mkdir(parents=True, exist_ok=True)
+            atomic_write_text(
+                paths.tasks_dir / f"{task.task_id}.json", json_dumps(task.model_dump(mode="json"))
+            )
+            tx.commit({"task_id": task.task_id})
+        config = load_config(paths.root)
+        exit_code = (
+            config.waiting_exit_code
+            if await_agent and config.enable_wait_exit_code
+            else 0
+        )
+        emit_success(
+            "task.start",
+            {
+                "task": task.model_dump(mode="json"),
+                "candidate_ranges": ranges,
+                "resume_command": (
+                    f"kc task resume --task-id {task.task_id} --event artifact_created --input @event.json"
+                    if await_agent
+                    else None
+                ),
+            },
+            exit_code=exit_code,
+        )
+
+    run("task.start", _run)
+
+
+@app.command("status", help="Show a compact task status and next commands.")
+def status(task_id: Annotated[str, typer.Option("--task-id", help="Task ID.")]) -> None:
+    def _run() -> None:
+        task = load_task(current_paths().sqlite_path, task_id)
+        if task is None:
+            raise KcError(code="KC_TASK_NOT_FOUND", message=f"Task not found: {task_id}")
+        emit_success(
+            "task.status",
+            {
+                "task_id": task.task_id,
+                "status": task.status,
+                "updated_at": task.updated_at,
+                "next_commands": task.next_commands,
+            },
+            target={"task_id": task_id},
+        )
+
+    run("task.status", _run)
+
+
+@app.command("next", help="Return state-specific next commands for a task.")
+def next_command(task_id: Annotated[str, typer.Option("--task-id", help="Task ID.")]) -> None:
+    def _run() -> None:
+        task = load_task(current_paths().sqlite_path, task_id)
+        if task is None:
+            raise KcError(code="KC_TASK_NOT_FOUND", message=f"Task not found: {task_id}")
+        _set_expected_event(task)
+        emit_success(
+            "task.next",
+            {
+                "task_id": task.task_id,
+                "status": task.status,
+                "expected_event_name": task.expected_event_name,
+                "expected_event_schema": task.expected_event_schema,
+                "next_commands": task.next_commands,
+            },
+            target={"task_id": task_id},
+        )
+
+    run("task.next", _run)
+
+
+@app.command("inspect", help="Show the full stored task record.")
+def inspect(task_id: Annotated[str, typer.Option("--task-id", help="Task ID.")]) -> None:
+    def _run() -> None:
+        task = load_task(current_paths().sqlite_path, task_id)
+        if task is None:
+            raise KcError(code="KC_TASK_NOT_FOUND", message=f"Task not found: {task_id}")
+        emit_success(
+            "task.inspect", {"task": task.model_dump(mode="json")}, target={"task_id": task_id}
+        )
+
+    run("task.inspect", _run)
+
+
+@app.command("resume", help="Resume an awaiting task with a structured event payload.")
+def resume(
+    task_id: Annotated[str, typer.Option("--task-id", help="Task ID.")],
+    event: Annotated[str, typer.Option("--event", help="Event name.")],
+    input_data: Annotated[str, typer.Option("--input", help="Inline JSON or @file.")],
+) -> None:
+    def _run() -> None:
+        paths = current_paths()
+        task = load_task(paths.sqlite_path, task_id)
+        if task is None:
+            raise KcError(code="KC_TASK_NOT_FOUND", message=f"Task not found: {task_id}")
+        if task.status not in {"awaiting_agent", "awaiting_validation", "awaiting_apply"}:
+            raise KcError(
+                code="KC_TASK_NOT_WAITING",
+                message=f"Task is not awaiting agent input: {task_id}",
+                details={"status": task.status},
+            )
+        if event != task.expected_event_name and event not in {"blocked_missing_source", "blocked_validation_failed"}:
+            raise KcError(
+                code="KC_EVENT_INVALID",
+                message=f"Expected event {task.expected_event_name}, got {event}",
+            )
+        payload = parse_input_json(input_data)
+        validate_payload_schema(payload, _event_schema(event) if event.startswith("blocked_") else task.expected_event_schema)
+        task.events.append({"event": event, "input": payload, "received_at": _now()})
+        if event == "artifact_created":
+            task.status = "awaiting_validation"
+        elif event in {"artifact_validated", "artifact_apply_dry_run"}:
+            task.status = "awaiting_apply"
+        elif event == "artifact_applied":
+            task.status = "completed"
+        elif event in {"blocked_missing_source", "blocked_validation_failed"}:
+            task.status = "blocked"
+        else:
+            task.status = "completed"
+        task.updated_at = _now()
+        _set_expected_event(task)
+        with mutation_transaction(paths, "task.resume", [paths.tasks_dir / f"{task.task_id}.json"]) as tx:
+            save_task(paths.sqlite_path, task)
+            atomic_write_text(
+                paths.tasks_dir / f"{task.task_id}.json", json_dumps(task.model_dump(mode="json"))
+            )
+            tx.commit({"task_id": task.task_id, "event": event, "status": task.status})
+        emit_success(
+            "task.resume", {"task": task.model_dump(mode="json")}, target={"task_id": task_id}
+        )
+
+    run("task.resume", _run)

kc/config.py

@@ -0,0 +1,127 @@
+"""kc.toml defaults and parsing."""
+
+from __future__ import annotations
+
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+
+from kc.errors import KcError
+
+DEFAULT_CONFIG = """schema_version = "kc.config.v1"
+project_id = "kc-project"
+data_dir = "knowledge"
+state_dir = ".kc"
+
+[output]
+default_format = "json"
+human_format = "table"
+llm_env_var = "LLM"
+
+[source_policy]
+copy_sources = false
+require_fingerprint = true
+require_locator = true
+allow_unregistered_citations = false
+
+[citation_policy]
+required_for_material_claims = true
+citation_token_pattern = "kc_v1"
+fail_on_stale_source_fingerprint = true
+
+[index]
+fts_enabled = true
+rrf_k = 60
+
+[index.semantic]
+provider = "model2vec"
+model = "potion-base-8M"
+dimension = 256
+checksum = "sha256:aef1c5e1fd70060804f5295ec8e9ab3ed62e50e79b208435fb77e15c5bf94bb8"
+purpose = "ranking_only"
+
+[mutation]
+default_dry_run = true
+require_yes_for_apply = true
+atomic_writes = true
+create_snapshots = true
+require_idempotency_key_for_apply = false
+update_log = true
+allow_skip_validate_in_llm = false
+
+[task]
+enable_wait_exit_code = false
+waiting_exit_code = 40
+"""
+
+
+@dataclass(frozen=True)
+class KcConfig:
+    schema_version: str = "kc.config.v1"
+    project_id: str = "kc-project"
+    data_dir: str = "knowledge"
+    state_dir: str = ".kc"
+    raw: dict | None = None
+
+    @property
+    def fail_on_stale_source_fingerprint(self) -> bool:
+        return bool(
+            (self.raw or {})
+            .get("citation_policy", {})
+            .get("fail_on_stale_source_fingerprint", True)
+        )
+
+    @property
+    def update_log(self) -> bool:
+        return bool((self.raw or {}).get("mutation", {}).get("update_log", True))
+
+    @property
+    def allow_skip_validate_in_llm(self) -> bool:
+        return bool((self.raw or {}).get("mutation", {}).get("allow_skip_validate_in_llm", False))
+
+    @property
+    def enable_wait_exit_code(self) -> bool:
+        return bool((self.raw or {}).get("task", {}).get("enable_wait_exit_code", False))
+
+    @property
+    def waiting_exit_code(self) -> int:
+        return int((self.raw or {}).get("task", {}).get("waiting_exit_code", 40))
+
+    @property
+    def rrf_k(self) -> int:
+        return int((self.raw or {}).get("index", {}).get("rrf_k", 60))
+
+
+def load_config(root: Path | None = None, *, required: bool = False) -> KcConfig:
+    root = root or Path.cwd()
+    path = root / "kc.toml"
+    if not path.exists():
+        if required:
+            raise KcError(
+                code="KC_CONFIG_NOT_FOUND",
+                message="kc.toml not found. Run kc init --yes first.",
+                details={"path": str(path)},
+            )
+        return KcConfig(raw={})
+    try:
+        data = tomllib.loads(path.read_text(encoding="utf-8"))
+    except tomllib.TOMLDecodeError as exc:
+        raise KcError(
+            code="KC_CONFIG_INVALID",
+            message=f"Invalid kc.toml: {exc}",
+            details={"path": str(path)},
+        ) from exc
+    schema_version = str(data.get("schema_version", "kc.config.v1"))
+    if schema_version != "kc.config.v1":
+        raise KcError(
+            code="KC_CONFIG_INVALID",
+            message=f"Unsupported kc.toml schema_version: {schema_version}",
+            details={"path": str(path), "schema_version": schema_version, "supported": ["kc.config.v1"]},
+        )
+    return KcConfig(
+        schema_version=schema_version,
+        project_id=str(data.get("project_id", "kc-project")),
+        data_dir=str(data.get("data_dir", "knowledge")),
+        state_dir=str(data.get("state_dir", ".kc")),
+        raw=data,
+    )

kc/embedding_models/potion-base-8M/README.md

@@ -0,0 +1,97 @@
+---
+library_name: model2vec
+license: mit
+model_name: potion-base-8M
+tags:
+- embeddings
+- static-embeddings
+- sentence-transformers
+---
+
+# potion-base-8M Model Card
+
+This [Model2Vec](https://github.com/MinishLab/model2vec) model is a distilled version of a Sentence Transformer. It uses static embeddings, allowing text embeddings to be computed orders of magnitude faster on both GPU and CPU. It is designed for applications where computational resources are limited or where real-time performance is critical. Model2Vec models are the smallest, fastest, and most performant static embedders available. The distilled models are up to 50 times smaller and 500 times faster than traditional Sentence Transformers.
+
+
+## Installation
+
+Install model2vec using pip:
+```
+pip install model2vec
+```
+
+## Usage
+
+### Using Model2Vec
+
+The [Model2Vec library](https://github.com/MinishLab/model2vec) is the fastest and most lightweight way to run Model2Vec models.
+
+Load this model using the `from_pretrained` method:
+```python
+from model2vec import StaticModel
+
+# Load a pretrained Model2Vec model
+model = StaticModel.from_pretrained("potion-base-8M")
+
+# Compute text embeddings
+embeddings = model.encode(["Example sentence"])
+```
+
+### Using Sentence Transformers
+
+You can also use the [Sentence Transformers library](https://github.com/UKPLab/sentence-transformers) to load and use the model:
+
+```python
+from sentence_transformers import SentenceTransformer
+
+# Load a pretrained Sentence Transformer model
+model = SentenceTransformer("potion-base-8M")
+
+# Compute text embeddings
+embeddings = model.encode(["Example sentence"])
+```
+
+### Distilling a Model2Vec model
+
+You can distill a Model2Vec model from a Sentence Transformer model using the `distill` method. First, install the `distill` extra with `pip install model2vec[distill]`. Then, run the following code:
+
+```python
+from model2vec.distill import distill
+
+# Distill a Sentence Transformer model, in this case the BAAI/bge-base-en-v1.5 model
+m2v_model = distill(model_name="BAAI/bge-base-en-v1.5", pca_dims=256)
+
+# Save the model
+m2v_model.save_pretrained("m2v_model")
+```
+
+## How it works
+
+Model2vec creates a small, fast, and powerful model that outperforms other static embedding models by a large margin on all tasks we could find, while being much faster to create than traditional static embedding models such as GloVe. Best of all, you don't need any data to distill a model using Model2Vec.
+
+It works by passing a vocabulary through a sentence transformer model, then reducing the dimensionality of the resulting embeddings using PCA, and finally weighting the embeddings using [SIF weighting](https://openreview.net/pdf?id=SyK00v5xx). During inference, we simply take the mean of all token embeddings occurring in a sentence.
+
+## Additional Resources
+
+- [Model2Vec Repo](https://github.com/MinishLab/model2vec)
+- [Model2Vec Base Models](https://huggingface.co/collections/minishlab/model2vec-base-models-66fd9dd9b7c3b3c0f25ca90e)
+- [Model2Vec Results](https://github.com/MinishLab/model2vec/tree/main/results)
+- [Model2Vec Tutorials](https://github.com/MinishLab/model2vec/tree/main/tutorials)
+- [Website](https://minishlab.github.io/)
+
+
+## Library Authors
+
+Model2Vec was developed by the [Minish Lab](https://github.com/MinishLab) team consisting of [Stephan Tulkens](https://github.com/stephantul) and [Thomas van Dongen](https://github.com/Pringled).
+
+## Citation
+
+Please cite the [Model2Vec repository](https://github.com/MinishLab/model2vec) if you use this model in your work.
+```
+@article{minishlab2024model2vec,
+  author = {Tulkens, Stephan and {van Dongen}, Thomas},
+  title = {Model2Vec: Fast State-of-the-Art Static Embeddings},
+  year = {2024},
+  url = {https://github.com/MinishLab/model2vec}
+}
+```

kc/embedding_models/potion-base-8M/config.json

@@ -0,0 +1,13 @@
+{
+    "model_type": "model2vec",
+    "architectures": [
+        "StaticModel"
+    ],
+    "tokenizer_name": "baai/bge-base-en-v1.5",
+    "apply_pca": 256,
+    "apply_zipf": true,
+    "hidden_dim": 256,
+    "seq_length": 1000000,
+    "normalize": true,
+    "embedding_dtype": "float32"
+}

kc/embedding_models/potion-base-8M/modules.json

@@ -0,0 +1,14 @@
+[
+    {
+        "idx": 0,
+        "name": "0",
+        "path": ".",
+        "type": "sentence_transformers.models.StaticEmbedding"
+    },
+    {
+        "idx": 1,
+        "name": "1",
+        "path": "1_Normalize",
+        "type": "sentence_transformers.models.Normalize"
+    }
+]