docu-devs-api-client 1.12.0__tar.gz → 1.12.1__tar.gz

{docu_devs_api_client-1.12.0 → docu_devs_api_client-1.12.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: docu-devs-api-client
-Version: 1.12.0
+Version: 1.12.1
 Summary: A client library for accessing DocuDevs API
 Project-URL: Homepage, https://www.docudevs.ai/
 Project-URL: Repository, https://github.com/piizei/docudevs.git
@@ -79,7 +79,7 @@ evaluation = await case.holdout_eval_and_wait({"baseline": {}, "no memo": {"memo
 print(evaluation.report.summary())
 ```
 
-`api.records_case(case_id)` returns the same handle for an existing case without making a request. `classify_accuracy(results, truth)` scores classify results per tree level against a `{key: code}` mapping.
+`api.records_case(case_id)` returns the same handle for an existing case without making a request; the first classify call with plain rows reads the dataset's field mapping from the server. `case.get_record(key)` reads one record back, and `case.pending_actions()` returns `AgentAction` objects with `confirm()` and `reject()`. `classify_scores(results, truth, tree=nodes)` scores classify results per tree level against a `{key: code}` mapping as `LevelScore` counts (`1/2 (50%)`); `classify_accuracy` returns the same as fractions.
 
 ## Workbook normalization (v1)
 

{docu_devs_api_client-1.12.0 → docu_devs_api_client-1.12.1}/README.md

@@ -45,7 +45,7 @@ evaluation = await case.holdout_eval_and_wait({"baseline": {}, "no memo": {"memo
 print(evaluation.report.summary())
 ```
 
-`api.records_case(case_id)` returns the same handle for an existing case without making a request. `classify_accuracy(results, truth)` scores classify results per tree level against a `{key: code}` mapping.
+`api.records_case(case_id)` returns the same handle for an existing case without making a request; the first classify call with plain rows reads the dataset's field mapping from the server. `case.get_record(key)` reads one record back, and `case.pending_actions()` returns `AgentAction` objects with `confirm()` and `reject()`. `classify_scores(results, truth, tree=nodes)` scores classify results per tree level against a `{key: code}` mapping as `LevelScore` counts (`1/2 (50%)`); `classify_accuracy` returns the same as fractions.
 
 ## Workbook normalization (v1)
 

{docu_devs_api_client-1.12.0 → docu_devs_api_client-1.12.1}/docudevs/__init__.py

@@ -13,6 +13,7 @@ from .docudevs_client import (
 )
 from .pipeline import NodeRef, P, Pipeline, PipelinePath, and_, eq, exists, ne, not_, or_
 from .records_case import (
+    AgentAction,
     AgentRun,
     AgentRunFailed,
     ChosenCategory,
@@ -26,13 +27,16 @@ from .records_case import (
     EvalReport,
     ImportFailed,
     ImportJob,
+    LevelScore,
     MemoEntry,
+    Record,
     RecordsCase,
     RecordsImportFailed,
     TreeActivation,
     TreeActivationFailed,
     TreeVersion,
     classify_accuracy,
+    classify_scores,
 )
 from .schema import array_schema, json_schema
 
@@ -53,7 +57,10 @@ __all__ = (
     "ImportFailed",
     "RecordsImportFailed",
     "ClassifyJob",
+    "AgentAction",
     "ClassifyRow",
+    "LevelScore",
+    "Record",
     "ChosenCategory",
     "ClassifyFailed",
     "ClassifyJobFailed",
@@ -68,6 +75,7 @@ __all__ = (
     "TreeActivationFailed",
     "DocuDevsJobError",
     "classify_accuracy",
+    "classify_scores",
     "P",
     "NodeRef",
     "Pipeline",

{docu_devs_api_client-1.12.0 → docu_devs_api_client-1.12.1}/docudevs/records_case.py

@@ -80,6 +80,21 @@ def _predicted_at(
     return path[level - 1] if len(path) >= level else None
 
 
+@dataclass(frozen=True)
+class LevelScore:
+    """Correct answers out of the scored rows at one tree level."""
+
+    correct: int
+    total: int
+
+    @property
+    def accuracy(self) -> float:
+        return self.correct / self.total if self.total else 0.0
+
+    def __str__(self) -> str:
+        return f"{self.correct}/{self.total} ({self.accuracy:.0%})"
+
+
 def classify_accuracy(
     results: Iterable[Union["ClassifyRow", Mapping[str, Any]]],
     truth: Mapping[str, str],
@@ -87,14 +102,28 @@ def classify_accuracy(
     *,
     tree: Optional[Sequence[Mapping[str, Any]]] = None,
 ) -> Dict[int, float]:
-    """Accuracy per tree level against a truth mapping of record key to category code.
+    """Accuracy per tree level as a fraction; classify_scores has the same rules with counts."""
+    return {
+        level: score.accuracy
+        for level, score in classify_scores(results, truth, levels, tree=tree).items()
+    }
+
+
+def classify_scores(
+    results: Iterable[Union["ClassifyRow", Mapping[str, Any]]],
+    truth: Mapping[str, str],
+    levels: Optional[Sequence[int]] = None,
+    *,
+    tree: Optional[Sequence[Mapping[str, Any]]] = None,
+) -> Dict[int, LevelScore]:
+    """Correct and scored rows per tree level against a truth mapping of record key to code.
 
     The level-L ancestor of a code comes from tree (the node list given to upload_tree) when
     given, otherwise from the dotted code itself ("1.2.6" is under "1.2" and "1"). A row's
     answer at level L is its chosen.levels[L] when present, else the level-L ancestor of its
     chosen code. Declined rows, truth keys missing from the results, and answers shallower
     than L count as wrong. Level L is scored only over truth codes at least L levels deep;
-    levels with no such truth code are omitted. Returns {level: accuracy}, or {} for an
+    levels with no such truth code are omitted. Returns {level: LevelScore}, or {} for an
     empty truth mapping.
     """
     if not truth:
@@ -106,7 +135,7 @@ def classify_accuracy(
         rows.setdefault(str(raw.get("key")), raw)
     truth_paths = {key: lineage(code) for key, code in truth.items()}
     deepest = max(len(path) for path in truth_paths.values())
-    accuracy: Dict[int, float] = {}
+    scores: Dict[int, LevelScore] = {}
     for level in range(1, deepest + 1) if levels is None else levels:
         eligible = [key for key, path in truth_paths.items() if len(path) >= level]
         if not eligible:
@@ -116,8 +145,8 @@ def classify_accuracy(
             for key in eligible
             if _predicted_at(rows.get(key), level, lineage) == truth_paths[key][level - 1]
         )
-        accuracy[level] = correct / len(eligible)
-    return accuracy
+        scores[level] = LevelScore(correct, len(eligible))
+    return scores
 
 
 def _percent(value: Optional[float]) -> str:
@@ -547,6 +576,82 @@ class AgentRun(_Job):
         return self
 
 
+class Record:
+    """A record of a case as the server stores it."""
+
+    def __init__(self, raw: Mapping[str, Any]):
+        self.raw: Dict[str, Any] = dict(raw)
+
+    @property
+    def key(self) -> str:
+        return str(self.raw["key"])
+
+    @property
+    def label(self) -> Optional[str]:
+        return self.raw.get("label")
+
+    @property
+    def label_path(self) -> List[str]:
+        return list(self.raw.get("labelPath") or [])
+
+    @property
+    def label_source(self) -> Optional[str]:
+        return self.raw.get("labelSource")
+
+    @property
+    def text(self) -> Optional[str]:
+        return self.raw.get("chunk")
+
+    @property
+    def attributes(self) -> List[str]:
+        return list(self.raw.get("attributes") or [])
+
+
+class AgentAction:
+    """An action an agent proposed. It takes effect only when a person confirms it. Indexing
+    and get() read the raw response, as the dicts of earlier releases did."""
+
+    def __init__(self, client: Any, raw: Mapping[str, Any]):
+        self._client = client
+        self.raw: Dict[str, Any] = dict(raw)
+
+    @property
+    def id(self) -> int:
+        return int(self.raw["id"])
+
+    @property
+    def status(self) -> Optional[str]:
+        return self.raw.get("status")
+
+    @property
+    def summary(self) -> Optional[str]:
+        return self.raw.get("summary")
+
+    @property
+    def tool(self) -> Optional[str]:
+        return self.raw.get("tool")
+
+    @property
+    def run_id(self) -> Optional[str]:
+        return self.raw.get("runId")
+
+    def __getitem__(self, name: str) -> Any:
+        return self.raw[name]
+
+    def get(self, name: str, default: Any = None) -> Any:
+        return self.raw.get(name, default)
+
+    async def confirm(self) -> "AgentAction":
+        """Confirm the action as the signed-in person; the stored request is replayed."""
+        self.raw = {**self.raw, **await self._client.confirm_agent_action(self.id)}
+        return self
+
+    async def reject(self, *, reason: Optional[str] = None) -> "AgentAction":
+        """Reject the action, optionally with a reason."""
+        self.raw = {**self.raw, **await self._client.reject_agent_action(self.id, reason=reason)}
+        return self
+
+
 def _classify_settings(
     llm_level: Optional[str] = None,
     target_depth: Optional[Sequence[int]] = None,
@@ -658,8 +763,8 @@ def _present(values: Mapping[str, Any]) -> Dict[str, str]:
     return {name: str(value) for name, value in values.items() if value is not None and value != ""}
 
 
-def _inline_row(row: Mapping[str, Any], mapping: Optional[Mapping[str, Any]]) -> Dict[str, Any]:
-    if mapping is None or "text" in row:
+def _inline_row(row: Mapping[str, Any], mapping: Mapping[str, Any]) -> Dict[str, Any]:
+    if "text" in row:
         return dict(row)
     if mapping["key"] not in row:
         raise ValueError(f"row has no {mapping['key']!r} key field")
@@ -703,6 +808,33 @@ class RecordsCase:
                 job=self,
             )
 
+    async def get_record(self, key: str) -> Record:
+        """Read one record with its label, label source and indexed text."""
+        return Record(await self._client.get_record(self.id, key))
+
+    async def _inline_rows(self, rows: Iterable[Mapping[str, Any]]) -> List[Dict[str, Any]]:
+        rows = list(rows)
+        if all("text" in row for row in rows):
+            return [dict(row) for row in rows]
+        mapping = await self._dataset_field_mapping()
+        return [_inline_row(row, mapping) for row in rows]
+
+    async def _dataset_field_mapping(self) -> Dict[str, Any]:
+        if self._field_mapping is None:
+            dataset = await self._client.get_dataset(self.id)
+            mapping = (dataset or {}).get("fieldMapping")
+            if not mapping:
+                raise ValueError(
+                    f"Case {self.id} has no dataset field mapping, so plain rows cannot be "
+                    "converted; configure the dataset or pass rows with key and text"
+                )
+            self._field_mapping = {
+                "key": mapping["key"],
+                "text": list(mapping.get("text") or []),
+                "metadata": dict(mapping.get("metadata") or {}),
+            }
+        return self._field_mapping
+
     async def search(self, query: str, **settings: Any) -> Dict[str, Any]:
         """Search the records of the case."""
         result: Dict[str, Any] = await self._client.search_records(self.id, query, **settings)
@@ -784,7 +916,7 @@ class RecordsCase:
         already carry "text" are sent as given. Settings are snake_case; config holds any other
         classify setting and is merged under the keyword settings.
         """
-        inline = None if rows is None else [_inline_row(row, self._field_mapping) for row in rows]
+        inline = None if rows is None else await self._inline_rows(rows)
         settings = _classify_settings(llm_level, target_depth, max_tokens, concurrency)
         accepted = await self._client.classify(
             self.id, rows=inline, keys=keys, config=_merged_config(config, settings)
@@ -907,13 +1039,13 @@ class RecordsCase:
         )
         return await run.wait(timeout=timeout, poll_interval=poll_interval)
 
-    async def pending_actions(self) -> List[Dict[str, Any]]:
-        """Every agent action of the case that waits for a person to confirm it."""
-        actions: List[Dict[str, Any]] = []
+    async def pending_actions(self) -> List[AgentAction]:
+        """Every agent action of the case that waits for a person to confirm or reject it."""
+        actions: List[AgentAction] = []
         before = None
         while True:
             page = await self._client.list_agent_actions(self.id, status="pending", before=before)
-            actions.extend(page.get("items") or [])
+            actions.extend(AgentAction(self._client, item) for item in page.get("items") or [])
             before = page.get("nextBefore")
             if before is None:
                 return actions

{docu_devs_api_client-1.12.0 → docu_devs_api_client-1.12.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "docu-devs-api-client"
-version = "1.12.0"
+version = "1.12.1"
 description = "A client library for accessing DocuDevs API"
 authors = [{name = "pj@docudevs.ai"}]
 readme = "README.md"