sws-api-client 2.26.0__tar.gz → 3.1.0__tar.gz

{sws_api_client-2.26.0 → sws_api_client-3.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: sws_api_client
-Version: 2.26.0
+Version: 3.1.0
 Author-email: Matteo <matteo.terrinoni@fao.org>, Daniele <daniele.mansillo@fao.org>, Luca <luca.simi@fao.org>
 License-File: LICENSE
 Requires-Python: <4,>=3.10
@@ -16,6 +16,29 @@ Description-Content-Type: text/markdown
 
 This library provides the user with a set of useful tools to easily interact with the FAO SWS (Statistical Working System) REST APIs.
 
+## Campaign validation outputs
+
+Campaign files use top-level `visibility: shared | internal`, defaulting to shared. Pass
+`visibility="internal"` to `create_file` before uploading internal content. `CampaignFile` reads
+`visibility` and `validation_report_id`; `RunOutput` uses `visibility` instead of `audience`.
+The migration is `audience="both"` → `visibility="shared"`, and
+`audience="internal"` → `visibility="internal"`.
+
+`register_validation_output(team_id, report_id, source_file_id, descriptor)` registers an internal
+artifact of the exact report. `validation_outputs` lists it and `download_content_to_file` streams it.
+Every manual harvesting event names one file. Worker methods `claim_manual_harvest`,
+`heartbeat_manual_harvest`, `manual_harvest_child_task` and `complete_manual_harvest` use the file ID,
+request ID and wrapper task ID. Record the child immediately after launch; completion has one outcome.
+Statistical writes use the existing `Datasets.import_file_via_plugin` and
+`Datatables.import_file_via_plugin` methods to start separate Save Data tasks.
+
+The packaged report schema supports a DATA ZIP with root `data.csv` and optional observation
+`metadata.csv`, plus explicit separator/quote options. METADATA outputs are independent datatable
+inputs; separate report files are not implicitly paired.
+
+Pass `payload.cycleId` from `ValidationTriggered` to `complete_validation(cycle_id=...)`, including
+when the plugin fails. This lets CM reject an old run's completion after a validation restart.
+
 ## Installation
 
 The module is available on Pypi:

{sws_api_client-2.26.0 → sws_api_client-3.1.0}/README.md

@@ -2,6 +2,29 @@
 
 This library provides the user with a set of useful tools to easily interact with the FAO SWS (Statistical Working System) REST APIs.
 
+## Campaign validation outputs
+
+Campaign files use top-level `visibility: shared | internal`, defaulting to shared. Pass
+`visibility="internal"` to `create_file` before uploading internal content. `CampaignFile` reads
+`visibility` and `validation_report_id`; `RunOutput` uses `visibility` instead of `audience`.
+The migration is `audience="both"` → `visibility="shared"`, and
+`audience="internal"` → `visibility="internal"`.
+
+`register_validation_output(team_id, report_id, source_file_id, descriptor)` registers an internal
+artifact of the exact report. `validation_outputs` lists it and `download_content_to_file` streams it.
+Every manual harvesting event names one file. Worker methods `claim_manual_harvest`,
+`heartbeat_manual_harvest`, `manual_harvest_child_task` and `complete_manual_harvest` use the file ID,
+request ID and wrapper task ID. Record the child immediately after launch; completion has one outcome.
+Statistical writes use the existing `Datasets.import_file_via_plugin` and
+`Datatables.import_file_via_plugin` methods to start separate Save Data tasks.
+
+The packaged report schema supports a DATA ZIP with root `data.csv` and optional observation
+`metadata.csv`, plus explicit separator/quote options. METADATA outputs are independent datatable
+inputs; separate report files are not implicitly paired.
+
+Pass `payload.cycleId` from `ValidationTriggered` to `complete_validation(cycle_id=...)`, including
+when the plugin fails. This lets CM reject an old run's completion after a validation restart.
+
 ## Installation
 
 The module is available on Pypi:

{sws_api_client-2.26.0 → sws_api_client-3.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sws_api_client"
-version = "2.26.0"
+version = "3.1.0"
 description = ""
 authors = [
     { name = "Matteo", email = "matteo.terrinoni@fao.org" },
@@ -28,6 +28,7 @@ dev = [
     "pytest>=8.3.2,<9",
     "pytest-cov>=5.0.0,<6",
     "pytest-mock>=3.14.0,<4",
+    "jsonschema>=4,<5",
 ]
 
 [build-system]

{sws_api_client-2.26.0 → sws_api_client-3.1.0}/sws_api_client/__init__.py

@@ -58,4 +58,4 @@ from .tasks import PluginPayload, TaskDataset, TaskInfo, TaskManager, TaskRespon
 from .users import User, Users
 from .harvester import Harvester
 
-__version__ = "2.26.0"
+__version__ = "3.1.0"

{sws_api_client-2.26.0 → sws_api_client-3.1.0}/sws_api_client/campaign_manager.py

@@ -107,17 +107,17 @@ class RunOutput:
     Attributes:
         file_id (int | str): the file already registered and uploaded
         role (str): what it is within the run — ``report``, ``annotated``, ``log``, …
-        audience (str): ``internal`` keeps it on the campaign side; ``both`` lets the country read
+        visibility (str): ``internal`` keeps it on the campaign side; ``shared`` lets the country read
             it. UNMARKED MEANS READABLE, so say ``internal`` when it is: the day this is forgotten,
             a country seeing a log is a smaller failure than a country losing a document it is owed
     """
 
     file_id: Union[int, str]
     role: str = "report"
-    audience: str = "both"
+    visibility: str = "shared"
 
     def to_wire(self) -> Dict[str, Any]:
-        return {"fileId": self.file_id, "role": self.role, "audience": self.audience}
+        return {"fileId": self.file_id, "role": self.role, "visibility": self.visibility}
 
 
 @dataclass
@@ -187,6 +187,8 @@ class CampaignFile:
     stage: Optional[str] = None
     storage_uri: Optional[str] = None
     raw: Dict[str, Any] = field(default_factory=dict)
+    visibility: str = "shared"
+    validation_report_id: Optional[int] = None
 
     @classmethod
     def from_wire(cls, raw: Dict[str, Any]) -> "CampaignFile":
@@ -199,6 +201,8 @@ class CampaignFile:
             stage=raw.get("stage"),
             storage_uri=raw.get("storageUri"),
             raw=raw,
+            visibility=raw.get("visibility", "shared"),
+            validation_report_id=raw.get("validationReportId"),
         )
 
 
@@ -321,6 +325,7 @@ class CampaignManager:
         country: Optional[str] = None,
         questionnaire_type_id: Optional[Union[int, str]] = None,
         stage: Optional[str] = "original",
+        visibility: str = "shared",
     ) -> CampaignFile:
         """Register a file of the campaign — the row, before its bytes.
 
@@ -330,7 +335,9 @@ class CampaignManager:
         a new version of the same chain. So a generation run may be repeated without leaving a
         country's case showing the same questionnaire four times, and no plugin has to be idempotent.
         """
-        body: Dict[str, Any] = {"campaignId": campaign_id, "name": name, "type": type}
+        if visibility not in ("shared", "internal"):
+            raise ValueError("visibility must be shared or internal")
+        body: Dict[str, Any] = {"campaignId": campaign_id, "name": name, "type": type, "visibility": visibility}
         if country:
             body["country"] = country
         if questionnaire_type_id is not None:
@@ -340,6 +347,39 @@ class CampaignManager:
         raw = self._call("POST", "/api/v1/files", params={"teamId": team_id}, json_body=body)
         return CampaignFile.from_wire(raw)
 
+    def register_validation_output(
+        self, team_id: Union[int, str], report_id: Union[int, str],
+        source_file_id: Union[int, str], descriptor: Dict[str, Any],
+    ) -> CampaignFile:
+        """Register an internal artifact of an exact report, preserving its schema descriptor."""
+        raw = self._call("POST", f"/api/v1/files/{report_id}/outputs", params={"teamId": team_id},
+                         json_body={"sourceFileId": source_file_id, "descriptor": descriptor})
+        return CampaignFile.from_wire(raw)
+
+    def validation_outputs(self, team_id: Union[int, str], report_id: Union[int, str]) -> List[CampaignFile]:
+        """List internal artifacts of one report (including historical reports)."""
+        return [CampaignFile.from_wire(raw) for raw in self._call(
+            "GET", f"/api/v1/files/{report_id}/outputs", params={"teamId": team_id})]
+
+    def claim_manual_harvest(self, team_id, file_id, request_id, task_id, expires_at):
+        """Claim one file's request; duplicate deliveries receive HTTP 409."""
+        return self._call("POST", f"/api/v1/files/{file_id}/manual-harvest/processing", params={"teamId": team_id},
+                         json_body={"requestId": request_id, "taskId": task_id, "expiresAt": expires_at})
+
+    def heartbeat_manual_harvest(self, team_id, file_id, request_id, task_id, expires_at):
+        return self._call("PUT", f"/api/v1/files/{file_id}/manual-harvest/status", params={"teamId": team_id},
+                         json_body={"requestId": request_id, "taskId": task_id, "expiresAt": expires_at})
+
+    def manual_harvest_child_task(self, team_id, file_id, request_id, task_id, child_task_id, destination):
+        """Record the Save Data task before waiting for it, allowing recovery after wrapper failure."""
+        return self._call("POST", f"/api/v1/files/{file_id}/manual-harvest/child-task", params={"teamId": team_id},
+                         json_body={"requestId": request_id, "taskId": task_id, "childTaskId": child_task_id, "destination": destination})
+
+    def complete_manual_harvest(self, team_id, file_id, request_id, task_id, outcome, reason=None, result=None):
+        return self._call("POST", f"/api/v1/files/{file_id}/manual-harvest/complete", params={"teamId": team_id},
+                         json_body={"requestId": request_id, "taskId": task_id, "outcome": outcome,
+                                    "reason": reason, "result": result or {}})
+
     def upload_content(
         self,
         team_id: Union[int, str],
@@ -387,6 +427,20 @@ class CampaignManager:
         )
         return response.content
 
+    def download_content_to_file(self, team_id: Union[int, str], file_id: Union[int, str],
+                                 path: Union[str, Path]) -> Path:
+        """Stream a stored artifact to disk without materialising the whole file in memory."""
+        destination = Path(path)
+        with self.sws_client.discoverable.get(
+            self.endpoint, f"/api/v1/files/{file_id}/content", params={"teamId": team_id},
+            options={"raw_response": True, "raise_on_error": True}, stream=True,
+        ) as response:
+            with destination.open("wb") as out:
+                for chunk in response.iter_content(chunk_size=1024 * 1024):
+                    if chunk:
+                        out.write(chunk)
+        return destination
+
     # ── validation ────────────────────────────────────────────────────────────
 
     def claim_validation(
@@ -433,6 +487,7 @@ class CampaignManager:
         issues: Optional[List[ValidationIssue]] = None,
         reason: Optional[str] = None,
         outputs: Optional[List["RunOutput"]] = None,
+        cycle_id: Optional[str] = None,
     ) -> Dict[str, Any]:
         """How the check ended, with the figures and the issues a reader will see.
 
@@ -446,6 +501,10 @@ class CampaignManager:
 
         ``outputs`` is everything ELSE the run left behind — an annotated workbook, a log — each with
         its role and its reader. ``file_report_id`` remains the one a screen opens first.
+
+        Pass the trigger's ``payload.cycleId`` as ``cycle_id`` on both success and failure. CM
+        rejects a completion from an earlier validation after a restart. Omit it only for legacy
+        invocations that supplied no cycle identifier.
         """
         return self._call(
             "POST",
@@ -458,6 +517,7 @@ class CampaignManager:
                 "errors": [i.to_wire() for i in (issues or [])],
                 "reason": reason,
                 "outputs": [o.to_wire() for o in (outputs or [])] or None,
+                **({"cycleId": cycle_id} if cycle_id is not None else {}),
             },
         )
 

{sws_api_client-2.26.0 → sws_api_client-3.1.0}/sws_api_client/schemas/validation-report.schema.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "validation-schema-v5.5",
-  "$comment": "v5.5 = v5.4 with the corrections agreed with the Campaign Manager team, 2026-09-01. (1) Every $ref pointed at fao-validation-plugin-proposal-schema-v5.3, so nothing resolved: they now point into this document. (2) findings[].required was [class, code, location, severity, values]; a code is optional because it is a team's own finer statement, a location is absent when the finding is about something that is not there, and values are absent when it is about no value in particular (a count). It is now [class, severity]. (3) The version patterns read ^\\\\d+\\\\.\\\\d+[\\\\.\\\\d+]?$, whose character class rejected 1.2.3 and accepted 1.2+. (4) submission.file.name rejected spaces and accents, which real questionnaire filenames have. (5) questionnaire.reference_years.items had a minimum and no type. Still open, deliberately not changed here: objects are not closed (additionalProperties), so a misspelt field validates silently; and Output enumerates ten near-identical allOf branches where an if/then on filetype would say the same thing once. (6) Six more required fields a plugin cannot always know: root.outputs (a run that leaves nothing has none), questionnaire.language and .version, submission.submitted_by and .submitted_at (who sent it is the platform's fact, not the validator's), and file.size. (7) questionnaire.kind had minLength 3, which rejects AQ; file.id had minLength 8, which rejects the platform's own ids. A required field a producer must invent is a field that will be invented. (8) findings[].values items were typed [string, integer], which rejects null — the value a finding about a MISSING entry carries — and every decimal. Carrying bad data is this document's job, so they are unconstrained.",
+  "$comment": "v5.5 = v5.4 with the corrections agreed with the Campaign Manager team, 2026-09-01. (1) Every $ref pointed at fao-validation-plugin-proposal-schema-v5.3, so nothing resolved: they now point into this document. (2) findings[].required was [class, code, location, severity, values]; a code is optional because it is a team's own finer statement, a location is absent when the finding is about something that is not there, and values are absent when it is about no value in particular (a count). It is now [class, severity]. (3) The version patterns read ^\\\\d+\\\\.\\\\d+[\\\\.\\\\d+]?$, whose character class rejected 1.2.3 and accepted 1.2+. (4) submission.file.name rejected spaces and accents, which real questionnaire filenames have. (5) questionnaire.reference_years.items had a minimum and no type. Still open, deliberately not changed here: objects are not closed (additionalProperties), so a misspelt field validates silently; and Output enumerates ten near-identical allOf branches where an if/then on filetype would say the same thing once. (6) Six more required fields a plugin cannot always know: root.outputs (a run that leaves nothing has none), questionnaire.language and .version, submission.submitted_by and .submitted_at (who sent it is the platform's fact, not the validator's), and file.size. (7) questionnaire.kind had minLength 3, which rejects AQ; file.id had minLength 8, which rejects the platform's own ids. A required field a producer must invent is a field that will be invented. (8) findings[].values items were typed [string, integer], which rejects null \u2014 the value a finding about a MISSING entry carries \u2014 and every decimal. Carrying bad data is this document's job, so they are unconstrained.",
   "type": "object",
   "required": [
     "findings",
@@ -65,7 +65,7 @@
             }
           },
           "values": {
-            "description": "The values the finding is about: the figure out of range, the flag nobody recognises, null where something required is missing. Deliberately unconstrained — reporting that a value is invalid means carrying the invalid value, so a schema that rejects the document when the data is bad makes it impossible to say the data is bad.",
+            "description": "The values the finding is about: the figure out of range, the flag nobody recognises, null where something required is missing. Deliberately unconstrained \u2014 reporting that a value is invalid means carrying the invalid value, so a schema that rejects the document when the data is bad makes it impossible to say the data is bad.",
             "type": "array",
             "items": {
               "type": [
@@ -515,7 +515,7 @@
                       "type": "string"
                     },
                     "kind": {
-                      "$ref": "#/$defs/OutputTargetKind"
+                      "const": "DATATABLE"
                     }
                   }
                 }
@@ -821,7 +821,7 @@
                       "type": "string"
                     },
                     "kind": {
-                      "$ref": "#/$defs/OutputTargetKind"
+                      "const": "DATATABLE"
                     }
                   }
                 }
@@ -960,6 +960,65 @@
               }
             }
           ]
+        },
+        {
+          "type": "object",
+          "description": "One dataset import package: data.csv and optional observation metadata.csv at the archive root; CSV options apply to both entries.",
+          "required": [
+            "id",
+            "kind",
+            "filetype",
+            "role",
+            "target",
+            "separator",
+            "delimiter"
+          ],
+          "properties": {
+            "id": {
+              "type": "string"
+            },
+            "kind": {
+              "const": "FILE"
+            },
+            "filetype": {
+              "const": "ZIP"
+            },
+            "role": {
+              "const": "DATA"
+            },
+            "separator": {
+              "enum": [
+                "\t",
+                ",",
+                ";"
+              ]
+            },
+            "delimiter": {
+              "enum": [
+                "\"",
+                "'",
+                "`"
+              ]
+            },
+            "target": {
+              "type": "object",
+              "required": [
+                "kind",
+                "id"
+              ],
+              "properties": {
+                "kind": {
+                  "enum": [
+                    "DATASET",
+                    "SESSION"
+                  ]
+                },
+                "id": {
+                  "type": "string"
+                }
+              }
+            }
+          }
         }
       ]
     },

{sws_api_client-2.26.0 → sws_api_client-3.1.0}/sws_api_client/tags.py

@@ -138,6 +138,7 @@ class BaseDisseminatedTagTable(TypedDict):
     layer: TableLayer
     private: bool
     debug: bool
+    to_upload: bool
     type: TableType
     database: str | None
     table: str | None