ds-caselaw-marklogic-api-client 19.1.0__tar.gz → 21.0.0__tar.gz

{ds_caselaw_marklogic_api_client-19.1.0 → ds_caselaw_marklogic_api_client-21.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ds-caselaw-marklogic-api-client
-Version: 19.1.0
+Version: 21.0.0
 Summary: An API client for interacting with the underlying data in Find Caselaw.
 Home-page: https://github.com/nationalarchives/ds-caselaw-custom-api-client
 Keywords: national archives,caselaw

{ds_caselaw_marklogic_api_client-19.1.0 → ds_caselaw_marklogic_api_client-21.0.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ds-caselaw-marklogic-api-client"
-version = "19.1.0"
+version = "21.0.0"
 description = "An API client for interacting with the underlying data in Find Caselaw."
 authors = ["The National Archives"]
 homepage = "https://github.com/nationalarchives/ds-caselaw-custom-api-client"
@@ -32,6 +32,7 @@ coverage = "^7.2.3"
 pytest = "^7.3.2"
 responses = "^0.24.0"
 python-dotenv = "^1.0.0"
+time-machine = "^2.13.0"
 
 [tool.poetry.group.docs]
 optional = true

{ds_caselaw_marklogic_api_client-19.1.0 → ds_caselaw_marklogic_api_client-21.0.0}/src/caselawclient/Client.py

@@ -446,6 +446,24 @@ class MarklogicApiClient:
 
         return self._send_to_eval(vars, "set_metadata_court.xqy")
 
+    def set_document_jurisdiction(
+        self, document_uri: DocumentURIString, content: str
+    ) -> requests.Response:
+        uri = self._format_uri_for_marklogic(document_uri)
+        vars: query_dicts.SetMetadataJurisdictionDict = {"uri": uri, "content": content}
+        return self._send_to_eval(vars, "set_metadata_jurisdiction.xqy")
+
+    def set_document_court_and_jurisdiction(
+        self, document_uri: DocumentURIString, content: str
+    ) -> requests.Response:
+        if "/" in content:
+            court, jurisdiction = re.split("\\s*/\\s*", content)
+            self.set_document_court(document_uri, court)
+            return self.set_document_jurisdiction(document_uri, jurisdiction)
+        else:
+            self.set_document_court(document_uri, content)
+            return self.set_document_jurisdiction(document_uri, "")
+
     def set_judgment_this_uri(
         self, judgment_uri: DocumentURIString
     ) -> requests.Response:
@@ -952,12 +970,32 @@ class MarklogicApiClient:
 
         return results
 
+    def get_highest_enrichment_version(self) -> tuple[int, int]:
+        """This gets the highest enrichment version in the database,
+        so if nothing has been enriched with the most recent version of enrichment,
+        this won't reflect that change."""
+        table = json.loads(
+            get_single_string_from_marklogic_response(
+                self._send_to_eval(
+                    {},
+                    "get_highest_enrichment_version.xqy",
+                )
+            )
+        )
+
+        return (int(table[1][1]), int(table[1][2]))
+
     def get_pending_enrichment_for_version(
-        self, target_version: int
+        self,
+        target_enrichment_version: tuple[int, int],
+        target_parser_version: tuple[int, int],
     ) -> list[list[Any]]:
         """Retrieve documents which are not yet enriched with a given version."""
         vars: query_dicts.GetPendingEnrichmentForVersionDict = {
-            "target_version": target_version
+            "target_enrichment_major_version": target_enrichment_version[0],
+            "target_enrichment_minor_version": target_enrichment_version[1],
+            "target_parser_major_version": target_parser_version[0],
+            "target_parser_minor_version": target_parser_version[1],
         }
         results: list[list[Any]] = json.loads(
             get_single_string_from_marklogic_response(
@@ -969,3 +1007,35 @@ class MarklogicApiClient:
         )
 
         return results
+
+    def get_highest_parser_version(self) -> tuple[int, int]:
+        """This gets the highest parser version in the database, so if nothing has been parsed with the most recent version of the parser, this won't reflect that change."""
+        table = json.loads(
+            get_single_string_from_marklogic_response(
+                self._send_to_eval(
+                    {},
+                    "get_highest_parser_version.xqy",
+                )
+            )
+        )
+
+        return (int(table[1][1]), int(table[1][2]))
+
+    def get_pending_parse_for_version(
+        self, target_version: tuple[int, int]
+    ) -> list[list[Any]]:
+        """Retrieve documents which are not yet parsed with a given version."""
+        vars: query_dicts.GetPendingParseForVersionDict = {
+            "target_major_version": target_version[0],
+            "target_minor_version": target_version[1],
+        }
+        results: list[list[Any]] = json.loads(
+            get_single_string_from_marklogic_response(
+                self._send_to_eval(
+                    vars,
+                    "get_pending_parse_for_version.xqy",
+                )
+            )
+        )
+
+        return results

{ds_caselaw_marklogic_api_client-19.1.0 → ds_caselaw_marklogic_api_client-21.0.0}/src/caselawclient/models/documents.py

@@ -154,6 +154,10 @@ class Document:
             )
         )
 
+    def __repr__(self) -> str:
+        name = self.name or "un-named"
+        return f"<{self.document_noun} {self.uri}: {name}>"
+
     def document_exists(self) -> bool:
         """Helper method to verify the existence of a document within MarkLogic.
 
@@ -194,6 +198,20 @@ class Document:
             },
         )
 
+    @cached_property
+    def jurisdiction(self) -> str:
+        return self.xml.get_xpath_match_string(
+            "/akn:akomaNtoso/akn:*/akn:meta/akn:proprietary/uk:jurisdiction/text()",
+            {
+                "uk": "https://caselaw.nationalarchives.gov.uk/akn",
+                "akn": "http://docs.oasis-open.org/legaldocml/ns/akn/3.0",
+            },
+        )
+
+    @property
+    def court_and_jurisdiction(self) -> str:
+        return "/".join((self.court, self.jurisdiction))
+
     @cached_property
     def document_date_as_string(self) -> str:
         return self.xml.get_xpath_match_string(
@@ -460,9 +478,9 @@ class Document:
         """
         Request enrichment of the document
         """
-
+        now = datetime.datetime.now(datetime.timezone.utc)
         self.api_client.set_property(
-            self.uri, "last_sent_to_enrichment", datetime.datetime.now().isoformat()
+            self.uri, "last_sent_to_enrichment", now.isoformat()
         )
 
         announce_document_event(
@@ -532,6 +550,9 @@ class Document:
     def reparse(self) -> None:
         "Send an SNS notification that triggers reparsing, also sending all editor-modifiable metadata and URI"
 
+        now = datetime.datetime.now(datetime.timezone.utc)
+        self.api_client.set_property(self.uri, "last_sent_to_parser", now.isoformat())
+
         parser_type_noun = {"judgment": "judgment", "press summary": "pressSummary"}[
             self.document_noun
         ]
@@ -546,13 +567,14 @@ class Document:
         # values are "" from the API, we should pass None instead in this case.
 
         parser_instructions: ParserInstructionsDict = {
-            "name": self.name or None,
-            "cite": self.best_human_identifier or None,
-            "court": self.court or None,
-            "date": checked_date,
-            "uri": self.uri,
             "documentType": parser_type_noun,
-            "published": self.is_published,
+            "metadata": {
+                "name": self.name or None,
+                "cite": self.best_human_identifier or None,
+                "court": self.court or None,
+                "date": checked_date,
+                "uri": self.uri,
+            },
         }
 
         request_parse(

{ds_caselaw_marklogic_api_client-19.1.0 → ds_caselaw_marklogic_api_client-21.0.0}/src/caselawclient/models/neutral_citation_mixin.py

@@ -19,20 +19,21 @@ class NeutralCitationMixin:
     """
 
     def __init__(self, document_noun: str, *args: Any, **kwargs: Any) -> None:
-        self.attributes_to_validate: list[
-            tuple[str, bool, str]
-        ] = self.attributes_to_validate + [
-            (
-                "has_ncn",
-                True,
-                f"This {document_noun} has no neutral citation number",
-            ),
-            (
-                "has_valid_ncn",
-                True,
-                f"The neutral citation number of this {document_noun} is not valid",
-            ),
-        ]
+        self.attributes_to_validate: list[tuple[str, bool, str]] = (
+            self.attributes_to_validate
+            + [
+                (
+                    "has_ncn",
+                    True,
+                    f"This {document_noun} has no neutral citation number",
+                ),
+                (
+                    "has_valid_ncn",
+                    True,
+                    f"The neutral citation number of this {document_noun} is not valid",
+                ),
+            ]
+        )
 
         super(NeutralCitationMixin, self).__init__(*args, **kwargs)
 

{ds_caselaw_marklogic_api_client-19.1.0 → ds_caselaw_marklogic_api_client-21.0.0}/src/caselawclient/models/utilities/aws.py

@@ -16,24 +16,25 @@ from typing_extensions import NotRequired
 env = environ.Env()
 
 
+class ParserInstructionsMetadataDict(TypedDict):
+    name: Optional[str]
+    cite: Optional[str]
+    court: Optional[str]
+    date: Optional[str]
+    uri: Optional[str]
+
+
 class ParserInstructionsDict(TypedDict):
-    name: NotRequired[Optional[str]]
-    cite: NotRequired[Optional[str]]
-    court: NotRequired[Optional[str]]
-    date: NotRequired[Optional[str]]
-    uri: NotRequired[Optional[str]]
     documentType: NotRequired[Optional[str]]
-    published: NotRequired[bool]
+    metadata: NotRequired[ParserInstructionsMetadataDict]
 
 
 @overload
-def create_aws_client(service: Literal["s3"]) -> S3Client:
-    ...
+def create_aws_client(service: Literal["s3"]) -> S3Client: ...
 
 
 @overload
-def create_aws_client(service: Literal["sns"]) -> SNSClient:
-    ...
+def create_aws_client(service: Literal["sns"]) -> SNSClient: ...
 
 
 def create_aws_client(service: Union[Literal["s3"], Literal["sns"]]) -> Any:

ds_caselaw_marklogic_api_client-21.0.0/src/caselawclient/xquery/get_highest_enrichment_version.xqy

@@ -0,0 +1,9 @@
+xquery version "1.0-ml";
+
+xdmp:to-json(xdmp:sql(
+  "SELECT enrich_version_string, enrich_major_version, enrich_minor_version
+   FROM documents.process_data
+   ORDER BY enrich_major_version DESC, enrich_minor_version DESC
+   LIMIT 1",
+  "array"
+))

ds_caselaw_marklogic_api_client-21.0.0/src/caselawclient/xquery/get_highest_parser_version.xqy

@@ -0,0 +1,9 @@
+xquery version "1.0-ml";
+
+xdmp:to-json(xdmp:sql(
+  "SELECT parser_version_string, parser_major_version, parser_minor_version
+   FROM documents.process_data
+   ORDER BY parser_major_version DESC, parser_minor_version DESC
+   LIMIT 1",
+  "array"
+))

ds_caselaw_marklogic_api_client-21.0.0/src/caselawclient/xquery/get_pending_enrichment_for_version.xqy

@@ -0,0 +1,35 @@
+xquery version "1.0-ml";
+
+declare variable $target_enrichment_major_version as xs:int external;
+declare variable $target_enrichment_minor_version as xs:int external;
+declare variable $target_parser_major_version as xs:int external;
+declare variable $target_parser_minor_version as xs:int external;
+
+xdmp:to-json(xdmp:sql(
+  "SELECT process_data.uri, enrich_version_string, minutes_since_enrichment_request
+  FROM (
+    SELECT
+      process_data.uri,
+      enrich_version_string, enrich_major_version, enrich_minor_version,
+      parser_major_version, parser_minor_version,
+      DATEDIFF('minute', last_sent_to_enrichment, CURRENT_TIMESTAMP) AS minutes_since_enrichment_request
+    FROM documents.process_data
+    JOIN documents.process_property_data ON process_data.uri = process_property_data.uri
+  )
+  WHERE (
+    (enrich_version_string IS NULL) OR
+    (enrich_major_version <= @target_enrichment_major_version AND enrich_minor_version < @target_enrichment_minor_version)
+  ) AND (
+    (parser_major_version = @target_parser_major_version AND parser_minor_version = @target_parser_minor_version)
+  )
+  AND (minutes_since_enrichment_request > 43200 OR minutes_since_enrichment_request IS NULL)
+  ORDER BY enrich_major_version ASC NULLS FIRST, enrich_minor_version ASC",
+  "array",
+  map:new((
+    map:entry("target_enrichment_major_version", $target_enrichment_major_version),
+    map:entry("target_enrichment_minor_version", $target_enrichment_minor_version),
+    map:entry("target_parser_major_version", $target_parser_major_version),
+    map:entry("target_parser_minor_version", $target_parser_minor_version)
+  ))
+))
+

ds_caselaw_marklogic_api_client-21.0.0/src/caselawclient/xquery/get_pending_parse_for_version.xqy

@@ -0,0 +1,28 @@
+xquery version "1.0-ml";
+
+declare variable $target_major_version as xs:int external;
+declare variable $target_minor_version as xs:int external;
+
+xdmp:to-json(xdmp:sql(
+  "SELECT process_data.uri, parser_version_string, minutes_since_parse_request
+  FROM (
+    SELECT 
+      process_data.uri,
+      parser_version_string, parser_major_version, parser_minor_version,
+      DATEDIFF('minute', last_sent_to_parser, CURRENT_TIMESTAMP) AS minutes_since_parse_request
+    FROM documents.process_data
+    JOIN documents.process_property_data ON process_data.uri = process_property_data.uri
+  )
+  WHERE (
+    (parser_version_string IS NULL) OR
+    (parser_major_version <= @target_major_version AND parser_minor_version < @target_minor_version)
+  )
+  AND (minutes_since_parse_request > 43200 OR minutes_since_parse_request IS NULL)
+  ORDER BY parser_major_version ASC NULLS FIRST, parser_minor_version ASC",
+  "array",
+  map:new((
+    map:entry("target_major_version", $target_major_version),
+    map:entry("target_minor_version", $target_minor_version)
+  ))
+))
+

ds_caselaw_marklogic_api_client-21.0.0/src/caselawclient/xquery/set_metadata_jurisdiction.xqy

@@ -0,0 +1,37 @@
+xquery version "1.0-ml";
+
+declare namespace akn = "http://docs.oasis-open.org/legaldocml/ns/akn/3.0";
+declare namespace uk = "https://caselaw.nationalarchives.gov.uk/akn";
+
+declare variable $uri as xs:string external;
+declare variable $content as xs:string external;
+declare variable $proprietary-node := document($uri)/akn:akomaNtoso/akn:*/akn:meta/akn:proprietary;
+declare variable $jurisdiction-node := $proprietary-node/uk:jurisdiction;
+
+declare function local:delete($uri)
+{
+   xdmp:node-delete($jurisdiction-node)
+};
+
+declare function local:edit($uri, $content)
+{
+   xdmp:node-replace(
+      $jurisdiction-node,
+     <uk:jurisdiction>{$content}</uk:jurisdiction>
+   )
+};
+
+declare function local:add($uri, $content)
+{
+   xdmp:node-insert-child(
+     $proprietary-node,
+     <uk:jurisdiction>{$content}</uk:jurisdiction>
+   )
+};
+
+if (fn:boolean(
+cts:search(doc($uri),
+cts:element-query(xs:QName('uk:jurisdiction'),cts:and-query(()))))) then
+    if ($content = "") then local:delete($uri) else local:edit($uri, $content)
+else
+    local:add($uri, $content)

{ds_caselaw_marklogic_api_client-19.1.0 → ds_caselaw_marklogic_api_client-21.0.0}/src/caselawclient/xquery_type_dicts.py

@@ -80,7 +80,16 @@ class GetLastModifiedDict(MarkLogicAPIDict):
 
 # get_pending_enrichment_for_version.xqy
 class GetPendingEnrichmentForVersionDict(MarkLogicAPIDict):
-    target_version: int
+    target_enrichment_major_version: int
+    target_enrichment_minor_version: int
+    target_parser_major_version: int
+    target_parser_minor_version: int
+
+
+# get_pending_parse_for_version.xqy
+class GetPendingParseForVersionDict(MarkLogicAPIDict):
+    target_major_version: int
+    target_minor_version: int
 
 
 # get_properties_for_search_results.xqy
@@ -135,6 +144,12 @@ class SetMetadataCourtDict(MarkLogicAPIDict):
     uri: MarkLogicDocumentURIString
 
 
+# set_metadata_jurisdiction.xqy
+class SetMetadataJurisdictionDict(MarkLogicAPIDict):
+    content: str
+    uri: MarkLogicDocumentURIString
+
+
 # set_metadata_name.xqy
 class SetMetadataNameDict(MarkLogicAPIDict):
     content: str

ds_caselaw_marklogic_api_client-19.1.0/src/caselawclient/xquery/get_pending_enrichment_for_version.xqy

@@ -1,18 +0,0 @@
-xquery version "1.0-ml";
-
-declare variable $target_version as xs:int external;
-
-xdmp:to-json(xdmp:sql(
-  "SELECT process_data.uri, enrich_version_string, minutes_since_enrichment_request
-  FROM (
-    SELECT process_data.uri, enrich_version_string, enrich_major_version, DATEDIFF('minute', last_sent_to_enrichment, CURRENT_TIMESTAMP) AS minutes_since_enrichment_request
-    FROM documents.process_data
-    JOIN documents.process_property_data ON process_data.uri = process_property_data.uri
-  )
-  WHERE ((enrich_version_string IS NULL) OR (enrich_major_version < @target_version))
-  AND (minutes_since_enrichment_request > 43200 OR minutes_since_enrichment_request IS NULL)
-  ORDER BY enrich_major_version ASC NULLS FIRST",
-  "array",
-  map:new(map:entry("target_version", $target_version))
-))
-