ds-caselaw-marklogic-api-client 17.1.0__tar.gz → 17.3.0__tar.gz

{ds_caselaw_marklogic_api_client-17.1.0 → ds_caselaw_marklogic_api_client-17.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ds-caselaw-marklogic-api-client
-Version: 17.1.0
+Version: 17.3.0
 Summary: An API client for interacting with the underlying data in Find Caselaw.
 Home-page: https://github.com/nationalarchives/ds-caselaw-custom-api-client
 Keywords: national archives,caselaw
@@ -14,7 +14,7 @@ Requires-Dist: boto3 (>=1.26.112,<2.0.0)
 Requires-Dist: certifi (>=2022.12.7,<2024.0.0)
 Requires-Dist: charset-normalizer (>=3.0.0,<4.0.0)
 Requires-Dist: django-environ (>=0.11.0,<0.12.0)
-Requires-Dist: ds-caselaw-utils (>=1.0.0,<2.0.0)
+Requires-Dist: ds-caselaw-utils (>=1.3.0,<2.0.0)
 Requires-Dist: idna (>=3.4,<4.0)
 Requires-Dist: lxml (>=4.9.2,<5.0.0)
 Requires-Dist: memoization (>=0.4.0,<0.5.0)

{ds_caselaw_marklogic_api_client-17.1.0 → ds_caselaw_marklogic_api_client-17.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ds-caselaw-marklogic-api-client"
-version = "17.1.0"
+version = "17.3.0"
 description = "An API client for interacting with the underlying data in Find Caselaw."
 authors = ["The National Archives"]
 homepage = "https://github.com/nationalarchives/ds-caselaw-custom-api-client"
@@ -21,7 +21,7 @@ requests-toolbelt = ">=0.10.1,<1.1.0"
 urllib3 = "^1.26.15"
 memoization = "^0.4.0"
 lxml = "^4.9.2"
-ds-caselaw-utils = "^1.0.0"
+ds-caselaw-utils = "^1.3.0"
 boto3 = "^1.26.112"
 typing-extensions = "^4.7.1"
 mypy-boto3-s3 = "^1.26.104"
@@ -30,7 +30,7 @@ mypy-boto3-sns = "^1.26.69"
 [tool.poetry.group.dev.dependencies]
 coverage = "^7.2.3"
 pytest = "^7.3.2"
-responses = "^0.23.1"
+responses = "^0.24.0"
 python-dotenv = "^1.0.0"
 
 [tool.poetry.group.docs]
@@ -47,3 +47,5 @@ build-backend = "poetry.core.masonry.api"
 markers = [
     "write: the test deliberately changes the Marklogic DB')",
 ]
+
+filterwarnings = ["ignore::DeprecationWarning"]

{ds_caselaw_marklogic_api_client-17.1.0 → ds_caselaw_marklogic_api_client-17.3.0}/src/caselawclient/Client.py

@@ -205,7 +205,9 @@ class MarklogicApiClient:
         self.session.headers.update({"User-Agent": user_agent})
         self.user_agent = user_agent
 
-    def get_document_by_uri(self, uri: DocumentURIString) -> Document:
+    def get_document_by_uri(
+        self, uri: DocumentURIString, query: Optional[str] = None
+    ) -> Document:
         document_type_class = self.get_document_type_from_uri(uri)
         return document_type_class(uri, self)
 
@@ -684,6 +686,7 @@ class MarklogicApiClient:
         version_uri: Optional[DocumentURIString] = None,
         show_unpublished: bool = False,
         xsl_filename: str = DEFAULT_XSL_TRANSFORM,
+        query: Optional[str] = None,
     ) -> requests.Response:
         marklogic_document_uri = self._format_uri_for_marklogic(judgment_uri)
         marklogic_document_version_uri = (
@@ -707,6 +710,7 @@ class MarklogicApiClient:
             "show_unpublished": show_unpublished,
             "img_location": image_location,
             "xsl_filename": xsl_filename,
+            "query": query,
         }
 
         return self._send_to_eval(vars, "xslt_transform.xqy")

{ds_caselaw_marklogic_api_client-17.1.0 → ds_caselaw_marklogic_api_client-17.3.0}/src/caselawclient/models/documents.py

@@ -6,23 +6,27 @@ from typing import TYPE_CHECKING, Any, Dict, NewType, Optional
 from ds_caselaw_utils import courts
 from ds_caselaw_utils.courts import CourtNotFoundException
 from lxml import etree
+from lxml import html as html_parser
 from requests_toolbelt.multipart import decoder
 
 from caselawclient.models.utilities import extract_version
 
 from ..errors import (
     DocumentNotFoundError,
+    GatewayTimeoutError,
     NotSupportedOnVersion,
     OnlySupportedOnVersion,
 )
 from ..xml_helpers import get_xpath_match_string, get_xpath_match_strings
 from .utilities import VersionsDict, get_judgment_root, render_versions
 from .utilities.aws import (
+    ParserInstructionsDict,
+    announce_document_event,
     delete_documents_from_private_bucket,
     generate_docx_url,
     generate_pdf_url,
-    notify_changed,
     publish_documents,
+    request_parse,
     unpublish_documents,
     uri_for_s3,
 )
@@ -32,6 +36,10 @@ class UnparsableDate(Warning):
     pass
 
 
+class GatewayTimeoutGettingHTMLWithQuery(RuntimeWarning):
+    pass
+
+
 DOCUMENT_STATUS_HOLD = "On hold"
 """ This document has been placed on hold to actively prevent publication. """
 
@@ -328,12 +336,36 @@ class Document:
     def content_as_xml_tree(self) -> Any:
         return etree.fromstring(self.content_as_xml_bytestring)
 
-    def content_as_html(self, version_uri: Optional[DocumentURIString] = None) -> str:
-        results = self.api_client.eval_xslt(
-            self.uri, version_uri, show_unpublished=True
-        )
-        multipart_data = decoder.MultipartDecoder.from_response(results)
-        return str(multipart_data.parts[0].text)
+    def content_as_html(
+        self,
+        version_uri: Optional[DocumentURIString] = None,
+        query: Optional[str] = None,
+    ) -> str:
+        try:
+            results = self.api_client.eval_xslt(
+                self.uri, version_uri, show_unpublished=True, query=query
+            )
+            multipart_data = decoder.MultipartDecoder.from_response(results)
+            return str(multipart_data.parts[0].text)
+        except GatewayTimeoutError as e:
+            if query is not None:
+                warnings.warn(
+                    (
+                        "Gateway timeout when getting content with query"
+                        "highlighting for document %s, version %s, and query"
+                        '"%s", falling back to unhighlighted content...'
+                    )
+                    % (self.uri, version_uri, query),
+                    GatewayTimeoutGettingHTMLWithQuery,
+                )
+                return self.content_as_html(version_uri)
+            else:
+                raise e
+
+    def number_of_mentions(self, query: str) -> int:
+        html = self.content_as_html(query=query)
+        tree = html_parser.fromstring(html.encode("utf-8"))
+        return len(tree.findall(".//mark"))
 
     @cached_property
     def is_failure(self) -> bool:
@@ -418,9 +450,12 @@ class Document:
         return DOCUMENT_STATUS_NEW
 
     def enrich(self) -> None:
-        notify_changed(
+        """
+        Announces to the ANNOUNCE SNS that the document is waiting to be enriched.
+        """
+        announce_document_event(
             uri=self.uri,
-            status="published",
+            status="enrich",
             enrich=True,
         )
 
@@ -434,20 +469,19 @@ class Document:
 
         publish_documents(uri_for_s3(self.uri))
         self.api_client.set_published(self.uri, True)
-        notify_changed(
+        announce_document_event(
             uri=self.uri,
-            status="published",
-            enrich=True,
+            status="publish",
         )
+        self.enrich()
 
     def unpublish(self) -> None:
         self.api_client.break_checkout(self.uri)
         unpublish_documents(uri_for_s3(self.uri))
         self.api_client.set_published(self.uri, False)
-        notify_changed(
+        announce_document_event(
             uri=self.uri,
-            status="not published",
-            enrich=False,
+            status="unpublish",
         )
 
     def hold(self) -> None:
@@ -490,3 +524,35 @@ class Document:
 
     def move(self, new_citation: str) -> None:
         self.api_client.update_document_uri(self.uri, new_citation)
+
+    def reparse(self) -> None:
+        "Send an SNS notification that triggers reparsing, also sending all editor-modifiable metadata and URI"
+
+        parser_type_noun = {"judgment": "judgment", "press summary": "pressSummary"}[
+            self.document_noun
+        ]
+        checked_date = (
+            self.document_date_as_string
+            if self.document_date_as_string > "1001"
+            else None
+        )
+
+        # the keys of parser_instructions should exactly match the parser output
+        # in the *-metadata.json files by the parser. Whilst typically empty
+        # values are "" from the API, we should pass None instead in this case.
+
+        parser_instructions: ParserInstructionsDict = {
+            "name": self.name or None,
+            "cite": self.best_human_identifier or None,
+            "court": self.court or None,
+            "date": checked_date,
+            "uri": self.uri,
+            "documentType": parser_type_noun,
+            "published": self.is_published,
+        }
+
+        request_parse(
+            uri=self.uri,
+            reference=self.consignment_reference,
+            parser_instructions=parser_instructions,
+        )

{ds_caselaw_marklogic_api_client-17.1.0 → ds_caselaw_marklogic_api_client-17.3.0}/src/caselawclient/models/utilities/aws.py

@@ -1,6 +1,8 @@
+import datetime
 import json
 import logging
-from typing import Any, Literal, Union, overload
+import uuid
+from typing import Any, Literal, Optional, TypedDict, Union, overload
 
 import boto3
 import botocore.client
@@ -9,10 +11,21 @@ from mypy_boto3_s3.client import S3Client
 from mypy_boto3_s3.type_defs import CopySourceTypeDef, ObjectIdentifierTypeDef
 from mypy_boto3_sns.client import SNSClient
 from mypy_boto3_sns.type_defs import MessageAttributeValueTypeDef
+from typing_extensions import NotRequired
 
 env = environ.Env()
 
 
+class ParserInstructionsDict(TypedDict):
+    name: NotRequired[Optional[str]]
+    cite: NotRequired[Optional[str]]
+    court: NotRequired[Optional[str]]
+    date: NotRequired[Optional[str]]
+    uri: NotRequired[Optional[str]]
+    documentType: NotRequired[Optional[str]]
+    published: NotRequired[bool]
+
+
 @overload
 def create_aws_client(service: Literal["s3"]) -> S3Client:
     ...
@@ -64,10 +77,14 @@ def generate_signed_asset_url(key: str) -> str:
     )
 
 
-def generate_docx_url(uri: str) -> str:
-    key = f'{uri}/{uri.replace("/", "_")}.docx'
+def generate_docx_key(uri: str) -> str:
+    """from a canonical caselaw URI (eat/2022/1) return the S3 key of the associated docx"""
+    return f'{uri}/{uri.replace("/", "_")}.docx'
 
-    return generate_signed_asset_url(key)
+
+def generate_docx_url(uri: str) -> str:
+    """from a canonical caselaw URI (eat/2022/1) return a signed S3 link for the front end"""
+    return generate_signed_asset_url(generate_docx_key(uri))
 
 
 def generate_pdf_url(uri: str) -> str:
@@ -110,7 +127,7 @@ def publish_documents(uri: str) -> None:
 
         if not key.endswith("parser.log") and not key.endswith(".tar.gz"):
             source: CopySourceTypeDef = {"Bucket": private_bucket, "Key": key}
-            extra_args = {"ACL": "public-read"}
+            extra_args: dict[str, str] = {}
             try:
                 client.copy(source, public_bucket, key, extra_args)
             except botocore.client.ClientError as e:
@@ -127,7 +144,7 @@ def delete_documents_from_private_bucket(uri: str) -> None:
     delete_from_bucket(uri, env("PRIVATE_ASSET_BUCKET"))
 
 
-def notify_changed(uri: str, status: str, enrich: bool = False) -> None:
+def announce_document_event(uri: str, status: str, enrich: bool = False) -> None:
     client = create_sns_client()
 
     message_attributes: dict[str, MessageAttributeValueTypeDef] = {}
@@ -146,7 +163,7 @@ def notify_changed(uri: str, status: str, enrich: bool = False) -> None:
         }
 
     client.publish(
-        TopicArn=env("SNS_TOPIC"),
+        TopicArn=env("SNS_TOPIC"),  # this is the ANNOUNCE SNS topic
         Message=json.dumps({"uri_reference": uri, "status": status}),
         Subject=f"Updated: {uri} {status}",
         MessageAttributes=message_attributes,
@@ -189,3 +206,40 @@ def build_new_key(old_key: str, new_uri: str) -> str:
         return f"{new_uri}/{new_filename}.{old_filename.split('.')[-1]}"
     else:
         return f"{new_uri}/{old_filename}"
+
+
+def request_parse(
+    uri: str,
+    reference: Optional[str],
+    parser_instructions: Optional[ParserInstructionsDict] = None,
+) -> None:
+    client = create_sns_client()
+
+    if parser_instructions is None:
+        parser_instructions = ParserInstructionsDict({})
+
+    message_to_send = {
+        "properties": {
+            "messageType": "uk.gov.nationalarchives.da.messages.request.courtdocument.parse.RequestCourtDocumentParse",
+            "timestamp": datetime.datetime.now(datetime.timezone.utc)
+            .isoformat()
+            .replace("+00:00", "Z"),
+            "function": "fcl-judgment-parse-request",
+            "producer": "FCL",
+            "executionId": f"fcl_ex_id_{uuid.uuid4()}",
+            "parentExecutionId": None,
+        },
+        "parameters": {
+            "s3Bucket": env("PRIVATE_ASSET_BUCKET"),
+            "s3Key": generate_docx_key(uri),
+            "reference": reference or f"FCL-{uuid.uuid4()}",
+            "originator": "FCL",
+            "parserInstructions": parser_instructions,
+        },
+    }
+
+    client.publish(
+        TopicArn=env("REPARSE_SNS_TOPIC"),
+        Message=json.dumps(message_to_send),
+        Subject=f"Reparse request: {uri}",
+    )

{ds_caselaw_marklogic_api_client-17.1.0 → ds_caselaw_marklogic_api_client-17.3.0}/src/caselawclient/search_parameters.py

@@ -73,6 +73,8 @@ class SearchParameters:
             "ewhc/kb": "ewhc/qb",
             "ewhc/scco": "ewhc/costs",
             "ewhc/costs": "ewhc/scco",
+            "ukait": "ukut/iac",
+            "ukut/iac": "ukait",
         }
         alternative_court_names = set()
         for primary_name, secondary_name in ALTERNATIVE_COURT_NAMES_MAP.items():

{ds_caselaw_marklogic_api_client-17.1.0 → ds_caselaw_marklogic_api_client-17.3.0}/src/caselawclient/xquery/xslt_transform.xqy

@@ -1,10 +1,13 @@
 xquery version "1.0-ml";
 
+import module namespace helper = "https://caselaw.nationalarchives.gov.uk/helper" at "/judgments/search/helper.xqy";
+
 declare variable $show_unpublished as xs:boolean? external;
 declare variable $uri as xs:string external;
 declare variable $version_uri as xs:string? external;
 declare variable $img_location as xs:string? external;
 declare variable $xsl_filename as xs:string? external;
+declare variable $query as xs:string? external;
 
 let $judgment_published_property := xdmp:document-get-properties($uri, xs:QName("published"))[1]
 let $is_published := $judgment_published_property/text()
@@ -14,6 +17,27 @@ let $xsl_path := fn:concat("judgments/xslts/", $xsl_filename)
 
 let $params := map:map()
 
+let $number_marks_xslt := (
+  <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                  version="2.0">
+    <xsl:output method="html" />
+    <xsl:template match="@*|node()">
+      <xsl:copy>
+        <xsl:apply-templates select="@*|node()"/>
+      </xsl:copy>
+    </xsl:template>
+    <xsl:template match="mark">
+      <xsl:copy>
+          <xsl:copy-of select="@*" />
+          <xsl:attribute name="id">
+              <xsl:text>mark_</xsl:text>
+              <xsl:value-of select="count(preceding::mark)"/>
+          </xsl:attribute>
+          <xsl:apply-templates />
+      </xsl:copy>
+    </xsl:template>
+  </xsl:stylesheet>
+)
 (: change the image-base of the document to match the location of the assets in $image_base
    so that references to images point to the correct places on the internet :)
 let $_put := map:put(
@@ -26,7 +50,7 @@ let $_ := if (not(exists($document_to_transform))) then
     fn:error(xs:QName("FCL_DOCUMENTNOTFOUND"), "No XML document was found to transform")
   ) else ()
 
-let $return_value := if (xs:boolean($is_published) or $show_unpublished) then
+let $retrieved_value := if (xs:boolean($is_published) or $show_unpublished) then
         xdmp:xslt-invoke($xsl_path,
           $document_to_transform,
           $params
@@ -34,4 +58,16 @@ let $return_value := if (xs:boolean($is_published) or $show_unpublished) then
     else
         ()
 
+let $return_value := if($query) then
+      xdmp:xslt-eval(
+        $number_marks_xslt,
+        cts:highlight(
+          $retrieved_value,
+          helper:make-q-query($query),
+          <mark>{$cts:text}</mark>
+        )
+      )
+    else
+      $retrieved_value
+
 return $return_value

{ds_caselaw_marklogic_api_client-17.1.0 → ds_caselaw_marklogic_api_client-17.3.0}/src/caselawclient/xquery_type_dicts.py

@@ -192,6 +192,7 @@ class XsltDict(MarkLogicAPIDict):
 # xslt_transform.xqy
 class XsltTransformDict(MarkLogicAPIDict):
     img_location: Optional[str]
+    query: Optional[str]
     show_unpublished: Optional[bool]
     uri: MarkLogicDocumentURIString
     version_uri: Optional[MarkLogicDocumentVersionURIString]