ds-caselaw-marklogic-api-client 39.2.1__tar.gz → 41.0.0__tar.gz

{ds_caselaw_marklogic_api_client-39.2.1 → ds_caselaw_marklogic_api_client-41.0.0}/PKG-INFO

@@ -1,13 +1,11 @@
 Metadata-Version: 2.3
 Name: ds-caselaw-marklogic-api-client
-Version: 39.2.1
+Version: 41.0.0
 Summary: An API client for interacting with the underlying data in Find Caselaw.
 Keywords: national archives,caselaw
 Author: The National Archives
-Requires-Python: >=3.10.0,<4.0.0
+Requires-Python: >=3.12.0,<4.0.0
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: boto3 (>=1.26.112,<2.0.0)

{ds_caselaw_marklogic_api_client-39.2.1 → ds_caselaw_marklogic_api_client-41.0.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ds-caselaw-marklogic-api-client"
-version = "39.2.1"
+version = "41.0.0"
 description = "An API client for interacting with the underlying data in Find Caselaw."
 authors = ["The National Archives"]
 homepage = "https://github.com/nationalarchives/ds-caselaw-custom-api-client"
@@ -11,7 +11,7 @@ packages = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.10.0"
+python = "^3.12.0"
 certifi = ">=2025.8.3,<2025.9.0"
 charset-normalizer = "^3.0.0"
 django-environ = ">=0.12.0"
@@ -32,14 +32,14 @@ sqids = "^0.5.0"
 defusedxml = "^0.7.1"
 
 [tool.poetry.group.dev.dependencies]
-coverage = "7.10.3"
+coverage = "7.10.6"
 pytest = "8.4.1"
 pytest-cov = "6.2.1"
-beautifulsoup4 = "4.13.4"
+beautifulsoup4 = "4.13.5"
 responses = "0.25.8"
 python-dotenv = "1.1.1"
-time-machine = "2.17.0"
-moto = {version = "5.1.10", extras = ["all"]}
+time-machine = "2.19.0"
+moto = {version = "5.1.11", extras = ["all"]}
 
 [tool.poetry.group.docs]
 optional = true

{ds_caselaw_marklogic_api_client-39.2.1 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/Client.py

@@ -11,6 +11,7 @@ from xml.etree.ElementTree import Element
 
 import environ
 import requests
+from dateutil.parser import isoparse
 from defusedxml import ElementTree
 from defusedxml.ElementTree import ParseError, fromstring
 from ds_caselaw_utils.types import NeutralCitationString
@@ -33,6 +34,7 @@ from caselawclient.models.utilities import move
 from caselawclient.search_parameters import SearchParameters
 from caselawclient.types import DocumentIdentifierSlug, DocumentIdentifierValue, DocumentURIString
 from caselawclient.xquery_type_dicts import (
+    CheckContentHashUniqueByUriDict,
     MarkLogicDocumentURIString,
     MarkLogicDocumentVersionURIString,
     MarkLogicPrivilegeURIString,
@@ -727,6 +729,14 @@ class MarklogicApiClient:
             == 0
         )
 
+    def has_unique_content_hash(self, judgment_uri: DocumentURIString) -> bool:
+        """
+        Returns True if the content hash for this document is unique (not shared with other documents).
+        """
+        uri = self._format_uri_for_marklogic(judgment_uri)
+        vars: CheckContentHashUniqueByUriDict = {"uri": uri}
+        return self._eval_and_decode(vars, "check_content_hash_unique_by_uri.xqy") == "true"
+
     def eval(
         self,
         xquery_path: str,
@@ -948,12 +958,50 @@ class MarklogicApiClient:
             "value": string_value,
             "name": name,
         }
+        """
+        Set a property within MarkLogic which is specifically a boolean.
+
+        Since XML has no concept of boolean, the actual value in the database is set to `"true"` or `"false"`.
+        """
         return self._send_to_eval(vars, "set_boolean_property.xqy")
 
     def get_boolean_property(self, judgment_uri: DocumentURIString, name: str) -> bool:
+        """
+        Get a property from MarkLogic which is specifically a boolean.
+
+        :return: `True` if the property exists and has a value of `"true"`, otherwise `False`
+        """
         content = self.get_property(judgment_uri, name)
         return content == "true"
 
+    def set_datetime_property(
+        self,
+        judgment_uri: DocumentURIString,
+        name: str,
+        value: datetime,
+    ) -> requests.Response:
+        """Set a property within MarkLogic which is specifically a datetime."""
+        uri = self._format_uri_for_marklogic(judgment_uri)
+        vars: query_dicts.SetDatetimePropertyDict = {
+            "uri": uri,
+            "value": value.isoformat(),
+            "name": name,
+        }
+        return self._send_to_eval(vars, "set_datetime_property.xqy")
+
+    def get_datetime_property(self, judgment_uri: DocumentURIString, name: str) -> Optional[datetime]:
+        """
+        Get a property from MarkLogic which is specifically a datetime.
+
+        :return: A datetime with the value of the property, or `None` if it does not exist
+        """
+        content = self.get_property(judgment_uri, name)
+
+        if content:
+            return isoparse(content)
+
+        return None
+
     def set_published(
         self,
         judgment_uri: DocumentURIString,

{ds_caselaw_marklogic_api_client-39.2.1 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/models/documents/__init__.py

@@ -95,6 +95,11 @@ class Document:
             True,
             "The court for this {document_noun} is not valid",
         ),
+        (
+            "has_unique_content_hash",
+            True,
+            "There is another document with identical content",
+        ),
     ]
     """
     A list of tuples in the form:
@@ -305,6 +310,14 @@ class Document:
         # An empty list (which is falsy) therefore means the judgment can be published safely.
         return not self.validation_failure_messages
 
+    @cached_property
+    def first_published_datetime(self) -> Optional[datetime.datetime]:
+        return self.api_client.get_datetime_property(self.uri, "first_published_datetime")
+
+    @cached_property
+    def has_ever_been_published(self) -> bool:
+        return self.is_published or self.first_published_datetime is not None
+
     @cached_property
     def validation_failure_messages(self) -> list[str]:
         exception_list = []
@@ -317,6 +330,11 @@ class Document:
     def annotation(self) -> str:
         return self.api_client.get_version_annotation(self.uri)
 
+    @cached_property
+    def has_unique_content_hash(self) -> bool:
+        """Check if the content hash of this document is unique compared to all other documents in MarkLogic."""
+        return self.api_client.has_unique_content_hash(self.uri)
+
     @cached_property
     def version_created_datetime(self) -> datetime.datetime:
         return self.api_client.get_version_created_datetime(self.uri)
@@ -407,6 +425,8 @@ class Document:
 
     def publish(self) -> None:
         """
+        Assuming that a document passes pre-publish checks, perform all necessary operations to put it into a published state.
+
         :raises CannotPublishUnpublishableDocument: This document has not passed the checks in `is_publishable`, and as
         such cannot be published.
         """
@@ -416,12 +436,25 @@ class Document:
         ## Make sure the document has an FCLID
         self.assign_fclid_if_missing()
 
+        ## Copy the document assets into the appropriate place in S3
         publish_documents(self.uri)
+
+        ## Set the fact the document is published
         self.api_client.set_published(self.uri, True)
+
+        ## If necessary, set the first published date
+        if not self.first_published_datetime:
+            self.api_client.set_datetime_property(
+                self.uri, "first_published_datetime", datetime.datetime.now(datetime.timezone.utc)
+            )
+
+        ## Announce the publication on the event bus
         announce_document_event(
             uri=self.uri,
             status="publish",
         )
+
+        ## Send the document off for enrichment, but accept if we can't for any reason
         self.enrich(accept_failures=True)
 
     def unpublish(self) -> None:
@@ -517,14 +550,14 @@ class Document:
         """
         Is it sensible to reparse this document?
         """
-        return self.docx_exists()
+        return self.docx_exists() and not self.body.has_external_data
 
     @cached_property
     def can_enrich(self) -> bool:
         """
         Is it possible to enrich this document?
         """
-        return self.body.has_content
+        return self.body.has_content and not self.body.has_external_data
 
     def validate_identifiers(self) -> SuccessFailureMessageTuple:
         return self.identifiers.perform_all_validations(document_type=type(self), api_client=self.api_client)

{ds_caselaw_marklogic_api_client-39.2.1 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/models/documents/body.py

@@ -6,9 +6,11 @@ from typing import Optional
 
 import pytz
 from ds_caselaw_utils.types import CourtCode
+from lxml import etree
 from saxonche import PySaxonProcessor
 
 from caselawclient.models.utilities.dates import parse_string_date_as_utc
+from caselawclient.types import DocumentCategory
 
 from .xml import XML
 
@@ -37,6 +39,9 @@ class DocumentBody:
     def get_xpath_match_strings(self, xpath: str, namespaces: dict[str, str] = DEFAULT_NAMESPACES) -> list[str]:
         return self._xml.get_xpath_match_strings(xpath, namespaces)
 
+    def get_xpath_nodes(self, xpath: str, namespaces: dict[str, str] = DEFAULT_NAMESPACES) -> list[etree._Element]:
+        return self._xml.get_xpath_nodes(xpath, namespaces)
+
     @cached_property
     def name(self) -> str:
         return self.get_xpath_match_string(
@@ -51,9 +56,46 @@ class DocumentBody:
     def jurisdiction(self) -> str:
         return self.get_xpath_match_string("/akn:akomaNtoso/akn:*/akn:meta/akn:proprietary/uk:jurisdiction/text()")
 
+    @cached_property
+    def categories(self) -> list[DocumentCategory]:
+        xpath = "/akn:akomaNtoso/akn:*/akn:meta/akn:proprietary/uk:category"
+        nodes = self.get_xpath_nodes(xpath, DEFAULT_NAMESPACES)
+
+        categories: dict[str, DocumentCategory] = {}
+        children_map: dict[str, list[DocumentCategory]] = {}
+
+        for node in nodes:
+            name = node.text
+            if name is None or not name.strip():
+                continue
+
+            category = DocumentCategory(name=name)
+            categories[name] = category
+
+            parent = node.get("parent")
+
+            if parent:
+                children_map.setdefault(parent, []).append(category)
+
+        for parent, subcategories in children_map.items():
+            if parent in categories:
+                categories[parent].subcategories.extend(subcategories)
+
+        top_level_categories = [
+            categories[name]
+            for node in nodes
+            if node.get("parent") is None
+            if (name := node.text) and name in categories
+        ]
+
+        return top_level_categories
+
+    # NOTE: Deprecated - use categories function
     @cached_property
     def category(self) -> Optional[str]:
-        return self.get_xpath_match_string("/akn:akomaNtoso/akn:*/akn:meta/akn:proprietary/uk:category/text()")
+        return self.get_xpath_match_string(
+            "/akn:akomaNtoso/akn:*/akn:meta/akn:proprietary/uk:category[not(@parent)][1]/text()"
+        )
 
     @cached_property
     def case_number(self) -> Optional[str]:
@@ -144,6 +186,13 @@ class DocumentBody:
 
         return False
 
+    @cached_property
+    def has_external_data(self) -> bool:
+        """Is there data which is not present within the source document:
+        is there a spreadsheet which has populated some fields. The current implementation
+        "is there a uk:party tag" is intended as a stopgap whilst we're not importing that data."""
+        return bool(self._xml.xml_as_tree.xpath("//uk:party", namespaces=DEFAULT_NAMESPACES))
+
     @cache
     def content_html(self, image_prefix: str) -> Optional[str]:
         """Convert the XML representation of the Document into HTML for rendering."""

{ds_caselaw_marklogic_api_client-39.2.1 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/models/documents/xml.py

@@ -2,7 +2,7 @@ import os
 
 from lxml import etree
 
-from caselawclient.xml_helpers import get_xpath_match_string, get_xpath_match_strings
+from caselawclient.xml_helpers import get_xpath_match_string, get_xpath_match_strings, get_xpath_nodes
 
 
 def _xslt_path(xslt_file_name: str) -> str:
@@ -50,6 +50,9 @@ class XML:
     ) -> list[str]:
         return get_xpath_match_strings(self.xml_as_tree, xpath, namespaces)
 
+    def get_xpath_nodes(self, xpath: str, namespaces: dict[str, str]) -> list[etree._Element]:
+        return get_xpath_nodes(self.xml_as_tree, xpath, namespaces)
+
     def _modified(
         self,
         xslt: str,

{ds_caselaw_marklogic_api_client-39.2.1 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/models/identifiers/__init__.py

@@ -46,7 +46,10 @@ class IdentifierSchema(ABC):
     """ Should editors be allowed to manually manipulate identifiers under this schema? """
 
     require_globally_unique: bool = True
-    """ Must this identifier be globally unique? """
+    """ Must this identifier be globally unique? (appear on no other documents) """
+
+    allow_multiple: bool = False
+    """ May documents have more than one non-deprecated identifier of this type? """
 
     document_types: Optional[list[str]] = None
     """

{ds_caselaw_marklogic_api_client-39.2.1 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/models/identifiers/collection.py

@@ -43,6 +43,8 @@ class IdentifiersCollection(dict[str, Identifier]):
         """Check that only one non-deprecated identifier exists per schema where that schema does not allow multiples."""
 
         for schema, identifiers in self._list_all_identifiers_by_schema().items():
+            if schema.allow_multiple:
+                continue
             non_deprecated_identifiers = [i for i in identifiers if not i.deprecated]
             if len(non_deprecated_identifiers) > 1:
                 return SuccessFailureMessageTuple(

{ds_caselaw_marklogic_api_client-39.2.1 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/models/utilities/aws.py

@@ -2,6 +2,7 @@ import datetime
 import json
 import logging
 import uuid
+from collections.abc import Callable
 from typing import Any, Literal, Optional, TypedDict, overload
 
 import boto3
@@ -118,11 +119,20 @@ def generate_pdf_url(uri: DocumentURIString) -> str:
 
 
 def delete_from_bucket(uri: DocumentURIString, bucket: str) -> None:
+    delete_some_from_bucket(uri=uri, bucket=bucket, filter=lambda x: True)
+
+
+def delete_some_from_bucket(
+    uri: DocumentURIString, bucket: str, filter: Callable[[ObjectIdentifierTypeDef], bool]
+) -> None:
     client = create_s3_client()
     response = client.list_objects(Bucket=bucket, Prefix=uri_for_s3(uri))
 
     if response.get("Contents"):
-        objects_to_delete: list[ObjectIdentifierTypeDef] = [{"Key": obj["Key"]} for obj in response.get("Contents", [])]
+        objects_to_maybe_delete: list[ObjectIdentifierTypeDef] = [
+            {"Key": obj["Key"]} for obj in response.get("Contents", [])
+        ]
+        objects_to_delete = [obj for obj in objects_to_maybe_delete if filter(obj)]
         client.delete_objects(
             Bucket=bucket,
             Delete={
@@ -131,6 +141,10 @@ def delete_from_bucket(uri: DocumentURIString, bucket: str) -> None:
         )
 
 
+def delete_non_targz_from_bucket(uri: DocumentURIString, bucket: str) -> None:
+    delete_some_from_bucket(uri=uri, bucket=bucket, filter=lambda x: not x["Key"].endswith(".tar.gz"))
+
+
 def publish_documents(uri: DocumentURIString) -> None:
     """
     Copy assets from the unpublished bucket to the published one.

{ds_caselaw_marklogic_api_client-39.2.1 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/responses/search_result.py

@@ -13,6 +13,8 @@ from lxml import etree
 
 from caselawclient.Client import MarklogicApiClient
 from caselawclient.models.identifiers.collection import IdentifiersCollection
+from caselawclient.models.identifiers.neutral_citation import NeutralCitationNumber
+from caselawclient.models.identifiers.press_summary_ncn import PressSummaryRelatedNCNIdentifier
 from caselawclient.models.identifiers.unpacker import unpack_all_identifiers_from_etree
 from caselawclient.types import DocumentURIString
 from caselawclient.xml_helpers import get_xpath_match_string
@@ -196,14 +198,18 @@ class SearchResult:
         return str(preferred.url_slug)
 
     @property
-    def neutral_citation(self) -> str:
+    def neutral_citation(self) -> Optional[str]:
         """
-        :return: The neutral citation of the search result, or the judgment it is a press summary of.
+        :return: If present, the value of preferred neutral citation of the document.
         """
 
-        return self._get_xpath_match_string(
-            "search:extracted/uk:cite/text()",
-        ) or self._get_xpath_match_string("search:extracted/akn:neutralCitation/text()")
+        preferred_ncn = self.identifiers.preferred(type=NeutralCitationNumber)
+
+        # If the result doesn't have a preferred NCN, maybe it has a preferred press summary related NCN?
+        if not preferred_ncn:
+            preferred_ncn = self.identifiers.preferred(type=PressSummaryRelatedNCNIdentifier)
+
+        return preferred_ncn.value if preferred_ncn else None
 
     @property
     def name(self) -> str:

{ds_caselaw_marklogic_api_client-39.2.1 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/types.py

@@ -1,6 +1,13 @@
+from dataclasses import dataclass, field
 from typing import NamedTuple
 
 
+@dataclass
+class DocumentCategory:
+    name: str
+    subcategories: list["DocumentCategory"] = field(default_factory=list)
+
+
 class InvalidDocumentURIException(Exception):
     """The document URI is not valid."""
 

{ds_caselaw_marklogic_api_client-39.2.1 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/xml_helpers.py

@@ -7,9 +7,25 @@ DEFAULT_NAMESPACES = {
     "akn": "http://docs.oasis-open.org/legaldocml/ns/akn/3.0",
 }
 
+# _Element is the only class lxml exposes, so need to use the private class for typing
+Element = etree._Element  # noqa: SLF001
+
+
+def get_xpath_nodes(
+    node: Element,
+    path: str,
+    namespaces: Optional[Dict[str, str]] = None,
+) -> list[Element]:
+    result = node.xpath(path, namespaces=namespaces)
+
+    if not isinstance(result, list) or not all(isinstance(x, Element) for x in result):
+        raise TypeError(f"Expected to return list[Element], got {type(result).__name__}")
+
+    return result
+
 
 def get_xpath_match_string(
-    node: etree._Element,
+    node: Element,
     path: str,
     namespaces: Optional[Dict[str, str]] = None,
     fallback: str = "",
@@ -18,7 +34,7 @@ def get_xpath_match_string(
 
 
 def get_xpath_match_strings(
-    node: etree._Element,
+    node: Element,
     path: str,
     namespaces: Optional[Dict[str, str]] = None,
 ) -> list[str]:

ds_caselaw_marklogic_api_client-41.0.0/src/caselawclient/xquery/check_content_hash_unique_by_uri.xqy

@@ -0,0 +1,9 @@
+xquery version "1.0-ml";
+declare namespace akn = "http://docs.oasis-open.org/legaldocml/ns/akn/3.0";
+declare namespace uk = "https://caselaw.nationalarchives.gov.uk/akn";
+declare variable $uri as xs:string external;
+
+let $doc := doc($uri)
+let $hash := $doc//uk:hash/text()
+let $count := count(cts:search(fn:doc(), cts:element-value-query(xs:QName("uk:hash"), $hash)))
+return $count = 1

ds_caselaw_marklogic_api_client-41.0.0/src/caselawclient/xquery/set_datetime_property.xqy

@@ -0,0 +1,11 @@
+xquery version "1.0-ml";
+
+import module namespace dls = "http://marklogic.com/xdmp/dls" at "/MarkLogic/dls.xqy";
+
+declare variable $uri as xs:string external;
+declare variable $value as xs:string external;
+declare variable $name as xs:string external;
+
+let $props := ( element {$name}{xs:dateTime($value)} )
+
+return dls:document-set-property($uri, $props)

{ds_caselaw_marklogic_api_client-39.2.1 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/xquery_type_dicts.py

@@ -23,6 +23,11 @@ class BreakJudgmentCheckoutDict(MarkLogicAPIDict):
     uri: MarkLogicDocumentURIString
 
 
+# check_content_hash_unique_by_uri.xqy
+class CheckContentHashUniqueByUriDict(MarkLogicAPIDict):
+    uri: MarkLogicDocumentURIString
+
+
 # checkin_judgment.xqy
 class CheckinJudgmentDict(MarkLogicAPIDict):
     uri: MarkLogicDocumentURIString
@@ -166,6 +171,13 @@ class SetBooleanPropertyDict(MarkLogicAPIDict):
     value: str
 
 
+# set_datetime_property.xqy
+class SetDatetimePropertyDict(MarkLogicAPIDict):
+    name: str
+    uri: MarkLogicDocumentURIString
+    value: str
+
+
 # set_metadata_citation.xqy
 class SetMetadataCitationDict(MarkLogicAPIDict):
     content: str