ds-caselaw-marklogic-api-client 40.0.0__tar.gz → 41.0.0__tar.gz

{ds_caselaw_marklogic_api_client-40.0.0 → ds_caselaw_marklogic_api_client-41.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ds-caselaw-marklogic-api-client
-Version: 40.0.0
+Version: 41.0.0
 Summary: An API client for interacting with the underlying data in Find Caselaw.
 Keywords: national archives,caselaw
 Author: The National Archives

{ds_caselaw_marklogic_api_client-40.0.0 → ds_caselaw_marklogic_api_client-41.0.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ds-caselaw-marklogic-api-client"
-version = "40.0.0"
+version = "41.0.0"
 description = "An API client for interacting with the underlying data in Find Caselaw."
 authors = ["The National Archives"]
 homepage = "https://github.com/nationalarchives/ds-caselaw-custom-api-client"
@@ -32,14 +32,14 @@ sqids = "^0.5.0"
 defusedxml = "^0.7.1"
 
 [tool.poetry.group.dev.dependencies]
-coverage = "7.10.4"
+coverage = "7.10.6"
 pytest = "8.4.1"
 pytest-cov = "6.2.1"
-beautifulsoup4 = "4.13.4"
+beautifulsoup4 = "4.13.5"
 responses = "0.25.8"
 python-dotenv = "1.1.1"
-time-machine = "2.17.0"
-moto = {version = "5.1.10", extras = ["all"]}
+time-machine = "2.19.0"
+moto = {version = "5.1.11", extras = ["all"]}
 
 [tool.poetry.group.docs]
 optional = true

{ds_caselaw_marklogic_api_client-40.0.0 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/Client.py

@@ -34,6 +34,7 @@ from caselawclient.models.utilities import move
 from caselawclient.search_parameters import SearchParameters
 from caselawclient.types import DocumentIdentifierSlug, DocumentIdentifierValue, DocumentURIString
 from caselawclient.xquery_type_dicts import (
+    CheckContentHashUniqueByUriDict,
     MarkLogicDocumentURIString,
     MarkLogicDocumentVersionURIString,
     MarkLogicPrivilegeURIString,
@@ -728,6 +729,14 @@ class MarklogicApiClient:
             == 0
         )
 
+    def has_unique_content_hash(self, judgment_uri: DocumentURIString) -> bool:
+        """
+        Returns True if the content hash for this document is unique (not shared with other documents).
+        """
+        uri = self._format_uri_for_marklogic(judgment_uri)
+        vars: CheckContentHashUniqueByUriDict = {"uri": uri}
+        return self._eval_and_decode(vars, "check_content_hash_unique_by_uri.xqy") == "true"
+
     def eval(
         self,
         xquery_path: str,

{ds_caselaw_marklogic_api_client-40.0.0 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/models/documents/__init__.py

@@ -95,6 +95,11 @@ class Document:
             True,
             "The court for this {document_noun} is not valid",
         ),
+        (
+            "has_unique_content_hash",
+            True,
+            "There is another document with identical content",
+        ),
     ]
     """
     A list of tuples in the form:
@@ -325,6 +330,11 @@ class Document:
     def annotation(self) -> str:
         return self.api_client.get_version_annotation(self.uri)
 
+    @cached_property
+    def has_unique_content_hash(self) -> bool:
+        """Check if the content hash of this document is unique compared to all other documents in MarkLogic."""
+        return self.api_client.has_unique_content_hash(self.uri)
+
     @cached_property
     def version_created_datetime(self) -> datetime.datetime:
         return self.api_client.get_version_created_datetime(self.uri)
@@ -540,14 +550,14 @@ class Document:
         """
         Is it sensible to reparse this document?
         """
-        return self.docx_exists()
+        return self.docx_exists() and not self.body.has_external_data
 
     @cached_property
     def can_enrich(self) -> bool:
         """
         Is it possible to enrich this document?
         """
-        return self.body.has_content
+        return self.body.has_content and not self.body.has_external_data
 
     def validate_identifiers(self) -> SuccessFailureMessageTuple:
         return self.identifiers.perform_all_validations(document_type=type(self), api_client=self.api_client)

{ds_caselaw_marklogic_api_client-40.0.0 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/models/documents/body.py

@@ -6,9 +6,11 @@ from typing import Optional
 
 import pytz
 from ds_caselaw_utils.types import CourtCode
+from lxml import etree
 from saxonche import PySaxonProcessor
 
 from caselawclient.models.utilities.dates import parse_string_date_as_utc
+from caselawclient.types import DocumentCategory
 
 from .xml import XML
 
@@ -37,6 +39,9 @@ class DocumentBody:
     def get_xpath_match_strings(self, xpath: str, namespaces: dict[str, str] = DEFAULT_NAMESPACES) -> list[str]:
         return self._xml.get_xpath_match_strings(xpath, namespaces)
 
+    def get_xpath_nodes(self, xpath: str, namespaces: dict[str, str] = DEFAULT_NAMESPACES) -> list[etree._Element]:
+        return self._xml.get_xpath_nodes(xpath, namespaces)
+
     @cached_property
     def name(self) -> str:
         return self.get_xpath_match_string(
@@ -51,9 +56,46 @@ class DocumentBody:
     def jurisdiction(self) -> str:
         return self.get_xpath_match_string("/akn:akomaNtoso/akn:*/akn:meta/akn:proprietary/uk:jurisdiction/text()")
 
+    @cached_property
+    def categories(self) -> list[DocumentCategory]:
+        xpath = "/akn:akomaNtoso/akn:*/akn:meta/akn:proprietary/uk:category"
+        nodes = self.get_xpath_nodes(xpath, DEFAULT_NAMESPACES)
+
+        categories: dict[str, DocumentCategory] = {}
+        children_map: dict[str, list[DocumentCategory]] = {}
+
+        for node in nodes:
+            name = node.text
+            if name is None or not name.strip():
+                continue
+
+            category = DocumentCategory(name=name)
+            categories[name] = category
+
+            parent = node.get("parent")
+
+            if parent:
+                children_map.setdefault(parent, []).append(category)
+
+        for parent, subcategories in children_map.items():
+            if parent in categories:
+                categories[parent].subcategories.extend(subcategories)
+
+        top_level_categories = [
+            categories[name]
+            for node in nodes
+            if node.get("parent") is None
+            if (name := node.text) and name in categories
+        ]
+
+        return top_level_categories
+
+    # NOTE: Deprecated - use categories function
     @cached_property
     def category(self) -> Optional[str]:
-        return self.get_xpath_match_string("/akn:akomaNtoso/akn:*/akn:meta/akn:proprietary/uk:category/text()")
+        return self.get_xpath_match_string(
+            "/akn:akomaNtoso/akn:*/akn:meta/akn:proprietary/uk:category[not(@parent)][1]/text()"
+        )
 
     @cached_property
     def case_number(self) -> Optional[str]:
@@ -144,6 +186,13 @@ class DocumentBody:
 
         return False
 
+    @cached_property
+    def has_external_data(self) -> bool:
+        """Is there data which is not present within the source document:
+        is there a spreadsheet which has populated some fields. The current implementation
+        "is there a uk:party tag" is intended as a stopgap whilst we're not importing that data."""
+        return bool(self._xml.xml_as_tree.xpath("//uk:party", namespaces=DEFAULT_NAMESPACES))
+
     @cache
     def content_html(self, image_prefix: str) -> Optional[str]:
         """Convert the XML representation of the Document into HTML for rendering."""

{ds_caselaw_marklogic_api_client-40.0.0 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/models/documents/xml.py

@@ -2,7 +2,7 @@ import os
 
 from lxml import etree
 
-from caselawclient.xml_helpers import get_xpath_match_string, get_xpath_match_strings
+from caselawclient.xml_helpers import get_xpath_match_string, get_xpath_match_strings, get_xpath_nodes
 
 
 def _xslt_path(xslt_file_name: str) -> str:
@@ -50,6 +50,9 @@ class XML:
     ) -> list[str]:
         return get_xpath_match_strings(self.xml_as_tree, xpath, namespaces)
 
+    def get_xpath_nodes(self, xpath: str, namespaces: dict[str, str]) -> list[etree._Element]:
+        return get_xpath_nodes(self.xml_as_tree, xpath, namespaces)
+
     def _modified(
         self,
         xslt: str,

{ds_caselaw_marklogic_api_client-40.0.0 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/models/identifiers/__init__.py

@@ -46,7 +46,10 @@ class IdentifierSchema(ABC):
     """ Should editors be allowed to manually manipulate identifiers under this schema? """
 
     require_globally_unique: bool = True
-    """ Must this identifier be globally unique? """
+    """ Must this identifier be globally unique? (appear on no other documents) """
+
+    allow_multiple: bool = False
+    """ May documents have more than one non-deprecated identifier of this type? """
 
     document_types: Optional[list[str]] = None
     """

{ds_caselaw_marklogic_api_client-40.0.0 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/models/identifiers/collection.py

@@ -43,6 +43,8 @@ class IdentifiersCollection(dict[str, Identifier]):
         """Check that only one non-deprecated identifier exists per schema where that schema does not allow multiples."""
 
         for schema, identifiers in self._list_all_identifiers_by_schema().items():
+            if schema.allow_multiple:
+                continue
             non_deprecated_identifiers = [i for i in identifiers if not i.deprecated]
             if len(non_deprecated_identifiers) > 1:
                 return SuccessFailureMessageTuple(

{ds_caselaw_marklogic_api_client-40.0.0 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/models/utilities/aws.py

@@ -2,6 +2,7 @@ import datetime
 import json
 import logging
 import uuid
+from collections.abc import Callable
 from typing import Any, Literal, Optional, TypedDict, overload
 
 import boto3
@@ -118,11 +119,20 @@ def generate_pdf_url(uri: DocumentURIString) -> str:
 
 
 def delete_from_bucket(uri: DocumentURIString, bucket: str) -> None:
+    delete_some_from_bucket(uri=uri, bucket=bucket, filter=lambda x: True)
+
+
+def delete_some_from_bucket(
+    uri: DocumentURIString, bucket: str, filter: Callable[[ObjectIdentifierTypeDef], bool]
+) -> None:
     client = create_s3_client()
     response = client.list_objects(Bucket=bucket, Prefix=uri_for_s3(uri))
 
     if response.get("Contents"):
-        objects_to_delete: list[ObjectIdentifierTypeDef] = [{"Key": obj["Key"]} for obj in response.get("Contents", [])]
+        objects_to_maybe_delete: list[ObjectIdentifierTypeDef] = [
+            {"Key": obj["Key"]} for obj in response.get("Contents", [])
+        ]
+        objects_to_delete = [obj for obj in objects_to_maybe_delete if filter(obj)]
         client.delete_objects(
             Bucket=bucket,
             Delete={
@@ -131,6 +141,10 @@ def delete_from_bucket(uri: DocumentURIString, bucket: str) -> None:
         )
 
 
+def delete_non_targz_from_bucket(uri: DocumentURIString, bucket: str) -> None:
+    delete_some_from_bucket(uri=uri, bucket=bucket, filter=lambda x: not x["Key"].endswith(".tar.gz"))
+
+
 def publish_documents(uri: DocumentURIString) -> None:
     """
     Copy assets from the unpublished bucket to the published one.

{ds_caselaw_marklogic_api_client-40.0.0 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/types.py

@@ -1,6 +1,13 @@
+from dataclasses import dataclass, field
 from typing import NamedTuple
 
 
+@dataclass
+class DocumentCategory:
+    name: str
+    subcategories: list["DocumentCategory"] = field(default_factory=list)
+
+
 class InvalidDocumentURIException(Exception):
     """The document URI is not valid."""
 

{ds_caselaw_marklogic_api_client-40.0.0 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/xml_helpers.py

@@ -7,9 +7,25 @@ DEFAULT_NAMESPACES = {
     "akn": "http://docs.oasis-open.org/legaldocml/ns/akn/3.0",
 }
 
+# _Element is the only class lxml exposes, so need to use the private class for typing
+Element = etree._Element  # noqa: SLF001
+
+
+def get_xpath_nodes(
+    node: Element,
+    path: str,
+    namespaces: Optional[Dict[str, str]] = None,
+) -> list[Element]:
+    result = node.xpath(path, namespaces=namespaces)
+
+    if not isinstance(result, list) or not all(isinstance(x, Element) for x in result):
+        raise TypeError(f"Expected to return list[Element], got {type(result).__name__}")
+
+    return result
+
 
 def get_xpath_match_string(
-    node: etree._Element,
+    node: Element,
     path: str,
     namespaces: Optional[Dict[str, str]] = None,
     fallback: str = "",
@@ -18,7 +34,7 @@ def get_xpath_match_string(
 
 
 def get_xpath_match_strings(
-    node: etree._Element,
+    node: Element,
     path: str,
     namespaces: Optional[Dict[str, str]] = None,
 ) -> list[str]:

ds_caselaw_marklogic_api_client-41.0.0/src/caselawclient/xquery/check_content_hash_unique_by_uri.xqy

@@ -0,0 +1,9 @@
+xquery version "1.0-ml";
+declare namespace akn = "http://docs.oasis-open.org/legaldocml/ns/akn/3.0";
+declare namespace uk = "https://caselaw.nationalarchives.gov.uk/akn";
+declare variable $uri as xs:string external;
+
+let $doc := doc($uri)
+let $hash := $doc//uk:hash/text()
+let $count := count(cts:search(fn:doc(), cts:element-value-query(xs:QName("uk:hash"), $hash)))
+return $count = 1

{ds_caselaw_marklogic_api_client-40.0.0 → ds_caselaw_marklogic_api_client-41.0.0}/src/caselawclient/xquery_type_dicts.py

@@ -23,6 +23,11 @@ class BreakJudgmentCheckoutDict(MarkLogicAPIDict):
     uri: MarkLogicDocumentURIString
 
 
+# check_content_hash_unique_by_uri.xqy
+class CheckContentHashUniqueByUriDict(MarkLogicAPIDict):
+    uri: MarkLogicDocumentURIString
+
+
 # checkin_judgment.xqy
 class CheckinJudgmentDict(MarkLogicAPIDict):
     uri: MarkLogicDocumentURIString