ds-caselaw-marklogic-api-client 27.3.0__tar.gz → 28.0.0__tar.gz

{ds_caselaw_marklogic_api_client-27.3.0 → ds_caselaw_marklogic_api_client-28.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ds-caselaw-marklogic-api-client
-Version: 27.3.0
+Version: 28.0.0
 Summary: An API client for interacting with the underlying data in Find Caselaw.
 Home-page: https://github.com/nationalarchives/ds-caselaw-custom-api-client
 Keywords: national archives,caselaw

{ds_caselaw_marklogic_api_client-27.3.0 → ds_caselaw_marklogic_api_client-28.0.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ds-caselaw-marklogic-api-client"
-version = "27.3.0"
+version = "28.0.0"
 description = "An API client for interacting with the underlying data in Find Caselaw."
 authors = ["The National Archives"]
 homepage = "https://github.com/nationalarchives/ds-caselaw-custom-api-client"

{ds_caselaw_marklogic_api_client-27.3.0 → ds_caselaw_marklogic_api_client-28.0.0}/src/caselawclient/Client.py

@@ -62,6 +62,8 @@ except importlib.metadata.PackageNotFoundError:
     VERSION = "0"
 DEFAULT_USER_AGENT = f"ds-caselaw-marklogic-api-client/{VERSION}"
 
+DEBUG: bool = bool(os.getenv("DEBUG", default=False))
+
 
 class NoResponse(Exception):
     """A requests HTTPError has no response. We expect this will never happen."""
@@ -207,12 +209,14 @@ class MarklogicApiClient:
         Returns a list of PressSummary objects associated with a given Document URI
         """
         vars: query_dicts.GetComponentsForDocumentDict = {
-            "parent_uri": DocumentURIString(uri if uri.startswith("/") else "/" + uri),
+            "parent_uri": uri,
             "component": "pressSummary",
         }
         response = self._send_to_eval(vars, "get_components_for_document.xqy")
         uris = get_multipart_strings_from_marklogic_response(response)
-        return [PressSummary(uri.strip(".xml"), self) for uri in uris]
+        return [
+            PressSummary(DocumentURIString(uri.strip("/").strip(".xml")), self) for uri in uris
+        ]  # TODO: Migrate this strip behaviour into proper manipulation of a MarkLogicURIString
 
     def get_document_by_uri(
         self,
@@ -728,6 +732,10 @@ class MarklogicApiClient:
             "vars": vars,
         }
         path = "LATEST/eval"
+
+        if DEBUG:
+            print(f"Sending {vars} to {xquery_path}")
+
         response = self.session.request(
             "POST",
             url=self._path_to_request_url(path),
@@ -1101,6 +1109,21 @@ class MarklogicApiClient:
 
         return results
 
+    def get_recently_enriched(
+        self,
+    ) -> list[list[Any]]:
+        """Retrieve documents which are not yet enriched with a given version."""
+        results: list[list[Any]] = json.loads(
+            get_single_string_from_marklogic_response(
+                self._send_to_eval(
+                    {},
+                    "get_recently_enriched.xqy",
+                ),
+            ),
+        )
+
+        return results
+
     def get_highest_parser_version(self) -> tuple[int, int]:
         """This gets the highest parser version in the database, so if nothing has been parsed with the most recent version of the parser, this won't reflect that change."""
         table = json.loads(
@@ -1135,3 +1158,18 @@ class MarklogicApiClient:
         )
 
         return results
+
+    def get_recently_parsed(
+        self,
+    ) -> list[list[Any]]:
+        """Retrieve documents which are not yet enriched with a given version."""
+        results: list[list[Any]] = json.loads(
+            get_single_string_from_marklogic_response(
+                self._send_to_eval(
+                    {},
+                    "get_recently_parsed.xqy",
+                ),
+            ),
+        )
+
+        return results

{ds_caselaw_marklogic_api_client-27.3.0 → ds_caselaw_marklogic_api_client-28.0.0}/src/caselawclient/factories.py

@@ -5,7 +5,7 @@ from unittest.mock import Mock
 from typing_extensions import TypeAlias
 
 from caselawclient.Client import MarklogicApiClient
-from caselawclient.models.documents import Document
+from caselawclient.models.documents import Document, DocumentURIString
 from caselawclient.models.documents.body import DocumentBody
 from caselawclient.models.judgments import Judgment
 from caselawclient.models.press_summaries import PressSummary
@@ -54,7 +54,7 @@ class DocumentFactory:
     @classmethod
     def build(
         cls,
-        uri: str = "test/2023/123",
+        uri: DocumentURIString = DocumentURIString("test/2023/123"),
         html: str = "<p>This is a judgment.</p>",
         api_client: Optional[MarklogicApiClient] = None,
         **kwargs: Any,

{ds_caselaw_marklogic_api_client-27.3.0 → ds_caselaw_marklogic_api_client-28.0.0}/src/caselawclient/models/documents/__init__.py

@@ -1,7 +1,7 @@
 import datetime
 import warnings
 from functools import cached_property
-from typing import TYPE_CHECKING, Any, NewType, Optional
+from typing import TYPE_CHECKING, Any, Optional
 
 from ds_caselaw_utils import courts
 from ds_caselaw_utils.courts import CourtNotFoundException
@@ -30,7 +30,7 @@ from caselawclient.models.utilities.aws import (
 )
 
 from .body import DocumentBody
-from .exceptions import CannotPublishUnpublishableDocument, DocumentNotSafeForDeletion
+from .exceptions import CannotPublishUnpublishableDocument, DocumentNotSafeForDeletion, InvalidDocumentURIException
 from .statuses import DOCUMENT_STATUS_HOLD, DOCUMENT_STATUS_IN_PROGRESS, DOCUMENT_STATUS_NEW, DOCUMENT_STATUS_PUBLISHED
 
 MINIMUM_ENRICHMENT_TIME = datetime.timedelta(minutes=20)
@@ -47,7 +47,26 @@ if TYPE_CHECKING:
     from caselawclient.Client import MarklogicApiClient
 
 
-DocumentURIString = NewType("DocumentURIString", str)
+class DocumentURIString(str):
+    """
+    This class checks that the string is actually a valid Document URI on creation. It does _not_ manipulate the string.
+    """
+
+    def __new__(cls, content: str) -> "DocumentURIString":
+        # Check that the URI doesn't begin or end with a slash
+        if content[0] == "/" or content[-1] == "/":
+            raise InvalidDocumentURIException(
+                f'"{content}" is not a valid document URI; URIs cannot begin or end with slashes.'
+            )
+
+        # Check that the URI doesn't contain a full stop
+        if "." in content:
+            raise InvalidDocumentURIException(
+                f'"{content}" is not a valid document URI; URIs cannot contain full stops.'
+            )
+
+        # If everything is good, return as usual
+        return str.__new__(cls, content)
 
 
 class Document:
@@ -105,13 +124,15 @@ class Document:
     Individual document classes should extend this list where necessary to validate document type-specific attributes.
     """
 
-    def __init__(self, uri: str, api_client: "MarklogicApiClient", search_query: Optional[str] = None):
+    def __init__(self, uri: DocumentURIString, api_client: "MarklogicApiClient", search_query: Optional[str] = None):
         """
-        :param uri: For historical reasons this accepts a pseudo-URI which may include leading or trailing slashes.
+        :param uri: The URI of the document to retrieve from MarkLogic.
+        :param api_client: An instance of the API client object to handle communication with the MarkLogic server.
+        :param search_query: Optionally, a search string which should be highlighted if it appears in the document body.
 
         :raises DocumentNotFoundError: The document does not exist within MarkLogic
         """
-        self.uri: DocumentURIString = DocumentURIString(uri.strip("/"))
+        self.uri: DocumentURIString = uri
         self.api_client: MarklogicApiClient = api_client
         if not self.document_exists():
             raise DocumentNotFoundError(f"Document {self.uri} does not exist")
@@ -123,7 +144,7 @@ class Document:
                 search_query=search_query,
             ),
         )
-        """ `Document.body` represents the XML of the document itself, without any information such as version tracking or properties. """
+        """ `Document.body` represents the body of the document itself, without any information such as version tracking or properties. """
 
     def __repr__(self) -> str:
         name = self.body.name or "un-named"

{ds_caselaw_marklogic_api_client-27.3.0 → ds_caselaw_marklogic_api_client-28.0.0}/src/caselawclient/models/documents/exceptions.py

@@ -4,3 +4,7 @@ class CannotPublishUnpublishableDocument(Exception):
 
 class DocumentNotSafeForDeletion(Exception):
     """A document which is not safe for deletion cannot be deleted."""
+
+
+class InvalidDocumentURIException(Exception):
+    """The document URI is not valid."""

{ds_caselaw_marklogic_api_client-27.3.0 → ds_caselaw_marklogic_api_client-28.0.0}/src/caselawclient/models/judgments.py

@@ -10,7 +10,7 @@ from caselawclient.models.neutral_citation_mixin import NeutralCitationMixin
 if TYPE_CHECKING:
     from caselawclient.models.press_summaries import PressSummary
 
-from .documents import Document
+from .documents import Document, DocumentURIString
 
 
 class Judgment(NeutralCitationMixin, Document):
@@ -21,8 +21,8 @@ class Judgment(NeutralCitationMixin, Document):
     document_noun = "judgment"
     document_noun_plural = "judgments"
 
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        super().__init__(self.document_noun, *args, **kwargs)
+    def __init__(self, uri: DocumentURIString, *args: Any, **kwargs: Any) -> None:
+        super().__init__(self.document_noun, uri, *args, **kwargs)
 
     @cached_property
     def neutral_citation(self) -> NeutralCitationString:
@@ -46,8 +46,9 @@ class Judgment(NeutralCitationMixin, Document):
         Attempt to fetch a linked press summary, and return it, if it exists
         """
         try:
-            uri = self.uri + "/press-summary/1"
-            PressSummary = importlib.import_module("caselawclient.models.press_summaries").PressSummary
-            return PressSummary(uri, self.api_client)  # type: ignore
+            uri = DocumentURIString(self.uri + "/press-summary/1")
+            if not TYPE_CHECKING:  # This isn't nice, but will be cleaned up when we refactor how related documents work
+                PressSummary = importlib.import_module("caselawclient.models.press_summaries").PressSummary
+            return PressSummary(uri, self.api_client)
         except DocumentNotFoundError:
             return None

{ds_caselaw_marklogic_api_client-27.3.0 → ds_caselaw_marklogic_api_client-28.0.0}/src/caselawclient/models/press_summaries.py

@@ -9,7 +9,7 @@ from ds_caselaw_utils.types import NeutralCitationString
 from caselawclient.errors import DocumentNotFoundError
 from caselawclient.models.neutral_citation_mixin import NeutralCitationMixin
 
-from .documents import Document
+from .documents import Document, DocumentURIString
 
 if TYPE_CHECKING:
     from caselawclient.models.judgments import Judgment
@@ -23,8 +23,8 @@ class PressSummary(NeutralCitationMixin, Document):
     document_noun = "press summary"
     document_noun_plural = "press summaries"
 
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        super().__init__(self.document_noun, *args, **kwargs)
+    def __init__(self, uri: DocumentURIString, *args: Any, **kwargs: Any) -> None:
+        super().__init__(self.document_noun, uri, *args, **kwargs)
 
     @cached_property
     def neutral_citation(self) -> NeutralCitationString:
@@ -47,8 +47,9 @@ class PressSummary(NeutralCitationMixin, Document):
         Attempt to fetch a linked judgement, and return it, if it exists
         """
         try:
-            uri = self.uri.removesuffix("/press-summary/1")
-            Judgment = importlib.import_module("caselawclient.models.judgments").Judgment
-            return Judgment(uri, self.api_client)  # type: ignore
+            uri = DocumentURIString(self.uri.removesuffix("/press-summary/1"))
+            if not TYPE_CHECKING:  # This isn't nice, but will be cleaned up when we refactor how related documents work
+                Judgment = importlib.import_module("caselawclient.models.judgments").Judgment
+            return Judgment(uri, self.api_client)
         except DocumentNotFoundError:
             return None

{ds_caselaw_marklogic_api_client-27.3.0 → ds_caselaw_marklogic_api_client-28.0.0}/src/caselawclient/models/utilities/__init__.py

@@ -12,14 +12,14 @@ uk_namespace = {"uk": "https://caselaw.nationalarchives.gov.uk/akn"}
 
 
 class VersionsDict(TypedDict):
-    uri: str
+    uri: str  ## TODO: This should be either a MarkLogicDocumentURIString (raw from ML) or a DocumentURIString (and we parse it out). Just a str is too vague.
     version: int
 
 
 def render_versions(decoded_versions: list[BodyPart]) -> list[VersionsDict]:
     versions: list[VersionsDict] = [
         {
-            "uri": part.text.rstrip(".xml"),
+            "uri": part.text.strip("/").rstrip(".xml"),
             "version": extract_version(part.text),
         }
         for part in decoded_versions

{ds_caselaw_marklogic_api_client-27.3.0 → ds_caselaw_marklogic_api_client-28.0.0}/src/caselawclient/xquery/get_components_for_document.xqy

@@ -14,7 +14,7 @@ let $docTypeQuery := cts:element-attribute-value-query(
     )
 let $refQuery := cts:element-query(
       xs:QName("uk:summaryOf"),
-      concat("https://caselaw.nationalarchives.gov.uk/id", $parent_uri)
+      concat("https://caselaw.nationalarchives.gov.uk/id/", $parent_uri)
     )
 
 return xdmp:node-uri(cts:search(//akn:akomaNtoso, cts:and-query(($refQuery, $collectionQuery, $docTypeQuery))))

{ds_caselaw_marklogic_api_client-27.3.0 → ds_caselaw_marklogic_api_client-28.0.0}/src/caselawclient/xquery/get_pending_enrichment_for_version.xqy

@@ -11,20 +11,26 @@ xdmp:to-json(xdmp:sql(
   "SELECT process_data.uri, enrich_version_string, minutes_since_enrichment_request
   FROM (
     SELECT
+      propertysummary.published,
       process_data.uri,
       enrich_version_string, enrich_major_version, enrich_minor_version,
       parser_major_version, parser_minor_version,
       DATEDIFF('minute', last_sent_to_enrichment, CURRENT_TIMESTAMP) AS minutes_since_enrichment_request
     FROM documents.process_data
     JOIN documents.process_property_data ON process_data.uri = process_property_data.uri
+    JOIN documents.propertysummary ON process_data.uri = propertysummary.uri
   )
-  WHERE (
-    (enrich_version_string IS NULL) OR
-    (enrich_major_version <= @target_enrichment_major_version AND enrich_minor_version < @target_enrichment_minor_version)
-  ) AND (
-    (parser_major_version = @target_parser_major_version AND parser_minor_version = @target_parser_minor_version)
-  )
-  AND (minutes_since_enrichment_request > 43200 OR minutes_since_enrichment_request IS NULL)
+  WHERE 
+    (minutes_since_enrichment_request > 43200 OR minutes_since_enrichment_request IS NULL) AND
+    (propertysummary.published = 'true') AND
+    ( enrich_version_string IS NULL 
+      OR
+      (
+        (enrich_major_version <= @target_enrichment_major_version AND enrich_minor_version < @target_enrichment_minor_version)
+           AND 
+        (parser_major_version = @target_parser_major_version AND parser_minor_version = @target_parser_minor_version)
+      )
+    )
   ORDER BY enrich_major_version ASC NULLS FIRST, enrich_minor_version ASC
   LIMIT @maximum_records",
   "array",

ds_caselaw_marklogic_api_client-28.0.0/src/caselawclient/xquery/get_recently_enriched.xqy

@@ -0,0 +1,18 @@
+xquery version "1.0-ml";
+
+declare namespace xdmp="http://marklogic.com/xdmp"; 
+xdmp:to-json(xdmp:sql(
+  "SELECT *, process_data.uri, hours_since_enrichment_request, enrich_major_version, enrich_minor_version
+  FROM (
+    SELECT 
+      process_data.uri, enrich_major_version, enrich_minor_version,
+      DATEDIFF('hour', last_sent_to_enrichment, CURRENT_TIMESTAMP) AS hours_since_enrichment_request
+    FROM documents.process_data
+    JOIN documents.process_property_data ON process_data.uri = process_property_data.uri
+  )
+  ORDER BY hours_since_enrichment_request ASC
+  LIMIT 1000",
+  "array",
+  map:new((
+  ))
+))

ds_caselaw_marklogic_api_client-28.0.0/src/caselawclient/xquery/get_recently_parsed.xqy

@@ -0,0 +1,19 @@
+xquery version "1.0-ml";
+
+declare namespace xdmp="http://marklogic.com/xdmp"; 
+xdmp:to-json(xdmp:sql(
+  "SELECT process_data.uri, hours_since_parse_request, parser_major_version, parser_minor_version
+  FROM (
+    SELECT 
+      process_data.uri, parser_major_version, parser_minor_version,
+      DATEDIFF('hour', last_sent_to_parser, CURRENT_TIMESTAMP) AS hours_since_parse_request
+    FROM documents.process_data
+    JOIN documents.process_property_data ON process_data.uri = process_property_data.uri
+  )
+  ORDER BY hours_since_parse_request ASC
+  LIMIT 1000",
+  "array",
+  map:new((
+  ))
+))
+