ds-caselaw-marklogic-api-client 43.1.0__py3-none-any.whl → 44.0.3__py3-none-any.whl

caselawclient/Client.py

@@ -1222,13 +1222,13 @@ class MarklogicApiClient:
 
         return (int(table[1][1]), int(table[1][2]))
 
-    def get_pending_parse_for_version(
+    def get_documents_pending_parse_for_version(
         self,
         target_version: tuple[int, int],
         maximum_records: int = 1000,
     ) -> list[list[Any]]:
-        """Retrieve documents which are not yet parsed with a given version."""
-        vars: query_dicts.GetPendingParseForVersionDict = {
+        """Retrieve a list of documents which are not yet parsed with a given version."""
+        vars: query_dicts.GetPendingParseForVersionDocumentsDict = {
             "target_major_version": target_version[0],
             "target_minor_version": target_version[1],
             "maximum_records": maximum_records,
@@ -1237,13 +1237,33 @@ class MarklogicApiClient:
             get_single_string_from_marklogic_response(
                 self._send_to_eval(
                     vars,
-                    "get_pending_parse_for_version.xqy",
+                    "get_pending_parse_for_version_documents.xqy",
                 ),
             ),
         )
 
         return results
 
+    def get_count_pending_parse_for_version(
+        self,
+        target_version: tuple[int, int],
+    ) -> int:
+        """Get the total number of documents which are not yet parsed with a given version."""
+        vars: query_dicts.GetPendingParseForVersionCountDict = {
+            "target_major_version": target_version[0],
+            "target_minor_version": target_version[1],
+        }
+        results = json.loads(
+            get_single_string_from_marklogic_response(
+                self._send_to_eval(
+                    vars,
+                    "get_pending_parse_for_version_count.xqy",
+                ),
+            ),
+        )
+
+        return int(results[1][0])
+
     def get_recently_parsed(
         self,
     ) -> list[list[Any]]:

caselawclient/factories.py

@@ -20,7 +20,7 @@ T = TypeVar("T")
 
 DEFAULT_DOCUMENT_BODY_XML = """<akomaNtoso xmlns="http://docs.oasis-open.org/legaldocml/ns/akn/3.0" xmlns:uk="https://caselaw.nationalarchives.gov.uk/akn">
             <judgment name="decision">
-                <meta/><header/>
+                <meta/><header><p>Header contains text</p></header>
                 <judgmentBody>
                 <decision>
                 <p>This is a document.</p>

caselawclient/models/documents/__init__.py

@@ -26,6 +26,7 @@ from caselawclient.models.utilities import VersionsDict, extract_version, render
 from caselawclient.models.utilities.aws import (
     ParserInstructionsDict,
     announce_document_event,
+    are_unpublished_assets_clean,
     check_docx_exists,
     delete_documents_from_private_bucket,
     generate_docx_url,
@@ -102,6 +103,11 @@ class Document:
             True,
             "There is another document with identical content",
         ),
+        (
+            "has_only_clean_assets",
+            True,
+            "An uncleaned asset exists for this document",
+        ),
     ]
     """
     A list of tuples in the form:
@@ -252,7 +258,7 @@ class Document:
         """
         if self.is_version:
             raise NotSupportedOnVersion(
-                "Cannot get versions of a version for {self.uri}",
+                f"Cannot get versions of a version for {self.uri}",
             )
         docs = []
         for version in self.versions:
@@ -370,6 +376,12 @@ class Document:
         """Check if the content hash of this document is unique compared to all other documents in MarkLogic."""
         return self.api_client.has_unique_content_hash(self.uri)
 
+    @cached_property
+    def has_only_clean_assets(self) -> bool:
+        """False if any non-tar.gz assets associated with this document have not been cleaned."""
+        return True  # TODO: Remove this once we have enabled the asset cleaning pipeline.
+        return are_unpublished_assets_clean(self.uri)
+
     @cached_property
     def version_created_datetime(self) -> datetime.datetime:
         return self.api_client.get_version_created_datetime(self.uri)

caselawclient/models/documents/body.py

@@ -177,14 +177,9 @@ class DocumentBody:
     def has_content(self) -> bool:
         """If we do not have a word document, the XML will not contain
         the contents of the judgment, but will contain a preamble."""
-        trailing_tags = self._xml.xml_as_tree.xpath("//*[preceding::akn:meta]", namespaces=DEFAULT_NAMESPACES)
-        for tag in trailing_tags:
-            if tag.tail and tag.tail.strip():
-                return True
-            if tag.text and tag.text.strip():
-                return True
-
-        return False
+        return bool(
+            self._xml.xml_as_tree.xpath("//akn:header[normalize-space(string(.))]", namespaces=DEFAULT_NAMESPACES)
+        )
 
     @cached_property
     def has_external_data(self) -> bool:

caselawclient/xquery/get_judgment.xqy

@@ -11,7 +11,7 @@ declare variable $version_uri as xs:string? external;
 declare variable $search_query as xs:string? external;
 
 (: Note that `xsl:output method` is changed from `html` to `xml` and we've namespaced the tag :)
-let $number_marks_xslt := (
+let $delete_meta_marks_xslt := (
   <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
                   xmlns:uk="https://caselaw.nationalarchives.gov.uk/akn"
                   xmlns:akn="http://docs.oasis-open.org/legaldocml/ns/akn/3.0"
@@ -25,16 +25,6 @@ let $number_marks_xslt := (
     <xsl:template match="//akn:meta//uk:mark">
           <xsl:apply-templates />
     </xsl:template>
-    <xsl:template match="uk:mark">
-      <xsl:copy>
-          <xsl:copy-of select="@*" />
-          <xsl:attribute name="id">
-              <xsl:text>mark_</xsl:text>
-              <xsl:number count="//uk:mark" level="any" from="//*[ancestor::akn:meta]" />
-          </xsl:attribute>
-          <xsl:apply-templates />
-      </xsl:copy>
-    </xsl:template>
   </xsl:stylesheet>
 )
 
@@ -56,7 +46,7 @@ let $raw_xml := if ($show_unpublished) then
 (: If a search query string is present, highlight instances :)
 let $transformed := if($search_query) then
       xdmp:xslt-eval(
-        $number_marks_xslt,
+        $delete_meta_marks_xslt,
         cts:highlight(
           $raw_xml,
           helper:make-q-query($search_query),

caselawclient/xquery/get_pending_parse_for_version_count.xqy

@@ -0,0 +1,26 @@
+xquery version "1.0-ml";
+
+declare variable $target_major_version as xs:int external;
+declare variable $target_minor_version as xs:int external;
+
+xdmp:to-json(xdmp:sql(
+  "SELECT COUNT(*) as count
+  FROM (
+    SELECT 
+      process_data.uri,
+      parser_version_string, parser_major_version, parser_minor_version,
+      DATEDIFF('minute', last_sent_to_parser, CURRENT_TIMESTAMP) AS minutes_since_parse_request
+    FROM documents.process_data
+    JOIN documents.process_property_data ON process_data.uri = process_property_data.uri
+  )
+  WHERE (
+    (parser_version_string IS NULL) OR
+    (parser_major_version <= @target_major_version AND parser_minor_version < @target_minor_version)
+  )
+  AND (minutes_since_parse_request > 43200 OR minutes_since_parse_request IS NULL)",
+  "array",
+  map:new((
+    map:entry("target_major_version", $target_major_version),
+    map:entry("target_minor_version", $target_minor_version)
+  ))
+))

caselawclient/xquery/{get_pending_parse_for_version.xqy → get_pending_parse_for_version_documents.xqy}

@@ -19,7 +19,7 @@ xdmp:to-json(xdmp:sql(
     (parser_major_version <= @target_major_version AND parser_minor_version < @target_minor_version)
   )
   AND (minutes_since_parse_request > 43200 OR minutes_since_parse_request IS NULL)
-  ORDER BY parser_major_version ASC NULLS FIRST, parser_minor_version ASC
+  ORDER BY parser_major_version ASC NULLS FIRST, parser_minor_version ASC, minutes_since_parse_request DESC
   LIMIT @maximum_records",
   "array",
   map:new((
@@ -28,4 +28,4 @@ xdmp:to-json(xdmp:sql(
     map:entry("maximum_records", $maximum_records)
     
   ))
-))
+))

caselawclient/xquery/xslt_transform.xqy

@@ -17,27 +17,6 @@ let $xsl_path := fn:concat("judgments/xslts/", $xsl_filename)
 
 let $params := map:map()
 
-let $number_marks_xslt := (
-  <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                  version="2.0">
-    <xsl:output method="html" />
-    <xsl:template match="@*|node()">
-      <xsl:copy>
-        <xsl:apply-templates select="@*|node()"/>
-      </xsl:copy>
-    </xsl:template>
-    <xsl:template match="mark">
-      <xsl:copy>
-          <xsl:copy-of select="*" />
-          <xsl:attribute name="id">
-              <xsl:text>mark_</xsl:text>
-              <xsl:value-of select="count(preceding::mark)"/>
-          </xsl:attribute>
-          <xsl:apply-templates />
-      </xsl:copy>
-    </xsl:template>
-  </xsl:stylesheet>
-)
 (: change the image-base of the document to match the location of the assets in $image_base
    so that references to images point to the correct places on the internet :)
 let $_put := map:put(
@@ -59,13 +38,10 @@ let $retrieved_value := if (xs:boolean($is_published) or $show_unpublished) then
         ()
 
 let $return_value := if($query) then
-      xdmp:xslt-eval(
-        $number_marks_xslt,
-        cts:highlight(
-          $retrieved_value,
-          helper:make-q-query($query),
-          <mark>{$cts:text}</mark>
-        )
+      cts:highlight(
+        $retrieved_value,
+        helper:make-q-query($query),
+        <mark>{$cts:text}</mark>
       )
     else
       $retrieved_value

caselawclient/xquery_type_dicts.py

@@ -105,8 +105,14 @@ class GetPendingEnrichmentForVersionDict(MarkLogicAPIDict):
     target_parser_minor_version: int
 
 
-# get_pending_parse_for_version.xqy
-class GetPendingParseForVersionDict(MarkLogicAPIDict):
+# get_pending_parse_for_version_count.xqy
+class GetPendingParseForVersionCountDict(MarkLogicAPIDict):
+    target_major_version: int
+    target_minor_version: int
+
+
+# get_pending_parse_for_version_documents.xqy
+class GetPendingParseForVersionDocumentsDict(MarkLogicAPIDict):
     maximum_records: Optional[int]
     target_major_version: int
     target_minor_version: int

{ds_caselaw_marklogic_api_client-43.1.0.dist-info → ds_caselaw_marklogic_api_client-44.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ds-caselaw-marklogic-api-client
-Version: 43.1.0
+Version: 44.0.3
 Summary: An API client for interacting with the underlying data in Find Caselaw.
 Keywords: national archives,caselaw
 Author: The National Archives
@@ -9,7 +9,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: boto3 (>=1.26.112,<2.0.0)
-Requires-Dist: certifi (>=2025.10.5,<2025.11.0)
+Requires-Dist: certifi (>=2025.11.12,<2025.12.0)
 Requires-Dist: charset-normalizer (>=3.0.0,<4.0.0)
 Requires-Dist: defusedxml (>=0.7.1,<0.8.0)
 Requires-Dist: django-environ (>=0.12.0)

{ds_caselaw_marklogic_api_client-43.1.0.dist-info → ds_caselaw_marklogic_api_client-44.0.3.dist-info}/RECORD

@@ -1,17 +1,17 @@
-caselawclient/Client.py,sha256=OZ7ee3QVvN59go2oHxndgRgAKSMaVAn-mmG7E9rpPgg,47415
+caselawclient/Client.py,sha256=aKe_SvSXpSIgHCc53DfQY6mgK4QyrUDqyPWLByzGxLI,48138
 caselawclient/__init__.py,sha256=QZtsOB_GR5XfFnWMJ6E9_fBany-JXFIrQmzs1mD_KVg,1225
 caselawclient/client_helpers/__init__.py,sha256=tpXWjwBAqOf8ChtSiEeMhdkiO7tVbfZ4FfQXsXaGJlI,1221
 caselawclient/client_helpers/search_helpers.py,sha256=R99HyRLeYHgsw2L3DOidEqlKLLvs6Tga5rKTuWQViig,1525
 caselawclient/content_hash.py,sha256=0cPC4OoABq0SC2wYFX9-24DodNigeOqksDxgxQH_hUA,2221
 caselawclient/errors.py,sha256=JC16fEGq_MRJX-_KFzfINCV2Cqx8o6OWOt3C16rQd84,3142
-caselawclient/factories.py,sha256=HXJeWpN7__X462joco07ziNqMOOMr-wUPJ91Y69gFk8,7466
+caselawclient/factories.py,sha256=5AiRrvtnvCkvHLU9SzD9MRZPKspEomFw9M2LDZjveJY,7501
 caselawclient/identifier_resolution.py,sha256=B5I1sD7o7YjzsXMECjbKjgiGLDda5bGhejsJ-lYpTIg,2429
 caselawclient/managers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 caselawclient/managers/merge/__init__.py,sha256=Rd6YIGifT3TP6UOf0gBrRoYzK5MJqTPeRaIJznS8dgI,2078
 caselawclient/managers/merge/checks.py,sha256=J9RBG6jZAKIZk27jdFq-BByoRLKWsViCfHDyA8ZM3qU,3205
 caselawclient/models/__init__.py,sha256=kd23EUpvaC7aLHdgk8farqKAQEx3lf7RvNT2jEatvlg,68
-caselawclient/models/documents/__init__.py,sha256=6OBZfMy_6FgLQ6ZnMvjKv1Poiq-UP7pb2TcxHx-uSTI,24705
-caselawclient/models/documents/body.py,sha256=pzk3bm9FGIWfI0Hs8dBuzk6RCiA9M4GHfgOYKpNlzyE,8455
+caselawclient/models/documents/__init__.py,sha256=nC1CoYUDVAlAgLT4vWdqmNj3yrpoUs7C73H588_PTSA,25185
+caselawclient/models/documents/body.py,sha256=IGRJcolJlkW_KWufNy4lLB97uPf9Ghhn6i_s0sptUrk,8286
 caselawclient/models/documents/comparison.py,sha256=KwFZQByOcYcZKe8csjAntttACKq4BZb28n2VeV5rK54,1355
 caselawclient/models/documents/exceptions.py,sha256=te7PPQTDHjZ9EYVg5pVaiZfF00lMBFy333PHj8_mkC4,443
 caselawclient/models/documents/statuses.py,sha256=Cp4dTQmJOtsU41EJcxy5dV1841pGD2PNWH0VrkDEv4Q,579
@@ -53,14 +53,15 @@ caselawclient/xquery/get_combined_stats_table.xqy,sha256=cclNqSzIB6sX6A_hgVOFZon
 caselawclient/xquery/get_components_for_document.xqy,sha256=qBOn5OI7ThK0OHizSm68oySfAdp2dsHFJaIMTI4iIC0,769
 caselawclient/xquery/get_highest_enrichment_version.xqy,sha256=a0dwVmEZuIMyRjIlvenSmbOaaN0WvgaCZvMtVWoLulQ,247
 caselawclient/xquery/get_highest_parser_version.xqy,sha256=LW3iSg4eWArbfBaCVWWOpr4MoUcDBz514nV48ElOsAM,247
-caselawclient/xquery/get_judgment.xqy,sha256=34fMHSBNPcXY3u3QIX3Bul8DMuWyGGHrhuYTzxGt3wk,2420
+caselawclient/xquery/get_judgment.xqy,sha256=YMDDTOvT2FaYXqhYPcJVNn95czDoDojauVJiSj1bVG0,2074
 caselawclient/xquery/get_judgment_checkout_status.xqy,sha256=mdY9UXLyzQdB7byEERPqentlr0YDLbXRVqH0h4UuZTQ,193
 caselawclient/xquery/get_judgment_version.xqy,sha256=wF9k9-CBrqo8VbxxyTrD-AGzR3-3jMm25tRVCjxPLrU,292
 caselawclient/xquery/get_last_modified.xqy,sha256=8fCm_7o_kkytCEmEeSTLWzLP7iOjuPV01IfHDgf6HaQ,172
 caselawclient/xquery/get_missing_fclid.xqy,sha256=FAZZMtqow0VAf1D9LjBydT9kcOxiEIQC7GQgs4o68sA,520
 caselawclient/xquery/get_next_document_sequence_number.xqy,sha256=LkGoaS7jZfaKDuZLi0apP5qHP1rpcM1HbqX3RUwquKY,450
 caselawclient/xquery/get_pending_enrichment_for_version.xqy,sha256=8J5Pi-jMXJd_BgtpK4g6C9uR99HP57JpFv5WkoPfNuo,2016
-caselawclient/xquery/get_pending_parse_for_version.xqy,sha256=9cjVZtHeBBjm-a7RMsn1PVJt_Ug78YFlmp5CN8VJ1jQ,1197
+caselawclient/xquery/get_pending_parse_for_version_count.xqy,sha256=8OrpwjfW3daG1wPox9Xyc5A6kqRfKFQZ8ogTqB8VIPA,928
+caselawclient/xquery/get_pending_parse_for_version_documents.xqy,sha256=g5r5Lnyi3r7eNOnX0eIYRTVBZjPd43KQRyrIAS8pIZY,1232
 caselawclient/xquery/get_properties_for_search_results.xqy,sha256=Tlv3EKwVV_q-JyQyhjWVHIleicPDpucxP4ScuQjpgSw,625
 caselawclient/xquery/get_property.xqy,sha256=RHlOTrK0aH-S7s_ykYzGmUeKOJxXlI4vE5sKRt556NY,209
 caselawclient/xquery/get_property_as_node.xqy,sha256=7EXNgjVD1QugJ1621pvg8PdjBRIuh7GugwARv04TuBk,202
@@ -89,11 +90,11 @@ caselawclient/xquery/user_has_role.xqy,sha256=52YuFZnXqaDDJs-j_UanpqcLNEiw_m9xb0
 caselawclient/xquery/validate_all_documents.xqy,sha256=z_0YEXmRcZ-FaJM0ouKiTjdI4tLKQ4FTssRihR07qFk,156
 caselawclient/xquery/validate_document.xqy,sha256=PgaDcnqCRJPIVqfmWsNlXmCLNKd21qkJrvY1RtNP7eA,140
 caselawclient/xquery/xslt.xqy,sha256=w57wNijH3dkwHkpKeAxqjlghVflQwo8cq6jS_sm-erM,199
-caselawclient/xquery/xslt_transform.xqy,sha256=cccaFiGkCcvSfDv007UriZ3I4ak2nTLP1trRZdbOoS8,2462
-caselawclient/xquery_type_dicts.py,sha256=f4PM8yZi5RRMdL2lQ8tsLUs0aJjBa5chvd-VVj40fJY,6767
+caselawclient/xquery/xslt_transform.xqy,sha256=3X8f7u5kRXKRMwnfZ2AO60LS9F3Gi3mFp_MHrw95x0w,1745
+caselawclient/xquery_type_dicts.py,sha256=caNLrQBytQFxfdVs5gpSTQEo-FEldKITZDqZtITKWJQ,6950
 caselawclient/xslt/modify_xml_live.xsl,sha256=gNjwBun2-UzOeeuf0wNjFtN3jXm1yrwqv_KT8r1slXw,2370
 caselawclient/xslt/sample.xsl,sha256=IG-v77stjwqiw25pguh391K-5DTKiX651WqILDZixm0,825
-ds_caselaw_marklogic_api_client-43.1.0.dist-info/LICENSE.md,sha256=fGMzyyLuQW-IAXUeDSCrRdsYW536aEWThdbpCjo6ZKg,1108
-ds_caselaw_marklogic_api_client-43.1.0.dist-info/METADATA,sha256=P-ddB6xJtOQa9vfR5Yl5iFIxXZL2mOgeLRg-YxVQRMY,4408
-ds_caselaw_marklogic_api_client-43.1.0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-ds_caselaw_marklogic_api_client-43.1.0.dist-info/RECORD,,
+ds_caselaw_marklogic_api_client-44.0.3.dist-info/LICENSE.md,sha256=fGMzyyLuQW-IAXUeDSCrRdsYW536aEWThdbpCjo6ZKg,1108
+ds_caselaw_marklogic_api_client-44.0.3.dist-info/METADATA,sha256=32qweXun6XgSOZhqDF4y-wKtPV9y90LrQsrL-IaSVv8,4409
+ds_caselaw_marklogic_api_client-44.0.3.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+ds_caselaw_marklogic_api_client-44.0.3.dist-info/RECORD,,