pathling 7.0.1__tar.gz → 7.2.0.dev0__tar.gz

{pathling-7.0.1/pathling.egg-info → pathling-7.2.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pathling
-Version: 7.0.1
+Version: 7.2.0.dev0
 Summary: Python API for Pathling
 Home-page: https://github.com/aehrc/pathling
 Author: Australian e-Health Research Centre, CSIRO
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3.10
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pyspark<3.6.0,>=3.5.0
+Requires-Dist: pyspark<3.6.0,>=3.5.3
 Requires-Dist: deprecated>=1.2.13
 
 Python API for Pathling

pathling-7.2.0.dev0/examples/bulk.py

@@ -0,0 +1,149 @@
+#  Copyright 2025 Commonwealth Scientific and Industrial Research
+#  Organisation (CSIRO) ABN 41 687 119 230.
+# 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import tempfile
+from datetime import datetime, timezone
+
+from pathling import PathlingContext
+
+jwk = """
+{
+  "kty": "RSA",
+  "alg": "RS384",
+  "n": "llps-ufRYIVplRdtF2FB1xn5iCuojOyHmCmkLsMLU2zsUNcpwxlFUmv9xQRHTArfGm_vRkKjtx4dsW8LMG45EvDK_a6TmLF5H5hDlCqr0aXuInpN-c3f6f9d0zRtBCc18IKHL_IBskoaHGK4LVdQypIPcLqMiKkPFXI-NRwtJLUpQt6NH_p8vW0fiIRbkdC1t2pSrPX0307et38IE_vv8_RZm3CAKef2pnbWzRUBleeQybqaR28VNNallixegt1Sh5ShQLfQvA0QmrST25Kzs5K0d_6eAKl4xPDp1Q_dC1N4mygMZAkbRXKdq49Pg9C-56pbzEmvOYiM_CtMWkzr9w",
+  "e": "AQAB",
+  "d": "By9zHdqOSwqVLSbdc8yWFO2M21Ea0QFMyZzT19hCZk5CTOq7eDNw-KtoiU3XCm9KkjzfNoBgypOJ37zqz_m0iI8xZEY_j4CLxVLFiAMyCubfJo6pw1JvbQNjPIC45QXqsf_K7iOmqRqZfNnK63_MwKGSU1TW-oD505COIIOkNKjQ7KpIOm56EfyH2_cPUfmlHsBCRGy6eQ2M8cSK-uxXchSrNqt46nD8ArCE8qtrGJn1zJTgWOkH2lS73uzkc_P6rGg3IdiAbmPl4HWU-PlJ2jwykFbbXhnzL3Tpruc8okR_cda5u7KSa8dfV5WPjnygTxPHNt5_iuszPKxa0X9nwQ",
+  "p": "1DcRbY_DevTMMni3WynbKGm-MXmnH7NMU-4IU1hdegZfrStoBC2DngP77JILRO_TApaMPiAkIpxIpgvovnWKtCZ3-2BXDWnd4x_Ews3BUUVzCjvxAatLTiSq_lZTAL93Htqf3FQPa86Q2x4lyvJ-rFWBfpONzMGr-5g9ut1sGbk",
+  "q": "tV_kQ3ggaBSYRkckrpWnKJI3-uREyZVI_-PTK8kUS43Glz12sxVpYIIRqt57XtArkpHG9_YjUxj_ROF_LjSFaGbCxmccPqu9tHr7JIsuVWQlz8ooxXNW3lURMCtKd3k2xm9FhoFmtncP7nLbCfVaBIlTLhaXZXVZSSUv-vDDSy8",
+  "dp": "dVk-OeeVoRhdEkvOmIq8tcxDb_hlghIT0xV9ZRkoF6IOpiOqkSTZ8zcgx-C6epRjirrVMkVzte_V_Hv5Z9h3qsba8haEDNbN7BpVI6PDkr1kr_QVgWbHbZ65L4tsuq0lodojLCMPo_3F_GTfYSpXAdUGlofhkahHAgldmUd3z4E",
+  "dq": "O6MdHiYombBz5V_NKu6gORHjAEcAazv_9cvGirYiSzmB3AbkubvHm2kJQCLJdAKE4Tu3rZ6sPM2SWea_d8TjPNHVJ4GN4vl7dhWd8IUnJgK5ABrbzxi-rnpQHYOOh7w-i37Y4II58LMzdNclOKAJCkbRJ-1buIueYROuNBfoTxc",
+  "qi": "CPlT4vGuJbV-WMLIRL4c-VW0H0fwRUljqvv-_nNDQyZ98uFlXYLtmQS2h3VX4WjK1UR8Ca3m9110JNe8Va_7Tepuk13p4CyMG0ccGojzl50fvfrINj1zN6jz0lRI4cAPWdfGwgEs0tpvtW1saVrg9y89XefEx8Iq2Z0bLrlKGrU",
+  "key_ops": [
+    "sign"
+  ],
+  "ext": true,
+  "kid": "b2979595c62deb396306ba3edbdfb4a0"
+}
+"""
+
+client_id = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6InJlZ2lzdHJhdGlvbi10b2tlbiJ9.eyJqd2tzIjp7ImtleXMiOlt7Imt0eSI6IlJTQSIsImFsZyI6IlJTMzg0IiwibiI6ImxscHMtdWZSWUlWcGxSZHRGMkZCMXhuNWlDdW9qT3lIbUNta0xzTUxVMnpzVU5jcHd4bEZVbXY5eFFSSFRBcmZHbV92UmtLanR4NGRzVzhMTUc0NUV2REtfYTZUbUxGNUg1aERsQ3FyMGFYdUlucE4tYzNmNmY5ZDB6UnRCQ2MxOElLSExfSUJza29hSEdLNExWZFF5cElQY0xxTWlLa1BGWEktTlJ3dEpMVXBRdDZOSF9wOHZXMGZpSVJia2RDMXQycFNyUFgwMzA3ZXQzOElFX3Z2OF9SWm0zQ0FLZWYycG5iV3pSVUJsZWVReWJxYVIyOFZOTmFsbGl4ZWd0MVNoNVNoUUxmUXZBMFFtclNUMjVLenM1SzBkXzZlQUtsNHhQRHAxUV9kQzFONG15Z01aQWtiUlhLZHE0OVBnOUMtNTZwYnpFbXZPWWlNX0N0TVdrenI5dyIsImUiOiJBUUFCIiwia2V5X29wcyI6WyJ2ZXJpZnkiXSwiZXh0Ijp0cnVlLCJraWQiOiJiMjk3OTU5NWM2MmRlYjM5NjMwNmJhM2VkYmRmYjRhMCJ9LHsia3R5IjoiUlNBIiwiYWxnIjoiUlMzODQiLCJuIjoibGxwcy11ZlJZSVZwbFJkdEYyRkIxeG41aUN1b2pPeUhtQ21rTHNNTFUyenNVTmNwd3hsRlVtdjl4UVJIVEFyZkdtX3ZSa0tqdHg0ZHNXOExNRzQ1RXZES19hNlRtTEY1SDVoRGxDcXIwYVh1SW5wTi1jM2Y2ZjlkMHpSdEJDYzE4SUtITF9JQnNrb2FIR0s0TFZkUXlwSVBjTHFNaUtrUEZYSS1OUnd0SkxVcFF0Nk5IX3A4dlcwZmlJUmJrZEMxdDJwU3JQWDAzMDdldDM4SUVfdnY4X1JabTNDQUtlZjJwbmJXelJVQmxlZVF5YnFhUjI4Vk5OYWxsaXhlZ3QxU2g1U2hRTGZRdkEwUW1yU1QyNUt6czVLMGRfNmVBS2w0eFBEcDFRX2RDMU40bXlnTVpBa2JSWEtkcTQ5UGc5Qy01NnBiekVtdk9ZaU1fQ3RNV2t6cjl3IiwiZSI6IkFRQUIiLCJkIjoiQnk5ekhkcU9Td3FWTFNiZGM4eVdGTzJNMjFFYTBRRk15WnpUMTloQ1prNUNUT3E3ZUROdy1LdG9pVTNYQ205S2tqemZOb0JneXBPSjM3enF6X20waUk4eFpFWV9qNENMeFZMRmlBTXlDdWJmSm82cHcxSnZiUU5qUElDNDVRWHFzZl9LN2lPbXFScVpmTm5LNjNfTXdLR1NVMVRXLW9ENTA1Q09JSU9rTktqUTdLcElPbTU2RWZ5SDJfY1BVZm1sSHNCQ1JHeTZlUTJNOGNTSy11eFhjaFNyTnF0NDZuRDhBckNFOHF0ckdKbjF6SlRnV09rSDJsUzczdXprY19QNnJHZzNJZGlBYm1QbDRIV1UtUGxKMmp3eWtGYmJYaG56TDNUcHJ1Yzhva1JfY2RhNXU3S1NhOGRmVjVXUGpueWdUeFBITnQ1X2l1c3pQS3hhMFg5bndRIiwicCI6IjFEY1JiWV9EZXZUTU1uaTNXeW5iS0dtLU1YbW5IN05NVS00SVUxaGRlZ1pmclN0b0JDMkRuZ1A3N0pJTFJPX1RBcGFNUGlBa0lweElwZ3Zvdm5XS3RDWjMtMkJYRFduZDR4X0V3czNCVVVWekNqdnhBYXRMVGlTcV9sWlRBTDkzSHRxZjNGUVBhODZRMng0bHl2Si1yRldCZnBPTnpNR3ItNWc5dXQxc0diayIsInEiOiJ0Vl9rUTNnZ2FCU1lSa2NrcnBXbktKSTMtdVJFeVpWSV8tUFRLOGtVUzQzR2x6MTJzeFZwWUlJUnF0NTdYdEFya3BIRzlfWWpVeGpfUk9GX0xqU0ZhR2JDeG1jY1BxdTl0SHI3SklzdVZXUWx6OG9veFhOVzNsVVJNQ3RLZDNrMnhtOUZob0ZtdG5jUDduTGJDZlZhQklsVExoYVhaWFZaU1NVdi12RERTeTgiLCJkcCI6ImRWay1PZWVWb1JoZEVrdk9tSXE4dGN4RGJfaGxnaElUMHhWOVpSa29GNklPcGlPcWtTVFo4emNneC1DNmVwUmppcnJWTWtWenRlX1ZfSHY1WjloM3FzYmE4aGFFRE5iTjdCcFZJNlBEa3Ixa3JfUVZnV2JIYlo2NUw0dHN1cTBsb2RvakxDTVBvXzNGX0dUZllTcFhBZFVHbG9maGthaEhBZ2xkbVVkM3o0RSIsImRxIjoiTzZNZEhpWW9tYkJ6NVZfTkt1NmdPUkhqQUVjQWF6dl85Y3ZHaXJZaVN6bUIzQWJrdWJ2SG0ya0pRQ0xKZEFLRTRUdTNyWjZzUE0yU1dlYV9kOFRqUE5IVko0R040dmw3ZGhXZDhJVW5KZ0s1QUJyYnp4aS1ybnBRSFlPT2g3dy1pMzdZNElJNThMTXpkTmNsT0tBSkNrYlJKLTFidUl1ZVlST3VOQmZvVHhjIiwicWkiOiJDUGxUNHZHdUpiVi1XTUxJUkw0Yy1WVzBIMGZ3UlVsanF2di1fbk5EUXlaOTh1RmxYWUx0bVFTMmgzVlg0V2pLMVVSOENhM205MTEwSk5lOFZhXzdUZXB1azEzcDRDeU1HMGNjR29qemw1MGZ2ZnJJTmoxek42anowbFJJNGNBUFdkZkd3Z0VzMHRwdnRXMXNhVnJnOXk4OVhlZkV4OElxMlowYkxybEtHclUiLCJrZXlfb3BzIjpbInNpZ24iXSwiZXh0Ijp0cnVlLCJraWQiOiJiMjk3OTU5NWM2MmRlYjM5NjMwNmJhM2VkYmRmYjRhMCJ9XX0sImFjY2Vzc1Rva2Vuc0V4cGlyZUluIjoxNSwiaWF0IjoxNzQwMzY3MDU0fQ.avoHoKI9g_2fmoRxZB0QnscRgEqb9xHip9CU_f-2U1I"
+
+
+def test_bulk_exports():
+    # Initialize PathlingContext.
+    pc = PathlingContext.create()
+
+    # Base parameters from the demo server
+    fhir_server = "https://bulk-data.smarthealthit.org/fhir"
+    output_base = os.path.join(tempfile.gettempdir(), "bulk_export_test")
+    
+    if os.path.exists(output_base):
+        import shutil
+        shutil.rmtree(output_base)
+    os.makedirs(output_base)
+
+    # Test 1: System level export with all parameters.
+    print("\n=== Testing system level export with all parameters ===")
+    pc.read.bulk(
+        fhir_endpoint_url=fhir_server,
+        output_dir=f"{output_base}/system_detailed",
+        output_format="application/fhir+ndjson",
+        since=datetime(2015, 1, 1, tzinfo=timezone.utc),
+        types=["Patient", "Observation"],
+        elements=["id", "status"],
+        include_associated_data=["LatestProvenanceResources"],
+        type_filters=["Patient?status=active"],
+        output_extension="ndjson",
+        timeout=3600,
+        auth_config={
+            "enabled": True,
+            "client_id": client_id,
+            "private_key_jwk": jwk,
+            "use_smart": True,
+            "use_form_for_basic_auth": False,
+            "scope": "system/*.read",
+            "token_expiry_tolerance": 30
+        }
+    )
+    print("System export completed successfully")
+
+    # Test 2: Group level export with minimal parameters.
+    print("\n=== Testing group level export with minimal parameters ===")
+    pc.read.bulk(
+        fhir_endpoint_url=fhir_server,
+        output_dir=f"{output_base}/group_basic",
+        group_id="BMCHealthNet"
+    )
+    print("Group export completed successfully")
+
+    # Test 3: Group level export with all parameters.
+    print("\n=== Testing group level export with all parameters ===")
+    pc.read.bulk(
+        fhir_endpoint_url=fhir_server,
+        output_dir=f"{output_base}/group_detailed",
+        group_id="BMCHealthNet",
+        output_format="application/fhir+ndjson",
+        since=datetime(2015, 1, 1, tzinfo=timezone.utc),
+        types=["Patient", "Condition", "Observation"],
+        elements=["id", "status"],
+        include_associated_data=["LatestProvenanceResources"],
+        type_filters=["Patient?status=active"],
+        output_extension="ndjson",
+        timeout=1800,
+        max_concurrent_downloads=8
+    )
+    print("Group export completed successfully")
+
+    # Test 4: Patient level export with minimal parameters.
+    print("\n=== Testing patient level export with minimal parameters ===")
+    pc.read.bulk(
+        fhir_endpoint_url=fhir_server,
+        output_dir=f"{output_base}/patient_basic",
+        patients=[
+            "Patient/58c297c4-d684-4677-8024-01131d93835e",
+            "Patient/118616a4-f0b2-411f-8050-39d5d27c738c"
+        ]
+    )
+    print("Patient export completed successfully")
+
+    # Test 5: Patient level export with all parameters.
+    print("\n=== Testing patient level export with all parameters ===")
+    pc.read.bulk(
+        fhir_endpoint_url=fhir_server,
+        output_dir=f"{output_base}/patient_detailed",
+        patients=[
+            "Patient/58c297c4-d684-4677-8024-01131d93835e",
+            "Patient/118616a4-f0b2-411f-8050-39d5d27c738c",
+            "Patient/21fba439-ca79-411f-a081-37a432a78f3a"
+        ],
+        output_format="application/fhir+ndjson",
+        since=datetime(2020, 1, 1, tzinfo=timezone.utc),
+        types=["Observation", "MedicationRequest"],
+        elements=["id", "status", "code"],
+        include_associated_data=["LatestProvenanceResources"],
+        type_filters=["Observation?category=vital-signs"],
+        output_extension="ndjson",
+        timeout=2400,
+        max_concurrent_downloads=3
+    )
+    print("Patient export completed successfully")
+
+    print("\nAll bulk exports completed successfully!")
+    print(f"Output written to: {output_base}")
+
+
+if __name__ == "__main__":
+    test_bulk_exports()

{pathling-7.0.1 → pathling-7.2.0.dev0}/pathling/_version.py

@@ -2,8 +2,8 @@
 # Auto generated from POM project version.
 # Please do not modify.
 #
-__version__="7.0.1"
-__java_version__="7.0.1"
+__version__="7.2.0.dev0"
+__java_version__="7.2.0-SNAPSHOT"
 __scala_version__="2.12"
-__delta_version__="3.2.0"
+__delta_version__="3.3.0"
 __hadoop_version__="3.3.4"

pathling-7.2.0.dev0/pathling/bulk.py

@@ -0,0 +1,213 @@
+#  Copyright 2025 Commonwealth Scientific and Industrial Research
+#  Organisation (CSIRO) ABN 41 687 119 230.
+# 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from datetime import datetime
+from typing import List, Optional
+
+from pathling import PathlingContext
+
+
+class BulkExportClient:
+    """
+    A client for exporting data from the FHIR Bulk Data Access API.
+    """
+    def __init__(self, java_client):
+        """
+        Create a new BulkExportClient that wraps a Java BulkExportClient.
+        
+        :param java_client: The Java BulkExportClient instance to wrap
+        """
+        self._java_client = java_client
+
+    def export(self):
+        """
+        Export data from the FHIR server.
+        
+        :return: The result of the export operation
+        """
+        return self._java_client.export()
+
+    @classmethod
+    def create(cls, jvm, fhir_endpoint_url: str, output_dir: str,
+               group_id: Optional[str] = None,
+               patients: Optional[List[str]] = None,
+               output_format: str = "application/fhir+ndjson",
+               since: Optional[datetime] = None,
+               types: Optional[List[str]] = None, 
+               elements: Optional[List[str]] = None,
+               include_associated_data: Optional[List[str]] = None,
+               type_filters: Optional[List[str]] = None,
+               output_extension: str = "ndjson",
+               timeout: Optional[int] = None,
+               max_concurrent_downloads: int = 10,
+               auth_config: Optional[dict] = None) -> 'BulkExportClient':
+        """
+        Create a BulkExportClient for the appropriate export level based on the provided arguments.
+        
+        :param jvm: The JVM instance
+        :param fhir_endpoint_url: The URL of the FHIR server to export from
+        :param output_dir: The directory to write the output files to
+        :param group_id: Optional group ID for group-level export
+        :param patients: Optional list of patient IDs for patient-level export
+        :param output_format: The format of the output data
+        :param since: Only include resources modified after this timestamp
+        :param types: List of FHIR resource types to include
+        :param elements: List of FHIR elements to include
+        :param include_associated_data: Pre-defined set of FHIR resources to include
+        :param type_filters: FHIR search queries to filter resources
+        :param output_extension: File extension for output files
+        :param timeout: Optional timeout duration in seconds
+        :param max_concurrent_downloads: Maximum number of concurrent downloads
+        :param auth_config: Optional authentication configuration dictionary with the following possible keys:
+            - enabled: Whether authentication is enabled (default: False)
+            - client_id: The client ID to use for authentication
+            - private_key_jwk: The private key in JWK format
+            - client_secret: The client secret to use for authentication
+            - token_endpoint: The token endpoint URL
+            - use_smart: Whether to use SMART authentication (default: True)
+            - use_form_for_basic_auth: Whether to use form-based basic auth (default: False)
+            - scope: The scope to request
+            - token_expiry_tolerance: The token expiry tolerance in seconds (default: 120)
+        :return: A BulkExportClient configured for the appropriate export level
+        """
+        client_class = jvm.au.csiro.fhir.export.BulkExportClient
+        
+        # Determine the export level based on the provided arguments
+        if group_id is not None:
+            # Group-level export
+            builder = client_class.groupBuilder(group_id)
+        elif patients is not None and len(patients) > 0:
+            # Patient-level export
+            builder = client_class.patientBuilder()
+            for patient in patients:
+                ref = jvm.au.csiro.fhir.model.Reference.of(patient)
+                builder.withPatient(ref)
+        else:
+            # System-level export
+            builder = client_class.systemBuilder()
+        
+        # Configure the builder with common settings
+        cls._configure_builder(jvm, builder, fhir_endpoint_url, output_dir, output_format, 
+                              output_extension, max_concurrent_downloads, since, types, 
+                              elements, include_associated_data, type_filters, timeout, auth_config)
+        
+        # Build and return the client
+        return cls(builder.build())
+
+    @staticmethod
+    def _configure_builder(jvm, builder, fhir_endpoint_url: str, output_dir: str,
+                          output_format: str, output_extension: str, max_concurrent_downloads: int,
+                          since: Optional[datetime], types: Optional[List[str]],
+                          elements: Optional[List[str]], include_associated_data: Optional[List[str]],
+                          type_filters: Optional[List[str]], timeout: Optional[int],
+                          auth_config: Optional[dict]):
+        """
+        Configure the builder with common settings.
+        
+        :param jvm: The JVM instance
+        :param builder: The builder to configure
+        :param fhir_endpoint_url: The URL of the FHIR server
+        :param output_dir: Output directory
+        :param output_format: Output format
+        :param output_extension: File extension for output files
+        :param max_concurrent_downloads: Maximum number of concurrent downloads
+        :param since: Timestamp filter
+        :param types: Resource types to include
+        :param elements: Elements to include
+        :param include_associated_data: Associated data to include
+        :param type_filters: Resource filters
+        :param timeout: Optional timeout duration in seconds
+        :param auth_config: Optional authentication configuration
+        """
+        # Configure basic settings
+        builder.withFhirEndpointUrl(fhir_endpoint_url)
+        builder.withOutputDir(output_dir)
+        builder.withOutputFormat(output_format)
+        builder.withOutputExtension(output_extension)
+        builder.withMaxConcurrentDownloads(max_concurrent_downloads)
+        
+        # Configure timeout if provided
+        if timeout is not None:
+            java_duration = jvm.java.time.Duration.ofSeconds(timeout)
+            builder.withTimeout(java_duration)
+        
+        # Configure since timestamp if provided
+        if since is not None:
+            if since.tzinfo is None:
+                raise ValueError("datetime must include timezone information")
+            # Format with microsecond precision and timezone offset
+            instant_str = since.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3]  # Truncate to milliseconds
+            if since.utcoffset() is None:
+                instant_str += 'Z'
+            else:
+                offset = since.strftime('%z')
+                # Insert colon in timezone offset
+                instant_str += f"{offset[:3]}:{offset[3:]}"
+            java_instant = jvm.java.time.Instant.parse(instant_str)
+            builder.withSince(java_instant)
+        
+        # Configure resource types if provided
+        if types is not None:
+            for type_ in types:
+                builder.withType(type_)
+        
+        # Configure elements if provided
+        if elements is not None:
+            for element in elements:
+                builder.withElement(element)
+        
+        # Configure associated data if provided
+        if include_associated_data is not None:
+            for data in include_associated_data:
+                j_object = jvm.au.csiro.fhir.export.ws.AssociatedData.fromCode(data)
+                builder.withIncludeAssociatedDatum(j_object)
+        
+        # Configure type filters if provided
+        if type_filters is not None:
+            for filter_ in type_filters:
+                builder.withTypeFilter(filter_)
+        
+        # Configure authentication if provided
+        if auth_config is not None:
+            auth_builder = jvm.au.csiro.fhir.auth.AuthConfig.builder()
+            
+            # Set defaults to match Java class
+            auth_builder.enabled(False)
+            auth_builder.useSMART(True)
+            auth_builder.useFormForBasicAuth(False)
+            auth_builder.tokenExpiryTolerance(120)
+            
+            # Map Python config to Java builder methods
+            if 'enabled' in auth_config:
+                auth_builder.enabled(auth_config['enabled'])
+            if 'use_smart' in auth_config:
+                auth_builder.useSMART(auth_config['use_smart'])
+            if 'token_endpoint' in auth_config:
+                auth_builder.tokenEndpoint(auth_config['token_endpoint'])
+            if 'client_id' in auth_config:
+                auth_builder.clientId(auth_config['client_id'])
+            if 'client_secret' in auth_config:
+                auth_builder.clientSecret(auth_config['client_secret'])
+            if 'private_key_jwk' in auth_config:
+                auth_builder.privateKeyJWK(auth_config['private_key_jwk'])
+            if 'use_form_for_basic_auth' in auth_config:
+                auth_builder.useFormForBasicAuth(auth_config['use_form_for_basic_auth'])
+            if 'scope' in auth_config:
+                auth_builder.scope(auth_config['scope'])
+            if 'token_expiry_tolerance' in auth_config:
+                auth_builder.tokenExpiryTolerance(auth_config['token_expiry_tolerance'])
+            
+            auth_config_obj = auth_builder.build()
+            builder.withAuthConfig(auth_config_obj)

{pathling-7.0.1 → pathling-7.2.0.dev0}/pathling/datasink.py

@@ -16,7 +16,9 @@
 from typing import Callable, Optional
 
 from pathling.core import SparkConversionsMixin, StringMapper
-from pathling.datasource import DataSource
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from pathling.datasource import DataSource
 
 
 class ImportMode:
@@ -49,7 +51,7 @@ class DataSinks(SparkConversionsMixin):
     A class for writing FHIR data to a variety of different targets.
     """
 
-    def __init__(self, datasource: DataSource):
+    def __init__(self, datasource: "DataSource"):
         SparkConversionsMixin.__init__(self, datasource.spark)
         self._datasinks = (
             self.spark._jvm.au.csiro.pathling.library.io.sink.DataSinkBuilder(

{pathling-7.0.1 → pathling-7.2.0.dev0}/pathling/datasource.py

@@ -13,17 +13,25 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-
+from datetime import datetime
 from typing import Dict, Sequence, Optional, Callable
+from typing import List, TYPE_CHECKING
 
 from py4j.java_collections import SetConverter
 from py4j.java_gateway import JavaObject
 from pyspark.sql import DataFrame
 
 from pathling import PathlingContext
-from pathling.core import ExpOrStr, StringToStringSetMapper, SparkConversionsMixin
+from pathling.core import (
+    ExpOrStr,
+    StringToStringSetMapper,
+    SparkConversionsMixin,
+)
 from pathling.fhir import MimeType
 
+if TYPE_CHECKING:
+    from pathling.datasink import DataSinks
+
 
 class DataSource(SparkConversionsMixin):
     """
@@ -45,13 +53,21 @@ class DataSource(SparkConversionsMixin):
         """
         return self._wrap_df(self._jds.read(resource_code))
 
+    def resource_types(self):
+        """
+        Returns a list of the resource types that are available in the data source.
+        
+        :return: A list of strings representing the resource types.
+        """
+        return [r.toCode() for r in self._jds.getResourceTypes()]
+
     @property
     def write(self) -> "DataSinks":
         """
         Provides access to a :class:`DataSinks` object that can be used to persist data.
         """
+        # Import here to avoid circular dependency
         from pathling.datasink import DataSinks
-
         return DataSinks(self)
 
     def extract(
@@ -225,3 +241,75 @@ class DataSources(SparkConversionsMixin):
             return self._wrap_ds(self._jdataSources.tables(schema))
         else:
             return self._wrap_ds(self._jdataSources.tables())
+
+    def bulk(
+        self,
+        fhir_endpoint_url: str,
+        output_dir: str,
+        group_id: Optional[str] = None,
+        patients: Optional[List[str]] = None,
+        output_format: str = "application/fhir+ndjson",
+        since: Optional[datetime] = None,
+        types: Optional[List[str]] = None,
+        elements: Optional[List[str]] = None,
+        include_associated_data: Optional[List[str]] = None,
+        type_filters: Optional[List[str]] = None,
+        output_extension: str = "ndjson",
+        timeout: Optional[int] = None,
+        max_concurrent_downloads: int = 10,
+        auth_config: Optional[Dict] = None
+    ) -> DataSource:
+        """
+        Creates a data source from a FHIR Bulk Data Access API endpoint.
+
+        :param fhir_endpoint_url: The URL of the FHIR server to export from
+        :param output_dir: The directory to write the output files to
+        :param group_id: Optional group ID for group-level export
+        :param patients: Optional list of patient references for patient-level export
+        :param output_format: The format of the output data
+        :param since: Only include resources modified after this timestamp
+        :param types: List of FHIR resource types to include
+        :param elements: List of FHIR elements to include
+        :param include_associated_data: Pre-defined set of FHIR resources to include
+        :param type_filters: FHIR search queries to filter resources
+        :param output_extension: File extension for output files. Defaults to "ndjson"
+        :param timeout: Optional timeout duration in seconds
+        :param max_concurrent_downloads: Maximum number of concurrent downloads. Defaults to 10
+        :param auth_config: Optional authentication configuration dictionary with the following possible keys:
+            - enabled: Whether authentication is enabled (default: False)
+            - client_id: The client ID to use for authentication
+            - private_key_jwk: The private key in JWK format
+            - client_secret: The client secret to use for authentication
+            - token_endpoint: The token endpoint URL
+            - use_smart: Whether to use SMART authentication (default: True)
+            - use_form_for_basic_auth: Whether to use form-based basic auth (default: False)
+            - scope: The scope to request
+            - token_expiry_tolerance: The token expiry tolerance in seconds (default: 120)
+        :return: A DataSource object that can be used to run queries against the data
+        """
+        from pathling.bulk import BulkExportClient
+
+        # Create client using the simplified API
+        client = BulkExportClient.create(
+            self.spark._jvm,
+            fhir_endpoint_url=fhir_endpoint_url,
+            output_dir=output_dir,
+            group_id=group_id,
+            patients=patients,
+            output_format=output_format,
+            since=since,
+            types=types,
+            elements=elements,
+            include_associated_data=include_associated_data,
+            type_filters=type_filters,
+            output_extension=output_extension,
+            timeout=timeout,
+            max_concurrent_downloads=max_concurrent_downloads,
+            auth_config=auth_config
+        )
+
+        # Perform the export
+        client.export()
+
+        # Return a DataSource that reads from the exported files
+        return self.ndjson(output_dir)

{pathling-7.0.1 → pathling-7.2.0.dev0/pathling.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pathling
-Version: 7.0.1
+Version: 7.2.0.dev0
 Summary: Python API for Pathling
 Home-page: https://github.com/aehrc/pathling
 Author: Australian e-Health Research Centre, CSIRO
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3.10
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pyspark<3.6.0,>=3.5.0
+Requires-Dist: pyspark<3.6.0,>=3.5.3
 Requires-Dist: deprecated>=1.2.13
 
 Python API for Pathling

{pathling-7.0.1 → pathling-7.2.0.dev0}/pathling.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@ MANIFEST.in
 README.md
 setup.cfg
 setup.py
+examples/bulk.py
 examples/designation.py
 examples/display.py
 examples/encode_bundles.py
@@ -21,6 +22,7 @@ examples/data/resources/Condition.ndjson
 examples/data/resources/Patient.ndjson
 pathling/__init__.py
 pathling/_version.py
+pathling/bulk.py
 pathling/coding.py
 pathling/context.py
 pathling/core.py

pathling-7.2.0.dev0/pathling.egg-info/requires.txt

@@ -0,0 +1,2 @@
+pyspark<3.6.0,>=3.5.3
+deprecated>=1.2.13

{pathling-7.0.1 → pathling-7.2.0.dev0}/setup.py

@@ -62,7 +62,7 @@ setup(
     ],
     license="Apache License, version 2.0",
     python_requires=">=3.8",
-    install_requires=["pyspark>=3.5.0,<3.6.0", "deprecated>=1.2.13"],
+    install_requires=["pyspark>=3.5.3,<3.6.0", "deprecated>=1.2.13"],
     include_package_data=True,
     data_files=[
         ("share/pathling/examples", glob.glob("examples/*.py")),

pathling-7.0.1/pathling.egg-info/requires.txt

@@ -1,2 +0,0 @@
-pyspark<3.6.0,>=3.5.0
-deprecated>=1.2.13