pathling 8.0.0.dev3__tar.gz → 8.0.0.dev5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pathling-8.0.0.dev3/pathling.egg-info → pathling-8.0.0.dev5}/PKG-INFO +1 -1
- pathling-8.0.0.dev5/examples/bulk.py +149 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling/_version.py +1 -1
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling/datasource.py +1 -1
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5/pathling.egg-info}/PKG-INFO +1 -1
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/tests/test_datasource.py +20 -3
- pathling-8.0.0.dev3/examples/bulk.py +0 -151
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/LICENSE +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/MANIFEST.in +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/README.md +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/examples/data/bundles/Bennett146_Swaniawski813_704c9750-f6e6-473b-ee83-fbd48e07fe3f.json +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/examples/data/bundles/Dino214_Parisian75_40d82b80-b682-cd8b-da6d-396809878641.json +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/examples/data/resources/Condition.ndjson +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/examples/data/resources/Patient.ndjson +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/examples/designation.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/examples/display.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/examples/encode_bundles.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/examples/encode_resources.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/examples/fhir_view.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/examples/member_of.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/examples/property_of.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/examples/subsumes.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/examples/translate.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling/__init__.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling/bulk.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling/coding.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling/context.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling/core.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling/datasink.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling/fhir.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling/functions.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling/spark.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling/udfs.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling.egg-info/SOURCES.txt +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling.egg-info/dependency_links.txt +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling.egg-info/requires.txt +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/pathling.egg-info/top_level.txt +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/setup.cfg +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/setup.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/tests/test_bulk.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/tests/test_encoders.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/tests/test_functions.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/tests/test_spark.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/tests/test_udfs.py +0 -0
- {pathling-8.0.0.dev3 → pathling-8.0.0.dev5}/tests/test_view.py +0 -0
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
# Copyright 2025 Commonwealth Scientific and Industrial Research
|
|
2
|
+
# Organisation (CSIRO) ABN 41 687 119 230.
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
import tempfile
|
|
18
|
+
from datetime import datetime, timezone
|
|
19
|
+
|
|
20
|
+
from pathling import PathlingContext
|
|
21
|
+
|
|
22
|
+
jwk = """
|
|
23
|
+
{
|
|
24
|
+
"kty": "RSA",
|
|
25
|
+
"alg": "RS384",
|
|
26
|
+
"n": "llps-ufRYIVplRdtF2FB1xn5iCuojOyHmCmkLsMLU2zsUNcpwxlFUmv9xQRHTArfGm_vRkKjtx4dsW8LMG45EvDK_a6TmLF5H5hDlCqr0aXuInpN-c3f6f9d0zRtBCc18IKHL_IBskoaHGK4LVdQypIPcLqMiKkPFXI-NRwtJLUpQt6NH_p8vW0fiIRbkdC1t2pSrPX0307et38IE_vv8_RZm3CAKef2pnbWzRUBleeQybqaR28VNNallixegt1Sh5ShQLfQvA0QmrST25Kzs5K0d_6eAKl4xPDp1Q_dC1N4mygMZAkbRXKdq49Pg9C-56pbzEmvOYiM_CtMWkzr9w",
|
|
27
|
+
"e": "AQAB",
|
|
28
|
+
"d": "By9zHdqOSwqVLSbdc8yWFO2M21Ea0QFMyZzT19hCZk5CTOq7eDNw-KtoiU3XCm9KkjzfNoBgypOJ37zqz_m0iI8xZEY_j4CLxVLFiAMyCubfJo6pw1JvbQNjPIC45QXqsf_K7iOmqRqZfNnK63_MwKGSU1TW-oD505COIIOkNKjQ7KpIOm56EfyH2_cPUfmlHsBCRGy6eQ2M8cSK-uxXchSrNqt46nD8ArCE8qtrGJn1zJTgWOkH2lS73uzkc_P6rGg3IdiAbmPl4HWU-PlJ2jwykFbbXhnzL3Tpruc8okR_cda5u7KSa8dfV5WPjnygTxPHNt5_iuszPKxa0X9nwQ",
|
|
29
|
+
"p": "1DcRbY_DevTMMni3WynbKGm-MXmnH7NMU-4IU1hdegZfrStoBC2DngP77JILRO_TApaMPiAkIpxIpgvovnWKtCZ3-2BXDWnd4x_Ews3BUUVzCjvxAatLTiSq_lZTAL93Htqf3FQPa86Q2x4lyvJ-rFWBfpONzMGr-5g9ut1sGbk",
|
|
30
|
+
"q": "tV_kQ3ggaBSYRkckrpWnKJI3-uREyZVI_-PTK8kUS43Glz12sxVpYIIRqt57XtArkpHG9_YjUxj_ROF_LjSFaGbCxmccPqu9tHr7JIsuVWQlz8ooxXNW3lURMCtKd3k2xm9FhoFmtncP7nLbCfVaBIlTLhaXZXVZSSUv-vDDSy8",
|
|
31
|
+
"dp": "dVk-OeeVoRhdEkvOmIq8tcxDb_hlghIT0xV9ZRkoF6IOpiOqkSTZ8zcgx-C6epRjirrVMkVzte_V_Hv5Z9h3qsba8haEDNbN7BpVI6PDkr1kr_QVgWbHbZ65L4tsuq0lodojLCMPo_3F_GTfYSpXAdUGlofhkahHAgldmUd3z4E",
|
|
32
|
+
"dq": "O6MdHiYombBz5V_NKu6gORHjAEcAazv_9cvGirYiSzmB3AbkubvHm2kJQCLJdAKE4Tu3rZ6sPM2SWea_d8TjPNHVJ4GN4vl7dhWd8IUnJgK5ABrbzxi-rnpQHYOOh7w-i37Y4II58LMzdNclOKAJCkbRJ-1buIueYROuNBfoTxc",
|
|
33
|
+
"qi": "CPlT4vGuJbV-WMLIRL4c-VW0H0fwRUljqvv-_nNDQyZ98uFlXYLtmQS2h3VX4WjK1UR8Ca3m9110JNe8Va_7Tepuk13p4CyMG0ccGojzl50fvfrINj1zN6jz0lRI4cAPWdfGwgEs0tpvtW1saVrg9y89XefEx8Iq2Z0bLrlKGrU",
|
|
34
|
+
"key_ops": [
|
|
35
|
+
"sign"
|
|
36
|
+
],
|
|
37
|
+
"ext": true,
|
|
38
|
+
"kid": "b2979595c62deb396306ba3edbdfb4a0"
|
|
39
|
+
}
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
client_id = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6InJlZ2lzdHJhdGlvbi10b2tlbiJ9.eyJqd2tzIjp7ImtleXMiOlt7Imt0eSI6IlJTQSIsImFsZyI6IlJTMzg0IiwibiI6ImxscHMtdWZSWUlWcGxSZHRGMkZCMXhuNWlDdW9qT3lIbUNta0xzTUxVMnpzVU5jcHd4bEZVbXY5eFFSSFRBcmZHbV92UmtLanR4NGRzVzhMTUc0NUV2REtfYTZUbUxGNUg1aERsQ3FyMGFYdUlucE4tYzNmNmY5ZDB6UnRCQ2MxOElLSExfSUJza29hSEdLNExWZFF5cElQY0xxTWlLa1BGWEktTlJ3dEpMVXBRdDZOSF9wOHZXMGZpSVJia2RDMXQycFNyUFgwMzA3ZXQzOElFX3Z2OF9SWm0zQ0FLZWYycG5iV3pSVUJsZWVReWJxYVIyOFZOTmFsbGl4ZWd0MVNoNVNoUUxmUXZBMFFtclNUMjVLenM1SzBkXzZlQUtsNHhQRHAxUV9kQzFONG15Z01aQWtiUlhLZHE0OVBnOUMtNTZwYnpFbXZPWWlNX0N0TVdrenI5dyIsImUiOiJBUUFCIiwia2V5X29wcyI6WyJ2ZXJpZnkiXSwiZXh0Ijp0cnVlLCJraWQiOiJiMjk3OTU5NWM2MmRlYjM5NjMwNmJhM2VkYmRmYjRhMCJ9LHsia3R5IjoiUlNBIiwiYWxnIjoiUlMzODQiLCJuIjoibGxwcy11ZlJZSVZwbFJkdEYyRkIxeG41aUN1b2pPeUhtQ21rTHNNTFUyenNVTmNwd3hsRlVtdjl4UVJIVEFyZkdtX3ZSa0tqdHg0ZHNXOExNRzQ1RXZES19hNlRtTEY1SDVoRGxDcXIwYVh1SW5wTi1jM2Y2ZjlkMHpSdEJDYzE4SUtITF9JQnNrb2FIR0s0TFZkUXlwSVBjTHFNaUtrUEZYSS1OUnd0SkxVcFF0Nk5IX3A4dlcwZmlJUmJrZEMxdDJwU3JQWDAzMDdldDM4SUVfdnY4X1JabTNDQUtlZjJwbmJXelJVQmxlZVF5YnFhUjI4Vk5OYWxsaXhlZ3QxU2g1U2hRTGZRdkEwUW1yU1QyNUt6czVLMGRfNmVBS2w0eFBEcDFRX2RDMU40bXlnTVpBa2JSWEtkcTQ5UGc5Qy01NnBiekVtdk9ZaU1fQ3RNV2t6cjl3IiwiZSI6IkFRQUIiLCJkIjoiQnk5ekhkcU9Td3FWTFNiZGM4eVdGTzJNMjFFYTBRRk15WnpUMTloQ1prNUNUT3E3ZUROdy1LdG9pVTNYQ205S2tqemZOb0JneXBPSjM3enF6X20waUk4eFpFWV9qNENMeFZMRmlBTXlDdWJmSm82cHcxSnZiUU5qUElDNDVRWHFzZl9LN2lPbXFScVpmTm5LNjNfTXdLR1NVMVRXLW9ENTA1Q09JSU9rTktqUTdLcElPbTU2RWZ5SDJfY1BVZm1sSHNCQ1JHeTZlUTJNOGNTSy11eFhjaFNyTnF0NDZuRDhBckNFOHF0ckdKbjF6SlRnV09rSDJsUzczdXprY19QNnJHZzNJZGlBYm1QbDRIV1UtUGxKMmp3eWtGYmJYaG56TDNUcHJ1Yzhva1JfY2RhNXU3S1NhOGRmVjVXUGpueWdUeFBITnQ1X2l1c3pQS3hhMFg5bndRIiwicCI6IjFEY1JiWV9EZXZUTU1uaTNXeW5iS0dtLU1YbW5IN05NVS00SVUxaGRlZ1pmclN0b0JDMkRuZ1A3N0pJTFJPX1RBcGFNUGlBa0lweElwZ3Zvdm5XS3RDWjMtMkJYRFduZDR4X0V3czNCVVVWekNqdnhBYXRMVGlTcV9sWlRBTDkzSHRxZjNGUVBhODZRMng0bHl2Si1yRldCZnBPTnpNR3ItNWc5dXQxc0diayIsInEiOiJ0Vl9rUTNnZ2FCU1lSa2NrcnBXbktKSTMtdVJFeVpWSV8tUFRLOGtVUzQzR2x6MTJzeFZwWUlJUnF0NTdYdEFya3BIRzlfWWpVeGpfUk9GX0xqU0ZhR2JDeG1jY1BxdTl0SHI3SklzdVZXUWx6OG9veFhOVzNsVVJNQ3RLZDNrMnhtOUZob0ZtdG5jUDduTGJDZlZhQklsVExoYVhaWFZaU1NVdi12RERTeTgiLCJkcCI6ImRWay1PZWVWb1JoZEVrdk9tSXE4dGN4RGJfaGxnaElUMHhWOVpSa29GNklPcGlPcWtTVFo4emNneC1DNmVwUmppcnJWTWtWenRlX1ZfSHY1WjloM3FzYmE4aGFFRE5iTjdCcFZJNlBEa3Ixa3JfUVZnV2JIYlo2NUw0dHN1cTBsb2RvakxDTVBvXzNGX0dUZllTcFhBZFVHbG9maGthaEhBZ2xkbVVkM3o0RSIsImRxIjoiTzZNZEhpWW9tYkJ6NVZfTkt1NmdPUkhqQUVjQWF6dl85Y3ZHaXJZaVN6bUIzQWJrdWJ2SG0ya0pRQ0xKZEFLRTRUdTNyWjZzUE0yU1dlYV9kOFRqUE5IVko0R040dmw3ZGhXZDhJVW5KZ0s1QUJyYnp4aS1ybnBRSFlPT2g3dy1pMzdZNElJNThMTXpkTmNsT0tBSkNrYlJKLTFidUl1ZVlST3VOQmZvVHhjIiwicWkiOiJDUGxUNHZHdUpiVi1XTUxJUkw0Yy1WVzBIMGZ3UlVsanF2di1fbk5EUXlaOTh1RmxYWUx0bVFTMmgzVlg0V2pLMVVSOENhM205MTEwSk5lOFZhXzdUZXB1azEzcDRDeU1HMGNjR29qemw1MGZ2ZnJJTmoxek42anowbFJJNGNBUFdkZkd3Z0VzMHRwdnRXMXNhVnJnOXk4OVhlZkV4OElxMlowYkxybEtHclUiLCJrZXlfb3BzIjpbInNpZ24iXSwiZXh0Ijp0cnVlLCJraWQiOiJiMjk3OTU5NWM2MmRlYjM5NjMwNmJhM2VkYmRmYjRhMCJ9XX0sImFjY2Vzc1Rva2Vuc0V4cGlyZUluIjoxNSwiaWF0IjoxNzQwMzY3MDU0fQ.avoHoKI9g_2fmoRxZB0QnscRgEqb9xHip9CU_f-2U1I"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_bulk_exports():
|
|
46
|
+
# Initialize PathlingContext.
|
|
47
|
+
pc = PathlingContext.create()
|
|
48
|
+
|
|
49
|
+
# Base parameters from the demo server
|
|
50
|
+
fhir_server = "https://bulk-data.smarthealthit.org/fhir"
|
|
51
|
+
output_base = os.path.join(tempfile.gettempdir(), "bulk_export_test")
|
|
52
|
+
|
|
53
|
+
if os.path.exists(output_base):
|
|
54
|
+
import shutil
|
|
55
|
+
shutil.rmtree(output_base)
|
|
56
|
+
os.makedirs(output_base)
|
|
57
|
+
|
|
58
|
+
# Test 1: System level export with all parameters.
|
|
59
|
+
print("\n=== Testing system level export with all parameters ===")
|
|
60
|
+
pc.read.bulk(
|
|
61
|
+
fhir_endpoint_url=fhir_server,
|
|
62
|
+
output_dir=f"{output_base}/system_detailed",
|
|
63
|
+
output_format="application/fhir+ndjson",
|
|
64
|
+
since=datetime(2015, 1, 1, tzinfo=timezone.utc),
|
|
65
|
+
types=["Patient", "Observation"],
|
|
66
|
+
elements=["id", "status"],
|
|
67
|
+
include_associated_data=["LatestProvenanceResources"],
|
|
68
|
+
type_filters=["Patient?status=active"],
|
|
69
|
+
output_extension="ndjson",
|
|
70
|
+
timeout=3600,
|
|
71
|
+
auth_config={
|
|
72
|
+
"enabled": True,
|
|
73
|
+
"client_id": client_id,
|
|
74
|
+
"private_key_jwk": jwk,
|
|
75
|
+
"use_smart": True,
|
|
76
|
+
"use_form_for_basic_auth": False,
|
|
77
|
+
"scope": "system/*.read",
|
|
78
|
+
"token_expiry_tolerance": 30
|
|
79
|
+
}
|
|
80
|
+
)
|
|
81
|
+
print("System export completed successfully")
|
|
82
|
+
|
|
83
|
+
# Test 2: Group level export with minimal parameters.
|
|
84
|
+
print("\n=== Testing group level export with minimal parameters ===")
|
|
85
|
+
pc.read.bulk(
|
|
86
|
+
fhir_endpoint_url=fhir_server,
|
|
87
|
+
output_dir=f"{output_base}/group_basic",
|
|
88
|
+
group_id="BMCHealthNet"
|
|
89
|
+
)
|
|
90
|
+
print("Group export completed successfully")
|
|
91
|
+
|
|
92
|
+
# Test 3: Group level export with all parameters.
|
|
93
|
+
print("\n=== Testing group level export with all parameters ===")
|
|
94
|
+
pc.read.bulk(
|
|
95
|
+
fhir_endpoint_url=fhir_server,
|
|
96
|
+
output_dir=f"{output_base}/group_detailed",
|
|
97
|
+
group_id="BMCHealthNet",
|
|
98
|
+
output_format="application/fhir+ndjson",
|
|
99
|
+
since=datetime(2015, 1, 1, tzinfo=timezone.utc),
|
|
100
|
+
types=["Patient", "Condition", "Observation"],
|
|
101
|
+
elements=["id", "status"],
|
|
102
|
+
include_associated_data=["LatestProvenanceResources"],
|
|
103
|
+
type_filters=["Patient?status=active"],
|
|
104
|
+
output_extension="ndjson",
|
|
105
|
+
timeout=1800,
|
|
106
|
+
max_concurrent_downloads=8
|
|
107
|
+
)
|
|
108
|
+
print("Group export completed successfully")
|
|
109
|
+
|
|
110
|
+
# Test 4: Patient level export with minimal parameters.
|
|
111
|
+
print("\n=== Testing patient level export with minimal parameters ===")
|
|
112
|
+
pc.read.bulk(
|
|
113
|
+
fhir_endpoint_url=fhir_server,
|
|
114
|
+
output_dir=f"{output_base}/patient_basic",
|
|
115
|
+
patients=[
|
|
116
|
+
"Patient/58c297c4-d684-4677-8024-01131d93835e",
|
|
117
|
+
"Patient/118616a4-f0b2-411f-8050-39d5d27c738c"
|
|
118
|
+
]
|
|
119
|
+
)
|
|
120
|
+
print("Patient export completed successfully")
|
|
121
|
+
|
|
122
|
+
# Test 5: Patient level export with all parameters.
|
|
123
|
+
print("\n=== Testing patient level export with all parameters ===")
|
|
124
|
+
pc.read.bulk(
|
|
125
|
+
fhir_endpoint_url=fhir_server,
|
|
126
|
+
output_dir=f"{output_base}/patient_detailed",
|
|
127
|
+
patients=[
|
|
128
|
+
"Patient/58c297c4-d684-4677-8024-01131d93835e",
|
|
129
|
+
"Patient/118616a4-f0b2-411f-8050-39d5d27c738c",
|
|
130
|
+
"Patient/21fba439-ca79-411f-a081-37a432a78f3a"
|
|
131
|
+
],
|
|
132
|
+
output_format="application/fhir+ndjson",
|
|
133
|
+
since=datetime(2020, 1, 1, tzinfo=timezone.utc),
|
|
134
|
+
types=["Observation", "MedicationRequest"],
|
|
135
|
+
elements=["id", "status", "code"],
|
|
136
|
+
include_associated_data=["LatestProvenanceResources"],
|
|
137
|
+
type_filters=["Observation?category=vital-signs"],
|
|
138
|
+
output_extension="ndjson",
|
|
139
|
+
timeout=2400,
|
|
140
|
+
max_concurrent_downloads=3
|
|
141
|
+
)
|
|
142
|
+
print("Patient export completed successfully")
|
|
143
|
+
|
|
144
|
+
print("\nAll bulk exports completed successfully!")
|
|
145
|
+
print(f"Output written to: {output_base}")
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
if __name__ == "__main__":
|
|
149
|
+
test_bulk_exports()
|
|
@@ -57,7 +57,7 @@ class DataSource(SparkConversionsMixin):
|
|
|
57
57
|
|
|
58
58
|
:return: A list of strings representing the resource types.
|
|
59
59
|
"""
|
|
60
|
-
return
|
|
60
|
+
return list(self._jds.getResourceTypes())
|
|
61
61
|
|
|
62
62
|
@property
|
|
63
63
|
def write(self) -> "DataSinks":
|
|
@@ -17,10 +17,9 @@ import os
|
|
|
17
17
|
from tempfile import TemporaryDirectory
|
|
18
18
|
|
|
19
19
|
from flask import Response
|
|
20
|
-
from pyspark.sql import Row, DataFrame
|
|
21
|
-
from pytest import fixture
|
|
22
|
-
|
|
23
20
|
from pathling.datasource import DataSource
|
|
21
|
+
from pyspark.sql import DataFrame, Row
|
|
22
|
+
from pytest import fixture
|
|
24
23
|
|
|
25
24
|
|
|
26
25
|
@fixture(scope="function", autouse=True)
|
|
@@ -290,3 +289,21 @@ def parquet_query(data_source: DataSource) -> DataFrame:
|
|
|
290
289
|
|
|
291
290
|
def delta_query(data_source: DataSource) -> DataFrame:
|
|
292
291
|
return ndjson_query(data_source)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def test_datasource_resource_types(ndjson_test_data_dir, pathling_ctx):
|
|
295
|
+
"""Test that resource_types() returns a list of strings."""
|
|
296
|
+
data_source = pathling_ctx.read.ndjson(ndjson_test_data_dir)
|
|
297
|
+
|
|
298
|
+
# Call the resource_types method.
|
|
299
|
+
resource_types = data_source.resource_types()
|
|
300
|
+
|
|
301
|
+
# Verify it returns a list.
|
|
302
|
+
assert isinstance(resource_types, list)
|
|
303
|
+
|
|
304
|
+
# Verify the list contains strings.
|
|
305
|
+
assert all(isinstance(rt, str) for rt in resource_types)
|
|
306
|
+
|
|
307
|
+
# Verify expected resource types are present.
|
|
308
|
+
assert "Patient" in resource_types
|
|
309
|
+
assert "Condition" in resource_types
|
|
@@ -1,151 +0,0 @@
|
|
|
1
|
-
# Copyright 2025 Commonwealth Scientific and Industrial Research
|
|
2
|
-
# Organisation (CSIRO) ABN 41 687 119 230.
|
|
3
|
-
#
|
|
4
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
-
# you may not use this file except in compliance with the License.
|
|
6
|
-
# You may obtain a copy of the License at
|
|
7
|
-
#
|
|
8
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
-
#
|
|
10
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
-
# See the License for the specific language governing permissions and
|
|
14
|
-
# limitations under the License.
|
|
15
|
-
|
|
16
|
-
import os
|
|
17
|
-
import tempfile
|
|
18
|
-
from datetime import datetime, timezone
|
|
19
|
-
|
|
20
|
-
from pathling import PathlingContext
|
|
21
|
-
|
|
22
|
-
jwk = """
|
|
23
|
-
{
|
|
24
|
-
"kty": "RSA",
|
|
25
|
-
"alg": "RS384",
|
|
26
|
-
"n": "jcrw7Jio4RVAMlo2clxqkmT9nmg_w1pXhpChg0jp41fKKfDXAtlIRhL_Ij8_N71l5KVxNQWeNeGsO0op73Rj28HR885fxJ2jimYFyD0fsftjjHvYkV_GskFubhcURbHAvx3lVrwLFyILq8sydF2G48A-XSfVAHPE6yEimusRRNihPmbM-MDlBuQkLBtwnT0bDXUEIlpDvlPB30Im2QOgvYTsMAnI-MzemOAtF5Xe5wCsj27nityK5AlnAJLFXfeeFqySoIyR7FaaQ1eay40MV-ZyDULSPtV4C-58eh3V2SL-qkQEsfQSuu3rqb-lgOz1-gl4FqTIz2JGtpEsTM7Uww",
|
|
27
|
-
"e": "AQAB",
|
|
28
|
-
"d": "kwNFEgpaxeAeHTtrypSZoXjLM7u-YM3czV9w8huCrjSg1SSXgFykAJX6zT40BHJbMv8xhcgEQZBMub69vBqoAOirWPky5KiNMPG7VirlRDaGJSJDH-UJQzaUJCM3c-bYzQXpDE3rxBBkCXHJcJQabAkwDa8-4F26YFjWGqUMsFOE1sxTXPnJG8qBTYxTSFxnWNf6U_kbGOQlWtHd1TgxPjXzmU6H472igte6SZEATh9eyYgPJrAqnw4qRNGy5pnAHkuIrCHIMaktR34LKFHHl3_xsLSHo9QmPfEdR5soKIKQIph2KRYArx4U03larr7vbZMSOypLqBtoRVlzvx0h",
|
|
29
|
-
"p": "wmc-aV7SbViyP50B0s_6wrDlOjGid9kO7QePuohFLmJWuC8TP8VYeSBscCPf6gX40O8agiCrsBuz2ZUTZDlYBXPHRiYprdV11SgCXkfTw6-G5CD2Xjq43gcTzFOy2q2FlU5YtBkPVTrsYMH8p6F09sZRu-4rnCOpgoahbAawGXs",
|
|
30
|
-
"q": "urhDpiHoZj0SBjmfn8GTHNh3FoUE8xiG3s0e64xSIBE3PzXCmskZpJuKGqgPX-wSXer2_WtmJUzOCucajcd4HQp222PWMKhc1HVZMj4073XQKDGqe_M7ZH29RbS9x93zhNgvFFiSdubQTg9SHJXL3Ja0f3IYxReha13G9YDSG1k",
|
|
31
|
-
"dp": "thWN15QA9HpHOl4M_y_eZ8zYZ5Fl42tjF5Alh0lrwu5I22r8VJa7L3i3GLIBYGkHjGroIUoIhYLtCbcf2pf7Yd_3njTQhQmSvHwk-7m7F2aoqbRWDhxiW1O1r4QV2cz9ecNQQh_WxLXUASyxQTFxJFLM64FBR5X_h0oil9QLzVE",
|
|
32
|
-
"dq": "W_t8L_JSR1Ncdr6aWRwGOdaVS_25g3wYrNeFnOoiZvO0MKpuNMxOmp2Y-irCcDGelq-yfwMSbduZQRu6JBAYps3J4agcExpNqMgqaarlbvWt1q8o2ijnoEilHhq8xyIa3d2Vy8MaXAK2qU242KYeqIuBXas6cpWCip7G7ZhJaPk",
|
|
33
|
-
"qi": "bZvyduEpMUYyGXhd-MnHyKOiJtCUF_kbM0hUGr8AfJ6_bi8MEjBNMt5qZKGYYT6bXFJWiTTUFq6nZLmQJ7cY5lv57gAQOTlLy6hp-nqkNrH1P-5UAzEbUMhIdnPQcDEkBEjpfObHlwtrmaFFVKgpm5vqFFD-szMHPuZ43o0vS98",
|
|
34
|
-
"key_ops": [
|
|
35
|
-
"sign"
|
|
36
|
-
],
|
|
37
|
-
"ext": true,
|
|
38
|
-
"kid": "b31ab1cd8db2c39287b3267a2914600c"
|
|
39
|
-
}
|
|
40
|
-
"""
|
|
41
|
-
|
|
42
|
-
client_id = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6InJlZ2lzdHJhdGlvbi10b2tlbiJ9.eyJqd2tzIjp7ImtleXMiOlt7Imt0eSI6IlJTQSIsImFsZyI6IlJTMzg0IiwibiI6Impjcnc3SmlvNFJWQU1sbzJjbHhxa21UOW5tZ193MXBYaHBDaGcwanA0MWZLS2ZEWEF0bElSaExfSWo4X043MWw1S1Z4TlFXZU5lR3NPMG9wNzNSajI4SFI4ODVmeEoyamltWUZ5RDBmc2Z0ampIdllrVl9Hc2tGdWJoY1VSYkhBdngzbFZyd0xGeUlMcThzeWRGMkc0OEEtWFNmVkFIUEU2eUVpbXVzUlJOaWhQbWJNLU1EbEJ1UWtMQnR3blQwYkRYVUVJbHBEdmxQQjMwSW0yUU9ndllUc01BbkktTXplbU9BdEY1WGU1d0NzajI3bml0eUs1QWxuQUpMRlhmZWVGcXlTb0l5UjdGYWFRMWVheTQwTVYtWnlEVUxTUHRWNEMtNThlaDNWMlNMLXFrUUVzZlFTdXUzcnFiLWxnT3oxLWdsNEZxVEl6MkpHdHBFc1RNN1V3dyIsImUiOiJBUUFCIiwia2V5X29wcyI6WyJ2ZXJpZnkiXSwiZXh0Ijp0cnVlLCJraWQiOiJiMzFhYjFjZDhkYjJjMzkyODdiMzI2N2EyOTE0NjAwYyJ9LHsia3R5IjoiUlNBIiwiYWxnIjoiUlMzODQiLCJuIjoiamNydzdKaW80UlZBTWxvMmNseHFrbVQ5bm1nX3cxcFhocENoZzBqcDQxZktLZkRYQXRsSVJoTF9JajhfTjcxbDVLVnhOUVdlTmVHc08wb3A3M1JqMjhIUjg4NWZ4SjJqaW1ZRnlEMGZzZnRqakh2WWtWX0dza0Z1YmhjVVJiSEF2eDNsVnJ3TEZ5SUxxOHN5ZEYyRzQ4QS1YU2ZWQUhQRTZ5RWltdXNSUk5paFBtYk0tTURsQnVRa0xCdHduVDBiRFhVRUlscER2bFBCMzBJbTJRT2d2WVRzTUFuSS1NemVtT0F0RjVYZTV3Q3NqMjduaXR5SzVBbG5BSkxGWGZlZUZxeVNvSXlSN0ZhYVExZWF5NDBNVi1aeURVTFNQdFY0Qy01OGVoM1YyU0wtcWtRRXNmUVN1dTNycWItbGdPejEtZ2w0RnFUSXoySkd0cEVzVE03VXd3IiwiZSI6IkFRQUIiLCJkIjoia3dORkVncGF4ZUFlSFR0cnlwU1pvWGpMTTd1LVlNM2N6Vjl3OGh1Q3JqU2cxU1NYZ0Z5a0FKWDZ6VDQwQkhKYk12OHhoY2dFUVpCTXViNjl2QnFvQU9pcldQa3k1S2lOTVBHN1ZpcmxSRGFHSlNKREgtVUpRemFVSkNNM2MtYll6UVhwREUzcnhCQmtDWEhKY0pRYWJBa3dEYTgtNEYyNllGaldHcVVNc0ZPRTFzeFRYUG5KRzhxQlRZeFRTRnhuV05mNlVfa2JHT1FsV3RIZDFUZ3hQalh6bVU2SDQ3MmlndGU2U1pFQVRoOWV5WWdQSnJBcW53NHFSTkd5NXBuQUhrdUlyQ0hJTWFrdFIzNExLRkhIbDNfeHNMU0hvOVFtUGZFZFI1c29LSUtRSXBoMktSWUFyeDRVMDNsYXJyN3ZiWk1TT3lwTHFCdG9SVmx6dngwaCIsInAiOiJ3bWMtYVY3U2JWaXlQNTBCMHNfNndyRGxPakdpZDlrTzdRZVB1b2hGTG1KV3VDOFRQOFZZZVNCc2NDUGY2Z1g0ME84YWdpQ3JzQnV6MlpVVFpEbFlCWFBIUmlZcHJkVjExU2dDWGtmVHc2LUc1Q0QyWGpxNDNnY1R6Rk95MnEyRmxVNVl0QmtQVlRyc1lNSDhwNkYwOXNaUnUtNHJuQ09wZ29haGJBYXdHWHMiLCJxIjoidXJoRHBpSG9aajBTQmptZm44R1RITmgzRm9VRTh4aUczczBlNjR4U0lCRTNQelhDbXNrWnBKdUtHcWdQWC13U1hlcjJfV3RtSlV6T0N1Y2FqY2Q0SFFwMjIyUFdNS2hjMUhWWk1qNDA3M1hRS0RHcWVfTTdaSDI5UmJTOXg5M3poTmd2RkZpU2R1YlFUZzlTSEpYTDNKYTBmM0lZeFJlaGExM0c5WURTRzFrIiwiZHAiOiJ0aFdOMTVRQTlIcEhPbDRNX3lfZVo4ellaNUZsNDJ0akY1QWxoMGxyd3U1STIycjhWSmE3TDNpM0dMSUJZR2tIakdyb0lVb0loWUx0Q2JjZjJwZjdZZF8zbmpUUWhRbVN2SHdrLTdtN0YyYW9xYlJXRGh4aVcxTzFyNFFWMmN6OWVjTlFRaF9XeExYVUFTeXhRVEZ4SkZMTTY0RkJSNVhfaDBvaWw5UUx6VkUiLCJkcSI6IldfdDhMX0pTUjFOY2RyNmFXUndHT2RhVlNfMjVnM3dZck5lRm5Pb2ladk8wTUtwdU5NeE9tcDJZLWlyQ2NER2VscS15ZndNU2JkdVpRUnU2SkJBWXBzM0o0YWdjRXhwTnFNZ3FhYXJsYnZXdDFxOG8yaWpub0VpbEhocTh4eUlhM2QyVnk4TWFYQUsycVUyNDJLWWVxSXVCWGFzNmNwV0NpcDdHN1poSmFQayIsInFpIjoiYlp2eWR1RXBNVVl5R1hoZC1Nbkh5S09pSnRDVUZfa2JNMGhVR3I4QWZKNl9iaThNRWpCTk10NXFaS0dZWVQ2YlhGSldpVFRVRnE2blpMbVFKN2NZNWx2NTdnQVFPVGxMeTZocC1ucWtOckgxUC01VUF6RWJVTWhJZG5QUWNERWtCRWpwZk9iSGx3dHJtYUZGVktncG01dnFGRkQtc3pNSFB1WjQzbzB2Uzk4Iiwia2V5X29wcyI6WyJzaWduIl0sImV4dCI6dHJ1ZSwia2lkIjoiYjMxYWIxY2Q4ZGIyYzM5Mjg3YjMyNjdhMjkxNDYwMGMifV19LCJhY2Nlc3NUb2tlbnNFeHBpcmVJbiI6MTUsImlhdCI6MTc0MDMwNTQ3OH0.qI-820847HN1S37IGMVMKJRGeXQBrgbx91UZ7Av9djs"
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
def test_bulk_exports():
|
|
46
|
-
# Initialize PathlingContext.
|
|
47
|
-
pc = PathlingContext.create()
|
|
48
|
-
|
|
49
|
-
# Base parameters from the demo server
|
|
50
|
-
fhir_server = "https://bulk-data.smarthealthit.org/eyJlcnIiOiIiLCJwYWdlIjoxMDAwMCwidGx0IjoxNSwibSI6MSwiZGVsIjowLCJzZWN1cmUiOjEsIm9wcCI6MTB9/fhir"
|
|
51
|
-
output_base = os.path.join(tempfile.gettempdir(), "bulk_export_test")
|
|
52
|
-
|
|
53
|
-
if os.path.exists(output_base):
|
|
54
|
-
import shutil
|
|
55
|
-
shutil.rmtree(output_base)
|
|
56
|
-
os.makedirs(output_base)
|
|
57
|
-
|
|
58
|
-
# Test 1: System level export with all parameters.
|
|
59
|
-
print("\n=== Testing system level export with all parameters ===")
|
|
60
|
-
pc.read.bulk(
|
|
61
|
-
fhir_endpoint_url=fhir_server,
|
|
62
|
-
output_dir=f"{output_base}/system_detailed",
|
|
63
|
-
output_format="application/fhir+ndjson",
|
|
64
|
-
since=datetime(2015, 1, 1, tzinfo=timezone.utc),
|
|
65
|
-
types=["Patient", "Observation"],
|
|
66
|
-
elements=["id", "status"],
|
|
67
|
-
include_associated_data=["LatestProvenanceResources"],
|
|
68
|
-
type_filters=["Patient?status=active"],
|
|
69
|
-
output_extension="ndjson",
|
|
70
|
-
timeout=3600,
|
|
71
|
-
max_concurrent_downloads=5,
|
|
72
|
-
auth_config={
|
|
73
|
-
"enabled": True,
|
|
74
|
-
"client_id": client_id,
|
|
75
|
-
"private_key_jwk": jwk,
|
|
76
|
-
"token_endpoint": "https://bulk-data.smarthealthit.org/auth/token",
|
|
77
|
-
"use_smart": True,
|
|
78
|
-
"use_form_for_basic_auth": False,
|
|
79
|
-
"scope": "system/Patient.r system/Observation.r",
|
|
80
|
-
"token_expiry_tolerance": 120
|
|
81
|
-
}
|
|
82
|
-
)
|
|
83
|
-
print("System export completed successfully")
|
|
84
|
-
|
|
85
|
-
# Test 2: Group level export with minimal parameters.
|
|
86
|
-
print("\n=== Testing group level export with minimal parameters ===")
|
|
87
|
-
pc.read.bulk(
|
|
88
|
-
fhir_endpoint_url=fhir_server,
|
|
89
|
-
output_dir=f"{output_base}/group_basic",
|
|
90
|
-
group_id="BMCHealthNet"
|
|
91
|
-
)
|
|
92
|
-
print("Group export completed successfully")
|
|
93
|
-
|
|
94
|
-
# Test 3: Group level export with all parameters.
|
|
95
|
-
print("\n=== Testing group level export with all parameters ===")
|
|
96
|
-
pc.read.bulk(
|
|
97
|
-
fhir_endpoint_url=fhir_server,
|
|
98
|
-
output_dir=f"{output_base}/group_detailed",
|
|
99
|
-
group_id="BMCHealthNet",
|
|
100
|
-
output_format="application/fhir+ndjson",
|
|
101
|
-
since=datetime(2015, 1, 1, tzinfo=timezone.utc),
|
|
102
|
-
types=["Patient", "Condition", "Observation"],
|
|
103
|
-
elements=["id", "status"],
|
|
104
|
-
include_associated_data=["LatestProvenanceResources"],
|
|
105
|
-
type_filters=["Patient?status=active"],
|
|
106
|
-
output_extension="ndjson",
|
|
107
|
-
timeout=1800,
|
|
108
|
-
max_concurrent_downloads=8
|
|
109
|
-
)
|
|
110
|
-
print("Group export completed successfully")
|
|
111
|
-
|
|
112
|
-
# Test 4: Patient level export with minimal parameters.
|
|
113
|
-
print("\n=== Testing patient level export with minimal parameters ===")
|
|
114
|
-
pc.read.bulk(
|
|
115
|
-
fhir_endpoint_url=fhir_server,
|
|
116
|
-
output_dir=f"{output_base}/patient_basic",
|
|
117
|
-
patients=[
|
|
118
|
-
"Patient/58c297c4-d684-4677-8024-01131d93835e",
|
|
119
|
-
"Patient/118616a4-f0b2-411f-8050-39d5d27c738c"
|
|
120
|
-
]
|
|
121
|
-
)
|
|
122
|
-
print("Patient export completed successfully")
|
|
123
|
-
|
|
124
|
-
# Test 5: Patient level export with all parameters.
|
|
125
|
-
print("\n=== Testing patient level export with all parameters ===")
|
|
126
|
-
pc.read.bulk(
|
|
127
|
-
fhir_endpoint_url=fhir_server,
|
|
128
|
-
output_dir=f"{output_base}/patient_detailed",
|
|
129
|
-
patients=[
|
|
130
|
-
"Patient/58c297c4-d684-4677-8024-01131d93835e",
|
|
131
|
-
"Patient/118616a4-f0b2-411f-8050-39d5d27c738c",
|
|
132
|
-
"Patient/21fba439-ca79-411f-a081-37a432a78f3a"
|
|
133
|
-
],
|
|
134
|
-
output_format="application/fhir+ndjson",
|
|
135
|
-
since=datetime(2020, 1, 1, tzinfo=timezone.utc),
|
|
136
|
-
types=["Observation", "MedicationRequest"],
|
|
137
|
-
elements=["id", "status", "code"],
|
|
138
|
-
include_associated_data=["LatestProvenanceResources"],
|
|
139
|
-
type_filters=["Observation?category=vital-signs"],
|
|
140
|
-
output_extension="ndjson",
|
|
141
|
-
timeout=2400,
|
|
142
|
-
max_concurrent_downloads=3
|
|
143
|
-
)
|
|
144
|
-
print("Patient export completed successfully")
|
|
145
|
-
|
|
146
|
-
print("\nAll bulk exports completed successfully!")
|
|
147
|
-
print(f"Output written to: {output_base}")
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
if __name__ == "__main__":
|
|
151
|
-
test_bulk_exports()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|