helix.fhir.client.sdk 4.2.3__py3-none-any.whl → 4.2.19__py3-none-any.whl

helix_fhir_client_sdk/graph/simulated_graph_processor_mixin.py

@@ -1,4 +1,5 @@
 import json
+import time
 from abc import ABC
 from collections.abc import AsyncGenerator
 from datetime import UTC, datetime
@@ -80,7 +81,6 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
         add_cached_bundles_to_result: bool = True,
         input_cache: RequestCache | None = None,
         compare_hash: bool = True,
-        make_persistent_connection: bool = False,
     ) -> AsyncGenerator[FhirGetResponse, None]:
         """
         Asynchronously simulate a FHIR $graph query with advanced processing capabilities.
@@ -120,11 +120,19 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
             add_cached_bundles_to_result: Optional flag to add cached bundles to result
             input_cache: Optional cache for resource retrieval
             compare_hash: Flag to compare resource hashes for changes
-            make_persistent_connection: Flag to make a persistent HTTP connection
 
         Yields:
             FhirGetResponse objects representing retrieved resources
         """
+
+        profiling: dict[str, Any] = {
+            "function": "process_simulate_graph_async",
+            "start_time": time.perf_counter(),
+            "steps": {},
+            "extend_calls": [],
+            "append_calls": [],
+        }
+
         # Validate graph definition input
         assert graph_json, "Graph JSON must be provided"
         graph_definition: GraphDefinition = GraphDefinition.from_dict(graph_json)
@@ -134,17 +142,6 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
         # Parse authentication scopes for resource access control
         scope_parser: FhirScopeParser = FhirScopeParser(scopes=auth_scopes)
 
-        persistent_session = None
-        if make_persistent_connection:
-            # Create a persistent HTTP session for reuse across all requests in this graph traversal
-            persistent_session = self.create_http_session()
-
-            # Store the origninal create_http_session method
-            original_create_http_session = self.create_http_session
-
-            # Override create_http_session to return the persistent session
-            self.create_http_session = lambda: persistent_session  # type: ignore[method-assign]
-
         # Ensure bundle resources are not separated by default
         self.separate_bundle_resources(False)
 
@@ -171,6 +168,7 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
         cache: RequestCache = input_cache if input_cache is not None else RequestCache()
         async with cache:
             # Retrieve start resources based on graph definition
+            step_start = time.perf_counter()
             start: str = graph_definition.start
             parent_response: FhirGetResponse
             cache_hits: int
@@ -184,10 +182,18 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                 add_cached_bundles_to_result=add_cached_bundles_to_result,
                 compare_hash=compare_hash,
             )
+            profiling["steps"]["get_start_resources"] = time.perf_counter() - step_start
 
             # If no parent resources found, yield empty response and exit
             parent_response_resource_count = parent_response.get_resource_count()
             if parent_response_resource_count == 0:
+                profiling["total_time"] = time.perf_counter() - profiling["start_time"]
+                if logger:
+                    logger.info(
+                        f"[PROFILING] process_simulate_graph_async: total={profiling['total_time']:.3f}s, "
+                        f"get_start_resources={profiling['steps'].get('get_start_resources', 0):.3f}s, "
+                        f"no parent resources found"
+                    )
                 yield parent_response
                 return  # no resources to process
 
@@ -209,6 +215,7 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                 )
 
             # now process the graph links
+            step_start = time.perf_counter()
             child_responses: list[FhirGetResponse] = []
             parent_link_map: list[tuple[list[GraphDefinitionLink], FhirBundleEntryList]] = []
 
@@ -217,6 +224,7 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                 parent_link_map.append((graph_definition.link, parent_bundle_entries))
 
             # Process graph links in parallel
+            link_processing_count = 0
             while len(parent_link_map):
                 new_parent_link_map: list[tuple[list[GraphDefinitionLink], FhirBundleEntryList]] = []
 
@@ -243,20 +251,38 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                         add_cached_bundles_to_result=add_cached_bundles_to_result,
                         ifModifiedSince=ifModifiedSince,
                     ):
+                        # Track extend operation
+                        extend_start = time.perf_counter()
                         child_responses.extend(link_responses)
+                        extend_time = time.perf_counter() - extend_start
+                        profiling["extend_calls"].append(
+                            {"location": "child_responses.extend", "count": len(link_responses), "time": extend_time}
+                        )
+                        link_processing_count += 1
 
                 # Update parent link map for next iteration
                 parent_link_map = new_parent_link_map
 
+            profiling["steps"]["process_graph_links"] = time.perf_counter() - step_start
+            profiling["steps"]["link_processing_iterations"] = link_processing_count
+
             # Combine and process responses
+            step_start = time.perf_counter()
             parent_response = cast(FhirGetBundleResponse, parent_response.extend(child_responses))
-            parent_response = parent_response.remove_duplicates()
+            extend_time = time.perf_counter() - step_start
+            profiling["steps"]["parent_response.extend"] = extend_time
+            profiling["extend_calls"].append(
+                {"location": "parent_response.extend", "count": len(child_responses), "time": extend_time}
+            )
 
             # Optional resource sorting
             if sort_resources:
+                step_start = time.perf_counter()
                 parent_response = parent_response.sort_resources()
+                profiling["steps"]["sort_resources"] = time.perf_counter() - step_start
 
             # Prepare final response based on bundling preferences
+            step_start = time.perf_counter()
             full_response: FhirGetResponse
             if separate_bundle_resources:
                 full_response = FhirGetListByResourceTypeResponse.from_response(other_response=parent_response)
@@ -264,10 +290,38 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                 full_response = FhirGetListResponse.from_response(other_response=parent_response)
             else:
                 full_response = parent_response
+            profiling["steps"]["prepare_final_response"] = time.perf_counter() - step_start
 
             # Set response URL
             full_response.url = url or parent_response.url
 
+            # Calculate profiling summary
+            profiling["total_time"] = time.perf_counter() - profiling["start_time"]
+            total_extend_time = sum(call["time"] for call in profiling["extend_calls"])
+            total_extend_count = sum(call["count"] for call in profiling["extend_calls"])
+
+            # Log profiling information
+            if logger:
+                logger.info(
+                    f"[PROFILING] process_simulate_graph_async for id={id_}: "
+                    f"total={profiling['total_time']:.3f}s, "
+                    f"get_start_resources={profiling['steps'].get('get_start_resources', 0):.3f}s, "
+                    f"process_graph_links={profiling['steps'].get('process_graph_links', 0):.3f}s, "
+                    f"parent_response.extend={profiling['steps'].get('parent_response.extend', 0):.3f}s, "
+                    f"sort_resources={profiling['steps'].get('sort_resources', 0):.3f}s, "
+                    f"prepare_final_response={profiling['steps'].get('prepare_final_response', 0):.3f}s"
+                )
+                logger.info(
+                    f"[PROFILING] process_simulate_graph_async extend operations: "
+                    f"total_calls={len(profiling['extend_calls'])}, "
+                    f"total_items={total_extend_count}, "
+                    f"total_time={total_extend_time:.3f}s"
+                )
+                for call in profiling["extend_calls"]:
+                    logger.info(
+                        f"[PROFILING]   extend at {call['location']}: items={call['count']}, time={call['time']:.3f}s"
+                    )
+
             # Log cache performance
             if logger:
                 logger.info(
@@ -280,13 +334,6 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
             # Yield the final response
             yield full_response
 
-        if persistent_session:
-            # Clean up: close the persistent session
-            await persistent_session.close()
-
-        if make_persistent_connection:
-            self.create_http_session = original_create_http_session  # type: ignore[method-assign]
-
     # noinspection PyUnusedLocal
     async def process_link_async_parallel_function(
         self,
@@ -297,26 +344,9 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
     ) -> list[FhirGetResponse]:
         """
         Parallel processing function for graph definition links.
-
-        This method is designed to be used with AsyncParallelProcessor to process
-        graph links concurrently, improving performance for complex FHIR resource
-        graph traversals.
-
-        Key Responsibilities:
-        - Process individual graph links in parallel
-        - Track and log processing details
-        - Handle resource retrieval for each link
-        - Manage parallel processing context
-
-        Args:
-            context: Parallel processing context information
-            row: Current GraphDefinitionLink being processed
-            parameters: Parameters for link processing
-            additional_parameters: Extra parameters for extended processing
-
-        Returns:
-            List of FhirGetResponse objects retrieved during link processing
         """
+        profiling_start = time.perf_counter()
+
         # Record the start time for performance tracking
         start_time: datetime = datetime.now()
 
@@ -347,11 +377,8 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
             logger=parameters.logger,
             cache=parameters.cache,
             scope_parser=parameters.scope_parser,
-            # Handle parent link map from additional parameters
             parent_link_map=(additional_parameters["parent_link_map"] if additional_parameters else []),
-            # Determine request size, default to 1 if not specified
             request_size=(additional_parameters["request_size"] if additional_parameters else 1),
-            # Track unsupported resources for ID-based search
             id_search_unsupported_resources=(
                 additional_parameters["id_search_unsupported_resources"] if additional_parameters else []
             ),
@@ -367,6 +394,8 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
         # Record end time for performance tracking
         end_time: datetime = datetime.now()
 
+        total_time = time.perf_counter() - profiling_start
+
         # Log detailed processing information
         if parameters.logger:
             parameters.logger.debug(
@@ -378,6 +407,11 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                 + f" | duration: {end_time - start_time}"
                 + f" | resource_count: {len(result)}"
             )
+            parameters.logger.info(
+                f"[PROFILING] process_link_async_parallel_function for path={row.path}: "
+                f"total={total_time:.3f}s, "
+                f"results={len(result)}"
+            )
 
         # Return the list of retrieved responses
         return result
@@ -863,9 +897,18 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
         logger: Logger | None,
         compare_hash: bool = True,
     ) -> FhirGetResponse | None:
+        profiling_start = time.perf_counter()
+        http_request_time = 0.0
+        http_request_count = 0
+        cache_check_time = 0.0
+        cache_update_time = 0.0
+        append_time = 0.0
+
         result: FhirGetResponse | None = None
         non_cached_id_list: list[str] = []
+
         # first check to see if we can find these in the cache
+        cache_check_start = time.perf_counter()
         if ids:
             for resource_id in ids:
                 cache_entry: RequestCacheEntry | None = await cache.get_async(
@@ -878,9 +921,12 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                     if logger:
                         logger.info(f"Cache entry not found for {resource_type}/{resource_id} (1by1)")
                     non_cached_id_list.append(resource_id)
+        cache_check_time = time.perf_counter() - cache_check_start
 
+        cache_update_start = time.perf_counter()
         for single_id in non_cached_id_list:
             result2: FhirGetResponse
+            http_start = time.perf_counter()
             async for result2 in self._get_with_session_async(
                 page_number=None,
                 ids=[single_id],
@@ -889,10 +935,15 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                 fn_handle_streaming_chunk=None,
                 resource_type=resource_type,
             ):
+                http_request_time += time.perf_counter() - http_start
+                http_request_count += 1
+
                 if result2.resource_type == "OperationOutcome":
                     result2 = FhirGetErrorResponse.from_response(other_response=result2)
                 if result:
+                    append_start = time.perf_counter()
                     result = result.append(result2)
+                    append_time += time.perf_counter() - append_start
                 else:
                     result = result2
                 if result2.successful:
@@ -926,6 +977,21 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                     )
                     if cache_updated and logger:
                         logger.info(f"Inserted {result2.status} for {resource_type}/{single_id} into cache (1by1)")
+        cache_update_time = time.perf_counter() - cache_update_start - http_request_time - append_time
+
+        total_time = time.perf_counter() - profiling_start
+        processing_time = total_time - http_request_time - cache_check_time - cache_update_time - append_time
+
+        if logger and http_request_count > 0:
+            logger.info(
+                f"[PROFILING] _get_resources_by_id_one_by_one_async for {resource_type}: "
+                f"total={total_time:.3f}s, "
+                f"http_requests={http_request_time:.3f}s ({http_request_count} calls), "
+                f"cache_check={cache_check_time:.3f}s, "
+                f"cache_update={cache_update_time:.3f}s, "
+                f"append={append_time:.3f}s, "
+                f"processing={processing_time:.3f}s"
+            )
 
         return result
 
@@ -942,6 +1008,13 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
         add_cached_bundles_to_result: bool = True,
         compare_hash: bool = True,
     ) -> tuple[FhirGetResponse, int]:
+        profiling_start = time.perf_counter()
+        http_request_time = 0.0
+        http_request_count = 0
+        cache_check_time = 0.0
+        cache_update_time = 0.0
+        append_time = 0.0
+
         assert resource_type
         if not scope_parser.scope_allows(resource_type=resource_type):
             if logger:
@@ -975,14 +1048,13 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
 
         non_cached_id_list: list[str] = []
         # get any cached resources
+        cache_check_start = time.perf_counter()
         if id_list:
             for resource_id in id_list:
                 cache_entry: RequestCacheEntry | None = await cache.get_async(
                     resource_type=resource_type, resource_id=resource_id
                 )
                 if cache_entry:
-                    # if there is an entry then it means we tried to get it in the past
-                    # so don't get it again whether we were successful or not
                     if logger:
                         logger.info(
                             f"{cache_entry.status} Returning {resource_type}/{resource_id} from cache (ByParam)"
@@ -991,6 +1063,7 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                     if logger:
                         logger.info(f"Cache entry not found for {resource_type}/{resource_id} (ByParam)")
                     non_cached_id_list.append(resource_id)
+        cache_check_time = time.perf_counter() - cache_check_start
 
         all_result: FhirGetResponse | None = None
         # either we have non-cached ids or this is a query without id but has other parameters
@@ -1002,6 +1075,7 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
             # call the server to get the resources
             result1: FhirGetResponse
             result: FhirGetResponse | None
+            http_start = time.perf_counter()
             async for result1 in self._get_with_session_async(
                 page_number=None,
                 ids=non_cached_id_list,
@@ -1010,6 +1084,8 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                 fn_handle_streaming_chunk=None,
                 resource_type=resource_type,
             ):
+                http_request_time += time.perf_counter() - http_start
+                http_request_count += 1
                 result = result1
                 # if we got a failure then check if we can get it one by one
                 if (not result or result.status != 200) and len(non_cached_id_list) > 1:
@@ -1022,6 +1098,7 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                                 f" Fetching one by one ids: {non_cached_id_list}"
                             )
                     # For some resources if search by _id doesn't work then fetch one by one.
+                    one_by_one_start = time.perf_counter()
                     result = await self._get_resources_by_id_one_by_one_async(
                         resource_type=resource_type,
                         ids=non_cached_id_list,
@@ -1030,6 +1107,9 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                         logger=logger,
                         compare_hash=compare_hash,
                     )
+                    one_by_one_time = time.perf_counter() - one_by_one_start
+                    http_request_time += one_by_one_time
+                    http_request_count += len(non_cached_id_list)
                 else:
                     if logger:
                         logger.info(f"Fetched {resource_type} resources using _id for url {self._url}")
@@ -1045,11 +1125,14 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
 
                     # append to the response
                     if all_result:
+                        append_start = time.perf_counter()
                         all_result = all_result.append(result)
+                        append_time += time.perf_counter() - append_start
                     else:
                         all_result = result
         # If non_cached_id_list is not empty and resource_type does not support ?_id search then fetch it one by one
         elif len(non_cached_id_list):
+            one_by_one_start = time.perf_counter()
             all_result = await self._get_resources_by_id_one_by_one_async(
                 resource_type=resource_type,
                 ids=non_cached_id_list,
@@ -1058,10 +1141,13 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                 logger=logger,
                 compare_hash=compare_hash,
             )
+            http_request_time += time.perf_counter() - one_by_one_start
+            http_request_count += len(non_cached_id_list)
 
         # This list tracks the non-cached ids that were found
         found_non_cached_id_list: list[str] = []
         # Cache the fetched entries
+        cache_update_start = time.perf_counter()
         if all_result:
             non_cached_bundle_entry: FhirBundleEntry
             for non_cached_bundle_entry in all_result.get_bundle_entries():
@@ -1095,7 +1181,6 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                             logger.debug(f"Inserted {resource_type}/{non_cached_resource_id} into cache (ByParam)")
                         found_non_cached_id_list.append(non_cached_resource_id)
 
-        # now add all the non-cached ids that were NOT found to the cache too so we don't look for them again
         for non_cached_id in non_cached_id_list:
             if non_cached_id not in found_non_cached_id_list:
                 cache_updated = await cache.add_async(
@@ -1110,6 +1195,7 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                 )
                 if cache_updated and logger:
                     logger.info(f"Inserted 404 for {resource_type}/{non_cached_id} into cache (ByParam)")
+        cache_update_time = time.perf_counter() - cache_update_start
 
         bundle_response: FhirGetBundleResponse = (
             FhirGetBundleResponse.from_response(other_response=all_result)
@@ -1151,6 +1237,21 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                 storage_mode=self._storage_mode,
             )
         )
+
+        total_time = time.perf_counter() - profiling_start
+        processing_time = total_time - http_request_time - cache_check_time - cache_update_time - append_time
+
+        if logger and http_request_count > 0:
+            logger.info(
+                f"[PROFILING] _get_resources_by_parameters_async for {resource_type}: "
+                f"total={total_time:.3f}s, "
+                f"http_requests={http_request_time:.3f}s ({http_request_count} calls), "
+                f"cache_check={cache_check_time:.3f}s, "
+                f"cache_update={cache_update_time:.3f}s, "
+                f"append={append_time:.3f}s, "
+                f"processing={processing_time:.3f}s"
+            )
+
         return bundle_response, cache.cache_hits
 
     # noinspection PyPep8Naming
@@ -1173,7 +1274,6 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
         add_cached_bundles_to_result: bool = True,
         input_cache: RequestCache | None = None,
         compare_hash: bool = True,
-        make_persistent_connection: bool = False,
     ) -> FhirGetResponse:
         """
         Simulates the $graph query on the FHIR server
@@ -1196,7 +1296,6 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
         :param add_cached_bundles_to_result: Optional flag to add cached bundles to result
         :param input_cache: Optional cache to use for input
         :param compare_hash: Optional flag to compare hash of the resources
-        :param make_persistent_connection: Whether to make the connection persistent for reuse across all requests in this graph traversal
         :return: FhirGetResponse
         """
         if contained:
@@ -1227,7 +1326,6 @@ class SimulatedGraphProcessorMixin(ABC, FhirClientProtocol):
                 add_cached_bundles_to_result=add_cached_bundles_to_result,
                 input_cache=input_cache,
                 compare_hash=compare_hash,
-                make_persistent_connection=make_persistent_connection,
             )
         )
         assert result, "No result returned from simulate_graph_async"

helix_fhir_client_sdk/open_telemetry/attribute_names.py

@@ -0,0 +1,7 @@
+class FhirClientSdkOpenTelemetryAttributeNames:
+    """Constants for OpenTelemetry attribute names used in the FHIR Client SDK."""
+
+    URL: str = "fhir.client_sdk.url"
+    RESOURCE: str = "fhir.client_sdk.resource"
+    JSON_DATA_COUNT: str = "fhir.client_sdk.json_data.count"
+    BATCH_SIZE: str = "fhir.client_sdk.batch.size"

helix_fhir_client_sdk/open_telemetry/span_names.py

@@ -0,0 +1,12 @@
+class FhirClientSdkOpenTelemetrySpanNames:
+    """Span names for OpenTelemetry tracing in the FHIR Client SDK."""
+
+    GET: str = "fhir.client_sdk.get"
+    GET_STREAMING: str = "fhir.client_sdk.streaming.get"
+    GET_ACCESS_TOKEN: str = "fhir.client_sdk.access_token.get"
+    HTTP_GET: str = "fhir.client_sdk.http.get"
+    HANDLE_RESPONSE: str = "fhir.client_sdk.handle_response"
+    DELETE: str = "fhir.client_sdk.delete"
+    UPDATE: str = "fhir.client_sdk.update"
+    PATCH: str = "fhir.client_sdk.patch"
+    MERGE: str = "fhir.client_sdk.merge"

helix_fhir_client_sdk/queue/request_queue_mixin.py

@@ -74,6 +74,15 @@ class RequestQueueMixin(ABC, FhirClientProtocol):
         request_id: str | None = None
         total_results = 0  # total number of resources returned so far
 
+        limit_count: int | None = self._limit
+
+        # if _count parameter is present in additional_parameters then set limit_count to it
+        if additional_parameters:
+            for param in additional_parameters:
+                if self._limit is None:
+                    if param.startswith("_count="):
+                        limit_count = int(param.split("=")[1])
+
         # create url and query to request from FHIR server
         resources_json: str = ""
         full_url = await self.build_url(
@@ -110,7 +119,8 @@ class RequestQueueMixin(ABC, FhirClientProtocol):
             )
 
             async with RetryableAioHttpClient(
-                fn_get_session=lambda: self.create_http_session(),
+                fn_get_session=self._fn_create_http_session or self.create_http_session,
+                caller_managed_session=self._fn_create_http_session is not None,
                 refresh_token_func=self._refresh_token_function,
                 tracer_request_func=self._trace_request_function,
                 retries=self._retry_count,
@@ -121,9 +131,6 @@ class RequestQueueMixin(ABC, FhirClientProtocol):
                 log_all_url_results=self._log_all_response_urls,
                 access_token=self._access_token,
                 access_token_expiry_date=self._access_token_expiry_date,
-                close_session_on_exit=self._close_session,
-                persistent_session=self._persistent_session,
-                use_persistent_session=self._use_persistent_session,
             ) as client:
                 while next_url:
                     # set access token in request if present
@@ -168,7 +175,7 @@ class RequestQueueMixin(ABC, FhirClientProtocol):
                     )
 
                     request_id = response.response_headers.get("X-Request-ID", None)
-                    self._internal_logger.info(f"X-Request-ID={request_id}")
+                    self._internal_logger.debug(f"X-Request-ID={request_id}")
 
                     async for r in FhirResponseProcessor.handle_response(
                         internal_logger=self._internal_logger,
@@ -198,9 +205,9 @@ class RequestQueueMixin(ABC, FhirClientProtocol):
                         total_results += resource_count
 
                         # Stop if limit reached
-                        if self._limit and total_results >= self._limit:
+                        if limit_count and total_results >= limit_count:
                             self._internal_logger.info(
-                                f"Reached limit={self._limit} after collecting {total_results} "
+                                f"Reached limit={limit_count} after collecting {total_results} "
                                 f"resources, stopping pagination"
                             )
                             return
@@ -281,7 +288,8 @@ class RequestQueueMixin(ABC, FhirClientProtocol):
             )
 
             async with RetryableAioHttpClient(
-                fn_get_session=lambda: self.create_http_session(),
+                fn_get_session=self._fn_create_http_session or self.create_http_session,
+                caller_managed_session=self._fn_create_http_session is not None,
                 refresh_token_func=self._refresh_token_function,
                 tracer_request_func=self._trace_request_function,
                 retries=self._retry_count,
@@ -292,9 +300,6 @@ class RequestQueueMixin(ABC, FhirClientProtocol):
                 log_all_url_results=self._log_all_response_urls,
                 access_token=self._access_token,
                 access_token_expiry_date=self._access_token_expiry_date,
-                close_session_on_exit=self._close_session,
-                persistent_session=self._persistent_session,
-                use_persistent_session=self._use_persistent_session,
             ) as client:
                 while next_url:
                     # set access token in request if present
@@ -339,7 +344,7 @@ class RequestQueueMixin(ABC, FhirClientProtocol):
                     )
 
                     request_id = response.response_headers.get("X-Request-ID", None)
-                    self._internal_logger.info(f"X-Request-ID={request_id}")
+                    self._internal_logger.debug(f"X-Request-ID={request_id}")
 
                     if response.status == 200:
                         response_next_url = None

helix_fhir_client_sdk/responses/fhir_client_protocol.py

@@ -1,5 +1,5 @@
 import uuid
-from collections.abc import AsyncGenerator
+from collections.abc import AsyncGenerator, Callable
 from datetime import datetime
 from logging import Logger
 from threading import Lock
@@ -89,10 +89,6 @@ class FhirClientProtocol(Protocol):
     _send_data_as_chunked: bool = False
     _last_page_lock: Lock
 
-    _persistent_session: ClientSession | None = None
-    _use_persistent_session: bool = False
-    _close_session: bool = True
-
     _use_post_for_search: bool = False
 
     _accept: str
@@ -124,12 +120,18 @@ class FhirClientProtocol(Protocol):
 
     _log_all_response_urls: bool
 
-    _storage_mode: CompressedDictStorageMode = CompressedDictStorageMode.raw()
+    _storage_mode: CompressedDictStorageMode
     """ storage mode to store the responses """
 
     _create_operation_outcome_for_error: bool | None
     """ whether to create OperationOutcome resource for errors """
 
+    _max_concurrent_requests: int | None
+    """ maximum number of concurrent requests to make to the FHIR server """
+
+    _fn_create_http_session: Callable[[], ClientSession] | None
+    """ optional callable to create HTTP sessions """
+
     async def get_access_token_async(self) -> GetAccessTokenResult: ...
 
     async def _send_fhir_request_async(
@@ -212,3 +214,5 @@ class FhirClientProtocol(Protocol):
     ) -> AsyncGenerator[FhirMergeResourceResponse, None]:
         # this is just here to tell Python this returns a generator
         yield None  # type: ignore[misc]
+
+    def use_http_session(self, fn_create_http_session: Callable[[], ClientSession] | None) -> "FhirClientProtocol": ...

helix_fhir_client_sdk/responses/fhir_get_response.py

@@ -119,7 +119,7 @@ class FhirGetResponse:
         """ Count of cache hits """
         self.results_by_url: list[RetryableAioHttpUrlResult] = results_by_url
         """ Count of errors in the response by status """
-        self.storage_mode: CompressedDictStorageMode = CompressedDictStorageMode.raw()
+        self.storage_mode: CompressedDictStorageMode = storage_mode
         """ Storage mode for the response """
 
     @abstractmethod
@@ -170,9 +170,8 @@ class FhirGetResponse:
         """
         result: FhirGetResponse = self._extend(others=others)
 
-        latest_chunk_number: list[int] = sorted([o.chunk_number for o in others if o.chunk_number], reverse=True)
-        if len(latest_chunk_number) > 0:
-            result.chunk_number = latest_chunk_number[0]
+        if others and others[-1].chunk_number:
+            result.chunk_number = others[-1].chunk_number
         return result
 
     @abstractmethod