pysof 0.2.0__tar.gz → 0.2.3__tar.gz

{pysof-0.2.0 → pysof-0.2.3}/Cargo.toml

@@ -17,13 +17,17 @@ crate-type = ["cdylib"]
 
 [dependencies]
 pyo3 = { version = ">=0.29", features = ["extension-module", "generate-import-lib"] }
-helios-sof = { path = "../sof", version = "0.2.0", default-features = false }
-helios-fhir = { path = "../fhir", version = "0.2.0", default-features = false }
+helios-sof = { path = "../sof", version = "0.2.3", default-features = false }
+helios-fhir = { path = "../fhir", version = "0.2.3", default-features = false }
 serde = { workspace = true }
 serde_json = { workspace = true }
 chrono = { version = "0.4", features = ["serde"] }
 pythonize = "0.29"
 csv = "1.3"
+# Drives helios-sof's async remote-resolve functions from the synchronous
+# PyO3 entry points via a current-thread runtime.
+tokio = { version = "1", default-features = false, features = ["rt", "net", "time"] }
+mimalloc = "0.1.52"
 
 [features]
 default = ["R4"]

{pysof-0.2.0 → pysof-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pysof
-Version: 0.2.0
+Version: 0.2.3
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -34,7 +34,7 @@ Project-URL: Source, https://github.com/HeliosSoftware/hfs/tree/main/crates/pyso
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Downloads](https://pepy.tech/badge/pysof)](https://pepy.tech/project/pysof)
 
-**High-performance FHIR data transformation for Python.** Transform FHIR resources into tabular formats (CSV, JSON, Parquet) using declarative ViewDefinitions from the [SQL on FHIR specification](https://build.fhir.org/ig/FHIR/sql-on-fhir-v2/).
+**High-performance FHIR data transformation for Python.** Transform FHIR resources into tabular formats (CSV, JSON, Parquet) using declarative ViewDefinitions from the [SQL on FHIR specification](http://hl7.org/fhir/uv/sql-on-fhir/).
 
 Built in Rust for speed, exposed to Python with a simple, Pythonic API. Part of the [Helios FHIR Server](https://github.com/HeliosSoftware/hfs) project.
 
@@ -46,7 +46,8 @@ Built in Rust for speed, exposed to Python with a simple, Pythonic API. Part of
 - 📦 **Streaming Support**: Memory-efficient chunked processing for large NDJSON files
 - 🌐 **Multi-Version FHIR**: Supports R4, R4B, R5, and R6 (based on build features)
 - 🎯 **Type-Safe**: Leverages Rust's type safety with a Pythonic interface
-- ⚡ **GIL-Free**: Python GIL released during processing for true parallelism
+- ⚡ **GIL-Friendly**: the GIL is released during parsing and processing, so other Python threads stay responsive
+- 📨 **Flexible Input**: ViewDefinitions and Bundles as Python dicts or pre-serialized JSON (`str`/`bytes`)
 
 ## 🎯 Why pysof?
 
@@ -176,6 +177,44 @@ result = pysof.run_view_definition_with_options(
 )
 ```
 
+### Remote `resolve()` (Trusted Servers)
+
+`resolve()` dereferences references against the input bundle by default. You can
+optionally let it fetch references that point at **explicitly trusted** FHIR
+servers and fold them into the resolution pool before rows are generated. This is
+**off by default** and gated by a strict allowlist (matching scheme + host + port
++ path-prefix; private/loopback addresses are blocked unless opted in).
+
+```python
+import pysof
+
+config = pysof.RemoteResolveConfig(
+    ["https://fhir.example.org/r4"],   # trusted base URLs (allowlist)
+    max_fetches=256,                    # per-run/-stream fetch cap
+    max_depth=1,                        # chained-reference rounds
+    allow_private_addresses=False,      # allow internal LBs (e.g. Traefik) by hostname
+    bearer_tokens={"fhir.example.org": "TOKEN"},  # optional per-host auth
+)
+
+# Or build it from SOF_RESOLVE_* environment variables:
+# config = pysof.RemoteResolveConfig.from_env()
+
+result = pysof.run_view_definition_remote(
+    view_definition, bundle, "json", config, fhir_version="R4"
+)
+
+# Streaming NDJSON with remote resolve() (one shared cache across chunks):
+stats = pysof.process_ndjson_to_file_remote(
+    view_definition, "input.ndjson", "output.json", "ndjson", config, chunk_size=1000
+)
+```
+
+When `config.is_active()` is `False` (disabled or empty allowlist) these behave
+exactly like their non-remote counterparts. Enabling remote resolution makes
+output depend on remote server state at run time. See the
+[SQL-on-FHIR guide](../../book/src/ch06-sql-on-fhir.md) for the full security
+model and configuration reference.
+
 ### Utility Functions
 
 ```python
@@ -276,18 +315,46 @@ pysof automatically processes FHIR resources in parallel using rayon:
 - **5-7x speedup** on typical batch workloads with multi-core CPUs
 - **Streaming benefits**: `ChunkedProcessor` and `process_ndjson_to_file` also use parallel processing
 - **Zero configuration** - parallelization is always enabled
-- **Python GIL released** during processing for true parallel execution
+- **Python GIL released** during parsing and processing, so other Python threads stay responsive
+
+Because each call already fans out across all cores via rayon's shared thread
+pool, running pysof from multiple Python threads does **not** multiply
+throughput — one call can saturate the machine. Use Python threads for
+concurrency (overlapping I/O, serving requests), not to speed up transforms.
+
+### Pass Pre-Serialized JSON for Maximum Speed
+
+If your data is already JSON text (from a file, an HTTP body, a database), pass
+the `str` or `bytes` directly instead of `json.loads`-ing it first:
+
+```python
+view_json: str = load_view_definition_text()
+bundle_json: bytes = fetch_bundle_bytes()
+
+result = pysof.run_view_definition(view_json, bundle_json, "csv")
+```
+
+This skips the Python-dict conversion entirely and parses the JSON with the
+GIL released — measured 20-27% faster per call on large bundles, with GIL hold
+time dropping from tens of milliseconds to microseconds.
 
 ### Performance Benchmarks
 
-| Mode | Dataset | Time | Memory | Notes |
-|------|---------|------|--------|-------|
-| **Batch** | 10k Patients | ~2.7s | 1.6 GB | All resources in memory |
-| **Streaming** | 10k Patients | ~0.9s | 45 MB | 35x less memory, 2.9x faster |
-| **Batch** | 93k Encounters | ~4s | 3.9 GB | All resources in memory |
-| **Streaming** | 93k Encounters | ~2.8s | 25 MB | 155x less memory, 1.4x faster |
+Measured with `scripts/bench_threading.py` (a 7-column Patient ViewDefinition;
+Windows 11, 12 logical CPUs, Python 3.11, release build with mimalloc):
+
+| Path | Dataset | Throughput |
+|------|---------|-----------|
+| **Batch** (dict input) | 2,000-patient bundle | ~28k patients/sec |
+| **Batch** (JSON `str` input) | 2,000-patient bundle | ~31k patients/sec |
+| **Streaming** (`process_ndjson_to_file`) | 5,000-patient NDJSON | ~31k patients/sec |
+
+Numbers vary with hardware and ViewDefinition complexity — re-run
+`scripts/bench_threading.py` on your target machine.
 
-Streaming mode (`ChunkedProcessor`, `process_ndjson_to_file`) is recommended for large NDJSON files.
+Streaming mode (`ChunkedProcessor`, `process_ndjson_to_file`) keeps memory
+bounded by `chunk_size` regardless of input size, so it is recommended for
+large NDJSON files; batch mode holds the whole bundle and result in memory.
 
 ### Controlling Thread Count (RAYON_NUM_THREADS)
 

{pysof-0.2.0/crates/pysof → pysof-0.2.3}/README.md

@@ -5,7 +5,7 @@
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Downloads](https://pepy.tech/badge/pysof)](https://pepy.tech/project/pysof)
 
-**High-performance FHIR data transformation for Python.** Transform FHIR resources into tabular formats (CSV, JSON, Parquet) using declarative ViewDefinitions from the [SQL on FHIR specification](https://build.fhir.org/ig/FHIR/sql-on-fhir-v2/).
+**High-performance FHIR data transformation for Python.** Transform FHIR resources into tabular formats (CSV, JSON, Parquet) using declarative ViewDefinitions from the [SQL on FHIR specification](http://hl7.org/fhir/uv/sql-on-fhir/).
 
 Built in Rust for speed, exposed to Python with a simple, Pythonic API. Part of the [Helios FHIR Server](https://github.com/HeliosSoftware/hfs) project.
 
@@ -17,7 +17,8 @@ Built in Rust for speed, exposed to Python with a simple, Pythonic API. Part of
 - 📦 **Streaming Support**: Memory-efficient chunked processing for large NDJSON files
 - 🌐 **Multi-Version FHIR**: Supports R4, R4B, R5, and R6 (based on build features)
 - 🎯 **Type-Safe**: Leverages Rust's type safety with a Pythonic interface
-- ⚡ **GIL-Free**: Python GIL released during processing for true parallelism
+- ⚡ **GIL-Friendly**: the GIL is released during parsing and processing, so other Python threads stay responsive
+- 📨 **Flexible Input**: ViewDefinitions and Bundles as Python dicts or pre-serialized JSON (`str`/`bytes`)
 
 ## 🎯 Why pysof?
 
@@ -147,6 +148,44 @@ result = pysof.run_view_definition_with_options(
 )
 ```
 
+### Remote `resolve()` (Trusted Servers)
+
+`resolve()` dereferences references against the input bundle by default. You can
+optionally let it fetch references that point at **explicitly trusted** FHIR
+servers and fold them into the resolution pool before rows are generated. This is
+**off by default** and gated by a strict allowlist (matching scheme + host + port
++ path-prefix; private/loopback addresses are blocked unless opted in).
+
+```python
+import pysof
+
+config = pysof.RemoteResolveConfig(
+    ["https://fhir.example.org/r4"],   # trusted base URLs (allowlist)
+    max_fetches=256,                    # per-run/-stream fetch cap
+    max_depth=1,                        # chained-reference rounds
+    allow_private_addresses=False,      # allow internal LBs (e.g. Traefik) by hostname
+    bearer_tokens={"fhir.example.org": "TOKEN"},  # optional per-host auth
+)
+
+# Or build it from SOF_RESOLVE_* environment variables:
+# config = pysof.RemoteResolveConfig.from_env()
+
+result = pysof.run_view_definition_remote(
+    view_definition, bundle, "json", config, fhir_version="R4"
+)
+
+# Streaming NDJSON with remote resolve() (one shared cache across chunks):
+stats = pysof.process_ndjson_to_file_remote(
+    view_definition, "input.ndjson", "output.json", "ndjson", config, chunk_size=1000
+)
+```
+
+When `config.is_active()` is `False` (disabled or empty allowlist) these behave
+exactly like their non-remote counterparts. Enabling remote resolution makes
+output depend on remote server state at run time. See the
+[SQL-on-FHIR guide](../../book/src/ch06-sql-on-fhir.md) for the full security
+model and configuration reference.
+
 ### Utility Functions
 
 ```python
@@ -247,18 +286,46 @@ pysof automatically processes FHIR resources in parallel using rayon:
 - **5-7x speedup** on typical batch workloads with multi-core CPUs
 - **Streaming benefits**: `ChunkedProcessor` and `process_ndjson_to_file` also use parallel processing
 - **Zero configuration** - parallelization is always enabled
-- **Python GIL released** during processing for true parallel execution
+- **Python GIL released** during parsing and processing, so other Python threads stay responsive
+
+Because each call already fans out across all cores via rayon's shared thread
+pool, running pysof from multiple Python threads does **not** multiply
+throughput — one call can saturate the machine. Use Python threads for
+concurrency (overlapping I/O, serving requests), not to speed up transforms.
+
+### Pass Pre-Serialized JSON for Maximum Speed
+
+If your data is already JSON text (from a file, an HTTP body, a database), pass
+the `str` or `bytes` directly instead of `json.loads`-ing it first:
+
+```python
+view_json: str = load_view_definition_text()
+bundle_json: bytes = fetch_bundle_bytes()
+
+result = pysof.run_view_definition(view_json, bundle_json, "csv")
+```
+
+This skips the Python-dict conversion entirely and parses the JSON with the
+GIL released — measured 20-27% faster per call on large bundles, with GIL hold
+time dropping from tens of milliseconds to microseconds.
 
 ### Performance Benchmarks
 
-| Mode | Dataset | Time | Memory | Notes |
-|------|---------|------|--------|-------|
-| **Batch** | 10k Patients | ~2.7s | 1.6 GB | All resources in memory |
-| **Streaming** | 10k Patients | ~0.9s | 45 MB | 35x less memory, 2.9x faster |
-| **Batch** | 93k Encounters | ~4s | 3.9 GB | All resources in memory |
-| **Streaming** | 93k Encounters | ~2.8s | 25 MB | 155x less memory, 1.4x faster |
+Measured with `scripts/bench_threading.py` (a 7-column Patient ViewDefinition;
+Windows 11, 12 logical CPUs, Python 3.11, release build with mimalloc):
+
+| Path | Dataset | Throughput |
+|------|---------|-----------|
+| **Batch** (dict input) | 2,000-patient bundle | ~28k patients/sec |
+| **Batch** (JSON `str` input) | 2,000-patient bundle | ~31k patients/sec |
+| **Streaming** (`process_ndjson_to_file`) | 5,000-patient NDJSON | ~31k patients/sec |
+
+Numbers vary with hardware and ViewDefinition complexity — re-run
+`scripts/bench_threading.py` on your target machine.
 
-Streaming mode (`ChunkedProcessor`, `process_ndjson_to_file`) is recommended for large NDJSON files.
+Streaming mode (`ChunkedProcessor`, `process_ndjson_to_file`) keeps memory
+bounded by `chunk_size` regardless of input size, so it is recommended for
+large NDJSON files; batch mode holds the whole bundle and result in memory.
 
 ### Controlling Thread Count (RAYON_NUM_THREADS)
 

{pysof-0.2.0 → pysof-0.2.3}/crates/fhir/Cargo.toml

@@ -24,9 +24,9 @@ xml = ["helios-serde-support/xml"]
 serde = { workspace = true }
 serde_json = { workspace = true, features = ["raw_value"] }
 clap = { version = "4.0", features = ["derive"] }
-helios-fhir-macro = { path = "../fhir-macro", version = "0.2.0" }
-helios-serde-support = { path = "../serde-support", version = "0.2.0" }
-helios-fhirpath-support = { path = "../fhirpath-support", version = "0.2.0" }
+helios-fhir-macro = { path = "../fhir-macro", version = "0.2.3" }
+helios-serde-support = { path = "../serde-support", version = "0.2.3" }
+helios-fhirpath-support = { path = "../fhirpath-support", version = "0.2.3" }
 time = "0.3"
 chrono = { workspace = true }
 # Re-add serde-with-arbitrary-precision, keep macros

pysof-0.2.3/crates/fhir/src/compartment/loader.rs

@@ -0,0 +1,178 @@
+//! CompartmentDefinition loader.
+//!
+//! Reads the FHIR spec `CompartmentDefinition` resources from the data
+//! directory, as stored-shape JSON ready to seed into storage. This is the
+//! compartment analogue of
+//! [`SearchParameterLoader::load_spec_resources`](crate::search::loader::SearchParameterLoader::load_spec_resources):
+//! the server seeds these into primary storage at startup so
+//! `GET /CompartmentDefinition` and the web UI read the same definitions the
+//! codegen'd membership table (`crate::get_compartment_params`) is built from.
+
+use std::path::Path;
+
+use serde_json::Value;
+
+use crate::FhirVersion;
+use crate::search::errors::LoaderError;
+
+/// Loader for CompartmentDefinition spec resources.
+pub struct CompartmentDefinitionLoader {
+    fhir_version: FhirVersion,
+}
+
+impl CompartmentDefinitionLoader {
+    /// Creates a new loader for the specified FHIR version.
+    pub fn new(fhir_version: FhirVersion) -> Self {
+        Self { fhir_version }
+    }
+
+    /// Returns the FHIR version.
+    pub fn version(&self) -> FhirVersion {
+        self.fhir_version
+    }
+
+    /// Returns the spec bundle filename for the configured FHIR version.
+    ///
+    /// Mirrors the `compartment-definitions-{version}.json` bundles shipped
+    /// under the data directory, alongside `search-parameters-{version}.json`.
+    #[allow(unreachable_patterns)]
+    pub fn spec_filename(&self) -> &'static str {
+        match self.fhir_version {
+            #[cfg(feature = "R4")]
+            FhirVersion::R4 => "compartment-definitions-r4.json",
+            #[cfg(feature = "R4B")]
+            FhirVersion::R4B => "compartment-definitions-r4b.json",
+            #[cfg(feature = "R5")]
+            FhirVersion::R5 => "compartment-definitions-r5.json",
+            #[cfg(feature = "R6")]
+            FhirVersion::R6 => "compartment-definitions-r6.json",
+            _ => "compartment-definitions-r4.json",
+        }
+    }
+
+    /// Returns the raw CompartmentDefinition resources from the spec bundle,
+    /// as stored-shape JSON.
+    ///
+    /// The seeding companion the server hands to storage: each entry carries
+    /// its bundle id (`patient`, `encounter`, …), so re-seeding is idempotent
+    /// via `create`'s `AlreadyExists`.
+    pub fn load_spec_resources(&self, data_dir: &Path) -> Result<Vec<Value>, LoaderError> {
+        let path = data_dir.join(self.spec_filename());
+        let content =
+            std::fs::read_to_string(&path).map_err(|e| LoaderError::ConfigLoadFailed {
+                path: path.display().to_string(),
+                message: e.to_string(),
+            })?;
+        let json: Value =
+            serde_json::from_str(&content).map_err(|e| LoaderError::ConfigLoadFailed {
+                path: path.display().to_string(),
+                message: format!("Invalid JSON: {}", e),
+            })?;
+
+        let mut resources = Vec::new();
+        if let Some(entries) = json.get("entry").and_then(|e| e.as_array()) {
+            for entry in entries {
+                if let Some(resource) = entry.get("resource")
+                    && resource.get("resourceType").and_then(|t| t.as_str())
+                        == Some("CompartmentDefinition")
+                {
+                    resources.push(resource.clone());
+                }
+            }
+        }
+        Ok(resources)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::path::PathBuf;
+
+    /// Repo-root `data/` directory, from this crate's manifest dir.
+    fn workspace_data_dir() -> PathBuf {
+        PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../data")
+    }
+
+    #[cfg(feature = "R4")]
+    #[test]
+    fn loads_r4_compartment_definitions() {
+        let loader = CompartmentDefinitionLoader::new(FhirVersion::R4);
+        let resources = loader.load_spec_resources(&workspace_data_dir()).unwrap();
+        // Device, Encounter, Patient, Practitioner, RelatedPerson (questionnaire
+        // has no code and is excluded from the bundle).
+        assert_eq!(resources.len(), 5);
+        assert!(
+            resources
+                .iter()
+                .all(|r| r["resourceType"] == "CompartmentDefinition")
+        );
+        assert!(resources.iter().any(|r| r["code"] == "Patient"));
+        assert!(
+            resources
+                .iter()
+                .all(|r| r.get("id").and_then(|i| i.as_str()).is_some())
+        );
+    }
+
+    #[test]
+    fn missing_dir_errors() {
+        let loader = CompartmentDefinitionLoader::new(FhirVersion::default_enabled());
+        assert!(
+            loader
+                .load_spec_resources(Path::new("/nonexistent/data/dir"))
+                .is_err()
+        );
+    }
+
+    /// The shipped `data/compartment-definitions-*.json` bundles must say
+    /// exactly what the codegen'd `get_compartment_params` table says — every
+    /// `(compartment, resource)` slot, every version. This is the guard against
+    /// the seeded copies drifting from `crates/fhir-gen/resources/`.
+    fn assert_parity(version: FhirVersion) {
+        let loader = CompartmentDefinitionLoader::new(version);
+        let defs = loader.load_spec_resources(&workspace_data_dir()).unwrap();
+        assert!(!defs.is_empty(), "{version:?} bundle is empty");
+        for def in &defs {
+            let code = def["code"].as_str().unwrap();
+            for resource in def["resource"].as_array().unwrap() {
+                let rtype = resource["code"].as_str().unwrap();
+                let params: Vec<&str> = resource
+                    .get("param")
+                    .and_then(|p| p.as_array())
+                    .map(|a| a.iter().filter_map(|v| v.as_str()).collect())
+                    .unwrap_or_default();
+                let runtime = crate::get_compartment_params(version, code, rtype);
+                assert_eq!(
+                    runtime,
+                    params.as_slice(),
+                    "{version:?} compartment {code} / {rtype}"
+                );
+            }
+        }
+    }
+
+    #[cfg(feature = "R4")]
+    #[test]
+    fn r4_bundle_matches_codegen_table() {
+        assert_parity(FhirVersion::R4);
+    }
+
+    #[cfg(feature = "R4B")]
+    #[test]
+    fn r4b_bundle_matches_codegen_table() {
+        assert_parity(FhirVersion::R4B);
+    }
+
+    #[cfg(feature = "R5")]
+    #[test]
+    fn r5_bundle_matches_codegen_table() {
+        assert_parity(FhirVersion::R5);
+    }
+
+    #[cfg(feature = "R6")]
+    #[test]
+    fn r6_bundle_matches_codegen_table() {
+        assert_parity(FhirVersion::R6);
+    }
+}

pysof-0.2.3/crates/fhir/src/compartment/mod.rs

@@ -0,0 +1,15 @@
+//! FHIR CompartmentDefinition spec data.
+//!
+//! Companion to [`crate::search`]: where SearchParameters drive search
+//! resolution, CompartmentDefinitions describe the compartment membership the
+//! REST compartment-search handler enforces (via the codegen'd
+//! [`crate::get_compartment_params`] table). This module only *loads* the spec
+//! resources so the server can seed them into storage — the source of truth the
+//! `GET /CompartmentDefinition` route and the web UI read from. Membership
+//! itself stays codegen-driven.
+//!
+//! - [`loader`] – [`CompartmentDefinitionLoader`] for reading the spec bundle.
+
+pub mod loader;
+
+pub use loader::CompartmentDefinitionLoader;