cdiscbuilder 1.2.0__tar.gz → 1.2.1__tar.gz

{cdiscbuilder-1.2.0/src/cdiscbuilder.egg-info → cdiscbuilder-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cdiscbuilder
-Version: 1.2.0
+Version: 1.2.1
 Summary: A package to convert ODM XML to SDTM/ADaM Datasets
 Author-email: Ming-Chun Chen <hellomingchun@gmail.com>
 Requires-Python: >=3.9

{cdiscbuilder-1.2.0 → cdiscbuilder-1.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cdiscbuilder"
-version = "1.2.0"
+version = "1.2.1"
 description = "A package to convert ODM XML to SDTM/ADaM Datasets"
 readme = "README.md"
 authors = [{name = "Ming-Chun Chen", email = "hellomingchun@gmail.com"}]

{cdiscbuilder-1.2.0 → cdiscbuilder-1.2.1}/src/cdiscbuilder/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.2.0"
+__version__ = "1.2.1"
 
 from . import adam as adam
 from . import tlf as tlf

{cdiscbuilder-1.2.0 → cdiscbuilder-1.2.1}/src/cdiscbuilder/cli.py

@@ -30,10 +30,24 @@ def main():
 
     args = parser.parse_args()
 
+    # Load defaults from configs directory if present
+    defaults = {}
+    if args.configs and os.path.exists(args.configs):
+        defaults_path = os.path.join(args.configs, "defaults.yaml")
+        if os.path.exists(defaults_path):
+            import yaml
+            try:
+                with open(defaults_path, "r") as f:
+                    defaults = yaml.safe_load(f) or {}
+            except Exception as e:
+                print(f"Warning: Failed to load defaults.yaml from {defaults_path}: {e}")
+
+    xml_mapping = defaults.get("xml_mapping")
+
     # Step 1: ODM XML -> Long CSV
     print(f"--- Step 1: Parsing ODM XML from {args.xml} ---")
     try:
-        df = parse_odm_to_long_df(args.xml)
+        df = parse_odm_to_long_df(args.xml, xml_mapping=xml_mapping)
         print(f"Parsed {len(df)} rows.")
         df.to_csv(args.csv, index=False)
         print(f"Saved intermediate data to {args.csv}")

{cdiscbuilder-1.2.0 → cdiscbuilder-1.2.1}/src/cdiscbuilder/sdtm/engine/classes/findings.py

@@ -10,7 +10,7 @@ class FindingsProcessor:
     def __init__(self):
         self.class_name = "FINDINGS"
 
-    def process(self, domain_name, sources, df_long, default_keys):
+    def process(self, domain_name, sources, df_long, default_keys, custom_to_standard=None):
         domain_dfs = []
 
         for settings in sources:
@@ -20,9 +20,8 @@ class FindingsProcessor:
             if form_oid:
                 if "FormOID" not in source_df.columns:
                     print(
-                        f"Warning: 'FormOID' column missing in source data for domain {domain_name}"
+                        f"Warning: 'FormOID' column missing in source data. Skipping FormOID filtering."
                     )
-                    source_df = pd.DataFrame(columns=source_df.columns)
                 else:
                     if isinstance(form_oid, list):
                         source_df = source_df[source_df["FormOID"].isin(form_oid)]
@@ -34,9 +33,8 @@ class FindingsProcessor:
             if item_group_match:
                 if "ItemGroupOID" not in source_df.columns:
                     print(
-                        f"Warning: 'ItemGroupOID' column missing in source data for domain {domain_name}"
+                        f"Warning: 'ItemGroupOID' column missing in source data. Skipping ItemGroupOID filtering."
                     )
-                    source_df = pd.DataFrame(columns=source_df.columns)
                 else:
                     source_df = source_df[
                         source_df["ItemGroupOID"].str.match(item_group_match, na=False)
@@ -47,9 +45,8 @@ class FindingsProcessor:
             if item_oid_match:
                 if "ItemOID" not in source_df.columns:
                     print(
-                        f"Warning: 'ItemOID' column missing in source data for domain {domain_name}"
+                        f"Warning: 'ItemOID' column missing in source data. Skipping ItemOID filtering."
                     )
-                    source_df = pd.DataFrame(columns=source_df.columns)
                 else:
                     source_df = source_df[
                         source_df["ItemOID"].str.match(item_oid_match, na=False)
@@ -60,6 +57,8 @@ class FindingsProcessor:
 
             # 3. Create Base DataFrame (No Pivot)
             keys = settings.get("keys", default_keys)
+            if custom_to_standard:
+                keys = [custom_to_standard.get(k, k) for k in keys]
 
             base_cols = keys + ["ItemOID", "Value"]
             if "Question" in source_df.columns:

{cdiscbuilder-1.2.0 → cdiscbuilder-1.2.1}/src/cdiscbuilder/sdtm/engine/classes/general.py

@@ -59,7 +59,7 @@ class GeneralProcessor:
 
         return expanded_list
 
-    def process(self, domain_name, sources, df_long, default_keys):
+    def process(self, domain_name, sources, df_long, default_keys, custom_to_standard=None):
         domain_dfs = []
 
         # Pre-expand sources if they contain lists
@@ -76,25 +76,34 @@ class GeneralProcessor:
             form_oid = settings.get("formoid")
             if form_oid:
                 try:
-                    # Filter for specific FormOID(s)
-                    if isinstance(form_oid, list):
-                        source_df = df_long[df_long["FormOID"].isin(form_oid)].copy()
+                    if "FormOID" in df_long.columns:
+                        # Filter for specific FormOID(s)
+                        if isinstance(form_oid, list):
+                            source_df = df_long[df_long["FormOID"].isin(form_oid)].copy()
+                        else:
+                            source_df = df_long[df_long["FormOID"] == form_oid].copy()
                     else:
-                        source_df = df_long[df_long["FormOID"] == form_oid].copy()
+                        print(f"Warning: 'FormOID' column missing in source data. Skipping FormOID filtering.")
+                        source_df = df_long.copy()
                 except Exception as e:
                     print(
                         f"Error filtering for {domain_name} (FormOID={form_oid}): {e}"
                     )
                     continue
             else:
-                print(f"Warning: No formoid specified for a block in {domain_name}")
-                continue
+                if "FormOID" in df_long.columns:
+                    print(f"Warning: No formoid specified for a block in {domain_name}")
+                    continue
+                else:
+                    source_df = df_long.copy()
 
             if source_df.empty:
                 continue
 
             # 2. Key columns for pivoting (use block keys or defaults)
             keys = settings.get("keys", default_keys)
+            if custom_to_standard:
+                keys = [custom_to_standard.get(k, k) for k in keys]
 
             # 3. Pivot
             try:

{cdiscbuilder-1.2.0 → cdiscbuilder-1.2.1}/src/cdiscbuilder/sdtm/engine/processor.py

@@ -8,7 +8,7 @@ from .classes.findings import FindingsProcessor
 from .classes.special_purpose import SpecialPurposeProcessor
 
 
-def process_domain(domain_name, sources, df_long, default_keys, output_dir):
+def process_domain(domain_name, sources, df_long, default_keys, output_dir, custom_to_standard=None):
     # Normalize to list
     if isinstance(sources, dict):
         sources = [sources]
@@ -31,7 +31,7 @@ def process_domain(domain_name, sources, df_long, default_keys, output_dir):
     else:
         processor = GeneralProcessor()
 
-    domain_dfs = processor.process(domain_name, sources, df_long, default_keys)
+    domain_dfs = processor.process(domain_name, sources, df_long, default_keys, custom_to_standard=custom_to_standard)
 
     if not domain_dfs:
         print(f"Warning: No data found for domain {domain_name}")

cdiscbuilder-1.2.1/src/cdiscbuilder/sdtm/engine/tests/test_boundary_standardization.py

@@ -0,0 +1,135 @@
+import os
+import tempfile
+import pandas as pd
+import pytest
+import yaml
+from cdiscbuilder.sdtm.odm_parser import parse_odm_to_long_df
+from cdiscbuilder.sdtm.sdtm import create_sdtm_datasets
+
+
+def test_parse_odm_to_long_df_boundary_standardization():
+    # Simple XML string with Medidata style attributes
+    xml_content = """<?xml version="1.0" encoding="UTF-8"?>
+    <ODM>
+      <ClinicalData StudyOID="STUDY_XYZ">
+        <SubjectData SubjectKey="SUBJ123">
+          <StudyEventData StudyEventOID="SE_VISIT" StartDate="2026-05-29">
+            <FormData FormOID="F_AE">
+              <ItemGroupData ItemGroupOID="IG_AE" RecordPosition="2">
+                <ItemData ItemOID="AE_TERM" Value="Headache"/>
+              </ItemGroupData>
+            </FormData>
+          </StudyEventData>
+        </SubjectData>
+      </ClinicalData>
+    </ODM>"""
+
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".xml", delete=False) as f:
+        f.write(xml_content)
+        temp_xml_path = f.name
+
+    try:
+        # Test parsing with custom xml_mapping.
+        # Should map the RecordPosition attribute to standard ItemGroupRepeatKey column in df
+        xml_mapping = {"item_group_repeat_key": "RecordPosition"}
+
+        df = parse_odm_to_long_df(temp_xml_path, xml_mapping=xml_mapping)
+
+        # Verify output columns are standard (no RecordPosition column is outputted;
+        # it is standardized to ItemGroupRepeatKey)
+        assert "ItemGroupRepeatKey" in df.columns
+        assert "RecordPosition" not in df.columns
+        assert df.iloc[0]["ItemGroupRepeatKey"] == "2"
+    finally:
+        try:
+            os.remove(temp_xml_path)
+        except Exception:
+            pass
+
+
+def test_create_sdtm_datasets_boundary_standardization(tmp_path):
+    # df_long with custom columns: RecordPosition, SubjectID, and no FormOID
+    df_long = pd.DataFrame(
+        [
+            {
+                "StudyOID": "STUDY01",
+                "SubjectID": "001",
+                "RecordPosition": 1,
+                "ItemOID": "AETERM",
+                "Value": "Headache",
+            },
+            {
+                "StudyOID": "STUDY01",
+                "SubjectID": "001",
+                "RecordPosition": 1,
+                "ItemOID": "AESTDTC",
+                "Value": "2026-05-29",
+            },
+            {
+                "StudyOID": "STUDY01",
+                "SubjectID": "001",
+                "RecordPosition": 2,
+                "ItemOID": "AETERM",
+                "Value": "Nausea",
+            },
+            {
+                "StudyOID": "STUDY01",
+                "SubjectID": "001",
+                "RecordPosition": 2,
+                "ItemOID": "AESTDTC",
+                "Value": "2026-05-30",
+            },
+        ]
+    )
+
+    input_csv = tmp_path / "long.csv"
+    df_long.to_csv(input_csv, index=False)
+
+    # Create domain spec directory
+    spec_dir = tmp_path / "specs"
+    os.makedirs(spec_dir)
+
+    # Create defaults.yaml
+    defaults = {
+        "keys": ["StudyOID", "SubjectID", "RecordPosition"],
+        "csv_columns": {
+            "item_group_repeat_key": "RecordPosition",
+            "subject_key": "SubjectID",
+        },
+    }
+    with open(spec_dir / "defaults.yaml", "w") as f:
+        yaml.dump(defaults, f)
+
+    # Create AE.yaml
+    ae_config = {
+        "AE": [
+            {
+                # Note: no formoid specified! So we test proceeding without FormOID filtering.
+                "type": "events",
+                "columns": {
+                    "STUDYID": {"source": "StudyOID"},
+                    "USUBJID": {"source": "SubjectKey"},
+                    "AETERM": {"source": "AETERM"},
+                    "AESTDTC": {"source": "AESTDTC"},
+                    "AESEQ": {"group": ["USUBJID"], "sort_by": ["AESTDTC"]},
+                },
+            }
+        ]
+    }
+    with open(spec_dir / "AE.yaml", "w") as f:
+        yaml.dump(ae_config, f)
+
+    output_dir = tmp_path / "sdtm_out"
+
+    # Generate datasets
+    create_sdtm_datasets(str(spec_dir), str(input_csv), str(output_dir))
+
+    # Verify dataset exists
+    out_file = output_dir / "AE.parquet"
+    assert out_file.exists()
+
+    res_df = pd.read_parquet(out_file)
+    assert len(res_df) == 2
+    assert "AETERM" in res_df.columns
+    assert list(res_df["AETERM"]) == ["Headache", "Nausea"]
+    assert list(res_df["AESEQ"]) == [1, 2]

{cdiscbuilder-1.2.0 → cdiscbuilder-1.2.1}/src/cdiscbuilder/sdtm/odm_parser.py

@@ -2,7 +2,26 @@ import xml.etree.ElementTree as ET
 import pandas as pd
 
 
-def parse_odm_to_long_df(xml_file):
+DEFAULT_XML_MAPPING = {
+    "study_oid": "StudyOID",
+    "subject_key": "SubjectKey",
+    "study_subject_id": "StudySubjectID",
+    "study_event_oid": "StudyEventOID",
+    "study_event_repeat_key": "StudyEventRepeatKey",
+    "study_event_start_date": "StartDate",
+    "form_oid": "FormOID",
+    "item_group_oid": "ItemGroupOID",
+    "item_group_repeat_key": "ItemGroupRepeatKey",
+    "item_oid": "ItemOID",
+    "value": "Value",
+}
+
+
+def parse_odm_to_long_df(xml_file, xml_mapping=None):
+    merged_xml_mapping = DEFAULT_XML_MAPPING.copy()
+    if xml_mapping:
+        merged_xml_mapping.update(xml_mapping)
+
     try:
         tree = ET.parse(xml_file)
         root = tree.getroot()
@@ -42,13 +61,15 @@ def parse_odm_to_long_df(xml_file):
 
     for cd in root:
         if get_local_name(cd.tag) == "ClinicalData":
-            study_oid = cd.get("StudyOID")
+            study_oid = cd.get(merged_xml_mapping["study_oid"])
             for sd in cd:
                 if get_local_name(sd.tag) == "SubjectData":
-                    subject_key = sd.get("SubjectKey")
+                    subject_key = sd.get(merged_xml_mapping["subject_key"])
 
                     # Helper for attributes
                     def get_attrib(elem, partial_name):
+                        if not partial_name:
+                            return None
                         if partial_name in elem.attrib:
                             return elem.attrib[partial_name]
                         for k, v in elem.attrib.items():
@@ -56,41 +77,46 @@ def parse_odm_to_long_df(xml_file):
                                 return v
                         return None
 
-                    study_subject_id = get_attrib(sd, "StudySubjectID") or get_attrib(
-                        sd, "studysubjectid"
-                    )
+                    study_subject_id_attr = merged_xml_mapping["study_subject_id"]
+                    study_subject_id = get_attrib(sd, study_subject_id_attr)
+                    if not study_subject_id and study_subject_id_attr:
+                        # try case-insensitive or fallback
+                        study_subject_id = get_attrib(sd, study_subject_id_attr.lower())
+                    if not study_subject_id:
+                        study_subject_id = get_attrib(sd, "studysubjectid")
+
                     if not subject_key:
                         subject_key = study_subject_id
 
                     for child in sd:
                         tag = get_local_name(child.tag)
                         if tag == "StudyEventData":
-                            study_event_oid = child.get("StudyEventOID")
+                            study_event_oid = child.get(merged_xml_mapping["study_event_oid"])
                             study_event_repeat_key = (
-                                child.get("StudyEventRepeatKey") or "1"
+                                child.get(merged_xml_mapping["study_event_repeat_key"]) or "1"
                             )
 
                             # Extract Namespaced StartDate
-                            start_date = get_attrib(child, "StartDate") or ""
+                            start_date = get_attrib(child, merged_xml_mapping["study_event_start_date"]) or ""
 
                             for form in child:
                                 f_tag = get_local_name(form.tag)
                                 if f_tag == "FormData":
-                                    form_oid = form.get("FormOID")
+                                    form_oid = form.get(merged_xml_mapping["form_oid"])
 
                                     for ig in form:
                                         ig_tag = get_local_name(ig.tag)
                                         if ig_tag == "ItemGroupData":
-                                            item_group_oid = ig.get("ItemGroupOID")
+                                            item_group_oid = ig.get(merged_xml_mapping["item_group_oid"])
                                             item_group_repeat_key = ig.get(
-                                                "ItemGroupRepeatKey"
+                                                merged_xml_mapping["item_group_repeat_key"]
                                             )
 
                                             for item in ig:
                                                 i_tag = get_local_name(item.tag)
                                                 if i_tag == "ItemData":
-                                                    item_oid = item.get("ItemOID")
-                                                    value = item.get("Value")
+                                                    item_oid = item.get(merged_xml_mapping["item_oid"])
+                                                    value = item.get(merged_xml_mapping["value"])
 
                                                     meta = item_metadata.get(
                                                         item_oid, {}

cdiscbuilder-1.2.1/src/cdiscbuilder/sdtm/sdtm.py

@@ -0,0 +1,71 @@
+import pandas as pd
+from .engine.config import load_config
+from .engine.processor import process_domain
+
+
+def create_sdtm_datasets(config_input, input_csv, output_dir):
+    if isinstance(config_input, dict):
+        config = config_input
+        # We assume it's already structured correctly or validated
+    else:
+        config = load_config(config_input)
+
+    # Get global defaults
+    defaults = config.get("defaults", {})
+    default_keys = defaults.get(
+        "keys", ["StudyOID", "SubjectKey", "ItemGroupRepeatKey", "StudyEventOID"]
+    )
+
+    print(f"Loading data from {input_csv}...")
+    df_long = pd.read_csv(input_csv)
+
+    # Invert mapping to go from custom CSV column name -> standard logical column name
+    # e.g., "RecordPosition" -> "ItemGroupRepeatKey"
+    csv_columns = defaults.get("csv_columns") or {}
+    STANDARD_LOGICAL_COLUMNS = {
+        "study_oid": "StudyOID",
+        "subject_key": "SubjectKey",
+        "study_subject_id": "StudySubjectID",
+        "study_event_oid": "StudyEventOID",
+        "study_event_repeat_key": "StudyEventRepeatKey",
+        "study_event_start_date": "StudyEventStartDate",
+        "form_oid": "FormOID",
+        "item_group_oid": "ItemGroupOID",
+        "item_group_repeat_key": "ItemGroupRepeatKey",
+        "item_oid": "ItemOID",
+        "value": "Value",
+        "question": "Question",
+        "item_name": "ItemName",
+    }
+
+    custom_to_standard = {}
+    rename_map = {}
+    for logical_key, custom_col in csv_columns.items():
+        if logical_key in STANDARD_LOGICAL_COLUMNS:
+            standard_col = STANDARD_LOGICAL_COLUMNS[logical_key]
+            rename_map[custom_col] = standard_col
+            custom_to_standard[custom_col] = standard_col
+
+    # Perform rename if map is not empty
+    if rename_map:
+        df_long.rename(columns=rename_map, inplace=True)
+        # Translate default_keys to match the standardized DataFrame
+        default_keys = [custom_to_standard.get(k, k) for k in default_keys]
+
+    # Prioritize DM domain processing
+    domains = list(config["domains"].keys())
+    if "DM" in domains:
+        domains.remove("DM")
+        domains.insert(0, "DM")
+
+    for domain in domains:
+        settings_entry = config["domains"][domain]
+        print(f"Processing domain: {domain}")
+
+        # Normalize to list.
+        if isinstance(settings_entry, list):
+            sources = settings_entry
+        else:
+            sources = [settings_entry]
+
+        process_domain(domain, sources, df_long, default_keys, output_dir, custom_to_standard=custom_to_standard)

{cdiscbuilder-1.2.0 → cdiscbuilder-1.2.1/src/cdiscbuilder.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cdiscbuilder
-Version: 1.2.0
+Version: 1.2.1
 Summary: A package to convert ODM XML to SDTM/ADaM Datasets
 Author-email: Ming-Chun Chen <hellomingchun@gmail.com>
 Requires-Python: >=3.9

{cdiscbuilder-1.2.0 → cdiscbuilder-1.2.1}/src/cdiscbuilder.egg-info/SOURCES.txt

@@ -59,6 +59,7 @@ src/cdiscbuilder/sdtm/engine/classes/findings.py
 src/cdiscbuilder/sdtm/engine/classes/general.py
 src/cdiscbuilder/sdtm/engine/classes/interventions.py
 src/cdiscbuilder/sdtm/engine/classes/special_purpose.py
+src/cdiscbuilder/sdtm/engine/tests/test_boundary_standardization.py
 src/cdiscbuilder/sdtm/engine/tests/test_config.py
 src/cdiscbuilder/sdtm/engine/tests/test_findings.py
 src/cdiscbuilder/sdtm/engine/tests/test_general.py

cdiscbuilder-1.2.0/src/cdiscbuilder/sdtm/sdtm.py

@@ -1,37 +0,0 @@
-import pandas as pd
-from .engine.config import load_config
-from .engine.processor import process_domain
-
-
-def create_sdtm_datasets(config_input, input_csv, output_dir):
-    if isinstance(config_input, dict):
-        config = config_input
-        # We assume it's already structured correctly or validated
-    else:
-        config = load_config(config_input)
-
-    # Get global defaults
-    default_keys = config.get("defaults", {}).get(
-        "keys", ["StudyOID", "SubjectKey", "ItemGroupRepeatKey", "StudyEventOID"]
-    )
-
-    print(f"Loading data from {input_csv}...")
-    df_long = pd.read_csv(input_csv)
-
-    # Prioritize DM domain processing
-    domains = list(config["domains"].keys())
-    if "DM" in domains:
-        domains.remove("DM")
-        domains.insert(0, "DM")
-
-    for domain in domains:
-        settings_entry = config["domains"][domain]
-        print(f"Processing domain: {domain}")
-
-        # Normalize to list.
-        if isinstance(settings_entry, list):
-            sources = settings_entry
-        else:
-            sources = [settings_entry]
-
-        process_domain(domain, sources, df_long, default_keys, output_dir)