cdiscbuilder 0.2.9__tar.gz

cdiscbuilder-0.2.9/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ming-Chun Chen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cdiscbuilder-0.2.9/MANIFEST.in

@@ -0,0 +1,2 @@
+include src/cdiscbuilder/specs/*.yaml
+include src/cdiscbuilder/metadata/*.yaml

cdiscbuilder-0.2.9/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 2.4
+Name: cdiscbuilder
+Version: 0.2.9
+Summary: A package to convert ODM XML to SDTM/ADaM Datasets
+Author-email: Ming-Chun Chen <hellomingchun@gmail.com>
+Requires-Python: >=3.8
+License-File: LICENSE
+Requires-Dist: pandas
+Requires-Dist: pyyaml
+Dynamic: license-file

cdiscbuilder-0.2.9/README.md

@@ -0,0 +1,132 @@
+# CDISC Builder
+
+**`cdiscbuilder`** is a Python package designed to simplify the transformation of clinical trial data from **ODM (Operational Data Model)** XML format into **CDISC SDTM (Study Data Tabulation Model)** and **ADaM (Analysis Data Model)** datasets.
+
+It provides a flexible, configuration-driven approach to data mapping, allowing users to define rules using simple YAML files or Python dictionaries without harcoding complex logic.
+
+## Key Features
+
+-   **ODM XML Parsing**: Efficiently parses CDISC ODM strings and files into workable dataframes.
+-   **Configurable Mappings**: Define your mapping rules (source columns, hardcoded values, custom logic) in YAML.
+-   **Schema Validation**: Ensures your configuration files adhere to strict standards before processing.
+-   **Metadata-Driven Findings**: Powerful processor for Findings domains (VS, LB, FA, etc.) using granular metadata.
+-   **Excel/Parquet Output**: Generates regulatory-compliant datasets in modern formats.
+
+## Installation
+
+```bash
+pip install cdiscbuilder
+```
+
+## Quick Start
+
+### 1. Command Line Interface
+
+You can generate datasets directly from your terminal:
+
+```bash
+# Generate SDTM datasets from an ODM XML file
+cdisc-sdtm --xml study_data.xml --output ./sdtm_data
+```
+
+### 2. Python API
+
+```python
+from cdiscbuilder.sdtm import create_sdtm_datasets
+
+# Define paths
+xml_file = "study_data.xml"
+config_dir = "path/to/my/specs" 
+output_dir = "./sdtm_outputs"
+
+# Generate Datasets
+create_sdtm_datasets(config_dir, xml_file, output_dir)
+```
+
+## Configuration
+
+The package comes with standard configurations for common domains (`DM`, `AE`, `VS`, etc.) in `src/cdisc_builder/specs`. You can override these or add new ones by creating your own configuration directory.
+
+### Example YAML (`DM.yaml`)
+
+```yaml
+DM:
+    - formoid: "FORM.DEMOG"
+      keys: ["StudyOID", "StudySubjectID"]
+      columns:
+          STUDYID:
+              source: StudyOID
+              type: str
+          USUBJID:
+              source: StudySubjectID
+              prefix: "PPT-"
+              type: str
+          AGE:
+              source: IT.AGE
+              type: int
+              type: str
+          SEX:
+              source: I_DEMOG_SEX
+              type: str
+              value_mapping:
+                  "M": "Male"
+                  "F": "Female"
+
+```
+### Finding Domains (Dynamic Mapping)
+
+For domains like `IE`, `LB`, `FA` where many input items map to a single `Test Code` / `Test Name` pair, use `type: finding`.
+
+```yaml
+IE:
+  - type: finding
+    formoid: "F_ELIGIBILITY"
+    # Filter rows using Regex
+    item_group_regex: "IG_ELIGI_.*"
+    item_oid_regex: "I_ELIGI_.*"
+    
+    columns:
+      # Extract part of the OID for the Short Code
+      IETESTCD:
+        source: ItemOID
+        regex_extract: "I_ELIGI_(.*)"
+      
+      # Use Metadata from parsed XML for the Description
+      IETEST:
+        source: Metadata.Question
+      
+      IEORRES:
+        source: Value
+```
+
+### Advanced Mapping Features
+
+**Prefixing**:
+```yaml
+USUBJID:
+  source: StudySubjectID
+  prefix: "PPT-"
+```
+
+**Substring Extraction** (extracts chars 3-5 before mapping):
+```yaml
+SITEID:
+  source: FULL_ID
+  substring_start: 3
+  substring_length: 3
+```
+
+**Default Values**:
+```yaml
+CUSTOM_COL:
+  source: ORG_COL
+  value_mapping:
+    "A": "Alpha"
+  mapping_default: "Other" # used if not A
+  # mapping_default_source: "AnotherCol" # Fallback to column value
+```
+```
+
+## License
+
+[MIT License](LICENSE)

cdiscbuilder-0.2.9/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "cdiscbuilder"
+version = "0.2.9"
+description = "A package to convert ODM XML to SDTM/ADaM Datasets"
+authors = [{name = "Ming-Chun Chen", email = "hellomingchun@gmail.com"}]
+dependencies = [
+    "pandas",
+    "pyyaml"
+]
+requires-python = ">=3.8"
+
+[project.scripts]
+cdisc-sdtm = "cdiscbuilder.cli:main"
+
+[tool.setuptools.package-data]
+cdiscbuilder = ["specs/*.yaml", "engine/classes/*.py"]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.mypy]
+ignore_missing_imports = true
+

cdiscbuilder-0.2.9/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

cdiscbuilder-0.2.9/src/cdiscbuilder/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.2"

cdiscbuilder-0.2.9/src/cdiscbuilder/cli.py

@@ -0,0 +1,42 @@
+import os
+import argparse
+from .odm import parse_odm_to_long_df
+from .sdtm import create_sdtm_datasets
+
+def main():
+    parser = argparse.ArgumentParser(description="Convert ODM XML to SDTM Datasets")
+    # Determine default config path inside package
+    current_dir = os.path.dirname(__file__)
+    default_config_path = os.path.join(current_dir, "specs")
+    
+    parser.add_argument("--xml", required=True, help="Path to input ODM XML file")
+    parser.add_argument("--csv", default="odm_long.csv", help="Path to intermediate long CSV file")
+    parser.add_argument("--configs", default=default_config_path, help="Path to SDTM configuration directory")
+    parser.add_argument("--output", default="sdtm_output", help="Path to output SDTM directory")
+    
+    args = parser.parse_args()
+
+    # Step 1: ODM XML -> Long CSV
+    print(f"--- Step 1: Parsing ODM XML from {args.xml} ---")
+    try:
+        df = parse_odm_to_long_df(args.xml)
+        print(f"Parsed {len(df)} rows.")
+        df.to_csv(args.csv, index=False)
+        print(f"Saved intermediate data to {args.csv}")
+    except Exception as e:
+        print(f"Error parsing XML: {e}")
+        return
+
+    # Step 2: Long CSV -> SDTM Datasets
+    print(f"\n--- Step 2: Generating SDTM Datasets using configs from {args.configs} ---")
+    if not os.path.exists(args.output):
+        os.makedirs(args.output)
+        
+    try:
+        create_sdtm_datasets(args.configs, args.csv, args.output)
+        print(f"\nSuccess! SDTM datasets created in {args.output}")
+    except Exception as e:
+        print(f"Error creating SDTM datasets: {e}")
+
+if __name__ == "__main__":
+    main()

cdiscbuilder-0.2.9/src/cdiscbuilder/engine/classes/finding.py

@@ -0,0 +1,113 @@
+import pandas as pd
+
+
+class FindingProcessor:
+    def __init__(self):
+        pass
+
+    def process(self, domain_name, sources, df_long, default_keys):
+        domain_dfs = []
+        
+        for settings in sources:
+            # 0. Filter by FormOID (optional but recommended)
+            form_oid = settings.get('formoid')
+            source_df = df_long.copy()
+            if form_oid:
+                if isinstance(form_oid, list):
+                    source_df = source_df[source_df['FormOID'].isin(form_oid)]
+                else:
+                    source_df = source_df[source_df['FormOID'] == form_oid]
+            
+            # 1. Filter by ItemGroupOID (regex or list)
+            item_group_match = settings.get('item_group_regex')
+            if item_group_match:
+                 source_df = source_df[source_df['ItemGroupOID'].str.match(item_group_match, na=False)]
+            
+            # 2. Filter by ItemOID (regex)
+            # This is crucial for "finding" domains - we want rows where ItemOID matches a pattern
+            item_oid_match = settings.get('item_oid_regex')
+            if item_oid_match:
+                source_df = source_df[source_df['ItemOID'].str.match(item_oid_match, na=False)]
+            
+            if source_df.empty:
+                continue
+                
+            # 3. Create Base DataFrame (No Pivot!)
+            # 3. Create Base DataFrame (No Pivot!)
+            # We treat every row as a potential finding
+            # Base columns: Keys + ItemOID + Value
+            keys = settings.get('keys', default_keys)
+            
+            # Check if Question exists in source_df (it should based on odm.py changes)
+            base_cols = keys + ['ItemOID', 'Value']
+            if 'Question' in source_df.columns:
+                 base_cols.append('Question')
+
+            final_df = source_df[base_cols].copy()
+            
+            # 4. Map Columns
+            mappings = settings.get('columns', {})
+            
+            for target_col, col_config in mappings.items():
+                series = None
+                
+                # Config can be simple string (source col) or dict
+                source_expr = None
+                literal_expr = None
+                target_type = None
+                regex_extract = None
+                
+                if isinstance(col_config, dict):
+                    source_expr = col_config.get('source')
+                    literal_expr = col_config.get('literal')
+                    target_type = col_config.get('type')
+                    regex_extract = col_config.get('regex_extract') # e.g. "I_ELIGI_(.*)"
+                else:
+                    source_expr = col_config # simplistic
+                
+                if literal_expr is not None:
+                     series = pd.Series([literal_expr] * len(final_df), index=final_df.index)
+                
+                elif source_expr:
+                    if source_expr == "ItemOID":
+                        series = final_df['ItemOID']
+                    elif source_expr == "Value":
+                        series = final_df['Value']
+                    elif source_expr in final_df.columns:
+                        series = final_df[source_expr]
+                    elif source_expr in source_df.columns:
+                         series = source_df[source_expr]
+                    
+                    # Regex Extraction from source
+                    if regex_extract and series is not None:
+                         # Extract group 1
+                         series = series.astype(str).str.extract(regex_extract)[0]
+                
+                # Apply Prefix
+                prefix = None
+                if isinstance(col_config, dict):
+                    prefix = col_config.get('prefix')
+                
+                if prefix and series is not None:
+                    series = prefix + series.astype(str)
+                
+                if series is not None:
+                    # Type Conversion
+                    if target_type:
+                        try:
+                            if target_type == 'int':
+                                series = pd.to_numeric(series, errors='coerce').astype('Int64')
+                            elif target_type == 'str':
+                                series = series.astype(str)
+                        except Exception:
+                            pass
+                    
+                    final_df[target_col] = series
+            
+            # Filter to keep only target columns
+            cols_to_keep = list(mappings.keys())
+            final_df = final_df[cols_to_keep]
+            
+            domain_dfs.append(final_df)
+            
+        return domain_dfs

cdiscbuilder-0.2.9/src/cdiscbuilder/engine/classes/general.py

@@ -0,0 +1,240 @@
+import pandas as pd
+
+class GeneralProcessor:
+    def _expand_settings(self, settings):
+        """
+        Expands a settings dict with list-based sources/literals into multiple settings dicts.
+        """
+        # Find all columns that have a list for source or literal
+        list_cols = {}
+        list_len = 0
+        
+        columns = settings.get('columns', {})
+        for col_name, col_cfg in columns.items():
+            if isinstance(col_cfg, dict):
+                src = col_cfg.get('source')
+                lit = col_cfg.get('literal')
+                
+                # Check source
+                if isinstance(src, list):
+                    if list_len > 0 and len(src) != list_len:
+                        raise ValueError(f"Column '{col_name}' source list length {len(src)} mismatch with others {list_len}")
+                    list_len = len(src)
+                    list_cols[col_name] = 'source'
+                
+                # Check literal
+                if isinstance(lit, list):
+                    if list_len > 0 and len(lit) != list_len:
+                        raise ValueError(f"Column '{col_name}' literal list length {len(lit)} mismatch with others {list_len}")
+                    list_len = len(lit)
+                    list_cols[col_name] = 'literal'
+                    
+        if list_len == 0:
+            return [settings]
+            
+        # Expand
+        expanded_list = []
+        for i in range(list_len):
+            new_settings = settings.copy()
+            new_cols = {}
+            for col_name, col_cfg in columns.items():
+                if isinstance(col_cfg, dict):
+                    new_cfg = col_cfg.copy()
+                    if col_name in list_cols:
+                        param = list_cols[col_name] # 'source' or 'literal'
+                         # Extract the i-th element
+                        val_list = col_cfg.get(param)
+                        new_cfg[param] = val_list[i]
+                    new_cols[col_name] = new_cfg
+                else:
+                    new_cols[col_name] = col_cfg
+            
+            new_settings['columns'] = new_cols
+            expanded_list.append(new_settings)
+            
+        return expanded_list
+
+    def process(self, domain_name, sources, df_long, default_keys):
+        domain_dfs = []
+        
+        # Pre-expand sources if they contain lists
+        expanded_sources = []
+        for s in sources:
+            try:
+                expanded_sources.extend(self._expand_settings(s))
+            except Exception as e:
+                print(f"Error expanding settings for {domain_name}: {e}")
+                continue # Skip invalid blocks
+        
+        for settings in expanded_sources:
+            # 1. Filter by FormOID
+            form_oid = settings.get('formoid')
+            if form_oid:
+                try:
+                    # Filter for specific FormOID(s)
+                    if isinstance(form_oid, list):
+                        source_df = df_long[df_long['FormOID'].isin(form_oid)].copy()
+                    else:
+                        source_df = df_long[df_long['FormOID'] == form_oid].copy()
+                except Exception as e:
+                    print(f"Error filtering for {domain_name} (FormOID={form_oid}): {e}")
+                    continue
+            else:
+                print(f"Warning: No formoid specified for a block in {domain_name}")
+                continue
+                
+            if source_df.empty:
+                continue
+
+            # 2. Key columns for pivoting (use block keys or defaults)
+            keys = settings.get('keys', default_keys)
+            
+            # 3. Pivot
+            try:
+                pivoted = source_df.pivot_table(
+                    index=keys, 
+                    columns='ItemOID', 
+                    values='Value', 
+                    aggfunc='first'
+                ).reset_index()
+            except Exception as e:
+                print(f"Error pivoting for {domain_name}: {e}")
+                continue
+                
+            # 4. Map columns
+            final_df = pd.DataFrame()
+            mappings = settings.get('columns', {})
+            
+            for target_col, col_config in mappings.items():
+                source_expr = None
+                literal_expr = None
+                target_type = None
+                value_map = None
+
+                # Check if simple string or object config
+                if isinstance(col_config, dict):
+                    source_expr = col_config.get('source')
+                    literal_expr = col_config.get('literal')
+                    target_type = col_config.get('type')
+                    value_map = col_config.get('value_mapping')
+                else:
+                    source_expr = col_config
+                    literal_expr = None
+                
+                # Extract Data
+                series = None
+                if literal_expr is not None:
+                    # Explicit literal value
+                    series = pd.Series([literal_expr] * len(pivoted))
+                elif source_expr:
+                    if source_expr in pivoted.columns:
+                        series = pivoted[source_expr].copy()
+                    elif source_expr in final_df.columns:
+                        series = final_df[source_expr].copy()
+                    else:
+                        # Source defined but not found.
+                        print(f"Warning: Source column '{source_expr}' not found for '{domain_name}.{target_col}'. Filling with NaN.")
+                        series = pd.Series([None] * len(pivoted))
+                else:
+                    print(f"Warning: No source or literal defined for '{domain_name}.{target_col}'. Filling with NaN.")
+                    series = pd.Series([None] * len(pivoted))
+                
+                # Apply Dependency Logic (Assign only if dependency column is not null)
+                dependency = col_config.get('dependency') if isinstance(col_config, dict) else None
+                if dependency:
+                    dep_series = None
+                    if dependency in pivoted.columns:
+                        dep_series = pivoted[dependency]
+                    elif dependency in final_df.columns:
+                        dep_series = final_df[dependency]
+                    
+                    if dep_series is not None:
+                         # Mask: Keep values where dependency is NOT null, else fill with False Value (default None)
+                         false_val = col_config.get('dependency_false_value')
+                         # Make sure false_val is treated as literal of correct type? pandas usually handles mixed.
+                         
+                         series = series.where(dep_series.notna(), false_val)
+                    else:
+                         print(f"Warning: Dependency column '{dependency}' not found for '{domain_name}.{target_col}'. Treating as all-null dependency.")
+                         false_val = col_config.get('dependency_false_value')
+                         series = pd.Series([false_val] * len(pivoted))
+                
+                # Apply Substring Extraction (Before Value Mapping)
+                if isinstance(col_config, dict):
+                    sub_start = col_config.get('substring_start')
+                    sub_len = col_config.get('substring_length')
+                    if sub_start is not None and sub_len is not None:
+                        # Ensure series is string
+                        series = series.astype(str)
+                        # Slice 0-indexed or 1-indexed? Python is 0-indexed.
+                        # User said "position 3-5". If string is '1110023565' and target is '002',
+                        # indices are 3,4,5. So slice[3:6].
+                        # Let's assume user provides 0-based start index and length.
+                        series = series.str[sub_start : sub_start + sub_len]
+
+                # Apply Value Mapping
+                mapping_default = col_config.get('mapping_default') if isinstance(col_config, dict) else None
+                mapping_default_source = col_config.get('mapping_default_source') if isinstance(col_config, dict) else None
+
+                if value_map:
+                    # Perform mapping (non-matches become NaN)
+                    mapped_series = series.map(value_map)
+                    
+                    if mapping_default is not None:
+                         # Strict mapping with default literal
+                         series = mapped_series.fillna(mapping_default)
+                    elif mapping_default_source is not None:
+                         # Strict mapping with default from another column
+                         fallback = None
+                         if mapping_default_source in final_df.columns:
+                             fallback = final_df[mapping_default_source]
+                         elif mapping_default_source in pivoted.columns:
+                             fallback = pivoted[mapping_default_source]
+                        
+                         if fallback is not None:
+                             series = mapped_series.fillna(fallback)
+                         else:
+                             print(f"Warning: Default source '{mapping_default_source}' not found for '{domain_name}.{target_col}'")
+                             series = mapped_series # Leave as NaN or original? mapped_series has NaNs.
+                    else:
+                         # Partial replacement (legacy: keep original values if not in map)
+                         series = series.replace(value_map)
+
+                # Apply Prefix
+                prefix = col_config.get('prefix') if isinstance(col_config, dict) else None
+                if prefix:
+                    series = prefix + series.astype(str)
+
+                # Apply Type Conversion
+                if target_type:
+                    try:
+                        if target_type == 'int':
+                            series = pd.to_numeric(series, errors='coerce').astype('Int64')
+                        elif target_type == 'float':
+                             series = pd.to_numeric(series, errors='coerce')
+                        elif target_type == 'str':
+                            series = series.astype(str)
+                        elif target_type == 'bool':
+                            series = series.astype(bool)
+                    except Exception as e:
+                        print(f"Error converting {target_col} to {target_type}: {e}")
+
+                final_df[target_col] = series
+                
+                # Validation: max_missing_pct
+                if isinstance(col_config, dict):
+                    max_missing = col_config.get('max_missing_pct')
+                    if max_missing is not None:
+                        missing_count = series.isna().sum()
+                        if target_type == 'str':
+                             missing_count += (series.isin(['nan', 'None'])).sum()
+                        
+                        total = len(series)
+                        if total > 0:
+                            pct = (missing_count / total) * 100
+                            if pct > max_missing:
+                                print(f"WARNING: [Validation] {domain_name}.{target_col} missing {pct:.2f}% (Limit: {max_missing:})")
+            
+            domain_dfs.append(final_df)
+            
+        return domain_dfs

cdiscbuilder-0.2.9/src/cdiscbuilder/engine/config.py

@@ -0,0 +1,57 @@
+import yaml
+import os
+
+from .validate import load_schema, validate_domain_config
+
+def load_config(config_dir):
+    """
+    Loads all YAML configuration files from the specified directory.
+    Validates them against schema.yaml.
+    """
+    config = {
+        'domains': {},
+        'defaults': {}
+    }
+
+    schema = load_schema()
+
+    if not os.path.exists(config_dir):
+        # Fallback to package data if default path doesn't exist?
+        # Assuming config_dir provided is valid or we expect empty.
+        return config
+
+    for filename in os.listdir(config_dir):
+        if filename.endswith(".yaml") or filename.endswith(".yml"):
+            file_path = os.path.join(config_dir, filename)
+            
+            # Skip schema itself if present in same dir
+            if filename == "schema.yaml":
+                continue
+
+            with open(file_path, "r") as f:
+                try:
+                    data = yaml.safe_load(f)
+                    
+                    if filename == 'defaults.yaml':
+                        # Defaults file - likely flat dict
+                        config['defaults'].update(data)
+                        continue
+
+                    # Merge data
+                    for key, value in data.items():
+                        if key == 'defaults':
+                             # Fallback if someone put defaults: inside another file
+                            config['defaults'].update(value)
+                        else:
+                            # It's a domain
+                            # Validate!
+                            if schema:
+                                if not validate_domain_config(key, value, schema):
+                                    print(f"Warning: {filename} failed schema validation. Proceeding with caution.")
+                                    
+                            config['domains'][key] = value
+
+                except yaml.YAMLError as exc:
+                    print(f"Error parsing YAML file {filename}: {exc}")
+    
+    return config