prelude-parser 0.1.0__tar.gz → 0.2.0__tar.gz

prelude_parser-0.2.0/.github/dependabot.yml

@@ -0,0 +1,23 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    labels:
+    - skip-changelog
+    - dependencies
+  - package-ecosystem: cargo
+    directory: "/"
+    schedule:
+      interval: daily
+    labels:
+    - skip-changelog
+    - dependencies
+  - package-ecosystem: github-actions
+    directory: '/'
+    schedule:
+      interval: daily
+    labels:
+    - skip-changelog
+    - dependencies

{prelude_parser-0.1.0 → prelude_parser-0.2.0}/.github/workflows/testing.yml

@@ -21,7 +21,7 @@ jobs:
         override: true
         components: clippy
     - name: Cache dependencies
-      uses: Swatinem/rust-cache@v2.2.1
+      uses: Swatinem/rust-cache@v2.3.0
     - name: Run cargo clippy
       uses: actions-rs/cargo@v1
       with:
@@ -39,7 +39,7 @@ jobs:
         override: true
         components: rustfmt
     - name: Cache dependencies
-      uses: Swatinem/rust-cache@v2.2.1
+      uses: Swatinem/rust-cache@v2.3.0
     - name: Run cargo fmt
       run: cargo fmt --all -- --check
   test:

{prelude_parser-0.1.0 → prelude_parser-0.2.0}/.pre-commit-config.yaml

@@ -15,11 +15,11 @@ repos:
       language_version: python3
       args: [--line-length=100]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.2.0
+    rev: v1.3.0
     hooks:
     - id: mypy
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.263
+    rev: v0.0.267
     hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]

{prelude_parser-0.1.0 → prelude_parser-0.2.0}/CONTRIBUTING.md

@@ -15,15 +15,18 @@ Please include:
 [GitHub markdown](https://docs.github.com/en/free-pro-team@latest/github/writing-on-github). For example:
 
     ```py
-    from prelude_parser import parse_flat_file
+    from prelude_parser import parse_to_classes
 
-    data = parse_flat_file("my_file.xml")
+    data = parse_to_classes("my_file.xml")
     ```
 
 2. Explain what is currently happening and what you expect instead.
 
 ## Working on the code
 
+In order to work on this project you will need to have [Rust](https://www.rust-lang.org/) and
+[just](https://github.com/casey/just) in addition to Python.
+
 ### Fork the project
 
 In order to work on the project you will need your own fork. To do this click the "Fork" button on
@@ -39,12 +42,7 @@ git remote add upstream git@github.com:pbs-data-solutions/prelude-parser
 
 This creates the directory prelude-parser and connects your repository to the upstream (main project) repository.
 
-### Working with the code
-
-In order to work on this project you will need to have [Rust](https://www.rust-lang.org/) in
-addition to Python.
-
-Once Rust is installed you should create a vitural environment and activate it.
+Next create a vitural environment and activate it.
 
 ```sh
 python -m venv .venv
@@ -52,16 +50,16 @@ python -m venv .venv
 . .venv/bin/activate
 ```
 
-Then install the dev requirements
+Then install the dev requirements. This will also build the Rust module in dev mode and install it.
 
 ```sh
-pip install -r requirements-dev.txt
+just install
 ```
 
-Now the Rust module can be built in dev mode.
+When changes are made to the Rust code, build and install the new module with:
 
 ```sh
-maturin develop
+just develop
 ```
 
 ### Creating a branch
@@ -95,20 +93,7 @@ formmating.
 You can run linting on your code at any time with:
 
 ```sh
-# Run ruff
-ruff check .
-
-# Run black
-black prelude_parser tests
-
-# Run mypy
-mypy .
-
-# clippy
-cargo clippy
-
-# rustfmt
-cargo fmt
+just lint
 ```
 
 It is also suggested that you setup [pre-commit](https://pre-commit.com/) in order to run linting
@@ -165,7 +150,7 @@ be accepted. You can view the current coverage level in the codecov badge on the
 the code coverage by running:
 
 ```sh
-pytest
+just test
 ```
 
 In additon to mainting the coverage percentage please ensure that all tests are passing before

{prelude_parser-0.1.0 → prelude_parser-0.2.0}/Cargo.lock

@@ -82,7 +82,7 @@ dependencies = [
 
 [[package]]
 name = "prelude-parser"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "pyo3",
  "roxmltree",

{prelude_parser-0.1.0 → prelude_parser-0.2.0}/Cargo.toml

@@ -1,6 +1,7 @@
 [package]
 name = "prelude-parser"
-version = "0.1.0"
+version = "0.2.0"
+description = "Parses XML files exported from Prelude EDC into formats usable by Python."
 edition = "2021"
 license = "MIT"
 homepage = "https://github.com/pbs-data-solutions/prelude-parser"

{prelude_parser-0.1.0 → prelude_parser-0.2.0}/PKG-INFO

@@ -1,10 +1,25 @@
 Metadata-Version: 2.1
 Name: prelude-parser
-Version: 0.1.0
+Version: 0.2.0
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
-Requires-Dist: camel-converter>=3.0.0
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Typing :: Typed
+Requires-Dist: camel-converter >=3.0.0
+Requires-Dist: pandas >=2.0.1 ; extra == 'pandas'
+Requires-Dist: polars >=0.17.14 ; extra == 'polars'
+Requires-Dist: pandas >=2.0.1 ; extra == 'all'
+Requires-Dist: polars >=0.17.14 ; extra == 'all'
+Provides-Extra: pandas
+Provides-Extra: polars
+Provides-Extra: all
 License-File: LICENSE
+Summary: Parses XML files exported from Prelude EDC into formats usable by Python.
+Keywords: parser,prelude-edc,xml,pandas,polars
 Home-Page: https://github.com/pbs-data-solutions/prelude-parser
 Author-email: Paul Sanders <paul@pbsdatasolutions.com>
 License: MIT
@@ -20,7 +35,7 @@ Project-URL: Source Code, https://github.com/pbs-data-solutions/prelude-parser
 [![PyPI version](https://badge.fury.io/py/prelude-parser.svg)](https://badge.fury.io/py/prelude-parser)
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/prelude-parser?color=5cc141)](https://github.com/pbs-data-solutions/prelude-parser)
 
-Parses XML files exported from [Prelude EDC](https://preludeedc.com/) into Python classes.
+Parses XML files exported from [Prelude EDC](https://preludeedc.com/) into formats Python can use.
 
 ## Installation
 
@@ -28,6 +43,24 @@ Parses XML files exported from [Prelude EDC](https://preludeedc.com/) into Pytho
 pip install prelude-parser
 ```
 
+Optionally the `pandas` extra can be installed to parse to a Pandas `DataFrame`
+
+```sh
+pip install prelude-parser[pandas]
+```
+
+Optionally the `polars` extra can be installed to parse to a Polars `DataFrame`
+
+```sh
+pip install prelude-parser[polars]
+```
+
+All extras can be install with
+
+```sh
+pip install prelude-parser[all]
+```
+
 ## Contributing
 
 Contributions to this project are welcome. If you are interesting in contributing please see our [contributing guide](CONTRIBUTING.md)

{prelude_parser-0.1.0 → prelude_parser-0.2.0}/README.md

@@ -6,7 +6,7 @@
 [![PyPI version](https://badge.fury.io/py/prelude-parser.svg)](https://badge.fury.io/py/prelude-parser)
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/prelude-parser?color=5cc141)](https://github.com/pbs-data-solutions/prelude-parser)
 
-Parses XML files exported from [Prelude EDC](https://preludeedc.com/) into Python classes.
+Parses XML files exported from [Prelude EDC](https://preludeedc.com/) into formats Python can use.
 
 ## Installation
 
@@ -14,6 +14,24 @@ Parses XML files exported from [Prelude EDC](https://preludeedc.com/) into Pytho
 pip install prelude-parser
 ```
 
+Optionally the `pandas` extra can be installed to parse to a Pandas `DataFrame`
+
+```sh
+pip install prelude-parser[pandas]
+```
+
+Optionally the `polars` extra can be installed to parse to a Polars `DataFrame`
+
+```sh
+pip install prelude-parser[polars]
+```
+
+All extras can be install with
+
+```sh
+pip install prelude-parser[all]
+```
+
 ## Contributing
 
 Contributions to this project are welcome. If you are interesting in contributing please see our [contributing guide](CONTRIBUTING.md)

prelude_parser-0.2.0/justfile

@@ -0,0 +1,40 @@
+@develop:
+  maturin develop
+
+@install: develop
+  pip install -r requirements-dev.txt
+
+@lint:
+  echo cargo check
+  just --justfile {{justfile()}} check
+  echo cargo clippy
+  just --justfile {{justfile()}} clippy
+  echo cargo fmt
+  just --justfile {{justfile()}} fmt
+  echo ruff
+  just --justfile {{justfile()}} ruff
+  echo black
+  just --justfile {{justfile()}} black
+  echo mypy
+  just --justfile {{justfile()}} mypy
+
+@check:
+  cargo check
+
+@clippy:
+  cargo clippy
+
+@fmt:
+  cargo fmt
+
+@black:
+  black prelude_parser tests
+
+@mypy:
+  mypy .
+
+@ruff:
+  ruff check . --fix
+
+@test:
+  pytest

prelude_parser-0.2.0/prelude_parser/__init__.py

@@ -0,0 +1,3 @@
+from prelude_parser.parser import parse_to_classes, parse_to_dict
+
+__all__ = ["parse_to_classes", "parse_to_dict"]

{prelude_parser-0.1.0 → prelude_parser-0.2.0}/prelude_parser/_prelude_parser.pyi

@@ -3,7 +3,8 @@ from __future__ import annotations
 from pathlib import Path
 from typing import Any
 
-def _parse_flat_file(xml_file: str | Path) -> dict[str, list[Any]]: ...
+def _parse_flat_file_to_dict(xml_file: str | Path) -> dict[str, list[dict[str, Any]]]: ...
+def _parse_flat_file_to_pandas_dict(xml_file: str | Path) -> dict[str, list[Any]]: ...
 
 class FileNotFoundError(Exception):
     pass

prelude_parser-0.2.0/prelude_parser/pandas.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from prelude_parser._prelude_parser import _parse_flat_file_to_pandas_dict
+
+try:
+    import pandas as pd
+except ImportError:  # pragma: no cover
+    raise ImportError("prelude-parser must be installed with the pandas or all extra to use pandas")
+
+
+def to_dataframe(xml_file: str | Path) -> pd.DataFrame:
+    """Parse a Prelude flat XML file into a Pandas DataFrame.
+
+    This works for Prelude flat XML files that were exported with the "write tables to seperate
+    files" option.
+
+    Args:
+        xml_file: The path to the XML file to parser.
+
+    Returns:
+        A Pandas DataFrame the data from the XML file.
+
+    Examples:
+        >>> from prelude_parser.pandas import to_dataframe
+        >>> df = to_dataframe("physical_examination.xml")
+    """
+    data = _parse_flat_file_to_pandas_dict(xml_file)
+    return pd.DataFrame.from_dict(data)

prelude_parser-0.2.0/prelude_parser/parser.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from camel_converter import to_pascal
+
+from prelude_parser._prelude_parser import _parse_flat_file_to_dict
+
+
+class _MetaCls(type):
+    def __new__(
+        cls, clsname: str, superclasses: tuple[type, ...], attributedict: dict[str, Any]
+    ) -> _MetaCls:
+        return super(_MetaCls, cls).__new__(cls, clsname, superclasses, attributedict)
+
+
+def parse_to_dict(xml_file: str | Path) -> dict[str, list[dict[str, Any]]]:
+    """Parse a Prelude flat XML file into a dict.
+
+    Args:
+        xml_file: The path to the XML file to parser.
+
+    Returns:
+        A Python dictionary containing the data from the XML file.
+
+    Examples:
+        >>> from prelude_parser import parse_to_dict
+        >>> data = parse_to_dict("physical_examination.xml")
+    """
+    parsed = _parse_flat_file_to_dict(xml_file)
+    for _, data in parsed.items():
+        for d in data:
+            for k, v in d.items():
+                try:
+                    if "." in v:
+                        d[k] = float(v)
+                    else:
+                        d[k] = int(v)
+                    continue
+                except (TypeError, ValueError):
+                    pass
+
+                try:
+                    d[k] = datetime.strptime(v, "%d-%b-%Y")
+                except (TypeError, ValueError):
+                    pass
+
+    return parsed
+
+
+def parse_to_classes(xml_file: str | Path) -> list[Any]:
+    """Parse a Prelude flat XML file into a list of Python class.
+
+    The name of the class is taken from the form name node in the XML file converted to pascal case.
+    For example a <physical_examination> node will result in a PhysicalExamination class being
+    created.
+
+    Args:
+        xml_file: The path to the XML file to parser.
+
+    Returns:
+        A list of Python classes containing the data from the XML file.
+
+    Examples:
+        >>> from prelude_parser import parse_to_classes
+        >>> data = parse_to_classes("physical_examination.xml")
+    """
+    parsed = parse_to_dict(xml_file)
+    formatted: list[Any] = []
+    for form, data in parsed.items():
+        class_name = to_pascal(form)
+        for d in data:
+            formatted.append(_MetaCls(class_name, (object,), d))
+
+    return formatted

prelude_parser-0.2.0/prelude_parser/polars.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from prelude_parser._prelude_parser import _parse_flat_file_to_pandas_dict
+
+try:
+    import polars as pl
+except ImportError:  # pragma: no cover
+    raise ImportError("prelude-parser must be installed with the polars or all extra to use pandas")
+
+
+def to_dataframe(xml_file: str | Path) -> pl.DataFrame:
+    """Parse a Prelude flat XML file into a Polars DataFrame.
+
+    This works for Prelude flat XML files that were exported with the "write tables to seperate
+    files" option.
+
+    Args:
+        xml_file: The path to the XML file to parser.
+
+    Returns:
+        A Polars DataFrame the data from the XML file.
+
+    Examples:
+        >>> from prelude_parser.polars import to_dataframe
+        >>> df = to_dataframe("physical_examination.xml")
+    """
+    data = _parse_flat_file_to_pandas_dict(xml_file)
+    return pl.from_dict(data)

{prelude_parser-0.1.0 → prelude_parser-0.2.0}/pyproject.toml

@@ -1,14 +1,22 @@
 [build-system]
-requires = ["maturin>=0.14,<0.15"]
+requires = ["maturin>=0.15"]
 build-backend = "maturin"
 
 [project]
 name = "prelude-parser"
 requires-python = ">=3.8"
+description = "Parses XML files exported from Prelude EDC into formats usable by Python."
 authors = [{name = "Paul Sanders", email = "paul@pbsdatasolutions.com"}]
+keywords = ["parser", "prelude-edc", "xml", "pandas", "polars"]
 classifiers = [
   "Programming Language :: Rust",
   "Programming Language :: Python :: Implementation :: CPython",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "License :: OSI Approved :: MIT License",
+  "Typing :: Typed",
 ]
 license = {file = "LICENSE"}
 readme = "README.md"
@@ -17,6 +25,11 @@ homepage = "https://github.com/pbs-data-solutions/prelude-parser"
 documentation = "https://github.com/pbs-data-solutions/prelude-parser"
 dependencies = ["camel-converter>=3.0.0"]
 
+[project.optional-dependencies]
+pandas = ["pandas>=2.0.1"]
+polars = ["polars>=0.17.14"]
+all = ["pandas>=2.0.1", "polars>=0.17.14"]
+
 [tool.maturin]
 module-name = "prelude_parser._prelude_parser"
 binding = "pyo3"

prelude_parser-0.2.0/requirements-dev.txt

@@ -0,0 +1,10 @@
+black==23.3.0
+maturin==0.15.2
+mypy==1.3.0
+pandas-stubs==2.0.1.230501
+pre-commit==3.3.2
+pandas
+polars
+pytest==7.3.1
+pytest-cov==4.0.0
+ruff==0.0.269

prelude_parser-0.2.0/src/lib.rs

@@ -0,0 +1,178 @@
+use std::collections::HashMap;
+use std::fs::read_to_string;
+use std::path::PathBuf;
+
+use pyo3::create_exception;
+use pyo3::exceptions::PyException;
+use pyo3::prelude::*;
+use pyo3::types::{IntoPyDict, PyDict};
+use roxmltree::Document;
+
+create_exception!(_prelude_parser, FileNotFoundError, PyException);
+create_exception!(_prelude_parser, InvalidFileTypeError, PyException);
+create_exception!(_prelude_parser, ParsingError, PyException);
+
+fn to_snake(camel_string: &str) -> String {
+    let snake_string: String = camel_string
+        .chars()
+        .map(|c| {
+            if c.is_uppercase() {
+                format!("_{}", c)
+            } else {
+                c.to_string()
+            }
+        })
+        .collect();
+
+    // There is some weirdness in the Prelude XML files that can result in names like
+    // `i_communications_Details`. Because of this we can end up with values like
+    // `i_communications__details` so we need to fix double `__` values when we return the string.
+
+    snake_string
+        .trim_start_matches('_')
+        .to_lowercase()
+        .replace("__", "_")
+}
+
+fn parse_xml<'py>(py: Python<'py>, xml_file: &PathBuf) -> PyResult<&'py PyDict> {
+    let reader = read_to_string(xml_file);
+
+    match reader {
+        Ok(r) => match Document::parse(&r) {
+            Ok(doc) => {
+                let mut data: HashMap<String, Vec<&PyDict>> = HashMap::new();
+                let tree = doc.root_element();
+                for form in tree.children() {
+                    let form_name = to_snake(form.tag_name().name());
+                    if !form_name.is_empty() {
+                        if let Some(d) = data.get_mut(&form_name) {
+                            let mut form_data: HashMap<String, Option<&str>> = HashMap::new();
+                            for child in form.children() {
+                                if child.is_element() && child.tag_name().name() != "" {
+                                    let key = to_snake(child.tag_name().name());
+                                    form_data.insert(key, child.text());
+                                }
+                            }
+                            d.push(form_data.into_py_dict(py));
+                        } else {
+                            let mut items: Vec<&PyDict> = Vec::new();
+                            let mut form_data: HashMap<String, Option<&str>> = HashMap::new();
+                            for child in form.children() {
+                                if child.is_element() && child.tag_name().name() != "" {
+                                    let key = to_snake(child.tag_name().name());
+                                    form_data.insert(key, child.text());
+                                }
+                            }
+                            items.push(form_data.into_py_dict(py));
+                            data.insert(form_name, items);
+                        }
+                    }
+                }
+                return Ok(data.into_py_dict(py));
+            }
+            Err(e) => Err(ParsingError::new_err(format!(
+                "Error parsing xml file: {:?}",
+                e
+            ))),
+        },
+        Err(e) => Err(ParsingError::new_err(format!(
+            "Error parsing xml file: {:?}",
+            e
+        ))),
+    }
+}
+
+fn parse_xml_pandas<'py>(py: Python<'py>, xml_file: &PathBuf) -> PyResult<&'py PyDict> {
+    let reader = read_to_string(xml_file);
+
+    match reader {
+        Ok(r) => match Document::parse(&r) {
+            Ok(doc) => {
+                let mut data: HashMap<&str, Vec<Option<&str>>> = HashMap::new();
+                let tree = doc.root_element();
+
+                for form in tree.children() {
+                    for child in form.children() {
+                        if child.is_element() && child.tag_name().name() != "" {
+                            let column = child.tag_name().name();
+                            if let Some(d) = data.get_mut(column) {
+                                d.push(child.text());
+                            } else {
+                                data.insert(column, vec![child.text()]);
+                            }
+                        }
+                    }
+                }
+                return Ok(data.into_py_dict(py));
+            }
+            Err(e) => Err(ParsingError::new_err(format!(
+                "Error parsing xml file: {:?}",
+                e
+            ))),
+        },
+        Err(e) => Err(ParsingError::new_err(format!(
+            "Error parsing xml file: {:?}",
+            e
+        ))),
+    }
+}
+
+fn validate_file(xml_file: &PathBuf) -> PyResult<()> {
+    if !xml_file.is_file() {
+        return Err(FileNotFoundError::new_err(format!(
+            "File not found: {:?}",
+            xml_file
+        )));
+    } else if xml_file.extension().unwrap() != "xml" {
+        return Err(InvalidFileTypeError::new_err(format!(
+            "{:?} is not an xml file",
+            xml_file
+        )));
+    }
+
+    Ok(())
+}
+
+#[pyfunction]
+fn _parse_flat_file_to_dict(py: Python, xml_file: PathBuf) -> PyResult<&PyDict> {
+    validate_file(&xml_file)?;
+    let data = parse_xml(py, &xml_file)?;
+
+    Ok(data)
+}
+
+#[pyfunction]
+fn _parse_flat_file_to_pandas_dict(py: Python, xml_file: PathBuf) -> PyResult<&PyDict> {
+    validate_file(&xml_file)?;
+    let data = parse_xml_pandas(py, &xml_file)?;
+
+    Ok(data)
+}
+
+#[pymodule]
+fn _prelude_parser(py: Python, m: &PyModule) -> PyResult<()> {
+    m.add_function(wrap_pyfunction!(_parse_flat_file_to_dict, m)?)?;
+    m.add_function(wrap_pyfunction!(_parse_flat_file_to_pandas_dict, m)?)?;
+    m.add("FileNotFoundError", py.get_type::<FileNotFoundError>())?;
+    m.add(
+        "InvalidFileTypeError",
+        py.get_type::<InvalidFileTypeError>(),
+    )?;
+    m.add("ParsingError", py.get_type::<ParsingError>())?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::assert_eq;
+
+    use super::*;
+
+    #[test]
+    fn test_to_snake() {
+        assert_eq!(
+            to_snake("i_communications_Details"),
+            String::from("i_communications_details")
+        );
+    }
+}

prelude_parser-0.2.0/tests/test_pandas.py

@@ -0,0 +1,26 @@
+import pandas as pd
+
+from prelude_parser.pandas import to_dataframe
+
+
+def test_pandas_to_dataframe(test_file_1):
+    result = to_dataframe(test_file_1)
+    data = {
+        "BaseForm": [
+            "communications.form.name.communications",
+            "communications.form.name.communications",
+        ],
+        "FormGroup": ["Communications", "Communications"],
+        "FormNumber": [None, None],
+        "FormState": ["In-Work", "In-Work"],
+        "FormTitle": ["Communications", "Communications"],
+        "PatientId": ["1681574905819", "1681574994823"],
+        "PatientName": ["ABC-001", "ABC-002"],
+        "SiteId": ["1681574834910", "1681574834910"],
+        "SiteName": ["Some Site", "Some Site"],
+        "StudyName": ["PBS", "PBS"],
+        "communications_made": ["Yes", "Yes"],
+    }
+    expected = pd.DataFrame.from_dict(data)
+    result = result.reindex(sorted(result.columns), axis=1)
+    assert expected.equals(result)

prelude_parser-0.2.0/tests/test_parser.py

@@ -0,0 +1,226 @@
+from datetime import datetime
+
+import pytest
+
+from prelude_parser import parse_to_classes, parse_to_dict
+from prelude_parser._prelude_parser import FileNotFoundError, InvalidFileTypeError, ParsingError
+
+
+def test_parse_to_classes(test_file_1):
+    result = parse_to_classes(test_file_1)
+    assert len(result) == 2
+    assert result[0].__name__ == "Communications"
+    assert result[0].study_name == "PBS"
+    assert result[0].site_name == "Some Site"
+    assert result[0].site_id == 1681574834910
+    assert result[0].patient_name == "ABC-001"
+    assert result[0].patient_id == 1681574905819
+    assert result[0].form_title == "Communications"
+    assert result[0].base_form == "communications.form.name.communications"
+    assert result[0].form_number is None
+    assert result[0].form_group == "Communications"
+    assert result[0].form_state == "In-Work"
+    assert result[0].communications_made == "Yes"
+
+
+def test_parse_to_classes_with_float(test_file_2):
+    result = parse_to_classes(test_file_2)
+    assert len(result) == 2
+    assert result[0].__name__ == "Demographics"
+    assert result[0].weight == 80.2
+    assert result[0].dob == datetime(2020, 4, 15)
+
+
+def test_parse_to_classes_i_form(test_file_3):
+    result = parse_to_classes(test_file_3)
+    assert len(result) == 3
+    assert result[0].__name__ == "ICommunicationsDetails"
+    assert result[0].study_name == "PBS"
+    assert result[0].site_name == "Some Site"
+    assert result[0].site_id == 1681574834910
+    assert result[0].patient_name == "ABC-001"
+    assert result[0].patient_id == 1681574905819
+    assert result[0].form_title == "Communications"
+    assert result[0].base_form == "communications.form.name.communications"
+    assert result[0].form_number is None
+    assert result[0].form_group == "Communications"
+    assert result[0].form_state == "In-Work"
+    assert result[0].i == 1
+    assert result[0].contacted_by == "You"
+    assert result[0].investigator == "Dr. Me"
+    assert result[0].communication == "Some random talk"
+
+
+def test_parse_to_classes_not_found_error():
+    with pytest.raises(FileNotFoundError):
+        parse_to_classes("bad.xml")
+
+
+def test_parse_to_classes_invalid_file_type_error(tmp_path):
+    bad = tmp_path / "bad.txt"
+    bad.touch()
+    with pytest.raises(InvalidFileTypeError):
+        parse_to_classes(bad)
+
+
+def test_parse_to_classes_parsing_error(tmp_path):
+    bad = tmp_path / "bad.xml"
+    bad.touch()
+    with pytest.raises(ParsingError):
+        parse_to_classes(bad)
+
+
+def test_parse_to_dict(test_file_1):
+    result = parse_to_dict(test_file_1)
+    expected = {
+        "communications": [
+            {
+                "base_form": "communications.form.name.communications",
+                "communications_made": "Yes",
+                "form_group": "Communications",
+                "form_number": None,
+                "form_state": "In-Work",
+                "form_title": "Communications",
+                "patient_id": 1681574905819,
+                "patient_name": "ABC-001",
+                "site_id": 1681574834910,
+                "site_name": "Some Site",
+                "study_name": "PBS",
+            },
+            {
+                "base_form": "communications.form.name.communications",
+                "communications_made": "Yes",
+                "form_group": "Communications",
+                "form_number": None,
+                "form_state": "In-Work",
+                "form_title": "Communications",
+                "patient_id": 1681574994823,
+                "patient_name": "ABC-002",
+                "site_id": 1681574834910,
+                "site_name": "Some Site",
+                "study_name": "PBS",
+            },
+        ]
+    }
+
+    result["communications"] = [dict(sorted(x.items())) for x in result["communications"]]
+
+    assert result == expected
+
+
+def test_parse_to_dict_with_float(test_file_2):
+    result = parse_to_dict(test_file_2)
+    expected = {
+        "demographics": [
+            {
+                "base_form": "day.0.form.name.demographics",
+                "breed": "Labrador",
+                "dob": datetime(2020, 4, 15),
+                "first_name": "Imma",
+                "form_group": "Day 0",
+                "form_number": None,
+                "form_state": "In-Work",
+                "form_title": "Demographics",
+                "gender": "Female Spayed",
+                "last_name": "Dog",
+                "patient_id": 1681574905819,
+                "patient_name": "ABC-001",
+                "screening_number": 1,
+                "site_id": 1681574834910,
+                "site_name": "Some Site",
+                "site_type": "Live",
+                "study_name": "PBS",
+                "subject_id": "ABC-001",
+                "visit_date": datetime(2023, 4, 15),
+                "weight": 80.2,
+            },
+            {
+                "base_form": "day.0.form.name.demographics",
+                "breed": "Golden",
+                "dob": datetime(2019, 4, 9),
+                "first_name": "Arthur",
+                "form_group": "Day 0",
+                "form_number": None,
+                "form_state": "In-Work",
+                "form_title": "Demographics",
+                "gender": "Male Neutered",
+                "last_name": "Dent",
+                "patient_id": 1681574994823,
+                "patient_name": "ABC-002",
+                "screening_number": 2,
+                "site_id": 1681574834910,
+                "site_name": "Some Site",
+                "site_type": "Live",
+                "study_name": "PBS",
+                "subject_id": "ABC-002",
+                "visit_date": datetime(2023, 4, 15),
+                "weight": 40.5,
+            },
+        ]
+    }
+
+    result["demographics"] = [dict(sorted(x.items())) for x in result["demographics"]]
+
+    assert result == expected
+
+
+def test_parse_to_dict_i_form(test_file_3):
+    result = parse_to_dict(test_file_3)
+    expected = {
+        "i_communications_details": [
+            {
+                "base_form": "communications.form.name.communications",
+                "communication": "Some random talk",
+                "contacted_by": "You",
+                "form_group": "Communications",
+                "form_number": None,
+                "form_state": "In-Work",
+                "form_title": "Communications",
+                "i": 1,
+                "investigator": "Dr. Me",
+                "patient_id": 1681574905819,
+                "patient_name": "ABC-001",
+                "site_id": 1681574834910,
+                "site_name": "Some Site",
+                "study_name": "PBS",
+            },
+            {
+                "base_form": "communications.form.name.communications",
+                "communication": "We talked",
+                "contacted_by": "Hi",
+                "form_group": "Communications",
+                "form_number": None,
+                "form_state": "In-Work",
+                "form_title": "Communications",
+                "i": 1,
+                "investigator": "There",
+                "patient_id": 1681574994823,
+                "patient_name": "ABC-002",
+                "site_id": 1681574834910,
+                "site_name": "Some Site",
+                "study_name": "PBS",
+            },
+            {
+                "base_form": "communications.form.name.communications",
+                "communication": "We talked again",
+                "contacted_by": "There",
+                "i": 2,
+                "investigator": "Hi",
+                "form_group": "Communications",
+                "form_number": None,
+                "form_state": "In-Work",
+                "form_title": "Communications",
+                "patient_id": 1681574994823,
+                "patient_name": "ABC-002",
+                "site_id": 1681574834910,
+                "site_name": "Some Site",
+                "study_name": "PBS",
+            },
+        ]
+    }
+
+    result["i_communications_details"] = [
+        dict(sorted(x.items())) for x in result["i_communications_details"]
+    ]
+
+    assert result == expected

prelude_parser-0.2.0/tests/test_polars.py

@@ -0,0 +1,27 @@
+import polars as pl
+from polars.testing import assert_frame_equal
+
+from prelude_parser.polars import to_dataframe
+
+
+def test_pandas_to_dataframe(test_file_1):
+    result = to_dataframe(test_file_1)
+    data = {
+        "BaseForm": [
+            "communications.form.name.communications",
+            "communications.form.name.communications",
+        ],
+        "FormGroup": ["Communications", "Communications"],
+        "FormNumber": [None, None],
+        "FormState": ["In-Work", "In-Work"],
+        "FormTitle": ["Communications", "Communications"],
+        "PatientId": ["1681574905819", "1681574994823"],
+        "PatientName": ["ABC-001", "ABC-002"],
+        "SiteId": ["1681574834910", "1681574834910"],
+        "SiteName": ["Some Site", "Some Site"],
+        "StudyName": ["PBS", "PBS"],
+        "communications_made": ["Yes", "Yes"],
+    }
+    expected = pl.from_dict(data)  # type: ignore
+    result = result.pipe(lambda x: x.select(sorted(x.columns)))
+    assert_frame_equal(expected, result)

prelude_parser-0.1.0/prelude_parser/__init__.py

@@ -1,3 +0,0 @@
-from prelude_parser.parser import parse_flat_file
-
-__all__ = ["parse_flat_file"]

prelude_parser-0.1.0/prelude_parser/parser.py

@@ -1,61 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime
-from pathlib import Path
-from typing import Any
-
-from camel_converter import to_pascal, to_snake
-
-from prelude_parser._prelude_parser import _parse_flat_file
-
-
-class _MetaCls(type):
-    def __new__(cls, clsname: str, superclasses: tuple[type, ...], attributedict: dict) -> _MetaCls:
-        return super(_MetaCls, cls).__new__(cls, clsname, superclasses, attributedict)
-
-
-def parse_flat_file(xml_file: str | Path) -> list[Any]:
-    """Parse a Prelude flat XML file into a Python class.
-
-    The name of the class is taken from the form name node in the XML file converted to pascal case.
-    For example a <physical_examination> node will result in a PhysicalExamination class being
-    created.
-
-    Args:
-        xml_file: The path to the XML file to parser.
-
-    Returns:
-        A list of Python classes containing the data from the XML file.
-
-    Examples:
-        >>> from prelude_parser import parse_flat_file
-        >>> data = parse_flat_file("physical_examination.xml")
-    """
-    parsed = _parse_flat_file(xml_file)
-    formatted: list[Any] = []
-    for form, data in parsed.items():
-        class_name = to_pascal(form)
-        formatted_data: dict[str, Any] = {}
-
-        for d in data:
-            for k, v in d.items():
-                key = to_snake(k)
-                try:
-                    if "." in v:
-                        formatted_data[key] = float(v)
-                    else:
-                        formatted_data[key] = int(v)
-                    continue
-                except (TypeError, ValueError):
-                    pass
-
-                try:
-                    formatted_data[key] = datetime.strptime(v, "%d-%b-%Y")
-                    continue
-                except (TypeError, ValueError):
-                    pass
-
-                formatted_data[key] = v
-            formatted.append(_MetaCls(class_name, (object,), formatted_data))
-
-    return formatted

prelude_parser-0.1.0/requirements-dev.txt

@@ -1,6 +0,0 @@
-black==23.3.0
-maturin==0.14.17
-mypy==1.2.0
-pytest==7.3.1
-pytest-cov==4.0.0
-ruff==0.0.263

prelude_parser-0.1.0/src/lib.rs

@@ -1,95 +0,0 @@
-use std::collections::HashMap;
-use std::fs::read_to_string;
-use std::path::PathBuf;
-
-use pyo3::create_exception;
-use pyo3::exceptions::PyException;
-use pyo3::prelude::*;
-use pyo3::types::{IntoPyDict, PyDict};
-use roxmltree::Document;
-
-create_exception!(_prelude_parser, FileNotFoundError, PyException);
-create_exception!(_prelude_parser, InvalidFileTypeError, PyException);
-create_exception!(_prelude_parser, ParsingError, PyException);
-
-fn parse_xml<'py>(py: Python<'py>, xml_file: &PathBuf) -> PyResult<&'py PyDict> {
-    let reader = read_to_string(xml_file);
-
-    match reader {
-        Ok(r) => match Document::parse(&r) {
-            Ok(doc) => {
-                let mut data: HashMap<&str, Vec<&PyDict>> = HashMap::new();
-                let tree = doc.root_element();
-                for form in tree.children() {
-                    let form_name = form.tag_name().name();
-                    if !form_name.is_empty() {
-                        if let Some(d) = data.get_mut(form_name) {
-                            let mut form_data: HashMap<&str, Option<&str>> = HashMap::new();
-                            for child in form.children() {
-                                if child.is_element() && child.tag_name().name() != "" {
-                                    form_data.insert(child.tag_name().name(), child.text());
-                                }
-                            }
-                            d.push(form_data.into_py_dict(py));
-                        } else {
-                            let mut items: Vec<&PyDict> = Vec::new();
-                            let mut form_data: HashMap<&str, Option<&str>> = HashMap::new();
-                            for child in form.children() {
-                                if child.is_element() && child.tag_name().name() != "" {
-                                    form_data.insert(child.tag_name().name(), child.text());
-                                }
-                            }
-                            items.push(form_data.into_py_dict(py));
-                            data.insert(form_name, items);
-                        }
-                    }
-                }
-                return Ok(data.into_py_dict(py));
-            }
-            Err(e) => Err(ParsingError::new_err(format!(
-                "Error parsing xml file: {:?}",
-                e
-            ))),
-        },
-        Err(e) => Err(ParsingError::new_err(format!(
-            "Error parsing xml file: {:?}",
-            e
-        ))),
-    }
-}
-
-fn validate_file(xml_file: &PathBuf) -> PyResult<()> {
-    if !xml_file.is_file() {
-        return Err(FileNotFoundError::new_err(format!(
-            "File not found: {:?}",
-            xml_file
-        )));
-    } else if xml_file.extension().unwrap() != "xml" {
-        return Err(InvalidFileTypeError::new_err(format!(
-            "{:?} is not an xml file",
-            xml_file
-        )));
-    }
-
-    Ok(())
-}
-
-#[pyfunction]
-fn _parse_flat_file(py: Python, xml_file: PathBuf) -> PyResult<&PyDict> {
-    validate_file(&xml_file)?;
-    let data = parse_xml(py, &xml_file)?;
-
-    Ok(data)
-}
-
-#[pymodule]
-fn _prelude_parser(py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_function(wrap_pyfunction!(_parse_flat_file, m)?)?;
-    m.add("FileNotFoundError", py.get_type::<FileNotFoundError>())?;
-    m.add(
-        "InvalidFileTypeError",
-        py.get_type::<InvalidFileTypeError>(),
-    )?;
-    m.add("ParsingError", py.get_type::<ParsingError>())?;
-    Ok(())
-}

prelude_parser-0.1.0/tests/test_parser.py

@@ -1,70 +0,0 @@
-from datetime import datetime
-
-import pytest
-
-from prelude_parser._prelude_parser import FileNotFoundError, InvalidFileTypeError, ParsingError
-from prelude_parser.parser import parse_flat_file
-
-
-def test_parse_flat_file(test_file_1):
-    result = parse_flat_file(test_file_1)
-    assert len(result) == 2
-    assert result[0].__name__ == "Communications"
-    assert result[0].study_name == "PBS"
-    assert result[0].site_name == "Some Site"
-    assert result[0].site_id == 1681574834910
-    assert result[0].patient_name == "ABC-001"
-    assert result[0].patient_id == 1681574905819
-    assert result[0].form_title == "Communications"
-    assert result[0].base_form == "communications.form.name.communications"
-    assert result[0].form_number is None
-    assert result[0].form_group == "Communications"
-    assert result[0].form_state == "In-Work"
-    assert result[0].communications_made == "Yes"
-
-
-def test_parse_flat_file_with_float(test_file_2):
-    result = parse_flat_file(test_file_2)
-    assert len(result) == 2
-    assert result[0].__name__ == "Demographics"
-    assert result[0].weight == 80.2
-    assert result[0].dob == datetime(2020, 4, 15)
-
-
-def test_parse_flat_file_i_form(test_file_3):
-    result = parse_flat_file(test_file_3)
-    assert len(result) == 3
-    assert result[0].__name__ == "ICommunicationsDetails"
-    assert result[0].study_name == "PBS"
-    assert result[0].site_name == "Some Site"
-    assert result[0].site_id == 1681574834910
-    assert result[0].patient_name == "ABC-001"
-    assert result[0].patient_id == 1681574905819
-    assert result[0].form_title == "Communications"
-    assert result[0].base_form == "communications.form.name.communications"
-    assert result[0].form_number is None
-    assert result[0].form_group == "Communications"
-    assert result[0].form_state == "In-Work"
-    assert result[0].i == 1
-    assert result[0].contacted_by == "You"
-    assert result[0].investigator == "Dr. Me"
-    assert result[0].communication == "Some random talk"
-
-
-def test_parse_flat_file_not_found_error():
-    with pytest.raises(FileNotFoundError):
-        parse_flat_file("bad.xml")
-
-
-def test_parse_flat_file_invalid_file_type_error(tmp_path):
-    bad = tmp_path / "bad.txt"
-    bad.touch()
-    with pytest.raises(InvalidFileTypeError):
-        parse_flat_file(bad)
-
-
-def test_parse_flat_file_parsing_error(tmp_path):
-    bad = tmp_path / "bad.xml"
-    bad.touch()
-    with pytest.raises(ParsingError):
-        parse_flat_file(bad)