tea-data-file-conversion 0.1.1__py3-none-any.whl

tea_data_file_conversion/__init__.py

@@ -0,0 +1,7 @@
+__version__ = "0.1.1"
+
+# fixedwidth_processor/__init__.py
+
+from .processor import export_templates, process_file, validate_yaml_config
+
+__all__ = ["export_templates", "process_file", "validate_yaml_config"]

tea_data_file_conversion/cli.py

@@ -0,0 +1,63 @@
+# file: src/tea_data_file_conversion/cli.py
+
+r"""Command-line interface for fixed\-width file processing.
+
+This module provides an entry point to either process a fixed\-width file
+into CSV format using a dynamic YAML schema or export default YAML templates.
+"""
+
+import argparse
+
+from .processor import export_templates, process_file
+
+
+def main():
+    r"""Parse command\-line arguments and execute the corresponding action.
+
+    Options:
+      \- Process a fixed\-width file to CSV.
+      \- Export YAML template files if the --export_templates flag is set.
+    """
+    # Set up the argument parser.
+    parser = argparse.ArgumentParser(
+        description=r"Process a fixed\-width file and output a CSV based on dynamic YAML schema."
+    )
+    # Input file (required).
+    parser.add_argument("input_file", help=r"Path to the input fixed\-width file.")
+    # Optional output file.
+    parser.add_argument(
+        "--output_file",
+        help=(
+            "Optional path for the output CSV file. "
+            "If not provided, defaults to the input file name with '_output.csv' appended."
+        ),
+        default=None,
+    )
+    # Optional schema folder location.
+    parser.add_argument(
+        "--schema_folder",
+        help="Path to the folder containing YAML schema files "
+        "(or where templates will be exported). Defaults to current directory.",
+        default=".",
+    )
+    # Flag to export templates.
+    parser.add_argument(
+        "--export_templates",
+        help=r"Export template YAML files from the built\-in "
+        r"default_schema folder to the specified schema_folder and exit.",
+        action="store_true",
+    )
+
+    # Parse the provided arguments.
+    args = parser.parse_args()
+
+    # If the export flag is set, export YAML templates and exit immediately.
+    if args.export_templates:
+        export_templates(args.schema_folder)
+
+    # Otherwise, process the file using the processed arguments.
+    process_file(args.input_file, args.output_file, schema_folder=args.schema_folder)
+
+
+if __name__ == "__main__":
+    main()

tea_data_file_conversion/processor.py

@@ -0,0 +1,340 @@
+# file: src/tea_data_file_conversion/processor.py
+
+r"""Processor module for fixed\-width file conversion.
+
+This module provides functions to:
+  \- Load and validate YAML schema configurations.
+  \- Process fixed\-width files into structured DataFrame objects.
+  \- Export template YAML schema files.
+  \- Convert CSV files into YAML schema files interactively.
+"""
+
+import os
+import shutil
+import sys
+
+import importlib_resources  # Used to locate package data.
+import pandas as pd
+import yaml
+
+
+def load_yaml_config(file_path):
+    """
+    Load a YAML configuration file for processing.
+
+    Parameters
+    ----------
+    file_path : str
+        The path to the YAML configuration file.
+
+    Returns
+    -------
+    dict
+        The parsed YAML configuration.
+
+    Raises
+    ------
+    ValueError
+        If there is an error parsing the YAML file.
+    """
+    try:
+        with open(file_path) as f:
+            config = yaml.safe_load(f)
+        return config
+    except yaml.YAMLError as ye:
+        # Raise an error with details of parsing issues.
+        raise ValueError(f"Error parsing YAML file {file_path}: {ye}") from ye
+
+
+def validate_yaml_config(config, file_path):
+    """
+    Validate the structure of the YAML configuration.
+
+    The configuration must be a dictionary containing a key 'fields' mapping to a list.
+    Each field in the list must contain 'start', 'end', and 'output_field' keys.
+
+    Parameters
+    ----------
+    config : dict
+        The YAML configuration dictionary.
+    file_path : str
+        File path used for reporting in error messages.
+
+    Raises
+    ------
+    ValueError
+        If the configuration does not adhere to the expected schema.
+    """
+    if not isinstance(config, dict):
+        raise ValueError(f"YAML file {file_path} should be a dictionary at the top level.")
+    if "fields" not in config:
+        raise ValueError(f"YAML file {file_path} is missing the required key 'fields'.")
+    if not isinstance(config["fields"], list):
+        raise ValueError(f"YAML file {file_path} key 'fields' should be a list.")
+
+    for index, field in enumerate(config["fields"]):
+        if not isinstance(field, dict):
+            raise ValueError(f"YAML file {file_path}, field at index {index} is not a dictionary.")
+        for key in ["start", "end", "output_field"]:
+            if key not in field:
+                raise ValueError(f"YAML file {file_path}, field at index {index} is missing required key '{key}'.")
+        if not isinstance(field["start"], int):
+            raise ValueError(f"YAML file {file_path}, field at index {index} key 'start' must be an integer.")
+        if not isinstance(field["end"], int):
+            raise ValueError(f"YAML file {file_path}, field at index {index} key 'end' must be an integer.")
+        if not isinstance(field["output_field"], str):
+            raise ValueError(f"YAML file {file_path}, field at index {index} key 'output_field' must be a string.")
+        if "keep" in field and not isinstance(field["keep"], bool):
+            raise ValueError(f"YAML file {file_path}, field at index {index} key 'keep' must be a boolean.")
+
+
+def process_fixed_width_file(input_file, schema_config, skip_header=False, filter_columns=False):
+    r"""
+    Process a fixed\-width file using the provided YAML schema configuration.
+
+    It determines column boundaries based on the schema, reads the file using pandas,
+    and applies optional filtering to only return columns marked to be kept.
+
+    Parameters
+    ----------
+    input_file : str
+        The path to the fixed\-width text file.
+    schema_config : dict
+        Schema configuration dictionary with field definitions.
+    skip_header : bool, optional
+        Skip the header row if True (default is False).
+    filter_columns : bool, optional
+        If True, return only DataFrame columns that are marked with "keep": true.
+
+    Returns
+    -------
+    pd.DataFrame
+        DataFrame with the processed data.
+    """
+    fields = schema_config["fields"]
+    colspecs = []  # List of tuples defining start and end positions for each field.
+    col_names = []  # List of column names derived from the schema.
+    keep_columns = []  # Track columns flagged to be retained.
+
+    for field in fields:
+        # Adjust the start position because the schema uses 1-based indexing.
+        start = field["start"] - 1
+        end = field["end"]
+        colspecs.append((start, end))
+        # Use 'mapped_field_name' when filtering columns if available.
+        if filter_columns:
+            col_name = (
+                field["mapped_field_name"] if not pd.isna(field.get("mapped_field_name")) else field["output_field"]
+            )
+        else:
+            col_name = field["output_field"]
+        col_names.append(col_name)
+        if field.get("keep", False):
+            keep_columns.append(col_name)
+
+    # Ensure each column name is unique by appending a counter if needed.
+    unique_col_names = []
+    for col_name in col_names:
+        if col_name in unique_col_names:
+            count = 1
+            new_col_name = f"{col_name}_{count}"
+            while new_col_name in unique_col_names:
+                count += 1
+                new_col_name = f"{col_name}_{count}"
+            unique_col_names.append(new_col_name)
+        else:
+            unique_col_names.append(col_name)
+
+    # Read the fixed\-width file into a DataFrame.
+    df = pd.read_fwf(input_file, colspecs=colspecs, header=None, names=unique_col_names)
+
+    if filter_columns:
+        df = df[keep_columns]
+
+    return df
+
+
+def process_file(input_file, output_file=None, schema_folder=None, filter_columns=False):
+    r"""
+    Process an input fixed\-width file and output a CSV file.
+
+    The function:
+      \- Determines the appropriate YAML schema based on header info.
+      \- Loads and validates the schema.
+      \- Processes the input file and writes the output DataFrame to CSV.
+
+    Parameters
+    ----------
+    input_file : str
+        The path to the fixed\-width input file.
+    output_file : str, optional
+        File path for the output CSV. Defaults to input file name with '_output.csv' appended.
+    schema_folder : str, optional
+        Folder where the YAML schema files are located; defaults to the current folder.
+    filter_columns : bool, optional
+        If True, only load columns flagged with "keep": true (default is False).
+
+    Returns
+    -------
+    pd.DataFrame
+        The processed DataFrame.
+    """
+    # Define the output CSV file name if not explicitly provided.
+    if output_file is None:
+        base, _ = os.path.splitext(input_file)
+        output_file = f"{base}_output.csv"
+
+    # Read and validate the header line.
+    with open(input_file) as f:
+        header_line = f.readline().strip()
+
+    if len(header_line) < 4:
+        raise ValueError("The header line must contain at least 4 characters.")
+
+    # Extract test month and abbreviated school year from header.
+    header = header_line[:4]
+    test_month = int(header[:2])
+    school_year_abbr = int(header[2:4])
+    full_school_year = 2000 + school_year_abbr
+
+    # Determine test type and adjust school year if necessary.
+    if test_month < 10:
+        test_name = "staar"
+    else:
+        test_name = "staar_eoc"
+        if test_month < 15:
+            full_school_year += 1
+
+    # Compose the path to the expected YAML schema file.
+    base_folder = schema_folder if schema_folder is not None else "default_schema"
+    schema_config_file = os.path.join(base_folder, test_name, f"{test_name}_{full_school_year}.yaml")
+    print(f"Loading schema config: {schema_config_file}")
+
+    # Load and validate the YAML configuration.
+    schema_config = load_yaml_config(schema_config_file)
+    try:
+        validate_yaml_config(schema_config, schema_config_file)
+    except ValueError as ve:
+        print(f"YAML validation error: {ve}")
+        sys.exit(1)
+
+    # Process the file using the loaded schema.
+    df = process_fixed_width_file(input_file, schema_config, skip_header=True, filter_columns=filter_columns)
+
+    # Write the processed data to a CSV file.
+    df.to_csv(output_file, index=False)
+    print(f"Data has been written to {output_file}")
+    return df
+
+
+def export_templates(schema_folder):
+    r"""
+    Export sample YAML template files to a specified folder.
+
+    The function copies files from the built\-in default_schema directory
+    (packaged with this module) into the target folder while preserving the
+    original directory structure.
+
+    Parameters
+    ----------
+    schema_folder : str
+        The destination folder for exporting the template YAML files.
+
+    Notes
+    -----
+    The function exits after exporting the template files.
+    """
+    # Locate the default_schema folder within the package.
+    with importlib_resources.path("fixedwidth_processor", "default_schema") as default_schema_path:
+        # Check if the default_schema_path is a valid directory.
+        if not os.path.isdir(str(default_schema_path)):
+            print("Default schema folder not found in package.")
+            sys.exit(1)
+
+        # Walk the directory using the string version of the path.
+        for root, _dirs, files in os.walk(str(default_schema_path)):
+            for file in files:
+                rel_path = os.path.relpath(os.path.join(root, file), str(default_schema_path))
+                target_file = os.path.join(schema_folder, rel_path)
+                os.makedirs(os.path.dirname(target_file), exist_ok=True)
+                shutil.copy(os.path.join(root, file), target_file)
+    print(f"Template YAML files exported to {schema_folder}.")
+    print(
+        "Please review and update the templates as needed, then run the script again using the --schema_folder option."
+    )
+    sys.exit(0)
+
+
+def csv_to_schema_yaml(csv_file, yaml_output_file=None):
+    r"""
+    Convert a CSV file into a YAML schema file for fixed\-width processing.
+
+    This function loads a CSV file, lists available columns, and interactively
+    prompts the user to select fields corresponding to start, end, and output
+    values, then writes out a YAML file with the chosen configuration.
+
+    Parameters
+    ----------
+    csv_file : str
+        Path to the input CSV file.
+    yaml_output_file : str, optional
+        Output file path for the YAML schema. If omitted, a default name is generated.
+    """
+    try:
+        df = pd.read_csv(csv_file)
+    except Exception as e:
+        print(f"Error loading CSV file: {e}")
+        return
+
+    # Display available CSV columns for user selection.
+    print("Available columns in the CSV:")
+    for col in df.columns:
+        print(f" - {col}")
+
+    # Request the user to enter the necessary columns.
+    start_col = input("Enter the name of the column representing the start value: ").strip()
+    end_col = input("Enter the name of the column representing the end value: ").strip()
+    output_field_col = input(
+        "Enter the name of the column representing the output field (e.g., 'Field Category - Field Title'): "
+    ).strip()
+
+    fields = []  # Prepare a list for schema field definitions.
+    for index, row in df.iterrows():
+        try:
+            start_value = int(row[start_col])
+        except (ValueError, TypeError):
+            print(f"Row {index}: Could not convert start value '{row[start_col]}' to int. Skipping this row.")
+            continue
+
+        try:
+            end_value = int(row[end_col])
+        except (ValueError, TypeError):
+            print(f"Row {index}: Could not convert end value '{row[end_col]}' to int. Skipping this row.")
+            continue
+
+        # Clean the output field by replacing special dash characters.
+        output_field_value = (
+            str(row[output_field_col]).replace("\u2010", "-").replace("\u2013", "-").replace("\n", "").replace("\r", "")
+        )
+        field_entry = {
+            "start": start_value,
+            "end": end_value,
+            "output_field": output_field_value,
+            "keep": row.get("keep", False),
+            "mapped_field_name": row.get("Mapped Field Title", output_field_value),
+        }
+        fields.append(field_entry)
+
+    data = {"fields": fields}
+
+    # Set default output YAML file name if none provided.
+    if yaml_output_file is None:
+        base, _ = os.path.splitext(csv_file)
+        yaml_output_file = f"{base}_schema.yaml"
+
+    try:
+        with open(yaml_output_file, "w") as f:
+            yaml.dump(data, f, sort_keys=False)
+        print(f"Schema YAML file successfully created: {yaml_output_file}")
+    except Exception as e:
+        print(f"Error writing YAML file: {e}")

tea_data_file_conversion-0.1.1.dist-info/LICENSE

@@ -0,0 +1,22 @@
+
+MIT License
+
+Copyright (c) 2025 Mark Moreno
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tea_data_file_conversion-0.1.1.dist-info/METADATA

@@ -0,0 +1,109 @@
+Metadata-Version: 2.2
+Name: tea-data-file-conversion
+Version: 0.1.1
+Summary: Fixedwidth Processor is a Python package designed to transform fixed-width text files into CSVs using dynamic YAML schema configurations.
+Author-email: Mark Moreno <mamoreno@aldineisd.org>
+License: MIT
+Project-URL: Bug Tracker, https://github.com/markm-io/tea-data-file-conversion/issues
+Project-URL: Changelog, https://github.com/markm-io/tea-data-file-conversion/blob/main/CHANGELOG.md
+Project-URL: documentation, https://tea-data-file-conversion.readthedocs.io
+Project-URL: repository, https://github.com/markm-io/tea-data-file-conversion
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: importlib-resources>=6.5.2
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: rich>=10
+Requires-Dist: typer<1,>=0.15
+
+# tea-data-file-conversion
+
+<p align="center">
+  <a href="https://github.com/markm-io/tea-data-file-conversion/actions/workflows/ci.yml?query=branch%3Amain">
+    <img src="https://img.shields.io/github/actions/workflow/status/markm-io/tea-data-file-conversion/ci.yml?branch=main&label=CI&logo=github&style=flat-square" alt="CI Status" >
+  </a>
+  <a href="https://tea-data-file-conversion.readthedocs.io">
+    <img src="https://img.shields.io/readthedocs/tea-data-file-conversion.svg?logo=read-the-docs&logoColor=fff&style=flat-square" alt="Documentation Status">
+  </a>
+  <a href="https://codecov.io/gh/markm-io/tea-data-file-conversion">
+    <img src="https://img.shields.io/codecov/c/github/markm-io/tea-data-file-conversion.svg?logo=codecov&logoColor=fff&style=flat-square" alt="Test coverage percentage">
+  </a>
+</p>
+<p align="center">
+  <a href="https://github.com/astral-sh/uv">
+    <img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json" alt="uv">
+  </a>
+  <a href="https://github.com/astral-sh/ruff">
+    <img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json" alt="Ruff">
+  </a>
+  <a href="https://github.com/pre-commit/pre-commit">
+    <img src="https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white&style=flat-square" alt="pre-commit">
+  </a>
+</p>
+<p align="center">
+  <a href="https://pypi.org/project/tea-data-file-conversion/">
+    <img src="https://img.shields.io/pypi/v/tea-data-file-conversion.svg?logo=python&logoColor=fff&style=flat-square" alt="PyPI Version">
+  </a>
+  <img src="https://img.shields.io/pypi/pyversions/tea-data-file-conversion.svg?style=flat-square&logo=python&amp;logoColor=fff" alt="Supported Python versions">
+  <img src="https://img.shields.io/pypi/l/tea-data-file-conversion.svg?style=flat-square" alt="License">
+</p>
+
+---
+
+**Documentation**: <a href="https://tea-data-file-conversion.readthedocs.io" target="_blank">https://tea-data-file-conversion.readthedocs.io </a>
+
+**Source Code**: <a href="https://github.com/markm-io/tea-data-file-conversion" target="_blank">https://github.com/markm-io/tea-data-file-conversion </a>
+
+---
+
+Fixedwidth Processor is a Python package designed to transform fixed-width text files into CSVs using dynamic YAML schema configurations.
+
+## Installation
+
+Install this via pip (or your favourite package manager):
+
+`pip install tea-data-file-conversion`
+
+## Contributors ✨
+
+Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
+
+<!-- prettier-ignore-start -->
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+<table>
+  <tbody>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/markm-io"><img src="https://avatars.githubusercontent.com/u/45011486?v=4?s=80" width="80px;" alt="Mark Moreno"/><br /><sub><b>Mark Moreno</b></sub></a><br /><a href="https://github.com/markm-io/tea-data-file-conversion/commits?author=markm-io" title="Code">💻</a> <a href="#ideas-markm-io" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/markm-io/tea-data-file-conversion/commits?author=markm-io" title="Documentation">📖</a></td>
+    </tr>
+  </tbody>
+</table>
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->
+<!-- prettier-ignore-end -->
+
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!
+
+## Credits
+
+[![Copier](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/copier-org/copier/master/img/badge/badge-grayscale-inverted-border-orange.json)](https://github.com/copier-org/copier)
+
+This package was created with
+[Copier](https://copier.readthedocs.io/) and the
+[browniebroke/pypackage-template](https://github.com/browniebroke/pypackage-template)
+project template.

tea_data_file_conversion-0.1.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+tea_data_file_conversion/__init__.py,sha256=_qK5HQcz-pMONPg9Yw6e-XI_Wj6LhpPBk4uz2QYBTlQ,207
+tea_data_file_conversion/cli.py,sha256=tL9Gnx9eBvCwAuvb3uUn2qH1lz56uM9JPbCpCEwrE9E,2070
+tea_data_file_conversion/processor.py,sha256=kak0qGCDYjaEW0QGtX_BsFVHLVRG-sSAmBnPFX0BxHc,12606
+tea_data_file_conversion/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tea_data_file_conversion-0.1.1.dist-info/LICENSE,sha256=PmXYgTt2lpACeHiHTQIHeMwcCdxA2dsOuiP7j4zq8sM,1069
+tea_data_file_conversion-0.1.1.dist-info/METADATA,sha256=MBAnDdGTc6ObgudNbeTx_20WzmG8_DBPkVwmjkbUwuI,5402
+tea_data_file_conversion-0.1.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tea_data_file_conversion-0.1.1.dist-info/entry_points.txt,sha256=eQyHdZ9rmyTUnqjMnDjL1f7cJvynhBhm9J4hY6X73eM,78
+tea_data_file_conversion-0.1.1.dist-info/top_level.txt,sha256=VrmxfkMdwWOVFan0XtZJnswHCaLkoFteFe-Ihd1zlNo,25
+tea_data_file_conversion-0.1.1.dist-info/RECORD,,

tea_data_file_conversion-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.8.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

tea_data_file_conversion-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+tea-data-file-conversion = tea_data_file_conversion.cli:app

tea_data_file_conversion-0.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+tea_data_file_conversion