datagrunt 0.0.0__py3-none-any.whl

datagrunt/__init__.py

@@ -0,0 +1,39 @@
+"""
+Datagrunt
+
+A Python library designed to simplify the way you work with CSV files.
+
+This module provides inferred CSV delimiters and helper methods for reading and writing CSV files.
+
+Example:
+    A simple example of how to use the main functionality of your package:
+
+    from datagrunt.csvfile import CSVReader
+
+    csv_file = 'electric_vehicle_population_data.csv'
+    engine = 'duckdb'
+
+    dg = CSVReader(csv_file, engine=engine)
+
+    dg.get_sample()
+
+Attributes:
+    __version__: A string representing the version of this module.
+    __author__: The name of the package author.
+    __license__: The license under which the package is released.
+"""
+
+__version__ = "0.0.0"
+__author__ = "Martin Graham"
+__license__ = "MIT"
+
+# Import key classes, functions, or submodules that should be available at the package level
+from .csvfile import CSVReader, CSVWriter
+
+# You can define __all__ to specify what gets imported with "from package import *"
+__all__ = ['CSVReader', 'CSVWriter']
+
+# Optionally, you can include a logger for your package
+import logging
+
+logging.getLogger(__name__).addHandler(logging.NullHandler())

datagrunt/core/databases.py

@@ -0,0 +1,53 @@
+"""Module for interfacing with databases."""
+
+# standard library
+import os
+from pathlib import Path
+import re
+
+# third party libraries
+import duckdb
+
+class DuckDBDatabase:
+    """Class to configure local database for file processing.
+       Utilizes duckdb as the processing engine.
+    """
+    DEFAULT_ENCODING = 'utf-8'
+    DEFAULT_THREAD_COUNT = 16
+
+    def __init__(self, filepath):
+        """
+        Initialize the FileDatabase class.
+
+        Args:
+            filepath (str): Path to the file.
+        """
+        self.filepath = filepath
+        self.database_filename = self._set_database_filename()
+        self.database_table_name = self._set_database_table_name()
+        self.database_connection = self._set_database_connection()
+
+    def __del__(self):
+        """Delete .db files after use."""
+        if os.path.exists(self.database_filename):
+            os.remove(self.database_filename)
+
+    def _format_filename_string(self):
+        """Remove all non alphanumeric characters from filename."""
+        return re.sub(r'[^a-zA-Z0-9]', '', Path(self.filepath).stem)
+
+    def _set_database_filename(self):
+        """Return name of duckdb file created at runtime."""
+        return f'{self._format_filename_string()}.db'
+
+    def _set_database_table_name(self):
+        """Return name of duckdb import table created during file import."""
+        return f'{self._format_filename_string()}'
+
+    def _set_database_connection(self, threads=DEFAULT_THREAD_COUNT):
+        """Establish a connection with duckdb.
+
+        Args:
+            threads (int): Number of threads to use for duckdb.
+        """
+        return duckdb.connect(self.database_filename, config = {'threads': threads})

datagrunt/core/engines.py

@@ -0,0 +1,231 @@
+"""Module engines to enable data processing."""
+
+# standard library
+
+# third party libraries
+import duckdb
+import polars as pl
+
+# local libraries
+from .fileproperties import CSVProperties
+from .queries import DuckDBQueries
+from .logger import show_large_file_warning, show_dataframe_sample
+
+class CSVReaderDuckDBEngine(CSVProperties):
+    """Class to read CSV files and convert CSV files powered by DuckDB."""
+
+    def __init__(self, filepath):
+        """
+        Initialize the CSVReader class.
+
+        Args:
+            filepath (str): Path to the file to read.
+        """
+        super().__init__(filepath)
+        self.db_table = DuckDBQueries(self.filepath).database_table_name
+
+    def _read_csv(self):
+        """Reads a CSV using DuckDB.
+
+        Returns:
+            A DuckDB DuckDBPyRelation.
+        """
+        return duckdb.read_csv(self.filepath,
+                               delimiter=self.delimiter,
+                               null_padding=True,
+                               all_varchar=True
+                            )
+
+    def get_sample(self):
+        """Return a sample of the CSV file."""
+        self._read_csv().show()
+
+    def to_dataframe(self):
+        """Converts CSV to a Polars dataframe.
+
+        Returns:
+            A Polars dataframe.
+        """
+        if self.is_large:
+            show_large_file_warning()
+        return self._read_csv().pl()
+
+    def to_arrow_table(self):
+        """Converts CSV to a PyArrow table.
+
+        Returns:
+            A PyArrow table.
+        """
+        arrow_table = self._read_csv().arrow()
+        return arrow_table
+
+    def to_dicts(self):
+        """Converts CSV to a list of Python dictionaries.
+
+        Returns:
+            A list of dictionaries.
+        """
+        dicts = self.to_dataframe().to_dicts()
+        return dicts
+
+class CSVReaderPolarsEngine(CSVProperties):
+    """Class to read CSV files and convert CSV files powered by Polars."""
+
+    def get_sample(self):
+        """Return a sample of the CSV file."""
+        df = pl.read_csv(self.filepath,
+                         separator=self.delimiter,
+                         truncate_ragged_lines=True,
+                         n_rows=self.DATAFRAME_SAMPLE_ROWS
+                        )
+        show_dataframe_sample(df)
+
+    def to_dataframe(self):
+        """Converts CSV to a Polars dataframe.
+
+        Returns:
+            A Polars dataframe.
+        """
+        if self.is_large:
+            show_large_file_warning()
+        return pl.read_csv(self.filepath,
+                           separator=self.delimiter,
+                           truncate_ragged_lines=True
+                           )
+
+    def to_arrow_table(self):
+        """Converts CSV to a PyArrow table.
+
+        Returns:
+            A PyArrow table.
+        """
+        df = self.to_dataframe().to_arrow()
+        return df
+
+    def to_dicts(self):
+        """Converts CSV to a list of Python dictionaries.
+
+        Returns:
+            A list of dictionaries.
+        """
+        dicts = self.to_dataframe().to_dicts()
+        return dicts
+
+class CSVWriterDuckDBEngine(CSVProperties):
+    """Class to convert CSV files to various other supported file types powered by DuckDB."""
+
+    def __init__(self, filepath):
+        """
+        Initialize the CSVWriter class.
+
+        Args:
+            filepath (str): Path to the file to write.
+        """
+        super().__init__(filepath)
+        self.queries = DuckDBQueries(self.filepath)
+
+    def write_csv(self, out_filename=None):
+        """Query to export a DuckDB table to a CSV file.
+
+            Args:
+                out_filename str: The name of the output file.
+            """
+        duckdb.sql(self.queries.import_csv_query(self.delimiter))
+        duckdb.sql(self.queries.export_csv_query(out_filename))
+
+    def write_excel(self, out_filename=None):
+        """Query to export a DuckDB table to an Excel file.
+
+        Args:
+            out_filename (optional, str): The name of the output file.
+        """
+        duckdb.sql(self.queries.import_csv_query(self.delimiter))
+        duckdb.sql(self.queries.export_excel_query(out_filename))
+
+    def write_json(self, out_filename=None):
+        """Query to export a DuckDB table to a JSON file.
+
+        Args:
+            out_filename (optional, str): The name of the output file.
+        """
+        duckdb.sql(self.queries.import_csv_query(self.delimiter))
+        duckdb.sql(self.queries.export_json_query(out_filename))
+
+    def write_json_newline_delimited(self, out_filename=None):
+        """Query to export a DuckDB table to a JSON newline delimited file.
+
+        Args:
+            out_filename (optional, str): The name of the output file.
+        """
+        duckdb.sql(self.queries.import_csv_query(self.delimiter))
+        duckdb.sql(self.queries.export_json_newline_delimited_query(out_filename))
+
+    def write_parquet(self, out_filename=None):
+        """Query to export a DuckDB table to a Parquet file.
+
+        Args:
+            out_filename (optional, str): The name of the output file.
+        """
+        duckdb.execute(self.queries.import_csv_query(self.delimiter))
+        duckdb.execute(self.queries.export_parquet_query(out_filename))
+
+class CSVWriterPolarsEngine(CSVProperties):
+    """Class to write CSVs to other file formats powered by Polars."""
+
+    def _set_out_filename(self, default_filename, out_filename=None):
+        """Evaluate if a filename is passed in and if not, return default filename."""
+        if out_filename:
+            filename = out_filename
+        else:
+            filename = default_filename
+        return filename
+
+    def write_csv(self, out_filename=None):
+        """Export a Polars dataframe to a CSV file.
+
+        Args:
+            out_filename (optional, str): The name of the output file.
+        """
+        filename = self._set_out_filename(self.CSV_OUT_FILENAME, out_filename)
+        df = CSVReaderPolarsEngine(self.filepath).to_dataframe()
+        df.write_csv(filename)
+
+    def write_excel(self, out_filename=None):
+        """Export a Polars dataframe to an Excel file.
+
+        Args:
+            out_filename (optional, str): The name of the output file.
+        """
+        filename = self._set_out_filename(self.EXCEL_OUT_FILENAME, out_filename)
+        df = CSVReaderPolarsEngine(self.filepath).to_dataframe()
+        df.write_excel(filename)
+
+    def write_json(self, out_filename=None):
+        """Export a Polars dataframe to a JSON file.
+
+        Args:
+            out_filename (optional, str): The name of the output file.
+        """
+        filename = self._set_out_filename(self.JSON_OUT_FILENAME, out_filename)
+        df = CSVReaderPolarsEngine(self.filepath).to_dataframe()
+        df.write_json(filename)
+
+    def write_json_newline_delimited(self, out_filename=None):
+        """Export a Polars dataframe to a JSON newline delimited file.
+
+        Args:
+            out_filename (optional, str): The name of the output file.
+        """
+        filename = self._set_out_filename(self.JSON_NEWLINE_OUT_FILENAME, out_filename)
+        df = CSVReaderPolarsEngine(self.filepath).to_dataframe()
+        df.write_ndjson(filename)
+
+    def write_parquet(self, out_filename=None):
+        """Export a Polars dataframe to a Parquet file.
+
+        Args:
+            out_filename (optional, str): The name of the output file.
+        """
+        filename = self._set_out_filename(self.PARQUET_OUT_FILENAME, out_filename)
+        df = CSVReaderPolarsEngine(self.filepath).to_dataframe()
+        df.write_parquet(filename)

datagrunt/core/fileproperties.py

@@ -0,0 +1,296 @@
+"""Module for deriving and evaluating file properties."""
+
+# standard library
+from collections import Counter
+import csv
+from functools import lru_cache
+import os
+from pathlib import Path
+import re
+
+class FileProperties:
+    """Base class for file objects."""
+
+    FILE_SIZE_DIVISOR = 1024
+    DEFAULT_ENCODING = 'utf-8'
+    EXCEL_FILE_EXTENSIONS = [
+        'xlsx',
+        'xlsm',
+        'xlsb',
+        'xltx',
+        'xltm',
+        'xls',
+        'xlt',
+        'xls'
+    ]
+
+    CSV_FILE_EXTENSIONS = ['csv']
+    TAB_SEPARATED_FILES = ['tsv']
+
+    TABULAR_FILES = list(set(CSV_FILE_EXTENSIONS +
+                             EXCEL_FILE_EXTENSIONS +
+                             TAB_SEPARATED_FILES
+                             )
+                        )
+    TABULAR_FILES.sort()
+
+    APACHE_FILE_EXTENSIONS = ['parquet', 'avro']
+
+    STRUCTURED_FILE_EXTENSIONS = list(set(CSV_FILE_EXTENSIONS +
+                                          EXCEL_FILE_EXTENSIONS +
+                                          TABULAR_FILES +
+                                          TAB_SEPARATED_FILES +
+                                          APACHE_FILE_EXTENSIONS
+                                          )
+                                    )
+    STRUCTURED_FILE_EXTENSIONS.sort()
+
+    SEMI_STRUCTURED_FILE_EXTENSIONS = ['json', 'jsonl']
+
+    STANDARD_FILE_EXTENSIONS = list(set(CSV_FILE_EXTENSIONS +
+                                        TAB_SEPARATED_FILES +
+                                        SEMI_STRUCTURED_FILE_EXTENSIONS +
+                                        APACHE_FILE_EXTENSIONS
+                                        )
+                                    )
+
+    STANDARD_FILE_EXTENSIONS.sort()
+
+    PROPRIETARY_FILE_EXTENSIONS = EXCEL_FILE_EXTENSIONS
+
+    EXCEL_ROW_LIMIT = 1_048_576
+
+    JSON_OUT_FILENAME = 'output.json'
+    JSON_NEWLINE_OUT_FILENAME = 'output.jsonl'
+    CSV_OUT_FILENAME = 'output.csv'
+    EXCEL_OUT_FILENAME = 'output.xlsx'
+    PARQUET_OUT_FILENAME = 'output.parquet'
+    AVRO_OUT_FILENAME = 'output.avro'
+
+    def __init__(self, filepath):
+        """
+        Initialize the FileBase class.
+
+        Args:
+            filepath (str): Path to the file.
+        """
+        self.filepath = filepath
+        self.filename = Path(filepath).name
+        self.extension = Path(filepath).suffix
+        self.extension_string = self.extension.replace('.', '')
+        self.size_in_bytes = os.path.getsize(filepath)
+        self.size_in_kb = round((self.size_in_bytes / self.FILE_SIZE_DIVISOR), 5)
+        self.size_in_mb = round((self.size_in_kb / self.FILE_SIZE_DIVISOR), 5)
+        self.size_in_gb = round((self.size_in_mb / self.FILE_SIZE_DIVISOR), 5)
+        self.size_in_tb = round((self.size_in_gb / self.FILE_SIZE_DIVISOR), 5)
+
+    @property
+    def is_structured(self):
+        """Check if the file is structured."""
+        return self.extension_string.lower() in self.STRUCTURED_FILE_EXTENSIONS
+
+    @property
+    def is_semi_structured(self):
+        """Check if the file is semi-structured."""
+        return self.extension_string.lower() in self.SEMI_STRUCTURED_FILE_EXTENSIONS
+
+    @property
+    def is_unstructured(self):
+        """Check if the file is unstructured."""
+        return self.extension_string.lower() not in self.STRUCTURED_FILE_EXTENSIONS and \
+               self.extension_string.lower() not in self.SEMI_STRUCTURED_FILE_EXTENSIONS
+
+    @property
+    def is_standard(self):
+        """Check if the file is standard."""
+        return self.extension_string.lower() in self.STANDARD_FILE_EXTENSIONS
+
+    @property
+    def is_proprietary(self):
+        """Check if the file is proprietary."""
+        return self.extension_string.lower() in self.PROPRIETARY_FILE_EXTENSIONS
+
+    @property
+    def is_csv(self):
+        """Check if the file is a CSV file."""
+        return self.extension_string.lower() in self.CSV_FILE_EXTENSIONS
+
+    @property
+    def is_excel(self):
+        """Check if the file is an Excel file."""
+        return self.extension_string.lower() in self.EXCEL_FILE_EXTENSIONS
+
+    @property
+    def is_apache(self):
+        """Check if the file is an Apache formatted file."""
+        return self.extension_string.lower() in self.APACHE_FILE_EXTENSIONS
+
+    @property
+    def is_empty(self):
+        """Check if the file is empty."""
+        return self.size_in_bytes == 0
+
+    @property
+    def is_large(self):
+        """Check if the file is greater than or equal to 1 GB."""
+        return self.size_in_gb >= 1.0
+
+    @property
+    def is_tabular(self):
+        """Check if the file is tabular."""
+        return self.extension_string.lower() in self.TABULAR_FILES
+
+class CSVProperties(FileProperties):
+    """Class for parsing CSV files. Mostly determining the delimiter."""
+
+    DELIMITER_REGEX_PATTERN = r'[^0-9a-zA-Z_ "-]'
+    DEFAULT_DELIMITER = ','
+    DEFAULT_SAMPLE_ROWS = 1
+    CSV_SNIFF_SAMPLE_ROWS = 5
+    DATAFRAME_SAMPLE_ROWS = 20
+
+    QUOTING_MAP = {
+        0: 'no quoting',
+        1: 'quote all',
+        2: 'quote minimal',
+        3: 'quote non-numeric'
+    }
+
+    def __init__(self, filepath):
+        """
+        Initialize the CSVParser class.
+
+        Args:
+            filepath (str): Path to the file to read.
+        """
+        super().__init__(filepath)
+        self.first_row = self._get_first_row_from_file()
+        self.delimiter = self._infer_csv_file_delimiter()
+        if not self.is_csv:
+            raise ValueError(
+                f"File extension '{self.extension_string}' is not a valid CSV file extension."
+                             )
+
+    def _get_first_row_from_file(self):
+        """Reads and returns the first line of a file.
+
+        Args:
+            filename: The path to the file.
+
+        Returns:
+            The first line of the file, stripped of leading/trailing whitespace,
+            or None if the file is empty.
+        """
+        with open(self.filepath, 'r', encoding=self.DEFAULT_ENCODING) as csv_file:
+            first_line = csv_file.readline().strip()
+        return first_line
+
+    def _get_most_common_non_alpha_numeric_character_from_string(self):
+        """Get the most common non-alpha-numeric character from a given string.
+
+        Args:
+            text (str): The string to get the most common non-alpha-numeric character from.
+
+        Returns:
+            str: The most common non-alpha-numeric character from the string.
+        """
+        columns_no_spaces = self.first_row.replace(' ', '')
+        regex = re.compile(self.DELIMITER_REGEX_PATTERN)
+        counts = Counter(char for char in regex.findall(columns_no_spaces))
+        most_common = counts.most_common()
+        return most_common
+
+    def _infer_csv_file_delimiter(self):
+        """Infer the delimiter of a CSV file.
+
+        Args:
+            csv_file (str): The path to the CSV file.
+
+        Returns:
+            str: The delimiter of the CSV file.
+        """
+        delimiter_candidates = self._get_most_common_non_alpha_numeric_character_from_string()
+
+        if self.is_empty:
+            delimiter = self.DEFAULT_DELIMITER
+        elif len(delimiter_candidates) == 0:
+            delimiter = ' '
+        else:
+            delimiter = delimiter_candidates[0][0]
+        return delimiter
+
+    def _get_attributes(self):
+        """Generate a dictionary of CSV attributes."""
+        columns_list = self.first_row.split(self.delimiter)
+        columns = {c: 'VARCHAR' for c in columns_list}
+        with open(self.filepath, 'r', encoding=self.DEFAULT_ENCODING) as csvfile:
+            # Sniff the file to detect parameters
+            dialect = csv.Sniffer().sniff(csvfile.read(self.CSV_SNIFF_SAMPLE_ROWS))
+            csvfile.seek(0)  # Reset file pointer to the beginning
+
+            attributes = {
+                'delimiter': self.delimiter,
+                'quotechar': dialect.quotechar,
+                'escapechar': dialect.escapechar,
+                'doublequote': dialect.doublequote,
+                'newline_delimiter': dialect.lineterminator,
+                'skipinitialspace': dialect.skipinitialspace,
+                'quoting': self.QUOTING_MAP.get(dialect.quoting),
+                'row_count_with_header': self.row_count_with_header,
+                'row_count_without_header': self.row_count_without_header,
+                'columns_schema': columns,
+                'columns_original_format': self.first_row,
+                'columns_list': columns_list,
+                'columns_string': ", ".join(columns_list),
+                'columns_byte_string': ", ".join(columns_list).encode(),
+                'column_count': len(columns_list)
+            }
+
+        return attributes
+
+    @property
+    @lru_cache()
+    def row_count_with_header(self):
+        """Return the number of lines in the CSV file including the header."""
+        with open(self.filepath, 'rb') as csv_file:
+            return sum(1 for _ in csv_file)
+
+    @property
+    def row_count_without_header(self):
+        """Return the number of lines in the CSV file excluding the header."""
+        return self.row_count_with_header - 1
+
+    @property
+    def columns(self):
+        """Return the schema of the columns in the CSV file."""
+        return self._get_attributes()['columns_list']
+
+    @property
+    def columns_string(self):
+        """Return the first row of a CSV file as a string."""
+        return self._get_attributes()['columns_string']
+
+    @property
+    def columns_byte_string(self):
+        """Return the first row of the CSV file as bytes."""
+        return self._get_attributes()['columns_byte_string']
+
+    @property
+    def column_count(self):
+        """Return the number of columns in the CSV file."""
+        return self._get_attributes()['column_count']
+
+    @property
+    def quotechar(self):
+        """Return the quote character used in the CSV file."""
+        return self._get_attributes()['quotechar']
+
+    @property
+    def escapechar(self):
+        """Return the escape character used in the CSV file."""
+        return self._get_attributes()['escapechar']
+
+    @property
+    def newline_delimiter(self):
+        """Return the newline delimiter used in the CSV file."""
+        return self._get_attributes()['newline_delimiter']

datagrunt/core/logger.py

@@ -0,0 +1,47 @@
+"""Module for showing logging messages."""
+
+import logging
+
+LARGE_FILE_WARNING = "File is large and may load into memory slowly or exceed memory capacity."
+DUCKDB_ENGINE_ERROR = """DuckDB engine failed due to the following error: {error}. \
+    Switching to Polars."""
+
+
+def show_warning(message):
+    """Show a warning message.
+
+    Args:
+        message (str): The message to show.
+    """
+    logging.basicConfig(level=logging.WARNING, format='%(levelname)s - %(message)s')
+    return logging.warning(message)
+
+
+def show_info_message(message):
+    """Show an info message.
+
+    Args:
+        message (str): The message to show.
+    """
+    logging.basicConfig(level=logging.INFO, format='%(levelname)s - %(message)s')
+    return logging.info(message)
+
+
+def show_large_file_warning():
+    """Show a warning message if the file is large."""
+    show_warning(LARGE_FILE_WARNING)
+
+
+def duckdb_query_error(error_message):
+    """Show error message if duckdb query fails."""
+    message = DUCKDB_ENGINE_ERROR.format(error=error_message)
+    return show_warning(message)
+
+
+def show_dataframe_sample(dataframe):
+    """Show dataframe output.
+
+    Args:
+        dataframe (dataframe): The dataframe to show.
+    """
+    return show_info_message(dataframe)

datagrunt/core/queries.py

@@ -0,0 +1,103 @@
+"""Module to store database queries and query strings."""
+
+# standard library
+
+# third party libraries
+
+# local libraries
+from .databases import DuckDBDatabase
+from .fileproperties import FileProperties
+
+class DuckDBQueries(DuckDBDatabase):
+    """Class to store DuckDB database queries and query strings."""
+
+    def __init__(self, filepath):
+        """
+        Initialize the DuckDBQueries class.
+
+        Args:
+            filepath (str): Path to the file.
+        """
+        super().__init__(filepath)
+        self.export_properties = FileProperties(self.filepath)
+
+    def _set_out_filename(self, default_filename, out_filename=None):
+        """Evaluate if a filename is passed in and if not, return default filename."""
+        if out_filename:
+            filename = out_filename
+        else:
+            filename = default_filename
+        return filename
+
+    def import_csv_query(self, delimiter):
+        """Query to import a CSV file into a DuckDB table.
+
+        Args:
+            filepath str: Path to the file.
+            delimiter str: The delimiter to use.
+        """
+        return f"""
+            CREATE OR REPLACE TABLE {self.database_table_name} AS
+            SELECT *
+            FROM read_csv('{self.filepath}',
+                            auto_detect=true,
+                            delim='{delimiter}',
+                            header=true,
+                            null_padding=true,
+                            all_varchar=True);
+            """
+
+    def select_from_duckdb_table(self):
+        """Query to select from a DuckDB table."""
+        return f"SELECT * FROM {self.database_table_name}"
+
+    def export_csv_query(self, out_filename=None):
+        """Query to export a DuckDB table to a CSV file.
+
+        Args:
+            out_filename (str, optional): The name of the output file.
+        """
+        filename = self._set_out_filename(self.export_properties.CSV_OUT_FILENAME, out_filename)
+        return f"COPY {self.database_table_name} TO '{filename}' (HEADER, DELIMITER ',');"
+
+    def export_excel_query(self, out_filename=None):
+        """Query to export a DuckDB table to an Excel file.
+
+        Args:
+            out_filename (str, optional): The name of the output file.
+        """
+        filename = self._set_out_filename(self.export_properties.EXCEL_OUT_FILENAME, out_filename)
+        return f"""
+            INSTALL spatial;
+            LOAD spatial;
+            COPY (SELECT * FROM {self.database_table_name})
+            TO '{filename}'(FORMAT GDAL, DRIVER 'xlsx')
+        """
+
+    def export_json_query(self, out_filename=None):
+        """Query to export a DuckDB table to a JSON file.
+
+        Args:
+            out_filename (str, optional): The name of the output file.
+        """
+        filename = self._set_out_filename(self.export_properties.JSON_OUT_FILENAME, out_filename)
+        return f"COPY (SELECT * FROM {self.database_table_name}) TO '{filename}' (ARRAY true) "
+
+    def export_json_newline_delimited_query(self, out_filename=None):
+        """Query to export a DuckDB table to a JSON file with newline delimited.
+
+        Args:
+            out_filename (str, optional): The name of the output file.
+        """
+        filename = self._set_out_filename(self.export_properties.JSON_NEWLINE_OUT_FILENAME,
+                                          out_filename)
+        return f"COPY (SELECT * FROM {self.database_table_name}) TO '{filename}'"
+
+    def export_parquet_query(self, out_filename=None):
+        """Query to export a DuckDB table to a Parquet file.
+
+        Args:
+            out_filename (str, optional): The name of the output file.
+        """
+        filename = self._set_out_filename(self.export_properties.PARQUET_OUT_FILENAME, out_filename)
+        return f"COPY (SELECT * FROM {self.database_table_name}) TO '{filename}'(FORMAT PARQUET)"

datagrunt/csvfile.py

@@ -0,0 +1,156 @@
+"""Module for reading CSV files and converting CSV files to different standard file formats."""
+
+# standard library
+
+# third party libraries
+import duckdb
+
+# local libraries
+from .core.fileproperties import CSVProperties
+from .core.engines import CSVReaderDuckDBEngine, CSVReaderPolarsEngine
+from .core.engines import CSVWriterDuckDBEngine, CSVWriterPolarsEngine
+from .core.queries import DuckDBQueries
+
+class CSVReader(CSVProperties):
+    """Class to unify the interface for reading CSV files."""
+
+    READER_ENGINES = ['duckdb', 'polars']
+    VALUE_ERROR_MESSAGE = """Reader engine '{engine}' is not 'duckdb' or 'polars'. Pass either 'duckdb' or 'polars' as valid engine params."""
+
+    def __init__(self, filepath, engine='polars'):
+        """Initialize the CSV Reader class.
+
+        Args:
+            filepath (str): Path to the file to read.
+            engine (str, default 'polars'): Determines which reader engine class to instantiate.
+        """
+        super().__init__(filepath)
+        self.db_table = DuckDBQueries(self.filepath).database_table_name
+        self.engine = engine.lower().replace(' ', '')
+        if self.engine not in self.READER_ENGINES:
+            raise ValueError(self.VALUE_ERROR_MESSAGE.format(engine=self.engine))
+
+    def _set_reader_engine(self):
+        """Sets the CSV reader engine as either DuckDB or Polars.
+           Default engine is Polars.
+        """
+        if self.engine != 'polars':
+            engine = CSVReaderDuckDBEngine(self.filepath)
+        else:
+            engine = CSVReaderPolarsEngine(self.filepath)
+        return engine
+
+    def get_sample(self):
+        """Return a sample of the CSV file."""
+        self._set_reader_engine().get_sample()
+
+    def to_dataframe(self):
+        """Converts CSV to a Polars dataframe.
+
+        Returns:
+            A Polars dataframe.
+        """
+        return self._set_reader_engine().to_dataframe()
+
+    def to_arrow_table(self):
+        """Converts CSV to a Polars dataframe.
+
+        Returns:
+            A PyArrow table.
+        """
+        return self._set_reader_engine().to_arrow_table()
+
+    def to_dicts(self):
+        """Converts CSV to a Polars dataframe.
+
+        Returns:
+            A list of dictionaries.
+        """
+        return self._set_reader_engine().to_dicts()
+
+    def query_data(self, sql_query):
+        """Queries as CSV file after importing into DuckDB.
+
+        Args:
+            sql_query (str): Query to run against DuckDB.
+
+        Returns:
+            A DuckDB DuckDBPyRelation with the query results.
+
+        Example if DuckDB Engine:
+            dg = CSVReader('myfile.csv')
+            query = "SELECT col1, col2 FROM {dg.db_table}" # f string assumed
+            dg.query_csv_data(query)
+        """
+        queries = DuckDBQueries(self.filepath)
+        duckdb.sql(queries.import_csv_query(self.delimiter))
+        return duckdb.sql(sql_query)
+
+class CSVWriter(CSVProperties):
+    """Class to unify the interface for converting CSV files to various other supported file types."""
+
+    WRITER_ENGINES = ['duckdb', 'polars']
+    VALUE_ERROR_MESSAGE = """Writer engine '{engine}' is not 'duckdb' or 'polars'. Pass either 'duckdb' or 'polars' as valid engine params."""
+
+    def __init__(self, filepath, engine='duckdb'):
+        """Initialize the CSV Writer class.
+
+        Args:
+            filepath (str): Path to the file to write.
+            engine (str, default 'duckdb'): Determines which writer engine class to instantiate.
+        """
+        super().__init__(filepath)
+        self.db_table = DuckDBQueries(self.filepath).database_table_name
+        self.engine = engine.lower().replace(' ', '')
+        if self.engine not in self.WRITER_ENGINES:
+            raise ValueError(self.VALUE_ERROR_MESSAGE.format(engine=self.engine))
+
+    def _set_writer_engine(self):
+        """Sets the CSV reader engine as either DuckDB or Polars.
+           Default engine is Polars.
+        """
+        if self.engine != 'polars':
+            engine = CSVWriterDuckDBEngine(self.filepath)
+        else:
+            engine = CSVWriterPolarsEngine(self.filepath)
+        return engine
+
+    def write_csv(self, out_filename=None):
+        """Query to export a DuckDB table to a CSV file.
+
+            Args:
+                out_filename str: The name of the output file.
+            """
+        return self._set_writer_engine().write_csv(out_filename)
+
+    def write_excel(self, out_filename=None):
+        """Query to export a DuckDB table to an Excel file.
+
+        Args:
+            out_filename str: The name of the output file.
+        """
+        return self._set_writer_engine().write_excel(out_filename)
+
+    def write_json(self, out_filename=None):
+        """Query to export a DuckDB table to a JSON file.
+
+        Args:
+            out_filename str: The name of the output file.
+        """
+        return self._set_writer_engine().write_json(out_filename)
+
+    def write_json_newline_delimited(self, out_filename=None):
+        """Query to export a DuckDB table to a JSON newline delimited file.
+
+        Args:
+            out_filename str: The name of the output file.
+        """
+        return self._set_writer_engine().write_json_newline_delimited(out_filename)
+
+    def write_parquet(self, out_filename=None):
+        """Query to export a DuckDB table to a Parquet file.
+
+        Args:
+            out_filename str: The name of the output file.
+        """
+        return self._set_writer_engine().write_parquet(out_filename)

datagrunt-0.0.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Martin Graham
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

datagrunt-0.0.0.dist-info/METADATA

@@ -0,0 +1,155 @@
+Metadata-Version: 2.1
+Name: datagrunt
+Version: 0.0.0
+Summary: Read CSV files and convert to other file formats easily
+Author-email: Martin Graham <datagrunt@datagrunt.io>
+License: MIT License
+Project-URL: Homepage, https://pmgraham.github.io/datagrunt-docs
+Project-URL: Bug Tracker, https://github.com/pmgraham/datagrunt/issues
+Project-URL: Documentation, https://pmgraham.github.io/datagrunt-docs
+Project-URL: Source Code, https://github.com/pmgraham/datagrunt
+Keywords: csv,data,duckdb,polars,pyarrow,xlsx,delimiter
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: duckdb>=1.1.0
+Requires-Dist: polars>=1.7.1
+Requires-Dist: pyarrow>=17.0.0
+Requires-Dist: XlsxWriter>=3.2.0
+Provides-Extra: build
+Requires-Dist: build; extra == "build"
+Requires-Dist: twine; extra == "build"
+Requires-Dist: bumpver; extra == "build"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=3.0; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+
+# Welcome To Datagrunt
+
+Datagrunt is a Python library designed to simplify the way you work with CSV files. It provides a streamlined approach to reading, processing, and transforming your data into various formats, making data manipulation efficient and intuitive.
+
+## Why Datagrunt?
+
+Born out of real-world frustration, Datagrunt eliminates the need For repetitive coding when handling CSV files. Whether you're a data analyst, data engineer, or data scientist, Datagrunt empowers you to focus on insights, not tedious data wrangling.
+
+## Key Features
+
+- **Intelligent Delimiter Inference:**  Datagrunt automatically detects and applies the correct delimiter for your csv files.
+- **Seamless Data Processing:** Leverage the robust capabilities of [DuckDB](https://duckdb.org) and [Polars](https://pola.rs) to perform advanced data processing tasks directly on your CSV data.
+- **Flexible Transformation:** Easily convert your processed CSV data into various formats to suit your needs.
+- **Pythonic API:** Enjoy a clean and intuitive API that integrates seamlessly into your existing Python workflows.
+
+## Installation
+
+Get started with Datagrunt in seconds using pip:
+
+```bash
+pip install datagrunt
+```
+
+## Getting Started
+
+```python
+from datagrunt import CSVReader
+
+# Load your CSV file
+csv_file = 'electric_vehicle_population_data.csv'
+engine = 'duckdb'
+
+# Set duckdb as the processing engine. Engine set to 'polars' by default
+dg = CSVReader(csv_file, engine=engine)
+
+# return sample of the data to get a peek at the schema
+dg.get_sample()
+┌────────────┬───────────┬──────────────┬───┬──────────────────────┬──────────────────────┬───────────────────┐
+│ VIN (1-10) │  County   │     City     │ … │   Vehicle Location   │   Electric Utility   │ 2020 Census Tract │
+│  varchar   │  varchar  │   varchar    │   │       varchar        │       varchar        │      varchar      │
+├────────────┼───────────┼──────────────┼───┼──────────────────────┼──────────────────────┼───────────────────┤
+│ 5YJSA1E28K │ Snohomish │ Mukilteo     │ … │ POINT (-122.29943 …  │ PUGET SOUND ENERGY…  │ 53061042001       │
+│ 1C4JJXP68P │ Yakima    │ Yakima       │ … │ POINT (-120.468875…  │ PACIFICORP           │ 53077001601       │
+│ WBY8P6C05L │ Kitsap    │ Kingston     │ … │ POINT (-122.517835…  │ PUGET SOUND ENERGY…  │ 53035090102       │
+│ JTDKARFP1J │ Kitsap    │ Port Orchard │ … │ POINT (-122.653005…  │ PUGET SOUND ENERGY…  │ 53035092802       │
+│ 5UXTA6C09N │ Snohomish │ Everett      │ … │ POINT (-122.203234…  │ PUGET SOUND ENERGY…  │ 53061041605       │
+│ 5YJYGDEF8L │ King      │ Seattle      │ … │ POINT (-122.378886…  │ CITY OF SEATTLE - …  │ 53033004703       │
+│ JTMAB3FV7P │ Thurston  │ Rainier      │ … │ POINT (-122.677141…  │ PUGET SOUND ENERGY…  │ 53067012530       │
+│ JN1AZ0CPXC │ King      │ Kirkland     │ … │ POINT (-122.192596…  │ PUGET SOUND ENERGY…  │ 53033022402       │
+│ JN1AZ0CP7B │ King      │ Kirkland     │ … │ POINT (-122.192596…  │ PUGET SOUND ENERGY…  │ 53033022603       │
+│ 1N4AZ0CP0F │ Thurston  │ Olympia      │ … │ POINT (-122.86491 …  │ PUGET SOUND ENERGY…  │ 53067010300       │
+│     ·      │   ·       │    ·         │ · │          ·           │          ·           │      ·            │
+│     ·      │   ·       │    ·         │ · │          ·           │          ·           │      ·            │
+│     ·      │   ·       │    ·         │ · │          ·           │          ·           │      ·            │
+│ 5YJYGDEE7M │ Clark     │ Vancouver    │ … │ POINT (-122.515805…  │ BONNEVILLE POWER A…  │ 53011041310       │
+│ 7SAYGAEE0P │ Snohomish │ Monroe       │ … │ POINT (-121.968385…  │ PUGET SOUND ENERGY…  │ 53061052203       │
+│ 2C4RC1N75P │ King      │ Burien       │ … │ POINT (-122.347227…  │ CITY OF SEATTLE - …  │ 53033027600       │
+│ 1FTVW1EVXP │ King      │ Kirkland     │ … │ POINT (-122.202653…  │ PUGET SOUND ENERGY…  │ 53033022300       │
+│ 4JGGM1CB2P │ King      │ Seattle      │ … │ POINT (-122.2453 4…  │ CITY OF SEATTLE - …  │ 53033011700       │
+│ 1N4BZ0CP0G │ King      │ Seattle      │ … │ POINT (-122.334079…  │ CITY OF SEATTLE - …  │ 53033008300       │
+│ 7SAYGDEF2N │ King      │ Bellevue     │ … │ POINT (-122.144149…  │ PUGET SOUND ENERGY…  │ 53033024704       │
+│ 1N4BZ1DP7L │ King      │ Bellevue     │ … │ POINT (-122.144149…  │ PUGET SOUND ENERGY…  │ 53033024902       │
+...
+├────────────┴───────────┴──────────────┴───┴──────────────────────┴──────────────────────┴───────────────────┤
+│ ? rows (>9999 rows, 20 shown)                                                          17 columns (6 shown) │
+└─────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
+```
+
+##  DuckDB Integration for Performant SQL Queries
+```python
+from datagrunt import CSVReader
+
+csv_file = 'electric_vehicle_population_data.csv'
+engine = 'duckdb'
+
+dg = CSVReader(csv_file, engine=engine)
+
+# Construct your SQL query
+query = f"""
+WITH core AS (
+    SELECT
+        City AS city,
+        "VIN (1-10)" AS vin
+    FROM {dg.db_table}
+)
+SELECT
+    city,
+    COUNT(vin) AS vehicle_count
+FROM core
+GROUP BY 1
+ORDER BY 2 DESC
+"""
+
+# Execute the query and get results as a Polars DataFrame
+df = dg.query_data(query).pl()
+print(df)
+┌────────────────┬───────────────┐
+│ city           ┆ vehicle_count │
+│ ---            ┆ ---           │
+│ str            ┆ i64           │
+╞════════════════╪═══════════════╡
+│ Seattle        ┆ 32602         │
+│ Bellevue       ┆ 9960          │
+│ Redmond        ┆ 7165          │
+│ Vancouver      ┆ 7081          │
+│ Bothell        ┆ 6602          │
+│ …              ┆ …             │
+│ Glenwood       ┆ 1             │
+│ Walla Walla Co ┆ 1             │
+│ Pittsburg      ┆ 1             │
+│ Decatur        ┆ 1             │
+│ Redwood City   ┆ 1             │
+└────────────────┴───────────────┘
+```
+## License
+This project is licensed under the [MIT License](https://opensource.org/license/mit)
+
+## Acknowledgements
+A HUGE thank you to the open source community and the creators of [DuckDB](https://duckdb.org) and [Polars](https://pola.rs) for their fantastic libraries that power Datagrunt.

datagrunt-0.0.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+datagrunt/__init__.py,sha256=rWgU2R8hvXqr9hw_DH_7j74PI2gnXlHCybQSpwDiZfg,1122
+datagrunt/csvfile.py,sha256=jbIvA9_0ucRJ4qCu6yQ1InSg0ENxN2dkCZcX8rVN4iI,5476
+datagrunt/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datagrunt/core/databases.py,sha256=yo8ILK9dtj8zusrM8HgoA5s629b-ghGRp2HumvJS4eE,1702
+datagrunt/core/engines.py,sha256=K__2kkdKwvxeL6QwmyEyZdeuUT5MDPpY0lzKwYHrouM,7543
+datagrunt/core/fileproperties.py,sha256=e5mP7bAsQzKpppryZqEV425AXNXsMGZnjrPhYz8gaJI,10198
+datagrunt/core/logger.py,sha256=ck5HmwHmk0SUmTjQudqLgpQk5mpC9T0UOlDKqi6qssI,1250
+datagrunt/core/queries.py,sha256=2OPD2Zz1Toryr5_nVANkJvifwg6MzCx4obah34m-IBY,3846
+datagrunt-0.0.0.dist-info/LICENSE,sha256=qY7AmIOAwE05aVD7siEo8-2HOUz5WYUEhxbX0Ts9fyg,1069
+datagrunt-0.0.0.dist-info/METADATA,sha256=K64t3rDDe8IiHUrJrcrweVYPJSa3COMhC0jWvZ01TpQ,9250
+datagrunt-0.0.0.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+datagrunt-0.0.0.dist-info/top_level.txt,sha256=kVGx0P9sIaw_RegsAR94f7fyE4Um4ayE5vf_aTImMoE,10
+datagrunt-0.0.0.dist-info/RECORD,,

datagrunt-0.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (74.1.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

datagrunt-0.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+datagrunt