datagrunt 0.0.0__tar.gz

datagrunt-0.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Martin Graham
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

datagrunt-0.0.0/PKG-INFO

@@ -0,0 +1,155 @@
+Metadata-Version: 2.1
+Name: datagrunt
+Version: 0.0.0
+Summary: Read CSV files and convert to other file formats easily
+Author-email: Martin Graham <datagrunt@datagrunt.io>
+License: MIT License
+Project-URL: Homepage, https://pmgraham.github.io/datagrunt-docs
+Project-URL: Bug Tracker, https://github.com/pmgraham/datagrunt/issues
+Project-URL: Documentation, https://pmgraham.github.io/datagrunt-docs
+Project-URL: Source Code, https://github.com/pmgraham/datagrunt
+Keywords: csv,data,duckdb,polars,pyarrow,xlsx,delimiter
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: duckdb>=1.1.0
+Requires-Dist: polars>=1.7.1
+Requires-Dist: pyarrow>=17.0.0
+Requires-Dist: XlsxWriter>=3.2.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=3.0; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Provides-Extra: build
+Requires-Dist: build; extra == "build"
+Requires-Dist: twine; extra == "build"
+Requires-Dist: bumpver; extra == "build"
+
+# Welcome To Datagrunt
+
+Datagrunt is a Python library designed to simplify the way you work with CSV files. It provides a streamlined approach to reading, processing, and transforming your data into various formats, making data manipulation efficient and intuitive.
+
+## Why Datagrunt?
+
+Born out of real-world frustration, Datagrunt eliminates the need For repetitive coding when handling CSV files. Whether you're a data analyst, data engineer, or data scientist, Datagrunt empowers you to focus on insights, not tedious data wrangling.
+
+## Key Features
+
+- **Intelligent Delimiter Inference:**  Datagrunt automatically detects and applies the correct delimiter for your csv files.
+- **Seamless Data Processing:** Leverage the robust capabilities of [DuckDB](https://duckdb.org) and [Polars](https://pola.rs) to perform advanced data processing tasks directly on your CSV data.
+- **Flexible Transformation:** Easily convert your processed CSV data into various formats to suit your needs.
+- **Pythonic API:** Enjoy a clean and intuitive API that integrates seamlessly into your existing Python workflows.
+
+## Installation
+
+Get started with Datagrunt in seconds using pip:
+
+```bash
+pip install datagrunt
+```
+
+## Getting Started
+
+```python
+from datagrunt import CSVReader
+
+# Load your CSV file
+csv_file = 'electric_vehicle_population_data.csv'
+engine = 'duckdb'
+
+# Set duckdb as the processing engine. Engine set to 'polars' by default
+dg = CSVReader(csv_file, engine=engine)
+
+# return sample of the data to get a peek at the schema
+dg.get_sample()
+┌────────────┬───────────┬──────────────┬───┬──────────────────────┬──────────────────────┬───────────────────┐
+│ VIN (1-10) │  County   │     City     │ … │   Vehicle Location   │   Electric Utility   │ 2020 Census Tract │
+│  varchar   │  varchar  │   varchar    │   │       varchar        │       varchar        │      varchar      │
+├────────────┼───────────┼──────────────┼───┼──────────────────────┼──────────────────────┼───────────────────┤
+│ 5YJSA1E28K │ Snohomish │ Mukilteo     │ … │ POINT (-122.29943 …  │ PUGET SOUND ENERGY…  │ 53061042001       │
+│ 1C4JJXP68P │ Yakima    │ Yakima       │ … │ POINT (-120.468875…  │ PACIFICORP           │ 53077001601       │
+│ WBY8P6C05L │ Kitsap    │ Kingston     │ … │ POINT (-122.517835…  │ PUGET SOUND ENERGY…  │ 53035090102       │
+│ JTDKARFP1J │ Kitsap    │ Port Orchard │ … │ POINT (-122.653005…  │ PUGET SOUND ENERGY…  │ 53035092802       │
+│ 5UXTA6C09N │ Snohomish │ Everett      │ … │ POINT (-122.203234…  │ PUGET SOUND ENERGY…  │ 53061041605       │
+│ 5YJYGDEF8L │ King      │ Seattle      │ … │ POINT (-122.378886…  │ CITY OF SEATTLE - …  │ 53033004703       │
+│ JTMAB3FV7P │ Thurston  │ Rainier      │ … │ POINT (-122.677141…  │ PUGET SOUND ENERGY…  │ 53067012530       │
+│ JN1AZ0CPXC │ King      │ Kirkland     │ … │ POINT (-122.192596…  │ PUGET SOUND ENERGY…  │ 53033022402       │
+│ JN1AZ0CP7B │ King      │ Kirkland     │ … │ POINT (-122.192596…  │ PUGET SOUND ENERGY…  │ 53033022603       │
+│ 1N4AZ0CP0F │ Thurston  │ Olympia      │ … │ POINT (-122.86491 …  │ PUGET SOUND ENERGY…  │ 53067010300       │
+│     ·      │   ·       │    ·         │ · │          ·           │          ·           │      ·            │
+│     ·      │   ·       │    ·         │ · │          ·           │          ·           │      ·            │
+│     ·      │   ·       │    ·         │ · │          ·           │          ·           │      ·            │
+│ 5YJYGDEE7M │ Clark     │ Vancouver    │ … │ POINT (-122.515805…  │ BONNEVILLE POWER A…  │ 53011041310       │
+│ 7SAYGAEE0P │ Snohomish │ Monroe       │ … │ POINT (-121.968385…  │ PUGET SOUND ENERGY…  │ 53061052203       │
+│ 2C4RC1N75P │ King      │ Burien       │ … │ POINT (-122.347227…  │ CITY OF SEATTLE - …  │ 53033027600       │
+│ 1FTVW1EVXP │ King      │ Kirkland     │ … │ POINT (-122.202653…  │ PUGET SOUND ENERGY…  │ 53033022300       │
+│ 4JGGM1CB2P │ King      │ Seattle      │ … │ POINT (-122.2453 4…  │ CITY OF SEATTLE - …  │ 53033011700       │
+│ 1N4BZ0CP0G │ King      │ Seattle      │ … │ POINT (-122.334079…  │ CITY OF SEATTLE - …  │ 53033008300       │
+│ 7SAYGDEF2N │ King      │ Bellevue     │ … │ POINT (-122.144149…  │ PUGET SOUND ENERGY…  │ 53033024704       │
+│ 1N4BZ1DP7L │ King      │ Bellevue     │ … │ POINT (-122.144149…  │ PUGET SOUND ENERGY…  │ 53033024902       │
+...
+├────────────┴───────────┴──────────────┴───┴──────────────────────┴──────────────────────┴───────────────────┤
+│ ? rows (>9999 rows, 20 shown)                                                          17 columns (6 shown) │
+└─────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
+```
+
+##  DuckDB Integration for Performant SQL Queries
+```python
+from datagrunt import CSVReader
+
+csv_file = 'electric_vehicle_population_data.csv'
+engine = 'duckdb'
+
+dg = CSVReader(csv_file, engine=engine)
+
+# Construct your SQL query
+query = f"""
+WITH core AS (
+    SELECT
+        City AS city,
+        "VIN (1-10)" AS vin
+    FROM {dg.db_table}
+)
+SELECT
+    city,
+    COUNT(vin) AS vehicle_count
+FROM core
+GROUP BY 1
+ORDER BY 2 DESC
+"""
+
+# Execute the query and get results as a Polars DataFrame
+df = dg.query_data(query).pl()
+print(df)
+┌────────────────┬───────────────┐
+│ city           ┆ vehicle_count │
+│ ---            ┆ ---           │
+│ str            ┆ i64           │
+╞════════════════╪═══════════════╡
+│ Seattle        ┆ 32602         │
+│ Bellevue       ┆ 9960          │
+│ Redmond        ┆ 7165          │
+│ Vancouver      ┆ 7081          │
+│ Bothell        ┆ 6602          │
+│ …              ┆ …             │
+│ Glenwood       ┆ 1             │
+│ Walla Walla Co ┆ 1             │
+│ Pittsburg      ┆ 1             │
+│ Decatur        ┆ 1             │
+│ Redwood City   ┆ 1             │
+└────────────────┴───────────────┘
+```
+## License
+This project is licensed under the [MIT License](https://opensource.org/license/mit)
+
+## Acknowledgements
+A HUGE thank you to the open source community and the creators of [DuckDB](https://duckdb.org) and [Polars](https://pola.rs) for their fantastic libraries that power Datagrunt.

datagrunt-0.0.0/README.md

@@ -0,0 +1,119 @@
+# Welcome To Datagrunt
+
+Datagrunt is a Python library designed to simplify the way you work with CSV files. It provides a streamlined approach to reading, processing, and transforming your data into various formats, making data manipulation efficient and intuitive.
+
+## Why Datagrunt?
+
+Born out of real-world frustration, Datagrunt eliminates the need For repetitive coding when handling CSV files. Whether you're a data analyst, data engineer, or data scientist, Datagrunt empowers you to focus on insights, not tedious data wrangling.
+
+## Key Features
+
+- **Intelligent Delimiter Inference:**  Datagrunt automatically detects and applies the correct delimiter for your csv files.
+- **Seamless Data Processing:** Leverage the robust capabilities of [DuckDB](https://duckdb.org) and [Polars](https://pola.rs) to perform advanced data processing tasks directly on your CSV data.
+- **Flexible Transformation:** Easily convert your processed CSV data into various formats to suit your needs.
+- **Pythonic API:** Enjoy a clean and intuitive API that integrates seamlessly into your existing Python workflows.
+
+## Installation
+
+Get started with Datagrunt in seconds using pip:
+
+```bash
+pip install datagrunt
+```
+
+## Getting Started
+
+```python
+from datagrunt import CSVReader
+
+# Load your CSV file
+csv_file = 'electric_vehicle_population_data.csv'
+engine = 'duckdb'
+
+# Set duckdb as the processing engine. Engine set to 'polars' by default
+dg = CSVReader(csv_file, engine=engine)
+
+# return sample of the data to get a peek at the schema
+dg.get_sample()
+┌────────────┬───────────┬──────────────┬───┬──────────────────────┬──────────────────────┬───────────────────┐
+│ VIN (1-10) │  County   │     City     │ … │   Vehicle Location   │   Electric Utility   │ 2020 Census Tract │
+│  varchar   │  varchar  │   varchar    │   │       varchar        │       varchar        │      varchar      │
+├────────────┼───────────┼──────────────┼───┼──────────────────────┼──────────────────────┼───────────────────┤
+│ 5YJSA1E28K │ Snohomish │ Mukilteo     │ … │ POINT (-122.29943 …  │ PUGET SOUND ENERGY…  │ 53061042001       │
+│ 1C4JJXP68P │ Yakima    │ Yakima       │ … │ POINT (-120.468875…  │ PACIFICORP           │ 53077001601       │
+│ WBY8P6C05L │ Kitsap    │ Kingston     │ … │ POINT (-122.517835…  │ PUGET SOUND ENERGY…  │ 53035090102       │
+│ JTDKARFP1J │ Kitsap    │ Port Orchard │ … │ POINT (-122.653005…  │ PUGET SOUND ENERGY…  │ 53035092802       │
+│ 5UXTA6C09N │ Snohomish │ Everett      │ … │ POINT (-122.203234…  │ PUGET SOUND ENERGY…  │ 53061041605       │
+│ 5YJYGDEF8L │ King      │ Seattle      │ … │ POINT (-122.378886…  │ CITY OF SEATTLE - …  │ 53033004703       │
+│ JTMAB3FV7P │ Thurston  │ Rainier      │ … │ POINT (-122.677141…  │ PUGET SOUND ENERGY…  │ 53067012530       │
+│ JN1AZ0CPXC │ King      │ Kirkland     │ … │ POINT (-122.192596…  │ PUGET SOUND ENERGY…  │ 53033022402       │
+│ JN1AZ0CP7B │ King      │ Kirkland     │ … │ POINT (-122.192596…  │ PUGET SOUND ENERGY…  │ 53033022603       │
+│ 1N4AZ0CP0F │ Thurston  │ Olympia      │ … │ POINT (-122.86491 …  │ PUGET SOUND ENERGY…  │ 53067010300       │
+│     ·      │   ·       │    ·         │ · │          ·           │          ·           │      ·            │
+│     ·      │   ·       │    ·         │ · │          ·           │          ·           │      ·            │
+│     ·      │   ·       │    ·         │ · │          ·           │          ·           │      ·            │
+│ 5YJYGDEE7M │ Clark     │ Vancouver    │ … │ POINT (-122.515805…  │ BONNEVILLE POWER A…  │ 53011041310       │
+│ 7SAYGAEE0P │ Snohomish │ Monroe       │ … │ POINT (-121.968385…  │ PUGET SOUND ENERGY…  │ 53061052203       │
+│ 2C4RC1N75P │ King      │ Burien       │ … │ POINT (-122.347227…  │ CITY OF SEATTLE - …  │ 53033027600       │
+│ 1FTVW1EVXP │ King      │ Kirkland     │ … │ POINT (-122.202653…  │ PUGET SOUND ENERGY…  │ 53033022300       │
+│ 4JGGM1CB2P │ King      │ Seattle      │ … │ POINT (-122.2453 4…  │ CITY OF SEATTLE - …  │ 53033011700       │
+│ 1N4BZ0CP0G │ King      │ Seattle      │ … │ POINT (-122.334079…  │ CITY OF SEATTLE - …  │ 53033008300       │
+│ 7SAYGDEF2N │ King      │ Bellevue     │ … │ POINT (-122.144149…  │ PUGET SOUND ENERGY…  │ 53033024704       │
+│ 1N4BZ1DP7L │ King      │ Bellevue     │ … │ POINT (-122.144149…  │ PUGET SOUND ENERGY…  │ 53033024902       │
+...
+├────────────┴───────────┴──────────────┴───┴──────────────────────┴──────────────────────┴───────────────────┤
+│ ? rows (>9999 rows, 20 shown)                                                          17 columns (6 shown) │
+└─────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
+```
+
+##  DuckDB Integration for Performant SQL Queries
+```python
+from datagrunt import CSVReader
+
+csv_file = 'electric_vehicle_population_data.csv'
+engine = 'duckdb'
+
+dg = CSVReader(csv_file, engine=engine)
+
+# Construct your SQL query
+query = f"""
+WITH core AS (
+    SELECT
+        City AS city,
+        "VIN (1-10)" AS vin
+    FROM {dg.db_table}
+)
+SELECT
+    city,
+    COUNT(vin) AS vehicle_count
+FROM core
+GROUP BY 1
+ORDER BY 2 DESC
+"""
+
+# Execute the query and get results as a Polars DataFrame
+df = dg.query_data(query).pl()
+print(df)
+┌────────────────┬───────────────┐
+│ city           ┆ vehicle_count │
+│ ---            ┆ ---           │
+│ str            ┆ i64           │
+╞════════════════╪═══════════════╡
+│ Seattle        ┆ 32602         │
+│ Bellevue       ┆ 9960          │
+│ Redmond        ┆ 7165          │
+│ Vancouver      ┆ 7081          │
+│ Bothell        ┆ 6602          │
+│ …              ┆ …             │
+│ Glenwood       ┆ 1             │
+│ Walla Walla Co ┆ 1             │
+│ Pittsburg      ┆ 1             │
+│ Decatur        ┆ 1             │
+│ Redwood City   ┆ 1             │
+└────────────────┴───────────────┘
+```
+## License
+This project is licensed under the [MIT License](https://opensource.org/license/mit)
+
+## Acknowledgements
+A HUGE thank you to the open source community and the creators of [DuckDB](https://duckdb.org) and [Polars](https://pola.rs) for their fantastic libraries that power Datagrunt.

datagrunt-0.0.0/pyproject.toml

@@ -0,0 +1,71 @@
+[build-system]
+requires = ["setuptools>=74.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "datagrunt"
+version = "0.0.0"
+description = "Read CSV files and convert to other file formats easily"
+readme = "README.md"
+authors = [{ name = "Martin Graham", email = "datagrunt@datagrunt.io" }]
+license = {text = "MIT License"}
+classifiers = [
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Intended Audience :: Developers",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+keywords = ["csv", "data", "duckdb", "polars", "pyarrow", "xlsx", "delimiter"]
+dependencies = [
+    "duckdb>=1.1.0",
+    "polars>=1.7.1",
+    "pyarrow>=17.0.0",
+    "XlsxWriter>=3.2.0"
+]
+requires-python = ">=3.10"
+
+[project.optional-dependencies]
+dev = ["pytest>=7.0", "pytest-cov>=3.0", "black", "isort", "flake8"]
+build = ["build", "twine", "bumpver"]
+
+[project.urls]
+Homepage = "https://pmgraham.github.io/datagrunt-docs"
+"Bug Tracker" = "https://github.com/pmgraham/datagrunt/issues"
+Documentation = "https://pmgraham.github.io/datagrunt-docs"
+"Source Code" = "https://github.com/pmgraham/datagrunt"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["datagrunt*"]
+exclude = ["tests*"]
+
+[tool.bumpver]
+current_version = "0.0.0"
+version_pattern = "MAJOR.MINOR.PATCH"
+commit_message = "bump version {old_version} -> {new_version}"
+commit = true
+tag = true
+push = false
+
+[tool.bumpver.file_patterns]
+"pyproject.toml" = [
+    'current_version = "{version}"',
+    'version = "{version}"',
+]
+"src/datagrunt/__init__.py" = [
+    '^__version__ = "{version}"$',
+]
+
+[tool.black]
+line-length = 88
+target-version = ['py310']
+
+[tool.isort]
+profile = "black"
+
+[tool.flake8]
+max-line-length = 88
+extend-ignore = "E203"

datagrunt-0.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

datagrunt-0.0.0/src/datagrunt/__init__.py

@@ -0,0 +1,39 @@
+"""
+Datagrunt
+
+A Python library designed to simplify the way you work with CSV files.
+
+This module provides inferred CSV delimiters and helper methods for reading and writing CSV files.
+
+Example:
+    A simple example of how to use the main functionality of your package:
+
+    from datagrunt.csvfile import CSVReader
+
+    csv_file = 'electric_vehicle_population_data.csv'
+    engine = 'duckdb'
+
+    dg = CSVReader(csv_file, engine=engine)
+
+    dg.get_sample()
+
+Attributes:
+    __version__: A string representing the version of this module.
+    __author__: The name of the package author.
+    __license__: The license under which the package is released.
+"""
+
+__version__ = "0.0.0"
+__author__ = "Martin Graham"
+__license__ = "MIT"
+
+# Import key classes, functions, or submodules that should be available at the package level
+from .csvfile import CSVReader, CSVWriter
+
+# You can define __all__ to specify what gets imported with "from package import *"
+__all__ = ['CSVReader', 'CSVWriter']
+
+# Optionally, you can include a logger for your package
+import logging
+
+logging.getLogger(__name__).addHandler(logging.NullHandler())

datagrunt-0.0.0/src/datagrunt/core/databases.py

@@ -0,0 +1,53 @@
+"""Module for interfacing with databases."""
+
+# standard library
+import os
+from pathlib import Path
+import re
+
+# third party libraries
+import duckdb
+
+class DuckDBDatabase:
+    """Class to configure local database for file processing.
+       Utilizes duckdb as the processing engine.
+    """
+    DEFAULT_ENCODING = 'utf-8'
+    DEFAULT_THREAD_COUNT = 16
+
+    def __init__(self, filepath):
+        """
+        Initialize the FileDatabase class.
+
+        Args:
+            filepath (str): Path to the file.
+        """
+        self.filepath = filepath
+        self.database_filename = self._set_database_filename()
+        self.database_table_name = self._set_database_table_name()
+        self.database_connection = self._set_database_connection()
+
+    def __del__(self):
+        """Delete .db files after use."""
+        if os.path.exists(self.database_filename):
+            os.remove(self.database_filename)
+
+    def _format_filename_string(self):
+        """Remove all non alphanumeric characters from filename."""
+        return re.sub(r'[^a-zA-Z0-9]', '', Path(self.filepath).stem)
+
+    def _set_database_filename(self):
+        """Return name of duckdb file created at runtime."""
+        return f'{self._format_filename_string()}.db'
+
+    def _set_database_table_name(self):
+        """Return name of duckdb import table created during file import."""
+        return f'{self._format_filename_string()}'
+
+    def _set_database_connection(self, threads=DEFAULT_THREAD_COUNT):
+        """Establish a connection with duckdb.
+
+        Args:
+            threads (int): Number of threads to use for duckdb.
+        """
+        return duckdb.connect(self.database_filename, config = {'threads': threads})