penwings 0.1.0__tar.gz

penwings-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

penwings-0.1.0/LICENSE

@@ -0,0 +1,9 @@
+The MIT License (MIT)
+
+Copyright 2026 Raf Blanckaert
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

penwings-0.1.0/PKG-INFO

@@ -0,0 +1,177 @@
+Metadata-Version: 2.4
+Name: penwings
+Version: 0.1.0
+Summary: Lightweight library to handle data and reproduce workflows
+Author-email: Raf Blanckaert <R.Blanckaert@outlook.com>
+License: LICENSE
+Project-URL: Homepage, https://github.com/Frissie/penwings
+Project-URL: Repository, https://github.com/Frissie/penwings
+Project-URL: Issues, https://github.com/Frissie/penwings/issues
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: sqlalchemy<3.0.0,>=2.0.46
+Requires-Dist: pyodbc<6.0.0,>=5.3.0
+Requires-Dist: pandas<4.0.0,>=3.0.0
+Requires-Dist: numpy<3.0.0,>=2.4.1
+Provides-Extra: scipy
+Requires-Dist: scipy<2.0.0,>=1.17.0; extra == "scipy"
+Provides-Extra: sklearn
+Requires-Dist: scikit-learn<2.0.0,>=1.8.0; extra == "sklearn"
+Provides-Extra: optuna
+Requires-Dist: optuna<5.0.0,>=4.7.0; extra == "optuna"
+Provides-Extra: all
+Requires-Dist: openpyxl<4.0.0,>=3.1.5; extra == "all"
+Dynamic: license-file
+
+# Penwings
+
+**Penwings** is a lightweight Python library designed to simplify SQL data workflows by automatically importing data from SQL and caching it as Parquet files. This ensures faster subsequent access and reproducible pipelines, while reducing database get.
+
+---
+
+## Table of Contents
+
+1. [Features](#features)  
+2. [Installation](#installation)  
+3. [Getting Started](#getting-started)  
+4. [Usage](#usage)  
+5. [Versioning](#versioning)  
+6. [Contributing](#contributing)  
+7. [License](#license)  
+
+---
+
+## Features
+
+- get data from SQL queries or SQL files  
+- Automatically save query results as Parquet files  
+- Reuse Parquet files to avoid redundant queries  
+- Simple, stable API for reproducible workflows  
+- Optimized for performance and ease of integration  
+
+---
+
+## Installation
+
+Install via pip:
+
+pip install penwings
+
+> Make sure you have Python 3.11+ installed.
+
+---
+
+## Getting Started
+
+### Importing the Library
+
+from penwings import SQLParquetCache
+
+### Initialize the Cache
+
+You can initialize the cache by providing either a SQL directory or a query string, along with a Parquet directory:
+
+```
+from sqlalchemy import create_engine
+```
+
+# SQL connection
+```
+engine = create_engine("postgresql://user:password@localhost/dbname")
+
+# Initialize the cache
+loader = SQLParquetCache(
+    sql_dir="sql_files",        # Optional if using query string
+    parquet_dir="parquet_cache",
+    conn=engine
+)
+```
+---
+
+## Usage
+
+### 1. Using SQL Files
+
+If you have SQL files stored in a directory:
+
+```
+# Run a SQL file and cache the result
+df = loader.get("monthly_sales.sql")
+```
+
+- `penwings` will automatically check if a Parquet version exists.
+- If it exists, the cached Parquet is loaded.
+- If not, the SQL query runs and the result is saved as a Parquet file.
+
+### 2. Using SQL Query Strings
+
+You can also pass queries directly:
+
+```
+query = "SELECT * FROM sales WHERE month='2026-02'"
+df = loader.get(sql=query, parquet_name="sales_feb2026")
+```
+
+- `parquet_name` determines the Parquet file name.
+- Works similarly to SQL file mode for caching.
+
+### 3. Automatic Parquet Management
+
+- All results are cached in the specified `parquet_dir`.
+- This reduces repeated database queries and ensures reproducibility.
+- Cached files can be reloaded for faster access.
+
+---
+
+## Versioning
+
+Penwings follows **semantic versioning**:
+
+- **MAJOR**: Breaking changes to API  
+- **MINOR**: New features, backward-compatible  
+- **PATCH**: Bug fixes  
+
+---
+
+## Contributing
+
+We welcome contributions!  
+
+1. Fork the repository  
+2. Create a feature branch (`git checkout -b feature/my-feature`)  
+3. Commit your changes (`git commit -m 'Add new feature'`)  
+4. Push to branch (`git push origin feature/my-feature`)  
+5. Open a pull request  
+
+Please ensure your code follows PEP8 standards.
+
+---
+
+## License
+
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.
+
+---
+
+## Example Workflow
+
+```
+from sqlalchemy import create_engine
+from penwings import SQLParquetCache
+
+engine = create_engine("sqlite:///example.db")
+
+loader = SQLParquetCache(
+    sql_dir="sql_queries",
+    parquet_dir="parquet_cache",
+    conn=engine
+)
+
+# get data
+df_jan = loader.get("sales_january.sql")
+df_feb = loader.get(sql="SELECT * FROM sales WHERE month='2026-02'", parquet_name="sales_feb")
+```
+
+- SQL files are automatically cached as Parquet  
+- Subsequent loads are fast and do not hit the database  

penwings-0.1.0/README.md

@@ -0,0 +1,151 @@
+# Penwings
+
+**Penwings** is a lightweight Python library designed to simplify SQL data workflows by automatically importing data from SQL and caching it as Parquet files. This ensures faster subsequent access and reproducible pipelines, while reducing database get.
+
+---
+
+## Table of Contents
+
+1. [Features](#features)  
+2. [Installation](#installation)  
+3. [Getting Started](#getting-started)  
+4. [Usage](#usage)  
+5. [Versioning](#versioning)  
+6. [Contributing](#contributing)  
+7. [License](#license)  
+
+---
+
+## Features
+
+- get data from SQL queries or SQL files  
+- Automatically save query results as Parquet files  
+- Reuse Parquet files to avoid redundant queries  
+- Simple, stable API for reproducible workflows  
+- Optimized for performance and ease of integration  
+
+---
+
+## Installation
+
+Install via pip:
+
+pip install penwings
+
+> Make sure you have Python 3.11+ installed.
+
+---
+
+## Getting Started
+
+### Importing the Library
+
+from penwings import SQLParquetCache
+
+### Initialize the Cache
+
+You can initialize the cache by providing either a SQL directory or a query string, along with a Parquet directory:
+
+```
+from sqlalchemy import create_engine
+```
+
+# SQL connection
+```
+engine = create_engine("postgresql://user:password@localhost/dbname")
+
+# Initialize the cache
+loader = SQLParquetCache(
+    sql_dir="sql_files",        # Optional if using query string
+    parquet_dir="parquet_cache",
+    conn=engine
+)
+```
+---
+
+## Usage
+
+### 1. Using SQL Files
+
+If you have SQL files stored in a directory:
+
+```
+# Run a SQL file and cache the result
+df = loader.get("monthly_sales.sql")
+```
+
+- `penwings` will automatically check if a Parquet version exists.
+- If it exists, the cached Parquet is loaded.
+- If not, the SQL query runs and the result is saved as a Parquet file.
+
+### 2. Using SQL Query Strings
+
+You can also pass queries directly:
+
+```
+query = "SELECT * FROM sales WHERE month='2026-02'"
+df = loader.get(sql=query, parquet_name="sales_feb2026")
+```
+
+- `parquet_name` determines the Parquet file name.
+- Works similarly to SQL file mode for caching.
+
+### 3. Automatic Parquet Management
+
+- All results are cached in the specified `parquet_dir`.
+- This reduces repeated database queries and ensures reproducibility.
+- Cached files can be reloaded for faster access.
+
+---
+
+## Versioning
+
+Penwings follows **semantic versioning**:
+
+- **MAJOR**: Breaking changes to API  
+- **MINOR**: New features, backward-compatible  
+- **PATCH**: Bug fixes  
+
+---
+
+## Contributing
+
+We welcome contributions!  
+
+1. Fork the repository  
+2. Create a feature branch (`git checkout -b feature/my-feature`)  
+3. Commit your changes (`git commit -m 'Add new feature'`)  
+4. Push to branch (`git push origin feature/my-feature`)  
+5. Open a pull request  
+
+Please ensure your code follows PEP8 standards.
+
+---
+
+## License
+
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.
+
+---
+
+## Example Workflow
+
+```
+from sqlalchemy import create_engine
+from penwings import SQLParquetCache
+
+engine = create_engine("sqlite:///example.db")
+
+loader = SQLParquetCache(
+    sql_dir="sql_queries",
+    parquet_dir="parquet_cache",
+    conn=engine
+)
+
+# get data
+df_jan = loader.get("sales_january.sql")
+df_feb = loader.get(sql="SELECT * FROM sales WHERE month='2026-02'", parquet_name="sales_feb")
+```
+
+- SQL files are automatically cached as Parquet  
+- Subsequent loads are fast and do not hit the database  

penwings-0.1.0/pyproject.toml

@@ -0,0 +1,59 @@
+[project]
+name = "penwings"
+dynamic = ["version"]
+description = "Lightweight library to handle data and reproduce workflows"
+readme = "README.md"
+license = {text = "LICENSE"}
+authors = [
+    {name = "Raf Blanckaert",email = "R.Blanckaert@outlook.com"}
+]
+requires-python = ">=3.11"
+dependencies = [
+    "sqlalchemy (>=2.0.46,<3.0.0)",
+    "pyodbc (>=5.3.0,<6.0.0)",
+    "pandas (>=3.0.0,<4.0.0)",
+    "numpy (>=2.4.1,<3.0.0)"
+]
+
+[project.urls]
+Homepage = "https://github.com/Frissie/penwings"
+Repository = "https://github.com/Frissie/penwings"
+Issues = "https://github.com/Frissie/penwings/issues"
+
+
+[tool.setuptools_scm]
+version_scheme = "guess-next-dev"
+local_scheme = "no-local-version"
+tag_regex = "^v(?P<version>.*)$"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["penwings*"]
+exclude = ["penwings._*"]
+
+[build-system]
+requires = ["setuptools>=68", "wheel", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[dependency-groups]
+dev = [
+    "openpyxl>=3.1.5",
+    "optuna>=4.7.0",
+    "scikit-learn>=1.8.0",
+    "scipy>=1.17.0",
+]
+
+[project.optional-dependencies]
+scipy = ["scipy (>=1.17.0,<2.0.0)"]
+sklearn = ["scikit-learn (>=1.8.0,<2.0.0)"]
+optuna= ["optuna (>=4.7.0,<5.0.0)"]
+all = ["openpyxl (>=3.1.5,<4.0.0)"]
+
+[[tool.uv.index]]
+name = "testpypi"
+url = "https://test.pypi.org/simple/"
+publish-url = "https://test.pypi.org/legacy/"
+explicit = true

penwings-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

penwings-0.1.0/src/penwings/__init__.py

@@ -0,0 +1,9 @@
+from .io.cache import SQLParquetCache
+from .paths import input_dir, output_dir, model_dir
+
+__all__ = [
+    "SQLParquetCache",
+    "input_dir",
+    "output_dir",
+    "model_dir",
+]

penwings-0.1.0/src/penwings/_utils/_decorators.py

@@ -0,0 +1,38 @@
+import time as t
+
+from functools import wraps
+from pathlib import Path
+
+
+def timing(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        start = t.perf_counter()
+        result = func(*args, **kwargs)
+        end = t.perf_counter()
+        print(f"{func.__name__} took {end - start: .2f}")
+        return result
+
+    return wrapper
+
+
+def timing_sql(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        sql_file = kwargs.get("sql_file", None)
+        verbose = getattr(args[0], "verbose", True)
+
+        if sql_file is None and len(args) > 1:
+            sql_file = args[1]
+
+        sql_file = Path(sql_file)
+
+        start = t.perf_counter()
+        result, source = func(*args, **kwargs)
+        end = t.perf_counter()
+
+        if verbose:
+            print(f"{sql_file.stem} -> {source} took {end - start: .2f} seconds to load")
+        return result
+
+    return wrapper

penwings-0.1.0/src/penwings/_utils/_typing.py

@@ -0,0 +1,7 @@
+from typing import TypedDict, List, Union
+
+
+class SQLParquetKwargs(TypedDict, total=False):
+    index_col: Union[str, List[str], None]
+    parse_dates: Union[List[str], None]
+    dtype: Union[dict, None]

penwings-0.1.0/src/penwings/io/cache.py

@@ -0,0 +1,93 @@
+import pandas as pd
+
+from sqlalchemy import Engine
+from pathlib import Path
+from datetime import datetime, timedelta
+from typing import Unpack, Union, Optional
+from .._utils._typing import SQLParquetKwargs
+from .._utils._decorators import timing_sql
+
+
+class SQLParquetCache:
+    def __init__(
+        self,
+        parquet_dir: Union[Path, str],
+        conn: Engine,
+        sql_dir: Optional[Union[Path, str]] = None,
+        refresh_days: int = 0,  # zero disables refresh when force == false
+        verbose: bool = True,
+        **kwargs: Unpack[SQLParquetKwargs],
+    ):
+
+        if sql_dir is not None:
+            self.sql_dir: Path = Path(sql_dir)
+        self.parquet_dir: Path = Path(parquet_dir)
+        self.refresh_days = refresh_days
+        self.conn = conn
+        self.global_kwargs = kwargs
+
+        self.verbose = verbose
+        self.source = "SQL"
+
+    def set_params(self, **params):
+        for key, value in params.items():
+            if not hasattr(self, key):
+                raise ValueError(f"Invalid parameter: {key}")
+            setattr(self, key, value)
+        return self
+
+    def _sql_path(self, sql_file: str) -> Path:
+        return self.sql_dir / sql_file
+
+    def _parquet_path(self, sql_file: str, parquet_name: str | None = None) -> Path:
+        name = parquet_name or Path(sql_file).stem
+        return self.parquet_dir / f"{name}.parquet"
+
+    def _is_new(self, path: Path, refresh_window: int) -> bool:
+        if not path.exists():
+            return False
+        if self.refresh_days == 0:
+            return True
+        last_modified = datetime.fromtimestamp(path.stat().st_mtime)
+        return datetime.now() - last_modified < timedelta(days=refresh_window)
+
+    def _read_sql(self, sql_file: str):
+        return self._sql_path(sql_file).read_text()
+
+    def _return_sql(self, query: str, conn, **kwargs: Unpack[SQLParquetKwargs]) -> pd.DataFrame:
+        return pd.read_sql(query, conn, **kwargs)
+
+    @timing_sql
+    def get(
+        self,
+        sql: str,
+        parquet_name: Union[str, None] = None,
+        conn: Engine | None = None,
+        refresh_days: int | None = None,
+        force: bool = False,
+        **kwargs: Unpack[SQLParquetKwargs],
+    ) -> tuple[pd.DataFrame, str]:
+        if isinstance(sql, str) and Path(sql).suffix == ".sql":
+            query = self._read_sql(sql)
+        elif isinstance(sql, str):
+            if parquet_name is None:
+                raise ValueError("parquet_name must be provided if query is passed directly")
+            query = sql
+        else:
+            raise ValueError("sql must be a SQL string or a path to a .sql file")
+
+        connection = conn or self.conn
+        refresh_window = refresh_days or self.refresh_days
+        parquet_path = self._parquet_path(query)
+        sql_kwargs = self.global_kwargs | kwargs
+
+        if not force and self._is_new(parquet_path, refresh_window):
+            source = "Parquet"
+            return pd.read_parquet(parquet_path), source
+
+        source = "SQL"
+        df = self._return_sql(query, connection, **sql_kwargs)
+        self.parquet_dir.mkdir(parents=True, exist_ok=True)
+        df.to_parquet(parquet_path, index=False)
+
+        return df, source

penwings-0.1.0/src/penwings/paths.py

@@ -0,0 +1,16 @@
+import pathlib
+
+home_dir = pathlib.Path.cwd()
+proj_dir = pathlib.Path.cwd().parent
+
+input_dir = home_dir / "input"
+sql_dir = input_dir / "sql"
+parquet_dir = input_dir / "parquet"
+output_dir = home_dir / "output"
+
+if __name__ == "__main__":
+    i = 1
+    for name, value in dict(locals()).items():
+        if isinstance(value, pathlib.Path):
+            print(f"{i} - {name}: {value}")
+            i += 1