cds-pyde-toolkit 1.1.0__tar.gz

cds_pyde_toolkit-1.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Your Name
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cds_pyde_toolkit-1.1.0/PKG-INFO

@@ -0,0 +1,156 @@
+Metadata-Version: 2.4
+Name: cds-pyde-toolkit
+Version: 1.1.0
+Summary: A growing toolkit of data-engineering helper functions and CLI commands — starting with schema inference (column standardisation, type inference, schema + DDL generation for Pandas/ANSI SQL or PySpark/Spark SQL).
+Author: Your Name
+License: MIT
+Project-URL: Homepage, https://github.com/your-org/cds-pyde-toolkit
+Project-URL: Issues, https://github.com/your-org/cds-pyde-toolkit/issues
+Keywords: pandas,pyspark,schema,ddl,data-engineering,delta-lake,databricks,toolkit
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas>=1.3
+Requires-Dist: numpy>=1.21
+Provides-Extra: excel
+Requires-Dist: openpyxl>=3.0; extra == "excel"
+Requires-Dist: xlrd>=2.0; extra == "excel"
+Requires-Dist: odfpy>=1.4; extra == "excel"
+Provides-Extra: memcheck
+Requires-Dist: psutil>=5.9; extra == "memcheck"
+Provides-Extra: all
+Requires-Dist: openpyxl>=3.0; extra == "all"
+Requires-Dist: xlrd>=2.0; extra == "all"
+Requires-Dist: odfpy>=1.4; extra == "all"
+Requires-Dist: psutil>=5.9; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: build>=1.0; extra == "dev"
+Requires-Dist: twine>=4.0; extra == "dev"
+Dynamic: license-file
+
+# cds-pyde-toolkit
+
+A growing toolkit of data-engineering helper functions and CLI commands.
+The first tool is **schema inference**: standardise column names, infer
+data types from sample data, and emit ready-to-use schema definitions and
+`CREATE TABLE` DDL — either Pandas/ANSI SQL or PySpark/Spark SQL (with
+bronze/silver/gold layer support for Databricks / Unity Catalog workflows).
+
+## Install
+
+```bash
+pip install cds-pyde-toolkit
+
+# with Excel support (.xlsx, .xls, .xlsm, .xlsb, .ods)
+pip install "cds-pyde-toolkit[excel]"
+
+# with the pre-flight memory check for large full-file reads
+pip install "cds-pyde-toolkit[memcheck]"
+
+# everything
+pip install "cds-pyde-toolkit[all]"
+```
+
+Already have it installed and want the latest release?
+
+```bash
+pip install --upgrade cds-pyde-toolkit
+```
+
+## Quick start — Python
+
+```python
+from cds_pyde_toolkit import infer_file          # top-level convenience re-export
+# or, namespaced (recommended as the toolkit grows):
+from cds_pyde_toolkit.schema_inferencer import infer_file
+
+result = infer_file(
+    "sales.csv",
+    pyspark=True,
+    casing="pascal",
+    table_name="sales_fact",
+    header_row=0,        # skip junk title rows if needed, e.g. header_row=4
+    type_threshold=0.95, # tolerate a few dirty values before falling back to string
+)
+
+print(result["schema"])         # PySpark StructType or pandas dtype dict
+print(result["create_table"])   # SQL DDL
+print(result["rename_code"])    # copy-paste column rename snippet
+print(result["report"])         # full formatted text report
+```
+
+## Quick start — CLI
+
+```bash
+cds-pyde-toolkit schema-infer sales.csv
+cds-pyde-toolkit schema-infer sales.csv --pyspark true --case pascal --layer bronze --catalog prod
+cds-pyde-toolkit schema-infer sales.xlsx --sheet Sheet2 --layer silver
+cds-pyde-toolkit schema-infer messy.csv --header-row 4 --type-threshold 0.80
+cds-pyde-toolkit --version
+```
+
+Run `cds-pyde-toolkit schema-infer --help` for the full flag reference, or see
+the module docstring in `cds_pyde_toolkit/schema_inferencer/core.py`.
+
+## Features
+
+- **Column standardisation** — camel, pascal, snake, screaming, kebab, or
+  skip casing, with symbol expansion (`/` → `or`, `%` → `pct`, etc.)
+- **Type inference** — bool, int32/int64, float, date, datetime, string,
+  with a configurable conformance threshold (`--type-threshold`) to tolerate
+  dirty data
+- **Header offset** — `--header-row` to skip junk/title rows above the real
+  header, for both CSV and Excel
+- **Dual output modes** — Pandas dtypes + ANSI SQL, or PySpark StructType +
+  Spark SQL
+- **Layered outputs** — bronze, parquet_bronze, silver, gold, gold_vw (view),
+  or all five at once
+- **Table types** — managed Delta, external, or external Delta tables
+- **Flexible input** — CSV/TSV (delimiter auto-detected), Excel
+  (`.xlsx .xls .xlsm .xlsb .ods`), or a pandas DataFrame directly
+
+## Project structure (for contributors)
+
+```
+src/cds_pyde_toolkit/
+├── __init__.py            # top-level re-exports + __version__
+├── cli.py                 # top-level CLI dispatcher (registers subcommands)
+└── schema_inferencer/     # one subpackage per feature
+    ├── __init__.py        # public API for this feature
+    ├── core.py            # logic only, no argparse
+    └── cli.py             # add_arguments(parser) + run(args) for this feature
+```
+
+**Adding a new feature later:** create `cds_pyde_toolkit/<your_feature>/` with
+the same three-file shape, then register it with one line in
+`cds_pyde_toolkit/cli.py`'s `build_parser()`. No other files need to change.
+
+## Releasing a new version
+
+Version lives in one place (`pyproject.toml`); the installed package's
+`__version__` is read live from package metadata, so there's nothing else
+to keep in sync.
+
+```bash
+python scripts/bump_version.py patch   # or minor / major / an exact X.Y.Z
+python -m build
+twine upload dist/*
+```
+
+Anyone with it already installed just runs `pip install --upgrade cds-pyde-toolkit`
+— no need to uninstall first.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

cds_pyde_toolkit-1.1.0/README.md

@@ -0,0 +1,115 @@
+# cds-pyde-toolkit
+
+A growing toolkit of data-engineering helper functions and CLI commands.
+The first tool is **schema inference**: standardise column names, infer
+data types from sample data, and emit ready-to-use schema definitions and
+`CREATE TABLE` DDL — either Pandas/ANSI SQL or PySpark/Spark SQL (with
+bronze/silver/gold layer support for Databricks / Unity Catalog workflows).
+
+## Install
+
+```bash
+pip install cds-pyde-toolkit
+
+# with Excel support (.xlsx, .xls, .xlsm, .xlsb, .ods)
+pip install "cds-pyde-toolkit[excel]"
+
+# with the pre-flight memory check for large full-file reads
+pip install "cds-pyde-toolkit[memcheck]"
+
+# everything
+pip install "cds-pyde-toolkit[all]"
+```
+
+Already have it installed and want the latest release?
+
+```bash
+pip install --upgrade cds-pyde-toolkit
+```
+
+## Quick start — Python
+
+```python
+from cds_pyde_toolkit import infer_file          # top-level convenience re-export
+# or, namespaced (recommended as the toolkit grows):
+from cds_pyde_toolkit.schema_inferencer import infer_file
+
+result = infer_file(
+    "sales.csv",
+    pyspark=True,
+    casing="pascal",
+    table_name="sales_fact",
+    header_row=0,        # skip junk title rows if needed, e.g. header_row=4
+    type_threshold=0.95, # tolerate a few dirty values before falling back to string
+)
+
+print(result["schema"])         # PySpark StructType or pandas dtype dict
+print(result["create_table"])   # SQL DDL
+print(result["rename_code"])    # copy-paste column rename snippet
+print(result["report"])         # full formatted text report
+```
+
+## Quick start — CLI
+
+```bash
+cds-pyde-toolkit schema-infer sales.csv
+cds-pyde-toolkit schema-infer sales.csv --pyspark true --case pascal --layer bronze --catalog prod
+cds-pyde-toolkit schema-infer sales.xlsx --sheet Sheet2 --layer silver
+cds-pyde-toolkit schema-infer messy.csv --header-row 4 --type-threshold 0.80
+cds-pyde-toolkit --version
+```
+
+Run `cds-pyde-toolkit schema-infer --help` for the full flag reference, or see
+the module docstring in `cds_pyde_toolkit/schema_inferencer/core.py`.
+
+## Features
+
+- **Column standardisation** — camel, pascal, snake, screaming, kebab, or
+  skip casing, with symbol expansion (`/` → `or`, `%` → `pct`, etc.)
+- **Type inference** — bool, int32/int64, float, date, datetime, string,
+  with a configurable conformance threshold (`--type-threshold`) to tolerate
+  dirty data
+- **Header offset** — `--header-row` to skip junk/title rows above the real
+  header, for both CSV and Excel
+- **Dual output modes** — Pandas dtypes + ANSI SQL, or PySpark StructType +
+  Spark SQL
+- **Layered outputs** — bronze, parquet_bronze, silver, gold, gold_vw (view),
+  or all five at once
+- **Table types** — managed Delta, external, or external Delta tables
+- **Flexible input** — CSV/TSV (delimiter auto-detected), Excel
+  (`.xlsx .xls .xlsm .xlsb .ods`), or a pandas DataFrame directly
+
+## Project structure (for contributors)
+
+```
+src/cds_pyde_toolkit/
+├── __init__.py            # top-level re-exports + __version__
+├── cli.py                 # top-level CLI dispatcher (registers subcommands)
+└── schema_inferencer/     # one subpackage per feature
+    ├── __init__.py        # public API for this feature
+    ├── core.py            # logic only, no argparse
+    └── cli.py             # add_arguments(parser) + run(args) for this feature
+```
+
+**Adding a new feature later:** create `cds_pyde_toolkit/<your_feature>/` with
+the same three-file shape, then register it with one line in
+`cds_pyde_toolkit/cli.py`'s `build_parser()`. No other files need to change.
+
+## Releasing a new version
+
+Version lives in one place (`pyproject.toml`); the installed package's
+`__version__` is read live from package metadata, so there's nothing else
+to keep in sync.
+
+```bash
+python scripts/bump_version.py patch   # or minor / major / an exact X.Y.Z
+python -m build
+twine upload dist/*
+```
+
+Anyone with it already installed just runs `pip install --upgrade cds-pyde-toolkit`
+— no need to uninstall first.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

cds_pyde_toolkit-1.1.0/pyproject.toml

@@ -0,0 +1,54 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "cds-pyde-toolkit"
+version = "1.1.0"
+description = "A growing toolkit of data-engineering helper functions and CLI commands — starting with schema inference (column standardisation, type inference, schema + DDL generation for Pandas/ANSI SQL or PySpark/Spark SQL)."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [
+    { name = "Your Name" }
+]
+keywords = ["pandas", "pyspark", "schema", "ddl", "data-engineering", "delta-lake", "databricks", "toolkit"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Topic :: Database",
+    "Topic :: Software Development :: Libraries",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+
+dependencies = [
+    "pandas>=1.3",
+    "numpy>=1.21",
+]
+
+[project.optional-dependencies]
+excel = ["openpyxl>=3.0", "xlrd>=2.0", "odfpy>=1.4"]
+memcheck = ["psutil>=5.9"]
+all = ["openpyxl>=3.0", "xlrd>=2.0", "odfpy>=1.4", "psutil>=5.9"]
+dev = ["pytest>=7.0", "build>=1.0", "twine>=4.0"]
+
+[project.urls]
+Homepage = "https://github.com/your-org/cds-pyde-toolkit"
+Issues = "https://github.com/your-org/cds-pyde-toolkit/issues"
+
+[project.scripts]
+cds-pyde-toolkit = "cds_pyde_toolkit.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+cds_pyde_toolkit = ["py.typed"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

cds_pyde_toolkit-1.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

cds_pyde_toolkit-1.1.0/src/cds_pyde_toolkit/__init__.py

@@ -0,0 +1,44 @@
+"""
+cds_pyde_toolkit
+=================
+A growing toolkit of data-engineering helper functions and CLI commands.
+
+Currently included
+-------------------
+schema_inferencer   Infer column names, data types, schema definitions, and
+                     CREATE TABLE / VIEW DDL from a file or a pandas DataFrame.
+                     (Pandas/ANSI SQL or PySpark/Spark SQL.)
+
+Usage
+-----
+Namespaced (recommended as the toolkit grows, to avoid name clashes between
+tools)::
+
+    from cds_pyde_toolkit.schema_inferencer import infer_file
+    result = infer_file(my_dataframe, pyspark=True, casing="snake")
+
+Top-level convenience re-exports are also provided for the most commonly
+used function of each tool — currently just `infer_file`::
+
+    from cds_pyde_toolkit import infer_file
+
+CLI
+---
+    cds-pyde-toolkit schema-infer Sales1.csv --pyspark true
+    cds-pyde-toolkit --help
+"""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _installed_version
+
+from .schema_inferencer import infer_file
+
+try:
+    __version__ = _installed_version("cds-pyde-toolkit")
+except PackageNotFoundError:  # pragma: no cover — running from source without an install
+    __version__ = "0.0.0+unknown"
+
+__all__ = [
+    "infer_file",
+    "__version__",
+]

cds_pyde_toolkit-1.1.0/src/cds_pyde_toolkit/cli.py

@@ -0,0 +1,62 @@
+"""
+cds_pyde_toolkit.cli
+======================
+Top-level command-line entry point for the whole toolkit, installed as the
+`cds-pyde-toolkit` console script. Each tool in the package contributes one
+subcommand here.
+
+Currently registered subcommands:
+    schema-infer    → cds_pyde_toolkit.schema_inferencer
+
+Adding a new tool later
+------------------------
+1. Create a new submodule, e.g. `cds_pyde_toolkit/data_profiler/` with its
+   own `core.py` and a `cli.py` that exposes `add_arguments(parser)` and
+   `run(args)` (see `schema_inferencer/cli.py` for the pattern).
+2. Register it below with one call to `subparsers.add_parser(...)` +
+   `<tool>.cli.add_arguments(...)`.
+That's it — no other changes needed; dispatch is generic via `args._run`.
+"""
+
+from __future__ import annotations
+
+import argparse
+from typing import Optional
+
+from . import __version__
+from .schema_inferencer import cli as schema_inferencer_cli
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog='cds-pyde-toolkit',
+        description='A growing toolkit of data-engineering helper commands.',
+    )
+    parser.add_argument(
+        '--version', action='version', version=f'cds-pyde-toolkit {__version__}'
+    )
+
+    subparsers = parser.add_subparsers(dest='command', required=True)
+
+    schema_infer_parser = subparsers.add_parser(
+        'schema-infer',
+        help='Infer column names, data types, schema, and CREATE TABLE/VIEW DDL '
+             'from a file or DataFrame.',
+    )
+    schema_inferencer_cli.add_arguments(schema_infer_parser)
+
+    # ── Future subcommands get registered here, e.g.: ─────────────────────────
+    # profile_parser = subparsers.add_parser('profile', help='...')
+    # data_profiler_cli.add_arguments(profile_parser)
+
+    return parser
+
+
+def main(argv: Optional[list] = None) -> None:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    args._run(args)
+
+
+if __name__ == '__main__':  # pragma: no cover
+    main()

cds_pyde_toolkit-1.1.0/src/cds_pyde_toolkit/schema_inferencer/__init__.py

@@ -0,0 +1,38 @@
+"""
+cds_pyde_toolkit.schema_inferencer
+===============================
+Infer column names, data types, schema definitions, and CREATE TABLE / VIEW
+DDL from a CSV/TSV/Excel file — or directly from a pandas DataFrame already
+in memory (e.g. a Spark DataFrame converted via `.toPandas()`).
+
+Quick start
+-----------
+    from cds_pyde_toolkit.schema_inferencer import infer_file
+
+    result = infer_file(my_dataframe, pyspark=True, casing="snake")
+    print(result["schema"])
+    print(result["create_table"])
+
+See `cds_pyde_toolkit.schema_inferencer.core.infer_file` for the full parameter
+reference, or run `pyde-toolkit schema-infer --help` for the CLI.
+"""
+
+from .core import (
+    VALID_CASINGS,
+    VALID_LAYERS,
+    VALID_TABLE_TYPES,
+    format_column_name,
+    infer_file,
+    standardise_columns,
+    to_camel_case,
+)
+
+__all__ = [
+    "infer_file",
+    "standardise_columns",
+    "format_column_name",
+    "to_camel_case",
+    "VALID_CASINGS",
+    "VALID_LAYERS",
+    "VALID_TABLE_TYPES",
+]