data-manipulation 0.54__tar.gz → 0.55__tar.gz

{data_manipulation-0.54 → data_manipulation-0.55}/.github/ISSUE_TEMPLATE/bug_report.md

@@ -12,6 +12,7 @@ A clear and concise description of what the bug is.
 
 **To Reproduce**
 Steps to reproduce the behavior:
+
 1. Go to '...'
 2. Click on '....'
 3. Scroll down to '....'
@@ -24,15 +25,17 @@ A clear and concise description of what you expected to happen.
 If applicable, add screenshots to help explain your problem.
 
 **Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Browser [e.g. chrome, safari]
- - Version [e.g. 22]
+
+- OS: [e.g. iOS]
+- Browser [e.g. chrome, safari]
+- Version [e.g. 22]
 
 **Smartphone (please complete the following information):**
- - Device: [e.g. iPhone6]
- - OS: [e.g. iOS8.1]
- - Browser [e.g. stock browser, safari]
- - Version [e.g. 22]
+
+- Device: [e.g. iPhone6]
+- OS: [e.g. iOS8.1]
+- Browser [e.g. stock browser, safari]
+- Version [e.g. 22]
 
 **Additional context**
 Add any other context about the problem here.

data_manipulation-0.55/.gitignore

@@ -0,0 +1,229 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+*.lcov
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi/*
+!.pixi/config.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule*
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer,
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+# Temporary file for partial code execution
+tempCodeRunnerFile.py
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml
+
+# custom
+.claude
+.opencode
+.specstory
+.vscode
+AGENTS.md
+CLAUDE.md
+data_manipulation/_version.py

{data_manipulation-0.54 → data_manipulation-0.55}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data_manipulation
-Version: 0.54
+Version: 0.55
 Summary: Powerful data manipulation
 Author-email: Shawn Ng <shawn.coding.acc@gmail.com>
 License-Expression: BSD-3-Clause
@@ -25,7 +25,9 @@ Requires-Dist: loguru
 Requires-Dist: mysql-connector-python
 Requires-Dist: numpy
 Requires-Dist: pandas
+Requires-Dist: polars
 Requires-Dist: psycopg
+Requires-Dist: pyiceberg
 Requires-Dist: pyspark
 Requires-Dist: python-dotenv
 Requires-Dist: sqlalchemy

{data_manipulation-0.54 → data_manipulation-0.55}/RELEASE.md

@@ -20,6 +20,16 @@ git status
 
 `git status` should report no changes before continuing.
 
+## Local Development Build
+
+Contributors do not need PyPI credentials or this release flow. To work on the package
+locally, install it editable and build with the standard tooling:
+
+```bash
+pip install -e .
+python -m build          # or: uv build
+```
+
 ## Tag The Release
 
 Use the next release version, for example `0.50`:
@@ -41,13 +51,13 @@ environment before running them.
 Build the MkDocs documentation (local preview only):
 
 ```bash
-./setup.sh create_update_docs
+mkdocs build
 ```
 
 Deploy to GitHub Pages:
 
 ```bash
-./setup.sh deploy_docs
+mkdocs gh-deploy
 ```
 
 This publishes the contents of `site/` to the `gh-pages` branch.
@@ -67,16 +77,23 @@ This publishes the contents of `site/` to the `gh-pages` branch.
 
 ## Publish To PyPI
 
+Build from a **clean checkout of the tagged commit** so the artifacts carry the tag
+version. `setuptools-scm` derives the version from git, so any uncommitted tracked change
+makes it emit the *next* version as `<next>.devN+d<date>` instead of the tag - confirm
+`git status` is clean before building.
+
 Build and validate the distribution artifacts:
 
 ```bash
-./setup.sh cleanup
+rm -rf build dist ./*.egg-info
+python -m build
+python -m twine check --strict dist/*
 ```
 
 Upload the artifacts:
 
 ```bash
-./setup.sh upload
+python -m twine upload dist/*
 ```
 
 When prompted by `twine`, enter the PyPI API token. The token should start with

{data_manipulation-0.54 → data_manipulation-0.55}/data_manipulation/__init__.py

@@ -3,6 +3,7 @@ from data_manipulation import (
     beautifulsoup_,
     boto3_,
     cryptography_,
+    iceberg,
     kerberos_,
     mysql_connector_python_,
     openldap_,

{data_manipulation-0.54 → data_manipulation-0.55}/data_manipulation/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.54'
-__version_tuple__ = version_tuple = (0, 54)
+__version__ = version = '0.55'
+__version_tuple__ = version_tuple = (0, 55)
 
-__commit_id__ = commit_id = 'g7826f58d8'
+__commit_id__ = commit_id = 'gb82db140a'

data_manipulation-0.55/data_manipulation/iceberg/__init__.py

@@ -0,0 +1,49 @@
+"""Generic, framework-free Iceberg ingestion toolkit.
+
+This subpackage has no Django or host-application dependencies, so it
+can be lifted into another project as a directory copy. It provides three layers:
+
+- :mod:`.ledger` - the S3 object registry + partition ingestion ledger (types,
+  the :class:`~data_manipulation.iceberg.ledger.IngestionRepository` protocol, and a psycopg
+  PostgreSQL adapter whose partition identity is declared per application).
+- :mod:`.catalog` - :class:`~data_manipulation.iceberg.catalog.IcebergCatalog`, a PyIceberg
+  SQL-catalog manager, plus :class:`~data_manipulation.iceberg.catalog.TableSpec`.
+- :mod:`.ingest` - :class:`~data_manipulation.iceberg.ingest.IngestionCoordinator`, the
+  restatement-aware state machine, driven by an injected
+  :class:`~data_manipulation.iceberg.ingest.SourcePlan`.
+"""
+
+from .catalog import DEFAULT_MAINTENANCE_PROPERTIES, IcebergCatalog, TableSpec
+from .ingest import IngestionCoordinator, SourcePlan
+from .ledger import (
+    CONNECT_OPTION_KEYS,
+    ERROR_CODES,
+    UNVERIFIABLE_COMMIT,
+    DimensionColumn,
+    IngestionOperation,
+    IngestionRepository,
+    IngestionStatus,
+    IngestionTarget,
+    PartitionLoad,
+    PostgresIngestionRepository,
+    S3ObjectRevision,
+)
+
+__all__ = [
+    "CONNECT_OPTION_KEYS",
+    "DEFAULT_MAINTENANCE_PROPERTIES",
+    "ERROR_CODES",
+    "UNVERIFIABLE_COMMIT",
+    "DimensionColumn",
+    "IcebergCatalog",
+    "IngestionCoordinator",
+    "IngestionOperation",
+    "IngestionRepository",
+    "IngestionStatus",
+    "IngestionTarget",
+    "PartitionLoad",
+    "PostgresIngestionRepository",
+    "S3ObjectRevision",
+    "SourcePlan",
+    "TableSpec",
+]