sqlalchemy-upsert-kit 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlalchemy_upsert_kit-0.1.1/AUTHORS.rst +15 -0
- sqlalchemy_upsert_kit-0.1.1/LICENSE.txt +21 -0
- sqlalchemy_upsert_kit-0.1.1/PKG-INFO +119 -0
- sqlalchemy_upsert_kit-0.1.1/README.rst +71 -0
- sqlalchemy_upsert_kit-0.1.1/pyproject.toml +151 -0
- sqlalchemy_upsert_kit-0.1.1/sqlalchemy_upsert_kit/__init__.py +9 -0
- sqlalchemy_upsert_kit-0.1.1/sqlalchemy_upsert_kit/_version.py +15 -0
- sqlalchemy_upsert_kit-0.1.1/sqlalchemy_upsert_kit/api.py +3 -0
- sqlalchemy_upsert_kit-0.1.1/sqlalchemy_upsert_kit/exc.py +13 -0
- sqlalchemy_upsert_kit-0.1.1/sqlalchemy_upsert_kit/paths.py +33 -0
- sqlalchemy_upsert_kit-0.1.1/sqlalchemy_upsert_kit/sqlite/__init__.py +34 -0
- sqlalchemy_upsert_kit-0.1.1/sqlalchemy_upsert_kit/sqlite/api.py +5 -0
- sqlalchemy_upsert_kit-0.1.1/sqlalchemy_upsert_kit/sqlite/executor.py +535 -0
- sqlalchemy_upsert_kit-0.1.1/sqlalchemy_upsert_kit/sqlite/insert_or_ignore.py +156 -0
- sqlalchemy_upsert_kit-0.1.1/sqlalchemy_upsert_kit/sqlite/insert_or_merge.py +254 -0
- sqlalchemy_upsert_kit-0.1.1/sqlalchemy_upsert_kit/sqlite/insert_or_replace.py +177 -0
- sqlalchemy_upsert_kit-0.1.1/sqlalchemy_upsert_kit/utils.py +92 -0
- sqlalchemy_upsert_kit-0.1.1/sqlalchemy_upsert_kit/vendor/__init__.py +2 -0
- sqlalchemy_upsert_kit-0.1.1/sqlalchemy_upsert_kit/vendor/pytest_cov_helper.py +148 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
.. _about_author:
|
|
2
|
+
|
|
3
|
+
About the Author
|
|
4
|
+
------------------------------------------------------------------------------
|
|
5
|
+
::
|
|
6
|
+
|
|
7
|
+
(\ (\
|
|
8
|
+
( -.-)o
|
|
9
|
+
o_(")(")
|
|
10
|
+
|
|
11
|
+
**Sanhe Hu** is a seasoned software engineer with a deep passion for Python development since 2010. As an author and maintainer of `150+ open-source Python projects <https://pypi.org/user/machugwu/>`_, with over `15 million monthly downloads <https://github.com/MacHu-GWU>`_, I bring a wealth of experience to the table. As a Senior Solution Architect and Subject Matter Expert in AI, Data, Amazon Web Services, Cloud Engineering, DevOps, I thrive on helping clients with platform design, enterprise architecture, and strategic roadmaps.
|
|
12
|
+
|
|
13
|
+
Talk is cheap, show me the code:
|
|
14
|
+
|
|
15
|
+
- My Github: https://github.com/MacHu-GWU
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Sanhe Hu <husanhe@email.com>
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: sqlalchemy_upsert_kit
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: High-performance SQLAlchemy bulk upsert operations using temporary table staging - 20x faster than row-by-row approaches.
|
|
5
|
+
License: MIT
|
|
6
|
+
Author: Sanhe Hu
|
|
7
|
+
Author-email: husanhe@email.com
|
|
8
|
+
Maintainer: Sanhe Hu
|
|
9
|
+
Maintainer-email: husanhe@email.com
|
|
10
|
+
Requires-Python: >=3.9,<4.0
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Provides-Extra: auto
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Provides-Extra: doc
|
|
21
|
+
Provides-Extra: test
|
|
22
|
+
Requires-Dist: Sphinx (>=7.4.7,<8.0.0) ; extra == "doc"
|
|
23
|
+
Requires-Dist: build (>=1.2.1,<2.0.0) ; extra == "dev"
|
|
24
|
+
Requires-Dist: docfly (==3.0.0) ; extra == "doc"
|
|
25
|
+
Requires-Dist: furo (==2024.8.6) ; extra == "doc"
|
|
26
|
+
Requires-Dist: ipython (>=8.18.1,<8.19.0) ; extra == "doc"
|
|
27
|
+
Requires-Dist: nbsphinx (>=0.8.12,<1.0.0) ; extra == "doc"
|
|
28
|
+
Requires-Dist: prettytable (>=3.0.0,<4.0.0) ; extra == "test"
|
|
29
|
+
Requires-Dist: pygments (>=2.18.0,<3.0.0) ; extra == "doc"
|
|
30
|
+
Requires-Dist: pytest (>=8.2.2,<9.0.0) ; extra == "test"
|
|
31
|
+
Requires-Dist: pytest-cov (>=6.0.0,<7.0.0) ; extra == "test"
|
|
32
|
+
Requires-Dist: rich (>=13.8.1,<14.0.0) ; extra == "dev"
|
|
33
|
+
Requires-Dist: rstobj (==1.2.1) ; extra == "doc"
|
|
34
|
+
Requires-Dist: sphinx-copybutton (>=0.5.2,<1.0.0) ; extra == "doc"
|
|
35
|
+
Requires-Dist: sphinx-design (>=0.6.1,<1.0.0) ; extra == "doc"
|
|
36
|
+
Requires-Dist: sphinx-jinja (>=2.0.2,<3.0.0) ; extra == "doc"
|
|
37
|
+
Requires-Dist: sqlalchemy (>=2.0.0,<3.0.0)
|
|
38
|
+
Requires-Dist: twine (>=6.0.0,<7.0.0) ; extra == "dev"
|
|
39
|
+
Requires-Dist: wheel (>=0.45.0,<1.0.0) ; extra == "dev"
|
|
40
|
+
Project-URL: Changelog, https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project/blob/main/release-history.rst
|
|
41
|
+
Project-URL: Documentation, https://sqlalchemy-upsert-kit.readthedocs.io/en/latest/
|
|
42
|
+
Project-URL: Download, https://pypi.org/pypi/sqlalchemy-upsert-kit#files
|
|
43
|
+
Project-URL: Homepage, https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project
|
|
44
|
+
Project-URL: Issues, https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project/issues
|
|
45
|
+
Project-URL: Repository, https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project
|
|
46
|
+
Description-Content-Type: text/x-rst
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
.. image:: https://readthedocs.org/projects/sqlalchemy-upsert-kit/badge/?version=latest
|
|
50
|
+
:target: https://sqlalchemy-upsert-kit.readthedocs.io/en/latest/
|
|
51
|
+
:alt: Documentation Status
|
|
52
|
+
|
|
53
|
+
.. image:: https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project/actions/workflows/main.yml/badge.svg
|
|
54
|
+
:target: https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project/actions?query=workflow:CI
|
|
55
|
+
|
|
56
|
+
.. image:: https://codecov.io/gh/MacHu-GWU/sqlalchemy_upsert_kit-project/branch/main/graph/badge.svg
|
|
57
|
+
:target: https://codecov.io/gh/MacHu-GWU/sqlalchemy_upsert_kit-project
|
|
58
|
+
|
|
59
|
+
.. image:: https://img.shields.io/pypi/v/sqlalchemy-upsert-kit.svg
|
|
60
|
+
:target: https://pypi.python.org/pypi/sqlalchemy-upsert-kit
|
|
61
|
+
|
|
62
|
+
.. image:: https://img.shields.io/pypi/l/sqlalchemy-upsert-kit.svg
|
|
63
|
+
:target: https://pypi.python.org/pypi/sqlalchemy-upsert-kit
|
|
64
|
+
|
|
65
|
+
.. image:: https://img.shields.io/pypi/pyversions/sqlalchemy-upsert-kit.svg
|
|
66
|
+
:target: https://pypi.python.org/pypi/sqlalchemy-upsert-kit
|
|
67
|
+
|
|
68
|
+
.. image:: https://img.shields.io/badge/✍️_Release_History!--None.svg?style=social&logo=github
|
|
69
|
+
:target: https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project/blob/main/release-history.rst
|
|
70
|
+
|
|
71
|
+
.. image:: https://img.shields.io/badge/⭐_Star_me_on_GitHub!--None.svg?style=social&logo=github
|
|
72
|
+
:target: https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project
|
|
73
|
+
|
|
74
|
+
------
|
|
75
|
+
|
|
76
|
+
.. image:: https://img.shields.io/badge/Link-API-blue.svg
|
|
77
|
+
:target: https://sqlalchemy-upsert-kit.readthedocs.io/en/latest/py-modindex.html
|
|
78
|
+
|
|
79
|
+
.. image:: https://img.shields.io/badge/Link-Install-blue.svg
|
|
80
|
+
:target: `install`_
|
|
81
|
+
|
|
82
|
+
.. image:: https://img.shields.io/badge/Link-GitHub-blue.svg
|
|
83
|
+
:target: https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project
|
|
84
|
+
|
|
85
|
+
.. image:: https://img.shields.io/badge/Link-Submit_Issue-blue.svg
|
|
86
|
+
:target: https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project/issues
|
|
87
|
+
|
|
88
|
+
.. image:: https://img.shields.io/badge/Link-Request_Feature-blue.svg
|
|
89
|
+
:target: https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project/issues
|
|
90
|
+
|
|
91
|
+
.. image:: https://img.shields.io/badge/Link-Download-blue.svg
|
|
92
|
+
:target: https://pypi.org/pypi/sqlalchemy-upsert-kit#files
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
Welcome to ``sqlalchemy_upsert_kit`` Documentation
|
|
96
|
+
==============================================================================
|
|
97
|
+
.. image:: https://sqlalchemy-upsert-kit.readthedocs.io/en/latest/_static/sqlalchemy_upsert_kit-logo.png
|
|
98
|
+
:target: https://sqlalchemy-upsert-kit.readthedocs.io/en/latest/
|
|
99
|
+
|
|
100
|
+
``sqlalchemy_upsert_kit`` provides high-performance bulk upsert operations for SQLAlchemy applications using temporary table staging. It offers three optimized strategies: ``insert_or_ignore`` for conflict-free insertion, ``insert_or_replace`` for complete record replacement, and ``insert_or_merge`` for selective column updates while preserving existing data. Each strategy leverages efficient SQL JOIN operations and temporary tables to achieve ~20x performance improvements over traditional row-by-row upserts, making it ideal for ETL processes, data synchronization, and bulk data operations where speed and reliability are critical.
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
.. _install:
|
|
104
|
+
|
|
105
|
+
Install
|
|
106
|
+
------------------------------------------------------------------------------
|
|
107
|
+
|
|
108
|
+
``sqlalchemy_upsert_kit`` is released on PyPI, so all you need is to:
|
|
109
|
+
|
|
110
|
+
.. code-block:: console
|
|
111
|
+
|
|
112
|
+
$ pip install sqlalchemy-upsert-kit
|
|
113
|
+
|
|
114
|
+
To upgrade to latest version:
|
|
115
|
+
|
|
116
|
+
.. code-block:: console
|
|
117
|
+
|
|
118
|
+
$ pip install --upgrade sqlalchemy-upsert-kit
|
|
119
|
+
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
|
|
2
|
+
.. image:: https://readthedocs.org/projects/sqlalchemy-upsert-kit/badge/?version=latest
|
|
3
|
+
:target: https://sqlalchemy-upsert-kit.readthedocs.io/en/latest/
|
|
4
|
+
:alt: Documentation Status
|
|
5
|
+
|
|
6
|
+
.. image:: https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project/actions/workflows/main.yml/badge.svg
|
|
7
|
+
:target: https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project/actions?query=workflow:CI
|
|
8
|
+
|
|
9
|
+
.. image:: https://codecov.io/gh/MacHu-GWU/sqlalchemy_upsert_kit-project/branch/main/graph/badge.svg
|
|
10
|
+
:target: https://codecov.io/gh/MacHu-GWU/sqlalchemy_upsert_kit-project
|
|
11
|
+
|
|
12
|
+
.. image:: https://img.shields.io/pypi/v/sqlalchemy-upsert-kit.svg
|
|
13
|
+
:target: https://pypi.python.org/pypi/sqlalchemy-upsert-kit
|
|
14
|
+
|
|
15
|
+
.. image:: https://img.shields.io/pypi/l/sqlalchemy-upsert-kit.svg
|
|
16
|
+
:target: https://pypi.python.org/pypi/sqlalchemy-upsert-kit
|
|
17
|
+
|
|
18
|
+
.. image:: https://img.shields.io/pypi/pyversions/sqlalchemy-upsert-kit.svg
|
|
19
|
+
:target: https://pypi.python.org/pypi/sqlalchemy-upsert-kit
|
|
20
|
+
|
|
21
|
+
.. image:: https://img.shields.io/badge/✍️_Release_History!--None.svg?style=social&logo=github
|
|
22
|
+
:target: https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project/blob/main/release-history.rst
|
|
23
|
+
|
|
24
|
+
.. image:: https://img.shields.io/badge/⭐_Star_me_on_GitHub!--None.svg?style=social&logo=github
|
|
25
|
+
:target: https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project
|
|
26
|
+
|
|
27
|
+
------
|
|
28
|
+
|
|
29
|
+
.. image:: https://img.shields.io/badge/Link-API-blue.svg
|
|
30
|
+
:target: https://sqlalchemy-upsert-kit.readthedocs.io/en/latest/py-modindex.html
|
|
31
|
+
|
|
32
|
+
.. image:: https://img.shields.io/badge/Link-Install-blue.svg
|
|
33
|
+
:target: `install`_
|
|
34
|
+
|
|
35
|
+
.. image:: https://img.shields.io/badge/Link-GitHub-blue.svg
|
|
36
|
+
:target: https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project
|
|
37
|
+
|
|
38
|
+
.. image:: https://img.shields.io/badge/Link-Submit_Issue-blue.svg
|
|
39
|
+
:target: https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project/issues
|
|
40
|
+
|
|
41
|
+
.. image:: https://img.shields.io/badge/Link-Request_Feature-blue.svg
|
|
42
|
+
:target: https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project/issues
|
|
43
|
+
|
|
44
|
+
.. image:: https://img.shields.io/badge/Link-Download-blue.svg
|
|
45
|
+
:target: https://pypi.org/pypi/sqlalchemy-upsert-kit#files
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
Welcome to ``sqlalchemy_upsert_kit`` Documentation
|
|
49
|
+
==============================================================================
|
|
50
|
+
.. image:: https://sqlalchemy-upsert-kit.readthedocs.io/en/latest/_static/sqlalchemy_upsert_kit-logo.png
|
|
51
|
+
:target: https://sqlalchemy-upsert-kit.readthedocs.io/en/latest/
|
|
52
|
+
|
|
53
|
+
``sqlalchemy_upsert_kit`` provides high-performance bulk upsert operations for SQLAlchemy applications using temporary table staging. It offers three optimized strategies: ``insert_or_ignore`` for conflict-free insertion, ``insert_or_replace`` for complete record replacement, and ``insert_or_merge`` for selective column updates while preserving existing data. Each strategy leverages efficient SQL JOIN operations and temporary tables to achieve ~20x performance improvements over traditional row-by-row upserts, making it ideal for ETL processes, data synchronization, and bulk data operations where speed and reliability are critical.
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
.. _install:
|
|
57
|
+
|
|
58
|
+
Install
|
|
59
|
+
------------------------------------------------------------------------------
|
|
60
|
+
|
|
61
|
+
``sqlalchemy_upsert_kit`` is released on PyPI, so all you need is to:
|
|
62
|
+
|
|
63
|
+
.. code-block:: console
|
|
64
|
+
|
|
65
|
+
$ pip install sqlalchemy-upsert-kit
|
|
66
|
+
|
|
67
|
+
To upgrade to latest version:
|
|
68
|
+
|
|
69
|
+
.. code-block:: console
|
|
70
|
+
|
|
71
|
+
$ pip install --upgrade sqlalchemy-upsert-kit
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
# ==============================================================================
|
|
2
|
+
# The [project] table defined by Official python.org
|
|
3
|
+
#
|
|
4
|
+
# Read: https://packaging.python.org/en/latest/guides/writing-pyproject-toml/
|
|
5
|
+
# ==============================================================================
|
|
6
|
+
[project]
|
|
7
|
+
name = "sqlalchemy_upsert_kit"
|
|
8
|
+
# Increment version before each release - follow `semantic versioning <https://semver.org/>`_
|
|
9
|
+
# Currently, poetry 2.1.X doesn't support dynamic versioning
|
|
10
|
+
# (Read https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#version)
|
|
11
|
+
# So this value has to be aligned with the one in ``sqlalchemy_upsert_kit/_version.py``
|
|
12
|
+
version = "0.1.1"
|
|
13
|
+
description = "High-performance SQLAlchemy bulk upsert operations using temporary table staging - 20x faster than row-by-row approaches."
|
|
14
|
+
# Read https://dev-exp-share.readthedocs.io/en/latest/search.html?q=Pick+An+Open+Source+License+For+Python+Project&check_keywords=yes&area=default
|
|
15
|
+
# To pick a license and update the ``license``, ``classifier`` field in ``pyproject.toml``
|
|
16
|
+
# And also update the ``LICENSE.txt`` file in the git repo.
|
|
17
|
+
license = "MIT"
|
|
18
|
+
license-files = ["LICENSE.txt", "AUTHORS.rst"]
|
|
19
|
+
authors = [
|
|
20
|
+
{ name = "Sanhe Hu", email = "husanhe@email.com" },
|
|
21
|
+
]
|
|
22
|
+
maintainers = [
|
|
23
|
+
{ name = "Sanhe Hu", email = "husanhe@email.com" },
|
|
24
|
+
]
|
|
25
|
+
keywords = []
|
|
26
|
+
readme = "README.rst"
|
|
27
|
+
requires-python = ">=3.9,<4.0"
|
|
28
|
+
# Full list of classifiers: https://pypi.org/classifiers/
|
|
29
|
+
classifier = [
|
|
30
|
+
"Development Status :: 4 - Beta",
|
|
31
|
+
"Intended Audience :: Developers",
|
|
32
|
+
"License :: OSI Approved :: MIT License",
|
|
33
|
+
"Natural Language :: English",
|
|
34
|
+
"Operating System :: Microsoft :: Windows",
|
|
35
|
+
"Operating System :: MacOS",
|
|
36
|
+
"Operating System :: Unix",
|
|
37
|
+
# Note: classfier like "Programming Language :: Python :: 3 :: Only",
|
|
38
|
+
# "Programming Language :: Python :: 3.X" is no longer needed,
|
|
39
|
+
# because we have ``requires-python`` field.
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
# ------------------------------------------------------------------------------
|
|
43
|
+
# Core Dependencies
|
|
44
|
+
# ------------------------------------------------------------------------------
|
|
45
|
+
dependencies = [
|
|
46
|
+
"sqlalchemy>=2.0.0,<3.0.0",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
# ------------------------------------------------------------------------------
|
|
50
|
+
# Optional dependency that can be used in ``pip install ${your_project_name}[${feature_name}]``
|
|
51
|
+
# Sometime this is also called "extras"
|
|
52
|
+
# Read: https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#dependencies-optional-dependencies
|
|
53
|
+
#
|
|
54
|
+
# IMPORTANT: all optional dependencies has to be compatible with the "requires-python" field
|
|
55
|
+
# ------------------------------------------------------------------------------
|
|
56
|
+
[project.optional-dependencies]
|
|
57
|
+
|
|
58
|
+
# ------------------------------------------------------------------------------
|
|
59
|
+
# Local Development dependenceies
|
|
60
|
+
# ------------------------------------------------------------------------------
|
|
61
|
+
dev = [
|
|
62
|
+
"twine>=6.0.0,<7.0.0", # distribute package to PyPI
|
|
63
|
+
"wheel>=0.45.0,<1.0.0", # build wheel package
|
|
64
|
+
"build>=1.2.1,<2.0.0", # build source distribution
|
|
65
|
+
"rich>=13.8.1,<14.0.0", # pretty print
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
# ------------------------------------------------------------------------------
|
|
69
|
+
# (Unit/Coverage/Integration/Load) Test dependenceies
|
|
70
|
+
# ------------------------------------------------------------------------------
|
|
71
|
+
test = [
|
|
72
|
+
"pytest>=8.2.2,<9.0.0", # Testing framework
|
|
73
|
+
"pytest-cov>=6.0.0,<7.0.0", # Coverage reporting
|
|
74
|
+
"prettytable>=3.0.0,<4.0.0", # Pretty print table in terminal
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
# ------------------------------------------------------------------------------
|
|
78
|
+
# Documentation build dependenceies
|
|
79
|
+
# ------------------------------------------------------------------------------
|
|
80
|
+
doc = [
|
|
81
|
+
"Sphinx>=7.4.7,<8.0.0",
|
|
82
|
+
"sphinx-copybutton>=0.5.2,<1.0.0", # add copy button to code block
|
|
83
|
+
"sphinx-design>=0.6.1,<1.0.0", # add additional design pattern to sphinx
|
|
84
|
+
"sphinx-jinja>=2.0.2,<3.0.0", # enable jinja syntax in reStructuredText
|
|
85
|
+
"furo==2024.8.6", # the furo sphinx theme
|
|
86
|
+
"pygments>=2.18.0,<3.0.0", # syntax highlight
|
|
87
|
+
"ipython>=8.18.1,<8.19.0", # interactive Python
|
|
88
|
+
"nbsphinx>=0.8.12,<1.0.0", # add jupyter notebook in sphinx doc
|
|
89
|
+
"rstobj==1.2.1", # generate reStructuredText from Python code
|
|
90
|
+
"docfly==3.0.0", # automaticall generate .. toctree directives and API reference doc
|
|
91
|
+
]
|
|
92
|
+
|
|
93
|
+
# ------------------------------------------------------------------------------
|
|
94
|
+
# Automation (devops) dependenceies
|
|
95
|
+
# ------------------------------------------------------------------------------
|
|
96
|
+
auto = [
|
|
97
|
+
]
|
|
98
|
+
|
|
99
|
+
# Quick Links
|
|
100
|
+
[project.urls]
|
|
101
|
+
Homepage = "https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project"
|
|
102
|
+
Documentation = "https://sqlalchemy-upsert-kit.readthedocs.io/en/latest/"
|
|
103
|
+
Repository = "https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project"
|
|
104
|
+
Issues = "https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project/issues"
|
|
105
|
+
Changelog = "https://github.com/MacHu-GWU/sqlalchemy_upsert_kit-project/blob/main/release-history.rst"
|
|
106
|
+
Download = "https://pypi.org/pypi/sqlalchemy-upsert-kit#files"
|
|
107
|
+
|
|
108
|
+
# For command line interface, read: https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#creating-executable-scripts
|
|
109
|
+
[project.scripts]
|
|
110
|
+
|
|
111
|
+
[tool.poetry.requires-plugins]
|
|
112
|
+
poetry-plugin-export = ">=1.9.0,<2.0.0"
|
|
113
|
+
|
|
114
|
+
[tool.poetry]
|
|
115
|
+
# Files to include in the package distribution
|
|
116
|
+
packages = [
|
|
117
|
+
{ include = "sqlalchemy_upsert_kit", from = ".", to = "." }
|
|
118
|
+
]
|
|
119
|
+
# Files to exclude from the package
|
|
120
|
+
exclude = [
|
|
121
|
+
"sqlalchemy_upsert_kit/tests/*",
|
|
122
|
+
"sqlalchemy_upsert_kit/docs/*",
|
|
123
|
+
"**/*.pyc",
|
|
124
|
+
"**/*.pyo",
|
|
125
|
+
]
|
|
126
|
+
|
|
127
|
+
# python workflow tool config
|
|
128
|
+
[tool.pywf]
|
|
129
|
+
# The specific python version you use for local development
|
|
130
|
+
dev_python = "3.11.8"
|
|
131
|
+
# --- github.com
|
|
132
|
+
github_account = "MacHu-GWU"
|
|
133
|
+
# Create GitHub token in https://github.com/settings/tokens and put the token in
|
|
134
|
+
# ``${HOME}/home_secret.json``
|
|
135
|
+
github_token_field = "providers.github.accounts.sh.users.sh.secrets.dev.value"
|
|
136
|
+
# --- codecov.io (for code coverage test report)
|
|
137
|
+
codecov_account = "MacHu-GWU"
|
|
138
|
+
# Create Codecov token in https://app.codecov.io/account/gh/${codecov_account}/access
|
|
139
|
+
# and put the token in ``${HOME}/home_secret.json``
|
|
140
|
+
codecov_token_field = "providers.codecov_io.accounts.sh.users.sh.secrets.dev.value"
|
|
141
|
+
# --- readthedocs.org (for documentation hosting)
|
|
142
|
+
# Create Readthedocs token in https://app.readthedocs.org/accounts/tokens/
|
|
143
|
+
# and put the token at ``${HOME}/home_secret.json``
|
|
144
|
+
readthedocs_token_field = "providers.readthedocs.accounts.sh.users.sh.secrets.dev.value"
|
|
145
|
+
# Readthedocs project name, usually it is the same as your project name
|
|
146
|
+
readthedocs_project_name = "sqlalchemy_upsert_kit"
|
|
147
|
+
|
|
148
|
+
# Read: https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#declaring-the-build-backend
|
|
149
|
+
[build-system]
|
|
150
|
+
requires = ["poetry-core>=2.0.0"]
|
|
151
|
+
build-backend = "poetry.core.masonry.api"
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
|
|
3
|
+
from ._version import __version__
|
|
4
|
+
from ._version import __short_description__
|
|
5
|
+
from ._version import __license__
|
|
6
|
+
from ._version import __author__
|
|
7
|
+
from ._version import __author_email__
|
|
8
|
+
from ._version import __maintainer__
|
|
9
|
+
from ._version import __maintainer_email__
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# This file is automatically generated by pywf script
|
|
3
|
+
# based on the content in pyproject.toml file
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.1"
|
|
6
|
+
__short_description__ = "High-performance SQLAlchemy bulk upsert operations using temporary table staging - 20x faster than row-by-row approaches."
|
|
7
|
+
__license__ = "MIT"
|
|
8
|
+
__author__ = "Sanhe Hu"
|
|
9
|
+
__author_email__ = "husanhe@email.com"
|
|
10
|
+
__maintainer__ = "Sanhe Hu"
|
|
11
|
+
__maintainer_email__ = "husanhe@email.com"
|
|
12
|
+
|
|
13
|
+
# run this script to print out the version number
|
|
14
|
+
if __name__ == "__main__": # pragma: no cover
|
|
15
|
+
print(__version__)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class UpsertTestError(Exception):
|
|
5
|
+
"""
|
|
6
|
+
Custom exception raised during testing to simulate failures.
|
|
7
|
+
|
|
8
|
+
This exception is used exclusively for testing error handling and cleanup
|
|
9
|
+
behavior in upsert operations. It allows tests to inject failures at specific
|
|
10
|
+
points in the operation flow to verify proper rollback and cleanup.
|
|
11
|
+
|
|
12
|
+
:param message: Descriptive error message indicating where the failure occurred
|
|
13
|
+
"""
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
dir_here = Path(__file__).absolute().parent
|
|
6
|
+
dir_package = dir_here
|
|
7
|
+
PACKAGE_NAME = dir_package.name
|
|
8
|
+
|
|
9
|
+
dir_project_root = dir_package.parent
|
|
10
|
+
|
|
11
|
+
# ------------------------------------------------------------------------------
|
|
12
|
+
# Virtual Environment Related
|
|
13
|
+
# ------------------------------------------------------------------------------
|
|
14
|
+
dir_venv = dir_project_root / ".venv"
|
|
15
|
+
dir_venv_bin = dir_venv / "bin"
|
|
16
|
+
|
|
17
|
+
# virtualenv executable paths
|
|
18
|
+
bin_pytest = dir_venv_bin / "pytest"
|
|
19
|
+
|
|
20
|
+
# ------------------------------------------------------------------------------
|
|
21
|
+
# Test Related
|
|
22
|
+
# ------------------------------------------------------------------------------
|
|
23
|
+
dir_htmlcov = dir_project_root / "htmlcov"
|
|
24
|
+
path_cov_index_html = dir_htmlcov / "index.html"
|
|
25
|
+
dir_unit_test = dir_project_root / "tests"
|
|
26
|
+
dir_int_test = dir_project_root / "tests_int"
|
|
27
|
+
dir_load_test = dir_project_root / "tests_load"
|
|
28
|
+
|
|
29
|
+
# ------------------------------------------------------------------------------
|
|
30
|
+
# Doc Related
|
|
31
|
+
# ------------------------------------------------------------------------------
|
|
32
|
+
dir_docs_source = dir_project_root / "docs" / "source"
|
|
33
|
+
dir_docs_build_html = dir_project_root / "docs" / "build" / "html"
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
High-Performance SQLAlchemy Bulk Operations Module for Sqlite
|
|
5
|
+
|
|
6
|
+
This module provides optimized bulk operations for SQLAlchemy using temporary tables.
|
|
7
|
+
These methods are specifically designed for large datasets and offer
|
|
8
|
+
superior performance compared to traditional row-by-row operations.
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
Implementation Notes
|
|
12
|
+
------------------------------------------------------------------------------
|
|
13
|
+
**Temporary Table Strategy**:
|
|
14
|
+
All bulk operations use temporary tables as staging areas to achieve optimal
|
|
15
|
+
performance. Temporary tables are created with unique names to avoid conflicts
|
|
16
|
+
in concurrent environments. Comprehensive cleanup ensures no temporary tables
|
|
17
|
+
are left behind, even when errors occur.
|
|
18
|
+
|
|
19
|
+
**SQLite DDL Behavior**:
|
|
20
|
+
SQLite DDL operations (CREATE/DROP TABLE) are not transactional and commit
|
|
21
|
+
immediately. The cleanup logic accounts for this by using fresh connections
|
|
22
|
+
when necessary to avoid database locks during error scenarios.
|
|
23
|
+
|
|
24
|
+
**Testing Infrastructure**:
|
|
25
|
+
Functions include boolean parameters prefixed with ``_raise_on_`` that are
|
|
26
|
+
exclusively for testing purposes. These parameters inject controlled failures
|
|
27
|
+
at specific points in the operation flow to verify error handling and cleanup
|
|
28
|
+
behavior. These parameters should never be used in production code.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from .insert_or_ignore import insert_or_ignore
|
|
32
|
+
from .insert_or_replace import insert_or_replace
|
|
33
|
+
|
|
34
|
+
__all__ = ["insert_or_ignore", "insert_or_replace"]
|