namidiff 0.6.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. namidiff-0.6.9/.gitignore +150 -0
  2. namidiff-0.6.9/LICENSE +106 -0
  3. namidiff-0.6.9/PKG-INFO +239 -0
  4. namidiff-0.6.9/README.md +158 -0
  5. namidiff-0.6.9/namidiff/__init__.py +219 -0
  6. namidiff-0.6.9/namidiff/__main__.py +459 -0
  7. namidiff-0.6.9/namidiff/config.py +116 -0
  8. namidiff-0.6.9/namidiff/databases/__init__.py +16 -0
  9. namidiff-0.6.9/namidiff/databases/_connect.py +55 -0
  10. namidiff-0.6.9/namidiff/databases/base.py +5 -0
  11. namidiff-0.6.9/namidiff/databases/bigquery.py +10 -0
  12. namidiff-0.6.9/namidiff/databases/clickhouse.py +10 -0
  13. namidiff-0.6.9/namidiff/databases/databricks.py +10 -0
  14. namidiff-0.6.9/namidiff/databases/duckdb.py +10 -0
  15. namidiff-0.6.9/namidiff/databases/mysql.py +10 -0
  16. namidiff-0.6.9/namidiff/databases/oracle.py +10 -0
  17. namidiff-0.6.9/namidiff/databases/postgresql.py +10 -0
  18. namidiff-0.6.9/namidiff/databases/presto.py +10 -0
  19. namidiff-0.6.9/namidiff/databases/redshift.py +10 -0
  20. namidiff-0.6.9/namidiff/databases/snowflake.py +10 -0
  21. namidiff-0.6.9/namidiff/databases/trino.py +10 -0
  22. namidiff-0.6.9/namidiff/databases/vertica.py +10 -0
  23. namidiff-0.6.9/namidiff/diff_tables.py +350 -0
  24. namidiff-0.6.9/namidiff/hashdiff_tables.py +301 -0
  25. namidiff-0.6.9/namidiff/info_tree.py +52 -0
  26. namidiff-0.6.9/namidiff/joindiff_tables.py +369 -0
  27. namidiff-0.6.9/namidiff/parse_time.py +74 -0
  28. namidiff-0.6.9/namidiff/query_utils.py +54 -0
  29. namidiff-0.6.9/namidiff/sqeleton/__init__.py +14 -0
  30. namidiff-0.6.9/namidiff/sqeleton/__main__.py +28 -0
  31. namidiff-0.6.9/namidiff/sqeleton/abcs/__init__.py +15 -0
  32. namidiff-0.6.9/namidiff/sqeleton/abcs/compiler.py +12 -0
  33. namidiff-0.6.9/namidiff/sqeleton/abcs/database_types.py +428 -0
  34. namidiff-0.6.9/namidiff/sqeleton/abcs/mixins.py +170 -0
  35. namidiff-0.6.9/namidiff/sqeleton/bound_exprs.py +81 -0
  36. namidiff-0.6.9/namidiff/sqeleton/conn_editor.py +292 -0
  37. namidiff-0.6.9/namidiff/sqeleton/databases/__init__.py +19 -0
  38. namidiff-0.6.9/namidiff/sqeleton/databases/_connect.py +276 -0
  39. namidiff-0.6.9/namidiff/sqeleton/databases/base.py +675 -0
  40. namidiff-0.6.9/namidiff/sqeleton/databases/bigquery.py +213 -0
  41. namidiff-0.6.9/namidiff/sqeleton/databases/clickhouse.py +197 -0
  42. namidiff-0.6.9/namidiff/sqeleton/databases/databricks.py +204 -0
  43. namidiff-0.6.9/namidiff/sqeleton/databases/dremio.py +207 -0
  44. namidiff-0.6.9/namidiff/sqeleton/databases/duckdb.py +193 -0
  45. namidiff-0.6.9/namidiff/sqeleton/databases/mssql.py +25 -0
  46. namidiff-0.6.9/namidiff/sqeleton/databases/mysql.py +157 -0
  47. namidiff-0.6.9/namidiff/sqeleton/databases/oracle.py +213 -0
  48. namidiff-0.6.9/namidiff/sqeleton/databases/postgresql.py +186 -0
  49. namidiff-0.6.9/namidiff/sqeleton/databases/presto.py +208 -0
  50. namidiff-0.6.9/namidiff/sqeleton/databases/redshift.py +128 -0
  51. namidiff-0.6.9/namidiff/sqeleton/databases/snowflake.py +246 -0
  52. namidiff-0.6.9/namidiff/sqeleton/databases/trino.py +71 -0
  53. namidiff-0.6.9/namidiff/sqeleton/databases/vertica.py +181 -0
  54. namidiff-0.6.9/namidiff/sqeleton/queries/__init__.py +25 -0
  55. namidiff-0.6.9/namidiff/sqeleton/queries/api.py +213 -0
  56. namidiff-0.6.9/namidiff/sqeleton/queries/ast_classes.py +1223 -0
  57. namidiff-0.6.9/namidiff/sqeleton/queries/base.py +27 -0
  58. namidiff-0.6.9/namidiff/sqeleton/queries/compiler.py +535 -0
  59. namidiff-0.6.9/namidiff/sqeleton/queries/extras.py +92 -0
  60. namidiff-0.6.9/namidiff/sqeleton/query_utils.py +54 -0
  61. namidiff-0.6.9/namidiff/sqeleton/repl.py +284 -0
  62. namidiff-0.6.9/namidiff/sqeleton/schema.py +73 -0
  63. namidiff-0.6.9/namidiff/sqeleton/utils.py +345 -0
  64. namidiff-0.6.9/namidiff/table_segment.py +378 -0
  65. namidiff-0.6.9/namidiff/thread_utils.py +107 -0
  66. namidiff-0.6.9/namidiff/utils.py +122 -0
  67. namidiff-0.6.9/pyproject.toml +138 -0
@@ -0,0 +1,150 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ pip-wheel-metadata/
24
+ share/python-wheels/
25
+ *.egg-info/
26
+ .installed.cfg
27
+ *.egg
28
+ MANIFEST
29
+
30
+ # PyInstaller
31
+ # Usually these files are written by a python script from a template
32
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
33
+ *.manifest
34
+ *.spec
35
+
36
+ # Installer logs
37
+ pip-log.txt
38
+ pip-delete-this-directory.txt
39
+
40
+ # Unit test / coverage reports
41
+ htmlcov/
42
+ .tox/
43
+ .nox/
44
+ .coverage
45
+ .coverage.*
46
+ .cache
47
+ nosetests.xml
48
+ coverage.xml
49
+ *.cover
50
+ *.py,cover
51
+ .hypothesis/
52
+ .pytest_cache/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ target/
76
+
77
+ # Jupyter Notebook
78
+ .ipynb_checkpoints
79
+
80
+ # IPython
81
+ profile_default/
82
+ ipython_config.py
83
+
84
+ # pyenv
85
+ .python-version
86
+
87
+ # pipenv
88
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
89
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
90
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
91
+ # install all needed dependencies.
92
+ #Pipfile.lock
93
+
94
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow
95
+ __pypackages__/
96
+
97
+ # Celery stuff
98
+ celerybeat-schedule
99
+ celerybeat.pid
100
+
101
+ # SageMath parsed files
102
+ *.sage.py
103
+
104
+ # Environments
105
+ .env
106
+ .venv
107
+ env/
108
+ venv/
109
+ ENV/
110
+ env.bak/
111
+ venv.bak/
112
+
113
+ # Spyder project settings
114
+ .spyderproject
115
+ .spyproject
116
+
117
+ # Rope project settings
118
+ .ropeproject
119
+
120
+ # mkdocs documentation
121
+ /site
122
+
123
+ # mypy
124
+ .mypy_cache/
125
+ .dmypy.json
126
+ dmypy.json
127
+
128
+ # Pyre type checker
129
+ .pyre/
130
+
131
+ # dev
132
+ ml-25m*
133
+ ratings*.csv
134
+ drive
135
+ mysqltuner.pl
136
+ benchmark_*.jsonl
137
+ benchmark_*.png
138
+
139
+ # Mac
140
+ .DS_Store
141
+
142
+ # IntelliJ
143
+ .idea
144
+
145
+ # VSCode
146
+ .vscode
147
+ .cecli*
148
+
149
+ # Local scratch copy of upstream sqeleton, not part of namidiff
150
+ sqeleton-project/
namidiff-0.6.9/LICENSE ADDED
@@ -0,0 +1,106 @@
1
+ namidiff is developed and maintained by NamiLink Kft.
2
+
3
+ It is a fork of reladiff (https://github.com/erezsh/reladiff) by Erez Shinan,
4
+ which is itself a fork of data-diff (https://github.com/datafold/data-diff) by
5
+ DataFold Inc.
6
+
7
+ The namidiff/sqeleton/ directory is a vendored fork of Sqeleton
8
+ (https://github.com/erezsh/sqeleton) by Erez Shinan.
9
+
10
+ All of these are released under the MIT License. The original copyright
11
+ notices are retained below, as required by the license.
12
+
13
+ --
14
+
15
+ MIT License
16
+
17
+ Copyright 2026 NamiLink Kft.
18
+
19
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
20
+ this software and associated documentation files (the "Software"), to deal in
21
+ the Software without restriction, including without limitation the rights to
22
+ use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
23
+ the Software, and to permit persons to whom the Software is furnished to do so,
24
+ subject to the following conditions:
25
+
26
+ The above copyright notice and this permission notice shall be included in all
27
+ copies or substantial portions of the Software.
28
+
29
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
30
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
31
+ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
32
+ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
33
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
34
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
35
+
36
+ --
37
+
38
+ Sqeleton (namidiff/sqeleton/)
39
+
40
+ MIT License
41
+
42
+ Copyright (c) 2023 Erez Shinan
43
+
44
+ Permission is hereby granted, free of charge, to any person obtaining a copy
45
+ of this software and associated documentation files (the "Software"), to deal
46
+ in the Software without restriction, including without limitation the rights
47
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
48
+ copies of the Software, and to permit persons to whom the Software is
49
+ furnished to do so, subject to the following conditions:
50
+
51
+ The above copyright notice and this permission notice shall be included in all
52
+ copies or substantial portions of the Software.
53
+
54
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
55
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
56
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
57
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
58
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
59
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
60
+ SOFTWARE.
61
+
62
+ --
63
+
64
+ Reladiff
65
+
66
+ Copyright 2024 Erez Shinnan
67
+
68
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
69
+ this software and associated documentation files (the "Software"), to deal in
70
+ the Software without restriction, including without limitation the rights to
71
+ use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
72
+ the Software, and to permit persons to whom the Software is furnished to do so,
73
+ subject to the following conditions:
74
+
75
+ The above copyright notice and this permission notice shall be included in all
76
+ copies or substantial portions of the Software.
77
+
78
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
79
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
80
+ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
81
+ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
82
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
83
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
84
+
85
+ --
86
+
87
+ data-diff
88
+
89
+ Copyright 2022 DataFold Inc.
90
+
91
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
92
+ this software and associated documentation files (the "Software"), to deal in
93
+ the Software without restriction, including without limitation the rights to
94
+ use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
95
+ the Software, and to permit persons to whom the Software is furnished to do so,
96
+ subject to the following conditions:
97
+
98
+ The above copyright notice and this permission notice shall be included in all
99
+ copies or substantial portions of the Software.
100
+
101
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
102
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
103
+ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
104
+ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
105
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
106
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,239 @@
1
+ Metadata-Version: 2.5
2
+ Name: namidiff
3
+ Version: 0.6.9
4
+ Summary: Efficiently diff rows across two different databases. Fork of reladiff (itself a fork of data-diff). Python package and CLI are both `namidiff`.
5
+ Project-URL: Homepage, https://github.com/NamiLinkLabs/namidiff
6
+ Project-URL: Repository, https://github.com/NamiLinkLabs/namidiff
7
+ Project-URL: Documentation, https://github.com/NamiLinkLabs/namidiff/tree/main/docs
8
+ Project-URL: Bug Tracker, https://github.com/NamiLinkLabs/namidiff/issues
9
+ Project-URL: Upstream (reladiff), https://github.com/erezsh/reladiff
10
+ Project-URL: Upstream (sqeleton), https://github.com/erezsh/sqeleton
11
+ Author-email: "NamiLink Kft." <mate@namilink.com>, Erez Shinan <erezshin@gmail.com>
12
+ Maintainer-email: "NamiLink Kft." <mate@namilink.com>
13
+ License: MIT
14
+ License-File: LICENSE
15
+ Classifier: Development Status :: 4 - Beta
16
+ Classifier: Environment :: Console
17
+ Classifier: Intended Audience :: Developers
18
+ Classifier: Intended Audience :: Information Technology
19
+ Classifier: Intended Audience :: System Administrators
20
+ Classifier: License :: OSI Approved :: MIT License
21
+ Classifier: Programming Language :: Python :: 3
22
+ Classifier: Programming Language :: Python :: 3.8
23
+ Classifier: Programming Language :: Python :: 3.9
24
+ Classifier: Programming Language :: Python :: 3.10
25
+ Classifier: Programming Language :: Python :: 3.11
26
+ Classifier: Programming Language :: Python :: 3.12
27
+ Classifier: Programming Language :: Python :: 3.13
28
+ Classifier: Programming Language :: Python :: 3.14
29
+ Classifier: Topic :: Database :: Database Engines/Servers
30
+ Classifier: Typing :: Typed
31
+ Requires-Python: >=3.8
32
+ Requires-Dist: click>=8.1
33
+ Requires-Dist: dsnparse
34
+ Requires-Dist: rich
35
+ Requires-Dist: runtype>=0.5.0
36
+ Requires-Dist: toml>=0.10.2
37
+ Provides-Extra: all
38
+ Requires-Dist: clickhouse-driver; extra == 'all'
39
+ Requires-Dist: cryptography; extra == 'all'
40
+ Requires-Dist: cx-oracle; extra == 'all'
41
+ Requires-Dist: duckdb>=0.7.0; extra == 'all'
42
+ Requires-Dist: mysql-connector-python>=8.0.29; extra == 'all'
43
+ Requires-Dist: presto-python-client; extra == 'all'
44
+ Requires-Dist: psycopg2-binary; extra == 'all'
45
+ Requires-Dist: snowflake-connector-python>=2.7.2; extra == 'all'
46
+ Requires-Dist: trino>=0.314.0; extra == 'all'
47
+ Requires-Dist: vertica-python; extra == 'all'
48
+ Provides-Extra: clickhouse
49
+ Requires-Dist: clickhouse-driver; extra == 'clickhouse'
50
+ Provides-Extra: dremio
51
+ Requires-Dist: numpy>=1.24.4; (python_version < '3.9') and extra == 'dremio'
52
+ Requires-Dist: numpy>=1.26.4; (python_version >= '3.9') and extra == 'dremio'
53
+ Requires-Dist: pandas; (python_version < '3.9') and extra == 'dremio'
54
+ Requires-Dist: pandas>=2.1; (python_version >= '3.9') and extra == 'dremio'
55
+ Requires-Dist: pyarrow<18.0.0; (python_version < '3.9') and extra == 'dremio'
56
+ Requires-Dist: pyarrow>=18.0.0; (python_version >= '3.9') and extra == 'dremio'
57
+ Requires-Dist: sqlalchemy-dremio; extra == 'dremio'
58
+ Provides-Extra: duckdb
59
+ Requires-Dist: duckdb>=0.7.0; extra == 'duckdb'
60
+ Provides-Extra: mysql
61
+ Requires-Dist: mysql-connector-python>=8.0.29; extra == 'mysql'
62
+ Provides-Extra: oracle
63
+ Requires-Dist: cx-oracle; extra == 'oracle'
64
+ Provides-Extra: postgresql
65
+ Requires-Dist: psycopg2-binary; extra == 'postgresql'
66
+ Provides-Extra: presto
67
+ Requires-Dist: presto-python-client; extra == 'presto'
68
+ Provides-Extra: snowflake
69
+ Requires-Dist: cryptography; extra == 'snowflake'
70
+ Requires-Dist: snowflake-connector-python>=2.7.2; extra == 'snowflake'
71
+ Provides-Extra: trino
72
+ Requires-Dist: trino>=0.314.0; extra == 'trino'
73
+ Provides-Extra: tui
74
+ Requires-Dist: prompt-toolkit>=3.0.36; extra == 'tui'
75
+ Requires-Dist: pygments>=2.13.0; extra == 'tui'
76
+ Requires-Dist: textual-select; extra == 'tui'
77
+ Requires-Dist: textual>=0.9.1; extra == 'tui'
78
+ Provides-Extra: vertica
79
+ Requires-Dist: vertica-python; extra == 'vertica'
80
+ Description-Content-Type: text/markdown
81
+
82
+ ![](reladiff_logo.svg)
83
+
84
+ &nbsp;
85
+ <br/>
86
+ <br/>
87
+ <span style="font-size:1.3em">**Namidiff**</span> is a fork of **Reladiff**, a high-performance tool and library designed for diffing large datasets across databases. By executing the diff calculation within the database itself, Namidiff minimizes data transfer and achieves optimal performance.
88
+
89
+ > **Lineage:** `namidiff` (NamiLink Kft.) is a fork of [reladiff](https://github.com/erezsh/reladiff) by Erez Shinan, which is a fork of the archived [data-diff](https://github.com/datafold/data-diff) by DataFold Inc. The PyPI distribution, Python package and CLI command are all named `namidiff`. The database layer, `namidiff.sqeleton`, is a vendored fork of [sqeleton](https://github.com/erezsh/sqeleton) by Erez Shinan. See [LICENSE](LICENSE) for the full copyright chain.
90
+
91
+ This tool is specifically tailored for data professionals, DevOps engineers, and system administrators.
92
+
93
+ Namidiff is free, open-source, user-friendly, extensively tested, and delivers fast results, even at massive scale.
94
+
95
+ ### Key Features:
96
+
97
+ 1. **Cross-Database Diff**: Namidiff employs a divide-and-conquer algorithm, based on matching hashes, to efficiently identify modified segments and download only the necessary data for comparison. This approach ensures exceptional performance when differences are minimal.
98
+
99
+ - ⇄ Diffs across over a dozen different databases (e.g. *PostgreSQL* -> *Snowflake*) !
100
+
101
+ - 🧠 Gracefully handles reduced precision (e.g., timestamp(9) -> timestamp(3)) by rounding according to the database specification.
102
+
103
+ - πŸ”₯ Benchmarked to diff over 25M rows in under 10 seconds and over 1B rows in approximately 5 minutes, given no differences.
104
+
105
+ - ♾️ Capable of handling tables with tens of billions of rows.
106
+
107
+
108
+ 2. **Intra-Database Diff**: When both tables reside in the same database, Namidiff compares them using a join operation, with additional optimizations for enhanced speed.
109
+
110
+ - Supports materializing the diff into a local table.
111
+ - Can collect various extra statistics about the tables.
112
+
113
+ 3. **Threaded**: Utilizes multiple threads to significantly boost performance during diffing operations.
114
+
115
+ 3. **Configurable**: Offers numerous options for power-users to customize and optimize their usage.
116
+
117
+ 4. **Automation-Friendly**: Outputs both JSON and git-like diffs (with + and -), facilitating easy integration into CI/CD pipelines.
118
+
119
+ 5. **Over a dozen databases supported**. MySQL, Postgres, Snowflake, Bigquery, Oracle, Clickhouse, and more. [See full list](docs/supported-databases.md)
120
+
121
+
122
+ Reladiff is a fork of an archived project called [data-diff](https://github.com/datafold/data-diff). Namidiff continues that lineage, with NamiLink-specific fixes for cross-database normalization (e.g. empty-string / NULL handling, timestamp precision).
123
+
124
+ ## Get Started
125
+
126
+ [**πŸ—Ž Read the Documentation**](docs/index.rst) - the `docs/` folder has everything you need to start diffing. (Upstream reladiff docs are also hosted at [reladiff.readthedocs.io](https://reladiff.readthedocs.io/en/latest/).)
127
+
128
+ ## Quickstart
129
+
130
+ For the impatient ;)
131
+
132
+ ### Install
133
+
134
+ Namidiff is available on [PyPI](https://pypi.org/project/namidiff/) as **`namidiff`** (the NamiLink Kft. fork). You may install it by running:
135
+
136
+ ```
137
+ pip install namidiff
138
+ ```
139
+
140
+ Requires Python 3.8+ with pip.
141
+
142
+ We advise to install it within a virtual-env.
143
+
144
+ ### How to Use
145
+
146
+ Once you've installed Namidiff, you can run it from the command-line:
147
+
148
+ ```bash
149
+ # Cross-DB diff, using hashes
150
+ namidiff DB1_URI TABLE1_NAME DB2_URI TABLE2_NAME [OPTIONS]
151
+ ```
152
+
153
+ When both tables belong to the same database, a shorter syntax is available:
154
+
155
+ ```bash
156
+ # Same-DB diff, using outer join
157
+ namidiff DB1_URI TABLE1_NAME TABLE2_NAME [OPTIONS]
158
+ ```
159
+
160
+ Or, you can import and run it from Python:
161
+
162
+ ```python
163
+ from namidiff import connect_to_table, diff_tables
164
+
165
+ table1 = connect_to_table("postgresql:///", "table_name", "id")
166
+ table2 = connect_to_table("mysql:///", "table_name", "id")
167
+
168
+ sign: Literal['+' | '-']
169
+ row: tuple[str, ...]
170
+ for sign, row in diff_tables(table1, table2):
171
+ print(sign, row)
172
+ ```
173
+
174
+ Read our detailed instructions:
175
+
176
+ * [How to use from the shell / command-line](docs/how-to-use.md#how-to-use-from-the-shell-or-command-line)
177
+ * [How to use with TOML configuration file](docs/how-to-use.md#how-to-use-with-a-configuration-file)
178
+ * [How to use from Python](docs/how-to-use.md#how-to-use-from-python)
179
+
180
+
181
+ #### "Real-world" example: Diff "events" table between Postgres and Snowflake
182
+
183
+ ```
184
+ namidiff \
185
+ postgresql:/// \
186
+ events \
187
+ "snowflake://<username>:<password>@<host>/<DATABASE>/<SCHEMA>?warehouse=<WAREHOUSE>&role=<ROLE>" \
188
+ events \
189
+ -k event_id \ # Identifier of event
190
+ -c event_data \ # Extra column to compare
191
+ -w "event_time < '2024-10-10'" # Filter the rows on both dbs
192
+ ```
193
+
194
+ #### "Real-world" example: Diff "events" and "old_events" tables in the same Postgres DB
195
+
196
+ Materializes the results into a new table, containing the current timestamp in its name.
197
+
198
+ ```
199
+ namidiff \
200
+ postgresql:/// events old_events \
201
+ -k org_id \
202
+ -c created_at -c is_internal \
203
+ -w "org_id != 1 and org_id < 2000" \
204
+ -m test_results_%t \
205
+ --materialize-all-rows \
206
+ --table-write-limit 10000
207
+ ```
208
+
209
+ ### Technical Explanation
210
+
211
+ This [technical explanation](docs/technical-explanation.md) provides a quick overview of how our cross-database diffing works.
212
+
213
+ For an in-depth explanation, this [blog post](https://eshsoft.com/blog/how-reladiff-works) details all the major technical choices we made in our implementation.
214
+
215
+
216
+ ### We're here to help!
217
+
218
+ * Confused? Got a cool idea? Just want to share your thoughts? Let's discuss it in [GitHub Discussions](https://github.com/NamiLinkLabs/namidiff/discussions).
219
+
220
+ * Did you encounter a bug? [Open an issue](https://github.com/NamiLinkLabs/namidiff/issues).
221
+
222
+ ## How to Contribute
223
+ * Please read the [contributing guidelines](CONTRIBUTING.md) to get started.
224
+ * Feel free to open a new issue or work on an existing one.
225
+
226
+ Big thanks to everyone who contributed so far:
227
+
228
+ <a href="https://github.com/NamiLinkLabs/namidiff/graphs/contributors">
229
+ <img src="https://contributors-img.web.app/image?repo=NamiLinkLabs/namidiff" />
230
+ </a>
231
+
232
+ And to the upstream [reladiff](https://github.com/erezsh/reladiff) and [data-diff](https://github.com/datafold/data-diff) contributors, whose work this fork builds on.
233
+
234
+
235
+ ## License
236
+
237
+ This project is licensed under the terms of the [MIT License](LICENSE).
238
+
239
+ Copyright 2026 NamiLink Kft. Contains code copyright Erez Shinan (reladiff, sqeleton) and DataFold Inc. (data-diff), also under MIT. The [LICENSE](LICENSE) file retains all original notices.
@@ -0,0 +1,158 @@
1
+ ![](reladiff_logo.svg)
2
+
3
+ &nbsp;
4
+ <br/>
5
+ <br/>
6
+ <span style="font-size:1.3em">**Namidiff**</span> is a fork of **Reladiff**, a high-performance tool and library designed for diffing large datasets across databases. By executing the diff calculation within the database itself, Namidiff minimizes data transfer and achieves optimal performance.
7
+
8
+ > **Lineage:** `namidiff` (NamiLink Kft.) is a fork of [reladiff](https://github.com/erezsh/reladiff) by Erez Shinan, which is a fork of the archived [data-diff](https://github.com/datafold/data-diff) by DataFold Inc. The PyPI distribution, Python package and CLI command are all named `namidiff`. The database layer, `namidiff.sqeleton`, is a vendored fork of [sqeleton](https://github.com/erezsh/sqeleton) by Erez Shinan. See [LICENSE](LICENSE) for the full copyright chain.
9
+
10
+ This tool is specifically tailored for data professionals, DevOps engineers, and system administrators.
11
+
12
+ Namidiff is free, open-source, user-friendly, extensively tested, and delivers fast results, even at massive scale.
13
+
14
+ ### Key Features:
15
+
16
+ 1. **Cross-Database Diff**: Namidiff employs a divide-and-conquer algorithm, based on matching hashes, to efficiently identify modified segments and download only the necessary data for comparison. This approach ensures exceptional performance when differences are minimal.
17
+
18
+ - ⇄ Diffs across over a dozen different databases (e.g. *PostgreSQL* -> *Snowflake*) !
19
+
20
+ - 🧠 Gracefully handles reduced precision (e.g., timestamp(9) -> timestamp(3)) by rounding according to the database specification.
21
+
22
+ - πŸ”₯ Benchmarked to diff over 25M rows in under 10 seconds and over 1B rows in approximately 5 minutes, given no differences.
23
+
24
+ - ♾️ Capable of handling tables with tens of billions of rows.
25
+
26
+
27
+ 2. **Intra-Database Diff**: When both tables reside in the same database, Namidiff compares them using a join operation, with additional optimizations for enhanced speed.
28
+
29
+ - Supports materializing the diff into a local table.
30
+ - Can collect various extra statistics about the tables.
31
+
32
+ 3. **Threaded**: Utilizes multiple threads to significantly boost performance during diffing operations.
33
+
34
+ 3. **Configurable**: Offers numerous options for power-users to customize and optimize their usage.
35
+
36
+ 4. **Automation-Friendly**: Outputs both JSON and git-like diffs (with + and -), facilitating easy integration into CI/CD pipelines.
37
+
38
+ 5. **Over a dozen databases supported**. MySQL, Postgres, Snowflake, Bigquery, Oracle, Clickhouse, and more. [See full list](docs/supported-databases.md)
39
+
40
+
41
+ Reladiff is a fork of an archived project called [data-diff](https://github.com/datafold/data-diff). Namidiff continues that lineage, with NamiLink-specific fixes for cross-database normalization (e.g. empty-string / NULL handling, timestamp precision).
42
+
43
+ ## Get Started
44
+
45
+ [**πŸ—Ž Read the Documentation**](docs/index.rst) - the `docs/` folder has everything you need to start diffing. (Upstream reladiff docs are also hosted at [reladiff.readthedocs.io](https://reladiff.readthedocs.io/en/latest/).)
46
+
47
+ ## Quickstart
48
+
49
+ For the impatient ;)
50
+
51
+ ### Install
52
+
53
+ Namidiff is available on [PyPI](https://pypi.org/project/namidiff/) as **`namidiff`** (the NamiLink Kft. fork). You may install it by running:
54
+
55
+ ```
56
+ pip install namidiff
57
+ ```
58
+
59
+ Requires Python 3.8+ with pip.
60
+
61
+ We advise to install it within a virtual-env.
62
+
63
+ ### How to Use
64
+
65
+ Once you've installed Namidiff, you can run it from the command-line:
66
+
67
+ ```bash
68
+ # Cross-DB diff, using hashes
69
+ namidiff DB1_URI TABLE1_NAME DB2_URI TABLE2_NAME [OPTIONS]
70
+ ```
71
+
72
+ When both tables belong to the same database, a shorter syntax is available:
73
+
74
+ ```bash
75
+ # Same-DB diff, using outer join
76
+ namidiff DB1_URI TABLE1_NAME TABLE2_NAME [OPTIONS]
77
+ ```
78
+
79
+ Or, you can import and run it from Python:
80
+
81
+ ```python
82
+ from namidiff import connect_to_table, diff_tables
83
+
84
+ table1 = connect_to_table("postgresql:///", "table_name", "id")
85
+ table2 = connect_to_table("mysql:///", "table_name", "id")
86
+
87
+ sign: Literal['+' | '-']
88
+ row: tuple[str, ...]
89
+ for sign, row in diff_tables(table1, table2):
90
+ print(sign, row)
91
+ ```
92
+
93
+ Read our detailed instructions:
94
+
95
+ * [How to use from the shell / command-line](docs/how-to-use.md#how-to-use-from-the-shell-or-command-line)
96
+ * [How to use with TOML configuration file](docs/how-to-use.md#how-to-use-with-a-configuration-file)
97
+ * [How to use from Python](docs/how-to-use.md#how-to-use-from-python)
98
+
99
+
100
+ #### "Real-world" example: Diff "events" table between Postgres and Snowflake
101
+
102
+ ```
103
+ namidiff \
104
+ postgresql:/// \
105
+ events \
106
+ "snowflake://<username>:<password>@<host>/<DATABASE>/<SCHEMA>?warehouse=<WAREHOUSE>&role=<ROLE>" \
107
+ events \
108
+ -k event_id \ # Identifier of event
109
+ -c event_data \ # Extra column to compare
110
+ -w "event_time < '2024-10-10'" # Filter the rows on both dbs
111
+ ```
112
+
113
+ #### "Real-world" example: Diff "events" and "old_events" tables in the same Postgres DB
114
+
115
+ Materializes the results into a new table, containing the current timestamp in its name.
116
+
117
+ ```
118
+ namidiff \
119
+ postgresql:/// events old_events \
120
+ -k org_id \
121
+ -c created_at -c is_internal \
122
+ -w "org_id != 1 and org_id < 2000" \
123
+ -m test_results_%t \
124
+ --materialize-all-rows \
125
+ --table-write-limit 10000
126
+ ```
127
+
128
+ ### Technical Explanation
129
+
130
+ This [technical explanation](docs/technical-explanation.md) provides a quick overview of how our cross-database diffing works.
131
+
132
+ For an in-depth explanation, this [blog post](https://eshsoft.com/blog/how-reladiff-works) details all the major technical choices we made in our implementation.
133
+
134
+
135
+ ### We're here to help!
136
+
137
+ * Confused? Got a cool idea? Just want to share your thoughts? Let's discuss it in [GitHub Discussions](https://github.com/NamiLinkLabs/namidiff/discussions).
138
+
139
+ * Did you encounter a bug? [Open an issue](https://github.com/NamiLinkLabs/namidiff/issues).
140
+
141
+ ## How to Contribute
142
+ * Please read the [contributing guidelines](CONTRIBUTING.md) to get started.
143
+ * Feel free to open a new issue or work on an existing one.
144
+
145
+ Big thanks to everyone who contributed so far:
146
+
147
+ <a href="https://github.com/NamiLinkLabs/namidiff/graphs/contributors">
148
+ <img src="https://contributors-img.web.app/image?repo=NamiLinkLabs/namidiff" />
149
+ </a>
150
+
151
+ And to the upstream [reladiff](https://github.com/erezsh/reladiff) and [data-diff](https://github.com/datafold/data-diff) contributors, whose work this fork builds on.
152
+
153
+
154
+ ## License
155
+
156
+ This project is licensed under the terms of the [MIT License](LICENSE).
157
+
158
+ Copyright 2026 NamiLink Kft. Contains code copyright Erez Shinan (reladiff, sqeleton) and DataFold Inc. (data-diff), also under MIT. The [LICENSE](LICENSE) file retains all original notices.