compare-schema 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. compare_schema-0.1.1/LICENSE +21 -0
  2. compare_schema-0.1.1/PKG-INFO +206 -0
  3. compare_schema-0.1.1/README.md +171 -0
  4. compare_schema-0.1.1/compare_schema.egg-info/PKG-INFO +206 -0
  5. compare_schema-0.1.1/compare_schema.egg-info/SOURCES.txt +30 -0
  6. compare_schema-0.1.1/compare_schema.egg-info/dependency_links.txt +1 -0
  7. compare_schema-0.1.1/compare_schema.egg-info/entry_points.txt +2 -0
  8. compare_schema-0.1.1/compare_schema.egg-info/requires.txt +21 -0
  9. compare_schema-0.1.1/compare_schema.egg-info/top_level.txt +2 -0
  10. compare_schema-0.1.1/db_client/queries/schema/data.yml +142 -0
  11. compare_schema-0.1.1/db_client/queries/schema/schema.yml +591 -0
  12. compare_schema-0.1.1/db_client/queries/source/import_data.yml +6 -0
  13. compare_schema-0.1.1/db_client/queries/source/import_schema.yml +53 -0
  14. compare_schema-0.1.1/db_client/schema.sql +56 -0
  15. compare_schema-0.1.1/db_client/schema_client.py +15 -0
  16. compare_schema-0.1.1/db_client/schema_settings.py +33 -0
  17. compare_schema-0.1.1/db_client/source_client.py +25 -0
  18. compare_schema-0.1.1/db_client/source_settings.py +36 -0
  19. compare_schema-0.1.1/pyproject.toml +90 -0
  20. compare_schema-0.1.1/setup.cfg +4 -0
  21. compare_schema-0.1.1/src/config.py +49 -0
  22. compare_schema-0.1.1/src/env_config.py +17 -0
  23. compare_schema-0.1.1/src/excel_helper.py +91 -0
  24. compare_schema-0.1.1/src/main.py +1344 -0
  25. compare_schema-0.1.1/src/markdown_helper.py +101 -0
  26. compare_schema-0.1.1/src/models.py +254 -0
  27. compare_schema-0.1.1/src/util_mysql.py +404 -0
  28. compare_schema-0.1.1/src/util_other.py +25 -0
  29. compare_schema-0.1.1/src/util_path.py +128 -0
  30. compare_schema-0.1.1/tests/test_config.py +36 -0
  31. compare_schema-0.1.1/tests/test_markdown.py +25 -0
  32. compare_schema-0.1.1/tests/test_models.py +86 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 doctorgu
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,206 @@
1
+ Metadata-Version: 2.4
2
+ Name: compare-schema
3
+ Version: 0.1.1
4
+ Summary: Compare schema and data between databases
5
+ Author-email: Gu Park <doctorgu@kakao.com>
6
+ License-Expression: MIT
7
+ Project-URL: homepage, https://github.com/doctorgu/compare_schema
8
+ Project-URL: repository, https://github.com/doctorgu/compare_schema
9
+ Project-URL: documentation, https://github.com/doctorgu/compare_schema
10
+ Keywords: compare schema,database comparison,schema diff,mysql,sqlite
11
+ Requires-Python: >=3.13
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ Requires-Dist: sqlite3-client>=1.2.0
15
+ Requires-Dist: mysqlclient-client>=1.2.0
16
+ Requires-Dist: pydantic>=2.0
17
+ Requires-Dist: pydantic-settings>=2.0
18
+ Requires-Dist: xlsxwriter>=3.0
19
+ Requires-Dist: tabulate>=0.9
20
+ Requires-Dist: types-tabulate>=0.10
21
+ Requires-Dist: requests>=2.28
22
+ Requires-Dist: PyYAML>=6.0
23
+ Requires-Dist: SQLAlchemy>=2.0
24
+ Requires-Dist: PyMySQL>=1.0
25
+ Requires-Dist: python-dotenv>=1.0.0
26
+ Requires-Dist: ruff
27
+ Provides-Extra: test
28
+ Requires-Dist: pytest; extra == "test"
29
+ Requires-Dist: pytest-cov; extra == "test"
30
+ Requires-Dist: pytest-env; extra == "test"
31
+ Requires-Dist: pytest-mock; extra == "test"
32
+ Requires-Dist: pytest-asyncio; extra == "test"
33
+ Requires-Dist: build; extra == "test"
34
+ Dynamic: license-file
35
+
36
+ # compare-schema — Database Schema and Data Comparison Tool
37
+
38
+ [English](README.md) | [한국어](README_KO.md)
39
+
40
+ A database schema and data comparison tool across different environments (e.g. dev, stg, prd).
41
+
42
+ > [!NOTE]
43
+ > Currently, only **MySQL** schema and data are supported. The comparison metadata and version history are stored locally in an embedded SQLite database.
44
+
45
+ ## Features
46
+
47
+ - **Schema Comparison**: Compare tables, views, columns, indices, and foreign key references across databases.
48
+ - **Data Comparison**: Compare key-based data records across environments.
49
+ - **Database History Tracking**: Track changes to the same database over time by comparing versions.
50
+ - **Multiple Output Formats**: Export comparison reports to Markdown (`.md`) or Excel (`.xlsx`).
51
+ - **History & Versioning**: Tracks versions in SQLite to detect schema drift over time.
52
+ - **Notifications**: Discord webhook notification support for schema and data changes.
53
+ - **CLI & Module Support**: Run via `compare-schema` CLI command or `python -m src.main`.
54
+
55
+ ## Installation
56
+
57
+ ```bash
58
+ pip install compare-schema
59
+ ```
60
+
61
+ ## Quick Start
62
+
63
+ ### 1. Configuration (`config/config.yaml`)
64
+
65
+ Create a configuration YAML file:
66
+
67
+ ```yaml
68
+ output_type: md # md or xlsx
69
+ output_dir: ./compare_schema/result
70
+ prev_version: ""
71
+
72
+ data_tables:
73
+ - table: users
74
+ columns:
75
+ - name
76
+ - email
77
+ - nick_name
78
+ key_columns:
79
+ - id
80
+
81
+ exclude_tables: []
82
+ exclude_columns: []
83
+ db_host: 127.0.0.1
84
+ envs:
85
+ - name: dev
86
+ port: 3306
87
+ db_names: [shop, logs]
88
+ - name: stg
89
+ port: 3306
90
+ db_names: [shop]
91
+ - name: prd
92
+ port: 3306
93
+ db_names: [shop]
94
+
95
+ compare_list:
96
+ # Compare between different databases or environments
97
+ - a: dev.shop
98
+ b: dev.logs
99
+ - a: dev.shop
100
+ b: stg.shop
101
+ - a: stg.shop
102
+ b: prd.shop
103
+ # If a and b is same, compares between recent version and new version (shows history of that DB)
104
+ - a: dev.shop
105
+ b: dev.shop
106
+ - a: stg.shop
107
+ b: stg.shop
108
+ - a: prd.shop
109
+ b: prd.shop
110
+ ```
111
+
112
+ ### 2. Version Comparison & History
113
+
114
+ The tool supports two comparison modes based on the entries defined in `compare_list` (`{env}.{database}`):
115
+
116
+ #### A. Cross-Database Comparison (`a != b`)
117
+
118
+ Compares schemas and data between two distinct environments or databases (e.g., `dev.shop` vs `stg.shop`):
119
+
120
+ - **Schema Differences**:
121
+ - `schema_not_exists`: Identifies missing tables, views, columns, indices, and foreign keys present in one database but absent in the other.
122
+ - `schema_diff`: Identifies structural discrepancies in matching objects:
123
+ - **Columns**: Data types, nullability (`is_nullable`), default values, character length, numeric precision/scale, and column comments.
124
+ - **Indices**: Index types (BTREE, etc.), uniqueness (`is_unique`), and indexed column combinations.
125
+ - **Foreign Keys**: Target referenced tables and columns.
126
+ - **Data Differences**:
127
+ - Compares rows in tables specified in `data_tables`.
128
+ - Matches rows between databases using the primary/unique keys configured in `key_columns`.
129
+ - `data_not_exists`: Rows present in one database but missing in the other.
130
+ - `data_diff`: Rows that exist in both databases but have differing values in the monitored `columns`.
131
+ - **Output**: Exported to `{version}_compare.md` (Markdown) or the `compare` sheets in Excel.
132
+
133
+ #### B. Database History & Drift Tracking (`a == b`)
134
+
135
+ When `a` and `b` reference the same database (e.g., `a: dev.shop, b: dev.shop`), the tool compares the new `version` (timestamped as `YYYYMMDDHHmm`) against `prev_version`:
136
+
137
+ - **Change Categories**:
138
+ - `schema_added` / `data_added`: Newly created tables, views, columns, indices, foreign keys, or inserted rows.
139
+ - `schema_removed` / `data_removed`: Dropped schema objects or deleted rows.
140
+ - `schema_changed` / `data_changed`: Modified column types/properties or updated data row values over time.
141
+ - **Output**: Exported to `{version}_log.md` (Markdown) or the `log` sheets in Excel.
142
+
143
+ ---
144
+
145
+ ### Configuration Reference (`config.yaml`)
146
+
147
+ | Parameter | Type | Default | Description |
148
+ | :--- | :--- | :--- | :--- |
149
+ | `output_type` | `Literal["xlsx", "md"]` | Required | Output format. `"md"` generates Markdown files; `"xlsx"` generates an Excel workbook. |
150
+ | `output_dir` | `str` | Required | Directory path where output reports are saved. |
151
+ | `prev_version` | `str` | `""` | Previous version (`YYYYMMDDHHmm`) to compare against. If empty (`""`), automatically uses the latest version recorded in SQLite. |
152
+ | `process_when_changed` | `bool` | Required | If `true`, stops execution without generating reports if no schema or data changes are detected against `prev_version`. If `false`, generates reports on every run. |
153
+ | `data_tables` | `list[DataTableConfig]` | `[]` | List of tables and columns to compare data for. |
154
+ | `data_tables[].table` | `str` | Required | Table name in MySQL. |
155
+ | `data_tables[].columns` | `list[str]` | Required | Columns to compare for value differences. |
156
+ | `data_tables[].key_columns` | `list[str]` | Required | Key columns (e.g., primary key) used to match corresponding rows across databases. |
157
+ | `exclude_tables` | `list[str]` | Required | Table names to exclude from cross-environment schema comparison (`a != b`), such as migration tables (e.g., `["alembic_version"]`). *(Not excluded during same-database history tracking `a == b`)*. |
158
+ | `exclude_columns` | `list[str]` | Required | Column names to ignore during cross-environment column comparisons. |
159
+ | `db_host` | `str` | Required | Hostname or IP address for MySQL database connections. |
160
+ | `envs` | `list[EnvConfig]` | Required | List of environments and databases to connect to and extract. |
161
+ | `envs[].name` | `str` | Required | Environment identifier (e.g., `dev`, `stg`, `prd`). |
162
+ | `envs[].port` | `int` | Required | MySQL port number for this environment. |
163
+ | `envs[].db_names` | `list[str]` | Required | List of database names to extract from this environment. |
164
+ | `compare_list` | `list[EnvCompareConfig]` | Required | Target database pairs to compare (e.g., `a: dev.shop`, `b: stg.shop`). |
165
+
166
+ ---
167
+
168
+ ### How It Works
169
+
170
+ 1. **Extraction & Snapshotting**:
171
+ - Connects to MySQL instances defined in `envs` using credentials from `.env`.
172
+ - Extracts schema definitions (`information_schema.TABLES`, `COLUMNS`, `STATISTICS`, `KEY_COLUMN_USAGE`) and records from `data_tables`.
173
+ - Saves a point-in-time snapshot into an embedded SQLite database (`db_client/schema_data.sqlite3`), tagged with the current version timestamp (`YYYYMMDDHHmm`).
174
+ 2. **Change Detection**:
175
+ - Queries SQLite to check whether any schema or data differences exist between the current version and `prev_version`.
176
+ - If `process_when_changed: true` and no changes are found, execution halts early.
177
+ 3. **Comparison Execution**:
178
+ - Compares all configured pairs in `compare_list` using parameterized queries with `#foreach` list filtering.
179
+ 4. **Report Export & Alerts**:
180
+ - Writes Markdown or Excel reports to `data/`.
181
+ - If schema or data changes are detected in log mode (`is_log: true`), an alert is automatically posted to Discord via `DISCORD_WEB_HOOK_URL`.
182
+ - Any runtime errors or exceptions are formatted and sent to Discord as alerts.
183
+
184
+ ### 3. Environment Variables (`.env`)
185
+
186
+ Set database credentials in `.env`:
187
+
188
+ ```env
189
+ DB_USERNAME=your_username
190
+ DB_PASSWORD=your_password
191
+ DISCORD_WEB_HOOK_URL=https://discord.com/api/webhooks/...
192
+ ```
193
+
194
+ ### 4. Run Comparison
195
+
196
+ ```bash
197
+ # Run via CLI
198
+ compare-schema --config_path config/config.yaml
199
+
200
+ # Or run as Python module
201
+ python -m src.main --config_path config/config.yaml
202
+ ```
203
+
204
+ ## License
205
+
206
+ MIT License. See [LICENSE](LICENSE) for details.
@@ -0,0 +1,171 @@
1
+ # compare-schema — Database Schema and Data Comparison Tool
2
+
3
+ [English](README.md) | [한국어](README_KO.md)
4
+
5
+ A database schema and data comparison tool across different environments (e.g. dev, stg, prd).
6
+
7
+ > [!NOTE]
8
+ > Currently, only **MySQL** schema and data are supported. The comparison metadata and version history are stored locally in an embedded SQLite database.
9
+
10
+ ## Features
11
+
12
+ - **Schema Comparison**: Compare tables, views, columns, indices, and foreign key references across databases.
13
+ - **Data Comparison**: Compare key-based data records across environments.
14
+ - **Database History Tracking**: Track changes to the same database over time by comparing versions.
15
+ - **Multiple Output Formats**: Export comparison reports to Markdown (`.md`) or Excel (`.xlsx`).
16
+ - **History & Versioning**: Tracks versions in SQLite to detect schema drift over time.
17
+ - **Notifications**: Discord webhook notification support for schema and data changes.
18
+ - **CLI & Module Support**: Run via `compare-schema` CLI command or `python -m src.main`.
19
+
20
+ ## Installation
21
+
22
+ ```bash
23
+ pip install compare-schema
24
+ ```
25
+
26
+ ## Quick Start
27
+
28
+ ### 1. Configuration (`config/config.yaml`)
29
+
30
+ Create a configuration YAML file:
31
+
32
+ ```yaml
33
+ output_type: md # md or xlsx
34
+ output_dir: ./compare_schema/result
35
+ prev_version: ""
36
+
37
+ data_tables:
38
+ - table: users
39
+ columns:
40
+ - name
41
+ - email
42
+ - nick_name
43
+ key_columns:
44
+ - id
45
+
46
+ exclude_tables: []
47
+ exclude_columns: []
48
+ db_host: 127.0.0.1
49
+ envs:
50
+ - name: dev
51
+ port: 3306
52
+ db_names: [shop, logs]
53
+ - name: stg
54
+ port: 3306
55
+ db_names: [shop]
56
+ - name: prd
57
+ port: 3306
58
+ db_names: [shop]
59
+
60
+ compare_list:
61
+ # Compare between different databases or environments
62
+ - a: dev.shop
63
+ b: dev.logs
64
+ - a: dev.shop
65
+ b: stg.shop
66
+ - a: stg.shop
67
+ b: prd.shop
68
+ # If a and b is same, compares between recent version and new version (shows history of that DB)
69
+ - a: dev.shop
70
+ b: dev.shop
71
+ - a: stg.shop
72
+ b: stg.shop
73
+ - a: prd.shop
74
+ b: prd.shop
75
+ ```
76
+
77
+ ### 2. Version Comparison & History
78
+
79
+ The tool supports two comparison modes based on the entries defined in `compare_list` (`{env}.{database}`):
80
+
81
+ #### A. Cross-Database Comparison (`a != b`)
82
+
83
+ Compares schemas and data between two distinct environments or databases (e.g., `dev.shop` vs `stg.shop`):
84
+
85
+ - **Schema Differences**:
86
+ - `schema_not_exists`: Identifies missing tables, views, columns, indices, and foreign keys present in one database but absent in the other.
87
+ - `schema_diff`: Identifies structural discrepancies in matching objects:
88
+ - **Columns**: Data types, nullability (`is_nullable`), default values, character length, numeric precision/scale, and column comments.
89
+ - **Indices**: Index types (BTREE, etc.), uniqueness (`is_unique`), and indexed column combinations.
90
+ - **Foreign Keys**: Target referenced tables and columns.
91
+ - **Data Differences**:
92
+ - Compares rows in tables specified in `data_tables`.
93
+ - Matches rows between databases using the primary/unique keys configured in `key_columns`.
94
+ - `data_not_exists`: Rows present in one database but missing in the other.
95
+ - `data_diff`: Rows that exist in both databases but have differing values in the monitored `columns`.
96
+ - **Output**: Exported to `{version}_compare.md` (Markdown) or the `compare` sheets in Excel.
97
+
98
+ #### B. Database History & Drift Tracking (`a == b`)
99
+
100
+ When `a` and `b` reference the same database (e.g., `a: dev.shop, b: dev.shop`), the tool compares the new `version` (timestamped as `YYYYMMDDHHmm`) against `prev_version`:
101
+
102
+ - **Change Categories**:
103
+ - `schema_added` / `data_added`: Newly created tables, views, columns, indices, foreign keys, or inserted rows.
104
+ - `schema_removed` / `data_removed`: Dropped schema objects or deleted rows.
105
+ - `schema_changed` / `data_changed`: Modified column types/properties or updated data row values over time.
106
+ - **Output**: Exported to `{version}_log.md` (Markdown) or the `log` sheets in Excel.
107
+
108
+ ---
109
+
110
+ ### Configuration Reference (`config.yaml`)
111
+
112
+ | Parameter | Type | Default | Description |
113
+ | :--- | :--- | :--- | :--- |
114
+ | `output_type` | `Literal["xlsx", "md"]` | Required | Output format. `"md"` generates Markdown files; `"xlsx"` generates an Excel workbook. |
115
+ | `output_dir` | `str` | Required | Directory path where output reports are saved. |
116
+ | `prev_version` | `str` | `""` | Previous version (`YYYYMMDDHHmm`) to compare against. If empty (`""`), automatically uses the latest version recorded in SQLite. |
117
+ | `process_when_changed` | `bool` | Required | If `true`, stops execution without generating reports if no schema or data changes are detected against `prev_version`. If `false`, generates reports on every run. |
118
+ | `data_tables` | `list[DataTableConfig]` | `[]` | List of tables and columns to compare data for. |
119
+ | `data_tables[].table` | `str` | Required | Table name in MySQL. |
120
+ | `data_tables[].columns` | `list[str]` | Required | Columns to compare for value differences. |
121
+ | `data_tables[].key_columns` | `list[str]` | Required | Key columns (e.g., primary key) used to match corresponding rows across databases. |
122
+ | `exclude_tables` | `list[str]` | Required | Table names to exclude from cross-environment schema comparison (`a != b`), such as migration tables (e.g., `["alembic_version"]`). *(Not excluded during same-database history tracking `a == b`)*. |
123
+ | `exclude_columns` | `list[str]` | Required | Column names to ignore during cross-environment column comparisons. |
124
+ | `db_host` | `str` | Required | Hostname or IP address for MySQL database connections. |
125
+ | `envs` | `list[EnvConfig]` | Required | List of environments and databases to connect to and extract. |
126
+ | `envs[].name` | `str` | Required | Environment identifier (e.g., `dev`, `stg`, `prd`). |
127
+ | `envs[].port` | `int` | Required | MySQL port number for this environment. |
128
+ | `envs[].db_names` | `list[str]` | Required | List of database names to extract from this environment. |
129
+ | `compare_list` | `list[EnvCompareConfig]` | Required | Target database pairs to compare (e.g., `a: dev.shop`, `b: stg.shop`). |
130
+
131
+ ---
132
+
133
+ ### How It Works
134
+
135
+ 1. **Extraction & Snapshotting**:
136
+ - Connects to MySQL instances defined in `envs` using credentials from `.env`.
137
+ - Extracts schema definitions (`information_schema.TABLES`, `COLUMNS`, `STATISTICS`, `KEY_COLUMN_USAGE`) and records from `data_tables`.
138
+ - Saves a point-in-time snapshot into an embedded SQLite database (`db_client/schema_data.sqlite3`), tagged with the current version timestamp (`YYYYMMDDHHmm`).
139
+ 2. **Change Detection**:
140
+ - Queries SQLite to check whether any schema or data differences exist between the current version and `prev_version`.
141
+ - If `process_when_changed: true` and no changes are found, execution halts early.
142
+ 3. **Comparison Execution**:
143
+ - Compares all configured pairs in `compare_list` using parameterized queries with `#foreach` list filtering.
144
+ 4. **Report Export & Alerts**:
145
+ - Writes Markdown or Excel reports to `data/`.
146
+ - If schema or data changes are detected in log mode (`is_log: true`), an alert is automatically posted to Discord via `DISCORD_WEB_HOOK_URL`.
147
+ - Any runtime errors or exceptions are formatted and sent to Discord as alerts.
148
+
149
+ ### 3. Environment Variables (`.env`)
150
+
151
+ Set database credentials in `.env`:
152
+
153
+ ```env
154
+ DB_USERNAME=your_username
155
+ DB_PASSWORD=your_password
156
+ DISCORD_WEB_HOOK_URL=https://discord.com/api/webhooks/...
157
+ ```
158
+
159
+ ### 4. Run Comparison
160
+
161
+ ```bash
162
+ # Run via CLI
163
+ compare-schema --config_path config/config.yaml
164
+
165
+ # Or run as Python module
166
+ python -m src.main --config_path config/config.yaml
167
+ ```
168
+
169
+ ## License
170
+
171
+ MIT License. See [LICENSE](LICENSE) for details.
@@ -0,0 +1,206 @@
1
+ Metadata-Version: 2.4
2
+ Name: compare-schema
3
+ Version: 0.1.1
4
+ Summary: Compare schema and data between databases
5
+ Author-email: Gu Park <doctorgu@kakao.com>
6
+ License-Expression: MIT
7
+ Project-URL: homepage, https://github.com/doctorgu/compare_schema
8
+ Project-URL: repository, https://github.com/doctorgu/compare_schema
9
+ Project-URL: documentation, https://github.com/doctorgu/compare_schema
10
+ Keywords: compare schema,database comparison,schema diff,mysql,sqlite
11
+ Requires-Python: >=3.13
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ Requires-Dist: sqlite3-client>=1.2.0
15
+ Requires-Dist: mysqlclient-client>=1.2.0
16
+ Requires-Dist: pydantic>=2.0
17
+ Requires-Dist: pydantic-settings>=2.0
18
+ Requires-Dist: xlsxwriter>=3.0
19
+ Requires-Dist: tabulate>=0.9
20
+ Requires-Dist: types-tabulate>=0.10
21
+ Requires-Dist: requests>=2.28
22
+ Requires-Dist: PyYAML>=6.0
23
+ Requires-Dist: SQLAlchemy>=2.0
24
+ Requires-Dist: PyMySQL>=1.0
25
+ Requires-Dist: python-dotenv>=1.0.0
26
+ Requires-Dist: ruff
27
+ Provides-Extra: test
28
+ Requires-Dist: pytest; extra == "test"
29
+ Requires-Dist: pytest-cov; extra == "test"
30
+ Requires-Dist: pytest-env; extra == "test"
31
+ Requires-Dist: pytest-mock; extra == "test"
32
+ Requires-Dist: pytest-asyncio; extra == "test"
33
+ Requires-Dist: build; extra == "test"
34
+ Dynamic: license-file
35
+
36
+ # compare-schema — Database Schema and Data Comparison Tool
37
+
38
+ [English](README.md) | [한국어](README_KO.md)
39
+
40
+ A database schema and data comparison tool across different environments (e.g. dev, stg, prd).
41
+
42
+ > [!NOTE]
43
+ > Currently, only **MySQL** schema and data are supported. The comparison metadata and version history are stored locally in an embedded SQLite database.
44
+
45
+ ## Features
46
+
47
+ - **Schema Comparison**: Compare tables, views, columns, indices, and foreign key references across databases.
48
+ - **Data Comparison**: Compare key-based data records across environments.
49
+ - **Database History Tracking**: Track changes to the same database over time by comparing versions.
50
+ - **Multiple Output Formats**: Export comparison reports to Markdown (`.md`) or Excel (`.xlsx`).
51
+ - **History & Versioning**: Tracks versions in SQLite to detect schema drift over time.
52
+ - **Notifications**: Discord webhook notification support for schema and data changes.
53
+ - **CLI & Module Support**: Run via `compare-schema` CLI command or `python -m src.main`.
54
+
55
+ ## Installation
56
+
57
+ ```bash
58
+ pip install compare-schema
59
+ ```
60
+
61
+ ## Quick Start
62
+
63
+ ### 1. Configuration (`config/config.yaml`)
64
+
65
+ Create a configuration YAML file:
66
+
67
+ ```yaml
68
+ output_type: md # md or xlsx
69
+ output_dir: ./compare_schema/result
70
+ prev_version: ""
71
+
72
+ data_tables:
73
+ - table: users
74
+ columns:
75
+ - name
76
+ - email
77
+ - nick_name
78
+ key_columns:
79
+ - id
80
+
81
+ exclude_tables: []
82
+ exclude_columns: []
83
+ db_host: 127.0.0.1
84
+ envs:
85
+ - name: dev
86
+ port: 3306
87
+ db_names: [shop, logs]
88
+ - name: stg
89
+ port: 3306
90
+ db_names: [shop]
91
+ - name: prd
92
+ port: 3306
93
+ db_names: [shop]
94
+
95
+ compare_list:
96
+ # Compare between different databases or environments
97
+ - a: dev.shop
98
+ b: dev.logs
99
+ - a: dev.shop
100
+ b: stg.shop
101
+ - a: stg.shop
102
+ b: prd.shop
103
+ # If a and b is same, compares between recent version and new version (shows history of that DB)
104
+ - a: dev.shop
105
+ b: dev.shop
106
+ - a: stg.shop
107
+ b: stg.shop
108
+ - a: prd.shop
109
+ b: prd.shop
110
+ ```
111
+
112
+ ### 2. Version Comparison & History
113
+
114
+ The tool supports two comparison modes based on the entries defined in `compare_list` (`{env}.{database}`):
115
+
116
+ #### A. Cross-Database Comparison (`a != b`)
117
+
118
+ Compares schemas and data between two distinct environments or databases (e.g., `dev.shop` vs `stg.shop`):
119
+
120
+ - **Schema Differences**:
121
+ - `schema_not_exists`: Identifies missing tables, views, columns, indices, and foreign keys present in one database but absent in the other.
122
+ - `schema_diff`: Identifies structural discrepancies in matching objects:
123
+ - **Columns**: Data types, nullability (`is_nullable`), default values, character length, numeric precision/scale, and column comments.
124
+ - **Indices**: Index types (BTREE, etc.), uniqueness (`is_unique`), and indexed column combinations.
125
+ - **Foreign Keys**: Target referenced tables and columns.
126
+ - **Data Differences**:
127
+ - Compares rows in tables specified in `data_tables`.
128
+ - Matches rows between databases using the primary/unique keys configured in `key_columns`.
129
+ - `data_not_exists`: Rows present in one database but missing in the other.
130
+ - `data_diff`: Rows that exist in both databases but have differing values in the monitored `columns`.
131
+ - **Output**: Exported to `{version}_compare.md` (Markdown) or the `compare` sheets in Excel.
132
+
133
+ #### B. Database History & Drift Tracking (`a == b`)
134
+
135
+ When `a` and `b` reference the same database (e.g., `a: dev.shop, b: dev.shop`), the tool compares the new `version` (timestamped as `YYYYMMDDHHmm`) against `prev_version`:
136
+
137
+ - **Change Categories**:
138
+ - `schema_added` / `data_added`: Newly created tables, views, columns, indices, foreign keys, or inserted rows.
139
+ - `schema_removed` / `data_removed`: Dropped schema objects or deleted rows.
140
+ - `schema_changed` / `data_changed`: Modified column types/properties or updated data row values over time.
141
+ - **Output**: Exported to `{version}_log.md` (Markdown) or the `log` sheets in Excel.
142
+
143
+ ---
144
+
145
+ ### Configuration Reference (`config.yaml`)
146
+
147
+ | Parameter | Type | Default | Description |
148
+ | :--- | :--- | :--- | :--- |
149
+ | `output_type` | `Literal["xlsx", "md"]` | Required | Output format. `"md"` generates Markdown files; `"xlsx"` generates an Excel workbook. |
150
+ | `output_dir` | `str` | Required | Directory path where output reports are saved. |
151
+ | `prev_version` | `str` | `""` | Previous version (`YYYYMMDDHHmm`) to compare against. If empty (`""`), automatically uses the latest version recorded in SQLite. |
152
+ | `process_when_changed` | `bool` | Required | If `true`, stops execution without generating reports if no schema or data changes are detected against `prev_version`. If `false`, generates reports on every run. |
153
+ | `data_tables` | `list[DataTableConfig]` | `[]` | List of tables and columns to compare data for. |
154
+ | `data_tables[].table` | `str` | Required | Table name in MySQL. |
155
+ | `data_tables[].columns` | `list[str]` | Required | Columns to compare for value differences. |
156
+ | `data_tables[].key_columns` | `list[str]` | Required | Key columns (e.g., primary key) used to match corresponding rows across databases. |
157
+ | `exclude_tables` | `list[str]` | Required | Table names to exclude from cross-environment schema comparison (`a != b`), such as migration tables (e.g., `["alembic_version"]`). *(Not excluded during same-database history tracking `a == b`)*. |
158
+ | `exclude_columns` | `list[str]` | Required | Column names to ignore during cross-environment column comparisons. |
159
+ | `db_host` | `str` | Required | Hostname or IP address for MySQL database connections. |
160
+ | `envs` | `list[EnvConfig]` | Required | List of environments and databases to connect to and extract. |
161
+ | `envs[].name` | `str` | Required | Environment identifier (e.g., `dev`, `stg`, `prd`). |
162
+ | `envs[].port` | `int` | Required | MySQL port number for this environment. |
163
+ | `envs[].db_names` | `list[str]` | Required | List of database names to extract from this environment. |
164
+ | `compare_list` | `list[EnvCompareConfig]` | Required | Target database pairs to compare (e.g., `a: dev.shop`, `b: stg.shop`). |
165
+
166
+ ---
167
+
168
+ ### How It Works
169
+
170
+ 1. **Extraction & Snapshotting**:
171
+ - Connects to MySQL instances defined in `envs` using credentials from `.env`.
172
+ - Extracts schema definitions (`information_schema.TABLES`, `COLUMNS`, `STATISTICS`, `KEY_COLUMN_USAGE`) and records from `data_tables`.
173
+ - Saves a point-in-time snapshot into an embedded SQLite database (`db_client/schema_data.sqlite3`), tagged with the current version timestamp (`YYYYMMDDHHmm`).
174
+ 2. **Change Detection**:
175
+ - Queries SQLite to check whether any schema or data differences exist between the current version and `prev_version`.
176
+ - If `process_when_changed: true` and no changes are found, execution halts early.
177
+ 3. **Comparison Execution**:
178
+ - Compares all configured pairs in `compare_list` using parameterized queries with `#foreach` list filtering.
179
+ 4. **Report Export & Alerts**:
180
+ - Writes Markdown or Excel reports to `data/`.
181
+ - If schema or data changes are detected in log mode (`is_log: true`), an alert is automatically posted to Discord via `DISCORD_WEB_HOOK_URL`.
182
+ - Any runtime errors or exceptions are formatted and sent to Discord as alerts.
183
+
184
+ ### 3. Environment Variables (`.env`)
185
+
186
+ Set database credentials in `.env`:
187
+
188
+ ```env
189
+ DB_USERNAME=your_username
190
+ DB_PASSWORD=your_password
191
+ DISCORD_WEB_HOOK_URL=https://discord.com/api/webhooks/...
192
+ ```
193
+
194
+ ### 4. Run Comparison
195
+
196
+ ```bash
197
+ # Run via CLI
198
+ compare-schema --config_path config/config.yaml
199
+
200
+ # Or run as Python module
201
+ python -m src.main --config_path config/config.yaml
202
+ ```
203
+
204
+ ## License
205
+
206
+ MIT License. See [LICENSE](LICENSE) for details.
@@ -0,0 +1,30 @@
1
+ LICENSE
2
+ README.md
3
+ pyproject.toml
4
+ compare_schema.egg-info/PKG-INFO
5
+ compare_schema.egg-info/SOURCES.txt
6
+ compare_schema.egg-info/dependency_links.txt
7
+ compare_schema.egg-info/entry_points.txt
8
+ compare_schema.egg-info/requires.txt
9
+ compare_schema.egg-info/top_level.txt
10
+ db_client/schema.sql
11
+ db_client/schema_client.py
12
+ db_client/schema_settings.py
13
+ db_client/source_client.py
14
+ db_client/source_settings.py
15
+ db_client/queries/schema/data.yml
16
+ db_client/queries/schema/schema.yml
17
+ db_client/queries/source/import_data.yml
18
+ db_client/queries/source/import_schema.yml
19
+ src/config.py
20
+ src/env_config.py
21
+ src/excel_helper.py
22
+ src/main.py
23
+ src/markdown_helper.py
24
+ src/models.py
25
+ src/util_mysql.py
26
+ src/util_other.py
27
+ src/util_path.py
28
+ tests/test_config.py
29
+ tests/test_markdown.py
30
+ tests/test_models.py
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ compare-schema = src.main:main
@@ -0,0 +1,21 @@
1
+ sqlite3-client>=1.2.0
2
+ mysqlclient-client>=1.2.0
3
+ pydantic>=2.0
4
+ pydantic-settings>=2.0
5
+ xlsxwriter>=3.0
6
+ tabulate>=0.9
7
+ types-tabulate>=0.10
8
+ requests>=2.28
9
+ PyYAML>=6.0
10
+ SQLAlchemy>=2.0
11
+ PyMySQL>=1.0
12
+ python-dotenv>=1.0.0
13
+ ruff
14
+
15
+ [test]
16
+ pytest
17
+ pytest-cov
18
+ pytest-env
19
+ pytest-mock
20
+ pytest-asyncio
21
+ build
@@ -0,0 +1,2 @@
1
+ db_client
2
+ src