mssql-ocp-migrate 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mssql_ocp_migrate-0.2.0/.readthedocs.yaml +12 -0
- mssql_ocp_migrate-0.2.0/CHANGELOG.md +26 -0
- mssql_ocp_migrate-0.2.0/LICENSE +21 -0
- mssql_ocp_migrate-0.2.0/MANIFEST.in +7 -0
- mssql_ocp_migrate-0.2.0/PKG-INFO +226 -0
- mssql_ocp_migrate-0.2.0/README.md +189 -0
- mssql_ocp_migrate-0.2.0/SECURITY.md +47 -0
- mssql_ocp_migrate-0.2.0/THIRD_PARTY_NOTICES.md +8 -0
- mssql_ocp_migrate-0.2.0/deploy/openshift/Containerfile +10 -0
- mssql_ocp_migrate-0.2.0/deploy/openshift/configmap.yaml +36 -0
- mssql_ocp_migrate-0.2.0/deploy/openshift/job.yaml +38 -0
- mssql_ocp_migrate-0.2.0/deploy/openshift/lift/01-backup-share.yaml +26 -0
- mssql_ocp_migrate-0.2.0/deploy/openshift/lift/02-sqlserver-volumes-patch.yaml +32 -0
- mssql_ocp_migrate-0.2.0/deploy/openshift/lift/03-lift-job.yaml +34 -0
- mssql_ocp_migrate-0.2.0/deploy/openshift/lift/04-ag-endpoint-service.yaml +9 -0
- mssql_ocp_migrate-0.2.0/deploy/openshift/scale-out/indexed-job.yaml +31 -0
- mssql_ocp_migrate-0.2.0/deploy/openshift/secret.yaml +9 -0
- mssql_ocp_migrate-0.2.0/docs/_generate.py +190 -0
- mssql_ocp_migrate-0.2.0/docs/_preview/make_preview.py +65 -0
- mssql_ocp_migrate-0.2.0/docs/_preview/momtheme/static/site.css +260 -0
- mssql_ocp_migrate-0.2.0/docs/_static/custom.css +5 -0
- mssql_ocp_migrate-0.2.0/docs/conf.py +85 -0
- mssql_ocp_migrate-0.2.0/docs/development/architecture.md +71 -0
- mssql_ocp_migrate-0.2.0/docs/development/index.md +17 -0
- mssql_ocp_migrate-0.2.0/docs/development/releasing.md +52 -0
- mssql_ocp_migrate-0.2.0/docs/development/testing.md +30 -0
- mssql_ocp_migrate-0.2.0/docs/getting_started/choosing.md +40 -0
- mssql_ocp_migrate-0.2.0/docs/getting_started/index.md +49 -0
- mssql_ocp_migrate-0.2.0/docs/getting_started/install.md +77 -0
- mssql_ocp_migrate-0.2.0/docs/getting_started/tour.md +123 -0
- mssql_ocp_migrate-0.2.0/docs/index.md +121 -0
- mssql_ocp_migrate-0.2.0/docs/reference/api.md +166 -0
- mssql_ocp_migrate-0.2.0/docs/reference/cli.md +20 -0
- mssql_ocp_migrate-0.2.0/docs/reference/config.md +22 -0
- mssql_ocp_migrate-0.2.0/docs/reference/index.md +19 -0
- mssql_ocp_migrate-0.2.0/docs/user_guide/configuration.md +109 -0
- mssql_ocp_migrate-0.2.0/docs/user_guide/connections.md +71 -0
- mssql_ocp_migrate-0.2.0/docs/user_guide/index.md +32 -0
- mssql_ocp_migrate-0.2.0/docs/user_guide/lift.md +238 -0
- mssql_ocp_migrate-0.2.0/docs/user_guide/limitations.md +37 -0
- mssql_ocp_migrate-0.2.0/docs/user_guide/logical.md +150 -0
- mssql_ocp_migrate-0.2.0/docs/user_guide/openshift.md +81 -0
- mssql_ocp_migrate-0.2.0/docs/user_guide/performance.md +93 -0
- mssql_ocp_migrate-0.2.0/docs/user_guide/security.md +51 -0
- mssql_ocp_migrate-0.2.0/docs/user_guide/troubleshooting.md +84 -0
- mssql_ocp_migrate-0.2.0/docs/user_guide/validation.md +97 -0
- mssql_ocp_migrate-0.2.0/docs/whatsnew/index.md +46 -0
- mssql_ocp_migrate-0.2.0/pyproject.toml +53 -0
- mssql_ocp_migrate-0.2.0/setup.cfg +4 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/__init__.py +16 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/__main__.py +5 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/LICENSE-python-tds.txt +21 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/__init__.py +7 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/__init__.py +553 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/collate.py +379 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/connection.py +307 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/connection_pool.py +43 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/cursor.py +711 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/extensions.py +18 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/fedauth.py +25 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/instance_browser_client.py +78 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/lcid.py +639 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/login.py +231 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/row_strategies.py +108 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/smp.py +348 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/sspi.py +508 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/tds.py +8 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/tds_base.py +1055 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/tds_reader.py +239 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/tds_session.py +1803 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/tds_socket.py +174 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/tds_types.py +2890 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/tds_writer.py +161 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/tls.py +172 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/tz.py +80 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/_vendor/pytds/utils.py +84 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/adapter.py +97 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/chunking.py +66 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/cli.py +343 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/config.py +217 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/ddl.py +128 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/drivers.py +260 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/engine.py +525 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/estimate.py +168 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/model.py +151 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/physical.py +873 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/py.typed +0 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/report.py +141 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/rowhash.py +75 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/sqlserver.py +691 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/state.py +106 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate/validate.py +238 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate.egg-info/PKG-INFO +226 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate.egg-info/SOURCES.txt +109 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate.egg-info/dependency_links.txt +1 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate.egg-info/entry_points.txt +2 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate.egg-info/requires.txt +10 -0
- mssql_ocp_migrate-0.2.0/src/mssql_ocp_migrate.egg-info/top_level.txt +1 -0
- mssql_ocp_migrate-0.2.0/tests/conftest.py +7 -0
- mssql_ocp_migrate-0.2.0/tests/e2e/docker-compose.yml +25 -0
- mssql_ocp_migrate-0.2.0/tests/e2e/seed_source.sql +142 -0
- mssql_ocp_migrate-0.2.0/tests/e2e/test_e2e_lift.py +51 -0
- mssql_ocp_migrate-0.2.0/tests/e2e/test_e2e_sqlserver.py +153 -0
- mssql_ocp_migrate-0.2.0/tests/sqlite_adapter.py +163 -0
- mssql_ocp_migrate-0.2.0/tests/test_bulk_types.py +105 -0
- mssql_ocp_migrate-0.2.0/tests/test_engine.py +204 -0
- mssql_ocp_migrate-0.2.0/tests/test_no_downloads.py +56 -0
- mssql_ocp_migrate-0.2.0/tests/test_physical.py +333 -0
- mssql_ocp_migrate-0.2.0/tests/test_pymssql.py +89 -0
- mssql_ocp_migrate-0.2.0/tests/test_tls.py +182 -0
- mssql_ocp_migrate-0.2.0/tests/test_units.py +173 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to `mssql-ocp-migrate` are documented here.
|
|
4
|
+
|
|
5
|
+
## [0.2.0] - 2026-11-16
|
|
6
|
+
|
|
7
|
+
First public release.
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
- **Physical migration (`mssql-migrate lift`)** for very large databases: striped, compressed,
|
|
11
|
+
copy-only backups; restore `WITH NORECOVERY` across several volumes; continuous log catch-up that
|
|
12
|
+
also applies log backups taken by existing jobs (from `msdb`, with `path_map`) and refuses broken
|
|
13
|
+
chains (gaps, NUL, third-party VDI backups); minutes-long cut-over with a tail-log backup and
|
|
14
|
+
automatic undo of replayed READ_ONLY/SINGLE_USER; logins copied with the same SID and password
|
|
15
|
+
hash; TDE certificates copied over the connection; `plan` from real `msdb` history; `verify`;
|
|
16
|
+
`ag-script` for availability-group seeding
|
|
17
|
+
- **Logical migration (`mssql-migrate migrate`)**: chunked, parallel, resumable copy with identity
|
|
18
|
+
preservation and lossless temporal/CLR transport; schema, indexes, constraints and programmable
|
|
19
|
+
objects; scale-out across pods with `--shard` and `finalize`; minimally logged loads
|
|
20
|
+
- **Validation**: schema parity, row counts, server-side checksums, SHA-256 row comparison naming the
|
|
21
|
+
exact keys that differ; HTML and JSON reports
|
|
22
|
+
- **Drivers**: bundled pure-Python TDS driver (python-tds 1.17.1, MIT) with TLS on the standard
|
|
23
|
+
library; optional `pymssql` and `pyodbc` when already installed
|
|
24
|
+
- OpenShift manifests (Jobs, Indexed Job, lift share/volumes), documentation (Sphinx, Read the Docs)
|
|
25
|
+
- CI: tests on Python 3.10–3.13, docs build, CodeQL, OpenSSF Scorecard, Dependabot; PyPI Trusted
|
|
26
|
+
Publishing (`python-publish.yml`, environment `release`); every GitHub Action pinned to a commit SHA
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Karthikeyan Balasundaram
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
include README.md LICENSE THIRD_PARTY_NOTICES.md SECURITY.md CHANGELOG.md .readthedocs.yaml
|
|
2
|
+
recursive-include docs *.md *.py *.css
|
|
3
|
+
recursive-include tests *.py *.sql *.yml
|
|
4
|
+
recursive-include deploy *
|
|
5
|
+
prune docs/_build
|
|
6
|
+
prune docs/reference/_generated
|
|
7
|
+
global-exclude __pycache__ *.pyc
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mssql-ocp-migrate
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Migrate SQL Server databases to SQL Server on OpenShift (or anywhere) with parallel chunked copy, resume and end-to-end validation. Zero dependencies: the TDS driver is bundled.
|
|
5
|
+
Author: Karthikeyan Balasundaram
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/karthikbd/mssql-ocp-migrate
|
|
8
|
+
Project-URL: Documentation, https://mssql-ocp-migrate.readthedocs.io
|
|
9
|
+
Project-URL: Source, https://github.com/karthikbd/mssql-ocp-migrate
|
|
10
|
+
Project-URL: Issues, https://github.com/karthikbd/mssql-ocp-migrate/issues
|
|
11
|
+
Project-URL: Changelog, https://github.com/karthikbd/mssql-ocp-migrate/blob/main/CHANGELOG.md
|
|
12
|
+
Keywords: sql server,mssql,migration,openshift,ocp,data validation,tds,bulk copy
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: System Administrators
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Database
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
License-File: src/mssql_ocp_migrate/_vendor/LICENSE-python-tds.txt
|
|
28
|
+
Provides-Extra: test
|
|
29
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
30
|
+
Provides-Extra: docs
|
|
31
|
+
Requires-Dist: sphinx>=7; extra == "docs"
|
|
32
|
+
Requires-Dist: pydata-sphinx-theme>=0.15; extra == "docs"
|
|
33
|
+
Requires-Dist: myst-parser>=2; extra == "docs"
|
|
34
|
+
Requires-Dist: sphinx-copybutton; extra == "docs"
|
|
35
|
+
Requires-Dist: sphinx-design; extra == "docs"
|
|
36
|
+
Dynamic: license-file
|
|
37
|
+
|
|
38
|
+
# mssql-ocp-migrate
|
|
39
|
+
|
|
40
|
+
[](https://pypi.org/project/mssql-ocp-migrate/)
|
|
41
|
+
[](https://pypi.org/project/mssql-ocp-migrate/)
|
|
42
|
+
[](https://github.com/karthikbd/mssql-ocp-migrate/actions/workflows/tests.yml)
|
|
43
|
+
[](https://mssql-ocp-migrate.readthedocs.io)
|
|
44
|
+
[](LICENSE)
|
|
45
|
+
[](https://securityscorecards.dev/viewer/?uri=github.com/karthikbd/mssql-ocp-migrate)
|
|
46
|
+
|
|
47
|
+
Move a SQL Server database to SQL Server running on **OpenShift (OCP)**, or to any other SQL Server,
|
|
48
|
+
with a parallel chunked copy, resume after failure, and end-to-end validation that proves the
|
|
49
|
+
target matches the source.
|
|
50
|
+
|
|
51
|
+
**No downloads, ever.** The package declares no dependencies: the SQL Server network driver
|
|
52
|
+
(python-tds, MIT) is bundled inside it, with TLS rebuilt on Python's own `ssl` module, so
|
|
53
|
+
`pip install` pulls in nothing else and the tool makes no HTTP calls at runtime. If `pyodbc` and a
|
|
54
|
+
Microsoft ODBC driver are already installed, it can use them; they are never required.
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
pip install mssql-ocp-migrate # from PyPI
|
|
58
|
+
pip install --no-index mssql_ocp_migrate-0.2.0-py3-none-any.whl # air-gapped: one file, nothing else
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Pick the path by size
|
|
62
|
+
|
|
63
|
+
| Data | Use | Why |
|
|
64
|
+
|---|---|---|
|
|
65
|
+
| up to a few TB, or subsets / schema changes / different collation | `migrate` (logical copy) | row-level control, per-table filters, scale out with `--shard` across pods |
|
|
66
|
+
| **tens to hundreds of TB, whole databases** | **`lift`** (physical) | SQL Server ships its own compressed pages: no row decoding, no index rebuilds, source stays online, cut-over takes minutes |
|
|
67
|
+
|
|
68
|
+
### `lift`: the 100 TB path
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
mssql-migrate lift plan -c migrate.toml # sizes, real compression ratio + speed from msdb, blockers, timeline
|
|
72
|
+
mssql-migrate lift run -c migrate.toml --follow-hours 168 # seed + keep caught up (source online)
|
|
73
|
+
mssql-migrate lift status -c migrate.toml
|
|
74
|
+
mssql-migrate lift cutover -c migrate.toml --confirm # maintenance window: minutes
|
|
75
|
+
mssql-migrate lift verify -c migrate.toml # row counts for every table + sampled checksums
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
1. **Seed:** a `COPY_ONLY`, compressed, `CHECKSUM` full backup striped over up to 64 files (one backup
|
|
79
|
+
thread per file) to a share both servers can reach. The tool sizes `MAXTRANSFERSIZE` and
|
|
80
|
+
`BUFFERCOUNT` to fit a memory cap. The restore runs `WITH NORECOVERY`, spreading the data files
|
|
81
|
+
over several OpenShift volumes so writes run in parallel, and verifies page checksums as it goes.
|
|
82
|
+
2. **Catch-up:** log backups every `log_interval_seconds`, restored continuously. Log backups taken
|
|
83
|
+
by your *existing* backup jobs are read from `msdb` and applied too (paths translated with
|
|
84
|
+
`path_map`), so the chain never breaks. A missing log, a backup sent to `NUL`, or a third-party
|
|
85
|
+
(VDI) log backup stops the loop with a clear message instead of silently corrupting the chain.
|
|
86
|
+
3. **Cut-over:** a final catch-up, then a tail-log backup `WITH NORECOVERY`. The source goes to
|
|
87
|
+
RESTORING, so nothing can write to it, and rollback is a single `RESTORE ... WITH RECOVERY` on the
|
|
88
|
+
source. Then the last restore, `RECOVERY`, and the post-steps: SQL logins with the same SID and
|
|
89
|
+
password hash (no orphaned users), database owner, TRUSTWORTHY / DB_CHAINING / Service Broker,
|
|
90
|
+
orphaned-user re-mapping, and optionally `DBCC CHECKDB WITH PHYSICAL_ONLY`.
|
|
91
|
+
4. **TDE:** the certificate and private key are copied over the connection
|
|
92
|
+
(`CERTENCODED` / `CERTPRIVATEKEY` → `CREATE CERTIFICATE ... FROM BINARY`), with no files on either
|
|
93
|
+
server.
|
|
94
|
+
|
|
95
|
+
Timeline for **100 TB, ~3:1 compression** (`lift plan` prints this from your own msdb history):
|
|
96
|
+
|
|
97
|
+
| Stage | 10 Gbps link, 2 GB/s target storage | 25 Gbps, 4 GB/s |
|
|
98
|
+
|---|---|---|
|
|
99
|
+
| full backup (source online; assumes the source can read ≥3 GB/s across its files) | ~10 h | ~4–7 h (source read speed bound) |
|
|
100
|
+
| restore on OCP (source online) | ~15 h | ~7 h |
|
|
101
|
+
| log catch-up | minutes to ~1 h | minutes |
|
|
102
|
+
| **cut-over downtime** | **~5–15 min** | **~5–15 min** |
|
|
103
|
+
|
|
104
|
+
The restore usually takes longest. It writes the full uncompressed size, so give the SQL Server pod
|
|
105
|
+
4–8 data volumes on the fastest storage class (`deploy/openshift/lift/02-sqlserver-volumes-patch.yaml`).
|
|
106
|
+
With S3, the HTTPS calls come from SQL Server itself, not from this package. S3 is supported for SQL Server 2022, but one backup set is capped at about 12 TB (64 URLs × 10,000 parts
|
|
107
|
+
× 20 MB), so a 100 TB database needs `transport = "share"` (NFS/SMB). `lift plan` reports this as a blocker.
|
|
108
|
+
|
|
109
|
+
**Option B, no backup files** (Enterprise/Developer on both sides): `mssql-migrate lift ag-script`
|
|
110
|
+
prints a cluster-less availability group (`CLUSTER_TYPE = NONE`, `SEEDING_MODE = AUTOMATIC`, trace
|
|
111
|
+
flag 9567 to compress the seeding stream) that streams the database straight into the OpenShift pod
|
|
112
|
+
and keeps it synchronized. Cut-over is a manual failover.
|
|
113
|
+
|
|
114
|
+
## Quick start
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
mssql-migrate init-config > migrate.toml # edit hosts, database, users
|
|
118
|
+
export SRC_DB_PASSWORD=... # or password_file = an OpenShift secret mount
|
|
119
|
+
mssql-migrate check -c migrate.toml # connectivity, versions, table inventory, warnings
|
|
120
|
+
mssql-migrate estimate -c migrate.toml # measures real throughput, projects total time
|
|
121
|
+
mssql-migrate migrate -c migrate.toml # schema -> data -> indexes/FKs -> objects -> validation
|
|
122
|
+
mssql-migrate validate -c migrate.toml # re-run any time (for example right before cut-over)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Python API:
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
from mssql_ocp_migrate import Config, Migrator
|
|
129
|
+
run = Migrator(Config.load("migrate.toml")).migrate()
|
|
130
|
+
print(run.ok, run.rows, run.validation.summary())
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Each run writes `migration_report/report.html` (evidence for sign-off) and `report.json`.
|
|
134
|
+
|
|
135
|
+
## What `migrate` does
|
|
136
|
+
|
|
137
|
+
| Phase | Details |
|
|
138
|
+
|---|---|
|
|
139
|
+
| Preflight | Reads the catalog: tables, columns, alias types, identity, computed, defaults, PK/unique/indexes (incl. filtered, INCLUDE, columnstore), FKs, checks, sequences, views, procs, functions, triggers, synonyms. Refuses a non-empty target unless told to `truncate` or `resume`. |
|
|
140
|
+
| Schema | Creates schemas, alias types, sequences (at the source's current value), and tables with their primary keys. |
|
|
141
|
+
| Copy | Splits each table into key ranges (`chunk_rows`, single or composite keys) and copies them in parallel with TDS bulk insert (`INSERT BULK`). Each chunk is one transaction that first clears its own range, so a retry or a resume never duplicates rows. Identity values are preserved, and the strategy is probed on the target at runtime. `datetime2(7)`, `time(7)`, `datetimeoffset(7)` and `datetime` are moved as ISO text, so no 100 ns ticks are lost. Geography, geometry and hierarchyid are moved as varbinary. |
|
|
142
|
+
| Post-load | Nonclustered/unique indexes, check constraints and foreign keys `WITH CHECK` (trusted), identity reseed, then views/functions/procs/synonyms with a dependency-retry loop, triggers last (disabled ones stay disabled). |
|
|
143
|
+
| Validate | See below. |
|
|
144
|
+
|
|
145
|
+
`mssql-migrate schema -c migrate.toml -o schema.sql` writes the DDL without running it, for DBA review.
|
|
146
|
+
|
|
147
|
+
## Validation (end-to-end testing)
|
|
148
|
+
|
|
149
|
+
| `validation.mode` | What is compared |
|
|
150
|
+
|---|---|
|
|
151
|
+
| `count` | Exact `COUNT_BIG(*)` per table, schema parity (columns, types, nullability, collation, identity, PK, indexes, FKs, checks), objects present, identity current values. |
|
|
152
|
+
| `checksum` (default) | The `count` checks, plus a per-chunk `COUNT` / `SUM(BINARY_CHECKSUM)` / `CHECKSUM_AGG` computed inside both servers (no data crosses the network; LOB, xml and CLR columns are hashed with SHA-256), plus an exact row-by-row SHA-256 comparison of a random `sample_full_pct` % of chunks. Any chunk whose checksum differs is compared row by row automatically, and the report lists the exact keys that are missing, extra or changed. |
|
|
153
|
+
| `full` | Every row of every table is hashed with SHA-256 on both sides and compared by key. Keyless tables use an order-independent multiset digest, which also catches duplicate-row differences. |
|
|
154
|
+
|
|
155
|
+
`rowversion` columns are excluded by design, because the target assigns new values.
|
|
156
|
+
|
|
157
|
+
## Scaling the logical copy out
|
|
158
|
+
|
|
159
|
+
The bundled pure-Python driver moves about 2–4 MB/s per worker process. To go wider, run the copy as
|
|
160
|
+
an OpenShift **Indexed Job**: `migrate --shard auto/16` on 16 pods. Each pod copies the chunks that
|
|
161
|
+
hash to its index. Shard 1 creates the schema and writes the shared chunk plan. Then
|
|
162
|
+
`mssql-migrate finalize --shard-count 16` builds indexes, constraints and objects and validates.
|
|
163
|
+
`target_recovery_simple = true` keeps bulk loads minimally logged, and the original recovery model is
|
|
164
|
+
restored at the end. See `deploy/openshift/scale-out/indexed-job.yaml`.
|
|
165
|
+
|
|
166
|
+
## How long will it take?
|
|
167
|
+
|
|
168
|
+
Run `mssql-migrate estimate`. It reads a sample from your largest table, bulk-loads it into a
|
|
169
|
+
`#temp` table on the target, times a server-side checksum, and projects copy, index and validation
|
|
170
|
+
time for your `workers` setting. Rough guide for the bundled pure-Python driver with
|
|
171
|
+
`executor = "auto"` (one process per worker):
|
|
172
|
+
|
|
173
|
+
| Per worker | 8 workers | 16 workers |
|
|
174
|
+
|---|---|---|
|
|
175
|
+
| ~15–25k rows/s (~2–4 MB/s) for a 10–15-column row | ~55–100 GB/hour | ~100–180 GB/hour, if the network and the target's log disk keep up |
|
|
176
|
+
|
|
177
|
+
Add about 20–30 % for index and FK builds. Checksum validation scans both databases on the server,
|
|
178
|
+
which usually takes minutes per 100 GB. With `pyodbc` + ODBC Driver 18 already installed,
|
|
179
|
+
per-stream throughput is typically 2–4× higher.
|
|
180
|
+
|
|
181
|
+
For multi-terabyte databases, a native `BACKUP` → copy into the pod → `RESTORE` is the fastest
|
|
182
|
+
lift. You can still run `mssql-migrate validate` afterwards to prove parity.
|
|
183
|
+
|
|
184
|
+
## Running against OpenShift
|
|
185
|
+
|
|
186
|
+
**From a workstation or VM:** `oc port-forward svc/mssql 14330:1433 -n <ns>`, then point `[target]` at
|
|
187
|
+
`127.0.0.1:14330`. Set `hostname_in_certificate` if you verify the server certificate.
|
|
188
|
+
|
|
189
|
+
**Inside the cluster (recommended for large data):** see `deploy/openshift/`:
|
|
190
|
+
`Containerfile` builds an offline image (`pip install --no-index` of the wheel), and `configmap.yaml`,
|
|
191
|
+
`secret.yaml` and `job.yaml` run it as a Job. A PVC keeps the checkpoint and the report, so a
|
|
192
|
+
failed or evicted Job resumes where it stopped (`if_target_not_empty = "resume"`). The source SQL
|
|
193
|
+
Server must be reachable from the cluster (egress / firewall on 1433).
|
|
194
|
+
|
|
195
|
+
## Cut-over pattern for a live source
|
|
196
|
+
|
|
197
|
+
1. `migrate` while the application is still running (bulk of the data, hours).
|
|
198
|
+
2. Stop writes (application maintenance window, or `ALTER DATABASE ... SET READ_ONLY`).
|
|
199
|
+
3. `migrate --truncate --include <tables that changed>`, or a full re-run for small databases.
|
|
200
|
+
4. `validate --mode checksum` (or `full`), keep the HTML report as evidence, then switch connection strings.
|
|
201
|
+
|
|
202
|
+
## Permissions
|
|
203
|
+
|
|
204
|
+
- **Source:** `db_datareader` + `VIEW DEFINITION` (+ `VIEW DATABASE STATE` for size estimates).
|
|
205
|
+
- **Target:** `db_owner` on the pre-created, empty database. The tool creates the schema, and bulk insert needs INSERT plus ALTER on each table. `ADMINISTER BULK OPERATIONS` is not needed, because no files are read on the server.
|
|
206
|
+
|
|
207
|
+
## Not migrated (reported as warnings)
|
|
208
|
+
|
|
209
|
+
Logins, users and permissions; SQL Agent jobs; memory-optimized tables are created disk-based;
|
|
210
|
+
system-versioned temporal tables arrive as two normal tables (re-enable `SYSTEM_VERSIONING`
|
|
211
|
+
afterwards); partitioning, XML and spatial indexes, full-text; FileTables; `WITH ENCRYPTION`
|
|
212
|
+
modules (their definitions cannot be read).
|
|
213
|
+
|
|
214
|
+
## Development
|
|
215
|
+
|
|
216
|
+
```bash
|
|
217
|
+
pip install -e . pytest && pytest # unit + engine (SQLite) + TLS wire tests
|
|
218
|
+
docker compose -f tests/e2e/docker-compose.yml up -d
|
|
219
|
+
MSSQL_E2E=1 pytest tests/e2e -v # real SQL Server 2022 source -> target
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
Releasing: bump the version in `pyproject.toml` and `src/mssql_ocp_migrate/__init__.py`, update
|
|
223
|
+
`CHANGELOG.md`, push `main`, then publish a GitHub release `vX.Y.Z`. `.github/workflows/python-publish.yml`
|
|
224
|
+
runs the tests and publishes to PyPI with Trusted Publishing (environment `release`, no stored token).
|
|
225
|
+
|
|
226
|
+
Third-party notices: see `THIRD_PARTY_NOTICES.md`.
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
# mssql-ocp-migrate
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/mssql-ocp-migrate/)
|
|
4
|
+
[](https://pypi.org/project/mssql-ocp-migrate/)
|
|
5
|
+
[](https://github.com/karthikbd/mssql-ocp-migrate/actions/workflows/tests.yml)
|
|
6
|
+
[](https://mssql-ocp-migrate.readthedocs.io)
|
|
7
|
+
[](LICENSE)
|
|
8
|
+
[](https://securityscorecards.dev/viewer/?uri=github.com/karthikbd/mssql-ocp-migrate)
|
|
9
|
+
|
|
10
|
+
Move a SQL Server database to SQL Server running on **OpenShift (OCP)**, or to any other SQL Server,
|
|
11
|
+
with a parallel chunked copy, resume after failure, and end-to-end validation that proves the
|
|
12
|
+
target matches the source.
|
|
13
|
+
|
|
14
|
+
**No downloads, ever.** The package declares no dependencies: the SQL Server network driver
|
|
15
|
+
(python-tds, MIT) is bundled inside it, with TLS rebuilt on Python's own `ssl` module, so
|
|
16
|
+
`pip install` pulls in nothing else and the tool makes no HTTP calls at runtime. If `pyodbc` and a
|
|
17
|
+
Microsoft ODBC driver are already installed, it can use them; they are never required.
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
pip install mssql-ocp-migrate # from PyPI
|
|
21
|
+
pip install --no-index mssql_ocp_migrate-0.2.0-py3-none-any.whl # air-gapped: one file, nothing else
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Pick the path by size
|
|
25
|
+
|
|
26
|
+
| Data | Use | Why |
|
|
27
|
+
|---|---|---|
|
|
28
|
+
| up to a few TB, or subsets / schema changes / different collation | `migrate` (logical copy) | row-level control, per-table filters, scale out with `--shard` across pods |
|
|
29
|
+
| **tens to hundreds of TB, whole databases** | **`lift`** (physical) | SQL Server ships its own compressed pages: no row decoding, no index rebuilds, source stays online, cut-over takes minutes |
|
|
30
|
+
|
|
31
|
+
### `lift`: the 100 TB path
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
mssql-migrate lift plan -c migrate.toml # sizes, real compression ratio + speed from msdb, blockers, timeline
|
|
35
|
+
mssql-migrate lift run -c migrate.toml --follow-hours 168 # seed + keep caught up (source online)
|
|
36
|
+
mssql-migrate lift status -c migrate.toml
|
|
37
|
+
mssql-migrate lift cutover -c migrate.toml --confirm # maintenance window: minutes
|
|
38
|
+
mssql-migrate lift verify -c migrate.toml # row counts for every table + sampled checksums
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
1. **Seed:** a `COPY_ONLY`, compressed, `CHECKSUM` full backup striped over up to 64 files (one backup
|
|
42
|
+
thread per file) to a share both servers can reach. The tool sizes `MAXTRANSFERSIZE` and
|
|
43
|
+
`BUFFERCOUNT` to fit a memory cap. The restore runs `WITH NORECOVERY`, spreading the data files
|
|
44
|
+
over several OpenShift volumes so writes run in parallel, and verifies page checksums as it goes.
|
|
45
|
+
2. **Catch-up:** log backups every `log_interval_seconds`, restored continuously. Log backups taken
|
|
46
|
+
by your *existing* backup jobs are read from `msdb` and applied too (paths translated with
|
|
47
|
+
`path_map`), so the chain never breaks. A missing log, a backup sent to `NUL`, or a third-party
|
|
48
|
+
(VDI) log backup stops the loop with a clear message instead of silently corrupting the chain.
|
|
49
|
+
3. **Cut-over:** a final catch-up, then a tail-log backup `WITH NORECOVERY`. The source goes to
|
|
50
|
+
RESTORING, so nothing can write to it, and rollback is a single `RESTORE ... WITH RECOVERY` on the
|
|
51
|
+
source. Then the last restore, `RECOVERY`, and the post-steps: SQL logins with the same SID and
|
|
52
|
+
password hash (no orphaned users), database owner, TRUSTWORTHY / DB_CHAINING / Service Broker,
|
|
53
|
+
orphaned-user re-mapping, and optionally `DBCC CHECKDB WITH PHYSICAL_ONLY`.
|
|
54
|
+
4. **TDE:** the certificate and private key are copied over the connection
|
|
55
|
+
(`CERTENCODED` / `CERTPRIVATEKEY` → `CREATE CERTIFICATE ... FROM BINARY`), with no files on either
|
|
56
|
+
server.
|
|
57
|
+
|
|
58
|
+
Timeline for **100 TB, ~3:1 compression** (`lift plan` prints this from your own msdb history):
|
|
59
|
+
|
|
60
|
+
| Stage | 10 Gbps link, 2 GB/s target storage | 25 Gbps, 4 GB/s |
|
|
61
|
+
|---|---|---|
|
|
62
|
+
| full backup (source online; assumes the source can read ≥3 GB/s across its files) | ~10 h | ~4–7 h (source read speed bound) |
|
|
63
|
+
| restore on OCP (source online) | ~15 h | ~7 h |
|
|
64
|
+
| log catch-up | minutes to ~1 h | minutes |
|
|
65
|
+
| **cut-over downtime** | **~5–15 min** | **~5–15 min** |
|
|
66
|
+
|
|
67
|
+
The restore usually takes longest. It writes the full uncompressed size, so give the SQL Server pod
|
|
68
|
+
4–8 data volumes on the fastest storage class (`deploy/openshift/lift/02-sqlserver-volumes-patch.yaml`).
|
|
69
|
+
With S3, the HTTPS calls come from SQL Server itself, not from this package. S3 is supported for SQL Server 2022, but one backup set is capped at about 12 TB (64 URLs × 10,000 parts
|
|
70
|
+
× 20 MB), so a 100 TB database needs `transport = "share"` (NFS/SMB). `lift plan` reports this as a blocker.
|
|
71
|
+
|
|
72
|
+
**Option B, no backup files** (Enterprise/Developer on both sides): `mssql-migrate lift ag-script`
|
|
73
|
+
prints a cluster-less availability group (`CLUSTER_TYPE = NONE`, `SEEDING_MODE = AUTOMATIC`, trace
|
|
74
|
+
flag 9567 to compress the seeding stream) that streams the database straight into the OpenShift pod
|
|
75
|
+
and keeps it synchronized. Cut-over is a manual failover.
|
|
76
|
+
|
|
77
|
+
## Quick start
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
mssql-migrate init-config > migrate.toml # edit hosts, database, users
|
|
81
|
+
export SRC_DB_PASSWORD=... # or password_file = an OpenShift secret mount
|
|
82
|
+
mssql-migrate check -c migrate.toml # connectivity, versions, table inventory, warnings
|
|
83
|
+
mssql-migrate estimate -c migrate.toml # measures real throughput, projects total time
|
|
84
|
+
mssql-migrate migrate -c migrate.toml # schema -> data -> indexes/FKs -> objects -> validation
|
|
85
|
+
mssql-migrate validate -c migrate.toml # re-run any time (for example right before cut-over)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Python API:
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from mssql_ocp_migrate import Config, Migrator
|
|
92
|
+
run = Migrator(Config.load("migrate.toml")).migrate()
|
|
93
|
+
print(run.ok, run.rows, run.validation.summary())
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Each run writes `migration_report/report.html` (evidence for sign-off) and `report.json`.
|
|
97
|
+
|
|
98
|
+
## What `migrate` does
|
|
99
|
+
|
|
100
|
+
| Phase | Details |
|
|
101
|
+
|---|---|
|
|
102
|
+
| Preflight | Reads the catalog: tables, columns, alias types, identity, computed, defaults, PK/unique/indexes (incl. filtered, INCLUDE, columnstore), FKs, checks, sequences, views, procs, functions, triggers, synonyms. Refuses a non-empty target unless told to `truncate` or `resume`. |
|
|
103
|
+
| Schema | Creates schemas, alias types, sequences (at the source's current value), and tables with their primary keys. |
|
|
104
|
+
| Copy | Splits each table into key ranges (`chunk_rows`, single or composite keys) and copies them in parallel with TDS bulk insert (`INSERT BULK`). Each chunk is one transaction that first clears its own range, so a retry or a resume never duplicates rows. Identity values are preserved, and the strategy is probed on the target at runtime. `datetime2(7)`, `time(7)`, `datetimeoffset(7)` and `datetime` are moved as ISO text, so no 100 ns ticks are lost. Geography, geometry and hierarchyid are moved as varbinary. |
|
|
105
|
+
| Post-load | Nonclustered/unique indexes, check constraints and foreign keys `WITH CHECK` (trusted), identity reseed, then views/functions/procs/synonyms with a dependency-retry loop, triggers last (disabled ones stay disabled). |
|
|
106
|
+
| Validate | See below. |
|
|
107
|
+
|
|
108
|
+
`mssql-migrate schema -c migrate.toml -o schema.sql` writes the DDL without running it, for DBA review.
|
|
109
|
+
|
|
110
|
+
## Validation (end-to-end testing)
|
|
111
|
+
|
|
112
|
+
| `validation.mode` | What is compared |
|
|
113
|
+
|---|---|
|
|
114
|
+
| `count` | Exact `COUNT_BIG(*)` per table, schema parity (columns, types, nullability, collation, identity, PK, indexes, FKs, checks), objects present, identity current values. |
|
|
115
|
+
| `checksum` (default) | The `count` checks, plus a per-chunk `COUNT` / `SUM(BINARY_CHECKSUM)` / `CHECKSUM_AGG` computed inside both servers (no data crosses the network; LOB, xml and CLR columns are hashed with SHA-256), plus an exact row-by-row SHA-256 comparison of a random `sample_full_pct` % of chunks. Any chunk whose checksum differs is compared row by row automatically, and the report lists the exact keys that are missing, extra or changed. |
|
|
116
|
+
| `full` | Every row of every table is hashed with SHA-256 on both sides and compared by key. Keyless tables use an order-independent multiset digest, which also catches duplicate-row differences. |
|
|
117
|
+
|
|
118
|
+
`rowversion` columns are excluded by design, because the target assigns new values.
|
|
119
|
+
|
|
120
|
+
## Scaling the logical copy out
|
|
121
|
+
|
|
122
|
+
The bundled pure-Python driver moves about 2–4 MB/s per worker process. To go wider, run the copy as
|
|
123
|
+
an OpenShift **Indexed Job**: `migrate --shard auto/16` on 16 pods. Each pod copies the chunks that
|
|
124
|
+
hash to its index. Shard 1 creates the schema and writes the shared chunk plan. Then
|
|
125
|
+
`mssql-migrate finalize --shard-count 16` builds indexes, constraints and objects and validates.
|
|
126
|
+
`target_recovery_simple = true` keeps bulk loads minimally logged, and the original recovery model is
|
|
127
|
+
restored at the end. See `deploy/openshift/scale-out/indexed-job.yaml`.
|
|
128
|
+
|
|
129
|
+
## How long will it take?
|
|
130
|
+
|
|
131
|
+
Run `mssql-migrate estimate`. It reads a sample from your largest table, bulk-loads it into a
|
|
132
|
+
`#temp` table on the target, times a server-side checksum, and projects copy, index and validation
|
|
133
|
+
time for your `workers` setting. Rough guide for the bundled pure-Python driver with
|
|
134
|
+
`executor = "auto"` (one process per worker):
|
|
135
|
+
|
|
136
|
+
| Per worker | 8 workers | 16 workers |
|
|
137
|
+
|---|---|---|
|
|
138
|
+
| ~15–25k rows/s (~2–4 MB/s) for a 10–15-column row | ~55–100 GB/hour | ~100–180 GB/hour, if the network and the target's log disk keep up |
|
|
139
|
+
|
|
140
|
+
Add about 20–30 % for index and FK builds. Checksum validation scans both databases on the server,
|
|
141
|
+
which usually takes minutes per 100 GB. With `pyodbc` + ODBC Driver 18 already installed,
|
|
142
|
+
per-stream throughput is typically 2–4× higher.
|
|
143
|
+
|
|
144
|
+
For multi-terabyte databases, a native `BACKUP` → copy into the pod → `RESTORE` is the fastest
|
|
145
|
+
lift. You can still run `mssql-migrate validate` afterwards to prove parity.
|
|
146
|
+
|
|
147
|
+
## Running against OpenShift
|
|
148
|
+
|
|
149
|
+
**From a workstation or VM:** `oc port-forward svc/mssql 14330:1433 -n <ns>`, then point `[target]` at
|
|
150
|
+
`127.0.0.1:14330`. Set `hostname_in_certificate` if you verify the server certificate.
|
|
151
|
+
|
|
152
|
+
**Inside the cluster (recommended for large data):** see `deploy/openshift/`:
|
|
153
|
+
`Containerfile` builds an offline image (`pip install --no-index` of the wheel), and `configmap.yaml`,
|
|
154
|
+
`secret.yaml` and `job.yaml` run it as a Job. A PVC keeps the checkpoint and the report, so a
|
|
155
|
+
failed or evicted Job resumes where it stopped (`if_target_not_empty = "resume"`). The source SQL
|
|
156
|
+
Server must be reachable from the cluster (egress / firewall on 1433).
|
|
157
|
+
|
|
158
|
+
## Cut-over pattern for a live source
|
|
159
|
+
|
|
160
|
+
1. `migrate` while the application is still running (bulk of the data, hours).
|
|
161
|
+
2. Stop writes (application maintenance window, or `ALTER DATABASE ... SET READ_ONLY`).
|
|
162
|
+
3. `migrate --truncate --include <tables that changed>`, or a full re-run for small databases.
|
|
163
|
+
4. `validate --mode checksum` (or `full`), keep the HTML report as evidence, then switch connection strings.
|
|
164
|
+
|
|
165
|
+
## Permissions
|
|
166
|
+
|
|
167
|
+
- **Source:** `db_datareader` + `VIEW DEFINITION` (+ `VIEW DATABASE STATE` for size estimates).
|
|
168
|
+
- **Target:** `db_owner` on the pre-created, empty database. The tool creates the schema, and bulk insert needs INSERT plus ALTER on each table. `ADMINISTER BULK OPERATIONS` is not needed, because no files are read on the server.
|
|
169
|
+
|
|
170
|
+
## Not migrated (reported as warnings)
|
|
171
|
+
|
|
172
|
+
Logins, users and permissions; SQL Agent jobs; memory-optimized tables are created disk-based;
|
|
173
|
+
system-versioned temporal tables arrive as two normal tables (re-enable `SYSTEM_VERSIONING`
|
|
174
|
+
afterwards); partitioning, XML and spatial indexes, full-text; FileTables; `WITH ENCRYPTION`
|
|
175
|
+
modules (their definitions cannot be read).
|
|
176
|
+
|
|
177
|
+
## Development
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
pip install -e . pytest && pytest # unit + engine (SQLite) + TLS wire tests
|
|
181
|
+
docker compose -f tests/e2e/docker-compose.yml up -d
|
|
182
|
+
MSSQL_E2E=1 pytest tests/e2e -v # real SQL Server 2022 source -> target
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
Releasing: bump the version in `pyproject.toml` and `src/mssql_ocp_migrate/__init__.py`, update
|
|
186
|
+
`CHANGELOG.md`, push `main`, then publish a GitHub release `vX.Y.Z`. `.github/workflows/python-publish.yml`
|
|
187
|
+
runs the tests and publishes to PyPI with Trusted Publishing (environment `release`, no stored token).
|
|
188
|
+
|
|
189
|
+
Third-party notices: see `THIRD_PARTY_NOTICES.md`.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Security Policy
|
|
2
|
+
|
|
3
|
+
## Supported Versions
|
|
4
|
+
|
|
5
|
+
| Version | Supported |
|
|
6
|
+
|---------|-----------|
|
|
7
|
+
| 0.2.x | ✅ Active |
|
|
8
|
+
| 0.1.x | ❌ |
|
|
9
|
+
|
|
10
|
+
## Reporting a Vulnerability
|
|
11
|
+
|
|
12
|
+
**Please do not open a public GitHub issue for security vulnerabilities.**
|
|
13
|
+
|
|
14
|
+
Report vulnerabilities privately via GitHub's [Security Advisories](https://github.com/karthikbd/mssql-ocp-migrate/security/advisories/new).
|
|
15
|
+
|
|
16
|
+
You can expect:
|
|
17
|
+
- **Acknowledgement** within 48 hours
|
|
18
|
+
- **Status update** within 7 days
|
|
19
|
+
- **Fix or mitigation** as soon as reasonably possible, coordinated with a disclosure date
|
|
20
|
+
|
|
21
|
+
## Network access
|
|
22
|
+
|
|
23
|
+
`mssql-ocp-migrate` has no telemetry, no update check and no HTTP calls. It declares **no runtime
|
|
24
|
+
dependencies** (the SQL Server driver is bundled), so `pip install` downloads nothing else. A test
|
|
25
|
+
fails the build if any module imports an HTTP client, `pip` or `subprocess`.
|
|
26
|
+
|
|
27
|
+
It opens connections only to the servers named in your configuration:
|
|
28
|
+
|
|
29
|
+
| Connection | Why |
|
|
30
|
+
|------------|-----|
|
|
31
|
+
| `[source]` host, TCP (TDS) | read the catalog and data; for `lift`, run BACKUP and read `msdb` history |
|
|
32
|
+
| `[target]` host, TCP (TDS) | create the schema, load data, validate; for `lift`, run RESTORE |
|
|
33
|
+
| SQL Browser, UDP 1434 | only when `instance` is set, to resolve a named instance |
|
|
34
|
+
|
|
35
|
+
With `lift.transport = "s3"`, **SQL Server itself** makes the HTTPS calls to your S3 endpoint; the
|
|
36
|
+
package only sends the T-SQL.
|
|
37
|
+
|
|
38
|
+
## Credentials and data
|
|
39
|
+
|
|
40
|
+
- Passwords come from `password_file` (e.g. an OpenShift Secret) or `password_env`; reports redact them.
|
|
41
|
+
- `lift` copies SQL logins with their existing SID and **password hash**; no clear-text password is
|
|
42
|
+
read or written. The TDE private key is exported encrypted with a one-time random password and
|
|
43
|
+
moved over the connection, never to disk.
|
|
44
|
+
- Backup files written by `lift` are compressed, not encrypted: protect the share.
|
|
45
|
+
|
|
46
|
+
See the [Security page](https://mssql-ocp-migrate.readthedocs.io/en/latest/user_guide/security.html)
|
|
47
|
+
of the documentation for permissions and TLS settings.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Third-party code bundled in mssql-ocp-migrate
|
|
2
|
+
|
|
3
|
+
## python-tds 1.17.1 — `src/mssql_ocp_migrate/_vendor/pytds/`
|
|
4
|
+
- Upstream: https://github.com/denisenkom/pytds — MIT License (`_vendor/LICENSE-python-tds.txt`)
|
|
5
|
+
- Modifications:
|
|
6
|
+
- absolute `pytds` imports rewritten to `mssql_ocp_migrate._vendor.pytds` (no clash with an installed pytds)
|
|
7
|
+
- `tls.py` re-implemented on the standard-library `ssl` module (MemoryBIO/SSLObject) instead of pyOpenSSL
|
|
8
|
+
- `connect()` gained a `tls_server_hostname` argument (certificate host-name override)
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Offline image: the wheel has zero dependencies, so `pip install --no-index` never touches a network.
|
|
2
|
+
# python -m pip wheel . -w dist --no-deps (on a build machine)
|
|
3
|
+
# podman build -f deploy/openshift/Containerfile -t mssql-ocp-migrate:0.2.0 .
|
|
4
|
+
FROM registry.access.redhat.com/ubi9/python-311:latest
|
|
5
|
+
COPY dist/mssql_ocp_migrate-*.whl /tmp/
|
|
6
|
+
RUN pip install --no-index --no-deps /tmp/mssql_ocp_migrate-*.whl && rm -f /tmp/*.whl
|
|
7
|
+
WORKDIR /work
|
|
8
|
+
USER 1001
|
|
9
|
+
ENTRYPOINT ["mssql-migrate"]
|
|
10
|
+
CMD ["--help"]
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
apiVersion: v1
|
|
2
|
+
kind: ConfigMap
|
|
3
|
+
metadata:
|
|
4
|
+
name: mssql-migrate-config
|
|
5
|
+
data:
|
|
6
|
+
migrate.toml: |
|
|
7
|
+
report_dir = "/work/data/report"
|
|
8
|
+
|
|
9
|
+
[source]
|
|
10
|
+
host = "sql01.corp.example.com" # must be reachable from the cluster (egress / firewall)
|
|
11
|
+
port = 1433
|
|
12
|
+
database = "SalesDB"
|
|
13
|
+
user = "migration_reader"
|
|
14
|
+
password_file = "/secrets/source-password"
|
|
15
|
+
encrypt = "optional"
|
|
16
|
+
trust_server_certificate = true
|
|
17
|
+
|
|
18
|
+
[target]
|
|
19
|
+
host = "mssql.sqlserver.svc.cluster.local" # the SQL Server Service inside OCP
|
|
20
|
+
port = 1433
|
|
21
|
+
database = "SalesDB"
|
|
22
|
+
user = "migration_writer"
|
|
23
|
+
password_file = "/secrets/target-password"
|
|
24
|
+
encrypt = "optional"
|
|
25
|
+
trust_server_certificate = true
|
|
26
|
+
|
|
27
|
+
[migration]
|
|
28
|
+
workers = 8
|
|
29
|
+
executor = "auto"
|
|
30
|
+
chunk_rows = 250000
|
|
31
|
+
if_target_not_empty = "resume" # safe to re-run the Job after a failure
|
|
32
|
+
state_file = "/work/data/migration_state.json"
|
|
33
|
+
|
|
34
|
+
[validation]
|
|
35
|
+
mode = "checksum"
|
|
36
|
+
sample_full_pct = 5
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# The PVC keeps the checkpoint + report, so a failed/evicted Job resumes where it stopped.
|
|
2
|
+
apiVersion: v1
|
|
3
|
+
kind: PersistentVolumeClaim
|
|
4
|
+
metadata:
|
|
5
|
+
name: mssql-migrate-work
|
|
6
|
+
spec:
|
|
7
|
+
accessModes: ["ReadWriteOnce"]
|
|
8
|
+
resources:
|
|
9
|
+
requests:
|
|
10
|
+
storage: 1Gi
|
|
11
|
+
---
|
|
12
|
+
apiVersion: batch/v1
|
|
13
|
+
kind: Job
|
|
14
|
+
metadata:
|
|
15
|
+
name: mssql-migrate
|
|
16
|
+
spec:
|
|
17
|
+
backoffLimit: 3
|
|
18
|
+
template:
|
|
19
|
+
spec:
|
|
20
|
+
restartPolicy: Never
|
|
21
|
+
containers:
|
|
22
|
+
- name: migrate
|
|
23
|
+
image: image-registry.openshift-image-registry.svc:5000/sqlserver/mssql-ocp-migrate:0.2.0
|
|
24
|
+
args: ["migrate", "-c", "/config/migrate.toml"]
|
|
25
|
+
resources:
|
|
26
|
+
requests: { cpu: "2", memory: "2Gi" }
|
|
27
|
+
limits: { cpu: "8", memory: "6Gi" }
|
|
28
|
+
volumeMounts:
|
|
29
|
+
- { name: config, mountPath: /config }
|
|
30
|
+
- { name: secrets, mountPath: /secrets, readOnly: true }
|
|
31
|
+
- { name: work, mountPath: /work/data }
|
|
32
|
+
volumes:
|
|
33
|
+
- name: config
|
|
34
|
+
configMap: { name: mssql-migrate-config }
|
|
35
|
+
- name: secrets
|
|
36
|
+
secret: { secretName: mssql-migrate-creds }
|
|
37
|
+
- name: work
|
|
38
|
+
persistentVolumeClaim: { claimName: mssql-migrate-work }
|