pydtsx-parser 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pydtsx_parser-0.1.0/CHANGELOG.md +36 -0
- pydtsx_parser-0.1.0/CONTRIBUTING.md +92 -0
- pydtsx_parser-0.1.0/LICENSE +21 -0
- pydtsx_parser-0.1.0/MANIFEST.in +3 -0
- pydtsx_parser-0.1.0/PKG-INFO +280 -0
- pydtsx_parser-0.1.0/README.md +251 -0
- pydtsx_parser-0.1.0/SECURITY.md +41 -0
- pydtsx_parser-0.1.0/pydtsx_parser/__init__.py +3 -0
- pydtsx_parser-0.1.0/pydtsx_parser/__main__.py +8 -0
- pydtsx_parser-0.1.0/pydtsx_parser/cli.py +120 -0
- pydtsx_parser-0.1.0/pydtsx_parser/constants.py +41 -0
- pydtsx_parser-0.1.0/pydtsx_parser/dispatcher.py +373 -0
- pydtsx_parser-0.1.0/pydtsx_parser/envelope.py +164 -0
- pydtsx_parser-0.1.0/pydtsx_parser/errors.py +37 -0
- pydtsx_parser-0.1.0/pydtsx_parser/extractors/__init__.py +1 -0
- pydtsx_parser-0.1.0/pydtsx_parser/extractors/columns.py +157 -0
- pydtsx_parser-0.1.0/pydtsx_parser/extractors/components.py +284 -0
- pydtsx_parser-0.1.0/pydtsx_parser/extractors/connections.py +349 -0
- pydtsx_parser-0.1.0/pydtsx_parser/extractors/executables.py +118 -0
- pydtsx_parser-0.1.0/pydtsx_parser/extractors/paths.py +133 -0
- pydtsx_parser-0.1.0/pydtsx_parser/extractors/pipeline.py +115 -0
- pydtsx_parser-0.1.0/pydtsx_parser/extractors/precedence.py +134 -0
- pydtsx_parser-0.1.0/pydtsx_parser/extractors/sort.py +156 -0
- pydtsx_parser-0.1.0/pydtsx_parser/extractors/sources.py +130 -0
- pydtsx_parser-0.1.0/pydtsx_parser/extractors/sql_tasks.py +195 -0
- pydtsx_parser-0.1.0/pydtsx_parser/extractors/transformations.py +371 -0
- pydtsx_parser-0.1.0/pydtsx_parser/extractors/variables.py +76 -0
- pydtsx_parser-0.1.0/pydtsx_parser/mcp_entry.py +31 -0
- pydtsx_parser-0.1.0/pydtsx_parser/mcp_server.py +499 -0
- pydtsx_parser-0.1.0/pydtsx_parser/parsers/__init__.py +1 -0
- pydtsx_parser-0.1.0/pydtsx_parser/parsers/conmgr.py +67 -0
- pydtsx_parser-0.1.0/pydtsx_parser/parsers/dtproj.py +470 -0
- pydtsx_parser-0.1.0/pydtsx_parser/parsers/dtsx.py +156 -0
- pydtsx_parser-0.1.0/pydtsx_parser/parsers/params.py +132 -0
- pydtsx_parser-0.1.0/pydtsx_parser/py.typed +0 -0
- pydtsx_parser-0.1.0/pydtsx_parser/redaction.py +282 -0
- pydtsx_parser-0.1.0/pydtsx_parser/xml_utils.py +185 -0
- pydtsx_parser-0.1.0/pydtsx_parser.egg-info/PKG-INFO +280 -0
- pydtsx_parser-0.1.0/pydtsx_parser.egg-info/SOURCES.txt +75 -0
- pydtsx_parser-0.1.0/pydtsx_parser.egg-info/dependency_links.txt +1 -0
- pydtsx_parser-0.1.0/pydtsx_parser.egg-info/entry_points.txt +3 -0
- pydtsx_parser-0.1.0/pydtsx_parser.egg-info/requires.txt +3 -0
- pydtsx_parser-0.1.0/pydtsx_parser.egg-info/top_level.txt +1 -0
- pydtsx_parser-0.1.0/pyproject.toml +101 -0
- pydtsx_parser-0.1.0/setup.cfg +4 -0
- pydtsx_parser-0.1.0/tests/test_cli.py +268 -0
- pydtsx_parser-0.1.0/tests/test_components_smoke.py +164 -0
- pydtsx_parser-0.1.0/tests/test_conmgr.py +274 -0
- pydtsx_parser-0.1.0/tests/test_connections.py +856 -0
- pydtsx_parser-0.1.0/tests/test_derived_columns.py +586 -0
- pydtsx_parser-0.1.0/tests/test_dispatcher.py +321 -0
- pydtsx_parser-0.1.0/tests/test_dtproj.py +369 -0
- pydtsx_parser-0.1.0/tests/test_dtsx.py +513 -0
- pydtsx_parser-0.1.0/tests/test_edge_cases.py +632 -0
- pydtsx_parser-0.1.0/tests/test_envelope.py +169 -0
- pydtsx_parser-0.1.0/tests/test_executables.py +421 -0
- pydtsx_parser-0.1.0/tests/test_integration.py +337 -0
- pydtsx_parser-0.1.0/tests/test_mcp_entry.py +60 -0
- pydtsx_parser-0.1.0/tests/test_mcp_server.py +145 -0
- pydtsx_parser-0.1.0/tests/test_merge_join.py +929 -0
- pydtsx_parser-0.1.0/tests/test_params.py +279 -0
- pydtsx_parser-0.1.0/tests/test_pipeline.py +252 -0
- pydtsx_parser-0.1.0/tests/test_precedence.py +333 -0
- pydtsx_parser-0.1.0/tests/test_property_connections.py +802 -0
- pydtsx_parser-0.1.0/tests/test_property_dataflow.py +795 -0
- pydtsx_parser-0.1.0/tests/test_property_dtsx.py +517 -0
- pydtsx_parser-0.1.0/tests/test_property_envelope.py +602 -0
- pydtsx_parser-0.1.0/tests/test_property_precedence.py +294 -0
- pydtsx_parser-0.1.0/tests/test_property_redaction.py +560 -0
- pydtsx_parser-0.1.0/tests/test_property_transformations.py +912 -0
- pydtsx_parser-0.1.0/tests/test_real_world.py +201 -0
- pydtsx_parser-0.1.0/tests/test_redaction.py +303 -0
- pydtsx_parser-0.1.0/tests/test_sort.py +557 -0
- pydtsx_parser-0.1.0/tests/test_sources.py +356 -0
- pydtsx_parser-0.1.0/tests/test_sql_tasks.py +325 -0
- pydtsx_parser-0.1.0/tests/test_variables.py +316 -0
- pydtsx_parser-0.1.0/tests/test_xml_utils.py +236 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.1.0] - 2026-08-29
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- Initial public release.
|
|
15
|
+
- Parsers for the four SSIS project file types: `.dtsx` packages, `.dtproj`
|
|
16
|
+
projects, `.conmgr` connection managers, and `.params` project parameters.
|
|
17
|
+
- Directory dispatcher that recursively discovers and cross-references all
|
|
18
|
+
files in an SSIS project.
|
|
19
|
+
- Data flow pipeline extraction: components, paths, error outputs, and
|
|
20
|
+
topological ordering.
|
|
21
|
+
- Control flow extraction: executables, precedence constraints, and Execute SQL
|
|
22
|
+
Task contents.
|
|
23
|
+
- Transformation extractors for derived columns, lookups, merge joins, sorts,
|
|
24
|
+
and column mappings.
|
|
25
|
+
- Self-describing JSON envelope carrying format/parser versions, source file
|
|
26
|
+
metadata, a data type map, and a completeness summary.
|
|
27
|
+
- Automatic credential redaction for sensitive fields and connection strings,
|
|
28
|
+
which deliberately preserves password-like *column* names as schema metadata.
|
|
29
|
+
- CLI entry point `pydtsx-parser` with `--output` and `--pretty`.
|
|
30
|
+
- Optional MCP server (`pip install "pydtsx-parser[mcp]"`) exposing package
|
|
31
|
+
summary, SQL extraction, and data lineage tools.
|
|
32
|
+
- Claude Skill for agent-driven use without a server.
|
|
33
|
+
- Type hints throughout, with a PEP 561 `py.typed` marker.
|
|
34
|
+
|
|
35
|
+
[Unreleased]: https://github.com/lamiskin/pydtsx-parser/compare/v0.1.0...HEAD
|
|
36
|
+
[0.1.0]: https://github.com/lamiskin/pydtsx-parser/releases/tag/v0.1.0
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Thanks for your interest in improving `pydtsx-parser`.
|
|
4
|
+
|
|
5
|
+
## The one rule that matters most
|
|
6
|
+
|
|
7
|
+
**Never contribute real SSIS packages, extracts, or production data.**
|
|
8
|
+
|
|
9
|
+
SSIS files are unusually leaky: they routinely embed internal server names, UNC
|
|
10
|
+
paths, database schemas, domain usernames, and sometimes credentials. Everything
|
|
11
|
+
in this repository — every test input, every example, every issue attachment —
|
|
12
|
+
must be synthetic.
|
|
13
|
+
|
|
14
|
+
If you are debugging against a real package, reduce it to a minimal synthetic
|
|
15
|
+
snippet that reproduces the behaviour before opening an issue or PR.
|
|
16
|
+
|
|
17
|
+
## Setup
|
|
18
|
+
|
|
19
|
+
This project uses [uv](https://docs.astral.sh/uv/).
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
git clone https://github.com/lamiskin/pydtsx-parser
|
|
23
|
+
cd pydtsx-parser
|
|
24
|
+
uv sync --all-extras
|
|
25
|
+
uv run pre-commit install
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Everyday commands
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
uv run pytest # run the test suite
|
|
32
|
+
uv run pytest --cov # with coverage
|
|
33
|
+
uv run ruff check --fix # lint
|
|
34
|
+
uv run ruff format # format
|
|
35
|
+
uv run mypy # type check
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
All four must pass before a PR can merge; CI runs them on Python 3.11–3.14
|
|
39
|
+
across Linux, Windows, and macOS.
|
|
40
|
+
|
|
41
|
+
## Testing against your own packages
|
|
42
|
+
|
|
43
|
+
Integration tests look for an optional `examples/` directory at the repository
|
|
44
|
+
root and skip cleanly when it is absent. You can drop your own SSIS projects
|
|
45
|
+
there to exercise the parser locally — `examples/` is gitignored, so they will
|
|
46
|
+
not be committed. Keep it that way.
|
|
47
|
+
|
|
48
|
+
## Commit messages
|
|
49
|
+
|
|
50
|
+
This project uses [Conventional Commits](https://www.conventionalcommits.org/).
|
|
51
|
+
Release automation reads them, so the prefix determines the version bump:
|
|
52
|
+
|
|
53
|
+
| Prefix | Effect |
|
|
54
|
+
|---|---|
|
|
55
|
+
| `fix:` | patch release |
|
|
56
|
+
| `feat:` | minor release |
|
|
57
|
+
| `feat!:` or `BREAKING CHANGE:` | major release |
|
|
58
|
+
| `docs:`, `ci:`, `test:`, `chore:`, `refactor:` | no release |
|
|
59
|
+
|
|
60
|
+
Example: `feat: extract Lookup transformation cache settings`
|
|
61
|
+
|
|
62
|
+
## Releases are automated
|
|
63
|
+
|
|
64
|
+
Do not bump versions or edit `CHANGELOG.md` by hand.
|
|
65
|
+
[release-please](https://github.com/googleapis/release-please) watches `main`,
|
|
66
|
+
maintains a release PR, and merging it tags a GitHub Release, which publishes to
|
|
67
|
+
PyPI via Trusted Publishing.
|
|
68
|
+
|
|
69
|
+
### Bootstrapping the first release (maintainers)
|
|
70
|
+
|
|
71
|
+
release-please treats `.release-please-manifest.json` as the last *released*
|
|
72
|
+
version, so it will only ever propose versions **after** it. The version
|
|
73
|
+
currently recorded there has to be tagged and released by hand once, or it never
|
|
74
|
+
reaches PyPI:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
git tag -a v1.0.0 -m "pydtsx-parser 1.0.0" && git push origin v1.0.0
|
|
78
|
+
gh release create v1.0.0 --notes-from-tag
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Before that, dry-run the whole pipeline without touching real PyPI: run the
|
|
82
|
+
**Publish** workflow manually (`workflow_dispatch`) with target `testpypi`. It
|
|
83
|
+
builds, validates the metadata, smoke-tests the wheel, and publishes to TestPyPI
|
|
84
|
+
only — the PyPI job is unreachable from that trigger.
|
|
85
|
+
|
|
86
|
+
## Adding a new extractor
|
|
87
|
+
|
|
88
|
+
Extractors live in `pydtsx_parser/extractors/` and take an `ET.Element`,
|
|
89
|
+
returning plain dicts and lists. Keep them pure — file I/O belongs in
|
|
90
|
+
`pydtsx_parser/parsers/`. Add the extractor to the relevant parser, then cover
|
|
91
|
+
it with both example-based tests and, where the shape allows, a Hypothesis
|
|
92
|
+
property test.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Lachlan Miskin
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pydtsx-parser
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Parse SQL Server Integration Services (SSIS) .dtsx, .dtproj, .conmgr and .params files into structured JSON
|
|
5
|
+
Author-email: Lachlan Miskin <lamiskin@users.noreply.github.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/lamiskin/pydtsx-parser
|
|
8
|
+
Project-URL: Repository, https://github.com/lamiskin/pydtsx-parser
|
|
9
|
+
Project-URL: Issues, https://github.com/lamiskin/pydtsx-parser/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/lamiskin/pydtsx-parser/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: ssis,dtsx,etl,sql-server,parser,data-engineering,lineage
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Topic :: Database
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.11
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Provides-Extra: mcp
|
|
27
|
+
Requires-Dist: mcp>=2.0.0; extra == "mcp"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
30
|
+
# pydtsx-parser
|
|
31
|
+
|
|
32
|
+
[](https://github.com/lamiskin/pydtsx-parser/actions/workflows/ci.yml)
|
|
33
|
+
[](https://codecov.io/gh/lamiskin/pydtsx-parser)
|
|
34
|
+
[](https://pypi.org/project/pydtsx-parser/)
|
|
35
|
+
[](https://pypi.org/project/pydtsx-parser/)
|
|
36
|
+
[](https://github.com/lamiskin/pydtsx-parser/actions/workflows/codeql.yml)
|
|
37
|
+
[](https://lamiskin.github.io/pydtsx-parser/)
|
|
38
|
+
[](LICENSE)
|
|
39
|
+
|
|
40
|
+
Parse SQL Server Integration Services (SSIS) project files into structured,
|
|
41
|
+
self-describing JSON — with no SSIS installation, no SQL Server, and no runtime
|
|
42
|
+
dependencies.
|
|
43
|
+
|
|
44
|
+
Handles the four file types that make up an SSIS project:
|
|
45
|
+
|
|
46
|
+
| File | What it holds |
|
|
47
|
+
|---|---|
|
|
48
|
+
| `.dtsx` | Packages: control flow, data flow pipelines, variables, connections |
|
|
49
|
+
| `.dtproj` | Project manifest, deployment model, package list |
|
|
50
|
+
| `.conmgr` | Project-level connection managers |
|
|
51
|
+
| `.params` | Project parameters |
|
|
52
|
+
|
|
53
|
+
## Why
|
|
54
|
+
|
|
55
|
+
SSIS packages are large, deeply nested XML files that are painful to read and
|
|
56
|
+
awkward to diff. If you are migrating away from SSIS, auditing what a package
|
|
57
|
+
actually does, or documenting an inherited ETL estate, you need the *structure*
|
|
58
|
+
out of that XML — task graphs, data lineage, embedded SQL, column mappings —
|
|
59
|
+
without opening Visual Studio.
|
|
60
|
+
|
|
61
|
+
`pydtsx-parser` extracts all of it into one JSON envelope designed to be
|
|
62
|
+
machine-readable and self-describing: the output carries its own data type map
|
|
63
|
+
and a completeness summary of how many elements and attributes were seen.
|
|
64
|
+
|
|
65
|
+
## Supported SSIS versions
|
|
66
|
+
|
|
67
|
+
The parser is **version-agnostic by design**: it has no version gates and no
|
|
68
|
+
version-specific branches. Version markers such as `LastModifiedProductVersion`
|
|
69
|
+
and the `.dtproj` `ProductVersion` are extracted and reported, but they never
|
|
70
|
+
change how a file is parsed. "Supported" below therefore means *verified*, not
|
|
71
|
+
*enabled* — an unlisted version is likely to parse.
|
|
72
|
+
|
|
73
|
+
| SQL Server / SSIS | Package format | Status | Verified by |
|
|
74
|
+
|---|---|---|---|
|
|
75
|
+
| **2012** (11.0) | `SSIS.Package.3` | Real package | `u2_toolkit/Package.dtsx`, `u2_toolkit/Project.dtproj` |
|
|
76
|
+
| **2014** (12.0) | `Microsoft.Package` | Real package | `u2_toolkit/PackageAzure.dtsx` |
|
|
77
|
+
| **2019** (15.0) | `Microsoft.Package` | Synthetic fixtures | hand-written packages across the suite |
|
|
78
|
+
| **2022** (16.0) | `Microsoft.Package` | Real packages | the four `ssis_examples/*.dtsx` |
|
|
79
|
+
| 2016 / 2017 (13.0 / 14.0) | `Microsoft.Package` | Expected to work, untested | bracketed by the 2014 and 2019 cases |
|
|
80
|
+
| 2008 and earlier | `SSIS.Package.2` and older | Unknown | no sample available; predates the `.dtproj` project deployment model |
|
|
81
|
+
|
|
82
|
+
Both package format generations are covered by real files: the older
|
|
83
|
+
`SSIS.Package.3` form used by SSIS 2012, and the `Microsoft.Package` form used
|
|
84
|
+
from 2014 onward. The real fixtures also span OLE DB, Flat File and ADO.NET
|
|
85
|
+
connection managers — including a third-party ADO.NET provider — and both
|
|
86
|
+
friendly-name and raw-GUID pipeline component class IDs.
|
|
87
|
+
|
|
88
|
+
Two gaps worth stating plainly:
|
|
89
|
+
|
|
90
|
+
- Only the **project deployment model** is verified. The single real `.dtproj`
|
|
91
|
+
declares `DeploymentModel=Project` (schema `9.0.1.0`); the legacy package
|
|
92
|
+
deployment model has no real-file coverage.
|
|
93
|
+
- No real `.conmgr` file was available, so project-level connection managers are
|
|
94
|
+
covered by synthetic fixtures only.
|
|
95
|
+
|
|
96
|
+
Provenance and the sanitisation applied to the real fixtures are documented in
|
|
97
|
+
[`tests/fixtures/real_world/README.md`](tests/fixtures/real_world/README.md).
|
|
98
|
+
|
|
99
|
+
### Handling unknown content
|
|
100
|
+
|
|
101
|
+
With no version gating, an unfamiliar package generally parses. The
|
|
102
|
+
`completeness_summary` on every parse result reports `total_elements` and
|
|
103
|
+
`total_attributes` actually seen, so you can check a package was read in full
|
|
104
|
+
rather than trusting silence. Note that its `skipped_items` field lists XML
|
|
105
|
+
comments and processing instructions — deliberately ignored content — not
|
|
106
|
+
elements the parser failed to understand.
|
|
107
|
+
|
|
108
|
+
Unrecognised executables and pipeline components are still emitted with their
|
|
109
|
+
attributes and properties intact, keyed by whatever `CreationName` or
|
|
110
|
+
`componentClassID` the file declares, so a third-party or newer component
|
|
111
|
+
appears in the output even when the parser has no special knowledge of it.
|
|
112
|
+
|
|
113
|
+
## Install
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
pip install pydtsx-parser
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
With the optional MCP server:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
pip install "pydtsx-parser[mcp]"
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Requires Python 3.11+.
|
|
126
|
+
|
|
127
|
+
## Quick start
|
|
128
|
+
|
|
129
|
+
Parse a single package:
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
pydtsx-parser Package.dtsx --pretty
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
Parse an entire project directory (recursively discovers all four file types and
|
|
136
|
+
cross-references them):
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
pydtsx-parser ./MyProject --pretty --output project.json
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
From Python:
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from pydtsx_parser.dispatcher import dispatch
|
|
146
|
+
|
|
147
|
+
result = dispatch("Package.dtsx")
|
|
148
|
+
print(result["content"]["package_attributes"]["object_name"])
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
### Output shape
|
|
152
|
+
|
|
153
|
+
```json
|
|
154
|
+
{
|
|
155
|
+
"format_version": "1.0.0",
|
|
156
|
+
"parser_version": "0.1.0",
|
|
157
|
+
"source_file_path": "/path/to/Package.dtsx",
|
|
158
|
+
"file_type": "dtsx_package",
|
|
159
|
+
"parsed_at": "2026-01-01T09:00:00+10:00",
|
|
160
|
+
"source_file_metadata": { "file_name": "Package.dtsx", "file_size_bytes": 724, "owner": "..." },
|
|
161
|
+
"data_type_map": { "130": "wstr", "131": "numeric", "...": "..." },
|
|
162
|
+
"redaction_summary": { "total_redacted": 0 },
|
|
163
|
+
"content": {
|
|
164
|
+
"package_attributes": { "object_name": "LoadCustomers", "...": "..." },
|
|
165
|
+
"variables": [ { "name": "BatchDate", "namespace": "User", "data_type": "7" } ],
|
|
166
|
+
"connection_managers": [ { "object_name": "DW", "creation_name": "OLEDB" } ],
|
|
167
|
+
"executables": [],
|
|
168
|
+
"completeness_summary": { "total_elements": 6, "total_attributes": 11, "skipped_items": [] }
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
## CLI
|
|
174
|
+
|
|
175
|
+
```
|
|
176
|
+
usage: pydtsx-parser [-h] [--output OUTPUT] [--pretty] path
|
|
177
|
+
|
|
178
|
+
Parse SSIS files (.dtsx, .dtproj, .conmgr, .params) into JSON.
|
|
179
|
+
|
|
180
|
+
positional arguments:
|
|
181
|
+
path File or directory path to parse
|
|
182
|
+
|
|
183
|
+
options:
|
|
184
|
+
-h, --help show this help message and exit
|
|
185
|
+
--output OUTPUT, -o OUTPUT
|
|
186
|
+
Output file path (default: stdout)
|
|
187
|
+
--pretty, -p Pretty-print JSON with 2-space indent
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
## Credential redaction
|
|
191
|
+
|
|
192
|
+
Passwords are redacted automatically — both as standalone fields and inside
|
|
193
|
+
connection strings:
|
|
194
|
+
|
|
195
|
+
```python
|
|
196
|
+
from pydtsx_parser.redaction import redact
|
|
197
|
+
|
|
198
|
+
redact({"connection_string": "Data Source=dbhost;User ID=svc;Password=hunter2;"})
|
|
199
|
+
# ({'connection_string': 'Data Source=dbhost;User ID=svc;Password=[SENSITIVE - REDACTED];'}, 1)
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
Schema metadata is deliberately left intact — a *column* named `PASSWORD_HASH`
|
|
203
|
+
is structure, not a secret, so it is not redacted.
|
|
204
|
+
|
|
205
|
+
### Handling real packages
|
|
206
|
+
|
|
207
|
+
Redaction covers credentials, not everything an SSIS file can reveal. Parser
|
|
208
|
+
output also includes the source file's absolute path and its filesystem owner,
|
|
209
|
+
and packages routinely embed internal server names, UNC paths, and schema names.
|
|
210
|
+
Review parser output before attaching it to a public issue or sharing it outside
|
|
211
|
+
your organisation.
|
|
212
|
+
|
|
213
|
+
## MCP server
|
|
214
|
+
|
|
215
|
+
`pydtsx-parser` ships an optional [MCP](https://modelcontextprotocol.io) server
|
|
216
|
+
so agents can explore SSIS packages directly. Install the extra, then point your
|
|
217
|
+
client at the `pydtsx-parser-mcp` command:
|
|
218
|
+
|
|
219
|
+
```json
|
|
220
|
+
{
|
|
221
|
+
"mcpServers": {
|
|
222
|
+
"pydtsx-parser": {
|
|
223
|
+
"command": "pydtsx-parser-mcp"
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Tools provided:
|
|
230
|
+
|
|
231
|
+
| Tool | Purpose |
|
|
232
|
+
|---|---|
|
|
233
|
+
| `get_package_summary` | High-level overview — best first call |
|
|
234
|
+
| `get_sql_code` | Extract embedded SQL statements |
|
|
235
|
+
| `get_data_lineage` | Control flow edges plus source → destination tracing |
|
|
236
|
+
| `get_data_flows` | Full data flow component detail and column mappings |
|
|
237
|
+
| `parse_dtsx_file` | Full structured JSON for one file |
|
|
238
|
+
| `parse_ssis_directory` | Full structured JSON for a project |
|
|
239
|
+
|
|
240
|
+
A [Claude Skill](skills/pydtsx-parser/SKILL.md) is also included, for a
|
|
241
|
+
portable, dependency-free way to teach an agent how to use the CLI.
|
|
242
|
+
|
|
243
|
+
## Documentation
|
|
244
|
+
|
|
245
|
+
Full documentation lives at
|
|
246
|
+
**[lamiskin.github.io/pydtsx-parser](https://lamiskin.github.io/pydtsx-parser/)** —
|
|
247
|
+
including the [LLM context guide](docs/LLM_CONTEXT.md), a deep reference for
|
|
248
|
+
interpreting the JSON output.
|
|
249
|
+
|
|
250
|
+
## Development
|
|
251
|
+
|
|
252
|
+
```bash
|
|
253
|
+
uv sync
|
|
254
|
+
uv run pytest
|
|
255
|
+
uv run ruff check
|
|
256
|
+
uv run ruff format --check
|
|
257
|
+
uv run mypy
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
The test suite is fully synthetic — every input is constructed in-memory or
|
|
261
|
+
written to a temp directory. No SSIS packages are bundled with this repository.
|
|
262
|
+
Integration tests look for an optional local `examples/` directory and skip
|
|
263
|
+
cleanly when it is absent, so you can point them at your own packages without
|
|
264
|
+
ever committing them.
|
|
265
|
+
|
|
266
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for the full workflow.
|
|
267
|
+
|
|
268
|
+
## Acknowledgements
|
|
269
|
+
|
|
270
|
+
This project was developed with AI assistance and validated against real-world
|
|
271
|
+
SSIS projects. **None of that data, its identifiers, or its history is included
|
|
272
|
+
in this repository** — no packages, no extracts, no connection details. The
|
|
273
|
+
tests run entirely on synthetic fixtures.
|
|
274
|
+
|
|
275
|
+
## License
|
|
276
|
+
|
|
277
|
+
MIT — see [LICENSE](LICENSE).
|
|
278
|
+
|
|
279
|
+
SQL Server and SQL Server Integration Services are trademarks of Microsoft
|
|
280
|
+
Corporation. This project is not affiliated with or endorsed by Microsoft.
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
# pydtsx-parser
|
|
2
|
+
|
|
3
|
+
[](https://github.com/lamiskin/pydtsx-parser/actions/workflows/ci.yml)
|
|
4
|
+
[](https://codecov.io/gh/lamiskin/pydtsx-parser)
|
|
5
|
+
[](https://pypi.org/project/pydtsx-parser/)
|
|
6
|
+
[](https://pypi.org/project/pydtsx-parser/)
|
|
7
|
+
[](https://github.com/lamiskin/pydtsx-parser/actions/workflows/codeql.yml)
|
|
8
|
+
[](https://lamiskin.github.io/pydtsx-parser/)
|
|
9
|
+
[](LICENSE)
|
|
10
|
+
|
|
11
|
+
Parse SQL Server Integration Services (SSIS) project files into structured,
|
|
12
|
+
self-describing JSON — with no SSIS installation, no SQL Server, and no runtime
|
|
13
|
+
dependencies.
|
|
14
|
+
|
|
15
|
+
Handles the four file types that make up an SSIS project:
|
|
16
|
+
|
|
17
|
+
| File | What it holds |
|
|
18
|
+
|---|---|
|
|
19
|
+
| `.dtsx` | Packages: control flow, data flow pipelines, variables, connections |
|
|
20
|
+
| `.dtproj` | Project manifest, deployment model, package list |
|
|
21
|
+
| `.conmgr` | Project-level connection managers |
|
|
22
|
+
| `.params` | Project parameters |
|
|
23
|
+
|
|
24
|
+
## Why
|
|
25
|
+
|
|
26
|
+
SSIS packages are large, deeply nested XML files that are painful to read and
|
|
27
|
+
awkward to diff. If you are migrating away from SSIS, auditing what a package
|
|
28
|
+
actually does, or documenting an inherited ETL estate, you need the *structure*
|
|
29
|
+
out of that XML — task graphs, data lineage, embedded SQL, column mappings —
|
|
30
|
+
without opening Visual Studio.
|
|
31
|
+
|
|
32
|
+
`pydtsx-parser` extracts all of it into one JSON envelope designed to be
|
|
33
|
+
machine-readable and self-describing: the output carries its own data type map
|
|
34
|
+
and a completeness summary of how many elements and attributes were seen.
|
|
35
|
+
|
|
36
|
+
## Supported SSIS versions
|
|
37
|
+
|
|
38
|
+
The parser is **version-agnostic by design**: it has no version gates and no
|
|
39
|
+
version-specific branches. Version markers such as `LastModifiedProductVersion`
|
|
40
|
+
and the `.dtproj` `ProductVersion` are extracted and reported, but they never
|
|
41
|
+
change how a file is parsed. "Supported" below therefore means *verified*, not
|
|
42
|
+
*enabled* — an unlisted version is likely to parse.
|
|
43
|
+
|
|
44
|
+
| SQL Server / SSIS | Package format | Status | Verified by |
|
|
45
|
+
|---|---|---|---|
|
|
46
|
+
| **2012** (11.0) | `SSIS.Package.3` | Real package | `u2_toolkit/Package.dtsx`, `u2_toolkit/Project.dtproj` |
|
|
47
|
+
| **2014** (12.0) | `Microsoft.Package` | Real package | `u2_toolkit/PackageAzure.dtsx` |
|
|
48
|
+
| **2019** (15.0) | `Microsoft.Package` | Synthetic fixtures | hand-written packages across the suite |
|
|
49
|
+
| **2022** (16.0) | `Microsoft.Package` | Real packages | the four `ssis_examples/*.dtsx` |
|
|
50
|
+
| 2016 / 2017 (13.0 / 14.0) | `Microsoft.Package` | Expected to work, untested | bracketed by the 2014 and 2019 cases |
|
|
51
|
+
| 2008 and earlier | `SSIS.Package.2` and older | Unknown | no sample available; predates the `.dtproj` project deployment model |
|
|
52
|
+
|
|
53
|
+
Both package format generations are covered by real files: the older
|
|
54
|
+
`SSIS.Package.3` form used by SSIS 2012, and the `Microsoft.Package` form used
|
|
55
|
+
from 2014 onward. The real fixtures also span OLE DB, Flat File and ADO.NET
|
|
56
|
+
connection managers — including a third-party ADO.NET provider — and both
|
|
57
|
+
friendly-name and raw-GUID pipeline component class IDs.
|
|
58
|
+
|
|
59
|
+
Two gaps worth stating plainly:
|
|
60
|
+
|
|
61
|
+
- Only the **project deployment model** is verified. The single real `.dtproj`
|
|
62
|
+
declares `DeploymentModel=Project` (schema `9.0.1.0`); the legacy package
|
|
63
|
+
deployment model has no real-file coverage.
|
|
64
|
+
- No real `.conmgr` file was available, so project-level connection managers are
|
|
65
|
+
covered by synthetic fixtures only.
|
|
66
|
+
|
|
67
|
+
Provenance and the sanitisation applied to the real fixtures are documented in
|
|
68
|
+
[`tests/fixtures/real_world/README.md`](tests/fixtures/real_world/README.md).
|
|
69
|
+
|
|
70
|
+
### Handling unknown content
|
|
71
|
+
|
|
72
|
+
With no version gating, an unfamiliar package generally parses. The
|
|
73
|
+
`completeness_summary` on every parse result reports `total_elements` and
|
|
74
|
+
`total_attributes` actually seen, so you can check a package was read in full
|
|
75
|
+
rather than trusting silence. Note that its `skipped_items` field lists XML
|
|
76
|
+
comments and processing instructions — deliberately ignored content — not
|
|
77
|
+
elements the parser failed to understand.
|
|
78
|
+
|
|
79
|
+
Unrecognised executables and pipeline components are still emitted with their
|
|
80
|
+
attributes and properties intact, keyed by whatever `CreationName` or
|
|
81
|
+
`componentClassID` the file declares, so a third-party or newer component
|
|
82
|
+
appears in the output even when the parser has no special knowledge of it.
|
|
83
|
+
|
|
84
|
+
## Install
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
pip install pydtsx-parser
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
With the optional MCP server:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
pip install "pydtsx-parser[mcp]"
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Requires Python 3.11+.
|
|
97
|
+
|
|
98
|
+
## Quick start
|
|
99
|
+
|
|
100
|
+
Parse a single package:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
pydtsx-parser Package.dtsx --pretty
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Parse an entire project directory (recursively discovers all four file types and
|
|
107
|
+
cross-references them):
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
pydtsx-parser ./MyProject --pretty --output project.json
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
From Python:
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
from pydtsx_parser.dispatcher import dispatch
|
|
117
|
+
|
|
118
|
+
result = dispatch("Package.dtsx")
|
|
119
|
+
print(result["content"]["package_attributes"]["object_name"])
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### Output shape
|
|
123
|
+
|
|
124
|
+
```json
|
|
125
|
+
{
|
|
126
|
+
"format_version": "1.0.0",
|
|
127
|
+
"parser_version": "0.1.0",
|
|
128
|
+
"source_file_path": "/path/to/Package.dtsx",
|
|
129
|
+
"file_type": "dtsx_package",
|
|
130
|
+
"parsed_at": "2026-01-01T09:00:00+10:00",
|
|
131
|
+
"source_file_metadata": { "file_name": "Package.dtsx", "file_size_bytes": 724, "owner": "..." },
|
|
132
|
+
"data_type_map": { "130": "wstr", "131": "numeric", "...": "..." },
|
|
133
|
+
"redaction_summary": { "total_redacted": 0 },
|
|
134
|
+
"content": {
|
|
135
|
+
"package_attributes": { "object_name": "LoadCustomers", "...": "..." },
|
|
136
|
+
"variables": [ { "name": "BatchDate", "namespace": "User", "data_type": "7" } ],
|
|
137
|
+
"connection_managers": [ { "object_name": "DW", "creation_name": "OLEDB" } ],
|
|
138
|
+
"executables": [],
|
|
139
|
+
"completeness_summary": { "total_elements": 6, "total_attributes": 11, "skipped_items": [] }
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
## CLI
|
|
145
|
+
|
|
146
|
+
```
|
|
147
|
+
usage: pydtsx-parser [-h] [--output OUTPUT] [--pretty] path
|
|
148
|
+
|
|
149
|
+
Parse SSIS files (.dtsx, .dtproj, .conmgr, .params) into JSON.
|
|
150
|
+
|
|
151
|
+
positional arguments:
|
|
152
|
+
path File or directory path to parse
|
|
153
|
+
|
|
154
|
+
options:
|
|
155
|
+
-h, --help show this help message and exit
|
|
156
|
+
--output OUTPUT, -o OUTPUT
|
|
157
|
+
Output file path (default: stdout)
|
|
158
|
+
--pretty, -p Pretty-print JSON with 2-space indent
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
## Credential redaction
|
|
162
|
+
|
|
163
|
+
Passwords are redacted automatically — both as standalone fields and inside
|
|
164
|
+
connection strings:
|
|
165
|
+
|
|
166
|
+
```python
|
|
167
|
+
from pydtsx_parser.redaction import redact
|
|
168
|
+
|
|
169
|
+
redact({"connection_string": "Data Source=dbhost;User ID=svc;Password=hunter2;"})
|
|
170
|
+
# ({'connection_string': 'Data Source=dbhost;User ID=svc;Password=[SENSITIVE - REDACTED];'}, 1)
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Schema metadata is deliberately left intact — a *column* named `PASSWORD_HASH`
|
|
174
|
+
is structure, not a secret, so it is not redacted.
|
|
175
|
+
|
|
176
|
+
### Handling real packages
|
|
177
|
+
|
|
178
|
+
Redaction covers credentials, not everything an SSIS file can reveal. Parser
|
|
179
|
+
output also includes the source file's absolute path and its filesystem owner,
|
|
180
|
+
and packages routinely embed internal server names, UNC paths, and schema names.
|
|
181
|
+
Review parser output before attaching it to a public issue or sharing it outside
|
|
182
|
+
your organisation.
|
|
183
|
+
|
|
184
|
+
## MCP server
|
|
185
|
+
|
|
186
|
+
`pydtsx-parser` ships an optional [MCP](https://modelcontextprotocol.io) server
|
|
187
|
+
so agents can explore SSIS packages directly. Install the extra, then point your
|
|
188
|
+
client at the `pydtsx-parser-mcp` command:
|
|
189
|
+
|
|
190
|
+
```json
|
|
191
|
+
{
|
|
192
|
+
"mcpServers": {
|
|
193
|
+
"pydtsx-parser": {
|
|
194
|
+
"command": "pydtsx-parser-mcp"
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
Tools provided:
|
|
201
|
+
|
|
202
|
+
| Tool | Purpose |
|
|
203
|
+
|---|---|
|
|
204
|
+
| `get_package_summary` | High-level overview — best first call |
|
|
205
|
+
| `get_sql_code` | Extract embedded SQL statements |
|
|
206
|
+
| `get_data_lineage` | Control flow edges plus source → destination tracing |
|
|
207
|
+
| `get_data_flows` | Full data flow component detail and column mappings |
|
|
208
|
+
| `parse_dtsx_file` | Full structured JSON for one file |
|
|
209
|
+
| `parse_ssis_directory` | Full structured JSON for a project |
|
|
210
|
+
|
|
211
|
+
A [Claude Skill](skills/pydtsx-parser/SKILL.md) is also included, for a
|
|
212
|
+
portable, dependency-free way to teach an agent how to use the CLI.
|
|
213
|
+
|
|
214
|
+
## Documentation
|
|
215
|
+
|
|
216
|
+
Full documentation lives at
|
|
217
|
+
**[lamiskin.github.io/pydtsx-parser](https://lamiskin.github.io/pydtsx-parser/)** —
|
|
218
|
+
including the [LLM context guide](docs/LLM_CONTEXT.md), a deep reference for
|
|
219
|
+
interpreting the JSON output.
|
|
220
|
+
|
|
221
|
+
## Development
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
uv sync
|
|
225
|
+
uv run pytest
|
|
226
|
+
uv run ruff check
|
|
227
|
+
uv run ruff format --check
|
|
228
|
+
uv run mypy
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
The test suite is fully synthetic — every input is constructed in-memory or
|
|
232
|
+
written to a temp directory. No SSIS packages are bundled with this repository.
|
|
233
|
+
Integration tests look for an optional local `examples/` directory and skip
|
|
234
|
+
cleanly when it is absent, so you can point them at your own packages without
|
|
235
|
+
ever committing them.
|
|
236
|
+
|
|
237
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for the full workflow.
|
|
238
|
+
|
|
239
|
+
## Acknowledgements
|
|
240
|
+
|
|
241
|
+
This project was developed with AI assistance and validated against real-world
|
|
242
|
+
SSIS projects. **None of that data, its identifiers, or its history is included
|
|
243
|
+
in this repository** — no packages, no extracts, no connection details. The
|
|
244
|
+
tests run entirely on synthetic fixtures.
|
|
245
|
+
|
|
246
|
+
## License
|
|
247
|
+
|
|
248
|
+
MIT — see [LICENSE](LICENSE).
|
|
249
|
+
|
|
250
|
+
SQL Server and SQL Server Integration Services are trademarks of Microsoft
|
|
251
|
+
Corporation. This project is not affiliated with or endorsed by Microsoft.
|