pydtsx-parser 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. pydtsx_parser-0.1.0/CHANGELOG.md +36 -0
  2. pydtsx_parser-0.1.0/CONTRIBUTING.md +92 -0
  3. pydtsx_parser-0.1.0/LICENSE +21 -0
  4. pydtsx_parser-0.1.0/MANIFEST.in +3 -0
  5. pydtsx_parser-0.1.0/PKG-INFO +280 -0
  6. pydtsx_parser-0.1.0/README.md +251 -0
  7. pydtsx_parser-0.1.0/SECURITY.md +41 -0
  8. pydtsx_parser-0.1.0/pydtsx_parser/__init__.py +3 -0
  9. pydtsx_parser-0.1.0/pydtsx_parser/__main__.py +8 -0
  10. pydtsx_parser-0.1.0/pydtsx_parser/cli.py +120 -0
  11. pydtsx_parser-0.1.0/pydtsx_parser/constants.py +41 -0
  12. pydtsx_parser-0.1.0/pydtsx_parser/dispatcher.py +373 -0
  13. pydtsx_parser-0.1.0/pydtsx_parser/envelope.py +164 -0
  14. pydtsx_parser-0.1.0/pydtsx_parser/errors.py +37 -0
  15. pydtsx_parser-0.1.0/pydtsx_parser/extractors/__init__.py +1 -0
  16. pydtsx_parser-0.1.0/pydtsx_parser/extractors/columns.py +157 -0
  17. pydtsx_parser-0.1.0/pydtsx_parser/extractors/components.py +284 -0
  18. pydtsx_parser-0.1.0/pydtsx_parser/extractors/connections.py +349 -0
  19. pydtsx_parser-0.1.0/pydtsx_parser/extractors/executables.py +118 -0
  20. pydtsx_parser-0.1.0/pydtsx_parser/extractors/paths.py +133 -0
  21. pydtsx_parser-0.1.0/pydtsx_parser/extractors/pipeline.py +115 -0
  22. pydtsx_parser-0.1.0/pydtsx_parser/extractors/precedence.py +134 -0
  23. pydtsx_parser-0.1.0/pydtsx_parser/extractors/sort.py +156 -0
  24. pydtsx_parser-0.1.0/pydtsx_parser/extractors/sources.py +130 -0
  25. pydtsx_parser-0.1.0/pydtsx_parser/extractors/sql_tasks.py +195 -0
  26. pydtsx_parser-0.1.0/pydtsx_parser/extractors/transformations.py +371 -0
  27. pydtsx_parser-0.1.0/pydtsx_parser/extractors/variables.py +76 -0
  28. pydtsx_parser-0.1.0/pydtsx_parser/mcp_entry.py +31 -0
  29. pydtsx_parser-0.1.0/pydtsx_parser/mcp_server.py +499 -0
  30. pydtsx_parser-0.1.0/pydtsx_parser/parsers/__init__.py +1 -0
  31. pydtsx_parser-0.1.0/pydtsx_parser/parsers/conmgr.py +67 -0
  32. pydtsx_parser-0.1.0/pydtsx_parser/parsers/dtproj.py +470 -0
  33. pydtsx_parser-0.1.0/pydtsx_parser/parsers/dtsx.py +156 -0
  34. pydtsx_parser-0.1.0/pydtsx_parser/parsers/params.py +132 -0
  35. pydtsx_parser-0.1.0/pydtsx_parser/py.typed +0 -0
  36. pydtsx_parser-0.1.0/pydtsx_parser/redaction.py +282 -0
  37. pydtsx_parser-0.1.0/pydtsx_parser/xml_utils.py +185 -0
  38. pydtsx_parser-0.1.0/pydtsx_parser.egg-info/PKG-INFO +280 -0
  39. pydtsx_parser-0.1.0/pydtsx_parser.egg-info/SOURCES.txt +75 -0
  40. pydtsx_parser-0.1.0/pydtsx_parser.egg-info/dependency_links.txt +1 -0
  41. pydtsx_parser-0.1.0/pydtsx_parser.egg-info/entry_points.txt +3 -0
  42. pydtsx_parser-0.1.0/pydtsx_parser.egg-info/requires.txt +3 -0
  43. pydtsx_parser-0.1.0/pydtsx_parser.egg-info/top_level.txt +1 -0
  44. pydtsx_parser-0.1.0/pyproject.toml +101 -0
  45. pydtsx_parser-0.1.0/setup.cfg +4 -0
  46. pydtsx_parser-0.1.0/tests/test_cli.py +268 -0
  47. pydtsx_parser-0.1.0/tests/test_components_smoke.py +164 -0
  48. pydtsx_parser-0.1.0/tests/test_conmgr.py +274 -0
  49. pydtsx_parser-0.1.0/tests/test_connections.py +856 -0
  50. pydtsx_parser-0.1.0/tests/test_derived_columns.py +586 -0
  51. pydtsx_parser-0.1.0/tests/test_dispatcher.py +321 -0
  52. pydtsx_parser-0.1.0/tests/test_dtproj.py +369 -0
  53. pydtsx_parser-0.1.0/tests/test_dtsx.py +513 -0
  54. pydtsx_parser-0.1.0/tests/test_edge_cases.py +632 -0
  55. pydtsx_parser-0.1.0/tests/test_envelope.py +169 -0
  56. pydtsx_parser-0.1.0/tests/test_executables.py +421 -0
  57. pydtsx_parser-0.1.0/tests/test_integration.py +337 -0
  58. pydtsx_parser-0.1.0/tests/test_mcp_entry.py +60 -0
  59. pydtsx_parser-0.1.0/tests/test_mcp_server.py +145 -0
  60. pydtsx_parser-0.1.0/tests/test_merge_join.py +929 -0
  61. pydtsx_parser-0.1.0/tests/test_params.py +279 -0
  62. pydtsx_parser-0.1.0/tests/test_pipeline.py +252 -0
  63. pydtsx_parser-0.1.0/tests/test_precedence.py +333 -0
  64. pydtsx_parser-0.1.0/tests/test_property_connections.py +802 -0
  65. pydtsx_parser-0.1.0/tests/test_property_dataflow.py +795 -0
  66. pydtsx_parser-0.1.0/tests/test_property_dtsx.py +517 -0
  67. pydtsx_parser-0.1.0/tests/test_property_envelope.py +602 -0
  68. pydtsx_parser-0.1.0/tests/test_property_precedence.py +294 -0
  69. pydtsx_parser-0.1.0/tests/test_property_redaction.py +560 -0
  70. pydtsx_parser-0.1.0/tests/test_property_transformations.py +912 -0
  71. pydtsx_parser-0.1.0/tests/test_real_world.py +201 -0
  72. pydtsx_parser-0.1.0/tests/test_redaction.py +303 -0
  73. pydtsx_parser-0.1.0/tests/test_sort.py +557 -0
  74. pydtsx_parser-0.1.0/tests/test_sources.py +356 -0
  75. pydtsx_parser-0.1.0/tests/test_sql_tasks.py +325 -0
  76. pydtsx_parser-0.1.0/tests/test_variables.py +316 -0
  77. pydtsx_parser-0.1.0/tests/test_xml_utils.py +236 -0
@@ -0,0 +1,36 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [Unreleased]
9
+
10
+ ## [0.1.0] - 2026-08-29
11
+
12
+ ### Added
13
+
14
+ - Initial public release.
15
+ - Parsers for the four SSIS project file types: `.dtsx` packages, `.dtproj`
16
+ projects, `.conmgr` connection managers, and `.params` project parameters.
17
+ - Directory dispatcher that recursively discovers and cross-references all
18
+ files in an SSIS project.
19
+ - Data flow pipeline extraction: components, paths, error outputs, and
20
+ topological ordering.
21
+ - Control flow extraction: executables, precedence constraints, and Execute SQL
22
+ Task contents.
23
+ - Transformation extractors for derived columns, lookups, merge joins, sorts,
24
+ and column mappings.
25
+ - Self-describing JSON envelope carrying format/parser versions, source file
26
+ metadata, a data type map, and a completeness summary.
27
+ - Automatic credential redaction for sensitive fields and connection strings,
28
+ which deliberately preserves password-like *column* names as schema metadata.
29
+ - CLI entry point `pydtsx-parser` with `--output` and `--pretty`.
30
+ - Optional MCP server (`pip install "pydtsx-parser[mcp]"`) exposing package
31
+ summary, SQL extraction, and data lineage tools.
32
+ - Claude Skill for agent-driven use without a server.
33
+ - Type hints throughout, with a PEP 561 `py.typed` marker.
34
+
35
+ [Unreleased]: https://github.com/lamiskin/pydtsx-parser/compare/v0.1.0...HEAD
36
+ [0.1.0]: https://github.com/lamiskin/pydtsx-parser/releases/tag/v0.1.0
@@ -0,0 +1,92 @@
1
+ # Contributing
2
+
3
+ Thanks for your interest in improving `pydtsx-parser`.
4
+
5
+ ## The one rule that matters most
6
+
7
+ **Never contribute real SSIS packages, extracts, or production data.**
8
+
9
+ SSIS files are unusually leaky: they routinely embed internal server names, UNC
10
+ paths, database schemas, domain usernames, and sometimes credentials. Everything
11
+ in this repository — every test input, every example, every issue attachment —
12
+ must be synthetic.
13
+
14
+ If you are debugging against a real package, reduce it to a minimal synthetic
15
+ snippet that reproduces the behaviour before opening an issue or PR.
16
+
17
+ ## Setup
18
+
19
+ This project uses [uv](https://docs.astral.sh/uv/).
20
+
21
+ ```bash
22
+ git clone https://github.com/lamiskin/pydtsx-parser
23
+ cd pydtsx-parser
24
+ uv sync --all-extras
25
+ uv run pre-commit install
26
+ ```
27
+
28
+ ## Everyday commands
29
+
30
+ ```bash
31
+ uv run pytest # run the test suite
32
+ uv run pytest --cov # with coverage
33
+ uv run ruff check --fix # lint
34
+ uv run ruff format # format
35
+ uv run mypy # type check
36
+ ```
37
+
38
+ All four must pass before a PR can merge; CI runs them on Python 3.11–3.14
39
+ across Linux, Windows, and macOS.
40
+
41
+ ## Testing against your own packages
42
+
43
+ Integration tests look for an optional `examples/` directory at the repository
44
+ root and skip cleanly when it is absent. You can drop your own SSIS projects
45
+ there to exercise the parser locally — `examples/` is gitignored, so they will
46
+ not be committed. Keep it that way.
47
+
48
+ ## Commit messages
49
+
50
+ This project uses [Conventional Commits](https://www.conventionalcommits.org/).
51
+ Release automation reads them, so the prefix determines the version bump:
52
+
53
+ | Prefix | Effect |
54
+ |---|---|
55
+ | `fix:` | patch release |
56
+ | `feat:` | minor release |
57
+ | `feat!:` or `BREAKING CHANGE:` | major release |
58
+ | `docs:`, `ci:`, `test:`, `chore:`, `refactor:` | no release |
59
+
60
+ Example: `feat: extract Lookup transformation cache settings`
61
+
62
+ ## Releases are automated
63
+
64
+ Do not bump versions or edit `CHANGELOG.md` by hand.
65
+ [release-please](https://github.com/googleapis/release-please) watches `main`,
66
+ maintains a release PR, and merging it tags a GitHub Release, which publishes to
67
+ PyPI via Trusted Publishing.
68
+
69
+ ### Bootstrapping the first release (maintainers)
70
+
71
+ release-please treats `.release-please-manifest.json` as the last *released*
72
+ version, so it will only ever propose versions **after** it. The version
73
+ currently recorded there has to be tagged and released by hand once, or it never
74
+ reaches PyPI:
75
+
76
+ ```bash
77
+ git tag -a v1.0.0 -m "pydtsx-parser 1.0.0" && git push origin v1.0.0
78
+ gh release create v1.0.0 --notes-from-tag
79
+ ```
80
+
81
+ Before that, dry-run the whole pipeline without touching real PyPI: run the
82
+ **Publish** workflow manually (`workflow_dispatch`) with target `testpypi`. It
83
+ builds, validates the metadata, smoke-tests the wheel, and publishes to TestPyPI
84
+ only — the PyPI job is unreachable from that trigger.
85
+
86
+ ## Adding a new extractor
87
+
88
+ Extractors live in `pydtsx_parser/extractors/` and take an `ET.Element`,
89
+ returning plain dicts and lists. Keep them pure — file I/O belongs in
90
+ `pydtsx_parser/parsers/`. Add the extractor to the relevant parser, then cover
91
+ it with both example-based tests and, where the shape allows, a Hypothesis
92
+ property test.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Lachlan Miskin
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,3 @@
1
+ include CHANGELOG.md
2
+ include CONTRIBUTING.md
3
+ include SECURITY.md
@@ -0,0 +1,280 @@
1
+ Metadata-Version: 2.4
2
+ Name: pydtsx-parser
3
+ Version: 0.1.0
4
+ Summary: Parse SQL Server Integration Services (SSIS) .dtsx, .dtproj, .conmgr and .params files into structured JSON
5
+ Author-email: Lachlan Miskin <lamiskin@users.noreply.github.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/lamiskin/pydtsx-parser
8
+ Project-URL: Repository, https://github.com/lamiskin/pydtsx-parser
9
+ Project-URL: Issues, https://github.com/lamiskin/pydtsx-parser/issues
10
+ Project-URL: Changelog, https://github.com/lamiskin/pydtsx-parser/blob/main/CHANGELOG.md
11
+ Keywords: ssis,dtsx,etl,sql-server,parser,data-engineering,lineage
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Programming Language :: Python :: 3.14
20
+ Classifier: Topic :: Database
21
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
22
+ Classifier: Typing :: Typed
23
+ Requires-Python: >=3.11
24
+ Description-Content-Type: text/markdown
25
+ License-File: LICENSE
26
+ Provides-Extra: mcp
27
+ Requires-Dist: mcp>=2.0.0; extra == "mcp"
28
+ Dynamic: license-file
29
+
30
+ # pydtsx-parser
31
+
32
+ [![CI](https://github.com/lamiskin/pydtsx-parser/actions/workflows/ci.yml/badge.svg)](https://github.com/lamiskin/pydtsx-parser/actions/workflows/ci.yml)
33
+ [![codecov](https://codecov.io/gh/lamiskin/pydtsx-parser/branch/main/graph/badge.svg)](https://codecov.io/gh/lamiskin/pydtsx-parser)
34
+ [![PyPI](https://img.shields.io/pypi/v/pydtsx-parser.svg)](https://pypi.org/project/pydtsx-parser/)
35
+ [![Python versions](https://img.shields.io/pypi/pyversions/pydtsx-parser.svg)](https://pypi.org/project/pydtsx-parser/)
36
+ [![CodeQL](https://github.com/lamiskin/pydtsx-parser/actions/workflows/codeql.yml/badge.svg)](https://github.com/lamiskin/pydtsx-parser/actions/workflows/codeql.yml)
37
+ [![Docs](https://img.shields.io/badge/docs-lamiskin.github.io-blue)](https://lamiskin.github.io/pydtsx-parser/)
38
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
39
+
40
+ Parse SQL Server Integration Services (SSIS) project files into structured,
41
+ self-describing JSON — with no SSIS installation, no SQL Server, and no runtime
42
+ dependencies.
43
+
44
+ Handles the four file types that make up an SSIS project:
45
+
46
+ | File | What it holds |
47
+ |---|---|
48
+ | `.dtsx` | Packages: control flow, data flow pipelines, variables, connections |
49
+ | `.dtproj` | Project manifest, deployment model, package list |
50
+ | `.conmgr` | Project-level connection managers |
51
+ | `.params` | Project parameters |
52
+
53
+ ## Why
54
+
55
+ SSIS packages are large, deeply nested XML files that are painful to read and
56
+ awkward to diff. If you are migrating away from SSIS, auditing what a package
57
+ actually does, or documenting an inherited ETL estate, you need the *structure*
58
+ out of that XML — task graphs, data lineage, embedded SQL, column mappings —
59
+ without opening Visual Studio.
60
+
61
+ `pydtsx-parser` extracts all of it into one JSON envelope designed to be
62
+ machine-readable and self-describing: the output carries its own data type map
63
+ and a completeness summary of how many elements and attributes were seen.
64
+
65
+ ## Supported SSIS versions
66
+
67
+ The parser is **version-agnostic by design**: it has no version gates and no
68
+ version-specific branches. Version markers such as `LastModifiedProductVersion`
69
+ and the `.dtproj` `ProductVersion` are extracted and reported, but they never
70
+ change how a file is parsed. "Supported" below therefore means *verified*, not
71
+ *enabled* — an unlisted version is likely to parse.
72
+
73
+ | SQL Server / SSIS | Package format | Status | Verified by |
74
+ |---|---|---|---|
75
+ | **2012** (11.0) | `SSIS.Package.3` | Real package | `u2_toolkit/Package.dtsx`, `u2_toolkit/Project.dtproj` |
76
+ | **2014** (12.0) | `Microsoft.Package` | Real package | `u2_toolkit/PackageAzure.dtsx` |
77
+ | **2019** (15.0) | `Microsoft.Package` | Synthetic fixtures | hand-written packages across the suite |
78
+ | **2022** (16.0) | `Microsoft.Package` | Real packages | the four `ssis_examples/*.dtsx` |
79
+ | 2016 / 2017 (13.0 / 14.0) | `Microsoft.Package` | Expected to work, untested | bracketed by the 2014 and 2019 cases |
80
+ | 2008 and earlier | `SSIS.Package.2` and older | Unknown | no sample available; predates the `.dtproj` project deployment model |
81
+
82
+ Both package format generations are covered by real files: the older
83
+ `SSIS.Package.3` form used by SSIS 2012, and the `Microsoft.Package` form used
84
+ from 2014 onward. The real fixtures also span OLE DB, Flat File and ADO.NET
85
+ connection managers — including a third-party ADO.NET provider — and both
86
+ friendly-name and raw-GUID pipeline component class IDs.
87
+
88
+ Two gaps worth stating plainly:
89
+
90
+ - Only the **project deployment model** is verified. The single real `.dtproj`
91
+ declares `DeploymentModel=Project` (schema `9.0.1.0`); the legacy package
92
+ deployment model has no real-file coverage.
93
+ - No real `.conmgr` file was available, so project-level connection managers are
94
+ covered by synthetic fixtures only.
95
+
96
+ Provenance and the sanitisation applied to the real fixtures are documented in
97
+ [`tests/fixtures/real_world/README.md`](tests/fixtures/real_world/README.md).
98
+
99
+ ### Handling unknown content
100
+
101
+ With no version gating, an unfamiliar package generally parses. The
102
+ `completeness_summary` on every parse result reports `total_elements` and
103
+ `total_attributes` actually seen, so you can check a package was read in full
104
+ rather than trusting silence. Note that its `skipped_items` field lists XML
105
+ comments and processing instructions — deliberately ignored content — not
106
+ elements the parser failed to understand.
107
+
108
+ Unrecognised executables and pipeline components are still emitted with their
109
+ attributes and properties intact, keyed by whatever `CreationName` or
110
+ `componentClassID` the file declares, so a third-party or newer component
111
+ appears in the output even when the parser has no special knowledge of it.
112
+
113
+ ## Install
114
+
115
+ ```bash
116
+ pip install pydtsx-parser
117
+ ```
118
+
119
+ With the optional MCP server:
120
+
121
+ ```bash
122
+ pip install "pydtsx-parser[mcp]"
123
+ ```
124
+
125
+ Requires Python 3.11+.
126
+
127
+ ## Quick start
128
+
129
+ Parse a single package:
130
+
131
+ ```bash
132
+ pydtsx-parser Package.dtsx --pretty
133
+ ```
134
+
135
+ Parse an entire project directory (recursively discovers all four file types and
136
+ cross-references them):
137
+
138
+ ```bash
139
+ pydtsx-parser ./MyProject --pretty --output project.json
140
+ ```
141
+
142
+ From Python:
143
+
144
+ ```python
145
+ from pydtsx_parser.dispatcher import dispatch
146
+
147
+ result = dispatch("Package.dtsx")
148
+ print(result["content"]["package_attributes"]["object_name"])
149
+ ```
150
+
151
+ ### Output shape
152
+
153
+ ```json
154
+ {
155
+ "format_version": "1.0.0",
156
+ "parser_version": "0.1.0",
157
+ "source_file_path": "/path/to/Package.dtsx",
158
+ "file_type": "dtsx_package",
159
+ "parsed_at": "2026-01-01T09:00:00+10:00",
160
+ "source_file_metadata": { "file_name": "Package.dtsx", "file_size_bytes": 724, "owner": "..." },
161
+ "data_type_map": { "130": "wstr", "131": "numeric", "...": "..." },
162
+ "redaction_summary": { "total_redacted": 0 },
163
+ "content": {
164
+ "package_attributes": { "object_name": "LoadCustomers", "...": "..." },
165
+ "variables": [ { "name": "BatchDate", "namespace": "User", "data_type": "7" } ],
166
+ "connection_managers": [ { "object_name": "DW", "creation_name": "OLEDB" } ],
167
+ "executables": [],
168
+ "completeness_summary": { "total_elements": 6, "total_attributes": 11, "skipped_items": [] }
169
+ }
170
+ }
171
+ ```
172
+
173
+ ## CLI
174
+
175
+ ```
176
+ usage: pydtsx-parser [-h] [--output OUTPUT] [--pretty] path
177
+
178
+ Parse SSIS files (.dtsx, .dtproj, .conmgr, .params) into JSON.
179
+
180
+ positional arguments:
181
+ path File or directory path to parse
182
+
183
+ options:
184
+ -h, --help show this help message and exit
185
+ --output OUTPUT, -o OUTPUT
186
+ Output file path (default: stdout)
187
+ --pretty, -p Pretty-print JSON with 2-space indent
188
+ ```
189
+
190
+ ## Credential redaction
191
+
192
+ Passwords are redacted automatically — both as standalone fields and inside
193
+ connection strings:
194
+
195
+ ```python
196
+ from pydtsx_parser.redaction import redact
197
+
198
+ redact({"connection_string": "Data Source=dbhost;User ID=svc;Password=hunter2;"})
199
+ # ({'connection_string': 'Data Source=dbhost;User ID=svc;Password=[SENSITIVE - REDACTED];'}, 1)
200
+ ```
201
+
202
+ Schema metadata is deliberately left intact — a *column* named `PASSWORD_HASH`
203
+ is structure, not a secret, so it is not redacted.
204
+
205
+ ### Handling real packages
206
+
207
+ Redaction covers credentials, not everything an SSIS file can reveal. Parser
208
+ output also includes the source file's absolute path and its filesystem owner,
209
+ and packages routinely embed internal server names, UNC paths, and schema names.
210
+ Review parser output before attaching it to a public issue or sharing it outside
211
+ your organisation.
212
+
213
+ ## MCP server
214
+
215
+ `pydtsx-parser` ships an optional [MCP](https://modelcontextprotocol.io) server
216
+ so agents can explore SSIS packages directly. Install the extra, then point your
217
+ client at the `pydtsx-parser-mcp` command:
218
+
219
+ ```json
220
+ {
221
+ "mcpServers": {
222
+ "pydtsx-parser": {
223
+ "command": "pydtsx-parser-mcp"
224
+ }
225
+ }
226
+ }
227
+ ```
228
+
229
+ Tools provided:
230
+
231
+ | Tool | Purpose |
232
+ |---|---|
233
+ | `get_package_summary` | High-level overview — best first call |
234
+ | `get_sql_code` | Extract embedded SQL statements |
235
+ | `get_data_lineage` | Control flow edges plus source → destination tracing |
236
+ | `get_data_flows` | Full data flow component detail and column mappings |
237
+ | `parse_dtsx_file` | Full structured JSON for one file |
238
+ | `parse_ssis_directory` | Full structured JSON for a project |
239
+
240
+ A [Claude Skill](skills/pydtsx-parser/SKILL.md) is also included, for a
241
+ portable, dependency-free way to teach an agent how to use the CLI.
242
+
243
+ ## Documentation
244
+
245
+ Full documentation lives at
246
+ **[lamiskin.github.io/pydtsx-parser](https://lamiskin.github.io/pydtsx-parser/)** —
247
+ including the [LLM context guide](docs/LLM_CONTEXT.md), a deep reference for
248
+ interpreting the JSON output.
249
+
250
+ ## Development
251
+
252
+ ```bash
253
+ uv sync
254
+ uv run pytest
255
+ uv run ruff check
256
+ uv run ruff format --check
257
+ uv run mypy
258
+ ```
259
+
260
+ The test suite is fully synthetic — every input is constructed in-memory or
261
+ written to a temp directory. No SSIS packages are bundled with this repository.
262
+ Integration tests look for an optional local `examples/` directory and skip
263
+ cleanly when it is absent, so you can point them at your own packages without
264
+ ever committing them.
265
+
266
+ See [CONTRIBUTING.md](CONTRIBUTING.md) for the full workflow.
267
+
268
+ ## Acknowledgements
269
+
270
+ This project was developed with AI assistance and validated against real-world
271
+ SSIS projects. **None of that data, its identifiers, or its history is included
272
+ in this repository** — no packages, no extracts, no connection details. The
273
+ tests run entirely on synthetic fixtures.
274
+
275
+ ## License
276
+
277
+ MIT — see [LICENSE](LICENSE).
278
+
279
+ SQL Server and SQL Server Integration Services are trademarks of Microsoft
280
+ Corporation. This project is not affiliated with or endorsed by Microsoft.
@@ -0,0 +1,251 @@
1
+ # pydtsx-parser
2
+
3
+ [![CI](https://github.com/lamiskin/pydtsx-parser/actions/workflows/ci.yml/badge.svg)](https://github.com/lamiskin/pydtsx-parser/actions/workflows/ci.yml)
4
+ [![codecov](https://codecov.io/gh/lamiskin/pydtsx-parser/branch/main/graph/badge.svg)](https://codecov.io/gh/lamiskin/pydtsx-parser)
5
+ [![PyPI](https://img.shields.io/pypi/v/pydtsx-parser.svg)](https://pypi.org/project/pydtsx-parser/)
6
+ [![Python versions](https://img.shields.io/pypi/pyversions/pydtsx-parser.svg)](https://pypi.org/project/pydtsx-parser/)
7
+ [![CodeQL](https://github.com/lamiskin/pydtsx-parser/actions/workflows/codeql.yml/badge.svg)](https://github.com/lamiskin/pydtsx-parser/actions/workflows/codeql.yml)
8
+ [![Docs](https://img.shields.io/badge/docs-lamiskin.github.io-blue)](https://lamiskin.github.io/pydtsx-parser/)
9
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
10
+
11
+ Parse SQL Server Integration Services (SSIS) project files into structured,
12
+ self-describing JSON — with no SSIS installation, no SQL Server, and no runtime
13
+ dependencies.
14
+
15
+ Handles the four file types that make up an SSIS project:
16
+
17
+ | File | What it holds |
18
+ |---|---|
19
+ | `.dtsx` | Packages: control flow, data flow pipelines, variables, connections |
20
+ | `.dtproj` | Project manifest, deployment model, package list |
21
+ | `.conmgr` | Project-level connection managers |
22
+ | `.params` | Project parameters |
23
+
24
+ ## Why
25
+
26
+ SSIS packages are large, deeply nested XML files that are painful to read and
27
+ awkward to diff. If you are migrating away from SSIS, auditing what a package
28
+ actually does, or documenting an inherited ETL estate, you need the *structure*
29
+ out of that XML — task graphs, data lineage, embedded SQL, column mappings —
30
+ without opening Visual Studio.
31
+
32
+ `pydtsx-parser` extracts all of it into one JSON envelope designed to be
33
+ machine-readable and self-describing: the output carries its own data type map
34
+ and a completeness summary of how many elements and attributes were seen.
35
+
36
+ ## Supported SSIS versions
37
+
38
+ The parser is **version-agnostic by design**: it has no version gates and no
39
+ version-specific branches. Version markers such as `LastModifiedProductVersion`
40
+ and the `.dtproj` `ProductVersion` are extracted and reported, but they never
41
+ change how a file is parsed. "Supported" below therefore means *verified*, not
42
+ *enabled* — an unlisted version is likely to parse.
43
+
44
+ | SQL Server / SSIS | Package format | Status | Verified by |
45
+ |---|---|---|---|
46
+ | **2012** (11.0) | `SSIS.Package.3` | Real package | `u2_toolkit/Package.dtsx`, `u2_toolkit/Project.dtproj` |
47
+ | **2014** (12.0) | `Microsoft.Package` | Real package | `u2_toolkit/PackageAzure.dtsx` |
48
+ | **2019** (15.0) | `Microsoft.Package` | Synthetic fixtures | hand-written packages across the suite |
49
+ | **2022** (16.0) | `Microsoft.Package` | Real packages | the four `ssis_examples/*.dtsx` |
50
+ | 2016 / 2017 (13.0 / 14.0) | `Microsoft.Package` | Expected to work, untested | bracketed by the 2014 and 2019 cases |
51
+ | 2008 and earlier | `SSIS.Package.2` and older | Unknown | no sample available; predates the `.dtproj` project deployment model |
52
+
53
+ Both package format generations are covered by real files: the older
54
+ `SSIS.Package.3` form used by SSIS 2012, and the `Microsoft.Package` form used
55
+ from 2014 onward. The real fixtures also span OLE DB, Flat File and ADO.NET
56
+ connection managers — including a third-party ADO.NET provider — and both
57
+ friendly-name and raw-GUID pipeline component class IDs.
58
+
59
+ Two gaps worth stating plainly:
60
+
61
+ - Only the **project deployment model** is verified. The single real `.dtproj`
62
+ declares `DeploymentModel=Project` (schema `9.0.1.0`); the legacy package
63
+ deployment model has no real-file coverage.
64
+ - No real `.conmgr` file was available, so project-level connection managers are
65
+ covered by synthetic fixtures only.
66
+
67
+ Provenance and the sanitisation applied to the real fixtures are documented in
68
+ [`tests/fixtures/real_world/README.md`](tests/fixtures/real_world/README.md).
69
+
70
+ ### Handling unknown content
71
+
72
+ With no version gating, an unfamiliar package generally parses. The
73
+ `completeness_summary` on every parse result reports `total_elements` and
74
+ `total_attributes` actually seen, so you can check a package was read in full
75
+ rather than trusting silence. Note that its `skipped_items` field lists XML
76
+ comments and processing instructions — deliberately ignored content — not
77
+ elements the parser failed to understand.
78
+
79
+ Unrecognised executables and pipeline components are still emitted with their
80
+ attributes and properties intact, keyed by whatever `CreationName` or
81
+ `componentClassID` the file declares, so a third-party or newer component
82
+ appears in the output even when the parser has no special knowledge of it.
83
+
84
+ ## Install
85
+
86
+ ```bash
87
+ pip install pydtsx-parser
88
+ ```
89
+
90
+ With the optional MCP server:
91
+
92
+ ```bash
93
+ pip install "pydtsx-parser[mcp]"
94
+ ```
95
+
96
+ Requires Python 3.11+.
97
+
98
+ ## Quick start
99
+
100
+ Parse a single package:
101
+
102
+ ```bash
103
+ pydtsx-parser Package.dtsx --pretty
104
+ ```
105
+
106
+ Parse an entire project directory (recursively discovers all four file types and
107
+ cross-references them):
108
+
109
+ ```bash
110
+ pydtsx-parser ./MyProject --pretty --output project.json
111
+ ```
112
+
113
+ From Python:
114
+
115
+ ```python
116
+ from pydtsx_parser.dispatcher import dispatch
117
+
118
+ result = dispatch("Package.dtsx")
119
+ print(result["content"]["package_attributes"]["object_name"])
120
+ ```
121
+
122
+ ### Output shape
123
+
124
+ ```json
125
+ {
126
+ "format_version": "1.0.0",
127
+ "parser_version": "0.1.0",
128
+ "source_file_path": "/path/to/Package.dtsx",
129
+ "file_type": "dtsx_package",
130
+ "parsed_at": "2026-01-01T09:00:00+10:00",
131
+ "source_file_metadata": { "file_name": "Package.dtsx", "file_size_bytes": 724, "owner": "..." },
132
+ "data_type_map": { "130": "wstr", "131": "numeric", "...": "..." },
133
+ "redaction_summary": { "total_redacted": 0 },
134
+ "content": {
135
+ "package_attributes": { "object_name": "LoadCustomers", "...": "..." },
136
+ "variables": [ { "name": "BatchDate", "namespace": "User", "data_type": "7" } ],
137
+ "connection_managers": [ { "object_name": "DW", "creation_name": "OLEDB" } ],
138
+ "executables": [],
139
+ "completeness_summary": { "total_elements": 6, "total_attributes": 11, "skipped_items": [] }
140
+ }
141
+ }
142
+ ```
143
+
144
+ ## CLI
145
+
146
+ ```
147
+ usage: pydtsx-parser [-h] [--output OUTPUT] [--pretty] path
148
+
149
+ Parse SSIS files (.dtsx, .dtproj, .conmgr, .params) into JSON.
150
+
151
+ positional arguments:
152
+ path File or directory path to parse
153
+
154
+ options:
155
+ -h, --help show this help message and exit
156
+ --output OUTPUT, -o OUTPUT
157
+ Output file path (default: stdout)
158
+ --pretty, -p Pretty-print JSON with 2-space indent
159
+ ```
160
+
161
+ ## Credential redaction
162
+
163
+ Passwords are redacted automatically — both as standalone fields and inside
164
+ connection strings:
165
+
166
+ ```python
167
+ from pydtsx_parser.redaction import redact
168
+
169
+ redact({"connection_string": "Data Source=dbhost;User ID=svc;Password=hunter2;"})
170
+ # ({'connection_string': 'Data Source=dbhost;User ID=svc;Password=[SENSITIVE - REDACTED];'}, 1)
171
+ ```
172
+
173
+ Schema metadata is deliberately left intact — a *column* named `PASSWORD_HASH`
174
+ is structure, not a secret, so it is not redacted.
175
+
176
+ ### Handling real packages
177
+
178
+ Redaction covers credentials, not everything an SSIS file can reveal. Parser
179
+ output also includes the source file's absolute path and its filesystem owner,
180
+ and packages routinely embed internal server names, UNC paths, and schema names.
181
+ Review parser output before attaching it to a public issue or sharing it outside
182
+ your organisation.
183
+
184
+ ## MCP server
185
+
186
+ `pydtsx-parser` ships an optional [MCP](https://modelcontextprotocol.io) server
187
+ so agents can explore SSIS packages directly. Install the extra, then point your
188
+ client at the `pydtsx-parser-mcp` command:
189
+
190
+ ```json
191
+ {
192
+ "mcpServers": {
193
+ "pydtsx-parser": {
194
+ "command": "pydtsx-parser-mcp"
195
+ }
196
+ }
197
+ }
198
+ ```
199
+
200
+ Tools provided:
201
+
202
+ | Tool | Purpose |
203
+ |---|---|
204
+ | `get_package_summary` | High-level overview — best first call |
205
+ | `get_sql_code` | Extract embedded SQL statements |
206
+ | `get_data_lineage` | Control flow edges plus source → destination tracing |
207
+ | `get_data_flows` | Full data flow component detail and column mappings |
208
+ | `parse_dtsx_file` | Full structured JSON for one file |
209
+ | `parse_ssis_directory` | Full structured JSON for a project |
210
+
211
+ A [Claude Skill](skills/pydtsx-parser/SKILL.md) is also included, for a
212
+ portable, dependency-free way to teach an agent how to use the CLI.
213
+
214
+ ## Documentation
215
+
216
+ Full documentation lives at
217
+ **[lamiskin.github.io/pydtsx-parser](https://lamiskin.github.io/pydtsx-parser/)** —
218
+ including the [LLM context guide](docs/LLM_CONTEXT.md), a deep reference for
219
+ interpreting the JSON output.
220
+
221
+ ## Development
222
+
223
+ ```bash
224
+ uv sync
225
+ uv run pytest
226
+ uv run ruff check
227
+ uv run ruff format --check
228
+ uv run mypy
229
+ ```
230
+
231
+ The test suite is fully synthetic — every input is constructed in-memory or
232
+ written to a temp directory. No SSIS packages are bundled with this repository.
233
+ Integration tests look for an optional local `examples/` directory and skip
234
+ cleanly when it is absent, so you can point them at your own packages without
235
+ ever committing them.
236
+
237
+ See [CONTRIBUTING.md](CONTRIBUTING.md) for the full workflow.
238
+
239
+ ## Acknowledgements
240
+
241
+ This project was developed with AI assistance and validated against real-world
242
+ SSIS projects. **None of that data, its identifiers, or its history is included
243
+ in this repository** — no packages, no extracts, no connection details. The
244
+ tests run entirely on synthetic fixtures.
245
+
246
+ ## License
247
+
248
+ MIT — see [LICENSE](LICENSE).
249
+
250
+ SQL Server and SQL Server Integration Services are trademarks of Microsoft
251
+ Corporation. This project is not affiliated with or endorsed by Microsoft.