dataspector 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. dataspector-1.0.0/.github/workflows/ci.yml +144 -0
  2. dataspector-1.0.0/.github/workflows/publish.yml +125 -0
  3. dataspector-1.0.0/.gitignore +38 -0
  4. dataspector-1.0.0/.python-version +1 -0
  5. dataspector-1.0.0/LICENSE +21 -0
  6. dataspector-1.0.0/Makefile +61 -0
  7. dataspector-1.0.0/PKG-INFO +181 -0
  8. dataspector-1.0.0/README.md +142 -0
  9. dataspector-1.0.0/docs/source/Makefile +14 -0
  10. dataspector-1.0.0/docs/source/_templates/package-version.html +1 -0
  11. dataspector-1.0.0/docs/source/_templates/version-badge.html +1 -0
  12. dataspector-1.0.0/docs/source/api/checkers.rst +25 -0
  13. dataspector-1.0.0/docs/source/api/cli.rst +16 -0
  14. dataspector-1.0.0/docs/source/api/coboller.rst +33 -0
  15. dataspector-1.0.0/docs/source/api/core.rst +7 -0
  16. dataspector-1.0.0/docs/source/api/index.rst +12 -0
  17. dataspector-1.0.0/docs/source/checks_guide.rst +84 -0
  18. dataspector-1.0.0/docs/source/coboller_guide.rst +293 -0
  19. dataspector-1.0.0/docs/source/conf.py +71 -0
  20. dataspector-1.0.0/docs/source/index.rst +53 -0
  21. dataspector-1.0.0/docs/source/make.bat +36 -0
  22. dataspector-1.0.0/docs/source/usage.rst +134 -0
  23. dataspector-1.0.0/docs/source/user_guide.rst +12 -0
  24. dataspector-1.0.0/pyproject.toml +131 -0
  25. dataspector-1.0.0/src/dataspector/__init__.py +64 -0
  26. dataspector-1.0.0/src/dataspector/__main__.py +10 -0
  27. dataspector-1.0.0/src/dataspector/_connectors.py +260 -0
  28. dataspector-1.0.0/src/dataspector/_reporting.py +480 -0
  29. dataspector-1.0.0/src/dataspector/checkers/__init__.py +85 -0
  30. dataspector-1.0.0/src/dataspector/checkers/record.py +671 -0
  31. dataspector-1.0.0/src/dataspector/checkers/source.py +344 -0
  32. dataspector-1.0.0/src/dataspector/checkers/utils.py +342 -0
  33. dataspector-1.0.0/src/dataspector/cli.py +207 -0
  34. dataspector-1.0.0/src/dataspector/coboller/__init__.py +123 -0
  35. dataspector-1.0.0/src/dataspector/coboller/codecs.py +841 -0
  36. dataspector-1.0.0/src/dataspector/coboller/constants.py +324 -0
  37. dataspector-1.0.0/src/dataspector/coboller/copybook.py +1448 -0
  38. dataspector-1.0.0/src/dataspector/coboller/parser.py +201 -0
  39. dataspector-1.0.0/src/dataspector/core.py +740 -0
  40. dataspector-1.0.0/src/dataspector/py.typed +0 -0
  41. dataspector-1.0.0/src/dataspector/settings.py +101 -0
  42. dataspector-1.0.0/test_driver.xlsx +0 -0
  43. dataspector-1.0.0/tests/conftest.py +82 -0
  44. dataspector-1.0.0/tests/data/sample_accessdb.mdb +0 -0
  45. dataspector-1.0.0/tests/data/sample_cobol_exotic.cbl +27 -0
  46. dataspector-1.0.0/tests/data/sample_cobol_exotic.dat +0 -0
  47. dataspector-1.0.0/tests/data/sample_cobol_mainframe.cbl +25 -0
  48. dataspector-1.0.0/tests/data/sample_cobol_mainframe.dat +0 -0
  49. dataspector-1.0.0/tests/data/sample_cobol_multiplesubgroups.cbl +17 -0
  50. dataspector-1.0.0/tests/data/sample_cobol_students.cbl +12 -0
  51. dataspector-1.0.0/tests/data/sample_cobol_students.dat +32 -0
  52. dataspector-1.0.0/tests/data/sample_cobol_studentsorig.cbl +13 -0
  53. dataspector-1.0.0/tests/data/sample_csv.csv +32 -0
  54. dataspector-1.0.0/tests/data/sample_csv0.csv +32 -0
  55. dataspector-1.0.0/tests/data/sample_csv1.csv +32 -0
  56. dataspector-1.0.0/tests/data/sample_excel.xlsx +0 -0
  57. dataspector-1.0.0/tests/data/sample_fwf.txt +32 -0
  58. dataspector-1.0.0/tests/data/sample_msaccess.accdb +0 -0
  59. dataspector-1.0.0/tests/data/sample_sqlite.sqlite +0 -0
  60. dataspector-1.0.0/tests/test_checkers.py +507 -0
  61. dataspector-1.0.0/tests/test_cli.py +151 -0
  62. dataspector-1.0.0/tests/test_coboller.py +808 -0
  63. dataspector-1.0.0/tests/test_core.py +273 -0
  64. dataspector-1.0.0/uv.lock +2050 -0
@@ -0,0 +1,144 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - main
7
+ tags:
8
+ - "v*"
9
+ pull_request:
10
+ branches:
11
+ - main
12
+ workflow_dispatch:
13
+
14
+ concurrency:
15
+ group: ${{ github.workflow }}-${{ github.ref }}
16
+ cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
17
+
18
+ permissions:
19
+ contents: read
20
+
21
+ jobs:
22
+ static-check:
23
+ name: Static Checks (lint & typecheck)
24
+ runs-on: ubuntu-latest
25
+ env:
26
+ UV_PYTHON: "3.12"
27
+ steps:
28
+ - name: Checkout repository
29
+ uses: actions/checkout@v4
30
+ with:
31
+ fetch-depth: 0
32
+
33
+ - name: Set up uv & Python
34
+ uses: astral-sh/setup-uv@v5
35
+ with:
36
+ python-version: "3.12"
37
+ enable-cache: true
38
+ cache-dependency-glob: "uv.lock"
39
+
40
+ - name: Bootstrap environment
41
+ run: make bootstrap UV_SYNC_FLAGS="--frozen"
42
+
43
+ - name: Run static checks
44
+ run: make static-check
45
+
46
+ dynamic-check:
47
+ name: Dynamic Checks (Python ${{ matrix.python-version }})
48
+ runs-on: ubuntu-latest
49
+ strategy:
50
+ fail-fast: false
51
+ matrix:
52
+ python-version: ["3.11", "3.12", "3.13"]
53
+ env:
54
+ UV_PYTHON: ${{ matrix.python-version }}
55
+ steps:
56
+ - name: Checkout repository
57
+ uses: actions/checkout@v4
58
+ with:
59
+ fetch-depth: 0
60
+
61
+ - name: Set up uv & Python ${{ matrix.python-version }}
62
+ uses: astral-sh/setup-uv@v5
63
+ with:
64
+ python-version: ${{ matrix.python-version }}
65
+ enable-cache: true
66
+ cache-dependency-glob: "uv.lock"
67
+
68
+ - name: Bootstrap environment
69
+ run: make bootstrap UV_SYNC_FLAGS="--frozen"
70
+
71
+ - name: Run tests and verify docs build
72
+ run: make dynamic-check
73
+
74
+ build:
75
+ name: Build Docs & Package
76
+ needs: [static-check, dynamic-check]
77
+ runs-on: ubuntu-latest
78
+ env:
79
+ UV_PYTHON: "3.12"
80
+ steps:
81
+ - name: Checkout repository
82
+ uses: actions/checkout@v4
83
+ with:
84
+ fetch-depth: 0
85
+
86
+ - name: Set up uv & Python
87
+ uses: astral-sh/setup-uv@v5
88
+ with:
89
+ python-version: "3.12"
90
+ enable-cache: true
91
+ cache-dependency-glob: "uv.lock"
92
+
93
+ - name: Bootstrap environment
94
+ run: make bootstrap UV_SYNC_FLAGS="--frozen"
95
+
96
+ - name: Build documentation and distributions
97
+ run: make build
98
+
99
+ - name: Smoke-test built wheel
100
+ run: |
101
+ WHEEL=$(ls dist/*.whl)
102
+ uv run --isolated --no-project --with "$WHEEL" python -c "import dataspector; print(dataspector.__version__)"
103
+ uv run --isolated --no-project --with "$WHEEL" dataspector --help
104
+ uv run --isolated --no-project --with "$WHEEL" coboller --help
105
+
106
+ - name: Upload package distributions
107
+ uses: actions/upload-artifact@v4
108
+ with:
109
+ name: dataspector-dist
110
+ path: dist/
111
+ if-no-files-found: error
112
+
113
+ - name: Upload HTML documentation
114
+ uses: actions/upload-artifact@v4
115
+ with:
116
+ name: dataspector-docs
117
+ path: docs/build/html/
118
+ if-no-files-found: error
119
+
120
+ - name: Add .nojekyll for GitHub Pages
121
+ if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
122
+ run: touch docs/build/html/.nojekyll
123
+
124
+ - name: Upload GitHub Pages artifact
125
+ if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
126
+ uses: actions/upload-pages-artifact@v3
127
+ with:
128
+ path: docs/build/html
129
+
130
+ deploy-docs:
131
+ name: Deploy Docs to GitHub Pages
132
+ needs: [build]
133
+ if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
134
+ runs-on: ubuntu-latest
135
+ permissions:
136
+ pages: write
137
+ id-token: write
138
+ environment:
139
+ name: github-pages
140
+ url: ${{ steps.deployment.outputs.page_url }}
141
+ steps:
142
+ - name: Deploy to GitHub Pages
143
+ id: deployment
144
+ uses: actions/deploy-pages@v4
@@ -0,0 +1,125 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ release:
5
+ types:
6
+ - published
7
+ workflow_dispatch:
8
+ inputs:
9
+ target:
10
+ description: Target package index
11
+ required: true
12
+ default: testpypi
13
+ type: choice
14
+ options:
15
+ - testpypi
16
+ - pypi
17
+
18
+ permissions:
19
+ contents: read
20
+
21
+ jobs:
22
+ build:
23
+ name: Verify & Build Distributions
24
+ runs-on: ubuntu-latest
25
+ env:
26
+ UV_PYTHON: "3.12"
27
+ steps:
28
+ - name: Checkout repository
29
+ uses: actions/checkout@v4
30
+ with:
31
+ fetch-depth: 0
32
+
33
+ - name: Set up uv & Python
34
+ uses: astral-sh/setup-uv@v5
35
+ with:
36
+ python-version: "3.12"
37
+ enable-cache: true
38
+ cache-dependency-glob: "uv.lock"
39
+
40
+ - name: Bootstrap environment
41
+ run: make bootstrap UV_SYNC_FLAGS="--frozen"
42
+
43
+ - name: Run static checks
44
+ run: make static-check
45
+
46
+ - name: Run test suite
47
+ run: make test
48
+
49
+ - name: Build sdist and wheel
50
+ run: make build-package
51
+
52
+ - name: Smoke-test built wheel
53
+ run: |
54
+ WHEEL=$(ls dist/*.whl)
55
+ uv run --isolated --no-project --with "$WHEEL" python -c "import dataspector; print(dataspector.__version__)"
56
+ uv run --isolated --no-project --with "$WHEEL" dataspector --help
57
+ uv run --isolated --no-project --with "$WHEEL" coboller --help
58
+
59
+ - name: Upload distribution artifacts
60
+ uses: actions/upload-artifact@v4
61
+ with:
62
+ name: release-dists
63
+ path: dist/
64
+ if-no-files-found: error
65
+
66
+ attach-release-assets:
67
+ name: Attach Distributions to GitHub Release
68
+ needs: [build]
69
+ if: github.event_name == 'release'
70
+ runs-on: ubuntu-latest
71
+ permissions:
72
+ contents: write
73
+ steps:
74
+ - name: Download distribution artifacts
75
+ uses: actions/download-artifact@v4
76
+ with:
77
+ name: release-dists
78
+ path: dist/
79
+
80
+ - name: Upload sdist and wheel to GitHub Release
81
+ env:
82
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
83
+ run: gh release upload "${{ github.event.release.tag_name }}" dist/* --clobber --repo "${{ github.repository }}"
84
+
85
+ publish-pypi:
86
+ name: Publish to PyPI
87
+ needs: [build]
88
+ if: github.event_name == 'release' || inputs.target == 'pypi'
89
+ runs-on: ubuntu-latest
90
+ environment:
91
+ name: pypi
92
+ url: https://pypi.org/p/dataspector
93
+ permissions:
94
+ id-token: write
95
+ steps:
96
+ - name: Download distribution artifacts
97
+ uses: actions/download-artifact@v4
98
+ with:
99
+ name: release-dists
100
+ path: dist/
101
+
102
+ - name: Publish distributions to PyPI
103
+ uses: pypa/gh-action-pypi-publish@release/v1
104
+
105
+ publish-testpypi:
106
+ name: Publish to TestPyPI
107
+ needs: [build]
108
+ if: github.event_name == 'workflow_dispatch' && inputs.target == 'testpypi'
109
+ runs-on: ubuntu-latest
110
+ environment:
111
+ name: testpypi
112
+ url: https://test.pypi.org/p/dataspector
113
+ permissions:
114
+ id-token: write
115
+ steps:
116
+ - name: Download distribution artifacts
117
+ uses: actions/download-artifact@v4
118
+ with:
119
+ name: release-dists
120
+ path: dist/
121
+
122
+ - name: Publish distributions to TestPyPI
123
+ uses: pypa/gh-action-pypi-publish@release/v1
124
+ with:
125
+ repository-url: https://test.pypi.org/legacy/
@@ -0,0 +1,38 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ *.egg-info/
7
+ dist/
8
+ build/
9
+ .eggs/
10
+
11
+ # Virtual environments
12
+ .venv/
13
+ venv/
14
+ ENV/
15
+
16
+ # Testing & coverage
17
+ .pytest_cache/
18
+ .coverage
19
+ .coverage.*
20
+ coverage.xml
21
+ htmlcov/
22
+ coverage_html_report/
23
+
24
+ # Linters & type checkers
25
+ .ruff_cache/
26
+ .mypy_cache/
27
+ .ty_cache/
28
+
29
+ # Docs build
30
+ docs/build/
31
+ docs/source/_build/
32
+
33
+ # OS / IDE / Local assistant notes
34
+ .DS_Store
35
+ Thumbs.db
36
+ .vscode/
37
+ .idea/
38
+ CLAUDE.md
@@ -0,0 +1 @@
1
+ 3.12
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Aaron Berndsen
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,61 @@
1
+ UV ?= uv
2
+ UV_SYNC_FLAGS ?=
3
+ VENV ?= .venv
4
+ BIN := $(VENV)/bin
5
+
6
+ DOCS_SRC ?= docs/source
7
+ DOCS_BUILD ?= docs/build/html
8
+ SPHINXOPTS ?=
9
+
10
+ LINT_TARGETS := src tests docs/source/conf.py
11
+ TYPECHECK_TARGETS := src tests
12
+
13
+ .DEFAULT_GOAL := help
14
+
15
+ .PHONY: help bootstrap lint format typecheck static-check test docs dynamic-check build-package build ci clean distclean
16
+
17
+ help: ## Show available targets
18
+ @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
19
+ awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-16s\033[0m %s\n", $$1, $$2}'
20
+
21
+ bootstrap: ## Create the virtual environment and install dataspector + dev dependencies
22
+ $(UV) sync --all-groups $(UV_SYNC_FLAGS)
23
+
24
+ lint: ## Check code formatting and linting with ruff
25
+ $(BIN)/ruff format --check $(LINT_TARGETS)
26
+ $(BIN)/ruff check $(LINT_TARGETS)
27
+
28
+ format: ## Auto-format code and fix lint issues with ruff
29
+ $(BIN)/ruff format $(LINT_TARGETS)
30
+ $(BIN)/ruff check --fix $(LINT_TARGETS)
31
+
32
+ typecheck: ## Run static type checking with ty
33
+ $(BIN)/ty check $(TYPECHECK_TARGETS)
34
+
35
+ static-check: lint typecheck ## Run all static checks (lint + type-hint)
36
+
37
+ test: ## Run the pytest suite with coverage
38
+ $(BIN)/pytest
39
+
40
+ docs: ## Build Sphinx HTML documentation
41
+ $(BIN)/sphinx-build -b html $(SPHINXOPTS) $(DOCS_SRC) $(DOCS_BUILD)
42
+
43
+ dynamic-check: test docs ## Run dynamic checks (test suite + docs build)
44
+
45
+ build-package: ## Build sdist and wheel distributions into dist/
46
+ $(UV) build
47
+
48
+ build: docs build-package ## Build documentation and dataspector package
49
+
50
+ ci: bootstrap static-check dynamic-check build ## Run full CI pipeline
51
+
52
+ clean: ## Remove build artifacts, docs output, and test/lint caches
53
+ rm -rf dist build docs/build htmlcov coverage_html_report
54
+ rm -rf .pytest_cache .ruff_cache .mypy_cache .ty_cache .eggs
55
+ rm -f .coverage .coverage.* coverage.xml
56
+ find . -path ./$(VENV) -prune -o -type d -name "__pycache__" -exec rm -rf {} +
57
+ find . -path ./$(VENV) -prune -o -type d -name "*.egg-info" -exec rm -rf {} +
58
+ find . -path ./$(VENV) -prune -o -type f \( -name "*.pyc" -o -name "*.pyo" \) -delete
59
+
60
+ distclean: clean ## Remove all cruft plus the virtual environment (.venv)
61
+ rm -rf $(VENV)
@@ -0,0 +1,181 @@
1
+ Metadata-Version: 2.5
2
+ Name: dataspector
3
+ Version: 1.0.0
4
+ Summary: Data-format-agnostic data-integrity checker with comprehensive COBOL copybook, binary data, and Pandera schema validation support. Convert your COBOL mainframe extracts to a modern format with ease!
5
+ Project-URL: Homepage, https://github.com/aberndsen/dataspector
6
+ Project-URL: Documentation, https://aberndsen.github.io/dataspector/
7
+ Project-URL: Repository, https://github.com/aberndsen/dataspector
8
+ Project-URL: Issues, https://github.com/aberndsen/dataspector/issues
9
+ Author-email: Aaron Berndsen <aaron.berndsen@gmail.com>
10
+ Maintainer-email: Aaron Berndsen <aaron.berndsen@gmail.com>
11
+ License-Expression: MIT
12
+ License-File: LICENSE
13
+ Keywords: cobol,comp-3,copybook,data-quality,data-validation,ebcdic,pandas,pandera
14
+ Classifier: Development Status :: 5 - Production/Stable
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: License :: OSI Approved :: MIT License
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Typing :: Typed
22
+ Requires-Python: >=3.11
23
+ Requires-Dist: chardet>=5.0.0
24
+ Requires-Dist: jsonargparse>=4.30.0
25
+ Requires-Dist: loguru>=0.7.0
26
+ Requires-Dist: numpy>=2.0.0
27
+ Requires-Dist: openpyxl>=3.1.0
28
+ Requires-Dist: pandas>=2.1.0
29
+ Requires-Dist: pandera>=0.36.0
30
+ Requires-Dist: sqlalchemy>=2.1.9
31
+ Requires-Dist: xlsxwriter>=3.3.3
32
+ Provides-Extra: docs
33
+ Requires-Dist: numpydoc>=1.7.0; extra == 'docs'
34
+ Requires-Dist: pydata-sphinx-theme>=0.15.0; extra == 'docs'
35
+ Requires-Dist: sphinx>=7.4.0; extra == 'docs'
36
+ Provides-Extra: odbc
37
+ Requires-Dist: pyodbc>=5.5.0; extra == 'odbc'
38
+ Description-Content-Type: text/markdown
39
+
40
+ # DataSpector
41
+
42
+ **DataSpector** is a format-agnostic Python toolkit for **converting** legacy fixed-width and compressed mainframe/COBOL files into modern human-readable formats and **enforcing data-integrity checks** across heterogeneous data sources (COBOL, CSV, fixed-width text, Excel, Parquet, JSON/JSONL, SQLite, and ODBC databases).
43
+
44
+ ## Highlights
45
+
46
+ - **Convert mainframe & fixed-width COBOL files effortlessly (`coboller`)**: turn opaque EBCDIC, fixed-width, and binary-compressed files (`COMP-3` packed decimal, `COMP`/`COMP-5`/`COMP-X`, `COMP-6`, `COMP-1`/`COMP-2` IEEE & IBM HFP, `BINARY-*`, `FLOAT-*`, `PIC U` UTF-8, `PIC G` DBCS, `PIC 1` bitfields, `OCCURS`, `REDEFINES`) directly into **CSV, TSV, JSON/JSONL, Excel, Parquet, or pandas DataFrames** — via CLI or Python.
47
+ - **Format-agnostic `GenericData` abstraction**: query existence, columns, modification timestamps, storage size, table/sheet lists, and stream records uniformly as `pandas.DataFrame` chunks across all supported formats.
48
+ - **Pandera schema integration**: run declarative `pandera.DataFrameSchema` validations on any format via `record_validate_schema`, or auto-generate a Pandera schema directly from a COBOL copybook with `CopyBook.to_pandera_schema()`.
49
+ - **Excel driver & reporting**: configure batch integrity checks in an Excel workbook, execute sequentially or in parallel, and export color-coded Excel reports.
50
+
51
+ ## Installation
52
+
53
+ DataSpector uses [`uv`](https://docs.astral.sh/uv/) for dependency and environment management:
54
+
55
+ ```bash
56
+ uv sync --dev
57
+ ```
58
+
59
+ Optional extras:
60
+
61
+ ```bash
62
+ uv sync --extra odbc # pyodbc support for SQL Server / MS Access
63
+ uv sync --extra docs # Sphinx + PyData theme documentation dependencies
64
+ ```
65
+
66
+ ## Converting Fixed-Width & Compressed COBOL Files (`coboller`)
67
+
68
+ Mainframe extracts are often unreadable binary/EBCDIC blobs packed with `COMP-3`, `COMP`, and floating-point fields:
69
+
70
+ ```text
71
+ b'\xd5\xc1\xf1\xf0\xf0\xf0\xf0\xf0\xf1\xf7@@\xc1\xf2\xf0\xf2\xf4\xf0\xf1\xf1\xf5\xf0\xf0\xf1\xf2\xf5\xf5\xc0\x00$\x00\x0b^(=.\x14{?\xf0'
72
+ ```
73
+
74
+ ### One-Line CLI Inspection & Conversion
75
+
76
+ Both `coboller` and `dataspector` CLIs are powered by `jsonargparse` from typed dataclasses in `dataspector.settings` (supporting `--config config.yaml` files as well as CLI flags):
77
+
78
+ ```bash
79
+ # Inspect field layout, byte offsets, PIC clauses, and storage types
80
+ uv run coboller inspect tests/data/sample_cobol_mainframe.cbl --omit_filler true
81
+
82
+ # Preview decoded records directly in the terminal
83
+ uv run coboller convert tests/data/sample_cobol_mainframe.cbl tests/data/sample_cobol_mainframe.dat --encoding cp500
84
+
85
+ # Convert binary EBCDIC/COMP-3 data directly to human-readable CSV, JSONL, JSON, or Excel
86
+ uv run coboller convert tests/data/sample_cobol_mainframe.cbl tests/data/sample_cobol_mainframe.dat \
87
+ --encoding cp500 --output accounts.csv
88
+ ```
89
+
90
+ ### Interactive Python Walkthrough
91
+
92
+ ```python
93
+ >>> from dataspector.coboller import Copybook
94
+ >>> cbl = Copybook("tests/data/sample_cobol_mainframe.cbl")
95
+ >>> cbl.record_length
96
+ 67
97
+
98
+ # 1. Inspect the decoded byte layout interactively
99
+ >>> cbl.summary(include_filler=False)[["name", "pic", "usage", "offset", "byte_length"]].head(6)
100
+ name pic usage offset byte_length
101
+ 0 ACCT-REGION XX DISPLAY 0 2
102
+ 1 ACCT-NUMBER X(7) DISPLAY 2 7
103
+ 2 ACCT-CHECK-DIGIT X DISPLAY 9 1
104
+ 3 ACCT-STATUS X DISPLAY 12 1
105
+ 4 ACCT-EFF-DATE 9(8) DISPLAY 13 8
106
+ 5 ACCT-ADJUSTMENT S9(5)V99 DISPLAY 21 7
107
+
108
+ # 2. Preview decoded mainframe records as a clean pandas DataFrame
109
+ >>> df = cbl.to_dataframe(
110
+ ... "tests/data/sample_cobol_mainframe.dat",
111
+ ... encoding="cp500",
112
+ ... include_filler=False,
113
+ ... nrows=5,
114
+ ... )
115
+ >>> df[["ACCT-REGION", "ACCT-NUMBER", "ACCT-STATUS", "ACCT-ADJUSTMENT", "ACCT-BALANCE"]]
116
+ ACCT-REGION ACCT-NUMBER ACCT-STATUS ACCT-ADJUSTMENT ACCT-BALANCE
117
+ 0 NA 1000001 A 125.50 250000.00
118
+ 1 EU 1000002 A -42.75 -15000.50
119
+ 2 AP 1000003 C 0.00 0.00
120
+ 3 NA 1000004 A 9999.99 1234567.89
121
+
122
+ # 3. Stream-convert to human-readable CSV, JSONL, JSON, Excel, or Parquet
123
+ >>> cbl.convert("tests/data/sample_cobol_mainframe.dat", "accounts.csv", encoding="cp500")
124
+ PosixPath('accounts.csv')
125
+ >>> cbl.convert("tests/data/sample_cobol_mainframe.dat", "accounts.jsonl", encoding="cp500")
126
+ PosixPath('accounts.jsonl')
127
+
128
+ # 4. Auto-generate a Pandera schema from the copybook and validate the decoded data
129
+ >>> schema = cbl.to_pandera_schema(include_filler=False, enforce_conditions=True)
130
+ >>> _ = schema.validate(df)
131
+ ```
132
+
133
+ ## Format-Agnostic Data-Integrity Checks
134
+
135
+ ### Python API
136
+
137
+ ```python
138
+ from pathlib import Path
139
+ import dataspector
140
+
141
+ results = dataspector.run_checks_on_location(
142
+ Path("tests/data/sample_csv.csv"),
143
+ [
144
+ ("source_exists", {}),
145
+ ("source_has_N_fields", {"n_fields": 5}),
146
+ ("record_count", {"nrows": 31}),
147
+ ("record_field_numeric_value_between", {"field": "Cost [CAD]", "constraint": (0, 200000)}),
148
+ ],
149
+ )
150
+ assert results.all_passed
151
+ ```
152
+
153
+ ### Batch Checks via Excel Driver CLI
154
+
155
+ ```bash
156
+ uv run dataspector --driver test_driver.xlsx --output report.xlsx
157
+ uv run dataspector --config run.yaml
158
+ ```
159
+
160
+ Key options (from `DataSpectorSettings`):
161
+ - `--driver`: path to the Excel driver workbook (`Location Details` and `Checks to Run` sheets).
162
+ - `--output`: path to write the formatted Excel report (auto-versioned unless `--overwrite true`).
163
+ - `--use_multiprocess true`: run location checks in parallel worker processes.
164
+ - `--log_level`: Loguru verbosity (`ERROR`, `WARNING`, `INFO`, `DEBUG`, `TRACE`).
165
+ - `--config`: optional YAML/JSON configuration file.
166
+
167
+ ## Development
168
+
169
+ ```bash
170
+ # Run tests with coverage
171
+ uv run pytest
172
+
173
+ # Lint & format check
174
+ uv run ruff check src tests
175
+
176
+ # Static type checking
177
+ uv run ty check src tests
178
+
179
+ # Build Sphinx HTML documentation (PyData theme)
180
+ uv run sphinx-build -b html docs/source docs/build/html
181
+ ```