moj-data-loader 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,124 @@
1
+ Metadata-Version: 2.3
2
+ Name: moj-data-loader
3
+ Version: 2.2.0
4
+ Summary: MoJ Data Loader (MoDaL) provides an easy-to-use, config-driven way to load files from S3 into Athena.
5
+ Requires-Dist: awswrangler>=3.17.0
6
+ Requires-Dist: charset-normalizer>=3.4.9
7
+ Requires-Dist: click>=8.4.2
8
+ Requires-Dist: cloudpathlib>=0.24.0
9
+ Requires-Dist: cryptography>=50.0.0
10
+ Requires-Dist: dlt[athena,pyiceberg]>=1.29.1
11
+ Requires-Dist: duckdb>=1.5.5
12
+ Requires-Dist: fsspec>=2026.7.0
13
+ Requires-Dist: gitpython>=3.1.58
14
+ Requires-Dist: loguru>=0.7.3
15
+ Requires-Dist: msoffcrypto-tool>=5.4.2
16
+ Requires-Dist: openpyxl>=3.1.5
17
+ Requires-Dist: pandas>=3.0.5
18
+ Requires-Dist: pathlib>=1.0.1
19
+ Requires-Dist: psutil>=7.2.2
20
+ Requires-Dist: pyarrow>=24.0.0
21
+ Requires-Dist: pydantic>=2.13.4
22
+ Requires-Dist: pydantic-settings>=2.14.2
23
+ Requires-Dist: pyiceberg>=0.9.0
24
+ Requires-Dist: python-calamine>=0.8.2
25
+ Requires-Dist: python-gnupg>=0.5.6
26
+ Requires-Dist: pyyaml>=6.0.2
27
+ Requires-Dist: setuptools>=83.0.0
28
+ Requires-Dist: sqlalchemy>=2.0.51
29
+ Requires-Dist: tornado>=6.6.1
30
+ Requires-Dist: xlrd>=2.0.2
31
+ Requires-Dist: ruff ; extra == 'linting'
32
+ Requires-Python: >=3.12
33
+ Provides-Extra: linting
34
+ Description-Content-Type: text/markdown
35
+
36
+ # MoJ Data Loader (MoDaL)
37
+ MoJ Data Loader provides an easy to use, config-driven way to load files from S3 into Athena.
38
+
39
+ ## Install in Your Project
40
+
41
+ MoDaL is distributed as the `moj-data-loader` Python package on PyPI. It requires Python 3.12 or newer. Install it in the repository that owns your pipelines; you do not need to clone this repository.
42
+
43
+ For a project managed with [uv](https://docs.astral.sh/uv/), run from your project root:
44
+
45
+ ```bash
46
+ uv add moj-data-loader
47
+ uv run moj-data-loader --help
48
+ ```
49
+
50
+ Alternatively, install it in a virtual environment with pip:
51
+
52
+ ```bash
53
+ python -m venv .venv
54
+ source .venv/bin/activate
55
+ python -m pip install moj-data-loader
56
+ moj-data-loader --help
57
+ ```
58
+
59
+ The package name and CLI command are `moj-data-loader`, not `modal`.
60
+
61
+ ## Use Your Own Pipeline Repository
62
+
63
+ Keep your pipeline YAMLs in your own repository:
64
+
65
+ ```text
66
+ your-project/
67
+ pyproject.toml
68
+ pipelines/
69
+ your_domain/
70
+ your_database/
71
+ your_table.yml
72
+ ```
73
+
74
+ Run from that repository's root:
75
+
76
+ ```bash
77
+ uv run moj-data-loader load your_table --env test
78
+ ```
79
+
80
+ With an activated pip environment, omit `uv run`. Configure AWS credentials and permissions for the source S3 data and Athena destination before loading data.
81
+
82
+ By default, MoDaL searches `./pipelines/`, not the installed package's directory. Use `--configs-dir /path/to/pipelines` or `MODAL_CONFIG_DIR` to select another config root, retaining the `domain/database/table.yml` layout.
83
+
84
+ Existing pipelines in this repository can continue to use the monorepo and its Airflow image during the transition. New pipelines will soon no longer be accepted here: create them in your own repository using the PyPI package, and plan to migrate existing pipelines there too. See the [migration guide](docs/migration_v2.md#move-pipelines-to-your-own-repository).
85
+
86
+ ## Releases
87
+
88
+ The Airflow container workflow runs on pushed tags without `/`. A monorepo tag such as `monorepo-v2.1.25` builds the container without publishing to PyPI; use this for interim pipeline-only releases.
89
+
90
+ For a package release, update the version in `pyproject.toml` and the lockfile, then push a version tag such as `v2.1.25` matching the package version. This publishes to PyPI and also builds the container. A manually dispatched PyPI workflow publishes to TestPyPI only. Configure the GitHub `release` environment and PyPI trusted publisher for production publishing.
91
+
92
+ ## Usage guidance
93
+
94
+ You can find usage guidance in [`docs/user_guide`](docs/user_guide.md)
95
+
96
+ Pipeline YAML specification is documented in [`docs/pipeline_config.md`](docs/pipeline_config.md).
97
+
98
+ ## Migration notes
99
+
100
+ v2.0 is a breaking release.
101
+
102
+ - Supported CLI: `moj-data-loader`
103
+ - Supported environment variables: `MODAL_*`
104
+ - Removed in v2.0: `airflow-file` command alias and `AFFILE_*` environment variables
105
+
106
+ For migration steps and old-to-new mappings, see:
107
+
108
+ - [`docs/migration_v2.md`](docs/migration_v2.md)
109
+ - [`docs/release_notes.md`](docs/release_notes.md)
110
+
111
+ ## Useful links
112
+ - Documentation can be found at the Data and Analytics Engineering internal Pages site [here]: TBC
113
+ - Development of this repo is managed via this [Project Board](https://github.com/orgs/moj-analytical-services/projects/262/views/1)
114
+
115
+ ## Contributing
116
+
117
+ To develop MoDaL itself, clone this repository and run:
118
+
119
+ ```bash
120
+ uv sync
121
+ uv run pytest tests/unit
122
+ ```
123
+
124
+ `uv sync` installs MoDaL in editable mode for development. Users should add the released package to their own project's dependencies instead. Commit your project's dependency manifest and lockfile to keep local, CI, and deployment environments consistent.
@@ -0,0 +1,89 @@
1
+ # MoJ Data Loader (MoDaL)
2
+ MoJ Data Loader provides an easy to use, config-driven way to load files from S3 into Athena.
3
+
4
+ ## Install in Your Project
5
+
6
+ MoDaL is distributed as the `moj-data-loader` Python package on PyPI. It requires Python 3.12 or newer. Install it in the repository that owns your pipelines; you do not need to clone this repository.
7
+
8
+ For a project managed with [uv](https://docs.astral.sh/uv/), run from your project root:
9
+
10
+ ```bash
11
+ uv add moj-data-loader
12
+ uv run moj-data-loader --help
13
+ ```
14
+
15
+ Alternatively, install it in a virtual environment with pip:
16
+
17
+ ```bash
18
+ python -m venv .venv
19
+ source .venv/bin/activate
20
+ python -m pip install moj-data-loader
21
+ moj-data-loader --help
22
+ ```
23
+
24
+ The package name and CLI command are `moj-data-loader`, not `modal`.
25
+
26
+ ## Use Your Own Pipeline Repository
27
+
28
+ Keep your pipeline YAMLs in your own repository:
29
+
30
+ ```text
31
+ your-project/
32
+ pyproject.toml
33
+ pipelines/
34
+ your_domain/
35
+ your_database/
36
+ your_table.yml
37
+ ```
38
+
39
+ Run from that repository's root:
40
+
41
+ ```bash
42
+ uv run moj-data-loader load your_table --env test
43
+ ```
44
+
45
+ With an activated pip environment, omit `uv run`. Configure AWS credentials and permissions for the source S3 data and Athena destination before loading data.
46
+
47
+ By default, MoDaL searches `./pipelines/`, not the installed package's directory. Use `--configs-dir /path/to/pipelines` or `MODAL_CONFIG_DIR` to select another config root, retaining the `domain/database/table.yml` layout.
48
+
49
+ Existing pipelines in this repository can continue to use the monorepo and its Airflow image during the transition. New pipelines will soon no longer be accepted here: create them in your own repository using the PyPI package, and plan to migrate existing pipelines there too. See the [migration guide](docs/migration_v2.md#move-pipelines-to-your-own-repository).
50
+
51
+ ## Releases
52
+
53
+ The Airflow container workflow runs on pushed tags without `/`. A monorepo tag such as `monorepo-v2.1.25` builds the container without publishing to PyPI; use this for interim pipeline-only releases.
54
+
55
+ For a package release, update the version in `pyproject.toml` and the lockfile, then push a version tag such as `v2.1.25` matching the package version. This publishes to PyPI and also builds the container. A manually dispatched PyPI workflow publishes to TestPyPI only. Configure the GitHub `release` environment and PyPI trusted publisher for production publishing.
56
+
57
+ ## Usage guidance
58
+
59
+ You can find usage guidance in [`docs/user_guide`](docs/user_guide.md)
60
+
61
+ Pipeline YAML specification is documented in [`docs/pipeline_config.md`](docs/pipeline_config.md).
62
+
63
+ ## Migration notes
64
+
65
+ v2.0 is a breaking release.
66
+
67
+ - Supported CLI: `moj-data-loader`
68
+ - Supported environment variables: `MODAL_*`
69
+ - Removed in v2.0: `airflow-file` command alias and `AFFILE_*` environment variables
70
+
71
+ For migration steps and old-to-new mappings, see:
72
+
73
+ - [`docs/migration_v2.md`](docs/migration_v2.md)
74
+ - [`docs/release_notes.md`](docs/release_notes.md)
75
+
76
+ ## Useful links
77
+ - Documentation can be found at the Data and Analytics Engineering internal Pages site [here]: TBC
78
+ - Development of this repo is managed via this [Project Board](https://github.com/orgs/moj-analytical-services/projects/262/views/1)
79
+
80
+ ## Contributing
81
+
82
+ To develop MoDaL itself, clone this repository and run:
83
+
84
+ ```bash
85
+ uv sync
86
+ uv run pytest tests/unit
87
+ ```
88
+
89
+ `uv sync` installs MoDaL in editable mode for development. Users should add the released package to their own project's dependencies instead. Commit your project's dependency manifest and lockfile to keep local, CI, and deployment environments consistent.
@@ -0,0 +1,145 @@
1
+ [build-system]
2
+ requires = ["uv_build>=0.13.1,<0.14"]
3
+ build-backend = "uv_build"
4
+
5
+ [project]
6
+ name = "moj-data-loader"
7
+ version = "2.2.0"
8
+ description = "MoJ Data Loader (MoDaL) provides an easy-to-use, config-driven way to load files from S3 into Athena."
9
+ readme = "README.md"
10
+ requires-python = ">=3.12"
11
+ dependencies = [
12
+ "awswrangler>=3.17.0",
13
+ "charset-normalizer>=3.4.9",
14
+ "click>=8.4.2",
15
+ "cloudpathlib>=0.24.0",
16
+ "cryptography>=50.0.0",
17
+ "dlt[athena,pyiceberg]>=1.29.1",
18
+ "duckdb>=1.5.5",
19
+ "fsspec>=2026.7.0",
20
+ "gitpython>=3.1.58",
21
+ "loguru>=0.7.3",
22
+ "msoffcrypto-tool>=5.4.2",
23
+ "openpyxl>=3.1.5",
24
+ "pandas>=3.0.5",
25
+ "pathlib>=1.0.1",
26
+ "psutil>=7.2.2",
27
+ "pyarrow>=24.0.0",
28
+ "pydantic>=2.13.4",
29
+ "pydantic-settings>=2.14.2",
30
+ "pyiceberg>=0.9.0",
31
+ "python-calamine>=0.8.2",
32
+ "python-gnupg>=0.5.6",
33
+ "pyyaml>=6.0.2",
34
+ "setuptools>=83.0.0",
35
+ "sqlalchemy>=2.0.51",
36
+ "tornado>=6.6.1",
37
+ "xlrd>=2.0.2",
38
+ ]
39
+
40
+ [project.scripts]
41
+ moj-data-loader = "moj_data_loader.main:cli"
42
+
43
+ [project.optional-dependencies]
44
+ linting = ["ruff"]
45
+
46
+ [tool.uv]
47
+ package = true
48
+ exclude-newer = "7 days"
49
+
50
+ [tool.uv.exclude-newer-package]
51
+ tornado = "2026-11-10T00:00:00Z"
52
+
53
+ [tool.uv.build-backend]
54
+ source-exclude = ["/pipelines"]
55
+
56
+ [tool.ruff]
57
+ line-length = 90
58
+ cache-dir = "~/.cache/ruff"
59
+ target-version = "py312"
60
+ fix = false
61
+ respect-gitignore = true
62
+ exclude = [
63
+ ".svn",
64
+ "CVS",
65
+ ".bzr",
66
+ ".hg",
67
+ ".git",
68
+ "__pycache__",
69
+ ".tox",
70
+ "venv/",
71
+ "env/",
72
+ ".github/",
73
+ "githooks/",
74
+ ]
75
+
76
+ [tool.ruff.lint]
77
+ ignore = [
78
+ "E731",
79
+ "W191",
80
+ "E111",
81
+ "E114",
82
+ "E117",
83
+ "D206",
84
+ "D300",
85
+ "Q000",
86
+ "Q001",
87
+ "Q002",
88
+ "Q003",
89
+ "COM812",
90
+ "COM819",
91
+ "ISC001",
92
+ "ISC002",
93
+ "E266",
94
+ ]
95
+ select = [
96
+ "E",
97
+ "F",
98
+ "W",
99
+ "I001",
100
+ "T20",
101
+ ]
102
+ fixable = ["ALL"]
103
+ unfixable = []
104
+
105
+ [tool.ruff.lint.isort]
106
+ combine-as-imports = true
107
+
108
+ [tool.ruff.lint.flake8-errmsg]
109
+ max-string-length = 50
110
+
111
+ [tool.ruff.format]
112
+ quote-style = "double"
113
+ indent-style = "space"
114
+ skip-magic-trailing-comma = false
115
+ line-ending = "auto"
116
+ docstring-code-format = true
117
+ docstring-code-line-length = "dynamic"
118
+ exclude = ["*.ipynb"]
119
+
120
+ [tool.pytest.ini_options]
121
+ pythonpath = ["."]
122
+ testpaths = ["tests"]
123
+ addopts = "-ra --strict-markers"
124
+ markers = [
125
+ "unit: mark a test as a unit test.",
126
+ "integration: mark a test as an integration test.",
127
+ "e2e: mark a test as end-to-end.",
128
+ ]
129
+
130
+ [dependency-groups]
131
+ dev = [
132
+ "moto[athena,glue,s3]>=5.2.2",
133
+ "pre-commit>=4.6.1",
134
+ "duckdb>=1.5.5",
135
+ "ipykernel>=7.3.0",
136
+ "pytest>=9.1.1",
137
+ "pytest-bdd>=8.1.0",
138
+ "pytest-mock>=3.15.1",
139
+ "ruff>=0.9.9",
140
+ "dotenv>=0.9.9",
141
+ "typer>=0.27.0",
142
+ "virtualenv>=21.14.13",
143
+ "uv==0.13.6",
144
+ ]
145
+ linting = ["ruff"]
@@ -0,0 +1,164 @@
1
+ [build-system]
2
+ requires = ["uv_build>=0.13.1,<0.14"]
3
+ build-backend = "uv_build"
4
+
5
+ [project]
6
+ name = "moj-data-loader"
7
+ version = "2.2.0"
8
+ description = "MoJ Data Loader (MoDaL) provides an easy-to-use, config-driven way to load files from S3 into Athena."
9
+ readme = "README.md"
10
+ requires-python = ">=3.12"
11
+ dependencies = [
12
+ "awswrangler>=3.17.0",
13
+ "charset-normalizer>=3.4.9",
14
+ "click>=8.4.2",
15
+ "cloudpathlib>=0.24.0",
16
+ "cryptography>=50.0.0",
17
+ "dlt[athena,pyiceberg]>=1.29.1",
18
+ "duckdb>=1.5.5",
19
+ "fsspec>=2026.7.0",
20
+ "gitpython>=3.1.58",
21
+ "loguru>=0.7.3",
22
+ "msoffcrypto-tool>=5.4.2",
23
+ "openpyxl>=3.1.5",
24
+ "pandas>=3.0.5",
25
+ "pathlib>=1.0.1",
26
+ "psutil>=7.2.2",
27
+ "pyarrow>=24.0.0",
28
+ "pydantic>=2.13.4",
29
+ "pydantic-settings>=2.14.2",
30
+ "pyiceberg>=0.9.0",
31
+ "python-calamine>=0.8.2",
32
+ "python-gnupg>=0.5.6",
33
+ "pyyaml>=6.0.2",
34
+ "setuptools>=83.0.0",
35
+ "sqlalchemy>=2.0.51",
36
+ "tornado>=6.6.1",
37
+ "xlrd>=2.0.2",
38
+ ]
39
+
40
+ [project.scripts]
41
+ moj-data-loader = "moj_data_loader.main:cli"
42
+
43
+ [tool.uv]
44
+ package = true
45
+ exclude-newer = "7 days"
46
+ exclude-newer-package = { tornado = "2026-11-10T00:00:00Z" }
47
+
48
+ [tool.uv.build-backend]
49
+ source-exclude = ["/pipelines"]
50
+
51
+ [dependency-groups]
52
+ dev = [
53
+ "moto[athena,glue,s3]>=5.2.2",
54
+ "pre-commit>=4.6.1",
55
+ "duckdb>=1.5.5",
56
+ "ipykernel>=7.3.0",
57
+ "pytest>=9.1.1",
58
+ "pytest-bdd>=8.1.0",
59
+ "pytest-mock>=3.15.1",
60
+ "ruff>=0.9.9",
61
+ "dotenv>=0.9.9",
62
+ "typer>=0.27.0",
63
+ "virtualenv>=21.14.13",
64
+ "uv==0.13.6",
65
+ ]
66
+ linting = ["ruff"]
67
+
68
+ [project.optional-dependencies]
69
+ linting = ["ruff"]
70
+
71
+ [tool.ruff]
72
+ # General settings
73
+ line-length = 90
74
+ cache-dir = "~/.cache/ruff"
75
+ target-version = "py312"
76
+ fix = false
77
+ respect-gitignore = true
78
+
79
+ # Exclude specific directories from being linted
80
+ exclude = [
81
+ ".svn",
82
+ "CVS",
83
+ ".bzr",
84
+ ".hg",
85
+ ".git",
86
+ "__pycache__",
87
+ ".tox",
88
+ "venv/",
89
+ "env/",
90
+ ".github/",
91
+ "githooks/",
92
+ ]
93
+
94
+ # Rules to ignore when linting
95
+ lint.ignore = [
96
+ "E731", # Do not assign a `lambda` expression, use a `def`
97
+ "W191", # Indentation contains tabs
98
+ "E111", # Indentation is not a multiple of four
99
+ "E114", # Indentation is not a multiple of four (comment)
100
+ "E117", # Over-indented
101
+ "D206", # Docstring should be indented with spaces, not tabs
102
+ "D300", # Use """triple double quotes""" (found '''-quotes)
103
+ # Q00x - Prefer either single or double quotes, but be consistent
104
+ "Q000",
105
+ "Q001",
106
+ "Q002",
107
+ "Q003",
108
+ "COM812", # Missing trailing comma
109
+ "COM819", # Trailing comma is missing
110
+ "ISC001", # isort found unsorted imports
111
+ "ISC002", # isort found ungrouped imports
112
+ "E266", # Too many leading '#' for block comment - ####
113
+ ]
114
+ # Rules to include in the linting selection
115
+ lint.select = [
116
+ # pycodestyle (default)
117
+ "E",
118
+ # Pyflakes (default)
119
+ "F",
120
+ # Whitespace warnings
121
+ "W",
122
+ # isort (sort imports)
123
+ "I001",
124
+ # flake8-print
125
+ "T20"
126
+ ]
127
+ lint.fixable = ["ALL"]
128
+ lint.unfixable = []
129
+
130
+ # isort settings
131
+ [tool.ruff.lint.isort]
132
+ combine-as-imports = true
133
+
134
+ # flake8-errmsg settings
135
+ [tool.ruff.lint.flake8-errmsg]
136
+ max-string-length = 50
137
+
138
+ # Formatting rules (inspired by Black)
139
+ [tool.ruff.format]
140
+ quote-style = "double"
141
+ indent-style = "space"
142
+ skip-magic-trailing-comma = false
143
+ line-ending = "auto"
144
+
145
+ # Docstring code formatting
146
+ docstring-code-format = true
147
+ docstring-code-line-length = "dynamic"
148
+
149
+ # Exclude formatting for jupyter notebooks
150
+ exclude = ["*.ipynb"]
151
+
152
+ [tool.pytest.ini_options]
153
+ pythonpath = [
154
+ "."
155
+ ]
156
+ testpaths = [
157
+ "tests"
158
+ ]
159
+ addopts = "-ra --strict-markers"
160
+ markers = [
161
+ "unit: mark a test as a unit test.",
162
+ "integration: mark a test as an integration test.",
163
+ "e2e: mark a test as end-to-end.",
164
+ ]