moj-data-loader 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moj_data_loader-2.2.0/PKG-INFO +124 -0
- moj_data_loader-2.2.0/README.md +89 -0
- moj_data_loader-2.2.0/pyproject.toml +145 -0
- moj_data_loader-2.2.0/pyproject.toml.orig +164 -0
- moj_data_loader-2.2.0/src/moj_data_loader/Pipeline.py +659 -0
- moj_data_loader-2.2.0/src/moj_data_loader/__init__.py +0 -0
- moj_data_loader-2.2.0/src/moj_data_loader/main.py +291 -0
- moj_data_loader-2.2.0/src/moj_data_loader/readers/__init__.py +0 -0
- moj_data_loader-2.2.0/src/moj_data_loader/readers/registry.py +194 -0
- moj_data_loader-2.2.0/src/moj_data_loader/readers/transforms.py +334 -0
- moj_data_loader-2.2.0/src/moj_data_loader/utils/__init__.py +0 -0
- moj_data_loader-2.2.0/src/moj_data_loader/utils/batch.py +221 -0
- moj_data_loader-2.2.0/src/moj_data_loader/utils/exceptions.py +6 -0
- moj_data_loader-2.2.0/src/moj_data_loader/utils/files.py +46 -0
- moj_data_loader-2.2.0/src/moj_data_loader/utils/pgp.py +223 -0
- moj_data_loader-2.2.0/src/moj_data_loader/utils/summary_log.py +81 -0
- moj_data_loader-2.2.0/src/moj_data_loader/utils/validators.py +15 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: moj-data-loader
|
|
3
|
+
Version: 2.2.0
|
|
4
|
+
Summary: MoJ Data Loader (MoDaL) provides an easy-to-use, config-driven way to load files from S3 into Athena.
|
|
5
|
+
Requires-Dist: awswrangler>=3.17.0
|
|
6
|
+
Requires-Dist: charset-normalizer>=3.4.9
|
|
7
|
+
Requires-Dist: click>=8.4.2
|
|
8
|
+
Requires-Dist: cloudpathlib>=0.24.0
|
|
9
|
+
Requires-Dist: cryptography>=50.0.0
|
|
10
|
+
Requires-Dist: dlt[athena,pyiceberg]>=1.29.1
|
|
11
|
+
Requires-Dist: duckdb>=1.5.5
|
|
12
|
+
Requires-Dist: fsspec>=2026.7.0
|
|
13
|
+
Requires-Dist: gitpython>=3.1.58
|
|
14
|
+
Requires-Dist: loguru>=0.7.3
|
|
15
|
+
Requires-Dist: msoffcrypto-tool>=5.4.2
|
|
16
|
+
Requires-Dist: openpyxl>=3.1.5
|
|
17
|
+
Requires-Dist: pandas>=3.0.5
|
|
18
|
+
Requires-Dist: pathlib>=1.0.1
|
|
19
|
+
Requires-Dist: psutil>=7.2.2
|
|
20
|
+
Requires-Dist: pyarrow>=24.0.0
|
|
21
|
+
Requires-Dist: pydantic>=2.13.4
|
|
22
|
+
Requires-Dist: pydantic-settings>=2.14.2
|
|
23
|
+
Requires-Dist: pyiceberg>=0.9.0
|
|
24
|
+
Requires-Dist: python-calamine>=0.8.2
|
|
25
|
+
Requires-Dist: python-gnupg>=0.5.6
|
|
26
|
+
Requires-Dist: pyyaml>=6.0.2
|
|
27
|
+
Requires-Dist: setuptools>=83.0.0
|
|
28
|
+
Requires-Dist: sqlalchemy>=2.0.51
|
|
29
|
+
Requires-Dist: tornado>=6.6.1
|
|
30
|
+
Requires-Dist: xlrd>=2.0.2
|
|
31
|
+
Requires-Dist: ruff ; extra == 'linting'
|
|
32
|
+
Requires-Python: >=3.12
|
|
33
|
+
Provides-Extra: linting
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
# MoJ Data Loader (MoDaL)
|
|
37
|
+
MoJ Data Loader provides an easy to use, config-driven way to load files from S3 into Athena.
|
|
38
|
+
|
|
39
|
+
## Install in Your Project
|
|
40
|
+
|
|
41
|
+
MoDaL is distributed as the `moj-data-loader` Python package on PyPI. It requires Python 3.12 or newer. Install it in the repository that owns your pipelines; you do not need to clone this repository.
|
|
42
|
+
|
|
43
|
+
For a project managed with [uv](https://docs.astral.sh/uv/), run from your project root:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
uv add moj-data-loader
|
|
47
|
+
uv run moj-data-loader --help
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Alternatively, install it in a virtual environment with pip:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
python -m venv .venv
|
|
54
|
+
source .venv/bin/activate
|
|
55
|
+
python -m pip install moj-data-loader
|
|
56
|
+
moj-data-loader --help
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
The package name and CLI command are `moj-data-loader`, not `modal`.
|
|
60
|
+
|
|
61
|
+
## Use Your Own Pipeline Repository
|
|
62
|
+
|
|
63
|
+
Keep your pipeline YAMLs in your own repository:
|
|
64
|
+
|
|
65
|
+
```text
|
|
66
|
+
your-project/
|
|
67
|
+
pyproject.toml
|
|
68
|
+
pipelines/
|
|
69
|
+
your_domain/
|
|
70
|
+
your_database/
|
|
71
|
+
your_table.yml
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Run from that repository's root:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
uv run moj-data-loader load your_table --env test
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
With an activated pip environment, omit `uv run`. Configure AWS credentials and permissions for the source S3 data and Athena destination before loading data.
|
|
81
|
+
|
|
82
|
+
By default, MoDaL searches `./pipelines/`, not the installed package's directory. Use `--configs-dir /path/to/pipelines` or `MODAL_CONFIG_DIR` to select another config root, retaining the `domain/database/table.yml` layout.
|
|
83
|
+
|
|
84
|
+
Existing pipelines in this repository can continue to use the monorepo and its Airflow image during the transition. New pipelines will soon no longer be accepted here: create them in your own repository using the PyPI package, and plan to migrate existing pipelines there too. See the [migration guide](docs/migration_v2.md#move-pipelines-to-your-own-repository).
|
|
85
|
+
|
|
86
|
+
## Releases
|
|
87
|
+
|
|
88
|
+
The Airflow container workflow runs on pushed tags without `/`. A monorepo tag such as `monorepo-v2.1.25` builds the container without publishing to PyPI; use this for interim pipeline-only releases.
|
|
89
|
+
|
|
90
|
+
For a package release, update the version in `pyproject.toml` and the lockfile, then push a version tag such as `v2.1.25` matching the package version. This publishes to PyPI and also builds the container. A manually dispatched PyPI workflow publishes to TestPyPI only. Configure the GitHub `release` environment and PyPI trusted publisher for production publishing.
|
|
91
|
+
|
|
92
|
+
## Usage guidance
|
|
93
|
+
|
|
94
|
+
You can find usage guidance in [`docs/user_guide`](docs/user_guide.md)
|
|
95
|
+
|
|
96
|
+
Pipeline YAML specification is documented in [`docs/pipeline_config.md`](docs/pipeline_config.md).
|
|
97
|
+
|
|
98
|
+
## Migration notes
|
|
99
|
+
|
|
100
|
+
v2.0 is a breaking release.
|
|
101
|
+
|
|
102
|
+
- Supported CLI: `moj-data-loader`
|
|
103
|
+
- Supported environment variables: `MODAL_*`
|
|
104
|
+
- Removed in v2.0: `airflow-file` command alias and `AFFILE_*` environment variables
|
|
105
|
+
|
|
106
|
+
For migration steps and old-to-new mappings, see:
|
|
107
|
+
|
|
108
|
+
- [`docs/migration_v2.md`](docs/migration_v2.md)
|
|
109
|
+
- [`docs/release_notes.md`](docs/release_notes.md)
|
|
110
|
+
|
|
111
|
+
## Useful links
|
|
112
|
+
- Documentation can be found at the Data and Analytics Engineering internal Pages site [here]: TBC
|
|
113
|
+
- Development of this repo is managed via this [Project Board](https://github.com/orgs/moj-analytical-services/projects/262/views/1)
|
|
114
|
+
|
|
115
|
+
## Contributing
|
|
116
|
+
|
|
117
|
+
To develop MoDaL itself, clone this repository and run:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
uv sync
|
|
121
|
+
uv run pytest tests/unit
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
`uv sync` installs MoDaL in editable mode for development. Users should add the released package to their own project's dependencies instead. Commit your project's dependency manifest and lockfile to keep local, CI, and deployment environments consistent.
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# MoJ Data Loader (MoDaL)
|
|
2
|
+
MoJ Data Loader provides an easy to use, config-driven way to load files from S3 into Athena.
|
|
3
|
+
|
|
4
|
+
## Install in Your Project
|
|
5
|
+
|
|
6
|
+
MoDaL is distributed as the `moj-data-loader` Python package on PyPI. It requires Python 3.12 or newer. Install it in the repository that owns your pipelines; you do not need to clone this repository.
|
|
7
|
+
|
|
8
|
+
For a project managed with [uv](https://docs.astral.sh/uv/), run from your project root:
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
uv add moj-data-loader
|
|
12
|
+
uv run moj-data-loader --help
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Alternatively, install it in a virtual environment with pip:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
python -m venv .venv
|
|
19
|
+
source .venv/bin/activate
|
|
20
|
+
python -m pip install moj-data-loader
|
|
21
|
+
moj-data-loader --help
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
The package name and CLI command are `moj-data-loader`, not `modal`.
|
|
25
|
+
|
|
26
|
+
## Use Your Own Pipeline Repository
|
|
27
|
+
|
|
28
|
+
Keep your pipeline YAMLs in your own repository:
|
|
29
|
+
|
|
30
|
+
```text
|
|
31
|
+
your-project/
|
|
32
|
+
pyproject.toml
|
|
33
|
+
pipelines/
|
|
34
|
+
your_domain/
|
|
35
|
+
your_database/
|
|
36
|
+
your_table.yml
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Run from that repository's root:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
uv run moj-data-loader load your_table --env test
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
With an activated pip environment, omit `uv run`. Configure AWS credentials and permissions for the source S3 data and Athena destination before loading data.
|
|
46
|
+
|
|
47
|
+
By default, MoDaL searches `./pipelines/`, not the installed package's directory. Use `--configs-dir /path/to/pipelines` or `MODAL_CONFIG_DIR` to select another config root, retaining the `domain/database/table.yml` layout.
|
|
48
|
+
|
|
49
|
+
Existing pipelines in this repository can continue to use the monorepo and its Airflow image during the transition. New pipelines will soon no longer be accepted here: create them in your own repository using the PyPI package, and plan to migrate existing pipelines there too. See the [migration guide](docs/migration_v2.md#move-pipelines-to-your-own-repository).
|
|
50
|
+
|
|
51
|
+
## Releases
|
|
52
|
+
|
|
53
|
+
The Airflow container workflow runs on pushed tags without `/`. A monorepo tag such as `monorepo-v2.1.25` builds the container without publishing to PyPI; use this for interim pipeline-only releases.
|
|
54
|
+
|
|
55
|
+
For a package release, update the version in `pyproject.toml` and the lockfile, then push a version tag such as `v2.1.25` matching the package version. This publishes to PyPI and also builds the container. A manually dispatched PyPI workflow publishes to TestPyPI only. Configure the GitHub `release` environment and PyPI trusted publisher for production publishing.
|
|
56
|
+
|
|
57
|
+
## Usage guidance
|
|
58
|
+
|
|
59
|
+
You can find usage guidance in [`docs/user_guide`](docs/user_guide.md)
|
|
60
|
+
|
|
61
|
+
Pipeline YAML specification is documented in [`docs/pipeline_config.md`](docs/pipeline_config.md).
|
|
62
|
+
|
|
63
|
+
## Migration notes
|
|
64
|
+
|
|
65
|
+
v2.0 is a breaking release.
|
|
66
|
+
|
|
67
|
+
- Supported CLI: `moj-data-loader`
|
|
68
|
+
- Supported environment variables: `MODAL_*`
|
|
69
|
+
- Removed in v2.0: `airflow-file` command alias and `AFFILE_*` environment variables
|
|
70
|
+
|
|
71
|
+
For migration steps and old-to-new mappings, see:
|
|
72
|
+
|
|
73
|
+
- [`docs/migration_v2.md`](docs/migration_v2.md)
|
|
74
|
+
- [`docs/release_notes.md`](docs/release_notes.md)
|
|
75
|
+
|
|
76
|
+
## Useful links
|
|
77
|
+
- Documentation can be found at the Data and Analytics Engineering internal Pages site [here]: TBC
|
|
78
|
+
- Development of this repo is managed via this [Project Board](https://github.com/orgs/moj-analytical-services/projects/262/views/1)
|
|
79
|
+
|
|
80
|
+
## Contributing
|
|
81
|
+
|
|
82
|
+
To develop MoDaL itself, clone this repository and run:
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
uv sync
|
|
86
|
+
uv run pytest tests/unit
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
`uv sync` installs MoDaL in editable mode for development. Users should add the released package to their own project's dependencies instead. Commit your project's dependency manifest and lockfile to keep local, CI, and deployment environments consistent.
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["uv_build>=0.13.1,<0.14"]
|
|
3
|
+
build-backend = "uv_build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "moj-data-loader"
|
|
7
|
+
version = "2.2.0"
|
|
8
|
+
description = "MoJ Data Loader (MoDaL) provides an easy-to-use, config-driven way to load files from S3 into Athena."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"awswrangler>=3.17.0",
|
|
13
|
+
"charset-normalizer>=3.4.9",
|
|
14
|
+
"click>=8.4.2",
|
|
15
|
+
"cloudpathlib>=0.24.0",
|
|
16
|
+
"cryptography>=50.0.0",
|
|
17
|
+
"dlt[athena,pyiceberg]>=1.29.1",
|
|
18
|
+
"duckdb>=1.5.5",
|
|
19
|
+
"fsspec>=2026.7.0",
|
|
20
|
+
"gitpython>=3.1.58",
|
|
21
|
+
"loguru>=0.7.3",
|
|
22
|
+
"msoffcrypto-tool>=5.4.2",
|
|
23
|
+
"openpyxl>=3.1.5",
|
|
24
|
+
"pandas>=3.0.5",
|
|
25
|
+
"pathlib>=1.0.1",
|
|
26
|
+
"psutil>=7.2.2",
|
|
27
|
+
"pyarrow>=24.0.0",
|
|
28
|
+
"pydantic>=2.13.4",
|
|
29
|
+
"pydantic-settings>=2.14.2",
|
|
30
|
+
"pyiceberg>=0.9.0",
|
|
31
|
+
"python-calamine>=0.8.2",
|
|
32
|
+
"python-gnupg>=0.5.6",
|
|
33
|
+
"pyyaml>=6.0.2",
|
|
34
|
+
"setuptools>=83.0.0",
|
|
35
|
+
"sqlalchemy>=2.0.51",
|
|
36
|
+
"tornado>=6.6.1",
|
|
37
|
+
"xlrd>=2.0.2",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[project.scripts]
|
|
41
|
+
moj-data-loader = "moj_data_loader.main:cli"
|
|
42
|
+
|
|
43
|
+
[project.optional-dependencies]
|
|
44
|
+
linting = ["ruff"]
|
|
45
|
+
|
|
46
|
+
[tool.uv]
|
|
47
|
+
package = true
|
|
48
|
+
exclude-newer = "7 days"
|
|
49
|
+
|
|
50
|
+
[tool.uv.exclude-newer-package]
|
|
51
|
+
tornado = "2026-11-10T00:00:00Z"
|
|
52
|
+
|
|
53
|
+
[tool.uv.build-backend]
|
|
54
|
+
source-exclude = ["/pipelines"]
|
|
55
|
+
|
|
56
|
+
[tool.ruff]
|
|
57
|
+
line-length = 90
|
|
58
|
+
cache-dir = "~/.cache/ruff"
|
|
59
|
+
target-version = "py312"
|
|
60
|
+
fix = false
|
|
61
|
+
respect-gitignore = true
|
|
62
|
+
exclude = [
|
|
63
|
+
".svn",
|
|
64
|
+
"CVS",
|
|
65
|
+
".bzr",
|
|
66
|
+
".hg",
|
|
67
|
+
".git",
|
|
68
|
+
"__pycache__",
|
|
69
|
+
".tox",
|
|
70
|
+
"venv/",
|
|
71
|
+
"env/",
|
|
72
|
+
".github/",
|
|
73
|
+
"githooks/",
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
[tool.ruff.lint]
|
|
77
|
+
ignore = [
|
|
78
|
+
"E731",
|
|
79
|
+
"W191",
|
|
80
|
+
"E111",
|
|
81
|
+
"E114",
|
|
82
|
+
"E117",
|
|
83
|
+
"D206",
|
|
84
|
+
"D300",
|
|
85
|
+
"Q000",
|
|
86
|
+
"Q001",
|
|
87
|
+
"Q002",
|
|
88
|
+
"Q003",
|
|
89
|
+
"COM812",
|
|
90
|
+
"COM819",
|
|
91
|
+
"ISC001",
|
|
92
|
+
"ISC002",
|
|
93
|
+
"E266",
|
|
94
|
+
]
|
|
95
|
+
select = [
|
|
96
|
+
"E",
|
|
97
|
+
"F",
|
|
98
|
+
"W",
|
|
99
|
+
"I001",
|
|
100
|
+
"T20",
|
|
101
|
+
]
|
|
102
|
+
fixable = ["ALL"]
|
|
103
|
+
unfixable = []
|
|
104
|
+
|
|
105
|
+
[tool.ruff.lint.isort]
|
|
106
|
+
combine-as-imports = true
|
|
107
|
+
|
|
108
|
+
[tool.ruff.lint.flake8-errmsg]
|
|
109
|
+
max-string-length = 50
|
|
110
|
+
|
|
111
|
+
[tool.ruff.format]
|
|
112
|
+
quote-style = "double"
|
|
113
|
+
indent-style = "space"
|
|
114
|
+
skip-magic-trailing-comma = false
|
|
115
|
+
line-ending = "auto"
|
|
116
|
+
docstring-code-format = true
|
|
117
|
+
docstring-code-line-length = "dynamic"
|
|
118
|
+
exclude = ["*.ipynb"]
|
|
119
|
+
|
|
120
|
+
[tool.pytest.ini_options]
|
|
121
|
+
pythonpath = ["."]
|
|
122
|
+
testpaths = ["tests"]
|
|
123
|
+
addopts = "-ra --strict-markers"
|
|
124
|
+
markers = [
|
|
125
|
+
"unit: mark a test as a unit test.",
|
|
126
|
+
"integration: mark a test as an integration test.",
|
|
127
|
+
"e2e: mark a test as end-to-end.",
|
|
128
|
+
]
|
|
129
|
+
|
|
130
|
+
[dependency-groups]
|
|
131
|
+
dev = [
|
|
132
|
+
"moto[athena,glue,s3]>=5.2.2",
|
|
133
|
+
"pre-commit>=4.6.1",
|
|
134
|
+
"duckdb>=1.5.5",
|
|
135
|
+
"ipykernel>=7.3.0",
|
|
136
|
+
"pytest>=9.1.1",
|
|
137
|
+
"pytest-bdd>=8.1.0",
|
|
138
|
+
"pytest-mock>=3.15.1",
|
|
139
|
+
"ruff>=0.9.9",
|
|
140
|
+
"dotenv>=0.9.9",
|
|
141
|
+
"typer>=0.27.0",
|
|
142
|
+
"virtualenv>=21.14.13",
|
|
143
|
+
"uv==0.13.6",
|
|
144
|
+
]
|
|
145
|
+
linting = ["ruff"]
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["uv_build>=0.13.1,<0.14"]
|
|
3
|
+
build-backend = "uv_build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "moj-data-loader"
|
|
7
|
+
version = "2.2.0"
|
|
8
|
+
description = "MoJ Data Loader (MoDaL) provides an easy-to-use, config-driven way to load files from S3 into Athena."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"awswrangler>=3.17.0",
|
|
13
|
+
"charset-normalizer>=3.4.9",
|
|
14
|
+
"click>=8.4.2",
|
|
15
|
+
"cloudpathlib>=0.24.0",
|
|
16
|
+
"cryptography>=50.0.0",
|
|
17
|
+
"dlt[athena,pyiceberg]>=1.29.1",
|
|
18
|
+
"duckdb>=1.5.5",
|
|
19
|
+
"fsspec>=2026.7.0",
|
|
20
|
+
"gitpython>=3.1.58",
|
|
21
|
+
"loguru>=0.7.3",
|
|
22
|
+
"msoffcrypto-tool>=5.4.2",
|
|
23
|
+
"openpyxl>=3.1.5",
|
|
24
|
+
"pandas>=3.0.5",
|
|
25
|
+
"pathlib>=1.0.1",
|
|
26
|
+
"psutil>=7.2.2",
|
|
27
|
+
"pyarrow>=24.0.0",
|
|
28
|
+
"pydantic>=2.13.4",
|
|
29
|
+
"pydantic-settings>=2.14.2",
|
|
30
|
+
"pyiceberg>=0.9.0",
|
|
31
|
+
"python-calamine>=0.8.2",
|
|
32
|
+
"python-gnupg>=0.5.6",
|
|
33
|
+
"pyyaml>=6.0.2",
|
|
34
|
+
"setuptools>=83.0.0",
|
|
35
|
+
"sqlalchemy>=2.0.51",
|
|
36
|
+
"tornado>=6.6.1",
|
|
37
|
+
"xlrd>=2.0.2",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[project.scripts]
|
|
41
|
+
moj-data-loader = "moj_data_loader.main:cli"
|
|
42
|
+
|
|
43
|
+
[tool.uv]
|
|
44
|
+
package = true
|
|
45
|
+
exclude-newer = "7 days"
|
|
46
|
+
exclude-newer-package = { tornado = "2026-11-10T00:00:00Z" }
|
|
47
|
+
|
|
48
|
+
[tool.uv.build-backend]
|
|
49
|
+
source-exclude = ["/pipelines"]
|
|
50
|
+
|
|
51
|
+
[dependency-groups]
|
|
52
|
+
dev = [
|
|
53
|
+
"moto[athena,glue,s3]>=5.2.2",
|
|
54
|
+
"pre-commit>=4.6.1",
|
|
55
|
+
"duckdb>=1.5.5",
|
|
56
|
+
"ipykernel>=7.3.0",
|
|
57
|
+
"pytest>=9.1.1",
|
|
58
|
+
"pytest-bdd>=8.1.0",
|
|
59
|
+
"pytest-mock>=3.15.1",
|
|
60
|
+
"ruff>=0.9.9",
|
|
61
|
+
"dotenv>=0.9.9",
|
|
62
|
+
"typer>=0.27.0",
|
|
63
|
+
"virtualenv>=21.14.13",
|
|
64
|
+
"uv==0.13.6",
|
|
65
|
+
]
|
|
66
|
+
linting = ["ruff"]
|
|
67
|
+
|
|
68
|
+
[project.optional-dependencies]
|
|
69
|
+
linting = ["ruff"]
|
|
70
|
+
|
|
71
|
+
[tool.ruff]
|
|
72
|
+
# General settings
|
|
73
|
+
line-length = 90
|
|
74
|
+
cache-dir = "~/.cache/ruff"
|
|
75
|
+
target-version = "py312"
|
|
76
|
+
fix = false
|
|
77
|
+
respect-gitignore = true
|
|
78
|
+
|
|
79
|
+
# Exclude specific directories from being linted
|
|
80
|
+
exclude = [
|
|
81
|
+
".svn",
|
|
82
|
+
"CVS",
|
|
83
|
+
".bzr",
|
|
84
|
+
".hg",
|
|
85
|
+
".git",
|
|
86
|
+
"__pycache__",
|
|
87
|
+
".tox",
|
|
88
|
+
"venv/",
|
|
89
|
+
"env/",
|
|
90
|
+
".github/",
|
|
91
|
+
"githooks/",
|
|
92
|
+
]
|
|
93
|
+
|
|
94
|
+
# Rules to ignore when linting
|
|
95
|
+
lint.ignore = [
|
|
96
|
+
"E731", # Do not assign a `lambda` expression, use a `def`
|
|
97
|
+
"W191", # Indentation contains tabs
|
|
98
|
+
"E111", # Indentation is not a multiple of four
|
|
99
|
+
"E114", # Indentation is not a multiple of four (comment)
|
|
100
|
+
"E117", # Over-indented
|
|
101
|
+
"D206", # Docstring should be indented with spaces, not tabs
|
|
102
|
+
"D300", # Use """triple double quotes""" (found '''-quotes)
|
|
103
|
+
# Q00x - Prefer either single or double quotes, but be consistent
|
|
104
|
+
"Q000",
|
|
105
|
+
"Q001",
|
|
106
|
+
"Q002",
|
|
107
|
+
"Q003",
|
|
108
|
+
"COM812", # Missing trailing comma
|
|
109
|
+
"COM819", # Trailing comma is missing
|
|
110
|
+
"ISC001", # isort found unsorted imports
|
|
111
|
+
"ISC002", # isort found ungrouped imports
|
|
112
|
+
"E266", # Too many leading '#' for block comment - ####
|
|
113
|
+
]
|
|
114
|
+
# Rules to include in the linting selection
|
|
115
|
+
lint.select = [
|
|
116
|
+
# pycodestyle (default)
|
|
117
|
+
"E",
|
|
118
|
+
# Pyflakes (default)
|
|
119
|
+
"F",
|
|
120
|
+
# Whitespace warnings
|
|
121
|
+
"W",
|
|
122
|
+
# isort (sort imports)
|
|
123
|
+
"I001",
|
|
124
|
+
# flake8-print
|
|
125
|
+
"T20"
|
|
126
|
+
]
|
|
127
|
+
lint.fixable = ["ALL"]
|
|
128
|
+
lint.unfixable = []
|
|
129
|
+
|
|
130
|
+
# isort settings
|
|
131
|
+
[tool.ruff.lint.isort]
|
|
132
|
+
combine-as-imports = true
|
|
133
|
+
|
|
134
|
+
# flake8-errmsg settings
|
|
135
|
+
[tool.ruff.lint.flake8-errmsg]
|
|
136
|
+
max-string-length = 50
|
|
137
|
+
|
|
138
|
+
# Formatting rules (inspired by Black)
|
|
139
|
+
[tool.ruff.format]
|
|
140
|
+
quote-style = "double"
|
|
141
|
+
indent-style = "space"
|
|
142
|
+
skip-magic-trailing-comma = false
|
|
143
|
+
line-ending = "auto"
|
|
144
|
+
|
|
145
|
+
# Docstring code formatting
|
|
146
|
+
docstring-code-format = true
|
|
147
|
+
docstring-code-line-length = "dynamic"
|
|
148
|
+
|
|
149
|
+
# Exclude formatting for jupyter notebooks
|
|
150
|
+
exclude = ["*.ipynb"]
|
|
151
|
+
|
|
152
|
+
[tool.pytest.ini_options]
|
|
153
|
+
pythonpath = [
|
|
154
|
+
"."
|
|
155
|
+
]
|
|
156
|
+
testpaths = [
|
|
157
|
+
"tests"
|
|
158
|
+
]
|
|
159
|
+
addopts = "-ra --strict-markers"
|
|
160
|
+
markers = [
|
|
161
|
+
"unit: mark a test as a unit test.",
|
|
162
|
+
"integration: mark a test as an integration test.",
|
|
163
|
+
"e2e: mark a test as end-to-end.",
|
|
164
|
+
]
|