data-linter 6.2.4__tar.gz → 6.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {data_linter-6.2.4 → data_linter-6.3.0}/PKG-INFO +7 -5
- data_linter-6.3.0/data_linter/__init__.py +1 -0
- {data_linter-6.2.4 → data_linter-6.3.0}/data_linter/logging_functions.py +2 -2
- {data_linter-6.2.4 → data_linter-6.3.0}/data_linter/utils.py +20 -1
- {data_linter-6.2.4 → data_linter-6.3.0}/data_linter/validation.py +1 -1
- {data_linter-6.2.4 → data_linter-6.3.0}/pyproject.toml +4 -2
- data_linter-6.2.4/data_linter/__init__.py +0 -1
- {data_linter-6.2.4 → data_linter-6.3.0}/README.md +0 -0
- {data_linter-6.2.4 → data_linter-6.3.0}/data_linter/command_line.py +0 -0
- {data_linter-6.2.4 → data_linter-6.3.0}/data_linter/constants.py +0 -0
- {data_linter-6.2.4 → data_linter-6.3.0}/data_linter/iam.py +0 -0
- {data_linter-6.2.4 → data_linter-6.3.0}/data_linter/log_explorer_utils.py +0 -0
- {data_linter-6.2.4 → data_linter-6.3.0}/data_linter/schemas/config-schema.json +0 -0
- {data_linter-6.2.4 → data_linter-6.3.0}/data_linter/validators/__init__.py +0 -0
- {data_linter-6.2.4 → data_linter-6.3.0}/data_linter/validators/base.py +0 -0
- {data_linter-6.2.4 → data_linter-6.3.0}/data_linter/validators/pandas_validator.py +0 -0
- {data_linter-6.2.4 → data_linter-6.3.0}/data_linter/validators/parquet_validator.py +0 -0
|
@@ -1,17 +1,16 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
2
|
Name: data_linter
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.3.0
|
|
4
4
|
Summary: data linter
|
|
5
|
-
Home-page: https://github.com/moj-analytical-services/data_linter
|
|
6
5
|
Author: Thomas Hirsch
|
|
7
6
|
Author-email: thomas.hirsch@digital.justice.gov.uk
|
|
8
|
-
Requires-Python: >=3.
|
|
7
|
+
Requires-Python: >=3.9,<4.0
|
|
9
8
|
Classifier: Programming Language :: Python :: 3
|
|
10
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
11
9
|
Classifier: Programming Language :: Python :: 3.9
|
|
12
10
|
Classifier: Programming Language :: Python :: 3.10
|
|
13
11
|
Classifier: Programming Language :: Python :: 3.11
|
|
14
12
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
14
|
Provides-Extra: frictionless
|
|
16
15
|
Provides-Extra: ge
|
|
17
16
|
Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
|
|
@@ -22,8 +21,11 @@ Requires-Dist: dataengineeringutils3 (>=1.0.1,<2.0.0)
|
|
|
22
21
|
Requires-Dist: iam_builder (>=4.1.0,<5.0.0)
|
|
23
22
|
Requires-Dist: jsonschema (>=4.10.0,<5.0.0)
|
|
24
23
|
Requires-Dist: mojap-metadata[arrow] (>=1.10.0,<2.0.0)
|
|
24
|
+
Requires-Dist: numpy (<2.0.0)
|
|
25
25
|
Requires-Dist: pandas (>=1.2)
|
|
26
|
+
Requires-Dist: setuptools (>=76.0.0)
|
|
26
27
|
Requires-Dist: toml (>=0.10,<0.11)
|
|
28
|
+
Project-URL: Homepage, https://github.com/moj-analytical-services/data_linter
|
|
27
29
|
Project-URL: Repository, https://github.com/moj-analytical-services/data_linter
|
|
28
30
|
Description-Content-Type: text/markdown
|
|
29
31
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "6.3.0"
|
|
@@ -22,7 +22,6 @@ class ContextFilter(logging.Filter):
|
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def logging_setup() -> Tuple[logging.Logger, io.StringIO]:
|
|
25
|
-
|
|
26
25
|
log = logging.getLogger("root")
|
|
27
26
|
log.setLevel(logging.DEBUG)
|
|
28
27
|
|
|
@@ -43,7 +42,8 @@ def logging_setup() -> Tuple[logging.Logger, io.StringIO]:
|
|
|
43
42
|
log.addHandler(console)
|
|
44
43
|
|
|
45
44
|
cf = ContextFilter()
|
|
46
|
-
log.
|
|
45
|
+
for handler in log.handlers:
|
|
46
|
+
handler.addFilter(cf)
|
|
47
47
|
|
|
48
48
|
return log, log_stringio
|
|
49
49
|
|
|
@@ -7,11 +7,11 @@ import tempfile
|
|
|
7
7
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
8
8
|
from typing import Tuple, Union, List
|
|
9
9
|
from pathlib import Path
|
|
10
|
+
import awswrangler as wr
|
|
10
11
|
|
|
11
12
|
from dataengineeringutils3.s3 import (
|
|
12
13
|
s3_path_to_bucket_key,
|
|
13
14
|
write_local_file_to_s3,
|
|
14
|
-
copy_s3_object,
|
|
15
15
|
check_for_s3_file,
|
|
16
16
|
)
|
|
17
17
|
|
|
@@ -112,6 +112,25 @@ def s3_to_local(s3_path: str, local_path: str):
|
|
|
112
112
|
s3_client.download_fileobj(bucket, key, opened_file)
|
|
113
113
|
|
|
114
114
|
|
|
115
|
+
def copy_s3_object(src_path: str, dst_path: str):
|
|
116
|
+
src_dirname = os.path.dirname(src_path)
|
|
117
|
+
dst_dirname = os.path.dirname(dst_path)
|
|
118
|
+
|
|
119
|
+
src_key = os.path.basename(src_path)
|
|
120
|
+
dst_key = os.path.basename(dst_path)
|
|
121
|
+
|
|
122
|
+
resp = wr.s3.copy_objects(
|
|
123
|
+
paths=[src_path],
|
|
124
|
+
source_path=src_dirname,
|
|
125
|
+
target_path=dst_dirname,
|
|
126
|
+
replace_filenames={
|
|
127
|
+
src_key: dst_key,
|
|
128
|
+
},
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
return resp
|
|
132
|
+
|
|
133
|
+
|
|
115
134
|
def copy_data(src_path: str, dst_path: str):
|
|
116
135
|
src_path_is_s3 = src_path.startswith("s3://")
|
|
117
136
|
dst_path_is_s3 = dst_path.startswith("s3://")
|
|
@@ -412,7 +412,7 @@ def validate_data(config: dict) -> ValidatorResult:
|
|
|
412
412
|
for i, matched_file in enumerate(table_params["matched_files"]):
|
|
413
413
|
|
|
414
414
|
log.info(
|
|
415
|
-
f"{matched_file} ...file {i+1} "
|
|
415
|
+
f"{matched_file} ...file {i + 1} "
|
|
416
416
|
f"of {len(table_params['matched_files'])}"
|
|
417
417
|
)
|
|
418
418
|
validator = get_validator[validator_engine](
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "data_linter"
|
|
3
|
-
version = "6.
|
|
3
|
+
version = "6.3.0"
|
|
4
4
|
description = "data linter"
|
|
5
5
|
authors = ["Thomas Hirsch <thomas.hirsch@digital.justice.gov.uk>",
|
|
6
6
|
"George Kelly <george.kelly@digital.justice.gov.uk>",
|
|
@@ -13,7 +13,7 @@ repository = "https://github.com/moj-analytical-services/data_linter"
|
|
|
13
13
|
homepage = "https://github.com/moj-analytical-services/data_linter"
|
|
14
14
|
|
|
15
15
|
[tool.poetry.dependencies]
|
|
16
|
-
python = ">=3.
|
|
16
|
+
python = ">=3.9,<4.0"
|
|
17
17
|
dataengineeringutils3 = "^1.0.1"
|
|
18
18
|
boto3 = "^1.14.7"
|
|
19
19
|
jsonschema = "^4.10.0"
|
|
@@ -24,6 +24,8 @@ mojap-metadata = {version = "^1.10.0", extras = ["arrow"]}
|
|
|
24
24
|
arrow-pd-parser = ">=1.3.0"
|
|
25
25
|
awswrangler = ">=2.12.0"
|
|
26
26
|
toml = "^0.10"
|
|
27
|
+
numpy = "<2.0.0"
|
|
28
|
+
setuptools = ">=76.0.0"
|
|
27
29
|
|
|
28
30
|
[tool.poetry.dev-dependencies]
|
|
29
31
|
pytest = ">=6.1"
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "6.2.4"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|