data-linter 6.2.4__tar.gz → 6.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,17 +1,16 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.3
2
2
  Name: data_linter
3
- Version: 6.2.4
3
+ Version: 6.3.0
4
4
  Summary: data linter
5
- Home-page: https://github.com/moj-analytical-services/data_linter
6
5
  Author: Thomas Hirsch
7
6
  Author-email: thomas.hirsch@digital.justice.gov.uk
8
- Requires-Python: >=3.8,<4.0
7
+ Requires-Python: >=3.9,<4.0
9
8
  Classifier: Programming Language :: Python :: 3
10
- Classifier: Programming Language :: Python :: 3.8
11
9
  Classifier: Programming Language :: Python :: 3.9
12
10
  Classifier: Programming Language :: Python :: 3.10
13
11
  Classifier: Programming Language :: Python :: 3.11
14
12
  Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Programming Language :: Python :: 3.13
15
14
  Provides-Extra: frictionless
16
15
  Provides-Extra: ge
17
16
  Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
@@ -22,8 +21,11 @@ Requires-Dist: dataengineeringutils3 (>=1.0.1,<2.0.0)
22
21
  Requires-Dist: iam_builder (>=4.1.0,<5.0.0)
23
22
  Requires-Dist: jsonschema (>=4.10.0,<5.0.0)
24
23
  Requires-Dist: mojap-metadata[arrow] (>=1.10.0,<2.0.0)
24
+ Requires-Dist: numpy (<2.0.0)
25
25
  Requires-Dist: pandas (>=1.2)
26
+ Requires-Dist: setuptools (>=76.0.0)
26
27
  Requires-Dist: toml (>=0.10,<0.11)
28
+ Project-URL: Homepage, https://github.com/moj-analytical-services/data_linter
27
29
  Project-URL: Repository, https://github.com/moj-analytical-services/data_linter
28
30
  Description-Content-Type: text/markdown
29
31
 
@@ -0,0 +1 @@
1
+ __version__ = "6.3.0"
@@ -22,7 +22,6 @@ class ContextFilter(logging.Filter):
22
22
 
23
23
 
24
24
  def logging_setup() -> Tuple[logging.Logger, io.StringIO]:
25
-
26
25
  log = logging.getLogger("root")
27
26
  log.setLevel(logging.DEBUG)
28
27
 
@@ -43,7 +42,8 @@ def logging_setup() -> Tuple[logging.Logger, io.StringIO]:
43
42
  log.addHandler(console)
44
43
 
45
44
  cf = ContextFilter()
46
- log.addFilter(cf)
45
+ for handler in log.handlers:
46
+ handler.addFilter(cf)
47
47
 
48
48
  return log, log_stringio
49
49
 
@@ -7,11 +7,11 @@ import tempfile
7
7
  from concurrent.futures import ThreadPoolExecutor, as_completed
8
8
  from typing import Tuple, Union, List
9
9
  from pathlib import Path
10
+ import awswrangler as wr
10
11
 
11
12
  from dataengineeringutils3.s3 import (
12
13
  s3_path_to_bucket_key,
13
14
  write_local_file_to_s3,
14
- copy_s3_object,
15
15
  check_for_s3_file,
16
16
  )
17
17
 
@@ -112,6 +112,25 @@ def s3_to_local(s3_path: str, local_path: str):
112
112
  s3_client.download_fileobj(bucket, key, opened_file)
113
113
 
114
114
 
115
+ def copy_s3_object(src_path: str, dst_path: str):
116
+ src_dirname = os.path.dirname(src_path)
117
+ dst_dirname = os.path.dirname(dst_path)
118
+
119
+ src_key = os.path.basename(src_path)
120
+ dst_key = os.path.basename(dst_path)
121
+
122
+ resp = wr.s3.copy_objects(
123
+ paths=[src_path],
124
+ source_path=src_dirname,
125
+ target_path=dst_dirname,
126
+ replace_filenames={
127
+ src_key: dst_key,
128
+ },
129
+ )
130
+
131
+ return resp
132
+
133
+
115
134
  def copy_data(src_path: str, dst_path: str):
116
135
  src_path_is_s3 = src_path.startswith("s3://")
117
136
  dst_path_is_s3 = dst_path.startswith("s3://")
@@ -412,7 +412,7 @@ def validate_data(config: dict) -> ValidatorResult:
412
412
  for i, matched_file in enumerate(table_params["matched_files"]):
413
413
 
414
414
  log.info(
415
- f"{matched_file} ...file {i+1} "
415
+ f"{matched_file} ...file {i + 1} "
416
416
  f"of {len(table_params['matched_files'])}"
417
417
  )
418
418
  validator = get_validator[validator_engine](
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "data_linter"
3
- version = "6.2.4"
3
+ version = "6.3.0"
4
4
  description = "data linter"
5
5
  authors = ["Thomas Hirsch <thomas.hirsch@digital.justice.gov.uk>",
6
6
  "George Kelly <george.kelly@digital.justice.gov.uk>",
@@ -13,7 +13,7 @@ repository = "https://github.com/moj-analytical-services/data_linter"
13
13
  homepage = "https://github.com/moj-analytical-services/data_linter"
14
14
 
15
15
  [tool.poetry.dependencies]
16
- python = ">=3.8,<4.0"
16
+ python = ">=3.9,<4.0"
17
17
  dataengineeringutils3 = "^1.0.1"
18
18
  boto3 = "^1.14.7"
19
19
  jsonschema = "^4.10.0"
@@ -24,6 +24,8 @@ mojap-metadata = {version = "^1.10.0", extras = ["arrow"]}
24
24
  arrow-pd-parser = ">=1.3.0"
25
25
  awswrangler = ">=2.12.0"
26
26
  toml = "^0.10"
27
+ numpy = "<2.0.0"
28
+ setuptools = ">=76.0.0"
27
29
 
28
30
  [tool.poetry.dev-dependencies]
29
31
  pytest = ">=6.1"
@@ -1 +0,0 @@
1
- __version__ = "6.2.4"
File without changes