basic-open-agent-tools 0.6.2__tar.gz → 0.6.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {basic_open_agent_tools-0.6.2/src/basic_open_agent_tools.egg-info → basic_open_agent_tools-0.6.4}/PKG-INFO +1 -1
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/pyproject.toml +1 -1
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/data/binary_processing.py +7 -7
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/data/config_processing.py +10 -3
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/data/csv_tools.py +59 -42
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/data/json_tools.py +9 -7
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/data/structures.py +15 -5
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/data/transform.py +96 -32
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/data/validation.py +11 -3
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4/src/basic_open_agent_tools.egg-info}/PKG-INFO +1 -1
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_data_csv_tools.py +4 -4
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_data_json_tools.py +1 -1
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/LICENSE +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/README.md +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/setup.cfg +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/setup.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/__init__.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/crypto/__init__.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/data/__init__.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/data/archive_processing.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/data/object_serialization.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/exceptions.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/file_system/__init__.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/file_system/info.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/file_system/operations.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/file_system/tree.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/file_system/validation.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/helpers.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/network/__init__.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/system/__init__.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/text/__init__.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/text/processing.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/types.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/utilities/__init__.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools.egg-info/SOURCES.txt +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools.egg-info/dependency_links.txt +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools.egg-info/requires.txt +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools.egg-info/top_level.txt +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_data_archive_processing.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_data_binary_processing.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_data_config_processing.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_data_object_serialization.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_data_structures.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_data_transform.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_data_validation.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_file_system_tools.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_helpers.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_read_only_tools.py +0 -0
- {basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_text_processing.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: basic-open-agent-tools
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.4
|
|
4
4
|
Summary: An open foundational toolkit providing essential components for building AI agents with minimal dependencies for local (non-HTTP/API) actions.
|
|
5
5
|
Home-page: https://github.com/open-agent-tools/basic-open-agent-tools
|
|
6
6
|
Author: Open Agent Tools
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta:__legacy__"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "basic-open-agent-tools"
|
|
7
|
-
version = "0.6.
|
|
7
|
+
version = "0.6.4" # Fixed binary and JSON function signatures for agent framework compatibility
|
|
8
8
|
description = "An open foundational toolkit providing essential components for building AI agents with minimal dependencies for local (non-HTTP/API) actions."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.8"
|
|
@@ -11,17 +11,17 @@ import imghdr
|
|
|
11
11
|
import mimetypes
|
|
12
12
|
import os
|
|
13
13
|
import struct
|
|
14
|
-
from typing import Any, Dict
|
|
14
|
+
from typing import Any, Dict
|
|
15
15
|
|
|
16
16
|
from ..exceptions import DataError
|
|
17
17
|
|
|
18
18
|
|
|
19
|
-
def read_binary_file(file_path: str, max_size:
|
|
19
|
+
def read_binary_file(file_path: str, max_size: int = 0) -> bytes:
|
|
20
20
|
"""Read binary data from a file.
|
|
21
21
|
|
|
22
22
|
Args:
|
|
23
23
|
file_path: Path to the binary file
|
|
24
|
-
max_size: Maximum file size in bytes to read (
|
|
24
|
+
max_size: Maximum file size in bytes to read (0 for no limit)
|
|
25
25
|
|
|
26
26
|
Returns:
|
|
27
27
|
Binary data as bytes
|
|
@@ -38,8 +38,8 @@ def read_binary_file(file_path: str, max_size: Optional[int] = None) -> bytes:
|
|
|
38
38
|
if not os.path.isfile(file_path):
|
|
39
39
|
raise FileNotFoundError(f"File not found: {file_path}")
|
|
40
40
|
|
|
41
|
-
# Check file size if max_size is specified
|
|
42
|
-
if max_size
|
|
41
|
+
# Check file size if max_size is specified (0 means no limit)
|
|
42
|
+
if max_size > 0:
|
|
43
43
|
file_size = os.path.getsize(file_path)
|
|
44
44
|
if file_size > max_size:
|
|
45
45
|
raise DataError(
|
|
@@ -146,11 +146,11 @@ def decode_binary_data(encoded_data: str, encoding: str = "base64") -> bytes:
|
|
|
146
146
|
raise ValueError(f"Invalid {encoding} data: {str(e)}")
|
|
147
147
|
|
|
148
148
|
|
|
149
|
-
def validate_binary_format(data:
|
|
149
|
+
def validate_binary_format(data: bytes, expected_format: str) -> bool:
|
|
150
150
|
"""Validate that binary data matches an expected format.
|
|
151
151
|
|
|
152
152
|
Args:
|
|
153
|
-
data: Binary data to validate
|
|
153
|
+
data: Binary data to validate as bytes
|
|
154
154
|
expected_format: Expected format ("png", "jpeg", "gif", "pdf", etc.)
|
|
155
155
|
|
|
156
156
|
Returns:
|
|
@@ -156,6 +156,7 @@ def read_ini_file(file_path: str) -> dict:
|
|
|
156
156
|
"""
|
|
157
157
|
# Check if file exists first (ConfigParser.read doesn't raise FileNotFoundError)
|
|
158
158
|
import os
|
|
159
|
+
|
|
159
160
|
if not os.path.isfile(file_path):
|
|
160
161
|
raise FileNotFoundError(f"INI file not found: {file_path}")
|
|
161
162
|
|
|
@@ -242,12 +243,16 @@ def validate_config_schema(config_data: dict, schema: dict) -> list:
|
|
|
242
243
|
if expected_type and not isinstance(config_data[field_name], expected_type):
|
|
243
244
|
actual_type = type(config_data[field_name]).__name__
|
|
244
245
|
expected_type_name = expected_type.__name__
|
|
245
|
-
errors.append(
|
|
246
|
+
errors.append(
|
|
247
|
+
f"Field '{field_name}' has incorrect type: expected {expected_type_name}, got {actual_type}"
|
|
248
|
+
)
|
|
246
249
|
|
|
247
250
|
# Check allowed values
|
|
248
251
|
allowed_values = field_spec.get("allowed_values")
|
|
249
252
|
if allowed_values and config_data[field_name] not in allowed_values:
|
|
250
|
-
errors.append(
|
|
253
|
+
errors.append(
|
|
254
|
+
f"Field '{field_name}' has invalid value: {config_data[field_name]}. Allowed values: {allowed_values}"
|
|
255
|
+
)
|
|
251
256
|
|
|
252
257
|
# Check for unknown fields
|
|
253
258
|
for field_name in config_data:
|
|
@@ -303,7 +308,9 @@ def merge_config_files(*config_paths: Any, format_type: Optional[str] = None) ->
|
|
|
303
308
|
raise DataError(f"Cannot determine format for file: {path}")
|
|
304
309
|
|
|
305
310
|
if len(detected_formats) > 1:
|
|
306
|
-
raise DataError(
|
|
311
|
+
raise DataError(
|
|
312
|
+
f"Cannot merge files with different formats: {detected_formats}"
|
|
313
|
+
)
|
|
307
314
|
|
|
308
315
|
merged_config: Dict[str, Any] = {}
|
|
309
316
|
|
|
@@ -2,17 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
import csv
|
|
4
4
|
import io
|
|
5
|
-
from
|
|
6
|
-
from typing import Optional, Union
|
|
5
|
+
from typing import Optional
|
|
7
6
|
|
|
8
7
|
from ..exceptions import DataError
|
|
9
8
|
|
|
10
9
|
|
|
11
|
-
def read_csv_simple(file_path:
|
|
10
|
+
def read_csv_simple(file_path: str, delimiter: str = ",", headers: bool = True) -> list:
|
|
12
11
|
"""Read CSV file and return as list of dictionaries.
|
|
13
12
|
|
|
14
13
|
Args:
|
|
15
|
-
file_path: Path to the CSV file
|
|
14
|
+
file_path: Path to the CSV file as a string
|
|
16
15
|
delimiter: CSV delimiter character (default: ',')
|
|
17
16
|
headers: Whether the CSV file has headers (default: True)
|
|
18
17
|
|
|
@@ -20,7 +19,7 @@ def read_csv_simple(file_path: Union[str, Path], delimiter: str = ',', headers:
|
|
|
20
19
|
List of dictionaries representing CSV rows with string values
|
|
21
20
|
|
|
22
21
|
Raises:
|
|
23
|
-
TypeError: If file_path is not a string
|
|
22
|
+
TypeError: If file_path is not a string
|
|
24
23
|
DataError: If file cannot be read or parsed
|
|
25
24
|
|
|
26
25
|
Example:
|
|
@@ -29,9 +28,13 @@ def read_csv_simple(file_path: Union[str, Path], delimiter: str = ',', headers:
|
|
|
29
28
|
>>> data
|
|
30
29
|
[{'name': 'Alice', 'age': '25'}, {'name': 'Bob', 'age': '30'}]
|
|
31
30
|
"""
|
|
32
|
-
# Check if file_path is a string or
|
|
33
|
-
if
|
|
34
|
-
|
|
31
|
+
# Check if file_path is a string or Path object and convert to string
|
|
32
|
+
if isinstance(file_path, str):
|
|
33
|
+
file_path_str = file_path
|
|
34
|
+
elif hasattr(file_path, "__fspath__"):
|
|
35
|
+
file_path_str = str(file_path)
|
|
36
|
+
else:
|
|
37
|
+
raise TypeError("file_path must be a string")
|
|
35
38
|
|
|
36
39
|
if not isinstance(delimiter, str):
|
|
37
40
|
raise TypeError("delimiter must be a string")
|
|
@@ -40,7 +43,7 @@ def read_csv_simple(file_path: Union[str, Path], delimiter: str = ',', headers:
|
|
|
40
43
|
raise TypeError("headers must be a boolean")
|
|
41
44
|
|
|
42
45
|
try:
|
|
43
|
-
with open(
|
|
46
|
+
with open(file_path_str, encoding="utf-8", newline="") as csvfile:
|
|
44
47
|
if headers:
|
|
45
48
|
dict_reader = csv.DictReader(csvfile, delimiter=delimiter)
|
|
46
49
|
return [dict(row) for row in dict_reader]
|
|
@@ -57,24 +60,26 @@ def read_csv_simple(file_path: Union[str, Path], delimiter: str = ',', headers:
|
|
|
57
60
|
result.append(row_dict)
|
|
58
61
|
return result
|
|
59
62
|
except FileNotFoundError:
|
|
60
|
-
raise DataError(f"CSV file not found: {
|
|
63
|
+
raise DataError(f"CSV file not found: {file_path_str}")
|
|
61
64
|
except UnicodeDecodeError as e:
|
|
62
|
-
raise DataError(f"Failed to decode CSV file {
|
|
65
|
+
raise DataError(f"Failed to decode CSV file {file_path_str}: {e}")
|
|
63
66
|
except csv.Error as e:
|
|
64
|
-
raise DataError(f"Failed to parse CSV file {
|
|
67
|
+
raise DataError(f"Failed to parse CSV file {file_path_str}: {e}")
|
|
65
68
|
|
|
66
69
|
|
|
67
|
-
def write_csv_simple(
|
|
70
|
+
def write_csv_simple(
|
|
71
|
+
data: list, file_path: str, delimiter: str = ",", headers: bool = True
|
|
72
|
+
) -> None:
|
|
68
73
|
"""Write list of dictionaries to CSV file.
|
|
69
74
|
|
|
70
75
|
Args:
|
|
71
76
|
data: List of dictionaries to write
|
|
72
|
-
file_path: Path where CSV file will be created
|
|
77
|
+
file_path: Path where CSV file will be created as a string
|
|
73
78
|
delimiter: CSV delimiter character (default: ',')
|
|
74
79
|
headers: Whether to write headers (default: True)
|
|
75
80
|
|
|
76
81
|
Raises:
|
|
77
|
-
TypeError: If data is not a list, contains non-dictionary items, or file_path is not a string
|
|
82
|
+
TypeError: If data is not a list, contains non-dictionary items, or file_path is not a string
|
|
78
83
|
DataError: If file cannot be written
|
|
79
84
|
|
|
80
85
|
Example:
|
|
@@ -85,9 +90,13 @@ def write_csv_simple(data: list, file_path: Union[str, Path], delimiter: str = '
|
|
|
85
90
|
if not isinstance(data, list):
|
|
86
91
|
raise TypeError("data must be a list")
|
|
87
92
|
|
|
88
|
-
# Check if file_path is a string or
|
|
89
|
-
if
|
|
90
|
-
|
|
93
|
+
# Check if file_path is a string or Path object and convert to string
|
|
94
|
+
if isinstance(file_path, str):
|
|
95
|
+
file_path_str = file_path
|
|
96
|
+
elif hasattr(file_path, "__fspath__"):
|
|
97
|
+
file_path_str = str(file_path)
|
|
98
|
+
else:
|
|
99
|
+
raise TypeError("file_path must be a string")
|
|
91
100
|
|
|
92
101
|
if not isinstance(delimiter, str):
|
|
93
102
|
raise TypeError("delimiter must be a string")
|
|
@@ -97,7 +106,7 @@ def write_csv_simple(data: list, file_path: Union[str, Path], delimiter: str = '
|
|
|
97
106
|
|
|
98
107
|
if not data:
|
|
99
108
|
# Write empty file for empty data
|
|
100
|
-
with open(
|
|
109
|
+
with open(file_path_str, "w", encoding="utf-8") as f:
|
|
101
110
|
f.write("")
|
|
102
111
|
return
|
|
103
112
|
|
|
@@ -114,16 +123,16 @@ def write_csv_simple(data: list, file_path: Union[str, Path], delimiter: str = '
|
|
|
114
123
|
if key not in fieldnames:
|
|
115
124
|
fieldnames.append(key)
|
|
116
125
|
|
|
117
|
-
with open(
|
|
126
|
+
with open(file_path_str, "w", encoding="utf-8", newline="") as csvfile:
|
|
118
127
|
writer = csv.DictWriter(csvfile, fieldnames=fieldnames, delimiter=delimiter)
|
|
119
128
|
if headers:
|
|
120
129
|
writer.writeheader()
|
|
121
130
|
writer.writerows(data)
|
|
122
131
|
except OSError as e:
|
|
123
|
-
raise DataError(f"Failed to write CSV file {
|
|
132
|
+
raise DataError(f"Failed to write CSV file {file_path_str}: {e}")
|
|
124
133
|
|
|
125
134
|
|
|
126
|
-
def csv_to_dict_list(csv_data: str, delimiter: str =
|
|
135
|
+
def csv_to_dict_list(csv_data: str, delimiter: str = ",") -> list:
|
|
127
136
|
"""Convert CSV string to list of dictionaries.
|
|
128
137
|
|
|
129
138
|
Args:
|
|
@@ -158,7 +167,7 @@ def csv_to_dict_list(csv_data: str, delimiter: str = ',') -> list:
|
|
|
158
167
|
raise DataError(f"Failed to parse CSV data: {e}")
|
|
159
168
|
|
|
160
169
|
|
|
161
|
-
def dict_list_to_csv(data: list, delimiter: str =
|
|
170
|
+
def dict_list_to_csv(data: list, delimiter: str = ",") -> str:
|
|
162
171
|
"""Convert list of dictionaries to CSV string.
|
|
163
172
|
|
|
164
173
|
Args:
|
|
@@ -203,18 +212,18 @@ def dict_list_to_csv(data: list, delimiter: str = ',') -> str:
|
|
|
203
212
|
return output.getvalue()
|
|
204
213
|
|
|
205
214
|
|
|
206
|
-
def detect_csv_delimiter(file_path:
|
|
215
|
+
def detect_csv_delimiter(file_path: str, sample_size: int = 1024) -> str:
|
|
207
216
|
"""Auto-detect CSV delimiter by analyzing file content.
|
|
208
217
|
|
|
209
218
|
Args:
|
|
210
|
-
file_path: Path to the CSV file
|
|
219
|
+
file_path: Path to the CSV file as a string
|
|
211
220
|
sample_size: Number of characters to sample for detection
|
|
212
221
|
|
|
213
222
|
Returns:
|
|
214
223
|
Detected delimiter character
|
|
215
224
|
|
|
216
225
|
Raises:
|
|
217
|
-
TypeError: If file_path is not a string
|
|
226
|
+
TypeError: If file_path is not a string, or sample_size is not a positive integer
|
|
218
227
|
DataError: If file cannot be read or delimiter cannot be detected
|
|
219
228
|
|
|
220
229
|
Example:
|
|
@@ -223,15 +232,19 @@ def detect_csv_delimiter(file_path: Union[str, Path], sample_size: int = 1024) -
|
|
|
223
232
|
>>> detect_csv_delimiter("data.tsv")
|
|
224
233
|
'\\t'
|
|
225
234
|
"""
|
|
226
|
-
# Check if file_path is a string or
|
|
227
|
-
if
|
|
228
|
-
|
|
235
|
+
# Check if file_path is a string or Path object and convert to string
|
|
236
|
+
if isinstance(file_path, str):
|
|
237
|
+
file_path_str = file_path
|
|
238
|
+
elif hasattr(file_path, "__fspath__"):
|
|
239
|
+
file_path_str = str(file_path)
|
|
240
|
+
else:
|
|
241
|
+
raise TypeError("file_path must be a string")
|
|
229
242
|
|
|
230
243
|
if not isinstance(sample_size, int) or sample_size <= 0:
|
|
231
244
|
raise TypeError("sample_size must be a positive integer")
|
|
232
245
|
|
|
233
246
|
try:
|
|
234
|
-
with open(
|
|
247
|
+
with open(file_path_str, encoding="utf-8") as csvfile:
|
|
235
248
|
sample = csvfile.read(sample_size)
|
|
236
249
|
|
|
237
250
|
if not sample:
|
|
@@ -241,27 +254,27 @@ def detect_csv_delimiter(file_path: Union[str, Path], sample_size: int = 1024) -
|
|
|
241
254
|
delimiter = sniffer.sniff(sample).delimiter
|
|
242
255
|
return delimiter
|
|
243
256
|
except FileNotFoundError:
|
|
244
|
-
raise DataError(f"CSV file not found: {
|
|
257
|
+
raise DataError(f"CSV file not found: {file_path_str}")
|
|
245
258
|
except UnicodeDecodeError as e:
|
|
246
|
-
raise DataError(f"Failed to decode CSV file {
|
|
259
|
+
raise DataError(f"Failed to decode CSV file {file_path_str}: {e}")
|
|
247
260
|
except csv.Error as e:
|
|
248
|
-
raise DataError(f"Failed to detect delimiter in {
|
|
261
|
+
raise DataError(f"Failed to detect delimiter in {file_path_str}: {e}")
|
|
249
262
|
|
|
250
263
|
|
|
251
264
|
def validate_csv_structure(
|
|
252
|
-
file_path:
|
|
265
|
+
file_path: str, expected_columns: Optional[list] = None
|
|
253
266
|
) -> bool:
|
|
254
267
|
"""Validate CSV file structure and column headers.
|
|
255
268
|
|
|
256
269
|
Args:
|
|
257
|
-
file_path: Path to the CSV file
|
|
270
|
+
file_path: Path to the CSV file as a string
|
|
258
271
|
expected_columns: List of expected column names (optional)
|
|
259
272
|
|
|
260
273
|
Returns:
|
|
261
274
|
True if CSV structure is valid
|
|
262
275
|
|
|
263
276
|
Raises:
|
|
264
|
-
TypeError: If file_path is not a string
|
|
277
|
+
TypeError: If file_path is not a string, or expected_columns is not a list
|
|
265
278
|
DataError: If file cannot be read or structure is invalid
|
|
266
279
|
|
|
267
280
|
Example:
|
|
@@ -270,9 +283,13 @@ def validate_csv_structure(
|
|
|
270
283
|
>>> validate_csv_structure("malformed.csv")
|
|
271
284
|
False
|
|
272
285
|
"""
|
|
273
|
-
# Check if file_path is a string or
|
|
274
|
-
if
|
|
275
|
-
|
|
286
|
+
# Check if file_path is a string or Path object and convert to string
|
|
287
|
+
if isinstance(file_path, str):
|
|
288
|
+
file_path_str = file_path
|
|
289
|
+
elif hasattr(file_path, "__fspath__"):
|
|
290
|
+
file_path_str = str(file_path)
|
|
291
|
+
else:
|
|
292
|
+
raise TypeError("file_path must be a string")
|
|
276
293
|
|
|
277
294
|
if expected_columns is not None and not isinstance(expected_columns, list):
|
|
278
295
|
raise TypeError("expected_columns must be a list or None")
|
|
@@ -281,11 +298,11 @@ def validate_csv_structure(
|
|
|
281
298
|
# Check if file is empty first
|
|
282
299
|
import os
|
|
283
300
|
|
|
284
|
-
if os.path.getsize(
|
|
301
|
+
if os.path.getsize(file_path_str) == 0:
|
|
285
302
|
return True # Empty file is considered valid
|
|
286
303
|
|
|
287
304
|
# Read first few rows to validate structure
|
|
288
|
-
data = read_csv_simple(
|
|
305
|
+
data = read_csv_simple(file_path_str)
|
|
289
306
|
|
|
290
307
|
if not data:
|
|
291
308
|
return True # Empty file is considered valid
|
|
@@ -305,7 +322,7 @@ def validate_csv_structure(
|
|
|
305
322
|
# Re-raise DataError as-is
|
|
306
323
|
raise
|
|
307
324
|
except Exception as e:
|
|
308
|
-
raise DataError(f"Invalid CSV structure in {
|
|
325
|
+
raise DataError(f"Invalid CSV structure in {file_path_str}: {e}")
|
|
309
326
|
|
|
310
327
|
|
|
311
328
|
def clean_csv_data(data: list, rules: Optional[dict] = None) -> list:
|
|
@@ -2,17 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
import gzip
|
|
4
4
|
import json
|
|
5
|
-
from typing import Any
|
|
5
|
+
from typing import Any
|
|
6
6
|
|
|
7
7
|
from ..exceptions import SerializationError
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
def safe_json_serialize(data:
|
|
10
|
+
def safe_json_serialize(data: dict, indent: int = 0) -> str:
|
|
11
11
|
"""Safely serialize data to JSON string with error handling.
|
|
12
12
|
|
|
13
13
|
Args:
|
|
14
|
-
data: Data to serialize to JSON
|
|
15
|
-
indent: Number of spaces for indentation (
|
|
14
|
+
data: Data to serialize to JSON (accepts any serializable type)
|
|
15
|
+
indent: Number of spaces for indentation (0 for compact)
|
|
16
16
|
|
|
17
17
|
Returns:
|
|
18
18
|
JSON string representation of the data
|
|
@@ -27,11 +27,13 @@ def safe_json_serialize(data: Any, indent: Optional[int] = None) -> str:
|
|
|
27
27
|
>>> safe_json_serialize({"a": 1, "b": 2}, indent=2)
|
|
28
28
|
'{\\n "a": 1,\\n "b": 2\\n}'
|
|
29
29
|
"""
|
|
30
|
-
if not isinstance(indent,
|
|
31
|
-
raise TypeError("indent must be an integer
|
|
30
|
+
if not isinstance(indent, int):
|
|
31
|
+
raise TypeError("indent must be an integer")
|
|
32
32
|
|
|
33
33
|
try:
|
|
34
|
-
|
|
34
|
+
# Use None for compact format when indent is 0
|
|
35
|
+
actual_indent = None if indent == 0 else indent
|
|
36
|
+
return json.dumps(data, indent=actual_indent, ensure_ascii=False)
|
|
35
37
|
except (TypeError, ValueError) as e:
|
|
36
38
|
raise SerializationError(f"Failed to serialize data to JSON: {e}")
|
|
37
39
|
|
|
@@ -96,7 +96,9 @@ def unflatten_dict(data: Dict[str, Any], separator: str = ".") -> Dict[str, Any]
|
|
|
96
96
|
return result
|
|
97
97
|
|
|
98
98
|
|
|
99
|
-
def get_nested_value_simple(
|
|
99
|
+
def get_nested_value_simple(
|
|
100
|
+
data: Dict[str, Any], key_path: str, default: Any = None
|
|
101
|
+
) -> Any:
|
|
100
102
|
"""Safely access nested dictionary values using dot notation.
|
|
101
103
|
|
|
102
104
|
Args:
|
|
@@ -461,7 +463,9 @@ def unflatten_dict_simple(data: Dict[str, Any], separator: str = ".") -> Dict[st
|
|
|
461
463
|
return unflatten_dict(data, separator)
|
|
462
464
|
|
|
463
465
|
|
|
464
|
-
def set_nested_value_simple(
|
|
466
|
+
def set_nested_value_simple(
|
|
467
|
+
data: Dict[str, Any], key_path: str, value: Any
|
|
468
|
+
) -> Dict[str, Any]:
|
|
465
469
|
"""Set nested dictionary value using dot notation (immutable).
|
|
466
470
|
|
|
467
471
|
This is an alias for set_nested_value for LLM agent compatibility.
|
|
@@ -482,7 +486,9 @@ def set_nested_value_simple(data: Dict[str, Any], key_path: str, value: Any) ->
|
|
|
482
486
|
return set_nested_value(data, key_path, value)
|
|
483
487
|
|
|
484
488
|
|
|
485
|
-
def compare_data_structures_simple(
|
|
489
|
+
def compare_data_structures_simple(
|
|
490
|
+
data1: Any, data2: Any, ignore_order: bool = False
|
|
491
|
+
) -> bool:
|
|
486
492
|
"""Compare two data structures for equality.
|
|
487
493
|
|
|
488
494
|
This is an alias for compare_data_structures for LLM agent compatibility.
|
|
@@ -524,7 +530,9 @@ def safe_get_simple(data: Dict[str, Any], key: str, default: Any = None) -> Any:
|
|
|
524
530
|
return safe_get(data, key, default)
|
|
525
531
|
|
|
526
532
|
|
|
527
|
-
def remove_empty_values_simple(
|
|
533
|
+
def remove_empty_values_simple(
|
|
534
|
+
data: Dict[str, Any], recursive: bool = True
|
|
535
|
+
) -> Dict[str, Any]:
|
|
528
536
|
"""Remove empty values from dictionary.
|
|
529
537
|
|
|
530
538
|
This is an alias for remove_empty_values for LLM agent compatibility.
|
|
@@ -564,7 +572,9 @@ def extract_keys_simple(data: Dict[str, Any], key_pattern: str) -> List[str]:
|
|
|
564
572
|
return extract_keys(data, key_pattern)
|
|
565
573
|
|
|
566
574
|
|
|
567
|
-
def rename_keys_simple(
|
|
575
|
+
def rename_keys_simple(
|
|
576
|
+
data: Dict[str, Any], key_mapping: Dict[str, str]
|
|
577
|
+
) -> Dict[str, Any]:
|
|
568
578
|
"""Rename dictionary keys according to mapping.
|
|
569
579
|
|
|
570
580
|
This is an alias for rename_keys for LLM agent compatibility.
|
|
@@ -5,7 +5,9 @@ from typing import Any, Callable, Dict, List, Optional, Union
|
|
|
5
5
|
from ..exceptions import DataError
|
|
6
6
|
|
|
7
7
|
|
|
8
|
-
def transform_data(
|
|
8
|
+
def transform_data(
|
|
9
|
+
data: List[Dict[str, Any]], mapping: Dict[str, Union[str, Callable[[Any], Any]]]
|
|
10
|
+
) -> List[Dict[str, Any]]:
|
|
9
11
|
"""Apply a transformation mapping to a list of data records.
|
|
10
12
|
|
|
11
13
|
Args:
|
|
@@ -46,11 +48,15 @@ def transform_data(data: List[Dict[str, Any]], mapping: Dict[str, Union[str, Cal
|
|
|
46
48
|
try:
|
|
47
49
|
transformed_record[source_field] = transform(value)
|
|
48
50
|
except Exception as e:
|
|
49
|
-
raise DataError(
|
|
51
|
+
raise DataError(
|
|
52
|
+
f"Error transforming field '{source_field}': {e}"
|
|
53
|
+
)
|
|
50
54
|
elif isinstance(transform, str):
|
|
51
55
|
transformed_record[transform] = value
|
|
52
56
|
else:
|
|
53
|
-
raise DataError(
|
|
57
|
+
raise DataError(
|
|
58
|
+
f"Invalid transform for field '{source_field}': must be string or callable"
|
|
59
|
+
)
|
|
54
60
|
|
|
55
61
|
# Include fields not in mapping as-is
|
|
56
62
|
for field, value in record.items():
|
|
@@ -62,7 +68,9 @@ def transform_data(data: List[Dict[str, Any]], mapping: Dict[str, Union[str, Cal
|
|
|
62
68
|
return transformed_data
|
|
63
69
|
|
|
64
70
|
|
|
65
|
-
def rename_fields(
|
|
71
|
+
def rename_fields(
|
|
72
|
+
data: List[Dict[str, Any]], field_mapping: Dict[str, str]
|
|
73
|
+
) -> List[Dict[str, Any]]:
|
|
66
74
|
"""Rename fields in a list of dictionaries.
|
|
67
75
|
|
|
68
76
|
Args:
|
|
@@ -89,7 +97,9 @@ def rename_fields(data: List[Dict[str, Any]], field_mapping: Dict[str, str]) ->
|
|
|
89
97
|
|
|
90
98
|
for i, record in enumerate(data):
|
|
91
99
|
if not isinstance(record, dict):
|
|
92
|
-
raise DataError(
|
|
100
|
+
raise DataError(
|
|
101
|
+
f"Record at index {i} is not a dictionary: {type(record).__name__}"
|
|
102
|
+
)
|
|
93
103
|
|
|
94
104
|
renamed_record = {}
|
|
95
105
|
for field, value in record.items():
|
|
@@ -101,7 +111,9 @@ def rename_fields(data: List[Dict[str, Any]], field_mapping: Dict[str, str]) ->
|
|
|
101
111
|
return renamed_data
|
|
102
112
|
|
|
103
113
|
|
|
104
|
-
def convert_data_types(
|
|
114
|
+
def convert_data_types(
|
|
115
|
+
data: List[Dict[str, Any]], type_mapping: Dict[str, Callable[[Any], Any]]
|
|
116
|
+
) -> List[Dict[str, Any]]:
|
|
105
117
|
"""Convert data types for specified fields.
|
|
106
118
|
|
|
107
119
|
Args:
|
|
@@ -143,7 +155,9 @@ def convert_data_types(data: List[Dict[str, Any]], type_mapping: Dict[str, Calla
|
|
|
143
155
|
return converted_data
|
|
144
156
|
|
|
145
157
|
|
|
146
|
-
def clean_data(
|
|
158
|
+
def clean_data(
|
|
159
|
+
data: List[Dict[str, Any]], rules: Dict[str, List[Callable[[Any], Any]]]
|
|
160
|
+
) -> List[Dict[str, Any]]:
|
|
147
161
|
"""Clean data according to specified rules.
|
|
148
162
|
|
|
149
163
|
Args:
|
|
@@ -186,12 +200,16 @@ def clean_data(data: List[Dict[str, Any]], rules: Dict[str, List[Callable[[Any],
|
|
|
186
200
|
|
|
187
201
|
for cleaner in cleaners:
|
|
188
202
|
if not callable(cleaner):
|
|
189
|
-
raise DataError(
|
|
203
|
+
raise DataError(
|
|
204
|
+
f"Cleaner for field '{field}' is not callable: {cleaner}"
|
|
205
|
+
)
|
|
190
206
|
|
|
191
207
|
try:
|
|
192
208
|
value = cleaner(value)
|
|
193
209
|
except Exception as e:
|
|
194
|
-
raise DataError(
|
|
210
|
+
raise DataError(
|
|
211
|
+
f"Error applying cleaner to field '{field}': {e}"
|
|
212
|
+
)
|
|
195
213
|
|
|
196
214
|
cleaned_record[field] = value
|
|
197
215
|
|
|
@@ -200,7 +218,9 @@ def clean_data(data: List[Dict[str, Any]], rules: Dict[str, List[Callable[[Any],
|
|
|
200
218
|
return cleaned_data
|
|
201
219
|
|
|
202
220
|
|
|
203
|
-
def deduplicate_records(
|
|
221
|
+
def deduplicate_records(
|
|
222
|
+
data: List[Dict[str, Any]], key_fields: Optional[List[str]] = None
|
|
223
|
+
) -> List[Dict[str, Any]]:
|
|
204
224
|
"""Remove duplicate records from a list.
|
|
205
225
|
|
|
206
226
|
Args:
|
|
@@ -229,14 +249,18 @@ def deduplicate_records(data: List[Dict[str, Any]], key_fields: Optional[List[st
|
|
|
229
249
|
|
|
230
250
|
for i, record in enumerate(data):
|
|
231
251
|
if not isinstance(record, dict):
|
|
232
|
-
raise DataError(
|
|
252
|
+
raise DataError(
|
|
253
|
+
f"Record at index {i} is not a dictionary: {type(record).__name__}"
|
|
254
|
+
)
|
|
233
255
|
|
|
234
256
|
# Create a key based on specified fields or all fields
|
|
235
257
|
if key_fields:
|
|
236
258
|
# Check if all key fields are present in the record
|
|
237
259
|
missing_fields = [field for field in key_fields if field not in record]
|
|
238
260
|
if missing_fields:
|
|
239
|
-
raise DataError(
|
|
261
|
+
raise DataError(
|
|
262
|
+
f"Record at index {i} is missing key fields: {missing_fields}"
|
|
263
|
+
)
|
|
240
264
|
|
|
241
265
|
key_values = tuple(record[field] for field in key_fields)
|
|
242
266
|
else:
|
|
@@ -249,7 +273,9 @@ def deduplicate_records(data: List[Dict[str, Any]], key_fields: Optional[List[st
|
|
|
249
273
|
return deduplicated_data
|
|
250
274
|
|
|
251
275
|
|
|
252
|
-
def normalize_data(
|
|
276
|
+
def normalize_data(
|
|
277
|
+
data: List[Dict[str, Any]], rules: Dict[str, Callable[[Any], Any]]
|
|
278
|
+
) -> List[Dict[str, Any]]:
|
|
253
279
|
"""Apply normalization functions to data fields.
|
|
254
280
|
|
|
255
281
|
Args:
|
|
@@ -278,7 +304,9 @@ def normalize_data(data: List[Dict[str, Any]], rules: Dict[str, Callable[[Any],
|
|
|
278
304
|
|
|
279
305
|
for i, record in enumerate(data):
|
|
280
306
|
if not isinstance(record, dict):
|
|
281
|
-
raise DataError(
|
|
307
|
+
raise DataError(
|
|
308
|
+
f"Record at index {i} is not a dictionary: {type(record).__name__}"
|
|
309
|
+
)
|
|
282
310
|
|
|
283
311
|
normalized_record = record.copy()
|
|
284
312
|
|
|
@@ -332,7 +360,9 @@ def pivot_data(
|
|
|
332
360
|
|
|
333
361
|
for i, record in enumerate(data):
|
|
334
362
|
if not isinstance(record, dict):
|
|
335
|
-
raise DataError(
|
|
363
|
+
raise DataError(
|
|
364
|
+
f"Record at index {i} is not a dictionary: {type(record).__name__}"
|
|
365
|
+
)
|
|
336
366
|
|
|
337
367
|
if (
|
|
338
368
|
index_field not in record
|
|
@@ -346,7 +376,9 @@ def pivot_data(
|
|
|
346
376
|
missing_fields.append(column_field)
|
|
347
377
|
if value_field not in record:
|
|
348
378
|
missing_fields.append(value_field)
|
|
349
|
-
raise DataError(
|
|
379
|
+
raise DataError(
|
|
380
|
+
f"Record at index {i} is missing required fields: {missing_fields}"
|
|
381
|
+
)
|
|
350
382
|
|
|
351
383
|
index_val = record[index_field]
|
|
352
384
|
column_val = record[column_field]
|
|
@@ -360,7 +392,9 @@ def pivot_data(
|
|
|
360
392
|
return pivoted
|
|
361
393
|
|
|
362
394
|
|
|
363
|
-
def transform_data_simple(
|
|
395
|
+
def transform_data_simple(
|
|
396
|
+
data: List[Dict[str, Any]], mapping: Dict[str, str]
|
|
397
|
+
) -> List[Dict[str, Any]]:
|
|
364
398
|
"""Apply a simple transformation mapping to a list of data records.
|
|
365
399
|
|
|
366
400
|
This is a simplified version of transform_data for LLM agent compatibility.
|
|
@@ -398,7 +432,9 @@ def transform_data_simple(data: List[Dict[str, Any]], mapping: Dict[str, str]) -
|
|
|
398
432
|
for source_field, new_field in mapping.items():
|
|
399
433
|
if source_field in record:
|
|
400
434
|
if not isinstance(new_field, str):
|
|
401
|
-
raise DataError(
|
|
435
|
+
raise DataError(
|
|
436
|
+
f"Invalid mapping for field '{source_field}': must be string"
|
|
437
|
+
)
|
|
402
438
|
transformed_record[new_field] = record[source_field]
|
|
403
439
|
|
|
404
440
|
# Include fields not in mapping as-is
|
|
@@ -411,7 +447,9 @@ def transform_data_simple(data: List[Dict[str, Any]], mapping: Dict[str, str]) -
|
|
|
411
447
|
return transformed_data
|
|
412
448
|
|
|
413
449
|
|
|
414
|
-
def rename_fields_simple(
|
|
450
|
+
def rename_fields_simple(
|
|
451
|
+
data: List[Dict[str, Any]], field_mapping: Dict[str, str]
|
|
452
|
+
) -> List[Dict[str, Any]]:
|
|
415
453
|
"""Rename fields in a list of dictionaries.
|
|
416
454
|
|
|
417
455
|
This is an alias for transform_data_simple for LLM agent compatibility.
|
|
@@ -431,7 +469,9 @@ def rename_fields_simple(data: List[Dict[str, Any]], field_mapping: Dict[str, st
|
|
|
431
469
|
return transform_data_simple(data, field_mapping)
|
|
432
470
|
|
|
433
471
|
|
|
434
|
-
def convert_data_types_simple(
|
|
472
|
+
def convert_data_types_simple(
|
|
473
|
+
data: List[Dict[str, Any]], type_mapping: Dict[str, str]
|
|
474
|
+
) -> List[Dict[str, Any]]:
|
|
435
475
|
"""Convert data types for specified fields using string type names.
|
|
436
476
|
|
|
437
477
|
This is a simplified version of convert_data_types for LLM agent compatibility.
|
|
@@ -462,31 +502,45 @@ def convert_data_types_simple(data: List[Dict[str, Any]], type_mapping: Dict[str
|
|
|
462
502
|
"str": str,
|
|
463
503
|
"int": int,
|
|
464
504
|
"float": float,
|
|
465
|
-
"bool": lambda x: bool(x)
|
|
505
|
+
"bool": lambda x: bool(x)
|
|
506
|
+
if x in (0, 1)
|
|
507
|
+
else x.lower() in ("true", "yes", "1")
|
|
508
|
+
if isinstance(x, str)
|
|
509
|
+
else bool(x),
|
|
466
510
|
}
|
|
467
511
|
|
|
468
512
|
for i, record in enumerate(data):
|
|
469
513
|
if not isinstance(record, dict):
|
|
470
|
-
raise DataError(
|
|
514
|
+
raise DataError(
|
|
515
|
+
f"Record at index {i} is not a dictionary: {type(record).__name__}"
|
|
516
|
+
)
|
|
471
517
|
|
|
472
518
|
converted_record = record.copy()
|
|
473
519
|
|
|
474
520
|
for field, type_name in type_mapping.items():
|
|
475
521
|
if field in converted_record and converted_record[field] is not None:
|
|
476
522
|
if type_name not in type_functions:
|
|
477
|
-
raise DataError(
|
|
523
|
+
raise DataError(
|
|
524
|
+
f"Invalid type name: {type_name}. Must be one of: str, int, float, bool"
|
|
525
|
+
)
|
|
478
526
|
|
|
479
527
|
try:
|
|
480
|
-
converted_record[field] = type_functions[type_name](
|
|
528
|
+
converted_record[field] = type_functions[type_name](
|
|
529
|
+
converted_record[field]
|
|
530
|
+
)
|
|
481
531
|
except ValueError as e:
|
|
482
|
-
raise DataError(
|
|
532
|
+
raise DataError(
|
|
533
|
+
f"Error converting field '{field}' to {type_name}: {e}"
|
|
534
|
+
)
|
|
483
535
|
|
|
484
536
|
converted_data.append(converted_record)
|
|
485
537
|
|
|
486
538
|
return converted_data
|
|
487
539
|
|
|
488
540
|
|
|
489
|
-
def clean_data_simple(
|
|
541
|
+
def clean_data_simple(
|
|
542
|
+
data: List[Dict[str, Any]], rules: Dict[str, str]
|
|
543
|
+
) -> List[Dict[str, Any]]:
|
|
490
544
|
"""Clean data according to simple string-based rules.
|
|
491
545
|
|
|
492
546
|
This is a simplified version of clean_data for LLM agent compatibility.
|
|
@@ -519,12 +573,14 @@ def clean_data_simple(data: List[Dict[str, Any]], rules: Dict[str, str]) -> List
|
|
|
519
573
|
"lower": str.lower,
|
|
520
574
|
"upper": str.upper,
|
|
521
575
|
"title": str.title,
|
|
522
|
-
"capitalize": str.capitalize
|
|
576
|
+
"capitalize": str.capitalize,
|
|
523
577
|
}
|
|
524
578
|
|
|
525
579
|
for i, record in enumerate(data):
|
|
526
580
|
if not isinstance(record, dict):
|
|
527
|
-
raise DataError(
|
|
581
|
+
raise DataError(
|
|
582
|
+
f"Record at index {i} is not a dictionary: {type(record).__name__}"
|
|
583
|
+
)
|
|
528
584
|
|
|
529
585
|
cleaned_record = record.copy()
|
|
530
586
|
|
|
@@ -534,20 +590,26 @@ def clean_data_simple(data: List[Dict[str, Any]], rules: Dict[str, str]) -> List
|
|
|
534
590
|
raise DataError(f"Rule for field '{field}' must be a string")
|
|
535
591
|
|
|
536
592
|
if rule not in cleaning_functions:
|
|
537
|
-
raise DataError(
|
|
593
|
+
raise DataError(
|
|
594
|
+
f"Invalid cleaning rule: {rule}. Must be one of: {', '.join(cleaning_functions.keys())}"
|
|
595
|
+
)
|
|
538
596
|
|
|
539
597
|
try:
|
|
540
598
|
value = str(cleaned_record[field])
|
|
541
599
|
cleaned_record[field] = cleaning_functions[rule](value)
|
|
542
600
|
except Exception as e:
|
|
543
|
-
raise DataError(
|
|
601
|
+
raise DataError(
|
|
602
|
+
f"Error applying rule '{rule}' to field '{field}': {e}"
|
|
603
|
+
)
|
|
544
604
|
|
|
545
605
|
cleaned_data.append(cleaned_record)
|
|
546
606
|
|
|
547
607
|
return cleaned_data
|
|
548
608
|
|
|
549
609
|
|
|
550
|
-
def deduplicate_records_simple(
|
|
610
|
+
def deduplicate_records_simple(
|
|
611
|
+
data: List[Dict[str, Any]], key_fields: Optional[List[str]] = None
|
|
612
|
+
) -> List[Dict[str, Any]]:
|
|
551
613
|
"""Remove duplicate records from a list.
|
|
552
614
|
|
|
553
615
|
This is a simplified version of deduplicate_records for LLM agent compatibility.
|
|
@@ -579,7 +641,9 @@ def deduplicate_records_simple(data: List[Dict[str, Any]], key_fields: Optional[
|
|
|
579
641
|
|
|
580
642
|
for i, record in enumerate(data):
|
|
581
643
|
if not isinstance(record, dict):
|
|
582
|
-
raise DataError(
|
|
644
|
+
raise DataError(
|
|
645
|
+
f"Record at index {i} is not a dictionary: {type(record).__name__}"
|
|
646
|
+
)
|
|
583
647
|
|
|
584
648
|
# Create a key based on specified fields
|
|
585
649
|
key_values = tuple(str(record.get(field, "")) for field in key_fields)
|
|
@@ -175,7 +175,11 @@ def validate_data_types_simple(data: Dict[str, Any], type_map: Dict[str, str]) -
|
|
|
175
175
|
return True
|
|
176
176
|
|
|
177
177
|
|
|
178
|
-
def validate_range_simple(
|
|
178
|
+
def validate_range_simple(
|
|
179
|
+
value: Union[int, float],
|
|
180
|
+
min_val: Optional[Union[int, float]] = None,
|
|
181
|
+
max_val: Optional[Union[int, float]] = None,
|
|
182
|
+
) -> bool:
|
|
179
183
|
"""Validate numeric value is within range.
|
|
180
184
|
|
|
181
185
|
Args:
|
|
@@ -214,7 +218,9 @@ def validate_range_simple(value: Union[int, float], min_val: Optional[Union[int,
|
|
|
214
218
|
return True
|
|
215
219
|
|
|
216
220
|
|
|
217
|
-
def create_validation_report(
|
|
221
|
+
def create_validation_report(
|
|
222
|
+
data: Dict[str, Any], rules: Dict[str, Any]
|
|
223
|
+
) -> Dict[str, Any]:
|
|
218
224
|
"""Create comprehensive validation report for data.
|
|
219
225
|
|
|
220
226
|
Args:
|
|
@@ -320,7 +326,9 @@ def check_required_fields_simple(data: Dict[str, Any], required: List[str]) -> b
|
|
|
320
326
|
return check_required_fields(data, required)
|
|
321
327
|
|
|
322
328
|
|
|
323
|
-
def create_validation_report_simple(
|
|
329
|
+
def create_validation_report_simple(
|
|
330
|
+
data: Dict[str, Any], rules: Dict[str, Any]
|
|
331
|
+
) -> Dict[str, Any]:
|
|
324
332
|
"""Create simplified validation report for data.
|
|
325
333
|
|
|
326
334
|
This is an alias for create_validation_report for LLM agent compatibility.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: basic-open-agent-tools
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.4
|
|
4
4
|
Summary: An open foundational toolkit providing essential components for building AI agents with minimal dependencies for local (non-HTTP/API) actions.
|
|
5
5
|
Home-page: https://github.com/open-agent-tools/basic-open-agent-tools
|
|
6
6
|
Author: Open Agent Tools
|
|
@@ -72,7 +72,7 @@ class TestReadCsvSimple:
|
|
|
72
72
|
|
|
73
73
|
def test_read_csv_invalid_types(self):
|
|
74
74
|
"""Test with invalid argument types."""
|
|
75
|
-
with pytest.raises(TypeError, match="file_path must be a string
|
|
75
|
+
with pytest.raises(TypeError, match="file_path must be a string"):
|
|
76
76
|
read_csv_simple(123)
|
|
77
77
|
|
|
78
78
|
with pytest.raises(TypeError, match="delimiter must be a string"):
|
|
@@ -152,7 +152,7 @@ class TestWriteCsvFile:
|
|
|
152
152
|
with pytest.raises(TypeError, match="data must be a list"):
|
|
153
153
|
write_csv_simple("not a list", csv_file)
|
|
154
154
|
|
|
155
|
-
with pytest.raises(TypeError, match="file_path must be a string
|
|
155
|
+
with pytest.raises(TypeError, match="file_path must be a string"):
|
|
156
156
|
write_csv_simple([], 123)
|
|
157
157
|
|
|
158
158
|
with pytest.raises(TypeError, match="All items in data must be dictionaries"):
|
|
@@ -295,7 +295,7 @@ class TestDetectCsvDelimiter:
|
|
|
295
295
|
|
|
296
296
|
def test_detect_invalid_types(self):
|
|
297
297
|
"""Test with invalid argument types."""
|
|
298
|
-
with pytest.raises(TypeError, match="file_path must be a string
|
|
298
|
+
with pytest.raises(TypeError, match="file_path must be a string"):
|
|
299
299
|
detect_csv_delimiter(123)
|
|
300
300
|
|
|
301
301
|
with pytest.raises(TypeError, match="sample_size must be a positive integer"):
|
|
@@ -342,7 +342,7 @@ class TestValidateCsvStructure:
|
|
|
342
342
|
|
|
343
343
|
def test_validate_invalid_types(self):
|
|
344
344
|
"""Test with invalid argument types."""
|
|
345
|
-
with pytest.raises(TypeError, match="file_path must be a string
|
|
345
|
+
with pytest.raises(TypeError, match="file_path must be a string"):
|
|
346
346
|
validate_csv_structure(123)
|
|
347
347
|
|
|
348
348
|
with pytest.raises(TypeError, match="expected_columns must be a list or None"):
|
|
@@ -47,7 +47,7 @@ class TestSafeJsonSerialize:
|
|
|
47
47
|
|
|
48
48
|
def test_serialize_invalid_indent_type(self):
|
|
49
49
|
"""Test with invalid indent type."""
|
|
50
|
-
with pytest.raises(TypeError, match="indent must be an integer
|
|
50
|
+
with pytest.raises(TypeError, match="indent must be an integer"):
|
|
51
51
|
safe_json_serialize({"test": "data"}, indent="invalid")
|
|
52
52
|
|
|
53
53
|
def test_serialize_non_serializable_object(self):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/helpers.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/src/basic_open_agent_tools/types.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_data_archive_processing.py
RENAMED
|
File without changes
|
{basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_data_binary_processing.py
RENAMED
|
File without changes
|
{basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_data_config_processing.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{basic_open_agent_tools-0.6.2 → basic_open_agent_tools-0.6.4}/tests/test_file_system_tools.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|