xploreds 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xploreds/__init__.py +7 -0
- xploreds/data_analysis/xds_drift.py +5 -0
- xploreds/data_analysis/xds_eda.py +5 -0
- xploreds/data_analysis/xds_outliers.py +5 -0
- xploreds/data_analysis/xds_quality.py +5 -0
- xploreds/data_analysis/xds_stats.py +5 -0
- xploreds/data_handle/xds_dataframe_pandas.py +5 -0
- xploreds/data_transformation/xds_categorical_encoding.py +5 -0
- xploreds/data_transformation/xds_datetime_encoding.py +5 -0
- xploreds/data_transformation/xds_missing_inputation.py +5 -0
- xploreds/data_transformation/xds_scaling.py +5 -0
- xploreds/data_visualization/xds_plotly.py +5 -0
- xploreds/io/xds_file.py +263 -0
- xploreds/log/xds_log.py +336 -0
- xploreds/models/xds_evaluate_model.py +5 -0
- xploreds/models/xds_linear_regression.py +5 -0
- xploreds/models/xds_logistic_regression.py +5 -0
- xploreds/time/xds_date_time.py +5 -0
- xploreds/utils/xds_memory.py +39 -0
- xploreds-0.1.0.dist-info/METADATA +88 -0
- xploreds-0.1.0.dist-info/RECORD +22 -0
- xploreds-0.1.0.dist-info/WHEEL +4 -0
xploreds/__init__.py
ADDED
xploreds/io/xds_file.py
ADDED
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
f"""
|
|
2
|
+
Xplore DS :: I/O Main Package
|
|
3
|
+
|
|
4
|
+
Utilities for folder management, JSON I/O, and pandas DataFrame persistence.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
from collections.abc import Sequence
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, Optional, Union
|
|
12
|
+
|
|
13
|
+
import pandas as pd
|
|
14
|
+
|
|
15
|
+
PathLike = Union[str, Path]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _as_path(path: PathLike) -> Path:
|
|
19
|
+
"""Normalize a path-like value to ``Path``."""
|
|
20
|
+
return Path(path)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _ensure_parent_dir(file_path: PathLike) -> None:
|
|
24
|
+
"""Create the parent directory of a file path when it does not exist."""
|
|
25
|
+
parent = _as_path(file_path).parent
|
|
26
|
+
# Ignora caminhos relativos na raiz (ex.: "arquivo.json")
|
|
27
|
+
if parent != Path("."):
|
|
28
|
+
parent.mkdir(parents=True, exist_ok=True)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _log_dataframe_summary(logger: logging.Logger, data: pd.DataFrame) -> None:
|
|
32
|
+
"""Log shape and column names of a DataFrame."""
|
|
33
|
+
logger.info("Total samples: %s", data.shape[0])
|
|
34
|
+
logger.info("Number of variables: %s", data.shape[1])
|
|
35
|
+
logger.info("Variables list: %s", data.columns.tolist())
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def create_folder(folder_path: PathLike) -> bool:
|
|
39
|
+
"""
|
|
40
|
+
Create a folder at the specified path when it does not exist.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
folder_path: Directory path to create. Accepts ``str`` or ``Path``.
|
|
44
|
+
|
|
45
|
+
Returns:
|
|
46
|
+
True when a new folder is created, False when it already exists.
|
|
47
|
+
|
|
48
|
+
Note:
|
|
49
|
+
Nested directories are created automatically (``parents=True``).
|
|
50
|
+
"""
|
|
51
|
+
path = _as_path(folder_path)
|
|
52
|
+
|
|
53
|
+
# Retorna cedo quando o diretorio ja existe
|
|
54
|
+
if path.exists():
|
|
55
|
+
return False
|
|
56
|
+
|
|
57
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
58
|
+
return True
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def get_name_and_extension_from_file(filename: str) -> tuple[str, str]:
|
|
62
|
+
"""
|
|
63
|
+
Split a filename into its base name and extension.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
filename: Full filename or path containing the file name.
|
|
67
|
+
|
|
68
|
+
Returns:
|
|
69
|
+
A tuple ``(name, extension)``. Extension includes the leading dot.
|
|
70
|
+
When there is no extension, the second element is an empty string.
|
|
71
|
+
|
|
72
|
+
Example:
|
|
73
|
+
``get_name_and_extension_from_file("archive.tar.gz")`` returns
|
|
74
|
+
``("archive.tar", ".gz")``.
|
|
75
|
+
"""
|
|
76
|
+
path = Path(Path(filename).name)
|
|
77
|
+
return path.stem, path.suffix
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def get_filename_from_path(path: PathLike) -> str:
|
|
81
|
+
"""
|
|
82
|
+
Extract the filename from a file path.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
path: Full path to the file. Accepts ``str`` or ``Path``.
|
|
86
|
+
|
|
87
|
+
Returns:
|
|
88
|
+
Filename including extension, without parent directories.
|
|
89
|
+
"""
|
|
90
|
+
return _as_path(path).name
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def load_dictionary_from_json(path_file: PathLike) -> dict[str, Any]:
|
|
94
|
+
"""
|
|
95
|
+
Load a dictionary from a JSON file.
|
|
96
|
+
|
|
97
|
+
Args:
|
|
98
|
+
path_file: Path to the JSON file. Accepts ``str`` or ``Path``.
|
|
99
|
+
|
|
100
|
+
Returns:
|
|
101
|
+
Dictionary parsed from the JSON content.
|
|
102
|
+
|
|
103
|
+
Raises:
|
|
104
|
+
FileNotFoundError: When the file does not exist.
|
|
105
|
+
json.JSONDecodeError: When the file content is not valid JSON.
|
|
106
|
+
PermissionError: When the file cannot be read due to permissions.
|
|
107
|
+
"""
|
|
108
|
+
path = _as_path(path_file)
|
|
109
|
+
if not path.is_file():
|
|
110
|
+
raise FileNotFoundError(f"The file {path} was not found.")
|
|
111
|
+
|
|
112
|
+
with path.open(encoding="utf-8") as json_file:
|
|
113
|
+
return json.load(json_file)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def save_dictionary_to_json(
|
|
117
|
+
data: dict[str, Any],
|
|
118
|
+
file_path: PathLike,
|
|
119
|
+
log: Optional[logging.Logger] = None,
|
|
120
|
+
) -> None:
|
|
121
|
+
"""
|
|
122
|
+
Save a dictionary to a JSON file.
|
|
123
|
+
|
|
124
|
+
Args:
|
|
125
|
+
data: Dictionary to serialize.
|
|
126
|
+
file_path: Destination file path. Accepts ``str`` or ``Path``.
|
|
127
|
+
log: Optional logger for progress messages.
|
|
128
|
+
|
|
129
|
+
Raises:
|
|
130
|
+
OSError: When the file cannot be written.
|
|
131
|
+
TypeError: When ``data`` is not JSON-serializable.
|
|
132
|
+
|
|
133
|
+
Note:
|
|
134
|
+
Creates parent directories automatically when they do not exist.
|
|
135
|
+
"""
|
|
136
|
+
path = _as_path(file_path)
|
|
137
|
+
|
|
138
|
+
if log:
|
|
139
|
+
log.info("Saving dictionary to json file...")
|
|
140
|
+
|
|
141
|
+
# Garante diretorio pai antes de gravar
|
|
142
|
+
_ensure_parent_dir(path)
|
|
143
|
+
|
|
144
|
+
with path.open("w", encoding="utf-8") as json_file:
|
|
145
|
+
json.dump(data, json_file, indent=4)
|
|
146
|
+
|
|
147
|
+
if log:
|
|
148
|
+
log.info("Dictionary saved to json file: %s", path)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def get_nrows_from_file(filepath: PathLike) -> int:
|
|
152
|
+
"""
|
|
153
|
+
Count the number of lines in a text file.
|
|
154
|
+
|
|
155
|
+
Args:
|
|
156
|
+
filepath: Path to the text file. Accepts ``str`` or ``Path``.
|
|
157
|
+
|
|
158
|
+
Returns:
|
|
159
|
+
Number of lines in the file. Returns ``0`` for empty files.
|
|
160
|
+
|
|
161
|
+
Raises:
|
|
162
|
+
OSError: When the file cannot be opened or read.
|
|
163
|
+
"""
|
|
164
|
+
with _as_path(filepath).open(encoding="utf-8") as file:
|
|
165
|
+
return sum(1 for _ in file)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def save_dataframe_to_parquet(
|
|
169
|
+
data: pd.DataFrame,
|
|
170
|
+
file_path: PathLike,
|
|
171
|
+
log: Optional[logging.Logger] = None,
|
|
172
|
+
) -> None:
|
|
173
|
+
"""
|
|
174
|
+
Save a pandas DataFrame to a Parquet file.
|
|
175
|
+
|
|
176
|
+
Args:
|
|
177
|
+
data: DataFrame to persist.
|
|
178
|
+
file_path: Destination file path. Accepts ``str`` or ``Path``.
|
|
179
|
+
log: Optional logger for progress and dataset summary.
|
|
180
|
+
|
|
181
|
+
Raises:
|
|
182
|
+
OSError: When the file cannot be written.
|
|
183
|
+
|
|
184
|
+
Note:
|
|
185
|
+
Requires ``pyarrow`` or ``fastparquet`` installed in the environment.
|
|
186
|
+
"""
|
|
187
|
+
path = _as_path(file_path)
|
|
188
|
+
|
|
189
|
+
if log:
|
|
190
|
+
log.info("Saving dataframe to parquet file...")
|
|
191
|
+
|
|
192
|
+
_ensure_parent_dir(path)
|
|
193
|
+
data.to_parquet(path)
|
|
194
|
+
|
|
195
|
+
if log:
|
|
196
|
+
log.info("Dataframe saved to parquet file: %s", path)
|
|
197
|
+
_log_dataframe_summary(log, data)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def load_dataframe_from_parquet(
|
|
201
|
+
file_path: PathLike,
|
|
202
|
+
selected_columns: Optional[Sequence[str]] = None,
|
|
203
|
+
log: Optional[logging.Logger] = None,
|
|
204
|
+
) -> pd.DataFrame:
|
|
205
|
+
"""
|
|
206
|
+
Load a pandas DataFrame from a Parquet file.
|
|
207
|
+
|
|
208
|
+
Args:
|
|
209
|
+
file_path: Path to the Parquet file. Accepts ``str`` or ``Path``.
|
|
210
|
+
selected_columns: Optional column subset to load. Loads all columns
|
|
211
|
+
when ``None``.
|
|
212
|
+
log: Optional logger for progress and dataset summary.
|
|
213
|
+
|
|
214
|
+
Returns:
|
|
215
|
+
DataFrame loaded from the Parquet file.
|
|
216
|
+
|
|
217
|
+
Raises:
|
|
218
|
+
OSError: When the file cannot be read.
|
|
219
|
+
"""
|
|
220
|
+
path = _as_path(file_path)
|
|
221
|
+
columns = list(selected_columns) if selected_columns else None
|
|
222
|
+
data = pd.read_parquet(path, columns=columns)
|
|
223
|
+
|
|
224
|
+
if log:
|
|
225
|
+
log.info("Dataframe loaded from parquet file: %s", path)
|
|
226
|
+
_log_dataframe_summary(log, data)
|
|
227
|
+
|
|
228
|
+
return data
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def save_dataframe_to_excel(
|
|
232
|
+
data: pd.DataFrame,
|
|
233
|
+
file_path: PathLike,
|
|
234
|
+
sheet_name: str = "data",
|
|
235
|
+
log: Optional[logging.Logger] = None,
|
|
236
|
+
) -> None:
|
|
237
|
+
"""
|
|
238
|
+
Save a pandas DataFrame to an Excel file.
|
|
239
|
+
|
|
240
|
+
Args:
|
|
241
|
+
data: DataFrame to persist.
|
|
242
|
+
file_path: Destination file path. Accepts ``str`` or ``Path``.
|
|
243
|
+
sheet_name: Worksheet name. Defaults to ``"data"``.
|
|
244
|
+
log: Optional logger for progress and dataset summary.
|
|
245
|
+
|
|
246
|
+
Raises:
|
|
247
|
+
OSError: When the file cannot be written.
|
|
248
|
+
|
|
249
|
+
Note:
|
|
250
|
+
Requires ``openpyxl`` installed in the environment.
|
|
251
|
+
The index is not written to the spreadsheet.
|
|
252
|
+
"""
|
|
253
|
+
path = _as_path(file_path)
|
|
254
|
+
|
|
255
|
+
if log:
|
|
256
|
+
log.info("Saving dataframe to excel file...")
|
|
257
|
+
|
|
258
|
+
_ensure_parent_dir(path)
|
|
259
|
+
data.to_excel(path, sheet_name=sheet_name, index=False)
|
|
260
|
+
|
|
261
|
+
if log:
|
|
262
|
+
log.info("Dataframe saved to excel file: %s", path)
|
|
263
|
+
_log_dataframe_summary(log, data)
|
xploreds/log/xds_log.py
ADDED
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Xplore DS :: Logging Tools Package
|
|
3
|
+
|
|
4
|
+
Structured logging for scripts and cookbook runs, including environment profiling.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import logging
|
|
8
|
+
import warnings
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
import time
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from xploreds.io.xds_file import create_folder
|
|
13
|
+
from xploreds.utils.xds_memory import convert_bytes
|
|
14
|
+
import sys
|
|
15
|
+
import os
|
|
16
|
+
import platform
|
|
17
|
+
import psutil
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class XploreDSLogging:
|
|
21
|
+
"""
|
|
22
|
+
Configure and write structured logs for a single script execution.
|
|
23
|
+
|
|
24
|
+
Each run creates a timestamped folder under ``runs/`` with a dedicated
|
|
25
|
+
``.log`` file. Use wrapper methods (``info``, ``title``, ``section``) to
|
|
26
|
+
keep log output consistent across projects.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
project_root: Root directory of the project.
|
|
30
|
+
script_name: Name of the script being executed (used in log paths).
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def __init__(self, project_root: str, script_name: str) -> None:
|
|
34
|
+
"""
|
|
35
|
+
Initialize the logging configuration for the project.
|
|
36
|
+
|
|
37
|
+
This constructor sets up the logging environment by configuring file paths,
|
|
38
|
+
creating necessary directories, and initializing the logger.
|
|
39
|
+
|
|
40
|
+
Args:
|
|
41
|
+
project_root (str): The root directory of the project.
|
|
42
|
+
script_name (str): The name of the script being executed.
|
|
43
|
+
|
|
44
|
+
Attributes:
|
|
45
|
+
dt_init (datetime): Timestamp of logger initialization.
|
|
46
|
+
dt_init_str (str): Formatted string of initialization timestamp.
|
|
47
|
+
log_name (str): Name of the log file.
|
|
48
|
+
log_run (str): Identifier for the current run.
|
|
49
|
+
log_path (str): Directory path where logs will be stored.
|
|
50
|
+
log_file (str): Full path to the log file.
|
|
51
|
+
logger: Configured logging object.
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
None
|
|
55
|
+
|
|
56
|
+
Note:
|
|
57
|
+
This method creates a new directory for each run under the 'runs' folder
|
|
58
|
+
in the project root, named with the script name and timestamp.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
# capturando referencias de caminhos
|
|
62
|
+
self.project_root = project_root
|
|
63
|
+
self.script_name = script_name
|
|
64
|
+
|
|
65
|
+
# capturando referencias de tempos de inicialização
|
|
66
|
+
self.dt_init = datetime.now()
|
|
67
|
+
self.dt_init_str = self.dt_init.strftime("%y%m%d%H%M%S")
|
|
68
|
+
self.ts_init = time.time()
|
|
69
|
+
|
|
70
|
+
# configurando o nome do arquivo de log
|
|
71
|
+
self.log_name = Path(str(script_name)).stem + "_" + self.dt_init_str + ".log"
|
|
72
|
+
self.log_run = Path(str(script_name)).stem + "_" + self.dt_init_str
|
|
73
|
+
|
|
74
|
+
# configurando locais de registros de logs
|
|
75
|
+
self.log_path = str(project_root) + "/runs/" + str(self.log_run) + "/"
|
|
76
|
+
|
|
77
|
+
# configurando o path completo de log
|
|
78
|
+
self.log_file = self.log_path + self.log_name
|
|
79
|
+
|
|
80
|
+
# criando a pasta caso não exista
|
|
81
|
+
create_folder(self.log_path)
|
|
82
|
+
|
|
83
|
+
# configurando o logger
|
|
84
|
+
self.logger = self._setup_logger()
|
|
85
|
+
|
|
86
|
+
# Configuracao de warnings
|
|
87
|
+
warnings.filterwarnings(os.getenv("PYTHON_WARNINGS"))
|
|
88
|
+
|
|
89
|
+
def _setup_logger(self) -> None:
|
|
90
|
+
"""
|
|
91
|
+
Set up and configure the logger for the class.
|
|
92
|
+
|
|
93
|
+
This method configures the logging system with specific settings:
|
|
94
|
+
- Sets the logging level to INFO
|
|
95
|
+
- Defines a custom format for log messages
|
|
96
|
+
- Sets up both file and stream handlers for logging
|
|
97
|
+
|
|
98
|
+
The configuration includes:
|
|
99
|
+
- Log level: INFO
|
|
100
|
+
- Format: timestamp - log level - message
|
|
101
|
+
- Date format: DD-MM-YYYY HH:MM:SS
|
|
102
|
+
- Handlers: FileHandler (writes to a file) and StreamHandler (writes to console)
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
None
|
|
106
|
+
|
|
107
|
+
Note:
|
|
108
|
+
This is an internal method, as indicated by the leading underscore.
|
|
109
|
+
The log file path should be set in self.log_file before calling this method.
|
|
110
|
+
"""
|
|
111
|
+
|
|
112
|
+
# configurando o logger
|
|
113
|
+
log_config = {
|
|
114
|
+
"level": logging.INFO,
|
|
115
|
+
"format": "%(asctime)s - %(levelname)s - %(message)s",
|
|
116
|
+
"datefmt": "%d-%m-%Y %H:%M:%S",
|
|
117
|
+
"handlers": [logging.FileHandler(self.log_file), logging.StreamHandler()],
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
logging.basicConfig(**log_config)
|
|
121
|
+
|
|
122
|
+
return logging.basicConfig(**log_config)
|
|
123
|
+
|
|
124
|
+
def info(self, message: str) -> None:
|
|
125
|
+
"""
|
|
126
|
+
Log an information-level message.
|
|
127
|
+
|
|
128
|
+
This method logs the provided message using the logging module's info level.
|
|
129
|
+
It serves as a wrapper around logging.info() within the context of the class.
|
|
130
|
+
|
|
131
|
+
Args:
|
|
132
|
+
message (str): The message to be logged.
|
|
133
|
+
|
|
134
|
+
Returns:
|
|
135
|
+
None
|
|
136
|
+
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
logging.info(message)
|
|
140
|
+
|
|
141
|
+
def warning(self, message: str) -> None:
|
|
142
|
+
"""
|
|
143
|
+
Log a warning-level message.
|
|
144
|
+
|
|
145
|
+
This method logs the provided message using the logging module's warning level.
|
|
146
|
+
It serves as a wrapper around logging.warning() within the context of the class.
|
|
147
|
+
|
|
148
|
+
Args:
|
|
149
|
+
message (str): The message to be logged as a warning.
|
|
150
|
+
|
|
151
|
+
Returns:
|
|
152
|
+
None
|
|
153
|
+
"""
|
|
154
|
+
logging.warning(message)
|
|
155
|
+
|
|
156
|
+
def error(self, message: str) -> None:
|
|
157
|
+
"""
|
|
158
|
+
Log an error-level message.
|
|
159
|
+
|
|
160
|
+
This method logs the provided message using the logging module's error level.
|
|
161
|
+
It serves as a wrapper around logging.error() within the context of the class.
|
|
162
|
+
|
|
163
|
+
Args:
|
|
164
|
+
message (str): The message to be logged as an error.
|
|
165
|
+
|
|
166
|
+
Returns:
|
|
167
|
+
None
|
|
168
|
+
"""
|
|
169
|
+
logging.error(message)
|
|
170
|
+
|
|
171
|
+
def title(self, message: str) -> None:
|
|
172
|
+
"""
|
|
173
|
+
Log a top-level title with a heavy separator.
|
|
174
|
+
|
|
175
|
+
Args:
|
|
176
|
+
message: Title text to log.
|
|
177
|
+
"""
|
|
178
|
+
logging.info(
|
|
179
|
+
"**********************************************************************************"
|
|
180
|
+
)
|
|
181
|
+
logging.info(message)
|
|
182
|
+
logging.info(
|
|
183
|
+
"**********************************************************************************"
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
def subtitle(self, message: str) -> None:
|
|
187
|
+
"""
|
|
188
|
+
Log a subtitle with a medium separator.
|
|
189
|
+
|
|
190
|
+
Args:
|
|
191
|
+
message: Subtitle text to log.
|
|
192
|
+
"""
|
|
193
|
+
logging.info(
|
|
194
|
+
"=================================================================================="
|
|
195
|
+
)
|
|
196
|
+
logging.info(message)
|
|
197
|
+
logging.info(
|
|
198
|
+
"=================================================================================="
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
def section(self, message: str) -> None:
|
|
202
|
+
"""
|
|
203
|
+
Log a section header with a light separator.
|
|
204
|
+
|
|
205
|
+
Args:
|
|
206
|
+
message: Section text to log.
|
|
207
|
+
"""
|
|
208
|
+
logging.info(
|
|
209
|
+
"----------------------------------------------------------------------------------"
|
|
210
|
+
)
|
|
211
|
+
logging.info(message)
|
|
212
|
+
logging.info(
|
|
213
|
+
"----------------------------------------------------------------------------------"
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
def init_run(self) -> None:
|
|
217
|
+
"""
|
|
218
|
+
Initialize a new run in the log.
|
|
219
|
+
|
|
220
|
+
This method logs the start of a new script execution by printing a
|
|
221
|
+
distinctive header in the log file. It creates a visual separation
|
|
222
|
+
between different runs, making it easier to identify the beginning
|
|
223
|
+
of a new execution in the log.
|
|
224
|
+
|
|
225
|
+
Returns:
|
|
226
|
+
None
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
"""
|
|
230
|
+
|
|
231
|
+
# configurando abertura de run
|
|
232
|
+
logging.info(
|
|
233
|
+
"**********************************************************************************"
|
|
234
|
+
)
|
|
235
|
+
logging.info("Starting script: " + self.log_run)
|
|
236
|
+
logging.info(
|
|
237
|
+
"**********************************************************************************"
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
def close_run(self) -> None:
|
|
241
|
+
"""
|
|
242
|
+
Finalize and close the current run in the log.
|
|
243
|
+
|
|
244
|
+
This method logs the end of the current script execution by printing a
|
|
245
|
+
distinctive footer in the log file. It creates a visual separation to
|
|
246
|
+
clearly mark the end of the current run, making it easier to identify
|
|
247
|
+
where one execution ends in the log.
|
|
248
|
+
|
|
249
|
+
"""
|
|
250
|
+
|
|
251
|
+
time_ref_end = datetime.now()
|
|
252
|
+
time_end = time.time()
|
|
253
|
+
exec_time = ((time_end - self.ts_init) / 60) / 60
|
|
254
|
+
|
|
255
|
+
logging.info(
|
|
256
|
+
"**********************************************************************************"
|
|
257
|
+
)
|
|
258
|
+
logging.info("Execution finished script: " + self.log_run)
|
|
259
|
+
logging.info("Conclusion at " + str(time_ref_end.strftime("%Y-%m-%d %H:%M")))
|
|
260
|
+
logging.info("Execution time: %.2f hours" % exec_time)
|
|
261
|
+
logging.info(
|
|
262
|
+
"**********************************************************************************"
|
|
263
|
+
)
|
|
264
|
+
logging.info(
|
|
265
|
+
" XploreDS"
|
|
266
|
+
)
|
|
267
|
+
logging.info(
|
|
268
|
+
"**********************************************************************************"
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
def log_environment_setup(self) -> None:
|
|
272
|
+
"""
|
|
273
|
+
Log hardware, OS, Python, and run configuration details.
|
|
274
|
+
|
|
275
|
+
Captures machine architecture, CPU and memory, platform information,
|
|
276
|
+
Python interpreter path, warning settings, and current run folders.
|
|
277
|
+
|
|
278
|
+
Note:
|
|
279
|
+
Requires ``psutil`` for memory and CPU reporting.
|
|
280
|
+
"""
|
|
281
|
+
logging.info(
|
|
282
|
+
"**********************************************************************************"
|
|
283
|
+
)
|
|
284
|
+
logging.info("Environment Setup")
|
|
285
|
+
logging.info(
|
|
286
|
+
"----------------------------------------------------------------------------------"
|
|
287
|
+
)
|
|
288
|
+
logging.info("Machine Setup")
|
|
289
|
+
logging.info(
|
|
290
|
+
"----------------------------------------------------------------------------------"
|
|
291
|
+
)
|
|
292
|
+
logging.info("Machine: " + str(platform.machine()))
|
|
293
|
+
logging.info("Processor: " + str(platform.processor()))
|
|
294
|
+
logging.info("Physical cores: " + str(psutil.cpu_count(logical=False)))
|
|
295
|
+
logging.info("Total cores: " + str(psutil.cpu_count(logical=True)))
|
|
296
|
+
logging.info(
|
|
297
|
+
"Physical memory: {gb:.2f}GB".format(
|
|
298
|
+
gb=convert_bytes(psutil.virtual_memory().total, "GB"),
|
|
299
|
+
)
|
|
300
|
+
)
|
|
301
|
+
logging.info(
|
|
302
|
+
"----------------------------------------------------------------------------------"
|
|
303
|
+
)
|
|
304
|
+
logging.info("OS Setup")
|
|
305
|
+
logging.info(
|
|
306
|
+
"----------------------------------------------------------------------------------"
|
|
307
|
+
)
|
|
308
|
+
logging.info("OS id: " + str(platform.platform()))
|
|
309
|
+
logging.info("OS System: " + str(platform.system()))
|
|
310
|
+
logging.info("OS Node Name: " + str(platform.node()))
|
|
311
|
+
logging.info("OS Release:" + str(platform.release()))
|
|
312
|
+
|
|
313
|
+
logging.info(
|
|
314
|
+
"----------------------------------------------------------------------------------"
|
|
315
|
+
)
|
|
316
|
+
logging.info("Python Setup")
|
|
317
|
+
logging.info(
|
|
318
|
+
"----------------------------------------------------------------------------------"
|
|
319
|
+
)
|
|
320
|
+
logging.info("Python version: " + sys.version)
|
|
321
|
+
logging.info("Python path: " + sys.executable)
|
|
322
|
+
logging.info("Python warnings: " + str(os.getenv("PYTHON_WARNINGS")))
|
|
323
|
+
logging.info(
|
|
324
|
+
"----------------------------------------------------------------------------------"
|
|
325
|
+
)
|
|
326
|
+
logging.info("Run Setup")
|
|
327
|
+
logging.info(
|
|
328
|
+
"----------------------------------------------------------------------------------"
|
|
329
|
+
)
|
|
330
|
+
logging.info("Root folder:" + str(self.project_root))
|
|
331
|
+
logging.info("Run name: " + str(self.log_run))
|
|
332
|
+
logging.info("Artifacts folder:" + str(self.log_path))
|
|
333
|
+
|
|
334
|
+
logging.info(
|
|
335
|
+
"**********************************************************************************"
|
|
336
|
+
)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Xplore DS :: Memory Utils Package
|
|
3
|
+
|
|
4
|
+
Utilities for converting and expressing memory sizes in binary units.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from typing import Final
|
|
8
|
+
|
|
9
|
+
BYTES_PER_UNIT: Final[int] = 1024
|
|
10
|
+
BINARY_UNITS: Final[tuple[str, ...]] = ("B", "KB", "MB", "GB", "TB")
|
|
11
|
+
_UNIT_EXPONENT: Final[dict[str, int]] = {
|
|
12
|
+
unit: exponent for exponent, unit in enumerate(BINARY_UNITS)
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def convert_bytes(size_bytes: int | float, to_unit: str = "GB") -> float:
|
|
17
|
+
"""
|
|
18
|
+
Convert a byte count to the requested binary unit (KiB-based).
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
size_bytes: Size in bytes.
|
|
22
|
+
to_unit: Target unit. One of B, KB, MB, GB, or TB (case-insensitive).
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
Size expressed in the requested unit.
|
|
26
|
+
|
|
27
|
+
Raises:
|
|
28
|
+
ValueError: When ``to_unit`` is not supported.
|
|
29
|
+
|
|
30
|
+
Example:
|
|
31
|
+
``convert_bytes(1073741824, "GB")`` returns ``1.0``.
|
|
32
|
+
"""
|
|
33
|
+
unit = to_unit.upper()
|
|
34
|
+
if unit not in _UNIT_EXPONENT:
|
|
35
|
+
supported = ", ".join(BINARY_UNITS)
|
|
36
|
+
raise ValueError(f"Invalid unit {to_unit!r}. Use one of: {supported}.")
|
|
37
|
+
|
|
38
|
+
exponent = _UNIT_EXPONENT[unit]
|
|
39
|
+
return float(size_bytes) / (BYTES_PER_UNIT**exponent)
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: xploreds
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Library and cookbook codes for Data Science Projects
|
|
5
|
+
Author-email: Daniel Dominguete <daniel.dominguete@gmail.com>
|
|
6
|
+
Requires-Python: <3.13,>=3.12
|
|
7
|
+
Requires-Dist: openpyxl>=3.1
|
|
8
|
+
Requires-Dist: pandas>=3.0.6
|
|
9
|
+
Requires-Dist: psutil>=6.0
|
|
10
|
+
Requires-Dist: pyarrow>=18.0
|
|
11
|
+
Requires-Dist: python-dotenv>=1.0
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# XploreDS
|
|
15
|
+
|
|
16
|
+
Library and cookbook codes for Data Science Projects.
|
|
17
|
+
|
|
18
|
+
## Estrutura
|
|
19
|
+
|
|
20
|
+
```
|
|
21
|
+
src/xploreds/ → biblioteca (publicavel no PyPI)
|
|
22
|
+
cookbook/ → scripts e receitas locais
|
|
23
|
+
tests/ → testes automatizados
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Environment Setup
|
|
27
|
+
|
|
28
|
+
Este projeto usa **[uv](https://docs.astral.sh/uv/)** para ambientes e dependencias.
|
|
29
|
+
Nao use `requirements.txt` nem `pip install` — tudo fica em `pyproject.toml` + `uv.lock`.
|
|
30
|
+
|
|
31
|
+
### 1. Instalar uv
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
### 2. Instalar dependencias (modo editavel)
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
uv sync --all-groups
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Isso cria `.venv/`, instala libs externas e registra `xploreds` em modo editavel apontando para `src/xploreds/`.
|
|
44
|
+
|
|
45
|
+
### 3. Hooks de commit (Commitizen)
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
uv run pre-commit install --hook-type commit-msg
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
### 4. Validar setup
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
uv run pytest
|
|
55
|
+
uv run python static/00_templates/script.py
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Versionamento semantico (Commitizen)
|
|
59
|
+
|
|
60
|
+
### Commits
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
git add .
|
|
64
|
+
uv run cz commit
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
| Tipo | Quando usar | Bump |
|
|
68
|
+
|------|-------------|------|
|
|
69
|
+
| `feat` | Nova funcionalidade | minor |
|
|
70
|
+
| `fix` | Correcao de bug | patch |
|
|
71
|
+
| `docs` | Documentacao | — |
|
|
72
|
+
| `refactor` | Refatoracao | patch |
|
|
73
|
+
| `test` | Testes | — |
|
|
74
|
+
| `chore` | Manutencao | — |
|
|
75
|
+
|
|
76
|
+
### Release e publicacao
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
uv run cz bump # bump de versao + CHANGELOG + tag
|
|
80
|
+
uv build # gera dist/*.whl
|
|
81
|
+
uv publish # publica no PyPI (quando configurado)
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Na primeira release (sem tags Git), use `--yes`:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
uv run cz bump --yes
|
|
88
|
+
```
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
xploreds/__init__.py,sha256=PmpIrQLq0HOqg2bY3hdcqVQ2OHJS4rktx7FjjIQ6U1I,176
|
|
2
|
+
xploreds/data_analysis/xds_drift.py,sha256=vCS3tKKHgB_DHWUY5R60yykQweIxCrnlo6-woR-eoTY,122
|
|
3
|
+
xploreds/data_analysis/xds_eda.py,sha256=TfQ-4JOCs9UmIYFncmfF8TA3gSbo78pakXqLoNkO7IQ,134
|
|
4
|
+
xploreds/data_analysis/xds_outliers.py,sha256=IE1m0HIp3VJ5yLqrvhvTtp1CZArgZMfEVn1Jdyn_lrI,116
|
|
5
|
+
xploreds/data_analysis/xds_quality.py,sha256=WQdTNVcSZ8Kx4AKEBiAzUXq-NmEjKCxiTNqv42b6Yzk,123
|
|
6
|
+
xploreds/data_analysis/xds_stats.py,sha256=cd3dEbA-jg7b_Y8u7bXHqaiIQIHxZNF_9umyK4TZCRE,133
|
|
7
|
+
xploreds/data_handle/xds_dataframe_pandas.py,sha256=GCuEJM2f3deuYP9PmIv9NTbpKiTRED7xtrVybfTP6tc,117
|
|
8
|
+
xploreds/data_transformation/xds_categorical_encoding.py,sha256=hOBmH3H69PEuW8eQ9OECnzLzRzMqbV52fBoRPLoP07E,130
|
|
9
|
+
xploreds/data_transformation/xds_datetime_encoding.py,sha256=UDA4dxYHy8KZDfmW-RPIXOnpaZeswg2Urlapje0zIdg,115
|
|
10
|
+
xploreds/data_transformation/xds_missing_inputation.py,sha256=dfe4A0aMBj9eZbmeEdQD4qwRAtmMgj7FBrSpDTw--S8,128
|
|
11
|
+
xploreds/data_transformation/xds_scaling.py,sha256=C6FMT7GPPTRhGJ9OD6VibrHS_cHO3AMafHij-eivY2M,102
|
|
12
|
+
xploreds/data_visualization/xds_plotly.py,sha256=VPAfsuvGgjTXI1a4wuUKMg5EQzMVDv8nYHQt8AoVXpc,121
|
|
13
|
+
xploreds/io/xds_file.py,sha256=P9TBYrQzIVh1yDPxWv3xOSOGiWoypaNEnaRFWYayuVE,7203
|
|
14
|
+
xploreds/log/xds_log.py,sha256=wwmui7LOKArJY1lTTfGuzlPNbKLjcKP6ATUbPaUy0TQ,11757
|
|
15
|
+
xploreds/models/xds_evaluate_model.py,sha256=vIzvmeaKMTff49Z8DSNejzLsl4serb6UJNBIQ5TID8o,123
|
|
16
|
+
xploreds/models/xds_linear_regression.py,sha256=02AN6Fdpy72of1xBod_-mlL3Sct-BuX9Spc5T1upKo4,127
|
|
17
|
+
xploreds/models/xds_logistic_regression.py,sha256=jNxSHnL0gn_BMI4V1UM-v8BvX7dcDyVC-G0qHtvOwOc,131
|
|
18
|
+
xploreds/time/xds_date_time.py,sha256=qRUs11O7zHUtqfih2bSVqhQfQhcx3LqS7WWNUi9wu8U,125
|
|
19
|
+
xploreds/utils/xds_memory.py,sha256=KPaJwb2yXhgRQ8kPKorbAJ4KKmAPvLqE7Ffj0_MQjmw,1119
|
|
20
|
+
xploreds-0.1.0.dist-info/METADATA,sha256=2Otkhou8OMz_3iu1fG6crXEb60WK8UyNC3TJeJBNxXM,1935
|
|
21
|
+
xploreds-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
22
|
+
xploreds-0.1.0.dist-info/RECORD,,
|