iragi 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- iragi/__init__.py +14 -0
- iragi/core/config.py +59 -0
- iragi/core/exceptions.py +67 -0
- iragi/core/models.py +201 -0
- iragi/scan/__init__.py +36 -0
- iragi/scan/engine.py +241 -0
- iragi/scan/profiler.py +229 -0
- iragi/scan/readers.py +167 -0
- iragi/scan/reports.py +78 -0
- iragi-0.1.0.dist-info/METADATA +435 -0
- iragi-0.1.0.dist-info/RECORD +15 -0
- iragi-0.1.0.dist-info/WHEEL +5 -0
- iragi-0.1.0.dist-info/entry_points.txt +2 -0
- iragi-0.1.0.dist-info/licenses/LICENSE +21 -0
- iragi-0.1.0.dist-info/top_level.txt +1 -0
iragi/__init__.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Iragi: Modular File Inspection, Extraction, and Analytics Toolkit.
|
|
2
|
+
|
|
3
|
+
Unifies Scan, PDF, and orchestration pipelines for reliable data quality workflows.
|
|
4
|
+
|
|
5
|
+
Example:
|
|
6
|
+
>>> import iragi
|
|
7
|
+
>>> print(iragi.__version__)
|
|
8
|
+
'0.1.0'
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from iragi.scan import scan, scan_dataframe
|
|
12
|
+
|
|
13
|
+
__version__ = "0.1.0"
|
|
14
|
+
__all__ = ["__version__", "scan", "scan_dataframe"]
|
iragi/core/config.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Configuration management for Iragi.
|
|
2
|
+
|
|
3
|
+
Provides system-wide defaults, tunable quality weights, and persistent settings.
|
|
4
|
+
|
|
5
|
+
Example:
|
|
6
|
+
>>> from iragi.core.config import IragiConfig
|
|
7
|
+
>>> config = IragiConfig()
|
|
8
|
+
>>> config.outlier_iqr_multiplier
|
|
9
|
+
1.5
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class IragiConfig:
|
|
19
|
+
"""Runtime configuration options for Iragi analyses.
|
|
20
|
+
|
|
21
|
+
:param missing_penalty_weight: Weight deducted for missing cells ratio.
|
|
22
|
+
:type missing_penalty_weight: float
|
|
23
|
+
:param duplicate_penalty_weight: Weight deducted for duplicate rows ratio.
|
|
24
|
+
:type duplicate_penalty_weight: float
|
|
25
|
+
:param outlier_penalty_weight: Weight deducted for numerical outlier ratio.
|
|
26
|
+
:type outlier_penalty_weight: float
|
|
27
|
+
:param invalid_date_penalty_weight: Weight deducted for unparseable dates.
|
|
28
|
+
:type invalid_date_penalty_weight: float
|
|
29
|
+
:param outlier_iqr_multiplier: IQR multiplier used to detect statistical outliers.
|
|
30
|
+
:type outlier_iqr_multiplier: float
|
|
31
|
+
:param db_path: Path to the local SQLite history database.
|
|
32
|
+
:type db_path: Path
|
|
33
|
+
|
|
34
|
+
Example:
|
|
35
|
+
>>> cfg = IragiConfig(outlier_iqr_multiplier=2.0)
|
|
36
|
+
>>> cfg.outlier_iqr_multiplier
|
|
37
|
+
2.0
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
missing_penalty_weight: float = 35.0
|
|
41
|
+
duplicate_penalty_weight: float = 25.0
|
|
42
|
+
outlier_penalty_weight: float = 20.0
|
|
43
|
+
invalid_date_penalty_weight: float = 20.0
|
|
44
|
+
outlier_iqr_multiplier: float = 1.5
|
|
45
|
+
db_path: Path = Path(os.environ.get("IRAGI_DB_PATH", Path.home() / ".iragi" / "history.db"))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def get_default_config() -> IragiConfig:
|
|
49
|
+
"""Retrieve the standard system default configuration.
|
|
50
|
+
|
|
51
|
+
:return: An initialized default IragiConfig instance.
|
|
52
|
+
:rtype: IragiConfig
|
|
53
|
+
|
|
54
|
+
Example:
|
|
55
|
+
>>> cfg = get_default_config()
|
|
56
|
+
>>> isinstance(cfg, IragiConfig)
|
|
57
|
+
True
|
|
58
|
+
"""
|
|
59
|
+
return IragiConfig()
|
iragi/core/exceptions.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Core exceptions for Iragi.
|
|
2
|
+
|
|
3
|
+
This module defines domain-specific exceptions used across inspection,
|
|
4
|
+
cleaning, parsing, and reporting subsystems.
|
|
5
|
+
|
|
6
|
+
Example:
|
|
7
|
+
>>> from iragi.core.exceptions import IragiDataError
|
|
8
|
+
>>> raise IragiDataError("Invalid tabular data format")
|
|
9
|
+
Traceback (most recent call last):
|
|
10
|
+
...
|
|
11
|
+
iragi.core.exceptions.IragiDataError: Invalid tabular data format
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class IragiError(Exception):
|
|
16
|
+
"""Base exception for all Iragi errors.
|
|
17
|
+
|
|
18
|
+
Example:
|
|
19
|
+
>>> err = IragiError("Something failed")
|
|
20
|
+
>>> str(err)
|
|
21
|
+
'Something failed'
|
|
22
|
+
"""
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class IragiDataError(IragiError):
|
|
27
|
+
"""Raised when data inspection or cleaning fails.
|
|
28
|
+
|
|
29
|
+
Example:
|
|
30
|
+
>>> err = IragiDataError("Unsupported file type")
|
|
31
|
+
>>> str(err)
|
|
32
|
+
'Unsupported file type'
|
|
33
|
+
"""
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class IragiLogError(IragiError):
|
|
38
|
+
"""Raised when log parsing or analysis encounters an invalid state.
|
|
39
|
+
|
|
40
|
+
Example:
|
|
41
|
+
>>> err = IragiLogError("Failed to parse log line")
|
|
42
|
+
>>> str(err)
|
|
43
|
+
'Failed to parse log line'
|
|
44
|
+
"""
|
|
45
|
+
pass
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class IragiConfigError(IragiError):
|
|
49
|
+
"""Raised when configuration values are missing or invalid.
|
|
50
|
+
|
|
51
|
+
Example:
|
|
52
|
+
>>> err = IragiConfigError("Invalid scoring threshold")
|
|
53
|
+
>>> str(err)
|
|
54
|
+
'Invalid scoring threshold'
|
|
55
|
+
"""
|
|
56
|
+
pass
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class IragiReportError(IragiError):
|
|
60
|
+
"""Raised when generating a report fails.
|
|
61
|
+
|
|
62
|
+
Example:
|
|
63
|
+
>>> err = IragiReportError("Template rendering failed")
|
|
64
|
+
>>> str(err)
|
|
65
|
+
'Template rendering failed'
|
|
66
|
+
"""
|
|
67
|
+
pass
|
iragi/core/models.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""Core domain models for Iragi.
|
|
2
|
+
|
|
3
|
+
Provides structured representations for tabular inspection results,
|
|
4
|
+
individual column profiles, log analysis summaries, and quality metrics.
|
|
5
|
+
|
|
6
|
+
Example:
|
|
7
|
+
>>> from iragi.core.models import QualityScore, DataInspectionResult
|
|
8
|
+
>>> score = QualityScore(total_score=85.5, breakdown={"missing_penalty": 10.0})
|
|
9
|
+
>>> score.total_score
|
|
10
|
+
85.5
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from datetime import datetime
|
|
15
|
+
from typing import Any, Dict, List, Optional
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class QualityScore:
|
|
20
|
+
"""Represents an explainable data quality score.
|
|
21
|
+
|
|
22
|
+
:param total_score: The final computed quality score between 0 and 100.
|
|
23
|
+
:type total_score: float
|
|
24
|
+
:param breakdown: Penalty and bonus components contributing to the score.
|
|
25
|
+
:type breakdown: Dict[str, float]
|
|
26
|
+
:param explanations: Human-readable explanations for applied deductions.
|
|
27
|
+
:type explanations: List[str]
|
|
28
|
+
|
|
29
|
+
Example:
|
|
30
|
+
>>> qs = QualityScore(total_score=90.0, breakdown={"missing": -10.0}, explanations=["10% missing rows"])
|
|
31
|
+
>>> qs.total_score
|
|
32
|
+
90.0
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
total_score: float
|
|
36
|
+
breakdown: Dict[str, float] = field(default_factory=dict)
|
|
37
|
+
explanations: List[str] = field(default_factory=list)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class ColumnProfile:
|
|
42
|
+
"""Detailed profile metrics for a single tabular column.
|
|
43
|
+
|
|
44
|
+
:param name: Column header name.
|
|
45
|
+
:type name: str
|
|
46
|
+
:param inferred_type: Inferred data type (e.g. numeric, datetime, categorical).
|
|
47
|
+
:type inferred_type: str
|
|
48
|
+
:param total_count: Total number of values processed.
|
|
49
|
+
:type total_count: int
|
|
50
|
+
:param missing_count: Number of missing/null values.
|
|
51
|
+
:type missing_count: int
|
|
52
|
+
:param unique_count: Number of unique distinct values.
|
|
53
|
+
:type unique_count: int
|
|
54
|
+
:param invalid_dates_count: Count of unparseable date strings if column is date-like.
|
|
55
|
+
:type invalid_dates_count: int
|
|
56
|
+
:param outliers_count: Count of potential numerical outliers detected.
|
|
57
|
+
:type outliers_count: int
|
|
58
|
+
:param summary_stats: Descriptive numerical or categorical summary statistics.
|
|
59
|
+
:type summary_stats: Dict[str, Any]
|
|
60
|
+
|
|
61
|
+
Example:
|
|
62
|
+
>>> col = ColumnProfile("age", "numeric", 100, 5, 45, 0, 2, {"mean": 32.4})
|
|
63
|
+
>>> col.name
|
|
64
|
+
'age'
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
name: str
|
|
68
|
+
inferred_type: str
|
|
69
|
+
total_count: int
|
|
70
|
+
missing_count: int
|
|
71
|
+
unique_count: int
|
|
72
|
+
invalid_dates_count: int = 0
|
|
73
|
+
outliers_count: int = 0
|
|
74
|
+
summary_stats: Dict[str, Any] = field(default_factory=dict)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass
|
|
78
|
+
class DataInspectionResult:
|
|
79
|
+
"""Consolidated findings from tabular data inspection.
|
|
80
|
+
|
|
81
|
+
:param file_path: Source path of the analyzed file.
|
|
82
|
+
:type file_path: str
|
|
83
|
+
:param row_count: Total row count in tabular dataset.
|
|
84
|
+
:type row_count: int
|
|
85
|
+
:param column_count: Total column count.
|
|
86
|
+
:type column_count: int
|
|
87
|
+
:param missing_values_count: Aggregated count of missing cells.
|
|
88
|
+
:type missing_values_count: int
|
|
89
|
+
:param duplicate_rows_count: Number of duplicate rows detected.
|
|
90
|
+
:type duplicate_rows_count: int
|
|
91
|
+
:param invalid_dates_count: Total invalid date formats encountered.
|
|
92
|
+
:type invalid_dates_count: int
|
|
93
|
+
:param potential_outliers_count: Total potential statistical outliers.
|
|
94
|
+
:type potential_outliers_count: int
|
|
95
|
+
:param quality_score: Computed quality score with breakdown.
|
|
96
|
+
:type quality_score: QualityScore
|
|
97
|
+
:param column_profiles: Detailed profiles per column.
|
|
98
|
+
:type column_profiles: List[ColumnProfile]
|
|
99
|
+
:param recommendations: Actionable cleaning and quality improvement recommendations.
|
|
100
|
+
:type recommendations: List[str]
|
|
101
|
+
:param analyzed_at: Timestamp of inspection completion.
|
|
102
|
+
:type analyzed_at: datetime
|
|
103
|
+
|
|
104
|
+
Example:
|
|
105
|
+
>>> score = QualityScore(total_score=95.0)
|
|
106
|
+
>>> res = DataInspectionResult("data.csv", 100, 3, 2, 0, 0, 1, score)
|
|
107
|
+
>>> res.row_count
|
|
108
|
+
100
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
file_path: str
|
|
112
|
+
row_count: int
|
|
113
|
+
column_count: int
|
|
114
|
+
missing_values_count: int
|
|
115
|
+
duplicate_rows_count: int
|
|
116
|
+
invalid_dates_count: int
|
|
117
|
+
potential_outliers_count: int
|
|
118
|
+
quality_score: QualityScore
|
|
119
|
+
column_profiles: List[ColumnProfile] = field(default_factory=list)
|
|
120
|
+
recommendations: List[str] = field(default_factory=list)
|
|
121
|
+
analyzed_at: datetime = field(default_factory=datetime.now)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@dataclass
|
|
125
|
+
class LogEntry:
|
|
126
|
+
"""Structured representation of a parsed log record.
|
|
127
|
+
|
|
128
|
+
:param raw: Original unmodified raw log line.
|
|
129
|
+
:type raw: str
|
|
130
|
+
:param timestamp: Parsed timestamp if available.
|
|
131
|
+
:type timestamp: Optional[datetime]
|
|
132
|
+
:param level: Log level (e.g. INFO, WARN, ERROR, CRITICAL).
|
|
133
|
+
:type level: str
|
|
134
|
+
:param message: Extracted log message or request payload.
|
|
135
|
+
:type message: str
|
|
136
|
+
:param status_code: HTTP status code if present in the line.
|
|
137
|
+
:type status_code: Optional[int]
|
|
138
|
+
:param ip_address: Client IP address if present.
|
|
139
|
+
:type ip_address: Optional[str]
|
|
140
|
+
:param source: Originating parser or format identifier (e.g. nginx, django).
|
|
141
|
+
:type source: str
|
|
142
|
+
|
|
143
|
+
Example:
|
|
144
|
+
>>> entry = LogEntry("line 1", None, "ERROR", "Failed db connection", None, None, "django")
|
|
145
|
+
>>> entry.level
|
|
146
|
+
'ERROR'
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
raw: str
|
|
150
|
+
timestamp: Optional[datetime]
|
|
151
|
+
level: str
|
|
152
|
+
message: str
|
|
153
|
+
status_code: Optional[int] = None
|
|
154
|
+
ip_address: Optional[str] = None
|
|
155
|
+
source: str = "generic"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
@dataclass
|
|
159
|
+
class LogAnalysisResult:
|
|
160
|
+
"""Consolidated output from technical log inspection.
|
|
161
|
+
|
|
162
|
+
:param file_path: Path of the inspected log file.
|
|
163
|
+
:type file_path: str
|
|
164
|
+
:param total_lines: Total lines processed.
|
|
165
|
+
:type total_lines: int
|
|
166
|
+
:param parsed_lines: Number of successfully parsed log lines.
|
|
167
|
+
:type parsed_lines: int
|
|
168
|
+
:param unparsed_lines: Number of lines failing parser structure.
|
|
169
|
+
:type unparsed_lines: int
|
|
170
|
+
:param level_counts: Distribution of records by log level.
|
|
171
|
+
:type level_counts: Dict[str, int]
|
|
172
|
+
:param http_status_counts: Distribution of HTTP response status codes.
|
|
173
|
+
:type http_status_counts: Dict[int, int]
|
|
174
|
+
:param top_errors: Frequently repeated error messages and their occurrence counts.
|
|
175
|
+
:type top_errors: List[Dict[str, Any]]
|
|
176
|
+
:param unique_ips: List of distinct anonymized or extracted IP addresses.
|
|
177
|
+
:type unique_ips: List[str]
|
|
178
|
+
:param peak_error_periods: Time windows exhibiting highest error densities.
|
|
179
|
+
:type peak_error_periods: List[Dict[str, Any]]
|
|
180
|
+
:param recommendations: Diagnostics summary and operational tips.
|
|
181
|
+
:type recommendations: List[str]
|
|
182
|
+
:param analyzed_at: Timestamp of inspection.
|
|
183
|
+
:type analyzed_at: datetime
|
|
184
|
+
|
|
185
|
+
Example:
|
|
186
|
+
>>> res = LogAnalysisResult("app.log", 100, 98, 2, {"ERROR": 5}, {500: 2}, [], [], [])
|
|
187
|
+
>>> res.total_lines
|
|
188
|
+
100
|
|
189
|
+
"""
|
|
190
|
+
|
|
191
|
+
file_path: str
|
|
192
|
+
total_lines: int
|
|
193
|
+
parsed_lines: int
|
|
194
|
+
unparsed_lines: int
|
|
195
|
+
level_counts: Dict[str, int] = field(default_factory=dict)
|
|
196
|
+
http_status_counts: Dict[int, int] = field(default_factory=dict)
|
|
197
|
+
top_errors: List[Dict[str, Any]] = field(default_factory=list)
|
|
198
|
+
unique_ips: List[str] = field(default_factory=list)
|
|
199
|
+
peak_error_periods: List[Dict[str, Any]] = field(default_factory=list)
|
|
200
|
+
recommendations: List[str] = field(default_factory=list)
|
|
201
|
+
analyzed_at: datetime = field(default_factory=datetime.now)
|
iragi/scan/__init__.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Iragi Scan engine: Tabular data inspection, profiling, and quality auditing.
|
|
2
|
+
|
|
3
|
+
Provides tools for inspecting CSV, Excel, and JSON files, computing column
|
|
4
|
+
distributions, identifying anomalies, and calculating explainable quality scores.
|
|
5
|
+
|
|
6
|
+
Example:
|
|
7
|
+
>>> from iragi import scan
|
|
8
|
+
>>> import tempfile
|
|
9
|
+
>>> tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False)
|
|
10
|
+
>>> _ = tmp.write("col_a,col_b\\n1,2\\n3,4")
|
|
11
|
+
>>> tmp.close()
|
|
12
|
+
>>> res = scan(tmp.name)
|
|
13
|
+
>>> res.quality_score.total_score
|
|
14
|
+
100.0
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from iragi.scan.engine import scan, scan_dataframe, compute_quality_score, generate_recommendations
|
|
18
|
+
from iragi.scan.readers import load_tabular_file, read_csv_file, read_excel_file, read_json_file
|
|
19
|
+
from iragi.scan.profiler import profile_dataframe, infer_column_type, detect_outliers_iqr
|
|
20
|
+
from iragi.scan.reports import to_json_report, inspection_result_to_dict
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"scan",
|
|
24
|
+
"scan_dataframe",
|
|
25
|
+
"compute_quality_score",
|
|
26
|
+
"generate_recommendations",
|
|
27
|
+
"load_tabular_file",
|
|
28
|
+
"read_csv_file",
|
|
29
|
+
"read_excel_file",
|
|
30
|
+
"read_json_file",
|
|
31
|
+
"profile_dataframe",
|
|
32
|
+
"infer_column_type",
|
|
33
|
+
"detect_outliers_iqr",
|
|
34
|
+
"to_json_report",
|
|
35
|
+
"inspection_result_to_dict",
|
|
36
|
+
]
|
iragi/scan/engine.py
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
"""Core execution engine for Iragi Scan.
|
|
2
|
+
|
|
3
|
+
Coordinates file loading, column profiling, explainable quality scoring,
|
|
4
|
+
and actionable cleaning recommendations.
|
|
5
|
+
|
|
6
|
+
Example:
|
|
7
|
+
>>> from iragi.scan.engine import scan
|
|
8
|
+
>>> import tempfile
|
|
9
|
+
>>> tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False)
|
|
10
|
+
>>> _ = tmp.write("name,score\\nAlice,95\\nBob,80")
|
|
11
|
+
>>> tmp.close()
|
|
12
|
+
>>> res = scan(tmp.name)
|
|
13
|
+
>>> res.quality_score.total_score >= 90.0
|
|
14
|
+
True
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import List, Tuple, Union, Optional
|
|
19
|
+
from datetime import datetime
|
|
20
|
+
import pandas as pd
|
|
21
|
+
from iragi.core.config import IragiConfig, get_default_config
|
|
22
|
+
from iragi.core.models import QualityScore, DataInspectionResult
|
|
23
|
+
from iragi.scan.readers import load_tabular_file
|
|
24
|
+
from iragi.scan.profiler import profile_dataframe
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def compute_quality_score(
|
|
28
|
+
row_count: int,
|
|
29
|
+
column_count: int,
|
|
30
|
+
missing_count: int,
|
|
31
|
+
duplicate_rows: int,
|
|
32
|
+
outliers_count: int,
|
|
33
|
+
invalid_dates_count: int,
|
|
34
|
+
config: IragiConfig,
|
|
35
|
+
) -> QualityScore:
|
|
36
|
+
"""Calculate an explainable data quality score on a 0-100 scale.
|
|
37
|
+
|
|
38
|
+
:param row_count: Number of rows in dataset.
|
|
39
|
+
:type row_count: int
|
|
40
|
+
:param column_count: Number of columns.
|
|
41
|
+
:type column_count: int
|
|
42
|
+
:param missing_count: Total missing cell values.
|
|
43
|
+
:type missing_count: int
|
|
44
|
+
:param duplicate_rows: Number of duplicated rows.
|
|
45
|
+
:type duplicate_rows: int
|
|
46
|
+
:param outliers_count: Total potential statistical outliers.
|
|
47
|
+
:type outliers_count: int
|
|
48
|
+
:param invalid_dates_count: Total invalid date formats.
|
|
49
|
+
:type invalid_dates_count: int
|
|
50
|
+
:param config: Runtime scoring weights configuration.
|
|
51
|
+
:type config: IragiConfig
|
|
52
|
+
:return: QualityScore instance with exact deduction breakdown.
|
|
53
|
+
:rtype: QualityScore
|
|
54
|
+
|
|
55
|
+
Example:
|
|
56
|
+
>>> from iragi.core.config import get_default_config
|
|
57
|
+
>>> qs = compute_quality_score(100, 5, 0, 0, 0, 0, get_default_config())
|
|
58
|
+
>>> qs.total_score
|
|
59
|
+
100.0
|
|
60
|
+
"""
|
|
61
|
+
if row_count == 0 or column_count == 0:
|
|
62
|
+
return QualityScore(
|
|
63
|
+
total_score=0.0,
|
|
64
|
+
breakdown={"empty_dataset": 100.0},
|
|
65
|
+
explanations=["Dataset contains zero rows or zero columns."],
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
total_cells = row_count * column_count
|
|
69
|
+
missing_ratio = missing_count / total_cells if total_cells > 0 else 0.0
|
|
70
|
+
duplicate_ratio = duplicate_rows / row_count if row_count > 0 else 0.0
|
|
71
|
+
outlier_ratio = min(1.0, outliers_count / row_count) if row_count > 0 else 0.0
|
|
72
|
+
invalid_date_ratio = min(1.0, invalid_dates_count / row_count) if row_count > 0 else 0.0
|
|
73
|
+
|
|
74
|
+
missing_penalty = round(missing_ratio * config.missing_penalty_weight, 2)
|
|
75
|
+
duplicate_penalty = round(duplicate_ratio * config.duplicate_penalty_weight, 2)
|
|
76
|
+
outlier_penalty = round(outlier_ratio * config.outlier_penalty_weight, 2)
|
|
77
|
+
invalid_date_penalty = round(invalid_date_ratio * config.invalid_date_penalty_weight, 2)
|
|
78
|
+
|
|
79
|
+
total_deductions = missing_penalty + duplicate_penalty + outlier_penalty + invalid_date_penalty
|
|
80
|
+
final_score = max(0.0, round(100.0 - total_deductions, 2))
|
|
81
|
+
|
|
82
|
+
breakdown = {
|
|
83
|
+
"missing_penalty": missing_penalty,
|
|
84
|
+
"duplicate_penalty": duplicate_penalty,
|
|
85
|
+
"outlier_penalty": outlier_penalty,
|
|
86
|
+
"invalid_date_penalty": invalid_date_penalty,
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
explanations: List[str] = []
|
|
90
|
+
if missing_penalty > 0:
|
|
91
|
+
explanations.append(f"Missing values deducted {missing_penalty} pts ({missing_count} null cells).")
|
|
92
|
+
if duplicate_penalty > 0:
|
|
93
|
+
explanations.append(f"Duplicate rows deducted {duplicate_penalty} pts ({duplicate_rows} duplicate rows).")
|
|
94
|
+
if outlier_penalty > 0:
|
|
95
|
+
explanations.append(f"Outliers deducted {outlier_penalty} pts ({outliers_count} extreme values).")
|
|
96
|
+
if invalid_date_penalty > 0:
|
|
97
|
+
explanations.append(f"Invalid dates deducted {invalid_date_penalty} pts ({invalid_dates_count} invalid entries).")
|
|
98
|
+
|
|
99
|
+
if not explanations:
|
|
100
|
+
explanations.append("Excellent data hygiene: zero missing, duplicate, or outlier entries detected.")
|
|
101
|
+
|
|
102
|
+
return QualityScore(total_score=final_score, breakdown=breakdown, explanations=explanations)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def generate_recommendations(
|
|
106
|
+
missing_count: int,
|
|
107
|
+
duplicate_rows: int,
|
|
108
|
+
outliers_count: int,
|
|
109
|
+
invalid_dates_count: int,
|
|
110
|
+
) -> List[str]:
|
|
111
|
+
"""Generate prioritized, actionable cleaning recommendations.
|
|
112
|
+
|
|
113
|
+
:param missing_count: Total missing cell values.
|
|
114
|
+
:type missing_count: int
|
|
115
|
+
:param duplicate_rows: Number of duplicated rows.
|
|
116
|
+
:type duplicate_rows: int
|
|
117
|
+
:param outliers_count: Total potential statistical outliers.
|
|
118
|
+
:type outliers_count: int
|
|
119
|
+
:param invalid_dates_count: Total invalid date formats.
|
|
120
|
+
:type invalid_dates_count: int
|
|
121
|
+
:return: List of recommended cleaning operations.
|
|
122
|
+
:rtype: List[str]
|
|
123
|
+
|
|
124
|
+
Example:
|
|
125
|
+
>>> recs = generate_recommendations(10, 2, 0, 0)
|
|
126
|
+
>>> len(recs)
|
|
127
|
+
2
|
|
128
|
+
"""
|
|
129
|
+
recommendations: List[str] = []
|
|
130
|
+
|
|
131
|
+
if duplicate_rows > 0:
|
|
132
|
+
recommendations.append(
|
|
133
|
+
f"Deduplicate dataset: {duplicate_rows} identical rows detected. Apply row deduplication."
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
if missing_count > 0:
|
|
137
|
+
recommendations.append(
|
|
138
|
+
f"Handle missing values: {missing_count} null cells found. Consider imputation (mean/median) or dropping empty records."
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
if invalid_dates_count > 0:
|
|
142
|
+
recommendations.append(
|
|
143
|
+
f"Standardize date formats: {invalid_dates_count} date strings could not be parsed to ISO standard."
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
if outliers_count > 0:
|
|
147
|
+
recommendations.append(
|
|
148
|
+
f"Audit statistical outliers: {outliers_count} numerical values exceed 1.5x IQR boundaries."
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
if not recommendations:
|
|
152
|
+
recommendations.append("Dataset passes all quality checks. No immediate cleaning actions required.")
|
|
153
|
+
|
|
154
|
+
return recommendations
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def scan_dataframe(
|
|
158
|
+
df: pd.DataFrame,
|
|
159
|
+
source_name: str = "in_memory_dataframe",
|
|
160
|
+
config: Optional[IragiConfig] = None,
|
|
161
|
+
) -> DataInspectionResult:
|
|
162
|
+
"""Inspect and profile an in-memory pandas DataFrame.
|
|
163
|
+
|
|
164
|
+
:param df: Target pandas DataFrame.
|
|
165
|
+
:type df: pd.DataFrame
|
|
166
|
+
:param source_name: Identifier for the data source.
|
|
167
|
+
:type source_name: str
|
|
168
|
+
:param config: Optional custom configuration instance.
|
|
169
|
+
:type config: Optional[IragiConfig]
|
|
170
|
+
:return: Comprehensive DataInspectionResult.
|
|
171
|
+
:rtype: DataInspectionResult
|
|
172
|
+
|
|
173
|
+
Example:
|
|
174
|
+
>>> import pandas as pd
|
|
175
|
+
>>> df = pd.DataFrame({"col1": [1, 2, 3]})
|
|
176
|
+
>>> res = scan_dataframe(df)
|
|
177
|
+
>>> res.column_count
|
|
178
|
+
1
|
|
179
|
+
"""
|
|
180
|
+
cfg = config or get_default_config()
|
|
181
|
+
|
|
182
|
+
row_count = int(df.shape[0])
|
|
183
|
+
column_count = int(df.shape[1])
|
|
184
|
+
missing_values_count = int(df.isna().sum().sum())
|
|
185
|
+
|
|
186
|
+
column_profiles, duplicate_rows, outliers_count, invalid_dates = profile_dataframe(df, config=cfg)
|
|
187
|
+
|
|
188
|
+
quality_score = compute_quality_score(
|
|
189
|
+
row_count=row_count,
|
|
190
|
+
column_count=column_count,
|
|
191
|
+
missing_count=missing_values_count,
|
|
192
|
+
duplicate_rows=duplicate_rows,
|
|
193
|
+
outliers_count=outliers_count,
|
|
194
|
+
invalid_dates_count=invalid_dates,
|
|
195
|
+
config=cfg,
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
recommendations = generate_recommendations(
|
|
199
|
+
missing_count=missing_values_count,
|
|
200
|
+
duplicate_rows=duplicate_rows,
|
|
201
|
+
outliers_count=outliers_count,
|
|
202
|
+
invalid_dates_count=invalid_dates,
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
return DataInspectionResult(
|
|
206
|
+
file_path=source_name,
|
|
207
|
+
row_count=row_count,
|
|
208
|
+
column_count=column_count,
|
|
209
|
+
missing_values_count=missing_values_count,
|
|
210
|
+
duplicate_rows_count=duplicate_rows,
|
|
211
|
+
invalid_dates_count=invalid_dates,
|
|
212
|
+
potential_outliers_count=outliers_count,
|
|
213
|
+
quality_score=quality_score,
|
|
214
|
+
column_profiles=column_profiles,
|
|
215
|
+
recommendations=recommendations,
|
|
216
|
+
analyzed_at=datetime.now(),
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def scan(file_path: Union[str, Path], config: Optional[IragiConfig] = None) -> DataInspectionResult:
|
|
221
|
+
"""Inspect, profile, and evaluate the quality of a CSV, Excel, or JSON file.
|
|
222
|
+
|
|
223
|
+
:param file_path: Path to target file.
|
|
224
|
+
:type file_path: Union[str, Path]
|
|
225
|
+
:param config: Optional custom configuration instance.
|
|
226
|
+
:type config: Optional[IragiConfig]
|
|
227
|
+
:return: Comprehensive DataInspectionResult.
|
|
228
|
+
:rtype: DataInspectionResult
|
|
229
|
+
|
|
230
|
+
Example:
|
|
231
|
+
>>> import tempfile
|
|
232
|
+
>>> tmp = tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False)
|
|
233
|
+
>>> _ = tmp.write("id,value\\n1,100\\n2,200")
|
|
234
|
+
>>> tmp.close()
|
|
235
|
+
>>> res = scan(tmp.name)
|
|
236
|
+
>>> res.row_count
|
|
237
|
+
2
|
|
238
|
+
"""
|
|
239
|
+
path = Path(file_path)
|
|
240
|
+
df, _ = load_tabular_file(path)
|
|
241
|
+
return scan_dataframe(df, source_name=str(path), config=config)
|