dq-engine 0.3.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dq-engine might be problematic. Click here for more details.
- dq_engine/__init__.py +6 -0
- dq_engine/config/__init__.py +6 -0
- dq_engine/config/conventions.py +164 -0
- dq_engine/execution/__init__.py +25 -0
- dq_engine/execution/engine.py +782 -0
- dq_engine/execution/exceptions.py +23 -0
- dq_engine/execution/outputs.py +166 -0
- dq_engine/execution/result.py +148 -0
- dq_engine/execution/rule_compiler.py +245 -0
- dq_engine/execution/treatment_plan.py +509 -0
- dq_engine/execution/validator.py +28 -0
- dq_engine/planning/__init__.py +15 -0
- dq_engine/planning/models.py +52 -0
- dq_engine/planning/resolver.py +72 -0
- dq_engine/registry/__init__.py +6 -0
- dq_engine/registry/registry.py +82 -0
- dq_engine/utils/__init__.py +3 -0
- dq_engine/utils/logging.py +37 -0
- dq_engine-0.3.14.dist-info/METADATA +307 -0
- dq_engine-0.3.14.dist-info/RECORD +22 -0
- dq_engine-0.3.14.dist-info/WHEEL +5 -0
- dq_engine-0.3.14.dist-info/top_level.txt +1 -0
dq_engine/__init__.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Convenções centrais do framework dq-engine.
|
|
3
|
+
|
|
4
|
+
Este módulo é a fonte única da verdade para:
|
|
5
|
+
- Valores canônicos de nulo por tipo de coluna
|
|
6
|
+
- Representações inválidas de nulo (lista única compartilhada por todos os tipos)
|
|
7
|
+
- Padrões regex nomeados
|
|
8
|
+
- Domínios de valores válidos
|
|
9
|
+
- Rule sets declarativos por nome
|
|
10
|
+
- Mapeamento prefixo de coluna → rule_set
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
CONVENTIONS: dict = {
|
|
16
|
+
# ------------------------------------------------------------------
|
|
17
|
+
# Valores canônicos de nulo por tipo de coluna
|
|
18
|
+
# ------------------------------------------------------------------
|
|
19
|
+
"null_values": {
|
|
20
|
+
"string": "N/D",
|
|
21
|
+
"cod": "0",
|
|
22
|
+
"numeric": 0,
|
|
23
|
+
"date": "1800-01-01",
|
|
24
|
+
},
|
|
25
|
+
|
|
26
|
+
# ------------------------------------------------------------------
|
|
27
|
+
# Representações inválidas de nulo — lista única, comparação UPPER+TRIM
|
|
28
|
+
# Obs: "N/D" é omitido — é o canônico de string e checado pontualmente
|
|
29
|
+
# no cod_null_pattern via condição hardcoded no rule builder.
|
|
30
|
+
# ------------------------------------------------------------------
|
|
31
|
+
"invalid_null_like": [
|
|
32
|
+
"0000-00-00",
|
|
33
|
+
"00/00/0000",
|
|
34
|
+
"99/99/9999",
|
|
35
|
+
"N/A",
|
|
36
|
+
"NA",
|
|
37
|
+
"ND",
|
|
38
|
+
"NONE",
|
|
39
|
+
"NULL",
|
|
40
|
+
"TESTE",
|
|
41
|
+
"VAZIO",
|
|
42
|
+
],
|
|
43
|
+
|
|
44
|
+
# ------------------------------------------------------------------
|
|
45
|
+
# Padrões regex nomeados
|
|
46
|
+
# ------------------------------------------------------------------
|
|
47
|
+
"patterns": {
|
|
48
|
+
"cpf": r"^\d{3}\.\d{3}\.\d{3}-\d{2}$",
|
|
49
|
+
"cnpj": r"^\d{2}\.\d{3}\.\d{3}/\d{4}-\d{2}$",
|
|
50
|
+
"cep": r"^\d{2}\.\d{3}-\d{3}$",
|
|
51
|
+
"telefone": r"^(\(\d{2}\)\s\d{5}-\d{4}|\d{2}\s\d{5}-\d{4})$",
|
|
52
|
+
},
|
|
53
|
+
|
|
54
|
+
# ------------------------------------------------------------------
|
|
55
|
+
# Domínios de valores válidos (enumerados)
|
|
56
|
+
# ------------------------------------------------------------------
|
|
57
|
+
"domains": {
|
|
58
|
+
"uf": [
|
|
59
|
+
"AC", "AL", "AP", "AM", "BA", "CE", "DF", "ES", "GO", "MA",
|
|
60
|
+
"MS", "MT", "MG", "PA", "PB", "PR", "PE", "PI", "RJ", "RN",
|
|
61
|
+
"RS", "RO", "RR", "SC", "SP", "SE", "TO",
|
|
62
|
+
],
|
|
63
|
+
},
|
|
64
|
+
|
|
65
|
+
# ------------------------------------------------------------------
|
|
66
|
+
# Rule sets declarativos — lista ordenada de regras por conjunto
|
|
67
|
+
# ------------------------------------------------------------------
|
|
68
|
+
"rule_sets": {
|
|
69
|
+
"string_default": [
|
|
70
|
+
{"type": "string_null_pattern"},
|
|
71
|
+
{"type": "trimmed"},
|
|
72
|
+
{"type": "no_test_values"},
|
|
73
|
+
],
|
|
74
|
+
"string_upper": [
|
|
75
|
+
{"type": "string_null_pattern"},
|
|
76
|
+
{"type": "trimmed"},
|
|
77
|
+
{"type": "upper"},
|
|
78
|
+
{"type": "no_test_values"},
|
|
79
|
+
],
|
|
80
|
+
|
|
81
|
+
"cod_default": [
|
|
82
|
+
{"type": "cod_null_pattern"},
|
|
83
|
+
],
|
|
84
|
+
|
|
85
|
+
"date_default": [
|
|
86
|
+
{"type": "date_null_pattern"},
|
|
87
|
+
],
|
|
88
|
+
"cpf_field": [
|
|
89
|
+
{"type": "string_null_pattern"},
|
|
90
|
+
{"type": "regex", "pattern_ref": "cpf"},
|
|
91
|
+
],
|
|
92
|
+
"cnpj_field": [
|
|
93
|
+
{"type": "string_null_pattern"},
|
|
94
|
+
{"type": "regex", "pattern_ref": "cnpj"},
|
|
95
|
+
],
|
|
96
|
+
"cep_field": [
|
|
97
|
+
{"type": "string_null_pattern"},
|
|
98
|
+
{"type": "regex", "pattern_ref": "cep"},
|
|
99
|
+
],
|
|
100
|
+
"uf_field": [
|
|
101
|
+
{"type": "string_null_pattern"},
|
|
102
|
+
{"type": "trimmed"},
|
|
103
|
+
{"type": "upper"},
|
|
104
|
+
{"type": "domain", "domain_ref": "uf"},
|
|
105
|
+
],
|
|
106
|
+
|
|
107
|
+
# Campos abertos (descrições livres): sem validação DQ, treatment apenas upper()
|
|
108
|
+
"dsc_default": [],
|
|
109
|
+
|
|
110
|
+
"val_default": [
|
|
111
|
+
{"type": "numeric_null_pattern"},
|
|
112
|
+
],
|
|
113
|
+
},
|
|
114
|
+
|
|
115
|
+
# ------------------------------------------------------------------
|
|
116
|
+
# Padrões de normalização — converte valores "sem pontuação" para o
|
|
117
|
+
# formato canônico. Indexado pelo mesmo nome-chave de 'patterns'.
|
|
118
|
+
# detect: regex que identifica valores ainda sem formatação.
|
|
119
|
+
# capture: grupos de captura para a substituição (Spark/Java).
|
|
120
|
+
# replacement: template de saída com backreferences ($1, $2...).
|
|
121
|
+
# ------------------------------------------------------------------
|
|
122
|
+
"normalization_patterns": {
|
|
123
|
+
"cpf": {
|
|
124
|
+
"detect": r"^\d{11}$",
|
|
125
|
+
"capture": r"(\d{3})(\d{3})(\d{3})(\d{2})",
|
|
126
|
+
"replacement": r"$1.$2.$3-$4",
|
|
127
|
+
},
|
|
128
|
+
"cnpj": {
|
|
129
|
+
"detect": r"^\d{14}$",
|
|
130
|
+
"capture": r"(\d{2})(\d{3})(\d{3})(\d{4})(\d{2})",
|
|
131
|
+
"replacement": r"$1.$2.$3/$4-$5",
|
|
132
|
+
},
|
|
133
|
+
"cep": {
|
|
134
|
+
"detect": r"^\d{8}$",
|
|
135
|
+
"capture": r"(\d{2})(\d{3})(\d{3})",
|
|
136
|
+
"replacement": r"$1.$2-$3",
|
|
137
|
+
},
|
|
138
|
+
},
|
|
139
|
+
|
|
140
|
+
# ------------------------------------------------------------------
|
|
141
|
+
# Mapeamento por substring — verificado antes do prefix_mapping
|
|
142
|
+
# Qualquer coluna cujo nome contenha a substring é mapeada para o rule_set.
|
|
143
|
+
# Exemplo: DSC_CPF_CLIENTE, COD_CPF_SEGURADO, DOC_CPF → cpf_field
|
|
144
|
+
# ------------------------------------------------------------------
|
|
145
|
+
"substring_mapping": {
|
|
146
|
+
"CNPJ": "cnpj_field",
|
|
147
|
+
"CPF": "cpf_field",
|
|
148
|
+
"UF": "uf_field",
|
|
149
|
+
},
|
|
150
|
+
|
|
151
|
+
# ------------------------------------------------------------------
|
|
152
|
+
# Mapeamento rule_set → [prefixos] — prefixos com o mesmo rule_set
|
|
153
|
+
# são agrupados. O resolver achata e ordena por comprimento (maior primeiro).
|
|
154
|
+
# ------------------------------------------------------------------
|
|
155
|
+
"prefix_mapping": {
|
|
156
|
+
"cep_field": ["END_CEP"],
|
|
157
|
+
"uf_field": ["COD_UF", "UF"],
|
|
158
|
+
"dsc_default": ["DSC"],
|
|
159
|
+
"string_upper": ["NOM", "TPO", "IDT", "NUM"],
|
|
160
|
+
"cod_default": ["COD"],
|
|
161
|
+
"date_default": ["DAT"],
|
|
162
|
+
"val_default": ["VAL"],
|
|
163
|
+
},
|
|
164
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Arquivo : __init__.py
|
|
2
|
+
# Caminho : data-quality-local/src/dq_engine/execution/__init__.py
|
|
3
|
+
|
|
4
|
+
from dq_engine.execution.engine import DataQualityEngine
|
|
5
|
+
from dq_engine.execution.result import DataQualityResult, compare_treatment_results
|
|
6
|
+
from dq_engine.execution.exceptions import (
|
|
7
|
+
DataQualityError,
|
|
8
|
+
RuleCompilationError,
|
|
9
|
+
InvalidRuleSetError,
|
|
10
|
+
InvalidConventionError,
|
|
11
|
+
MissingPatternRefError,
|
|
12
|
+
MissingDomainRefError,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"DataQualityEngine",
|
|
17
|
+
"DataQualityResult",
|
|
18
|
+
"compare_treatment_results",
|
|
19
|
+
"DataQualityError",
|
|
20
|
+
"RuleCompilationError",
|
|
21
|
+
"InvalidRuleSetError",
|
|
22
|
+
"InvalidConventionError",
|
|
23
|
+
"MissingPatternRefError",
|
|
24
|
+
"MissingDomainRefError",
|
|
25
|
+
]
|