nini-datagen 0.2.0__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nini_datagen/__init__.py +230 -0
- nini_datagen/_native.py +122 -0
- nini_datagen/_signatures.py +58 -0
- nini_datagen/api.py +187 -0
- nini_datagen/config/__init__.py +83 -0
- nini_datagen/config/address.py +78 -0
- nini_datagen/config/base.py +42 -0
- nini_datagen/config/business.py +37 -0
- nini_datagen/config/choice.py +30 -0
- nini_datagen/config/contact.py +79 -0
- nini_datagen/config/dirty.py +89 -0
- nini_datagen/config/documents.py +114 -0
- nini_datagen/config/network.py +33 -0
- nini_datagen/config/numbers.py +133 -0
- nini_datagen/config/person.py +81 -0
- nini_datagen/config/relations.py +91 -0
- nini_datagen/config/text.py +65 -0
- nini_datagen/core/__init__.py +1 -0
- nini_datagen/core/arrow.py +164 -0
- nini_datagen/core/blocks.py +84 -0
- nini_datagen/core/grammar.py +270 -0
- nini_datagen/core/rng.py +74 -0
- nini_datagen/core/tables.py +132 -0
- nini_datagen/core/views.py +56 -0
- nini_datagen/datagen.dll +0 -0
- nini_datagen/datasets/__init__.py +115 -0
- nini_datagen/datasets/manifest.json +10 -0
- nini_datagen/errors.py +11 -0
- nini_datagen/generator.py +325 -0
- nini_datagen/io/__init__.py +7 -0
- nini_datagen/io/parquet.py +143 -0
- nini_datagen/locales/__init__.py +46 -0
- nini_datagen/locales/base.py +177 -0
- nini_datagen/locales/pt_BR/__init__.py +228 -0
- nini_datagen/locales/pt_BR/data/cnefe_lite/addresses.parquet +0 -0
- nini_datagen/locales/pt_BR/data/cnefe_lite/districts.parquet +0 -0
- nini_datagen/locales/pt_BR/data/cnefe_lite/format.json +7 -0
- nini_datagen/locales/pt_BR/data/cnefe_lite/streets.parquet +0 -0
- nini_datagen/locales/pt_BR/data/first_names.tsv +1670 -0
- nini_datagen/locales/pt_BR/data/last_names.tsv +7357 -0
- nini_datagen/locales/pt_BR/data/municipalities.tsv +5572 -0
- nini_datagen/locales/pt_BR/grammar.py +575 -0
- nini_datagen/locales/pt_BR/lists.py +102 -0
- nini_datagen/providers/__init__.py +23 -0
- nini_datagen/providers/address.py +280 -0
- nini_datagen/providers/base.py +304 -0
- nini_datagen/providers/business.py +134 -0
- nini_datagen/providers/choice.py +117 -0
- nini_datagen/providers/contact.py +173 -0
- nini_datagen/providers/documents.py +186 -0
- nini_datagen/providers/network.py +106 -0
- nini_datagen/providers/numbers.py +133 -0
- nini_datagen/providers/person.py +370 -0
- nini_datagen/providers/relations.py +230 -0
- nini_datagen/providers/text.py +135 -0
- nini_datagen/py.typed +0 -0
- nini_datagen/registry.py +96 -0
- nini_datagen/schema/__init__.py +21 -0
- nini_datagen/schema/lazy.py +72 -0
- nini_datagen/schema/plan.py +279 -0
- nini_datagen/schema/spec.py +158 -0
- nini_datagen/transforms/__init__.py +8 -0
- nini_datagen/transforms/dirty.py +190 -0
- nini_datagen/transforms/duplicates.py +67 -0
- nini_datagen/validators.py +144 -0
- nini_datagen-0.2.0.dist-info/METADATA +277 -0
- nini_datagen-0.2.0.dist-info/RECORD +70 -0
- nini_datagen-0.2.0.dist-info/WHEEL +5 -0
- nini_datagen-0.2.0.dist-info/licenses/DATA_LICENSES.md +20 -0
- nini_datagen-0.2.0.dist-info/licenses/LICENSE +21 -0
nini_datagen/__init__.py
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""nini-datagen: dados sintéticos realistas do Brasil em escala, gerados por kernels C++ direto em Polars.
|
|
2
|
+
|
|
3
|
+
>>> import nini_datagen as dg
|
|
4
|
+
>>> dg.first_name(5, seed=42) # doctest: +SKIP
|
|
5
|
+
>>> g = dg.DataGen(seed=42)
|
|
6
|
+
>>> g.integer(1_000_000, low=1, high=6) # doctest: +SKIP
|
|
7
|
+
>>> g.choice(10, ["a", "b", "c"], weights=[5, 3, 2]) # doctest: +SKIP
|
|
8
|
+
|
|
9
|
+
Toda configuração é um modelo Pydantic (``dg.IntegerConfig(low=1, high=6)``); dicts e kwargs são convertidos para ele,
|
|
10
|
+
então um parâmetro com nome errado vira erro em vez de ser ignorado.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from importlib import metadata
|
|
16
|
+
|
|
17
|
+
from . import datasets, validators
|
|
18
|
+
from .api import (
|
|
19
|
+
address,
|
|
20
|
+
age,
|
|
21
|
+
birthdate,
|
|
22
|
+
boolean,
|
|
23
|
+
cep,
|
|
24
|
+
choice,
|
|
25
|
+
choice_by,
|
|
26
|
+
city,
|
|
27
|
+
cnh,
|
|
28
|
+
cnpj,
|
|
29
|
+
company,
|
|
30
|
+
correlated,
|
|
31
|
+
cpf,
|
|
32
|
+
credit_card,
|
|
33
|
+
date,
|
|
34
|
+
datetime,
|
|
35
|
+
ean13,
|
|
36
|
+
email,
|
|
37
|
+
first_name,
|
|
38
|
+
full_name,
|
|
39
|
+
generate,
|
|
40
|
+
iban,
|
|
41
|
+
integer,
|
|
42
|
+
ipv4,
|
|
43
|
+
ipv6,
|
|
44
|
+
last_name,
|
|
45
|
+
license_plate,
|
|
46
|
+
lognormal,
|
|
47
|
+
mac,
|
|
48
|
+
normal,
|
|
49
|
+
occupation,
|
|
50
|
+
paragraph,
|
|
51
|
+
phone,
|
|
52
|
+
pis,
|
|
53
|
+
rg,
|
|
54
|
+
scan,
|
|
55
|
+
sentence,
|
|
56
|
+
sex,
|
|
57
|
+
state,
|
|
58
|
+
table,
|
|
59
|
+
uniform,
|
|
60
|
+
url,
|
|
61
|
+
username,
|
|
62
|
+
uuid,
|
|
63
|
+
vehicle,
|
|
64
|
+
voter_id,
|
|
65
|
+
write_parquet_dataset,
|
|
66
|
+
)
|
|
67
|
+
from .config import (
|
|
68
|
+
AddressConfig,
|
|
69
|
+
AgeConfig,
|
|
70
|
+
BaseConfig,
|
|
71
|
+
BirthdateConfig,
|
|
72
|
+
BooleanConfig,
|
|
73
|
+
CepConfig,
|
|
74
|
+
ChoiceByConfig,
|
|
75
|
+
ChoiceConfig,
|
|
76
|
+
CityConfig,
|
|
77
|
+
CNHConfig,
|
|
78
|
+
CNPJConfig,
|
|
79
|
+
CompanyConfig,
|
|
80
|
+
CorrelatedConfig,
|
|
81
|
+
CPFConfig,
|
|
82
|
+
CreditCardConfig,
|
|
83
|
+
DateConfig,
|
|
84
|
+
DatetimeConfig,
|
|
85
|
+
EAN13Config,
|
|
86
|
+
EmailConfig,
|
|
87
|
+
FirstNameConfig,
|
|
88
|
+
FullNameConfig,
|
|
89
|
+
IBANConfig,
|
|
90
|
+
IntegerConfig,
|
|
91
|
+
IPv4Config,
|
|
92
|
+
IPv6Config,
|
|
93
|
+
LastNameConfig,
|
|
94
|
+
LicensePlateConfig,
|
|
95
|
+
LognormalConfig,
|
|
96
|
+
MACConfig,
|
|
97
|
+
NormalConfig,
|
|
98
|
+
OccupationConfig,
|
|
99
|
+
ParagraphConfig,
|
|
100
|
+
PhoneConfig,
|
|
101
|
+
PISConfig,
|
|
102
|
+
RGConfig,
|
|
103
|
+
SentenceConfig,
|
|
104
|
+
SexConfig,
|
|
105
|
+
StateConfig,
|
|
106
|
+
UniformConfig,
|
|
107
|
+
URLConfig,
|
|
108
|
+
UsernameConfig,
|
|
109
|
+
UUIDConfig,
|
|
110
|
+
VehicleConfig,
|
|
111
|
+
VoterIDConfig,
|
|
112
|
+
)
|
|
113
|
+
from .config.dirty import DirtyConfig
|
|
114
|
+
from .errors import DatagenError, NativeLibraryError
|
|
115
|
+
from .generator import DataGen
|
|
116
|
+
from .registry import registry
|
|
117
|
+
from .schema import Schema, column, ref
|
|
118
|
+
from .transforms import add_duplicates, dirty
|
|
119
|
+
|
|
120
|
+
try:
|
|
121
|
+
__version__ = metadata.version("nini-datagen")
|
|
122
|
+
except metadata.PackageNotFoundError: # rodando de uma cópia sem instalar
|
|
123
|
+
__version__ = "0.0.0"
|
|
124
|
+
|
|
125
|
+
__all__ = [
|
|
126
|
+
"AddressConfig",
|
|
127
|
+
"AgeConfig",
|
|
128
|
+
"BaseConfig",
|
|
129
|
+
"BirthdateConfig",
|
|
130
|
+
"BooleanConfig",
|
|
131
|
+
"CNHConfig",
|
|
132
|
+
"CNPJConfig",
|
|
133
|
+
"CPFConfig",
|
|
134
|
+
"CepConfig",
|
|
135
|
+
"ChoiceByConfig",
|
|
136
|
+
"ChoiceConfig",
|
|
137
|
+
"CityConfig",
|
|
138
|
+
"CompanyConfig",
|
|
139
|
+
"CorrelatedConfig",
|
|
140
|
+
"CreditCardConfig",
|
|
141
|
+
"DataGen",
|
|
142
|
+
"DatagenError",
|
|
143
|
+
"DateConfig",
|
|
144
|
+
"DatetimeConfig",
|
|
145
|
+
"DirtyConfig",
|
|
146
|
+
"EAN13Config",
|
|
147
|
+
"EmailConfig",
|
|
148
|
+
"FirstNameConfig",
|
|
149
|
+
"FullNameConfig",
|
|
150
|
+
"IBANConfig",
|
|
151
|
+
"IPv4Config",
|
|
152
|
+
"IPv6Config",
|
|
153
|
+
"IntegerConfig",
|
|
154
|
+
"LastNameConfig",
|
|
155
|
+
"LicensePlateConfig",
|
|
156
|
+
"LognormalConfig",
|
|
157
|
+
"MACConfig",
|
|
158
|
+
"NativeLibraryError",
|
|
159
|
+
"NormalConfig",
|
|
160
|
+
"OccupationConfig",
|
|
161
|
+
"PISConfig",
|
|
162
|
+
"ParagraphConfig",
|
|
163
|
+
"PhoneConfig",
|
|
164
|
+
"RGConfig",
|
|
165
|
+
"Schema",
|
|
166
|
+
"SentenceConfig",
|
|
167
|
+
"SexConfig",
|
|
168
|
+
"StateConfig",
|
|
169
|
+
"URLConfig",
|
|
170
|
+
"UUIDConfig",
|
|
171
|
+
"UniformConfig",
|
|
172
|
+
"UsernameConfig",
|
|
173
|
+
"VehicleConfig",
|
|
174
|
+
"VoterIDConfig",
|
|
175
|
+
"__version__",
|
|
176
|
+
"add_duplicates",
|
|
177
|
+
"address",
|
|
178
|
+
"age",
|
|
179
|
+
"birthdate",
|
|
180
|
+
"boolean",
|
|
181
|
+
"cep",
|
|
182
|
+
"choice",
|
|
183
|
+
"choice_by",
|
|
184
|
+
"city",
|
|
185
|
+
"cnh",
|
|
186
|
+
"cnpj",
|
|
187
|
+
"column",
|
|
188
|
+
"company",
|
|
189
|
+
"correlated",
|
|
190
|
+
"cpf",
|
|
191
|
+
"credit_card",
|
|
192
|
+
"datasets",
|
|
193
|
+
"date",
|
|
194
|
+
"datetime",
|
|
195
|
+
"dirty",
|
|
196
|
+
"ean13",
|
|
197
|
+
"email",
|
|
198
|
+
"first_name",
|
|
199
|
+
"full_name",
|
|
200
|
+
"generate",
|
|
201
|
+
"iban",
|
|
202
|
+
"integer",
|
|
203
|
+
"ipv4",
|
|
204
|
+
"ipv6",
|
|
205
|
+
"last_name",
|
|
206
|
+
"license_plate",
|
|
207
|
+
"lognormal",
|
|
208
|
+
"mac",
|
|
209
|
+
"normal",
|
|
210
|
+
"occupation",
|
|
211
|
+
"paragraph",
|
|
212
|
+
"phone",
|
|
213
|
+
"pis",
|
|
214
|
+
"ref",
|
|
215
|
+
"registry",
|
|
216
|
+
"rg",
|
|
217
|
+
"scan",
|
|
218
|
+
"sentence",
|
|
219
|
+
"sex",
|
|
220
|
+
"state",
|
|
221
|
+
"table",
|
|
222
|
+
"uniform",
|
|
223
|
+
"url",
|
|
224
|
+
"username",
|
|
225
|
+
"uuid",
|
|
226
|
+
"validators",
|
|
227
|
+
"vehicle",
|
|
228
|
+
"voter_id",
|
|
229
|
+
"write_parquet_dataset",
|
|
230
|
+
]
|
nini_datagen/_native.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Carrega a biblioteca C++ (libdatagen) via ctypes e aplica as assinaturas de `_signatures.py`.
|
|
2
|
+
|
|
3
|
+
Ordem de busca:
|
|
4
|
+
1. variável de ambiente ``NINI_DATAGEN_LIB`` (caminho do arquivo; útil para testar um build manual);
|
|
5
|
+
2. a pasta do pacote (instalação normal por wheel);
|
|
6
|
+
3. os arquivos registrados na instalação (instalação editável do ``uv sync``, em que o pacote aponta para ``src/`` e a
|
|
7
|
+
biblioteca compilada fica no site-packages).
|
|
8
|
+
|
|
9
|
+
O ctypes solta o GIL durante cada chamada, então os blocos rodam em paralelo de verdade numa ThreadPool.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import ctypes
|
|
15
|
+
import os
|
|
16
|
+
import sys
|
|
17
|
+
import threading
|
|
18
|
+
from importlib import metadata
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
import numpy as np
|
|
22
|
+
|
|
23
|
+
from ._signatures import ABI_VERSION, SIGNATURES
|
|
24
|
+
from .errors import NativeLibraryError
|
|
25
|
+
|
|
26
|
+
__all__ = ["ENV_VAR", "LIB_NAMES", "lib", "library_path", "ptr"]
|
|
27
|
+
|
|
28
|
+
ENV_VAR = "NINI_DATAGEN_LIB"
|
|
29
|
+
if sys.platform == "win32":
|
|
30
|
+
LIB_NAMES: tuple[str, ...] = ("datagen.dll",)
|
|
31
|
+
elif sys.platform == "darwin":
|
|
32
|
+
LIB_NAMES = ("libdatagen.dylib", "libdatagen.so")
|
|
33
|
+
else:
|
|
34
|
+
LIB_NAMES = ("libdatagen.so",)
|
|
35
|
+
|
|
36
|
+
_lock = threading.Lock()
|
|
37
|
+
_lib: ctypes.CDLL | None = None
|
|
38
|
+
_path: Path | None = None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _candidates() -> list[Path]:
|
|
42
|
+
env = os.environ.get(ENV_VAR)
|
|
43
|
+
if env:
|
|
44
|
+
return [Path(env)] # pedido explícito: não procura em outro lugar
|
|
45
|
+
here = Path(__file__).resolve().parent
|
|
46
|
+
found = [here / name for name in LIB_NAMES]
|
|
47
|
+
try:
|
|
48
|
+
files = metadata.files("nini-datagen") or []
|
|
49
|
+
except metadata.PackageNotFoundError:
|
|
50
|
+
files = []
|
|
51
|
+
found += [Path(str(f.locate())) for f in files if f.name in LIB_NAMES]
|
|
52
|
+
return found
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _load(path: Path) -> ctypes.CDLL:
|
|
56
|
+
try:
|
|
57
|
+
handle = ctypes.CDLL(str(path))
|
|
58
|
+
except OSError as e:
|
|
59
|
+
raise NativeLibraryError(f"não consegui carregar {path}: {e}") from e
|
|
60
|
+
try:
|
|
61
|
+
version = handle.dg_abi_version()
|
|
62
|
+
except AttributeError as e:
|
|
63
|
+
raise NativeLibraryError(f"{path} não é a biblioteca do nini-datagen (falta dg_abi_version)") from e
|
|
64
|
+
if version != ABI_VERSION:
|
|
65
|
+
raise NativeLibraryError(
|
|
66
|
+
f"{path} tem ABI {version}, mas o pacote Python espera {ABI_VERSION}: recompile"
|
|
67
|
+
" (uv sync --reinstall-package nini-datagen) ou reinstale o pacote"
|
|
68
|
+
)
|
|
69
|
+
missing = [name for name in SIGNATURES if not hasattr(handle, name)]
|
|
70
|
+
if missing:
|
|
71
|
+
raise NativeLibraryError(f"{path} não exporta {missing}")
|
|
72
|
+
for name, (restype, argtypes) in SIGNATURES.items():
|
|
73
|
+
fn = getattr(handle, name)
|
|
74
|
+
fn.restype = restype
|
|
75
|
+
fn.argtypes = argtypes
|
|
76
|
+
# inicializa todas as tabelas antes de qualquer thread (veja dg_init em cpp/tables.cpp)
|
|
77
|
+
handle.dg_init()
|
|
78
|
+
return handle
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def lib() -> ctypes.CDLL:
|
|
82
|
+
"""A biblioteca carregada (carrega na primeira chamada; seguro entre threads)."""
|
|
83
|
+
global _lib, _path
|
|
84
|
+
if _lib is not None:
|
|
85
|
+
return _lib
|
|
86
|
+
with _lock:
|
|
87
|
+
if _lib is None:
|
|
88
|
+
tried = []
|
|
89
|
+
for path in _candidates():
|
|
90
|
+
if path.is_file():
|
|
91
|
+
_lib, _path = _load(path), path
|
|
92
|
+
break
|
|
93
|
+
tried.append(str(path))
|
|
94
|
+
else:
|
|
95
|
+
raise NativeLibraryError(
|
|
96
|
+
"biblioteca C++ do nini-datagen não encontrada (procurei em: "
|
|
97
|
+
+ ", ".join(tried)
|
|
98
|
+
+ "). Numa cópia do repositório, rode `uv sync`; para um build manual, aponte "
|
|
99
|
+
+ f"{ENV_VAR} para o arquivo."
|
|
100
|
+
)
|
|
101
|
+
return _lib
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def library_path() -> Path:
|
|
105
|
+
"""Caminho do arquivo da biblioteca carregada."""
|
|
106
|
+
lib()
|
|
107
|
+
assert _path is not None
|
|
108
|
+
return _path
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _reset() -> None:
|
|
112
|
+
"""Só para testes: esquece a biblioteca carregada (a próxima chamada a `lib()` procura de novo)."""
|
|
113
|
+
global _lib, _path
|
|
114
|
+
with _lock:
|
|
115
|
+
_lib, _path = None, None
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def ptr(a: np.ndarray) -> int:
|
|
119
|
+
"""Endereço do primeiro elemento de um array contíguo (o C++ escreve/lê nele como um vetor plano)."""
|
|
120
|
+
if not a.flags.c_contiguous:
|
|
121
|
+
raise ValueError("o array precisa ser contíguo (C order) para ir ao C++")
|
|
122
|
+
return a.ctypes.data
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Tipos dos argumentos de cada função da biblioteca C++ (GERADO por tools/gen_bindings.py; não edite)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from ctypes import c_double, c_float, c_int, c_int64, c_uint8, c_uint16, c_uint32, c_uint64, c_void_p
|
|
5
|
+
|
|
6
|
+
ABI_VERSION = 10
|
|
7
|
+
|
|
8
|
+
# nome: (tipo de retorno, [tipos dos argumentos])
|
|
9
|
+
SIGNATURES = {
|
|
10
|
+
"dg_cep_views": (None, [c_void_p, c_uint32, c_void_p, c_int, c_int64, c_void_p]),
|
|
11
|
+
"dg_cep_views_uf": (None, [c_void_p, c_void_p, c_void_p, c_void_p, c_void_p, c_int, c_int64, c_void_p]),
|
|
12
|
+
"dg_empresa_views": (None, [c_void_p, c_uint32, c_void_p, c_void_p, c_void_p, c_uint32, c_void_p, c_void_p, c_void_p, c_uint32, c_void_p, c_void_p, c_void_p, c_uint32, c_void_p, c_void_p, c_void_p, c_int64, c_void_p, c_void_p]),
|
|
13
|
+
"dg_veiculo_idx": (None, [c_void_p, c_uint32, c_void_p, c_void_p, c_void_p, c_void_p, c_uint32, c_void_p, c_int64, c_void_p, c_void_p, c_void_p]),
|
|
14
|
+
"dg_cep_pick": (None, [c_void_p, c_uint32, c_void_p, c_void_p, c_int, c_int64, c_void_p]),
|
|
15
|
+
"dg_address_core": (None, [c_void_p, c_uint32, c_void_p, c_void_p, c_void_p, c_void_p, c_void_p, c_void_p, c_int64, c_void_p, c_void_p, c_void_p, c_void_p, c_void_p, c_void_p, c_void_p]),
|
|
16
|
+
"dg_house_number": (None, [c_void_p, c_void_p, c_int64, c_void_p]),
|
|
17
|
+
"dg_pick_in_group": (None, [c_void_p, c_void_p, c_int, c_void_p, c_void_p, c_void_p, c_int64, c_void_p]),
|
|
18
|
+
"dg_int_views": (None, [c_void_p, c_int64, c_void_p]),
|
|
19
|
+
"dg_concat_size": (c_int64, [c_int64, c_int, c_void_p, c_void_p]),
|
|
20
|
+
"dg_concat_views": (None, [c_int64, c_int, c_void_p, c_void_p, c_void_p, c_void_p, c_void_p, c_void_p]),
|
|
21
|
+
"dg_cpf_from_base": (None, [c_void_p, c_int64, c_int, c_void_p, c_void_p]),
|
|
22
|
+
"dg_cpf_perm": (None, [c_uint64, c_uint64, c_int64, c_int, c_void_p, c_void_p]),
|
|
23
|
+
"dg_cpf_random": (None, [c_void_p, c_int64, c_int, c_void_p, c_void_p]),
|
|
24
|
+
"dg_cnpj_gen": (None, [c_void_p, c_int, c_uint32, c_uint32, c_int, c_int, c_uint64, c_uint64, c_int64, c_void_p, c_void_p]),
|
|
25
|
+
"dg_id_views": (None, [c_void_p, c_int, c_int, c_int64, c_void_p, c_void_p, c_int]),
|
|
26
|
+
"dg_uuid_views": (None, [c_void_p, c_int64, c_void_p, c_void_p]),
|
|
27
|
+
"dg_ipv4_views": (None, [c_void_p, c_int, c_int64, c_void_p, c_void_p]),
|
|
28
|
+
"dg_url_views": (None, [c_void_p, c_uint32, c_void_p, c_void_p, c_uint32, c_void_p, c_void_p, c_void_p, c_int64, c_void_p, c_void_p]),
|
|
29
|
+
"dg_alias_build": (None, [c_void_p, c_uint32, c_void_p]),
|
|
30
|
+
"dg_bool_bits": (None, [c_void_p, c_uint64, c_int64, c_void_p]),
|
|
31
|
+
"dg_rand_i64": (None, [c_void_p, c_int64, c_uint64, c_int64, c_void_p]),
|
|
32
|
+
"dg_rand_i32": (None, [c_void_p, c_int, c_uint32, c_int64, c_void_p]),
|
|
33
|
+
"dg_rand_f64": (None, [c_void_p, c_double, c_double, c_int64, c_void_p]),
|
|
34
|
+
"dg_rand_f32": (None, [c_void_p, c_float, c_float, c_int64, c_void_p]),
|
|
35
|
+
"dg_rand_normal": (None, [c_void_p, c_double, c_double, c_int, c_int64, c_void_p]),
|
|
36
|
+
"dg_correlated": (None, [c_void_p, c_void_p, c_double, c_double, c_double, c_double, c_double, c_int, c_int64, c_void_p]),
|
|
37
|
+
"dg_birthdate": (None, [c_void_p, c_void_p, c_void_p, c_void_p, c_int64, c_void_p]),
|
|
38
|
+
"dg_det_exp": (c_double, [c_double]),
|
|
39
|
+
"dg_alias_idx": (None, [c_uint32, c_void_p, c_void_p, c_int64, c_void_p]),
|
|
40
|
+
"dg_choice_views": (None, [c_void_p, c_uint32, c_void_p, c_int64, c_void_p]),
|
|
41
|
+
"dg_choice_alias_views": (None, [c_void_p, c_uint32, c_void_p, c_void_p, c_int64, c_void_p]),
|
|
42
|
+
"dg_choice_alias_views_by": (None, [c_void_p, c_uint32, c_uint32, c_void_p, c_void_p, c_void_p, c_int64, c_void_p]),
|
|
43
|
+
"dg_phone_views": (None, [c_void_p, c_uint32, c_void_p, c_void_p, c_uint64, c_int, c_int64, c_void_p, c_void_p, c_void_p]),
|
|
44
|
+
"dg_email_views": (None, [c_void_p, c_uint32, c_void_p, c_void_p, c_uint32, c_void_p, c_void_p, c_void_p, c_void_p, c_int, c_uint32, c_void_p, c_void_p, c_void_p, c_int64, c_void_p, c_void_p, c_void_p, c_void_p, c_void_p]),
|
|
45
|
+
"dg_gather_views": (None, [c_void_p, c_void_p, c_int64, c_void_p]),
|
|
46
|
+
"dg_alias_idx_by": (None, [c_uint32, c_uint32, c_void_p, c_void_p, c_void_p, c_int64, c_void_p]),
|
|
47
|
+
"dg_join_size": (c_int64, [c_int64, c_int, c_void_p, c_void_p, c_uint32]),
|
|
48
|
+
"dg_join_views": (None, [c_int64, c_int, c_void_p, c_void_p, c_void_p, c_void_p, c_uint32, c_uint32, c_void_p, c_void_p]),
|
|
49
|
+
"dg_sex_mask": (c_int64, [c_void_p, c_void_p, c_int64, c_int64, c_void_p]),
|
|
50
|
+
"dg_xoshiro_streams": (None, [c_void_p, c_int64, c_int64, c_void_p]),
|
|
51
|
+
"dg_views_used": (c_int64, [c_void_p, c_int64, c_int]),
|
|
52
|
+
"dg_init": (c_int, []),
|
|
53
|
+
"dg_abi_version": (c_int, []),
|
|
54
|
+
"dg_text_views": (None, [c_void_p, c_uint32, c_void_p, c_void_p, c_void_p, c_uint32, c_uint32, c_uint32, c_uint32, c_int64, c_void_p, c_void_p]),
|
|
55
|
+
"dg_template_views": (c_int64, [c_void_p, c_void_p, c_void_p, c_void_p, c_void_p, c_void_p, c_void_p, c_void_p, c_uint32, c_uint32, c_int, c_int64, c_int64, c_int64, c_void_p, c_void_p]),
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
__all__ = ['ABI_VERSION', 'SIGNATURES', 'c_double', 'c_float', 'c_int', 'c_int64', 'c_uint8', 'c_uint16', 'c_uint32', 'c_uint64', 'c_void_p']
|
nini_datagen/api.py
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Funções soltas: ``nini_datagen.integer(n, ...)`` é um atalho para ``DataGen(seed, ...).integer(n, ...)``.
|
|
2
|
+
|
|
3
|
+
Sem ``seed``, cada chamada sorteia uma semente nova (valores diferentes a cada vez). Com ``seed``, o resultado é sempre
|
|
4
|
+
o mesmo.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Mapping
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import polars as pl
|
|
14
|
+
|
|
15
|
+
from .generator import DataGen
|
|
16
|
+
from .locales import Locale
|
|
17
|
+
from .schema import Schema
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"address",
|
|
21
|
+
"age",
|
|
22
|
+
"birthdate",
|
|
23
|
+
"boolean",
|
|
24
|
+
"cep",
|
|
25
|
+
"choice",
|
|
26
|
+
"choice_by",
|
|
27
|
+
"city",
|
|
28
|
+
"cnh",
|
|
29
|
+
"cnpj",
|
|
30
|
+
"company",
|
|
31
|
+
"correlated",
|
|
32
|
+
"cpf",
|
|
33
|
+
"credit_card",
|
|
34
|
+
"date",
|
|
35
|
+
"datetime",
|
|
36
|
+
"ean13",
|
|
37
|
+
"email",
|
|
38
|
+
"first_name",
|
|
39
|
+
"full_name",
|
|
40
|
+
"generate",
|
|
41
|
+
"iban",
|
|
42
|
+
"integer",
|
|
43
|
+
"ipv4",
|
|
44
|
+
"ipv6",
|
|
45
|
+
"last_name",
|
|
46
|
+
"license_plate",
|
|
47
|
+
"lognormal",
|
|
48
|
+
"mac",
|
|
49
|
+
"normal",
|
|
50
|
+
"occupation",
|
|
51
|
+
"paragraph",
|
|
52
|
+
"phone",
|
|
53
|
+
"pis",
|
|
54
|
+
"rg",
|
|
55
|
+
"scan",
|
|
56
|
+
"sentence",
|
|
57
|
+
"sex",
|
|
58
|
+
"state",
|
|
59
|
+
"table",
|
|
60
|
+
"uniform",
|
|
61
|
+
"url",
|
|
62
|
+
"username",
|
|
63
|
+
"uuid",
|
|
64
|
+
"vehicle",
|
|
65
|
+
"voter_id",
|
|
66
|
+
"write_parquet_dataset",
|
|
67
|
+
]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def generate(
|
|
71
|
+
kind: str,
|
|
72
|
+
n: int,
|
|
73
|
+
config: Any = None,
|
|
74
|
+
/,
|
|
75
|
+
*,
|
|
76
|
+
seed: int | None = None,
|
|
77
|
+
locale: str | Locale = "pt_BR",
|
|
78
|
+
threads: int | None = None,
|
|
79
|
+
**kwargs: Any,
|
|
80
|
+
) -> pl.Series:
|
|
81
|
+
"""Gera ``n`` valores do tipo ``kind``; veja `DataGen.generate`."""
|
|
82
|
+
return DataGen(seed, locale=locale, threads=threads).generate(kind, n, config, **kwargs)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def table(
|
|
86
|
+
n: int,
|
|
87
|
+
schema: Schema | Mapping[str, Any],
|
|
88
|
+
*,
|
|
89
|
+
seed: int | None = None,
|
|
90
|
+
locale: str | Locale = "pt_BR",
|
|
91
|
+
threads: int | None = None,
|
|
92
|
+
) -> pl.DataFrame:
|
|
93
|
+
"""Tabela de ``n`` linhas; veja `DataGen.table`."""
|
|
94
|
+
return DataGen(seed, locale=locale, threads=threads).table(n, schema)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def scan(
|
|
98
|
+
n: int,
|
|
99
|
+
schema: Schema | Mapping[str, Any],
|
|
100
|
+
*,
|
|
101
|
+
seed: int | None = None,
|
|
102
|
+
locale: str | Locale = "pt_BR",
|
|
103
|
+
threads: int | None = None,
|
|
104
|
+
batch_size: int | None = None,
|
|
105
|
+
) -> pl.LazyFrame:
|
|
106
|
+
"""Tabela como ``pl.LazyFrame``, gerada sob demanda; veja `DataGen.scan`.
|
|
107
|
+
|
|
108
|
+
Sem ``seed``, a semente é sorteada uma vez aqui: coletar o mesmo LazyFrame duas vezes dá a mesma tabela.
|
|
109
|
+
"""
|
|
110
|
+
return DataGen(seed, locale=locale, threads=threads).scan(n, schema, batch_size=batch_size)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def write_parquet_dataset(
|
|
114
|
+
path: str | Path,
|
|
115
|
+
n: int,
|
|
116
|
+
schema: Schema | Mapping[str, Any],
|
|
117
|
+
*,
|
|
118
|
+
seed: int | None = None,
|
|
119
|
+
locale: str | Locale = "pt_BR",
|
|
120
|
+
threads: int | None = None,
|
|
121
|
+
**options: Any,
|
|
122
|
+
) -> list[Path]:
|
|
123
|
+
"""Grava ``n`` linhas em Parquet, em lotes; veja `DataGen.write_parquet_dataset` (as opções são as mesmas)."""
|
|
124
|
+
return DataGen(seed, locale=locale, threads=threads).write_parquet_dataset(path, n, schema, **options)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _shortcut(kind: str, doc: str) -> Any:
|
|
128
|
+
def fn(
|
|
129
|
+
n: int,
|
|
130
|
+
config: Any = None,
|
|
131
|
+
/,
|
|
132
|
+
*,
|
|
133
|
+
seed: int | None = None,
|
|
134
|
+
locale: str | Locale = "pt_BR",
|
|
135
|
+
threads: int | None = None,
|
|
136
|
+
**kwargs: Any,
|
|
137
|
+
) -> pl.Series:
|
|
138
|
+
return generate(kind, n, config, seed=seed, locale=locale, threads=threads, **kwargs)
|
|
139
|
+
|
|
140
|
+
fn.__name__ = fn.__qualname__ = kind
|
|
141
|
+
fn.__doc__ = doc + "\n\n``seed``, ``locale`` e ``threads`` como em `DataGen`; o resto vai para a configuração."
|
|
142
|
+
return fn
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
integer = _shortcut("integer", DataGen.integer.__doc__ or "")
|
|
146
|
+
uniform = _shortcut("uniform", DataGen.uniform.__doc__ or "")
|
|
147
|
+
normal = _shortcut("normal", DataGen.normal.__doc__ or "")
|
|
148
|
+
lognormal = _shortcut("lognormal", DataGen.lognormal.__doc__ or "")
|
|
149
|
+
boolean = _shortcut("boolean", DataGen.boolean.__doc__ or "")
|
|
150
|
+
date = _shortcut("date", DataGen.date.__doc__ or "")
|
|
151
|
+
datetime = _shortcut("datetime", DataGen.datetime.__doc__ or "")
|
|
152
|
+
choice = _shortcut("choice", DataGen.choice.__doc__ or "")
|
|
153
|
+
sex = _shortcut("sex", DataGen.sex.__doc__ or "")
|
|
154
|
+
age = _shortcut("age", DataGen.age.__doc__ or "")
|
|
155
|
+
first_name = _shortcut("first_name", DataGen.first_name.__doc__ or "")
|
|
156
|
+
last_name = _shortcut("last_name", DataGen.last_name.__doc__ or "")
|
|
157
|
+
full_name = _shortcut("full_name", DataGen.full_name.__doc__ or "")
|
|
158
|
+
cpf = _shortcut("cpf", DataGen.cpf.__doc__ or "")
|
|
159
|
+
cnpj = _shortcut("cnpj", DataGen.cnpj.__doc__ or "")
|
|
160
|
+
pis = _shortcut("pis", DataGen.pis.__doc__ or "")
|
|
161
|
+
voter_id = _shortcut("voter_id", DataGen.voter_id.__doc__ or "")
|
|
162
|
+
cnh = _shortcut("cnh", DataGen.cnh.__doc__ or "")
|
|
163
|
+
rg = _shortcut("rg", DataGen.rg.__doc__ or "")
|
|
164
|
+
license_plate = _shortcut("license_plate", DataGen.license_plate.__doc__ or "")
|
|
165
|
+
credit_card = _shortcut("credit_card", DataGen.credit_card.__doc__ or "")
|
|
166
|
+
iban = _shortcut("iban", DataGen.iban.__doc__ or "")
|
|
167
|
+
ean13 = _shortcut("ean13", DataGen.ean13.__doc__ or "")
|
|
168
|
+
uuid = _shortcut("uuid", DataGen.uuid.__doc__ or "")
|
|
169
|
+
ipv4 = _shortcut("ipv4", DataGen.ipv4.__doc__ or "")
|
|
170
|
+
ipv6 = _shortcut("ipv6", DataGen.ipv6.__doc__ or "")
|
|
171
|
+
mac = _shortcut("mac", DataGen.mac.__doc__ or "")
|
|
172
|
+
url = _shortcut("url", DataGen.url.__doc__ or "")
|
|
173
|
+
phone = _shortcut("phone", DataGen.phone.__doc__ or "")
|
|
174
|
+
email = _shortcut("email", DataGen.email.__doc__ or "")
|
|
175
|
+
username = _shortcut("username", DataGen.username.__doc__ or "")
|
|
176
|
+
company = _shortcut("company", DataGen.company.__doc__ or "")
|
|
177
|
+
vehicle = _shortcut("vehicle", DataGen.vehicle.__doc__ or "")
|
|
178
|
+
occupation = _shortcut("occupation", DataGen.occupation.__doc__ or "")
|
|
179
|
+
sentence = _shortcut("sentence", DataGen.sentence.__doc__ or "")
|
|
180
|
+
paragraph = _shortcut("paragraph", DataGen.paragraph.__doc__ or "")
|
|
181
|
+
cep = _shortcut("cep", DataGen.cep.__doc__ or "")
|
|
182
|
+
address = _shortcut("address", DataGen.address.__doc__ or "")
|
|
183
|
+
city = _shortcut("city", DataGen.city.__doc__ or "")
|
|
184
|
+
state = _shortcut("state", DataGen.state.__doc__ or "")
|
|
185
|
+
birthdate = _shortcut("birthdate", DataGen.birthdate.__doc__ or "")
|
|
186
|
+
correlated = _shortcut("correlated", DataGen.correlated.__doc__ or "")
|
|
187
|
+
choice_by = _shortcut("choice_by", DataGen.choice_by.__doc__ or "")
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Configurações dos geradores (modelos Pydantic imutáveis; parâmetro com nome errado vira erro)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .address import AddressConfig, CepConfig, CityConfig, StateConfig
|
|
6
|
+
from .base import BaseConfig
|
|
7
|
+
from .business import CompanyConfig, OccupationConfig, VehicleConfig
|
|
8
|
+
from .choice import ChoiceConfig
|
|
9
|
+
from .contact import EmailConfig, PhoneConfig, UsernameConfig
|
|
10
|
+
from .dirty import DirtyConfig
|
|
11
|
+
from .documents import (
|
|
12
|
+
CNHConfig,
|
|
13
|
+
CNPJConfig,
|
|
14
|
+
CPFConfig,
|
|
15
|
+
CreditCardConfig,
|
|
16
|
+
EAN13Config,
|
|
17
|
+
IBANConfig,
|
|
18
|
+
LicensePlateConfig,
|
|
19
|
+
PISConfig,
|
|
20
|
+
RGConfig,
|
|
21
|
+
VoterIDConfig,
|
|
22
|
+
)
|
|
23
|
+
from .network import IPv4Config, IPv6Config, MACConfig, URLConfig, UUIDConfig
|
|
24
|
+
from .numbers import (
|
|
25
|
+
BooleanConfig,
|
|
26
|
+
DateConfig,
|
|
27
|
+
DatetimeConfig,
|
|
28
|
+
IntegerConfig,
|
|
29
|
+
LognormalConfig,
|
|
30
|
+
NormalConfig,
|
|
31
|
+
UniformConfig,
|
|
32
|
+
)
|
|
33
|
+
from .person import AgeConfig, FirstNameConfig, FullNameConfig, LastNameConfig, SexConfig
|
|
34
|
+
from .relations import BirthdateConfig, ChoiceByConfig, CorrelatedConfig
|
|
35
|
+
from .text import ParagraphConfig, SentenceConfig
|
|
36
|
+
|
|
37
|
+
__all__ = [
|
|
38
|
+
"AddressConfig",
|
|
39
|
+
"AgeConfig",
|
|
40
|
+
"BaseConfig",
|
|
41
|
+
"BirthdateConfig",
|
|
42
|
+
"BooleanConfig",
|
|
43
|
+
"CNHConfig",
|
|
44
|
+
"CNPJConfig",
|
|
45
|
+
"CPFConfig",
|
|
46
|
+
"CepConfig",
|
|
47
|
+
"ChoiceByConfig",
|
|
48
|
+
"ChoiceConfig",
|
|
49
|
+
"CityConfig",
|
|
50
|
+
"CompanyConfig",
|
|
51
|
+
"CorrelatedConfig",
|
|
52
|
+
"CreditCardConfig",
|
|
53
|
+
"DateConfig",
|
|
54
|
+
"DatetimeConfig",
|
|
55
|
+
"DirtyConfig",
|
|
56
|
+
"EAN13Config",
|
|
57
|
+
"EmailConfig",
|
|
58
|
+
"FirstNameConfig",
|
|
59
|
+
"FullNameConfig",
|
|
60
|
+
"IBANConfig",
|
|
61
|
+
"IPv4Config",
|
|
62
|
+
"IPv6Config",
|
|
63
|
+
"IntegerConfig",
|
|
64
|
+
"LastNameConfig",
|
|
65
|
+
"LicensePlateConfig",
|
|
66
|
+
"LognormalConfig",
|
|
67
|
+
"MACConfig",
|
|
68
|
+
"NormalConfig",
|
|
69
|
+
"OccupationConfig",
|
|
70
|
+
"PISConfig",
|
|
71
|
+
"ParagraphConfig",
|
|
72
|
+
"PhoneConfig",
|
|
73
|
+
"RGConfig",
|
|
74
|
+
"SentenceConfig",
|
|
75
|
+
"SexConfig",
|
|
76
|
+
"StateConfig",
|
|
77
|
+
"URLConfig",
|
|
78
|
+
"UUIDConfig",
|
|
79
|
+
"UniformConfig",
|
|
80
|
+
"UsernameConfig",
|
|
81
|
+
"VehicleConfig",
|
|
82
|
+
"VoterIDConfig",
|
|
83
|
+
]
|