dtex 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dtex/__init__.py +129 -0
- dtex/cli/__init__.py +1255 -0
- dtex/cli/_discovery.py +141 -0
- dtex/cli/_format.py +216 -0
- dtex/cli/_runs.py +301 -0
- dtex/cli/_scaffold.py +383 -0
- dtex/cli/_secrets.py +262 -0
- dtex/cli/_state.py +202 -0
- dtex/destinations/__init__.py +2 -0
- dtex/destinations/bigquery/README.md +162 -0
- dtex/destinations/bigquery/__init__.py +12 -0
- dtex/destinations/bigquery/client.py +324 -0
- dtex/destinations/bigquery/ddl.py +450 -0
- dtex/destinations/bigquery/destination.py +1140 -0
- dtex/destinations/bigquery/register.yaml +72 -0
- dtex/destinations/duckdb/__init__.py +11 -0
- dtex/destinations/duckdb/ddl.py +140 -0
- dtex/destinations/duckdb/destination.py +856 -0
- dtex/destinations/duckdb/register.yaml +27 -0
- dtex/engine/__init__.py +53 -0
- dtex/engine/config.py +672 -0
- dtex/engine/configs.py +151 -0
- dtex/engine/discovery.py +411 -0
- dtex/engine/logger.py +315 -0
- dtex/engine/runner.py +1469 -0
- dtex/py.typed +0 -0
- dtex/registry.py +834 -0
- dtex/secrets/__init__.py +65 -0
- dtex/secrets/_aws.py +432 -0
- dtex/secrets/_gcp.py +312 -0
- dtex/secrets/_vault.py +337 -0
- dtex/secrets/resolvers.py +687 -0
- dtex/sources/__init__.py +2 -0
- dtex/sources/filesystem/README.md +70 -0
- dtex/sources/filesystem/__init__.py +10 -0
- dtex/sources/filesystem/backends.py +437 -0
- dtex/sources/filesystem/readers.py +260 -0
- dtex/sources/filesystem/register.yaml +124 -0
- dtex/sources/filesystem/source.py +236 -0
- dtex/sources/postgres/README.md +109 -0
- dtex/sources/postgres/__init__.py +12 -0
- dtex/sources/postgres/client.py +256 -0
- dtex/sources/postgres/register.yaml +99 -0
- dtex/sources/postgres/source.py +381 -0
- dtex/sources/postgres/type_mapping.py +178 -0
- dtex/sources/rest/README.md +133 -0
- dtex/sources/rest/__init__.py +15 -0
- dtex/sources/rest/client.py +271 -0
- dtex/sources/rest/extractors.py +207 -0
- dtex/sources/rest/pagination.py +469 -0
- dtex/sources/rest/register.yaml +103 -0
- dtex/sources/rest/source.py +239 -0
- dtex/sources/shiphero/README.md +97 -0
- dtex/sources/shiphero/__init__.py +10 -0
- dtex/sources/shiphero/client.py +278 -0
- dtex/sources/shiphero/pagination.py +121 -0
- dtex/sources/shiphero/queries.py +105 -0
- dtex/sources/shiphero/register.yaml +160 -0
- dtex/sources/shiphero/source.py +241 -0
- dtex/sources/shiphero/windows.py +122 -0
- dtex/sources/stripe/README.md +128 -0
- dtex/sources/stripe/__init__.py +9 -0
- dtex/sources/stripe/client.py +336 -0
- dtex/sources/stripe/pagination.py +106 -0
- dtex/sources/stripe/register.yaml +206 -0
- dtex/sources/stripe/source.py +222 -0
- dtex/types.py +1861 -0
- dtex-0.1.0.dist-info/METADATA +140 -0
- dtex-0.1.0.dist-info/RECORD +72 -0
- dtex-0.1.0.dist-info/WHEEL +4 -0
- dtex-0.1.0.dist-info/entry_points.txt +7 -0
- dtex-0.1.0.dist-info/licenses/LICENSE +201 -0
dtex/__init__.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Copyright 2026 Albinas Plesnys
|
|
3
|
+
|
|
4
|
+
"""dtex — a simple, open-source Python extract-load (EL) tool.
|
|
5
|
+
|
|
6
|
+
dtex moves data from a **source** into a **destination** and nothing more.
|
|
7
|
+
Pipelines are configs, connectors are folders of plain Python; projects are
|
|
8
|
+
folders of plain files run from the ``dtex`` CLI or the importable ``dtex``
|
|
9
|
+
library. Architectural inspiration is dbt.
|
|
10
|
+
|
|
11
|
+
This module is the **public API** a connector author imports. After build
|
|
12
|
+
stage 3 it exposes the decorator surface a connector body binds to —
|
|
13
|
+
``stream``, ``resource``, ``destination``, ``Connector``, ``stream_method`` —
|
|
14
|
+
plus the contract types an author references (``Capability``, ``Schema``,
|
|
15
|
+
``Field``, ``Config``, ``State``, ``Cursor``, ``Batch``, ``StateRecord`` and
|
|
16
|
+
the enums). A connector body should need to import only from ``dtex``::
|
|
17
|
+
|
|
18
|
+
from dtex import stream, destination, Capability, Schema
|
|
19
|
+
|
|
20
|
+
As of build stage 5 it also exposes the engine entry point :func:`run` — the
|
|
21
|
+
library front door onto the run lifecycle (docs/02 §The triad). The CLI is a
|
|
22
|
+
thin shell over this same function. Stage 8.B made *configs* the runtime
|
|
23
|
+
unit — a config (``configs/<name>.yml``) names one source-to-destination
|
|
24
|
+
pipeline; ``run`` takes the config's name::
|
|
25
|
+
|
|
26
|
+
import dtex
|
|
27
|
+
result = dtex.run(config="shiphero_prod")
|
|
28
|
+
|
|
29
|
+
``run`` returns a :class:`~dtex.types.RunResult` and never raises on a
|
|
30
|
+
connector/destination failure (docs/07 §4.1).
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
# Imported last: dtex.engine pulls in dtex.registry / dtex.types,
|
|
36
|
+
# which are already bound above — no import cycle. The engine is the library's
|
|
37
|
+
# run entry point (docs/02).
|
|
38
|
+
from dtex.engine import run, run_tag
|
|
39
|
+
from dtex.registry import (
|
|
40
|
+
Connector,
|
|
41
|
+
destination,
|
|
42
|
+
resource,
|
|
43
|
+
stream,
|
|
44
|
+
stream_method,
|
|
45
|
+
)
|
|
46
|
+
from dtex.secrets import (
|
|
47
|
+
SecretResolutionError,
|
|
48
|
+
SecretResolver,
|
|
49
|
+
register_secret_resolver,
|
|
50
|
+
)
|
|
51
|
+
from dtex.types import (
|
|
52
|
+
Batch,
|
|
53
|
+
Capability,
|
|
54
|
+
Config,
|
|
55
|
+
ConnectorKind,
|
|
56
|
+
Cursor,
|
|
57
|
+
CursorType,
|
|
58
|
+
Field,
|
|
59
|
+
FieldMode,
|
|
60
|
+
FieldType,
|
|
61
|
+
PartitionConfig,
|
|
62
|
+
PartitionRange,
|
|
63
|
+
PartitionType,
|
|
64
|
+
PipelineConfig,
|
|
65
|
+
Record,
|
|
66
|
+
RunConfig,
|
|
67
|
+
RunRecord,
|
|
68
|
+
RunResult,
|
|
69
|
+
RunStatus,
|
|
70
|
+
Schema,
|
|
71
|
+
SchemaContract,
|
|
72
|
+
State,
|
|
73
|
+
StateBackend,
|
|
74
|
+
StateRecord,
|
|
75
|
+
StreamMeta,
|
|
76
|
+
StreamResult,
|
|
77
|
+
StreamStatus,
|
|
78
|
+
TimeGranularity,
|
|
79
|
+
WriteDisposition,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
__version__ = "0.1.0"
|
|
83
|
+
|
|
84
|
+
__all__ = [
|
|
85
|
+
# Version
|
|
86
|
+
"__version__",
|
|
87
|
+
# Engine entry point (dtex.engine) — the library front door
|
|
88
|
+
"run",
|
|
89
|
+
"run_tag",
|
|
90
|
+
# Decorator API surface (dtex.registry)
|
|
91
|
+
"stream",
|
|
92
|
+
"resource",
|
|
93
|
+
"destination",
|
|
94
|
+
"Connector",
|
|
95
|
+
"stream_method",
|
|
96
|
+
# Secret-resolver plugin surface (dtex.secrets — stage 9a, docs/08 §3)
|
|
97
|
+
"SecretResolver",
|
|
98
|
+
"SecretResolutionError",
|
|
99
|
+
"register_secret_resolver",
|
|
100
|
+
# Contract types a connector author references (dtex.types)
|
|
101
|
+
"Batch",
|
|
102
|
+
"Capability",
|
|
103
|
+
"Config",
|
|
104
|
+
"ConnectorKind",
|
|
105
|
+
"Cursor",
|
|
106
|
+
"CursorType",
|
|
107
|
+
"Field",
|
|
108
|
+
"FieldMode",
|
|
109
|
+
"FieldType",
|
|
110
|
+
"PartitionConfig",
|
|
111
|
+
"PartitionRange",
|
|
112
|
+
"PartitionType",
|
|
113
|
+
"PipelineConfig",
|
|
114
|
+
"Record",
|
|
115
|
+
"RunConfig",
|
|
116
|
+
"RunRecord",
|
|
117
|
+
"RunResult",
|
|
118
|
+
"RunStatus",
|
|
119
|
+
"Schema",
|
|
120
|
+
"SchemaContract",
|
|
121
|
+
"State",
|
|
122
|
+
"StateBackend",
|
|
123
|
+
"StateRecord",
|
|
124
|
+
"StreamMeta",
|
|
125
|
+
"StreamResult",
|
|
126
|
+
"StreamStatus",
|
|
127
|
+
"TimeGranularity",
|
|
128
|
+
"WriteDisposition",
|
|
129
|
+
]
|