interloper-core 0.2.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- interloper_core-0.2.0rc1/PKG-INFO +35 -0
- interloper_core-0.2.0rc1/README.md +20 -0
- interloper_core-0.2.0rc1/pyproject.toml +64 -0
- interloper_core-0.2.0rc1/src/interloper/__init__.py +140 -0
- interloper_core-0.2.0rc1/src/interloper/assets/__init__.py +8 -0
- interloper_core-0.2.0rc1/src/interloper/assets/base.py +594 -0
- interloper_core-0.2.0rc1/src/interloper/assets/context.py +163 -0
- interloper_core-0.2.0rc1/src/interloper/assets/decorator.py +92 -0
- interloper_core-0.2.0rc1/src/interloper/assets/keys.py +22 -0
- interloper_core-0.2.0rc1/src/interloper/backfillers/__init__.py +8 -0
- interloper_core-0.2.0rc1/src/interloper/backfillers/base.py +254 -0
- interloper_core-0.2.0rc1/src/interloper/backfillers/results.py +99 -0
- interloper_core-0.2.0rc1/src/interloper/backfillers/serial.py +38 -0
- interloper_core-0.2.0rc1/src/interloper/backfillers/state.py +141 -0
- interloper_core-0.2.0rc1/src/interloper/cli/__init__.py +5 -0
- interloper_core-0.2.0rc1/src/interloper/cli/config.py +50 -0
- interloper_core-0.2.0rc1/src/interloper/cli/display.py +1068 -0
- interloper_core-0.2.0rc1/src/interloper/cli/main.py +265 -0
- interloper_core-0.2.0rc1/src/interloper/dag/__init__.py +6 -0
- interloper_core-0.2.0rc1/src/interloper/dag/base.py +404 -0
- interloper_core-0.2.0rc1/src/interloper/errors.py +155 -0
- interloper_core-0.2.0rc1/src/interloper/events/__init__.py +29 -0
- interloper_core-0.2.0rc1/src/interloper/events/base.py +480 -0
- interloper_core-0.2.0rc1/src/interloper/events/server.py +148 -0
- interloper_core-0.2.0rc1/src/interloper/io/__init__.py +21 -0
- interloper_core-0.2.0rc1/src/interloper/io/adapter.py +106 -0
- interloper_core-0.2.0rc1/src/interloper/io/base.py +73 -0
- interloper_core-0.2.0rc1/src/interloper/io/context.py +31 -0
- interloper_core-0.2.0rc1/src/interloper/io/csv.py +140 -0
- interloper_core-0.2.0rc1/src/interloper/io/database.py +378 -0
- interloper_core-0.2.0rc1/src/interloper/io/file.py +153 -0
- interloper_core-0.2.0rc1/src/interloper/io/memory.py +149 -0
- interloper_core-0.2.0rc1/src/interloper/normalizer/__init__.py +6 -0
- interloper_core-0.2.0rc1/src/interloper/normalizer/base.py +228 -0
- interloper_core-0.2.0rc1/src/interloper/normalizer/strategy.py +21 -0
- interloper_core-0.2.0rc1/src/interloper/partitioning/__init__.py +21 -0
- interloper_core-0.2.0rc1/src/interloper/partitioning/base.py +50 -0
- interloper_core-0.2.0rc1/src/interloper/partitioning/time.py +83 -0
- interloper_core-0.2.0rc1/src/interloper/rest/__init__.py +12 -0
- interloper_core-0.2.0rc1/src/interloper/rest/auth.py +270 -0
- interloper_core-0.2.0rc1/src/interloper/rest/client.py +66 -0
- interloper_core-0.2.0rc1/src/interloper/rest/paginator.py +120 -0
- interloper_core-0.2.0rc1/src/interloper/runners/__init__.py +14 -0
- interloper_core-0.2.0rc1/src/interloper/runners/base.py +279 -0
- interloper_core-0.2.0rc1/src/interloper/runners/multi_process.py +158 -0
- interloper_core-0.2.0rc1/src/interloper/runners/multi_thread.py +100 -0
- interloper_core-0.2.0rc1/src/interloper/runners/results.py +135 -0
- interloper_core-0.2.0rc1/src/interloper/runners/serial.py +42 -0
- interloper_core-0.2.0rc1/src/interloper/runners/state.py +229 -0
- interloper_core-0.2.0rc1/src/interloper/schema/__init__.py +5 -0
- interloper_core-0.2.0rc1/src/interloper/schema/base.py +179 -0
- interloper_core-0.2.0rc1/src/interloper/serialization/__init__.py +21 -0
- interloper_core-0.2.0rc1/src/interloper/serialization/asset.py +100 -0
- interloper_core-0.2.0rc1/src/interloper/serialization/backfiller.py +29 -0
- interloper_core-0.2.0rc1/src/interloper/serialization/base.py +43 -0
- interloper_core-0.2.0rc1/src/interloper/serialization/config.py +45 -0
- interloper_core-0.2.0rc1/src/interloper/serialization/dag.py +29 -0
- interloper_core-0.2.0rc1/src/interloper/serialization/io.py +28 -0
- interloper_core-0.2.0rc1/src/interloper/serialization/runner.py +29 -0
- interloper_core-0.2.0rc1/src/interloper/serialization/source.py +68 -0
- interloper_core-0.2.0rc1/src/interloper/source/__init__.py +7 -0
- interloper_core-0.2.0rc1/src/interloper/source/base.py +389 -0
- interloper_core-0.2.0rc1/src/interloper/source/config.py +20 -0
- interloper_core-0.2.0rc1/src/interloper/source/decorator.py +77 -0
- interloper_core-0.2.0rc1/src/interloper/utils/__init__.py +6 -0
- interloper_core-0.2.0rc1/src/interloper/utils/imports.py +107 -0
- interloper_core-0.2.0rc1/src/interloper/utils/text.py +94 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: interloper-core
|
|
3
|
+
Version: 0.2.0rc1
|
|
4
|
+
Summary: Interloper
|
|
5
|
+
Author: Guillaume Onfroy
|
|
6
|
+
Author-email: Guillaume Onfroy <guillaume@digitlcloud.com>
|
|
7
|
+
Requires-Dist: pydantic>=2.11.7
|
|
8
|
+
Requires-Dist: pydantic-settings>=2.11.0
|
|
9
|
+
Requires-Dist: httpx>=0.28.1
|
|
10
|
+
Requires-Dist: pyyaml>=6.0.0 ; extra == 'cli'
|
|
11
|
+
Requires-Dist: rich>=14.3.2 ; extra == 'cli'
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Provides-Extra: cli
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# Interloper Core
|
|
17
|
+
|
|
18
|
+
## Features
|
|
19
|
+
|
|
20
|
+
### Assets
|
|
21
|
+
* Functional or OOP definition
|
|
22
|
+
* Direct execution
|
|
23
|
+
* Materialization with IO config
|
|
24
|
+
* Multi IO config
|
|
25
|
+
* Upstream asset dependency
|
|
26
|
+
|
|
27
|
+
### Sources
|
|
28
|
+
* Functional or OOP definition
|
|
29
|
+
* Auto resolution of asset dependency
|
|
30
|
+
|
|
31
|
+
### Execution & materialization
|
|
32
|
+
* Custom parameters with deferred resolution (currently called asset_params)
|
|
33
|
+
* Asset graph resolution
|
|
34
|
+
* Single or multi IO config
|
|
35
|
+
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Interloper Core
|
|
2
|
+
|
|
3
|
+
## Features
|
|
4
|
+
|
|
5
|
+
### Assets
|
|
6
|
+
* Functional or OOP definition
|
|
7
|
+
* Direct execution
|
|
8
|
+
* Materialization with IO config
|
|
9
|
+
* Multi IO config
|
|
10
|
+
* Upstream asset dependency
|
|
11
|
+
|
|
12
|
+
### Sources
|
|
13
|
+
* Functional or OOP definition
|
|
14
|
+
* Auto resolution of asset dependency
|
|
15
|
+
|
|
16
|
+
### Execution & materialization
|
|
17
|
+
* Custom parameters with deferred resolution (currently called asset_params)
|
|
18
|
+
* Asset graph resolution
|
|
19
|
+
* Single or multi IO config
|
|
20
|
+
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# ###############
|
|
2
|
+
# PROJECT / UV
|
|
3
|
+
# ###############
|
|
4
|
+
[project]
|
|
5
|
+
name = "interloper-core"
|
|
6
|
+
version = "0.2.0-rc.1"
|
|
7
|
+
description = "Interloper"
|
|
8
|
+
authors = [{ name = "Guillaume Onfroy", email = "guillaume@digitlcloud.com" }]
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"pydantic>=2.11.7",
|
|
13
|
+
"pydantic-settings>=2.11.0",
|
|
14
|
+
"httpx>=0.28.1",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
[project.optional-dependencies]
|
|
18
|
+
cli = ["pyyaml>=6.0.0", "rich>=14.3.2"]
|
|
19
|
+
|
|
20
|
+
[dependency-groups]
|
|
21
|
+
dev = ["pytest>=8.0.0", "pytest-cov>=7.0.0"]
|
|
22
|
+
|
|
23
|
+
[build-system]
|
|
24
|
+
requires = ["uv_build>=0.9.5,<0.10.0"]
|
|
25
|
+
build-backend = "uv_build"
|
|
26
|
+
|
|
27
|
+
[tool.uv.build-backend]
|
|
28
|
+
module-name = "interloper"
|
|
29
|
+
|
|
30
|
+
[project.scripts]
|
|
31
|
+
interloper = "interloper.cli.main:main"
|
|
32
|
+
|
|
33
|
+
# ###############
|
|
34
|
+
# RUFF
|
|
35
|
+
# ###############
|
|
36
|
+
[tool.ruff]
|
|
37
|
+
line-length = 120
|
|
38
|
+
|
|
39
|
+
[tool.ruff.lint]
|
|
40
|
+
preview = true
|
|
41
|
+
extend-select = ["E", "I", "UP", "ANN001", "ANN201", "ANN202", "DOC", "D"]
|
|
42
|
+
ignore = ["D104"]
|
|
43
|
+
|
|
44
|
+
[tool.ruff.lint.pydocstyle]
|
|
45
|
+
convention = "google"
|
|
46
|
+
|
|
47
|
+
[tool.ruff.lint.per-file-ignores]
|
|
48
|
+
"__init__.py" = ["F401", "F403"]
|
|
49
|
+
"tests/**" = ["ANN", "D101", "D102", "B018"]
|
|
50
|
+
|
|
51
|
+
# ###############
|
|
52
|
+
# PYRIGHT
|
|
53
|
+
# ###############
|
|
54
|
+
[tool.pyright]
|
|
55
|
+
include = ["src"]
|
|
56
|
+
typeCheckingMode = "basic"
|
|
57
|
+
reportMissingParameterType = true
|
|
58
|
+
ignore = ["libs/**", "tests/**", "scripts/**"]
|
|
59
|
+
|
|
60
|
+
# ###############
|
|
61
|
+
# PYTEST
|
|
62
|
+
# ###############
|
|
63
|
+
[tool.pytest.ini_options]
|
|
64
|
+
filterwarnings = ["ignore::DeprecationWarning"]
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""Interloper - A Python framework for building and executing data pipelines."""
|
|
2
|
+
|
|
3
|
+
from interloper.assets import Asset, AssetDefinition, asset
|
|
4
|
+
from interloper.assets.context import EventLogger, ExecutionContext
|
|
5
|
+
from interloper.assets.keys import AssetDefinitionKey, AssetInstanceKey
|
|
6
|
+
from interloper.backfillers import Backfiller
|
|
7
|
+
from interloper.backfillers.results import BackfillResult
|
|
8
|
+
from interloper.backfillers.serial import SerialBackfiller
|
|
9
|
+
from interloper.dag.base import DAG
|
|
10
|
+
from interloper.errors import (
|
|
11
|
+
AdapterError,
|
|
12
|
+
AssetError,
|
|
13
|
+
AssetNotFoundError,
|
|
14
|
+
AuthenticationError,
|
|
15
|
+
BackfillError,
|
|
16
|
+
CircularDependencyError,
|
|
17
|
+
ConfigError,
|
|
18
|
+
DAGError,
|
|
19
|
+
DataNotFoundError,
|
|
20
|
+
DependencyNotFoundError,
|
|
21
|
+
EventError,
|
|
22
|
+
InterloperError,
|
|
23
|
+
InterloperIOError,
|
|
24
|
+
NormalizerError,
|
|
25
|
+
PartitionError,
|
|
26
|
+
RunnerError,
|
|
27
|
+
SchemaError,
|
|
28
|
+
ScriptLoadError,
|
|
29
|
+
SourceError,
|
|
30
|
+
TableNotFoundError,
|
|
31
|
+
)
|
|
32
|
+
from interloper.events.base import (
|
|
33
|
+
Event,
|
|
34
|
+
EventBus,
|
|
35
|
+
EventType,
|
|
36
|
+
LogLevel,
|
|
37
|
+
disable_event_forwarding,
|
|
38
|
+
emit,
|
|
39
|
+
enable_event_forwarding,
|
|
40
|
+
subscribe,
|
|
41
|
+
unsubscribe,
|
|
42
|
+
)
|
|
43
|
+
from interloper.io import IO, CsvIO, FileIO, IOContext, MemoryIO
|
|
44
|
+
from interloper.normalizer import MaterializationStrategy, Normalizer
|
|
45
|
+
from interloper.partitioning import (
|
|
46
|
+
Partition,
|
|
47
|
+
PartitionConfig,
|
|
48
|
+
PartitionWindow,
|
|
49
|
+
TimePartition,
|
|
50
|
+
TimePartitionConfig,
|
|
51
|
+
TimePartitionWindow,
|
|
52
|
+
)
|
|
53
|
+
from interloper.rest import HTTPBearerAuth, OAuth2Auth, OAuth2ClientCredentialsAuth, OAuth2RefreshTokenAuth, RESTClient
|
|
54
|
+
from interloper.runners import MultiProcessRunner, MultiThreadRunner, Runner, SerialRunner
|
|
55
|
+
from interloper.runners.results import AssetExecutionInfo, ExecutionStatus, RunResult
|
|
56
|
+
from interloper.schema import AssetSchema
|
|
57
|
+
from interloper.serialization import AssetSpec, BackfillerSpec, ConfigSpec, DAGSpec, IOSpec, RunnerSpec
|
|
58
|
+
from interloper.source import Source, SourceDefinition, source
|
|
59
|
+
from interloper.source.config import Config
|
|
60
|
+
|
|
61
|
+
__version__ = "0.1.0"
|
|
62
|
+
|
|
63
|
+
__all__ = [
|
|
64
|
+
"DAG",
|
|
65
|
+
"IO",
|
|
66
|
+
"AdapterError",
|
|
67
|
+
"Asset",
|
|
68
|
+
"AssetDefinition",
|
|
69
|
+
"AssetDefinitionKey",
|
|
70
|
+
"AssetError",
|
|
71
|
+
"AssetExecutionInfo",
|
|
72
|
+
"AssetInstanceKey",
|
|
73
|
+
"AssetNotFoundError",
|
|
74
|
+
"AssetSchema",
|
|
75
|
+
"AssetSpec",
|
|
76
|
+
"AuthenticationError",
|
|
77
|
+
"BackfillError",
|
|
78
|
+
"BackfillResult",
|
|
79
|
+
"Backfiller",
|
|
80
|
+
"BackfillerSpec",
|
|
81
|
+
"CircularDependencyError",
|
|
82
|
+
"Config",
|
|
83
|
+
"ConfigError",
|
|
84
|
+
"ConfigSpec",
|
|
85
|
+
"CsvIO",
|
|
86
|
+
"DAGError",
|
|
87
|
+
"DAGSpec",
|
|
88
|
+
"DataNotFoundError",
|
|
89
|
+
"DependencyNotFoundError",
|
|
90
|
+
"Event",
|
|
91
|
+
"EventBus",
|
|
92
|
+
"EventError",
|
|
93
|
+
"EventLogger",
|
|
94
|
+
"EventType",
|
|
95
|
+
"ExecutionContext",
|
|
96
|
+
"ExecutionStatus",
|
|
97
|
+
"FileIO",
|
|
98
|
+
"HTTPBearerAuth",
|
|
99
|
+
"IOContext",
|
|
100
|
+
"IOSpec",
|
|
101
|
+
"InterloperError",
|
|
102
|
+
"InterloperIOError",
|
|
103
|
+
"LogLevel",
|
|
104
|
+
"MaterializationStrategy",
|
|
105
|
+
"MemoryIO",
|
|
106
|
+
"MultiProcessRunner",
|
|
107
|
+
"MultiThreadRunner",
|
|
108
|
+
"Normalizer",
|
|
109
|
+
"NormalizerError",
|
|
110
|
+
"OAuth2Auth",
|
|
111
|
+
"OAuth2ClientCredentialsAuth",
|
|
112
|
+
"OAuth2RefreshTokenAuth",
|
|
113
|
+
"Partition",
|
|
114
|
+
"PartitionConfig",
|
|
115
|
+
"PartitionError",
|
|
116
|
+
"PartitionWindow",
|
|
117
|
+
"RESTClient",
|
|
118
|
+
"RunResult",
|
|
119
|
+
"Runner",
|
|
120
|
+
"RunnerError",
|
|
121
|
+
"RunnerSpec",
|
|
122
|
+
"SchemaError",
|
|
123
|
+
"ScriptLoadError",
|
|
124
|
+
"SerialBackfiller",
|
|
125
|
+
"SerialRunner",
|
|
126
|
+
"Source",
|
|
127
|
+
"SourceDefinition",
|
|
128
|
+
"SourceError",
|
|
129
|
+
"TableNotFoundError",
|
|
130
|
+
"TimePartition",
|
|
131
|
+
"TimePartitionConfig",
|
|
132
|
+
"TimePartitionWindow",
|
|
133
|
+
"asset",
|
|
134
|
+
"disable_event_forwarding",
|
|
135
|
+
"emit",
|
|
136
|
+
"enable_event_forwarding",
|
|
137
|
+
"source",
|
|
138
|
+
"subscribe",
|
|
139
|
+
"unsubscribe",
|
|
140
|
+
]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Asset definitions and decorators."""
|
|
2
|
+
|
|
3
|
+
from interloper.assets.base import Asset, AssetDefinition
|
|
4
|
+
from interloper.assets.context import ExecutionContext
|
|
5
|
+
from interloper.assets.decorator import asset
|
|
6
|
+
|
|
7
|
+
__all__ = ["Asset", "AssetDefinition", "ExecutionContext", "asset"]
|
|
8
|
+
|