tiozin 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tiozin/__init__.py +59 -0
- tiozin/api/__init__.py +28 -0
- tiozin/api/loggable.py +55 -0
- tiozin/api/metadata/docs.py +60 -0
- tiozin/api/metadata/input_manifest.py +32 -0
- tiozin/api/metadata/job_manifest.py +46 -0
- tiozin/api/metadata/manifest.py +104 -0
- tiozin/api/metadata/output_manifest.py +27 -0
- tiozin/api/metadata/runner_manifest.py +20 -0
- tiozin/api/metadata/transform_manifest.py +25 -0
- tiozin/api/plugin.py +161 -0
- tiozin/api/processors/context.py +127 -0
- tiozin/api/processors/input.py +79 -0
- tiozin/api/processors/job.py +133 -0
- tiozin/api/processors/job_context.py +87 -0
- tiozin/api/processors/output.py +71 -0
- tiozin/api/processors/runner.py +58 -0
- tiozin/api/processors/runner_context.py +45 -0
- tiozin/api/processors/step_context.py +55 -0
- tiozin/api/processors/transform.py +105 -0
- tiozin/api/registries/job_registry.py +10 -0
- tiozin/api/registries/lineage_registry.py +10 -0
- tiozin/api/registries/metric_registry.py +10 -0
- tiozin/api/registries/schema_registry.py +10 -0
- tiozin/api/registries/secret_registry.py +10 -0
- tiozin/api/registries/setting_registry.py +10 -0
- tiozin/api/registries/transaction_registry.py +11 -0
- tiozin/api/registry.py +40 -0
- tiozin/api/typehint.py +16 -0
- tiozin/app.py +133 -0
- tiozin/assembly/__init__.py +10 -0
- tiozin/assembly/job_builder.py +213 -0
- tiozin/assembly/job_proxy.py +99 -0
- tiozin/assembly/plugin_factory.py +165 -0
- tiozin/assembly/plugin_scanner.py +89 -0
- tiozin/assembly/plugin_template.py +100 -0
- tiozin/assembly/policies.py +155 -0
- tiozin/assembly/proxying.py +91 -0
- tiozin/assembly/registry_factory.py +84 -0
- tiozin/assembly/runner_proxy.py +88 -0
- tiozin/assembly/step_proxy.py +110 -0
- tiozin/assembly/template_context_builder.py +118 -0
- tiozin/cli.py +43 -0
- tiozin/config.py +39 -0
- tiozin/env.py +28 -0
- tiozin/exceptions.py +327 -0
- tiozin/family/tio_kernel/__init__.py +16 -0
- tiozin/family/tio_kernel/inputs/__init__.py +0 -0
- tiozin/family/tio_kernel/inputs/noop_input.py +32 -0
- tiozin/family/tio_kernel/jobs/__init__.py +0 -0
- tiozin/family/tio_kernel/jobs/linear_job.py +68 -0
- tiozin/family/tio_kernel/outputs/__init__.py +0 -0
- tiozin/family/tio_kernel/outputs/noop_output.py +32 -0
- tiozin/family/tio_kernel/registries/__init__.py +0 -0
- tiozin/family/tio_kernel/registries/file_job_registry.py +72 -0
- tiozin/family/tio_kernel/registries/noop_lineage_registry.py +21 -0
- tiozin/family/tio_kernel/registries/noop_metric_registry.py +21 -0
- tiozin/family/tio_kernel/registries/noop_schema_registry.py +21 -0
- tiozin/family/tio_kernel/registries/noop_secret_registry.py +21 -0
- tiozin/family/tio_kernel/registries/noop_settings_registry.py +21 -0
- tiozin/family/tio_kernel/registries/noop_transaction_registry.py +21 -0
- tiozin/family/tio_kernel/runners/__init__.py +0 -0
- tiozin/family/tio_kernel/runners/noop_runner.py +32 -0
- tiozin/family/tio_kernel/transforms/__init__.py +0 -0
- tiozin/family/tio_kernel/transforms/noop_transform.py +32 -0
- tiozin/lifecycle.py +38 -0
- tiozin/logs.py +34 -0
- tiozin/utils/app_status.py +140 -0
- tiozin/utils/helpers.py +488 -0
- tiozin/utils/jinja.py +217 -0
- tiozin/utils/messages.py +215 -0
- tiozin/utils/relative_date.py +357 -0
- tiozin-1.1.0.dist-info/METADATA +132 -0
- tiozin-1.1.0.dist-info/RECORD +77 -0
- tiozin-1.1.0.dist-info/WHEEL +4 -0
- tiozin-1.1.0.dist-info/entry_points.txt +5 -0
- tiozin-1.1.0.dist-info/licenses/LICENSE +373 -0
tiozin/__init__.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# isort: skip_file
|
|
2
|
+
# flake8: noqa
|
|
3
|
+
|
|
4
|
+
"""
|
|
5
|
+
Tiozin - A friendly ETL framework
|
|
6
|
+
|
|
7
|
+
Public API for building data pipelines with Tiozin.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
# Bases
|
|
11
|
+
from tiozin.api import Registry, PlugIn
|
|
12
|
+
|
|
13
|
+
# Metadata
|
|
14
|
+
from tiozin.api import JobManifest
|
|
15
|
+
|
|
16
|
+
# Registries
|
|
17
|
+
from tiozin.api import (
|
|
18
|
+
JobRegistry,
|
|
19
|
+
LineageRegistry,
|
|
20
|
+
MetricRegistry,
|
|
21
|
+
SchemaRegistry,
|
|
22
|
+
SecretRegistry,
|
|
23
|
+
SettingRegistry,
|
|
24
|
+
TransactionRegistry,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
# Processors
|
|
28
|
+
from tiozin.api import Runner, Transform, Input, Output, Job
|
|
29
|
+
|
|
30
|
+
# Auxiliary
|
|
31
|
+
from tiozin.api import JobContext, StepContext, RunnerContext
|
|
32
|
+
from tiozin.app import TiozinApp
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
# Bases
|
|
36
|
+
"PlugIn",
|
|
37
|
+
"Registry",
|
|
38
|
+
# Metadata
|
|
39
|
+
"JobManifest",
|
|
40
|
+
# Registries
|
|
41
|
+
"JobRegistry",
|
|
42
|
+
"LineageRegistry",
|
|
43
|
+
"MetricRegistry",
|
|
44
|
+
"SchemaRegistry",
|
|
45
|
+
"SecretRegistry",
|
|
46
|
+
"SettingRegistry",
|
|
47
|
+
"TransactionRegistry",
|
|
48
|
+
# Processors
|
|
49
|
+
"Runner",
|
|
50
|
+
"Transform",
|
|
51
|
+
"Input",
|
|
52
|
+
"Output",
|
|
53
|
+
"Job",
|
|
54
|
+
# Auxiliary
|
|
55
|
+
"JobContext",
|
|
56
|
+
"RunnerContext",
|
|
57
|
+
"StepContext",
|
|
58
|
+
"TiozinApp",
|
|
59
|
+
]
|
tiozin/api/__init__.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# isort: skip_file
|
|
2
|
+
# flake8: noqa
|
|
3
|
+
|
|
4
|
+
from .loggable import Loggable as Loggable
|
|
5
|
+
from .plugin import PlugIn as PlugIn
|
|
6
|
+
from .registry import Registry as Registry
|
|
7
|
+
|
|
8
|
+
from .metadata.job_manifest import JobManifest as JobManifest
|
|
9
|
+
|
|
10
|
+
from .registries.job_registry import JobRegistry as JobRegistry
|
|
11
|
+
from .registries.lineage_registry import LineageRegistry as LineageRegistry
|
|
12
|
+
from .registries.metric_registry import MetricRegistry as MetricRegistry
|
|
13
|
+
from .registries.schema_registry import SchemaRegistry as SchemaRegistry
|
|
14
|
+
from .registries.secret_registry import SecretRegistry as SecretRegistry
|
|
15
|
+
from .registries.setting_registry import SettingRegistry as SettingRegistry
|
|
16
|
+
from .registries.transaction_registry import TransactionRegistry as TransactionRegistry
|
|
17
|
+
|
|
18
|
+
from .processors.runner_context import RunnerContext as RunnerContext
|
|
19
|
+
from .processors.runner import Runner as Runner
|
|
20
|
+
|
|
21
|
+
from .processors.step_context import StepContext as StepContext
|
|
22
|
+
from .processors.input import Input as Input
|
|
23
|
+
from .processors.transform import Transform as Transform
|
|
24
|
+
from .processors.transform import CoTransform as CoTransform
|
|
25
|
+
from .processors.output import Output as Output
|
|
26
|
+
|
|
27
|
+
from .processors.job_context import JobContext as JobContext
|
|
28
|
+
from .processors.job import Job as Job
|
tiozin/api/loggable.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Unpack
|
|
3
|
+
|
|
4
|
+
from tiozin import config, logs
|
|
5
|
+
|
|
6
|
+
from .typehint import LogKwargs
|
|
7
|
+
|
|
8
|
+
BLUE = "\033[34m"
|
|
9
|
+
RESET = "\033[0m"
|
|
10
|
+
|
|
11
|
+
PADDING_MAX_LENGTH = 15
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Loggable:
|
|
15
|
+
"""
|
|
16
|
+
Mixin that provides logging capabilities to any class.
|
|
17
|
+
|
|
18
|
+
Classes using this mixin gain access to convenience logging methods
|
|
19
|
+
(debug, info, warning, error, exception, critical) that delegate
|
|
20
|
+
to a logger instance scoped to the class name.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
__slots__ = ("_logger", "_logger_name")
|
|
24
|
+
|
|
25
|
+
@property
|
|
26
|
+
def logger(self) -> logging.Logger:
|
|
27
|
+
if not hasattr(self, "_logger"):
|
|
28
|
+
name = getattr(self, "name", None) or type(self).__name__
|
|
29
|
+
self._logger = logs.get_logger(name)
|
|
30
|
+
self._logger_name = name
|
|
31
|
+
return self._logger
|
|
32
|
+
|
|
33
|
+
def debug(self, msg: str, *args, **kwargs: Unpack[LogKwargs]) -> None:
|
|
34
|
+
self.logger.debug(self._fmt(msg), *args, **kwargs)
|
|
35
|
+
|
|
36
|
+
def info(self, msg: str, *args, **kwargs: Unpack[LogKwargs]) -> None:
|
|
37
|
+
self.logger.info(self._fmt(msg), *args, **kwargs)
|
|
38
|
+
|
|
39
|
+
def warning(self, msg: str, *args, **kwargs: Unpack[LogKwargs]) -> None:
|
|
40
|
+
self.logger.warning(self._fmt(msg), *args, **kwargs)
|
|
41
|
+
|
|
42
|
+
def error(self, msg: str, *args, **kwargs: Unpack[LogKwargs]) -> None:
|
|
43
|
+
self.logger.error(self._fmt(msg), *args, **kwargs)
|
|
44
|
+
|
|
45
|
+
def exception(self, msg: str, *args, **kwargs: Unpack[LogKwargs]) -> None:
|
|
46
|
+
self.logger.exception(self._fmt(msg), *args, **kwargs)
|
|
47
|
+
|
|
48
|
+
def critical(self, msg: str, *args, **kwargs: Unpack[LogKwargs]) -> None:
|
|
49
|
+
self.logger.critical(self._fmt(msg), *args, **kwargs)
|
|
50
|
+
|
|
51
|
+
def _fmt(self, msg: str) -> str:
|
|
52
|
+
if config.log_json:
|
|
53
|
+
return msg
|
|
54
|
+
name = self._logger_name.ljust(PADDING_MAX_LENGTH)
|
|
55
|
+
return f"[{BLUE}{name}{RESET}] {msg}"
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Documentation strings for manifest fields."""
|
|
2
|
+
|
|
3
|
+
# Common fields
|
|
4
|
+
KIND = "Type of plugin, e.g., 'spark', 'kafka', 'bigquery'"
|
|
5
|
+
|
|
6
|
+
# RunnerManifest
|
|
7
|
+
RUNNER_NAME = "Unique identifier for this runner (optional)"
|
|
8
|
+
RUNNER_DESCRIPTION = "Short description of the runner's execution backend (optional)"
|
|
9
|
+
RUNNER_STREAMING = "Whether this runner executes streaming workloads (optional, defaults to False)"
|
|
10
|
+
|
|
11
|
+
# InputManifest
|
|
12
|
+
INPUT_NAME = "Unique identifier for this input within the job"
|
|
13
|
+
INPUT_DESCRIPTION = "Short description of the data source (optional)"
|
|
14
|
+
INPUT_ORG = "Organization owning the source data (optional)"
|
|
15
|
+
INPUT_REGION = "Business region of the source data domain (optional)"
|
|
16
|
+
INPUT_DOMAIN = "Domain team owning the source data (optional)"
|
|
17
|
+
INPUT_PRODUCT = "Data product being consumed (optional)"
|
|
18
|
+
INPUT_MODEL = "Data model being read (e.g., table, topic, collection) (optional)"
|
|
19
|
+
INPUT_LAYER = "Data layer of the source (e.g., raw, trusted, refined) (optional)"
|
|
20
|
+
INPUT_SCHEMA = "The schema definition of input data (optional)"
|
|
21
|
+
INPUT_SCHEMA_SUBJECT = "Schema registry subject name (optional)"
|
|
22
|
+
INPUT_SCHEMA_VERSION = "Specific schema version (optional)"
|
|
23
|
+
|
|
24
|
+
# TransformManifest
|
|
25
|
+
TRANSFORM_NAME = "Unique identifier for this transform within the job"
|
|
26
|
+
TRANSFORM_DESCRIPTION = "Short description of the transformation logic (optional)"
|
|
27
|
+
TRANSFORM_ORG = "Organization owning the transformation logic (optional)"
|
|
28
|
+
TRANSFORM_REGION = "Business region of the transformation domain (optional)"
|
|
29
|
+
TRANSFORM_DOMAIN = "Domain team owning the transformation (optional)"
|
|
30
|
+
TRANSFORM_PRODUCT = "Data product being transformed (optional)"
|
|
31
|
+
TRANSFORM_MODEL = "Data model being transformed (optional)"
|
|
32
|
+
TRANSFORM_LAYER = "Data layer of the transformation output (optional)"
|
|
33
|
+
|
|
34
|
+
# OutputManifest
|
|
35
|
+
OUTPUT_NAME = "Unique identifier for this output within the job"
|
|
36
|
+
OUTPUT_DESCRIPTION = "Short description of the data destination (optional)"
|
|
37
|
+
OUTPUT_ORG = "Organization owning the destination data (optional)"
|
|
38
|
+
OUTPUT_REGION = "Business region of the destination data domain (optional)"
|
|
39
|
+
OUTPUT_DOMAIN = "Domain team owning the destination (optional)"
|
|
40
|
+
OUTPUT_PRODUCT = "Data product being produced (optional)"
|
|
41
|
+
OUTPUT_MODEL = "Data model being written (e.g., table, topic, collection) (optional)"
|
|
42
|
+
OUTPUT_LAYER = "Data layer of the destination (e.g., raw, trusted, refined) (optional)"
|
|
43
|
+
|
|
44
|
+
# JobManifest - Identity & Ownership
|
|
45
|
+
JOB_NAME = "Unique name for the job (it is not the execution ID)"
|
|
46
|
+
JOB_DESCRIPTION = "Short description of the pipeline (optional)"
|
|
47
|
+
JOB_OWNER = "Team that required for the job (optional)"
|
|
48
|
+
JOB_MAINTAINER = "Team that maintains this job (optional)"
|
|
49
|
+
JOB_COST_CENTER = "Team that pays for this job (optional)"
|
|
50
|
+
JOB_LABELS = "Additional metadata as key-value pairs (optional, defaults to empty dict)"
|
|
51
|
+
JOB_ORG = "Organization producing the data product"
|
|
52
|
+
JOB_REGION = "Business region of the domain team"
|
|
53
|
+
JOB_DOMAIN = "Domain team following the Data Mesh concept"
|
|
54
|
+
JOB_PRODUCT = "Data product being produced"
|
|
55
|
+
JOB_MODEL = "Data model being produced (e.g., table, topic, collection)"
|
|
56
|
+
JOB_LAYER = "Data layer this job represents (e.g., raw, trusted, refined)"
|
|
57
|
+
JOB_RUNNER = "Runtime environment where the job runs"
|
|
58
|
+
JOB_INPUTS = "Sources that provide data to the job"
|
|
59
|
+
JOB_TRANSFORMS = "Steps that modify the data (optional, defaults to empty list)"
|
|
60
|
+
JOB_OUTPUTS = "Destinations where data is written (optional, defaults to empty list)"
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pydantic import Field
|
|
4
|
+
|
|
5
|
+
from . import docs
|
|
6
|
+
from .manifest import Manifest
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class InputManifest(Manifest):
|
|
10
|
+
"""
|
|
11
|
+
Declarative data source definition.
|
|
12
|
+
|
|
13
|
+
Specifies how and where data is read into the pipeline.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
# Identity
|
|
17
|
+
kind: str = Field(description=docs.KIND)
|
|
18
|
+
name: str = Field(description=docs.INPUT_NAME)
|
|
19
|
+
description: str | None = Field(None, description=docs.INPUT_DESCRIPTION)
|
|
20
|
+
|
|
21
|
+
# Business Taxonomy
|
|
22
|
+
org: str | None = Field(None, description=docs.INPUT_ORG)
|
|
23
|
+
region: str | None = Field(None, description=docs.INPUT_REGION)
|
|
24
|
+
domain: str | None = Field(None, description=docs.INPUT_DOMAIN)
|
|
25
|
+
product: str | None = Field(None, description=docs.INPUT_PRODUCT)
|
|
26
|
+
model: str | None = Field(None, description=docs.INPUT_MODEL)
|
|
27
|
+
layer: str | None = Field(None, description=docs.INPUT_LAYER)
|
|
28
|
+
|
|
29
|
+
# Specific
|
|
30
|
+
schema: str | None = Field(None, description=docs.INPUT_SCHEMA)
|
|
31
|
+
schema_subject: str | None = Field(None, description=docs.INPUT_SCHEMA_SUBJECT)
|
|
32
|
+
schema_version: str | None = Field(None, description=docs.INPUT_SCHEMA_VERSION)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pydantic import Field
|
|
4
|
+
|
|
5
|
+
from . import docs
|
|
6
|
+
from .input_manifest import InputManifest
|
|
7
|
+
from .manifest import Manifest
|
|
8
|
+
from .output_manifest import OutputManifest
|
|
9
|
+
from .runner_manifest import RunnerManifest
|
|
10
|
+
from .transform_manifest import TransformManifest
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class JobManifest(Manifest):
|
|
14
|
+
"""
|
|
15
|
+
Declarative job definition.
|
|
16
|
+
|
|
17
|
+
Describes a job as structured data including metadata, taxonomy, and pipeline components.
|
|
18
|
+
Can be stored, versioned, and transferred as code.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
# Identity
|
|
22
|
+
kind: str = Field(description=docs.KIND)
|
|
23
|
+
name: str = Field(description=docs.JOB_NAME)
|
|
24
|
+
description: str | None = Field(None, description=docs.JOB_DESCRIPTION)
|
|
25
|
+
|
|
26
|
+
# Ownership
|
|
27
|
+
owner: str | None = Field(None, description=docs.JOB_OWNER)
|
|
28
|
+
maintainer: str | None = Field(None, description=docs.JOB_MAINTAINER)
|
|
29
|
+
cost_center: str | None = Field(None, description=docs.JOB_COST_CENTER)
|
|
30
|
+
labels: dict[str, str] | None = Field(default_factory=dict, description=docs.JOB_LABELS)
|
|
31
|
+
|
|
32
|
+
# Business Taxonomy
|
|
33
|
+
org: str = Field(description=docs.JOB_ORG)
|
|
34
|
+
region: str = Field(description=docs.JOB_REGION)
|
|
35
|
+
domain: str = Field(description=docs.JOB_DOMAIN)
|
|
36
|
+
product: str = Field(description=docs.JOB_PRODUCT)
|
|
37
|
+
model: str = Field(description=docs.JOB_MODEL)
|
|
38
|
+
layer: str = Field(description=docs.JOB_LAYER)
|
|
39
|
+
|
|
40
|
+
# Pipeline Components
|
|
41
|
+
runner: RunnerManifest = Field(description=docs.JOB_RUNNER)
|
|
42
|
+
inputs: list[InputManifest] = Field(description=docs.JOB_INPUTS, min_length=1)
|
|
43
|
+
transforms: list[TransformManifest] | None = Field(
|
|
44
|
+
default_factory=list, description=docs.JOB_TRANSFORMS
|
|
45
|
+
)
|
|
46
|
+
outputs: list[OutputManifest] | None = Field(default_factory=list, description=docs.JOB_OUTPUTS)
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from io import StringIO
|
|
5
|
+
from typing import Any, Self
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel, ConfigDict, ValidationError
|
|
8
|
+
from ruamel.yaml import YAML
|
|
9
|
+
from ruamel.yaml.constructor import DuplicateKeyError
|
|
10
|
+
|
|
11
|
+
from tiozin.exceptions import ManifestError
|
|
12
|
+
from tiozin.utils.helpers import try_get
|
|
13
|
+
|
|
14
|
+
_yaml = YAML(typ="safe")
|
|
15
|
+
_yaml.allow_duplicate_keys = False
|
|
16
|
+
_yaml.explicit_start = False
|
|
17
|
+
_yaml.sort_base_mapping_type_on_output = False
|
|
18
|
+
_yaml.default_flow_style = False
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Manifest(BaseModel):
|
|
22
|
+
"""
|
|
23
|
+
Base manifest for pipeline resources.
|
|
24
|
+
|
|
25
|
+
Provides identity and business context for runners, inputs, transforms, and outputs.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
model_config = ConfigDict(extra="allow")
|
|
29
|
+
|
|
30
|
+
@classmethod
|
|
31
|
+
def model_validate(cls, obj, **kwargs) -> None:
|
|
32
|
+
try:
|
|
33
|
+
return super().model_validate(obj, **kwargs)
|
|
34
|
+
except ValidationError as e:
|
|
35
|
+
name = try_get(obj, "name", cls.__name__)
|
|
36
|
+
raise ManifestError.from_pydantic(e, name=name) from e
|
|
37
|
+
|
|
38
|
+
@classmethod
|
|
39
|
+
def from_yaml_or_json(cls, data: str) -> Self:
|
|
40
|
+
"""
|
|
41
|
+
Load manifest from YAML or JSON string.
|
|
42
|
+
JSON is parsed as YAML since JSON is a valid YAML subset.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
data: YAML or JSON formatted string to parse.
|
|
46
|
+
failfast: returns None instead of raising an exception.
|
|
47
|
+
|
|
48
|
+
Returns:
|
|
49
|
+
Validated manifest instance.
|
|
50
|
+
|
|
51
|
+
Raises:
|
|
52
|
+
ManifestError: If the data contains duplicate keys or validation fails.
|
|
53
|
+
"""
|
|
54
|
+
try:
|
|
55
|
+
manifest = _yaml.load(data)
|
|
56
|
+
return cls.model_validate(manifest)
|
|
57
|
+
except DuplicateKeyError as e:
|
|
58
|
+
raise ManifestError.from_ruamel(e, cls.__name__) from e
|
|
59
|
+
|
|
60
|
+
@classmethod
|
|
61
|
+
def try_from_yaml_or_json(cls, data: str | Manifest | Any) -> Self | None:
|
|
62
|
+
"""
|
|
63
|
+
Alias of from_yaml_or_json() that suppresses all exceptions.
|
|
64
|
+
Try to load manifest from YAML or JSON string, returning None on failure.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
data: YAML/JSON string, manifest instance, or any other object type.
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
Manifest instance if data is already a manifest or valid YAML/JSON string.
|
|
71
|
+
Returns None if parsing/validation fails or data is an unsupported type.
|
|
72
|
+
"""
|
|
73
|
+
if isinstance(data, cls):
|
|
74
|
+
return data
|
|
75
|
+
|
|
76
|
+
if not isinstance(data, str):
|
|
77
|
+
return None
|
|
78
|
+
|
|
79
|
+
try:
|
|
80
|
+
return cls.from_yaml_or_json(data)
|
|
81
|
+
except Exception:
|
|
82
|
+
return None
|
|
83
|
+
|
|
84
|
+
def to_yaml(self) -> str:
|
|
85
|
+
"""
|
|
86
|
+
Serialize manifest to YAML string.
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
YAML formatted string with None values excluded.
|
|
90
|
+
"""
|
|
91
|
+
manifest = self.model_dump(mode="json", exclude_unset=True)
|
|
92
|
+
data = StringIO()
|
|
93
|
+
_yaml.dump(manifest, data)
|
|
94
|
+
return data.getvalue()
|
|
95
|
+
|
|
96
|
+
def to_json(self) -> str:
|
|
97
|
+
"""
|
|
98
|
+
Serialize manifest to JSON string.
|
|
99
|
+
|
|
100
|
+
Returns:
|
|
101
|
+
Pretty-printed JSON string with 2-space indentation and None values excluded.
|
|
102
|
+
"""
|
|
103
|
+
manifest = self.model_dump(mode="json", exclude_unset=True)
|
|
104
|
+
return json.dumps(manifest, indent=2, ensure_ascii=False) + "\n"
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pydantic import Field
|
|
4
|
+
|
|
5
|
+
from . import docs
|
|
6
|
+
from .manifest import Manifest
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class OutputManifest(Manifest):
|
|
10
|
+
"""
|
|
11
|
+
Declarative data destination definition.
|
|
12
|
+
|
|
13
|
+
Specifies where and how processed data is written.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
# Identity
|
|
17
|
+
kind: str = Field(description=docs.KIND)
|
|
18
|
+
name: str = Field(description=docs.OUTPUT_NAME)
|
|
19
|
+
description: str | None = Field(None, description=docs.OUTPUT_DESCRIPTION)
|
|
20
|
+
|
|
21
|
+
# Business Taxonomy
|
|
22
|
+
org: str | None = Field(None, description=docs.OUTPUT_ORG)
|
|
23
|
+
region: str | None = Field(None, description=docs.OUTPUT_REGION)
|
|
24
|
+
domain: str | None = Field(None, description=docs.OUTPUT_DOMAIN)
|
|
25
|
+
product: str | None = Field(None, description=docs.OUTPUT_PRODUCT)
|
|
26
|
+
model: str | None = Field(None, description=docs.OUTPUT_MODEL)
|
|
27
|
+
layer: str | None = Field(None, description=docs.OUTPUT_LAYER)
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from pydantic import Field
|
|
2
|
+
|
|
3
|
+
from . import docs
|
|
4
|
+
from .manifest import Manifest
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class RunnerManifest(Manifest):
|
|
8
|
+
"""
|
|
9
|
+
Declarative runtime environment definition.
|
|
10
|
+
|
|
11
|
+
Specifies the execution backend and runtime behavior for the job.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
# Identity
|
|
15
|
+
kind: str = Field(description=docs.KIND)
|
|
16
|
+
name: str | None = Field(None, description=docs.RUNNER_NAME)
|
|
17
|
+
description: str | None = Field(None, description=docs.RUNNER_DESCRIPTION)
|
|
18
|
+
|
|
19
|
+
# Specific
|
|
20
|
+
streaming: bool = Field(False, description=docs.RUNNER_STREAMING)
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from pydantic import Field
|
|
2
|
+
|
|
3
|
+
from . import docs
|
|
4
|
+
from .manifest import Manifest
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class TransformManifest(Manifest):
|
|
8
|
+
"""
|
|
9
|
+
Declarative transformation definition.
|
|
10
|
+
|
|
11
|
+
Specifies operations that modify or enrich data.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
# Identity
|
|
15
|
+
kind: str = Field(description=docs.KIND)
|
|
16
|
+
name: str = Field(description=docs.TRANSFORM_NAME)
|
|
17
|
+
description: str | None = Field(None, description=docs.TRANSFORM_DESCRIPTION)
|
|
18
|
+
|
|
19
|
+
# Business Taxonomy
|
|
20
|
+
org: str | None = Field(None, description=docs.TRANSFORM_ORG)
|
|
21
|
+
region: str | None = Field(None, description=docs.TRANSFORM_REGION)
|
|
22
|
+
domain: str | None = Field(None, description=docs.TRANSFORM_DOMAIN)
|
|
23
|
+
product: str | None = Field(None, description=docs.TRANSFORM_PRODUCT)
|
|
24
|
+
model: str | None = Field(None, description=docs.TRANSFORM_MODEL)
|
|
25
|
+
layer: str | None = Field(None, description=docs.TRANSFORM_LAYER)
|
tiozin/api/plugin.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any, ClassVar
|
|
5
|
+
|
|
6
|
+
from tiozin import config
|
|
7
|
+
from tiozin.api import Loggable
|
|
8
|
+
from tiozin.assembly import ProxyMeta
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class PlugIn(Loggable, metaclass=ProxyMeta):
|
|
12
|
+
"""
|
|
13
|
+
Base class for resources that can be discovered and loaded as plugins.
|
|
14
|
+
|
|
15
|
+
Provides plugin metadata and discovery capabilities for resources that are
|
|
16
|
+
dynamically loaded by the framework. Used by Jobs, Inputs, Transforms,
|
|
17
|
+
Outputs, Runners, and Registries.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class Metadata:
|
|
22
|
+
name: str
|
|
23
|
+
kind: str
|
|
24
|
+
kind_class: type[PlugIn]
|
|
25
|
+
provider: str
|
|
26
|
+
uri: str
|
|
27
|
+
tio_path: str
|
|
28
|
+
python_path: str
|
|
29
|
+
|
|
30
|
+
__tiometa__: ClassVar[Metadata]
|
|
31
|
+
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
name: str = None,
|
|
35
|
+
description: str = None,
|
|
36
|
+
**options,
|
|
37
|
+
) -> None:
|
|
38
|
+
self.kind = self.plugin_name
|
|
39
|
+
self.name = name or self.kind
|
|
40
|
+
self.description = description
|
|
41
|
+
self.options = options
|
|
42
|
+
|
|
43
|
+
def __init_subclass__(plugin, **kwargs) -> None:
|
|
44
|
+
super().__init_subclass__(**kwargs)
|
|
45
|
+
name = plugin.__name__
|
|
46
|
+
kind_class = plugin._detect_category()
|
|
47
|
+
kind = kind_class.__name__
|
|
48
|
+
provider = plugin._detect_provider()
|
|
49
|
+
plugin.__tiometa__ = PlugIn.Metadata(
|
|
50
|
+
name=name,
|
|
51
|
+
kind=kind,
|
|
52
|
+
kind_class=kind_class,
|
|
53
|
+
provider=provider,
|
|
54
|
+
uri=f"tiozin://{provider}/{kind.lower()}/{name}",
|
|
55
|
+
tio_path=f"{provider}:{name}",
|
|
56
|
+
python_path=f"{plugin.__module__}.{plugin.__qualname__}",
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
@classmethod
|
|
60
|
+
def _detect_category(plugin) -> type:
|
|
61
|
+
for clazz in reversed(plugin.__mro__):
|
|
62
|
+
if clazz is not PlugIn and issubclass(clazz, PlugIn):
|
|
63
|
+
return clazz
|
|
64
|
+
|
|
65
|
+
@classmethod
|
|
66
|
+
def _detect_provider(plugin) -> str:
|
|
67
|
+
module_path: list[str] = plugin.__module__.split(".")
|
|
68
|
+
prefixes = tuple(config.plugin_provider_prefixes)
|
|
69
|
+
|
|
70
|
+
for part in module_path:
|
|
71
|
+
if part.startswith(prefixes):
|
|
72
|
+
return part
|
|
73
|
+
|
|
74
|
+
return config.plugin_provider_unknown
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def plugin_name(self) -> str:
|
|
78
|
+
return self.__tiometa__.name
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def plugin_kind(self) -> str:
|
|
82
|
+
return self.__tiometa__.kind
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def plugin_kind_class(self) -> type[PlugIn]:
|
|
86
|
+
return self.__tiometa__.kind_class
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def plugin_provider(self) -> str:
|
|
90
|
+
return self.__tiometa__.provider
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def plugin_uri(self) -> str:
|
|
94
|
+
return self.__tiometa__.uri
|
|
95
|
+
|
|
96
|
+
@property
|
|
97
|
+
def plugin_tio_path(self) -> str:
|
|
98
|
+
return self.__tiometa__.tio_path
|
|
99
|
+
|
|
100
|
+
@property
|
|
101
|
+
def plugin_python_path(self) -> str:
|
|
102
|
+
return self.__tiometa__.python_path
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def uri(self) -> str:
|
|
106
|
+
if self.plugin_uri.endswith(self.name):
|
|
107
|
+
return self.plugin_uri
|
|
108
|
+
return f"{self.plugin_uri}/{self.name}"
|
|
109
|
+
|
|
110
|
+
def setup(self, *args, **kwargs) -> None:
|
|
111
|
+
"""
|
|
112
|
+
Optional initialization hook.
|
|
113
|
+
|
|
114
|
+
Called when the resource enters its execution context.
|
|
115
|
+
Override if the resource requires setup logic such as establishing
|
|
116
|
+
connections, initializing sessions, or allocating resources.
|
|
117
|
+
"""
|
|
118
|
+
return None
|
|
119
|
+
|
|
120
|
+
def teardown(self, *args, **kwargs) -> None:
|
|
121
|
+
"""
|
|
122
|
+
Optional cleanup hook.
|
|
123
|
+
|
|
124
|
+
Called when the resource exits its execution context.
|
|
125
|
+
Override if the resource requires cleanup logic such as closing
|
|
126
|
+
connections, releasing resources, or performing final operations.
|
|
127
|
+
"""
|
|
128
|
+
return None
|
|
129
|
+
|
|
130
|
+
def to_dict(
|
|
131
|
+
self,
|
|
132
|
+
*,
|
|
133
|
+
exclude: set[str] | None = None,
|
|
134
|
+
exclude_none: bool = False,
|
|
135
|
+
) -> dict[str, Any]:
|
|
136
|
+
"""
|
|
137
|
+
Returns a shallow dictionary representation of the resource state.
|
|
138
|
+
|
|
139
|
+
Args:
|
|
140
|
+
exclude: Field names to exclude from the output.
|
|
141
|
+
exclude_none: If True, fields with None values are excluded.
|
|
142
|
+
"""
|
|
143
|
+
result: dict[str, Any] = {}
|
|
144
|
+
exclude = exclude or set()
|
|
145
|
+
|
|
146
|
+
for key, value in vars(self).items():
|
|
147
|
+
if key in exclude:
|
|
148
|
+
continue
|
|
149
|
+
if exclude_none and value is None:
|
|
150
|
+
continue
|
|
151
|
+
result[key] = value
|
|
152
|
+
|
|
153
|
+
return result
|
|
154
|
+
|
|
155
|
+
def __str__(self) -> str:
|
|
156
|
+
"""Returns a simple string representation of the resource."""
|
|
157
|
+
return self.name
|
|
158
|
+
|
|
159
|
+
def __repr__(self) -> str:
|
|
160
|
+
"""Returns a concise string representation of the resource."""
|
|
161
|
+
return f"{self.name}"
|