tiozin 1.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. tiozin/__init__.py +59 -0
  2. tiozin/api/__init__.py +28 -0
  3. tiozin/api/loggable.py +55 -0
  4. tiozin/api/metadata/docs.py +60 -0
  5. tiozin/api/metadata/input_manifest.py +32 -0
  6. tiozin/api/metadata/job_manifest.py +46 -0
  7. tiozin/api/metadata/manifest.py +104 -0
  8. tiozin/api/metadata/output_manifest.py +27 -0
  9. tiozin/api/metadata/runner_manifest.py +20 -0
  10. tiozin/api/metadata/transform_manifest.py +25 -0
  11. tiozin/api/plugin.py +161 -0
  12. tiozin/api/processors/context.py +127 -0
  13. tiozin/api/processors/input.py +79 -0
  14. tiozin/api/processors/job.py +133 -0
  15. tiozin/api/processors/job_context.py +87 -0
  16. tiozin/api/processors/output.py +71 -0
  17. tiozin/api/processors/runner.py +58 -0
  18. tiozin/api/processors/runner_context.py +45 -0
  19. tiozin/api/processors/step_context.py +55 -0
  20. tiozin/api/processors/transform.py +105 -0
  21. tiozin/api/registries/job_registry.py +10 -0
  22. tiozin/api/registries/lineage_registry.py +10 -0
  23. tiozin/api/registries/metric_registry.py +10 -0
  24. tiozin/api/registries/schema_registry.py +10 -0
  25. tiozin/api/registries/secret_registry.py +10 -0
  26. tiozin/api/registries/setting_registry.py +10 -0
  27. tiozin/api/registries/transaction_registry.py +11 -0
  28. tiozin/api/registry.py +40 -0
  29. tiozin/api/typehint.py +16 -0
  30. tiozin/app.py +133 -0
  31. tiozin/assembly/__init__.py +10 -0
  32. tiozin/assembly/job_builder.py +213 -0
  33. tiozin/assembly/job_proxy.py +99 -0
  34. tiozin/assembly/plugin_factory.py +165 -0
  35. tiozin/assembly/plugin_scanner.py +89 -0
  36. tiozin/assembly/plugin_template.py +100 -0
  37. tiozin/assembly/policies.py +155 -0
  38. tiozin/assembly/proxying.py +91 -0
  39. tiozin/assembly/registry_factory.py +84 -0
  40. tiozin/assembly/runner_proxy.py +88 -0
  41. tiozin/assembly/step_proxy.py +110 -0
  42. tiozin/assembly/template_context_builder.py +118 -0
  43. tiozin/cli.py +43 -0
  44. tiozin/config.py +39 -0
  45. tiozin/env.py +28 -0
  46. tiozin/exceptions.py +327 -0
  47. tiozin/family/tio_kernel/__init__.py +16 -0
  48. tiozin/family/tio_kernel/inputs/__init__.py +0 -0
  49. tiozin/family/tio_kernel/inputs/noop_input.py +32 -0
  50. tiozin/family/tio_kernel/jobs/__init__.py +0 -0
  51. tiozin/family/tio_kernel/jobs/linear_job.py +68 -0
  52. tiozin/family/tio_kernel/outputs/__init__.py +0 -0
  53. tiozin/family/tio_kernel/outputs/noop_output.py +32 -0
  54. tiozin/family/tio_kernel/registries/__init__.py +0 -0
  55. tiozin/family/tio_kernel/registries/file_job_registry.py +72 -0
  56. tiozin/family/tio_kernel/registries/noop_lineage_registry.py +21 -0
  57. tiozin/family/tio_kernel/registries/noop_metric_registry.py +21 -0
  58. tiozin/family/tio_kernel/registries/noop_schema_registry.py +21 -0
  59. tiozin/family/tio_kernel/registries/noop_secret_registry.py +21 -0
  60. tiozin/family/tio_kernel/registries/noop_settings_registry.py +21 -0
  61. tiozin/family/tio_kernel/registries/noop_transaction_registry.py +21 -0
  62. tiozin/family/tio_kernel/runners/__init__.py +0 -0
  63. tiozin/family/tio_kernel/runners/noop_runner.py +32 -0
  64. tiozin/family/tio_kernel/transforms/__init__.py +0 -0
  65. tiozin/family/tio_kernel/transforms/noop_transform.py +32 -0
  66. tiozin/lifecycle.py +38 -0
  67. tiozin/logs.py +34 -0
  68. tiozin/utils/app_status.py +140 -0
  69. tiozin/utils/helpers.py +488 -0
  70. tiozin/utils/jinja.py +217 -0
  71. tiozin/utils/messages.py +215 -0
  72. tiozin/utils/relative_date.py +357 -0
  73. tiozin-1.1.0.dist-info/METADATA +132 -0
  74. tiozin-1.1.0.dist-info/RECORD +77 -0
  75. tiozin-1.1.0.dist-info/WHEEL +4 -0
  76. tiozin-1.1.0.dist-info/entry_points.txt +5 -0
  77. tiozin-1.1.0.dist-info/licenses/LICENSE +373 -0
tiozin/__init__.py ADDED
@@ -0,0 +1,59 @@
1
+ # isort: skip_file
2
+ # flake8: noqa
3
+
4
+ """
5
+ Tiozin - A friendly ETL framework
6
+
7
+ Public API for building data pipelines with Tiozin.
8
+ """
9
+
10
+ # Bases
11
+ from tiozin.api import Registry, PlugIn
12
+
13
+ # Metadata
14
+ from tiozin.api import JobManifest
15
+
16
+ # Registries
17
+ from tiozin.api import (
18
+ JobRegistry,
19
+ LineageRegistry,
20
+ MetricRegistry,
21
+ SchemaRegistry,
22
+ SecretRegistry,
23
+ SettingRegistry,
24
+ TransactionRegistry,
25
+ )
26
+
27
+ # Processors
28
+ from tiozin.api import Runner, Transform, Input, Output, Job
29
+
30
+ # Auxiliary
31
+ from tiozin.api import JobContext, StepContext, RunnerContext
32
+ from tiozin.app import TiozinApp
33
+
34
+ __all__ = [
35
+ # Bases
36
+ "PlugIn",
37
+ "Registry",
38
+ # Metadata
39
+ "JobManifest",
40
+ # Registries
41
+ "JobRegistry",
42
+ "LineageRegistry",
43
+ "MetricRegistry",
44
+ "SchemaRegistry",
45
+ "SecretRegistry",
46
+ "SettingRegistry",
47
+ "TransactionRegistry",
48
+ # Processors
49
+ "Runner",
50
+ "Transform",
51
+ "Input",
52
+ "Output",
53
+ "Job",
54
+ # Auxiliary
55
+ "JobContext",
56
+ "RunnerContext",
57
+ "StepContext",
58
+ "TiozinApp",
59
+ ]
tiozin/api/__init__.py ADDED
@@ -0,0 +1,28 @@
1
+ # isort: skip_file
2
+ # flake8: noqa
3
+
4
+ from .loggable import Loggable as Loggable
5
+ from .plugin import PlugIn as PlugIn
6
+ from .registry import Registry as Registry
7
+
8
+ from .metadata.job_manifest import JobManifest as JobManifest
9
+
10
+ from .registries.job_registry import JobRegistry as JobRegistry
11
+ from .registries.lineage_registry import LineageRegistry as LineageRegistry
12
+ from .registries.metric_registry import MetricRegistry as MetricRegistry
13
+ from .registries.schema_registry import SchemaRegistry as SchemaRegistry
14
+ from .registries.secret_registry import SecretRegistry as SecretRegistry
15
+ from .registries.setting_registry import SettingRegistry as SettingRegistry
16
+ from .registries.transaction_registry import TransactionRegistry as TransactionRegistry
17
+
18
+ from .processors.runner_context import RunnerContext as RunnerContext
19
+ from .processors.runner import Runner as Runner
20
+
21
+ from .processors.step_context import StepContext as StepContext
22
+ from .processors.input import Input as Input
23
+ from .processors.transform import Transform as Transform
24
+ from .processors.transform import CoTransform as CoTransform
25
+ from .processors.output import Output as Output
26
+
27
+ from .processors.job_context import JobContext as JobContext
28
+ from .processors.job import Job as Job
tiozin/api/loggable.py ADDED
@@ -0,0 +1,55 @@
1
+ import logging
2
+ from typing import Unpack
3
+
4
+ from tiozin import config, logs
5
+
6
+ from .typehint import LogKwargs
7
+
8
+ BLUE = "\033[34m"
9
+ RESET = "\033[0m"
10
+
11
+ PADDING_MAX_LENGTH = 15
12
+
13
+
14
+ class Loggable:
15
+ """
16
+ Mixin that provides logging capabilities to any class.
17
+
18
+ Classes using this mixin gain access to convenience logging methods
19
+ (debug, info, warning, error, exception, critical) that delegate
20
+ to a logger instance scoped to the class name.
21
+ """
22
+
23
+ __slots__ = ("_logger", "_logger_name")
24
+
25
+ @property
26
+ def logger(self) -> logging.Logger:
27
+ if not hasattr(self, "_logger"):
28
+ name = getattr(self, "name", None) or type(self).__name__
29
+ self._logger = logs.get_logger(name)
30
+ self._logger_name = name
31
+ return self._logger
32
+
33
+ def debug(self, msg: str, *args, **kwargs: Unpack[LogKwargs]) -> None:
34
+ self.logger.debug(self._fmt(msg), *args, **kwargs)
35
+
36
+ def info(self, msg: str, *args, **kwargs: Unpack[LogKwargs]) -> None:
37
+ self.logger.info(self._fmt(msg), *args, **kwargs)
38
+
39
+ def warning(self, msg: str, *args, **kwargs: Unpack[LogKwargs]) -> None:
40
+ self.logger.warning(self._fmt(msg), *args, **kwargs)
41
+
42
+ def error(self, msg: str, *args, **kwargs: Unpack[LogKwargs]) -> None:
43
+ self.logger.error(self._fmt(msg), *args, **kwargs)
44
+
45
+ def exception(self, msg: str, *args, **kwargs: Unpack[LogKwargs]) -> None:
46
+ self.logger.exception(self._fmt(msg), *args, **kwargs)
47
+
48
+ def critical(self, msg: str, *args, **kwargs: Unpack[LogKwargs]) -> None:
49
+ self.logger.critical(self._fmt(msg), *args, **kwargs)
50
+
51
+ def _fmt(self, msg: str) -> str:
52
+ if config.log_json:
53
+ return msg
54
+ name = self._logger_name.ljust(PADDING_MAX_LENGTH)
55
+ return f"[{BLUE}{name}{RESET}] {msg}"
@@ -0,0 +1,60 @@
1
+ """Documentation strings for manifest fields."""
2
+
3
+ # Common fields
4
+ KIND = "Type of plugin, e.g., 'spark', 'kafka', 'bigquery'"
5
+
6
+ # RunnerManifest
7
+ RUNNER_NAME = "Unique identifier for this runner (optional)"
8
+ RUNNER_DESCRIPTION = "Short description of the runner's execution backend (optional)"
9
+ RUNNER_STREAMING = "Whether this runner executes streaming workloads (optional, defaults to False)"
10
+
11
+ # InputManifest
12
+ INPUT_NAME = "Unique identifier for this input within the job"
13
+ INPUT_DESCRIPTION = "Short description of the data source (optional)"
14
+ INPUT_ORG = "Organization owning the source data (optional)"
15
+ INPUT_REGION = "Business region of the source data domain (optional)"
16
+ INPUT_DOMAIN = "Domain team owning the source data (optional)"
17
+ INPUT_PRODUCT = "Data product being consumed (optional)"
18
+ INPUT_MODEL = "Data model being read (e.g., table, topic, collection) (optional)"
19
+ INPUT_LAYER = "Data layer of the source (e.g., raw, trusted, refined) (optional)"
20
+ INPUT_SCHEMA = "The schema definition of input data (optional)"
21
+ INPUT_SCHEMA_SUBJECT = "Schema registry subject name (optional)"
22
+ INPUT_SCHEMA_VERSION = "Specific schema version (optional)"
23
+
24
+ # TransformManifest
25
+ TRANSFORM_NAME = "Unique identifier for this transform within the job"
26
+ TRANSFORM_DESCRIPTION = "Short description of the transformation logic (optional)"
27
+ TRANSFORM_ORG = "Organization owning the transformation logic (optional)"
28
+ TRANSFORM_REGION = "Business region of the transformation domain (optional)"
29
+ TRANSFORM_DOMAIN = "Domain team owning the transformation (optional)"
30
+ TRANSFORM_PRODUCT = "Data product being transformed (optional)"
31
+ TRANSFORM_MODEL = "Data model being transformed (optional)"
32
+ TRANSFORM_LAYER = "Data layer of the transformation output (optional)"
33
+
34
+ # OutputManifest
35
+ OUTPUT_NAME = "Unique identifier for this output within the job"
36
+ OUTPUT_DESCRIPTION = "Short description of the data destination (optional)"
37
+ OUTPUT_ORG = "Organization owning the destination data (optional)"
38
+ OUTPUT_REGION = "Business region of the destination data domain (optional)"
39
+ OUTPUT_DOMAIN = "Domain team owning the destination (optional)"
40
+ OUTPUT_PRODUCT = "Data product being produced (optional)"
41
+ OUTPUT_MODEL = "Data model being written (e.g., table, topic, collection) (optional)"
42
+ OUTPUT_LAYER = "Data layer of the destination (e.g., raw, trusted, refined) (optional)"
43
+
44
+ # JobManifest - Identity & Ownership
45
+ JOB_NAME = "Unique name for the job (it is not the execution ID)"
46
+ JOB_DESCRIPTION = "Short description of the pipeline (optional)"
47
+ JOB_OWNER = "Team that required for the job (optional)"
48
+ JOB_MAINTAINER = "Team that maintains this job (optional)"
49
+ JOB_COST_CENTER = "Team that pays for this job (optional)"
50
+ JOB_LABELS = "Additional metadata as key-value pairs (optional, defaults to empty dict)"
51
+ JOB_ORG = "Organization producing the data product"
52
+ JOB_REGION = "Business region of the domain team"
53
+ JOB_DOMAIN = "Domain team following the Data Mesh concept"
54
+ JOB_PRODUCT = "Data product being produced"
55
+ JOB_MODEL = "Data model being produced (e.g., table, topic, collection)"
56
+ JOB_LAYER = "Data layer this job represents (e.g., raw, trusted, refined)"
57
+ JOB_RUNNER = "Runtime environment where the job runs"
58
+ JOB_INPUTS = "Sources that provide data to the job"
59
+ JOB_TRANSFORMS = "Steps that modify the data (optional, defaults to empty list)"
60
+ JOB_OUTPUTS = "Destinations where data is written (optional, defaults to empty list)"
@@ -0,0 +1,32 @@
1
+ from __future__ import annotations
2
+
3
+ from pydantic import Field
4
+
5
+ from . import docs
6
+ from .manifest import Manifest
7
+
8
+
9
+ class InputManifest(Manifest):
10
+ """
11
+ Declarative data source definition.
12
+
13
+ Specifies how and where data is read into the pipeline.
14
+ """
15
+
16
+ # Identity
17
+ kind: str = Field(description=docs.KIND)
18
+ name: str = Field(description=docs.INPUT_NAME)
19
+ description: str | None = Field(None, description=docs.INPUT_DESCRIPTION)
20
+
21
+ # Business Taxonomy
22
+ org: str | None = Field(None, description=docs.INPUT_ORG)
23
+ region: str | None = Field(None, description=docs.INPUT_REGION)
24
+ domain: str | None = Field(None, description=docs.INPUT_DOMAIN)
25
+ product: str | None = Field(None, description=docs.INPUT_PRODUCT)
26
+ model: str | None = Field(None, description=docs.INPUT_MODEL)
27
+ layer: str | None = Field(None, description=docs.INPUT_LAYER)
28
+
29
+ # Specific
30
+ schema: str | None = Field(None, description=docs.INPUT_SCHEMA)
31
+ schema_subject: str | None = Field(None, description=docs.INPUT_SCHEMA_SUBJECT)
32
+ schema_version: str | None = Field(None, description=docs.INPUT_SCHEMA_VERSION)
@@ -0,0 +1,46 @@
1
+ from __future__ import annotations
2
+
3
+ from pydantic import Field
4
+
5
+ from . import docs
6
+ from .input_manifest import InputManifest
7
+ from .manifest import Manifest
8
+ from .output_manifest import OutputManifest
9
+ from .runner_manifest import RunnerManifest
10
+ from .transform_manifest import TransformManifest
11
+
12
+
13
+ class JobManifest(Manifest):
14
+ """
15
+ Declarative job definition.
16
+
17
+ Describes a job as structured data including metadata, taxonomy, and pipeline components.
18
+ Can be stored, versioned, and transferred as code.
19
+ """
20
+
21
+ # Identity
22
+ kind: str = Field(description=docs.KIND)
23
+ name: str = Field(description=docs.JOB_NAME)
24
+ description: str | None = Field(None, description=docs.JOB_DESCRIPTION)
25
+
26
+ # Ownership
27
+ owner: str | None = Field(None, description=docs.JOB_OWNER)
28
+ maintainer: str | None = Field(None, description=docs.JOB_MAINTAINER)
29
+ cost_center: str | None = Field(None, description=docs.JOB_COST_CENTER)
30
+ labels: dict[str, str] | None = Field(default_factory=dict, description=docs.JOB_LABELS)
31
+
32
+ # Business Taxonomy
33
+ org: str = Field(description=docs.JOB_ORG)
34
+ region: str = Field(description=docs.JOB_REGION)
35
+ domain: str = Field(description=docs.JOB_DOMAIN)
36
+ product: str = Field(description=docs.JOB_PRODUCT)
37
+ model: str = Field(description=docs.JOB_MODEL)
38
+ layer: str = Field(description=docs.JOB_LAYER)
39
+
40
+ # Pipeline Components
41
+ runner: RunnerManifest = Field(description=docs.JOB_RUNNER)
42
+ inputs: list[InputManifest] = Field(description=docs.JOB_INPUTS, min_length=1)
43
+ transforms: list[TransformManifest] | None = Field(
44
+ default_factory=list, description=docs.JOB_TRANSFORMS
45
+ )
46
+ outputs: list[OutputManifest] | None = Field(default_factory=list, description=docs.JOB_OUTPUTS)
@@ -0,0 +1,104 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from io import StringIO
5
+ from typing import Any, Self
6
+
7
+ from pydantic import BaseModel, ConfigDict, ValidationError
8
+ from ruamel.yaml import YAML
9
+ from ruamel.yaml.constructor import DuplicateKeyError
10
+
11
+ from tiozin.exceptions import ManifestError
12
+ from tiozin.utils.helpers import try_get
13
+
14
+ _yaml = YAML(typ="safe")
15
+ _yaml.allow_duplicate_keys = False
16
+ _yaml.explicit_start = False
17
+ _yaml.sort_base_mapping_type_on_output = False
18
+ _yaml.default_flow_style = False
19
+
20
+
21
+ class Manifest(BaseModel):
22
+ """
23
+ Base manifest for pipeline resources.
24
+
25
+ Provides identity and business context for runners, inputs, transforms, and outputs.
26
+ """
27
+
28
+ model_config = ConfigDict(extra="allow")
29
+
30
+ @classmethod
31
+ def model_validate(cls, obj, **kwargs) -> None:
32
+ try:
33
+ return super().model_validate(obj, **kwargs)
34
+ except ValidationError as e:
35
+ name = try_get(obj, "name", cls.__name__)
36
+ raise ManifestError.from_pydantic(e, name=name) from e
37
+
38
+ @classmethod
39
+ def from_yaml_or_json(cls, data: str) -> Self:
40
+ """
41
+ Load manifest from YAML or JSON string.
42
+ JSON is parsed as YAML since JSON is a valid YAML subset.
43
+
44
+ Args:
45
+ data: YAML or JSON formatted string to parse.
46
+ failfast: returns None instead of raising an exception.
47
+
48
+ Returns:
49
+ Validated manifest instance.
50
+
51
+ Raises:
52
+ ManifestError: If the data contains duplicate keys or validation fails.
53
+ """
54
+ try:
55
+ manifest = _yaml.load(data)
56
+ return cls.model_validate(manifest)
57
+ except DuplicateKeyError as e:
58
+ raise ManifestError.from_ruamel(e, cls.__name__) from e
59
+
60
+ @classmethod
61
+ def try_from_yaml_or_json(cls, data: str | Manifest | Any) -> Self | None:
62
+ """
63
+ Alias of from_yaml_or_json() that suppresses all exceptions.
64
+ Try to load manifest from YAML or JSON string, returning None on failure.
65
+
66
+ Args:
67
+ data: YAML/JSON string, manifest instance, or any other object type.
68
+
69
+ Returns:
70
+ Manifest instance if data is already a manifest or valid YAML/JSON string.
71
+ Returns None if parsing/validation fails or data is an unsupported type.
72
+ """
73
+ if isinstance(data, cls):
74
+ return data
75
+
76
+ if not isinstance(data, str):
77
+ return None
78
+
79
+ try:
80
+ return cls.from_yaml_or_json(data)
81
+ except Exception:
82
+ return None
83
+
84
+ def to_yaml(self) -> str:
85
+ """
86
+ Serialize manifest to YAML string.
87
+
88
+ Returns:
89
+ YAML formatted string with None values excluded.
90
+ """
91
+ manifest = self.model_dump(mode="json", exclude_unset=True)
92
+ data = StringIO()
93
+ _yaml.dump(manifest, data)
94
+ return data.getvalue()
95
+
96
+ def to_json(self) -> str:
97
+ """
98
+ Serialize manifest to JSON string.
99
+
100
+ Returns:
101
+ Pretty-printed JSON string with 2-space indentation and None values excluded.
102
+ """
103
+ manifest = self.model_dump(mode="json", exclude_unset=True)
104
+ return json.dumps(manifest, indent=2, ensure_ascii=False) + "\n"
@@ -0,0 +1,27 @@
1
+ from __future__ import annotations
2
+
3
+ from pydantic import Field
4
+
5
+ from . import docs
6
+ from .manifest import Manifest
7
+
8
+
9
+ class OutputManifest(Manifest):
10
+ """
11
+ Declarative data destination definition.
12
+
13
+ Specifies where and how processed data is written.
14
+ """
15
+
16
+ # Identity
17
+ kind: str = Field(description=docs.KIND)
18
+ name: str = Field(description=docs.OUTPUT_NAME)
19
+ description: str | None = Field(None, description=docs.OUTPUT_DESCRIPTION)
20
+
21
+ # Business Taxonomy
22
+ org: str | None = Field(None, description=docs.OUTPUT_ORG)
23
+ region: str | None = Field(None, description=docs.OUTPUT_REGION)
24
+ domain: str | None = Field(None, description=docs.OUTPUT_DOMAIN)
25
+ product: str | None = Field(None, description=docs.OUTPUT_PRODUCT)
26
+ model: str | None = Field(None, description=docs.OUTPUT_MODEL)
27
+ layer: str | None = Field(None, description=docs.OUTPUT_LAYER)
@@ -0,0 +1,20 @@
1
+ from pydantic import Field
2
+
3
+ from . import docs
4
+ from .manifest import Manifest
5
+
6
+
7
+ class RunnerManifest(Manifest):
8
+ """
9
+ Declarative runtime environment definition.
10
+
11
+ Specifies the execution backend and runtime behavior for the job.
12
+ """
13
+
14
+ # Identity
15
+ kind: str = Field(description=docs.KIND)
16
+ name: str | None = Field(None, description=docs.RUNNER_NAME)
17
+ description: str | None = Field(None, description=docs.RUNNER_DESCRIPTION)
18
+
19
+ # Specific
20
+ streaming: bool = Field(False, description=docs.RUNNER_STREAMING)
@@ -0,0 +1,25 @@
1
+ from pydantic import Field
2
+
3
+ from . import docs
4
+ from .manifest import Manifest
5
+
6
+
7
+ class TransformManifest(Manifest):
8
+ """
9
+ Declarative transformation definition.
10
+
11
+ Specifies operations that modify or enrich data.
12
+ """
13
+
14
+ # Identity
15
+ kind: str = Field(description=docs.KIND)
16
+ name: str = Field(description=docs.TRANSFORM_NAME)
17
+ description: str | None = Field(None, description=docs.TRANSFORM_DESCRIPTION)
18
+
19
+ # Business Taxonomy
20
+ org: str | None = Field(None, description=docs.TRANSFORM_ORG)
21
+ region: str | None = Field(None, description=docs.TRANSFORM_REGION)
22
+ domain: str | None = Field(None, description=docs.TRANSFORM_DOMAIN)
23
+ product: str | None = Field(None, description=docs.TRANSFORM_PRODUCT)
24
+ model: str | None = Field(None, description=docs.TRANSFORM_MODEL)
25
+ layer: str | None = Field(None, description=docs.TRANSFORM_LAYER)
tiozin/api/plugin.py ADDED
@@ -0,0 +1,161 @@
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Any, ClassVar
5
+
6
+ from tiozin import config
7
+ from tiozin.api import Loggable
8
+ from tiozin.assembly import ProxyMeta
9
+
10
+
11
+ class PlugIn(Loggable, metaclass=ProxyMeta):
12
+ """
13
+ Base class for resources that can be discovered and loaded as plugins.
14
+
15
+ Provides plugin metadata and discovery capabilities for resources that are
16
+ dynamically loaded by the framework. Used by Jobs, Inputs, Transforms,
17
+ Outputs, Runners, and Registries.
18
+ """
19
+
20
+ @dataclass(frozen=True)
21
+ class Metadata:
22
+ name: str
23
+ kind: str
24
+ kind_class: type[PlugIn]
25
+ provider: str
26
+ uri: str
27
+ tio_path: str
28
+ python_path: str
29
+
30
+ __tiometa__: ClassVar[Metadata]
31
+
32
+ def __init__(
33
+ self,
34
+ name: str = None,
35
+ description: str = None,
36
+ **options,
37
+ ) -> None:
38
+ self.kind = self.plugin_name
39
+ self.name = name or self.kind
40
+ self.description = description
41
+ self.options = options
42
+
43
+ def __init_subclass__(plugin, **kwargs) -> None:
44
+ super().__init_subclass__(**kwargs)
45
+ name = plugin.__name__
46
+ kind_class = plugin._detect_category()
47
+ kind = kind_class.__name__
48
+ provider = plugin._detect_provider()
49
+ plugin.__tiometa__ = PlugIn.Metadata(
50
+ name=name,
51
+ kind=kind,
52
+ kind_class=kind_class,
53
+ provider=provider,
54
+ uri=f"tiozin://{provider}/{kind.lower()}/{name}",
55
+ tio_path=f"{provider}:{name}",
56
+ python_path=f"{plugin.__module__}.{plugin.__qualname__}",
57
+ )
58
+
59
+ @classmethod
60
+ def _detect_category(plugin) -> type:
61
+ for clazz in reversed(plugin.__mro__):
62
+ if clazz is not PlugIn and issubclass(clazz, PlugIn):
63
+ return clazz
64
+
65
+ @classmethod
66
+ def _detect_provider(plugin) -> str:
67
+ module_path: list[str] = plugin.__module__.split(".")
68
+ prefixes = tuple(config.plugin_provider_prefixes)
69
+
70
+ for part in module_path:
71
+ if part.startswith(prefixes):
72
+ return part
73
+
74
+ return config.plugin_provider_unknown
75
+
76
+ @property
77
+ def plugin_name(self) -> str:
78
+ return self.__tiometa__.name
79
+
80
+ @property
81
+ def plugin_kind(self) -> str:
82
+ return self.__tiometa__.kind
83
+
84
+ @property
85
+ def plugin_kind_class(self) -> type[PlugIn]:
86
+ return self.__tiometa__.kind_class
87
+
88
+ @property
89
+ def plugin_provider(self) -> str:
90
+ return self.__tiometa__.provider
91
+
92
+ @property
93
+ def plugin_uri(self) -> str:
94
+ return self.__tiometa__.uri
95
+
96
+ @property
97
+ def plugin_tio_path(self) -> str:
98
+ return self.__tiometa__.tio_path
99
+
100
+ @property
101
+ def plugin_python_path(self) -> str:
102
+ return self.__tiometa__.python_path
103
+
104
+ @property
105
+ def uri(self) -> str:
106
+ if self.plugin_uri.endswith(self.name):
107
+ return self.plugin_uri
108
+ return f"{self.plugin_uri}/{self.name}"
109
+
110
+ def setup(self, *args, **kwargs) -> None:
111
+ """
112
+ Optional initialization hook.
113
+
114
+ Called when the resource enters its execution context.
115
+ Override if the resource requires setup logic such as establishing
116
+ connections, initializing sessions, or allocating resources.
117
+ """
118
+ return None
119
+
120
+ def teardown(self, *args, **kwargs) -> None:
121
+ """
122
+ Optional cleanup hook.
123
+
124
+ Called when the resource exits its execution context.
125
+ Override if the resource requires cleanup logic such as closing
126
+ connections, releasing resources, or performing final operations.
127
+ """
128
+ return None
129
+
130
+ def to_dict(
131
+ self,
132
+ *,
133
+ exclude: set[str] | None = None,
134
+ exclude_none: bool = False,
135
+ ) -> dict[str, Any]:
136
+ """
137
+ Returns a shallow dictionary representation of the resource state.
138
+
139
+ Args:
140
+ exclude: Field names to exclude from the output.
141
+ exclude_none: If True, fields with None values are excluded.
142
+ """
143
+ result: dict[str, Any] = {}
144
+ exclude = exclude or set()
145
+
146
+ for key, value in vars(self).items():
147
+ if key in exclude:
148
+ continue
149
+ if exclude_none and value is None:
150
+ continue
151
+ result[key] = value
152
+
153
+ return result
154
+
155
+ def __str__(self) -> str:
156
+ """Returns a simple string representation of the resource."""
157
+ return self.name
158
+
159
+ def __repr__(self) -> str:
160
+ """Returns a concise string representation of the resource."""
161
+ return f"{self.name}"