kpops 3.2.0__tar.gz → 3.2.2__tar.gz

{kpops-3.2.0 → kpops-3.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kpops
-Version: 3.2.0
+Version: 3.2.2
 Summary: KPOps is a tool to deploy Kafka pipelines to Kubernetes
 Home-page: https://github.com/bakdata/kpops
 License: MIT

{kpops-3.2.0 → kpops-3.2.2}/kpops/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "3.2.0"
+__version__ = "3.2.2"
 
 # export public API functions
 from kpops.cli.main import clean, deploy, destroy, generate, manifest, reset

{kpops-3.2.0 → kpops-3.2.2}/kpops/cli/main.py

@@ -440,7 +440,7 @@ def reset(
             pipeline_tasks = pipeline.build_execution_graph(reset_runner, reverse=True)
             await pipeline_tasks
         else:
-            for component in pipeline.components:
+            for component in reversed(pipeline.components):
                 await reset_runner(component)
 
     asyncio.run(async_reset())
@@ -481,7 +481,7 @@ def clean(
             pipeline_tasks = pipeline.build_execution_graph(clean_runner, reverse=True)
             await pipeline_tasks
         else:
-            for component in pipeline.components:
+            for component in reversed(pipeline.components):
                 await clean_runner(component)
 
     asyncio.run(async_clean())

{kpops-3.2.0 → kpops-3.2.2}/kpops/components/base_components/base_defaults_component.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import logging
 from abc import ABC
 from collections.abc import Sequence
@@ -21,11 +22,16 @@ from pydantic.json_schema import SkipJsonSchema
 from kpops.component_handlers import ComponentHandlers
 from kpops.config import KpopsConfig
 from kpops.utils import cached_classproperty
-from kpops.utils.dict_ops import update_nested, update_nested_pair
+from kpops.utils.dict_ops import (
+    generate_substitution,
+    update_nested,
+    update_nested_pair,
+)
 from kpops.utils.docstring import describe_attr
 from kpops.utils.environment import ENV
 from kpops.utils.pydantic import DescConfigModel, issubclass_patched, to_dash
-from kpops.utils.yaml import load_yaml_file
+from kpops.utils.types import JsonType
+from kpops.utils.yaml import load_yaml_file, substitute_nested
 
 try:
     from typing import Self
@@ -54,7 +60,7 @@ class BaseDefaultsComponent(DescConfigModel, ABC):
     )
 
     enrich: SkipJsonSchema[bool] = Field(
-        default=False,
+        default=True,
         description=describe_attr("enrich", __doc__),
         exclude=True,
     )
@@ -70,21 +76,31 @@ class BaseDefaultsComponent(DescConfigModel, ABC):
     )
     validate_: SkipJsonSchema[bool] = Field(
         validation_alias=AliasChoices("validate", "validate_"),
-        default=True,
+        default=False,
         description=describe_attr("validate", __doc__),
         exclude=True,
     )
 
-    @pydantic.model_validator(mode="before")
-    @classmethod
-    def enrich_component(cls, values: dict[str, Any]) -> dict[str, Any]:
+    def __init__(self, **values: Any) -> None:
         if values.get("enrich", True):
+            cls = self.__class__
             values = cls.extend_with_defaults(**values)
-        return values
+            tmp_self = cls(**values, enrich=False)
+            values = tmp_self.model_dump(mode="json", by_alias=True)
+            values = cls.substitute_in_component(tmp_self.config, **values)
+            self.__init__(
+                enrich=False,
+                validate=True,
+                config=tmp_self.config,
+                handlers=tmp_self.handlers,
+                **values,
+            )
+        else:
+            super().__init__(**values)
 
     @pydantic.model_validator(mode="after")
     def validate_component(self) -> Self:
-        if self.validate_:
+        if not self.enrich and self.validate_:
             self._validate_custom()
         return self
 
@@ -113,6 +129,42 @@ class BaseDefaultsComponent(DescConfigModel, ABC):
 
         return tuple(gen_parents())
 
+    @classmethod
+    def substitute_in_component(
+        cls, config: KpopsConfig, **component_data: Any
+    ) -> dict[str, Any]:
+        """Substitute all $-placeholders in a component in dict representation.
+
+        :param component_as_dict: Component represented as dict
+        :return: Updated component
+        """
+        # Leftover variables that were previously introduced in the component by the substitution
+        # functions, still hardcoded, because of their names.
+        # TODO(Ivan Yordanov): Get rid of them
+        substitution_hardcoded: dict[str, JsonType] = {
+            "error_topic_name": config.topic_name_config.default_error_topic_name,
+            "output_topic_name": config.topic_name_config.default_output_topic_name,
+        }
+        component_substitution = generate_substitution(
+            component_data,
+            "component",
+            substitution_hardcoded,
+            separator=".",
+        )
+        substitution = generate_substitution(
+            config.model_dump(mode="json"),
+            "config",
+            existing_substitution=component_substitution,
+            separator=".",
+        )
+
+        return json.loads(
+            substitute_nested(
+                json.dumps(component_data),
+                **update_nested_pair(substitution, ENV),
+            )
+        )
+
     @classmethod
     def extend_with_defaults(cls, config: KpopsConfig, **kwargs: Any) -> dict[str, Any]:
         """Merge parent components' defaults with own.

{kpops-3.2.0 → kpops-3.2.2}/kpops/components/base_components/kafka_app.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import logging
 from abc import ABC
 
-from pydantic import ConfigDict, Field
+from pydantic import AliasChoices, ConfigDict, Field
 from typing_extensions import override
 
 from kpops.component_handlers.helm_wrapper.model import (
@@ -28,7 +28,11 @@ class KafkaStreamsConfig(CamelCaseConfigModel, DescConfigModel):
 
     brokers: str = Field(default=..., description=describe_attr("brokers", __doc__))
     schema_registry_url: str | None = Field(
-        default=None, description=describe_attr("schema_registry_url", __doc__)
+        default=None,
+        validation_alias=AliasChoices(
+            "schema_registry_url", "schemaRegistryUrl"
+        ),  # TODO: same for other camelcase fields, avoids duplicates during enrichment
+        description=describe_attr("schema_registry_url", __doc__),
     )
 
     model_config = ConfigDict(

{kpops-3.2.0 → kpops-3.2.2}/kpops/components/base_components/kafka_connector.py

@@ -148,7 +148,6 @@ class KafkaConnector(PipelineComponent, ABC):
         app["name"] = component_name
         return KafkaConnectorConfig(**app)
 
-    @computed_field
     @cached_property
     def _resetter(self) -> KafkaConnectorResetter:
         kwargs: dict[str, Any] = {}
@@ -159,7 +158,8 @@ class KafkaConnector(PipelineComponent, ABC):
             handlers=self.handlers,
             **kwargs,
             **self.model_dump(
-                exclude={"_resetter", "resetter_values", "resetter_namespace", "app"}
+                by_alias=True,
+                exclude={"_resetter", "resetter_values", "resetter_namespace", "app"},
             ),
             app=KafkaConnectorResetterValues(
                 connector_type=self._connector_type.value,
@@ -218,6 +218,11 @@ class KafkaSourceConnector(KafkaConnector):
 
     _connector_type: KafkaConnectorType = PrivateAttr(KafkaConnectorType.SOURCE)
 
+    @computed_field
+    @cached_property
+    def _resetter(self) -> KafkaConnectorResetter:
+        return super()._resetter
+
     @override
     def apply_from_inputs(self, name: str, topic: FromTopic) -> NoReturn:
         msg = "Kafka source connector doesn't support FromSection"
@@ -240,6 +245,11 @@ class KafkaSinkConnector(KafkaConnector):
 
     _connector_type: KafkaConnectorType = PrivateAttr(KafkaConnectorType.SINK)
 
+    @computed_field
+    @cached_property
+    def _resetter(self) -> KafkaConnectorResetter:
+        return super()._resetter
+
     @property
     @override
     def input_topics(self) -> list[str]:

{kpops-3.2.0 → kpops-3.2.2}/kpops/components/streams_bootstrap/producer/producer_app.py

@@ -1,6 +1,6 @@
 from functools import cached_property
 
-from pydantic import Field
+from pydantic import Field, computed_field
 from typing_extensions import override
 
 from kpops.components.base_components.kafka_app import (
@@ -51,12 +51,13 @@ class ProducerApp(KafkaApp, StreamsBootstrap):
         description=describe_attr("from_", __doc__),
     )
 
+    @computed_field
     @cached_property
     def _cleaner(self) -> ProducerAppCleaner:
         return ProducerAppCleaner(
             config=self.config,
             handlers=self.handlers,
-            **self.model_dump(),
+            **self.model_dump(by_alias=True, exclude={"_cleaner"}),
         )
 
     @override

{kpops-3.2.0 → kpops-3.2.2}/kpops/components/streams_bootstrap/streams/streams_app.py

@@ -1,6 +1,6 @@
 from functools import cached_property
 
-from pydantic import Field
+from pydantic import Field, computed_field
 from typing_extensions import override
 
 from kpops.components.base_components.kafka_app import (
@@ -33,12 +33,13 @@ class StreamsApp(KafkaApp, StreamsBootstrap):
         description=describe_attr("app", __doc__),
     )
 
+    @computed_field
     @cached_property
     def _cleaner(self) -> StreamsAppCleaner:
         return StreamsAppCleaner(
             config=self.config,
             handlers=self.handlers,
-            **self.model_dump(),
+            **self.model_dump(by_alias=True, exclude={"_cleaner"}),
         )
 
     @property

{kpops-3.2.0 → kpops-3.2.2}/kpops/pipeline.py

@@ -1,21 +1,24 @@
 from __future__ import annotations
 
 import asyncio
-import json
 import logging
-from collections.abc import Callable, Iterable
+from collections.abc import Callable
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, TypeAlias
+from typing import TYPE_CHECKING, Any, TypeAlias
 
 import networkx as nx
 import yaml
-from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny, computed_field
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    SerializeAsAny,
+    computed_field,
+)
 
 from kpops.components.base_components.pipeline_component import PipelineComponent
-from kpops.utils.dict_ops import generate_substitution, update_nested_pair
+from kpops.utils.dict_ops import update_nested_pair
 from kpops.utils.environment import ENV
-from kpops.utils.types import JsonType
-from kpops.utils.yaml import load_yaml_file, substitute_nested
+from kpops.utils.yaml import load_yaml_file
 
 if TYPE_CHECKING:
     from collections.abc import Awaitable, Coroutine, Iterator
@@ -42,8 +45,8 @@ ComponentFilterPredicate: TypeAlias = Callable[[PipelineComponent], bool]
 class Pipeline(BaseModel):
     """Pipeline representation."""
 
-    graph: nx.DiGraph = Field(default_factory=nx.DiGraph, exclude=True)
     _component_index: dict[str, PipelineComponent] = {}
+    _graph: nx.DiGraph = nx.DiGraph()
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
@@ -93,6 +96,9 @@ class Pipeline(BaseModel):
             if not predicate(component):
                 self.remove(component.id)
 
+    def validate(self) -> None:
+        self.__validate_graph()
+
     def to_yaml(self) -> str:
         return yaml.dump(
             self.model_dump(mode="json", by_alias=True, exclude_none=True)["components"]
@@ -101,42 +107,33 @@ class Pipeline(BaseModel):
     def build_execution_graph(
         self, runner: Callable[[PipelineComponent], Coroutine], /, reverse: bool = False
     ) -> Awaitable:
-        sub_graph_nodes = self.__collect_graph_nodes(
-            reversed(self.components) if reverse else self.components
-        )
-
         async def run_parallel_tasks(coroutines: list[Coroutine]) -> None:
             tasks = []
             for coro in coroutines:
                 tasks.append(asyncio.create_task(coro))
             await asyncio.gather(*tasks)
 
-        async def run_graph_tasks(pending_tasks: list[Awaitable]):
+        async def run_graph_tasks(pending_tasks: list[Awaitable]) -> None:
             for pending_task in pending_tasks:
                 await pending_task
 
-        sub_graph = self.graph.subgraph(sub_graph_nodes)
-        transformed_graph = sub_graph.copy()
+        graph: nx.DiGraph = self._graph.copy()  # pyright: ignore[reportAssignmentType, reportGeneralTypeIssues] imprecise type hint in networkx
 
-        root_node = "root_node_bfs"
         # We add an extra node to the graph, connecting all the leaf nodes to it
         # in that way we make this node the root of the graph, avoiding backtracking
-        transformed_graph.add_node(root_node)
+        root_node = "root_node_bfs"
+        graph.add_node(root_node)
 
-        for node in sub_graph:
-            predecessors = list(sub_graph.predecessors(node))
+        for node in graph:
+            predecessors = list(graph.predecessors(node))
             if not predecessors:
-                transformed_graph.add_edge(root_node, node)
+                graph.add_edge(root_node, node)
 
-        layers_graph: list[list[str]] = list(
-            nx.bfs_layers(transformed_graph, root_node)
-        )
+        layers_graph: list[list[str]] = list(nx.bfs_layers(graph, root_node))
 
         sorted_tasks = []
         for layer in layers_graph[1:]:
-            parallel_tasks = self.__get_parallel_tasks_from(layer, runner)
-
-            if parallel_tasks:
+            if parallel_tasks := self.__get_parallel_tasks_from(layer, runner):
                 sorted_tasks.append(run_parallel_tasks(parallel_tasks))
 
         if reverse:
@@ -161,7 +158,7 @@ class Pipeline(BaseModel):
         return len(self.components)
 
     def __add_to_graph(self, component: PipelineComponent):
-        self.graph.add_node(component.id)
+        self._graph.add_node(component.id)
 
         for input_topic in component.inputs:
             self.__add_input(input_topic, component.id)
@@ -169,12 +166,13 @@ class Pipeline(BaseModel):
         for output_topic in component.outputs:
             self.__add_output(output_topic, component.id)
 
-    @staticmethod
-    def __collect_graph_nodes(components: Iterable[PipelineComponent]) -> Iterator[str]:
-        for component in components:
-            yield component.id
-            yield from component.inputs
-            yield from component.outputs
+    def __add_output(self, topic_id: str, source: str) -> None:
+        self._graph.add_node(topic_id)
+        self._graph.add_edge(source, topic_id)
+
+    def __add_input(self, topic_id: str, target: str) -> None:
+        self._graph.add_node(topic_id)
+        self._graph.add_edge(topic_id, target)
 
     def __get_parallel_tasks_from(
         self, layer: list[str], runner: Callable[[PipelineComponent], Coroutine]
@@ -188,36 +186,25 @@ class Pipeline(BaseModel):
         return list(gen_parallel_tasks())
 
     def __validate_graph(self) -> None:
-        if not nx.is_directed_acyclic_graph(self.graph):
+        if not nx.is_directed_acyclic_graph(self._graph):
             msg = "Pipeline is not a valid DAG."
             raise ValueError(msg)
 
-    def validate(self) -> None:
-        self.__validate_graph()
-
-    def __add_output(self, topic_id: str, source: str) -> None:
-        self.graph.add_node(topic_id)
-        self.graph.add_edge(source, topic_id)
-
-    def __add_input(self, topic_id: str, target: str) -> None:
-        self.graph.add_node(topic_id)
-        self.graph.add_edge(topic_id, target)
-
 
 def create_env_components_index(
-    environment_components: list[dict],
-) -> dict[str, dict]:
+    environment_components: list[dict[str, Any]],
+) -> dict[str, dict[str, Any]]:
     """Create an index for all registered components in the project.
 
     :param environment_components: List of all components to be included
     :return: component index
     """
-    index: dict[str, dict] = {}
+    index: dict[str, dict[str, Any]] = {}
     for component in environment_components:
         if "type" not in component or "name" not in component:
             msg = "To override components per environment, every component should at least have a type and a name."
             raise ValueError(msg)
-        index[component["name"]] = component
+        index[component["name"]] = component  # TODO: id
     return index
 
 
@@ -305,7 +292,7 @@ class PipelineGenerator:
                     raise ParsingException from ex
 
     def apply_component(
-        self, component_class: type[PipelineComponent], component_data: dict
+        self, component_class: type[PipelineComponent], component_data: dict[str, Any]
     ) -> None:
         """Instantiate, enrich and inflate pipeline component.
 
@@ -339,90 +326,51 @@ class PipelineGenerator:
         component = component_class(
             config=self.config,
             handlers=self.handlers,
-            validate=False,
             **component_data,
         )
-        component = self.enrich_component(component)
+        component = self.enrich_component_with_env(component)
         # inflate & enrich components
         for inflated_component in component.inflate():  # TODO: recursively
-            enriched_component = self.enrich_component(inflated_component)
-            if enriched_component.from_:
+            if inflated_component.from_:
                 # read from specified components
                 for (
                     original_from_component_name,
                     from_topic,
-                ) in enriched_component.from_.components.items():
+                ) in inflated_component.from_.components.items():
                     original_from_component = find(original_from_component_name)
 
                     inflated_from_component = original_from_component.inflate()[-1]
                     resolved_from_component = find(inflated_from_component.name)
 
-                    enriched_component.weave_from_topics(
+                    inflated_component.weave_from_topics(
                         resolved_from_component.to, from_topic
                     )
             elif self.pipeline:
                 # read from previous component
                 prev_component = self.pipeline.last
-                enriched_component.weave_from_topics(prev_component.to)
-            self.pipeline.add(enriched_component)
+                inflated_component.weave_from_topics(prev_component.to)
+            self.pipeline.add(inflated_component)
 
-    def enrich_component(
-        self,
-        component: PipelineComponent,
+    def enrich_component_with_env(
+        self, component: PipelineComponent
     ) -> PipelineComponent:
-        """Enrich a pipeline component with env-specific config and substitute variables.
+        """Enrich a pipeline component with env-specific config.
 
         :param component: Component to be enriched
         :returns: Enriched component
         """
-        component.validate_ = True
+        env_component = self.env_components_index.get(component.name)
+        if not env_component:
+            return component
         env_component_as_dict = update_nested_pair(
-            self.env_components_index.get(component.name, {}),
+            env_component,
             component.model_dump(mode="json", by_alias=True),
         )
 
-        component_data = self.substitute_in_component(env_component_as_dict)
-
-        component_class = type(component)
-        return component_class(
-            enrich=False,
+        return component.__class__(
             config=self.config,
             handlers=self.handlers,
-            **component_data,
-        )
-
-    def substitute_in_component(self, component_as_dict: dict) -> dict:
-        """Substitute all $-placeholders in a component in dict representation.
-
-        :param component_as_dict: Component represented as dict
-        :return: Updated component
-        """
-        config = self.config
-        # Leftover variables that were previously introduced in the component by the substitution
-        # functions, still hardcoded, because of their names.
-        # TODO(Ivan Yordanov): Get rid of them
-        substitution_hardcoded: dict[str, JsonType] = {
-            "error_topic_name": config.topic_name_config.default_error_topic_name,
-            "output_topic_name": config.topic_name_config.default_output_topic_name,
-        }
-        component_substitution = generate_substitution(
-            component_as_dict,
-            "component",
-            substitution_hardcoded,
-            separator=".",
-        )
-        substitution = generate_substitution(
-            config.model_dump(mode="json"),
-            "config",
-            existing_substitution=component_substitution,
-            separator=".",
-        )
-
-        return json.loads(
-            substitute_nested(
-                json.dumps(component_as_dict),
-                **update_nested_pair(substitution, ENV),
-            )
+            **env_component_as_dict,
         )
 
     @staticmethod

{kpops-3.2.0 → kpops-3.2.2}/kpops/utils/gen_schema.py

@@ -138,9 +138,7 @@ def gen_pipeline_schema(
         )
         core_schema: DefinitionsSchema = component.__pydantic_core_schema__  # pyright:ignore[reportGeneralTypeIssues]
 
-        model_schema: ModelFieldsSchema = core_schema["schema"]["schema"]["schema"][  # pyright:ignore[reportGeneralTypeIssues,reportTypedDictNotRequiredAccess]
-            "schema"
-        ]
+        model_schema: ModelFieldsSchema = core_schema["schema"]["schema"]["schema"]  # pyright:ignore[reportGeneralTypeIssues,reportTypedDictNotRequiredAccess]
         model_schema["fields"]["type"] = ModelField(
             type="model-field",
             schema=LiteralSchema(

{kpops-3.2.0 → kpops-3.2.2}/kpops/utils/yaml.py

@@ -1,3 +1,4 @@
+import json
 from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
@@ -44,14 +45,22 @@ def substitute(input: str, substitution: Mapping[str, Any] | None = None) -> str
     return ImprovedTemplate(input).safe_substitute(**prepare_substitution(substitution))
 
 
+def _diff_substituted_str(s1: str, s2: str):
+    """Compare 2 strings, raise exception if not equal.
+
+    :param s1: String to compare
+    :param s2: String to compare
+    :raises ValueError: An infinite loop condition detected. Check substitution variables.
+    """
+    if s1 != s2:
+        msg = "An infinite loop condition detected. Check substitution variables."
+        raise ValueError(msg)
+
+
 def substitute_nested(input: str, **kwargs) -> str:
     """Allow for multiple substitutions to be passed.
 
     Will make as many passes as needed to substitute all possible placeholders.
-    A ceiling is set to avoid infinite loops.
-
-    HINT: If :param input: is a ``Mapping`` that you converted into ``str``,
-    You can pass it as a string, and as a ``Mapping`` to enable self-reference.
 
     :Example:
 
@@ -63,26 +72,43 @@ def substitute_nested(input: str, **kwargs) -> str:
         }
     >>> input = "${a}, ${b}, ${c}, ${d}"
     >>> print("Substituted string: " + substitute_nested(input, **substitution))
-    0, 0, 0, 0
+    "0, 0, 0, 0"
 
     :param input: The raw input containing $-placeholders
     :param **kwargs: Substitutions
-    :raises Exception: An infinite loop condition detected. Check substitution variables.
+    :raises ValueError: An infinite loop condition detected. Check substitution variables.
     :return: Substituted input string
     """
     if not kwargs:
         return input
+    kwargs = substitute_in_self(kwargs)
     old_str, new_str = "", substitute(input, kwargs)
     steps = set()
     while new_str not in steps:
         steps.add(new_str)
         old_str, new_str = new_str, substitute(new_str, kwargs)
-    if new_str != old_str:
-        msg = "An infinite loop condition detected. Check substitution variables."
-        raise ValueError(msg)
+    _diff_substituted_str(new_str, old_str)
     return old_str
 
 
+def substitute_in_self(input: dict[str, Any]) -> dict[str, Any]:
+    """Substitute all self-references in mapping.
+
+    Will make as many passes as needed to substitute all possible placeholders.
+
+    :param input: Mapping containing $-placeholders
+    :raises ValueError: An infinite loop condition detected. Check substitution variables.
+    :return: Substituted input mapping as dict
+    """
+    old_str, new_str = "", substitute(json.dumps(input), input)
+    steps = set()
+    while new_str not in steps:
+        steps.add(new_str)
+        old_str, new_str = new_str, substitute(new_str, json.loads(new_str))
+    _diff_substituted_str(new_str, old_str)
+    return json.loads(old_str)
+
+
 def print_yaml(data: Mapping | str, *, substitution: dict | None = None) -> None:
     """Print YAML object with syntax highlighting.
 

{kpops-3.2.0 → kpops-3.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "kpops"
-version = "3.2.0"
+version = "3.2.2"
 description = "KPOps is a tool to deploy Kafka pipelines to Kubernetes"
 authors = ["bakdata <opensource@bakdata.com>"]
 license = "MIT"