fabricks 3.0.19__py3-none-any.whl → 4.0.1__py3-none-any.whl

fabricks/core/jobs/base/invoker.py

@@ -5,10 +5,12 @@ from pyspark.sql import DataFrame
 
 from fabricks.context import PATH_RUNTIME
 from fabricks.context.log import DEFAULT_LOGGER
+from fabricks.core.extenders import get_extender
 from fabricks.core.jobs.base.checker import Checker
 from fabricks.core.jobs.base.exception import PostRunInvokeException, PreRunInvokeException
 from fabricks.core.jobs.get_schedule import get_schedule
-from fabricks.utils.path import Path
+from fabricks.models.common import BaseInvokerOptions, ExtenderOptions
+from fabricks.utils.path import GitPath
 
 
 class Invoker(Checker):
@@ -27,8 +29,37 @@ class Invoker(Checker):
         self._invoke_job(position="post_run", schedule=schedule)
         self._invoke_step(position="post_run", schedule=schedule)
 
+    def _invoke_notebook(
+        self,
+        invoker: dict | BaseInvokerOptions,
+        schedule: Optional[str] = None,
+        **kwargs,
+    ):
+        path = kwargs.get("path")
+        if path is None:
+            notebook = invoker.get("notebook") if isinstance(invoker, dict) else invoker.notebook
+            assert notebook, "notebook mandatory"
+            path = PATH_RUNTIME.joinpath(notebook)
+
+        assert path is not None, "path could not be resolved"
+
+        timeout = invoker.get("timeout") if isinstance(invoker, dict) else invoker.timeout
+        arguments = invoker.get("arguments") if isinstance(invoker, dict) else invoker.arguments
+        arguments = arguments or {}
+
+        schema_only = kwargs.get("schema_only")
+        if schema_only is not None:
+            arguments["schema_only"] = schema_only
+
+        return self._run_notebook(
+            path=path,
+            arguments=arguments,
+            schedule=schedule,
+            timeout=timeout,
+        )
+
     def _invoke_job(self, position: str, schedule: Optional[str] = None, **kwargs):
-        invokers = self.options.invokers.get_list(position)
+        invokers = getattr(self.invoker_options, position, None) or [] if self.invoker_options else []
         if position == "run":
             invokers = invokers if len(invokers) > 0 else [{}]  # run must work even without run invoker options
 
@@ -38,35 +69,10 @@ class Invoker(Checker):
             for i, invoker in enumerate(invokers):
                 DEFAULT_LOGGER.debug(f"invoke ({i}, {position})", extra={"label": self})
                 try:
-                    path = kwargs.get("path")
-                    if path is None:
-                        notebook = invoker.get("notebook")
-                        assert notebook, "notebook mandatory"
-                        path = PATH_RUNTIME.joinpath(notebook)
-
-                    assert path is not None, "path mandatory"
-
-                    arguments = invoker.get("arguments") or {}
-                    timeout = invoker.get("timeout")
-
-                    schema_only = kwargs.get("schema_only")
-                    if schema_only is not None:
-                        arguments["schema_only"] = schema_only
-
                     if len(invokers) == 1 and position == "run":
-                        return self._run_notebook(
-                            path=path,
-                            arguments=arguments,
-                            timeout=timeout,
-                            schedule=schedule,
-                        )
+                        return self._invoke_notebook(invoker, schedule=schedule, **kwargs)
                     else:
-                        self._run_notebook(
-                            path=path,
-                            arguments=arguments,
-                            timeout=timeout,
-                            schedule=schedule,
-                        )
+                        self._invoke_notebook(invoker=invoker, schedule=schedule, **kwargs)
 
                 except Exception as e:
                     DEFAULT_LOGGER.warning(f"fail to run invoker ({i}, {position})", extra={"label": self})
@@ -82,7 +88,7 @@ class Invoker(Checker):
             raise Exception(errors)
 
     def _invoke_step(self, position: str, schedule: Optional[str] = None):
-        invokers = self.step_conf.get("invoker_options", {}).get(position, [])
+        invokers = getattr(self.step_conf.invoker_options, position, []) if self.step_conf.invoker_options else []
 
         errors = []
 
@@ -90,19 +96,7 @@ class Invoker(Checker):
             for i, invoker in enumerate(invokers):
                 DEFAULT_LOGGER.debug(f"invoke by step ({i}, {position})", extra={"label": self})
                 try:
-                    notebook = invoker.get("notebook")
-                    assert notebook, "notebook mandatory"
-                    path = PATH_RUNTIME.joinpath(notebook)
-
-                    arguments = invoker.get("arguments", {})
-                    timeout = invoker.get("timeout")
-
-                    self._run_notebook(
-                        path=path,
-                        arguments=arguments,
-                        timeout=timeout,
-                        schedule=schedule,
-                    )
+                    self._invoke_notebook(invoker=invoker, schedule=schedule)
 
                 except Exception as e:
                     DEFAULT_LOGGER.warning(f"fail to run invoker by step ({i}, {position})", extra={"label": self})
@@ -119,7 +113,7 @@ class Invoker(Checker):
 
     def _run_notebook(
         self,
-        path: Path,
+        path: GitPath,
         arguments: Optional[dict] = None,
         timeout: Optional[int] = None,
         schedule: Optional[str] = None,
@@ -128,7 +122,7 @@ class Invoker(Checker):
         Invokes a notebook job.
 
         Args:
-            path (Optional[Path]): The path to the notebook file. If not provided, it will be retrieved from the invoker options.
+            path (Optional[GitPath]): The path to the notebook file. If not provided, it will be retrieved from the invoker options.
             arguments (Optional[dict]): Additional arguments to pass to the notebook job. If not provided, it will be retrieved from the invoker options.
             schedule (Optional[str]): The schedule for the job. If provided, schedule variables will be retrieved.
 
@@ -167,33 +161,24 @@ class Invoker(Checker):
                 "topic": self.topic,
                 "item": self.item,
                 **arguments,
-                "job_options": json.dumps(self.options.job.options),
+                "job_options": json.dumps(self.options.model_dump()),
                 "schedule_variables": json.dumps(variables),
             },
         )
 
     def extend_job(self, df: DataFrame) -> DataFrame:
-        from fabricks.core.extenders import get_extender
-
-        extenders = self.options.extenders
-        for e in extenders:
-            name = e.get("extender")
-            DEFAULT_LOGGER.debug(f"extend ({name})", extra={"label": self})
-            arguments = e.get("arguments") or {}
-
-            extender = get_extender(name)
-            df = extender(df, **arguments)
-
-        return df
+        extenders = self.extender_options or []
+        return self._extend(df, extenders, extended="job")
 
     def extend_step(self, df: DataFrame) -> DataFrame:
-        from fabricks.core.extenders import get_extender
+        extenders = self.step_conf.extender_options or []
+        return self._extend(df, extenders, extended="step")
 
-        extenders = self.step_conf.get("extender_options", {})
+    def _extend(self, df: DataFrame, extenders: list[ExtenderOptions], extended: str) -> DataFrame:
         for e in extenders:
-            name = e.get("extender")
-            DEFAULT_LOGGER.debug(f"extend by step ({name})", extra={"label": self})
-            arguments = e.get("arguments", {})
+            name = e.extender
+            DEFAULT_LOGGER.debug(f"extend {extended} ({name})", extra={"label": self})
+            arguments = e.arguments or {}
 
             extender = get_extender(name)
             df = extender(df, **arguments)

fabricks/core/jobs/base/processor.py

@@ -3,9 +3,8 @@ from functools import partial
 from typing import Optional
 
 from pyspark.sql import DataFrame
-from pyspark.sql.functions import expr
 
-from fabricks.context import IS_TYPE_WIDENING, IS_UNITY_CATALOG, SECRET_SCOPE
+from fabricks.context import IS_TYPE_WIDENING
 from fabricks.context.log import DEFAULT_LOGGER
 from fabricks.core.jobs.base.exception import (
     PostRunCheckException,
@@ -18,39 +17,21 @@ from fabricks.core.jobs.base.exception import (
     SkipRunCheckWarning,
 )
 from fabricks.core.jobs.base.invoker import Invoker
+from fabricks.models import JobBronzeOptions, JobSilverOptions
 from fabricks.utils.write import write_stream
 
 
 class Processor(Invoker):
     def filter_where(self, df: DataFrame) -> DataFrame:
-        f = self.options.job.get("filter_where")
+        assert isinstance(self.options, (JobBronzeOptions, JobSilverOptions))
 
+        f = self.options.filter_where
         if f:
             DEFAULT_LOGGER.debug(f"filter where {f}", extra={"label": self})
             df = df.where(f"{f}")
 
         return df
 
-    def encrypt(self, df: DataFrame) -> DataFrame:
-        encrypted_columns = self.options.job.get_list("encrypted_columns")
-        if encrypted_columns:
-            if not IS_UNITY_CATALOG:
-                from databricks.sdk.runtime import dbutils
-
-                key = dbutils.secrets.get(scope=SECRET_SCOPE, key="encryption-key")
-            else:
-                import os
-
-                key = os.environ["FABRICKS_ENCRYPTION_KEY"]
-
-            assert key, "key not found"
-
-            for col in encrypted_columns:
-                DEFAULT_LOGGER.debug(f"encrypt column: {col}", extra={"label": self})
-                df = df.withColumn(col, expr(f"aes_encrypt({col}, '{key}')"))
-
-        return df
-
     def restore(self, last_version: Optional[str] = None, last_batch: Optional[str] = None):
         """
         Restores the processor to a specific version and batch.
@@ -70,7 +51,7 @@ class Processor(Invoker):
                 self.rm_commit(current_batch)
 
                 assert last_batch == self.table.get_property("fabricks.last_batch")
-                assert self.paths.commits.joinpath(last_batch).exists()
+                assert self.paths.to_commits.joinpath(last_batch).exists()
 
     def _for_each_batch(self, df: DataFrame, batch: Optional[int] = None, **kwargs):
         DEFAULT_LOGGER.debug("start (for each batch)", extra={"label": self})
@@ -118,7 +99,7 @@ class Processor(Invoker):
                 DEFAULT_LOGGER.debug("use streaming", extra={"label": self})
                 write_stream(
                     df,
-                    checkpoints_path=self.paths.checkpoints,
+                    checkpoints_path=self.paths.to_checkpoints,
                     func=self._for_each_batch,
                     timeout=self.timeout,
                 )
@@ -198,11 +179,15 @@ class Processor(Invoker):
                 raise exception
 
             if vacuum is None:
-                vacuum = self.options.job.get("vacuum", False)
+                vacuum = self.options.vacuum if self.options and self.options.vacuum is not None else False
             if optimize is None:
-                optimize = self.options.job.get("optimize", False)
+                optimize = self.options.optimize if self.options and self.options.optimize is not None else False
             if compute_statistics is None:
-                compute_statistics = self.options.job.get("compute_statistics", False)
+                compute_statistics = (
+                    self.options.compute_statistics
+                    if self.options and self.options.compute_statistics is not None
+                    else False
+                )
 
             if vacuum or optimize or compute_statistics:
                 self.maintain(

fabricks/core/jobs/bronze.py

@@ -7,21 +7,20 @@ from pyspark.sql.types import Row, TimestampType
 from fabricks.cdc.nocdc import NoCDC
 from fabricks.context import VARIABLES
 from fabricks.context.log import DEFAULT_LOGGER
-from fabricks.core.jobs.base._types import JobDependency, TBronze
 from fabricks.core.jobs.base.job import BaseJob
-from fabricks.core.parsers import BaseParser
 from fabricks.core.parsers.get_parser import get_parser
 from fabricks.core.parsers.utils import clean
 from fabricks.metastore.view import create_or_replace_global_temp_view
+from fabricks.models import JobBronzeOptions, JobDependency, StepBronzeConf, StepBronzeOptions
 from fabricks.utils.helpers import concat_ws
-from fabricks.utils.path import Path
+from fabricks.utils.path import FileSharePath
 from fabricks.utils.read import read
 
 
 class Bronze(BaseJob):
     def __init__(
         self,
-        step: TBronze,
+        step: str,
         topic: Optional[str] = None,
         item: Optional[str] = None,
         job_id: Optional[str] = None,
@@ -36,7 +35,7 @@ class Bronze(BaseJob):
             conf=conf,
         )
 
-    _parser: Optional[BaseParser] = None
+    _parser: Optional[str] = None
 
     @property
     def stream(self) -> bool:
@@ -54,25 +53,40 @@ class Bronze(BaseJob):
     def virtual(self) -> bool:
         return False
 
+    @property
+    def options(self) -> JobBronzeOptions:
+        """Direct access to typed bronze job options."""
+        return self.conf.options  # type: ignore
+
+    @property
+    def step_conf(self) -> StepBronzeConf:
+        """Direct access to typed bronze step conf."""
+        return self.base_step_conf  # type: ignore
+
+    @property
+    def step_options(self) -> StepBronzeOptions:
+        """Direct access to typed bronze step options."""
+        return self.base_step_conf.options  # type: ignore
+
     @classmethod
     def from_job_id(cls, step: str, job_id: str, *, conf: Optional[Union[dict, Row]] = None):
-        return cls(step=cast(TBronze, step), job_id=job_id, conf=conf)
+        return cls(step=step, job_id=job_id, conf=conf)
 
     @classmethod
     def from_step_topic_item(cls, step: str, topic: str, item: str, *, conf: Optional[Union[dict, Row]] = None):
-        return cls(step=cast(TBronze, step), topic=topic, item=item, conf=conf)
+        return cls(step=step, topic=topic, item=item, conf=conf)
 
     @property
-    def data_path(self) -> Path:
-        uri = self.options.job.get("uri")
+    def data_path(self) -> FileSharePath:
+        uri = self.options.uri
         assert uri is not None, "no uri provided in options"
-        path = Path.from_uri(uri, regex=VARIABLES)
+        path = FileSharePath.from_uri(uri, regex=VARIABLES)
         return path
 
     def get_dependencies(self, *s) -> Sequence[JobDependency]:
         dependencies = []
 
-        parents = self.options.job.get_list("parents")
+        parents = self.options.parents or []
         if parents:
             for p in parents:
                 dependencies.append(JobDependency.from_parts(self.job_id, p, "job"))
@@ -81,8 +95,8 @@ class Bronze(BaseJob):
 
     def register_external_table(self):
         options = self.conf.parser_options  # type: ignore
-        if options:
-            file_format = options.get("file_format")
+        if options and options.file_format:
+            file_format = options.file_format
         else:
             file_format = "delta"
 
@@ -136,17 +150,14 @@ class Bronze(BaseJob):
             self.compute_statistics_external_table()
 
     @property
-    def parser(self) -> BaseParser:
+    def parser(self) -> str:
         if not self._parser:
             assert self.mode not in ["register"], f"{self.mode} not allowed"
 
-            name = self.options.job.get("parser")
-            assert name is not None, "parser not found"
-
-            options = self.conf.parser_options or None  # type: ignore
-            p = get_parser(name, options)
+            parser = self.options.parser
+            assert parser is not None, "parser not found"
 
-            self._parser = p
+            self._parser = cast(str, parser)
 
         return self._parser
 
@@ -171,19 +182,52 @@ class Bronze(BaseJob):
             else:
                 df = self.spark.sql(f"select * from {self}")
 
-            # cleaning should done by parser
-            df = clean(df)
+            if self.step_options.clean is not False:
+                # cleaning should done by parser but for delta we do it here
+                df = clean(df)
 
         else:
-            df = self.parser.get_data(
+            options = self.conf.parser_options or None  # type: ignore
+            parse = get_parser(self.parser, options)
+
+            df = parse(
                 stream=stream,
                 data_path=self.data_path,
-                schema_path=self.paths.schema,
+                schema_path=self.paths.to_schema,
                 spark=self.spark,
             )
 
         return df
 
+    def encrypt(self, df: DataFrame) -> DataFrame:
+        encrypted_columns = self.options.encrypted_columns or []
+        if encrypted_columns:
+            if self.runtime_options.encryption_key is not None:
+                from databricks.sdk.runtime import dbutils
+
+                key = dbutils.secrets.get(
+                    scope=self.runtime_options.secret_scope,
+                    key=self.runtime_options.encryption_key,
+                )
+                if self.runtime_options.unity_catalog:
+                    DEFAULT_LOGGER.warning(
+                        "Unity Catalog enabled, use FABRICKS_ENCRYPTION_KEY instead",
+                        extra={"label": self},
+                    )
+
+            else:
+                import os
+
+                key = os.environ.get("FABRICKS_ENCRYPTION_KEY")
+
+            assert key, "encryption key not found in secrets nor in environment"
+
+            for col in encrypted_columns:
+                DEFAULT_LOGGER.debug(f"encrypt column: {col}", extra={"label": self})
+                df = df.withColumn(col, expr(f"aes_encrypt({col}, '{key}')"))
+
+        return df
+
     def get_data(
         self,
         stream: bool = False,
@@ -204,7 +248,7 @@ class Bronze(BaseJob):
         return df
 
     def add_calculated_columns(self, df: DataFrame) -> DataFrame:
-        calculated_columns = self.options.job.get_dict("calculated_columns")
+        calculated_columns = self.options.calculated_columns or {}
 
         if calculated_columns:
             for key, value in calculated_columns.items():
@@ -230,7 +274,7 @@ class Bronze(BaseJob):
 
     def add_key(self, df: DataFrame) -> DataFrame:
         if "__key" not in df.columns:
-            fields = self.options.job.get_list("keys")
+            fields = self.options.keys or []
             if fields:
                 DEFAULT_LOGGER.debug(f"add key ({', '.join(fields)})", extra={"label": self})
 
@@ -244,7 +288,7 @@ class Bronze(BaseJob):
 
     def add_source(self, df: DataFrame) -> DataFrame:
         if "__source" not in df.columns:
-            source = self.options.job.get("source")
+            source = self.options.source
             if source:
                 DEFAULT_LOGGER.debug(f"add source ({source})", extra={"label": self})
                 df = df.withColumn("__source", lit(source))
@@ -253,7 +297,7 @@ class Bronze(BaseJob):
 
     def add_operation(self, df: DataFrame) -> DataFrame:
         if "__operation" not in df.columns:
-            operation = self.options.job.get("operation")
+            operation = self.options.operation
             if operation:
                 DEFAULT_LOGGER.debug(f"add operation ({operation})", extra={"label": self})
                 df = df.withColumn("__operation", lit(operation))
@@ -263,15 +307,10 @@ class Bronze(BaseJob):
 
         return df
 
-    def base_transform(self, df: DataFrame) -> DataFrame:
-        df = df.transform(self.extend)
-        df = df.transform(self.add_calculated_columns)
-        df = df.transform(self.add_hash)
-        df = df.transform(self.add_operation)
-        df = df.transform(self.add_source)
-        df = df.transform(self.add_key)
-
+    def add_metadata(self, df: DataFrame) -> DataFrame:
         if "__metadata" in df.columns:
+            DEFAULT_LOGGER.debug("add metadata", extra={"label": self})
+
             if self.mode == "register":
                 #  https://github.com/delta-io/delta/issues/2014 (BUG)
                 df = df.withColumn(
@@ -307,6 +346,17 @@ class Bronze(BaseJob):
 
         return df
 
+    def base_transform(self, df: DataFrame) -> DataFrame:
+        df = df.transform(self.extend)
+        df = df.transform(self.add_calculated_columns)
+        df = df.transform(self.add_hash)
+        df = df.transform(self.add_operation)
+        df = df.transform(self.add_source)
+        df = df.transform(self.add_key)
+        df = df.transform(self.add_metadata)
+
+        return df
+
     def create_or_replace_view(self):
         DEFAULT_LOGGER.warning("create or replace view not allowed", extra={"label": self})
 
@@ -395,6 +445,6 @@ class Bronze(BaseJob):
         else:
             super().vacuum()
 
-    def overwrite(self, schedule: Optional[str] = None):
+    def overwrite(self, schedule: Optional[str] = None, invoke: Optional[bool] = False):
         self.truncate()
-        self.run(schedule=schedule)
+        self.run(schedule=schedule, invoke=invoke)

fabricks/core/jobs/get_job.py

@@ -1,10 +1,10 @@
-from typing import Optional, Union, cast, overload
+from typing import Optional, Union, overload
 
 from pyspark.sql.types import Row
 
-from fabricks.core.jobs.base._types import Bronzes, Golds, Silvers, TBronze, TGold, TSilver
+from fabricks.context import Bronzes, Golds, Silvers
 from fabricks.core.jobs.base.job import BaseJob
-from fabricks.core.jobs.get_job_id import get_job_id
+from fabricks.models import get_job_id
 
 
 @overload
@@ -91,7 +91,6 @@ def get_job_internal(
     if step in Bronzes:
         from fabricks.core.jobs.bronze import Bronze
 
-        step = cast(TBronze, step)
         if job_id is not None:
             job = Bronze.from_job_id(step=step, job_id=job_id, conf=conf)
         else:
@@ -102,7 +101,6 @@ def get_job_internal(
     elif step in Silvers:
         from fabricks.core.jobs.silver import Silver
 
-        step = cast(TSilver, step)
         if job_id is not None:
             job = Silver.from_job_id(step=step, job_id=job_id, conf=conf)
         else:
@@ -113,7 +111,6 @@ def get_job_internal(
     elif step in Golds:
         from fabricks.core.jobs.gold import Gold
 
-        step = cast(TGold, step)
         if job_id is not None:
             job = Gold.from_job_id(step=step, job_id=job_id, conf=conf)
         else:

fabricks/core/jobs/get_job_conf.py

@@ -1,97 +1,48 @@
-from typing import Optional, Union, cast, overload
+from typing import Optional, Union, overload
 
 from pyspark.sql.types import Row
 
-from fabricks.context import IS_JOB_CONFIG_FROM_YAML, SPARK
-from fabricks.core.jobs.base._types import Bronzes, Golds, JobConf, Silvers, TBronze, TGold, TSilver, TStep
-from fabricks.core.jobs.get_job_id import get_job_id
+from fabricks.context import IS_JOB_CONFIG_FROM_YAML, SPARK, Bronzes, Golds, Silvers
+from fabricks.models import JobConf, get_job_id
 
 
-def get_job_conf_internal(step: TStep, row: Union[Row, dict]) -> JobConf:
+def get_job_conf_internal(step: str, row: Union[Row, dict]) -> JobConf:
     if isinstance(row, Row):
         row = row.asDict(recursive=True)
 
-    options = row.get("options")
-    table_options = row.get("table_options")
-    check_options = row.get("check_options")
-    spark_options = row.get("spark_options")
-    invoker_options = row.get("invoker_options")
-    extender_options = row.get("extender_options")
-
-    job_id = row.get("job_id", get_job_id(step=step, topic=row["topic"], item=row["item"]))
+    # Add step to row data (job_id will be computed automatically)
+    row["step"] = step
 
+    # Use Pydantic validation - handles nested models and validation automatically
     if step in Bronzes:
-        from fabricks.core.jobs.base._types import JobConfBronze
-
-        assert options is not None, "no option"
-        step = cast(TBronze, step)
-        return JobConfBronze(
-            job_id=job_id,
-            topic=row["topic"],
-            item=row["item"],
-            step=step,
-            options=options,
-            parser_options=row.get("parser_options"),
-            table_options=table_options,
-            check_options=check_options,
-            invoker_options=invoker_options,
-            extender_options=extender_options,
-            spark_options=spark_options,
-            tags=row.get("tags"),
-        )
+        from fabricks.models import JobConfBronze
+
+        return JobConfBronze.model_validate(row)
 
     elif step in Silvers:
-        from fabricks.core.jobs.base._types import JobConfSilver
-
-        assert options is not None, "no option"
-        step = cast(TSilver, step)
-        return JobConfSilver(
-            job_id=job_id,
-            topic=row["topic"],
-            item=row["item"],
-            step=step,
-            options=options,
-            table_options=table_options,
-            check_options=check_options,
-            invoker_options=invoker_options,
-            extender_options=extender_options,
-            spark_options=spark_options,
-            tags=row.get("tags"),
-        )
+        from fabricks.models import JobConfSilver
+
+        return JobConfSilver.model_validate(row)
 
     elif step in Golds:
-        from fabricks.core.jobs.base._types import JobConfGold
-
-        assert options is not None, "no option"
-        step = cast(TGold, step)
-        return JobConfGold(
-            job_id=job_id,
-            topic=row["topic"],
-            item=row["item"],
-            step=step,
-            options=options,
-            table_options=table_options,
-            check_options=check_options,
-            invoker_options=invoker_options,
-            extender_options=extender_options,
-            spark_options=spark_options,
-            tags=row.get("tags"),
-        )
+        from fabricks.models import JobConfGold
+
+        return JobConfGold.model_validate(row)
 
     else:
         raise ValueError(f"{step} not found")
 
 
 @overload
-def get_job_conf(step: TStep, *, job_id: str, row: Optional[Union[Row, dict]] = None) -> JobConf: ...
+def get_job_conf(step: str, *, job_id: str, row: Optional[Union[Row, dict]] = None) -> JobConf: ...
 
 
 @overload
-def get_job_conf(step: TStep, *, topic: str, item: str, row: Optional[Union[Row, dict]] = None) -> JobConf: ...
+def get_job_conf(step: str, *, topic: str, item: str, row: Optional[Union[Row, dict]] = None) -> JobConf: ...
 
 
 def get_job_conf(
-    step: TStep,
+    step: str,
     job_id: Optional[str] = None,
     topic: Optional[str] = None,
     item: Optional[str] = None,