fabricks 3.0.19__py3-none-any.whl → 4.0.1__py3-none-any.whl

fabricks/core/jobs/base/_types.py

@@ -1,284 +0,0 @@
-from dataclasses import dataclass
-from typing import List, Literal, Optional, TypedDict, Union
-
-from pydantic import BaseModel, ConfigDict, model_validator
-from pyspark.sql.types import StringType, StructField, StructType
-
-from fabricks.cdc.base._types import AllowedChangeDataCaptures
-from fabricks.context import BRONZE, GOLD, SILVER
-from fabricks.core.jobs.get_job_id import get_dependency_id, get_job_id
-from fabricks.core.parsers import ParserOptions
-from fabricks.utils.fdict import FDict
-from fabricks.utils.path import Path
-
-TBronze = Literal["bronze"]
-TSilver = Literal["silver"]
-TGold = Literal["gold"]
-TStep = Literal[TBronze, TSilver, TGold]
-
-Bronzes: List[TBronze] = [b.get("name") for b in BRONZE]
-Silvers: List[TSilver] = [s.get("name") for s in SILVER]
-Golds: List[TGold] = [g.get("name") for g in GOLD]
-Steps: List[TStep] = Bronzes + Silvers + Golds
-
-AllowedModesBronze = Literal["memory", "append", "register"]
-AllowedModesSilver = Literal["memory", "append", "latest", "update", "combine"]
-AllowedModesGold = Literal["memory", "append", "complete", "update", "invoke"]
-AllowedModes = Literal[AllowedModesBronze, AllowedModesSilver, AllowedModesGold]
-
-AllowedFileFormats = Literal["json_array", "json", "jsonl", "csv", "parquet", "delta"]
-AllowedOperations = Literal["upsert", "reload", "delete"]
-AllowedTypes = Literal["manual", "default"]
-AllowedOrigins = Literal["parser", "job"]
-
-AllowedConstraintOptions = Literal["not enforced", "deferrable", "initially deferred", "norely", "rely"]
-AllowedForeignKeyOptions = Literal["match full", "on update no action", "on delete no action"]
-
-
-class SparkOptions(TypedDict):
-    sql: Optional[dict[str, str]]
-    conf: Optional[dict[str, str]]
-
-
-class ForeignKeyOptions(TypedDict):
-    foreign_key: Optional[AllowedForeignKeyOptions]
-    constraint: Optional[AllowedConstraintOptions]
-
-
-class PrimaryKeyOptions(TypedDict):
-    constraint: Optional[AllowedConstraintOptions]
-
-
-class ForeignKey(TypedDict):
-    keys: List[str]
-    reference: str
-    options: Optional[ForeignKeyOptions]
-
-
-class PrimaryKey(TypedDict):
-    keys: List[str]
-    options: Optional[PrimaryKeyOptions]
-
-
-class TableOptions(TypedDict):
-    identity: Optional[bool]
-    liquid_clustering: Optional[bool]
-    partition_by: Optional[List[str]]
-    zorder_by: Optional[List[str]]
-    cluster_by: Optional[List[str]]
-    powerbi: Optional[bool]
-    maximum_compatibility: Optional[bool]
-    bloomfilter_by: Optional[List[str]]
-    constraints: Optional[dict[str, str]]
-    properties: Optional[dict[str, str]]
-    comment: Optional[str]
-    calculated_columns: Optional[dict[str, str]]
-    masks: Optional[dict[str, str]]
-    comments: Optional[dict[str, str]]
-    retention_days: Optional[int]
-    primary_key: Optional[dict[str, PrimaryKey]]
-    foreign_keys: Optional[dict[str, ForeignKey]]
-
-
-class _InvokeOptions(TypedDict):
-    notebook: str
-    timeout: int
-    arguments: Optional[dict[str, str]]
-
-
-class InvokerOptions(TypedDict):
-    pre_run: Optional[List[_InvokeOptions]]
-    run: Optional[List[_InvokeOptions]]
-    post_run: Optional[List[_InvokeOptions]]
-
-
-class ExtenderOptions(TypedDict):
-    extender: str
-    arguments: Optional[dict[str, str]]
-
-
-class CheckOptions(TypedDict):
-    skip: Optional[bool]
-    pre_run: Optional[bool]
-    post_run: Optional[bool]
-    min_rows: Optional[int]
-    max_rows: Optional[int]
-    count_must_equal: Optional[str]
-
-
-class BronzeOptions(TypedDict):
-    type: Optional[AllowedTypes]
-    mode: AllowedModesBronze
-    uri: str
-    parser: str
-    source: str
-    keys: Optional[List[str]]
-    # default
-    parents: Optional[List[str]]
-    filter_where: Optional[str]
-    optimize: Optional[bool]
-    compute_statistics: Optional[bool]
-    vacuum: Optional[bool]
-    no_drop: Optional[bool]
-    # extra
-    encrypted_columns: Optional[List[str]]
-    calculated_columns: Optional[dict[str, str]]
-    operation: Optional[AllowedOperations]
-    timeout: Optional[int]
-
-
-class SilverOptions(TypedDict):
-    type: Optional[AllowedTypes]
-    mode: AllowedModesSilver
-    change_data_capture: AllowedChangeDataCaptures
-    # default
-    parents: Optional[List[str]]
-    filter_where: Optional[str]
-    optimize: Optional[bool]
-    compute_statistics: Optional[bool]
-    vacuum: Optional[bool]
-    no_drop: Optional[bool]
-    # extra
-    deduplicate: Optional[bool]
-    stream: Optional[bool]
-    # else
-    order_duplicate_by: Optional[dict[str, str]]
-    timeout: Optional[int]
-
-
-class GoldOptions(TypedDict):
-    type: Optional[AllowedTypes]
-    mode: AllowedModesGold
-    change_data_capture: AllowedChangeDataCaptures
-    update_where: Optional[str]
-    # default
-    parents: Optional[List[str]]
-    optimize: Optional[bool]
-    compute_statistics: Optional[bool]
-    vacuum: Optional[bool]
-    no_drop: Optional[bool]
-    # extra
-    deduplicate: Optional[bool]  # remove duplicates on the keys and on the hash
-    rectify_as_upserts: Optional[bool]  # convert reloads into upserts and deletes
-    correct_valid_from: Optional[bool]  # update valid_from to '1900-01-01' for the first timestamp
-    persist_last_timestamp: Optional[bool]  # persist the last timestamp to be used as a watermark for the next run
-    # delete_missing: Optional[bool]  # delete missing records on update (to be implemented)
-    # else
-    table: Optional[str]
-    notebook: Optional[bool]
-    requirements: Optional[bool]
-    timeout: Optional[int]
-    metadata: Optional[bool]
-
-
-StepOptions = Union[BronzeOptions, SilverOptions, GoldOptions]
-
-
-@dataclass
-class BaseJobConf:
-    job_id: str
-    topic: str
-    item: str
-
-
-@dataclass
-class JobConfBronze(BaseJobConf):
-    step: TBronze
-    options: BronzeOptions
-    table_options: Optional[TableOptions] = None
-    parser_options: Optional[ParserOptions] = None
-    check_options: Optional[CheckOptions] = None
-    spark_options: Optional[SparkOptions] = None
-    invoker_options: Optional[InvokerOptions] = None
-    extender_options: Optional[List[ExtenderOptions]] = None
-    tags: Optional[List[str]] = None
-    comment: Optional[str] = None
-
-
-@dataclass
-class JobConfSilver(BaseJobConf):
-    step: TSilver
-    options: SilverOptions
-    table_options: Optional[TableOptions] = None
-    check_options: Optional[CheckOptions] = None
-    spark_options: Optional[SparkOptions] = None
-    invoker_options: Optional[InvokerOptions] = None
-    extender_options: Optional[List[ExtenderOptions]] = None
-    tags: Optional[List[str]] = None
-    comment: Optional[str] = None
-
-
-@dataclass
-class JobConfGold(BaseJobConf):
-    step: TGold
-    options: Optional[GoldOptions]
-    table_options: Optional[TableOptions] = None
-    check_options: Optional[CheckOptions] = None
-    spark_options: Optional[SparkOptions] = None
-    invoker_options: Optional[InvokerOptions] = None
-    extender_options: Optional[List[ExtenderOptions]] = None
-    tags: Optional[List[str]] = None
-    comment: Optional[str] = None
-
-
-JobConf = Union[JobConfBronze, JobConfSilver, JobConfGold]
-
-
-@dataclass
-class Paths:
-    storage: Path
-    tmp: Path
-    checkpoints: Path
-    commits: Path
-    schema: Path
-    runtime: Path
-
-
-@dataclass
-class Options:
-    job: FDict
-    check: FDict
-    table: FDict
-    spark: FDict
-    invokers: FDict
-    extenders: List
-
-
-class JobDependency(BaseModel):
-    model_config = ConfigDict(extra="forbid", frozen=True)
-    origin: AllowedOrigins
-    job_id: str
-    parent: str
-    parent_id: str
-    dependency_id: str
-
-    def __str__(self) -> str:
-        return f"{self.job_id} -> {self.parent}"
-
-    @model_validator(mode="after")
-    def check_no_circular_dependency(self):
-        if self.job_id == self.parent_id:
-            raise ValueError("Circular dependency detected")
-        return self
-
-    @staticmethod
-    def from_parts(job_id: str, parent: str, origin: AllowedOrigins):
-        parent = parent.removesuffix("__current")
-        return JobDependency(
-            job_id=job_id,
-            origin=origin,
-            parent=parent,
-            parent_id=get_job_id(job=parent),
-            dependency_id=get_dependency_id(parent=parent, job_id=job_id),
-        )
-
-
-SchemaDependencies = StructType(
-    [
-        StructField("dependency_id", StringType(), True),
-        StructField("origin", StringType(), True),
-        StructField("job_id", StringType(), True),
-        StructField("parent_id", StringType(), True),
-        StructField("parent", StringType(), True),
-    ]
-)

fabricks/core/parsers/_types.py

@@ -1,6 +0,0 @@
-from typing import Optional, TypedDict
-
-
-class ParserOptions(TypedDict):
-    file_format: Optional[str]
-    read_options: Optional[dict[str, str]]

fabricks/utils/fdict.py

@@ -1,240 +0,0 @@
-from typing import Any, Callable, Dict, List, Optional, TypeVar, Union, overload
-
-T = TypeVar("T")
-
-
-class FDict:
-    """
-    A flexible dictionary wrapper that provides type-safe access to nested data structures
-    with convenient conversion methods.
-    """
-
-    def __init__(self, options: Union[Dict[str, Any], Any, None] = None):
-        """
-        Initialize FDict with a dictionary of options.
-
-        Args:
-            options: Input dictionary. If None, creates an empty dictionary.
-        """
-        self.options = options if options is not None else {}
-
-    def __getitem__(self, key: str) -> Any:
-        """Enable dictionary-like access with [] operator."""
-        return self.options[key]
-
-    def __setitem__(self, key: str, value: Any) -> None:
-        """Enable dictionary-like value setting with [] operator."""
-        self.options[key] = value
-
-    def __contains__(self, key: str) -> bool:
-        """Enable 'in' operator for membership testing."""
-        return key in self.options
-
-    def __repr__(self) -> str:
-        """Return string representation of the FDict."""
-        return f"FDict({self.options})"
-
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert FDict to a regular dictionary."""
-        return self.options
-
-    @overload
-    def get(self, key: str) -> Optional[Any]: ...
-
-    @overload
-    def get(self, key: str, default: T) -> Union[Any, T]: ...
-
-    def get(self, key: str, default: Any = None) -> Any:
-        """
-        Get a value from the dictionary with an optional default.
-
-        Args:
-            key: The key to look up
-            default: Value to return if key is not found
-
-        Returns:
-            The value associated with the key or the default value
-        """
-        return self.options.get(key, default)
-
-    def get_list(self, key: str, default: Optional[List[Any]] = None) -> List[Any]:
-        """
-        Get a value as a list, converting single items to a single-item list.
-
-        Args:
-            key: The key to look up
-            default: Default value if key is not found
-
-        Returns:
-            A list containing the value(s)
-        """
-        values = self.options.get(key, default if default is not None else [])
-        if values is None:
-            return []
-
-        return [values] if not isinstance(values, list) else values
-
-    def get_boolean(self, key: str, default: Optional[bool] = None) -> Optional[bool]:
-        """
-        Get a value as a boolean, with string conversion support.
-
-        Args:
-            key: The key to look up
-            default: Default value if key is not found
-
-        Returns:
-            Boolean value of the key, or default if key not found
-        """
-        value = self.options.get(key)
-
-        if value is None:
-            return default
-        if isinstance(value, bool):
-            return value
-        if isinstance(value, str):
-            return value.lower() in ("true", "1", "yes", "on")
-
-        return bool(value)
-
-    def get_dict(self, key: str, default: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
-        """
-        Get a nested dictionary, with a default empty dict if not found.
-
-        Args:
-            key: The key to look up
-            default: Default value if key is not found
-
-        Returns:
-            Dictionary value of the key, or default if key not found
-        """
-        return self.options.get(key, default if default is not None else {})
-
-    def get_nested(self, *keys: str, default: Any = None) -> Any:
-        """
-        Access nested dictionary values using a sequence of keys.
-
-        Args:
-            *keys: Sequence of keys to traverse
-            default: Default value if path not found
-
-        Returns:
-            Value at the nested path, or default if path not found
-        """
-        current = self.options
-        for key in keys:
-            if not isinstance(current, dict):
-                return default
-            if key not in current:
-                return default
-            current = current[key]
-
-        return current
-
-    def set_nested(self, *keys: str, value: Any) -> None:
-        """
-        Set a value in a nested dictionary path, creating intermediate dictionaries as needed.
-
-        Args:
-            *keys: Sequence of keys defining the path
-            value: Value to set at the path
-        """
-        current = self.options
-        for key in keys[:-1]:
-            current = current.setdefault(key, {})
-
-        current[keys[-1]] = value
-
-    def filter(self, predicate: Callable[[str, Any], bool]) -> "FDict":
-        """
-        Create a new FDict with key-value pairs that satisfy the predicate function.
-
-        Args:
-            predicate: Lambda function that takes key and value as arguments and returns bool
-
-        Returns:
-            New FDict containing only the filtered key-value pairs
-
-        Example:
-            # Get all items with numeric values greater than 10
-            filtered = fdict.filter(lambda k, v: isinstance(v, (int, float)) and v > 10)
-        """
-        filtered_dict = {k: v for k, v in self.options.items() if predicate(k, v)}
-        return FDict(filtered_dict)
-
-    def filter_keys(self, predicate: Callable[[str], bool]) -> "FDict":
-        """
-        Create a new FDict with keys that satisfy the predicate function.
-
-        Args:
-            predicate: Lambda function that takes key as argument and returns bool
-
-        Returns:
-            New FDict containing only the filtered keys
-
-        Example:
-            # Get all items with keys starting with 'user_'
-            filtered = fdict.filter_keys(lambda k: k.startswith('user_'))
-        """
-        return self.filter(lambda k, _: predicate(k))
-
-    def filter_values(self, predicate: Callable[[Any], bool]) -> "FDict":
-        """
-        Create a new FDict with values that satisfy the predicate function.
-
-        Args:
-            predicate: Lambda function that takes value as argument and returns bool
-
-        Returns:
-            New FDict containing only the filtered values
-
-        Example:
-            # Get all items with string values
-            filtered = fdict.filter_values(lambda v: isinstance(v, str))
-        """
-        return self.filter(lambda _, v: predicate(v))
-
-    def map_values(self, transform: Callable[[Any], Any]) -> "FDict":
-        """
-        Create a new FDict with transformed values using the provided function.
-
-        Args:
-            transform: Lambda function that takes a value and returns transformed value
-
-        Returns:
-            New FDict containing transformed values
-
-        Example:
-            # Convert all string values to uppercase
-            transformed = fdict.map_values(lambda v: v.upper() if isinstance(v, str) else v)
-        """
-        transformed_dict = {k: transform(v) for k, v in self.options.items()}
-        return FDict(transformed_dict)
-
-    def deep_filter(self, predicate: Callable[[str, Any], bool]) -> "FDict":
-        """
-        Recursively filter nested dictionaries using the predicate function.
-
-        Args:
-            predicate: Lambda function that takes key and value as arguments and returns bool
-
-        Returns:
-            New FDict with filtered nested structure
-
-        Example:
-            # Filter all nested numeric values greater than 10
-            filtered = fdict.deep_filter(lambda k, v:
-                not isinstance(v, dict) and isinstance(v, (int, float)) and v > 10)
-        """
-
-        def filter_recursive(d: Dict[str, Any]) -> Dict[str, Any]:
-            result = {}
-            for k, v in d.items():
-                if isinstance(v, dict):
-                    filtered_nested = filter_recursive(v)
-                    if filtered_nested:  # Only include non-empty nested dicts
-                        result[k] = filtered_nested
-                elif predicate(k, v):
-                    result[k] = v
-            return result
-
-        return FDict(filter_recursive(self.options))

fabricks/utils/pydantic.py

@@ -1,94 +0,0 @@
-from typing import List, Literal, Type, TypeVar, Union, get_args, get_origin
-
-import yaml
-from pydantic import BaseModel as PydanticBaseModel
-from pydantic import parse_obj_as
-from pyspark.sql import DataFrame
-from pyspark.sql.types import (
-    ArrayType,
-    BooleanType,
-    DoubleType,
-    LongType,
-    MapType,
-    Row,
-    StringType,
-    StructField,
-    StructType,
-)
-
-from fabricks.context import SPARK
-
-types_ = {
-    str: StringType(),
-    bool: BooleanType(),
-    float: DoubleType(),
-    int: LongType(),
-    dict: MapType(StringType(), StringType()),
-}
-T = TypeVar("T")
-
-
-def _to_spark_type(type_):
-    if type_ in types_:
-        return types_[type_]
-
-    origin = get_origin(type_)
-    args = get_args(type_)
-    if origin is Literal:
-        return StringType()
-    if origin is list:
-        return ArrayType(_to_spark_type(args[0]))
-    if origin is dict:
-        return MapType(
-            _to_spark_type(args[0]),
-            _to_spark_type(args[1]),
-        )
-
-    if issubclass(type_, PydanticBaseModel):
-        return _schema_pyspark(type_)
-
-    raise ValueError(type_)
-
-
-def _schema_pyspark(model):
-    fields = []
-    for field in model.__fields__.values():
-        type_ = field.outer_type_
-        spark_type_ = _to_spark_type(type_)
-        f = StructField(
-            name=field.name,
-            dataType=spark_type_,  # type: ignore
-            nullable=not field.required,
-        )
-        fields.append(f)
-    return StructType(fields)
-
-
-class FBaseModel(PydanticBaseModel):
-    @classmethod
-    def from_yaml(cls: Type[T], path: str) -> Union[T, List[T]]:
-        with open(path, encoding="utf-8") as f:
-            y = yaml.safe_load(f)
-            if isinstance(y, List):
-                return parse_obj_as(List[cls], y)
-            else:
-                return parse_obj_as(cls, y)
-
-    @classmethod
-    def from_row(cls: Type[T], row: Row) -> T:
-        return parse_obj_as(cls, row.asDict(True))
-
-    @classmethod
-    def from_dataframe(cls: Type[T], df: DataFrame) -> List[T]:
-        return [parse_obj_as(cls, row.asDict(True)) for row in df.collect()]
-
-    def schema_pyspark(self):
-        return _schema_pyspark(self)
-
-    @staticmethod
-    def get_dataframe(data: Union[T, List[T]]) -> DataFrame:
-        if isinstance(data, List):
-            df = SPARK.createDataFrame([d.dict() for d in data], data[0].schema_pyspark())  # type: ignore
-        else:
-            df = SPARK.createDataFrame([data.dict()], data.schema_pyspark())  # type: ignore
-        return df

fabricks/utils/schema/__init__.py

@@ -1,7 +0,0 @@
-from fabricks.utils.schema.get_json_schema_for_type import get_json_schema_for_type
-from fabricks.utils.schema.get_schema_for_type import get_schema_for_type
-
-__all__ = [
-    "get_json_schema_for_type",
-    "get_schema_for_type",
-]