pytrilogy 0.3.148__cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

trilogy/constants.py

@@ -0,0 +1,119 @@
+import random
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from enum import Enum
+from logging import getLogger
+from typing import Any
+
+logger = getLogger("trilogy")
+
+DEFAULT_NAMESPACE = "local"
+
+RECURSIVE_GATING_CONCEPT = "_terminal"
+
+VIRTUAL_CONCEPT_PREFIX = "_virt"
+
+ENV_CACHE_NAME = ".preql_cache.json"
+
+
+class MagicConstants(Enum):
+    NULL = "null"
+    LINE_SEPARATOR = "\n"
+
+
+NULL_VALUE = MagicConstants.NULL
+
+
+@dataclass
+class Optimizations:
+    predicate_pushdown: bool = True
+    datasource_inlining: bool = True
+    constant_inlining: bool = True
+    constant_inline_cutoff: int = 10
+    direct_return: bool = True
+    hide_unused_concepts: bool = True
+
+
+@dataclass
+class Generation:
+    datasource_build_cache: bool = True
+
+
+@dataclass
+class Comments:
+    """Control what is placed in CTE comments"""
+
+    show: bool = False
+    basic: bool = True
+    joins: bool = False
+    nullable: bool = False
+    partial: bool = False
+    source_map: bool = False
+
+
+@dataclass
+class Rendering:
+    """Control how the SQL is rendered"""
+
+    parameters: bool = True
+    concise: bool = False
+
+    @contextmanager
+    def temporary(self, **kwargs: Any):
+        """
+        Context manager to temporarily set attributes and revert them afterwards.
+
+        Usage:
+            r = Rendering()
+            with r.temporary(parameters=False, concise=True):
+                # parameters is False, concise is True here
+                do_something()
+            # parameters and concise are back to their original values
+        """
+        # Store original values
+        original_values = {key: getattr(self, key) for key in kwargs}
+
+        # Set new values
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+        try:
+            yield self
+        finally:
+            # Restore original values
+            for key, value in original_values.items():
+                setattr(self, key, value)
+
+
+@dataclass
+class Parsing:
+    """Control Parsing"""
+
+    strict_name_shadow_enforcement: bool = False
+    select_as_definition: bool = True
+
+
+# TODO: support loading from environments
+@dataclass
+class Config:
+    strict_mode: bool = True
+    human_identifiers: bool = True
+    randomize_cte_names: bool = False
+    validate_missing: bool = True
+    comments: Comments = field(default_factory=Comments)
+    optimizations: Optimizations = field(default_factory=Optimizations)
+    rendering: Rendering = field(default_factory=Rendering)
+    parsing: Parsing = field(default_factory=Parsing)
+    generation: Generation = field(default_factory=Generation)
+
+    @property
+    def show_comments(self) -> bool:
+        return self.comments.show
+
+    def set_random_seed(self, seed: int):
+        random.seed(seed)
+
+
+CONFIG = Config()
+
+CONFIG.set_random_seed(42)

trilogy/core/README.md

@@ -0,0 +1,52 @@
+
+
+
+### Implicit Filtering Design
+
+An implicit filter is any case where the where clause of a statement references more fields than are in the output of a select.
+
+In those cases, those concepts are "promoted" up into a virtual CTE before the final output is returned.
+
+THe `wrinkle` is aggregates - aggregates must have the implicit filter pushed "inside" the aggregate via the select_context
+to ensure that they aggregate appropriately. Applying the condition after creating the aggregate would result in an incorrect 
+aggregation result.
+
+### Case 1 - Basic Scalar Filtering
+
+SELECT
+    abc,
+    def
+where
+    gde = 1
+
+
+We should source
+    ABC,
+    DEF,
+    GDE
+
+filter on GDE = 1;
+
+return results with optional grouping.
+
+
+### Case 2 - Aggregate Filtering
+
+SELECT
+    sum(abc)
+WHERE
+    gde = 1
+
+We should source
+
+    abc | gde =1
+
+;
+
+### Case 3 - Mixed filter
+
+SELECT
+    sum(abc) by def,
+    def,
+    gde
+

trilogy/core/constants.py

@@ -0,0 +1,6 @@
+CONSTANT_DATASET: str = "preql_internal_constant_dataset"
+ALL_ROWS_CONCEPT = "all_rows"
+INTERNAL_NAMESPACE = "__preql_internal"
+PERSISTED_CONCEPT_PREFIX = "__pre_persist"
+UNNEST_NAME = "_unnest_alias"
+WORKING_PATH_CONCEPT = "_env_working_path"

trilogy/core/enums.py

@@ -0,0 +1,454 @@
+from enum import Enum
+
+InfiniteFunctionArgs = -1
+
+
+class DatasourceState(Enum):
+    PUBLISHED = "published"
+    UNPUBLISHED = "unpublished"
+
+
+class CreateMode(Enum):
+    CREATE = "create"
+    CREATE_IF_NOT_EXISTS = "create_if_not_exists"
+    CREATE_OR_REPLACE = "create_or_replace"
+
+
+class UnnestMode(Enum):
+    DIRECT = "direct"
+    CROSS_APPLY = "cross_apply"
+    CROSS_JOIN = "cross_join"
+    CROSS_JOIN_UNNEST = "cross_join_unnest"
+    CROSS_JOIN_ALIAS = "cross_join_alias"
+    PRESTO = "presto"
+    SNOWFLAKE = "snowflake"
+
+
+class GroupMode(Enum):
+    AUTO = "auto"
+    BY_INDEX = "by_index"
+
+
+class ConceptSource(Enum):
+    MANUAL = "manual"
+    CTE = "cte"
+    SELECT = "select"
+    PERSIST_STATEMENT = "persist_statement"
+    AUTO_DERIVED = "auto_derived"
+
+
+class StatementType(Enum):
+    QUERY = "query"
+
+
+class Purpose(Enum):
+    CONSTANT = "const"
+    KEY = "key"
+    PROPERTY = "property"
+    UNIQUE_PROPERTY = "unique_property"
+    METRIC = "metric"
+    ROWSET = "rowset"
+    AUTO = "auto"
+    PARAMETER = "parameter"
+    UNKNOWN = "unknown"
+
+    @classmethod
+    def _missing_(cls, value):
+        if value.lower() == "constant":
+            return Purpose.CONSTANT
+        if value.lower() == "param":
+            return Purpose.PARAMETER
+        return super()._missing_(value)
+
+
+class Derivation(Enum):
+    BASIC = "basic"
+    GROUP_TO = "group_to"
+    WINDOW = "window"
+    AGGREGATE = "aggregate"
+    FILTER = "filter"
+    CONSTANT = "constant"
+    UNNEST = "unnest"
+    UNION = "union"
+    ROOT = "root"
+    ROWSET = "rowset"
+    MULTISELECT = "multiselect"
+    RECURSIVE = "recursive"
+
+
+class Granularity(Enum):
+    SINGLE_ROW = "single_row"
+    MULTI_ROW = "multi_row"
+
+
+class PersistMode(Enum):
+    OVERWRITE = "overwrite"
+    APPEND = "append"
+
+
+class PublishAction(Enum):
+    PUBLISH = "publish"
+    UNPUBLISH = "unpublish"
+
+
+class AddressType(Enum):
+    TABLE = "table"
+    QUERY = "query"
+    PYTHON_SCRIPT = "python_script"
+    CSV = "csv"
+    TSV = "tsv"
+    PARQUET = "parquet"
+    SQL = "sql"
+
+
+class Modifier(Enum):
+    PARTIAL = "Partial"
+    OPTIONAL = "Optional"
+    HIDDEN = "Hidden"
+    NULLABLE = "Nullable"
+
+    @classmethod
+    def _missing_(cls, value):
+        strval = str(value)
+        if strval == "~":
+            return Modifier.PARTIAL
+        elif strval == "?":
+            return Modifier.NULLABLE
+        return super()._missing_(value=strval.capitalize())
+
+    def __lt__(self, other):
+        order = [
+            Modifier.HIDDEN,
+            Modifier.PARTIAL,
+            Modifier.NULLABLE,
+            Modifier.OPTIONAL,
+        ]
+        return order.index(self) < order.index(other)
+
+
+class JoinType(Enum):
+    INNER = "inner"
+    LEFT_OUTER = "left outer"
+    FULL = "full"
+    RIGHT_OUTER = "right outer"
+    CROSS = "cross"
+
+
+class Ordering(Enum):
+    ASCENDING = "asc"
+    DESCENDING = "desc"
+    ASC_NULLS_AUTO = "asc nulls auto"
+    ASC_NULLS_FIRST = "asc nulls first"
+    ASC_NULLS_LAST = "asc nulls last"
+    DESC_NULLS_FIRST = "desc nulls first"
+    DESC_NULLS_LAST = "desc nulls last"
+    DESC_NULLS_AUTO = "desc nulls auto"
+
+
+class WindowType(Enum):
+    ROW_NUMBER = "row_number"
+    RANK = "rank"
+    LAG = "lag"
+    LEAD = "lead"
+    SUM = "sum"
+    MAX = "max"
+    MIN = "min"
+    AVG = "avg"
+    COUNT = "count"
+    COUNT_DISTINCT = "count_distinct"
+
+
+class WindowOrder(Enum):
+    ASCENDING = "top"
+    DESCENDING = "bottom"
+
+
+class FunctionType(Enum):
+    # custom
+    CUSTOM = "custom"
+
+    # structural
+    UNNEST = "unnest"
+    RECURSE_EDGE = "recurse_edge"
+
+    UNION = "union"
+
+    ALIAS = "alias"
+
+    PARENTHETICAL = "parenthetical"
+
+    # Generic
+    CASE = "case"
+    CAST = "cast"
+    CONCAT = "concat"
+    CONSTANT = "constant"
+    TYPED_CONSTANT = "typed_constant"
+    COALESCE = "coalesce"
+    IS_NULL = "isnull"
+    NULLIF = "nullif"
+    BOOL = "bool"
+
+    # COMPLEX
+    INDEX_ACCESS = "index_access"
+    MAP_ACCESS = "map_access"
+    ATTR_ACCESS = "attr_access"
+    STRUCT = "struct"
+    ARRAY = "array"
+    DATE_LITERAL = "date_literal"
+    DATETIME_LITERAL = "datetime_literal"
+
+    # ARRAY
+    ARRAY_DISTINCT = "array_distinct"
+    ARRAY_SUM = "array_sum"
+    ARRAY_SORT = "array_sort"
+    ARRAY_TRANSFORM = "array_transform"
+    ARRAY_TO_STRING = "array_to_string"
+    ARRAY_FILTER = "array_filter"
+    GENERATE_ARRAY = "generate_array"
+
+    # MAP
+    MAP_KEYS = "map_keys"
+    MAP_VALUES = "map_values"
+
+    # TEXT AND MAYBE MORE
+    SPLIT = "split"
+    LENGTH = "len"
+
+    # Maths
+    DIVIDE = "divide"
+    MULTIPLY = "multiply"
+    ADD = "add"
+    SUBTRACT = "subtract"
+    MOD = "mod"
+    ROUND = "round"
+    ABS = "abs"
+    SQRT = "sqrt"
+    RANDOM = "random"
+    FLOOR = "floor"
+    CEIL = "ceil"
+    LOG = "log"
+    POWER = "power"
+
+    # Aggregates
+    ## group is not a real aggregate - it just means group by this + some other set of fields
+    ## but is here as syntax is identical
+    GROUP = "group"
+
+    COUNT = "count"
+    COUNT_DISTINCT = "count_distinct"
+    SUM = "sum"
+    MAX = "max"
+    MIN = "min"
+    AVG = "avg"
+    ARRAY_AGG = "array_agg"
+    BOOL_OR = "bool_or"
+    BOOL_AND = "bool_and"
+    ANY = "any"
+
+    # String
+    LIKE = "like"
+    ILIKE = "ilike"
+    LOWER = "lower"
+    UPPER = "upper"
+    SUBSTRING = "substring"
+    STRPOS = "strpos"
+    CONTAINS = "contains"
+    TRIM = "trim"
+    REPLACE = "replace"
+    HASH = "hash"
+
+    # STRING REGEX
+    REGEXP_CONTAINS = "regexp_contains"
+    REGEXP_EXTRACT = "regexp_extract"
+    REGEXP_REPLACE = "regexp_replace"
+
+    # Dates
+    DATE = "date"
+    DATETIME = "datetime"
+    TIMESTAMP = "timestamp"
+
+    # time
+    SECOND = "second"
+    MINUTE = "minute"
+    HOUR = "hour"
+    DAY = "day"
+    DAY_OF_WEEK = "day_of_week"
+    WEEK = "week"
+    MONTH = "month"
+    QUARTER = "quarter"
+    YEAR = "year"
+    MONTH_NAME = "month_name"
+    DAY_NAME = "day_name"
+
+    DATE_PART = "date_part"
+    DATE_TRUNCATE = "date_truncate"
+    DATE_ADD = "date_add"
+    DATE_SUB = "date_sub"
+    DATE_DIFF = "date_diff"
+    DATE_SPINE = "date_spine"
+
+    # Geography
+    GEO_POINT = "geo_point"
+    GEO_DISTANCE = "geo_distance"
+
+    # UNIX
+    UNIX_TO_TIMESTAMP = "unix_to_timestamp"
+
+    # CONSTANTS
+    CURRENT_DATE = "current_date"
+    CURRENT_DATETIME = "current_datetime"
+    CURRENT_TIMESTAMP = "current_timestamp"
+
+
+class FunctionClass(Enum):
+    AGGREGATE_FUNCTIONS = [
+        FunctionType.MAX,
+        FunctionType.MIN,
+        FunctionType.SUM,
+        FunctionType.AVG,
+        FunctionType.ARRAY_AGG,
+        FunctionType.COUNT,
+        FunctionType.COUNT_DISTINCT,
+        FunctionType.ANY,
+        FunctionType.BOOL_OR,
+        FunctionType.BOOL_AND,
+    ]
+    SINGLE_ROW = [
+        FunctionType.CONSTANT,
+        FunctionType.CURRENT_DATE,
+        FunctionType.CURRENT_DATETIME,
+    ]
+
+    ONE_TO_MANY = [FunctionType.UNNEST, FunctionType.DATE_SPINE]
+
+    RECURSIVE = [FunctionType.RECURSE_EDGE]
+
+
+class Boolean(Enum):
+    TRUE = "true"
+    FALSE = "false"
+
+    @classmethod
+    def _missing_(cls, value):
+        if value is True:
+            return Boolean.TRUE
+        elif value is False:
+            return Boolean.FALSE
+        strval = str(value)
+        if strval.lower() != strval:
+            return Boolean(strval.lower())
+
+
+class BooleanOperator(Enum):
+    AND = "and"
+    OR = "or"
+
+    @classmethod
+    def _missing_(cls, value):
+        strval = str(value)
+        if strval.lower() != strval:
+            return BooleanOperator(strval.lower())
+        return None
+
+
+class ComparisonOperator(Enum):
+    LT = "<"
+    GT = ">"
+    EQ = "="
+    IS = "is"
+    IS_NOT = "is not"
+    GTE = ">="
+    LTE = "<="
+    NE = "!="
+    IN = "in"
+    NOT_IN = "not in"
+    # TODO: deprecate for contains?
+    LIKE = "like"
+    ILIKE = "ilike"
+    CONTAINS = "contains"
+    ELSE = "else"
+
+    def __eq__(self, other):
+        if isinstance(other, str):
+            return self.value == other
+        if not isinstance(other, ComparisonOperator):
+            return False
+        return self.value == other.value
+
+    @classmethod
+    def _missing_(cls, value):
+        if not isinstance(value, list) and " " in str(value):
+            value = str(value).split()
+        if isinstance(value, list):
+            processed = [str(v).lower() for v in value]
+            if processed == ["not", "in"]:
+                return ComparisonOperator.NOT_IN
+            if processed == ["is", "not"]:
+                return ComparisonOperator.IS_NOT
+            if value == ["in"]:
+                return ComparisonOperator.IN
+        if str(value).lower() != str(value):
+            return ComparisonOperator(str(value).lower())
+        return super()._missing_(str(value).lower())
+
+
+class DatePart(Enum):
+    MONTH = "month"
+    YEAR = "year"
+    WEEK = "week"
+    DAY = "day"
+    QUARTER = "quarter"
+    HOUR = "hour"
+    MINUTE = "minute"
+    SECOND = "second"
+    DAY_OF_WEEK = "day_of_week"
+
+    @classmethod
+    def _missing_(cls, value):
+        if isinstance(value, str) and value.lower() != value:
+            return DatePart(value.lower())
+        return super()._missing_(value)
+
+
+class SourceType(Enum):
+    FILTER = "filter"
+    SELECT = "select"
+    ABSTRACT = "abstract"
+    DIRECT_SELECT = "direct_select"
+    GROUP = "group"
+    WINDOW = "window"
+    UNNEST = "unnest"
+    CONSTANT = "constant"
+    ROWSET = "rowset"
+    MERGE = "merge"
+    BASIC = "basic"
+    UNION = "union"
+    RECURSIVE = "recursive"
+
+
+class ShowCategory(Enum):
+    DATASOURCES = "datasources"
+    CONCEPTS = "concepts"
+
+
+class IOType(Enum):
+    CSV = "csv"
+    PARQUET = "parquet"
+
+    @classmethod
+    def _missing_(cls, value):
+        if isinstance(value, str) and value.lower() != value:
+            return IOType(value.lower())
+        return super()._missing_(value)
+
+
+class ValidationScope(Enum):
+    ALL = "all"
+    CONCEPTS = "concepts"
+    DATASOURCES = "datasources"
+
+    @classmethod
+    def _missing_(cls, value):
+        if isinstance(value, str) and value.lower() != value:
+            return ValidationScope(value.lower())
+        return super()._missing_(value)