pytrilogy 0.3.138__cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

trilogy/core/README.md

@@ -0,0 +1,52 @@
+
+
+
+### Implicit Filtering Design
+
+An implicit filter is any case where the where clause of a statement references more fields than are in the output of a select.
+
+In those cases, those concepts are "promoted" up into a virtual CTE before the final output is returned.
+
+THe `wrinkle` is aggregates - aggregates must have the implicit filter pushed "inside" the aggregate via the select_context
+to ensure that they aggregate appropriately. Applying the condition after creating the aggregate would result in an incorrect 
+aggregation result.
+
+### Case 1 - Basic Scalar Filtering
+
+SELECT
+    abc,
+    def
+where
+    gde = 1
+
+
+We should source
+    ABC,
+    DEF,
+    GDE
+
+filter on GDE = 1;
+
+return results with optional grouping.
+
+
+### Case 2 - Aggregate Filtering
+
+SELECT
+    sum(abc)
+WHERE
+    gde = 1
+
+We should source
+
+    abc | gde =1
+
+;
+
+### Case 3 - Mixed filter
+
+SELECT
+    sum(abc) by def,
+    def,
+    gde
+

trilogy/core/constants.py

@@ -0,0 +1,6 @@
+CONSTANT_DATASET: str = "preql_internal_constant_dataset"
+ALL_ROWS_CONCEPT = "all_rows"
+INTERNAL_NAMESPACE = "__preql_internal"
+PERSISTED_CONCEPT_PREFIX = "__pre_persist"
+UNNEST_NAME = "_unnest_alias"
+WORKING_PATH_CONCEPT = "_env_working_path"

trilogy/core/enums.py

@@ -0,0 +1,443 @@
+from enum import Enum
+
+InfiniteFunctionArgs = -1
+
+
+class DatasourceState(Enum):
+    PUBLISHED = "published"
+    UNPUBLISHED = "unpublished"
+
+
+class CreateMode(Enum):
+    CREATE = "create"
+    CREATE_IF_NOT_EXISTS = "create_if_not_exists"
+    CREATE_OR_REPLACE = "create_or_replace"
+
+
+class UnnestMode(Enum):
+    DIRECT = "direct"
+    CROSS_APPLY = "cross_apply"
+    CROSS_JOIN = "cross_join"
+    CROSS_JOIN_UNNEST = "cross_join_unnest"
+    CROSS_JOIN_ALIAS = "cross_join_alias"
+    PRESTO = "presto"
+    SNOWFLAKE = "snowflake"
+
+
+class GroupMode(Enum):
+    AUTO = "auto"
+    BY_INDEX = "by_index"
+
+
+class ConceptSource(Enum):
+    MANUAL = "manual"
+    CTE = "cte"
+    SELECT = "select"
+    PERSIST_STATEMENT = "persist_statement"
+    AUTO_DERIVED = "auto_derived"
+
+
+class StatementType(Enum):
+    QUERY = "query"
+
+
+class Purpose(Enum):
+    CONSTANT = "const"
+    KEY = "key"
+    PROPERTY = "property"
+    UNIQUE_PROPERTY = "unique_property"
+    METRIC = "metric"
+    ROWSET = "rowset"
+    AUTO = "auto"
+    PARAMETER = "parameter"
+    UNKNOWN = "unknown"
+
+    @classmethod
+    def _missing_(cls, value):
+        if value.lower() == "constant":
+            return Purpose.CONSTANT
+        if value.lower() == "param":
+            return Purpose.PARAMETER
+        return super()._missing_(value)
+
+
+class Derivation(Enum):
+    BASIC = "basic"
+    GROUP_TO = "group_to"
+    WINDOW = "window"
+    AGGREGATE = "aggregate"
+    FILTER = "filter"
+    CONSTANT = "constant"
+    UNNEST = "unnest"
+    UNION = "union"
+    ROOT = "root"
+    ROWSET = "rowset"
+    MULTISELECT = "multiselect"
+    RECURSIVE = "recursive"
+
+
+class Granularity(Enum):
+    SINGLE_ROW = "single_row"
+    MULTI_ROW = "multi_row"
+
+
+class PersistMode(Enum):
+    OVERWRITE = "overwrite"
+    APPEND = "append"
+
+
+class PublishAction(Enum):
+    PUBLISH = "publish"
+    UNPUBLISH = "unpublish"
+
+
+class Modifier(Enum):
+    PARTIAL = "Partial"
+    OPTIONAL = "Optional"
+    HIDDEN = "Hidden"
+    NULLABLE = "Nullable"
+
+    @classmethod
+    def _missing_(cls, value):
+        strval = str(value)
+        if strval == "~":
+            return Modifier.PARTIAL
+        elif strval == "?":
+            return Modifier.NULLABLE
+        return super()._missing_(value=strval.capitalize())
+
+    def __lt__(self, other):
+        order = [
+            Modifier.HIDDEN,
+            Modifier.PARTIAL,
+            Modifier.NULLABLE,
+            Modifier.OPTIONAL,
+        ]
+        return order.index(self) < order.index(other)
+
+
+class JoinType(Enum):
+    INNER = "inner"
+    LEFT_OUTER = "left outer"
+    FULL = "full"
+    RIGHT_OUTER = "right outer"
+    CROSS = "cross"
+
+
+class Ordering(Enum):
+    ASCENDING = "asc"
+    DESCENDING = "desc"
+    ASC_NULLS_AUTO = "asc nulls auto"
+    ASC_NULLS_FIRST = "asc nulls first"
+    ASC_NULLS_LAST = "asc nulls last"
+    DESC_NULLS_FIRST = "desc nulls first"
+    DESC_NULLS_LAST = "desc nulls last"
+    DESC_NULLS_AUTO = "desc nulls auto"
+
+
+class WindowType(Enum):
+    ROW_NUMBER = "row_number"
+    RANK = "rank"
+    LAG = "lag"
+    LEAD = "lead"
+    SUM = "sum"
+    MAX = "max"
+    MIN = "min"
+    AVG = "avg"
+    COUNT = "count"
+    COUNT_DISTINCT = "count_distinct"
+
+
+class WindowOrder(Enum):
+    ASCENDING = "top"
+    DESCENDING = "bottom"
+
+
+class FunctionType(Enum):
+    # custom
+    CUSTOM = "custom"
+
+    # structural
+    UNNEST = "unnest"
+    RECURSE_EDGE = "recurse_edge"
+
+    UNION = "union"
+
+    ALIAS = "alias"
+
+    PARENTHETICAL = "parenthetical"
+
+    # Generic
+    CASE = "case"
+    CAST = "cast"
+    CONCAT = "concat"
+    CONSTANT = "constant"
+    TYPED_CONSTANT = "typed_constant"
+    COALESCE = "coalesce"
+    IS_NULL = "isnull"
+    NULLIF = "nullif"
+    BOOL = "bool"
+
+    # COMPLEX
+    INDEX_ACCESS = "index_access"
+    MAP_ACCESS = "map_access"
+    ATTR_ACCESS = "attr_access"
+    STRUCT = "struct"
+    ARRAY = "array"
+    DATE_LITERAL = "date_literal"
+    DATETIME_LITERAL = "datetime_literal"
+
+    # ARRAY
+    ARRAY_DISTINCT = "array_distinct"
+    ARRAY_SUM = "array_sum"
+    ARRAY_SORT = "array_sort"
+    ARRAY_TRANSFORM = "array_transform"
+    ARRAY_TO_STRING = "array_to_string"
+    ARRAY_FILTER = "array_filter"
+    GENERATE_ARRAY = "generate_array"
+
+    # MAP
+    MAP_KEYS = "map_keys"
+    MAP_VALUES = "map_values"
+
+    # TEXT AND MAYBE MORE
+    SPLIT = "split"
+    LENGTH = "len"
+
+    # Maths
+    DIVIDE = "divide"
+    MULTIPLY = "multiply"
+    ADD = "add"
+    SUBTRACT = "subtract"
+    MOD = "mod"
+    ROUND = "round"
+    ABS = "abs"
+    SQRT = "sqrt"
+    RANDOM = "random"
+    FLOOR = "floor"
+    CEIL = "ceil"
+    LOG = "log"
+    POWER = "power"
+
+    # Aggregates
+    ## group is not a real aggregate - it just means group by this + some other set of fields
+    ## but is here as syntax is identical
+    GROUP = "group"
+
+    COUNT = "count"
+    COUNT_DISTINCT = "count_distinct"
+    SUM = "sum"
+    MAX = "max"
+    MIN = "min"
+    AVG = "avg"
+    ARRAY_AGG = "array_agg"
+    BOOL_OR = "bool_or"
+    BOOL_AND = "bool_and"
+    ANY = "any"
+
+    # String
+    LIKE = "like"
+    ILIKE = "ilike"
+    LOWER = "lower"
+    UPPER = "upper"
+    SUBSTRING = "substring"
+    STRPOS = "strpos"
+    CONTAINS = "contains"
+    TRIM = "trim"
+    REPLACE = "replace"
+    HASH = "hash"
+
+    # STRING REGEX
+    REGEXP_CONTAINS = "regexp_contains"
+    REGEXP_EXTRACT = "regexp_extract"
+    REGEXP_REPLACE = "regexp_replace"
+
+    # Dates
+    DATE = "date"
+    DATETIME = "datetime"
+    TIMESTAMP = "timestamp"
+
+    # time
+    SECOND = "second"
+    MINUTE = "minute"
+    HOUR = "hour"
+    DAY = "day"
+    DAY_OF_WEEK = "day_of_week"
+    WEEK = "week"
+    MONTH = "month"
+    QUARTER = "quarter"
+    YEAR = "year"
+    MONTH_NAME = "month_name"
+    DAY_NAME = "day_name"
+
+    DATE_PART = "date_part"
+    DATE_TRUNCATE = "date_truncate"
+    DATE_ADD = "date_add"
+    DATE_SUB = "date_sub"
+    DATE_DIFF = "date_diff"
+    DATE_SPINE = "date_spine"
+
+    # Geography
+    GEO_POINT = "geo_point"
+    GEO_DISTANCE = "geo_distance"
+
+    # UNIX
+    UNIX_TO_TIMESTAMP = "unix_to_timestamp"
+
+    # CONSTANTS
+    CURRENT_DATE = "current_date"
+    CURRENT_DATETIME = "current_datetime"
+    CURRENT_TIMESTAMP = "current_timestamp"
+
+
+class FunctionClass(Enum):
+    AGGREGATE_FUNCTIONS = [
+        FunctionType.MAX,
+        FunctionType.MIN,
+        FunctionType.SUM,
+        FunctionType.AVG,
+        FunctionType.ARRAY_AGG,
+        FunctionType.COUNT,
+        FunctionType.COUNT_DISTINCT,
+        FunctionType.ANY,
+        FunctionType.BOOL_OR,
+        FunctionType.BOOL_AND,
+    ]
+    SINGLE_ROW = [
+        FunctionType.CONSTANT,
+        FunctionType.CURRENT_DATE,
+        FunctionType.CURRENT_DATETIME,
+    ]
+
+    ONE_TO_MANY = [FunctionType.UNNEST, FunctionType.DATE_SPINE]
+
+    RECURSIVE = [FunctionType.RECURSE_EDGE]
+
+
+class Boolean(Enum):
+    TRUE = "true"
+    FALSE = "false"
+
+    @classmethod
+    def _missing_(cls, value):
+        if value is True:
+            return Boolean.TRUE
+        elif value is False:
+            return Boolean.FALSE
+        strval = str(value)
+        if strval.lower() != strval:
+            return Boolean(strval.lower())
+
+
+class BooleanOperator(Enum):
+    AND = "and"
+    OR = "or"
+
+    @classmethod
+    def _missing_(cls, value):
+        strval = str(value)
+        if strval.lower() != strval:
+            return BooleanOperator(strval.lower())
+        return None
+
+
+class ComparisonOperator(Enum):
+    LT = "<"
+    GT = ">"
+    EQ = "="
+    IS = "is"
+    IS_NOT = "is not"
+    GTE = ">="
+    LTE = "<="
+    NE = "!="
+    IN = "in"
+    NOT_IN = "not in"
+    # TODO: deprecate for contains?
+    LIKE = "like"
+    ILIKE = "ilike"
+    CONTAINS = "contains"
+    ELSE = "else"
+
+    def __eq__(self, other):
+        if isinstance(other, str):
+            return self.value == other
+        if not isinstance(other, ComparisonOperator):
+            return False
+        return self.value == other.value
+
+    @classmethod
+    def _missing_(cls, value):
+        if not isinstance(value, list) and " " in str(value):
+            value = str(value).split()
+        if isinstance(value, list):
+            processed = [str(v).lower() for v in value]
+            if processed == ["not", "in"]:
+                return ComparisonOperator.NOT_IN
+            if processed == ["is", "not"]:
+                return ComparisonOperator.IS_NOT
+            if value == ["in"]:
+                return ComparisonOperator.IN
+        if str(value).lower() != str(value):
+            return ComparisonOperator(str(value).lower())
+        return super()._missing_(str(value).lower())
+
+
+class DatePart(Enum):
+    MONTH = "month"
+    YEAR = "year"
+    WEEK = "week"
+    DAY = "day"
+    QUARTER = "quarter"
+    HOUR = "hour"
+    MINUTE = "minute"
+    SECOND = "second"
+    DAY_OF_WEEK = "day_of_week"
+
+    @classmethod
+    def _missing_(cls, value):
+        if isinstance(value, str) and value.lower() != value:
+            return DatePart(value.lower())
+        return super()._missing_(value)
+
+
+class SourceType(Enum):
+    FILTER = "filter"
+    SELECT = "select"
+    ABSTRACT = "abstract"
+    DIRECT_SELECT = "direct_select"
+    GROUP = "group"
+    WINDOW = "window"
+    UNNEST = "unnest"
+    CONSTANT = "constant"
+    ROWSET = "rowset"
+    MERGE = "merge"
+    BASIC = "basic"
+    UNION = "union"
+    RECURSIVE = "recursive"
+
+
+class ShowCategory(Enum):
+    DATASOURCES = "datasources"
+    CONCEPTS = "concepts"
+
+
+class IOType(Enum):
+    CSV = "csv"
+
+    @classmethod
+    def _missing_(cls, value):
+        if isinstance(value, str) and value.lower() != value:
+            return IOType(value.lower())
+        return super()._missing_(value)
+
+
+class ValidationScope(Enum):
+    ALL = "all"
+    CONCEPTS = "concepts"
+    DATASOURCES = "datasources"
+
+    @classmethod
+    def _missing_(cls, value):
+        if isinstance(value, str) and value.lower() != value:
+            return ValidationScope(value.lower())
+        return super()._missing_(value)

trilogy/core/env_processor.py

@@ -0,0 +1,120 @@
+from trilogy.core.graph_models import (
+    ReferenceGraph,
+    concept_to_node,
+    datasource_to_node,
+)
+from trilogy.core.models.build import BuildConcept, BuildDatasource
+from trilogy.core.models.build_environment import BuildEnvironment
+
+
+def add_concept(
+    concept: BuildConcept,
+    g: ReferenceGraph,
+    concept_mapping: dict[str, BuildConcept],
+    default_concept_graph: dict[str, BuildConcept],
+    seen: set[str],
+):
+
+    # if we have sources, recursively add them
+    node_name = concept_to_node(concept)
+    if node_name in seen:
+        return
+    seen.add(node_name)
+    g.concepts[node_name] = concept
+    g.add_node(node_name)
+    if concept.concept_arguments:
+        for source in concept.concept_arguments:
+            if not isinstance(source, BuildConcept):
+                raise ValueError(
+                    f"Invalid non-build concept {source} passed into graph generation from {concept}"
+                )
+            generic = get_default_grain_concept(source, default_concept_graph)
+            generic_node = concept_to_node(generic)
+            add_concept(generic, g, concept_mapping, default_concept_graph, seen)
+
+            g.add_edge(generic_node, node_name, fast=True)
+    for ps_address in concept.pseudonyms:
+        if ps_address not in concept_mapping:
+            raise SyntaxError(f"Concept {concept} has invalid pseudonym {ps_address}")
+        pseudonym = concept_mapping[ps_address]
+        pseudonym = get_default_grain_concept(pseudonym, default_concept_graph)
+        pseudonym_node = concept_to_node(pseudonym)
+        if (pseudonym_node, node_name) in g.edges and (
+            node_name,
+            pseudonym_node,
+        ) in g.edges:
+            continue
+        if pseudonym_node.split("@")[0] == node_name.split("@")[0]:
+            continue
+        g.add_edge(pseudonym_node, node_name, fast=True)
+        g.add_edge(node_name, pseudonym_node, fast=True)
+        g.pseudonyms.add((pseudonym_node, node_name))
+        g.pseudonyms.add((node_name, pseudonym_node))
+        add_concept(pseudonym, g, concept_mapping, default_concept_graph, seen)
+
+
+def get_default_grain_concept(
+    concept: BuildConcept, default_concept_graph: dict[str, BuildConcept]
+) -> BuildConcept:
+    """Get the default grain concept from the graph."""
+    if concept.address in default_concept_graph:
+        return default_concept_graph[concept.address]
+    default = concept.with_default_grain()
+    default_concept_graph[concept.address] = default
+    return default
+
+
+def generate_adhoc_graph(
+    concepts: list[BuildConcept],
+    datasources: list[BuildDatasource],
+    default_concept_graph: dict[str, BuildConcept],
+    restrict_to_listed: bool = False,
+) -> ReferenceGraph:
+    g = ReferenceGraph()
+    concept_mapping = {x.address: x for x in concepts}
+    seen: set[str] = set()
+    for concept in concepts:
+        if not isinstance(concept, BuildConcept):
+            raise ValueError(f"Invalid non-build concept {concept}")
+
+    # add all parsed concepts
+    for concept in concepts:
+
+        add_concept(concept, g, concept_mapping, default_concept_graph, seen)
+
+    for dataset in datasources:
+        node = datasource_to_node(dataset)
+        g.add_datasource_node(node, dataset)
+        for concept in dataset.concepts:
+            cnode = concept_to_node(concept)
+            g.concepts[cnode] = concept
+            g.add_node(cnode)
+            if restrict_to_listed:
+                if cnode not in g.nodes:
+                    continue
+            g.add_edge(node, cnode, fast=True)
+            g.add_edge(cnode, node, fast=True)
+            # if there is a key on a table at a different grain
+            # add an FK edge to the canonical source, if it exists
+            # for example, order ID on order product table
+            default = get_default_grain_concept(concept, default_concept_graph)
+
+            if concept != default:
+                dcnode = concept_to_node(default)
+                g.concepts[dcnode] = default
+                g.add_node(dcnode)
+                g.add_edge(cnode, dcnode, fast=True)
+                g.add_edge(dcnode, cnode, fast=True)
+    return g
+
+
+def generate_graph(
+    environment: BuildEnvironment,
+) -> ReferenceGraph:
+    default_concept_graph: dict[str, BuildConcept] = {}
+    return generate_adhoc_graph(
+        list(environment.concepts.values())
+        + list(environment.alias_origin_lookup.values()),
+        list(environment.datasources.values()),
+        default_concept_graph=default_concept_graph,
+    )