pytrilogy 0.0.1.102__py3-none-any.whl

trilogy/core/enums.py

@@ -0,0 +1,270 @@
+from enum import Enum
+
+InfiniteFunctionArgs = -1
+
+
+class UnnestMode(Enum):
+    DIRECT = "direct"
+    CROSS_APPLY = "cross_apply"
+    CROSS_JOIN = "cross_join"
+
+
+class ConceptSource(Enum):
+    MANUAL = "manual"
+    AUTO_DERIVED = "auto_derived"
+
+
+class StatementType(Enum):
+    QUERY = "query"
+
+
+class Purpose(Enum):
+    CONSTANT = "const"
+    KEY = "key"
+    PROPERTY = "property"
+    METRIC = "metric"
+    ROWSET = "rowset"
+    AUTO = "auto"
+
+    @classmethod
+    def _missing_(cls, value):
+        if value == "constant":
+            return Purpose.CONSTANT
+        return super()._missing_(value)
+
+
+class PurposeLineage(Enum):
+    BASIC = "basic"
+    WINDOW = "window"
+    AGGREGATE = "aggregate"
+    FILTER = "filter"
+    CONSTANT = "constant"
+    UNNEST = "unnest"
+    ROOT = "root"
+    ROWSET = "rowset"
+    MULTISELECT = "multiselect"
+    MERGE = "merge"
+
+
+class Granularity(Enum):
+    SINGLE_ROW = "single_row"
+    MULTI_ROW = "multi_row"
+
+
+class Modifier(Enum):
+    PARTIAL = "Partial"
+    OPTIONAL = "Optional"
+    HIDDEN = "Hidden"
+    NULLABLE = "Nullable"
+
+    @classmethod
+    def _missing_(cls, value):
+        strval = str(value)
+        if strval == "~":
+            return Modifier.PARTIAL
+        return super()._missing_(value=strval.capitalize())
+
+
+class JoinType(Enum):
+    INNER = "inner"
+    LEFT_OUTER = "left outer"
+    FULL = "full"
+    RIGHT_OUTER = "right outer"
+    CROSS = "cross"
+
+
+class Ordering(Enum):
+    ASCENDING = "asc"
+    DESCENDING = "desc"
+
+
+class WindowType(Enum):
+    ROW_NUMBER = "row_number"
+    RANK = "rank"
+    LAG = "lag"
+    LEAD = "lead"
+    SUM = "sum"
+    MAX = "max"
+    MIN = "min"
+    AVG = "avg"
+    COUNT = "count"
+    COUNT_DISTINCT = "count_distinct"
+
+
+class WindowOrder(Enum):
+    ASCENDING = "top"
+    DESCENDING = "bottom"
+
+
+class FunctionType(Enum):
+    # custom
+    CUSTOM = "custom"
+
+    # structural
+    UNNEST = "unnest"
+    ALIAS = "alias"
+
+    # Generic
+    CASE = "case"
+    CAST = "cast"
+    CONCAT = "concat"
+    CONSTANT = "constant"
+    COALESCE = "coalesce"
+    IS_NULL = "isnull"
+
+    # COMPLEX
+    INDEX_ACCESS = "index_access"
+    ATTR_ACCESS = "attr_access"
+
+    # TEXT AND MAYBE MORE
+    SPLIT = "split"
+
+    # Math
+    DIVIDE = "divide"
+    MULTIPLY = "multiply"
+    ADD = "add"
+    SUBTRACT = "subtract"
+    MOD = "mod"
+    ROUND = "round"
+    ABS = "abs"
+
+    # Aggregates
+    ## group is not a real aggregate - it just means group by this + some other set of fields
+    ## but is here as syntax is identical
+    GROUP = "group"
+
+    COUNT = "count"
+    COUNT_DISTINCT = "count_distinct"
+    SUM = "sum"
+    MAX = "max"
+    MIN = "min"
+    AVG = "avg"
+    LENGTH = "len"
+
+    # String
+    LIKE = "like"
+    ILIKE = "ilike"
+    LOWER = "lower"
+    UPPER = "upper"
+    SUBSTRING = "substring"
+    STRPOS = "strpos"
+
+    # Dates
+    DATE = "date"
+    DATETIME = "datetime"
+    TIMESTAMP = "timestamp"
+
+    # time
+    SECOND = "second"
+    MINUTE = "minute"
+    HOUR = "hour"
+    DAY = "day"
+    DAY_OF_WEEK = "day_of_week"
+    WEEK = "week"
+    MONTH = "month"
+    QUARTER = "quarter"
+    YEAR = "year"
+
+    DATE_PART = "date_part"
+    DATE_TRUNCATE = "date_truncate"
+    DATE_ADD = "date_add"
+    DATE_DIFF = "date_diff"
+
+    # UNIX
+    UNIX_TO_TIMESTAMP = "unix_to_timestamp"
+
+    # CONSTANTS
+    CURRENT_DATE = "current_date"
+    CURRENT_DATETIME = "current_datetime"
+
+
+class FunctionClass(Enum):
+    AGGREGATE_FUNCTIONS = [
+        FunctionType.MAX,
+        FunctionType.MIN,
+        FunctionType.SUM,
+        FunctionType.AVG,
+        FunctionType.COUNT,
+        FunctionType.COUNT_DISTINCT,
+    ]
+    SINGLE_ROW = [
+        FunctionType.CONSTANT,
+        FunctionType.CURRENT_DATE,
+        FunctionType.CURRENT_DATETIME,
+    ]
+
+
+class Boolean(Enum):
+    TRUE = "true"
+    FALSE = "false"
+
+
+class BooleanOperator(Enum):
+    AND = "and"
+    OR = "or"
+
+
+class ComparisonOperator(Enum):
+    LT = "<"
+    GT = ">"
+    EQ = "="
+    IS = "is"
+    IS_NOT = "is not"
+    GTE = ">="
+    LTE = "<="
+    NE = "!="
+    IN = "in"
+    NOT_IN = "not in"
+    # TODO: deprecate for contains?
+    LIKE = "like"
+    ILIKE = "ilike"
+    CONTAINS = "contains"
+    ELSE = "else"
+
+    @classmethod
+    def _missing_(cls, value):
+        if not isinstance(value, list) and " " in str(value):
+            value = str(value).split()
+        if isinstance(value, list):
+            processed = [str(v).lower() for v in value]
+            if processed == ["not", "in"]:
+                return ComparisonOperator.NOT_IN
+            if processed == ["is", "not"]:
+                return ComparisonOperator.IS_NOT
+            if value == ["in"]:
+                return ComparisonOperator.IN
+        return super()._missing_(str(value).lower())
+
+
+class DatePart(Enum):
+    MONTH = "month"
+    YEAR = "year"
+    WEEK = "week"
+    DAY = "day"
+    QUARTER = "quarter"
+    HOUR = "hour"
+    MINUTE = "minute"
+    SECOND = "second"
+
+    @classmethod
+    def _missing_(cls, value):
+        if isinstance(value, str) and value.lower() != value:
+            return DatePart(value.lower())
+        return super()._missing_(value)
+
+
+class SourceType(Enum):
+    FILTER = "filter"
+    SELECT = "select"
+    MERGE = "merge"
+    ABSTRACT = "abstract"
+    DIRECT_SELECT = "direct_select"
+    GROUP = "group"
+    WINDOW = "window"
+    UNNEST = "unnest"
+    CONSTANT = "constant"
+
+
+class ShowCategory(Enum):
+    MODELS = "models"
+    CONCEPTS = "concepts"

trilogy/core/env_processor.py

@@ -0,0 +1,33 @@
+from trilogy.core.graph_models import ReferenceGraph, concept_to_node, datasource_to_node
+from trilogy.core.models import Environment
+from trilogy.core.enums import PurposeLineage
+
+
+def generate_graph(
+    environment: Environment,
+) -> ReferenceGraph:
+    g = ReferenceGraph()
+
+    # add all parsed concepts
+    for _, concept in environment.concepts.items():
+        g.add_node(concept)
+        # if we have sources, recursively add them
+        if concept.sources:
+            node_name = concept_to_node(concept)
+            for source in concept.sources:
+                generic = source.with_default_grain()
+                g.add_edge(generic, node_name)
+                if concept.derivation == PurposeLineage.MERGE:
+                    g.add_edge(node_name, generic)
+    for _, dataset in environment.datasources.items():
+        node = datasource_to_node(dataset)
+        g.add_node(dataset, type="datasource", datasource=dataset)
+        for concept in dataset.concepts:
+            g.add_edge(node, concept)
+            g.add_edge(concept, node)
+            # if there is a key on a table at a different grain
+            # add an FK edge to the canonical source, if it exists
+            # for example, order ID on order product table
+            g.add_edge(concept, concept.with_default_grain())
+            g.add_edge(concept.with_default_grain(), concept)
+    return g

trilogy/core/environment_helpers.py

@@ -0,0 +1,156 @@
+from trilogy.core.models import DataType, Concept, Environment, Function, Metadata
+from trilogy.core.enums import Purpose, FunctionType, ConceptSource
+from trilogy.constants import DEFAULT_NAMESPACE
+
+
+def generate_date_concepts(concept: Concept, environment: Environment):
+    if concept.metadata and concept.metadata.description:
+        base_description = concept.metadata.description
+    else:
+        base_description = f"a {concept.datatype.value}"
+    if concept.metadata and concept.metadata.line_number:
+        base_line_number = concept.metadata.line_number
+    else:
+        base_line_number = None
+    for ftype in [
+        FunctionType.MONTH,
+        FunctionType.YEAR,
+        FunctionType.QUARTER,
+        FunctionType.DAY,
+        FunctionType.DAY_OF_WEEK,
+    ]:
+        fname = ftype.name.lower()
+        default_type = (
+            Purpose.CONSTANT
+            if concept.purpose == Purpose.CONSTANT
+            else Purpose.PROPERTY
+        )
+        const_function: Function = Function(
+            operator=ftype,
+            output_datatype=DataType.INTEGER,
+            output_purpose=default_type,
+            arguments=[concept],
+        )
+        namespace = (
+            None if concept.namespace == DEFAULT_NAMESPACE else concept.namespace
+        )
+        new_concept = Concept(
+            name=f"{concept.name}.{fname}",
+            datatype=DataType.INTEGER,
+            purpose=default_type,
+            lineage=const_function,
+            grain=const_function.output_grain,
+            namespace=namespace,
+            keys=(concept,),
+            metadata=Metadata(
+                description=f"Auto-derived. Integer format. The {ftype.value} derived from {concept.name}, {base_description}",
+                line_number=base_line_number,
+                concept_source=ConceptSource.AUTO_DERIVED,
+            ),
+        )
+        if new_concept.name in environment.concepts:
+            continue
+        environment.add_concept(new_concept, add_derived=False)
+
+
+def generate_datetime_concepts(concept: Concept, environment: Environment):
+    if concept.metadata and concept.metadata.description:
+        base_description = concept.metadata.description
+    else:
+        base_description = f"a {concept.datatype.value}"
+    if concept.metadata and concept.metadata.line_number:
+        base_line_number = concept.metadata.line_number
+    else:
+        base_line_number = None
+    for ftype in [
+        FunctionType.DATE,
+        FunctionType.HOUR,
+        FunctionType.MINUTE,
+        FunctionType.SECOND,
+    ]:
+        fname = ftype.name.lower()
+        default_type = (
+            Purpose.CONSTANT
+            if concept.purpose == Purpose.CONSTANT
+            else Purpose.PROPERTY
+        )
+        const_function: Function = Function(
+            operator=ftype,
+            output_datatype=DataType.INTEGER,
+            output_purpose=default_type,
+            arguments=[concept],
+        )
+        namespace = (
+            None if concept.namespace == DEFAULT_NAMESPACE else concept.namespace
+        )
+        new_concept = Concept(
+            name=f"{concept.name}.{fname}",
+            datatype=DataType.INTEGER,
+            purpose=default_type,
+            lineage=const_function,
+            grain=const_function.output_grain,
+            namespace=namespace,
+            keys=(concept,),
+            metadata=Metadata(
+                description=f"Auto-derived. Integer format. The {ftype.value} derived from {concept.name}, {base_description}",
+                line_number=base_line_number,
+                concept_source=ConceptSource.AUTO_DERIVED,
+            ),
+        )
+        if new_concept.name in environment.concepts:
+            continue
+        environment.add_concept(new_concept, add_derived=False)
+
+
+def generate_key_concepts(concept: Concept, environment: Environment):
+    if concept.metadata and concept.metadata.description:
+        base_description = concept.metadata.description
+    else:
+        base_description = f"a {concept.datatype.value}"
+    if concept.metadata and concept.metadata.line_number:
+        base_line_number = concept.metadata.line_number
+    else:
+        base_line_number = None
+    for ftype in [FunctionType.COUNT]:
+        fname = ftype.name.lower()
+        default_type = Purpose.METRIC
+        const_function: Function = Function(
+            operator=ftype,
+            output_datatype=DataType.INTEGER,
+            output_purpose=default_type,
+            arguments=[concept],
+        )
+        namespace = (
+            None if concept.namespace == DEFAULT_NAMESPACE else concept.namespace
+        )
+        new_concept = Concept(
+            name=f"{concept.name}.{fname}",
+            datatype=DataType.INTEGER,
+            purpose=default_type,
+            lineage=const_function,
+            grain=const_function.output_grain,
+            namespace=namespace,
+            keys=(concept,),
+            metadata=Metadata(
+                description=f"Auto-derived. Integer format. The {ftype.value} derived from {concept.name}, {base_description}",
+                line_number=base_line_number,
+                concept_source=ConceptSource.AUTO_DERIVED,
+            ),
+        )
+        if new_concept.name in environment.concepts:
+            continue
+        environment.add_concept(new_concept, add_derived=False)
+
+
+def generate_related_concepts(concept: Concept, environment: Environment):
+    """Auto populate common derived concepts on types"""
+    if concept.purpose == Purpose.KEY:
+        generate_key_concepts(concept, environment)
+    if concept.datatype == DataType.DATE:
+        generate_date_concepts(concept, environment)
+    elif concept.datatype == DataType.DATETIME:
+        generate_date_concepts(concept, environment)
+        generate_datetime_concepts(concept, environment)
+    elif concept.datatype == DataType.TIMESTAMP:
+        generate_date_concepts(concept, environment)
+        generate_datetime_concepts(concept, environment)

trilogy/core/ergonomics.py

@@ -0,0 +1,187 @@
+# source: https://github.com/aaronbassett/Pass-phrase
+CTE_NAMES = """quizzical
+highfalutin
+dynamic
+wakeful
+cheerful
+thoughtful
+cooperative
+questionable
+abundant
+uneven
+yummy
+juicy
+vacuous
+concerned
+young
+sparkling
+abhorrent
+sweltering
+late
+macho
+scrawny
+friendly
+kaput
+divergent
+busy
+charming
+protective
+premium
+puzzled
+waggish
+rambunctious
+puffy
+hard
+sedate
+yellow
+resonant
+dapper
+courageous
+vast
+cool
+elated
+wary
+bewildered
+level
+wooden
+ceaseless
+tearful
+cloudy
+gullible
+flashy
+trite
+quick
+nondescript
+round
+slow
+spiritual
+brave
+tenuous
+abstracted
+colossal
+sloppy
+obsolete
+elegant
+fabulous
+vivacious
+exuberant
+faithful
+helpless
+odd
+sordid
+blue
+imported
+ugly
+ruthless
+deeply
+eminent
+badger
+barracuda
+bear
+boa
+cheetah
+chimpanzee
+civet
+cobra
+cougar
+coyote
+crocodile
+dingo
+eagle
+eel
+fossa
+fox
+human
+jackal
+jaguar
+komodo
+leopard
+lion
+lynx
+mamba
+mandrill
+marlin
+monitor
+ocelot
+osprey
+owl
+petrel
+python
+ray
+salamander
+serval
+shark
+skua
+tiger
+viper
+wolf
+wolverine
+albatross
+avocet
+budgie
+canary
+chick
+chickadee
+chicken
+cockatiel
+cockatoo
+coot
+covey
+crow
+cuckoo
+darter
+dove
+duck
+eagle
+falcon
+finch
+flamingo
+fowl
+goldfinch
+goose
+grouse
+hawk
+heron
+jackdaw
+jay
+kestrel
+lark
+loon
+macaw
+magpie
+martin
+osprey
+ostrich
+owl
+parakeet
+parrot
+pelican
+penguin
+pigeon
+pintail
+puffin
+quail
+quetzal
+rail
+raven
+razorbill
+rhea
+rook
+shrike
+skylark
+snipe
+sparrow
+starling
+stork
+swallow
+swift
+tanager
+thrush
+toucan
+turkey
+vulture
+warbler""".split(
+    "\n"
+)
+
+CTE_NAMES = list(set(CTE_NAMES))

trilogy/core/exceptions.py

@@ -0,0 +1,23 @@
+from typing import List
+
+
+class UndefinedConceptException(Exception):
+    def __init__(self, message, suggestions: List[str]):
+        super().__init__(self, message)
+        self.message = message
+        self.suggestions = suggestions
+
+
+class InvalidSyntaxException(Exception):
+    pass
+
+
+class NoDatasourceException(Exception):
+    pass
+
+
+class AmbiguousRelationshipResolutionException(Exception):
+    def __init__(self, message, parents: List[set[str]]):
+        super().__init__(self, message)
+        self.message = message
+        self.parents = parents