pytrilogy 0.3.142__cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

trilogy/core/env_processor.py

@@ -0,0 +1,120 @@
+from trilogy.core.graph_models import (
+    ReferenceGraph,
+    concept_to_node,
+    datasource_to_node,
+)
+from trilogy.core.models.build import BuildConcept, BuildDatasource
+from trilogy.core.models.build_environment import BuildEnvironment
+
+
+def add_concept(
+    concept: BuildConcept,
+    g: ReferenceGraph,
+    concept_mapping: dict[str, BuildConcept],
+    default_concept_graph: dict[str, BuildConcept],
+    seen: set[str],
+):
+
+    # if we have sources, recursively add them
+    node_name = concept_to_node(concept)
+    if node_name in seen:
+        return
+    seen.add(node_name)
+    g.concepts[node_name] = concept
+    g.add_node(node_name)
+    if concept.concept_arguments:
+        for source in concept.concept_arguments:
+            if not isinstance(source, BuildConcept):
+                raise ValueError(
+                    f"Invalid non-build concept {source} passed into graph generation from {concept}"
+                )
+            generic = get_default_grain_concept(source, default_concept_graph)
+            generic_node = concept_to_node(generic)
+            add_concept(generic, g, concept_mapping, default_concept_graph, seen)
+
+            g.add_edge(generic_node, node_name, fast=True)
+    for ps_address in concept.pseudonyms:
+        if ps_address not in concept_mapping:
+            raise SyntaxError(f"Concept {concept} has invalid pseudonym {ps_address}")
+        pseudonym = concept_mapping[ps_address]
+        pseudonym = get_default_grain_concept(pseudonym, default_concept_graph)
+        pseudonym_node = concept_to_node(pseudonym)
+        if (pseudonym_node, node_name) in g.edges and (
+            node_name,
+            pseudonym_node,
+        ) in g.edges:
+            continue
+        if pseudonym_node.split("@")[0] == node_name.split("@")[0]:
+            continue
+        g.add_edge(pseudonym_node, node_name, fast=True)
+        g.add_edge(node_name, pseudonym_node, fast=True)
+        g.pseudonyms.add((pseudonym_node, node_name))
+        g.pseudonyms.add((node_name, pseudonym_node))
+        add_concept(pseudonym, g, concept_mapping, default_concept_graph, seen)
+
+
+def get_default_grain_concept(
+    concept: BuildConcept, default_concept_graph: dict[str, BuildConcept]
+) -> BuildConcept:
+    """Get the default grain concept from the graph."""
+    if concept.address in default_concept_graph:
+        return default_concept_graph[concept.address]
+    default = concept.with_default_grain()
+    default_concept_graph[concept.address] = default
+    return default
+
+
+def generate_adhoc_graph(
+    concepts: list[BuildConcept],
+    datasources: list[BuildDatasource],
+    default_concept_graph: dict[str, BuildConcept],
+    restrict_to_listed: bool = False,
+) -> ReferenceGraph:
+    g = ReferenceGraph()
+    concept_mapping = {x.address: x for x in concepts}
+    seen: set[str] = set()
+    for concept in concepts:
+        if not isinstance(concept, BuildConcept):
+            raise ValueError(f"Invalid non-build concept {concept}")
+
+    # add all parsed concepts
+    for concept in concepts:
+
+        add_concept(concept, g, concept_mapping, default_concept_graph, seen)
+
+    for dataset in datasources:
+        node = datasource_to_node(dataset)
+        g.add_datasource_node(node, dataset)
+        for concept in dataset.concepts:
+            cnode = concept_to_node(concept)
+            g.concepts[cnode] = concept
+            g.add_node(cnode)
+            if restrict_to_listed:
+                if cnode not in g.nodes:
+                    continue
+            g.add_edge(node, cnode, fast=True)
+            g.add_edge(cnode, node, fast=True)
+            # if there is a key on a table at a different grain
+            # add an FK edge to the canonical source, if it exists
+            # for example, order ID on order product table
+            default = get_default_grain_concept(concept, default_concept_graph)
+
+            if concept != default:
+                dcnode = concept_to_node(default)
+                g.concepts[dcnode] = default
+                g.add_node(dcnode)
+                g.add_edge(cnode, dcnode, fast=True)
+                g.add_edge(dcnode, cnode, fast=True)
+    return g
+
+
+def generate_graph(
+    environment: BuildEnvironment,
+) -> ReferenceGraph:
+    default_concept_graph: dict[str, BuildConcept] = {}
+    return generate_adhoc_graph(
+        list(environment.concepts.values())
+        + list(environment.alias_origin_lookup.values()),
+        list(environment.datasources.values()),
+        default_concept_graph=default_concept_graph,
+    )

trilogy/core/environment_helpers.py

@@ -0,0 +1,320 @@
+from trilogy.constants import DEFAULT_NAMESPACE
+from trilogy.core.enums import ConceptSource, DatePart, FunctionType, Purpose
+from trilogy.core.functions import AttrAccess
+from trilogy.core.models.author import Concept, Function, Grain, Metadata, TraitDataType
+from trilogy.core.models.core import DataType, StructType, arg_to_datatype
+from trilogy.core.models.environment import Environment
+from trilogy.parsing.common import Meta
+
+
+def generate_date_concepts(concept: Concept, environment: Environment):
+    if concept.metadata and concept.metadata.line_number:
+        base_line_number = concept.metadata.line_number
+    else:
+        base_line_number = None
+    arg_tuples: list[tuple[FunctionType, TraitDataType]] = [
+        (FunctionType.MONTH, TraitDataType(type=DataType.INTEGER, traits=["month"])),
+        (FunctionType.YEAR, TraitDataType(type=DataType.INTEGER, traits=["year"])),
+        (
+            FunctionType.QUARTER,
+            TraitDataType(type=DataType.INTEGER, traits=["quarter"]),
+        ),
+        (FunctionType.DAY, TraitDataType(type=DataType.INTEGER, traits=["day"])),
+        (
+            FunctionType.DAY_OF_WEEK,
+            TraitDataType(type=DataType.INTEGER, traits=["day_of_week"]),
+        ),
+    ]
+    for ftype, dtype in arg_tuples:
+        fname = ftype.name.lower()
+        address = concept.address + f".{fname}"
+        if address in environment.concepts:
+            continue
+        default_type = (
+            Purpose.CONSTANT
+            if concept.purpose == Purpose.CONSTANT
+            else Purpose.PROPERTY
+        )
+        function = Function.model_construct(
+            operator=ftype,
+            arguments=[concept.reference],
+            output_datatype=dtype,
+            output_purpose=default_type,
+        )
+        new_concept = Concept.model_construct(
+            name=f"{concept.name}.{fname}",
+            datatype=function.output_datatype,
+            purpose=default_type,
+            lineage=function,
+            grain=concept.grain,
+            namespace=concept.namespace,
+            keys=set(
+                [concept.address],
+            ),
+            metadata=Metadata(
+                line_number=base_line_number,
+                concept_source=ConceptSource.AUTO_DERIVED,
+            ),
+        )
+        environment.add_concept(new_concept, add_derived=False)
+    for grain in [DatePart.MONTH, DatePart.YEAR]:
+        address = concept.address + f".{grain.value}_start"
+        if address in environment.concepts:
+            continue
+        function = Function.model_construct(
+            operator=FunctionType.DATE_TRUNCATE,
+            arguments=[concept.reference, grain],
+            output_datatype=DataType.DATE,
+            output_purpose=default_type,
+            arg_count=2,
+        )
+        new_concept = Concept.model_construct(
+            name=f"{concept.name}.{grain.value}_start",
+            datatype=DataType.DATE,
+            purpose=Purpose.PROPERTY,
+            lineage=function,
+            grain=concept.grain.model_copy(),
+            namespace=concept.namespace,
+            keys=set(
+                [concept.address],
+            ),
+            metadata=Metadata(
+                # description=f"Auto-derived from {base_description}. The date truncated to the {grain.value}.",
+                line_number=base_line_number,
+                concept_source=ConceptSource.AUTO_DERIVED,
+            ),
+        )
+
+        environment.add_concept(new_concept, add_derived=False)
+
+
+def generate_datetime_concepts(concept: Concept, environment: Environment):
+    if concept.metadata and concept.metadata.line_number:
+        base_line_number = concept.metadata.line_number
+    else:
+        base_line_number = None
+    setup_tuples: list[tuple[FunctionType, DataType | TraitDataType]] = [
+        (FunctionType.DATE, DataType.DATE),
+        (FunctionType.HOUR, TraitDataType(type=DataType.INTEGER, traits=["hour"])),
+        (FunctionType.MINUTE, TraitDataType(type=DataType.INTEGER, traits=["minute"])),
+        (FunctionType.SECOND, TraitDataType(type=DataType.INTEGER, traits=["second"])),
+    ]
+    for ftype, datatype in setup_tuples:
+        fname = ftype.name.lower()
+        address = concept.address + f".{fname}"
+        if address in environment.concepts:
+            continue
+        default_type = (
+            Purpose.CONSTANT
+            if concept.purpose == Purpose.CONSTANT
+            else Purpose.PROPERTY
+        )
+        const_function = Function.model_construct(
+            operator=ftype,
+            arguments=[concept.reference],
+            output_datatype=datatype,
+            output_purpose=default_type,
+        )
+        new_concept = Concept.model_construct(
+            name=f"{concept.name}.{fname}",
+            datatype=datatype,
+            purpose=default_type,
+            lineage=const_function,
+            grain=concept.grain.model_copy(),
+            namespace=concept.namespace,
+            keys=set(
+                [concept.address],
+            ),
+            metadata=Metadata(
+                line_number=base_line_number,
+                concept_source=ConceptSource.AUTO_DERIVED,
+            ),
+        )
+        if new_concept.name in environment.concepts:
+            continue
+        environment.add_concept(new_concept, add_derived=False)
+
+
+def generate_key_concepts(concept: Concept, environment: Environment):
+    if concept.metadata and concept.metadata.line_number:
+        base_line_number = concept.metadata.line_number
+    else:
+        base_line_number = None
+    for ftype in [FunctionType.COUNT]:
+        address = concept.address + f".{ftype.name.lower()}"
+        if address in environment.concepts:
+            continue
+        fname = ftype.name.lower()
+        default_type = Purpose.METRIC
+        const_function: Function = Function.model_construct(
+            operator=ftype,
+            output_datatype=DataType.INTEGER,
+            output_purpose=default_type,
+            arguments=[concept.reference],
+        )
+        new_concept = Concept.model_construct(
+            name=f"{concept.name}.{fname}",
+            datatype=DataType.INTEGER,
+            purpose=default_type,
+            lineage=const_function,
+            grain=Grain(),
+            namespace=concept.namespace,
+            keys=set(),
+            metadata=Metadata(
+                # description=f"Auto-derived integer. The {ftype.value} of {concept.address}, {base_description}",
+                line_number=base_line_number,
+                concept_source=ConceptSource.AUTO_DERIVED,
+            ),
+        )
+        environment.add_concept(new_concept, add_derived=False)
+
+
+def remove_date_concepts(concept: Concept, environment: Environment):
+    """Remove auto-generated date-related concepts for the given concept"""
+    date_suffixes = ["month", "year", "quarter", "day", "day_of_week"]
+    grain_suffixes = ["month_start", "year_start"]
+
+    for suffix in date_suffixes + grain_suffixes:
+        address = concept.address + f".{suffix}"
+        if address in environment.concepts:
+            derived_concept = environment.concepts[address]
+            # Only remove if it was auto-derived from this concept
+            if (
+                derived_concept.metadata
+                and derived_concept.metadata.concept_source
+                == ConceptSource.AUTO_DERIVED
+                and derived_concept.keys
+                and concept.address in derived_concept.keys
+            ):
+                environment.remove_concept(address)
+
+
+def remove_datetime_concepts(concept: Concept, environment: Environment):
+    """Remove auto-generated datetime-related concepts for the given concept"""
+    datetime_suffixes = ["date", "hour", "minute", "second"]
+
+    for suffix in datetime_suffixes:
+        address = concept.address + f".{suffix}"
+        if address in environment.concepts:
+            derived_concept = environment.concepts[address]
+            # Only remove if it was auto-derived from this concept
+            if (
+                derived_concept.metadata
+                and derived_concept.metadata.concept_source
+                == ConceptSource.AUTO_DERIVED
+                and derived_concept.keys
+                and concept.address in derived_concept.keys
+            ):
+                environment.remove_concept(address)
+
+
+def remove_key_concepts(concept: Concept, environment: Environment):
+    """Remove auto-generated key-related concepts for the given concept"""
+    key_suffixes = ["count"]
+
+    for suffix in key_suffixes:
+        address = concept.address + f".{suffix}"
+        if address in environment.concepts:
+            derived_concept = environment.concepts[address]
+            if (
+                derived_concept.metadata
+                and derived_concept.metadata.concept_source
+                == ConceptSource.AUTO_DERIVED
+            ):
+                environment.remove_concept(address)
+
+
+def remove_struct_concepts(concept: Concept, environment: Environment):
+    """Remove auto-generated struct field concepts for the given concept"""
+    if not isinstance(concept.datatype, StructType):
+        return
+
+    target_namespace = (
+        environment.namespace + "." + concept.name
+        if environment.namespace and environment.namespace != DEFAULT_NAMESPACE
+        else concept.name
+    )
+
+    # Get all concepts in the target namespace that were auto-derived
+    concepts_to_remove = []
+    for address, derived_concept in environment.concepts.items():
+        if (
+            derived_concept.namespace == target_namespace
+            and derived_concept.metadata
+            and derived_concept.metadata.concept_source == ConceptSource.AUTO_DERIVED
+            and isinstance(derived_concept.lineage, Function)
+            and derived_concept.lineage.operator == FunctionType.ATTR_ACCESS
+            and len(derived_concept.lineage.arguments) >= 1
+            and derived_concept.lineage.arguments[0] == concept.reference
+        ):
+            concepts_to_remove.append(address)
+
+    for address in concepts_to_remove:
+        environment.remove_concept(address)
+
+
+def remove_related_concepts(concept: Concept, environment: Environment):
+    """Remove all auto-generated concepts that were derived from the given concept"""
+
+    # Remove key-related concepts
+    if concept.purpose == Purpose.KEY:
+        remove_key_concepts(concept, environment)
+
+    # Remove datatype-specific concepts
+    if concept.datatype == DataType.DATE:
+        remove_date_concepts(concept, environment)
+    elif concept.datatype == DataType.DATETIME:
+        remove_date_concepts(concept, environment)
+        remove_datetime_concepts(concept, environment)
+    elif concept.datatype == DataType.TIMESTAMP:
+        remove_date_concepts(concept, environment)
+        remove_datetime_concepts(concept, environment)
+
+    # Remove struct field concepts
+    if isinstance(concept.datatype, StructType):
+        remove_struct_concepts(concept, environment)
+
+
+def generate_related_concepts(
+    concept: Concept,
+    environment: Environment,
+    meta: Meta | None = None,
+    add_derived: bool = False,
+):
+    """Auto populate common derived concepts on types"""
+    if concept.purpose == Purpose.KEY and add_derived:
+        generate_key_concepts(concept, environment)
+
+    # datatype types
+    if concept.datatype == DataType.DATE and add_derived:
+        generate_date_concepts(concept, environment)
+    elif concept.datatype == DataType.DATETIME and add_derived:
+
+        generate_date_concepts(concept, environment)
+        generate_datetime_concepts(concept, environment)
+    elif concept.datatype == DataType.TIMESTAMP and add_derived:
+        generate_date_concepts(concept, environment)
+        generate_datetime_concepts(concept, environment)
+
+    if isinstance(concept.datatype, StructType):
+        for key, value in concept.datatype.fields_map.items():
+            auto = Concept.model_construct(
+                name=key,
+                datatype=arg_to_datatype(value),
+                purpose=Purpose.PROPERTY,
+                namespace=(
+                    environment.namespace + "." + concept.name
+                    if environment.namespace
+                    and environment.namespace != DEFAULT_NAMESPACE
+                    else concept.name
+                ),
+                lineage=AttrAccess([concept.reference, key], environment=environment),
+                grain=concept.grain,
+                metadata=Metadata(
+                    concept_source=ConceptSource.AUTO_DERIVED,
+                ),
+                keys=concept.keys,
+            )
+            environment.add_concept(auto, meta=meta)
+            if isinstance(value, Concept):
+                environment.merge_concept(auto, value, modifiers=[])

trilogy/core/ergonomics.py

@@ -0,0 +1,193 @@
+from trilogy.constants import CONFIG
+
+# source: https://github.com/aaronbassett/Pass-phrase
+CTE_NAMES = """quizzical
+highfalutin
+wakeful
+cheerful
+thoughtful
+cooperative
+questionable
+abundant
+uneven
+yummy
+juicy
+vacuous
+concerned
+young
+sparkling
+abhorrent
+sweltering
+late
+macho
+scrawny
+friendly
+kaput
+divergent
+busy
+charming
+protective
+premium
+puzzled
+waggish
+rambunctious
+puffy
+hard
+sedate
+yellow
+resonant
+dapper
+courageous
+vast
+cool
+elated
+wary
+bewildered
+level
+wooden
+ceaseless
+tearful
+cloudy
+gullible
+flashy
+trite
+quick
+nondescript
+round
+slow
+spiritual
+brave
+tenuous
+abstracted
+colossal
+sloppy
+obsolete
+elegant
+fabulous
+vivacious
+exuberant
+faithful
+helpless
+odd
+sordid
+blue
+imported
+ugly
+ruthless
+deeply
+eminent
+badger
+barracuda
+bear
+boa
+cheetah
+chimpanzee
+civet
+cobra
+cougar
+coyote
+crocodile
+dingo
+eagle
+eel
+fossa
+fox
+human
+jackal
+jaguar
+komodo
+leopard
+lion
+lynx
+mamba
+mandrill
+marlin
+monitor
+ocelot
+petrel
+python
+ray
+salamander
+serval
+shark
+skua
+tiger
+viper
+wolf
+wolverine
+albatross
+avocet
+budgie
+canary
+chick
+chickadee
+chicken
+cockatiel
+cockatoo
+coot
+covey
+crow
+cuckoo
+darter
+dove
+duck
+falcon
+finch
+flamingo
+fowl
+goldfinch
+goose
+grouse
+hawk
+heron
+jackdaw
+jay
+kestrel
+lark
+loon
+macaw
+magpie
+martin
+osprey
+ostrich
+owl
+parakeet
+parrot
+pelican
+penguin
+pigeon
+pintail
+puffin
+quail
+quetzal
+rail
+raven
+razorbill
+rhea
+rook
+shrike
+skylark
+snipe
+sparrow
+starling
+stork
+swallow
+swift
+tanager
+thrush
+toucan
+turkey
+vulture
+warbler""".split(
+    "\n"
+)
+
+
+def generate_cte_names():
+    if CONFIG.randomize_cte_names:
+        from random import shuffle
+
+        new = [*CTE_NAMES]
+        shuffle(new)
+        return new
+    return CTE_NAMES