pytrilogy 0.3.148__cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

trilogy/core/env_processor.py

@@ -0,0 +1,239 @@
+from dataclasses import dataclass
+from typing import Iterator
+
+from trilogy.core.enums import Derivation
+from trilogy.core.graph_models import (
+    ReferenceGraph,
+    concept_to_node,
+    datasource_to_node,
+)
+from trilogy.core.models.build import BuildConcept, BuildDatasource
+from trilogy.core.models.build_environment import BuildEnvironment
+
+
+@dataclass(slots=True)
+class BasicConceptGraph:
+    """Pre-computed dependency graph for BASIC derived concepts."""
+
+    # concept address -> concept
+    concepts: dict[str, BuildConcept]
+    # concept address -> set of input addresses required
+    dependencies: dict[str, frozenset[str]]
+    # input address -> concepts that depend on it
+    reverse_deps: dict[str, list[str]]
+    # concepts with no dependencies (can be computed from any base concept)
+    roots: list[str]
+
+
+def build_basic_concept_graph(concepts: list[BuildConcept]) -> BasicConceptGraph:
+    """Build dependency graph for BASIC derived concepts.
+
+    Returns a structure that allows efficient single-pass computation
+    of which derived concepts can be added to a datasource.
+    """
+    concept_map: dict[str, BuildConcept] = {}
+    dependencies: dict[str, frozenset[str]] = {}
+    reverse_deps: dict[str, list[str]] = {}
+    roots: list[str] = []
+
+    for concept in concepts:
+        if concept.derivation != Derivation.BASIC or not concept.concept_arguments:
+            continue
+
+        addr = concept.canonical_address
+        concept_map[addr] = concept
+        input_addrs = frozenset(c.canonical_address for c in concept.concept_arguments)
+        dependencies[addr] = input_addrs
+
+        # Build reverse dependency map
+        for input_addr in input_addrs:
+            if input_addr not in reverse_deps:
+                reverse_deps[input_addr] = []
+            reverse_deps[input_addr].append(addr)
+
+    # Find roots - concepts whose inputs are all non-BASIC
+    for addr, deps in dependencies.items():
+        if all(d not in dependencies for d in deps):
+            roots.append(addr)
+
+    return BasicConceptGraph(
+        concepts=concept_map,
+        dependencies=dependencies,
+        reverse_deps=reverse_deps,
+        roots=roots,
+    )
+
+
+def get_derivable_concepts(
+    graph: BasicConceptGraph,
+    available: set[str],
+    already_present: set[str],
+) -> Iterator[BuildConcept]:
+    """Yield concepts derivable from available concepts in topological order.
+
+    Uses the pre-computed dependency graph to traverse in a single pass,
+    yielding each concept as soon as its dependencies are satisfied.
+
+    Args:
+        graph: Pre-computed dependency graph for BASIC concepts
+        available: Set of canonical addresses of complete concepts (will be mutated)
+        already_present: Set of canonical addresses already in the datasource (skip these)
+    """
+    if not graph.roots:
+        return
+
+    # Track which concepts we've already processed
+    processed: set[str] = set()
+    # Queue of concept addresses to check
+    to_check: list[str] = list(graph.roots)
+
+    while to_check:
+        addr = to_check.pop()
+        if addr in processed:
+            continue
+
+        deps = graph.dependencies[addr]
+        if deps.issubset(available):
+            processed.add(addr)
+            available.add(addr)
+
+            # Only yield if not already in the datasource
+            if addr not in already_present:
+                yield graph.concepts[addr]
+
+            # Add concepts that depend on this one to the check queue
+            for dependent in graph.reverse_deps.get(addr, []):
+                if dependent not in processed:
+                    to_check.append(dependent)
+
+
+def add_concept(
+    concept: BuildConcept,
+    g: ReferenceGraph,
+    concept_mapping: dict[str, BuildConcept],
+    default_concept_graph: dict[str, BuildConcept],
+    seen: set[str],
+    node_stash: dict[str, str] = {},
+):
+
+    # if we have sources, recursively add them
+    node_name = concept_to_node(concept, node_stash)
+    if node_name in seen:
+        return
+    seen.add(node_name)
+    g.concepts[node_name] = concept
+    g.add_node(node_name)
+    root_name = node_name.split("@", 1)[0]
+    if concept.concept_arguments:
+        for source in concept.concept_arguments:
+            if not isinstance(source, BuildConcept):
+                raise ValueError(
+                    f"Invalid non-build concept {source} passed into graph generation from {concept}"
+                )
+            generic = get_default_grain_concept(source, default_concept_graph)
+            generic_node = concept_to_node(generic, stash=node_stash)
+            add_concept(
+                generic, g, concept_mapping, default_concept_graph, seen, node_stash
+            )
+
+            g.add_edge(generic_node, node_name)
+    for ps_address in concept.pseudonyms:
+        if ps_address not in concept_mapping:
+            raise SyntaxError(f"Concept {concept} has invalid pseudonym {ps_address}")
+        pseudonym = concept_mapping[ps_address]
+        pseudonym = get_default_grain_concept(pseudonym, default_concept_graph)
+        pseudonym_node = concept_to_node(pseudonym, stash=node_stash)
+        if (pseudonym_node, node_name) in g.edges and (
+            node_name,
+            pseudonym_node,
+        ) in g.edges:
+            continue
+        if pseudonym_node.split("@", 1)[0] == root_name:
+            continue
+        g.add_edge(pseudonym_node, node_name)
+        g.add_edge(node_name, pseudonym_node)
+        g.pseudonyms.add((pseudonym_node, node_name))
+        g.pseudonyms.add((node_name, pseudonym_node))
+        add_concept(
+            pseudonym, g, concept_mapping, default_concept_graph, seen, node_stash
+        )
+
+
+def get_default_grain_concept(
+    concept: BuildConcept, default_concept_graph: dict[str, BuildConcept]
+) -> BuildConcept:
+    """Get the default grain concept from the graph."""
+    if concept.address in default_concept_graph:
+        return default_concept_graph[concept.address]
+    default = concept.with_default_grain()
+    default_concept_graph[concept.address] = default
+    return default
+
+
+def generate_adhoc_graph(
+    concepts: list[BuildConcept],
+    datasources: list[BuildDatasource],
+    default_concept_graph: dict[str, BuildConcept],
+) -> ReferenceGraph:
+    g = ReferenceGraph()
+    concept_mapping = {x.address: x for x in concepts}
+    node_stash: dict[str, str] = {}
+    seen: set[str] = set()
+    for concept in concepts:
+        if not isinstance(concept, BuildConcept):
+            raise ValueError(f"Invalid non-build concept {concept}")
+
+    # add all parsed concepts
+    for concept in concepts:
+        add_concept(
+            concept, g, concept_mapping, default_concept_graph, seen, node_stash
+        )
+
+    basic_graph = build_basic_concept_graph(concepts)
+
+    for dataset in datasources:
+        node = datasource_to_node(dataset)
+        g.datasources[node] = dataset
+        g.add_datasource_node(node, dataset)
+        eligible = dataset.concepts
+        already_present = set(x.canonical_address for x in eligible)
+        complete_contains = set(
+            c.concept.canonical_address for c in dataset.columns if c.is_complete
+        )
+        for derived in get_derivable_concepts(
+            basic_graph, complete_contains, already_present
+        ):
+            eligible.append(derived)
+
+        for concept in eligible:
+            cnode = concept_to_node(concept, node_stash)
+            g.concepts[cnode] = concept
+
+            g.add_node(cnode)
+            g.add_edge(node, cnode)
+            g.add_edge(cnode, node)
+            # if there is a key on a table at a different grain
+            # add an FK edge to the canonical source, if it exists
+            # for example, order ID on order product table
+            default = get_default_grain_concept(concept, default_concept_graph)
+
+            if concept != default:
+
+                dcnode = concept_to_node(default, node_stash)
+                g.concepts[dcnode] = default
+                g.add_node(dcnode)
+                g.add_edge(cnode, dcnode)
+                g.add_edge(dcnode, cnode)
+    return g
+
+
+def generate_graph(
+    environment: BuildEnvironment,
+) -> ReferenceGraph:
+    default_concept_graph: dict[str, BuildConcept] = {}
+    return generate_adhoc_graph(
+        list(environment.concepts.values())
+        + list(environment.alias_origin_lookup.values()),
+        list(environment.datasources.values()),
+        default_concept_graph=default_concept_graph,
+    )

trilogy/core/environment_helpers.py

@@ -0,0 +1,320 @@
+from trilogy.constants import DEFAULT_NAMESPACE
+from trilogy.core.enums import ConceptSource, DatePart, FunctionType, Purpose
+from trilogy.core.functions import AttrAccess
+from trilogy.core.models.author import Concept, Function, Grain, Metadata, TraitDataType
+from trilogy.core.models.core import DataType, StructType, arg_to_datatype
+from trilogy.core.models.environment import Environment
+from trilogy.parsing.common import Meta
+
+
+def generate_date_concepts(concept: Concept, environment: Environment):
+    if concept.metadata and concept.metadata.line_number:
+        base_line_number = concept.metadata.line_number
+    else:
+        base_line_number = None
+    arg_tuples: list[tuple[FunctionType, TraitDataType]] = [
+        (FunctionType.MONTH, TraitDataType(type=DataType.INTEGER, traits=["month"])),
+        (FunctionType.YEAR, TraitDataType(type=DataType.INTEGER, traits=["year"])),
+        (
+            FunctionType.QUARTER,
+            TraitDataType(type=DataType.INTEGER, traits=["quarter"]),
+        ),
+        (FunctionType.DAY, TraitDataType(type=DataType.INTEGER, traits=["day"])),
+        (
+            FunctionType.DAY_OF_WEEK,
+            TraitDataType(type=DataType.INTEGER, traits=["day_of_week"]),
+        ),
+    ]
+    for ftype, dtype in arg_tuples:
+        fname = ftype.name.lower()
+        address = concept.address + f".{fname}"
+        if address in environment.concepts:
+            continue
+        default_type = (
+            Purpose.CONSTANT
+            if concept.purpose == Purpose.CONSTANT
+            else Purpose.PROPERTY
+        )
+        function = Function.model_construct(
+            operator=ftype,
+            arguments=[concept.reference],
+            output_datatype=dtype,
+            output_purpose=default_type,
+        )
+        new_concept = Concept.model_construct(
+            name=f"{concept.name}.{fname}",
+            datatype=function.output_datatype,
+            purpose=default_type,
+            lineage=function,
+            grain=concept.grain,
+            namespace=concept.namespace,
+            keys=set(
+                [concept.address],
+            ),
+            metadata=Metadata(
+                line_number=base_line_number,
+                concept_source=ConceptSource.AUTO_DERIVED,
+            ),
+        )
+        environment.add_concept(new_concept, add_derived=False)
+    for grain in [DatePart.MONTH, DatePart.YEAR]:
+        address = concept.address + f".{grain.value}_start"
+        if address in environment.concepts:
+            continue
+        function = Function.model_construct(
+            operator=FunctionType.DATE_TRUNCATE,
+            arguments=[concept.reference, grain],
+            output_datatype=DataType.DATE,
+            output_purpose=default_type,
+            arg_count=2,
+        )
+        new_concept = Concept.model_construct(
+            name=f"{concept.name}.{grain.value}_start",
+            datatype=DataType.DATE,
+            purpose=Purpose.PROPERTY,
+            lineage=function,
+            grain=concept.grain.model_copy(),
+            namespace=concept.namespace,
+            keys=set(
+                [concept.address],
+            ),
+            metadata=Metadata(
+                # description=f"Auto-derived from {base_description}. The date truncated to the {grain.value}.",
+                line_number=base_line_number,
+                concept_source=ConceptSource.AUTO_DERIVED,
+            ),
+        )
+
+        environment.add_concept(new_concept, add_derived=False)
+
+
+def generate_datetime_concepts(concept: Concept, environment: Environment):
+    if concept.metadata and concept.metadata.line_number:
+        base_line_number = concept.metadata.line_number
+    else:
+        base_line_number = None
+    setup_tuples: list[tuple[FunctionType, DataType | TraitDataType]] = [
+        (FunctionType.DATE, DataType.DATE),
+        (FunctionType.HOUR, TraitDataType(type=DataType.INTEGER, traits=["hour"])),
+        (FunctionType.MINUTE, TraitDataType(type=DataType.INTEGER, traits=["minute"])),
+        (FunctionType.SECOND, TraitDataType(type=DataType.INTEGER, traits=["second"])),
+    ]
+    for ftype, datatype in setup_tuples:
+        fname = ftype.name.lower()
+        address = concept.address + f".{fname}"
+        if address in environment.concepts:
+            continue
+        default_type = (
+            Purpose.CONSTANT
+            if concept.purpose == Purpose.CONSTANT
+            else Purpose.PROPERTY
+        )
+        const_function = Function.model_construct(
+            operator=ftype,
+            arguments=[concept.reference],
+            output_datatype=datatype,
+            output_purpose=default_type,
+        )
+        new_concept = Concept.model_construct(
+            name=f"{concept.name}.{fname}",
+            datatype=datatype,
+            purpose=default_type,
+            lineage=const_function,
+            grain=concept.grain.model_copy(),
+            namespace=concept.namespace,
+            keys=set(
+                [concept.address],
+            ),
+            metadata=Metadata(
+                line_number=base_line_number,
+                concept_source=ConceptSource.AUTO_DERIVED,
+            ),
+        )
+        if new_concept.name in environment.concepts:
+            continue
+        environment.add_concept(new_concept, add_derived=False)
+
+
+def generate_key_concepts(concept: Concept, environment: Environment):
+    if concept.metadata and concept.metadata.line_number:
+        base_line_number = concept.metadata.line_number
+    else:
+        base_line_number = None
+    for ftype in [FunctionType.COUNT]:
+        address = concept.address + f".{ftype.name.lower()}"
+        if address in environment.concepts:
+            continue
+        fname = ftype.name.lower()
+        default_type = Purpose.METRIC
+        const_function: Function = Function.model_construct(
+            operator=ftype,
+            output_datatype=DataType.INTEGER,
+            output_purpose=default_type,
+            arguments=[concept.reference],
+        )
+        new_concept = Concept.model_construct(
+            name=f"{concept.name}.{fname}",
+            datatype=DataType.INTEGER,
+            purpose=default_type,
+            lineage=const_function,
+            grain=Grain(),
+            namespace=concept.namespace,
+            keys=set(),
+            metadata=Metadata(
+                # description=f"Auto-derived integer. The {ftype.value} of {concept.address}, {base_description}",
+                line_number=base_line_number,
+                concept_source=ConceptSource.AUTO_DERIVED,
+            ),
+        )
+        environment.add_concept(new_concept, add_derived=False)
+
+
+def remove_date_concepts(concept: Concept, environment: Environment):
+    """Remove auto-generated date-related concepts for the given concept"""
+    date_suffixes = ["month", "year", "quarter", "day", "day_of_week"]
+    grain_suffixes = ["month_start", "year_start"]
+
+    for suffix in date_suffixes + grain_suffixes:
+        address = concept.address + f".{suffix}"
+        if address in environment.concepts:
+            derived_concept = environment.concepts[address]
+            # Only remove if it was auto-derived from this concept
+            if (
+                derived_concept.metadata
+                and derived_concept.metadata.concept_source
+                == ConceptSource.AUTO_DERIVED
+                and derived_concept.keys
+                and concept.address in derived_concept.keys
+            ):
+                environment.remove_concept(address)
+
+
+def remove_datetime_concepts(concept: Concept, environment: Environment):
+    """Remove auto-generated datetime-related concepts for the given concept"""
+    datetime_suffixes = ["date", "hour", "minute", "second"]
+
+    for suffix in datetime_suffixes:
+        address = concept.address + f".{suffix}"
+        if address in environment.concepts:
+            derived_concept = environment.concepts[address]
+            # Only remove if it was auto-derived from this concept
+            if (
+                derived_concept.metadata
+                and derived_concept.metadata.concept_source
+                == ConceptSource.AUTO_DERIVED
+                and derived_concept.keys
+                and concept.address in derived_concept.keys
+            ):
+                environment.remove_concept(address)
+
+
+def remove_key_concepts(concept: Concept, environment: Environment):
+    """Remove auto-generated key-related concepts for the given concept"""
+    key_suffixes = ["count"]
+
+    for suffix in key_suffixes:
+        address = concept.address + f".{suffix}"
+        if address in environment.concepts:
+            derived_concept = environment.concepts[address]
+            if (
+                derived_concept.metadata
+                and derived_concept.metadata.concept_source
+                == ConceptSource.AUTO_DERIVED
+            ):
+                environment.remove_concept(address)
+
+
+def remove_struct_concepts(concept: Concept, environment: Environment):
+    """Remove auto-generated struct field concepts for the given concept"""
+    if not isinstance(concept.datatype, StructType):
+        return
+
+    target_namespace = (
+        environment.namespace + "." + concept.name
+        if environment.namespace and environment.namespace != DEFAULT_NAMESPACE
+        else concept.name
+    )
+
+    # Get all concepts in the target namespace that were auto-derived
+    concepts_to_remove = []
+    for address, derived_concept in environment.concepts.items():
+        if (
+            derived_concept.namespace == target_namespace
+            and derived_concept.metadata
+            and derived_concept.metadata.concept_source == ConceptSource.AUTO_DERIVED
+            and isinstance(derived_concept.lineage, Function)
+            and derived_concept.lineage.operator == FunctionType.ATTR_ACCESS
+            and len(derived_concept.lineage.arguments) >= 1
+            and derived_concept.lineage.arguments[0] == concept.reference
+        ):
+            concepts_to_remove.append(address)
+
+    for address in concepts_to_remove:
+        environment.remove_concept(address)
+
+
+def remove_related_concepts(concept: Concept, environment: Environment):
+    """Remove all auto-generated concepts that were derived from the given concept"""
+
+    # Remove key-related concepts
+    if concept.purpose == Purpose.KEY:
+        remove_key_concepts(concept, environment)
+
+    # Remove datatype-specific concepts
+    if concept.datatype == DataType.DATE:
+        remove_date_concepts(concept, environment)
+    elif concept.datatype == DataType.DATETIME:
+        remove_date_concepts(concept, environment)
+        remove_datetime_concepts(concept, environment)
+    elif concept.datatype == DataType.TIMESTAMP:
+        remove_date_concepts(concept, environment)
+        remove_datetime_concepts(concept, environment)
+
+    # Remove struct field concepts
+    if isinstance(concept.datatype, StructType):
+        remove_struct_concepts(concept, environment)
+
+
+def generate_related_concepts(
+    concept: Concept,
+    environment: Environment,
+    meta: Meta | None = None,
+    add_derived: bool = False,
+):
+    """Auto populate common derived concepts on types"""
+    if concept.purpose == Purpose.KEY and add_derived:
+        generate_key_concepts(concept, environment)
+
+    # datatype types
+    if concept.datatype == DataType.DATE and add_derived:
+        generate_date_concepts(concept, environment)
+    elif concept.datatype == DataType.DATETIME and add_derived:
+
+        generate_date_concepts(concept, environment)
+        generate_datetime_concepts(concept, environment)
+    elif concept.datatype == DataType.TIMESTAMP and add_derived:
+        generate_date_concepts(concept, environment)
+        generate_datetime_concepts(concept, environment)
+
+    if isinstance(concept.datatype, StructType):
+        for key, value in concept.datatype.fields_map.items():
+            auto = Concept.model_construct(
+                name=key,
+                datatype=arg_to_datatype(value),
+                purpose=Purpose.PROPERTY,
+                namespace=(
+                    environment.namespace + "." + concept.name
+                    if environment.namespace
+                    and environment.namespace != DEFAULT_NAMESPACE
+                    else concept.name
+                ),
+                lineage=AttrAccess([concept.reference, key], environment=environment),
+                grain=concept.grain,
+                metadata=Metadata(
+                    concept_source=ConceptSource.AUTO_DERIVED,
+                ),
+                keys=concept.keys,
+            )
+            environment.add_concept(auto, meta=meta)
+            if isinstance(value, Concept):
+                environment.merge_concept(auto, value, modifiers=[])