pytrilogy 0.0.3.55__py3-none-any.whl → 0.0.3.56__py3-none-any.whl

trilogy/core/processing/discovery_validation.py

@@ -0,0 +1,155 @@
+from collections import defaultdict
+from enum import Enum
+from typing import List
+
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildWhereClause,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.nodes import (
+    StrategyNode,
+)
+from trilogy.core.processing.utility import (
+    get_disconnected_components,
+)
+
+
+class ValidationResult(Enum):
+    COMPLETE = 1
+    DISCONNECTED = 2
+    INCOMPLETE = 3
+    INCOMPLETE_CONDITION = 4
+
+
+def validate_concept(
+    concept: BuildConcept,
+    node: StrategyNode,
+    found_addresses: set[str],
+    non_partial_addresses: set[str],
+    partial_addresses: set[str],
+    virtual_addresses: set[str],
+    found_map: dict[str, set[BuildConcept]],
+    accept_partial: bool,
+    seen: set[str],
+    environment: BuildEnvironment,
+):
+    found_map[str(node)].add(concept)
+    seen.add(concept.address)
+    if concept not in node.partial_concepts:
+        found_addresses.add(concept.address)
+        non_partial_addresses.add(concept.address)
+        # remove it from our partial tracking
+        if concept.address in partial_addresses:
+            partial_addresses.remove(concept.address)
+        if concept.address in virtual_addresses:
+            virtual_addresses.remove(concept.address)
+    if concept in node.partial_concepts:
+        if concept.address in non_partial_addresses:
+            return None
+        partial_addresses.add(concept.address)
+        if accept_partial:
+            found_addresses.add(concept.address)
+            found_map[str(node)].add(concept)
+    for v_address in concept.pseudonyms:
+        if v_address in seen:
+            return
+        v = environment.concepts[v_address]
+        if v.address in seen:
+            return
+        if v.address == concept.address:
+            return
+        validate_concept(
+            v,
+            node,
+            found_addresses,
+            non_partial_addresses,
+            partial_addresses,
+            virtual_addresses,
+            found_map,
+            accept_partial,
+            seen=seen,
+            environment=environment,
+        )
+
+
+def validate_stack(
+    environment: BuildEnvironment,
+    stack: List[StrategyNode],
+    concepts: List[BuildConcept],
+    mandatory_with_filter: List[BuildConcept],
+    conditions: BuildWhereClause | None = None,
+    accept_partial: bool = False,
+) -> tuple[ValidationResult, set[str], set[str], set[str], set[str]]:
+    found_map: dict[str, set[BuildConcept]] = defaultdict(set)
+    found_addresses: set[str] = set()
+    non_partial_addresses: set[str] = set()
+    partial_addresses: set[str] = set()
+    virtual_addresses: set[str] = set()
+    seen: set[str] = set()
+
+    for node in stack:
+        resolved = node.resolve()
+
+        for concept in resolved.output_concepts:
+            if concept.address in resolved.hidden_concepts:
+                continue
+
+            validate_concept(
+                concept,
+                node,
+                found_addresses,
+                non_partial_addresses,
+                partial_addresses,
+                virtual_addresses,
+                found_map,
+                accept_partial,
+                seen,
+                environment,
+            )
+        for concept in node.virtual_output_concepts:
+            if concept.address in non_partial_addresses:
+                continue
+            found_addresses.add(concept.address)
+            virtual_addresses.add(concept.address)
+    if not conditions:
+        conditions_met = True
+    else:
+        conditions_met = all(
+            [node.preexisting_conditions == conditions.conditional for node in stack]
+        ) or all([c.address in found_addresses for c in mandatory_with_filter])
+    # zip in those we know we found
+    if not all([c.address in found_addresses for c in concepts]) or not conditions_met:
+        if not all([c.address in found_addresses for c in concepts]):
+            return (
+                ValidationResult.INCOMPLETE,
+                found_addresses,
+                {c.address for c in concepts if c.address not in found_addresses},
+                partial_addresses,
+                virtual_addresses,
+            )
+        return (
+            ValidationResult.INCOMPLETE_CONDITION,
+            found_addresses,
+            {c.address for c in concepts if c.address not in mandatory_with_filter},
+            partial_addresses,
+            virtual_addresses,
+        )
+
+    graph_count, _ = get_disconnected_components(found_map)
+    if graph_count in (0, 1):
+        return (
+            ValidationResult.COMPLETE,
+            found_addresses,
+            set(),
+            partial_addresses,
+            virtual_addresses,
+        )
+    # if we have too many subgraphs, we need to keep searching
+    return (
+        ValidationResult.DISCONNECTED,
+        found_addresses,
+        set(),
+        partial_addresses,
+        virtual_addresses,
+    )

trilogy/core/processing/node_generators/select_node.py

@@ -19,8 +19,7 @@ LOGGER_PREFIX = "[GEN_SELECT_NODE]"
 
 
 def gen_select_node(
-    concept: BuildConcept,
-    local_optional: list[BuildConcept],
+    concepts: list[BuildConcept],
     environment: BuildEnvironment,
     g,
     depth: int,
@@ -28,12 +27,11 @@ def gen_select_node(
     fail_if_not_found: bool = True,
     conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
-    all_concepts = [concept] + local_optional
-    all_lcl = LooseBuildConceptList(concepts=all_concepts)
+    all_lcl = LooseBuildConceptList(concepts=concepts)
     materialized_lcl = LooseBuildConceptList(
         concepts=[
             x
-            for x in all_concepts
+            for x in concepts
             if x.address in environment.materialized_concepts
             or x.derivation == Derivation.CONSTANT
         ]
@@ -41,15 +39,15 @@ def gen_select_node(
     if materialized_lcl != all_lcl:
         missing = all_lcl.difference(materialized_lcl)
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} Skipping select node generation for {concept.address}"
+            f"{padding(depth)}{LOGGER_PREFIX} Skipping select node generation for {concepts}"
             f" as it + optional includes non-materialized concepts (looking for all {all_lcl}, missing {missing}) "
         )
         if fail_if_not_found:
-            raise NoDatasourceException(f"No datasource exists for {concept}")
+            raise NoDatasourceException(f"No datasource exists for {concepts}")
         return None
 
     return gen_select_merge_node(
-        [concept] + local_optional,
+        concepts,
         g=g,
         environment=environment,
         depth=depth,

trilogy/core/processing/nodes/__init__.py

@@ -150,7 +150,6 @@ class History(BaseModel):
         environment: BuildEnvironment,
         g,
         depth: int,
-        source_concepts,
         fail_if_not_found: bool = False,
         accept_partial: bool = False,
         accept_partial_optional: bool = False,
@@ -169,8 +168,7 @@ class History(BaseModel):
         if fingerprint in self.select_history:
             return self.select_history[fingerprint]
         gen = gen_select_node(
-            concept,
-            local_optional,
+            [concept] + local_optional,
             environment,
             g,
             depth + 1,
@@ -190,8 +188,8 @@ __all__ = [
     "WindowNode",
     "StrategyNode",
     "NodeJoin",
-    "ConstantNode",
     "UnnestNode",
+    "ConstantNode",
     "UnionNode",
     "History",
     "WhereSafetyNode",

trilogy/dialect/snowflake.py

@@ -47,7 +47,7 @@ SNOWFLAKE_SQL_TEMPLATE = Template(
 CREATE OR REPLACE TABLE {{ output.address.location }} AS
 {% endif %}{%- if ctes %}
 WITH {% if recursive%}RECURSIVE{% endif %}{% for cte in ctes %}
-{{cte.name}} as ({{cte.statement}}){% if not loop.last %},{% endif %}{% else %}
+"{{cte.name}}" as ({{cte.statement}}){% if not loop.last %},{% endif %}{% else %}
 {% endfor %}{% endif %}
 {%- if full_select -%}
 {{full_select}}

trilogy/parsing/common.py

@@ -397,7 +397,7 @@ def group_function_to_concept(
             modifiers=modifiers,
             grain=grain,
             metadata=fmetadata,
-            derivation=Derivation.BASIC,
+            derivation=Derivation.GROUP_TO,
             granularity=granularity,
         )
         return r
@@ -654,7 +654,6 @@ def agg_wrapper_to_concept(
     fmetadata = metadata or Metadata()
     aggfunction = parent.function
     modifiers = get_upstream_modifiers(parent.concept_arguments, environment)
-    # derivation = Concept.calculate_derivation(parent, Purpose.PROPERTY)
     grain = Grain.from_concepts(parent.by, environment) if parent.by else Grain()
     granularity = Concept.calculate_granularity(Derivation.AGGREGATE, grain, parent)
 
@@ -778,7 +777,6 @@ def rowset_to_concepts(rowset: RowsetDerivationStatement, environment: Environme
     for x in pre_output:
         x.lineage = RowsetItem(
             content=orig_map[x.address].reference,
-            # where=rowset.select.where_clause,
             rowset=RowsetLineage(
                 name=rowset.name,
                 derived_concepts=[x.reference for x in pre_output],

trilogy/parsing/parse_engine.py

@@ -74,6 +74,7 @@ from trilogy.core.models.author import (
     Parenthetical,
     RowsetItem,
     SubselectComparison,
+    UndefinedConceptFull,
     WhereClause,
     Window,
     WindowItem,
@@ -962,6 +963,11 @@ class ParseToObjects(Transformer):
             targets = {sources[0].address: self.environment.concepts[target]}
 
         if self.parse_pass == ParsePass.VALIDATION:
+            for source_c in sources:
+                if isinstance(source_c, UndefinedConceptFull):
+                    raise SyntaxError(
+                        f"Cannot merge non-existent source concept {source_c.address} on line: {meta.line}"
+                    )
             new = MergeStatementV2(
                 sources=sources,
                 targets=targets,

trilogy/std/date.preql

@@ -2,7 +2,9 @@
 
 type year int;
 type month int;
+type week int;
 type day int;
 type hour int;
 type minute int;
-type second int;
+type second int;
+type day_of_week int;

trilogy/std/geography.preql

@@ -15,3 +15,7 @@ type city string; # City name
 type country string; # Country name
 type timezone string; # Timezone name
 type region string; # Region name
+
+
+## special formats
+type geojson string; # GeoJSON format for geographic data

trilogy/std/money.preql

@@ -1,6 +1,67 @@
+# generic currency types
 
+# Major global currencies
+type usd numeric;  # US Dollar
+type eur numeric;  # Euro
+type gbp numeric;  # British Pound Sterling
+type jpy numeric;  # Japanese Yen
+type chf numeric;  # Swiss Franc
+type cad numeric;  # Canadian Dollar
+type aud numeric;  # Australian Dollar
+type nzd numeric;  # New Zealand Dollar
 
-# generic currency types
-type usd numeric;
-type eur numeric;
-type gbp numeric;
+# Asian currencies
+type cny numeric;  # Chinese Yuan
+type hkd numeric;  # Hong Kong Dollar
+type sgd numeric;  # Singapore Dollar
+type krw numeric;  # South Korean Won
+type inr numeric;  # Indian Rupee
+type thb numeric;  # Thai Baht
+type php numeric;  # Philippine Peso
+type myr numeric;  # Malaysian Ringgit
+type idr numeric;  # Indonesian Rupiah
+type vnd numeric;  # Vietnamese Dong
+
+# European currencies (non-Euro)
+type sek numeric;  # Swedish Krona
+type nok numeric;  # Norwegian Krone
+type dkk numeric;  # Danish Krone
+type pln numeric;  # Polish Zloty
+type czk numeric;  # Czech Koruna
+type huf numeric;  # Hungarian Forint
+type ron numeric;  # Romanian Leu
+type bgn numeric;  # Bulgarian Lev
+type rub numeric;  # Russian Ruble
+
+# Middle Eastern currencies
+type aed numeric;  # UAE Dirham
+type sar numeric;  # Saudi Riyal
+type qar numeric;  # Qatari Riyal
+type kwd numeric;  # Kuwaiti Dinar
+type bhd numeric;  # Bahraini Dinar
+type omr numeric;  # Omani Rial
+type jod numeric;  # Jordanian Dinar
+type ils numeric;  # Israeli Shekel
+type try numeric;  # Turkish Lira
+
+# African currencies
+type zar numeric;  # South African Rand
+type egp numeric;  # Egyptian Pound
+type ngn numeric;  # Nigerian Naira
+type mad numeric;  # Moroccan Dirham
+type kes numeric;  # Kenyan Shilling
+type ghs numeric;  # Ghanaian Cedi
+
+# Latin American currencies
+type mxn numeric;  # Mexican Peso
+type brl numeric;  # Brazilian Real
+type ars numeric;  # Argentine Peso
+type cop numeric;  # Colombian Peso
+type pen numeric;  # Peruvian Sol
+type clp numeric;  # Chilean Peso
+type uyu numeric;  # Uruguayan Peso
+
+# Other notable currencies
+type isk numeric;  # Icelandic Krona
+type xof numeric;  # West African CFA Franc
+type xaf numeric;  # Central African CFA Franc

trilogy/std/net.preql

@@ -0,0 +1,8 @@
+
+
+type url string;
+type domain string;
+type ip_net_mask string;
+type ipv6_address string;
+type ipv4_address string;
+type suffix string;