dataforge-core 1.4.0__tar.gz → 1.5.0__tar.gz

{dataforge_core-1.4.0/cli/dataforge_core.egg-info → dataforge_core-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataforge-core
-Version: 1.4.0
+Version: 1.5.0
 Summary: Command line compiler for dataforge core projects
 Author-email: Vadim Orlov <vorlov@dataforgelabs.com>
 Project-URL: Homepage, https://github.com/dataforgelabs/dataforge-core

{dataforge_core-1.4.0 → dataforge_core-1.5.0}/cli/dataforge/resources/pg_deploy.sql

@@ -1,4 +1,4 @@
---Built: Fri Aug  8 16:10:46 UTC 2025
+--Built: Wed Jan 21 20:38:48 UTC 2026
 CREATE SCHEMA IF NOT EXISTS meta;
 CREATE SCHEMA IF NOT EXISTS log;
 DO $$
@@ -187,8 +187,6 @@ CREATE TABLE IF NOT EXISTS meta.output_column
     updated_userid text COLLATE pg_catalog."default",
     update_datetime timestamp without time zone,
     description text COLLATE pg_catalog."default",
-    partition_ordinal integer,
-    zorder_ordinal integer,
     CONSTRAINT pk_output_column PRIMARY KEY (output_column_id),
     CONSTRAINT fk_output_column_output_id FOREIGN KEY (output_id)
         REFERENCES meta.output (output_id) MATCH SIMPLE
@@ -210,7 +208,6 @@ CREATE TABLE IF NOT EXISTS meta.aggregate (
     numeric_flag   BOOLEAN,
     data_type      TEXT
 );
-
 TRUNCATE TABLE meta.aggregate;
 INSERT INTO meta.aggregate(aggregate_name, description, numeric_flag, data_type)
 VALUES ('any', 'Returns true if at least one value of `expr` is true.', FALSE, 'boolean'),
@@ -293,7 +290,6 @@ CREATE TABLE IF NOT EXISTS meta.enrichment
         ON DELETE NO ACTION,
     CONSTRAINT enrichment_rule_type_code_check CHECK (rule_type_code = ANY (ARRAY['V'::bpchar, 'E'::bpchar])),
     CONSTRAINT enrichment_datatype_fkey FOREIGN KEY (datatype) REFERENCES meta.attribute_type(hive_type),
-    CONSTRAINT enrichment_cast_datatype_fkey FOREIGN KEY (datatype) REFERENCES meta.attribute_type(hive_type),
     CONSTRAINT enrichment_parent_enrichment_id_fkey FOREIGN KEY (parent_enrichment_id)
         REFERENCES meta.enrichment (enrichment_id) ON DELETE CASCADE
 );
@@ -689,6 +685,17 @@ v_ret := v_ret || jsonb_build_object('source_cardinality',v_decoded[1]) || jsonb
 
 RETURN v_ret  - 'name' - 'cardinality';
 
+END;
+$function$;CREATE OR REPLACE FUNCTION meta.imp_map_connection_type(in_body jsonb)
+    RETURNS jsonb
+    LANGUAGE plpgsql
+AS
+$function$
+DECLARE
+    v_connection_type text = in_body->>'connection_type';
+
+BEGIN
+    RETURN CASE v_connection_type WHEN 'custom' THEN in_body || jsonb_build_object('connection_type','custom_ingestion') ELSE in_body END;
 END;
 $function$;CREATE OR REPLACE FUNCTION meta.imp_map_relations(in_relation_uids jsonb)
     RETURNS int[]
@@ -740,7 +747,7 @@ BEGIN
         SELECT import_object_id, body_text::jsonb body
         FROM meta.import_object WHERE import_id = in_import_id)
     UPDATE meta.import_object io
-    SET body = p.body, 
+    SET body = CASE WHEN object_type IN ('source') THEN meta.imp_map_connection_type(p.body) ELSE p.body END, 
         name =  CASE WHEN object_type IN ('source' ,'output', 'group', 'token') THEN p.body->>(object_type || '_name')
                 WHEN object_type IN ('output_template','source_template') THEN p.body->>'object_name'
         ELSE p.body->>'name' END
@@ -2277,6 +2284,7 @@ DECLARE
     v_processing_type           text;
     v_loop_check                text;
     v_upstream_keep_current_flag boolean;
+    v_platform text = meta.u_sys_config('lakehouse-platform');
 
 BEGIN
 
@@ -2293,11 +2301,14 @@ THEN
     RETURN json_build_object('error', 'Please use the description section for any comments.', 'expression', NULL);
 END IF;
 
+
 -- check attribute name syntax
-IF NOT in_enr.attribute_name ~ '^[a-z_]+[a-z0-9_]*$'
-THEN
-    v_attribute_name_error := 'Invalid attribute name syntax. Attribute name has to start with lowercase letter or _ It may contain lowercase letters, numbers and _';
-END IF;
+v_attribute_name_error := CASE 
+    WHEN v_platform = 'snowflake' AND NOT in_enr.attribute_name ~ '^[A-Z_]+[A-Z0-9_]*$' THEN 
+        'Invalid attribute name syntax. Attribute name has to start with uppercase letter or _ It may contain uppercase letters, numbers and _'
+    WHEN v_platform = 'databricks' AND NOT in_enr.attribute_name ~ '^[a-z_]+[a-z0-9_]*$' THEN 
+        'Invalid attribute name syntax. Attribute name has to start with lowercase letter or _ It may contain lowercase letters, numbers and _'
+    END;
 
 v_template_match_flag := in_template_check_flag and exists (select 1 from meta.enrichment WHERE source_id = in_enr.source_id AND expression = in_enr.expression AND attribute_name = in_enr.attribute_name AND name = in_enr.name);
 
@@ -2955,7 +2966,11 @@ BEGIN
         END IF;
 
         v_ret_expression := v_ret_expression || CASE WHEN v_aggregates_exist_flag AND v_aggregate_id IS NULL 
-            THEN  'first_value(' || v_attribute_name || ')' -- wrap non-aggregated parameter into aggregate for data type testing purposes only
+            THEN  -- wrap non-aggregated parameter into aggregate for data type testing purposes only
+                CASE meta.u_sys_config('lakehouse-platform') 
+                    WHEN 'databricks' THEN  'first_value(' || v_attribute_name || ')' 
+                    WHEN 'snowflake' THEN  'ARRAY_AGG(' || v_attribute_name || ')[0]' 
+                END
             ELSE v_attribute_name END;
 
         v_last_end = v_end + 1;
@@ -2964,9 +2979,13 @@ BEGIN
     v_ret_expression := v_ret_expression || substr(in_expression,v_last_end);
 
     IF NULLIF(in_cast_datatype,'') IS NOT NULL THEN
+        IF in_cast_datatype = 'DECIMAL' THEN
+            in_cast_datatype = 'NUMERIC(38,12)';
+        END IF;
         v_ret_expression := format('CAST(%s as %s)',v_ret_expression,in_cast_datatype);
     END IF;
 
+
     RAISE DEBUG 'v_exp_test_select_list: %',v_exp_test_select_list;
     
     IF cardinality(v_exp_test_select_list) > 0 THEN
@@ -3034,7 +3053,11 @@ BEGIN
 
         v_ret_expression := replace(v_ret_expression, format('P<%s>', v_ep.enrichment_parameter_id), 
             CASE WHEN v_aggregates_exist_flag AND v_ep.aggregation_id IS NULL 
-            THEN  'first_value(' || v_attribute_name || ')' -- wrap non-aggregated parameter into aggregate for data type testing purposes only
+            THEN  -- wrap non-aggregated parameter into aggregate for data type testing purposes only
+                CASE meta.u_sys_config('lakehouse-platform') 
+                    WHEN 'databricks' THEN  'first_value(' || v_attribute_name || ')' 
+                    WHEN 'snowflake' THEN  'ARRAY_AGG(' || v_attribute_name || ')[0]' 
+                END
             ELSE v_attribute_name END);
 
         IF in_root_id IS NOT NULL AND v_param.type IN ('raw','enrichment') THEN 
@@ -3168,21 +3191,21 @@ BEGIN
   --Recursively get all downstream rules. 
   -- Include rules used by relations
   --Return rule chain if enrichment contains downstream self-reference
+  -- Stop check on first duplicate rule in chain
 
 
   WITH RECURSIVE ds AS (
-    SELECT enrichment_id, path, 0 level
-    FROM meta.u_get_upstream_rules(in_enrichment_id) ur
+    SELECT in_enrichment_id enrichment_id, ARRAY[v_start_node_id] path, 0 level, false loop_flag
     UNION ALL 
-    SELECT der.enrichment_id, ds.path || der.path, ds.level + 1
+    SELECT der.enrichment_id, ds.path || der.path, ds.level + 1, der.path && ds.path
     FROM ds
     CROSS JOIN LATERAL meta.u_get_upstream_rules(ds.enrichment_id, ds.level + 1) der
-    WHERE NOT v_start_node_id = ANY(ds.path)
+    WHERE NOT ds.loop_flag AND ds.level < 10
   ) 
-  SELECT ARRAY['EN' || in_enrichment_id] || ds.path 
+  SELECT ds.path 
   INTO v_circular_path
   FROM ds
-  WHERE enrichment_id = in_enrichment_id;
+  WHERE loop_flag;
 
 
 
@@ -3282,7 +3305,8 @@ ELSEIF in_datatype = 'array' THEN
 ELSEIF in_datatype like 'decimal(%' THEN 
     v_exp :=  format('CAST(`decimal` AS decimal(38,12))',in_datatype);
 ELSE
-   v_exp :=  '`' || in_datatype || '`';
+   v_exp := CASE meta.u_sys_config('lakehouse-platform') WHEN 'databricks' THEN '`' || in_datatype || '`'
+   WHEN 'snowflake' THEN '"' || in_datatype || '"' END;
 END IF;   
 
 RETURN v_exp;
@@ -3596,9 +3620,7 @@ v_in_path_length := cardinality(in_start_path);
 IF to_regclass('meta.system_configuration') IS NULL THEN
 	v_max_length := greatest(4, v_in_path_length + 2);
 ELSE
-	SELECT greatest(sc.value::int, v_in_path_length + 2) INTO v_max_length
-	FROM meta.system_configuration sc
-	WHERE sc.name = 'max-relation-hops';
+	v_max_length := greatest(meta.u_sys_config('max-relation-hops')::int, v_in_path_length + 2);
 END IF;
 
 SELECT array_agg(r.id) INTO v_missing_relation_ids
@@ -3892,7 +3914,30 @@ END;
 
 $function$;
 
--- build and return datatype_schema if it's null (legacy pre-8.0 attribute)
+
+CREATE OR REPLACE FUNCTION meta.u_get_schema_from_ddl_type(in_datatype text)
+    RETURNS jsonb
+    LANGUAGE 'plpgsql'
+    
+AS $BODY$
+
+DECLARE
+    v_type text;
+
+BEGIN
+
+v_type := CASE WHEN in_datatype LIKE 'VARCHAR%' THEN 'VARCHAR'
+    WHEN in_datatype LIKE 'TIMESTAMP%' THEN 'TIMESTAMP'
+    WHEN in_datatype = 'NUMBER%' THEN  CASE WHEN in_datatype ~ ',0\)' THEN 'INTEGER' ELSE 'DECIMAL' END
+    ELSE (SELECT hive_type FROM meta.attribute_type WHERE hive_ddl_type = in_datatype)
+    END;
+
+RETURN to_jsonb(v_type);
+
+
+END;
+
+$BODY$;-- build and return datatype_schema if it's null (legacy pre-8.0 attribute)
 CREATE OR REPLACE FUNCTION meta.u_get_schema_from_type(in_schema jsonb, in_datatype text)
     RETURNS jsonb
     LANGUAGE 'plpgsql'
@@ -3903,11 +3948,14 @@ DECLARE
 
      v_type text;
 
+
 BEGIN
 
 IF in_schema IS NOT NULL THEN
     IF jsonb_typeof(in_schema) = 'string' AND in_schema->>0 like 'decimal%' THEN
         RETURN to_jsonb('decimal(38,12)'::text);        
+    ELSEIF jsonb_typeof(in_schema) = 'string' AND in_schema->>0 like 'DECIMAL%' THEN
+        RETURN to_jsonb('DECIMAL(38,12)'::text);        
     ELSE
         RETURN in_schema;
     END IF;
@@ -3918,6 +3966,8 @@ IF in_datatype = 'int' THEN
     RETURN to_jsonb('integer'::text);
 ELSEIF in_datatype LIKE 'decimal%' THEN 
     RETURN to_jsonb('decimal(38,12)'::text);
+ELSEIF in_datatype LIKE 'DECIMAL%' THEN 
+    RETURN to_jsonb('DECIMAL(38,12)'::text);
 ELSEIF in_datatype IN ('struct','array') THEN 
     RETURN null;
 END IF;
@@ -4054,6 +4104,8 @@ IF v_type IN ('integer', 'byte','short') THEN
     v_type := 'int';
 ELSEIF v_type LIKE 'decimal%' THEN 
     v_type := 'decimal';
+ELSEIF v_type LIKE 'DECIMAL%' THEN 
+    v_type := 'DECIMAL';
 END IF;
 
 RETURN v_type;
@@ -4075,7 +4127,7 @@ BEGIN
   -- Include rules used by relations
   --Return rule chain if enrichment contains downstream self-reference
 
-    IF in_level = 0 AND EXISTS(SELECT 1 FROM information_schema.columns where table_name = '_params' AND column_name = 'type')
+    IF in_level = 1 AND EXISTS(SELECT 1 FROM information_schema.columns where table_name = '_params' AND column_name = 'type')
         THEN
         
         RETURN QUERY (
@@ -4588,7 +4640,7 @@ CREATE TEMP TABLE _aggs_parsed
             THEN
                 -- Check for window function
                 v_follow_text := substring(v_expression, v_expression_position + 1, length(v_expression));
-                v_window_function_flag := v_follow_text ~* '^\s*over\s*\(';
+                v_window_function_flag := v_follow_text ~* '(^|\s*within group\s*\([^\)]+\))\s*over\s*\(';
 
                 RAISE DEBUG 'Out of parenthesis. position=% v_window_function_flag=% v_follow_text=%',v_expression_position,v_window_function_flag, v_follow_text;
                 v_inner_text := substring(v_expression, v_aggregate_start_position, v_expression_position - v_aggregate_start_position);
@@ -4844,7 +4896,18 @@ END LOOP;
 RETURN v_ret;
 END;
 
-$function$;CREATE OR REPLACE FUNCTION meta.u_validate_expression_parameters(in_enr meta.enrichment)
+$function$;CREATE OR REPLACE FUNCTION meta.u_sys_config(in_name text)
+    RETURNS text
+    LANGUAGE 'plpgsql'
+AS $BODY$
+
+BEGIN
+
+RETURN CASE WHEN in_name = 'lakehouse-platform' THEN 'databricks' END;
+
+END;
+$BODY$;
+CREATE OR REPLACE FUNCTION meta.u_validate_expression_parameters(in_enr meta.enrichment)
     RETURNS text
     LANGUAGE 'plpgsql'
 AS $BODY$

{dataforge_core-1.4.0 → dataforge_core-1.5.0/cli/dataforge_core.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataforge-core
-Version: 1.4.0
+Version: 1.5.0
 Summary: Command line compiler for dataforge core projects
 Author-email: Vadim Orlov <vorlov@dataforgelabs.com>
 Project-URL: Homepage, https://github.com/dataforgelabs/dataforge-core

{dataforge_core-1.4.0 → dataforge_core-1.5.0}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 [project]
 name = "dataforge-core"
-version = "v1.4.0"
+version = "v1.5.0"
 authors = [
     {name="Vadim Orlov", email="vorlov@dataforgelabs.com"}
 ]