recce-nightly 0.59.0.20250325__py3-none-any.whl → 0.59.0.20250326__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of recce-nightly might be problematic. Click here for more details.
- recce/VERSION +1 -1
- recce/adapter/base.py +1 -1
- recce/adapter/dbt_adapter/__init__.py +210 -126
- recce/data/404.html +1 -1
- recce/data/_next/static/5XO9Q5RJ7mNkcELBP9S4D/_buildManifest.js +1 -0
- recce/data/_next/static/chunks/269-9a6f1447f09759ef.js +65 -0
- recce/data/_next/static/chunks/{29e3cc0d-b9131230fdb267e7.js → 29e3cc0d-6e3b0cf904a6874d.js} +1 -1
- recce/data/_next/static/chunks/36e1c10d-800596efe83e74a6.js +1 -0
- recce/data/_next/static/chunks/{523b8219-cd28a6efac8e1cf1.js → 38610ee6-24a85e2dfb424ddb.js} +1 -1
- recce/data/_next/static/chunks/{3998a672-18454f939c052b39.js → 3998a672-63fc54b2b517ee2b.js} +1 -1
- recce/data/_next/static/chunks/{450c323b-ff537c106edf3f59.js → 450c323b-b36e745822edebe2.js} +1 -1
- recce/data/_next/static/chunks/{47d8844f-cac2989cb39ba350.js → 47d8844f-0516cea2ef568380.js} +1 -1
- recce/data/_next/static/chunks/6dc81886-9871b611a22bd997.js +1 -0
- recce/data/_next/static/chunks/783-784ce10322400a48.js +2 -0
- recce/data/_next/static/chunks/{7a8a3e83-277b00caf162a419.js → 7a8a3e83-dd26fafa0a7421e5.js} +1 -1
- recce/data/_next/static/chunks/{7f27ae6c-9c6bdf82ccd2f646.js → 7f27ae6c-4ddea120696db611.js} +1 -1
- recce/data/_next/static/chunks/{9746af58-344daabf5df08121.js → 9746af58-527fd8cbd2d17eee.js} +1 -1
- recce/data/_next/static/chunks/{a30376cd-0e2abcaded0e4cfc.js → a30376cd-1ad5a3f522da896c.js} +1 -1
- recce/data/_next/static/chunks/app/_not-found/page-76f691066176b63a.js +1 -0
- recce/data/_next/static/chunks/app/layout-fdacc83a3bf974a3.js +1 -0
- recce/data/_next/static/chunks/app/page-139ac00bab13bd3c.js +1 -0
- recce/data/_next/static/chunks/{b63b1b3f-fbe2ac2c00b4eb2f.js → b63b1b3f-fe714039749c1250.js} +1 -1
- recce/data/_next/static/chunks/c132bf7d-4cb4c4a123e27340.js +1 -0
- recce/data/_next/static/chunks/c1ceaa8b-64142a5184f1b8b2.js +1 -0
- recce/data/_next/static/chunks/{cd9f8d63-ba9b342f3d5e92ac.js → cd9f8d63-543677b0ef49da87.js} +3 -3
- recce/data/_next/static/chunks/ce84277d-4bca7695ec1ada94.js +1 -0
- recce/data/_next/static/chunks/e24bf851-5db5db8459b0b291.js +1 -0
- recce/data/_next/static/chunks/{fee69bc6-4490eb47d2aaaa72.js → fee69bc6-1df8306af9d11293.js} +1 -1
- recce/data/_next/static/chunks/main-7183c49fe89a1f9b.js +1 -0
- recce/data/_next/static/chunks/main-app-cfd795a5fe58bf37.js +1 -0
- recce/data/_next/static/chunks/pages/_app-9f1049bbd7b90455.js +1 -0
- recce/data/_next/static/chunks/pages/_error-71ed60313c005df1.js +1 -0
- recce/data/_next/static/chunks/{webpack-5ab3237140d2c0f8.js → webpack-8a8709fe9d22c323.js} +1 -1
- recce/data/index.html +2 -2
- recce/data/index.txt +5 -5
- recce/server.py +24 -2
- recce/util/lineage.py +34 -0
- {recce_nightly-0.59.0.20250325.dist-info → recce_nightly-0.59.0.20250326.dist-info}/METADATA +1 -1
- {recce_nightly-0.59.0.20250325.dist-info → recce_nightly-0.59.0.20250326.dist-info}/RECORD +48 -47
- {recce_nightly-0.59.0.20250325.dist-info → recce_nightly-0.59.0.20250326.dist-info}/WHEEL +1 -1
- tests/adapter/dbt_adapter/dbt_test_helper.py +3 -1
- tests/adapter/dbt_adapter/test_dbt_cll.py +11 -9
- tests/tasks/test_schema.py +0 -3
- tests/test_dbt.py +1 -5
- recce/data/_next/static/H9OYwyR7yb7CjKNEdDwCX/_buildManifest.js +0 -1
- recce/data/_next/static/chunks/36e1c10d-28bb82b9a3d959e7.js +0 -1
- recce/data/_next/static/chunks/6dc81886-d45acadb4b6674c7.js +0 -1
- recce/data/_next/static/chunks/727-81ddfe5839f762d5.js +0 -65
- recce/data/_next/static/chunks/921-4158ef2dc23e4a01.js +0 -2
- recce/data/_next/static/chunks/app/_not-found/page-8fb2dd95da4548ae.js +0 -1
- recce/data/_next/static/chunks/app/layout-2ba8d6f3664176aa.js +0 -1
- recce/data/_next/static/chunks/app/page-db98e82acba724d7.js +0 -1
- recce/data/_next/static/chunks/c132bf7d-3375ff4277eceb81.js +0 -1
- recce/data/_next/static/chunks/c1ceaa8b-c4f0232ef5c2c9ba.js +0 -1
- recce/data/_next/static/chunks/ce84277d-3aa717f588901c96.js +0 -1
- recce/data/_next/static/chunks/e24bf851-38fffe87a7b8784a.js +0 -1
- recce/data/_next/static/chunks/main-535eacaa53931854.js +0 -1
- recce/data/_next/static/chunks/main-app-6f45568a6a7cd7b0.js +0 -1
- recce/data/_next/static/chunks/pages/_app-bbd1966052cb1e5d.js +0 -1
- recce/data/_next/static/chunks/pages/_error-2456b9a2e0600a52.js +0 -1
- /recce/data/_next/static/{H9OYwyR7yb7CjKNEdDwCX → 5XO9Q5RJ7mNkcELBP9S4D}/_ssgManifest.js +0 -0
- {recce_nightly-0.59.0.20250325.dist-info → recce_nightly-0.59.0.20250326.dist-info}/entry_points.txt +0 -0
- {recce_nightly-0.59.0.20250325.dist-info → recce_nightly-0.59.0.20250326.dist-info}/licenses/LICENSE +0 -0
- {recce_nightly-0.59.0.20250325.dist-info → recce_nightly-0.59.0.20250326.dist-info}/top_level.txt +0 -0
recce/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.59.0.
|
|
1
|
+
0.59.0.20250326
|
recce/adapter/base.py
CHANGED
|
@@ -42,7 +42,7 @@ class BaseAdapter(ABC):
|
|
|
42
42
|
|
|
43
43
|
diff[key] = NodeDiff(change_status='modified', change_category='breaking')
|
|
44
44
|
elif base_node:
|
|
45
|
-
diff[key] = NodeDiff(
|
|
45
|
+
diff[key] = NodeDiff(change_status='removed')
|
|
46
46
|
elif curr_node:
|
|
47
47
|
diff[key] = NodeDiff(change_status='added')
|
|
48
48
|
return LineageDiff(
|
|
@@ -3,6 +3,7 @@ import logging
|
|
|
3
3
|
import os
|
|
4
4
|
import uuid
|
|
5
5
|
from contextlib import contextmanager
|
|
6
|
+
from copy import deepcopy
|
|
6
7
|
from dataclasses import dataclass, fields
|
|
7
8
|
from errno import ENOENT
|
|
8
9
|
from functools import lru_cache
|
|
@@ -13,6 +14,7 @@ from recce.event import log_performance
|
|
|
13
14
|
from recce.exceptions import RecceException
|
|
14
15
|
from recce.util.cll import cll, CLLPerformanceTracking
|
|
15
16
|
from ...tasks.profile import ProfileTask
|
|
17
|
+
from recce.util.lineage import find_upstream, find_downstream
|
|
16
18
|
|
|
17
19
|
try:
|
|
18
20
|
import agate
|
|
@@ -571,22 +573,12 @@ class DbtAdapter(BaseAdapter):
|
|
|
571
573
|
|
|
572
574
|
return parent_map
|
|
573
575
|
|
|
574
|
-
|
|
575
|
-
def build_table_map(self, base: Optional[bool] = False) -> Dict[str, str]:
|
|
576
|
+
def build_parent_list_per_node(self, node_id: str, base: Optional[bool] = False) -> List[str]:
|
|
576
577
|
manifest = self.curr_manifest if base is False else self.base_manifest
|
|
577
578
|
manifest_dict = manifest.to_dict()
|
|
578
579
|
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
resource_type = node['resource_type']
|
|
582
|
-
if resource_type not in ['model', 'seed', 'exposure', 'snapshot']:
|
|
583
|
-
continue
|
|
584
|
-
table_map[node['unique_id']] = node.get('alias')
|
|
585
|
-
|
|
586
|
-
for source in manifest_dict['sources'].values():
|
|
587
|
-
table_map[source['unique_id']] = source.get('identifier')
|
|
588
|
-
|
|
589
|
-
return table_map
|
|
580
|
+
if node_id in manifest_dict['parent_map']:
|
|
581
|
+
return manifest_dict['parent_map'][node_id]
|
|
590
582
|
|
|
591
583
|
def get_lineage(self, base: Optional[bool] = False):
|
|
592
584
|
manifest = self.curr_manifest if base is False else self.base_manifest
|
|
@@ -735,16 +727,10 @@ class DbtAdapter(BaseAdapter):
|
|
|
735
727
|
|
|
736
728
|
parent_map = self.build_parent_map(nodes, base)
|
|
737
729
|
|
|
738
|
-
# Handle column-level lineage, only enable if env var is set to true
|
|
739
|
-
if os.getenv('RECCE_CLL_ENABLED') != 'false':
|
|
740
|
-
if base is False:
|
|
741
|
-
cll_tracker.start_column_lineage()
|
|
742
|
-
self.append_column_lineage(nodes, parent_map, base)
|
|
743
|
-
cll_tracker.end_column_lineage()
|
|
744
|
-
|
|
745
730
|
if base is False:
|
|
746
731
|
cll_tracker.end_lineage()
|
|
747
|
-
|
|
732
|
+
cll_tracker.set_total_nodes(len(nodes))
|
|
733
|
+
log_performance('model lineage', cll_tracker.to_dict())
|
|
748
734
|
cll_tracker.reset()
|
|
749
735
|
|
|
750
736
|
return dict(
|
|
@@ -754,111 +740,6 @@ class DbtAdapter(BaseAdapter):
|
|
|
754
740
|
catalog_metadata=catalog_metadata,
|
|
755
741
|
)
|
|
756
742
|
|
|
757
|
-
def append_column_lineage(self, nodes: Dict, parent_map: Dict, base: Optional[bool] = False):
|
|
758
|
-
def _apply_all_columns(node, trans_type, depends_on):
|
|
759
|
-
for col in node.get('columns', {}).values():
|
|
760
|
-
col['transformation_type'] = trans_type
|
|
761
|
-
col['depends_on'] = depends_on
|
|
762
|
-
|
|
763
|
-
def _depend_node_to_id(column_lineage, nodes):
|
|
764
|
-
for cl in column_lineage.values():
|
|
765
|
-
for depend_on in cl.depends_on:
|
|
766
|
-
if depend_on.node.startswith('__'):
|
|
767
|
-
for n in nodes.values():
|
|
768
|
-
if n.get('resource_type') != 'source':
|
|
769
|
-
continue
|
|
770
|
-
# __source__table -> source.table
|
|
771
|
-
source_table = depend_on.node.lstrip("_").replace("__", ".", 1).lower()
|
|
772
|
-
if source_table in n.get('id'):
|
|
773
|
-
depend_on.node = n.get('id')
|
|
774
|
-
break
|
|
775
|
-
else:
|
|
776
|
-
for n in nodes.values():
|
|
777
|
-
if n.get('name') == depend_on.node.lower():
|
|
778
|
-
depend_on.node = n.get('id')
|
|
779
|
-
break
|
|
780
|
-
|
|
781
|
-
cll_tracker = CLLPerformanceTracking()
|
|
782
|
-
cll_tracker.set_total_nodes(len(nodes))
|
|
783
|
-
manifest = as_manifest(self.get_manifest(base))
|
|
784
|
-
for node in nodes.values():
|
|
785
|
-
resource_type = node.get('resource_type')
|
|
786
|
-
if resource_type not in {'model', 'seed', 'source', 'snapshot'}:
|
|
787
|
-
continue
|
|
788
|
-
|
|
789
|
-
if resource_type == 'source' or resource_type == 'seed':
|
|
790
|
-
_apply_all_columns(node, 'source', [])
|
|
791
|
-
continue
|
|
792
|
-
|
|
793
|
-
if node.get('raw_code') is None or self.is_python_model(node.get('id'), base=base):
|
|
794
|
-
_apply_all_columns(node, 'unknown', [])
|
|
795
|
-
continue
|
|
796
|
-
|
|
797
|
-
# dbt <= 1.8, MetricFlow expects the time spine table to be named metricflow_time_spine
|
|
798
|
-
if node.get('name') == 'metricflow_time_spine':
|
|
799
|
-
_apply_all_columns(node, 'source', [])
|
|
800
|
-
continue
|
|
801
|
-
|
|
802
|
-
if not node.get('columns', {}):
|
|
803
|
-
# no catalog
|
|
804
|
-
continue
|
|
805
|
-
|
|
806
|
-
def ref_func(*args):
|
|
807
|
-
if len(args) == 1:
|
|
808
|
-
node = args[0]
|
|
809
|
-
elif len(args) > 1:
|
|
810
|
-
node = args[1]
|
|
811
|
-
else:
|
|
812
|
-
return None
|
|
813
|
-
return node
|
|
814
|
-
|
|
815
|
-
def source_func(source_name, table_name):
|
|
816
|
-
return f"__{source_name}__{table_name}"
|
|
817
|
-
|
|
818
|
-
raw_code = node.get('raw_code')
|
|
819
|
-
jinja_context = dict(
|
|
820
|
-
ref=ref_func,
|
|
821
|
-
source=source_func,
|
|
822
|
-
)
|
|
823
|
-
|
|
824
|
-
schema = {}
|
|
825
|
-
for parent_id in parent_map[node.get('id')]:
|
|
826
|
-
parent_node = nodes.get(parent_id)
|
|
827
|
-
if parent_node is None:
|
|
828
|
-
continue
|
|
829
|
-
columns = parent_node.get('columns') or {}
|
|
830
|
-
name = parent_node.get('name')
|
|
831
|
-
if parent_node.get('resource_type') == 'source':
|
|
832
|
-
parts = parent_id.split('.')
|
|
833
|
-
source = parts[2]
|
|
834
|
-
table = parts[3]
|
|
835
|
-
name = f"__{source}__{table}"
|
|
836
|
-
schema[name] = {
|
|
837
|
-
name: column.get('type') for name, column in columns.items()
|
|
838
|
-
}
|
|
839
|
-
|
|
840
|
-
try:
|
|
841
|
-
# provide a manifest to speedup and not pollute the manifest
|
|
842
|
-
compiled_sql = self.generate_sql(raw_code, base=base, context=jinja_context, provided_manifest=manifest)
|
|
843
|
-
dialect = self.adapter.type()
|
|
844
|
-
column_lineage = cll(compiled_sql, schema=schema, dialect=dialect)
|
|
845
|
-
except RecceException:
|
|
846
|
-
# TODO: provide parsing error message if needed
|
|
847
|
-
_apply_all_columns(node, 'unknown', [])
|
|
848
|
-
cll_tracker.increment_sqlglot_error_nodes()
|
|
849
|
-
continue
|
|
850
|
-
except Exception:
|
|
851
|
-
_apply_all_columns(node, 'unknown', [])
|
|
852
|
-
cll_tracker.increment_other_error_nodes()
|
|
853
|
-
continue
|
|
854
|
-
|
|
855
|
-
_depend_node_to_id(column_lineage, nodes)
|
|
856
|
-
|
|
857
|
-
for name, column in node.get('columns', {}).items():
|
|
858
|
-
if name in column_lineage:
|
|
859
|
-
column['depends_on'] = column_lineage[name].depends_on
|
|
860
|
-
column['transformation_type'] = column_lineage[name].type
|
|
861
|
-
|
|
862
743
|
@lru_cache(maxsize=1)
|
|
863
744
|
def _get_lineage_diff_cached(self, cache_key) -> LineageDiff:
|
|
864
745
|
base = self.get_lineage(base=True)
|
|
@@ -902,6 +783,206 @@ class DbtAdapter(BaseAdapter):
|
|
|
902
783
|
diff=diff,
|
|
903
784
|
)
|
|
904
785
|
|
|
786
|
+
def get_cll_by_node_id(self, node_id: str, base: Optional[bool] = False):
|
|
787
|
+
cll_tracker = CLLPerformanceTracking()
|
|
788
|
+
cll_tracker.start_column_lineage()
|
|
789
|
+
|
|
790
|
+
manifest = self.curr_manifest if base is False else self.base_manifest
|
|
791
|
+
manifest_dict = manifest.to_dict()
|
|
792
|
+
|
|
793
|
+
parent_ids = find_upstream(node_id, manifest_dict.get('parent_map'))
|
|
794
|
+
child_ids = find_downstream(node_id, manifest_dict.get('child_map'))
|
|
795
|
+
cll_node_ids = parent_ids.union(child_ids)
|
|
796
|
+
cll_node_ids.add(node_id)
|
|
797
|
+
|
|
798
|
+
node_manifest = self.get_lineage_nodes_metadata(base=base)
|
|
799
|
+
nodes = {}
|
|
800
|
+
for node_id in cll_node_ids:
|
|
801
|
+
if node_id not in node_manifest:
|
|
802
|
+
continue
|
|
803
|
+
nodes[node_id] = self.get_cll_cached(node_id, base=base)
|
|
804
|
+
|
|
805
|
+
cll_tracker.end_column_lineage()
|
|
806
|
+
cll_tracker.set_total_nodes(len(nodes))
|
|
807
|
+
log_performance('column level lineage', cll_tracker.to_dict())
|
|
808
|
+
cll_tracker.reset()
|
|
809
|
+
|
|
810
|
+
return dict(nodes=nodes)
|
|
811
|
+
|
|
812
|
+
@lru_cache(maxsize=128)
|
|
813
|
+
def get_cll_cached(self, node_id: str, base: Optional[bool] = False):
|
|
814
|
+
nodes = self.get_lineage_nodes_metadata(base=base)
|
|
815
|
+
|
|
816
|
+
manifest = self.curr_manifest if base is False else self.base_manifest
|
|
817
|
+
manifest_dict = manifest.to_dict()
|
|
818
|
+
parent_list = []
|
|
819
|
+
if node_id in manifest_dict['parent_map']:
|
|
820
|
+
parent_list = manifest_dict['parent_map'][node_id]
|
|
821
|
+
|
|
822
|
+
node = deepcopy(nodes[node_id])
|
|
823
|
+
self.append_column_lineage(node, parent_list, base)
|
|
824
|
+
return node
|
|
825
|
+
|
|
826
|
+
def append_column_lineage(self, node: Dict, parent_list: List, base: Optional[bool] = False):
|
|
827
|
+
def _apply_all_columns(node, trans_type, depends_on):
|
|
828
|
+
for col in node.get('columns', {}).values():
|
|
829
|
+
col['transformation_type'] = trans_type
|
|
830
|
+
col['depends_on'] = depends_on
|
|
831
|
+
|
|
832
|
+
def _depend_node_to_id(column_lineage, nodes):
|
|
833
|
+
for cl in column_lineage.values():
|
|
834
|
+
for depend_on in cl.depends_on:
|
|
835
|
+
if depend_on.node.startswith('__'):
|
|
836
|
+
for n in nodes.values():
|
|
837
|
+
if n.get('resource_type') != 'source':
|
|
838
|
+
continue
|
|
839
|
+
# __source__table -> source.table
|
|
840
|
+
source_table = depend_on.node.lstrip("_").replace("__", ".", 1).lower()
|
|
841
|
+
if source_table in n.get('id'):
|
|
842
|
+
depend_on.node = n.get('id')
|
|
843
|
+
break
|
|
844
|
+
else:
|
|
845
|
+
for n in nodes.values():
|
|
846
|
+
if n.get('name') == depend_on.node.lower():
|
|
847
|
+
depend_on.node = n.get('id')
|
|
848
|
+
break
|
|
849
|
+
|
|
850
|
+
cll_tracker = CLLPerformanceTracking()
|
|
851
|
+
nodes = self.get_lineage_nodes_metadata(base=base)
|
|
852
|
+
manifest = as_manifest(self.get_manifest(base))
|
|
853
|
+
resource_type = node.get('resource_type')
|
|
854
|
+
if resource_type not in {'model', 'seed', 'source', 'snapshot'}:
|
|
855
|
+
return
|
|
856
|
+
|
|
857
|
+
if resource_type == 'source' or resource_type == 'seed':
|
|
858
|
+
_apply_all_columns(node, 'source', [])
|
|
859
|
+
return
|
|
860
|
+
|
|
861
|
+
if node.get('raw_code') is None or self.is_python_model(node.get('id'), base=base):
|
|
862
|
+
_apply_all_columns(node, 'unknown', [])
|
|
863
|
+
return
|
|
864
|
+
|
|
865
|
+
# dbt <= 1.8, MetricFlow expects the time spine table to be named metricflow_time_spine
|
|
866
|
+
if node.get('name') == 'metricflow_time_spine':
|
|
867
|
+
_apply_all_columns(node, 'source', [])
|
|
868
|
+
return
|
|
869
|
+
|
|
870
|
+
if not node.get('columns', {}):
|
|
871
|
+
# no catalog
|
|
872
|
+
return
|
|
873
|
+
|
|
874
|
+
def ref_func(*args):
|
|
875
|
+
if len(args) == 1:
|
|
876
|
+
node = args[0]
|
|
877
|
+
elif len(args) > 1:
|
|
878
|
+
node = args[1]
|
|
879
|
+
else:
|
|
880
|
+
return None
|
|
881
|
+
return node
|
|
882
|
+
|
|
883
|
+
def source_func(source_name, table_name):
|
|
884
|
+
return f"__{source_name}__{table_name}"
|
|
885
|
+
|
|
886
|
+
raw_code = node.get('raw_code')
|
|
887
|
+
jinja_context = dict(
|
|
888
|
+
ref=ref_func,
|
|
889
|
+
source=source_func,
|
|
890
|
+
)
|
|
891
|
+
|
|
892
|
+
schema = {}
|
|
893
|
+
for parent_id in parent_list:
|
|
894
|
+
parent_node = nodes.get(parent_id)
|
|
895
|
+
if parent_node is None:
|
|
896
|
+
continue
|
|
897
|
+
columns = parent_node.get('columns') or {}
|
|
898
|
+
name = parent_node.get('name')
|
|
899
|
+
if parent_node.get('resource_type') == 'source':
|
|
900
|
+
parts = parent_id.split('.')
|
|
901
|
+
source = parts[2]
|
|
902
|
+
table = parts[3]
|
|
903
|
+
name = f"__{source}__{table}"
|
|
904
|
+
schema[name] = {
|
|
905
|
+
name: column.get('type') for name, column in columns.items()
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
try:
|
|
909
|
+
# provide a manifest to speedup and not pollute the manifest
|
|
910
|
+
compiled_sql = self.generate_sql(raw_code, base=base, context=jinja_context, provided_manifest=manifest)
|
|
911
|
+
dialect = self.adapter.type()
|
|
912
|
+
column_lineage = cll(compiled_sql, schema=schema, dialect=dialect)
|
|
913
|
+
except RecceException:
|
|
914
|
+
# TODO: provide parsing error message if needed
|
|
915
|
+
_apply_all_columns(node, 'unknown', [])
|
|
916
|
+
cll_tracker.increment_sqlglot_error_nodes()
|
|
917
|
+
return
|
|
918
|
+
except Exception:
|
|
919
|
+
_apply_all_columns(node, 'unknown', [])
|
|
920
|
+
cll_tracker.increment_other_error_nodes()
|
|
921
|
+
return
|
|
922
|
+
|
|
923
|
+
_depend_node_to_id(column_lineage, nodes)
|
|
924
|
+
|
|
925
|
+
for name, column in node.get('columns', {}).items():
|
|
926
|
+
if name in column_lineage:
|
|
927
|
+
column['depends_on'] = column_lineage[name].depends_on
|
|
928
|
+
column['transformation_type'] = column_lineage[name].type
|
|
929
|
+
|
|
930
|
+
@lru_cache(maxsize=2)
|
|
931
|
+
def get_lineage_nodes_metadata(self, base: Optional[bool] = False):
|
|
932
|
+
manifest = self.curr_manifest if base is False else self.base_manifest
|
|
933
|
+
catalog = self.curr_catalog if base is False else self.base_catalog
|
|
934
|
+
manifest_dict = manifest.to_dict()
|
|
935
|
+
|
|
936
|
+
nodes = {}
|
|
937
|
+
for node in manifest_dict['nodes'].values():
|
|
938
|
+
unique_id = node['unique_id']
|
|
939
|
+
resource_type = node['resource_type']
|
|
940
|
+
|
|
941
|
+
if resource_type not in ['model', 'seed', 'exposure', 'snapshot']:
|
|
942
|
+
continue
|
|
943
|
+
|
|
944
|
+
nodes[unique_id] = {
|
|
945
|
+
'id': node['unique_id'],
|
|
946
|
+
'name': node['name'],
|
|
947
|
+
'resource_type': node['resource_type'],
|
|
948
|
+
'raw_code': node['raw_code'],
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
if catalog is not None and unique_id in catalog.nodes:
|
|
952
|
+
columns = {}
|
|
953
|
+
for col_name, col_metadata in catalog.nodes[unique_id].columns.items():
|
|
954
|
+
col = dict(name=col_name, type=col_metadata.type)
|
|
955
|
+
columns[col_name] = col
|
|
956
|
+
nodes[unique_id]['columns'] = columns
|
|
957
|
+
|
|
958
|
+
for source in manifest_dict['sources'].values():
|
|
959
|
+
unique_id = source['unique_id']
|
|
960
|
+
|
|
961
|
+
nodes[unique_id] = {
|
|
962
|
+
'id': source['unique_id'],
|
|
963
|
+
'name': source['name'],
|
|
964
|
+
'resource_type': source['resource_type'],
|
|
965
|
+
'columns': {
|
|
966
|
+
col.get('name'): {
|
|
967
|
+
'name': col.get('name'),
|
|
968
|
+
'type': col.get('data_type')
|
|
969
|
+
}
|
|
970
|
+
for col in source.get('columns', {}).values()
|
|
971
|
+
}
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
if catalog is not None and unique_id in catalog.sources:
|
|
975
|
+
columns = {
|
|
976
|
+
col_name: {
|
|
977
|
+
'name': col_name,
|
|
978
|
+
'type': col_metadata.type
|
|
979
|
+
}
|
|
980
|
+
for col_name, col_metadata in catalog.sources[unique_id].columns.items()
|
|
981
|
+
}
|
|
982
|
+
nodes[unique_id]['columns'].update(columns)
|
|
983
|
+
|
|
984
|
+
return nodes
|
|
985
|
+
|
|
905
986
|
def get_manifests_by_id(self, unique_id: str):
|
|
906
987
|
curr_manifest = self.get_manifest(base=False)
|
|
907
988
|
base_manifest = self.get_manifest(base=True)
|
|
@@ -997,8 +1078,11 @@ class DbtAdapter(BaseAdapter):
|
|
|
997
1078
|
if refresh_file_path.endswith('manifest.json'):
|
|
998
1079
|
self.curr_manifest = load_manifest(path=refresh_file_path)
|
|
999
1080
|
self.manifest = as_manifest(self.curr_manifest)
|
|
1081
|
+
self.get_cll_cached.cache_clear()
|
|
1082
|
+
self.get_lineage_nodes_metadata.cache_clear()
|
|
1000
1083
|
elif refresh_file_path.endswith('catalog.json'):
|
|
1001
1084
|
self.curr_catalog = load_catalog(path=refresh_file_path)
|
|
1085
|
+
self.get_lineage_nodes_metadata.cache_clear()
|
|
1002
1086
|
elif self.base_path and target_type == os.path.basename(self.base_path):
|
|
1003
1087
|
if refresh_file_path.endswith('manifest.json'):
|
|
1004
1088
|
self.base_manifest = load_manifest(path=refresh_file_path)
|
recce/data/404.html
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-8a8709fe9d22c323.js"/><script src="/_next/static/chunks/38610ee6-24a85e2dfb424ddb.js" async=""></script><script src="/_next/static/chunks/783-784ce10322400a48.js" async=""></script><script src="/_next/static/chunks/main-app-cfd795a5fe58bf37.js" async=""></script><meta name="robots" content="noindex"/><title>404: This page could not be found.</title><title>recce</title><meta name="description" content="Recce: Data validation toolkit for comprehensive PR review"/><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body><div style="font-family:system-ui,"Segoe UI",Roboto,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji";height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding:0 23px 0 0;font-size:24px;font-weight:500;vertical-align:top;line-height:49px">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:49px;margin:0">This page could not be found.</h2></div></div></div><script src="/_next/static/chunks/webpack-8a8709fe9d22c323.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:I[83798,[],\"\"]\n3:I[26375,[],\"\"]\n4:I[22587,[],\"\"]\na:I[114,[],\"\"]\n5:{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"}\n6:{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"}\n7:{\"display\":\"inline-block\"}\n8:{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0}\nb:[]\n"])</script><script>self.__next_f.push([1,"0:[\"$\",\"$L1\",null,{\"buildId\":\"5XO9Q5RJ7mNkcELBP9S4D\",\"assetPrefix\":\"\",\"urlParts\":[\"\",\"_not-found\"],\"initialTree\":[\"\",{\"children\":[\"/_not-found\",{\"children\":[\"__PAGE__\",{}]}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"/_not-found\",{\"children\":[\"__PAGE__\",{},[[\"$L2\",[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],null],null],null]},[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"/_not-found\",\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\"}]],null]},[[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"suppressHydrationWarning\":true,\"children\":[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":\"$5\",\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":\"$6\",\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":\"$7\",\"children\":[\"$\",\"h2\",null,{\"style\":\"$8\",\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[]}]}]}]],null],null],\"couldBeIntercepted\":false,\"initialHead\":[[\"$\",\"meta\",null,{\"name\":\"robots\",\"content\":\"noindex\"}],\"$L9\"],\"globalErrorComponent\":\"$a\",\"missingSlots\":\"$Wb\"}]\n"])</script><script>self.__next_f.push([1,"9:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"recce\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"Recce: Data validation toolkit for comprehensive PR review\"}]]\n2:null\n"])</script></body></html>
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
self.__BUILD_MANIFEST={__rewrites:{afterFiles:[],beforeFiles:[],fallback:[]},"/_error":["static/chunks/pages/_error-71ed60313c005df1.js"],sortedPages:["/_app","/_error"]},self.__BUILD_MANIFEST_CB&&self.__BUILD_MANIFEST_CB();
|