sqlmesh 0.225.1.dev13__tar.gz → 0.225.1.dev15__tar.gz

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.225.1.dev13
+Version: 0.225.1.dev15
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/sqlmesh/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.225.1.dev13'
-__version_tuple__ = version_tuple = (0, 225, 1, 'dev13')
+__version__ = version = '0.225.1.dev15'
+__version_tuple__ = version_tuple = (0, 225, 1, 'dev15')
 
-__commit_id__ = commit_id = 'g4b210f204'
+__commit_id__ = commit_id = 'g053052079'

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/sqlmesh.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.225.1.dev13
+Version: 0.225.1.dev15
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/tests/conftest.py

@@ -261,9 +261,12 @@ def rescope_lineage_cache(request):
 
 @pytest.fixture(autouse=True)
 def reset_console():
-    from sqlmesh.core.console import set_console, NoopConsole
+    from sqlmesh.core.console import set_console, NoopConsole, get_console
 
+    orig_console = get_console()
     set_console(NoopConsole())
+    yield
+    set_console(orig_console)
 
 
 @pytest.fixture

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/tests/dbt/conftest.py

@@ -127,3 +127,26 @@ def dbt_dummy_postgres_config() -> PostgresConfig:
         port=5432,
         schema="schema",
     )
+
+
+@pytest.fixture(scope="function", autouse=True)
+def reset_dbt_globals():
+    # This fixture is used to clear the memoized cache for _get_package_with_retries
+    # in dbt.clients.registry. This is necessary because the cache is shared across
+    # tests and can cause unexpected behavior if not cleared as some tests depend on
+    # the deprecation warning that _get_package_with_retries fires
+    yield
+    # https://github.com/dbt-labs/dbt-core/blob/main/tests/functional/conftest.py#L9
+    try:
+        from dbt.clients.registry import _get_cached
+
+        _get_cached.cache = {}
+    except Exception:
+        pass
+    # https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/tests/util.py#L82
+    try:
+        from dbt_common.events.functions import reset_metadata_vars
+
+        reset_metadata_vars()
+    except Exception:
+        pass

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/tests/dbt/test_model.py

@@ -6,6 +6,7 @@ from pathlib import Path
 from sqlglot import exp
 from sqlglot.errors import SchemaError
 from sqlmesh import Context
+from sqlmesh.core.console import NoopConsole, get_console
 from sqlmesh.core.model import TimeColumn, IncrementalByTimeRangeKind
 from sqlmesh.core.model.kind import OnDestructiveChange, OnAdditiveChange
 from sqlmesh.core.state_sync.db.snapshot import _snapshot_to_json
@@ -537,6 +538,7 @@ def test_load_deprecated_incremental_time_column(
         f.write(incremental_time_range_contents)
 
     snapshot_fqn = '"local"."main"."incremental_time_range"'
+    assert isinstance(get_console(), NoopConsole)
     context = Context(paths=project_dir)
     model = context.snapshots[snapshot_fqn].model
     # Validate model-level attributes

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/web/common/src/components/Lineage/LineageColumnLevel/FactoryColumn.tsx

@@ -233,7 +233,7 @@ export function FactoryColumn<
             <NodeBadge className="FactoryColumn__NodeBadge">{type}</NodeBadge>
           }
           className={cn(
-            'FactoryColumn__Metadata relative overflow-visible group',
+            'FactoryColumn__Metadata relative overflow-visible',
             isDisabledColumn && 'cursor-not-allowed',
             className,
           )}

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/web/common/src/components/Lineage/LineageColumnLevel/help.ts

@@ -236,9 +236,9 @@ export function calculateColumnsHeight({
   const hasColumns = columnsCount > 0
   const columnHeight = 24 // tailwind h-6
   const columnsTopSeparator = 1
-  const columnSeparator = 1
+  const columnSeparator = 0
   const columnsContainerPadding = 4
-  const columnsPadding = 4
+  const columnsPadding = 0
   const columnsFilterHeight = hasColumnsFilter && hasColumns ? columnHeight : 0
   const columnsSeparators = columnsCount > 1 ? columnsCount - 1 : 0
 

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/web/common/src/components/Lineage/LineageContext.ts

@@ -64,7 +64,6 @@ export interface LineageContextValue<
   >
   currentNodeId: TNodeID | null
   selectedNode: LineageNode<TNodeData, TNodeID> | null
-  currentNode: LineageNode<TNodeData, TNodeID> | null
 }
 
 export function getInitial<
@@ -89,7 +88,6 @@ export function getInitial<
     selectedNodeId: null,
     selectedNode: null,
     currentNodeId: null,
-    currentNode: null,
   }
 }
 

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/web/common/src/components/Lineage/LineageLayout.tsx

@@ -35,14 +35,11 @@ export function LineageLayout<
   edgeTypes,
   className,
   controls,
-  nodesDraggable,
-  nodesConnectable,
   isBuildingLayout,
   useLineage,
   onNodeClick,
   onNodeDoubleClick,
   showControlOnlySelectedNodes,
-  showControlZoomToCurrentNode,
   showControlZoomToSelectedNode,
 }: {
   useLineage: LineageContextHook<
@@ -59,10 +56,7 @@ export function LineageLayout<
   nodeTypes?: NodeTypes
   edgeTypes?: EdgeTypes
   className?: string
-  nodesDraggable?: boolean
-  nodesConnectable?: boolean
   showControlOnlySelectedNodes?: boolean
-  showControlZoomToCurrentNode?: boolean
   showControlZoomToSelectedNode?: boolean
   controls?:
     | React.ReactNode
@@ -85,14 +79,11 @@ export function LineageLayout<
         <LineageLayoutBase
           nodeTypes={nodeTypes}
           edgeTypes={edgeTypes}
-          nodesDraggable={nodesDraggable}
-          nodesConnectable={nodesConnectable}
           controls={controls}
           useLineage={useLineage}
           onNodeClick={onNodeClick}
           onNodeDoubleClick={onNodeDoubleClick}
           showControlOnlySelectedNodes={showControlOnlySelectedNodes}
-          showControlZoomToCurrentNode={showControlZoomToCurrentNode}
           showControlZoomToSelectedNode={showControlZoomToSelectedNode}
         />
       </LineageLayoutContainer>

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/web/common/src/components/Lineage/LineageLayoutBase.tsx

@@ -2,9 +2,7 @@ import {
   Background,
   BackgroundVariant,
   Controls,
-  type EdgeChange,
   type EdgeTypes,
-  type NodeChange,
   type NodeTypes,
   ReactFlow,
   type SetCenter,
@@ -13,14 +11,12 @@ import {
   getOutgoers,
   useReactFlow,
   useViewport,
-  applyNodeChanges,
-  applyEdgeChanges,
 } from '@xyflow/react'
 
 import '@xyflow/react/dist/style.css'
 import './Lineage.css'
 
-import { CircuitBoard, Crosshair, LocateFixed, RotateCcw } from 'lucide-react'
+import { CircuitBoard, LocateFixed, RotateCcw } from 'lucide-react'
 import React from 'react'
 
 import { type LineageContextHook } from './LineageContext'
@@ -60,13 +56,10 @@ export function LineageLayoutBase<
   edgeTypes,
   className,
   controls,
-  nodesDraggable = false,
-  nodesConnectable = false,
   useLineage,
   onNodeClick,
   onNodeDoubleClick,
   showControlOnlySelectedNodes = true,
-  showControlZoomToCurrentNode = true,
   showControlZoomToSelectedNode = true,
 }: {
   useLineage: LineageContextHook<
@@ -79,13 +72,10 @@ export function LineageLayoutBase<
     TSourceHandleID,
     TTargetHandleID
   >
-  nodesDraggable?: boolean
-  nodesConnectable?: boolean
   nodeTypes?: NodeTypes
   edgeTypes?: EdgeTypes
   className?: string
   showControlOnlySelectedNodes?: boolean
-  showControlZoomToCurrentNode?: boolean
   showControlZoomToSelectedNode?: boolean
   controls?:
     | React.ReactNode
@@ -103,74 +93,20 @@ export function LineageLayoutBase<
   const { setCenter } = useReactFlow()
 
   const {
-    currentNode,
     zoom,
-    nodes: initialNodes,
+    nodes,
     edges,
-    setEdges,
     selectedNode,
     showOnlySelectedNodes,
     selectedNodeId,
     setZoom,
-    setSelectedNodeId,
     setShowOnlySelectedNodes,
     setSelectedNodes,
     setSelectedEdges,
   } = useLineage()
 
-  const [nodes, setNodes] =
-    React.useState<LineageNode<TNodeData, TNodeID>[]>(initialNodes)
-
-  const onNodesChange = React.useCallback(
-    (changes: NodeChange<LineageNode<TNodeData, TNodeID>>[]) => {
-      setNodes(applyNodeChanges(changes, nodes))
-    },
-    [nodes],
-  )
-
-  const onEdgesChange = React.useCallback(
-    (
-      changes: EdgeChange<
-        LineageEdge<
-          TEdgeData,
-          TEdgeID,
-          TSourceID,
-          TTargetID,
-          TSourceHandleID,
-          TTargetHandleID
-        >
-      >[],
-    ) => {
-      setEdges(
-        applyEdgeChanges<
-          LineageEdge<
-            TEdgeData,
-            TEdgeID,
-            TSourceID,
-            TTargetID,
-            TSourceHandleID,
-            TTargetHandleID
-          >
-        >(changes, edges),
-      )
-    },
-    [edges],
-  )
-
   const updateZoom = React.useMemo(() => debounce(setZoom, 200), [setZoom])
 
-  const zoomToCurrentNode = React.useCallback(
-    (zoom: number = DEFAULT_ZOOM) => {
-      if (currentNode) {
-        setCenter(currentNode.position.x, currentNode.position.y, {
-          zoom,
-          duration: 0,
-        })
-      }
-    },
-    [currentNode?.position.x, currentNode?.position.y],
-  )
-
   const zoomToSelectedNode = React.useCallback(
     (zoom: number = DEFAULT_ZOOM) => {
       if (selectedNode) {
@@ -195,13 +131,13 @@ export function LineageLayoutBase<
       return Array.from(
         new Set<LineageNode<TNodeData, TNodeID>>([
           node,
-          ...getIncomers(node, initialNodes, edges)
+          ...getIncomers(node, nodes, edges)
             .map(n => getAllIncomers(n, visited))
             .flat(),
         ]),
       )
     },
-    [initialNodes, edges],
+    [nodes, edges],
   )
 
   const getAllOutgoers = React.useCallback(
@@ -216,13 +152,13 @@ export function LineageLayoutBase<
       return Array.from(
         new Set<LineageNode<TNodeData, TNodeID>>([
           node,
-          ...getOutgoers(node, initialNodes, edges)
+          ...getOutgoers(node, nodes, edges)
             .map(n => getAllOutgoers(n, visited))
             .flat(),
         ]),
       )
     },
-    [initialNodes, edges],
+    [nodes, edges],
   )
 
   const connectedNodes = React.useMemo(() => {
@@ -233,12 +169,8 @@ export function LineageLayoutBase<
       ...getAllOutgoers(selectedNode),
     ]
 
-    if (currentNode) {
-      all.push(currentNode)
-    }
-
     return all
-  }, [selectedNode, currentNode, getAllIncomers, getAllOutgoers])
+  }, [selectedNode, getAllIncomers, getAllOutgoers])
 
   const connectedEdges = React.useMemo(() => {
     return getConnectedEdges<
@@ -254,19 +186,11 @@ export function LineageLayoutBase<
     >(connectedNodes, edges)
   }, [connectedNodes, edges])
 
-  React.useEffect(() => {
-    setNodes(initialNodes)
-  }, [initialNodes])
-
   React.useEffect(() => {
     if (selectedNodeId == null) {
       setShowOnlySelectedNodes(false)
       setSelectedNodes(new Set())
       setSelectedEdges(new Set())
-    } else {
-      if (selectedNode == null) {
-        setSelectedNodeId(null)
-      }
     }
   }, [selectedNodeId, selectedNode])
 
@@ -317,17 +241,12 @@ export function LineageLayoutBase<
       edges={edges}
       nodeTypes={nodeTypes}
       edgeTypes={edgeTypes}
-      onNodesChange={onNodesChange}
-      onEdgesChange={onEdgesChange}
-      nodesDraggable={nodesDraggable}
-      nodesConnectable={nodesConnectable}
       zoomOnDoubleClick={false}
       panOnScroll={true}
       zoomOnScroll={true}
       minZoom={nodes.length > NODES_TRESHOLD ? NODES_TRESHOLD_ZOOM : MIN_ZOOM}
       maxZoom={MAX_ZOOM}
       fitView={false}
-      nodeOrigin={[0.5, 0.5]}
       onlyRenderVisibleElements
       onNodeClick={onNodeClick}
       onNodeDoubleClick={onNodeDoubleClick}
@@ -346,14 +265,6 @@ export function LineageLayoutBase<
         position="top-right"
         className="m-1 border-2 border-lineage-control-border rounded-sm overflow-hidden"
       >
-        {currentNode && showControlZoomToCurrentNode && (
-          <LineageControlButton
-            text="Zoom to current node"
-            onClick={() => zoomToCurrentNode(DEFAULT_ZOOM)}
-          >
-            <LineageControlIcon Icon={Crosshair} />
-          </LineageControlButton>
-        )}
         {selectedNodeId && (
           <>
             {showControlOnlySelectedNodes && (

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/web/common/src/components/Lineage/help.ts

@@ -41,17 +41,17 @@ export function getTransformedNodes<
   adjacencyListKeys: TAdjacencyListKey[],
   lineageDetails: LineageDetails<TAdjacencyListKey, TDetailsNode>,
   transformNode: TransformNodeFn<TDetailsNode, TNodeData, TNodeID>,
+  allNodesMap?: LineageNodesMap<TNodeData, TNodeID>,
 ): LineageNodesMap<TNodeData, TNodeID> {
   const nodesCount = adjacencyListKeys.length
   const nodesMap: LineageNodesMap<TNodeData, TNodeID> = Object.create(null)
 
   for (let i = 0; i < nodesCount; i++) {
     const adjacencyListKey = adjacencyListKeys[i]
-    const encodedNodeId = toNodeID<TNodeID>(adjacencyListKey)
-    nodesMap[encodedNodeId] = transformNode(
-      encodedNodeId,
-      lineageDetails[adjacencyListKey],
-    )
+    const nodeId = toNodeID<TNodeID>(adjacencyListKey)
+    nodesMap[nodeId] =
+      allNodesMap?.[nodeId] ||
+      transformNode(nodeId, lineageDetails[adjacencyListKey])
   }
 
   return nodesMap

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/web/common/src/components/Lineage/layout/dagreLayout.ts

@@ -21,6 +21,7 @@ export function buildLayout<
 >({
   edges,
   nodesMap,
+  shouldReuseExistingPosition = true,
 }: {
   edges: LineageEdge<
     TEdgeData,
@@ -31,6 +32,7 @@ export function buildLayout<
     TTargetHandleID
   >[]
   nodesMap: LineageNodesMap<TNodeData>
+  shouldReuseExistingPosition?: boolean
 }) {
   const nodes = Object.values(nodesMap)
   const nodeCount = nodes.length
@@ -46,7 +48,7 @@ export function buildLayout<
 
   g.setGraph({
     rankdir: 'LR',
-    nodesep: 0,
+    nodesep: 12,
     ranksep: 48,
     edgesep: 0,
     ranker: 'longest-path',
@@ -78,12 +80,19 @@ export function buildLayout<
     const nodeWithPosition = g.node(nodeId)
     const halfWidth = width / 2
     const halfHeight = height / 2
+    const isDefaultPosition = node.position.x === 0 && node.position.y === 0
 
     nodesMap[nodeId] = {
       ...node,
       position: {
-        x: nodeWithPosition.x - halfWidth,
-        y: nodeWithPosition.y - halfHeight,
+        x:
+          shouldReuseExistingPosition && isDefaultPosition
+            ? nodeWithPosition.x - halfWidth
+            : node.position.x,
+        y:
+          shouldReuseExistingPosition && isDefaultPosition
+            ? nodeWithPosition.y - halfHeight
+            : node.position.y,
       },
     }
   }

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/web/common/src/components/Lineage/node/useNodeMetadata.tsx

@@ -4,17 +4,14 @@ import {
   useNodeConnections,
 } from '@xyflow/react'
 
-import { type LineageNode, type LineageNodeData, type NodeId } from '../utils'
+import { type LineageNodeData, type NodeId } from '../utils'
 
 export type NodeProps<TNodeData extends LineageNodeData = LineageNodeData> =
   ReactFlowNodeProps<Node<TNodeData>>
 
-export function useNodeMetadata<
-  TNodeData extends LineageNodeData = LineageNodeData,
-  TNodeID extends string = NodeId,
->(
+export function useNodeMetadata<TNodeID extends string = NodeId>(
   nodeId: TNodeID,
-  currentNode: LineageNode<TNodeData, TNodeID> | null,
+  currentNodeId: TNodeID | null,
   selectedNodeId: TNodeID | null,
   selectedNodes: Set<TNodeID>,
 ) {
@@ -29,7 +26,7 @@ export function useNodeMetadata<
 
   const leftId = targets.length > 0 ? nodeId : undefined
   const rightId = sources.length > 0 ? nodeId : undefined
-  const isCurrent = currentNode?.id === nodeId
+  const isCurrent = currentNodeId === nodeId
   const isSelected = selectedNodeId === nodeId
   const isActive = selectedNodes.has(nodeId)
 

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/web/common/src/components/Lineage/stories/Lineage.stories.tsx

@@ -1,5 +1,3 @@
-import type { LineageAdjacencyList, LineageDetails } from '../utils'
-
 import { ModelLineage } from './ModelLineage'
 import type {
   BrandedLineageAdjacencyList,

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/web/common/src/components/Lineage/stories/ModelLineage.tsx

@@ -1,4 +1,4 @@
-import { Focus, LockOpen, Rows2, Rows3, Lock } from 'lucide-react'
+import { Focus, Rows2, Rows3 } from 'lucide-react'
 import React from 'react'
 
 import {
@@ -82,7 +82,6 @@ export const ModelLineage = ({
 
   const [zoom, setZoom] = React.useState(ZOOM_THRESHOLD)
   const [isBuildingLayout, setIsBuildingLayout] = React.useState(false)
-  const [nodesDraggable, setNodesDraggable] = React.useState(false)
   const [edges, setEdges] = React.useState<
     LineageEdge<
       EdgeData,
@@ -311,6 +310,7 @@ export const ModelLineage = ({
         })
         .catch(error => {
           console.error('Layout processing failed:', error)
+
           setEdges([])
           setNodesMap({})
         })
@@ -324,7 +324,6 @@ export const ModelLineage = ({
     return Object.values(nodesMap)
   }, [nodesMap])
 
-  const currentNode = currentNodeId ? nodesMap[currentNodeId] : null
   const selectedNode = selectedNodeId ? nodesMap[selectedNodeId] : null
 
   const handleReset = React.useCallback(() => {
@@ -384,7 +383,6 @@ export const ModelLineage = ({
         edges,
         nodes,
         nodesMap,
-        currentNode,
         setFetchingColumns,
         setColumnLevelLineage,
         setShowColumns,
@@ -412,7 +410,6 @@ export const ModelLineage = ({
         nodeTypes={nodeTypes}
         edgeTypes={edgeTypes}
         className={className}
-        nodesDraggable={nodesDraggable}
         controls={
           <>
             <LineageControlButton
@@ -433,13 +430,6 @@ export const ModelLineage = ({
             >
               <LineageControlIcon Icon={Focus} />
             </LineageControlButton>
-            <LineageControlButton
-              text={nodesDraggable ? 'Lock nodes' : 'Unlock nodes'}
-              onClick={() => setNodesDraggable(prev => !prev)}
-              disabled={isBuildingLayout}
-            >
-              <LineageControlIcon Icon={nodesDraggable ? Lock : LockOpen} />
-            </LineageControlButton>
           </>
         }
       />

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/web/common/src/components/Lineage/stories/ModelNode.tsx

@@ -51,7 +51,7 @@ export const ModelNode = React.memo(function ModelNode({
   const {
     selectedColumns,
     zoom,
-    currentNode,
+    currentNodeId,
     selectedNodeId,
     selectedNodes,
     showColumns,
@@ -70,7 +70,7 @@ export const ModelNode = React.memo(function ModelNode({
     isCurrent,
     isSelected, // if selected from inside the lineage and node is selcted
     isActive, // if selected from inside the lineage and node is not selected but in path
-  } = useNodeMetadata(nodeId, currentNode, selectedNodeId, selectedNodes)
+  } = useNodeMetadata(nodeId, currentNodeId, selectedNodeId, selectedNodes)
 
   const {
     columns,
@@ -132,7 +132,7 @@ export const ModelNode = React.memo(function ModelNode({
     <NodeContainer
       className={cn(
         'hover:opacity-100 group',
-        selectedNodeId == null || isActive || isSelected
+        isActive || isSelected || hasSelectedColumns
           ? 'opacity-100'
           : 'opacity-10',
       )}

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/web/common/src/components/Lineage/utils.ts

@@ -99,7 +99,7 @@ export const DEFAULT_NODE_WIDTH = 300
 export const DEFAULT_ZOOM = 0.85
 export const MIN_ZOOM = 0.01
 export const MAX_ZOOM = 1.75
-export const ZOOM_THRESHOLD = 0.75
+export const ZOOM_THRESHOLD = 0.5
 export const NODES_TRESHOLD = 200
 export const NODES_TRESHOLD_ZOOM = 0.1
 

{sqlmesh-0.225.1.dev13 → sqlmesh-0.225.1.dev15}/web/common/src/components/VirtualList/VirtualList.tsx

@@ -106,7 +106,7 @@ export function VirtualList<TItem>({
       )}
       <ScrollContainer
         ref={scrollableAreaRef}
-        className="h-auto"
+        className="h-auto overflow-auto"
       >
         <div
           style={{