dvt-core 1.11.0b4__py3-none-any.whl

dvt/contracts/graph/manifest.py

@@ -0,0 +1,1904 @@
+import enum
+from collections import defaultdict
+from dataclasses import dataclass, field, replace
+from itertools import chain
+from multiprocessing.synchronize import Lock
+from typing import (
+    Any,
+    Callable,
+    ClassVar,
+    DefaultDict,
+    Dict,
+    Generic,
+    List,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Set,
+    Tuple,
+    TypeVar,
+    Union,
+)
+
+from dvt import deprecations, tracking
+
+# to preserve import paths
+from dvt.artifacts.resources import (
+    BaseResource,
+    DeferRelation,
+    NodeConfig,
+    NodeVersion,
+    RefArgs,
+)
+from dvt.artifacts.schemas.manifest import ManifestMetadata, UniqueID, WritableManifest
+from dvt.clients.jinja_static import statically_parse_ref_or_source
+from dvt.contracts.files import (
+    AnySourceFile,
+    FileHash,
+    FixtureSourceFile,
+    SchemaSourceFile,
+    SourceFile,
+)
+from dvt.contracts.graph.nodes import (
+    RESOURCE_CLASS_TO_NODE_CLASS,
+    BaseNode,
+    Documentation,
+    Exposure,
+    FunctionNode,
+    GenericTestNode,
+    GraphMemberNode,
+    Group,
+    Macro,
+    ManifestNode,
+    Metric,
+    ModelNode,
+    SavedQuery,
+    SeedNode,
+    SemanticModel,
+    SingularTestNode,
+    SnapshotNode,
+    SourceDefinition,
+    UnitTestDefinition,
+    UnitTestFileFixture,
+    UnpatchedSourceDefinition,
+)
+from dvt.contracts.graph.unparsed import SourcePatch, UnparsedVersion
+from dvt.contracts.util import SourceKey
+from dvt.events.types import ArtifactWritten, UnpinnedRefNewVersionAvailable
+from dvt.exceptions import (
+    AmbiguousResourceNameRefError,
+    CompilationError,
+    DuplicateResourceNameError,
+)
+from dvt.flags import get_flags
+from dvt.mp_context import get_mp_context
+from dvt.node_types import (
+    REFABLE_NODE_TYPES,
+    VERSIONED_NODE_TYPES,
+    AccessType,
+    NodeType,
+)
+from typing_extensions import Protocol
+
+import dbt_common.exceptions
+import dbt_common.utils
+from dbt.adapters.exceptions import (
+    DuplicateMacroInPackageError,
+    DuplicateMaterializationNameError,
+)
+from dbt.adapters.factory import get_adapter_package_names
+from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.events.contextvars import get_node_info
+from dbt_common.events.functions import fire_event
+from dbt_common.helper_types import PathSet
+
+PackageName = str
+DocName = str
+RefName = str
+
+
+def find_unique_id_for_package(storage, key, package: Optional[PackageName]) -> Optional[UniqueID]:
+    if key not in storage:
+        return None
+
+    pkg_dct: Mapping[PackageName, UniqueID] = storage[key]
+
+    if package is None:
+        if not pkg_dct:
+            return None
+        else:
+            return next(iter(pkg_dct.values()))
+    elif package in pkg_dct:
+        return pkg_dct[package]
+    else:
+        return None
+
+
+class DocLookup(dbtClassMixin):
+    def __init__(self, manifest: "Manifest") -> None:
+        self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
+        self.populate(manifest)
+
+    def get_unique_id(self, key, package: Optional[PackageName]):
+        return find_unique_id_for_package(self.storage, key, package)
+
+    def find(self, key, package: Optional[PackageName], manifest: "Manifest"):
+        unique_id = self.get_unique_id(key, package)
+        if unique_id is not None:
+            return self.perform_lookup(unique_id, manifest)
+        return None
+
+    def add_doc(self, doc: Documentation):
+        if doc.name not in self.storage:
+            self.storage[doc.name] = {}
+        self.storage[doc.name][doc.package_name] = doc.unique_id
+
+    def populate(self, manifest):
+        for doc in manifest.docs.values():
+            self.add_doc(doc)
+
+    def perform_lookup(self, unique_id: UniqueID, manifest) -> Documentation:
+        if unique_id not in manifest.docs:
+            raise dbt_common.exceptions.DbtInternalError(
+                f"Doc {unique_id} found in cache but not found in manifest"
+            )
+        return manifest.docs[unique_id]
+
+
+class SourceLookup(dbtClassMixin):
+    def __init__(self, manifest: "Manifest") -> None:
+        self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
+        self.populate(manifest)
+
+    def get_unique_id(self, search_name, package: Optional[PackageName]):
+        return find_unique_id_for_package(self.storage, search_name, package)
+
+    def find(self, search_name, package: Optional[PackageName], manifest: "Manifest"):
+        unique_id = self.get_unique_id(search_name, package)
+        if unique_id is not None:
+            return self.perform_lookup(unique_id, manifest)
+        return None
+
+    def add_source(self, source: SourceDefinition):
+        if source.search_name not in self.storage:
+            self.storage[source.search_name] = {}
+
+        self.storage[source.search_name][source.package_name] = source.unique_id
+
+    def populate(self, manifest):
+        for source in manifest.sources.values():
+            if hasattr(source, "source_name"):
+                self.add_source(source)
+
+    def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SourceDefinition:
+        if unique_id not in manifest.sources:
+            raise dbt_common.exceptions.DbtInternalError(
+                f"Source {unique_id} found in cache but not found in manifest"
+            )
+        return manifest.sources[unique_id]
+
+
+class FunctionLookup(dbtClassMixin):
+    def __init__(self, manifest: "Manifest") -> None:
+        self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
+        self.populate(manifest)
+
+    def get_unique_id(self, search_name, package: Optional[PackageName]):
+        return find_unique_id_for_package(self.storage, search_name, package)
+
+    def find(self, search_name, package: Optional[PackageName], manifest: "Manifest"):
+        unique_id = self.get_unique_id(search_name, package)
+        if unique_id is not None:
+            return self.perform_lookup(unique_id, manifest)
+        return None
+
+    def add_function(self, function: FunctionNode):
+        if function.search_name not in self.storage:
+            self.storage[function.search_name] = {}
+
+        self.storage[function.search_name][function.package_name] = function.unique_id
+
+    def populate(self, manifest):
+        for function in manifest.functions.values():
+            if hasattr(function, "name"):
+                self.add_function(function)
+
+    def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> FunctionNode:
+        if unique_id not in manifest.functions:
+            raise dbt_common.exceptions.DbtInternalError(
+                f"Function {unique_id} found in cache but not found in manifest"
+            )
+        return manifest.functions[unique_id]
+
+
+class RefableLookup(dbtClassMixin):
+    # model, seed, snapshot, function
+    _lookup_types: ClassVar[set] = set(REFABLE_NODE_TYPES)
+    _versioned_types: ClassVar[set] = set(VERSIONED_NODE_TYPES)
+
+    def __init__(self, manifest: "Manifest") -> None:
+        self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
+        self.populate(manifest)
+
+    def get_unique_id(
+        self,
+        key: str,
+        package: Optional[PackageName],
+        version: Optional[NodeVersion],
+        node: Optional[GraphMemberNode] = None,
+    ):
+        if version:
+            key = f"{key}.v{version}"
+
+        unique_ids = self._find_unique_ids_for_package(key, package)
+        if len(unique_ids) > 1:
+            raise AmbiguousResourceNameRefError(key, unique_ids, node)
+        else:
+            return unique_ids[0] if unique_ids else None
+
+    def find(
+        self,
+        key: str,
+        package: Optional[PackageName],
+        version: Optional[NodeVersion],
+        manifest: "Manifest",
+        source_node: Optional[GraphMemberNode] = None,
+    ):
+        unique_id = self.get_unique_id(key, package, version, source_node)
+        if unique_id is not None:
+            node = self.perform_lookup(unique_id, manifest)
+            # If this is an unpinned ref (no 'version' arg was passed),
+            # AND this is a versioned node,
+            # AND this ref is being resolved at runtime -- get_node_info != {}
+            # Only ModelNodes can be versioned.
+            if (
+                isinstance(node, ModelNode)
+                and version is None
+                and node.is_versioned
+                and get_node_info()
+            ):
+                # Check to see if newer versions are available, and log an "FYI" if so
+                max_version: UnparsedVersion = max(
+                    [
+                        UnparsedVersion(v.version)
+                        for v in manifest.nodes.values()
+                        if isinstance(v, ModelNode)
+                        and v.name == node.name
+                        and v.version is not None
+                    ]
+                )
+                assert node.latest_version is not None  # for mypy, whenever i may find it
+                if max_version > UnparsedVersion(node.latest_version):
+                    fire_event(
+                        UnpinnedRefNewVersionAvailable(
+                            node_info=get_node_info(),
+                            ref_node_name=node.name,
+                            ref_node_package=node.package_name,
+                            ref_node_version=str(node.version),
+                            ref_max_version=str(max_version.v),
+                        )
+                    )
+
+            return node
+        return None
+
+    def add_node(self, node: ManifestNode):
+        if node.resource_type in self._lookup_types:
+            if node.name not in self.storage:
+                self.storage[node.name] = {}
+
+            if node.is_versioned:
+                if node.search_name not in self.storage:
+                    self.storage[node.search_name] = {}
+                self.storage[node.search_name][node.package_name] = node.unique_id
+                if node.is_latest_version:  # type: ignore
+                    self.storage[node.name][node.package_name] = node.unique_id
+            else:
+                self.storage[node.name][node.package_name] = node.unique_id
+
+    def populate(self, manifest):
+        for node in manifest.nodes.values():
+            self.add_node(node)
+
+    def perform_lookup(self, unique_id: UniqueID, manifest) -> ManifestNode:
+        if unique_id in manifest.nodes:
+            node = manifest.nodes[unique_id]
+        else:
+            raise dbt_common.exceptions.DbtInternalError(
+                f"Node {unique_id} found in cache but not found in manifest"
+            )
+        return node
+
+    def _find_unique_ids_for_package(self, key, package: Optional[PackageName]) -> List[str]:
+        if key not in self.storage:
+            return []
+
+        pkg_dct: Mapping[PackageName, UniqueID] = self.storage[key]
+
+        if package is None:
+            if not pkg_dct:
+                return []
+            else:
+                return list(pkg_dct.values())
+        elif package in pkg_dct:
+            return [pkg_dct[package]]
+        else:
+            return []
+
+
+class MetricLookup(dbtClassMixin):
+    def __init__(self, manifest: "Manifest") -> None:
+        self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
+        self.populate(manifest)
+
+    def get_unique_id(self, search_name, package: Optional[PackageName]):
+        return find_unique_id_for_package(self.storage, search_name, package)
+
+    def find(self, search_name, package: Optional[PackageName], manifest: "Manifest"):
+        unique_id = self.get_unique_id(search_name, package)
+        if unique_id is not None:
+            return self.perform_lookup(unique_id, manifest)
+        return None
+
+    def add_metric(self, metric: Metric):
+        if metric.search_name not in self.storage:
+            self.storage[metric.search_name] = {}
+
+        self.storage[metric.search_name][metric.package_name] = metric.unique_id
+
+    def populate(self, manifest):
+        for metric in manifest.metrics.values():
+            if hasattr(metric, "name"):
+                self.add_metric(metric)
+
+    def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> Metric:
+        if unique_id not in manifest.metrics:
+            raise dbt_common.exceptions.DbtInternalError(
+                f"Metric {unique_id} found in cache but not found in manifest"
+            )
+        return manifest.metrics[unique_id]
+
+
+class SavedQueryLookup(dbtClassMixin):
+    """Lookup utility for finding SavedQuery nodes"""
+
+    def __init__(self, manifest: "Manifest") -> None:
+        self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
+        self.populate(manifest)
+
+    def get_unique_id(self, search_name, package: Optional[PackageName]):
+        return find_unique_id_for_package(self.storage, search_name, package)
+
+    def find(self, search_name, package: Optional[PackageName], manifest: "Manifest"):
+        unique_id = self.get_unique_id(search_name, package)
+        if unique_id is not None:
+            return self.perform_lookup(unique_id, manifest)
+        return None
+
+    def add_saved_query(self, saved_query: SavedQuery):
+        if saved_query.search_name not in self.storage:
+            self.storage[saved_query.search_name] = {}
+
+        self.storage[saved_query.search_name][saved_query.package_name] = saved_query.unique_id
+
+    def populate(self, manifest):
+        for saved_query in manifest.saved_queries.values():
+            if hasattr(saved_query, "name"):
+                self.add_saved_query(saved_query)
+
+    def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SavedQuery:
+        if unique_id not in manifest.saved_queries:
+            raise dbt_common.exceptions.DbtInternalError(
+                f"SavedQUery {unique_id} found in cache but not found in manifest"
+            )
+        return manifest.saved_queries[unique_id]
+
+
+class SemanticModelByMeasureLookup(dbtClassMixin):
+    """Lookup utility for finding SemanticModel by measure
+
+    This is possible because measure names are supposed to be unique across
+    the semantic models in a manifest.
+    """
+
+    def __init__(self, manifest: "Manifest") -> None:
+        self.storage: DefaultDict[str, Dict[PackageName, UniqueID]] = defaultdict(dict)
+        self.populate(manifest)
+
+    def get_unique_id(self, search_name: str, package: Optional[PackageName]):
+        return find_unique_id_for_package(self.storage, search_name, package)
+
+    def find(
+        self, search_name: str, package: Optional[PackageName], manifest: "Manifest"
+    ) -> Optional[SemanticModel]:
+        """Tries to find a SemanticModel based on a measure name"""
+        unique_id = self.get_unique_id(search_name, package)
+        if unique_id is not None:
+            return self.perform_lookup(unique_id, manifest)
+        return None
+
+    def add(self, semantic_model: SemanticModel):
+        """Sets all measures for a SemanticModel as paths to the SemanticModel's `unique_id`"""
+        for measure in semantic_model.measures:
+            self.storage[measure.name][semantic_model.package_name] = semantic_model.unique_id
+
+    def populate(self, manifest: "Manifest"):
+        """Populate storage with all the measure + package paths to the Manifest's SemanticModels"""
+        for semantic_model in manifest.semantic_models.values():
+            self.add(semantic_model=semantic_model)
+        for disabled in manifest.disabled.values():
+            for node in disabled:
+                if isinstance(node, SemanticModel):
+                    self.add(semantic_model=node)
+
+    def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SemanticModel:
+        """Tries to get a SemanticModel from the Manifest"""
+        enabled_semantic_model: Optional[SemanticModel] = manifest.semantic_models.get(unique_id)
+        disabled_semantic_model: Optional[List] = manifest.disabled.get(unique_id)
+
+        if isinstance(enabled_semantic_model, SemanticModel):
+            return enabled_semantic_model
+        elif disabled_semantic_model is not None and isinstance(
+            disabled_semantic_model[0], SemanticModel
+        ):
+            return disabled_semantic_model[0]
+        else:
+            raise dbt_common.exceptions.DbtInternalError(
+                f"Semantic model `{unique_id}` found in cache but not found in manifest"
+            )
+
+
+# This handles both models/seeds/snapshots and sources/metrics/exposures/semantic_models
+class DisabledLookup(dbtClassMixin):
+    def __init__(self, manifest: "Manifest") -> None:
+        self.storage: Dict[str, Dict[PackageName, List[Any]]] = {}
+        self.populate(manifest)
+
+    def populate(self, manifest: "Manifest"):
+        for node in list(chain.from_iterable(manifest.disabled.values())):
+            self.add_node(node)
+
+    def add_node(self, node: GraphMemberNode) -> None:
+        if node.search_name not in self.storage:
+            self.storage[node.search_name] = {}
+        if node.package_name not in self.storage[node.search_name]:
+            self.storage[node.search_name][node.package_name] = []
+        self.storage[node.search_name][node.package_name].append(node)
+
+    # This should return a list of disabled nodes. It's different from
+    # the other Lookup functions in that it returns full nodes, not just unique_ids
+    def find(
+        self,
+        search_name,
+        package: Optional[PackageName],
+        version: Optional[NodeVersion] = None,
+        resource_types: Optional[List[NodeType]] = None,
+    ) -> Optional[List[Any]]:
+        if version:
+            search_name = f"{search_name}.v{version}"
+
+        if search_name not in self.storage:
+            return None
+
+        pkg_dct: Mapping[PackageName, List[Any]] = self.storage[search_name]
+
+        nodes = []
+        if package is None:
+            if not pkg_dct:
+                return None
+            else:
+                nodes = next(iter(pkg_dct.values()))
+        elif package in pkg_dct:
+            nodes = pkg_dct[package]
+        else:
+            return None
+
+        if resource_types is None:
+            return nodes
+        else:
+            new_nodes = []
+            for node in nodes:
+                if node.resource_type in resource_types:
+                    new_nodes.append(node)
+            if not new_nodes:
+                return None
+            else:
+                return new_nodes
+
+
+class AnalysisLookup(RefableLookup):
+    _lookup_types: ClassVar[set] = set([NodeType.Analysis])
+    _versioned_types: ClassVar[set] = set()
+
+
+class SingularTestLookup(dbtClassMixin):
+    def __init__(self, manifest: "Manifest") -> None:
+        self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
+        self.populate(manifest)
+
+    def get_unique_id(self, search_name, package: Optional[PackageName]) -> Optional[UniqueID]:
+        return find_unique_id_for_package(self.storage, search_name, package)
+
+    def find(
+        self, search_name, package: Optional[PackageName], manifest: "Manifest"
+    ) -> Optional[SingularTestNode]:
+        unique_id = self.get_unique_id(search_name, package)
+        if unique_id is not None:
+            return self.perform_lookup(unique_id, manifest)
+        return None
+
+    def add_singular_test(self, source: SingularTestNode) -> None:
+        if source.search_name not in self.storage:
+            self.storage[source.search_name] = {}
+
+        self.storage[source.search_name][source.package_name] = source.unique_id
+
+    def populate(self, manifest: "Manifest") -> None:
+        for node in manifest.nodes.values():
+            if isinstance(node, SingularTestNode):
+                self.add_singular_test(node)
+
+    def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SingularTestNode:
+        if unique_id not in manifest.nodes:
+            raise dbt_common.exceptions.DbtInternalError(
+                f"Singular test {unique_id} found in cache but not found in manifest"
+            )
+        node = manifest.nodes[unique_id]
+        assert isinstance(node, SingularTestNode)
+        return node
+
+
+def _packages_to_search(
+    current_project: str,
+    node_package: str,
+    target_package: Optional[str] = None,
+) -> List[Optional[str]]:
+    if target_package is not None:
+        return [target_package]
+    elif current_project == node_package:
+        return [current_project, None]
+    else:
+        return [current_project, node_package, None]
+
+
+def _sort_values(dct):
+    """Given a dictionary, sort each value. This makes output deterministic,
+    which helps for tests.
+    """
+    return {k: sorted(v) for k, v in dct.items()}
+
+
+def build_node_edges(nodes: List[ManifestNode]):
+    """Build the forward and backward edges on the given list of ManifestNodes
+    and return them as two separate dictionaries, each mapping unique IDs to
+    lists of edges.
+    """
+    backward_edges: Dict[str, List[str]] = {}
+    # pre-populate the forward edge dict for simplicity
+    forward_edges: Dict[str, List[str]] = {n.unique_id: [] for n in nodes}
+    for node in nodes:
+        backward_edges[node.unique_id] = node.depends_on_nodes[:]
+        for unique_id in backward_edges[node.unique_id]:
+            if unique_id in forward_edges.keys():
+                forward_edges[unique_id].append(node.unique_id)
+    return _sort_values(forward_edges), _sort_values(backward_edges)
+
+
+# Build a map of children of macros and generic tests
+def build_macro_edges(nodes: List[Any]):
+    forward_edges: Dict[str, List[str]] = {
+        n.unique_id: [] for n in nodes if n.unique_id.startswith("macro") or n.depends_on_macros
+    }
+    for node in nodes:
+        for unique_id in node.depends_on_macros:
+            if unique_id in forward_edges.keys():
+                forward_edges[unique_id].append(node.unique_id)
+    return _sort_values(forward_edges)
+
+
+def _deepcopy(value):
+    return value.from_dict(value.to_dict(omit_none=True))
+
+
+class Locality(enum.IntEnum):
+    Core = 1
+    Imported = 2
+    Root = 3
+
+
+@dataclass
+class MacroCandidate:
+    locality: Locality
+    macro: Macro
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, MacroCandidate):
+            return NotImplemented
+        return self.locality == other.locality
+
+    def __lt__(self, other: object) -> bool:
+        if not isinstance(other, MacroCandidate):
+            return NotImplemented
+        if self.locality < other.locality:
+            return True
+        if self.locality > other.locality:
+            return False
+        return False
+
+
+@dataclass
+class MaterializationCandidate(MacroCandidate):
+    # specificity describes where in the inheritance chain this materialization candidate is
+    # a specificity of 0 means a materialization defined by the current adapter
+    # the highest the specificity describes a default materialization. the value itself depends on
+    # how many adapters there are in the inheritance chain
+    specificity: int
+
+    @classmethod
+    def from_macro(cls, candidate: MacroCandidate, specificity: int) -> "MaterializationCandidate":
+        return cls(
+            locality=candidate.locality,
+            macro=candidate.macro,
+            specificity=specificity,
+        )
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, MaterializationCandidate):
+            return NotImplemented
+        equal = self.specificity == other.specificity and self.locality == other.locality
+        if equal:
+            raise DuplicateMaterializationNameError(self.macro, other)
+
+        return equal
+
+    def __lt__(self, other: object) -> bool:
+        if not isinstance(other, MaterializationCandidate):
+            return NotImplemented
+        if self.specificity > other.specificity:
+            return True
+        if self.specificity < other.specificity:
+            return False
+        if self.locality < other.locality:
+            return True
+        if self.locality > other.locality:
+            return False
+        return False
+
+
+M = TypeVar("M", bound=MacroCandidate)
+
+
+class CandidateList(List[M]):
+    def last_candidate(
+        self, valid_localities: Optional[List[Locality]] = None
+    ) -> Optional[MacroCandidate]:
+        """
+        Obtain the last (highest precedence) MacroCandidate from the CandidateList of any locality in valid_localities.
+        If valid_localities is not specified, return the last MacroCandidate of any locality.
+        """
+        if not self:
+            return None
+        self.sort()
+
+        if valid_localities is None:
+            return self[-1]
+
+        for candidate in reversed(self):
+            if candidate.locality in valid_localities:
+                return candidate
+
+        return None
+
+    def last(self) -> Optional[Macro]:
+        last_candidate = self.last_candidate()
+        return last_candidate.macro if last_candidate is not None else None
+
+
+def _get_locality(macro: Macro, root_project_name: str, internal_packages: Set[str]) -> Locality:
+    if macro.package_name == root_project_name:
+        return Locality.Root
+    elif macro.package_name in internal_packages:
+        return Locality.Core
+    else:
+        return Locality.Imported
+
+
+class Searchable(Protocol):
+    resource_type: NodeType
+    package_name: str
+
+    @property
+    def search_name(self) -> str:
+        raise NotImplementedError("search_name not implemented")
+
+
+D = TypeVar("D")
+
+
+@dataclass
+class Disabled(Generic[D]):
+    target: D
+
+
+MaybeFunctionNode = Optional[Union[FunctionNode, Disabled[FunctionNode]]]
+
+
+MaybeMetricNode = Optional[Union[Metric, Disabled[Metric]]]
+
+
+MaybeSavedQueryNode = Optional[Union[SavedQuery, Disabled[SavedQuery]]]
+
+
+MaybeDocumentation = Optional[Documentation]
+
+
+MaybeParsedSource = Optional[
+    Union[
+        SourceDefinition,
+        Disabled[SourceDefinition],
+    ]
+]
+
+
+MaybeNonSource = Optional[Union[ManifestNode, Disabled[ManifestNode]]]
+
+
+T = TypeVar("T", bound=GraphMemberNode)
+
+
+# This contains macro methods that are in both the Manifest
+# and the MacroManifest
+class MacroMethods:
+    # Just to make mypy happy. There must be a better way.
+    def __init__(self):
+        self.macros = []
+        self.metadata = {}
+        self._macros_by_name = {}
+        self._macros_by_package = {}
+
+    def find_macro_candidate_by_name(
+        self, name: str, root_project_name: str, package: Optional[str]
+    ) -> Optional[MacroCandidate]:
+        """Find a MacroCandidate in the graph by its name and package name, or None for
+        any package. The root project name is used to determine priority:
+         - locally defined macros come first
+         - then imported macros
+         - then macros defined in the root project
+        """
+        filter: Optional[Callable[[MacroCandidate], bool]] = None
+        if package is not None:
+
+            def filter(candidate: MacroCandidate) -> bool:
+                return package == candidate.macro.package_name
+
+        candidates: CandidateList = self._find_macros_by_name(
+            name=name,
+            root_project_name=root_project_name,
+            filter=filter,
+        )
+
+        return candidates.last_candidate()
+
+    def find_macro_by_name(
+        self, name: str, root_project_name: str, package: Optional[str]
+    ) -> Optional[Macro]:
+        macro_candidate = self.find_macro_candidate_by_name(
+            name=name, root_project_name=root_project_name, package=package
+        )
+        return macro_candidate.macro if macro_candidate else None
+
+    def find_generate_macro_by_name(
+        self, component: str, root_project_name: str, imported_package: Optional[str] = None
+    ) -> Optional[Macro]:
+        """
+        The default `generate_X_name` macros are similar to regular ones, but only
+        includes imported packages when searching for a package.
+        - if package is not provided:
+            - if there is a `generate_{component}_name` macro in the root
+              project, return it
+            - return the `generate_{component}_name` macro from the 'dbt'
+              internal project
+        - if package is provided
+            - return the `generate_{component}_name` macro from the imported
+              package, if one exists
+        """
+
+        def filter(candidate: MacroCandidate) -> bool:
+            if imported_package:
+                return (
+                    candidate.locality == Locality.Imported
+                    and imported_package == candidate.macro.package_name
+                )
+            else:
+                return candidate.locality != Locality.Imported
+
+        candidates: CandidateList = self._find_macros_by_name(
+            name=f"generate_{component}_name",
+            root_project_name=root_project_name,
+            filter=filter,
+        )
+
+        return candidates.last()
+
+    def _find_macros_by_name(
+        self,
+        name: str,
+        root_project_name: str,
+        filter: Optional[Callable[[MacroCandidate], bool]] = None,
+    ) -> CandidateList:
+        """Find macros by their name."""
+        candidates: CandidateList = CandidateList()
+
+        macros_by_name = self.get_macros_by_name()
+        if name not in macros_by_name:
+            return candidates
+
+        packages = set(get_adapter_package_names(self.metadata.adapter_type))
+        for macro in macros_by_name[name]:
+            candidate = MacroCandidate(
+                locality=_get_locality(macro, root_project_name, packages),
+                macro=macro,
+            )
+            if filter is None or filter(candidate):
+                candidates.append(candidate)
+
+        return candidates
+
+    def get_macros_by_name(self) -> Dict[str, List[Macro]]:
+        if self._macros_by_name is None:
+            # The by-name mapping doesn't exist yet (perhaps because the manifest
+            # was deserialized), so we build it.
+            self._macros_by_name = self._build_macros_by_name(self.macros)
+
+        return self._macros_by_name
+
+    @staticmethod
+    def _build_macros_by_name(macros: Mapping[str, Macro]) -> Dict[str, List[Macro]]:
+        # Convert a macro dictionary keyed on unique id to a flattened version
+        # keyed on macro name for faster lookup by name. Since macro names are
+        # not necessarily unique, the dict value is a list.
+        macros_by_name: Dict[str, List[Macro]] = {}
+        for macro in macros.values():
+            if macro.name not in macros_by_name:
+                macros_by_name[macro.name] = []
+
+            macros_by_name[macro.name].append(macro)
+
+        return macros_by_name
+
+    def get_macros_by_package(self) -> Dict[str, Dict[str, Macro]]:
+        if self._macros_by_package is None:
+            # The by-package mapping doesn't exist yet (perhaps because the manifest
+            # was deserialized), so we build it.
+            self._macros_by_package = self._build_macros_by_package(self.macros)
+
+        return self._macros_by_package
+
+    @staticmethod
+    def _build_macros_by_package(macros: Mapping[str, Macro]) -> Dict[str, Dict[str, Macro]]:
+        # Convert a macro dictionary keyed on unique id to a flattened version
+        # keyed on package name for faster lookup by name.
+        macros_by_package: Dict[str, Dict[str, Macro]] = {}
+        for macro in macros.values():
+            if macro.package_name not in macros_by_package:
+                macros_by_package[macro.package_name] = {}
+            macros_by_name = macros_by_package[macro.package_name]
+            macros_by_name[macro.name] = macro
+
+        return macros_by_package
+
+
+@dataclass
+class ParsingInfo:
+    static_analysis_parsed_path_count: int = 0
+    static_analysis_path_count: int = 0
+
+
+@dataclass
+class ManifestStateCheck(dbtClassMixin):
+    vars_hash: FileHash = field(default_factory=FileHash.empty)
+    project_env_vars_hash: FileHash = field(default_factory=FileHash.empty)
+    profile_env_vars_hash: FileHash = field(default_factory=FileHash.empty)
+    profile_hash: FileHash = field(default_factory=FileHash.empty)
+    project_hashes: MutableMapping[str, FileHash] = field(default_factory=dict)
+
+
+NodeClassT = TypeVar("NodeClassT", bound="BaseNode")
+ResourceClassT = TypeVar("ResourceClassT", bound="BaseResource")
+
+
+@dataclass
+class Manifest(MacroMethods, dbtClassMixin):
+    """The manifest for the full graph, after parsing and during compilation."""
+
+    # These attributes are both positional and by keyword. If an attribute
+    # is added it must all be added in the __reduce_ex__ method in the
+    # args tuple in the right position.
+    nodes: MutableMapping[str, ManifestNode] = field(default_factory=dict)
+    sources: MutableMapping[str, SourceDefinition] = field(default_factory=dict)
+    macros: MutableMapping[str, Macro] = field(default_factory=dict)
+    docs: MutableMapping[str, Documentation] = field(default_factory=dict)
+    exposures: MutableMapping[str, Exposure] = field(default_factory=dict)
+    functions: MutableMapping[str, FunctionNode] = field(default_factory=dict)
+    metrics: MutableMapping[str, Metric] = field(default_factory=dict)
+    groups: MutableMapping[str, Group] = field(default_factory=dict)
+    selectors: MutableMapping[str, Any] = field(default_factory=dict)
+    files: MutableMapping[str, AnySourceFile] = field(default_factory=dict)
+    metadata: ManifestMetadata = field(default_factory=ManifestMetadata)
+    flat_graph: Dict[str, Any] = field(default_factory=dict)
+    state_check: ManifestStateCheck = field(default_factory=ManifestStateCheck)
+    source_patches: MutableMapping[SourceKey, SourcePatch] = field(default_factory=dict)
+    disabled: MutableMapping[str, List[GraphMemberNode]] = field(default_factory=dict)
+    env_vars: MutableMapping[str, str] = field(default_factory=dict)
+    semantic_models: MutableMapping[str, SemanticModel] = field(default_factory=dict)
+    unit_tests: MutableMapping[str, UnitTestDefinition] = field(default_factory=dict)
+    saved_queries: MutableMapping[str, SavedQuery] = field(default_factory=dict)
+    fixtures: MutableMapping[str, UnitTestFileFixture] = field(default_factory=dict)
+
+    _doc_lookup: Optional[DocLookup] = field(
+        default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
+    )
+    _source_lookup: Optional[SourceLookup] = field(
+        default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
+    )
+    _ref_lookup: Optional[RefableLookup] = field(
+        default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
+    )
+    _metric_lookup: Optional[MetricLookup] = field(
+        default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
+    )
+    _saved_query_lookup: Optional[SavedQueryLookup] = field(
+        default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
+    )
+    _semantic_model_by_measure_lookup: Optional[SemanticModelByMeasureLookup] = field(
+        default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
+    )
+    _disabled_lookup: Optional[DisabledLookup] = field(
+        default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
+    )
+    _analysis_lookup: Optional[AnalysisLookup] = field(
+        default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
+    )
+    _singular_test_lookup: Optional[SingularTestLookup] = field(
+        default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
+    )
+    _function_lookup: Optional[FunctionLookup] = field(
+        default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
+    )
+    _parsing_info: ParsingInfo = field(
+        default_factory=ParsingInfo,
+        metadata={"serialize": lambda x: None, "deserialize": lambda x: None},
+    )
+    _lock: Lock = field(
+        default_factory=get_mp_context().Lock,
+        metadata={"serialize": lambda x: None, "deserialize": lambda x: None},
+    )
+    _macros_by_name: Optional[Dict[str, List[Macro]]] = field(
+        default=None,
+        metadata={"serialize": lambda x: None, "deserialize": lambda x: None},
+    )
+    _macros_by_package: Optional[Dict[str, Dict[str, Macro]]] = field(
+        default=None,
+        metadata={"serialize": lambda x: None, "deserialize": lambda x: None},
+    )
+
+    def __pre_serialize__(self, context: Optional[Dict] = None):
+        # serialization won't work with anything except an empty source_patches because
+        # tuple keys are not supported, so ensure it's empty
+        self.source_patches = {}
+        return self
+
+    @classmethod
+    def __post_deserialize__(cls, obj):
+        obj._lock = get_mp_context().Lock()
+        return obj
+
+    def build_flat_graph(self):
+        """This attribute is used in context.common by each node, so we want to
+        only build it once and avoid any concurrency issues around it.
+        Make sure you don't call this until you're done with building your
+        manifest!
+        """
+        self.flat_graph = {
+            "exposures": {k: v.to_dict(omit_none=False) for k, v in self.exposures.items()},
+            "functions": {k: v.to_dict(omit_none=False) for k, v in self.functions.items()},
+            "groups": {k: v.to_dict(omit_none=False) for k, v in self.groups.items()},
+            "metrics": {k: v.to_dict(omit_none=False) for k, v in self.metrics.items()},
+            "nodes": {k: v.to_dict(omit_none=False) for k, v in self.nodes.items()},
+            "sources": {k: v.to_dict(omit_none=False) for k, v in self.sources.items()},
+            "semantic_models": {
+                k: v.to_dict(omit_none=False) for k, v in self.semantic_models.items()
+            },
+            "saved_queries": {
+                k: v.to_dict(omit_none=False) for k, v in self.saved_queries.items()
+            },
+        }
+
+    def build_disabled_by_file_id(self):
+        disabled_by_file_id = {}
+        for node_list in self.disabled.values():
+            for node in node_list:
+                disabled_by_file_id[node.file_id] = node
+        return disabled_by_file_id
+
+    def _get_parent_adapter_types(self, adapter_type: str) -> List[str]:
+        # This is duplicated logic from core/dbt/context/providers.py
+        # Ideally this would instead be incorporating actual dispatch logic
+        from dbt.adapters.factory import get_adapter_type_names
+
+        # order matters for dispatch:
+        #  1. current adapter
+        #  2. any parent adapters (dependencies)
+        #  3. 'default'
+        return get_adapter_type_names(adapter_type) + ["default"]
+
+    def _materialization_candidates_for(
+        self,
+        project_name: str,
+        materialization_name: str,
+        adapter_type: str,
+        specificity: int,
+    ) -> CandidateList:
+        full_name = dbt_common.utils.get_materialization_macro_name(
+            materialization_name=materialization_name,
+            adapter_type=adapter_type,
+            with_prefix=False,
+        )
+        return CandidateList(
+            MaterializationCandidate.from_macro(m, specificity)
+            for m in self._find_macros_by_name(full_name, project_name)
+        )
+
+    def find_materialization_macro_by_name(
+        self, project_name: str, materialization_name: str, adapter_type: str
+    ) -> Optional[Macro]:
+        candidates: CandidateList = CandidateList(
+            chain.from_iterable(
+                self._materialization_candidates_for(
+                    project_name=project_name,
+                    materialization_name=materialization_name,
+                    adapter_type=atype,
+                    specificity=specificity,  # where in the inheritance chain this candidate is
+                )
+                for specificity, atype in enumerate(self._get_parent_adapter_types(adapter_type))
+            )
+        )
+        core_candidates = [
+            candidate for candidate in candidates if candidate.locality == Locality.Core
+        ]
+
+        materialization_candidate = candidates.last_candidate()
+        # If an imported materialization macro was found that also had a core candidate, fire a deprecation
+        if (
+            materialization_candidate is not None
+            and materialization_candidate.locality == Locality.Imported
+            and core_candidates
+        ):
+            # preserve legacy behaviour - allow materialization override
+            if (
+                get_flags().require_explicit_package_overrides_for_builtin_materializations
+                is False
+            ):
+                deprecations.warn(
+                    "package-materialization-override",
+                    package_name=materialization_candidate.macro.package_name,
+                    materialization_name=materialization_name,
+                )
+            else:
+                materialization_candidate = candidates.last_candidate(
+                    valid_localities=[Locality.Core, Locality.Root]
+                )
+
+        return materialization_candidate.macro if materialization_candidate else None
+
+    def get_resource_fqns(self) -> Mapping[str, PathSet]:
+        resource_fqns: Dict[str, Set[Tuple[str, ...]]] = {}
+        all_resources = chain(
+            self.exposures.values(),
+            self.functions.values(),
+            self.nodes.values(),
+            self.sources.values(),
+            self.metrics.values(),
+            self.semantic_models.values(),
+            self.saved_queries.values(),
+            self.unit_tests.values(),
+        )
+        for resource in all_resources:
+            resource_type_plural = resource.resource_type.pluralize()
+            if resource_type_plural not in resource_fqns:
+                resource_fqns[resource_type_plural] = set()
+            resource_fqns[resource_type_plural].add(tuple(resource.fqn))
+        return resource_fqns
+
+    def get_used_schemas(self, resource_types=None):
+        return frozenset(
+            {
+                (node.database, node.schema)
+                for node in chain(self.nodes.values(), self.sources.values())
+                if not resource_types or node.resource_type in resource_types
+            }
+        )
+
+    def get_used_databases(self):
+        return frozenset(x.database for x in chain(self.nodes.values(), self.sources.values()))
+
+    def deepcopy(self):
+        copy = Manifest(
+            nodes={k: _deepcopy(v) for k, v in self.nodes.items()},
+            sources={k: _deepcopy(v) for k, v in self.sources.items()},
+            macros={k: _deepcopy(v) for k, v in self.macros.items()},
+            docs={k: _deepcopy(v) for k, v in self.docs.items()},
+            exposures={k: _deepcopy(v) for k, v in self.exposures.items()},
+            functions={k: _deepcopy(v) for k, v in self.functions.items()},
+            metrics={k: _deepcopy(v) for k, v in self.metrics.items()},
+            groups={k: _deepcopy(v) for k, v in self.groups.items()},
+            selectors={k: _deepcopy(v) for k, v in self.selectors.items()},
+            metadata=self.metadata,
+            disabled={k: _deepcopy(v) for k, v in self.disabled.items()},
+            files={k: _deepcopy(v) for k, v in self.files.items()},
+            state_check=_deepcopy(self.state_check),
+            semantic_models={k: _deepcopy(v) for k, v in self.semantic_models.items()},
+            unit_tests={k: _deepcopy(v) for k, v in self.unit_tests.items()},
+            saved_queries={k: _deepcopy(v) for k, v in self.saved_queries.items()},
+        )
+        copy.build_flat_graph()
+        return copy
+
+    def build_parent_and_child_maps(self):
+        edge_members = list(
+            chain(
+                self.nodes.values(),
+                self.sources.values(),
+                self.exposures.values(),
+                self.functions.values(),
+                self.metrics.values(),
+                self.semantic_models.values(),
+                self.saved_queries.values(),
+                self.unit_tests.values(),
+            )
+        )
+        forward_edges, backward_edges = build_node_edges(edge_members)
+        self.child_map = forward_edges
+        self.parent_map = backward_edges
+
+    def build_macro_child_map(self):
+        edge_members = list(
+            chain(
+                self.nodes.values(),
+                self.macros.values(),
+            )
+        )
+        forward_edges = build_macro_edges(edge_members)
+        return forward_edges
+
+    def build_group_map(self):
+        groupable_nodes = list(
+            chain(
+                self.nodes.values(),
+                self.saved_queries.values(),
+                self.semantic_models.values(),
+                self.metrics.values(),
+            )
+        )
+        group_map = {group.name: [] for group in self.groups.values()}
+        for node in groupable_nodes:
+            if node.group is not None:
+                # group updates are not included with state:modified and
+                # by ignoring the groups that aren't in the group map we
+                # can avoid hitting errors for groups that are not getting
+                # updated.  This is a hack but any groups that are not
+                # valid will be caught in
+                # parser.manifest.ManifestLoader.check_valid_group_config_node
+                if node.group in group_map:
+                    group_map[node.group].append(node.unique_id)
+        self.group_map = group_map
+
+    def fill_tracking_metadata(self):
+        self.metadata.user_id = tracking.active_user.id if tracking.active_user else None
+        self.metadata.send_anonymous_usage_stats = get_flags().SEND_ANONYMOUS_USAGE_STATS
+
+    @classmethod
+    def from_writable_manifest(cls, writable_manifest: WritableManifest) -> "Manifest":
+        manifest = Manifest(
+            nodes=cls._map_resources_to_map_nodes(writable_manifest.nodes),
+            disabled=cls._map_list_resources_to_map_list_nodes(writable_manifest.disabled),
+            unit_tests=cls._map_resources_to_map_nodes(writable_manifest.unit_tests),
+            sources=cls._map_resources_to_map_nodes(writable_manifest.sources),
+            macros=cls._map_resources_to_map_nodes(writable_manifest.macros),
+            docs=cls._map_resources_to_map_nodes(writable_manifest.docs),
+            exposures=cls._map_resources_to_map_nodes(writable_manifest.exposures),
+            functions=cls._map_resources_to_map_nodes(writable_manifest.functions),
+            metrics=cls._map_resources_to_map_nodes(writable_manifest.metrics),
+            groups=cls._map_resources_to_map_nodes(writable_manifest.groups),
+            semantic_models=cls._map_resources_to_map_nodes(writable_manifest.semantic_models),
+            saved_queries=cls._map_resources_to_map_nodes(writable_manifest.saved_queries),
+            selectors={
+                selector_id: selector
+                for selector_id, selector in writable_manifest.selectors.items()
+            },
+            metadata=writable_manifest.metadata,
+        )
+
+        return manifest
+
+    def _map_nodes_to_map_resources(cls, nodes_map: MutableMapping[str, NodeClassT]):
+        return {node_id: node.to_resource() for node_id, node in nodes_map.items()}
+
+    def _map_list_nodes_to_map_list_resources(
+        cls, nodes_map: MutableMapping[str, List[NodeClassT]]
+    ):
+        return {
+            node_id: [node.to_resource() for node in node_list]
+            for node_id, node_list in nodes_map.items()
+        }
+
+    @classmethod
+    def _map_resources_to_map_nodes(cls, resources_map: Mapping[str, ResourceClassT]):
+        return {
+            node_id: RESOURCE_CLASS_TO_NODE_CLASS[type(resource)].from_resource(resource)
+            for node_id, resource in resources_map.items()
+        }
+
+    @classmethod
+    def _map_list_resources_to_map_list_nodes(
+        cls, resources_map: Optional[Mapping[str, List[ResourceClassT]]]
+    ):
+        if resources_map is None:
+            return {}
+
+        return {
+            node_id: [
+                RESOURCE_CLASS_TO_NODE_CLASS[type(resource)].from_resource(resource)
+                for resource in resource_list
+            ]
+            for node_id, resource_list in resources_map.items()
+        }
+
+    def writable_manifest(self) -> "WritableManifest":
+        self.build_parent_and_child_maps()
+        self.build_group_map()
+        self.fill_tracking_metadata()
+
+        return WritableManifest(
+            nodes=self._map_nodes_to_map_resources(self.nodes),
+            sources=self._map_nodes_to_map_resources(self.sources),
+            macros=self._map_nodes_to_map_resources(self.macros),
+            docs=self._map_nodes_to_map_resources(self.docs),
+            exposures=self._map_nodes_to_map_resources(self.exposures),
+            functions=self._map_nodes_to_map_resources(self.functions),
+            metrics=self._map_nodes_to_map_resources(self.metrics),
+            groups=self._map_nodes_to_map_resources(self.groups),
+            selectors=self.selectors,
+            metadata=self.metadata,
+            disabled=self._map_list_nodes_to_map_list_resources(self.disabled),
+            child_map=self.child_map,
+            parent_map=self.parent_map,
+            group_map=self.group_map,
+            semantic_models=self._map_nodes_to_map_resources(self.semantic_models),
+            unit_tests=self._map_nodes_to_map_resources(self.unit_tests),
+            saved_queries=self._map_nodes_to_map_resources(self.saved_queries),
+        )
+
+    def write(self, path):
+        writable = self.writable_manifest()
+        writable.write(path)
+        fire_event(ArtifactWritten(artifact_type=writable.__class__.__name__, artifact_path=path))
+
+    # Called in dbt.compilation.Linker.write_graph and
+    # dbt.graph.queue.get and ._include_in_cost
+    def expect(self, unique_id: str) -> GraphMemberNode:
+        if unique_id in self.nodes:
+            return self.nodes[unique_id]
+        elif unique_id in self.sources:
+            return self.sources[unique_id]
+        elif unique_id in self.exposures:
+            return self.exposures[unique_id]
+        elif unique_id in self.functions:
+            return self.functions[unique_id]
+        elif unique_id in self.metrics:
+            return self.metrics[unique_id]
+        elif unique_id in self.semantic_models:
+            return self.semantic_models[unique_id]
+        elif unique_id in self.unit_tests:
+            return self.unit_tests[unique_id]
+        elif unique_id in self.saved_queries:
+            return self.saved_queries[unique_id]
+        else:
+            # something terrible has happened
+            raise dbt_common.exceptions.DbtInternalError(
+                "Expected node {} not found in manifest".format(unique_id)
+            )
+
+    @property
+    def doc_lookup(self) -> DocLookup:
+        if self._doc_lookup is None:
+            self._doc_lookup = DocLookup(self)
+        return self._doc_lookup
+
+    def rebuild_doc_lookup(self):
+        self._doc_lookup = DocLookup(self)
+
+    @property
+    def source_lookup(self) -> SourceLookup:
+        if self._source_lookup is None:
+            self._source_lookup = SourceLookup(self)
+        return self._source_lookup
+
+    def rebuild_source_lookup(self):
+        self._source_lookup = SourceLookup(self)
+
+    @property
+    def ref_lookup(self) -> RefableLookup:
+        if self._ref_lookup is None:
+            self._ref_lookup = RefableLookup(self)
+        return self._ref_lookup
+
+    @property
+    def metric_lookup(self) -> MetricLookup:
+        if self._metric_lookup is None:
+            self._metric_lookup = MetricLookup(self)
+        return self._metric_lookup
+
+    @property
+    def saved_query_lookup(self) -> SavedQueryLookup:
+        """Retuns a SavedQueryLookup, instantiating it first if necessary."""
+        if self._saved_query_lookup is None:
+            self._saved_query_lookup = SavedQueryLookup(self)
+        return self._saved_query_lookup
+
+    @property
+    def semantic_model_by_measure_lookup(self) -> SemanticModelByMeasureLookup:
+        """Gets (and creates if necessary) the lookup utility for getting SemanticModels by measures"""
+        if self._semantic_model_by_measure_lookup is None:
+            self._semantic_model_by_measure_lookup = SemanticModelByMeasureLookup(self)
+        return self._semantic_model_by_measure_lookup
+
+    def rebuild_ref_lookup(self):
+        self._ref_lookup = RefableLookup(self)
+
+    @property
+    def disabled_lookup(self) -> DisabledLookup:
+        if self._disabled_lookup is None:
+            self._disabled_lookup = DisabledLookup(self)
+        return self._disabled_lookup
+
+    def rebuild_disabled_lookup(self):
+        self._disabled_lookup = DisabledLookup(self)
+
+    @property
+    def analysis_lookup(self) -> AnalysisLookup:
+        if self._analysis_lookup is None:
+            self._analysis_lookup = AnalysisLookup(self)
+        return self._analysis_lookup
+
+    @property
+    def singular_test_lookup(self) -> SingularTestLookup:
+        if self._singular_test_lookup is None:
+            self._singular_test_lookup = SingularTestLookup(self)
+        return self._singular_test_lookup
+
+    @property
+    def function_lookup(self) -> FunctionLookup:
+        if self._function_lookup is None:
+            self._function_lookup = FunctionLookup(self)
+        return self._function_lookup
+
+    @property
+    def external_node_unique_ids(self):
+        return [node.unique_id for node in self.nodes.values() if node.is_external_node]
+
+    # Called by dbt.parser.manifest._process_refs & ManifestLoader.check_for_model_deprecations
+    def resolve_ref(
+        self,
+        source_node: GraphMemberNode,
+        target_model_name: str,
+        target_model_package: Optional[str],
+        target_model_version: Optional[NodeVersion],
+        current_project: str,
+        node_package: str,
+    ) -> MaybeNonSource:
+
+        node: Optional[ManifestNode] = None
+        disabled: Optional[List[ManifestNode]] = None
+
+        candidates = _packages_to_search(current_project, node_package, target_model_package)
+        for pkg in candidates:
+            node = self.ref_lookup.find(
+                target_model_name, pkg, target_model_version, self, source_node
+            )
+
+            if node is not None and hasattr(node, "config") and node.config.enabled:
+                return node
+
+            # it's possible that the node is disabled
+            if disabled is None:
+                disabled = self.disabled_lookup.find(
+                    target_model_name,
+                    pkg,
+                    version=target_model_version,
+                    resource_types=REFABLE_NODE_TYPES,
+                )
+
+        if disabled:
+            return Disabled(disabled[0])
+        return None
+
+    # Called by dbt.parser.manifest._resolve_sources_for_exposure
+    # and dbt.parser.manifest._process_source_for_node
+    def resolve_source(
+        self,
+        target_source_name: str,
+        target_table_name: str,
+        current_project: str,
+        node_package: str,
+    ) -> MaybeParsedSource:
+        search_name = f"{target_source_name}.{target_table_name}"
+        candidates = _packages_to_search(current_project, node_package)
+
+        source: Optional[SourceDefinition] = None
+        disabled: Optional[List[SourceDefinition]] = None
+
+        for pkg in candidates:
+            source = self.source_lookup.find(search_name, pkg, self)
+            if source is not None and source.config.enabled:
+                return source
+
+            if disabled is None:
+                disabled = self.disabled_lookup.find(
+                    f"{target_source_name}.{target_table_name}", pkg
+                )
+
+        if disabled:
+            return Disabled(disabled[0])
+        return None
+
+    def resolve_function(
+        self,
+        target_function_name: str,
+        target_function_package: Optional[str],
+        current_project: str,
+        node_package: str,
+    ) -> MaybeFunctionNode:
+        package_candidates = _packages_to_search(
+            current_project, node_package, target_function_package
+        )
+        disabled: Optional[List[FunctionNode]] = None
+        for package in package_candidates:
+            function = self.function_lookup.find(target_function_name, package, self)
+            if function is not None and function.config.enabled:
+                return function
+
+            # it's possible that the function is disabled
+            if disabled is None:
+                disabled = self.disabled_lookup.find(target_function_name, package)
+        if disabled:
+            return Disabled(disabled[0])
+        return None
+
+    def resolve_metric(
+        self,
+        target_metric_name: str,
+        target_metric_package: Optional[str],
+        current_project: str,
+        node_package: str,
+    ) -> MaybeMetricNode:
+
+        metric: Optional[Metric] = None
+        disabled: Optional[List[Metric]] = None
+
+        candidates = _packages_to_search(current_project, node_package, target_metric_package)
+        for pkg in candidates:
+            metric = self.metric_lookup.find(target_metric_name, pkg, self)
+
+            if metric is not None and metric.config.enabled:
+                return metric
+
+            # it's possible that the node is disabled
+            if disabled is None:
+                disabled = self.disabled_lookup.find(f"{target_metric_name}", pkg)
+        if disabled:
+            return Disabled(disabled[0])
+        return None
+
+    def resolve_saved_query(
+        self,
+        target_saved_query_name: str,
+        target_saved_query_package: Optional[str],
+        current_project: str,
+        node_package: str,
+    ) -> MaybeSavedQueryNode:
+        """Tries to find the SavedQuery by name within the available project and packages.
+
+        Will return the first enabled SavedQuery matching the name found while iterating over
+        the scoped packages. If no enabled SavedQuery node match is found, returns the last
+        disabled SavedQuery node. Otherwise it returns None.
+        """
+        disabled: Optional[List[SavedQuery]] = None
+        candidates = _packages_to_search(current_project, node_package, target_saved_query_package)
+        for pkg in candidates:
+            saved_query = self.saved_query_lookup.find(target_saved_query_name, pkg, self)
+
+            if saved_query is not None and saved_query.config.enabled:
+                return saved_query
+
+            # it's possible that the node is disabled
+            if disabled is None:
+                disabled = self.disabled_lookup.find(f"{target_saved_query_name}", pkg)
+        if disabled:
+            return Disabled(disabled[0])
+
+        return None
+
+    def resolve_semantic_model_for_measure(
+        self,
+        target_measure_name: str,
+        current_project: str,
+        node_package: str,
+        target_package: Optional[str] = None,
+    ) -> Optional[SemanticModel]:
+        """Tries to find the SemanticModel that a measure belongs to"""
+        candidates = _packages_to_search(current_project, node_package, target_package)
+
+        for pkg in candidates:
+            semantic_model = self.semantic_model_by_measure_lookup.find(
+                target_measure_name, pkg, self
+            )
+            # need to return it even if it's disabled so know it's not fully missing
+            if semantic_model is not None:
+                return semantic_model
+
+        return None
+
+    # Called by DocsRuntimeContext.doc
+    def resolve_doc(
+        self,
+        name: str,
+        package: Optional[str],
+        current_project: str,
+        node_package: str,
+    ) -> Optional[Documentation]:
+        """Resolve the given documentation. This follows the same algorithm as
+        resolve_ref except the is_enabled checks are unnecessary as docs are
+        always enabled.
+        """
+        candidates = _packages_to_search(current_project, node_package, package)
+
+        for pkg in candidates:
+            result = self.doc_lookup.find(name, pkg, self)
+            if result is not None:
+                return result
+        return None
+
+    def is_invalid_private_ref(
+        self, node: GraphMemberNode, target_model: MaybeNonSource, dependencies: Optional[Mapping]
+    ) -> bool:
+        dependencies = dependencies or {}
+        if not isinstance(target_model, ModelNode):
+            return False
+
+        is_private_ref = (
+            target_model.access == AccessType.Private
+            # don't raise this reference error for ad hoc 'preview' queries
+            and node.resource_type != NodeType.SqlOperation
+            and node.resource_type != NodeType.RPCCall  # TODO: rm
+        )
+        target_dependency = dependencies.get(target_model.package_name)
+        restrict_package_access = target_dependency.restrict_access if target_dependency else False
+
+        # TODO: SemanticModel and SourceDefinition do not have group, and so should not be able to make _any_ private ref.
+        return is_private_ref and (
+            not hasattr(node, "group")
+            or not node.group
+            # Invalid reference because group does not match
+            or node.group != target_model.group
+            # Or, invalid because these are different namespaces (project/package) and restrict-access is enforced
+            or (node.package_name != target_model.package_name and restrict_package_access)
+        )
+
+    def is_invalid_protected_ref(
+        self, node: GraphMemberNode, target_model: MaybeNonSource, dependencies: Optional[Mapping]
+    ) -> bool:
+        dependencies = dependencies or {}
+        if not isinstance(target_model, ModelNode):
+            return False
+
+        is_protected_ref = (
+            target_model.access == AccessType.Protected
+            # don't raise this reference error for ad hoc 'preview' queries
+            and node.resource_type != NodeType.SqlOperation
+            and node.resource_type != NodeType.RPCCall  # TODO: rm
+        )
+        target_dependency = dependencies.get(target_model.package_name)
+        restrict_package_access = target_dependency.restrict_access if target_dependency else False
+
+        return is_protected_ref and (
+            node.package_name != target_model.package_name and restrict_package_access
+        )
+
+    # Called in GraphRunnableTask.before_run, RunTask.before_run, CloneTask.before_run
+    def merge_from_artifact(self, other: "Manifest") -> None:
+        """Update this manifest by adding the 'defer_relation' attribute to all nodes
+        with a counterpart in the stateful manifest used for deferral.
+
+        Only non-ephemeral refable nodes are examined.
+        """
+        refables = set(REFABLE_NODE_TYPES)
+        for unique_id, node in other.nodes.items():
+            current = self.nodes.get(unique_id)
+            if current and node.resource_type in refables and not node.is_ephemeral:
+                assert isinstance(node.config, NodeConfig)  # this makes mypy happy
+                defer_relation = DeferRelation(
+                    database=node.database,
+                    schema=node.schema,
+                    alias=node.alias,
+                    relation_name=node.relation_name,
+                    resource_type=node.resource_type,
+                    name=node.name,
+                    description=node.description,
+                    compiled_code=(node.compiled_code if not isinstance(node, SeedNode) else None),
+                    meta=node.meta,
+                    tags=node.tags,
+                    config=node.config,
+                )
+                self.nodes[unique_id] = replace(current, defer_relation=defer_relation)
+
+        # Rebuild the flat_graph, which powers the 'graph' context variable
+        self.build_flat_graph()
+
+    # Methods that were formerly in ParseResult
+    def add_macro(self, source_file: SourceFile, macro: Macro):
+        if macro.unique_id in self.macros:
+            # detect that the macro exists and emit an error
+            raise DuplicateMacroInPackageError(macro=macro, macro_mapping=self.macros)
+
+        self.macros[macro.unique_id] = macro
+
+        if self._macros_by_name is None:
+            self._macros_by_name = self._build_macros_by_name(self.macros)
+
+        if macro.name not in self._macros_by_name:
+            self._macros_by_name[macro.name] = []
+
+        self._macros_by_name[macro.name].append(macro)
+
+        if self._macros_by_package is None:
+            self._macros_by_package = self._build_macros_by_package(self.macros)
+
+        if macro.package_name not in self._macros_by_package:
+            self._macros_by_package[macro.package_name] = {}
+
+        self._macros_by_package[macro.package_name][macro.name] = macro
+
+        source_file.macros.append(macro.unique_id)
+
+    def has_file(self, source_file: SourceFile) -> bool:
+        key = source_file.file_id
+        if key is None:
+            return False
+        if key not in self.files:
+            return False
+        my_checksum = self.files[key].checksum
+        return my_checksum == source_file.checksum
+
+    def add_source(self, source_file: SchemaSourceFile, source: UnpatchedSourceDefinition):
+        # sources can't be overwritten!
+        _check_duplicates(source, self.sources)
+        self.sources[source.unique_id] = source  # type: ignore
+        source_file.sources.append(source.unique_id)
+
+    def add_node_nofile(self, node: ManifestNode):
+        # nodes can't be overwritten!
+        _check_duplicates(node, self.nodes)
+        self.nodes[node.unique_id] = node
+
+    def add_node(self, source_file: AnySourceFile, node: ManifestNode, test_from=None):
+        self.add_node_nofile(node)
+        if isinstance(source_file, SchemaSourceFile):
+            if isinstance(node, GenericTestNode):
+                assert test_from
+                source_file.add_test(node.unique_id, test_from)
+            elif isinstance(node, Metric):
+                source_file.metrics.append(node.unique_id)
+            elif isinstance(node, Exposure):
+                source_file.exposures.append(node.unique_id)
+            elif isinstance(node, Group):
+                source_file.groups.append(node.unique_id)
+            elif isinstance(node, SnapshotNode):
+                source_file.snapshots.append(node.unique_id)
+        elif isinstance(source_file, FixtureSourceFile):
+            pass
+        else:
+            source_file.nodes.append(node.unique_id)
+
+    def add_exposure(self, source_file: SchemaSourceFile, exposure: Exposure):
+        _check_duplicates(exposure, self.exposures)
+        self.exposures[exposure.unique_id] = exposure
+        source_file.exposures.append(exposure.unique_id)
+
+    def add_function(self, function: FunctionNode):
+        _check_duplicates(function, self.functions)
+        self.functions[function.unique_id] = function
+
+    def add_metric(
+        self, source_file: SchemaSourceFile, metric: Metric, generated_from: Optional[str] = None
+    ):
+        _check_duplicates(metric, self.metrics)
+        self.metrics[metric.unique_id] = metric
+        if not generated_from:
+            source_file.metrics.append(metric.unique_id)
+        else:
+            source_file.add_metrics_from_measures(generated_from, metric.unique_id)
+
+    def add_group(self, source_file: SchemaSourceFile, group: Group):
+        _check_duplicates(group, self.groups)
+        self.groups[group.unique_id] = group
+        source_file.groups.append(group.unique_id)
+
+    def add_disabled_nofile(self, node: GraphMemberNode):
+        # There can be multiple disabled nodes for the same unique_id
+        if node.unique_id in self.disabled:
+            self.disabled[node.unique_id].append(node)
+        else:
+            self.disabled[node.unique_id] = [node]
+
+    def add_disabled(self, source_file: AnySourceFile, node: GraphMemberNode, test_from=None):
+        self.add_disabled_nofile(node)
+        if isinstance(source_file, SchemaSourceFile):
+            if isinstance(node, GenericTestNode):
+                assert test_from
+                source_file.add_test(node.unique_id, test_from)
+            if isinstance(node, Metric):
+                source_file.metrics.append(node.unique_id)
+            if isinstance(node, SavedQuery):
+                source_file.saved_queries.append(node.unique_id)
+            if isinstance(node, SemanticModel):
+                source_file.semantic_models.append(node.unique_id)
+            if isinstance(node, Exposure):
+                source_file.exposures.append(node.unique_id)
+            if isinstance(node, FunctionNode):
+                source_file.functions.append(node.unique_id)
+            if isinstance(node, UnitTestDefinition):
+                source_file.unit_tests.append(node.unique_id)
+        elif isinstance(source_file, FixtureSourceFile):
+            pass
+        else:
+            source_file.nodes.append(node.unique_id)
+
+    def add_doc(self, source_file: SourceFile, doc: Documentation):
+        _check_duplicates(doc, self.docs)
+        self.docs[doc.unique_id] = doc
+        source_file.docs.append(doc.unique_id)
+
+    def add_semantic_model(self, source_file: SchemaSourceFile, semantic_model: SemanticModel):
+        _check_duplicates(semantic_model, self.semantic_models)
+        self.semantic_models[semantic_model.unique_id] = semantic_model
+        source_file.semantic_models.append(semantic_model.unique_id)
+
+    def add_unit_test(self, source_file: SchemaSourceFile, unit_test: UnitTestDefinition):
+        if unit_test.unique_id in self.unit_tests:
+            raise DuplicateResourceNameError(unit_test, self.unit_tests[unit_test.unique_id])
+        self.unit_tests[unit_test.unique_id] = unit_test
+        source_file.unit_tests.append(unit_test.unique_id)
+
+    def add_fixture(self, source_file: FixtureSourceFile, fixture: UnitTestFileFixture):
+        if fixture.unique_id in self.fixtures:
+            raise DuplicateResourceNameError(fixture, self.fixtures[fixture.unique_id])
+        self.fixtures[fixture.unique_id] = fixture
+        source_file.fixture = fixture.unique_id
+
+    def add_saved_query(self, source_file: SchemaSourceFile, saved_query: SavedQuery) -> None:
+        _check_duplicates(saved_query, self.saved_queries)
+        self.saved_queries[saved_query.unique_id] = saved_query
+        source_file.saved_queries.append(saved_query.unique_id)
+
+    # end of methods formerly in ParseResult
+
+    def find_node_from_ref_or_source(
+        self, expression: str
+    ) -> Optional[Union[ModelNode, SourceDefinition]]:
+        ref_or_source = statically_parse_ref_or_source(expression)
+
+        node = None
+        if isinstance(ref_or_source, RefArgs):
+            node = self.ref_lookup.find(
+                ref_or_source.name, ref_or_source.package, ref_or_source.version, self
+            )
+        else:
+            source_name, source_table_name = ref_or_source[0], ref_or_source[1]
+            node = self.source_lookup.find(f"{source_name}.{source_table_name}", None, self)
+
+        return node
+
+    # Provide support for copy.deepcopy() - we just need to avoid the lock!
+    # pickle and deepcopy use this. It returns a callable object used to
+    # create the initial version of the object and a tuple of arguments
+    # for the object, i.e. the Manifest.
+    # The order of the arguments must match the order of the attributes
+    # in the Manifest class declaration, because they are used as
+    # positional arguments to construct a Manifest.
+    def __reduce_ex__(self, protocol):
+        args = (
+            self.nodes,
+            self.sources,
+            self.macros,
+            self.docs,
+            self.exposures,
+            self.functions,
+            self.metrics,
+            self.groups,
+            self.selectors,
+            self.files,
+            self.metadata,
+            self.flat_graph,
+            self.state_check,
+            self.source_patches,
+            self.disabled,
+            self.env_vars,
+            self.semantic_models,
+            self.unit_tests,
+            self.saved_queries,
+            self._doc_lookup,
+            self._source_lookup,
+            self._ref_lookup,
+            self._metric_lookup,
+            self._semantic_model_by_measure_lookup,
+            self._disabled_lookup,
+            self._analysis_lookup,
+            self._singular_test_lookup,
+        )
+        return self.__class__, args
+
+    def _microbatch_macro_is_core(self, project_name: str) -> bool:
+        microbatch_is_core = False
+        candidate = self.find_macro_candidate_by_name(
+            name="get_incremental_microbatch_sql", root_project_name=project_name, package=None
+        )
+
+        # We want to check for "Core", because "Core" basically means "builtin"
+        if candidate is not None and candidate.locality == Locality.Core:
+            microbatch_is_core = True
+
+        return microbatch_is_core
+
+    def use_microbatch_batches(self, project_name: str) -> bool:
+        return (
+            get_flags().require_batched_execution_for_custom_microbatch_strategy
+            or self._microbatch_macro_is_core(project_name=project_name)
+        )
+
+
+class MacroManifest(MacroMethods):
+    def __init__(self, macros) -> None:
+        self.macros = macros
+        self.metadata = ManifestMetadata(
+            user_id=tracking.active_user.id if tracking.active_user else None,
+            send_anonymous_usage_stats=(
+                get_flags().SEND_ANONYMOUS_USAGE_STATS if tracking.active_user else None
+            ),
+        )
+        # This is returned by the 'graph' context property
+        # in the ProviderContext class.
+        self.flat_graph: Dict[str, Any] = {}
+        self._macros_by_name: Optional[Dict[str, List[Macro]]] = None
+        self._macros_by_package: Optional[Dict[str, Dict[str, Macro]]] = None
+
+
+AnyManifest = Union[Manifest, MacroManifest]
+
+
+def _check_duplicates(value: BaseNode, src: Mapping[str, BaseNode]):
+    if value.unique_id in src:
+        raise DuplicateResourceNameError(value, src[value.unique_id])
+
+
+K_T = TypeVar("K_T")
+V_T = TypeVar("V_T")
+
+
+def _expect_value(key: K_T, src: Mapping[K_T, V_T], old_file: SourceFile, name: str) -> V_T:
+    if key not in src:
+        raise CompilationError(
+            'Expected to find "{}" in cached "result.{}" based '
+            "on cached file information: {}!".format(key, name, old_file)
+        )
+    return src[key]