dvt-core 1.11.0b4__py3-none-any.whl

dvt/parser/search.py

@@ -0,0 +1,149 @@
+import os
+from dataclasses import dataclass
+from typing import (
+    Callable,
+    Generic,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Set,
+    TypeVar,
+    Union,
+)
+
+from dvt import deprecations
+from dvt.config import Project
+from dvt.contracts.files import AnySourceFile, FilePath
+from dvt.exceptions import DbtInternalError, ParsingError
+from pathspec import PathSpec  # type: ignore
+
+from dbt_common.clients._jinja_blocks import ExtractWarning
+from dbt_common.clients.jinja import BlockTag, extract_toplevel_blocks
+from dbt_common.clients.system import find_matching
+
+
+# What's the point of wrapping a SourceFile with this class?
+# Could it be removed?
+@dataclass
+class FileBlock:
+    file: AnySourceFile
+
+    @property
+    def name(self):
+        base = os.path.basename(self.file.path.relative_path)
+        name, _ = os.path.splitext(base)
+        return name
+
+    @property
+    def contents(self):
+        return self.file.contents
+
+    @property
+    def path(self):
+        return self.file.path
+
+
+# The BlockTag is used in Jinja processing
+# Why do we have different classes where the only
+# difference is what 'contents' returns?
+@dataclass
+class BlockContents(FileBlock):
+    file: AnySourceFile  # if you remove this, mypy will get upset
+    block: BlockTag
+
+    @property
+    def name(self):
+        return self.block.block_name
+
+    @property
+    def contents(self):
+        return self.block.contents
+
+
+@dataclass
+class FullBlock(FileBlock):
+    file: AnySourceFile  # if you remove this, mypy will get upset
+    block: BlockTag
+
+    @property
+    def name(self):
+        return self.block.block_name
+
+    @property
+    def contents(self):
+        return self.block.full_block
+
+
+def filesystem_search(
+    project: Project,
+    relative_dirs: List[str],
+    extension: str,
+    ignore_spec: Optional[PathSpec] = None,
+):
+    ext = "[!.#~]*" + extension
+    root = project.project_root
+    file_path_list = []
+    for result in find_matching(root, relative_dirs, ext, ignore_spec):
+        if "searched_path" not in result or "relative_path" not in result:
+            raise DbtInternalError("Invalid result from find_matching: {}".format(result))
+        file_match = FilePath(
+            searched_path=result["searched_path"],
+            relative_path=result["relative_path"],
+            modification_time=result["modification_time"],
+            project_root=root,
+        )
+        file_path_list.append(file_match)
+
+    return file_path_list
+
+
+Block = Union[BlockContents, FullBlock]
+
+BlockSearchResult = TypeVar("BlockSearchResult", BlockContents, FullBlock)
+
+BlockSearchResultFactory = Callable[[AnySourceFile, BlockTag], BlockSearchResult]
+
+
+class BlockSearcher(Generic[BlockSearchResult], Iterable[BlockSearchResult]):
+    def __init__(
+        self,
+        source: List[FileBlock],
+        allowed_blocks: Set[str],
+        source_tag_factory: BlockSearchResultFactory,
+        check_jinja: bool = True,
+    ) -> None:
+        self.source = source
+        self.allowed_blocks = allowed_blocks
+        self.source_tag_factory: BlockSearchResultFactory = source_tag_factory
+        self.check_jinja = check_jinja
+
+    def extract_blocks(self, source_file: FileBlock) -> Iterable[BlockTag]:
+        # This is a bit of a hack to get the file path to the deprecation
+        def wrap_handle_extract_warning(warning: ExtractWarning) -> None:
+            self._handle_extract_warning(warning=warning, file=source_file.path.relative_path)
+
+        try:
+            blocks = extract_toplevel_blocks(
+                source_file.contents,
+                allowed_blocks=self.allowed_blocks,
+                collect_raw_data=False,
+                warning_callback=wrap_handle_extract_warning if self.check_jinja else None,
+            )
+            # this makes mypy happy, and this is an invariant we really need
+            for block in blocks:
+                assert isinstance(block, BlockTag)
+                yield block
+
+        except ParsingError as exc:
+            if exc.node is None:
+                exc.add_node(source_file)
+            raise
+
+    def _handle_extract_warning(self, warning: ExtractWarning, file: str) -> None:
+        deprecations.warn("unexpected-jinja-block-deprecation", msg=warning.msg, file=file)
+
+    def __iter__(self) -> Iterator[BlockSearchResult]:
+        for entry in self.source:
+            for block in self.extract_blocks(entry):
+                yield self.source_tag_factory(entry.file, block)

dvt/parser/seeds.py

@@ -0,0 +1,28 @@
+from dvt.context.context_config import ContextConfig
+from dvt.contracts.graph.nodes import SeedNode
+from dvt.node_types import NodeType
+from dvt.parser.base import SimpleSQLParser
+from dvt.parser.search import FileBlock
+
+
+class SeedParser(SimpleSQLParser[SeedNode]):
+    def parse_from_dict(self, dct, validate=True) -> SeedNode:
+        # seeds need the root_path because the contents are not loaded
+        dct["root_path"] = self.project.project_root
+        if "language" in dct:
+            del dct["language"]
+        # raw_code is not currently used, but it might be in the future
+        if validate:
+            SeedNode.validate(dct)
+        return SeedNode.from_dict(dct)
+
+    @property
+    def resource_type(self) -> NodeType:
+        return NodeType.Seed
+
+    @classmethod
+    def get_compiled_path(cls, block: FileBlock):
+        return block.path.relative_path
+
+    def render_with_context(self, parsed_node: SeedNode, config: ContextConfig) -> None:
+        """Seeds don't need to do any rendering."""

dvt/parser/singular_test.py

@@ -0,0 +1,20 @@
+from dvt.contracts.graph.nodes import SingularTestNode
+from dvt.node_types import NodeType
+from dvt.parser.base import SimpleSQLParser
+from dvt.parser.search import FileBlock
+from dvt.utils import get_pseudo_test_path
+
+
+class SingularTestParser(SimpleSQLParser[SingularTestNode]):
+    def parse_from_dict(self, dct, validate=True) -> SingularTestNode:
+        if validate:
+            SingularTestNode.validate(dct)
+        return SingularTestNode.from_dict(dct)
+
+    @property
+    def resource_type(self) -> NodeType:
+        return NodeType.Test
+
+    @classmethod
+    def get_compiled_path(cls, block: FileBlock):
+        return get_pseudo_test_path(block.name, block.path.relative_path)

dvt/parser/snapshots.py

@@ -0,0 +1,44 @@
+import os
+from typing import List
+
+from dvt.contracts.graph.nodes import SnapshotNode
+from dvt.node_types import NodeType
+from dvt.parser.base import SQLParser
+from dvt.parser.search import BlockContents, BlockSearcher, FileBlock
+from dvt.utils import split_path
+
+
+class SnapshotParser(SQLParser[SnapshotNode]):
+    def parse_from_dict(self, dct, validate=True) -> SnapshotNode:
+        if validate:
+            SnapshotNode.validate(dct)
+        return SnapshotNode.from_dict(dct)
+
+    @property
+    def resource_type(self) -> NodeType:
+        return NodeType.Snapshot
+
+    @classmethod
+    def get_compiled_path(cls, block: FileBlock):
+        return block.path.relative_path
+
+    def get_fqn(self, path: str, name: str) -> List[str]:
+        """Get the FQN for the node. This impacts node selection and config
+        application.
+
+        On snapshots, the fqn includes the filename.
+        """
+        no_ext = os.path.splitext(path)[0]
+        fqn = [self.project.project_name]
+        fqn.extend(split_path(no_ext))
+        fqn.append(name)
+        return fqn
+
+    def parse_file(self, file_block: FileBlock) -> None:
+        blocks = BlockSearcher(
+            source=[file_block],
+            allowed_blocks={"snapshot"},
+            source_tag_factory=BlockContents,
+        )
+        for block in blocks:
+            self.parse_node(block)