dvt-core 1.11.0b4__py3-none-any.whl

dvt/parser/common.py

@@ -0,0 +1,267 @@
+from dataclasses import dataclass
+from typing import Any, Dict, Generic, List, Optional, TypeVar, Union
+
+from dvt.artifacts.resources import ColumnConfig, ColumnInfo, NodeVersion
+from dvt.contracts.graph.nodes import UnpatchedSourceDefinition
+from dvt.contracts.graph.unparsed import (
+    HasColumnDocs,
+    HasColumnProps,
+    HasColumnTests,
+    UnparsedAnalysisUpdate,
+    UnparsedColumn,
+    UnparsedExposure,
+    UnparsedFunctionUpdate,
+    UnparsedMacroUpdate,
+    UnparsedModelUpdate,
+    UnparsedNodeUpdate,
+    UnparsedSingularTestUpdate,
+)
+from dvt.exceptions import ParsingError
+from dvt.node_types import NodeType
+from dvt.parser.search import FileBlock
+
+from dbt_common.contracts.constraints import ColumnLevelConstraint, ConstraintType
+from dbt_common.exceptions import DbtInternalError
+from dbt_semantic_interfaces.type_enums import TimeGranularity
+
+schema_file_keys_to_resource_types = {
+    "models": NodeType.Model,
+    "seeds": NodeType.Seed,
+    "snapshots": NodeType.Snapshot,
+    "sources": NodeType.Source,
+    "macros": NodeType.Macro,
+    "analyses": NodeType.Analysis,
+    "exposures": NodeType.Exposure,
+    "metrics": NodeType.Metric,
+    "semantic_models": NodeType.SemanticModel,
+    "saved_queries": NodeType.SavedQuery,
+    "functions": NodeType.Function,
+}
+
+resource_types_to_schema_file_keys = {
+    v: k for (k, v) in schema_file_keys_to_resource_types.items()
+}
+
+schema_file_keys = list(schema_file_keys_to_resource_types.keys())
+
+
+def trimmed(inp: str) -> str:
+    if len(inp) < 50:
+        return inp
+    return inp[:44] + "..." + inp[-3:]
+
+
+TestDef = Union[str, Dict[str, Any]]
+
+
+Target = TypeVar(
+    "Target",
+    UnparsedNodeUpdate,
+    UnparsedMacroUpdate,
+    UnparsedAnalysisUpdate,
+    UnpatchedSourceDefinition,
+    UnparsedExposure,
+    UnparsedModelUpdate,
+    UnparsedFunctionUpdate,
+    UnparsedSingularTestUpdate,
+)
+
+
+ColumnTarget = TypeVar(
+    "ColumnTarget",
+    UnparsedModelUpdate,
+    UnparsedNodeUpdate,
+    UnparsedAnalysisUpdate,
+    UnpatchedSourceDefinition,
+)
+
+Versioned = TypeVar("Versioned", bound=UnparsedModelUpdate)
+
+Testable = TypeVar("Testable", UnparsedNodeUpdate, UnpatchedSourceDefinition, UnparsedModelUpdate)
+
+
+@dataclass
+class YamlBlock(FileBlock):
+    data: Dict[str, Any]
+
+    @classmethod
+    def from_file_block(cls, src: FileBlock, data: Dict[str, Any]):
+        return cls(
+            file=src.file,
+            data=data,
+        )
+
+
+@dataclass
+class TargetBlock(YamlBlock, Generic[Target]):
+    target: Target
+
+    @property
+    def name(self):
+        return self.target.name
+
+    @property
+    def columns(self):
+        return []
+
+    @property
+    def data_tests(self) -> List[TestDef]:
+        return []
+
+    @property
+    def tests(self) -> List[TestDef]:
+        return []
+
+    @classmethod
+    def from_yaml_block(cls, src: YamlBlock, target: Target) -> "TargetBlock[Target]":
+        return cls(
+            file=src.file,
+            data=src.data,
+            target=target,
+        )
+
+
+@dataclass
+class TargetColumnsBlock(TargetBlock[ColumnTarget], Generic[ColumnTarget]):
+    @property
+    def columns(self):
+        if self.target.columns is None:
+            return []
+        else:
+            return self.target.columns
+
+
+@dataclass
+class TestBlock(TargetColumnsBlock[Testable], Generic[Testable]):
+    @property
+    def data_tests(self) -> List[TestDef]:
+        if self.target.data_tests is None:
+            return []
+        else:
+            return self.target.data_tests
+
+    @property
+    def quote_columns(self) -> Optional[bool]:
+        return self.target.quote_columns
+
+    @classmethod
+    def from_yaml_block(cls, src: YamlBlock, target: Testable) -> "TestBlock[Testable]":
+        return cls(
+            file=src.file,
+            data=src.data,
+            target=target,
+        )
+
+
+@dataclass
+class VersionedTestBlock(TestBlock, Generic[Versioned]):
+    @property
+    def columns(self):
+        if not self.target.versions:
+            return super().columns
+        else:
+            raise DbtInternalError(".columns for VersionedTestBlock with versions")
+
+    @property
+    def data_tests(self) -> List[TestDef]:
+        if not self.target.versions:
+            return super().data_tests
+        else:
+            raise DbtInternalError(".data_tests for VersionedTestBlock with versions")
+
+    @classmethod
+    def from_yaml_block(cls, src: YamlBlock, target: Versioned) -> "VersionedTestBlock[Versioned]":
+        return cls(
+            file=src.file,
+            data=src.data,
+            target=target,
+        )
+
+
+@dataclass
+class GenericTestBlock(TestBlock[Testable], Generic[Testable]):
+    data_test: Dict[str, Any]
+    column_name: Optional[str]
+    tags: List[str]
+    version: Optional[NodeVersion]
+
+    @classmethod
+    def from_test_block(
+        cls,
+        src: TestBlock,
+        data_test: Dict[str, Any],
+        column_name: Optional[str],
+        tags: List[str],
+        version: Optional[NodeVersion],
+    ) -> "GenericTestBlock":
+        return cls(
+            file=src.file,
+            data=src.data,
+            target=src.target,
+            data_test=data_test,
+            column_name=column_name,
+            tags=tags,
+            version=version,
+        )
+
+
+class ParserRef:
+    """A helper object to hold parse-time references."""
+
+    def __init__(self) -> None:
+        self.column_info: Dict[str, ColumnInfo] = {}
+
+    def _add(self, column: HasColumnProps) -> None:
+        tags: List[str] = getattr(column, "tags", [])
+        quote: Optional[bool] = None
+        granularity: Optional[TimeGranularity] = None
+        if isinstance(column, UnparsedColumn):
+            quote = column.quote
+            granularity = TimeGranularity(column.granularity) if column.granularity else None
+
+        if any(
+            c
+            for c in column.constraints
+            if "type" not in c or not ConstraintType.is_valid(c["type"])
+        ):
+            raise ParsingError(f"Invalid constraint type on column {column.name}")
+
+        # Merge meta and tags from column and config
+        column_config_meta = (
+            column.config["meta"] if isinstance(column.config.get("meta"), dict) else {}
+        )
+        column_config_tags = []
+        if "tags" in column.config:
+            if isinstance(column.config["tags"], list):
+                column_config_tags = column.config["tags"]
+            elif isinstance(column.config["tags"], str):
+                column_config_tags = [column.config["tags"]]
+
+        column_meta = {**column.meta, **column_config_meta}
+        column_tags = list(set(tags + column_config_tags))
+        self.column_info[column.name] = ColumnInfo(
+            name=column.name,
+            description=column.description,
+            data_type=column.data_type,
+            constraints=[ColumnLevelConstraint.from_dict(c) for c in column.constraints],
+            meta=column_meta,
+            tags=column_tags,
+            quote=quote,
+            _extra=column.extra,
+            granularity=granularity,
+            config=ColumnConfig(meta=column_meta, tags=column_tags),
+        )
+
+    @classmethod
+    def from_target(cls, target: Union[HasColumnDocs, HasColumnTests]) -> "ParserRef":
+        refs = cls()
+        for column in target.columns:
+            refs._add(column)
+        return refs
+
+    @classmethod
+    def from_versioned_target(cls, target: Versioned, version: NodeVersion) -> "ParserRef":
+        refs = cls()
+        for base_column in target.get_columns_for_version(version):
+            refs._add(base_column)
+        return refs

dvt/parser/docs.py

@@ -0,0 +1,52 @@
+import re
+from typing import Iterable, Optional
+
+from dvt.clients.jinja import get_rendered
+from dvt.contracts.files import SourceFile
+from dvt.contracts.graph.nodes import Documentation
+from dvt.node_types import NodeType
+from dvt.parser.base import Parser
+from dvt.parser.search import BlockContents, BlockSearcher, FileBlock
+
+SHOULD_PARSE_RE = re.compile(r"{[{%]")
+
+
+class DocumentationParser(Parser[Documentation]):
+    @property
+    def resource_type(self) -> NodeType:
+        return NodeType.Documentation
+
+    @classmethod
+    def get_compiled_path(cls, block: FileBlock):
+        return block.path.relative_path
+
+    def generate_unique_id(self, resource_name: str, _: Optional[str] = None) -> str:
+        # For consistency, use the same format for doc unique_ids
+        return f"doc.{self.project.project_name}.{resource_name}"
+
+    def parse_block(self, block: BlockContents) -> Iterable[Documentation]:
+        unique_id = self.generate_unique_id(block.name)
+        contents = get_rendered(block.contents, {}).strip()
+
+        doc = Documentation(
+            path=block.file.path.relative_path,
+            original_file_path=block.path.original_file_path,
+            package_name=self.project.project_name,
+            unique_id=unique_id,
+            name=block.name,
+            block_contents=contents,
+            resource_type=NodeType.Documentation,
+        )
+        return [doc]
+
+    def parse_file(self, file_block: FileBlock):
+        assert isinstance(file_block.file, SourceFile)
+        searcher: Iterable[BlockContents] = BlockSearcher(
+            source=[file_block],
+            allowed_blocks={"docs"},
+            source_tag_factory=BlockContents,
+            check_jinja=False,
+        )
+        for block in searcher:
+            for parsed in self.parse_block(block):
+                self.manifest.add_doc(file_block.file, parsed)

dvt/parser/fixtures.py

@@ -0,0 +1,51 @@
+import csv
+from io import StringIO
+from typing import Any, Dict, List, Optional
+
+from dvt.contracts.files import FixtureSourceFile
+from dvt.contracts.graph.nodes import UnitTestFileFixture
+from dvt.node_types import NodeType
+from dvt.parser.base import Parser
+from dvt.parser.search import FileBlock
+
+
+class FixtureParser(Parser[UnitTestFileFixture]):
+    @property
+    def resource_type(self) -> NodeType:
+        return NodeType.Fixture
+
+    @classmethod
+    def get_compiled_path(cls, block: FileBlock):
+        # Is this necessary?
+        return block.path.relative_path
+
+    def generate_unique_id(self, resource_name: str, _: Optional[str] = None) -> str:
+        return f"fixture.{self.project.project_name}.{resource_name}"
+
+    def parse_file(self, file_block: FileBlock):
+        assert isinstance(file_block.file, FixtureSourceFile)
+        unique_id = self.generate_unique_id(file_block.name)
+
+        if file_block.file.path.relative_path.endswith(".sql"):
+            rows = file_block.file.contents  # type: ignore
+        else:  # endswith('.csv')
+            rows = self.get_rows(file_block.file.contents)  # type: ignore
+
+        fixture = UnitTestFileFixture(
+            name=file_block.name,
+            path=file_block.file.path.relative_path,
+            original_file_path=file_block.path.original_file_path,
+            package_name=self.project.project_name,
+            unique_id=unique_id,
+            resource_type=NodeType.Fixture,
+            rows=rows,
+        )
+        self.manifest.add_fixture(file_block.file, fixture)
+
+    def get_rows(self, contents) -> List[Dict[str, Any]]:
+        rows = []
+        dummy_file = StringIO(contents)
+        reader = csv.DictReader(dummy_file)
+        for row in reader:
+            rows.append(row)
+        return rows

dvt/parser/functions.py

@@ -0,0 +1,30 @@
+from dvt.artifacts.resources.types import NodeType
+from dvt.contracts.graph.nodes import FunctionNode, ManifestNode
+from dvt.parser.base import SimpleParser
+from dvt.parser.search import FileBlock
+
+
+class FunctionParser(SimpleParser[FileBlock, FunctionNode]):
+    def parse_from_dict(self, dct, validate=True) -> FunctionNode:
+        if validate:
+            FunctionNode.validate(dct)
+        return FunctionNode.from_dict(dct)
+
+    @property
+    def resource_type(self) -> NodeType:
+        return NodeType.Function
+
+    @classmethod
+    def get_compiled_path(cls, block: FileBlock):
+        return block.path.relative_path
+
+    # overrides SimpleSQLParser.add_result_node
+    def add_result_node(self, block: FileBlock, node: ManifestNode):
+        assert isinstance(node, FunctionNode), "Got non FunctionNode in FunctionParser"
+        if node.config.enabled:
+            self.manifest.add_function(node)
+        else:
+            self.manifest.add_disabled(block.file, node)
+
+    def parse_file(self, file_block: FileBlock) -> None:
+        self.parse_node(file_block)

dvt/parser/generic_test.py

@@ -0,0 +1,100 @@
+from typing import Iterable, List
+
+import jinja2
+from dvt.contracts.files import SourceFile
+from dvt.contracts.graph.nodes import GenericTestNode, Macro
+from dvt.contracts.graph.unparsed import UnparsedMacro
+from dvt.exceptions import ParsingError
+from dvt.node_types import NodeType
+from dvt.parser.base import BaseParser
+from dvt.parser.search import FileBlock
+
+from dbt_common.clients import jinja
+from dbt_common.utils import MACRO_PREFIX
+
+
+class GenericTestParser(BaseParser[GenericTestNode]):
+    @property
+    def resource_type(self) -> NodeType:
+        return NodeType.Macro
+
+    @classmethod
+    def get_compiled_path(cls, block: FileBlock):
+        return block.path.relative_path
+
+    def create_generic_test_macro(
+        self, block: jinja.BlockTag, base_node: UnparsedMacro, name: str
+    ) -> Macro:
+        unique_id = self.generate_unique_id(name)
+        macro_sql = block.full_block or ""
+
+        return Macro(
+            path=base_node.path,
+            macro_sql=macro_sql,
+            original_file_path=base_node.original_file_path,
+            package_name=base_node.package_name,
+            resource_type=base_node.resource_type,
+            name=name,
+            unique_id=unique_id,
+        )
+
+    def parse_unparsed_generic_test(self, base_node: UnparsedMacro) -> Iterable[Macro]:
+        try:
+            blocks: List[jinja.BlockTag] = [
+                t
+                for t in jinja.extract_toplevel_blocks(
+                    base_node.raw_code,
+                    allowed_blocks={"test", "data_test"},
+                    collect_raw_data=False,
+                )
+                if isinstance(t, jinja.BlockTag)
+            ]
+        except ParsingError as exc:
+            exc.add_node(base_node)
+            raise
+
+        for block in blocks:
+            try:
+                ast = jinja.parse(block.full_block)
+            except ParsingError as e:
+                e.add_node(base_node)
+                raise
+
+            # generic tests are structured as macros so we want to count the number of macro blocks
+            generic_test_nodes = list(ast.find_all(jinja2.nodes.Macro))
+
+            if len(generic_test_nodes) != 1:
+                # things have gone disastrously wrong, we thought we only
+                # parsed one block!
+                raise ParsingError(
+                    f"Found multiple generic tests in {block.full_block}, expected 1",
+                    node=base_node,
+                )
+
+            generic_test_name = generic_test_nodes[0].name
+
+            if not generic_test_name.startswith(MACRO_PREFIX):
+                continue
+
+            name: str = generic_test_name.replace(MACRO_PREFIX, "")
+            node = self.create_generic_test_macro(block, base_node, name)
+            yield node
+
+    def parse_file(self, block: FileBlock):
+        assert isinstance(block.file, SourceFile)
+        source_file = block.file
+        assert isinstance(source_file.contents, str)
+        original_file_path = source_file.path.original_file_path
+
+        # this is really only used for error messages
+        base_node = UnparsedMacro(
+            path=original_file_path,
+            original_file_path=original_file_path,
+            package_name=self.project.project_name,
+            raw_code=source_file.contents,
+            resource_type=NodeType.Macro,
+            language="sql",
+        )
+
+        for node in self.parse_unparsed_generic_test(base_node):
+            self.manifest.add_macro(block.file, node)