mario-pipeline-tools 0.1__tar.gz

mario-pipeline-tools-0.1/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 2.1
+Name: mario-pipeline-tools
+Version: 0.1
+Summary: Base classes and helpers for common data pipeline tasks
+Home-page: https://github.com/JiscDACT/mario
+Author: scottbw
+Author-email: scott.wilson@jisc.ac.uk
+License: all rights reserved
+
+A set of base classes and helpers for common data pipeline tasks, whether using in local control scripts or in Airflow tasks.

mario-pipeline-tools-0.1/README.md

@@ -0,0 +1,10 @@
+# General purpose data pipelines 
+
+A set of base classes and helpers for 
+common data pipeline tasks, whether using
+in local control scripts or in Airflow tasks.
+
+## !WARNING!
+This is a public repository for ease of deployment.
+Be especially careful not to commit anything
+that shouldn't be here.

mario-pipeline-tools-0.1/mario/data_extractor.py

@@ -0,0 +1,235 @@
+import logging
+import shutil
+
+import pandas as pd
+import pantab
+from pandas import DataFrame
+from tableauhyperapi import TableName
+
+from mario.dataset_specification import DatasetSpecification
+from mario.metadata import Metadata
+
+logger = logging.getLogger(__name__)
+
+
+class Configuration:
+    """
+    Configuration for a data extractor. This can include:
+    * A connection string: an ODBC connection string for SQL extraction
+    * A view name: A view as the base for the SQL code (H+, TDSA only)
+    * A hook: An object or function used to access data, e.g. S3
+    * A file path: A data file to load as the source of data, e.g. data.csv
+    """
+
+    def __init__(self,
+                 connection_string: str = None,
+                 hook=None,
+                 view: str = None,
+                 file_path: str = None
+                 ):
+        self.connection_string = connection_string
+        self.hook = hook
+        self.view = view
+        self.file_path = file_path
+
+
+class DataExtractor:
+
+    def __init__(self,
+                 configuration: Configuration,
+                 dataset_specification: DatasetSpecification,
+                 metadata: Metadata):
+        self.configuration = configuration
+        self.dataset_specification = dataset_specification
+        self.metadata = metadata
+        self._data = pd.DataFrame()
+        if configuration.file_path is not None:
+            if configuration.file_path.endswith('.csv'):
+                self.__load_from_csv__()
+            elif configuration.file_path.endswith('.hyper'):
+                self.__load_from_hyper__()
+            else:
+                raise ValueError("Unsupported file type")
+        elif configuration.hook is not None:
+            # TODO something here for Airflow hooks
+            pass
+        else:
+            self.__load_from_sql__()
+
+    def __load_from_sql__(self):
+        query = self.__build_query__()
+        # TODO connect using ODBC connection string
+        # TODO run query using pyodbc
+        logger.info("Executing query: " + query)
+
+    def __build_query__(self):
+        query = ''
+        if self.configuration.view is not None:
+            query = 'SELECT * FROM ' + self.configuration.view + ' WITH(NOLOCK)'
+        else:
+            # TODO: Automation 2.0 or TDSA query builder
+            pass
+        return query
+
+    def __load_from_csv__(self):
+        self._data = pd.read_csv(self.configuration.file_path)
+
+    def __load_from_hyper__(self):
+        from tableau_builder import hyper_utils
+        table_parts = hyper_utils.get_default_table_and_schema(hyper_path=self.configuration.file_path)
+        table = TableName(table_parts['schema'], table_parts['table'])
+        self._data = pantab.frame_from_hyper(
+            source=self.configuration.file_path,
+            table=table
+        )
+
+    def __minimise_data__(self):
+        """ Minimise data so we only keep the columns in the spec """
+        columns_to_keep = []
+        for item in self.dataset_specification.items:
+            if not self.metadata.get_metadata(item).get_property('formula'):
+                columns_to_keep.append(item)
+        self._data = self._data[columns_to_keep]
+
+    def validate_data(self):
+        validation_errors = []
+        for item in self.dataset_specification.items:
+            metadata = self.metadata.get_metadata(item)
+            # Ignore calculated fields
+            if metadata.get_property('formula') is None:
+                # Check all columns present
+                if item not in self._data.columns:
+                    validation_errors.append("Item missing: " + item)
+                else:
+                    # Check domain
+                    if metadata.get_property('domain') is not None:
+                        data_domain = self._data[item].unique()
+                        metadata_domain = metadata.get_property('domain')
+                        for element in data_domain:
+                            if element not in metadata_domain:
+                                validation_errors.append("Domain validation failed for " + item)
+                                logger.error("Validation error: '" + str(element) + "' is not in domain of " + item)
+                    # Check range
+                    if metadata.get_property('range') is not None:
+                        min_value = metadata.get_property('range')[0]
+                        max_value = metadata.get_property('range')[1]
+                        data_min = self._data[item].min()
+                        data_max = self._data[item].max()
+                        if data_min < min_value:
+                            validation_errors.append("Range validation failed for " + item)
+                            logger.error("Validation error: '" + str(data_min) + "' is less than " + str(min_value))
+                        if data_max > max_value:
+                            validation_errors.append("Range validation failed for " + item)
+                            logger.error("Validation error: '" + str(data_max) + "' is greater than " + str(max_value))
+
+        if len(validation_errors) != 0:
+            for validation_error in validation_errors:
+                logger.error(validation_error)
+            raise ValueError("Data validation failed - check the logs for details")
+        return True
+
+    def get_data_frame(self) -> DataFrame:
+        return self._data
+
+    def save_data_as_excel(self, file_path: str, minimise=True):
+        if minimise:
+            self.__minimise_data__()
+        # TODO use OpenPyxl here, or drop this method
+        self._data.to_csv(file_path)
+        pass
+
+    def save_data_as_csv(self, file_path: str, minimise=True):
+        if minimise:
+            self.__minimise_data__()
+        self._data.to_csv(file_path)
+
+    def save_data_as_hyper(self, file_path: str, table: str = 'Extract', schema: str = 'Extract', minimise=True):
+        table_name = TableName(schema, table)
+        if minimise:
+            self.__minimise_data__()
+        pantab.frame_to_hyper(df=self._data, database=file_path, table=table_name)
+
+
+class HyperFile(DataExtractor):
+    """
+    Wrapper for a HyperFile as a data extractor - use when no data needs to be extracted,
+    and we just want to treat a hyper as a hyper with no conversion to/from dataframe
+    """
+
+    def __init__(self,
+                 configuration: Configuration,
+                 dataset_specification: DatasetSpecification,
+                 metadata: Metadata):
+        super().__init__(configuration, dataset_specification, metadata)
+
+    def validate_data(self):
+        from tableau_builder.hyper_utils import get_default_table_and_schema, check_column_exists, check_type, \
+            check_domain, check_range
+
+        validation_errors = []
+
+        table_schema = get_default_table_and_schema(self.configuration.file_path)
+        table = table_schema['table']
+        schema = table_schema['schema']
+
+        for item in self.dataset_specification.items:
+            metadata = self.metadata.get_metadata(item)
+            if metadata.get_property('formula') is None:
+                if not check_column_exists(
+                        hyper_path=self.configuration.file_path,
+                        column_name=item,
+                        table_name=table,
+                        schema_name=schema
+                ):
+                    validation_errors.append("Item missing: " + item)
+
+                if metadata.get_property('datatype') and not check_type(
+                        hyper_path=self.configuration.file_path,
+                        column_name=item,
+                        expected_type=metadata.get_property('datatype'),
+                        table_name=table,
+                        schema_name=schema
+                ):
+                    validation_errors.append("Item is wrong type: " + item)
+
+            if metadata.get_property('domain') is not None:
+                if not check_domain(
+                        hyper_path=self.configuration.file_path,
+                        field=item,
+                        domain=metadata.get_property('domain'),
+                        table_name=table,
+                        schema_name=schema
+                ):
+                    validation_errors.append("Domain validation failed for " + item)
+            if metadata.get_property('range') is not None:
+                if not check_range(
+                        hyper_path=self.configuration.file_path,
+                        field=item,
+                        min_value=metadata.get_property('range')[0],
+                        max_value=metadata.get_property('range')[1],
+                        table_name=table,
+                        schema_name=schema
+                ):
+                    validation_errors.append("Range validation failed for " + item)
+
+        if len(validation_errors) != 0:
+            for validation_error in validation_errors:
+                logger.error(validation_error)
+            raise ValueError("Domain validation failed - check the logs for details")
+        return True
+
+    def __minimise_data__(self):
+        from tableau_builder import hyper_utils
+        columns_to_keep = self.dataset_specification.items
+        table = hyper_utils.get_default_table_and_schema(self.configuration.file_path)
+        hyper_utils.subset_columns(
+            columns_to_keep=columns_to_keep,
+            hyper_path=self.configuration.file_path,
+            schema_name=table['schema'],
+            table_name=table['table']
+        )
+
+    def save_data_as_hyper(self, file_path: str, table: str = 'Extract', schema: str = 'Extract', minimise=True):
+        if minimise:
+            self.__minimise_data__()
+        shutil.copyfile(self.configuration.file_path, file_path)

mario-pipeline-tools-0.1/mario/dataset_builder.py

@@ -0,0 +1,99 @@
+import logging
+import os.path
+import shutil
+import tempfile
+from enum import Enum
+
+from tableau_builder.hyper_utils import get_default_table_and_schema
+from tableau_builder.json_metadata import JsonRepository
+
+from mario.data_extractor import DataExtractor
+from mario.dataset_specification import DatasetSpecification
+from mario.metadata import Metadata
+
+logger = logging.getLogger(__name__)
+
+
+class Format(Enum):
+    TABLEAU_PACKAGED_DATASOURCE = 'tdsx'
+    POWER_BI_PACKAGE = 'pbix'
+    EXCEL_PIVOT = 'xlsx'
+    CSV = 'csv'
+
+
+class DatasetBuilder:
+
+    def __init__(self,
+                 dataset_specification: DatasetSpecification = None,
+                 metadata: Metadata = None,
+                 data: DataExtractor = None
+                 ):
+        self.dataset_specification = dataset_specification
+        self.metadata = metadata
+        self.data = data
+
+    def validate_metadata(self):
+        """ Checks whether the metadata matches the specification"""
+        not_included = []
+        fields = self.dataset_specification.items
+
+        for field in fields:
+            try:
+                self.metadata.get_metadata(field)
+            except ValueError:
+                not_included.append(field)
+
+        if len(not_included) > 0:
+            logger.error(self.dataset_specification.name + ": Some metadata is missing")
+            logger.error(not_included)
+            return False
+        else:
+            logger.info(self.dataset_specification.name + ": All metadata present")
+            return True
+
+    def build(self, output_format: Format, file_path: str):
+        if output_format == Format.TABLEAU_PACKAGED_DATASOURCE:
+            self.__build_tdsx__(file_path)
+
+    def __build_excel_pivot__(self, file_path: str):
+        # TODO export Excel and Info sheet using OpenPyxl - see code in Automation2.0 and TDSA.
+        pass
+
+    def __build_csv__(self, file_path: str):
+        # TODO export Info sheet as well - see code in Automation2.0 and TDSA.
+        self.data.save_data_as_csv(file_path=file_path)
+
+    def __build_tdsx__(self, file_path: str):
+        with tempfile.TemporaryDirectory() as temp_folder:
+
+            # Metadata - convert to internal format used for Tableau
+            metadata_file_path = os.path.join(temp_folder, 'metadata.json')
+            self.metadata.save(metadata_file_path)
+            repository = JsonRepository(metadata_file_path)
+
+            # Move the hyper extract
+            data_file_path = os.path.join(temp_folder, 'data.hyper')
+            self.data.save_data_as_hyper(file_path=data_file_path)
+
+            # Get the table and schema name from the extract
+            table_schema = get_default_table_and_schema(data_file_path)
+
+            # Save the spec
+            dataset_file_path = os.path.join(temp_folder, 'dataset.json')
+            self.dataset_specification.save(dataset_file_path)
+
+            # Build a tdsx in a temp file
+            output_path = os.path.join(temp_folder, 'build')
+            from tableau_builder.dataset import create_tdsx, HYPER
+            create_tdsx(
+                dataset_file=dataset_file_path,
+                metadata_repository=repository,
+                data_file=data_file_path,
+                output_file=output_path,
+                data_source_type=HYPER,
+                table_name=table_schema['table'],
+                schema_name=table_schema['schema'],
+                use_metadata_groups=True
+            )
+            shutil.copyfile(src=output_path + '.tdsx', dst=file_path)
+

mario-pipeline-tools-0.1/mario/dataset_specification.py

@@ -0,0 +1,87 @@
+import json
+from typing import List
+
+
+class DatasetSpecification:
+    """
+    Base class/interface for dataset specifications. Use an implementation specific to the
+    format of specification used.
+    """
+
+    def __init__(self):
+        self._name = ''
+        self._collection: str = ''
+        self._measures: List[str] = []
+        self._dimensions: List[str] = []
+        self._properties = []
+
+    @property
+    def name(self):
+        return self._name
+
+    @name.setter
+    def name(self, value):
+        self._name = value
+
+    @property
+    def collection(self):
+        return self._collection
+
+    @collection.setter
+    def collection(self, value):
+        self._collection = value
+
+    @property
+    def enquiry(self):
+        return ''
+
+    @property
+    def measures(self):
+        return self._measures
+
+    @measures.setter
+    def measures(self, value):
+        self._measures = value
+
+    @property
+    def dimensions(self):
+        return self._dimensions
+
+    @dimensions.setter
+    def dimensions(self, value):
+        self._dimensions = value
+
+    @property
+    def items(self):
+        return self.dimensions + self.measures
+
+    def set_property(self, name, value):
+        self._properties[name] = value
+
+    def get_property(self, name):
+        if name in self._properties:
+            return self._properties[name]
+        return None
+
+    def save(self, file_path: str):
+        json_representation = {
+            "name": self.name,
+            "collection": self.collection,
+            "dimensions": self.dimensions,
+            "measures": self.measures
+        }
+        with open(file_path, 'w') as file:
+            json.dump(json_representation, file, default=vars)
+
+
+def dataset_from_json(file_path: str = None) -> DatasetSpecification:
+    with open(file_path, mode='r') as source_file:
+        spec = json.load(source_file)
+
+    dataset_specification = DatasetSpecification()
+    dataset_specification.name = spec['name']
+    dataset_specification.collection = spec['collection']
+    dataset_specification.measures = spec['measures']
+    dataset_specification.dimensions = spec['dimensions']
+
+    return dataset_specification

mario-pipeline-tools-0.1/mario/example.py

@@ -0,0 +1,28 @@
+import os
+
+from mario.data_extractor import DataExtractor, Configuration
+from mario.dataset_specification import dataset_from_json
+from mario.dataset_builder import DatasetBuilder, Format
+from mario.metadata import metadata_from_json
+
+# Build a TD extract in Excel
+dataset = dataset_from_json("data_spec.xlsx")
+metadata = metadata_from_json('metadata.json')
+configuration = Configuration(connection_string=os.environ.get('odbc_connection_string'))
+extractor = DataExtractor(configuration=configuration, dataset_specification=dataset, metadata=metadata)
+# Either put automation code in here, or override the base class
+builder = DatasetBuilder(dataset_specification=dataset, metadata=metadata, data=extractor)
+path = os.path.join(dataset.enquiry, dataset.name + '.xlsx')
+builder.build(file_path=path, output_format=Format.EXCEL_PIVOT)
+
+# Build a Heidi Plus Datasource in TDSX
+dataset = dataset_from_json("dataset.json")
+metadata = metadata_from_json('metadata.json')
+configuration = Configuration(
+    connection_string=os.environ.get('odbc_connection_string'),
+    view='dbo.v_heidi_plus_student_fpe'
+)
+extractor = DataExtractor(configuration=configuration, dataset_specification=dataset, metadata=metadata)
+builder = DatasetBuilder(dataset_specification=dataset, metadata=metadata, data=extractor)
+path = os.path.join('datasources', dataset.collection, dataset.name + '.tdsx')
+builder.build(file_path=path, output_format=Format.TABLEAU_PACKAGED_DATASOURCE)

mario-pipeline-tools-0.1/mario/metadata.py

@@ -0,0 +1,113 @@
+import json
+from typing import List
+
+
+class Item:
+
+    def __init__(self):
+        self._name = ''
+        self._description = ''
+        self._properties = {}
+
+    @property
+    def name(self):
+        return self._name
+
+    @name.setter
+    def name(self, value):
+        self._name = value
+
+    @property
+    def description(self):
+        return self._description
+
+    @description.setter
+    def description(self, value):
+        self._description = value
+
+    def set_property(self, name, value):
+        self._properties[name] = value
+
+    def get_property(self, name):
+        if name in self._properties:
+            return self._properties[name]
+        return None
+
+    def to_json(self):
+        json_representation = {
+            "name": self.name,
+            "description": self.description
+        }
+        for key, value in self._properties.items():
+            json_representation[key] = value
+        return json_representation
+
+
+class Metadata(Item):
+
+    def __init__(self, name: str = None):
+        super().__init__()
+        self._items: List[Item] = []
+        self.name = name
+        if self.name is None:
+            self.name = 'Metadata'
+
+    def get_metadata(self, name: str):
+        for item in self._items:
+            if item.name == name:
+                return item
+        return None
+
+    @property
+    def items(self):
+        return self._items
+
+    def add_item(self, item: Item) -> None:
+        self._items.append(item)
+
+    def merge_items(self, metadata) -> None:
+        for item in metadata.items:
+            self.add_item(item)
+
+    def save(self, file_path: str = None) -> None:
+        json_representation = {
+            "collection": {
+                "name": self.name,
+                "items": []
+            }
+        }
+        for item in self._items:
+            json_representation['collection']['items'].append(item.to_json())
+
+        with open(file_path, mode='w') as file:
+            json.dump(json_representation, file, default=vars)
+
+
+def metadata_from_json(file_path: str = None):
+    """ Factory method for creating a Metadata instance from a JSON file"""
+    metadata = Metadata()
+
+    with open(file_path) as metadata_file:
+        metadata_json = json.load(metadata_file)
+
+    if 'collection' in metadata_json:
+        collection = 'collection'
+        items = 'items'
+        name = 'name'
+    else:
+        collection = 'datasource'
+        items = 'fields'
+        name = 'fieldName'
+
+    metadata.name = metadata_json[collection]['name']
+
+    for item in metadata_json[collection][items]:
+        metadata_item = Item()
+        metadata_item.name = item[name]
+        metadata_item.description = item['description']
+        for prop in item:
+            if prop not in [name, 'description']:
+                metadata_item.set_property(prop, item[prop])
+        metadata.add_item(metadata_item)
+
+    return metadata

mario-pipeline-tools-0.1/mario/utils.py

@@ -0,0 +1,9 @@
+import os
+from datetime import datetime
+
+
+def append_current_date_to_file_name(file_name: str) -> str:
+    date_time = datetime.now().strftime("%Y_%m_%d")
+    filename = os.path.splitext(os.path.basename(file_name))[0]
+    extension = os.path.splitext(os.path.basename(file_name))[1]
+    return filename + '_' + date_time + extension

mario-pipeline-tools-0.1/mario_pipeline_tools.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 2.1
+Name: mario-pipeline-tools
+Version: 0.1
+Summary: Base classes and helpers for common data pipeline tasks
+Home-page: https://github.com/JiscDACT/mario
+Author: scottbw
+Author-email: scott.wilson@jisc.ac.uk
+License: all rights reserved
+
+A set of base classes and helpers for common data pipeline tasks, whether using in local control scripts or in Airflow tasks.

mario-pipeline-tools-0.1/mario_pipeline_tools.egg-info/SOURCES.txt

@@ -0,0 +1,18 @@
+README.md
+setup.py
+mario/__init__.py
+mario/data_extractor.py
+mario/dataset_builder.py
+mario/dataset_specification.py
+mario/example.py
+mario/metadata.py
+mario/utils.py
+mario_pipeline_tools.egg-info/PKG-INFO
+mario_pipeline_tools.egg-info/SOURCES.txt
+mario_pipeline_tools.egg-info/dependency_links.txt
+mario_pipeline_tools.egg-info/requires.txt
+mario_pipeline_tools.egg-info/top_level.txt
+test/test_data_extractor.py
+test/test_dataset_specification.py
+test/test_integrated.py
+test/test_metadata.py

mario-pipeline-tools-0.1/mario_pipeline_tools.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

mario-pipeline-tools-0.1/mario_pipeline_tools.egg-info/requires.txt

@@ -0,0 +1,4 @@
+pandas
+tableauhyperapi
+pantab
+tableau-builder==0.18

mario-pipeline-tools-0.1/mario_pipeline_tools.egg-info/top_level.txt

@@ -0,0 +1 @@
+mario

mario-pipeline-tools-0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

mario-pipeline-tools-0.1/setup.py

@@ -0,0 +1,20 @@
+from setuptools import setup
+
+setup(
+    name='mario-pipeline-tools',
+    version='0.1',
+    packages=['mario'],
+    url='https://github.com/JiscDACT/mario',
+    license='all rights reserved',
+    author='scottbw',
+    author_email='scott.wilson@jisc.ac.uk',
+    description='Base classes and helpers for common data pipeline tasks',
+    long_description='A set of base classes and helpers for common data pipeline tasks, '
+                     'whether using in local control scripts or in Airflow tasks.',
+    package_data={
+        # If any package contains *.ini files, include them
+        '': ['*.ini']
+    },
+    include_package_data=True,
+    install_requires=['pandas', 'tableauhyperapi', 'pantab', 'tableau-builder==0.18']
+)

mario-pipeline-tools-0.1/test/test_data_extractor.py

@@ -0,0 +1,53 @@
+import os
+import tempfile
+
+from mario.data_extractor import DataExtractor, Configuration
+from mario.dataset_specification import dataset_from_json
+from mario.metadata import metadata_from_json
+
+
+def test_csv_to_csv():
+    dataset = dataset_from_json(os.path.join('test', 'dataset.json'))
+    metadata = metadata_from_json(os.path.join('test', 'metadata.json'))
+    configuration = Configuration(
+        file_path=os.path.join('test', 'orders.csv')
+    )
+    extractor = DataExtractor(
+        dataset_specification=dataset,
+        metadata=metadata,
+        configuration=configuration
+    )
+    extractor.validate_data()
+    with tempfile.NamedTemporaryFile(suffix='.csv') as file:
+        extractor.save_data_as_csv(file_path=file.name)
+
+
+def test_csv_to_hyper():
+    dataset = dataset_from_json(os.path.join('test', 'dataset.json'))
+    metadata = metadata_from_json(os.path.join('test', 'metadata.json'))
+    configuration = Configuration(
+        file_path=os.path.join('test', 'orders.csv')
+    )
+    extractor = DataExtractor(
+        dataset_specification=dataset,
+        metadata=metadata,
+        configuration=configuration
+    )
+    with tempfile.NamedTemporaryFile(suffix='.hyper') as file:
+        extractor.save_data_as_hyper(file_path=file.name)
+
+
+def test_hyper_to_csv():
+    dataset = dataset_from_json(os.path.join('test', 'dataset.json'))
+    metadata = metadata_from_json(os.path.join('test', 'metadata.json'))
+    configuration = Configuration(
+        file_path=os.path.join('test', 'orders.hyper')
+    )
+    extractor = DataExtractor(
+        dataset_specification=dataset,
+        metadata=metadata,
+        configuration=configuration
+    )
+    extractor.validate_data()
+    with tempfile.NamedTemporaryFile(suffix='.csv') as file:
+        extractor.save_data_as_csv(file_path=file.name)

mario-pipeline-tools-0.1/test/test_dataset_specification.py

@@ -0,0 +1,11 @@
+import os
+
+from mario.dataset_specification import dataset_from_json
+
+
+def test_import_dataset():
+    specification_file = os.path.join('test', 'dataset.json')
+    specification = dataset_from_json(file_path=specification_file)
+
+    assert specification.collection == 'superstore'
+    assert len(specification.measures) == 4

mario-pipeline-tools-0.1/test/test_integrated.py

@@ -0,0 +1,27 @@
+import os
+
+from mario.data_extractor import Configuration, HyperFile
+from mario.dataset_builder import DatasetBuilder, Format
+from mario.dataset_specification import dataset_from_json
+from mario.metadata import metadata_from_json
+
+
+def test_integration_tdsx():
+    dataset = dataset_from_json(os.path.join('test', 'dataset.json'))
+    metadata = metadata_from_json(os.path.join('test', 'metadata.json'))
+    configuration = Configuration(file_path=os.path.join('test', 'orders.hyper'))
+    extractor = HyperFile(configuration=configuration, dataset_specification=dataset, metadata=metadata)
+    builder = DatasetBuilder(dataset_specification=dataset, metadata=metadata, data=extractor)
+    path = os.path.join('output', dataset.collection, dataset.name + '.tdsx')
+    os.makedirs(os.path.join('output', dataset.collection), exist_ok=True)
+    builder.build(file_path=path, output_format=Format.TABLEAU_PACKAGED_DATASOURCE)
+
+def test_integration_csv():
+    dataset = dataset_from_json(os.path.join('test', 'dataset.json'))
+    metadata = metadata_from_json(os.path.join('test', 'metadata.json'))
+    configuration = Configuration(file_path=os.path.join('test', 'orders.hyper'))
+    extractor = HyperFile(configuration=configuration, dataset_specification=dataset, metadata=metadata)
+    builder = DatasetBuilder(dataset_specification=dataset, metadata=metadata, data=extractor)
+    path = os.path.join('output', dataset.collection, dataset.name + '.csv')
+    os.makedirs(os.path.join('output', dataset.collection), exist_ok=True)
+    builder.build(file_path=path, output_format=Format.CSV)

mario-pipeline-tools-0.1/test/test_metadata.py

@@ -0,0 +1,24 @@
+import os
+import tempfile
+
+from mario.metadata import metadata_from_json
+
+
+def test_load_metadata():
+    metadata_file = os.path.join('test', 'metadata.json')
+    metadata = metadata_from_json(file_path=metadata_file)
+    assert metadata.get_metadata('Ship Mode') is not None
+    assert metadata.get_metadata('Ship Mode').description is not None
+    assert metadata.get_metadata('Ship Mode').get_property('groups') == ['Shipping']
+
+
+def test_save_metadata():
+    metadata_file = os.path.join('test', 'metadata.json')
+    metadata = metadata_from_json(file_path=metadata_file)
+    with tempfile.NamedTemporaryFile() as file:
+        metadata.save(file_path=file.name)
+
+
+def test_load_tdsa():
+    metadata_file = os.path.join('test', 'tdsa.json')
+    metadata = metadata_from_json(file_path=metadata_file)