datakiln 0.3.0__py3-none-any.whl

datakiln/__init__.py

@@ -0,0 +1 @@
+"""Build a database from a repository of files."""

datakiln/__main__.py

@@ -0,0 +1,7 @@
+"""
+Runs the command line with python -m datakiln.
+"""
+from datakiln.cli import main
+
+
+main()

datakiln/arguments.py

@@ -0,0 +1,76 @@
+"""
+Options that the commands share, and the log they write.
+"""
+import logging
+import sys
+from argparse import ArgumentParser, BooleanOptionalAction
+
+
+def add_directory(parser: ArgumentParser):
+    """Adds the directory of the dataset, which every command takes.
+
+    :param parser: Parser of a command.
+    :type parser: argparse.ArgumentParser
+    """
+    parser.add_argument("directory", metavar="directory", type=str,
+                        help="Path of the directory of the dataset.")
+
+
+def add_log(parser: ArgumentParser):
+    """Adds the option to show the log, which commands write to a file.
+
+    :param parser: Parser of a command.
+    :type parser: argparse.ArgumentParser
+    """
+    parser.add_argument("--stdout", action="store_true",
+                        help="Output logs to standard output as well as to "
+                             "the log file.")
+
+
+def add_include(parser: ArgumentParser, what: str):
+    """Adds the option that limits a command to some of the things it works
+    on.
+
+    :param parser: Parser of a command.
+    :type parser: argparse.ArgumentParser
+
+    :param what: What the command does with the names, as a sentence.
+    :type what: str
+    """
+    parser.add_argument("--include", metavar="name", type=str, nargs="+",
+                        help=what)
+
+
+def add_export(parser: ArgumentParser):
+    """Adds the options that tell what a command that reads from a source
+    writes to the dataset.
+
+    :param parser: Parser of a command.
+    :type parser: argparse.ArgumentParser
+    """
+    parser.add_argument("--data", action=BooleanOptionalAction, default=True,
+                        help="Whether to write the data files.")
+    parser.add_argument("--metadata", action=BooleanOptionalAction,
+                        default=False,
+                        help="Whether to write the metadata files of the "
+                             "tables.")
+
+
+def setup_logging(stdout: bool, log_file: str):
+    """Sets up logging for a command.
+
+    :param stdout: Whether to output logs to standard output.
+    :type stdout: bool
+
+    :param log_file: Path to the log file.
+    :type log_file: str
+    """
+    handlers = [logging.FileHandler(log_file, mode="w")]
+    if stdout:
+        handlers.append(logging.StreamHandler(sys.stdout))
+
+    logging.basicConfig(
+        level=logging.INFO, datefmt="%Y-%m-%d %H:%M:%S",
+        format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s",
+        handlers=handlers
+    )

datakiln/build_database.py

@@ -0,0 +1,110 @@
+"""
+Creates table and loads data.
+"""
+import logging
+from pathlib import Path
+
+from datakiln.arguments import add_directory, add_include, add_log
+from datakiln.metadata.database import DatabaseMeta, Session
+
+
+logger = logging.getLogger(__name__)
+
+
+def add_arguments(parser):
+    """Adds the options of the command.
+
+    :param parser: Parser of the command.
+    :type parser: argparse.ArgumentParser
+    """
+    add_directory(parser)
+    add_log(parser)
+    add_include(parser, "Build only these tables and scripts, and "
+                        "what they need.")
+    parser.add_argument("--grant", metavar="grant", type=str, nargs='+',
+                        help="Grant read permissions to specified users.")
+    parser.add_argument("--rebuild-all", action="store_true",
+                        help="Rebuild all tables and scripts, even if "
+                             "up to date.")
+    parser.add_argument("--prune", action="store_true",
+                        help="Drop the objects removed from the repo.")
+
+
+def run(args):
+    """Creates the tables, loads the data and runs the scripts.
+
+    :param args: Arguments of the command.
+    :type args: argparse.Namespace
+    """
+
+    # Get absolute of specified path
+    directory = Path(args.directory).absolute()
+
+    # Get database parameters
+    db_params = DatabaseMeta.load_env(directory)
+
+    # Tables and scripts to build, with what they use
+    database = DatabaseMeta(directory, db_params["database"])
+    include = None
+    if args.include:
+        selected = database.graph.find(args.include)
+        include = database.graph.requirements(selected)
+
+    # Connect to database
+    session = Session.create(**db_params)
+
+    # Create metadata
+    database.create_metadata(session)
+    database.load_stale(session)
+
+    # Create role for database read access
+    database.create_role(session)
+
+    # Stage and flush to database
+    updated = []
+    for table in database.graph.tables:
+        if include is not None and table not in include:
+            continue
+
+        # Skip the tables that did not change, unless all are rebuilt. A table
+        # is also rebuilt if a table it references is, since that drops its
+        # foreign keys
+        if (
+            not args.rebuild_all
+            and not table.needs_update(session)
+            and database.graph.dependencies(table).isdisjoint(updated)
+        ):
+            logger.info("Table '%s'.'%s': Already up to date.",
+                        table.schema, table.name)
+            continue
+        table.pull()
+        table.create(session)
+        table.stage_meta.load(session)
+        updated.append(table)
+
+    # Run transforms
+    database.execute_directory("transform", session, max_statements=0)
+
+    # Load to final schema and flush to database
+    for table in updated:
+        table.resolve_columns(session)
+        table.create_uniques(session)
+        table.load(session)
+        table.enforce_fks(session)
+        table.create_indexes(session)
+        table.create_metadata(session)
+        database.forget_dependents(session, [table])
+        session.commit()
+
+    # Run views
+    session.close()
+    database.execute_derive(
+        session, force=args.rebuild_all, include=include)
+
+    # Objects whose metadata or script was removed
+    database.prune(session, drop=args.prune)
+
+    # Grant read permissions on entire database
+    database.grant(session, args.grant)
+    session.commit()
+    session.close()

datakiln/build_dbms.py

@@ -0,0 +1,136 @@
+"""
+Builds table metadata from DBMS.
+"""
+import logging
+from getpass import getuser
+from pathlib import Path
+
+from datakiln.metadata.database import DatabaseMeta, DRIVER
+from datakiln.arguments import (
+    add_directory, add_export, add_include, add_log
+)
+from datakiln.extraction.models import ExtractMeta, ExtractGraph
+from datakiln.extraction.strategies import (
+    StreamStrategy, TempTableStrategy, ParallelExecutor
+)
+from datakiln.extraction.exporters import MetadataExtractor
+
+
+logger = logging.getLogger(__name__)
+
+
+def add_arguments(parser):
+    """Adds the options of the command.
+
+    :param parser: Parser of the command.
+    :type parser: argparse.ArgumentParser
+    """
+    add_directory(parser)
+    add_log(parser)
+    add_include(parser, "Extract only these tables, and the ones they "
+                        "need.")
+    add_export(parser)
+    parser.add_argument('--dialect', metavar="dialect", type=str,
+                        required=True, choices=DRIVER.keys(),
+                        help='Dialect of the source DBMS.')
+    parser.add_argument('--host', metavar="host", type=str,
+                        required=True, help='Hostname of the source DBMS.')
+    parser.add_argument('--port', metavar="port", type=str,
+                        required=True,
+                        help='Port to use when connecting to the source '
+                             'DBMS.')
+    parser.add_argument('--user', metavar="user", type=str,
+                        required=False, default=getuser(),
+                        help='Username to use when connecting to the source '
+                             'DBMS.')
+    parser.add_argument('--password', metavar="password", type=str,
+                        required=False,
+                        help='Password to use when connecting to the source '
+                             'DBMS.')
+    parser.add_argument('--database', metavar="database", type=str,
+                        required=True,
+                        help='Database to connect to in the source DBMS.')
+    parser.add_argument('--read-only', action='store_true', default=False,
+                        help='Run in read-only mode (no temp tables).')
+    parser.add_argument('--parallel', type=int, default=4,
+                        help='Number of parallel extraction jobs.')
+
+
+def run(args):
+    # pylint: disable=too-many-locals, too-many-branches, too-many-statements
+    """Downloads data and creates sample metadata for tables.
+
+    :param args: Arguments of the command.
+    :type args: argparse.Namespace
+    """
+    directory = Path(args.directory).absolute()
+
+    # Create output directory
+    meta_dir = Path(directory) / "metadata"
+    meta_dir.mkdir(exist_ok=True)
+
+    # Connect parameters
+    db_params = {
+        "dialect": args.dialect,
+        "user": args.user,
+        "password": args.password,
+        "host": args.host,
+        "database": args.database,
+        "port": args.port
+    }
+
+    # Load all extraction metadata
+    tasks = []
+    for file in sorted(directory.glob("extract/*.json")):
+        tasks.append(ExtractMeta.from_json(file))
+
+    # Build dependency graph of the tables to export, and what they need
+    graph = ExtractGraph.select(tasks, args.include)
+
+    # Initialize execution strategy
+    if args.read_only:
+        strategy = StreamStrategy(db_params, directory)
+    else:
+        strategy = TempTableStrategy(db_params, directory)
+
+    # Track exported tables for metadata generation
+    exported_schemas = {}
+    for task in graph.sort():
+        if task in graph.export:
+            if task.schema not in exported_schemas:
+                exported_schemas[task.schema] = []
+            exported_schemas[task.schema].append(task.table)
+
+    if args.data:
+        # Create data directory if not exists
+        data_dir = directory / "data"
+        data_dir.mkdir(exist_ok=True)
+
+        # Initialize and run parallel executor
+        executor = ParallelExecutor(strategy, graph, max_workers=args.parallel)
+        try:
+            executor.execute(data_dir=data_dir)
+        finally:
+            strategy.cleanup()
+
+    # Skip metadata generation
+    if not args.metadata:
+        return
+
+    # Get target database params
+    db_params = DatabaseMeta.load_env(directory)
+
+    # Generate table metadata
+    with strategy.session() as session:
+        for schema, tables in exported_schemas.items():
+            # Retrieve metadata from DBMS
+            metadata = MetadataExtractor.get_metadata(session, schema, tables)
+
+            for _, table in metadata.tables.items():
+                # Generate table metadata
+                logger.info("Table '%s': Generating metadata.", table)
+                MetadataExtractor.generate_table_meta(
+                    meta_dir,
+                    table,
+                    db_params["dialect"]
+                )

datakiln/build_metadata.py

@@ -0,0 +1,165 @@
+"""
+Generate metadata from data file.
+"""
+import json
+import logging
+from pathlib import Path
+from itertools import starmap
+from functools import partial
+
+import pandas as pd
+from slugify import slugify
+
+from datakiln.arguments import add_directory, add_log
+
+
+logger = logging.getLogger(__name__)
+
+SUPPORTED_EXTENSIONS = {
+    ".csv.gz": partial(pd.read_csv, engine="python", nrows=10000),
+    ".csv": partial(pd.read_csv, engine="python", nrows=10000),
+    ".xlsx": partial(pd.read_excel, sheet_name=None, nrows=10000)
+}
+
+
+def dtype_to_psql(dtype):
+    """Define PostgreSQL type from pandas inferred type.
+
+    :param dtype: Type inferred by pandas.
+    :type dtype: type
+
+    :return: PostgreSQL data type.
+    :rtype: str
+    """
+    if dtype == object:
+        return "TEXT"
+    if dtype == int:
+        return "INTEGER"
+    if dtype == float:
+        return "DOUBLE PRECISION"
+    if dtype == bool:
+        return "BOOLEAN"
+    return None
+
+
+def dtype_to_meta(column, dtype):
+    """Generates metadata for column from tuple.
+
+    :param column: Column name as it is in data file.
+    :type column: str
+
+    :param dtype: Type inferred by pandas.
+    :type dtype: type
+
+    :return: Column metadata.
+    :rtype: dict
+    """
+    return {
+        "name": column,
+        "rename": slugify(column, separator="_"),
+        "dtype": {
+            "postgresql": dtype_to_psql(dtype)
+        }
+    }
+
+
+def generate_table_meta(meta_dir, table_name, data):
+    """Generates metadata for column from tuple.
+
+    :param meta_dir: Medatada directory.
+    :type meta_dir: Path
+
+    :param table_name: Name of the table.
+    :type table_name: str
+
+    :param data: DataFrame containing sample data for the table.
+    :type data: pandas.DataFrame
+
+    :return: None
+    """
+    # Ensure table name is database friendly
+    table_name = slugify(table_name, separator="_")
+
+    # Remove trailing white spaces from data
+    data = data.map(lambda x: x.strip() if isinstance(x, str) else x)
+
+    # Generate metadata
+    meta = {
+        "name": table_name,
+        "schema": "public",
+        "encoding": "utf-8",
+        "columns": list(starmap(dtype_to_meta, data.dtypes.items())),
+        "constraints": {
+            "primary_key": ["meta_row_id"]
+        }
+    }
+
+    # Write metadata to file
+    meta_path = meta_dir / f"{table_name}.json"
+    with open(meta_path, "w", encoding="utf-8") as file:
+        json.dump(meta, file, indent=4)
+
+
+def add_arguments(parser):
+    """Adds the options of the command.
+
+    :param parser: Parser of the command.
+    :type parser: argparse.ArgumentParser
+    """
+    add_directory(parser)
+    add_log(parser)
+    parser.add_argument('--delimiter', metavar="delimiter", type=str,
+                        required=False,
+                        help='Character to treat as delimiter.')
+
+
+def run(args):
+    """Generates metadata from the data files.
+
+    :param args: Arguments of the command.
+    :type args: argparse.Namespace
+    """
+
+    # Get absolute path for specified directory
+    directory = Path(args.directory).absolute()
+
+    # Ensure metadata directory exists
+    meta_dir = directory / "metadata"
+    meta_dir.mkdir(exist_ok=True)
+
+    # Generate metadata for each data file
+    for data_path in directory.glob("data/*.*"):
+        # Skip if the extension is not supported
+        extension = "".join(data_path.suffixes).lower()
+        if extension not in SUPPORTED_EXTENSIONS:
+            continue
+        logger.info("File '%s': Generating metadata.", data_path)
+
+        # Read file
+        params = {}
+        reader = SUPPORTED_EXTENSIONS[extension]
+        if args.delimiter and reader.func == pd.read_csv:
+            params["delimiter"] = args.delimiter
+        data = reader(data_path.absolute(), **params)
+
+        # Excel files can have more than one sheet
+        # so we handle the data as a dictionary
+        table_name = data_path.name[:-len(extension)]
+        if isinstance(data, pd.DataFrame):
+            data = {table_name: data}
+        # Only consider tables with header row
+        # when multiple tables are returned
+        elif isinstance(data, dict):
+            data = {
+                name: df
+                for name, df in data.items()
+                if not df.columns.str.startswith("Unnamed:").all()
+            }
+
+            # If only one table is left, use file name
+            if len(data) == 1:
+                data = {data_path.stem: list(data.values())[0]}
+
+        # Generate metadata
+        for table_name, df in data.items():
+            generate_table_meta(meta_dir, table_name, df)