tableconv 1.8__py3-none-any.whl

tableconv/__init__.py

@@ -0,0 +1,20 @@
+import logging
+
+# One gripe of python is some libraries immediately start logging unhelp stuff the moment you import them.
+# This triggers flake8 E402 (code before imports error). How are you supposed to resolve this cleanly?
+logging.getLogger("numexpr").setLevel(logging.ERROR)
+
+from .__version__ import __version__  # noqa: E402
+from .core import IntermediateExchangeTable, load_url  # noqa: E402
+from .exceptions import DataError, EmptyDataError, InvalidQueryError, InvalidURLError, SuppliedDataError  # noqa: E402
+
+__all__ = [
+    "IntermediateExchangeTable",
+    "load_url",
+    "EmptyDataError",
+    "DataError",
+    "InvalidQueryError",
+    "InvalidURLError",
+    "SuppliedDataError",
+    "__version__",
+]

tableconv/__version__.py

@@ -0,0 +1,3 @@
+VERSION = (1, 9985, 20250218)
+
+__version__ = ".".join(map(str, VERSION))

tableconv/adapters/df/__init__.py

@@ -0,0 +1,25 @@
+from tableconv.adapters.df.base import adapters, read_adapters, write_adapters  # noqa: F401
+
+from .ascii import *  # noqa: F401 F403
+from .aws_athena import *  # noqa: F401 F403
+from .aws_dynamodb import *  # noqa: F401 F403
+from .aws_logs import *  # noqa: F401 F403
+from .gsheets import *  # noqa: F401 F403
+from .jc import *  # noqa: F401 F403
+from .jira import *  # noqa: F401 F403
+from .json import *  # noqa: F401 F403
+from .leveldb import *  # noqa: F401 F403
+from .nested_list import *  # noqa: F401 F403
+from .numbers import *  # noqa: F401 F403
+from .osquery import *  # noqa: F401 F403
+from .pandas_io import *  # noqa: F401 F403
+from .pcap import *  # noqa: F401 F403
+from .python import *  # noqa: F401 F403
+from .rdbms import *  # noqa: F401 F403
+from .smart_sheet import *  # noqa: F401 F403
+from .sql_literal import *  # noqa: F401 F403
+from .sumo_logic import *  # noqa: F401 F403
+from .text_array import *  # noqa: F401 F403
+from .yaml import *  # noqa: F401 F403
+
+# TODO: Register adapters in a cleaner way (dynamic adapter loading?). Just get rid of the `import *`.

tableconv/adapters/df/ascii.py

@@ -0,0 +1,130 @@
+from tableconv.adapters.df.base import Adapter, register_adapter
+from tableconv.adapters.df.file_adapter_mixin import FileAdapterMixin
+
+
+def _render_value(value):
+    if value is None:
+        return ""
+    return str(value).replace("\n", "\\n")
+
+
+def _get_serialized_rows(rows):
+    return [{key: _render_value(value) for key, value in row.items()} for row in rows]
+
+
+def _get_column_max_lengths(rows, column_names):
+    return {column: max([len(row[column]) for row in rows] + [len(column)]) for column in column_names}
+
+
+def render_asciilite(ordered_fields, rows):
+    """Text table rendering inspired by sqlite CLI."""
+    output_lines = []
+    for row in rows:
+        sorted_values = [row[field] for field in ordered_fields]
+        serialized_value = [_render_value(value) for value in sorted_values]
+        output_lines.append("|".join(serialized_value))
+    return "\n".join(output_lines)
+
+
+def render_unicodebox(ordered_fields, rows):
+    """Text table rendering inspired by ClickHouse."""
+    serialized_rows = _get_serialized_rows(rows)
+    max_lengths = _get_column_max_lengths(serialized_rows, ordered_fields)
+
+    output_lines = []
+    output_lines.append("┌─" + "─┬─".join([field.ljust(max_lengths[field], "─") for field in ordered_fields]) + "─┐")
+    for row in serialized_rows:
+        rendered_values_list = []
+        for field in ordered_fields:
+            rendered_values_list.append(row[field].ljust(max_lengths[field]))
+        output_lines.append("│ " + " │ ".join(rendered_values_list) + " │")
+    output_lines.append("└─" + "─┴─".join(["─" * max_lengths[field] for field in ordered_fields]) + "─┘")
+    return "\n".join(output_lines)
+
+
+@register_adapter(
+    [
+        "ascii",
+        "asciiplain",
+        "asciisimple",
+        "asciigrid",
+        "asciifancygrid",
+        "asciipipe",
+        "asciipresto",
+        "asciipretty",
+        "asciipsql",
+        "asciilite",
+        "asciibox",
+        "mediawikiformat",
+        "moinmoinformat",
+        "jiraformat",
+        "markdown",
+        "md",
+        "rst",
+        "html",
+        "latex",
+        "tex",
+    ],
+    write_only=True,
+)
+class ASCIIAdapter(FileAdapterMixin, Adapter):
+    """
+    Adapter focused on outputting ascii art style table renderings, such as those found in database CLIs.
+    """
+
+    text_based = True
+
+    # @staticmethod
+    # def _transform_df(df):
+    #     def transform(obj):
+    #         if isinstance(obj, datetime.datetime):
+    #             if obj.tzinfo is not None:
+    #                 obj = obj.astimezone(datetime.timezone.utc)
+    #             # Warning: Interpret naive TS as being UTC.
+    #             return obj.strftime('%Y-%m-%d %H:%M:%S')
+    #         elif isinstance(obj, list) or isinstance(obj, dict):
+    #             return str(obj)
+    #         return obj
+    #     df = df.applymap(transform)
+    #     df = df.replace({np.nan: None})
+
+    @staticmethod
+    def get_example_url(scheme):
+        return f"{scheme}:-"
+
+    @staticmethod
+    def dump_text_data(df, scheme, params):
+        TABULATE_TABLEFMT = {
+            "ascii": "simple",
+            "asciiplain": "plain",
+            "asciisimple": "simple",
+            "md": "github",
+            "markdown": "github",
+            "asciigrid": "grid",
+            "asciifancygrid": "fancy_grid",
+            "asciipipe": "pipe",
+            "asciipresto": "presto",
+            "asciipretty": "pretty",
+            "asciipsql": "psql",
+            "mediawikiformat": "mediawiki",
+            "moinmoinformat": "moinmoin",
+            "jiraformat": "jira",
+            "rst": "rst",
+            "latex": "latex",
+            "tex": "latex",
+        }
+        if scheme in TABULATE_TABLEFMT:
+            from tabulate import tabulate
+
+            return tabulate(
+                df.values.tolist(),
+                list(df.columns),
+                tablefmt=TABULATE_TABLEFMT[scheme],
+                disable_numparse=True,
+            )
+        elif scheme == "asciilite":
+            return render_asciilite(list(df.columns), df.to_dict("records"))
+        elif scheme == "asciibox":
+            return render_unicodebox(list(df.columns), df.to_dict("records"))
+        else:
+            raise AssertionError()

tableconv/adapters/df/aws_athena.py

@@ -0,0 +1,333 @@
+import contextlib
+import datetime
+import logging
+import os
+import tempfile
+import textwrap
+import time
+import uuid
+
+from tableconv.adapters.df.base import Adapter, register_adapter
+from tableconv.adapters.df.pandas_io import CSVAdapter, ParquetAdapter
+from tableconv.exceptions import (
+    AppendSchemeConflictError,
+    InvalidParamsError,
+    InvalidQueryError,
+    TableAlreadyExistsError,
+)
+from tableconv.uri import parse_uri
+
+logger = logging.getLogger(__name__)
+
+
+FORMAT_SQL_MAPPING = {
+    "parquet": textwrap.dedent(
+        """
+        ROW FORMAT SERDE
+          'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+        STORED AS INPUTFORMAT
+          'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+        OUTPUTFORMAT
+          'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+        """
+    ),
+    "csv": textwrap.dedent(
+        """
+        ROW FORMAT DELIMITED
+          FIELDS TERMINATED BY ','
+        STORED AS INPUTFORMAT
+          'org.apache.hadoop.mapred.TextInputFormat'
+        OUTPUTFORMAT
+          'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+        """
+    ),
+}
+
+
+@register_adapter(["awsathena"])
+class AWSAthenaAdapter(Adapter):
+    @staticmethod
+    def get_example_url(scheme):
+        return f"{scheme}://eu-central-1"
+
+    @staticmethod
+    def load(uri, query):
+        uri = parse_uri(uri)
+        aws_region = uri.authority
+
+        return AWSAthenaAdapter._run_athena_query(
+            query=query, aws_region=aws_region, catalog="AwsDataCatalog", database=None, return_results_df=True
+        )
+
+    @staticmethod
+    def _run_athena_query(
+        query, aws_region, catalog, database, return_results_raw=False, return_results_df=False, athena_client=None
+    ):
+        import boto3
+
+        if not athena_client:
+            athena_client = boto3.client("athena", region_name=aws_region)
+        sts = boto3.client("sts", region_name=aws_region)
+        s3 = boto3.client("s3", region_name=aws_region)
+
+        aws_account_id = sts.get_caller_identity()["Account"]
+        output_s3_bucket = f"aws-athena-query-results-{aws_account_id}-{aws_region}"
+
+        logger.debug(f"Querying.. aws_region={aws_region}, catalog={catalog}, output_s3_bucket={output_s3_bucket}.")
+        query_execution_context = {"Catalog": catalog}
+        if database:
+            query_execution_context["Database"] = database
+        query_req_resp = athena_client.start_query_execution(
+            QueryString=query,
+            QueryExecutionContext=query_execution_context,
+            WorkGroup="primary",
+            ResultConfiguration={"OutputLocation": f"s3://{output_s3_bucket}/"},
+        )
+        query_execution_id = query_req_resp["QueryExecutionId"]
+        logger.info(f"Waiting for AWS Athena query {query_execution_id}...")
+
+        while True:
+            details = athena_client.get_query_execution(QueryExecutionId=query_execution_id)
+            status = details["QueryExecution"]["Status"]["State"]
+            if status in ("FAILED", "CANCELLED"):
+                error_message = details["QueryExecution"]["Status"].get("StateChangeReason")
+                raise InvalidQueryError(f"AWS Athena Query {status.lower()}: {error_message}")
+            elif status == "SUCCEEDED":
+                break
+            else:
+                POLL_INTERVAL = datetime.timedelta(seconds=2)
+                time.sleep(POLL_INTERVAL.total_seconds())
+
+        if return_results_raw:
+            response = athena_client.get_query_results(QueryExecutionId=query_execution_id)
+            return response["ResultSet"]
+        if return_results_df:
+            output_s3_key = f"{query_execution_id}.csv"
+            local_filename = f"/tmp/awsathena-{query_execution_id}.csv"
+            try:
+                s3.download_file(output_s3_bucket, output_s3_key, local_filename)
+                df = CSVAdapter.load(f"csv://{local_filename}", None)
+            finally:
+                with contextlib.suppress(FileNotFoundError):
+                    os.remove(local_filename)
+            return df
+
+    @staticmethod
+    def _get_json_schema(df):
+        """
+        TODO: Update aws_athena to not be a df-level adapter, and instead use IntermediateExchangeTable, so we don't
+        have to duplicate this logic.
+        """
+        from genson import SchemaBuilder
+
+        builder = SchemaBuilder()
+        builder.add_schema({"type": "object", "properties": {}})
+        for row in df.to_dict(orient="records"):
+            builder.add_object(row)
+        return builder.to_schema()
+
+    @staticmethod
+    def resolve_presto_type(json_schema, column_name=None, top_level=False):
+        presto_types = set()
+        if "type" in json_schema:
+            if isinstance(json_schema["type"], str):
+                json_types = {json_schema["type"]}
+            else:
+                assert isinstance(json_schema["type"], list)
+                json_types = json_schema["type"]
+            for json_type in json_types:
+                if json_type == "array":
+                    if "items" in json_schema:
+                        array_type = AWSAthenaAdapter.resolve_presto_type(json_schema["items"])
+                    else:
+                        array_type = "string"
+                    presto_types.add(f"array<{array_type}>")
+                elif json_type == "null":
+                    pass
+                else:
+                    presto_types.add(
+                        {
+                            "integer": "bigint",
+                            "string": "string",
+                            "boolean": "boolean",
+                            "number": "double",
+                            "object": "string",
+                        }[json_type]
+                    )
+        else:
+            assert "anyOf" in json_schema
+            for sub_definition in json_schema["anyOf"]:
+                presto_types.add(AWSAthenaAdapter.resolve_presto_type(sub_definition))
+        if "null" in presto_types and presto_types != {"null"}:
+            presto_types.remove("null")
+        if "double" in presto_types and presto_types != {"double"}:
+            # hide NaN corruption added by pandas (pandas converts nulls to NaN, which then cause the column to get
+            # misidentified (or so I argue) as containing doubles)
+            presto_types.remove("double")
+        if len(presto_types) > 1:
+            if top_level:
+                logger.warning(
+                    f"Identified multiple conflicting types for {column_name}: {presto_types}. Picking one "
+                    + "arbitrarily."
+                )
+            presto_types = {presto_types.pop()}
+        if len(presto_types) == 0:
+            if top_level:
+                logger.warning(f"Unable to identify type of column {column_name}. Picking string.")
+            presto_types = {"string"}
+        return presto_types.pop()
+
+    @staticmethod
+    def _gen_schema(df, data_format, table_name, s3_base_url):
+        schema = f"CREATE EXTERNAL TABLE `{table_name}` (\n"
+        field_schema_lines = []
+        columns = []
+        for column, json_schema in AWSAthenaAdapter._get_json_schema(df)["properties"].items():
+            presto_type = AWSAthenaAdapter.resolve_presto_type(json_schema, column_name=column, top_level=True)
+            field_schema_lines.append(f"  `{column}` {presto_type}")
+            columns.append(column)
+        schema += ",\n".join(field_schema_lines)
+        schema += "\n)"
+        schema += FORMAT_SQL_MAPPING[data_format]
+        schema += f"LOCATION\n  '{s3_base_url}'"
+        return schema, columns
+
+    @staticmethod
+    def dump(df, uri):
+        import boto3
+
+        uri = parse_uri(uri)
+        if "if_exists" in uri.query:
+            if_exists = uri.query["if_exists"]
+        elif "append" in uri.query and uri.query["append"].lower() != "false":
+            if_exists = "append"
+        elif "overwrite" in uri.query and uri.query["overwrite"].lower() != "false":
+            if_exists = "replace"
+        else:
+            if_exists = "fail"
+
+        if if_exists not in ("replace", "append", "fail"):
+            raise InvalidParamsError("valid values for if_exists are replace, append, or fail (default)")
+
+        aws_region = uri.authority
+
+        athena_client = boto3.client("athena", region_name=aws_region)
+        catalog = "AwsDataCatalog"
+        database = uri.path.strip("/")
+        table_name = uri.query["table"]
+
+        s3_bucket_path = uri.query["s3_bucket_path"]
+        s3_bucket_path_split = os.path.split(s3_bucket_path)
+        if s3_bucket_path_split[0]:
+            s3_bucket = s3_bucket_path_split[0]
+            s3_bucket_prefix = os.path.join(s3_bucket_path_split[1], table_name)
+        else:
+            s3_bucket = s3_bucket_path_split[1]
+            s3_bucket_prefix = table_name
+        data_format = uri.query["data_format"]
+        if data_format not in FORMAT_SQL_MAPPING:
+            raise InvalidParamsError(f"Only formats {FORMAT_SQL_MAPPING.keys()} supported")
+        s3_base_url = f"s3://{os.path.join(s3_bucket, s3_bucket_prefix)}"
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Dump to temp file on  disk
+            filename = f"{uuid.uuid4()}.{data_format}"
+            temp_file_path = os.path.join(temp_dir, filename)
+            if data_format == "csv":
+                CSVAdapter.dump(df, uri=temp_file_path)
+            elif data_format == "parquet":
+                ParquetAdapter.dump(df, uri=temp_file_path)
+            else:
+                raise AssertionError
+
+            # Manage Table DDL
+            schema_ddl, columns = AWSAthenaAdapter._gen_schema(df, data_format, table_name, s3_base_url)
+
+            try:
+                table_metadata = athena_client.get_table_metadata(
+                    CatalogName=catalog, DatabaseName=database, TableName=table_name
+                )
+                table_exists = True
+            except athena_client.exceptions.MetadataException as exc:
+                if "EntityNotFoundException" in exc.response["Error"]["Message"]:
+                    table_exists = False
+                else:
+                    raise
+            if table_exists:
+                if if_exists == "fail":
+                    raise TableAlreadyExistsError(f"{database}{table_name} already exists")
+                elif if_exists == "append":
+                    pre_existing_columns = [col["Name"] for col in table_metadata["TableMetadata"]["Columns"]]
+                    if not pre_existing_columns == columns:
+                        raise AppendSchemeConflictError("Cannot append to existing table - schema mismatch")
+                    pre_existing_s3_base_url = table_metadata["TableMetadata"]["Parameters"]["location"].strip("/")
+                    if pre_existing_s3_base_url != s3_base_url.strip("/"):
+                        existing_uri = parse_uri(pre_existing_s3_base_url)
+                        existing_bucket = existing_uri.authority
+                        existing_prefix = existing_uri.path.strip("/")
+                        if existing_bucket != s3_bucket:
+                            raise AppendSchemeConflictError(
+                                "Cannot append to existing table - s3 bucket mismatch "
+                                + f"(pre-existing location is {pre_existing_s3_base_url})"
+                            )
+                        if existing_prefix.startswith(s3_bucket_prefix):
+                            # Discovered prefix is more restrictive than our requested prefix - this is safe, we can
+                            # just adopt it.
+                            logger.warning(f"Appending to found pre-existing prefix at {existing_prefix}")
+                            s3_bucket_prefix = existing_prefix
+                            s3_base_url = f"s3://{os.path.join(s3_bucket, s3_bucket_prefix)}"
+                            schema_ddl = None  # Invalidate the now-outdated schema
+                elif if_exists == "replace":
+                    s3_bucket_prefix = os.path.join(s3_bucket_prefix, str(uuid.uuid4()))
+                    s3_base_url = f"s3://{os.path.join(s3_bucket, s3_bucket_prefix)}"
+                    schema_ddl, _ = AWSAthenaAdapter._gen_schema(df, data_format, table_name, s3_base_url)
+                    logger.warning(
+                        f"Deleting table definition for {database}.{table_name}. Leaving old data behind and changing "
+                        + f"prefix to {s3_bucket_prefix}/."
+                    )
+                    assert s3_base_url and schema_ddl and s3_bucket_prefix  # safety check
+                    old_table_schema_query_result = AWSAthenaAdapter._run_athena_query(
+                        query=f"SHOW CREATE TABLE `{table_name}`",
+                        return_results_raw=True,
+                        aws_region=aws_region,
+                        catalog="AwsDataCatalog",
+                        database=database,
+                        athena_client=athena_client,
+                    )
+                    old_table_schema = "\n".join(
+                        [x["Data"][0]["VarCharValue"] for x in old_table_schema_query_result["Rows"]]
+                    )
+                    logger.debug(
+                        "Backup of old table schema before deleting:\n" + textwrap.indent(old_table_schema, "  ")
+                    )
+                    AWSAthenaAdapter._run_athena_query(
+                        query=f"DROP TABLE `{table_name}`",
+                        aws_region=aws_region,
+                        catalog="AwsDataCatalog",
+                        database=database,
+                        athena_client=athena_client,
+                    )
+                    # It's just too dangerous to actually delete any data; commented out.
+                    # s3 = boto3.resource('s3')
+                    # bucket = s3.Bucket(s3_bucket)
+                    # bucket.objects.filter(Prefix=f'{s3_bucket_prefix}/').delete()
+                    table_exists = False
+                else:
+                    raise AssertionError
+            if not table_exists:
+                logger.info(f"Creating new table {database}.{table_name} in {aws_region}")
+                logger.debug("\n" + textwrap.indent(schema_ddl, "  "))
+                AWSAthenaAdapter._run_athena_query(
+                    query=schema_ddl,
+                    return_results_raw=True,
+                    aws_region=aws_region,
+                    catalog="AwsDataCatalog",
+                    database=database,
+                    athena_client=athena_client,
+                )
+
+            # Upload temp file to s3
+            s3_client = boto3.client("s3")
+            s3_object_key = os.path.join(s3_bucket_prefix, filename)
+            logger.info(f"Uploading data to s3://{os.path.join(s3_bucket, s3_object_key)}")
+            s3_client.upload_file(temp_file_path, s3_bucket, s3_object_key)

tableconv/adapters/df/aws_dynamodb.py

@@ -0,0 +1,37 @@
+import logging
+
+import pandas as pd
+
+from tableconv.adapters.df.base import Adapter, register_adapter
+from tableconv.uri import parse_uri
+
+logger = logging.getLogger(__name__)
+
+
+@register_adapter(["awsdynamodb"], read_only=True)
+class AWSDynamoDB(Adapter):
+    @staticmethod
+    def get_example_url(scheme):
+        return f"{scheme}://eu-central-1/example_table"
+
+    @staticmethod
+    def load(uri, query):
+        import boto3
+
+        uri = parse_uri(uri)
+        aws_region = uri.authority
+        table_name = uri.path.strip("/")
+
+        dynamodb = boto3.client("dynamodb", region_name=aws_region)
+
+        if query:
+            result = dynamodb.execute_statement(Statement=query)
+            raw_array = result["Items"]
+        else:
+            logger.info("Sequentially querying DynamoDB scan results...")
+            scan_results = dynamodb.get_paginator("scan").paginate(TableName=table_name)
+            raw_array = []
+            for response in scan_results:
+                raw_array.extend(response["Items"])
+
+        return pd.DataFrame.from_records(raw_array)

tableconv/adapters/df/aws_logs.py

@@ -0,0 +1,68 @@
+import contextlib
+import datetime
+import logging
+import time
+
+import pandas as pd
+
+from tableconv.adapters.df.base import Adapter, register_adapter
+from tableconv.exceptions import InvalidQueryError
+from tableconv.parse_time import parse_input_time
+from tableconv.uri import parse_uri
+
+logger = logging.getLogger(__name__)
+
+
+@register_adapter(["awslogs"], read_only=True)
+class AWSLogs(Adapter):
+    """AWS Cloudwatch Logs (Disclaimer: Only supports Logs Insights queries for now)"""
+
+    @staticmethod
+    def get_example_url(scheme):
+        return f"{scheme}://eu-central-1//aws/lambda/example-function"
+
+    @staticmethod
+    def load(uri, query):
+        import boto3
+
+        uri = parse_uri(uri)
+        aws_region = uri.authority
+
+        from_time = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(days=1)
+        to_time = datetime.datetime.now(tz=datetime.timezone.utc)
+        if "from" in uri.query:
+            from_time = parse_input_time(uri.query["from"])
+        if "to" in uri.query:
+            to_time = parse_input_time(uri.query["to"])
+        client = boto3.client("logs", region_name=aws_region)
+
+        path = uri.path
+        if path[0] == "/":
+            path = path[1:]
+
+        query_id = client.start_query(
+            logGroupName=path,
+            startTime=int(from_time.timestamp()),
+            endTime=int(to_time.timestamp()),
+            queryString=query,
+            limit=int(uri.query.get("limit", 1000)),
+        )["queryId"]
+
+        try:
+            while True:
+                results = client.get_query_results(queryId=query_id)
+                if results["status"] in ("Failed", "Timeout", "Unknown"):
+                    raise InvalidQueryError(f'AWS CloudWatch Logs Insights Query {results["status"]}.')
+                elif results["status"] == "Complete":
+                    raw_array = [{item["field"]: item["value"] for item in row} for row in results["results"]]
+                    break
+                else:
+                    assert results["status"] in ("Running", "Scheduled")
+                    POLL_INTERVAL = datetime.timedelta(seconds=2)
+                    time.sleep(POLL_INTERVAL.total_seconds())
+        except Exception as exc:
+            with contextlib.suppress(Exception):
+                client.stop_query(query_id)
+            raise exc
+
+        return pd.DataFrame.from_records(raw_array)