sql2csv 1.4__py3-none-any.whl

sql2csv/__main__.py

@@ -0,0 +1,4 @@
+from . import sql2csv
+
+if __name__ == '__main__':
+    sql2csv.main()

sql2csv/sql2csv.py

@@ -0,0 +1,340 @@
+#!/usr/bin/env python3
+
+import sys
+import csv
+import tempfile
+import os
+from os.path import expanduser
+import json
+
+import argparse
+import pymysql.cursors
+import pymysql.constants.CLIENT
+import psycopg2.extras
+import psycopg2
+
+file_ = None
+
+
+def get_mysql_connection(host, user, port, password, database):
+    """ MySQL connection """
+
+    return pymysql.connect(host=host,
+                           user=user,
+                           port=port,
+                           password=password,
+                           database=database,
+                           charset='utf8mb4',
+                           client_flag=pymysql.constants.CLIENT.MULTI_STATEMENTS
+                           )
+
+
+def get_pg_connection(host, user, port, password, database):
+    """ PostgreSQL connection """
+
+    return psycopg2.connect(host=host,
+                            user=user,
+                            port=port,
+                            password=password,
+                            dbname=database
+                            )
+
+
+def get_connection(engine, host, user, port, password, database):
+    """ Get SQL connection """
+
+    if engine == 'mysql':
+        return get_mysql_connection(host, user, port, password, database)
+    elif engine == 'postgresql':
+        return get_pg_connection(host, user, port, password, database)
+    else:
+        raise RuntimeError(
+            '"%s" engine is not supported.' % (engine))
+
+
+def get_cursor(connection):
+    """ Return connection cursor """
+
+    return connection.cursor()
+
+
+def execute_query(cursor, query):
+    """ Run a query and yield each row """
+
+    cursor.execute(query)
+
+
+def fetch_rows(cursor):
+    """ Fetch and yield rows """
+
+    for row in cursor.fetchall():
+        yield row
+
+
+def fetch_headers(cursor):
+    """ Return list of headers """
+
+    return [i[0] for i in cursor.description]
+
+
+def discard_line(line):
+    """ Decide whether we should keep or discard a line """
+
+    if line[:1] in ['', '+', '(', '-']:
+        return True
+
+    return False
+
+
+def remove_leading_trailing_pipe(line):
+    """ Remove optional leading and trailing pipe """
+
+    return line.strip('|')
+
+
+def get_column_separator(input_):
+    """ Return the column separator """
+    """ This logic needs to be improved """
+
+    if input_.count('|') > input_.count('\t'):
+        return '|'
+
+    return '\t'
+
+
+def split_columns(line, separator='\t'):
+    """ Split a line with a "separator" """
+
+    return line.split(separator)
+
+
+def strip_whitespaces(tpl):
+    """ Strip white spaces before and after each item """
+
+    return [item.strip() for item in tpl]
+
+
+def stringify_items(row):
+    """ Loop thru each item and Json-encode dicts """
+
+    row = list(row)
+    for k, item in enumerate(row):
+        if (isinstance(item, dict)):
+            row[k] = json.dumps(item)
+
+    return tuple(row)
+
+
+def has_stdin_input():
+    """ Return `True` if there is an stdin input """
+
+    if not sys.stdin.isatty():
+        return True
+
+    return False
+
+
+def resolve_home_dir(destination):
+    """ Resolve `~` to a full path """
+
+    if destination[:1] == '~':
+        return expanduser("~") + destination[1:]
+
+    return destination
+
+
+def open_file(destination):
+    """ Open file """
+
+    global file_
+
+    file_ = open(destination, 'w+', newline='')
+
+    return file_
+
+
+def open_tempfile():
+    """ Open a temporary file """
+
+    global file_
+
+    file_ = tempfile.NamedTemporaryFile('w+', newline='', delete=False)
+
+    return file_
+
+
+def get_writer(file_, delimiter=',', quotechar='"'):
+    """ Return a writer object """
+
+    return csv.writer(
+        file_,
+        delimiter=delimiter,
+        quotechar=quotechar,
+        quoting=csv.QUOTE_MINIMAL
+    )
+
+
+def file_to_stdout(remove=False):
+    """ Print file content to stdout """
+
+    with open(file_.name) as f:
+        print(f.read())
+
+    # Remove the temporary file once it has been printed
+    if remove:
+        os.remove(file_.name)
+
+
+def stdin_to_csv(delimiter=',', quotechar='"'):
+    """ Parse stdin and return output in a CSV format """
+
+    # Open CSV
+    with open_tempfile() as file_:
+        writer = get_writer(file_, delimiter=delimiter, quotechar=quotechar)
+
+        # Parse lines and add to file
+        separator = None
+        for line in sys.stdin:
+
+            if not discard_line(line):
+                # Get column separator
+                separator = get_column_separator(
+                    line) if not separator else separator
+
+                # Remove leading and trailing |
+                line = remove_leading_trailing_pipe(line)
+
+                if line.strip():
+                    # Split columns with separator
+                    row = split_columns(line, separator)
+
+                    # Add to CSV
+                    row = strip_whitespaces(row)
+
+                    # Write row
+                    writer.writerow(row)
+
+    file_to_stdout(remove=True)
+
+
+def query_to_csv(engine, host, user, port, password, database, query, headers=False, out_type='stdout', destination_file=None, delimiter=',', quotechar='"', print_info=1000):
+    """ Run a query and store the result to a CSV file """
+
+    # Get SQL connection
+    connection = get_connection(
+        engine=engine,
+        host=host,
+        user=user,
+        port=port,
+        password=password,
+        database=database
+    )
+    cursor = get_cursor(connection)
+
+    if out_type == 'file':
+        print('\n* Exporting rows...')
+
+    with open_tempfile() if out_type == 'stdout' else open_file(resolve_home_dir(destination_file)) as file_:
+        writer = get_writer(file_, delimiter=delimiter, quotechar=quotechar)
+
+        # Execute query
+        execute_query(cursor=cursor, query=query)
+
+        # Write headers if requested
+        if headers:
+            writer.writerow(fetch_headers(cursor=cursor))
+
+        # Write rows to CSV
+        i = 0
+        for row in fetch_rows(cursor=cursor):
+            # Increment row counter
+            i += 1
+
+            if out_type == 'file' and i % print_info == 0:
+                print('  ...%s rows written' % "{:,}".format(i))
+
+            row = stringify_items(row)
+
+            writer.writerow(row)
+
+        if out_type == 'file':
+            print('  ...done')
+            print('* The result has been exported to %s.\n' %
+                  (destination_file))
+
+    # Print stdout
+    if out_type == 'stdout':
+        file_to_stdout(remove=True)
+
+
+def main():
+    """ Parses arguments and run module """
+
+    # Intercept and parse stdin input
+    if has_stdin_input():
+        # Parse arguments
+        parser = argparse.ArgumentParser()
+        parser.add_argument("-D", "--delimiter",
+                            help="CSV delimiter", default=',')
+        parser.add_argument("-Q", "--quotechar",
+                            help="CSV quote character", default='"')
+        args = parser.parse_args()
+
+        return stdin_to_csv(delimiter=args.delimiter, quotechar=args.quotechar)
+
+    # Parse arguments
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-e", "--engine", type=str, help="Database engine",
+                        choices=['mysql', 'postgresql'], default='mysql')
+    parser.add_argument("-H", "--host", default="127.0.0.1",
+                        help="Database host")
+    parser.add_argument("-P", "--port", type=int,
+                        help="Database port")
+    parser.add_argument("-u", "--user", required=True, help="Database user")
+    parser.add_argument("-p", "--password", default='',
+                        help="Database password")
+    parser.add_argument("-d", "--database", required=True,
+                        help="Database name")
+    parser.add_argument("-q", "--query", required=True,
+                        help="SQL query")
+    parser.add_argument("-o", "--out",
+                        help="CSV destination", choices=['stdout', 'file'],
+                        default='stdout')
+    parser.add_argument("-f", "--destination_file",
+                        help="CSV destination file")
+    parser.add_argument("-D", "--delimiter", help="CSV delimiter", default=',')
+    parser.add_argument("-Q", "--quotechar",
+                        help="CSV quote character", default='"')
+    parser.add_argument("-t", "--headers", action='store_true',
+                        help="Include headers")
+    args = parser.parse_args()
+
+    # Set default port
+    if not args.port:
+        if args.engine == 'postgresql':
+            args.port = 5432  # PG default
+        else:
+            args.port = 3306  # MySQL default
+
+    # Force output to file if there is a file
+    if args.destination_file:
+        args.out = 'file'
+
+    query_to_csv(
+        engine=args.engine,
+        host=args.host,
+        user=args.user,
+        port=args.port,
+        password=args.password,
+        database=args.database,
+        query=args.query,
+        headers=args.headers,
+        out_type=args.out,
+        destination_file=args.destination_file,
+        delimiter=args.delimiter,
+        quotechar=args.quotechar
+    )
+
+
+if __name__ == '__main__':
+    main()

sql2csv-1.4.dist-info/METADATA

@@ -0,0 +1,144 @@
+Metadata-Version: 2.4
+Name: sql2csv
+Version: 1.4
+Summary: Run MySQL and PostgreSQL queries and store result in CSV
+Author-email: Gabriel Bordeaux <pypi@gab.lc>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/gabfl/sql2csv
+Classifier: Topic :: Database
+Classifier: Topic :: Database :: Database Engines/Servers
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Development Status :: 4 - Beta
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: PyMySQL>=1.1
+Requires-Dist: psycopg2-binary>=2.9.10
+Dynamic: license-file
+
+# sql2csv
+
+[![Pypi](https://img.shields.io/pypi/v/sql2csv.svg)](https://pypi.org/project/sql2csv)
+[![Build Status](https://github.com/gabfl/sql2csv/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/gabfl/sql2csv/actions)
+[![codecov](https://codecov.io/gh/gabfl/sql2csv/branch/main/graph/badge.svg)](https://codecov.io/gh/gabfl/sql2csv)
+[![MIT licensed](https://img.shields.io/badge/license-MIT-green.svg)](https://raw.githubusercontent.com/gabfl/sql2csv/main/LICENSE)
+
+Run MySQL and PostgreSQL queries and store the results in CSV.
+
+## Why sql2csv
+
+`sql2csv` is a small utility to run MySQL and PostgreSQL queries and store the output in a CSV file.
+
+In some environments like when using MySQL or Aurora in AWS RDS, exporting queries' results to CSV is not available with native tools. `sql2csv` is a simple module that offers this feature.
+
+## Installation
+
+```bash
+pip3 install sql2csv
+
+# Basic usage
+mysql [...] -e "SELECT * FROM table" | sql2csv
+# or
+psql [...] -c "SELECT * FROM table" | sql2csv
+```
+
+## Example
+
+### From stdin
+
+For simple queries you can pipe a result directly from `mysql` or `psql` to `sql2csv`.
+
+For more complex queries, it is recommended to use the CLI (see below) to ensure a properly formatted CSV.
+
+```bash
+mysql -U root -p"secret" my_db -e "SELECT * FROM some_mysql_table;" | sql2csv
+
+id,some_int,some_str,some_date
+1,12,hello world,2018-12-01 12:23:12
+2,15,hello,2018-12-05 12:18:12
+3,18,world,2018-12-08 12:17:12
+```
+
+```bash
+psql -U postgres my_db -c "SELECT * FROM some_pg_table" | sql2csv
+
+id,some_int,some_str,some_date
+1,12,hello world,2018-12-01 12:23:12
+2,15,hello,2018-12-05 12:18:12
+3,18,world,2018-12-08 12:17:12
+```
+
+### Using `sql2csv` CLI
+
+#### Output to stdout
+
+```bash
+$ sql2csv --engine mysql \
+  --database my_db --user root --password "secret" \
+  --query "SELECT * FROM some_mysql_table"
+
+1,12,hello world,2018-12-01 12:23:12
+2,15,hello,2018-12-05 12:18:12
+3,18,world,2018-12-08 12:17:12
+```
+
+#### Output saved in a file
+
+```bash
+$ sql2csv --engine mysql \
+  --database my_db --user root --password "secret" \
+  --query "SELECT * FROM some_mysql_table" \
+  --headers \
+  --out file --destination_file export.csv
+
+# * Exporting rows...
+#   ...done
+# * The result has been exported to export.csv.
+
+$ cat export.csv 
+id,some_int,some_str,some_date
+1,12,hello world,2018-12-01 12:23:12
+2,15,hello,2018-12-05 12:18:12
+3,18,world,2018-12-08 12:17:12
+```
+
+## Usage
+
+```bash
+usage: sql2csv [-h] [-e {mysql,postgresql}] [-H HOST] [-P PORT] -u USER
+               [-p PASSWORD] -d DATABASE -q QUERY [-o {stdout,file}]
+               [-f DESTINATION_FILE] [-D DELIMITER] [-Q QUOTECHAR] [-t]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -e {mysql,postgresql}, --engine {mysql,postgresql}
+                        Database engine
+  -H HOST, --host HOST  Database host
+  -P PORT, --port PORT  Database port
+  -u USER, --user USER  Database user
+  -p PASSWORD, --password PASSWORD
+                        Database password
+  -d DATABASE, --database DATABASE
+                        Database name
+  -q QUERY, --query QUERY
+                        SQL query
+  -o {stdout,file}, --out {stdout,file}
+                        CSV destination
+  -f DESTINATION_FILE, --destination_file DESTINATION_FILE
+                        CSV destination file
+  -D DELIMITER, --delimiter DELIMITER
+                        CSV delimiter
+  -Q QUOTECHAR, --quotechar QUOTECHAR
+                        CSV quote character
+  -t, --headers         Include headers
+```

sql2csv-1.4.dist-info/RECORD

@@ -0,0 +1,9 @@
+sql2csv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sql2csv/__main__.py,sha256=elJffGxPxqOYcg0q8wZAUcysRB31g3mMOxuEOJYOyWw,69
+sql2csv/sql2csv.py,sha256=CDZpJJsIsih91juzf5OMQSitIl7jrmJdTNRZkqwvfSw,9054
+sql2csv-1.4.dist-info/licenses/LICENSE,sha256=WqS7GgVJk_T1GouVCLKKsR1q83EOah9Hpb72mq0pQ64,1073
+sql2csv-1.4.dist-info/METADATA,sha256=n2caXnT9zGhqP2wCabzqKEkE2cgfPccsvqar2uvfmik,4605
+sql2csv-1.4.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+sql2csv-1.4.dist-info/entry_points.txt,sha256=r6P7X-kTfb-_i1IIPX2R6e8v9ypJeVQSHO9nMXT5teU,49
+sql2csv-1.4.dist-info/top_level.txt,sha256=cNCzLiAnuxrr1OQUoDDLb_6mkjGLOwEcqi0ErtFrfl0,8
+sql2csv-1.4.dist-info/RECORD,,

sql2csv-1.4.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sql2csv-1.4.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+sql2csv = sql2csv.sql2csv:main

sql2csv-1.4.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Gabriel Bordeaux
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sql2csv-1.4.dist-info/top_level.txt

@@ -0,0 +1 @@
+sql2csv