pyathena 1.0.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- PyAthena-1.0.0.dist-info/DESCRIPTION.rst +228 -0
- PyAthena-1.0.0.dist-info/METADATA +257 -0
- PyAthena-1.0.0.dist-info/RECORD +15 -0
- PyAthena-1.0.0.dist-info/WHEEL +6 -0
- PyAthena-1.0.0.dist-info/entry_points.txt +3 -0
- PyAthena-1.0.0.dist-info/metadata.json +1 -0
- PyAthena-1.0.0.dist-info/top_level.txt +1 -0
- pyathena/__init__.py +56 -0
- pyathena/connection.py +64 -0
- pyathena/converter.py +95 -0
- pyathena/cursor.py +288 -0
- pyathena/error.py +49 -0
- pyathena/formatter.py +134 -0
- pyathena/sqlalchemy_athena.py +171 -0
- pyathena/util.py +24 -0
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
.. image:: https://img.shields.io/pypi/pyversions/PyAthena.svg
|
|
2
|
+
:target: https://pypi.python.org/pypi/PyAthena/
|
|
3
|
+
|
|
4
|
+
.. image:: https://circleci.com/gh/laughingman7743/PyAthena.svg?style=shield
|
|
5
|
+
:target: https://circleci.com/gh/laughingman7743/PyAthena
|
|
6
|
+
|
|
7
|
+
.. image:: https://codecov.io/gh/laughingman7743/PyAthena/branch/master/graph/badge.svg
|
|
8
|
+
:target: https://codecov.io/gh/laughingman7743/PyAthena
|
|
9
|
+
|
|
10
|
+
.. image:: https://img.shields.io/pypi/l/PyAthena.svg
|
|
11
|
+
:target: https://github.com/laughingman7743/PyAthena/blob/master/LICENSE
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
PyAthena
|
|
15
|
+
========
|
|
16
|
+
|
|
17
|
+
PyAthena is a Python `DB API 2.0 (PEP 249)`_ compliant client for `Amazon Athena`_.
|
|
18
|
+
|
|
19
|
+
.. _`DB API 2.0 (PEP 249)`: https://www.python.org/dev/peps/pep-0249/
|
|
20
|
+
.. _`Amazon Athena`: http://docs.aws.amazon.com/athena/latest/APIReference/Welcome.html
|
|
21
|
+
|
|
22
|
+
Requirements
|
|
23
|
+
------------
|
|
24
|
+
|
|
25
|
+
* Python
|
|
26
|
+
|
|
27
|
+
- CPython 2.6, 2,7, 3,4, 3.5, 3.6
|
|
28
|
+
|
|
29
|
+
Installation
|
|
30
|
+
------------
|
|
31
|
+
|
|
32
|
+
.. code:: bash
|
|
33
|
+
|
|
34
|
+
$ pip install PyAthena
|
|
35
|
+
|
|
36
|
+
Extra packages:
|
|
37
|
+
|
|
38
|
+
+---------------+--------------------------------------+----------+
|
|
39
|
+
| Package | Install command | Version |
|
|
40
|
+
+===============+======================================+==========+
|
|
41
|
+
| Pandas | ``pip install PyAthena[Pandas]`` | >=0.19.0 |
|
|
42
|
+
+---------------+--------------------------------------+----------+
|
|
43
|
+
| SQLAlchemy | ``pip install PyAthena[SQLAlchemy]`` | >=1.0.0 |
|
|
44
|
+
+---------------+--------------------------------------+----------+
|
|
45
|
+
|
|
46
|
+
Usage
|
|
47
|
+
-----
|
|
48
|
+
|
|
49
|
+
Basic usage
|
|
50
|
+
~~~~~~~~~~~
|
|
51
|
+
|
|
52
|
+
.. code:: python
|
|
53
|
+
|
|
54
|
+
from pyathena import connect
|
|
55
|
+
|
|
56
|
+
cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
|
|
57
|
+
aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
|
|
58
|
+
s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
|
|
59
|
+
region_name='us-west-2').cursor()
|
|
60
|
+
cursor.execute("SELECT * FROM one_row")
|
|
61
|
+
print(cursor.description)
|
|
62
|
+
print(cursor.fetchall())
|
|
63
|
+
|
|
64
|
+
Cursor iteration
|
|
65
|
+
~~~~~~~~~~~~~~~~
|
|
66
|
+
|
|
67
|
+
.. code:: python
|
|
68
|
+
|
|
69
|
+
from pyathena import connect
|
|
70
|
+
|
|
71
|
+
cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
|
|
72
|
+
aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
|
|
73
|
+
s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
|
|
74
|
+
region_name='us-west-2').cursor()
|
|
75
|
+
cursor.execute("SELECT * FROM many_rows LIMIT 10")
|
|
76
|
+
for row in cursor:
|
|
77
|
+
print(row)
|
|
78
|
+
|
|
79
|
+
Query with parameter
|
|
80
|
+
~~~~~~~~~~~~~~~~~~~~
|
|
81
|
+
|
|
82
|
+
Supported `DB API paramstyle`_ is only ``PyFormat``.
|
|
83
|
+
``PyFormat`` only supports `named placeholders`_ with old ``%`` operator style and parameters specify dictionary format.
|
|
84
|
+
|
|
85
|
+
.. code:: python
|
|
86
|
+
|
|
87
|
+
from pyathena import connect
|
|
88
|
+
|
|
89
|
+
cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
|
|
90
|
+
aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
|
|
91
|
+
s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
|
|
92
|
+
region_name='us-west-2').cursor()
|
|
93
|
+
cursor.execute("""
|
|
94
|
+
SELECT col_string FROM one_row_complex
|
|
95
|
+
WHERE col_string = %(param)s
|
|
96
|
+
""", {'param': 'a string'})
|
|
97
|
+
print(cursor.fetchall())
|
|
98
|
+
|
|
99
|
+
if ``%`` character is contained in your query, it must be escaped with ``%%`` like the following:
|
|
100
|
+
|
|
101
|
+
.. code:: sql
|
|
102
|
+
|
|
103
|
+
SELECT col_string FROM one_row_complex
|
|
104
|
+
WHERE col_string = %(param)s OR col_string LIKE 'a%%'
|
|
105
|
+
|
|
106
|
+
.. _`DB API paramstyle`: https://www.python.org/dev/peps/pep-0249/#paramstyle
|
|
107
|
+
.. _`named placeholders`: https://pyformat.info/#named_placeholders
|
|
108
|
+
|
|
109
|
+
SQLAlchemy
|
|
110
|
+
~~~~~~~~~~
|
|
111
|
+
|
|
112
|
+
Install SQLAlchemy with ``pip install SQLAlchemy>=1.0.0`` or ``pip install PyAthena[SQLAlchemy]``.
|
|
113
|
+
Supported SQLAlchemy is 1.0.0 or higher.
|
|
114
|
+
|
|
115
|
+
.. code:: python
|
|
116
|
+
|
|
117
|
+
from urllib.parse import quote_plus # PY2: from urllib import quote_plus
|
|
118
|
+
from sqlalchemy.engine import create_engine
|
|
119
|
+
from sqlalchemy.sql.expression import select
|
|
120
|
+
from sqlalchemy.sql.functions import func
|
|
121
|
+
from sqlalchemy.sql.schema import Table, MetaData
|
|
122
|
+
|
|
123
|
+
conn_str = 'awsathena+rest://{aws_access_key_id}:{aws_secret_access_key}@athena.{region_name}.amazonaws.com:443/'\
|
|
124
|
+
'{schema_name}?s3_staging_dir={s3_staging_dir}'
|
|
125
|
+
engine = create_engine(conn_str.format(
|
|
126
|
+
aws_access_key_id=quote_plus('YOUR_ACCESS_KEY_ID'),
|
|
127
|
+
aws_secret_access_key=quote_plus('YOUR_SECRET_ACCESS_KEY'),
|
|
128
|
+
region_name='us-west-2',
|
|
129
|
+
schema_name='default',
|
|
130
|
+
s3_staging_dir=quote_plus('s3://YOUR_S3_BUCKET/path/to/')))
|
|
131
|
+
many_rows = Table('many_rows', MetaData(bind=engine), autoload=True)
|
|
132
|
+
print(select([func.count('*')], from_obj=many_rows).scalar())
|
|
133
|
+
|
|
134
|
+
The connection string has the following format:
|
|
135
|
+
|
|
136
|
+
.. code:: python
|
|
137
|
+
|
|
138
|
+
awsathena+rest://{aws_access_key_id}:{aws_secret_access_key}@athena.{region_name}.amazonaws.com:443/{schema_name}?s3_staging_dir={s3_staging_dir}&...
|
|
139
|
+
|
|
140
|
+
NOTE: ``s3_staging_dir`` requires quote. If ``aws_access_key_id``, ``aws_secret_access_key`` and other parameter contain special characters, quote is also required.
|
|
141
|
+
|
|
142
|
+
Pandas
|
|
143
|
+
~~~~~~
|
|
144
|
+
|
|
145
|
+
Minimal example for Pandas DataFrame:
|
|
146
|
+
|
|
147
|
+
.. code:: python
|
|
148
|
+
|
|
149
|
+
from pyathena import connect
|
|
150
|
+
import pandas as pd
|
|
151
|
+
|
|
152
|
+
conn = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
|
|
153
|
+
aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
|
|
154
|
+
s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
|
|
155
|
+
region_name='us-west-2')
|
|
156
|
+
df = pd.read_sql("SELECT * FROM many_rows", conn)
|
|
157
|
+
print(df.head())
|
|
158
|
+
|
|
159
|
+
As Pandas DataFrame:
|
|
160
|
+
|
|
161
|
+
.. code:: python
|
|
162
|
+
|
|
163
|
+
from pyathena import connect
|
|
164
|
+
from pyathena.util import as_pandas
|
|
165
|
+
|
|
166
|
+
cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
|
|
167
|
+
aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
|
|
168
|
+
s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
|
|
169
|
+
region_name='us-west-2').cursor()
|
|
170
|
+
cursor.execute("SELECT * FROM many_rows")
|
|
171
|
+
df = as_pandas(cursor)
|
|
172
|
+
print(df.describe())
|
|
173
|
+
|
|
174
|
+
Credentials
|
|
175
|
+
-----------
|
|
176
|
+
|
|
177
|
+
Support `Boto3 credentials`_.
|
|
178
|
+
|
|
179
|
+
.. _`Boto3 credentials`: http://boto3.readthedocs.io/en/latest/guide/configuration.html
|
|
180
|
+
|
|
181
|
+
Additional environment variable:
|
|
182
|
+
|
|
183
|
+
.. code:: bash
|
|
184
|
+
|
|
185
|
+
$ export AWS_ATHENA_S3_STAGING_DIR=s3://YOUR_S3_BUCKET/path/to/
|
|
186
|
+
|
|
187
|
+
Testing
|
|
188
|
+
-------
|
|
189
|
+
|
|
190
|
+
Depends on the AWS CLI credentials and the following environment variables:
|
|
191
|
+
|
|
192
|
+
~/.aws/credentials
|
|
193
|
+
|
|
194
|
+
.. code:: cfg
|
|
195
|
+
|
|
196
|
+
[default]
|
|
197
|
+
aws_access_key_id=YOUR_ACCESS_KEY_ID
|
|
198
|
+
aws_secret_access_key=YOUR_SECRET_ACCESS_KEY
|
|
199
|
+
|
|
200
|
+
Environment variables
|
|
201
|
+
|
|
202
|
+
.. code:: bash
|
|
203
|
+
|
|
204
|
+
$ export AWS_DEFAULT_REGION=us-west-2
|
|
205
|
+
$ export AWS_ATHENA_S3_STAGING_DIR=s3://YOUR_S3_BUCKET/path/to/
|
|
206
|
+
|
|
207
|
+
Run test
|
|
208
|
+
~~~~~~~~
|
|
209
|
+
|
|
210
|
+
.. code:: bash
|
|
211
|
+
|
|
212
|
+
$ pip install pytest awscli
|
|
213
|
+
$ scripts/upload_test_data.sh
|
|
214
|
+
$ py.test
|
|
215
|
+
$ scripts/delete_test_data.sh
|
|
216
|
+
|
|
217
|
+
Run test multiple Python versions
|
|
218
|
+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
219
|
+
|
|
220
|
+
.. code:: bash
|
|
221
|
+
|
|
222
|
+
$ pip install tox awscli
|
|
223
|
+
$ scripts/upload_test_data.sh
|
|
224
|
+
$ pyenv local 2.6.9 2.7.13 3.4.6 3.5.3 3.6.1
|
|
225
|
+
$ tox
|
|
226
|
+
$ scripts/delete_test_data.sh
|
|
227
|
+
|
|
228
|
+
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
Metadata-Version: 2.0
|
|
2
|
+
Name: PyAthena
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Python DB API 2.0 (PEP 249) compliant client for Amazon Athena
|
|
5
|
+
Home-page: https://github.com/laughingman7743/PyAthena/
|
|
6
|
+
Author: laughingman7743
|
|
7
|
+
Author-email: laughingman7743@gmail.com
|
|
8
|
+
License: MIT License
|
|
9
|
+
Platform: UNKNOWN
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Topic :: Database :: Front-Ends
|
|
15
|
+
Classifier: Programming Language :: Python :: 2
|
|
16
|
+
Classifier: Programming Language :: Python :: 2.6
|
|
17
|
+
Classifier: Programming Language :: Python :: 2.7
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.4
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.5
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.6
|
|
22
|
+
Requires-Dist: boto3 (>=1.4.4)
|
|
23
|
+
Requires-Dist: botocore (>=1.5.52)
|
|
24
|
+
Requires-Dist: future
|
|
25
|
+
Provides-Extra: Pandas
|
|
26
|
+
Requires-Dist: pandas (>=0.19.0); extra == 'Pandas'
|
|
27
|
+
Provides-Extra: SQLAlchemy
|
|
28
|
+
Requires-Dist: SQLAlchemy (>=1.0.0); extra == 'SQLAlchemy'
|
|
29
|
+
|
|
30
|
+
.. image:: https://img.shields.io/pypi/pyversions/PyAthena.svg
|
|
31
|
+
:target: https://pypi.python.org/pypi/PyAthena/
|
|
32
|
+
|
|
33
|
+
.. image:: https://circleci.com/gh/laughingman7743/PyAthena.svg?style=shield
|
|
34
|
+
:target: https://circleci.com/gh/laughingman7743/PyAthena
|
|
35
|
+
|
|
36
|
+
.. image:: https://codecov.io/gh/laughingman7743/PyAthena/branch/master/graph/badge.svg
|
|
37
|
+
:target: https://codecov.io/gh/laughingman7743/PyAthena
|
|
38
|
+
|
|
39
|
+
.. image:: https://img.shields.io/pypi/l/PyAthena.svg
|
|
40
|
+
:target: https://github.com/laughingman7743/PyAthena/blob/master/LICENSE
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
PyAthena
|
|
44
|
+
========
|
|
45
|
+
|
|
46
|
+
PyAthena is a Python `DB API 2.0 (PEP 249)`_ compliant client for `Amazon Athena`_.
|
|
47
|
+
|
|
48
|
+
.. _`DB API 2.0 (PEP 249)`: https://www.python.org/dev/peps/pep-0249/
|
|
49
|
+
.. _`Amazon Athena`: http://docs.aws.amazon.com/athena/latest/APIReference/Welcome.html
|
|
50
|
+
|
|
51
|
+
Requirements
|
|
52
|
+
------------
|
|
53
|
+
|
|
54
|
+
* Python
|
|
55
|
+
|
|
56
|
+
- CPython 2.6, 2,7, 3,4, 3.5, 3.6
|
|
57
|
+
|
|
58
|
+
Installation
|
|
59
|
+
------------
|
|
60
|
+
|
|
61
|
+
.. code:: bash
|
|
62
|
+
|
|
63
|
+
$ pip install PyAthena
|
|
64
|
+
|
|
65
|
+
Extra packages:
|
|
66
|
+
|
|
67
|
+
+---------------+--------------------------------------+----------+
|
|
68
|
+
| Package | Install command | Version |
|
|
69
|
+
+===============+======================================+==========+
|
|
70
|
+
| Pandas | ``pip install PyAthena[Pandas]`` | >=0.19.0 |
|
|
71
|
+
+---------------+--------------------------------------+----------+
|
|
72
|
+
| SQLAlchemy | ``pip install PyAthena[SQLAlchemy]`` | >=1.0.0 |
|
|
73
|
+
+---------------+--------------------------------------+----------+
|
|
74
|
+
|
|
75
|
+
Usage
|
|
76
|
+
-----
|
|
77
|
+
|
|
78
|
+
Basic usage
|
|
79
|
+
~~~~~~~~~~~
|
|
80
|
+
|
|
81
|
+
.. code:: python
|
|
82
|
+
|
|
83
|
+
from pyathena import connect
|
|
84
|
+
|
|
85
|
+
cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
|
|
86
|
+
aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
|
|
87
|
+
s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
|
|
88
|
+
region_name='us-west-2').cursor()
|
|
89
|
+
cursor.execute("SELECT * FROM one_row")
|
|
90
|
+
print(cursor.description)
|
|
91
|
+
print(cursor.fetchall())
|
|
92
|
+
|
|
93
|
+
Cursor iteration
|
|
94
|
+
~~~~~~~~~~~~~~~~
|
|
95
|
+
|
|
96
|
+
.. code:: python
|
|
97
|
+
|
|
98
|
+
from pyathena import connect
|
|
99
|
+
|
|
100
|
+
cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
|
|
101
|
+
aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
|
|
102
|
+
s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
|
|
103
|
+
region_name='us-west-2').cursor()
|
|
104
|
+
cursor.execute("SELECT * FROM many_rows LIMIT 10")
|
|
105
|
+
for row in cursor:
|
|
106
|
+
print(row)
|
|
107
|
+
|
|
108
|
+
Query with parameter
|
|
109
|
+
~~~~~~~~~~~~~~~~~~~~
|
|
110
|
+
|
|
111
|
+
Supported `DB API paramstyle`_ is only ``PyFormat``.
|
|
112
|
+
``PyFormat`` only supports `named placeholders`_ with old ``%`` operator style and parameters specify dictionary format.
|
|
113
|
+
|
|
114
|
+
.. code:: python
|
|
115
|
+
|
|
116
|
+
from pyathena import connect
|
|
117
|
+
|
|
118
|
+
cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
|
|
119
|
+
aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
|
|
120
|
+
s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
|
|
121
|
+
region_name='us-west-2').cursor()
|
|
122
|
+
cursor.execute("""
|
|
123
|
+
SELECT col_string FROM one_row_complex
|
|
124
|
+
WHERE col_string = %(param)s
|
|
125
|
+
""", {'param': 'a string'})
|
|
126
|
+
print(cursor.fetchall())
|
|
127
|
+
|
|
128
|
+
if ``%`` character is contained in your query, it must be escaped with ``%%`` like the following:
|
|
129
|
+
|
|
130
|
+
.. code:: sql
|
|
131
|
+
|
|
132
|
+
SELECT col_string FROM one_row_complex
|
|
133
|
+
WHERE col_string = %(param)s OR col_string LIKE 'a%%'
|
|
134
|
+
|
|
135
|
+
.. _`DB API paramstyle`: https://www.python.org/dev/peps/pep-0249/#paramstyle
|
|
136
|
+
.. _`named placeholders`: https://pyformat.info/#named_placeholders
|
|
137
|
+
|
|
138
|
+
SQLAlchemy
|
|
139
|
+
~~~~~~~~~~
|
|
140
|
+
|
|
141
|
+
Install SQLAlchemy with ``pip install SQLAlchemy>=1.0.0`` or ``pip install PyAthena[SQLAlchemy]``.
|
|
142
|
+
Supported SQLAlchemy is 1.0.0 or higher.
|
|
143
|
+
|
|
144
|
+
.. code:: python
|
|
145
|
+
|
|
146
|
+
from urllib.parse import quote_plus # PY2: from urllib import quote_plus
|
|
147
|
+
from sqlalchemy.engine import create_engine
|
|
148
|
+
from sqlalchemy.sql.expression import select
|
|
149
|
+
from sqlalchemy.sql.functions import func
|
|
150
|
+
from sqlalchemy.sql.schema import Table, MetaData
|
|
151
|
+
|
|
152
|
+
conn_str = 'awsathena+rest://{aws_access_key_id}:{aws_secret_access_key}@athena.{region_name}.amazonaws.com:443/'\
|
|
153
|
+
'{schema_name}?s3_staging_dir={s3_staging_dir}'
|
|
154
|
+
engine = create_engine(conn_str.format(
|
|
155
|
+
aws_access_key_id=quote_plus('YOUR_ACCESS_KEY_ID'),
|
|
156
|
+
aws_secret_access_key=quote_plus('YOUR_SECRET_ACCESS_KEY'),
|
|
157
|
+
region_name='us-west-2',
|
|
158
|
+
schema_name='default',
|
|
159
|
+
s3_staging_dir=quote_plus('s3://YOUR_S3_BUCKET/path/to/')))
|
|
160
|
+
many_rows = Table('many_rows', MetaData(bind=engine), autoload=True)
|
|
161
|
+
print(select([func.count('*')], from_obj=many_rows).scalar())
|
|
162
|
+
|
|
163
|
+
The connection string has the following format:
|
|
164
|
+
|
|
165
|
+
.. code:: python
|
|
166
|
+
|
|
167
|
+
awsathena+rest://{aws_access_key_id}:{aws_secret_access_key}@athena.{region_name}.amazonaws.com:443/{schema_name}?s3_staging_dir={s3_staging_dir}&...
|
|
168
|
+
|
|
169
|
+
NOTE: ``s3_staging_dir`` requires quote. If ``aws_access_key_id``, ``aws_secret_access_key`` and other parameter contain special characters, quote is also required.
|
|
170
|
+
|
|
171
|
+
Pandas
|
|
172
|
+
~~~~~~
|
|
173
|
+
|
|
174
|
+
Minimal example for Pandas DataFrame:
|
|
175
|
+
|
|
176
|
+
.. code:: python
|
|
177
|
+
|
|
178
|
+
from pyathena import connect
|
|
179
|
+
import pandas as pd
|
|
180
|
+
|
|
181
|
+
conn = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
|
|
182
|
+
aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
|
|
183
|
+
s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
|
|
184
|
+
region_name='us-west-2')
|
|
185
|
+
df = pd.read_sql("SELECT * FROM many_rows", conn)
|
|
186
|
+
print(df.head())
|
|
187
|
+
|
|
188
|
+
As Pandas DataFrame:
|
|
189
|
+
|
|
190
|
+
.. code:: python
|
|
191
|
+
|
|
192
|
+
from pyathena import connect
|
|
193
|
+
from pyathena.util import as_pandas
|
|
194
|
+
|
|
195
|
+
cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
|
|
196
|
+
aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
|
|
197
|
+
s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
|
|
198
|
+
region_name='us-west-2').cursor()
|
|
199
|
+
cursor.execute("SELECT * FROM many_rows")
|
|
200
|
+
df = as_pandas(cursor)
|
|
201
|
+
print(df.describe())
|
|
202
|
+
|
|
203
|
+
Credentials
|
|
204
|
+
-----------
|
|
205
|
+
|
|
206
|
+
Support `Boto3 credentials`_.
|
|
207
|
+
|
|
208
|
+
.. _`Boto3 credentials`: http://boto3.readthedocs.io/en/latest/guide/configuration.html
|
|
209
|
+
|
|
210
|
+
Additional environment variable:
|
|
211
|
+
|
|
212
|
+
.. code:: bash
|
|
213
|
+
|
|
214
|
+
$ export AWS_ATHENA_S3_STAGING_DIR=s3://YOUR_S3_BUCKET/path/to/
|
|
215
|
+
|
|
216
|
+
Testing
|
|
217
|
+
-------
|
|
218
|
+
|
|
219
|
+
Depends on the AWS CLI credentials and the following environment variables:
|
|
220
|
+
|
|
221
|
+
~/.aws/credentials
|
|
222
|
+
|
|
223
|
+
.. code:: cfg
|
|
224
|
+
|
|
225
|
+
[default]
|
|
226
|
+
aws_access_key_id=YOUR_ACCESS_KEY_ID
|
|
227
|
+
aws_secret_access_key=YOUR_SECRET_ACCESS_KEY
|
|
228
|
+
|
|
229
|
+
Environment variables
|
|
230
|
+
|
|
231
|
+
.. code:: bash
|
|
232
|
+
|
|
233
|
+
$ export AWS_DEFAULT_REGION=us-west-2
|
|
234
|
+
$ export AWS_ATHENA_S3_STAGING_DIR=s3://YOUR_S3_BUCKET/path/to/
|
|
235
|
+
|
|
236
|
+
Run test
|
|
237
|
+
~~~~~~~~
|
|
238
|
+
|
|
239
|
+
.. code:: bash
|
|
240
|
+
|
|
241
|
+
$ pip install pytest awscli
|
|
242
|
+
$ scripts/upload_test_data.sh
|
|
243
|
+
$ py.test
|
|
244
|
+
$ scripts/delete_test_data.sh
|
|
245
|
+
|
|
246
|
+
Run test multiple Python versions
|
|
247
|
+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
248
|
+
|
|
249
|
+
.. code:: bash
|
|
250
|
+
|
|
251
|
+
$ pip install tox awscli
|
|
252
|
+
$ scripts/upload_test_data.sh
|
|
253
|
+
$ pyenv local 2.6.9 2.7.13 3.4.6 3.5.3 3.6.1
|
|
254
|
+
$ tox
|
|
255
|
+
$ scripts/delete_test_data.sh
|
|
256
|
+
|
|
257
|
+
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
PyAthena-1.0.0.dist-info/DESCRIPTION.rst,sha256=EvZHmmTpejhi1-QDdJrTbd4o3BEu1yEC7-J3HmZb1l4,6838
|
|
2
|
+
PyAthena-1.0.0.dist-info/METADATA,sha256=9jlmteDaLjsYTwjj9m2VZXu9UmWwviu7Sg5pDd81mLo,7946
|
|
3
|
+
PyAthena-1.0.0.dist-info/RECORD,,
|
|
4
|
+
PyAthena-1.0.0.dist-info/WHEEL,sha256=o2k-Qa-RMNIJmUdIc7KU6VWR_ErNRbWNlxDIpl7lm34,110
|
|
5
|
+
PyAthena-1.0.0.dist-info/entry_points.txt,sha256=hv9yQKXWU8PfSjt6VW1TehGibxQ7YMxA8rjzcymsu14,81
|
|
6
|
+
PyAthena-1.0.0.dist-info/metadata.json,sha256=wdSLmcH25CoRskLOJ5GhBSTC1RPxK1MVhozNsAn17IY,1432
|
|
7
|
+
PyAthena-1.0.0.dist-info/top_level.txt,sha256=OlJSojWdSb65h3SVnNPAjdz3J_9AGK3etvFKw5wxAyk,9
|
|
8
|
+
pyathena/__init__.py,sha256=hnTtiH2zOKXyWyvZ_F8ddr1NjlX_elnLAIQskJtyaC8,1654
|
|
9
|
+
pyathena/connection.py,sha256=S5rLuaCbLklZPJ0CLVgwUZIWf5QA4mtGjkid-dblstg,2131
|
|
10
|
+
pyathena/converter.py,sha256=4EUmV4VpJ5qi-HN0jkQFH2thpRC-Sg_sbUfNf0RsT9Y,2107
|
|
11
|
+
pyathena/cursor.py,sha256=yRCy9bXnDQ19GLTraiWeki3sZccue_cEh_jeFaYIAl0,8995
|
|
12
|
+
pyathena/error.py,sha256=hz5wBgfQGf5PSrWK31LVvWc0gCrg0WXkCPgDfCeeZUI,713
|
|
13
|
+
pyathena/formatter.py,sha256=IN6y3eh0lafHyhXs-SRTIguPcQHJaNY5Y_zm-4TgjTM,3718
|
|
14
|
+
pyathena/sqlalchemy_athena.py,sha256=Ec5bjxM9isgd_EFumjMFS64ZrBWhu8gf77NNG2mwEOE,5789
|
|
15
|
+
pyathena/util.py,sha256=5pZ8wFJbkNPbKSgMOS2Ml43qjjXWF58qsQBXusREUXU,605
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"classifiers": ["Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Topic :: Database :: Front-Ends", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6"], "extensions": {"python.details": {"contacts": [{"email": "laughingman7743@gmail.com", "name": "laughingman7743", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}, "project_urls": {"Home": "https://github.com/laughingman7743/PyAthena/"}}, "python.exports": {"sqlalchemy.dialects": {"awsathena.rest": "pyathena.sqlalchemy_athena:AthenaDialect"}}}, "extras": ["Pandas", "SQLAlchemy"], "generator": "bdist_wheel (0.29.0)", "license": "MIT License", "metadata_version": "2.0", "name": "PyAthena", "run_requires": [{"extra": "SQLAlchemy", "requires": ["SQLAlchemy (>=1.0.0)"]}, {"requires": ["boto3 (>=1.4.4)", "botocore (>=1.5.52)", "future"]}, {"extra": "Pandas", "requires": ["pandas (>=0.19.0)"]}], "summary": "Python DB API 2.0 (PEP 249) compliant client for Amazon Athena", "test_requires": [{"requires": ["SQLAlchemy (>=1.0.0)", "futures", "pytest", "pytest-catchlog", "pytest-cov", "pytest-flake8"]}], "version": "1.0.0"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
pyathena
|
pyathena/__init__.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
from __future__ import absolute_import
|
|
3
|
+
from __future__ import unicode_literals
|
|
4
|
+
import datetime
|
|
5
|
+
|
|
6
|
+
from pyathena.error import * # noqa
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
__version__ = '1.0.0'
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# Globals https://www.python.org/dev/peps/pep-0249/#globals
|
|
13
|
+
apilevel = '2.0'
|
|
14
|
+
threadsafety = 3
|
|
15
|
+
paramstyle = 'pyformat'
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class DBAPITypeObject:
|
|
19
|
+
"""Type Objects and Constructors
|
|
20
|
+
|
|
21
|
+
https://www.python.org/dev/peps/pep-0249/#type-objects-and-constructors
|
|
22
|
+
"""
|
|
23
|
+
def __init__(self, *values):
|
|
24
|
+
self.values = values
|
|
25
|
+
|
|
26
|
+
def __cmp__(self, other):
|
|
27
|
+
if other in self.values:
|
|
28
|
+
return 0
|
|
29
|
+
if other < self.values:
|
|
30
|
+
return 1
|
|
31
|
+
else:
|
|
32
|
+
return -1
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
STRING = DBAPITypeObject('CHAR', 'NCHAR',
|
|
36
|
+
'VARCHAR', 'NVARCHAR',
|
|
37
|
+
'LONGVARCHAR', 'LONGNVARCHAR')
|
|
38
|
+
BINARY = DBAPITypeObject('BINARY', 'VARBINARY', 'LONGVARBINARY')
|
|
39
|
+
NUMBER = DBAPITypeObject('BOOLEAN', 'TINYINT', 'SMALLINT', 'BIGINT', 'INTEGER',
|
|
40
|
+
'REAL', 'DOUBLE', 'FLOAT', 'DECIMAL', 'NUMERIC')
|
|
41
|
+
DATETIME = DBAPITypeObject('TIMESTAMP')
|
|
42
|
+
ROWID = DBAPITypeObject('')
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
Date = datetime.date
|
|
46
|
+
Time = datetime.time
|
|
47
|
+
Timestamp = datetime.datetime
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def connect(s3_staging_dir=None, region_name=None, schema_name='default',
|
|
51
|
+
poll_interval=1, encryption_option=None, kms_key=None, profile_name=None,
|
|
52
|
+
converter=None, formatter=None, **kwargs):
|
|
53
|
+
from pyathena.connection import Connection
|
|
54
|
+
return Connection(s3_staging_dir, region_name, schema_name,
|
|
55
|
+
poll_interval, encryption_option, kms_key, profile_name,
|
|
56
|
+
converter, formatter, **kwargs)
|
pyathena/connection.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
from __future__ import absolute_import
|
|
3
|
+
from __future__ import unicode_literals
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
import boto3
|
|
8
|
+
from boto3.session import Session
|
|
9
|
+
|
|
10
|
+
from pyathena.converter import TypeConverter
|
|
11
|
+
from pyathena.cursor import Cursor
|
|
12
|
+
from pyathena.error import NotSupportedError
|
|
13
|
+
from pyathena.formatter import ParameterFormatter
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
_logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Connection(object):
|
|
20
|
+
|
|
21
|
+
_ENV_S3_STAGING_DIR = 'AWS_ATHENA_S3_STAGING_DIR'
|
|
22
|
+
|
|
23
|
+
def __init__(self, s3_staging_dir=None, region_name=None, schema_name='default',
|
|
24
|
+
poll_interval=1, encryption_option=None, kms_key=None, profile_name=None,
|
|
25
|
+
converter=None, formatter=None, **kwargs):
|
|
26
|
+
if s3_staging_dir:
|
|
27
|
+
self.s3_staging_dir = s3_staging_dir
|
|
28
|
+
else:
|
|
29
|
+
self.s3_staging_dir = os.getenv(self._ENV_S3_STAGING_DIR, None)
|
|
30
|
+
assert self.s3_staging_dir, 'Required argument `s3_staging_dir` not found.'
|
|
31
|
+
assert schema_name, 'Required argument `schema_name` not found.'
|
|
32
|
+
self.region_name = region_name
|
|
33
|
+
self.schema_name = schema_name
|
|
34
|
+
self.poll_interval = poll_interval
|
|
35
|
+
self.encryption_option = encryption_option
|
|
36
|
+
self.kms_key = kms_key
|
|
37
|
+
|
|
38
|
+
if profile_name:
|
|
39
|
+
session = Session(profile_name=profile_name, **kwargs)
|
|
40
|
+
self._client = session.client('athena', region_name=region_name, **kwargs)
|
|
41
|
+
else:
|
|
42
|
+
self._client = boto3.client('athena', region_name=region_name, **kwargs)
|
|
43
|
+
|
|
44
|
+
self._converter = converter if converter else TypeConverter()
|
|
45
|
+
self._formatter = formatter if formatter else ParameterFormatter()
|
|
46
|
+
|
|
47
|
+
def __enter__(self):
|
|
48
|
+
return self.cursor()
|
|
49
|
+
|
|
50
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
51
|
+
self.close()
|
|
52
|
+
|
|
53
|
+
def cursor(self):
|
|
54
|
+
return Cursor(self._client, self.s3_staging_dir, self.schema_name, self.poll_interval,
|
|
55
|
+
self.encryption_option, self.kms_key, self._converter, self._formatter)
|
|
56
|
+
|
|
57
|
+
def close(self):
|
|
58
|
+
pass
|
|
59
|
+
|
|
60
|
+
def commit(self):
|
|
61
|
+
pass
|
|
62
|
+
|
|
63
|
+
def rollback(self):
|
|
64
|
+
raise NotSupportedError
|