pyathena 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1 @@
1
+ include README.rst
@@ -0,0 +1,248 @@
1
+ Metadata-Version: 1.1
2
+ Name: PyAthena
3
+ Version: 1.0.0
4
+ Summary: Python DB API 2.0 (PEP 249) compliant client for Amazon Athena
5
+ Home-page: https://github.com/laughingman7743/PyAthena/
6
+ Author: laughingman7743
7
+ Author-email: laughingman7743@gmail.com
8
+ License: MIT License
9
+ Description: .. image:: https://img.shields.io/pypi/pyversions/PyAthena.svg
10
+ :target: https://pypi.python.org/pypi/PyAthena/
11
+
12
+ .. image:: https://circleci.com/gh/laughingman7743/PyAthena.svg?style=shield
13
+ :target: https://circleci.com/gh/laughingman7743/PyAthena
14
+
15
+ .. image:: https://codecov.io/gh/laughingman7743/PyAthena/branch/master/graph/badge.svg
16
+ :target: https://codecov.io/gh/laughingman7743/PyAthena
17
+
18
+ .. image:: https://img.shields.io/pypi/l/PyAthena.svg
19
+ :target: https://github.com/laughingman7743/PyAthena/blob/master/LICENSE
20
+
21
+
22
+ PyAthena
23
+ ========
24
+
25
+ PyAthena is a Python `DB API 2.0 (PEP 249)`_ compliant client for `Amazon Athena`_.
26
+
27
+ .. _`DB API 2.0 (PEP 249)`: https://www.python.org/dev/peps/pep-0249/
28
+ .. _`Amazon Athena`: http://docs.aws.amazon.com/athena/latest/APIReference/Welcome.html
29
+
30
+ Requirements
31
+ ------------
32
+
33
+ * Python
34
+
35
+ - CPython 2.6, 2,7, 3,4, 3.5, 3.6
36
+
37
+ Installation
38
+ ------------
39
+
40
+ .. code:: bash
41
+
42
+ $ pip install PyAthena
43
+
44
+ Extra packages:
45
+
46
+ +---------------+--------------------------------------+----------+
47
+ | Package | Install command | Version |
48
+ +===============+======================================+==========+
49
+ | Pandas | ``pip install PyAthena[Pandas]`` | >=0.19.0 |
50
+ +---------------+--------------------------------------+----------+
51
+ | SQLAlchemy | ``pip install PyAthena[SQLAlchemy]`` | >=1.0.0 |
52
+ +---------------+--------------------------------------+----------+
53
+
54
+ Usage
55
+ -----
56
+
57
+ Basic usage
58
+ ~~~~~~~~~~~
59
+
60
+ .. code:: python
61
+
62
+ from pyathena import connect
63
+
64
+ cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
65
+ aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
66
+ s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
67
+ region_name='us-west-2').cursor()
68
+ cursor.execute("SELECT * FROM one_row")
69
+ print(cursor.description)
70
+ print(cursor.fetchall())
71
+
72
+ Cursor iteration
73
+ ~~~~~~~~~~~~~~~~
74
+
75
+ .. code:: python
76
+
77
+ from pyathena import connect
78
+
79
+ cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
80
+ aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
81
+ s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
82
+ region_name='us-west-2').cursor()
83
+ cursor.execute("SELECT * FROM many_rows LIMIT 10")
84
+ for row in cursor:
85
+ print(row)
86
+
87
+ Query with parameter
88
+ ~~~~~~~~~~~~~~~~~~~~
89
+
90
+ Supported `DB API paramstyle`_ is only ``PyFormat``.
91
+ ``PyFormat`` only supports `named placeholders`_ with old ``%`` operator style and parameters specify dictionary format.
92
+
93
+ .. code:: python
94
+
95
+ from pyathena import connect
96
+
97
+ cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
98
+ aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
99
+ s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
100
+ region_name='us-west-2').cursor()
101
+ cursor.execute("""
102
+ SELECT col_string FROM one_row_complex
103
+ WHERE col_string = %(param)s
104
+ """, {'param': 'a string'})
105
+ print(cursor.fetchall())
106
+
107
+ if ``%`` character is contained in your query, it must be escaped with ``%%`` like the following:
108
+
109
+ .. code:: sql
110
+
111
+ SELECT col_string FROM one_row_complex
112
+ WHERE col_string = %(param)s OR col_string LIKE 'a%%'
113
+
114
+ .. _`DB API paramstyle`: https://www.python.org/dev/peps/pep-0249/#paramstyle
115
+ .. _`named placeholders`: https://pyformat.info/#named_placeholders
116
+
117
+ SQLAlchemy
118
+ ~~~~~~~~~~
119
+
120
+ Install SQLAlchemy with ``pip install SQLAlchemy>=1.0.0`` or ``pip install PyAthena[SQLAlchemy]``.
121
+ Supported SQLAlchemy is 1.0.0 or higher.
122
+
123
+ .. code:: python
124
+
125
+ from urllib.parse import quote_plus # PY2: from urllib import quote_plus
126
+ from sqlalchemy.engine import create_engine
127
+ from sqlalchemy.sql.expression import select
128
+ from sqlalchemy.sql.functions import func
129
+ from sqlalchemy.sql.schema import Table, MetaData
130
+
131
+ conn_str = 'awsathena+rest://{aws_access_key_id}:{aws_secret_access_key}@athena.{region_name}.amazonaws.com:443/'\
132
+ '{schema_name}?s3_staging_dir={s3_staging_dir}'
133
+ engine = create_engine(conn_str.format(
134
+ aws_access_key_id=quote_plus('YOUR_ACCESS_KEY_ID'),
135
+ aws_secret_access_key=quote_plus('YOUR_SECRET_ACCESS_KEY'),
136
+ region_name='us-west-2',
137
+ schema_name='default',
138
+ s3_staging_dir=quote_plus('s3://YOUR_S3_BUCKET/path/to/')))
139
+ many_rows = Table('many_rows', MetaData(bind=engine), autoload=True)
140
+ print(select([func.count('*')], from_obj=many_rows).scalar())
141
+
142
+ The connection string has the following format:
143
+
144
+ .. code:: python
145
+
146
+ awsathena+rest://{aws_access_key_id}:{aws_secret_access_key}@athena.{region_name}.amazonaws.com:443/{schema_name}?s3_staging_dir={s3_staging_dir}&...
147
+
148
+ NOTE: ``s3_staging_dir`` requires quote. If ``aws_access_key_id``, ``aws_secret_access_key`` and other parameter contain special characters, quote is also required.
149
+
150
+ Pandas
151
+ ~~~~~~
152
+
153
+ Minimal example for Pandas DataFrame:
154
+
155
+ .. code:: python
156
+
157
+ from pyathena import connect
158
+ import pandas as pd
159
+
160
+ conn = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
161
+ aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
162
+ s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
163
+ region_name='us-west-2')
164
+ df = pd.read_sql("SELECT * FROM many_rows", conn)
165
+ print(df.head())
166
+
167
+ As Pandas DataFrame:
168
+
169
+ .. code:: python
170
+
171
+ from pyathena import connect
172
+ from pyathena.util import as_pandas
173
+
174
+ cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
175
+ aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
176
+ s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
177
+ region_name='us-west-2').cursor()
178
+ cursor.execute("SELECT * FROM many_rows")
179
+ df = as_pandas(cursor)
180
+ print(df.describe())
181
+
182
+ Credentials
183
+ -----------
184
+
185
+ Support `Boto3 credentials`_.
186
+
187
+ .. _`Boto3 credentials`: http://boto3.readthedocs.io/en/latest/guide/configuration.html
188
+
189
+ Additional environment variable:
190
+
191
+ .. code:: bash
192
+
193
+ $ export AWS_ATHENA_S3_STAGING_DIR=s3://YOUR_S3_BUCKET/path/to/
194
+
195
+ Testing
196
+ -------
197
+
198
+ Depends on the AWS CLI credentials and the following environment variables:
199
+
200
+ ~/.aws/credentials
201
+
202
+ .. code:: cfg
203
+
204
+ [default]
205
+ aws_access_key_id=YOUR_ACCESS_KEY_ID
206
+ aws_secret_access_key=YOUR_SECRET_ACCESS_KEY
207
+
208
+ Environment variables
209
+
210
+ .. code:: bash
211
+
212
+ $ export AWS_DEFAULT_REGION=us-west-2
213
+ $ export AWS_ATHENA_S3_STAGING_DIR=s3://YOUR_S3_BUCKET/path/to/
214
+
215
+ Run test
216
+ ~~~~~~~~
217
+
218
+ .. code:: bash
219
+
220
+ $ pip install pytest awscli
221
+ $ scripts/upload_test_data.sh
222
+ $ py.test
223
+ $ scripts/delete_test_data.sh
224
+
225
+ Run test multiple Python versions
226
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
227
+
228
+ .. code:: bash
229
+
230
+ $ pip install tox awscli
231
+ $ scripts/upload_test_data.sh
232
+ $ pyenv local 2.6.9 2.7.13 3.4.6 3.5.3 3.6.1
233
+ $ tox
234
+ $ scripts/delete_test_data.sh
235
+
236
+ Platform: UNKNOWN
237
+ Classifier: Development Status :: 4 - Beta
238
+ Classifier: Intended Audience :: Developers
239
+ Classifier: License :: OSI Approved :: MIT License
240
+ Classifier: Operating System :: OS Independent
241
+ Classifier: Topic :: Database :: Front-Ends
242
+ Classifier: Programming Language :: Python :: 2
243
+ Classifier: Programming Language :: Python :: 2.6
244
+ Classifier: Programming Language :: Python :: 2.7
245
+ Classifier: Programming Language :: Python :: 3
246
+ Classifier: Programming Language :: Python :: 3.4
247
+ Classifier: Programming Language :: Python :: 3.5
248
+ Classifier: Programming Language :: Python :: 3.6
@@ -0,0 +1,248 @@
1
+ Metadata-Version: 1.1
2
+ Name: PyAthena
3
+ Version: 1.0.0
4
+ Summary: Python DB API 2.0 (PEP 249) compliant client for Amazon Athena
5
+ Home-page: https://github.com/laughingman7743/PyAthena/
6
+ Author: laughingman7743
7
+ Author-email: laughingman7743@gmail.com
8
+ License: MIT License
9
+ Description: .. image:: https://img.shields.io/pypi/pyversions/PyAthena.svg
10
+ :target: https://pypi.python.org/pypi/PyAthena/
11
+
12
+ .. image:: https://circleci.com/gh/laughingman7743/PyAthena.svg?style=shield
13
+ :target: https://circleci.com/gh/laughingman7743/PyAthena
14
+
15
+ .. image:: https://codecov.io/gh/laughingman7743/PyAthena/branch/master/graph/badge.svg
16
+ :target: https://codecov.io/gh/laughingman7743/PyAthena
17
+
18
+ .. image:: https://img.shields.io/pypi/l/PyAthena.svg
19
+ :target: https://github.com/laughingman7743/PyAthena/blob/master/LICENSE
20
+
21
+
22
+ PyAthena
23
+ ========
24
+
25
+ PyAthena is a Python `DB API 2.0 (PEP 249)`_ compliant client for `Amazon Athena`_.
26
+
27
+ .. _`DB API 2.0 (PEP 249)`: https://www.python.org/dev/peps/pep-0249/
28
+ .. _`Amazon Athena`: http://docs.aws.amazon.com/athena/latest/APIReference/Welcome.html
29
+
30
+ Requirements
31
+ ------------
32
+
33
+ * Python
34
+
35
+ - CPython 2.6, 2,7, 3,4, 3.5, 3.6
36
+
37
+ Installation
38
+ ------------
39
+
40
+ .. code:: bash
41
+
42
+ $ pip install PyAthena
43
+
44
+ Extra packages:
45
+
46
+ +---------------+--------------------------------------+----------+
47
+ | Package | Install command | Version |
48
+ +===============+======================================+==========+
49
+ | Pandas | ``pip install PyAthena[Pandas]`` | >=0.19.0 |
50
+ +---------------+--------------------------------------+----------+
51
+ | SQLAlchemy | ``pip install PyAthena[SQLAlchemy]`` | >=1.0.0 |
52
+ +---------------+--------------------------------------+----------+
53
+
54
+ Usage
55
+ -----
56
+
57
+ Basic usage
58
+ ~~~~~~~~~~~
59
+
60
+ .. code:: python
61
+
62
+ from pyathena import connect
63
+
64
+ cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
65
+ aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
66
+ s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
67
+ region_name='us-west-2').cursor()
68
+ cursor.execute("SELECT * FROM one_row")
69
+ print(cursor.description)
70
+ print(cursor.fetchall())
71
+
72
+ Cursor iteration
73
+ ~~~~~~~~~~~~~~~~
74
+
75
+ .. code:: python
76
+
77
+ from pyathena import connect
78
+
79
+ cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
80
+ aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
81
+ s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
82
+ region_name='us-west-2').cursor()
83
+ cursor.execute("SELECT * FROM many_rows LIMIT 10")
84
+ for row in cursor:
85
+ print(row)
86
+
87
+ Query with parameter
88
+ ~~~~~~~~~~~~~~~~~~~~
89
+
90
+ Supported `DB API paramstyle`_ is only ``PyFormat``.
91
+ ``PyFormat`` only supports `named placeholders`_ with old ``%`` operator style and parameters specify dictionary format.
92
+
93
+ .. code:: python
94
+
95
+ from pyathena import connect
96
+
97
+ cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
98
+ aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
99
+ s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
100
+ region_name='us-west-2').cursor()
101
+ cursor.execute("""
102
+ SELECT col_string FROM one_row_complex
103
+ WHERE col_string = %(param)s
104
+ """, {'param': 'a string'})
105
+ print(cursor.fetchall())
106
+
107
+ if ``%`` character is contained in your query, it must be escaped with ``%%`` like the following:
108
+
109
+ .. code:: sql
110
+
111
+ SELECT col_string FROM one_row_complex
112
+ WHERE col_string = %(param)s OR col_string LIKE 'a%%'
113
+
114
+ .. _`DB API paramstyle`: https://www.python.org/dev/peps/pep-0249/#paramstyle
115
+ .. _`named placeholders`: https://pyformat.info/#named_placeholders
116
+
117
+ SQLAlchemy
118
+ ~~~~~~~~~~
119
+
120
+ Install SQLAlchemy with ``pip install SQLAlchemy>=1.0.0`` or ``pip install PyAthena[SQLAlchemy]``.
121
+ Supported SQLAlchemy is 1.0.0 or higher.
122
+
123
+ .. code:: python
124
+
125
+ from urllib.parse import quote_plus # PY2: from urllib import quote_plus
126
+ from sqlalchemy.engine import create_engine
127
+ from sqlalchemy.sql.expression import select
128
+ from sqlalchemy.sql.functions import func
129
+ from sqlalchemy.sql.schema import Table, MetaData
130
+
131
+ conn_str = 'awsathena+rest://{aws_access_key_id}:{aws_secret_access_key}@athena.{region_name}.amazonaws.com:443/'\
132
+ '{schema_name}?s3_staging_dir={s3_staging_dir}'
133
+ engine = create_engine(conn_str.format(
134
+ aws_access_key_id=quote_plus('YOUR_ACCESS_KEY_ID'),
135
+ aws_secret_access_key=quote_plus('YOUR_SECRET_ACCESS_KEY'),
136
+ region_name='us-west-2',
137
+ schema_name='default',
138
+ s3_staging_dir=quote_plus('s3://YOUR_S3_BUCKET/path/to/')))
139
+ many_rows = Table('many_rows', MetaData(bind=engine), autoload=True)
140
+ print(select([func.count('*')], from_obj=many_rows).scalar())
141
+
142
+ The connection string has the following format:
143
+
144
+ .. code:: python
145
+
146
+ awsathena+rest://{aws_access_key_id}:{aws_secret_access_key}@athena.{region_name}.amazonaws.com:443/{schema_name}?s3_staging_dir={s3_staging_dir}&...
147
+
148
+ NOTE: ``s3_staging_dir`` requires quote. If ``aws_access_key_id``, ``aws_secret_access_key`` and other parameter contain special characters, quote is also required.
149
+
150
+ Pandas
151
+ ~~~~~~
152
+
153
+ Minimal example for Pandas DataFrame:
154
+
155
+ .. code:: python
156
+
157
+ from pyathena import connect
158
+ import pandas as pd
159
+
160
+ conn = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
161
+ aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
162
+ s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
163
+ region_name='us-west-2')
164
+ df = pd.read_sql("SELECT * FROM many_rows", conn)
165
+ print(df.head())
166
+
167
+ As Pandas DataFrame:
168
+
169
+ .. code:: python
170
+
171
+ from pyathena import connect
172
+ from pyathena.util import as_pandas
173
+
174
+ cursor = connect(aws_access_key_id='YOUR_ACCESS_KEY_ID',
175
+ aws_secret_access_key='YOUR_SECRET_ACCESS_KEY',
176
+ s3_staging_dir='s3://YOUR_S3_BUCKET/path/to/',
177
+ region_name='us-west-2').cursor()
178
+ cursor.execute("SELECT * FROM many_rows")
179
+ df = as_pandas(cursor)
180
+ print(df.describe())
181
+
182
+ Credentials
183
+ -----------
184
+
185
+ Support `Boto3 credentials`_.
186
+
187
+ .. _`Boto3 credentials`: http://boto3.readthedocs.io/en/latest/guide/configuration.html
188
+
189
+ Additional environment variable:
190
+
191
+ .. code:: bash
192
+
193
+ $ export AWS_ATHENA_S3_STAGING_DIR=s3://YOUR_S3_BUCKET/path/to/
194
+
195
+ Testing
196
+ -------
197
+
198
+ Depends on the AWS CLI credentials and the following environment variables:
199
+
200
+ ~/.aws/credentials
201
+
202
+ .. code:: cfg
203
+
204
+ [default]
205
+ aws_access_key_id=YOUR_ACCESS_KEY_ID
206
+ aws_secret_access_key=YOUR_SECRET_ACCESS_KEY
207
+
208
+ Environment variables
209
+
210
+ .. code:: bash
211
+
212
+ $ export AWS_DEFAULT_REGION=us-west-2
213
+ $ export AWS_ATHENA_S3_STAGING_DIR=s3://YOUR_S3_BUCKET/path/to/
214
+
215
+ Run test
216
+ ~~~~~~~~
217
+
218
+ .. code:: bash
219
+
220
+ $ pip install pytest awscli
221
+ $ scripts/upload_test_data.sh
222
+ $ py.test
223
+ $ scripts/delete_test_data.sh
224
+
225
+ Run test multiple Python versions
226
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
227
+
228
+ .. code:: bash
229
+
230
+ $ pip install tox awscli
231
+ $ scripts/upload_test_data.sh
232
+ $ pyenv local 2.6.9 2.7.13 3.4.6 3.5.3 3.6.1
233
+ $ tox
234
+ $ scripts/delete_test_data.sh
235
+
236
+ Platform: UNKNOWN
237
+ Classifier: Development Status :: 4 - Beta
238
+ Classifier: Intended Audience :: Developers
239
+ Classifier: License :: OSI Approved :: MIT License
240
+ Classifier: Operating System :: OS Independent
241
+ Classifier: Topic :: Database :: Front-Ends
242
+ Classifier: Programming Language :: Python :: 2
243
+ Classifier: Programming Language :: Python :: 2.6
244
+ Classifier: Programming Language :: Python :: 2.7
245
+ Classifier: Programming Language :: Python :: 3
246
+ Classifier: Programming Language :: Python :: 3.4
247
+ Classifier: Programming Language :: Python :: 3.5
248
+ Classifier: Programming Language :: Python :: 3.6
@@ -0,0 +1,19 @@
1
+ MANIFEST.in
2
+ README.rst
3
+ setup.cfg
4
+ setup.py
5
+ PyAthena.egg-info/PKG-INFO
6
+ PyAthena.egg-info/SOURCES.txt
7
+ PyAthena.egg-info/dependency_links.txt
8
+ PyAthena.egg-info/entry_points.txt
9
+ PyAthena.egg-info/not-zip-safe
10
+ PyAthena.egg-info/requires.txt
11
+ PyAthena.egg-info/top_level.txt
12
+ pyathena/__init__.py
13
+ pyathena/connection.py
14
+ pyathena/converter.py
15
+ pyathena/cursor.py
16
+ pyathena/error.py
17
+ pyathena/formatter.py
18
+ pyathena/sqlalchemy_athena.py
19
+ pyathena/util.py
@@ -0,0 +1,3 @@
1
+ [sqlalchemy.dialects]
2
+ awsathena.rest = pyathena.sqlalchemy_athena:AthenaDialect
3
+
@@ -0,0 +1,9 @@
1
+ future
2
+ botocore>=1.5.52
3
+ boto3>=1.4.4
4
+
5
+ [Pandas]
6
+ pandas>=0.19.0
7
+
8
+ [SQLAlchemy]
9
+ SQLAlchemy>=1.0.0
@@ -0,0 +1 @@
1
+ pyathena