fullscript.spellbook 2.1.7__tar.gz → 3.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fullscript_spellbook-3.0.0/.gitignore +9 -0
- fullscript_spellbook-3.0.0/AGENTS.md +98 -0
- fullscript_spellbook-3.0.0/CLAUDE.md +1 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/PKG-INFO +15 -2
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/README.md +10 -0
- fullscript_spellbook-3.0.0/documentation/db_magic.md +254 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/fullscript.spellbook.egg-info/PKG-INFO +15 -2
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/fullscript.spellbook.egg-info/SOURCES.txt +9 -3
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/fullscript.spellbook.egg-info/requires.txt +4 -0
- fullscript_spellbook-3.0.0/fullscript.spellbook.egg-info/scm_file_list.json +43 -0
- fullscript_spellbook-3.0.0/fullscript.spellbook.egg-info/scm_version.json +8 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/pyproject.toml +10 -2
- fullscript_spellbook-3.0.0/spellbook/__init__.py +19 -0
- fullscript_spellbook-3.0.0/spellbook/db_magic.py +589 -0
- fullscript_spellbook-3.0.0/spellbook/exceptions.py +41 -0
- fullscript_spellbook-3.0.0/spellbook/utils.py +218 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook_config.yaml +11 -0
- fullscript_spellbook-3.0.0/tests/test_db_magic.py +499 -0
- fullscript_spellbook-3.0.0/tests/test_utils.py +224 -0
- fullscript_spellbook-2.1.7/.DS_Store +0 -0
- fullscript_spellbook-2.1.7/.gitignore +0 -3
- fullscript_spellbook-2.1.7/documentation/db_magic.md +0 -98
- fullscript_spellbook-2.1.7/spellbook/.DS_Store +0 -0
- fullscript_spellbook-2.1.7/spellbook/db_magic.py +0 -286
- fullscript_spellbook-2.1.7/spellbook/utils.py +0 -123
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/.gitlab-ci.yml +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/.python-version +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/LICENSE +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/MANIFEST.in +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/documentation/drive_magic.md +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/documentation/sheet_magic.md +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/documentation/snowflake_key_pair_setup.md +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/documentation/stats_magic.md +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/fullscript.spellbook.egg-info/dependency_links.txt +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/fullscript.spellbook.egg-info/top_level.txt +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/requirements.txt +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/setup.cfg +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/drive_magic.py +0 -0
- /fullscript_spellbook-2.1.7/spellbook/__init__.py → /fullscript_spellbook-3.0.0/spellbook/py.typed +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/sheet_magic.py +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/stats_magic.py +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/wizards/__init__.py +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/wizards/wizard1.txt +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/wizards/wizard10.txt +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/wizards/wizard11.txt +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/wizards/wizard12.txt +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/wizards/wizard2.txt +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/wizards/wizard3.txt +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/wizards/wizard4.txt +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/wizards/wizard5.txt +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/wizards/wizard6.txt +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/wizards/wizard7.txt +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/wizards/wizard8.txt +0 -0
- {fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/spellbook/wizards/wizard9.txt +0 -0
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# Working on Spellbook
|
|
2
|
+
|
|
3
|
+
Notes for anyone - person or coding agent - changing this package. This file covers the
|
|
4
|
+
things that are not obvious from reading the code.
|
|
5
|
+
|
|
6
|
+
## What this package is
|
|
7
|
+
|
|
8
|
+
A thin, friendly wrapper over Snowflake, MySQL, Google Sheets, Google Drive and some
|
|
9
|
+
statistics helpers, published publicly to PyPI as `fullscript.spellbook`.
|
|
10
|
+
|
|
11
|
+
**It is a public package.** Never commit Fullscript hostnames, database names, RDS instance
|
|
12
|
+
identifiers, usernames or internal URLs - not in code, examples, docs or tests. Use
|
|
13
|
+
placeholders (`${MYSQL_HOST}`, `your_rds_instance_name`) instead.
|
|
14
|
+
|
|
15
|
+
## Running the tests
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pip install -e '.[test]'
|
|
19
|
+
pytest
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Tests mock the database and `subprocess`, so they need no network and no credentials.
|
|
23
|
+
`tests/test_db_magic.py` covers connections and auth, `tests/test_utils.py` covers config
|
|
24
|
+
loading. Testing against a real database needs AWS SSO, which no automated run can do.
|
|
25
|
+
|
|
26
|
+
## Versioning
|
|
27
|
+
|
|
28
|
+
The version is **not** in a file. `setuptools_scm` derives it from git tags, so releasing
|
|
29
|
+
means tagging:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
git tag 3.0.0 && git push origin main --tags
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Pushing a tag triggers `.gitlab-ci.yml`, which publishes to both the internal registry and
|
|
36
|
+
public PyPI. Do not tag unless you intend to publish.
|
|
37
|
+
|
|
38
|
+
## Conventions worth keeping
|
|
39
|
+
|
|
40
|
+
**Errors raise, they do not return empty.** Every failure path raises a subclass of
|
|
41
|
+
`SpellbookError` from `spellbook/exceptions.py`. Returning `pd.DataFrame()` or `None` on
|
|
42
|
+
failure is the thing this package deliberately moved away from: a caller cannot tell
|
|
43
|
+
"matched no rows" from "never ran", which produces confidently wrong analysis. An empty
|
|
44
|
+
DataFrame must only ever mean zero rows.
|
|
45
|
+
|
|
46
|
+
**Decorative output goes to stderr, never stdout.** The wizard ASCII art is charm, not
|
|
47
|
+
data. `utils.show_wizard()` honours `SPELLBOOK_QUIET=1`; set that when calling Spellbook
|
|
48
|
+
from a script or agent. Use `logging` for diagnostics rather than `print`.
|
|
49
|
+
|
|
50
|
+
**Function signatures are `(db_name, query)`, in that order.** `execute_mysql` once took
|
|
51
|
+
`(query, dbname)`, which silently broke every `run_sql` call against MySQL. Keep the order
|
|
52
|
+
and the `db_name` spelling consistent across every function.
|
|
53
|
+
|
|
54
|
+
**Bind parameters in catalogue queries.** `list_tables` and `describe_table` pass values
|
|
55
|
+
through SQLAlchemy parameters so a `%` or a quote in user input cannot reshape the query.
|
|
56
|
+
`preview_table` cannot bind a table name, so it validates it against a strict identifier
|
|
57
|
+
pattern instead.
|
|
58
|
+
|
|
59
|
+
## Traps that have cost time before
|
|
60
|
+
|
|
61
|
+
**Config file precedence.** `utils.find_config_file()` prefers `spellbook_config.yaml` in
|
|
62
|
+
the *current working directory* over the one in your home directory. Running anything from
|
|
63
|
+
this repository picks up the placeholder example config, not your real one. If a connection
|
|
64
|
+
fails with odd credentials, check which file was loaded.
|
|
65
|
+
|
|
66
|
+
**Two MySQL drivers.** Password auth uses `mysql+mysqlconnector`; AWS RDS IAM auth uses
|
|
67
|
+
`mysql+pymysql`, because only pymysql completes the IAM token handshake. They are not
|
|
68
|
+
interchangeable:
|
|
69
|
+
|
|
70
|
+
- pymysql substitutes parameters whenever `args is not None`, and pandas passes an empty
|
|
71
|
+
tuple, so a literal `%` in `like '%value%'` is read as a format character and raises.
|
|
72
|
+
mysql-connector guards on `if params:` instead. `db_magic._iam_cursor_class()` normalises
|
|
73
|
+
empty parameters to `None` so both drivers behave the same. Do not remove it without a
|
|
74
|
+
test that runs a `like '%...%'` query.
|
|
75
|
+
- An empty `ssl` dict makes TLS merely *preferred* in pymysql. The dict passed must be
|
|
76
|
+
truthy for TLS to be required.
|
|
77
|
+
|
|
78
|
+
**RDS IAM specifics.** Tokens expire in about 15 minutes, so they are fetched per
|
|
79
|
+
connection and never cached. `rx rds password` takes the *RDS instance* name, which usually
|
|
80
|
+
differs from the config alias - hence the optional `rds_db_name` field. The token is URL
|
|
81
|
+
shaped and must be `quote_plus`-escaped into the connection string.
|
|
82
|
+
|
|
83
|
+
**`iam_auth` is internal only.** It shells out to `rx`, which is Fullscript internal
|
|
84
|
+
tooling, so the feature cannot work for public users of this package. Keep password auth
|
|
85
|
+
working and keep the docs honest about the limitation.
|
|
86
|
+
|
|
87
|
+
## Repository layout
|
|
88
|
+
|
|
89
|
+
| Path | Purpose |
|
|
90
|
+
| --- | --- |
|
|
91
|
+
| `spellbook/db_magic.py` | Snowflake and MySQL connections, queries, schema helpers |
|
|
92
|
+
| `spellbook/exceptions.py` | The `SpellbookError` hierarchy |
|
|
93
|
+
| `spellbook/utils.py` | Config loading, env substitution, wizards, SQL file loading |
|
|
94
|
+
| `spellbook/sheet_magic.py` | Google Sheets |
|
|
95
|
+
| `spellbook/drive_magic.py` | Google Drive |
|
|
96
|
+
| `spellbook/stats_magic.py` | Statistical tests |
|
|
97
|
+
| `spellbook_config.yaml` | **Example** config, published - placeholders only |
|
|
98
|
+
| `documentation/` | User-facing docs per module |
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
See [AGENTS.md](AGENTS.md) for how to work on this package.
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fullscript.spellbook
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.0
|
|
4
4
|
Summary: A simple package to make your life easier.
|
|
5
|
-
Author-email: Thomas Edwards <thomas.edwards@fullscript.com>, Gui Chagas <gui.chagas@fullscript.com>, David King <david.king@fullscript.com>,
|
|
5
|
+
Author-email: Meinhard Mare <meinhard.mare@fullscript.com>, Thomas Edwards <thomas.edwards@fullscript.com>, Gui Chagas <gui.chagas@fullscript.com>, David King <david.king@fullscript.com>, Karsten Melchers <karsten.melchers@fullscript.com>
|
|
6
6
|
License: MIT
|
|
7
7
|
Project-URL: homepage, https://github.com/Fullscript/spellbook
|
|
8
8
|
Requires-Python: <3.12,>=3.10
|
|
@@ -46,6 +46,7 @@ Requires-Dist: pycparser==2.22
|
|
|
46
46
|
Requires-Dist: Pygments==2.18.0
|
|
47
47
|
Requires-Dist: PyJWT==2.9.0
|
|
48
48
|
Requires-Dist: pyOpenSSL==24.2.1
|
|
49
|
+
Requires-Dist: PyMySQL==1.2.3
|
|
49
50
|
Requires-Dist: python-dateutil==2.9.0.post0
|
|
50
51
|
Requires-Dist: pytz==2024.2
|
|
51
52
|
Requires-Dist: PyYAML==6.0.2
|
|
@@ -116,6 +117,8 @@ Requires-Dist: scipy==1.14.1; extra == "statsmagic"
|
|
|
116
117
|
Requires-Dist: seaborn==0.13.2; extra == "statsmagic"
|
|
117
118
|
Requires-Dist: six==1.16.0; extra == "statsmagic"
|
|
118
119
|
Requires-Dist: statsmodels==0.14.4; extra == "statsmagic"
|
|
120
|
+
Provides-Extra: test
|
|
121
|
+
Requires-Dist: pytest==8.3.4; extra == "test"
|
|
119
122
|
Dynamic: license-file
|
|
120
123
|
|
|
121
124
|
# Spellbook
|
|
@@ -186,6 +189,16 @@ databases:
|
|
|
186
189
|
database: snowflake_database
|
|
187
190
|
schema: snowflake_schema
|
|
188
191
|
|
|
192
|
+
# MySQL Database Connection using AWS RDS IAM authentication (no stored password).
|
|
193
|
+
# Fullscript internal only: requires the `rx` CLI. See documentation/db_magic.md.
|
|
194
|
+
- name: alias_of_your_connection
|
|
195
|
+
type: mysql
|
|
196
|
+
host: your_host.rds.amazonaws.com
|
|
197
|
+
port: 3306
|
|
198
|
+
user: ${DB_USER}
|
|
199
|
+
iam_auth: true
|
|
200
|
+
database: ${MYSQL_DATABASE}
|
|
201
|
+
|
|
189
202
|
google_accounts:
|
|
190
203
|
# Google Sheets Service Account 1
|
|
191
204
|
- name: google_sheets_service_account_1
|
|
@@ -66,6 +66,16 @@ databases:
|
|
|
66
66
|
database: snowflake_database
|
|
67
67
|
schema: snowflake_schema
|
|
68
68
|
|
|
69
|
+
# MySQL Database Connection using AWS RDS IAM authentication (no stored password).
|
|
70
|
+
# Fullscript internal only: requires the `rx` CLI. See documentation/db_magic.md.
|
|
71
|
+
- name: alias_of_your_connection
|
|
72
|
+
type: mysql
|
|
73
|
+
host: your_host.rds.amazonaws.com
|
|
74
|
+
port: 3306
|
|
75
|
+
user: ${DB_USER}
|
|
76
|
+
iam_auth: true
|
|
77
|
+
database: ${MYSQL_DATABASE}
|
|
78
|
+
|
|
69
79
|
google_accounts:
|
|
70
80
|
# Google Sheets Service Account 1
|
|
71
81
|
- name: google_sheets_service_account_1
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
A magical wrapper to perform work with databases.
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
##### Key Features
|
|
5
|
+
* **Unified Interface**: Abstracts the complexities of connecting and interacting with Snowflake and MySQL databases.
|
|
6
|
+
* **Seamless Data Handling**: Fetch, manipulate, and write data with Pandas DataFrames.
|
|
7
|
+
* **Flexibility**: Supports secure credentials management via configuration files and environment variables.
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
from spellbook import db_magic
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
### Error Handling
|
|
14
|
+
|
|
15
|
+
Every failure raises an exception. An empty DataFrame always means "the query matched no
|
|
16
|
+
rows" and never "something went wrong", so results can be trusted without extra checks.
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
from spellbook import db_magic
|
|
20
|
+
from spellbook.exceptions import SpellbookError, QueryError, DatabaseNotFoundError
|
|
21
|
+
|
|
22
|
+
try:
|
|
23
|
+
df = db_magic.get_data('reporting_db', 'select * from users limit 10')
|
|
24
|
+
except DatabaseNotFoundError:
|
|
25
|
+
... # the name is not in spellbook_config.yaml
|
|
26
|
+
except QueryError:
|
|
27
|
+
... # the database rejected the query
|
|
28
|
+
except SpellbookError:
|
|
29
|
+
... # anything Spellbook raises
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
| Exception | Raised when |
|
|
33
|
+
| --- | --- |
|
|
34
|
+
| `SpellbookError` | Base class for all of the below |
|
|
35
|
+
| `ConfigError` | The config file is missing, malformed, or references an unset `${VAR}` |
|
|
36
|
+
| `DatabaseNotFoundError` | No database in the config matches the requested name |
|
|
37
|
+
| `AuthenticationError` | Credentials could not be obtained or were rejected |
|
|
38
|
+
| `ConnectionFailedError` | The database could not be reached |
|
|
39
|
+
| `QueryError` | The query was rejected or failed while running |
|
|
40
|
+
|
|
41
|
+
> **Upgrading from 2.x:** these functions used to print a message and return an empty
|
|
42
|
+
> DataFrame or `None` on failure. Code that inspected `df.empty` to detect *errors* now
|
|
43
|
+
> needs a `try`/`except`. Code that used `df.empty` to detect *zero rows* is unaffected.
|
|
44
|
+
|
|
45
|
+
### Quiet Mode
|
|
46
|
+
|
|
47
|
+
Spellbook prints a wizard for company on each query. It goes to stderr, never stdout, so it
|
|
48
|
+
cannot contaminate results. Set `SPELLBOOK_QUIET=1` to turn it off - recommended for
|
|
49
|
+
scripts, scheduled jobs and anything driven by an AI agent.
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
export SPELLBOOK_QUIET=1
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
### MySQL Authentication
|
|
56
|
+
|
|
57
|
+
MySQL connections support two authentication methods, chosen per database in `spellbook_config.yaml`.
|
|
58
|
+
|
|
59
|
+
**Password authentication** is the default and needs no changes to an existing config:
|
|
60
|
+
|
|
61
|
+
```yaml
|
|
62
|
+
- name: legacy_mysql
|
|
63
|
+
type: mysql
|
|
64
|
+
host: your_host
|
|
65
|
+
port: 3306
|
|
66
|
+
user: mysql_user_name
|
|
67
|
+
password: ${MYSQL_PASSWORD}
|
|
68
|
+
database: mysql_database
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
**AWS RDS IAM authentication** stores no password at all. Add `iam_auth: true` and Spellbook
|
|
72
|
+
calls `rx rds password <rds_db_name>` for a temporary token each time it connects, because the
|
|
73
|
+
tokens expire after about 15 minutes:
|
|
74
|
+
|
|
75
|
+
> **Internal use only.** `iam_auth` depends on the `rx` CLI, which is Fullscript internal
|
|
76
|
+
> tooling and is not publicly available. Password authentication is the only MySQL option
|
|
77
|
+
> outside Fullscript, and it remains fully supported.
|
|
78
|
+
|
|
79
|
+
```yaml
|
|
80
|
+
- name: reporting_db
|
|
81
|
+
type: mysql
|
|
82
|
+
host: ${MYSQL_HOST} # e.g. your-instance.abc123.us-east-1.rds.amazonaws.com
|
|
83
|
+
port: 3306
|
|
84
|
+
user: ${DB_USER}
|
|
85
|
+
iam_auth: true
|
|
86
|
+
rds_db_name: your_rds_instance_name # Optional, defaults to the value of `name`
|
|
87
|
+
database: ${MYSQL_DATABASE}
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
| Field | Required | Notes |
|
|
91
|
+
| --- | --- | --- |
|
|
92
|
+
| `iam_auth` | No | `true` switches the database to IAM tokens. Omitting it keeps password authentication. |
|
|
93
|
+
| `user` | Yes | Your IAM database user. |
|
|
94
|
+
| `rds_db_name` | No | The remote database as named by `rx rds list`. Defaults to `name`, so set it whenever your config alias differs from the RDS name. |
|
|
95
|
+
| `ssl_ca` | No | Not needed in normal use; see below. |
|
|
96
|
+
| `password` | No | Ignored when `iam_auth` is true. |
|
|
97
|
+
|
|
98
|
+
Requirements for the IAM path:
|
|
99
|
+
* The `rx` CLI must be installed and on your `PATH`.
|
|
100
|
+
* You must be signed in to AWS. If a connection hangs or fails on authentication, run
|
|
101
|
+
`rx rds password <rds_db_name>` in a terminal once to complete the AWS SSO prompt.
|
|
102
|
+
* IAM connections use the `pymysql` driver (installed with Spellbook). Password connections
|
|
103
|
+
continue to use `mysql-connector`.
|
|
104
|
+
|
|
105
|
+
##### A note on TLS and `ssl_ca`
|
|
106
|
+
|
|
107
|
+
AWS requires TLS for IAM authentication, so Spellbook always negotiates an encrypted
|
|
108
|
+
connection. You do **not** need to set `ssl_ca` — leave it out and you get TLS without
|
|
109
|
+
verifying which server you reached, which is the normal setup.
|
|
110
|
+
|
|
111
|
+
Set `ssl_ca` only if you also want the server certificate verified. If you do:
|
|
112
|
+
* Point it at the [AWS RDS CA bundle](https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem).
|
|
113
|
+
Your operating system's bundle will not work, because RDS certificates do not chain to a
|
|
114
|
+
publicly trusted root; using it fails with `CERTIFICATE_VERIFY_FAILED`.
|
|
115
|
+
* Use the instance's own RDS endpoint as `host`. Verification also checks the hostname, so a
|
|
116
|
+
CNAME that points at the instance is likely to be rejected even with the right bundle.
|
|
117
|
+
|
|
118
|
+
### Methods Overview
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
##### get_data(db_name, query)
|
|
122
|
+
|
|
123
|
+
Fetches data from a specified database and returns it as a Pandas DataFrame. This method abstracts the connection and retrieval process for both Snowflake and MySQL databases, allowing you to focus on the SQL query.
|
|
124
|
+
|
|
125
|
+
Parameters:
|
|
126
|
+
* `db_name` (str): The database name defined in the configuration file.
|
|
127
|
+
* `query` (str): The SQL query to execute.
|
|
128
|
+
|
|
129
|
+
Returns:
|
|
130
|
+
* A Pandas DataFrame containing the result of the query.
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
# Fetch data from Snowflake
|
|
134
|
+
# Option 1:
|
|
135
|
+
query = "SELECT * FROM my_table LIMIT 10;"
|
|
136
|
+
df = db_magic.get_data("snowflake_db", query)
|
|
137
|
+
|
|
138
|
+
# Option 2:
|
|
139
|
+
df = db_magic.get_data("snowflake_db", 'my_sql_query.sql')
|
|
140
|
+
|
|
141
|
+
# Option 3:
|
|
142
|
+
df = db_magic.get_data("snowflake_db", 'user/project/my_query.sql')
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# Fetch data from MySQL
|
|
146
|
+
query = "SELECT * FROM users WHERE is_active = 1;"
|
|
147
|
+
df = db_magic.get_data("mysql_db", query)
|
|
148
|
+
|
|
149
|
+
# Display results
|
|
150
|
+
print(df.head())
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
##### write_data(db_name, schema, df, table_name, overwrite=True, chunk_size=1000, **kwargs)
|
|
154
|
+
|
|
155
|
+
Writes a Pandas DataFrame to a table in Snowflake. It supports creating tables automatically and overwriting existing tables.
|
|
156
|
+
|
|
157
|
+
Parameters:
|
|
158
|
+
* `db_name` (str): The database name defined in the configuration file.
|
|
159
|
+
* `schema` (str): The schema where the table is located.
|
|
160
|
+
* `df` (Pandas DataFrame): The data to write.
|
|
161
|
+
* `table_name` (str): The name of the destination table.
|
|
162
|
+
* `overwrite` (bool): Whether to overwrite the table if it exists (default: True).
|
|
163
|
+
* `chunk_size` (int): Number of rows to write per chunk (default: 1000).
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
import pandas as pd
|
|
167
|
+
|
|
168
|
+
# Create a sample DataFrame
|
|
169
|
+
data = {"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"]}
|
|
170
|
+
df = pd.DataFrame(data)
|
|
171
|
+
|
|
172
|
+
# Write data to Snowflake
|
|
173
|
+
db_magic.write_data("snowflake_db", "my_schema", df, "my_table", overwrite=True)
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
##### run_sql(db_name, query)
|
|
177
|
+
|
|
178
|
+
Executes a SQL query directly on a specified database without returning results. Ideal for operations like creating tables, updating rows, or running stored procedures.
|
|
179
|
+
|
|
180
|
+
Parameters:
|
|
181
|
+
* `db_name` (str): The database name defined in the configuration file.
|
|
182
|
+
* `query` (str): The SQL query to execute.
|
|
183
|
+
|
|
184
|
+
Returns:
|
|
185
|
+
* `None`
|
|
186
|
+
|
|
187
|
+
```python
|
|
188
|
+
|
|
189
|
+
# Execute SQL in Snowflake
|
|
190
|
+
create_table_query = """
|
|
191
|
+
CREATE TABLE my_table (
|
|
192
|
+
id INT,
|
|
193
|
+
name STRING
|
|
194
|
+
);
|
|
195
|
+
"""
|
|
196
|
+
db_magic.run_sql("snowflake_db", create_table_query)
|
|
197
|
+
|
|
198
|
+
# Execute SQL in MySQL
|
|
199
|
+
update_query = "UPDATE users SET is_active = 1 WHERE last_login > '2024-01-01';"
|
|
200
|
+
db_magic.run_sql("mysql_db", update_query)
|
|
201
|
+
|
|
202
|
+
print("SQL executed successfully!")
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
### Exploring a database
|
|
206
|
+
|
|
207
|
+
These helpers answer "what is in here?" without hand-writing `information_schema` queries.
|
|
208
|
+
They are useful interactively, and they are what an AI agent needs before it can write a
|
|
209
|
+
sensible query.
|
|
210
|
+
|
|
211
|
+
##### list_databases()
|
|
212
|
+
|
|
213
|
+
Lists the databases in your configuration file, with no credentials included.
|
|
214
|
+
|
|
215
|
+
```python
|
|
216
|
+
db_magic.list_databases()
|
|
217
|
+
# name type database iam_auth
|
|
218
|
+
# 0 reporting_db mysql mysql_database True
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
##### list_tables(db_name, pattern=None)
|
|
222
|
+
|
|
223
|
+
Lists the tables in a database, optionally filtered by a case-insensitive substring.
|
|
224
|
+
|
|
225
|
+
```python
|
|
226
|
+
db_magic.list_tables("mysql_db")
|
|
227
|
+
db_magic.list_tables("mysql_db", pattern="order")
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
The pattern is passed as a bound parameter, so `%` and quotes in it are treated as literal
|
|
231
|
+
text rather than changing the query.
|
|
232
|
+
|
|
233
|
+
##### describe_table(db_name, table)
|
|
234
|
+
|
|
235
|
+
Returns one row per column, with its type, nullability and default. Raises `QueryError` if
|
|
236
|
+
the table does not exist.
|
|
237
|
+
|
|
238
|
+
```python
|
|
239
|
+
db_magic.describe_table("mysql_db", "users")
|
|
240
|
+
# column_name data_type is_nullable column_default
|
|
241
|
+
# 0 id bigint NO None
|
|
242
|
+
# 1 name varchar YES None
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
##### preview_table(db_name, table, limit=10)
|
|
246
|
+
|
|
247
|
+
Reads the first few rows of a table.
|
|
248
|
+
|
|
249
|
+
```python
|
|
250
|
+
db_magic.preview_table("mysql_db", "users", limit=5)
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
`table` must be a plain identifier - letters, digits, underscores, dots or dollar signs -
|
|
254
|
+
because a table name cannot be passed as a bound parameter.
|
{fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/fullscript.spellbook.egg-info/PKG-INFO
RENAMED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fullscript.spellbook
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.0
|
|
4
4
|
Summary: A simple package to make your life easier.
|
|
5
|
-
Author-email: Thomas Edwards <thomas.edwards@fullscript.com>, Gui Chagas <gui.chagas@fullscript.com>, David King <david.king@fullscript.com>,
|
|
5
|
+
Author-email: Meinhard Mare <meinhard.mare@fullscript.com>, Thomas Edwards <thomas.edwards@fullscript.com>, Gui Chagas <gui.chagas@fullscript.com>, David King <david.king@fullscript.com>, Karsten Melchers <karsten.melchers@fullscript.com>
|
|
6
6
|
License: MIT
|
|
7
7
|
Project-URL: homepage, https://github.com/Fullscript/spellbook
|
|
8
8
|
Requires-Python: <3.12,>=3.10
|
|
@@ -46,6 +46,7 @@ Requires-Dist: pycparser==2.22
|
|
|
46
46
|
Requires-Dist: Pygments==2.18.0
|
|
47
47
|
Requires-Dist: PyJWT==2.9.0
|
|
48
48
|
Requires-Dist: pyOpenSSL==24.2.1
|
|
49
|
+
Requires-Dist: PyMySQL==1.2.3
|
|
49
50
|
Requires-Dist: python-dateutil==2.9.0.post0
|
|
50
51
|
Requires-Dist: pytz==2024.2
|
|
51
52
|
Requires-Dist: PyYAML==6.0.2
|
|
@@ -116,6 +117,8 @@ Requires-Dist: scipy==1.14.1; extra == "statsmagic"
|
|
|
116
117
|
Requires-Dist: seaborn==0.13.2; extra == "statsmagic"
|
|
117
118
|
Requires-Dist: six==1.16.0; extra == "statsmagic"
|
|
118
119
|
Requires-Dist: statsmodels==0.14.4; extra == "statsmagic"
|
|
120
|
+
Provides-Extra: test
|
|
121
|
+
Requires-Dist: pytest==8.3.4; extra == "test"
|
|
119
122
|
Dynamic: license-file
|
|
120
123
|
|
|
121
124
|
# Spellbook
|
|
@@ -186,6 +189,16 @@ databases:
|
|
|
186
189
|
database: snowflake_database
|
|
187
190
|
schema: snowflake_schema
|
|
188
191
|
|
|
192
|
+
# MySQL Database Connection using AWS RDS IAM authentication (no stored password).
|
|
193
|
+
# Fullscript internal only: requires the `rx` CLI. See documentation/db_magic.md.
|
|
194
|
+
- name: alias_of_your_connection
|
|
195
|
+
type: mysql
|
|
196
|
+
host: your_host.rds.amazonaws.com
|
|
197
|
+
port: 3306
|
|
198
|
+
user: ${DB_USER}
|
|
199
|
+
iam_auth: true
|
|
200
|
+
database: ${MYSQL_DATABASE}
|
|
201
|
+
|
|
189
202
|
google_accounts:
|
|
190
203
|
# Google Sheets Service Account 1
|
|
191
204
|
- name: google_sheets_service_account_1
|
{fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/fullscript.spellbook.egg-info/SOURCES.txt
RENAMED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
.DS_Store
|
|
2
1
|
.gitignore
|
|
3
2
|
.gitlab-ci.yml
|
|
4
3
|
.python-version
|
|
4
|
+
AGENTS.md
|
|
5
|
+
CLAUDE.md
|
|
5
6
|
LICENSE
|
|
6
7
|
MANIFEST.in
|
|
7
8
|
README.md
|
|
@@ -17,11 +18,14 @@ fullscript.spellbook.egg-info/PKG-INFO
|
|
|
17
18
|
fullscript.spellbook.egg-info/SOURCES.txt
|
|
18
19
|
fullscript.spellbook.egg-info/dependency_links.txt
|
|
19
20
|
fullscript.spellbook.egg-info/requires.txt
|
|
21
|
+
fullscript.spellbook.egg-info/scm_file_list.json
|
|
22
|
+
fullscript.spellbook.egg-info/scm_version.json
|
|
20
23
|
fullscript.spellbook.egg-info/top_level.txt
|
|
21
|
-
spellbook/.DS_Store
|
|
22
24
|
spellbook/__init__.py
|
|
23
25
|
spellbook/db_magic.py
|
|
24
26
|
spellbook/drive_magic.py
|
|
27
|
+
spellbook/exceptions.py
|
|
28
|
+
spellbook/py.typed
|
|
25
29
|
spellbook/sheet_magic.py
|
|
26
30
|
spellbook/stats_magic.py
|
|
27
31
|
spellbook/utils.py
|
|
@@ -37,4 +41,6 @@ spellbook/wizards/wizard5.txt
|
|
|
37
41
|
spellbook/wizards/wizard6.txt
|
|
38
42
|
spellbook/wizards/wizard7.txt
|
|
39
43
|
spellbook/wizards/wizard8.txt
|
|
40
|
-
spellbook/wizards/wizard9.txt
|
|
44
|
+
spellbook/wizards/wizard9.txt
|
|
45
|
+
tests/test_db_magic.py
|
|
46
|
+
tests/test_utils.py
|
{fullscript_spellbook-2.1.7 → fullscript_spellbook-3.0.0}/fullscript.spellbook.egg-info/requires.txt
RENAMED
|
@@ -36,6 +36,7 @@ pycparser==2.22
|
|
|
36
36
|
Pygments==2.18.0
|
|
37
37
|
PyJWT==2.9.0
|
|
38
38
|
pyOpenSSL==24.2.1
|
|
39
|
+
PyMySQL==1.2.3
|
|
39
40
|
python-dateutil==2.9.0.post0
|
|
40
41
|
pytz==2024.2
|
|
41
42
|
PyYAML==6.0.2
|
|
@@ -109,3 +110,6 @@ scipy==1.14.1
|
|
|
109
110
|
seaborn==0.13.2
|
|
110
111
|
six==1.16.0
|
|
111
112
|
statsmodels==0.14.4
|
|
113
|
+
|
|
114
|
+
[test]
|
|
115
|
+
pytest==8.3.4
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
{
|
|
2
|
+
"files": [
|
|
3
|
+
".gitignore",
|
|
4
|
+
".gitlab-ci.yml",
|
|
5
|
+
".python-version",
|
|
6
|
+
"AGENTS.md",
|
|
7
|
+
"CLAUDE.md",
|
|
8
|
+
"LICENSE",
|
|
9
|
+
"MANIFEST.in",
|
|
10
|
+
"README.md",
|
|
11
|
+
"documentation/db_magic.md",
|
|
12
|
+
"documentation/drive_magic.md",
|
|
13
|
+
"documentation/sheet_magic.md",
|
|
14
|
+
"documentation/snowflake_key_pair_setup.md",
|
|
15
|
+
"documentation/stats_magic.md",
|
|
16
|
+
"pyproject.toml",
|
|
17
|
+
"requirements.txt",
|
|
18
|
+
"spellbook/__init__.py",
|
|
19
|
+
"spellbook/db_magic.py",
|
|
20
|
+
"spellbook/drive_magic.py",
|
|
21
|
+
"spellbook/exceptions.py",
|
|
22
|
+
"spellbook/py.typed",
|
|
23
|
+
"spellbook/sheet_magic.py",
|
|
24
|
+
"spellbook/stats_magic.py",
|
|
25
|
+
"spellbook/utils.py",
|
|
26
|
+
"spellbook/wizards/__init__.py",
|
|
27
|
+
"spellbook/wizards/wizard1.txt",
|
|
28
|
+
"spellbook/wizards/wizard10.txt",
|
|
29
|
+
"spellbook/wizards/wizard11.txt",
|
|
30
|
+
"spellbook/wizards/wizard12.txt",
|
|
31
|
+
"spellbook/wizards/wizard2.txt",
|
|
32
|
+
"spellbook/wizards/wizard3.txt",
|
|
33
|
+
"spellbook/wizards/wizard4.txt",
|
|
34
|
+
"spellbook/wizards/wizard5.txt",
|
|
35
|
+
"spellbook/wizards/wizard6.txt",
|
|
36
|
+
"spellbook/wizards/wizard7.txt",
|
|
37
|
+
"spellbook/wizards/wizard8.txt",
|
|
38
|
+
"spellbook/wizards/wizard9.txt",
|
|
39
|
+
"spellbook_config.yaml",
|
|
40
|
+
"tests/test_db_magic.py",
|
|
41
|
+
"tests/test_utils.py"
|
|
42
|
+
]
|
|
43
|
+
}
|
|
@@ -10,10 +10,10 @@ readme = "README.md"
|
|
|
10
10
|
license = {text = "MIT"}
|
|
11
11
|
requires-python = ">=3.10, <3.12"
|
|
12
12
|
authors = [
|
|
13
|
+
{name = "Meinhard Mare", email = "meinhard.mare@fullscript.com"},
|
|
13
14
|
{name = "Thomas Edwards", email = "thomas.edwards@fullscript.com"},
|
|
14
15
|
{name = "Gui Chagas", email = "gui.chagas@fullscript.com"},
|
|
15
16
|
{name = "David King", email = "david.king@fullscript.com"},
|
|
16
|
-
{name = "Meinhard Mare", email = "meinhard.mare@fullscript.com"},
|
|
17
17
|
{name = "Karsten Melchers", email = "karsten.melchers@fullscript.com"}
|
|
18
18
|
]
|
|
19
19
|
urls = {homepage = "https://github.com/Fullscript/spellbook"}
|
|
@@ -56,6 +56,7 @@ dependencies = [
|
|
|
56
56
|
"Pygments==2.18.0",
|
|
57
57
|
"PyJWT==2.9.0",
|
|
58
58
|
"pyOpenSSL==24.2.1",
|
|
59
|
+
"PyMySQL==1.2.3",
|
|
59
60
|
"python-dateutil==2.9.0.post0",
|
|
60
61
|
"pytz==2024.2",
|
|
61
62
|
"PyYAML==6.0.2",
|
|
@@ -133,11 +134,18 @@ statsmagic = [
|
|
|
133
134
|
"statsmodels==0.14.4"
|
|
134
135
|
]
|
|
135
136
|
|
|
137
|
+
test = [
|
|
138
|
+
"pytest==8.3.4"
|
|
139
|
+
]
|
|
140
|
+
|
|
141
|
+
[tool.pytest.ini_options]
|
|
142
|
+
testpaths = ["tests"]
|
|
143
|
+
|
|
136
144
|
[tool.setuptools.packages.find]
|
|
137
145
|
where = ["."]
|
|
138
146
|
include = ["spellbook"]
|
|
139
147
|
|
|
140
148
|
[tool.setuptools.package-data]
|
|
141
|
-
"spellbook" = ["wizards/*.txt"]
|
|
149
|
+
"spellbook" = ["wizards/*.txt", "py.typed"]
|
|
142
150
|
|
|
143
151
|
[tool.setuptools_scm]
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Spellbook: connect to databases, spreadsheets and Drive, and run analyses."""
|
|
2
|
+
|
|
3
|
+
from spellbook.exceptions import (
|
|
4
|
+
AuthenticationError,
|
|
5
|
+
ConfigError,
|
|
6
|
+
ConnectionFailedError,
|
|
7
|
+
DatabaseNotFoundError,
|
|
8
|
+
QueryError,
|
|
9
|
+
SpellbookError,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
'SpellbookError',
|
|
14
|
+
'ConfigError',
|
|
15
|
+
'DatabaseNotFoundError',
|
|
16
|
+
'AuthenticationError',
|
|
17
|
+
'ConnectionFailedError',
|
|
18
|
+
'QueryError',
|
|
19
|
+
]
|