squirrels 0.1.0rc1__tar.gz → 0.1.0rc2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of squirrels might be problematic. Click here for more details.
- {squirrels-0.1.0rc1 → squirrels-0.1.0rc2}/LICENSE +1 -0
- squirrels-0.1.0rc2/PKG-INFO +56 -0
- squirrels-0.1.0rc2/README.md +46 -0
- squirrels-0.1.0rc2/setup.py +42 -0
- squirrels-0.1.0rc2/squirrels/__init__.py +19 -0
- squirrels-0.1.0rc2/squirrels/_command_line.py +105 -0
- squirrels-0.1.0rc2/squirrels/api_server.py +128 -0
- squirrels-0.1.0rc2/squirrels/connection_set.py +65 -0
- squirrels-0.1.0rc2/squirrels/constants.py +53 -0
- squirrels-0.1.0rc2/squirrels/credentials_manager.py +87 -0
- squirrels-0.1.0rc2/squirrels/dateutils.py +272 -0
- {squirrels-0.1.0rc1 → squirrels-0.1.0rc2}/squirrels/initializer.py +37 -27
- squirrels-0.1.0rc2/squirrels/manifest.py +123 -0
- {squirrels-0.1.0rc1 → squirrels-0.1.0rc2}/squirrels/module_loader.py +4 -7
- {squirrels-0.1.0rc1/squirrels → squirrels-0.1.0rc2/squirrels/package_data}/base_project/.gitignore +1 -0
- squirrels-0.1.0rc2/squirrels/package_data/base_project/connections.py +21 -0
- squirrels-0.1.0rc2/squirrels/package_data/base_project/database/sample_database.db +0 -0
- squirrels-0.1.0rc2/squirrels/package_data/base_project/datasets/sample_dataset/context.py +8 -0
- squirrels-0.1.0rc2/squirrels/package_data/base_project/datasets/sample_dataset/database_view1.py +21 -0
- squirrels-0.1.0rc2/squirrels/package_data/base_project/datasets/sample_dataset/database_view1.sql.j2 +5 -0
- squirrels-0.1.0rc2/squirrels/package_data/base_project/datasets/sample_dataset/final_view.py +9 -0
- squirrels-0.1.0rc2/squirrels/package_data/base_project/datasets/sample_dataset/parameters.py +31 -0
- squirrels-0.1.0rc2/squirrels/package_data/base_project/datasets/sample_dataset/selections.cfg +6 -0
- squirrels-0.1.0rc2/squirrels/package_data/base_project/project_vars.yaml +3 -0
- squirrels-0.1.0rc2/squirrels/package_data/base_project/squirrels.yaml +15 -0
- {squirrels-0.1.0rc1/squirrels → squirrels-0.1.0rc2/squirrels/package_data}/static/script.js +20 -9
- {squirrels-0.1.0rc1/squirrels → squirrels-0.1.0rc2/squirrels/package_data}/templates/index.html +2 -2
- squirrels-0.1.0rc2/squirrels/param_configs/__init__.py +0 -0
- squirrels-0.1.0rc2/squirrels/param_configs/data_sources.py +329 -0
- squirrels-0.1.0rc2/squirrels/param_configs/parameter_options.py +189 -0
- squirrels-0.1.0rc2/squirrels/param_configs/parameter_set.py +36 -0
- squirrels-0.1.0rc2/squirrels/param_configs/parameters.py +435 -0
- squirrels-0.1.0rc2/squirrels/renderer.py +279 -0
- squirrels-0.1.0rc2/squirrels/timed_imports.py +36 -0
- squirrels-0.1.0rc2/squirrels/utils.py +94 -0
- squirrels-0.1.0rc2/squirrels/version.py +3 -0
- squirrels-0.1.0rc2/squirrels.egg-info/PKG-INFO +56 -0
- squirrels-0.1.0rc2/squirrels.egg-info/SOURCES.txt +45 -0
- squirrels-0.1.0rc2/squirrels.egg-info/entry_points.txt +2 -0
- {squirrels-0.1.0rc1 → squirrels-0.1.0rc2}/squirrels.egg-info/requires.txt +2 -2
- squirrels-0.1.0rc1/.gitignore +0 -82
- squirrels-0.1.0rc1/PKG-INFO +0 -47
- squirrels-0.1.0rc1/README.md +0 -37
- squirrels-0.1.0rc1/requirements.txt +0 -12
- squirrels-0.1.0rc1/sample_project/.gitignore +0 -2
- squirrels-0.1.0rc1/sample_project/datasets/sample_lu_data.csv +0 -4
- squirrels-0.1.0rc1/sample_project/datasets/stock_price_history/context.py +0 -22
- squirrels-0.1.0rc1/sample_project/datasets/stock_price_history/final_view.py +0 -7
- squirrels-0.1.0rc1/sample_project/datasets/stock_price_history/final_view.sql.j2 +0 -6
- squirrels-0.1.0rc1/sample_project/datasets/stock_price_history/parameters.py +0 -41
- squirrels-0.1.0rc1/sample_project/datasets/stock_price_history/selections.cfg +0 -6
- squirrels-0.1.0rc1/sample_project/datasets/stock_price_history/ticker_history.sql.j2 +0 -8
- squirrels-0.1.0rc1/sample_project/functions.py +0 -43
- squirrels-0.1.0rc1/sample_project/parameter_options.py +0 -53
- squirrels-0.1.0rc1/setup.py +0 -38
- squirrels-0.1.0rc1/squirrels/__init__.py +0 -10
- squirrels-0.1.0rc1/squirrels/api_server.py +0 -136
- squirrels-0.1.0rc1/squirrels/base_project/datasets/sample_dataset/context.py +0 -6
- squirrels-0.1.0rc1/squirrels/base_project/datasets/sample_dataset/database_view1.py +0 -10
- squirrels-0.1.0rc1/squirrels/base_project/datasets/sample_dataset/database_view1.sql.j2 +0 -4
- squirrels-0.1.0rc1/squirrels/base_project/datasets/sample_dataset/final_view.py +0 -9
- squirrels-0.1.0rc1/squirrels/base_project/datasets/sample_dataset/parameters.py +0 -8
- squirrels-0.1.0rc1/squirrels/base_project/squirrels.yaml +0 -20
- squirrels-0.1.0rc1/squirrels/command_line.py +0 -114
- squirrels-0.1.0rc1/squirrels/constants.py +0 -72
- squirrels-0.1.0rc1/squirrels/db_conn.py +0 -41
- squirrels-0.1.0rc1/squirrels/manifest.py +0 -38
- squirrels-0.1.0rc1/squirrels/parameter_configs.py +0 -512
- squirrels-0.1.0rc1/squirrels/profile_manager.py +0 -57
- squirrels-0.1.0rc1/squirrels/renderer.py +0 -225
- squirrels-0.1.0rc1/squirrels/templates/sandbox.html +0 -25
- squirrels-0.1.0rc1/squirrels/version.txt +0 -1
- squirrels-0.1.0rc1/squirrels.egg-info/PKG-INFO +0 -47
- squirrels-0.1.0rc1/squirrels.egg-info/SOURCES.txt +0 -52
- squirrels-0.1.0rc1/squirrels.egg-info/entry_points.txt +0 -2
- squirrels-0.1.0rc1/tests/conftest.py +0 -9
- squirrels-0.1.0rc1/tests/test_db_conn.py +0 -14
- squirrels-0.1.0rc1/tests/test_module_loader.py +0 -17
- squirrels-0.1.0rc1/tests/test_parameter_configs.py +0 -120
- squirrels-0.1.0rc1/tests/test_profile_manager.py +0 -28
- {squirrels-0.1.0rc1 → squirrels-0.1.0rc2}/setup.cfg +0 -0
- {squirrels-0.1.0rc1/squirrels → squirrels-0.1.0rc2/squirrels/package_data}/base_project/database/seattle_weather.db +0 -0
- {squirrels-0.1.0rc1/squirrels → squirrels-0.1.0rc2/squirrels/package_data}/base_project/datasets/sample_dataset/final_view.sql.j2 +0 -0
- {squirrels-0.1.0rc1/squirrels → squirrels-0.1.0rc2/squirrels/package_data}/static/favicon.ico +0 -0
- {squirrels-0.1.0rc1/squirrels → squirrels-0.1.0rc2/squirrels/package_data}/static/style.css +0 -0
- {squirrels-0.1.0rc1 → squirrels-0.1.0rc2}/squirrels.egg-info/dependency_links.txt +0 -0
- {squirrels-0.1.0rc1 → squirrels-0.1.0rc2}/squirrels.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: squirrels
|
|
3
|
+
Version: 0.1.0rc2
|
|
4
|
+
Summary: Python Package for Configuring SQL Generating APIs
|
|
5
|
+
Author: Tim Huang
|
|
6
|
+
Author-email: tim.yuting@hotmail.com
|
|
7
|
+
License: MIT
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
|
|
11
|
+
# Squirrels
|
|
12
|
+
|
|
13
|
+
Squirrels is an API framework for creating REST APIs that generate sql queries & dataframes dynamically from query parameters.
|
|
14
|
+
|
|
15
|
+
## Setup
|
|
16
|
+
|
|
17
|
+
First, install the library and all dependencies **in an independent virtual environment**.
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
# activate virtual environment...
|
|
21
|
+
pip install -e .
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
To confirm that the setup worked, run this to show the help page for all squirrels CLI commands:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
squirrels -h
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Testing
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
python setup.py pytest
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Documentation
|
|
37
|
+
|
|
38
|
+
Check the documentation website [here](https://squirrels-nest.github.io/squirrels-docs/) or get started [here](https://squirrels-nest.github.io/squirrels-docs/getting-started/) to learn how to use the squirrels library to make an API project.
|
|
39
|
+
|
|
40
|
+
## Developer Guide
|
|
41
|
+
|
|
42
|
+
From the root of the git repo, the source code can be found in the `squirrels` folder and unit tests can be found in the `tests` folder.
|
|
43
|
+
|
|
44
|
+
To understand what a specific squirrels command line utility is doing, start from the `_command_line.py` file as your entry point.
|
|
45
|
+
|
|
46
|
+
The library version is maintained in both the `setup.py` file (for the next release or release-candidate version) and the `squirrels/version.py` file (for the next release version only).
|
|
47
|
+
|
|
48
|
+
When a user initializes a squirrels project using `squirrels init`, the files are copied from the `squirrels/package_data/base_project` folder. The contents in the `database` subfolder were constructed from the scripts in the `database_elt` folder at the top level.
|
|
49
|
+
|
|
50
|
+
For the Squirrels UI activated by `squirrels run`, the HTML, CSS, and Javascript files can be found in the `static` and `templates` subfolders of `squirrels/package_data`.
|
|
51
|
+
|
|
52
|
+
## License
|
|
53
|
+
|
|
54
|
+
Squirrels is released under the MIT license.
|
|
55
|
+
|
|
56
|
+
See the file LICENSE for more details.
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Squirrels
|
|
2
|
+
|
|
3
|
+
Squirrels is an API framework for creating REST APIs that generate sql queries & dataframes dynamically from query parameters.
|
|
4
|
+
|
|
5
|
+
## Setup
|
|
6
|
+
|
|
7
|
+
First, install the library and all dependencies **in an independent virtual environment**.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
# activate virtual environment...
|
|
11
|
+
pip install -e .
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
To confirm that the setup worked, run this to show the help page for all squirrels CLI commands:
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
squirrels -h
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Testing
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
python setup.py pytest
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Documentation
|
|
27
|
+
|
|
28
|
+
Check the documentation website [here](https://squirrels-nest.github.io/squirrels-docs/) or get started [here](https://squirrels-nest.github.io/squirrels-docs/getting-started/) to learn how to use the squirrels library to make an API project.
|
|
29
|
+
|
|
30
|
+
## Developer Guide
|
|
31
|
+
|
|
32
|
+
From the root of the git repo, the source code can be found in the `squirrels` folder and unit tests can be found in the `tests` folder.
|
|
33
|
+
|
|
34
|
+
To understand what a specific squirrels command line utility is doing, start from the `_command_line.py` file as your entry point.
|
|
35
|
+
|
|
36
|
+
The library version is maintained in both the `setup.py` file (for the next release or release-candidate version) and the `squirrels/version.py` file (for the next release version only).
|
|
37
|
+
|
|
38
|
+
When a user initializes a squirrels project using `squirrels init`, the files are copied from the `squirrels/package_data/base_project` folder. The contents in the `database` subfolder were constructed from the scripts in the `database_elt` folder at the top level.
|
|
39
|
+
|
|
40
|
+
For the Squirrels UI activated by `squirrels run`, the HTML, CSS, and Javascript files can be found in the `static` and `templates` subfolders of `squirrels/package_data`.
|
|
41
|
+
|
|
42
|
+
## License
|
|
43
|
+
|
|
44
|
+
Squirrels is released under the MIT license.
|
|
45
|
+
|
|
46
|
+
See the file LICENSE for more details.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
from setuptools import setup, find_packages
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
# The directory containing this file
|
|
5
|
+
HERE = os.path.abspath(os.path.dirname(__file__))
|
|
6
|
+
|
|
7
|
+
# Get the long description from the README file
|
|
8
|
+
with open(os.path.join(HERE, 'README.md'), encoding='utf-8') as f:
|
|
9
|
+
long_description = f.read()
|
|
10
|
+
|
|
11
|
+
# Recursively get package data
|
|
12
|
+
def package_files(directory):
|
|
13
|
+
paths = []
|
|
14
|
+
for (path, directories, filenames) in os.walk(directory):
|
|
15
|
+
for filename in filenames:
|
|
16
|
+
paths.append(os.path.join('..', path, filename))
|
|
17
|
+
return paths
|
|
18
|
+
|
|
19
|
+
extra_files = package_files(os.path.join('squirrels', 'package_data'))
|
|
20
|
+
|
|
21
|
+
setup(
|
|
22
|
+
name='squirrels',
|
|
23
|
+
version='0.1.0rc2',
|
|
24
|
+
packages=find_packages(),
|
|
25
|
+
description='Python Package for Configuring SQL Generating APIs',
|
|
26
|
+
long_description=long_description,
|
|
27
|
+
long_description_content_type="text/markdown",
|
|
28
|
+
author='Tim Huang',
|
|
29
|
+
author_email='tim.yuting@hotmail.com',
|
|
30
|
+
license='MIT',
|
|
31
|
+
install_requires=[
|
|
32
|
+
'openpyxl', 'inquirer', 'pwinput', 'cachetools', 'fastapi', 'uvicorn',
|
|
33
|
+
'Jinja2', 'GitPython', 'sqlalchemy', 'pandas', 'pyyaml'
|
|
34
|
+
],
|
|
35
|
+
setup_requires=['pytest-runner==6.0.0'],
|
|
36
|
+
tests_require=['pytest==7.2.0'],
|
|
37
|
+
test_suite='tests',
|
|
38
|
+
package_data= {'squirrels': extra_files},
|
|
39
|
+
entry_points= {
|
|
40
|
+
'console_scripts': ['squirrels=squirrels._command_line:main']
|
|
41
|
+
}
|
|
42
|
+
)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from .param_configs.parameter_options import SelectParameterOption, DateParameterOption, NumberParameterOption, NumRangeParameterOption
|
|
2
|
+
from .param_configs.parameters import WidgetType, Parameter, SingleSelectParameter, MultiSelectParameter, DateParameter, NumberParameter, NumRangeParameter
|
|
3
|
+
from .param_configs.data_sources import SelectionDataSource, DateDataSource, NumberDataSource, NumRangeDataSource, DataSourceParameter
|
|
4
|
+
from .param_configs.parameter_set import ParameterSet
|
|
5
|
+
from .connection_set import ConnectionSet
|
|
6
|
+
from .version import __version__, major_version, minor_version, patch_version
|
|
7
|
+
|
|
8
|
+
def get_credential(key: str):
|
|
9
|
+
"""
|
|
10
|
+
Gets the username and password that was set through "$squirrels set-credential [key]"
|
|
11
|
+
|
|
12
|
+
Parameters:
|
|
13
|
+
key (str): The credential key
|
|
14
|
+
|
|
15
|
+
Returns:
|
|
16
|
+
Credential: Object with attributes "username" and "password"
|
|
17
|
+
"""
|
|
18
|
+
from .credentials_manager import squirrels_config_io
|
|
19
|
+
return squirrels_config_io.get_credential(key)
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
from typing import List, Tuple, Optional
|
|
2
|
+
from argparse import ArgumentParser
|
|
3
|
+
import sys, time, pwinput
|
|
4
|
+
sys.path.append('.')
|
|
5
|
+
|
|
6
|
+
from squirrels import constants as c, __version__
|
|
7
|
+
from squirrels import module_loader as ml, manifest as mf, credentials_manager as cm, connection_set as cs
|
|
8
|
+
from squirrels.api_server import ApiServer
|
|
9
|
+
from squirrels.renderer import RendererIOWrapper
|
|
10
|
+
from squirrels.initializer import Initializer
|
|
11
|
+
from squirrels.timed_imports import timer
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _prompt_user_pw(args_values: Optional[List[str]]) -> Tuple[str, str]:
|
|
15
|
+
if args_values is not None:
|
|
16
|
+
user, pw = args_values
|
|
17
|
+
else:
|
|
18
|
+
user = input("Enter username: ")
|
|
19
|
+
pw = pwinput.pwinput("Enter password: ")
|
|
20
|
+
return user, pw
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def main():
|
|
24
|
+
"""
|
|
25
|
+
Main entry point for the squirrels command line utilities.
|
|
26
|
+
"""
|
|
27
|
+
start = time.time()
|
|
28
|
+
parser = ArgumentParser(description="Command line utilities from the squirrels python package")
|
|
29
|
+
parser.add_argument('-v', '--version', action='store_true', help='Show the version and exit')
|
|
30
|
+
parser.add_argument('--verbose', action='store_true', help='Enable verbose output')
|
|
31
|
+
subparsers = parser.add_subparsers(title='commands', dest='command')
|
|
32
|
+
|
|
33
|
+
init_parser = subparsers.add_parser(c.INIT_CMD, help='Initialize a squirrels project')
|
|
34
|
+
init_parser.add_argument('--no-overwrite', action='store_true', help="Don't overwrite files if they already exist")
|
|
35
|
+
init_parser.add_argument('--core', action='store_true', help='Include all core files')
|
|
36
|
+
init_parser.add_argument('--context', action='store_true', help=f'Include the {c.CONTEXT_FILE} file')
|
|
37
|
+
init_parser.add_argument('--selections-cfg', action='store_true', help=f'Include the {c.SELECTIONS_CFG_FILE} file')
|
|
38
|
+
init_parser.add_argument('--db-view', type=str, choices=['sql', 'py'], help='Create database view as sql (default) or python file')
|
|
39
|
+
init_parser.add_argument('--final-view', type=str, choices=['sql', 'py'], help='Include final view as sql or python file')
|
|
40
|
+
init_parser.add_argument('--sample-db', type=str, choices=['seattle-weather'], help='Sample sqlite database to include')
|
|
41
|
+
|
|
42
|
+
subparsers.add_parser(c.LOAD_MODULES_CMD, help='Load all the modules specified in squirrels.yaml from git')
|
|
43
|
+
|
|
44
|
+
def _add_profile_argument(parser: ArgumentParser):
|
|
45
|
+
parser.add_argument('key', type=str, help='Key to the database connection credential')
|
|
46
|
+
|
|
47
|
+
subparsers.add_parser(c.GET_CREDS_CMD, help='Get all database connection credential keys')
|
|
48
|
+
|
|
49
|
+
set_cred_parser = subparsers.add_parser(c.SET_CRED_CMD, help='Set a database connection credential key')
|
|
50
|
+
_add_profile_argument(set_cred_parser)
|
|
51
|
+
set_cred_parser.add_argument('--values', type=str, nargs=2, help='The username and password')
|
|
52
|
+
|
|
53
|
+
delete_cred_parser = subparsers.add_parser(c.DELETE_CRED_CMD, help='Delete a database connection credential key')
|
|
54
|
+
_add_profile_argument(delete_cred_parser)
|
|
55
|
+
|
|
56
|
+
test_parser = subparsers.add_parser(c.TEST_CMD, help='For a given dataset, create outputs for parameter API response and rendered sql queries')
|
|
57
|
+
test_parser.add_argument('dataset', type=str, help='Name of dataset (provided in squirrels.yaml) to test. Results are written in an "outputs" folder')
|
|
58
|
+
test_parser.add_argument('-c', '--cfg', type=str, help="Configuration file for parameter selections. Path is relative to the dataset's folder")
|
|
59
|
+
test_parser.add_argument('-d', '--data', type=str, help="Excel file with lookup data to avoid making a database connection. Path is relative to the dataset's folder")
|
|
60
|
+
test_parser.add_argument('-r', '--runquery', action='store_true', help='Runs all database queries and final view, and produce the results as csv files')
|
|
61
|
+
|
|
62
|
+
run_parser = subparsers.add_parser(c.RUN_CMD, help='Run the builtin API server')
|
|
63
|
+
run_parser.add_argument('--no-cache', action='store_true', help='Do not cache any api results')
|
|
64
|
+
run_parser.add_argument('--debug', action='store_true', help='In debug mode, all "hidden parameters" show in the parameters response')
|
|
65
|
+
run_parser.add_argument('--host', type=str, default='127.0.0.1')
|
|
66
|
+
run_parser.add_argument('--port', type=int, default=8000)
|
|
67
|
+
|
|
68
|
+
args, _ = parser.parse_known_args()
|
|
69
|
+
timer.verbose = args.verbose
|
|
70
|
+
timer.add_activity_time('parsing arguments', start)
|
|
71
|
+
|
|
72
|
+
if args.version:
|
|
73
|
+
print(__version__)
|
|
74
|
+
elif args.command == c.INIT_CMD:
|
|
75
|
+
Initializer(not args.no_overwrite).init_project(args)
|
|
76
|
+
elif args.command == c.LOAD_MODULES_CMD:
|
|
77
|
+
manifest = mf.from_file()
|
|
78
|
+
ml.load_modules(manifest)
|
|
79
|
+
elif args.command == c.GET_CREDS_CMD:
|
|
80
|
+
cm.squirrels_config_io.print_all_credentials()
|
|
81
|
+
elif args.command == c.SET_CRED_CMD:
|
|
82
|
+
user, pw = _prompt_user_pw(args.values)
|
|
83
|
+
cm.squirrels_config_io.set_credential(args.key, user, pw)
|
|
84
|
+
elif args.command == c.DELETE_CRED_CMD:
|
|
85
|
+
cm.squirrels_config_io.delete_credential(args.key)
|
|
86
|
+
elif args.command in [c.RUN_CMD, c.TEST_CMD]:
|
|
87
|
+
manifest = mf.from_file()
|
|
88
|
+
conn_set = cs.from_file(manifest)
|
|
89
|
+
if args.command == c.RUN_CMD:
|
|
90
|
+
server = ApiServer(manifest, conn_set, args.no_cache, args.debug)
|
|
91
|
+
server.run(args)
|
|
92
|
+
elif args.command == c.TEST_CMD:
|
|
93
|
+
rendererIO = RendererIOWrapper(args.dataset, manifest, conn_set, args.data)
|
|
94
|
+
rendererIO.write_outputs(args.cfg, args.runquery)
|
|
95
|
+
conn_set.dispose()
|
|
96
|
+
elif args.command is None:
|
|
97
|
+
print(f'Command is missing. Enter "squirrels -h" for help.')
|
|
98
|
+
else:
|
|
99
|
+
print(f'Error: No such command "{args.command}". Enter "squirrels -h" for help.')
|
|
100
|
+
|
|
101
|
+
timer.report_times()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
if __name__ == '__main__':
|
|
105
|
+
main()
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
from typing import Dict, List, Tuple, Set
|
|
2
|
+
from fastapi import FastAPI, Request
|
|
3
|
+
from fastapi.datastructures import QueryParams
|
|
4
|
+
from fastapi.responses import HTMLResponse, JSONResponse
|
|
5
|
+
from fastapi.templating import Jinja2Templates
|
|
6
|
+
from fastapi.staticfiles import StaticFiles
|
|
7
|
+
from cachetools.func import ttl_cache
|
|
8
|
+
import os, json
|
|
9
|
+
|
|
10
|
+
from squirrels import major_version, constants as c, utils
|
|
11
|
+
from squirrels.manifest import Manifest
|
|
12
|
+
from squirrels.connection_set import ConnectionSet
|
|
13
|
+
from squirrels.renderer import RendererIOWrapper, Renderer
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ApiServer:
|
|
17
|
+
def __init__(self, manifest: Manifest, conn_set: ConnectionSet, no_cache: bool, debug: bool) -> None:
|
|
18
|
+
"""
|
|
19
|
+
Constructor for ApiServer
|
|
20
|
+
|
|
21
|
+
Parameters:
|
|
22
|
+
manifest (Manifest): Manifest object produced from squirrels.yaml
|
|
23
|
+
conn_set (ConnectionSet): Set of all connection pools defined in connections.py
|
|
24
|
+
no_cache (bool): Whether to disable caching
|
|
25
|
+
debug (bool): Set to True to show "hidden" parameters in the /parameters endpoint response
|
|
26
|
+
"""
|
|
27
|
+
self.manifest = manifest
|
|
28
|
+
self.conn_set = conn_set
|
|
29
|
+
self.no_cache = no_cache
|
|
30
|
+
self.debug = debug
|
|
31
|
+
|
|
32
|
+
self.datasets = manifest.get_all_dataset_names()
|
|
33
|
+
self.renderers: Dict[str, Renderer] = {}
|
|
34
|
+
for dataset in self.datasets:
|
|
35
|
+
rendererIO = RendererIOWrapper(dataset, manifest, conn_set)
|
|
36
|
+
self.renderers[dataset] = rendererIO.renderer
|
|
37
|
+
|
|
38
|
+
def _get_parameters_helper(self, dataset: str, query_params: Set[Tuple[str, str]]) -> Dict:
|
|
39
|
+
if len(query_params) > 1:
|
|
40
|
+
raise utils.InvalidInputError("The /parameters endpoint takes at most 1 query parameter")
|
|
41
|
+
renderer = self.renderers[dataset]
|
|
42
|
+
parameters = renderer.apply_selections(dict(query_params), updates_only = True)
|
|
43
|
+
return parameters.to_dict(self.debug)
|
|
44
|
+
|
|
45
|
+
def _get_results_helper(self, dataset: str, query_params: Set[Tuple[str, str]]) -> Dict:
|
|
46
|
+
renderer = self.renderers[dataset]
|
|
47
|
+
_, _, _, _, df = renderer.load_results(dict(query_params))
|
|
48
|
+
return json.loads(df.to_json(orient='table', index=False))
|
|
49
|
+
|
|
50
|
+
def _apply_dataset_api_function(self, api_function, dataset: str, raw_query_params: QueryParams):
|
|
51
|
+
dataset = utils.normalize_name(dataset)
|
|
52
|
+
query_params = set()
|
|
53
|
+
for key, val in raw_query_params.items():
|
|
54
|
+
query_params.add((utils.normalize_name(key), val))
|
|
55
|
+
query_params = frozenset(query_params)
|
|
56
|
+
return api_function(dataset, query_params)
|
|
57
|
+
|
|
58
|
+
def run(self, uvicorn_args: List[str]) -> None:
|
|
59
|
+
"""
|
|
60
|
+
Runs the API server with uvicorn for CLI "squirrels run"
|
|
61
|
+
|
|
62
|
+
Parameters:
|
|
63
|
+
uvicorn_args (List[str]): List of arguments to pass to uvicorn.run. Currently only supports "host" and "port"
|
|
64
|
+
"""
|
|
65
|
+
app = FastAPI()
|
|
66
|
+
|
|
67
|
+
squirrels_version_path = f'/squirrels{major_version}'
|
|
68
|
+
config_base_path = utils.normalize_name_for_api(self.manifest.get_base_path())
|
|
69
|
+
base_path = squirrels_version_path + config_base_path
|
|
70
|
+
|
|
71
|
+
static_dir = utils.join_paths(os.path.dirname(__file__), 'package_data', 'static')
|
|
72
|
+
app.mount('/static', StaticFiles(directory=static_dir), name='static')
|
|
73
|
+
|
|
74
|
+
templates_dir = utils.join_paths(os.path.dirname(__file__), 'package_data', 'templates')
|
|
75
|
+
templates = Jinja2Templates(directory=templates_dir)
|
|
76
|
+
|
|
77
|
+
# Parameters API
|
|
78
|
+
parameters_path = base_path + '/{dataset}/parameters'
|
|
79
|
+
|
|
80
|
+
parameters_cache_size = self.manifest.get_setting(c.PARAMETERS_CACHE_SIZE_SETTING, 1024)
|
|
81
|
+
parameters_cache_ttl = self.manifest.get_setting(c.PARAMETERS_CACHE_TTL_SETTING, 24*60*60)
|
|
82
|
+
|
|
83
|
+
@ttl_cache(maxsize=parameters_cache_size, ttl=parameters_cache_ttl)
|
|
84
|
+
def get_parameters_cachable(*args):
|
|
85
|
+
return self._get_parameters_helper(*args)
|
|
86
|
+
|
|
87
|
+
@app.get(parameters_path, response_class=JSONResponse)
|
|
88
|
+
async def get_parameters(dataset: str, request: Request):
|
|
89
|
+
api_function = self._get_parameters_helper if self.no_cache else get_parameters_cachable
|
|
90
|
+
return self._apply_dataset_api_function(api_function, dataset, request.query_params)
|
|
91
|
+
|
|
92
|
+
# Results API
|
|
93
|
+
results_path = base_path + '/{dataset}'
|
|
94
|
+
|
|
95
|
+
results_cache_size = self.manifest.get_setting(c.RESULTS_CACHE_SIZE_SETTING, 128)
|
|
96
|
+
results_cache_ttl = self.manifest.get_setting(c.RESULTS_CACHE_TTL_SETTING, 60*60)
|
|
97
|
+
|
|
98
|
+
@ttl_cache(maxsize=results_cache_size, ttl=results_cache_ttl)
|
|
99
|
+
def get_results_cachable(*args):
|
|
100
|
+
return self._get_results_helper(*args)
|
|
101
|
+
|
|
102
|
+
@app.get(results_path, response_class=JSONResponse)
|
|
103
|
+
async def get_results(dataset: str, request: Request):
|
|
104
|
+
api_function = self._get_results_helper if self.no_cache else get_results_cachable
|
|
105
|
+
return self._apply_dataset_api_function(api_function, dataset, request.query_params)
|
|
106
|
+
|
|
107
|
+
# Catalog API
|
|
108
|
+
@app.get(base_path, response_class=JSONResponse)
|
|
109
|
+
async def get_catalog():
|
|
110
|
+
datasets_info = []
|
|
111
|
+
for dataset in self.datasets:
|
|
112
|
+
dataset_normalized = utils.normalize_name_for_api(dataset)
|
|
113
|
+
datasets_info.append({
|
|
114
|
+
'dataset': dataset,
|
|
115
|
+
'label': self.manifest.get_dataset_label(dataset),
|
|
116
|
+
'parameters_path': parameters_path.format(dataset=dataset_normalized),
|
|
117
|
+
'result_path': results_path.format(dataset=dataset_normalized)
|
|
118
|
+
})
|
|
119
|
+
return {'project_variables': self.manifest.get_proj_vars(), 'resource_paths': datasets_info}
|
|
120
|
+
|
|
121
|
+
# Squirrels UI
|
|
122
|
+
@app.get('/', response_class=HTMLResponse)
|
|
123
|
+
async def get_ui(request: Request):
|
|
124
|
+
return templates.TemplateResponse('index.html', {'request': request, 'base_path': base_path})
|
|
125
|
+
|
|
126
|
+
# Run API server
|
|
127
|
+
import uvicorn
|
|
128
|
+
uvicorn.run(app, host=uvicorn_args.host, port=uvicorn_args.port)
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
from typing import Dict, Union
|
|
2
|
+
from importlib.machinery import SourceFileLoader
|
|
3
|
+
from sqlalchemy import Engine, Pool
|
|
4
|
+
import sqlite3
|
|
5
|
+
|
|
6
|
+
from squirrels import manifest as mf, constants as c
|
|
7
|
+
from squirrels.timed_imports import pandas as pd
|
|
8
|
+
from squirrels.utils import ConfigurationError
|
|
9
|
+
|
|
10
|
+
ConnectionPool = Union[Engine, Pool]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ConnectionSet:
|
|
14
|
+
def __init__(self, conn_pools: Dict[str, ConnectionPool]) -> None:
|
|
15
|
+
self._conn_pools = conn_pools
|
|
16
|
+
|
|
17
|
+
def get_connection_pool(self, conn_name: str) -> ConnectionPool:
|
|
18
|
+
try:
|
|
19
|
+
connection_pool = self._conn_pools[conn_name]
|
|
20
|
+
except KeyError as e:
|
|
21
|
+
raise ConfigurationError(f'Connection name "{conn_name}" was not configured') from e
|
|
22
|
+
return connection_pool
|
|
23
|
+
|
|
24
|
+
def get_dataframe_from_query(self, conn_name: str, query: str) -> pd.DataFrame:
|
|
25
|
+
connector = self.get_connection_pool(conn_name)
|
|
26
|
+
if isinstance(connector, Pool):
|
|
27
|
+
conn = connector.connect()
|
|
28
|
+
elif isinstance(connector, Engine):
|
|
29
|
+
conn = connector.raw_connection()
|
|
30
|
+
else:
|
|
31
|
+
raise TypeError(f'Type for connection name "{conn_name}" not supported')
|
|
32
|
+
|
|
33
|
+
try:
|
|
34
|
+
cur = conn.cursor()
|
|
35
|
+
cur.execute(query)
|
|
36
|
+
data = cur.fetchall()
|
|
37
|
+
columns = [x[0] for x in cur.description]
|
|
38
|
+
df = pd.DataFrame(data, columns=columns)
|
|
39
|
+
finally:
|
|
40
|
+
conn.close()
|
|
41
|
+
|
|
42
|
+
return df
|
|
43
|
+
|
|
44
|
+
def dispose(self) -> None:
|
|
45
|
+
for pool in self._conn_pools.values():
|
|
46
|
+
pool.dispose()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def from_file(manifest: mf.Manifest) -> ConnectionSet:
|
|
50
|
+
module = SourceFileLoader(c.CONNECTIONS_FILE, c.CONNECTIONS_FILE).load_module()
|
|
51
|
+
proj_vars = manifest.get_proj_vars()
|
|
52
|
+
try:
|
|
53
|
+
return module.main(proj_vars)
|
|
54
|
+
except Exception as e:
|
|
55
|
+
raise ConfigurationError(f'Error in the {c.CONNECTIONS_FILE} file') from e
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def sqldf(query: str, df_by_db_views: Dict[str, pd.DataFrame]) -> pd.DataFrame:
|
|
59
|
+
conn = sqlite3.connect(":memory:")
|
|
60
|
+
try:
|
|
61
|
+
for db_view, df in df_by_db_views.items():
|
|
62
|
+
df.to_sql(db_view, conn, index=False)
|
|
63
|
+
return pd.read_sql(query, conn)
|
|
64
|
+
finally:
|
|
65
|
+
conn.close()
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Squirrels CLI commands
|
|
2
|
+
INIT_CMD = 'init'
|
|
3
|
+
LOAD_MODULES_CMD = 'load-modules'
|
|
4
|
+
GET_CREDS_CMD = 'get-all-credentials'
|
|
5
|
+
SET_CRED_CMD = 'set-credential'
|
|
6
|
+
DELETE_CRED_CMD = 'delete-credential'
|
|
7
|
+
TEST_CMD = 'test'
|
|
8
|
+
RUN_CMD = 'run'
|
|
9
|
+
|
|
10
|
+
# Manifest file keys
|
|
11
|
+
DB_CONNECTION_KEY = 'db_connection'
|
|
12
|
+
MODULES_KEY = 'modules'
|
|
13
|
+
DATASET_LABEL_KEY = 'label'
|
|
14
|
+
DATASETS_KEY = 'datasets'
|
|
15
|
+
HEADERS_KEY = 'headers'
|
|
16
|
+
DATABASE_VIEWS_KEY = 'database_views'
|
|
17
|
+
DB_VIEW_NAME_KEY = 'name'
|
|
18
|
+
DB_VIEW_FILE_KEY = 'file'
|
|
19
|
+
FINAL_VIEW_KEY = 'final_view'
|
|
20
|
+
BASE_PATH_KEY = 'base_path'
|
|
21
|
+
SETTINGS_KEY = 'settings'
|
|
22
|
+
|
|
23
|
+
# Database credentials keys
|
|
24
|
+
CREDENTIALS_KEY = 'credentials'
|
|
25
|
+
USERNAME_KEY = 'username'
|
|
26
|
+
PASSWORD_KEY = 'password'
|
|
27
|
+
|
|
28
|
+
# Folder/File names
|
|
29
|
+
MANIFEST_FILE = 'squirrels.yaml'
|
|
30
|
+
PROJ_VARS_FILE = 'project_vars.yaml'
|
|
31
|
+
CONNECTIONS_FILE = 'connections.py'
|
|
32
|
+
OUTPUTS_FOLDER = 'outputs'
|
|
33
|
+
MODULES_FOLDER = 'modules'
|
|
34
|
+
DATASETS_FOLDER = 'datasets'
|
|
35
|
+
PARAMETERS_FILE = 'parameters.py'
|
|
36
|
+
PARAMETERS_OUTPUT = 'parameters.json'
|
|
37
|
+
DATABASE_VIEW_SQL_FILE = 'database_view1.sql.j2'
|
|
38
|
+
DATABASE_VIEW_PY_FILE = 'database_view1.py'
|
|
39
|
+
FINAL_VIEW_SQL_NAME = 'final_view.sql.j2'
|
|
40
|
+
FINAL_VIEW_PY_NAME = 'final_view.py'
|
|
41
|
+
FINAL_VIEW_OUT_STEM = 'final_view'
|
|
42
|
+
CONTEXT_FILE = 'context.py'
|
|
43
|
+
SELECTIONS_CFG_FILE = 'selections.cfg'
|
|
44
|
+
|
|
45
|
+
# Dataset setting names
|
|
46
|
+
PARAMETERS_CACHE_SIZE_SETTING = 'parameters.cache.size'
|
|
47
|
+
PARAMETERS_CACHE_TTL_SETTING = 'parameters.cache.ttl'
|
|
48
|
+
RESULTS_CACHE_SIZE_SETTING = 'results.cache.size'
|
|
49
|
+
RESULTS_CACHE_TTL_SETTING = 'results.cache.ttl'
|
|
50
|
+
|
|
51
|
+
# Selection cfg sections
|
|
52
|
+
PARAMETERS_SECTION = 'parameters'
|
|
53
|
+
HEADERS_SECTION = 'headers'
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
from typing import Dict
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from configparser import ConfigParser
|
|
4
|
+
import os, json
|
|
5
|
+
|
|
6
|
+
from squirrels.utils import ConfigurationError
|
|
7
|
+
from squirrels import constants as c, utils
|
|
8
|
+
|
|
9
|
+
_SQUIRRELS_CFG_PATH = utils.join_paths(os.path.expanduser('~'), '.squirrelscfg')
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class Credential:
|
|
14
|
+
username: str
|
|
15
|
+
password: str
|
|
16
|
+
|
|
17
|
+
def __str__(self) -> str:
|
|
18
|
+
redacted_pass = '*'*len(self.password)
|
|
19
|
+
return f'username={self.username}, password={redacted_pass}'
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SquirrelsConfigParser(ConfigParser):
|
|
23
|
+
def _get_creds_section(self):
|
|
24
|
+
section_name: str = c.CREDENTIALS_KEY
|
|
25
|
+
if not self.has_section(section_name):
|
|
26
|
+
self.add_section(section_name)
|
|
27
|
+
return self[section_name]
|
|
28
|
+
|
|
29
|
+
def _json_str_to_credential(self, json_str: str) -> Credential:
|
|
30
|
+
cred_dict = json.loads(json_str)
|
|
31
|
+
return Credential(cred_dict[c.USERNAME_KEY], cred_dict[c.PASSWORD_KEY])
|
|
32
|
+
|
|
33
|
+
def get_credential(self, key: str) -> Credential:
|
|
34
|
+
section = self._get_creds_section()
|
|
35
|
+
try:
|
|
36
|
+
value = section[key]
|
|
37
|
+
except KeyError as e:
|
|
38
|
+
raise ConfigurationError(f'Credential key "{key}" has not been set. To set it, use $ squirrels set-credential {key}')
|
|
39
|
+
return self._json_str_to_credential(value)
|
|
40
|
+
|
|
41
|
+
def get_all_credentials(self) -> Dict[str, Credential]:
|
|
42
|
+
section = self._get_creds_section()
|
|
43
|
+
result = {}
|
|
44
|
+
for key, value in section.items():
|
|
45
|
+
result[key] = self._json_str_to_credential(value)
|
|
46
|
+
return result
|
|
47
|
+
|
|
48
|
+
def set_credential(self, key: str, credential: Credential) -> ConfigParser:
|
|
49
|
+
section = self._get_creds_section()
|
|
50
|
+
section[key] = json.dumps(credential.__dict__)
|
|
51
|
+
return self
|
|
52
|
+
|
|
53
|
+
def delete_credential(self, key: str) -> ConfigParser:
|
|
54
|
+
section = self._get_creds_section()
|
|
55
|
+
section.pop(key)
|
|
56
|
+
return self
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class SquirrelsConfigIOWrapper:
|
|
60
|
+
def __init__(self) -> None:
|
|
61
|
+
self.config = SquirrelsConfigParser()
|
|
62
|
+
self.config.read(_SQUIRRELS_CFG_PATH)
|
|
63
|
+
|
|
64
|
+
def get_credential(self, key: str) -> Credential:
|
|
65
|
+
return self.config.get_credential(key)
|
|
66
|
+
|
|
67
|
+
def print_all_credentials(self) -> None:
|
|
68
|
+
credentials_dict = self.config.get_all_credentials()
|
|
69
|
+
for key, cred in credentials_dict.items():
|
|
70
|
+
print(f'{key}:', cred)
|
|
71
|
+
|
|
72
|
+
def _write_config(self) -> None:
|
|
73
|
+
with open(_SQUIRRELS_CFG_PATH, 'w') as f:
|
|
74
|
+
self.config.write(f)
|
|
75
|
+
|
|
76
|
+
def set_credential(self, key: str, user: str, pw: str) -> None:
|
|
77
|
+
credential = Credential(user, pw)
|
|
78
|
+
self.config.set_credential(key, credential)
|
|
79
|
+
print(f'Credential key "{key}" set to: {credential}')
|
|
80
|
+
self._write_config()
|
|
81
|
+
|
|
82
|
+
def delete_credential(self, key: str) -> None:
|
|
83
|
+
self.config.delete_credential(key)
|
|
84
|
+
print(f'Credential key "{key}" has been deleted')
|
|
85
|
+
self._write_config()
|
|
86
|
+
|
|
87
|
+
squirrels_config_io = SquirrelsConfigIOWrapper()
|