squirrels 0.1.0rc1__tar.gz → 0.1.0rc2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of squirrels might be problematic. Click here for more details.

Files changed (87) hide show
  1. {squirrels-0.1.0rc1 → squirrels-0.1.0rc2}/LICENSE +1 -0
  2. squirrels-0.1.0rc2/PKG-INFO +56 -0
  3. squirrels-0.1.0rc2/README.md +46 -0
  4. squirrels-0.1.0rc2/setup.py +42 -0
  5. squirrels-0.1.0rc2/squirrels/__init__.py +19 -0
  6. squirrels-0.1.0rc2/squirrels/_command_line.py +105 -0
  7. squirrels-0.1.0rc2/squirrels/api_server.py +128 -0
  8. squirrels-0.1.0rc2/squirrels/connection_set.py +65 -0
  9. squirrels-0.1.0rc2/squirrels/constants.py +53 -0
  10. squirrels-0.1.0rc2/squirrels/credentials_manager.py +87 -0
  11. squirrels-0.1.0rc2/squirrels/dateutils.py +272 -0
  12. {squirrels-0.1.0rc1 → squirrels-0.1.0rc2}/squirrels/initializer.py +37 -27
  13. squirrels-0.1.0rc2/squirrels/manifest.py +123 -0
  14. {squirrels-0.1.0rc1 → squirrels-0.1.0rc2}/squirrels/module_loader.py +4 -7
  15. {squirrels-0.1.0rc1/squirrels → squirrels-0.1.0rc2/squirrels/package_data}/base_project/.gitignore +1 -0
  16. squirrels-0.1.0rc2/squirrels/package_data/base_project/connections.py +21 -0
  17. squirrels-0.1.0rc2/squirrels/package_data/base_project/database/sample_database.db +0 -0
  18. squirrels-0.1.0rc2/squirrels/package_data/base_project/datasets/sample_dataset/context.py +8 -0
  19. squirrels-0.1.0rc2/squirrels/package_data/base_project/datasets/sample_dataset/database_view1.py +21 -0
  20. squirrels-0.1.0rc2/squirrels/package_data/base_project/datasets/sample_dataset/database_view1.sql.j2 +5 -0
  21. squirrels-0.1.0rc2/squirrels/package_data/base_project/datasets/sample_dataset/final_view.py +9 -0
  22. squirrels-0.1.0rc2/squirrels/package_data/base_project/datasets/sample_dataset/parameters.py +31 -0
  23. squirrels-0.1.0rc2/squirrels/package_data/base_project/datasets/sample_dataset/selections.cfg +6 -0
  24. squirrels-0.1.0rc2/squirrels/package_data/base_project/project_vars.yaml +3 -0
  25. squirrels-0.1.0rc2/squirrels/package_data/base_project/squirrels.yaml +15 -0
  26. {squirrels-0.1.0rc1/squirrels → squirrels-0.1.0rc2/squirrels/package_data}/static/script.js +20 -9
  27. {squirrels-0.1.0rc1/squirrels → squirrels-0.1.0rc2/squirrels/package_data}/templates/index.html +2 -2
  28. squirrels-0.1.0rc2/squirrels/param_configs/__init__.py +0 -0
  29. squirrels-0.1.0rc2/squirrels/param_configs/data_sources.py +329 -0
  30. squirrels-0.1.0rc2/squirrels/param_configs/parameter_options.py +189 -0
  31. squirrels-0.1.0rc2/squirrels/param_configs/parameter_set.py +36 -0
  32. squirrels-0.1.0rc2/squirrels/param_configs/parameters.py +435 -0
  33. squirrels-0.1.0rc2/squirrels/renderer.py +279 -0
  34. squirrels-0.1.0rc2/squirrels/timed_imports.py +36 -0
  35. squirrels-0.1.0rc2/squirrels/utils.py +94 -0
  36. squirrels-0.1.0rc2/squirrels/version.py +3 -0
  37. squirrels-0.1.0rc2/squirrels.egg-info/PKG-INFO +56 -0
  38. squirrels-0.1.0rc2/squirrels.egg-info/SOURCES.txt +45 -0
  39. squirrels-0.1.0rc2/squirrels.egg-info/entry_points.txt +2 -0
  40. {squirrels-0.1.0rc1 → squirrels-0.1.0rc2}/squirrels.egg-info/requires.txt +2 -2
  41. squirrels-0.1.0rc1/.gitignore +0 -82
  42. squirrels-0.1.0rc1/PKG-INFO +0 -47
  43. squirrels-0.1.0rc1/README.md +0 -37
  44. squirrels-0.1.0rc1/requirements.txt +0 -12
  45. squirrels-0.1.0rc1/sample_project/.gitignore +0 -2
  46. squirrels-0.1.0rc1/sample_project/datasets/sample_lu_data.csv +0 -4
  47. squirrels-0.1.0rc1/sample_project/datasets/stock_price_history/context.py +0 -22
  48. squirrels-0.1.0rc1/sample_project/datasets/stock_price_history/final_view.py +0 -7
  49. squirrels-0.1.0rc1/sample_project/datasets/stock_price_history/final_view.sql.j2 +0 -6
  50. squirrels-0.1.0rc1/sample_project/datasets/stock_price_history/parameters.py +0 -41
  51. squirrels-0.1.0rc1/sample_project/datasets/stock_price_history/selections.cfg +0 -6
  52. squirrels-0.1.0rc1/sample_project/datasets/stock_price_history/ticker_history.sql.j2 +0 -8
  53. squirrels-0.1.0rc1/sample_project/functions.py +0 -43
  54. squirrels-0.1.0rc1/sample_project/parameter_options.py +0 -53
  55. squirrels-0.1.0rc1/setup.py +0 -38
  56. squirrels-0.1.0rc1/squirrels/__init__.py +0 -10
  57. squirrels-0.1.0rc1/squirrels/api_server.py +0 -136
  58. squirrels-0.1.0rc1/squirrels/base_project/datasets/sample_dataset/context.py +0 -6
  59. squirrels-0.1.0rc1/squirrels/base_project/datasets/sample_dataset/database_view1.py +0 -10
  60. squirrels-0.1.0rc1/squirrels/base_project/datasets/sample_dataset/database_view1.sql.j2 +0 -4
  61. squirrels-0.1.0rc1/squirrels/base_project/datasets/sample_dataset/final_view.py +0 -9
  62. squirrels-0.1.0rc1/squirrels/base_project/datasets/sample_dataset/parameters.py +0 -8
  63. squirrels-0.1.0rc1/squirrels/base_project/squirrels.yaml +0 -20
  64. squirrels-0.1.0rc1/squirrels/command_line.py +0 -114
  65. squirrels-0.1.0rc1/squirrels/constants.py +0 -72
  66. squirrels-0.1.0rc1/squirrels/db_conn.py +0 -41
  67. squirrels-0.1.0rc1/squirrels/manifest.py +0 -38
  68. squirrels-0.1.0rc1/squirrels/parameter_configs.py +0 -512
  69. squirrels-0.1.0rc1/squirrels/profile_manager.py +0 -57
  70. squirrels-0.1.0rc1/squirrels/renderer.py +0 -225
  71. squirrels-0.1.0rc1/squirrels/templates/sandbox.html +0 -25
  72. squirrels-0.1.0rc1/squirrels/version.txt +0 -1
  73. squirrels-0.1.0rc1/squirrels.egg-info/PKG-INFO +0 -47
  74. squirrels-0.1.0rc1/squirrels.egg-info/SOURCES.txt +0 -52
  75. squirrels-0.1.0rc1/squirrels.egg-info/entry_points.txt +0 -2
  76. squirrels-0.1.0rc1/tests/conftest.py +0 -9
  77. squirrels-0.1.0rc1/tests/test_db_conn.py +0 -14
  78. squirrels-0.1.0rc1/tests/test_module_loader.py +0 -17
  79. squirrels-0.1.0rc1/tests/test_parameter_configs.py +0 -120
  80. squirrels-0.1.0rc1/tests/test_profile_manager.py +0 -28
  81. {squirrels-0.1.0rc1 → squirrels-0.1.0rc2}/setup.cfg +0 -0
  82. {squirrels-0.1.0rc1/squirrels → squirrels-0.1.0rc2/squirrels/package_data}/base_project/database/seattle_weather.db +0 -0
  83. {squirrels-0.1.0rc1/squirrels → squirrels-0.1.0rc2/squirrels/package_data}/base_project/datasets/sample_dataset/final_view.sql.j2 +0 -0
  84. {squirrels-0.1.0rc1/squirrels → squirrels-0.1.0rc2/squirrels/package_data}/static/favicon.ico +0 -0
  85. {squirrels-0.1.0rc1/squirrels → squirrels-0.1.0rc2/squirrels/package_data}/static/style.css +0 -0
  86. {squirrels-0.1.0rc1 → squirrels-0.1.0rc2}/squirrels.egg-info/dependency_links.txt +0 -0
  87. {squirrels-0.1.0rc1 → squirrels-0.1.0rc2}/squirrels.egg-info/top_level.txt +0 -0
@@ -19,3 +19,4 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
19
  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
20
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
21
  SOFTWARE.
22
+
@@ -0,0 +1,56 @@
1
+ Metadata-Version: 2.1
2
+ Name: squirrels
3
+ Version: 0.1.0rc2
4
+ Summary: Python Package for Configuring SQL Generating APIs
5
+ Author: Tim Huang
6
+ Author-email: tim.yuting@hotmail.com
7
+ License: MIT
8
+ Description-Content-Type: text/markdown
9
+ License-File: LICENSE
10
+
11
+ # Squirrels
12
+
13
+ Squirrels is an API framework for creating REST APIs that generate sql queries & dataframes dynamically from query parameters.
14
+
15
+ ## Setup
16
+
17
+ First, install the library and all dependencies **in an independent virtual environment**.
18
+
19
+ ```bash
20
+ # activate virtual environment...
21
+ pip install -e .
22
+ ```
23
+
24
+ To confirm that the setup worked, run this to show the help page for all squirrels CLI commands:
25
+
26
+ ```bash
27
+ squirrels -h
28
+ ```
29
+
30
+ ## Testing
31
+
32
+ ```
33
+ python setup.py pytest
34
+ ```
35
+
36
+ ## Documentation
37
+
38
+ Check the documentation website [here](https://squirrels-nest.github.io/squirrels-docs/) or get started [here](https://squirrels-nest.github.io/squirrels-docs/getting-started/) to learn how to use the squirrels library to make an API project.
39
+
40
+ ## Developer Guide
41
+
42
+ From the root of the git repo, the source code can be found in the `squirrels` folder and unit tests can be found in the `tests` folder.
43
+
44
+ To understand what a specific squirrels command line utility is doing, start from the `_command_line.py` file as your entry point.
45
+
46
+ The library version is maintained in both the `setup.py` file (for the next release or release-candidate version) and the `squirrels/version.py` file (for the next release version only).
47
+
48
+ When a user initializes a squirrels project using `squirrels init`, the files are copied from the `squirrels/package_data/base_project` folder. The contents in the `database` subfolder were constructed from the scripts in the `database_elt` folder at the top level.
49
+
50
+ For the Squirrels UI activated by `squirrels run`, the HTML, CSS, and Javascript files can be found in the `static` and `templates` subfolders of `squirrels/package_data`.
51
+
52
+ ## License
53
+
54
+ Squirrels is released under the MIT license.
55
+
56
+ See the file LICENSE for more details.
@@ -0,0 +1,46 @@
1
+ # Squirrels
2
+
3
+ Squirrels is an API framework for creating REST APIs that generate sql queries & dataframes dynamically from query parameters.
4
+
5
+ ## Setup
6
+
7
+ First, install the library and all dependencies **in an independent virtual environment**.
8
+
9
+ ```bash
10
+ # activate virtual environment...
11
+ pip install -e .
12
+ ```
13
+
14
+ To confirm that the setup worked, run this to show the help page for all squirrels CLI commands:
15
+
16
+ ```bash
17
+ squirrels -h
18
+ ```
19
+
20
+ ## Testing
21
+
22
+ ```
23
+ python setup.py pytest
24
+ ```
25
+
26
+ ## Documentation
27
+
28
+ Check the documentation website [here](https://squirrels-nest.github.io/squirrels-docs/) or get started [here](https://squirrels-nest.github.io/squirrels-docs/getting-started/) to learn how to use the squirrels library to make an API project.
29
+
30
+ ## Developer Guide
31
+
32
+ From the root of the git repo, the source code can be found in the `squirrels` folder and unit tests can be found in the `tests` folder.
33
+
34
+ To understand what a specific squirrels command line utility is doing, start from the `_command_line.py` file as your entry point.
35
+
36
+ The library version is maintained in both the `setup.py` file (for the next release or release-candidate version) and the `squirrels/version.py` file (for the next release version only).
37
+
38
+ When a user initializes a squirrels project using `squirrels init`, the files are copied from the `squirrels/package_data/base_project` folder. The contents in the `database` subfolder were constructed from the scripts in the `database_elt` folder at the top level.
39
+
40
+ For the Squirrels UI activated by `squirrels run`, the HTML, CSS, and Javascript files can be found in the `static` and `templates` subfolders of `squirrels/package_data`.
41
+
42
+ ## License
43
+
44
+ Squirrels is released under the MIT license.
45
+
46
+ See the file LICENSE for more details.
@@ -0,0 +1,42 @@
1
+ from setuptools import setup, find_packages
2
+ import os
3
+
4
+ # The directory containing this file
5
+ HERE = os.path.abspath(os.path.dirname(__file__))
6
+
7
+ # Get the long description from the README file
8
+ with open(os.path.join(HERE, 'README.md'), encoding='utf-8') as f:
9
+ long_description = f.read()
10
+
11
+ # Recursively get package data
12
+ def package_files(directory):
13
+ paths = []
14
+ for (path, directories, filenames) in os.walk(directory):
15
+ for filename in filenames:
16
+ paths.append(os.path.join('..', path, filename))
17
+ return paths
18
+
19
+ extra_files = package_files(os.path.join('squirrels', 'package_data'))
20
+
21
+ setup(
22
+ name='squirrels',
23
+ version='0.1.0rc2',
24
+ packages=find_packages(),
25
+ description='Python Package for Configuring SQL Generating APIs',
26
+ long_description=long_description,
27
+ long_description_content_type="text/markdown",
28
+ author='Tim Huang',
29
+ author_email='tim.yuting@hotmail.com',
30
+ license='MIT',
31
+ install_requires=[
32
+ 'openpyxl', 'inquirer', 'pwinput', 'cachetools', 'fastapi', 'uvicorn',
33
+ 'Jinja2', 'GitPython', 'sqlalchemy', 'pandas', 'pyyaml'
34
+ ],
35
+ setup_requires=['pytest-runner==6.0.0'],
36
+ tests_require=['pytest==7.2.0'],
37
+ test_suite='tests',
38
+ package_data= {'squirrels': extra_files},
39
+ entry_points= {
40
+ 'console_scripts': ['squirrels=squirrels._command_line:main']
41
+ }
42
+ )
@@ -0,0 +1,19 @@
1
+ from .param_configs.parameter_options import SelectParameterOption, DateParameterOption, NumberParameterOption, NumRangeParameterOption
2
+ from .param_configs.parameters import WidgetType, Parameter, SingleSelectParameter, MultiSelectParameter, DateParameter, NumberParameter, NumRangeParameter
3
+ from .param_configs.data_sources import SelectionDataSource, DateDataSource, NumberDataSource, NumRangeDataSource, DataSourceParameter
4
+ from .param_configs.parameter_set import ParameterSet
5
+ from .connection_set import ConnectionSet
6
+ from .version import __version__, major_version, minor_version, patch_version
7
+
8
+ def get_credential(key: str):
9
+ """
10
+ Gets the username and password that was set through "$squirrels set-credential [key]"
11
+
12
+ Parameters:
13
+ key (str): The credential key
14
+
15
+ Returns:
16
+ Credential: Object with attributes "username" and "password"
17
+ """
18
+ from .credentials_manager import squirrels_config_io
19
+ return squirrels_config_io.get_credential(key)
@@ -0,0 +1,105 @@
1
+ from typing import List, Tuple, Optional
2
+ from argparse import ArgumentParser
3
+ import sys, time, pwinput
4
+ sys.path.append('.')
5
+
6
+ from squirrels import constants as c, __version__
7
+ from squirrels import module_loader as ml, manifest as mf, credentials_manager as cm, connection_set as cs
8
+ from squirrels.api_server import ApiServer
9
+ from squirrels.renderer import RendererIOWrapper
10
+ from squirrels.initializer import Initializer
11
+ from squirrels.timed_imports import timer
12
+
13
+
14
+ def _prompt_user_pw(args_values: Optional[List[str]]) -> Tuple[str, str]:
15
+ if args_values is not None:
16
+ user, pw = args_values
17
+ else:
18
+ user = input("Enter username: ")
19
+ pw = pwinput.pwinput("Enter password: ")
20
+ return user, pw
21
+
22
+
23
+ def main():
24
+ """
25
+ Main entry point for the squirrels command line utilities.
26
+ """
27
+ start = time.time()
28
+ parser = ArgumentParser(description="Command line utilities from the squirrels python package")
29
+ parser.add_argument('-v', '--version', action='store_true', help='Show the version and exit')
30
+ parser.add_argument('--verbose', action='store_true', help='Enable verbose output')
31
+ subparsers = parser.add_subparsers(title='commands', dest='command')
32
+
33
+ init_parser = subparsers.add_parser(c.INIT_CMD, help='Initialize a squirrels project')
34
+ init_parser.add_argument('--no-overwrite', action='store_true', help="Don't overwrite files if they already exist")
35
+ init_parser.add_argument('--core', action='store_true', help='Include all core files')
36
+ init_parser.add_argument('--context', action='store_true', help=f'Include the {c.CONTEXT_FILE} file')
37
+ init_parser.add_argument('--selections-cfg', action='store_true', help=f'Include the {c.SELECTIONS_CFG_FILE} file')
38
+ init_parser.add_argument('--db-view', type=str, choices=['sql', 'py'], help='Create database view as sql (default) or python file')
39
+ init_parser.add_argument('--final-view', type=str, choices=['sql', 'py'], help='Include final view as sql or python file')
40
+ init_parser.add_argument('--sample-db', type=str, choices=['seattle-weather'], help='Sample sqlite database to include')
41
+
42
+ subparsers.add_parser(c.LOAD_MODULES_CMD, help='Load all the modules specified in squirrels.yaml from git')
43
+
44
+ def _add_profile_argument(parser: ArgumentParser):
45
+ parser.add_argument('key', type=str, help='Key to the database connection credential')
46
+
47
+ subparsers.add_parser(c.GET_CREDS_CMD, help='Get all database connection credential keys')
48
+
49
+ set_cred_parser = subparsers.add_parser(c.SET_CRED_CMD, help='Set a database connection credential key')
50
+ _add_profile_argument(set_cred_parser)
51
+ set_cred_parser.add_argument('--values', type=str, nargs=2, help='The username and password')
52
+
53
+ delete_cred_parser = subparsers.add_parser(c.DELETE_CRED_CMD, help='Delete a database connection credential key')
54
+ _add_profile_argument(delete_cred_parser)
55
+
56
+ test_parser = subparsers.add_parser(c.TEST_CMD, help='For a given dataset, create outputs for parameter API response and rendered sql queries')
57
+ test_parser.add_argument('dataset', type=str, help='Name of dataset (provided in squirrels.yaml) to test. Results are written in an "outputs" folder')
58
+ test_parser.add_argument('-c', '--cfg', type=str, help="Configuration file for parameter selections. Path is relative to the dataset's folder")
59
+ test_parser.add_argument('-d', '--data', type=str, help="Excel file with lookup data to avoid making a database connection. Path is relative to the dataset's folder")
60
+ test_parser.add_argument('-r', '--runquery', action='store_true', help='Runs all database queries and final view, and produce the results as csv files')
61
+
62
+ run_parser = subparsers.add_parser(c.RUN_CMD, help='Run the builtin API server')
63
+ run_parser.add_argument('--no-cache', action='store_true', help='Do not cache any api results')
64
+ run_parser.add_argument('--debug', action='store_true', help='In debug mode, all "hidden parameters" show in the parameters response')
65
+ run_parser.add_argument('--host', type=str, default='127.0.0.1')
66
+ run_parser.add_argument('--port', type=int, default=8000)
67
+
68
+ args, _ = parser.parse_known_args()
69
+ timer.verbose = args.verbose
70
+ timer.add_activity_time('parsing arguments', start)
71
+
72
+ if args.version:
73
+ print(__version__)
74
+ elif args.command == c.INIT_CMD:
75
+ Initializer(not args.no_overwrite).init_project(args)
76
+ elif args.command == c.LOAD_MODULES_CMD:
77
+ manifest = mf.from_file()
78
+ ml.load_modules(manifest)
79
+ elif args.command == c.GET_CREDS_CMD:
80
+ cm.squirrels_config_io.print_all_credentials()
81
+ elif args.command == c.SET_CRED_CMD:
82
+ user, pw = _prompt_user_pw(args.values)
83
+ cm.squirrels_config_io.set_credential(args.key, user, pw)
84
+ elif args.command == c.DELETE_CRED_CMD:
85
+ cm.squirrels_config_io.delete_credential(args.key)
86
+ elif args.command in [c.RUN_CMD, c.TEST_CMD]:
87
+ manifest = mf.from_file()
88
+ conn_set = cs.from_file(manifest)
89
+ if args.command == c.RUN_CMD:
90
+ server = ApiServer(manifest, conn_set, args.no_cache, args.debug)
91
+ server.run(args)
92
+ elif args.command == c.TEST_CMD:
93
+ rendererIO = RendererIOWrapper(args.dataset, manifest, conn_set, args.data)
94
+ rendererIO.write_outputs(args.cfg, args.runquery)
95
+ conn_set.dispose()
96
+ elif args.command is None:
97
+ print(f'Command is missing. Enter "squirrels -h" for help.')
98
+ else:
99
+ print(f'Error: No such command "{args.command}". Enter "squirrels -h" for help.')
100
+
101
+ timer.report_times()
102
+
103
+
104
+ if __name__ == '__main__':
105
+ main()
@@ -0,0 +1,128 @@
1
+ from typing import Dict, List, Tuple, Set
2
+ from fastapi import FastAPI, Request
3
+ from fastapi.datastructures import QueryParams
4
+ from fastapi.responses import HTMLResponse, JSONResponse
5
+ from fastapi.templating import Jinja2Templates
6
+ from fastapi.staticfiles import StaticFiles
7
+ from cachetools.func import ttl_cache
8
+ import os, json
9
+
10
+ from squirrels import major_version, constants as c, utils
11
+ from squirrels.manifest import Manifest
12
+ from squirrels.connection_set import ConnectionSet
13
+ from squirrels.renderer import RendererIOWrapper, Renderer
14
+
15
+
16
+ class ApiServer:
17
+ def __init__(self, manifest: Manifest, conn_set: ConnectionSet, no_cache: bool, debug: bool) -> None:
18
+ """
19
+ Constructor for ApiServer
20
+
21
+ Parameters:
22
+ manifest (Manifest): Manifest object produced from squirrels.yaml
23
+ conn_set (ConnectionSet): Set of all connection pools defined in connections.py
24
+ no_cache (bool): Whether to disable caching
25
+ debug (bool): Set to True to show "hidden" parameters in the /parameters endpoint response
26
+ """
27
+ self.manifest = manifest
28
+ self.conn_set = conn_set
29
+ self.no_cache = no_cache
30
+ self.debug = debug
31
+
32
+ self.datasets = manifest.get_all_dataset_names()
33
+ self.renderers: Dict[str, Renderer] = {}
34
+ for dataset in self.datasets:
35
+ rendererIO = RendererIOWrapper(dataset, manifest, conn_set)
36
+ self.renderers[dataset] = rendererIO.renderer
37
+
38
+ def _get_parameters_helper(self, dataset: str, query_params: Set[Tuple[str, str]]) -> Dict:
39
+ if len(query_params) > 1:
40
+ raise utils.InvalidInputError("The /parameters endpoint takes at most 1 query parameter")
41
+ renderer = self.renderers[dataset]
42
+ parameters = renderer.apply_selections(dict(query_params), updates_only = True)
43
+ return parameters.to_dict(self.debug)
44
+
45
+ def _get_results_helper(self, dataset: str, query_params: Set[Tuple[str, str]]) -> Dict:
46
+ renderer = self.renderers[dataset]
47
+ _, _, _, _, df = renderer.load_results(dict(query_params))
48
+ return json.loads(df.to_json(orient='table', index=False))
49
+
50
+ def _apply_dataset_api_function(self, api_function, dataset: str, raw_query_params: QueryParams):
51
+ dataset = utils.normalize_name(dataset)
52
+ query_params = set()
53
+ for key, val in raw_query_params.items():
54
+ query_params.add((utils.normalize_name(key), val))
55
+ query_params = frozenset(query_params)
56
+ return api_function(dataset, query_params)
57
+
58
+ def run(self, uvicorn_args: List[str]) -> None:
59
+ """
60
+ Runs the API server with uvicorn for CLI "squirrels run"
61
+
62
+ Parameters:
63
+ uvicorn_args (List[str]): List of arguments to pass to uvicorn.run. Currently only supports "host" and "port"
64
+ """
65
+ app = FastAPI()
66
+
67
+ squirrels_version_path = f'/squirrels{major_version}'
68
+ config_base_path = utils.normalize_name_for_api(self.manifest.get_base_path())
69
+ base_path = squirrels_version_path + config_base_path
70
+
71
+ static_dir = utils.join_paths(os.path.dirname(__file__), 'package_data', 'static')
72
+ app.mount('/static', StaticFiles(directory=static_dir), name='static')
73
+
74
+ templates_dir = utils.join_paths(os.path.dirname(__file__), 'package_data', 'templates')
75
+ templates = Jinja2Templates(directory=templates_dir)
76
+
77
+ # Parameters API
78
+ parameters_path = base_path + '/{dataset}/parameters'
79
+
80
+ parameters_cache_size = self.manifest.get_setting(c.PARAMETERS_CACHE_SIZE_SETTING, 1024)
81
+ parameters_cache_ttl = self.manifest.get_setting(c.PARAMETERS_CACHE_TTL_SETTING, 24*60*60)
82
+
83
+ @ttl_cache(maxsize=parameters_cache_size, ttl=parameters_cache_ttl)
84
+ def get_parameters_cachable(*args):
85
+ return self._get_parameters_helper(*args)
86
+
87
+ @app.get(parameters_path, response_class=JSONResponse)
88
+ async def get_parameters(dataset: str, request: Request):
89
+ api_function = self._get_parameters_helper if self.no_cache else get_parameters_cachable
90
+ return self._apply_dataset_api_function(api_function, dataset, request.query_params)
91
+
92
+ # Results API
93
+ results_path = base_path + '/{dataset}'
94
+
95
+ results_cache_size = self.manifest.get_setting(c.RESULTS_CACHE_SIZE_SETTING, 128)
96
+ results_cache_ttl = self.manifest.get_setting(c.RESULTS_CACHE_TTL_SETTING, 60*60)
97
+
98
+ @ttl_cache(maxsize=results_cache_size, ttl=results_cache_ttl)
99
+ def get_results_cachable(*args):
100
+ return self._get_results_helper(*args)
101
+
102
+ @app.get(results_path, response_class=JSONResponse)
103
+ async def get_results(dataset: str, request: Request):
104
+ api_function = self._get_results_helper if self.no_cache else get_results_cachable
105
+ return self._apply_dataset_api_function(api_function, dataset, request.query_params)
106
+
107
+ # Catalog API
108
+ @app.get(base_path, response_class=JSONResponse)
109
+ async def get_catalog():
110
+ datasets_info = []
111
+ for dataset in self.datasets:
112
+ dataset_normalized = utils.normalize_name_for_api(dataset)
113
+ datasets_info.append({
114
+ 'dataset': dataset,
115
+ 'label': self.manifest.get_dataset_label(dataset),
116
+ 'parameters_path': parameters_path.format(dataset=dataset_normalized),
117
+ 'result_path': results_path.format(dataset=dataset_normalized)
118
+ })
119
+ return {'project_variables': self.manifest.get_proj_vars(), 'resource_paths': datasets_info}
120
+
121
+ # Squirrels UI
122
+ @app.get('/', response_class=HTMLResponse)
123
+ async def get_ui(request: Request):
124
+ return templates.TemplateResponse('index.html', {'request': request, 'base_path': base_path})
125
+
126
+ # Run API server
127
+ import uvicorn
128
+ uvicorn.run(app, host=uvicorn_args.host, port=uvicorn_args.port)
@@ -0,0 +1,65 @@
1
+ from typing import Dict, Union
2
+ from importlib.machinery import SourceFileLoader
3
+ from sqlalchemy import Engine, Pool
4
+ import sqlite3
5
+
6
+ from squirrels import manifest as mf, constants as c
7
+ from squirrels.timed_imports import pandas as pd
8
+ from squirrels.utils import ConfigurationError
9
+
10
+ ConnectionPool = Union[Engine, Pool]
11
+
12
+
13
+ class ConnectionSet:
14
+ def __init__(self, conn_pools: Dict[str, ConnectionPool]) -> None:
15
+ self._conn_pools = conn_pools
16
+
17
+ def get_connection_pool(self, conn_name: str) -> ConnectionPool:
18
+ try:
19
+ connection_pool = self._conn_pools[conn_name]
20
+ except KeyError as e:
21
+ raise ConfigurationError(f'Connection name "{conn_name}" was not configured') from e
22
+ return connection_pool
23
+
24
+ def get_dataframe_from_query(self, conn_name: str, query: str) -> pd.DataFrame:
25
+ connector = self.get_connection_pool(conn_name)
26
+ if isinstance(connector, Pool):
27
+ conn = connector.connect()
28
+ elif isinstance(connector, Engine):
29
+ conn = connector.raw_connection()
30
+ else:
31
+ raise TypeError(f'Type for connection name "{conn_name}" not supported')
32
+
33
+ try:
34
+ cur = conn.cursor()
35
+ cur.execute(query)
36
+ data = cur.fetchall()
37
+ columns = [x[0] for x in cur.description]
38
+ df = pd.DataFrame(data, columns=columns)
39
+ finally:
40
+ conn.close()
41
+
42
+ return df
43
+
44
+ def dispose(self) -> None:
45
+ for pool in self._conn_pools.values():
46
+ pool.dispose()
47
+
48
+
49
+ def from_file(manifest: mf.Manifest) -> ConnectionSet:
50
+ module = SourceFileLoader(c.CONNECTIONS_FILE, c.CONNECTIONS_FILE).load_module()
51
+ proj_vars = manifest.get_proj_vars()
52
+ try:
53
+ return module.main(proj_vars)
54
+ except Exception as e:
55
+ raise ConfigurationError(f'Error in the {c.CONNECTIONS_FILE} file') from e
56
+
57
+
58
+ def sqldf(query: str, df_by_db_views: Dict[str, pd.DataFrame]) -> pd.DataFrame:
59
+ conn = sqlite3.connect(":memory:")
60
+ try:
61
+ for db_view, df in df_by_db_views.items():
62
+ df.to_sql(db_view, conn, index=False)
63
+ return pd.read_sql(query, conn)
64
+ finally:
65
+ conn.close()
@@ -0,0 +1,53 @@
1
+ # Squirrels CLI commands
2
+ INIT_CMD = 'init'
3
+ LOAD_MODULES_CMD = 'load-modules'
4
+ GET_CREDS_CMD = 'get-all-credentials'
5
+ SET_CRED_CMD = 'set-credential'
6
+ DELETE_CRED_CMD = 'delete-credential'
7
+ TEST_CMD = 'test'
8
+ RUN_CMD = 'run'
9
+
10
+ # Manifest file keys
11
+ DB_CONNECTION_KEY = 'db_connection'
12
+ MODULES_KEY = 'modules'
13
+ DATASET_LABEL_KEY = 'label'
14
+ DATASETS_KEY = 'datasets'
15
+ HEADERS_KEY = 'headers'
16
+ DATABASE_VIEWS_KEY = 'database_views'
17
+ DB_VIEW_NAME_KEY = 'name'
18
+ DB_VIEW_FILE_KEY = 'file'
19
+ FINAL_VIEW_KEY = 'final_view'
20
+ BASE_PATH_KEY = 'base_path'
21
+ SETTINGS_KEY = 'settings'
22
+
23
+ # Database credentials keys
24
+ CREDENTIALS_KEY = 'credentials'
25
+ USERNAME_KEY = 'username'
26
+ PASSWORD_KEY = 'password'
27
+
28
+ # Folder/File names
29
+ MANIFEST_FILE = 'squirrels.yaml'
30
+ PROJ_VARS_FILE = 'project_vars.yaml'
31
+ CONNECTIONS_FILE = 'connections.py'
32
+ OUTPUTS_FOLDER = 'outputs'
33
+ MODULES_FOLDER = 'modules'
34
+ DATASETS_FOLDER = 'datasets'
35
+ PARAMETERS_FILE = 'parameters.py'
36
+ PARAMETERS_OUTPUT = 'parameters.json'
37
+ DATABASE_VIEW_SQL_FILE = 'database_view1.sql.j2'
38
+ DATABASE_VIEW_PY_FILE = 'database_view1.py'
39
+ FINAL_VIEW_SQL_NAME = 'final_view.sql.j2'
40
+ FINAL_VIEW_PY_NAME = 'final_view.py'
41
+ FINAL_VIEW_OUT_STEM = 'final_view'
42
+ CONTEXT_FILE = 'context.py'
43
+ SELECTIONS_CFG_FILE = 'selections.cfg'
44
+
45
+ # Dataset setting names
46
+ PARAMETERS_CACHE_SIZE_SETTING = 'parameters.cache.size'
47
+ PARAMETERS_CACHE_TTL_SETTING = 'parameters.cache.ttl'
48
+ RESULTS_CACHE_SIZE_SETTING = 'results.cache.size'
49
+ RESULTS_CACHE_TTL_SETTING = 'results.cache.ttl'
50
+
51
+ # Selection cfg sections
52
+ PARAMETERS_SECTION = 'parameters'
53
+ HEADERS_SECTION = 'headers'
@@ -0,0 +1,87 @@
1
+ from typing import Dict
2
+ from dataclasses import dataclass
3
+ from configparser import ConfigParser
4
+ import os, json
5
+
6
+ from squirrels.utils import ConfigurationError
7
+ from squirrels import constants as c, utils
8
+
9
+ _SQUIRRELS_CFG_PATH = utils.join_paths(os.path.expanduser('~'), '.squirrelscfg')
10
+
11
+
12
+ @dataclass
13
+ class Credential:
14
+ username: str
15
+ password: str
16
+
17
+ def __str__(self) -> str:
18
+ redacted_pass = '*'*len(self.password)
19
+ return f'username={self.username}, password={redacted_pass}'
20
+
21
+
22
+ class SquirrelsConfigParser(ConfigParser):
23
+ def _get_creds_section(self):
24
+ section_name: str = c.CREDENTIALS_KEY
25
+ if not self.has_section(section_name):
26
+ self.add_section(section_name)
27
+ return self[section_name]
28
+
29
+ def _json_str_to_credential(self, json_str: str) -> Credential:
30
+ cred_dict = json.loads(json_str)
31
+ return Credential(cred_dict[c.USERNAME_KEY], cred_dict[c.PASSWORD_KEY])
32
+
33
+ def get_credential(self, key: str) -> Credential:
34
+ section = self._get_creds_section()
35
+ try:
36
+ value = section[key]
37
+ except KeyError as e:
38
+ raise ConfigurationError(f'Credential key "{key}" has not been set. To set it, use $ squirrels set-credential {key}')
39
+ return self._json_str_to_credential(value)
40
+
41
+ def get_all_credentials(self) -> Dict[str, Credential]:
42
+ section = self._get_creds_section()
43
+ result = {}
44
+ for key, value in section.items():
45
+ result[key] = self._json_str_to_credential(value)
46
+ return result
47
+
48
+ def set_credential(self, key: str, credential: Credential) -> ConfigParser:
49
+ section = self._get_creds_section()
50
+ section[key] = json.dumps(credential.__dict__)
51
+ return self
52
+
53
+ def delete_credential(self, key: str) -> ConfigParser:
54
+ section = self._get_creds_section()
55
+ section.pop(key)
56
+ return self
57
+
58
+
59
+ class SquirrelsConfigIOWrapper:
60
+ def __init__(self) -> None:
61
+ self.config = SquirrelsConfigParser()
62
+ self.config.read(_SQUIRRELS_CFG_PATH)
63
+
64
+ def get_credential(self, key: str) -> Credential:
65
+ return self.config.get_credential(key)
66
+
67
+ def print_all_credentials(self) -> None:
68
+ credentials_dict = self.config.get_all_credentials()
69
+ for key, cred in credentials_dict.items():
70
+ print(f'{key}:', cred)
71
+
72
+ def _write_config(self) -> None:
73
+ with open(_SQUIRRELS_CFG_PATH, 'w') as f:
74
+ self.config.write(f)
75
+
76
+ def set_credential(self, key: str, user: str, pw: str) -> None:
77
+ credential = Credential(user, pw)
78
+ self.config.set_credential(key, credential)
79
+ print(f'Credential key "{key}" set to: {credential}')
80
+ self._write_config()
81
+
82
+ def delete_credential(self, key: str) -> None:
83
+ self.config.delete_credential(key)
84
+ print(f'Credential key "{key}" has been deleted')
85
+ self._write_config()
86
+
87
+ squirrels_config_io = SquirrelsConfigIOWrapper()