kiba-core 0.5.3.dev20__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kiba_core-0.5.3.dev20/.github/pull_request_template.md +19 -0
- kiba_core-0.5.3.dev20/.github/workflows/deploy.yml +20 -0
- kiba_core-0.5.3.dev20/.github/workflows/pull-request.yml +58 -0
- kiba_core-0.5.3.dev20/.github/workflows/release.yml +39 -0
- kiba_core-0.5.3.dev20/.gitignore +67 -0
- kiba_core-0.5.3.dev20/CHANGELOG.md +206 -0
- kiba_core-0.5.3.dev20/Dockerfile +13 -0
- kiba_core-0.5.3.dev20/PKG-INFO +40 -0
- kiba_core-0.5.3.dev20/README.md +1 -0
- kiba_core-0.5.3.dev20/core/__init__.py +0 -0
- kiba_core-0.5.3.dev20/core/api/__init__.py +0 -0
- kiba_core-0.5.3.dev20/core/api/api_request.py +15 -0
- kiba_core-0.5.3.dev20/core/api/api_respnse.py +9 -0
- kiba_core-0.5.3.dev20/core/api/authorizer.py +45 -0
- kiba_core-0.5.3.dev20/core/api/default_routes.py +14 -0
- kiba_core-0.5.3.dev20/core/api/health.py +11 -0
- kiba_core-0.5.3.dev20/core/api/json_route.py +57 -0
- kiba_core-0.5.3.dev20/core/api/middleware/__init__.py +0 -0
- kiba_core-0.5.3.dev20/core/api/middleware/database_connection_middleware.py +19 -0
- kiba_core-0.5.3.dev20/core/api/middleware/exception_handling_middleware.py +33 -0
- kiba_core-0.5.3.dev20/core/api/middleware/logging_middleware.py +32 -0
- kiba_core-0.5.3.dev20/core/api/middleware/server_headers_middleware.py +23 -0
- kiba_core-0.5.3.dev20/core/aws_requester.py +91 -0
- kiba_core-0.5.3.dev20/core/caching/__init__.py +0 -0
- kiba_core-0.5.3.dev20/core/caching/cache.py +24 -0
- kiba_core-0.5.3.dev20/core/caching/dict_cache.py +47 -0
- kiba_core-0.5.3.dev20/core/caching/file_cache.py +60 -0
- kiba_core-0.5.3.dev20/core/exceptions.py +260 -0
- kiba_core-0.5.3.dev20/core/http/__init__.py +2 -0
- kiba_core-0.5.3.dev20/core/http/basic_authentication.py +25 -0
- kiba_core-0.5.3.dev20/core/http/jwt.py +129 -0
- kiba_core-0.5.3.dev20/core/http/rest_method.py +10 -0
- kiba_core-0.5.3.dev20/core/logging.py +236 -0
- kiba_core-0.5.3.dev20/core/notifications/__init__.py +0 -0
- kiba_core-0.5.3.dev20/core/notifications/discord_client.py +14 -0
- kiba_core-0.5.3.dev20/core/notifications/notification_client.py +10 -0
- kiba_core-0.5.3.dev20/core/notifications/slack_client.py +24 -0
- kiba_core-0.5.3.dev20/core/py.typed +1 -0
- kiba_core-0.5.3.dev20/core/queues/__init__.py +0 -0
- kiba_core-0.5.3.dev20/core/queues/aqs.py +81 -0
- kiba_core-0.5.3.dev20/core/queues/message_queue.py +39 -0
- kiba_core-0.5.3.dev20/core/queues/message_queue_processor.py +95 -0
- kiba_core-0.5.3.dev20/core/queues/model.py +41 -0
- kiba_core-0.5.3.dev20/core/queues/sqs.py +96 -0
- kiba_core-0.5.3.dev20/core/requester/__init__.py +1 -0
- kiba_core-0.5.3.dev20/core/requester/requester.py +158 -0
- kiba_core-0.5.3.dev20/core/s3_manager.py +240 -0
- kiba_core-0.5.3.dev20/core/service_client.py +17 -0
- kiba_core-0.5.3.dev20/core/store/__init__.py +0 -0
- kiba_core-0.5.3.dev20/core/store/database.py +82 -0
- kiba_core-0.5.3.dev20/core/store/retriever.py +205 -0
- kiba_core-0.5.3.dev20/core/store/saver.py +68 -0
- kiba_core-0.5.3.dev20/core/util/__init__.py +0 -0
- kiba_core-0.5.3.dev20/core/util/chain_util.py +14 -0
- kiba_core-0.5.3.dev20/core/util/date_util.py +107 -0
- kiba_core-0.5.3.dev20/core/util/dict_util.py +10 -0
- kiba_core-0.5.3.dev20/core/util/file_util.py +108 -0
- kiba_core-0.5.3.dev20/core/util/hashing_util.py +6 -0
- kiba_core-0.5.3.dev20/core/util/http_util.py +1 -0
- kiba_core-0.5.3.dev20/core/util/json_util.py +28 -0
- kiba_core-0.5.3.dev20/core/util/list_util.py +10 -0
- kiba_core-0.5.3.dev20/core/util/string_util.py +9 -0
- kiba_core-0.5.3.dev20/core/util/typing_util.py +7 -0
- kiba_core-0.5.3.dev20/core/util/url_util.py +17 -0
- kiba_core-0.5.3.dev20/core/util/value_holder.py +59 -0
- kiba_core-0.5.3.dev20/core/web3/__init__.py +0 -0
- kiba_core-0.5.3.dev20/core/web3/eth_client.py +477 -0
- kiba_core-0.5.3.dev20/kiba_core.egg-info/PKG-INFO +40 -0
- kiba_core-0.5.3.dev20/kiba_core.egg-info/SOURCES.txt +84 -0
- kiba_core-0.5.3.dev20/kiba_core.egg-info/dependency_links.txt +1 -0
- kiba_core-0.5.3.dev20/kiba_core.egg-info/requires.txt +42 -0
- kiba_core-0.5.3.dev20/kiba_core.egg-info/top_level.txt +1 -0
- kiba_core-0.5.3.dev20/makefile +57 -0
- kiba_core-0.5.3.dev20/pyproject.toml +75 -0
- kiba_core-0.5.3.dev20/setup.cfg +4 -0
- kiba_core-0.5.3.dev20/tests/__init__.py +1 -0
- kiba_core-0.5.3.dev20/tests/caching/test_dict_cache.py +67 -0
- kiba_core-0.5.3.dev20/tests/caching/test_file_cache.py +99 -0
- kiba_core-0.5.3.dev20/tests/util/__init__.py +1 -0
- kiba_core-0.5.3.dev20/tests/util/test_date_util.py +321 -0
- kiba_core-0.5.3.dev20/tests/util/test_dict_util.py +39 -0
- kiba_core-0.5.3.dev20/tests/util/test_json_util.py +149 -0
- kiba_core-0.5.3.dev20/tests/util/test_list_util.py +39 -0
- kiba_core-0.5.3.dev20/tests/util/test_string_util.py +33 -0
- kiba_core-0.5.3.dev20/tests/util/test_url_util.py +76 -0
- kiba_core-0.5.3.dev20/uv.lock +2262 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
<!--- Title format: [Feature | Fix | Task#]: <summary of your changes> -->
|
|
2
|
+
|
|
3
|
+
## Description
|
|
4
|
+
|
|
5
|
+
<!--- Describe your changes -->
|
|
6
|
+
<!--- Why is this change required? What problem does it solve? -->
|
|
7
|
+
<!--- If it fixes an open issue, please link to the issue here. -->
|
|
8
|
+
|
|
9
|
+
## Screenshots:
|
|
10
|
+
|
|
11
|
+
<!--- If not relevant delete the sub-heading above -->
|
|
12
|
+
|
|
13
|
+
## Checklist:
|
|
14
|
+
|
|
15
|
+
<!--- Please put an `x` in all the following boxes that apply to these changes. -->
|
|
16
|
+
|
|
17
|
+
- [ ] I have updated the CHANGELOG with a summary of my changes
|
|
18
|
+
<!-- - [ ] I have updated the documentation accordingly -->
|
|
19
|
+
<!-- - [ ] My changes have tests around them -->
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
name: Deploy
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches: [main]
|
|
5
|
+
jobs:
|
|
6
|
+
deploy-package-next:
|
|
7
|
+
runs-on: ubuntu-latest
|
|
8
|
+
env:
|
|
9
|
+
DOCKER_IMAGE: package
|
|
10
|
+
PYPI_PUBLISH_TOKEN: ${{ secrets.PYPI_PUBLISH_TOKEN }}
|
|
11
|
+
steps:
|
|
12
|
+
- name: Check out the repo
|
|
13
|
+
uses: actions/checkout@v2
|
|
14
|
+
with:
|
|
15
|
+
# NOTE(krishan711): need full history to calculate difference
|
|
16
|
+
fetch-depth: 0
|
|
17
|
+
- name: Build docker image
|
|
18
|
+
run: docker build --tag $DOCKER_IMAGE .
|
|
19
|
+
- name: Publish next package to pypi
|
|
20
|
+
run: docker run -e UV_PUBLISH_TOKEN=$PYPI_PUBLISH_TOKEN $DOCKER_IMAGE /bin/bash -c " make publish-dev"
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
name: Build
|
|
2
|
+
on:
|
|
3
|
+
pull_request:
|
|
4
|
+
branches: ["*"]
|
|
5
|
+
concurrency:
|
|
6
|
+
group: ${{ github.workflow }}-${{ github.ref }}-pull-request
|
|
7
|
+
cancel-in-progress: true
|
|
8
|
+
jobs:
|
|
9
|
+
core-build:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
env:
|
|
12
|
+
DOCKER_IMAGE: core
|
|
13
|
+
steps:
|
|
14
|
+
- name: Check out the repo
|
|
15
|
+
uses: actions/checkout@v2
|
|
16
|
+
- name: Set up Docker Buildx
|
|
17
|
+
uses: docker/setup-buildx-action@v1
|
|
18
|
+
- name: Build docker image
|
|
19
|
+
uses: docker/build-push-action@v2
|
|
20
|
+
with:
|
|
21
|
+
context: .
|
|
22
|
+
load: true
|
|
23
|
+
cache-from: type=gha,scope=core
|
|
24
|
+
cache-to: type=gha,mode=max,scope=core
|
|
25
|
+
tags: ${{ env.DOCKER_IMAGE }}
|
|
26
|
+
- name: Run lint-check
|
|
27
|
+
run: |
|
|
28
|
+
docker run --name lint-check $DOCKER_IMAGE make lint-check-ci
|
|
29
|
+
docker cp lint-check:/app/lint-check-results.json ./lint-check-results.json
|
|
30
|
+
- name: Upload lint-check results
|
|
31
|
+
uses: kibalabs/github-action-create-annotations@main
|
|
32
|
+
with:
|
|
33
|
+
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
34
|
+
json-file-path: ./lint-check-results.json
|
|
35
|
+
check-name: core-lint-check
|
|
36
|
+
fail-on-error: false
|
|
37
|
+
- name: Run type-check
|
|
38
|
+
run: |
|
|
39
|
+
docker run --name type-check $DOCKER_IMAGE make type-check-ci
|
|
40
|
+
docker cp type-check:/app/type-check-results.json ./type-check-results.json
|
|
41
|
+
- name: Upload type-check results
|
|
42
|
+
uses: kibalabs/github-action-create-annotations@main
|
|
43
|
+
with:
|
|
44
|
+
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
45
|
+
json-file-path: ./type-check-results.json
|
|
46
|
+
check-name: core-type-check
|
|
47
|
+
fail-on-error: false
|
|
48
|
+
- name: Run tests
|
|
49
|
+
run: |
|
|
50
|
+
docker run --name tests $DOCKER_IMAGE make test-ci
|
|
51
|
+
# docker cp tests:/app/test-results.json ./test-results.json
|
|
52
|
+
# - name: Upload test results
|
|
53
|
+
# uses: kibalabs/github-action-create-annotations@main
|
|
54
|
+
# with:
|
|
55
|
+
# github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
56
|
+
# json-file-path: ./test-results.json
|
|
57
|
+
# check-name: core-tests
|
|
58
|
+
# fail-on-error: true
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
tags: ["v*"]
|
|
5
|
+
jobs:
|
|
6
|
+
deploy-package:
|
|
7
|
+
runs-on: ubuntu-latest
|
|
8
|
+
env:
|
|
9
|
+
DOCKER_IMAGE: build
|
|
10
|
+
PYPI_PUBLISH_TOKEN: ${{ secrets.PYPI_PUBLISH_TOKEN }}
|
|
11
|
+
steps:
|
|
12
|
+
- name: Check out the repo
|
|
13
|
+
uses: actions/checkout@v2
|
|
14
|
+
- name: Check the package version is the same as the tag
|
|
15
|
+
run: |
|
|
16
|
+
version=$(python3 setup.py --version)
|
|
17
|
+
if [ "${{ github.ref }}" != "refs/tags/v${version}" ]; then
|
|
18
|
+
echo "version in setup.py (v${version}) is different to the given tag (${{ github.ref }})"
|
|
19
|
+
exit 1;
|
|
20
|
+
fi
|
|
21
|
+
- name: Build docker image
|
|
22
|
+
run: docker build --tag $DOCKER_IMAGE .
|
|
23
|
+
- name: Publish package to pypi
|
|
24
|
+
run: docker run -e UV_PUBLISH_TOKEN=$PYPI_PUBLISH_TOKEN $DOCKER_IMAGE /bin/bash -c " make publish"
|
|
25
|
+
create-release:
|
|
26
|
+
needs: deploy-package
|
|
27
|
+
runs-on: ubuntu-latest
|
|
28
|
+
env:
|
|
29
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
30
|
+
steps:
|
|
31
|
+
- name: Check out the repo
|
|
32
|
+
uses: actions/checkout@v2
|
|
33
|
+
- name: Create Release
|
|
34
|
+
uses: actions/create-release@v1
|
|
35
|
+
with:
|
|
36
|
+
tag_name: ${{ github.ref }}
|
|
37
|
+
release_name: Release ${{ github.ref }}
|
|
38
|
+
draft: false
|
|
39
|
+
prerelease: false
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Mac
|
|
2
|
+
*.DS_Store
|
|
3
|
+
.Spotlight-V100
|
|
4
|
+
.Trashes
|
|
5
|
+
Thumbs.db
|
|
6
|
+
Desktop.ini
|
|
7
|
+
|
|
8
|
+
# IDEs
|
|
9
|
+
.idea/
|
|
10
|
+
.vscode/
|
|
11
|
+
*.swp
|
|
12
|
+
*.log
|
|
13
|
+
|
|
14
|
+
# Protobufs
|
|
15
|
+
*_pb2.*
|
|
16
|
+
|
|
17
|
+
# Testing
|
|
18
|
+
coverage
|
|
19
|
+
.coverage
|
|
20
|
+
tests.xml
|
|
21
|
+
lint.xml
|
|
22
|
+
.noseids
|
|
23
|
+
nosetests.xml
|
|
24
|
+
eslint.xml
|
|
25
|
+
jest.xml
|
|
26
|
+
local_testing*
|
|
27
|
+
|
|
28
|
+
# Generated
|
|
29
|
+
generated/
|
|
30
|
+
dist/
|
|
31
|
+
build/
|
|
32
|
+
tmp/
|
|
33
|
+
output/
|
|
34
|
+
|
|
35
|
+
# Python
|
|
36
|
+
__pycache__/
|
|
37
|
+
eggs/
|
|
38
|
+
dist/
|
|
39
|
+
build/
|
|
40
|
+
*.egg-info/
|
|
41
|
+
.env*
|
|
42
|
+
.mypy_cache
|
|
43
|
+
|
|
44
|
+
# JS
|
|
45
|
+
node_modules/
|
|
46
|
+
npm-debug.log*
|
|
47
|
+
yarn-debug.log*
|
|
48
|
+
yarn-error.log*
|
|
49
|
+
*.d.ts
|
|
50
|
+
.next
|
|
51
|
+
|
|
52
|
+
# Terraform
|
|
53
|
+
**/.terraform/*
|
|
54
|
+
*.tfstate
|
|
55
|
+
*.tfstate.*
|
|
56
|
+
crash.log
|
|
57
|
+
*.tfvars
|
|
58
|
+
*override.tf
|
|
59
|
+
override.tf.json
|
|
60
|
+
*_override.tf
|
|
61
|
+
*_override.tf.json
|
|
62
|
+
.terraformrc
|
|
63
|
+
terraform.rc
|
|
64
|
+
|
|
65
|
+
# Ethereum dev
|
|
66
|
+
artifacts/
|
|
67
|
+
cache/
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) with some additions:
|
|
6
|
+
- For all changes include one of [PATCH | MINOR | MAJOR] with the scope of the change being made.
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- [MAJOR] Update web3 to 7.8.0
|
|
12
|
+
- [MINOR] Added `call_function_by_name`, `send_transaction_by_name` and `wait_for_transaction_receipt` to `RestEthClient`
|
|
13
|
+
- [MINOR] Updated `RestEthClient` to use `maxPriorityFeePerGas` and `maxFeePerGas` instead of `gasPrice`
|
|
14
|
+
- [MAJOR] Make all datetimes timezone aware
|
|
15
|
+
- [MINOR] Added `Cache`, `FileCache` and `DictCache`
|
|
16
|
+
- [MINOR] Added `url_util.encode_url`, `url_util.encode_url_part`, `url_util.decode_url` and `url_util.decode_url_part`
|
|
17
|
+
- [MINOR] Added `http_util.CACHABLE_STATUS_CODES`
|
|
18
|
+
- [PATCH] Moved `Requester` into a folder with same path as original
|
|
19
|
+
- [MINOR] Added `date_util.datetime_from_date`
|
|
20
|
+
- [MINOR] Add `json_util.dumps` and `json_util.loads` to use faster json serialization
|
|
21
|
+
|
|
22
|
+
### Changed
|
|
23
|
+
|
|
24
|
+
### Removed
|
|
25
|
+
|
|
26
|
+
## [0.5.2] - 2024-08-22
|
|
27
|
+
|
|
28
|
+
### Added
|
|
29
|
+
- [MAJOR] Updated `MessageQueueProcessor` to use a list of `NotificationClients` to send server messages
|
|
30
|
+
- [MAJOR] Updated `SlackClient` to post `messageText` instead of text
|
|
31
|
+
- [MINOR] Added `NotificationClient.py` abstract class to help send data to multiple platforms
|
|
32
|
+
- [MINOR] Added `DiscordClient.py` to send data discord server
|
|
33
|
+
- [MINOR] Added `calculate_diff_days` and `calculate_diff_years` to `date_util`
|
|
34
|
+
- [MINOR] Added `datetime_to_utc` to `date_util`
|
|
35
|
+
- [MINOR] Added `init_external_loggers` to `logging`
|
|
36
|
+
- [MINOR] Added `end_of_day` to `date_util`
|
|
37
|
+
- [MINOR] Added `aiosqlite engine` to `database`
|
|
38
|
+
- [MINOR] Added core-api extra to server APIs with starlette only, no fastapi:
|
|
39
|
+
- Added `core.http.jwt.Jwt`
|
|
40
|
+
- Added `core.http.rest_method.RestMethod`
|
|
41
|
+
- Added `core.api.api_request.KibaApiRequest`
|
|
42
|
+
- Added `core.api.api_response.KibaJSONResponse`
|
|
43
|
+
- Added `core.api.authorizer.authorize_bearer_jwt`
|
|
44
|
+
- Added `core.api.json_route.json_route`
|
|
45
|
+
|
|
46
|
+
### Changed
|
|
47
|
+
- [MAJOR] Moved SlackClient to `core.notifications`
|
|
48
|
+
- [MAJOR] Moved DiscordClient to `core.notifications`
|
|
49
|
+
- [MAJOR] Moved NotificationClient to `core.notifications`
|
|
50
|
+
- [MINOR] Updated `Requester` to send data correctly for PUT requests
|
|
51
|
+
- [MINOR] Updated `Requester` to send data correctly for PATCH requests
|
|
52
|
+
|
|
53
|
+
### Removed
|
|
54
|
+
|
|
55
|
+
## [0.5.1] - 2023-02-16
|
|
56
|
+
|
|
57
|
+
### Added
|
|
58
|
+
- [MINOR] Added `call_contract_function` to `EthClientInterface`
|
|
59
|
+
- [MINOR] Added `_insert_record`, `_update_records`, and `_delete_records` to `Saver`
|
|
60
|
+
- [MINOR] Added `core.queues.aqs.AqsMessage` and `core.queues.aqs.AqsMessageQueue` to work with Azure Queues
|
|
61
|
+
- [MINOR] Added `BooleanFieldFilter` to `Retriever`
|
|
62
|
+
|
|
63
|
+
### Changed
|
|
64
|
+
- [MAJOR] Moved `SqsMessage` and `SqsMessageQueue` to `core.queues.sqs`
|
|
65
|
+
|
|
66
|
+
## [0.5.0] - 2022-11-08
|
|
67
|
+
|
|
68
|
+
### Added
|
|
69
|
+
- [MINOR] Added `requestId` to `Message`
|
|
70
|
+
|
|
71
|
+
### Changed
|
|
72
|
+
- [MINOR] Updated `Saver` to throw `SavingException` when saving fails
|
|
73
|
+
- [MINOR] Added `MessageNeedsReprocessingException` to `MessageQueueProcessor` to support rescheduling a message
|
|
74
|
+
- [MINOR] Use `www-authenticate` as message (if available) in 401 responses
|
|
75
|
+
- [MAJOR] Upgraded to sqlalchemy2
|
|
76
|
+
|
|
77
|
+
## [0.4.0] - 2022-07-19
|
|
78
|
+
|
|
79
|
+
### Added
|
|
80
|
+
- [MINOR] Added `head_file` to `S3Manager`
|
|
81
|
+
- [MAJOR] Added exception lists to `exceptions`
|
|
82
|
+
- [MINOR] Added `generate_clock_hour_intervals`, `generate_hourly_intervals`, `generate_datetime_intervals`, `generate_dates_in_range` to `date_util`
|
|
83
|
+
|
|
84
|
+
### Changed
|
|
85
|
+
- [MINOR] Updated `DatabaseConnectionMiddleware` to only create a transaction for non-GET requests
|
|
86
|
+
- [MAJOR] Updated `Requester` to throw specific exception types
|
|
87
|
+
- [MINOR] Updated `Requester.post_form` and `Requester.make_request` to accept `formFiles` parameter
|
|
88
|
+
- [MINOR] Added `shouldAddCacheHeader` to relevant `RedirectException`s
|
|
89
|
+
|
|
90
|
+
## [0.3.0] - 2022-04-23
|
|
91
|
+
|
|
92
|
+
### Added
|
|
93
|
+
- [MINOR] Added `send_messages` to `SqsMessageQueue` for sending multiple messages in one request
|
|
94
|
+
- [MINOR] Added `Database` to make facilitate easy migration from databases package
|
|
95
|
+
- [MINOR] Added `get_block_uncle_count` to `EthClient`
|
|
96
|
+
- [MINOR] Added `shouldFollowRedirects` to `Requester`
|
|
97
|
+
- [MINOR] Added `DatabaseConnectionMiddleware` to manage api database connections
|
|
98
|
+
- [MAJOR] Added `logging` module for custom kiba log formatting
|
|
99
|
+
- [MAJOR] Added `ExceptionHandlingMiddleware` to replace KibaRoute
|
|
100
|
+
- [MAJOR] Added `ServerHeadersMiddleware` to replace KibaRoute
|
|
101
|
+
- [MAJOR] Added `LoggingMiddleware` to replace KibaRoute
|
|
102
|
+
- [MAJOR] Added `requestIdHolder` to `MessageQueueProcessor`
|
|
103
|
+
|
|
104
|
+
### Changed
|
|
105
|
+
- [MAJOR] Replaced use of databases package in `Saver` and `Retriever`
|
|
106
|
+
- [MINOR] Update `Requester` to follow redirects by default
|
|
107
|
+
- [MAJOR] Replaced use of `boto3` with `aiobotocore` for S3Manager
|
|
108
|
+
- [MAJOR] Replaced use of `boto3` with `aiobotocore` for SqsMessageQueue
|
|
109
|
+
|
|
110
|
+
### Removed
|
|
111
|
+
- [MAJOR] Removed `KibaRouter` and `KibaRoute` - Use middlewares instead
|
|
112
|
+
|
|
113
|
+
## [0.2.10] - 2022-01-24
|
|
114
|
+
|
|
115
|
+
### Changed
|
|
116
|
+
- [MINOR] Updated `RestEthClient` to throw `BadRequestException` for malformed responses
|
|
117
|
+
|
|
118
|
+
## [0.2.9] - 2022-01-20
|
|
119
|
+
|
|
120
|
+
### Added
|
|
121
|
+
- [MINOR] Added `shouldHydrateTransactions` to `EthClient.get_block`
|
|
122
|
+
|
|
123
|
+
## [0.2.8] - 2022-01-17
|
|
124
|
+
|
|
125
|
+
### Added
|
|
126
|
+
- [MINOR] Added `postDate` to `Message`
|
|
127
|
+
|
|
128
|
+
### Changed
|
|
129
|
+
- [MINOR] Set `Message.postDate` in `SqsMessageQueue` when posting to queue
|
|
130
|
+
|
|
131
|
+
## [0.2.7] - 2022-01-15
|
|
132
|
+
|
|
133
|
+
### Added
|
|
134
|
+
- [MINOR] Added parallel processing to `MessageQueueProcessor.execute_batch`
|
|
135
|
+
|
|
136
|
+
## [0.2.6] - 2022-01-06
|
|
137
|
+
|
|
138
|
+
### Changed
|
|
139
|
+
- [PATCH] Fixed `MessageQueueProcessor` to allow limitless processing
|
|
140
|
+
|
|
141
|
+
## [0.2.5] - 2022-01-06
|
|
142
|
+
|
|
143
|
+
### Added
|
|
144
|
+
- [MINOR] Added batch processing to `MessageQueueProcessor`
|
|
145
|
+
- [MINOR] Added one-off processing to `MessageQueueProcessor`
|
|
146
|
+
|
|
147
|
+
## [0.2.4] - 2021-11-19
|
|
148
|
+
|
|
149
|
+
### Added
|
|
150
|
+
- [MINOR] Added `BURN_ADDRESS` to `chain_util`
|
|
151
|
+
- [MINOR] Added `RedirectException` and subclasses
|
|
152
|
+
|
|
153
|
+
### Changed
|
|
154
|
+
- [MINOR] Handle converting `RedirectException` to correct response in `KibaRoute`
|
|
155
|
+
|
|
156
|
+
## [0.2.3] - 2021-10-11
|
|
157
|
+
|
|
158
|
+
### Added
|
|
159
|
+
- [MINOR] Added `chain_util` with `normalize_address`
|
|
160
|
+
|
|
161
|
+
### Changed
|
|
162
|
+
- [MINOR] Updated to databases v0.5.1
|
|
163
|
+
- [MINOR] Updated `MessageQueueProcessor` to make `slackClient` optional
|
|
164
|
+
|
|
165
|
+
## [0.2.2] - 2021-08-21
|
|
166
|
+
|
|
167
|
+
### Changed
|
|
168
|
+
- [MINOR] Reverted `Message.COMMAND` change from 0.2.1
|
|
169
|
+
- [MINOR] Added `Message.get_command()` to prevent private access lint error
|
|
170
|
+
|
|
171
|
+
## [0.2.1] - 2021-08-18
|
|
172
|
+
|
|
173
|
+
### Changed
|
|
174
|
+
- [MINOR] Added `Message.COMMAND` and deprecation note on `Message._COMMAND` to suit linters
|
|
175
|
+
|
|
176
|
+
## [0.2.0] - 2021-08-08
|
|
177
|
+
|
|
178
|
+
### Added
|
|
179
|
+
- [MINOR] Added `post_form` to `Requester`
|
|
180
|
+
- [MINOR] Added `BasicAuthentication`
|
|
181
|
+
|
|
182
|
+
### Changed
|
|
183
|
+
- [MAJOR] Updated `Requester.post_json` to not accept data
|
|
184
|
+
- [MAJOR] Updated `file_util.create_directory` to be async
|
|
185
|
+
|
|
186
|
+
## [0.1.3] - 2021-07-14
|
|
187
|
+
|
|
188
|
+
### Added
|
|
189
|
+
- [MINOR] Added `create_directory` to `file_util`
|
|
190
|
+
|
|
191
|
+
### Changed
|
|
192
|
+
- [MINOR] Fix `Requester` to use dataDict correctly for GET params
|
|
193
|
+
|
|
194
|
+
## [0.1.2] - 2021-06-09
|
|
195
|
+
|
|
196
|
+
### Added
|
|
197
|
+
- [MINOR] Added `EthClient`
|
|
198
|
+
|
|
199
|
+
## [0.1.1] - 2021-05-20
|
|
200
|
+
|
|
201
|
+
### Added
|
|
202
|
+
- [MINOR] Added `IntegerFieldFilter` and `FloatFieldFilter` to `Saver`
|
|
203
|
+
|
|
204
|
+
## [0.1.0] - 2021-05-18
|
|
205
|
+
|
|
206
|
+
Initial Commit
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: kiba-core
|
|
3
|
+
Version: 0.5.3.dev20
|
|
4
|
+
Summary: Kiba Labs' python utilities
|
|
5
|
+
Project-URL: repository, https://github.com/kibalabs/core-py
|
|
6
|
+
Requires-Python: ~=3.12
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: aiofiles>=24.1.0
|
|
9
|
+
Requires-Dist: pydantic>=2.10.6
|
|
10
|
+
Requires-Dist: orjson>=3.10.15
|
|
11
|
+
Requires-Dist: kiba-build
|
|
12
|
+
Provides-Extra: api
|
|
13
|
+
Requires-Dist: fastapi>=0.115.8; extra == "api"
|
|
14
|
+
Requires-Dist: uvicorn[standard]>=0.34.0; extra == "api"
|
|
15
|
+
Provides-Extra: core-api
|
|
16
|
+
Requires-Dist: starlette>=0.45.3; extra == "core-api"
|
|
17
|
+
Requires-Dist: uvicorn[standard]>=0.34.0; extra == "core-api"
|
|
18
|
+
Provides-Extra: storage
|
|
19
|
+
Requires-Dist: aiobotocore>=2.19.0; extra == "storage"
|
|
20
|
+
Provides-Extra: queues
|
|
21
|
+
Requires-Dist: aiobotocore>=2.19.0; extra == "queues"
|
|
22
|
+
Provides-Extra: queue-sqs
|
|
23
|
+
Requires-Dist: aiobotocore>=2.19.0; extra == "queue-sqs"
|
|
24
|
+
Provides-Extra: queue-aqs
|
|
25
|
+
Requires-Dist: azure-storage-queue[aio]>=12.12.0; extra == "queue-aqs"
|
|
26
|
+
Provides-Extra: database-psql
|
|
27
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0.38; extra == "database-psql"
|
|
28
|
+
Requires-Dist: asyncpg>=0.30.0; extra == "database-psql"
|
|
29
|
+
Provides-Extra: database-sqlite
|
|
30
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0.38; extra == "database-sqlite"
|
|
31
|
+
Requires-Dist: aiosqlite>=0.21.0; extra == "database-sqlite"
|
|
32
|
+
Provides-Extra: requester
|
|
33
|
+
Requires-Dist: httpx>=0.28.1; extra == "requester"
|
|
34
|
+
Provides-Extra: web3
|
|
35
|
+
Requires-Dist: web3>=7.8.0; extra == "web3"
|
|
36
|
+
Provides-Extra: types
|
|
37
|
+
Requires-Dist: types-aiobotocore[essential]>=2.19.0; extra == "types"
|
|
38
|
+
Requires-Dist: types-aiofiles>=24.1.0.20241221; extra == "types"
|
|
39
|
+
|
|
40
|
+
# Core-Py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Core-Py
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import typing
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel
|
|
4
|
+
from starlette.requests import Request
|
|
5
|
+
|
|
6
|
+
from core.http.basic_authentication import BasicAuthentication
|
|
7
|
+
from core.http.jwt import Jwt
|
|
8
|
+
|
|
9
|
+
ApiRequestDataType = typing.TypeVar('ApiRequestDataType', bound=BaseModel)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class KibaApiRequest(Request, typing.Generic[ApiRequestDataType]):
|
|
13
|
+
data: ApiRequestDataType
|
|
14
|
+
authJwt: Jwt | None = None
|
|
15
|
+
authBasic: BasicAuthentication | None = None
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
import typing
|
|
3
|
+
from typing import ParamSpec
|
|
4
|
+
|
|
5
|
+
from mypy_extensions import Arg
|
|
6
|
+
from pydantic import BaseModel
|
|
7
|
+
|
|
8
|
+
from core.api.api_request import KibaApiRequest
|
|
9
|
+
from core.exceptions import ForbiddenException
|
|
10
|
+
from core.http.jwt import Jwt
|
|
11
|
+
|
|
12
|
+
_P = ParamSpec('_P')
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Authorizer:
|
|
16
|
+
async def validate_jwt(self, jwtString: str) -> Jwt:
|
|
17
|
+
raise NotImplementedError
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
ApiRequest = typing.TypeVar('ApiRequest', bound=BaseModel)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def authorize_bearer_jwt( # type: ignore[explicit-any]
|
|
24
|
+
authorizer: Authorizer,
|
|
25
|
+
) -> typing.Callable[[typing.Callable[[Arg(KibaApiRequest[ApiRequest], 'request')], typing.Awaitable[typing.Any]]], typing.Callable[_P, typing.Any]]:
|
|
26
|
+
def decorator(func: typing.Callable[[Arg(KibaApiRequest[ApiRequest], 'request')], typing.Awaitable[typing.Any]]) -> typing.Callable[_P, typing.Any]: # type: ignore[explicit-any]
|
|
27
|
+
@functools.wraps(func)
|
|
28
|
+
async def async_wrapper(request: KibaApiRequest[ApiRequest]) -> typing.Any: # type: ignore[explicit-any, misc]
|
|
29
|
+
authorization = request.headers.get('Authorization')
|
|
30
|
+
if not authorization:
|
|
31
|
+
raise ForbiddenException(message='AUTH_NOT_PROVIDED')
|
|
32
|
+
if not authorization.startswith('Bearer '):
|
|
33
|
+
raise ForbiddenException(message='AUTH_INVALID')
|
|
34
|
+
jwtString = authorization.replace('Bearer ', '')
|
|
35
|
+
try:
|
|
36
|
+
jwt = await authorizer.validate_jwt(jwtString=jwtString)
|
|
37
|
+
except BaseException: # noqa: BLE001
|
|
38
|
+
raise ForbiddenException(message='AUTH_INVALID')
|
|
39
|
+
request.authJwt = jwt
|
|
40
|
+
return await func(request=request)
|
|
41
|
+
|
|
42
|
+
# TODO(krishan711): figure out correct typing here
|
|
43
|
+
return async_wrapper # type: ignore[return-value]
|
|
44
|
+
|
|
45
|
+
return decorator
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from starlette.requests import Request
|
|
2
|
+
from starlette.responses import JSONResponse
|
|
3
|
+
from starlette.responses import Response
|
|
4
|
+
from starlette.routing import BaseRoute
|
|
5
|
+
from starlette.routing import Route
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def create_default_routes(name: str, version: str, environment: str) -> list[BaseRoute]:
|
|
9
|
+
async def root(request: Request) -> Response: # noqa: ARG001
|
|
10
|
+
return JSONResponse(content={'server': name, 'version': version, 'environment': environment})
|
|
11
|
+
|
|
12
|
+
return [
|
|
13
|
+
Route('/', methods=['GET'], endpoint=root),
|
|
14
|
+
]
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from fastapi import APIRouter
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def create_api(name: str, version: str, environment: str) -> APIRouter:
|
|
5
|
+
router = APIRouter()
|
|
6
|
+
|
|
7
|
+
@router.get('/')
|
|
8
|
+
async def root() -> dict[str, str]:
|
|
9
|
+
return {'server': name, 'version': version, 'environment': environment}
|
|
10
|
+
|
|
11
|
+
return router
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
import typing
|
|
3
|
+
from typing import ParamSpec
|
|
4
|
+
|
|
5
|
+
from mypy_extensions import Arg
|
|
6
|
+
from pydantic import BaseModel
|
|
7
|
+
from pydantic import ValidationError
|
|
8
|
+
|
|
9
|
+
from core.api.api_request import KibaApiRequest
|
|
10
|
+
from core.api.api_respnse import KibaJSONResponse
|
|
11
|
+
from core.exceptions import BadRequestException
|
|
12
|
+
from core.exceptions import InternalServerErrorException
|
|
13
|
+
from core.util import json_util
|
|
14
|
+
from core.util.typing_util import JsonObject
|
|
15
|
+
|
|
16
|
+
_P = ParamSpec('_P')
|
|
17
|
+
|
|
18
|
+
ApiRequest = typing.TypeVar('ApiRequest', bound=BaseModel)
|
|
19
|
+
ApiResponse = typing.TypeVar('ApiResponse', bound=BaseModel)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def json_route(
|
|
23
|
+
requestType: typing.Type[ApiRequest],
|
|
24
|
+
responseType: typing.Type[ApiResponse],
|
|
25
|
+
) -> typing.Callable[[typing.Callable[[Arg(KibaApiRequest[ApiRequest], 'request')], typing.Awaitable[ApiResponse]]], typing.Callable[_P, KibaJSONResponse]]:
|
|
26
|
+
def decorator(func: typing.Callable[[Arg(KibaApiRequest[ApiRequest], 'request')], typing.Awaitable[ApiResponse]]) -> typing.Callable[_P, KibaJSONResponse]:
|
|
27
|
+
@functools.wraps(func)
|
|
28
|
+
async def async_wrapper(*args: typing.Any) -> KibaJSONResponse: # type: ignore[explicit-any, misc]
|
|
29
|
+
receivedRequest = args[0]
|
|
30
|
+
pathParams = receivedRequest.path_params
|
|
31
|
+
# TODO(krishan711): move these to get request only
|
|
32
|
+
queryParams = receivedRequest.query_params
|
|
33
|
+
bodyBytes = await args[0].body()
|
|
34
|
+
if len(bodyBytes) == 0:
|
|
35
|
+
body: JsonObject = {}
|
|
36
|
+
else:
|
|
37
|
+
try:
|
|
38
|
+
body = typing.cast(JsonObject, json_util.loads(bodyBytes.decode()))
|
|
39
|
+
except json_util.JsonDecodeException as exception:
|
|
40
|
+
raise BadRequestException(f'Invalid JSON body: {exception}')
|
|
41
|
+
allParams = {**pathParams, **body, **queryParams}
|
|
42
|
+
try:
|
|
43
|
+
requestParams = requestType(**allParams)
|
|
44
|
+
except ValidationError as exception:
|
|
45
|
+
validationErrorMessage = ', '.join([f'{".".join([str(value) for value in error["loc"]])}: {error["msg"]}' for error in exception.errors()])
|
|
46
|
+
raise BadRequestException(f'Invalid request: {validationErrorMessage}')
|
|
47
|
+
kibaRequest: KibaApiRequest[ApiRequest] = KibaApiRequest(scope=receivedRequest.scope, receive=receivedRequest._receive, send=receivedRequest._send) # noqa: SLF001
|
|
48
|
+
kibaRequest.data = requestParams
|
|
49
|
+
receivedResponse = await func(request=kibaRequest)
|
|
50
|
+
if not isinstance(receivedResponse, responseType):
|
|
51
|
+
raise InternalServerErrorException(f'Expected response to be of type {responseType}, got {type(receivedResponse)}')
|
|
52
|
+
return KibaJSONResponse(content=receivedResponse.model_dump())
|
|
53
|
+
|
|
54
|
+
# TODO(krishan711): figure out correct typing here
|
|
55
|
+
return async_wrapper # type: ignore[return-value]
|
|
56
|
+
|
|
57
|
+
return decorator
|
|
File without changes
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from starlette.middleware.base import BaseHTTPMiddleware
|
|
2
|
+
from starlette.middleware.base import RequestResponseEndpoint
|
|
3
|
+
from starlette.requests import Request
|
|
4
|
+
from starlette.responses import Response
|
|
5
|
+
from starlette.types import ASGIApp
|
|
6
|
+
|
|
7
|
+
from core.store.database import Database
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class DatabaseConnectionMiddleware(BaseHTTPMiddleware):
|
|
11
|
+
def __init__(self, app: ASGIApp, database: Database) -> None:
|
|
12
|
+
super().__init__(app=app)
|
|
13
|
+
self.database = database
|
|
14
|
+
|
|
15
|
+
async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
|
|
16
|
+
# isReadonly = request.method in {'GET', 'OPTIONS', 'HEAD'}
|
|
17
|
+
async with self.database.create_context_connection():
|
|
18
|
+
response = await call_next(request)
|
|
19
|
+
return response
|