intentkit 0.7.5.dev3__py3-none-any.whl → 0.8.34.dev7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- intentkit/MANIFEST.in +14 -0
- intentkit/README.md +88 -0
- intentkit/__init__.py +6 -4
- intentkit/abstracts/agent.py +4 -5
- intentkit/abstracts/engine.py +5 -5
- intentkit/abstracts/graph.py +15 -8
- intentkit/abstracts/skill.py +6 -144
- intentkit/abstracts/twitter.py +4 -5
- intentkit/clients/__init__.py +9 -2
- intentkit/clients/cdp.py +129 -153
- intentkit/{utils → clients}/s3.py +109 -34
- intentkit/clients/twitter.py +83 -62
- intentkit/clients/web3.py +4 -7
- intentkit/config/config.py +123 -90
- intentkit/core/account_checking.py +802 -0
- intentkit/core/agent.py +313 -498
- intentkit/core/asset.py +267 -0
- intentkit/core/chat.py +5 -3
- intentkit/core/client.py +1 -1
- intentkit/core/credit.py +49 -41
- intentkit/core/draft.py +201 -0
- intentkit/core/draft_chat.py +118 -0
- intentkit/core/engine.py +378 -287
- intentkit/core/manager/__init__.py +25 -0
- intentkit/core/manager/engine.py +220 -0
- intentkit/core/manager/service.py +172 -0
- intentkit/core/manager/skills.py +178 -0
- intentkit/core/middleware.py +231 -0
- intentkit/core/prompt.py +74 -114
- intentkit/core/scheduler.py +143 -0
- intentkit/core/statistics.py +168 -0
- intentkit/models/agent.py +931 -518
- intentkit/models/agent_data.py +165 -106
- intentkit/models/agent_schema.json +38 -251
- intentkit/models/app_setting.py +15 -13
- intentkit/models/chat.py +86 -140
- intentkit/models/credit.py +182 -162
- intentkit/models/db.py +42 -23
- intentkit/models/db_mig.py +120 -3
- intentkit/models/draft.py +222 -0
- intentkit/models/llm.csv +31 -0
- intentkit/models/llm.py +262 -370
- intentkit/models/redis.py +6 -4
- intentkit/models/skill.py +222 -101
- intentkit/models/skills.csv +173 -0
- intentkit/models/team.py +189 -0
- intentkit/models/user.py +103 -31
- intentkit/skills/acolyt/__init__.py +2 -9
- intentkit/skills/acolyt/ask.py +3 -4
- intentkit/skills/acolyt/base.py +4 -9
- intentkit/skills/acolyt/schema.json +4 -3
- intentkit/skills/aixbt/__init__.py +2 -13
- intentkit/skills/aixbt/base.py +1 -7
- intentkit/skills/aixbt/projects.py +14 -15
- intentkit/skills/aixbt/schema.json +4 -4
- intentkit/skills/allora/__init__.py +2 -9
- intentkit/skills/allora/base.py +4 -9
- intentkit/skills/allora/price.py +3 -4
- intentkit/skills/allora/schema.json +3 -2
- intentkit/skills/base.py +241 -41
- intentkit/skills/basename/__init__.py +51 -0
- intentkit/skills/basename/base.py +11 -0
- intentkit/skills/basename/basename.svg +11 -0
- intentkit/skills/basename/schema.json +58 -0
- intentkit/skills/carv/__init__.py +115 -121
- intentkit/skills/carv/base.py +184 -185
- intentkit/skills/carv/fetch_news.py +3 -3
- intentkit/skills/carv/onchain_query.py +4 -4
- intentkit/skills/carv/schema.json +134 -137
- intentkit/skills/carv/token_info_and_price.py +6 -6
- intentkit/skills/casino/__init__.py +4 -15
- intentkit/skills/casino/base.py +1 -7
- intentkit/skills/casino/deck_draw.py +5 -8
- intentkit/skills/casino/deck_shuffle.py +6 -6
- intentkit/skills/casino/dice_roll.py +2 -4
- intentkit/skills/casino/schema.json +0 -1
- intentkit/skills/cdp/__init__.py +22 -84
- intentkit/skills/cdp/base.py +1 -7
- intentkit/skills/cdp/schema.json +11 -314
- intentkit/skills/chainlist/__init__.py +2 -7
- intentkit/skills/chainlist/base.py +1 -7
- intentkit/skills/chainlist/chain_lookup.py +18 -18
- intentkit/skills/chainlist/schema.json +3 -5
- intentkit/skills/common/__init__.py +2 -9
- intentkit/skills/common/base.py +1 -7
- intentkit/skills/common/current_time.py +1 -2
- intentkit/skills/common/schema.json +2 -2
- intentkit/skills/cookiefun/__init__.py +6 -9
- intentkit/skills/cookiefun/base.py +2 -7
- intentkit/skills/cookiefun/get_account_details.py +7 -7
- intentkit/skills/cookiefun/get_account_feed.py +19 -19
- intentkit/skills/cookiefun/get_account_smart_followers.py +7 -7
- intentkit/skills/cookiefun/get_sectors.py +3 -3
- intentkit/skills/cookiefun/schema.json +1 -3
- intentkit/skills/cookiefun/search_accounts.py +9 -9
- intentkit/skills/cryptocompare/__init__.py +7 -24
- intentkit/skills/cryptocompare/api.py +2 -3
- intentkit/skills/cryptocompare/base.py +10 -24
- intentkit/skills/cryptocompare/fetch_news.py +4 -5
- intentkit/skills/cryptocompare/fetch_price.py +6 -7
- intentkit/skills/cryptocompare/fetch_top_exchanges.py +4 -5
- intentkit/skills/cryptocompare/fetch_top_market_cap.py +4 -5
- intentkit/skills/cryptocompare/fetch_top_volume.py +4 -5
- intentkit/skills/cryptocompare/fetch_trading_signals.py +5 -6
- intentkit/skills/cryptocompare/schema.json +3 -3
- intentkit/skills/cryptopanic/__init__.py +7 -10
- intentkit/skills/cryptopanic/base.py +51 -55
- intentkit/skills/cryptopanic/fetch_crypto_news.py +4 -8
- intentkit/skills/cryptopanic/fetch_crypto_sentiment.py +5 -7
- intentkit/skills/cryptopanic/schema.json +105 -103
- intentkit/skills/dapplooker/__init__.py +2 -9
- intentkit/skills/dapplooker/base.py +4 -9
- intentkit/skills/dapplooker/dapplooker_token_data.py +7 -7
- intentkit/skills/dapplooker/schema.json +3 -5
- intentkit/skills/defillama/__init__.py +24 -74
- intentkit/skills/defillama/api.py +6 -9
- intentkit/skills/defillama/base.py +8 -19
- intentkit/skills/defillama/coins/fetch_batch_historical_prices.py +8 -10
- intentkit/skills/defillama/coins/fetch_block.py +6 -8
- intentkit/skills/defillama/coins/fetch_current_prices.py +8 -10
- intentkit/skills/defillama/coins/fetch_first_price.py +7 -9
- intentkit/skills/defillama/coins/fetch_historical_prices.py +9 -11
- intentkit/skills/defillama/coins/fetch_price_chart.py +9 -11
- intentkit/skills/defillama/coins/fetch_price_percentage.py +7 -9
- intentkit/skills/defillama/config/chains.py +1 -3
- intentkit/skills/defillama/fees/fetch_fees_overview.py +24 -26
- intentkit/skills/defillama/schema.json +5 -1
- intentkit/skills/defillama/stablecoins/fetch_stablecoin_chains.py +16 -18
- intentkit/skills/defillama/stablecoins/fetch_stablecoin_charts.py +8 -10
- intentkit/skills/defillama/stablecoins/fetch_stablecoin_prices.py +5 -7
- intentkit/skills/defillama/stablecoins/fetch_stablecoins.py +7 -9
- intentkit/skills/defillama/tests/api_integration.test.py +1 -1
- intentkit/skills/defillama/tvl/fetch_chain_historical_tvl.py +4 -6
- intentkit/skills/defillama/tvl/fetch_chains.py +9 -11
- intentkit/skills/defillama/tvl/fetch_historical_tvl.py +4 -6
- intentkit/skills/defillama/tvl/fetch_protocol.py +32 -38
- intentkit/skills/defillama/tvl/fetch_protocol_current_tvl.py +3 -5
- intentkit/skills/defillama/tvl/fetch_protocols.py +37 -45
- intentkit/skills/defillama/volumes/fetch_dex_overview.py +42 -48
- intentkit/skills/defillama/volumes/fetch_dex_summary.py +35 -37
- intentkit/skills/defillama/volumes/fetch_options_overview.py +24 -28
- intentkit/skills/defillama/yields/fetch_pool_chart.py +10 -12
- intentkit/skills/defillama/yields/fetch_pools.py +26 -30
- intentkit/skills/dexscreener/__init__.py +97 -102
- intentkit/skills/dexscreener/base.py +125 -130
- intentkit/skills/dexscreener/get_pair_info.py +4 -5
- intentkit/skills/dexscreener/get_token_pairs.py +4 -5
- intentkit/skills/dexscreener/get_tokens_info.py +7 -8
- intentkit/skills/dexscreener/model/search_token_response.py +80 -82
- intentkit/skills/dexscreener/schema.json +91 -93
- intentkit/skills/dexscreener/search_token.py +182 -184
- intentkit/skills/dexscreener/utils.py +15 -14
- intentkit/skills/dune_analytics/__init__.py +7 -9
- intentkit/skills/dune_analytics/base.py +48 -52
- intentkit/skills/dune_analytics/fetch_kol_buys.py +5 -7
- intentkit/skills/dune_analytics/fetch_nation_metrics.py +6 -8
- intentkit/skills/dune_analytics/schema.json +104 -99
- intentkit/skills/elfa/__init__.py +5 -18
- intentkit/skills/elfa/base.py +10 -14
- intentkit/skills/elfa/mention.py +19 -21
- intentkit/skills/elfa/schema.json +3 -2
- intentkit/skills/elfa/stats.py +4 -4
- intentkit/skills/elfa/tokens.py +12 -12
- intentkit/skills/elfa/utils.py +26 -28
- intentkit/skills/enso/__init__.py +11 -31
- intentkit/skills/enso/base.py +54 -35
- intentkit/skills/enso/best_yield.py +16 -24
- intentkit/skills/enso/networks.py +6 -11
- intentkit/skills/enso/prices.py +11 -13
- intentkit/skills/enso/route.py +34 -38
- intentkit/skills/enso/schema.json +3 -2
- intentkit/skills/enso/tokens.py +29 -38
- intentkit/skills/enso/wallet.py +76 -191
- intentkit/skills/erc20/__init__.py +50 -0
- intentkit/skills/erc20/base.py +11 -0
- intentkit/skills/erc20/erc20.svg +5 -0
- intentkit/skills/erc20/schema.json +74 -0
- intentkit/skills/erc721/__init__.py +53 -0
- intentkit/skills/erc721/base.py +11 -0
- intentkit/skills/erc721/erc721.svg +5 -0
- intentkit/skills/erc721/schema.json +90 -0
- intentkit/skills/firecrawl/__init__.py +5 -18
- intentkit/skills/firecrawl/base.py +4 -9
- intentkit/skills/firecrawl/clear.py +4 -8
- intentkit/skills/firecrawl/crawl.py +19 -19
- intentkit/skills/firecrawl/query.py +4 -3
- intentkit/skills/firecrawl/schema.json +2 -6
- intentkit/skills/firecrawl/scrape.py +17 -22
- intentkit/skills/firecrawl/utils.py +50 -42
- intentkit/skills/github/__init__.py +2 -7
- intentkit/skills/github/base.py +1 -7
- intentkit/skills/github/github_search.py +1 -2
- intentkit/skills/github/schema.json +3 -4
- intentkit/skills/heurist/__init__.py +8 -27
- intentkit/skills/heurist/base.py +4 -9
- intentkit/skills/heurist/image_generation_animagine_xl.py +13 -15
- intentkit/skills/heurist/image_generation_arthemy_comics.py +13 -15
- intentkit/skills/heurist/image_generation_arthemy_real.py +13 -15
- intentkit/skills/heurist/image_generation_braindance.py +13 -15
- intentkit/skills/heurist/image_generation_cyber_realistic_xl.py +13 -15
- intentkit/skills/heurist/image_generation_flux_1_dev.py +13 -15
- intentkit/skills/heurist/image_generation_sdxl.py +13 -15
- intentkit/skills/heurist/schema.json +2 -2
- intentkit/skills/http/__init__.py +4 -15
- intentkit/skills/http/base.py +1 -7
- intentkit/skills/http/get.py +21 -16
- intentkit/skills/http/post.py +23 -18
- intentkit/skills/http/put.py +23 -18
- intentkit/skills/http/schema.json +4 -5
- intentkit/skills/lifi/__init__.py +8 -13
- intentkit/skills/lifi/base.py +3 -9
- intentkit/skills/lifi/schema.json +17 -8
- intentkit/skills/lifi/token_execute.py +150 -60
- intentkit/skills/lifi/token_quote.py +8 -10
- intentkit/skills/lifi/utils.py +104 -51
- intentkit/skills/moralis/__init__.py +6 -10
- intentkit/skills/moralis/api.py +6 -7
- intentkit/skills/moralis/base.py +5 -10
- intentkit/skills/moralis/fetch_chain_portfolio.py +10 -11
- intentkit/skills/moralis/fetch_nft_portfolio.py +22 -22
- intentkit/skills/moralis/fetch_solana_portfolio.py +11 -12
- intentkit/skills/moralis/fetch_wallet_portfolio.py +8 -9
- intentkit/skills/moralis/schema.json +7 -2
- intentkit/skills/morpho/__init__.py +52 -0
- intentkit/skills/morpho/base.py +11 -0
- intentkit/skills/morpho/morpho.svg +12 -0
- intentkit/skills/morpho/schema.json +73 -0
- intentkit/skills/nation/__init__.py +4 -9
- intentkit/skills/nation/base.py +5 -10
- intentkit/skills/nation/nft_check.py +3 -4
- intentkit/skills/nation/schema.json +4 -3
- intentkit/skills/onchain.py +30 -0
- intentkit/skills/openai/__init__.py +17 -18
- intentkit/skills/openai/base.py +10 -14
- intentkit/skills/openai/dalle_image_generation.py +4 -9
- intentkit/skills/openai/gpt_avatar_generator.py +102 -0
- intentkit/skills/openai/gpt_image_generation.py +5 -9
- intentkit/skills/openai/gpt_image_mini_generator.py +92 -0
- intentkit/skills/openai/gpt_image_to_image.py +5 -9
- intentkit/skills/openai/image_to_text.py +3 -7
- intentkit/skills/openai/schema.json +34 -3
- intentkit/skills/portfolio/__init__.py +11 -35
- intentkit/skills/portfolio/base.py +33 -19
- intentkit/skills/portfolio/schema.json +3 -5
- intentkit/skills/portfolio/token_balances.py +21 -21
- intentkit/skills/portfolio/wallet_approvals.py +17 -18
- intentkit/skills/portfolio/wallet_defi_positions.py +3 -3
- intentkit/skills/portfolio/wallet_history.py +31 -31
- intentkit/skills/portfolio/wallet_net_worth.py +13 -13
- intentkit/skills/portfolio/wallet_nfts.py +19 -19
- intentkit/skills/portfolio/wallet_profitability.py +18 -18
- intentkit/skills/portfolio/wallet_profitability_summary.py +5 -5
- intentkit/skills/portfolio/wallet_stats.py +3 -3
- intentkit/skills/portfolio/wallet_swaps.py +19 -19
- intentkit/skills/pyth/__init__.py +50 -0
- intentkit/skills/pyth/base.py +11 -0
- intentkit/skills/pyth/pyth.svg +6 -0
- intentkit/skills/pyth/schema.json +75 -0
- intentkit/skills/skills.toml +36 -0
- intentkit/skills/slack/__init__.py +5 -17
- intentkit/skills/slack/base.py +3 -9
- intentkit/skills/slack/get_channel.py +8 -8
- intentkit/skills/slack/get_message.py +9 -9
- intentkit/skills/slack/schedule_message.py +5 -5
- intentkit/skills/slack/schema.json +2 -2
- intentkit/skills/slack/send_message.py +3 -5
- intentkit/skills/supabase/__init__.py +7 -23
- intentkit/skills/supabase/base.py +1 -7
- intentkit/skills/supabase/delete_data.py +4 -4
- intentkit/skills/supabase/fetch_data.py +12 -12
- intentkit/skills/supabase/insert_data.py +4 -4
- intentkit/skills/supabase/invoke_function.py +6 -6
- intentkit/skills/supabase/schema.json +2 -3
- intentkit/skills/supabase/update_data.py +6 -6
- intentkit/skills/supabase/upsert_data.py +4 -4
- intentkit/skills/superfluid/__init__.py +53 -0
- intentkit/skills/superfluid/base.py +11 -0
- intentkit/skills/superfluid/schema.json +89 -0
- intentkit/skills/superfluid/superfluid.svg +6 -0
- intentkit/skills/system/__init__.py +7 -24
- intentkit/skills/system/add_autonomous_task.py +10 -12
- intentkit/skills/system/delete_autonomous_task.py +2 -2
- intentkit/skills/system/edit_autonomous_task.py +14 -18
- intentkit/skills/system/list_autonomous_tasks.py +3 -5
- intentkit/skills/system/read_agent_api_key.py +6 -4
- intentkit/skills/system/regenerate_agent_api_key.py +6 -4
- intentkit/skills/system/schema.json +6 -8
- intentkit/skills/tavily/__init__.py +3 -12
- intentkit/skills/tavily/base.py +4 -9
- intentkit/skills/tavily/schema.json +3 -5
- intentkit/skills/tavily/tavily_extract.py +2 -4
- intentkit/skills/tavily/tavily_search.py +4 -6
- intentkit/skills/token/__init__.py +5 -10
- intentkit/skills/token/base.py +7 -11
- intentkit/skills/token/erc20_transfers.py +19 -19
- intentkit/skills/token/schema.json +3 -6
- intentkit/skills/token/token_analytics.py +3 -3
- intentkit/skills/token/token_price.py +13 -13
- intentkit/skills/token/token_search.py +9 -9
- intentkit/skills/twitter/__init__.py +11 -35
- intentkit/skills/twitter/base.py +22 -34
- intentkit/skills/twitter/follow_user.py +2 -6
- intentkit/skills/twitter/get_mentions.py +5 -12
- intentkit/skills/twitter/get_timeline.py +4 -12
- intentkit/skills/twitter/get_user_by_username.py +2 -6
- intentkit/skills/twitter/get_user_tweets.py +5 -13
- intentkit/skills/twitter/like_tweet.py +2 -6
- intentkit/skills/twitter/post_tweet.py +6 -9
- intentkit/skills/twitter/reply_tweet.py +6 -9
- intentkit/skills/twitter/retweet.py +2 -6
- intentkit/skills/twitter/schema.json +1 -0
- intentkit/skills/twitter/search_tweets.py +4 -12
- intentkit/skills/unrealspeech/__init__.py +2 -7
- intentkit/skills/unrealspeech/base.py +2 -8
- intentkit/skills/unrealspeech/schema.json +2 -5
- intentkit/skills/unrealspeech/text_to_speech.py +8 -8
- intentkit/skills/venice_audio/__init__.py +98 -106
- intentkit/skills/venice_audio/base.py +117 -121
- intentkit/skills/venice_audio/input.py +41 -41
- intentkit/skills/venice_audio/schema.json +151 -152
- intentkit/skills/venice_audio/venice_audio.py +38 -21
- intentkit/skills/venice_image/__init__.py +147 -154
- intentkit/skills/venice_image/api.py +138 -138
- intentkit/skills/venice_image/base.py +185 -192
- intentkit/skills/venice_image/config.py +33 -35
- intentkit/skills/venice_image/image_enhance/image_enhance.py +2 -3
- intentkit/skills/venice_image/image_enhance/image_enhance_base.py +21 -23
- intentkit/skills/venice_image/image_enhance/image_enhance_input.py +38 -40
- intentkit/skills/venice_image/image_generation/image_generation_base.py +11 -10
- intentkit/skills/venice_image/image_generation/image_generation_fluently_xl.py +26 -26
- intentkit/skills/venice_image/image_generation/image_generation_flux_dev.py +27 -27
- intentkit/skills/venice_image/image_generation/image_generation_flux_dev_uncensored.py +26 -26
- intentkit/skills/venice_image/image_generation/image_generation_input.py +158 -158
- intentkit/skills/venice_image/image_generation/image_generation_lustify_sdxl.py +26 -26
- intentkit/skills/venice_image/image_generation/image_generation_pony_realism.py +26 -26
- intentkit/skills/venice_image/image_generation/image_generation_stable_diffusion_3_5.py +28 -28
- intentkit/skills/venice_image/image_generation/image_generation_venice_sd35.py +28 -28
- intentkit/skills/venice_image/image_upscale/image_upscale.py +3 -3
- intentkit/skills/venice_image/image_upscale/image_upscale_base.py +21 -23
- intentkit/skills/venice_image/image_upscale/image_upscale_input.py +22 -22
- intentkit/skills/venice_image/image_vision/image_vision.py +2 -2
- intentkit/skills/venice_image/image_vision/image_vision_base.py +17 -17
- intentkit/skills/venice_image/image_vision/image_vision_input.py +9 -9
- intentkit/skills/venice_image/schema.json +267 -267
- intentkit/skills/venice_image/utils.py +77 -78
- intentkit/skills/web_scraper/__init__.py +5 -18
- intentkit/skills/web_scraper/base.py +21 -7
- intentkit/skills/web_scraper/document_indexer.py +7 -6
- intentkit/skills/web_scraper/schema.json +2 -6
- intentkit/skills/web_scraper/scrape_and_index.py +15 -15
- intentkit/skills/web_scraper/utils.py +62 -63
- intentkit/skills/web_scraper/website_indexer.py +17 -19
- intentkit/skills/weth/__init__.py +49 -0
- intentkit/skills/weth/base.py +11 -0
- intentkit/skills/weth/schema.json +58 -0
- intentkit/skills/weth/weth.svg +6 -0
- intentkit/skills/wow/__init__.py +51 -0
- intentkit/skills/wow/base.py +11 -0
- intentkit/skills/wow/schema.json +89 -0
- intentkit/skills/wow/wow.svg +7 -0
- intentkit/skills/x402/__init__.py +58 -0
- intentkit/skills/x402/base.py +99 -0
- intentkit/skills/x402/http_request.py +117 -0
- intentkit/skills/x402/schema.json +40 -0
- intentkit/skills/x402/x402.webp +0 -0
- intentkit/skills/xmtp/__init__.py +4 -15
- intentkit/skills/xmtp/base.py +5 -5
- intentkit/skills/xmtp/price.py +7 -6
- intentkit/skills/xmtp/schema.json +69 -71
- intentkit/skills/xmtp/swap.py +6 -8
- intentkit/skills/xmtp/transfer.py +4 -6
- intentkit/utils/__init__.py +4 -0
- intentkit/utils/chain.py +198 -96
- intentkit/utils/ens.py +135 -0
- intentkit/utils/error.py +5 -2
- intentkit/utils/logging.py +9 -11
- intentkit/utils/schema.py +100 -0
- intentkit/utils/slack_alert.py +8 -8
- intentkit/utils/tx.py +16 -8
- intentkit/uv.lock +3377 -0
- {intentkit-0.7.5.dev3.dist-info → intentkit-0.8.34.dev7.dist-info}/METADATA +13 -15
- intentkit-0.8.34.dev7.dist-info/RECORD +478 -0
- intentkit-0.8.34.dev7.dist-info/licenses/LICENSE +21 -0
- intentkit/core/node.py +0 -215
- intentkit/models/conversation.py +0 -286
- intentkit/models/generator.py +0 -347
- intentkit/skills/cdp/get_balance.py +0 -110
- intentkit/skills/cdp/swap.py +0 -121
- intentkit/skills/moralis/tests/__init__.py +0 -0
- intentkit/skills/moralis/tests/test_wallet.py +0 -511
- intentkit-0.7.5.dev3.dist-info/RECORD +0 -424
- {intentkit-0.7.5.dev3.dist-info/licenses → intentkit}/LICENSE +0 -0
- {intentkit-0.7.5.dev3.dist-info → intentkit-0.8.34.dev7.dist-info}/WHEEL +0 -0
|
@@ -0,0 +1,802 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import logging
|
|
3
|
+
from datetime import UTC, datetime, timedelta
|
|
4
|
+
from decimal import Decimal
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from sqlalchemy import select, text
|
|
8
|
+
|
|
9
|
+
from intentkit.models.credit import (
|
|
10
|
+
CreditAccount,
|
|
11
|
+
CreditAccountTable,
|
|
12
|
+
CreditEvent,
|
|
13
|
+
CreditEventTable,
|
|
14
|
+
CreditTransaction,
|
|
15
|
+
CreditTransactionTable,
|
|
16
|
+
)
|
|
17
|
+
from intentkit.models.db import get_session
|
|
18
|
+
from intentkit.utils.slack_alert import send_slack_message
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class AccountCheckingResult:
|
|
24
|
+
"""Result of an account checking operation."""
|
|
25
|
+
|
|
26
|
+
def __init__(
|
|
27
|
+
self, check_type: str, status: bool, details: dict[str, Any] | None = None
|
|
28
|
+
):
|
|
29
|
+
self.check_type = check_type
|
|
30
|
+
self.status = status # True if check passed, False if failed
|
|
31
|
+
self.details = details or {}
|
|
32
|
+
self.timestamp = datetime.now(UTC)
|
|
33
|
+
|
|
34
|
+
def __str__(self) -> str:
|
|
35
|
+
status_str = "PASSED" if self.status else "FAILED"
|
|
36
|
+
return f"[{self.timestamp.isoformat()}] {self.check_type}: {status_str} - {self.details}"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
async def check_account_balance_consistency(
|
|
40
|
+
check_recent_only: bool = True, recent_hours: int = 24
|
|
41
|
+
) -> list[AccountCheckingResult]:
|
|
42
|
+
"""Check if all account balances are consistent with their transactions.
|
|
43
|
+
|
|
44
|
+
This verifies that the total balance in each account matches the sum of all transactions
|
|
45
|
+
for that account, properly accounting for credits and debits.
|
|
46
|
+
|
|
47
|
+
To ensure consistency during system operation, this function processes accounts in batches
|
|
48
|
+
using ID-based pagination and uses the last_event_id from each account to limit
|
|
49
|
+
transaction queries, ensuring that only transactions from events up to and including
|
|
50
|
+
the last recorded event for that account are considered.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
check_recent_only: If True, only check accounts updated within recent_hours. Default True.
|
|
54
|
+
recent_hours: Number of hours to look back for recent updates. Default 24.
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
List of checking results
|
|
58
|
+
"""
|
|
59
|
+
results = []
|
|
60
|
+
batch_size = 1000 # Process 1000 accounts at a time
|
|
61
|
+
total_processed = 0
|
|
62
|
+
batch_count = 0
|
|
63
|
+
last_id = "" # Starting ID for pagination (empty string comes before all valid IDs)
|
|
64
|
+
|
|
65
|
+
# Calculate time threshold for recent updates if needed
|
|
66
|
+
time_threshold = None
|
|
67
|
+
if check_recent_only:
|
|
68
|
+
time_threshold = datetime.now(UTC) - timedelta(hours=recent_hours)
|
|
69
|
+
|
|
70
|
+
while True:
|
|
71
|
+
# Create a new session for each batch to prevent timeouts
|
|
72
|
+
async with get_session() as session:
|
|
73
|
+
# Get accounts in batches using ID-based pagination
|
|
74
|
+
query = (
|
|
75
|
+
select(CreditAccountTable)
|
|
76
|
+
.where(CreditAccountTable.id > last_id) # ID-based pagination
|
|
77
|
+
.order_by(CreditAccountTable.id)
|
|
78
|
+
.limit(batch_size)
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
# Add time filter if checking recent updates only
|
|
82
|
+
if check_recent_only and time_threshold:
|
|
83
|
+
query = query.where(CreditAccountTable.updated_at >= time_threshold)
|
|
84
|
+
accounts_result = await session.execute(query)
|
|
85
|
+
batch_accounts = [
|
|
86
|
+
CreditAccount.model_validate(acc)
|
|
87
|
+
for acc in accounts_result.scalars().all()
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
# If no more accounts to process, break the loop
|
|
91
|
+
if not batch_accounts:
|
|
92
|
+
break
|
|
93
|
+
|
|
94
|
+
# Update counters and last_id for next iteration
|
|
95
|
+
batch_count += 1
|
|
96
|
+
current_batch_size = len(batch_accounts)
|
|
97
|
+
total_processed += current_batch_size
|
|
98
|
+
last_id = batch_accounts[-1].id # Update last_id for next batch
|
|
99
|
+
|
|
100
|
+
logger.info(
|
|
101
|
+
f"Processing account balance batch: {batch_count}, accounts: {current_batch_size}"
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
# Process each account in the batch
|
|
105
|
+
for account in batch_accounts:
|
|
106
|
+
# Sleep for 10ms to reduce database load
|
|
107
|
+
await asyncio.sleep(0.01)
|
|
108
|
+
|
|
109
|
+
# Calculate the total balance across all credit types
|
|
110
|
+
total_balance = (
|
|
111
|
+
account.free_credits + account.reward_credits + account.credits
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
# Calculate the expected balance from all transactions, regardless of credit type
|
|
115
|
+
# If account has last_event_id, only include transactions from events up to and including that event
|
|
116
|
+
# If no last_event_id, include all transactions for the account
|
|
117
|
+
if account.last_event_id:
|
|
118
|
+
query = text("""
|
|
119
|
+
SELECT
|
|
120
|
+
SUM(CASE WHEN ct.credit_debit = 'credit' THEN ct.change_amount ELSE 0 END) as credits,
|
|
121
|
+
SUM(CASE WHEN ct.credit_debit = 'debit' THEN ct.change_amount ELSE 0 END) as debits,
|
|
122
|
+
SUM(CASE WHEN ct.credit_debit = 'credit' THEN ct.free_amount ELSE -ct.free_amount END) as free_credits_sum,
|
|
123
|
+
SUM(CASE WHEN ct.credit_debit = 'credit' THEN ct.reward_amount ELSE -ct.reward_amount END) as reward_credits_sum,
|
|
124
|
+
SUM(CASE WHEN ct.credit_debit = 'credit' THEN ct.permanent_amount ELSE -ct.permanent_amount END) as permanent_credits_sum
|
|
125
|
+
FROM credit_transactions ct
|
|
126
|
+
JOIN credit_events ce ON ct.event_id = ce.id
|
|
127
|
+
WHERE ct.account_id = :account_id
|
|
128
|
+
AND ce.id <= :last_event_id
|
|
129
|
+
""")
|
|
130
|
+
|
|
131
|
+
tx_result = await session.execute(
|
|
132
|
+
query,
|
|
133
|
+
{
|
|
134
|
+
"account_id": account.id,
|
|
135
|
+
"last_event_id": account.last_event_id,
|
|
136
|
+
},
|
|
137
|
+
)
|
|
138
|
+
else:
|
|
139
|
+
query = text("""
|
|
140
|
+
SELECT
|
|
141
|
+
SUM(CASE WHEN ct.credit_debit = 'credit' THEN ct.change_amount ELSE 0 END) as credits,
|
|
142
|
+
SUM(CASE WHEN ct.credit_debit = 'debit' THEN ct.change_amount ELSE 0 END) as debits,
|
|
143
|
+
SUM(CASE WHEN ct.credit_debit = 'credit' THEN ct.free_amount ELSE -ct.free_amount END) as free_credits_sum,
|
|
144
|
+
SUM(CASE WHEN ct.credit_debit = 'credit' THEN ct.reward_amount ELSE -ct.reward_amount END) as reward_credits_sum,
|
|
145
|
+
SUM(CASE WHEN ct.credit_debit = 'credit' THEN ct.permanent_amount ELSE -ct.permanent_amount END) as permanent_credits_sum
|
|
146
|
+
FROM credit_transactions ct
|
|
147
|
+
WHERE ct.account_id = :account_id
|
|
148
|
+
""")
|
|
149
|
+
|
|
150
|
+
tx_result = await session.execute(
|
|
151
|
+
query,
|
|
152
|
+
{"account_id": account.id},
|
|
153
|
+
)
|
|
154
|
+
tx_data = tx_result.fetchone()
|
|
155
|
+
|
|
156
|
+
if tx_data is None:
|
|
157
|
+
credits = Decimal("0")
|
|
158
|
+
debits = Decimal("0")
|
|
159
|
+
expected_free_credits = Decimal("0")
|
|
160
|
+
expected_reward_credits = Decimal("0")
|
|
161
|
+
expected_permanent_credits = Decimal("0")
|
|
162
|
+
else:
|
|
163
|
+
credits = tx_data.credits or Decimal("0")
|
|
164
|
+
debits = tx_data.debits or Decimal("0")
|
|
165
|
+
expected_free_credits = tx_data.free_credits_sum or Decimal("0")
|
|
166
|
+
expected_reward_credits = tx_data.reward_credits_sum or Decimal("0")
|
|
167
|
+
expected_permanent_credits = (
|
|
168
|
+
tx_data.permanent_credits_sum or Decimal("0")
|
|
169
|
+
)
|
|
170
|
+
expected_balance = credits - debits
|
|
171
|
+
|
|
172
|
+
# Compare total balances and individual credit type balances with tolerance
|
|
173
|
+
tolerance = Decimal("0.01")
|
|
174
|
+
|
|
175
|
+
total_balance_diff = abs(total_balance - expected_balance)
|
|
176
|
+
free_credits_diff = abs(account.free_credits - expected_free_credits)
|
|
177
|
+
reward_credits_diff = abs(
|
|
178
|
+
account.reward_credits - expected_reward_credits
|
|
179
|
+
)
|
|
180
|
+
permanent_credits_diff = abs(
|
|
181
|
+
account.credits - expected_permanent_credits
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
is_total_consistent = total_balance_diff <= tolerance
|
|
185
|
+
is_free_consistent = free_credits_diff <= tolerance
|
|
186
|
+
is_reward_consistent = reward_credits_diff <= tolerance
|
|
187
|
+
is_permanent_consistent = permanent_credits_diff <= tolerance
|
|
188
|
+
|
|
189
|
+
is_consistent = (
|
|
190
|
+
is_total_consistent
|
|
191
|
+
and is_free_consistent
|
|
192
|
+
and is_reward_consistent
|
|
193
|
+
and is_permanent_consistent
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
result = AccountCheckingResult(
|
|
197
|
+
check_type="account_balance_consistency",
|
|
198
|
+
status=is_consistent,
|
|
199
|
+
details={
|
|
200
|
+
"account_id": account.id,
|
|
201
|
+
"owner_type": account.owner_type,
|
|
202
|
+
"owner_id": account.owner_id,
|
|
203
|
+
"current_total_balance": float(total_balance),
|
|
204
|
+
"free_credits": float(account.free_credits),
|
|
205
|
+
"reward_credits": float(account.reward_credits),
|
|
206
|
+
"permanent_credits": float(account.credits),
|
|
207
|
+
"expected_total_balance": float(expected_balance),
|
|
208
|
+
"expected_free_credits": float(expected_free_credits),
|
|
209
|
+
"expected_reward_credits": float(expected_reward_credits),
|
|
210
|
+
"expected_permanent_credits": float(expected_permanent_credits),
|
|
211
|
+
"total_credits": float(credits),
|
|
212
|
+
"total_debits": float(debits),
|
|
213
|
+
"total_balance_difference": float(
|
|
214
|
+
total_balance - expected_balance
|
|
215
|
+
),
|
|
216
|
+
"free_credits_difference": float(
|
|
217
|
+
account.free_credits - expected_free_credits
|
|
218
|
+
),
|
|
219
|
+
"reward_credits_difference": float(
|
|
220
|
+
account.reward_credits - expected_reward_credits
|
|
221
|
+
),
|
|
222
|
+
"permanent_credits_difference": float(
|
|
223
|
+
account.credits - expected_permanent_credits
|
|
224
|
+
),
|
|
225
|
+
"is_total_consistent": is_total_consistent,
|
|
226
|
+
"is_free_consistent": is_free_consistent,
|
|
227
|
+
"is_reward_consistent": is_reward_consistent,
|
|
228
|
+
"is_permanent_consistent": is_permanent_consistent,
|
|
229
|
+
"last_event_id": account.last_event_id,
|
|
230
|
+
"batch": batch_count,
|
|
231
|
+
"check_recent_only": check_recent_only,
|
|
232
|
+
"recent_hours": recent_hours if check_recent_only else None,
|
|
233
|
+
},
|
|
234
|
+
)
|
|
235
|
+
results.append(result)
|
|
236
|
+
|
|
237
|
+
if not is_consistent:
|
|
238
|
+
inconsistency_details = []
|
|
239
|
+
if not is_total_consistent:
|
|
240
|
+
inconsistency_details.append(
|
|
241
|
+
f"Total: {total_balance} vs {expected_balance}"
|
|
242
|
+
)
|
|
243
|
+
if not is_free_consistent:
|
|
244
|
+
inconsistency_details.append(
|
|
245
|
+
f"Free: {account.free_credits} vs {expected_free_credits}"
|
|
246
|
+
)
|
|
247
|
+
if not is_reward_consistent:
|
|
248
|
+
inconsistency_details.append(
|
|
249
|
+
f"Reward: {account.reward_credits} vs {expected_reward_credits}"
|
|
250
|
+
)
|
|
251
|
+
if not is_permanent_consistent:
|
|
252
|
+
inconsistency_details.append(
|
|
253
|
+
f"Permanent: {account.credits} vs {expected_permanent_credits}"
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
logger.warning(
|
|
257
|
+
f"Account balance inconsistency detected: {account.id} ({account.owner_type}:{account.owner_id}) - "
|
|
258
|
+
f"{'; '.join(inconsistency_details)}"
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
filter_info = (
|
|
262
|
+
f" (recent {recent_hours}h only)" if check_recent_only else " (all accounts)"
|
|
263
|
+
)
|
|
264
|
+
logger.info(
|
|
265
|
+
f"Completed account balance consistency check{filter_info}: processed {total_processed} accounts in {batch_count} batches"
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
return results
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
async def check_transaction_balance() -> list[AccountCheckingResult]:
|
|
272
|
+
"""Check if all credit events have balanced transactions.
|
|
273
|
+
|
|
274
|
+
For each credit event, the sum of all credit transactions should equal the sum of all debit transactions.
|
|
275
|
+
Events are processed in batches to prevent memory overflow issues using ID-based pagination for better performance.
|
|
276
|
+
|
|
277
|
+
Returns:
|
|
278
|
+
List of checking results
|
|
279
|
+
"""
|
|
280
|
+
results = []
|
|
281
|
+
batch_size = 1000 # Process 1000 events at a time
|
|
282
|
+
total_processed = 0
|
|
283
|
+
batch_count = 0
|
|
284
|
+
last_id = "" # Starting ID for pagination (empty string comes before all valid IDs)
|
|
285
|
+
|
|
286
|
+
# Time window for events (last 3 days for performance)
|
|
287
|
+
three_days_ago = datetime.now(UTC) - timedelta(hours=4)
|
|
288
|
+
|
|
289
|
+
while True:
|
|
290
|
+
# Create a new session for each batch to prevent timeouts
|
|
291
|
+
async with get_session() as session:
|
|
292
|
+
# Get events in batches using ID-based pagination
|
|
293
|
+
query = (
|
|
294
|
+
select(CreditEventTable)
|
|
295
|
+
.where(CreditEventTable.created_at >= three_days_ago)
|
|
296
|
+
.where(
|
|
297
|
+
CreditEventTable.id > last_id
|
|
298
|
+
) # Key change: ID-based pagination with string comparison
|
|
299
|
+
.order_by(CreditEventTable.id)
|
|
300
|
+
.limit(batch_size)
|
|
301
|
+
)
|
|
302
|
+
events_result = await session.execute(query)
|
|
303
|
+
batch_events = [
|
|
304
|
+
CreditEvent.model_validate(event)
|
|
305
|
+
for event in events_result.scalars().all()
|
|
306
|
+
]
|
|
307
|
+
|
|
308
|
+
# If no more events to process, break the loop
|
|
309
|
+
if not batch_events:
|
|
310
|
+
break
|
|
311
|
+
|
|
312
|
+
# Update counters and last_id for next iteration
|
|
313
|
+
batch_count += 1
|
|
314
|
+
current_batch_size = len(batch_events)
|
|
315
|
+
total_processed += current_batch_size
|
|
316
|
+
last_id = batch_events[-1].id # Update last_id for next batch
|
|
317
|
+
|
|
318
|
+
logger.info(
|
|
319
|
+
f"Processing transaction balance batch: {batch_count}, events: {current_batch_size}"
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
# Process each event in the batch
|
|
323
|
+
for event in batch_events:
|
|
324
|
+
# Sleep for 10ms to reduce database load
|
|
325
|
+
await asyncio.sleep(0.01)
|
|
326
|
+
|
|
327
|
+
# Get all transactions for this event
|
|
328
|
+
tx_query = select(CreditTransactionTable).where(
|
|
329
|
+
CreditTransactionTable.event_id == event.id
|
|
330
|
+
)
|
|
331
|
+
tx_result = await session.execute(tx_query)
|
|
332
|
+
transactions = [
|
|
333
|
+
CreditTransaction.model_validate(tx)
|
|
334
|
+
for tx in tx_result.scalars().all()
|
|
335
|
+
]
|
|
336
|
+
|
|
337
|
+
# Calculate credit and debit sums
|
|
338
|
+
credit_sum = sum(
|
|
339
|
+
tx.change_amount
|
|
340
|
+
for tx in transactions
|
|
341
|
+
if tx.credit_debit == "credit"
|
|
342
|
+
)
|
|
343
|
+
debit_sum = sum(
|
|
344
|
+
tx.change_amount
|
|
345
|
+
for tx in transactions
|
|
346
|
+
if tx.credit_debit == "debit"
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
# Check if they balance
|
|
350
|
+
is_balanced = credit_sum == debit_sum
|
|
351
|
+
|
|
352
|
+
result = AccountCheckingResult(
|
|
353
|
+
check_type="transaction_balance",
|
|
354
|
+
status=is_balanced,
|
|
355
|
+
details={
|
|
356
|
+
"event_id": event.id,
|
|
357
|
+
"event_type": event.event_type,
|
|
358
|
+
"credit_sum": float(credit_sum),
|
|
359
|
+
"debit_sum": float(debit_sum),
|
|
360
|
+
"difference": float(credit_sum - debit_sum),
|
|
361
|
+
"created_at": event.created_at.isoformat()
|
|
362
|
+
if event.created_at
|
|
363
|
+
else None,
|
|
364
|
+
"batch": batch_count,
|
|
365
|
+
},
|
|
366
|
+
)
|
|
367
|
+
results.append(result)
|
|
368
|
+
|
|
369
|
+
if not is_balanced:
|
|
370
|
+
logger.warning(
|
|
371
|
+
f"Transaction imbalance detected for event {event.id} ({event.event_type}). "
|
|
372
|
+
f"Credit: {credit_sum}, Debit: {debit_sum}"
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
logger.info(
|
|
376
|
+
f"Completed transaction balance check: processed {total_processed} events in {batch_count} batches"
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
return results
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
async def check_orphaned_transactions() -> list[AccountCheckingResult]:
|
|
383
|
+
"""Check for orphaned transactions that don't have a corresponding event.
|
|
384
|
+
|
|
385
|
+
Returns:
|
|
386
|
+
List of checking results
|
|
387
|
+
"""
|
|
388
|
+
# Create a new session for this function
|
|
389
|
+
async with get_session() as session:
|
|
390
|
+
# Find transactions with event_ids that don't exist in the events table
|
|
391
|
+
query = text("""
|
|
392
|
+
SELECT t.id, t.account_id, t.event_id, t.tx_type, t.credit_debit, t.change_amount, t.credit_type, t.created_at
|
|
393
|
+
FROM credit_transactions t
|
|
394
|
+
LEFT JOIN credit_events e ON t.event_id = e.id
|
|
395
|
+
WHERE e.id IS NULL
|
|
396
|
+
""")
|
|
397
|
+
|
|
398
|
+
result = await session.execute(query)
|
|
399
|
+
orphaned_txs = result.fetchall()
|
|
400
|
+
|
|
401
|
+
# Process orphaned transactions with a sleep to reduce database load
|
|
402
|
+
orphaned_tx_details = []
|
|
403
|
+
for tx in orphaned_txs[:100]: # Limit to first 100 for report size
|
|
404
|
+
# Sleep for 10ms to reduce database load
|
|
405
|
+
await asyncio.sleep(0.01)
|
|
406
|
+
|
|
407
|
+
# Add transaction details to the list
|
|
408
|
+
orphaned_tx_details.append(
|
|
409
|
+
{
|
|
410
|
+
"id": tx.id,
|
|
411
|
+
"account_id": tx.account_id,
|
|
412
|
+
"event_id": tx.event_id,
|
|
413
|
+
"tx_type": tx.tx_type,
|
|
414
|
+
"credit_debit": tx.credit_debit,
|
|
415
|
+
"change_amount": float(tx.change_amount),
|
|
416
|
+
"credit_type": tx.credit_type,
|
|
417
|
+
"created_at": tx.created_at.isoformat() if tx.created_at else None,
|
|
418
|
+
}
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
check_result = AccountCheckingResult(
|
|
422
|
+
check_type="orphaned_transactions",
|
|
423
|
+
status=(len(orphaned_txs) == 0),
|
|
424
|
+
details={
|
|
425
|
+
"orphaned_count": len(orphaned_txs),
|
|
426
|
+
"orphaned_transactions": orphaned_tx_details,
|
|
427
|
+
},
|
|
428
|
+
)
|
|
429
|
+
|
|
430
|
+
if orphaned_txs:
|
|
431
|
+
logger.warning(
|
|
432
|
+
f"Found {len(orphaned_txs)} orphaned transactions without corresponding events"
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
return [check_result]
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
async def check_orphaned_events() -> list[AccountCheckingResult]:
|
|
439
|
+
"""Check for orphaned events that don't have any transactions.
|
|
440
|
+
|
|
441
|
+
Returns:
|
|
442
|
+
List of checking results
|
|
443
|
+
"""
|
|
444
|
+
# Create a new session for this function
|
|
445
|
+
async with get_session() as session:
|
|
446
|
+
# Find events that don't have any transactions
|
|
447
|
+
query = text("""
|
|
448
|
+
SELECT e.id, e.event_type, e.account_id, e.total_amount, e.credit_type, e.created_at
|
|
449
|
+
FROM credit_events e
|
|
450
|
+
LEFT JOIN credit_transactions t ON e.id = t.event_id
|
|
451
|
+
WHERE t.id IS NULL
|
|
452
|
+
""")
|
|
453
|
+
|
|
454
|
+
result = await session.execute(query)
|
|
455
|
+
orphaned_events = result.fetchall()
|
|
456
|
+
|
|
457
|
+
if not orphaned_events:
|
|
458
|
+
return [
|
|
459
|
+
AccountCheckingResult(
|
|
460
|
+
check_type="orphaned_events",
|
|
461
|
+
status=True,
|
|
462
|
+
details={"message": "No orphaned events found"},
|
|
463
|
+
)
|
|
464
|
+
]
|
|
465
|
+
|
|
466
|
+
# If we found orphaned events, report them
|
|
467
|
+
orphaned_event_ids = [event.id for event in orphaned_events]
|
|
468
|
+
orphaned_event_details = []
|
|
469
|
+
for event in orphaned_events:
|
|
470
|
+
# Sleep for 10ms to reduce database load
|
|
471
|
+
await asyncio.sleep(0.01)
|
|
472
|
+
|
|
473
|
+
# Add event details to the list
|
|
474
|
+
orphaned_event_details.append(
|
|
475
|
+
{
|
|
476
|
+
"event_id": event.id,
|
|
477
|
+
"event_type": event.event_type,
|
|
478
|
+
"account_id": event.account_id,
|
|
479
|
+
"total_amount": float(event.total_amount),
|
|
480
|
+
"credit_type": event.credit_type,
|
|
481
|
+
"created_at": event.created_at.isoformat()
|
|
482
|
+
if event.created_at
|
|
483
|
+
else None,
|
|
484
|
+
}
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
logger.warning(
|
|
488
|
+
f"Found {len(orphaned_events)} orphaned events with no transactions: {orphaned_event_ids}"
|
|
489
|
+
)
|
|
490
|
+
|
|
491
|
+
return [
|
|
492
|
+
AccountCheckingResult(
|
|
493
|
+
check_type="orphaned_events",
|
|
494
|
+
status=False,
|
|
495
|
+
details={
|
|
496
|
+
"orphaned_count": len(orphaned_events),
|
|
497
|
+
"orphaned_events": orphaned_event_details,
|
|
498
|
+
},
|
|
499
|
+
)
|
|
500
|
+
]
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
async def check_total_credit_balance() -> list[AccountCheckingResult]:
|
|
504
|
+
"""Check if the sum of all free_credits, reward_credits, and credits across all accounts is 0.
|
|
505
|
+
|
|
506
|
+
This verifies that the overall credit system is balanced, with all credits accounted for.
|
|
507
|
+
|
|
508
|
+
Returns:
|
|
509
|
+
List of checking results
|
|
510
|
+
"""
|
|
511
|
+
# Create a new session for this function
|
|
512
|
+
async with get_session() as session:
|
|
513
|
+
# Query to sum all credit types across all accounts
|
|
514
|
+
query = text("""
|
|
515
|
+
SELECT
|
|
516
|
+
SUM(free_credits) as total_free_credits,
|
|
517
|
+
SUM(reward_credits) as total_reward_credits,
|
|
518
|
+
SUM(credits) as total_permanent_credits,
|
|
519
|
+
SUM(free_credits) + SUM(reward_credits) + SUM(credits) as grand_total
|
|
520
|
+
FROM credit_accounts
|
|
521
|
+
""")
|
|
522
|
+
|
|
523
|
+
result = await session.execute(query)
|
|
524
|
+
balance_data = result.fetchone()
|
|
525
|
+
|
|
526
|
+
total_free_credits = balance_data.total_free_credits or Decimal("0")
|
|
527
|
+
total_reward_credits = balance_data.total_reward_credits or Decimal("0")
|
|
528
|
+
total_permanent_credits = balance_data.total_permanent_credits or Decimal("0")
|
|
529
|
+
grand_total = balance_data.grand_total or Decimal("0")
|
|
530
|
+
|
|
531
|
+
# Check if the grand total is zero (or very close to zero due to potential floating point issues)
|
|
532
|
+
is_balanced = grand_total == Decimal("0")
|
|
533
|
+
|
|
534
|
+
# If not exactly zero but very close (due to potential rounding issues), log a warning but still consider it balanced
|
|
535
|
+
if not is_balanced and abs(grand_total) < Decimal("0.01"):
|
|
536
|
+
logger.warning(
|
|
537
|
+
f"Total credit balance is very close to zero but not exact: {grand_total}. "
|
|
538
|
+
f"This might be due to rounding issues."
|
|
539
|
+
)
|
|
540
|
+
is_balanced = True
|
|
541
|
+
|
|
542
|
+
result = AccountCheckingResult(
|
|
543
|
+
check_type="total_credit_balance",
|
|
544
|
+
status=is_balanced,
|
|
545
|
+
details={
|
|
546
|
+
"total_free_credits": float(total_free_credits),
|
|
547
|
+
"total_reward_credits": float(total_reward_credits),
|
|
548
|
+
"total_permanent_credits": float(total_permanent_credits),
|
|
549
|
+
"grand_total": float(grand_total),
|
|
550
|
+
},
|
|
551
|
+
)
|
|
552
|
+
|
|
553
|
+
if not is_balanced:
|
|
554
|
+
logger.warning(
|
|
555
|
+
f"Total credit balance inconsistency detected. System is not balanced. "
|
|
556
|
+
f"Total: {grand_total} (Free: {total_free_credits}, Reward: {total_reward_credits}, "
|
|
557
|
+
f"Permanent: {total_permanent_credits})"
|
|
558
|
+
)
|
|
559
|
+
|
|
560
|
+
return [result]
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
async def check_transaction_total_balance() -> list[AccountCheckingResult]:
|
|
564
|
+
"""Check if the total credit and debit amounts in the CreditTransaction table are balanced.
|
|
565
|
+
|
|
566
|
+
This verifies that across all transactions in the system, the total credits equal the total debits.
|
|
567
|
+
|
|
568
|
+
Returns:
|
|
569
|
+
List of checking results
|
|
570
|
+
"""
|
|
571
|
+
# Create a new session for this function
|
|
572
|
+
async with get_session() as session:
|
|
573
|
+
# Query to sum all credit and debit transactions
|
|
574
|
+
query = text("""
|
|
575
|
+
SELECT
|
|
576
|
+
SUM(CASE WHEN credit_debit = 'credit' THEN change_amount ELSE 0 END) as total_credits,
|
|
577
|
+
SUM(CASE WHEN credit_debit = 'debit' THEN change_amount ELSE 0 END) as total_debits
|
|
578
|
+
FROM credit_transactions
|
|
579
|
+
""")
|
|
580
|
+
|
|
581
|
+
result = await session.execute(query)
|
|
582
|
+
balance_data = result.fetchone()
|
|
583
|
+
|
|
584
|
+
total_credits = balance_data.total_credits or Decimal("0")
|
|
585
|
+
total_debits = balance_data.total_debits or Decimal("0")
|
|
586
|
+
difference = total_credits - total_debits
|
|
587
|
+
|
|
588
|
+
# Check if credits and debits are balanced (difference should be zero)
|
|
589
|
+
is_balanced = difference == Decimal("0")
|
|
590
|
+
|
|
591
|
+
# If not exactly zero but very close (due to potential rounding issues), log a warning but still consider it balanced
|
|
592
|
+
if not is_balanced and abs(difference) < Decimal("0.001"):
|
|
593
|
+
logger.warning(
|
|
594
|
+
f"Transaction total balance is very close to zero but not exact: {difference}. "
|
|
595
|
+
f"This might be due to rounding issues."
|
|
596
|
+
)
|
|
597
|
+
is_balanced = True
|
|
598
|
+
|
|
599
|
+
result = AccountCheckingResult(
|
|
600
|
+
check_type="transaction_total_balance",
|
|
601
|
+
status=is_balanced,
|
|
602
|
+
details={
|
|
603
|
+
"total_credits": float(total_credits),
|
|
604
|
+
"total_debits": float(total_debits),
|
|
605
|
+
"difference": float(difference),
|
|
606
|
+
},
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
if not is_balanced:
|
|
610
|
+
logger.warning(
|
|
611
|
+
f"Transaction total balance inconsistency detected. System is not balanced. "
|
|
612
|
+
f"Credits: {total_credits}, Debits: {total_debits}, Difference: {difference}"
|
|
613
|
+
)
|
|
614
|
+
|
|
615
|
+
return [result]
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
async def run_quick_checks() -> dict[str, list[AccountCheckingResult]]:
|
|
619
|
+
"""Run quick account checking procedures and return results.
|
|
620
|
+
|
|
621
|
+
These checks are designed to be fast and can be run frequently.
|
|
622
|
+
|
|
623
|
+
Returns:
|
|
624
|
+
Dictionary mapping check names to their results
|
|
625
|
+
"""
|
|
626
|
+
logger.info("Starting quick account checking procedures")
|
|
627
|
+
|
|
628
|
+
results = {}
|
|
629
|
+
# Quick checks don't need a session at this level as each function creates its own session
|
|
630
|
+
results["transaction_balance"] = await check_transaction_balance()
|
|
631
|
+
results["orphaned_transactions"] = await check_orphaned_transactions()
|
|
632
|
+
results["orphaned_events"] = await check_orphaned_events()
|
|
633
|
+
results["total_credit_balance"] = await check_total_credit_balance()
|
|
634
|
+
results["transaction_total_balance"] = await check_transaction_total_balance()
|
|
635
|
+
|
|
636
|
+
# Log summary
|
|
637
|
+
all_passed = True
|
|
638
|
+
failed_count = 0
|
|
639
|
+
for check_name, check_results in results.items():
|
|
640
|
+
check_failed_count = sum(1 for result in check_results if not result.status)
|
|
641
|
+
failed_count += check_failed_count
|
|
642
|
+
|
|
643
|
+
if check_failed_count > 0:
|
|
644
|
+
logger.warning(
|
|
645
|
+
f"{check_name}: {check_failed_count} of {len(check_results)} checks failed"
|
|
646
|
+
)
|
|
647
|
+
all_passed = False
|
|
648
|
+
else:
|
|
649
|
+
logger.info(f"{check_name}: All {len(check_results)} checks passed")
|
|
650
|
+
|
|
651
|
+
if all_passed:
|
|
652
|
+
logger.info("All quick account checks passed successfully")
|
|
653
|
+
else:
|
|
654
|
+
logger.warning(
|
|
655
|
+
f"Quick account checking summary: {failed_count} checks failed - see logs for details"
|
|
656
|
+
)
|
|
657
|
+
|
|
658
|
+
# Create a summary message with color based on status
|
|
659
|
+
total_checks = sum(len(check_results) for check_results in results.values())
|
|
660
|
+
|
|
661
|
+
if all_passed:
|
|
662
|
+
color = "good" # Green color
|
|
663
|
+
title = "✅ Quick Account Checking Completed Successfully"
|
|
664
|
+
text = f"All {total_checks} quick account checks passed successfully."
|
|
665
|
+
notify = "" # No notification needed for success
|
|
666
|
+
else:
|
|
667
|
+
color = "danger" # Red color
|
|
668
|
+
title = "❌ Quick Account Checking Found Issues"
|
|
669
|
+
text = f"Quick account checking found {failed_count} issues out of {total_checks} checks."
|
|
670
|
+
notify = "<!channel> " # Notify channel for failures
|
|
671
|
+
|
|
672
|
+
# Create attachments with check details
|
|
673
|
+
attachments = [{"color": color, "title": title, "text": text, "fields": []}]
|
|
674
|
+
|
|
675
|
+
# Add fields for each check type
|
|
676
|
+
for check_name, check_results in results.items():
|
|
677
|
+
check_failed_count = sum(1 for result in check_results if not result.status)
|
|
678
|
+
check_status = (
|
|
679
|
+
"✅ Passed"
|
|
680
|
+
if check_failed_count == 0
|
|
681
|
+
else f"❌ Failed ({check_failed_count} issues)"
|
|
682
|
+
)
|
|
683
|
+
|
|
684
|
+
attachments[0]["fields"].append(
|
|
685
|
+
{
|
|
686
|
+
"title": check_name.replace("_", " ").title(),
|
|
687
|
+
"value": check_status,
|
|
688
|
+
"short": True,
|
|
689
|
+
}
|
|
690
|
+
)
|
|
691
|
+
|
|
692
|
+
# Send the message
|
|
693
|
+
send_slack_message(
|
|
694
|
+
message=f"{notify}Quick Account Checking Results", attachments=attachments
|
|
695
|
+
)
|
|
696
|
+
|
|
697
|
+
return results
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
async def run_slow_checks() -> dict[str, list[AccountCheckingResult]]:
|
|
701
|
+
"""Run slow account checking procedures and return results.
|
|
702
|
+
|
|
703
|
+
These checks are more resource-intensive and should be run less frequently.
|
|
704
|
+
|
|
705
|
+
Returns:
|
|
706
|
+
Dictionary mapping check names to their results
|
|
707
|
+
"""
|
|
708
|
+
logger.info("Starting slow account checking procedures")
|
|
709
|
+
|
|
710
|
+
results = {}
|
|
711
|
+
# Slow checks don't need a session at this level as each function creates its own session
|
|
712
|
+
results["account_balance"] = await check_account_balance_consistency()
|
|
713
|
+
|
|
714
|
+
# Log summary
|
|
715
|
+
all_passed = True
|
|
716
|
+
failed_count = 0
|
|
717
|
+
for check_name, check_results in results.items():
|
|
718
|
+
check_failed_count = sum(1 for result in check_results if not result.status)
|
|
719
|
+
failed_count += check_failed_count
|
|
720
|
+
|
|
721
|
+
if check_failed_count > 0:
|
|
722
|
+
logger.warning(
|
|
723
|
+
f"{check_name}: {check_failed_count} of {len(check_results)} checks failed"
|
|
724
|
+
)
|
|
725
|
+
all_passed = False
|
|
726
|
+
else:
|
|
727
|
+
logger.info(f"{check_name}: All {len(check_results)} checks passed")
|
|
728
|
+
|
|
729
|
+
if all_passed:
|
|
730
|
+
logger.info("All slow account checks passed successfully")
|
|
731
|
+
else:
|
|
732
|
+
logger.warning(
|
|
733
|
+
f"Slow account checking summary: {failed_count} checks failed - see logs for details"
|
|
734
|
+
)
|
|
735
|
+
|
|
736
|
+
# Send summary to Slack
|
|
737
|
+
|
|
738
|
+
# Create a summary message with color based on status
|
|
739
|
+
total_checks = sum(len(check_results) for check_results in results.values())
|
|
740
|
+
|
|
741
|
+
if all_passed:
|
|
742
|
+
color = "good" # Green color
|
|
743
|
+
title = "✅ Slow Account Checking Completed Successfully"
|
|
744
|
+
text = f"All {total_checks} slow account checks passed successfully."
|
|
745
|
+
notify = "" # No notification needed for success
|
|
746
|
+
else:
|
|
747
|
+
color = "danger" # Red color
|
|
748
|
+
title = "❌ Slow Account Checking Found Issues"
|
|
749
|
+
text = f"Slow account checking found {failed_count} issues out of {total_checks} checks."
|
|
750
|
+
notify = "<!channel> " # Notify channel for failures
|
|
751
|
+
|
|
752
|
+
# Create attachments with check details
|
|
753
|
+
attachments = [{"color": color, "title": title, "text": text, "fields": []}]
|
|
754
|
+
|
|
755
|
+
# Add fields for each check type
|
|
756
|
+
for check_name, check_results in results.items():
|
|
757
|
+
check_failed_count = sum(1 for result in check_results if not result.status)
|
|
758
|
+
check_status = (
|
|
759
|
+
"✅ Passed"
|
|
760
|
+
if check_failed_count == 0
|
|
761
|
+
else f"❌ Failed ({check_failed_count} issues)"
|
|
762
|
+
)
|
|
763
|
+
|
|
764
|
+
attachments[0]["fields"].append(
|
|
765
|
+
{
|
|
766
|
+
"title": check_name.replace("_", " ").title(),
|
|
767
|
+
"value": check_status,
|
|
768
|
+
"short": True,
|
|
769
|
+
}
|
|
770
|
+
)
|
|
771
|
+
|
|
772
|
+
# If there are failed account balance checks, add details of first 5 failed accounts
|
|
773
|
+
if "account_balance" in results:
|
|
774
|
+
failed_account_results = [r for r in results["account_balance"] if not r.status]
|
|
775
|
+
if failed_account_results:
|
|
776
|
+
# Add a separate attachment for failed account details
|
|
777
|
+
failed_details_text = "First 5 inconsistent accounts:\n"
|
|
778
|
+
for i, result in enumerate(failed_account_results[:5]):
|
|
779
|
+
details = result.details
|
|
780
|
+
failed_details_text += (
|
|
781
|
+
f"{i + 1}. Account {details['account_id']} ({details['owner_type']}:{details['owner_id']}):\n"
|
|
782
|
+
f" • Total: {details['current_total_balance']:.4f} vs {details['expected_total_balance']:.4f} (diff: {details['total_balance_difference']:.4f})\n"
|
|
783
|
+
f" • Free: {details['free_credits']:.4f} vs {details['expected_free_credits']:.4f} (diff: {details['free_credits_difference']:.4f})\n"
|
|
784
|
+
f" • Reward: {details['reward_credits']:.4f} vs {details['expected_reward_credits']:.4f} (diff: {details['reward_credits_difference']:.4f})\n"
|
|
785
|
+
f" • Permanent: {details['permanent_credits']:.4f} vs {details['expected_permanent_credits']:.4f} (diff: {details['permanent_credits_difference']:.4f})\n"
|
|
786
|
+
)
|
|
787
|
+
|
|
788
|
+
attachments.append(
|
|
789
|
+
{
|
|
790
|
+
"color": "warning",
|
|
791
|
+
"title": "Account Balance Inconsistencies Details",
|
|
792
|
+
"text": failed_details_text,
|
|
793
|
+
"mrkdwn_in": ["text"],
|
|
794
|
+
}
|
|
795
|
+
)
|
|
796
|
+
|
|
797
|
+
# Send the message
|
|
798
|
+
send_slack_message(
|
|
799
|
+
message=f"{notify}Slow Account Checking Results", attachments=attachments
|
|
800
|
+
)
|
|
801
|
+
|
|
802
|
+
return results
|