django-cfg 1.2.29__py3-none-any.whl → 1.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- django_cfg/__init__.py +1 -1
- django_cfg/apps/api/health/views.py +4 -2
- django_cfg/apps/knowbase/config/settings.py +16 -15
- django_cfg/apps/payments/README.md +326 -0
- django_cfg/apps/payments/admin/__init__.py +20 -9
- django_cfg/apps/payments/admin/api_keys_admin.py +521 -237
- django_cfg/apps/payments/admin/balance_admin.py +592 -297
- django_cfg/apps/payments/admin/currencies_admin.py +600 -108
- django_cfg/apps/payments/admin/filters.py +306 -199
- django_cfg/apps/payments/admin/payments_admin.py +470 -64
- django_cfg/apps/payments/admin/subscriptions_admin.py +578 -128
- django_cfg/apps/payments/admin_interface/__init__.py +18 -0
- django_cfg/apps/payments/admin_interface/templates/payments/base.html +162 -0
- django_cfg/apps/payments/admin_interface/templates/payments/components/dev_tool_card.html +38 -0
- django_cfg/apps/payments/admin_interface/templates/payments/components/loading_spinner.html +16 -0
- django_cfg/apps/payments/admin_interface/templates/payments/components/notification.html +27 -0
- django_cfg/apps/payments/admin_interface/templates/payments/components/provider_card.html +86 -0
- django_cfg/apps/payments/admin_interface/templates/payments/components/status_card.html +39 -0
- django_cfg/apps/payments/admin_interface/templates/payments/currency_converter.html +382 -0
- django_cfg/apps/payments/admin_interface/templates/payments/payment_dashboard.html +300 -0
- django_cfg/apps/payments/admin_interface/templates/payments/payment_form.html +303 -0
- django_cfg/apps/payments/admin_interface/templates/payments/payment_list.html +382 -0
- django_cfg/apps/payments/admin_interface/templates/payments/payment_status.html +500 -0
- django_cfg/apps/payments/admin_interface/templates/payments/webhook_dashboard.html +594 -0
- django_cfg/apps/payments/admin_interface/views/__init__.py +23 -0
- django_cfg/apps/payments/admin_interface/views/payment_views.py +259 -0
- django_cfg/apps/payments/admin_interface/views/webhook_dashboard.py +37 -0
- django_cfg/apps/payments/apps.py +34 -9
- django_cfg/apps/payments/config/__init__.py +28 -51
- django_cfg/apps/payments/config/constance/__init__.py +22 -0
- django_cfg/apps/payments/config/constance/config_service.py +123 -0
- django_cfg/apps/payments/config/constance/fields.py +69 -0
- django_cfg/apps/payments/config/constance/settings.py +160 -0
- django_cfg/apps/payments/config/django_cfg_integration.py +202 -0
- django_cfg/apps/payments/config/helpers.py +130 -0
- django_cfg/apps/payments/management/__init__.py +1 -3
- django_cfg/apps/payments/management/commands/__init__.py +1 -3
- django_cfg/apps/payments/management/commands/manage_currencies.py +381 -0
- django_cfg/apps/payments/management/commands/manage_providers.py +408 -0
- django_cfg/apps/payments/middleware/__init__.py +3 -1
- django_cfg/apps/payments/middleware/api_access.py +329 -222
- django_cfg/apps/payments/middleware/rate_limiting.py +343 -163
- django_cfg/apps/payments/middleware/usage_tracking.py +250 -238
- django_cfg/apps/payments/migrations/0001_initial.py +708 -536
- django_cfg/apps/payments/models/__init__.py +16 -20
- django_cfg/apps/payments/models/api_keys.py +121 -43
- django_cfg/apps/payments/models/balance.py +150 -115
- django_cfg/apps/payments/models/base.py +68 -15
- django_cfg/apps/payments/models/currencies.py +207 -67
- django_cfg/apps/payments/models/managers/__init__.py +44 -0
- django_cfg/apps/payments/models/managers/api_key_managers.py +329 -0
- django_cfg/apps/payments/models/managers/balance_managers.py +599 -0
- django_cfg/apps/payments/models/managers/currency_managers.py +385 -0
- django_cfg/apps/payments/models/managers/payment_managers.py +511 -0
- django_cfg/apps/payments/models/managers/subscription_managers.py +641 -0
- django_cfg/apps/payments/models/payments.py +235 -284
- django_cfg/apps/payments/models/subscriptions.py +257 -177
- django_cfg/apps/payments/models/tariffs.py +147 -40
- django_cfg/apps/payments/services/__init__.py +209 -56
- django_cfg/apps/payments/services/cache/__init__.py +6 -6
- django_cfg/apps/payments/services/cache/{simple_cache.py → cache_service.py} +112 -12
- django_cfg/apps/payments/services/core/__init__.py +10 -6
- django_cfg/apps/payments/services/core/balance_service.py +435 -360
- django_cfg/apps/payments/services/core/base.py +166 -0
- django_cfg/apps/payments/services/core/currency_service.py +478 -0
- django_cfg/apps/payments/services/core/payment_service.py +344 -468
- django_cfg/apps/payments/services/core/subscription_service.py +425 -484
- django_cfg/apps/payments/services/core/webhook_service.py +410 -0
- django_cfg/apps/payments/services/integrations/__init__.py +29 -0
- django_cfg/apps/payments/services/integrations/ngrok_service.py +47 -0
- django_cfg/apps/payments/services/integrations/providers_config.py +107 -0
- django_cfg/apps/payments/services/providers/__init__.py +9 -14
- django_cfg/apps/payments/services/providers/base.py +232 -71
- django_cfg/apps/payments/services/providers/nowpayments.py +404 -219
- django_cfg/apps/payments/services/providers/registry.py +429 -80
- django_cfg/apps/payments/services/types/__init__.py +78 -0
- django_cfg/apps/payments/services/types/data.py +177 -0
- django_cfg/apps/payments/services/types/requests.py +150 -0
- django_cfg/apps/payments/services/types/responses.py +156 -0
- django_cfg/apps/payments/services/types/webhooks.py +232 -0
- django_cfg/apps/payments/signals/__init__.py +33 -8
- django_cfg/apps/payments/signals/api_key_signals.py +211 -130
- django_cfg/apps/payments/signals/balance_signals.py +174 -0
- django_cfg/apps/payments/signals/payment_signals.py +129 -98
- django_cfg/apps/payments/signals/subscription_signals.py +195 -143
- django_cfg/apps/payments/static/payments/css/components.css +380 -0
- django_cfg/apps/payments/static/payments/css/dashboard.css +188 -0
- django_cfg/apps/payments/static/payments/js/components.js +545 -0
- django_cfg/apps/payments/static/payments/js/utils.js +412 -0
- django_cfg/apps/payments/templatetags/__init__.py +1 -1
- django_cfg/apps/payments/templatetags/payment_tags.py +466 -0
- django_cfg/apps/payments/urls.py +46 -47
- django_cfg/apps/payments/urls_admin.py +49 -0
- django_cfg/apps/payments/views/api/__init__.py +101 -0
- django_cfg/apps/payments/views/api/api_keys.py +387 -0
- django_cfg/apps/payments/views/api/balances.py +381 -0
- django_cfg/apps/payments/views/api/base.py +298 -0
- django_cfg/apps/payments/views/api/currencies.py +402 -0
- django_cfg/apps/payments/views/api/payments.py +415 -0
- django_cfg/apps/payments/views/api/subscriptions.py +475 -0
- django_cfg/apps/payments/views/api/webhooks.py +476 -0
- django_cfg/apps/payments/views/serializers/__init__.py +99 -0
- django_cfg/apps/payments/views/serializers/api_keys.py +424 -0
- django_cfg/apps/payments/views/serializers/balances.py +300 -0
- django_cfg/apps/payments/views/serializers/currencies.py +335 -0
- django_cfg/apps/payments/views/serializers/payments.py +387 -0
- django_cfg/apps/payments/views/serializers/subscriptions.py +429 -0
- django_cfg/apps/payments/views/serializers/webhooks.py +137 -0
- django_cfg/apps/tasks/urls.py +0 -2
- django_cfg/apps/tasks/urls_admin.py +14 -0
- django_cfg/apps/urls.py +4 -4
- django_cfg/config.py +1 -1
- django_cfg/core/config.py +75 -4
- django_cfg/core/generation.py +25 -4
- django_cfg/core/integration/README.md +363 -0
- django_cfg/core/integration/__init__.py +47 -0
- django_cfg/core/integration/commands_collector.py +239 -0
- django_cfg/core/integration/display/__init__.py +15 -0
- django_cfg/core/integration/display/base.py +157 -0
- django_cfg/core/integration/display/ngrok.py +164 -0
- django_cfg/core/integration/display/startup.py +815 -0
- django_cfg/core/integration/url_integration.py +123 -0
- django_cfg/core/integration/version_checker.py +160 -0
- django_cfg/management/commands/auto_generate.py +4 -0
- django_cfg/management/commands/check_settings.py +6 -0
- django_cfg/management/commands/clear_constance.py +5 -2
- django_cfg/management/commands/create_token.py +6 -0
- django_cfg/management/commands/list_urls.py +6 -0
- django_cfg/management/commands/migrate_all.py +6 -0
- django_cfg/management/commands/migrator.py +3 -0
- django_cfg/management/commands/rundramatiq.py +6 -0
- django_cfg/management/commands/runserver_ngrok.py +51 -29
- django_cfg/management/commands/script.py +6 -0
- django_cfg/management/commands/show_config.py +12 -2
- django_cfg/management/commands/show_urls.py +4 -0
- django_cfg/management/commands/superuser.py +6 -0
- django_cfg/management/commands/task_clear.py +4 -1
- django_cfg/management/commands/task_status.py +3 -1
- django_cfg/management/commands/test_email.py +3 -0
- django_cfg/management/commands/test_telegram.py +6 -0
- django_cfg/management/commands/test_twilio.py +6 -0
- django_cfg/management/commands/tree.py +6 -0
- django_cfg/management/commands/validate_config.py +155 -149
- django_cfg/models/constance.py +31 -11
- django_cfg/models/payments.py +175 -498
- django_cfg/modules/django_currency/__init__.py +16 -11
- django_cfg/modules/django_currency/clients/__init__.py +4 -4
- django_cfg/modules/django_currency/clients/coinpaprika_client.py +289 -0
- django_cfg/modules/django_currency/clients/yahoo_client.py +157 -0
- django_cfg/modules/django_currency/core/__init__.py +1 -7
- django_cfg/modules/django_currency/core/converter.py +18 -23
- django_cfg/modules/django_currency/core/models.py +122 -11
- django_cfg/modules/django_currency/database/__init__.py +4 -4
- django_cfg/modules/django_currency/database/database_loader.py +190 -309
- django_cfg/modules/django_logger.py +160 -146
- django_cfg/modules/django_unfold/dashboard.py +65 -12
- django_cfg/registry/core.py +1 -0
- django_cfg/template_archive/django_sample.zip +0 -0
- django_cfg/templates/admin/components/action_grid.html +9 -9
- django_cfg/templates/admin/components/metric_card.html +5 -5
- django_cfg/templates/admin/components/status_badge.html +2 -2
- django_cfg/templates/admin/layouts/dashboard_with_tabs.html +152 -24
- django_cfg/templates/admin/snippets/components/quick_actions.html +3 -3
- django_cfg/templates/admin/snippets/components/system_health.html +1 -1
- django_cfg/templates/admin/snippets/tabs/overview_tab.html +49 -52
- django_cfg/utils/smart_defaults.py +222 -571
- django_cfg/utils/toolkit.py +51 -11
- {django_cfg-1.2.29.dist-info → django_cfg-1.3.1.dist-info}/METADATA +5 -4
- {django_cfg-1.2.29.dist-info → django_cfg-1.3.1.dist-info}/RECORD +172 -182
- django_cfg/apps/payments/__init__.py +0 -8
- django_cfg/apps/payments/admin/tariffs_admin.py +0 -199
- django_cfg/apps/payments/config/module.py +0 -70
- django_cfg/apps/payments/config/providers.py +0 -105
- django_cfg/apps/payments/config/settings.py +0 -96
- django_cfg/apps/payments/config/utils.py +0 -52
- django_cfg/apps/payments/decorators.py +0 -291
- django_cfg/apps/payments/management/commands/README.md +0 -178
- django_cfg/apps/payments/management/commands/currency_stats.py +0 -323
- django_cfg/apps/payments/management/commands/populate_currencies.py +0 -246
- django_cfg/apps/payments/management/commands/update_currencies.py +0 -336
- django_cfg/apps/payments/managers/__init__.py +0 -22
- django_cfg/apps/payments/managers/api_key_manager.py +0 -35
- django_cfg/apps/payments/managers/balance_manager.py +0 -361
- django_cfg/apps/payments/managers/currency_manager.py +0 -83
- django_cfg/apps/payments/managers/payment_manager.py +0 -44
- django_cfg/apps/payments/managers/subscription_manager.py +0 -37
- django_cfg/apps/payments/managers/tariff_manager.py +0 -29
- django_cfg/apps/payments/models/events.py +0 -73
- django_cfg/apps/payments/serializers/__init__.py +0 -56
- django_cfg/apps/payments/serializers/api_keys.py +0 -51
- django_cfg/apps/payments/serializers/balance.py +0 -59
- django_cfg/apps/payments/serializers/currencies.py +0 -55
- django_cfg/apps/payments/serializers/payments.py +0 -62
- django_cfg/apps/payments/serializers/subscriptions.py +0 -71
- django_cfg/apps/payments/serializers/tariffs.py +0 -56
- django_cfg/apps/payments/services/billing/__init__.py +0 -8
- django_cfg/apps/payments/services/cache/base.py +0 -30
- django_cfg/apps/payments/services/core/fallback_service.py +0 -432
- django_cfg/apps/payments/services/internal_types.py +0 -297
- django_cfg/apps/payments/services/middleware/__init__.py +0 -8
- django_cfg/apps/payments/services/monitoring/__init__.py +0 -22
- django_cfg/apps/payments/services/monitoring/api_schemas.py +0 -222
- django_cfg/apps/payments/services/monitoring/provider_health.py +0 -372
- django_cfg/apps/payments/services/providers/cryptapi.py +0 -273
- django_cfg/apps/payments/services/providers/cryptomus.py +0 -311
- django_cfg/apps/payments/services/security/__init__.py +0 -34
- django_cfg/apps/payments/services/security/error_handler.py +0 -637
- django_cfg/apps/payments/services/security/payment_notifications.py +0 -342
- django_cfg/apps/payments/services/security/webhook_validator.py +0 -475
- django_cfg/apps/payments/services/validators/__init__.py +0 -8
- django_cfg/apps/payments/static/payments/css/payments.css +0 -340
- django_cfg/apps/payments/static/payments/js/notifications.js +0 -202
- django_cfg/apps/payments/static/payments/js/payment-utils.js +0 -318
- django_cfg/apps/payments/static/payments/js/theme.js +0 -86
- django_cfg/apps/payments/tasks/__init__.py +0 -12
- django_cfg/apps/payments/tasks/webhook_processing.py +0 -177
- django_cfg/apps/payments/templates/payments/base.html +0 -182
- django_cfg/apps/payments/templates/payments/components/payment_card.html +0 -201
- django_cfg/apps/payments/templates/payments/components/payment_qr_code.html +0 -109
- django_cfg/apps/payments/templates/payments/components/progress_bar.html +0 -36
- django_cfg/apps/payments/templates/payments/components/provider_stats.html +0 -40
- django_cfg/apps/payments/templates/payments/components/status_badge.html +0 -27
- django_cfg/apps/payments/templates/payments/components/status_overview.html +0 -144
- django_cfg/apps/payments/templates/payments/dashboard.html +0 -346
- django_cfg/apps/payments/templatetags/payments_tags.py +0 -315
- django_cfg/apps/payments/urls_templates.py +0 -52
- django_cfg/apps/payments/utils/__init__.py +0 -45
- django_cfg/apps/payments/utils/billing_utils.py +0 -342
- django_cfg/apps/payments/utils/config_utils.py +0 -245
- django_cfg/apps/payments/utils/middleware_utils.py +0 -228
- django_cfg/apps/payments/utils/validation_utils.py +0 -94
- django_cfg/apps/payments/views/__init__.py +0 -62
- django_cfg/apps/payments/views/api_key_views.py +0 -164
- django_cfg/apps/payments/views/balance_views.py +0 -75
- django_cfg/apps/payments/views/currency_views.py +0 -111
- django_cfg/apps/payments/views/payment_views.py +0 -149
- django_cfg/apps/payments/views/subscription_views.py +0 -135
- django_cfg/apps/payments/views/tariff_views.py +0 -131
- django_cfg/apps/payments/views/templates/__init__.py +0 -25
- django_cfg/apps/payments/views/templates/ajax.py +0 -312
- django_cfg/apps/payments/views/templates/base.py +0 -204
- django_cfg/apps/payments/views/templates/dashboard.py +0 -60
- django_cfg/apps/payments/views/templates/payment_detail.py +0 -102
- django_cfg/apps/payments/views/templates/payment_management.py +0 -164
- django_cfg/apps/payments/views/templates/qr_code.py +0 -174
- django_cfg/apps/payments/views/templates/stats.py +0 -240
- django_cfg/apps/payments/views/templates/utils.py +0 -181
- django_cfg/apps/payments/views/webhook_views.py +0 -266
- django_cfg/apps/payments/viewsets.py +0 -65
- django_cfg/core/integration.py +0 -160
- django_cfg/modules/django_currency/clients/coingecko_client.py +0 -257
- django_cfg/modules/django_currency/clients/yfinance_client.py +0 -246
- django_cfg/template_archive/.gitignore +0 -1
- django_cfg/template_archive/__init__.py +0 -0
- django_cfg/urls.py +0 -33
- {django_cfg-1.2.29.dist-info → django_cfg-1.3.1.dist-info}/WHEEL +0 -0
- {django_cfg-1.2.29.dist-info → django_cfg-1.3.1.dist-info}/entry_points.txt +0 -0
- {django_cfg-1.2.29.dist-info → django_cfg-1.3.1.dist-info}/licenses/LICENSE +0 -0
@@ -1,5 +1,5 @@
|
|
1
1
|
"""
|
2
|
-
Database loader for populating currency data
|
2
|
+
Database loader for populating currency data using Yahoo Finance + CoinPaprika.
|
3
3
|
"""
|
4
4
|
|
5
5
|
import logging
|
@@ -11,9 +11,8 @@ from dataclasses import dataclass
|
|
11
11
|
from pydantic import BaseModel, Field, validator
|
12
12
|
from cachetools import TTLCache
|
13
13
|
|
14
|
-
#
|
15
|
-
from
|
16
|
-
import yfinance as yf
|
14
|
+
# Our new clients
|
15
|
+
from ..clients import YahooFinanceClient, CoinPaprikaClient
|
17
16
|
|
18
17
|
logger = logging.getLogger(__name__)
|
19
18
|
|
@@ -22,14 +21,14 @@ logger = logging.getLogger(__name__)
|
|
22
21
|
# PYDANTIC MODELS
|
23
22
|
# ============================================================================
|
24
23
|
|
25
|
-
class
|
26
|
-
"""Single coin information from
|
27
|
-
id: str = Field(description="
|
24
|
+
class CoinPaprikaCoinInfo(BaseModel):
|
25
|
+
"""Single coin information from CoinPaprika."""
|
26
|
+
id: str = Field(description="CoinPaprika coin ID")
|
28
27
|
symbol: str = Field(description="Currency symbol (e.g., BTC)")
|
29
28
|
name: str = Field(description="Full coin name")
|
30
29
|
|
31
30
|
|
32
|
-
class
|
31
|
+
class YahooFinanceCurrencyInfo(BaseModel):
|
33
32
|
"""Single fiat currency information."""
|
34
33
|
code: str = Field(description="Currency code (e.g., USD)")
|
35
34
|
name: str = Field(description="Full currency name")
|
@@ -45,404 +44,285 @@ class CurrencyRateInfo(BaseModel):
|
|
45
44
|
decimal_places: int = Field(default=2, description="Decimal places")
|
46
45
|
usd_rate: float = Field(description="Rate to USD")
|
47
46
|
min_payment_amount: float = Field(default=1.0, description="Minimum payment amount")
|
48
|
-
is_active: bool = Field(default=True, description="Is currency active")
|
49
47
|
|
50
48
|
|
51
|
-
class DatabaseLoaderConfig(BaseModel):
|
52
|
-
"""Configuration for database loader."""
|
53
|
-
|
54
|
-
# Rate limiting
|
55
|
-
coingecko_delay: float = Field(default=1.5, description="Delay between CoinGecko requests (seconds)")
|
56
|
-
yfinance_delay: float = Field(default=0.5, description="Delay between YFinance requests (seconds)")
|
57
|
-
max_requests_per_minute: int = Field(default=30, description="Max requests per minute")
|
58
|
-
|
59
|
-
# Limits
|
60
|
-
max_cryptocurrencies: int = Field(default=500, description="Max cryptocurrencies to load")
|
61
|
-
max_fiat_currencies: int = Field(default=50, description="Max fiat currencies to load")
|
62
|
-
|
63
|
-
# Filtering
|
64
|
-
min_market_cap_usd: float = Field(default=1000000, description="Minimum market cap in USD")
|
65
|
-
exclude_stablecoins: bool = Field(default=False, description="Exclude stablecoins")
|
66
|
-
|
67
|
-
# Cache
|
68
|
-
cache_ttl_hours: int = Field(default=24, description="Cache TTL in hours")
|
69
|
-
|
70
|
-
|
71
|
-
# ============================================================================
|
72
|
-
# RATE LIMITER
|
73
|
-
# ============================================================================
|
74
|
-
|
75
|
-
@dataclass
|
76
49
|
class RateLimiter:
|
77
|
-
"""Simple rate limiter
|
50
|
+
"""Simple rate limiter for API calls."""
|
78
51
|
|
79
52
|
def __init__(self, max_requests_per_minute: int = 30):
|
80
53
|
self.max_requests = max_requests_per_minute
|
81
|
-
self.
|
82
|
-
|
83
|
-
|
84
|
-
def wait_if_needed(self, delay: float = 1.0):
|
54
|
+
self.request_times = []
|
55
|
+
|
56
|
+
def __call__(self):
|
85
57
|
"""Wait if necessary to respect rate limits."""
|
86
|
-
|
58
|
+
now = time.time()
|
87
59
|
|
88
|
-
# Remove
|
89
|
-
self.
|
90
|
-
if current_time - req_time < 60]
|
60
|
+
# Remove requests older than 1 minute
|
61
|
+
self.request_times = [t for t in self.request_times if now - t < 60]
|
91
62
|
|
92
|
-
#
|
93
|
-
if len(self.
|
94
|
-
sleep_time = 60 - (
|
63
|
+
# If we're at the limit, wait
|
64
|
+
if len(self.request_times) >= self.max_requests:
|
65
|
+
sleep_time = 60 - (now - self.request_times[0]) + 1
|
95
66
|
if sleep_time > 0:
|
96
|
-
logger.
|
67
|
+
logger.debug(f"Rate limiting: sleeping for {sleep_time:.1f}s")
|
97
68
|
time.sleep(sleep_time)
|
98
|
-
|
99
|
-
|
100
|
-
time_since_last = current_time - self.last_request_time
|
101
|
-
if time_since_last < delay:
|
102
|
-
sleep_time = delay - time_since_last
|
103
|
-
logger.debug(f"Rate limiting: sleeping for {sleep_time:.2f}s")
|
104
|
-
time.sleep(sleep_time)
|
105
|
-
|
106
|
-
# Record this request
|
107
|
-
self.requests.append(time.time())
|
108
|
-
self.last_request_time = time.time()
|
69
|
+
|
70
|
+
self.request_times.append(now)
|
109
71
|
|
110
72
|
|
111
|
-
|
112
|
-
|
113
|
-
|
73
|
+
class DatabaseLoaderConfig(BaseModel):
|
74
|
+
"""Configuration for database loader."""
|
75
|
+
|
76
|
+
yahoo_delay: float = Field(default=1.0, description="Delay between Yahoo requests")
|
77
|
+
coinpaprika_delay: float = Field(default=0.5, description="Delay between CoinPaprika requests")
|
78
|
+
max_requests_per_minute: int = Field(default=30, description="Max requests per minute")
|
79
|
+
max_cryptocurrencies: int = Field(default=500, description="Max cryptocurrencies to load")
|
80
|
+
max_fiat_currencies: int = Field(default=50, description="Max fiat currencies to load")
|
81
|
+
min_market_cap_usd: float = Field(default=1000000, description="Minimum market cap for crypto")
|
82
|
+
exclude_stablecoins: bool = Field(default=False, description="Exclude stablecoins")
|
83
|
+
cache_ttl_hours: int = Field(default=24, description="Cache TTL in hours")
|
84
|
+
|
114
85
|
|
115
86
|
class CurrencyDatabaseLoader:
|
116
87
|
"""
|
117
|
-
|
118
|
-
|
119
|
-
Features:
|
120
|
-
- Rate limiting to prevent API throttling
|
121
|
-
- Configurable limits on number of currencies
|
122
|
-
- Market cap filtering for cryptocurrencies
|
123
|
-
- Caching to avoid repeated API calls
|
124
|
-
- Type safety with Pydantic models
|
88
|
+
Database loader for populating currency data from Yahoo Finance and CoinPaprika.
|
125
89
|
"""
|
126
90
|
|
127
91
|
def __init__(self, config: DatabaseLoaderConfig = None):
|
128
92
|
"""Initialize the database loader."""
|
129
93
|
self.config = config or DatabaseLoaderConfig()
|
130
94
|
|
131
|
-
# Initialize
|
132
|
-
self.
|
95
|
+
# Initialize clients
|
96
|
+
self.yahoo = YahooFinanceClient(cache_ttl=self.config.cache_ttl_hours * 3600)
|
97
|
+
self.coinpaprika = CoinPaprikaClient(cache_ttl=self.config.cache_ttl_hours * 3600)
|
133
98
|
|
134
99
|
# Rate limiters
|
135
|
-
self.
|
136
|
-
self.
|
100
|
+
self.yahoo_limiter = RateLimiter(self.config.max_requests_per_minute)
|
101
|
+
self.coinpaprika_limiter = RateLimiter(self.config.max_requests_per_minute)
|
137
102
|
|
138
|
-
#
|
103
|
+
# Caches
|
139
104
|
cache_ttl = self.config.cache_ttl_hours * 3600
|
140
|
-
self.crypto_cache = TTLCache(maxsize=
|
141
|
-
self.fiat_cache = TTLCache(maxsize=
|
105
|
+
self.crypto_cache = TTLCache(maxsize=1000, ttl=cache_ttl)
|
106
|
+
self.fiat_cache = TTLCache(maxsize=100, ttl=cache_ttl)
|
142
107
|
|
143
108
|
logger.info(f"Initialized CurrencyDatabaseLoader with config: {self.config}")
|
144
109
|
|
145
|
-
def
|
110
|
+
def get_fiat_currency_list(self) -> List[YahooFinanceCurrencyInfo]:
|
146
111
|
"""
|
147
|
-
Get list of
|
112
|
+
Get list of supported fiat currencies.
|
148
113
|
|
149
114
|
Returns:
|
150
|
-
List of
|
151
|
-
"""
|
152
|
-
cache_key = "crypto_list"
|
153
|
-
if cache_key in self.crypto_cache:
|
154
|
-
logger.debug("Retrieved cryptocurrency list from cache")
|
155
|
-
return self.crypto_cache[cache_key]
|
156
|
-
|
157
|
-
logger.info("Fetching cryptocurrency list from CoinGecko...")
|
158
|
-
|
159
|
-
try:
|
160
|
-
# Get coins with market data for filtering
|
161
|
-
self.coingecko_limiter.wait_if_needed(self.config.coingecko_delay)
|
162
|
-
|
163
|
-
coins_markets = self.coingecko.get_coins_markets(
|
164
|
-
vs_currency='usd',
|
165
|
-
order='market_cap_desc',
|
166
|
-
per_page=self.config.max_cryptocurrencies,
|
167
|
-
page=1,
|
168
|
-
sparkline=False
|
169
|
-
)
|
170
|
-
|
171
|
-
cryptocurrencies = []
|
172
|
-
for coin in coins_markets:
|
173
|
-
# Filter by market cap
|
174
|
-
market_cap = coin.get('market_cap', 0) or 0
|
175
|
-
if market_cap < self.config.min_market_cap_usd:
|
176
|
-
continue
|
177
|
-
|
178
|
-
# Filter stablecoins if requested
|
179
|
-
if self.config.exclude_stablecoins:
|
180
|
-
categories = coin.get('categories', []) or []
|
181
|
-
if any('stablecoin' in str(cat).lower() for cat in categories):
|
182
|
-
continue
|
183
|
-
|
184
|
-
crypto_info = CoinGeckoCoinInfo(
|
185
|
-
id=coin['id'],
|
186
|
-
symbol=coin['symbol'].upper(),
|
187
|
-
name=coin['name']
|
188
|
-
)
|
189
|
-
cryptocurrencies.append(crypto_info)
|
190
|
-
|
191
|
-
logger.info(f"Loaded {len(cryptocurrencies)} cryptocurrencies")
|
192
|
-
self.crypto_cache[cache_key] = cryptocurrencies
|
193
|
-
return cryptocurrencies
|
194
|
-
|
195
|
-
except Exception as e:
|
196
|
-
logger.error(f"Failed to fetch cryptocurrency list: {e}")
|
197
|
-
raise
|
198
|
-
|
199
|
-
def get_fiat_currency_list(self) -> List[YFinanceCurrencyInfo]:
|
115
|
+
List of fiat currency info objects
|
200
116
|
"""
|
201
|
-
|
117
|
+
cache_key = "fiat_currencies"
|
202
118
|
|
203
|
-
Returns:
|
204
|
-
List of fiat currency information
|
205
|
-
"""
|
206
|
-
cache_key = "fiat_list"
|
207
119
|
if cache_key in self.fiat_cache:
|
208
|
-
logger.debug("Retrieved fiat
|
120
|
+
logger.debug("Retrieved fiat currencies from cache")
|
209
121
|
return self.fiat_cache[cache_key]
|
210
122
|
|
211
|
-
|
212
|
-
|
213
|
-
# Major fiat currencies with their symbols
|
214
|
-
fiat_currencies_data = [
|
215
|
-
("USD", "US Dollar", "$"),
|
216
|
-
("EUR", "Euro", "€"),
|
217
|
-
("GBP", "British Pound", "£"),
|
218
|
-
("JPY", "Japanese Yen", "¥"),
|
219
|
-
("CNY", "Chinese Yuan", "¥"),
|
220
|
-
("KRW", "South Korean Won", "₩"),
|
221
|
-
("CAD", "Canadian Dollar", "C$"),
|
222
|
-
("AUD", "Australian Dollar", "A$"),
|
223
|
-
("CHF", "Swiss Franc", "₣"),
|
224
|
-
("RUB", "Russian Ruble", "₽"),
|
225
|
-
("BRL", "Brazilian Real", "R$"),
|
226
|
-
("INR", "Indian Rupee", "₹"),
|
227
|
-
("MXN", "Mexican Peso", "$"),
|
228
|
-
("SGD", "Singapore Dollar", "S$"),
|
229
|
-
("HKD", "Hong Kong Dollar", "HK$"),
|
230
|
-
("SEK", "Swedish Krona", "kr"),
|
231
|
-
("NOK", "Norwegian Krone", "kr"),
|
232
|
-
("DKK", "Danish Krone", "kr"),
|
233
|
-
("PLN", "Polish Zloty", "zł"),
|
234
|
-
("CZK", "Czech Koruna", "Kč"),
|
235
|
-
("HUF", "Hungarian Forint", "Ft"),
|
236
|
-
("TRY", "Turkish Lira", "₺"),
|
237
|
-
("ZAR", "South African Rand", "R"),
|
238
|
-
("THB", "Thai Baht", "฿"),
|
239
|
-
("MYR", "Malaysian Ringgit", "RM"),
|
240
|
-
("PHP", "Philippine Peso", "₱"),
|
241
|
-
("IDR", "Indonesian Rupiah", "Rp"),
|
242
|
-
("VND", "Vietnamese Dong", "₫"),
|
243
|
-
("TWD", "Taiwan Dollar", "NT$"),
|
244
|
-
("NZD", "New Zealand Dollar", "NZ$")
|
245
|
-
]
|
123
|
+
# Get supported currencies from Yahoo
|
124
|
+
supported_currencies = self.yahoo.get_all_supported_currencies()
|
246
125
|
|
126
|
+
# Convert to our format and limit count
|
247
127
|
fiat_currencies = []
|
248
|
-
for code, name
|
249
|
-
|
128
|
+
for code, name in list(supported_currencies.items())[:self.config.max_fiat_currencies]:
|
129
|
+
currency_info = YahooFinanceCurrencyInfo(
|
250
130
|
code=code,
|
251
131
|
name=name,
|
252
|
-
symbol=
|
132
|
+
symbol="" # Yahoo doesn't provide symbols
|
253
133
|
)
|
254
|
-
fiat_currencies.append(
|
134
|
+
fiat_currencies.append(currency_info)
|
255
135
|
|
256
|
-
logger.info(f"Built list of {len(fiat_currencies)} fiat currencies")
|
257
136
|
self.fiat_cache[cache_key] = fiat_currencies
|
137
|
+
logger.info(f"Loaded {len(fiat_currencies)} fiat currencies")
|
138
|
+
|
258
139
|
return fiat_currencies
|
259
140
|
|
260
|
-
def
|
141
|
+
def get_cryptocurrency_list(self) -> List[CoinPaprikaCoinInfo]:
|
261
142
|
"""
|
262
|
-
Get
|
143
|
+
Get list of supported cryptocurrencies from CoinPaprika.
|
263
144
|
|
264
|
-
Args:
|
265
|
-
currencies: List of currency codes/IDs
|
266
|
-
vs_currency: Quote currency (default: usd)
|
267
|
-
|
268
145
|
Returns:
|
269
|
-
|
146
|
+
List of cryptocurrency info objects
|
270
147
|
"""
|
271
|
-
|
272
|
-
return {}
|
148
|
+
cache_key = "crypto_currencies"
|
273
149
|
|
274
|
-
|
150
|
+
if cache_key in self.crypto_cache:
|
151
|
+
logger.debug("Retrieved cryptocurrencies from cache")
|
152
|
+
return self.crypto_cache[cache_key]
|
275
153
|
|
276
154
|
try:
|
277
|
-
#
|
278
|
-
|
279
|
-
all_rates = {}
|
155
|
+
# Get all tickers from CoinPaprika
|
156
|
+
all_tickers = self.coinpaprika._fetch_all_tickers()
|
280
157
|
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
158
|
+
crypto_currencies = []
|
159
|
+
for symbol, data in all_tickers.items():
|
160
|
+
# Skip if no price data
|
161
|
+
if data.get('price_usd') is None:
|
162
|
+
continue
|
285
163
|
|
286
|
-
#
|
287
|
-
|
164
|
+
# Apply market cap filter if needed
|
165
|
+
# Note: CoinPaprika doesn't provide market cap in tickers endpoint
|
166
|
+
# We'd need to use a different endpoint for that
|
288
167
|
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
168
|
+
coin_info = CoinPaprikaCoinInfo(
|
169
|
+
id=data['id'],
|
170
|
+
symbol=symbol,
|
171
|
+
name=data['name']
|
293
172
|
)
|
173
|
+
crypto_currencies.append(coin_info)
|
294
174
|
|
295
|
-
#
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
175
|
+
# Limit count
|
176
|
+
if len(crypto_currencies) >= self.config.max_cryptocurrencies:
|
177
|
+
break
|
178
|
+
|
179
|
+
self.crypto_cache[cache_key] = crypto_currencies
|
180
|
+
logger.info(f"Loaded {len(crypto_currencies)} cryptocurrencies")
|
301
181
|
|
302
|
-
|
303
|
-
return all_rates
|
182
|
+
return crypto_currencies
|
304
183
|
|
305
184
|
except Exception as e:
|
306
|
-
logger.error(f"Failed to
|
307
|
-
|
185
|
+
logger.error(f"Failed to get cryptocurrency list: {e}")
|
186
|
+
return []
|
308
187
|
|
309
|
-
def
|
188
|
+
def get_currency_rates(self, currency_ids: List[str], quote: str = 'usd') -> Dict[str, float]:
|
310
189
|
"""
|
311
|
-
Get
|
190
|
+
Get current rates for multiple cryptocurrencies.
|
312
191
|
|
313
192
|
Args:
|
314
|
-
|
315
|
-
|
193
|
+
currency_ids: List of currency symbols
|
194
|
+
quote: Quote currency (usually 'usd')
|
316
195
|
|
317
196
|
Returns:
|
318
|
-
|
197
|
+
Dict mapping currency ID to rate
|
319
198
|
"""
|
199
|
+
self.coinpaprika_limiter()
|
200
|
+
|
320
201
|
try:
|
321
|
-
|
202
|
+
rates = {}
|
203
|
+
all_tickers = self.coinpaprika._fetch_all_tickers()
|
322
204
|
|
323
|
-
|
324
|
-
|
205
|
+
for currency_id in currency_ids:
|
206
|
+
if currency_id.upper() in all_tickers:
|
207
|
+
price = all_tickers[currency_id.upper()].get('price_usd')
|
208
|
+
if price is not None:
|
209
|
+
rates[currency_id] = price
|
325
210
|
|
326
|
-
|
327
|
-
|
211
|
+
logger.debug(f"Retrieved rates for {len(rates)} currencies")
|
212
|
+
return rates
|
328
213
|
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
214
|
+
except Exception as e:
|
215
|
+
logger.error(f"Failed to get currency rates: {e}")
|
216
|
+
return {}
|
217
|
+
|
218
|
+
def get_fiat_rate(self, base: str, quote: str) -> Optional[float]:
|
219
|
+
"""
|
220
|
+
Get exchange rate for fiat currency pair.
|
221
|
+
|
222
|
+
Args:
|
223
|
+
base: Base currency code
|
224
|
+
quote: Quote currency code
|
340
225
|
|
226
|
+
Returns:
|
227
|
+
Exchange rate or None if failed
|
228
|
+
"""
|
229
|
+
# Handle same currency
|
230
|
+
if base.upper() == quote.upper():
|
231
|
+
return 1.0
|
232
|
+
|
233
|
+
self.yahoo_limiter()
|
234
|
+
|
235
|
+
try:
|
236
|
+
rate_obj = self.yahoo.fetch_rate(base, quote)
|
237
|
+
return rate_obj.rate
|
341
238
|
except Exception as e:
|
342
|
-
logger.debug(f"Failed to get fiat rate
|
239
|
+
logger.debug(f"Failed to get fiat rate {base}/{quote}: {e}")
|
343
240
|
return None
|
344
241
|
|
345
242
|
def build_currency_database_data(self) -> List[CurrencyRateInfo]:
|
346
243
|
"""
|
347
|
-
Build complete currency data
|
244
|
+
Build complete currency database data combining fiat and crypto.
|
348
245
|
|
349
246
|
Returns:
|
350
|
-
List of currency rate
|
247
|
+
List of currency rate info objects
|
351
248
|
"""
|
352
|
-
|
249
|
+
currencies = []
|
353
250
|
|
354
|
-
|
355
|
-
|
356
|
-
# 1. Get fiat currencies
|
357
|
-
logger.info("Processing fiat currencies...")
|
251
|
+
# Get fiat currencies
|
252
|
+
logger.info("Loading fiat currencies...")
|
358
253
|
fiat_currencies = self.get_fiat_currency_list()
|
359
254
|
|
360
255
|
for fiat in fiat_currencies:
|
361
|
-
# Get USD rate
|
362
|
-
if fiat.code ==
|
256
|
+
# Get USD rate
|
257
|
+
if fiat.code == 'USD':
|
363
258
|
usd_rate = 1.0
|
364
259
|
else:
|
365
|
-
usd_rate = self.get_fiat_rate(fiat.code,
|
260
|
+
usd_rate = self.get_fiat_rate(fiat.code, 'USD')
|
366
261
|
if usd_rate is None:
|
367
|
-
logger.warning(f"Could not get USD rate for {fiat.code}
|
262
|
+
logger.warning(f"Could not get USD rate for {fiat.code}")
|
368
263
|
continue
|
369
264
|
|
370
265
|
currency_info = CurrencyRateInfo(
|
371
266
|
code=fiat.code,
|
372
267
|
name=fiat.name,
|
373
|
-
symbol=fiat.symbol,
|
268
|
+
symbol=fiat.symbol or "$" if fiat.code == "USD" else "",
|
374
269
|
currency_type="fiat",
|
375
270
|
decimal_places=2,
|
376
271
|
usd_rate=usd_rate,
|
377
|
-
min_payment_amount=1.0
|
378
|
-
is_active=True
|
272
|
+
min_payment_amount=1.0
|
379
273
|
)
|
380
|
-
|
381
|
-
|
382
|
-
logger.debug(f"Added fiat currency: {fiat.code} = {usd_rate} USD")
|
274
|
+
currencies.append(currency_info)
|
383
275
|
|
384
|
-
#
|
385
|
-
logger.info("
|
276
|
+
# Get cryptocurrencies
|
277
|
+
logger.info("Loading cryptocurrencies...")
|
386
278
|
crypto_currencies = self.get_cryptocurrency_list()
|
387
279
|
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
for crypto in crypto_currencies:
|
393
|
-
if crypto.id not in crypto_rates:
|
394
|
-
logger.warning(f"No USD rate found for {crypto.symbol}, skipping")
|
395
|
-
continue
|
280
|
+
if crypto_currencies:
|
281
|
+
# Get rates for all cryptos
|
282
|
+
crypto_symbols = [crypto.symbol for crypto in crypto_currencies]
|
283
|
+
rates = self.get_currency_rates(crypto_symbols, 'usd')
|
396
284
|
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
min_payment_amount=min_payment,
|
423
|
-
is_active=True
|
424
|
-
)
|
425
|
-
all_currencies.append(currency_info)
|
426
|
-
|
427
|
-
logger.debug(f"Added cryptocurrency: {crypto.symbol} = {usd_rate} USD")
|
285
|
+
for crypto in crypto_currencies:
|
286
|
+
if crypto.symbol in rates:
|
287
|
+
usd_rate = rates[crypto.symbol]
|
288
|
+
|
289
|
+
# Calculate appropriate decimal places based on USD value
|
290
|
+
if usd_rate >= 1:
|
291
|
+
decimal_places = 2
|
292
|
+
elif usd_rate >= 0.01:
|
293
|
+
decimal_places = 4
|
294
|
+
else:
|
295
|
+
decimal_places = 8
|
296
|
+
|
297
|
+
# Calculate minimum payment amount (roughly $1 worth)
|
298
|
+
min_payment = max(1.0 / usd_rate, 0.000001)
|
299
|
+
|
300
|
+
currency_info = CurrencyRateInfo(
|
301
|
+
code=crypto.symbol,
|
302
|
+
name=crypto.name,
|
303
|
+
symbol="", # We don't have crypto symbols
|
304
|
+
currency_type="crypto",
|
305
|
+
decimal_places=decimal_places,
|
306
|
+
usd_rate=usd_rate,
|
307
|
+
min_payment_amount=min_payment
|
308
|
+
)
|
309
|
+
currencies.append(currency_info)
|
428
310
|
|
429
|
-
logger.info(f"Built
|
430
|
-
|
431
|
-
|
432
|
-
return all_currencies
|
311
|
+
logger.info(f"Built database data for {len(currencies)} currencies")
|
312
|
+
return currencies
|
433
313
|
|
434
314
|
def get_statistics(self) -> Dict[str, int]:
|
435
|
-
"""Get statistics
|
315
|
+
"""Get loader statistics."""
|
436
316
|
fiat_count = len(self.get_fiat_currency_list())
|
437
317
|
crypto_count = len(self.get_cryptocurrency_list())
|
438
318
|
|
439
319
|
return {
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
320
|
+
'total_fiat_currencies': fiat_count,
|
321
|
+
'total_cryptocurrencies': crypto_count,
|
322
|
+
'total_currencies': fiat_count + crypto_count,
|
323
|
+
'max_cryptocurrencies': self.config.max_cryptocurrencies,
|
324
|
+
'max_fiat_currencies': self.config.max_fiat_currencies,
|
325
|
+
'min_market_cap_usd': self.config.min_market_cap_usd
|
446
326
|
}
|
447
327
|
|
448
328
|
|
@@ -454,7 +334,8 @@ def create_database_loader(
|
|
454
334
|
max_cryptocurrencies: int = 500,
|
455
335
|
max_fiat_currencies: int = 50,
|
456
336
|
min_market_cap_usd: float = 1000000,
|
457
|
-
|
337
|
+
yahoo_delay: float = 1.0,
|
338
|
+
coinpaprika_delay: float = 0.5
|
458
339
|
) -> CurrencyDatabaseLoader:
|
459
340
|
"""
|
460
341
|
Create a configured database loader.
|
@@ -463,7 +344,8 @@ def create_database_loader(
|
|
463
344
|
max_cryptocurrencies: Maximum number of cryptocurrencies to load
|
464
345
|
max_fiat_currencies: Maximum number of fiat currencies to load
|
465
346
|
min_market_cap_usd: Minimum market cap for cryptocurrencies
|
466
|
-
|
347
|
+
yahoo_delay: Delay between Yahoo Finance requests
|
348
|
+
coinpaprika_delay: Delay between CoinPaprika requests
|
467
349
|
|
468
350
|
Returns:
|
469
351
|
Configured CurrencyDatabaseLoader instance
|
@@ -472,24 +354,24 @@ def create_database_loader(
|
|
472
354
|
max_cryptocurrencies=max_cryptocurrencies,
|
473
355
|
max_fiat_currencies=max_fiat_currencies,
|
474
356
|
min_market_cap_usd=min_market_cap_usd,
|
475
|
-
|
357
|
+
yahoo_delay=yahoo_delay,
|
358
|
+
coinpaprika_delay=coinpaprika_delay
|
476
359
|
)
|
477
|
-
|
478
360
|
return CurrencyDatabaseLoader(config)
|
479
361
|
|
480
362
|
|
481
363
|
def load_currencies_to_database_format() -> List[Dict]:
|
482
364
|
"""
|
483
|
-
|
365
|
+
Load currencies and convert to database format.
|
484
366
|
|
485
367
|
Returns:
|
486
|
-
List of dictionaries ready for
|
368
|
+
List of currency dictionaries ready for database insertion
|
487
369
|
"""
|
488
370
|
loader = create_database_loader()
|
489
371
|
currencies = loader.build_currency_database_data()
|
490
372
|
|
491
|
-
# Convert to
|
492
|
-
|
373
|
+
# Convert to dict format
|
374
|
+
result = []
|
493
375
|
for currency in currencies:
|
494
376
|
currency_dict = {
|
495
377
|
'code': currency.code,
|
@@ -499,9 +381,8 @@ def load_currencies_to_database_format() -> List[Dict]:
|
|
499
381
|
'decimal_places': currency.decimal_places,
|
500
382
|
'usd_rate': currency.usd_rate,
|
501
383
|
'min_payment_amount': currency.min_payment_amount,
|
502
|
-
'is_active': currency.is_active,
|
503
384
|
'rate_updated_at': datetime.now()
|
504
385
|
}
|
505
|
-
|
386
|
+
result.append(currency_dict)
|
506
387
|
|
507
|
-
return
|
388
|
+
return result
|