opencode-pyneruntime 6.6.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opencode_pyneruntime-6.6.4.dist-info/METADATA +281 -0
- opencode_pyneruntime-6.6.4.dist-info/RECORD +261 -0
- opencode_pyneruntime-6.6.4.dist-info/WHEEL +5 -0
- opencode_pyneruntime-6.6.4.dist-info/entry_points.txt +6 -0
- opencode_pyneruntime-6.6.4.dist-info/licenses/LICENSE +201 -0
- opencode_pyneruntime-6.6.4.dist-info/licenses/NOTICE +21 -0
- opencode_pyneruntime-6.6.4.dist-info/top_level.txt +1 -0
- pynecore/__init__.py +6 -0
- pynecore/cli/__init__.py +2 -0
- pynecore/cli/app.py +238 -0
- pynecore/cli/commands/__init__.py +343 -0
- pynecore/cli/commands/benchmark.py +186 -0
- pynecore/cli/commands/compile.py +198 -0
- pynecore/cli/commands/data.py +857 -0
- pynecore/cli/commands/debug.py +63 -0
- pynecore/cli/commands/optimize.py +956 -0
- pynecore/cli/commands/plugin.py +242 -0
- pynecore/cli/commands/run.py +2006 -0
- pynecore/cli/pluggable.py +132 -0
- pynecore/cli/utils/__init__.py +0 -0
- pynecore/cli/utils/api_error_handler.py +168 -0
- pynecore/cli/utils/broker_picker.py +330 -0
- pynecore/cli/utils/error_hook.py +28 -0
- pynecore/cli/utils/keyreader.py +178 -0
- pynecore/cli/utils/provider_picker.py +19 -0
- pynecore/cli/utils/symbol_browser.py +1149 -0
- pynecore/core/__init__.py +0 -0
- pynecore/core/aggregator.py +257 -0
- pynecore/core/bar_magnifier.py +168 -0
- pynecore/core/broker/__init__.py +64 -0
- pynecore/core/broker/defaults.py +113 -0
- pynecore/core/broker/disappearance.py +927 -0
- pynecore/core/broker/emulator.py +345 -0
- pynecore/core/broker/exceptions.py +346 -0
- pynecore/core/broker/idempotency.py +401 -0
- pynecore/core/broker/intent_builder.py +334 -0
- pynecore/core/broker/journal.py +1785 -0
- pynecore/core/broker/models.py +1600 -0
- pynecore/core/broker/native_failsafe_manager.py +1436 -0
- pynecore/core/broker/one_way_emulator.py +1128 -0
- pynecore/core/broker/position.py +787 -0
- pynecore/core/broker/run_identity.py +126 -0
- pynecore/core/broker/software_entry_stop_engine.py +351 -0
- pynecore/core/broker/software_partial_bracket_engine.py +1379 -0
- pynecore/core/broker/spot_inventory.py +1327 -0
- pynecore/core/broker/storage.py +2655 -0
- pynecore/core/broker/store_helpers.py +2161 -0
- pynecore/core/broker/sync_engine.py +16070 -0
- pynecore/core/broker/validation.py +382 -0
- pynecore/core/class_property.py +7 -0
- pynecore/core/config.py +392 -0
- pynecore/core/csv_file.py +547 -0
- pynecore/core/currency.py +262 -0
- pynecore/core/data_converter.py +1002 -0
- pynecore/core/datetime.py +296 -0
- pynecore/core/download_info.py +71 -0
- pynecore/core/download_runner.py +274 -0
- pynecore/core/htf_aggregator.py +181 -0
- pynecore/core/import_hook.py +358 -0
- pynecore/core/instance_state.py +494 -0
- pynecore/core/live_ltf_collector.py +442 -0
- pynecore/core/live_ltf_window.py +189 -0
- pynecore/core/live_runner.py +1347 -0
- pynecore/core/module_property.py +26 -0
- pynecore/core/ohlcv_file.py +1888 -0
- pynecore/core/overload.py +371 -0
- pynecore/core/pine_cast.py +113 -0
- pynecore/core/pine_export.py +95 -0
- pynecore/core/pine_method.py +244 -0
- pynecore/core/pine_range.py +86 -0
- pynecore/core/pine_udt.py +69 -0
- pynecore/core/plugin/__init__.py +394 -0
- pynecore/core/plugin/broker.py +781 -0
- pynecore/core/plugin/cli.py +96 -0
- pynecore/core/plugin/live_provider.py +208 -0
- pynecore/core/plugin/provider.py +331 -0
- pynecore/core/provider_string.py +148 -0
- pynecore/core/random.py +40 -0
- pynecore/core/resampler.py +686 -0
- pynecore/core/safe_convert.py +64 -0
- pynecore/core/script.py +1011 -0
- pynecore/core/script_runner.py +3202 -0
- pynecore/core/security.py +1749 -0
- pynecore/core/security_process.py +1253 -0
- pynecore/core/security_shm.py +456 -0
- pynecore/core/series.py +417 -0
- pynecore/core/strategy_stats.py +669 -0
- pynecore/core/symbol_map.py +134 -0
- pynecore/core/syminfo.py +505 -0
- pynecore/core/viz.py +591 -0
- pynecore/lib/__init__.py +1771 -0
- pynecore/lib/_fixnan.py +32 -0
- pynecore/lib/_math_stateful.py +202 -0
- pynecore/lib/_timeframe_change.py +101 -0
- pynecore/lib/adjustment.py +6 -0
- pynecore/lib/alert.py +39 -0
- pynecore/lib/alert.pyi +14 -0
- pynecore/lib/array.py +1051 -0
- pynecore/lib/barmerge.py +60 -0
- pynecore/lib/barstate.py +30 -0
- pynecore/lib/box.py +415 -0
- pynecore/lib/chart.py +128 -0
- pynecore/lib/color.py +152 -0
- pynecore/lib/color.pyi +50 -0
- pynecore/lib/currency.py +62 -0
- pynecore/lib/dayofweek.py +36 -0
- pynecore/lib/dayofweek.pyi +18 -0
- pynecore/lib/display.py +8 -0
- pynecore/lib/dividends.py +9 -0
- pynecore/lib/earnings.py +11 -0
- pynecore/lib/extend.py +6 -0
- pynecore/lib/font.py +5 -0
- pynecore/lib/footprint.py +79 -0
- pynecore/lib/format.py +11 -0
- pynecore/lib/hline.py +67 -0
- pynecore/lib/hline.pyi +24 -0
- pynecore/lib/label.py +409 -0
- pynecore/lib/line.py +433 -0
- pynecore/lib/linefill.py +93 -0
- pynecore/lib/location.py +11 -0
- pynecore/lib/log.py +362 -0
- pynecore/lib/map.py +150 -0
- pynecore/lib/math.py +385 -0
- pynecore/lib/matrix.py +708 -0
- pynecore/lib/order.py +8 -0
- pynecore/lib/pivotpointtype.py +8 -0
- pynecore/lib/plot.py +95 -0
- pynecore/lib/plot.pyi +33 -0
- pynecore/lib/polyline.py +91 -0
- pynecore/lib/position.py +15 -0
- pynecore/lib/request.py +281 -0
- pynecore/lib/runtime.py +5 -0
- pynecore/lib/scale.py +9 -0
- pynecore/lib/session.py +267 -0
- pynecore/lib/session.pyi +12 -0
- pynecore/lib/shape.py +18 -0
- pynecore/lib/size.py +12 -0
- pynecore/lib/splits.py +4 -0
- pynecore/lib/strategy/__init__.py +4778 -0
- pynecore/lib/strategy/closedtrades.py +347 -0
- pynecore/lib/strategy/closedtrades.pyi +53 -0
- pynecore/lib/strategy/commission.py +9 -0
- pynecore/lib/strategy/direction.py +9 -0
- pynecore/lib/strategy/oca.py +13 -0
- pynecore/lib/strategy/opentrades.py +281 -0
- pynecore/lib/strategy/opentrades.pyi +49 -0
- pynecore/lib/strategy/risk.py +109 -0
- pynecore/lib/string.py +649 -0
- pynecore/lib/syminfo.py +84 -0
- pynecore/lib/ta.py +2230 -0
- pynecore/lib/table.py +290 -0
- pynecore/lib/text.py +17 -0
- pynecore/lib/ticker.py +207 -0
- pynecore/lib/timeframe.py +293 -0
- pynecore/lib/volume_row.py +67 -0
- pynecore/lib/xloc.py +4 -0
- pynecore/lib/yloc.py +5 -0
- pynecore/providers/__init__.py +0 -0
- pynecore/providers/ccxt.py +664 -0
- pynecore/providers/replay.py +187 -0
- pynecore/pynesys/__init__.py +0 -0
- pynecore/pynesys/api.py +498 -0
- pynecore/pynesys/compiler.py +112 -0
- pynecore/standalone.py +99 -0
- pynecore/testing/__init__.py +1 -0
- pynecore/testing/broker_lab/__init__.py +41 -0
- pynecore/testing/broker_lab/__main__.py +5 -0
- pynecore/testing/broker_lab/cli.py +87 -0
- pynecore/testing/broker_lab/generate.py +47 -0
- pynecore/testing/broker_lab/model.py +84 -0
- pynecore/testing/broker_lab/reference.py +645 -0
- pynecore/testing/broker_lab/runner.py +372 -0
- pynecore/testing/broker_lab/scheduler.py +50 -0
- pynecore/testing/broker_lab/subprocess.py +73 -0
- pynecore/transformers/__init__.py +0 -0
- pynecore/transformers/builtin_shadow.py +136 -0
- pynecore/transformers/closure_arguments_transformer.py +428 -0
- pynecore/transformers/display_rewrite.py +140 -0
- pynecore/transformers/dynamic_default.py +147 -0
- pynecore/transformers/function_isolation.py +757 -0
- pynecore/transformers/import_lifter.py +61 -0
- pynecore/transformers/import_normalizer.py +328 -0
- pynecore/transformers/inline_series_hoist.py +178 -0
- pynecore/transformers/input_transformer.py +175 -0
- pynecore/transformers/lib_series.py +201 -0
- pynecore/transformers/locations.py +70 -0
- pynecore/transformers/module_properties.json +3387 -0
- pynecore/transformers/module_property.py +221 -0
- pynecore/transformers/ne_guard.py +70 -0
- pynecore/transformers/persistent.py +320 -0
- pynecore/transformers/persistent_series.py +76 -0
- pynecore/transformers/safe_convert_transformer.py +97 -0
- pynecore/transformers/safe_division_transformer.py +95 -0
- pynecore/transformers/script_requirements.py +308 -0
- pynecore/transformers/security.py +752 -0
- pynecore/transformers/security_instantiation.py +274 -0
- pynecore/transformers/series.py +275 -0
- pynecore/transformers/slot_layout.py +381 -0
- pynecore/transformers/type_checking_stripper.py +25 -0
- pynecore/transformers/unused_series_detector.py +267 -0
- pynecore/types/__init__.py +21 -0
- pynecore/types/alert.py +5 -0
- pynecore/types/barmerge.py +5 -0
- pynecore/types/base.py +39 -0
- pynecore/types/box.py +37 -0
- pynecore/types/chart.py +17 -0
- pynecore/types/color.py +107 -0
- pynecore/types/currency.py +5 -0
- pynecore/types/datetime.py +6 -0
- pynecore/types/display.py +5 -0
- pynecore/types/dividends.py +5 -0
- pynecore/types/earnings.py +5 -0
- pynecore/types/extend.py +5 -0
- pynecore/types/font.py +5 -0
- pynecore/types/footprint.py +41 -0
- pynecore/types/format.py +5 -0
- pynecore/types/hline.py +24 -0
- pynecore/types/ib_persistent.py +8 -0
- pynecore/types/ib_persistent.pyi +10 -0
- pynecore/types/label.py +35 -0
- pynecore/types/line.py +32 -0
- pynecore/types/linefill.py +13 -0
- pynecore/types/location.py +5 -0
- pynecore/types/matrix.py +999 -0
- pynecore/types/na.py +237 -0
- pynecore/types/na.pyi +83 -0
- pynecore/types/ohlcv.py +12 -0
- pynecore/types/order.py +5 -0
- pynecore/types/persistent.py +8 -0
- pynecore/types/persistent.pyi +13 -0
- pynecore/types/pine_types.py +11 -0
- pynecore/types/pine_types.pyi +15 -0
- pynecore/types/pivotpointtype.py +5 -0
- pynecore/types/plot.py +12 -0
- pynecore/types/plot_meta.py +60 -0
- pynecore/types/polyline.py +40 -0
- pynecore/types/position.py +5 -0
- pynecore/types/scale.py +5 -0
- pynecore/types/script_type.py +15 -0
- pynecore/types/series.py +23 -0
- pynecore/types/series.pyi +19 -0
- pynecore/types/session.py +35 -0
- pynecore/types/shape.py +5 -0
- pynecore/types/size.py +5 -0
- pynecore/types/source.py +33 -0
- pynecore/types/splits.py +5 -0
- pynecore/types/strategy.py +45 -0
- pynecore/types/table.py +87 -0
- pynecore/types/text.py +13 -0
- pynecore/types/type_checker.py +7 -0
- pynecore/types/type_checker.pyi +48 -0
- pynecore/types/volume_row.py +36 -0
- pynecore/types/weekdays.py +11 -0
- pynecore/types/xloc.py +5 -0
- pynecore/types/yloc.py +5 -0
- pynecore/utils/__init__.py +0 -0
- pynecore/utils/file_utils.py +50 -0
- pynecore/utils/rich/__init__.py +0 -0
- pynecore/utils/rich/date_column.py +25 -0
- pynecore/utils/sequence_view.py +92 -0
- pynecore/utils/stdlib_checker.py +17 -0
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import sys
|
|
3
|
+
from zoneinfo import ZoneInfo
|
|
4
|
+
from datetime import datetime, UTC
|
|
5
|
+
from functools import cache, lru_cache
|
|
6
|
+
|
|
7
|
+
# Standard formats for non-ISO dates
|
|
8
|
+
# %b = abbreviated month (Jan, Feb), %B = full month (January, February)
|
|
9
|
+
STANDARD_FORMATS = [
|
|
10
|
+
"%d %b %Y %H:%M:%S %z", # "20 Feb 2020 15:30:00 +0200"
|
|
11
|
+
"%d %b %Y %H:%M %z", # "01 Jan 2018 00:00 +0000"
|
|
12
|
+
"%d %B %Y %H:%M:%S %z", # "20 February 2020 15:30:00 +0200"
|
|
13
|
+
"%d %B %Y %H:%M %z", # "1 January 2018 00:00 +0000"
|
|
14
|
+
"%Y-%m-%d %H:%M:%S %z", # "2021-01-01 00:00:00 +0000"
|
|
15
|
+
"%Y-%m-%d %H:%M %z", # "2021-01-01 00:00 +0000"
|
|
16
|
+
"%m %d %Y %H:%M:%S %z", # "05 12 2000 10:20:30 +0000" (month-first)
|
|
17
|
+
"%m %d %Y %H:%M %z", # "01 1 2000 00:00 +0000" (month-first)
|
|
18
|
+
"%m %d %Y %z", # "01 1 2000 +0000" (month-first)
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
# Pine Script specific formats (without timezone)
|
|
22
|
+
# %b = abbreviated month (Jan, Feb), %B = full month (January, February)
|
|
23
|
+
# Numeric dates are MONTH-FIRST (MM-DD-YYYY) with '-', '/', '.' or ' ' separators:
|
|
24
|
+
# TradingView parses "03-04-2023" (and "05 12 2000") as March 4 / May 12 and
|
|
25
|
+
# rejects a day-first "13-04-2023" ("31 1 2000") outright ("timestamp(s):
|
|
26
|
+
# unrecognized datetime format"), so there is intentionally no day-first
|
|
27
|
+
# fallback here.
|
|
28
|
+
PINE_FORMATS = [
|
|
29
|
+
"%b %d %Y %H:%M:%S", # "Feb 01 2020 22:10:05"
|
|
30
|
+
"%d %b %Y %H:%M:%S", # "04 Dec 1995 00:12:00"
|
|
31
|
+
"%d %b %Y %H:%M", # "01 Jan 2018 00:00"
|
|
32
|
+
"%b %d %Y", # "Feb 01 2020"
|
|
33
|
+
"%d %b %Y", # "04 Dec 1995"
|
|
34
|
+
"%B %d %Y %H:%M:%S", # "February 01 2020 22:10:05"
|
|
35
|
+
"%d %B %Y %H:%M:%S", # "04 December 1995 00:12:00"
|
|
36
|
+
"%d %B %Y %H:%M", # "01 January 2018 00:00"
|
|
37
|
+
"%B %d %Y", # "February 01 2020"
|
|
38
|
+
"%d %B %Y", # "04 December 1995"
|
|
39
|
+
"%Y-%m-%d", # "2020-02-20"
|
|
40
|
+
"%Y-%m-%d %H:%M:%S", # "2021-01-01 00:00:00"
|
|
41
|
+
"%Y-%m-%d %H:%M", # "2021-01-01 00:00"
|
|
42
|
+
"%m %d %Y %H:%M:%S", # "05 12 2000 10:20:30"
|
|
43
|
+
"%m %d %Y %H:%M", # "01 1 2000 00:00"
|
|
44
|
+
"%m %d %Y", # "05 12 2000", "3 4 2023"
|
|
45
|
+
"%m-%d-%Y %H:%M:%S", # "03-04-2023 10:20:30"
|
|
46
|
+
"%m-%d-%Y %H:%M", # "03-04-2023 10:20"
|
|
47
|
+
"%m-%d-%Y", # "03-04-2023", "3-4-2023"
|
|
48
|
+
"%m/%d/%Y %H:%M:%S", # "03/04/2023 10:20:30"
|
|
49
|
+
"%m/%d/%Y %H:%M", # "03/04/2023 10:20"
|
|
50
|
+
"%m/%d/%Y", # "03/04/2023"
|
|
51
|
+
"%m.%d.%Y %H:%M:%S", # "03.04.2023 10:20:30"
|
|
52
|
+
"%m.%d.%Y %H:%M", # "03.04.2023 10:20"
|
|
53
|
+
"%m.%d.%Y", # "03.04.2023"
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def normalize_timezone(datestring: str) -> str:
|
|
58
|
+
"""
|
|
59
|
+
Normalize timezone format to be compatible with Python's datetime.
|
|
60
|
+
Converts formats like "+00:00" to "+0000"
|
|
61
|
+
|
|
62
|
+
:param datestring: Input date string
|
|
63
|
+
:return: Normalized date string
|
|
64
|
+
"""
|
|
65
|
+
tz_match = re.search(r'([+-])(\d{2}):(\d{2})(?:\s|$)', datestring)
|
|
66
|
+
if tz_match:
|
|
67
|
+
sign, hours, minutes = tz_match.groups()
|
|
68
|
+
new_tz = f"{sign}{hours}{minutes}"
|
|
69
|
+
return datestring[:tz_match.start()] + new_tz + datestring[tz_match.end():]
|
|
70
|
+
return datestring
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# Matches UTC/GMT±HHMM offset forms with optional colon: "UTC-5", "GMT+0530", "+05:30"
|
|
74
|
+
_OFFSET_RE = re.compile(r'^(UTC|GMT)?([+-])(\d{1,2})(?::?(\d{2})?)?$')
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class TimezoneNotFoundError(ValueError):
|
|
78
|
+
"""
|
|
79
|
+
Raised when a timezone string cannot be resolved to a ``ZoneInfo``.
|
|
80
|
+
|
|
81
|
+
Subclasses ``ValueError`` so existing ``except ValueError`` handlers keep
|
|
82
|
+
working, but is a distinct type so callers (e.g. :func:`pynecore.lib.time`)
|
|
83
|
+
can surface it as an actionable error instead of silently degrading to ``na``.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@cache
|
|
88
|
+
def _timezone_db_available() -> bool:
|
|
89
|
+
"""
|
|
90
|
+
Return whether an IANA timezone database is reachable on this system.
|
|
91
|
+
|
|
92
|
+
Probes a canonical zone name. On Windows without the ``tzdata`` package and
|
|
93
|
+
without a system zoneinfo database, even standard names fail to resolve.
|
|
94
|
+
|
|
95
|
+
:return: True if standard IANA names can be resolved
|
|
96
|
+
"""
|
|
97
|
+
try:
|
|
98
|
+
ZoneInfo("America/New_York")
|
|
99
|
+
return True
|
|
100
|
+
except Exception: # noqa - any failure means the database is unusable
|
|
101
|
+
return False
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _missing_timezone_message(timezone: str) -> str:
|
|
105
|
+
"""
|
|
106
|
+
Build an actionable error message for an unresolved timezone when the IANA
|
|
107
|
+
database is missing.
|
|
108
|
+
|
|
109
|
+
:param timezone: The timezone string that could not be resolved
|
|
110
|
+
:return: Multi-line, platform-aware error message
|
|
111
|
+
"""
|
|
112
|
+
lines = [
|
|
113
|
+
f"Timezone {timezone!r} could not be resolved: the IANA timezone database "
|
|
114
|
+
"is not available on this system.",
|
|
115
|
+
"",
|
|
116
|
+
"Install it with:",
|
|
117
|
+
" pip install tzdata",
|
|
118
|
+
]
|
|
119
|
+
if sys.platform.startswith("win"):
|
|
120
|
+
lines += [
|
|
121
|
+
"",
|
|
122
|
+
"Windows has no built-in timezone database, so the 'tzdata' package is "
|
|
123
|
+
"required. PyneCore's [cli] and [all] installs include it automatically.",
|
|
124
|
+
]
|
|
125
|
+
return "\n".join(lines)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@lru_cache(maxsize=128)
|
|
129
|
+
def _parse_timezone_cached(timezone: str) -> ZoneInfo:
|
|
130
|
+
"""
|
|
131
|
+
Parse a concrete, non-empty timezone string into a ZoneInfo object.
|
|
132
|
+
|
|
133
|
+
Kept separate from :func:`parse_timezone` so the cache is only ever keyed on
|
|
134
|
+
an explicit timezone string. The ``None`` -> exchange-timezone fallback must
|
|
135
|
+
NOT be cached: it resolves against the mutable ``syminfo.timezone`` global,
|
|
136
|
+
so a cached ``None`` entry would leak one script's timezone into the next run
|
|
137
|
+
in the same process.
|
|
138
|
+
|
|
139
|
+
:param timezone: Concrete timezone string (IANA name or UTC/GMT±HHMM offset)
|
|
140
|
+
:return: ZoneInfo object
|
|
141
|
+
:raises TimezoneNotFoundError: If the timezone cannot be resolved
|
|
142
|
+
"""
|
|
143
|
+
# Try as IANA timezone first
|
|
144
|
+
try:
|
|
145
|
+
return ZoneInfo(timezone)
|
|
146
|
+
except KeyError:
|
|
147
|
+
# ZoneInfoNotFoundError is a KeyError subclass: the name is not in the IANA
|
|
148
|
+
# database. UTC/GMT±HHMM offset forms are parsed below; any other name is an
|
|
149
|
+
# IANA name whose lookup genuinely failed.
|
|
150
|
+
pass
|
|
151
|
+
|
|
152
|
+
# Parse UTC/GMT±HHMM offset format with optional colon
|
|
153
|
+
match = _OFFSET_RE.match(timezone)
|
|
154
|
+
if match is None:
|
|
155
|
+
# Not an offset form -> the timezone name could not be resolved. The most
|
|
156
|
+
# common cause is a missing IANA database (Windows ships none by default).
|
|
157
|
+
if not _timezone_db_available():
|
|
158
|
+
raise TimezoneNotFoundError(_missing_timezone_message(timezone))
|
|
159
|
+
raise TimezoneNotFoundError(
|
|
160
|
+
f"Unknown timezone {timezone!r}. Use a valid IANA name "
|
|
161
|
+
"(e.g. 'America/New_York') or a UTC/GMT±HHMM offset (e.g. 'UTC-5', 'GMT+0530')."
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
prefix, sign, hours, minutes = match.groups()
|
|
165
|
+
offset = int(hours)
|
|
166
|
+
if minutes:
|
|
167
|
+
offset += int(minutes) / 60
|
|
168
|
+
|
|
169
|
+
# UTC/GMT+X maps to Etc/GMT-X and vice versa
|
|
170
|
+
# Special case: offset 0 should use UTC directly
|
|
171
|
+
if offset == 0:
|
|
172
|
+
return ZoneInfo("UTC")
|
|
173
|
+
zone = f"Etc/GMT{'-' if sign == '+' else '+'}{int(abs(offset))}"
|
|
174
|
+
return ZoneInfo(zone)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
# Lazily bound to the ``lib.syminfo`` module on first use. Importing it at module
|
|
178
|
+
# top would create a datetime <-> lib import cycle (lib pulls in timeframe, which
|
|
179
|
+
# imports parse_timezone), so the reference is fetched once on the first fallback
|
|
180
|
+
# call and reused -- keeping the hot path a plain attribute read with no per-call
|
|
181
|
+
# import cost.
|
|
182
|
+
_syminfo = None
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def parse_timezone(timezone: str | None) -> ZoneInfo:
|
|
186
|
+
"""
|
|
187
|
+
Parse timezone string into ZoneInfo object. Supports:
|
|
188
|
+
- IANA timezone names (e.g. "America/New_York")
|
|
189
|
+
- UTC±HHMM format (e.g. "UTC-5", "UTC+0530")
|
|
190
|
+
- GMT±HHMM format (e.g. "GMT-5", "GMT+0530")
|
|
191
|
+
- Raw offset (e.g. "+0530", "-05:00")
|
|
192
|
+
|
|
193
|
+
When ``timezone`` is falsy the exchange timezone (``syminfo.timezone``) is
|
|
194
|
+
used, defaulting to UTC when that is unset too. This fallback value is read on
|
|
195
|
+
every call -- never cached -- so changing the active symbol's timezone takes
|
|
196
|
+
effect immediately instead of returning a previous run's cached zone.
|
|
197
|
+
|
|
198
|
+
:param timezone: Timezone string, or None to use the exchange timezone
|
|
199
|
+
:return: ZoneInfo object
|
|
200
|
+
:raises TimezoneNotFoundError: If the timezone cannot be resolved
|
|
201
|
+
"""
|
|
202
|
+
if not timezone:
|
|
203
|
+
global _syminfo
|
|
204
|
+
if _syminfo is None:
|
|
205
|
+
from ..lib import syminfo
|
|
206
|
+
_syminfo = syminfo
|
|
207
|
+
timezone = _syminfo.timezone or 'UTC'
|
|
208
|
+
return _parse_timezone_cached(timezone)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def parse_datestring(datestring: str) -> datetime:
|
|
212
|
+
"""
|
|
213
|
+
Parse date string using multiple formats.
|
|
214
|
+
Handles ISO 8601 with microseconds and timezone offsets.
|
|
215
|
+
If no time is supplied, "00:00" is used.
|
|
216
|
+
If no timezone is supplied, GMT+0 is used.
|
|
217
|
+
|
|
218
|
+
:param datestring: Date string to parse
|
|
219
|
+
:return: Parsed datetime object
|
|
220
|
+
:raises ValueError: If the date format is invalid
|
|
221
|
+
"""
|
|
222
|
+
datestring = datestring.strip()
|
|
223
|
+
if not datestring:
|
|
224
|
+
return datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0)
|
|
225
|
+
|
|
226
|
+
# Try parsing ISO 8601 style dates WITH TIME first (handles both T and space
|
|
227
|
+
# separator; seconds are optional -- "2021-01-01 00:00" is accepted too)
|
|
228
|
+
iso_match = re.match(
|
|
229
|
+
r'(\d{4}-\d{2}-\d{2}[T\s]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?)' # datetime part
|
|
230
|
+
r'([+-]\d{2}:\d{2})?$', # timezone part
|
|
231
|
+
datestring
|
|
232
|
+
)
|
|
233
|
+
if iso_match:
|
|
234
|
+
dt_part, tz_part = iso_match.groups()
|
|
235
|
+
if tz_part:
|
|
236
|
+
datestring = normalize_timezone(datestring)
|
|
237
|
+
dt_str = datestring.replace(' ', 'T') # Normalize to T for parsing
|
|
238
|
+
for fmt in ("%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M%z"):
|
|
239
|
+
try:
|
|
240
|
+
return datetime.strptime(dt_str, fmt)
|
|
241
|
+
except ValueError:
|
|
242
|
+
continue
|
|
243
|
+
else:
|
|
244
|
+
dt_str = dt_part.replace(' ', 'T')
|
|
245
|
+
for fmt in ("%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M"):
|
|
246
|
+
try:
|
|
247
|
+
return datetime.strptime(dt_str, fmt).replace(tzinfo=UTC)
|
|
248
|
+
except ValueError:
|
|
249
|
+
continue
|
|
250
|
+
|
|
251
|
+
# Try parsing ISO 8601 DATE ONLY format (YYYY-MM-DD) before timezone extraction
|
|
252
|
+
# This prevents the timezone regex from incorrectly matching date parts like -09 in 2025-01-09
|
|
253
|
+
iso_date_match = re.match(r'^\d{4}-\d{2}-\d{2}$', datestring)
|
|
254
|
+
if iso_date_match:
|
|
255
|
+
dt = datetime.strptime(datestring, "%Y-%m-%d")
|
|
256
|
+
# Use exchange timezone (from syminfo) when no timezone is specified
|
|
257
|
+
default_tz = parse_timezone(None) # This will return syminfo.timezone
|
|
258
|
+
return dt.replace(tzinfo=default_tz)
|
|
259
|
+
|
|
260
|
+
# Extract timezone if present at the end for other formats
|
|
261
|
+
# The regex requires whitespace before timezone to avoid matching date parts
|
|
262
|
+
tz_match = re.search(r'\s+((?:UTC|GMT)?[+-]\d{1,2}(?::?\d{2})?)\s*$', datestring)
|
|
263
|
+
if tz_match:
|
|
264
|
+
tz = parse_timezone(
|
|
265
|
+
f"UTC{tz_match.group(1)}" if not tz_match.group(1).startswith(('UTC', 'GMT')) else tz_match.group(1))
|
|
266
|
+
datestring = datestring[:tz_match.start()].strip()
|
|
267
|
+
else:
|
|
268
|
+
# Use exchange timezone (from syminfo) when no timezone is specified
|
|
269
|
+
tz = parse_timezone(None) # This will return syminfo.timezone
|
|
270
|
+
|
|
271
|
+
# Try standard formats (with timezone)
|
|
272
|
+
if tz_match:
|
|
273
|
+
normalized = normalize_timezone(f"{datestring} {tz_match.group(1)}")
|
|
274
|
+
for fmt in STANDARD_FORMATS:
|
|
275
|
+
try:
|
|
276
|
+
return datetime.strptime(normalized, fmt)
|
|
277
|
+
except ValueError:
|
|
278
|
+
continue
|
|
279
|
+
|
|
280
|
+
# Try Pine formats (without timezone)
|
|
281
|
+
for fmt in PINE_FORMATS:
|
|
282
|
+
try:
|
|
283
|
+
dt = datetime.strptime(datestring, fmt)
|
|
284
|
+
return dt.replace(tzinfo=tz)
|
|
285
|
+
except ValueError:
|
|
286
|
+
continue
|
|
287
|
+
|
|
288
|
+
raise ValueError(
|
|
289
|
+
f"Invalid date format: {datestring}\n"
|
|
290
|
+
"Supported formats:\n"
|
|
291
|
+
"- ISO Style: '2020-02-20T15:30:00+02:00', '2025-01-01 01:23:45-05:00'\n"
|
|
292
|
+
"- With fraction: '2024-08-01T04:38:47.731215+00:00'\n"
|
|
293
|
+
"- RFC Style: '20 Feb 2020 15:30:00 GMT+0200', '1 January 2018 00:00 +0000'\n"
|
|
294
|
+
"- Simple Pine: 'Feb 01 2020 22:10:05', '1 January 2018', '2020-02-20'\n"
|
|
295
|
+
"- Numeric, month first: '01-01-2023', '03/04/2023', '03.04.2023 10:20:30'"
|
|
296
|
+
)
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Persist the originating provider string of a download in the syminfo TOML.
|
|
3
|
+
|
|
4
|
+
``pyne data download`` appends a ``[download]`` section to the ``.toml`` saved
|
|
5
|
+
next to the ``.ohlcv`` file::
|
|
6
|
+
|
|
7
|
+
[download]
|
|
8
|
+
provider = "ccxt:BYBIT:BTC/USDT:USDT@1D"
|
|
9
|
+
|
|
10
|
+
The provider string is the canonical CLI form, so the file can be re-downloaded
|
|
11
|
+
or resumed without reconstructing it from ``SymInfo`` fields (``prefix`` and the
|
|
12
|
+
flattened filename are both lossy). ``SymInfo.load_toml`` reads only the
|
|
13
|
+
``[symbol]`` section, so older pynecore versions ignore this section;
|
|
14
|
+
``SymInfo.save_toml`` preserves it verbatim when rewriting the file.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
__all__ = ['read_download_provider', 'write_download_provider', 'extract_download_section']
|
|
22
|
+
|
|
23
|
+
# [download] up to (not including) the next section header or EOF
|
|
24
|
+
_SECTION_RE = re.compile(r'(?ms)^\[download\][^\n]*\n.*?(?=^\[|\Z)')
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def read_download_provider(toml_path: Path) -> str | None:
|
|
28
|
+
"""
|
|
29
|
+
Read the persisted provider string from a syminfo TOML.
|
|
30
|
+
|
|
31
|
+
:param toml_path: Path to the ``.toml`` next to the ``.ohlcv`` file.
|
|
32
|
+
:return: The provider string, or None if the file or section is missing
|
|
33
|
+
or unparsable.
|
|
34
|
+
"""
|
|
35
|
+
import tomllib
|
|
36
|
+
try:
|
|
37
|
+
with open(toml_path, 'rb') as f:
|
|
38
|
+
data = tomllib.load(f)
|
|
39
|
+
except (OSError, tomllib.TOMLDecodeError):
|
|
40
|
+
return None
|
|
41
|
+
value = data.get('download', {}).get('provider')
|
|
42
|
+
return value if isinstance(value, str) and value else None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def extract_download_section(text: str) -> str | None:
|
|
46
|
+
"""
|
|
47
|
+
Extract the ``[download]`` section verbatim from TOML text.
|
|
48
|
+
|
|
49
|
+
:param text: Full TOML file content.
|
|
50
|
+
:return: The section text (without trailing whitespace), or None.
|
|
51
|
+
"""
|
|
52
|
+
m = _SECTION_RE.search(text)
|
|
53
|
+
return m.group(0).rstrip() if m else None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def write_download_provider(toml_path: Path, provider_string: str) -> None:
|
|
57
|
+
"""
|
|
58
|
+
Write (or replace) the ``[download]`` section in a syminfo TOML.
|
|
59
|
+
|
|
60
|
+
Missing files are left untouched: the section rides along with the symbol
|
|
61
|
+
info and makes no sense on its own.
|
|
62
|
+
|
|
63
|
+
:param toml_path: Path to the ``.toml`` next to the ``.ohlcv`` file.
|
|
64
|
+
:param provider_string: Canonical provider string to persist.
|
|
65
|
+
"""
|
|
66
|
+
if not toml_path.exists():
|
|
67
|
+
return
|
|
68
|
+
text = toml_path.read_text(encoding='utf-8')
|
|
69
|
+
text = _SECTION_RE.sub('', text).rstrip()
|
|
70
|
+
section = f'[download]\nprovider = "{provider_string}"'
|
|
71
|
+
toml_path.write_text(f'{text}\n\n{section}\n', encoding='utf-8')
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Shared OHLCV download core.
|
|
3
|
+
|
|
4
|
+
The ``.ohlcv`` download flow (start-date resolution, range clamping, truncate
|
|
5
|
+
handling, progress accounting, symbol-info persistence) is driven by several
|
|
6
|
+
front-ends: the ``pyne data download`` CLI, the symbol-browser TUI and the
|
|
7
|
+
IDE bridge's provider service. They only differ in how they render progress
|
|
8
|
+
and how they answer the "the file would have to be truncated" question, so
|
|
9
|
+
everything else lives here.
|
|
10
|
+
|
|
11
|
+
This module is part of the non-interactive ``core`` layer: it never prompts,
|
|
12
|
+
never prints and knows nothing about typer/rich. Front-ends pass callbacks.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from typing import Callable, Literal, TypeAlias
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from datetime import datetime, timedelta, UTC
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
from .download_info import write_download_provider
|
|
22
|
+
from .ohlcv_file import OHLCVWriter
|
|
23
|
+
from .plugin.provider import ProviderPlugin
|
|
24
|
+
from .syminfo import SymInfo
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
'ConflictAction', 'DownloadConflict', 'DownloadPlan', 'DownloadProgress',
|
|
28
|
+
'DownloadResult', 'DownloadError', 'DownloadConflictError',
|
|
29
|
+
'InvalidTimeRangeError', 'download_to_file',
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
ConflictAction: TypeAlias = Literal['truncate', 'abort']
|
|
33
|
+
"""What to do when the requested start date precedes the first bar of the
|
|
34
|
+
existing file: drop the file's content or raise. Appending before the existing
|
|
35
|
+
first bar is not possible, the writer only ever appends at the end."""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class DownloadConflict:
|
|
40
|
+
"""The requested start date is before the first bar of the existing file."""
|
|
41
|
+
ohlcv_path: Path
|
|
42
|
+
time_from: datetime
|
|
43
|
+
existing_start: datetime
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class DownloadPlan:
|
|
48
|
+
"""The resolved download range, handed to ``on_start`` before the first
|
|
49
|
+
provider request so front-ends can size their progress display."""
|
|
50
|
+
ohlcv_path: Path
|
|
51
|
+
time_from: datetime
|
|
52
|
+
time_to: datetime
|
|
53
|
+
fetch_all: bool
|
|
54
|
+
total_seconds: int
|
|
55
|
+
"""Length of the range in seconds, 0 for a ``fetch_all`` download (whose
|
|
56
|
+
end is unknown, so no proportional progress can be shown)."""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True)
|
|
60
|
+
class DownloadProgress:
|
|
61
|
+
"""One progress tick. Progress is time-proportional: the provider reports
|
|
62
|
+
the timestamp it has reached, not a bar count."""
|
|
63
|
+
current: datetime
|
|
64
|
+
elapsed_seconds: int
|
|
65
|
+
total_seconds: int
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True)
|
|
69
|
+
class DownloadResult:
|
|
70
|
+
"""Outcome of a finished download."""
|
|
71
|
+
ohlcv_path: Path
|
|
72
|
+
time_from: datetime
|
|
73
|
+
time_to: datetime
|
|
74
|
+
fetch_all: bool
|
|
75
|
+
bars_written: int
|
|
76
|
+
"""Number of records the file grew by. Bars that overwrote already present
|
|
77
|
+
records do not count, so this is a lower bound of the fetched bars."""
|
|
78
|
+
syminfo: SymInfo | None
|
|
79
|
+
"""The symbol info belonging to the file. When the caller passed one in, it
|
|
80
|
+
is the very same object, whose ``mincontract`` may have been refined."""
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class DownloadError(Exception):
|
|
84
|
+
"""Base class of the errors raised by :func:`download_to_file`."""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class InvalidTimeRangeError(DownloadError):
|
|
88
|
+
"""The resolved end date is before the resolved start date."""
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class DownloadConflictError(DownloadError):
|
|
92
|
+
"""The download would have truncated an existing file and the caller's
|
|
93
|
+
``on_conflict`` policy was ``'abort'``."""
|
|
94
|
+
|
|
95
|
+
def __init__(self, conflict: DownloadConflict):
|
|
96
|
+
super().__init__(
|
|
97
|
+
f"The start date (from: {conflict.time_from}) is before the start of the "
|
|
98
|
+
f"existing file ({conflict.existing_start}); "
|
|
99
|
+
f"downloading it would truncate the file."
|
|
100
|
+
)
|
|
101
|
+
self.conflict = conflict
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def download_to_file(
|
|
105
|
+
provider: ProviderPlugin, *,
|
|
106
|
+
time_from: datetime | str,
|
|
107
|
+
time_to: datetime,
|
|
108
|
+
truncate: bool = False,
|
|
109
|
+
chunk_size: int | None = None,
|
|
110
|
+
extra_data: bool = False,
|
|
111
|
+
syminfo: SymInfo | None = None,
|
|
112
|
+
on_start: Callable[[DownloadPlan], None] | None = None,
|
|
113
|
+
on_progress: Callable[[DownloadProgress], None] | None = None,
|
|
114
|
+
on_conflict: "ConflictAction | Callable[[DownloadConflict], ConflictAction]" = 'abort',
|
|
115
|
+
provider_string: str | None = None,
|
|
116
|
+
) -> DownloadResult:
|
|
117
|
+
"""
|
|
118
|
+
Download OHLCV data into the provider's ``.ohlcv`` file.
|
|
119
|
+
|
|
120
|
+
The provider must already be bound to a symbol (built with one, so its
|
|
121
|
+
``ohlcv_path`` and writer are set); every front-end constructs it that way,
|
|
122
|
+
which for multi-broker providers keeps the broker in the filename.
|
|
123
|
+
|
|
124
|
+
:param provider: The provider instance to download with.
|
|
125
|
+
:param time_from: Start date, or the ``"continue"`` sentinel to resume the
|
|
126
|
+
existing file (falling back to all available data for
|
|
127
|
+
``fetch_all_by_default`` providers, otherwise one year back). Aware
|
|
128
|
+
datetimes are converted to UTC, naive ones are taken as UTC.
|
|
129
|
+
:param time_to: End date (same timezone handling as ``time_from``); clamped
|
|
130
|
+
to "now", as the future takes forever to download.
|
|
131
|
+
:param truncate: Drop the existing file content before downloading.
|
|
132
|
+
:param chunk_size: Override the provider's per-request bar count.
|
|
133
|
+
:param extra_data: Also fetch the provider's extra fields (``.extra.csv``).
|
|
134
|
+
:param syminfo: Already-loaded symbol info. When None it is fetched (and
|
|
135
|
+
persisted) *before* the download, best-effort, so an interrupted
|
|
136
|
+
download still leaves a resumable file.
|
|
137
|
+
:param on_start: Called once with the resolved :class:`DownloadPlan`, before
|
|
138
|
+
the first OHLCV request.
|
|
139
|
+
:param on_progress: Called with :class:`DownloadProgress` as the download
|
|
140
|
+
advances. Never called for a ``fetch_all`` download.
|
|
141
|
+
:param on_conflict: Policy or callback deciding what to do when the start
|
|
142
|
+
date precedes the existing file's first bar. A callback may prompt (the
|
|
143
|
+
CLI does) and must return one of the :data:`ConflictAction` values.
|
|
144
|
+
:param provider_string: Canonical provider string to persist in the
|
|
145
|
+
``[download]`` section of the syminfo TOML, written *before* the
|
|
146
|
+
download so the file stays re-downloadable by path even if the download
|
|
147
|
+
is cut short. None skips it.
|
|
148
|
+
:return: The :class:`DownloadResult` of the finished download.
|
|
149
|
+
:raises InvalidTimeRangeError: If the end date is before the start date.
|
|
150
|
+
:raises DownloadConflictError: If a truncating conflict was answered with
|
|
151
|
+
``'abort'``.
|
|
152
|
+
"""
|
|
153
|
+
assert provider.ohlcv_path is not None
|
|
154
|
+
ohlcv_path = provider.ohlcv_path
|
|
155
|
+
|
|
156
|
+
with provider as ohlcv_writer:
|
|
157
|
+
if truncate:
|
|
158
|
+
ohlcv_writer.seek(0)
|
|
159
|
+
ohlcv_writer.truncate()
|
|
160
|
+
|
|
161
|
+
resolved_from, fetch_all = _resolve_from(provider, time_from, ohlcv_writer)
|
|
162
|
+
|
|
163
|
+
# The rest of the flow works with naive UTC datetimes
|
|
164
|
+
resolved_from = _to_naive_utc(resolved_from)
|
|
165
|
+
resolved_to = _to_naive_utc(time_to)
|
|
166
|
+
|
|
167
|
+
# We cannot download data from the future otherwise it would take very long
|
|
168
|
+
now_naive = datetime.now(UTC).replace(tzinfo=None)
|
|
169
|
+
if resolved_to > now_naive:
|
|
170
|
+
resolved_to = now_naive
|
|
171
|
+
|
|
172
|
+
if not fetch_all and resolved_to < resolved_from:
|
|
173
|
+
raise InvalidTimeRangeError(
|
|
174
|
+
"End date (to) must be greater than start date (from)!")
|
|
175
|
+
|
|
176
|
+
if ohlcv_writer.start_timestamp and not fetch_all:
|
|
177
|
+
existing_start = ohlcv_writer.start_datetime.replace(tzinfo=None)
|
|
178
|
+
if resolved_from < existing_start:
|
|
179
|
+
conflict = DownloadConflict(ohlcv_path=ohlcv_path, time_from=resolved_from,
|
|
180
|
+
existing_start=existing_start)
|
|
181
|
+
action = on_conflict(conflict) if callable(on_conflict) else on_conflict
|
|
182
|
+
if action == 'abort':
|
|
183
|
+
raise DownloadConflictError(conflict)
|
|
184
|
+
if action != 'truncate':
|
|
185
|
+
raise ValueError(f"Invalid conflict action: {action!r}")
|
|
186
|
+
ohlcv_writer.seek(0)
|
|
187
|
+
ohlcv_writer.truncate()
|
|
188
|
+
|
|
189
|
+
# Persist the symbol info and the originating provider string BEFORE the
|
|
190
|
+
# (potentially long) download, so an interrupted or user-aborted download
|
|
191
|
+
# still leaves a resumable file: the [download] section lets it be
|
|
192
|
+
# continued by path, and "continue" picks up from the last written bar.
|
|
193
|
+
if syminfo is None:
|
|
194
|
+
# noinspection PyBroadException
|
|
195
|
+
try:
|
|
196
|
+
syminfo = provider.get_symbol_info() # save_toml() side effect
|
|
197
|
+
except Exception:
|
|
198
|
+
syminfo = None # Symbol info is best-effort, don't block the download
|
|
199
|
+
if provider_string is not None:
|
|
200
|
+
# No-op until the syminfo TOML exists; get_symbol_info() above (or the
|
|
201
|
+
# caller) is what creates it.
|
|
202
|
+
write_download_provider(ohlcv_path.with_suffix('.toml'), provider_string)
|
|
203
|
+
|
|
204
|
+
total_seconds = 0 if fetch_all else max(0, int((resolved_to - resolved_from).total_seconds()))
|
|
205
|
+
if on_start is not None:
|
|
206
|
+
on_start(DownloadPlan(ohlcv_path=ohlcv_path, time_from=resolved_from,
|
|
207
|
+
time_to=resolved_to, fetch_all=fetch_all,
|
|
208
|
+
total_seconds=total_seconds))
|
|
209
|
+
|
|
210
|
+
progress_cb = on_progress
|
|
211
|
+
|
|
212
|
+
def cb_progress(current_time: datetime) -> None:
|
|
213
|
+
""" Callback to report time-proportional progress """
|
|
214
|
+
assert progress_cb is not None
|
|
215
|
+
elapsed = min(max(int((current_time - resolved_from).total_seconds()), 0), total_seconds)
|
|
216
|
+
progress_cb(DownloadProgress(current=current_time, elapsed_seconds=elapsed,
|
|
217
|
+
total_seconds=total_seconds))
|
|
218
|
+
|
|
219
|
+
cb = cb_progress if (progress_cb is not None and not fetch_all) else None
|
|
220
|
+
|
|
221
|
+
size_before = ohlcv_writer.size
|
|
222
|
+
provider.download_ohlcv(resolved_from, resolved_to, on_progress=cb,
|
|
223
|
+
limit=chunk_size, with_extra=extra_data)
|
|
224
|
+
bars_written = ohlcv_writer.size - size_before
|
|
225
|
+
|
|
226
|
+
# Refine the heuristic mincontract from the downloaded volume data
|
|
227
|
+
# (only when the provider had no exchange value for it). save_toml()
|
|
228
|
+
# preserves the [download] section written before the download.
|
|
229
|
+
if syminfo is not None and provider.mincontract_estimated:
|
|
230
|
+
qty_step = ohlcv_writer.analyzed_qty_step or 0.0
|
|
231
|
+
if qty_step > 0.0 and qty_step != syminfo.mincontract:
|
|
232
|
+
syminfo.mincontract = qty_step
|
|
233
|
+
syminfo.save_toml(ohlcv_path.with_suffix('.toml'))
|
|
234
|
+
|
|
235
|
+
return DownloadResult(ohlcv_path=ohlcv_path, time_from=resolved_from, time_to=resolved_to,
|
|
236
|
+
fetch_all=fetch_all, bars_written=bars_written, syminfo=syminfo)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _resolve_from(provider: ProviderPlugin, time_from: datetime | str,
|
|
240
|
+
ohlcv_writer: OHLCVWriter) -> tuple[datetime, bool]:
|
|
241
|
+
"""
|
|
242
|
+
Resolve the ``"continue"`` start-date sentinel against the existing file.
|
|
243
|
+
|
|
244
|
+
:param provider: The provider instance (its ``fetch_all_by_default``
|
|
245
|
+
decides what "no data yet" means).
|
|
246
|
+
:param time_from: Start date or the ``"continue"`` sentinel.
|
|
247
|
+
:param ohlcv_writer: The already opened writer of the target file.
|
|
248
|
+
:return: ``(start_date, fetch_all)``.
|
|
249
|
+
"""
|
|
250
|
+
if isinstance(time_from, datetime):
|
|
251
|
+
return time_from, False
|
|
252
|
+
|
|
253
|
+
assert time_from == "continue", f"Unexpected from value: {time_from!r}"
|
|
254
|
+
|
|
255
|
+
end_ts = ohlcv_writer.end_timestamp
|
|
256
|
+
interval = ohlcv_writer.interval
|
|
257
|
+
if end_ts and interval: # Resume from last download
|
|
258
|
+
# One interval ahead, otherwise the last bar would be downloaded again
|
|
259
|
+
return datetime.fromtimestamp(end_ts, UTC) + timedelta(seconds=interval), False
|
|
260
|
+
if provider.fetch_all_by_default:
|
|
261
|
+
return datetime.fromtimestamp(0, UTC), True
|
|
262
|
+
return datetime.now(UTC) - timedelta(days=365), False
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _to_naive_utc(dt: datetime) -> datetime:
|
|
266
|
+
"""
|
|
267
|
+
Convert a datetime to naive UTC, the form the download flow works with.
|
|
268
|
+
|
|
269
|
+
:param dt: Aware or naive (already UTC) datetime.
|
|
270
|
+
:return: Naive UTC datetime.
|
|
271
|
+
"""
|
|
272
|
+
if dt.tzinfo is None:
|
|
273
|
+
return dt
|
|
274
|
+
return dt.astimezone(UTC).replace(tzinfo=None)
|