mt5cli 1.4.1__tar.gz → 1.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mt5cli-1.4.1 → mt5cli-1.5.0}/.agents/skills/local-qa/scripts/qa.sh +3 -2
- {mt5cli-1.4.1 → mt5cli-1.5.0}/.agents/skills/mt5cli/SKILL.md +4 -2
- {mt5cli-1.4.1 → mt5cli-1.5.0}/.github/workflows/claude.yml +2 -3
- {mt5cli-1.4.1 → mt5cli-1.5.0}/PKG-INFO +21 -14
- {mt5cli-1.4.1 → mt5cli-1.5.0}/README.md +19 -12
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/api/history.md +43 -109
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/api/public-contract.md +41 -43
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/api/trading.md +23 -17
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/index.md +16 -9
- mt5cli-1.5.0/mt5cli/__init__.py +18 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mt5cli/cli.py +39 -52
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mt5cli/client.py +24 -23
- mt5cli-1.5.0/mt5cli/contract.py +103 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mt5cli/converters.py +70 -14
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mt5cli/history.py +209 -687
- mt5cli-1.5.0/mt5cli/lifecycle.py +87 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mt5cli/marketdata.py +1 -1
- mt5cli-1.5.0/mt5cli/rates.py +538 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mt5cli/schemas.py +102 -44
- mt5cli-1.4.1/mt5cli/__init__.py → mt5cli-1.5.0/mt5cli/sdk.py +16 -20
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mt5cli/trading.py +6 -6
- {mt5cli-1.4.1 → mt5cli-1.5.0}/pyproject.toml +3 -3
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/cli/test_cli.bats +8 -3
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/unit/test_cli.py +37 -110
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/unit/test_client.py +105 -29
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/unit/test_contracts.py +118 -27
- mt5cli-1.5.0/tests/unit/test_docs_contract.py +40 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/unit/test_history.py +588 -2013
- mt5cli-1.5.0/tests/unit/test_history_contract.py +371 -0
- mt5cli-1.5.0/tests/unit/test_history_edge_contract.py +122 -0
- mt5cli-1.5.0/tests/unit/test_lifecycle.py +145 -0
- mt5cli-1.5.0/tests/unit/test_rate_timestamp_contract.py +291 -0
- mt5cli-1.5.0/tests/unit/test_rates.py +380 -0
- mt5cli-1.5.0/tests/unit/test_rates_contract.py +220 -0
- mt5cli-1.5.0/tests/unit/test_stable_sdk_cleanup.py +52 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/uv.lock +4 -4
- mt5cli-1.4.1/AGENTS.md +0 -29
- mt5cli-1.4.1/CLAUDE.md +0 -1
- mt5cli-1.4.1/mt5cli/contract.py +0 -177
- mt5cli-1.4.1/tests/unit/test_stable_sdk_cleanup.py +0 -31
- {mt5cli-1.4.1 → mt5cli-1.5.0}/.agents/skills/local-qa/SKILL.md +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/.claude/agents/codex.md +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/.claude/settings.json +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/.github/CODEOWNERS +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/.github/FUNDING.yml +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/.github/dependabot.yml +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/.github/renovate.json +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/.github/workflows/ci.yml +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/.github/workflows/release.yml +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/.gitignore +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/LICENSE +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/api/cli.md +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/api/client.md +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/api/converters.md +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/api/exceptions.md +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/api/grafana.md +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/api/index.md +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/api/marketdata.md +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/api/observability.md +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/api/schemas.md +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/api/telemetry.md +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/docs/api/utils.md +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/examples/grafana/README.md +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/examples/grafana/compose.yml +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/examples/grafana/dashboards/mt5cli-market.json +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/examples/grafana/dashboards/mt5cli-overview.json +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/examples/grafana/dashboards/mt5cli-trades.json +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/examples/grafana/provisioning/dashboards/mt5cli.yml +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/examples/grafana/provisioning/datasources/mt5cli-sqlite.yml +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mkdocs.yml +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mt5cli/__main__.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mt5cli/exceptions.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mt5cli/grafana.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mt5cli/observability.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mt5cli/retry.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mt5cli/telemetry.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/mt5cli/utils.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/unit/__init__.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/unit/conftest.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/unit/test_examples.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/unit/test_grafana.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/unit/test_marketdata.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/unit/test_observability.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/unit/test_packaging.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/unit/test_telemetry.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/unit/test_trading.py +0 -0
- {mt5cli-1.4.1 → mt5cli-1.5.0}/tests/unit/test_utils.py +0 -0
|
@@ -9,6 +9,7 @@ export NPM_CONFIG_MIN_RELEASE_AGE="${COOLDOWN_DAYS}"
|
|
|
9
9
|
export PNPM_CONFIG_MINIMUM_RELEASE_AGE=$((COOLDOWN_DAYS * 24 * 60))
|
|
10
10
|
|
|
11
11
|
# Python
|
|
12
|
+
uv sync
|
|
12
13
|
uv run ruff format .
|
|
13
14
|
uv run ruff check --fix .
|
|
14
15
|
uv run pyright .
|
|
@@ -31,9 +32,9 @@ case "${OSTYPE}" in
|
|
|
31
32
|
| xargs -0 -t shellcheck
|
|
32
33
|
|
|
33
34
|
# GitHub Actions
|
|
34
|
-
zizmor --fix=safe .github/workflows
|
|
35
|
+
uvx zizmor --fix=safe .github/workflows
|
|
35
36
|
git ls-files -z -- '.github/workflows/*.yml' | xargs -0 -t actionlint
|
|
36
|
-
checkov --framework=all --output=github_failed_only --directory=.
|
|
37
|
+
uvx checkov --framework=all --output=github_failed_only --directory=.
|
|
37
38
|
;;
|
|
38
39
|
*)
|
|
39
40
|
echo 'GitHub Actions and shell script linting is only supported on Linux and macOS.'
|
|
@@ -41,8 +41,10 @@ Global options MUST precede the subcommand.
|
|
|
41
41
|
|
|
42
42
|
### Parameter value formats
|
|
43
43
|
|
|
44
|
-
- **Datetimes** (`--date-from`, `--date-to`): ISO 8601
|
|
45
|
-
`2024-01-01T12:00:00
|
|
44
|
+
- **Datetimes** (`--date-from`, `--date-to`): offset-free ISO 8601 values
|
|
45
|
+
(`2024-01-01` or `2024-01-01T12:00:00`). MT5 bounds are timezone-naive
|
|
46
|
+
trade-server wall-clock values; offset-bearing values are rejected and no
|
|
47
|
+
UTC-to-server-time conversion is inferred.
|
|
46
48
|
- **Timeframe** (`--timeframe`): `M1`, `M2`, `M3`, `M4`, `M5`, `M6`, `M10`,
|
|
47
49
|
`M12`, `M15`, `M20`, `M30`, `H1`, `H2`, `H3`, `H4`, `H6`, `H8`, `H12`,
|
|
48
50
|
`D1`, `W1`, `MN1`, or the raw integer.
|
|
@@ -27,8 +27,7 @@ jobs:
|
|
|
27
27
|
if: >
|
|
28
28
|
github.event_name == 'pull_request'
|
|
29
29
|
&& (! github.event.pull_request.draft)
|
|
30
|
-
&&
|
|
31
|
-
&& (! startsWith(github.head_ref, 'renovate/'))
|
|
30
|
+
&& github.event.pull_request.user.type != 'Bot'
|
|
32
31
|
permissions:
|
|
33
32
|
contents: read
|
|
34
33
|
pull-requests: write
|
|
@@ -46,7 +45,7 @@ jobs:
|
|
|
46
45
|
|| (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude'))
|
|
47
46
|
|| (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
|
48
47
|
permissions:
|
|
49
|
-
contents:
|
|
48
|
+
contents: write
|
|
50
49
|
pull-requests: write
|
|
51
50
|
issues: write
|
|
52
51
|
id-token: write
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: mt5cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.5.0
|
|
4
4
|
Summary: Generic MT5 data and execution infrastructure for Python applications
|
|
5
5
|
Project-URL: Repository, https://github.com/dceoy/mt5cli.git
|
|
6
6
|
Author-email: dceoy <dceoy@users.noreply.github.com>
|
|
@@ -50,7 +50,7 @@ Built on top of [pdmt5](https://github.com/dceoy/pdmt5), a pandas-based data han
|
|
|
50
50
|
- **Comprehensive data access**: Rates, ticks, account info, symbols, orders, positions, and trading history
|
|
51
51
|
- **Flexible timeframes**: Named timeframes (M1, H1, D1, etc.) and numeric values
|
|
52
52
|
- **Connection management**: Optional credentials, server, and timeout configuration
|
|
53
|
-
- **SQLite rate loading**: Load
|
|
53
|
+
- **SQLite rate loading**: Load canonical normalized `rates` storage for offline workflows
|
|
54
54
|
|
|
55
55
|
## Installation
|
|
56
56
|
|
|
@@ -69,7 +69,7 @@ pip install -U "mt5cli[parquet]" MetaTrader5
|
|
|
69
69
|
Import `MT5Client` for generic MT5 data access, schema normalization, and optional order primitives.
|
|
70
70
|
|
|
71
71
|
```python
|
|
72
|
-
from datetime import
|
|
72
|
+
from datetime import datetime
|
|
73
73
|
from pathlib import Path
|
|
74
74
|
|
|
75
75
|
from mt5cli import (
|
|
@@ -103,8 +103,8 @@ export_dataframe(closed_rates, Path("rates.csv"), "csv")
|
|
|
103
103
|
collect_history(
|
|
104
104
|
Path("history.db"),
|
|
105
105
|
symbols=["EURUSD"],
|
|
106
|
-
date_from=datetime(2024, 1, 1
|
|
107
|
-
date_to=datetime(2024, 2, 1
|
|
106
|
+
date_from=datetime(2024, 1, 1),
|
|
107
|
+
date_to=datetime(2024, 2, 1),
|
|
108
108
|
datasets={Dataset.rates, Dataset.history_deals},
|
|
109
109
|
)
|
|
110
110
|
|
|
@@ -113,9 +113,16 @@ update_history_with_config(
|
|
|
113
113
|
output="history.db",
|
|
114
114
|
symbols=["EURUSD"],
|
|
115
115
|
config=build_config(login=12345),
|
|
116
|
+
date_to=datetime(2024, 2, 1),
|
|
116
117
|
)
|
|
117
118
|
```
|
|
118
119
|
|
|
120
|
+
MT5 query bounds are timezone-naive trade-server wall-clock datetimes, as
|
|
121
|
+
required by pdmt5. Incremental history updates require an explicit `date_to`
|
|
122
|
+
because mt5cli cannot determine the current trade-server time. Offset-bearing
|
|
123
|
+
strings and timezone-aware `datetime` objects are rejected; mt5cli does not
|
|
124
|
+
infer or strip offsets to perform a conversion.
|
|
125
|
+
|
|
119
126
|
Schema contracts live in `mt5cli.schemas` (`DataKind`, `validate_schema`, `normalize_dataframe`). Export and storage helpers are in `mt5cli.utils` (`Dataset`, `export_dataframe`) and `mt5cli.history`.
|
|
120
127
|
|
|
121
128
|
`MT5Client.order_send()` is a live execution primitive: it can place real trades on the connected account. mt5cli does not implement strategy logic, signal generation, backtesting, or optimization — downstream applications must gate live execution explicitly.
|
|
@@ -335,6 +342,8 @@ OpenTelemetry metrics are documented in [`docs/api/telemetry.md`](docs/api/telem
|
|
|
335
342
|
For automated pipelines, use the importable incremental API instead of re-fetching fixed date ranges:
|
|
336
343
|
|
|
337
344
|
```python
|
|
345
|
+
from datetime import datetime
|
|
346
|
+
|
|
338
347
|
from pdmt5 import Mt5Config
|
|
339
348
|
from mt5cli import update_history_with_config
|
|
340
349
|
from mt5cli.utils import Dataset
|
|
@@ -347,19 +356,17 @@ update_history_with_config(
|
|
|
347
356
|
datasets={Dataset.rates, Dataset.history_deals},
|
|
348
357
|
timeframes=["M1", "H1"], # default: all fixed MT5 timeframes
|
|
349
358
|
lookback_hours=24,
|
|
350
|
-
|
|
359
|
+
date_to=datetime(2024, 2, 1),
|
|
351
360
|
with_views=True,
|
|
352
361
|
include_account_events=True,
|
|
353
362
|
)
|
|
354
363
|
```
|
|
355
364
|
|
|
356
365
|
- **`collect-history`**: explicit date-range export into SQLite.
|
|
357
|
-
- **`update_history`**: incremental append based on existing SQLite `MAX(time)` per symbol (and timeframe for rates); account-level deals use a separate cursor when `include_account_events=True`.
|
|
366
|
+
- **`update_history`**: incremental append based on existing SQLite `MAX(time)` per symbol (and timeframe for rates); account-level deals use a separate cursor when `include_account_events=True`. Pass an explicit naive trade-server `date_to`; mt5cli cannot determine the current server time. Rates are persisted only in the canonical normalized `rates` table.
|
|
358
367
|
- **`rates` table**: normalized storage with `symbol` and `timeframe` columns.
|
|
359
|
-
- **
|
|
360
|
-
- **
|
|
361
|
-
- **Rate view loading**: use `load_rate_data()` / `load_rate_data_from_connection()` to load a SQLite rate table or view into a `DatetimeIndex` DataFrame.
|
|
362
|
-
- **Multi-series rate loading**: use `build_rate_targets()` to build neutral `RateTarget(symbol, timeframe)` pairs, `resolve_rate_tables()` to map them to table/view names (pass `require_existing=True` for strict resolution), and `load_rate_series_from_sqlite()` to load them into a mapping keyed by `(symbol, integer timeframe)`. The loader requires existing managed views unless `explicit_tables` is supplied, and rejects duplicate `(symbol, timeframe)` targets.
|
|
368
|
+
- **Explicit table loading**: use `load_rate_data()` / `load_rate_data_from_connection()` for intentionally named custom SQLite rate tables.
|
|
369
|
+
- **Multi-series rate loading**: use `build_rate_targets()` to build neutral `RateTarget(symbol, timeframe)` pairs and `load_rate_series_from_sqlite()` to load canonical `rates` rows into a mapping keyed by `(symbol, integer timeframe)`. Pass `explicit_tables` only for intentionally named custom tables; duplicate `(symbol, timeframe)` targets are rejected.
|
|
363
370
|
- **Multi-account latest rates**: use `collect_latest_rates_for_accounts()` with `AccountSpec` to read the latest bars for several account groups, merged into a `(symbol, integer timeframe)` mapping. For long-running pollers, `collect_latest_rates_for_accounts_with_retries()` adds bounded exponential backoff that retries only recoverable MT5 errors and re-raises once `retry_count` is exhausted.
|
|
364
371
|
- **Latest closed bars**: use `collect_latest_closed_rates_for_accounts()` when downstream logic must exclude the still-forming current bar. It fetches `count + 1` bars at `start_pos=0`, drops the last row with `drop_forming_rate_bar()`, and validates each series is non-empty. `collect_latest_closed_rates_by_granularity()` returns the same data keyed by `(symbol, granularity_name)` such as `("EURUSD", "M1")`.
|
|
365
372
|
|
|
@@ -376,7 +383,7 @@ eurusd_m1 = rates["EURUSD", "M1"] # closed bars only
|
|
|
376
383
|
```
|
|
377
384
|
|
|
378
385
|
- **Credential resolution**: use `resolve_account_spec()` / `resolve_account_specs()` to merge explicit override values over `AccountSpec` fields and expand `${ENV_VAR}` placeholders (via `substitute_env_placeholders()`), raising `ValueError` for missing variables. This keeps secrets out of plan/config files without coupling to any strategy code. For config dicts or nested structures loaded from YAML/TOML, use `substitute_mapping_values(data, keys={"login", "password"})` to expand placeholders only for caller-specified keys — key names are never hard-coded in mt5cli.
|
|
379
|
-
- **Throttled history updates**: use `ThrottledHistoryUpdater` to wrap `update_history()` with a minimum `interval_seconds` between successful runs (monotonic clock). Call `should_update()` / `update(client, symbols)` from an application loop; errors propagate by default, or pass `suppress_errors=True` to swallow recoverable `Mt5*Error`, `sqlite3.Error`, `ValueError`, and `OSError` without advancing the throttle (`AttributeError` / `TypeError` always propagate, since the client passed to `update()` must implement the canonical `HistoryClient` method names). Pass `update_backend` to inject a custom history update callable (same keyword arguments as `update_history`) instead of monkey-patching `mt5cli.history.update_history`.
|
|
386
|
+
- **Throttled history updates**: use `ThrottledHistoryUpdater` to wrap `update_history()` with a minimum `interval_seconds` between successful runs (monotonic clock). Call `should_update()` / `update(client, symbols, date_to=...)` from an application loop; errors propagate by default, or pass `suppress_errors=True` to swallow recoverable `Mt5*Error`, `sqlite3.Error`, `ValueError`, and `OSError` without advancing the throttle (`AttributeError` / `TypeError` always propagate, since the client passed to `update()` must implement the canonical `HistoryClient` method names). Pass `update_backend` to inject a custom history update callable (same keyword arguments as `update_history`) instead of monkey-patching `mt5cli.history.update_history`.
|
|
380
387
|
- **Trading helpers**: use `mt5_session()` for both market data and generic execution helpers. It owns initialization and shutdown for sessions it creates; caller-supplied clients remain caller-owned.
|
|
381
388
|
- **Granularity-keyed rate loading**: `load_rate_series_by_granularity()` builds targets with `build_rate_targets()`, loads them with `load_rate_series_from_sqlite()`, and returns a mapping keyed by `(symbol | None, granularity_name)` such as `("EURUSD", "M1")` to reduce downstream boilerplate.
|
|
382
389
|
- **MT5 session helper**: use the `mt5_session()` context manager to attach to (or, when `Mt5Config.path` is set, launch) an MT5 terminal, log in, and yield a connected `MT5Client` that shuts down on exit.
|
|
@@ -434,7 +441,7 @@ updater = ThrottledHistoryUpdater(
|
|
|
434
441
|
output="history.db", interval_seconds=60, suppress_errors=True
|
|
435
442
|
)
|
|
436
443
|
with mt5_session(Mt5Config(login=login)) as client:
|
|
437
|
-
updater.update(client, ["EURUSD"])
|
|
444
|
+
updater.update(client, ["EURUSD"], date_to="2024-02-01")
|
|
438
445
|
```
|
|
439
446
|
|
|
440
447
|
Read-only collectors can keep using `mt5_session()` and `MT5Client`.
|
|
@@ -21,7 +21,7 @@ Built on top of [pdmt5](https://github.com/dceoy/pdmt5), a pandas-based data han
|
|
|
21
21
|
- **Comprehensive data access**: Rates, ticks, account info, symbols, orders, positions, and trading history
|
|
22
22
|
- **Flexible timeframes**: Named timeframes (M1, H1, D1, etc.) and numeric values
|
|
23
23
|
- **Connection management**: Optional credentials, server, and timeout configuration
|
|
24
|
-
- **SQLite rate loading**: Load
|
|
24
|
+
- **SQLite rate loading**: Load canonical normalized `rates` storage for offline workflows
|
|
25
25
|
|
|
26
26
|
## Installation
|
|
27
27
|
|
|
@@ -40,7 +40,7 @@ pip install -U "mt5cli[parquet]" MetaTrader5
|
|
|
40
40
|
Import `MT5Client` for generic MT5 data access, schema normalization, and optional order primitives.
|
|
41
41
|
|
|
42
42
|
```python
|
|
43
|
-
from datetime import
|
|
43
|
+
from datetime import datetime
|
|
44
44
|
from pathlib import Path
|
|
45
45
|
|
|
46
46
|
from mt5cli import (
|
|
@@ -74,8 +74,8 @@ export_dataframe(closed_rates, Path("rates.csv"), "csv")
|
|
|
74
74
|
collect_history(
|
|
75
75
|
Path("history.db"),
|
|
76
76
|
symbols=["EURUSD"],
|
|
77
|
-
date_from=datetime(2024, 1, 1
|
|
78
|
-
date_to=datetime(2024, 2, 1
|
|
77
|
+
date_from=datetime(2024, 1, 1),
|
|
78
|
+
date_to=datetime(2024, 2, 1),
|
|
79
79
|
datasets={Dataset.rates, Dataset.history_deals},
|
|
80
80
|
)
|
|
81
81
|
|
|
@@ -84,9 +84,16 @@ update_history_with_config(
|
|
|
84
84
|
output="history.db",
|
|
85
85
|
symbols=["EURUSD"],
|
|
86
86
|
config=build_config(login=12345),
|
|
87
|
+
date_to=datetime(2024, 2, 1),
|
|
87
88
|
)
|
|
88
89
|
```
|
|
89
90
|
|
|
91
|
+
MT5 query bounds are timezone-naive trade-server wall-clock datetimes, as
|
|
92
|
+
required by pdmt5. Incremental history updates require an explicit `date_to`
|
|
93
|
+
because mt5cli cannot determine the current trade-server time. Offset-bearing
|
|
94
|
+
strings and timezone-aware `datetime` objects are rejected; mt5cli does not
|
|
95
|
+
infer or strip offsets to perform a conversion.
|
|
96
|
+
|
|
90
97
|
Schema contracts live in `mt5cli.schemas` (`DataKind`, `validate_schema`, `normalize_dataframe`). Export and storage helpers are in `mt5cli.utils` (`Dataset`, `export_dataframe`) and `mt5cli.history`.
|
|
91
98
|
|
|
92
99
|
`MT5Client.order_send()` is a live execution primitive: it can place real trades on the connected account. mt5cli does not implement strategy logic, signal generation, backtesting, or optimization — downstream applications must gate live execution explicitly.
|
|
@@ -306,6 +313,8 @@ OpenTelemetry metrics are documented in [`docs/api/telemetry.md`](docs/api/telem
|
|
|
306
313
|
For automated pipelines, use the importable incremental API instead of re-fetching fixed date ranges:
|
|
307
314
|
|
|
308
315
|
```python
|
|
316
|
+
from datetime import datetime
|
|
317
|
+
|
|
309
318
|
from pdmt5 import Mt5Config
|
|
310
319
|
from mt5cli import update_history_with_config
|
|
311
320
|
from mt5cli.utils import Dataset
|
|
@@ -318,19 +327,17 @@ update_history_with_config(
|
|
|
318
327
|
datasets={Dataset.rates, Dataset.history_deals},
|
|
319
328
|
timeframes=["M1", "H1"], # default: all fixed MT5 timeframes
|
|
320
329
|
lookback_hours=24,
|
|
321
|
-
|
|
330
|
+
date_to=datetime(2024, 2, 1),
|
|
322
331
|
with_views=True,
|
|
323
332
|
include_account_events=True,
|
|
324
333
|
)
|
|
325
334
|
```
|
|
326
335
|
|
|
327
336
|
- **`collect-history`**: explicit date-range export into SQLite.
|
|
328
|
-
- **`update_history`**: incremental append based on existing SQLite `MAX(time)` per symbol (and timeframe for rates); account-level deals use a separate cursor when `include_account_events=True`.
|
|
337
|
+
- **`update_history`**: incremental append based on existing SQLite `MAX(time)` per symbol (and timeframe for rates); account-level deals use a separate cursor when `include_account_events=True`. Pass an explicit naive trade-server `date_to`; mt5cli cannot determine the current server time. Rates are persisted only in the canonical normalized `rates` table.
|
|
329
338
|
- **`rates` table**: normalized storage with `symbol` and `timeframe` columns.
|
|
330
|
-
- **
|
|
331
|
-
- **
|
|
332
|
-
- **Rate view loading**: use `load_rate_data()` / `load_rate_data_from_connection()` to load a SQLite rate table or view into a `DatetimeIndex` DataFrame.
|
|
333
|
-
- **Multi-series rate loading**: use `build_rate_targets()` to build neutral `RateTarget(symbol, timeframe)` pairs, `resolve_rate_tables()` to map them to table/view names (pass `require_existing=True` for strict resolution), and `load_rate_series_from_sqlite()` to load them into a mapping keyed by `(symbol, integer timeframe)`. The loader requires existing managed views unless `explicit_tables` is supplied, and rejects duplicate `(symbol, timeframe)` targets.
|
|
339
|
+
- **Explicit table loading**: use `load_rate_data()` / `load_rate_data_from_connection()` for intentionally named custom SQLite rate tables.
|
|
340
|
+
- **Multi-series rate loading**: use `build_rate_targets()` to build neutral `RateTarget(symbol, timeframe)` pairs and `load_rate_series_from_sqlite()` to load canonical `rates` rows into a mapping keyed by `(symbol, integer timeframe)`. Pass `explicit_tables` only for intentionally named custom tables; duplicate `(symbol, timeframe)` targets are rejected.
|
|
334
341
|
- **Multi-account latest rates**: use `collect_latest_rates_for_accounts()` with `AccountSpec` to read the latest bars for several account groups, merged into a `(symbol, integer timeframe)` mapping. For long-running pollers, `collect_latest_rates_for_accounts_with_retries()` adds bounded exponential backoff that retries only recoverable MT5 errors and re-raises once `retry_count` is exhausted.
|
|
335
342
|
- **Latest closed bars**: use `collect_latest_closed_rates_for_accounts()` when downstream logic must exclude the still-forming current bar. It fetches `count + 1` bars at `start_pos=0`, drops the last row with `drop_forming_rate_bar()`, and validates each series is non-empty. `collect_latest_closed_rates_by_granularity()` returns the same data keyed by `(symbol, granularity_name)` such as `("EURUSD", "M1")`.
|
|
336
343
|
|
|
@@ -347,7 +354,7 @@ eurusd_m1 = rates["EURUSD", "M1"] # closed bars only
|
|
|
347
354
|
```
|
|
348
355
|
|
|
349
356
|
- **Credential resolution**: use `resolve_account_spec()` / `resolve_account_specs()` to merge explicit override values over `AccountSpec` fields and expand `${ENV_VAR}` placeholders (via `substitute_env_placeholders()`), raising `ValueError` for missing variables. This keeps secrets out of plan/config files without coupling to any strategy code. For config dicts or nested structures loaded from YAML/TOML, use `substitute_mapping_values(data, keys={"login", "password"})` to expand placeholders only for caller-specified keys — key names are never hard-coded in mt5cli.
|
|
350
|
-
- **Throttled history updates**: use `ThrottledHistoryUpdater` to wrap `update_history()` with a minimum `interval_seconds` between successful runs (monotonic clock). Call `should_update()` / `update(client, symbols)` from an application loop; errors propagate by default, or pass `suppress_errors=True` to swallow recoverable `Mt5*Error`, `sqlite3.Error`, `ValueError`, and `OSError` without advancing the throttle (`AttributeError` / `TypeError` always propagate, since the client passed to `update()` must implement the canonical `HistoryClient` method names). Pass `update_backend` to inject a custom history update callable (same keyword arguments as `update_history`) instead of monkey-patching `mt5cli.history.update_history`.
|
|
357
|
+
- **Throttled history updates**: use `ThrottledHistoryUpdater` to wrap `update_history()` with a minimum `interval_seconds` between successful runs (monotonic clock). Call `should_update()` / `update(client, symbols, date_to=...)` from an application loop; errors propagate by default, or pass `suppress_errors=True` to swallow recoverable `Mt5*Error`, `sqlite3.Error`, `ValueError`, and `OSError` without advancing the throttle (`AttributeError` / `TypeError` always propagate, since the client passed to `update()` must implement the canonical `HistoryClient` method names). Pass `update_backend` to inject a custom history update callable (same keyword arguments as `update_history`) instead of monkey-patching `mt5cli.history.update_history`.
|
|
351
358
|
- **Trading helpers**: use `mt5_session()` for both market data and generic execution helpers. It owns initialization and shutdown for sessions it creates; caller-supplied clients remain caller-owned.
|
|
352
359
|
- **Granularity-keyed rate loading**: `load_rate_series_by_granularity()` builds targets with `build_rate_targets()`, loads them with `load_rate_series_from_sqlite()`, and returns a mapping keyed by `(symbol | None, granularity_name)` such as `("EURUSD", "M1")` to reduce downstream boilerplate.
|
|
353
360
|
- **MT5 session helper**: use the `mt5_session()` context manager to attach to (or, when `Mt5Config.path` is set, launch) an MT5 terminal, log in, and yield a connected `MT5Client` that shuts down on exit.
|
|
@@ -405,7 +412,7 @@ updater = ThrottledHistoryUpdater(
|
|
|
405
412
|
output="history.db", interval_seconds=60, suppress_errors=True
|
|
406
413
|
)
|
|
407
414
|
with mt5_session(Mt5Config(login=login)) as client:
|
|
408
|
-
updater.update(client, ["EURUSD"])
|
|
415
|
+
updater.update(client, ["EURUSD"], date_to="2024-02-01")
|
|
409
416
|
```
|
|
410
417
|
|
|
411
418
|
Read-only collectors can keep using `mt5_session()` and `MT5Client`.
|
|
@@ -20,6 +20,13 @@ append using dataset-specific keys (for example `ticket` on history tables, or
|
|
|
20
20
|
Optional views are created when `--with-views` is set and the `history-deals` dataset
|
|
21
21
|
was written.
|
|
22
22
|
|
|
23
|
+
All MT5 history query bounds are timezone-naive trade-server wall-clock
|
|
24
|
+
datetimes, matching pdmt5. Offset-bearing strings and timezone-aware
|
|
25
|
+
`datetime` values are rejected; incremental cursors and returned history
|
|
26
|
+
frames use the same naive contract, and no broker-time conversion is inferred.
|
|
27
|
+
Incremental updates require an explicit `date_to` because mt5cli cannot
|
|
28
|
+
determine the current MT5 trade-server time.
|
|
29
|
+
|
|
23
30
|
`ticks` and `symbols` are opt-in datasets (pass `--dataset ticks` /
|
|
24
31
|
`--dataset symbols`); they are excluded from the default `rates`,
|
|
25
32
|
`history-orders`, `history-deals` selection. Unlike the other datasets,
|
|
@@ -157,135 +164,56 @@ when required columns are missing.
|
|
|
157
164
|
|
|
158
165
|
### Incremental collection
|
|
159
166
|
|
|
160
|
-
The `update_history` SDK path uses the
|
|
161
|
-
`cash_events` / `positions_reconstructed` views. It
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
### Rate view resolution
|
|
165
|
-
|
|
166
|
-
Downstream tools can resolve mt5cli-managed compatibility view names from an
|
|
167
|
-
existing SQLite history database without creating files or guessing naming
|
|
168
|
-
schemes:
|
|
169
|
-
|
|
170
|
-
```python
|
|
171
|
-
from pathlib import Path
|
|
172
|
-
|
|
173
|
-
from mt5cli.history import resolve_rate_view_name, resolve_rate_view_names
|
|
174
|
-
|
|
175
|
-
# Single symbol and granularity
|
|
176
|
-
view = resolve_rate_view_name(Path("history.db"), "EURUSD", "M1")
|
|
177
|
-
|
|
178
|
-
# Batch resolution in row-major order
|
|
179
|
-
views = resolve_rate_view_names(
|
|
180
|
-
Path("history.db"),
|
|
181
|
-
["EURUSD", "GBPUSD"],
|
|
182
|
-
["M1", "H1"],
|
|
183
|
-
)
|
|
184
|
-
```
|
|
185
|
-
|
|
186
|
-
Resolution rules:
|
|
187
|
-
|
|
188
|
-
- Returns `rate_<symbol>__<timeframe>` when a symbol stores one timeframe.
|
|
189
|
-
- Returns `rate_<symbol>__<granularity>_<timeframe>` when multiple timeframes
|
|
190
|
-
are stored for the same symbol.
|
|
191
|
-
- When multiple naming candidates apply, prefers an existing managed
|
|
192
|
-
`rate_*__*` view from the candidate list.
|
|
193
|
-
- Falls back to single-timeframe naming when the database path is missing or
|
|
194
|
-
`rates` metadata is unavailable.
|
|
195
|
-
- Pass `require_existing=True` to raise `ValueError` instead of returning a
|
|
196
|
-
best-guess name when the database or view is missing.
|
|
197
|
-
- Accepts either a SQLite path or an open `sqlite3.Connection`.
|
|
167
|
+
The package-root `mt5cli.update_history` SDK path uses the normalized `rates`
|
|
168
|
+
table and optional `cash_events` / `positions_reconstructed` views. It writes the canonical normalized `rates` table directly and does not
|
|
169
|
+
maintain per-series compatibility views.
|
|
198
170
|
|
|
199
171
|
### Rate data loading
|
|
200
172
|
|
|
201
|
-
The canonical
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
timeframes. `resolve_rate_table_name()` returns `rates`, while
|
|
205
|
-
`resolve_rate_view_name()` returns the per-symbol compatibility view name.
|
|
173
|
+
The canonical managed rate store is the normalized `rates` table. Stable
|
|
174
|
+
symbol/timeframe reads query that table directly; mt5cli does not create,
|
|
175
|
+
discover, resolve, or maintain per-series `rate_*` compatibility views.
|
|
206
176
|
|
|
207
|
-
Use `
|
|
208
|
-
|
|
209
|
-
`load_rate_series_by_granularity()` to load multiple instrument/granularity
|
|
210
|
-
targets without hard-coding view names:
|
|
177
|
+
Use `load_rate_series_from_sqlite()` or
|
|
178
|
+
`load_rate_series_by_granularity()` for canonical managed data:
|
|
211
179
|
|
|
212
180
|
```python
|
|
213
181
|
from pathlib import Path
|
|
214
182
|
|
|
215
183
|
from mt5cli import (
|
|
184
|
+
build_rate_targets,
|
|
216
185
|
load_rate_series_by_granularity,
|
|
217
186
|
load_rate_series_from_sqlite,
|
|
218
187
|
)
|
|
219
|
-
from mt5cli.history import (
|
|
220
|
-
load_rate_data,
|
|
221
|
-
resolve_rate_table_name,
|
|
222
|
-
resolve_rate_view_name,
|
|
223
|
-
)
|
|
224
188
|
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
189
|
+
path = Path("history.db")
|
|
190
|
+
targets = build_rate_targets(["EURUSD", "GBPUSD"], ["M1", "H1"])
|
|
191
|
+
series = load_rate_series_from_sqlite(path, targets, count=1000)
|
|
192
|
+
eurusd_m1 = series["EURUSD", 1]
|
|
228
193
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
Path("history.db"),
|
|
194
|
+
by_name = load_rate_series_by_granularity(
|
|
195
|
+
path,
|
|
232
196
|
symbols=["EURUSD", "GBPUSD"],
|
|
233
197
|
granularities=["M1", "H1"],
|
|
234
198
|
count=500,
|
|
235
199
|
)
|
|
200
|
+
eurusd_m1_named = by_name["EURUSD", "M1"]
|
|
236
201
|
```
|
|
237
202
|
|
|
238
|
-
`count`
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
validates that `time` exists, parses timestamps with pandas, and returns a
|
|
244
|
-
DataFrame indexed by ascending `DatetimeIndex` named `time`.
|
|
245
|
-
|
|
246
|
-
### Multi-series rate loading
|
|
247
|
-
|
|
248
|
-
For loading many rate series at once, build neutral `RateTarget` pairs and load
|
|
249
|
-
them from SQLite in one call. View names are resolved via the same
|
|
250
|
-
compatibility-view rules, or you can pass `explicit_tables` to bypass resolution:
|
|
251
|
-
|
|
252
|
-
```python
|
|
253
|
-
from pathlib import Path
|
|
254
|
-
|
|
255
|
-
from mt5cli import build_rate_targets, load_rate_series_from_sqlite
|
|
203
|
+
Canonical reads apply `count` in SQLite and return the selected rows in
|
|
204
|
+
chronological order. Each returned series uses an ascending `DatetimeIndex`
|
|
205
|
+
named `time`. Timezone-naive MT5 trade-server wall-clock timestamps remain
|
|
206
|
+
naive; explicitly timezone-aware timestamps preserve their instant and are
|
|
207
|
+
normalized to UTC.
|
|
256
208
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
209
|
+
`explicit_tables` and the single-table `table=` form are reserved for
|
|
210
|
+
intentionally named custom tables. They do not discover or fall back to
|
|
211
|
+
managed per-series views. Explicit table counts must match target counts, and
|
|
212
|
+
duplicate `(symbol, timeframe)` targets are rejected.
|
|
261
213
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
`explicit_tables` (targets carry `symbol=None`).
|
|
266
|
-
- `resolve_rate_tables()` maps targets to table or view names and validates that
|
|
267
|
-
any `explicit_tables` count matches the target count. Pass
|
|
268
|
-
`require_existing=True` to raise `ValueError` instead of returning a
|
|
269
|
-
best-guess name when the database or managed view is missing. When
|
|
270
|
-
`explicit_tables` is provided, names are returned as-is and
|
|
271
|
-
`require_existing` is ignored.
|
|
272
|
-
- `load_rate_series_from_sqlite()` returns a mapping keyed by
|
|
273
|
-
`(symbol, integer timeframe)`. Unless `explicit_tables` is supplied, it
|
|
274
|
-
requires existing managed `rate_*__*` compatibility views and raises
|
|
275
|
-
`ValueError` when they are missing. Duplicate `(symbol, timeframe)` targets
|
|
276
|
-
are rejected.
|
|
277
|
-
- `load_rate_series_by_granularity()` is a thin wrapper that builds the targets,
|
|
278
|
-
loads the series, and rekeys the result by granularity name to avoid
|
|
279
|
-
converting integer timeframes downstream:
|
|
280
|
-
|
|
281
|
-
```python
|
|
282
|
-
from mt5cli import load_rate_series_by_granularity
|
|
283
|
-
|
|
284
|
-
series = load_rate_series_by_granularity(
|
|
285
|
-
"history.db", ["EURUSD"], ["M1", "H1"], count=1000
|
|
286
|
-
)
|
|
287
|
-
frame = series["EURUSD", "M1"] # keyed by (symbol | None, granularity_name)
|
|
288
|
-
```
|
|
214
|
+
`load_rate_data()` / `load_rate_data_from_connection()` remain available for
|
|
215
|
+
low-level explicit-table reads. These helpers validate that `time` exists and
|
|
216
|
+
return an ascending `DatetimeIndex` named `time`.
|
|
289
217
|
|
|
290
218
|
## Throttled incremental history updates
|
|
291
219
|
|
|
@@ -294,6 +222,8 @@ between successful runs (using a monotonic clock), so an application loop can
|
|
|
294
222
|
call it every iteration without over-fetching.
|
|
295
223
|
|
|
296
224
|
```python
|
|
225
|
+
from datetime import datetime
|
|
226
|
+
|
|
297
227
|
from pdmt5 import Mt5Config
|
|
298
228
|
|
|
299
229
|
from mt5cli import ThrottledHistoryUpdater, mt5_session
|
|
@@ -308,7 +238,11 @@ updater = ThrottledHistoryUpdater(
|
|
|
308
238
|
|
|
309
239
|
with mt5_session(Mt5Config(login=12345)) as client:
|
|
310
240
|
while True:
|
|
311
|
-
updater.update(
|
|
241
|
+
updater.update(
|
|
242
|
+
client,
|
|
243
|
+
["EURUSD", "GBPUSD"],
|
|
244
|
+
date_to=datetime(2024, 2, 1),
|
|
245
|
+
) # no-op until 60s elapse
|
|
312
246
|
# ... do other work; break when shutting down ...
|
|
313
247
|
```
|
|
314
248
|
|
|
@@ -316,7 +250,7 @@ Pass `update_backend` to substitute the default `update_history` implementation
|
|
|
316
250
|
without monkey-patching `mt5cli.history.update_history`. The callable receives the
|
|
317
251
|
same keyword arguments as `update_history` (`client`, `output`, `symbols`,
|
|
318
252
|
`datasets`, `timeframes`, `flags`, `lookback_hours`, `with_views`,
|
|
319
|
-
`include_account_events`). The resolved backend is stored on
|
|
253
|
+
`date_to`, `include_account_events`). The resolved backend is stored on
|
|
320
254
|
`updater.update_backend` for inspection or subclassing.
|
|
321
255
|
|
|
322
256
|
```python
|