mtcli 3.8.0.dev3__tar.gz → 3.8.0.dev5__tar.gz

{mtcli-3.8.0.dev3 → mtcli-3.8.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mtcli
-Version: 3.8.0.dev3
+Version: 3.8.0.dev5
 Summary: Aplicativo CLI para exibir gráficos do MetaTrader 5 screen reader friendly
 License-Expression: MIT
 License-File: LICENSE

{mtcli-3.8.0.dev3 → mtcli-3.8.0.dev5}/mtcli/cli.py

@@ -1,63 +1,74 @@
 """
 CLI principal do mtcli.
+
+Responsável por:
+
+- inicializar o ambiente do CLI
+- carregar plugins
+- iniciar captura automática de ticks (opcional)
+
+A captura automática é ativada se a variável de ambiente
+MTCLI_SYMBOL estiver definida.
+
+Exemplo:
+
+    MTCLI_SYMBOL=WIN$N mt bars
+
+Nesse caso o mtcli inicia um TickEngine em background
+para manter o histórico de ticks atualizado.
 """
 
 import os
-import threading
 import click
 
 from mtcli.plugin_loader import load_plugins
 from mtcli.logger import setup_logger
-from mtcli.marketdata.tick_streamer import TickStreamer
+from mtcli.marketdata.tick_engine import TickEngine
 
 from .commands.bars import bars
 from .commands.doctor import doctor
 from .commands.migrate import migrate
+from .commands.ticks import ticks
 
 
 logger = setup_logger(__name__)
 
-_tick_streamer = None
+_tick_engine = None
 
 
 def start_tick_capture():
     """
     Inicia captura contínua de ticks em background.
 
-    O símbolo deve ser definido via variável:
+    Se a variável de ambiente MTCLI_SYMBOL estiver definida,
+    o mtcli iniciará automaticamente um TickEngine para
+    coletar ticks continuamente.
 
-        MTCLI_SYMBOL=WINJ26
+    Isso permite manter um histórico próprio de ticks
+    independente do histórico do broker.
     """
 
-    global _tick_streamer
+    global _tick_engine
 
-    if _tick_streamer:
+    if _tick_engine:
         return
 
     symbol = os.getenv("MTCLI_SYMBOL")
 
     if not symbol:
-        logger.info("Captura de ticks desativada (MTCLI_SYMBOL não definido).")
+        logger.info("Captura automática de ticks desativada (MTCLI_SYMBOL não definido).")
         return
 
     logger.info("Iniciando captura contínua de ticks para %s", symbol)
 
     try:
 
-        _tick_streamer = TickStreamer(symbol)
-
-        thread = threading.Thread(
-            target=_tick_streamer.start,
-            daemon=True,
-            name="mtcli-tick-streamer",
-        )
-
-        thread.start()
+        _tick_engine = TickEngine([symbol])
+        _tick_engine.start()
 
         logger.info("Captura de ticks iniciada em background.")
 
     except Exception:
-
         logger.exception("Falha ao iniciar captura de ticks")
 
 
@@ -79,6 +90,7 @@ mt.add_command(doctor, name="doctor")
 mt.add_command(bars, name="bars")
 mt.add_command(doctor, name="dr")
 mt.add_command(migrate)
+mt.add_command(ticks)
 
 loaded_plugins = load_plugins(mt)
 
@@ -88,7 +100,7 @@ logger.info("Plugins carregados: %s", loaded_plugins)
 @mt.command(name="plugins")
 def list_plugins():
     """
-    Lista os plugins carregados.
+    Lista os plugins carregados no mtcli.
     """
 
     if not loaded_plugins:

mtcli-3.8.0.dev5/mtcli/commands/ticks.py

@@ -0,0 +1,47 @@
+"""
+Comando CLI para captura contínua de ticks.
+
+Permite iniciar um TickEngine manualmente
+para um ou mais símbolos.
+
+Exemplo:
+
+    mt ticks WIN$N
+    mt ticks WIN$N WDO$N PETR4
+
+A captura continua até o usuário interromper
+com Ctrl+C.
+"""
+
+import time
+import click
+
+from mtcli.marketdata.tick_engine import TickEngine
+
+
+@click.command()
+@click.argument("symbols", nargs=-1)
+def ticks(symbols):
+    """
+    Inicia captura contínua de ticks para os símbolos informados.
+    """
+
+    if not symbols:
+        click.echo("Informe ao menos um símbolo.")
+        return
+
+    engine = TickEngine(symbols)
+
+    click.echo("Iniciando captura de ticks...")
+
+    engine.start()
+
+    try:
+
+        while True:
+            time.sleep(1)
+
+    except KeyboardInterrupt:
+
+        click.echo("\nEncerrando captura de ticks...")
+        engine.stop()

{mtcli-3.8.0.dev3 → mtcli-3.8.0.dev5}/mtcli/database.py

@@ -15,28 +15,23 @@ from pathlib import Path
 from datetime import datetime
 from .conf import DB_NAME
 
+
 DB_PATH = Path.home() / ".mtcli" / DB_NAME
 BACKUP_DIR = Path.home() / ".mtcli" / "backups"
 
+_connection = None
+
 
 def get_connection():
     """
-    Cria ou retorna uma conexão SQLite otimizada para ingestão
-    contínua de ticks de mercado.
-
-    Configurações aplicadas:
+    Retorna conexão singleton SQLite otimizada para ingestão
+    contínua de ticks.
+    """
 
-    - WAL (Write Ahead Logging)
-    - synchronous=NORMAL
-    - temp_store em memória
-    - mmap para leitura rápida
-    - cache expandido
+    global _connection
 
-    Returns
-    -------
-    sqlite3.Connection
-        Conexão ativa com o banco SQLite.
-    """
+    if _connection:
+        return _connection
 
     DB_PATH.parent.mkdir(parents=True, exist_ok=True)
     BACKUP_DIR.mkdir(parents=True, exist_ok=True)
@@ -50,6 +45,9 @@ def get_connection():
     conn.execute("PRAGMA cache_size=-200000")
     conn.execute("PRAGMA journal_size_limit=67108864")
 
+    # checkpoint automático
+    conn.execute("PRAGMA wal_autocheckpoint=1000")
+
     conn.execute("""
     CREATE TABLE IF NOT EXISTS schema_migrations(
         version INTEGER PRIMARY KEY,
@@ -59,22 +57,9 @@ def get_connection():
 
     conn.commit()
 
-    return conn
-
+    _connection = conn
 
-# ==========================================================
-# CHECKPOINT
-# ==========================================================
-
-def wal_checkpoint(conn):
-    """
-    Executa checkpoint do WAL.
-
-    Move os dados do arquivo `.wal` para o banco principal
-    e reduz seu tamanho.
-    """
-
-    conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+    return conn
 
 
 # ==========================================================
@@ -84,9 +69,6 @@ def wal_checkpoint(conn):
 def backup_database(conn):
     """
     Realiza backup diário seguro do banco SQLite.
-
-    O backup utiliza a API nativa do SQLite,
-    permitindo cópia consistente mesmo com o banco em uso.
     """
 
     now = datetime.now().strftime("%Y%m%d")

mtcli-3.8.0.dev5/mtcli/marketdata/tick_engine.py

@@ -0,0 +1,134 @@
+"""
+TickEngine - motor de captura contínua de ticks.
+
+Responsável por:
+
+- sincronizar histórico inicial
+- capturar ticks em tempo real
+- persistir ticks via TickRepository
+"""
+
+import time
+import threading
+import MetaTrader5 as mt5
+
+from datetime import datetime
+
+from mtcli.mt5_context import mt5_conexao
+from .tick_repository import TickRepository
+
+
+class TickEngine:
+
+    POLL_INTERVAL = 0.2
+    BATCH_SIZE = 1000
+    OVERLAP_MS = 5
+
+    def __init__(self, symbols):
+
+        self.symbols = symbols
+
+        self.repositories = {
+            symbol: TickRepository()
+            for symbol in symbols
+        }
+
+        self.running = False
+        self.thread = None
+
+    def start(self):
+
+        if self.running:
+            return
+
+        self.running = True
+
+        self.thread = threading.Thread(
+            target=self._run,
+            daemon=True,
+            name="mtcli-tick-engine",
+        )
+
+        self.thread.start()
+
+    def stop(self):
+
+        self.running = False
+
+        if self.thread:
+            self.thread.join()
+
+    def _run(self):
+
+        with mt5_conexao():
+
+            last_positions = {}
+
+            for symbol in self.symbols:
+
+                repo = self.repositories[symbol]
+
+                # sincroniza histórico
+                repo.sync(symbol)
+
+                last_msc = repo._get_last_tick_msc(symbol)
+
+                if last_msc:
+                    last_positions[symbol] = last_msc
+                else:
+                    last_positions[symbol] = int(time.time() * 1000)
+
+            while self.running:
+
+                for symbol in self.symbols:
+                    self._drain_symbol(symbol, last_positions)
+
+                time.sleep(self.POLL_INTERVAL)
+
+    def _drain_symbol(self, symbol, last_positions):
+
+        repo = self.repositories[symbol]
+
+        last_msc = last_positions[symbol]
+
+        start_dt = datetime.fromtimestamp(
+            (last_msc - self.OVERLAP_MS) / 1000
+        )
+
+        while True:
+
+            ticks = mt5.copy_ticks_from(
+                symbol,
+                start_dt,
+                self.BATCH_SIZE,
+                mt5.COPY_TICKS_ALL,
+            )
+
+            if ticks is None or len(ticks) == 0:
+                break
+
+            repo.conn.execute("BEGIN")
+
+            try:
+
+                repo._insert_ticks(symbol, ticks)
+
+                repo.cache.add_many(ticks)
+
+                repo.conn.commit()
+
+            except Exception:
+
+                repo.conn.rollback()
+                raise
+
+            last_msc = int(ticks[-1]["time_msc"])
+
+            last_positions[symbol] = last_msc + 1
+
+            start_dt = datetime.fromtimestamp(
+                (last_msc - self.OVERLAP_MS) / 1000
+            )
+
+            if len(ticks) < self.BATCH_SIZE:
+                break

{mtcli-3.8.0.dev3 → mtcli-3.8.0.dev5}/mtcli/marketdata/tick_repository.py

@@ -3,91 +3,90 @@ TickRepository.
 
 Responsável por:
 
-- Persistir ticks no SQLite
-- Sincronizar histórico inicial
-- Consultas rápidas para engines (Renko etc)
+- persistência de ticks
+- sincronização histórica
+- consultas rápidas
 """
 
 import MetaTrader5 as mt5
+
 from datetime import datetime, timedelta
 
-from ..database import get_connection, wal_checkpoint, backup_database
+from ..database import get_connection, backup_database
 from .tick_cache import TickCache
 from mtcli.mt5_context import mt5_conexao
 
 
 class TickRepository:
 
-    BATCH_SIZE = 200000
+    RANGE_WINDOW_MINUTES = 10
 
     def __init__(self):
 
         self.conn = get_connection()
         self.cache = TickCache()
 
-        self.insert_counter = 0
         self.last_backup_day = None
 
     # ==========================================================
-    # SINCRONIZAÇÃO HISTÓRICA
+    # SYNC HISTÓRICO
     # ==========================================================
 
     def sync(self, symbol: str, days_back: int = 1):
-        """
-        Sincroniza histórico de ticks a partir do broker.
-        """
 
         total_inserted = 0
 
-        with mt5_conexao():
+        end = datetime.now()
+
+        last_msc = self._get_last_tick_msc(symbol)
 
-            last_msc = self._get_last_tick_msc(symbol)
+        if last_msc:
+            start = datetime.fromtimestamp((last_msc + 1) / 1000)
+        else:
+            start = end - timedelta(days=days_back)
 
-            if last_msc:
-                start = datetime.fromtimestamp((last_msc + 1) / 1000)
-            else:
-                start = datetime.now() - timedelta(days=days_back)
+        window = timedelta(minutes=self.RANGE_WINDOW_MINUTES)
 
-            self.conn.execute("BEGIN")
+        with mt5_conexao():
+
+            while start < end:
 
-            try:
+                chunk_end = min(start + window, end)
 
-                while True:
+                ticks = mt5.copy_ticks_range(
+                    symbol,
+                    start,
+                    chunk_end,
+                    mt5.COPY_TICKS_ALL
+                )
 
-                    ticks = mt5.copy_ticks_from(
-                        symbol,
-                        start,
-                        self.BATCH_SIZE,
-                        mt5.COPY_TICKS_ALL,
-                    )
+                if ticks is not None and len(ticks) > 0:
 
-                    if ticks is None or len(ticks) == 0:
-                        break
+                    self.conn.execute("BEGIN")
 
-                    inserted = self._insert_ticks(symbol, ticks)
+                    try:
 
-                    total_inserted += inserted
+                        inserted = self._insert_ticks(symbol, ticks)
 
-                    self.cache.add_many(ticks)
+                        total_inserted += inserted
 
-                    last_msc = int(ticks[-1]["time_msc"])
+                        self.cache.add_many(ticks)
 
-                    start = datetime.fromtimestamp((last_msc + 1) / 1000)
+                        self.conn.commit()
 
-                    if len(ticks) < self.BATCH_SIZE:
-                        break
+                    except Exception:
 
-                self.conn.commit()
+                        self.conn.rollback()
+                        raise
 
-            except Exception:
+                start = chunk_end
 
-                self.conn.rollback()
-                raise
+        self._daily_backup()
 
         return total_inserted
 
     # ==========================================================
-    # INSERÇÃO
+    # INSERT
     # ==========================================================
 
     def _insert_ticks(self, symbol, ticks):
@@ -114,28 +113,12 @@ class TickRepository:
                 symbol,time,time_msc,bid,ask,last,volume,flags
             )
             VALUES (?,?,?,?,?,?,?,?)
-            ON CONFLICT(symbol,time) DO NOTHING
+            ON CONFLICT(symbol,time_msc) DO NOTHING
             """,
             data,
         )
 
-        inserted = len(data)
-
-        self.insert_counter += inserted
-
-        if self.insert_counter >= 200000:
-
-            wal_checkpoint(self.conn)
-            self.insert_counter = 0
-
-        today = datetime.now().date()
-
-        if self.last_backup_day != today:
-
-            backup_database(self.conn)
-            self.last_backup_day = today
-
-        return inserted
+        return len(data)
 
     # ==========================================================
     # CONSULTAS
@@ -180,7 +163,7 @@ class TickRepository:
         return cursor.fetchall()
 
     # ==========================================================
-    # UTILITÁRIOS
+    # UTIL
     # ==========================================================
 
     def _get_last_tick_msc(self, symbol):
@@ -196,6 +179,15 @@ class TickRepository:
             (symbol,),
         )
 
-        result = cursor.fetchone()
+        row = cursor.fetchone()
+
+        return row[0] if row and row[0] else None
+
+    def _daily_backup(self):
 
-        return result[0] if result and result[0] else None
+        today = datetime.now().date()
+
+        if self.last_backup_day != today:
+
+            backup_database(self.conn)
+            self.last_backup_day = today

mtcli-3.8.0.dev5/mtcli/migrations/004_ticks_pk_time_msc.py

@@ -0,0 +1,56 @@
+def upgrade(conn):
+
+    cursor = conn.execute("""
+        SELECT sql
+        FROM sqlite_master
+        WHERE type='table'
+        AND name='ticks'
+    """)
+
+    row = cursor.fetchone()
+
+    if not row:
+        return
+
+    if "PRIMARY KEY(symbol, time_msc)" in row[0]:
+        return
+
+    print("Rebuilding ticks table with PK(symbol,time_msc)...")
+
+    conn.execute("PRAGMA foreign_keys=OFF")
+
+    conn.execute("""
+    CREATE TABLE ticks_new(
+        symbol TEXT NOT NULL,
+        time INTEGER NOT NULL,
+        time_msc INTEGER NOT NULL,
+        bid REAL,
+        ask REAL,
+        last REAL,
+        volume REAL,
+        flags INTEGER,
+        PRIMARY KEY(symbol, time_msc)
+    ) WITHOUT ROWID
+    """)
+
+    conn.execute("""
+    INSERT INTO ticks_new
+    SELECT
+        symbol,
+        time,
+        COALESCE(time_msc,time),
+        bid,
+        ask,
+        last,
+        volume,
+        flags
+    FROM ticks
+    """)
+
+    conn.execute("DROP TABLE ticks")
+
+    conn.execute("ALTER TABLE ticks_new RENAME TO ticks")
+
+    conn.commit()
+
+    print("ticks table rebuilt with PK(symbol,time_msc)")

{mtcli-3.8.0.dev3 → mtcli-3.8.0.dev5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mtcli"
-version = "3.8.0.dev3"
+version = "3.8.0.dev5"
 description = "Aplicativo CLI para exibir gráficos do MetaTrader 5 screen reader friendly"
 authors = [
     {name = "Valmir França",email = "vfranca3@gmail.com"}

mtcli-3.8.0.dev3/mtcli/marketdata/tick_engine.py

@@ -1,93 +0,0 @@
-"""
-Motor de captura de ticks multi-ativo.
-"""
-
-import time
-import threading
-import MetaTrader5 as mt5
-from datetime import datetime
-
-from mtcli.mt5_context import mt5_conexao
-from .tick_repository import TickRepository
-
-
-class TickEngine:
-
-    POLL_INTERVAL = 0.2
-
-    def __init__(self, symbols):
-
-        self.symbols = symbols
-
-        self.repositories = {
-            symbol: TickRepository()
-            for symbol in symbols
-        }
-
-        self.running = False
-
-    def start(self):
-
-        thread = threading.Thread(
-            target=self._run,
-            daemon=True
-        )
-
-        self.running = True
-        thread.start()
-
-    def stop(self):
-
-        self.running = False
-
-    def _run(self):
-
-        with mt5_conexao():
-
-            last_positions = {}
-
-            for symbol in self.symbols:
-
-                repo = self.repositories[symbol]
-                last_msc = repo._get_last_tick_msc(symbol)
-
-                if last_msc:
-                    last_positions[symbol] = last_msc + 1
-                else:
-                    last_positions[symbol] = int(time.time() * 1000)
-
-            while self.running:
-
-                for symbol in self.symbols:
-
-                    repo = self.repositories[symbol]
-
-                    start = last_positions[symbol]
-
-                    ticks = mt5.copy_ticks_from(
-                        symbol,
-                        datetime.fromtimestamp(start / 1000),
-                        1000,
-                        mt5.COPY_TICKS_ALL
-                    )
-
-                    if ticks is None or len(ticks) == 0:
-                        continue
-
-                    repo.conn.execute("BEGIN")
-
-                    try:
-
-                        repo._insert_ticks(symbol, ticks)
-
-                        repo.cache.add_many(ticks)
-
-                        repo.conn.commit()
-
-                    except Exception:
-
-                        repo.conn.rollback()
-
-                    last_positions[symbol] = int(ticks[-1]["time_msc"]) + 1
-
-                time.sleep(self.POLL_INTERVAL)

mtcli-3.8.0.dev3/mtcli/marketdata/tick_streamer.py

@@ -1,75 +0,0 @@
-"""
-Captura contínua de ticks.
-
-Objetivo:
-
-- eliminar dependência do histórico do broker
-- manter histórico próprio
-"""
-
-import time
-import MetaTrader5 as mt5
-from datetime import datetime
-
-from mtcli.mt5_context import mt5_conexao
-from .tick_repository import TickRepository
-
-
-class TickStreamer:
-
-    def __init__(self, symbol):
-
-        self.symbol = symbol
-        self.repo = TickRepository()
-
-        self.running = False
-
-    def start(self):
-        """
-        Inicia captura contínua de ticks.
-        """
-
-        self.running = True
-
-        with mt5_conexao():
-
-            last_msc = self.repo._get_last_tick_msc(self.symbol)
-
-            if last_msc:
-                start = last_msc + 1
-            else:
-                start = int(time.time() * 1000)
-
-            while self.running:
-
-                ticks = mt5.copy_ticks_from(
-                    self.symbol,
-                    datetime.fromtimestamp(start / 1000),
-                    1000,
-                    mt5.COPY_TICKS_ALL,
-                )
-
-                if ticks is None or len(ticks) == 0:
-
-                    time.sleep(0.1)
-                    continue
-
-                self.repo.conn.execute("BEGIN")
-
-                try:
-
-                    self.repo._insert_ticks(self.symbol, ticks)
-
-                    self.repo.cache.add_many(ticks)
-
-                    self.repo.conn.commit()
-
-                except Exception:
-
-                    self.repo.conn.rollback()
-
-                start = int(ticks[-1]["time_msc"]) + 1
-
-    def stop(self):
-
-        self.running = False