ib-connect 0.2.0__py3-none-any.whl

ib_connect/__init__.py

@@ -0,0 +1 @@
+# IB Connect Package

ib_connect/gui/__init__.py

@@ -0,0 +1 @@
+# GUI Package

ib_connect/gui/app.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+"""
+IB Data Downloader GUI - Flask Web App
+"""
+
+from flask import Flask, render_template, request, jsonify
+import subprocess
+import json
+import os
+import sys
+import asyncio
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+
+
+from ib_connect.skills.ib_query.query import query_ib
+from ib_connect.skills.ib_download.job_queue import JobQueue
+
+app = Flask(__name__)
+logging.getLogger('werkzeug').setLevel(logging.ERROR)
+
+# Config
+CONFIG_FILE = 'config.json'
+DEFAULT_CONFIG = {
+    'contracts_folder': './data/contracts',
+    'ib_host': '127.0.0.1',
+    'ib_port': 7497,
+    'ib_client_id': 77
+}
+
+def load_config():
+    if os.path.exists(CONFIG_FILE):
+        with open(CONFIG_FILE, 'r') as f:
+            return json.load(f)
+    return DEFAULT_CONFIG
+
+def save_config(config):
+    with open(CONFIG_FILE, 'w') as f:
+        json.dump(config, f, indent=2)
+
+config = load_config()
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+@app.route('/query', methods=['POST'])
+def query_contracts():
+    data = request.json
+    # Include configured IB params to avoid conflicts
+    data['host'] = config.get('ib_host', '127.0.0.1')
+    data['port'] = config.get('ib_port', 7497)
+    data['client_id'] = config.get('ib_client_id', 1)
+    def run_query():
+        return asyncio.run(query_ib(data))
+
+    try:
+        with ThreadPoolExecutor() as executor:
+            future = executor.submit(run_query)
+            result = future.result()
+        if isinstance(result, dict) and 'error' in result:
+            return jsonify({'success': False, 'error': result['error']})
+        contracts = result if isinstance(result, list) else [result] if result else []
+        contracts = [c for c in contracts if c is not None]  # Filter out nulls
+        return jsonify({'success': True, 'contracts': contracts})
+    except Exception as e:
+        return jsonify({'success': False, 'error': str(e)})
+
+@app.route('/save_contracts', methods=['POST'])
+def save_contracts():
+    data = request.json
+    selected = data.get('selected', [])
+    # Add defaults for missing required fields
+    for contract in selected:
+        if 'time_zone_id' not in contract or not contract['time_zone_id']:
+            contract['time_zone_id'] = 'US/Eastern'
+        if 'min_tick' not in contract or not contract['min_tick']:
+            contract['min_tick'] = 0.01
+        if 'tick_value' not in contract or not contract['tick_value']:
+            contract['tick_value'] = contract.get('min_tick', 0.01) * contract.get('multiplier', 1)
+        if 'multiplier' not in contract or not contract['multiplier']:
+            contract['multiplier'] = 1
+    # Omit empty strings and zero values, keep None for db defaults
+    cleaned_selected = []
+    for contract in selected:
+        cleaned = {k: v for k, v in contract.items() if v not in ('', 0)}
+        cleaned_selected.append(cleaned)
+    folder = config['contracts_folder']
+    os.makedirs(folder, exist_ok=True)
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    file_path = os.path.join(folder, f'selected_contracts_{timestamp}.json')
+    with open(file_path, 'w') as f:
+        json.dump(cleaned_selected, f, indent=2)
+    return jsonify({'success': True, 'file': file_path})
+
+@app.route('/download', methods=['POST'])
+def download_data():
+    data = request.json
+    contracts = data.get('contracts', [])
+    downloads_folder = config.get('downloads_folder', './data/downloads')
+    job_queue_db = config.get('job_queue_db', './jobs.db')
+    queue = JobQueue(job_queue_db)
+    jobs = []
+    for contract in contracts:
+        params = {
+            'conid': contract['conid'],
+            'start': contract['start_date'],
+            'end': contract['end_date'],
+            'bar_size': contract['bar_size'],
+            'show': contract.get('show', 'TRADES'),
+            'msg': f"Download {contract['symbol']} data ({contract.get('show', 'TRADES')})"
+        }
+        try:
+            job_key = queue.submit_job(params)
+            jobs.append({'contract': contract['symbol'], 'job_key': job_key, 'status': 'submitted'})
+        except Exception as e:
+            jobs.append({'contract': contract['symbol'], 'error': str(e)})
+    return jsonify({'jobs': jobs})
+
+@app.route('/job_status/<job_key>')
+def job_status(job_key):
+    try:
+        job_queue_db = config['job_queue_db']
+        queue = JobQueue(job_queue_db)
+        status_data = queue.get_status(job_key)
+        if status_data['status'] == 'not_found':
+            return jsonify({'status': 'error', 'details': 'Job not found'})
+        details = f"Status: {status_data.get('status', 'unknown')}"
+        if status_data.get('message'):
+            details += f" - {status_data['message']}"
+        if status_data.get('error'):
+            details += f" - Error: {status_data['error']}"
+        return jsonify({'status': 'ok', 'details': details})
+    except Exception as e:
+        return jsonify({'status': 'error', 'details': str(e)})
+
+@app.route('/cancel_job/<job_key>', methods=['POST'])
+def cancel_job(job_key):
+    try:
+        job_queue_db = config['job_queue_db']
+        queue = JobQueue(job_queue_db)
+        status_data = queue.get_status(job_key)
+        if status_data['status'] == 'pending':
+            queue.remove_job(job_key)
+            return jsonify({'success': True})
+        else:
+            return jsonify({'success': False, 'error': 'Job not pending'})
+    except Exception as e:
+        return jsonify({'success': False, 'error': str(e)})
+
+if __name__ == '__main__':
+    app.run(debug=True, host=config.get('host', '127.0.0.1'), port=config.get('port', 5000))

ib_connect/shared/__init__.py

@@ -0,0 +1 @@
+# Shared utilities for IB Connect

ib_connect/shared/ib_connection.py

@@ -0,0 +1,34 @@
+from ib_insync import IB
+
+class IBConnection:
+    """
+    Connection management for Interactive Brokers using ib_insync.
+    Provides a connected IB instance that can be used as a context manager.
+    """
+
+    @staticmethod
+    def connect(host='127.0.0.1', port=7497, clientId=1, timeout=4.0, readonly=False, account=''):
+        """
+        Establish a connection to IB and return the connected IB instance (which is a context manager).
+        
+        Args:
+            host (str): IB Gateway/TWS host. Default: '127.0.0.1'
+            port (int): IB Gateway/TWS port. Default: 7497
+            clientId (int): Client ID for the connection. Default: 1
+            timeout (float): Connection timeout in seconds. Default: 4.0
+            readonly (bool): Read-only connection. Default: False
+            account (str): Account to use (optional). Default: ''
+        
+        Returns:
+            IB: Connected IB instance (supports context manager: with ib: ...)
+        
+        Raises:
+            ConnectionError: If connection fails.
+        """
+        ib = IB()
+        try:
+            ib.connect(host=host, port=port, clientId=clientId, timeout=timeout, readonly=readonly, account=account)
+        except Exception as e:
+            raise ConnectionError(f"Failed to connect to Interactive Brokers: {e}") from e
+        
+        return ib

ib_connect/skills/__init__.py

@@ -0,0 +1 @@
+# Skills for IB Connect

ib_connect/skills/data_upload/__init__.py

@@ -0,0 +1 @@
+# Data Upload skill

ib_connect/skills/data_upload/data_upload.py

@@ -0,0 +1,371 @@
+#!/usr/bin/env python3
+"""
+Data Upload Service
+
+Monitors folder for JSON/CSV files, processes contracts and OHLCV data, inserts to PostgreSQL.
+"""
+
+import os
+import json
+import logging
+import hashlib
+import shutil
+import time
+import pandas as pd
+import psycopg2
+import psutil
+from psycopg2.extras import execute_values
+import pytz
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+# Expected contract schema keys
+CONTRACT_KEYS = {
+    'conid', 'symbol', 'local_symbol', 'exchange', 'currency', 'sec_type',
+    'long_name', 'industry', 'category', 'sub_category', 'min_tick', 'tick_value',
+    'contract_month', 'expiration_date', 'under_conid', 'strike', 'right', 'multiplier', 'time_zone_id'
+}
+
+def compute_conid(symbol, exchange, currency, sec_type):
+    """Compute hash as conid for non-IB contracts (fits in signed BIGINT)."""
+    key = f"{symbol}{exchange}{currency}{sec_type}".encode('utf-8')
+    hash_obj = hashlib.sha256(key)
+    # Take first 7 bytes (56 bits) to fit in signed BIGINT
+    return int.from_bytes(hash_obj.digest()[:7], byteorder='big', signed=False)
+
+# Required contract schema keys
+REQUIRED_KEYS = {'symbol', 'exchange', 'currency', 'sec_type', 'min_tick', 'tick_value', 'multiplier', 'time_zone_id'}
+
+def validate_contract(data):
+    """Validate contract JSON against schema."""
+    if not isinstance(data, dict):
+        return False
+    # Check required keys are present
+    if not REQUIRED_KEYS.issubset(data.keys()):
+        missing = REQUIRED_KEYS - set(data.keys())
+        logging.error(f"Missing required keys: {missing}")
+        return False
+    # Check no invalid keys
+    invalid = set(data.keys()) - CONTRACT_KEYS
+    if invalid:
+        logging.error(f"Invalid keys: {invalid}")
+        return False
+    # Basic type checks
+    if 'conid' in data and not isinstance(data['conid'], (int, type(None))):
+        logging.error("conid must be int or None")
+        return False
+    if not isinstance(data['min_tick'], (int, float)):
+        logging.error("min_tick must be number")
+        return False
+    if not isinstance(data['tick_value'], (int, float)):
+        logging.error("tick_value must be number")
+        return False
+    if not isinstance(data['multiplier'], (int, float)):
+        logging.error("multiplier must be number")
+        return False
+    return True
+
+def process_contract_file(filepath, conn, schema, table):
+    """Process a contract JSON file (single object or array)."""
+    try:
+        with open(filepath, 'r') as f:
+            data = json.load(f)
+
+        if isinstance(data, list):
+            # Array of contracts
+            success = True
+            for item in data:
+                if not process_single_contract(item, conn, schema, table):
+                    success = False
+            return success
+        else:
+            # Single contract
+            return process_single_contract(data, conn, schema, table)
+    except Exception as e:
+        logging.error(f"Error processing {filepath}: {e}", exc_info=True)
+        return False
+
+def normalize_tz(tz_str):
+    """Normalize timezone string to pytz format."""
+    parts = tz_str.split('-')
+    if len(parts) == 2 and parts[0].lower() == 'us':
+        return f"US/{parts[1].capitalize()}"
+    else:
+        return tz_str.upper()
+
+def parse_csv_header(filepath, barsize_tables):
+    """Parse CSV header and return metadata."""
+    filename = os.path.basename(filepath)
+    if not filename.endswith('.csv'):
+        raise ValueError("Not a CSV file")
+
+    parts = filename[:-4].split('.')
+    if len(parts) == 5:
+        symbol, conid_str, barsize_file, tz_str, _ = parts
+    elif len(parts) == 4:
+        symbol, conid_str, barsize_file, tz_str = parts
+    else:
+        raise ValueError(f"Invalid filename format: {filename}")
+    try:
+        conid = int(conid_str)
+    except ValueError:
+        raise ValueError(f"Invalid conid in filename: {conid_str}")
+
+    # Map barsize to table
+    table = barsize_tables.get(barsize_file)
+    if not table:
+        raise ValueError(f"Unsupported barsize in filename: {barsize_file}")
+
+    # Map barsize to time_col_name
+    if barsize_file == '1min':
+        time_col_name = 'time_1m'
+    elif barsize_file == '1day':
+        time_col_name = 'day'
+    else:
+        time_col_name = 'time'  # fallback
+
+    # Normalize timezone
+    tz = normalize_tz(tz_str)
+    try:
+        timezone = pytz.timezone(tz)
+    except pytz.exceptions.UnknownTimeZoneError:
+        raise ValueError(f"Invalid timezone: {tz_str}")
+
+    # Detect separator
+    with open(filepath, 'r') as f:
+        sample = f.read(1024)
+        sep = '\t' if '\t' in sample and sample.count('\t') > sample.count(',') else ','
+
+    # Read CSV header
+    df = pd.read_csv(filepath, sep=sep, nrows=0)
+    columns = df.columns.str.lower().tolist()
+
+    # Find time column
+    time_cols = ['date', 'datetime', 'time', 'timestamp']
+    time_col = next((col for col in time_cols if col in columns), None)
+    if not time_col:
+        raise ValueError(f"No time column found in {filename}")
+
+    # Check mandatory columns
+    mandatory = ['open', 'high', 'low', 'close']
+    if not all(col in columns for col in mandatory):
+        missing = [col for col in mandatory if col not in columns]
+        raise ValueError(f"Missing mandatory columns: {missing}")
+
+    # Map optional columns
+    optional = ['volume', 'trades', 'adjusted_close', 'adj_close']
+    col_map = {col: col for col in mandatory + [time_col]}
+    for opt in optional:
+        if opt in columns:
+            col_map[opt] = opt
+        elif opt == 'adj_close' and 'adjusted_close' in columns:
+            col_map['adjusted_close'] = 'adjusted_close'
+
+    insert_cols = ['conid', 'symbol', time_col_name] + mandatory + [k for k in optional if k in col_map]
+
+    return symbol, conid, table, time_col_name, sep, time_col, col_map, insert_cols, timezone
+
+def process_csv_data(filepath, sep, time_col, time_col_name, col_map, table, symbol, conid, timezone):
+    """Process CSV data into DataFrame."""
+    df = pd.read_csv(filepath, sep=sep)
+    df.columns = df.columns.str.lower()
+
+    # Rename columns
+    rename_map = {v: k for k, v in col_map.items() if k != 'time'}
+    rename_map[time_col] = time_col_name
+    if 'adj_close' in rename_map:
+        rename_map['adjusted_close'] = 'adj_close'
+    df = df.rename(columns=rename_map)
+    
+    # Add conid and symbol
+    df['conid'] = conid
+    df['symbol'] = symbol
+    
+    # Localize time
+    df[time_col_name] = pd.to_datetime(df[time_col_name])
+    if df[time_col_name].dt.tz is None:
+        df[time_col_name] = df[time_col_name].dt.tz_localize(timezone)
+    else:
+        df[time_col_name] = df[time_col_name].dt.tz_convert(timezone)
+
+    # Cast types
+    if table == 'ohlcv_1d':
+        if 'volume' in df.columns:
+            df['volume'] = df['volume'].astype(int)
+        if 'trades' in df.columns:
+            df['trades'] = df['trades'].astype(int)
+
+    return df
+
+def insert_ohlcv_data(df, conn, schema, table, insert_cols, time_col_name):
+    """Insert DataFrame into database."""
+    quoted_cols = [f'"{col}"' for col in insert_cols]
+    conflict_cols = ['conid', time_col_name]
+    query = f"INSERT INTO {schema}.{table} ({', '.join(quoted_cols)}) VALUES %s ON CONFLICT ({', '.join(conflict_cols)}) DO NOTHING"
+
+    batch_size = 1000
+    with conn.cursor() as cur:
+        for i in range(0, len(df), batch_size):
+            batch = df.iloc[i:i+batch_size]
+            values = [tuple(row) for row in batch[insert_cols].values]
+            execute_values(cur, query, values)
+    conn.commit()
+
+def process_single_contract(data, conn, schema, table):
+    """Process a single contract dict."""
+    try:
+        if not validate_contract(data):
+            logging.error(f"Validation failed for {data.get('symbol', 'unknown')}")
+            return False
+
+        # Compute conid if missing
+        if 'conid' not in data or data['conid'] is None:
+            data['conid'] = compute_conid(data['symbol'], data['exchange'], data['currency'], data['sec_type'])
+
+        # Insert to DB
+        with conn.cursor() as cur:
+            columns = list(data.keys())
+            values = [data[k] for k in columns]
+            placeholders = ', '.join(['%s'] * len(columns))
+            quoted_columns = [f'"{col}"' for col in columns]
+            query = f"INSERT INTO {schema}.{table} ({', '.join(quoted_columns)}) VALUES ({placeholders}) ON CONFLICT (conid) DO NOTHING"
+            logging.debug(f"Executing query: {query} with values: {values}")
+            cur.execute(query, values)
+        conn.commit()
+        logging.info(f"Inserted contract {json.dumps(data)}")
+        return True
+    except Exception as e:
+        logging.error(f"Error inserting contract {data.get('symbol', 'unknown')}: {e}", exc_info=True)
+        return False
+
+def process_ohlcv_file(filepath, conn, schema, barsize_tables):
+    """Process an OHLCV CSV file using parse_csv_header and process_csv_data."""
+    try:
+        symbol, conid, table, time_col_name, sep, time_col, col_map, insert_cols, timezone = parse_csv_header(filepath, barsize_tables)
+        df = process_csv_data(filepath, sep, time_col, time_col_name, col_map, table, symbol, conid, timezone)
+        insert_ohlcv_data(df, conn, schema, table, insert_cols, time_col_name)
+        logging.info(f"Inserted {len(df)} OHLCV bars for {symbol} into {table}")
+        return True
+    except Exception as e:
+        logging.error(f"Error processing OHLCV file {filepath}: {e}", exc_info=True)
+        return False
+
+class DataFileHandler(FileSystemEventHandler):
+    """Watchdog handler for new data files."""
+    def __init__(self, input_folder, processed_folder, error_folder, conn, config):
+        self.input_folder = input_folder
+        self.processed_folder = processed_folder
+        self.error_folder = error_folder
+        self.conn = conn
+        self.schema = config.get('schema', 'finance')
+        self.contract_table = config.get('contract_table', 'contracts')
+        self.barsize_tables = {
+            '1min': config.get('ohlcv_1m_table', 'ohlcv_1m'),
+            '1day': config.get('ohlcv_1d_table', 'ohlcv_1d')
+        }
+        os.makedirs(self.processed_folder, exist_ok=True)
+        os.makedirs(self.error_folder, exist_ok=True)
+
+    def on_created(self, event):
+        if event.is_directory:
+            return
+        filepath = event.src_path
+        filename = os.path.basename(filepath)
+        if not (filename.endswith('.json') or filename.endswith('.csv')):
+            return
+
+        try:
+            if filename.endswith('.json'):
+                success = process_contract_file(filepath, self.conn, self.schema, 'contracts')
+                if success:
+                    shutil.move(filepath, os.path.join(self.processed_folder, filename))
+                    logging.info(f"Processed and moved {filename} to processed")
+                else:
+                    shutil.move(filepath, os.path.join(self.error_folder, filename))
+                    logging.error(f"Moved {filename} to errors")
+            elif filename.endswith('.csv'):
+                success = process_ohlcv_file(filepath, self.conn, self.schema, self.barsize_tables)
+                if success:
+                    shutil.move(filepath, os.path.join(self.processed_folder, filename))
+                    logging.info(f"Processed and moved {filename} to processed")
+                else:
+                    shutil.move(filepath, os.path.join(self.error_folder, filename))
+                    logging.error(f"Moved {filename} to errors")
+        except Exception as e:
+            logging.error(f"Unexpected error processing {filename}: {e}", exc_info=True)
+            # Move to errors if not already
+            try:
+                shutil.move(filepath, os.path.join(self.error_folder, filename))
+            except:
+                pass
+
+def main():
+    lock_file = os.path.join(os.path.dirname(__file__), 'service.lock')
+    if os.path.exists(lock_file):
+        with open(lock_file, 'r') as f:
+            pid = f.read().strip()
+        try:
+            pid_int = int(pid)
+            if any(p.pid == pid_int for p in psutil.process_iter() if 'python' in p.name().lower()):
+                logging.error("Another instance is running")
+                return
+        except (ValueError, psutil.NoSuchProcess):
+            pass  # Lock file stale
+    with open(lock_file, 'w') as f:
+        f.write(str(os.getpid()))
+
+    try:
+        with open('config.json', 'r') as f:
+            config = json.load(f)
+
+        input_folder = config['input_folder']
+        processed_folder = config['processed_folder']
+        error_folder = config['error_folder']
+        db_uri = os.environ.get('PG_URI', os.path.expandvars(config.get('db_uri', '')))
+
+        if not db_uri:
+            print("No DB URI provided")
+            return
+
+        logging.basicConfig(
+            filename='data_upload.log',
+            level=logging.INFO,
+            format='%(asctime)s - %(levelname)s - %(message)s',
+            datefmt='%Y-%m-%d %H:%M:%S'
+        )
+
+        try:
+            conn = psycopg2.connect(db_uri)
+            logging.info("Connected to DB")
+        except Exception as e:
+            logging.error(f"DB connection failed: {e}")
+            return
+
+        # Set up file watcher
+        event_handler = DataFileHandler(input_folder, processed_folder, error_folder, conn, config)
+        observer = Observer()
+        observer.schedule(event_handler, input_folder, recursive=False)
+
+        try:
+            observer.start()
+            logging.info("Data upload service started successfully - monitoring input folder for .json and .csv files")
+            # Process any existing files on startup
+            for filename in os.listdir(input_folder):
+                filepath = os.path.join(input_folder, filename)
+                if os.path.isfile(filepath) and (filename.endswith('.json') or filename.endswith('.csv')):
+                    event_handler.on_created(type('Event', (), {'is_directory': False, 'src_path': filepath})())
+            while True:
+                time.sleep(1)
+        except KeyboardInterrupt:
+            observer.stop()
+        observer.join()
+        conn.close()
+
+    except KeyboardInterrupt:
+        pass
+    finally:
+        if os.path.exists(lock_file):
+            os.remove(lock_file)
+
+if __name__ == '__main__':
+    main()

ib_connect/skills/ib_download/__init__.py

@@ -0,0 +1 @@
+# IB Download Skill

ib_connect/skills/ib_download/download.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""
+IB Download Module
+"""
+
+from .job_queue import JobQueue
+
+def submit_download_job(params_dict):
+    """
+    Submit a download job. Params as dict.
+    Returns job_key.
+    """
+    queue = JobQueue()
+    
+    # Convert dict to namespace
+    class Args:
+        pass
+    args = Args()
+    for k, v in params_dict.items():
+        setattr(args, k, v)
+    
+    # Set defaults for job params (not connection params — service uses its own config)
+    if not hasattr(args, 'show') or not args.show:
+        args.show = 'TRADES'
+    if not hasattr(args, 'timeout'):
+        args.timeout = 4.0
+    if not hasattr(args, 'max_retries'):
+        args.max_retries = 3
+    if not hasattr(args, 'use_rth'):
+        args.use_rth = False
+    if not hasattr(args, 'format'):
+        args.format = 'ib'
+    
+    return submit_single_job(queue, args)
+
+def submit_single_job(queue, args):
+    params = {
+        'conid': args.conid,
+        'start': args.start,
+        'end': args.end,
+        'bar_size': args.bar_size,
+        'show': args.show,
+        'timeout': args.timeout,
+        'max_retries': args.max_retries,
+        'use_rth': args.use_rth,
+        'format': args.format,
+        'agent': args.agent,
+        'msg': args.msg
+    }
+    # Only store connection params if explicitly provided — absent means service uses its own config
+    for attr in ('host', 'port', 'client_id'):
+        if hasattr(args, attr):
+            params[attr] = getattr(args, attr)
+    job_key = queue.submit_job(params)
+    return job_key
+
+def get_job_status(job_key):
+    """
+    Get status of a job.
+    Returns dict.
+    """
+    queue = JobQueue()
+    return queue.get_status(job_key)