grafit 0.1.0__tar.gz

grafit-0.1.0/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: grafit
+Version: 0.1.0
+Summary: Import Garmin FIT files into SQLite database and visualize with Grafana
+Author-email: nbr23 <max@23.tf>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/nbr23/grafit
+Project-URL: Repository, https://github.com/nbr23/grafit
+Project-URL: Issues, https://github.com/nbr23/grafit/issues
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: python-fitparse>=2.0.4
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: black>=23.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+
+# GraFIT
+
+Visualize your FIT files in Grafana

grafit-0.1.0/README.md

@@ -0,0 +1,3 @@
+# GraFIT
+
+Visualize your FIT files in Grafana

grafit-0.1.0/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "grafit"
+version = "0.1.0"
+description = "Import Garmin FIT files into SQLite database and visualize with Grafana"
+readme = "README.md"
+authors = [
+    {name = "nbr23", email = "max@23.tf"}
+]
+license = "MIT"
+requires-python = ">=3.11"
+dependencies = [
+    "python-fitparse>=2.0.4",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "black>=23.0",
+    "ruff>=0.1.0",
+]
+
+[project.scripts]
+grafit = "grafit.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/nbr23/grafit"
+Repository = "https://github.com/nbr23/grafit"
+Issues = "https://github.com/nbr23/grafit/issues"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.black]
+line-length = 100
+target-version = ["py38"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py38"

grafit-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

grafit-0.1.0/src/grafit/__init__.py

@@ -0,0 +1,3 @@
+"""GraFIT - Import Garmin FIT files into SQLite database."""
+
+__version__ = "0.1.0"

grafit-0.1.0/src/grafit/cli.py

@@ -0,0 +1,258 @@
+"""Command-line interface for GraFIT."""
+
+import argparse
+import sys
+import logging
+import json
+from pathlib import Path
+
+from .schema_generator import generate_schema
+from .database import GraFITDatabase
+from .fit_importer import FitImporter
+
+
+def cmd_generate_schema(args):
+    """Generate SQLite schema from profile.py."""
+    profile_path = Path(args.profile)
+    output_path = Path(args.output)
+
+    if not profile_path.exists():
+        print(f"Error: Profile file {profile_path} not found")
+        sys.exit(1)
+
+    try:
+        schema = generate_schema(str(profile_path))
+
+        # Create output directory if it doesn't exist
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        with open(output_path, "w") as f:
+            f.write(schema)
+
+        print(f"Schema generated successfully: {output_path}")
+
+    except Exception as e:
+        print(f"Error generating schema: {e}")
+        sys.exit(1)
+
+
+def cmd_init_database(args):
+    """Initialize database with schema."""
+    db_path = Path(args.database)
+    schema_path = Path(args.schema)
+
+    if not schema_path.exists():
+        print(f"Error: Schema file {schema_path} not found")
+        sys.exit(1)
+
+    try:
+        db = GraFITDatabase(str(db_path))
+        db.initialize_schema(str(schema_path))
+
+        version = db.get_schema_version()
+        print(f"Database initialized successfully: {db_path}")
+        print(f"Schema version: {version}")
+
+        db.close()
+
+    except Exception as e:
+        print(f"Error initializing database: {e}")
+        sys.exit(1)
+
+
+def cmd_import(args):
+    """Import FIT files - automatically detects if path is file or directory."""
+    input_path = Path(args.path)
+    db_path = Path(args.database)
+
+    if not input_path.exists():
+        print(f"Error: Path {input_path} not found")
+        sys.exit(1)
+
+    # Setup logging
+    setup_logging(args.verbose)
+
+    try:
+        # Auto-create database if it doesn't exist
+        if not db_path.exists():
+            print(f"Database {db_path} not found, creating with schema...")
+
+            # Generate schema on the fly
+            profile_path = Path(__file__).parent.parent.parent / "extra" / "profile.py"
+            if not profile_path.exists():
+                print(f"Error: Profile file {profile_path} not found")
+                sys.exit(1)
+
+            schema = generate_schema(str(profile_path))
+
+            # Create database and initialize with schema
+            db = GraFITDatabase(str(db_path))
+            db.execute_schema(schema)
+            print(f"✓ Database created and initialized: {db_path}")
+        else:
+            db = GraFITDatabase(str(db_path))
+
+        importer = FitImporter(db)
+
+        if input_path.is_file():
+            # Import single file
+            print(f"Importing file: {input_path}")
+            result = importer.import_fit_file(str(input_path), args.force)
+
+            # Print results
+            if result["status"] == "success":
+                print(f"✓ Imported {input_path.name}")
+                print(f"  Messages: {result['messages_imported']}")
+                print(f"  Tables: {', '.join(result['tables_affected'])}")
+            elif result["status"] == "skipped":
+                print(f"⏭ Skipped {input_path.name} (already imported)")
+            else:
+                print(
+                    f"✗ Failed to import {input_path.name}: {result.get('error', 'Unknown error')}"
+                )
+                sys.exit(1)
+
+        elif input_path.is_dir():
+            # Import directory
+            print(f"Importing directory: {input_path}")
+            result = importer.import_directory(
+                str(input_path), args.pattern, args.force, args.recursive
+            )
+
+            # Print results
+            print(f"Import complete:")
+            print(f"  Files processed: {result['files_processed']}")
+            print(f"  Files imported: {result['files_imported']}")
+            print(f"  Files skipped: {result['files_skipped']}")
+            print(f"  Files failed: {result['files_failed']}")
+            print(f"  Total messages: {result['total_messages']}")
+
+            if result["failed_files"]:
+                print("\\nFailed files:")
+                for failure in result["failed_files"]:
+                    print(f"  ✗ {failure['file']}: {failure['error']}")
+
+            if result["tables_affected"]:
+                print(f"\\nTables updated: {', '.join(sorted(result['tables_affected']))}")
+        else:
+            print(f"Error: {input_path} is neither a file nor a directory")
+            sys.exit(1)
+
+        db.close()
+
+    except Exception as e:
+        print(f"Error importing: {e}")
+        sys.exit(1)
+
+
+def cmd_database_stats(args):
+    """Show database statistics."""
+    db_path = Path(args.database)
+
+    if not db_path.exists():
+        print(f"Error: Database {db_path} not found")
+        sys.exit(1)
+
+    try:
+        db = GraFITDatabase(str(db_path))
+
+        version = db.get_schema_version()
+        stats = db.get_import_stats()
+
+        print(f"Database: {db_path}")
+        print(f"Schema version: {version}")
+        print(f"Tables: {len(stats)}")
+
+        total_records = sum(stats.values())
+        print(f"Total records: {total_records:,}")
+
+        if args.detailed:
+            print("\\nTable breakdown:")
+            for table, count in sorted(stats.items(), key=lambda x: x[1], reverse=True):
+                if count > 0:
+                    print(f"  {table}: {count:,}")
+
+        db.close()
+
+    except Exception as e:
+        print(f"Error reading database stats: {e}")
+        sys.exit(1)
+
+
+def setup_logging(verbose: bool):
+    """Setup logging configuration."""
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(level=level, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        prog="grafit", description="Import Garmin FIT files into SQLite database"
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # Schema generation command
+    schema_parser = subparsers.add_parser(
+        "generate-schema", help="Generate SQLite schema from FIT profile"
+    )
+    schema_parser.add_argument("profile", help="Path to profile.py file")
+    schema_parser.add_argument("output", help="Output SQL file path")
+    schema_parser.set_defaults(func=cmd_generate_schema)
+
+    # Database initialization command
+    init_parser = subparsers.add_parser("init-database", help="Initialize database with schema")
+    init_parser.add_argument("database", help="Path to SQLite database file")
+    init_parser.add_argument("schema", help="Path to schema SQL file")
+    init_parser.set_defaults(func=cmd_init_database)
+
+    # Import command (handles both files and directories)
+    import_parser = subparsers.add_parser(
+        "import", help="Import FIT file(s) - automatically detects file or directory"
+    )
+    import_parser.add_argument("path", help="Path to FIT file or directory containing FIT files")
+    import_parser.add_argument(
+        "database",
+        nargs="?",
+        default="fit_data.db",
+        help="Path to SQLite database file (default: fit_data.db)",
+    )
+    import_parser.add_argument(
+        "--pattern", default="*.fit", help="File pattern to match for directories (default: *.fit)"
+    )
+    import_parser.add_argument(
+        "--force", action="store_true", help="Force reimport even if files already exist"
+    )
+    import_parser.add_argument(
+        "--no-recursive",
+        dest="recursive",
+        action="store_false",
+        help="Don't search subdirectories when importing directories",
+    )
+    import_parser.add_argument(
+        "--verbose", "-v", action="store_true", help="Enable verbose logging"
+    )
+    import_parser.set_defaults(func=cmd_import)
+
+    # Database stats command
+    stats_parser = subparsers.add_parser("stats", help="Show database statistics")
+    stats_parser.add_argument("database", help="Path to SQLite database file")
+    stats_parser.add_argument(
+        "--detailed", action="store_true", help="Show detailed table breakdown"
+    )
+    stats_parser.set_defaults(func=cmd_database_stats)
+
+    # Parse arguments
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
+    # Execute the command
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

grafit-0.1.0/src/grafit/database.py

@@ -0,0 +1,297 @@
+"""Database connection and initialization for GraFIT."""
+
+import sqlite3
+import hashlib
+import logging
+from pathlib import Path
+from typing import Optional, Dict, Any, List
+from contextlib import contextmanager
+
+logger = logging.getLogger(__name__)
+
+
+class GraFITDatabase:
+    """Manages SQLite database connections and operations for GraFIT."""
+
+    def __init__(self, db_path: str):
+        """Initialize database connection."""
+        self.db_path = Path(db_path)
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Configure SQLite connection
+        self.connection = sqlite3.connect(
+            str(self.db_path), isolation_level=None  # Autocommit mode
+        )
+        self.connection.row_factory = sqlite3.Row  # Access columns by name
+
+        # Enable WAL mode and other optimizations
+        self._configure_database()
+
+    def _configure_database(self):
+        """Configure SQLite database settings."""
+        cursor = self.connection.cursor()
+
+        # Enable optimizations
+        cursor.execute("PRAGMA foreign_keys = ON")
+        cursor.execute("PRAGMA journal_mode = WAL")
+        cursor.execute("PRAGMA synchronous = NORMAL")
+        cursor.execute("PRAGMA cache_size = -64000")  # 64MB cache
+        cursor.execute("PRAGMA temp_store = MEMORY")
+
+        cursor.close()
+
+    def initialize_schema(self, schema_file: str):
+        """Initialize database schema from SQL file."""
+        schema_path = Path(schema_file)
+        if not schema_path.exists():
+            raise FileNotFoundError(f"Schema file not found: {schema_file}")
+
+        with open(schema_path, "r") as f:
+            schema_sql = f.read()
+
+        self.execute_schema(schema_sql)
+        logger.info(f"Database schema initialized from {schema_file}")
+
+    def execute_schema(self, schema_sql: str):
+        """Execute schema SQL directly."""
+        cursor = self.connection.cursor()
+        try:
+            cursor.executescript(schema_sql)
+            logger.info("Database schema executed successfully")
+        except sqlite3.Error as e:
+            logger.error(f"Failed to execute schema: {e}")
+            raise
+        finally:
+            cursor.close()
+
+    def is_file_imported(self, file_hash: str) -> bool:
+        """Check if a file has already been imported."""
+        cursor = self.connection.cursor()
+        try:
+            # Check any table for this _file_hash
+            cursor.execute(
+                """
+                SELECT 1 FROM file_id WHERE _file_hash = ? LIMIT 1
+            """,
+                (file_hash,),
+            )
+            return cursor.fetchone() is not None
+        except sqlite3.Error:
+            # If file_id table doesn't exist, file hasn't been imported
+            return False
+        finally:
+            cursor.close()
+
+    def insert_message_data(
+        self, table_name: str, data: Dict[str, Any], file_hash: str, file_id: str
+    ):
+        """Insert message data into the specified table."""
+        if not data:
+            return
+
+        # Add _file_hash, _file_id and current timestamp
+        data = data.copy()
+        data["_file_hash"] = file_hash
+        data["_file_id"] = file_id
+
+        # Get valid columns for this table to filter out unknown fields
+        valid_columns = self._get_table_columns(table_name)
+        if not valid_columns:
+            logger.warning(f"Could not get column info for table {table_name}")
+            return
+
+        # Filter data to only include columns that exist in the table
+        filtered_data = {}
+        for key, value in data.items():
+            if key in valid_columns:
+                filtered_data[key] = value
+            else:
+                logger.debug(f"Skipping unknown column {key} for table {table_name}")
+
+        if not filtered_data:
+            logger.debug(f"No valid data to insert into {table_name}")
+            return
+
+        # Build INSERT statement
+        columns = list(filtered_data.keys())
+        placeholders = ["?" for _ in columns]
+        values = [filtered_data[col] for col in columns]
+
+        # Handle quoted table names
+        table_ref = (
+            f'"{table_name}"' if table_name in ["set", "user", "group", "order"] else table_name
+        )
+
+        sql = f"""
+            INSERT INTO {table_ref} ({', '.join(columns)})
+            VALUES ({', '.join(placeholders)})
+        """
+
+        cursor = self.connection.cursor()
+        try:
+            cursor.execute(sql, values)
+            logger.debug(f"Inserted data into {table_name}: {len(values)} fields")
+        except sqlite3.Error as e:
+            logger.error(f"Failed to insert into {table_name}: {e}")
+            logger.debug(f"Filtered data: {filtered_data}")
+            raise
+        finally:
+            cursor.close()
+
+    def _get_table_columns(self, table_name: str) -> set:
+        """Get set of column names for a table."""
+        cursor = self.connection.cursor()
+        try:
+            # Handle quoted table names
+            table_ref = (
+                f'"{table_name}"' if table_name in ["set", "user", "group", "order"] else table_name
+            )
+            cursor.execute(f"PRAGMA table_info({table_ref})")
+            return {row[1] for row in cursor.fetchall()}  # row[1] is column name
+        except sqlite3.Error as e:
+            logger.error(f"Failed to get column info for {table_name}: {e}")
+            return set()
+        finally:
+            cursor.close()
+
+    def bulk_insert_message_data(
+        self, table_name: str, data_list: List[Dict[str, Any]], file_hash: str, file_id: str
+    ):
+        """Bulk insert multiple records for better performance."""
+        if not data_list:
+            return
+
+        # Get valid columns for this table
+        valid_columns = self._get_table_columns(table_name)
+        if not valid_columns:
+            logger.warning(f"Could not get column info for table {table_name}")
+            return
+
+        # Filter and prepare all records
+        filtered_data_list = []
+        for data in data_list:
+            # Add _file_hash and _file_id
+            data = data.copy()
+            data["_file_hash"] = file_hash
+            data["_file_id"] = file_id
+
+            # Filter to valid columns
+            filtered_data = {k: v for k, v in data.items() if k in valid_columns}
+            if filtered_data:
+                filtered_data_list.append(filtered_data)
+
+        if not filtered_data_list:
+            logger.debug(f"No valid data to bulk insert into {table_name}")
+            return
+
+        # Collect all possible columns from all records
+        all_columns = set()
+        for data in filtered_data_list:
+            all_columns.update(data.keys())
+        
+        columns = sorted(list(all_columns))  # Sort for consistent ordering
+        placeholders = ["?" for _ in columns]
+
+        # Handle quoted table names
+        table_ref = (
+            f'"{table_name}"' if table_name in ["set", "user", "group", "order"] else table_name
+        )
+
+        sql = f"""
+            INSERT INTO {table_ref} ({', '.join(columns)})
+            VALUES ({', '.join(placeholders)})
+        """
+
+        # Prepare all value tuples
+        values_list = [[data.get(col) for col in columns] for data in filtered_data_list]
+
+        cursor = self.connection.cursor()
+        try:
+            cursor.executemany(sql, values_list)
+            logger.debug(f"Bulk inserted {len(filtered_data_list)} records into {table_name}")
+        except sqlite3.Error as e:
+            logger.error(f"Failed to bulk insert into {table_name}: {e}")
+            raise
+        finally:
+            cursor.close()
+
+    @contextmanager
+    def transaction(self):
+        """Context manager for database transactions."""
+        cursor = self.connection.cursor()
+        try:
+            cursor.execute("BEGIN")
+            yield cursor
+            cursor.execute("COMMIT")
+        except Exception:
+            cursor.execute("ROLLBACK")
+            raise
+        finally:
+            cursor.close()
+
+    def get_schema_version(self) -> Optional[str]:
+        """Get current schema version from metadata table."""
+        cursor = self.connection.cursor()
+        try:
+            cursor.execute(
+                """
+                SELECT value FROM _grafit_metadata
+                WHERE key = 'schema_version'
+            """
+            )
+            result = cursor.fetchone()
+            return result[0] if result else None
+        except sqlite3.Error:
+            return None
+        finally:
+            cursor.close()
+
+    def get_table_names(self) -> List[str]:
+        """Get list of all table names in the database."""
+        cursor = self.connection.cursor()
+        try:
+            cursor.execute(
+                """
+                SELECT name FROM sqlite_master 
+                WHERE type='table' AND name NOT LIKE 'sqlite_%'
+                AND name != '_grafit_metadata'  -- Exclude only our metadata table
+                ORDER BY name
+            """
+            )
+            return [row[0] for row in cursor.fetchall()]
+        finally:
+            cursor.close()
+
+    def get_import_stats(self) -> Dict[str, int]:
+        """Get statistics about imported data."""
+        stats = {}
+        tables = self.get_table_names()
+
+        cursor = self.connection.cursor()
+        try:
+            for table in tables:
+                # Handle reserved words like 'set'
+                table_name = f'"{table}"' if table in ["set", "user", "group", "order"] else table
+                cursor.execute(f"SELECT COUNT(*) FROM {table_name}")
+                stats[table] = cursor.fetchone()[0]
+        finally:
+            cursor.close()
+
+        return stats
+
+    def close(self):
+        """Close database connection."""
+        if self.connection:
+            self.connection.close()
+
+
+def calculate_file_hash(file_path: str) -> str:
+    """Calculate SHA-256 hash of a file for deduplication."""
+    hash_sha256 = hashlib.sha256()
+
+    with open(file_path, "rb") as f:
+        # Read in chunks to handle large files
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_sha256.update(chunk)
+
+    return hash_sha256.hexdigest()