rclone-api 1.3.7__tar.gz → 1.3.9__tar.gz

{rclone_api-1.3.7 → rclone_api-1.3.9}/.gitignore

@@ -151,3 +151,6 @@ mount
 t2.py
 chunk_store
 state.json
+database.db
+data.db
+test.db

{rclone_api-1.3.7 → rclone_api-1.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.3.7
+Version: 1.3.9
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License
@@ -14,6 +14,8 @@ Requires-Dist: python-dotenv>=1.0.0
 Requires-Dist: certifi>=2025.1.31
 Requires-Dist: psutil
 Requires-Dist: boto3<=1.35.99,>=1.20.1
+Requires-Dist: sqlmodel>=0.0.23
+Requires-Dist: psycopg2-binary>=2.9.10
 Dynamic: home-page
 
 # rclone-api

{rclone_api-1.3.7 → rclone_api-1.3.9}/pyproject.toml

@@ -15,15 +15,16 @@ dependencies = [
     "python-dotenv>=1.0.0",
     "certifi>=2025.1.31",
     "psutil",
-
     # BOTO3 Library needs to be pinned to a specific version
     # BackBlaze S3 fails with checksum header which it doesn't support after 1.35.99
     # The 1.20.1 was the earliest one I checked that worked and is not the true lower bound.
     "boto3>=1.20.1,<=1.35.99",
+    "sqlmodel>=0.0.23",
+    "psycopg2-binary>=2.9.10",
 ]
 
 # Change this with the version number bump.
-version = "1.3.7"
+version = "1.3.9"
 
 [tool.setuptools]
 package-dir = {"" = "src"}
@@ -55,4 +56,5 @@ disable_error_code = ["import-untyped"]
 [project.scripts]
 rclone-api-listfiles = "rclone_api.cmd.list_files:main"
 rclone-api-copylarge-s3 = "rclone_api.cmd.copy_large_s3:main"
-rclone-api-profile-mount = "rclone_api.profile.mount_copy_bytes:main"
+rclone-api-profile-mount = "rclone_api.profile.mount_copy_bytes:main"
+rclone-api-save-to-db = "rclone_api.cmd.save_to_db:main"

rclone_api-1.3.9/src/rclone_api/cmd/analyze.py

@@ -0,0 +1,51 @@
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api import Rclone
+
+
+@dataclass
+class Args:
+    config: Path
+    path: str
+
+    def __post_init__(self):
+        if not self.config.exists():
+            raise FileNotFoundError(f"Config file not found: {self.config}")
+
+
+def list_files(rclone: Rclone, path: str):
+    """List files in a remote path."""
+
+    with rclone.ls_stream(path, fast_list=True) as files:
+        for file_item in files:
+            print(file_item.path, "", file_item.size, file_item.mod_time)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, default="rclone.conf"
+    )
+    parser.add_argument("path", help="Remote path to list")
+    tmp = parser.parse_args()
+    return Args(config=tmp.config, path=tmp.path)
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    path = args.path
+    rclone = Rclone(Path(args.config))
+    list_files(rclone, path)
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+
+    cwd = Path(".").absolute()
+    print(f"cwd: {cwd}")
+    sys.argv.append("dst:TorrentBooks")
+    main()

rclone_api-1.3.9/src/rclone_api/cmd/save_to_db.py

@@ -0,0 +1,68 @@
+import argparse
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+from rclone_api import Rclone
+
+# load_dotenv()
+
+
+# DB_URL = "sqlite:///data.db"
+
+# os.environ["DB_URL"] = "sqlite:///data.db"
+
+
+def _db_url_from_env_or_raise() -> str:
+    load_dotenv()
+    db_url = os.getenv("DB_URL")
+    if db_url is None:
+        raise ValueError("DB_URL not set")
+    return db_url
+
+
+@dataclass
+class Args:
+    config: Path
+    path: str
+
+    def __post_init__(self):
+        if not self.config.exists():
+            raise FileNotFoundError(f"Config file not found: {self.config}")
+
+
+def fill_db(rclone: Rclone, path: str):
+    """List files in a remote path."""
+    # db = DB(_db_url_from_env_or_raise())
+    db_url = _db_url_from_env_or_raise()
+    rclone.save_to_db(src=path, db_url=db_url, fast_list=True)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, default="rclone.conf"
+    )
+    parser.add_argument("path", help="Remote path to list")
+    tmp = parser.parse_args()
+    return Args(config=tmp.config, path=tmp.path)
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    path = args.path
+    rclone = Rclone(Path(args.config))
+    fill_db(rclone, path)
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+
+    cwd = Path(".").absolute()
+    print(f"cwd: {cwd}")
+    sys.argv.append("dst:TorrentBooks/meta")
+    main()

rclone_api-1.3.9/src/rclone_api/db/__init__.py

@@ -0,0 +1,3 @@
+from .db import DB
+
+__all__ = ["DB"]

rclone_api-1.3.9/src/rclone_api/db/db.py

@@ -0,0 +1,268 @@
+"""
+Database module for rclone_api.
+"""
+
+import os
+from threading import Lock
+from typing import Optional
+
+from sqlmodel import Session, SQLModel, create_engine, select
+
+from rclone_api.db.models import RepositoryMeta, create_file_entry_model
+from rclone_api.file import FileItem
+
+
+def _to_table_name(remote_name: str) -> str:
+    return (
+        "files_"
+        + remote_name.replace(":", "_").replace(" ", "_").replace("/", "_").lower()
+    )
+
+
+class DB:
+    """Database class for rclone_api."""
+
+    def __init__(self, db_path_url: str):
+        """Initialize the database.
+
+        Args:
+            db_path: Path to the database file
+        """
+        self.db_path_url = db_path_url
+        self.engine = create_engine(db_path_url)
+
+        # Create the meta table
+        SQLModel.metadata.create_all(self.engine)
+        self._cache: dict[str, DBRepo] = {}
+        self._cache_lock = Lock()
+
+    def drop_all(self) -> None:
+        """Drop all tables in the database."""
+        SQLModel.metadata.drop_all(self.engine)
+
+    def close(self) -> None:
+        """Close the database connection and release resources."""
+        if hasattr(self, "engine") and self.engine is not None:
+            self.engine.dispose()
+
+    def add_files(self, files: list[FileItem]) -> None:
+        """Add files to the database.
+
+        Args:
+            remote_name: Name of the remote
+            files: List of file entries
+        """
+
+        partition: dict[str, list[FileItem]] = {}
+        for file in files:
+            partition.setdefault(file.remote, []).append(file)
+
+        for remote_name, files in partition.items():
+            repo = self.get_or_create_repo(remote_name)
+            repo.insert_files(files)
+
+    def query_files(self, remote_name: str) -> list[FileItem]:
+        """Query files from the database.
+
+        Args:
+            remote_name: Name of the remote
+        """
+        repo = self.get_or_create_repo(remote_name)
+        files = repo.get_files()
+        out: list[FileItem] = []
+        for file in files:
+            out.append(file)
+        return out
+
+    def get_or_create_repo(self, remote_name: str) -> "DBRepo":
+        """Get a table section for a remote.
+
+        Args:
+            remote_name: Name of the remote
+            table_name: Optional table name, will be derived from remote_name if not provided
+
+        Returns:
+            DBRepo: A table section for the remote
+        """
+        with self._cache_lock:
+            if remote_name in self._cache:
+                return self._cache[remote_name]
+            table_name = _to_table_name(remote_name)
+            out = DBRepo(self.engine, remote_name, table_name)
+            self._cache[remote_name] = out
+            return out
+
+
+class DBRepo:
+    """Table repo remote."""
+
+    def __init__(self, engine, remote_name: str, table_name: Optional[str] = None):
+        """Initialize a table section.
+
+        Args:
+            engine: SQLAlchemy engine
+            remote_name: Name of the remote
+            table_name: Optional table name, will be derived from remote_name if not provided
+        """
+        self.engine = engine
+        self.remote_name = remote_name
+
+        # If table_name is not provided, derive one from the remote name.
+        if table_name is None:
+            # table_name = (
+            #     "file_entries_"
+            #     + remote_name.replace(":", "_").replace(" ", "_").replace("/", "_").lower()
+            # )
+            table_name = _to_table_name(remote_name)
+        self.table_name = table_name
+
+        # Check if repository exists in RepositoryMeta; if not, create a new entry.
+        with Session(self.engine) as session:
+            existing_repo = session.exec(
+                select(RepositoryMeta).where(
+                    RepositoryMeta.repo_name == self.remote_name
+                )
+            ).first()
+            if not existing_repo:
+                repo_meta = RepositoryMeta(
+                    repo_name=self.remote_name, file_table_name=self.table_name
+                )
+                session.add(repo_meta)
+                session.commit()
+
+        # Dynamically create the file entry model and its table.
+        self.FileEntryModel = create_file_entry_model(self.table_name)
+        SQLModel.metadata.create_all(self.engine, tables=[self.FileEntryModel.__table__])  # type: ignore
+
+    def insert_file(self, file: FileItem) -> None:
+        """Insert a file entry into the table.
+
+        Args:
+            file: File entry
+        """
+        return self.insert_files([file])
+
+    def insert_files(self, files: list[FileItem]) -> None:
+        """
+        Insert multiple file entries into the table.
+
+        Three bulk operations are performed:
+        1. Select: Determine which files already exist.
+        2. Insert: Bulk-insert new file entries.
+        3. Update: Bulk-update existing file entries.
+
+        The FileEntryModel must define a unique constraint on (path, name) and have a primary key "id".
+        """
+        # Step 1: Bulk select existing records.
+        # get_exists() returns a set of FileItem objects (based on path_no_remote and name) that already exist.
+        existing_files = self.get_exists(files)
+
+        # Determine which files need to be updated vs. inserted.
+        needs_update = existing_files
+        is_new = set(files) - existing_files
+
+        # Step 2: Bulk insert new rows.
+        new_values = [
+            {
+                "path": file.path_no_remote,
+                "name": file.name,
+                "size": file.size,
+                "mime_type": file.mime_type,
+                "mod_time": file.mod_time,
+                "suffix": file.suffix,
+            }
+            for file in is_new
+        ]
+        with Session(self.engine) as session:
+            if new_values:
+                session.bulk_insert_mappings(self.FileEntryModel, new_values)  # type: ignore
+                session.commit()
+
+        # Step 3: Bulk update existing rows.
+        # First, query the database for the primary keys of rows that match the unique keys in needs_update.
+        with Session(self.engine) as session:
+            # Collect all unique paths from files needing update.
+            update_paths = [file.path_no_remote for file in needs_update]
+            # Query for existing rows matching any of these paths.
+            db_entries = session.exec(
+                select(self.FileEntryModel).where(
+                    self.FileEntryModel.path.in_(update_paths)  # type: ignore
+                )
+            ).all()
+
+            # Build a mapping from the unique key (path, name) to the primary key (id).
+            id_map = {(entry.path, entry.name): entry.id for entry in db_entries}
+
+            # Prepare bulk update mappings.
+            update_values = []
+            for file in needs_update:
+                key = (file.path_no_remote, file.name)
+                if key in id_map:
+                    update_values.append(
+                        {
+                            "id": id_map[key],
+                            "size": file.size,
+                            "mime_type": file.mime_type,
+                            "mod_time": file.mod_time,
+                            "suffix": file.suffix,
+                        }
+                    )
+            if update_values:
+                session.bulk_update_mappings(self.FileEntryModel, update_values)  # type: ignore
+                session.commit()
+
+    def get_exists(self, files: list[FileItem]) -> set[FileItem]:
+        """Get file entries from the table that exist among the given files.
+
+        Args:
+            files: List of file entries
+
+        Returns:
+            Set of FileItem instances whose 'path_no_remote' exists in the table.
+        """
+        # Extract unique paths from the input files.
+        paths = {file.path_no_remote for file in files}
+
+        with Session(self.engine) as session:
+            # Execute a single query to fetch all file paths in the table that match the input paths.
+            result = session.exec(
+                select(self.FileEntryModel.path).where(
+                    self.FileEntryModel.path.in_(paths)  # type: ignore
+                )
+            ).all()
+            # Convert the result to a set for fast membership tests.
+            existing_paths = set(result)
+
+        # Return the set of FileItem objects that have a path in the existing_paths.
+        return {file for file in files if file.path_no_remote in existing_paths}
+
+    def get_files(self) -> list[FileItem]:
+        """Get all files in the table.
+
+        Returns:
+            list: List of file entries
+        """
+        # with Session(self.engine) as session:
+        #     return session.exec(select(self.FileEntryModel)).all()
+        out: list[FileItem] = []
+        with Session(self.engine) as session:
+            query = session.exec(select(self.FileEntryModel)).all()
+            for item in query:
+                name = item.name  # type: ignore
+                size = item.size  # type: ignore
+                mime_type = item.mime_type  # type: ignore
+                mod_time = item.mod_time  # type: ignore
+                path = item.path  # type: ignore
+                parent = os.path.dirname(path)
+                if parent == "/" or parent == ".":
+                    parent = ""
+                o = FileItem(
+                    remote=self.remote_name,
+                    parent=parent,
+                    name=name,
+                    size=size,
+                    mime_type=mime_type,
+                    mod_time=mod_time,
+                )
+                out.append(o)
+        return out

rclone_api-1.3.9/src/rclone_api/db/models.py

@@ -0,0 +1,57 @@
+"""
+Database models for rclone_api.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Optional, Type
+
+from sqlalchemy import BigInteger, Column
+from sqlmodel import Field, SQLModel
+
+
+# Meta table that indexes all repositories
+class RepositoryMeta(SQLModel, table=True):
+    """Repository metadata table."""
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    repo_name: str
+    file_table_name: str  # The dedicated table name for file entries
+
+
+# Base FileEntry model that will be extended
+class FileEntry(SQLModel, ABC):
+    """Base file entry model with common fields."""
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    path: str = Field(index=True, unique=True)
+    suffix: str = Field(index=True)
+    name: str
+    size: int = Field(sa_column=Column(BigInteger))
+    mime_type: str
+    mod_time: str
+    hash: Optional[str] = Field(default=None)
+
+    @abstractmethod
+    def table_name(self) -> str:
+        """Return the table name for this file entry model."""
+        pass
+
+
+# Factory to dynamically create a FileEntry model with a given table name
+def create_file_entry_model(_table_name: str) -> Type[FileEntry]:
+    """Create a file entry model with a given table name.
+
+    Args:
+        table_name: Table name
+
+    Returns:
+        Type[FileEntryBase]: File entry model class with specified table name
+    """
+
+    class FileEntryConcrete(FileEntry, table=True):
+        __tablename__ = _table_name  # type: ignore # dynamically set table name
+
+        def table_name(self) -> str:
+            return _table_name
+
+    return FileEntryConcrete

{rclone_api-1.3.7 → rclone_api-1.3.9}/src/rclone_api/file.py

@@ -5,41 +5,85 @@ from pathlib import Path
 
 from rclone_api.rpath import RPath
 
+_STRING_INTERNER: dict[str, str] = {}
+
+
+def _intern(s: str) -> str:
+    return _STRING_INTERNER.setdefault(s, s)
+
 
 # File is too complex, this is a simple dataclass that can be streamed out.
 @dataclass
 class FileItem:
     """Remote file dataclass."""
 
-    path: str
+    remote: str
+    parent: str
     name: str
     size: int
     mime_type: str
     mod_time: str
+    hash: str | None = None
+
+    @property
+    def path(self) -> str:
+        if self.parent == ".":
+            return f"{self.remote}/{self.name}"
+        else:
+            return f"{self.remote}/{self.parent}/{self.name}"
+
+    @property
+    def path_no_remote(self) -> str:
+        if self.parent == ".":
+            return f"{self.name}"
+        else:
+            return f"{self.parent}/{self.name}"
+
+    @property
+    def suffix(self) -> str:
+        return self._suffix
+
+    def __post_init__(self):
+        self.parent = _intern(self.parent)
+        self.mime_type = _intern(self.mime_type)
+        self.remote = _intern(self.remote)
+        self._suffix = _intern(Path(self.name).suffix)
 
     @staticmethod
-    def from_json(data: dict) -> "FileItem | None":
+    def from_json(remote: str, data: dict) -> "FileItem | None":
         try:
+            path_str: str = data["Path"]
+            parent_path = Path(path_str).parent.as_posix()
+            name = data["Name"]
+            size = data["Size"]
+            mime_type = data["MimeType"]
+            mod_time = data["ModTime"]
+
             return FileItem(
-                data["Path"],
-                data["Name"],
-                data["Size"],
-                data["MimeType"],
-                data["ModTime"],
+                remote=remote,
+                parent=parent_path,
+                name=name,
+                size=size,
+                mime_type=mime_type,
+                mod_time=mod_time,
             )
         except KeyError:
             warnings.warn(f"Invalid data: {data}")
             return None
 
     @staticmethod
-    def from_json_str(data: str) -> "FileItem | None":
+    def from_json_str(remote: str, data: str) -> "FileItem | None":
         try:
             data_dict = json.loads(data)
-            return FileItem.from_json(data_dict)
+            return FileItem.from_json(remote, data_dict)
         except json.JSONDecodeError:
             warnings.warn(f"Invalid JSON data: {data}")
             return None
 
+    # hasher for set membership
+    def __hash__(self) -> int:
+        return hash(self.path_no_remote)
+
 
 class File:
     """Remote file dataclass."""

rclone_api-1.3.9/src/rclone_api/file_item.py

@@ -0,0 +1,68 @@
+import json
+import warnings
+from dataclasses import dataclass
+from pathlib import Path
+
+_STRING_INTERNER: dict[str, str] = {}
+
+
+def _intern(s: str) -> str:
+    return _STRING_INTERNER.setdefault(s, s)
+
+
+# File is too complex, this is a simple dataclass that can be streamed out.
+@dataclass
+class FileItem:
+    """Remote file dataclass."""
+
+    remote: str
+    parent: str
+    name: str
+    size: int
+    mime_type: str
+    mod_time: str
+
+    @property
+    def path(self) -> str:
+        return f"{self.remote}/{self.parent}/{self.name}"
+
+    @property
+    def suffix(self) -> str:
+        return self._suffix
+
+    def __post_init__(self):
+        self.parent = _intern(self.parent)
+        self.mime_type = _intern(self.mime_type)
+        suffix = Path(self.name).suffix
+        self._suffix = _intern(suffix)
+
+    @staticmethod
+    def from_json(data: dict) -> "FileItem | None":
+        try:
+            path_str: str = data["Path"]
+            parent_path = Path(path_str).parent.as_posix()
+            name = data["Name"]
+            size = data["Size"]
+            mime_type = data["MimeType"]
+            mod_time = data["ModTime"]
+
+            return FileItem(
+                remote="DUMMY",
+                parent=parent_path,
+                name=name,
+                size=size,
+                mime_type=mime_type,
+                mod_time=mod_time,
+            )
+        except KeyError:
+            warnings.warn(f"Invalid data: {data}")
+            return None
+
+    @staticmethod
+    def from_json_str(data: str) -> "FileItem | None":
+        try:
+            data_dict = json.loads(data)
+            return FileItem.from_json(data_dict)
+        except json.JSONDecodeError:
+            warnings.warn(f"Invalid JSON data: {data}")
+            return None

{rclone_api-1.3.7 → rclone_api-1.3.9}/src/rclone_api/process.py

@@ -76,6 +76,8 @@ class Process:
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        self.terminate()
+        self.wait()
         self.cleanup()
 
     def cleanup(self) -> None: