ebk 0.3.1__py3-none-any.whl → 0.3.2__py3-none-any.whl

ebk/services/tag_service.py

@@ -0,0 +1,282 @@
+"""Service for managing hierarchical user tags.
+
+Tags provide user-defined organization separate from bibliographic subjects.
+"""
+
+from typing import List, Optional
+from sqlalchemy.orm import Session
+from datetime import datetime
+
+from ebk.db.models import Tag, Book, book_tags
+
+
+class TagService:
+    """Service for CRUD operations on hierarchical tags."""
+
+    def __init__(self, session: Session):
+        """Initialize tag service.
+
+        Args:
+            session: SQLAlchemy session
+        """
+        self.session = session
+
+    def get_or_create_tag(self, path: str, description: Optional[str] = None,
+                          color: Optional[str] = None) -> Tag:
+        """Get existing tag or create new one with full hierarchy.
+
+        Args:
+            path: Full tag path (e.g., "Work/Project-2024")
+            description: Optional description
+            color: Optional hex color code
+
+        Returns:
+            Tag instance
+
+        Examples:
+            >>> service.get_or_create_tag("Work/Project-2024")
+            # Creates: "Work" and "Work/Project-2024" if they don't exist
+        """
+        # Check if tag already exists
+        existing = self.session.query(Tag).filter_by(path=path).first()
+        if existing:
+            return existing
+
+        # Parse path into components
+        parts = path.split('/')
+        parent_tag = None
+        current_path = ""
+
+        # Create hierarchy from root to leaf
+        for i, name in enumerate(parts):
+            # Build current path
+            if current_path:
+                current_path += f"/{name}"
+            else:
+                current_path = name
+
+            # Check if this level exists
+            tag = self.session.query(Tag).filter_by(path=current_path).first()
+
+            if not tag:
+                # Create new tag at this level
+                tag = Tag(
+                    name=name,
+                    path=current_path,
+                    parent_id=parent_tag.id if parent_tag else None
+                )
+
+                # Only set description and color on the leaf node
+                if i == len(parts) - 1:
+                    tag.description = description
+                    tag.color = color
+
+                self.session.add(tag)
+
+            parent_tag = tag
+
+        self.session.commit()
+        return parent_tag
+
+    def get_tag(self, path: str) -> Optional[Tag]:
+        """Get tag by path.
+
+        Args:
+            path: Full tag path
+
+        Returns:
+            Tag instance or None
+        """
+        return self.session.query(Tag).filter_by(path=path).first()
+
+    def get_all_tags(self) -> List[Tag]:
+        """Get all tags ordered by path.
+
+        Returns:
+            List of all tags
+        """
+        return self.session.query(Tag).order_by(Tag.path).all()
+
+    def get_root_tags(self) -> List[Tag]:
+        """Get top-level tags (no parent).
+
+        Returns:
+            List of root tags
+        """
+        return self.session.query(Tag).filter(Tag.parent_id.is_(None)).order_by(Tag.name).all()
+
+    def get_children(self, tag: Tag) -> List[Tag]:
+        """Get immediate children of a tag.
+
+        Args:
+            tag: Parent tag
+
+        Returns:
+            List of child tags
+        """
+        return self.session.query(Tag).filter_by(parent_id=tag.id).order_by(Tag.name).all()
+
+    def delete_tag(self, path: str, delete_children: bool = False) -> bool:
+        """Delete a tag.
+
+        Args:
+            path: Full tag path
+            delete_children: If True, delete children and all descendants too
+
+        Returns:
+            True if deleted, False if not found
+        """
+        tag = self.get_tag(path)
+        if not tag:
+            return False
+
+        # Check if tag has children
+        children = self.get_children(tag)
+        if children and not delete_children:
+            raise ValueError(f"Tag '{path}' has {len(children)} children. "
+                           "Use delete_children=True to delete them too.")
+
+        # If delete_children=True, explicitly delete all descendants
+        if delete_children:
+            # Find all tags that start with this path + "/"
+            descendants = self.session.query(Tag).filter(
+                Tag.path.like(f"{path}/%")
+            ).all()
+            for desc in descendants:
+                self.session.delete(desc)
+
+        self.session.delete(tag)
+        self.session.commit()
+        return True
+
+    def rename_tag(self, old_path: str, new_path: str) -> Tag:
+        """Rename a tag and update all descendant paths.
+
+        Args:
+            old_path: Current tag path
+            new_path: New tag path
+
+        Returns:
+            Updated tag
+
+        Raises:
+            ValueError: If tag doesn't exist or new path already exists
+        """
+        tag = self.get_tag(old_path)
+        if not tag:
+            raise ValueError(f"Tag '{old_path}' not found")
+
+        # Check if new path already exists
+        if self.get_tag(new_path):
+            raise ValueError(f"Tag '{new_path}' already exists")
+
+        # Update this tag
+        old_name = tag.name
+        new_parts = new_path.split('/')
+        tag.name = new_parts[-1]
+        tag.path = new_path
+
+        # Update all descendant paths
+        descendants = self.session.query(Tag).filter(
+            Tag.path.like(f"{old_path}/%")
+        ).all()
+
+        for desc in descendants:
+            # Replace the old path prefix with new path
+            desc.path = desc.path.replace(old_path, new_path, 1)
+
+        self.session.commit()
+        return tag
+
+    def add_tag_to_book(self, book: Book, tag_path: str) -> Tag:
+        """Add a tag to a book (creates tag if it doesn't exist).
+
+        Args:
+            book: Book instance
+            tag_path: Full tag path
+
+        Returns:
+            Tag instance
+        """
+        tag = self.get_or_create_tag(tag_path)
+
+        if tag not in book.tags:
+            book.tags.append(tag)
+            self.session.commit()
+
+        return tag
+
+    def remove_tag_from_book(self, book: Book, tag_path: str) -> bool:
+        """Remove a tag from a book.
+
+        Args:
+            book: Book instance
+            tag_path: Full tag path
+
+        Returns:
+            True if removed, False if book didn't have that tag
+        """
+        tag = self.get_tag(tag_path)
+        if not tag:
+            return False
+
+        if tag in book.tags:
+            book.tags.remove(tag)
+            self.session.commit()
+            return True
+
+        return False
+
+    def get_books_with_tag(self, tag_path: str, include_subtags: bool = False) -> List[Book]:
+        """Get all books with a specific tag.
+
+        Args:
+            tag_path: Full tag path
+            include_subtags: If True, include books from descendant tags
+
+        Returns:
+            List of books
+        """
+        tag = self.get_tag(tag_path)
+        if not tag:
+            return []
+
+        if not include_subtags:
+            return tag.books
+
+        # Get all descendant tags
+        descendant_paths = self.session.query(Tag.id).filter(
+            Tag.path.like(f"{tag_path}/%")
+        ).all()
+
+        all_tag_ids = [tag.id] + [t[0] for t in descendant_paths]
+
+        # Get books with any of these tags
+        books = self.session.query(Book).join(book_tags).filter(
+            book_tags.c.tag_id.in_(all_tag_ids)
+        ).distinct().all()
+
+        return books
+
+    def get_tag_stats(self, tag_path: str) -> dict:
+        """Get statistics for a tag.
+
+        Args:
+            tag_path: Full tag path
+
+        Returns:
+            Dict with stats: {book_count, subtag_count, depth}
+        """
+        tag = self.get_tag(tag_path)
+        if not tag:
+            return {}
+
+        children = self.get_children(tag)
+
+        return {
+            'path': tag.path,
+            'book_count': len(tag.books),
+            'subtag_count': len(children),
+            'depth': tag.depth,
+            'created_at': tag.created_at,
+        }

ebk/services/text_extraction.py

@@ -0,0 +1,317 @@
+"""
+Text extraction service for ebook files.
+
+Handles extraction from PDF, EPUB, TXT, MD and stores in database with FTS indexing.
+"""
+
+import re
+from pathlib import Path
+from typing import List, Optional, Tuple
+import logging
+
+import pypdf
+import fitz  # PyMuPDF
+from ebooklib import epub
+from bs4 import BeautifulSoup
+
+from ..db.models import File, ExtractedText, TextChunk
+from sqlalchemy.orm import Session
+from sqlalchemy import text
+
+logger = logging.getLogger(__name__)
+
+
+class TextExtractionService:
+    """Service for extracting and chunking text from ebook files."""
+
+    def __init__(self, library_root: Path):
+        self.library_root = Path(library_root)
+
+    def extract_full_text(self, file: File, session: Session) -> Optional[ExtractedText]:
+        """
+        Extract complete text from ebook file and store in database.
+
+        Args:
+            file: File model instance
+            session: Database session
+
+        Returns:
+            ExtractedText instance or None if extraction failed
+        """
+        file_path = self.library_root / file.path
+
+        if not file_path.exists():
+            logger.error(f"File not found: {file_path}")
+            return None
+
+        try:
+            # Extract based on format
+            if file.format.lower() in ['txt', 'md', 'text']:
+                text = self._extract_plaintext(file_path)
+            elif file.format.lower() == 'pdf':
+                text = self._extract_pdf_text(file_path)
+            elif file.format.lower() == 'epub':
+                text = self._extract_epub_text(file_path)
+            else:
+                logger.warning(f"Unsupported format for text extraction: {file.format}")
+                return None
+
+            if not text or len(text.strip()) < 100:
+                logger.warning(f"Extracted text too short for {file.path}")
+                return None
+
+            # Store in database
+            extracted = ExtractedText(
+                file_id=file.id,
+                content=text,
+                content_hash=self._hash_text(text)
+            )
+            session.add(extracted)
+
+            # Update file status
+            file.text_extracted = True
+            file.extraction_date = extracted.extracted_at
+
+            # Update FTS index
+            self._update_fts_index(session, file.book_id, text)
+
+            logger.info(f"Extracted {len(text)} chars from {file.path}")
+            return extracted
+
+        except Exception as e:
+            logger.error(f"Error extracting text from {file.path}: {e}")
+            return None
+
+    def create_chunks(self, extracted: ExtractedText, file: File,
+                     session: Session, chunk_size: int = 500,
+                     overlap: int = 100) -> List[TextChunk]:
+        """
+        Split extracted text into overlapping chunks for semantic search.
+
+        Args:
+            extracted: ExtractedText instance
+            file: File instance
+            session: Database session
+            chunk_size: Number of words per chunk
+            overlap: Number of overlapping words between chunks
+
+        Returns:
+            List of TextChunk instances
+        """
+        text = extracted.content
+        words = text.split()
+
+        chunks = []
+        for i in range(0, len(words), chunk_size - overlap):
+            chunk_words = words[i:i + chunk_size]
+            chunk_text = ' '.join(chunk_words)
+
+            if len(chunk_text.strip()) < 50:  # Skip tiny chunks
+                continue
+
+            chunk = TextChunk(
+                file_id=file.id,
+                chunk_index=len(chunks),
+                content=chunk_text,
+                has_embedding=False
+            )
+            chunks.append(chunk)
+
+        session.add_all(chunks)
+        logger.info(f"Created {len(chunks)} chunks from {file.path}")
+        return chunks
+
+    def _extract_plaintext(self, file_path: Path) -> str:
+        """Extract text from plain text files."""
+        try:
+            return file_path.read_text(encoding='utf-8')
+        except UnicodeDecodeError:
+            # Try with different encoding
+            return file_path.read_text(encoding='latin-1')
+
+    def _extract_pdf_text(self, file_path: Path) -> str:
+        """
+        Extract text from PDF using PyMuPDF (primary) with pypdf fallback.
+        """
+        try:
+            # Try PyMuPDF first (better quality)
+            doc = fitz.open(str(file_path))
+            text = ""
+            for page in doc:
+                text += page.get_text()
+            doc.close()
+
+            if text.strip():
+                return self._clean_text(text)
+
+        except Exception as e:
+            logger.warning(f"PyMuPDF extraction failed: {e}, trying pypdf")
+
+        try:
+            # Fallback to pypdf
+            with open(file_path, 'rb') as f:
+                reader = pypdf.PdfReader(f)
+                text = ""
+                for page in reader.pages:
+                    text += page.extract_text()
+
+            return self._clean_text(text)
+
+        except Exception as e:
+            logger.error(f"PDF text extraction failed: {e}")
+            return ""
+
+    def _extract_epub_text(self, file_path: Path) -> str:
+        """Extract text from EPUB file."""
+        try:
+            book = epub.read_epub(str(file_path))
+            text_parts = []
+
+            for item in book.get_items():
+                # Handle different ebooklib versions
+                # Type 9 is ITEM_DOCUMENT in ebooklib
+                item_type = item.get_type()
+
+                # Check if this is a document item (HTML/XHTML content)
+                is_document = False
+                if hasattr(epub, 'ITEM_DOCUMENT'):
+                    is_document = item_type == epub.ITEM_DOCUMENT
+                else:
+                    # Fallback: type 9 is document, or check media type
+                    is_document = (item_type == 9 or
+                                 'html' in item.get_name().lower() or
+                                 (hasattr(item, 'media_type') and
+                                  item.media_type and
+                                  'html' in item.media_type.lower()))
+
+                if is_document:
+                    try:
+                        soup = BeautifulSoup(item.content, 'html.parser')
+
+                        # Remove script and style elements
+                        for script in soup(["script", "style"]):
+                            script.decompose()
+
+                        text = soup.get_text(separator='\n')
+                        text_parts.append(text)
+                    except Exception as e:
+                        logger.debug(f"Failed to extract text from item {item.get_name()}: {e}")
+                        continue
+
+            full_text = '\n\n'.join(text_parts)
+            return self._clean_text(full_text)
+
+        except Exception as e:
+            logger.error(f"EPUB text extraction failed: {e}")
+            return ""
+
+    def _clean_text(self, text: str) -> str:
+        """Clean extracted text."""
+        # Remove excessive whitespace
+        text = re.sub(r'\n\s*\n', '\n\n', text)
+        text = re.sub(r' +', ' ', text)
+
+        # Remove page headers/footers (common patterns)
+        text = re.sub(r'^\d+\s*$', '', text, flags=re.MULTILINE)
+
+        # Strip leading/trailing whitespace
+        text = text.strip()
+
+        return text
+
+    def _hash_text(self, text: str) -> str:
+        """Generate hash of text content."""
+        import hashlib
+        return hashlib.sha256(text.encode()).hexdigest()
+
+    def _update_fts_index(self, session: Session, book_id: int, extracted_text: str):
+        """
+        Update full-text search index.
+
+        Args:
+            session: Database session
+            book_id: Book ID
+            extracted_text: Extracted text content
+        """
+        try:
+            # Get book title and description for FTS
+            from ..db.models import Book
+            book = session.query(Book).get(book_id)
+
+            if not book:
+                return
+
+            # Delete existing FTS entry if exists
+            session.execute(
+                text("DELETE FROM books_fts WHERE book_id = :book_id"),
+                {"book_id": book_id}
+            )
+
+            # Insert into FTS table
+            session.execute(
+                text("""
+                INSERT INTO books_fts (book_id, title, description, extracted_text)
+                VALUES (:book_id, :title, :description, :extracted_text)
+                """),
+                {
+                    "book_id": book_id,
+                    "title": book.title or '',
+                    "description": book.description or '',
+                    "extracted_text": extracted_text[:50000]  # Limit FTS content to first 50k chars
+                }
+            )
+
+            logger.info(f"Updated FTS index for book {book_id}")
+
+        except Exception as e:
+            logger.error(f"Error updating FTS index: {e}")
+
+    def extract_page_content(self, file_path: Path, page_number: int) -> Optional[str]:
+        """
+        Extract text from a specific page (PDF only).
+
+        Args:
+            file_path: Path to PDF file
+            page_number: Page number (0-indexed)
+
+        Returns:
+            Page text or None
+        """
+        try:
+            if file_path.suffix.lower() == '.pdf':
+                doc = fitz.open(str(file_path))
+                if 0 <= page_number < len(doc):
+                    page_text = doc[page_number].get_text()
+                    doc.close()
+                    return self._clean_text(page_text)
+                doc.close()
+        except Exception as e:
+            logger.error(f"Error extracting page {page_number}: {e}")
+
+        return None
+
+    def get_word_count(self, text: str) -> int:
+        """Get word count from text."""
+        return len(text.split())
+
+    def extract_and_chunk_all(self, file: File, session: Session,
+                              chunk_size: int = 500) -> Tuple[Optional[ExtractedText], List[TextChunk]]:
+        """
+        Extract full text and create chunks in one operation.
+
+        Args:
+            file: File instance
+            session: Database session
+            chunk_size: Words per chunk
+
+        Returns:
+            Tuple of (ExtractedText, List[TextChunk])
+        """
+        extracted = self.extract_full_text(file, session)
+
+        if not extracted:
+            return None, []
+
+        chunks = self.create_chunks(extracted, file, session, chunk_size)
+
+        return extracted, chunks

ebk/similarity/__init__.py

@@ -0,0 +1,77 @@
+"""Book similarity system.
+
+This module provides a flexible system for computing similarity between books
+using multiple features (content, metadata, etc.).
+
+Basic usage:
+    >>> from ebk.similarity import BookSimilarity
+    >>>
+    >>> # Configure similarity
+    >>> sim = BookSimilarity().balanced()
+    >>>
+    >>> # Fit on corpus for performance
+    >>> sim.fit(all_books)
+    >>>
+    >>> # Find similar books
+    >>> similar = sim.find_similar(my_book, all_books, top_k=10)
+
+Advanced usage:
+    >>> # Custom configuration
+    >>> sim = (BookSimilarity()
+    ...     .content(weight=4.0)
+    ...     .authors(weight=2.0, metric=CustomMetric())
+    ...     .temporal(weight=1.0, sigma=5.0))
+    >>>
+    >>> # Compute similarity matrix for batch processing
+    >>> matrix = sim.similarity_matrix(books)
+    >>>
+    >>> # Save/load fitted state
+    >>> sim.save(Path("cache/similarity"))
+    >>> sim.load(Path("cache/similarity"))
+"""
+
+from ebk.similarity.base import Extractor, Feature, Metric
+from ebk.similarity.core import BookSimilarity
+from ebk.similarity.extractors import (
+    AuthorsExtractor,
+    ContentExtractor,
+    DescriptionExtractor,
+    LanguageExtractor,
+    PageCountExtractor,
+    PublicationYearExtractor,
+    PublisherExtractor,
+    SubjectsExtractor,
+)
+from ebk.similarity.metrics import (
+    CosineMetric,
+    ExactMatchMetric,
+    JaccardMetric,
+    NumericProximityMetric,
+    TemporalDecayMetric,
+    TfidfMetric,
+)
+
+__all__ = [
+    # Core
+    "BookSimilarity",
+    # Base classes
+    "Extractor",
+    "Metric",
+    "Feature",
+    # Extractors
+    "ContentExtractor",
+    "DescriptionExtractor",
+    "AuthorsExtractor",
+    "SubjectsExtractor",
+    "PublicationYearExtractor",
+    "LanguageExtractor",
+    "PublisherExtractor",
+    "PageCountExtractor",
+    # Metrics
+    "TfidfMetric",
+    "CosineMetric",
+    "JaccardMetric",
+    "ExactMatchMetric",
+    "TemporalDecayMetric",
+    "NumericProximityMetric",
+]