smartlex-search 1.0.0__tar.gz

smartlex_search-1.0.0/LICENSE

@@ -0,0 +1,14 @@
+Copyright (c) 2025 DAKSH GOEL
+All rights reserved.
+
+This software and associated documentation files (the “SmartLex”) are the
+intellectual property of DAKSH GOEL.
+
+Unauthorized copying, modification, distribution, or use of this software,
+in whole or in part, is strictly prohibited without explicit written
+permission from the copyright holder.
+
+The Software may not be used for commercial, educational, or research
+purposes without prior consent.
+
+For inquiries about licensing or usage, contact: `goel.daksh2008@gmail.com`.

smartlex_search-1.0.0/PKG-INFO

@@ -0,0 +1,188 @@
+Metadata-Version: 2.4
+Name: smartlex-search
+Version: 1.0.0
+Summary: A lexical, ultra-fast cross-platform search engine
+Author-email: Daksh Goel <daksh@example.com>
+License: Custom Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: PyQt5==5.15.9
+Requires-Dist: PyMuPDF==1.23.8
+Requires-Dist: rake-nltk==1.0.6
+Requires-Dist: nltk==3.8.1
+Requires-Dist: python-docx==1.1.0
+Requires-Dist: tqdm==4.66.1
+Requires-Dist: watchdog==3.0.0
+
+# Smart Lexical Search Engine
+
+## A powerful desktop search application that searches **inside** PDF and Word documents based on content keywords given by the user.
+
+##  Features
+
+### Core Features
+
+-    **Content-based search**: Find documents by keywords inside them
+-    **Multi-format support**: PDF and DOCX files
+-    **Fast search**: ~0.02 seconds after initial indexing
+-    **Smart ranking**: Results sorted by relevance
+-    **Auto-complete**: Suggests keywords dynamically as you type
+
+### Enhanced Features (New)
+
+-    **Standalone Executable**: No Python needed! Run straight from your desktop.
+-    **Cross-Platform Package**: Install via `pip install smartlex-search`
+-    **Modern UI**: Clean, beautifully designed Qt interface with dynamic status indicators.
+-    **Parallel processing**: Uses multithreading for blazingly fast background indexing.
+-    **Automated Setup**: Completely self-contained, handles NLTK and indexing without scripts!
+
+---
+
+##  Installation
+
+### Method 1: The Standalone App (No Python Required!)
+
+1. Go to the **Releases** page of this GitHub repository.
+2. Download `SmartLex.exe`.
+3. Double-click it! The app will automatically handle indexing and open right up.
+
+### Method 2: Terminal Package (For Developers)
+
+You can install this directly into your Python environment as a global package.
+
+1. **Install via pip**
+```bash
+pip install smartlex-search
+```
+
+2. **Run it from anywhere**
+```bash
+smartlex
+```
+
+*(Note: To install the developer version from source, simply clone the repo and run `pip install -e .`)*
+
+---
+
+##  Usage
+
+### First Run (Indexing)
+
+Whether you use the `.exe` or the `smartlex` terminal command, the application will automatically detect if it is your first time running it. It will launch a background scanner that sweeps your drives for PDFs and Word documents, extracts their keywords using advanced NLP, and builds your local index.
+
+⏱️ **First run**: Depends on the number of files (usually a few minutes)
+⏱️ **Subsequent runs**: Instant! The UI will pop open in less than a second.
+
+### Regular Usage
+
+1. Type keywords in the modern search bar (e.g., "machine learning").
+2. The dynamic auto-complete will suggest words based specifically on the contents of your own files!
+3. Press Enter or click Search.
+4. Click or double-click any beautifully formatted result to instantly open the document.
+
+### Keyboard Shortcuts
+
+-   `Enter`: Search / Open selected document
+-   `Tab`: Switch between search bar and results
+-   `↑/↓`: Navigate through results
+-   `Ctrl+F`: Focus the search bar
+-   `Esc`: Clear search / Close application
+
+---
+
+##  How It Works
+
+### Architecture
+
+```
+User Query → RAKE Extraction → Keyword Matching → Ranking → Results
+                                       ↑
+                Document Index ← Multithreading ← Automated File Scanner
+```
+
+### Step-by-Step Process
+
+#### 1️ **File Collection** (First Run Only)
+-   The internal Python scanner recursively sweeps all connected system drives.
+-   It strictly filters out system folders (`Windows`, `Program Files`, `.git`) for maximum speed.
+-   It collects all PDF and DOCX file paths.
+
+#### 2️ **Parallel Indexing** (First Run Only)
+-   Multiple background threads parse the documents simultaneously to maximize CPU efficiency.
+
+#### 3️ **Keyword Extraction** (RAKE Algorithm)
+```text
+Input: "Machine learning algorithms use neural networks"
+        ↓
+Remove stopwords: "Machine learning algorithms use neural networks"
+        ↓
+Extract phrases: ["machine learning algorithms", "neural networks"]
+        ↓
+Split to words: ["machine", "learning", "algorithms", "neural", "networks"]
+```
+
+#### 4️ **Index Storage**
+The system builds an ultra-fast lookup dictionary (`output.json`):
+```json
+{
+    "C:/docs/paper1.pdf": ["machine", "learning", "neural", "network"],
+    "C:/docs/paper2.pdf": ["algorithm", "optimization", "training"]
+}
+```
+
+#### 5️ **Search Process**
+```text
+User types: "neural network training"
+        ↓
+Extract keywords: ["neural", "network", "training"]
+        ↓
+Find matching documents:
+  paper1.pdf: 2 matches (neural, network)
+  paper2.pdf: 1 match (training)
+        ↓
+Rank by relevance:
+  1. paper1.pdf (score: 2)
+  2. paper2.pdf (score: 1)
+```
+
+---
+
+##  Project Structure
+
+```text
+SmartLex/
+│
+├── pyproject.toml         # Package configuration
+├── run.py                 # PyInstaller entry point
+│
+├── src/smartlex/
+│   ├── main.py            # Core application entry
+│   ├── core/              # NLP, parsing, and indexing logic
+│   └── gui/               # PyQt5 interface and background threads
+│
+├── requirements.txt
+├── config.json
+└── README.md
+```
+
+---
+
+## Reference
+
+-   [Base Paper / Reference PDF](https://drive.google.com/file/d/10f3bUmaTRzAZ2jOq6oWFu0ilP6Q6dyJx/view?usp=sharing)
+
+---
+
+## Sample Screenshots
+- ### 🪟 Initial Window
+![Initial Window](images/Initial_Window.png)
+
+### 📄 Output Window
+
+![Output Window](images/Output_Window.png)
+
+### Future enhancement
+- Working on adding support for additional file types such as TXT, CSV, XLSX, PPTX, HTML, etc.
+- Currently working on similar functionality for images, audios etc.

smartlex_search-1.0.0/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "smartlex-search"
+version = "1.0.0"
+authors = [
+  { name="Daksh Goel", email="daksh@example.com" },
+]
+description = "A lexical, ultra-fast cross-platform search engine"
+readme = "readme.md"
+license = { text = "Custom Proprietary License" }
+requires-python = ">=3.8"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "PyQt5==5.15.9",
+    "PyMuPDF==1.23.8",
+    "rake-nltk==1.0.6",
+    "nltk==3.8.1",
+    "python-docx==1.1.0",
+    "tqdm==4.66.1",
+    "watchdog==3.0.0"
+]
+
+[project.scripts]
+smartlex = "smartlex.main:main"
+
+[tool.setuptools]
+license-files = []

smartlex_search-1.0.0/readme.md

@@ -0,0 +1,170 @@
+# Smart Lexical Search Engine
+
+## A powerful desktop search application that searches **inside** PDF and Word documents based on content keywords given by the user.
+
+##  Features
+
+### Core Features
+
+-    **Content-based search**: Find documents by keywords inside them
+-    **Multi-format support**: PDF and DOCX files
+-    **Fast search**: ~0.02 seconds after initial indexing
+-    **Smart ranking**: Results sorted by relevance
+-    **Auto-complete**: Suggests keywords dynamically as you type
+
+### Enhanced Features (New)
+
+-    **Standalone Executable**: No Python needed! Run straight from your desktop.
+-    **Cross-Platform Package**: Install via `pip install smartlex-search`
+-    **Modern UI**: Clean, beautifully designed Qt interface with dynamic status indicators.
+-    **Parallel processing**: Uses multithreading for blazingly fast background indexing.
+-    **Automated Setup**: Completely self-contained, handles NLTK and indexing without scripts!
+
+---
+
+##  Installation
+
+### Method 1: The Standalone App (No Python Required!)
+
+1. Go to the **Releases** page of this GitHub repository.
+2. Download `SmartLex.exe`.
+3. Double-click it! The app will automatically handle indexing and open right up.
+
+### Method 2: Terminal Package (For Developers)
+
+You can install this directly into your Python environment as a global package.
+
+1. **Install via pip**
+```bash
+pip install smartlex-search
+```
+
+2. **Run it from anywhere**
+```bash
+smartlex
+```
+
+*(Note: To install the developer version from source, simply clone the repo and run `pip install -e .`)*
+
+---
+
+##  Usage
+
+### First Run (Indexing)
+
+Whether you use the `.exe` or the `smartlex` terminal command, the application will automatically detect if it is your first time running it. It will launch a background scanner that sweeps your drives for PDFs and Word documents, extracts their keywords using advanced NLP, and builds your local index.
+
+⏱️ **First run**: Depends on the number of files (usually a few minutes)
+⏱️ **Subsequent runs**: Instant! The UI will pop open in less than a second.
+
+### Regular Usage
+
+1. Type keywords in the modern search bar (e.g., "machine learning").
+2. The dynamic auto-complete will suggest words based specifically on the contents of your own files!
+3. Press Enter or click Search.
+4. Click or double-click any beautifully formatted result to instantly open the document.
+
+### Keyboard Shortcuts
+
+-   `Enter`: Search / Open selected document
+-   `Tab`: Switch between search bar and results
+-   `↑/↓`: Navigate through results
+-   `Ctrl+F`: Focus the search bar
+-   `Esc`: Clear search / Close application
+
+---
+
+##  How It Works
+
+### Architecture
+
+```
+User Query → RAKE Extraction → Keyword Matching → Ranking → Results
+                                       ↑
+                Document Index ← Multithreading ← Automated File Scanner
+```
+
+### Step-by-Step Process
+
+#### 1️ **File Collection** (First Run Only)
+-   The internal Python scanner recursively sweeps all connected system drives.
+-   It strictly filters out system folders (`Windows`, `Program Files`, `.git`) for maximum speed.
+-   It collects all PDF and DOCX file paths.
+
+#### 2️ **Parallel Indexing** (First Run Only)
+-   Multiple background threads parse the documents simultaneously to maximize CPU efficiency.
+
+#### 3️ **Keyword Extraction** (RAKE Algorithm)
+```text
+Input: "Machine learning algorithms use neural networks"
+        ↓
+Remove stopwords: "Machine learning algorithms use neural networks"
+        ↓
+Extract phrases: ["machine learning algorithms", "neural networks"]
+        ↓
+Split to words: ["machine", "learning", "algorithms", "neural", "networks"]
+```
+
+#### 4️ **Index Storage**
+The system builds an ultra-fast lookup dictionary (`output.json`):
+```json
+{
+    "C:/docs/paper1.pdf": ["machine", "learning", "neural", "network"],
+    "C:/docs/paper2.pdf": ["algorithm", "optimization", "training"]
+}
+```
+
+#### 5️ **Search Process**
+```text
+User types: "neural network training"
+        ↓
+Extract keywords: ["neural", "network", "training"]
+        ↓
+Find matching documents:
+  paper1.pdf: 2 matches (neural, network)
+  paper2.pdf: 1 match (training)
+        ↓
+Rank by relevance:
+  1. paper1.pdf (score: 2)
+  2. paper2.pdf (score: 1)
+```
+
+---
+
+##  Project Structure
+
+```text
+SmartLex/
+│
+├── pyproject.toml         # Package configuration
+├── run.py                 # PyInstaller entry point
+│
+├── src/smartlex/
+│   ├── main.py            # Core application entry
+│   ├── core/              # NLP, parsing, and indexing logic
+│   └── gui/               # PyQt5 interface and background threads
+│
+├── requirements.txt
+├── config.json
+└── README.md
+```
+
+---
+
+## Reference
+
+-   [Base Paper / Reference PDF](https://drive.google.com/file/d/10f3bUmaTRzAZ2jOq6oWFu0ilP6Q6dyJx/view?usp=sharing)
+
+---
+
+## Sample Screenshots
+- ### 🪟 Initial Window
+![Initial Window](images/Initial_Window.png)
+
+### 📄 Output Window
+
+![Output Window](images/Output_Window.png)
+
+### Future enhancement
+- Working on adding support for additional file types such as TXT, CSV, XLSX, PPTX, HTML, etc.
+- Currently working on similar functionality for images, audios etc.

smartlex_search-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

smartlex_search-1.0.0/src/smartlex/core/config.py

@@ -0,0 +1,23 @@
+# core/config.py
+import json
+from pathlib import Path
+
+CONFIG_FILE = Path("config.json")
+
+DEFAULT_CONFIG = {
+    "NUM_PROCESSES": 8,
+    "TOP_KEYWORDS": 150,
+    "AUTOCOMPLETE_WORDS": 100,
+    "SUPPORTED_FORMATS": [".pdf", ".docx"],
+    "INDEX_FOLDER": "all",
+    "OUTPUT_FILE": "output.json",
+    "AUTOCOMPLETE_FILE": "autocomplete_words.json",
+}
+
+
+def load_config():
+    if CONFIG_FILE.exists():
+        with open(CONFIG_FILE, "r") as f:
+            user_config = json.load(f)
+        return {**DEFAULT_CONFIG, **user_config}
+    return DEFAULT_CONFIG

smartlex_search-1.0.0/src/smartlex/core/index_manager.py

@@ -0,0 +1,21 @@
+import json
+from collections import Counter
+from pathlib import Path
+
+
+def save_index(data, output_file):
+    with open(output_file, "w") as f:
+        json.dump(data, f, indent=2)
+
+
+def load_index(output_file):
+    with open(output_file, "r") as f:
+        return json.load(f)
+
+
+def generate_autocomplete(data, top_n):
+    words = []
+    for kws in data.values():
+        words.extend(kws)
+    freq = Counter(words)
+    return [w for w, _ in freq.most_common(top_n)]

smartlex_search-1.0.0/src/smartlex/core/keyword_extraction.py

@@ -0,0 +1,29 @@
+import re
+
+from rake_nltk import Rake
+
+
+def rake_keywords(text):
+    if not text or not text.strip():
+        return []
+
+    r = Rake()
+    r.extract_keywords_from_text(text)
+    ranked_phrases = r.get_ranked_phrases()
+    filtered = []
+
+    for phrase in ranked_phrases:
+        if any(
+            [
+                re.search(r"\b\w\b", phrase),
+                re.search(r"\d", phrase),
+                re.search(r"[*&!()?/>.<,:;\"\]\[\}\{]", phrase),
+            ]
+        ):
+            continue
+
+        for word in phrase.split():
+            if len(word) > 2:
+                filtered.append(word.lower())
+
+    return filtered

smartlex_search-1.0.0/src/smartlex/core/logger.py

@@ -0,0 +1,23 @@
+# core/logger.py
+import logging
+from pathlib import Path
+
+
+def setup_logger(name="search_engine", log_file="search_engine.log"):
+    log_path = Path("logs")
+    log_path.mkdir(exist_ok=True)
+
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+
+    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+
+    fh = logging.FileHandler(log_path / log_file)
+    sh = logging.StreamHandler()
+
+    fh.setFormatter(formatter)
+    sh.setFormatter(formatter)
+
+    logger.addHandler(fh)
+    logger.addHandler(sh)
+    return logger

smartlex_search-1.0.0/src/smartlex/core/processor.py

@@ -0,0 +1,54 @@
+import concurrent.futures
+import os
+from collections import Counter
+from pathlib import Path
+
+from smartlex.core.keyword_extraction import rake_keywords
+from smartlex.core.logger import setup_logger
+from smartlex.core.text_extraction import extract_text_docx, extract_text_pdf
+
+logger = setup_logger(__name__)
+
+
+def extract_keywords_from_file(file_path):
+    if file_path.endswith(".pdf"):
+        text = extract_text_pdf(file_path)
+    elif file_path.endswith(".docx"):
+        text = extract_text_docx(file_path)
+    else:
+        return []
+    return rake_keywords(text)
+
+
+def process_batch(batch_file):
+    result = {}
+    if not os.path.exists(batch_file):
+        logger.error(f"Batch file not found: {batch_file}")
+        return result
+
+    with open(batch_file, "r", encoding="utf-8") as f:
+        paths = [line.strip() for line in f]
+
+    for path in paths:
+        if os.path.exists(path):
+            kws = extract_keywords_from_file(path)
+            if kws:
+                result[path] = kws
+    return result
+
+
+def refine_keywords(data, top_n):
+    for k, v in data.items():
+        c = Counter(v)
+        top_values = [w for w, _ in c.most_common(top_n)]
+        data[k] = list(set(top_values))
+    return data
+
+
+def process_all_batches(batch_files, top_n):
+    D = {}
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        results = list(executor.map(process_batch, batch_files))
+    for res in results:
+        D.update(res)
+    return refine_keywords(D, top_n)