scitex 2.13.0__py3-none-any.whl → 2.15.1__py3-none-any.whl

scitex/scholar/.legacy/metadata/urls/zotero_translators/zotero.org.js

@@ -1,311 +0,0 @@
-{
-	"translatorID": "c82c574d-7fe8-49ca-a360-a05d6e34fec0",
-	"label": "zotero.org",
-	"creator": "Dan Stillman, Aurimas Vinckevicius, Tomasz Najdek",
-	"target": "^https?://[^/]*zotero\\.org(:\\d+)?/.+/(items|collections|library|publications|trash|search|item-list|item-details)(/|$)",
-	"minVersion": "3.0",
-	"maxVersion": "",
-	"priority": 100,
-	"inRepository": true,
-	"translatorType": 4,
-	"browserSupport": "gcsv",
-	"lastUpdated": "2021-07-18 22:33:42"
-}
-
-/*
-  ***** BEGIN LICENSE BLOCK *****
-
-  Copyright © 2017-2019 Dan Stillman and Aurimas Vinckevicius
-
-  This file is part of Zotero.
-
-  Zotero is free software: you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published by
-  the Free Software Foundation, either version 3 of the License, or
-  (at your option) any later version.
-
-  Zotero is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-  GNU Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with Zotero. If not, see <http://www.gnu.org/licenses/>.
-
-  ***** END LICENSE BLOCK *****
-*/
-
-function scrape(text) {
-	var item = JSON.parse(text);
-	var newItem = new Zotero.Item();
-	for (var prop in item.data) {
-		switch (prop) {
-			case 'key':
-			case 'version':
-			case 'collections':
-			case 'relations':
-			case 'dateAdded':
-			case 'dateModified':
-				continue;
-		}
-		newItem[prop] = item.data[prop];
-	}
-	// TODO: Don't pass for notes, once client no longer requires it
-	if (!newItem.title) newItem.title = "[Untitled]";
-	newItem.complete();
-}
-
-function getListTitles(doc) {
-	return ZU.xpath(doc, '//table[@id="field-table"]//td[@class="title"]'
-		+ '[./a[not(contains(text(), "Unpublished Note"))]'
-			+ '/span[not(contains(@class,"sprite-treeitem-attachment"))]]');
-}
-
-function isNewWebLibrary(doc) {
-	return !!doc.querySelector('#zotero-web-library');
-}
-
-function legacyGetLibraryURI(doc) {
-	var feed = ZU.xpath(doc, '//a[@type="application/atom+xml" and @rel="alternate"]')[0];
-	if (!feed) return false;
-	var url = feed.href.match(/^.+?\/(?:users|groups)\/\w+/);
-	if (!url) {
-		url = decodeURIComponent(feed.href)
-			.match(/https?:\/\/[^/]+\/(?:users|groups)\/\w+/);
-		if (!url) return false;
-	}
-	if (!url) return false;
-	return (url[0] + '/items/').replace("https://api.zotero.org", "https://www.zotero.org/api");
-}
-
-function getLibraryURI(doc, url) {
-	if (!isNewWebLibrary(doc)) {
-		return legacyGetLibraryURI(doc);
-	}
-
-	var groupMatch = url.match(/zotero.org\/groups\/(\d+)\/.*?/i);
-	if (groupMatch) {
-		const groupId = groupMatch[1];
-		return `https://www.zotero.org/api/groups/${groupId}/items`;
-	}
-	else {
-		const config = JSON.parse(doc.querySelector('#zotero-web-library-config').textContent);
-		const { userId } = config;
-		return `https://www.zotero.org/api/users/${userId}/items`;
-	}
-}
-
-function getSessionKey(doc) {
-	var matches = doc.cookie.match(/zotero_www_session_v2=([a-z0-9]+)/);
-	return matches ? matches[1] : null;
-}
-
-function legacyDetectWeb(doc, url) {
-	// single item
-	if (url.match(/\/itemKey\/\w+/)) {
-		return ZU.xpathText(doc, '//div[@id="item-details-div"]//td[preceding-sibling::th[text()="Item Type"]]/@class')
-				|| false;
-	}
-
-	// Library and collections
-	if ((url.match(/\/items\/?([?#].*)?$/)
-		|| url.includes('/collectionKey/')
-		|| url.match(/\/collection\/\w+/)
-		|| url.includes('/tag/'))
-		&& getListTitles(doc).length) {
-		return "multiple";
-	}
-
-	return false;
-}
-
-function detectWeb(doc, url) {
-	// disable for libraries where we can't get a library URI or an apiKey
-	if (!getLibraryURI(doc, url)) {
-		return false;
-	}
-
-	if (!isNewWebLibrary(doc)) {
-		return legacyDetectWeb(doc, url);
-	}
-
-	const singleItemDataEl = doc.querySelector('script#translator-current-item');
-
-	if (singleItemDataEl) {
-		const { itemType } = JSON.parse(singleItemDataEl.textContent);
-		return itemType;
-	}
-
-	const multipleItemsDataEl = doc.querySelector('script#translator-items-list');
-
-	if (multipleItemsDataEl
-		&& Object.keys(JSON.parse(multipleItemsDataEl.textContent)).length) {
-		return 'multiple';
-	}
-
-	return null;
-}
-
-function getCommon(doc, url) {
-	var headers = {
-		'Zotero-API-Version': 3
-	};
-	var libraryURI = getLibraryURI(doc, url);
-	// Pass session key in URL, as required for CSRF protection. This allows saving of items
-	// in private libraries.
-	var sessionKey = getSessionKey(doc);
-	var suffix = sessionKey ? '?session=' + sessionKey : "";
-
-	return { headers, libraryURI, sessionKey, suffix };
-}
-
-function legacyDoWeb(doc, url) {
-	const { headers, libraryURI, suffix } = getCommon(doc, url);
-	var itemRe = /\/itemKey\/(\w+)/;
-
-	if (detectWeb(doc, url) == "multiple") {
-		var elems = getListTitles(doc);
-		var items = ZU.getItemArray(doc, elems);
-
-		Zotero.selectItems(items, (selectedItems) => {
-			if (!selectedItems) {
-				return;
-			}
-
-			var apiURIs = [], itemID;
-			for (var url in selectedItems) {
-				itemID = url.match(itemRe)[1];
-				apiURIs.push(libraryURI + itemID + suffix);
-			}
-
-			Zotero.Utilities.HTTP.doGet(apiURIs, scrape, null, null, headers);
-		});
-	}
-	else {
-		var itemID = url.match(itemRe)[1];
-		var itemURI = libraryURI + itemID + suffix;
-		Zotero.Utilities.doGet(itemURI, scrape, null, null, headers);
-	}
-}
-
-function doWeb(doc, url) {
-	if (!isNewWebLibrary(doc)) {
-		legacyDoWeb(doc, url);
-	}
-
-	const { headers, libraryURI, suffix } = getCommon(doc, url);
-
-	const singleItemDataEl = doc.querySelector('script#translator-current-item');
-
-	if (singleItemDataEl) {
-		const item = JSON.parse(singleItemDataEl.textContent);
-		const itemURI = `${libraryURI}/${item.key}${suffix}`;
-		Zotero.Utilities.doGet(itemURI, scrape, null, null, headers);
-	}
-	else {
-		const multipleItemsDataEl = doc.querySelector('script#translator-items-list');
-
-		if (multipleItemsDataEl) {
-			const options = JSON.parse(multipleItemsDataEl.textContent);
-			Zotero.selectItems(options, (selectedItems) => {
-				if (!selectedItems) {
-					return;
-				}
-
-				const selectedItemsURIs = Object.keys(selectedItems).map(itemKey => `${libraryURI}/${itemKey}${suffix}`);
-				Zotero.Utilities.HTTP.doGet(selectedItemsURIs, scrape, null, null, headers);
-			});
-		}
-	}
-}
-
-/** BEGIN TEST CASES **/
-var testCases = [
-	{
-		"type": "web",
-		"url": "https://www.zotero.org/groups/729/all_things_zotero/items/itemKey/HXTTNJGD?usenewlibrary=0",
-		"defer": true,
-		"items": [
-			{
-				"itemType": "journalArticle",
-				"title": "Expert Searching, Zotero: A New Breed of Search Tool",
-				"creators": [
-					{
-						"creatorType": "author",
-						"firstName": "Mark",
-						"lastName": "Desirto"
-					}
-				],
-				"date": "April 2007",
-				"extra": "00000 \nCited by 0000",
-				"publicationTitle": "Medical Library Association Newsletter",
-				"attachments": [],
-				"tags": [],
-				"notes": [],
-				"seeAlso": []
-			}
-		]
-	},
-	{
-		"type": "web",
-		"url": "https://www.zotero.org/groups/729/all_things_zotero/items/HXTTNJGD?usenewlibrary=1",
-		"defer": true,
-		"items": [
-			{
-				"itemType": "journalArticle",
-				"title": "Expert Searching, Zotero: A New Breed of Search Tool",
-				"creators": [
-					{
-						"creatorType": "author",
-						"firstName": "Mark",
-						"lastName": "Desirto"
-					}
-				],
-				"date": "April 2007",
-				"extra": "00000 \nCited by 0000",
-				"publicationTitle": "Medical Library Association Newsletter",
-				"attachments": [],
-				"tags": [],
-				"notes": [],
-				"seeAlso": []
-			}
-		]
-	},
-	{
-		"type": "web",
-		"url": "https://www.zotero.org/groups/all_things_zotero/items?usenewlibrary=0",
-		"defer": true,
-		"items": "multiple"
-	},
-	{
-		"type": "web",
-		"url": "https://www.zotero.org/groups/729/all_things_zotero/items/BBNKRTN9,BVRM2S5S,VMUNGCPG/item-list?usenewlibrary=1",
-		"defer": true,
-		"items": "multiple"
-	},
-	{
-		"type": "web",
-		"url": "https://www.zotero.org/groups/all_things_zotero/items/collectionKey/XX99JMW8?usenewlibrary=0",
-		"defer": true,
-		"items": "multiple"
-	},
-	{
-		"type": "web",
-		"url": "https://www.zotero.org/groups/729/all_things_zotero/collections/XX99JMW8/items/BVRM2S5S,HXTTNJGD,92N4P9MH/item-list?usenewlibrary=1",
-		"defer": true,
-		"items": "multiple"
-	},
-	{
-		"type": "web",
-		"url": "https://www.zotero.org/groups/devtesting/items/tag/tag2?usenewlibrary=0",
-		"defer": true,
-		"items": "multiple"
-	},
-	{
-		"type": "web",
-		"url": "https://www.zotero.org/groups/729/all_things_zotero/tags/blogs/item-list?usenewlibrary=1",
-		"defer": true,
-		"items": "multiple"
-	}
-]
-/** END TEST CASES **/

scitex/scholar/.legacy/search/README.md

@@ -1,205 +0,0 @@
-# Scholar Semantic Search Module
-
-Find semantically similar papers using embeddings and vector search.
-
-## Features
-
-### Embedder
-- Multiple embedding models (Sentence Transformers, TF-IDF)
-- GPU acceleration support
-- Embedding caching for performance
-- Paper-specific embedding (title, abstract, keywords)
-
-### VectorDatabase
-- Fast nearest neighbor search (FAISS/NumPy)
-- Multiple index types (flat, IVF, HNSW)
-- Persistent storage
-- Metadata support
-- Batch operations
-
-### SemanticSearchEngine
-- Index papers from database
-- Find similar papers
-- Search by text query
-- Multi-paper recommendations
-- Hybrid search (semantic + keyword)
-- Metadata filtering
-
-## Installation
-
-```bash
-# Basic (NumPy/TF-IDF)
-pip install scikit-learn
-
-# Recommended (Sentence Transformers)
-pip install sentence-transformers
-
-# Advanced (FAISS for fast search)
-pip install faiss-cpu  # or faiss-gpu
-```
-
-## Usage
-
-```python
-from scitex.scholar.database import PaperDatabase
-from scitex.scholar.search import SemanticSearchEngine
-
-# Initialize with database
-db = PaperDatabase()
-engine = SemanticSearchEngine(
-    database=db,
-    model_name="all-MiniLM-L6-v2",  # Fast and good
-    index_type="flat",  # Exact search
-    use_gpu=False
-)
-
-# Index all papers
-stats = engine.index_papers(
-    fields=["title", "abstract", "keywords"]
-)
-print(f"Indexed {stats['indexed']} papers")
-
-# Search by text
-results = engine.search_by_text(
-    "deep learning for climate modeling",
-    k=10,
-    search_mode="hybrid"  # Semantic + keyword
-)
-
-for paper, score in results:
-    print(f"{score:.3f}: {paper.title} ({paper.year})")
-
-# Find similar papers
-paper = db.get_entry("doi_10.1038_nature12345")
-similar = engine.find_similar_papers(paper, k=5)
-
-# Get recommendations based on multiple papers
-recommendations = engine.recommend_papers(
-    entry_ids=["doi_1", "doi_2", "doi_3"],
-    k=10,
-    method="average"
-)
-
-# Filter results
-ml_papers_2024 = engine.search_similar(
-    "machine learning",
-    k=20,
-    filters={
-        "year_min": 2024,
-        "has_pdf": True,
-        "tag": "reviewed"
-    }
-)
-```
-
-## Embedding Models
-
-### Sentence Transformers (Recommended)
-- `all-MiniLM-L6-v2`: Fast, good quality (384 dim)
-- `all-mpnet-base-v2`: Higher quality (768 dim)
-- `allenai-specter`: Scientific papers (768 dim)
-
-### TF-IDF (Fallback)
-- No dependencies beyond scikit-learn
-- Vocabulary-based embeddings
-- Good for keyword matching
-
-## Index Types
-
-### Flat (Default)
-- Exact nearest neighbor search
-- Best for < 100k papers
-- No training required
-
-### IVF (Inverted File)
-- Approximate search
-- Good for 100k-1M papers
-- Requires training
-
-### HNSW (Hierarchical NSW)
-- Very fast approximate search
-- Good for > 1M papers
-- Higher memory usage
-
-## Performance Tips
-
-1. **Batch Indexing**: Index papers in batches
-   ```python
-   engine.index_papers(batch_size=1000)
-   ```
-
-2. **GPU Acceleration**: Use GPU for faster search
-   ```python
-   engine = SemanticSearchEngine(use_gpu=True)
-   ```
-
-3. **Caching**: Embeddings are cached automatically
-   ```python
-   # Cache location: ~/.scitex/scholar/embeddings_cache/
-   ```
-
-4. **Incremental Updates**: Index only new papers
-   ```python
-   engine.index_papers()  # Only indexes new entries
-   ```
-
-## Advanced Features
-
-### Custom Fields
-```python
-# Index with custom fields
-engine.index_papers(
-    fields=["title", "abstract", "authors"]
-)
-```
-
-### Similarity Metrics
-```python
-# In Embedder class
-similarities = embedder.compute_similarity(
-    embeddings1, 
-    embeddings2,
-    metric="cosine"  # or "dot", "euclidean"
-)
-```
-
-### Update Index
-```python
-# Update after paper modification
-engine.update_index(entry_id)
-```
-
-### Search Modes
-- `semantic`: Pure embedding search
-- `keyword`: Title/abstract keyword match
-- `hybrid`: Combines both approaches
-
-## Storage
-
-Default location: `~/.scitex/scholar/vector_db/`
-
-Structure:
-```
-vector_db/
-├── embeddings.npy       # Embedding vectors
-├── entry_ids.json       # Paper IDs
-├── metadata.json        # Metadata for each embedding
-├── faiss.index         # FAISS index (if available)
-├── config.json         # Database configuration
-└── indexed_entries.json # List of indexed papers
-```
-
-## Troubleshooting
-
-### No sentence-transformers
-- Falls back to TF-IDF automatically
-- Install with: `pip install sentence-transformers`
-
-### FAISS not available
-- Uses NumPy for search (slower but works)
-- Install with: `pip install faiss-cpu`
-
-### Out of memory
-- Use smaller model: `all-MiniLM-L6-v2`
-- Index in smaller batches
-- Use approximate index (IVF/HNSW)