BibexPy 2.0.0__py3-none-any.whl

bibexpy/_server/config.py

@@ -0,0 +1,92 @@
+import os
+from pathlib import Path
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+def _resolve_env_file() -> str:
+    """`.env` dosyasının yolu.
+
+    `BIBEXPY_CONFIG_DIR` env var'ı set ise oradaki `.env` kullanılır (paket
+    modu: ~/.bibexpy/.env gibi yazılabilir bir konum). Aksi halde CWD'deki
+    `.env` (geliştirme: uvicorn apps/api'den çalışır, CWD = apps/api).
+    """
+    cfg_dir = os.environ.get("BIBEXPY_CONFIG_DIR")
+    if cfg_dir:
+        return str(Path(cfg_dir).expanduser() / ".env")
+    return ".env"
+
+
+def _default_storage_dir() -> str:
+    """Yazilabilir varsayilan depolama yolu.
+
+    `STORAGE_DIR` env'i (cli.py pakette set eder) varsa pydantic onu okur.
+    Aksi halde: gelistirme agacindaki mevcut `./storage` varsa onu (dev'i
+    bozmamak icin); yoksa kullanici-yazilabilir `~/.bibexpy/storage`
+    (pip/rastgele CWD'de veriyi rastgele yere yazmamak icin).
+    """
+    if Path("./storage").is_dir():
+        return "./storage"
+    return str(Path.home() / ".bibexpy" / "storage")
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=_resolve_env_file(),
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    api_host: str = "127.0.0.1"
+    api_port: int = 8001
+    api_cors_origins: str = "http://localhost:3000,http://127.0.0.1:3000,http://localhost:3001,http://127.0.0.1:3001"
+
+    storage_dir: str = _default_storage_dir()
+
+    scopus_api_key: str = ""
+    semantic_scholar_api_key: str = ""
+    unpaywall_email: str = ""
+    crossref_email: str = ""
+
+    # LLM yapılandırması — yeni birleşik alanlar
+    llm_provider: str = "deepseek"                  # "deepseek" | "openai" | "custom"
+    llm_api_key: str = ""
+    llm_base_url: str = "https://api.deepseek.com"
+    llm_model: str = "deepseek-v4-flash"
+
+    # Geriye dönük uyumluluk — eski .env'lerden okuyacak
+    deepseek_api_key: str = ""
+    deepseek_model: str = ""
+    deepseek_base_url: str = ""
+
+    disambiguation_enabled: bool = True
+    disambiguation_blocking_threshold: float = 0.75
+    disambiguation_auto_approve_threshold: float = 0.95
+    # ORCID-öncelikli disambiguation: ambigu gruplar için OpenAlex'e anlık ORCID
+    # sorgusu (veride OI yoksa) ve çekileni veriye geri yazma.
+    disambiguation_orcid_lookup: bool = True
+    disambiguation_orcid_persist: bool = True
+
+    @property
+    def effective_llm_api_key(self) -> str:
+        return self.llm_api_key or self.deepseek_api_key
+
+    @property
+    def effective_llm_base_url(self) -> str:
+        return self.llm_base_url or self.deepseek_base_url or "https://api.deepseek.com"
+
+    @property
+    def effective_llm_model(self) -> str:
+        return self.llm_model or self.deepseek_model or "deepseek-v4-flash"
+
+    @property
+    def storage_path(self) -> Path:
+        p = Path(self.storage_dir).resolve()
+        p.mkdir(parents=True, exist_ok=True)
+        return p
+
+    @property
+    def cors_origins_list(self) -> list[str]:
+        return [o.strip() for o in self.api_cors_origins.split(",") if o.strip()]
+
+
+settings = Settings()

bibexpy/_server/jobs/__init__.py

@@ -0,0 +1,3 @@
+from .runner import JobRunner, Job, JobStatus, job_runner
+
+__all__ = ["JobRunner", "Job", "JobStatus", "job_runner"]

bibexpy/_server/jobs/runner.py

@@ -0,0 +1,301 @@
+"""Basit, tek-kullanıcılı job runner.
+
+- Tek aktif iş + sıralı kuyruk (self-hosted için yeterli)
+- Bellekte log + progress; opsiyonel olarak proje klasörüne persist
+- SSE üzerinden canlı stream
+"""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import time
+import traceback
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Awaitable, Callable, Optional
+
+
+# Ağır CPU işi (disambiguation blocking builder'ları) için AYRILMIŞ, sınırlı havuz.
+# Neden: bu işler `asyncio.to_thread` ile çalışırsa PAYLAŞILAN default thread havuzunu
+# kullanır. Bir scan iptal edilince `task.cancel()` yalnız asyncio task'ı iptal eder —
+# `build_author_blocks` gibi fonksiyonu çalıştıran thread Python'da ÖLDÜRÜLEMEZ. İptal
+# edilen scan'lerin orphan thread'leri default havuzu tüketir → export/merge/conversion
+# ve SPA-serving dâhil TÜM `to_thread` işleri kuyruğa girer = uygulama kilitlenir
+# (test bulgusu #7). Ağır işi ayrı havuza alınca orphan'lar burada izole kalır; default
+# havuz (uygulamanın geri kalanı) yanıt vermeye devam eder. blocking.py'deki kooperatif
+# iptal (should_cancel) sayesinde orphan'lar zaten ~bir iterasyonda biter.
+_CPU_EXECUTOR = ThreadPoolExecutor(max_workers=4, thread_name_prefix="bibex-cpu")
+
+
+async def run_cpu(fn: Callable, *args, **kwargs):
+    """Ağır/iptal-edilebilir CPU işini AYRILMIŞ havuzda çalıştır (default havuzu koru).
+
+    `asyncio.to_thread` yerine bunu kullan: orphan thread'ler izole havuzda kalır,
+    uygulamanın geri kalanını (export/merge/serving) kilitlemez."""
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(_CPU_EXECUTOR, functools.partial(fn, *args, **kwargs))
+
+
+class JobStatus(str, Enum):
+    queued = "queued"
+    running = "running"
+    completed = "completed"
+    failed = "failed"
+    cancelled = "cancelled"
+
+
+@dataclass
+class Job:
+    id: str
+    project_id: str
+    kind: str
+    title: str
+    # i18n: title_key + title_params → frontend (JobProgress) display-time çevirir.
+    # title alanı okunabilir fallback (markdown/legacy) olarak kalır.
+    title_key: Optional[str] = None
+    title_params: Optional[dict[str, Any]] = None
+    # exclusive: aynı projede aynı anda başka bir exclusive iş varsa REDDEDİLİR
+    # (ağır scan/fill işlerinin üst üste binip thread havuzunu tüketmesini önler — #7).
+    exclusive: bool = False
+    status: JobStatus = JobStatus.queued
+    progress: float = 0.0
+    log: list[str] = field(default_factory=list)
+    result: Any = None
+    error: Optional[str] = None
+    started_at: Optional[float] = None
+    finished_at: Optional[float] = None
+    created_at: float = field(default_factory=time.time)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "project_id": self.project_id,
+            "kind": self.kind,
+            "title": self.title,
+            "title_key": self.title_key,
+            "title_params": self.title_params or None,
+            "status": self.status.value,
+            "progress": round(self.progress, 3),
+            "log_tail": self.log[-50:],
+            "result": self.result,
+            "error": self.error,
+            "started_at": self.started_at,
+            "finished_at": self.finished_at,
+            "created_at": self.created_at,
+        }
+
+
+class JobContext:
+    """Worker fonksiyonuna geçirilen kontrol nesnesi."""
+
+    def __init__(self, job: Job, runner: "JobRunner"):
+        self._job = job
+        self._runner = runner
+
+    def log(self, line: str) -> None:
+        self._job.log.append(line)
+        self._runner._notify(self._job.id)
+
+    def progress(self, value: float) -> None:
+        self._job.progress = max(0.0, min(1.0, value))
+        self._runner._notify(self._job.id)
+
+    @property
+    def cancelled(self) -> bool:
+        return self._job.status == JobStatus.cancelled
+
+
+JobWorker = Callable[[JobContext], Awaitable[Any]]
+
+
+class JobRunner:
+    """Bellekte job kayıt + asyncio task'lar."""
+
+    def __init__(self):
+        self._jobs: dict[str, Job] = {}
+        self._tasks: dict[str, asyncio.Task] = {}
+        self._listeners: dict[str, set[asyncio.Queue]] = {}
+
+    # --- CRUD ---
+    def list(self, project_id: Optional[str] = None) -> list[Job]:
+        items = list(self._jobs.values())
+        if project_id:
+            items = [j for j in items if j.project_id == project_id]
+        items.sort(key=lambda j: j.created_at, reverse=True)
+        return items
+
+    def get(self, job_id: str) -> Optional[Job]:
+        return self._jobs.get(job_id)
+
+    def cancel(self, job_id: str) -> bool:
+        job = self._jobs.get(job_id)
+        if not job or job.status not in (JobStatus.queued, JobStatus.running):
+            return False
+        job.status = JobStatus.cancelled
+        task = self._tasks.get(job_id)
+        if task and not task.done():
+            task.cancel()
+        self._notify(job_id, done=True)
+        return True
+
+    def has_active_exclusive(self, project_id: str) -> bool:
+        """Bu projede aktif (kuyrukta/çalışan) bir exclusive iş var mı?"""
+        return any(
+            j.project_id == project_id and j.exclusive
+            and j.status in (JobStatus.queued, JobStatus.running)
+            for j in self._jobs.values()
+        )
+
+    # --- enqueue ---
+    def submit(
+        self,
+        project_id: str,
+        kind: str,
+        title: str,
+        worker: JobWorker,
+        *,
+        title_key: Optional[str] = None,
+        title_params: Optional[dict[str, Any]] = None,
+        exclusive: bool = False,
+    ) -> Job:
+        # Tek-aktif-iş guard (#7): exclusive bir iş istenirken aynı projede zaten
+        # exclusive bir iş varsa reddet. Frontend buton kilidi desenkronize olsa bile
+        # (eski donma bug'ının tetikleyicisi) ağır işler üst üste binmez.
+        if exclusive and self.has_active_exclusive(project_id):
+            from fastapi import HTTPException
+            raise HTTPException(409, "job_already_running")
+        job = Job(
+            id=uuid.uuid4().hex[:12], project_id=project_id, kind=kind,
+            title=title, title_key=title_key, title_params=title_params,
+            exclusive=exclusive,
+        )
+        self._jobs[job.id] = job
+        task = asyncio.create_task(self._run(job, worker))
+        self._tasks[job.id] = task
+        return job
+
+    async def _run(self, job: Job, worker: JobWorker) -> None:
+        ctx = JobContext(job, self)
+        try:
+            job.status = JobStatus.running
+            job.started_at = time.time()
+            self._notify(job.id)
+            result = await worker(ctx)
+            if job.status == JobStatus.cancelled:
+                return
+            job.result = result
+            job.progress = 1.0
+            job.status = JobStatus.completed
+        except asyncio.CancelledError:
+            job.status = JobStatus.cancelled
+            raise
+        except Exception as e:
+            job.status = JobStatus.failed
+            job.error = str(e)
+            job.log.append("HATA: " + str(e))
+            job.log.append(traceback.format_exc())
+        finally:
+            job.finished_at = time.time()
+            self._notify(job.id, done=True)
+            # Otomatik audit log yazımı — her job sonunda
+            self._write_audit(job)
+
+    def _write_audit(self, job: Job) -> None:
+        """Job tamamlandıktan sonra audit log'a yaz."""
+        try:
+            from services import audit  # circular import'ı önle
+            duration = (job.finished_at - job.started_at) if (job.started_at and job.finished_at) else None
+            details: dict = {
+                "job_id": job.id,
+                "duration_seconds": round(duration, 2) if duration else None,
+                "status": job.status.value,
+            }
+            if isinstance(job.result, dict):
+                # Result alanlarının ilgili kısımlarını da ekle (verbose olmasın)
+                for k in ("method", "scopus_input", "wos_input", "merged_count", "duplicates_removed",
+                          "stats", "enriched_count", "updated_count", "replacements", "snapshot",
+                          "candidates", "clusters", "uncertain", "output_xlsx",
+                          "clusters_proposed", "splits_proposed", "auto",
+                          "orcid_rows_filled", "dois_fetched",  # ORCID disambiguation write-back
+                          # Field/selected enrichment
+                          "field", "label", "total", "missing_before", "no_doi", "target",
+                          "enriched", "api_no_data", "failed", "missing_after",
+                          "fill_rate_before", "fill_rate_after", "per_field_fill",
+                          "api", "doi",  # fill_all alt-özetleri (enrichment raporu için)
+                          "api_success_by_source", "fields_updated", "doi_missing",
+                          # Smart Merge alanları
+                          "matched_pairs", "borderline_count", "borderline_pending",
+                          "conflict_count", "match_stages", "field_source_distribution",
+                          "lost_wos_count", "lost_scopus_count", "output_files",
+                          # Analiz klasörü
+                          "analysis_id"):
+                    if k in job.result:
+                        v = job.result[k]
+                        if isinstance(v, list):
+                            details[k] = f"{len(v)} öğe" if len(v) > 8 else v
+                        elif isinstance(v, float):
+                            details[k] = round(v, 4)
+                        else:
+                            details[k] = v
+            if job.error:
+                details["error"] = job.error[:300]
+            audit.write(
+                job.project_id,
+                kind=job.kind,
+                title=f"{job.title} — {self._status_label(job.status)}",
+                title_key=job.title_key,
+                title_params=job.title_params,
+                details=details,
+                user_action=f"job_{job.status.value}",
+            )
+        except Exception:
+            pass  # audit yazımı kritik değil — ana akışı bozmasın
+
+    @staticmethod
+    def _status_label(s: JobStatus) -> str:
+        return {
+            JobStatus.completed: "tamamlandı",
+            JobStatus.failed: "başarısız",
+            JobStatus.cancelled: "iptal edildi",
+            JobStatus.running: "çalışıyor",
+            JobStatus.queued: "kuyrukta",
+        }.get(s, str(s))
+
+    # --- SSE listeners ---
+    def subscribe(self, job_id: str) -> asyncio.Queue:
+        q: asyncio.Queue = asyncio.Queue(maxsize=500)
+        self._listeners.setdefault(job_id, set()).add(q)
+        # İlk frame: mevcut state
+        job = self._jobs.get(job_id)
+        if job:
+            q.put_nowait(job.to_dict())
+        return q
+
+    def unsubscribe(self, job_id: str, q: asyncio.Queue) -> None:
+        listeners = self._listeners.get(job_id)
+        if listeners:
+            listeners.discard(q)
+
+    def _notify(self, job_id: str, done: bool = False) -> None:
+        job = self._jobs.get(job_id)
+        if not job:
+            return
+        payload = job.to_dict()
+        for q in list(self._listeners.get(job_id, ())):
+            try:
+                q.put_nowait(payload)
+            except asyncio.QueueFull:
+                pass
+        if done:
+            # done sentinel
+            for q in list(self._listeners.get(job_id, ())):
+                try:
+                    q.put_nowait(None)
+                except asyncio.QueueFull:
+                    pass
+
+
+job_runner = JobRunner()

bibexpy/_server/main.py

@@ -0,0 +1,211 @@
+"""BibexPy v2 — FastAPI uygulama girişi.
+
+Geliştirme:
+  cd apps/api && python -m uvicorn main:app --reload --port 8001
+  cd apps/web && npm run dev                                       # ayrı terminal
+
+Backend tüm router'ları `/api/` prefix altında mount eder; bu, ileride
+frontend ile aynı host'tan serve edilse bile URL çakışmasını önler.
+
+`_frontend_root()` — opsiyonel statik UI mount (varsa). Geliştirmede
+genelde frontend ayrı çalıştığı için `None` döner ve devre dışı kalır.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Optional
+
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse, JSONResponse
+from fastapi.staticfiles import StaticFiles
+
+from config import settings
+from routers import (
+    projects, upload, convert, merge, jobs, downloads,
+    filter as filter_router, export as export_router, enrich,
+    disambiguate, prepare, export_folder, audit as audit_router,
+    quality as quality_router, records as records_router,
+    settings as settings_router, system as system_router,
+    tools as tools_router, report as report_router,
+)
+
+
+def _frontend_root() -> Optional[Path]:
+    """Next.js static export klasörünü bul. Yoksa None — API-only mode.
+
+    Geliştirmede genelde frontend ayrı (npm run dev, port 3000) çalışır;
+    bu fonksiyon None döner, backend sadece API serve eder.
+    """
+    # 1. Açık override (env var)
+    env = os.environ.get("BIBEXPY_FRONTEND_DIST")
+    if env:
+        p = Path(env)
+        if (p / "index.html").is_file():
+            return p
+
+    # 2. apps/web/out/ — eğer biri `npm run build:static` yaptıysa
+    here = Path(__file__).resolve().parent
+    p = here.parent / "web" / "out"
+    if (p / "index.html").is_file():
+        return p
+
+    return None
+
+
+def _spa_fallback_response(path: str, root: Path) -> Optional[FileResponse]:
+    """Verilen URL path'i için doğru statik HTML dosyasını bul.
+
+    Dinamik project ID (`projects/<id>/...`) varsa placeholder `_` ile
+    eşleştirmeyi dener. Sonra root index.html'e düşer.
+    """
+    # 1. Tam dosya
+    full = (root / path).resolve()
+    # Path traversal guard
+    try:
+        full.relative_to(root)
+    except ValueError:
+        return None
+
+    if full.is_file():
+        return FileResponse(full)
+
+    # 2. Dizin + index.html
+    idx = full / "index.html"
+    if idx.is_file():
+        return FileResponse(idx)
+
+    # 3. Dinamik [id] segment: `projects/abc123/upload` → `projects/_/upload/index.html`
+    parts = path.strip("/").split("/")
+    if len(parts) >= 2 and parts[0] == "projects":
+        # parts[1] proje ID — `_` ile değiştir
+        ph_path = root.joinpath("projects", "_", *parts[2:]) / "index.html"
+        if ph_path.is_file():
+            return FileResponse(ph_path)
+
+    # 4. SPA son fallback — root index.html (404.html da denenebilir ama gerek yok)
+    fallback = root / "index.html"
+    if fallback.is_file():
+        return FileResponse(fallback)
+    return None
+
+
+def create_app() -> FastAPI:
+    app = FastAPI(
+        title="BibexPy v2 API",
+        version="0.1.0",
+        description="Self-hosted bibliometric data preparation tool",
+    )
+
+    # CORS — dev modunda 3000'den 8001'e istek atan frontend için.
+    # Packaged binary'de tek port olduğu için CORS gereksiz ama zarar vermez.
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=settings.cors_origins_list,
+        # Yerel araç: localhost / 127.0.0.1 hangi portta olursa olsun kabul et.
+        # (Açık liste + regex birlikte değerlendirilir.) Port değişse bile çalışır.
+        allow_origin_regex=r"https?://(localhost|127\.0\.0\.1)(:\d+)?",
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    # API router'ları — `/api/` prefix altında. Bu, frontend route'larıyla
+    # (örn. `/projects` UI sayfası vs `/api/projects` API endpoint'i) URL
+    # çakışmasını önler. Hem dev hem packaged binary için aynı prefix.
+    for r in (projects, upload, convert, merge, jobs, downloads, filter_router,
+              export_router, enrich, disambiguate, prepare, export_folder,
+              audit_router, quality_router, records_router, settings_router,
+              system_router, tools_router, report_router):
+        app.include_router(r.router, prefix="/api")
+
+    # Startup: soft-deleted analiz klasörlerini fiziksel temizle.
+    @app.on_event("startup")
+    def _purge_soft_deleted_on_startup() -> None:
+        try:
+            from services import storage as _storage, analyses as _analyses
+            total = 0
+            for proj in _storage.list_projects():
+                pid = getattr(proj, "id", None) or (
+                    proj.get("id") if hasattr(proj, "get") else None
+                )
+                if not pid:
+                    continue
+                try:
+                    total += _analyses.purge_soft_deleted(pid)
+                except Exception:
+                    pass
+            if total > 0:
+                import logging
+                logging.getLogger("uvicorn").info(
+                    f"Purged {total} soft-deleted analysis folder(s)"
+                )
+        except Exception:
+            pass
+
+    # Meta endpoint'leri — frontend mount'tan önce, `/api/` altında
+    @app.get("/api/health", tags=["meta"])
+    def health() -> dict:
+        return {
+            "status": "ok",
+            "storage": str(settings.storage_path),
+            "disambiguation_enabled": settings.disambiguation_enabled,
+            "frontend_bundled": _frontend_root() is not None,
+        }
+
+    # Frontend mount — paketlenmiş veya dev'de export edilmiş ise
+    frontend_root = _frontend_root()
+    if frontend_root is not None:
+        # _next/static/* statik asset'leri — direkt mount, en hızlı yol
+        next_static = frontend_root / "_next"
+        if next_static.is_dir():
+            app.mount("/_next", StaticFiles(directory=next_static), name="_next")
+        # public/* asset'leri (görseller, vs.)
+        for sub in ("images", "tools", "favicon.ico"):
+            sub_path = frontend_root / sub
+            if sub_path.is_dir():
+                app.mount(f"/{sub}", StaticFiles(directory=sub_path), name=sub)
+
+        # Catch-all GET — SPA fallback (API ve mount'lardan SONRA register edilir,
+        # sadece yukarıdaki hiçbir route ile eşleşmeyen GET isteklerini yakalar)
+        @app.get("/{path:path}", include_in_schema=False)
+        async def spa_catchall(path: str, request: Request) -> FileResponse:
+            resp = _spa_fallback_response(path, frontend_root)
+            if resp is not None:
+                return resp
+            return JSONResponse({"detail": "Not Found"}, status_code=404)
+
+        # Root da SPA için
+        @app.get("/", include_in_schema=False)
+        async def spa_root() -> FileResponse:
+            return FileResponse(frontend_root / "index.html")
+    else:
+        # API-only mode (dev veya frontend build edilmemiş)
+        @app.get("/", tags=["meta"])
+        def root() -> dict:
+            return {
+                "name": "BibexPy v2 API",
+                "version": "0.1.0",
+                "description": "Self-hosted bibliometric data preparation tool",
+                "docs": "/docs",
+                "redoc": "/redoc",
+                "health": "/health",
+                "frontend_hint": "Frontend bundled değil. Geliştirme için: cd apps/web && npm run dev → http://localhost:3000",
+            }
+
+    return app
+
+
+app = create_app()
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(
+        "main:app",
+        host=settings.api_host,
+        port=settings.api_port,
+        reload=True,
+    )

bibexpy/_server/models/project.py

@@ -0,0 +1,25 @@
+from datetime import datetime
+from typing import Optional
+from pydantic import BaseModel, Field
+
+
+class ProjectCreate(BaseModel):
+    name: str = Field(min_length=1, max_length=120)
+    description: Optional[str] = None
+
+
+class ProjectMeta(BaseModel):
+    id: str
+    name: str
+    description: Optional[str] = None
+    created_at: datetime
+    updated_at: datetime
+    file_count: int = 0
+    raw_size_bytes: int = 0
+
+
+class UploadedFile(BaseModel):
+    name: str
+    size: int
+    kind: str  # "scopus_csv" | "wos_txt" | "xlsx" | "unknown"
+    saved_path: str