BibexPy 2.0.0__py3-none-any.whl

bibexpy/_server/services/disambiguation/deepseek_client.py

@@ -0,0 +1,203 @@
+"""DeepSeek API minimum istemcisi (OpenAI uyumlu chat completions)."""
+
+from __future__ import annotations
+
+import json
+import asyncio
+from typing import Any
+
+import requests
+
+
+class DeepSeekError(Exception):
+    pass
+
+
+class DeepSeekClient:
+    def __init__(self, api_key: str, base_url: str = "https://api.deepseek.com", model: str = "deepseek-v4-flash"):
+        if not api_key:
+            raise DeepSeekError("DEEPSEEK_API_KEY is not set")
+        self.api_key = api_key
+        self.base_url = base_url.rstrip("/")
+        self.model = model
+
+    def _post(self, payload: dict) -> dict:
+        url = f"{self.base_url}/chat/completions"
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+        r = requests.post(url, headers=headers, json=payload, timeout=60)
+        if not r.ok:
+            raise DeepSeekError(f"DeepSeek {r.status_code}: {r.text[:200]}")
+        return r.json()
+
+    def chat_json(self, system: str, user_payload: dict, temperature: float = 0.0) -> dict:
+        """JSON yanıt iste. Prompt 'json' kelimesini içermek zorunda (DeepSeek kuralı)."""
+        body = {
+            "model": self.model,
+            "messages": [
+                {"role": "system", "content": system},
+                {"role": "user", "content": json.dumps(user_payload, ensure_ascii=False)},
+            ],
+            "response_format": {"type": "json_object"},
+            "temperature": temperature,
+        }
+        data = self._post(body)
+        try:
+            content = data["choices"][0]["message"]["content"]
+            return json.loads(content)
+        except (KeyError, IndexError, json.JSONDecodeError) as e:
+            raise DeepSeekError(f"Could not parse response: {e}; raw={data}")
+
+    async def chat_json_async(self, system: str, user_payload: dict, temperature: float = 0.0) -> dict:
+        return await asyncio.to_thread(self.chat_json, system, user_payload, temperature)
+
+    def chat_text(self, system: str, user_text: str, temperature: float = 0.3) -> str:
+        """Serbest metin yanıt iste (JSON zorlaması yok). Metodoloji raporu gibi
+        düz prose çıktılar için kullanılır."""
+        body = {
+            "model": self.model,
+            "messages": [
+                {"role": "system", "content": system},
+                {"role": "user", "content": user_text},
+            ],
+            "temperature": temperature,
+        }
+        data = self._post(body)
+        try:
+            return data["choices"][0]["message"]["content"]
+        except (KeyError, IndexError) as e:
+            raise DeepSeekError(f"Could not parse response: {e}; raw={data}")
+
+    async def chat_text_async(self, system: str, user_text: str, temperature: float = 0.3) -> str:
+        return await asyncio.to_thread(self.chat_text, system, user_text, temperature)
+
+
+AUTHOR_SYSTEM_PROMPT = """You are a bibliometric data-quality expert. Your task: review the candidate author set provided by the user and determine which variants belong to the SAME person.
+
+Signals to use when deciding:
+- Name variants (initial variation, transliteration, married surname)
+- Overlapping affiliations or known mobility (change of university)
+- Shared coauthors (strong signal)
+- Consistency of research field
+- Chronological consistency of the year range
+
+Return the answer in this JSON schema:
+{
+  "clusters": [
+    {
+      "cluster_id": "c1",
+      "member_ids": ["v1", "v3"],   (use the EXACT candidate "id" values provided, e.g. "v1","v2"; do NOT invent new labels)
+      "confidence": 0.0-1.0,
+      "reason": "short explanation (English)"
+    }
+  ],
+  "uncertain": [
+    {"id": "z", "reason": "why it is uncertain (English)"}
+  ]
+}
+
+Only place decisions with confidence>=0.7 into the same cluster. List uncertain ones in the "uncertain" list.
+Respond with JSON only, write no other text."""
+
+AUTHOR_SPLIT_SYSTEM_PROMPT = """You are a bibliometric data-quality expert. Your task: decide whether a SINGLE author name spelling (e.g., "Mehmet A") should be split into record groups belonging to different research fields.
+
+Question: Do these records belong to DIFFERENT people sharing the SAME name (split), or to a SINGLE person working across disciplines (keep)?
+
+Signals:
+- If the research fields/categories are completely far apart (e.g., pure Physics vs pure Literature), the probability of different people is high -> split.
+- If the fields are neighboring/related or there is an interdisciplinary topic, it may be a single person -> keep.
+- Some records may lack field information; take this uncertainty into account.
+
+Return the answer in this JSON schema:
+{
+  "decision": "split" | "keep",
+  "confidence": 0.0-1.0,
+  "reason": "short explanation (English)"
+}
+Respond with JSON only, write no other text."""
+
+AFFILIATION_SYSTEM_PROMPT = """You are an institution-name normalization expert. Your task: determine which of the given affiliation variants belong to the SAME institution.
+
+Even if the institutional hierarchy (university > faculty > department) looks different, variants may be in the same cluster when they share the same parent institution; however, separate campuses/cities belong to different clusters.
+
+Return the answer in this JSON schema:
+{
+  "clusters": [
+    {
+      "cluster_id": "c1",
+      "member_ids": ["v1", "v3"],   (use the EXACT candidate "id" values provided, e.g. "v1","v2"; do NOT invent new labels)
+      "canonical_name": "Suggested standard spelling (official English form)",
+      "country": "ISO 3166-1 alpha-2 or empty",
+      "confidence": 0.0-1.0,
+      "reason": "short explanation (English)"
+    }
+  ],
+  "uncertain": [
+    {"id": "z", "reason": "..."}
+  ]
+}
+
+Respond with JSON only, write no other text."""
+
+
+METHODOLOGY_SYSTEM_PROMPT = """You are an academic writing assistant. You help researchers draft the "Data Collection and Preparation" (Methodology) subsection of a bibliometric study.
+
+You will receive a chronological JSON log of operations the researcher performed in BibexPy, a tool that merges, cleans, filters, enriches and disambiguates bibliographic records exported from Web of Science (WoS) and Scopus.
+
+Write a clear, formal methodology narrative IN ENGLISH describing the data preparation workflow. Requirements:
+- Use academic past tense and passive voice (e.g., "A total of N records were retrieved from...", "After deduplication, M unique records remained...", "Records published before YYYY were excluded...").
+- Report the SPECIFIC numbers that appear in the log (input counts, merged/unique counts, removed duplicates, excluded/filtered records, enriched fields, normalized/split author names).
+- Follow the actual chronological order of the steps as logged.
+- Name the source databases (Web of Science and/or Scopus) and state that data preparation was performed using BibexPy, with an APA-style in-text citation "(Kara et al., 2025)".
+- When duplicate removal or record linkage is described, you may note that it followed established record-linkage practice; when author name handling is described, refer to it as author name disambiguation (normalization of name variants and separation of homonymous names).
+- Do NOT invent steps, numbers, criteria, or external citations that are not present in the log — the ONLY citation you must add is the BibexPy reference specified below. If a detail is missing, omit it rather than fabricating.
+- Be concise and publication-ready: one to a few short paragraphs. You MAY use brief Markdown subheadings (e.g., "## Data sources", "## Deduplication and merging", "## Screening and enrichment", "## Author disambiguation") if it improves clarity.
+- End with a short "## Reference" section that contains EXACTLY this reference (do not alter it): Kara, B. C., Şahin, A., & Dirsehan, T. (2025). BibexPy: Harmonizing the bibliometric symphony of Scopus and Web of Science. SoftwareX, 30, 102098. https://doi.org/10.1016/j.softx.2025.102098
+
+Output Markdown text only. Do not add a preamble like "Here is the methodology"; start directly with the content."""
+
+
+COUNTRY_SYSTEM_PROMPT = """You are a bibliometric data-quality expert. You receive tokens taken from the LAST component of author addresses (the C1 field) that did NOT match the automatic country dictionary.
+
+Some tokens ARE country-name spellings the dictionary missed (abbreviation, typo, local or former name, e.g. "U.S.A.", "Peoples R China", "Deutschland") — group THESE by the country they denote.
+
+Other tokens are NOT countries at all (institutions, cities, fragments, e.g. "Michigan State University", "Hangzhou") — put EACH of these in "uncertain". NEVER cluster a non-country token as a country and NEVER map an institution/city to a country here. (Missing countries are completed separately from the API, not guessed here.)
+
+Return the answer in this JSON schema:
+{
+  "clusters": [
+    {
+      "cluster_id": "c1",
+      "member_ids": ["v1", "v3"],   (use the EXACT candidate "id" values provided, e.g. "v1","v2"; do NOT invent new labels)
+      "canonical_name": "Canonical English country name (ISO 3166, e.g., United States)",
+      "confidence": 0.0-1.0,
+      "reason": "short explanation (English)"
+    }
+  ],
+  "uncertain": [ {"id": "z", "reason": "not a country (e.g., institution / city / fragment)"} ]
+}
+Respond with JSON only, write no other text."""
+
+
+ORG_ROLLUP_SYSTEM_PROMPT = """You are a bibliometric data-quality expert. Your task: group the given FULL address (affiliation) strings by the PARENT INSTITUTION (university / research institution) they belong to.
+
+Goal: build the collaboration network through the PARENT INSTITUTION rather than the department/sub-unit. Different departments or spelling variants of the same parent institution should be in a single cluster; DIFFERENT campuses/cities or different institutions must stay in SEPARATE clusters.
+
+canonical_name: the standard name of the parent institution — WITHOUT department and geography (e.g., "University of Oxford").
+
+Return the answer in this JSON schema:
+{
+  "clusters": [
+    {
+      "cluster_id": "c1",
+      "member_ids": ["v1", "v3"],   (use the EXACT candidate "id" values provided, e.g. "v1","v2"; do NOT invent new labels)
+      "canonical_name": "Parent institution standard name",
+      "confidence": 0.0-1.0,
+      "reason": "short explanation (English)"
+    }
+  ],
+  "uncertain": [ {"id": "z", "reason": "why uncertain (English)"} ]
+}
+Respond with JSON only, write no other text."""

bibexpy/_server/services/disambiguation/orcid.py

@@ -0,0 +1,261 @@
+"""ORCID-öncelikli yazar disambiguation yardımcıları.
+
+Mantık: bir ambigu yazar grubu için kimlik kararını ÖNCE veride var olan ORCID
+(OI sütunu) ile, yoksa o grubun çalışmalarının DOI'lerinden OpenAlex'e ANLIK
+sorgu atarak verir. ORCID altın standarttır:
+  • Aynı ORCID  → kesin aynı kişi (alan/isim farkı önemsiz).
+  • Farklı ORCID → kesin farklı kişi (isim aynı olsa bile).
+
+Bu modül framework-bağımsız tutulmaya çalışılır; yalnız api_utils'in HTTP
+yardımcısını (timeout + 429 backoff) ve similarity.normalize_name'i kullanır.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+
+from bibex_core.modules.api_utils import _get_with_retry
+from .similarity import normalize_name
+from .blocking import _SPLIT_SUFFIX_RE
+
+
+def _clean_orcid(v) -> str:
+    """https://orcid.org/0000-... → 0000-... (sade ORCID)."""
+    return (
+        str(v or "")
+        .replace("https://orcid.org/", "")
+        .replace("http://orcid.org/", "")
+        .strip()
+    )
+
+
+def _norm(name: str) -> str:
+    """İsim eşleştirme anahtarı: split eki ('(b)') soyulmuş + normalize."""
+    return normalize_name(_SPLIT_SUFFIX_RE.sub("", str(name or "")))
+
+
+@lru_cache(maxsize=8192)
+def fetch_orcids_for_doi(doi: str, email: str | None = None) -> tuple:
+    """OpenAlex'ten bir DOI için YALNIZ yazar-ORCID/ROR/ülke çek (hafif: select=authorships).
+
+    Dönüş: ((display_name, orcid, ror, cc), ...) tuple'ı (lru_cache hashable olsun diye).
+    Hata/boş → boş tuple. Tam metadata çekmez (~%95 daha küçük payload).
+    """
+    d = str(doi or "").strip()
+    if not d:
+        return tuple()
+    url = f"https://api.openalex.org/works/doi:{d}?select=authorships,doi"
+    if email:
+        url += f"&mailto={email}"
+    try:
+        r = _get_with_retry(url)
+    except Exception:
+        return tuple()
+    if getattr(r, "status_code", None) != 200:
+        return tuple()
+    try:
+        work = r.json()
+    except Exception:
+        return tuple()
+
+    out: list[tuple] = []
+    for a in (work.get("authorships") or []):
+        au = a.get("author") or {}
+        name = au.get("display_name") or ""
+        orcid = _clean_orcid(au.get("orcid"))
+        rors, ccs = [], []
+        for inst in (a.get("institutions") or []):
+            if inst.get("ror"):
+                rors.append(str(inst["ror"]).replace("https://ror.org/", "").strip())
+            if inst.get("country_code"):
+                ccs.append(str(inst["country_code"]).strip().upper())
+        for cc in (a.get("countries") or []):
+            if cc:
+                ccs.append(str(cc).strip().upper())
+        out.append((name, orcid, "; ".join(dict.fromkeys(rors)), "; ".join(dict.fromkeys(ccs))))
+    return tuple(out)
+
+
+@lru_cache(maxsize=8192)
+def fetch_affiliations_for_doi(doi: str, email: str | None = None) -> tuple:
+    """OpenAlex'ten bir DOI için (kurum_adı, kanonik_ülke) çiftlerini çek (adres tamamlama).
+
+    Dönüş: ((institution_display_name, country_name), ...) — country_name kanonik
+    İngilizce ad (country_code'dan çevrilir; bilinmiyorsa ''). Hata/boş → boş tuple.
+    Hafif: select=authorships. lru_cache ile DOI tekrar çekilmez.
+    """
+    from bibex_core.modules.c1_utils import country_from_code
+
+    d = str(doi or "").strip()
+    if not d:
+        return tuple()
+    url = f"https://api.openalex.org/works/doi:{d}?select=authorships,doi"
+    if email:
+        url += f"&mailto={email}"
+    try:
+        r = _get_with_retry(url)
+    except Exception:
+        return tuple()
+    if getattr(r, "status_code", None) != 200:
+        return tuple()
+    try:
+        work = r.json()
+    except Exception:
+        return tuple()
+
+    out: list[tuple] = []
+    seen: set = set()
+    for a in (work.get("authorships") or []):
+        for inst in (a.get("institutions") or []):
+            name = str(inst.get("display_name") or "").strip()
+            country = country_from_code(inst.get("country_code") or "") or ""
+            key = (name.lower(), country)
+            if (name or country) and key not in seen:
+                seen.add(key)
+                out.append((name, country))
+    return tuple(out)
+
+
+def parse_oi_cell(cell) -> dict[str, str]:
+    """WoS OI hücresini ('Ad/0000-...; Ad2/0000-...') {normalize_name(ad): orcid} yap.
+
+    Hem '/' (WoS) hem ', ' varyantlarını tolere eder; ad yoksa atlar.
+    """
+    s = str(cell or "").strip()
+    if not s or s.lower() in ("nan", "none"):
+        return {}
+    out: dict[str, str] = {}
+    for part in s.split(";"):
+        part = part.strip()
+        if not part:
+            continue
+        # 'Ad/ORCID' biçimi
+        if "/" in part:
+            name, _, oid = part.rpartition("/")
+        else:
+            # yalnız ORCID olabilir — ada bağlanamaz, atla
+            continue
+        oid = _clean_orcid(oid)
+        nm = _norm(name)
+        if nm and oid:
+            out[nm] = oid
+    return out
+
+
+def orcids_for_candidate(df, candidate: dict, *, email: str | None, allow_api: bool) -> set[str]:
+    """Bir adayın (candidate) ORCID kümesi.
+
+    candidate['records'] satır indeksleri + candidate['name_variants'] (ya da
+    'variants') üzerinden: ÖNCE her satırın OI hücresinden, eksikse (allow_api ise)
+    DOI'den OpenAlex sorgusuyla, adayın isim varyantlarıyla EŞLEŞEN yazarların
+    ORCID'lerini toplar. İsim eşleşmesi şart — yoksa ko-yazarın ORCID'i çekilmez.
+
+    Yan etki YOK (veriye yazmaz); yalnız okur. (Write-back ayrı yapılır.)
+    """
+    variants = candidate.get("name_variants") or candidate.get("variants") or []
+    want = {_norm(v) for v in variants if v}
+    if candidate.get("id"):
+        want.add(_norm(candidate["id"]))
+    want.discard("")
+    if not want:
+        return set()
+
+    records = candidate.get("records") or []
+    has_oi = "OI" in df.columns
+    has_di = "DI" in df.columns
+    found: set[str] = set()
+
+    for idx in records:
+        if idx not in df.index:
+            continue
+        # 1) Veride var olan OI
+        if has_oi:
+            for nm, oid in parse_oi_cell(df.at[idx, "OI"]).items():
+                if nm in want:
+                    found.add(oid)
+        # 2) Eksikse DOI'den anlık (yalnız bu satırda eşleşen isim için)
+        if allow_api and has_di:
+            doi = df.at[idx, "DI"]
+            if doi is not None and str(doi).strip() and str(doi).strip().lower() != "nan":
+                for name, orcid, _ror, _cc in fetch_orcids_for_doi(str(doi).strip(), email):
+                    if orcid and _norm(name) in want:
+                        found.add(orcid)
+    return found
+
+
+def _is_blank(v) -> bool:
+    s = "" if v is None else str(v).strip()
+    return s == "" or s.lower() in ("nan", "none")
+
+
+def enrich_rows_from_doi(df, row_indices, *, email: str | None) -> dict:
+    """Verilen satırların BOŞ OI/RI/ROR/CC hücrelerini DOI üzerinden OpenAlex ile doldur.
+
+    Yalnız boş hücreye yazar (mevcut/özgün değeri ezmez). df YERİNDE güncellenir.
+    Dönüş: {'rows_filled': int, 'dois_fetched': int}. Yan etki: df mutasyonu.
+    """
+    if "DI" not in df.columns:
+        return {"rows_filled": 0, "dois_fetched": 0}
+    # Yeni kolonlar yoksa oluştur (object dtype) — boş hücreye yazım için
+    for col in ("OI", "RI", "ROR", "CC"):
+        if col not in df.columns:
+            df[col] = None
+
+    rows_filled = 0
+    dois_fetched = 0
+    seen_doi: set[str] = set()
+    for idx in dict.fromkeys(row_indices):  # tekilleştir, sırayı koru
+        if idx not in df.index:
+            continue
+        doi = df.at[idx, "DI"]
+        if doi is None or not str(doi).strip() or str(doi).strip().lower() == "nan":
+            continue
+        d = str(doi).strip()
+        rows = fetch_orcids_for_doi(d, email)  # lru_cache'li
+        if d not in seen_doi:
+            seen_doi.add(d)
+            if rows:
+                dois_fetched += 1
+        if not rows:
+            continue
+        # Satır-seviyesi birleşik stringler (WoS formatı)
+        ois = [f"{n}/{o}" for (n, o, _r, _c) in rows if n and o]
+        ris = [f"{n}/{o}" for (n, o, _r, _c) in rows if n]  # RI yok; ad/id yerine ad/orcid yedeği
+        rors, ccs = [], []
+        for (_n, _o, r, c) in rows:
+            if r:
+                rors += [t.strip() for t in r.split(";") if t.strip()]
+            if c:
+                ccs += [t.strip() for t in c.split(";") if t.strip()]
+        updates = {
+            "OI": "; ".join(dict.fromkeys(ois)),
+            "ROR": "; ".join(dict.fromkeys(rors)),
+            "CC": "; ".join(dict.fromkeys(ccs)),
+        }
+        changed = False
+        for col, val in updates.items():
+            if val and _is_blank(df.at[idx, col]):
+                df.at[idx, col] = val
+                changed = True
+        if changed:
+            rows_filled += 1
+    return {"rows_filled": rows_filled, "dois_fetched": dois_fetched}
+
+
+def classify_by_orcid(orcid_sets: list[set[str]]) -> str:
+    """Üye-başına ORCID kümelerinden grup kararı.
+
+      'merge'    — ORCID'i bilinen üyeler TEK ortak ORCID'de buluşuyor → aynı kişi
+      'conflict' — iki üye ayrık (kesişimsiz) ORCID taşıyor → farklı kişi
+      'unknown'  — yetersiz ORCID kanıtı → isim+alan heuristiği + LLM'e bırak
+    """
+    known = [s for s in orcid_sets if s]
+    if len(known) < 2:
+        return "unknown"
+    # Herhangi iki bilinen küme kesişmiyorsa → çelişki (farklı kişiler)
+    for i in range(len(known)):
+        for j in range(i + 1, len(known)):
+            if not (known[i] & known[j]):
+                return "conflict"
+    # Tüm bilinen kümeler ikili kesişiyor → ortak ORCID(ler) var → aynı kişi
+    return "merge"