django-pbn-client 0.1.0__tar.gz

django_pbn_client-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,41 @@
+name: CI
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - name: Check out django-pbn-client
+        uses: actions/checkout@v4
+
+      # django-pbn-client declares `pbn-client` as a dependency. Until that
+      # package is published to PyPI, the [tool.uv.sources] override resolves
+      # it from a sibling checkout, so CI clones it next to this repo.
+      # (actions/checkout cannot target a path outside the workspace, so we
+      # clone the sibling directly.) Remove this step and the
+      # [tool.uv.sources] entry once pbn-client is available on PyPI.
+      - name: Check out sibling pbn-client
+        run: git clone --depth 1 https://github.com/iplweb/pbn-client.git ../pbn-client
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Run tests
+        run: uv run --group dev pytest

django_pbn_client-0.1.0/.gitignore

@@ -0,0 +1,19 @@
+__pycache__/
+*.pyc
+.pytest_cache/
+.ruff_cache/
+.venv/
+dist/
+build/
+*.egg-info/
+uv.lock
+
+# Editors / IDEs
+.idea/
+.vscode/
+*.sublime-workspace
+
+# OS turds
+.DS_Store
+Thumbs.db
+desktop.ini

django_pbn_client-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-added-large-files
+      - id: check-merge-conflict
+      - id: detect-private-key
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.21
+    hooks:
+      - id: ruff-check
+        args: [--fix]
+      - id: ruff-format

django_pbn_client-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017-2026 Michal Pasternak
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

django_pbn_client-0.1.0/PKG-INFO

@@ -0,0 +1,102 @@
+Metadata-Version: 2.4
+Name: django-pbn-client
+Version: 0.1.0
+Summary: Django persistence helpers for data downloaded from PBN
+Project-URL: Homepage, https://github.com/iplweb/django-pbn-client
+Project-URL: Repository, https://github.com/iplweb/django-pbn-client
+Project-URL: Issues, https://github.com/iplweb/django-pbn-client/issues
+Author-email: Michał Pasternak <michal.dtz@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: bibliography,django,pbn,persistence,polon
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: Django
+Classifier: Framework :: Django :: 5.2
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: <3.15,>=3.10
+Requires-Dist: django<5.3,>=5.2
+Requires-Dist: pbn-client<0.2,>=0.1.0
+Description-Content-Type: text/markdown
+
+# django-pbn-client
+
+[![CI](https://github.com/iplweb/django-pbn-client/actions/workflows/ci.yml/badge.svg)](https://github.com/iplweb/django-pbn-client/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python](https://img.shields.io/badge/python-3.10%20%E2%80%93%203.14-blue.svg)](https://www.python.org/)
+
+Reusable Django models and persistence services for data downloaded from the
+Polish Bibliography Network (PBN).
+
+The package deliberately does not define concrete application models or
+migrations. Applications subclass `BasePBNMongoDBModel` and retain ownership
+of their database schema:
+
+```python
+from django.db import models
+from django_pbn_client import BasePBNMongoDBModel
+
+
+class Publication(BasePBNMongoDBModel):
+    title = models.TextField(blank=True, default="")
+    pull_up_on_save = ["title"]
+```
+
+Downloaded objects can then be stored atomically:
+
+```python
+from django_pbn_client import download_pbn_objects, upsert_pbn_object
+
+publication = upsert_pbn_object(payload, Publication)
+download_pbn_objects(client.get_publications(), Publication)
+```
+
+`django-pbn-client` depends only on Django and the transport-level
+`pbn-client` package. Progress bars, background jobs, concrete relationships,
+and application-specific integration remain the responsibility of the host
+application.
+
+Concurrent page downloads use threads by default. The optional
+`method="processes"` mode uses the POSIX `fork` start method and is therefore
+not available on Windows.
+
+## Installation
+
+```console
+pip install django-pbn-client
+```
+
+## Development
+
+This package depends on the transport-level
+[`pbn-client`](https://github.com/iplweb/pbn-client) package. Until `pbn-client`
+is published to PyPI, a dev-only `[tool.uv.sources]` entry in `pyproject.toml`
+resolves it from a sibling checkout, so clone both repositories side by side:
+
+```console
+git clone https://github.com/iplweb/pbn-client.git
+git clone https://github.com/iplweb/django-pbn-client.git
+cd django-pbn-client
+```
+
+The source override is ignored when building or publishing wheels, so end users
+still get `pbn-client` from PyPI via the version constraint in
+`[project.dependencies]`.
+
+Run the standalone package tests with:
+
+```console
+uv run --group dev pytest
+```
+
+## License
+
+Released under the [MIT License](LICENSE).

django_pbn_client-0.1.0/README.md

@@ -0,0 +1,73 @@
+# django-pbn-client
+
+[![CI](https://github.com/iplweb/django-pbn-client/actions/workflows/ci.yml/badge.svg)](https://github.com/iplweb/django-pbn-client/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python](https://img.shields.io/badge/python-3.10%20%E2%80%93%203.14-blue.svg)](https://www.python.org/)
+
+Reusable Django models and persistence services for data downloaded from the
+Polish Bibliography Network (PBN).
+
+The package deliberately does not define concrete application models or
+migrations. Applications subclass `BasePBNMongoDBModel` and retain ownership
+of their database schema:
+
+```python
+from django.db import models
+from django_pbn_client import BasePBNMongoDBModel
+
+
+class Publication(BasePBNMongoDBModel):
+    title = models.TextField(blank=True, default="")
+    pull_up_on_save = ["title"]
+```
+
+Downloaded objects can then be stored atomically:
+
+```python
+from django_pbn_client import download_pbn_objects, upsert_pbn_object
+
+publication = upsert_pbn_object(payload, Publication)
+download_pbn_objects(client.get_publications(), Publication)
+```
+
+`django-pbn-client` depends only on Django and the transport-level
+`pbn-client` package. Progress bars, background jobs, concrete relationships,
+and application-specific integration remain the responsibility of the host
+application.
+
+Concurrent page downloads use threads by default. The optional
+`method="processes"` mode uses the POSIX `fork` start method and is therefore
+not available on Windows.
+
+## Installation
+
+```console
+pip install django-pbn-client
+```
+
+## Development
+
+This package depends on the transport-level
+[`pbn-client`](https://github.com/iplweb/pbn-client) package. Until `pbn-client`
+is published to PyPI, a dev-only `[tool.uv.sources]` entry in `pyproject.toml`
+resolves it from a sibling checkout, so clone both repositories side by side:
+
+```console
+git clone https://github.com/iplweb/pbn-client.git
+git clone https://github.com/iplweb/django-pbn-client.git
+cd django-pbn-client
+```
+
+The source override is ignored when building or publishing wheels, so end users
+still get `pbn-client` from PyPI via the version constraint in
+`[project.dependencies]`.
+
+Run the standalone package tests with:
+
+```console
+uv run --group dev pytest
+```
+
+## License
+
+Released under the [MIT License](LICENSE).

django_pbn_client-0.1.0/pyproject.toml

@@ -0,0 +1,70 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "django-pbn-client"
+version = "0.1.0"
+description = "Django persistence helpers for data downloaded from PBN"
+readme = "README.md"
+requires-python = ">=3.10,<3.15"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [
+    { name = "Michał Pasternak", email = "michal.dtz@gmail.com" },
+]
+keywords = ["django", "pbn", "bibliography", "polon", "persistence"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Framework :: Django",
+    "Framework :: Django :: 5.2",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "Django>=5.2,<5.3",
+    "pbn-client>=0.1.0,<0.2",
+]
+
+[project.urls]
+Homepage = "https://github.com/iplweb/django-pbn-client"
+Repository = "https://github.com/iplweb/django-pbn-client"
+Issues = "https://github.com/iplweb/django-pbn-client/issues"
+
+[dependency-groups]
+dev = [
+    "pytest>=9.1.1,<10",
+    "pytest-django>=4.9,<5",
+]
+
+# Dev-only source override. The published `pbn-client` dependency above is what
+# real installs resolve from PyPI once that package is released. Until then this
+# points uv at the sibling checkout so the package is developable and testable
+# standalone. uv ignores [tool.uv.sources] when building/publishing wheels, so
+# the constraint in [project.dependencies] is unaffected for downstream users.
+[tool.uv.sources]
+pbn-client = { path = "../pbn-client", editable = true }
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/django_pbn_client"]
+
+[tool.pytest.ini_options]
+addopts = "-ra"
+pythonpath = ["src", "tests"]
+DJANGO_SETTINGS_MODULE = "package_test_settings"
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 88
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "I"]

django_pbn_client-0.1.0/src/django_pbn_client/__init__.py

@@ -0,0 +1,33 @@
+"""Django persistence helpers for data downloaded from PBN."""
+
+from django_pbn_client.models import (
+    MAX_TEXT_FIELD_LENGTH,
+    BasePBNModel,
+    BasePBNMongoDBModel,
+)
+from django_pbn_client.pages import (
+    ThreadedModelSaver,
+    ThreadedMongoDBSaver,
+    ThreadedPageGetter,
+    download_pages,
+    simple_page_getter,
+)
+from django_pbn_client.persistence import (
+    download_pbn_objects,
+    get_total_count,
+    upsert_pbn_object,
+)
+
+__all__ = [
+    "MAX_TEXT_FIELD_LENGTH",
+    "BasePBNModel",
+    "BasePBNMongoDBModel",
+    "ThreadedModelSaver",
+    "ThreadedMongoDBSaver",
+    "ThreadedPageGetter",
+    "download_pages",
+    "download_pbn_objects",
+    "get_total_count",
+    "simple_page_getter",
+    "upsert_pbn_object",
+]

django_pbn_client-0.1.0/src/django_pbn_client/models.py

@@ -0,0 +1,106 @@
+"""Abstract Django models for PBN objects."""
+
+import warnings
+
+from django.db import models
+from django.utils.functional import cached_property
+
+MAX_TEXT_FIELD_LENGTH = 350
+
+
+class BasePBNModel(models.Model):
+    """Base timestamps shared by locally persisted PBN objects."""
+
+    created_on = models.DateTimeField(auto_now_add=True)
+    last_updated_on = models.DateTimeField(auto_now=True)
+
+    class Meta:
+        abstract = True
+
+
+class BasePBNMongoDBModel(BasePBNModel):
+    """Abstract representation of a versioned object returned by PBN."""
+
+    mongoId = models.CharField(max_length=32, primary_key=True)
+    status = models.CharField(max_length=32, db_index=True, default="")
+    verificationLevel = models.CharField(
+        max_length=32,
+        db_index=True,
+        default="",
+    )
+    verified = models.BooleanField(default=False, db_index=True)
+    versions = models.JSONField(default=list)
+
+    # Field names copied from the current version's ``object`` dictionary.
+    pull_up_on_save = None
+
+    class Meta:
+        abstract = True
+
+    def save(
+        self,
+        force_insert=False,
+        force_update=False,
+        using=None,
+        update_fields=None,
+    ):
+        if self.pull_up_on_save:
+            self._pull_up_on_save()
+        return super().save(
+            force_insert=force_insert,
+            force_update=force_update,
+            using=using,
+            update_fields=update_fields,
+        )
+
+    def _pull_up_on_save(self):
+        for attr in self.pull_up_on_save:
+            pull_up = getattr(self, f"pull_up_{attr}", None)
+            if pull_up is not None:
+                value = pull_up()
+            else:
+                value = self.value_or_none("object", attr)
+
+            if value is not None:
+                if isinstance(value, str) and len(value) >= MAX_TEXT_FIELD_LENGTH:
+                    value = value[:MAX_TEXT_FIELD_LENGTH]
+                setattr(self, attr, value)
+
+    @cached_property
+    def current_version(self):
+        """Return the version marked as current, if one exists."""
+        if self.versions:
+            for element in self.versions:
+                if element.get("current"):
+                    return element
+        return None
+
+    def value(self, *path, return_none=False):
+        """Read a nested value from the current version."""
+        value = self.current_version
+        if value is None:
+            warnings.warn(
+                f"Model {self.__class__} with id {self.mongoId} "
+                "has NO current_version!",
+                stacklevel=2,
+            )
+            if return_none:
+                return None
+            return "[brak current_version]"
+
+        for element in path:
+            if element in value:
+                value = value[element]
+            else:
+                if return_none:
+                    return None
+                return f"[brak {element}]"
+        return value
+
+    def value_or_none(self, *path):
+        """Read a nested value, returning ``None`` when it is absent."""
+        return self.value(*path, return_none=True)
+
+    def website(self):
+        """Return the website stored in the current object version."""
+        return self.value("object", "website")

django_pbn_client-0.1.0/src/django_pbn_client/pages.py

@@ -0,0 +1,172 @@
+"""Concurrent page download services for PBN iterators."""
+
+from __future__ import annotations
+
+import multiprocessing
+from collections.abc import Callable, Iterable
+from multiprocessing.dummy import Pool as ThreadPool
+from typing import TypeVar
+
+from django import setup as django_setup
+from django.db import close_old_connections, connections
+from pbn_client.exceptions import HttpException
+
+from django_pbn_client.persistence import upsert_pbn_object
+
+PageResult = TypeVar("PageResult")
+PageProgressWrapper = Callable[
+    [Iterable[PageResult], int | None, str],
+    Iterable[PageResult],
+]
+
+
+def simple_page_getter(
+    client,
+    data,
+    repeat_on_failure=False,
+    skip_page_on_failure=False,
+):
+    """Yield paginator pages sequentially with optional HTTP 500 retries."""
+    del client
+
+    for page_number in range(data.total_pages):
+        while True:
+            try:
+                yield data.fetch_page(page_number)
+                break
+            except HttpException as error:
+                if skip_page_on_failure:
+                    break
+                if not repeat_on_failure or error.status_code != 500:
+                    raise
+
+
+class ThreadedPageGetter:
+    """Fetch and process individual pages supplied by a PBN paginator."""
+
+    # Subclasses bind the target model by overriding this class attribute
+    # (``model_class = MyModel``); the constructor argument takes precedence
+    # when supplied.
+    model_class = None
+
+    def __init__(self, max_workers=None, model_class=None):
+        self.max_workers = max_workers
+        if model_class is not None:
+            self.model_class = model_class
+        self.client = None
+        self.data = None
+
+    def pool_init(self, client, data):
+        django_setup()
+        self.client = client
+        self.data = data
+
+    def process_element(self, element):
+        raise NotImplementedError
+
+    def get_single_page(self, page_number):
+        for element in self.data.fetch_page(page_number):
+            self.process_element(element)
+
+
+class ThreadedModelSaver(ThreadedPageGetter):
+    """Page getter that atomically stores each element in a Django model."""
+
+    save_function = staticmethod(upsert_pbn_object)
+
+    def process_element(self, element):
+        self.save_function(element, self.model_class)
+
+
+# Historical name: describes PBN's upstream storage, not the local database.
+# Retained as an alias for existing consumers of the older API.
+ThreadedMongoDBSaver = ThreadedModelSaver
+
+
+_process_getter = None
+
+
+def _initialize_process_getter(getter, client, data):
+    """Initialize the getter instance owned by one worker process."""
+    global _process_getter
+
+    getter.pool_init(client, data)
+    _process_getter = getter
+
+
+def _get_single_page_in_process(page_number):
+    """Dispatch a page through the getter initialized in this process."""
+    return _process_getter.get_single_page(page_number)
+
+
+def _make_getter(getter_class, workers):
+    try:
+        return getter_class(max_workers=workers)
+    except TypeError:
+        # Compatibility with existing getter classes that predate max_workers.
+        return getter_class()
+
+
+def _make_pool(method, workers, getter, client, data):
+    if method == "threads":
+        pool = ThreadPool(
+            processes=workers,
+            initializer=getter.pool_init,
+            initargs=(client, data),
+        )
+        return pool, getter.get_single_page
+
+    if method == "processes":
+        close_old_connections()
+        connections.close_all()
+        pool = multiprocessing.get_context("fork").Pool(
+            processes=workers,
+            initializer=_initialize_process_getter,
+            initargs=(getter, client, data),
+        )
+        return pool, _get_single_page_in_process
+
+    raise ValueError(f"Unsupported page download method: {method!r}")
+
+
+def download_pages(
+    client,
+    data,
+    *,
+    getter_class=ThreadedPageGetter,
+    workers=12,
+    label="getting...",
+    method="threads",
+    progress: PageProgressWrapper | None = None,
+):
+    """Fetch all paginator pages concurrently and process their elements."""
+    getter = _make_getter(getter_class, workers)
+    effective_workers = getattr(getter, "max_workers", None) or workers
+    pool, get_single_page = _make_pool(
+        method,
+        effective_workers,
+        getter,
+        client,
+        data,
+    )
+
+    try:
+        completed_pages = pool.imap_unordered(
+            get_single_page,
+            range(data.total_pages),
+        )
+        if progress is not None:
+            completed_pages = progress(
+                completed_pages,
+                data.total_pages,
+                label,
+            )
+        for _completed_page in completed_pages:
+            pass
+    except BaseException:
+        pool.terminate()
+        raise
+    else:
+        pool.close()
+    finally:
+        pool.join()

django_pbn_client-0.1.0/src/django_pbn_client/persistence.py

@@ -0,0 +1,116 @@
+"""Atomic persistence and sequential download services for PBN objects."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable, Iterable
+from typing import Any, TypeVar
+
+from django.db import IntegrityError, transaction
+
+logger = logging.getLogger(__name__)
+
+Element = TypeVar("Element")
+ProgressWrapper = Callable[
+    [Iterable[Element], int | None, str],
+    Iterable[Element],
+]
+
+
+def _update_changed_fields(instance, values):
+    changed = False
+    for field_name, value in values.items():
+        if getattr(instance, field_name) != value:
+            setattr(instance, field_name, value)
+            changed = True
+
+    if changed:
+        # Deliberately save the complete model: BasePBNMongoDBModel.save()
+        # recalculates fields named by pull_up_on_save.
+        instance.save()
+    return instance
+
+
+@transaction.atomic
+def upsert_pbn_object(element, model_class, client=None, **extra_fields):
+    """Create or update one versioned PBN object atomically.
+
+    ``client`` is accepted so this function can be used interchangeably with
+    application-specific download callbacks. It is not needed for a generic
+    model upsert.
+    """
+    del client
+
+    values = {
+        "status": element["status"],
+        "verificationLevel": element["verificationLevel"],
+        "verified": element["verified"],
+        "versions": element["versions"],
+        **extra_fields,
+    }
+    object_id = element["mongoId"]
+    existing = model_class.objects.select_for_update().filter(pk=object_id)
+
+    try:
+        instance = existing.get()
+    except model_class.DoesNotExist:
+        try:
+            # The savepoint is essential. If another worker wins the insert
+            # race, its IntegrityError must not poison the enclosing atomic
+            # block before we read and update the winning row.
+            with transaction.atomic():
+                return model_class.objects.create(pk=object_id, **values)
+        except IntegrityError as create_error:
+            try:
+                instance = model_class.objects.select_for_update().get(pk=object_id)
+            except model_class.DoesNotExist:
+                raise create_error
+
+    return _update_changed_fields(instance, values)
+
+
+def get_total_count(elements) -> int | None:
+    """Best-effort total used by download progress integrations."""
+    total_elements = getattr(elements, "total_elements", None)
+    if total_elements is not None:
+        return total_elements
+
+    length = getattr(elements, "__len__", None)
+    if length is not None:
+        try:
+            return len(elements)
+        except TypeError:
+            logger.debug("Element collection rejected len()", exc_info=True)
+
+    count = getattr(elements, "count", None)
+    if count is None:
+        return None
+    if not callable(count):
+        return count
+
+    try:
+        return count()
+    except Exception:
+        # Counting is optional and must not abort a data download. Keep the
+        # traceback in standard logging for hosts that need diagnostics.
+        logger.debug("Could not determine element count", exc_info=True)
+        return None
+
+
+def download_pbn_objects(
+    elements,
+    model_class,
+    *,
+    label="download_pbn_objects",
+    save: Callable[..., Any] | None = None,
+    client=None,
+    progress: ProgressWrapper | None = None,
+):
+    """Persist every object from an iterable, optionally wrapping progress."""
+    if save is None:
+        save = upsert_pbn_object
+
+    total = get_total_count(elements)
+    iterable = progress(elements, total, label) if progress else elements
+    for element in iterable:
+        save(element, model_class, client=client)

django_pbn_client-0.1.0/tests/package_test_settings.py

@@ -0,0 +1,10 @@
+SECRET_KEY = "django-pbn-client-tests"
+INSTALLED_APPS = []
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": ":memory:",
+    }
+}
+DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
+USE_TZ = True

django_pbn_client-0.1.0/tests/test_django_pbn_client_models.py

@@ -0,0 +1,109 @@
+import pytest
+from django.db import connection, models
+
+from django_pbn_client.models import (
+    MAX_TEXT_FIELD_LENGTH,
+    BasePBNMongoDBModel,
+)
+from django_pbn_client.persistence import upsert_pbn_object
+
+
+class MirrorRecord(BasePBNMongoDBModel):
+    title = models.TextField(blank=True, default="")
+    year = models.IntegerField(blank=True, null=True)
+    source = models.CharField(max_length=32, blank=True, default="")
+    pull_up_on_save = ["title", "year"]
+
+    class Meta:
+        app_label = "django_pbn_client_package_tests"
+
+
+@pytest.fixture(scope="module", autouse=True)
+def mirror_record_table(django_db_setup, django_db_blocker):
+    with django_db_blocker.unblock(), connection.schema_editor() as editor:
+        editor.create_model(MirrorRecord)
+
+    yield
+
+    with django_db_blocker.unblock(), connection.schema_editor() as editor:
+        editor.delete_model(MirrorRecord)
+
+
+def _element(
+    *,
+    status="ACTIVE",
+    verification_level="VERIFIED",
+    verified=True,
+    title="A title",
+    year=2026,
+):
+    return {
+        "mongoId": "mongo-1",
+        "status": status,
+        "verificationLevel": verification_level,
+        "verified": verified,
+        "versions": [
+            {
+                "current": True,
+                "object": {
+                    "title": title,
+                    "year": year,
+                    "website": "https://pbn.example/object",
+                },
+            }
+        ],
+    }
+
+
+@pytest.mark.django_db
+def test_base_model_reads_current_version_and_pulls_up_text():
+    record = upsert_pbn_object(_element(title="Downloaded title"), MirrorRecord)
+
+    assert record.current_version["current"] is True
+    assert record.value("object", "title") == "Downloaded title"
+    assert record.value_or_none("object", "missing") is None
+    assert record.website() == "https://pbn.example/object"
+    assert record.title == "Downloaded title"
+    assert record.year == 2026
+
+
+@pytest.mark.django_db
+def test_pull_up_truncates_pathological_indexed_text():
+    title = "x" * (MAX_TEXT_FIELD_LENGTH + 50)
+
+    record = upsert_pbn_object(_element(title=title), MirrorRecord)
+
+    assert record.title == "x" * MAX_TEXT_FIELD_LENGTH
+
+
+@pytest.mark.django_db
+def test_upsert_updates_all_mirrored_state_and_extra_fields():
+    upsert_pbn_object(_element(title="Before"), MirrorRecord, source="api")
+
+    record = upsert_pbn_object(
+        _element(
+            status="DELETED",
+            verification_level="REJECTED",
+            verified=False,
+            title="After",
+        ),
+        MirrorRecord,
+        source="offline",
+    )
+
+    assert record.status == "DELETED"
+    assert record.verificationLevel == "REJECTED"
+    assert record.verified is False
+    assert record.source == "offline"
+    assert record.title == "After"
+
+
+@pytest.mark.django_db
+def test_unchanged_upsert_does_not_touch_last_updated_timestamp():
+    payload = _element()
+    original = upsert_pbn_object(payload, MirrorRecord)
+    original_timestamp = original.last_updated_on
+
+    unchanged = upsert_pbn_object(payload, MirrorRecord)
+
+    assert unchanged.last_updated_on == original_timestamp

django_pbn_client-0.1.0/tests/test_django_pbn_client_pages.py

@@ -0,0 +1,115 @@
+import pytest
+
+from django_pbn_client.pages import (
+    ThreadedModelSaver,
+    ThreadedMongoDBSaver,
+    ThreadedPageGetter,
+    download_pages,
+    simple_page_getter,
+)
+
+
+def test_threaded_model_saver_uses_injected_model_and_save_function():
+    calls = []
+    model_class = object()
+
+    class Saver(ThreadedModelSaver):
+        save_function = staticmethod(
+            lambda element, model: calls.append((element, model))
+        )
+
+    saver = Saver(model_class=model_class)
+    saver.process_element({"mongoId": "one"})
+
+    assert calls == [({"mongoId": "one"}, model_class)]
+    assert ThreadedMongoDBSaver is ThreadedModelSaver
+
+
+def test_threaded_model_saver_binds_model_via_class_attribute():
+    # download_pages() instantiates the getter class without passing
+    # model_class, so subclasses that bind the model as a class attribute
+    # must keep working — the constructor must not shadow it with None.
+    calls = []
+    sentinel_model = object()
+
+    class Saver(ThreadedModelSaver):
+        model_class = sentinel_model
+        save_function = staticmethod(
+            lambda element, model: calls.append((element, model))
+        )
+
+    Saver().process_element({"mongoId": "one"})
+
+    assert calls == [({"mongoId": "one"}, sentinel_model)]
+
+
+def test_download_pages_processes_each_page_with_host_progress():
+    processed = []
+    progress_calls = []
+
+    class Data:
+        total_pages = 3
+
+        def fetch_page(self, page_number):
+            return [page_number * 2, page_number * 2 + 1]
+
+    class Getter(ThreadedPageGetter):
+        def process_element(self, element):
+            processed.append((self.client, element))
+
+    def progress(pages, total, label):
+        progress_calls.append((total, label))
+        return pages
+
+    client = object()
+    download_pages(
+        client,
+        Data(),
+        getter_class=Getter,
+        workers=2,
+        label="Pages",
+        progress=progress,
+    )
+
+    assert progress_calls == [(3, "Pages")]
+    assert sorted(processed, key=lambda item: item[1]) == [
+        (client, 0),
+        (client, 1),
+        (client, 2),
+        (client, 3),
+        (client, 4),
+        (client, 5),
+    ]
+
+
+def test_download_pages_rejects_unknown_concurrency_method():
+    class Data:
+        total_pages = 0
+
+    with pytest.raises(ValueError, match="greenlets"):
+        download_pages(None, Data(), method="greenlets")
+
+
+@pytest.mark.django_db
+def test_process_workers_use_their_initialized_getter():
+    class Data:
+        total_pages = 2
+
+        def fetch_page(self, _page_number):
+            return []
+
+    download_pages(None, Data(), method="processes", workers=1)
+
+
+def test_simple_page_getter_yields_each_page_in_order():
+    class Data:
+        total_pages = 3
+
+        def fetch_page(self, page_number):
+            return [f"page-{page_number}"]
+
+    assert list(simple_page_getter(None, Data())) == [
+        ["page-0"],
+        ["page-1"],
+        ["page-2"],
+    ]

django_pbn_client-0.1.0/tests/test_django_pbn_client_persistence.py

@@ -0,0 +1,116 @@
+import pytest
+from django.core.exceptions import ObjectDoesNotExist
+from django.db import IntegrityError, connection
+
+from django_pbn_client.persistence import (
+    download_pbn_objects,
+    get_total_count,
+    upsert_pbn_object,
+)
+
+
+def _element():
+    return {
+        "mongoId": "race-winner",
+        "status": "ACTIVE",
+        "verificationLevel": "VERIFIED",
+        "verified": True,
+        "versions": [{"current": True, "object": {}}],
+    }
+
+
+def test_download_uses_paginator_total_and_host_progress_wrapper():
+    saved = []
+    progress_calls = []
+
+    class Elements:
+        total_elements = 2
+
+        def __iter__(self):
+            yield {"mongoId": "one"}
+            yield {"mongoId": "two"}
+
+    def save(element, model_class, client=None):
+        saved.append((element["mongoId"], model_class, client))
+
+    def progress(elements, total, label):
+        progress_calls.append((total, label))
+        return elements
+
+    model_class = object()
+    client = object()
+    download_pbn_objects(
+        Elements(),
+        model_class,
+        label="Downloading",
+        save=save,
+        client=client,
+        progress=progress,
+    )
+
+    assert progress_calls == [(2, "Downloading")]
+    assert saved == [
+        ("one", model_class, client),
+        ("two", model_class, client),
+    ]
+
+
+def test_total_count_supports_sized_and_counted_iterables():
+    class Counted:
+        count = 7
+
+    assert get_total_count([1, 2, 3]) == 3
+    assert get_total_count(Counted()) == 7
+    assert get_total_count(iter([1, 2])) is None
+
+
+@pytest.mark.django_db
+def test_insert_race_fallback_keeps_outer_transaction_usable():
+    class RaceDoesNotExist(ObjectDoesNotExist):
+        pass
+
+    class WinningInstance:
+        status = "OLD"
+        verificationLevel = "OLD"
+        verified = False
+        versions = []
+        save_calls = 0
+
+        def save(self):
+            self.save_calls += 1
+
+    winner = WinningInstance()
+
+    class MissingQuery:
+        def get(self):
+            raise RaceDoesNotExist
+
+    class RaceManager:
+        def select_for_update(self):
+            return self
+
+        def filter(self, **kwargs):
+            assert kwargs == {"pk": "race-winner"}
+            return MissingQuery()
+
+        def create(self, **kwargs):
+            raise IntegrityError("another worker inserted the row")
+
+        def get(self, **kwargs):
+            assert kwargs == {"pk": "race-winner"}
+            with connection.cursor() as cursor:
+                cursor.execute("SELECT 1")
+            return winner
+
+    class RaceModel:
+        objects = RaceManager()
+        DoesNotExist = RaceDoesNotExist
+
+    result = upsert_pbn_object(_element(), RaceModel)
+
+    assert result is winner
+    assert winner.status == "ACTIVE"
+    assert winner.verificationLevel == "VERIFIED"
+    assert winner.verified is True
+    assert winner.versions == [{"current": True, "object": {}}]
+    assert winner.save_calls == 1