django-pfx 1.7.6.dev35__tar.gz → 1.7.6.dev39__tar.gz

{django_pfx-1.7.6.dev35 → django_pfx-1.7.6.dev39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: django-pfx
-Version: 1.7.6.dev35
+Version: 1.7.6.dev39
 Summary: Django PFX is a toolkit designed to streamline the development of RESTful APIs using the Django framework.
 Author: Hervé Martinet
 Author-email: herve.martinet@gmail.com
@@ -41,6 +41,8 @@ Requires-Dist: pyotp>=2.10.0; extra == "otp"
 Provides-Extra: heic-converter
 Requires-Dist: python-magic; extra == "heic-converter"
 Requires-Dist: pillow-heif; extra == "heic-converter"
+Provides-Extra: xlsx
+Requires-Dist: openpyxl; extra == "xlsx"
 Dynamic: license-file
 
 # Django PFX

{django_pfx-1.7.6.dev35 → django_pfx-1.7.6.dev39}/django_pfx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: django-pfx
-Version: 1.7.6.dev35
+Version: 1.7.6.dev39
 Summary: Django PFX is a toolkit designed to streamline the development of RESTful APIs using the Django framework.
 Author: Hervé Martinet
 Author-email: herve.martinet@gmail.com
@@ -41,6 +41,8 @@ Requires-Dist: pyotp>=2.10.0; extra == "otp"
 Provides-Extra: heic-converter
 Requires-Dist: python-magic; extra == "heic-converter"
 Requires-Dist: pillow-heif; extra == "heic-converter"
+Provides-Extra: xlsx
+Requires-Dist: openpyxl; extra == "xlsx"
 Dynamic: license-file
 
 # Django PFX

{django_pfx-1.7.6.dev35 → django_pfx-1.7.6.dev39}/django_pfx.egg-info/SOURCES.txt

@@ -220,6 +220,7 @@ tests/tests/__init__.py
 tests/tests/basic_api_errors.py
 tests/tests/basic_api_test.py
 tests/tests/mfa_test_utils.py
+tests/tests/parallel_scenarios.py
 tests/tests/test_api_doc.py
 tests/tests/test_api_doc_search.py
 tests/tests/test_auth_api.py
@@ -252,6 +253,7 @@ tests/tests/test_post_migrate_groups_update.py
 tests/tests/test_profiling_middleware.py
 tests/tests/test_settings.py
 tests/tests/test_shortcuts.py
+tests/tests/test_table_file_helpers.py
 tests/tests/test_test_runner.py
 tests/tests/test_timezone_middleware.py
 tests/tests/test_tools.py

{django_pfx-1.7.6.dev35 → django_pfx-1.7.6.dev39}/django_pfx.egg-info/requires.txt

@@ -16,3 +16,6 @@ pillow-heif
 
 [otp]
 pyotp>=2.10.0
+
+[xlsx]
+openpyxl

{django_pfx-1.7.6.dev35 → django_pfx-1.7.6.dev39}/django_pfx.egg-info/scm_file_list.json

@@ -210,6 +210,7 @@
     "tests/tests/basic_api_errors.py",
     "tests/tests/basic_api_test.py",
     "tests/tests/mfa_test_utils.py",
+    "tests/tests/parallel_scenarios.py",
     "tests/tests/test_api_doc.py",
     "tests/tests/test_api_doc_search.py",
     "tests/tests/test_auth_api.py",
@@ -242,6 +243,7 @@
     "tests/tests/test_profiling_middleware.py",
     "tests/tests/test_settings.py",
     "tests/tests/test_shortcuts.py",
+    "tests/tests/test_table_file_helpers.py",
     "tests/tests/test_test_runner.py",
     "tests/tests/test_timezone_middleware.py",
     "tests/tests/test_tools.py",

django_pfx-1.7.6.dev39/django_pfx.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "1.7.5",
+  "distance": 39,
+  "node": "g471828c66bafecdb4544e8e23075df5d7b128c66",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-10-21"
+}

{django_pfx-1.7.6.dev35 → django_pfx-1.7.6.dev39}/doc/source/testing.md

@@ -430,3 +430,39 @@ class ATestClass(EmailTestMixin, SMSTestMixin, TestCase):
         [(to, message)] = self.sms_outbox()
         self.assertEqual(to, '+41791234567')
 ```
+
+## CSV and Excel files
+
+`CSVFileHelper` and `XLSXFileHelper` read back a table file of a test: an
+export, a file attached to a user job, a file to import.
+
+```python
+from pfx.pfxcore.test import CSVFileHelper, XLSXFileHelper
+
+f = XLSXFileHelper(response)            # or CSVFileHelper(response)
+f.header()                              # ['Name', 'Birth']
+f.dicts()                               # [{'Name': 'Alice', 'Birth': datetime(…)}, …]
+f.lines()[0][f.column('Name')]          # 'Alice'
+```
+
+The file can be given as bytes, a string (its content), a `Path`, an HTTP
+response (streamed or not),
+an iterable of chunks, or the value of a `MediaField` on the local storage
+(`XLSXFileHelper(job.attachments.get().file)`).
+
+- `header=2`: the number of the line of the headers, for a file with a title
+  above them. The lines after it are the data.
+- `header()`, `lines()`, `dicts()`: the headers, the data as lists, the data
+  as dicts by header. `line(n)` and `line_dict(n)` count from 0, the first
+  line of the file; `len()` counts every line.
+- `column(title)`: the index of a column.
+- `write_file(path)` and `pretty_print()`: to look at the file while writing a
+  test.
+
+`CSVFileHelper(data, delimiter=';', encoding='utf-8-sig')` reads the CSV of a
+French Excel; its values are strings.
+
+`XLSXFileHelper(data, sheet='Totals')` reads a sheet by name or by index, the
+active one by default. Its values keep their type (numbers, dates), and
+`workbook` and `sheet` give the formats and the styles. It needs
+[openpyxl](https://openpyxl.readthedocs.io/): install `django-pfx[xlsx]`.

{django_pfx-1.7.6.dev35 → django_pfx-1.7.6.dev39}/pfx/pfxcore/test.py

@@ -1,15 +1,25 @@
+import copy
+import csv
+import io
 import json
 import logging
 import re
+import traceback
 from functools import reduce
 from http.cookies import SimpleCookie
 from json import JSONDecodeError
+from pathlib import Path
 
 from django.apps import apps
 from django.db import transaction
 from django.test import override_settings
 from django.test.client import Client as DjangoClient
-from django.test.runner import DiscoverRunner, ParallelTestSuite
+from django.test.runner import (
+    DiscoverRunner,
+    ParallelTestSuite,
+    RemoteTestResult,
+    RemoteTestRunner,
+)
 from django.test.runner import _init_worker as django_init_worker
 from django.utils.module_loading import import_string
 
@@ -320,6 +330,130 @@ class TestPermsAssertMixin(TestAssertMixin):
             self.assertPerms(**perms)
 
 
+def file_content(data):
+    """The bytes of a file given to a test.
+
+    ``data`` can be bytes, a string, a ``Path``, an HTTP response (streamed
+    or not), an iterable of chunks, or the value of a ``MediaField`` on the
+    local storage, such as the attachment of a user job.
+    """
+    if isinstance(data, bytes):
+        return data
+    if isinstance(data, Path):
+        return data.read_bytes()
+    if isinstance(data, str):
+        return data.encode()
+    if isinstance(data, dict):
+        return Path(settings.STORAGE_LOCAL_ROOT, data['key']).read_bytes()
+    if hasattr(data, 'streaming_content'):
+        data = data.streaming_content
+    elif hasattr(data, 'content'):
+        return data.content
+    return b"".join(
+        chunk.encode() if isinstance(chunk, str) else chunk
+        for chunk in data)
+
+
+class TableFileResponseHelper:
+    """A table file of a test, read back as rows of values.
+
+    ``header`` is the number of the line of the headers, the first by
+    default: the lines after it are the data. :meth:`line` counts from 0,
+    the first line of the file.
+    """
+
+    def __init__(self, data, header=1):
+        self.content = file_content(data)
+        self.header_index = header - 1
+        self.rows = self.read_rows()
+
+    def read_rows(self):
+        raise NotImplementedError  # pragma: no cover
+
+    def write_file(self, path):
+        """Write the file, to open it while writing a test."""
+        Path(path).write_bytes(self.content)
+
+    def pretty_print(self):
+        """Print the rows as a table, to read them while writing a test."""
+        cells = [
+            ['' if v is None else str(v) for v in row] for row in self.rows]
+        widths = [
+            max(len(row[i]) for row in cells if i < len(row))
+            for i in range(max((len(row) for row in cells), default=0))]
+        print()
+        for row in cells:
+            print('  '.join(v.ljust(w) for v, w in zip(row, widths)).rstrip())
+
+    def header(self):
+        return self.rows[self.header_index]
+
+    def line(self, n):
+        return self.rows[n]
+
+    def len(self):
+        return len(self.rows)
+
+    def lines(self):
+        """The rows after the headers."""
+        return self.rows[self.header_index + 1:]
+
+    def column(self, title):
+        """The index of the column ``title``."""
+        return self.header().index(title)
+
+    def line_dict(self, n):
+        """The line ``n`` by header."""
+        return dict(zip(self.header(), self.line(n)))
+
+    def dicts(self):
+        """The rows after the headers, by header."""
+        return [dict(zip(self.header(), row)) for row in self.lines()]
+
+
+class CSVFileHelper(TableFileResponseHelper):
+    """A CSV file of a test; the values are strings."""
+
+    def __init__(self, data, header=1, delimiter=',', encoding='utf-8-sig'):
+        self.delimiter = delimiter
+        self.encoding = encoding
+        super().__init__(data, header)
+
+    def read_rows(self):
+        return list(csv.reader(
+            io.StringIO(self.content.decode(self.encoding)),
+            delimiter=self.delimiter))
+
+
+class XLSXFileHelper(TableFileResponseHelper):
+    """An Excel file of a test, read with openpyxl.
+
+    ``sheet`` is the name or the index of the sheet read, the active one
+    by default. The values keep their type (numbers, dates); ``workbook``
+    and ``sheet`` give the formats and the styles.
+    """
+
+    def __init__(self, data, header=1, sheet=None):
+        self.sheet_name = sheet
+        super().__init__(data, header)
+
+    def read_rows(self):
+        try:
+            import openpyxl
+        except ImportError:  # pragma: no cover
+            raise ImportError(
+                "XLSXFileHelper needs openpyxl: install django-pfx[xlsx].")
+
+        self.workbook = openpyxl.load_workbook(io.BytesIO(self.content))
+        if self.sheet_name is None:
+            self.sheet = self.workbook.active
+        elif isinstance(self.sheet_name, int):
+            self.sheet = self.workbook.worksheets[self.sheet_name]
+        else:
+            self.sheet = self.workbook[self.sheet_name]
+        return [list(row) for row in self.sheet.iter_rows(values_only=True)]
+
+
 class MockBoto3Client:
     def generate_presigned_url(self, *args, **kwargs):
         params = kwargs.get('Params', {})
@@ -351,8 +485,72 @@ def _init_worker(*args, **kwargs):
     override_settings(**get_test_backends()).enable()
 
 
+class UnpicklableTestError(Exception):
+    """An error a parallel worker could not send back, as text."""
+
+
+class UnpicklableTestFailure(UnpicklableTestError, AssertionError):
+    """A failure a parallel worker could not send back, as text."""
+
+
+class _Repr(str):
+    def __repr__(self):
+        return str(self)
+
+
+class PicklableRemoteTestResult(RemoteTestResult):
+    """Send every outcome of a parallel worker back to the parent.
+
+    Django aborts the whole run, without the traceback, on an exception it
+    cannot pickle (one holding a request, a lock, a local class…). Such an
+    exception is sent as its formatted traceback instead, and the run goes
+    on."""
+
+    def _picklable(self, err):
+        try:
+            self._confirm_picklable(err)
+        except Exception:
+            cls = (
+                UnpicklableTestFailure if issubclass(err[0], AssertionError)
+                else UnpicklableTestError)
+            return cls, cls(''.join(traceback.format_exception(*err))), None
+        return err
+
+    def _picklable_subtest(self, test, subtest):
+        # The subtest holds the running test case, which can become
+        # unpicklable after the failure: a fresh one is always sent.
+        subtest = copy.copy(subtest)
+        subtest.test_case = type(test)(test._testMethodName)
+        try:
+            self._confirm_picklable(subtest)
+        except Exception:
+            subtest.params = {
+                k: _Repr(repr(v)) for k, v in subtest.params.items()}
+        return subtest
+
+    def addError(self, test, err):
+        super().addError(test, self._picklable(err))
+
+    def addFailure(self, test, err):
+        super().addFailure(test, self._picklable(err))
+
+    def addExpectedFailure(self, test, err):
+        super().addExpectedFailure(test, self._picklable(err))
+
+    def addSubTest(self, test, subtest, err):
+        if err is not None:
+            err = self._picklable(err)
+            subtest = self._picklable_subtest(test, subtest)
+        super().addSubTest(test, subtest, err)
+
+
+class PicklableRemoteTestRunner(RemoteTestRunner):
+    resultclass = PicklableRemoteTestResult
+
+
 class MemoryBackendsParallelTestSuite(ParallelTestSuite):
     init_worker = _init_worker
+    runner_class = PicklableRemoteTestRunner
 
 
 class MemoryBackendsRunnerMixin:
@@ -371,7 +569,10 @@ class MemoryBackendsRunnerMixin:
       validated, which ``locmem`` never does.
 
     A test class decorated with ``override_settings`` keeps its own
-    backends."""
+    backends.
+
+    With ``--parallel``, the workers also send back the errors Django
+    cannot pickle (see :class:`PicklableRemoteTestResult`)."""
 
     parallel_test_suite = MemoryBackendsParallelTestSuite
 

{django_pfx-1.7.6.dev35 → django_pfx-1.7.6.dev39}/requirements.txt

@@ -4,6 +4,8 @@ pre-commit
 flake8
 isort
 coverage
+# Pickles the tracebacks of the --parallel workers.
+tblib
 psycopg
 Babel
 pyjwt>=2.1
@@ -31,3 +33,4 @@ munch
 # bypasses the error handling of pfx.emails.models.Email.send.
 mailjet_rest<1.8
 responses
+openpyxl

{django_pfx-1.7.6.dev35 → django_pfx-1.7.6.dev39}/setup.cfg

@@ -46,6 +46,7 @@ install_requires =
 [options.extras_require]
 otp = pyotp>=2.10.0
 heic-converter = python-magic; pillow-heif
+xlsx = openpyxl
 
 [options.packages.find]
 exclude = 

django_pfx-1.7.6.dev39/tests/tests/parallel_scenarios.py

@@ -0,0 +1,89 @@
+"""Test cases that fail in ways a parallel worker cannot pickle.
+
+Run by test_test_runner.py, not by the discovery (the name does not start
+with ``test``). They live at module level as real test classes do: the
+parent process rebuilds a test case from its class."""
+import threading
+import unittest
+
+
+class Unpicklable(Exception):
+    def __init__(self):
+        super().__init__("holds a lock")
+        self.lock = threading.Lock()
+
+
+class KeywordOnly(Exception):
+    """Pickles, but cannot be rebuilt from its args."""
+
+    def __init__(self, *, code):
+        super().__init__(f"code {code}")
+        self.code = code
+
+
+class UnpicklableAssertion(AssertionError):
+    def __init__(self):
+        super().__init__("assertion holding a lock")
+        self.lock = threading.Lock()
+
+
+class Scenarios(unittest.TestCase):
+
+    def test_plain_failure(self):
+        self.assertEqual(1, 2)
+
+    def test_lock_in_the_exception(self):
+        raise Unpicklable()
+
+    def test_unpicklable_cause(self):
+        try:
+            raise Unpicklable()
+        except Unpicklable as e:
+            raise ValueError("wrapped") from e
+
+    def test_local_class(self):
+        class Local(Exception):
+            pass
+        raise Local("local class")
+
+    def test_keyword_only_init(self):
+        raise KeywordOnly(code=42)
+
+    def test_unpicklable_assertion(self):
+        raise UnpicklableAssertion()
+
+    @unittest.expectedFailure
+    def test_expected_failure(self):
+        raise Unpicklable()
+
+    def test_subtest_then_a_lock(self):
+        with self.subTest(i=1):
+            self.assertEqual(1, 5)
+        self.lock = threading.Lock()
+
+    def test_subtest_with_a_lock_param(self):
+        with self.subTest(lock=threading.Lock()):
+            self.fail("with a lock as param")
+
+    def test_subtest_unpicklable_error(self):
+        with self.subTest(n=1):
+            raise Unpicklable()
+
+
+class TearDownError(unittest.TestCase):
+
+    def tearDown(self):
+        raise Unpicklable()
+
+    def test_ok(self):
+        pass
+
+
+class SetUpClassError(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        raise Unpicklable()
+
+    def test_never_runs(self):
+        pass

django_pfx-1.7.6.dev39/tests/tests/test_table_file_helpers.py

@@ -0,0 +1,144 @@
+import io
+import os
+import tempfile
+from contextlib import redirect_stdout
+from datetime import date
+from pathlib import Path
+
+from django.http import HttpResponse, StreamingHttpResponse
+from django.test import SimpleTestCase, override_settings
+
+from openpyxl import Workbook
+
+from pfx.pfxcore.test import CSVFileHelper, XLSXFileHelper
+
+CSV = "Name,Age\nAlice,30\nBob,\n"
+
+
+def xlsx(*sheets):
+    """An Excel file of `sheets`, (title, rows) pairs."""
+    workbook = Workbook()
+    workbook.remove(workbook.active)
+    for title, rows in sheets:
+        sheet = workbook.create_sheet(title)
+        for row in rows:
+            sheet.append(row)
+    output = io.BytesIO()
+    workbook.save(output)
+    return output.getvalue()
+
+
+PEOPLE = [['Name', 'Birth'], ['Alice', date(1990, 5, 1)], ['Bob', None]]
+
+
+class TestCSVFileHelper(SimpleTestCase):
+    def test_the_sources_of_a_file(self):
+        """Bytes, a string, a response, a streamed one, chunks of text."""
+        for data in [
+                CSV.encode(), CSV, HttpResponse(CSV),
+                StreamingHttpResponse(iter([CSV[:10], CSV[10:]])),
+                [CSV[:10].encode(), CSV[10:].encode()]]:
+            with self.subTest(data=type(data).__name__):
+                f = CSVFileHelper(data)
+                self.assertEqual(f.header(), ['Name', 'Age'])
+                self.assertEqual(f.len(), 3)
+
+    def test_lines(self):
+        """`line` counts from the headers, `lines` and `dicts` are the data;
+        a CSV value is a string."""
+        f = CSVFileHelper(CSV)
+
+        self.assertEqual(f.line(1), ['Alice', '30'])
+        self.assertEqual(f.lines(), [['Alice', '30'], ['Bob', '']])
+        self.assertEqual(f.column('Age'), 1)
+        self.assertEqual(f.line_dict(2), {'Name': 'Bob', 'Age': ''})
+        self.assertEqual(
+            f.dicts(),
+            [{'Name': 'Alice', 'Age': '30'}, {'Name': 'Bob', 'Age': ''}])
+
+    def test_a_french_excel_csv(self):
+        """Semicolons, and the byte order mark Excel writes."""
+        f = CSVFileHelper(
+            "Nom;Âge\nAlice;30\n".encode(), delimiter=';')
+
+        self.assertEqual(f.dicts(), [{'Nom': 'Alice', 'Âge': '30'}])
+
+    def test_headers_below_a_title(self):
+        f = CSVFileHelper("Report\nName,Age\nAlice,30\n", header=2)
+
+        self.assertEqual(f.header(), ['Name', 'Age'])
+        self.assertEqual(f.dicts(), [{'Name': 'Alice', 'Age': '30'}])
+
+    def test_unknown_column(self):
+        with self.assertRaises(ValueError):
+            CSVFileHelper(CSV).column('Size')
+
+    def test_a_path(self):
+        """A Path is a file to read; a string is the content itself."""
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp, 'file.csv')
+            path.write_text(CSV)
+
+            self.assertEqual(CSVFileHelper(path).len(), 3)
+
+    def test_write_file_and_pretty_print(self):
+        f = CSVFileHelper(CSV)
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, 'file.csv')
+            f.write_file(path)
+            with open(path, 'rb') as written:
+                self.assertEqual(written.read(), CSV.encode())
+
+        out = io.StringIO()
+        with redirect_stdout(out):
+            f.pretty_print()
+        self.assertEqual(
+            out.getvalue(), "\nName   Age\nAlice  30\nBob\n")
+
+
+class TestXLSXFileHelper(SimpleTestCase):
+    def test_values_keep_their_type(self):
+        """A date is a datetime, an empty cell None."""
+        f = XLSXFileHelper(xlsx(('People', PEOPLE)))
+
+        self.assertEqual(f.header(), ['Name', 'Birth'])
+        self.assertEqual(f.line_dict(1)['Birth'].date(), date(1990, 5, 1))
+        self.assertEqual(f.dicts()[1], {'Name': 'Bob', 'Birth': None})
+        self.assertEqual(f.len(), 3)
+        self.assertEqual(f.column('Birth'), 1)
+
+    def test_the_sheet_read(self):
+        """The active one by default, else by name or by index; the sheet
+        is there for the formats and the styles."""
+        content = xlsx(('People', PEOPLE), ('Totals', [['Total'], [2]]))
+
+        self.assertEqual(XLSXFileHelper(content).header(), ['Name', 'Birth'])
+        f = XLSXFileHelper(content, sheet='Totals')
+        self.assertEqual(f.dicts(), [{'Total': 2}])
+        self.assertEqual(f.sheet.title, 'Totals')
+        self.assertEqual(
+            XLSXFileHelper(content, sheet=1).header(), ['Total'])
+
+    def test_a_response(self):
+        f = XLSXFileHelper(StreamingHttpResponse(
+            iter([xlsx(('People', PEOPLE))])))
+
+        self.assertEqual(f.lines()[0][0], 'Alice')
+
+    def test_headers_below_a_title(self):
+        f = XLSXFileHelper(
+            xlsx(('Report', [['Report 2026'], *PEOPLE])), header=2)
+
+        self.assertEqual(f.header(), ['Name', 'Birth'])
+        self.assertEqual(len(f.dicts()), 2)
+
+    def test_a_file_of_the_local_storage(self):
+        """The value of a MediaField, such as the attachment of a job."""
+        with tempfile.TemporaryDirectory() as tmp:
+            with open(os.path.join(tmp, 'export.xlsx'), 'wb') as f:
+                f.write(xlsx(('People', PEOPLE)))
+            with override_settings(STORAGE_LOCAL_ROOT=tmp):
+                helper = XLSXFileHelper(
+                    {'key': 'export.xlsx', 'name': 'export.xlsx'})
+
+        self.assertEqual(helper.header(), ['Name', 'Birth'])