girder-jsonforms 2.1.2__tar.gz → 2.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {girder_jsonforms-2.1.2/girder_jsonforms.egg-info → girder_jsonforms-2.1.3}/PKG-INFO +2 -10
  2. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/igsn_client.py +45 -4
  3. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/igsn_vocab.py +43 -1
  4. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/rest/deposition.py +5 -1
  5. girder_jsonforms-2.1.3/girder_jsonforms/scripts/reconcile_igsn.py +118 -0
  6. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/worker_plugin/igsn_registry.py +123 -4
  7. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3/girder_jsonforms.egg-info}/PKG-INFO +2 -10
  8. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms.egg-info/SOURCES.txt +2 -4
  9. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms.egg-info/entry_points.txt +1 -0
  10. girder_jsonforms-2.1.3/pyproject.toml +75 -0
  11. girder_jsonforms-2.1.2/.github/workflows/release.yml +0 -102
  12. girder_jsonforms-2.1.2/.gitignore +0 -136
  13. girder_jsonforms-2.1.2/girder_jsonforms.egg-info/not-zip-safe +0 -1
  14. girder_jsonforms-2.1.2/setup.py +0 -67
  15. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/LICENSE +0 -0
  16. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/README.md +0 -0
  17. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/doc/file-uploads.md +0 -0
  18. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/doc/preload.md +0 -0
  19. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/__init__.py +0 -0
  20. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/asgi.py +0 -0
  21. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/__init__.py +0 -0
  22. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/announcement.py +0 -0
  23. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/events.py +0 -0
  24. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/google_drive.py +0 -0
  25. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/jq.py +0 -0
  26. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/locks.py +0 -0
  27. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/mail.py +0 -0
  28. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/metadata_dates.py +0 -0
  29. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/orcid.py +0 -0
  30. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/project_helpers.py +0 -0
  31. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/project_tasks.py +0 -0
  32. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/response_cache.py +0 -0
  33. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/lib/timing.py +0 -0
  34. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/mail_assets/caimee-shield-80.png +0 -0
  35. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/mail_templates/_projectsFooter.mako +0 -0
  36. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/mail_templates/_projectsFooter.txt.mako +0 -0
  37. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/mail_templates/_projectsHeader.mako +0 -0
  38. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/mail_templates/projectDecision.mako +0 -0
  39. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/mail_templates/projectDecision.txt.mako +0 -0
  40. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/mail_templates/projectSubmitted.mako +0 -0
  41. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/mail_templates/projectSubmitted.txt.mako +0 -0
  42. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/models/__init__.py +0 -0
  43. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/models/deposition.py +0 -0
  44. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/models/entry.py +0 -0
  45. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/models/form.py +0 -0
  46. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/models/project.py +0 -0
  47. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/rest/__init__.py +0 -0
  48. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/rest/aimdl.py +0 -0
  49. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/rest/entry.py +0 -0
  50. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/rest/form.py +0 -0
  51. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/rest/project.py +0 -0
  52. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/schemas/__init__.py +0 -0
  53. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/schemas/datacite-v4.5.json +0 -0
  54. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/scripts/__init__.py +0 -0
  55. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/scripts/migrate_metadata_dates.py +0 -0
  56. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/scripts/preview_mail.py +0 -0
  57. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/settings.py +0 -0
  58. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/web_client/dist/girder-plugin-jsonforms.umd.cjs +0 -0
  59. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/web_client/dist/style.css +0 -0
  60. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/worker_plugin/__init__.py +0 -0
  61. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/worker_plugin/amdee.py +0 -0
  62. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/worker_plugin/folder_ops.py +0 -0
  63. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/worker_plugin/handle_enumsource.py +0 -0
  64. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/worker_plugin/htmax.py +0 -0
  65. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/worker_plugin/orcid.py +0 -0
  66. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/worker_plugin/projects.py +0 -0
  67. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/worker_plugin/pull_related_ids.py +0 -0
  68. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms/worker_plugin/relate_entry_to_igsn.py +0 -0
  69. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms.egg-info/dependency_links.txt +0 -0
  70. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms.egg-info/requires.txt +0 -0
  71. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/girder_jsonforms.egg-info/top_level.txt +0 -0
  72. {girder_jsonforms-2.1.2 → girder_jsonforms-2.1.3}/setup.cfg +0 -0
@@ -1,11 +1,10 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: girder-jsonforms
3
- Version: 2.1.2
3
+ Version: 2.1.3
4
4
  Summary: Girder plugin adding forms based on JSON-editor
5
- License: BSD-3-Clause
5
+ License-Expression: BSD-3-Clause
6
6
  Classifier: Development Status :: 4 - Beta
7
7
  Classifier: Environment :: Web Environment
8
- Classifier: License :: OSI Approved :: BSD License
9
8
  Classifier: Operating System :: POSIX :: Linux
10
9
  Classifier: Programming Language :: Python
11
10
  Classifier: Programming Language :: Python :: 3
@@ -23,14 +22,7 @@ Requires-Dist: google-auth-oauthlib
23
22
  Requires-Dist: jsondiff
24
23
  Requires-Dist: pandas
25
24
  Requires-Dist: openpyxl
26
- Dynamic: classifier
27
- Dynamic: description
28
- Dynamic: description-content-type
29
- Dynamic: license
30
25
  Dynamic: license-file
31
- Dynamic: requires-dist
32
- Dynamic: requires-python
33
- Dynamic: summary
34
26
 
35
27
  ### Girder JSONForms Plugin
36
28
 
@@ -19,6 +19,7 @@ Two rules matter here:
19
19
  import logging
20
20
 
21
21
  import requests
22
+ from girder.exceptions import RestException
22
23
  from girder.models.setting import Setting
23
24
  from requests.adapters import HTTPAdapter
24
25
  from urllib3.util.retry import Retry
@@ -29,16 +30,40 @@ logger = logging.getLogger(__name__)
29
30
 
30
31
  DEFAULT_TIMEOUT = 30
31
32
 
33
+ # Upstream statuses that mean "the caller asked for something the registry will
34
+ # not do": a prefix outside the tenant's vocabulary, an identifier that already
35
+ # exists, a published record that cannot be withdrawn. The service's own
36
+ # explanation is the useful part and the person filling in the form can act on
37
+ # it, so these become a 400 carrying that message.
38
+ #
39
+ # Everything else -- our service token rejected, a 5xx, an unreachable host --
40
+ # is this deployment's problem rather than the caller's, so it becomes a 502
41
+ # and the detail is left to the log.
42
+ CLIENT_ERROR_STATUSES = frozenset({400, 404, 409, 422})
32
43
 
33
- class IGSNServiceError(Exception):
44
+
45
+ def _rest_code(status_code):
46
+ return 400 if status_code in CLIENT_ERROR_STATUSES else 502
47
+
48
+
49
+ class IGSNServiceError(RestException):
34
50
  """The IGSN registry service could not satisfy a request.
35
51
 
36
52
  Deliberately fatal to whatever operation raised it: a Girder deposition must
37
53
  never exist without a matching record in the registry.
54
+
55
+ It is a ``RestException`` so that the fatality is a *useful* one. Identifier
56
+ allocation happens several levels below the endpoint -- in
57
+ ``PrefixCounter.get_next``, and off the ``model.entry.save`` event for the
58
+ form path -- and most of those call sites have no business knowing about
59
+ HTTP. Left as a plain exception it reached the REST layer as an unhandled
60
+ error, so "Institution JH is not permitted for this tenant" showed up in the
61
+ UI as a bare 500. Subclassing means every path reports the registry's reason
62
+ without each one having to catch and translate.
38
63
  """
39
64
 
40
65
  def __init__(self, message, status_code=None, detail=None):
41
- super().__init__(message)
66
+ super().__init__(message, code=_rest_code(status_code))
42
67
  self.status_code = status_code
43
68
  self.detail = detail
44
69
 
@@ -82,9 +107,25 @@ class IGSNClient:
82
107
  detail = response.json().get("detail")
83
108
  except ValueError:
84
109
  detail = response.text[:500]
110
+ # The verb and path are what an operator needs, so they go to the
111
+ # log; the message itself is shown to the user verbatim for a 4xx,
112
+ # where they would only be noise.
113
+ logger.warning(
114
+ "IGSN service returned %s for %s %s: %s",
115
+ response.status_code,
116
+ method,
117
+ path,
118
+ detail,
119
+ )
120
+ if response.status_code in CLIENT_ERROR_STATUSES:
121
+ message = f"IGSN registry rejected the request: {detail}"
122
+ else:
123
+ message = (
124
+ f"IGSN service returned {response.status_code} for "
125
+ f"{method} {path}: {detail}"
126
+ )
85
127
  raise IGSNServiceError(
86
- f"IGSN service returned {response.status_code} for "
87
- f"{method} {path}: {detail}",
128
+ message,
88
129
  status_code=response.status_code,
89
130
  detail=detail,
90
131
  )
@@ -10,6 +10,13 @@ answer briefly. The local settings remain the fallback for local mode and for
10
10
  when the service is momentarily unreachable: rejecting a prefix that was valid
11
11
  five minutes ago is worse than using a slightly stale vocabulary, since the
12
12
  service validates every allocation again anyway.
13
+
14
+ The registry's vocabulary is global but allocation is tenant-scoped, so the two
15
+ are not the same list. ``GET /vocabularies`` returns every institution the
16
+ registry knows; ``GET /whoami`` says which of them this instance's tenant may
17
+ allocate under. Without the intersection the form happily offers JH to a tenant
18
+ that only holds AB, and the refusal arrives after the metadata has been filled
19
+ in -- readable now, but still a dead end.
13
20
  """
14
21
 
15
22
  import logging
@@ -43,6 +50,41 @@ def invalidate():
43
50
  _cache["value"] = None
44
51
 
45
52
 
53
+ def _permitted(institutions, client):
54
+ """Narrow the registry's institutions to the ones this tenant may use.
55
+
56
+ An empty ``allowed_institutions`` means unrestricted -- the service reads it
57
+ that way too (``validate_prefix`` only checks when the list is non-empty),
58
+ and the two must agree or the form offers less than the registry accepts.
59
+
60
+ A ``whoami`` that fails is not worth discarding a good vocabulary over: the
61
+ unfiltered list is what was shown before, and the allocation is validated
62
+ again at the service anyway.
63
+ """
64
+ try:
65
+ allowed = set((client.whoami() or {}).get("allowed_institutions") or ())
66
+ except IGSNServiceError:
67
+ logger.exception("Could not read the tenant from the IGSN service")
68
+ return institutions
69
+
70
+ if not allowed:
71
+ return institutions
72
+
73
+ permitted = {
74
+ code: value for code, value in institutions.items() if code in allowed
75
+ }
76
+ if not permitted:
77
+ # Nothing the tenant is allowed to use exists in the vocabulary, so the
78
+ # dropdown will be empty. That is the honest answer -- every prefix
79
+ # would be refused -- but it is a misconfiguration, not a normal state.
80
+ logger.warning(
81
+ "Tenant is allowed institutions %s, none of which the IGSN "
82
+ "vocabulary defines",
83
+ sorted(allowed),
84
+ )
85
+ return permitted
86
+
87
+
46
88
  def get_vocabularies():
47
89
  """Return ``{"institutions": ..., "materials": ...}``."""
48
90
  client = None
@@ -62,7 +104,7 @@ def get_vocabularies():
62
104
  try:
63
105
  remote = client.vocabularies()
64
106
  value = {
65
- "institutions": remote["institutions"],
107
+ "institutions": _permitted(remote["institutions"], client),
66
108
  "materials": remote["materials"],
67
109
  }
68
110
  except (IGSNServiceError, KeyError):
@@ -350,7 +350,11 @@ class Deposition(Resource):
350
350
  except pymongo.errors.BulkWriteError as e:
351
351
  raise RestException(f"Error creating child depositions: {e.details}")
352
352
  except IGSNServiceError as e:
353
- raise RestException(f"IGSN registry rejected the batch: {e}")
353
+ # Re-raised only to name the operation; ``e`` already carries the
354
+ # registry's reason and the status it should surface as.
355
+ raise RestException(
356
+ f"IGSN registry rejected the batch: {e.detail or e}", code=e.code
357
+ )
354
358
  return DepositionModel().load(
355
359
  result.inserted_ids[0], user=self.getCurrentUser()
356
360
  )
@@ -0,0 +1,118 @@
1
+ """Pull IGSN registry state onto local depositions.
2
+
3
+ Publication is asynchronous the whole way down: Girder queues it with the
4
+ registry, the registry queues it with DataCite, and nothing pushes the outcome
5
+ back. ``publish_deposition`` polls for a while after queuing one, which covers
6
+ the ordinary case, but not a record published straight against the registry's
7
+ API, not one that settled while Girder was down, and not one that took longer
8
+ than that poll chain was willing to wait. This is the backstop for those, and
9
+ is meant to run on a timer::
10
+
11
+ */15 * * * * girder-jsonforms-reconcile-igsn --quiet
12
+
13
+ It runs the ``reconcile`` task in-process -- no broker, no worker -- against
14
+ whatever database the local Girder configuration points at, so it is also the
15
+ quickest way to repair one record by hand.
16
+
17
+ By default it only looks at depositions that have not settled locally, which is
18
+ what a timer is for and what keeps ``--limit`` meaningful: the scan has no
19
+ ordering, so a flat limit over every deposition would re-check the same first
20
+ thousand for ever. ``--all`` does the full pass, which is the one that also
21
+ reports depositions the registry has never heard of.
22
+
23
+ Usage::
24
+
25
+ girder-jsonforms-reconcile-igsn # unsettled records
26
+ girder-jsonforms-reconcile-igsn --all # full audit, to --limit
27
+ girder-jsonforms-reconcile-igsn --prefix JHAMAL
28
+ girder-jsonforms-reconcile-igsn --igsn JHAMAL00001 --igsn JHAMAL00002
29
+ girder-jsonforms-reconcile-igsn --quiet # summary only on trouble
30
+
31
+ Exits non-zero when a record could not be read from the registry, so a cron
32
+ run that hit a broken service says so. A deposition the registry has never
33
+ heard of is logged and counted, but is not a failure: it is a finding for a
34
+ human, and reconcile deliberately does not invent the missing record.
35
+ """
36
+
37
+ import argparse
38
+ import logging
39
+ import sys
40
+
41
+ logger = logging.getLogger("girder_jsonforms.reconcile_igsn")
42
+
43
+
44
+ def main(argv=None):
45
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
46
+ parser.add_argument(
47
+ "--prefix",
48
+ help="Only reconcile depositions whose IGSN starts with this prefix.",
49
+ )
50
+ parser.add_argument(
51
+ "--igsn",
52
+ action="append",
53
+ dest="igsns",
54
+ metavar="IGSN",
55
+ help="Reconcile one IGSN. Repeatable; overrides --prefix.",
56
+ )
57
+ parser.add_argument(
58
+ "--all",
59
+ action="store_true",
60
+ dest="check_all",
61
+ help=(
62
+ "Also re-check records that have already settled locally, and "
63
+ "report any the registry does not have."
64
+ ),
65
+ )
66
+ parser.add_argument(
67
+ "--limit",
68
+ type=int,
69
+ default=1000,
70
+ help="Maximum number of depositions to check (default: 1000).",
71
+ )
72
+ parser.add_argument(
73
+ "-q",
74
+ "--quiet",
75
+ action="store_true",
76
+ help="Log warnings and errors only -- a quiet cron run stays quiet.",
77
+ )
78
+ parser.add_argument("-v", "--verbose", action="store_true", help="Debug logging.")
79
+ args = parser.parse_args(argv)
80
+
81
+ level = logging.INFO
82
+ if args.quiet:
83
+ level = logging.WARNING
84
+ if args.verbose:
85
+ level = logging.DEBUG
86
+ logging.basicConfig(level=level, format="%(levelname)s %(message)s")
87
+
88
+ # Imported here so ``--help`` works without a configured Girder/DB.
89
+ from girder_jsonforms.worker_plugin.igsn_registry import reconcile
90
+
91
+ summary = reconcile(
92
+ prefix=args.prefix,
93
+ igsns=args.igsns,
94
+ limit=args.limit,
95
+ pending_only=not args.check_all,
96
+ )
97
+ if summary is None:
98
+ # Local mode. Not an error: an instance that allocates its own IGSNs has
99
+ # no registry to reconcile against, and a cron entry left behind after
100
+ # switching modes should not start failing.
101
+ logger.warning("No IGSN registry is configured; nothing to reconcile.")
102
+ return 0
103
+
104
+ logger.info(
105
+ "Reconciled: %(updated)s updated, %(unchanged)s unchanged, "
106
+ "%(missing)s missing, %(errors)s error(s).",
107
+ summary,
108
+ )
109
+ if summary["missing"]:
110
+ logger.warning(
111
+ "%s deposition(s) are not in the registry; see the lines above.",
112
+ summary["missing"],
113
+ )
114
+ return 1 if summary["errors"] else 0
115
+
116
+
117
+ if __name__ == "__main__":
118
+ sys.exit(main())
@@ -6,7 +6,16 @@ metadata with raw ``collection.update_*`` -- ``Deposition.updateRelations``,
6
6
  so never reach ``Model.save()``. Each of those has to push explicitly, and doing
7
7
  it inline would put an HTTP round-trip in the middle of a form submission.
8
8
 
9
- All three tasks are no-ops when ``jsonforms.igsn_service_url`` is unset, so
9
+ Publication state comes back the same way, by polling. ``POST /records/<igsn>/
10
+ publish`` only queues an attempt on the service, which talks to DataCite later
11
+ and updates its own record; there is no callback. So ``publish_deposition``
12
+ hands the deposition to ``refresh_publication_state``, which checks until the
13
+ record settles. ``reconcile`` is the backstop for everything that never went
14
+ through here -- a record published against the registry's API directly, or one
15
+ that settled while Girder was down -- and is meant to be run on a timer
16
+ (``girder-jsonforms-reconcile-igsn``).
17
+
18
+ All of these tasks are no-ops when ``jsonforms.igsn_service_url`` is unset, so
10
19
  enqueueing them is harmless on an instance still in local mode.
11
20
  """
12
21
 
@@ -20,6 +29,19 @@ from ..models.deposition import Deposition
20
29
 
21
30
  logger = logging.getLogger(__name__)
22
31
 
32
+ # Statuses the registry will not leave on its own. Anything else means the
33
+ # publication is still in flight, or being retried against DataCite.
34
+ TERMINAL_STATUSES = frozenset({"registered", "findable", "failed", "revoked"})
35
+
36
+ # Seconds to wait before each successive check after a publication is queued.
37
+ # DataCite usually answers within seconds; the tail covers the service backing
38
+ # off through an outage. The chain gives up after ~18 minutes rather than
39
+ # polling forever -- past that it is reconcile's problem, and a record that has
40
+ # not settled in eighteen minutes needs a human looking at its attempts. Each
41
+ # delay stays well under Redis's default one-hour visibility timeout, beyond
42
+ # which a countdown'd task gets redelivered to a second worker.
43
+ REFRESH_DELAYS = (5, 15, 45, 120, 300, 600)
44
+
23
45
 
24
46
  @app.task(queue="local")
25
47
  def sync_deposition_metadata(depositionId):
@@ -75,13 +97,99 @@ def publish_deposition(depositionId, target="findable", recurse=False, metadata_
75
97
  target,
76
98
  result.get("queued"),
77
99
  )
78
- # Publication is asynchronous on the service side too, so the local mirror
79
- # is refreshed by `reconcile` once DataCite has actually accepted it.
100
+ # Publication is asynchronous on the service side too, and nothing pushes
101
+ # the outcome back, so the mirror is worthless unless something asks. The
102
+ # children go in as well: a recursive publish settles each of them
103
+ # separately.
104
+ ids = [deposition["_id"]]
105
+ if recurse:
106
+ ids += [
107
+ child["_id"]
108
+ for child in Deposition().collection.find(
109
+ {"parentId": deposition["_id"]}, {"_id": 1}
110
+ )
111
+ ]
112
+ schedule_refresh(ids)
80
113
  return result
81
114
 
82
115
 
116
+ def schedule_refresh(deposition_ids, attempt=0):
117
+ """Queue a status check for depositions whose publication is in flight.
118
+
119
+ Separate from the task so the delay schedule lives in one place and the
120
+ chain has a defined end: past the last delay this logs and stops rather
121
+ than retrying forever.
122
+ """
123
+ ids = [str(_id) for _id in deposition_ids if _id]
124
+ if not ids:
125
+ return None
126
+ if attempt >= len(REFRESH_DELAYS):
127
+ logger.warning(
128
+ "%s still not settled after %s checks; leaving it to reconcile",
129
+ ", ".join(ids),
130
+ attempt,
131
+ )
132
+ return None
133
+ return refresh_publication_state.apply_async(
134
+ (ids, attempt),
135
+ countdown=REFRESH_DELAYS[attempt],
136
+ # No girder job per check: six of them per publish would bury the job
137
+ # that actually did something.
138
+ girder_job_disable=True,
139
+ )
140
+
141
+
142
+ @app.task(queue="local")
143
+ def refresh_publication_state(depositionIds, attempt=0):
144
+ """Mirror the registry's status onto depositions mid-publication.
145
+
146
+ Re-queues itself for whatever is still pending, so a record that takes a
147
+ DataCite retry or two to settle still lands locally. A record the registry
148
+ does not have is dropped from the chain, not retried: that is a reconcile
149
+ finding, and polling will not conjure it.
150
+ """
151
+ client = get_client()
152
+ if client is None:
153
+ return None
154
+
155
+ pending = []
156
+ summary = {"settled": 0, "pending": 0, "missing": 0, "errors": 0}
157
+ for depositionId in depositionIds:
158
+ deposition = Deposition().load(depositionId, force=True)
159
+ if deposition is None:
160
+ logger.warning("Deposition %s no longer exists", depositionId)
161
+ continue
162
+ try:
163
+ record = client.get_record(deposition["igsn"])
164
+ except IGSNServiceError:
165
+ logger.exception(
166
+ "Could not read %s from the registry", deposition["igsn"]
167
+ )
168
+ summary["errors"] += 1
169
+ pending.append(depositionId)
170
+ continue
171
+
172
+ if record is None:
173
+ logger.warning(
174
+ "%s exists in Girder but not in the IGSN registry",
175
+ deposition["igsn"],
176
+ )
177
+ summary["missing"] += 1
178
+ continue
179
+
180
+ Deposition().apply_registry_state(deposition["_id"], record)
181
+ if record.get("status") in TERMINAL_STATUSES:
182
+ summary["settled"] += 1
183
+ else:
184
+ summary["pending"] += 1
185
+ pending.append(depositionId)
186
+
187
+ schedule_refresh(pending, attempt + 1)
188
+ return summary
189
+
190
+
83
191
  @app.task(queue="local")
84
- def reconcile(prefix=None, igsns=None, limit=1000):
192
+ def reconcile(prefix=None, igsns=None, limit=1000, pending_only=False):
85
193
  """Pull registry state onto local depositions and report divergence.
86
194
 
87
195
  Two kinds of drift are possible and only one is fixable here:
@@ -91,6 +199,13 @@ def reconcile(prefix=None, igsns=None, limit=1000):
91
199
  * missing records -- a deposition exists locally with no registry record at
92
200
  all. Reported, not invented: creating one would mean guessing which tenant
93
201
  and prefix it belongs to, and a duplicate IGSN is worse than a warning.
202
+
203
+ ``pending_only`` skips depositions already sitting on a status the registry
204
+ will not move off by itself. That is one request per deposition saved on a
205
+ run that is only chasing publications, and it is what makes ``limit``
206
+ honest on a large instance: the scan has no ordering, so a flat limit over
207
+ every deposition would check the same first N for ever. It gives up the
208
+ missing-record audit in exchange, which wants the full pass.
94
209
  """
95
210
  client = get_client()
96
211
  if client is None:
@@ -102,6 +217,10 @@ def reconcile(prefix=None, igsns=None, limit=1000):
102
217
  query["igsn"] = {"$in": list(igsns)}
103
218
  elif prefix:
104
219
  query["igsn"] = {"$regex": f"^{prefix}"}
220
+ if pending_only:
221
+ # Matches a missing serviceStatus too, which is what every deposition
222
+ # minted before the mirror existed has.
223
+ query["serviceStatus"] = {"$nin": list(TERMINAL_STATUSES)}
105
224
 
106
225
  updated = missing = unchanged = errors = 0
107
226
  cursor = Deposition().collection.find(
@@ -1,11 +1,10 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: girder-jsonforms
3
- Version: 2.1.2
3
+ Version: 2.1.3
4
4
  Summary: Girder plugin adding forms based on JSON-editor
5
- License: BSD-3-Clause
5
+ License-Expression: BSD-3-Clause
6
6
  Classifier: Development Status :: 4 - Beta
7
7
  Classifier: Environment :: Web Environment
8
- Classifier: License :: OSI Approved :: BSD License
9
8
  Classifier: Operating System :: POSIX :: Linux
10
9
  Classifier: Programming Language :: Python
11
10
  Classifier: Programming Language :: Python :: 3
@@ -23,14 +22,7 @@ Requires-Dist: google-auth-oauthlib
23
22
  Requires-Dist: jsondiff
24
23
  Requires-Dist: pandas
25
24
  Requires-Dist: openpyxl
26
- Dynamic: classifier
27
- Dynamic: description
28
- Dynamic: description-content-type
29
- Dynamic: license
30
25
  Dynamic: license-file
31
- Dynamic: requires-dist
32
- Dynamic: requires-python
33
- Dynamic: summary
34
26
 
35
27
  ### Girder JSONForms Plugin
36
28
 
@@ -1,8 +1,6 @@
1
- .gitignore
2
1
  LICENSE
3
2
  README.md
4
- setup.py
5
- .github/workflows/release.yml
3
+ pyproject.toml
6
4
  doc/file-uploads.md
7
5
  doc/preload.md
8
6
  girder_jsonforms/__init__.py
@@ -12,7 +10,6 @@ girder_jsonforms.egg-info/PKG-INFO
12
10
  girder_jsonforms.egg-info/SOURCES.txt
13
11
  girder_jsonforms.egg-info/dependency_links.txt
14
12
  girder_jsonforms.egg-info/entry_points.txt
15
- girder_jsonforms.egg-info/not-zip-safe
16
13
  girder_jsonforms.egg-info/requires.txt
17
14
  girder_jsonforms.egg-info/top_level.txt
18
15
  girder_jsonforms/lib/__init__.py
@@ -54,6 +51,7 @@ girder_jsonforms/schemas/datacite-v4.5.json
54
51
  girder_jsonforms/scripts/__init__.py
55
52
  girder_jsonforms/scripts/migrate_metadata_dates.py
56
53
  girder_jsonforms/scripts/preview_mail.py
54
+ girder_jsonforms/scripts/reconcile_igsn.py
57
55
  girder_jsonforms/web_client/dist/girder-plugin-jsonforms.umd.cjs
58
56
  girder_jsonforms/web_client/dist/style.css
59
57
  girder_jsonforms/worker_plugin/__init__.py
@@ -1,6 +1,7 @@
1
1
  [console_scripts]
2
2
  girder-jsonforms-migrate-metadata-dates = girder_jsonforms.scripts.migrate_metadata_dates:main
3
3
  girder-jsonforms-preview-mail = girder_jsonforms.scripts.preview_mail:main
4
+ girder-jsonforms-reconcile-igsn = girder_jsonforms.scripts.reconcile_igsn:main
4
5
 
5
6
  [girder.plugin]
6
7
  jsonforms = girder_jsonforms:JSONFormsPlugin
@@ -0,0 +1,75 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "girder-jsonforms"
7
+ version = "2.1.3"
8
+ description = "Girder plugin adding forms based on JSON-editor"
9
+ readme = "README.md"
10
+ license = "BSD-3-Clause"
11
+ license-files = ["LICENSE"]
12
+ requires-python = ">=3.10"
13
+ classifiers = [
14
+ "Development Status :: 4 - Beta",
15
+ "Environment :: Web Environment",
16
+ "Operating System :: POSIX :: Linux",
17
+ "Programming Language :: Python",
18
+ "Programming Language :: Python :: 3",
19
+ ]
20
+ dependencies = [
21
+ "girder>=5.0.18",
22
+ "girder-worker>=5.0.18",
23
+ "girder-wholetale>=2.0.13",
24
+ "girder-jobs>=5.0.18",
25
+ "girder-sample-tracker>=2.0.0", # Required for the sample tracker to test JSONForms
26
+ "girder-oauth>=5.0.18",
27
+ "google-api-python-client",
28
+ "google-auth-oauthlib",
29
+ "jsondiff",
30
+ "pandas",
31
+ "openpyxl",
32
+ ]
33
+
34
+ [project.scripts]
35
+ girder-jsonforms-migrate-metadata-dates = "girder_jsonforms.scripts.migrate_metadata_dates:main"
36
+ girder-jsonforms-preview-mail = "girder_jsonforms.scripts.preview_mail:main"
37
+ girder-jsonforms-reconcile-igsn = "girder_jsonforms.scripts.reconcile_igsn:main"
38
+
39
+ [project.entry-points."girder.plugin"]
40
+ jsonforms = "girder_jsonforms:JSONFormsPlugin"
41
+
42
+ [project.entry-points."girder_worker_plugins"]
43
+ jsonforms = "girder_jsonforms.worker_plugin:JSONFormsWorkerPlugin"
44
+
45
+ [dependency-groups]
46
+ dev = [
47
+ "pytest",
48
+ "pytest-cov",
49
+ "pytest-mock",
50
+ "pytest-girder>=5.0.0a5.dev0",
51
+ "pytest-xdist",
52
+ "responses",
53
+ ]
54
+
55
+ # Pinned to the package itself: a bare find_packages() also walks build/ and
56
+ # .eggs/ when they are lying around.
57
+ [tool.setuptools.packages.find]
58
+ include = ["girder_jsonforms*"]
59
+
60
+ # MANIFEST.in alone covers the sdist; package-data is what puts these in
61
+ # the wheel, which is how the plugin actually gets installed.
62
+ [tool.setuptools.package-data]
63
+ girder_jsonforms = [
64
+ "mail_templates/*.mako",
65
+ "mail_assets/*.png",
66
+ "schemas/*.json",
67
+ ]
68
+
69
+ [tool.setuptools.data-files]
70
+ schemas = ["girder_jsonforms/schemas/datacite-v4.5.json"]
71
+
72
+ [tool.pytest.ini_options]
73
+ cache_dir = "build/test/pytest_cache"
74
+ junit_family = "xunit2"
75
+ testpaths = ["girder_jsonforms/tests"]
@@ -1,102 +0,0 @@
1
- name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
2
-
3
- on: push
4
-
5
- jobs:
6
- build:
7
- name: Build distribution 📦
8
- runs-on: ubuntu-latest
9
-
10
- steps:
11
- - uses: actions/checkout@v7
12
- - name: Set up Python
13
- uses: actions/setup-python@v7
14
- with:
15
- python-version: "3.x"
16
- - name: Set up Node.js
17
- uses: actions/setup-node@v7
18
- with:
19
- node-version: 24
20
- - name: Install pypa/build
21
- run: >-
22
- python3 -m
23
- pip install
24
- build
25
- --user
26
- - name: Build js component in package/web_client
27
- run: >-
28
- cd $(find . -maxdepth 2 -mindepth 2 -type d -name web_client)
29
- && npm ci
30
- && npm run build
31
- - name: Build a binary wheel and a source tarball
32
- run: python3 -m build
33
- - name: install check-manifest
34
- run: >-
35
- python3 -m
36
- pip install
37
- check-manifest
38
- --user
39
- - name: Check the package manifest
40
- run: check-manifest -vvv
41
- - name: Store the distribution packages
42
- uses: actions/upload-artifact@v7
43
- with:
44
- name: python-package-distributions
45
- path: dist/
46
-
47
- publish-to-pypi:
48
- name: >-
49
- Publish Python 🐍 distribution 📦 to PyPI
50
- if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes
51
- needs:
52
- - build
53
- runs-on: ubuntu-latest
54
- environment:
55
- name: pypi
56
- url: https://pypi.org/p/girder-jsonforms
57
- permissions:
58
- id-token: write # IMPORTANT: mandatory for trusted publishing
59
-
60
- steps:
61
- - name: Download all the dists
62
- uses: actions/download-artifact@v7
63
- with:
64
- name: python-package-distributions
65
- path: dist/
66
- - name: Publish distribution 📦 to PyPI
67
- uses: pypa/gh-action-pypi-publish@release/v1
68
-
69
- github-release:
70
- name: >-
71
- Sign the Python 🐍 distribution 📦 with Sigstore
72
- and upload them to GitHub Release
73
- needs:
74
- - publish-to-pypi
75
- runs-on: ubuntu-latest
76
-
77
- permissions:
78
- contents: write # IMPORTANT: mandatory for making GitHub Releases
79
- id-token: write # IMPORTANT: mandatory for sigstore
80
-
81
- steps:
82
- - name: Download all the dists
83
- uses: actions/download-artifact@v7
84
- with:
85
- name: python-package-distributions
86
- path: dist/
87
- - name: Sign the dists with Sigstore
88
- uses: sigstore/gh-action-sigstore-python@v3.5.0
89
- with:
90
- inputs: >-
91
- ./dist/*.tar.gz
92
- ./dist/*.whl
93
- - name: Upload artifact signatures to GitHub Release
94
- env:
95
- GITHUB_TOKEN: ${{ github.token }}
96
- # Upload to GitHub Release using the `gh` CLI.
97
- # `dist/` contains the built packages, and the
98
- # sigstore-produced signatures and certificates.
99
- run: >-
100
- gh release upload
101
- '${{ github.ref_name }}' dist/**
102
- --repo '${{ github.repository }}'
@@ -1,136 +0,0 @@
1
- Testing
2
-
3
- *ala*
4
-
5
- # Byte-compiled / optimized / DLL files
6
- __pycache__/
7
- *.py[cod]
8
- *$py.class
9
-
10
- # C extensions
11
- *.so
12
-
13
- # Distribution / packaging
14
- .Python
15
- build/
16
- develop-eggs/
17
- dist/
18
- downloads/
19
- eggs/
20
- .eggs/
21
- parts/
22
- sdist/
23
- var/
24
- wheels/
25
- pip-wheel-metadata/
26
- share/python-wheels/
27
- *.egg-info/
28
- .installed.cfg
29
- *.egg
30
- MANIFEST
31
-
32
- # PyInstaller
33
- # Usually these files are written by a python script from a template
34
- # before PyInstaller builds the exe, so as to inject date/other infos into it.
35
- *.manifest
36
- *.spec
37
-
38
- # Installer logs
39
- pip-log.txt
40
- pip-delete-this-directory.txt
41
-
42
- # Unit test / coverage reports
43
- htmlcov/
44
- .tox/
45
- .nox/
46
- .coverage
47
- .coverage.*
48
- .cache
49
- nosetests.xml
50
- coverage.xml
51
- *.cover
52
- *.py,cover
53
- .hypothesis/
54
- .pytest_cache/
55
-
56
- # Translations
57
- *.mo
58
- *.pot
59
-
60
- # Django stuff:
61
- *.log
62
- local_settings.py
63
- db.sqlite3
64
- db.sqlite3-journal
65
-
66
- # Flask stuff:
67
- instance/
68
- .webassets-cache
69
-
70
- # Scrapy stuff:
71
- .scrapy
72
-
73
- # Sphinx documentation
74
- docs/_build/
75
-
76
- # PyBuilder
77
- target/
78
-
79
- # Jupyter Notebook
80
- .ipynb_checkpoints
81
-
82
- # IPython
83
- profile_default/
84
- ipython_config.py
85
-
86
- # pyenv
87
- .python-version
88
-
89
- # pipenv
90
- # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
91
- # However, in case of collaboration, if having platform-specific dependencies or dependencies
92
- # having no cross-platform support, pipenv may install dependencies that don't work, or not
93
- # install all needed dependencies.
94
- #Pipfile.lock
95
-
96
- # PEP 582; used by e.g. github.com/David-OConnor/pyflow
97
- __pypackages__/
98
-
99
- # Celery stuff
100
- celerybeat-schedule
101
- celerybeat.pid
102
-
103
- # SageMath parsed files
104
- *.sage.py
105
-
106
- # Environments
107
- .env
108
- .venv
109
- env/
110
- venv/
111
- ENV/
112
- env.bak/
113
- venv.bak/
114
-
115
- # Spyder project settings
116
- .spyderproject
117
- .spyproject
118
-
119
- # Rope project settings
120
- .ropeproject
121
-
122
- # mkdocs documentation
123
- /site
124
-
125
- # mypy
126
- .mypy_cache/
127
- .dmypy.json
128
- dmypy.json
129
-
130
- # Pyre type checker
131
- .pyre/
132
-
133
- node_modules/
134
-
135
- # Local email template previews
136
- mail-preview/
@@ -1,67 +0,0 @@
1
- from pathlib import Path
2
-
3
- from setuptools import find_packages, setup
4
-
5
- this_directory = Path(__file__).parent
6
- long_description = (this_directory / "README.md").read_text()
7
-
8
- girder_version = "5.0.18"
9
-
10
- setup(
11
- name="girder-jsonforms",
12
- long_description=long_description,
13
- long_description_content_type="text/markdown",
14
- version="2.1.2",
15
- description="Girder plugin adding forms based on JSON-editor",
16
- packages=find_packages(),
17
- data_files=[
18
- ("schemas", ["girder_jsonforms/schemas/datacite-v4.5.json"]),
19
- ],
20
- # MANIFEST.in alone covers the sdist; package_data is what puts these in
21
- # the wheel, which is how the plugin actually gets installed.
22
- package_data={
23
- "girder_jsonforms": [
24
- "mail_templates/*.mako",
25
- "mail_assets/*.png",
26
- "schemas/*.json",
27
- ],
28
- },
29
- include_package_data=True,
30
- license="BSD-3-Clause",
31
- classifiers=[
32
- "Development Status :: 4 - Beta",
33
- "Environment :: Web Environment",
34
- "License :: OSI Approved :: BSD License",
35
- "Operating System :: POSIX :: Linux",
36
- "Programming Language :: Python",
37
- "Programming Language :: Python :: 3",
38
- ],
39
- python_requires=">=3.10",
40
- setup_requires=["setuptools-git"],
41
- install_requires=[
42
- f"girder>={girder_version}",
43
- f"girder-worker>={girder_version}",
44
- "girder-wholetale>=2.0.13",
45
- f"girder-jobs>={girder_version}",
46
- "girder-sample-tracker>=2.0.0", # Required for the sample tracker to test JSONForms
47
- f"girder-oauth>={girder_version}",
48
- "google-api-python-client",
49
- "google-auth-oauthlib",
50
- "jsondiff",
51
- "pandas",
52
- "openpyxl",
53
- ],
54
- entry_points={
55
- "console_scripts": [
56
- "girder-jsonforms-migrate-metadata-dates = "
57
- "girder_jsonforms.scripts.migrate_metadata_dates:main",
58
- "girder-jsonforms-preview-mail = "
59
- "girder_jsonforms.scripts.preview_mail:main",
60
- ],
61
- "girder.plugin": ["jsonforms = girder_jsonforms:JSONFormsPlugin"],
62
- "girder_worker_plugins": [
63
- "jsonforms = girder_jsonforms.worker_plugin:JSONFormsWorkerPlugin"
64
- ],
65
- },
66
- zip_safe=False,
67
- )