dcor-shared 0.11.4__tar.gz → 0.12.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/CHANGELOG +4 -0
- {dcor_shared-0.11.4/dcor_shared.egg-info → dcor_shared-0.12.0}/PKG-INFO +1 -1
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/__init__.py +1 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/_version.py +2 -2
- dcor_shared-0.12.0/dcor_shared/rqjob.py +108 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/s3.py +1 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0/dcor_shared.egg-info}/PKG-INFO +1 -1
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared.egg-info/SOURCES.txt +1 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/.github/workflows/check.yml +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/.github/workflows/deploy_pypi.yml +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/.gitignore +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/LICENSE +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/MANIFEST.in +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/README.rst +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/Vagrantfile +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/ckan.py +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/data.py +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/dcinst.py +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/mime.py +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/parse.py +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/paths.py +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/s3cc.py +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/testing.py +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared.egg-info/dependency_links.txt +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared.egg-info/requires.txt +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared.egg-info/top_level.txt +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/pyproject.toml +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/setup.cfg +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/data/calibration_beads_47.rtdc +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/requirements.txt +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/test_basic.py +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/test_ckan.py +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/test_data.py +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/test_s3.py +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/test_s3cc.py +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/test_testing.py +0 -0
- {dcor_shared-0.11.4 → dcor_shared-0.12.0}/vagrant-run-tests.sh +0 -0
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import collections
|
|
2
|
+
from typing import Callable, Dict, List
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class RQJob:
|
|
6
|
+
"""Helper class for managing CKAN background jobs on DCOR
|
|
7
|
+
|
|
8
|
+
Instead of manually enqueueing jobs in plugin.py, you can use
|
|
9
|
+
this class in combination with the decorator `rqjob_register`
|
|
10
|
+
below to run all jobs of your extension.
|
|
11
|
+
|
|
12
|
+
1. Decorate each job function in your jobs.py with `@rqjob_register`,
|
|
13
|
+
optionally setting keyword arguments for `RQJob`. Make sure to
|
|
14
|
+
import `RQJob` in your jobs.py as well.
|
|
15
|
+
2. In your plugin.py, import your `jobs.py` (to make sure that
|
|
16
|
+
all methods are registered) and then in `after_resource_create`
|
|
17
|
+
run `jobs.RQJob.enqueue_all_jobs(resource, ckanext="extension_name")`
|
|
18
|
+
"""
|
|
19
|
+
_instances = []
|
|
20
|
+
|
|
21
|
+
def __init__(self,
|
|
22
|
+
method: Callable,
|
|
23
|
+
ckanext: str,
|
|
24
|
+
queue: str = "default",
|
|
25
|
+
timeout: int = 60,
|
|
26
|
+
depends_on: List[str] = None,
|
|
27
|
+
at_front: bool = False,
|
|
28
|
+
):
|
|
29
|
+
self.method = method
|
|
30
|
+
self.name = method.__name__
|
|
31
|
+
self.title = method.__doc__.split("\n")[0].strip()
|
|
32
|
+
self.ckanext = ckanext
|
|
33
|
+
self.depends_on = depends_on or []
|
|
34
|
+
self.timeout = timeout
|
|
35
|
+
self.at_front = at_front
|
|
36
|
+
self.queue = queue
|
|
37
|
+
self._instances.append(self)
|
|
38
|
+
|
|
39
|
+
def enqueue_job(self, resource: Dict, redis_connect=None):
|
|
40
|
+
# Import here so module can be loaded without ckan available
|
|
41
|
+
import ckan.plugins.toolkit as toolkit
|
|
42
|
+
from ckan.lib.jobs import _connect as ckan_redis_connect
|
|
43
|
+
from rq.job import Job
|
|
44
|
+
|
|
45
|
+
if redis_connect is None:
|
|
46
|
+
redis_connect = ckan_redis_connect()
|
|
47
|
+
|
|
48
|
+
jid = f"{resource['package_id']}_{resource['position']}_"
|
|
49
|
+
jid_cur = jid + self.name
|
|
50
|
+
rq_args = {
|
|
51
|
+
"timeout": self.timeout,
|
|
52
|
+
"at_front": self.at_front,
|
|
53
|
+
"job_id": jid_cur,
|
|
54
|
+
}
|
|
55
|
+
if self.depends_on:
|
|
56
|
+
rq_args["depends_on"] = [f"{jid}_{dep}" for dep in self.depends_on]
|
|
57
|
+
|
|
58
|
+
if not Job.exists(jid_cur, connection=redis_connect):
|
|
59
|
+
toolkit.enqueue_job(self.method,
|
|
60
|
+
[resource],
|
|
61
|
+
title=self.title,
|
|
62
|
+
queue=self.queue,
|
|
63
|
+
rq_kwargs=rq_args)
|
|
64
|
+
|
|
65
|
+
@classmethod
|
|
66
|
+
def enqueue_all_jobs(cls,
|
|
67
|
+
resource: Dict,
|
|
68
|
+
ckanext: str):
|
|
69
|
+
from ckan.lib.jobs import _connect as ckan_redis_connect
|
|
70
|
+
redis_connect = ckan_redis_connect()
|
|
71
|
+
|
|
72
|
+
for inst in cls._instances:
|
|
73
|
+
if inst.ckanext == ckanext:
|
|
74
|
+
inst.enqueue_job(resource, redis_connect=redis_connect)
|
|
75
|
+
|
|
76
|
+
@classmethod
|
|
77
|
+
def get_all_job_methods_in_order(cls, ckanext: str):
|
|
78
|
+
job_dict = {}
|
|
79
|
+
for job in cls._instances:
|
|
80
|
+
if job.ckanext == ckanext:
|
|
81
|
+
job_dict[job.name] = job
|
|
82
|
+
|
|
83
|
+
# Order the jobs according to depends_on
|
|
84
|
+
job_dict_ordered = collections.OrderedDict()
|
|
85
|
+
for ii in range(len(job_dict)):
|
|
86
|
+
for key in list(job_dict.keys()):
|
|
87
|
+
job = job_dict[key]
|
|
88
|
+
if set(job.depends_on) <= set(job_dict_ordered.keys()):
|
|
89
|
+
# All jobs this job depends on are in the ordered list
|
|
90
|
+
job_dict_ordered[key] = job
|
|
91
|
+
job_dict.pop(key)
|
|
92
|
+
|
|
93
|
+
if job_dict:
|
|
94
|
+
raise NotImplementedError(
|
|
95
|
+
f"The following jobs have unmet dependencies and will not "
|
|
96
|
+
f"be returned: {list(job_dict.keys())}")
|
|
97
|
+
|
|
98
|
+
return list(job_dict_ordered.values())
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class rqjob_register:
|
|
102
|
+
"""A decorator for background jobs to register them with RQJob"""
|
|
103
|
+
def __init__(self, **kwargs):
|
|
104
|
+
self.kwargs = kwargs
|
|
105
|
+
|
|
106
|
+
def __call__(self, func):
|
|
107
|
+
RQJob(method=func, **self.kwargs)
|
|
108
|
+
return func
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|