dcor-shared 0.11.4__tar.gz → 0.12.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/CHANGELOG +4 -0
  2. {dcor_shared-0.11.4/dcor_shared.egg-info → dcor_shared-0.12.0}/PKG-INFO +1 -1
  3. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/__init__.py +1 -0
  4. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/_version.py +2 -2
  5. dcor_shared-0.12.0/dcor_shared/rqjob.py +108 -0
  6. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/s3.py +1 -0
  7. {dcor_shared-0.11.4 → dcor_shared-0.12.0/dcor_shared.egg-info}/PKG-INFO +1 -1
  8. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared.egg-info/SOURCES.txt +1 -0
  9. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/.github/workflows/check.yml +0 -0
  10. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/.github/workflows/deploy_pypi.yml +0 -0
  11. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/.gitignore +0 -0
  12. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/LICENSE +0 -0
  13. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/MANIFEST.in +0 -0
  14. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/README.rst +0 -0
  15. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/Vagrantfile +0 -0
  16. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/ckan.py +0 -0
  17. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/data.py +0 -0
  18. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/dcinst.py +0 -0
  19. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/mime.py +0 -0
  20. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/parse.py +0 -0
  21. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/paths.py +0 -0
  22. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/s3cc.py +0 -0
  23. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared/testing.py +0 -0
  24. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared.egg-info/dependency_links.txt +0 -0
  25. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared.egg-info/requires.txt +0 -0
  26. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/dcor_shared.egg-info/top_level.txt +0 -0
  27. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/pyproject.toml +0 -0
  28. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/setup.cfg +0 -0
  29. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/data/calibration_beads_47.rtdc +0 -0
  30. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/requirements.txt +0 -0
  31. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/test_basic.py +0 -0
  32. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/test_ckan.py +0 -0
  33. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/test_data.py +0 -0
  34. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/test_s3.py +0 -0
  35. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/test_s3cc.py +0 -0
  36. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/tests/test_testing.py +0 -0
  37. {dcor_shared-0.11.4 → dcor_shared-0.12.0}/vagrant-run-tests.sh +0 -0
@@ -1,3 +1,7 @@
1
+ 0.12.0
2
+ - feat: introduce rqjob submodule for managing CKAN background jobs
3
+ 0.11.5
4
+ - ref: add minor caching
1
5
  0.11.4
2
6
  - ref: wait longer in wait_for_resource
3
7
  0.11.3
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcor_shared
3
- Version: 0.11.4
3
+ Version: 0.12.0
4
4
  Summary: Functionalities shared by the DCOR CKAN extensions
5
5
  Author: Paul Müller
6
6
  Maintainer-email: Paul Müller <dev@craban.de>
@@ -6,4 +6,5 @@ from .data import DUMMY_BYTES, sha256sum, wait_for_resource
6
6
  from .dcinst import get_dc_instance
7
7
  from .mime import DC_MIME_TYPES, VALID_FORMATS
8
8
  from . import paths
9
+ from .rqjob import RQJob, rqjob_register
9
10
  from ._version import version as __version__
@@ -12,5 +12,5 @@ __version__: str
12
12
  __version_tuple__: VERSION_TUPLE
13
13
  version_tuple: VERSION_TUPLE
14
14
 
15
- __version__ = version = '0.11.4'
16
- __version_tuple__ = version_tuple = (0, 11, 4)
15
+ __version__ = version = '0.12.0'
16
+ __version_tuple__ = version_tuple = (0, 12, 0)
@@ -0,0 +1,108 @@
1
+ import collections
2
+ from typing import Callable, Dict, List
3
+
4
+
5
+ class RQJob:
6
+ """Helper class for managing CKAN background jobs on DCOR
7
+
8
+ Instead of manually enqueueing jobs in plugin.py, you can use
9
+ this class in combination with the decorator `rqjob_register`
10
+ below to run all jobs of your extension.
11
+
12
+ 1. Decorate each job function in your jobs.py with `@rqjob_register`,
13
+ optionally setting keyword arguments for `RQJob`. Make sure to
14
+ import `RQJob` in your jobs.py as well.
15
+ 2. In your plugin.py, import your `jobs.py` (to make sure that
16
+ all methods are registered) and then in `after_resource_create`
17
+ run `jobs.RQJob.enqueue_all_jobs(resource, ckanext="extension_name")`
18
+ """
19
+ _instances = []
20
+
21
+ def __init__(self,
22
+ method: Callable,
23
+ ckanext: str,
24
+ queue: str = "default",
25
+ timeout: int = 60,
26
+ depends_on: List[str] = None,
27
+ at_front: bool = False,
28
+ ):
29
+ self.method = method
30
+ self.name = method.__name__
31
+ self.title = method.__doc__.split("\n")[0].strip()
32
+ self.ckanext = ckanext
33
+ self.depends_on = depends_on or []
34
+ self.timeout = timeout
35
+ self.at_front = at_front
36
+ self.queue = queue
37
+ self._instances.append(self)
38
+
39
+ def enqueue_job(self, resource: Dict, redis_connect=None):
40
+ # Import here so module can be loaded without ckan available
41
+ import ckan.plugins.toolkit as toolkit
42
+ from ckan.lib.jobs import _connect as ckan_redis_connect
43
+ from rq.job import Job
44
+
45
+ if redis_connect is None:
46
+ redis_connect = ckan_redis_connect()
47
+
48
+ jid = f"{resource['package_id']}_{resource['position']}_"
49
+ jid_cur = jid + self.name
50
+ rq_args = {
51
+ "timeout": self.timeout,
52
+ "at_front": self.at_front,
53
+ "job_id": jid_cur,
54
+ }
55
+ if self.depends_on:
56
+ rq_args["depends_on"] = [f"{jid}_{dep}" for dep in self.depends_on]
57
+
58
+ if not Job.exists(jid_cur, connection=redis_connect):
59
+ toolkit.enqueue_job(self.method,
60
+ [resource],
61
+ title=self.title,
62
+ queue=self.queue,
63
+ rq_kwargs=rq_args)
64
+
65
+ @classmethod
66
+ def enqueue_all_jobs(cls,
67
+ resource: Dict,
68
+ ckanext: str):
69
+ from ckan.lib.jobs import _connect as ckan_redis_connect
70
+ redis_connect = ckan_redis_connect()
71
+
72
+ for inst in cls._instances:
73
+ if inst.ckanext == ckanext:
74
+ inst.enqueue_job(resource, redis_connect=redis_connect)
75
+
76
+ @classmethod
77
+ def get_all_job_methods_in_order(cls, ckanext: str):
78
+ job_dict = {}
79
+ for job in cls._instances:
80
+ if job.ckanext == ckanext:
81
+ job_dict[job.name] = job
82
+
83
+ # Order the jobs according to depends_on
84
+ job_dict_ordered = collections.OrderedDict()
85
+ for ii in range(len(job_dict)):
86
+ for key in list(job_dict.keys()):
87
+ job = job_dict[key]
88
+ if set(job.depends_on) <= set(job_dict_ordered.keys()):
89
+ # All jobs this job depends on are in the ordered list
90
+ job_dict_ordered[key] = job
91
+ job_dict.pop(key)
92
+
93
+ if job_dict:
94
+ raise NotImplementedError(
95
+ f"The following jobs have unmet dependencies and will not "
96
+ f"be returned: {list(job_dict.keys())}")
97
+
98
+ return list(job_dict_ordered.values())
99
+
100
+
101
+ class rqjob_register:
102
+ """A decorator for background jobs to register them with RQJob"""
103
+ def __init__(self, **kwargs):
104
+ self.kwargs = kwargs
105
+
106
+ def __call__(self, func):
107
+ RQJob(method=func, **self.kwargs)
108
+ return func
@@ -315,6 +315,7 @@ def get_s3():
315
315
  return s3_client, s3_session, s3_resource
316
316
 
317
317
 
318
+ @functools.lru_cache()
318
319
  def is_available():
319
320
  """Return True if S3 credentials have been specified"""
320
321
  s3_key_id = get_ckan_config_option("dcor_object_store.access_key_id")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcor_shared
3
- Version: 0.11.4
3
+ Version: 0.12.0
4
4
  Summary: Functionalities shared by the DCOR CKAN extensions
5
5
  Author: Paul Müller
6
6
  Maintainer-email: Paul Müller <dev@craban.de>
@@ -16,6 +16,7 @@ dcor_shared/dcinst.py
16
16
  dcor_shared/mime.py
17
17
  dcor_shared/parse.py
18
18
  dcor_shared/paths.py
19
+ dcor_shared/rqjob.py
19
20
  dcor_shared/s3.py
20
21
  dcor_shared/s3cc.py
21
22
  dcor_shared/testing.py
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes