karton-core 5.8.0__py3-none-any.whl → 5.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- karton/core/__version__.py +1 -1
- karton/core/asyncio/backend.py +27 -18
- karton/core/asyncio/karton.py +20 -15
- karton/core/backend.py +62 -18
- karton/core/base.py +3 -2
- karton/core/config.py +8 -1
- karton/core/karton.py +22 -10
- karton/core/main.py +26 -6
- karton/core/test.py +6 -2
- {karton_core-5.8.0.dist-info → karton_core-5.9.0.dist-info}/METADATA +30 -7
- {karton_core-5.8.0.dist-info → karton_core-5.9.0.dist-info}/RECORD +15 -17
- {karton_core-5.8.0.dist-info → karton_core-5.9.0.dist-info}/WHEEL +1 -1
- karton_core-5.8.0-nspkg.pth +0 -1
- karton_core-5.8.0.dist-info/namespace_packages.txt +0 -1
- {karton_core-5.8.0.dist-info → karton_core-5.9.0.dist-info}/entry_points.txt +0 -0
- {karton_core-5.8.0.dist-info → karton_core-5.9.0.dist-info/licenses}/LICENSE +0 -0
- {karton_core-5.8.0.dist-info → karton_core-5.9.0.dist-info}/top_level.txt +0 -0
karton/core/__version__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "5.
|
1
|
+
__version__ = "5.9.0"
|
karton/core/asyncio/backend.py
CHANGED
@@ -12,7 +12,7 @@ from redis.asyncio.client import Pipeline
|
|
12
12
|
from redis.exceptions import AuthenticationError
|
13
13
|
|
14
14
|
from karton.core import Config, Task
|
15
|
-
from karton.core.asyncio.resource import RemoteResource
|
15
|
+
from karton.core.asyncio.resource import LocalResource, RemoteResource
|
16
16
|
from karton.core.backend import (
|
17
17
|
KARTON_BINDS_HSET,
|
18
18
|
KARTON_TASK_NAMESPACE,
|
@@ -22,6 +22,7 @@ from karton.core.backend import (
|
|
22
22
|
KartonMetrics,
|
23
23
|
KartonServiceInfo,
|
24
24
|
)
|
25
|
+
from karton.core.resource import LocalResource as SyncLocalResource
|
25
26
|
from karton.core.task import TaskState
|
26
27
|
|
27
28
|
logger = logging.getLogger(__name__)
|
@@ -117,8 +118,9 @@ class KartonAsyncBackend(KartonBackendBase):
|
|
117
118
|
boto_session._credentials = creds # type: ignore
|
118
119
|
return aioboto3.Session(botocore_session=boto_session)
|
119
120
|
|
120
|
-
@
|
121
|
+
@classmethod
|
121
122
|
async def make_redis(
|
123
|
+
cls,
|
122
124
|
config,
|
123
125
|
identity: Optional[str] = None,
|
124
126
|
service_info: Optional[KartonServiceInfo] = None,
|
@@ -131,22 +133,9 @@ class KartonAsyncBackend(KartonBackendBase):
|
|
131
133
|
:param service_info: Additional service identity metadata
|
132
134
|
:return: Redis connection
|
133
135
|
"""
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
client_name = identity
|
138
|
-
|
139
|
-
redis_args = {
|
140
|
-
"host": config["redis"]["host"],
|
141
|
-
"port": config.getint("redis", "port", 6379),
|
142
|
-
"db": config.getint("redis", "db", 0),
|
143
|
-
"username": config.get("redis", "username"),
|
144
|
-
"password": config.get("redis", "password"),
|
145
|
-
"client_name": client_name,
|
146
|
-
# set socket_timeout to None if set to 0
|
147
|
-
"socket_timeout": config.getint("redis", "socket_timeout", 30) or None,
|
148
|
-
"decode_responses": True,
|
149
|
-
}
|
136
|
+
redis_args = cls.get_redis_configuration(
|
137
|
+
config, identity=identity, service_info=service_info
|
138
|
+
)
|
150
139
|
try:
|
151
140
|
rs = Redis(**redis_args)
|
152
141
|
await rs.ping()
|
@@ -154,6 +143,7 @@ class KartonAsyncBackend(KartonBackendBase):
|
|
154
143
|
# Maybe we've sent a wrong password.
|
155
144
|
# Or maybe the server is not (yet) password protected
|
156
145
|
# To make smooth transition possible, try to login insecurely
|
146
|
+
del redis_args["username"]
|
157
147
|
del redis_args["password"]
|
158
148
|
rs = Redis(**redis_args)
|
159
149
|
await rs.ping()
|
@@ -168,6 +158,25 @@ class KartonAsyncBackend(KartonBackendBase):
|
|
168
158
|
"""
|
169
159
|
return RemoteResource.from_dict(resource_spec, backend=self)
|
170
160
|
|
161
|
+
async def declare_task(self, task: Task) -> None:
|
162
|
+
"""
|
163
|
+
Declares a new task to send it to the queue.
|
164
|
+
|
165
|
+
:param task: Task to declare
|
166
|
+
"""
|
167
|
+
# Ensure all local resources have good buckets
|
168
|
+
for resource in task.iterate_resources():
|
169
|
+
if isinstance(resource, LocalResource) and not resource.bucket:
|
170
|
+
resource.bucket = self.default_bucket_name
|
171
|
+
if isinstance(resource, SyncLocalResource):
|
172
|
+
raise RuntimeError(
|
173
|
+
"Synchronous resources are not supported. "
|
174
|
+
"Use karton.core.asyncio.resource module instead."
|
175
|
+
)
|
176
|
+
|
177
|
+
# Register new task
|
178
|
+
await self.register_task(task)
|
179
|
+
|
171
180
|
async def register_task(self, task: Task, pipe: Optional[Pipeline] = None) -> None:
|
172
181
|
"""
|
173
182
|
Register or update task in Redis.
|
karton/core/asyncio/karton.py
CHANGED
@@ -12,7 +12,6 @@ from karton.core.__version__ import __version__
|
|
12
12
|
from karton.core.backend import KartonBind, KartonMetrics
|
13
13
|
from karton.core.config import Config
|
14
14
|
from karton.core.exceptions import TaskTimeoutError
|
15
|
-
from karton.core.resource import LocalResource as SyncLocalResource
|
16
15
|
from karton.core.task import Task, TaskState
|
17
16
|
|
18
17
|
from .backend import KartonAsyncBackend
|
@@ -81,18 +80,8 @@ class Producer(KartonAsyncBase):
|
|
81
80
|
task.last_update = time.time()
|
82
81
|
task.headers.update({"origin": self.identity})
|
83
82
|
|
84
|
-
# Ensure all local resources have good buckets
|
85
|
-
for resource in task.iterate_resources():
|
86
|
-
if isinstance(resource, LocalResource) and not resource.bucket:
|
87
|
-
resource.bucket = self.backend.default_bucket_name
|
88
|
-
if isinstance(resource, SyncLocalResource):
|
89
|
-
raise RuntimeError(
|
90
|
-
"Synchronous resources are not supported. "
|
91
|
-
"Use karton.core.asyncio.resource module instead."
|
92
|
-
)
|
93
|
-
|
94
83
|
# Register new task
|
95
|
-
await self.backend.
|
84
|
+
await self.backend.declare_task(task)
|
96
85
|
|
97
86
|
# Upload local resources
|
98
87
|
for resource in task.iterate_resources():
|
@@ -222,7 +211,11 @@ class Consumer(KartonAsyncServiceBase):
|
|
222
211
|
self.current_task = task
|
223
212
|
|
224
213
|
if not task.matches_filters(self.filters):
|
225
|
-
self.log.info(
|
214
|
+
self.log.info(
|
215
|
+
"Task rejected because binds are no longer valid. "
|
216
|
+
"Rejected ask headers: %s",
|
217
|
+
task.headers,
|
218
|
+
)
|
226
219
|
await self.backend.set_task_status(task, TaskState.FINISHED)
|
227
220
|
# Task rejected: end of processing
|
228
221
|
return
|
@@ -301,7 +294,13 @@ class Consumer(KartonAsyncServiceBase):
|
|
301
294
|
if not old_bind:
|
302
295
|
self.log.info("Service binds created.")
|
303
296
|
elif old_bind != self._bind:
|
304
|
-
self.log.info(
|
297
|
+
self.log.info(
|
298
|
+
"Binds changed, old service instances should exit soon. "
|
299
|
+
"Old binds: %s "
|
300
|
+
"New binds: %s",
|
301
|
+
old_bind,
|
302
|
+
self._bind,
|
303
|
+
)
|
305
304
|
|
306
305
|
for task_filter in self.filters:
|
307
306
|
self.log.info("Binding on: %s", task_filter)
|
@@ -312,7 +311,13 @@ class Consumer(KartonAsyncServiceBase):
|
|
312
311
|
while True:
|
313
312
|
current_bind = await self.backend.get_bind(self.identity)
|
314
313
|
if current_bind != self._bind:
|
315
|
-
self.log.info(
|
314
|
+
self.log.info(
|
315
|
+
"Binds changed, shutting down. "
|
316
|
+
"Old binds: %s "
|
317
|
+
"New binds: %s",
|
318
|
+
self._bind,
|
319
|
+
current_bind,
|
320
|
+
)
|
316
321
|
break
|
317
322
|
if self.concurrency_semaphore is not None:
|
318
323
|
await self.concurrency_semaphore.acquire()
|
karton/core/backend.py
CHANGED
@@ -17,11 +17,12 @@ from botocore.credentials import (
|
|
17
17
|
from botocore.session import get_session
|
18
18
|
from redis import AuthenticationError, StrictRedis
|
19
19
|
from redis.client import Pipeline
|
20
|
+
from redis.connection import parse_url as parse_redis_url
|
20
21
|
from urllib3.response import HTTPResponse
|
21
22
|
|
22
23
|
from .config import Config
|
23
24
|
from .exceptions import InvalidIdentityError
|
24
|
-
from .resource import RemoteResource
|
25
|
+
from .resource import LocalResource, RemoteResource
|
25
26
|
from .task import Task, TaskPriority, TaskState
|
26
27
|
from .utils import chunks, chunks_iter
|
27
28
|
|
@@ -142,6 +143,41 @@ class KartonBackendBase:
|
|
142
143
|
raise RuntimeError("S3 default bucket is not defined in configuration")
|
143
144
|
return bucket_name
|
144
145
|
|
146
|
+
@staticmethod
|
147
|
+
def get_redis_configuration(
|
148
|
+
config: Config,
|
149
|
+
identity: Optional[str] = None,
|
150
|
+
service_info: Optional[KartonServiceInfo] = None,
|
151
|
+
) -> Dict[str, Any]:
|
152
|
+
if service_info is not None:
|
153
|
+
client_name: Optional[str] = service_info.make_client_name()
|
154
|
+
else:
|
155
|
+
client_name = identity
|
156
|
+
|
157
|
+
redis_url = config.get("redis", "url")
|
158
|
+
if redis_url is not None:
|
159
|
+
redis_conf = parse_redis_url(redis_url)
|
160
|
+
else:
|
161
|
+
redis_conf = {
|
162
|
+
"host": config["redis"]["host"],
|
163
|
+
"port": config.getint("redis", "port", 6379),
|
164
|
+
"db": config.getint("redis", "db", 0),
|
165
|
+
"ssl": config.getboolean("redis", "ssl", False),
|
166
|
+
}
|
167
|
+
|
168
|
+
if username := config.get("redis", "username"):
|
169
|
+
redis_conf["username"] = username
|
170
|
+
password = config.get("redis", "password")
|
171
|
+
if password is None:
|
172
|
+
raise RuntimeError("You must set both username and password, or none")
|
173
|
+
redis_conf["password"] = password
|
174
|
+
# Don't set if set to 0
|
175
|
+
if socket_timeout := config.get("redis", "socket_timeout", 30):
|
176
|
+
redis_conf["socket_timeout"] = socket_timeout
|
177
|
+
redis_conf["client_name"] = client_name
|
178
|
+
redis_conf["decode_responses"] = True
|
179
|
+
return redis_conf
|
180
|
+
|
145
181
|
@staticmethod
|
146
182
|
def get_queue_name(identity: str, priority: TaskPriority) -> str:
|
147
183
|
"""
|
@@ -302,8 +338,9 @@ class KartonBackend(KartonBackendBase):
|
|
302
338
|
endpoint_url=endpoint,
|
303
339
|
)
|
304
340
|
|
305
|
-
@
|
341
|
+
@classmethod
|
306
342
|
def make_redis(
|
343
|
+
cls,
|
307
344
|
config,
|
308
345
|
identity: Optional[str] = None,
|
309
346
|
service_info: Optional[KartonServiceInfo] = None,
|
@@ -316,22 +353,9 @@ class KartonBackend(KartonBackendBase):
|
|
316
353
|
:param service_info: Additional service identity metadata
|
317
354
|
:return: Redis connection
|
318
355
|
"""
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
client_name = identity
|
323
|
-
|
324
|
-
redis_args = {
|
325
|
-
"host": config["redis"]["host"],
|
326
|
-
"port": config.getint("redis", "port", 6379),
|
327
|
-
"db": config.getint("redis", "db", 0),
|
328
|
-
"username": config.get("redis", "username"),
|
329
|
-
"password": config.get("redis", "password"),
|
330
|
-
"client_name": client_name,
|
331
|
-
# set socket_timeout to None if set to 0
|
332
|
-
"socket_timeout": config.getint("redis", "socket_timeout", 30) or None,
|
333
|
-
"decode_responses": True,
|
334
|
-
}
|
356
|
+
redis_args = cls.get_redis_configuration(
|
357
|
+
config, identity=identity, service_info=service_info
|
358
|
+
)
|
335
359
|
try:
|
336
360
|
redis = StrictRedis(**redis_args)
|
337
361
|
redis.ping()
|
@@ -339,6 +363,7 @@ class KartonBackend(KartonBackendBase):
|
|
339
363
|
# Maybe we've sent a wrong password.
|
340
364
|
# Or maybe the server is not (yet) password protected
|
341
365
|
# To make smooth transition possible, try to login insecurely
|
366
|
+
del redis_args["username"]
|
342
367
|
del redis_args["password"]
|
343
368
|
redis = StrictRedis(**redis_args)
|
344
369
|
redis.ping()
|
@@ -631,10 +656,29 @@ class KartonBackend(KartonBackendBase):
|
|
631
656
|
task_keys, chunk_size=chunk_size, parse_resources=parse_resources
|
632
657
|
)
|
633
658
|
|
659
|
+
def declare_task(self, task: Task) -> None:
|
660
|
+
"""
|
661
|
+
Declares a new task to send it to the queue.
|
662
|
+
|
663
|
+
Task producers should use this method for new tasks.
|
664
|
+
|
665
|
+
:param task: Task to declare
|
666
|
+
"""
|
667
|
+
# Ensure all local resources have good buckets
|
668
|
+
for resource in task.iterate_resources():
|
669
|
+
if isinstance(resource, LocalResource) and not resource.bucket:
|
670
|
+
resource.bucket = self.default_bucket_name
|
671
|
+
|
672
|
+
# Register new task
|
673
|
+
self.register_task(task)
|
674
|
+
|
634
675
|
def register_task(self, task: Task, pipe: Optional[Pipeline] = None) -> None:
|
635
676
|
"""
|
636
677
|
Register or update task in Redis.
|
637
678
|
|
679
|
+
This method is used internally to alter task data. If you want to declare new
|
680
|
+
task in Redis, use declare_task.
|
681
|
+
|
638
682
|
:param task: Task object
|
639
683
|
:param pipe: Optional pipeline object if operation is a part of pipeline
|
640
684
|
"""
|
karton/core/base.py
CHANGED
@@ -131,11 +131,11 @@ class LoggingMixin:
|
|
131
131
|
if not self.identity:
|
132
132
|
raise ValueError("Can't setup logger without identity")
|
133
133
|
|
134
|
+
task_context_filter = TaskContextFilter()
|
134
135
|
self._log_handler.setFormatter(logging.Formatter())
|
136
|
+
self._log_handler.addFilter(task_context_filter)
|
135
137
|
|
136
138
|
logger = logging.getLogger(self.identity)
|
137
|
-
logger.addFilter(TaskContextFilter())
|
138
|
-
|
139
139
|
if logger.handlers:
|
140
140
|
# If logger already have handlers set: clear them
|
141
141
|
logger.handlers.clear()
|
@@ -148,6 +148,7 @@ class LoggingMixin:
|
|
148
148
|
logger.setLevel(log_level)
|
149
149
|
stream_handler = logging.StreamHandler()
|
150
150
|
stream_handler.setFormatter(logging.Formatter(self._log_format))
|
151
|
+
stream_handler.addFilter(task_context_filter)
|
151
152
|
logger.addHandler(stream_handler)
|
152
153
|
|
153
154
|
if not self.debug and self.enable_publish_log:
|
karton/core/config.py
CHANGED
@@ -14,6 +14,7 @@ class Config(object):
|
|
14
14
|
- ``/etc/karton/karton.ini`` (global)
|
15
15
|
- ``~/.config/karton/karton.ini`` (user local)
|
16
16
|
- ``./karton.ini`` (subsystem local)
|
17
|
+
- path from ``KARTON_CONFIG_FILE`` environment variable
|
17
18
|
- ``<path>`` optional, additional path provided in arguments
|
18
19
|
|
19
20
|
It is also possible to pass configuration via environment variables.
|
@@ -38,6 +39,12 @@ class Config(object):
|
|
38
39
|
) -> None:
|
39
40
|
self._config: Dict[str, Dict[str, Any]] = {}
|
40
41
|
|
42
|
+
path_from_env = os.getenv("KARTON_CONFIG_FILE")
|
43
|
+
if path_from_env:
|
44
|
+
if not os.path.isfile(path_from_env):
|
45
|
+
raise IOError(f"Configuration file not found in {path_from_env}")
|
46
|
+
self.SEARCH_PATHS = self.SEARCH_PATHS + [path_from_env]
|
47
|
+
|
41
48
|
if path is not None:
|
42
49
|
if not os.path.isfile(path):
|
43
50
|
raise IOError("Configuration file not found in " + path)
|
@@ -222,7 +229,7 @@ class Config(object):
|
|
222
229
|
for name, value in os.environ.items():
|
223
230
|
# Load env variables named KARTON_[section]_[key]
|
224
231
|
# to match ConfigParser structure
|
225
|
-
result = re.fullmatch(r"KARTON_([A-Z0-9
|
232
|
+
result = re.fullmatch(r"KARTON_([A-Z0-9\-\.]+)_([A-Z0-9_]+)", name)
|
226
233
|
|
227
234
|
if not result:
|
228
235
|
continue
|
karton/core/karton.py
CHANGED
@@ -80,13 +80,8 @@ class Producer(KartonBase):
|
|
80
80
|
task.last_update = time.time()
|
81
81
|
task.headers.update({"origin": self.identity})
|
82
82
|
|
83
|
-
# Ensure all local resources have good buckets
|
84
|
-
for resource in task.iterate_resources():
|
85
|
-
if isinstance(resource, LocalResource) and not resource.bucket:
|
86
|
-
resource.bucket = self.backend.default_bucket_name
|
87
|
-
|
88
83
|
# Register new task
|
89
|
-
self.backend.
|
84
|
+
self.backend.declare_task(task)
|
90
85
|
|
91
86
|
# Upload local resources
|
92
87
|
for resource in task.iterate_resources():
|
@@ -172,7 +167,11 @@ class Consumer(KartonServiceBase):
|
|
172
167
|
self.current_task = task
|
173
168
|
|
174
169
|
if not task.matches_filters(self.filters):
|
175
|
-
self.log.info(
|
170
|
+
self.log.info(
|
171
|
+
"Task rejected because binds are no longer valid. "
|
172
|
+
"Rejected ask headers: %s",
|
173
|
+
task.headers,
|
174
|
+
)
|
176
175
|
self.backend.set_task_status(task, TaskState.FINISHED)
|
177
176
|
# Task rejected: end of processing
|
178
177
|
return
|
@@ -339,15 +338,28 @@ class Consumer(KartonServiceBase):
|
|
339
338
|
if not old_bind:
|
340
339
|
self.log.info("Service binds created.")
|
341
340
|
elif old_bind != self._bind:
|
342
|
-
self.log.info(
|
341
|
+
self.log.info(
|
342
|
+
"Binds changed, old service instances should exit soon. "
|
343
|
+
"Old binds: %s "
|
344
|
+
"New binds: %s",
|
345
|
+
old_bind,
|
346
|
+
self._bind,
|
347
|
+
)
|
343
348
|
|
344
349
|
for task_filter in self.filters:
|
345
350
|
self.log.info("Binding on: %s", task_filter)
|
346
351
|
|
347
352
|
with self.graceful_killer():
|
348
353
|
while not self.shutdown:
|
349
|
-
|
350
|
-
|
354
|
+
current_bind = self.backend.get_bind(self.identity)
|
355
|
+
if current_bind != self._bind:
|
356
|
+
self.log.info(
|
357
|
+
"Binds changed, shutting down. "
|
358
|
+
"Old binds: %s "
|
359
|
+
"New binds: %s",
|
360
|
+
self._bind,
|
361
|
+
current_bind,
|
362
|
+
)
|
351
363
|
break
|
352
364
|
task = self.backend.consume_routed_task(self.identity)
|
353
365
|
if task:
|
karton/core/main.py
CHANGED
@@ -144,10 +144,23 @@ def configuration_wizard(config_filename: str) -> None:
|
|
144
144
|
log.info("Saved the new configuration file in %s", os.path.abspath(config_filename))
|
145
145
|
|
146
146
|
|
147
|
-
def print_bind_list(config: Config) -> None:
|
147
|
+
def print_bind_list(config: Config, output_format: str) -> None:
|
148
148
|
backend = KartonBackend(config=config)
|
149
|
-
|
150
|
-
|
149
|
+
|
150
|
+
if output_format == "table":
|
151
|
+
# Print a human-readable table-like version
|
152
|
+
print(f"{'karton name':50} {'version':10} {'karton':10}")
|
153
|
+
print("-" * 72)
|
154
|
+
for bind in backend.get_binds():
|
155
|
+
print(
|
156
|
+
f"{bind.identity:50} {bind.service_version or "-":10} {bind.version:10}"
|
157
|
+
)
|
158
|
+
elif output_format == "json":
|
159
|
+
# Use JSONL, each line is a JSON representing next bind
|
160
|
+
for bind in backend.get_binds():
|
161
|
+
print(backend.serialize_bind(bind))
|
162
|
+
else:
|
163
|
+
raise RuntimeError(f"Invalid output format: {output_format}")
|
151
164
|
|
152
165
|
|
153
166
|
def delete_bind(config: Config, karton_name: str) -> None:
|
@@ -180,7 +193,7 @@ def delete_bind(config: Config, karton_name: str) -> None:
|
|
180
193
|
|
181
194
|
def main() -> None:
|
182
195
|
|
183
|
-
parser = argparse.ArgumentParser(description="
|
196
|
+
parser = argparse.ArgumentParser(description="Karton-core management utility")
|
184
197
|
parser.add_argument("--version", action="version", version=__version__)
|
185
198
|
parser.add_argument("-c", "--config-file", help="Alternative configuration path")
|
186
199
|
parser.add_argument(
|
@@ -189,7 +202,14 @@ def main() -> None:
|
|
189
202
|
|
190
203
|
subparsers = parser.add_subparsers(dest="command", help="sub-command help")
|
191
204
|
|
192
|
-
subparsers.add_parser("list", help="List active karton binds")
|
205
|
+
list_parser = subparsers.add_parser("list", help="List active karton binds")
|
206
|
+
list_parser.add_argument(
|
207
|
+
"-o",
|
208
|
+
"--output",
|
209
|
+
help="Short, human readable output, with names and versions only.",
|
210
|
+
default="table",
|
211
|
+
choices=("table", "json"),
|
212
|
+
)
|
193
213
|
|
194
214
|
logs_parser = subparsers.add_parser("logs", help="Start streaming logs")
|
195
215
|
logs_parser.add_argument(
|
@@ -253,7 +273,7 @@ def main() -> None:
|
|
253
273
|
return
|
254
274
|
|
255
275
|
if args.command == "list":
|
256
|
-
print_bind_list(config)
|
276
|
+
print_bind_list(config, args.output)
|
257
277
|
elif args.command == "delete":
|
258
278
|
karton_name = args.identity
|
259
279
|
print(
|
karton/core/test.py
CHANGED
@@ -35,8 +35,12 @@ class BackendMock:
|
|
35
35
|
def default_bucket_name(self) -> str:
|
36
36
|
return "karton.test"
|
37
37
|
|
38
|
-
def
|
39
|
-
|
38
|
+
def declare_task(self, task: Task) -> None:
|
39
|
+
# Ensure all local resources have good buckets
|
40
|
+
for resource in task.iterate_resources():
|
41
|
+
if isinstance(resource, LocalResource) and not resource.bucket:
|
42
|
+
resource.bucket = self.default_bucket_name
|
43
|
+
log.debug("Declaring a new task in Redis: %s", task.serialize())
|
40
44
|
|
41
45
|
def set_task_status(self, task: Task, status: TaskState, pipe=None) -> None:
|
42
46
|
log.debug("Setting task %s status to %s", task.uid, status)
|
@@ -1,18 +1,21 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.4
|
2
2
|
Name: karton-core
|
3
|
-
Version: 5.
|
3
|
+
Version: 5.9.0
|
4
4
|
Summary: Distributed malware analysis orchestration framework
|
5
|
-
|
5
|
+
License-Expression: BSD-3-Clause
|
6
|
+
Project-URL: Homepage, https://github.com/CERT-Polska/karton
|
7
|
+
Project-URL: Documentation, https://karton-core.readthedocs.io/
|
8
|
+
Project-URL: Repository, https://github.com/CERT-Polska/karton
|
6
9
|
Classifier: Programming Language :: Python :: 3
|
7
10
|
Classifier: Operating System :: OS Independent
|
8
|
-
Classifier: License :: OSI Approved :: BSD License
|
9
11
|
Requires-Python: >=3.8
|
10
12
|
Description-Content-Type: text/markdown
|
11
13
|
License-File: LICENSE
|
12
|
-
Requires-Dist: aioboto3 ==14.3.0
|
13
|
-
Requires-Dist: boto3 <1.37.4,>=1.37.2
|
14
|
-
Requires-Dist: orjson
|
15
14
|
Requires-Dist: redis
|
15
|
+
Requires-Dist: orjson
|
16
|
+
Requires-Dist: boto3<=1.35.81
|
17
|
+
Requires-Dist: aioboto3==13.3.0
|
18
|
+
Dynamic: license-file
|
16
19
|
|
17
20
|
# Karton <img src="img/logo.svg" width="64">
|
18
21
|
|
@@ -82,6 +85,26 @@ if __name__ == "__main__":
|
|
82
85
|
GenericUnpacker.main()
|
83
86
|
```
|
84
87
|
|
88
|
+
## Command line
|
89
|
+
|
90
|
+
This package also provies a command-line utility called "karton". You can use it for simple management tasks (but it's not designed as a fully capable management tool).
|
91
|
+
|
92
|
+
```
|
93
|
+
$ karton configure # create a new configuration file
|
94
|
+
|
95
|
+
$ karton list -s # list current binds
|
96
|
+
karton name version karton
|
97
|
+
------------------------------------------------------------------------
|
98
|
+
karton.yaramatcher 1.2.0 5.3.0
|
99
|
+
karton.autoit-ripper 1.2.1 5.3.3
|
100
|
+
karton.mwdb-reporter 1.3.0 5.3.2
|
101
|
+
|
102
|
+
$ karton logs # start streaming all system logs
|
103
|
+
|
104
|
+
$ karton delete karton.something # remove unused bind (will be GCed by system during the next operation)
|
105
|
+
```
|
106
|
+
|
107
|
+
|
85
108
|
## Karton systems
|
86
109
|
|
87
110
|
Some Karton systems are universal and useful to everyone. We decided to share them with the community.
|
@@ -1,33 +1,31 @@
|
|
1
|
-
karton_core-5.8.0-nspkg.pth,sha256=vHa-jm6pBTeInFrmnsHMg9AOeD88czzQy-6QCFbpRcM,539
|
2
1
|
karton/core/__init__.py,sha256=QuT0BWZyp799eY90tK3H1OD2hwuusqMJq8vQwpB3kG4,337
|
3
|
-
karton/core/__version__.py,sha256=
|
4
|
-
karton/core/backend.py,sha256=
|
5
|
-
karton/core/base.py,sha256=
|
6
|
-
karton/core/config.py,sha256=
|
2
|
+
karton/core/__version__.py,sha256=K8QUWqtzm0RvyvMi0shZurlcbRE1MiHY43oJIJLcGF0,22
|
3
|
+
karton/core/backend.py,sha256=IhqK-Pia3RjUGHdWEtV7ruFr3w6rhDETrewU3DQo-Pw,41547
|
4
|
+
karton/core/base.py,sha256=mbsZKna9TNqGpvDMwp2nuc9KvlevwOH94ubu0rJvVgk,9178
|
5
|
+
karton/core/config.py,sha256=7nZHQ_25k-dyofohiubOPUBISY3f362gJTh80Igyh4U,8580
|
7
6
|
karton/core/exceptions.py,sha256=8i9WVzi4PinNlX10Cb-lQQC35Hl-JB5R_UKXa9AUKoQ,153
|
8
7
|
karton/core/inspect.py,sha256=aIJQEOEkD5q2xLlV8nhxY5qL5zqcnprP-2DdP6ecKlE,6150
|
9
|
-
karton/core/karton.py,sha256=
|
8
|
+
karton/core/karton.py,sha256=SUcu1V0Xehq5X5EFp91uaulaefZoF1ObYlEV6VsBZrQ,15522
|
10
9
|
karton/core/logger.py,sha256=UhYCoVARXaatvoJ2lO2mfBHeODOS7z8O-vqdeQhNmV4,2654
|
11
|
-
karton/core/main.py,sha256=
|
10
|
+
karton/core/main.py,sha256=2teV0W4W672fzDk6zNcRF77qhhac1_peN3aEWpGXOdQ,9109
|
12
11
|
karton/core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
13
12
|
karton/core/query.py,sha256=sf24DweVlXfJuBbBD_ns2LXhOV-IBwuPG3jBfTJu77s,12063
|
14
13
|
karton/core/resource.py,sha256=GkU3JGSP1kOsAoblon4BWbQp31eZiDsCjaaDKanEokk,20872
|
15
14
|
karton/core/task.py,sha256=QvAsSUIsyYiRmkgxgugrYkHWH2gczrFDfw2V4izdt1E,19566
|
16
|
-
karton/core/test.py,sha256=
|
15
|
+
karton/core/test.py,sha256=TwLXnyNvTvZbnYTjiB4PhW0F_mXiiov4fhLL5Cs2-6I,9349
|
17
16
|
karton/core/utils.py,sha256=sEVqGdVPyYswWuVn8wYXBQmln8Az826N_2HgC__pmW8,4090
|
18
17
|
karton/core/asyncio/__init__.py,sha256=ZgndeKzS3Yg2o8hebwFYJWlCRdW3ImdCOShK4EVmZ14,457
|
19
|
-
karton/core/asyncio/backend.py,sha256=
|
18
|
+
karton/core/asyncio/backend.py,sha256=fyOl5kA_KJ-SUyTPzR9ZU5Iz_6vvUb0Je3SVaBQrs6U,13057
|
20
19
|
karton/core/asyncio/base.py,sha256=YDNGyWzgVvt2TnfKvHYbJbcNJaQl95bdBq45YGEo-3Q,4246
|
21
|
-
karton/core/asyncio/karton.py,sha256=
|
20
|
+
karton/core/asyncio/karton.py,sha256=sWzwsBBbAhO32TIu7hi0R9HAWlcN_SzgrmaloEgwsgY,12844
|
22
21
|
karton/core/asyncio/logger.py,sha256=BjkbuAeWylTmFjWv8-ckmOGf4nL2Tma96W0nIOc2vwk,1752
|
23
22
|
karton/core/asyncio/resource.py,sha256=86AYm7JeVjEYRNw--h02HIS9xFvgddhktmDUp0qvTO4,12517
|
24
23
|
karton/system/__init__.py,sha256=JF51OqRU_Y4c0unOulvmv1KzSHSq4ZpXU8ZsH4nefRM,63
|
25
24
|
karton/system/__main__.py,sha256=QJkwIlSwaPRdzwKlNmCAL41HtDAa73db9MZKWmOfxGM,56
|
26
25
|
karton/system/system.py,sha256=d_5hhLTthJdr_4gZEGQ6Y-kHvxeBqyQxjjx_wRs3xMA,17285
|
27
|
-
karton_core-5.
|
28
|
-
karton_core-5.
|
29
|
-
karton_core-5.
|
30
|
-
karton_core-5.
|
31
|
-
karton_core-5.
|
32
|
-
karton_core-5.
|
33
|
-
karton_core-5.8.0.dist-info/RECORD,,
|
26
|
+
karton_core-5.9.0.dist-info/licenses/LICENSE,sha256=o8h7hYhn7BJC_-DmrfqWwLjaR_Gbe0TZOOQJuN2ca3I,1519
|
27
|
+
karton_core-5.9.0.dist-info/METADATA,sha256=r6CDk__eoUHVC2zufKRsKr64ZGgz_xII1tFSZulMkv4,7788
|
28
|
+
karton_core-5.9.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
29
|
+
karton_core-5.9.0.dist-info/entry_points.txt,sha256=OgLlsXy61GP6-Yob3oXqeJ2hlRU6LBLj33fr0NufKz0,98
|
30
|
+
karton_core-5.9.0.dist-info/top_level.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
|
31
|
+
karton_core-5.9.0.dist-info/RECORD,,
|
karton_core-5.8.0-nspkg.pth
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
import sys, types, os;has_mfs = sys.version_info > (3, 5);p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('karton',));importlib = has_mfs and __import__('importlib.util');has_mfs and __import__('importlib.machinery');m = has_mfs and sys.modules.setdefault('karton', importlib.util.module_from_spec(importlib.machinery.PathFinder.find_spec('karton', [os.path.dirname(p)])));m = m or sys.modules.setdefault('karton', types.ModuleType('karton'));mp = (m or []) and m.__dict__.setdefault('__path__',[]);(p not in mp) and mp.append(p)
|
@@ -1 +0,0 @@
|
|
1
|
-
karton
|
File without changes
|
File without changes
|
File without changes
|