runnable 0.17.1__py3-none-any.whl → 0.19.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. extensions/README.md +0 -0
  2. extensions/__init__.py +0 -0
  3. extensions/catalog/README.md +0 -0
  4. extensions/catalog/file_system.py +253 -0
  5. extensions/catalog/pyproject.toml +14 -0
  6. extensions/job_executor/README.md +0 -0
  7. extensions/job_executor/__init__.py +160 -0
  8. extensions/job_executor/k8s.py +484 -0
  9. extensions/job_executor/k8s_job_spec.yaml +37 -0
  10. extensions/job_executor/local.py +61 -0
  11. extensions/job_executor/local_container.py +192 -0
  12. extensions/job_executor/pyproject.toml +16 -0
  13. extensions/nodes/README.md +0 -0
  14. extensions/nodes/nodes.py +954 -0
  15. extensions/nodes/pyproject.toml +15 -0
  16. extensions/pipeline_executor/README.md +0 -0
  17. extensions/pipeline_executor/__init__.py +644 -0
  18. extensions/pipeline_executor/argo.py +1307 -0
  19. extensions/pipeline_executor/argo_specification.yaml +51 -0
  20. extensions/pipeline_executor/local.py +62 -0
  21. extensions/pipeline_executor/local_container.py +362 -0
  22. extensions/pipeline_executor/mocked.py +161 -0
  23. extensions/pipeline_executor/pyproject.toml +16 -0
  24. extensions/pipeline_executor/retry.py +180 -0
  25. extensions/run_log_store/README.md +0 -0
  26. extensions/run_log_store/__init__.py +0 -0
  27. extensions/run_log_store/chunked_fs.py +113 -0
  28. extensions/run_log_store/db/implementation_FF.py +163 -0
  29. extensions/run_log_store/db/integration_FF.py +0 -0
  30. extensions/run_log_store/file_system.py +145 -0
  31. extensions/run_log_store/generic_chunked.py +599 -0
  32. extensions/run_log_store/pyproject.toml +15 -0
  33. extensions/secrets/README.md +0 -0
  34. extensions/secrets/dotenv.py +62 -0
  35. extensions/secrets/pyproject.toml +15 -0
  36. runnable/__init__.py +1 -0
  37. runnable/catalog.py +1 -2
  38. runnable/entrypoints.py +1 -5
  39. runnable/executor.py +1 -1
  40. runnable/parameters.py +0 -9
  41. runnable/utils.py +5 -25
  42. {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/METADATA +1 -7
  43. runnable-0.19.0.dist-info/RECORD +58 -0
  44. {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/entry_points.txt +1 -0
  45. runnable-0.17.1.dist-info/RECORD +0 -23
  46. {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/WHEEL +0 -0
  47. {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,599 @@
1
+ import logging
2
+ import time
3
+ from abc import abstractmethod
4
+ from enum import Enum
5
+ from pathlib import Path
6
+ from string import Template
7
+ from typing import Any, Dict, Optional, Sequence, Union
8
+
9
+ from runnable import defaults, exceptions
10
+ from runnable.datastore import (
11
+ BaseRunLogStore,
12
+ BranchLog,
13
+ JsonParameter,
14
+ MetricParameter,
15
+ ObjectParameter,
16
+ Parameter,
17
+ RunLog,
18
+ StepLog,
19
+ )
20
+
21
+ logger = logging.getLogger(defaults.LOGGER_NAME)
22
+
23
+
24
+ T = Union[str, Path] # Holds str, path
25
+
26
+
27
+ class EntityNotFoundError(Exception):
28
+ pass
29
+
30
+
31
+ class ChunkedRunLogStore(BaseRunLogStore):
32
+ """
33
+ A generic implementation of a RunLogStore that stores RunLogs in chunks.
34
+ """
35
+
36
+ service_name: str = ""
37
+
38
+ class LogTypes(Enum):
39
+ RUN_LOG = "RunLog"
40
+ PARAMETER = "Parameter"
41
+ STEP_LOG = "StepLog"
42
+ BRANCH_LOG = "BranchLog"
43
+
44
+ class ModelTypes(Enum):
45
+ RUN_LOG = RunLog
46
+ PARAMETER = dict
47
+ STEP_LOG = StepLog
48
+ BRANCH_LOG = BranchLog
49
+
50
+ def naming_pattern(self, log_type: LogTypes, name: str = "") -> str:
51
+ """
52
+ Naming pattern to store RunLog, Parameter, StepLog or BranchLog.
53
+
54
+ The reasoning for name to be defaulted to empty string:
55
+ Its actually conditionally empty. For RunLog and Parameter it is empty.
56
+ For StepLog and BranchLog it should be provided.
57
+
58
+ Args:
59
+ log_type (LogTypes): One of RunLog, Parameter, StepLog or BranchLog
60
+ name (str, optional): The name to be included or left. Defaults to ''.
61
+
62
+ Raises:
63
+ Exception: If log_type is not recognized
64
+
65
+ Returns:
66
+ str: The naming pattern
67
+ """
68
+ if log_type == self.LogTypes.RUN_LOG:
69
+ return f"{self.LogTypes.RUN_LOG.value}"
70
+
71
+ if log_type == self.LogTypes.PARAMETER:
72
+ return "-".join([self.LogTypes.PARAMETER.value, name])
73
+
74
+ if not name:
75
+ raise Exception(
76
+ f"Name should be provided for naming pattern for {log_type}"
77
+ )
78
+
79
+ if log_type == self.LogTypes.STEP_LOG:
80
+ return "-".join([self.LogTypes.STEP_LOG.value, name, "${creation_time}"])
81
+
82
+ if log_type == self.LogTypes.BRANCH_LOG:
83
+ return "-".join([self.LogTypes.BRANCH_LOG.value, name, "${creation_time}"])
84
+
85
+ raise Exception("Unexpected log type")
86
+
87
+ @abstractmethod
88
+ def get_matches(
89
+ self, run_id: str, name: str, multiple_allowed: bool = False
90
+ ) -> Optional[Union[Sequence[T], T]]:
91
+ """
92
+ Get contents of persistence layer matching the pattern name*
93
+
94
+ Args:
95
+ run_id (str): The run id
96
+ name (str): The suffix of the entity name to check in the run log store.
97
+ """
98
+ ...
99
+
100
+ @abstractmethod
101
+ def _store(self, run_id: str, contents: dict, name: T, insert: bool = False):
102
+ """
103
+ Store the contents against the name in the persistence layer.
104
+
105
+ Args:
106
+ run_id (str): The run id
107
+ contents (dict): The dict to store
108
+ name (str): The name to store as
109
+ """
110
+ ...
111
+
112
+ @abstractmethod
113
+ def _retrieve(self, name: T) -> dict:
114
+ """
115
+ Does the job of retrieving from the persistent layer.
116
+
117
+ Args:
118
+ name (str): the name of the file to retrieve
119
+
120
+ Returns:
121
+ dict: The contents
122
+ """
123
+ ...
124
+
125
+ def store(self, run_id: str, log_type: LogTypes, contents: dict, name: str = ""):
126
+ """Store a SINGLE log type in the file system
127
+
128
+ Args:
129
+ run_id (str): The run id to store against
130
+ log_type (LogTypes): The type of log to store
131
+ contents (dict): The dict of contents to store
132
+ name (str, optional): The name against the contents have to be stored. Defaults to ''.
133
+ """
134
+ naming_pattern = self.naming_pattern(log_type=log_type, name=name)
135
+ match = self.get_matches(
136
+ run_id=run_id, name=naming_pattern, multiple_allowed=False
137
+ )
138
+ # The boolean multiple allowed confuses mypy a lot!
139
+ name_to_give: str = ""
140
+ insert = False
141
+
142
+ if match:
143
+ existing_contents = self._retrieve(name=match) # type: ignore
144
+ contents = dict(existing_contents, **contents)
145
+ name_to_give = match # type: ignore
146
+ else:
147
+ name_to_give = Template(naming_pattern).safe_substitute(
148
+ {"creation_time": str(int(time.time_ns()))}
149
+ )
150
+ insert = True
151
+
152
+ self._store(run_id=run_id, contents=contents, name=name_to_give, insert=insert)
153
+
154
+ def retrieve(
155
+ self, run_id: str, log_type: LogTypes, name: str = "", multiple_allowed=False
156
+ ) -> Any:
157
+ """
158
+ Retrieve the model given a log_type and a name.
159
+ Use multiple_allowed to control if you are expecting multiple of them.
160
+ eg: There could be multiple of Parameters- but only one of StepLog-stepname
161
+
162
+ The reasoning for name to be defaulted to empty string:
163
+ Its actually conditionally empty. For RunLog and Parameter it is empty.
164
+ For StepLog and BranchLog it should be provided.
165
+
166
+ Args:
167
+ run_id (str): The run id
168
+ log_type (LogTypes): One of RunLog, Parameter, StepLog, BranchLog
169
+ name (str, optional): The name to match. Defaults to ''.
170
+ multiple_allowed (bool, optional): Are multiple allowed. Defaults to False.
171
+
172
+ Raises:
173
+ FileNotFoundError: If there is no match found
174
+
175
+ Returns:
176
+ Any: One of StepLog, BranchLog, Parameter or RunLog
177
+ """
178
+ # The reason of any is it could be one of Logs or dict or list of the
179
+ if not name and log_type not in [
180
+ self.LogTypes.RUN_LOG,
181
+ self.LogTypes.PARAMETER,
182
+ ]:
183
+ raise Exception(f"Name is required during retrieval for {log_type}")
184
+
185
+ naming_pattern = self.naming_pattern(log_type=log_type, name=name)
186
+
187
+ matches = self.get_matches(
188
+ run_id=run_id, name=naming_pattern, multiple_allowed=multiple_allowed
189
+ )
190
+
191
+ if matches:
192
+ if not multiple_allowed:
193
+ contents = self._retrieve(name=matches) # type: ignore
194
+ model = self.ModelTypes[log_type.name].value
195
+ return model(**contents)
196
+
197
+ models = []
198
+ for match in matches: # type: ignore
199
+ contents = self._retrieve(name=match)
200
+ model = self.ModelTypes[log_type.name].value
201
+ models.append(model(**contents))
202
+ return models
203
+
204
+ raise EntityNotFoundError()
205
+
206
+ def orderly_retrieve(
207
+ self, run_id: str, log_type: LogTypes
208
+ ) -> Dict[str, Union[StepLog, BranchLog]]:
209
+ """Should only be used by prepare full run log.
210
+
211
+ Retrieves the StepLog or BranchLog sorted according to creation time.
212
+
213
+ Args:
214
+ run_id (str): _description_
215
+ log_type (LogTypes): _description_
216
+ """
217
+ prefix: str = self.LogTypes.STEP_LOG.value
218
+
219
+ if log_type == self.LogTypes.BRANCH_LOG:
220
+ prefix = self.LogTypes.BRANCH_LOG.value
221
+
222
+ matches = self.get_matches(run_id=run_id, name=prefix, multiple_allowed=True)
223
+
224
+ if log_type == self.LogTypes.BRANCH_LOG and not matches:
225
+ # No branch logs are found
226
+ return {}
227
+ # Forcing get_matches to always return a list is a better design
228
+ epoch_created = [str(match).split("-")[-1] for match in matches] # type: ignore
229
+
230
+ # sort matches by epoch created
231
+ epoch_created, matches = zip(*sorted(zip(epoch_created, matches))) # type: ignore
232
+
233
+ logs: Dict[str, Union[StepLog, BranchLog]] = {}
234
+
235
+ for match in matches:
236
+ model = self.ModelTypes[log_type.name].value
237
+ log_model = model(**self._retrieve(match))
238
+ logs[log_model.internal_name] = log_model # type: ignore
239
+
240
+ return logs
241
+
242
+ def _get_parent_branch(self, name: str) -> Union[str, None]:
243
+ """
244
+ Returns the name of the parent branch.
245
+ If the step is part of main dag, return None.
246
+
247
+ Args:
248
+ name (str): The name of the step.
249
+
250
+ Returns:
251
+ str: The name of the branch containing the step.
252
+ """
253
+ dot_path = name.split(".")
254
+
255
+ if len(dot_path) == 1:
256
+ return None
257
+ # Ignore the step name
258
+ return ".".join(dot_path[:-1])
259
+
260
+ def _get_parent_step(self, name: str) -> Union[str, None]:
261
+ """
262
+ Returns the step containing the step, useful when we have steps within a branch.
263
+ Returns None, if the step belongs to parent dag.
264
+
265
+ Args:
266
+ name (str): The name of the step to find the parent step it belongs to.
267
+
268
+ Returns:
269
+ str: The parent step the step belongs to, None if the step belongs to parent dag.
270
+ """
271
+ dot_path = name.split(".")
272
+
273
+ if len(dot_path) == 1:
274
+ return None
275
+ # Ignore the branch.step_name
276
+ return ".".join(dot_path[:-2])
277
+
278
+ def _prepare_full_run_log(self, run_log: RunLog):
279
+ """
280
+ Populates the run log with the branches and steps.
281
+
282
+ Args:
283
+ run_log (RunLog): The partial run log containing empty step logs
284
+ """
285
+ run_id = run_log.run_id
286
+ run_log.parameters = self.get_parameters(run_id=run_id)
287
+
288
+ ordered_steps = self.orderly_retrieve(
289
+ run_id=run_id, log_type=self.LogTypes.STEP_LOG
290
+ )
291
+ ordered_branches = self.orderly_retrieve(
292
+ run_id=run_id, log_type=self.LogTypes.BRANCH_LOG
293
+ )
294
+
295
+ current_branch: Any = None # It could be str, None, RunLog
296
+ for step_internal_name in ordered_steps:
297
+ current_branch = self._get_parent_branch(step_internal_name)
298
+ step_to_add_branch = self._get_parent_step(step_internal_name)
299
+
300
+ if not current_branch:
301
+ current_branch = run_log
302
+ else:
303
+ current_branch = ordered_branches[current_branch]
304
+ step_to_add_branch = ordered_steps[step_to_add_branch] # type: ignore
305
+ step_to_add_branch.branches[current_branch.internal_name] = ( # type: ignore
306
+ current_branch
307
+ )
308
+
309
+ current_branch.steps[step_internal_name] = ordered_steps[step_internal_name]
310
+
311
+ def create_run_log(
312
+ self,
313
+ run_id: str,
314
+ dag_hash: str = "",
315
+ use_cached: bool = False,
316
+ tag: str = "",
317
+ original_run_id: str = "",
318
+ status: str = defaults.CREATED,
319
+ **kwargs,
320
+ ):
321
+ """
322
+ Creates a Run Log object by using the config
323
+
324
+ Logically the method should do the following:
325
+ * Creates a Run log
326
+ * Adds it to the db
327
+ * Return the log
328
+ """
329
+ try:
330
+ self.get_run_log_by_id(run_id=run_id, full=False)
331
+ raise exceptions.RunLogExistsError(run_id=run_id)
332
+ except exceptions.RunLogNotFoundError:
333
+ pass
334
+
335
+ logger.info(f"{self.service_name} Creating a Run Log for : {run_id}")
336
+ run_log = RunLog(
337
+ run_id=run_id,
338
+ dag_hash=dag_hash,
339
+ tag=tag,
340
+ status=status,
341
+ )
342
+
343
+ self.store(
344
+ run_id=run_id, contents=run_log.model_dump(), log_type=self.LogTypes.RUN_LOG
345
+ )
346
+ return run_log
347
+
348
+ def get_run_log_by_id(self, run_id: str, full: bool = False) -> RunLog:
349
+ """
350
+ Retrieves a Run log from the database using the config and the run_id
351
+
352
+ Args:
353
+ run_id (str): The run_id of the run
354
+ full (bool): return the full run log store or only the RunLog object
355
+
356
+ Returns:
357
+ RunLog: The RunLog object identified by the run_id
358
+
359
+ Logically the method should:
360
+ * Returns the run_log defined by id from the data store defined by the config
361
+
362
+ """
363
+ try:
364
+ logger.info(f"{self.service_name} Getting a Run Log for : {run_id}")
365
+ run_log = self.retrieve(
366
+ run_id=run_id, log_type=self.LogTypes.RUN_LOG, multiple_allowed=False
367
+ )
368
+
369
+ if full:
370
+ self._prepare_full_run_log(run_log=run_log)
371
+
372
+ return run_log
373
+ except EntityNotFoundError as e:
374
+ raise exceptions.RunLogNotFoundError(run_id) from e
375
+
376
+ def put_run_log(self, run_log: RunLog):
377
+ """
378
+ Puts the Run Log in the database as defined by the config
379
+
380
+ Args:
381
+ run_log (RunLog): The Run log of the run
382
+
383
+ Logically the method should:
384
+ Puts the run_log into the database
385
+
386
+ Raises:
387
+ NotImplementedError: This is a base class and therefore has no default implementation
388
+ """
389
+ run_id = run_log.run_id
390
+ self.store(
391
+ run_id=run_id, contents=run_log.model_dump(), log_type=self.LogTypes.RUN_LOG
392
+ )
393
+
394
+ def get_parameters(self, run_id: str) -> dict:
395
+ """
396
+ Get the parameters from the Run log defined by the run_id
397
+
398
+ Args:
399
+ run_id (str): The run_id of the run
400
+
401
+ The method should:
402
+ * Call get_run_log_by_id(run_id) to retrieve the run_log
403
+ * Return the parameters as identified in the run_log
404
+
405
+ Returns:
406
+ dict: A dictionary of the run_log parameters
407
+ Raises:
408
+ RunLogNotFoundError: If the run log for run_id is not found in the datastore
409
+ """
410
+ parameters: Dict[str, Parameter] = {}
411
+ try:
412
+ parameters_list = self.retrieve(
413
+ run_id=run_id, log_type=self.LogTypes.PARAMETER, multiple_allowed=True
414
+ )
415
+ for param in parameters_list:
416
+ for key, value in param.items():
417
+ if value["kind"] == "json":
418
+ parameters[key] = JsonParameter(**value)
419
+ if value["kind"] == "metric":
420
+ parameters[key] = MetricParameter(**value)
421
+ if value["kind"] == "object":
422
+ parameters[key] = ObjectParameter(**value)
423
+ except EntityNotFoundError:
424
+ # No parameters are set
425
+ pass
426
+
427
+ return parameters
428
+
429
+ def set_parameters(self, run_id: str, parameters: dict):
430
+ """
431
+ Update the parameters of the Run log with the new parameters
432
+
433
+ This method would over-write the parameters, if the parameter exists in the run log already
434
+
435
+ The method should:
436
+ * Call get_run_log_by_id(run_id) to retrieve the run_log
437
+ * Update the parameters of the run_log
438
+ * Call put_run_log(run_log) to put the run_log in the datastore
439
+
440
+ Args:
441
+ run_id (str): The run_id of the run
442
+ parameters (dict): The parameters to update in the run log
443
+ Raises:
444
+ RunLogNotFoundError: If the run log for run_id is not found in the datastore
445
+ """
446
+ for key, value in parameters.items():
447
+ self.store(
448
+ run_id=run_id,
449
+ log_type=self.LogTypes.PARAMETER,
450
+ contents={key: value.model_dump(by_alias=True)},
451
+ name=key,
452
+ )
453
+
454
+ def get_run_config(self, run_id: str) -> dict:
455
+ """
456
+ Given a run_id, return the run_config used to perform the run.
457
+
458
+ Args:
459
+ run_id (str): The run_id of the run
460
+
461
+ Returns:
462
+ dict: The run config used for the run
463
+ """
464
+
465
+ run_log = self.get_run_log_by_id(run_id=run_id)
466
+ return run_log.run_config
467
+
468
+ def set_run_config(self, run_id: str, run_config: dict):
469
+ """Set the run config used to run the run_id
470
+
471
+ Args:
472
+ run_id (str): The run_id of the run
473
+ run_config (dict): The run_config of the run
474
+ """
475
+
476
+ run_log = self.get_run_log_by_id(run_id=run_id)
477
+ run_log.run_config.update(run_config)
478
+ self.put_run_log(run_log=run_log)
479
+
480
+ def get_step_log(self, internal_name: str, run_id: str) -> StepLog:
481
+ """
482
+ Get a step log from the datastore for run_id and the internal naming of the step log
483
+
484
+ The internal naming of the step log is a dot path convention.
485
+
486
+ The method should:
487
+ * Call get_run_log_by_id(run_id) to retrieve the run_log
488
+ * Identify the step location by decoding the internal naming
489
+ * Return the step log
490
+
491
+ Args:
492
+ internal_name (str): The internal name of the step log
493
+ run_id (str): The run_id of the run
494
+
495
+ Returns:
496
+ StepLog: The step log object for the step defined by the internal naming and run_id
497
+
498
+ Raises:
499
+ RunLogNotFoundError: If the run log for run_id is not found in the datastore
500
+ StepLogNotFoundError: If the step log for internal_name is not found in the datastore for run_id
501
+ """
502
+ logger.info(
503
+ f"{self.service_name} Getting the step log: {internal_name} of {run_id}"
504
+ )
505
+
506
+ step_log = self.retrieve(
507
+ run_id=run_id,
508
+ log_type=self.LogTypes.STEP_LOG,
509
+ name=internal_name,
510
+ multiple_allowed=False,
511
+ )
512
+
513
+ return step_log
514
+
515
+ def add_step_log(self, step_log: StepLog, run_id: str):
516
+ """
517
+ Add the step log in the run log as identified by the run_id in the datastore
518
+
519
+ The method should:
520
+ * Call get_run_log_by_id(run_id) to retrieve the run_log
521
+ * Identify the branch to add the step by decoding the step_logs internal name
522
+ * Add the step log to the identified branch log
523
+ * Call put_run_log(run_log) to put the run_log in the datastore
524
+
525
+ Args:
526
+ step_log (StepLog): The Step log to add to the database
527
+ run_id (str): The run id of the run
528
+
529
+ Raises:
530
+ RunLogNotFoundError: If the run log for run_id is not found in the datastore
531
+ BranchLogNotFoundError: If the branch of the step log for internal_name is not found in the datastore
532
+ for run_id
533
+ """
534
+ logger.info(
535
+ f"{self.service_name} Adding the step log to DB: {step_log.internal_name}"
536
+ )
537
+
538
+ self.store(
539
+ run_id=run_id,
540
+ log_type=self.LogTypes.STEP_LOG,
541
+ contents=step_log.model_dump(),
542
+ name=step_log.internal_name,
543
+ )
544
+
545
+ def get_branch_log(
546
+ self, internal_branch_name: str, run_id: str, **kwargs
547
+ ) -> Union[BranchLog, RunLog]:
548
+ """
549
+ Returns the branch log by the internal branch name for the run id
550
+
551
+ If the internal branch name is none, returns the run log
552
+
553
+ Args:
554
+ internal_branch_name (str): The internal branch name to retrieve.
555
+ run_id (str): The run id of interest
556
+
557
+ Returns:
558
+ BranchLog: The branch log or the run log as requested.
559
+ """
560
+ if not internal_branch_name:
561
+ return self.get_run_log_by_id(run_id=run_id)
562
+ branch = self.retrieve(
563
+ run_id=run_id, log_type=self.LogTypes.BRANCH_LOG, name=internal_branch_name
564
+ )
565
+ return branch
566
+
567
+ def add_branch_log(
568
+ self,
569
+ branch_log: Union[BranchLog, RunLog],
570
+ run_id: str,
571
+ ):
572
+ """
573
+ The method should:
574
+ # Get the run log
575
+ # Get the branch and step containing the branch
576
+ # Add the branch to the step
577
+ # Write the run_log
578
+
579
+ The branch log could some times be a Run log and should be handled appropriately
580
+
581
+ Args:
582
+ branch_log (BranchLog): The branch log/run log to add to the database
583
+ run_id (str): The run id to which the branch/run log is added
584
+ """
585
+ if not isinstance(branch_log, BranchLog):
586
+ self.put_run_log(branch_log)
587
+ return
588
+
589
+ internal_branch_name = branch_log.internal_name
590
+
591
+ logger.info(
592
+ f"{self.service_name} Adding the branch log to DB: {branch_log.internal_name}"
593
+ )
594
+ self.store(
595
+ run_id=run_id,
596
+ log_type=self.LogTypes.BRANCH_LOG,
597
+ contents=branch_log.model_dump(),
598
+ name=internal_branch_name,
599
+ )
@@ -0,0 +1,15 @@
1
+ [project]
2
+ name = "run_log_store"
3
+ version = "0.0.0"
4
+ description = "Extensions to run log store"
5
+ readme = "README.md"
6
+ requires-python = ">=3.10"
7
+ dependencies = []
8
+
9
+
10
+ [build-system]
11
+ requires = ["hatchling"]
12
+ build-backend = "hatchling.build"
13
+
14
+ [tool.hatch.build.targets.wheel]
15
+ packages = ["."]
File without changes
@@ -0,0 +1,62 @@
1
+ import logging
2
+
3
+ from dotenv import dotenv_values
4
+
5
+ from runnable import defaults, exceptions
6
+ from runnable.secrets import BaseSecrets
7
+
8
+ logger = logging.getLogger(defaults.LOGGER_NAME)
9
+
10
+
11
+ class DotEnvSecrets(BaseSecrets):
12
+ """
13
+ A secret manager which uses .env files for secrets.
14
+
15
+ We recommend this secrets manager only for local development and should not be used for anything close to
16
+ production.
17
+ """
18
+
19
+ service_name: str = "dotenv"
20
+ location: str = defaults.DOTENV_FILE_LOCATION
21
+ secrets: dict = {}
22
+
23
+ @property
24
+ def secrets_location(self):
25
+ """
26
+ Return the location of the .env file.
27
+ If the user has not over-ridden it, it defaults to .env file in the project root.
28
+
29
+ Returns:
30
+ str: The location of the secrets file
31
+ """
32
+ return self.location
33
+
34
+ def _load_secrets(self):
35
+ """
36
+ Use dotenv to load the secrets
37
+ """
38
+ self.secrets = dotenv_values(self.secrets_location)
39
+
40
+ def get(self, name: str = "", **kwargs) -> str:
41
+ """
42
+ Get a secret of name from the secrets file.
43
+
44
+
45
+ Args:
46
+ name (str): The name of the secret to retrieve
47
+
48
+ Raises:
49
+ Exception: If the secret by the name is not found.
50
+
51
+ Returns:
52
+ str: The value of the secret
53
+ """
54
+ if not self.secrets:
55
+ self._load_secrets()
56
+
57
+ if name in self.secrets:
58
+ return self.secrets[name]
59
+
60
+ raise exceptions.SecretNotFoundError(
61
+ secret_name=name, secret_setting=self.secrets_location
62
+ )
@@ -0,0 +1,15 @@
1
+ [project]
2
+ name = "secrets"
3
+ version = "0.0.0"
4
+ description = "Extension to manage secrets"
5
+ readme = "README.md"
6
+ requires-python = ">=3.10"
7
+ dependencies = []
8
+
9
+
10
+ [build-system]
11
+ requires = ["hatchling"]
12
+ build-backend = "hatchling.build"
13
+
14
+ [tool.hatch.build.targets.wheel]
15
+ packages = ["."]
runnable/__init__.py CHANGED
@@ -1,5 +1,6 @@
1
1
  # ruff: noqa
2
2
 
3
+
3
4
  import logging
4
5
  import os
5
6
  from logging.config import dictConfig