fred-oss 0.53.0__tar.gz → 0.55.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. fred_oss-0.55.0/MANIFEST.in +3 -0
  2. {fred_oss-0.53.0/src/main/fred_oss.egg-info → fred_oss-0.55.0}/PKG-INFO +1 -1
  3. {fred_oss-0.53.0 → fred_oss-0.55.0}/setup.py +2 -0
  4. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/dao/comp/_keyval.py +46 -1
  5. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/dao/comp/interface.py +7 -6
  6. fred_oss-0.55.0/src/main/fred/dao/service/_minio/__init__.py +1 -0
  7. fred_oss-0.55.0/src/main/fred/dao/service/_minio/policy/builder.py +16 -0
  8. fred_oss-0.55.0/src/main/fred/dao/service/_minio/policy/catalog.py +28 -0
  9. fred_oss-0.55.0/src/main/fred/dao/service/_minio/policy/loader.py +27 -0
  10. fred_oss-0.55.0/src/main/fred/dao/service/_minio/policy/templates/public_ro.json +24 -0
  11. fred_oss-0.55.0/src/main/fred/dao/service/_minio/policy/templates/public_rw.json +31 -0
  12. fred_oss-0.55.0/src/main/fred/dao/service/_minio/pool.py +55 -0
  13. fred_oss-0.53.0/src/main/fred/dao/service/_minio.py → fred_oss-0.55.0/src/main/fred/dao/service/_minio/service.py +13 -52
  14. fred_oss-0.55.0/src/main/fred/integrations/databricks/runtimes/16.4LTS.json +10 -0
  15. fred_oss-0.55.0/src/main/fred/version +1 -0
  16. fred_oss-0.55.0/src/main/fred/worker/runner/backend.py +42 -0
  17. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/client.py +27 -7
  18. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/handler.py +6 -4
  19. fred_oss-0.55.0/src/main/fred/worker/runner/rest/routers/__init__.py +0 -0
  20. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/status.py +6 -6
  21. {fred_oss-0.53.0 → fred_oss-0.55.0/src/main/fred_oss.egg-info}/PKG-INFO +1 -1
  22. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred_oss.egg-info/SOURCES.txt +10 -1
  23. fred_oss-0.53.0/MANIFEST.in +0 -2
  24. fred_oss-0.53.0/src/main/fred/version +0 -1
  25. fred_oss-0.53.0/src/main/fred/worker/runner/backend.py +0 -33
  26. {fred_oss-0.53.0 → fred_oss-0.55.0}/NOTICE.txt +0 -0
  27. {fred_oss-0.53.0 → fred_oss-0.55.0}/README.md +0 -0
  28. {fred_oss-0.53.0 → fred_oss-0.55.0}/requirements.txt +0 -0
  29. {fred_oss-0.53.0 → fred_oss-0.55.0}/setup.cfg +0 -0
  30. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/cli/__init__.py +0 -0
  31. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/cli/__main__.py +0 -0
  32. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/cli/interface.py +0 -0
  33. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/cli/main.py +0 -0
  34. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/dao/__init__.py +0 -0
  35. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/dao/comp/__init__.py +0 -0
  36. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/dao/comp/_pubsub.py +0 -0
  37. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/dao/comp/_queue.py +0 -0
  38. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/dao/comp/catalog.py +0 -0
  39. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/dao/service/__init__.py +0 -0
  40. {fred_oss-0.53.0/src/main/fred/future/callback → fred_oss-0.55.0/src/main/fred/dao/service/_minio/policy}/__init__.py +0 -0
  41. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/dao/service/_redis.py +0 -0
  42. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/dao/service/_stdlib.py +0 -0
  43. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/dao/service/catalog.py +0 -0
  44. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/dao/service/interface.py +0 -0
  45. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/dao/service/utils.py +0 -0
  46. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/future/__init__.py +0 -0
  47. {fred_oss-0.53.0/src/main/fred/integrations/databricks/runtimes → fred_oss-0.55.0/src/main/fred/future/callback}/__init__.py +0 -0
  48. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/future/callback/_function.py +0 -0
  49. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/future/callback/catalog.py +0 -0
  50. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/future/callback/interface.py +0 -0
  51. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/future/impl.py +0 -0
  52. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/future/result.py +0 -0
  53. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/future/settings.py +0 -0
  54. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/future/utils.py +0 -0
  55. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/integrations/databricks/__init__.py +0 -0
  56. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/integrations/databricks/cli_ext.py +0 -0
  57. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/integrations/databricks/runtime.py +0 -0
  58. {fred_oss-0.53.0/src/main/fred/integrations/databricks/wrappers → fred_oss-0.55.0/src/main/fred/integrations/databricks/runtimes}/__init__.py +0 -0
  59. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/integrations/databricks/runtimes/scanner.py +0 -0
  60. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/integrations/databricks/runtimes/sync.py +0 -0
  61. {fred_oss-0.53.0/src/main/fred/utils → fred_oss-0.55.0/src/main/fred/integrations/databricks/wrappers}/__init__.py +0 -0
  62. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/integrations/databricks/wrappers/dbutils.py +0 -0
  63. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/integrations/runpod/__init__.py +0 -0
  64. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/integrations/runpod/cli_ext.py +0 -0
  65. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/integrations/runpod/helper.py +0 -0
  66. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/maturity.py +0 -0
  67. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/monad/__init__.py +0 -0
  68. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/monad/_either.py +0 -0
  69. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/monad/catalog.py +0 -0
  70. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/monad/interface.py +0 -0
  71. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/settings.py +0 -0
  72. {fred_oss-0.53.0/src/main/fred/utils/imout → fred_oss-0.55.0/src/main/fred/utils}/__init__.py +0 -0
  73. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/utils/dateops.py +0 -0
  74. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/utils/imops.py +0 -0
  75. {fred_oss-0.53.0/src/main/fred/worker/runner/model → fred_oss-0.55.0/src/main/fred/utils/imout}/__init__.py +0 -0
  76. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/utils/imout/_filesystem.py +0 -0
  77. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/utils/imout/_minio.py +0 -0
  78. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/utils/imout/_string.py +0 -0
  79. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/utils/imout/catalog.py +0 -0
  80. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/utils/imout/interface.py +0 -0
  81. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/utils/mlops/__init__.py +0 -0
  82. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/utils/mlops/auto.py +0 -0
  83. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/utils/runtime.py +0 -0
  84. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/version.py +0 -0
  85. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/__init__.py +0 -0
  86. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/interface.py +0 -0
  87. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/__init__.py +0 -0
  88. {fred_oss-0.53.0/src/main/fred/worker/runner/plugins → fred_oss-0.55.0/src/main/fred/worker/runner/model}/__init__.py +0 -0
  89. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/model/_handler.py +0 -0
  90. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/model/_item.py +0 -0
  91. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/model/_request.py +0 -0
  92. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/model/_runner_spec.py +0 -0
  93. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/model/catalog.py +0 -0
  94. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/model/interface.py +0 -0
  95. {fred_oss-0.53.0/src/main/fred/worker/runner/rest → fred_oss-0.55.0/src/main/fred/worker/runner/plugins}/__init__.py +0 -0
  96. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/plugins/_local.py +0 -0
  97. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/plugins/_runpod.py +0 -0
  98. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/plugins/catalog.py +0 -0
  99. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/plugins/interface.py +0 -0
  100. {fred_oss-0.53.0/src/main/fred/worker/runner/rest/routers → fred_oss-0.55.0/src/main/fred/worker/runner/rest}/__init__.py +0 -0
  101. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/rest/auth.py +0 -0
  102. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/rest/cli_ext.py +0 -0
  103. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/rest/routers/_runner.py +0 -0
  104. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/rest/routers/catalog.py +0 -0
  105. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/rest/routers/interface.py +0 -0
  106. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/rest/server.py +0 -0
  107. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/rest/settings.py +0 -0
  108. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/settings.py +0 -0
  109. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/signal.py +0 -0
  110. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/runner/utils.py +0 -0
  111. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred/worker/settings.py +0 -0
  112. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred_oss.egg-info/dependency_links.txt +0 -0
  113. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred_oss.egg-info/entry_points.txt +0 -0
  114. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred_oss.egg-info/requires.txt +0 -0
  115. {fred_oss-0.53.0 → fred_oss-0.55.0}/src/main/fred_oss.egg-info/top_level.txt +0 -0
@@ -0,0 +1,3 @@
1
+ include requirements.txt
2
+ include src/main/fred/version
3
+ include src/main/fred/dao/service/_minio/policy/templates/*.json
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fred-oss
3
- Version: 0.53.0
3
+ Version: 0.55.0
4
4
  Summary: FREDOSS
5
5
  Home-page: https://fred.fahera.mx
6
6
  Author: Fahera Research, Education, and Development
@@ -35,6 +35,8 @@ setup(
35
35
  package_data={
36
36
  "": [
37
37
  version_filepath,
38
+ "**/*.json",
39
+ "**/*.yaml",
38
40
  ]
39
41
  },
40
42
  entry_points={
@@ -1,5 +1,5 @@
1
1
  from dataclasses import dataclass
2
- from typing import Optional
2
+ from typing import Iterator, Optional
3
3
 
4
4
  from fred.settings import logger_manager, get_environ_variable
5
5
  from fred.dao.service.catalog import ServiceCatalog
@@ -36,6 +36,51 @@ class FredKeyVal(ComponentInterface):
36
36
  """
37
37
  key: str
38
38
 
39
+ @classmethod
40
+ def keys(cls, pattern: Optional[str] = None, **kwargs) -> Iterator[str]:
41
+ """Returns a list of keys matching the given pattern.
42
+ The implementation of this method depends on the underlying service.
43
+ For example, if the service is Redis, it uses the KEYS command to get the
44
+ list of keys matching the pattern.
45
+ Args:
46
+ pattern (str): The pattern to match keys against. Defaults to "*".
47
+ Returns:
48
+ list[str]: A list of keys matching the pattern.
49
+ Raises:
50
+ NotImplementedError: If the method is not implemented for the current service.
51
+ """
52
+ match cls._cat:
53
+ case ServiceCatalog.REDIS:
54
+ return (
55
+ key if isinstance(key, str) else key.decode("utf-8")
56
+ for key in cls._srv.client.scan_iter(match=pattern, **kwargs)
57
+ )
58
+ case ServiceCatalog.STDLIB:
59
+ import fnmatch
60
+ pattern = pattern or "*"
61
+ return (
62
+ key
63
+ for key in cls._srv.client._memstore_keyval.keys()
64
+ if fnmatch.fnmatch(key, pattern=pattern)
65
+ )
66
+ case ServiceCatalog.MINIO:
67
+ import fnmatch
68
+ pattern = pattern or "*"
69
+ bucket_name = (
70
+ kwargs.get("bucket", None)
71
+ or kwargs.get("minio_bucket", None)
72
+ or get_environ_variable("MINIO_BUCKET")
73
+ )
74
+ if not bucket_name:
75
+ raise ValueError("Missing bucket info to list keys in MinIO service.")
76
+ return (
77
+ key
78
+ for obj in cls._srv.objects(bucket_name, **kwargs)
79
+ if fnmatch.fnmatch((key := obj.object_name), pattern=pattern)
80
+ )
81
+ case _:
82
+ raise NotImplementedError(f"Keys method not implemented for service {cls._nme}")
83
+
39
84
  def set(self, value: str, key: Optional[str] = None, **kwargs) -> None:
40
85
  """Sets a key-value pair in the store.
41
86
  The implementation of this method depends on the underlying service.
@@ -7,6 +7,7 @@ SRV_REF_TYPE = str | ServiceInterface | ServiceCatalog
7
7
 
8
8
  class SrvCompanionMixin:
9
9
  _srv: ServiceInterface
10
+ _cat: ServiceCatalog
10
11
 
11
12
  @classmethod
12
13
  def _set_srv(cls, srv_ref: Optional[SRV_REF_TYPE] = None, **kwargs):
@@ -19,11 +20,16 @@ class SrvCompanionMixin:
19
20
  """
20
21
  match (srv_ref or "REDIS"):
21
22
  case str() as name:
22
- cls._srv = ServiceCatalog[name.upper()].auto(**kwargs)
23
+ cls._cat = ServiceCatalog[name.upper()]
24
+ cls._srv = cls._cat.auto(**kwargs)
23
25
  case ServiceCatalog() as cat:
26
+ cls._cat = cat
24
27
  cls._srv = cat.auto(**kwargs)
25
28
  case ServiceInterface() as instance:
26
29
  cls._srv = instance
30
+ cls._cat = ServiceCatalog.from_classname(
31
+ classname=instance.__class__.__name__
32
+ )
27
33
  case _:
28
34
  raise ValueError(f"Invalid service '{srv_ref}' type: {type(srv_ref)}")
29
35
 
@@ -32,11 +38,6 @@ class SrvCompanionMixin:
32
38
  """Returns the class name of the current service instance."""
33
39
  return self._srv.__class__.__name__
34
40
 
35
- @property
36
- def _cat(self) -> ServiceCatalog:
37
- """Returns the ServiceCatalog enum member corresponding to the current service instance."""
38
- return ServiceCatalog.from_classname(self._nme)
39
-
40
41
 
41
42
  class ComponentInterface(SrvCompanionMixin):
42
43
 
@@ -0,0 +1 @@
1
+ from .service import MinioService
@@ -0,0 +1,16 @@
1
+ import enum
2
+
3
+
4
+ class MinioPolicyBuilderActionBucket(enum.StrEnum):
5
+ ALL = "s3:*"
6
+ LIST = "s3:ListBucket"
7
+ LOCATION = "s3:GetBucketLocation"
8
+ DELETE = "s3:DeleteBucket"
9
+ CREATE = "s3:CreateBucket"
10
+
11
+
12
+ class MinioPolicyBuilderActionObject(enum.StrEnum):
13
+ GET = "s3:GetObject"
14
+ PUT = "s3:PutObject"
15
+ DELETE = "s3:DeleteObject"
16
+ ALL = "s3:*"
@@ -0,0 +1,28 @@
1
+ import enum
2
+ from typing import Optional
3
+
4
+ from fred.dao.service._minio.policy.loader import MinioPolicyLoader
5
+
6
+
7
+ class MinioPolicyCatalog(enum.Enum):
8
+ BUCKET_PUBLIC_RO = MinioPolicyLoader(
9
+ title="[Bucket Policy] Public Read-Only",
10
+ filename="public_ro.json",
11
+ requires=[
12
+ "bucket_name",
13
+ ],
14
+ )
15
+ BUCKET_PUBLIC_RW = MinioPolicyLoader(
16
+ title="[Bucket Policy] Public Read-Write",
17
+ filename="public_rw.json",
18
+ requires=[
19
+ "bucket_name",
20
+ ],
21
+ )
22
+
23
+ def load(self, path: Optional[str] = None, **params) -> dict:
24
+ return self.value.load(path=path, **params)
25
+
26
+ def content(self, path: Optional[str] = None, **params) -> str:
27
+ import json
28
+ return json.dumps(self.load(path=path, **params))
@@ -0,0 +1,27 @@
1
+ import os
2
+ import json
3
+ from dataclasses import dataclass, field
4
+ from typing import Optional
5
+ def load_policy(filename: str, path: Optional[str] = None, **kwargs) -> dict:
6
+ path = path or os.path.join(os.path.dirname(__file__), "templates")
7
+ filepath = os.path.join(path, filename)
8
+ # Read the policy file as a string
9
+ with open(filepath, "r", encoding="utf-8") as file:
10
+ policy_str = file.read()
11
+ # Replace placeholders in the policy with actual values from kwargs
12
+ for param_key, param_val in kwargs.items():
13
+ param_ref = "${{param_key}}".replace("param_key", param_key)
14
+ policy_str = policy_str.replace(param_ref, param_val)
15
+ return json.loads(policy_str)
16
+
17
+
18
+ @dataclass(frozen=True, slots=True)
19
+ class MinioPolicyLoader:
20
+ title: str
21
+ filename: str
22
+ requires: list[str] = field(default_factory=list)
23
+
24
+ def load(self, path: Optional[str] = None, **kwargs) -> dict:
25
+ if (missing := set(self.requires) - set(kwargs.keys())):
26
+ raise ValueError(f"Missing required parameters for policy '{self.filename}': {missing}")
27
+ return load_policy(filename=self.filename, path=path, **kwargs)
@@ -0,0 +1,24 @@
1
+ {
2
+ "Version": "2012-10-17",
3
+ "Statement": [
4
+ {
5
+ "Effect": "Allow",
6
+ "Principal": {
7
+ "AWS": "*"
8
+ },
9
+ "Action": [
10
+ "s3:GetBucketLocation",
11
+ "s3:ListBucket"
12
+ ],
13
+ "Resource": "arn:aws:s3:::${{bucket_name}}"
14
+ },
15
+ {
16
+ "Effect": "Allow",
17
+ "Principal": {
18
+ "AWS": "*"
19
+ },
20
+ "Action": "s3:GetObject",
21
+ "Resource": "arn:aws:s3:::${{bucket_name}}/*"
22
+ }
23
+ ]
24
+ }
@@ -0,0 +1,31 @@
1
+ {
2
+ "Version": "2012-10-17",
3
+ "Statement": [
4
+ {
5
+ "Effect": "Allow",
6
+ "Principal": {
7
+ "AWS": "*"
8
+ },
9
+ "Action": [
10
+ "s3:GetBucketLocation",
11
+ "s3:ListBucket",
12
+ "s3:ListBucketMultipartUploads"
13
+ ],
14
+ "Resource": "arn:aws:s3:::${{bucket_name}}"
15
+ },
16
+ {
17
+ "Effect": "Allow",
18
+ "Principal": {
19
+ "AWS": "*"
20
+ },
21
+ "Action": [
22
+ "s3:GetObject",
23
+ "s3:PutObject",
24
+ "s3:DeleteObject",
25
+ "s3:ListMultipartUploadParts",
26
+ "s3:AbortMultipartUpload"
27
+ ],
28
+ "Resource": "arn:aws:s3:::${{bucket_name}}/*"
29
+ }
30
+ ]
31
+ }
@@ -0,0 +1,55 @@
1
+ from urllib3 import PoolManager, Retry
2
+ from urllib3.util import Timeout
3
+
4
+ from fred.settings import get_environ_variable, logger_manager
5
+ from fred.dao.service.interface import ServiceConnectionPoolInterface
6
+
7
+ logger = logger_manager.get_logger(name=__name__)
8
+
9
+
10
+ class MinioConnectionPool(ServiceConnectionPoolInterface[PoolManager]):
11
+
12
+ @classmethod
13
+ def _create_pool(cls, disable_cert: bool = False, **kwargs) -> PoolManager:
14
+ """Create a urllib3 PoolManager with the given configurations.
15
+
16
+ TODO: Consider using the inverse of 'require_cert' as the default to ensure we do have cert-check automatically.
17
+ For now, we keep it as is to avoid breaking changes.
18
+
19
+ Args:
20
+ require_cert (bool): Whether to require SSL certificate verification.
21
+ **kwargs: Additional keyword arguments to pass to the PoolManager constructor.
22
+ Returns:
23
+ PoolManager: A configured PoolManager instance.
24
+ """
25
+ num_pools = kwargs.pop("num_pools", 10)
26
+ maxsize = kwargs.pop("maxsize", 10)
27
+ # Default timeout of 5 minutes
28
+ timeout_seconds = kwargs.pop("timeout", 300)
29
+ timeout = Timeout(
30
+ connect=timeout_seconds,
31
+ read=timeout_seconds,
32
+ )
33
+ # Default retries of 5 with exponential backoff
34
+ retry = Retry(
35
+ total=kwargs.pop("retries", 5),
36
+ backoff_factor=kwargs.pop("backoff_factor", 0.25),
37
+ status_forcelist=[500, 502, 503, 504],
38
+ )
39
+ # Configure certificate requirements for SSL connections
40
+ cert_reqs = "CERT_NONE"
41
+ ca_certs = None
42
+ if not disable_cert:
43
+ import certifi
44
+ cert_reqs = "CERT_REQUIRED"
45
+ ca_certs = get_environ_variable("SSL_CERT_FILE") or certifi.where()
46
+ # Finally, create and return the PoolManager instance
47
+ return PoolManager(
48
+ num_pools=num_pools,
49
+ maxsize=maxsize,
50
+ timeout=timeout,
51
+ retries=retry,
52
+ cert_reqs=cert_reqs,
53
+ ca_certs=ca_certs,
54
+ **kwargs
55
+ )
@@ -1,62 +1,14 @@
1
1
  from minio import Minio
2
- from urllib3 import PoolManager, Retry
3
- from urllib3.util import Timeout
4
2
 
5
- from fred.settings import get_environ_variable, logger_manager
3
+ from fred.settings import logger_manager
4
+ from fred.dao.service.interface import ServiceInterface
6
5
  from fred.dao.service.utils import get_minio_from_payload
7
- from fred.dao.service.interface import ServiceInterface, ServiceConnectionPoolInterface
6
+ from fred.dao.service._minio.pool import MinioConnectionPool
7
+ from fred.dao.service._minio.policy.catalog import MinioPolicyCatalog
8
8
 
9
9
  logger = logger_manager.get_logger(name=__name__)
10
10
 
11
11
 
12
- class MinioConnectionPool(ServiceConnectionPoolInterface[PoolManager]):
13
-
14
- @classmethod
15
- def _create_pool(cls, disable_cert: bool = False, **kwargs) -> PoolManager:
16
- """Create a urllib3 PoolManager with the given configurations.
17
-
18
- TODO: Consider using the inverse of 'require_cert' as the default to ensure we do have cert-check automatically.
19
- For now, we keep it as is to avoid breaking changes.
20
-
21
- Args:
22
- require_cert (bool): Whether to require SSL certificate verification.
23
- **kwargs: Additional keyword arguments to pass to the PoolManager constructor.
24
- Returns:
25
- PoolManager: A configured PoolManager instance.
26
- """
27
- num_pools = kwargs.pop("num_pools", 10)
28
- maxsize = kwargs.pop("maxsize", 10)
29
- # Default timeout of 5 minutes
30
- timeout_seconds = kwargs.pop("timeout", 300)
31
- timeout = Timeout(
32
- connect=timeout_seconds,
33
- read=timeout_seconds,
34
- )
35
- # Default retries of 5 with exponential backoff
36
- retry = Retry(
37
- total=kwargs.pop("retries", 5),
38
- backoff_factor=kwargs.pop("backoff_factor", 0.25),
39
- status_forcelist=[500, 502, 503, 504],
40
- )
41
- # Configure certificate requirements for SSL connections
42
- cert_reqs = "CERT_NONE"
43
- ca_certs = None
44
- if not disable_cert:
45
- import certifi
46
- cert_reqs = "CERT_REQUIRED"
47
- ca_certs = get_environ_variable("SSL_CERT_FILE") or certifi.where()
48
- # Finally, create and return the PoolManager instance
49
- return PoolManager(
50
- num_pools=num_pools,
51
- maxsize=maxsize,
52
- timeout=timeout,
53
- retries=retry,
54
- cert_reqs=cert_reqs,
55
- ca_certs=ca_certs,
56
- **kwargs
57
- )
58
-
59
-
60
12
  class MinioService(ServiceInterface[Minio]):
61
13
  instance: Minio
62
14
  metadata: dict = {}
@@ -120,3 +72,12 @@ class MinioService(ServiceInterface[Minio]):
120
72
  except S3Error:
121
73
  logger.debug(f"Object {object_name} in bucket {bucket_name} does not exist.")
122
74
  return False
75
+
76
+ def make_bucket_public(self, bucket_name: str, readonly: bool = False):
77
+ """Make a bucket public with either read-only or read-write access."""
78
+ policy = MinioPolicyCatalog.BUCKET_PUBLIC_RO \
79
+ if readonly else MinioPolicyCatalog.BUCKET_PUBLIC_RW
80
+ self.client.set_bucket_policy(
81
+ bucket_name=bucket_name,
82
+ policy=policy.content(bucket_name=bucket_name)
83
+ )
@@ -0,0 +1,10 @@
1
+ {
2
+ "python_version": "3.12.3",
3
+ "databricks_runtime": "16.4 LTS",
4
+ "libraries": [
5
+ {
6
+ "name": "pydantic",
7
+ "version": "2.8.2"
8
+ }
9
+ ]
10
+ }
@@ -0,0 +1 @@
1
+ 0.55.0
@@ -0,0 +1,42 @@
1
+ from dataclasses import dataclass
2
+
3
+ from fred.settings import logger_manager
4
+ from fred.dao.comp.catalog import FredKeyVal, FredQueue, CompCatalog
5
+ from fred.dao.service.interface import ServiceInterface
6
+ from fred.dao.service.catalog import ServiceCatalog
7
+
8
+ logger = logger_manager.get_logger(name=__name__)
9
+
10
+
11
+ @dataclass(frozen=True, slots=False)
12
+ class RunnerBackend:
13
+ keyval: FredKeyVal
14
+ queue: FredQueue
15
+ # NOTE: Catalog and service instance references (for internal use)
16
+ # should be a temporal fix until we refactor the backend system
17
+ # according to a more precise usage-patter (still to be defined/identified).
18
+ _cat: ServiceCatalog # Contains complementary service info and simple references
19
+ _srv: ServiceInterface # Allows direct access to a client instance
20
+
21
+ @classmethod
22
+ def auto(cls, service_name: str, **kwargs) -> 'RunnerBackend':
23
+ match (srv_catalog := ServiceCatalog[service_name.upper()]):
24
+ case ServiceCatalog.REDIS:
25
+ from fred.dao.service.utils import get_redis_configs_from_payload
26
+ service_kwargs = get_redis_configs_from_payload(kwargs)
27
+ case ServiceCatalog.STDLIB:
28
+ service_kwargs = {}
29
+ case _:
30
+ logger.error(
31
+ f"Unknown service '{service_name}'... "
32
+ "will attempt to use provided kwargs as-is."
33
+ )
34
+ service_kwargs = kwargs
35
+ logger.info(f"Initializing RunnerBackend using service '{service_name}'")
36
+ srv_instance = srv_catalog.auto(**service_kwargs)
37
+ return cls(
38
+ keyval=CompCatalog.KEYVAL.value.mount(srv_ref=srv_instance),
39
+ queue=CompCatalog.QUEUE.value.mount(srv_ref=srv_instance),
40
+ _cat=srv_catalog,
41
+ _srv=srv_instance,
42
+ )
@@ -61,14 +61,34 @@ class RunnerClient:
61
61
  signal = RunnerSignal[signal.upper()] if isinstance(signal, str) else signal
62
62
  return signal.send(self.req_queue)
63
63
 
64
- def runner_status(self, runner_id: str) -> RunnerStatus:
65
- runner_status=self._runner_backend.keyval(
66
- key=RunnerStatus.get_key(runner_id=runner_id)
67
- )
68
- if not (value := runner_status.get()):
64
+ def runner_info(self, runner_id: str) -> tuple[str, RunnerStatus]:
65
+ runner_status = self._runner_backend.keyval(
66
+ key=RunnerStatus.get_key(runner_id=runner_id)
67
+ )
68
+ if not (out := runner_status.get()):
69
69
  logger.warning(f"No status found for runner_id: '{runner_id}'")
70
- return RunnerStatus.UNDEFINED
71
- return RunnerStatus.parse_value(value=value)
70
+ return ("", RunnerStatus.UNDEFINED)
71
+ return RunnerStatus.parse_value(value=out)
72
+
73
+ def runner_status(self, runner_id: str) -> RunnerStatus:
74
+ _, status = self.runner_info(runner_id=runner_id)
75
+ return status
76
+
77
+ def runner_queue(self, runner_id: str) -> str:
78
+ queue_slug, _ = self.runner_info(runner_id=runner_id)
79
+ return queue_slug
80
+
81
+ def runners(self) -> dict[str, Optional[str]]:
82
+ return {
83
+ key: self._runner_backend.keyval(key=key).get()
84
+ for key in self._runner_backend.keyval.keys(pattern="frd:runner:*")
85
+ }
86
+
87
+ def futures(self) -> dict[str, Optional[str]]:
88
+ return {
89
+ key: self._runner_backend.keyval(key=key).get()
90
+ for key in self._runner_backend.keyval.keys(pattern="frd:future:*:status")
91
+ }
72
92
 
73
93
  def send(
74
94
  self,
@@ -140,9 +140,11 @@ class RunnerHandler(HandlerInterface):
140
140
  runner_status = runner_backend.keyval(
141
141
  key=RunnerStatus.get_key(runner_id=runner_id)
142
142
  )
143
- logger.info(f"Starting runner with ID '{runner_id}' using request-queue '{req_queue.name}'")
143
+ # Start the runner loop in a future and track its status
144
+ on_start_queue_size = req_queue.size()
145
+ logger.info(f"Starting runner '{runner_id}' using req-queue '{req_queue.name}': {on_start_queue_size}")
144
146
  runner_status.set(
145
- value=RunnerStatus.STARTED.get_val(),
147
+ value=RunnerStatus.STARTED.get_val(spec.queue_slug, f"Q({on_start_queue_size})"),
146
148
  expire=None,
147
149
  )
148
150
  runner_loop = Future(
@@ -156,7 +158,7 @@ class RunnerHandler(HandlerInterface):
156
158
  future_id=runner_id,
157
159
  )
158
160
  runner_status.set(
159
- value=RunnerStatus.RUNNING.get_val(),
161
+ value=RunnerStatus.RUNNING.get_val(spec.queue_slug, f"Q({req_queue.size()})"),
160
162
  expire=None,
161
163
  )
162
164
  results = {
@@ -180,7 +182,7 @@ class RunnerHandler(HandlerInterface):
180
182
  }
181
183
  results["pending_requests"] = pending_requests = req_queue.size()
182
184
  runner_status.set(
183
- value=RunnerStatus.STOPPED.get_val(str(pending_requests)),
185
+ value=RunnerStatus.STOPPED.get_val(spec.queue_slug, f"Q({pending_requests})"),
184
186
  expire=3600, # Keep the stopped status for 1 hour
185
187
  )
186
188
  if pending_requests:
@@ -14,14 +14,14 @@ class RunnerStatus(Enum):
14
14
  @staticmethod
15
15
  def get_key(runner_id: str) -> str:
16
16
  """Get the status key for the specified runner ID."""
17
- return f"runner:status:{runner_id}"
17
+ return f"frd:runner:{runner_id}"
18
18
 
19
- def get_val(self, *include) -> str:
19
+ def get_val(self, queue_slug: str, *include) -> str:
20
20
  """Get the string representation of the status."""
21
- return ":".join([self.name, datetime_utcnow().isoformat(), *include])
21
+ return ":".join([queue_slug, self.name, datetime_utcnow().isoformat(), *include])
22
22
 
23
23
  @classmethod
24
- def parse_value(cls, value: str) -> "RunnerStatus":
24
+ def parse_value(cls, value: str) -> tuple[str, "RunnerStatus"]:
25
25
  """Parse the status from the stored value."""
26
- val, *_ = value.split(":")
27
- return cls[val.upper()]
26
+ queue_slug, status, *_ = value.split(":")
27
+ return (queue_slug, cls[status.upper()])
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fred-oss
3
- Version: 0.53.0
3
+ Version: 0.55.0
4
4
  Summary: FREDOSS
5
5
  Home-page: https://fred.fahera.mx
6
6
  Author: Fahera Research, Education, and Development
@@ -19,12 +19,20 @@ src/main/fred/dao/comp/_queue.py
19
19
  src/main/fred/dao/comp/catalog.py
20
20
  src/main/fred/dao/comp/interface.py
21
21
  src/main/fred/dao/service/__init__.py
22
- src/main/fred/dao/service/_minio.py
23
22
  src/main/fred/dao/service/_redis.py
24
23
  src/main/fred/dao/service/_stdlib.py
25
24
  src/main/fred/dao/service/catalog.py
26
25
  src/main/fred/dao/service/interface.py
27
26
  src/main/fred/dao/service/utils.py
27
+ src/main/fred/dao/service/_minio/__init__.py
28
+ src/main/fred/dao/service/_minio/pool.py
29
+ src/main/fred/dao/service/_minio/service.py
30
+ src/main/fred/dao/service/_minio/policy/__init__.py
31
+ src/main/fred/dao/service/_minio/policy/builder.py
32
+ src/main/fred/dao/service/_minio/policy/catalog.py
33
+ src/main/fred/dao/service/_minio/policy/loader.py
34
+ src/main/fred/dao/service/_minio/policy/templates/public_ro.json
35
+ src/main/fred/dao/service/_minio/policy/templates/public_rw.json
28
36
  src/main/fred/future/__init__.py
29
37
  src/main/fred/future/impl.py
30
38
  src/main/fred/future/result.py
@@ -37,6 +45,7 @@ src/main/fred/future/callback/interface.py
37
45
  src/main/fred/integrations/databricks/__init__.py
38
46
  src/main/fred/integrations/databricks/cli_ext.py
39
47
  src/main/fred/integrations/databricks/runtime.py
48
+ src/main/fred/integrations/databricks/runtimes/16.4LTS.json
40
49
  src/main/fred/integrations/databricks/runtimes/__init__.py
41
50
  src/main/fred/integrations/databricks/runtimes/scanner.py
42
51
  src/main/fred/integrations/databricks/runtimes/sync.py
@@ -1,2 +0,0 @@
1
- include requirements.txt
2
- include src/main/fred/version
@@ -1 +0,0 @@
1
- 0.53.0
@@ -1,33 +0,0 @@
1
- from dataclasses import dataclass
2
-
3
- from fred.settings import logger_manager
4
- from fred.dao.comp.catalog import FredKeyVal, FredQueue
5
- from fred.dao.service.catalog import ServiceCatalog
6
-
7
- logger = logger_manager.get_logger(name=__name__)
8
-
9
-
10
- @dataclass(frozen=True, slots=False)
11
- class RunnerBackend:
12
- keyval: FredKeyVal
13
- queue: FredQueue
14
- _cat: ServiceCatalog
15
-
16
- @classmethod
17
- def auto(cls, service_name: str, **kwargs) -> 'RunnerBackend':
18
- service = ServiceCatalog[service_name.upper()]
19
- match service:
20
- case ServiceCatalog.REDIS:
21
- from fred.dao.service.utils import get_redis_configs_from_payload
22
- service_kwargs = get_redis_configs_from_payload(kwargs)
23
- case ServiceCatalog.STDLIB:
24
- service_kwargs = {}
25
- case _:
26
- logger.error(f"Unknown service '{service_name}'... will attempt to use provided kwargs as-is.")
27
- service_kwargs = kwargs
28
- components = service.component_catalog(**service_kwargs)
29
- return cls(
30
- keyval=components.KEYVAL.value,
31
- queue=components.QUEUE.value,
32
- _cat=service,
33
- )
File without changes
File without changes
File without changes
File without changes