fred-oss 0.48.0__tar.gz → 0.50.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. {fred_oss-0.48.0/src/main/fred_oss.egg-info → fred_oss-0.50.0}/PKG-INFO +2 -1
  2. {fred_oss-0.48.0 → fred_oss-0.50.0}/requirements.txt +2 -0
  3. fred_oss-0.50.0/src/main/fred/dao/comp/_keyval.py +176 -0
  4. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/dao/comp/catalog.py +13 -0
  5. fred_oss-0.50.0/src/main/fred/dao/service/_minio.py +122 -0
  6. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/dao/service/catalog.py +2 -0
  7. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/dao/service/interface.py +1 -0
  8. fred_oss-0.50.0/src/main/fred/dao/service/utils.py +76 -0
  9. fred_oss-0.50.0/src/main/fred/utils/imout/_filesystem.py +11 -0
  10. fred_oss-0.50.0/src/main/fred/utils/imout/_string.py +10 -0
  11. fred_oss-0.50.0/src/main/fred/utils/imout/catalog.py +18 -0
  12. fred_oss-0.50.0/src/main/fred/utils/imout/interface.py +37 -0
  13. fred_oss-0.50.0/src/main/fred/version +1 -0
  14. fred_oss-0.50.0/src/main/fred/worker/runner/rest/routers/__init__.py +0 -0
  15. {fred_oss-0.48.0 → fred_oss-0.50.0/src/main/fred_oss.egg-info}/PKG-INFO +2 -1
  16. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred_oss.egg-info/SOURCES.txt +6 -0
  17. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred_oss.egg-info/requires.txt +1 -0
  18. fred_oss-0.48.0/src/main/fred/dao/comp/_keyval.py +0 -93
  19. fred_oss-0.48.0/src/main/fred/dao/service/utils.py +0 -37
  20. fred_oss-0.48.0/src/main/fred/version +0 -1
  21. {fred_oss-0.48.0 → fred_oss-0.50.0}/MANIFEST.in +0 -0
  22. {fred_oss-0.48.0 → fred_oss-0.50.0}/NOTICE.txt +0 -0
  23. {fred_oss-0.48.0 → fred_oss-0.50.0}/README.md +0 -0
  24. {fred_oss-0.48.0 → fred_oss-0.50.0}/setup.cfg +0 -0
  25. {fred_oss-0.48.0 → fred_oss-0.50.0}/setup.py +0 -0
  26. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/cli/__init__.py +0 -0
  27. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/cli/__main__.py +0 -0
  28. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/cli/interface.py +0 -0
  29. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/cli/main.py +0 -0
  30. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/dao/__init__.py +0 -0
  31. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/dao/comp/__init__.py +0 -0
  32. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/dao/comp/_pubsub.py +0 -0
  33. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/dao/comp/_queue.py +0 -0
  34. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/dao/comp/interface.py +0 -0
  35. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/dao/service/__init__.py +0 -0
  36. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/dao/service/_redis.py +0 -0
  37. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/dao/service/_stdlib.py +0 -0
  38. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/future/__init__.py +0 -0
  39. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/future/callback/__init__.py +0 -0
  40. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/future/callback/_function.py +0 -0
  41. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/future/callback/catalog.py +0 -0
  42. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/future/callback/interface.py +0 -0
  43. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/future/impl.py +0 -0
  44. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/future/result.py +0 -0
  45. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/future/settings.py +0 -0
  46. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/future/utils.py +0 -0
  47. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/integrations/databricks/__init__.py +0 -0
  48. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/integrations/databricks/cli_ext.py +0 -0
  49. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/integrations/databricks/runtime.py +0 -0
  50. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/integrations/databricks/runtimes/__init__.py +0 -0
  51. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/integrations/databricks/runtimes/scanner.py +0 -0
  52. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/integrations/databricks/runtimes/sync.py +0 -0
  53. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/integrations/databricks/wrappers/__init__.py +0 -0
  54. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/integrations/databricks/wrappers/dbutils.py +0 -0
  55. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/integrations/runpod/__init__.py +0 -0
  56. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/integrations/runpod/cli_ext.py +0 -0
  57. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/integrations/runpod/helper.py +0 -0
  58. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/maturity.py +0 -0
  59. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/monad/__init__.py +0 -0
  60. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/monad/_either.py +0 -0
  61. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/monad/catalog.py +0 -0
  62. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/monad/interface.py +0 -0
  63. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/settings.py +0 -0
  64. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/utils/__init__.py +0 -0
  65. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/utils/dateops.py +0 -0
  66. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/utils/imops.py +0 -0
  67. {fred_oss-0.48.0/src/main/fred/worker/runner/model → fred_oss-0.50.0/src/main/fred/utils/imout}/__init__.py +0 -0
  68. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/utils/runtime.py +0 -0
  69. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/version.py +0 -0
  70. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/__init__.py +0 -0
  71. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/interface.py +0 -0
  72. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/__init__.py +0 -0
  73. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/backend.py +0 -0
  74. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/client.py +0 -0
  75. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/handler.py +0 -0
  76. {fred_oss-0.48.0/src/main/fred/worker/runner/plugins → fred_oss-0.50.0/src/main/fred/worker/runner/model}/__init__.py +0 -0
  77. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/model/_handler.py +0 -0
  78. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/model/_item.py +0 -0
  79. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/model/_request.py +0 -0
  80. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/model/_runner_spec.py +0 -0
  81. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/model/catalog.py +0 -0
  82. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/model/interface.py +0 -0
  83. {fred_oss-0.48.0/src/main/fred/worker/runner/rest → fred_oss-0.50.0/src/main/fred/worker/runner/plugins}/__init__.py +0 -0
  84. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/plugins/_local.py +0 -0
  85. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/plugins/_runpod.py +0 -0
  86. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/plugins/catalog.py +0 -0
  87. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/plugins/interface.py +0 -0
  88. {fred_oss-0.48.0/src/main/fred/worker/runner/rest/routers → fred_oss-0.50.0/src/main/fred/worker/runner/rest}/__init__.py +0 -0
  89. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/rest/auth.py +0 -0
  90. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/rest/cli_ext.py +0 -0
  91. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/rest/routers/_runner.py +0 -0
  92. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/rest/routers/catalog.py +0 -0
  93. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/rest/routers/interface.py +0 -0
  94. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/rest/server.py +0 -0
  95. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/rest/settings.py +0 -0
  96. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/settings.py +0 -0
  97. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/signal.py +0 -0
  98. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/status.py +0 -0
  99. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/runner/utils.py +0 -0
  100. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred/worker/settings.py +0 -0
  101. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred_oss.egg-info/dependency_links.txt +0 -0
  102. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred_oss.egg-info/entry_points.txt +0 -0
  103. {fred_oss-0.48.0 → fred_oss-0.50.0}/src/main/fred_oss.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fred-oss
3
- Version: 0.48.0
3
+ Version: 0.50.0
4
4
  Summary: FREDOSS
5
5
  Home-page: https://fred.fahera.mx
6
6
  Author: Fahera Research, Education, and Development
@@ -14,6 +14,7 @@ Requires-Dist: redis==6.4.0
14
14
  Requires-Dist: requests==2.32.5
15
15
  Requires-Dist: fastapi==0.116.2
16
16
  Requires-Dist: uvicorn[standard]==0.35.0
17
+ Requires-Dist: minio==7.2.18
17
18
  Requires-Dist: pillow==11.3.0
18
19
  Dynamic: author
19
20
  Dynamic: author-email
@@ -7,6 +7,8 @@ redis==6.4.0
7
7
  requests==2.32.5
8
8
  fastapi==0.116.2
9
9
  uvicorn[standard]==0.35.0
10
+ # Services
11
+ minio==7.2.18
10
12
  # Fred Utils
11
13
  pillow==11.3.0
12
14
  # Databricks Runtime: 16.4 LTS (Python 3.12.3)
@@ -0,0 +1,176 @@
1
+ from dataclasses import dataclass
2
+ from typing import Optional
3
+
4
+ from fred.settings import logger_manager, get_environ_variable
5
+ from fred.dao.service.catalog import ServiceCatalog
6
+ from fred.dao.comp.interface import ComponentInterface
7
+
8
+ logger = logger_manager.get_logger(name=__name__)
9
+
10
+
11
+ def _get_minio_elements_from_key(key: str, **kwargs) -> tuple[str, str]:
12
+ import os
13
+
14
+ fullpath = os.path.join(
15
+ kwargs.pop("bucket_name", get_environ_variable("MINIO_BUCKET")) or "",
16
+ key
17
+ )
18
+ bucket_name = os.path.dirname(fullpath)
19
+ object_name = os.path.basename(fullpath)
20
+ if not bucket_name:
21
+ raise ValueError(
22
+ "Bucket name must be specified either in kwargs, environment variable MINIO_BUCKET, "
23
+ "or implicitly as part of the key."
24
+ )
25
+ if not object_name:
26
+ raise ValueError("Object name cannot be empty.")
27
+ return bucket_name, object_name
28
+
29
+
30
+ @dataclass(frozen=True, slots=True)
31
+ class FredKeyVal(ComponentInterface):
32
+ """A simple key-value store implementation using a backend service.
33
+ This class provides methods to interact with a key-value store, such as setting,
34
+ getting, and deleting key-value pairs. The actual implementation of these methods
35
+ depends on the underlying service being used (e.g., Redis).
36
+ """
37
+ key: str
38
+
39
+ def set(self, value: str, key: Optional[str] = None, **kwargs) -> None:
40
+ """Sets a key-value pair in the store.
41
+ The implementation of this method depends on the underlying service.
42
+ For example, if the service is Redis, it uses the SET command to store the
43
+ key-value pair.
44
+ Args:
45
+ key (str): The key to set.
46
+ value (str): The value to associate with the key.
47
+ **kwargs: Additional keyword arguments for setting the key-value pair,
48
+ such as expiration time.
49
+ Raises:
50
+ NotImplementedError: If the method is not implemented for the current service.
51
+ """
52
+ key = key or self.key
53
+ match self._cat:
54
+ case ServiceCatalog.REDIS:
55
+ self._srv.client.set(key, value)
56
+ expire = kwargs.get("expire")
57
+ if expire and isinstance(expire, int):
58
+ self._srv.client.expire(key, expire)
59
+ case ServiceCatalog.STDLIB:
60
+ self._srv.client._memstore_keyval[key] = value
61
+ # TODO: Implement expiration logic
62
+ if "expire" in kwargs:
63
+ logger.warning("Expiration not implemented for STDLIB service.")
64
+ case ServiceCatalog.MINIO:
65
+ # MinIO is not a key-value store, but we can simulate it:
66
+ # the key will be the object name, and the value will be the object content.
67
+ import io
68
+
69
+ bucket_name, object_name = _get_minio_elements_from_key(key, **kwargs)
70
+ # Ensure the bucket exists or create otherwise
71
+ if not self._srv.bucket_exists(bucket_name):
72
+ logger.warning(f"Creating bucket since doesn't exists: {bucket_name}")
73
+ self._srv.client.make_bucket(bucket_name)
74
+ if "expire" in kwargs:
75
+ # TODO: Implement expiration logic
76
+ logger.warning("Expiration not implemented for MINIO service.")
77
+ # Prepare the value as a byte stream
78
+ value_bytes = value.encode("utf-8")
79
+ value_stream = io.BytesIO(value_bytes)
80
+ value_stream.seek(0) # Ensure the stream is at the beginning
81
+ # Put the object into the bucket
82
+ self._srv.client.put_object(
83
+ bucket_name=bucket_name,
84
+ object_name=object_name,
85
+ data=value_stream,
86
+ length=len(value_bytes),
87
+ )
88
+ case _:
89
+ raise NotImplementedError(f"Set method not implemented for service {self._nme}")
90
+
91
+ def get(self, key: Optional[str] = None, fail: bool = False) -> Optional[str]:
92
+ """Gets the value associated with a key from the store.
93
+ The implementation of this method depends on the underlying service.
94
+ For example, if the service is Redis, it uses the GET command to retrieve the
95
+ value associated with the key.
96
+ Args:
97
+ key (str): The key to retrieve.
98
+ fail (bool): If True, raises a KeyError if the key is not found. Defaults to False.
99
+ Returns:
100
+ Optional[str]: The value associated with the key, or None if the key is not found
101
+ and fail is False.
102
+ Raises:
103
+ KeyError: If the key is not found and fail is True.
104
+ NotImplementedError: If the method is not implemented for the current service.
105
+ """
106
+ key = key or self.key
107
+ result = None
108
+ match self._cat:
109
+ case ServiceCatalog.REDIS:
110
+ result = self._srv.client.get(key)
111
+ case ServiceCatalog.STDLIB:
112
+ result = self._srv.client._memstore_keyval.get(key)
113
+ case ServiceCatalog.MINIO:
114
+ bucket_name, object_name = _get_minio_elements_from_key(key)
115
+ # Verify if the bucket exists
116
+ if not self._srv.bucket_exists(bucket_name):
117
+ logger.warning(f"Bucket {bucket_name} does not exist.")
118
+ if fail:
119
+ raise KeyError(f"Bucket {bucket_name} does not exist.")
120
+ return None
121
+ # Verify if the object exists
122
+ if not self._srv.object_exists(bucket_name, object_name):
123
+ logger.warning(f"Object {object_name} in bucket {bucket_name} does not exist.")
124
+ if fail:
125
+ raise KeyError(f"Object {object_name} not found in bucket {bucket_name}.")
126
+ return None
127
+ try:
128
+ response = self._srv.client.get_object(bucket_name, object_name)
129
+ result_bytes = response.read()
130
+ result = result_bytes.decode("utf-8")
131
+ except Exception as e:
132
+ logger.error(f"Error retrieving object {object_name} from bucket {bucket_name}: {e}")
133
+ result = None
134
+ if fail:
135
+ raise KeyError(f"Object {object_name} not found in bucket {bucket_name}.")
136
+ finally:
137
+ response.close()
138
+ response.release_conn()
139
+ case _:
140
+ raise NotImplementedError(f"Get method not implemented for service {self._nme}")
141
+ if fail and result is None:
142
+ raise KeyError(f"Key {key} not found.")
143
+ return result
144
+
145
+ def delete(self, key: Optional[str] = None) -> None:
146
+ """Deletes a key-value pair from the store.
147
+ The implementation of this method depends on the underlying service.
148
+ For example, if the service is Redis, it uses the DEL command to remove the
149
+ key-value pair.
150
+ Args:
151
+ key (str): The key to delete.
152
+ Raises:
153
+ NotImplementedError: If the method is not implemented for the current service.
154
+ """
155
+ key = key or self.key
156
+ match self._cat:
157
+ case ServiceCatalog.REDIS:
158
+ self._srv.client.delete(key)
159
+ case ServiceCatalog.STDLIB:
160
+ self._srv.client._memstore_keyval.pop(key, None)
161
+ case ServiceCatalog.MINIO:
162
+ bucket_name, object_name = _get_minio_elements_from_key(key)
163
+ # Verify if the bucket exists
164
+ if not self._srv.bucket_exists(bucket_name):
165
+ logger.warning(f"Bucket {bucket_name} does not exist.")
166
+ return
167
+ # Verify if the object exists
168
+ if not self._srv.object_exists(bucket_name, object_name):
169
+ logger.warning(f"Object {object_name} in bucket {bucket_name} does not exist.")
170
+ return
171
+ try:
172
+ self._srv.client.remove_object(bucket_name, object_name)
173
+ except Exception as e:
174
+ logger.error(f"Error deleting object {object_name} from bucket {bucket_name}: {e}")
175
+ case _:
176
+ raise NotImplementedError(f"Delete method not implemented for service {self._nme}")
@@ -87,6 +87,19 @@ class CompCatalog(enum.Enum):
87
87
  """
88
88
  return self.value
89
89
 
90
+ def mount(self, srv_ref: Optional[SRV_REF_TYPE] = None, **kwargs) -> type[ComponentInterface]:
91
+ """Mounts the component to a specific service instance.
92
+ This method configures the component to use a service instance
93
+ identified by `srv_name` and any additional parameters passed via `kwargs`.
94
+
95
+ Args:
96
+ srv_ref (Optional[SRV_REF_TYPE]): The service reference to mount the component to.
97
+ **kwargs: Additional keyword arguments to configure the service instance.
98
+ Returns:
99
+ type[ComponentInterface]: The component class mounted to the specified service.
100
+ """
101
+ return self.value.mount(srv_ref=srv_ref, **kwargs)
102
+
90
103
  def auto(self, srv_ref: Optional[SRV_REF_TYPE] = None, **kwargs) -> ComponentInterface:
91
104
  """Automatically creates an instance of the component, mounting it to a service.
92
105
  This method is a convenience wrapper that first mounts the component to a service
@@ -0,0 +1,122 @@
1
+ from minio import Minio
2
+ from urllib3 import PoolManager, Retry
3
+ from urllib3.util import Timeout
4
+
5
+ from fred.settings import get_environ_variable, logger_manager
6
+ from fred.dao.service.utils import get_minio_from_payload
7
+ from fred.dao.service.interface import ServiceInterface, ServiceConnectionPoolInterface
8
+
9
+ logger = logger_manager.get_logger(name=__name__)
10
+
11
+
12
+ class MinioConnectionPool(ServiceConnectionPoolInterface[PoolManager]):
13
+
14
+ @classmethod
15
+ def _create_pool(cls, disable_cert: bool = False, **kwargs) -> PoolManager:
16
+ """Create a urllib3 PoolManager with the given configurations.
17
+
18
+ TODO: Consider using the inverse of 'require_cert' as the default to ensure we do have cert-check automatically.
19
+ For now, we keep it as is to avoid breaking changes.
20
+
21
+ Args:
22
+ require_cert (bool): Whether to require SSL certificate verification.
23
+ **kwargs: Additional keyword arguments to pass to the PoolManager constructor.
24
+ Returns:
25
+ PoolManager: A configured PoolManager instance.
26
+ """
27
+ num_pools = kwargs.pop("num_pools", 10)
28
+ maxsize = kwargs.pop("maxsize", 10)
29
+ # Default timeout of 5 minutes
30
+ timeout_seconds = kwargs.pop("timeout", 300)
31
+ timeout = Timeout(
32
+ connect=timeout_seconds,
33
+ read=timeout_seconds,
34
+ )
35
+ # Default retries of 5 with exponential backoff
36
+ retry = Retry(
37
+ total=kwargs.pop("retries", 5),
38
+ backoff_factor=kwargs.pop("backoff_factor", 0.25),
39
+ status_forcelist=[500, 502, 503, 504],
40
+ )
41
+ # Configure certificate requirements for SSL connections
42
+ cert_reqs = "CERT_NONE"
43
+ ca_certs = None
44
+ if not disable_cert:
45
+ import certifi
46
+ cert_reqs = "CERT_REQUIRED"
47
+ ca_certs = get_environ_variable("SSL_CERT_FILE") or certifi.where()
48
+ # Finally, create and return the PoolManager instance
49
+ return PoolManager(
50
+ num_pools=num_pools,
51
+ maxsize=maxsize,
52
+ timeout=timeout,
53
+ retries=retry,
54
+ cert_reqs=cert_reqs,
55
+ ca_certs=ca_certs,
56
+ **kwargs
57
+ )
58
+
59
+
60
+ class MinioService(ServiceInterface[Minio]):
61
+ instance: Minio
62
+ metadata: dict = {}
63
+
64
+ @classmethod
65
+ def _create_instance(cls, disable_cert: bool = False, **kwargs) -> Minio:
66
+ pool_configs = kwargs.pop("pool_configs", {})
67
+ minio_configs = get_minio_from_payload(kwargs)
68
+ if "http_client" not in minio_configs:
69
+ logger.warning("Creating a new HTTP client for MinIO with connection pooling.")
70
+ minio_configs["http_client"] = MinioConnectionPool.get_or_create_pool(
71
+ disable_cert=disable_cert,
72
+ **pool_configs
73
+ )
74
+ cls.metadata["minio_endpoint"] = minio_configs.get("endpoint")
75
+ return Minio(cert_check=not disable_cert, **minio_configs)
76
+
77
+ @classmethod
78
+ def auto(cls, disable_cert: bool = False, **kwargs) -> "MinioService":
79
+ cls.instance = cls._create_instance(disable_cert=disable_cert, **kwargs)
80
+ return cls(**kwargs)
81
+
82
+ def buckets(self) -> list[str]:
83
+ """List all buckets in the MinIO instance."""
84
+ return [
85
+ bucket.name
86
+ for bucket in self.client.list_buckets()
87
+ ]
88
+
89
+ def objects(self, bucket_name: str, prefix: str = "", shallow: bool = False) -> list[str]:
90
+ """List all objects in a specific bucket in the MinIO instance."""
91
+ return [
92
+ obj.object_name
93
+ for obj in self.client.list_objects(bucket_name, prefix=prefix, recursive=not shallow)
94
+ ]
95
+
96
+ def bucket_exists(self, bucket_name: str) -> bool:
97
+ """Check if a bucket exists in the MinIO instance."""
98
+ return self.client.bucket_exists(bucket_name)
99
+
100
+ def object_info(self, bucket_name: str, object_name: str) -> dict:
101
+ """Get metadata of an object in a specific bucket in the MinIO instance."""
102
+ stat = self.client.stat_object(bucket_name, object_name)
103
+ return {
104
+ "bucket_name": bucket_name,
105
+ "object_name": object_name,
106
+ "size": stat.size,
107
+ "last_modified": stat.last_modified,
108
+ "etag": stat.etag,
109
+ "content_type": stat.content_type,
110
+ "metadata": stat.metadata,
111
+ }
112
+
113
+ def object_exists(self, bucket_name: str, object_name: str) -> bool:
114
+ """Check if an object exists in a specific bucket in the MinIO instance."""
115
+ from minio.error import S3Error
116
+
117
+ try:
118
+ self.client.stat_object(bucket_name, object_name)
119
+ return True
120
+ except S3Error:
121
+ logger.debug(f"Object {object_name} in bucket {bucket_name} does not exist.")
122
+ return False
@@ -3,6 +3,7 @@ from functools import lru_cache
3
3
  from typing import TypeVar, Optional
4
4
 
5
5
  from fred.dao.service.interface import ServiceInterface
6
+ from fred.dao.service._minio import MinioService
6
7
  from fred.dao.service._redis import RedisService
7
8
  from fred.dao.service._stdlib import StdLibService
8
9
 
@@ -12,6 +13,7 @@ T = TypeVar("T")
12
13
  class ServiceCatalog(enum.Enum):
13
14
  STDLIB = StdLibService
14
15
  REDIS = RedisService
16
+ MINIO = MinioService
15
17
 
16
18
  @classmethod
17
19
  def from_classname(cls, classname: str) -> "ServiceCatalog":
@@ -29,6 +29,7 @@ class ServiceConnectionPoolInterface(Generic[T]):
29
29
 
30
30
  class ServiceInterface(Generic[T]):
31
31
  instance: T
32
+ metadata: dict
32
33
 
33
34
  def __init__(self, **kwargs):
34
35
  self.config = kwargs
@@ -0,0 +1,76 @@
1
+ from fred.settings import get_environ_variable
2
+
3
+
4
+ def get_redis_configs_from_payload(
5
+ payload: dict,
6
+ keep: bool = False,
7
+ ) -> dict:
8
+ """Extract Redis configuration from the given payload dictionary.
9
+ This function looks for common Redis configuration keys in the payload
10
+ dictionary. If a key is not found, it falls back to environment variables.
11
+ Args:
12
+ payload (dict): The dictionary from which to extract Redis configuration.
13
+ keep (bool): If True, the original keys are retained in the payload. If False, they are removed.
14
+ Returns:
15
+ dict: A dictionary containing Redis configuration parameters.
16
+ """
17
+ host = port = password = db = None
18
+ for host_key in ["host", "redis_host"]:
19
+ if (host := payload.get(host_key) if keep else payload.pop(host_key, None)):
20
+ break
21
+ for port_key in ["port", "redis_port"]:
22
+ if (port := payload.get(port_key) if keep else payload.pop(port_key, None)):
23
+ break
24
+ for password_key in ["password", "redis_password"]:
25
+ if (password := payload.get(password_key) if keep else payload.pop(password_key, None)):
26
+ break
27
+ for db_key in ["db", "redis_db"]:
28
+ if (db := payload.get(db_key) if keep else payload.pop(db_key, None)):
29
+ break
30
+ return {
31
+ "host": host or get_environ_variable(name="REDIS_HOST", default=None) or "localhost",
32
+ "port": int(port or get_environ_variable(name="REDIS_PORT", default=None) or 6379),
33
+ "password": password or get_environ_variable(name="REDIS_PASSWORD", default=None),
34
+ "db": int(db or get_environ_variable(name="REDIS_DB", default=None) or 0),
35
+ "decode_responses": True,
36
+ **(payload.get("redis_configs", {}) if keep else payload.pop("redis_configs", {})),
37
+ }
38
+
39
+
40
+ def get_minio_from_payload(
41
+ payload: dict,
42
+ keep: bool = False,
43
+ ):
44
+ """Extract MinIO configuration from the given payload dictionary.
45
+ This function looks for common MinIO configuration keys in the payload
46
+ dictionary. If a key is not found, it falls back to environment variables.
47
+ Args:
48
+ payload (dict): The dictionary from which to extract MinIO configuration.
49
+ keep (bool): If True, the original keys are retained in the payload. If False, they are removed.
50
+ Returns:
51
+ dict: A dictionary containing MinIO configuration parameters.
52
+ """
53
+ endpoint = access_key = secret_key = region = secure = None
54
+ for endpoint_key in ["endpoint", "minio_endpoint"]:
55
+ if (endpoint := payload.get(endpoint_key) if keep else payload.pop(endpoint_key, None)):
56
+ break
57
+ for access_key_key in ["access_key", "minio_access_key"]:
58
+ if (access_key := payload.get(access_key_key) if keep else payload.pop(access_key_key, None)):
59
+ break
60
+ for secret_key_key in ["secret_key", "minio_secret_key"]:
61
+ if (secret_key := payload.get(secret_key_key) if keep else payload.pop(secret_key_key, None)):
62
+ break
63
+ for secure_key in ["secure", "minio_secure"]:
64
+ if (secure := payload.get(secure_key) if keep else payload.pop(secure_key, None)):
65
+ break
66
+ for region_key in ["region", "minio_region"]:
67
+ if (region := payload.get(region_key) if keep else payload.pop(region_key, None)):
68
+ break
69
+ return {
70
+ "endpoint": endpoint or get_environ_variable(name="MINIO_ENDPOINT", default=None) or "localhost:9000",
71
+ "access_key": access_key or get_environ_variable(name="MINIO_ACCESS_KEY", default=None) or "minioadmin",
72
+ "secret_key": secret_key or get_environ_variable(name="MINIO_SECRET_KEY", default=None) or "minioadmin",
73
+ "secure": bool(secure or get_environ_variable(name="MINIO_SECURE", default=None) or True), # Default to True
74
+ "region": region or get_environ_variable(name="MINIO_REGION", default=None) or "us-east-1",
75
+ **(payload.get("minio_configs", {}) if keep else payload.pop("minio_configs", {})),
76
+ }
@@ -0,0 +1,11 @@
1
+ from fred.utils.imout.interface import ImageOutputInterface
2
+
3
+
4
+ class OutputFilesystem(ImageOutputInterface):
5
+ """Filesystem output handler for images."""
6
+
7
+ def out(self, path: str) -> str:
8
+ from fred.utils.imops import save_image_to_path
9
+
10
+ save_image_to_path(image=self.image, path=path)
11
+ return path
@@ -0,0 +1,10 @@
1
+ from fred.utils.imout.interface import ImageOutputInterface
2
+
3
+
4
+ class OutputString(ImageOutputInterface):
5
+ """String output handler for images."""
6
+
7
+ def out(self) -> str:
8
+ from fred.utils.imops import image_to_b64
9
+
10
+ return image_to_b64(self.image)
@@ -0,0 +1,18 @@
1
+ import enum
2
+
3
+ from PIL.Image import Image
4
+
5
+ from fred.utils.imout.interface import ImageOutputInterface
6
+ from fred.utils.imout._filesystem import OutputFilesystem
7
+ from fred.utils.imout._string import OutputString
8
+
9
+
10
+ class ImageOutputCatalog(enum.Enum):
11
+ B64 = OutputString
12
+ STRING = OutputString
13
+ FILESYSTEM = OutputFilesystem
14
+
15
+ def __call__(self, image: Image, **kwargs) -> ImageOutputInterface:
16
+ if getattr(self.value, "auto", None):
17
+ return self.value.auto(image=image, **kwargs)
18
+ return self.value(image=image, **kwargs)
@@ -0,0 +1,37 @@
1
+ from dataclasses import dataclass
2
+ from typing import Any, Optional
3
+
4
+ from PIL.Image import Image
5
+
6
+
7
+ class ImageOutputBackendInterface:
8
+ metadata: dict = {}
9
+ client: Optional[Any] = None # Placeholder for clients like MinIO, S3, etc.
10
+
11
+ @classmethod
12
+ def set_shared_client(cls, client: Any) -> None:
13
+ cls.client = client
14
+
15
+
16
+ @dataclass(frozen=True, slots=True)
17
+ class ImageOutputInterface(ImageOutputBackendInterface):
18
+ """Interface for image output handling."""
19
+ image: Image
20
+
21
+ @classmethod
22
+ def from_path(cls, path: str) -> "ImageOutputInterface":
23
+ from fred.utils.imops import get_image_from_path
24
+
25
+ image = get_image_from_path(path=path)
26
+ return cls(image=image)
27
+
28
+ def save(self, path: str, format: str = "PNG"):
29
+ """Save the image to a file."""
30
+ self.image.save(path, format=format)
31
+
32
+ def show(self):
33
+ """Display the image."""
34
+ self.image.show()
35
+
36
+ def out(self, **kwargs) -> str:
37
+ raise NotImplementedError("Subclasses must implement out method.")
@@ -0,0 +1 @@
1
+ 0.50.0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fred-oss
3
- Version: 0.48.0
3
+ Version: 0.50.0
4
4
  Summary: FREDOSS
5
5
  Home-page: https://fred.fahera.mx
6
6
  Author: Fahera Research, Education, and Development
@@ -14,6 +14,7 @@ Requires-Dist: redis==6.4.0
14
14
  Requires-Dist: requests==2.32.5
15
15
  Requires-Dist: fastapi==0.116.2
16
16
  Requires-Dist: uvicorn[standard]==0.35.0
17
+ Requires-Dist: minio==7.2.18
17
18
  Requires-Dist: pillow==11.3.0
18
19
  Dynamic: author
19
20
  Dynamic: author-email
@@ -19,6 +19,7 @@ src/main/fred/dao/comp/_queue.py
19
19
  src/main/fred/dao/comp/catalog.py
20
20
  src/main/fred/dao/comp/interface.py
21
21
  src/main/fred/dao/service/__init__.py
22
+ src/main/fred/dao/service/_minio.py
22
23
  src/main/fred/dao/service/_redis.py
23
24
  src/main/fred/dao/service/_stdlib.py
24
25
  src/main/fred/dao/service/catalog.py
@@ -52,6 +53,11 @@ src/main/fred/utils/__init__.py
52
53
  src/main/fred/utils/dateops.py
53
54
  src/main/fred/utils/imops.py
54
55
  src/main/fred/utils/runtime.py
56
+ src/main/fred/utils/imout/__init__.py
57
+ src/main/fred/utils/imout/_filesystem.py
58
+ src/main/fred/utils/imout/_string.py
59
+ src/main/fred/utils/imout/catalog.py
60
+ src/main/fred/utils/imout/interface.py
55
61
  src/main/fred/worker/__init__.py
56
62
  src/main/fred/worker/interface.py
57
63
  src/main/fred/worker/settings.py
@@ -4,4 +4,5 @@ redis==6.4.0
4
4
  requests==2.32.5
5
5
  fastapi==0.116.2
6
6
  uvicorn[standard]==0.35.0
7
+ minio==7.2.18
7
8
  pillow==11.3.0
@@ -1,93 +0,0 @@
1
- from dataclasses import dataclass
2
- from typing import Optional
3
-
4
- from fred.settings import logger_manager
5
- from fred.dao.service.catalog import ServiceCatalog
6
- from fred.dao.comp.interface import ComponentInterface
7
-
8
- logger = logger_manager.get_logger(name=__name__)
9
-
10
-
11
- @dataclass(frozen=True, slots=True)
12
- class FredKeyVal(ComponentInterface):
13
- """A simple key-value store implementation using a backend service.
14
- This class provides methods to interact with a key-value store, such as setting,
15
- getting, and deleting key-value pairs. The actual implementation of these methods
16
- depends on the underlying service being used (e.g., Redis).
17
- """
18
- key: str
19
-
20
- def set(self, value: str, key: Optional[str] = None, **kwargs) -> None:
21
- """Sets a key-value pair in the store.
22
- The implementation of this method depends on the underlying service.
23
- For example, if the service is Redis, it uses the SET command to store the
24
- key-value pair.
25
- Args:
26
- key (str): The key to set.
27
- value (str): The value to associate with the key.
28
- **kwargs: Additional keyword arguments for setting the key-value pair,
29
- such as expiration time.
30
- Raises:
31
- NotImplementedError: If the method is not implemented for the current service.
32
- """
33
- key = key or self.key
34
- match self._cat:
35
- case ServiceCatalog.REDIS:
36
- self._srv.client.set(key, value)
37
- expire = kwargs.get("expire")
38
- if expire and isinstance(expire, int):
39
- self._srv.client.expire(key, expire)
40
- case ServiceCatalog.STDLIB:
41
- self._srv.client._memstore_keyval[key] = value
42
- # TODO: Implement expiration logic
43
- if "expire" in kwargs:
44
- logger.warning("Expiration not implemented for STDLIB service.")
45
- case _:
46
- raise NotImplementedError(f"Set method not implemented for service {self._nme}")
47
-
48
- def get(self, key: Optional[str] = None, fail: bool = False) -> Optional[str]:
49
- """Gets the value associated with a key from the store.
50
- The implementation of this method depends on the underlying service.
51
- For example, if the service is Redis, it uses the GET command to retrieve the
52
- value associated with the key.
53
- Args:
54
- key (str): The key to retrieve.
55
- fail (bool): If True, raises a KeyError if the key is not found. Defaults to False.
56
- Returns:
57
- Optional[str]: The value associated with the key, or None if the key is not found
58
- and fail is False.
59
- Raises:
60
- KeyError: If the key is not found and fail is True.
61
- NotImplementedError: If the method is not implemented for the current service.
62
- """
63
- key = key or self.key
64
- result = None
65
- match self._cat:
66
- case ServiceCatalog.REDIS:
67
- result = self._srv.client.get(key)
68
- case ServiceCatalog.STDLIB:
69
- result = self._srv.client._memstore_keyval.get(key)
70
- case _:
71
- raise NotImplementedError(f"Get method not implemented for service {self._nme}")
72
- if fail and result is None:
73
- raise KeyError(f"Key {key} not found.")
74
- return result
75
-
76
- def delete(self, key: Optional[str] = None) -> None:
77
- """Deletes a key-value pair from the store.
78
- The implementation of this method depends on the underlying service.
79
- For example, if the service is Redis, it uses the DEL command to remove the
80
- key-value pair.
81
- Args:
82
- key (str): The key to delete.
83
- Raises:
84
- NotImplementedError: If the method is not implemented for the current service.
85
- """
86
- key = key or self.key
87
- match self._cat:
88
- case ServiceCatalog.REDIS:
89
- self._srv.client.delete(key)
90
- case ServiceCatalog.STDLIB:
91
- self._srv.client._memstore_keyval.pop(key, None)
92
- case _:
93
- raise NotImplementedError(f"Delete method not implemented for service {self._nme}")
@@ -1,37 +0,0 @@
1
- from fred.settings import get_environ_variable
2
-
3
-
4
- def get_redis_configs_from_payload(
5
- payload: dict,
6
- keep: bool = False,
7
- ) -> dict:
8
- """Extract Redis configuration from the given payload dictionary.
9
- This function looks for common Redis configuration keys in the payload
10
- dictionary. If a key is not found, it falls back to environment variables.
11
- Args:
12
- payload (dict): The dictionary from which to extract Redis configuration.
13
- keep (bool): If True, the original keys are retained in the payload. If False, they are removed.
14
- Returns:
15
- dict: A dictionary containing Redis configuration parameters.
16
- """
17
- host = port = password = db = None
18
- for host_key in ["host", "redis_host"]:
19
- if (host := payload.get(host_key) if keep else payload.pop(host_key, None)):
20
- break
21
- for port_key in ["port", "redis_port"]:
22
- if (port := payload.get(port_key) if keep else payload.pop(port_key, None)):
23
- break
24
- for password_key in ["password", "redis_password"]:
25
- if (password := payload.get(password_key) if keep else payload.pop(password_key, None)):
26
- break
27
- for db_key in ["db", "redis_db"]:
28
- if (db := payload.get(db_key) if keep else payload.pop(db_key, None)):
29
- break
30
- return {
31
- "host": host or get_environ_variable(name="REDIS_HOST", default=None) or "localhost",
32
- "port": int(port or get_environ_variable(name="REDIS_PORT", default=None) or 6379),
33
- "password": password or get_environ_variable(name="REDIS_PASSWORD", default=None),
34
- "db": int(db or get_environ_variable(name="REDIS_DB", default=None) or 0),
35
- "decode_responses": True,
36
- **(payload.get("redis_configs", {}) if keep else payload.pop("redis_configs", {})),
37
- }
@@ -1 +0,0 @@
1
- 0.48.0
File without changes
File without changes
File without changes
File without changes
File without changes