fred-oss 0.61.0__tar.gz → 0.63.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. {fred_oss-0.61.0/src/main/fred_oss.egg-info → fred_oss-0.63.0}/PKG-INFO +2 -1
  2. {fred_oss-0.61.0 → fred_oss-0.63.0}/requirements.txt +2 -0
  3. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/comp/_keyval.py +7 -8
  4. fred_oss-0.63.0/src/main/fred/edag/__init__.py +15 -0
  5. fred_oss-0.63.0/src/main/fred/edag/comp/_group.py +9 -0
  6. fred_oss-0.63.0/src/main/fred/edag/comp/_node.py +189 -0
  7. fred_oss-0.63.0/src/main/fred/edag/comp/catalog.py +26 -0
  8. fred_oss-0.63.0/src/main/fred/edag/comp/interface.py +20 -0
  9. fred_oss-0.63.0/src/main/fred/edag/conn/catalog.py +21 -0
  10. fred_oss-0.63.0/src/main/fred/edag/decorator.py +48 -0
  11. fred_oss-0.63.0/src/main/fred/edag/executor.py +127 -0
  12. fred_oss-0.63.0/src/main/fred/edag/plan.py +66 -0
  13. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/future/result.py +4 -4
  14. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/rest/router/endpoint.py +3 -1
  15. fred_oss-0.63.0/src/main/fred/version +1 -0
  16. fred_oss-0.63.0/src/main/fred/worker/runner/rest/__init__.py +0 -0
  17. fred_oss-0.63.0/src/main/fred/worker/runner/rest/router/__init__.py +0 -0
  18. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/rest/router/_runner.py +8 -0
  19. {fred_oss-0.61.0 → fred_oss-0.63.0/src/main/fred_oss.egg-info}/PKG-INFO +2 -1
  20. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred_oss.egg-info/SOURCES.txt +11 -0
  21. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred_oss.egg-info/requires.txt +1 -0
  22. fred_oss-0.61.0/src/main/fred/version +0 -1
  23. {fred_oss-0.61.0 → fred_oss-0.63.0}/MANIFEST.in +0 -0
  24. {fred_oss-0.61.0 → fred_oss-0.63.0}/NOTICE.txt +0 -0
  25. {fred_oss-0.61.0 → fred_oss-0.63.0}/README.md +0 -0
  26. {fred_oss-0.61.0 → fred_oss-0.63.0}/setup.cfg +0 -0
  27. {fred_oss-0.61.0 → fred_oss-0.63.0}/setup.py +0 -0
  28. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/cli/__init__.py +0 -0
  29. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/cli/__main__.py +0 -0
  30. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/cli/interface.py +0 -0
  31. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/cli/main.py +0 -0
  32. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/__init__.py +0 -0
  33. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/comp/__init__.py +0 -0
  34. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/comp/_pubsub.py +0 -0
  35. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/comp/_queue.py +0 -0
  36. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/comp/catalog.py +0 -0
  37. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/comp/interface.py +0 -0
  38. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/__init__.py +0 -0
  39. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/_minio/__init__.py +0 -0
  40. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/_minio/policy/__init__.py +0 -0
  41. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/_minio/policy/builder.py +0 -0
  42. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/_minio/policy/catalog.py +0 -0
  43. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/_minio/policy/loader.py +0 -0
  44. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/_minio/policy/templates/public_ro.json +0 -0
  45. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/_minio/policy/templates/public_rw.json +0 -0
  46. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/_minio/pool.py +0 -0
  47. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/_minio/service.py +0 -0
  48. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/_redis.py +0 -0
  49. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/_stdlib.py +0 -0
  50. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/catalog.py +0 -0
  51. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/interface.py +0 -0
  52. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/dao/service/utils.py +0 -0
  53. {fred_oss-0.61.0/src/main/fred/future/callback → fred_oss-0.63.0/src/main/fred/edag/comp}/__init__.py +0 -0
  54. {fred_oss-0.61.0/src/main/fred/integrations/databricks/runtimes → fred_oss-0.63.0/src/main/fred/edag/conn}/__init__.py +0 -0
  55. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/future/__init__.py +0 -0
  56. {fred_oss-0.61.0/src/main/fred/integrations/databricks/wrappers → fred_oss-0.63.0/src/main/fred/future/callback}/__init__.py +0 -0
  57. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/future/callback/_function.py +0 -0
  58. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/future/callback/catalog.py +0 -0
  59. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/future/callback/interface.py +0 -0
  60. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/future/impl.py +0 -0
  61. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/future/settings.py +0 -0
  62. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/future/utils.py +0 -0
  63. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/integrations/databricks/__init__.py +0 -0
  64. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/integrations/databricks/cli_ext.py +0 -0
  65. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/integrations/databricks/runtime.py +0 -0
  66. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/integrations/databricks/runtimes/16.4LTS.json +0 -0
  67. {fred_oss-0.61.0/src/main/fred/rest/router → fred_oss-0.63.0/src/main/fred/integrations/databricks/runtimes}/__init__.py +0 -0
  68. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/integrations/databricks/runtimes/scanner.py +0 -0
  69. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/integrations/databricks/runtimes/sync.py +0 -0
  70. {fred_oss-0.61.0/src/main/fred/rest/router/catalog → fred_oss-0.63.0/src/main/fred/integrations/databricks/wrappers}/__init__.py +0 -0
  71. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/integrations/databricks/wrappers/dbutils.py +0 -0
  72. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/integrations/runpod/__init__.py +0 -0
  73. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/integrations/runpod/cli_ext.py +0 -0
  74. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/integrations/runpod/helper.py +0 -0
  75. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/maturity.py +0 -0
  76. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/monad/__init__.py +0 -0
  77. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/monad/_either.py +0 -0
  78. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/monad/catalog.py +0 -0
  79. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/monad/interface.py +0 -0
  80. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/rest/__init__.py +0 -0
  81. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/rest/auth.py +0 -0
  82. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/rest/config.py +0 -0
  83. {fred_oss-0.61.0/src/main/fred/utils → fred_oss-0.63.0/src/main/fred/rest/router}/__init__.py +0 -0
  84. {fred_oss-0.61.0/src/main/fred/utils/imout → fred_oss-0.63.0/src/main/fred/rest/router/catalog}/__init__.py +0 -0
  85. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/rest/router/catalog/default/__init__.py +0 -0
  86. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/rest/router/catalog/default/_base.py +0 -0
  87. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/rest/router/catalog/default/_example.py +0 -0
  88. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/rest/router/catalog/default/catalog.py +0 -0
  89. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/rest/router/catalog/interface.py +0 -0
  90. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/rest/router/config.py +0 -0
  91. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/rest/router/interface.py +0 -0
  92. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/rest/server.py +0 -0
  93. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/rest/settings.py +0 -0
  94. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/settings.py +0 -0
  95. {fred_oss-0.61.0/src/main/fred/worker/runner/model → fred_oss-0.63.0/src/main/fred/utils}/__init__.py +0 -0
  96. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/utils/dateops.py +0 -0
  97. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/utils/imops.py +0 -0
  98. {fred_oss-0.61.0/src/main/fred/worker/runner/plugins → fred_oss-0.63.0/src/main/fred/utils/imout}/__init__.py +0 -0
  99. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/utils/imout/_filesystem.py +0 -0
  100. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/utils/imout/_minio.py +0 -0
  101. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/utils/imout/_string.py +0 -0
  102. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/utils/imout/catalog.py +0 -0
  103. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/utils/imout/interface.py +0 -0
  104. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/utils/mlops/__init__.py +0 -0
  105. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/utils/mlops/auto.py +0 -0
  106. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/utils/runtime.py +0 -0
  107. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/version.py +0 -0
  108. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/__init__.py +0 -0
  109. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/interface.py +0 -0
  110. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/__init__.py +0 -0
  111. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/backend.py +0 -0
  112. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/client.py +0 -0
  113. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/handler.py +0 -0
  114. {fred_oss-0.61.0/src/main/fred/worker/runner/rest → fred_oss-0.63.0/src/main/fred/worker/runner/model}/__init__.py +0 -0
  115. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/model/_handler.py +0 -0
  116. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/model/_item.py +0 -0
  117. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/model/_request.py +0 -0
  118. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/model/_runner_spec.py +0 -0
  119. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/model/catalog.py +0 -0
  120. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/model/interface.py +0 -0
  121. {fred_oss-0.61.0/src/main/fred/worker/runner/rest/router → fred_oss-0.63.0/src/main/fred/worker/runner/plugins}/__init__.py +0 -0
  122. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/plugins/_local.py +0 -0
  123. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/plugins/_runpod.py +0 -0
  124. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/plugins/catalog.py +0 -0
  125. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/plugins/interface.py +0 -0
  126. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/rest/router/_base.py +0 -0
  127. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/rest/router/catalog.py +0 -0
  128. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/settings.py +0 -0
  129. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/signal.py +0 -0
  130. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/status.py +0 -0
  131. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/runner/utils.py +0 -0
  132. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred/worker/settings.py +0 -0
  133. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred_oss.egg-info/dependency_links.txt +0 -0
  134. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred_oss.egg-info/entry_points.txt +0 -0
  135. {fred_oss-0.61.0 → fred_oss-0.63.0}/src/main/fred_oss.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fred-oss
3
- Version: 0.61.0
3
+ Version: 0.63.0
4
4
  Summary: FREDOSS
5
5
  Home-page: https://fred.fahera.mx
6
6
  Author: Fahera Research, Education, and Development
@@ -10,6 +10,7 @@ Description-Content-Type: text/markdown
10
10
  License-File: NOTICE.txt
11
11
  Requires-Dist: fire==0.7.1
12
12
  Requires-Dist: psutil==7.0.0
13
+ Requires-Dist: dill==0.4.0
13
14
  Requires-Dist: redis==6.4.0
14
15
  Requires-Dist: requests==2.32.5
15
16
  Requires-Dist: fastapi==0.116.2
@@ -2,6 +2,8 @@
2
2
  fire==0.7.1
3
3
  # Minimal requirements
4
4
  psutil==7.0.0
5
+ # Fred Core Implementation
6
+ dill==0.4.0
5
7
  # Fred Worker/Runner Infra
6
8
  redis==6.4.0
7
9
  requests==2.32.5
@@ -13,17 +13,16 @@ def _get_minio_elements_from_key(key: str, **kwargs) -> tuple[str, str]:
13
13
 
14
14
  for bucket_key in ("bucket", "minio_bucket", "bucket_name"):
15
15
  if bucket_key in kwargs:
16
- bucket_name = kwargs.pop(bucket_key)
16
+ bucket_name_candidate = kwargs.pop(bucket_key)
17
17
  break
18
18
  else:
19
- bucket_name = get_environ_variable("MINIO_BUCKET") or os.path.dirname(key)
19
+ bucket_name_candidate = get_environ_variable("MINIO_BUCKET") or os.path.dirname(key)
20
20
 
21
- fullpath = key if key.startswith(bucket_name + '/') else os.path.join(
22
- bucket_name,
23
- key,
24
- )
25
- bucket_name = os.path.dirname(fullpath)
26
- object_name = os.path.basename(fullpath)
21
+ fullpath = (key if key.startswith(bucket_name_candidate + os.sep) else os.path.join(bucket_name_candidate, key)).strip(os.sep)
22
+
23
+ # We only want the first component as bucket name, the rest is the object name
24
+ bucket_name, *obj_components = os.path.normpath(fullpath).split(os.sep)
25
+ object_name = os.path.join(*obj_components)
27
26
  if not bucket_name:
28
27
  raise ValueError(
29
28
  "Bucket name must be specified either in kwargs, environment variable MINIO_BUCKET, "
@@ -0,0 +1,15 @@
1
+ from fred.maturity import Maturity, MaturityLevel
2
+ from fred.edag.decorator import NodeDecorator
3
+
4
+
5
+ module_maturity = Maturity(
6
+ level=MaturityLevel.ALPHA,
7
+ reference=__name__,
8
+ message=(
9
+ "Fred-eDAG implementation is in early development "
10
+ "and therefore currently with incomplete and unstable features."
11
+ )
12
+ )
13
+
14
+
15
+ node = NodeDecorator
@@ -0,0 +1,9 @@
1
+ from dataclasses import dataclass
2
+
3
+ from fred.edag.comp.interface import ComponentInterface
4
+ from fred.edag.comp._node import Node
5
+
6
+
7
+ @dataclass(frozen=True, slots=True, unsafe_hash=True)
8
+ class Group(ComponentInterface):
9
+ nodes: list[Node] # TODO: let's make this a frozenset instead of a list to ensure immutability
@@ -0,0 +1,189 @@
1
+ import uuid
2
+ from inspect import Signature, signature, Parameter
3
+ from dataclasses import dataclass, field, asdict
4
+ from typing import Callable, Optional
5
+
6
+ from fred.edag.comp.interface import ComponentInterface
7
+
8
+
9
+ @dataclass(frozen=True, slots=True)
10
+ class NodeFun:
11
+ fname: str
12
+ inner: Callable
13
+ signature: Signature
14
+
15
+
16
+ @classmethod
17
+ def auto(cls, function: Callable, name: Optional[str] = None) -> "NodeFun":
18
+ fname = name or getattr(function, "__name__", "undefined")
19
+ return cls(
20
+ fname=fname,
21
+ inner=function,
22
+ signature=signature(function),
23
+ )
24
+
25
+ def validate_parameter_compliance(self, *args, **kwargs) -> dict:
26
+ # Determine if the signature accepts **kwargs
27
+ var_kwargs = any(
28
+ p.kind == Parameter.VAR_KEYWORD
29
+ for p in self.signature.parameters.values()
30
+ )
31
+ # Validate keywords against signature if not accepting **kwargs
32
+ clean_kwargs = {
33
+ k: v
34
+ for k, v in kwargs.items()
35
+ if k in self.signature.parameters
36
+ } if not var_kwargs else kwargs
37
+ # Bind arguments to signature
38
+ bound = self.signature.bind_partial(*args, **clean_kwargs)
39
+ bound.apply_defaults()
40
+ # Return the bound arguments as a dictionary
41
+ return {
42
+ "args": bound.args,
43
+ "kwargs": bound.kwargs,
44
+ }
45
+
46
+ def __call__(self, *args, **kwargs):
47
+ params = self.validate_parameter_compliance(*args, **kwargs)
48
+ return self.inner(*params["args"], **params["kwargs"])
49
+
50
+ def __name__(self):
51
+ return self.fname
52
+
53
+ def __hash__(self):
54
+ return hash((
55
+ self.inner,
56
+ tuple(self.signature.parameters.items()),
57
+ ))
58
+
59
+
60
+ @dataclass(frozen=True, slots=True)
61
+ class Node(ComponentInterface):
62
+ name: str
63
+ key: str # Output key
64
+ nfun: NodeFun
65
+ # TODO: let's make the 'params' a frozenset (i.e., frozenparams) instead of a dict to ensure immutability
66
+ params: dict = field(default_factory=dict)
67
+ nid: str = field(default_factory=lambda: str(uuid.uuid4()))
68
+ _inplace: bool = False
69
+ _explode: bool = False # Whether this node's output should be exploded when used as input to another node
70
+
71
+ def __hash__(self):
72
+ obj = asdict(self)
73
+ obj["nfun"] = self.nfun.__hash__()
74
+ obj["params"] = frozenset((obj.get("params") or {}).keys()) # only hash keys to avoid unhashable values
75
+ return hash(frozenset(obj.items()))
76
+
77
+ def clone(self, **kwargs) -> "Node":
78
+ # Verify if 'inplace' is set via '_inplace' or 'inplace' keys; otherwise, keep current value
79
+ for key in ("inplace", "_inplace"):
80
+ value = kwargs.pop(key, None)
81
+ if isinstance(value, bool):
82
+ kwargs["_inplace"] = value
83
+ break
84
+ else:
85
+ kwargs["_inplace"] = self._inplace
86
+ # Verify if 'explode' is set via '_explode' or 'explode' keys; otherwise, keep current value
87
+ for key in ("explode", "_explode"):
88
+ value = kwargs.pop(key, None)
89
+ if isinstance(value, bool):
90
+ kwargs["_explode"] = value
91
+ break
92
+ else:
93
+ kwargs["_explode"] = self._explode
94
+ # Create a new Node with updated attributes
95
+ return self.__class__(
96
+ **{
97
+ "name": self.name,
98
+ "key": self.key,
99
+ "nfun": self.nfun,
100
+ "params": self.params,
101
+ **kwargs,
102
+ },
103
+ nid=str(uuid.uuid4()), # Must have a new ID
104
+ )
105
+
106
+ def wrap(self, function: Callable) -> "Node":
107
+ fname = getattr(function, "__name__", "undef_wrapper_function")
108
+ return self.clone(
109
+ nfun=NodeFun.auto(
110
+ name=fname,
111
+ function=lambda *args, **kwargs: function(self.fun(*args, **kwargs))
112
+ ),
113
+ )
114
+
115
+ @classmethod
116
+ def auto(
117
+ cls,
118
+ function: Callable,
119
+ inplace: bool = False,
120
+ explode: bool = False,
121
+ fname: Optional[str] = None,
122
+ name: Optional[str] = None,
123
+ key: Optional[str] = None,
124
+ **params,
125
+ ):
126
+ name = name or getattr(function, "__name__", "undefined")
127
+ return cls(
128
+ name=name,
129
+ key=key or name,
130
+ nfun=NodeFun.auto(function=function, name=fname),
131
+ _inplace=inplace,
132
+ _explode=explode,
133
+ params=params,
134
+ )
135
+
136
+ @property
137
+ def fun(self) -> Callable:
138
+ return self.nfun
139
+
140
+ def inplace(self) -> "Node":
141
+ return self.clone(_inplace=True)
142
+
143
+ def explode(self) -> "Node":
144
+ return self.clone(_explode=True)
145
+
146
+ @property
147
+ def E(self) -> "Node":
148
+ # Shortcut to set explode=True
149
+ return self.explode()
150
+
151
+ def __invert__(self) -> "Node":
152
+ # Unary ~ operator to set explode=True
153
+ return self.explode()
154
+
155
+ def with_output(self, key: str) -> "Node":
156
+ return self.with_alias(alias=self.name, key=key, keep_key=False)
157
+
158
+ def with_alias(self, alias: str, key: Optional[str] = None, keep_key: bool = False) -> "Node":
159
+ return self.__class__(
160
+ name=alias,
161
+ key=key or (self.key if keep_key else alias),
162
+ nfun=self.nfun,
163
+ params=self.params,
164
+ _inplace=self._inplace,
165
+ _explode=self._explode,
166
+ )
167
+
168
+ def with_params(self, update_key: Optional[str] = None, **params) -> "Node":
169
+ return self.__class__(
170
+ name=self.name,
171
+ nfun=self.nfun,
172
+ key=update_key or self.key,
173
+ params={
174
+ **self.params,
175
+ **params,
176
+ },
177
+ _inplace=self._inplace,
178
+ _explode=self._explode,
179
+ )
180
+
181
+ def execute(self, **kwargs):
182
+ params = {
183
+ **self.params,
184
+ **kwargs
185
+ }
186
+ if self._inplace:
187
+ return self.fun(**params)
188
+ from fred.future.impl import Future
189
+ return Future(self.fun, **params)
@@ -0,0 +1,26 @@
1
+ import enum
2
+
3
+ from fred.edag.comp.interface import ComponentInterface
4
+ from fred.edag.comp._group import Group
5
+ from fred.edag.comp._node import Node
6
+ from fred.edag.plan import Plan
7
+
8
+
9
+ class PlanCompositionMixin:
10
+
11
+ def __rshift__(self, other: ComponentInterface | Plan) -> Plan:
12
+ left = Plan.as_plan(self)
13
+ right = Plan.as_plan(other)
14
+ return left >> right
15
+
16
+
17
+ class CompCatalog(enum.Enum):
18
+ NODE = type("Node", (Node, PlanCompositionMixin), {})
19
+ GROUP = type("Group", (Group, PlanCompositionMixin), {})
20
+
21
+ @property
22
+ def ref(self) -> type[ComponentInterface]:
23
+ return self.value
24
+
25
+ def __call__(self, *args, **kwargs) -> ComponentInterface:
26
+ return self.value(*args, **kwargs)
@@ -0,0 +1,20 @@
1
+
2
+ class ComponentInterface:
3
+
4
+ def __or__(self, other: 'ComponentInterface') -> 'ComponentInterface':
5
+ # union: d1 | d2 | d3 (build a group)
6
+ # allow Node | Group and Group | Node chaining
7
+ from fred.edag.comp._group import Group
8
+ from fred.edag.comp._node import Node
9
+
10
+ match self, other:
11
+ case Node(), Node():
12
+ return Group(nodes=[self, other])
13
+ case Node(), Group():
14
+ return Group(nodes=[self, *other.nodes])
15
+ case Group(), Node():
16
+ return Group(nodes=[*self.nodes, other])
17
+ case Group(), Group():
18
+ return Group(nodes=[*self.nodes, *other.nodes])
19
+ case _:
20
+ raise TypeError("| expects Node or Group")
@@ -0,0 +1,21 @@
1
+ import enum
2
+
3
+ from fred.edag.comp.catalog import CompCatalog
4
+
5
+
6
+ class ConnCatalog(enum.Enum):
7
+ PASS = CompCatalog.NODE.ref.auto(
8
+ name="conn:passthrough",
9
+ function=lambda **kwargs: kwargs,
10
+ explode=True,
11
+ inplace=True,
12
+ )
13
+ PASS_PRINT = CompCatalog.NODE.ref.auto(
14
+ name="conn:passthrough_print",
15
+ function=lambda **kwargs: print(kwargs) or kwargs,
16
+ explode=True,
17
+ inplace=True,
18
+ )
19
+
20
+ def __call__(self, *args, **kwargs) -> CompCatalog.NODE.ref:
21
+ return self.value.clone(*args, **kwargs)
@@ -0,0 +1,48 @@
1
+ from typing import Callable, Optional
2
+
3
+ from fred.edag.comp.catalog import CompCatalog
4
+
5
+
6
+ class NodeDecorator:
7
+ """Decorator to create Node instances from functions.
8
+ Can be used with or without parameters.
9
+
10
+ Example usage:
11
+
12
+ @NodeDecorator
13
+ def my_function(...):
14
+ ...
15
+ or
16
+ @NodeDecorator(name="example", param1=value1, param2=value2)
17
+ def my_function(...):
18
+ """
19
+
20
+ def __new__(cls, func: Optional[Callable] = None, **kwargs):
21
+ if not func:
22
+ # Create an instance of NodeDecorator to hold kwargs until __call__ is invoked
23
+ instance = super(NodeDecorator, cls).__new__(cls)
24
+ instance.kwargs = kwargs
25
+ return instance
26
+ # Return a Node instance directly if func is provided
27
+ return CompCatalog.NODE.ref.auto(
28
+ function=func,
29
+ **kwargs,
30
+ )
31
+
32
+ def __init__(self, func: Optional[Callable] = None, **kwargs):
33
+ self.func = func
34
+ self.kwargs = kwargs
35
+
36
+ def __call__(self, func: Callable, **kwargs) -> CompCatalog.NODE.ref:
37
+ self.func = func
38
+ self.kwargs = {**self.kwargs, **kwargs}
39
+ return self.get_node()
40
+
41
+ def get_node(self, **kwargs) -> CompCatalog.NODE.ref:
42
+ return CompCatalog.NODE.ref.auto(
43
+ function=self.func,
44
+ **{
45
+ **self.kwargs,
46
+ **kwargs,
47
+ },
48
+ )
@@ -0,0 +1,127 @@
1
+ import uuid
2
+ from graphlib import TopologicalSorter
3
+ from dataclasses import dataclass, field
4
+ from typing import Any, Optional
5
+
6
+ from fred.future.impl import Future
7
+ from fred.edag.comp.catalog import CompCatalog
8
+ from fred.edag.plan import Plan
9
+
10
+
11
+ @dataclass(frozen=True, slots=True)
12
+ class Executor:
13
+ predmap: dict[CompCatalog.NODE.ref, set[CompCatalog.NODE.ref]]
14
+ results: dict[str, dict[str, Any]] = field(default_factory=dict)
15
+
16
+ @classmethod
17
+ def from_plan(cls, plan: Plan, **kwargs) -> "Executor":
18
+ return cls(predmap=plan.as_predmap(**kwargs))
19
+
20
+ def get_tsort(self) -> TopologicalSorter:
21
+ return TopologicalSorter(self.predmap)
22
+
23
+ def loop(
24
+ self,
25
+ run_id: str,
26
+ tsort: TopologicalSorter,
27
+ prev_layer: list[list[str]],
28
+ start_with: Optional[dict] = None,
29
+ unrestricted: bool = False,
30
+ non_destructive_node_explosion: bool = False,
31
+ ) -> list[list[str]]:
32
+ start_with = start_with or {}
33
+ if not (nodes := tsort.get_ready()):
34
+ return prev_layer
35
+ # You can only get access to results of previous layers unless unrestricted is requested.
36
+ prev_layer_results = self.results[run_id] if unrestricted else {
37
+ key: val
38
+ for key, val in self.results[run_id].items()
39
+ if key in prev_layer[-1]
40
+ }
41
+ curr_layer = []
42
+ for node in nodes:
43
+ parents = [
44
+ parent.name
45
+ for parent in self.predmap.get(node, [])
46
+ ]
47
+ accessible_results = prev_layer_results if unrestricted else {
48
+ key: val
49
+ for key, val in prev_layer_results.items()
50
+ if key in parents
51
+ }
52
+ kwargs = {
53
+ **start_with,
54
+ **{
55
+ arg: val
56
+ for node_key, node_out in accessible_results.items()
57
+ for arg, val in node_out.items()
58
+ }
59
+ }
60
+ # Execute node function
61
+ match node.execute(**kwargs):
62
+ case Future() as future:
63
+ # Can't we just build the whole graph in the future and 'wait_and_resolve' only at the end?
64
+ # Or at least per layer/generation?
65
+ output = {node.key: future.wait_and_resolve()}
66
+ case present:
67
+ output = {node.key: present}
68
+ # We can only explode if requested and the output result is a dict
69
+ if node._explode and isinstance(output.get(node.key), dict):
70
+ output = {
71
+ # Keep original output if non-destrictive-explode is requested;
72
+ # The original key can be overwritten if key collides during explosion.
73
+ **(output if non_destructive_node_explosion else {}),
74
+ # Explode keys into the output dict
75
+ **output[node.key],
76
+ }
77
+ # Store output in results
78
+ self.results[run_id][node.name] = {
79
+ #**self.results[run_id].get(node.name, {}),
80
+ **output,
81
+ }
82
+ # Mark node as done
83
+ tsort.done(node)
84
+ curr_layer.append(node.name)
85
+ prev_layer.append(curr_layer)
86
+ return self.loop(
87
+ run_id=run_id,
88
+ tsort=tsort,
89
+ prev_layer=prev_layer,
90
+ unrestricted=unrestricted,
91
+ start_with={}, # Only availabe during the first layer call
92
+ non_destructive_node_explosion=non_destructive_node_explosion,
93
+ )
94
+
95
+ def execute(
96
+ self,
97
+ keep: bool = False,
98
+ unrestricted: bool = False,
99
+ start_with: Optional[dict] = None,
100
+ non_destructive_node_explosion: bool = False,
101
+ ) -> dict:
102
+ from fred.utils.dateops import datetime_utcnow
103
+
104
+ run_id = str(uuid.uuid4())
105
+ run_start = datetime_utcnow()
106
+ # Initialize in-memory result storage for this run
107
+ # TODO: Swap the result-store to our fred-keyval implementation
108
+ self.results[run_id] = {}
109
+ # Prepare TopologicalSorter
110
+ tsort = self.get_tsort()
111
+ tsort.prepare()
112
+ # Execute nodes in topological order
113
+ layers = self.loop(
114
+ run_id=run_id,
115
+ tsort=tsort,
116
+ prev_layer=[[]],
117
+ unrestricted=unrestricted,
118
+ start_with=start_with or {},
119
+ non_destructive_node_explosion=non_destructive_node_explosion,
120
+ )
121
+ return {
122
+ "run_id": run_id,
123
+ "run_start": run_start,
124
+ "run_end": datetime_utcnow(),
125
+ "results": self.results[run_id] if keep else self.results.pop(run_id),
126
+ "layers": layers,
127
+ }
@@ -0,0 +1,66 @@
1
+ from dataclasses import dataclass, field
2
+ from typing import FrozenSet, Tuple, Union
3
+
4
+ from fred.edag.comp._node import Node
5
+ from fred.edag.comp._group import Group
6
+ from fred.edag.comp.interface import ComponentInterface
7
+
8
+
9
+ @dataclass(frozen=True, slots=True)
10
+ class Plan:
11
+ nodes: list[Node] = field(default_factory=list)
12
+ edges: list[Tuple[Node, Node]] = field(default_factory=list) # directed edges: (from/src, to/dst)
13
+ heads: list[Node] = field(default_factory=list) # current entry set of chaining
14
+ tails: list[Node] = field(default_factory=list) # current exit set of chaining
15
+
16
+ @staticmethod
17
+ def empty() -> 'Plan':
18
+ return Plan()
19
+
20
+ @classmethod
21
+ def as_plan(cls, other: Union[ComponentInterface, 'Plan']) -> 'Plan':
22
+ match other:
23
+ case Plan() as plan:
24
+ return plan
25
+ case Node():
26
+ return Plan(
27
+ nodes=[other],
28
+ edges=[],
29
+ heads=[other],
30
+ tails=[other],
31
+ )
32
+ case Group():
33
+ return Plan(
34
+ nodes=[*other.nodes],
35
+ edges=[],
36
+ heads=[*other.nodes],
37
+ tails=[*other.nodes],
38
+ )
39
+ case _:
40
+ raise TypeError("as_plan expects Node, Group, or Plan")
41
+
42
+ def __rshift__(self, other: Union[ComponentInterface, 'Plan']) -> "Plan":
43
+ other_plan = self.as_plan(other)
44
+ # combine edges, nodes, and link tails -> heads (i.e., connect current tails to new heads)
45
+ new_edges = [
46
+ *self.edges,
47
+ *other_plan.edges,
48
+ *(
49
+ (src, dst)
50
+ for src in self.tails
51
+ for dst in other_plan.heads
52
+ ),
53
+ ]
54
+ new_nodes = [*self.nodes, *other_plan.nodes]
55
+ return Plan(
56
+ nodes=new_nodes,
57
+ edges=new_edges,
58
+ heads=self.heads,
59
+ tails=other_plan.tails,
60
+ )
61
+
62
+ def as_predmap(self) -> dict[Node, set[Node]]:
63
+ predmap = {n: set() for n in self.nodes} # Ensure all nodes are keys
64
+ for src, dst in self.edges:
65
+ predmap[dst].add(src)
66
+ return predmap
@@ -133,15 +133,15 @@ class FutureResult(Generic[A], FutureBackend.infer_backend()):
133
133
  return self._get_obj_key(future_id=self.future_id)
134
134
 
135
135
  def stringify(self) -> str:
136
- import pickle
136
+ import dill
137
137
  import base64
138
- return base64.b64encode(pickle.dumps(self)).decode("ascii")
138
+ return base64.b64encode(dill.dumps(self)).decode("ascii")
139
139
 
140
140
  @classmethod
141
141
  def from_string(cls, payload: str) -> 'FutureResult[A]':
142
- import pickle
142
+ import dill
143
143
  import base64
144
- return pickle.loads(base64.b64decode(payload))
144
+ return dill.loads(base64.b64decode(payload))
145
145
 
146
146
  def _from_backend(self) -> Optional['FutureResult[A]']:
147
147
  payload = self.obj.get()
@@ -31,6 +31,7 @@ class RouterEndpointConfig:
31
31
  class RouterEndpoint:
32
32
  function: Callable
33
33
  configs: RouterEndpointConfig
34
+ _og: Optional[Callable] = None # Original function before any binding... this is mainly for testing purposes.
34
35
 
35
36
  def __call__(self, *args, **kwargs):
36
37
  return self.function(*args, **kwargs)
@@ -93,5 +94,6 @@ class RouterEndpointAnnotation(RouterEndpoint):
93
94
  return self.function(other_class, **params)
94
95
  return RouterEndpoint(
95
96
  function=closure,
96
- configs=self.configs
97
+ configs=self.configs,
98
+ _og=self.function,
97
99
  )
@@ -0,0 +1 @@
1
+ 0.63.0
@@ -109,11 +109,19 @@ class RunnerRouterMixin(RouterInterfaceMixin):
109
109
  request.dispatch(
110
110
  request_queue=self.runner_backend.queue(f"req:{queue_slug}")
111
111
  )
112
+ # Starting the runner to process the request if requested; this should always be BEFORE placing the request in the queue
113
+ # to avoid race conditions where a blocking runner is spawned before the request is enqueued.
114
+ # TODO: Let's optimize this by identifying if there's already an on-going runner for the same spec.
115
+ # If so, we can potentially reuse it instead of starting a new one each time.
116
+ # TODO: Let's remove the '_og' reference and find a cleaner way to handle this...
117
+ runner_start_output = self.runner_start._og(self, **start_configs) \
118
+ if (start_configs := kwargs.pop("start_configs", {})) else {}
112
119
  return {
113
120
  "item_id": item.item_id,
114
121
  "request_id": request.request_id,
115
122
  "queue_slug": queue_slug,
116
123
  "dispatched_at": datetime_utcnow().isoformat(),
124
+ "runner_start_output": runner_start_output,
117
125
  }
118
126
 
119
127
  @RouterEndpointAnnotation.set(
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fred-oss
3
- Version: 0.61.0
3
+ Version: 0.63.0
4
4
  Summary: FREDOSS
5
5
  Home-page: https://fred.fahera.mx
6
6
  Author: Fahera Research, Education, and Development
@@ -10,6 +10,7 @@ Description-Content-Type: text/markdown
10
10
  License-File: NOTICE.txt
11
11
  Requires-Dist: fire==0.7.1
12
12
  Requires-Dist: psutil==7.0.0
13
+ Requires-Dist: dill==0.4.0
13
14
  Requires-Dist: redis==6.4.0
14
15
  Requires-Dist: requests==2.32.5
15
16
  Requires-Dist: fastapi==0.116.2
@@ -33,6 +33,17 @@ src/main/fred/dao/service/_minio/policy/catalog.py
33
33
  src/main/fred/dao/service/_minio/policy/loader.py
34
34
  src/main/fred/dao/service/_minio/policy/templates/public_ro.json
35
35
  src/main/fred/dao/service/_minio/policy/templates/public_rw.json
36
+ src/main/fred/edag/__init__.py
37
+ src/main/fred/edag/decorator.py
38
+ src/main/fred/edag/executor.py
39
+ src/main/fred/edag/plan.py
40
+ src/main/fred/edag/comp/__init__.py
41
+ src/main/fred/edag/comp/_group.py
42
+ src/main/fred/edag/comp/_node.py
43
+ src/main/fred/edag/comp/catalog.py
44
+ src/main/fred/edag/comp/interface.py
45
+ src/main/fred/edag/conn/__init__.py
46
+ src/main/fred/edag/conn/catalog.py
36
47
  src/main/fred/future/__init__.py
37
48
  src/main/fred/future/impl.py
38
49
  src/main/fred/future/result.py
@@ -1,5 +1,6 @@
1
1
  fire==0.7.1
2
2
  psutil==7.0.0
3
+ dill==0.4.0
3
4
  redis==6.4.0
4
5
  requests==2.32.5
5
6
  fastapi==0.116.2
@@ -1 +0,0 @@
1
- 0.61.0
File without changes
File without changes
File without changes
File without changes
File without changes