olas-operate-middleware 0.13.0__tar.gz → 0.13.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/PKG-INFO +1 -1
  2. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/cli.py +1 -1
  3. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/services/deployment_runner.py +107 -82
  4. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/services/health_checker.py +38 -2
  5. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/services/manage.py +14 -0
  6. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/wallet/master.py +11 -3
  7. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/wallet/wallet_recovery_manager.py +108 -56
  8. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/pyproject.toml +1 -1
  9. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/LICENSE +0 -0
  10. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/README.md +0 -0
  11. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/__init__.py +0 -0
  12. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/account/__init__.py +0 -0
  13. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/account/user.py +0 -0
  14. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/bridge/bridge_manager.py +0 -0
  15. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/bridge/providers/lifi_provider.py +0 -0
  16. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/bridge/providers/native_bridge_provider.py +0 -0
  17. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/bridge/providers/provider.py +0 -0
  18. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/bridge/providers/relay_provider.py +0 -0
  19. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/constants.py +0 -0
  20. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/README.md +0 -0
  21. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/__init__.py +0 -0
  22. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/__init__.py +0 -0
  23. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/dual_staking_token/__init__.py +0 -0
  24. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/dual_staking_token/build/DualStakingToken.json +0 -0
  25. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/dual_staking_token/contract.py +0 -0
  26. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/dual_staking_token/contract.yaml +0 -0
  27. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/foreign_omnibridge/__init__.py +0 -0
  28. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/foreign_omnibridge/build/ForeignOmnibridge.json +0 -0
  29. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/foreign_omnibridge/contract.py +0 -0
  30. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/foreign_omnibridge/contract.yaml +0 -0
  31. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/home_omnibridge/__init__.py +0 -0
  32. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/home_omnibridge/build/HomeOmnibridge.json +0 -0
  33. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/home_omnibridge/contract.py +0 -0
  34. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/home_omnibridge/contract.yaml +0 -0
  35. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/l1_standard_bridge/__init__.py +0 -0
  36. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/l1_standard_bridge/build/L1StandardBridge.json +0 -0
  37. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/l1_standard_bridge/contract.py +0 -0
  38. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/l1_standard_bridge/contract.yaml +0 -0
  39. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/l2_standard_bridge/__init__.py +0 -0
  40. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/l2_standard_bridge/build/L2StandardBridge.json +0 -0
  41. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/l2_standard_bridge/contract.py +0 -0
  42. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/l2_standard_bridge/contract.yaml +0 -0
  43. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/mech_activity/__init__.py +0 -0
  44. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/mech_activity/build/MechActivity.json +0 -0
  45. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/mech_activity/contract.py +0 -0
  46. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/mech_activity/contract.yaml +0 -0
  47. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/optimism_mintable_erc20/__init__.py +0 -0
  48. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/optimism_mintable_erc20/build/OptimismMintableERC20.json +0 -0
  49. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/optimism_mintable_erc20/contract.py +0 -0
  50. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/optimism_mintable_erc20/contract.yaml +0 -0
  51. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/recovery_module/__init__.py +0 -0
  52. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/recovery_module/build/RecoveryModule.json +0 -0
  53. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/recovery_module/contract.py +0 -0
  54. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/recovery_module/contract.yaml +0 -0
  55. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/requester_activity_checker/__init__.py +0 -0
  56. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/requester_activity_checker/build/RequesterActivityChecker.json +0 -0
  57. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/requester_activity_checker/contract.py +0 -0
  58. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/requester_activity_checker/contract.yaml +0 -0
  59. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/staking_token/__init__.py +0 -0
  60. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/staking_token/build/StakingToken.json +0 -0
  61. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/staking_token/contract.py +0 -0
  62. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/staking_token/contract.yaml +0 -0
  63. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/uniswap_v2_erc20/__init__.py +0 -0
  64. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/uniswap_v2_erc20/build/IUniswapV2ERC20.json +0 -0
  65. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/uniswap_v2_erc20/contract.py +0 -0
  66. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/uniswap_v2_erc20/contract.yaml +0 -0
  67. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/uniswap_v2_erc20/tests/__init__.py +0 -0
  68. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/data/contracts/uniswap_v2_erc20/tests/test_contract.py +0 -0
  69. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/keys.py +0 -0
  70. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/ledger/__init__.py +0 -0
  71. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/ledger/profiles.py +0 -0
  72. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/migration.py +0 -0
  73. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/operate_http/__init__.py +0 -0
  74. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/operate_http/exceptions.py +0 -0
  75. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/operate_types.py +0 -0
  76. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/pearl.py +0 -0
  77. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/quickstart/analyse_logs.py +0 -0
  78. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/quickstart/claim_staking_rewards.py +0 -0
  79. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/quickstart/reset_configs.py +0 -0
  80. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/quickstart/reset_password.py +0 -0
  81. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/quickstart/reset_staking.py +0 -0
  82. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/quickstart/run_service.py +0 -0
  83. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/quickstart/stop_service.py +0 -0
  84. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/quickstart/terminate_on_chain_service.py +0 -0
  85. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/quickstart/utils.py +0 -0
  86. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/resource.py +0 -0
  87. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/services/__init__.py +0 -0
  88. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/services/agent_runner.py +0 -0
  89. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/services/funding_manager.py +0 -0
  90. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/services/protocol.py +0 -0
  91. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/services/service.py +0 -0
  92. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/services/utils/__init__.py +0 -0
  93. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/services/utils/mech.py +0 -0
  94. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/services/utils/tendermint.py +0 -0
  95. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/settings.py +0 -0
  96. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/utils/__init__.py +0 -0
  97. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/utils/gnosis.py +0 -0
  98. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/utils/single_instance.py +0 -0
  99. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/utils/ssl.py +0 -0
  100. {olas_operate_middleware-0.13.0 → olas_operate_middleware-0.13.2}/operate/wallet/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: olas-operate-middleware
3
- Version: 0.13.0
3
+ Version: 0.13.2
4
4
  Summary:
5
5
  License-File: LICENSE
6
6
  Author: David Vilela
@@ -223,7 +223,7 @@ class OperateApp: # pylint: disable=too-many-instance-attributes
223
223
  )
224
224
 
225
225
  logger.info(f"Backing up existing {OPERATE} directory to {backup_path}")
226
- shutil.copytree(self._path, backup_path)
226
+ shutil.copytree(self._path, backup_path, ignore_dangling_symlinks=True)
227
227
  version_file.write_text(str(current_version))
228
228
 
229
229
  # remove recoverable files from the backup to save space
@@ -112,14 +112,30 @@ class BaseDeploymentRunner(AbstractDeploymentRunner, metaclass=ABCMeta):
112
112
 
113
113
  def _open_agent_runner_log_file(self) -> TextIOWrapper:
114
114
  """Open agent_runner.log file."""
115
- return (
116
- Path(self._work_directory).parent.parent.parent / "agent_runner.log"
117
- ).open("w+")
115
+ return (self._get_operate_dir() / "agent_runner.log").open("w+")
116
+
117
+ def _open_tendermint_log_file(self) -> TextIOWrapper:
118
+ """Open tm.log file."""
119
+ return (self._get_operate_dir() / "tm.log").open("w+")
120
+
121
+ def _get_operate_dir(self) -> Path:
122
+ """Get .operate dir."""
123
+ return Path(self._work_directory).parent.parent.parent
118
124
 
119
125
  def _run_aea_command(self, *args: str, cwd: Path) -> Any:
120
126
  """Run aea command."""
121
- cmd = " ".join(args)
122
- self.logger.info(f"Running aea command: {cmd} at {str(cwd)}")
127
+ no_password_args = []
128
+ for i, arg in enumerate(args):
129
+ if i > 0 and args[i - 1] == "--password":
130
+ no_password_args.append("******")
131
+ elif arg.startswith("--password="):
132
+ no_password_args.append("--password=******")
133
+ else:
134
+ no_password_args.append(arg)
135
+
136
+ self.logger.info(
137
+ f"Running aea command: {' '.join(no_password_args)} at {str(cwd)}"
138
+ )
123
139
  p = multiprocessing.Process(
124
140
  target=self.__class__._call_aea_command, # pylint: disable=protected-access
125
141
  args=(cwd, args),
@@ -128,7 +144,7 @@ class BaseDeploymentRunner(AbstractDeploymentRunner, metaclass=ABCMeta):
128
144
  p.join()
129
145
  if p.exitcode != 0:
130
146
  raise RuntimeError(
131
- f"aea command `{cmd}`execution failed with exit code: {p.exitcode}"
147
+ f"aea command `{' '.join(no_password_args)}` execution failed with exit code: {p.exitcode}"
132
148
  )
133
149
 
134
150
  @staticmethod
@@ -188,82 +204,90 @@ class BaseDeploymentRunner(AbstractDeploymentRunner, metaclass=ABCMeta):
188
204
  return env
189
205
 
190
206
  def _setup_agent(self, password: str) -> None:
191
- """Setup agent."""
192
- working_dir = self._work_directory
193
- env = self._prepare_agent_env()
194
- agent_alias_name = "agent"
195
- agent_dir_full_path = Path(working_dir) / agent_alias_name
196
- if not self._is_aea:
197
- if agent_dir_full_path.exists():
198
- # remove if exists before fetching! can have issues with retry mechanism of multiple start attempts
199
- with suppress(Exception):
200
- shutil.rmtree(agent_dir_full_path, ignore_errors=True)
201
-
202
- # Add keys
203
- agent_dir_full_path.mkdir(exist_ok=True, parents=True)
204
- shutil.copy(
205
- working_dir / "ethereum_private_key.txt",
206
- working_dir / "agent" / "ethereum_private_key.txt",
207
- )
208
- return
207
+ """Setup agent with retries for network operations."""
208
+ max_attempts = 10
209
+ for attempt in range(1, max_attempts + 1):
210
+ try:
211
+ working_dir = self._work_directory
212
+ env = self._prepare_agent_env()
213
+
214
+ # Clear agent directory before each attempt to avoid partial state
215
+ agent_alias_name = "agent"
216
+ agent_dir_full_path = Path(working_dir) / agent_alias_name
217
+ if agent_dir_full_path.exists():
218
+ with suppress(Exception):
219
+ shutil.rmtree(agent_dir_full_path, ignore_errors=True)
220
+
221
+ self._run_aea_command(
222
+ "init",
223
+ "--reset",
224
+ "--author",
225
+ "valory",
226
+ "--remote",
227
+ "--ipfs",
228
+ "--ipfs-node",
229
+ "/dns/registry.autonolas.tech/tcp/443/https",
230
+ cwd=working_dir,
231
+ )
209
232
 
210
- self._run_aea_command(
211
- "init",
212
- "--reset",
213
- "--author",
214
- "valory",
215
- "--remote",
216
- "--ipfs",
217
- "--ipfs-node",
218
- "/dns/registry.autonolas.tech/tcp/443/https",
219
- cwd=working_dir,
220
- )
233
+ self._run_aea_command(
234
+ "-s",
235
+ "fetch",
236
+ env["AEA_AGENT"],
237
+ "--alias",
238
+ agent_alias_name,
239
+ cwd=working_dir,
240
+ )
221
241
 
222
- if agent_dir_full_path.exists():
223
- # remove if exists before fetching! can have issues with retry mechanism of multiple start attempts
224
- with suppress(Exception):
225
- shutil.rmtree(agent_dir_full_path, ignore_errors=True)
226
-
227
- self._run_aea_command(
228
- "-s",
229
- "fetch",
230
- env["AEA_AGENT"],
231
- "--alias",
232
- agent_alias_name,
233
- cwd=working_dir,
234
- )
242
+ # Add keys
243
+ shutil.copy(
244
+ working_dir / "ethereum_private_key.txt",
245
+ working_dir / "agent" / "ethereum_private_key.txt",
246
+ )
235
247
 
236
- # Add keys
237
- shutil.copy(
238
- working_dir / "ethereum_private_key.txt",
239
- working_dir / "agent" / "ethereum_private_key.txt",
240
- )
248
+ self._run_aea_command(
249
+ "-s",
250
+ "add-key",
251
+ "--password",
252
+ password,
253
+ "ethereum",
254
+ cwd=working_dir / "agent",
255
+ )
256
+ self._run_aea_command(
257
+ "-s",
258
+ "add-key",
259
+ "--password",
260
+ password,
261
+ "ethereum",
262
+ "--connection",
263
+ cwd=working_dir / "agent",
264
+ )
241
265
 
242
- self._run_aea_command(
243
- "-s",
244
- "add-key",
245
- "--password",
246
- password,
247
- "ethereum",
248
- cwd=working_dir / "agent",
249
- )
250
- self._run_aea_command(
251
- "-s",
252
- "add-key",
253
- "--password",
254
- password,
255
- "ethereum",
256
- "--connection",
257
- cwd=working_dir / "agent",
258
- )
266
+ self._run_aea_command(
267
+ "-s",
268
+ "issue-certificates",
269
+ "--password",
270
+ password,
271
+ cwd=working_dir / "agent",
272
+ )
259
273
 
260
- self._run_aea_command(
261
- "-s",
262
- "issue-certificates",
263
- "--password",
264
- password,
265
- cwd=working_dir / "agent",
266
- )
274
+ # Success - break out of retry loop
275
+ self.logger.info(
276
+ f"Agent setup completed successfully on attempt {attempt}"
277
+ )
278
+ break
279
+
280
+ except Exception as e: # pylint: disable=broad-except
281
+ self.logger.warning(
282
+ f"Agent setup attempt {attempt}/{max_attempts} failed: {e}"
283
+ )
284
+ if attempt < max_attempts:
285
+ sleep_time = attempt * 5
286
+ self.logger.info(f"Retrying agent setup in {sleep_time} seconds...")
287
+ time.sleep(sleep_time)
288
+ else:
289
+ self.logger.error(f"All {max_attempts} agent setup attempts failed")
290
+ raise
267
291
 
268
292
  def start(self, password: str) -> None:
269
293
  """Start the deployment with retries."""
@@ -439,12 +463,12 @@ class PyInstallerHostDeploymentRunnerMac(PyInstallerHostDeploymentRunner):
439
463
  **env,
440
464
  }
441
465
  env["PATH"] = os.path.dirname(sys.executable) + ":" + os.environ["PATH"]
442
-
466
+ tm_log_file = self._open_tendermint_log_file()
443
467
  process = subprocess.Popen( # pylint: disable=consider-using-with,subprocess-popen-preexec-fn # nosec
444
468
  args=[self._tendermint_bin],
445
469
  cwd=working_dir,
446
- stdout=subprocess.DEVNULL,
447
- stderr=subprocess.DEVNULL,
470
+ stdout=tm_log_file,
471
+ stderr=tm_log_file,
448
472
  env=env,
449
473
  preexec_fn=os.setpgrp, # pylint: disable=subprocess-popen-preexec-fn # nosec
450
474
  )
@@ -559,13 +583,14 @@ class PyInstallerHostDeploymentRunnerWindows(PyInstallerHostDeploymentRunner):
559
583
  env = {
560
584
  **env,
561
585
  }
586
+ tm_log_file = self._open_tendermint_log_file()
562
587
  env["PATH"] = os.path.dirname(sys.executable) + ";" + os.environ["PATH"]
563
588
 
564
589
  process = subprocess.Popen( # pylint: disable=consider-using-with # nosec
565
590
  args=[self._tendermint_bin],
566
591
  cwd=working_dir,
567
- stdout=subprocess.DEVNULL,
568
- stderr=subprocess.DEVNULL,
592
+ stdout=tm_log_file,
593
+ stderr=tm_log_file,
569
594
  env=env,
570
595
  creationflags=0x00000200, # Detach process from the main process
571
596
  )
@@ -21,6 +21,7 @@
21
21
  import asyncio
22
22
  import json
23
23
  import logging
24
+ import time
24
25
  import typing as t
25
26
  from concurrent.futures import ThreadPoolExecutor
26
27
  from http import HTTPStatus
@@ -40,6 +41,8 @@ class HealthChecker:
40
41
  PORT_UP_TIMEOUT_DEFAULT = 300 # seconds
41
42
  REQUEST_TIMEOUT_DEFAULT = 90 # seconds
42
43
  NUMBER_OF_FAILS_DEFAULT = 60
44
+ FAILFAST_NUM = 15
45
+ FAILFAST_TIMEOUT = 15 * 60 # 15 minutes
43
46
 
44
47
  def __init__(
45
48
  self,
@@ -121,7 +124,7 @@ class HealthChecker:
121
124
  )
122
125
  return False
123
126
 
124
- async def healthcheck_job(
127
+ async def healthcheck_job( # pylint: disable=too-many-statements
125
128
  self,
126
129
  service_config_id: str,
127
130
  ) -> None:
@@ -214,7 +217,24 @@ class HealthChecker:
214
217
  if exception is not None:
215
218
  raise exception
216
219
 
220
+ async def _stop(
221
+ service_manager: ServiceManager, service_config_id: str
222
+ ) -> None:
223
+ def _do_stop() -> None:
224
+ service_manager.stop_service_locally(
225
+ service_config_id=service_config_id
226
+ )
227
+
228
+ loop = asyncio.get_event_loop()
229
+ with ThreadPoolExecutor() as executor:
230
+ future = loop.run_in_executor(executor, _do_stop)
231
+ await future
232
+ exception = future.exception()
233
+ if exception is not None:
234
+ raise exception
235
+
217
236
  # upper cycle
237
+ failfast_records: t.List[float] = []
218
238
  while True:
219
239
  self.logger.info(
220
240
  f"[HEALTH_CHECKER] {service_config_id} wait for port ready"
@@ -224,6 +244,7 @@ class HealthChecker:
224
244
  self.logger.info(
225
245
  f"[HEALTH_CHECKER] {service_config_id} port is ready, checking health every {self.sleep_period}"
226
246
  )
247
+ failfast_records = []
227
248
  await _check_health(
228
249
  number_of_fails=self.number_of_fails,
229
250
  sleep_period=self.sleep_period,
@@ -236,7 +257,22 @@ class HealthChecker:
236
257
 
237
258
  # perform restart
238
259
  # TODO: blocking!!!!!!!
239
- await _restart(self._service_manager, service_config_id)
260
+ while True:
261
+ # we count every restart till success (port is up and healtcheck started)
262
+ failfast_records.append(time.time())
263
+ try:
264
+ await _restart(self._service_manager, service_config_id)
265
+ break
266
+ except Exception: # pylint: disable=broad-except
267
+ if (len(failfast_records) >= self.FAILFAST_NUM) or (
268
+ time.time() - failfast_records[0]
269
+ ) > self.FAILFAST_TIMEOUT:
270
+ await _stop(self._service_manager, service_config_id)
271
+ raise
272
+
273
+ self.logger.exception(f"Restart problem: {service_config_id}")
274
+ await asyncio.sleep(30)
275
+
240
276
  except Exception:
241
277
  self.logger.exception(
242
278
  f"Problems running healthcheck job for {service_config_id}"
@@ -22,6 +22,7 @@
22
22
  import json
23
23
  import logging
24
24
  import os
25
+ import time
25
26
  import traceback
26
27
  import typing as t
27
28
  from collections import Counter, defaultdict
@@ -105,6 +106,8 @@ from operate.wallet.master import InsufficientFundsException, MasterWalletManage
105
106
  # If multiple agents are provided in the service.yaml file, only the 0th index config will be used.
106
107
  NUM_LOCAL_AGENT_INSTANCES = 1
107
108
 
109
+ RPC_SYNC_TIMEOUT = 15
110
+
108
111
 
109
112
  class ServiceManager:
110
113
  """Service manager."""
@@ -964,6 +967,10 @@ class ServiceManager:
964
967
  chain_data.token = event_data["args"]["serviceId"]
965
968
  service.store()
966
969
 
970
+ if is_first_mint: # Hotfix to prevent RPC out-of-sync issues
971
+ time.sleep(RPC_SYNC_TIMEOUT)
972
+
973
+ # Activate service
967
974
  if (
968
975
  self._get_on_chain_state(service=service, chain=chain)
969
976
  == OnChainState.PRE_REGISTRATION
@@ -1028,6 +1035,10 @@ class ServiceManager:
1028
1035
  )
1029
1036
  ).settle()
1030
1037
 
1038
+ if is_first_mint: # Hotfix to prevent RPC out-of-sync issues
1039
+ time.sleep(RPC_SYNC_TIMEOUT)
1040
+
1041
+ # Register agent instances
1031
1042
  if (
1032
1043
  self._get_on_chain_state(service=service, chain=chain)
1033
1044
  == OnChainState.ACTIVE_REGISTRATION
@@ -1094,6 +1105,9 @@ class ServiceManager:
1094
1105
  )
1095
1106
  ).settle()
1096
1107
 
1108
+ if is_first_mint: # Hotfix to prevent RPC out-of-sync issues
1109
+ time.sleep(RPC_SYNC_TIMEOUT)
1110
+
1097
1111
  # Deploy service
1098
1112
  is_initial_funding = False
1099
1113
  if (
@@ -722,6 +722,9 @@ class EthereumMasterWallet(MasterWallet):
722
722
  "The master wallet cannot be set as the Safe backup owner."
723
723
  )
724
724
 
725
+ if self.address not in owners:
726
+ return False
727
+
725
728
  owners.remove(self.address)
726
729
  old_backup_owner = owners[0] if owners else None
727
730
 
@@ -769,7 +772,9 @@ class EthereumMasterWallet(MasterWallet):
769
772
  chain_str = chain.value
770
773
  ledger_api = self.ledger_api(chain=chain, rpc=rpc)
771
774
  owners = get_owners(ledger_api=ledger_api, safe=safe)
772
- owners.remove(self.address)
775
+
776
+ if self.address in owners:
777
+ owners.remove(self.address)
773
778
 
774
779
  balances[chain_str] = {self.address: {}, safe: {}}
775
780
 
@@ -781,8 +786,8 @@ class EthereumMasterWallet(MasterWallet):
781
786
  balances[chain_str][safe][asset] = self.get_balance(
782
787
  chain=chain, asset=asset, from_safe=True
783
788
  )
784
- wallet_json["safes"][chain.value] = {
785
- wallet_json["safes"][chain.value]: {
789
+ wallet_json["safes"][chain_str] = {
790
+ safe: {
786
791
  "backup_owners": owners,
787
792
  "balances": balances[chain_str][safe],
788
793
  }
@@ -791,6 +796,9 @@ class EthereumMasterWallet(MasterWallet):
791
796
 
792
797
  wallet_json["balances"] = balances
793
798
  wallet_json["extended_json"] = True
799
+ wallet_json["all_safes_have_backup_owner"] = all(
800
+ len(owners) > 0 for owners in owner_sets
801
+ )
794
802
  wallet_json["consistent_safe_address"] = len(set(self.safes.values())) == 1
795
803
  wallet_json["consistent_backup_owner"] = len(owner_sets) == 1
796
804
  wallet_json["consistent_backup_owner_count"] = all(
@@ -19,6 +19,7 @@
19
19
 
20
20
  """Wallet recovery manager"""
21
21
 
22
+ import enum
22
23
  import shutil
23
24
  import typing as t
24
25
  import uuid
@@ -49,6 +50,19 @@ RECOVERY_NEW_OBJECTS_DIR = "new"
49
50
  RECOVERY_OLD_OBJECTS_DIR = "old"
50
51
 
51
52
 
53
+ class WalletRecoveryStatus(str, enum.Enum):
54
+ """ProviderRequestStatus"""
55
+
56
+ NOT_PREPARED = "NOT_PREPARED"
57
+ PREPARED = "PREPARED"
58
+ IN_PROGRESS = "IN_PROGRESS"
59
+ COMPLETED = "COMPLETED"
60
+
61
+ def __str__(self) -> str:
62
+ """__str__"""
63
+ return self.value
64
+
65
+
52
66
  class WalletRecoveryError(Exception):
53
67
  """WalletRecoveryError"""
54
68
 
@@ -125,20 +139,16 @@ class WalletRecoveryManager:
125
139
  )
126
140
 
127
141
  last_prepared_bundle_id = self.data.last_prepared_bundle_id
128
- if last_prepared_bundle_id is not None:
129
- (
130
- _,
131
- num_safes_with_new_wallet,
132
- _,
133
- _,
134
- ) = self._get_swap_status(last_prepared_bundle_id)
135
- if num_safes_with_new_wallet > 0:
136
- self.logger.info(
137
- f"[WALLET RECOVERY MANAGER] Uncompleted bundle {last_prepared_bundle_id} has Safes with new wallet."
138
- )
139
- return self._load_bundle(
140
- bundle_id=last_prepared_bundle_id, new_password=new_password
141
- )
142
+ if (
143
+ last_prepared_bundle_id is not None
144
+ and self.status()["num_safes_with_new_wallet"] > 0
145
+ ):
146
+ self.logger.info(
147
+ f"[WALLET RECOVERY MANAGER] Uncompleted bundle {last_prepared_bundle_id} has Safes with new wallet."
148
+ )
149
+ return self._load_bundle(
150
+ bundle_id=last_prepared_bundle_id, new_password=new_password
151
+ )
142
152
 
143
153
  # Create new recovery bundle
144
154
  bundle_id = f"{RECOVERY_BUNDLE_PREFIX}{str(uuid.uuid4())}"
@@ -178,23 +188,45 @@ class WalletRecoveryManager:
178
188
  )
179
189
  return self._load_bundle(bundle_id=bundle_id, new_password=new_password)
180
190
 
181
- def _get_swap_status(self, bundle_id: str) -> t.Tuple[int, int, int, int]:
191
+ def _load_bundle( # pylint: disable=too-many-locals
192
+ self, bundle_id: str, new_password: t.Optional[str] = None
193
+ ) -> t.Dict:
182
194
  new_root = self.path / bundle_id / RECOVERY_NEW_OBJECTS_DIR
195
+
196
+ new_user_account = UserAccount.load(new_root / USER_JSON)
197
+ if new_password is not None and not new_user_account.is_valid(
198
+ password=new_password
199
+ ):
200
+ raise ValueError(MSG_INVALID_PASSWORD)
201
+
183
202
  new_wallets_path = new_root / WALLETS_DIR
184
- new_wallet_manager = MasterWalletManager(path=new_wallets_path, password=None)
203
+ new_wallet_manager = MasterWalletManager(
204
+ path=new_wallets_path, password=new_password
205
+ )
185
206
 
186
207
  num_safes = 0
187
208
  num_safes_with_new_wallet = 0
188
209
  num_safes_with_old_wallet = 0
189
210
  num_safes_with_both_wallets = 0
211
+ backup_owner_sets = set()
190
212
 
213
+ wallets = []
191
214
  for wallet in self.wallet_manager:
192
215
  new_wallet = next(
193
216
  (w for w in new_wallet_manager if w.ledger_type == wallet.ledger_type)
194
217
  )
218
+ new_mnemonic = None
219
+ if new_password:
220
+ new_mnemonic = new_wallet.decrypt_mnemonic(password=new_password)
221
+
222
+ wallet_json = wallet.json
223
+
195
224
  for chain, safe in wallet.safes.items():
225
+ chain_str = chain.value
196
226
  ledger_api = get_default_ledger_api(chain)
197
227
  owners = get_owners(ledger_api=ledger_api, safe=safe)
228
+ backup_owners = list(set(owners) - {wallet.address, new_wallet.address})
229
+ backup_owner_sets.add(frozenset(backup_owners))
198
230
 
199
231
  num_safes += 1
200
232
  if new_wallet.address in owners and wallet.address in owners:
@@ -204,42 +236,52 @@ class WalletRecoveryManager:
204
236
  if wallet.address in owners:
205
237
  num_safes_with_old_wallet += 1
206
238
 
207
- return (
208
- num_safes,
209
- num_safes_with_new_wallet,
210
- num_safes_with_old_wallet,
211
- num_safes_with_both_wallets,
212
- )
213
-
214
- def _load_bundle(self, bundle_id: str, new_password: str) -> t.Dict:
215
- new_root = self.path / bundle_id / RECOVERY_NEW_OBJECTS_DIR
216
-
217
- new_user_account = UserAccount.load(new_root / USER_JSON)
218
- if not new_user_account.is_valid(password=new_password):
219
- raise ValueError(MSG_INVALID_PASSWORD)
220
-
221
- new_wallets_path = new_root / WALLETS_DIR
222
- new_wallet_manager = MasterWalletManager(
223
- path=new_wallets_path, password=new_password
224
- )
239
+ wallet_json["safes"][chain_str] = {
240
+ safe: {
241
+ "owners": owners,
242
+ "backup_owners": backup_owners,
243
+ "owner_to_remove": wallet.address
244
+ if wallet.address in owners
245
+ else None,
246
+ "owner_to_add": new_wallet.address
247
+ if new_wallet.address not in owners
248
+ else None,
249
+ }
250
+ }
225
251
 
226
- wallets = []
227
- for wallet in self.wallet_manager:
228
- ledger_type = wallet.ledger_type
229
- new_wallet = new_wallet_manager.load(ledger_type=ledger_type)
230
- new_mnemonic = None
231
- if new_password:
232
- new_mnemonic = new_wallet.decrypt_mnemonic(password=new_password)
233
252
  wallets.append(
234
253
  {
235
- "current_wallet": wallet.json,
254
+ "current_wallet": wallet_json,
236
255
  "new_wallet": new_wallet.json,
237
256
  "new_mnemonic": new_mnemonic,
238
257
  }
239
258
  )
259
+
260
+ if num_safes_with_new_wallet == 0:
261
+ status = WalletRecoveryStatus.PREPARED
262
+ elif num_safes_with_new_wallet < num_safes:
263
+ status = WalletRecoveryStatus.IN_PROGRESS
264
+ else:
265
+ status = WalletRecoveryStatus.COMPLETED
266
+
240
267
  return {
241
268
  "id": bundle_id,
242
269
  "wallets": wallets,
270
+ "status": status,
271
+ "all_safes_have_backup_owner": all(
272
+ len(owners) >= 1 for owners in backup_owner_sets
273
+ ),
274
+ "consistent_backup_owner": len(backup_owner_sets) == 1,
275
+ "consistent_backup_owner_count": all(
276
+ len(owners) == 1 for owners in backup_owner_sets
277
+ ),
278
+ "prepared": bundle_id is not None,
279
+ "has_swaps": num_safes_with_new_wallet > 0,
280
+ "has_pending_swaps": num_safes_with_new_wallet < num_safes,
281
+ "num_safes": num_safes,
282
+ "num_safes_with_new_wallet": num_safes_with_new_wallet,
283
+ "num_safes_with_old_wallet": num_safes_with_old_wallet,
284
+ "num_safes_with_both_wallets": num_safes_with_both_wallets,
243
285
  }
244
286
 
245
287
  def recovery_requirements( # pylint: disable=too-many-locals
@@ -317,27 +359,37 @@ class WalletRecoveryManager:
317
359
  def status(self) -> t.Dict[str, t.Any]:
318
360
  """Get recovery status."""
319
361
  bundle_id = self.data.last_prepared_bundle_id
320
- if not bundle_id:
362
+ if bundle_id is None:
363
+ backup_owner_sets = set()
364
+ for wallet in self.wallet_manager:
365
+ for chain, safe in wallet.safes.items():
366
+ ledger_api = get_default_ledger_api(chain)
367
+ owners = get_owners(ledger_api=ledger_api, safe=safe)
368
+ backup_owners = list(set(owners) - {wallet.address})
369
+ backup_owner_sets.add(frozenset(backup_owners))
370
+
321
371
  return {
372
+ "id": None,
373
+ "wallets": [],
374
+ "status": WalletRecoveryStatus.NOT_PREPARED,
375
+ "all_safes_have_backup_owner": all(
376
+ len(owners) >= 1 for owners in backup_owner_sets
377
+ ),
378
+ "consistent_backup_owner": len(backup_owner_sets) == 1,
379
+ "consistent_backup_owner_count": all(
380
+ len(owners) == 1 for owners in backup_owner_sets
381
+ ),
322
382
  "prepared": False,
323
383
  "bundle_id": bundle_id,
324
384
  "has_swaps": False,
325
385
  "has_pending_swaps": False,
386
+ "num_safes": 0,
387
+ "num_safes_with_new_wallet": 0,
388
+ "num_safes_with_old_wallet": 0,
389
+ "num_safes_with_both_wallets": 0,
326
390
  }
327
391
 
328
- (
329
- num_safes,
330
- num_safes_with_new_wallet,
331
- _,
332
- _,
333
- ) = self._get_swap_status(bundle_id)
334
-
335
- return {
336
- "prepared": bundle_id is not None,
337
- "bundle_id": bundle_id,
338
- "has_swaps": num_safes_with_new_wallet > 0,
339
- "has_pending_swaps": num_safes_with_new_wallet < num_safes,
340
- }
392
+ return self._load_bundle(bundle_id=bundle_id)
341
393
 
342
394
  def complete_recovery( # pylint: disable=too-many-locals,too-many-statements
343
395
  self, raise_if_inconsistent_owners: bool = True
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "olas-operate-middleware"
3
- version = "0.13.0"
3
+ version = "0.13.2"
4
4
  description = ""
5
5
  authors = ["David Vilela <dvilelaf@gmail.com>", "Viraj Patel <vptl185@gmail.com>"]
6
6
  readme = "README.md"