port-ocean 0.9.14__py3-none-any.whl → 0.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of port-ocean might be problematic. Click here for more details.
- port_ocean/clients/port/client.py +17 -0
- port_ocean/config/settings.py +3 -3
- port_ocean/core/event_listener/base.py +37 -0
- port_ocean/core/event_listener/http.py +1 -1
- port_ocean/core/event_listener/kafka.py +8 -4
- port_ocean/core/event_listener/once.py +96 -1
- port_ocean/core/event_listener/polling.py +14 -10
- port_ocean/core/handlers/resync_state_updater/__init__.py +5 -0
- port_ocean/core/handlers/resync_state_updater/updater.py +84 -0
- port_ocean/core/models.py +5 -2
- port_ocean/core/utils.py +3 -2
- port_ocean/ocean.py +25 -7
- port_ocean/utils/misc.py +7 -1
- port_ocean/utils/time.py +54 -0
- {port_ocean-0.9.14.dist-info → port_ocean-0.10.0.dist-info}/METADATA +1 -1
- {port_ocean-0.9.14.dist-info → port_ocean-0.10.0.dist-info}/RECORD +19 -16
- {port_ocean-0.9.14.dist-info → port_ocean-0.10.0.dist-info}/LICENSE.md +0 -0
- {port_ocean-0.9.14.dist-info → port_ocean-0.10.0.dist-info}/WHEEL +0 -0
- {port_ocean-0.9.14.dist-info → port_ocean-0.10.0.dist-info}/entry_points.txt +0 -0
|
@@ -13,6 +13,7 @@ from port_ocean.clients.port.utils import (
|
|
|
13
13
|
get_internal_http_client,
|
|
14
14
|
)
|
|
15
15
|
from port_ocean.exceptions.clients import KafkaCredentialsNotFound
|
|
16
|
+
from typing import Any
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
class PortClient(
|
|
@@ -75,3 +76,19 @@ class PortClient(
|
|
|
75
76
|
handle_status_code(response)
|
|
76
77
|
|
|
77
78
|
return response.json()["organization"]["id"]
|
|
79
|
+
|
|
80
|
+
async def update_integration_state(
|
|
81
|
+
self, state: dict[str, Any], should_raise: bool = True, should_log: bool = True
|
|
82
|
+
) -> dict[str, Any]:
|
|
83
|
+
if should_log:
|
|
84
|
+
logger.debug(f"Updating integration resync state with: {state}")
|
|
85
|
+
response = await self.client.patch(
|
|
86
|
+
f"{self.api_url}/integration/{self.integration_identifier}/resync-state",
|
|
87
|
+
headers=await self.auth.headers(),
|
|
88
|
+
json=state,
|
|
89
|
+
)
|
|
90
|
+
handle_status_code(response, should_raise, should_log)
|
|
91
|
+
if response.is_success and should_log:
|
|
92
|
+
logger.info("Integration resync state updated successfully")
|
|
93
|
+
|
|
94
|
+
return response.json().get("integration", {})
|
port_ocean/config/settings.py
CHANGED
|
@@ -76,7 +76,7 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
|
|
|
76
76
|
integration: IntegrationSettings = Field(
|
|
77
77
|
default_factory=lambda: IntegrationSettings(type="", identifier="")
|
|
78
78
|
)
|
|
79
|
-
runtime: Runtime =
|
|
79
|
+
runtime: Runtime = Runtime.OnPrem
|
|
80
80
|
|
|
81
81
|
@root_validator()
|
|
82
82
|
def validate_integration_config(cls, values: dict[str, Any]) -> dict[str, Any]:
|
|
@@ -100,8 +100,8 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
|
|
|
100
100
|
return values
|
|
101
101
|
|
|
102
102
|
@validator("runtime")
|
|
103
|
-
def validate_runtime(cls, runtime:
|
|
104
|
-
if runtime ==
|
|
103
|
+
def validate_runtime(cls, runtime: Runtime) -> Runtime:
|
|
104
|
+
if runtime == Runtime.Saas:
|
|
105
105
|
spec = get_spec_file()
|
|
106
106
|
if spec is None:
|
|
107
107
|
raise ValueError(
|
|
@@ -5,6 +5,8 @@ from pydantic import Extra
|
|
|
5
5
|
|
|
6
6
|
from port_ocean.config.base import BaseOceanModel
|
|
7
7
|
from port_ocean.utils.signal import signal_handler
|
|
8
|
+
from port_ocean.context.ocean import ocean
|
|
9
|
+
from port_ocean.utils.misc import IntegrationStateStatus
|
|
8
10
|
|
|
9
11
|
|
|
10
12
|
class EventListenerEvents(TypedDict):
|
|
@@ -36,6 +38,41 @@ class BaseEventListener:
|
|
|
36
38
|
"""
|
|
37
39
|
pass
|
|
38
40
|
|
|
41
|
+
async def _before_resync(self) -> None:
|
|
42
|
+
"""
|
|
43
|
+
Can be used for event listeners that need to perform some action before resync.
|
|
44
|
+
"""
|
|
45
|
+
await ocean.app.resync_state_updater.update_before_resync()
|
|
46
|
+
|
|
47
|
+
async def _after_resync(self) -> None:
|
|
48
|
+
"""
|
|
49
|
+
Can be used for event listeners that need to perform some action after resync.
|
|
50
|
+
"""
|
|
51
|
+
await ocean.app.resync_state_updater.update_after_resync()
|
|
52
|
+
|
|
53
|
+
async def _on_resync_failure(self, e: Exception) -> None:
|
|
54
|
+
"""
|
|
55
|
+
Can be used for event listeners that need to handle resync failures.
|
|
56
|
+
"""
|
|
57
|
+
await ocean.app.resync_state_updater.update_after_resync(
|
|
58
|
+
IntegrationStateStatus.Failed
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
async def _resync(
|
|
62
|
+
self,
|
|
63
|
+
resync_args: dict[Any, Any],
|
|
64
|
+
) -> None:
|
|
65
|
+
"""
|
|
66
|
+
Triggers the "on_resync" event.
|
|
67
|
+
"""
|
|
68
|
+
await self._before_resync()
|
|
69
|
+
try:
|
|
70
|
+
await self.events["on_resync"](resync_args)
|
|
71
|
+
await self._after_resync()
|
|
72
|
+
except Exception as e:
|
|
73
|
+
await self._on_resync_failure(e)
|
|
74
|
+
raise e
|
|
75
|
+
|
|
39
76
|
|
|
40
77
|
class EventListenerSettings(BaseOceanModel, extra=Extra.allow):
|
|
41
78
|
type: str
|
|
@@ -99,9 +99,13 @@ class KafkaEventListener(BaseEventListener):
|
|
|
99
99
|
return False
|
|
100
100
|
|
|
101
101
|
integration_identifier = after.get("identifier")
|
|
102
|
-
if integration_identifier
|
|
103
|
-
|
|
104
|
-
|
|
102
|
+
if integration_identifier != self.integration_identifier:
|
|
103
|
+
return False
|
|
104
|
+
|
|
105
|
+
if after.get("updatedAt") == after.get("resyncState", {}).get("updatedAt"):
|
|
106
|
+
return False
|
|
107
|
+
|
|
108
|
+
if "change.log" in topic:
|
|
105
109
|
return msg_value.get("changelogDestination", {}).get("type", "") == "KAFKA"
|
|
106
110
|
|
|
107
111
|
return False
|
|
@@ -122,7 +126,7 @@ class KafkaEventListener(BaseEventListener):
|
|
|
122
126
|
|
|
123
127
|
if "change.log" in topic and message is not None:
|
|
124
128
|
try:
|
|
125
|
-
await self.
|
|
129
|
+
await self._resync(message)
|
|
126
130
|
except Exception as e:
|
|
127
131
|
_type, _, tb = sys.exc_info()
|
|
128
132
|
logger.opt(exception=(_type, None, tb)).error(
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import datetime
|
|
1
2
|
import signal
|
|
2
3
|
from typing import Literal, Any
|
|
3
4
|
|
|
@@ -9,6 +10,9 @@ from port_ocean.core.event_listener.base import (
|
|
|
9
10
|
EventListenerSettings,
|
|
10
11
|
)
|
|
11
12
|
from port_ocean.utils.repeat import repeat_every
|
|
13
|
+
from port_ocean.context.ocean import ocean
|
|
14
|
+
from port_ocean.utils.time import convert_str_to_utc_datetime, convert_to_minutes
|
|
15
|
+
from port_ocean.utils.misc import IntegrationStateStatus
|
|
12
16
|
|
|
13
17
|
|
|
14
18
|
class OnceEventListenerSettings(EventListenerSettings):
|
|
@@ -41,6 +45,97 @@ class OnceEventListener(BaseEventListener):
|
|
|
41
45
|
):
|
|
42
46
|
super().__init__(events)
|
|
43
47
|
self.event_listener_config = event_listener_config
|
|
48
|
+
self.cached_integration: dict[str, Any] | None = None
|
|
49
|
+
|
|
50
|
+
async def get_current_integration_cached(self) -> dict[str, Any]:
|
|
51
|
+
if self.cached_integration:
|
|
52
|
+
return self.cached_integration
|
|
53
|
+
|
|
54
|
+
self.cached_integration = await ocean.port_client.get_current_integration()
|
|
55
|
+
return self.cached_integration
|
|
56
|
+
|
|
57
|
+
async def get_saas_resync_initialization_and_interval(
|
|
58
|
+
self,
|
|
59
|
+
) -> tuple[int | None, datetime.datetime | None]:
|
|
60
|
+
"""
|
|
61
|
+
Get the scheduled resync interval and the last updated time of the integration config for the saas application.
|
|
62
|
+
interval is the saas configured resync interval time.
|
|
63
|
+
start_time is the last updated time of the integration config.
|
|
64
|
+
return: (interval, start_time)
|
|
65
|
+
"""
|
|
66
|
+
if not ocean.app.is_saas():
|
|
67
|
+
return (None, None)
|
|
68
|
+
|
|
69
|
+
try:
|
|
70
|
+
integration = await self.get_current_integration_cached()
|
|
71
|
+
except Exception as e:
|
|
72
|
+
logger.exception(f"Error occurred while getting current integration {e}")
|
|
73
|
+
return (None, None)
|
|
74
|
+
|
|
75
|
+
interval_str = (
|
|
76
|
+
integration.get("spec", {})
|
|
77
|
+
.get("appSpec", {})
|
|
78
|
+
.get("scheduledResyncInterval")
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
if not interval_str:
|
|
82
|
+
logger.error(
|
|
83
|
+
"Unexpected: scheduledResyncInterval not found for Saas integration, Cannot predict the next resync"
|
|
84
|
+
)
|
|
85
|
+
return (None, None)
|
|
86
|
+
|
|
87
|
+
last_updated_saas_integration_config_str = integration.get(
|
|
88
|
+
"statusInfo", {}
|
|
89
|
+
).get("updatedAt")
|
|
90
|
+
|
|
91
|
+
# we use the last updated time of the integration config as the start time since in saas application the interval is configured by the user from the portal
|
|
92
|
+
if not last_updated_saas_integration_config_str:
|
|
93
|
+
logger.error(
|
|
94
|
+
"Unexpected: updatedAt not found for Saas integration, Cannot predict the next resync"
|
|
95
|
+
)
|
|
96
|
+
return (None, None)
|
|
97
|
+
|
|
98
|
+
return (
|
|
99
|
+
convert_to_minutes(interval_str),
|
|
100
|
+
convert_str_to_utc_datetime(last_updated_saas_integration_config_str),
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
async def _before_resync(self) -> None:
|
|
104
|
+
if not ocean.app.is_saas():
|
|
105
|
+
# in case of non-saas, we still want to update the state before and after the resync
|
|
106
|
+
await super()._before_resync()
|
|
107
|
+
return
|
|
108
|
+
|
|
109
|
+
(interval, start_time) = (
|
|
110
|
+
await self.get_saas_resync_initialization_and_interval()
|
|
111
|
+
)
|
|
112
|
+
await ocean.app.resync_state_updater.update_before_resync(interval, start_time)
|
|
113
|
+
|
|
114
|
+
async def _after_resync(self) -> None:
|
|
115
|
+
if not ocean.app.is_saas():
|
|
116
|
+
# in case of non-saas, we still want to update the state before and after the resync
|
|
117
|
+
await super()._after_resync()
|
|
118
|
+
return
|
|
119
|
+
|
|
120
|
+
(interval, start_time) = (
|
|
121
|
+
await self.get_saas_resync_initialization_and_interval()
|
|
122
|
+
)
|
|
123
|
+
await ocean.app.resync_state_updater.update_after_resync(
|
|
124
|
+
IntegrationStateStatus.Completed, interval, start_time
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
async def _on_resync_failure(self, e: Exception) -> None:
|
|
128
|
+
if not ocean.app.is_saas():
|
|
129
|
+
# in case of non-saas, we still want to update the state before and after the resync
|
|
130
|
+
await super()._after_resync()
|
|
131
|
+
return
|
|
132
|
+
|
|
133
|
+
(interval, start_time) = (
|
|
134
|
+
await self.get_saas_resync_initialization_and_interval()
|
|
135
|
+
)
|
|
136
|
+
await ocean.app.resync_state_updater.update_after_resync(
|
|
137
|
+
IntegrationStateStatus.Failed, interval, start_time
|
|
138
|
+
)
|
|
44
139
|
|
|
45
140
|
async def _start(self) -> None:
|
|
46
141
|
"""
|
|
@@ -53,7 +148,7 @@ class OnceEventListener(BaseEventListener):
|
|
|
53
148
|
async def resync_and_exit() -> None:
|
|
54
149
|
logger.info("Once event listener started")
|
|
55
150
|
try:
|
|
56
|
-
await self.
|
|
151
|
+
await self._resync({})
|
|
57
152
|
except Exception:
|
|
58
153
|
# we catch all exceptions here to make sure the application will exit gracefully
|
|
59
154
|
logger.exception("Error occurred while resyncing")
|
|
@@ -49,7 +49,16 @@ class PollingEventListener(BaseEventListener):
|
|
|
49
49
|
):
|
|
50
50
|
super().__init__(events)
|
|
51
51
|
self.event_listener_config = event_listener_config
|
|
52
|
-
|
|
52
|
+
|
|
53
|
+
def should_resync(self, last_updated_at: str) -> bool:
|
|
54
|
+
_last_updated_at = (
|
|
55
|
+
ocean.app.resync_state_updater.last_integration_state_updated_at
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
if _last_updated_at is None:
|
|
59
|
+
return self.event_listener_config.resync_on_start
|
|
60
|
+
|
|
61
|
+
return _last_updated_at != last_updated_at
|
|
53
62
|
|
|
54
63
|
async def _start(self) -> None:
|
|
55
64
|
"""
|
|
@@ -69,17 +78,12 @@ class PollingEventListener(BaseEventListener):
|
|
|
69
78
|
integration = await ocean.app.port_client.get_current_integration()
|
|
70
79
|
last_updated_at = integration["updatedAt"]
|
|
71
80
|
|
|
72
|
-
should_resync
|
|
73
|
-
self._last_updated_at is not None
|
|
74
|
-
or self.event_listener_config.resync_on_start
|
|
75
|
-
) and self._last_updated_at != last_updated_at
|
|
76
|
-
|
|
77
|
-
if should_resync:
|
|
81
|
+
if self.should_resync(last_updated_at):
|
|
78
82
|
logger.info("Detected change in integration, resyncing")
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
self.events["on_resync"]({}) # type: ignore
|
|
83
|
+
ocean.app.resync_state_updater.last_integration_state_updated_at = (
|
|
84
|
+
last_updated_at
|
|
82
85
|
)
|
|
86
|
+
running_task: Task[Any] = get_event_loop().create_task(self._resync({}))
|
|
83
87
|
signal_handler.register(running_task.cancel)
|
|
84
88
|
|
|
85
89
|
await running_task
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
from typing import Any, Literal
|
|
3
|
+
from port_ocean.clients.port.client import PortClient
|
|
4
|
+
from port_ocean.utils.misc import IntegrationStateStatus
|
|
5
|
+
from port_ocean.utils.time import get_next_occurrence
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ResyncStateUpdater:
|
|
9
|
+
def __init__(self, port_client: PortClient, scheduled_resync_interval: int | None):
|
|
10
|
+
self.port_client = port_client
|
|
11
|
+
self.initiated_at = datetime.datetime.now(tz=datetime.timezone.utc)
|
|
12
|
+
self.scheduled_resync_interval = scheduled_resync_interval
|
|
13
|
+
|
|
14
|
+
# This is used to differ between integration changes that require a full resync and state changes
|
|
15
|
+
# So that the polling event-listener can decide whether to perform a full resync or not
|
|
16
|
+
# TODO: remove this once we separate the state from the integration
|
|
17
|
+
self.last_integration_state_updated_at: str = ""
|
|
18
|
+
|
|
19
|
+
def _calculate_next_scheduled_resync(
|
|
20
|
+
self,
|
|
21
|
+
interval: int | None = None,
|
|
22
|
+
custom_start_time: datetime.datetime | None = None,
|
|
23
|
+
) -> str | None:
|
|
24
|
+
if interval is None:
|
|
25
|
+
return None
|
|
26
|
+
return get_next_occurrence(
|
|
27
|
+
interval * 60, custom_start_time or self.initiated_at
|
|
28
|
+
).isoformat()
|
|
29
|
+
|
|
30
|
+
async def update_before_resync(
|
|
31
|
+
self,
|
|
32
|
+
interval: int | None = None,
|
|
33
|
+
custom_start_time: datetime.datetime | None = None,
|
|
34
|
+
) -> None:
|
|
35
|
+
_interval = interval or self.scheduled_resync_interval
|
|
36
|
+
nest_resync = self._calculate_next_scheduled_resync(
|
|
37
|
+
_interval, custom_start_time
|
|
38
|
+
)
|
|
39
|
+
state: dict[str, Any] = {
|
|
40
|
+
"status": IntegrationStateStatus.Running.value,
|
|
41
|
+
"lastResyncEnd": None,
|
|
42
|
+
"lastResyncStart": datetime.datetime.now(
|
|
43
|
+
tz=datetime.timezone.utc
|
|
44
|
+
).isoformat(),
|
|
45
|
+
"nextResync": nest_resync,
|
|
46
|
+
"intervalInMinuets": _interval,
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
integration = await self.port_client.update_integration_state(
|
|
50
|
+
state, should_raise=False
|
|
51
|
+
)
|
|
52
|
+
if integration:
|
|
53
|
+
self.last_integration_state_updated_at = integration["resyncState"][
|
|
54
|
+
"updatedAt"
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
async def update_after_resync(
|
|
58
|
+
self,
|
|
59
|
+
status: Literal[
|
|
60
|
+
IntegrationStateStatus.Completed, IntegrationStateStatus.Failed
|
|
61
|
+
] = IntegrationStateStatus.Completed,
|
|
62
|
+
interval: int | None = None,
|
|
63
|
+
custom_start_time: datetime.datetime | None = None,
|
|
64
|
+
) -> None:
|
|
65
|
+
_interval = interval or self.scheduled_resync_interval
|
|
66
|
+
nest_resync = self._calculate_next_scheduled_resync(
|
|
67
|
+
_interval, custom_start_time
|
|
68
|
+
)
|
|
69
|
+
state: dict[str, Any] = {
|
|
70
|
+
"status": status.value,
|
|
71
|
+
"lastResyncEnd": datetime.datetime.now(
|
|
72
|
+
tz=datetime.timezone.utc
|
|
73
|
+
).isoformat(),
|
|
74
|
+
"nextResync": nest_resync,
|
|
75
|
+
"intervalInMinuets": _interval,
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
integration = await self.port_client.update_integration_state(
|
|
79
|
+
state, should_raise=False
|
|
80
|
+
)
|
|
81
|
+
if integration:
|
|
82
|
+
self.last_integration_state_updated_at = integration["resyncState"][
|
|
83
|
+
"updatedAt"
|
|
84
|
+
]
|
port_ocean/core/models.py
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
from dataclasses import dataclass, field
|
|
2
|
-
from
|
|
2
|
+
from enum import Enum
|
|
3
|
+
from typing import Any
|
|
3
4
|
|
|
4
5
|
from pydantic import BaseModel
|
|
5
6
|
from pydantic.fields import Field
|
|
6
7
|
|
|
7
8
|
|
|
8
|
-
Runtime
|
|
9
|
+
class Runtime(Enum):
|
|
10
|
+
Saas = "Saas"
|
|
11
|
+
OnPrem = "OnPrem"
|
|
9
12
|
|
|
10
13
|
|
|
11
14
|
class Entity(BaseModel):
|
port_ocean/core/utils.py
CHANGED
|
@@ -35,12 +35,13 @@ def is_same_entity(first_entity: Entity, second_entity: Entity) -> bool:
|
|
|
35
35
|
|
|
36
36
|
|
|
37
37
|
async def validate_integration_runtime(
|
|
38
|
-
port_client: PortClient,
|
|
38
|
+
port_client: PortClient,
|
|
39
|
+
requested_runtime: Runtime,
|
|
39
40
|
) -> None:
|
|
40
41
|
logger.debug("Validating integration runtime")
|
|
41
42
|
current_integration = await port_client.get_current_integration(should_raise=False)
|
|
42
43
|
current_runtime = current_integration.get("installationType", "OnPrem")
|
|
43
|
-
if current_integration and current_runtime != requested_runtime:
|
|
44
|
+
if current_integration and current_runtime != requested_runtime.value:
|
|
44
45
|
raise IntegrationRuntimeException(
|
|
45
46
|
f"Invalid Runtime! Requested to run existing {current_runtime} integration in {requested_runtime} runtime."
|
|
46
47
|
)
|
port_ocean/ocean.py
CHANGED
|
@@ -9,6 +9,8 @@ from loguru import logger
|
|
|
9
9
|
from pydantic import BaseModel
|
|
10
10
|
from starlette.types import Scope, Receive, Send
|
|
11
11
|
|
|
12
|
+
from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
|
|
13
|
+
from port_ocean.core.models import Runtime
|
|
12
14
|
from port_ocean.clients.port.client import PortClient
|
|
13
15
|
from port_ocean.config.settings import (
|
|
14
16
|
IntegrationConfiguration,
|
|
@@ -24,6 +26,7 @@ from port_ocean.middlewares import request_handler
|
|
|
24
26
|
from port_ocean.utils.repeat import repeat_every
|
|
25
27
|
from port_ocean.utils.signal import signal_handler
|
|
26
28
|
from port_ocean.version import __integration_version__
|
|
29
|
+
from port_ocean.utils.misc import IntegrationStateStatus
|
|
27
30
|
|
|
28
31
|
|
|
29
32
|
class Ocean:
|
|
@@ -63,16 +66,27 @@ class Ocean:
|
|
|
63
66
|
integration_class(ocean) if integration_class else BaseIntegration(ocean)
|
|
64
67
|
)
|
|
65
68
|
|
|
69
|
+
self.resync_state_updater = ResyncStateUpdater(
|
|
70
|
+
self.port_client, self.config.scheduled_resync_interval
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
def is_saas(self) -> bool:
|
|
74
|
+
return self.config.runtime == Runtime.Saas
|
|
75
|
+
|
|
66
76
|
async def _setup_scheduled_resync(
|
|
67
77
|
self,
|
|
68
78
|
) -> None:
|
|
69
|
-
def execute_resync_all() -> None:
|
|
70
|
-
|
|
71
|
-
asyncio.set_event_loop(loop)
|
|
72
|
-
|
|
79
|
+
async def execute_resync_all() -> None:
|
|
80
|
+
await self.resync_state_updater.update_before_resync()
|
|
73
81
|
logger.info("Starting a new scheduled resync")
|
|
74
|
-
|
|
75
|
-
|
|
82
|
+
try:
|
|
83
|
+
await self.integration.sync_raw_all()
|
|
84
|
+
await self.resync_state_updater.update_after_resync()
|
|
85
|
+
except Exception as e:
|
|
86
|
+
await self.resync_state_updater.update_after_resync(
|
|
87
|
+
IntegrationStateStatus.Failed
|
|
88
|
+
)
|
|
89
|
+
raise e
|
|
76
90
|
|
|
77
91
|
interval = self.config.scheduled_resync_interval
|
|
78
92
|
if interval is not None:
|
|
@@ -83,7 +97,11 @@ class Ocean:
|
|
|
83
97
|
seconds=interval * 60,
|
|
84
98
|
# Not running the resync immediately because the event listener should run resync on startup
|
|
85
99
|
wait_first=True,
|
|
86
|
-
)(
|
|
100
|
+
)(
|
|
101
|
+
lambda: threading.Thread(
|
|
102
|
+
target=lambda: asyncio.run(execute_resync_all())
|
|
103
|
+
).start()
|
|
104
|
+
)
|
|
87
105
|
await repeated_function()
|
|
88
106
|
|
|
89
107
|
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
port_ocean/utils/misc.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from enum import Enum
|
|
1
2
|
import inspect
|
|
2
3
|
from importlib.util import spec_from_file_location, module_from_spec
|
|
3
4
|
from pathlib import Path
|
|
@@ -5,11 +6,16 @@ from time import time
|
|
|
5
6
|
from types import ModuleType
|
|
6
7
|
from typing import Callable, Any
|
|
7
8
|
from uuid import uuid4
|
|
8
|
-
|
|
9
9
|
import tomli
|
|
10
10
|
import yaml
|
|
11
11
|
|
|
12
12
|
|
|
13
|
+
class IntegrationStateStatus(Enum):
|
|
14
|
+
Running = "running"
|
|
15
|
+
Failed = "failed"
|
|
16
|
+
Completed = "completed"
|
|
17
|
+
|
|
18
|
+
|
|
13
19
|
def get_time(seconds_precision: bool = True) -> float:
|
|
14
20
|
"""Return current time as Unix/Epoch timestamp, in seconds.
|
|
15
21
|
:param seconds_precision: if True, return with seconds precision as integer (default).
|
port_ocean/utils/time.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
from loguru import logger
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def convert_str_to_utc_datetime(time_str: str) -> datetime.datetime | None:
|
|
6
|
+
"""
|
|
7
|
+
Convert a string representing time to a datetime object.
|
|
8
|
+
:param time_str: a string representing time in the format "2021-09-01T12:00:00Z"
|
|
9
|
+
"""
|
|
10
|
+
aware_date = datetime.datetime.fromisoformat(time_str)
|
|
11
|
+
if time_str.endswith("Z"):
|
|
12
|
+
aware_date = datetime.datetime.fromisoformat(time_str.replace("Z", "+00:00"))
|
|
13
|
+
return aware_date.astimezone(datetime.timezone.utc)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def convert_to_minutes(s: str) -> int:
|
|
17
|
+
minutes_per_unit = {"s": 1 / 60, "m": 1, "h": 60, "d": 1440, "w": 10080}
|
|
18
|
+
try:
|
|
19
|
+
return int(int(s[:-1]) * minutes_per_unit[s[-1]])
|
|
20
|
+
except Exception:
|
|
21
|
+
logger.error(f"Failed converting string to minutes, {s}")
|
|
22
|
+
raise ValueError(
|
|
23
|
+
f"Invalid format. Expected a string ending with {minutes_per_unit.keys()}"
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def get_next_occurrence(
|
|
28
|
+
interval_seconds: int,
|
|
29
|
+
start_time: datetime.datetime,
|
|
30
|
+
now: datetime.datetime | None = None,
|
|
31
|
+
) -> datetime.datetime:
|
|
32
|
+
"""
|
|
33
|
+
Predict the next occurrence of an event based on interval, start time, and current time.
|
|
34
|
+
|
|
35
|
+
:param interval_minutes: Interval between occurrences in minutes.
|
|
36
|
+
:param start_time: Start time of the event as a datetime object.
|
|
37
|
+
:param now: Current time as a datetime object.
|
|
38
|
+
:return: The next occurrence time as a datetime object.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
if now is None:
|
|
42
|
+
now = datetime.datetime.now(tz=datetime.timezone.utc)
|
|
43
|
+
# Calculate the total seconds elapsed since the start time
|
|
44
|
+
elapsed_seconds = (now - start_time).total_seconds()
|
|
45
|
+
|
|
46
|
+
# Calculate the number of intervals that have passed
|
|
47
|
+
intervals_passed = int(elapsed_seconds // interval_seconds)
|
|
48
|
+
|
|
49
|
+
# Calculate the next occurrence time
|
|
50
|
+
next_occurrence = start_time + datetime.timedelta(
|
|
51
|
+
seconds=(intervals_passed + 1) * interval_seconds
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
return next_occurrence
|
|
@@ -37,7 +37,7 @@ port_ocean/cli/utils.py,sha256=IUK2UbWqjci-lrcDdynZXqVP5B5TcjF0w5CpEVUks-k,54
|
|
|
37
37
|
port_ocean/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
38
38
|
port_ocean/clients/port/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
39
39
|
port_ocean/clients/port/authentication.py,sha256=t3z6h4vld-Tzkpth15sstaMJg0rccX-pXXjNtOa-nCY,2949
|
|
40
|
-
port_ocean/clients/port/client.py,sha256=
|
|
40
|
+
port_ocean/clients/port/client.py,sha256=Xd8Jk25Uh4WXY_WW-z1Qbv6F3ZTBFPoOolsxHMfozKw,3366
|
|
41
41
|
port_ocean/clients/port/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
42
42
|
port_ocean/clients/port/mixins/blueprints.py,sha256=8ZVC5i8K1WKQMJJiPqZmgcOlF3OyxWz1aAQ_WA5UW3c,4500
|
|
43
43
|
port_ocean/clients/port/mixins/entities.py,sha256=j3YqLb1zMmJrIutYsYsZ_WCT4xh9VXEIH1A0kYMUUtk,8104
|
|
@@ -49,7 +49,7 @@ port_ocean/clients/port/utils.py,sha256=O9mBu6zp4TfpS4SQ3qCPpn9ZVyYF8GKnji4UnYhM
|
|
|
49
49
|
port_ocean/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
50
50
|
port_ocean/config/base.py,sha256=x1gFbzujrxn7EJudRT81C6eN9WsYAb3vOHwcpcpX8Tc,6370
|
|
51
51
|
port_ocean/config/dynamic.py,sha256=qOFkRoJsn_BW7581omi_AoMxoHqasf_foxDQ_G11_SI,2030
|
|
52
|
-
port_ocean/config/settings.py,sha256=
|
|
52
|
+
port_ocean/config/settings.py,sha256=EguKa8idZWylil_dQucip6W_q4_WzfTr4_XQjtyE4ds,4187
|
|
53
53
|
port_ocean/consumers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
54
54
|
port_ocean/consumers/kafka_consumer.py,sha256=N8KocjBi9aR0BOPG8hgKovg-ns_ggpEjrSxqSqF_BSo,4710
|
|
55
55
|
port_ocean/context/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -62,12 +62,12 @@ port_ocean/core/defaults/clean.py,sha256=S3UAfca-oU89WJKIB4OgGjGjPr0vxBQ2aRZsLTZ
|
|
|
62
62
|
port_ocean/core/defaults/common.py,sha256=uVUg6VEn4RqtXQwLwMNGfkmT5zYRN_h5USfKw3poVyo,3561
|
|
63
63
|
port_ocean/core/defaults/initialize.py,sha256=U1JWdpG_snAyqBXEikCjA7JzL7UtpOyx5rLfPhrBIZk,8721
|
|
64
64
|
port_ocean/core/event_listener/__init__.py,sha256=mzJ33wRq0kh60fpVdOHVmvMTUQIvz3vxmifyBgwDn0E,889
|
|
65
|
-
port_ocean/core/event_listener/base.py,sha256=
|
|
65
|
+
port_ocean/core/event_listener/base.py,sha256=1Nmpg00OfT2AD2L8eFm4VQEcdG2TClpSWJMhWhAjkEE,2356
|
|
66
66
|
port_ocean/core/event_listener/factory.py,sha256=AYYfSHPAF7P5H-uQECXT0JVJjKDHrYkWJJBSL4mGkg8,3697
|
|
67
|
-
port_ocean/core/event_listener/http.py,sha256=
|
|
68
|
-
port_ocean/core/event_listener/kafka.py,sha256=
|
|
69
|
-
port_ocean/core/event_listener/once.py,sha256=
|
|
70
|
-
port_ocean/core/event_listener/polling.py,sha256=
|
|
67
|
+
port_ocean/core/event_listener/http.py,sha256=N8HrfFqR3KGKz96pWdp_pP-m30jGtcz_1CkijovkBN8,2565
|
|
68
|
+
port_ocean/core/event_listener/kafka.py,sha256=ulidnp4sz-chXwHsbH9JayVjcxy_mG6ts_Im3YKmLpI,6983
|
|
69
|
+
port_ocean/core/event_listener/once.py,sha256=iL3NkujZOw-7LpxT-EAUJUcAuiAZPm4ZzjHTSt9EdHs,5918
|
|
70
|
+
port_ocean/core/event_listener/polling.py,sha256=d9E3oRLy-Ogb0oadZNxSDgSLIHe4z92uMVwztscZycg,3667
|
|
71
71
|
port_ocean/core/handlers/__init__.py,sha256=d7ShmS90gLRzGKJA6oNy2Zs_dF2yjkmYZInRhBnO9Rw,572
|
|
72
72
|
port_ocean/core/handlers/base.py,sha256=cTarblazu8yh8xz2FpB-dzDKuXxtoi143XJgPbV_DcM,157
|
|
73
73
|
port_ocean/core/handlers/entities_state_applier/__init__.py,sha256=kgLZDCeCEzi4r-0nzW9k78haOZNf6PX7mJOUr34A4c8,173
|
|
@@ -83,6 +83,8 @@ port_ocean/core/handlers/port_app_config/__init__.py,sha256=8AAT5OthiVM7KCcM34iE
|
|
|
83
83
|
port_ocean/core/handlers/port_app_config/api.py,sha256=6VbKPwFzsWG0IYsVD81hxSmfqtHUFqrfUuj1DBX5g4w,853
|
|
84
84
|
port_ocean/core/handlers/port_app_config/base.py,sha256=4Nxt2g8voEIHJ4Y1Km5NJcaG2iSbCklw5P8-Kus7Y9k,3007
|
|
85
85
|
port_ocean/core/handlers/port_app_config/models.py,sha256=4dw6HbgjMG3advpN3x6XF35xsgScnWm0KKTERG4CYZ8,2201
|
|
86
|
+
port_ocean/core/handlers/resync_state_updater/__init__.py,sha256=kG6y-JQGpPfuTHh912L_bctIDCzAK4DN-d00S7rguWU,81
|
|
87
|
+
port_ocean/core/handlers/resync_state_updater/updater.py,sha256=Yg9ET6ZV5B9GW7u6zZA6GlB_71kmvxvYX2FWgQNzMvo,3182
|
|
86
88
|
port_ocean/core/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
87
89
|
port_ocean/core/integrations/base.py,sha256=KHsRYFZ38ff3AkkqIQu45883ovgKOJn_fZfnTNy7HWY,2952
|
|
88
90
|
port_ocean/core/integrations/mixins/__init__.py,sha256=FA1FEKMM6P-L2_m7Q4L20mFa4_RgZnwSRmTCreKcBVM,220
|
|
@@ -91,9 +93,9 @@ port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcO
|
|
|
91
93
|
port_ocean/core/integrations/mixins/sync.py,sha256=B9fEs8faaYLLikH9GBjE_E61vo0bQDjIGQsQ1SRXOlA,3931
|
|
92
94
|
port_ocean/core/integrations/mixins/sync_raw.py,sha256=2vyHhGWyPchVfSUKRRfSJ2XsX55ygLWCFrKDaqkNd0o,18386
|
|
93
95
|
port_ocean/core/integrations/mixins/utils.py,sha256=7y1rGETZIjOQadyIjFJXIHKkQFKx_SwiP-TrAIsyyLY,2303
|
|
94
|
-
port_ocean/core/models.py,sha256=
|
|
96
|
+
port_ocean/core/models.py,sha256=8yYyF4DQ4jMmQJPsmh5-XoF9gfHUTuBit6zuT-bxZ7Y,1228
|
|
95
97
|
port_ocean/core/ocean_types.py,sha256=3_d8-n626f1kWLQ_Jxw194LEyrOVupz05qs_Y1pvB-A,990
|
|
96
|
-
port_ocean/core/utils.py,sha256=
|
|
98
|
+
port_ocean/core/utils.py,sha256=GjAm66FiGR48-hT3Mo66G8B4_jYTqSEGh80zs39qYQs,3599
|
|
97
99
|
port_ocean/exceptions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
98
100
|
port_ocean/exceptions/api.py,sha256=TLmTMqn4uHGaHgZK8PMIJ0TVJlPB4iP7xl9rx7GtCyY,426
|
|
99
101
|
port_ocean/exceptions/base.py,sha256=uY4DX7fIITDFfemCJDWpaZi3bD51lcANc5swpoNvMJA,46
|
|
@@ -110,7 +112,7 @@ port_ocean/log/handlers.py,sha256=k9G_Mb4ga2-Jke9irpdlYqj6EYiwv0gEsh4TgyqqOmI,28
|
|
|
110
112
|
port_ocean/log/logger_setup.py,sha256=BaXt-mh9CVXhneh37H46d04lqOdIBixG1pFyGfotuZs,2328
|
|
111
113
|
port_ocean/log/sensetive.py,sha256=wkyvkKMbyLTjZDSbvvLHL9bv4RvD0DPAyL3uWSttUOA,2916
|
|
112
114
|
port_ocean/middlewares.py,sha256=6GrhldYAazxSwK2TbS-J28XdZ-9wO3PgCcyIMhnnJvI,2480
|
|
113
|
-
port_ocean/ocean.py,sha256=
|
|
115
|
+
port_ocean/ocean.py,sha256=0L8sj5TaIJSeYydO7jyANNinB1k0WwN-Q-ChWNnhJOs,4729
|
|
114
116
|
port_ocean/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
115
117
|
port_ocean/run.py,sha256=rTxBlrQd4yyrtgErCFJCHCEHs7d1OXrRiJehUYmIbN0,2212
|
|
116
118
|
port_ocean/sonar-project.properties,sha256=X_wLzDOkEVmpGLRMb2fg9Rb0DxWwUFSvESId8qpvrPI,73
|
|
@@ -120,13 +122,14 @@ port_ocean/utils/__init__.py,sha256=KMGnCPXZJbNwtgxtyMycapkDz8tpSyw23MSYT3iVeHs,
|
|
|
120
122
|
port_ocean/utils/async_http.py,sha256=arnH458TExn2Dju_Sy6pHas_vF5RMWnOp-jBz5WAAcE,1226
|
|
121
123
|
port_ocean/utils/async_iterators.py,sha256=buFBiPdsqkNMCk91h6ZG8hJa181j7RjgHajbfgeB8A8,1608
|
|
122
124
|
port_ocean/utils/cache.py,sha256=3KItZDE2yVrbVDr-hoM8lNna8s2dlpxhP4ICdLjH4LQ,2231
|
|
123
|
-
port_ocean/utils/misc.py,sha256=
|
|
125
|
+
port_ocean/utils/misc.py,sha256=0q2cJ5psqxn_5u_56pT7vOVQ3shDM02iC1lzyWQ_zl0,2098
|
|
124
126
|
port_ocean/utils/queue_utils.py,sha256=KWWl8YVnG-glcfIHhM6nefY-2sou_C6DVP1VynQwzB4,2762
|
|
125
127
|
port_ocean/utils/repeat.py,sha256=0EFWM9d8lLXAhZmAyczY20LAnijw6UbIECf5lpGbOas,3231
|
|
126
128
|
port_ocean/utils/signal.py,sha256=K-6kKFQTltcmKDhtyZAcn0IMa3sUpOHGOAUdWKgx0_E,1369
|
|
129
|
+
port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
|
|
127
130
|
port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
|
|
128
|
-
port_ocean-0.
|
|
129
|
-
port_ocean-0.
|
|
130
|
-
port_ocean-0.
|
|
131
|
-
port_ocean-0.
|
|
132
|
-
port_ocean-0.
|
|
131
|
+
port_ocean-0.10.0.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
|
|
132
|
+
port_ocean-0.10.0.dist-info/METADATA,sha256=1T_SKzgGSYS3uiNJ6B29dszAsryEHYt0lm4r6TmyFng,6616
|
|
133
|
+
port_ocean-0.10.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
134
|
+
port_ocean-0.10.0.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
|
|
135
|
+
port_ocean-0.10.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|