polyaxon 2.4.0rc1__py3-none-any.whl → 2.6.0__py3-none-any.whl

polyaxon/_flow/run/kubeflow/xgboost_job.py

@@ -117,12 +117,17 @@ class V1XGBoostJob(BaseRun, DestinationImageMixin):
     """
 
     _IDENTIFIER = V1RunKind.XGBJOB
+    _CUSTOM_DUMP_FIELDS = {"master", "worker"}
 
     kind: Literal[_IDENTIFIER] = _IDENTIFIER
-    clean_pod_policy: Optional[V1CleanPodPolicy] = Field(alias="cleanPodPolicy")
-    scheduling_policy: Optional[V1SchedulingPolicy] = Field(alias="schedulingPolicy")
-    master: Optional[Union[V1KFReplica, RefField]]
-    worker: Optional[Union[V1KFReplica, RefField]]
+    clean_pod_policy: Optional[V1CleanPodPolicy] = Field(
+        alias="cleanPodPolicy", default=None
+    )
+    scheduling_policy: Optional[V1SchedulingPolicy] = Field(
+        alias="schedulingPolicy", default=None
+    )
+    master: Optional[Union[V1KFReplica, RefField]] = None
+    worker: Optional[Union[V1KFReplica, RefField]] = None
 
     def apply_image_destination(self, image: str):
         if self.chief:

polyaxon/_flow/run/ray/ray.py

@@ -135,14 +135,17 @@ class V1RayJob(BaseRun, DestinationImageMixin):
     """
 
     _IDENTIFIER = V1RunKind.RAYJOB
+    _CUSTOM_DUMP_FIELDS = {"head", "workers"}
 
     kind: Literal[_IDENTIFIER] = _IDENTIFIER
-    entrypoint: Optional[str]
-    runtime_env: Optional[Union[Dict[str, Any], RefField]] = Field(alias="runtimeEnv")
-    metadata: Optional[Union[Dict[str, str], RefField]]
-    ray_version: Optional[str] = Field(alias="rayVersion")
-    head: Optional[Union[V1RayReplica, RefField]]
-    workers: Optional[Dict[str, Union[V1RayReplica, RefField]]]
+    entrypoint: Optional[str] = None
+    runtime_env: Optional[Union[Dict[str, Any], RefField]] = Field(
+        alias="runtimeEnv", default=None
+    )
+    metadata: Optional[Union[Dict[str, str], RefField]] = Field(default=None)
+    ray_version: Optional[str] = Field(alias="rayVersion", default=None)
+    head: Optional[Union[V1RayReplica, RefField]] = None
+    workers: Optional[Dict[str, Union[V1RayReplica, RefField]]] = Field(default=None)
 
     def apply_image_destination(self, image: str):
         if self.head:

polyaxon/_flow/run/ray/replica.py

@@ -1,6 +1,12 @@
 from typing import Dict, List, Optional, Union
 
-from clipped.compact.pydantic import Field, StrictStr, validator
+from clipped.compact.pydantic import (
+    Field,
+    StrictStr,
+    field_validator,
+    validation_always,
+    validation_before,
+)
 from clipped.types.ref_or_obj import IntOrRef, RefField
 
 from polyaxon._flow.environment import V1Environment
@@ -210,31 +216,34 @@ class V1RayReplica(BaseSchemaModel):
 
     _IDENTIFIER = "replica"
     _SWAGGER_FIELDS = ["volumes", "sidecars", "container"]
-
-    replicas: Optional[IntOrRef]
-    min_replicas: Optional[IntOrRef] = Field(alias="minReplicas")
-    max_replicas: Optional[IntOrRef] = Field(alias="maxReplicas")
-    ray_start_params: Optional[Dict[str, str]] = Field(alias="rayStartParams")
-    environment: Optional[Union[V1Environment, RefField]]
-    connections: Optional[Union[List[StrictStr], RefField]]
-    volumes: Optional[Union[List[k8s_schemas.V1Volume], RefField]]
-    init: Optional[Union[List[V1Init], RefField]]
-    sidecars: Optional[Union[List[k8s_schemas.V1Container], RefField]]
-    container: Optional[Union[k8s_schemas.V1Container, RefField]]
-
-    @validator("volumes", always=True, pre=True)
+    _CUSTOM_DUMP_FIELDS = {"environment"}
+
+    replicas: Optional[IntOrRef] = None
+    min_replicas: Optional[IntOrRef] = Field(alias="minReplicas", default=None)
+    max_replicas: Optional[IntOrRef] = Field(alias="maxReplicas", default=None)
+    ray_start_params: Optional[Dict[str, str]] = Field(
+        alias="rayStartParams", default=None
+    )
+    environment: Optional[Union[V1Environment, RefField]] = None
+    connections: Optional[Union[List[StrictStr], RefField]] = None
+    volumes: Optional[Union[List[k8s_schemas.V1Volume], RefField]] = None
+    init: Optional[Union[List[V1Init], RefField]] = None
+    sidecars: Optional[Union[List[k8s_schemas.V1Container], RefField]] = None
+    container: Optional[Union[k8s_schemas.V1Container, RefField]] = None
+
+    @field_validator("volumes", **validation_always, **validation_before)
     def validate_volumes(cls, v):
         if not v:
             return v
         return [k8s_validation.validate_k8s_volume(vi) for vi in v]
 
-    @validator("sidecars", always=True, pre=True)
+    @field_validator("sidecars", **validation_always, **validation_before)
     def validate_helper_containers(cls, v):
         if not v:
             return v
         return [k8s_validation.validate_k8s_container(vi) for vi in v]
 
-    @validator("container", always=True, pre=True)
+    @field_validator("container", **validation_always, **validation_before)
     def validate_container(cls, v):
         return k8s_validation.validate_k8s_container(v)
 

polyaxon/_flow/run/resources.py

@@ -1,4 +1,4 @@
-from typing import Optional, Union
+from typing import ClassVar, Optional, Set, Union
 
 from clipped.compact.pydantic import StrictStr
 from clipped.types.numbers import StrictIntOrFloat
@@ -9,18 +9,19 @@ from polyaxon._schemas.base import BaseSchemaModel
 
 
 class V1RunResources(BaseSchemaModel):
-    cpu: Optional[Union[StrictIntOrFloat, StrictStr]]
-    memory: Optional[Union[StrictIntOrFloat, StrictStr]]
-    gpu: Optional[Union[StrictIntOrFloat, StrictStr]]
-    custom: Optional[Union[StrictIntOrFloat, StrictStr]]
-    cost: Optional[Union[StrictIntOrFloat, StrictStr]]
-
-    _MEMORY = "memory"
-    _CPU = "cpu"
-    _GPU = "gpu"
-    _CUSTOM_RESOURCE = "custom"
-    _COST = "cost"
-    _VALUES = {_MEMORY, _CPU, _GPU, _CUSTOM_RESOURCE, _COST}
+    cpu: Optional[Union[StrictIntOrFloat, StrictStr]] = None
+    memory: Optional[Union[StrictIntOrFloat, StrictStr]] = None
+    gpu: Optional[Union[StrictIntOrFloat, StrictStr]] = None
+    custom: Optional[Union[StrictIntOrFloat, StrictStr]] = None
+    cost: Optional[Union[StrictIntOrFloat, StrictStr]] = None
+
+    _MEMORY: ClassVar[str] = "memory"
+    _CPU: ClassVar[str] = "cpu"
+    _GPU: ClassVar[str] = "gpu"
+    _CUSTOM_RESOURCE: ClassVar[str] = "custom"
+    _COST: ClassVar[str] = "cost"
+
+    _VALUES: ClassVar[Set[str]] = {_MEMORY, _CPU, _GPU, _CUSTOM_RESOURCE, _COST}
 
     @classmethod
     def validate_memory(cls, value):

polyaxon/_flow/run/service.py

@@ -272,7 +272,7 @@ class V1Service(V1Job):
     _IDENTIFIER = V1RunKind.SERVICE
 
     kind: Literal[_IDENTIFIER] = _IDENTIFIER
-    ports: Optional[Union[List[StrictInt], RefField]]
-    rewrite_path: Optional[BoolOrRef] = Field(alias="rewritePath")
-    is_external: Optional[BoolOrRef] = Field(alias="isExternal")
-    replicas: Optional[IntOrRef]
+    ports: Optional[Union[List[StrictInt], RefField]] = None
+    rewrite_path: Optional[BoolOrRef] = Field(alias="rewritePath", default=None)
+    is_external: Optional[BoolOrRef] = Field(alias="isExternal", default=None)
+    replicas: Optional[IntOrRef] = None

polyaxon/_flow/schedules/cron.py

@@ -114,7 +114,7 @@ class V1CronSchedule(BaseSchemaModel):
 
     kind: Literal[_IDENTIFIER] = _IDENTIFIER
     cron: StrictStr
-    start_at: Optional[DatetimeOrRef] = Field(alias="startAt")
-    end_at: Optional[DatetimeOrRef] = Field(alias="endAt")
-    max_runs: Optional[IntOrRef] = Field(alias="maxRuns")
-    depends_on_past: Optional[BoolOrRef] = Field(alias="dependsOnPast")
+    start_at: Optional[DatetimeOrRef] = Field(alias="startAt", default=None)
+    end_at: Optional[DatetimeOrRef] = Field(alias="endAt", default=None)
+    max_runs: Optional[IntOrRef] = Field(alias="maxRuns", default=None)
+    depends_on_past: Optional[BoolOrRef] = Field(alias="dependsOnPast", default=None)

polyaxon/_flow/schedules/interval.py

@@ -117,8 +117,8 @@ class V1IntervalSchedule(BaseSchemaModel):
     _USE_DISCRIMINATOR = True
 
     kind: Literal[_IDENTIFIER] = _IDENTIFIER
-    start_at: Optional[DatetimeOrRef] = Field(alias="startAt")
-    end_at: Optional[DatetimeOrRef] = Field(alias="endAt")
-    max_runs: Optional[IntOrRef] = Field(alias="maxRuns")
+    start_at: Optional[DatetimeOrRef] = Field(alias="startAt", default=None)
+    end_at: Optional[DatetimeOrRef] = Field(alias="endAt", default=None)
+    max_runs: Optional[IntOrRef] = Field(alias="maxRuns", default=None)
     frequency: TimeDeltaOrRef
-    depends_on_past: Optional[BoolOrRef] = Field(alias="dependsOnPast")
+    depends_on_past: Optional[BoolOrRef] = Field(alias="dependsOnPast", default=None)

polyaxon/_flow/templates/__init__.py

@@ -80,9 +80,9 @@ class V1Template(BaseSchemaModel):
 
     _IDENTIFIER = "template"
 
-    enabled: Optional[BoolOrRef]
-    description: Optional[StrictStr]
-    fields: Optional[Union[List[StrictStr], RefField]]
+    enabled: Optional[BoolOrRef] = None
+    description: Optional[StrictStr] = None
+    fields: Optional[Union[List[StrictStr], RefField]] = None
 
 
 class TemplateMixinConfig:

polyaxon/_flow/termination/__init__.py

@@ -99,6 +99,6 @@ class V1Termination(BaseSchemaModel):
 
     _IDENTIFIER = "termination"
 
-    max_retries: Optional[IntOrRef] = Field(alias="maxRetries")
-    ttl: Optional[IntOrRef]
-    timeout: Optional[IntOrRef]
+    max_retries: Optional[IntOrRef] = Field(alias="maxRetries", default=None)
+    ttl: Optional[IntOrRef] = None
+    timeout: Optional[IntOrRef] = None

polyaxon/_fs/async_manager.py

@@ -314,7 +314,7 @@ async def list_files(
 
 
 async def delete_file_or_dir(
-    fs: FSSystem, store_path: str, subpath: Union[str], is_file: bool
+    fs: FSSystem, store_path: str, subpath: Optional[str], is_file: bool
 ) -> bool:
     try:
         await ensure_async_execution(

polyaxon/_fs/watcher.py

@@ -3,36 +3,43 @@ import os
 from datetime import datetime
 from typing import Dict, List, Optional, Set, Tuple
 
+from clipped.compact.pydantic import PYDANTIC_VERSION
 from clipped.utils.dates import path_last_modified
 from clipped.utils.paths import get_files_and_dirs_in_path
 
 from polyaxon._contexts import paths as ctx_paths
-from polyaxon._schemas.base import BaseSchemaModel
+from polyaxon._schemas.base import BaseSchemaModel, RootModel
 
 
-class PathData(BaseSchemaModel):
-    __root__: Tuple[str, datetime, str]
+class PathData(RootModel):
+    if PYDANTIC_VERSION.startswith("2."):
+        root: Tuple[str, datetime, str]
+    else:
+        __root__: Tuple[str, datetime, str]
 
-    class Config(BaseSchemaModel.Config):
+    class Config:
         validate_assignment = False
 
     @property
     def base(self) -> str:
-        return self.__root__[0]
+        return self.get_root()[0]
 
     @property
     def ts(self) -> datetime:
-        if isinstance(self.__root__[1], str):
-            self.__root__ = (
-                self.__root__[0],
-                datetime.fromisoformat(self.__root__[1]),
-                self.__root__[1],
+        root = self.get_root()
+        if isinstance(root[1], str):
+            self.set_root(
+                (
+                    root[0],
+                    datetime.fromisoformat(root[1]),
+                    root[1],
+                )
             )
-        return self.__root__[1]
+        return root[1]
 
     @property
     def op(self) -> str:
-        return self.__root__[2]
+        return self.get_root()[2]
 
 
 class FSWatcher(BaseSchemaModel):
@@ -42,8 +49,8 @@ class FSWatcher(BaseSchemaModel):
     _RM = "rm"
     _NOOP = ""
 
-    dir_mapping: Optional[Dict[str, PathData]]
-    file_mapping: Optional[Dict[str, PathData]]
+    dir_mapping: Optional[Dict[str, PathData]] = None
+    file_mapping: Optional[Dict[str, PathData]] = None
 
     @property
     def dirs_mp(self) -> Dict[str, PathData]:
@@ -71,17 +78,11 @@ class FSWatcher(BaseSchemaModel):
         data = mapping.get(rel_path)
         if data:
             if current_ts > data.ts:
-                mapping[rel_path] = PathData.construct(
-                    __root__=(base_path, current_ts, self._PUT)
-                )
+                mapping[rel_path] = PathData.make((base_path, current_ts, self._PUT))
             else:
-                mapping[rel_path] = PathData.construct(
-                    __root__=(base_path, data.ts, self._NOOP)
-                )
+                mapping[rel_path] = PathData.make((base_path, data.ts, self._NOOP))
         else:
-            mapping[rel_path] = PathData.construct(
-                __root__=(base_path, current_ts, self._PUT)
-            )
+            mapping[rel_path] = PathData.make((base_path, current_ts, self._PUT))
         return mapping
 
     def sync_file(self, path: str, base_path: str):
@@ -92,12 +93,10 @@ class FSWatcher(BaseSchemaModel):
 
     def init(self):
         self.dir_mapping = {
-            p: PathData.construct(__root__=(d.base, d.ts, self._RM))
-            for p, d in self.dirs_mp.items()
+            p: PathData.make((d.base, d.ts, self._RM)) for p, d in self.dirs_mp.items()
         }
         self.file_mapping = {
-            p: PathData.construct(__root__=(d.base, d.ts, self._RM))
-            for p, d in self.files_mp.items()
+            p: PathData.make((d.base, d.ts, self._RM)) for p, d in self.files_mp.items()
         }
 
     def sync(self, path: str, exclude: Optional[List[str]] = None):

polyaxon/_k8s/k8s_validation.py

@@ -14,7 +14,7 @@ def _validate_schema(value: Optional[Union[Swagger, Dict]], cls: Type[Swagger]):
         return cls(**{to_snake_case(k): value[k] for k in value})
     if isinstance(value, cls):
         return value
-    raise TypeError(
+    raise ValueError(
         "This field expects a dict or an instance of {}.".format(cls.__name__)
     )
 

polyaxon/_k8s/logging/async_monitor.py

@@ -1,4 +1,5 @@
 import datetime
+import logging
 
 from typing import List, Optional, Tuple
 
@@ -10,6 +11,8 @@ from polyaxon._flow import V1RunKind
 from polyaxon._k8s.manager.async_manager import AsyncK8sManager
 from traceml.logging import V1Log, V1Logs
 
+_logger = logging.getLogger("haupt.k8s.logs")
+
 
 async def handle_container_logs(
     k8s_manager: AsyncK8sManager, pod: V1Pod, container_name: str, **params
@@ -23,8 +26,20 @@ async def handle_container_logs(
             timestamps=True,
             **params,
         )
-    except ApiException:
-        pass
+    except ApiException as e:
+        _logger.warning(
+            "Error collecting logs for %s - container %s: %s",
+            pod.metadata.name,
+            container_name,
+            e,
+        )
+    except Exception as e:
+        _logger.warning(
+            "Unexpected error collecting logs for %s - container %s: %s",
+            pod.metadata.name,
+            container_name,
+            e,
+        )
     if not resp:
         return []
 
@@ -108,7 +123,7 @@ async def collect_agent_service_logs(
 async def query_k8s_pod_logs(
     k8s_manager: AsyncK8sManager,
     pod: V1Pod,
-    last_time: Optional[datetime.datetime],
+    last_time: Optional[datetime.datetime] = None,
     stream: bool = False,
 ) -> Tuple[List[V1Log], Optional[datetime.datetime]]:
     new_time = now()

polyaxon/_local_process/converter/base/containers.py

@@ -36,11 +36,8 @@ class ContainerMixin(BaseConverter):
         env: List[process_types.V1EnvVar],
     ) -> List[process_types.V1EnvVar]:
         def sanitize_env_dict(d: Dict):
-            return process_types.V1EnvVar(
-                __root__={
-                    d_k: sanitize_value(d_v, handle_dict=False)
-                    for d_k, d_v in d.items()
-                }
+            return process_types.V1EnvVar.make(
+                {d_k: sanitize_value(d_v, handle_dict=False) for d_k, d_v in d.items()}
             )
 
         results = []
@@ -50,8 +47,8 @@ class ContainerMixin(BaseConverter):
                 results.append(e)
             elif isinstance(e, tuple):
                 if e[1] is not None:
-                    e = process_types.V1EnvVar(
-                        __root__=(e[0], sanitize_value(e[1], handle_dict=False))
+                    e = process_types.V1EnvVar.make(
+                        (e[0], sanitize_value(e[1], handle_dict=False))
                     )
                 results.append(e)
             elif isinstance(e, process_types.V1EnvVar):

polyaxon/_local_process/converter/base/env_vars.py

@@ -42,7 +42,7 @@ class EnvMixin(BaseConverter):
             except (ValueError, TypeError) as e:
                 raise PolyaxonConverterError(e)
 
-        return process_types.V1EnvVar(__root__=(name, value))
+        return process_types.V1EnvVar.make((name, value))
 
     @staticmethod
     def _get_from_json_resource(
@@ -59,7 +59,7 @@ class EnvMixin(BaseConverter):
         except Exception as e:
             raise PolyaxonConverterError from e
 
-        return [process_types.V1EnvVar(__root__=k) for k in (secret_value.items())]
+        return [process_types.V1EnvVar.make(k) for k in (secret_value.items())]
 
     @classmethod
     def _get_env_from_json_resources(
@@ -82,10 +82,10 @@ class EnvMixin(BaseConverter):
         try:
             secret_value = orjson_loads(secret)
         except orjson.JSONDecodeError:
-            return process_types.V1EnvVar(__root__=(key, secret))
+            return process_types.V1EnvVar.make((key, secret))
 
         value = secret_value.get(key)
-        return process_types.V1EnvVar(__root__=(key, value))
+        return process_types.V1EnvVar.make((key, value))
 
     @classmethod
     def _get_items_from_json_resource(
@@ -104,7 +104,7 @@ class EnvMixin(BaseConverter):
         for item in resource.items:
             value = secret_value.get(item)
             if value:
-                items_from.append(process_types.V1EnvVar(__root__=(item, value)))
+                items_from.append(process_types.V1EnvVar.make((item, value)))
         return items_from
 
     @classmethod

polyaxon/_local_process/process_types.py

@@ -1,27 +1,30 @@
 from typing import Dict, List, Optional, Tuple, Union
 
-from clipped.compact.pydantic import Field
+from clipped.compact.pydantic import PYDANTIC_VERSION, Field
 
-from polyaxon._schemas.base import BaseSchemaModel
+from polyaxon._schemas.base import BaseSchemaModel, RootModel
 
 
-class V1EnvVar(BaseSchemaModel):
-    __root__: Union[Tuple[str, str], Dict[str, str]]
+class V1EnvVar(RootModel):
+    if PYDANTIC_VERSION.startswith("2."):
+        root: Union[Tuple[str, str], Dict[str, str]]
+    else:
+        __root__: Union[Tuple[str, str], Dict[str, str]]
 
     def to_cmd(self):
-        if isinstance(self.__root__, tuple):
-            value = self.__root__
+        if isinstance(self._root, tuple):
+            value = self._root
         else:
-            value = self.__root__.items()
+            value = self._root.items()
         return [f"{value[0]}={value[1]}"]
 
 
 class V1Container(BaseSchemaModel):
-    name: Optional[str]
-    command: Optional[List[str]]
-    args: Optional[List[str]]
-    env: Optional[List[V1EnvVar]]
-    working_dir: Optional[str] = Field(alias="workingDir")
+    name: Optional[str] = None
+    command: Optional[List[str]] = None
+    args: Optional[List[str]] = None
+    env: Optional[List[V1EnvVar]] = None
+    working_dir: Optional[str] = Field(alias="workingDir", default=None)
 
     def get_cmd_args(self):
         cmd_args = ["run", "--rm"]

polyaxon/_polyaxonfile/specs/compiled_operation.py

@@ -348,7 +348,7 @@ class CompiledOperationSpecification(BaseSpecification):
             "conditions",
             "skip_on_upstream_skip",
         }
-        patch_keys = patch_keys.intersection(preset.__fields_set__)
+        patch_keys = patch_keys.intersection(preset.model_fields_set)
         patch_data = {k: getattr(preset, k) for k in patch_keys}
         patch_compiled = V1CompiledOperation.construct(**patch_data)
         return config.patch(patch_compiled, strategy=preset.patch_strategy)

polyaxon/_polyaxonfile/specs/libs/parser.py

@@ -93,7 +93,7 @@ class PolyaxonfileParser:
         # Check workflow
         for section in Sections.PARSING_SECTIONS:
             config_section = cls._get_section(config, section)
-            if config_section:
+            if config_section is not None:
                 parsed_data[section] = cls.parse_expression(
                     config_section, parsed_params
                 )

polyaxon/_polyaxonfile/specs/libs/validator.py

@@ -11,7 +11,7 @@ def validate(spec, data):
 
     def validate_keys(section, config, section_data):
         extra_args = [
-            key for key in section_data.keys() if key not in config.__fields__.keys()
+            key for key in section_data.keys() if key not in config.model_fields.keys()
         ]
         if extra_args:
             raise PolyaxonfileError(

polyaxon/_polyaxonfile/specs/operation.py

@@ -107,7 +107,7 @@ class OperationSpecification(BaseSpecification):
             "conditions",
             "skip_on_upstream_skip",
         }
-        patch_keys = patch_keys.intersection(config.__fields_set__)
+        patch_keys = patch_keys.intersection(config.model_fields_set)
         patch_data = {k: getattr(config, k) for k in patch_keys}
         patch_compiled = V1CompiledOperation.construct(contexts=contexts, **patch_data)
 

polyaxon/_polyaxonfile/specs/sections.py

@@ -99,6 +99,14 @@ class Sections:
         CONDITIONS,
         SKIP_ON_UPSTREAM_SKIP,
         PATCH_STRATEGY,
+        "is_approved",
+        "patch_strategy",
+        "is_preset",
+        "hub_ref",
+        "dag_ref",
+        "path_ref",
+        "url_ref",
+        "skip_on_upstream_skip",
     )
 
     REQUIRED_SECTIONS = (VERSION, KIND)

polyaxon/_runner/agent/async_agent.py

@@ -24,10 +24,10 @@ class BaseAsyncAgent(BaseAgent):
     IS_ASYNC = True
 
     async def _enter(self):
-        if not self.client._is_managed:
-            return self
         logger.warning("Agent is starting.")
         await self.executor.refresh()
+        if not self.client._is_managed:
+            return self
         try:
             agent = await self.client.get_info()
             self._check_status(agent)
@@ -81,13 +81,16 @@ class BaseAsyncAgent(BaseAgent):
 
     async def reconcile(self):
         if (
-            now() - self._last_reconciled_at
+            now() - self._last_data_collected_at
         ).total_seconds() > self.SLEEP_AGENT_DATA_COLLECT_TIME:
+            await self.collect_agent_data()
+        if (
+            now() - self._last_reconciled_at
+        ).total_seconds() < self.SLEEP_AGENT_DATA_RECONCILE_TIME:
             return
 
-        # Collect data
-        await self.collect_agent_data()
-
+        logger.info("Checking cluster state.")
+        self._last_reconciled_at = now()
         # Update reconcile
         namespaces = [settings.AGENT_CONFIG.namespace]
         namespaces += settings.AGENT_CONFIG.additional_namespaces or []

polyaxon/_runner/agent/base_agent.py

@@ -4,7 +4,7 @@ import traceback
 from concurrent.futures import ThreadPoolExecutor
 from typing import Any, Dict, Optional, Tuple, Type
 
-from clipped.utils.tz import now
+from clipped.utils.tz import get_datetime_from_now, now
 
 from polyaxon import settings
 from polyaxon._auxiliaries import V1PolyaxonInitContainer, V1PolyaxonSidecarContainer
@@ -24,7 +24,8 @@ class BaseAgent:
     HEALTH_FILE = "/tmp/.healthz"
     SLEEP_STOP_TIME = 60 * 5
     SLEEP_ARCHIVED_TIME = 60 * 60
-    SLEEP_AGENT_DATA_COLLECT_TIME = 60 * 30
+    SLEEP_AGENT_DATA_COLLECT_TIME = 60 * 15
+    SLEEP_AGENT_DATA_RECONCILE_TIME = 60 * 5
     IS_ASYNC = False
 
     def __init__(
@@ -38,11 +39,13 @@ class BaseAgent:
             self.max_interval = max(max_interval, 3)
         if not agent_uuid and not owner:
             owner = DEFAULT
+        last_hour = get_datetime_from_now(days=0, hours=1)
         self.executor = None
         self._default_auth = bool(agent_uuid)
-        self._executor_refreshed_at = now()
+        self._executor_refreshed_at = last_hour
         self._graceful_shutdown = False
-        self._last_reconciled_at = now()
+        self._last_data_collected_at = last_hour
+        self._last_reconciled_at = last_hour
         self.client = AgentClient(
             owner=owner, agent_uuid=agent_uuid, is_async=self.IS_ASYNC
         )
@@ -60,7 +63,7 @@ class BaseAgent:
 
     def collect_agent_data(self):
         logger.info("Collecting agent data.")
-        self._last_reconciled_at = now()
+        self._last_data_collected_at = now()
         try:
             return self.client.collect_agent_data(
                 namespace=settings.CLIENT_CONFIG.namespace