watchmen-collector-kernel 16.4.8__tar.gz → 16.4.10__tar.gz

watchmen_collector_kernel-16.4.10/PKG-INFO

@@ -0,0 +1,29 @@
+Metadata-Version: 2.1
+Name: watchmen-collector-kernel
+Version: 16.4.10
+Summary: 
+License: MIT
+Author: botlikes
+Author-email: 75356972+botlikes456@users.noreply.github.com
+Requires-Python: >=3.9,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Provides-Extra: mongodb
+Provides-Extra: mssql
+Provides-Extra: mysql
+Provides-Extra: oracle
+Provides-Extra: oss
+Provides-Extra: postgresql
+Provides-Extra: s3
+Requires-Dist: numpy (>=1.23.3,<2.0.0)
+Requires-Dist: watchmen-data-kernel (==16.4.10)
+Requires-Dist: watchmen-storage-mongodb (==16.4.10) ; extra == "mongodb"
+Requires-Dist: watchmen-storage-mssql (==16.4.10) ; extra == "mssql"
+Requires-Dist: watchmen-storage-mysql (==16.4.10) ; extra == "mysql"
+Requires-Dist: watchmen-storage-oracle (==16.4.10) ; extra == "oracle"
+Requires-Dist: watchmen-storage-oss (==16.4.10) ; extra == "oss"
+Requires-Dist: watchmen-storage-postgresql (==16.4.10) ; extra == "postgresql"
+Requires-Dist: watchmen-storage-s3 (==16.4.10) ; extra == "s3"

{watchmen_collector_kernel-16.4.8 → watchmen_collector_kernel-16.4.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "watchmen-collector-kernel"
-version = "16.4.8"
+version = "16.4.10"
 description = ""
 authors = ["botlikes <75356972+botlikes456@users.noreply.github.com>"]
 license = "MIT"
@@ -10,14 +10,15 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.9"
-watchmen-data-kernel = "16.4.8"
-watchmen-storage-mysql = { version = "16.4.8", optional = true }
-watchmen-storage-oracle = { version = "16.4.8", optional = true }
-watchmen-storage-mongodb = { version = "16.4.8", optional = true }
-watchmen-storage-mssql = { version = "16.4.8", optional = true }
-watchmen-storage-postgresql = { version = "16.4.8", optional = true }
-watchmen-storage-oss = { version = "16.4.8", optional = true }
-watchmen-storage-s3 = { version = "16.4.8", optional = true }
+numpy = "^1.23.3"
+watchmen-data-kernel = "16.4.10"
+watchmen-storage-mysql = { version = "16.4.10", optional = true }
+watchmen-storage-oracle = { version = "16.4.10", optional = true }
+watchmen-storage-mongodb = { version = "16.4.10", optional = true }
+watchmen-storage-mssql = { version = "16.4.10", optional = true }
+watchmen-storage-postgresql = { version = "16.4.10", optional = true }
+watchmen-storage-oss = { version = "16.4.10", optional = true }
+watchmen-storage-s3 = { version = "16.4.10", optional = true }
 
 [tool.poetry.dev-dependencies]
 

{watchmen_collector_kernel-16.4.8 → watchmen_collector_kernel-16.4.10}/src/watchmen_collector_kernel/common/__init__.py

@@ -1,2 +1,3 @@
 from .constants import CHANGE_RECORD_ID, TENANT_ID, IS_MERGED, LEFT_BRACE, RIGHT_BRACE, COMMA, \
 	IS_POSTED, CHANGE_JSON_ID, WAVE, IS_FINISHED, IS_EXTRACTED, MODEL_TRIGGER_ID
+from .settings import ask_lock_clean_interval, ask_lock_clean_timeout, ask_partial_size

watchmen_collector_kernel-16.4.10/src/watchmen_collector_kernel/common/settings.py

@@ -0,0 +1,32 @@
+from pydantic import BaseSettings
+from logging import getLogger
+
+logger = getLogger(__name__)
+
+
+class CollectorSettings(BaseSettings):
+	LOCK_CLEAN_INTERVAL: int = 60
+	LOCK_CLEAN_TIMEOUT: int = 3600
+	PARTIAL_SIZE: int = 10000
+
+	class Config:
+		# secrets_dir = '/var/run'
+		env_file = '.env'
+		env_file_encoding = 'utf-8'
+		case_sensitive = True
+
+
+collector_settings = CollectorSettings()
+logger.info(f'Collector settings[{collector_settings.dict()}].')
+
+
+def ask_lock_clean_interval() -> int:
+	return collector_settings.LOCK_CLEAN_INTERVAL
+
+
+def ask_lock_clean_timeout() -> int:
+	return collector_settings.LOCK_CLEAN_TIMEOUT
+
+
+def ask_partial_size() -> int:
+	return collector_settings.PARTIAL_SIZE

{watchmen_collector_kernel-16.4.8 → watchmen_collector_kernel-16.4.10}/src/watchmen_collector_kernel/model/__init__.py

@@ -1,5 +1,5 @@
 from .competitive_lock import CompetitiveLock
-from .scheduled_task import ScheduledTask, TaskStatus
+from .scheduled_task import ScheduledTask
 
 from .collector_model_config import CollectorModelConfig
 from .collector_table_config import CollectorTableConfig
@@ -9,7 +9,9 @@ from .trigger_model import TriggerModel
 from .trigger_table import TriggerTable
 
 from .change_data_record import ChangeDataRecord
+from .change_data_record_history import ChangeDataRecordHistory
 from .change_data_json import ChangeDataJson
+from .change_data_json_history import ChangeDataJsonHistory
 
 from .condition import construct_conditions, Condition, ConditionJoint, ConditionExpression, \
 	ConditionJointConjunction

watchmen_collector_kernel-16.4.10/src/watchmen_collector_kernel/model/change_data_json_history.py

@@ -0,0 +1,6 @@
+from .change_data_json import ChangeDataJson
+
+
+class ChangeDataJsonHistory(ChangeDataJson):
+	pass
+

watchmen_collector_kernel-16.4.10/src/watchmen_collector_kernel/model/change_data_record_history.py

@@ -0,0 +1,5 @@
+from .change_data_record import ChangeDataRecord
+
+
+class ChangeDataRecordHistory(ChangeDataRecord):
+	pass

watchmen_collector_kernel-16.4.10/src/watchmen_collector_kernel/model/scheduled_task.py

@@ -0,0 +1,53 @@
+from typing import Dict, List, Optional, Union
+
+from watchmen_utilities import ArrayHelper
+
+from watchmen_model.common import TenantBasedTuple, ScheduledTaskId, Storable
+from pydantic import BaseModel
+from enum import Enum
+
+
+class Result(str, Enum):
+	DEPENDENCY_FAILED = "DEPENDENCY_FAILED"
+	PROCESS_TASK_SUCCESS = "PROCESS_TASK_SUCCESS"
+	PROCESS_TASK_FAILED = "PROCESS_TASK_FAILED"
+
+
+class Dependence(Storable, BaseModel):
+	modelName: str
+	objectId: str
+
+
+def construct_dependence(dependence: Union[Dependence, Dict]) -> Optional[Dependence]:
+	if dependence is None:
+		return None
+	elif isinstance(dependence, Dependence):
+		return dependence
+	else:
+		return Dependence(**dependence)
+
+
+def construct_depend_on(depend_on: Optional[List[Union[Dependence, Dict]]]) -> Optional[List[Dependence]]:
+	if depend_on is None:
+		return None
+	else:
+		return ArrayHelper(depend_on).map(lambda x: construct_dependence(x)).to_list()
+
+
+class ScheduledTask(TenantBasedTuple, BaseModel):
+	taskId: ScheduledTaskId
+	resourceId: str  # global unique, monotonous increase
+	topicCode: str
+	content: Dict
+	modelName: str
+	objectId: str
+	dependOn: List[Dependence]
+	parentTaskId: List[int]
+	isFinished: bool
+	result: Dict
+
+	def __setattr__(self, name, value):
+		if name == 'dependOn':
+			super().__setattr__(name, construct_depend_on(value))
+		else:
+			super().__setattr__(name, value)

watchmen_collector_kernel-16.4.10/src/watchmen_collector_kernel/model/scheduled_task_history.py

@@ -0,0 +1,5 @@
+from .scheduled_task import ScheduledTask
+
+
+class ScheduledTaskHistory(ScheduledTask):
+	pass

{watchmen_collector_kernel-16.4.8 → watchmen_collector_kernel-16.4.10}/src/watchmen_collector_kernel/service/lock_clean.py

@@ -4,6 +4,7 @@ from threading import Thread
 
 from time import sleep
 
+from watchmen_collector_kernel.common import ask_lock_clean_interval, ask_lock_clean_timeout
 from watchmen_collector_kernel.storage import get_competitive_lock_service
 from watchmen_meta.common import ask_meta_storage
 
@@ -12,7 +13,8 @@ class LockClean:
 
 	def __init__(self):
 		self.lock_service = get_competitive_lock_service(ask_meta_storage())
-		self.cleanInterval = 60
+		self.cleanInterval = ask_lock_clean_interval()
+		self.cleanTimeout = ask_lock_clean_timeout()
 
 	def run(self):
 		try:
@@ -25,7 +27,7 @@ class LockClean:
 			self.restart()
 
 	def clean_lock(self):
-		query_time = datetime.now() - timedelta(minutes=120)
+		query_time = datetime.now() - timedelta(seconds=self.cleanTimeout)
 		locks = self.lock_service.find_overtime_lock(query_time)
 		for lock in locks:
 			self.lock_service.delete_by_id(lock.lockId)

{watchmen_collector_kernel-16.4.8 → watchmen_collector_kernel-16.4.10}/src/watchmen_collector_kernel/service/task_service.py

@@ -3,9 +3,9 @@ from traceback import format_exc
 from typing import Callable, Dict, Optional, List
 
 from watchmen_auth import PrincipalService
-from watchmen_collector_kernel.model import ScheduledTask, TaskStatus
+from watchmen_collector_kernel.model import ScheduledTask
 from watchmen_collector_kernel.model.scheduled_task import Dependence
-from watchmen_collector_kernel.storage import get_scheduled_task_service
+from watchmen_collector_kernel.storage import get_scheduled_task_service, get_scheduled_task_history_service
 from watchmen_model.common import ScheduledTaskId
 from watchmen_storage import TransactionalStorageSPI, SnowflakeGenerator
 from watchmen_utilities import ArrayHelper
@@ -26,33 +26,45 @@ class TaskService:
 		self.scheduled_task_service = get_scheduled_task_service(self.storage,
 		                                                         self.snowflake_generator,
 		                                                         self.principal_service)
+		self.scheduled_task_history_service = get_scheduled_task_history_service(self.storage,
+		                                                                         self.snowflake_generator,
+		                                                                         self.principal_service)
 
 	# noinspection PyMethodMayBeStatic
 	def consume_task(self, task: ScheduledTask, executed: Callable[[str, Dict, str], None]) -> ScheduledTask:
 		try:
 			executed(task.topicCode, task.content, task.tenantId)
-			task.status = int(TaskStatus.SUCCESS)
-			return self.scheduled_task_service.update_task(task)
+			return self.update_task_result(task)
 		except Exception as e:
 			logger.error(e, exc_info=True, stack_info=True)
-			task.status = int(TaskStatus.FAILED)
+			task.isFinished = True
 			task.result = format_exc()
 			return self.scheduled_task_service.update_task(task)
 
+	def update_task_result(self, task: ScheduledTask) -> ScheduledTask:
+		self.scheduled_task_service.begin_transaction()
+		try:
+			task.isFinished = True
+			self.scheduled_task_history_service.create(task)
+			self.scheduled_task_service.delete(task.taskId)
+			self.scheduled_task_service.commit_transaction()
+			return task
+		except Exception as e:
+			raise e
+
 	def is_dependencies_finished(self, task: ScheduledTask) -> bool:
-		return ArrayHelper(task.parentTaskId).every(self.is_parent_task_finished) \
-		       and self.is_dependence_finished(task.dependOn, task.tenantId)
+		return ArrayHelper(task.parentTaskId).every(self.is_parent_task_finished) and self.is_dependence_finished(task.dependOn, task.tenantId)
 
 	def is_parent_task_finished(self, task_id: ScheduledTaskId) -> bool:
 		existed_task = self.scheduled_task_service.find_task_by_id(task_id)
-		return self.is_finished(existed_task)
+		if existed_task:
+			return self.is_finished(existed_task)
+		else:
+			return True
 
 	# noinspection PyMethodMayBeStatic
 	def is_finished(self, task: ScheduledTask) -> bool:
-		if task.status == TaskStatus.SUCCESS or task.status == TaskStatus.FAILED:
-			return True
-		else:
-			return False
+		return task.isFinished
 
 	def is_dependence_finished(self, depend_on: Optional[List[Dependence]], tenant_id: str) -> bool:
 		if ArrayHelper(depend_on).every(lambda dependence: self.is_dependent_task_finished(dependence, tenant_id)):

{watchmen_collector_kernel-16.4.8 → watchmen_collector_kernel-16.4.10}/src/watchmen_collector_kernel/storage/__init__.py

@@ -1,5 +1,6 @@
 from .competitive_lock_service import get_competitive_lock_service, CompetitiveLockService
 from .scheduled_task_service import get_scheduled_task_service, ScheduledTaskService
+from .scheduled_task_history_service import get_scheduled_task_history_service, ScheduledTaskHistoryService
 
 
 from .collector_model_config_service import get_collector_model_config_service, CollectorModelConfigService
@@ -9,8 +10,11 @@ from .collector_table_config_service import get_collector_table_config_service,
 from .trigger_event_service import get_trigger_event_service, TriggerEventService
 from .trigger_model_service import get_trigger_model_service, TriggerModelService
 from .trigger_table_service import get_trigger_table_service, TriggerTableService
+
 from .change_data_record_service import get_change_data_record_service, ChangeDataRecordService
+from .change_data_record_history_service import get_change_data_record_history_service, ChangeDataRecordHistoryService
 from .change_data_json_service import get_change_data_json_service, ChangeDataJsonService
+from .change_data_json_history_service import get_change_data_json_history_service, ChangeDataJsonHistoryService
 
 
 

watchmen_collector_kernel-16.4.10/src/watchmen_collector_kernel/storage/change_data_json_history_service.py

@@ -0,0 +1,48 @@
+from watchmen_collector_kernel.storage.change_data_json_service import ChangeDataJsonShaper, ChangeDataJsonService
+
+from watchmen_auth import PrincipalService
+
+from watchmen_collector_kernel.model import ChangeDataJson
+from watchmen_meta.common.storage_service import StorableId
+from watchmen_model.common import Storable, ChangeJsonId
+from watchmen_storage import EntityName, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator
+
+
+class ChangeDataJsonHistoryShaper(ChangeDataJsonShaper):
+	pass
+
+
+CHANGE_DATA_JSON_HISTORY_TABLE = 'change_data_json_history'
+CHANGE_DATA_JSON_HISTORY_ENTITY_SHAPER = ChangeDataJsonHistoryShaper()
+
+
+class ChangeDataJsonHistoryService(ChangeDataJsonService):
+
+	def should_record_operation(self) -> bool:
+		return False
+
+	def get_entity_name(self) -> EntityName:
+		return CHANGE_DATA_JSON_HISTORY_TABLE
+
+	def get_entity_shaper(self) -> EntityShaper:
+		return CHANGE_DATA_JSON_HISTORY_ENTITY_SHAPER
+
+	# noinspection SpellCheckingInspection
+	def get_storable_id_column_name(self) -> EntityName:
+		return 'change_json_id'
+
+	def get_storable_id(self, storable: ChangeDataJson) -> StorableId:
+		# noinspection PyTypeChecker
+		return storable.changeJsonId
+
+	# noinspection SpellCheckingInspection
+	def set_storable_id(self, storable: ChangeDataJson, storable_id: ChangeJsonId) -> Storable:
+		storable.changeRecordId = storable_id
+		return storable
+
+
+def get_change_data_json_history_service(storage: TransactionalStorageSPI,
+                                         snowflake_generator: SnowflakeGenerator,
+                                         principal_service: PrincipalService
+                                         ) -> ChangeDataJsonHistoryService:
+	return ChangeDataJsonHistoryService(storage, snowflake_generator, principal_service)

{watchmen_collector_kernel-16.4.8 → watchmen_collector_kernel-16.4.10}/src/watchmen_collector_kernel/storage/change_data_json_service.py

@@ -1,11 +1,11 @@
 from typing import List, Dict, Any, Optional
 
 from watchmen_auth import PrincipalService
-from watchmen_collector_kernel.common import IS_POSTED, CHANGE_JSON_ID, TENANT_ID, MODEL_TRIGGER_ID
+from watchmen_collector_kernel.common import IS_POSTED, CHANGE_JSON_ID, TENANT_ID, MODEL_TRIGGER_ID, ask_partial_size
 from watchmen_collector_kernel.model import ChangeDataJson
 from watchmen_meta.common import TupleService, TupleShaper
 from watchmen_meta.common.storage_service import StorableId
-from watchmen_model.common import Storable, ChangeJsonId
+from watchmen_model.common import Storable, ChangeJsonId, Pageable
 from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
 	ColumnNameLiteral, EntityCriteriaExpression, EntityStraightValuesFinder, EntityStraightColumn, EntitySortColumn, \
 	EntitySortMethod
@@ -24,6 +24,7 @@ class ChangeDataJsonShaper(EntityShaper):
 			                                          'content': entity.content,
 			                                          'depend_on': entity.dependOn,
 			                                          'is_posted': entity.isPosted,
+			                                          'result': entity.result,
 			                                          'task_id': entity.taskId,
 			                                          'table_trigger_id': entity.tableTriggerId,
 			                                          'model_trigger_id': entity.modelTriggerId,
@@ -43,6 +44,7 @@ class ChangeDataJsonShaper(EntityShaper):
 			                                            content=row.get('content'),
 			                                            dependOn=row.get('depend_on'),
 			                                            isPosted=row.get('is_posted'),
+			                                            result=row.get('result'),
 			                                            taskId=row.get('task_id'),
 			                                            tableTriggerId=row.get('table_trigger_id'),
 			                                            modelTriggerId=row.get('model_trigger_id'),
@@ -104,6 +106,35 @@ class ChangeDataJsonService(TupleService):
 		finally:
 			self.close_transaction()
 
+	def find_partial_json(self, model_trigger_id: int) -> List[ChangeDataJson]:
+		self.begin_transaction()
+		try:
+			return self.storage.page(self.get_entity_pager(
+				criteria=[
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName=IS_POSTED), right=False),
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName=MODEL_TRIGGER_ID),
+					                         right=model_trigger_id)
+				],
+				pageable=Pageable(pageNumber=1, pageSize=ask_partial_size())
+			)).data
+		finally:
+			self.close_transaction()
+
+	def is_existed(self, change_json: ChangeDataJson) -> bool:
+		self.begin_transaction()
+		try:
+			return self.storage.exists(self.get_entity_finder(
+				criteria=[
+					EntityCriteriaExpression(
+						left=ColumnNameLiteral(
+							columnName=self.get_storable_id_column_name()
+						),
+						right=change_json.changeJsonId)
+				]
+			))
+		finally:
+			self.close_transaction()
+
 	def find_json_by_id(self, change_json_id: str) -> ChangeDataJson:
 		self.begin_transaction()
 		try:
@@ -112,7 +143,7 @@ class ChangeDataJsonService(TupleService):
 		finally:
 			self.close_transaction()
 
-	def find_id_by_resource_id(self, resource_id: str) -> Optional[ChangeDataJson]:
+	def find_by_resource_id(self, resource_id: str) -> Optional[ChangeDataJson]:
 		try:
 			self.storage.connect()
 			return self.storage.find_one(self.get_entity_finder(

watchmen_collector_kernel-16.4.10/src/watchmen_collector_kernel/storage/change_data_record_history_service.py

@@ -0,0 +1,47 @@
+from .change_data_record_service import ChangeDataRecordShaper, ChangeDataRecordService
+from watchmen_auth import PrincipalService
+from watchmen_collector_kernel.model import ChangeDataRecordHistory
+from watchmen_meta.common.storage_service import StorableId
+from watchmen_model.common import Storable, ChangeRecordId
+from watchmen_storage import EntityName, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator
+
+
+class ChangeDataRecordHistoryShaper(ChangeDataRecordShaper):
+	pass
+
+
+CHANGE_DATA_RECORD_HISTORY_TABLE = 'change_data_record_history'
+CHANGE_DATA_RECORD_HISTORY_ENTITY_SHAPER = ChangeDataRecordHistoryShaper()
+
+
+class ChangeDataRecordHistoryService(ChangeDataRecordService):
+
+	def should_record_operation(self) -> bool:
+		return False
+
+	def get_entity_name(self) -> EntityName:
+		return CHANGE_DATA_RECORD_HISTORY_TABLE
+
+	def get_entity_shaper(self) -> EntityShaper:
+		return CHANGE_DATA_RECORD_HISTORY_ENTITY_SHAPER
+
+	# noinspection SpellCheckingInspection
+	def get_storable_id_column_name(self) -> EntityName:
+		return 'change_record_id'
+
+	# noinspection SpellCheckingInspection
+	def get_storable_id(self, storable: ChangeDataRecordHistory) -> StorableId:
+		# noinspection PyTypeChecker
+		return storable.changeRecordId
+
+	# noinspection SpellCheckingInspection
+	def set_storable_id(self, storable: ChangeDataRecordHistory, storable_id: ChangeRecordId) -> Storable:
+		storable.changeRecordId = storable_id
+		return storable
+
+
+def get_change_data_record_history_service(storage: TransactionalStorageSPI,
+                                           snowflake_generator: SnowflakeGenerator,
+                                           principal_service: PrincipalService
+                                           ) -> ChangeDataRecordService:
+	return ChangeDataRecordHistoryService(storage, snowflake_generator, principal_service)

{watchmen_collector_kernel-16.4.8 → watchmen_collector_kernel-16.4.10}/src/watchmen_collector_kernel/storage/change_data_record_service.py

@@ -1,13 +1,14 @@
 from typing import List, Optional, Dict
 
 from watchmen_auth import PrincipalService
-from watchmen_collector_kernel.common import CHANGE_RECORD_ID, TENANT_ID, IS_MERGED
+from watchmen_collector_kernel.common import CHANGE_RECORD_ID, TENANT_ID, IS_MERGED, ask_partial_size
 from watchmen_collector_kernel.model import ChangeDataRecord
 from watchmen_meta.common import TupleService, TupleShaper
 from watchmen_meta.common.storage_service import StorableId
-from watchmen_model.common import Storable, ChangeRecordId
+from watchmen_model.common import Storable, ChangeRecordId, Pageable
 from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
-	EntityCriteriaExpression, ColumnNameLiteral, EntityStraightValuesFinder, EntityStraightColumn, EntityColumnType
+	EntityCriteriaExpression, ColumnNameLiteral, EntityStraightValuesFinder, EntityStraightColumn, EntityColumnType, \
+	EntityPager
 from watchmen_utilities import ArrayHelper
 
 
@@ -110,19 +111,34 @@ class ChangeDataRecordService(TupleService):
 		finally:
 			self.close_transaction()
 
-	def count_unmerged_records(self) -> int:
+	def find_partial_records(self) -> List[ChangeDataRecord]:
 		self.begin_transaction()
 		try:
-			# noinspection PyTypeChecker
-			return self.storage.count(self.get_entity_finder(
+			return self.storage.page(self.get_entity_pager(
 				criteria=[
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName=IS_MERGED), right=False)
+				],
+				pageable=Pageable(pageNumber=1, pageSize=ask_partial_size())
+			)).data
+		finally:
+			self.close_transaction()
+
+	def is_existed(self, change_record: ChangeDataRecord) -> bool:
+		self.begin_transaction()
+		try:
+			return self.storage.exists(self.get_entity_finder(
+				criteria=[
+					EntityCriteriaExpression(
+						left=ColumnNameLiteral(
+							columnName=self.get_storable_id_column_name()
+						),
+						right=change_record.changeRecordId)
 				]
 			))
 		finally:
 			self.close_transaction()
 
-	def find_change_record_by_id(self, change_record_id: ChangeRecordId) -> ChangeDataRecord:
+	def find_change_record_by_id(self, change_record_id: ChangeRecordId) -> Optional[ChangeDataRecord]:
 		self.begin_transaction()
 		try:
 			# noinspection PyTypeChecker
@@ -143,8 +159,8 @@ class ChangeDataRecordService(TupleService):
 						                         right=table_trigger_id)
 					],
 					straightColumns=[EntityStraightColumn(columnName='data_id', columnType=EntityColumnType.JSON)]
-					)
 				)
+			)
 			return ArrayHelper(result).map(lambda x: x.get('data_id')).to_list()
 		finally:
 			self.close_transaction()

watchmen_collector_kernel-16.4.10/src/watchmen_collector_kernel/storage/scheduled_task_history_service.py

@@ -0,0 +1,45 @@
+from .scheduled_task_service import ScheduledTaskShaper, ScheduledTaskService
+
+from watchmen_auth import PrincipalService
+from watchmen_collector_kernel.model import ScheduledTask
+from watchmen_meta.common.storage_service import StorableId
+from watchmen_model.common import Storable, ScheduledTaskId
+from watchmen_storage import EntityName, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator
+
+
+class ScheduledTaskHistoryShaper(ScheduledTaskShaper):
+	pass
+
+
+SCHEDULED_TASK_HISTORY_TABLE = 'scheduled_task_history'
+SCHEDULED_TASK_HISTORY_ENTITY_SHAPER = ScheduledTaskHistoryShaper()
+
+
+class ScheduledTaskHistoryService(ScheduledTaskService):
+
+	def should_record_operation(self) -> bool:
+		return False
+
+	def get_entity_name(self) -> EntityName:
+		return SCHEDULED_TASK_HISTORY_TABLE
+
+	def get_entity_shaper(self) -> EntityShaper:
+		return SCHEDULED_TASK_HISTORY_ENTITY_SHAPER
+
+	def get_storable_id_column_name(self) -> EntityName:
+		return 'task_id'
+
+	def get_storable_id(self, storable: ScheduledTask) -> StorableId:
+		return storable.taskId
+
+	def set_storable_id(
+			self, storable: ScheduledTask, storable_id: ScheduledTaskId) -> Storable:
+		storable.taskId = storable_id
+		return storable
+
+
+def get_scheduled_task_history_service(storage: TransactionalStorageSPI,
+                                       snowflake_generator: SnowflakeGenerator,
+                                       principal_service: PrincipalService
+                                       ) -> ScheduledTaskHistoryService:
+	return ScheduledTaskHistoryService(storage, snowflake_generator, principal_service)

{watchmen_collector_kernel-16.4.8 → watchmen_collector_kernel-16.4.10}/src/watchmen_collector_kernel/storage/scheduled_task_service.py

@@ -1,17 +1,35 @@
 from typing import Optional, List, Any, Dict
 
+from watchmen_collector_kernel.common import ask_partial_size
+from watchmen_utilities import ArrayHelper
+
+from watchmen_collector_kernel.model.scheduled_task import Dependence
+
 from watchmen_auth import PrincipalService
 from watchmen_collector_kernel.model import ScheduledTask
 from watchmen_meta.common import TupleService, TupleShaper
 from watchmen_meta.common.storage_service import StorableId
-from watchmen_model.common import Storable, ScheduledTaskId
+from watchmen_model.common import Storable, ScheduledTaskId, Pageable
 from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, \
 	EntityCriteriaExpression, ColumnNameLiteral, SnowflakeGenerator, EntitySortColumn, EntitySortMethod, \
-	EntityCriteriaJoint, EntityCriteriaJointConjunction, EntityStraightValuesFinder, EntityStraightColumn, \
-	EntityCriteriaOperator
+	EntityCriteriaJoint, EntityStraightValuesFinder, EntityStraightColumn
 
 
 class ScheduledTaskShaper(EntityShaper):
+
+	@staticmethod
+	def serialize_dependence(dependence: Dependence) -> dict:
+		if isinstance(dependence, dict):
+			return dependence
+		else:
+			return dependence.dict()
+
+	@staticmethod
+	def serialize_depend_on(depend_on: Optional[List[Dependence]]) -> Optional[list]:
+		if depend_on is None:
+			return None
+		return ArrayHelper(depend_on).map(lambda x: ScheduledTaskShaper.serialize_dependence(x)).to_list()
+
 	def serialize(self, entity: ScheduledTask) -> EntityRow:
 		return TupleShaper.serialize_tenant_based(entity, {
 			'task_id': entity.taskId,
@@ -20,10 +38,10 @@ class ScheduledTaskShaper(EntityShaper):
 			'content': entity.content,
 			'model_name': entity.modelName,
 			'object_id': entity.objectId,
-			'depend_on': entity.dependOn,
+			'depend_on': ScheduledTaskShaper.serialize_depend_on(entity.dependOn),
 			'parent_task_id': entity.parentTaskId,
 			'tenant_id': entity.tenantId,
-			'status': entity.status,
+			'is_finished': entity.isFinished,
 			'result': entity.result
 		})
 
@@ -39,7 +57,7 @@ class ScheduledTaskShaper(EntityShaper):
 			dependOn=row.get('depend_on'),
 			parentTaskId=row.get('parent_task_id'),
 			tenantId=row.get('tenant_id'),
-			status=row.get('status'),
+			isFinished=row.get('is_finished'),
 			result=row.get('result')
 		))
 
@@ -107,7 +125,7 @@ class ScheduledTaskService(TupleService):
 				criteria=[EntityCriteriaJoint(
 					children=[
 						EntityCriteriaExpression(
-							left=ColumnNameLiteral(columnName='status'), right=0)
+							left=ColumnNameLiteral(columnName='is_finished'), right=False)
 					]
 				)],
 				straightColumns=[EntityStraightColumn(columnName='task_id'),
@@ -118,13 +136,43 @@ class ScheduledTaskService(TupleService):
 		finally:
 			self.close_transaction()
 
+	def find_partial_tasks(self) -> List[ScheduledTask]:
+		self.begin_transaction()
+		try:
+			return self.storage.page(self.get_entity_pager(
+				criteria=[
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='is_finished'), right=False)
+				],
+				pageable=Pageable(pageNumber=1, pageSize=ask_partial_size())
+			)).data
+		finally:
+			self.close_transaction()
+
+	def is_existed(self, task: ScheduledTask) -> bool:
+		self.begin_transaction()
+		try:
+			return self.storage.exists(self.get_entity_finder(
+				criteria=[
+					EntityCriteriaExpression(
+						left=ColumnNameLiteral(
+							columnName=self.get_storable_id_column_name()
+						),
+						right=task.taskId)
+				]
+			))
+		finally:
+			self.close_transaction()
+
 	def find_by_resource_id(self, resource_id: str) -> List[Dict[str, Any]]:
 		self.begin_transaction()
 		try:
 			return self.storage.find_straight_values(EntityStraightValuesFinder(
 				name=self.get_entity_name(),
-				criteria=[EntityCriteriaExpression(left=ColumnNameLiteral(columnName='resource_id'), right=resource_id)],
-				straightColumns=['task_id', 'resource_id', 'tenant_id']
+				criteria=[
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='resource_id'), right=resource_id)],
+				straightColumns=[EntityStraightColumn(columnName='task_id'),
+				                 EntityStraightColumn(columnName='resource_id'),
+				                 EntityStraightColumn(columnName='tenant_id')]
 			))
 		finally:
 			self.close_transaction()
@@ -139,8 +187,8 @@ class ScheduledTaskService(TupleService):
 					                         right=model_name),
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='object_id'),
 					                         right=object_id),
-					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='status'),
-					                         right=0),
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='is_finished'),
+					                         right=False),
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'),
 					                         right=tenant_id)
 				]
@@ -148,6 +196,7 @@ class ScheduledTaskService(TupleService):
 		finally:
 			self.close_transaction()
 
+
 def get_scheduled_task_service(storage: TransactionalStorageSPI,
                                snowflake_generator: SnowflakeGenerator,
                                principal_service: PrincipalService

watchmen_collector_kernel-16.4.8/PKG-INFO

@@ -1,28 +0,0 @@
-Metadata-Version: 2.1
-Name: watchmen-collector-kernel
-Version: 16.4.8
-Summary: 
-License: MIT
-Author: botlikes
-Author-email: 75356972+botlikes456@users.noreply.github.com
-Requires-Python: >=3.9,<4.0
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Provides-Extra: mongodb
-Provides-Extra: mssql
-Provides-Extra: mysql
-Provides-Extra: oracle
-Provides-Extra: oss
-Provides-Extra: postgresql
-Provides-Extra: s3
-Requires-Dist: watchmen-data-kernel (==16.4.8)
-Requires-Dist: watchmen-storage-mongodb (==16.4.8) ; extra == "mongodb"
-Requires-Dist: watchmen-storage-mssql (==16.4.8) ; extra == "mssql"
-Requires-Dist: watchmen-storage-mysql (==16.4.8) ; extra == "mysql"
-Requires-Dist: watchmen-storage-oracle (==16.4.8) ; extra == "oracle"
-Requires-Dist: watchmen-storage-oss (==16.4.8) ; extra == "oss"
-Requires-Dist: watchmen-storage-postgresql (==16.4.8) ; extra == "postgresql"
-Requires-Dist: watchmen-storage-s3 (==16.4.8) ; extra == "s3"

watchmen_collector_kernel-16.4.8/setup.py

@@ -1,48 +0,0 @@
-# -*- coding: utf-8 -*-
-from setuptools import setup
-
-package_dir = \
-{'': 'src'}
-
-packages = \
-['watchmen_collector_kernel',
- 'watchmen_collector_kernel.common',
- 'watchmen_collector_kernel.model',
- 'watchmen_collector_kernel.service',
- 'watchmen_collector_kernel.storage']
-
-package_data = \
-{'': ['*']}
-
-install_requires = \
-['watchmen-data-kernel==16.4.8']
-
-extras_require = \
-{'mongodb': ['watchmen-storage-mongodb==16.4.8'],
- 'mssql': ['watchmen-storage-mssql==16.4.8'],
- 'mysql': ['watchmen-storage-mysql==16.4.8'],
- 'oracle': ['watchmen-storage-oracle==16.4.8'],
- 'oss': ['watchmen-storage-oss==16.4.8'],
- 'postgresql': ['watchmen-storage-postgresql==16.4.8'],
- 's3': ['watchmen-storage-s3==16.4.8']}
-
-setup_kwargs = {
-    'name': 'watchmen-collector-kernel',
-    'version': '16.4.8',
-    'description': '',
-    'long_description': 'None',
-    'author': 'botlikes',
-    'author_email': '75356972+botlikes456@users.noreply.github.com',
-    'maintainer': 'None',
-    'maintainer_email': 'None',
-    'url': 'None',
-    'package_dir': package_dir,
-    'packages': packages,
-    'package_data': package_data,
-    'install_requires': install_requires,
-    'extras_require': extras_require,
-    'python_requires': '>=3.9,<4.0',
-}
-
-
-setup(**setup_kwargs)

watchmen_collector_kernel-16.4.8/src/watchmen_collector_kernel/model/scheduled_task.py

@@ -1,36 +0,0 @@
-from typing import Dict, List
-
-from watchmen_model.common import TenantBasedTuple, ScheduledTaskId, Storable
-from pydantic import BaseModel
-from enum import Enum, IntEnum
-
-
-class Result(str, Enum):
-	DEPENDENCY_FAILED = "DEPENDENCY_FAILED"
-	PROCESS_TASK_SUCCESS = "PROCESS_TASK_SUCCESS"
-	PROCESS_TASK_FAILED = "PROCESS_TASK_FAILED"
-
-
-class TaskStatus(IntEnum):
-	INITIAL = 0
-	SUCCESS = 1
-	FAILED = 2
-
-
-class Dependence(Storable, BaseModel):
-	modelName: str
-	objectId: str
-
-
-class ScheduledTask(TenantBasedTuple, BaseModel):
-	taskId: ScheduledTaskId
-	resourceId: str  # global unique, monotonous increase
-	topicCode: str
-	content: Dict
-	modelName: str
-	objectId: str
-	dependOn: List[Dependence]
-	parentTaskId: List[int]
-	status: TaskStatus
-	result: Dict
-