watchmen-collector-kernel 16.6.12__tar.gz → 16.6.14__tar.gz

{watchmen_collector_kernel-16.6.12 → watchmen_collector_kernel-16.6.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: watchmen-collector-kernel
-Version: 16.6.12
+Version: 16.6.14
 Summary: 
 License: MIT
 Author: botlikes
@@ -20,11 +20,11 @@ Provides-Extra: oss
 Provides-Extra: postgresql
 Provides-Extra: s3
 Requires-Dist: numpy (>=1.23.3,<2.0.0)
-Requires-Dist: watchmen-data-kernel (==16.6.12)
-Requires-Dist: watchmen-storage-mongodb (==16.6.12) ; extra == "mongodb"
-Requires-Dist: watchmen-storage-mssql (==16.6.12) ; extra == "mssql"
-Requires-Dist: watchmen-storage-mysql (==16.6.12) ; extra == "mysql"
-Requires-Dist: watchmen-storage-oracle (==16.6.12) ; extra == "oracle"
-Requires-Dist: watchmen-storage-oss (==16.6.12) ; extra == "oss"
-Requires-Dist: watchmen-storage-postgresql (==16.6.12) ; extra == "postgresql"
-Requires-Dist: watchmen-storage-s3 (==16.6.12) ; extra == "s3"
+Requires-Dist: watchmen-data-kernel (==16.6.14)
+Requires-Dist: watchmen-storage-mongodb (==16.6.14) ; extra == "mongodb"
+Requires-Dist: watchmen-storage-mssql (==16.6.14) ; extra == "mssql"
+Requires-Dist: watchmen-storage-mysql (==16.6.14) ; extra == "mysql"
+Requires-Dist: watchmen-storage-oracle (==16.6.14) ; extra == "oracle"
+Requires-Dist: watchmen-storage-oss (==16.6.14) ; extra == "oss"
+Requires-Dist: watchmen-storage-postgresql (==16.6.14) ; extra == "postgresql"
+Requires-Dist: watchmen-storage-s3 (==16.6.14) ; extra == "s3"

{watchmen_collector_kernel-16.6.12 → watchmen_collector_kernel-16.6.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "watchmen-collector-kernel"
-version = "16.6.12"
+version = "16.6.14"
 description = ""
 authors = ["botlikes <75356972+botlikes456@users.noreply.github.com>"]
 license = "MIT"
@@ -11,14 +11,14 @@ packages = [
 [tool.poetry.dependencies]
 python = "^3.9"
 numpy = "^1.23.3"
-watchmen-data-kernel = "16.6.12"
-watchmen-storage-mysql = { version = "16.6.12", optional = true }
-watchmen-storage-oracle = { version = "16.6.12", optional = true }
-watchmen-storage-mongodb = { version = "16.6.12", optional = true }
-watchmen-storage-mssql = { version = "16.6.12", optional = true }
-watchmen-storage-postgresql = { version = "16.6.12", optional = true }
-watchmen-storage-oss = { version = "16.6.12", optional = true }
-watchmen-storage-s3 = { version = "16.6.12", optional = true }
+watchmen-data-kernel = "16.6.14"
+watchmen-storage-mysql = { version = "16.6.14", optional = true }
+watchmen-storage-oracle = { version = "16.6.14", optional = true }
+watchmen-storage-mongodb = { version = "16.6.14", optional = true }
+watchmen-storage-mssql = { version = "16.6.14", optional = true }
+watchmen-storage-postgresql = { version = "16.6.14", optional = true }
+watchmen-storage-oss = { version = "16.6.14", optional = true }
+watchmen-storage-s3 = { version = "16.6.14", optional = true }
 
 [tool.poetry.dev-dependencies]
 

{watchmen_collector_kernel-16.6.12 → watchmen_collector_kernel-16.6.14}/src/watchmen_collector_kernel/common/__init__.py

@@ -3,5 +3,6 @@ from .constants import CHANGE_RECORD_ID, TENANT_ID, IS_MERGED, LEFT_BRACE, RIGHT
 from .settings import ask_clean_of_timeout_interval, ask_lock_timeout, ask_partial_size, \
 	ask_collector_config_cache_enabled, ask_collector_timeout, ask_clean_up_lock_timeout, \
 	ask_trigger_event_lock_timeout, ask_extract_table_lock_timeout, ask_s3_connector_lock_timeout, \
-	ask_collector_task_timeout, ask_exception_max_length
+	ask_collector_task_timeout, ask_exception_max_length, ask_grouped_task_data_size_threshold, \
+	ask_task_partial_size
 from .exception import CollectorKernelException

{watchmen_collector_kernel-16.6.12 → watchmen_collector_kernel-16.6.14}/src/watchmen_collector_kernel/common/settings.py

@@ -15,7 +15,9 @@ class CollectorSettings(BaseSettings):
 	S3_CONNECTOR_LOCK_TIMEOUT = 300
 	PARTIAL_SIZE: int = 100
 	COLLECTOR_CONFIG_CACHE_ENABLED: bool = True
-	EXCEPTION_MAX_LENGTH = 1000  # character
+	EXCEPTION_MAX_LENGTH = 3000  # character
+	GROUPED_TASK_DATA_SIZE_THRESHOLD = 100
+	TASK_PARTIAL_SIZE: int = 100
 
 	class Config:
 		# secrets_dir = '/var/run'
@@ -70,3 +72,11 @@ def ask_collector_task_timeout() -> int:
 
 def ask_exception_max_length() -> int:
 	return collector_settings.EXCEPTION_MAX_LENGTH
+
+
+def ask_grouped_task_data_size_threshold() -> int:
+	return collector_settings.GROUPED_TASK_DATA_SIZE_THRESHOLD
+
+
+def ask_task_partial_size() -> int:
+	return collector_settings.TASK_PARTIAL_SIZE

{watchmen_collector_kernel-16.6.12 → watchmen_collector_kernel-16.6.14}/src/watchmen_collector_kernel/model/scheduled_task.py

@@ -9,7 +9,8 @@ from enum import Enum
 
 class TaskType(int, Enum):
 	DEFAULT = 1,
-	RUN_PIPELINE = 2
+	RUN_PIPELINE = 2,
+	GROUP = 3
 
 
 class Result(str, Enum):

{watchmen_collector_kernel-16.6.12 → watchmen_collector_kernel-16.6.14}/src/watchmen_collector_kernel/service/task_service.py

@@ -39,6 +39,10 @@ class TaskService:
 			executed(task.topicCode, task.content, task.tenantId)
 		elif task.type == TaskType.RUN_PIPELINE.value:
 			executed(task.topicCode, task.content, task.tenantId, task.pipelineId)
+		elif task.type == TaskType.GROUP.value:
+			raw_data_list = task.content.get("data")
+			for raw_data in raw_data_list:
+				executed(task.topicCode, raw_data, task.tenantId)
 
 	def update_task_result(self, task: ScheduledTask, status: int, result=None) -> ScheduledTask:
 		try:

{watchmen_collector_kernel-16.6.12 → watchmen_collector_kernel-16.6.14}/src/watchmen_collector_kernel/storage/change_data_json_service.py

@@ -144,6 +144,23 @@ class ChangeDataJsonService(TupleService):
 				limit=limit if limit is not None else ask_partial_size()
 			))
 
+	def find_json(self, model_trigger_id: int, limit: int = None) -> List:
+		try:
+			self.begin_transaction()
+			return self.storage.find_limited(
+				EntityLimitedFinder(
+					name=self.get_entity_name(),
+					shaper=self.get_entity_shaper(),
+					criteria=[
+						EntityCriteriaExpression(left=ColumnNameLiteral(columnName=STATUS), right=0),
+						EntityCriteriaExpression(left=ColumnNameLiteral(columnName=MODEL_TRIGGER_ID),
+						                         right=model_trigger_id)
+					],
+					limit=limit if limit is not None else ask_partial_size()
+				))
+		finally:
+			self.close_transaction()
+
 	def is_existed(self, change_json: ChangeDataJson) -> bool:
 		self.begin_transaction()
 		try:
@@ -187,8 +204,7 @@ class ChangeDataJsonService(TupleService):
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='object_id'), right=object_id),
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='model_trigger_id'),
 					                         right=model_trigger_id)
-				],
-				sort=[EntitySortColumn(name='sequence', method=EntitySortMethod.ASC)]
+				]
 			))
 		finally:
 			self.storage.close()

{watchmen_collector_kernel-16.6.12 → watchmen_collector_kernel-16.6.14}/src/watchmen_collector_kernel/storage/scheduled_task_service.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import Optional, List, Any, Dict
 
-from watchmen_collector_kernel.common import ask_partial_size, STATUS
+from watchmen_collector_kernel.common import ask_task_partial_size, STATUS
 from watchmen_utilities import ArrayHelper
 
 from watchmen_auth import PrincipalService
@@ -124,6 +124,14 @@ class ScheduledTaskService(TupleService):
 		else:
 			return None
 
+	def find_one_and_lock_nowait(self, task_id: int) -> Optional[ScheduledTask]:
+		return self.storage.find_one_and_lock_nowait(self.get_entity_finder(
+			criteria=[
+				EntityCriteriaExpression(left=ColumnNameLiteral(columnName='task_id'), right=task_id),
+				EntityCriteriaExpression(left=ColumnNameLiteral(columnName='status'), right=0),
+			])
+		)
+
 	def find_unfinished_tasks(self) -> List[Dict[str, Any]]:
 		self.begin_transaction()
 		try:
@@ -150,7 +158,7 @@ class ScheduledTaskService(TupleService):
 				criteria=[
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='is_finished'), right=False)
 				],
-				pageable=Pageable(pageNumber=1, pageSize=ask_partial_size())
+				pageable=Pageable(pageNumber=1, pageSize=ask_task_partial_size())
 			)).data
 		finally:
 			self.close_transaction()
@@ -163,7 +171,7 @@ class ScheduledTaskService(TupleService):
 				criteria=[
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName=STATUS), right=0)
 				],
-				limit=limit if limit is not None else ask_partial_size()
+				limit=limit if limit is not None else ask_task_partial_size()
 			))
 
 	def is_existed(self, task: ScheduledTask) -> bool:
@@ -209,7 +217,8 @@ class ScheduledTaskService(TupleService):
 					                         right=event_id),
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'),
 					                         right=tenant_id)
-				]
+				],
+				sort=[EntitySortColumn(name='task_id', method=EntitySortMethod.ASC)]
 			))
 		finally:
 			self.close_transaction()