watchmen-collector-kernel 16.6.13__py3-none-any.whl → 16.6.14__py3-none-any.whl

watchmen_collector_kernel/common/__init__.py

@@ -3,5 +3,6 @@ from .constants import CHANGE_RECORD_ID, TENANT_ID, IS_MERGED, LEFT_BRACE, RIGHT
 from .settings import ask_clean_of_timeout_interval, ask_lock_timeout, ask_partial_size, \
 	ask_collector_config_cache_enabled, ask_collector_timeout, ask_clean_up_lock_timeout, \
 	ask_trigger_event_lock_timeout, ask_extract_table_lock_timeout, ask_s3_connector_lock_timeout, \
-	ask_collector_task_timeout, ask_exception_max_length
+	ask_collector_task_timeout, ask_exception_max_length, ask_grouped_task_data_size_threshold, \
+	ask_task_partial_size
 from .exception import CollectorKernelException

watchmen_collector_kernel/common/settings.py

@@ -15,7 +15,9 @@ class CollectorSettings(BaseSettings):
 	S3_CONNECTOR_LOCK_TIMEOUT = 300
 	PARTIAL_SIZE: int = 100
 	COLLECTOR_CONFIG_CACHE_ENABLED: bool = True
-	EXCEPTION_MAX_LENGTH = 1000  # character
+	EXCEPTION_MAX_LENGTH = 3000  # character
+	GROUPED_TASK_DATA_SIZE_THRESHOLD = 100
+	TASK_PARTIAL_SIZE: int = 100
 
 	class Config:
 		# secrets_dir = '/var/run'
@@ -70,3 +72,11 @@ def ask_collector_task_timeout() -> int:
 
 def ask_exception_max_length() -> int:
 	return collector_settings.EXCEPTION_MAX_LENGTH
+
+
+def ask_grouped_task_data_size_threshold() -> int:
+	return collector_settings.GROUPED_TASK_DATA_SIZE_THRESHOLD
+
+
+def ask_task_partial_size() -> int:
+	return collector_settings.TASK_PARTIAL_SIZE

watchmen_collector_kernel/model/scheduled_task.py

@@ -9,7 +9,8 @@ from enum import Enum
 
 class TaskType(int, Enum):
 	DEFAULT = 1,
-	RUN_PIPELINE = 2
+	RUN_PIPELINE = 2,
+	GROUP = 3
 
 
 class Result(str, Enum):

watchmen_collector_kernel/service/task_service.py

@@ -39,6 +39,10 @@ class TaskService:
 			executed(task.topicCode, task.content, task.tenantId)
 		elif task.type == TaskType.RUN_PIPELINE.value:
 			executed(task.topicCode, task.content, task.tenantId, task.pipelineId)
+		elif task.type == TaskType.GROUP.value:
+			raw_data_list = task.content.get("data")
+			for raw_data in raw_data_list:
+				executed(task.topicCode, raw_data, task.tenantId)
 
 	def update_task_result(self, task: ScheduledTask, status: int, result=None) -> ScheduledTask:
 		try:

watchmen_collector_kernel/storage/change_data_json_service.py

@@ -144,6 +144,23 @@ class ChangeDataJsonService(TupleService):
 				limit=limit if limit is not None else ask_partial_size()
 			))
 
+	def find_json(self, model_trigger_id: int, limit: int = None) -> List:
+		try:
+			self.begin_transaction()
+			return self.storage.find_limited(
+				EntityLimitedFinder(
+					name=self.get_entity_name(),
+					shaper=self.get_entity_shaper(),
+					criteria=[
+						EntityCriteriaExpression(left=ColumnNameLiteral(columnName=STATUS), right=0),
+						EntityCriteriaExpression(left=ColumnNameLiteral(columnName=MODEL_TRIGGER_ID),
+						                         right=model_trigger_id)
+					],
+					limit=limit if limit is not None else ask_partial_size()
+				))
+		finally:
+			self.close_transaction()
+
 	def is_existed(self, change_json: ChangeDataJson) -> bool:
 		self.begin_transaction()
 		try:
@@ -187,8 +204,7 @@ class ChangeDataJsonService(TupleService):
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='object_id'), right=object_id),
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='model_trigger_id'),
 					                         right=model_trigger_id)
-				],
-				sort=[EntitySortColumn(name='sequence', method=EntitySortMethod.ASC)]
+				]
 			))
 		finally:
 			self.storage.close()

watchmen_collector_kernel/storage/scheduled_task_service.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import Optional, List, Any, Dict
 
-from watchmen_collector_kernel.common import ask_partial_size, STATUS
+from watchmen_collector_kernel.common import ask_task_partial_size, STATUS
 from watchmen_utilities import ArrayHelper
 
 from watchmen_auth import PrincipalService
@@ -124,6 +124,14 @@ class ScheduledTaskService(TupleService):
 		else:
 			return None
 
+	def find_one_and_lock_nowait(self, task_id: int) -> Optional[ScheduledTask]:
+		return self.storage.find_one_and_lock_nowait(self.get_entity_finder(
+			criteria=[
+				EntityCriteriaExpression(left=ColumnNameLiteral(columnName='task_id'), right=task_id),
+				EntityCriteriaExpression(left=ColumnNameLiteral(columnName='status'), right=0),
+			])
+		)
+
 	def find_unfinished_tasks(self) -> List[Dict[str, Any]]:
 		self.begin_transaction()
 		try:
@@ -150,7 +158,7 @@ class ScheduledTaskService(TupleService):
 				criteria=[
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='is_finished'), right=False)
 				],
-				pageable=Pageable(pageNumber=1, pageSize=ask_partial_size())
+				pageable=Pageable(pageNumber=1, pageSize=ask_task_partial_size())
 			)).data
 		finally:
 			self.close_transaction()
@@ -163,7 +171,7 @@ class ScheduledTaskService(TupleService):
 				criteria=[
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName=STATUS), right=0)
 				],
-				limit=limit if limit is not None else ask_partial_size()
+				limit=limit if limit is not None else ask_task_partial_size()
 			))
 
 	def is_existed(self, task: ScheduledTask) -> bool:
@@ -209,7 +217,8 @@ class ScheduledTaskService(TupleService):
 					                         right=event_id),
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'),
 					                         right=tenant_id)
-				]
+				],
+				sort=[EntitySortColumn(name='task_id', method=EntitySortMethod.ASC)]
 			))
 		finally:
 			self.close_transaction()

{watchmen_collector_kernel-16.6.13.dist-info → watchmen_collector_kernel-16.6.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: watchmen-collector-kernel
-Version: 16.6.13
+Version: 16.6.14
 Summary: 
 License: MIT
 Author: botlikes
@@ -20,11 +20,11 @@ Provides-Extra: oss
 Provides-Extra: postgresql
 Provides-Extra: s3
 Requires-Dist: numpy (>=1.23.3,<2.0.0)
-Requires-Dist: watchmen-data-kernel (==16.6.13)
-Requires-Dist: watchmen-storage-mongodb (==16.6.13) ; extra == "mongodb"
-Requires-Dist: watchmen-storage-mssql (==16.6.13) ; extra == "mssql"
-Requires-Dist: watchmen-storage-mysql (==16.6.13) ; extra == "mysql"
-Requires-Dist: watchmen-storage-oracle (==16.6.13) ; extra == "oracle"
-Requires-Dist: watchmen-storage-oss (==16.6.13) ; extra == "oss"
-Requires-Dist: watchmen-storage-postgresql (==16.6.13) ; extra == "postgresql"
-Requires-Dist: watchmen-storage-s3 (==16.6.13) ; extra == "s3"
+Requires-Dist: watchmen-data-kernel (==16.6.14)
+Requires-Dist: watchmen-storage-mongodb (==16.6.14) ; extra == "mongodb"
+Requires-Dist: watchmen-storage-mssql (==16.6.14) ; extra == "mssql"
+Requires-Dist: watchmen-storage-mysql (==16.6.14) ; extra == "mysql"
+Requires-Dist: watchmen-storage-oracle (==16.6.14) ; extra == "oracle"
+Requires-Dist: watchmen-storage-oss (==16.6.14) ; extra == "oss"
+Requires-Dist: watchmen-storage-postgresql (==16.6.14) ; extra == "postgresql"
+Requires-Dist: watchmen-storage-s3 (==16.6.14) ; extra == "s3"

{watchmen_collector_kernel-16.6.13.dist-info → watchmen_collector_kernel-16.6.14.dist-info}/RECORD

@@ -6,10 +6,10 @@ watchmen_collector_kernel/cache/collector_topic_cache.py,sha256=jpr6-T4ETH5Oi7RI
 watchmen_collector_kernel/cache/model_config_cache.py,sha256=WW2PmILHm0WbC9gr73uIq1B-poGx78Xpf1w9WEZLbJ8,1906
 watchmen_collector_kernel/cache/module_config_cache.py,sha256=lGx-eIZLcLTa1RiH_kGU8soU1ilcfbE6KrYixim4Uhw,1393
 watchmen_collector_kernel/cache/table_config_cache.py,sha256=oOjyatujD-J6hFcj42hxdmMhZylRdj8v5UXypTcqJQE,4390
-watchmen_collector_kernel/common/__init__.py,sha256=tiP9kGZQhtRCLlwHfyBe6hu09OfwsqddUC8V-A9KKn0,563
+watchmen_collector_kernel/common/__init__.py,sha256=KCifKsnn6NPHkUMSq2q__fL9aWg52eK35a_mIfLKYRM,627
 watchmen_collector_kernel/common/constants.py,sha256=mBc0dVjMXKp2pluN2HMNcArcOWvycNzLNu7zcZO0Eao,320
 watchmen_collector_kernel/common/exception.py,sha256=AY4JAL0fBP6fAkgT7bfMTABp3x3oYtvV5vdPO5jI6S4,49
-watchmen_collector_kernel/common/settings.py,sha256=xlPOZ0U4iQgYC_yOAZMdL_w11sOQBoSM91MHqkWMdeE,1746
+watchmen_collector_kernel/common/settings.py,sha256=Sivg_S8lEaEn6IdJFP6lrTB3JnQi5m_FCqoP37A9wqQ,2012
 watchmen_collector_kernel/model/__init__.py,sha256=7FUgGxozQ3TMHBllZmds7g6nQbN2H6fID8cCLED7u04,845
 watchmen_collector_kernel/model/change_data_json.py,sha256=ckCj2vvDoZCWryYE8lGjJnZ_WEfynsVrL2RvmKA2IUk,530
 watchmen_collector_kernel/model/change_data_json_history.py,sha256=L9ACRe5tCP5Pc5MrpAVkLHe-6EfgZsUBfnQ2BXIYu4M,99
@@ -20,7 +20,7 @@ watchmen_collector_kernel/model/collector_module_config.py,sha256=0knkFhBdx4gpRX
 watchmen_collector_kernel/model/collector_table_config.py,sha256=apxZRQmLEMsVK3Z8KnAUy61LRSITDhTEs1EVNTvC-M0,3266
 watchmen_collector_kernel/model/competitive_lock.py,sha256=fxDRuzZpsgrZjGeuADS3zRYqQioqWhg72Pr592U_Yvw,264
 watchmen_collector_kernel/model/condition.py,sha256=I0QSsz1pP2DRKsgfjw_M7Z_seaPFfzOCvlg6Ud2Te0s,1603
-watchmen_collector_kernel/model/scheduled_task.py,sha256=Yj5ALS4_PPm-Cgf3N4SNIFGj4TEq2X_ix1Oi4PLO5dU,1479
+watchmen_collector_kernel/model/scheduled_task.py,sha256=Fzw_IeHGFYeEmWSQMajOb2WVCoX80wfmIL0scFG-6Ak,1491
 watchmen_collector_kernel/model/scheduled_task_history.py,sha256=0lgWIQWN2r_HcMoLGwKxzLmaxGNtr0NZF7NaT_DeOyE,93
 watchmen_collector_kernel/model/status.py,sha256=ctv8szNlnij5ZNQq2QPu5lWJds-QYwBX4kMAI6HqB3k,196
 watchmen_collector_kernel/model/trigger_event.py,sha256=vERU5ebEAy_2d7Au6WAEkpszou9Svb9tnk4rjMMKEOk,479
@@ -37,12 +37,12 @@ watchmen_collector_kernel/service/lock_helper.py,sha256=Jwh9J4alPReXpuKmhNxFbZ6z
 watchmen_collector_kernel/service/model_config_service.py,sha256=1tFd_MUldwl37JyRJF4FX6ywmFQ2caFYfslAlqnv2Jk,2235
 watchmen_collector_kernel/service/module_config_service.py,sha256=M4Mn45WEV9rllpX4A9LhtVE7VjBo-Q9LrHnR4_U2UYA,2027
 watchmen_collector_kernel/service/table_config_service.py,sha256=1jmaaOEM0ETRrTDGgGK7S9SCX85bPIP_jJ2zqsHH4DU,3962
-watchmen_collector_kernel/service/task_service.py,sha256=jThTe0EuViCq-EsKggr8rX2xg2N61EM0iDhyn1Je_Ds,4673
+watchmen_collector_kernel/service/task_service.py,sha256=MRpdcMAH6N3M8WyJ3g9O47ZrHNsq9643bXVlIWqjQyg,4847
 watchmen_collector_kernel/service/trigger_collector.py,sha256=p07SEQsfrBYIT02Ea5w9R70Yh7NAMC1TlFnn7hn79-o,10852
 watchmen_collector_kernel/service/trigger_event_helper.py,sha256=l01bMQrHM-bij4_S0vdzHDuERaTZ4PFo7jQQMRw7dTc,17784
 watchmen_collector_kernel/storage/__init__.py,sha256=064rAoXd3IkhPL6aVisW4BGYImKItD6h3btXsHkrBZo,1369
 watchmen_collector_kernel/storage/change_data_json_history_service.py,sha256=8WPdazvG6EPTOtmiAXtb1yAMRwUaBmD-C6S0ZwsCMrw,1774
-watchmen_collector_kernel/storage/change_data_json_service.py,sha256=sczu7rWdlA4JqcOx8cnvL1wUv0zV8zxGKeJostegHo4,12089
+watchmen_collector_kernel/storage/change_data_json_service.py,sha256=ILEqtAs2vQzfzEtMd7w6lMkmHbf-hQa9f0GpWhcbXTo,12611
 watchmen_collector_kernel/storage/change_data_record_history_service.py,sha256=q_41H2AQGLf_XD2W6ZsATLWvxpdK50swOWt5IMCTWjg,1845
 watchmen_collector_kernel/storage/change_data_record_service.py,sha256=6-b12-NmGPeO2LOUpHVc6uHtuUjU4cgIHe4LdjoG1KU,10458
 watchmen_collector_kernel/storage/collector_model_config_service.py,sha256=vIgiqVfxx94DuJVmZ9m-P7cKgfH5py54WfCn_BlYKqU,4555
@@ -50,12 +50,12 @@ watchmen_collector_kernel/storage/collector_module_config_service.py,sha256=OB4q
 watchmen_collector_kernel/storage/collector_table_config_service.py,sha256=PcXEgW4igoEqeTbYI8uikudRFuom4a-G9eGUAzjH9Vo,7705
 watchmen_collector_kernel/storage/competitive_lock_service.py,sha256=SIXpoeRP1S88ZQJy_-iZgGV9HTJwwM5r3vJn7J9WJbU,3154
 watchmen_collector_kernel/storage/scheduled_task_history_service.py,sha256=f-JXZ5G87oy7FdYPkbbfTKhI_oRm4wauWNvTw9hxGjM,1586
-watchmen_collector_kernel/storage/scheduled_task_service.py,sha256=NaGM7i4IRf0QtxLcIehc6VOKuKxnZtlbhkLYGp9o0c8,10206
+watchmen_collector_kernel/storage/scheduled_task_service.py,sha256=ENDhcjPzFkwr2YlLtOMiEerROyeUGufQM9G_LEw4-BE,10644
 watchmen_collector_kernel/storage/trigger_event_service.py,sha256=pQwfrwt6-6qkJvorO28dTR7cGuvcmvFQUMPUfTujjWg,6836
 watchmen_collector_kernel/storage/trigger_model_service.py,sha256=FE_88sYzh9ibUS1bOyDmGvryfPQraqkMbZc3BKtzfpI,3760
 watchmen_collector_kernel/storage/trigger_module_service.py,sha256=Ca4J8vnBzS90qP3ljwDJ7lt6MzaOushGjoFRwsSKfHI,3377
 watchmen_collector_kernel/storage/trigger_table_service.py,sha256=TRAFknaX6IW8lX7dMCR4y_3yjA2bYFFH1HdwW21--Kc,3915
-watchmen_collector_kernel-16.6.13.dist-info/LICENSE,sha256=iuuG7ErblOdcEZi5u89VXS0VIUiTb4flerGp_PAS9E4,1061
-watchmen_collector_kernel-16.6.13.dist-info/METADATA,sha256=MJ6UXJMttz1LHFMthQaXYgq0xTeRtmJjEkp3pc6OAv4,1234
-watchmen_collector_kernel-16.6.13.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-watchmen_collector_kernel-16.6.13.dist-info/RECORD,,
+watchmen_collector_kernel-16.6.14.dist-info/LICENSE,sha256=iuuG7ErblOdcEZi5u89VXS0VIUiTb4flerGp_PAS9E4,1061
+watchmen_collector_kernel-16.6.14.dist-info/METADATA,sha256=mt-ZJYPm9rayBe8c-cBWBuBpdwUxjg700cJBksL7Iuw,1234
+watchmen_collector_kernel-16.6.14.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+watchmen_collector_kernel-16.6.14.dist-info/RECORD,,