watchmen-collector-kernel 16.5.8__py3-none-any.whl → 16.5.9__py3-none-any.whl

watchmen_collector_kernel/common/settings.py

@@ -7,7 +7,7 @@ logger = getLogger(__name__)
 class CollectorSettings(BaseSettings):
 	LOCK_CLEAN_INTERVAL: int = 60
 	LOCK_CLEAN_TIMEOUT: int = 3600
-	PARTIAL_SIZE: int = 10000
+	PARTIAL_SIZE: int = 100
 
 	class Config:
 		# secrets_dir = '/var/run'

watchmen_collector_kernel/model/__init__.py

@@ -1,10 +1,12 @@
 from .competitive_lock import CompetitiveLock
 from .scheduled_task import ScheduledTask
 
+from .collector_module_config import CollectorModuleConfig
 from .collector_model_config import CollectorModelConfig
 from .collector_table_config import CollectorTableConfig
 
 from .trigger_event import TriggerEvent
+from .trigger_module import TriggerModule
 from .trigger_model import TriggerModel
 from .trigger_table import TriggerTable
 

watchmen_collector_kernel/model/change_data_json.py

@@ -24,5 +24,6 @@ class ChangeDataJson(TenantBasedTuple, BaseModel):
 	taskId: int
 	tableTriggerId: int
 	modelTriggerId: int
+	moduleTriggerId: int
 	eventTriggerId: int
 

watchmen_collector_kernel/model/change_data_record.py

@@ -14,6 +14,7 @@ class ChangeDataRecord(TenantBasedTuple):
 	result: Dict
 	tableTriggerId: int
 	modelTriggerId: int
+	moduleTriggerId: int
 	eventTriggerId: int
 
 

watchmen_collector_kernel/model/collector_model_config.py

@@ -6,7 +6,9 @@ from pydantic import BaseModel
 class CollectorModelConfig(TenantBasedTuple, OptimisticLock, BaseModel):
 	modelId: str
 	modelName: str
+	moduleId: str
 	dependOn: List[str]
+	priority: int = 0
 	rawTopicCode: str
 	isParalleled: bool
 

watchmen_collector_kernel/model/collector_module_config.py

@@ -0,0 +1,10 @@
+from typing import List
+from watchmen_model.common import TenantBasedTuple, OptimisticLock
+from pydantic import BaseModel
+
+
+class CollectorModuleConfig(TenantBasedTuple, OptimisticLock, BaseModel):
+	moduleId: str
+	moduleName: str
+	priority: int = 0
+

watchmen_collector_kernel/model/collector_table_config.py

@@ -22,6 +22,7 @@ class JsonColumn(Storable, BaseModel):
 	ignoredPath: List[str] = None
 	needFlatten: bool = None
 	flattenPath: List[str] = None
+	jsonPath: List[str] = None
 
 
 def construct_json_column(json_column: Union[JsonColumn, Dict]) -> Optional[JsonColumn]:

watchmen_collector_kernel/model/trigger_model.py

@@ -5,4 +5,7 @@ class TriggerModel(TenantBasedTuple):
 	modelTriggerId: int
 	modelName: str
 	isFinished: bool = False
+	priority: int
+	moduleTriggerId: int
 	eventTriggerId: int
+

watchmen_collector_kernel/model/trigger_module.py

@@ -0,0 +1,9 @@
+from watchmen_model.common import TenantBasedTuple
+
+
+class TriggerModule(TenantBasedTuple):
+	moduleTriggerId: int
+	moduleName: str
+	isFinished: bool = False
+	priority: int
+	eventTriggerId: int

watchmen_collector_kernel/model/trigger_table.py

@@ -8,4 +8,5 @@ class TriggerTable(TenantBasedTuple):
 	modelName: str
 	isExtracted: bool = False
 	modelTriggerId: int
+	moduleTriggerId: int
 	eventTriggerId: int

watchmen_collector_kernel/service/extract_source.py

@@ -58,7 +58,7 @@ class SourceTableExtractor:
 		              tenantId=self.principal_service.tenantId
 		              )
 		topic_storage = ask_topic_storage(topic, self.principal_service)
-		factors = topic_storage.ask_synonym_factors(config.tableName)
+		factors = topic_storage.ask_reflect_factors(config.tableName)
 		topic.factors = factors
 		now = get_current_time_in_seconds()
 		topic.createdAt = now
@@ -157,6 +157,34 @@ class SourceTableExtractor:
 
 			return ArrayHelper(flatten_path_list).reduce(flatten_path, data_dict)
 
+		def process_json_path(inner_json_path_list: List[str], data_dict: Dict) -> Dict:
+
+			def json_path(data_need_load: Dict, inner_json_path: str) -> Dict:
+				json_path_list = inner_json_path.split(".")
+				json_path_link_head = LinkList().create_tail(json_path_list)
+
+				def load(data_loaded: Dict, node: LinkNode) -> Optional[Dict]:
+					if data_loaded is None:
+						return None
+					if node.item in data_loaded:
+						if node.next is not None:
+							temp = data_loaded[node.item]
+							data_loaded[node.item] = load(temp, node.next)
+							return data_loaded
+						else:
+							if data_loaded[node.item]:
+								try:
+									data_loaded[node.item] = json.loads(data_loaded[node.item])
+									return data_loaded
+								except ValueError:
+									logger.error(f'json node name: {node.item}')
+					else:
+						return data_loaded
+
+				return load(data_need_load, json_path_link_head)
+
+			return ArrayHelper(inner_json_path_list).reduce(json_path, data_dict)
+
 		def change_column_value(row: Dict) -> Dict:
 			for key, value in row.items():
 				for column in json_columns:
@@ -175,6 +203,10 @@ class SourceTableExtractor:
 
 								if column.flattenPath:
 									tmp_data = process_flatten_path(column.flattenPath, tmp_data)
+
+								if column.jsonPath:
+									tmp_data = process_json_path(column.jsonPath, tmp_data)
+
 							row[key] = tmp_data
 						else:
 							pass

watchmen_collector_kernel/service/lock_helper.py

@@ -1,6 +1,6 @@
 from typing import Union
 
-from sqlalchemy.exc import IntegrityError
+from sqlalchemy.exc import IntegrityError, OperationalError
 
 from watchmen_collector_kernel.model import CompetitiveLock
 from watchmen_collector_kernel.storage.competitive_lock_service import CompetitiveLockService
@@ -11,6 +11,8 @@ def try_lock_nowait(lock_service: CompetitiveLockService, lock: CompetitiveLock)
 	try:
 		lock_service.insert_one(lock)
 		return True
+	except OperationalError:
+		return False
 	except IntegrityError:
 		return False
 

watchmen_collector_kernel/service/task_service.py

@@ -38,9 +38,8 @@ class TaskService:
 			return self.update_task_result(task)
 		except Exception as e:
 			logger.error(e, exc_info=True, stack_info=True)
-			task.isFinished = True
 			task.result = format_exc()
-			return self.scheduled_task_service.update_task(task)
+			return self.update_task_result(task)
 
 	def update_task_result(self, task: ScheduledTask) -> ScheduledTask:
 		self.scheduled_task_service.begin_transaction()

watchmen_collector_kernel/service/trigger_collector.py

@@ -2,11 +2,12 @@ from typing import Callable, List
 
 from watchmen_auth import PrincipalService
 from watchmen_collector_kernel.model import TriggerEvent, CollectorModelConfig, TriggerModel, TriggerTable, \
-	CollectorTableConfig
+	CollectorTableConfig, TriggerModule, CollectorModuleConfig
 
 from watchmen_collector_kernel.storage import TriggerModelService, TriggerTableService, TriggerEventService, \
 	get_trigger_model_service, get_collector_model_config_service, get_trigger_table_service, \
-	CollectorTableConfigService, get_collector_table_config_service
+	CollectorTableConfigService, get_collector_table_config_service, get_collector_module_config_service, \
+	get_trigger_module_service, TriggerModuleService, CollectorModelConfigService
 from watchmen_meta.common import TupleService
 from watchmen_model.common import Storable
 
@@ -21,16 +22,57 @@ def redress_storable_id(tuple_service: TupleService,
 		tuple_service.redress_storable_id(a_tuple)
 
 
-def new_trigger_model(model_name: str, event_trigger_id: str) -> TriggerModel:
+def new_trigger_module(module_name: str, priority: int, event_trigger_id: int) -> TriggerModule:
+	return TriggerModule(
+		moduleName=module_name,
+		priority=priority,
+		is_finished=False,
+		eventTriggerId=event_trigger_id
+	)
+
+
+def get_trigger_module(event_trigger: TriggerEvent, module_config: CollectorModuleConfig) -> TriggerModule:
+	return new_trigger_module(module_config.moduleName, module_config.priority, event_trigger.eventTriggerId)
+
+
+def save_trigger_module(trigger_module_service: TriggerModuleService,
+                        trigger_module: TriggerModule,
+                        principal_service: PrincipalService) -> TriggerModule:
+	validate_tenant_id(trigger_module, principal_service)
+	redress_storable_id(trigger_module_service, trigger_module)
+	# noinspection PyTypeChecker
+	return trigger_module_service.create(trigger_module)
+
+
+def get_trigger_module_action(trigger_event_service: TriggerEventService,
+                              trigger_event: TriggerEvent) -> Callable[[CollectorModuleConfig], TriggerModule]:
+	def create_trigger_module_action(module_config: CollectorModuleConfig) -> TriggerModule:
+		trigger_module = get_trigger_module(trigger_event, module_config)
+		trigger_module_service = get_trigger_module_service(trigger_event_service.storage,
+		                                                    trigger_event_service.snowflakeGenerator,
+		                                                    trigger_event_service.principalService)
+		return save_trigger_module(trigger_module_service,
+		                           trigger_module,
+		                           trigger_event_service.principalService)
+
+	return create_trigger_module_action
+
+
+def new_trigger_model(model_name: str, priority: int, module_trigger_id: int, event_trigger_id: int) -> TriggerModel:
 	return TriggerModel(
 		modelName=model_name,
+		priority=priority,
 		is_finished=False,
+		moduleTriggerId=module_trigger_id,
 		eventTriggerId=event_trigger_id
 	)
 
 
-def get_trigger_model(event_trigger: TriggerEvent, model_config: CollectorModelConfig) -> TriggerModel:
-	return new_trigger_model(model_config.modelName, event_trigger.eventTriggerId)
+def get_trigger_model(module_trigger: TriggerModule, model_config: CollectorModelConfig) -> TriggerModel:
+	return new_trigger_model(model_config.modelName,
+	                         model_config.priority,
+	                         module_trigger.moduleTriggerId,
+	                         module_trigger.eventTriggerId)
 
 
 def save_trigger_model(trigger_model_service: TriggerModelService,
@@ -42,30 +84,37 @@ def save_trigger_model(trigger_model_service: TriggerModelService,
 	return trigger_model_service.create(trigger_model)
 
 
-def get_trigger_model_action(trigger_event_service: TriggerEventService,
-                             trigger_event: TriggerEvent) -> Callable[[CollectorModelConfig], TriggerModel]:
+def get_trigger_model_action(trigger_module_service: TriggerModuleService,
+                             trigger_module: TriggerModule) -> Callable[[CollectorModelConfig], TriggerModel]:
 	def create_trigger_model_action(model_config: CollectorModelConfig) -> TriggerModel:
-		trigger_model = get_trigger_model(trigger_event, model_config)
-		trigger_model_service = get_trigger_model_service(trigger_event_service.storage,
-		                                                  trigger_event_service.snowflakeGenerator,
-		                                                  trigger_event_service.principalService)
+		trigger_model = get_trigger_model(trigger_module, model_config)
+		trigger_model_service = get_trigger_model_service(trigger_module_service.storage,
+		                                                  trigger_module_service.snowflakeGenerator,
+		                                                  trigger_module_service.principalService)
 		return save_trigger_model(trigger_model_service,
 		                          trigger_model,
-		                          trigger_event_service.principalService)
+		                          trigger_module_service.principalService)
 
 	return create_trigger_model_action
 
 
+def get_model_configs_by_module(collector_model_config_service: CollectorModelConfigService,
+                                module_config: CollectorModuleConfig) -> List[CollectorModelConfig]:
+	return collector_model_config_service.find_by_module_id(module_config.moduleId)
+
+
 def new_trigger_table(table_name: str,
                       model_name: str,
-                      model_trigger_id: str,
-                      event_trigger_id: str) -> TriggerTable:
+                      model_trigger_id: int,
+                      module_trigger_id: int,
+                      event_trigger_id: int) -> TriggerTable:
 	return TriggerTable(
 		tableName=table_name,
 		dataCount=0,
 		modelName=model_name,
 		isExtracted=False,
 		modelTriggerId=model_trigger_id,
+		moduleTriggerId=module_trigger_id,
 		eventTriggerId=event_trigger_id
 	)
 
@@ -74,6 +123,7 @@ def get_trigger_table(model_trigger: TriggerModel, table_config: CollectorTableC
 	return new_trigger_table(table_config.tableName,
 	                         table_config.modelName,
 	                         model_trigger.modelTriggerId,
+	                         model_trigger.moduleTriggerId,
 	                         model_trigger.eventTriggerId)
 
 
@@ -112,10 +162,19 @@ def trigger_collector(trigger_event_service: TriggerEventService,
 	try:
 		# noinspection PyTypeChecker
 		trigger_event = trigger_event_service.create(trigger_event)
+		module_config_service = get_collector_module_config_service(trigger_event_service.storage,
+		                                                            trigger_event_service.snowflakeGenerator,
+		                                                            trigger_event_service.principalService)
+		module_configs = module_config_service.find_by_tenant(principal_service.get_tenant_id())
+		# noinspection PyTypeChecker
+		trigger_module_action = get_trigger_module_action(trigger_event_service, trigger_event)
+		trigger_module_service = get_trigger_module_service(trigger_event_service.storage,
+		                                                    trigger_event_service.snowflakeGenerator,
+		                                                    trigger_event_service.principalService)
 		model_config_service = get_collector_model_config_service(trigger_event_service.storage,
 		                                                          trigger_event_service.snowflakeGenerator,
 		                                                          trigger_event_service.principalService)
-		model_configs = model_config_service.find_by_tenant(principal_service.get_tenant_id())
+		# model_configs = model_config_service.find_by_tenant(principal_service.get_tenant_id())
 		# noinspection PyTypeChecker
 		trigger_model_action = get_trigger_model_action(trigger_event_service, trigger_event)
 		trigger_model_service = get_trigger_model_service(trigger_event_service.storage,
@@ -125,12 +184,17 @@ def trigger_collector(trigger_event_service: TriggerEventService,
 		table_config_service = get_collector_table_config_service(trigger_event_service.storage,
 		                                                          trigger_event_service.snowflakeGenerator,
 		                                                          trigger_event_service.principalService)
-		for model_config in model_configs:
-			trigger_model = trigger_model_action(model_config)
-			trigger_table_action = get_trigger_table_action(trigger_model_service, trigger_model)
-			table_configs = get_table_configs_by_model(table_config_service, model_config)
-			for table_config in table_configs:
-				trigger_table_action(table_config)
+
+		for module_config in module_configs:
+			trigger_module = trigger_module_action(module_config)
+			trigger_model_action = get_trigger_model_action(trigger_module_service, trigger_module)
+			model_configs = get_model_configs_by_module(model_config_service, module_config)
+			for model_config in model_configs:
+				trigger_model = trigger_model_action(model_config)
+				trigger_table_action = get_trigger_table_action(trigger_model_service, trigger_model)
+				table_configs = get_table_configs_by_model(table_config_service, model_config)
+				for table_config in table_configs:
+					trigger_table_action(table_config)
 
 		trigger_event_service.commit_transaction()
 	except Exception as e:

watchmen_collector_kernel/storage/__init__.py

@@ -3,11 +3,13 @@ from .scheduled_task_service import get_scheduled_task_service, ScheduledTaskSer
 from .scheduled_task_history_service import get_scheduled_task_history_service, ScheduledTaskHistoryService
 
 
+from .collector_module_config_service import get_collector_module_config_service, CollectorModuleConfigService
 from .collector_model_config_service import get_collector_model_config_service, CollectorModelConfigService
 from .collector_table_config_service import get_collector_table_config_service, CollectorTableConfigService
 
 
 from .trigger_event_service import get_trigger_event_service, TriggerEventService
+from .trigger_module_service import get_trigger_module_service, TriggerModuleService
 from .trigger_model_service import get_trigger_model_service, TriggerModelService
 from .trigger_table_service import get_trigger_table_service, TriggerTableService
 

watchmen_collector_kernel/storage/change_data_json_service.py

@@ -28,6 +28,7 @@ class ChangeDataJsonShaper(EntityShaper):
 			                                          'task_id': entity.taskId,
 			                                          'table_trigger_id': entity.tableTriggerId,
 			                                          'model_trigger_id': entity.modelTriggerId,
+			                                          'module_trigger_id': entity.moduleTriggerId,
 			                                          'event_trigger_id': entity.eventTriggerId
 		                                          })
 
@@ -48,6 +49,7 @@ class ChangeDataJsonShaper(EntityShaper):
 			                                            taskId=row.get('task_id'),
 			                                            tableTriggerId=row.get('table_trigger_id'),
 			                                            modelTriggerId=row.get('model_trigger_id'),
+			                                            moduleTriggerId=row.get('module_trigger_id'),
 			                                            eventTriggerId=row.get('event_trigger_id')
 		                                            ))
 
@@ -194,6 +196,19 @@ class ChangeDataJsonService(TupleService):
 		finally:
 			self.close_transaction()
 
+	def is_module_finished(self, module_trigger_id: int) -> bool:
+		self.begin_transaction()
+		try:
+			# noinspection PyTypeChecker
+			return self.storage.count(self.get_entity_finder(
+				criteria=[
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='module_trigger_id'), right=module_trigger_id),
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName=IS_POSTED), right=False)
+				]
+			)) == 0
+		finally:
+			self.close_transaction()
+
 
 def get_change_data_json_service(storage: TransactionalStorageSPI,
                                  snowflake_generator: SnowflakeGenerator,

watchmen_collector_kernel/storage/change_data_record_service.py

@@ -26,6 +26,7 @@ class ChangeDataRecordShaper(EntityShaper):
 			                                          'result': entity.result,
 			                                          'table_trigger_id': entity.tableTriggerId,
 			                                          'model_trigger_id': entity.modelTriggerId,
+			                                          'module_trigger_id': entity.moduleTriggerId,
 			                                          'event_trigger_id': entity.eventTriggerId
 		                                          })
 
@@ -43,6 +44,7 @@ class ChangeDataRecordShaper(EntityShaper):
 			                                            result=row.get('result'),
 			                                            tableTriggerId=row.get('table_trigger_id'),
 			                                            modelTriggerId=row.get('model_trigger_id'),
+			                                            moduleTriggerId=row.get('module_trigger_id'),
 			                                            eventTriggerId=row.get('event_trigger_id')
 		                                            ))
 
@@ -193,6 +195,20 @@ class ChangeDataRecordService(TupleService):
 		finally:
 			self.close_transaction()
 
+	def is_module_finished(self, module_trigger_id: int) -> bool:
+		self.begin_transaction()
+		try:
+			# noinspection PyTypeChecker
+			return self.storage.count(self.get_entity_finder(
+				criteria=[
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='module_trigger_id'),
+					                         right=module_trigger_id),
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName=IS_MERGED), right=False)
+				]
+			)) == 0
+		finally:
+			self.close_transaction()
+
 
 def get_change_data_record_service(storage: TransactionalStorageSPI,
                                    snowflake_generator: SnowflakeGenerator,

watchmen_collector_kernel/storage/collector_model_config_service.py

@@ -14,7 +14,9 @@ class CollectorModelConfigShaper(EntityShaper):
 		return TupleShaper.serialize_tenant_based(config, {
 			'model_id': config.modelId,
 			'model_name': config.modelName,
+			'module_id': config.moduleId,
 			'depend_on': config.dependOn,
+			'priority': config.priority,
 			'raw_topic_code': config.rawTopicCode,
 			'is_paralleled': config.isParalleled
 		})
@@ -24,7 +26,9 @@ class CollectorModelConfigShaper(EntityShaper):
 		return TupleShaper.deserialize_tenant_based(row, CollectorModelConfig(
 			modelId=row.get('model_id'),
 			modelName=row.get('model_name'),
+			moduleId=row.get('module_id'),
 			dependOn=row.get('depend_on'),
+			priority=row.get('priority'),
 			rawTopicCode=row.get('raw_topic_code'),
 			isParalleled=row.get('is_paralleled')
 		))
@@ -99,6 +103,13 @@ class CollectorModelConfigService(TupleService):
 		finally:
 			self.close_transaction()
 
+	def find_by_module_id(self, module_id: str) -> Optional[List[CollectorModelConfig]]:
+		# noinspection PyTypeChecker
+		return self.storage.find(self.get_entity_finder(
+			criteria=[
+				EntityCriteriaExpression(left=ColumnNameLiteral(columnName='module_id'), right=module_id)]
+		))
+
 
 def get_collector_model_config_service(storage: TransactionalStorageSPI,
                                        snowflake_generator: SnowflakeGenerator,

watchmen_collector_kernel/storage/collector_module_config_service.py

@@ -0,0 +1,103 @@
+from typing import Optional, List
+
+from watchmen_auth import PrincipalService
+from watchmen_collector_kernel.model import CollectorModuleConfig
+from watchmen_meta.common import TupleService, TupleShaper
+from watchmen_meta.common.storage_service import StorableId
+from watchmen_model.common import Storable, TenantId, CollectorModelConfigId
+from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
+	EntityCriteriaExpression, ColumnNameLiteral
+
+
+class CollectorModuleConfigShaper(EntityShaper):
+	def serialize(self, config: CollectorModuleConfig) -> EntityRow:
+		return TupleShaper.serialize_tenant_based(config, {
+			'module_id': config.moduleId,
+			'module_name': config.moduleName,
+			'priority': config.priority
+		})
+
+	def deserialize(self, row: EntityRow) -> CollectorModuleConfig:
+		# noinspection PyTypeChecker
+		return TupleShaper.deserialize_tenant_based(row, CollectorModuleConfig(
+			moduleId=row.get('module_id'),
+			moduleName=row.get('module_name'),
+			priority=row.get('priority')
+		))
+
+
+COLLECTOR_MODULE_CONFIG_ENTITY_NAME = 'collector_module_config'
+COLLECTOR_MODULE_CONFIG_ENTITY_SHAPER = CollectorModuleConfigShaper()
+
+
+class CollectorModuleConfigService(TupleService):
+	def should_record_operation(self) -> bool:
+		return False
+
+	def get_entity_name(self) -> EntityName:
+		return COLLECTOR_MODULE_CONFIG_ENTITY_NAME
+
+	def get_entity_shaper(self) -> EntityShaper:
+		return COLLECTOR_MODULE_CONFIG_ENTITY_SHAPER
+
+	# noinspection SpellCheckingInspection
+	def get_storable_id_column_name(self) -> EntityName:
+		return "module_id"
+
+	def get_storable_id(self, storable: CollectorModuleConfig) -> StorableId:
+		return storable.moduleId
+
+	# noinspection SpellCheckingInspection
+	def set_storable_id(self, storable: CollectorModuleConfig, storable_id: CollectorModelConfigId) -> Storable:
+		storable.moduleId = storable_id
+		return storable
+
+	def create_module_config(self, module_config: CollectorModuleConfig) -> CollectorModuleConfig:
+		try:
+			self.storage.connect()
+			# noinspection PyTypeChecker
+			return self.create(module_config)
+		finally:
+			self.storage.close()
+
+	def update_module_config(self, module_config: CollectorModuleConfig) -> CollectorModuleConfig:
+		try:
+			self.storage.connect()
+			# noinspection PyTypeChecker
+			return self.update(module_config)
+		finally:
+			self.storage.close()
+
+	def find_by_module_id(self, module_id: str) -> Optional[CollectorModuleConfig]:
+		self.begin_transaction()
+		try:
+			return self.storage.find_one(self.get_entity_finder(
+				criteria=[
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='module_id'), right=module_id)]
+			))
+		finally:
+			self.close_transaction()
+
+	def find_by_tenant(self, tenant_id: TenantId) -> Optional[List[CollectorModuleConfig]]:
+		# noinspection PyTypeChecker
+		return self.storage.find(self.get_entity_finder(
+			criteria=[
+				EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id)]
+		))
+
+	def find_by_name(self, module_name: str) -> Optional[CollectorModuleConfig]:
+		self.begin_transaction()
+		try:
+			return self.storage.find_one(self.get_entity_finder(
+				criteria=[
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='module_name'), right=module_name)]
+			))
+		finally:
+			self.close_transaction()
+
+
+def get_collector_module_config_service(storage: TransactionalStorageSPI,
+                                        snowflake_generator: SnowflakeGenerator,
+                                        principal_service: PrincipalService
+                                        ) -> CollectorModuleConfigService:
+	return CollectorModuleConfigService(storage, snowflake_generator, principal_service)

watchmen_collector_kernel/storage/trigger_model_service.py

@@ -16,7 +16,9 @@ class TriggerModelShaper(EntityShaper):
 		return TupleShaper.serialize_tenant_based(entity, {
 			'model_trigger_id': entity.modelTriggerId,
 			'model_name': entity.modelName,
+			'priority': entity.priority,
 			'is_finished': entity.isFinished,
+			'module_trigger_id': entity.moduleTriggerId,
 			'event_trigger_id': entity.eventTriggerId
 		})
 
@@ -25,7 +27,9 @@ class TriggerModelShaper(EntityShaper):
 		return TupleShaper.deserialize_tenant_based(row, TriggerModel(
 			modelTriggerId=row.get('model_trigger_id'),
 			modelName=row.get('model_name'),
+			priority=row.get('priority'),
 			isFinished=row.get('is_finished'),
+			moduleTriggerId=row.get('module_trigger_id'),
 			eventTriggerId=row.get('event_trigger_id')
 		))
 
@@ -88,6 +92,17 @@ class TriggerModelService(TupleService):
 		finally:
 			self.close_transaction()
 
+	def find_by_module_trigger_id(self, module_trigger_id: int) -> List[TriggerModel]:
+		self.begin_transaction()
+		try:
+			# noinspection PyTypeChecker
+			return self.storage.find(self.get_entity_finder(
+				criteria=[
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='module_trigger_id'), right=module_trigger_id)
+				]
+			))
+		finally:
+			self.close_transaction()
 
 def get_trigger_model_service(storage: TransactionalStorageSPI,
                               snowflake_generator: SnowflakeGenerator,

watchmen_collector_kernel/storage/trigger_module_service.py

@@ -0,0 +1,100 @@
+from typing import Optional, List
+
+from watchmen_auth import PrincipalService
+from watchmen_collector_kernel.common import IS_FINISHED, TENANT_ID
+from watchmen_collector_kernel.model import TriggerModule
+from watchmen_meta.common import TupleShaper, TupleService
+from watchmen_meta.common.storage_service import StorableId
+from watchmen_model.common import Storable, ModelTriggerId
+from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
+	EntityCriteriaExpression, ColumnNameLiteral
+
+
+class TriggerModuleShaper(EntityShaper):
+
+	def serialize(self, entity: TriggerModule) -> EntityRow:
+		return TupleShaper.serialize_tenant_based(entity, {
+			'module_trigger_id': entity.moduleTriggerId,
+			'module_name': entity.moduleName,
+			'priority': entity.priority,
+			'is_finished': entity.isFinished,
+			'event_trigger_id': entity.eventTriggerId
+		})
+
+	def deserialize(self, row: EntityRow) -> TriggerModule:
+		# noinspection PyTypeChecker
+		return TupleShaper.deserialize_tenant_based(row, TriggerModule(
+			moduleTriggerId=row.get('module_trigger_id'),
+			moduleName=row.get('module_name'),
+			priority=row.get('priority'),
+			isFinished=row.get('is_finished'),
+			eventTriggerId=row.get('event_trigger_id')
+		))
+
+
+TRIGGER_MODULE_TABLE = 'trigger_module'
+TRIGGER_MODULE_ENTITY_SHAPER = TriggerModuleShaper()
+
+
+class TriggerModuleService(TupleService):
+
+	def should_record_operation(self) -> bool:
+		return False
+
+	def get_entity_name(self) -> EntityName:
+		return TRIGGER_MODULE_TABLE
+
+	def get_entity_shaper(self) -> EntityShaper:
+		return TRIGGER_MODULE_ENTITY_SHAPER
+
+	def get_storable_id_column_name(self) -> EntityName:
+		return 'module_trigger_id'
+
+	def get_storable_id(self, storable: TriggerModule) -> StorableId:
+		# noinspection PyTypeChecker
+		return storable.moduleTriggerId
+
+	def set_storable_id(
+			self, storable: TriggerModule, storable_id: ModelTriggerId) -> Storable:
+		storable.moduleTriggerId = storable_id
+		return storable
+
+	def find_trigger_by_id(self, trigger_id: int) -> Optional[TriggerModule]:
+		self.begin_transaction()
+		try:
+			return self.find_by_id(trigger_id)
+		finally:
+			self.close_transaction()
+
+	def is_finished(self, event_trigger_id: str) -> bool:
+		self.begin_transaction()
+		try:
+			return self.storage.count(self.get_entity_finder(
+				criteria=[
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName=IS_FINISHED), right=False),
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='event_trigger_id'),
+					                         right=event_trigger_id)
+				]
+			)) == 0
+		finally:
+			self.close_transaction()
+
+	def find_by_event_trigger_id(self, event_trigger_id: int) -> List[TriggerModule]:
+		self.begin_transaction()
+		try:
+			# noinspection PyTypeChecker
+			return self.storage.find(self.get_entity_finder(
+				criteria=[
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='event_trigger_id'),
+					                         right=event_trigger_id)
+				]
+			))
+		finally:
+			self.close_transaction()
+
+
+def get_trigger_module_service(storage: TransactionalStorageSPI,
+                               snowflake_generator: SnowflakeGenerator,
+                               principal_service: PrincipalService
+                               ) -> TriggerModuleService:
+	return TriggerModuleService(storage, snowflake_generator, principal_service)

watchmen_collector_kernel/storage/trigger_table_service.py

@@ -19,6 +19,7 @@ class TriggerTableShaper(EntityShaper):
 			'is_extracted': entity.isExtracted,
 			'data_count': entity.dataCount,
 			'model_trigger_id': entity.modelTriggerId,
+			'module_trigger_id': entity.moduleTriggerId,
 			'event_trigger_id': entity.eventTriggerId
 		})
 
@@ -31,6 +32,7 @@ class TriggerTableShaper(EntityShaper):
 			isExtracted=row.get('is_extracted'),
 			dataCount=row.get('data_count'),
 			modelTriggerId=row.get('model_trigger_id'),
+			moduleTriggerId=row.get('module_trigger_id'),
 			eventTriggerId=row.get('event_trigger_id')
 		))
 

{watchmen_collector_kernel-16.5.8.dist-info → watchmen_collector_kernel-16.5.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: watchmen-collector-kernel
-Version: 16.5.8
+Version: 16.5.9
 Summary: 
 License: MIT
 Author: botlikes
@@ -19,11 +19,11 @@ Provides-Extra: oss
 Provides-Extra: postgresql
 Provides-Extra: s3
 Requires-Dist: numpy (>=1.23.3,<2.0.0)
-Requires-Dist: watchmen-data-kernel (==16.5.8)
-Requires-Dist: watchmen-storage-mongodb (==16.5.8) ; extra == "mongodb"
-Requires-Dist: watchmen-storage-mssql (==16.5.8) ; extra == "mssql"
-Requires-Dist: watchmen-storage-mysql (==16.5.8) ; extra == "mysql"
-Requires-Dist: watchmen-storage-oracle (==16.5.8) ; extra == "oracle"
-Requires-Dist: watchmen-storage-oss (==16.5.8) ; extra == "oss"
-Requires-Dist: watchmen-storage-postgresql (==16.5.8) ; extra == "postgresql"
-Requires-Dist: watchmen-storage-s3 (==16.5.8) ; extra == "s3"
+Requires-Dist: watchmen-data-kernel (==16.5.9)
+Requires-Dist: watchmen-storage-mongodb (==16.5.9) ; extra == "mongodb"
+Requires-Dist: watchmen-storage-mssql (==16.5.9) ; extra == "mssql"
+Requires-Dist: watchmen-storage-mysql (==16.5.9) ; extra == "mysql"
+Requires-Dist: watchmen-storage-oracle (==16.5.9) ; extra == "oracle"
+Requires-Dist: watchmen-storage-oss (==16.5.9) ; extra == "oss"
+Requires-Dist: watchmen-storage-postgresql (==16.5.9) ; extra == "postgresql"
+Requires-Dist: watchmen-storage-s3 (==16.5.9) ; extra == "s3"

{watchmen_collector_kernel-16.5.8.dist-info → watchmen_collector_kernel-16.5.9.dist-info}/RECORD

@@ -1,45 +1,49 @@
 watchmen_collector_kernel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 watchmen_collector_kernel/common/__init__.py,sha256=xDuOxA-p9hVm0IzAsPlrnsohNJgZ0qF6g8Kx8bn1p-E,263
 watchmen_collector_kernel/common/constants.py,sha256=mIsGBhAjHbmeuVeNYDOKLhiBqG2sSX-0vs8ipEEIo9E,302
-watchmen_collector_kernel/common/settings.py,sha256=xOOrpMe2E8MNek2Z4CLFTQp_yw2NOJCVi_Y28kgukOc,697
-watchmen_collector_kernel/model/__init__.py,sha256=HyMAWNLN1ZIppJtLbqVIIgyFEqrQZshNZk5It5fqKbY,666
-watchmen_collector_kernel/model/change_data_json.py,sha256=-oNIGsvYo8KJJsSFusbGa9kIuFfE0otLnWe-zz9O42A,495
+watchmen_collector_kernel/common/settings.py,sha256=p1I-LpMYRyg_Wx-C15vLYsJf54iwBxH5brqg8s9CJDk,695
+watchmen_collector_kernel/model/__init__.py,sha256=E6niSOjVdFtNfpvghy9z8WoPqXfIOnsbc_HBXWbKvCk,767
+watchmen_collector_kernel/model/change_data_json.py,sha256=M3hYIAoL1ZmlcfGjurQIA7sGiXmAoD12UFby1yNNWDs,517
 watchmen_collector_kernel/model/change_data_json_history.py,sha256=L9ACRe5tCP5Pc5MrpAVkLHe-6EfgZsUBfnQ2BXIYu4M,99
-watchmen_collector_kernel/model/change_data_record.py,sha256=vadzugcX4nl6G3F-jt7DtuVqjE8vtlasu86DDjpaGGM,330
+watchmen_collector_kernel/model/change_data_record.py,sha256=Vfy-wpy-TaXwOFrilq3pbZ1AqldBpT1apMkKnTaVl7M,352
 watchmen_collector_kernel/model/change_data_record_history.py,sha256=87XLw51oT8TXDRbQLWKy4U1Tyn3KtM22G5wPlMSYCv8,106
-watchmen_collector_kernel/model/collector_model_config.py,sha256=oE0T9I_9uGppttoytvzl7-35WgwlWu7s6awNtsK4Q20,288
-watchmen_collector_kernel/model/collector_table_config.py,sha256=gW7B26LdJt7jXbC6nNSNpQTooeSLGSVGhdgRHFLKD3M,2907
+watchmen_collector_kernel/model/collector_model_config.py,sha256=t31ZWoe_ITeemrzEaXtHVPiuK_n-jiHG9UlpqjxPS4s,322
+watchmen_collector_kernel/model/collector_module_config.py,sha256=0knkFhBdx4gpRXMfrnNWJlmuKMxqC643TJ8-fYDr1nA,250
+watchmen_collector_kernel/model/collector_table_config.py,sha256=GEY71Ofkkzx-vAQK3xNJT2LPwIPE0Y4Hy6E5mwCtV4E,2935
 watchmen_collector_kernel/model/competitive_lock.py,sha256=fxDRuzZpsgrZjGeuADS3zRYqQioqWhg72Pr592U_Yvw,264
 watchmen_collector_kernel/model/condition.py,sha256=jGsjvb5z9XRFW0B55k-iC2_Zle4rl-A5J_6PB6EV6mA,1942
 watchmen_collector_kernel/model/scheduled_task.py,sha256=-HBwZVbCkuZaTedixeH0s3z-d09nXSlERzQBPfNGfsY,1362
 watchmen_collector_kernel/model/scheduled_task_history.py,sha256=0lgWIQWN2r_HcMoLGwKxzLmaxGNtr0NZF7NaT_DeOyE,93
 watchmen_collector_kernel/model/trigger_event.py,sha256=ZjVgWQS_qqILchw4fGFzxn1tJUMx77s2_UpyiBd7azA,251
-watchmen_collector_kernel/model/trigger_model.py,sha256=NpnIbdax8vQeGYeGcOMpWpjYUhkeZUil--6-yTNpv3k,175
-watchmen_collector_kernel/model/trigger_table.py,sha256=aPYJlIb-8aX_-O6vZQp3DU58zYtyosYLGFjpL81ijds,229
+watchmen_collector_kernel/model/trigger_model.py,sha256=LGhtT8xmeDGB1UivAiaQ9FaHYYkaIjKztRUIuyHGBes,213
+watchmen_collector_kernel/model/trigger_module.py,sha256=LVhhS-tD5Gl7GLxKDmadSYJxkSJYwlKrlnl93h4-yZQ,193
+watchmen_collector_kernel/model/trigger_table.py,sha256=5ARZAJ9W5qTnvlH0Zsx4NZ8bYUqWauiZxY63M9-JTYk,251
 watchmen_collector_kernel/service/__init__.py,sha256=hsg-38u7V3QizNmmuEtOw8gC7QEOopVInLbY97MiwuM,426
 watchmen_collector_kernel/service/criteria_builder.py,sha256=NAk0yQBn4KARttqEOwWozxIlqkVuGPTQ8n3Zk6gjwcQ,2360
 watchmen_collector_kernel/service/data_capture.py,sha256=45asoQQlv016NKsPoiUVgaSCOJzUBHDCmWr7XPt-8VI,2873
-watchmen_collector_kernel/service/extract_source.py,sha256=7B6DTxD6iYlBiaPYGgQ-W14Ty_4YcPbO97w1kF-vunE,6560
+watchmen_collector_kernel/service/extract_source.py,sha256=8Yy3J2Cb62jskWEXewyeu4TL1luu-MzHaOIrCSQdMv0,7622
 watchmen_collector_kernel/service/extract_utils.py,sha256=PIyxFG7U37ZXLYlOdIK-Dh-kBptXPFqk-9-b1LuPL1A,2452
 watchmen_collector_kernel/service/lock_clean.py,sha256=-TVsJjr61MtgBU67S9hBrcH4Mb7xFEG61njq9GA5obY,1142
-watchmen_collector_kernel/service/lock_helper.py,sha256=GxqjIsCD-jn5kDICVWS1LlTFCuMpnRSpcrQtPLv5bxs,842
+watchmen_collector_kernel/service/lock_helper.py,sha256=SJEbaj2-qKNYcu0oboEw1SSRYnqsE_l2CypFkQNL5EE,901
 watchmen_collector_kernel/service/task_housekeeping.py,sha256=i1lnZ4jo-RLjLQD0Z7HSLQqjXuSqdL3zqHDfFVaNbe8,1673
-watchmen_collector_kernel/service/task_service.py,sha256=7YhuM1xiQhXEROiFezidyA8UaAVREingdzZhlydwfg0,3821
-watchmen_collector_kernel/service/trigger_collector.py,sha256=IP_ILJNMhahBogtvfS1iHDOJAxKCDBDJFXTDi98R11M,6790
-watchmen_collector_kernel/storage/__init__.py,sha256=9jj793SaeAJ637JisLMXDnp_59Zs20Q4hldYzBkdgjA,1173
+watchmen_collector_kernel/service/task_service.py,sha256=pcJODNYAjxesuymSgedDuPSMaDxzpee-KcnpWk4ktHI,3779
+watchmen_collector_kernel/service/trigger_collector.py,sha256=Aq7YCuAkJ2qZixCYjhmG0FWFtm2zng7g6QTmX1Oii10,10348
+watchmen_collector_kernel/storage/__init__.py,sha256=064rAoXd3IkhPL6aVisW4BGYImKItD6h3btXsHkrBZo,1369
 watchmen_collector_kernel/storage/change_data_json_history_service.py,sha256=8WPdazvG6EPTOtmiAXtb1yAMRwUaBmD-C6S0ZwsCMrw,1774
-watchmen_collector_kernel/storage/change_data_json_service.py,sha256=v_NslDdCRgKjCqIOgcQwD7hA-Ea14NOB4Zkr_g3zqBo,8464
+watchmen_collector_kernel/storage/change_data_json_service.py,sha256=ERGIKs8j71shpmYnvvie-Hsf8F9ZEX0MF8uy9aGkeDI,9102
 watchmen_collector_kernel/storage/change_data_record_history_service.py,sha256=q_41H2AQGLf_XD2W6ZsATLWvxpdK50swOWt5IMCTWjg,1845
-watchmen_collector_kernel/storage/change_data_record_service.py,sha256=QUwqn-lTIf0dYWfEKEJXy16nY-Iz6NLnihWYWP1BoQ4,7953
-watchmen_collector_kernel/storage/collector_model_config_service.py,sha256=kWLjmppPpRuN0VptiAEGA38BJ20fFD5Q5gxPZJRbQYs,3830
+watchmen_collector_kernel/storage/change_data_record_service.py,sha256=trmvO8KwuECC5nFKBLMPX_LE_gtbdXGV5JWLcWz2Lmw,8621
+watchmen_collector_kernel/storage/collector_model_config_service.py,sha256=vl04Wm-PBl-qePGhNYVAqfW6Rq513lDGwikK_tGyPOs,4245
+watchmen_collector_kernel/storage/collector_module_config_service.py,sha256=iHWYlOlzTn4iomtKu0e3SCxV1ubbdxak0bOm9tFjcBo,3711
 watchmen_collector_kernel/storage/collector_table_config_service.py,sha256=GX8zCReNd7xM07dnBEQal7NeFUgpZAWQInjdW8awmvY,5882
 watchmen_collector_kernel/storage/competitive_lock_service.py,sha256=jjxxHb4Dr0DIDX_g26wtCa8vGLGLnc4BJQGl0d3J31M,2534
 watchmen_collector_kernel/storage/scheduled_task_history_service.py,sha256=f-JXZ5G87oy7FdYPkbbfTKhI_oRm4wauWNvTw9hxGjM,1586
 watchmen_collector_kernel/storage/scheduled_task_service.py,sha256=gMv3jxAYJg_sqV4pD5WmV4LXF9Wz8-41DnbkEgwoW8o,6825
 watchmen_collector_kernel/storage/trigger_event_service.py,sha256=DKSCQ0sYSH3uAtGmokvncD9FIYvjoVsk7QtqdRMQGhM,3099
-watchmen_collector_kernel/storage/trigger_model_service.py,sha256=0-_YcL5FNGHy2MKn8vTYrWFaYOPW_u4iKQRG00AOfCI,3219
-watchmen_collector_kernel/storage/trigger_table_service.py,sha256=RHiFJaT3UQmCpz-gxnfsoBPV7lS8JM1i9DCQn3BnfA4,3779
-watchmen_collector_kernel-16.5.8.dist-info/LICENSE,sha256=iuuG7ErblOdcEZi5u89VXS0VIUiTb4flerGp_PAS9E4,1061
-watchmen_collector_kernel-16.5.8.dist-info/METADATA,sha256=jmiycosMKKac6fzPoV3KM51tSeFHbiMAfpp4paDhMQ4,1174
-watchmen_collector_kernel-16.5.8.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-watchmen_collector_kernel-16.5.8.dist-info/RECORD,,
+watchmen_collector_kernel/storage/trigger_model_service.py,sha256=FE_88sYzh9ibUS1bOyDmGvryfPQraqkMbZc3BKtzfpI,3760
+watchmen_collector_kernel/storage/trigger_module_service.py,sha256=Ca4J8vnBzS90qP3ljwDJ7lt6MzaOushGjoFRwsSKfHI,3377
+watchmen_collector_kernel/storage/trigger_table_service.py,sha256=sQXHyszSqUxYcstnc59r7fTIrGknyDsX3qivDOymZ4A,3876
+watchmen_collector_kernel-16.5.9.dist-info/LICENSE,sha256=iuuG7ErblOdcEZi5u89VXS0VIUiTb4flerGp_PAS9E4,1061
+watchmen_collector_kernel-16.5.9.dist-info/METADATA,sha256=ThU7GXuAHSAKbbxo0qujl1g5fO1r6kJQzq0PTspSwMI,1174
+watchmen_collector_kernel-16.5.9.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+watchmen_collector_kernel-16.5.9.dist-info/RECORD,,