watchmen-collector-kernel 17.0.3__py3-none-any.whl → 17.1.0__py3-none-any.whl

watchmen_collector_kernel/cache/collector_cache_manger.py

@@ -28,10 +28,6 @@ class TableConfigsByTenantAndParentNameCache(Cache):
 	pass
 
 
-class TableConfigByTenantAndTableNameCache(Cache):
-	pass
-
-
 class CollectorTopicByIdCache(Cache):
 	pass
 
@@ -60,10 +56,6 @@ configure_cache('TABLE_CONFIGS_BY_TENANT_AND_PARENT_NAME',
                 {'cache_class': TableConfigsByTenantAndParentNameCache, 'maxsize': 512})
 
 
-configure_cache('TABLE_CONFIG_BY_TENANT_AND_TABLE_NAME',
-                {'cache_class': TableConfigByTenantAndTableNameCache, 'maxsize': 512})
-
-
 configure_cache('COLLECTOR_TOPIC_BY_ID',
                 {'cache_class': CollectorTopicByIdCache, 'maxsize': 512})
 
@@ -84,10 +76,6 @@ def get_table_configs_by_tenant_and_parent_name_cache() -> TableConfigsByTenantA
 	return cache_set['TABLE_CONFIGS_BY_TENANT_AND_PARENT_NAME']
 
 
-def get_table_config_by_tenant_and_table_name_cache() -> TableConfigByTenantAndTableNameCache:
-	return cache_set['TABLE_CONFIG_BY_TENANT_AND_TABLE_NAME']
-
-
 def get_collector_topic_by_id_cache() -> CollectorTopicByIdCache:
 	return cache_set['COLLECTOR_TOPIC_BY_ID']
 

watchmen_collector_kernel/cache/table_config_cache.py

@@ -6,8 +6,7 @@ from watchmen_collector_kernel.model import CollectorTableConfig
 
 from watchmen_data_kernel.cache import InternalCache
 
-from .collector_cache_manger import get_table_config_by_tenant_and_name_cache, \
-	get_table_config_by_tenant_and_table_name_cache, get_table_configs_by_tenant_and_parent_name_cache
+from .collector_cache_manger import get_table_config_by_tenant_and_name_cache, get_table_configs_by_tenant_and_parent_name_cache
 from watchmen_collector_kernel.common import ask_collector_config_cache_enabled
 
 
@@ -16,7 +15,6 @@ class TableConfigCache:
 	def __init__(self):
 		self.ByTenantAndNameCache = InternalCache(cache=get_table_config_by_tenant_and_name_cache)
 		self.ByTenantAndParentNameCache = InternalCache(cache=get_table_configs_by_tenant_and_parent_name_cache)
-		self.ByTenantAndTableNameCache = InternalCache(cache=get_table_config_by_tenant_and_table_name_cache)
 
 	# noinspection PyMethodMayBeStatic
 	def to_tenant_and_name_key(self, name: str, tenant_id: TenantId) -> str:
@@ -44,30 +42,6 @@ class TableConfigCache:
 		else:
 			return None
 
-	def put_config_by_table_name(self, table_config: CollectorTableConfig) -> Optional[CollectorTableConfig]:
-		if ask_collector_config_cache_enabled():
-			self.ByTenantAndTableNameCache.remove(table_config.tableName)
-			existing_config = self.ByTenantAndTableNameCache.put(self.to_tenant_and_name_key(table_config.tableName, table_config.tenantId),
-			                                                     table_config)
-			return existing_config
-		else:
-			return None
-
-	def get_config_by_table_name(self, table_name: str, tenant_id: str) -> Optional[CollectorTableConfig]:
-		if ask_collector_config_cache_enabled():
-			return self.ByTenantAndTableNameCache.get(self.to_tenant_and_name_key(table_name, tenant_id))
-		else:
-			return None
-
-	def remove_config_by_table_name(self, table_name: str, tenant_id: str) -> Optional[CollectorTableConfig]:
-		if ask_collector_config_cache_enabled():
-			existing: Optional[CollectorTableConfig] = self.ByTenantAndTableNameCache.remove(
-				self.to_tenant_and_name_key(table_name, tenant_id)
-			)
-			return existing
-		else:
-			return None
-
 	def put_configs_by_parent_name(self, parent_name: str, tenant_id: str,
 	                               configs: List[CollectorTableConfig]) -> Optional[List[CollectorTableConfig]]:
 		if ask_collector_config_cache_enabled():
@@ -101,7 +75,6 @@ class TableConfigCache:
 
 	def clear(self) -> None:
 		self.ByTenantAndNameCache.clear()
-		self.ByTenantAndTableNameCache.clear()
 		self.ByTenantAndParentNameCache.clear()
 
 

watchmen_collector_kernel/common/settings.py

@@ -1,30 +1,24 @@
-from pydantic import BaseSettings
+from watchmen_utilities import ExtendedBaseSettings
 from logging import getLogger
 
 logger = getLogger(__name__)
 
 
-class CollectorSettings(BaseSettings):
+class CollectorSettings(ExtendedBaseSettings):
 	CLEAN_OF_TIMEOUT_INTERVAL: int = 300
 	LOCK_TIMEOUT: int = 1800
-	TRIGGER_EVENT_LOCK_TIMEOUT = 300
-	EXTRACT_TABLE_LOCK_TIMEOUT = 7200
-	CLEAN_UP_LOCK_TIMEOUT = 300
+	TRIGGER_EVENT_LOCK_TIMEOUT: int = 300
+	EXTRACT_TABLE_LOCK_TIMEOUT: int = 7200
+	CLEAN_UP_LOCK_TIMEOUT: int = 300
 	COLLECTOR_TIMEOUT: int = 600
 	COLLECTOR_TASK_TIMEOUT: int = 900
-	S3_CONNECTOR_LOCK_TIMEOUT = 300
+	S3_CONNECTOR_LOCK_TIMEOUT: int = 300
 	PARTIAL_SIZE: int = 100
 	COLLECTOR_CONFIG_CACHE_ENABLED: bool = True
-	EXCEPTION_MAX_LENGTH = 5000  # character
-	GROUPED_TASK_DATA_SIZE_THRESHOLD = 100
+	EXCEPTION_MAX_LENGTH: int = 5000  # character
+	GROUPED_TASK_DATA_SIZE_THRESHOLD: int = 100
 	TASK_PARTIAL_SIZE: int = 100
 
-	class Config:
-		# secrets_dir = '/var/run'
-		env_file = '.env'
-		env_file_encoding = 'utf-8'
-		case_sensitive = True
-
 
 collector_settings = CollectorSettings()
 logger.info(f'Collector settings[{collector_settings.dict()}].')

watchmen_collector_kernel/model/__init__.py

@@ -5,7 +5,7 @@ from .collector_module_config import CollectorModuleConfig
 from .collector_model_config import CollectorModelConfig
 from .collector_table_config import CollectorTableConfig
 
-from .trigger_event import TriggerEvent, EventType
+from .trigger_event import TriggerEvent, EventType, QueryParam
 from .trigger_module import TriggerModule
 from .trigger_model import TriggerModel
 from .trigger_table import TriggerTable

watchmen_collector_kernel/model/change_data_json.py

@@ -1,30 +1,30 @@
-from typing import Dict, List
+from typing import Dict, List, Optional
 
-from pydantic import BaseModel
+from watchmen_utilities import ExtendedBaseModel
 
 from watchmen_model.common import TenantBasedTuple, Storable
 
 
-class Dependence(Storable, BaseModel):
-	modelName: str
-	objectId: str
+class Dependence(Storable, ExtendedBaseModel):
+	modelName: Optional[str] = None
+	objectId: Optional[str] = None
 
 
-class ChangeDataJson(TenantBasedTuple, BaseModel):
-	changeJsonId: int
-	resourceId: str
-	modelName: str
-	objectId: str
-	sequence: int
-	content: Dict
-	dataId: Dict
-	dependOn: List[Dependence]
-	isPosted: bool
-	status: int
-	result: Dict
-	taskId: int
-	tableTriggerId: int
-	modelTriggerId: int
-	moduleTriggerId: int
-	eventTriggerId: int
+class ChangeDataJson(TenantBasedTuple, ExtendedBaseModel):
+	changeJsonId: Optional[int] = None
+	resourceId: Optional[str] = None
+	modelName: Optional[str] = None
+	objectId: Optional[str] = None
+	sequence: Optional[int] = None
+	content: Optional[Dict] = None
+	dataId: Optional[Dict] = None
+	dependOn: Optional[List[Dependence]] = None
+	isPosted: Optional[bool] = None
+	status: Optional[int] = None
+	result: Optional[Dict] = None
+	taskId: Optional[int] = None
+	tableTriggerId: Optional[int] = None
+	modelTriggerId: Optional[int] = None
+	moduleTriggerId: Optional[int] = None
+	eventTriggerId: Optional[int] = None
 

watchmen_collector_kernel/model/collector_model_config.py

@@ -1,14 +1,14 @@
-from typing import List
+from typing import List, Optional
 from watchmen_model.common import TenantBasedTuple, OptimisticLock
-from pydantic import BaseModel
+from watchmen_utilities import ExtendedBaseModel
 
 
-class CollectorModelConfig(TenantBasedTuple, OptimisticLock, BaseModel):
-	modelId: str
-	modelName: str
-	moduleId: str
-	dependOn: List[str]
+class CollectorModelConfig(TenantBasedTuple, OptimisticLock, ExtendedBaseModel):
+	modelId: Optional[str] = None
+	modelName: Optional[str] = None
+	moduleId: Optional[str] = None
+	dependOn: Optional[List[str]] = None
 	priority: int = 0
-	rawTopicCode: str
-	isParalleled: bool
+	rawTopicCode: Optional[str] = None
+	isParalleled: Optional[bool] = None
 

watchmen_collector_kernel/model/collector_module_config.py

@@ -1,10 +1,10 @@
-from typing import List
+from typing import List, Optional
 from watchmen_model.common import TenantBasedTuple, OptimisticLock
-from pydantic import BaseModel
+from watchmen_utilities import ExtendedBaseModel
 
 
-class CollectorModuleConfig(TenantBasedTuple, OptimisticLock, BaseModel):
-	moduleId: str
-	moduleName: str
+class CollectorModuleConfig(TenantBasedTuple, OptimisticLock, ExtendedBaseModel):
+	moduleId: Optional[str] = None
+	moduleName: Optional[str] = None
 	priority: int = 0
 

watchmen_collector_kernel/model/collector_table_config.py

@@ -1,15 +1,12 @@
 from typing import Dict, List, Optional, Union
-
-from pydantic import BaseModel
-
 from .condition import Condition, construct_conditions, construct_condition
 from watchmen_model.common import TenantBasedTuple, Storable, OptimisticLock
-from watchmen_utilities import ArrayHelper
+from watchmen_utilities import ArrayHelper, ExtendedBaseModel
 
 
-class JoinCondition(Storable, BaseModel):
-	parentKey: Condition = None
-	childKey: Condition = None
+class JoinCondition(Storable, ExtendedBaseModel):
+	parentKey: Optional[Condition] = None
+	childKey: Optional[Condition] = None
 
 	def __setattr__(self, name, value):
 		if name == 'parentKey':
@@ -18,17 +15,17 @@ class JoinCondition(Storable, BaseModel):
 			super().__setattr__(name, construct_condition(value))
 
 
-class Dependence(Storable, BaseModel):
-	modelName: str
-	objectKey: str  # the dependent column
+class Dependence(Storable, ExtendedBaseModel):
+	modelName: Optional[str] = None
+	objectKey: Optional[str] = None  # the dependent column
 
 
-class JsonColumn(Storable, BaseModel):
-	columnName: str = None
-	ignoredPath: List[str] = None
-	needFlatten: bool = None
-	flattenPath: List[str] = None
-	jsonPath: List[str] = None
+class JsonColumn(Storable, ExtendedBaseModel):
+	columnName: Optional[str] = None
+	ignoredPath: Optional[List[str]] = None
+	needFlatten: Optional[bool] = None
+	flattenPath: Optional[List[str]] = None
+	jsonPath: Optional[List[str]] = None
 
 
 def construct_json_column(json_column: Union[JsonColumn, Dict]) -> Optional[JsonColumn]:
@@ -77,23 +74,23 @@ def construct_depend_on(depend_on: Optional[List[Union[Dependence, Dict]]]) -> O
 		return ArrayHelper(depend_on).map(lambda x: construct_dependence(x)).to_list()
 
 
-class CollectorTableConfig(TenantBasedTuple, OptimisticLock, BaseModel):
-	configId: str = None
-	name: str = None
-	tableName: str = None
-	primaryKey: List[str] = None
-	objectKey: str = None
-	sequenceKey: str = None
-	modelName: str = None
-	parentName: str = None
-	label: str = None
-	joinKeys: List[JoinCondition] = None
-	dependOn: List[Dependence] = []
-	auditColumn: str = None
-	ignoredColumns: List[str] = None
-	jsonColumns: List[JsonColumn] = None
-	conditions: List[Condition] = []
-	dataSourceId: str = None
+class CollectorTableConfig(TenantBasedTuple, OptimisticLock, ExtendedBaseModel):
+	configId: Optional[str] = None
+	name: Optional[str] = None
+	tableName: Optional[str] = None
+	primaryKey: Optional[List[str]] = None
+	objectKey: Optional[str] = None
+	sequenceKey: Optional[str] = None
+	modelName: Optional[str] = None
+	parentName: Optional[str] = None
+	label: Optional[str] = None
+	joinKeys: Optional[List[JoinCondition]] = None
+	dependOn: Optional[List[Dependence]] = []
+	auditColumn: Optional[str] = None
+	ignoredColumns: Optional[List[str]] = None
+	jsonColumns: Optional[List[JsonColumn]] = None
+	conditions: Optional[List[Condition]] = []
+	dataSourceId: Optional[str] = None
 	isList: bool = False
 	triggered: bool = False
 

watchmen_collector_kernel/model/condition.py

@@ -4,14 +4,12 @@ from datetime import date, datetime
 from enum import Enum
 from typing import Optional, List, Union, Dict
 
-from pydantic import BaseModel
 from watchmen_storage import EntityCriteriaOperator
-
 from watchmen_model.common import DataModel
-from watchmen_utilities import ArrayHelper
+from watchmen_utilities import ArrayHelper, ExtendedBaseModel
 
 
-class Condition(DataModel, BaseModel):
+class Condition(DataModel, ExtendedBaseModel):
 	pass
 
 
@@ -28,7 +26,7 @@ class ConditionJointConjunction(str, Enum):
 
 class ConditionJoint(Condition):
 	conjunction: ConditionJointConjunction = ConditionJointConjunction.AND
-	children: List[Condition]
+	children: Optional[List[Condition]] = None
 
 	def __setattr__(self, name, value):
 		if name == 'children':

watchmen_collector_kernel/model/scheduled_task.py

@@ -1,9 +1,8 @@
 from typing import Dict, List, Optional, Union
 
-from watchmen_utilities import ArrayHelper
-
+from watchmen_utilities import ArrayHelper, ExtendedBaseModel
 from watchmen_model.common import TenantBasedTuple, ScheduledTaskId, Storable
-from pydantic import BaseModel
+
 from enum import Enum
 
 
@@ -19,7 +18,7 @@ class Result(str, Enum):
 	PROCESS_TASK_FAILED = "PROCESS_TASK_FAILED"
 
 
-class Dependence(Storable, BaseModel):
+class Dependence(Storable, ExtendedBaseModel):
 	modelName: str
 	objectId: str
 
@@ -40,23 +39,23 @@ def construct_depend_on(depend_on: Optional[List[Union[Dependence, Dict]]]) -> O
 		return ArrayHelper(depend_on).map(lambda x: construct_dependence(x)).to_list()
 
 
-class ScheduledTask(TenantBasedTuple, BaseModel):
-	taskId: ScheduledTaskId
-	resourceId: str  # global unique, monotonous increase
-	topicCode: str
-	content: Dict
-	changeJsonIds: List[int]
-	modelName: str
-	objectId: str
-	dependOn: List[Dependence]
-	parentTaskId: List[int]
-	isFinished: bool
-	status: int
-	result: Dict
-	eventId: str  # Deprecated
-	eventTriggerId: int
-	pipelineId: str
-	type: int
+class ScheduledTask(TenantBasedTuple, ExtendedBaseModel):
+	taskId: Optional[ScheduledTaskId] = None
+	resourceId:  Optional[str] = None  # global unique, monotonous increase
+	topicCode:  Optional[str] = None
+	content:  Optional[Dict] = None
+	changeJsonIds:  Optional[List[int]] = None
+	modelName:  Optional[str] = None
+	objectId:  Optional[str] = None
+	dependOn:  Optional[List[Dependence]] = None
+	parentTaskId:  Optional[List[int]] = None
+	isFinished:  Optional[bool] = None
+	status:  Optional[int] = None
+	result:  Optional[Dict] = None
+	eventId:  Optional[str] = None  # Deprecated
+	eventTriggerId:  Optional[int] = None
+	pipelineId:  Optional[str] = None
+	type:  Optional[int] = None
 
 	def __setattr__(self, name, value):
 		if name == 'dependOn':

watchmen_collector_kernel/model/trigger_event.py

@@ -1,9 +1,10 @@
 from datetime import datetime
 from enum import Enum
-from typing import List, Dict
+from typing import List, Dict, Optional, Union
 
-from watchmen_model.common import TenantBasedTuple
-from pydantic import BaseModel
+from .condition import Condition, construct_conditions
+from watchmen_model.common import TenantBasedTuple, DataModel
+from watchmen_utilities import ExtendedBaseModel, ArrayHelper
 
 
 class EventType(int, Enum):
@@ -13,14 +14,47 @@ class EventType(int, Enum):
 	BY_PIPELINE = 4
 
 
-class TriggerEvent(TenantBasedTuple, BaseModel):
-	eventTriggerId: int
-	startTime: datetime
-	endTime: datetime
+class QueryParam(DataModel, ExtendedBaseModel):
+	name: str
+	filter: List[Condition]
+
+	def __setattr__(self, name, value):
+		if name == 'filter':
+			super().__setattr__(name, construct_conditions(value))
+		else:
+			super().__setattr__(name, value)
+
+
+class TriggerEvent(TenantBasedTuple, ExtendedBaseModel):
+	eventTriggerId: Optional[int] = None
+	startTime: Optional[datetime] = None
+	endTime: Optional[datetime] = None
 	isFinished: bool = False
-	status: int
-	type: int
-	tableName: str
-	records: List[Dict] = []
-	pipelineId: str
+	status: Optional[int] = None
+	type: Optional[int] = None
+	tableName: Optional[str] = None
+	records: Optional[List[Dict]] = []
+	pipelineId: Optional[str] = None
+	params: Optional[List[QueryParam]] = None
+
+	def __setattr__(self, name, value):
+		if name == 'params':
+			super().__setattr__(name, construct_params(value))
+		else:
+			super().__setattr__(name, value)
+
+
+def construct_params(params: Optional[List[Union[dict, QueryParam]]]) -> Optional[List[QueryParam]]:
+	if params is None:
+		return None
+	else:
+		return ArrayHelper(params).map(lambda x: construct_param(x)).to_list()
+
 
+def construct_param(param: Optional[Union[dict, QueryParam]]) -> Optional[QueryParam]:
+	if param is None:
+		return None
+	elif isinstance(param, QueryParam):
+		return param
+	else:
+		return QueryParam(**param)

watchmen_collector_kernel/service/table_config_service.py

@@ -18,24 +18,6 @@ class TableConfigService:
 	def __init__(self, principal_service: PrincipalService):
 		self.principalService = principal_service
 
-	def find_by_table_name(self, table_name: str, tenant_id: str) -> Optional[CollectorTableConfig]:
-		config = CollectorCacheService.table_config().get_config_by_table_name(table_name, tenant_id)
-		if config is not None:
-			return config
-
-		storage_service = get_collector_table_config_service(
-			ask_meta_storage(), ask_snowflake_generator(), self.principalService
-		)
-		table_config: CollectorTableConfig = storage_service.find_by_table_name(table_name, tenant_id)
-		if table_config is None:
-			table_config = self.find_by_topic_name(table_name, tenant_id)
-
-		if table_config is None:
-			return None
-
-		CollectorCacheService.table_config().put_config_by_table_name(table_config)
-		return table_config
-
 	def find_by_name(self, name: str, tenant_id: str) -> Optional[CollectorTableConfig]:
 		config = CollectorCacheService.table_config().get_config_by_name(name, tenant_id)
 		if config is not None:
@@ -45,6 +27,9 @@ class TableConfigService:
 			ask_meta_storage(), ask_snowflake_generator(), self.principalService
 		)
 		table_config: CollectorTableConfig = storage_service.find_by_name(name, tenant_id)
+		if table_config is None:
+			table_config = self.find_by_topic_name(name, tenant_id)
+
 		if table_config is None:
 			return None
 
@@ -77,15 +62,19 @@ class TableConfigService:
 	def get_topic_name_by_table_name(self, table_name: str) -> str:
 		return table_name.removeprefix("topic_")
 
-	def find_by_topic_name(self, table_name: str, tenant_id: str) -> Optional[CollectorTableConfig]:
-		topic_name = self.get_topic_name_by_table_name(table_name)
+	# noinspection PyMethodMayBeStatic
+	def get_table_name_by_topic_name(self, topic_name: str) -> str:
+		return "topic_" + topic_name
+
+	def find_by_topic_name(self, name: str, tenant_id: str) -> Optional[CollectorTableConfig]:
+		topic_name = name
 		schema = TopicService(self.principalService).find_schema_by_name(topic_name, tenant_id)
 		now = get_current_time_in_seconds()
 		if schema:
 			return CollectorTableConfig(
 				configId=self.fake_config_id(schema.topic.topicId),
 				name=topic_name,
-				tableName=table_name,
+				tableName=self.get_table_name_by_topic_name(name),
 				primaryKey=["id_"],
 				objectKey="id_",
 				sequenceKey=None,

watchmen_collector_kernel/service/trigger_collector.py

@@ -132,7 +132,7 @@ def new_trigger_table(table_name: str,
 
 
 def get_trigger_table(model_trigger: TriggerModel, table_config: CollectorTableConfig) -> TriggerTable:
-	return new_trigger_table(table_config.tableName,
+	return new_trigger_table(table_config.name,
 	                         table_config.modelName,
 	                         model_trigger.modelTriggerId,
 	                         model_trigger.moduleTriggerId,

watchmen_collector_kernel/service/trigger_event_helper.py

@@ -86,8 +86,7 @@ def trigger_event_by_table(trigger_event: TriggerEvent):
 	                                                          trigger_event_service.snowflakeGenerator,
 	                                                          trigger_event_service.principalService)
 
-	table_config = table_config_service.find_by_table_name_and_tenant_id(trigger_event.tableName,
-	                                                                     trigger_event.tenantId)
+	table_config = table_config_service.find_by_name(trigger_event.tableName, trigger_event.tenantId)
 	model_config = model_config_service.find_by_name(table_config.modelName, table_config.tenantId)
 
 	module_config = module_config_service.find_by_module_id(model_config.moduleId)
@@ -140,8 +139,7 @@ def trigger_event_by_records(trigger_event: TriggerEvent):
 	                                                          trigger_event_service.snowflakeGenerator,
 	                                                          trigger_event_service.principalService)
 
-	table_config = table_config_service.find_by_table_name_and_tenant_id(trigger_event.tableName,
-	                                                                     trigger_event.tenantId)
+	table_config = table_config_service.find_by_name(trigger_event.tableName, trigger_event.tenantId)
 	model_config = model_config_service.find_by_name(table_config.modelName, table_config.tenantId)
 	module_config = module_config_service.find_by_module_id(model_config.moduleId)
 
@@ -276,7 +274,7 @@ def trigger_event_by_pipeline(trigger_event: TriggerEvent):
 	model_config_service = get_model_config_service(principal_service)
 	table_config_service = get_table_config_service(principal_service)
 
-	table_config = table_config_service.find_by_table_name(trigger_event.tableName, trigger_event.tenantId)
+	table_config = table_config_service.find_by_name(trigger_event.tableName, trigger_event.tenantId)
 	model_config = model_config_service.find_by_name(table_config.modelName, trigger_event.tenantId)
 	module_config = module_config_service.find_by_module_id(model_config.moduleId, trigger_event.tenantId)
 

watchmen_collector_kernel/storage/trigger_event_service.py

@@ -1,17 +1,31 @@
 from typing import List, Dict, Any, Optional
 
 from watchmen_auth import PrincipalService
-from watchmen_collector_kernel.model import TriggerEvent
+from watchmen_collector_kernel.model import TriggerEvent, QueryParam
 from watchmen_meta.common import TupleShaper, TupleService
 from watchmen_meta.common.storage_service import StorableId
 from watchmen_model.common import Storable, EventTriggerId
 from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
 	EntityStraightValuesFinder, EntityCriteriaExpression, ColumnNameLiteral, \
 	EntityStraightColumn, EntitySortColumn, EntitySortMethod
+from watchmen_utilities import ArrayHelper
 
 
 class TriggerEventShaper(EntityShaper):
 
+	@staticmethod
+	def serialize_param(param: Optional[QueryParam]) -> dict:
+		if isinstance(param, dict):
+			return param
+		else:
+			return param.to_dict()
+
+	@staticmethod
+	def serialize_params(params: Optional[List[QueryParam]]) -> Optional[list]:
+		if params is None:
+			return None
+		return ArrayHelper(params).map(lambda x: TriggerEventShaper.serialize_param(x)).to_list()
+
 	def serialize(self, entity: TriggerEvent) -> EntityRow:
 		return TupleShaper.serialize_tenant_based(entity, {
 			'event_trigger_id': entity.eventTriggerId,
@@ -22,7 +36,8 @@ class TriggerEventShaper(EntityShaper):
 			'type': entity.type,
 			'table_name': entity.tableName,
 			'records': entity.records,
-			'pipeline_id': entity.pipelineId
+			'pipeline_id': entity.pipelineId,
+			'params': TriggerEventShaper.serialize_params(entity.params)
 		})
 
 	def deserialize(self, row: EntityRow) -> TriggerEvent:
@@ -36,7 +51,8 @@ class TriggerEventShaper(EntityShaper):
 			type=row.get('type'),
 			tableName=row.get('table_name'),
 			records=row.get('records'),
-			pipelineId=row.get('pipeline_id')
+			pipelineId=row.get('pipeline_id'),
+			params=row.get('params')
 		))
 
 

{watchmen_collector_kernel-17.0.3.dist-info → watchmen_collector_kernel-17.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: watchmen-collector-kernel
-Version: 17.0.3
+Version: 17.1.0
 Summary: 
 License: MIT
 Author: botlikes
@@ -17,11 +17,11 @@ Provides-Extra: oss
 Provides-Extra: postgresql
 Provides-Extra: s3
 Requires-Dist: numpy (>=1.26.4,<2.0.0)
-Requires-Dist: watchmen-data-kernel (==17.0.3)
-Requires-Dist: watchmen-storage-mongodb (==17.0.3) ; extra == "mongodb"
-Requires-Dist: watchmen-storage-mssql (==17.0.3) ; extra == "mssql"
-Requires-Dist: watchmen-storage-mysql (==17.0.3) ; extra == "mysql"
-Requires-Dist: watchmen-storage-oracle (==17.0.3) ; extra == "oracle"
-Requires-Dist: watchmen-storage-oss (==17.0.3) ; extra == "oss"
-Requires-Dist: watchmen-storage-postgresql (==17.0.3) ; extra == "postgresql"
-Requires-Dist: watchmen-storage-s3 (==17.0.3) ; extra == "s3"
+Requires-Dist: watchmen-data-kernel (==17.1.0)
+Requires-Dist: watchmen-storage-mongodb (==17.1.0) ; extra == "mongodb"
+Requires-Dist: watchmen-storage-mssql (==17.1.0) ; extra == "mssql"
+Requires-Dist: watchmen-storage-mysql (==17.1.0) ; extra == "mysql"
+Requires-Dist: watchmen-storage-oracle (==17.1.0) ; extra == "oracle"
+Requires-Dist: watchmen-storage-oss (==17.1.0) ; extra == "oss"
+Requires-Dist: watchmen-storage-postgresql (==17.1.0) ; extra == "postgresql"
+Requires-Dist: watchmen-storage-s3 (==17.1.0) ; extra == "s3"

{watchmen_collector_kernel-17.0.3.dist-info → watchmen_collector_kernel-17.1.0.dist-info}/RECORD

@@ -1,29 +1,29 @@
 watchmen_collector_kernel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 watchmen_collector_kernel/cache/__init__.py,sha256=ypQLQQtAanUybsWuwBpBjdD2yMC-c2Zub4g8zxj2CrY,59
-watchmen_collector_kernel/cache/collector_cache_manger.py,sha256=VtJnDkEDnCxvsQbZ0Wpkmqss5bCRuV_wmPBwaYEjbO8,2381
+watchmen_collector_kernel/cache/collector_cache_manger.py,sha256=FjK2hpN8g5kA4GQFzcRCt1msIYiTgNwFsRvIdVnrURM,2020
 watchmen_collector_kernel/cache/collector_cache_service.py,sha256=Zktnbkj5KtGzYiQo3_tNldYr24Ir6vyhGm4Cgc_Gz9k,905
 watchmen_collector_kernel/cache/collector_topic_cache.py,sha256=jpr6-T4ETH5Oi7RIrGm9jWdfaNbSCYdz8bgG6Sap4Lw,1191
 watchmen_collector_kernel/cache/model_config_cache.py,sha256=WW2PmILHm0WbC9gr73uIq1B-poGx78Xpf1w9WEZLbJ8,1906
 watchmen_collector_kernel/cache/module_config_cache.py,sha256=lGx-eIZLcLTa1RiH_kGU8soU1ilcfbE6KrYixim4Uhw,1393
-watchmen_collector_kernel/cache/table_config_cache.py,sha256=oOjyatujD-J6hFcj42hxdmMhZylRdj8v5UXypTcqJQE,4390
+watchmen_collector_kernel/cache/table_config_cache.py,sha256=dM0dU7_3NpqmShiILiaU3EYaBBfDDoWFwfr3OR7Tpzo,3120
 watchmen_collector_kernel/common/__init__.py,sha256=KCifKsnn6NPHkUMSq2q__fL9aWg52eK35a_mIfLKYRM,627
 watchmen_collector_kernel/common/constants.py,sha256=mBc0dVjMXKp2pluN2HMNcArcOWvycNzLNu7zcZO0Eao,320
 watchmen_collector_kernel/common/exception.py,sha256=AY4JAL0fBP6fAkgT7bfMTABp3x3oYtvV5vdPO5jI6S4,49
-watchmen_collector_kernel/common/settings.py,sha256=h5k9PDhd5QG-iDTTXLPFmVXNeaV9-eFZdiIZyfqCr_A,2012
-watchmen_collector_kernel/model/__init__.py,sha256=7FUgGxozQ3TMHBllZmds7g6nQbN2H6fID8cCLED7u04,845
-watchmen_collector_kernel/model/change_data_json.py,sha256=ckCj2vvDoZCWryYE8lGjJnZ_WEfynsVrL2RvmKA2IUk,530
+watchmen_collector_kernel/common/settings.py,sha256=ckkes4rBObImsWaUaJak6msowSsrAAtGXa8EupQtNHk,1949
+watchmen_collector_kernel/model/__init__.py,sha256=nyF4heHswdII2Gw4Kp70AhbknxjmNHRk4wyCcVcYrVU,857
+watchmen_collector_kernel/model/change_data_json.py,sha256=8i-69zBdjaEa3OAtzOpy40g-mCer2stDV_mN5I2t6Mc,880
 watchmen_collector_kernel/model/change_data_json_history.py,sha256=L9ACRe5tCP5Pc5MrpAVkLHe-6EfgZsUBfnQ2BXIYu4M,99
 watchmen_collector_kernel/model/change_data_record.py,sha256=zZe_QaYGg6JQRhjtX5dLZ5TIb6vok0G7fdtmWZvTDpo,355
 watchmen_collector_kernel/model/change_data_record_history.py,sha256=87XLw51oT8TXDRbQLWKy4U1Tyn3KtM22G5wPlMSYCv8,106
-watchmen_collector_kernel/model/collector_model_config.py,sha256=t31ZWoe_ITeemrzEaXtHVPiuK_n-jiHG9UlpqjxPS4s,322
-watchmen_collector_kernel/model/collector_module_config.py,sha256=0knkFhBdx4gpRXMfrnNWJlmuKMxqC643TJ8-fYDr1nA,250
-watchmen_collector_kernel/model/collector_table_config.py,sha256=apxZRQmLEMsVK3Z8KnAUy61LRSITDhTEs1EVNTvC-M0,3266
+watchmen_collector_kernel/model/collector_model_config.py,sha256=MtbfugwwB_In04Wgi1ndSuj9udGgdtfASaS-0lxOpaM,460
+watchmen_collector_kernel/model/collector_module_config.py,sha256=l_6OcMheFycz1o51-qdxzySCmQ_wgJRAv788EcriOew,320
+watchmen_collector_kernel/model/collector_table_config.py,sha256=eMvF_g4obnQqNh9_AUfcP9RYTxmf1dzXasDC3GdrzBg,3548
 watchmen_collector_kernel/model/competitive_lock.py,sha256=fxDRuzZpsgrZjGeuADS3zRYqQioqWhg72Pr592U_Yvw,264
-watchmen_collector_kernel/model/condition.py,sha256=I0QSsz1pP2DRKsgfjw_M7Z_seaPFfzOCvlg6Ud2Te0s,1603
-watchmen_collector_kernel/model/scheduled_task.py,sha256=xJCSzo95mXItOsoLJ0Uglh-FmWR4MFeouTLtlxDdLtw,1552
+watchmen_collector_kernel/model/condition.py,sha256=DN7AxbaOPNJ4peiMQpsBeLR_waD9cAl_FJtfzHGJ_G4,1615
+watchmen_collector_kernel/model/scheduled_task.py,sha256=mMgCmPbUZOE2YDQ1gC0dsOx6-6KFJx58LZ2L71NSEe0,1843
 watchmen_collector_kernel/model/scheduled_task_history.py,sha256=0lgWIQWN2r_HcMoLGwKxzLmaxGNtr0NZF7NaT_DeOyE,93
 watchmen_collector_kernel/model/status.py,sha256=7JgTF8SZJIKH0acwzJdFCAaW3O2g_dpH0wcnN6DNVjU,210
-watchmen_collector_kernel/model/trigger_event.py,sha256=vERU5ebEAy_2d7Au6WAEkpszou9Svb9tnk4rjMMKEOk,479
+watchmen_collector_kernel/model/trigger_event.py,sha256=zlJp3_eisMftMTE1qX9Y40aDiTpVryokGZHWsP5EO54,1607
 watchmen_collector_kernel/model/trigger_model.py,sha256=LGhtT8xmeDGB1UivAiaQ9FaHYYkaIjKztRUIuyHGBes,213
 watchmen_collector_kernel/model/trigger_module.py,sha256=LVhhS-tD5Gl7GLxKDmadSYJxkSJYwlKrlnl93h4-yZQ,193
 watchmen_collector_kernel/model/trigger_table.py,sha256=5ARZAJ9W5qTnvlH0Zsx4NZ8bYUqWauiZxY63M9-JTYk,251
@@ -36,10 +36,10 @@ watchmen_collector_kernel/service/extract_utils.py,sha256=pmeUKc9dret_mfjqbaBDJt
 watchmen_collector_kernel/service/lock_helper.py,sha256=rr37IFj80OAPdP7TBhGLdmG8hSOkm_L6r6RTmJxNNbw,925
 watchmen_collector_kernel/service/model_config_service.py,sha256=1tFd_MUldwl37JyRJF4FX6ywmFQ2caFYfslAlqnv2Jk,2235
 watchmen_collector_kernel/service/module_config_service.py,sha256=M4Mn45WEV9rllpX4A9LhtVE7VjBo-Q9LrHnR4_U2UYA,2027
-watchmen_collector_kernel/service/table_config_service.py,sha256=1jmaaOEM0ETRrTDGgGK7S9SCX85bPIP_jJ2zqsHH4DU,3962
+watchmen_collector_kernel/service/table_config_service.py,sha256=oqMKWQj2Fskp9AvLSqbOAmXOB-PfQMi4WmC48BCnsXY,3461
 watchmen_collector_kernel/service/task_service.py,sha256=7bKbRmHwyWRV6cff8V9rfVlf8EXa2DidVlocuVaHGpc,4705
-watchmen_collector_kernel/service/trigger_collector.py,sha256=p07SEQsfrBYIT02Ea5w9R70Yh7NAMC1TlFnn7hn79-o,10852
-watchmen_collector_kernel/service/trigger_event_helper.py,sha256=l01bMQrHM-bij4_S0vdzHDuERaTZ4PFo7jQQMRw7dTc,17784
+watchmen_collector_kernel/service/trigger_collector.py,sha256=bb_SIW8FMcj_zUId5zApnHIEH6fQy9vK1bNdwJE6_qA,10847
+watchmen_collector_kernel/service/trigger_event_helper.py,sha256=7HYYIDrG10plDvhYVG1YDDl33GjKkSVrwawlE0pr0aE,17598
 watchmen_collector_kernel/storage/__init__.py,sha256=064rAoXd3IkhPL6aVisW4BGYImKItD6h3btXsHkrBZo,1369
 watchmen_collector_kernel/storage/change_data_json_history_service.py,sha256=8WPdazvG6EPTOtmiAXtb1yAMRwUaBmD-C6S0ZwsCMrw,1774
 watchmen_collector_kernel/storage/change_data_json_service.py,sha256=ILEqtAs2vQzfzEtMd7w6lMkmHbf-hQa9f0GpWhcbXTo,12611
@@ -51,11 +51,11 @@ watchmen_collector_kernel/storage/collector_table_config_service.py,sha256=PcXEg
 watchmen_collector_kernel/storage/competitive_lock_service.py,sha256=SIXpoeRP1S88ZQJy_-iZgGV9HTJwwM5r3vJn7J9WJbU,3154
 watchmen_collector_kernel/storage/scheduled_task_history_service.py,sha256=f-JXZ5G87oy7FdYPkbbfTKhI_oRm4wauWNvTw9hxGjM,1586
 watchmen_collector_kernel/storage/scheduled_task_service.py,sha256=PGH8Qz8nDtUEBIqrKkItjQWSM7cFh13_fkdptdztE14,10906
-watchmen_collector_kernel/storage/trigger_event_service.py,sha256=pQwfrwt6-6qkJvorO28dTR7cGuvcmvFQUMPUfTujjWg,6836
+watchmen_collector_kernel/storage/trigger_event_service.py,sha256=8fnz2T2FB2QFa24YJYITcoSsG4AkCnl7H3u7rDC7Wcs,7361
 watchmen_collector_kernel/storage/trigger_model_service.py,sha256=FE_88sYzh9ibUS1bOyDmGvryfPQraqkMbZc3BKtzfpI,3760
 watchmen_collector_kernel/storage/trigger_module_service.py,sha256=Ca4J8vnBzS90qP3ljwDJ7lt6MzaOushGjoFRwsSKfHI,3377
 watchmen_collector_kernel/storage/trigger_table_service.py,sha256=TRAFknaX6IW8lX7dMCR4y_3yjA2bYFFH1HdwW21--Kc,3915
-watchmen_collector_kernel-17.0.3.dist-info/LICENSE,sha256=iuuG7ErblOdcEZi5u89VXS0VIUiTb4flerGp_PAS9E4,1061
-watchmen_collector_kernel-17.0.3.dist-info/METADATA,sha256=W_ARkRSuSjWkdFMXnoQX_l2J3CsSMr_sv57k-UF_ehc,1075
-watchmen_collector_kernel-17.0.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-watchmen_collector_kernel-17.0.3.dist-info/RECORD,,
+watchmen_collector_kernel-17.1.0.dist-info/LICENSE,sha256=iuuG7ErblOdcEZi5u89VXS0VIUiTb4flerGp_PAS9E4,1061
+watchmen_collector_kernel-17.1.0.dist-info/METADATA,sha256=A8eL6jRJ6yInrt8ppeYSBIEVH4oJ_aVVrORdibNXC68,1075
+watchmen_collector_kernel-17.1.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+watchmen_collector_kernel-17.1.0.dist-info/RECORD,,

{watchmen_collector_kernel-17.0.3.dist-info → watchmen_collector_kernel-17.1.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 1.9.1
 Root-Is-Purelib: true
 Tag: py3-none-any