watchmen-collector-kernel 17.0.2__tar.gz → 17.1.0__tar.gz

{watchmen_collector_kernel-17.0.2 → watchmen_collector_kernel-17.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: watchmen-collector-kernel
-Version: 17.0.2
+Version: 17.1.0
 Summary: 
 License: MIT
 Author: botlikes
@@ -17,11 +17,11 @@ Provides-Extra: oss
 Provides-Extra: postgresql
 Provides-Extra: s3
 Requires-Dist: numpy (>=1.26.4,<2.0.0)
-Requires-Dist: watchmen-data-kernel (==17.0.2)
-Requires-Dist: watchmen-storage-mongodb (==17.0.2) ; extra == "mongodb"
-Requires-Dist: watchmen-storage-mssql (==17.0.2) ; extra == "mssql"
-Requires-Dist: watchmen-storage-mysql (==17.0.2) ; extra == "mysql"
-Requires-Dist: watchmen-storage-oracle (==17.0.2) ; extra == "oracle"
-Requires-Dist: watchmen-storage-oss (==17.0.2) ; extra == "oss"
-Requires-Dist: watchmen-storage-postgresql (==17.0.2) ; extra == "postgresql"
-Requires-Dist: watchmen-storage-s3 (==17.0.2) ; extra == "s3"
+Requires-Dist: watchmen-data-kernel (==17.1.0)
+Requires-Dist: watchmen-storage-mongodb (==17.1.0) ; extra == "mongodb"
+Requires-Dist: watchmen-storage-mssql (==17.1.0) ; extra == "mssql"
+Requires-Dist: watchmen-storage-mysql (==17.1.0) ; extra == "mysql"
+Requires-Dist: watchmen-storage-oracle (==17.1.0) ; extra == "oracle"
+Requires-Dist: watchmen-storage-oss (==17.1.0) ; extra == "oss"
+Requires-Dist: watchmen-storage-postgresql (==17.1.0) ; extra == "postgresql"
+Requires-Dist: watchmen-storage-s3 (==17.1.0) ; extra == "s3"

{watchmen_collector_kernel-17.0.2 → watchmen_collector_kernel-17.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "watchmen-collector-kernel"
-version = "17.0.2"
+version = "17.1.0"
 description = ""
 authors = ["botlikes <75356972+botlikes456@users.noreply.github.com>"]
 license = "MIT"
@@ -11,14 +11,14 @@ packages = [
 [tool.poetry.dependencies]
 python = "~3.12"
 numpy = "^1.26.4"
-watchmen-data-kernel = "17.0.2"
-watchmen-storage-mysql = { version = "17.0.2", optional = true }
-watchmen-storage-oracle = { version = "17.0.2", optional = true }
-watchmen-storage-mongodb = { version = "17.0.2", optional = true }
-watchmen-storage-mssql = { version = "17.0.2", optional = true }
-watchmen-storage-postgresql = { version = "17.0.2", optional = true }
-watchmen-storage-oss = { version = "17.0.2", optional = true }
-watchmen-storage-s3 = { version = "17.0.2", optional = true }
+watchmen-data-kernel = "17.1.0"
+watchmen-storage-mysql = { version = "17.1.0", optional = true }
+watchmen-storage-oracle = { version = "17.1.0", optional = true }
+watchmen-storage-mongodb = { version = "17.1.0", optional = true }
+watchmen-storage-mssql = { version = "17.1.0", optional = true }
+watchmen-storage-postgresql = { version = "17.1.0", optional = true }
+watchmen-storage-oss = { version = "17.1.0", optional = true }
+watchmen-storage-s3 = { version = "17.1.0", optional = true }
 
 [tool.poetry.dev-dependencies]
 

{watchmen_collector_kernel-17.0.2 → watchmen_collector_kernel-17.1.0}/src/watchmen_collector_kernel/cache/collector_cache_manger.py

@@ -28,10 +28,6 @@ class TableConfigsByTenantAndParentNameCache(Cache):
 	pass
 
 
-class TableConfigByTenantAndTableNameCache(Cache):
-	pass
-
-
 class CollectorTopicByIdCache(Cache):
 	pass
 
@@ -60,10 +56,6 @@ configure_cache('TABLE_CONFIGS_BY_TENANT_AND_PARENT_NAME',
                 {'cache_class': TableConfigsByTenantAndParentNameCache, 'maxsize': 512})
 
 
-configure_cache('TABLE_CONFIG_BY_TENANT_AND_TABLE_NAME',
-                {'cache_class': TableConfigByTenantAndTableNameCache, 'maxsize': 512})
-
-
 configure_cache('COLLECTOR_TOPIC_BY_ID',
                 {'cache_class': CollectorTopicByIdCache, 'maxsize': 512})
 
@@ -84,10 +76,6 @@ def get_table_configs_by_tenant_and_parent_name_cache() -> TableConfigsByTenantA
 	return cache_set['TABLE_CONFIGS_BY_TENANT_AND_PARENT_NAME']
 
 
-def get_table_config_by_tenant_and_table_name_cache() -> TableConfigByTenantAndTableNameCache:
-	return cache_set['TABLE_CONFIG_BY_TENANT_AND_TABLE_NAME']
-
-
 def get_collector_topic_by_id_cache() -> CollectorTopicByIdCache:
 	return cache_set['COLLECTOR_TOPIC_BY_ID']
 

{watchmen_collector_kernel-17.0.2 → watchmen_collector_kernel-17.1.0}/src/watchmen_collector_kernel/cache/table_config_cache.py

@@ -6,8 +6,7 @@ from watchmen_collector_kernel.model import CollectorTableConfig
 
 from watchmen_data_kernel.cache import InternalCache
 
-from .collector_cache_manger import get_table_config_by_tenant_and_name_cache, \
-	get_table_config_by_tenant_and_table_name_cache, get_table_configs_by_tenant_and_parent_name_cache
+from .collector_cache_manger import get_table_config_by_tenant_and_name_cache, get_table_configs_by_tenant_and_parent_name_cache
 from watchmen_collector_kernel.common import ask_collector_config_cache_enabled
 
 
@@ -16,7 +15,6 @@ class TableConfigCache:
 	def __init__(self):
 		self.ByTenantAndNameCache = InternalCache(cache=get_table_config_by_tenant_and_name_cache)
 		self.ByTenantAndParentNameCache = InternalCache(cache=get_table_configs_by_tenant_and_parent_name_cache)
-		self.ByTenantAndTableNameCache = InternalCache(cache=get_table_config_by_tenant_and_table_name_cache)
 
 	# noinspection PyMethodMayBeStatic
 	def to_tenant_and_name_key(self, name: str, tenant_id: TenantId) -> str:
@@ -44,30 +42,6 @@ class TableConfigCache:
 		else:
 			return None
 
-	def put_config_by_table_name(self, table_config: CollectorTableConfig) -> Optional[CollectorTableConfig]:
-		if ask_collector_config_cache_enabled():
-			self.ByTenantAndTableNameCache.remove(table_config.tableName)
-			existing_config = self.ByTenantAndTableNameCache.put(self.to_tenant_and_name_key(table_config.tableName, table_config.tenantId),
-			                                                     table_config)
-			return existing_config
-		else:
-			return None
-
-	def get_config_by_table_name(self, table_name: str, tenant_id: str) -> Optional[CollectorTableConfig]:
-		if ask_collector_config_cache_enabled():
-			return self.ByTenantAndTableNameCache.get(self.to_tenant_and_name_key(table_name, tenant_id))
-		else:
-			return None
-
-	def remove_config_by_table_name(self, table_name: str, tenant_id: str) -> Optional[CollectorTableConfig]:
-		if ask_collector_config_cache_enabled():
-			existing: Optional[CollectorTableConfig] = self.ByTenantAndTableNameCache.remove(
-				self.to_tenant_and_name_key(table_name, tenant_id)
-			)
-			return existing
-		else:
-			return None
-
 	def put_configs_by_parent_name(self, parent_name: str, tenant_id: str,
 	                               configs: List[CollectorTableConfig]) -> Optional[List[CollectorTableConfig]]:
 		if ask_collector_config_cache_enabled():
@@ -101,7 +75,6 @@ class TableConfigCache:
 
 	def clear(self) -> None:
 		self.ByTenantAndNameCache.clear()
-		self.ByTenantAndTableNameCache.clear()
 		self.ByTenantAndParentNameCache.clear()
 
 

{watchmen_collector_kernel-17.0.2 → watchmen_collector_kernel-17.1.0}/src/watchmen_collector_kernel/common/settings.py

@@ -1,30 +1,24 @@
-from pydantic import BaseSettings
+from watchmen_utilities import ExtendedBaseSettings
 from logging import getLogger
 
 logger = getLogger(__name__)
 
 
-class CollectorSettings(BaseSettings):
+class CollectorSettings(ExtendedBaseSettings):
 	CLEAN_OF_TIMEOUT_INTERVAL: int = 300
 	LOCK_TIMEOUT: int = 1800
-	TRIGGER_EVENT_LOCK_TIMEOUT = 300
-	EXTRACT_TABLE_LOCK_TIMEOUT = 7200
-	CLEAN_UP_LOCK_TIMEOUT = 300
+	TRIGGER_EVENT_LOCK_TIMEOUT: int = 300
+	EXTRACT_TABLE_LOCK_TIMEOUT: int = 7200
+	CLEAN_UP_LOCK_TIMEOUT: int = 300
 	COLLECTOR_TIMEOUT: int = 600
 	COLLECTOR_TASK_TIMEOUT: int = 900
-	S3_CONNECTOR_LOCK_TIMEOUT = 300
+	S3_CONNECTOR_LOCK_TIMEOUT: int = 300
 	PARTIAL_SIZE: int = 100
 	COLLECTOR_CONFIG_CACHE_ENABLED: bool = True
-	EXCEPTION_MAX_LENGTH = 5000  # character
-	GROUPED_TASK_DATA_SIZE_THRESHOLD = 100
+	EXCEPTION_MAX_LENGTH: int = 5000  # character
+	GROUPED_TASK_DATA_SIZE_THRESHOLD: int = 100
 	TASK_PARTIAL_SIZE: int = 100
 
-	class Config:
-		# secrets_dir = '/var/run'
-		env_file = '.env'
-		env_file_encoding = 'utf-8'
-		case_sensitive = True
-
 
 collector_settings = CollectorSettings()
 logger.info(f'Collector settings[{collector_settings.dict()}].')

{watchmen_collector_kernel-17.0.2 → watchmen_collector_kernel-17.1.0}/src/watchmen_collector_kernel/model/__init__.py

@@ -5,7 +5,7 @@ from .collector_module_config import CollectorModuleConfig
 from .collector_model_config import CollectorModelConfig
 from .collector_table_config import CollectorTableConfig
 
-from .trigger_event import TriggerEvent, EventType
+from .trigger_event import TriggerEvent, EventType, QueryParam
 from .trigger_module import TriggerModule
 from .trigger_model import TriggerModel
 from .trigger_table import TriggerTable

watchmen_collector_kernel-17.1.0/src/watchmen_collector_kernel/model/change_data_json.py

@@ -0,0 +1,30 @@
+from typing import Dict, List, Optional
+
+from watchmen_utilities import ExtendedBaseModel
+
+from watchmen_model.common import TenantBasedTuple, Storable
+
+
+class Dependence(Storable, ExtendedBaseModel):
+	modelName: Optional[str] = None
+	objectId: Optional[str] = None
+
+
+class ChangeDataJson(TenantBasedTuple, ExtendedBaseModel):
+	changeJsonId: Optional[int] = None
+	resourceId: Optional[str] = None
+	modelName: Optional[str] = None
+	objectId: Optional[str] = None
+	sequence: Optional[int] = None
+	content: Optional[Dict] = None
+	dataId: Optional[Dict] = None
+	dependOn: Optional[List[Dependence]] = None
+	isPosted: Optional[bool] = None
+	status: Optional[int] = None
+	result: Optional[Dict] = None
+	taskId: Optional[int] = None
+	tableTriggerId: Optional[int] = None
+	modelTriggerId: Optional[int] = None
+	moduleTriggerId: Optional[int] = None
+	eventTriggerId: Optional[int] = None
+

watchmen_collector_kernel-17.1.0/src/watchmen_collector_kernel/model/collector_model_config.py

@@ -0,0 +1,14 @@
+from typing import List, Optional
+from watchmen_model.common import TenantBasedTuple, OptimisticLock
+from watchmen_utilities import ExtendedBaseModel
+
+
+class CollectorModelConfig(TenantBasedTuple, OptimisticLock, ExtendedBaseModel):
+	modelId: Optional[str] = None
+	modelName: Optional[str] = None
+	moduleId: Optional[str] = None
+	dependOn: Optional[List[str]] = None
+	priority: int = 0
+	rawTopicCode: Optional[str] = None
+	isParalleled: Optional[bool] = None
+

watchmen_collector_kernel-17.1.0/src/watchmen_collector_kernel/model/collector_module_config.py

@@ -0,0 +1,10 @@
+from typing import List, Optional
+from watchmen_model.common import TenantBasedTuple, OptimisticLock
+from watchmen_utilities import ExtendedBaseModel
+
+
+class CollectorModuleConfig(TenantBasedTuple, OptimisticLock, ExtendedBaseModel):
+	moduleId: Optional[str] = None
+	moduleName: Optional[str] = None
+	priority: int = 0
+

{watchmen_collector_kernel-17.0.2 → watchmen_collector_kernel-17.1.0}/src/watchmen_collector_kernel/model/collector_table_config.py

@@ -1,15 +1,12 @@
 from typing import Dict, List, Optional, Union
-
-from pydantic import BaseModel
-
 from .condition import Condition, construct_conditions, construct_condition
 from watchmen_model.common import TenantBasedTuple, Storable, OptimisticLock
-from watchmen_utilities import ArrayHelper
+from watchmen_utilities import ArrayHelper, ExtendedBaseModel
 
 
-class JoinCondition(Storable, BaseModel):
-	parentKey: Condition = None
-	childKey: Condition = None
+class JoinCondition(Storable, ExtendedBaseModel):
+	parentKey: Optional[Condition] = None
+	childKey: Optional[Condition] = None
 
 	def __setattr__(self, name, value):
 		if name == 'parentKey':
@@ -18,17 +15,17 @@ class JoinCondition(Storable, BaseModel):
 			super().__setattr__(name, construct_condition(value))
 
 
-class Dependence(Storable, BaseModel):
-	modelName: str
-	objectKey: str  # the dependent column
+class Dependence(Storable, ExtendedBaseModel):
+	modelName: Optional[str] = None
+	objectKey: Optional[str] = None  # the dependent column
 
 
-class JsonColumn(Storable, BaseModel):
-	columnName: str = None
-	ignoredPath: List[str] = None
-	needFlatten: bool = None
-	flattenPath: List[str] = None
-	jsonPath: List[str] = None
+class JsonColumn(Storable, ExtendedBaseModel):
+	columnName: Optional[str] = None
+	ignoredPath: Optional[List[str]] = None
+	needFlatten: Optional[bool] = None
+	flattenPath: Optional[List[str]] = None
+	jsonPath: Optional[List[str]] = None
 
 
 def construct_json_column(json_column: Union[JsonColumn, Dict]) -> Optional[JsonColumn]:
@@ -77,23 +74,23 @@ def construct_depend_on(depend_on: Optional[List[Union[Dependence, Dict]]]) -> O
 		return ArrayHelper(depend_on).map(lambda x: construct_dependence(x)).to_list()
 
 
-class CollectorTableConfig(TenantBasedTuple, OptimisticLock, BaseModel):
-	configId: str = None
-	name: str = None
-	tableName: str = None
-	primaryKey: List[str] = None
-	objectKey: str = None
-	sequenceKey: str = None
-	modelName: str = None
-	parentName: str = None
-	label: str = None
-	joinKeys: List[JoinCondition] = None
-	dependOn: List[Dependence] = []
-	auditColumn: str = None
-	ignoredColumns: List[str] = None
-	jsonColumns: List[JsonColumn] = None
-	conditions: List[Condition] = []
-	dataSourceId: str = None
+class CollectorTableConfig(TenantBasedTuple, OptimisticLock, ExtendedBaseModel):
+	configId: Optional[str] = None
+	name: Optional[str] = None
+	tableName: Optional[str] = None
+	primaryKey: Optional[List[str]] = None
+	objectKey: Optional[str] = None
+	sequenceKey: Optional[str] = None
+	modelName: Optional[str] = None
+	parentName: Optional[str] = None
+	label: Optional[str] = None
+	joinKeys: Optional[List[JoinCondition]] = None
+	dependOn: Optional[List[Dependence]] = []
+	auditColumn: Optional[str] = None
+	ignoredColumns: Optional[List[str]] = None
+	jsonColumns: Optional[List[JsonColumn]] = None
+	conditions: Optional[List[Condition]] = []
+	dataSourceId: Optional[str] = None
 	isList: bool = False
 	triggered: bool = False
 

{watchmen_collector_kernel-17.0.2 → watchmen_collector_kernel-17.1.0}/src/watchmen_collector_kernel/model/condition.py

@@ -4,14 +4,12 @@ from datetime import date, datetime
 from enum import Enum
 from typing import Optional, List, Union, Dict
 
-from pydantic import BaseModel
 from watchmen_storage import EntityCriteriaOperator
-
 from watchmen_model.common import DataModel
-from watchmen_utilities import ArrayHelper
+from watchmen_utilities import ArrayHelper, ExtendedBaseModel
 
 
-class Condition(DataModel, BaseModel):
+class Condition(DataModel, ExtendedBaseModel):
 	pass
 
 
@@ -28,7 +26,7 @@ class ConditionJointConjunction(str, Enum):
 
 class ConditionJoint(Condition):
 	conjunction: ConditionJointConjunction = ConditionJointConjunction.AND
-	children: List[Condition]
+	children: Optional[List[Condition]] = None
 
 	def __setattr__(self, name, value):
 		if name == 'children':

{watchmen_collector_kernel-17.0.2 → watchmen_collector_kernel-17.1.0}/src/watchmen_collector_kernel/model/scheduled_task.py

@@ -1,9 +1,8 @@
 from typing import Dict, List, Optional, Union
 
-from watchmen_utilities import ArrayHelper
-
+from watchmen_utilities import ArrayHelper, ExtendedBaseModel
 from watchmen_model.common import TenantBasedTuple, ScheduledTaskId, Storable
-from pydantic import BaseModel
+
 from enum import Enum
 
 
@@ -19,7 +18,7 @@ class Result(str, Enum):
 	PROCESS_TASK_FAILED = "PROCESS_TASK_FAILED"
 
 
-class Dependence(Storable, BaseModel):
+class Dependence(Storable, ExtendedBaseModel):
 	modelName: str
 	objectId: str
 
@@ -40,22 +39,23 @@ def construct_depend_on(depend_on: Optional[List[Union[Dependence, Dict]]]) -> O
 		return ArrayHelper(depend_on).map(lambda x: construct_dependence(x)).to_list()
 
 
-class ScheduledTask(TenantBasedTuple, BaseModel):
-	taskId: ScheduledTaskId
-	resourceId: str  # global unique, monotonous increase
-	topicCode: str
-	content: Dict
-	changeJsonIds: List[int]
-	modelName: str
-	objectId: str
-	dependOn: List[Dependence]
-	parentTaskId: List[int]
-	isFinished: bool
-	status: int
-	result: Dict
-	eventId: str
-	pipelineId: str
-	type: int
+class ScheduledTask(TenantBasedTuple, ExtendedBaseModel):
+	taskId: Optional[ScheduledTaskId] = None
+	resourceId:  Optional[str] = None  # global unique, monotonous increase
+	topicCode:  Optional[str] = None
+	content:  Optional[Dict] = None
+	changeJsonIds:  Optional[List[int]] = None
+	modelName:  Optional[str] = None
+	objectId:  Optional[str] = None
+	dependOn:  Optional[List[Dependence]] = None
+	parentTaskId:  Optional[List[int]] = None
+	isFinished:  Optional[bool] = None
+	status:  Optional[int] = None
+	result:  Optional[Dict] = None
+	eventId:  Optional[str] = None  # Deprecated
+	eventTriggerId:  Optional[int] = None
+	pipelineId:  Optional[str] = None
+	type:  Optional[int] = None
 
 	def __setattr__(self, name, value):
 		if name == 'dependOn':

watchmen_collector_kernel-17.1.0/src/watchmen_collector_kernel/model/trigger_event.py

@@ -0,0 +1,60 @@
+from datetime import datetime
+from enum import Enum
+from typing import List, Dict, Optional, Union
+
+from .condition import Condition, construct_conditions
+from watchmen_model.common import TenantBasedTuple, DataModel
+from watchmen_utilities import ExtendedBaseModel, ArrayHelper
+
+
+class EventType(int, Enum):
+	DEFAULT = 1,
+	BY_TABLE = 2,
+	BY_RECORD = 3,
+	BY_PIPELINE = 4
+
+
+class QueryParam(DataModel, ExtendedBaseModel):
+	name: str
+	filter: List[Condition]
+
+	def __setattr__(self, name, value):
+		if name == 'filter':
+			super().__setattr__(name, construct_conditions(value))
+		else:
+			super().__setattr__(name, value)
+
+
+class TriggerEvent(TenantBasedTuple, ExtendedBaseModel):
+	eventTriggerId: Optional[int] = None
+	startTime: Optional[datetime] = None
+	endTime: Optional[datetime] = None
+	isFinished: bool = False
+	status: Optional[int] = None
+	type: Optional[int] = None
+	tableName: Optional[str] = None
+	records: Optional[List[Dict]] = []
+	pipelineId: Optional[str] = None
+	params: Optional[List[QueryParam]] = None
+
+	def __setattr__(self, name, value):
+		if name == 'params':
+			super().__setattr__(name, construct_params(value))
+		else:
+			super().__setattr__(name, value)
+
+
+def construct_params(params: Optional[List[Union[dict, QueryParam]]]) -> Optional[List[QueryParam]]:
+	if params is None:
+		return None
+	else:
+		return ArrayHelper(params).map(lambda x: construct_param(x)).to_list()
+
+
+def construct_param(param: Optional[Union[dict, QueryParam]]) -> Optional[QueryParam]:
+	if param is None:
+		return None
+	elif isinstance(param, QueryParam):
+		return param
+	else:
+		return QueryParam(**param)

{watchmen_collector_kernel-17.0.2 → watchmen_collector_kernel-17.1.0}/src/watchmen_collector_kernel/service/table_config_service.py

@@ -18,24 +18,6 @@ class TableConfigService:
 	def __init__(self, principal_service: PrincipalService):
 		self.principalService = principal_service
 
-	def find_by_table_name(self, table_name: str, tenant_id: str) -> Optional[CollectorTableConfig]:
-		config = CollectorCacheService.table_config().get_config_by_table_name(table_name, tenant_id)
-		if config is not None:
-			return config
-
-		storage_service = get_collector_table_config_service(
-			ask_meta_storage(), ask_snowflake_generator(), self.principalService
-		)
-		table_config: CollectorTableConfig = storage_service.find_by_table_name(table_name, tenant_id)
-		if table_config is None:
-			table_config = self.find_by_topic_name(table_name, tenant_id)
-
-		if table_config is None:
-			return None
-
-		CollectorCacheService.table_config().put_config_by_table_name(table_config)
-		return table_config
-
 	def find_by_name(self, name: str, tenant_id: str) -> Optional[CollectorTableConfig]:
 		config = CollectorCacheService.table_config().get_config_by_name(name, tenant_id)
 		if config is not None:
@@ -45,6 +27,9 @@ class TableConfigService:
 			ask_meta_storage(), ask_snowflake_generator(), self.principalService
 		)
 		table_config: CollectorTableConfig = storage_service.find_by_name(name, tenant_id)
+		if table_config is None:
+			table_config = self.find_by_topic_name(name, tenant_id)
+
 		if table_config is None:
 			return None
 
@@ -77,15 +62,19 @@ class TableConfigService:
 	def get_topic_name_by_table_name(self, table_name: str) -> str:
 		return table_name.removeprefix("topic_")
 
-	def find_by_topic_name(self, table_name: str, tenant_id: str) -> Optional[CollectorTableConfig]:
-		topic_name = self.get_topic_name_by_table_name(table_name)
+	# noinspection PyMethodMayBeStatic
+	def get_table_name_by_topic_name(self, topic_name: str) -> str:
+		return "topic_" + topic_name
+
+	def find_by_topic_name(self, name: str, tenant_id: str) -> Optional[CollectorTableConfig]:
+		topic_name = name
 		schema = TopicService(self.principalService).find_schema_by_name(topic_name, tenant_id)
 		now = get_current_time_in_seconds()
 		if schema:
 			return CollectorTableConfig(
 				configId=self.fake_config_id(schema.topic.topicId),
 				name=topic_name,
-				tableName=table_name,
+				tableName=self.get_table_name_by_topic_name(name),
 				primaryKey=["id_"],
 				objectKey="id_",
 				sequenceKey=None,

{watchmen_collector_kernel-17.0.2 → watchmen_collector_kernel-17.1.0}/src/watchmen_collector_kernel/service/trigger_collector.py

@@ -132,7 +132,7 @@ def new_trigger_table(table_name: str,
 
 
 def get_trigger_table(model_trigger: TriggerModel, table_config: CollectorTableConfig) -> TriggerTable:
-	return new_trigger_table(table_config.tableName,
+	return new_trigger_table(table_config.name,
 	                         table_config.modelName,
 	                         model_trigger.modelTriggerId,
 	                         model_trigger.moduleTriggerId,

{watchmen_collector_kernel-17.0.2 → watchmen_collector_kernel-17.1.0}/src/watchmen_collector_kernel/service/trigger_event_helper.py

@@ -86,8 +86,7 @@ def trigger_event_by_table(trigger_event: TriggerEvent):
 	                                                          trigger_event_service.snowflakeGenerator,
 	                                                          trigger_event_service.principalService)
 
-	table_config = table_config_service.find_by_table_name_and_tenant_id(trigger_event.tableName,
-	                                                                     trigger_event.tenantId)
+	table_config = table_config_service.find_by_name(trigger_event.tableName, trigger_event.tenantId)
 	model_config = model_config_service.find_by_name(table_config.modelName, table_config.tenantId)
 
 	module_config = module_config_service.find_by_module_id(model_config.moduleId)
@@ -140,8 +139,7 @@ def trigger_event_by_records(trigger_event: TriggerEvent):
 	                                                          trigger_event_service.snowflakeGenerator,
 	                                                          trigger_event_service.principalService)
 
-	table_config = table_config_service.find_by_table_name_and_tenant_id(trigger_event.tableName,
-	                                                                     trigger_event.tenantId)
+	table_config = table_config_service.find_by_name(trigger_event.tableName, trigger_event.tenantId)
 	model_config = model_config_service.find_by_name(table_config.modelName, table_config.tenantId)
 	module_config = module_config_service.find_by_module_id(model_config.moduleId)
 
@@ -276,7 +274,7 @@ def trigger_event_by_pipeline(trigger_event: TriggerEvent):
 	model_config_service = get_model_config_service(principal_service)
 	table_config_service = get_table_config_service(principal_service)
 
-	table_config = table_config_service.find_by_table_name(trigger_event.tableName, trigger_event.tenantId)
+	table_config = table_config_service.find_by_name(trigger_event.tableName, trigger_event.tenantId)
 	model_config = model_config_service.find_by_name(table_config.modelName, trigger_event.tenantId)
 	module_config = module_config_service.find_by_module_id(model_config.moduleId, trigger_event.tenantId)
 

{watchmen_collector_kernel-17.0.2 → watchmen_collector_kernel-17.1.0}/src/watchmen_collector_kernel/storage/scheduled_task_service.py

@@ -32,6 +32,7 @@ class ScheduledTaskShaper(EntityShaper):
 			'status': entity.status,
 			'result': entity.result,
 			'event_id': entity.eventId,
+			'event_trigger_id': entity.eventTriggerId,
 			'pipeline_id': entity.pipelineId,
 			'type': entity.type
 		})
@@ -53,6 +54,7 @@ class ScheduledTaskShaper(EntityShaper):
 			status=row.get('status'),
 			result=row.get('result'),
 			eventId=row.get('event_id'),
+			eventTriggerId=row.get('event_trigger_id'),
 			pipelineId=row.get('pipeline_id'),
 			type=row.get('type')
 		))
@@ -205,7 +207,7 @@ class ScheduledTaskService(TupleService):
 		finally:
 			self.close_transaction()
 
-	def find_model_dependent_tasks(self, model_name: str, object_id: str, event_id: str, tenant_id: str) -> List[ScheduledTask]:
+	def find_model_dependent_tasks(self, model_name: str, object_id: str, event_trigger_id: int, tenant_id: str) -> List[ScheduledTask]:
 		self.begin_transaction()
 		try:
 			# noinspection PyTypeChecker
@@ -215,8 +217,8 @@ class ScheduledTaskService(TupleService):
 					                         right=model_name),
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='object_id'),
 					                         right=object_id),
-					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='event_id'),
-					                         right=event_id),
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='event_trigger_id'),
+					                         right=event_trigger_id),
 					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'),
 					                         right=tenant_id)
 				],
@@ -247,7 +249,7 @@ class ScheduledTaskService(TupleService):
 	def count_scheduled_task(self, event_trigger_id: int) -> int:
 		return self.storage.count(self.get_entity_finder(
 			criteria=[
-				EntityCriteriaExpression(left=ColumnNameLiteral(columnName='event_id'),
+				EntityCriteriaExpression(left=ColumnNameLiteral(columnName='event_trigger_id'),
 				                         right=event_trigger_id)
 			]
 		))
@@ -264,7 +266,7 @@ class ScheduledTaskService(TupleService):
 		finally:
 			self.storage.close()
 
-	def is_model_finished(self, model_name: str, event_id: str) -> bool:
+	def is_model_finished(self, model_name: str, event_trigger_id: int) -> bool:
 		try:
 			self.begin_transaction()
 			results = self.storage.find_limited(
@@ -273,7 +275,7 @@ class ScheduledTaskService(TupleService):
 					shaper=self.get_entity_shaper(),
 					criteria=[
 						EntityCriteriaExpression(left=ColumnNameLiteral(columnName='model_name'), right=model_name),
-						EntityCriteriaExpression(left=ColumnNameLiteral(columnName='event_id'), right=event_id)
+						EntityCriteriaExpression(left=ColumnNameLiteral(columnName='event_trigger_id'), right=event_trigger_id)
 					],
 					limit=1
 				)
@@ -289,7 +291,7 @@ class ScheduledTaskService(TupleService):
 		finally:
 			self.close_transaction()
 
-	def is_event_finished(self, event_id: str) -> bool:
+	def is_event_finished(self, event_trigger_id: int) -> bool:
 		try:
 			self.begin_transaction()
 			results = self.storage.find_limited(
@@ -297,7 +299,7 @@ class ScheduledTaskService(TupleService):
 					name=self.get_entity_name(),
 					shaper=self.get_entity_shaper(),
 					criteria=[
-						EntityCriteriaExpression(left=ColumnNameLiteral(columnName='event_id'), right=event_id)
+						EntityCriteriaExpression(left=ColumnNameLiteral(columnName='event_trigger_id'), right=event_trigger_id)
 					],
 					limit=1
 				)

{watchmen_collector_kernel-17.0.2 → watchmen_collector_kernel-17.1.0}/src/watchmen_collector_kernel/storage/trigger_event_service.py

@@ -1,17 +1,31 @@
 from typing import List, Dict, Any, Optional
 
 from watchmen_auth import PrincipalService
-from watchmen_collector_kernel.model import TriggerEvent
+from watchmen_collector_kernel.model import TriggerEvent, QueryParam
 from watchmen_meta.common import TupleShaper, TupleService
 from watchmen_meta.common.storage_service import StorableId
 from watchmen_model.common import Storable, EventTriggerId
 from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
 	EntityStraightValuesFinder, EntityCriteriaExpression, ColumnNameLiteral, \
 	EntityStraightColumn, EntitySortColumn, EntitySortMethod
+from watchmen_utilities import ArrayHelper
 
 
 class TriggerEventShaper(EntityShaper):
 
+	@staticmethod
+	def serialize_param(param: Optional[QueryParam]) -> dict:
+		if isinstance(param, dict):
+			return param
+		else:
+			return param.to_dict()
+
+	@staticmethod
+	def serialize_params(params: Optional[List[QueryParam]]) -> Optional[list]:
+		if params is None:
+			return None
+		return ArrayHelper(params).map(lambda x: TriggerEventShaper.serialize_param(x)).to_list()
+
 	def serialize(self, entity: TriggerEvent) -> EntityRow:
 		return TupleShaper.serialize_tenant_based(entity, {
 			'event_trigger_id': entity.eventTriggerId,
@@ -22,7 +36,8 @@ class TriggerEventShaper(EntityShaper):
 			'type': entity.type,
 			'table_name': entity.tableName,
 			'records': entity.records,
-			'pipeline_id': entity.pipelineId
+			'pipeline_id': entity.pipelineId,
+			'params': TriggerEventShaper.serialize_params(entity.params)
 		})
 
 	def deserialize(self, row: EntityRow) -> TriggerEvent:
@@ -36,7 +51,8 @@ class TriggerEventShaper(EntityShaper):
 			type=row.get('type'),
 			tableName=row.get('table_name'),
 			records=row.get('records'),
-			pipelineId=row.get('pipeline_id')
+			pipelineId=row.get('pipeline_id'),
+			params=row.get('params')
 		))
 
 

watchmen_collector_kernel-17.0.2/src/watchmen_collector_kernel/model/change_data_json.py

@@ -1,30 +0,0 @@
-from typing import Dict, List
-
-from pydantic import BaseModel
-
-from watchmen_model.common import TenantBasedTuple, Storable
-
-
-class Dependence(Storable, BaseModel):
-	modelName: str
-	objectId: str
-
-
-class ChangeDataJson(TenantBasedTuple, BaseModel):
-	changeJsonId: int
-	resourceId: str
-	modelName: str
-	objectId: str
-	sequence: int
-	content: Dict
-	dataId: Dict
-	dependOn: List[Dependence]
-	isPosted: bool
-	status: int
-	result: Dict
-	taskId: int
-	tableTriggerId: int
-	modelTriggerId: int
-	moduleTriggerId: int
-	eventTriggerId: int
-

watchmen_collector_kernel-17.0.2/src/watchmen_collector_kernel/model/collector_model_config.py

@@ -1,14 +0,0 @@
-from typing import List
-from watchmen_model.common import TenantBasedTuple, OptimisticLock
-from pydantic import BaseModel
-
-
-class CollectorModelConfig(TenantBasedTuple, OptimisticLock, BaseModel):
-	modelId: str
-	modelName: str
-	moduleId: str
-	dependOn: List[str]
-	priority: int = 0
-	rawTopicCode: str
-	isParalleled: bool
-

watchmen_collector_kernel-17.0.2/src/watchmen_collector_kernel/model/collector_module_config.py

@@ -1,10 +0,0 @@
-from typing import List
-from watchmen_model.common import TenantBasedTuple, OptimisticLock
-from pydantic import BaseModel
-
-
-class CollectorModuleConfig(TenantBasedTuple, OptimisticLock, BaseModel):
-	moduleId: str
-	moduleName: str
-	priority: int = 0
-

watchmen_collector_kernel-17.0.2/src/watchmen_collector_kernel/model/trigger_event.py

@@ -1,26 +0,0 @@
-from datetime import datetime
-from enum import Enum
-from typing import List, Dict
-
-from watchmen_model.common import TenantBasedTuple
-from pydantic import BaseModel
-
-
-class EventType(int, Enum):
-	DEFAULT = 1,
-	BY_TABLE = 2,
-	BY_RECORD = 3,
-	BY_PIPELINE = 4
-
-
-class TriggerEvent(TenantBasedTuple, BaseModel):
-	eventTriggerId: int
-	startTime: datetime
-	endTime: datetime
-	isFinished: bool = False
-	status: int
-	type: int
-	tableName: str
-	records: List[Dict] = []
-	pipelineId: str
-