watchmen-collector-kernel 16.4.6__py3-none-any.whl → 16.4.8__py3-none-any.whl

watchmen_collector_kernel/storage/trigger_table_service.py

@@ -0,0 +1,113 @@
+from typing import Optional, List
+
+from watchmen_auth import PrincipalService
+from watchmen_collector_kernel.model import TriggerTable
+from watchmen_meta.common import TupleShaper, TupleService
+from watchmen_meta.common.storage_service import StorableId
+from watchmen_model.common import Storable, TableTriggerId
+from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
+	EntityCriteriaJoint, EntityCriteriaExpression, ColumnNameLiteral, EntitySortColumn
+
+
+class TriggerTableShaper(EntityShaper):
+
+	def serialize(self, entity: TriggerTable) -> EntityRow:
+		return TupleShaper.serialize_tenant_based(entity, {
+			'table_trigger_id': entity.tableTriggerId,
+			'table_name': entity.tableName,
+			'model_name': entity.modelName,
+			'is_extracted': entity.isExtracted,
+			'data_count': entity.dataCount,
+			'model_trigger_id': entity.modelTriggerId,
+			'event_trigger_id': entity.eventTriggerId
+		})
+
+	def deserialize(self, row: EntityRow) -> TriggerTable:
+		# noinspection PyTypeChecker
+		return TupleShaper.deserialize_tenant_based(row, TriggerTable(
+			tableTriggerId=row.get('table_trigger_id'),
+			tableName=row.get('table_name'),
+			modelName=row.get('model_name'),
+			isExtracted=row.get('is_extracted'),
+			dataCount=row.get('data_count'),
+			modelTriggerId=row.get('model_trigger_id'),
+			eventTriggerId=row.get('event_trigger_id')
+		))
+
+
+TRIGGER_TABLE_TABLE = 'trigger_table'
+TRIGGER_TABLE_ENTITY_SHAPER = TriggerTableShaper()
+
+
+class TriggerTableService(TupleService):
+
+	def should_record_operation(self) -> bool:
+		return False
+
+	def get_entity_name(self) -> EntityName:
+		return TRIGGER_TABLE_TABLE
+
+	def get_entity_shaper(self) -> EntityShaper:
+		return TRIGGER_TABLE_ENTITY_SHAPER
+
+	def get_storable_id_column_name(self) -> EntityName:
+		return 'table_trigger_id'
+
+	def get_storable_id(self, storable: TriggerTable) -> StorableId:
+		# noinspection PyTypeChecker
+		return storable.tableTriggerId
+
+	def set_storable_id(
+			self, storable: TriggerTable, storable_id: TableTriggerId) -> Storable:
+		storable.tableTriggerId = storable_id
+		return storable
+
+	def update_table_trigger(self, trigger: TriggerTable):
+		self.begin_transaction()
+		try:
+			result = self.update(trigger)
+			self.commit_transaction()
+			return result
+		except Exception as e:
+			self.rollback_transaction()
+			raise e
+
+	def find_unfinished(self) -> Optional[List[TriggerTable]]:
+		self.begin_transaction()
+		try:
+			# noinspection PyTypeChecker
+			return self.storage.find_distinct_values(
+				self.get_entity_finder_for_columns(
+					criteria=[EntityCriteriaExpression(left=ColumnNameLiteral(columnName='is_extracted'), right=False)],
+					distinctColumnNames=['table_trigger_id',
+					                     'tenant_id'],
+					distinctValueOnSingleColumn=False)
+			)
+		finally:
+			self.close_transaction()
+
+	def find_by_id(self, trigger_id: TableTriggerId) -> Optional[TriggerTable]:
+		self.begin_transaction()
+		try:
+			return self.storage.find_by_id(trigger_id, self.get_entity_id_helper())
+		finally:
+			self.close_transaction()
+
+	def find_by_model_trigger_id(self, model_trigger_id: int) -> List[TriggerTable]:
+		self.begin_transaction()
+		try:
+			# noinspection PyTypeChecker
+			return self.storage.find(self.get_entity_finder(
+				criteria=[
+					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='model_trigger_id'), right=model_trigger_id)
+				]
+			))
+		finally:
+			self.close_transaction()
+
+
+def get_trigger_table_service(storage: TransactionalStorageSPI,
+                              snowflake_generator: SnowflakeGenerator,
+                              principal_service: PrincipalService
+                              ) -> TriggerTableService:
+	return TriggerTableService(storage, snowflake_generator, principal_service)

{watchmen_collector_kernel-16.4.6.dist-info → watchmen_collector_kernel-16.4.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: watchmen-collector-kernel
-Version: 16.4.6
+Version: 16.4.8
 Summary: 
 License: MIT
 Author: botlikes
@@ -18,11 +18,11 @@ Provides-Extra: oracle
 Provides-Extra: oss
 Provides-Extra: postgresql
 Provides-Extra: s3
-Requires-Dist: watchmen-pipeline-kernel (==16.4.6)
-Requires-Dist: watchmen-storage-mongodb (==16.4.6) ; extra == "mongodb"
-Requires-Dist: watchmen-storage-mssql (==16.4.6) ; extra == "mssql"
-Requires-Dist: watchmen-storage-mysql (==16.4.6) ; extra == "mysql"
-Requires-Dist: watchmen-storage-oracle (==16.4.6) ; extra == "oracle"
-Requires-Dist: watchmen-storage-oss (==16.4.6) ; extra == "oss"
-Requires-Dist: watchmen-storage-postgresql (==16.4.6) ; extra == "postgresql"
-Requires-Dist: watchmen-storage-s3 (==16.4.6) ; extra == "s3"
+Requires-Dist: watchmen-data-kernel (==16.4.8)
+Requires-Dist: watchmen-storage-mongodb (==16.4.8) ; extra == "mongodb"
+Requires-Dist: watchmen-storage-mssql (==16.4.8) ; extra == "mssql"
+Requires-Dist: watchmen-storage-mysql (==16.4.8) ; extra == "mysql"
+Requires-Dist: watchmen-storage-oracle (==16.4.8) ; extra == "oracle"
+Requires-Dist: watchmen-storage-oss (==16.4.8) ; extra == "oss"
+Requires-Dist: watchmen-storage-postgresql (==16.4.8) ; extra == "postgresql"
+Requires-Dist: watchmen-storage-s3 (==16.4.8) ; extra == "s3"

watchmen_collector_kernel-16.4.8.dist-info/RECORD

@@ -0,0 +1,38 @@
+watchmen_collector_kernel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+watchmen_collector_kernel/common/__init__.py,sha256=QJHylPJAMoSbSa6EfgiQTLNtkQKSCUJucf7ZG3Nqswg,175
+watchmen_collector_kernel/common/constants.py,sha256=mIsGBhAjHbmeuVeNYDOKLhiBqG2sSX-0vs8ipEEIo9E,302
+watchmen_collector_kernel/model/__init__.py,sha256=ZhsKn7a2KW7LaVj87coIRC4Z8wsysE2RXn6v8AmoUsg,554
+watchmen_collector_kernel/model/change_data_json.py,sha256=-oNIGsvYo8KJJsSFusbGa9kIuFfE0otLnWe-zz9O42A,495
+watchmen_collector_kernel/model/change_data_record.py,sha256=vadzugcX4nl6G3F-jt7DtuVqjE8vtlasu86DDjpaGGM,330
+watchmen_collector_kernel/model/collector_model_config.py,sha256=oE0T9I_9uGppttoytvzl7-35WgwlWu7s6awNtsK4Q20,288
+watchmen_collector_kernel/model/collector_table_config.py,sha256=R80d2rqfVRj_rZ6-mognaLYe26R7r3g4nFR3Wj-4YLY,2143
+watchmen_collector_kernel/model/competitive_lock.py,sha256=fxDRuzZpsgrZjGeuADS3zRYqQioqWhg72Pr592U_Yvw,264
+watchmen_collector_kernel/model/condition.py,sha256=jGsjvb5z9XRFW0B55k-iC2_Zle4rl-A5J_6PB6EV6mA,1942
+watchmen_collector_kernel/model/scheduled_task.py,sha256=Zb6blDfDmxX90vZwAODQLQtyfI_0JTb1aWGi80voAyY,752
+watchmen_collector_kernel/model/trigger_event.py,sha256=ZjVgWQS_qqILchw4fGFzxn1tJUMx77s2_UpyiBd7azA,251
+watchmen_collector_kernel/model/trigger_model.py,sha256=NpnIbdax8vQeGYeGcOMpWpjYUhkeZUil--6-yTNpv3k,175
+watchmen_collector_kernel/model/trigger_table.py,sha256=aPYJlIb-8aX_-O6vZQp3DU58zYtyosYLGFjpL81ijds,229
+watchmen_collector_kernel/service/__init__.py,sha256=hsg-38u7V3QizNmmuEtOw8gC7QEOopVInLbY97MiwuM,426
+watchmen_collector_kernel/service/criteria_builder.py,sha256=NAk0yQBn4KARttqEOwWozxIlqkVuGPTQ8n3Zk6gjwcQ,2360
+watchmen_collector_kernel/service/data_capture.py,sha256=45asoQQlv016NKsPoiUVgaSCOJzUBHDCmWr7XPt-8VI,2873
+watchmen_collector_kernel/service/extract_source.py,sha256=MYldG4gR6t-2JeVJ6nL5aAb59AjxkQWj3dVwzJKXN9Q,2928
+watchmen_collector_kernel/service/extract_utils.py,sha256=PIyxFG7U37ZXLYlOdIK-Dh-kBptXPFqk-9-b1LuPL1A,2452
+watchmen_collector_kernel/service/lock_clean.py,sha256=qeNc8vZtfxXTNpGhGc_ZnUD9iRGK2VXNEpv1VglVWCA,965
+watchmen_collector_kernel/service/lock_helper.py,sha256=GxqjIsCD-jn5kDICVWS1LlTFCuMpnRSpcrQtPLv5bxs,842
+watchmen_collector_kernel/service/task_housekeeping.py,sha256=i1lnZ4jo-RLjLQD0Z7HSLQqjXuSqdL3zqHDfFVaNbe8,1673
+watchmen_collector_kernel/service/task_service.py,sha256=XNc-nudTpk569ibqd5wLpzLGlcHUAf71Acn9_iKuS2Y,3091
+watchmen_collector_kernel/service/trigger_collector.py,sha256=IP_ILJNMhahBogtvfS1iHDOJAxKCDBDJFXTDi98R11M,6790
+watchmen_collector_kernel/storage/__init__.py,sha256=sstizR2fcwo4ZhqfiqceTxxS49HTtfExCPMeEaYc9bc,832
+watchmen_collector_kernel/storage/change_data_json_service.py,sha256=JvaIFWPDzq6CHKNbMaUvbIaXs0iAT4IMUFj-zeZFPcw,7393
+watchmen_collector_kernel/storage/change_data_record_service.py,sha256=MOju5vTl-ZQrpEfL7DytvB_T_JM0dYyXGAzrhNnzb2Y,7463
+watchmen_collector_kernel/storage/collector_model_config_service.py,sha256=kWLjmppPpRuN0VptiAEGA38BJ20fFD5Q5gxPZJRbQYs,3830
+watchmen_collector_kernel/storage/collector_table_config_service.py,sha256=aVZeN5AhzUORHaZQUDo_MlKcX6AkgcHcQeI6UwAoAT0,5659
+watchmen_collector_kernel/storage/competitive_lock_service.py,sha256=jjxxHb4Dr0DIDX_g26wtCa8vGLGLnc4BJQGl0d3J31M,2534
+watchmen_collector_kernel/storage/scheduled_task_service.py,sha256=ROHYW-pTQaL8kWkLgk9h18XsxfKGjsReMoBY0hoD4SM,5340
+watchmen_collector_kernel/storage/trigger_event_service.py,sha256=DKSCQ0sYSH3uAtGmokvncD9FIYvjoVsk7QtqdRMQGhM,3099
+watchmen_collector_kernel/storage/trigger_model_service.py,sha256=0-_YcL5FNGHy2MKn8vTYrWFaYOPW_u4iKQRG00AOfCI,3219
+watchmen_collector_kernel/storage/trigger_table_service.py,sha256=RHiFJaT3UQmCpz-gxnfsoBPV7lS8JM1i9DCQn3BnfA4,3779
+watchmen_collector_kernel-16.4.8.dist-info/LICENSE,sha256=iuuG7ErblOdcEZi5u89VXS0VIUiTb4flerGp_PAS9E4,1061
+watchmen_collector_kernel-16.4.8.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
+watchmen_collector_kernel-16.4.8.dist-info/METADATA,sha256=7-zGOrH90-yWi6UZGqtvw4qbSvOFs6a934l9YrQoLVM,1135
+watchmen_collector_kernel-16.4.8.dist-info/RECORD,,

watchmen_collector_kernel/common/settings.py

@@ -1,14 +0,0 @@
-from watchmen_model.common import SettingsModel
-
-
-class S3CollectorSettings(SettingsModel):
-	access_key_id: str
-	secret_access_key: str
-	bucket_name: str
-	region: str
-	token: str
-	tenant_id: int
-	consume_prefix: str
-	dead_prefix: str
-	max_keys: int = 10
-	clean_task_interval: int = 3600

watchmen_collector_kernel/connector/__init__.py

@@ -1 +0,0 @@
-from .s3_connector import init_s3_collector

watchmen_collector_kernel/connector/handler.py

@@ -1,42 +0,0 @@
-
-from watchmen_data_kernel.storage import TopicTrigger
-from watchmen_meta.common import ask_snowflake_generator, ask_super_admin
-
-from watchmen_model.pipeline_kernel import PipelineTriggerData
-from watchmen_pipeline_kernel.pipeline import create_monitor_log_pipeline_invoker, PipelineTrigger
-
-
-from watchmen_data_kernel.meta import TopicService
-from watchmen_data_kernel.service import ask_topic_data_service, ask_topic_storage
-
-
-async def handle_trigger_data(trigger_data: PipelineTriggerData, topic_trigger: TopicTrigger) -> None:
-	# use super admin
-	principal_service = ask_super_admin()
-	# change the tenant_id
-	principal_service.tenantId = trigger_data.tenantId
-	schema = TopicService(principal_service).find_schema_by_name(trigger_data.code, trigger_data.tenantId)
-	trace_id = str(ask_snowflake_generator().next_id())
-	await PipelineTrigger(
-		trigger_topic_schema=schema,
-		trigger_type=trigger_data.triggerType,
-		trigger_data=trigger_data.data,
-		trace_id=trace_id,
-		principal_service=principal_service,
-		asynchronized=False,
-		handle_monitor_log=create_monitor_log_pipeline_invoker(trace_id, principal_service)
-	).start(topic_trigger)
-
-
-def save_topic_data(trigger_data: PipelineTriggerData) -> TopicTrigger:
-	# use super admin
-	principal_service = ask_super_admin()
-	# change the tenant_id
-	principal_service.tenantId = trigger_data.tenantId
-	schema = TopicService(principal_service).find_schema_by_name(trigger_data.code, trigger_data.tenantId)
-	if schema is None:
-		raise Exception("schema is not not found. %s, %s", trigger_data.code, trigger_data.tenantId)
-	data = schema.prepare_data(trigger_data.data, principal_service)
-	storage = ask_topic_storage(schema, principal_service)
-	service = ask_topic_data_service(schema, storage, principal_service)
-	return service.trigger_by_insert(data)

watchmen_collector_kernel/connector/housekeeping.py

@@ -1,58 +0,0 @@
-from datetime import datetime, date, timedelta
-from logging import getLogger
-from threading import Thread
-from typing import List
-
-from watchmen_collector_kernel.common import S3CollectorSettings
-from watchmen_collector_kernel.lock import get_oss_collector_lock_service
-from watchmen_collector_kernel.model import OSSCollectorCompetitiveLock
-from time import sleep
-
-
-class CleanTask:
-	
-	def __init__(self,  settings: S3CollectorSettings):
-		self.lock_service = get_oss_collector_lock_service()
-		self.processed_date = []
-		self.cleanTaskInterval = settings.clean_task_interval
-	
-	def run(self):
-		try:
-			while True:
-				self.clean_task()
-				sleep(self.cleanTaskInterval)
-		except Exception as e:
-			getLogger(__name__).error(e, exc_info=True, stack_info=True)
-			self.restart()
-	
-	def clean_task(self):
-		query_datetime = datetime.now()
-		query_date = query_datetime.date() - timedelta(hours=query_datetime.hour,
-		                                               minutes=query_datetime.minute,
-		                                               seconds=query_datetime.second,
-		                                               microseconds=query_datetime.microsecond)
-		if self.is_processed(query_date):
-			return "Done"
-		else:
-			tasks = self.get_task_list(query_date)
-			for task in tasks:
-				self.lock_service.delete_by_id(task.lockId)
-			self.processed_date.append(query_date.strftime('%Y-%m-%d'))
-	
-	def get_task_list(self, clean_date) -> List[OSSCollectorCompetitiveLock]:
-		return self.lock_service.find_completed_task(clean_date)
-	
-	def is_processed(self, query_date: date) -> bool:
-		date_str = query_date.strftime('%Y-%m-%d')
-		if date_str in self.processed_date:
-			return True
-		else:
-			return False
-	
-	def restart(self):
-		Thread(target=CleanTask.run, args=(self,), daemon=True).start()
-
-
-def init_task_housekeeping(settings: S3CollectorSettings):
-	cleanTask = CleanTask(settings)
-	Thread(target=CleanTask.run, args=(cleanTask,), daemon=True).start()

watchmen_collector_kernel/connector/s3_connector.py

@@ -1,210 +0,0 @@
-import asyncio
-from enum import Enum
-from logging import getLogger
-from threading import Thread
-from time import sleep
-from typing import Optional, Dict, Any
-
-from watchmen_collector_kernel.common import S3CollectorSettings
-from watchmen_collector_kernel.lock import get_oss_collector_lock_service, get_unique_key_distributed_lock, \
-	DistributedLock
-from watchmen_collector_kernel.model import OSSCollectorCompetitiveLock
-from watchmen_data_kernel.storage import TopicTrigger
-from watchmen_meta.common import ask_snowflake_generator
-
-from watchmen_storage_s3 import SimpleStorageService, ObjectContent
-
-from watchmen_model.pipeline_kernel import PipelineTriggerData
-from watchmen_model.common import Storable
-
-from .handler import save_topic_data, handle_trigger_data
-from .housekeeping import init_task_housekeeping
-
-logger = getLogger(__name__)
-
-identifier_delimiter = "~"
-
-
-class STATUS(str, Enum):
-	CHECK_KEY_FAILED = "CHECK_KEY_FAILED"
-	DEPENDENCY_FAILED = "DEPENDENCY_FAILED"
-	CREATE_TASK_FAILED = "CREATE_TASK_FAILED"
-	EMPTY_PAYLOAD = "EMPTY_PAYLOAD"
-	COMPLETED_TASK = "COMPLETED_TASK"
-	PROCESS_TASK_FAILED = "PROCESS_TASK_FAILED"
-
-
-def init_s3_collector(settings: S3CollectorSettings):
-	S3Connector(settings).create_connector()
-	init_task_housekeeping(settings)
-
-
-class Dependency(Storable):
-	model_name: str
-	object_id: str
-
-
-class S3Connector:
-	
-	def __init__(self, settings: S3CollectorSettings):
-		self.simpleStorageService = SimpleStorageService(access_key_id=settings.access_key_id,
-		                                                 access_key_secret=settings.secret_access_key,
-		                                                 endpoint=settings.region,
-		                                                 bucket_name=settings.bucket_name,
-		                                                 params=None)
-		self.lock_service = get_oss_collector_lock_service()
-		self.snowflakeGenerator = ask_snowflake_generator()
-		self.token = settings.token
-		self.tenant_id = settings.tenant_id
-		self.consume_prefix = settings.consume_prefix
-		self.dead_prefix = settings.dead_prefix
-		self.maxKeys = settings.max_keys
-	
-	def create_connector(self) -> None:
-		Thread(target=S3Connector.run, args=(self,), daemon=True).start()
-	
-	def run(self):
-		try:
-			while True:
-				objects = self.simpleStorageService.list_objects(max_keys=self.maxKeys, prefix=self.consume_prefix)
-				logger.info("objects size {}".format(len(objects)))
-				if len(objects) == 0:
-					sleep(5)
-				else:
-					for object_ in objects:
-						result = self.consume(object_)
-						if result == STATUS.CREATE_TASK_FAILED or result == STATUS.DEPENDENCY_FAILED:
-							# logger.info("CREATE_TASK_FAILED or DEPENDENCY_FAILED , key is  {}".format(object_.key))
-							continue
-						elif result == STATUS.CHECK_KEY_FAILED or result == STATUS.COMPLETED_TASK or \
-								STATUS.EMPTY_PAYLOAD or result == STATUS.PROCESS_TASK_FAILED:
-							logger.info(
-								"CHECK_KEY_FAILED or COMPLETED_TASK or EMPTY_PAYLOAD or PROCESS_TASK_FAILED, key is  {}".format(
-									object_.key))
-							break
-		except Exception as e:
-			logger.error(e, exc_info=True, stack_info=True)
-			sleep(300)
-			self.create_connector()
-	
-	def consume(self, object_: ObjectContent) -> str:
-		object_key = self.get_identifier(self.consume_prefix, object_.key)
-		if self.validate_key_pattern(object_key):
-			dependency = self.get_dependency(object_key)
-			if self.check_dependency_finished(dependency):
-				distributed_lock = get_unique_key_distributed_lock(self.get_resource_lock(object_.key),
-				                                                   self.lock_service)
-				try:
-					if not self.ask_lock(distributed_lock):
-						return STATUS.CREATE_TASK_FAILED
-					else:
-						payload = self.get_payload(object_.key)
-						if payload:
-							try:
-								trigger_data = PipelineTriggerData(code=self.get_code(object_key), data=payload,
-								                                   tenantId=self.tenant_id)
-								topic_trigger = self.save_data(trigger_data)
-								self.trigger_pipeline(trigger_data, topic_trigger)
-								self.simpleStorageService.delete_object(object_.key)
-								return STATUS.COMPLETED_TASK
-							except Exception as e:
-								logger.error(e, exc_info=True, stack_info=True)
-								self.move_to_dead_queue(object_.key, payload)
-								return STATUS.PROCESS_TASK_FAILED
-						else:
-							self.move_to_dead_queue(object_.key, payload)
-							return STATUS.EMPTY_PAYLOAD
-				finally:
-					self.ask_unlock(distributed_lock)
-			else:
-				return STATUS.DEPENDENCY_FAILED
-		else:
-			distributed_lock = get_unique_key_distributed_lock(self.get_resource_lock(object_.key),
-			                                                   self.lock_service)
-			try:
-				if not self.ask_lock(distributed_lock):
-					return STATUS.CREATE_TASK_FAILED
-				else:
-					payload = self.simpleStorageService.get_object(object_.key)
-					self.move_to_dead_queue(object_.key, payload)
-					return STATUS.CHECK_KEY_FAILED
-			finally:
-				self.ask_unlock(distributed_lock)
-	
-	def get_payload(self, key: str) -> Dict:
-		return self.simpleStorageService.get_object(key)
-	
-	def ask_lock(self, lock: DistributedLock) -> bool:
-		return lock.try_lock_nowait()
-	
-	def ask_unlock(self, lock: DistributedLock) -> bool:
-		return lock.unlock()
-	
-	def process(self, key: str, code: str, payload: Dict[str, Any] = None):
-		logger.info("start to process %s and %s", code, key)
-		trigger_data = PipelineTriggerData(code=code, data=payload, tenantId=self.tenant_id)
-		result = save_topic_data(trigger_data)
-		asyncio.run(handle_trigger_data(trigger_data, result))
-	
-	def save_data(self, trigger_data: PipelineTriggerData) -> TopicTrigger:
-		return save_topic_data(trigger_data)
-	
-	def trigger_pipeline(self, trigger_data: PipelineTriggerData, topic_trigger: TopicTrigger):
-		try:
-			asyncio.run(handle_trigger_data(trigger_data, topic_trigger))
-		except Exception as e:
-			logger.error(e, exc_info=True, stack_info=True)
-	
-	def get_resource_lock(self, key: str) -> OSSCollectorCompetitiveLock:
-		object_key = self.get_identifier(self.consume_prefix, key)
-		key_parts = object_key.split(identifier_delimiter)
-		return OSSCollectorCompetitiveLock(lockId=self.snowflakeGenerator.next_id(),
-		                                   resourceId=key,
-		                                   modelName=key_parts[1],
-		                                   objectId=key_parts[2],
-		                                   tenantId=self.tenant_id,
-		                                   status=0)
-	
-	def get_dependency(self, key: str) -> Optional[Dependency]:
-		key_parts = key.split(identifier_delimiter)
-		if len(key_parts) == 5:
-			return Dependency(model_name=key_parts[3], object_id=key_parts[4])
-		elif len(key_parts) == 3:
-			return Dependency(model_name=key_parts[1], object_id=key_parts[2])
-		else:
-			return None
-	
-	def check_dependency_finished(self, dependency: Optional[Dependency]) -> bool:
-		if dependency:
-			result = self.lock_service.find_by_dependency(dependency.model_name, dependency.object_id)
-			if result == 0:
-				return True
-			else:
-				return False
-		else:
-			return True
-	
-	def get_code(self, identifier: str) -> str:
-		key_parts = identifier.split(identifier_delimiter)
-		return 'raw_' + key_parts[1].lower()
-	
-	def validate_key_pattern(self, identifier: str) -> bool:
-		key_parts = identifier.split(identifier_delimiter)
-		if len(key_parts) == 3:
-			return True
-		elif len(key_parts) == 5:
-			return True
-		else:
-			return False
-	
-	def move_to_dead_queue(self, key: str, payload: Optional[Dict]):
-		dead_queue_key = self.generate_dead_file_key(key)
-		self.simpleStorageService.put_object(dead_queue_key, payload)
-		self.simpleStorageService.delete_object(key)
-	
-	def generate_dead_file_key(self, key_: str):
-		return self.dead_prefix + self.get_identifier(self.consume_prefix, key_)
-	
-	@staticmethod
-	def get_identifier(prefix, key) -> str:
-		return key.removeprefix(prefix)

watchmen_collector_kernel/lock/__init__.py

@@ -1,3 +0,0 @@
-from .distributed_lock import DistributedLock
-from .oss_collector_lock_service import get_oss_collector_lock_service
-from .unique_key_distributed_lock import get_unique_key_distributed_lock

watchmen_collector_kernel/lock/distributed_lock.py

@@ -1,23 +0,0 @@
-from abc import ABC, abstractmethod
-
-
-class DistributedLock(ABC):
-
-	def __init__(self):
-		pass
-
-	@abstractmethod
-	def lock(self):
-		pass
-
-	@abstractmethod
-	def try_lock(self, timeout: int):
-		pass
-
-	@abstractmethod
-	def try_lock_nowait(self):
-		pass
-
-	@abstractmethod
-	def unlock(self):
-		pass

watchmen_collector_kernel/lock/oss_collector_lock_service.py

@@ -1,127 +0,0 @@
-from datetime import datetime
-from typing import List
-
-from watchmen_meta.common import EntityService, ask_meta_storage
-from watchmen_meta.common.storage_service import StorableId
-from watchmen_model.common import Storable, OssCollectorCompetitiveLockId
-from watchmen_storage import EntityShaper, EntityRow, EntityName, TransactionalStorageSPI, \
-	EntityHelper, EntityIdHelper, EntityFinder, ColumnNameLiteral, EntityCriteriaExpression, Entity
-from watchmen_collector_kernel.model import OSSCollectorCompetitiveLock
-
-
-class OSSCollectorCompetitiveLockShaper(EntityShaper):
-	def serialize(self, entity: OSSCollectorCompetitiveLock) -> EntityRow:
-		return {
-			'lock_id': entity.lockId,
-			'resource_id': entity.resourceId,
-			'model_name': entity.modelName,
-			'object_id': entity.objectId,
-			'registered_at': entity.registeredAt,
-			'tenant_id': entity.tenantId,
-			'status': entity.status
-		}
-	
-	def deserialize(self, row: EntityRow) -> OSSCollectorCompetitiveLock:
-		return OSSCollectorCompetitiveLock(
-			lockId=row.get('lock_id'),
-			resourceId=row.get('resource_id'),
-			modelName=row.get('model_name'),
-			objectId=row.get('object_id'),
-			registeredAt=row.get('registered_at'),
-			tenantId=row.get('tenant_id'),
-			status=row.get('status')
-		)
-
-
-OSS_COLLECTOR_COMPETITIVE_LOCK_TABLE = 'oss_collector_competitive_lock'
-OSS_COLLECTOR_COMPETITIVE_LOCK_ENTITY_SHAPER = OSSCollectorCompetitiveLockShaper()
-
-
-class OssCollectorLockService(EntityService):
-	
-	def __init__(self, storage: TransactionalStorageSPI):
-		super().__init__(storage)
-	
-	def get_entity_name(self) -> EntityName:
-		return OSS_COLLECTOR_COMPETITIVE_LOCK_TABLE
-	
-	def get_entity_shaper(self) -> EntityShaper:
-		return OSS_COLLECTOR_COMPETITIVE_LOCK_ENTITY_SHAPER
-	
-	def get_storable_id_column_name(self) -> EntityName:
-		return 'lock_id'
-	
-	def get_storable_id(self, storable: OSSCollectorCompetitiveLock) -> StorableId:
-		return storable.lockId
-	
-	def set_storable_id(self, storable: OSSCollectorCompetitiveLock,
-	                    storable_id: OssCollectorCompetitiveLockId) -> Storable:
-		storable.lockId = storable_id
-		return storable
-	
-	def insert_one(self, lock: OSSCollectorCompetitiveLock):
-		try:
-			self.storage.connect()
-			self.storage.insert_one(
-				lock,
-				EntityHelper(name=OSS_COLLECTOR_COMPETITIVE_LOCK_TABLE, shaper=OSS_COLLECTOR_COMPETITIVE_LOCK_ENTITY_SHAPER)
-			)
-		finally:
-			self.storage.close()
-	
-	def delete_by_id(self, id_: OssCollectorCompetitiveLockId):
-		try:
-			self.storage.connect()
-			self.storage.delete_by_id(id_,
-			                          EntityIdHelper(idColumnName='lock_id',
-			                                         name=OSS_COLLECTOR_COMPETITIVE_LOCK_TABLE,
-			                                         shaper=OSS_COLLECTOR_COMPETITIVE_LOCK_ENTITY_SHAPER)
-			                          )
-		finally:
-			self.storage.close()
-	
-	def update_one(self, one: Entity) -> int:
-		try:
-			self.storage.connect()
-			self.storage.update_one(one,
-			                        EntityIdHelper(idColumnName='lock_id',
-			                                       name=OSS_COLLECTOR_COMPETITIVE_LOCK_TABLE,
-			                                       shaper=OSS_COLLECTOR_COMPETITIVE_LOCK_ENTITY_SHAPER)
-			                        )
-		finally:
-			self.storage.close()
-	
-	def find_by_dependency(self, model_name: str, object_id: str) -> int:
-		try:
-			self.storage.connect()
-			return self.storage.count(EntityFinder(
-				name=self.get_entity_name(),
-				shaper=self.get_entity_shaper(),
-				criteria=[
-					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='model_name'), right=model_name),
-					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='object_id'), right=object_id),
-					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='status'), right=0)
-				]
-			))
-		finally:
-			self.storage.close()
-
-	def find_completed_task(self, query_date: datetime) -> List:
-		try:
-			self.storage.connect()
-			return self.storage.find(EntityFinder(
-				name=self.get_entity_name(),
-				shaper=self.get_entity_shaper(),
-				criteria=[
-					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='registered_at'),
-					                         operator="less-than",
-					                         right=query_date),
-					EntityCriteriaExpression(left=ColumnNameLiteral(columnName='status'), right=1)
-				]
-			))
-		finally:
-			self.storage.close()
-	
-
-def get_oss_collector_lock_service() -> OssCollectorLockService:
-	return OssCollectorLockService(ask_meta_storage())