watchmen-collector-kernel 16.4.6__py3-none-any.whl → 16.4.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. watchmen_collector_kernel/common/__init__.py +2 -2
  2. watchmen_collector_kernel/common/constants.py +16 -0
  3. watchmen_collector_kernel/model/__init__.py +15 -1
  4. watchmen_collector_kernel/model/change_data_json.py +28 -0
  5. watchmen_collector_kernel/model/change_data_record.py +19 -0
  6. watchmen_collector_kernel/model/collector_model_config.py +12 -0
  7. watchmen_collector_kernel/model/collector_table_config.py +78 -0
  8. watchmen_collector_kernel/model/competitive_lock.py +11 -0
  9. watchmen_collector_kernel/model/condition.py +73 -0
  10. watchmen_collector_kernel/model/scheduled_task.py +36 -0
  11. watchmen_collector_kernel/model/trigger_event.py +11 -0
  12. watchmen_collector_kernel/model/trigger_model.py +8 -0
  13. watchmen_collector_kernel/model/trigger_table.py +11 -0
  14. watchmen_collector_kernel/service/__init__.py +8 -0
  15. watchmen_collector_kernel/service/criteria_builder.py +53 -0
  16. watchmen_collector_kernel/service/data_capture.py +60 -0
  17. watchmen_collector_kernel/service/extract_source.py +71 -0
  18. watchmen_collector_kernel/service/extract_utils.py +70 -0
  19. watchmen_collector_kernel/service/lock_clean.py +39 -0
  20. watchmen_collector_kernel/service/lock_helper.py +28 -0
  21. watchmen_collector_kernel/service/task_housekeeping.py +61 -0
  22. watchmen_collector_kernel/service/task_service.py +72 -0
  23. watchmen_collector_kernel/service/trigger_collector.py +147 -0
  24. watchmen_collector_kernel/storage/__init__.py +17 -0
  25. watchmen_collector_kernel/storage/change_data_json_service.py +171 -0
  26. watchmen_collector_kernel/storage/change_data_record_service.py +185 -0
  27. watchmen_collector_kernel/storage/collector_model_config_service.py +107 -0
  28. watchmen_collector_kernel/storage/collector_table_config_service.py +168 -0
  29. watchmen_collector_kernel/storage/competitive_lock_service.py +83 -0
  30. watchmen_collector_kernel/storage/scheduled_task_service.py +155 -0
  31. watchmen_collector_kernel/storage/trigger_event_service.py +86 -0
  32. watchmen_collector_kernel/storage/trigger_model_service.py +96 -0
  33. watchmen_collector_kernel/storage/trigger_table_service.py +113 -0
  34. {watchmen_collector_kernel-16.4.6.dist-info → watchmen_collector_kernel-16.4.8.dist-info}/LICENSE +0 -0
  35. {watchmen_collector_kernel-16.4.6.dist-info → watchmen_collector_kernel-16.4.8.dist-info}/METADATA +9 -9
  36. watchmen_collector_kernel-16.4.8.dist-info/RECORD +38 -0
  37. {watchmen_collector_kernel-16.4.6.dist-info → watchmen_collector_kernel-16.4.8.dist-info}/WHEEL +0 -0
  38. watchmen_collector_kernel/common/settings.py +0 -14
  39. watchmen_collector_kernel/connector/__init__.py +0 -1
  40. watchmen_collector_kernel/connector/handler.py +0 -42
  41. watchmen_collector_kernel/connector/housekeeping.py +0 -58
  42. watchmen_collector_kernel/connector/s3_connector.py +0 -210
  43. watchmen_collector_kernel/lock/__init__.py +0 -3
  44. watchmen_collector_kernel/lock/distributed_lock.py +0 -23
  45. watchmen_collector_kernel/lock/oss_collector_lock_service.py +0 -127
  46. watchmen_collector_kernel/lock/unique_key_distributed_lock.py +0 -39
  47. watchmen_collector_kernel/model/oss_collector_competitive_lock.py +0 -15
  48. watchmen_collector_kernel-16.4.6.dist-info/RECORD +0 -17
@@ -0,0 +1,107 @@
1
+ from typing import Optional, List
2
+
3
+ from watchmen_auth import PrincipalService
4
+ from watchmen_collector_kernel.model import CollectorModelConfig
5
+ from watchmen_meta.common import TupleService, TupleShaper
6
+ from watchmen_meta.common.storage_service import StorableId
7
+ from watchmen_model.common import Storable, TenantId, CollectorModelConfigId
8
+ from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
9
+ EntityCriteriaExpression, ColumnNameLiteral
10
+
11
+
12
+ class CollectorModelConfigShaper(EntityShaper):
13
+ def serialize(self, config: CollectorModelConfig) -> EntityRow:
14
+ return TupleShaper.serialize_tenant_based(config, {
15
+ 'model_id': config.modelId,
16
+ 'model_name': config.modelName,
17
+ 'depend_on': config.dependOn,
18
+ 'raw_topic_code': config.rawTopicCode,
19
+ 'is_paralleled': config.isParalleled
20
+ })
21
+
22
+ def deserialize(self, row: EntityRow) -> CollectorModelConfig:
23
+ # noinspection PyTypeChecker
24
+ return TupleShaper.deserialize_tenant_based(row, CollectorModelConfig(
25
+ modelId=row.get('model_id'),
26
+ modelName=row.get('model_name'),
27
+ dependOn=row.get('depend_on'),
28
+ rawTopicCode=row.get('raw_topic_code'),
29
+ isParalleled=row.get('is_paralleled')
30
+ ))
31
+
32
+
33
+ COLLECTOR_MODEL_CONFIG_ENTITY_NAME = 'collector_model_config'
34
+ COLLECTOR_MODEL_CONFIG_ENTITY_SHAPER = CollectorModelConfigShaper()
35
+
36
+
37
+ class CollectorModelConfigService(TupleService):
38
+ def should_record_operation(self) -> bool:
39
+ return False
40
+
41
+ def get_entity_name(self) -> EntityName:
42
+ return COLLECTOR_MODEL_CONFIG_ENTITY_NAME
43
+
44
+ def get_entity_shaper(self) -> EntityShaper:
45
+ return COLLECTOR_MODEL_CONFIG_ENTITY_SHAPER
46
+
47
+ # noinspection SpellCheckingInspection
48
+ def get_storable_id_column_name(self) -> EntityName:
49
+ return "model_id"
50
+
51
+ def get_storable_id(self, storable: CollectorModelConfig) -> StorableId:
52
+ return storable.modelId
53
+
54
+ # noinspection SpellCheckingInspection
55
+ def set_storable_id(self, storable: CollectorModelConfig, storable_id: CollectorModelConfigId) -> Storable:
56
+ storable.modelId = storable_id
57
+ return storable
58
+
59
+ def create_model_config(self, model_config: CollectorModelConfig) -> CollectorModelConfig:
60
+ try:
61
+ self.storage.connect()
62
+ # noinspection PyTypeChecker
63
+ return self.create(model_config)
64
+ finally:
65
+ self.storage.close()
66
+
67
+ def update_model_config(self, model_config: CollectorModelConfig) -> CollectorModelConfig:
68
+ try:
69
+ self.storage.connect()
70
+ # noinspection PyTypeChecker
71
+ return self.update(model_config)
72
+ finally:
73
+ self.storage.close()
74
+
75
+ def find_by_model_id(self, model_id: str) -> Optional[CollectorModelConfig]:
76
+ self.begin_transaction()
77
+ try:
78
+ return self.storage.find_one(self.get_entity_finder(
79
+ criteria=[
80
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='model_id'), right=model_id)]
81
+ ))
82
+ finally:
83
+ self.close_transaction()
84
+
85
+ def find_by_tenant(self, tenant_id: TenantId) -> Optional[List[CollectorModelConfig]]:
86
+ # noinspection PyTypeChecker
87
+ return self.storage.find(self.get_entity_finder(
88
+ criteria=[
89
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id)]
90
+ ))
91
+
92
+ def find_by_name(self, model_name: str) -> Optional[CollectorModelConfig]:
93
+ self.begin_transaction()
94
+ try:
95
+ return self.storage.find_one(self.get_entity_finder(
96
+ criteria=[
97
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='model_name'), right=model_name)]
98
+ ))
99
+ finally:
100
+ self.close_transaction()
101
+
102
+
103
+ def get_collector_model_config_service(storage: TransactionalStorageSPI,
104
+ snowflake_generator: SnowflakeGenerator,
105
+ principal_service: PrincipalService
106
+ ) -> CollectorModelConfigService:
107
+ return CollectorModelConfigService(storage, snowflake_generator, principal_service)
@@ -0,0 +1,168 @@
1
+ from typing import Optional, List
2
+
3
+ from watchmen_auth import PrincipalService
4
+ from watchmen_collector_kernel.model import CollectorTableConfig
5
+ from watchmen_meta.common import TupleService, TupleShaper
6
+ from watchmen_meta.common.storage_service import StorableId
7
+ from watchmen_model.common import Storable
8
+ from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
9
+ EntityCriteriaExpression, ColumnNameLiteral
10
+ from watchmen_utilities import ArrayHelper
11
+
12
+
13
+ class CollectorTableConfigShaper(EntityShaper):
14
+ def serialize(self, config: CollectorTableConfig) -> EntityRow:
15
+ return TupleShaper.serialize_tenant_based(config, {
16
+ 'config_id': config.configId,
17
+ 'name': config.name,
18
+ 'table_name': config.tableName,
19
+ 'primary_key': config.primaryKey,
20
+ 'object_key': config.objectKey,
21
+ 'model_name': config.modelName,
22
+ 'parent_name': config.parentName,
23
+ 'join_keys': ArrayHelper(config.joinKeys).map(lambda x: x.to_dict()).to_list(),
24
+ 'depend_on': ArrayHelper(config.dependOn).map(lambda x: x.to_dict()).to_list(),
25
+ 'conditions': ArrayHelper(config.conditions).map(lambda x: x.to_dict()).to_list(),
26
+ 'label': config.label,
27
+ 'audit_column': config.auditColumn,
28
+ 'data_source_id': config.dataSourceId,
29
+ 'is_list': config.isList,
30
+ 'triggered': config.triggered
31
+ })
32
+
33
+ def deserialize(self, row: EntityRow) -> CollectorTableConfig:
34
+ # noinspection PyTypeChecker
35
+ return TupleShaper.deserialize_tenant_based(row, CollectorTableConfig(
36
+ configId=row.get('config_id'),
37
+ name=row.get('name'),
38
+ tableName=row.get('table_name'),
39
+ primaryKey=row.get('primary_key'),
40
+ objectKey=row.get('object_key'),
41
+ modelName=row.get('model_name'),
42
+ parentName=row.get('parent_name'),
43
+ joinKeys=row.get('join_keys'),
44
+ dependOn=row.get('depend_on'),
45
+ conditions=row.get('conditions'),
46
+ label=row.get('label'),
47
+ auditColumn=row.get('audit_column'),
48
+ dataSourceId=row.get('data_source_id'),
49
+ isList=row.get('is_list'),
50
+ triggered=row.get('triggered')
51
+ ))
52
+
53
+
54
+ COLLECTOR_TABLE_CONFIG_ENTITY_NAME = 'collector_table_config'
55
+ COLLECTOR_TABLE_CONFIG_ENTITY_SHAPER = CollectorTableConfigShaper()
56
+
57
+
58
+ class CollectorTableConfigService(TupleService):
59
+ def should_record_operation(self) -> bool:
60
+ return False
61
+
62
+ def get_entity_name(self) -> EntityName:
63
+ return COLLECTOR_TABLE_CONFIG_ENTITY_NAME
64
+
65
+ def get_entity_shaper(self) -> EntityShaper:
66
+ return COLLECTOR_TABLE_CONFIG_ENTITY_SHAPER
67
+
68
+ # noinspection SpellCheckingInspection
69
+ def get_storable_id_column_name(self) -> EntityName:
70
+ return "config_id"
71
+
72
+ def get_storable_id(self, storable: CollectorTableConfig) -> StorableId:
73
+ return storable.configId
74
+
75
+ # noinspection SpellCheckingInspection
76
+ def set_storable_id(self, storable: CollectorTableConfig, storable_id: StorableId) -> Storable:
77
+ storable.configId = storable_id
78
+ return storable
79
+
80
+ # noinspection PyTypeChecker
81
+ def create_config(self, config: CollectorTableConfig) -> CollectorTableConfig:
82
+ self.begin_transaction()
83
+ try:
84
+ config = self.create(config)
85
+ self.commit_transaction()
86
+ return config
87
+ except Exception as e:
88
+ self.rollback_transaction()
89
+ raise e
90
+
91
+ # noinspection PyTypeChecker
92
+ def update_config(self, config: CollectorTableConfig) -> CollectorTableConfig:
93
+ self.begin_transaction()
94
+ try:
95
+ config = self.update(config)
96
+ self.commit_transaction()
97
+ return config
98
+ except Exception as e:
99
+ self.rollback_transaction()
100
+ raise e
101
+
102
+ def find_config_by_id(self, config_id: str) -> Optional[CollectorTableConfig]:
103
+ self.begin_transaction()
104
+ try:
105
+ return self.find_by_id(config_id)
106
+ finally:
107
+ self.close_transaction()
108
+
109
+ def find_by_table_name(self, table_name: str) -> Optional[CollectorTableConfig]:
110
+ try:
111
+ self.storage.connect()
112
+ # noinspection PyTypeChecker
113
+ return self.storage.find_one(
114
+ self.get_entity_finder(
115
+ criteria=[
116
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='table_name'),
117
+ right=table_name)
118
+ ]
119
+ )
120
+ )
121
+ finally:
122
+ self.storage.close()
123
+
124
+ def find_by_name(self, name: str) -> Optional[CollectorTableConfig]:
125
+ try:
126
+ self.storage.connect()
127
+ # noinspection PyTypeChecker
128
+ return self.storage.find_one(
129
+ self.get_entity_finder(
130
+ criteria=[
131
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='name'),
132
+ right=name)
133
+ ]
134
+ )
135
+ )
136
+ finally:
137
+ self.storage.close()
138
+
139
+ def find_by_parent_name(self, parent_name: str) -> Optional[List[CollectorTableConfig]]:
140
+ try:
141
+ self.storage.connect()
142
+ # noinspection PyTypeChecker
143
+ return self.storage.find(
144
+ self.get_entity_finder(
145
+ criteria=[
146
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='parent_name'),
147
+ right=parent_name)
148
+ ]
149
+ )
150
+ )
151
+ finally:
152
+ self.storage.close()
153
+
154
+ def find_by_model_name(self,
155
+ model_name: str) -> Optional[List[CollectorTableConfig]]:
156
+ # noinspection PyTypeChecker
157
+ return self.storage.find(self.get_entity_finder(
158
+ criteria=[
159
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'), right=self.principalService.get_tenant_id()),
160
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='model_name'), right=model_name)]
161
+ ))
162
+
163
+
164
+ def get_collector_table_config_service(storage: TransactionalStorageSPI,
165
+ snowflake_generator: SnowflakeGenerator,
166
+ principal_service: PrincipalService
167
+ ) -> CollectorTableConfigService:
168
+ return CollectorTableConfigService(storage, snowflake_generator, principal_service)
@@ -0,0 +1,83 @@
1
+ from datetime import datetime
2
+ from typing import List
3
+
4
+ from watchmen_collector_kernel.model import CompetitiveLock
5
+ from watchmen_meta.common import ask_meta_storage, EntityService
6
+ from watchmen_meta.common.storage_service import StorableId
7
+ from watchmen_model.common import CompetitiveLockId, Storable
8
+ from watchmen_storage import ColumnNameLiteral, Entity, EntityCriteriaExpression, EntityCriteriaOperator, \
9
+ EntityName, EntityRow, EntityShaper, TransactionalStorageSPI
10
+
11
+
12
+ class CompetitiveLockShaper(EntityShaper):
13
+ def serialize(self, entity: CompetitiveLock) -> EntityRow:
14
+ return {
15
+ 'lock_id': entity.lockId,
16
+ 'resource_id': entity.resourceId,
17
+ 'registered_at': entity.registeredAt,
18
+ 'tenant_id': entity.tenantId
19
+ }
20
+
21
+ def deserialize(self, row: EntityRow) -> CompetitiveLock:
22
+ return CompetitiveLock(
23
+ lockId=row.get('lock_id'),
24
+ resourceId=row.get('resource_id'),
25
+ registeredAt=row.get('registered_at'),
26
+ tenantId=row.get('tenant_id')
27
+ )
28
+
29
+
30
+ COMPETITIVE_LOCK_TABLE = 'competitive_lock'
31
+ COMPETITIVE_LOCK_ENTITY_SHAPER = CompetitiveLockShaper()
32
+
33
+
34
+ class CompetitiveLockService(EntityService):
35
+
36
+ def __init__(self, storage: TransactionalStorageSPI):
37
+ super().__init__(storage)
38
+
39
+ def get_entity_name(self) -> EntityName:
40
+ return COMPETITIVE_LOCK_TABLE
41
+
42
+ def get_entity_shaper(self) -> EntityShaper:
43
+ return COMPETITIVE_LOCK_ENTITY_SHAPER
44
+
45
+ def get_storable_id_column_name(self) -> EntityName:
46
+ return 'lock_id'
47
+
48
+ def get_storable_id(self, storable: CompetitiveLock) -> StorableId:
49
+ return storable.lockId
50
+
51
+ def set_storable_id(
52
+ self, storable: CompetitiveLock, storable_id: CompetitiveLockId) -> Storable:
53
+ storable.lockId = storable_id
54
+ return storable
55
+
56
+ def insert_one(self, lock: CompetitiveLock):
57
+ try:
58
+ self.storage.connect()
59
+ self.storage.insert_one(lock, self.get_entity_helper())
60
+ finally:
61
+ self.storage.close()
62
+
63
+ def delete_by_id(self, lock_id: CompetitiveLockId):
64
+ try:
65
+ self.storage.connect()
66
+ self.storage.delete_by_id(lock_id, self.get_entity_id_helper())
67
+ finally:
68
+ self.storage.close()
69
+
70
+ def find_overtime_lock(self, query_time: datetime) -> List:
71
+ try:
72
+ self.storage.connect()
73
+ return self.storage.find(self.get_entity_finder(criteria=[
74
+ EntityCriteriaExpression(
75
+ left=ColumnNameLiteral(columnName='registered_at'),
76
+ operator=EntityCriteriaOperator.LESS_THAN, right=query_time)
77
+ ]))
78
+ finally:
79
+ self.storage.close()
80
+
81
+
82
+ def get_competitive_lock_service(storage: TransactionalStorageSPI) -> CompetitiveLockService:
83
+ return CompetitiveLockService(storage)
@@ -0,0 +1,155 @@
1
+ from typing import Optional, List, Any, Dict
2
+
3
+ from watchmen_auth import PrincipalService
4
+ from watchmen_collector_kernel.model import ScheduledTask
5
+ from watchmen_meta.common import TupleService, TupleShaper
6
+ from watchmen_meta.common.storage_service import StorableId
7
+ from watchmen_model.common import Storable, ScheduledTaskId
8
+ from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, \
9
+ EntityCriteriaExpression, ColumnNameLiteral, SnowflakeGenerator, EntitySortColumn, EntitySortMethod, \
10
+ EntityCriteriaJoint, EntityCriteriaJointConjunction, EntityStraightValuesFinder, EntityStraightColumn, \
11
+ EntityCriteriaOperator
12
+
13
+
14
+ class ScheduledTaskShaper(EntityShaper):
15
+ def serialize(self, entity: ScheduledTask) -> EntityRow:
16
+ return TupleShaper.serialize_tenant_based(entity, {
17
+ 'task_id': entity.taskId,
18
+ 'resource_id': entity.resourceId,
19
+ 'topic_code': entity.topicCode,
20
+ 'content': entity.content,
21
+ 'model_name': entity.modelName,
22
+ 'object_id': entity.objectId,
23
+ 'depend_on': entity.dependOn,
24
+ 'parent_task_id': entity.parentTaskId,
25
+ 'tenant_id': entity.tenantId,
26
+ 'status': entity.status,
27
+ 'result': entity.result
28
+ })
29
+
30
+ def deserialize(self, row: EntityRow) -> ScheduledTask:
31
+ # noinspection PyTypeChecker
32
+ return TupleShaper.deserialize_tenant_based(row, ScheduledTask(
33
+ taskId=row.get('task_id'),
34
+ resourceId=row.get('resource_id'),
35
+ topicCode=row.get('topic_code'),
36
+ content=row.get('content'),
37
+ modelName=row.get('model_name'),
38
+ objectId=row.get('object_id'),
39
+ dependOn=row.get('depend_on'),
40
+ parentTaskId=row.get('parent_task_id'),
41
+ tenantId=row.get('tenant_id'),
42
+ status=row.get('status'),
43
+ result=row.get('result')
44
+ ))
45
+
46
+
47
+ SCHEDULED_TASK_TABLE = 'scheduled_task'
48
+ SCHEDULED_TASK_ENTITY_SHAPER = ScheduledTaskShaper()
49
+
50
+
51
+ class ScheduledTaskService(TupleService):
52
+
53
+ def should_record_operation(self) -> bool:
54
+ return False
55
+
56
+ def get_entity_name(self) -> EntityName:
57
+ return SCHEDULED_TASK_TABLE
58
+
59
+ def get_entity_shaper(self) -> EntityShaper:
60
+ return SCHEDULED_TASK_ENTITY_SHAPER
61
+
62
+ def get_storable_id_column_name(self) -> EntityName:
63
+ return 'task_id'
64
+
65
+ def get_storable_id(self, storable: ScheduledTask) -> StorableId:
66
+ return storable.taskId
67
+
68
+ def set_storable_id(
69
+ self, storable: ScheduledTask, storable_id: ScheduledTaskId) -> Storable:
70
+ storable.taskId = storable_id
71
+ return storable
72
+
73
+ def create_task(self, task: ScheduledTask) -> ScheduledTask:
74
+ self.begin_transaction()
75
+ try:
76
+ task = self.create(task)
77
+ self.commit_transaction()
78
+ # noinspection PyTypeChecker
79
+ return task
80
+ except Exception as e:
81
+ self.rollback_transaction()
82
+ raise e
83
+
84
+ def update_task(self, task: ScheduledTask) -> ScheduledTask:
85
+ self.begin_transaction()
86
+ try:
87
+ task = self.update(task)
88
+ self.commit_transaction()
89
+ # noinspection PyTypeChecker
90
+ return task
91
+ except Exception as e:
92
+ self.rollback_transaction()
93
+ raise e
94
+
95
+ def find_task_by_id(self, task_id: ScheduledTaskId) -> Optional[ScheduledTask]:
96
+ self.begin_transaction()
97
+ try:
98
+ return self.find_by_id(task_id)
99
+ finally:
100
+ self.close_transaction()
101
+
102
+ def find_unfinished_tasks(self) -> List[Dict[str, Any]]:
103
+ self.begin_transaction()
104
+ try:
105
+ return self.storage.find_straight_values(EntityStraightValuesFinder(
106
+ name=self.get_entity_name(),
107
+ criteria=[EntityCriteriaJoint(
108
+ children=[
109
+ EntityCriteriaExpression(
110
+ left=ColumnNameLiteral(columnName='status'), right=0)
111
+ ]
112
+ )],
113
+ straightColumns=[EntityStraightColumn(columnName='task_id'),
114
+ EntityStraightColumn(columnName='resource_id'),
115
+ EntityStraightColumn(columnName='tenant_id')],
116
+ sort=[EntitySortColumn(name='resource_id', method=EntitySortMethod.ASC)]
117
+ ))
118
+ finally:
119
+ self.close_transaction()
120
+
121
+ def find_by_resource_id(self, resource_id: str) -> List[Dict[str, Any]]:
122
+ self.begin_transaction()
123
+ try:
124
+ return self.storage.find_straight_values(EntityStraightValuesFinder(
125
+ name=self.get_entity_name(),
126
+ criteria=[EntityCriteriaExpression(left=ColumnNameLiteral(columnName='resource_id'), right=resource_id)],
127
+ straightColumns=['task_id', 'resource_id', 'tenant_id']
128
+ ))
129
+ finally:
130
+ self.close_transaction()
131
+
132
+ def is_dependent_task_finished(self, model_name: str, object_id: str, tenant_id: str) -> bool:
133
+ self.begin_transaction()
134
+ try:
135
+ # noinspection PyTypeChecker
136
+ return self.storage.count(self.get_entity_finder(
137
+ criteria=[
138
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='model_name'),
139
+ right=model_name),
140
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='object_id'),
141
+ right=object_id),
142
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='status'),
143
+ right=0),
144
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'),
145
+ right=tenant_id)
146
+ ]
147
+ )) == 0
148
+ finally:
149
+ self.close_transaction()
150
+
151
+ def get_scheduled_task_service(storage: TransactionalStorageSPI,
152
+ snowflake_generator: SnowflakeGenerator,
153
+ principal_service: PrincipalService
154
+ ) -> ScheduledTaskService:
155
+ return ScheduledTaskService(storage, snowflake_generator, principal_service)
@@ -0,0 +1,86 @@
1
+ from typing import List, Dict, Any
2
+
3
+ from watchmen_auth import PrincipalService
4
+ from watchmen_collector_kernel.model import TriggerEvent
5
+ from watchmen_meta.common import TupleShaper, TupleService
6
+ from watchmen_meta.common.storage_service import StorableId
7
+ from watchmen_model.common import Storable, EventTriggerId
8
+ from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
9
+ EntityStraightValuesFinder, EntityCriteriaExpression, ColumnNameLiteral, \
10
+ EntityStraightColumn, EntitySortColumn, EntitySortMethod
11
+
12
+
13
+ class TriggerEventShaper(EntityShaper):
14
+
15
+ def serialize(self, entity: TriggerEvent) -> EntityRow:
16
+ return TupleShaper.serialize_tenant_based(entity, {
17
+ 'event_trigger_id': entity.eventTriggerId,
18
+ 'start_time': entity.startTime,
19
+ 'end_time': entity.endTime,
20
+ 'is_finished': entity.isFinished
21
+ })
22
+
23
+ def deserialize(self, row: EntityRow) -> TriggerEvent:
24
+ # noinspection PyTypeChecker
25
+ return TupleShaper.deserialize_tenant_based(row, TriggerEvent(
26
+ eventTriggerId=row.get('event_trigger_id'),
27
+ startTime=row.get('start_time'),
28
+ endTime=row.get('end_time'),
29
+ isFinished=row.get('is_finished')
30
+ ))
31
+
32
+
33
+ TRIGGER_EVENT_TABLE = 'trigger_event'
34
+ TRIGGER_EVENT_ENTITY_SHAPER = TriggerEventShaper()
35
+
36
+
37
+ class TriggerEventService(TupleService):
38
+
39
+ def should_record_operation(self) -> bool:
40
+ return False
41
+
42
+ def get_entity_name(self) -> EntityName:
43
+ return TRIGGER_EVENT_TABLE
44
+
45
+ def get_entity_shaper(self) -> EntityShaper:
46
+ return TRIGGER_EVENT_ENTITY_SHAPER
47
+
48
+ def get_storable_id_column_name(self) -> EntityName:
49
+ return 'event_trigger_id'
50
+
51
+ def get_storable_id(self, storable: TriggerEvent) -> StorableId:
52
+ # noinspection PyTypeChecker
53
+ return storable.eventTriggerId
54
+
55
+ def set_storable_id(
56
+ self, storable: TriggerEvent, storable_id: EventTriggerId) -> Storable:
57
+ storable.eventTriggerId = storable_id
58
+ return storable
59
+
60
+ def find_event_by_id(self, event_trigger_id: EventTriggerId) -> TriggerEvent:
61
+ try:
62
+ self.storage.connect()
63
+ # noinspection PyTypeChecker
64
+ return self.storage.find_by_id(event_trigger_id, self.get_entity_id_helper())
65
+ finally:
66
+ self.storage.close()
67
+
68
+ def find_unfinished_events(self) -> List[Dict[str, Any]]:
69
+ self.begin_transaction()
70
+ try:
71
+ return self.storage.find_straight_values(EntityStraightValuesFinder(
72
+ name=self.get_entity_name(),
73
+ criteria=[EntityCriteriaExpression(left=ColumnNameLiteral(columnName='is_finished'), right=False)],
74
+ straightColumns=[EntityStraightColumn(columnName=self.get_storable_id_column_name()),
75
+ EntityStraightColumn(columnName='tenant_id')],
76
+ sort=[EntitySortColumn(name=self.get_storable_id_column_name(), method=EntitySortMethod.ASC)]
77
+ ))
78
+ finally:
79
+ self.close_transaction()
80
+
81
+
82
+ def get_trigger_event_service(storage: TransactionalStorageSPI,
83
+ snowflake_generator: SnowflakeGenerator,
84
+ principal_service: PrincipalService
85
+ ) -> TriggerEventService:
86
+ return TriggerEventService(storage, snowflake_generator, principal_service)
@@ -0,0 +1,96 @@
1
+ from typing import Optional, List
2
+
3
+ from watchmen_auth import PrincipalService
4
+ from watchmen_collector_kernel.common import IS_FINISHED, TENANT_ID
5
+ from watchmen_collector_kernel.model import TriggerModel
6
+ from watchmen_meta.common import TupleShaper, TupleService
7
+ from watchmen_meta.common.storage_service import StorableId
8
+ from watchmen_model.common import Storable, ModelTriggerId
9
+ from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
10
+ EntityCriteriaExpression, ColumnNameLiteral
11
+
12
+
13
+ class TriggerModelShaper(EntityShaper):
14
+
15
+ def serialize(self, entity: TriggerModel) -> EntityRow:
16
+ return TupleShaper.serialize_tenant_based(entity, {
17
+ 'model_trigger_id': entity.modelTriggerId,
18
+ 'model_name': entity.modelName,
19
+ 'is_finished': entity.isFinished,
20
+ 'event_trigger_id': entity.eventTriggerId
21
+ })
22
+
23
+ def deserialize(self, row: EntityRow) -> TriggerModel:
24
+ # noinspection PyTypeChecker
25
+ return TupleShaper.deserialize_tenant_based(row, TriggerModel(
26
+ modelTriggerId=row.get('model_trigger_id'),
27
+ modelName=row.get('model_name'),
28
+ isFinished=row.get('is_finished'),
29
+ eventTriggerId=row.get('event_trigger_id')
30
+ ))
31
+
32
+
33
+ TRIGGER_MODEL_TABLE = 'trigger_model'
34
+ TRIGGER_MODEL_ENTITY_SHAPER = TriggerModelShaper()
35
+
36
+
37
+ class TriggerModelService(TupleService):
38
+
39
+ def should_record_operation(self) -> bool:
40
+ return False
41
+
42
+ def get_entity_name(self) -> EntityName:
43
+ return TRIGGER_MODEL_TABLE
44
+
45
+ def get_entity_shaper(self) -> EntityShaper:
46
+ return TRIGGER_MODEL_ENTITY_SHAPER
47
+
48
+ def get_storable_id_column_name(self) -> EntityName:
49
+ return 'model_trigger_id'
50
+
51
+ def get_storable_id(self, storable: TriggerModel) -> StorableId:
52
+ # noinspection PyTypeChecker
53
+ return storable.modelTriggerId
54
+
55
+ def set_storable_id(
56
+ self, storable: TriggerModel, storable_id: ModelTriggerId) -> Storable:
57
+ storable.modelTriggerId = storable_id
58
+ return storable
59
+
60
+ def find_trigger_by_id(self, trigger_id: int) -> Optional[TriggerModel]:
61
+ self.begin_transaction()
62
+ try:
63
+ return self.find_by_id(trigger_id)
64
+ finally:
65
+ self.close_transaction()
66
+
67
+ def is_finished(self, event_trigger_id: str) -> bool:
68
+ self.begin_transaction()
69
+ try:
70
+ return self.storage.count(self.get_entity_finder(
71
+ criteria=[
72
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName=IS_FINISHED), right=False),
73
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='event_trigger_id'), right=event_trigger_id)
74
+ ]
75
+ )) == 0
76
+ finally:
77
+ self.close_transaction()
78
+
79
+ def find_by_event_trigger_id(self, event_trigger_id: int) -> List[TriggerModel]:
80
+ self.begin_transaction()
81
+ try:
82
+ # noinspection PyTypeChecker
83
+ return self.storage.find(self.get_entity_finder(
84
+ criteria=[
85
+ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='event_trigger_id'), right=event_trigger_id)
86
+ ]
87
+ ))
88
+ finally:
89
+ self.close_transaction()
90
+
91
+
92
+ def get_trigger_model_service(storage: TransactionalStorageSPI,
93
+ snowflake_generator: SnowflakeGenerator,
94
+ principal_service: PrincipalService
95
+ ) -> TriggerModelService:
96
+ return TriggerModelService(storage, snowflake_generator, principal_service)