watchmen-collector-kernel 16.4.6__py3-none-any.whl → 16.4.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- watchmen_collector_kernel/common/__init__.py +2 -2
- watchmen_collector_kernel/common/constants.py +16 -0
- watchmen_collector_kernel/model/__init__.py +15 -1
- watchmen_collector_kernel/model/change_data_json.py +28 -0
- watchmen_collector_kernel/model/change_data_record.py +19 -0
- watchmen_collector_kernel/model/collector_model_config.py +12 -0
- watchmen_collector_kernel/model/collector_table_config.py +78 -0
- watchmen_collector_kernel/model/competitive_lock.py +11 -0
- watchmen_collector_kernel/model/condition.py +73 -0
- watchmen_collector_kernel/model/scheduled_task.py +36 -0
- watchmen_collector_kernel/model/trigger_event.py +11 -0
- watchmen_collector_kernel/model/trigger_model.py +8 -0
- watchmen_collector_kernel/model/trigger_table.py +11 -0
- watchmen_collector_kernel/service/__init__.py +8 -0
- watchmen_collector_kernel/service/criteria_builder.py +53 -0
- watchmen_collector_kernel/service/data_capture.py +60 -0
- watchmen_collector_kernel/service/extract_source.py +71 -0
- watchmen_collector_kernel/service/extract_utils.py +70 -0
- watchmen_collector_kernel/service/lock_clean.py +39 -0
- watchmen_collector_kernel/service/lock_helper.py +28 -0
- watchmen_collector_kernel/service/task_housekeeping.py +61 -0
- watchmen_collector_kernel/service/task_service.py +72 -0
- watchmen_collector_kernel/service/trigger_collector.py +147 -0
- watchmen_collector_kernel/storage/__init__.py +17 -0
- watchmen_collector_kernel/storage/change_data_json_service.py +171 -0
- watchmen_collector_kernel/storage/change_data_record_service.py +185 -0
- watchmen_collector_kernel/storage/collector_model_config_service.py +107 -0
- watchmen_collector_kernel/storage/collector_table_config_service.py +168 -0
- watchmen_collector_kernel/storage/competitive_lock_service.py +83 -0
- watchmen_collector_kernel/storage/scheduled_task_service.py +155 -0
- watchmen_collector_kernel/storage/trigger_event_service.py +86 -0
- watchmen_collector_kernel/storage/trigger_model_service.py +96 -0
- watchmen_collector_kernel/storage/trigger_table_service.py +113 -0
- {watchmen_collector_kernel-16.4.6.dist-info → watchmen_collector_kernel-16.4.8.dist-info}/LICENSE +0 -0
- {watchmen_collector_kernel-16.4.6.dist-info → watchmen_collector_kernel-16.4.8.dist-info}/METADATA +9 -9
- watchmen_collector_kernel-16.4.8.dist-info/RECORD +38 -0
- {watchmen_collector_kernel-16.4.6.dist-info → watchmen_collector_kernel-16.4.8.dist-info}/WHEEL +0 -0
- watchmen_collector_kernel/common/settings.py +0 -14
- watchmen_collector_kernel/connector/__init__.py +0 -1
- watchmen_collector_kernel/connector/handler.py +0 -42
- watchmen_collector_kernel/connector/housekeeping.py +0 -58
- watchmen_collector_kernel/connector/s3_connector.py +0 -210
- watchmen_collector_kernel/lock/__init__.py +0 -3
- watchmen_collector_kernel/lock/distributed_lock.py +0 -23
- watchmen_collector_kernel/lock/oss_collector_lock_service.py +0 -127
- watchmen_collector_kernel/lock/unique_key_distributed_lock.py +0 -39
- watchmen_collector_kernel/model/oss_collector_competitive_lock.py +0 -15
- watchmen_collector_kernel-16.4.6.dist-info/RECORD +0 -17
@@ -0,0 +1,107 @@
|
|
1
|
+
from typing import Optional, List
|
2
|
+
|
3
|
+
from watchmen_auth import PrincipalService
|
4
|
+
from watchmen_collector_kernel.model import CollectorModelConfig
|
5
|
+
from watchmen_meta.common import TupleService, TupleShaper
|
6
|
+
from watchmen_meta.common.storage_service import StorableId
|
7
|
+
from watchmen_model.common import Storable, TenantId, CollectorModelConfigId
|
8
|
+
from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
|
9
|
+
EntityCriteriaExpression, ColumnNameLiteral
|
10
|
+
|
11
|
+
|
12
|
+
class CollectorModelConfigShaper(EntityShaper):
|
13
|
+
def serialize(self, config: CollectorModelConfig) -> EntityRow:
|
14
|
+
return TupleShaper.serialize_tenant_based(config, {
|
15
|
+
'model_id': config.modelId,
|
16
|
+
'model_name': config.modelName,
|
17
|
+
'depend_on': config.dependOn,
|
18
|
+
'raw_topic_code': config.rawTopicCode,
|
19
|
+
'is_paralleled': config.isParalleled
|
20
|
+
})
|
21
|
+
|
22
|
+
def deserialize(self, row: EntityRow) -> CollectorModelConfig:
|
23
|
+
# noinspection PyTypeChecker
|
24
|
+
return TupleShaper.deserialize_tenant_based(row, CollectorModelConfig(
|
25
|
+
modelId=row.get('model_id'),
|
26
|
+
modelName=row.get('model_name'),
|
27
|
+
dependOn=row.get('depend_on'),
|
28
|
+
rawTopicCode=row.get('raw_topic_code'),
|
29
|
+
isParalleled=row.get('is_paralleled')
|
30
|
+
))
|
31
|
+
|
32
|
+
|
33
|
+
COLLECTOR_MODEL_CONFIG_ENTITY_NAME = 'collector_model_config'
|
34
|
+
COLLECTOR_MODEL_CONFIG_ENTITY_SHAPER = CollectorModelConfigShaper()
|
35
|
+
|
36
|
+
|
37
|
+
class CollectorModelConfigService(TupleService):
|
38
|
+
def should_record_operation(self) -> bool:
|
39
|
+
return False
|
40
|
+
|
41
|
+
def get_entity_name(self) -> EntityName:
|
42
|
+
return COLLECTOR_MODEL_CONFIG_ENTITY_NAME
|
43
|
+
|
44
|
+
def get_entity_shaper(self) -> EntityShaper:
|
45
|
+
return COLLECTOR_MODEL_CONFIG_ENTITY_SHAPER
|
46
|
+
|
47
|
+
# noinspection SpellCheckingInspection
|
48
|
+
def get_storable_id_column_name(self) -> EntityName:
|
49
|
+
return "model_id"
|
50
|
+
|
51
|
+
def get_storable_id(self, storable: CollectorModelConfig) -> StorableId:
|
52
|
+
return storable.modelId
|
53
|
+
|
54
|
+
# noinspection SpellCheckingInspection
|
55
|
+
def set_storable_id(self, storable: CollectorModelConfig, storable_id: CollectorModelConfigId) -> Storable:
|
56
|
+
storable.modelId = storable_id
|
57
|
+
return storable
|
58
|
+
|
59
|
+
def create_model_config(self, model_config: CollectorModelConfig) -> CollectorModelConfig:
|
60
|
+
try:
|
61
|
+
self.storage.connect()
|
62
|
+
# noinspection PyTypeChecker
|
63
|
+
return self.create(model_config)
|
64
|
+
finally:
|
65
|
+
self.storage.close()
|
66
|
+
|
67
|
+
def update_model_config(self, model_config: CollectorModelConfig) -> CollectorModelConfig:
|
68
|
+
try:
|
69
|
+
self.storage.connect()
|
70
|
+
# noinspection PyTypeChecker
|
71
|
+
return self.update(model_config)
|
72
|
+
finally:
|
73
|
+
self.storage.close()
|
74
|
+
|
75
|
+
def find_by_model_id(self, model_id: str) -> Optional[CollectorModelConfig]:
|
76
|
+
self.begin_transaction()
|
77
|
+
try:
|
78
|
+
return self.storage.find_one(self.get_entity_finder(
|
79
|
+
criteria=[
|
80
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='model_id'), right=model_id)]
|
81
|
+
))
|
82
|
+
finally:
|
83
|
+
self.close_transaction()
|
84
|
+
|
85
|
+
def find_by_tenant(self, tenant_id: TenantId) -> Optional[List[CollectorModelConfig]]:
|
86
|
+
# noinspection PyTypeChecker
|
87
|
+
return self.storage.find(self.get_entity_finder(
|
88
|
+
criteria=[
|
89
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id)]
|
90
|
+
))
|
91
|
+
|
92
|
+
def find_by_name(self, model_name: str) -> Optional[CollectorModelConfig]:
|
93
|
+
self.begin_transaction()
|
94
|
+
try:
|
95
|
+
return self.storage.find_one(self.get_entity_finder(
|
96
|
+
criteria=[
|
97
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='model_name'), right=model_name)]
|
98
|
+
))
|
99
|
+
finally:
|
100
|
+
self.close_transaction()
|
101
|
+
|
102
|
+
|
103
|
+
def get_collector_model_config_service(storage: TransactionalStorageSPI,
|
104
|
+
snowflake_generator: SnowflakeGenerator,
|
105
|
+
principal_service: PrincipalService
|
106
|
+
) -> CollectorModelConfigService:
|
107
|
+
return CollectorModelConfigService(storage, snowflake_generator, principal_service)
|
@@ -0,0 +1,168 @@
|
|
1
|
+
from typing import Optional, List
|
2
|
+
|
3
|
+
from watchmen_auth import PrincipalService
|
4
|
+
from watchmen_collector_kernel.model import CollectorTableConfig
|
5
|
+
from watchmen_meta.common import TupleService, TupleShaper
|
6
|
+
from watchmen_meta.common.storage_service import StorableId
|
7
|
+
from watchmen_model.common import Storable
|
8
|
+
from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
|
9
|
+
EntityCriteriaExpression, ColumnNameLiteral
|
10
|
+
from watchmen_utilities import ArrayHelper
|
11
|
+
|
12
|
+
|
13
|
+
class CollectorTableConfigShaper(EntityShaper):
|
14
|
+
def serialize(self, config: CollectorTableConfig) -> EntityRow:
|
15
|
+
return TupleShaper.serialize_tenant_based(config, {
|
16
|
+
'config_id': config.configId,
|
17
|
+
'name': config.name,
|
18
|
+
'table_name': config.tableName,
|
19
|
+
'primary_key': config.primaryKey,
|
20
|
+
'object_key': config.objectKey,
|
21
|
+
'model_name': config.modelName,
|
22
|
+
'parent_name': config.parentName,
|
23
|
+
'join_keys': ArrayHelper(config.joinKeys).map(lambda x: x.to_dict()).to_list(),
|
24
|
+
'depend_on': ArrayHelper(config.dependOn).map(lambda x: x.to_dict()).to_list(),
|
25
|
+
'conditions': ArrayHelper(config.conditions).map(lambda x: x.to_dict()).to_list(),
|
26
|
+
'label': config.label,
|
27
|
+
'audit_column': config.auditColumn,
|
28
|
+
'data_source_id': config.dataSourceId,
|
29
|
+
'is_list': config.isList,
|
30
|
+
'triggered': config.triggered
|
31
|
+
})
|
32
|
+
|
33
|
+
def deserialize(self, row: EntityRow) -> CollectorTableConfig:
|
34
|
+
# noinspection PyTypeChecker
|
35
|
+
return TupleShaper.deserialize_tenant_based(row, CollectorTableConfig(
|
36
|
+
configId=row.get('config_id'),
|
37
|
+
name=row.get('name'),
|
38
|
+
tableName=row.get('table_name'),
|
39
|
+
primaryKey=row.get('primary_key'),
|
40
|
+
objectKey=row.get('object_key'),
|
41
|
+
modelName=row.get('model_name'),
|
42
|
+
parentName=row.get('parent_name'),
|
43
|
+
joinKeys=row.get('join_keys'),
|
44
|
+
dependOn=row.get('depend_on'),
|
45
|
+
conditions=row.get('conditions'),
|
46
|
+
label=row.get('label'),
|
47
|
+
auditColumn=row.get('audit_column'),
|
48
|
+
dataSourceId=row.get('data_source_id'),
|
49
|
+
isList=row.get('is_list'),
|
50
|
+
triggered=row.get('triggered')
|
51
|
+
))
|
52
|
+
|
53
|
+
|
54
|
+
COLLECTOR_TABLE_CONFIG_ENTITY_NAME = 'collector_table_config'
|
55
|
+
COLLECTOR_TABLE_CONFIG_ENTITY_SHAPER = CollectorTableConfigShaper()
|
56
|
+
|
57
|
+
|
58
|
+
class CollectorTableConfigService(TupleService):
|
59
|
+
def should_record_operation(self) -> bool:
|
60
|
+
return False
|
61
|
+
|
62
|
+
def get_entity_name(self) -> EntityName:
|
63
|
+
return COLLECTOR_TABLE_CONFIG_ENTITY_NAME
|
64
|
+
|
65
|
+
def get_entity_shaper(self) -> EntityShaper:
|
66
|
+
return COLLECTOR_TABLE_CONFIG_ENTITY_SHAPER
|
67
|
+
|
68
|
+
# noinspection SpellCheckingInspection
|
69
|
+
def get_storable_id_column_name(self) -> EntityName:
|
70
|
+
return "config_id"
|
71
|
+
|
72
|
+
def get_storable_id(self, storable: CollectorTableConfig) -> StorableId:
|
73
|
+
return storable.configId
|
74
|
+
|
75
|
+
# noinspection SpellCheckingInspection
|
76
|
+
def set_storable_id(self, storable: CollectorTableConfig, storable_id: StorableId) -> Storable:
|
77
|
+
storable.configId = storable_id
|
78
|
+
return storable
|
79
|
+
|
80
|
+
# noinspection PyTypeChecker
|
81
|
+
def create_config(self, config: CollectorTableConfig) -> CollectorTableConfig:
|
82
|
+
self.begin_transaction()
|
83
|
+
try:
|
84
|
+
config = self.create(config)
|
85
|
+
self.commit_transaction()
|
86
|
+
return config
|
87
|
+
except Exception as e:
|
88
|
+
self.rollback_transaction()
|
89
|
+
raise e
|
90
|
+
|
91
|
+
# noinspection PyTypeChecker
|
92
|
+
def update_config(self, config: CollectorTableConfig) -> CollectorTableConfig:
|
93
|
+
self.begin_transaction()
|
94
|
+
try:
|
95
|
+
config = self.update(config)
|
96
|
+
self.commit_transaction()
|
97
|
+
return config
|
98
|
+
except Exception as e:
|
99
|
+
self.rollback_transaction()
|
100
|
+
raise e
|
101
|
+
|
102
|
+
def find_config_by_id(self, config_id: str) -> Optional[CollectorTableConfig]:
|
103
|
+
self.begin_transaction()
|
104
|
+
try:
|
105
|
+
return self.find_by_id(config_id)
|
106
|
+
finally:
|
107
|
+
self.close_transaction()
|
108
|
+
|
109
|
+
def find_by_table_name(self, table_name: str) -> Optional[CollectorTableConfig]:
|
110
|
+
try:
|
111
|
+
self.storage.connect()
|
112
|
+
# noinspection PyTypeChecker
|
113
|
+
return self.storage.find_one(
|
114
|
+
self.get_entity_finder(
|
115
|
+
criteria=[
|
116
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='table_name'),
|
117
|
+
right=table_name)
|
118
|
+
]
|
119
|
+
)
|
120
|
+
)
|
121
|
+
finally:
|
122
|
+
self.storage.close()
|
123
|
+
|
124
|
+
def find_by_name(self, name: str) -> Optional[CollectorTableConfig]:
|
125
|
+
try:
|
126
|
+
self.storage.connect()
|
127
|
+
# noinspection PyTypeChecker
|
128
|
+
return self.storage.find_one(
|
129
|
+
self.get_entity_finder(
|
130
|
+
criteria=[
|
131
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='name'),
|
132
|
+
right=name)
|
133
|
+
]
|
134
|
+
)
|
135
|
+
)
|
136
|
+
finally:
|
137
|
+
self.storage.close()
|
138
|
+
|
139
|
+
def find_by_parent_name(self, parent_name: str) -> Optional[List[CollectorTableConfig]]:
|
140
|
+
try:
|
141
|
+
self.storage.connect()
|
142
|
+
# noinspection PyTypeChecker
|
143
|
+
return self.storage.find(
|
144
|
+
self.get_entity_finder(
|
145
|
+
criteria=[
|
146
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='parent_name'),
|
147
|
+
right=parent_name)
|
148
|
+
]
|
149
|
+
)
|
150
|
+
)
|
151
|
+
finally:
|
152
|
+
self.storage.close()
|
153
|
+
|
154
|
+
def find_by_model_name(self,
|
155
|
+
model_name: str) -> Optional[List[CollectorTableConfig]]:
|
156
|
+
# noinspection PyTypeChecker
|
157
|
+
return self.storage.find(self.get_entity_finder(
|
158
|
+
criteria=[
|
159
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'), right=self.principalService.get_tenant_id()),
|
160
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='model_name'), right=model_name)]
|
161
|
+
))
|
162
|
+
|
163
|
+
|
164
|
+
def get_collector_table_config_service(storage: TransactionalStorageSPI,
|
165
|
+
snowflake_generator: SnowflakeGenerator,
|
166
|
+
principal_service: PrincipalService
|
167
|
+
) -> CollectorTableConfigService:
|
168
|
+
return CollectorTableConfigService(storage, snowflake_generator, principal_service)
|
@@ -0,0 +1,83 @@
|
|
1
|
+
from datetime import datetime
|
2
|
+
from typing import List
|
3
|
+
|
4
|
+
from watchmen_collector_kernel.model import CompetitiveLock
|
5
|
+
from watchmen_meta.common import ask_meta_storage, EntityService
|
6
|
+
from watchmen_meta.common.storage_service import StorableId
|
7
|
+
from watchmen_model.common import CompetitiveLockId, Storable
|
8
|
+
from watchmen_storage import ColumnNameLiteral, Entity, EntityCriteriaExpression, EntityCriteriaOperator, \
|
9
|
+
EntityName, EntityRow, EntityShaper, TransactionalStorageSPI
|
10
|
+
|
11
|
+
|
12
|
+
class CompetitiveLockShaper(EntityShaper):
|
13
|
+
def serialize(self, entity: CompetitiveLock) -> EntityRow:
|
14
|
+
return {
|
15
|
+
'lock_id': entity.lockId,
|
16
|
+
'resource_id': entity.resourceId,
|
17
|
+
'registered_at': entity.registeredAt,
|
18
|
+
'tenant_id': entity.tenantId
|
19
|
+
}
|
20
|
+
|
21
|
+
def deserialize(self, row: EntityRow) -> CompetitiveLock:
|
22
|
+
return CompetitiveLock(
|
23
|
+
lockId=row.get('lock_id'),
|
24
|
+
resourceId=row.get('resource_id'),
|
25
|
+
registeredAt=row.get('registered_at'),
|
26
|
+
tenantId=row.get('tenant_id')
|
27
|
+
)
|
28
|
+
|
29
|
+
|
30
|
+
COMPETITIVE_LOCK_TABLE = 'competitive_lock'
|
31
|
+
COMPETITIVE_LOCK_ENTITY_SHAPER = CompetitiveLockShaper()
|
32
|
+
|
33
|
+
|
34
|
+
class CompetitiveLockService(EntityService):
|
35
|
+
|
36
|
+
def __init__(self, storage: TransactionalStorageSPI):
|
37
|
+
super().__init__(storage)
|
38
|
+
|
39
|
+
def get_entity_name(self) -> EntityName:
|
40
|
+
return COMPETITIVE_LOCK_TABLE
|
41
|
+
|
42
|
+
def get_entity_shaper(self) -> EntityShaper:
|
43
|
+
return COMPETITIVE_LOCK_ENTITY_SHAPER
|
44
|
+
|
45
|
+
def get_storable_id_column_name(self) -> EntityName:
|
46
|
+
return 'lock_id'
|
47
|
+
|
48
|
+
def get_storable_id(self, storable: CompetitiveLock) -> StorableId:
|
49
|
+
return storable.lockId
|
50
|
+
|
51
|
+
def set_storable_id(
|
52
|
+
self, storable: CompetitiveLock, storable_id: CompetitiveLockId) -> Storable:
|
53
|
+
storable.lockId = storable_id
|
54
|
+
return storable
|
55
|
+
|
56
|
+
def insert_one(self, lock: CompetitiveLock):
|
57
|
+
try:
|
58
|
+
self.storage.connect()
|
59
|
+
self.storage.insert_one(lock, self.get_entity_helper())
|
60
|
+
finally:
|
61
|
+
self.storage.close()
|
62
|
+
|
63
|
+
def delete_by_id(self, lock_id: CompetitiveLockId):
|
64
|
+
try:
|
65
|
+
self.storage.connect()
|
66
|
+
self.storage.delete_by_id(lock_id, self.get_entity_id_helper())
|
67
|
+
finally:
|
68
|
+
self.storage.close()
|
69
|
+
|
70
|
+
def find_overtime_lock(self, query_time: datetime) -> List:
|
71
|
+
try:
|
72
|
+
self.storage.connect()
|
73
|
+
return self.storage.find(self.get_entity_finder(criteria=[
|
74
|
+
EntityCriteriaExpression(
|
75
|
+
left=ColumnNameLiteral(columnName='registered_at'),
|
76
|
+
operator=EntityCriteriaOperator.LESS_THAN, right=query_time)
|
77
|
+
]))
|
78
|
+
finally:
|
79
|
+
self.storage.close()
|
80
|
+
|
81
|
+
|
82
|
+
def get_competitive_lock_service(storage: TransactionalStorageSPI) -> CompetitiveLockService:
|
83
|
+
return CompetitiveLockService(storage)
|
@@ -0,0 +1,155 @@
|
|
1
|
+
from typing import Optional, List, Any, Dict
|
2
|
+
|
3
|
+
from watchmen_auth import PrincipalService
|
4
|
+
from watchmen_collector_kernel.model import ScheduledTask
|
5
|
+
from watchmen_meta.common import TupleService, TupleShaper
|
6
|
+
from watchmen_meta.common.storage_service import StorableId
|
7
|
+
from watchmen_model.common import Storable, ScheduledTaskId
|
8
|
+
from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, \
|
9
|
+
EntityCriteriaExpression, ColumnNameLiteral, SnowflakeGenerator, EntitySortColumn, EntitySortMethod, \
|
10
|
+
EntityCriteriaJoint, EntityCriteriaJointConjunction, EntityStraightValuesFinder, EntityStraightColumn, \
|
11
|
+
EntityCriteriaOperator
|
12
|
+
|
13
|
+
|
14
|
+
class ScheduledTaskShaper(EntityShaper):
|
15
|
+
def serialize(self, entity: ScheduledTask) -> EntityRow:
|
16
|
+
return TupleShaper.serialize_tenant_based(entity, {
|
17
|
+
'task_id': entity.taskId,
|
18
|
+
'resource_id': entity.resourceId,
|
19
|
+
'topic_code': entity.topicCode,
|
20
|
+
'content': entity.content,
|
21
|
+
'model_name': entity.modelName,
|
22
|
+
'object_id': entity.objectId,
|
23
|
+
'depend_on': entity.dependOn,
|
24
|
+
'parent_task_id': entity.parentTaskId,
|
25
|
+
'tenant_id': entity.tenantId,
|
26
|
+
'status': entity.status,
|
27
|
+
'result': entity.result
|
28
|
+
})
|
29
|
+
|
30
|
+
def deserialize(self, row: EntityRow) -> ScheduledTask:
|
31
|
+
# noinspection PyTypeChecker
|
32
|
+
return TupleShaper.deserialize_tenant_based(row, ScheduledTask(
|
33
|
+
taskId=row.get('task_id'),
|
34
|
+
resourceId=row.get('resource_id'),
|
35
|
+
topicCode=row.get('topic_code'),
|
36
|
+
content=row.get('content'),
|
37
|
+
modelName=row.get('model_name'),
|
38
|
+
objectId=row.get('object_id'),
|
39
|
+
dependOn=row.get('depend_on'),
|
40
|
+
parentTaskId=row.get('parent_task_id'),
|
41
|
+
tenantId=row.get('tenant_id'),
|
42
|
+
status=row.get('status'),
|
43
|
+
result=row.get('result')
|
44
|
+
))
|
45
|
+
|
46
|
+
|
47
|
+
SCHEDULED_TASK_TABLE = 'scheduled_task'
|
48
|
+
SCHEDULED_TASK_ENTITY_SHAPER = ScheduledTaskShaper()
|
49
|
+
|
50
|
+
|
51
|
+
class ScheduledTaskService(TupleService):
|
52
|
+
|
53
|
+
def should_record_operation(self) -> bool:
|
54
|
+
return False
|
55
|
+
|
56
|
+
def get_entity_name(self) -> EntityName:
|
57
|
+
return SCHEDULED_TASK_TABLE
|
58
|
+
|
59
|
+
def get_entity_shaper(self) -> EntityShaper:
|
60
|
+
return SCHEDULED_TASK_ENTITY_SHAPER
|
61
|
+
|
62
|
+
def get_storable_id_column_name(self) -> EntityName:
|
63
|
+
return 'task_id'
|
64
|
+
|
65
|
+
def get_storable_id(self, storable: ScheduledTask) -> StorableId:
|
66
|
+
return storable.taskId
|
67
|
+
|
68
|
+
def set_storable_id(
|
69
|
+
self, storable: ScheduledTask, storable_id: ScheduledTaskId) -> Storable:
|
70
|
+
storable.taskId = storable_id
|
71
|
+
return storable
|
72
|
+
|
73
|
+
def create_task(self, task: ScheduledTask) -> ScheduledTask:
|
74
|
+
self.begin_transaction()
|
75
|
+
try:
|
76
|
+
task = self.create(task)
|
77
|
+
self.commit_transaction()
|
78
|
+
# noinspection PyTypeChecker
|
79
|
+
return task
|
80
|
+
except Exception as e:
|
81
|
+
self.rollback_transaction()
|
82
|
+
raise e
|
83
|
+
|
84
|
+
def update_task(self, task: ScheduledTask) -> ScheduledTask:
|
85
|
+
self.begin_transaction()
|
86
|
+
try:
|
87
|
+
task = self.update(task)
|
88
|
+
self.commit_transaction()
|
89
|
+
# noinspection PyTypeChecker
|
90
|
+
return task
|
91
|
+
except Exception as e:
|
92
|
+
self.rollback_transaction()
|
93
|
+
raise e
|
94
|
+
|
95
|
+
def find_task_by_id(self, task_id: ScheduledTaskId) -> Optional[ScheduledTask]:
|
96
|
+
self.begin_transaction()
|
97
|
+
try:
|
98
|
+
return self.find_by_id(task_id)
|
99
|
+
finally:
|
100
|
+
self.close_transaction()
|
101
|
+
|
102
|
+
def find_unfinished_tasks(self) -> List[Dict[str, Any]]:
|
103
|
+
self.begin_transaction()
|
104
|
+
try:
|
105
|
+
return self.storage.find_straight_values(EntityStraightValuesFinder(
|
106
|
+
name=self.get_entity_name(),
|
107
|
+
criteria=[EntityCriteriaJoint(
|
108
|
+
children=[
|
109
|
+
EntityCriteriaExpression(
|
110
|
+
left=ColumnNameLiteral(columnName='status'), right=0)
|
111
|
+
]
|
112
|
+
)],
|
113
|
+
straightColumns=[EntityStraightColumn(columnName='task_id'),
|
114
|
+
EntityStraightColumn(columnName='resource_id'),
|
115
|
+
EntityStraightColumn(columnName='tenant_id')],
|
116
|
+
sort=[EntitySortColumn(name='resource_id', method=EntitySortMethod.ASC)]
|
117
|
+
))
|
118
|
+
finally:
|
119
|
+
self.close_transaction()
|
120
|
+
|
121
|
+
def find_by_resource_id(self, resource_id: str) -> List[Dict[str, Any]]:
|
122
|
+
self.begin_transaction()
|
123
|
+
try:
|
124
|
+
return self.storage.find_straight_values(EntityStraightValuesFinder(
|
125
|
+
name=self.get_entity_name(),
|
126
|
+
criteria=[EntityCriteriaExpression(left=ColumnNameLiteral(columnName='resource_id'), right=resource_id)],
|
127
|
+
straightColumns=['task_id', 'resource_id', 'tenant_id']
|
128
|
+
))
|
129
|
+
finally:
|
130
|
+
self.close_transaction()
|
131
|
+
|
132
|
+
def is_dependent_task_finished(self, model_name: str, object_id: str, tenant_id: str) -> bool:
|
133
|
+
self.begin_transaction()
|
134
|
+
try:
|
135
|
+
# noinspection PyTypeChecker
|
136
|
+
return self.storage.count(self.get_entity_finder(
|
137
|
+
criteria=[
|
138
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='model_name'),
|
139
|
+
right=model_name),
|
140
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='object_id'),
|
141
|
+
right=object_id),
|
142
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='status'),
|
143
|
+
right=0),
|
144
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'),
|
145
|
+
right=tenant_id)
|
146
|
+
]
|
147
|
+
)) == 0
|
148
|
+
finally:
|
149
|
+
self.close_transaction()
|
150
|
+
|
151
|
+
def get_scheduled_task_service(storage: TransactionalStorageSPI,
|
152
|
+
snowflake_generator: SnowflakeGenerator,
|
153
|
+
principal_service: PrincipalService
|
154
|
+
) -> ScheduledTaskService:
|
155
|
+
return ScheduledTaskService(storage, snowflake_generator, principal_service)
|
@@ -0,0 +1,86 @@
|
|
1
|
+
from typing import List, Dict, Any
|
2
|
+
|
3
|
+
from watchmen_auth import PrincipalService
|
4
|
+
from watchmen_collector_kernel.model import TriggerEvent
|
5
|
+
from watchmen_meta.common import TupleShaper, TupleService
|
6
|
+
from watchmen_meta.common.storage_service import StorableId
|
7
|
+
from watchmen_model.common import Storable, EventTriggerId
|
8
|
+
from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
|
9
|
+
EntityStraightValuesFinder, EntityCriteriaExpression, ColumnNameLiteral, \
|
10
|
+
EntityStraightColumn, EntitySortColumn, EntitySortMethod
|
11
|
+
|
12
|
+
|
13
|
+
class TriggerEventShaper(EntityShaper):
|
14
|
+
|
15
|
+
def serialize(self, entity: TriggerEvent) -> EntityRow:
|
16
|
+
return TupleShaper.serialize_tenant_based(entity, {
|
17
|
+
'event_trigger_id': entity.eventTriggerId,
|
18
|
+
'start_time': entity.startTime,
|
19
|
+
'end_time': entity.endTime,
|
20
|
+
'is_finished': entity.isFinished
|
21
|
+
})
|
22
|
+
|
23
|
+
def deserialize(self, row: EntityRow) -> TriggerEvent:
|
24
|
+
# noinspection PyTypeChecker
|
25
|
+
return TupleShaper.deserialize_tenant_based(row, TriggerEvent(
|
26
|
+
eventTriggerId=row.get('event_trigger_id'),
|
27
|
+
startTime=row.get('start_time'),
|
28
|
+
endTime=row.get('end_time'),
|
29
|
+
isFinished=row.get('is_finished')
|
30
|
+
))
|
31
|
+
|
32
|
+
|
33
|
+
TRIGGER_EVENT_TABLE = 'trigger_event'
|
34
|
+
TRIGGER_EVENT_ENTITY_SHAPER = TriggerEventShaper()
|
35
|
+
|
36
|
+
|
37
|
+
class TriggerEventService(TupleService):
|
38
|
+
|
39
|
+
def should_record_operation(self) -> bool:
|
40
|
+
return False
|
41
|
+
|
42
|
+
def get_entity_name(self) -> EntityName:
|
43
|
+
return TRIGGER_EVENT_TABLE
|
44
|
+
|
45
|
+
def get_entity_shaper(self) -> EntityShaper:
|
46
|
+
return TRIGGER_EVENT_ENTITY_SHAPER
|
47
|
+
|
48
|
+
def get_storable_id_column_name(self) -> EntityName:
|
49
|
+
return 'event_trigger_id'
|
50
|
+
|
51
|
+
def get_storable_id(self, storable: TriggerEvent) -> StorableId:
|
52
|
+
# noinspection PyTypeChecker
|
53
|
+
return storable.eventTriggerId
|
54
|
+
|
55
|
+
def set_storable_id(
|
56
|
+
self, storable: TriggerEvent, storable_id: EventTriggerId) -> Storable:
|
57
|
+
storable.eventTriggerId = storable_id
|
58
|
+
return storable
|
59
|
+
|
60
|
+
def find_event_by_id(self, event_trigger_id: EventTriggerId) -> TriggerEvent:
|
61
|
+
try:
|
62
|
+
self.storage.connect()
|
63
|
+
# noinspection PyTypeChecker
|
64
|
+
return self.storage.find_by_id(event_trigger_id, self.get_entity_id_helper())
|
65
|
+
finally:
|
66
|
+
self.storage.close()
|
67
|
+
|
68
|
+
def find_unfinished_events(self) -> List[Dict[str, Any]]:
|
69
|
+
self.begin_transaction()
|
70
|
+
try:
|
71
|
+
return self.storage.find_straight_values(EntityStraightValuesFinder(
|
72
|
+
name=self.get_entity_name(),
|
73
|
+
criteria=[EntityCriteriaExpression(left=ColumnNameLiteral(columnName='is_finished'), right=False)],
|
74
|
+
straightColumns=[EntityStraightColumn(columnName=self.get_storable_id_column_name()),
|
75
|
+
EntityStraightColumn(columnName='tenant_id')],
|
76
|
+
sort=[EntitySortColumn(name=self.get_storable_id_column_name(), method=EntitySortMethod.ASC)]
|
77
|
+
))
|
78
|
+
finally:
|
79
|
+
self.close_transaction()
|
80
|
+
|
81
|
+
|
82
|
+
def get_trigger_event_service(storage: TransactionalStorageSPI,
|
83
|
+
snowflake_generator: SnowflakeGenerator,
|
84
|
+
principal_service: PrincipalService
|
85
|
+
) -> TriggerEventService:
|
86
|
+
return TriggerEventService(storage, snowflake_generator, principal_service)
|
@@ -0,0 +1,96 @@
|
|
1
|
+
from typing import Optional, List
|
2
|
+
|
3
|
+
from watchmen_auth import PrincipalService
|
4
|
+
from watchmen_collector_kernel.common import IS_FINISHED, TENANT_ID
|
5
|
+
from watchmen_collector_kernel.model import TriggerModel
|
6
|
+
from watchmen_meta.common import TupleShaper, TupleService
|
7
|
+
from watchmen_meta.common.storage_service import StorableId
|
8
|
+
from watchmen_model.common import Storable, ModelTriggerId
|
9
|
+
from watchmen_storage import EntityName, EntityRow, EntityShaper, TransactionalStorageSPI, SnowflakeGenerator, \
|
10
|
+
EntityCriteriaExpression, ColumnNameLiteral
|
11
|
+
|
12
|
+
|
13
|
+
class TriggerModelShaper(EntityShaper):
|
14
|
+
|
15
|
+
def serialize(self, entity: TriggerModel) -> EntityRow:
|
16
|
+
return TupleShaper.serialize_tenant_based(entity, {
|
17
|
+
'model_trigger_id': entity.modelTriggerId,
|
18
|
+
'model_name': entity.modelName,
|
19
|
+
'is_finished': entity.isFinished,
|
20
|
+
'event_trigger_id': entity.eventTriggerId
|
21
|
+
})
|
22
|
+
|
23
|
+
def deserialize(self, row: EntityRow) -> TriggerModel:
|
24
|
+
# noinspection PyTypeChecker
|
25
|
+
return TupleShaper.deserialize_tenant_based(row, TriggerModel(
|
26
|
+
modelTriggerId=row.get('model_trigger_id'),
|
27
|
+
modelName=row.get('model_name'),
|
28
|
+
isFinished=row.get('is_finished'),
|
29
|
+
eventTriggerId=row.get('event_trigger_id')
|
30
|
+
))
|
31
|
+
|
32
|
+
|
33
|
+
TRIGGER_MODEL_TABLE = 'trigger_model'
|
34
|
+
TRIGGER_MODEL_ENTITY_SHAPER = TriggerModelShaper()
|
35
|
+
|
36
|
+
|
37
|
+
class TriggerModelService(TupleService):
|
38
|
+
|
39
|
+
def should_record_operation(self) -> bool:
|
40
|
+
return False
|
41
|
+
|
42
|
+
def get_entity_name(self) -> EntityName:
|
43
|
+
return TRIGGER_MODEL_TABLE
|
44
|
+
|
45
|
+
def get_entity_shaper(self) -> EntityShaper:
|
46
|
+
return TRIGGER_MODEL_ENTITY_SHAPER
|
47
|
+
|
48
|
+
def get_storable_id_column_name(self) -> EntityName:
|
49
|
+
return 'model_trigger_id'
|
50
|
+
|
51
|
+
def get_storable_id(self, storable: TriggerModel) -> StorableId:
|
52
|
+
# noinspection PyTypeChecker
|
53
|
+
return storable.modelTriggerId
|
54
|
+
|
55
|
+
def set_storable_id(
|
56
|
+
self, storable: TriggerModel, storable_id: ModelTriggerId) -> Storable:
|
57
|
+
storable.modelTriggerId = storable_id
|
58
|
+
return storable
|
59
|
+
|
60
|
+
def find_trigger_by_id(self, trigger_id: int) -> Optional[TriggerModel]:
|
61
|
+
self.begin_transaction()
|
62
|
+
try:
|
63
|
+
return self.find_by_id(trigger_id)
|
64
|
+
finally:
|
65
|
+
self.close_transaction()
|
66
|
+
|
67
|
+
def is_finished(self, event_trigger_id: str) -> bool:
|
68
|
+
self.begin_transaction()
|
69
|
+
try:
|
70
|
+
return self.storage.count(self.get_entity_finder(
|
71
|
+
criteria=[
|
72
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName=IS_FINISHED), right=False),
|
73
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='event_trigger_id'), right=event_trigger_id)
|
74
|
+
]
|
75
|
+
)) == 0
|
76
|
+
finally:
|
77
|
+
self.close_transaction()
|
78
|
+
|
79
|
+
def find_by_event_trigger_id(self, event_trigger_id: int) -> List[TriggerModel]:
|
80
|
+
self.begin_transaction()
|
81
|
+
try:
|
82
|
+
# noinspection PyTypeChecker
|
83
|
+
return self.storage.find(self.get_entity_finder(
|
84
|
+
criteria=[
|
85
|
+
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='event_trigger_id'), right=event_trigger_id)
|
86
|
+
]
|
87
|
+
))
|
88
|
+
finally:
|
89
|
+
self.close_transaction()
|
90
|
+
|
91
|
+
|
92
|
+
def get_trigger_model_service(storage: TransactionalStorageSPI,
|
93
|
+
snowflake_generator: SnowflakeGenerator,
|
94
|
+
principal_service: PrincipalService
|
95
|
+
) -> TriggerModelService:
|
96
|
+
return TriggerModelService(storage, snowflake_generator, principal_service)
|