datacom-device-api 0.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. datacom_device_api-0.1.5/PKG-INFO +11 -0
  2. datacom_device_api-0.1.5/README.md +0 -0
  3. datacom_device_api-0.1.5/pyproject.toml +13 -0
  4. datacom_device_api-0.1.5/setup.cfg +4 -0
  5. datacom_device_api-0.1.5/src/datacom_device_api/__init__.py +0 -0
  6. datacom_device_api-0.1.5/src/datacom_device_api/data/__init__.py +0 -0
  7. datacom_device_api-0.1.5/src/datacom_device_api/data/table.py +25 -0
  8. datacom_device_api-0.1.5/src/datacom_device_api/operation/__init__.py +0 -0
  9. datacom_device_api-0.1.5/src/datacom_device_api/operation/table_operation.py +89 -0
  10. datacom_device_api-0.1.5/src/datacom_device_api/operation/table_read_from_xlsx.py +103 -0
  11. datacom_device_api-0.1.5/src/datacom_device_api/operation/table_save_to_db.py +20 -0
  12. datacom_device_api-0.1.5/src/datacom_device_api/process/__init__.py +0 -0
  13. datacom_device_api-0.1.5/src/datacom_device_api/process/terminal_process.py +244 -0
  14. datacom_device_api-0.1.5/src/datacom_device_api/utils/__init__.py +0 -0
  15. datacom_device_api-0.1.5/src/datacom_device_api/utils/directory_util.py +13 -0
  16. datacom_device_api-0.1.5/src/datacom_device_api/utils/logger.py +14 -0
  17. datacom_device_api-0.1.5/src/datacom_device_api.egg-info/PKG-INFO +11 -0
  18. datacom_device_api-0.1.5/src/datacom_device_api.egg-info/SOURCES.txt +20 -0
  19. datacom_device_api-0.1.5/src/datacom_device_api.egg-info/dependency_links.txt +1 -0
  20. datacom_device_api-0.1.5/src/datacom_device_api.egg-info/requires.txt +5 -0
  21. datacom_device_api-0.1.5/src/datacom_device_api.egg-info/top_level.txt +1 -0
  22. datacom_device_api-0.1.5/test/test_table_matching.py +270 -0
@@ -0,0 +1,11 @@
1
+ Metadata-Version: 2.4
2
+ Name: datacom-device-api
3
+ Version: 0.1.5
4
+ Summary: Add your description here
5
+ Requires-Python: >=3.14
6
+ Description-Content-Type: text/markdown
7
+ Requires-Dist: dataclasses>=0.8
8
+ Requires-Dist: openpyxl>=3.1.5
9
+ Requires-Dist: ops>=3.8.0
10
+ Requires-Dist: pymysql>=1.2.0
11
+ Requires-Dist: rich>=15.0.0
File without changes
@@ -0,0 +1,13 @@
1
+ [project]
2
+ name = "datacom-device-api"
3
+ version = "0.1.5"
4
+ description = "Add your description here"
5
+ readme = "README.md"
6
+ requires-python = ">=3.14"
7
+ dependencies = [
8
+ "dataclasses>=0.8",
9
+ "openpyxl>=3.1.5",
10
+ "ops>=3.8.0",
11
+ "pymysql>=1.2.0",
12
+ "rich>=15.0.0",
13
+ ]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,25 @@
1
+ from dataclasses import dataclass
2
+ from typing import Any, Generic, TypeVarTuple, Unpack
3
+
4
+ Ts = TypeVarTuple('Ts')
5
+
6
+ @dataclass
7
+ class Table(Generic[Unpack[Ts]]):
8
+ table_name: str
9
+ headers: list[str]
10
+ fragment_refrence_size: int
11
+ total_fragment: int
12
+ total_row: int
13
+ fragments: list[TableFragment[Unpack[Ts]]]
14
+ def __post_init__(self):
15
+ for fragment in self.fragments:
16
+ if fragment.fragment_offset < 0 or fragment.fragment_size <= 0:
17
+ raise ValueError('Invalid fragment offset or size')
18
+ if len(fragment.rows) > fragment.fragment_size:
19
+ raise ValueError('Invalid fragment size or row count')
20
+
21
+ @dataclass
22
+ class TableFragment(Generic[Unpack[Ts]]):
23
+ fragment_offset: int
24
+ fragment_size: int
25
+ rows: list[tuple[Unpack[Ts]]]
@@ -0,0 +1,89 @@
1
+
2
+ from typing import Any, Callable, Generic, Optional, TypeVar, TypeVarTuple, Union, Unpack, overload
3
+
4
+ from datacom_device_api.data.table import Table, TableFragment
5
+
6
+ Ts = TypeVarTuple('Ts')
7
+
8
+ class TableOperation(Generic[Unpack[Ts]]):
9
+ def __init__(
10
+ self,
11
+ table_name: str,
12
+ headers: list[str],
13
+ fragment_refrence_size: int = 1000
14
+ ) -> None:
15
+ self.__table = Table[Unpack[Ts]](
16
+ table_name=table_name,
17
+ headers=headers,
18
+ fragment_refrence_size=fragment_refrence_size,
19
+ total_fragment=0,
20
+ total_row=0,
21
+ fragments=[],
22
+ )
23
+
24
+ def add_fragment(self, fragment: TableFragment[Unpack[Ts]]) -> None:
25
+ if fragment.fragment_size > self.__table.fragment_refrence_size:
26
+ raise ValueError('fragment size is too large')
27
+
28
+ self.__table.fragments.append(fragment)
29
+ self.__table.total_fragment += 1
30
+ self.__table.total_row += len(fragment.rows)
31
+
32
+ @property
33
+ def fragment_refrence_size(self) -> int:
34
+ return self.__table.fragment_refrence_size
35
+
36
+ def get_row_by_row_index(self, row_index: int) -> Optional[tuple[Unpack[Ts]]]:
37
+ if row_index >= self.__table.total_row:
38
+ raise ValueError('row index is out of range')
39
+
40
+ self.__table.fragments.sort(key=lambda fragment: fragment.fragment_offset)
41
+
42
+ fragment_index = row_index // self.__table.fragment_refrence_size
43
+ fragment = self.__table.fragments[fragment_index]
44
+
45
+ return fragment.rows[row_index % fragment.fragment_size]
46
+
47
+ @overload
48
+ def get_row_by_header(self, header: str, value: Any) -> list[tuple[*Ts]]: ...
49
+ @overload
50
+ def get_row_by_header(self, header: int, value: Any) -> list[tuple[*Ts]]: ...
51
+
52
+ def get_row_by_header(self, header: Any, value: Any) -> list[tuple[*Ts]]:
53
+ if isinstance(header, int):
54
+ header_index = header
55
+ elif isinstance(header, str):
56
+ if header not in self.__table.headers:
57
+ raise ValueError('header name not found')
58
+ header_index = self.__table.headers.index(header)
59
+ else:
60
+ raise ValueError('header must be int or str')
61
+
62
+ if type(value) not in [*Ts]:
63
+ raise ValueError('value type of value')
64
+
65
+ result = []
66
+ for fragment in self.__table.fragments:
67
+ for row in fragment.rows:
68
+ if row[header_index] == value:
69
+ result.append(row)
70
+
71
+ return result
72
+
73
+ @property
74
+ def table(self) -> Table[Unpack[Ts]]:
75
+ return self.__table
76
+
77
+ @property
78
+ def headers(self) -> list[str]:
79
+ return self.__table.headers
80
+
81
+ def for_each_row_by_fragment_index(self, fragment_index: int, callback: Callable[[tuple[*Ts]], None]):
82
+ fragment = self.__table.fragments[fragment_index]
83
+ for row in fragment.rows:
84
+ callback(row)
85
+
86
+ def for_each_all(self, callback: Callable[[tuple[*Ts]], None]):
87
+ for frament in self.__table.fragments:
88
+ for row in frament.rows:
89
+ callback(row)
@@ -0,0 +1,103 @@
1
+ from typing import Any, Callable, Generic, TypeVarTuple
2
+
3
+ from openpyxl import Workbook, load_workbook
4
+
5
+ from datacom_device_api.data.table import Table, TableFragment
6
+ from datacom_device_api.operation.table_operation import TableOperation
7
+
8
+
9
+ Ts = TypeVarTuple('Ts')
10
+
11
+ class TableReadFromXlsx(Generic[*Ts]):
12
+ def __init__(self, xlsx_file_path: str) -> None:
13
+ try:
14
+ self.__workbook = load_workbook(xlsx_file_path, read_only=True)
15
+ except FileNotFoundError:
16
+ raise FileNotFoundError(f"{xlsx_file_path} is not found.")
17
+ except Exception as e:
18
+ raise e
19
+
20
+ @property
21
+ def get_sheet_names(self) -> list[str]:
22
+ return self.__workbook.sheetnames
23
+
24
+
25
+ def _generate_table_fragment(self, worksheet, min_row: int, max_row: int, convert_func: Callable[[str, Any], Any], headers: list[str]) -> TableFragment[*Ts]:
26
+ if not hasattr(worksheet, 'iter_rows'):
27
+ raise AttributeError("worksheet does not have iter_rows method.")
28
+
29
+ row_datas: list[tuple] = []
30
+ for row_data in worksheet.iter_rows(min_row=min_row, max_row=max_row):
31
+ tmp = tuple(convert_func(headers[i], row_data[i].value ) for i in range(len(headers)))
32
+ print(tmp)
33
+ row_datas.append(tmp)
34
+
35
+ if min_row == 1 or min_row == 2:
36
+ fragment_offset = 0
37
+ else:
38
+ fragment_offset = min_row - 1
39
+
40
+ return TableFragment[*Ts](
41
+ rows=row_datas,
42
+ fragment_offset=fragment_offset,
43
+ fragment_size=len(row_datas),
44
+ )
45
+
46
+ def generate_table_data(self, sheet_name: str, convert_func: Callable[[str, Any], Any]) -> Table[*Ts]:
47
+ if sheet_name not in self.__workbook.sheetnames:
48
+ raise ValueError(f"{sheet_name} is not found.")
49
+
50
+ worksheet = self.__workbook[sheet_name]
51
+ total_rows = worksheet.max_row - 1
52
+
53
+ headers = []
54
+ for row_data in worksheet.iter_rows(min_row=1, max_row=1):
55
+ headers.extend([item.value for item in row_data])
56
+
57
+ table = TableOperation[*Ts](
58
+ table_name=sheet_name,
59
+ headers=headers,
60
+ )
61
+
62
+ fragments_count = -(-total_rows // table.fragment_refrence_size)
63
+ fragments_size = []
64
+
65
+ remaining_count = total_rows
66
+ for _ in range(fragments_count):
67
+ remaining_count -= (table.fragment_refrence_size)
68
+ if remaining_count < 0:
69
+ fragments_size.append(remaining_count + table.fragment_refrence_size)
70
+ else:
71
+ fragments_size.append(table.fragment_refrence_size)
72
+
73
+ offset = 2
74
+ for size in fragments_size:
75
+ fragment = self._generate_table_fragment(
76
+ worksheet=worksheet,
77
+ min_row=offset,
78
+ max_row=offset + size - 1,
79
+ convert_func=convert_func,
80
+ headers=headers,
81
+ )
82
+
83
+ table.add_fragment(fragment)
84
+ offset += size
85
+
86
+ return table.table
87
+
88
+ def _convert_func(header: str, value: Any) -> Any:
89
+ if header == 'USER_NAME':
90
+ return str(value)
91
+ return value
92
+
93
+ if __name__ == "__main__":
94
+ xlsx_file = "C:\\Users\\24168\\Projects\\Datacom_tools\\tables\\test.xlsx"
95
+ sheet_name = "Sheet1"
96
+ xlsx = TableReadFromXlsx(xlsx_file)
97
+ xlsx.generate_table_data(sheet_name, _convert_func)
98
+
99
+ # workbook = load_workbook(xlsx_file, read_only=True)
100
+ # sheet = workbook[sheet_name]
101
+
102
+ # for row_data in sheet.iter_rows(min_row=1, max_row=10):
103
+ # print(row_data)
@@ -0,0 +1,20 @@
1
+ from typing import Generic, TypeVarTuple, Unpack
2
+ from pymysql.cursors import Cursor
3
+ from pymysql.connections import Connection
4
+ from datacom_device_api.data.table import Table
5
+
6
+ Ts = TypeVarTuple('Ts')
7
+
8
+ class TableSaveToDB(Generic[Unpack[Ts]]):
9
+ def __init__(self, connect: Connection) -> None:
10
+ self.__cursor = connect.cursor()
11
+ self.__conn = connect
12
+ def save_to_db(self, db_table_name: str, table: Table[Unpack[Ts]]) -> None:
13
+ sql = f'INSERT INTO {db_table_name} VALUES ({",".join(["%s"] * len(table.headers))})'
14
+
15
+ for fragment in table.fragments:
16
+ values_list = fragment.rows
17
+ self.__cursor.executemany(sql, values_list)
18
+ self.__conn.commit()
19
+ print(table)
20
+
@@ -0,0 +1,244 @@
1
+ """多进程任务执行与进度展示封装。
2
+
3
+ 提供 TerminalProcess 类,封装 ProcessPoolExecutor 和 Rich Progress,
4
+ 支持任务提交、进度监听、异常处理、上下文管理器等能力。
5
+ """
6
+
7
+ from concurrent.futures import Future, ProcessPoolExecutor
8
+ from dataclasses import dataclass
9
+ from functools import partial
10
+ from multiprocessing import Manager
11
+ import os
12
+ import queue
13
+ import threading
14
+ from typing import Any, Callable, Optional
15
+ from uuid import UUID
16
+ import uuid
17
+
18
+ from rich.progress import Progress, TaskID
19
+
20
+ REPORT_FUNC_TYPE = Callable[[float, float, str], None]
21
+ EXECUTE_FUNC_TYPE = Callable[[REPORT_FUNC_TYPE], Any]
22
+
23
+
24
+ @dataclass
25
+ class ProgressMessage:
26
+ """进程间传递的进度消息。
27
+
28
+ Attributes:
29
+ type: 消息类型,取值为 'progress' / 'done' / 'error'
30
+ task_id: 任务唯一标识
31
+ message: 进度描述文本
32
+ progress: 当前进度值
33
+ total: 总进度值
34
+ """
35
+
36
+ type: str
37
+ task_id: UUID
38
+ message: str
39
+ progress: float
40
+ total: float
41
+
42
+
43
+ __PROGRESS_UPDATE_FN_TYPE = Callable[[ProgressMessage], None]
44
+
45
+
46
+ class TerminalProcess:
47
+ """多进程任务执行器,支持进度展示与监听器机制。
48
+
49
+ 每个实例拥有独立的进程池、进度队列和进度条,
50
+ 多实例之间状态互不干扰。
51
+
52
+ 用法:
53
+ with TerminalProcess() as process:
54
+ process.submit(task1)
55
+ process.submit(task2)
56
+ """
57
+
58
+ def __init__(self, max_workers: Optional[int] = None):
59
+ """初始化终端进程执行器。
60
+
61
+ Args:
62
+ max_workers: 进程池最大并发数,默认为 CPU 核心数
63
+ """
64
+ self.__pool = ProcessPoolExecutor(max_workers=max_workers or os.cpu_count())
65
+ self.__manager = Manager()
66
+ self.__progress_queue: queue.Queue[ProgressMessage] = self.__manager.Queue()
67
+
68
+ self.__task_future_count = 0
69
+ self.__tasks: dict[UUID, Future[Any]] = {}
70
+ self.__task_update_lisening_fn_dict: dict[UUID, list[__PROGRESS_UPDATE_FN_TYPE]] = {}
71
+
72
+ self.__progress_thread_stop_event = threading.Event()
73
+ self.__progress_tasks: dict[UUID, TaskID] = {}
74
+ self.__progress = Progress()
75
+ self.__progress_thread: Optional[threading.Thread] = None
76
+ self.__progress_main_task: TaskID = self.__progress.add_task(
77
+ 'main', total=self.__task_future_count
78
+ )
79
+
80
+ def trigger_progress(self, task_id: UUID, message: ProgressMessage) -> None:
81
+ """触发指定任务的所有进度监听器。
82
+
83
+ 若 task_id 未注册监听器则静默跳过。
84
+ """
85
+ listening_fns = self.__task_update_lisening_fn_dict.get(task_id, [])
86
+ for fn in listening_fns:
87
+ fn(message)
88
+
89
+ def listen_progress(self, task_id: UUID, fn: __PROGRESS_UPDATE_FN_TYPE) -> None:
90
+ """为指定任务注册一个进度监听器。
91
+
92
+ 同一任务可注册多个监听器,按注册顺序依次调用。
93
+ """
94
+ self.__task_update_lisening_fn_dict.setdefault(task_id, []).append(fn)
95
+
96
+ def update_progress(self, message: ProgressMessage) -> None:
97
+ """更新内置进度条的显示。
98
+
99
+ 任务完成时自动推进主进度条计数。
100
+ 若消息对应的任务不存在则静默跳过。
101
+ """
102
+ task_id = self.__progress_tasks.get(message.task_id)
103
+ if task_id is None:
104
+ return
105
+ self.__progress.update(task_id, completed=message.progress, total=message.total)
106
+ self.__progress.refresh()
107
+
108
+ if message.type == 'done':
109
+ self.__progress.update(self.__progress_main_task, advance=1)
110
+
111
+ @staticmethod
112
+ def _warrper_fn(
113
+ fn: EXECUTE_FUNC_TYPE,
114
+ progress_queue: queue.Queue[ProgressMessage],
115
+ task_id: UUID,
116
+ *args,
117
+ **kwargs,
118
+ ) -> Any:
119
+ """任务包装函数,在子进程中执行。
120
+
121
+ 封装进度上报、完成标记和异常处理,通过队列将进度消息
122
+ 发送回主进程。任务正常结束或异常均会发送对应类型的消息。
123
+ """
124
+ done_send = False
125
+
126
+ def report(progress: float, total: float, message: str, msg_type: str = 'progress'):
127
+ nonlocal done_send
128
+
129
+ if done_send:
130
+ return
131
+
132
+ current_type = msg_type
133
+ if progress >= total:
134
+ current_type = 'done'
135
+ done_send = True
136
+
137
+ progress_queue.put(ProgressMessage(
138
+ type=current_type,
139
+ task_id=task_id,
140
+ message=message,
141
+ progress=progress,
142
+ total=total,
143
+ ))
144
+
145
+ try:
146
+ result = fn(report, *args, **kwargs)
147
+ if not done_send:
148
+ report(100, 100, 'done', msg_type='done')
149
+ return result
150
+ except Exception as e:
151
+ progress_queue.put(ProgressMessage(
152
+ type='error',
153
+ task_id=task_id,
154
+ message=str(e),
155
+ progress=0,
156
+ total=0,
157
+ ))
158
+ raise
159
+
160
+ def run_display_progress(self) -> None:
161
+ """进度消费线程主循环。
162
+
163
+ 持续从队列中取出进度消息,依次更新内置进度条并触发监听器。
164
+ 使用带超时的轮询避免阻塞,确保停止事件触发后能及时退出。
165
+ """
166
+ while not self.__progress_thread_stop_event.is_set() or not self.__progress_queue.empty():
167
+ try:
168
+ message = self.__progress_queue.get(timeout=0.1)
169
+ self.update_progress(message)
170
+ self.trigger_progress(message.task_id, message)
171
+ except queue.Empty:
172
+ if (
173
+ self.__progress_thread_stop_event.is_set()
174
+ and all(futrue.done() for futrue in self.__tasks.values())
175
+ ):
176
+ break
177
+
178
+ def submit(
179
+ self,
180
+ fn: EXECUTE_FUNC_TYPE,
181
+ task_name: Optional[str] = None,
182
+ *args,
183
+ **kwargs,
184
+ ) -> Optional[Future[Any]]:
185
+ """提交一个任务到进程池执行。
186
+
187
+ Args:
188
+ fn: 任务函数,第一个参数为 report 回调,签名为 (progress, total, message)
189
+ task_name: 进度条显示的任务名,默认为 task_{uuid}
190
+ *args: 传递给任务函数的位置参数
191
+ **kwargs: 传递给任务函数的关键字参数
192
+
193
+ Returns:
194
+ 任务的 Future 对象,可用于获取结果或异常
195
+ """
196
+ task_id = uuid.uuid4()
197
+ task_fn = partial(TerminalProcess._warrper_fn, fn, self.__progress_queue, task_id)
198
+ future = self.__pool.submit(task_fn, *args, **kwargs)
199
+ self.__tasks[task_id] = future
200
+ self.__task_update_lisening_fn_dict[task_id] = []
201
+
202
+ self.__task_future_count += 1
203
+ progress_task = self.__progress.add_task(task_name or f'task_{task_id}', total=1)
204
+ self.__progress_tasks[task_id] = progress_task
205
+ self.__progress.refresh()
206
+
207
+ self.__progress.update(self.__progress_main_task, total=self.__task_future_count)
208
+ return future
209
+
210
+ def start(self) -> None:
211
+ """启动进度条显示与消息消费线程。"""
212
+ self.__progress.start()
213
+ self.__progress_thread_stop_event.clear()
214
+ self.__progress_thread = threading.Thread(target=self.run_display_progress)
215
+ self.__progress_thread.start()
216
+
217
+ def shutdown(self) -> None:
218
+ """停止任务执行并释放资源。
219
+
220
+ 按顺序执行:等待所有任务完成 → 消费完剩余进度消息 →
221
+ 关闭进程池 → 停止进度条。确保不会因消息未消费完而丢失进度。
222
+ """
223
+ for future in self.__tasks.values():
224
+ try:
225
+ future.result()
226
+ except Exception:
227
+ pass
228
+
229
+ self.__progress_thread_stop_event.set()
230
+ if self.__progress_thread is not None:
231
+ self.__progress_thread.join()
232
+ self.__progress_thread = None
233
+
234
+ self.__pool.shutdown(wait=True)
235
+ self.__progress.stop()
236
+
237
+ def __enter__(self) -> 'TerminalProcess':
238
+ """进入上下文管理器时自动启动。"""
239
+ self.start()
240
+ return self
241
+
242
+ def __exit__(self, exc_type, exc_val, exc_tb) -> None:
243
+ """退出上下文管理器时自动释放资源。"""
244
+ self.shutdown()
@@ -0,0 +1,13 @@
1
+ import os
2
+
3
+ def get_project_root_dir() -> str:
4
+ return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
5
+ def get_work_dir() -> str:
6
+ return os.getcwd()
7
+
8
+ def test():
9
+ print(get_project_root_dir())
10
+ print(get_work_dir())
11
+
12
+ if __name__ == '__main__':
13
+ test()
@@ -0,0 +1,14 @@
1
+ import logging
2
+ class Logger(object):
3
+ def __init__(self, name: str) -> None:
4
+ self.__logger = logging.getLogger(name)
5
+ self.__logger.setLevel(logging.INFO)
6
+ formatter = logging.Formatter('[%(asctime)s-%(levelname)s](%(name)s): %(message)s')
7
+
8
+ consolo_handler = logging.StreamHandler()
9
+ consolo_handler.setLevel(logging.INFO)
10
+ consolo_handler.setFormatter(formatter)
11
+ self.__logger.addHandler(consolo_handler)
12
+
13
+ def __getattr__(self, name):
14
+ return getattr(self.__logger, name)
@@ -0,0 +1,11 @@
1
+ Metadata-Version: 2.4
2
+ Name: datacom-device-api
3
+ Version: 0.1.5
4
+ Summary: Add your description here
5
+ Requires-Python: >=3.14
6
+ Description-Content-Type: text/markdown
7
+ Requires-Dist: dataclasses>=0.8
8
+ Requires-Dist: openpyxl>=3.1.5
9
+ Requires-Dist: ops>=3.8.0
10
+ Requires-Dist: pymysql>=1.2.0
11
+ Requires-Dist: rich>=15.0.0
@@ -0,0 +1,20 @@
1
+ README.md
2
+ pyproject.toml
3
+ src/datacom_device_api/__init__.py
4
+ src/datacom_device_api.egg-info/PKG-INFO
5
+ src/datacom_device_api.egg-info/SOURCES.txt
6
+ src/datacom_device_api.egg-info/dependency_links.txt
7
+ src/datacom_device_api.egg-info/requires.txt
8
+ src/datacom_device_api.egg-info/top_level.txt
9
+ src/datacom_device_api/data/__init__.py
10
+ src/datacom_device_api/data/table.py
11
+ src/datacom_device_api/operation/__init__.py
12
+ src/datacom_device_api/operation/table_operation.py
13
+ src/datacom_device_api/operation/table_read_from_xlsx.py
14
+ src/datacom_device_api/operation/table_save_to_db.py
15
+ src/datacom_device_api/process/__init__.py
16
+ src/datacom_device_api/process/terminal_process.py
17
+ src/datacom_device_api/utils/__init__.py
18
+ src/datacom_device_api/utils/directory_util.py
19
+ src/datacom_device_api/utils/logger.py
20
+ test/test_table_matching.py
@@ -0,0 +1,5 @@
1
+ dataclasses>=0.8
2
+ openpyxl>=3.1.5
3
+ ops>=3.8.0
4
+ pymysql>=1.2.0
5
+ rich>=15.0.0
@@ -0,0 +1 @@
1
+ datacom_device_api
@@ -0,0 +1,270 @@
1
+ import re
2
+ from datacom_device_api.operation.table_matching import StringTableMatching
3
+
4
+ CPU_OUTPUT = """CPU Usage Stat. Cycle: 60 (Second)
5
+ CPU Usage : 11% Max: 100%
6
+ CPU Usage Stat. Time : 2026-07-26 22:27:23
7
+ CPU utilization for five seconds: 11%: one minute: 8%: five minutes: 4%.
8
+
9
+ TaskName CPU Runtime(CPU Tick High/Tick Low) Task Explanation
10
+ BOX 0% 0/ 676cb3a BOX Output
11
+ _TIL 0% 0/ 0 Infinite loop event task
12
+ VCLK 0% 0/ 2b0c04b
13
+ TICK 0% 0/ 766bb97
14
+ co0 0% 0/ 3fee5e co0 Line user's task
15
+ U0 0% 0/ 0 U0 user command process
16
+ task
17
+ RTMR 0% 0/ 13842b6 RTMR
18
+ IPCR 0% 0/ 0 IPCR
19
+ VPR 0% 0/ 0 VPR
20
+ VPS 0% 0/ 0 VPS
21
+ EDBG 0% 0/ 0 EDBG
22
+ ECM 0% 0/ 0 ECM
23
+ DEV 0% 0/ 0 DEV Device
24
+ FCAT 0% 0/ 187e3a FCAT FECD task for catch
25
+ packet
26
+ FOAM 0% 0/ 0 FOAM
27
+ FTS 0% 0/ 0 FTS
28
+ IPCQ 0% 0/ 557dfd0 IPCQIPC task for single queue
29
+ IPCK 0% 0/ 0 IPCKIPC task for ack message
30
+ VP 0% 0/ 1b28d VP Virtual path task
31
+ RPCQ 0% 0/ ca3abd RPCQRemote procedure call
32
+ VFSD 0% 0/ 0 VFSDVFS flash task for delete
33
+ file block
34
+ VFS 0% 0/ 0 VFS Virtual file system
35
+ VMON 0% 0/ 0 VMONSystem monitor
36
+ VMSH 0% 0/ 0 VMSH
37
+ PAT 0% 0/ 0 PAT
38
+ HACK 0% 0/ 0 HACKtask for HA ACK
39
+ CLKI 0% 0/ 0 CLKI
40
+ STND 0% 0/ 0 STNDStandby task
41
+ INFO 0% 0/ f714a INFOInformation center
42
+ SAPP 0% 0/ 724205 SAPP
43
+ NQAC 0% 0/ 0 NQAC
44
+ NQAS 0% 0/ 0 NQAS
45
+ ALM 0% 0/ 36bdb ALM Alarm
46
+ ENSP 0% 0/ 11ba520 ENSP
47
+ DEVA 0% 0/ 0 DEVA Device assistant
48
+ SRM 0% 0/ 4fae8c5 SRM System Resource Manage
49
+ METH 0% 0/ 2cab20 METH
50
+ FIB6 0% 0/ 0 FIB6IPv6 FIB
51
+ BFD 0% 0/ 1e08781 BFD Bidirection Forwarding
52
+ Detect
53
+ TNLM 0% 0/ 1e927d2 TNLM
54
+ LSPA 0% 0/ 0 LSPA
55
+ L2V 0% 0/ 0 L2V
56
+ MACL 0% 0/ 7ff9c0 MACL
57
+ CCTL 0% 0/ 0 CCTLBulk stat connect control
58
+ TCTL 0% 0/ 0 TCTLBulk stat transmit
59
+ control
60
+ NAP 0% 0/ 0 NAP
61
+ TRAF 0% 0/ 55de90 TRAFTraffic Statistics
62
+ SLAG 0% 0/ 0 SLAG
63
+ ITSK 0% 0/ 3c398f ITSKIPOS common task
64
+ SNPG 0% 0/ 2a2f7a SNPG Multicast Snooping
65
+ MCSW 0% 0/ 2ee900 MCSW Mulitcast Switch
66
+ L3M4 0% 0/ 0 L3M4
67
+ L3I4 0% 0/ 0 L3I4
68
+ NDMB 0% 0/ 0 NDMB
69
+ NDIO 0% 0/ 0 NDIO
70
+ L3MB 0% 0/ 0 L3MB
71
+ L3IO 0% 0/ 0 L3IO
72
+ PNGM 0% 0/ 0 PNGM
73
+ FECD 0% 0/ 8ec6bf FECD Forward Equal Class
74
+ Develope
75
+ MPSI 0% 0/ a129b MPSI
76
+ MPSM 0% 0/ 0 MPSM
77
+ PPI 0% 0/ 268afa PPI Product Process Interface
78
+ MAC 0% 0/ 178cbf MAC Media Access Control
79
+ IFPD 0% 0/ 5ad822e IFPD Ifnet Product Adapt
80
+ TRUN 0% 0/ 0 TRUNK Product Adapt
81
+ STFW 0% 0/ 0 STFW Super task forward
82
+ XQOS 0% 0/ fe4207 XQOS Quality of service
83
+ MIRR 0% 0/ 0 MIRR
84
+ SMLK 0% 0/ 9902dc SMLK Smart Link Protocol
85
+ CDM 0% 0/ 1d895a CDM
86
+ NFPT 0% 0/ 222e67d NFPTNFP timer task
87
+ SOCK 0% 0/ 2fc739 SOCKPacket schedule and
88
+ process
89
+ FIB 0% 0/ 0 FIB Forward Information Base
90
+ MFIB 0% 0/ 1d53a MFIBMulticast forward info
91
+ IFNT 0% 0/ 0 IFNTIfnet task
92
+ VTYD 0% 0/ 7c0612 VTYDVirtual terminal
93
+ RSA 0% 0/ 0 RSA RSA public-key algorithms
94
+ AGNT 0% 0/ 0 AGNTSNMP agent task
95
+ TRAP 0% 0/ 0 TRAPSNMP trap task
96
+ FMAT 0% 0/ 174892 FMATFault Manage task
97
+ MDMT 0% 0/ 606d03e MDMTModem task
98
+ CFM 0% 0/ 0 CFM Configuration file
99
+ management
100
+ HS2M 0% 0/ 0 HS2MHigh available task
101
+ TACH 0% 0/ 2ef6727 TACHWTACACS
102
+ ACL 0% 0/ d5c88 ACL
103
+ SECE 0% 0/ 2051f1d SECE Security
104
+ DEFD 0% 0/ 9eb24 DEFD CPU Defend
105
+ STRA 0% 0/ 2731b STRA Source Track
106
+ MFF 0% 0/ 278c66 MFF MAC Forced Forwarding
107
+ LBDT 0% 0/ 74c44c5 LBDT Loopback Detect Mpu
108
+ BFDA 0% 0/ 0 BFDA BFD Adapter
109
+ DAD- 0% 0/ 0 DAD-proxy Task
110
+ MSYN 0% 0/ f12139 MSYN Mac Synchronization
111
+ AAA 0% 0/ 0 AAA
112
+ RDS 0% 0/ 0 RDS
113
+ UCM 0% 0/ c8e49 UCM User Control Management
114
+ DHCP 0% 0/ 4ea069 DHCP Dynamic Host Config
115
+ Protocol
116
+ AM 0% 0/ 661794 AM Address Management
117
+ PTAL 0% 0/ 0 PTAL Portal
118
+ PS 0% 0/ 7e140 PS Portal local server
119
+ HTPD 0% 0/ 23ac4b HTPD HTTP process for portal
120
+ EAP 0% 0/ e652b EAP Extensible Authen
121
+ Protocol
122
+ TM 0% 0/ 0 TM Transmission Management
123
+ SAM 0% 0/ 4cbf8 SAM Service Agent Module
124
+ WEB 0% 0/ 0 WEB Web
125
+ POE+ 0% 0/ 0 POE+ PPP Over Ethernet Plus
126
+ SMAG 0% 0/ 0 SMAG Smart Link Agent
127
+ GVRP 0% 0/ 0 GVRP Protocol
128
+ DLDP 0% 0/ 1e7e64 DLDP Protocol
129
+ LLDP 0% 0/ 3edb7e LLDP Protocol
130
+ UDPH 0% 0/ 0 UDPH
131
+ BPDU 0% 0/ 3df08 BPDU Adapter
132
+ EFMT 0% 0/ 28d270 EFMTEST 802.3AH Test
133
+ ADPT 0% 0/ 0 ADPT Adapter
134
+ ETHA 0% 0/ 0 ETHA
135
+ SPM 0% 0/ 25f5a48 SPM
136
+ ROUT 0% 0/ 4a529b7 ROUTRoute task
137
+ LSPM 0% 0/ 1ebb95 LSPMLsp management
138
+ RSVP 0% 0/ 0 RSVP task
139
+ LDP 0% 0/ 0 LDP task
140
+ GRES 0% 0/ 0 GRESM task
141
+ GEM 0% 0/ 0 GEM task
142
+ GEMR 0% 0/ 0 GEMRun task
143
+ UTSK 0% 0/ 0 UTSK
144
+ APP 0% 0/ 0 APP
145
+ IP 0% 0/ 21e93f IP
146
+ EOAM 0% 0/ 19df19 EOAM1AG
147
+ LINK 0% 0/ 67afb5 LINK
148
+ STP 0% 0/ 22ef8e2 STP
149
+ VRPT 0% 0/ 273415 VRPT
150
+ HOTT 0% 0/ 0 HOTT
151
+ TNQA 0% 0/ 2a0e5d TNQAC
152
+ TTNQ 0% 0/ 0 TTNQAS
153
+ TARP 0% 0/ 0 TARPING
154
+ TTVP 0% 0/ 0 TTVPLS
155
+ L2 0% 0/ f07caa L2
156
+ VRRP 0% 0/ 169431e VRRP
157
+ L2_P 0% 0/ 17b2e2a L2_PR
158
+ ARP 0% 0/ 0 ARP
159
+ NTPT 0% 0/ 207e8e NTPT task
160
+ SRMT 0% 0/ 880f55 SRMT System Resource Manage
161
+ Timer
162
+ SRMI 0% 0/ 0 SRMI External Interrupt
163
+ RMON 0% 0/ 23a925 RMONRemote monitoring
164
+ IFLP 0% 0/ de4ef IFLP
165
+ OS 100% 19/2ccb3fa0 Operation System
166
+ """
167
+
168
+
169
+ def test_extract_key_value_single():
170
+ print("=" * 60)
171
+ print("测试 1: extract_key_value_single - 提取单个 key:value")
172
+ print("=" * 60)
173
+
174
+ pattern = re.compile(r'CPU Usage\s+:\s*(?P<value>\d+)%')
175
+ result = StringTableMatching.extract_key_value_single(pattern, CPU_OUTPUT, key_group="value", value_group="value")
176
+ print(f"CPU 当前使用率: {result}")
177
+ assert result is not None
178
+ print("PASS\n")
179
+
180
+
181
+ def test_extract_key_value_multiple():
182
+ print("=" * 60)
183
+ print("测试 2: extract_key_value_multiple - 提取多个 key:value")
184
+ print("=" * 60)
185
+
186
+ pattern = re.compile(r'(?P<key>CPU Usage(?:\s+Stat\.\s+\w+)?)\s*:\s*(?P<value>[^\n]+?)\s*$', re.MULTILINE)
187
+ result = StringTableMatching.extract_key_value_multiple(pattern, CPU_OUTPUT)
188
+ print(f"提取到 {len(result)} 个 key:value 对:")
189
+ for k, v in result.items():
190
+ print(f" {k}: {v}")
191
+ assert len(result) >= 2
192
+ print("PASS\n")
193
+
194
+
195
+ def test_extract_values_by_groups():
196
+ print("=" * 60)
197
+ print("测试 3: extract_values_by_groups - 按命名分组提取任务表前几列")
198
+ print("=" * 60)
199
+
200
+ pattern = re.compile(
201
+ r'^(?P<task_name>\S+)\s+(?P<cpu>\d+)%\s+(?P<tick_high>\w+)\s*/\s*(?P<tick_low>\w+)',
202
+ re.MULTILINE
203
+ )
204
+ result = StringTableMatching.extract_values_by_groups(pattern, CPU_OUTPUT)
205
+ print(f"提取到 {len(result)} 个任务:")
206
+ for row in result[:5]:
207
+ print(f" {row}")
208
+ print(" ...")
209
+ assert len(result) > 0
210
+ print("PASS\n")
211
+
212
+
213
+ def test_extract_table_to_list():
214
+ print("=" * 60)
215
+ print("测试 4: extract_table_to_list - 提取表格为二维列表")
216
+ print("=" * 60)
217
+
218
+ lines = CPU_OUTPUT.strip().split('\n')
219
+ table_start = next(i for i, line in enumerate(lines) if line.startswith('TaskName'))
220
+ table_lines = lines[table_start + 1:]
221
+ table_text = '\n'.join(table_lines)
222
+
223
+ pattern = re.compile(r'(?P<value>\S+)')
224
+ result = StringTableMatching.extract_table_to_list(pattern, table_text)
225
+ print(f"提取到 {len(result)} 行:")
226
+ for row in result[:5]:
227
+ print(f" {row[:3]}...")
228
+ print(" ...")
229
+ assert len(result) > 0
230
+ print("PASS\n")
231
+
232
+
233
+ def test_cpu_utilization_line():
234
+ print("=" * 60)
235
+ print("测试 5: 提取 CPU 利用率行(5秒/1分钟/5分钟)")
236
+ print("=" * 60)
237
+
238
+ pattern = re.compile(
239
+ r'five seconds:\s*(?P<five_sec>\d+)%.*?one minute:\s*(?P<one_min>\d+)%.*?five minutes:\s*(?P<five_min>\d+)%'
240
+ )
241
+ result = StringTableMatching.extract_values_by_groups(pattern, CPU_OUTPUT)
242
+ print(f"CPU 利用率: {result}")
243
+ assert len(result) == 1
244
+ assert 'five_sec' in result[0]
245
+ print("PASS\n")
246
+
247
+
248
+ def test_os_task():
249
+ print("=" * 60)
250
+ print("测试 6: 提取 OS 任务(CPU 100% 的那行)")
251
+ print("=" * 60)
252
+
253
+ pattern = re.compile(r'^(?P<task>OS)\s+(?P<cpu>100)%\s+(?P<tick_high>\w+)\s*/\s*(?P<tick_low>\w+)\s+(?P<desc>.+)$', re.MULTILINE)
254
+ result = StringTableMatching.extract_values_by_groups(pattern, CPU_OUTPUT)
255
+ print(f"OS 任务: {result}")
256
+ assert len(result) == 1
257
+ assert result[0]['cpu'] == '100'
258
+ print("PASS\n")
259
+
260
+
261
+ if __name__ == '__main__':
262
+ test_extract_key_value_single()
263
+ test_extract_key_value_multiple()
264
+ test_extract_values_by_groups()
265
+ test_extract_table_to_list()
266
+ test_cpu_utilization_line()
267
+ test_os_task()
268
+ print("=" * 60)
269
+ print("所有测试通过!")
270
+ print("=" * 60)