atomicshop 2.14.3__py3-none-any.whl → 2.14.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of atomicshop might be problematic. Click here for more details.
- atomicshop/__init__.py +1 -1
- atomicshop/datetimes.py +3 -2
- atomicshop/etws/providers.py +18 -2
- atomicshop/etws/sessions.py +1 -1
- atomicshop/etws/trace.py +0 -1
- atomicshop/file_io/csvs.py +1 -1
- atomicshop/mitm/statistic_analyzer.py +376 -3
- atomicshop/wrappers/ctyping/etw_winapi/const.py +130 -20
- atomicshop/wrappers/ctyping/etw_winapi/etw_functions.py +141 -5
- atomicshop/wrappers/loggingw/reading.py +20 -19
- {atomicshop-2.14.3.dist-info → atomicshop-2.14.4.dist-info}/METADATA +1 -1
- {atomicshop-2.14.3.dist-info → atomicshop-2.14.4.dist-info}/RECORD +15 -15
- {atomicshop-2.14.3.dist-info → atomicshop-2.14.4.dist-info}/LICENSE.txt +0 -0
- {atomicshop-2.14.3.dist-info → atomicshop-2.14.4.dist-info}/WHEEL +0 -0
- {atomicshop-2.14.3.dist-info → atomicshop-2.14.4.dist-info}/top_level.txt +0 -0
atomicshop/__init__.py
CHANGED
atomicshop/datetimes.py
CHANGED
|
@@ -47,7 +47,7 @@ class MonthToNumber:
|
|
|
47
47
|
'דצמבר': '12'}
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
def get_datetime_from_complex_string_by_pattern(complex_string: str, date_pattern: str):
|
|
50
|
+
def get_datetime_from_complex_string_by_pattern(complex_string: str, date_pattern: str) -> tuple[datetime, str, float]:
|
|
51
51
|
"""
|
|
52
52
|
Function will get datetime object from a complex string by pattern.
|
|
53
53
|
|
|
@@ -65,7 +65,8 @@ def get_datetime_from_complex_string_by_pattern(complex_string: str, date_patter
|
|
|
65
65
|
if date_str:
|
|
66
66
|
# Convert the date string to a datetime object based on the given pattern
|
|
67
67
|
date_obj = datetime.datetime.strptime(date_str.group(), date_pattern)
|
|
68
|
-
|
|
68
|
+
date_timestamp = date_obj.timestamp()
|
|
69
|
+
return date_obj, date_str.group(), date_timestamp
|
|
69
70
|
else:
|
|
70
71
|
raise ValueError("No valid date found in the string")
|
|
71
72
|
|
atomicshop/etws/providers.py
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
|
+
from typing import Literal
|
|
2
|
+
|
|
1
3
|
from ..wrappers.ctyping.etw_winapi import etw_functions
|
|
2
4
|
|
|
3
5
|
|
|
4
|
-
def get_providers():
|
|
5
|
-
return etw_functions.get_all_providers()
|
|
6
|
+
def get_providers(key_as: Literal['name', 'guid'] = 'name'):
|
|
7
|
+
return etw_functions.get_all_providers(key_as=key_as)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def get_provider_guid_by_name(provider_name):
|
|
11
|
+
providers = get_providers(key_as='name')
|
|
12
|
+
|
|
13
|
+
try:
|
|
14
|
+
provider_guid = providers[provider_name]
|
|
15
|
+
except KeyError:
|
|
16
|
+
provider_guid = None
|
|
17
|
+
|
|
18
|
+
if not provider_guid:
|
|
19
|
+
raise ValueError(f"Provider '{provider_name}' not found")
|
|
20
|
+
|
|
21
|
+
return provider_guid
|
atomicshop/etws/sessions.py
CHANGED
atomicshop/etws/trace.py
CHANGED
atomicshop/file_io/csvs.py
CHANGED
|
@@ -36,7 +36,7 @@ def read_csv_to_list_of_dicts_by_header(
|
|
|
36
36
|
All the lines of the CSV file will be considered as content.
|
|
37
37
|
:param file_object: file object of the 'open()' function in the decorator. Decorator executes the 'with open()'
|
|
38
38
|
statement and passes to this function. That's why the default is 'None', since we get it from the decorator.
|
|
39
|
-
:return: list.
|
|
39
|
+
:return: tuple(list of entries, header(list of cell names)).
|
|
40
40
|
"""
|
|
41
41
|
|
|
42
42
|
# The header fields will be separated to list of "csv_reader.fieldnames".
|
|
@@ -1,8 +1,12 @@
|
|
|
1
|
+
import os
|
|
1
2
|
import datetime
|
|
3
|
+
import statistics
|
|
4
|
+
import json
|
|
5
|
+
from typing import Literal
|
|
2
6
|
|
|
3
7
|
from .. import filesystem, domains, datetimes, urls
|
|
4
8
|
from ..basics import dicts
|
|
5
|
-
from ..file_io import tomls, xlsxs
|
|
9
|
+
from ..file_io import tomls, xlsxs, csvs, jsons
|
|
6
10
|
from ..wrappers.loggingw import reading
|
|
7
11
|
from ..print_api import print_api
|
|
8
12
|
|
|
@@ -154,10 +158,10 @@ def analyze(main_file_path: str):
|
|
|
154
158
|
summary_path: str = filesystem.check_absolute_path___add_full(config['report_file_path'], script_directory)
|
|
155
159
|
|
|
156
160
|
# Get the content from statistics files.
|
|
157
|
-
statistics_content: list = reading.
|
|
161
|
+
statistics_content: list = reading.get_all_log_files_into_list(
|
|
158
162
|
config['statistic_files_path'],
|
|
159
163
|
file_name_pattern='statistics*.csv',
|
|
160
|
-
log_type='csv'
|
|
164
|
+
log_type='csv'
|
|
161
165
|
)
|
|
162
166
|
|
|
163
167
|
# Initialize loop.
|
|
@@ -465,3 +469,372 @@ def analyze(main_file_path: str):
|
|
|
465
469
|
xlsxs.write_xlsx(combined_sorted_stats, file_path=summary_path)
|
|
466
470
|
|
|
467
471
|
return
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
# ======================================================================================================================
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def calculate_moving_average(
|
|
478
|
+
file_path: str,
|
|
479
|
+
moving_average_window_days,
|
|
480
|
+
top_bottom_deviation_percentage: float,
|
|
481
|
+
print_kwargs: dict = None
|
|
482
|
+
):
|
|
483
|
+
"""
|
|
484
|
+
This function calculates the moving average of the daily statistics.
|
|
485
|
+
|
|
486
|
+
:param file_path: string, the path to the 'statistics.csv' file.
|
|
487
|
+
:param moving_average_window_days: integer, the window size for the moving average.
|
|
488
|
+
:param top_bottom_deviation_percentage: float, the percentage of deviation from the moving average to the top or
|
|
489
|
+
bottom.
|
|
490
|
+
:param print_kwargs: dict, the print_api arguments.
|
|
491
|
+
"""
|
|
492
|
+
|
|
493
|
+
date_pattern: str = '%Y_%m_%d'
|
|
494
|
+
|
|
495
|
+
# Get all the file paths and their midnight rotations.
|
|
496
|
+
logs_paths: list = reading.get_logs_paths(
|
|
497
|
+
log_file_path=file_path,
|
|
498
|
+
date_pattern=date_pattern
|
|
499
|
+
)
|
|
500
|
+
|
|
501
|
+
statistics_content: dict = {}
|
|
502
|
+
# Read each file to its day.
|
|
503
|
+
for log_path_dict in logs_paths:
|
|
504
|
+
date_string = log_path_dict['date_string']
|
|
505
|
+
statistics_content[date_string] = {}
|
|
506
|
+
|
|
507
|
+
statistics_content[date_string]['file'] = log_path_dict
|
|
508
|
+
|
|
509
|
+
log_file_content, log_file_header = (
|
|
510
|
+
csvs.read_csv_to_list_of_dicts_by_header(log_path_dict['file_path'], **(print_kwargs or {})))
|
|
511
|
+
statistics_content[date_string]['content'] = log_file_content
|
|
512
|
+
statistics_content[date_string]['header'] = log_file_header
|
|
513
|
+
|
|
514
|
+
statistics_content[date_string]['content_no_errors'] = get_content_without_errors(log_file_content)
|
|
515
|
+
|
|
516
|
+
# Get the data dictionary from the statistics content.
|
|
517
|
+
statistics_content[date_string]['statistics_daily'] = compute_statistics_from_content(
|
|
518
|
+
statistics_content[date_string]['content_no_errors']
|
|
519
|
+
)
|
|
520
|
+
|
|
521
|
+
moving_average_dict: dict = compute_moving_averages_from_average_statistics(
|
|
522
|
+
statistics_content,
|
|
523
|
+
moving_average_window_days
|
|
524
|
+
)
|
|
525
|
+
|
|
526
|
+
# Add the moving average to the statistics content.
|
|
527
|
+
for day, day_dict in statistics_content.items():
|
|
528
|
+
try:
|
|
529
|
+
day_dict['moving_average'] = moving_average_dict[day]
|
|
530
|
+
except KeyError:
|
|
531
|
+
day_dict['moving_average'] = {}
|
|
532
|
+
|
|
533
|
+
# Find deviation from the moving average to the bottom or top by specified percentage.
|
|
534
|
+
deviation_list: list = find_deviation_from_moving_average(
|
|
535
|
+
statistics_content, top_bottom_deviation_percentage)
|
|
536
|
+
|
|
537
|
+
return deviation_list
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def get_content_without_errors(content: list) -> list:
|
|
541
|
+
"""
|
|
542
|
+
This function gets the 'statistics.csv' file content without errors from the 'content' list.
|
|
543
|
+
|
|
544
|
+
:param content: list, the content list.
|
|
545
|
+
:return: list, the content without errors.
|
|
546
|
+
"""
|
|
547
|
+
|
|
548
|
+
traffic_statistics_without_errors: list = []
|
|
549
|
+
for line in content:
|
|
550
|
+
# Skip empty lines, headers and errors.
|
|
551
|
+
if line['host'] == 'host' or line['command'] == '':
|
|
552
|
+
continue
|
|
553
|
+
|
|
554
|
+
traffic_statistics_without_errors.append(line)
|
|
555
|
+
|
|
556
|
+
return traffic_statistics_without_errors
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def get_data_dict_from_statistics_content(content: list) -> dict:
|
|
560
|
+
"""
|
|
561
|
+
This function gets the data dictionary from the 'statistics.csv' file content.
|
|
562
|
+
|
|
563
|
+
:param content: list, the content list.
|
|
564
|
+
:return: dict, the data dictionary.
|
|
565
|
+
"""
|
|
566
|
+
|
|
567
|
+
hosts_requests_responses: dict = {}
|
|
568
|
+
for line in content:
|
|
569
|
+
# If subdomain is not in the dictionary, add it.
|
|
570
|
+
if line['host'] not in hosts_requests_responses:
|
|
571
|
+
hosts_requests_responses[line['host']] = {
|
|
572
|
+
'request_sizes': [],
|
|
573
|
+
'response_sizes': []
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
# Append the sizes.
|
|
577
|
+
try:
|
|
578
|
+
hosts_requests_responses[line['host']]['request_sizes'].append(int(line['request_size_bytes']))
|
|
579
|
+
hosts_requests_responses[line['host']]['response_sizes'].append(
|
|
580
|
+
int(line['response_size_bytes']))
|
|
581
|
+
except ValueError:
|
|
582
|
+
print_api(line, color='yellow')
|
|
583
|
+
raise
|
|
584
|
+
|
|
585
|
+
return hosts_requests_responses
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
def compute_statistics_from_data_dict(data_dict: dict):
|
|
589
|
+
"""
|
|
590
|
+
This function computes the statistics from the data dictionary.
|
|
591
|
+
|
|
592
|
+
:param data_dict: dict, the data dictionary.
|
|
593
|
+
:return: dict, the statistics dictionary.
|
|
594
|
+
"""
|
|
595
|
+
|
|
596
|
+
for host, host_dict in data_dict.items():
|
|
597
|
+
count = len(host_dict['request_sizes'])
|
|
598
|
+
avg_request_size = statistics.mean(host_dict['request_sizes']) if count > 0 else 0
|
|
599
|
+
median_request_size = statistics.median(host_dict['request_sizes']) if count > 0 else 0
|
|
600
|
+
avg_response_size = statistics.mean(host_dict['response_sizes']) if count > 0 else 0
|
|
601
|
+
median_response_size = statistics.median(host_dict['response_sizes']) if count > 0 else 0
|
|
602
|
+
|
|
603
|
+
data_dict[host]['count'] = count
|
|
604
|
+
data_dict[host]['avg_request_size'] = avg_request_size
|
|
605
|
+
data_dict[host]['median_request_size'] = median_request_size
|
|
606
|
+
data_dict[host]['avg_response_size'] = avg_response_size
|
|
607
|
+
data_dict[host]['median_response_size'] = median_response_size
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def compute_statistics_from_content(content: list):
|
|
611
|
+
"""
|
|
612
|
+
This function computes the statistics from the 'statistics.csv' file content.
|
|
613
|
+
|
|
614
|
+
:param content: list, the content list.
|
|
615
|
+
:return: dict, the statistics dictionary.
|
|
616
|
+
"""
|
|
617
|
+
|
|
618
|
+
hosts_requests_responses: dict = get_data_dict_from_statistics_content(content)
|
|
619
|
+
compute_statistics_from_data_dict(hosts_requests_responses)
|
|
620
|
+
|
|
621
|
+
return hosts_requests_responses
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
def compute_moving_averages_from_average_statistics(
|
|
625
|
+
average_statistics_dict: dict,
|
|
626
|
+
moving_average_window_days: int
|
|
627
|
+
):
|
|
628
|
+
"""
|
|
629
|
+
This function computes the moving averages from the average statistics dictionary.
|
|
630
|
+
|
|
631
|
+
:param average_statistics_dict: dict, the average statistics dictionary.
|
|
632
|
+
:param moving_average_window_days: integer, the window size for the moving average.
|
|
633
|
+
:return: dict, the moving averages dictionary.
|
|
634
|
+
"""
|
|
635
|
+
|
|
636
|
+
moving_average: dict = {}
|
|
637
|
+
for day_index, (day, day_dict) in enumerate(average_statistics_dict.items()):
|
|
638
|
+
current_day = day_index + 1
|
|
639
|
+
if current_day < moving_average_window_days:
|
|
640
|
+
continue
|
|
641
|
+
|
|
642
|
+
# Create list of the previous 'moving_average_window_days' days.
|
|
643
|
+
previous_days_content_list = (
|
|
644
|
+
list(average_statistics_dict.values()))[current_day-moving_average_window_days:current_day]
|
|
645
|
+
|
|
646
|
+
# Compute the moving averages.
|
|
647
|
+
moving_average[day] = compute_average_for_current_day_from_past_x_days(previous_days_content_list)
|
|
648
|
+
|
|
649
|
+
return moving_average
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
def compute_average_for_current_day_from_past_x_days(previous_days_content_list: list) -> dict:
|
|
653
|
+
"""
|
|
654
|
+
This function computes the average for the current day from the past x days.
|
|
655
|
+
|
|
656
|
+
:param previous_days_content_list: list, the list of the previous days content.
|
|
657
|
+
:return: dict, the average dictionary.
|
|
658
|
+
"""
|
|
659
|
+
|
|
660
|
+
moving_average: dict = {}
|
|
661
|
+
for entry in previous_days_content_list:
|
|
662
|
+
statistics_daily = entry['statistics_daily']
|
|
663
|
+
for host, host_dict in statistics_daily.items():
|
|
664
|
+
if host not in moving_average:
|
|
665
|
+
moving_average[host] = {
|
|
666
|
+
'counts': [],
|
|
667
|
+
'avg_request_sizes': [],
|
|
668
|
+
'avg_response_sizes': [],
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
moving_average[host]['counts'].append(int(host_dict['count']))
|
|
672
|
+
moving_average[host]['avg_request_sizes'].append(float(host_dict['avg_request_size']))
|
|
673
|
+
moving_average[host]['avg_response_sizes'].append(float(host_dict['avg_response_size']))
|
|
674
|
+
|
|
675
|
+
# Compute the moving average.
|
|
676
|
+
moving_average_results: dict = {}
|
|
677
|
+
for host, host_dict in moving_average.items():
|
|
678
|
+
ma_count = statistics.mean(host_dict['counts'])
|
|
679
|
+
ma_request_size = statistics.mean(host_dict['avg_request_sizes'])
|
|
680
|
+
ma_response_size = statistics.mean(host_dict['avg_response_sizes'])
|
|
681
|
+
|
|
682
|
+
moving_average_results[host] = {
|
|
683
|
+
'ma_count': ma_count,
|
|
684
|
+
'ma_request_size': ma_request_size,
|
|
685
|
+
'ma_response_size': ma_response_size,
|
|
686
|
+
'counts': host_dict['counts'],
|
|
687
|
+
'avg_request_sizes': host_dict['avg_request_sizes'],
|
|
688
|
+
'avg_response_sizes': host_dict['avg_response_sizes']
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
return moving_average_results
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def find_deviation_from_moving_average(
|
|
695
|
+
statistics_content: dict,
|
|
696
|
+
top_bottom_deviation_percentage: float
|
|
697
|
+
) -> list:
|
|
698
|
+
"""
|
|
699
|
+
This function finds the deviation from the moving average to the bottom or top by specified percentage.
|
|
700
|
+
|
|
701
|
+
:param statistics_content: dict, the statistics content dictionary.
|
|
702
|
+
:param top_bottom_deviation_percentage: float, the percentage of deviation from the moving average to the top or
|
|
703
|
+
bottom.
|
|
704
|
+
:return: list, the deviation list.
|
|
705
|
+
"""
|
|
706
|
+
|
|
707
|
+
def _check_deviation(
|
|
708
|
+
check_type: Literal['count', 'avg_request_size', 'avg_response_size'],
|
|
709
|
+
ma_check_type: Literal['ma_count', 'ma_request_size', 'ma_response_size'],
|
|
710
|
+
day_statistics_content_dict: dict,
|
|
711
|
+
moving_averages_dict: dict
|
|
712
|
+
):
|
|
713
|
+
"""
|
|
714
|
+
This function checks the deviation for the host.
|
|
715
|
+
"""
|
|
716
|
+
|
|
717
|
+
nonlocal message
|
|
718
|
+
|
|
719
|
+
host_moving_average_by_type = moving_averages_dict[host][ma_check_type]
|
|
720
|
+
check_type_moving_by_percent = (
|
|
721
|
+
host_moving_average_by_type * top_bottom_deviation_percentage)
|
|
722
|
+
check_type_moving_above = host_moving_average_by_type + check_type_moving_by_percent
|
|
723
|
+
check_type_moving_below = host_moving_average_by_type - check_type_moving_by_percent
|
|
724
|
+
|
|
725
|
+
deviation_type = None
|
|
726
|
+
if day_statistics_content_dict[check_type] > check_type_moving_above:
|
|
727
|
+
deviation_type = 'above'
|
|
728
|
+
elif day_statistics_content_dict[check_type] < check_type_moving_below:
|
|
729
|
+
deviation_type = 'below'
|
|
730
|
+
|
|
731
|
+
if deviation_type:
|
|
732
|
+
message = f'[{check_type}] is [{deviation_type}] the moving average.'
|
|
733
|
+
deviation_list.append({
|
|
734
|
+
'day': day,
|
|
735
|
+
'host': host,
|
|
736
|
+
'message': message,
|
|
737
|
+
'value': day_statistics_content_dict[check_type],
|
|
738
|
+
'ma_value': host_moving_average_by_type,
|
|
739
|
+
'check_type': check_type,
|
|
740
|
+
'percentage': top_bottom_deviation_percentage,
|
|
741
|
+
'ma_value_checked': check_type_moving_above,
|
|
742
|
+
'deviation_type': deviation_type,
|
|
743
|
+
'data': day_statistics_content_dict,
|
|
744
|
+
'ma_data': moving_averages_dict[host]
|
|
745
|
+
})
|
|
746
|
+
|
|
747
|
+
deviation_list: list = []
|
|
748
|
+
for day_index, (day, day_dict) in enumerate(statistics_content.items()):
|
|
749
|
+
# If it's the first day, there is no previous day moving average.
|
|
750
|
+
if day_index == 0:
|
|
751
|
+
previous_day_moving_average_dict = {}
|
|
752
|
+
else:
|
|
753
|
+
previous_day_moving_average_dict = list(statistics_content.values())[day_index-1].get('moving_average', {})
|
|
754
|
+
|
|
755
|
+
# If there is no moving average for previous day continue to the next day.
|
|
756
|
+
if not previous_day_moving_average_dict:
|
|
757
|
+
continue
|
|
758
|
+
|
|
759
|
+
for host, host_dict in day_dict['statistics_daily'].items():
|
|
760
|
+
# If the host is not in the moving averages, then this is clear deviation.
|
|
761
|
+
# It means that in the current day, there were no requests for this host.
|
|
762
|
+
if host not in previous_day_moving_average_dict:
|
|
763
|
+
message = f'Host not in the moving averages: {host}'
|
|
764
|
+
deviation_list.append({
|
|
765
|
+
'day': day,
|
|
766
|
+
'host': host,
|
|
767
|
+
'data': host_dict,
|
|
768
|
+
'message': message,
|
|
769
|
+
'type': 'clear'
|
|
770
|
+
})
|
|
771
|
+
continue
|
|
772
|
+
|
|
773
|
+
_check_deviation(
|
|
774
|
+
'count', 'ma_count', host_dict, previous_day_moving_average_dict)
|
|
775
|
+
_check_deviation(
|
|
776
|
+
'avg_request_size', 'ma_request_size', host_dict, previous_day_moving_average_dict)
|
|
777
|
+
_check_deviation(
|
|
778
|
+
'avg_response_size', 'ma_response_size', host_dict, previous_day_moving_average_dict)
|
|
779
|
+
|
|
780
|
+
return deviation_list
|
|
781
|
+
|
|
782
|
+
|
|
783
|
+
def moving_average_calculator_main(
|
|
784
|
+
statistics_file_path: str,
|
|
785
|
+
output_directory: str,
|
|
786
|
+
moving_average_window_days: int,
|
|
787
|
+
top_bottom_deviation_percentage: float
|
|
788
|
+
) -> int:
|
|
789
|
+
"""
|
|
790
|
+
This function is the main function for the moving average calculator.
|
|
791
|
+
|
|
792
|
+
:param statistics_file_path: string, the statistics file path.
|
|
793
|
+
:param output_directory: string, the output directory.
|
|
794
|
+
:param moving_average_window_days: integer, the moving average window days.
|
|
795
|
+
:param top_bottom_deviation_percentage: float, the top bottom deviation percentage. Example: 0.1 for 10%.
|
|
796
|
+
:return: integer, the return code.
|
|
797
|
+
-----------------------------
|
|
798
|
+
|
|
799
|
+
Example:
|
|
800
|
+
import sys
|
|
801
|
+
from atomicshop.mitm import statistic_analyzer
|
|
802
|
+
|
|
803
|
+
|
|
804
|
+
def main():
|
|
805
|
+
return statistic_analyzer.moving_average_calculator_main(
|
|
806
|
+
statistics_file_path='statistics.csv',
|
|
807
|
+
output_directory='output',
|
|
808
|
+
moving_average_window_days=7,
|
|
809
|
+
top_bottom_deviation_percentage=0.1
|
|
810
|
+
)
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
if __name__ == '__main__':
|
|
814
|
+
sys.exit(main())
|
|
815
|
+
"""
|
|
816
|
+
|
|
817
|
+
def convert_data_value_to_string(value_key: str, list_index: int) -> None:
|
|
818
|
+
deviation_list[list_index]['data'][value_key] = json.dumps(deviation_list[list_index]['data'][value_key])
|
|
819
|
+
|
|
820
|
+
def convert_value_to_string(value_key: str, list_index: int) -> None:
|
|
821
|
+
if value_key in deviation_list[list_index]:
|
|
822
|
+
deviation_list[list_index][value_key] = json.dumps(deviation_list[list_index][value_key])
|
|
823
|
+
|
|
824
|
+
deviation_list = calculate_moving_average(
|
|
825
|
+
statistics_file_path,
|
|
826
|
+
moving_average_window_days,
|
|
827
|
+
top_bottom_deviation_percentage
|
|
828
|
+
)
|
|
829
|
+
|
|
830
|
+
if deviation_list:
|
|
831
|
+
for deviation_list_index, deviation in enumerate(deviation_list):
|
|
832
|
+
convert_data_value_to_string('request_sizes', deviation_list_index)
|
|
833
|
+
convert_data_value_to_string('response_sizes', deviation_list_index)
|
|
834
|
+
convert_value_to_string('ma_data', deviation_list_index)
|
|
835
|
+
|
|
836
|
+
file_path = output_directory + os.sep + 'deviation.json'
|
|
837
|
+
print_api(f'Deviation Found, saving to file: {file_path}', color='blue')
|
|
838
|
+
jsons.write_json_file(deviation_list, file_path, use_default_indent=True)
|
|
839
|
+
|
|
840
|
+
return 0
|
|
@@ -6,8 +6,15 @@ from ctypes.wintypes import ULONG
|
|
|
6
6
|
# Constants
|
|
7
7
|
EVENT_TRACE_CONTROL_STOP = 1
|
|
8
8
|
WNODE_FLAG_TRACED_GUID = 0x00020000
|
|
9
|
+
EVENT_TRACE_REAL_TIME_MODE = 0x00000100
|
|
10
|
+
EVENT_CONTROL_CODE_ENABLE_PROVIDER = 1
|
|
9
11
|
|
|
10
12
|
MAXIMUM_LOGGERS = 64
|
|
13
|
+
ULONG64 = ctypes.c_uint64
|
|
14
|
+
|
|
15
|
+
INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value
|
|
16
|
+
TRACEHANDLE = ULONG64
|
|
17
|
+
|
|
11
18
|
|
|
12
19
|
|
|
13
20
|
"""
|
|
@@ -43,7 +50,6 @@ class WNODE_HEADER(ctypes.Structure):
|
|
|
43
50
|
]
|
|
44
51
|
|
|
45
52
|
|
|
46
|
-
# Define EVENT_TRACE_PROPERTIES structure
|
|
47
53
|
class EVENT_TRACE_PROPERTIES(ctypes.Structure):
|
|
48
54
|
_fields_ = [
|
|
49
55
|
("Wnode", WNODE_HEADER),
|
|
@@ -70,29 +76,32 @@ class EVENT_TRACE_PROPERTIES(ctypes.Structure):
|
|
|
70
76
|
]
|
|
71
77
|
|
|
72
78
|
|
|
73
|
-
|
|
74
|
-
class EVENT_TRACE_LOGFILE(ctypes.Structure):
|
|
79
|
+
class TRACE_LOGFILE_HEADER(ctypes.Structure):
|
|
75
80
|
_fields_ = [
|
|
76
|
-
("LogFileName", wintypes.LPWSTR),
|
|
77
|
-
("LoggerName", wintypes.LPWSTR),
|
|
78
|
-
("CurrentTime", wintypes.LARGE_INTEGER),
|
|
79
|
-
("BuffersRead", wintypes.ULONG),
|
|
80
|
-
("ProcessTraceMode", wintypes.ULONG),
|
|
81
|
-
("EventRecordCallback", wintypes.LPVOID),
|
|
82
81
|
("BufferSize", wintypes.ULONG),
|
|
83
|
-
("
|
|
84
|
-
("
|
|
85
|
-
("
|
|
86
|
-
("
|
|
87
|
-
("
|
|
88
|
-
("
|
|
82
|
+
("Version", wintypes.ULONG),
|
|
83
|
+
("ProviderVersion", wintypes.ULONG),
|
|
84
|
+
("NumberOfProcessors", wintypes.ULONG),
|
|
85
|
+
("EndTime", wintypes.LARGE_INTEGER),
|
|
86
|
+
("TimerResolution", wintypes.ULONG),
|
|
87
|
+
("MaximumFileSize", wintypes.ULONG),
|
|
89
88
|
("LogFileMode", wintypes.ULONG),
|
|
90
|
-
("
|
|
91
|
-
("
|
|
89
|
+
("BuffersWritten", wintypes.ULONG),
|
|
90
|
+
("StartBuffers", wintypes.ULONG),
|
|
91
|
+
("PointerSize", wintypes.ULONG),
|
|
92
|
+
("EventsLost", wintypes.ULONG),
|
|
93
|
+
("CpuSpeedInMHz", wintypes.ULONG),
|
|
94
|
+
("LoggerName", wintypes.WCHAR * 256),
|
|
95
|
+
("LogFileName", wintypes.WCHAR * 256),
|
|
96
|
+
("TimeZone", wintypes.LPVOID),
|
|
97
|
+
("BootTime", wintypes.LARGE_INTEGER),
|
|
98
|
+
("PerfFreq", wintypes.LARGE_INTEGER),
|
|
99
|
+
("StartTime", wintypes.LARGE_INTEGER),
|
|
100
|
+
("ReservedFlags", wintypes.ULONG),
|
|
101
|
+
("BuffersLost", wintypes.ULONG)
|
|
92
102
|
]
|
|
93
103
|
|
|
94
104
|
|
|
95
|
-
# Define the EVENT_TRACE_HEADER structure
|
|
96
105
|
class EVENT_TRACE_HEADER(ctypes.Structure):
|
|
97
106
|
_fields_ = [
|
|
98
107
|
("Size", wintypes.USHORT),
|
|
@@ -116,10 +125,67 @@ class EVENT_TRACE_HEADER(ctypes.Structure):
|
|
|
116
125
|
]
|
|
117
126
|
|
|
118
127
|
|
|
119
|
-
|
|
128
|
+
class EVENT_TRACE(ctypes.Structure):
|
|
129
|
+
_fields_ = [
|
|
130
|
+
("Header", EVENT_TRACE_HEADER),
|
|
131
|
+
("InstanceId", wintypes.DWORD),
|
|
132
|
+
("ParentInstanceId", wintypes.DWORD),
|
|
133
|
+
("ParentGuid", GUID),
|
|
134
|
+
("MofData", ctypes.c_void_p),
|
|
135
|
+
("MofLength", wintypes.ULONG),
|
|
136
|
+
("ClientContext", wintypes.ULONG)
|
|
137
|
+
]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class EVENT_TRACE_LOGFILEW(ctypes.Structure):
|
|
141
|
+
_fields_ = [
|
|
142
|
+
("LogFileName", ctypes.c_wchar_p),
|
|
143
|
+
("LoggerName", ctypes.c_wchar_p),
|
|
144
|
+
("CurrentTime", wintypes.LARGE_INTEGER),
|
|
145
|
+
("BuffersRead", wintypes.ULONG),
|
|
146
|
+
("ProcessTraceMode", wintypes.ULONG),
|
|
147
|
+
("CurrentEvent", EVENT_TRACE),
|
|
148
|
+
("LogfileHeader", TRACE_LOGFILE_HEADER),
|
|
149
|
+
("BufferCallback", ctypes.c_void_p), # Placeholder for buffer callback
|
|
150
|
+
("BufferSize", wintypes.ULONG),
|
|
151
|
+
("Filled", wintypes.ULONG),
|
|
152
|
+
("EventsLost", wintypes.ULONG),
|
|
153
|
+
("EventCallback", ctypes.CFUNCTYPE(None, ctypes.POINTER(EVENT_TRACE))),
|
|
154
|
+
("Context", ULONG64)
|
|
155
|
+
]
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
class EVENT_DESCRIPTOR(ctypes.Structure):
|
|
159
|
+
_fields_ = [
|
|
160
|
+
("Id", wintypes.USHORT),
|
|
161
|
+
("Version", wintypes.BYTE),
|
|
162
|
+
("Channel", wintypes.BYTE),
|
|
163
|
+
("Level", wintypes.BYTE),
|
|
164
|
+
("Opcode", wintypes.BYTE),
|
|
165
|
+
("Task", wintypes.USHORT),
|
|
166
|
+
("Keyword", ULONG64),
|
|
167
|
+
]
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class EVENT_HEADER(ctypes.Structure):
|
|
171
|
+
_fields_ = [
|
|
172
|
+
("Size", wintypes.USHORT),
|
|
173
|
+
("HeaderType", wintypes.USHORT),
|
|
174
|
+
("Flags", wintypes.USHORT),
|
|
175
|
+
("EventProperty", wintypes.USHORT),
|
|
176
|
+
("ThreadId", wintypes.ULONG),
|
|
177
|
+
("ProcessId", wintypes.ULONG),
|
|
178
|
+
("TimeStamp", wintypes.LARGE_INTEGER),
|
|
179
|
+
("ProviderId", GUID),
|
|
180
|
+
("EventDescriptor", EVENT_DESCRIPTOR),
|
|
181
|
+
("ProcessorTime", ULONG64),
|
|
182
|
+
("ActivityId", GUID),
|
|
183
|
+
("RelatedActivityId", GUID),
|
|
184
|
+
]
|
|
185
|
+
|
|
120
186
|
class EVENT_RECORD(ctypes.Structure):
|
|
121
187
|
_fields_ = [
|
|
122
|
-
("EventHeader",
|
|
188
|
+
("EventHeader", EVENT_HEADER),
|
|
123
189
|
("BufferContext", wintypes.ULONG),
|
|
124
190
|
("ExtendedDataCount", wintypes.USHORT),
|
|
125
191
|
("UserDataLength", wintypes.USHORT),
|
|
@@ -129,6 +195,50 @@ class EVENT_RECORD(ctypes.Structure):
|
|
|
129
195
|
]
|
|
130
196
|
|
|
131
197
|
|
|
198
|
+
# class EVENT_TRACE_LOGFILE(ctypes.Structure):
|
|
199
|
+
# _fields_ = [
|
|
200
|
+
# ("LogFileName", wintypes.LPWSTR),
|
|
201
|
+
# ("LoggerName", wintypes.LPWSTR),
|
|
202
|
+
# ("CurrentTime", wintypes.LARGE_INTEGER),
|
|
203
|
+
# ("BuffersRead", wintypes.ULONG),
|
|
204
|
+
# ("ProcessTraceMode", wintypes.ULONG),
|
|
205
|
+
# ("EventRecordCallback", wintypes.LPVOID),
|
|
206
|
+
# ("BufferSize", wintypes.ULONG),
|
|
207
|
+
# ("Filled", wintypes.ULONG),
|
|
208
|
+
# ("EventsLost", wintypes.ULONG),
|
|
209
|
+
# ("BuffersLost", wintypes.ULONG),
|
|
210
|
+
# ("RealTimeBuffersLost", wintypes.ULONG),
|
|
211
|
+
# ("LogBuffersLost", wintypes.ULONG),
|
|
212
|
+
# ("BuffersWritten", wintypes.ULONG),
|
|
213
|
+
# ("LogFileMode", wintypes.ULONG),
|
|
214
|
+
# ("IsKernelTrace", wintypes.ULONG),
|
|
215
|
+
# ("Context", wintypes.ULONG) # Placeholder for context pointer
|
|
216
|
+
# ]
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
class EVENT_TRACE_LOGFILE(ctypes.Structure):
|
|
220
|
+
_fields_ = [
|
|
221
|
+
("LogFileName", wintypes.LPWSTR),
|
|
222
|
+
("LoggerName", wintypes.LPWSTR),
|
|
223
|
+
("CurrentTime", wintypes.LARGE_INTEGER),
|
|
224
|
+
("BuffersRead", wintypes.ULONG),
|
|
225
|
+
("ProcessTraceMode", wintypes.ULONG),
|
|
226
|
+
("CurrentEvent", EVENT_RECORD),
|
|
227
|
+
("LogfileHeader", TRACE_LOGFILE_HEADER),
|
|
228
|
+
("BufferCallback", wintypes.LPVOID),
|
|
229
|
+
("BufferSize", wintypes.ULONG),
|
|
230
|
+
("Filled", wintypes.ULONG),
|
|
231
|
+
("EventsLost", wintypes.ULONG),
|
|
232
|
+
("EventCallback", ctypes.c_void_p),
|
|
233
|
+
("IsKernelTrace", wintypes.ULONG),
|
|
234
|
+
("Context", wintypes.LPVOID)
|
|
235
|
+
]
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
# Define the callback type for processing events
|
|
239
|
+
EVENT_CALLBACK_TYPE = ctypes.WINFUNCTYPE(None, ctypes.POINTER(EVENT_RECORD))
|
|
240
|
+
|
|
241
|
+
|
|
132
242
|
class PROVIDER_ENUMERATION_INFO(ctypes.Structure):
|
|
133
243
|
_fields_ = [
|
|
134
244
|
("NumberOfProviders", ULONG),
|
|
@@ -1,12 +1,137 @@
|
|
|
1
1
|
import ctypes
|
|
2
|
+
from ctypes import wintypes
|
|
2
3
|
from ctypes.wintypes import ULONG
|
|
3
4
|
import uuid
|
|
5
|
+
from typing import Literal
|
|
4
6
|
|
|
5
7
|
from . import const
|
|
8
|
+
from ....etws import providers
|
|
9
|
+
|
|
10
|
+
class ETWSessionExists(Exception):
|
|
11
|
+
pass
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
# Convert the GUID string to a GUID structure
|
|
15
|
+
def _string_to_guid(guid_string):
|
|
16
|
+
guid_string = guid_string.strip('{}') # Remove curly braces
|
|
17
|
+
parts = guid_string.split('-')
|
|
18
|
+
return const.GUID(
|
|
19
|
+
Data1=int(parts[0], 16),
|
|
20
|
+
Data2=int(parts[1], 16),
|
|
21
|
+
Data3=int(parts[2], 16),
|
|
22
|
+
Data4=(ctypes.c_ubyte * 8)(*[
|
|
23
|
+
int(parts[3][i:i+2], 16) for i in range(0, 4, 2)
|
|
24
|
+
] + [
|
|
25
|
+
int(parts[4][i:i+2], 16) for i in range(0, 12, 2)
|
|
26
|
+
])
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# Set up the ETW session
|
|
31
|
+
def start_etw_session(
|
|
32
|
+
session_name: str,
|
|
33
|
+
provider_guid_list: list = None,
|
|
34
|
+
provider_name_list: list = None,
|
|
35
|
+
verbosity_mode: int = 4,
|
|
36
|
+
maximum_buffers: int = 38
|
|
37
|
+
):
|
|
38
|
+
"""
|
|
39
|
+
Start an ETW session and enable the specified provider.
|
|
40
|
+
|
|
41
|
+
:param session_name: The name of the session to start.
|
|
42
|
+
:param provider_guid_list: The GUID list of the providers to enable.
|
|
43
|
+
:param provider_name_list: The name list of the providers to enable.
|
|
44
|
+
:param verbosity_mode: The verbosity level of the events to capture.
|
|
45
|
+
0 - Always: Capture all events. This is typically used for critical events that should always be logged.
|
|
46
|
+
1 - Critical: Capture critical events that indicate a severe problem.
|
|
47
|
+
2 - Error: Capture error events that indicate a problem but are not critical.
|
|
48
|
+
3 - Warning: Capture warning events that indicate a potential problem.
|
|
49
|
+
4 - Information: Capture informational events that are not indicative of problems.
|
|
50
|
+
5 - Verbose: Capture detailed trace events for diagnostic purposes.
|
|
51
|
+
:param maximum_buffers: The maximum number of buffers to use.
|
|
52
|
+
0 or 16: The default value of ETW class. If you put 0, it will be converted to 16 by default by ETW itself.
|
|
53
|
+
38: The maximum number of buffers that can be used.
|
|
54
|
+
|
|
55
|
+
Event Handling Capacity:
|
|
56
|
+
16 Buffers: With fewer buffers, the session can handle a smaller volume of events before needing to flush
|
|
57
|
+
the buffers to the log file or before a real-time consumer needs to process them. If the buffers fill up
|
|
58
|
+
quickly and cannot be processed in time, events might be lost.
|
|
59
|
+
38 Buffers: With more buffers, the session can handle a higher volume of events. This reduces the
|
|
60
|
+
likelihood of losing events in high-traffic scenarios because more events can be held in memory before
|
|
61
|
+
they need to be processed or written to a log file.
|
|
62
|
+
Performance Considerations:
|
|
63
|
+
16 Buffers: Requires less memory, but may be prone to event loss under heavy load if the buffers fill up
|
|
64
|
+
faster than they can be processed.
|
|
65
|
+
38 Buffers: Requires more memory, but can improve reliability in capturing all events under heavy load by
|
|
66
|
+
providing more buffer space. However, it can also increase the memory footprint of the application or
|
|
67
|
+
system running the ETW session.
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
if not provider_guid_list and not provider_name_list:
|
|
71
|
+
raise ValueError("Either 'provider_guid_list' or 'provider_name_list' must be provided")
|
|
72
|
+
elif provider_guid_list and provider_name_list:
|
|
73
|
+
raise ValueError("Only one of 'provider_guid_list' or 'provider_name_list' must be provided")
|
|
74
|
+
|
|
75
|
+
if provider_name_list:
|
|
76
|
+
provider_guid_list = []
|
|
77
|
+
for provider_name in provider_name_list:
|
|
78
|
+
provider_guid_list.append(providers.get_provider_guid_by_name(provider_name))
|
|
79
|
+
|
|
80
|
+
properties_size = ctypes.sizeof(const.EVENT_TRACE_PROPERTIES) + (2 * wintypes.MAX_PATH)
|
|
81
|
+
properties = (ctypes.c_byte * properties_size)()
|
|
82
|
+
properties_ptr = ctypes.cast(properties, ctypes.POINTER(const.EVENT_TRACE_PROPERTIES))
|
|
83
|
+
|
|
84
|
+
# Initialize the EVENT_TRACE_PROPERTIES structure
|
|
85
|
+
properties_ptr.contents.Wnode.BufferSize = properties_size
|
|
86
|
+
properties_ptr.contents.Wnode.Guid = const.GUID()
|
|
87
|
+
properties_ptr.contents.Wnode.Flags = const.WNODE_FLAG_TRACED_GUID
|
|
88
|
+
properties_ptr.contents.Wnode.ClientContext = 1 # QPC clock resolution
|
|
89
|
+
properties_ptr.contents.BufferSize = 1024
|
|
90
|
+
properties_ptr.contents.MinimumBuffers = 1
|
|
91
|
+
properties_ptr.contents.MaximumBuffers = maximum_buffers
|
|
92
|
+
properties_ptr.contents.MaximumFileSize = 0
|
|
93
|
+
properties_ptr.contents.LogFileMode = const.EVENT_TRACE_REAL_TIME_MODE
|
|
94
|
+
properties_ptr.contents.FlushTimer = 1
|
|
95
|
+
properties_ptr.contents.EnableFlags = 0
|
|
96
|
+
|
|
97
|
+
# Start the ETW session
|
|
98
|
+
session_handle = wintypes.HANDLE()
|
|
99
|
+
status = ctypes.windll.advapi32.StartTraceW(
|
|
100
|
+
ctypes.byref(session_handle),
|
|
101
|
+
ctypes.c_wchar_p(session_name),
|
|
102
|
+
properties_ptr
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
if status != 0:
|
|
106
|
+
if status == 183:
|
|
107
|
+
raise ETWSessionExists(f"ETW session [{session_name}] already exists")
|
|
108
|
+
else:
|
|
109
|
+
raise Exception(f"StartTraceW failed with error {status}")
|
|
110
|
+
|
|
111
|
+
# Enable each provider
|
|
112
|
+
for provider_guid in provider_guid_list:
|
|
113
|
+
provider_guid_struct = _string_to_guid(provider_guid)
|
|
114
|
+
status = ctypes.windll.advapi32.EnableTraceEx2(
|
|
115
|
+
session_handle,
|
|
116
|
+
ctypes.byref(provider_guid_struct),
|
|
117
|
+
const.EVENT_CONTROL_CODE_ENABLE_PROVIDER,
|
|
118
|
+
verbosity_mode,
|
|
119
|
+
0,
|
|
120
|
+
0,
|
|
121
|
+
0,
|
|
122
|
+
None
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
if status != 0:
|
|
126
|
+
raise Exception(f"EnableTraceEx2 failed for provider {provider_guid} with error {status}")
|
|
127
|
+
|
|
128
|
+
print("ETW session started successfully")
|
|
129
|
+
|
|
130
|
+
return session_handle
|
|
6
131
|
|
|
7
132
|
|
|
8
133
|
# Function to stop and delete ETW session
|
|
9
|
-
def stop_and_delete_etw_session(session_name) -> tuple[bool, int]:
|
|
134
|
+
def stop_and_delete_etw_session(session_name: str) -> tuple[bool, int]:
|
|
10
135
|
"""
|
|
11
136
|
Stop and delete ETW session.
|
|
12
137
|
|
|
@@ -40,13 +165,19 @@ def stop_and_delete_etw_session(session_name) -> tuple[bool, int]:
|
|
|
40
165
|
return True, status
|
|
41
166
|
|
|
42
167
|
|
|
43
|
-
def get_all_providers(
|
|
168
|
+
def get_all_providers(key_as: Literal['name', 'guid'] = 'name') -> dict:
|
|
44
169
|
"""
|
|
45
170
|
Get all ETW providers available on the system.
|
|
46
171
|
|
|
47
|
-
:
|
|
172
|
+
:param key_as: The key to use in the dictionary, either 'name' or 'guid'.
|
|
173
|
+
'name': The provider name is used as the key, the guid as the value.
|
|
174
|
+
'guid': The provider guid is used as the key, the name as the value.
|
|
175
|
+
:return: dict containing the provider name and GUID.
|
|
48
176
|
"""
|
|
49
177
|
|
|
178
|
+
if key_as not in ['name', 'guid']:
|
|
179
|
+
raise ValueError("key_as must be either 'name' or 'guid'")
|
|
180
|
+
|
|
50
181
|
providers_info_size = ULONG(0)
|
|
51
182
|
status = const.tdh.TdhEnumerateProviders(None, ctypes.byref(providers_info_size))
|
|
52
183
|
|
|
@@ -71,7 +202,7 @@ def get_all_providers() -> list[tuple[any, uuid.UUID]]:
|
|
|
71
202
|
ctypes.addressof(providers_info.contents) + ctypes.sizeof(const.PROVIDER_ENUMERATION_INFO),
|
|
72
203
|
ctypes.POINTER(const.PROVIDER_INFORMATION * provider_count))
|
|
73
204
|
|
|
74
|
-
providers =
|
|
205
|
+
providers: dict = {}
|
|
75
206
|
for i in range(provider_count):
|
|
76
207
|
provider = provider_array.contents[i]
|
|
77
208
|
provider_name_offset = provider.ProviderNameOffset
|
|
@@ -79,7 +210,12 @@ def get_all_providers() -> list[tuple[any, uuid.UUID]]:
|
|
|
79
210
|
ctypes.addressof(providers_info.contents) + provider_name_offset, ctypes.c_wchar_p)
|
|
80
211
|
provider_name = provider_name_ptr.value
|
|
81
212
|
provider_guid = uuid.UUID(bytes_le=bytes(provider.ProviderId))
|
|
82
|
-
|
|
213
|
+
provider_guid_string = str(provider_guid)
|
|
214
|
+
|
|
215
|
+
if key_as == 'name':
|
|
216
|
+
providers[provider_name] = provider_guid_string
|
|
217
|
+
elif key_as == 'guid':
|
|
218
|
+
providers[provider_guid_string] = provider_name
|
|
83
219
|
|
|
84
220
|
return providers
|
|
85
221
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import os
|
|
2
2
|
from typing import Literal, Union
|
|
3
3
|
from pathlib import Path
|
|
4
|
+
import datetime
|
|
4
5
|
|
|
5
6
|
from ... import filesystem, datetimes
|
|
6
7
|
from ...file_io import csvs
|
|
@@ -11,7 +12,6 @@ def get_logs_paths(
|
|
|
11
12
|
log_file_path: str = None,
|
|
12
13
|
file_name_pattern: str = '*.*',
|
|
13
14
|
date_pattern: str = None,
|
|
14
|
-
log_type: Literal['csv'] = 'csv',
|
|
15
15
|
latest_only: bool = False,
|
|
16
16
|
previous_day_only: bool = False
|
|
17
17
|
):
|
|
@@ -36,7 +36,6 @@ def get_logs_paths(
|
|
|
36
36
|
:param date_pattern: Pattern to match the date in the log file name.
|
|
37
37
|
If specified, the function will get the log file by the date pattern.
|
|
38
38
|
If not specified, the function will get the file date by file last modified time.
|
|
39
|
-
:param log_type: Type of log to get.
|
|
40
39
|
:param latest_only: Boolean, if True, only the latest log file path will be returned.
|
|
41
40
|
:param previous_day_only: Boolean, if True, only the log file path from the previous day will be returned.
|
|
42
41
|
"""
|
|
@@ -46,9 +45,6 @@ def get_logs_paths(
|
|
|
46
45
|
elif log_files_directory_path and log_file_path:
|
|
47
46
|
raise ValueError('Both "log_files_directory_path" and "log_file_path" cannot be specified at the same time.')
|
|
48
47
|
|
|
49
|
-
if log_type != 'csv':
|
|
50
|
-
raise ValueError('Only "csv" log type is supported.')
|
|
51
|
-
|
|
52
48
|
if latest_only and previous_day_only:
|
|
53
49
|
raise ValueError('Both "latest_only" and "previous_day_only" cannot be True at the same time.')
|
|
54
50
|
|
|
@@ -75,16 +71,22 @@ def get_logs_paths(
|
|
|
75
71
|
for file_index, single_file in enumerate(logs_files):
|
|
76
72
|
# Get file name from current loop file path.
|
|
77
73
|
current_file_name: str = Path(single_file['file_path']).name
|
|
74
|
+
logs_files[file_index]['file_name'] = current_file_name
|
|
75
|
+
|
|
78
76
|
# Get the datetime object from the file name by the date pattern.
|
|
79
77
|
try:
|
|
80
|
-
datetime_object = datetimes.get_datetime_from_complex_string_by_pattern(
|
|
78
|
+
datetime_object, date_string, timestamp_float = datetimes.get_datetime_from_complex_string_by_pattern(
|
|
81
79
|
current_file_name, date_pattern)
|
|
82
|
-
timestamp_float = datetime_object.timestamp()
|
|
83
80
|
# ValueError will be raised if the date pattern does not match the file name.
|
|
84
81
|
except ValueError:
|
|
85
82
|
timestamp_float = 0
|
|
83
|
+
datetime_object = None
|
|
84
|
+
date_string = None
|
|
85
|
+
|
|
86
86
|
# Update the last modified time to the dictionary.
|
|
87
87
|
logs_files[file_index]['last_modified'] = timestamp_float
|
|
88
|
+
logs_files[file_index]['datetime'] = datetime_object
|
|
89
|
+
logs_files[file_index]['date_string'] = date_string
|
|
88
90
|
|
|
89
91
|
if timestamp_float > latest_timestamp:
|
|
90
92
|
latest_timestamp = timestamp_float
|
|
@@ -95,6 +97,8 @@ def get_logs_paths(
|
|
|
95
97
|
if single_file['last_modified'] == 0:
|
|
96
98
|
latest_timestamp += 86400
|
|
97
99
|
logs_files[file_index]['last_modified'] = latest_timestamp
|
|
100
|
+
logs_files[file_index]['datetime'] = datetime.datetime.fromtimestamp(latest_timestamp)
|
|
101
|
+
logs_files[file_index]['date_string'] = logs_files[file_index]['datetime'].strftime(date_pattern)
|
|
98
102
|
break
|
|
99
103
|
|
|
100
104
|
# Sort the files by the last modified time.
|
|
@@ -117,7 +121,7 @@ def get_logs_paths(
|
|
|
117
121
|
return logs_files
|
|
118
122
|
|
|
119
123
|
|
|
120
|
-
def
|
|
124
|
+
def get_all_log_files_into_list(
|
|
121
125
|
log_files_directory_path: str = None,
|
|
122
126
|
log_file_path: str = None,
|
|
123
127
|
file_name_pattern: str = '*.*',
|
|
@@ -127,9 +131,10 @@ def get_logs(
|
|
|
127
131
|
remove_logs: bool = False,
|
|
128
132
|
move_to_path: str = None,
|
|
129
133
|
print_kwargs: dict = None
|
|
130
|
-
):
|
|
134
|
+
) -> list:
|
|
131
135
|
"""
|
|
132
|
-
This function gets the logs from the log files. Supports rotating files to get the logs by time.
|
|
136
|
+
This function gets the logs contents from the log files. Supports rotating files to get the logs by time.
|
|
137
|
+
All the contents will be merged into one list.
|
|
133
138
|
|
|
134
139
|
:param log_files_directory_path: Path to the log files. Check the 'get_logs_paths' function for more details.
|
|
135
140
|
:param log_file_path: Path to the log file. Check the 'get_logs_paths' function for more details.
|
|
@@ -144,8 +149,9 @@ def get_logs(
|
|
|
144
149
|
'all' - Each CSV file has a header. Get the header from each file.
|
|
145
150
|
:param remove_logs: Boolean, if True, the logs will be removed after getting them.
|
|
146
151
|
:param move_to_path: Path to move the logs to.
|
|
147
|
-
|
|
148
152
|
:param print_kwargs: Keyword arguments dict for 'print_api' function.
|
|
153
|
+
|
|
154
|
+
:return: List of dictionaries with the logs content.
|
|
149
155
|
"""
|
|
150
156
|
|
|
151
157
|
if not print_kwargs:
|
|
@@ -162,8 +168,7 @@ def get_logs(
|
|
|
162
168
|
log_files_directory_path=log_files_directory_path,
|
|
163
169
|
log_file_path=log_file_path,
|
|
164
170
|
file_name_pattern=file_name_pattern,
|
|
165
|
-
date_pattern=date_pattern
|
|
166
|
-
log_type=log_type)
|
|
171
|
+
date_pattern=date_pattern)
|
|
167
172
|
|
|
168
173
|
# Read all the logs.
|
|
169
174
|
logs_content: list = list()
|
|
@@ -294,8 +299,7 @@ class LogReader:
|
|
|
294
299
|
# If the existing logs file count is 0, it means that this is the first check. We need to get the current count.
|
|
295
300
|
if self._existing_logs_file_count == 0:
|
|
296
301
|
self._existing_logs_file_count = len(get_logs_paths(
|
|
297
|
-
log_file_path=self.log_file_path
|
|
298
|
-
log_type='csv'
|
|
302
|
+
log_file_path=self.log_file_path
|
|
299
303
|
))
|
|
300
304
|
|
|
301
305
|
# If the count is still 0, then there are no logs to read.
|
|
@@ -311,7 +315,6 @@ class LogReader:
|
|
|
311
315
|
latest_statistics_file_path_object = get_logs_paths(
|
|
312
316
|
log_file_path=self.log_file_path,
|
|
313
317
|
date_pattern=self.date_pattern,
|
|
314
|
-
log_type='csv',
|
|
315
318
|
latest_only=True
|
|
316
319
|
)
|
|
317
320
|
|
|
@@ -327,7 +330,6 @@ class LogReader:
|
|
|
327
330
|
previous_day_statistics_file_path = get_logs_paths(
|
|
328
331
|
log_file_path=self.log_file_path,
|
|
329
332
|
date_pattern=self.date_pattern,
|
|
330
|
-
log_type='csv',
|
|
331
333
|
previous_day_only=True
|
|
332
334
|
)[0]['file_path']
|
|
333
335
|
# If you get IndexError, it means that there are no previous day logs to read.
|
|
@@ -336,8 +338,7 @@ class LogReader:
|
|
|
336
338
|
|
|
337
339
|
# Count all the rotated files.
|
|
338
340
|
current_log_files_count: int = len(get_logs_paths(
|
|
339
|
-
log_file_path=self.log_file_path
|
|
340
|
-
log_type='csv'
|
|
341
|
+
log_file_path=self.log_file_path
|
|
341
342
|
))
|
|
342
343
|
|
|
343
344
|
# If the count of the log files is greater than the existing logs file count, it means that the rotation
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
atomicshop/__init__.py,sha256=
|
|
1
|
+
atomicshop/__init__.py,sha256=tunTjRW84u7Nao5t0Z6ZWFYrQDUW_zun_a86yEIubts,123
|
|
2
2
|
atomicshop/_basics_temp.py,sha256=6cu2dd6r2dLrd1BRNcVDKTHlsHs_26Gpw8QS6v32lQ0,3699
|
|
3
3
|
atomicshop/_create_pdf_demo.py,sha256=Yi-PGZuMg0RKvQmLqVeLIZYadqEZwUm-4A9JxBl_vYA,3713
|
|
4
4
|
atomicshop/_patch_import.py,sha256=ENp55sKVJ0e6-4lBvZnpz9PQCt3Otbur7F6aXDlyje4,6334
|
|
@@ -8,7 +8,7 @@ atomicshop/command_line_processing.py,sha256=u5yT9Ger_cu7ni5ID0VFlRbVD46ARHeNC9t
|
|
|
8
8
|
atomicshop/config_init.py,sha256=z2RXD_mw9nQlAOpuGry1h9QT-2LhNscXgGAktN3dCVQ,2497
|
|
9
9
|
atomicshop/console_output.py,sha256=AOSJjrRryE97PAGtgDL03IBtWSi02aNol8noDnW3k6M,4667
|
|
10
10
|
atomicshop/console_user_response.py,sha256=31HIy9QGXa7f-GVR8MzJauQ79E_ZqAeagF3Ks4GGdDU,3234
|
|
11
|
-
atomicshop/datetimes.py,sha256=
|
|
11
|
+
atomicshop/datetimes.py,sha256=wZ75JS6Aq5kTQrCSrjCwBrT-KPO7Xu9_BRWKrY8uU3c,15645
|
|
12
12
|
atomicshop/diff_check.py,sha256=RJvzJhyYAZyRPKVDk1dS7UwZCx0kq__WDZ6N0rNfZUY,27110
|
|
13
13
|
atomicshop/dns.py,sha256=h4uZKoz4wbBlLOOduL1GtRcTm-YpiPnGOEGxUm7hhOI,2140
|
|
14
14
|
atomicshop/domains.py,sha256=Rxu6JhhMqFZRcoFs69IoEd1PtYca0lMCG6F1AomP7z4,3197
|
|
@@ -104,14 +104,14 @@ atomicshop/basics/timeit_template.py,sha256=fYLrk-X_dhdVtnPU22tarrhhvlggeW6FdKCX
|
|
|
104
104
|
atomicshop/basics/tracebacks.py,sha256=cNfh_oAwF55kSIdqtv3boHZQIoQI8TajxkTnwJwpweI,535
|
|
105
105
|
atomicshop/etws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
106
106
|
atomicshop/etws/const.py,sha256=v3x_IdCYeSKbCGywiZFOZln80ldpwKW5nuMDuUe51Jg,1257
|
|
107
|
-
atomicshop/etws/providers.py,sha256=
|
|
108
|
-
atomicshop/etws/sessions.py,sha256=
|
|
109
|
-
atomicshop/etws/trace.py,sha256=
|
|
107
|
+
atomicshop/etws/providers.py,sha256=CXNx8pYdjtpLIpA66IwrnE64XhY4U5ExnFBMLEb8Uzk,547
|
|
108
|
+
atomicshop/etws/sessions.py,sha256=b_KeiOvgOBJezJokN81TRlrvJiQNJlIWN4Z6UVjuxP0,1335
|
|
109
|
+
atomicshop/etws/trace.py,sha256=WMOjdazK97UcIdhVgcnjh98OCbuEJcnm1Z_yPp_nE2c,7258
|
|
110
110
|
atomicshop/etws/traces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
111
111
|
atomicshop/etws/traces/trace_dns.py,sha256=rWQ8bv8eMHBRRkA8oxO9caYqj0h4Emw4aZXmoI3Q6fg,6292
|
|
112
112
|
atomicshop/etws/traces/trace_sysmon_process_creation.py,sha256=OM-bkK38uYMwWLZKNOTDa0Xdk3sO6sqsxoMUIiPvm5g,4656
|
|
113
113
|
atomicshop/file_io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
114
|
-
atomicshop/file_io/csvs.py,sha256=
|
|
114
|
+
atomicshop/file_io/csvs.py,sha256=eS2SSGwcC1MlRPPgoqyFyE-qqH2esUWQvv3wWLMiOuA,5876
|
|
115
115
|
atomicshop/file_io/docxs.py,sha256=6tcYFGp0vRsHR47VwcRqwhdt2DQOwrAUYhrwN996n9U,5117
|
|
116
116
|
atomicshop/file_io/file_io.py,sha256=FOZ6_PjOASxSDHESe4fwDv5miXYR10OHTxkxtEHoZYQ,6555
|
|
117
117
|
atomicshop/file_io/jsons.py,sha256=q9ZU8slBKnHLrtn3TnbK1qxrRpj5ZvCm6AlsFzoANjo,5303
|
|
@@ -126,7 +126,7 @@ atomicshop/mitm/initialize_engines.py,sha256=UGdT5DKYNri3MNOxESP7oeSxYiUDrVilJ4j
|
|
|
126
126
|
atomicshop/mitm/initialize_mitm_server.py,sha256=5JGkyvAvz1sJVeRGMJWSQiQ-VOdrU-NJn633oxQe0cw,13143
|
|
127
127
|
atomicshop/mitm/message.py,sha256=u2U2f2SOHdBNU-6r1Ik2W14ai2EOwxUV4wVfGZA098k,1732
|
|
128
128
|
atomicshop/mitm/shared_functions.py,sha256=PaK_sbnEA5zo9k2ktEOKLmvo-6wRUunxzSNRr41uXIQ,1924
|
|
129
|
-
atomicshop/mitm/statistic_analyzer.py,sha256=
|
|
129
|
+
atomicshop/mitm/statistic_analyzer.py,sha256=ctsf-MBIUvG4-R0K4gFQyi_b42-VCq-5s7hgO9jMOes,38415
|
|
130
130
|
atomicshop/mitm/engines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
131
131
|
atomicshop/mitm/engines/create_module_template.py,sha256=tRjVSm1sD6FzML71Qbuwvita0qsusdFGm8NZLsZ-XMs,4853
|
|
132
132
|
atomicshop/mitm/engines/create_module_template_example.py,sha256=X5xhvbV6-g9jU_bQVhf_crZmaH50LRWz3bS-faQ18ds,489
|
|
@@ -184,8 +184,8 @@ atomicshop/wrappers/certauthw/certauthw.py,sha256=4WvhjANI7Kzqrr_nKmtA8Kf7B6rute
|
|
|
184
184
|
atomicshop/wrappers/ctyping/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
185
185
|
atomicshop/wrappers/ctyping/process_winapi.py,sha256=QcXL-ETtlSSkoT8F7pYle97ubGWsjYp8cx8HxkVMgAc,2762
|
|
186
186
|
atomicshop/wrappers/ctyping/etw_winapi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
187
|
-
atomicshop/wrappers/ctyping/etw_winapi/const.py,sha256=
|
|
188
|
-
atomicshop/wrappers/ctyping/etw_winapi/etw_functions.py,sha256=
|
|
187
|
+
atomicshop/wrappers/ctyping/etw_winapi/const.py,sha256=stZHZ7tSiSAs04ikr7uH-Td_yBXxsF-bp2Q0F3K2fsM,9543
|
|
188
|
+
atomicshop/wrappers/ctyping/etw_winapi/etw_functions.py,sha256=Iwd0wIuoxpjMaaOfZZtT1bPtDTsMO8jjItBE5bvkocM,11546
|
|
189
189
|
atomicshop/wrappers/ctyping/msi_windows_installer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
190
190
|
atomicshop/wrappers/ctyping/msi_windows_installer/base.py,sha256=Uu9SlWLsQQ6mjE-ek-ptHcmgiI3Ruah9bdZus70EaVY,4884
|
|
191
191
|
atomicshop/wrappers/ctyping/msi_windows_installer/cabs.py,sha256=htzwb2ROYI8yyc82xApStckPS2yCcoyaw32yC15KROs,3285
|
|
@@ -237,7 +237,7 @@ atomicshop/wrappers/loggingw/formatters.py,sha256=mUtcJJfmhLNrwUVYShXTmdu40dBaJu
|
|
|
237
237
|
atomicshop/wrappers/loggingw/handlers.py,sha256=2A_3Qy1B0RvVWZmQocAB6CmpqlXoKJ-yi6iBWG2jNLo,8274
|
|
238
238
|
atomicshop/wrappers/loggingw/loggers.py,sha256=DHOOTAtqkwn1xgvLHSkOiBm6yFGNuQy1kvbhG-TDog8,2374
|
|
239
239
|
atomicshop/wrappers/loggingw/loggingw.py,sha256=m6YySEedP3_4Ik1S_uGMxETSbmRkmMYmAZxhHBlXSlo,16616
|
|
240
|
-
atomicshop/wrappers/loggingw/reading.py,sha256=
|
|
240
|
+
atomicshop/wrappers/loggingw/reading.py,sha256=wse-38zUDHB3HUB28R8Ah_Ig3Wxt2tChapKtu-yyy2E,17036
|
|
241
241
|
atomicshop/wrappers/nodejsw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
242
242
|
atomicshop/wrappers/nodejsw/install_nodejs.py,sha256=QZg-R2iTQt7kFb8wNtnTmwraSGwvUs34JIasdbNa7ZU,5154
|
|
243
243
|
atomicshop/wrappers/playwrightw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -281,8 +281,8 @@ atomicshop/wrappers/socketw/socket_server_tester.py,sha256=AhpurHJmP2kgzHaUbq5ey
|
|
|
281
281
|
atomicshop/wrappers/socketw/socket_wrapper.py,sha256=aXBwlEIJhFT0-c4i8iNlFx2It9VpCEpsv--5Oqcpxao,11624
|
|
282
282
|
atomicshop/wrappers/socketw/ssl_base.py,sha256=k4V3gwkbq10MvOH4btU4onLX2GNOsSfUAdcHmL1rpVE,2274
|
|
283
283
|
atomicshop/wrappers/socketw/statistics_csv.py,sha256=t3dtDEfN47CfYVi0CW6Kc2QHTEeZVyYhc57IYYh5nmA,826
|
|
284
|
-
atomicshop-2.14.
|
|
285
|
-
atomicshop-2.14.
|
|
286
|
-
atomicshop-2.14.
|
|
287
|
-
atomicshop-2.14.
|
|
288
|
-
atomicshop-2.14.
|
|
284
|
+
atomicshop-2.14.4.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
|
|
285
|
+
atomicshop-2.14.4.dist-info/METADATA,sha256=fjxtj_SzmyjVK6o_Z08jX-lLJoz7m_vYNPvn0fCoFYU,10478
|
|
286
|
+
atomicshop-2.14.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
|
287
|
+
atomicshop-2.14.4.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
|
|
288
|
+
atomicshop-2.14.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|