atomicshop 2.16.11__py3-none-any.whl → 2.16.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of atomicshop might be problematic. Click here for more details.
- atomicshop/__init__.py +1 -1
- atomicshop/basics/enums.py +2 -2
- atomicshop/basics/list_of_classes.py +29 -0
- atomicshop/file_io/docxs.py +4 -4
- atomicshop/filesystem.py +265 -198
- atomicshop/mitm/config_static.py +1 -1
- atomicshop/mitm/initialize_engines.py +6 -6
- atomicshop/mitm/{initialize_mitm_server.py → mitm_main.py} +4 -3
- atomicshop/mitm/recs_files.py +17 -16
- atomicshop/mitm/statistic_analyzer.py +2 -2
- atomicshop/wrappers/factw/install/pre_install_and_install_before_restart.py +1 -1
- atomicshop/wrappers/loggingw/reading.py +63 -100
- atomicshop/wrappers/socketw/socket_server_tester.py +5 -5
- {atomicshop-2.16.11.dist-info → atomicshop-2.16.13.dist-info}/METADATA +1 -1
- {atomicshop-2.16.11.dist-info → atomicshop-2.16.13.dist-info}/RECORD +18 -17
- {atomicshop-2.16.11.dist-info → atomicshop-2.16.13.dist-info}/LICENSE.txt +0 -0
- {atomicshop-2.16.11.dist-info → atomicshop-2.16.13.dist-info}/WHEEL +0 -0
- {atomicshop-2.16.11.dist-info → atomicshop-2.16.13.dist-info}/top_level.txt +0 -0
atomicshop/filesystem.py
CHANGED
|
@@ -9,10 +9,11 @@ from contextlib import contextmanager
|
|
|
9
9
|
from typing import Literal, Union
|
|
10
10
|
import tempfile
|
|
11
11
|
|
|
12
|
+
# noinspection PyPackageRequirements
|
|
12
13
|
import psutil
|
|
13
14
|
|
|
14
15
|
from .print_api import print_api, print_status_of_list
|
|
15
|
-
from .basics import strings, list_of_dicts
|
|
16
|
+
from .basics import strings, list_of_dicts, list_of_classes
|
|
16
17
|
from .file_io import file_io
|
|
17
18
|
from . import hashing, datetimes
|
|
18
19
|
|
|
@@ -65,14 +66,18 @@ FILE_NAME_REPLACEMENT_DICT: dict = {
|
|
|
65
66
|
}
|
|
66
67
|
|
|
67
68
|
|
|
69
|
+
# class TimeCouldNotBeFoundInFileNameError(Exception):
|
|
70
|
+
# pass
|
|
71
|
+
|
|
72
|
+
|
|
68
73
|
def get_home_directory(return_sudo_user: bool = False) -> str:
|
|
69
74
|
"""
|
|
70
75
|
Returns the home directory of the current user or the user who invoked sudo.
|
|
71
76
|
|
|
72
77
|
:param return_sudo_user: bool, if 'False', then the function will return the home directory of the user who invoked
|
|
73
78
|
sudo (if the script was invoked with sudo).
|
|
74
|
-
If 'True', then the function will return the home directory of the current user, doesn't matter if the script
|
|
75
|
-
invoked with sudo or not, if so home directory of the sudo user will be returned.
|
|
79
|
+
If 'True', then the function will return the home directory of the current user, doesn't matter if the script
|
|
80
|
+
was invoked with sudo or not, if so home directory of the sudo user will be returned.
|
|
76
81
|
"""
|
|
77
82
|
|
|
78
83
|
def return_home_directory_of_current_user():
|
|
@@ -209,16 +214,6 @@ def get_list_of_directories_in_file_path(
|
|
|
209
214
|
return directory_list
|
|
210
215
|
|
|
211
216
|
|
|
212
|
-
def get_file_name(file_path: str) -> str:
|
|
213
|
-
"""
|
|
214
|
-
The function will return file name of the file.
|
|
215
|
-
|
|
216
|
-
:param file_path: string, full file path.
|
|
217
|
-
:return: string.
|
|
218
|
-
"""
|
|
219
|
-
return str(Path(file_path).name)
|
|
220
|
-
|
|
221
|
-
|
|
222
217
|
def check_absolute_path(filesystem_path) -> bool:
|
|
223
218
|
"""
|
|
224
219
|
The function checks if the path provided is a full path (absolute) or relative.
|
|
@@ -245,16 +240,6 @@ def check_absolute_path___add_full(filesystem_path: str, full_path_to_add: str)
|
|
|
245
240
|
return filesystem_path
|
|
246
241
|
|
|
247
242
|
|
|
248
|
-
def check_file_existence(file_path: str) -> bool:
|
|
249
|
-
"""This will be removed in future versions. Use 'is_file_exists' instead."""
|
|
250
|
-
return is_file_exists(file_path)
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
def check_directory_existence(directory_path: str) -> bool:
|
|
254
|
-
"""This will be removed in future versions. Use 'is_directory_exists' instead."""
|
|
255
|
-
return is_directory_exists(directory_path)
|
|
256
|
-
|
|
257
|
-
|
|
258
243
|
def is_file_exists(file_path: str) -> bool:
|
|
259
244
|
"""
|
|
260
245
|
Function to check if the path is a file.
|
|
@@ -369,25 +354,26 @@ def rename_file(source_file_path: str, target_file_path: str) -> None:
|
|
|
369
354
|
|
|
370
355
|
@contextmanager
|
|
371
356
|
def temporary_rename(file_path: str, temp_file_path) -> None:
|
|
357
|
+
# noinspection GrazieInspection
|
|
372
358
|
"""
|
|
373
|
-
|
|
359
|
+
The function will rename the file to temporary name and then rename it back to original name.
|
|
374
360
|
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
361
|
+
:param file_path: string, full path to file.
|
|
362
|
+
:param temp_file_path: string, temporary name to rename the file to.
|
|
363
|
+
:return: None.
|
|
378
364
|
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
365
|
+
Usage:
|
|
366
|
+
original_file = 'example.txt'
|
|
367
|
+
temporary_file = 'temp_example.txt'
|
|
382
368
|
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
369
|
+
with temporary_rename(original_file, temporary_file):
|
|
370
|
+
# Inside this block, the file exists as 'temp_example.txt'
|
|
371
|
+
print(f"File is temporarily renamed to {temporary_file}")
|
|
372
|
+
# Perform operations with the temporarily named file here
|
|
387
373
|
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
374
|
+
# Outside the block, it's back to 'example.txt'
|
|
375
|
+
print(f"File is renamed back to {original_file}")
|
|
376
|
+
"""
|
|
391
377
|
|
|
392
378
|
original_name = file_path
|
|
393
379
|
try:
|
|
@@ -511,17 +497,17 @@ def move_files_from_folder_to_folder(
|
|
|
511
497
|
move_file(source_file_path=source_item, target_file_path=destination_item, overwrite=overwrite)
|
|
512
498
|
|
|
513
499
|
# # Get all file names without full paths in source folder.
|
|
514
|
-
# file_list_in_source: list =
|
|
500
|
+
# file_list_in_source: list = get_paths_from_directory(source_directory, get_file=True)
|
|
515
501
|
#
|
|
516
502
|
# # Iterate through all the files.
|
|
517
503
|
# for file_path in file_list_in_source:
|
|
518
504
|
# # Move the file from source to target.
|
|
519
|
-
# if file_path
|
|
520
|
-
# create_directory(target_directory + os.sep + file_path
|
|
521
|
-
# relative_file_path: str = file_path
|
|
505
|
+
# if file_path.relative_dir:
|
|
506
|
+
# create_directory(target_directory + os.sep + file_path.relative_dir)
|
|
507
|
+
# relative_file_path: str = file_path.relative_dir + os.sep + Path(file_path.path).name
|
|
522
508
|
# else:
|
|
523
|
-
# relative_file_path: str = Path(file_path
|
|
524
|
-
# move_file(file_path
|
|
509
|
+
# relative_file_path: str = Path(file_path.path).name
|
|
510
|
+
# move_file(file_path.path, target_directory + os.sep + relative_file_path)
|
|
525
511
|
|
|
526
512
|
|
|
527
513
|
def copy_file(
|
|
@@ -601,49 +587,101 @@ def copy_files_from_folder_to_folder(source_directory: str, target_directory: st
|
|
|
601
587
|
shutil.copy2(s, d)
|
|
602
588
|
|
|
603
589
|
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
590
|
+
class AtomicPath:
|
|
591
|
+
def __init__(self, path: str):
|
|
592
|
+
self.path: str = path
|
|
593
|
+
|
|
594
|
+
self.is_file: bool = os.path.isfile(path)
|
|
595
|
+
self.is_directory: bool = os.path.isdir(path)
|
|
596
|
+
self.name: str = Path(path).name
|
|
597
|
+
|
|
598
|
+
self.queried_directory: str = ''
|
|
599
|
+
# noinspection PyTypeChecker
|
|
600
|
+
self.last_modified: float = None
|
|
601
|
+
self.relative_dir: str = ''
|
|
602
|
+
self.binary: bytes = b''
|
|
603
|
+
self.hash: str = ''
|
|
604
|
+
self.datetime_datetime = None
|
|
605
|
+
self.datetime_string: str = ''
|
|
606
|
+
# noinspection PyTypeChecker
|
|
607
|
+
self.datetime_float: float = None
|
|
608
|
+
self.datetime_format: str = ''
|
|
609
|
+
|
|
610
|
+
def __str__(self):
|
|
611
|
+
return self.path
|
|
612
|
+
|
|
613
|
+
def update(
|
|
614
|
+
self,
|
|
615
|
+
path: str = None,
|
|
616
|
+
datetime_format: str = None,
|
|
617
|
+
update_datetime: bool = False,
|
|
618
|
+
update_last_modified: bool = False,
|
|
619
|
+
update_binary: bool = False,
|
|
620
|
+
update_hash: bool = False
|
|
621
|
+
):
|
|
622
|
+
if path:
|
|
623
|
+
if path != self.path:
|
|
624
|
+
self.queried_directory = ''
|
|
625
|
+
self.last_modified = None
|
|
626
|
+
self.relative_dir = ''
|
|
627
|
+
self.binary = b''
|
|
628
|
+
self.hash = ''
|
|
629
|
+
self.datetime_datetime = None
|
|
630
|
+
self.datetime_string = ''
|
|
631
|
+
self.datetime_float = None
|
|
632
|
+
self.datetime_format = ''
|
|
633
|
+
|
|
634
|
+
self.path = path
|
|
635
|
+
self.is_file = os.path.isfile(path)
|
|
636
|
+
self.is_directory = os.path.isdir(path)
|
|
637
|
+
self.name = Path(path).name
|
|
638
|
+
|
|
639
|
+
# Update the datetime format only if it is provided without the update_datetime boolean.
|
|
640
|
+
# Since, we don't want this variable if there is no relation between the datetime format and the filename.
|
|
641
|
+
# If the user want to put it manually, then we will not stop him, but this case is useless if filename
|
|
642
|
+
# doesn't contain the datetime.
|
|
643
|
+
if datetime_format and not update_datetime:
|
|
644
|
+
self.datetime_format = datetime_format
|
|
645
|
+
|
|
646
|
+
if update_datetime and not datetime_format and not self.datetime_format:
|
|
647
|
+
raise ValueError('If "update_datetime" is True, then "datetime_format" must be provided.')
|
|
648
|
+
|
|
649
|
+
if update_datetime:
|
|
650
|
+
self.datetime_datetime, self.datetime_string, self.datetime_float = (
|
|
651
|
+
datetimes.get_datetime_from_complex_string_by_pattern(self.name, datetime_format))
|
|
652
|
+
# If the provided datetime format is correct, then we will update the datetime format.
|
|
653
|
+
if self.datetime_string:
|
|
654
|
+
self.datetime_format = datetime_format
|
|
655
|
+
|
|
656
|
+
if update_last_modified:
|
|
657
|
+
self.last_modified = get_file_modified_time(self.path)
|
|
658
|
+
|
|
659
|
+
if update_binary:
|
|
660
|
+
self.binary = file_io.read_file(self.path, file_mode='rb', stdout=False)
|
|
661
|
+
|
|
662
|
+
if update_hash:
|
|
663
|
+
if self.binary:
|
|
664
|
+
self.hash = hashing.hash_bytes(self.binary)
|
|
665
|
+
else:
|
|
666
|
+
self.hash = hashing.hash_file(self.path)
|
|
636
667
|
|
|
637
668
|
|
|
638
|
-
def
|
|
669
|
+
def get_paths_from_directory(
|
|
639
670
|
directory_path: str,
|
|
671
|
+
simple_list: bool = False,
|
|
672
|
+
get_file: bool = False,
|
|
673
|
+
get_directory: bool = False,
|
|
640
674
|
recursive: bool = True,
|
|
641
675
|
file_name_check_pattern: str = '*',
|
|
676
|
+
datetime_format: str = None,
|
|
677
|
+
specific_date: str = None,
|
|
642
678
|
add_relative_directory: bool = False,
|
|
643
679
|
relative_file_name_as_directory: bool = False,
|
|
644
680
|
add_last_modified_time: bool = False,
|
|
645
|
-
sort_by_last_modified_time: bool = False
|
|
646
|
-
|
|
681
|
+
sort_by_last_modified_time: bool = False,
|
|
682
|
+
add_file_binary: bool = False,
|
|
683
|
+
add_file_hash: bool = False,
|
|
684
|
+
) -> list[AtomicPath]:
|
|
647
685
|
"""
|
|
648
686
|
Recursive, by option.
|
|
649
687
|
The function receives a filesystem directory as string, scans it recursively for files and returns list of
|
|
@@ -652,15 +690,23 @@ def get_file_paths_from_directory(
|
|
|
652
690
|
of that tuple.
|
|
653
691
|
|
|
654
692
|
:param directory_path: string to full path to directory on the filesystem to scan.
|
|
693
|
+
:param simple_list: boolean, if 'True', then the function will return only full file paths.
|
|
694
|
+
:param get_file: boolean, if 'True', then the function will return files.
|
|
695
|
+
:param get_directory: boolean, if 'True', then the function will return directories.
|
|
655
696
|
:param recursive: boolean.
|
|
656
697
|
'True', then the function will scan recursively in subdirectories.
|
|
657
698
|
'False', then the function will scan only in the directory that was passed.
|
|
658
699
|
:param file_name_check_pattern: string, if specified, the function will return only files that match the pattern.
|
|
659
700
|
The string can contain part of file name to check or full file name with extension.
|
|
660
701
|
Can contain wildcards.
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
702
|
+
:param datetime_format: datetime format string pattern to match the date in the file name.
|
|
703
|
+
If specified, the function will get the files by the date pattern.
|
|
704
|
+
|
|
705
|
+
Example:
|
|
706
|
+
datetime_format = '%Y-%m-%d'
|
|
707
|
+
:param specific_date: Specific date to get the file path.
|
|
708
|
+
If specified, the function will get the file by the specific date.
|
|
709
|
+
Meaning that 'datetime_format' must be specified.
|
|
664
710
|
:param add_relative_directory: boolean, if
|
|
665
711
|
'True', then the function will add relative directory to the output list.
|
|
666
712
|
In this case the output list will contain dictionaries with keys 'path' and 'relative_dir'.
|
|
@@ -672,47 +718,72 @@ def get_file_paths_from_directory(
|
|
|
672
718
|
to the output list.
|
|
673
719
|
:param sort_by_last_modified_time: boolean, if 'True', then the function will sort the output list by last
|
|
674
720
|
modified time of the file.
|
|
721
|
+
:param add_file_binary: boolean, if 'True', then the function will add binary content of the file to each file
|
|
722
|
+
object of the output list.
|
|
723
|
+
:param add_file_hash: boolean, if 'True', then the function will add hash of the file to each file object of the
|
|
724
|
+
output list.
|
|
675
725
|
|
|
676
726
|
:return: list of all found filenames with full file paths, list with relative folders to file excluding the
|
|
677
727
|
main folder.
|
|
678
728
|
"""
|
|
679
729
|
|
|
680
|
-
def
|
|
730
|
+
def get_path(file_or_directory: str):
|
|
681
731
|
"""
|
|
682
732
|
Function gets the full file path, adds it to the found 'object_list' and gets the relative path to that
|
|
683
733
|
file, against the main path to directory that was passed to the parent function.
|
|
684
734
|
"""
|
|
685
735
|
|
|
686
|
-
|
|
736
|
+
if strings.match_pattern_against_string(file_name_check_pattern, file_or_directory):
|
|
737
|
+
file_or_dir_path: str = os.path.join(dir_path, file_or_directory)
|
|
687
738
|
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
file_result: dict = dict()
|
|
739
|
+
if simple_list:
|
|
740
|
+
object_list.append(file_or_dir_path)
|
|
741
|
+
return
|
|
692
742
|
|
|
693
|
-
|
|
694
|
-
|
|
743
|
+
path_object: AtomicPath = AtomicPath(path=file_or_dir_path)
|
|
744
|
+
path_object.queried_directory = directory_path
|
|
695
745
|
|
|
696
746
|
if add_relative_directory:
|
|
697
747
|
# if 'relative_file_name_as_directory' was passed.
|
|
698
748
|
if relative_file_name_as_directory:
|
|
699
749
|
# Output the path with filename.
|
|
700
|
-
|
|
701
|
-
directory_path,
|
|
750
|
+
path_object.relative_dir = _get_relative_output_path_from_input_path(
|
|
751
|
+
directory_path, dir_path, file_or_directory)
|
|
702
752
|
# if 'relative_file_name_as_directory' wasn't passed.
|
|
703
753
|
else:
|
|
704
754
|
# Output the path without filename.
|
|
705
|
-
|
|
755
|
+
path_object.relative_dir = _get_relative_output_path_from_input_path(
|
|
756
|
+
directory_path, dir_path)
|
|
706
757
|
|
|
707
758
|
# Remove separator from the beginning if exists.
|
|
708
|
-
|
|
759
|
+
path_object.relative_dir = path_object.relative_dir.removeprefix(os.sep)
|
|
709
760
|
|
|
710
761
|
# If 'add_last_modified_time' was passed.
|
|
711
762
|
if add_last_modified_time:
|
|
712
763
|
# Get last modified time of the file.
|
|
713
|
-
|
|
764
|
+
path_object.update(update_last_modified=True)
|
|
714
765
|
|
|
715
|
-
|
|
766
|
+
if datetime_format:
|
|
767
|
+
# Get the datetime object from the file name by the date format pattern.
|
|
768
|
+
path_object.update(datetime_format=datetime_format, update_datetime=True)
|
|
769
|
+
# If the datetime string is empty, then the file doesn't contain the date in the filename.
|
|
770
|
+
if not path_object.datetime_string:
|
|
771
|
+
return
|
|
772
|
+
|
|
773
|
+
if specific_date:
|
|
774
|
+
if path_object.datetime_string != specific_date:
|
|
775
|
+
return
|
|
776
|
+
|
|
777
|
+
object_list.append(path_object)
|
|
778
|
+
|
|
779
|
+
if get_file and get_directory:
|
|
780
|
+
raise ValueError('Parameters "get_file" and "get_directory" cannot be both "True".')
|
|
781
|
+
elif not get_file and not get_directory:
|
|
782
|
+
raise ValueError('Parameters "get_file" and "get_directory" cannot be both "False".')
|
|
783
|
+
|
|
784
|
+
if get_directory and (add_file_binary or add_file_hash):
|
|
785
|
+
raise ValueError(
|
|
786
|
+
'While "get_directory" is True, Parameters "add_file_binary" or "add_file_hash" cannot be "True".')
|
|
716
787
|
|
|
717
788
|
if sort_by_last_modified_time and not add_last_modified_time:
|
|
718
789
|
raise ValueError('Parameter "sort_by_last_modified_time" cannot be "True" if parameter '
|
|
@@ -721,18 +792,25 @@ def get_file_paths_from_directory(
|
|
|
721
792
|
raise ValueError('Parameter "relative_file_name_as_directory" cannot be "True" if parameter '
|
|
722
793
|
'"add_relative_directory" is not "True".')
|
|
723
794
|
|
|
795
|
+
if not datetime_format and specific_date:
|
|
796
|
+
raise ValueError('If "specific_date" is specified, "datetime_format" must be specified.')
|
|
797
|
+
|
|
724
798
|
# === Function main ================
|
|
725
799
|
# Define locals.
|
|
726
800
|
object_list: list = list()
|
|
727
801
|
|
|
728
802
|
# "Walk" over all the directories and subdirectories - make list of full file paths inside the directory
|
|
729
803
|
# recursively.
|
|
730
|
-
for
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
804
|
+
for dir_path, sub_dirs, files in os.walk(directory_path):
|
|
805
|
+
if get_file:
|
|
806
|
+
# Iterate through all the file names that were found in the folder.
|
|
807
|
+
for path in files:
|
|
808
|
+
# If 'file_name_check_pattern' was passed.
|
|
809
|
+
get_path(path)
|
|
810
|
+
elif get_directory:
|
|
811
|
+
# Iterate through all the directory names that were found in the folder.
|
|
812
|
+
for path in sub_dirs:
|
|
813
|
+
get_path(path)
|
|
736
814
|
|
|
737
815
|
if not recursive:
|
|
738
816
|
break
|
|
@@ -740,7 +818,34 @@ def get_file_paths_from_directory(
|
|
|
740
818
|
# If 'sort_by_last_modified_time' was passed.
|
|
741
819
|
if sort_by_last_modified_time:
|
|
742
820
|
# Sort the list by last modified time.
|
|
743
|
-
object_list =
|
|
821
|
+
object_list = list_of_classes.sort_by_attributes(object_list, attribute_list=['last_modified'])
|
|
822
|
+
|
|
823
|
+
if add_file_binary or add_file_hash:
|
|
824
|
+
if add_file_binary and not add_file_hash:
|
|
825
|
+
prefix_string = 'Reading Binary of File: '
|
|
826
|
+
elif add_file_hash and not add_file_binary:
|
|
827
|
+
prefix_string = 'Reading Hash of File: '
|
|
828
|
+
elif add_file_binary and add_file_hash:
|
|
829
|
+
prefix_string = 'Reading Binary and Hash of File: '
|
|
830
|
+
else:
|
|
831
|
+
prefix_string = 'Reading File: '
|
|
832
|
+
|
|
833
|
+
for file_index, file_path in enumerate(object_list):
|
|
834
|
+
print_status_of_list(
|
|
835
|
+
list_instance=object_list, prefix_string=prefix_string, current_state=(file_index + 1))
|
|
836
|
+
|
|
837
|
+
# If 'add_binary' was passed.
|
|
838
|
+
if add_file_binary and file_path.is_file:
|
|
839
|
+
# Get binary content of the file.
|
|
840
|
+
object_list[file_index].binary = file_io.read_file(file_path.path, file_mode='rb', stdout=False)
|
|
841
|
+
|
|
842
|
+
# If 'add_file_hash' was passed.
|
|
843
|
+
if add_file_hash and file_path.is_file:
|
|
844
|
+
# Get hash of the file.
|
|
845
|
+
if file_path.binary:
|
|
846
|
+
object_list[file_index].hash = hashing.hash_bytes(file_path.binary)
|
|
847
|
+
else:
|
|
848
|
+
object_list[file_index].hash = hashing.hash_file(file_path.path)
|
|
744
849
|
|
|
745
850
|
return object_list
|
|
746
851
|
|
|
@@ -815,18 +920,6 @@ def remove_last_separator(directory_path: str) -> str:
|
|
|
815
920
|
return directory_path.removesuffix(os.sep)
|
|
816
921
|
|
|
817
922
|
|
|
818
|
-
def remove_last_separator(filesystem_path: str) -> str:
|
|
819
|
-
"""
|
|
820
|
-
The function removes the last character in 'filesystem_path' if it is a system separator ('/' or '\')
|
|
821
|
-
returning the processed string.
|
|
822
|
-
If the character is not a separator, nothing happens.
|
|
823
|
-
|
|
824
|
-
:param filesystem_path:
|
|
825
|
-
:return:
|
|
826
|
-
"""
|
|
827
|
-
return filesystem_path.removesuffix(os.sep)
|
|
828
|
-
|
|
829
|
-
|
|
830
923
|
def add_last_separator(filesystem_path: str) -> str:
|
|
831
924
|
"""
|
|
832
925
|
The function adds a separator to the end of the path if it doesn't exist.
|
|
@@ -862,14 +955,14 @@ def get_files_and_folders(directory_path: str, string_contains: str = str()):
|
|
|
862
955
|
return files_folders_list
|
|
863
956
|
|
|
864
957
|
|
|
865
|
-
def get_file_modified_time(
|
|
958
|
+
def get_file_modified_time(file_or_dir_path: str) -> float:
|
|
866
959
|
"""
|
|
867
960
|
The function returns the time of last modification of the file in seconds since the epoch.
|
|
868
961
|
|
|
869
|
-
:param
|
|
962
|
+
:param file_or_dir_path: string, full path to file or directory.
|
|
870
963
|
:return: float, time of last modification of the file in seconds since the epoch.
|
|
871
964
|
"""
|
|
872
|
-
return os.path.getmtime(
|
|
965
|
+
return os.path.getmtime(file_or_dir_path)
|
|
873
966
|
|
|
874
967
|
|
|
875
968
|
def change_last_modified_date_of_file(file_path: str, new_date: float) -> None:
|
|
@@ -891,43 +984,6 @@ def change_last_modified_date_of_file(file_path: str, new_date: float) -> None:
|
|
|
891
984
|
os.utime(file_path, (new_date, new_date))
|
|
892
985
|
|
|
893
986
|
|
|
894
|
-
def get_file_hashes_from_directory(directory_path: str, recursive: bool = False, add_binary: bool = False) -> list:
|
|
895
|
-
"""
|
|
896
|
-
The function scans a directory for files and returns a list of dictionaries with file path and hash of the file.
|
|
897
|
-
Binary option can be specified.
|
|
898
|
-
|
|
899
|
-
:param directory_path: string, of full path to directory you want to return file names of.
|
|
900
|
-
:param recursive: boolean.
|
|
901
|
-
'True', then the function will scan recursively in subdirectories.
|
|
902
|
-
'False', then the function will scan only in the directory that was passed.
|
|
903
|
-
:param add_binary: boolean, if 'True', then the function will add the binary of the file to the output list.
|
|
904
|
-
|
|
905
|
-
:return: list of dicts with full file paths, hashes and binaries (if specified).
|
|
906
|
-
"""
|
|
907
|
-
|
|
908
|
-
# Get all the files.
|
|
909
|
-
file_paths_list = get_file_paths_from_directory(directory_path, recursive=recursive)
|
|
910
|
-
|
|
911
|
-
# Create a list of dictionaries, each dictionary is a file with its hash.
|
|
912
|
-
files: list = list()
|
|
913
|
-
for file_index, file_path in enumerate(file_paths_list):
|
|
914
|
-
print_status_of_list(
|
|
915
|
-
list_instance=file_paths_list, prefix_string=f'Reading File: ', current_state=(file_index + 1))
|
|
916
|
-
|
|
917
|
-
file_info: dict = dict()
|
|
918
|
-
file_info['path'] = file_path['path']
|
|
919
|
-
|
|
920
|
-
if add_binary:
|
|
921
|
-
file_info['binary'] = file_io.read_file(file_path['path'], file_mode='rb', stdout=False)
|
|
922
|
-
file_info['hash'] = hashing.hash_bytes(file_info['binary'])
|
|
923
|
-
else:
|
|
924
|
-
file_info['hash'] = hashing.hash_file(file_path['path'])
|
|
925
|
-
|
|
926
|
-
files.append(file_info)
|
|
927
|
-
|
|
928
|
-
return files
|
|
929
|
-
|
|
930
|
-
|
|
931
987
|
def find_duplicates_by_hash(
|
|
932
988
|
directory_path: str,
|
|
933
989
|
recursive: bool = False,
|
|
@@ -946,33 +1002,34 @@ def find_duplicates_by_hash(
|
|
|
946
1002
|
"""
|
|
947
1003
|
|
|
948
1004
|
# Get all the files.
|
|
949
|
-
files: list =
|
|
1005
|
+
files: list = get_paths_from_directory(
|
|
1006
|
+
directory_path, get_file=True, recursive=recursive, add_file_binary=add_binary)
|
|
950
1007
|
|
|
951
1008
|
same_hash_files: list = list()
|
|
952
1009
|
# Check if there are files that have exactly the same hash.
|
|
953
|
-
for
|
|
1010
|
+
for atomic_path in files:
|
|
954
1011
|
# Create a list of files that have the same hash for current 'firmware'.
|
|
955
1012
|
current_run_list: list = list()
|
|
956
|
-
for
|
|
1013
|
+
for atomic_path_compare in files:
|
|
957
1014
|
# Add all the 'firmware_compare' that have the same hash to the list.
|
|
958
|
-
if (
|
|
959
|
-
|
|
1015
|
+
if (atomic_path.hash == atomic_path_compare.hash and
|
|
1016
|
+
atomic_path.path != atomic_path_compare.path):
|
|
960
1017
|
# Check if current 'firmware' is already in the 'same_hash_files' list. If not, add 'firmware_compare'
|
|
961
1018
|
# to the 'current_run_list'.
|
|
962
1019
|
if not any(list_of_dicts.is_value_exist_in_key(
|
|
963
|
-
list_of_dicts=test_hash, key='path', value_to_match=
|
|
1020
|
+
list_of_dicts=test_hash, key='path', value_to_match=atomic_path.path) for
|
|
964
1021
|
test_hash in same_hash_files):
|
|
965
1022
|
current_run_list.append({
|
|
966
|
-
'path':
|
|
967
|
-
'hash':
|
|
1023
|
+
'path': atomic_path_compare.path,
|
|
1024
|
+
'hash': atomic_path_compare.hash
|
|
968
1025
|
})
|
|
969
1026
|
|
|
970
1027
|
if current_run_list:
|
|
971
1028
|
# After the iteration of the 'firmware_compare' finished and the list is not empty, add the 'firmware'
|
|
972
1029
|
# to the list.
|
|
973
1030
|
current_run_list.append({
|
|
974
|
-
'path':
|
|
975
|
-
'hash':
|
|
1031
|
+
'path': atomic_path.path,
|
|
1032
|
+
'hash': atomic_path.hash
|
|
976
1033
|
})
|
|
977
1034
|
same_hash_files.append(current_run_list)
|
|
978
1035
|
|
|
@@ -1053,39 +1110,40 @@ def get_directory_size(directory_path: str):
|
|
|
1053
1110
|
|
|
1054
1111
|
|
|
1055
1112
|
def get_subpaths_between(start_path: str, end_path: str) -> list[str]:
|
|
1113
|
+
# noinspection GrazieInspection
|
|
1056
1114
|
"""
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1115
|
+
Get the subpaths between two paths.
|
|
1116
|
+
:param start_path: string, start path.
|
|
1117
|
+
:param end_path: string, end path.
|
|
1118
|
+
:return:
|
|
1061
1119
|
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1120
|
+
Example Linux:
|
|
1121
|
+
start_path = '/test/1'
|
|
1122
|
+
end_path = '/test/1/2/3/4'
|
|
1065
1123
|
|
|
1066
|
-
|
|
1124
|
+
subpaths = get_subpaths_between(start_path, end_path)
|
|
1067
1125
|
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1126
|
+
subpaths = [
|
|
1127
|
+
'/test/1'
|
|
1128
|
+
'/test/1/2',
|
|
1129
|
+
'/test/1/2/3',
|
|
1130
|
+
'/test/1/2/3/4',
|
|
1131
|
+
]
|
|
1074
1132
|
|
|
1075
1133
|
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1134
|
+
Example Windows:
|
|
1135
|
+
start_path = 'C:\\test\\1'
|
|
1136
|
+
end_path = 'C:\\test\\1\\2\\3\\4'
|
|
1079
1137
|
|
|
1080
|
-
|
|
1138
|
+
subpaths = get_subpaths_between(start_path, end_path)
|
|
1081
1139
|
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1140
|
+
subpaths = [
|
|
1141
|
+
'C:\\test\\1',
|
|
1142
|
+
'C:\\test\\1\\2',
|
|
1143
|
+
'C:\\test\\1\\2\\3',
|
|
1144
|
+
'C:\\test\\1\\2\\3\\4',
|
|
1145
|
+
]
|
|
1146
|
+
"""
|
|
1089
1147
|
|
|
1090
1148
|
# Detect slash type based on the input (default to forward slash)
|
|
1091
1149
|
slash_type = "\\" if "\\" in start_path else "/"
|
|
@@ -1126,13 +1184,13 @@ def get_subpaths_between(start_path: str, end_path: str) -> list[str]:
|
|
|
1126
1184
|
# else:
|
|
1127
1185
|
# raise ValueError("Start path must be a parent of the end path")
|
|
1128
1186
|
#
|
|
1129
|
-
# # Reverse the list so it goes from start to end.
|
|
1187
|
+
# # Reverse the list, so it goes from start to end.
|
|
1130
1188
|
# subpaths.reverse()
|
|
1131
1189
|
#
|
|
1132
1190
|
# return subpaths
|
|
1133
1191
|
|
|
1134
1192
|
|
|
1135
|
-
def create_dict_of_paths_list(list_of_paths: list) ->
|
|
1193
|
+
def create_dict_of_paths_list(list_of_paths: list) -> list:
|
|
1136
1194
|
"""
|
|
1137
1195
|
The function receives a list of paths and returns a dictionary with keys as the paths and values as the
|
|
1138
1196
|
subpaths of the key path.
|
|
@@ -1168,7 +1226,7 @@ def create_dict_of_paths_list(list_of_paths: list) -> dict:
|
|
|
1168
1226
|
:return: dictionary.
|
|
1169
1227
|
"""
|
|
1170
1228
|
|
|
1171
|
-
structure = []
|
|
1229
|
+
structure: list = []
|
|
1172
1230
|
for path in list_of_paths:
|
|
1173
1231
|
create_dict_of_path(path, structure)
|
|
1174
1232
|
return structure
|
|
@@ -1251,6 +1309,7 @@ def create_dict_of_path(
|
|
|
1251
1309
|
is_last_part = (i == len(parts) - 1)
|
|
1252
1310
|
|
|
1253
1311
|
# Try to find an existing entry for this part
|
|
1312
|
+
# noinspection PyTypeChecker
|
|
1254
1313
|
existing_entry = next((item for item in current_level if item["entry"] == part), None)
|
|
1255
1314
|
|
|
1256
1315
|
if existing_entry is None:
|
|
@@ -1285,6 +1344,7 @@ def list_open_files_in_directory(directory):
|
|
|
1285
1344
|
proc_open_files = proc.open_files()
|
|
1286
1345
|
for file in proc_open_files:
|
|
1287
1346
|
if file.path.startswith(directory):
|
|
1347
|
+
# noinspection PyUnresolvedReferences
|
|
1288
1348
|
open_files.append((proc.info['pid'], proc.info['name'], file.path))
|
|
1289
1349
|
except (psutil.AccessDenied, psutil.NoSuchProcess):
|
|
1290
1350
|
# Ignore processes that can't be accessed
|
|
@@ -1363,7 +1423,13 @@ def is_file_open_by_process(file_path: str) -> bool:
|
|
|
1363
1423
|
return False
|
|
1364
1424
|
|
|
1365
1425
|
|
|
1366
|
-
def get_download_directory(
|
|
1426
|
+
def get_download_directory(
|
|
1427
|
+
place: Literal[
|
|
1428
|
+
'temp',
|
|
1429
|
+
'script',
|
|
1430
|
+
'working'] = 'temp',
|
|
1431
|
+
script_path: str = None
|
|
1432
|
+
) -> str:
|
|
1367
1433
|
"""
|
|
1368
1434
|
The function returns the default download directory based on place.
|
|
1369
1435
|
|
|
@@ -1395,11 +1461,12 @@ def backup_folder(directory_path: str, backup_directory: str) -> None:
|
|
|
1395
1461
|
"""
|
|
1396
1462
|
Backup the specified directory.
|
|
1397
1463
|
|
|
1398
|
-
:param directory_path: The directory path to
|
|
1399
|
-
:param backup_directory: The directory to
|
|
1464
|
+
:param directory_path: The directory path to back up.
|
|
1465
|
+
:param backup_directory: The directory to back up the directory to.
|
|
1400
1466
|
|
|
1401
1467
|
Example:
|
|
1402
|
-
backup_folder(
|
|
1468
|
+
backup_folder(
|
|
1469
|
+
directory_path='C:\\Users\\user1\\Downloads\\folder1', backup_directory='C:\\Users\\user1\\Downloads\\backup')
|
|
1403
1470
|
|
|
1404
1471
|
Backed up folder will be moved to 'C:\\Users\\user1\\Downloads\\backup' with timestamp in the name.
|
|
1405
1472
|
Final path will look like: 'C:\\Users\\user1\\Downloads\\backup\\20231003-120000-000000_folder1'
|
|
@@ -1471,8 +1538,8 @@ def find_file(file_name: str, directory_path: str):
|
|
|
1471
1538
|
:param directory_path: string, The directory to search in.
|
|
1472
1539
|
:return:
|
|
1473
1540
|
"""
|
|
1474
|
-
for
|
|
1541
|
+
for dir_path, dir_names, filenames in os.walk(directory_path):
|
|
1475
1542
|
for filename in filenames:
|
|
1476
1543
|
if filename == file_name:
|
|
1477
|
-
return os.path.join(
|
|
1544
|
+
return os.path.join(dir_path, filename)
|
|
1478
1545
|
return None
|