atomicshop 2.11.47__py3-none-any.whl → 3.10.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atomicshop/__init__.py +1 -1
- atomicshop/{addons/mains → a_mains}/FACT/update_extract.py +3 -2
- atomicshop/a_mains/addons/process_list/compile.cmd +7 -0
- atomicshop/a_mains/addons/process_list/compiled/Win10x64/process_list.dll +0 -0
- atomicshop/a_mains/addons/process_list/compiled/Win10x64/process_list.exp +0 -0
- atomicshop/a_mains/addons/process_list/compiled/Win10x64/process_list.lib +0 -0
- atomicshop/{addons → a_mains/addons}/process_list/process_list.cpp +8 -1
- atomicshop/a_mains/dns_gateway_setting.py +11 -0
- atomicshop/a_mains/get_local_tcp_ports.py +85 -0
- atomicshop/a_mains/github_wrapper.py +11 -0
- atomicshop/a_mains/install_ca_certificate.py +172 -0
- atomicshop/{addons/mains → a_mains}/msi_unpacker.py +3 -1
- atomicshop/a_mains/process_from_port.py +119 -0
- atomicshop/a_mains/set_default_dns_gateway.py +90 -0
- atomicshop/a_mains/update_config_toml.py +38 -0
- atomicshop/appointment_management.py +5 -3
- atomicshop/basics/ansi_escape_codes.py +3 -1
- atomicshop/basics/argparse_template.py +2 -0
- atomicshop/basics/booleans.py +27 -30
- atomicshop/basics/bytes_arrays.py +43 -0
- atomicshop/basics/classes.py +149 -1
- atomicshop/basics/dicts.py +12 -0
- atomicshop/basics/enums.py +2 -2
- atomicshop/basics/exceptions.py +5 -1
- atomicshop/basics/list_of_classes.py +29 -0
- atomicshop/basics/list_of_dicts.py +69 -5
- atomicshop/basics/lists.py +14 -0
- atomicshop/basics/multiprocesses.py +374 -50
- atomicshop/basics/package_module.py +10 -0
- atomicshop/basics/strings.py +160 -7
- atomicshop/basics/threads.py +14 -0
- atomicshop/basics/tracebacks.py +13 -4
- atomicshop/certificates.py +153 -52
- atomicshop/config_init.py +12 -7
- atomicshop/console_user_response.py +7 -14
- atomicshop/consoles.py +9 -0
- atomicshop/datetimes.py +98 -0
- atomicshop/diff_check.py +340 -40
- atomicshop/dns.py +128 -12
- atomicshop/etws/_pywintrace_fix.py +17 -0
- atomicshop/etws/const.py +38 -0
- atomicshop/etws/providers.py +21 -0
- atomicshop/etws/sessions.py +43 -0
- atomicshop/etws/trace.py +168 -0
- atomicshop/etws/traces/trace_dns.py +162 -0
- atomicshop/etws/traces/trace_sysmon_process_creation.py +126 -0
- atomicshop/etws/traces/trace_tcp.py +130 -0
- atomicshop/file_io/csvs.py +222 -24
- atomicshop/file_io/docxs.py +35 -18
- atomicshop/file_io/file_io.py +35 -19
- atomicshop/file_io/jsons.py +49 -0
- atomicshop/file_io/tomls.py +139 -0
- atomicshop/filesystem.py +864 -293
- atomicshop/get_process_list.py +133 -0
- atomicshop/{process_name_cmd.py → get_process_name_cmd_dll.py} +52 -19
- atomicshop/http_parse.py +149 -93
- atomicshop/ip_addresses.py +6 -1
- atomicshop/mitm/centered_settings.py +132 -0
- atomicshop/mitm/config_static.py +207 -0
- atomicshop/mitm/config_toml_editor.py +55 -0
- atomicshop/mitm/connection_thread_worker.py +875 -357
- atomicshop/mitm/engines/__parent/parser___parent.py +4 -17
- atomicshop/mitm/engines/__parent/recorder___parent.py +108 -51
- atomicshop/mitm/engines/__parent/requester___parent.py +116 -0
- atomicshop/mitm/engines/__parent/responder___parent.py +75 -114
- atomicshop/mitm/engines/__reference_general/parser___reference_general.py +10 -7
- atomicshop/mitm/engines/__reference_general/recorder___reference_general.py +5 -5
- atomicshop/mitm/engines/__reference_general/requester___reference_general.py +47 -0
- atomicshop/mitm/engines/__reference_general/responder___reference_general.py +95 -13
- atomicshop/mitm/engines/create_module_template.py +58 -14
- atomicshop/mitm/import_config.py +359 -139
- atomicshop/mitm/initialize_engines.py +160 -74
- atomicshop/mitm/message.py +64 -23
- atomicshop/mitm/mitm_main.py +892 -0
- atomicshop/mitm/recs_files.py +183 -0
- atomicshop/mitm/shared_functions.py +4 -10
- atomicshop/mitm/ssh_tester.py +82 -0
- atomicshop/mitm/statistic_analyzer.py +257 -166
- atomicshop/mitm/statistic_analyzer_helper/analyzer_helper.py +136 -0
- atomicshop/mitm/statistic_analyzer_helper/moving_average_helper.py +525 -0
- atomicshop/monitor/change_monitor.py +96 -120
- atomicshop/monitor/checks/dns.py +139 -70
- atomicshop/monitor/checks/file.py +77 -0
- atomicshop/monitor/checks/network.py +81 -77
- atomicshop/monitor/checks/process_running.py +33 -34
- atomicshop/monitor/checks/url.py +94 -0
- atomicshop/networks.py +671 -0
- atomicshop/on_exit.py +205 -0
- atomicshop/package_mains_processor.py +84 -0
- atomicshop/permissions/permissions.py +22 -0
- atomicshop/permissions/ubuntu_permissions.py +239 -0
- atomicshop/permissions/win_permissions.py +33 -0
- atomicshop/print_api.py +24 -41
- atomicshop/process.py +63 -17
- atomicshop/process_poller/__init__.py +0 -0
- atomicshop/process_poller/pollers/__init__.py +0 -0
- atomicshop/process_poller/pollers/psutil_pywin32wmi_dll.py +95 -0
- atomicshop/process_poller/process_pool.py +207 -0
- atomicshop/process_poller/simple_process_pool.py +311 -0
- atomicshop/process_poller/tracer_base.py +45 -0
- atomicshop/process_poller/tracers/__init__.py +0 -0
- atomicshop/process_poller/tracers/event_log.py +46 -0
- atomicshop/process_poller/tracers/sysmon_etw.py +68 -0
- atomicshop/python_file_patcher.py +1 -1
- atomicshop/python_functions.py +27 -75
- atomicshop/question_answer_engine.py +2 -2
- atomicshop/scheduling.py +24 -5
- atomicshop/sound.py +4 -2
- atomicshop/speech_recognize.py +8 -0
- atomicshop/ssh_remote.py +158 -172
- atomicshop/startup/__init__.py +0 -0
- atomicshop/startup/win/__init__.py +0 -0
- atomicshop/startup/win/startup_folder.py +53 -0
- atomicshop/startup/win/task_scheduler.py +119 -0
- atomicshop/system_resource_monitor.py +61 -46
- atomicshop/system_resources.py +8 -8
- atomicshop/tempfiles.py +1 -2
- atomicshop/timer.py +30 -11
- atomicshop/urls.py +41 -0
- atomicshop/venvs.py +28 -0
- atomicshop/versioning.py +27 -0
- atomicshop/web.py +110 -25
- atomicshop/web_apis/__init__.py +0 -0
- atomicshop/web_apis/google_custom_search.py +44 -0
- atomicshop/web_apis/google_llm.py +188 -0
- atomicshop/websocket_parse.py +450 -0
- atomicshop/wrappers/certauthw/certauth.py +1 -0
- atomicshop/wrappers/cryptographyw.py +29 -8
- atomicshop/wrappers/ctyping/etw_winapi/__init__.py +0 -0
- atomicshop/wrappers/ctyping/etw_winapi/const.py +335 -0
- atomicshop/wrappers/ctyping/etw_winapi/etw_functions.py +393 -0
- atomicshop/wrappers/ctyping/file_details_winapi.py +67 -0
- atomicshop/wrappers/ctyping/msi_windows_installer/cabs.py +2 -1
- atomicshop/wrappers/ctyping/msi_windows_installer/extract_msi_main.py +13 -9
- atomicshop/wrappers/ctyping/msi_windows_installer/tables.py +35 -0
- atomicshop/wrappers/ctyping/setup_device.py +466 -0
- atomicshop/wrappers/ctyping/win_console.py +39 -0
- atomicshop/wrappers/dockerw/dockerw.py +113 -2
- atomicshop/wrappers/elasticsearchw/config_basic.py +0 -12
- atomicshop/wrappers/elasticsearchw/elastic_infra.py +75 -0
- atomicshop/wrappers/elasticsearchw/elasticsearchw.py +2 -20
- atomicshop/wrappers/factw/get_file_data.py +12 -5
- atomicshop/wrappers/factw/install/install_after_restart.py +89 -5
- atomicshop/wrappers/factw/install/pre_install_and_install_before_restart.py +20 -14
- atomicshop/wrappers/factw/postgresql/firmware.py +4 -6
- atomicshop/wrappers/githubw.py +583 -51
- atomicshop/wrappers/loggingw/consts.py +49 -0
- atomicshop/wrappers/loggingw/filters.py +102 -0
- atomicshop/wrappers/loggingw/formatters.py +58 -71
- atomicshop/wrappers/loggingw/handlers.py +459 -40
- atomicshop/wrappers/loggingw/loggers.py +19 -0
- atomicshop/wrappers/loggingw/loggingw.py +1010 -178
- atomicshop/wrappers/loggingw/reading.py +344 -19
- atomicshop/wrappers/mongodbw/__init__.py +0 -0
- atomicshop/wrappers/mongodbw/mongo_infra.py +31 -0
- atomicshop/wrappers/mongodbw/mongodbw.py +1432 -0
- atomicshop/wrappers/netshw.py +271 -0
- atomicshop/wrappers/playwrightw/engine.py +34 -19
- atomicshop/wrappers/playwrightw/infra.py +5 -0
- atomicshop/wrappers/playwrightw/javascript.py +7 -3
- atomicshop/wrappers/playwrightw/keyboard.py +14 -0
- atomicshop/wrappers/playwrightw/scenarios.py +172 -5
- atomicshop/wrappers/playwrightw/waits.py +9 -7
- atomicshop/wrappers/powershell_networking.py +80 -0
- atomicshop/wrappers/psutilw/processes.py +81 -0
- atomicshop/wrappers/psutilw/psutil_networks.py +85 -0
- atomicshop/wrappers/psutilw/psutilw.py +9 -0
- atomicshop/wrappers/pyopensslw.py +9 -2
- atomicshop/wrappers/pywin32w/__init__.py +0 -0
- atomicshop/wrappers/pywin32w/cert_store.py +116 -0
- atomicshop/wrappers/pywin32w/console.py +34 -0
- atomicshop/wrappers/pywin32w/win_event_log/__init__.py +0 -0
- atomicshop/wrappers/pywin32w/win_event_log/fetch.py +174 -0
- atomicshop/wrappers/pywin32w/win_event_log/subscribe.py +212 -0
- atomicshop/wrappers/pywin32w/win_event_log/subscribes/__init__.py +0 -0
- atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_create.py +57 -0
- atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_terminate.py +49 -0
- atomicshop/wrappers/pywin32w/win_event_log/subscribes/schannel_logging.py +97 -0
- atomicshop/wrappers/pywin32w/winshell.py +19 -0
- atomicshop/wrappers/pywin32w/wmis/__init__.py +0 -0
- atomicshop/wrappers/pywin32w/wmis/msft_netipaddress.py +113 -0
- atomicshop/wrappers/pywin32w/wmis/win32_networkadapterconfiguration.py +259 -0
- atomicshop/wrappers/pywin32w/wmis/win32networkadapter.py +112 -0
- atomicshop/wrappers/pywin32w/wmis/wmi_helpers.py +236 -0
- atomicshop/wrappers/socketw/accepter.py +21 -7
- atomicshop/wrappers/socketw/certificator.py +216 -150
- atomicshop/wrappers/socketw/creator.py +190 -50
- atomicshop/wrappers/socketw/dns_server.py +500 -173
- atomicshop/wrappers/socketw/exception_wrapper.py +45 -52
- atomicshop/wrappers/socketw/process_getter.py +86 -0
- atomicshop/wrappers/socketw/receiver.py +144 -102
- atomicshop/wrappers/socketw/sender.py +65 -35
- atomicshop/wrappers/socketw/sni.py +334 -165
- atomicshop/wrappers/socketw/socket_base.py +134 -0
- atomicshop/wrappers/socketw/socket_client.py +137 -95
- atomicshop/wrappers/socketw/socket_server_tester.py +14 -9
- atomicshop/wrappers/socketw/socket_wrapper.py +717 -116
- atomicshop/wrappers/socketw/ssl_base.py +15 -14
- atomicshop/wrappers/socketw/statistics_csv.py +148 -17
- atomicshop/wrappers/sysmonw.py +157 -0
- atomicshop/wrappers/ubuntu_terminal.py +65 -26
- atomicshop/wrappers/win_auditw.py +189 -0
- atomicshop/wrappers/winregw/__init__.py +0 -0
- atomicshop/wrappers/winregw/winreg_installed_software.py +58 -0
- atomicshop/wrappers/winregw/winreg_network.py +232 -0
- {atomicshop-2.11.47.dist-info → atomicshop-3.10.5.dist-info}/METADATA +31 -49
- atomicshop-3.10.5.dist-info/RECORD +306 -0
- {atomicshop-2.11.47.dist-info → atomicshop-3.10.5.dist-info}/WHEEL +1 -1
- atomicshop/_basics_temp.py +0 -101
- atomicshop/addons/a_setup_scripts/install_psycopg2_ubuntu.sh +0 -3
- atomicshop/addons/a_setup_scripts/install_pywintrace_0.3.cmd +0 -2
- atomicshop/addons/mains/install_docker_rootless_ubuntu.py +0 -11
- atomicshop/addons/mains/install_docker_ubuntu_main_sudo.py +0 -11
- atomicshop/addons/mains/install_elastic_search_and_kibana_ubuntu.py +0 -10
- atomicshop/addons/mains/install_wsl_ubuntu_lts_admin.py +0 -9
- atomicshop/addons/package_setup/CreateWheel.cmd +0 -7
- atomicshop/addons/package_setup/Setup in Edit mode.cmd +0 -6
- atomicshop/addons/package_setup/Setup.cmd +0 -7
- atomicshop/addons/process_list/compile.cmd +0 -2
- atomicshop/addons/process_list/compiled/Win10x64/process_list.dll +0 -0
- atomicshop/addons/process_list/compiled/Win10x64/process_list.exp +0 -0
- atomicshop/addons/process_list/compiled/Win10x64/process_list.lib +0 -0
- atomicshop/archiver/_search_in_zip.py +0 -189
- atomicshop/archiver/archiver.py +0 -34
- atomicshop/archiver/search_in_archive.py +0 -250
- atomicshop/archiver/sevenz_app_w.py +0 -86
- atomicshop/archiver/sevenzs.py +0 -44
- atomicshop/archiver/zips.py +0 -293
- atomicshop/etw/dns_trace.py +0 -118
- atomicshop/etw/etw.py +0 -61
- atomicshop/file_types.py +0 -24
- atomicshop/mitm/engines/create_module_template_example.py +0 -13
- atomicshop/mitm/initialize_mitm_server.py +0 -240
- atomicshop/monitor/checks/hash.py +0 -44
- atomicshop/monitor/checks/hash_checks/file.py +0 -55
- atomicshop/monitor/checks/hash_checks/url.py +0 -62
- atomicshop/pbtkmultifile_argparse.py +0 -88
- atomicshop/permissions.py +0 -110
- atomicshop/process_poller.py +0 -237
- atomicshop/script_as_string_processor.py +0 -38
- atomicshop/ssh_scripts/process_from_ipv4.py +0 -37
- atomicshop/ssh_scripts/process_from_port.py +0 -27
- atomicshop/wrappers/_process_wrapper_curl.py +0 -27
- atomicshop/wrappers/_process_wrapper_tar.py +0 -21
- atomicshop/wrappers/dockerw/install_docker.py +0 -209
- atomicshop/wrappers/elasticsearchw/infrastructure.py +0 -265
- atomicshop/wrappers/elasticsearchw/install_elastic.py +0 -232
- atomicshop/wrappers/ffmpegw.py +0 -125
- atomicshop/wrappers/loggingw/checks.py +0 -20
- atomicshop/wrappers/nodejsw/install_nodejs.py +0 -139
- atomicshop/wrappers/process_wrapper_pbtk.py +0 -16
- atomicshop/wrappers/socketw/base.py +0 -59
- atomicshop/wrappers/socketw/get_process.py +0 -107
- atomicshop/wrappers/wslw.py +0 -191
- atomicshop-2.11.47.dist-info/RECORD +0 -251
- /atomicshop/{addons/mains → a_mains}/FACT/factw_fact_extractor_docker_image_main_sudo.py +0 -0
- /atomicshop/{addons → a_mains/addons}/PlayWrightCodegen.cmd +0 -0
- /atomicshop/{addons → a_mains/addons}/ScriptExecution.cmd +0 -0
- /atomicshop/{addons/mains → a_mains/addons}/inits/init_to_import_all_modules.py +0 -0
- /atomicshop/{addons → a_mains/addons}/process_list/ReadMe.txt +0 -0
- /atomicshop/{addons/mains → a_mains}/search_for_hyperlinks_in_docx.py +0 -0
- /atomicshop/{archiver → etws}/__init__.py +0 -0
- /atomicshop/{etw → etws/traces}/__init__.py +0 -0
- /atomicshop/{monitor/checks/hash_checks → mitm/statistic_analyzer_helper}/__init__.py +0 -0
- /atomicshop/{wrappers/nodejsw → permissions}/__init__.py +0 -0
- /atomicshop/wrappers/pywin32w/{wmi_win32process.py → wmis/win32process.py} +0 -0
- {atomicshop-2.11.47.dist-info → atomicshop-3.10.5.dist-info/licenses}/LICENSE.txt +0 -0
- {atomicshop-2.11.47.dist-info → atomicshop-3.10.5.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,1432 @@
|
|
|
1
|
+
from typing import Union, Literal
|
|
2
|
+
import datetime
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
# noinspection PyPackageRequirements
|
|
6
|
+
import pymongo
|
|
7
|
+
# noinspection PyPackageRequirements
|
|
8
|
+
import pymongo.database
|
|
9
|
+
|
|
10
|
+
from ...basics import dicts
|
|
11
|
+
|
|
12
|
+
from . import mongo_infra
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
"""
|
|
16
|
+
DISCLAIMER: you should use the pymongo library directly instead of using the atomicshop/wrappers/mongodbw/mongodbw.py.
|
|
17
|
+
These are good examples to get you started, but can't really cover all the use cases you might have.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class MongoDBReplaceOneError(Exception):
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class MongoDBUpdateOneError(Exception):
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class MongoDBUpdateManyError(Exception):
|
|
30
|
+
pass
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class MongoDBWrapper:
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
db_name: str,
|
|
37
|
+
uri: str = mongo_infra.MONGODB_DEFAULT_URI
|
|
38
|
+
):
|
|
39
|
+
self.db_name: str = db_name
|
|
40
|
+
self.uri: str = uri
|
|
41
|
+
|
|
42
|
+
# noinspection PyTypeChecker
|
|
43
|
+
self.client: pymongo.MongoClient = None
|
|
44
|
+
# noinspection PyTypeChecker
|
|
45
|
+
self.db: pymongo.database.Database = None
|
|
46
|
+
|
|
47
|
+
def connect(self):
|
|
48
|
+
"""
|
|
49
|
+
Connect to a MongoDB database.
|
|
50
|
+
:return: pymongo.MongoClient, the client object.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
if not self.client:
|
|
54
|
+
self.client = connect(uri=self.uri)
|
|
55
|
+
self.db = get_db(database=self.db_name, mongo_client=self.client)
|
|
56
|
+
|
|
57
|
+
def disconnect(self):
|
|
58
|
+
"""
|
|
59
|
+
Disconnect from a MongoDB database.
|
|
60
|
+
:return: None
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
if self.client:
|
|
64
|
+
self.client.close()
|
|
65
|
+
self.client = None
|
|
66
|
+
|
|
67
|
+
def insert(
|
|
68
|
+
self,
|
|
69
|
+
object_instance: Union[list[dict], dict],
|
|
70
|
+
collection_name: str,
|
|
71
|
+
add_timestamp: bool = False,
|
|
72
|
+
convert_mixed_lists_to_strings: bool = False
|
|
73
|
+
):
|
|
74
|
+
"""
|
|
75
|
+
Add a dictionary or list dictionaries to a MongoDB collection.
|
|
76
|
+
:param object_instance: list of dictionaries or dictionary to add to the collection.
|
|
77
|
+
:param collection_name: str, the name of the collection.
|
|
78
|
+
:param add_timestamp: bool, if True, a current time timestamp will be added to the object.
|
|
79
|
+
:param convert_mixed_lists_to_strings: bool, if True, mixed lists or tuples when entries are
|
|
80
|
+
strings and integers, the integers will be converted to strings.
|
|
81
|
+
|
|
82
|
+
:return: None
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
self.connect()
|
|
86
|
+
|
|
87
|
+
insert(
|
|
88
|
+
object_instance=object_instance,
|
|
89
|
+
database=self.db, collection_name=collection_name,
|
|
90
|
+
add_timestamp=add_timestamp, convert_mixed_lists_to_strings=convert_mixed_lists_to_strings,
|
|
91
|
+
mongo_client=self.client, close_client=False)
|
|
92
|
+
|
|
93
|
+
def delete(
|
|
94
|
+
self,
|
|
95
|
+
filter_instance: Union[list[dict], dict],
|
|
96
|
+
collection_name: str
|
|
97
|
+
):
|
|
98
|
+
"""
|
|
99
|
+
Remove a dict or list of dictionaries or a dictionary from a MongoDB collection.
|
|
100
|
+
For pure mongo, this is the list of queries to remove.
|
|
101
|
+
Each query for a single item.
|
|
102
|
+
|
|
103
|
+
:param filter_instance: dict or list of dictionaries (the list of filters to remove from the collection).
|
|
104
|
+
:param collection_name: str, the name of the collection.
|
|
105
|
+
|
|
106
|
+
:return: None
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
self.connect()
|
|
110
|
+
|
|
111
|
+
delete(
|
|
112
|
+
filter_instance=filter_instance,
|
|
113
|
+
database=self.db, collection_name=collection_name,
|
|
114
|
+
mongo_client=self.client, close_client=False)
|
|
115
|
+
|
|
116
|
+
def delete_many(
|
|
117
|
+
self,
|
|
118
|
+
filter_query: dict,
|
|
119
|
+
collection_name: str
|
|
120
|
+
):
|
|
121
|
+
"""
|
|
122
|
+
Remove all entries that match the filter query from a MongoDB collection.
|
|
123
|
+
|
|
124
|
+
:param filter_query: dict, the filter query to search for.
|
|
125
|
+
Example, search for all entries with column name 'name' equal to 'John':
|
|
126
|
+
filter query = {'name': 'John'}
|
|
127
|
+
:param collection_name: str, the name of the collection.
|
|
128
|
+
|
|
129
|
+
:return: result of the operation.
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
self.connect()
|
|
133
|
+
|
|
134
|
+
return delete_many(
|
|
135
|
+
filter_query=filter_query,
|
|
136
|
+
database=self.db, collection_name=collection_name,
|
|
137
|
+
mongo_client=self.client, close_client=False)
|
|
138
|
+
|
|
139
|
+
def create_index(
|
|
140
|
+
self,
|
|
141
|
+
collection_name: str,
|
|
142
|
+
fields_list: list[tuple[str, int]],
|
|
143
|
+
name: str = None
|
|
144
|
+
):
|
|
145
|
+
"""
|
|
146
|
+
Create an index in a MongoDB collection.
|
|
147
|
+
:param collection_name: str, the name of the collection.
|
|
148
|
+
:param fields_list: list of tuples, each tuple will contain
|
|
149
|
+
[0] string of the field name and
|
|
150
|
+
[1] the integer value of the order
|
|
151
|
+
to sort by, this is pymongo default, 1 for ascending and -1 for descending.
|
|
152
|
+
Example:
|
|
153
|
+
[
|
|
154
|
+
('vendor', 1),
|
|
155
|
+
('model', -1)
|
|
156
|
+
]
|
|
157
|
+
|
|
158
|
+
Explanation:
|
|
159
|
+
This will create a compound index that will sort the collection by the field 'vendor'
|
|
160
|
+
in ascending order, and then by the field 'model' in descending order.
|
|
161
|
+
:param name: str, the name of the index.
|
|
162
|
+
|
|
163
|
+
:return: None
|
|
164
|
+
"""
|
|
165
|
+
|
|
166
|
+
self.connect()
|
|
167
|
+
|
|
168
|
+
create_index(
|
|
169
|
+
database=self.db, collection_name=collection_name,
|
|
170
|
+
fields_list=fields_list, name=name,
|
|
171
|
+
mongo_client=self.client, close_client=False)
|
|
172
|
+
|
|
173
|
+
def find(
|
|
174
|
+
self,
|
|
175
|
+
collection_name: str,
|
|
176
|
+
filter_query: dict = None,
|
|
177
|
+
projection: dict = None,
|
|
178
|
+
page: int = None,
|
|
179
|
+
items: int = None,
|
|
180
|
+
sort: dict[str, Literal[
|
|
181
|
+
'asc', 'desc',
|
|
182
|
+
1, -1]] = None,
|
|
183
|
+
convert_object_id_to_str: bool = False,
|
|
184
|
+
keys_convert_to_dict: list[str] = None
|
|
185
|
+
) -> list[dict]:
|
|
186
|
+
"""
|
|
187
|
+
Find entries in a MongoDB collection by query.
|
|
188
|
+
:param collection_name: str, the name of the collection.
|
|
189
|
+
:param filter_query: dict, the query to search for.
|
|
190
|
+
Example, search for all entries with column name 'name' equal to 'John':
|
|
191
|
+
filter_query = {'name': 'John'}
|
|
192
|
+
Example, return all entries from collection:
|
|
193
|
+
filter_query = None
|
|
194
|
+
|
|
195
|
+
CHECK MORE EXAMPLES IN THE DOCSTRING OF THE FUNCTION 'find' BELOW which is not in this class.
|
|
196
|
+
:param projection: dict, the only fields to return or exclude.
|
|
197
|
+
:param page: int, the page number (Optional).
|
|
198
|
+
The results are filtered after results are fetched from db.
|
|
199
|
+
:param items: int, the number of results per page (Optional).
|
|
200
|
+
The results are filtered after results are fetched from db.
|
|
201
|
+
:param sort: dict, the name of the field and the order to sort the containers by.
|
|
202
|
+
You can use several fields to sort the containers by several fields.
|
|
203
|
+
In this case the containers will be sorted by the first field, then by the second field, etc.
|
|
204
|
+
You can also use only singular field to sort the containers by only one field.
|
|
205
|
+
Usage:
|
|
206
|
+
{
|
|
207
|
+
field_name: order
|
|
208
|
+
}
|
|
209
|
+
Example:
|
|
210
|
+
{
|
|
211
|
+
'vendor': 'asc',
|
|
212
|
+
'model': 'desc'
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
Or example using integers:
|
|
216
|
+
{
|
|
217
|
+
'vendor': 1,
|
|
218
|
+
'model': -1
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
:param convert_object_id_to_str: bool, if True, the '_id' field will be converted to a string.
|
|
222
|
+
The '_id' field is an ObjectId type, which is a complex object, it can be converted to a string for simpler
|
|
223
|
+
processing.
|
|
224
|
+
:param keys_convert_to_dict: list, the keys of the documents that should be converted from string to dict.
|
|
225
|
+
Recursively searches for keys in specified list in a nested dictionary in result entries list,
|
|
226
|
+
and converts their values using 'json.loads' if found.
|
|
227
|
+
:return: list of dictionaries, the list of entries that match the query.
|
|
228
|
+
"""
|
|
229
|
+
|
|
230
|
+
self.connect()
|
|
231
|
+
|
|
232
|
+
entries: list[dict] = find(
|
|
233
|
+
database=self.db, collection_name=collection_name,
|
|
234
|
+
filter_query=filter_query, projection=projection,
|
|
235
|
+
page=page, items=items, sort=sort,
|
|
236
|
+
convert_object_id_to_str=convert_object_id_to_str, key_convert_to_dict=keys_convert_to_dict,
|
|
237
|
+
mongo_client=self.client, close_client=False)
|
|
238
|
+
|
|
239
|
+
return entries
|
|
240
|
+
|
|
241
|
+
def distinct(
|
|
242
|
+
self,
|
|
243
|
+
collection_name: str,
|
|
244
|
+
field_name: str,
|
|
245
|
+
filter_query: dict = None
|
|
246
|
+
) -> list:
|
|
247
|
+
"""
|
|
248
|
+
Get distinct values of a field from a MongoDB collection.
|
|
249
|
+
Example:
|
|
250
|
+
Example database:
|
|
251
|
+
{
|
|
252
|
+
'users': [
|
|
253
|
+
{'name': 'John', 'age': 25},
|
|
254
|
+
{'name': 'John', 'age': 30},
|
|
255
|
+
{'name': 'Alice', 'age': 25}
|
|
256
|
+
]
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
Get distinct values of the field 'name' from the collection 'users':
|
|
260
|
+
distinct('users', 'name')
|
|
261
|
+
|
|
262
|
+
Output:
|
|
263
|
+
['John', 'Alice']
|
|
264
|
+
|
|
265
|
+
:param collection_name: str, the name of the collection.
|
|
266
|
+
:param field_name: str, the name of the field.
|
|
267
|
+
:param filter_query: dict, the filter query to search for. If None, the filter query will not be executed.
|
|
268
|
+
|
|
269
|
+
:return: list, the list of distinct values.
|
|
270
|
+
"""
|
|
271
|
+
|
|
272
|
+
self.connect()
|
|
273
|
+
|
|
274
|
+
distinct_values = distinct(
|
|
275
|
+
database=self.db, collection_name=collection_name,
|
|
276
|
+
field_name=field_name, filter_query=filter_query, mongo_client=self.client, close_client=False)
|
|
277
|
+
|
|
278
|
+
return distinct_values
|
|
279
|
+
|
|
280
|
+
def update(
|
|
281
|
+
self,
|
|
282
|
+
collection_name: str,
|
|
283
|
+
filter_query: dict,
|
|
284
|
+
update_instance: Union[dict, list[dict]],
|
|
285
|
+
add_timestamp: bool = False,
|
|
286
|
+
convert_mixed_lists_to_strings: bool = False
|
|
287
|
+
):
|
|
288
|
+
"""
|
|
289
|
+
Update one entry in a MongoDB collection by filter query.
|
|
290
|
+
:param collection_name: str, the name of the collection.
|
|
291
|
+
:param filter_query: dict, the filter query to search for.
|
|
292
|
+
Example, search for all entries with column name 'name' equal to 'John':
|
|
293
|
+
filter_query = {'name': 'John'}
|
|
294
|
+
Find by Object id:
|
|
295
|
+
filter_query = {'_id': ObjectId('5f3e3b3b4b9f3b3b4b9f3b3b')}
|
|
296
|
+
:param update_instance: dict or list of dicts, the update to apply.
|
|
297
|
+
Get examples for operators for each dict in the docstring of the function 'update' below.
|
|
298
|
+
:param add_timestamp: bool, if True, a current time timestamp will be added to the object.
|
|
299
|
+
:param convert_mixed_lists_to_strings: bool, if True, mixed lists or tuples when entries are
|
|
300
|
+
strings and integers, the integers will be converted to strings.
|
|
301
|
+
:return: result of the operation.
|
|
302
|
+
"""
|
|
303
|
+
|
|
304
|
+
self.connect()
|
|
305
|
+
|
|
306
|
+
return update(
|
|
307
|
+
database=self.db, collection_name=collection_name,
|
|
308
|
+
filter_query=filter_query, update_instance=update_instance, add_timestamp=add_timestamp,
|
|
309
|
+
convert_mixed_lists_to_strings=convert_mixed_lists_to_strings,
|
|
310
|
+
mongo_client=self.client, close_client=False)
|
|
311
|
+
|
|
312
|
+
def replace(
|
|
313
|
+
self,
|
|
314
|
+
collection_name: str,
|
|
315
|
+
filter_query: dict,
|
|
316
|
+
replacement: dict,
|
|
317
|
+
add_timestamp: bool = False,
|
|
318
|
+
convert_mixed_lists_to_strings: bool = False
|
|
319
|
+
):
|
|
320
|
+
"""
|
|
321
|
+
Replace one entry in a MongoDB collection by filter query.
|
|
322
|
+
:param collection_name: str, the name of the collection.
|
|
323
|
+
:param filter_query: dict, the filter query to search for.
|
|
324
|
+
Example, search for all entries with column name 'name' equal to 'John':
|
|
325
|
+
filter_query = {'name': 'John'}
|
|
326
|
+
Find by Object id:
|
|
327
|
+
filter_query = {'_id': ObjectId('5f3e3b3b4b9f3b3b4b9f3b3b')}
|
|
328
|
+
:param replacement: dict, the replacement to apply.
|
|
329
|
+
:param add_timestamp: bool, if True, a current time timestamp will be added to the object.
|
|
330
|
+
:param convert_mixed_lists_to_strings: bool, if True, mixed lists or tuples when entries are
|
|
331
|
+
|
|
332
|
+
:return: result of the operation.
|
|
333
|
+
"""
|
|
334
|
+
|
|
335
|
+
self.connect()
|
|
336
|
+
|
|
337
|
+
return replace(
|
|
338
|
+
database=self.db, collection_name=collection_name,
|
|
339
|
+
filter_query=filter_query, replacement=replacement,
|
|
340
|
+
add_timestamp=add_timestamp, convert_mixed_lists_to_strings=convert_mixed_lists_to_strings,
|
|
341
|
+
mongo_client=self.client, close_client=False)
|
|
342
|
+
|
|
343
|
+
def get_all_indexes_in_collection(
|
|
344
|
+
self,
|
|
345
|
+
collection_name: str
|
|
346
|
+
) -> dict:
|
|
347
|
+
"""
|
|
348
|
+
Get all indexes in a MongoDB collection.
|
|
349
|
+
:param collection_name: str, the name of the collection.
|
|
350
|
+
:return: list of dictionaries, the list of indexes.
|
|
351
|
+
"""
|
|
352
|
+
|
|
353
|
+
self.connect()
|
|
354
|
+
|
|
355
|
+
indexes: dict = get_all_indexes_in_collection(
|
|
356
|
+
database=self.db, collection_name=collection_name,
|
|
357
|
+
mongo_client=self.client, close_client=False)
|
|
358
|
+
|
|
359
|
+
return indexes
|
|
360
|
+
|
|
361
|
+
def is_index_name_in_collection(
|
|
362
|
+
self,
|
|
363
|
+
collection_name: str,
|
|
364
|
+
index_name: str
|
|
365
|
+
) -> bool:
|
|
366
|
+
"""
|
|
367
|
+
Check if an index name exists in a MongoDB collection.
|
|
368
|
+
:param collection_name: str, the name of the collection.
|
|
369
|
+
:param index_name: str, the name of the index.
|
|
370
|
+
:return: bool, if the index name exists in the collection.
|
|
371
|
+
"""
|
|
372
|
+
|
|
373
|
+
self.connect()
|
|
374
|
+
|
|
375
|
+
exists: bool = is_index_name_in_collection(
|
|
376
|
+
database=self.db, collection_name=collection_name,
|
|
377
|
+
index_name=index_name, mongo_client=self.client, close_client=False)
|
|
378
|
+
|
|
379
|
+
return exists
|
|
380
|
+
|
|
381
|
+
def count_entries_in_collection(
|
|
382
|
+
self,
|
|
383
|
+
collection_name: str,
|
|
384
|
+
filter_query: dict = None
|
|
385
|
+
) -> int:
|
|
386
|
+
"""
|
|
387
|
+
Count entries in a MongoDB collection by query.
|
|
388
|
+
|
|
389
|
+
:param collection_name: str, the name of the collection.
|
|
390
|
+
:param filter_query: dict, the query to search for.
|
|
391
|
+
Example, search for all entries with column name 'name' equal to 'John':
|
|
392
|
+
filter_query = {'name': 'John'}
|
|
393
|
+
Example, return all entries from collection:
|
|
394
|
+
filter_query = None
|
|
395
|
+
|
|
396
|
+
:return: int, the number of entries that match the query.
|
|
397
|
+
"""
|
|
398
|
+
|
|
399
|
+
self.connect()
|
|
400
|
+
|
|
401
|
+
count = count_entries_in_collection(
|
|
402
|
+
database=self.db, collection_name=collection_name,
|
|
403
|
+
filter_query=filter_query, mongo_client=self.client, close_client=False)
|
|
404
|
+
|
|
405
|
+
return count
|
|
406
|
+
|
|
407
|
+
def aggregate_entries_in_collection(
|
|
408
|
+
self,
|
|
409
|
+
collection_name: str,
|
|
410
|
+
pipeline: list[dict]
|
|
411
|
+
) -> list[dict]:
|
|
412
|
+
"""
|
|
413
|
+
Aggregate entries in a MongoDB collection by query.
|
|
414
|
+
|
|
415
|
+
:param collection_name: str, the name of the collection.
|
|
416
|
+
:param pipeline: list of dictionaries, the pipeline to search for.
|
|
417
|
+
Example, search for all entries with column name 'name' equal to 'John':
|
|
418
|
+
pipeline = [{'$match': {'name': 'John'}}]
|
|
419
|
+
Example, return all entries from collection:
|
|
420
|
+
pipeline = []
|
|
421
|
+
|
|
422
|
+
:return: list of dictionaries, the list of entries that match the query.
|
|
423
|
+
"""
|
|
424
|
+
|
|
425
|
+
self.connect()
|
|
426
|
+
|
|
427
|
+
aggregation: list[dict] = aggregate_entries_in_collection(
|
|
428
|
+
database=self.db, collection_name=collection_name,
|
|
429
|
+
pipeline=pipeline, mongo_client=self.client, close_client=False)
|
|
430
|
+
|
|
431
|
+
return aggregation
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def get_client(self):
|
|
435
|
+
return self.client
|
|
436
|
+
|
|
437
|
+
def get_stats_db(
|
|
438
|
+
self
|
|
439
|
+
):
|
|
440
|
+
"""
|
|
441
|
+
Get the stats of a MongoDB database.
|
|
442
|
+
|
|
443
|
+
:return: dict, the stats of the collection.
|
|
444
|
+
"""
|
|
445
|
+
|
|
446
|
+
self.connect()
|
|
447
|
+
|
|
448
|
+
stats = get_stats_db(
|
|
449
|
+
database=self.db, mongo_client=self.client, close_client=False)
|
|
450
|
+
|
|
451
|
+
return stats
|
|
452
|
+
|
|
453
|
+
def get_stats_db_size(
|
|
454
|
+
self
|
|
455
|
+
):
|
|
456
|
+
"""
|
|
457
|
+
Get the size of a MongoDB database in bytes.
|
|
458
|
+
|
|
459
|
+
:return: int, the size of the database in bytes.
|
|
460
|
+
"""
|
|
461
|
+
|
|
462
|
+
self.connect()
|
|
463
|
+
|
|
464
|
+
size = get_stats_db_size(
|
|
465
|
+
database=self.db, mongo_client=self.client, close_client=False)
|
|
466
|
+
|
|
467
|
+
return size
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def connect(uri: str = mongo_infra.MONGODB_DEFAULT_URI):
|
|
471
|
+
"""
|
|
472
|
+
Connect to a MongoDB database.
|
|
473
|
+
:param uri: str, the URI of the MongoDB database.
|
|
474
|
+
:return: pymongo.MongoClient, the client object.
|
|
475
|
+
"""
|
|
476
|
+
return pymongo.MongoClient(uri)
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def get_db(
|
|
480
|
+
database: str,
|
|
481
|
+
mongo_client: pymongo.MongoClient = None
|
|
482
|
+
) -> pymongo.database.Database:
|
|
483
|
+
"""
|
|
484
|
+
Get a MongoDB database object.
|
|
485
|
+
:param database: String, the name of the database.
|
|
486
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
487
|
+
If None, a new connection will be created to default URI.
|
|
488
|
+
:return: pymongo.database.Database, the database object.
|
|
489
|
+
"""
|
|
490
|
+
|
|
491
|
+
if not mongo_client:
|
|
492
|
+
mongo_client = connect()
|
|
493
|
+
|
|
494
|
+
return mongo_client[database]
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def insert(
|
|
498
|
+
object_instance: Union[list[dict], dict],
|
|
499
|
+
database: Union[str, pymongo.database.Database],
|
|
500
|
+
collection_name: str,
|
|
501
|
+
add_timestamp: bool = False,
|
|
502
|
+
convert_mixed_lists_to_strings: bool = False,
|
|
503
|
+
mongo_client: pymongo.MongoClient = None,
|
|
504
|
+
close_client: bool = False
|
|
505
|
+
):
|
|
506
|
+
"""
|
|
507
|
+
Add a dictionary or list dictionaries to a MongoDB collection.
|
|
508
|
+
:param object_instance: list of dictionaries or dictionary to add to the collection.
|
|
509
|
+
:param database: String or the database object.
|
|
510
|
+
str - the name of the database. In this case the database object will be created.
|
|
511
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
512
|
+
:param collection_name: str, the name of the collection.
|
|
513
|
+
:param add_timestamp: bool, if True, a current time timestamp will be added to the object.
|
|
514
|
+
:param convert_mixed_lists_to_strings: bool, if True, mixed lists or tuples when entries are strings and integers,
|
|
515
|
+
the integers will be converted to strings.
|
|
516
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
517
|
+
If None, a new connection will be created to default URI.
|
|
518
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
519
|
+
|
|
520
|
+
:return: None
|
|
521
|
+
"""
|
|
522
|
+
|
|
523
|
+
_is_object_list_of_dicts_or_dict(object_instance)
|
|
524
|
+
|
|
525
|
+
if not mongo_client:
|
|
526
|
+
mongo_client = connect()
|
|
527
|
+
close_client = True
|
|
528
|
+
|
|
529
|
+
db = _get_pymongo_db_from_string_or_pymongo_db(database, mongo_client)
|
|
530
|
+
collection = db[collection_name]
|
|
531
|
+
|
|
532
|
+
if convert_mixed_lists_to_strings:
|
|
533
|
+
if isinstance(object_instance, dict):
|
|
534
|
+
object_instance = dicts.convert_int_to_str_in_mixed_lists(object_instance)
|
|
535
|
+
elif isinstance(object_instance, list):
|
|
536
|
+
for doc_index, doc in enumerate(object_instance):
|
|
537
|
+
object_instance[doc_index] = dicts.convert_int_to_str_in_mixed_lists(doc)
|
|
538
|
+
|
|
539
|
+
if add_timestamp:
|
|
540
|
+
timestamp = datetime.datetime.now()
|
|
541
|
+
if isinstance(object_instance, dict):
|
|
542
|
+
object_instance['timestamp'] = timestamp
|
|
543
|
+
elif isinstance(object_instance, list):
|
|
544
|
+
for doc in object_instance:
|
|
545
|
+
doc['timestamp'] = timestamp
|
|
546
|
+
|
|
547
|
+
if isinstance(object_instance, dict):
|
|
548
|
+
collection.insert_one(object_instance)
|
|
549
|
+
elif isinstance(object_instance, list):
|
|
550
|
+
collection.insert_many(object_instance)
|
|
551
|
+
|
|
552
|
+
if close_client:
|
|
553
|
+
mongo_client.close()
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def delete(
|
|
557
|
+
filter_instance: Union[list[dict], dict],
|
|
558
|
+
database: Union[str, pymongo.database.Database],
|
|
559
|
+
collection_name: str,
|
|
560
|
+
mongo_client: pymongo.MongoClient = None,
|
|
561
|
+
close_client: bool = False
|
|
562
|
+
):
|
|
563
|
+
"""
|
|
564
|
+
Remove a dict or list of dictionaries or a dictionary from a MongoDB collection.
|
|
565
|
+
|
|
566
|
+
:param filter_instance: dict or list of dictionaries,
|
|
567
|
+
dict, the regular filter for pymongo.
|
|
568
|
+
list of dictionaries to remove from the collection, for pure mongo, this is the list of filtered to remove.
|
|
569
|
+
Each filter for a single item.
|
|
570
|
+
:param database: String or the database object.
|
|
571
|
+
str - the name of the database. In this case the database object will be created.
|
|
572
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
573
|
+
:param collection_name: str, the name of the collection.
|
|
574
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
575
|
+
If None, a new connection will be created to default URI.
|
|
576
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
577
|
+
|
|
578
|
+
:return: None
|
|
579
|
+
"""
|
|
580
|
+
|
|
581
|
+
_is_object_list_of_dicts_or_dict(filter_instance)
|
|
582
|
+
|
|
583
|
+
if not mongo_client:
|
|
584
|
+
mongo_client = connect()
|
|
585
|
+
close_client = True
|
|
586
|
+
|
|
587
|
+
db = _get_pymongo_db_from_string_or_pymongo_db(database, mongo_client)
|
|
588
|
+
collection = db[collection_name]
|
|
589
|
+
|
|
590
|
+
if isinstance(filter_instance, dict):
|
|
591
|
+
collection.delete_one(filter_instance)
|
|
592
|
+
elif isinstance(filter_instance, list):
|
|
593
|
+
for doc in filter_instance:
|
|
594
|
+
collection.delete_one(doc)
|
|
595
|
+
|
|
596
|
+
if close_client:
|
|
597
|
+
mongo_client.close()
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def delete_many(
|
|
601
|
+
filter_query: dict,
|
|
602
|
+
database: Union[str, pymongo.database.Database],
|
|
603
|
+
collection_name: str,
|
|
604
|
+
mongo_client: pymongo.MongoClient = None,
|
|
605
|
+
close_client: bool = False
|
|
606
|
+
):
|
|
607
|
+
"""
|
|
608
|
+
Remove all entries that match the filter query from a MongoDB collection.
|
|
609
|
+
|
|
610
|
+
:param filter_query: dict, the filter query to search for.
|
|
611
|
+
Example, search for all entries with column name 'name' equal to 'John':
|
|
612
|
+
filter_query = {'name': 'John'}
|
|
613
|
+
:param database: String or the database object.
|
|
614
|
+
str - the name of the database. In this case the database object will be created.
|
|
615
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
616
|
+
:param collection_name: str, the name of the collection.
|
|
617
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
618
|
+
If None, a new connection will be created to default URI.
|
|
619
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
620
|
+
|
|
621
|
+
:return: result of the operation.
|
|
622
|
+
"""
|
|
623
|
+
|
|
624
|
+
if not mongo_client:
|
|
625
|
+
mongo_client = connect()
|
|
626
|
+
close_client = True
|
|
627
|
+
|
|
628
|
+
db = _get_pymongo_db_from_string_or_pymongo_db(database, mongo_client)
|
|
629
|
+
collection = db[collection_name]
|
|
630
|
+
|
|
631
|
+
result = collection.delete_many(filter_query)
|
|
632
|
+
|
|
633
|
+
if close_client:
|
|
634
|
+
mongo_client.close()
|
|
635
|
+
|
|
636
|
+
return result
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def create_index(
|
|
640
|
+
database: Union[str, pymongo.database.Database],
|
|
641
|
+
collection_name: str,
|
|
642
|
+
fields_list: list[tuple[str, int]],
|
|
643
|
+
name: str = None,
|
|
644
|
+
mongo_client: pymongo.MongoClient = None,
|
|
645
|
+
close_client: bool = False
|
|
646
|
+
):
|
|
647
|
+
"""
|
|
648
|
+
Create an index in a MongoDB collection.
|
|
649
|
+
:param database: String or the database object.
|
|
650
|
+
str - the name of the database. In this case the database object will be created.
|
|
651
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
652
|
+
:param collection_name: str, the name of the collection.
|
|
653
|
+
:param fields_list: list of tuples, each tuple will contain
|
|
654
|
+
[0] string of the field name and
|
|
655
|
+
[1] the integer value of the order
|
|
656
|
+
to sort by, this is pymongo default, 1 for ascending and -1 for descending.
|
|
657
|
+
Example:
|
|
658
|
+
[
|
|
659
|
+
('vendor', 1),
|
|
660
|
+
('model', -1)
|
|
661
|
+
]
|
|
662
|
+
|
|
663
|
+
Explanation:
|
|
664
|
+
This will create a compound index that will sort the collection by the field 'vendor' in ascending order,
|
|
665
|
+
and then by the field 'model' in descending order.
|
|
666
|
+
:param name: str, the name of the index.
|
|
667
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
668
|
+
If None, a new connection will be created to default URI.
|
|
669
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
670
|
+
|
|
671
|
+
:return: None
|
|
672
|
+
"""
|
|
673
|
+
|
|
674
|
+
if not mongo_client:
|
|
675
|
+
mongo_client = connect()
|
|
676
|
+
close_client = True
|
|
677
|
+
|
|
678
|
+
db = _get_pymongo_db_from_string_or_pymongo_db(database, mongo_client)
|
|
679
|
+
collection = db[collection_name]
|
|
680
|
+
|
|
681
|
+
collection.create_index(fields_list, name=name)
|
|
682
|
+
|
|
683
|
+
if close_client:
|
|
684
|
+
mongo_client.close()
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
def find(
|
|
688
|
+
database: Union[str, pymongo.database.Database],
|
|
689
|
+
collection_name: str,
|
|
690
|
+
filter_query: dict = None,
|
|
691
|
+
projection: dict = None,
|
|
692
|
+
page: int = None,
|
|
693
|
+
items: int = None,
|
|
694
|
+
sort: Union[
|
|
695
|
+
dict[str, Literal[
|
|
696
|
+
'asc', 'desc',
|
|
697
|
+
'ASC', 'DESC',
|
|
698
|
+
1, -1]],
|
|
699
|
+
list[tuple[
|
|
700
|
+
str, Literal[1, -1]]],
|
|
701
|
+
None] = None,
|
|
702
|
+
convert_object_id_to_str: bool = False,
|
|
703
|
+
key_convert_to_dict: list[str] = None,
|
|
704
|
+
mongo_client: pymongo.MongoClient = None,
|
|
705
|
+
close_client: bool = False
|
|
706
|
+
) -> list[dict]:
|
|
707
|
+
"""
|
|
708
|
+
Find entries in a MongoDB collection by query.
|
|
709
|
+
:param database: String or the database object.
|
|
710
|
+
str - the name of the database. In this case the database object will be created.
|
|
711
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
712
|
+
:param collection_name: str, the name of the collection.
|
|
713
|
+
:param filter_query: dict, the query to search for.
|
|
714
|
+
Example, return all entries from collection:
|
|
715
|
+
filter_query = None
|
|
716
|
+
Example, search for all entries with column name 'name' equal to 'John':
|
|
717
|
+
filter_query = {'name': 'John'}
|
|
718
|
+
|
|
719
|
+
Additional parameters to use in the value of the query:
|
|
720
|
+
$regex: Will search for a regex pattern in the field.
|
|
721
|
+
Example for searching for a value that contains 'test':
|
|
722
|
+
filter_query = {'field_name': {'$regex': 'test'}}
|
|
723
|
+
This will return all entries where the field 'field_name' contains the word 'test':
|
|
724
|
+
'test', 'test1', '2test', etc.
|
|
725
|
+
|
|
726
|
+
Example for searching for a value that starts with 'test':
|
|
727
|
+
filter_query = {'field_name': {'$regex': '^test'}}
|
|
728
|
+
|
|
729
|
+
If you need to escape the string for regex special characters you will typically use:
|
|
730
|
+
re.escape(test)
|
|
731
|
+
If you're string contains characters like parentheses "()", you will need to escape them.
|
|
732
|
+
$options: The options for the regex search.
|
|
733
|
+
'i': case-insensitive search.
|
|
734
|
+
Example for case-insensitive search:
|
|
735
|
+
filter_query = {'field_name': {'$regex': 'test', '$options': 'i'}}
|
|
736
|
+
$and: Will search for entries that match all the conditions.
|
|
737
|
+
Example for searching for entries that match all the conditions:
|
|
738
|
+
filter_query = {'$and': [
|
|
739
|
+
{'field_name1': 'value1'},
|
|
740
|
+
{'field_name2': 'value2'}
|
|
741
|
+
]}
|
|
742
|
+
$or: Will search for entries that match at least one of the conditions.
|
|
743
|
+
Example for searching for entries that match at least one of the conditions:
|
|
744
|
+
filter_query = {'$or': [
|
|
745
|
+
{'field_name1': 'value1'},
|
|
746
|
+
{'field_name2': 'value2'}
|
|
747
|
+
]}
|
|
748
|
+
$in: Will search for a value in a list of values.
|
|
749
|
+
Example for searching for a value that is in a list of values:
|
|
750
|
+
filter_query = {'field_name': {'$in': ['value1', 'value2', 'value3']}}
|
|
751
|
+
$nin: Will search for a value not in a list of values.
|
|
752
|
+
Example for searching for a value that is not in a list of values:
|
|
753
|
+
filter_query = {'field_name': {'$nin': ['value1', 'value2', 'value3']}}
|
|
754
|
+
$exists: Will search for entries where the field exists or not.
|
|
755
|
+
Example for searching for entries where the field exists:
|
|
756
|
+
filter_query = {'field_name': {'$exists': True}}
|
|
757
|
+
Example for searching for entries where the field does not exist:
|
|
758
|
+
filter_query = {'field_name': {'$exists': False}}
|
|
759
|
+
$ne: Will search for entries where the field is not equal to the value.
|
|
760
|
+
Example for searching for entries where the field is not equal to the value:
|
|
761
|
+
filter_query = {'field_name': {'$ne': 'value'}}
|
|
762
|
+
|
|
763
|
+
:param projection: dict, the only fields to return or exclude.
|
|
764
|
+
Example, return only the field 'name' and 'age':
|
|
765
|
+
projection = {'name': 1, 'age': 1}
|
|
766
|
+
Example, return all fields except the field 'age':
|
|
767
|
+
projection = {'age': 0}
|
|
768
|
+
Example, return all fields except the field 'age' and 'name':
|
|
769
|
+
projection = {'age': 0, 'name': 0}
|
|
770
|
+
:param page: int, the page number (Optional).
|
|
771
|
+
:param items: int, the number of results per page (Optional).
|
|
772
|
+
:param sort: dict or list of tuples:
|
|
773
|
+
dict, the name of the field and the order to sort the containers by.
|
|
774
|
+
You can use several fields to sort the containers by several fields.
|
|
775
|
+
In this case the containers will be sorted by the first field, then by the second field, etc.
|
|
776
|
+
You can also use only singular field to sort the containers by only one field.
|
|
777
|
+
Usage:
|
|
778
|
+
{
|
|
779
|
+
field_name: order
|
|
780
|
+
}
|
|
781
|
+
Example:
|
|
782
|
+
{
|
|
783
|
+
'vendor': 'asc',
|
|
784
|
+
'model': 'desc'
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
Or example using integers:
|
|
788
|
+
{
|
|
789
|
+
'vendor': 1,
|
|
790
|
+
'model': -1
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
list of tuples, each tuple will contain [0] string of the field name and [1] the integer value of the order
|
|
794
|
+
to sort by, this is pymongo default, 1 for ascending and -1 for descending.
|
|
795
|
+
:param convert_object_id_to_str: bool, if True, the '_id' field will be converted to a string.
|
|
796
|
+
The '_id' field is an ObjectId type, which is a complex object, it can be converted to a string for simpler
|
|
797
|
+
processing.
|
|
798
|
+
:param key_convert_to_dict: list, the keys of the documents that should be converted from string to dict.
|
|
799
|
+
Recursively searches for keys in specified list in a nested dictionary in result entries list,
|
|
800
|
+
and converts their values using 'json.loads' if found.
|
|
801
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
802
|
+
If None, a new connection will be created to default URI.
|
|
803
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
804
|
+
|
|
805
|
+
:return: list of dictionaries, the list of entries that match the query.
|
|
806
|
+
"""
|
|
807
|
+
|
|
808
|
+
if page and not items:
|
|
809
|
+
raise ValueError("If 'page' is provided, 'items' must be provided as well.")
|
|
810
|
+
elif items and not page:
|
|
811
|
+
page = 1
|
|
812
|
+
|
|
813
|
+
if sort and isinstance(sort, dict):
|
|
814
|
+
for key_to_sort_by, order in sort.items():
|
|
815
|
+
if order.lower() not in ['asc', 'desc', 1, -1]:
|
|
816
|
+
raise ValueError("The order must be 'asc', 'desc', 1 or -1.")
|
|
817
|
+
|
|
818
|
+
if not mongo_client:
|
|
819
|
+
mongo_client = connect()
|
|
820
|
+
close_client = True
|
|
821
|
+
|
|
822
|
+
db = _get_pymongo_db_from_string_or_pymongo_db(database, mongo_client)
|
|
823
|
+
collection = db[collection_name]
|
|
824
|
+
|
|
825
|
+
if filter_query is None:
|
|
826
|
+
filter_query = {}
|
|
827
|
+
|
|
828
|
+
# 'skip_items' can be 0, if we ask for the first page, so we still need to cut the number of items.
|
|
829
|
+
# In this case checking if 'items' is not None is enough.
|
|
830
|
+
if items is None:
|
|
831
|
+
items = 0
|
|
832
|
+
|
|
833
|
+
# Calculate the number of documents to skip
|
|
834
|
+
skip_items = 0
|
|
835
|
+
if page and items:
|
|
836
|
+
skip_items = (page - 1) * items
|
|
837
|
+
|
|
838
|
+
# noinspection PyTypeChecker
|
|
839
|
+
sorting_list_of_tuples: list[tuple[str, int]] = None
|
|
840
|
+
if sort:
|
|
841
|
+
sorting_list_of_tuples = []
|
|
842
|
+
if isinstance(sort, dict):
|
|
843
|
+
for key_to_sort_by, order in sort.items():
|
|
844
|
+
if order.lower() == 'asc':
|
|
845
|
+
order = pymongo.ASCENDING
|
|
846
|
+
elif order.lower() == 'desc':
|
|
847
|
+
order = pymongo.DESCENDING
|
|
848
|
+
|
|
849
|
+
sorting_list_of_tuples.append((key_to_sort_by, order))
|
|
850
|
+
elif sort and isinstance(sort, list):
|
|
851
|
+
sorting_list_of_tuples = sort
|
|
852
|
+
|
|
853
|
+
# collection_items = collection_items.sort(sorting_list_of_tuples)
|
|
854
|
+
collection_items = collection.find(
|
|
855
|
+
filter_query, projection=projection, sort=sorting_list_of_tuples, skip=skip_items, limit=items)
|
|
856
|
+
|
|
857
|
+
# # 'skip_items' can be 0, if we ask for the first page, so we still need to cut the number of items.
|
|
858
|
+
# # In this case checking if 'items' is not None is enough.
|
|
859
|
+
# if items:
|
|
860
|
+
# collection_items = collection_items.skip(skip_items).limit(items)
|
|
861
|
+
|
|
862
|
+
# List consolidates the results into a list of dictionaries, collection_items cursor will not be available after this.
|
|
863
|
+
entries: list[dict] = list(collection_items)
|
|
864
|
+
|
|
865
|
+
if entries and convert_object_id_to_str and '_id' in entries[0]:
|
|
866
|
+
for entry_index, entry in enumerate(entries):
|
|
867
|
+
entries[entry_index]['_id'] = str(entry['_id'])
|
|
868
|
+
|
|
869
|
+
if key_convert_to_dict and entries:
|
|
870
|
+
entries = convert_key_values_to_objects(keys_convert_to_dict=key_convert_to_dict, returned_data=entries)
|
|
871
|
+
|
|
872
|
+
if close_client:
|
|
873
|
+
mongo_client.close()
|
|
874
|
+
|
|
875
|
+
return entries
|
|
876
|
+
|
|
877
|
+
|
|
878
|
+
def distinct(
|
|
879
|
+
database: Union[str, pymongo.database.Database],
|
|
880
|
+
collection_name: str,
|
|
881
|
+
field_name: str,
|
|
882
|
+
filter_query: dict = None,
|
|
883
|
+
mongo_client: pymongo.MongoClient = None,
|
|
884
|
+
close_client: bool = False
|
|
885
|
+
) -> list:
|
|
886
|
+
"""
|
|
887
|
+
Get distinct values of a field from a MongoDB collection.
|
|
888
|
+
Example:
|
|
889
|
+
Example database:
|
|
890
|
+
{
|
|
891
|
+
'users': [
|
|
892
|
+
{'name': 'John', 'age': 25},
|
|
893
|
+
{'name': 'John', 'age': 30},
|
|
894
|
+
{'name': 'Alice', 'age': 25}
|
|
895
|
+
]
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
Get distinct values of the field 'name' from the collection 'users':
|
|
899
|
+
distinct('my_db', 'users', 'name')
|
|
900
|
+
|
|
901
|
+
Output:
|
|
902
|
+
['John', 'Alice']
|
|
903
|
+
|
|
904
|
+
:param database: String or the database object.
|
|
905
|
+
str - the name of the database. In this case the database object will be created.
|
|
906
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
907
|
+
:param collection_name: str, the name of the collection.
|
|
908
|
+
:param field_name: str, the name of the field.
|
|
909
|
+
:param filter_query: dict, the filter query to search for.
|
|
910
|
+
If None, the filter query will not be executed.
|
|
911
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
912
|
+
If None, a new connection will be created to default URI.
|
|
913
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
914
|
+
|
|
915
|
+
:return: list, the list of distinct values.
|
|
916
|
+
"""
|
|
917
|
+
|
|
918
|
+
if not mongo_client:
|
|
919
|
+
mongo_client = connect()
|
|
920
|
+
close_client = True
|
|
921
|
+
|
|
922
|
+
db = _get_pymongo_db_from_string_or_pymongo_db(database, mongo_client)
|
|
923
|
+
collection = db[collection_name]
|
|
924
|
+
|
|
925
|
+
distinct_values = collection.distinct(field_name, filter_query)
|
|
926
|
+
|
|
927
|
+
if close_client:
|
|
928
|
+
mongo_client.close()
|
|
929
|
+
|
|
930
|
+
return distinct_values
|
|
931
|
+
|
|
932
|
+
|
|
933
|
+
def update(
|
|
934
|
+
database: Union[str, pymongo.database.Database],
|
|
935
|
+
collection_name: str,
|
|
936
|
+
filter_query: dict,
|
|
937
|
+
update_instance: Union[dict, list[dict]],
|
|
938
|
+
add_timestamp: bool = False,
|
|
939
|
+
convert_mixed_lists_to_strings: bool = False,
|
|
940
|
+
mongo_client: pymongo.MongoClient = None,
|
|
941
|
+
close_client: bool = False
|
|
942
|
+
):
|
|
943
|
+
"""
|
|
944
|
+
Update one entry in a MongoDB collection by filter query.
|
|
945
|
+
:param database: String or the database object.
|
|
946
|
+
str - the name of the database. In this case the database object will be created.
|
|
947
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
948
|
+
:param collection_name: str, the name of the collection.
|
|
949
|
+
:param filter_query: dict, the filter query to search for.
|
|
950
|
+
Example, search for all entries with column name 'name' equal to 'John':
|
|
951
|
+
filter_query = {'name': 'John'}
|
|
952
|
+
Find by Object id:
|
|
953
|
+
filter_query = {'_id': ObjectId('5f3e3b3b4b9f3b3b4b9f3b3b')}
|
|
954
|
+
:param update_instance: dict or list of dicts, the update to apply.
|
|
955
|
+
If dict, the update will be applied to one entry using 'update_one'.
|
|
956
|
+
If list of dicts, the update will be applied to multiple entries using 'update_many'.
|
|
957
|
+
|
|
958
|
+
Examples for operators for each dict:
|
|
959
|
+
$set: update the column 'name' to 'Alice':
|
|
960
|
+
update_instance = {'$set': {'name': 'Alice'}}
|
|
961
|
+
$inc: increment the column 'age' by 1:
|
|
962
|
+
update_instance = {'$inc': {'age': 1}}
|
|
963
|
+
$unset: remove the column 'name':
|
|
964
|
+
update_instance = {'$unset': {'name': ''}}
|
|
965
|
+
$push: add a value to the list 'hobbies':
|
|
966
|
+
update_instance = {'$push': {'hobbies': 'swimming'}}
|
|
967
|
+
$pull: remove a value from the list 'hobbies':
|
|
968
|
+
update_instance = {'$pull': {'hobbies': 'swimming'}}
|
|
969
|
+
:param add_timestamp: bool, if True, a current time timestamp will be added to the object.
|
|
970
|
+
:param convert_mixed_lists_to_strings: bool, if True, mixed lists or tuples when entries are
|
|
971
|
+
strings and integers, the integers will be converted to strings.
|
|
972
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
973
|
+
If None, a new connection will be created to default URI.
|
|
974
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
975
|
+
|
|
976
|
+
:return: None
|
|
977
|
+
"""
|
|
978
|
+
|
|
979
|
+
if not mongo_client:
|
|
980
|
+
mongo_client = connect()
|
|
981
|
+
close_client = True
|
|
982
|
+
|
|
983
|
+
db = _get_pymongo_db_from_string_or_pymongo_db(database, mongo_client)
|
|
984
|
+
collection = db[collection_name]
|
|
985
|
+
|
|
986
|
+
if convert_mixed_lists_to_strings:
|
|
987
|
+
if isinstance(update_instance, dict):
|
|
988
|
+
update_instance = dicts.convert_int_to_str_in_mixed_lists(update_instance)
|
|
989
|
+
elif isinstance(update_instance, list):
|
|
990
|
+
for doc_index, doc in enumerate(update_instance):
|
|
991
|
+
update_instance[doc_index] = dicts.convert_int_to_str_in_mixed_lists(doc)
|
|
992
|
+
|
|
993
|
+
if add_timestamp:
|
|
994
|
+
timestamp = datetime.datetime.now()
|
|
995
|
+
if isinstance(update_instance, dict):
|
|
996
|
+
update_instance['timestamp'] = timestamp
|
|
997
|
+
elif isinstance(update_instance, list):
|
|
998
|
+
for doc in update_instance:
|
|
999
|
+
doc['timestamp'] = timestamp
|
|
1000
|
+
|
|
1001
|
+
result = None
|
|
1002
|
+
if isinstance(update_instance, dict):
|
|
1003
|
+
result = collection.update_one(filter_query, update_instance)
|
|
1004
|
+
elif isinstance(update_instance, list):
|
|
1005
|
+
result = collection.update_many(filter_query, update_instance)
|
|
1006
|
+
|
|
1007
|
+
if result.matched_count == 0:
|
|
1008
|
+
raise MongoDBUpdateOneError("No document found to update.")
|
|
1009
|
+
|
|
1010
|
+
if close_client:
|
|
1011
|
+
mongo_client.close()
|
|
1012
|
+
|
|
1013
|
+
return result
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
def replace(
|
|
1017
|
+
database: Union[str, pymongo.database.Database],
|
|
1018
|
+
collection_name: str,
|
|
1019
|
+
filter_query: dict,
|
|
1020
|
+
replacement: dict,
|
|
1021
|
+
add_timestamp: bool = False,
|
|
1022
|
+
convert_mixed_lists_to_strings: bool = False,
|
|
1023
|
+
mongo_client: pymongo.MongoClient = None,
|
|
1024
|
+
close_client: bool = False
|
|
1025
|
+
):
|
|
1026
|
+
"""
|
|
1027
|
+
Replace one entry in a MongoDB collection by filter query.
|
|
1028
|
+
:param database: String or the database object.
|
|
1029
|
+
str - the name of the database. In this case the database object will be created.
|
|
1030
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
1031
|
+
:param collection_name: str, the name of the collection.
|
|
1032
|
+
:param filter_query: dict, the filter query to search for.
|
|
1033
|
+
Example, search for all entries with column name 'name' equal to 'John':
|
|
1034
|
+
filter_query = {'name': 'John'}
|
|
1035
|
+
Find by Object id:
|
|
1036
|
+
filter_query = {'_id': ObjectId('5f3e3b3b4b9f3b3b4b9f3b3b')}
|
|
1037
|
+
:param replacement: dict, the replacement to apply.
|
|
1038
|
+
:param add_timestamp: bool, if True, a current time timestamp will be added to the object.
|
|
1039
|
+
:param convert_mixed_lists_to_strings: bool, if True, mixed lists or tuples when entries are strings and integers,
|
|
1040
|
+
the integers will be converted to strings.
|
|
1041
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
1042
|
+
If None, a new connection will be created to default URI.
|
|
1043
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
1044
|
+
|
|
1045
|
+
:return: None
|
|
1046
|
+
"""
|
|
1047
|
+
|
|
1048
|
+
if not mongo_client:
|
|
1049
|
+
mongo_client = connect()
|
|
1050
|
+
close_client = True
|
|
1051
|
+
|
|
1052
|
+
db = _get_pymongo_db_from_string_or_pymongo_db(database, mongo_client)
|
|
1053
|
+
collection = db[collection_name]
|
|
1054
|
+
|
|
1055
|
+
if convert_mixed_lists_to_strings:
|
|
1056
|
+
replacement = dicts.convert_int_to_str_in_mixed_lists(replacement)
|
|
1057
|
+
|
|
1058
|
+
if add_timestamp:
|
|
1059
|
+
timestamp = datetime.datetime.now()
|
|
1060
|
+
replacement['timestamp'] = timestamp
|
|
1061
|
+
|
|
1062
|
+
result = collection.replace_one(filter_query, replacement)
|
|
1063
|
+
if result.matched_count == 0:
|
|
1064
|
+
raise MongoDBReplaceOneError("No document found to replace.")
|
|
1065
|
+
|
|
1066
|
+
if close_client:
|
|
1067
|
+
mongo_client.close()
|
|
1068
|
+
|
|
1069
|
+
return result
|
|
1070
|
+
|
|
1071
|
+
|
|
1072
|
+
def get_all_indexes_in_collection(
|
|
1073
|
+
database: Union[str, pymongo.database.Database],
|
|
1074
|
+
collection_name: str,
|
|
1075
|
+
mongo_client: pymongo.MongoClient = None,
|
|
1076
|
+
close_client: bool = False
|
|
1077
|
+
) -> dict:
|
|
1078
|
+
"""
|
|
1079
|
+
Get all indexes in a MongoDB collection.
|
|
1080
|
+
:param database: String or the database object.
|
|
1081
|
+
str - the name of the database. In this case the database object will be created.
|
|
1082
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
1083
|
+
:param collection_name: str, the name of the collection.
|
|
1084
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
1085
|
+
If None, a new connection will be created to default URI.
|
|
1086
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
1087
|
+
|
|
1088
|
+
:return: list, the list of indexes.
|
|
1089
|
+
"""
|
|
1090
|
+
|
|
1091
|
+
if not mongo_client:
|
|
1092
|
+
mongo_client = connect()
|
|
1093
|
+
close_client = True
|
|
1094
|
+
|
|
1095
|
+
db = _get_pymongo_db_from_string_or_pymongo_db(database, mongo_client)
|
|
1096
|
+
collection = db[collection_name]
|
|
1097
|
+
|
|
1098
|
+
# noinspection PyTypeChecker
|
|
1099
|
+
indexes: dict = collection.index_information()
|
|
1100
|
+
|
|
1101
|
+
if close_client:
|
|
1102
|
+
mongo_client.close()
|
|
1103
|
+
|
|
1104
|
+
return indexes
|
|
1105
|
+
|
|
1106
|
+
|
|
1107
|
+
def is_index_name_in_collection(
|
|
1108
|
+
database: Union[str, pymongo.database.Database],
|
|
1109
|
+
collection_name: str,
|
|
1110
|
+
index_name: str,
|
|
1111
|
+
mongo_client: pymongo.MongoClient = None,
|
|
1112
|
+
close_client: bool = False
|
|
1113
|
+
) -> bool:
|
|
1114
|
+
"""
|
|
1115
|
+
Check if an index name is in a MongoDB collection.
|
|
1116
|
+
:param database: String or the database object.
|
|
1117
|
+
str - the name of the database. In this case the database object will be created.
|
|
1118
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
1119
|
+
:param collection_name: str, the name of the collection.
|
|
1120
|
+
:param index_name: str, the name of the index.
|
|
1121
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
1122
|
+
If None, a new connection will be created to default URI.
|
|
1123
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
1124
|
+
|
|
1125
|
+
:return: bool, if the index name is in the collection.
|
|
1126
|
+
"""
|
|
1127
|
+
|
|
1128
|
+
indexes = get_all_indexes_in_collection(
|
|
1129
|
+
database=database, collection_name=collection_name,
|
|
1130
|
+
mongo_client=mongo_client, close_client=close_client)
|
|
1131
|
+
|
|
1132
|
+
return index_name in indexes
|
|
1133
|
+
|
|
1134
|
+
|
|
1135
|
+
def count_entries_in_collection(
|
|
1136
|
+
database: Union[str, pymongo.database.Database],
|
|
1137
|
+
collection_name: str,
|
|
1138
|
+
filter_query: dict = None,
|
|
1139
|
+
mongo_client: pymongo.MongoClient = None,
|
|
1140
|
+
close_client: bool = False
|
|
1141
|
+
) -> int:
|
|
1142
|
+
"""
|
|
1143
|
+
Count entries in a MongoDB collection by query.
|
|
1144
|
+
|
|
1145
|
+
:param database: String or the database object.
|
|
1146
|
+
str - the name of the database. In this case the database object will be created.
|
|
1147
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
1148
|
+
:param collection_name: str, the name of the collection.
|
|
1149
|
+
:param filter_query: dict, the query to search for.
|
|
1150
|
+
Example, search for all entries with column name 'name' equal to 'John':
|
|
1151
|
+
filter_query = {'name': 'John'}
|
|
1152
|
+
Example, return all entries from collection:
|
|
1153
|
+
filter_query = None
|
|
1154
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
1155
|
+
If None, a new connection will be created to default URI.
|
|
1156
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
1157
|
+
|
|
1158
|
+
:return: int, the number of entries that match the query.
|
|
1159
|
+
"""
|
|
1160
|
+
|
|
1161
|
+
if not mongo_client:
|
|
1162
|
+
mongo_client = connect()
|
|
1163
|
+
close_client = True
|
|
1164
|
+
|
|
1165
|
+
db = _get_pymongo_db_from_string_or_pymongo_db(database, mongo_client)
|
|
1166
|
+
collection = db[collection_name]
|
|
1167
|
+
|
|
1168
|
+
if filter_query is None:
|
|
1169
|
+
filter_query = {}
|
|
1170
|
+
|
|
1171
|
+
count = collection.count_documents(filter_query)
|
|
1172
|
+
|
|
1173
|
+
if close_client:
|
|
1174
|
+
mongo_client.close()
|
|
1175
|
+
|
|
1176
|
+
return count
|
|
1177
|
+
|
|
1178
|
+
|
|
1179
|
+
def aggregate_entries_in_collection(
|
|
1180
|
+
database: Union[str, pymongo.database.Database],
|
|
1181
|
+
collection_name: str,
|
|
1182
|
+
pipeline: list,
|
|
1183
|
+
mongo_client: pymongo.MongoClient = None,
|
|
1184
|
+
close_client: bool = False
|
|
1185
|
+
) -> list:
|
|
1186
|
+
"""
|
|
1187
|
+
Perform an aggregation pipeline operation on a MongoDB collection.
|
|
1188
|
+
For example, we count the number of entries with the same 'sha256' value that is provided in a list:
|
|
1189
|
+
pipeline = [
|
|
1190
|
+
{"$match": {"sha256": {"$in": ["hash1", "hash2"]}}},
|
|
1191
|
+
{"$group": {"_id": "$sha256", "count": {"$sum": 1}}}
|
|
1192
|
+
]
|
|
1193
|
+
And we will get the result:
|
|
1194
|
+
[
|
|
1195
|
+
{"_id": "hash1", "count": 1},
|
|
1196
|
+
{"_id": "hash2", "count": 1}
|
|
1197
|
+
]
|
|
1198
|
+
Meaning we will get separate counts for each 'sha256' value in the list.
|
|
1199
|
+
|
|
1200
|
+
:param database: String or the database object.
|
|
1201
|
+
str - the name of the database. In this case the database object will be created.
|
|
1202
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
1203
|
+
:param collection_name: str, the name of the collection.
|
|
1204
|
+
:param pipeline: list, the aggregation pipeline to execute.
|
|
1205
|
+
Example:
|
|
1206
|
+
pipeline = [
|
|
1207
|
+
{"$match": {"sha256": {"$in": ["hash1", "hash2"]}}},
|
|
1208
|
+
{"$group": {"_id": "$sha256", "count": {"$sum": 1}}}
|
|
1209
|
+
]
|
|
1210
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
1211
|
+
If None, a new connection will be created to default URI.
|
|
1212
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
1213
|
+
|
|
1214
|
+
:return: list, the results of the aggregation pipeline.
|
|
1215
|
+
"""
|
|
1216
|
+
if not mongo_client:
|
|
1217
|
+
mongo_client = connect()
|
|
1218
|
+
close_client = True
|
|
1219
|
+
|
|
1220
|
+
db = _get_pymongo_db_from_string_or_pymongo_db(database, mongo_client)
|
|
1221
|
+
collection = db[collection_name]
|
|
1222
|
+
|
|
1223
|
+
# Perform aggregation
|
|
1224
|
+
results = collection.aggregate(pipeline)
|
|
1225
|
+
|
|
1226
|
+
if close_client:
|
|
1227
|
+
mongo_client.close()
|
|
1228
|
+
|
|
1229
|
+
# Return the results as a list
|
|
1230
|
+
return list(results)
|
|
1231
|
+
|
|
1232
|
+
|
|
1233
|
+
def delete_all_entries_from_collection(
|
|
1234
|
+
database: Union[str, pymongo.database.Database],
|
|
1235
|
+
collection_name: str,
|
|
1236
|
+
mongo_client: pymongo.MongoClient = None,
|
|
1237
|
+
close_client: bool = False
|
|
1238
|
+
):
|
|
1239
|
+
"""
|
|
1240
|
+
Remove all entries from a MongoDB collection.
|
|
1241
|
+
:param database: String or the database object.
|
|
1242
|
+
str - the name of the database. In this case the database object will be created.
|
|
1243
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
1244
|
+
:param collection_name: str, the name of the collection.
|
|
1245
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
1246
|
+
If None, a new connection will be created to default URI.
|
|
1247
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
1248
|
+
|
|
1249
|
+
:return: None
|
|
1250
|
+
"""
|
|
1251
|
+
|
|
1252
|
+
if not mongo_client:
|
|
1253
|
+
mongo_client = connect()
|
|
1254
|
+
close_client = True
|
|
1255
|
+
|
|
1256
|
+
db = _get_pymongo_db_from_string_or_pymongo_db(database, mongo_client)
|
|
1257
|
+
collection = db[collection_name]
|
|
1258
|
+
|
|
1259
|
+
collection.delete_many({})
|
|
1260
|
+
|
|
1261
|
+
if close_client:
|
|
1262
|
+
mongo_client.close()
|
|
1263
|
+
|
|
1264
|
+
|
|
1265
|
+
def overwrite_collection(
|
|
1266
|
+
object_instance: list,
|
|
1267
|
+
database: Union[str, pymongo.database.Database],
|
|
1268
|
+
collection_name: str,
|
|
1269
|
+
add_timestamp: bool = False,
|
|
1270
|
+
convert_mixed_lists_to_strings: bool = False,
|
|
1271
|
+
mongo_client: pymongo.MongoClient = None,
|
|
1272
|
+
close_client: bool = False
|
|
1273
|
+
):
|
|
1274
|
+
"""
|
|
1275
|
+
Overwrite a MongoDB collection with list of dicts or a dict.
|
|
1276
|
+
:param object_instance: list of dictionaries, the list of dictionaries to overwrite in the collection.
|
|
1277
|
+
:param database: String or the database object.
|
|
1278
|
+
str - the name of the database. In this case the database object will be created.
|
|
1279
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
1280
|
+
:param collection_name: str, the name of the collection.
|
|
1281
|
+
:param add_timestamp: bool, if True, a current time timestamp will be added to the object.
|
|
1282
|
+
:param convert_mixed_lists_to_strings: bool, if True, mixed lists or tuples when entries are strings and integers,
|
|
1283
|
+
the integers will be converted to strings.
|
|
1284
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
1285
|
+
If None, a new connection will be created to default URI.
|
|
1286
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
1287
|
+
|
|
1288
|
+
:return: None
|
|
1289
|
+
"""
|
|
1290
|
+
|
|
1291
|
+
_is_object_list_of_dicts_or_dict(object_instance)
|
|
1292
|
+
|
|
1293
|
+
if not mongo_client:
|
|
1294
|
+
mongo_client = connect()
|
|
1295
|
+
close_client = True
|
|
1296
|
+
|
|
1297
|
+
delete_all_entries_from_collection(
|
|
1298
|
+
database=database, collection_name=collection_name,
|
|
1299
|
+
mongo_client=mongo_client
|
|
1300
|
+
)
|
|
1301
|
+
|
|
1302
|
+
insert(
|
|
1303
|
+
object_instance=object_instance,
|
|
1304
|
+
database=database, collection_name=collection_name,
|
|
1305
|
+
add_timestamp=add_timestamp, convert_mixed_lists_to_strings=convert_mixed_lists_to_strings,
|
|
1306
|
+
mongo_client=mongo_client, close_client=close_client)
|
|
1307
|
+
|
|
1308
|
+
|
|
1309
|
+
def _is_object_list_of_dicts_or_dict(
|
|
1310
|
+
object_instance: Union[list[dict], dict]
|
|
1311
|
+
):
|
|
1312
|
+
if isinstance(object_instance, dict):
|
|
1313
|
+
return True
|
|
1314
|
+
elif isinstance(object_instance, list):
|
|
1315
|
+
if object_instance and isinstance(object_instance[0], dict):
|
|
1316
|
+
return True
|
|
1317
|
+
else:
|
|
1318
|
+
raise ValueError("List must contain dictionaries.")
|
|
1319
|
+
else:
|
|
1320
|
+
raise ValueError("Object must be a dictionary or a list of dictionaries.")
|
|
1321
|
+
|
|
1322
|
+
|
|
1323
|
+
def _get_pymongo_db_from_string_or_pymongo_db(
|
|
1324
|
+
database: Union[str, pymongo.database.Database],
|
|
1325
|
+
mongo_client: pymongo.MongoClient
|
|
1326
|
+
) -> pymongo.database.Database:
|
|
1327
|
+
"""
|
|
1328
|
+
Get a pymongo.database.Database object from a string or a pymongo.database.Database object.
|
|
1329
|
+
|
|
1330
|
+
:param database: Union[str, pymongo.database.Database], the database object or the name of the database.
|
|
1331
|
+
If the database is a string, the database object will be created.
|
|
1332
|
+
If the database is a pymongo.database.Database object, it will be returned as is.
|
|
1333
|
+
:param mongo_client: mongodb.MongoClient, the connection object.
|
|
1334
|
+
:return: pymongo.database.Database, the database object.
|
|
1335
|
+
"""
|
|
1336
|
+
|
|
1337
|
+
if isinstance(database, str):
|
|
1338
|
+
return mongo_client[database]
|
|
1339
|
+
elif isinstance(database, pymongo.database.Database):
|
|
1340
|
+
return database
|
|
1341
|
+
else:
|
|
1342
|
+
raise ValueError("Database must be a string (database name) or a pymongo.database.Database object.")
|
|
1343
|
+
|
|
1344
|
+
|
|
1345
|
+
def get_stats_db(
|
|
1346
|
+
database: Union[str, pymongo.database.Database],
|
|
1347
|
+
mongo_client: pymongo.MongoClient = None,
|
|
1348
|
+
close_client: bool = False
|
|
1349
|
+
):
|
|
1350
|
+
"""
|
|
1351
|
+
Get the stats of a MongoDB database.
|
|
1352
|
+
|
|
1353
|
+
:param database: String or the database object.
|
|
1354
|
+
str - the name of the database. In this case the database object will be created.
|
|
1355
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
1356
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
1357
|
+
If None, a new connection will be created to default URI.
|
|
1358
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
1359
|
+
|
|
1360
|
+
:return: dict, the stats of the collection.
|
|
1361
|
+
"""
|
|
1362
|
+
|
|
1363
|
+
if not mongo_client:
|
|
1364
|
+
mongo_client = connect()
|
|
1365
|
+
close_client = True
|
|
1366
|
+
|
|
1367
|
+
db = _get_pymongo_db_from_string_or_pymongo_db(database, mongo_client)
|
|
1368
|
+
|
|
1369
|
+
stats = db.command("dbStats")
|
|
1370
|
+
|
|
1371
|
+
if close_client:
|
|
1372
|
+
mongo_client.close()
|
|
1373
|
+
|
|
1374
|
+
return stats
|
|
1375
|
+
|
|
1376
|
+
|
|
1377
|
+
def get_stats_db_size(
|
|
1378
|
+
database: Union[str, pymongo.database.Database],
|
|
1379
|
+
mongo_client: pymongo.MongoClient = None,
|
|
1380
|
+
close_client: bool = False
|
|
1381
|
+
):
|
|
1382
|
+
"""
|
|
1383
|
+
Get the size of a MongoDB database in bytes.
|
|
1384
|
+
|
|
1385
|
+
:param database: String or the database object.
|
|
1386
|
+
str - the name of the database. In this case the database object will be created.
|
|
1387
|
+
pymongo.database.Database - the database object that will be used instead of creating a new one.
|
|
1388
|
+
:param mongo_client: pymongo.MongoClient, the connection object.
|
|
1389
|
+
If None, a new connection will be created to default URI.
|
|
1390
|
+
:param close_client: bool, if True, the connection will be closed after the operation.
|
|
1391
|
+
|
|
1392
|
+
:return: int, the size of the database in bytes.
|
|
1393
|
+
"""
|
|
1394
|
+
|
|
1395
|
+
stats = get_stats_db(
|
|
1396
|
+
database=database, mongo_client=mongo_client, close_client=close_client)
|
|
1397
|
+
|
|
1398
|
+
return stats['dataSize']
|
|
1399
|
+
|
|
1400
|
+
|
|
1401
|
+
def convert_key_values_to_objects(
|
|
1402
|
+
keys_convert_to_dict: list[str],
|
|
1403
|
+
returned_data: Union[dict, list]
|
|
1404
|
+
) -> Union[dict, list]:
|
|
1405
|
+
"""
|
|
1406
|
+
Recursively searches for provided keys from the 'keys_convert_to_dict' list like 'test1' and 'test2'
|
|
1407
|
+
in a nested dictionary 'returned_data' and converts their values using "json.loads" if found.
|
|
1408
|
+
|
|
1409
|
+
:param keys_convert_to_dict: list, the keys of the documents that should be converted from string to dict.
|
|
1410
|
+
:param returned_data: The nested dictionary to search through.
|
|
1411
|
+
:type returned_data: dict
|
|
1412
|
+
"""
|
|
1413
|
+
|
|
1414
|
+
if isinstance(returned_data, dict):
|
|
1415
|
+
for key, value in returned_data.items():
|
|
1416
|
+
if key in keys_convert_to_dict:
|
|
1417
|
+
# Can be that the value is None, so we don't need to convert it.
|
|
1418
|
+
if value is None:
|
|
1419
|
+
continue
|
|
1420
|
+
|
|
1421
|
+
try:
|
|
1422
|
+
returned_data[key] = json.loads(value)
|
|
1423
|
+
except (ValueError, TypeError):
|
|
1424
|
+
# This is needed only to know the possible exception types.
|
|
1425
|
+
raise
|
|
1426
|
+
else:
|
|
1427
|
+
convert_key_values_to_objects(keys_convert_to_dict, value)
|
|
1428
|
+
elif isinstance(returned_data, list):
|
|
1429
|
+
for i, item in enumerate(returned_data):
|
|
1430
|
+
returned_data[i] = convert_key_values_to_objects(keys_convert_to_dict, item)
|
|
1431
|
+
|
|
1432
|
+
return returned_data
|