datalake-scripts 2.7.3__tar.gz → 2.7.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/PKG-INFO +25 -2
  2. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/README.md +24 -1
  3. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/token_manager.py +13 -2
  4. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/datalake.py +20 -6
  5. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/endpoint.py +25 -3
  6. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/cli.py +1 -1
  7. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/common/base_engine.py +15 -3
  8. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/common/mixins.py +1 -0
  9. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/PKG-INFO +25 -2
  10. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/__init__.py +0 -0
  11. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/api_objects/__init__.py +0 -0
  12. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/api_objects/bulk_search_task.py +0 -0
  13. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/__init__.py +0 -0
  14. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/atom.py +0 -0
  15. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/atom_type.py +0 -0
  16. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/config.py +0 -0
  17. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/logger.py +0 -0
  18. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/output.py +0 -0
  19. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/throttler.py +0 -0
  20. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/utils.py +0 -0
  21. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/common/warn.py +0 -0
  22. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/__init__.py +0 -0
  23. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/advanced_search.py +0 -0
  24. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/bulk_search.py +0 -0
  25. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/filtered_tag_subcategory.py +0 -0
  26. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/sightings.py +0 -0
  27. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/tags.py +0 -0
  28. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/endpoints/threats.py +0 -0
  29. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/miscellaneous/__init__.py +0 -0
  30. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake/miscellaneous/search_watch.py +0 -0
  31. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/__init__.py +0 -0
  32. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/common/__init__.py +0 -0
  33. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/common/base_script.py +0 -0
  34. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/engines/__init__.py +0 -0
  35. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/engines/get_engine.py +0 -0
  36. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/engines/post_engine.py +0 -0
  37. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/helper_scripts/__init__.py +0 -0
  38. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/helper_scripts/output_builder.py +0 -0
  39. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/helper_scripts/utils.py +0 -0
  40. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/__init__.py +0 -0
  41. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/add_comments.py +0 -0
  42. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/add_tags.py +0 -0
  43. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/add_threats.py +0 -0
  44. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/advanced_search.py +0 -0
  45. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
  46. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/edit_score.py +0 -0
  47. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_atom_values.py +0 -0
  48. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_filtered_tag_subcategory.py +0 -0
  49. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_query_hash.py +0 -0
  50. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_threats_by_hashkey.py +0 -0
  51. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
  52. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/lookup_threats.py +0 -0
  53. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts/scripts/search_watch.py +0 -0
  54. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/SOURCES.txt +0 -0
  55. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/dependency_links.txt +0 -0
  56. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/entry_points.txt +0 -0
  57. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/requires.txt +0 -0
  58. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/datalake_scripts.egg-info/top_level.txt +0 -0
  59. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/setup.cfg +0 -0
  60. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/setup.py +0 -0
  61. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/__init__.py +0 -0
  62. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/common/__init__.py +0 -0
  63. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/common/fixture.py +0 -0
  64. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/common/test_output.py +0 -0
  65. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/common/test_utils.py +0 -0
  66. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/conftest.py +0 -0
  67. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/__init__.py +0 -0
  68. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_advanced_search.py +0 -0
  69. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_atom_type.py +0 -0
  70. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_auth_via_tokens.py +0 -0
  71. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_bulk_search.py +0 -0
  72. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_filtered_tag_subcategory.py +0 -0
  73. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_search_watch.py +0 -0
  74. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_sightings.py +0 -0
  75. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_tags.py +0 -0
  76. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/lib/test_threats.py +0 -0
  77. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/scripts/__init__.py +0 -0
  78. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/scripts/test_add_new_threats.py +0 -0
  79. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/scripts/test_base_engine.py +0 -0
  80. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/scripts/test_cli.py +0 -0
  81. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/scripts/test_csv_builder.py +0 -0
  82. {datalake-scripts-2.7.3 → datalake-scripts-2.7.4}/tests/scripts/test_endpoint_config.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: datalake-scripts
3
- Version: 2.7.3
3
+ Version: 2.7.4
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -40,7 +40,19 @@ $ pip install datalake-scripts
40
40
  $ pip3 install datalake-scripts
41
41
  ```
42
42
  ### Using as a library
43
- see [the following link](tutorial.md)
43
+ ```python
44
+ from datalake import Datalake, AtomType, Output
45
+
46
+ dtl = Datalake(username='username', password='password')
47
+ dtl.Threats.lookup(
48
+ atom_value='mayoclinic.org',
49
+ atom_type=AtomType.DOMAIN,
50
+ hashkey_only=False,
51
+ output=Output.JSON
52
+ )
53
+ ```
54
+
55
+ see [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
44
56
 
45
57
  ### Using as a CLI
46
58
 
@@ -72,6 +84,17 @@ or
72
84
  * `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
73
85
  > These last two are independent and one can be used without the other if you wish.
74
86
 
87
+ #### Using a Proxy
88
+
89
+ You can set up following environment variables :
90
+
91
+ * `HTTP_PROXY`
92
+ * `HTTPS_PROXY`
93
+
94
+ We use the format accepted by the requests python library.
95
+ See its documenation for other possible kinds of proxy to set up.
96
+
97
+
75
98
  #### Throttling
76
99
  For throttling the request, those two environment variable can be used:
77
100
  * `OCD_DTL_QUOTA_TIME` define, in seconds, the time before resetting the requests limit, *default is 1 second*.
@@ -20,7 +20,19 @@ $ pip install datalake-scripts
20
20
  $ pip3 install datalake-scripts
21
21
  ```
22
22
  ### Using as a library
23
- see [the following link](tutorial.md)
23
+ ```python
24
+ from datalake import Datalake, AtomType, Output
25
+
26
+ dtl = Datalake(username='username', password='password')
27
+ dtl.Threats.lookup(
28
+ atom_value='mayoclinic.org',
29
+ atom_type=AtomType.DOMAIN,
30
+ hashkey_only=False,
31
+ output=Output.JSON
32
+ )
33
+ ```
34
+
35
+ see [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
24
36
 
25
37
  ### Using as a CLI
26
38
 
@@ -52,6 +64,17 @@ or
52
64
  * `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
53
65
  > These last two are independent and one can be used without the other if you wish.
54
66
 
67
+ #### Using a Proxy
68
+
69
+ You can set up following environment variables :
70
+
71
+ * `HTTP_PROXY`
72
+ * `HTTPS_PROXY`
73
+
74
+ We use the format accepted by the requests python library.
75
+ See its documenation for other possible kinds of proxy to set up.
76
+
77
+
55
78
  #### Throttling
56
79
  For throttling the request, those two environment variable can be used:
57
80
  * `OCD_DTL_QUOTA_TIME` define, in seconds, the time before resetting the requests limit, *default is 1 second*.
@@ -25,6 +25,8 @@ class TokenManager:
25
25
  username=None,
26
26
  password=None,
27
27
  longterm_token=None,
28
+ proxies: dict = None,
29
+ verify: bool = True,
28
30
  ):
29
31
  """environment can be either prod or preprod"""
30
32
  base_url = urljoin(
@@ -40,6 +42,8 @@ class TokenManager:
40
42
  self.username = username
41
43
  self.password = password
42
44
  self.longterm_token = longterm_token
45
+ self.proxies = proxies
46
+ self.verify = verify
43
47
  self.access_token = None
44
48
  self.refresh_token = None
45
49
  self.get_token()
@@ -81,7 +85,9 @@ class TokenManager:
81
85
  else:
82
86
  data = {"email": self.username, "password": self.password}
83
87
 
84
- response = requests.post(url=self.url_token, json=data)
88
+ response = requests.post(
89
+ url=self.url_token, json=data, proxies=self.proxies, verify=self.verify
90
+ )
85
91
  json_response = json.loads(response.text)
86
92
  try:
87
93
  self.access_token = f'Token {json_response["access_token"]}'
@@ -96,7 +102,12 @@ class TokenManager:
96
102
  def fetch_new_token(self):
97
103
  logger.debug("Token will be refreshed")
98
104
  headers = {"Authorization": self.refresh_token}
99
- response = requests.post(url=self.url_refresh, headers=headers)
105
+ response = requests.post(
106
+ url=self.url_refresh,
107
+ headers=headers,
108
+ proxies=self.proxies,
109
+ verify=self.verify,
110
+ )
100
111
 
101
112
  json_response = response.json()
102
113
  if (
@@ -33,6 +33,8 @@ class Datalake:
33
33
  longterm_token: str = None,
34
34
  env="prod",
35
35
  log_level=logging.WARNING,
36
+ proxies: dict = None,
37
+ verify: bool = True,
36
38
  ):
37
39
  configure_logging(log_level)
38
40
  endpoint_config = Config().load_config()
@@ -43,6 +45,8 @@ class Datalake:
43
45
  username=username,
44
46
  password=password,
45
47
  longterm_token=longterm_token,
48
+ proxies=proxies,
49
+ verify=verify,
46
50
  )
47
51
  except Exception as e:
48
52
  if "Failed to resolve" in str(e) or "Failed to establish" in str(e):
@@ -52,14 +56,24 @@ class Datalake:
52
56
  else:
53
57
  raise
54
58
 
55
- self.Threats = Threats(endpoint_config, env, token_manager)
56
- self.BulkSearch = BulkSearch(endpoint_config, env, token_manager)
57
- self.Tags = Tags(endpoint_config, env, token_manager)
59
+ self.Threats = Threats(
60
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
61
+ )
62
+ self.BulkSearch = BulkSearch(
63
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
64
+ )
65
+ self.Tags = Tags(
66
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
67
+ )
58
68
  self.FilteredTagSubcategory = FilteredTagSubcategory(
59
- endpoint_config, env, token_manager
69
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
70
+ )
71
+ self.AdvancedSearch = AdvancedSearch(
72
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
73
+ )
74
+ self.Sightings = Sightings(
75
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
60
76
  )
61
- self.AdvancedSearch = AdvancedSearch(endpoint_config, env, token_manager)
62
- self.Sightings = Sightings(endpoint_config, env, token_manager)
63
77
  self.SearchWatch = SearchWatch(self.BulkSearch)
64
78
 
65
79
  def sightings_filtered_from_atom_value(
@@ -28,13 +28,20 @@ class Endpoint:
28
28
  SET_MAX_RETRY = 3
29
29
 
30
30
  def __init__(
31
- self, endpoint_config: dict, environment: str, token_manager: TokenManager
31
+ self,
32
+ endpoint_config: dict,
33
+ environment: str,
34
+ token_manager: TokenManager,
35
+ proxies: dict = None,
36
+ verify: bool = True,
32
37
  ):
33
38
  self.endpoint_config = endpoint_config
34
39
  self.environment = environment
35
40
  self.terminal_size = self._get_terminal_size()
36
41
  self.token_manager = token_manager
37
42
  self.SET_MAX_RETRY = 3
43
+ self.proxies = proxies
44
+ self.verify = verify
38
45
 
39
46
  @staticmethod
40
47
  def _get_terminal_size() -> int:
@@ -71,7 +78,13 @@ class Endpoint:
71
78
  logger.debug(self._pretty_debug_request(url, method, post_body, headers))
72
79
 
73
80
  response = self._send_request(
74
- url, method, headers, post_body, stream=stream
81
+ url,
82
+ method,
83
+ headers,
84
+ post_body,
85
+ stream=stream,
86
+ proxies=self.proxies,
87
+ verify=self.verify,
75
88
  )
76
89
 
77
90
  if logger.isEnabledFor(logging.DEBUG):
@@ -111,7 +124,13 @@ class Endpoint:
111
124
 
112
125
  @staticmethod
113
126
  def _send_request(
114
- url: str, method: str, headers: dict, data: dict, stream=False
127
+ url: str,
128
+ method: str,
129
+ headers: dict,
130
+ data: dict,
131
+ stream=False,
132
+ proxies: dict = None,
133
+ verify: bool = True,
115
134
  ) -> Response:
116
135
  """
117
136
  Send the correct http request to url from method [get, post, delete, patch, put].
@@ -121,7 +140,10 @@ class Endpoint:
121
140
  "url": url,
122
141
  "headers": headers,
123
142
  "stream": stream,
143
+ "verify": verify,
124
144
  }
145
+ if proxies:
146
+ common_kwargs["proxies"] = proxies
125
147
 
126
148
  if method == "get":
127
149
  api_response = requests.get(**common_kwargs)
@@ -20,7 +20,7 @@ from datalake_scripts.scripts import (
20
20
 
21
21
  class Cli:
22
22
  CLI_NAME = "ocd-dtl"
23
- VERSION = "2.7.3"
23
+ VERSION = "2.7.4"
24
24
 
25
25
  def __init__(self):
26
26
  parser = argparse.ArgumentParser(
@@ -33,14 +33,26 @@ class BaseEngine:
33
33
  Json = Union[dict, list] # json like object that can be a dict or root level array
34
34
 
35
35
  def __init__(
36
- self, endpoint_config: dict, environment: str, token_manager: TokenManager
36
+ self,
37
+ endpoint_config: dict,
38
+ environment: str,
39
+ token_manager: TokenManager,
40
+ proxies: dict = None,
41
+ verify: bool = True,
37
42
  ):
38
43
  self.endpoint_config = endpoint_config
39
44
  self.environment = environment
40
- self.requests_ssl_verify = suppress_insecure_request_warns(environment)
45
+ self.requests_ssl_verify = (
46
+ verify
47
+ if suppress_insecure_request_warns(environment)
48
+ else suppress_insecure_request_warns(environment)
49
+ )
41
50
  self.url = self._build_url(endpoint_config, environment)
42
51
  self.token_manager = token_manager
43
- self.endpoint = Endpoint(endpoint_config, environment, token_manager)
52
+ self.proxies = proxies
53
+ self.endpoint = Endpoint(
54
+ endpoint_config, environment, token_manager, proxies, verify
55
+ )
44
56
 
45
57
  def datalake_requests(
46
58
  self, url: str, method: str, headers: dict, post_body: dict = None
@@ -55,6 +55,7 @@ class HandleBulkTaskMixin(BaseEngine):
55
55
  url=retrieve_bulk_result_url,
56
56
  headers=headers,
57
57
  verify=self.requests_ssl_verify,
58
+ proxies=self.proxies,
58
59
  )
59
60
  if response.status_code == 200:
60
61
  potential_json_response = response.json()
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: datalake-scripts
3
- Version: 2.7.3
3
+ Version: 2.7.4
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -40,7 +40,19 @@ $ pip install datalake-scripts
40
40
  $ pip3 install datalake-scripts
41
41
  ```
42
42
  ### Using as a library
43
- see [the following link](tutorial.md)
43
+ ```python
44
+ from datalake import Datalake, AtomType, Output
45
+
46
+ dtl = Datalake(username='username', password='password')
47
+ dtl.Threats.lookup(
48
+ atom_value='mayoclinic.org',
49
+ atom_type=AtomType.DOMAIN,
50
+ hashkey_only=False,
51
+ output=Output.JSON
52
+ )
53
+ ```
54
+
55
+ see [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
44
56
 
45
57
  ### Using as a CLI
46
58
 
@@ -72,6 +84,17 @@ or
72
84
  * `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
73
85
  > These last two are independent and one can be used without the other if you wish.
74
86
 
87
+ #### Using a Proxy
88
+
89
+ You can set up following environment variables :
90
+
91
+ * `HTTP_PROXY`
92
+ * `HTTPS_PROXY`
93
+
94
+ We use the format accepted by the requests python library.
95
+ See its documenation for other possible kinds of proxy to set up.
96
+
97
+
75
98
  #### Throttling
76
99
  For throttling the request, those two environment variable can be used:
77
100
  * `OCD_DTL_QUOTA_TIME` define, in seconds, the time before resetting the requests limit, *default is 1 second*.