datalake-scripts 2.7.3__py3-none-any.whl → 2.7.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -25,6 +25,8 @@ class TokenManager:
25
25
  username=None,
26
26
  password=None,
27
27
  longterm_token=None,
28
+ proxies: dict = None,
29
+ verify: bool = True,
28
30
  ):
29
31
  """environment can be either prod or preprod"""
30
32
  base_url = urljoin(
@@ -40,6 +42,8 @@ class TokenManager:
40
42
  self.username = username
41
43
  self.password = password
42
44
  self.longterm_token = longterm_token
45
+ self.proxies = proxies
46
+ self.verify = verify
43
47
  self.access_token = None
44
48
  self.refresh_token = None
45
49
  self.get_token()
@@ -81,7 +85,9 @@ class TokenManager:
81
85
  else:
82
86
  data = {"email": self.username, "password": self.password}
83
87
 
84
- response = requests.post(url=self.url_token, json=data)
88
+ response = requests.post(
89
+ url=self.url_token, json=data, proxies=self.proxies, verify=self.verify
90
+ )
85
91
  json_response = json.loads(response.text)
86
92
  try:
87
93
  self.access_token = f'Token {json_response["access_token"]}'
@@ -96,7 +102,12 @@ class TokenManager:
96
102
  def fetch_new_token(self):
97
103
  logger.debug("Token will be refreshed")
98
104
  headers = {"Authorization": self.refresh_token}
99
- response = requests.post(url=self.url_refresh, headers=headers)
105
+ response = requests.post(
106
+ url=self.url_refresh,
107
+ headers=headers,
108
+ proxies=self.proxies,
109
+ verify=self.verify,
110
+ )
100
111
 
101
112
  json_response = response.json()
102
113
  if (
datalake/datalake.py CHANGED
@@ -33,6 +33,8 @@ class Datalake:
33
33
  longterm_token: str = None,
34
34
  env="prod",
35
35
  log_level=logging.WARNING,
36
+ proxies: dict = None,
37
+ verify: bool = True,
36
38
  ):
37
39
  configure_logging(log_level)
38
40
  endpoint_config = Config().load_config()
@@ -43,6 +45,8 @@ class Datalake:
43
45
  username=username,
44
46
  password=password,
45
47
  longterm_token=longterm_token,
48
+ proxies=proxies,
49
+ verify=verify,
46
50
  )
47
51
  except Exception as e:
48
52
  if "Failed to resolve" in str(e) or "Failed to establish" in str(e):
@@ -52,14 +56,24 @@ class Datalake:
52
56
  else:
53
57
  raise
54
58
 
55
- self.Threats = Threats(endpoint_config, env, token_manager)
56
- self.BulkSearch = BulkSearch(endpoint_config, env, token_manager)
57
- self.Tags = Tags(endpoint_config, env, token_manager)
59
+ self.Threats = Threats(
60
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
61
+ )
62
+ self.BulkSearch = BulkSearch(
63
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
64
+ )
65
+ self.Tags = Tags(
66
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
67
+ )
58
68
  self.FilteredTagSubcategory = FilteredTagSubcategory(
59
- endpoint_config, env, token_manager
69
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
70
+ )
71
+ self.AdvancedSearch = AdvancedSearch(
72
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
73
+ )
74
+ self.Sightings = Sightings(
75
+ endpoint_config, env, token_manager, proxies=proxies, verify=verify
60
76
  )
61
- self.AdvancedSearch = AdvancedSearch(endpoint_config, env, token_manager)
62
- self.Sightings = Sightings(endpoint_config, env, token_manager)
63
77
  self.SearchWatch = SearchWatch(self.BulkSearch)
64
78
 
65
79
  def sightings_filtered_from_atom_value(
@@ -28,13 +28,20 @@ class Endpoint:
28
28
  SET_MAX_RETRY = 3
29
29
 
30
30
  def __init__(
31
- self, endpoint_config: dict, environment: str, token_manager: TokenManager
31
+ self,
32
+ endpoint_config: dict,
33
+ environment: str,
34
+ token_manager: TokenManager,
35
+ proxies: dict = None,
36
+ verify: bool = True,
32
37
  ):
33
38
  self.endpoint_config = endpoint_config
34
39
  self.environment = environment
35
40
  self.terminal_size = self._get_terminal_size()
36
41
  self.token_manager = token_manager
37
42
  self.SET_MAX_RETRY = 3
43
+ self.proxies = proxies
44
+ self.verify = verify
38
45
 
39
46
  @staticmethod
40
47
  def _get_terminal_size() -> int:
@@ -71,7 +78,13 @@ class Endpoint:
71
78
  logger.debug(self._pretty_debug_request(url, method, post_body, headers))
72
79
 
73
80
  response = self._send_request(
74
- url, method, headers, post_body, stream=stream
81
+ url,
82
+ method,
83
+ headers,
84
+ post_body,
85
+ stream=stream,
86
+ proxies=self.proxies,
87
+ verify=self.verify,
75
88
  )
76
89
 
77
90
  if logger.isEnabledFor(logging.DEBUG):
@@ -111,7 +124,13 @@ class Endpoint:
111
124
 
112
125
  @staticmethod
113
126
  def _send_request(
114
- url: str, method: str, headers: dict, data: dict, stream=False
127
+ url: str,
128
+ method: str,
129
+ headers: dict,
130
+ data: dict,
131
+ stream=False,
132
+ proxies: dict = None,
133
+ verify: bool = True,
115
134
  ) -> Response:
116
135
  """
117
136
  Send the correct http request to url from method [get, post, delete, patch, put].
@@ -121,7 +140,10 @@ class Endpoint:
121
140
  "url": url,
122
141
  "headers": headers,
123
142
  "stream": stream,
143
+ "verify": verify,
124
144
  }
145
+ if proxies:
146
+ common_kwargs["proxies"] = proxies
125
147
 
126
148
  if method == "get":
127
149
  api_response = requests.get(**common_kwargs)
datalake_scripts/cli.py CHANGED
@@ -20,7 +20,7 @@ from datalake_scripts.scripts import (
20
20
 
21
21
  class Cli:
22
22
  CLI_NAME = "ocd-dtl"
23
- VERSION = "2.7.3"
23
+ VERSION = "2.7.4"
24
24
 
25
25
  def __init__(self):
26
26
  parser = argparse.ArgumentParser(
@@ -33,14 +33,26 @@ class BaseEngine:
33
33
  Json = Union[dict, list] # json like object that can be a dict or root level array
34
34
 
35
35
  def __init__(
36
- self, endpoint_config: dict, environment: str, token_manager: TokenManager
36
+ self,
37
+ endpoint_config: dict,
38
+ environment: str,
39
+ token_manager: TokenManager,
40
+ proxies: dict = None,
41
+ verify: bool = True,
37
42
  ):
38
43
  self.endpoint_config = endpoint_config
39
44
  self.environment = environment
40
- self.requests_ssl_verify = suppress_insecure_request_warns(environment)
45
+ self.requests_ssl_verify = (
46
+ verify
47
+ if suppress_insecure_request_warns(environment)
48
+ else suppress_insecure_request_warns(environment)
49
+ )
41
50
  self.url = self._build_url(endpoint_config, environment)
42
51
  self.token_manager = token_manager
43
- self.endpoint = Endpoint(endpoint_config, environment, token_manager)
52
+ self.proxies = proxies
53
+ self.endpoint = Endpoint(
54
+ endpoint_config, environment, token_manager, proxies, verify
55
+ )
44
56
 
45
57
  def datalake_requests(
46
58
  self, url: str, method: str, headers: dict, post_body: dict = None
@@ -55,6 +55,7 @@ class HandleBulkTaskMixin(BaseEngine):
55
55
  url=retrieve_bulk_result_url,
56
56
  headers=headers,
57
57
  verify=self.requests_ssl_verify,
58
+ proxies=self.proxies,
58
59
  )
59
60
  if response.status_code == 200:
60
61
  potential_json_response = response.json()
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: datalake-scripts
3
- Version: 2.7.3
3
+ Version: 2.7.4
4
4
  Summary: A collection of scripts to easily use the API of OCD Datalake
5
5
  Home-page: https://github.com/cert-orangecyberdefense/datalake/
6
6
  Author: OCD
@@ -43,7 +43,19 @@ $ pip install datalake-scripts
43
43
  $ pip3 install datalake-scripts
44
44
  ```
45
45
  ### Using as a library
46
- see [the following link](tutorial.md)
46
+ ```python
47
+ from datalake import Datalake, AtomType, Output
48
+
49
+ dtl = Datalake(username='username', password='password')
50
+ dtl.Threats.lookup(
51
+ atom_value='mayoclinic.org',
52
+ atom_type=AtomType.DOMAIN,
53
+ hashkey_only=False,
54
+ output=Output.JSON
55
+ )
56
+ ```
57
+
58
+ see [the following link](https://github.com/cert-orangecyberdefense/datalake/blob/master/tutorial.md)
47
59
 
48
60
  ### Using as a CLI
49
61
 
@@ -75,6 +87,17 @@ or
75
87
  * `OCD_DTL_PASSWORD` password used to login on Datalake API/GUI.
76
88
  > These last two are independent and one can be used without the other if you wish.
77
89
 
90
+ #### Using a Proxy
91
+
92
+ You can set up following environment variables :
93
+
94
+ * `HTTP_PROXY`
95
+ * `HTTPS_PROXY`
96
+
97
+ We use the format accepted by the requests python library.
98
+ See its documenation for other possible kinds of proxy to set up.
99
+
100
+
78
101
  #### Throttling
79
102
  For throttling the request, those two environment variable can be used:
80
103
  * `OCD_DTL_QUOTA_TIME` define, in seconds, the time before resetting the requests limit, *default is 1 second*.
@@ -1,5 +1,5 @@
1
1
  datalake/__init__.py,sha256=A2Cmrxun_uvpfjPC_9kwwAFKTpUI5uBcln2-ortksMY,699
2
- datalake/datalake.py,sha256=fxlti1_8otlvy4AG5BMd7E_9njGDyf5jbKf7wlKo5o0,4078
2
+ datalake/datalake.py,sha256=mFUEXYJ89YIw8eYR7nZyfl9l0TeXvvRMqw_7DkqKiQA,4503
3
3
  datalake/api_objects/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
4
4
  datalake/api_objects/bulk_search_task.py,sha256=2H1EKFNR6ttf5yJWW0F8kpHgWjA-Pawpu2i2Z3xP-kg,4627
5
5
  datalake/common/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
@@ -9,14 +9,14 @@ datalake/common/config.py,sha256=b9PAfwL2qhJzGBZXaUTe1HtH_WZole1xnbawFq9LbyI,476
9
9
  datalake/common/logger.py,sha256=vC03L_VZ6kN5YHGCRJX2_iSNlTNqx010bFH90Cyxf9A,460
10
10
  datalake/common/output.py,sha256=sU6wZLznokKjMvtNNcGq5BRn9nDKtE_3N2RrFM8NDqg,1537
11
11
  datalake/common/throttler.py,sha256=O9h1DsGiloatsnOrwJdZIZEFCGL4pPqF6dafHmMFvYA,1882
12
- datalake/common/token_manager.py,sha256=q-0qCNl5Bw0QIQdAJcm1ml0RWQlLwuc3xdKp16lijdY,5906
12
+ datalake/common/token_manager.py,sha256=uzcHGUzFrjTvlNosRU-wsycKob_gCRrsmtaEF-CYt_Q,6198
13
13
  datalake/common/utils.py,sha256=b2wUo6t9H-l9ctCpthKpS9CgOMvWiSJGOci3t6-M330,3504
14
14
  datalake/common/warn.py,sha256=8-YFJHlH3q91vp4TDyifmG05a5uvlVjHzcztbOiVP44,271
15
15
  datalake/config/endpoints.json,sha256=D2XUhnOEoZsIUWhlqOnMTFl9QJdpI0OnTksHTi27BzQ,1426
16
16
  datalake/endpoints/__init__.py,sha256=4GqELe9-njbc5BikldIX9RLDvVqQ96RqKRW9ykUsAL0,49
17
17
  datalake/endpoints/advanced_search.py,sha256=zS4iSOkUsZES59NvlrPsmmy9mF2w2pi8It1NBcWkH8g,2538
18
18
  datalake/endpoints/bulk_search.py,sha256=KGHd3zSM-A_2oXCm6HO81xJ4xVS9eZfCJNCrxD445ms,2570
19
- datalake/endpoints/endpoint.py,sha256=8UodFEdERlFS9ilBqENzw0foi2JVu4eH_B2wYfo3Fvk,5993
19
+ datalake/endpoints/endpoint.py,sha256=pa6_X9cJ3vt_919WlGW97uiG2JytIqqo_OAgdA0XCXg,6465
20
20
  datalake/endpoints/filtered_tag_subcategory.py,sha256=8ERU3PtSrNHM5A72gXGe6S6iAdHWnCKDT2ByU4oAuuI,3096
21
21
  datalake/endpoints/sightings.py,sha256=bXl_hEm8hrvyCG8vg1944jCnCgOfl_lVe7vvaDe1Bw8,7801
22
22
  datalake/endpoints/tags.py,sha256=zYKBjaNTuVOwUmdzDC2Og0IAvZSUirXTMVlJCPNQMoY,1027
@@ -24,11 +24,11 @@ datalake/endpoints/threats.py,sha256=ZfDJA7SX4pJglpK42oWJdRriRkzVYPoqvN8RRX4pG2g
24
24
  datalake/miscellaneous/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
25
25
  datalake/miscellaneous/search_watch.py,sha256=078kLpkDPW1fx9u1ANKg_haFSvUxjsFz9T3P-_nqFtI,6897
26
26
  datalake_scripts/__init__.py,sha256=EKgNnE5TNzttTOC4vPKID6gWB4S2Zj-qpKxI8ZA8mCw,147
27
- datalake_scripts/cli.py,sha256=0w7aWknip2R6ZAiR9s60nwDOoY_zriQdZGazUsWtqyg,2702
27
+ datalake_scripts/cli.py,sha256=ZbvP3_OP22oe9XrV8j07fXkRUUKsCB6Xp2Z9fc2WQn8,2702
28
28
  datalake_scripts/common/__init__.py,sha256=CeFL01aRaUnr4v36sE_CKoTQ9rDRAGYZBOMLsoKKsLc,803
29
- datalake_scripts/common/base_engine.py,sha256=wUEKIr1zvLWWHrCc4UNtUO78OEaNq-NodzowdfyNa2o,4321
29
+ datalake_scripts/common/base_engine.py,sha256=SgN6a5d5gJ5KFiTgjIwdtTBxUXX1Q0qNfcv1v9JzMKM,4583
30
30
  datalake_scripts/common/base_script.py,sha256=M6yRiUWwkuwj6iS9Bcg16D--tGA_UgAQCwUp_yqkUAo,2520
31
- datalake_scripts/common/mixins.py,sha256=3Srf2TOA2FXxziqWNs4fhlSbEjsiU7mwjGwhYcoy3P8,3101
31
+ datalake_scripts/common/mixins.py,sha256=3gEDzQs4CAmsf8SO2weXUxSe5u9lZvX_xKHNaQYdPxc,3139
32
32
  datalake_scripts/engines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
33
33
  datalake_scripts/engines/get_engine.py,sha256=CAkU1phNuFUYKdNob_keI2Qjwh_-fgVFhoLaD2J6vkc,3816
34
34
  datalake_scripts/engines/post_engine.py,sha256=H9FkwbWcW98S0tBlQ9ddojurB5muPKM6oBA9s6DLq_c,6136
@@ -71,8 +71,8 @@ tests/scripts/test_base_engine.py,sha256=vk-qqVHXwo9riQUQccoMhebw3GQSMD456rsB_FQ
71
71
  tests/scripts/test_cli.py,sha256=X0zS6z1begL4304IpQ0AK861FuY_RFM9B4RoUZkUGYI,1652
72
72
  tests/scripts/test_csv_builder.py,sha256=QEhbzH4w_epcFXYieZ2rTij7TGGMYRLTR-7LbiwSA60,7997
73
73
  tests/scripts/test_endpoint_config.py,sha256=gqE12mfuh_qGx6e-bTArcWxhhg4MUQEIYNZyrcy3m8g,987
74
- datalake_scripts-2.7.3.dist-info/METADATA,sha256=v9o7jv8xOHJK1Ei1ok4is9sqRe7vOu2NT8h41aaOeX4,4276
75
- datalake_scripts-2.7.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
76
- datalake_scripts-2.7.3.dist-info/entry_points.txt,sha256=DlfjQgUDIGnK_Ef_35515ACzOw1Vpdx9cI3sI0pMEL0,471
77
- datalake_scripts-2.7.3.dist-info/top_level.txt,sha256=uU48AOj64UXu2kR5yxRNQrTU133Il-sI6FrwEYBjxkk,32
78
- datalake_scripts-2.7.3.dist-info/RECORD,,
74
+ datalake_scripts-2.7.4.dist-info/METADATA,sha256=dDZepY5Dko5suOAtCJsV1yX3nAlFD7gDMQZens_qeTY,4825
75
+ datalake_scripts-2.7.4.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
76
+ datalake_scripts-2.7.4.dist-info/entry_points.txt,sha256=DlfjQgUDIGnK_Ef_35515ACzOw1Vpdx9cI3sI0pMEL0,471
77
+ datalake_scripts-2.7.4.dist-info/top_level.txt,sha256=uU48AOj64UXu2kR5yxRNQrTU133Il-sI6FrwEYBjxkk,32
78
+ datalake_scripts-2.7.4.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: bdist_wheel (0.43.0)
2
+ Generator: bdist_wheel (0.45.1)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5