howler-api 3.0.0.dev374__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of howler-api might be problematic. Click here for more details.

Files changed (198) hide show
  1. howler/__init__.py +0 -0
  2. howler/actions/__init__.py +168 -0
  3. howler/actions/add_label.py +111 -0
  4. howler/actions/add_to_bundle.py +159 -0
  5. howler/actions/change_field.py +76 -0
  6. howler/actions/demote.py +160 -0
  7. howler/actions/example_plugin.py +104 -0
  8. howler/actions/prioritization.py +93 -0
  9. howler/actions/promote.py +147 -0
  10. howler/actions/remove_from_bundle.py +133 -0
  11. howler/actions/remove_label.py +111 -0
  12. howler/actions/transition.py +200 -0
  13. howler/api/__init__.py +249 -0
  14. howler/api/base.py +88 -0
  15. howler/api/socket.py +114 -0
  16. howler/api/v1/__init__.py +97 -0
  17. howler/api/v1/action.py +372 -0
  18. howler/api/v1/analytic.py +748 -0
  19. howler/api/v1/auth.py +382 -0
  20. howler/api/v1/clue.py +99 -0
  21. howler/api/v1/configs.py +58 -0
  22. howler/api/v1/dossier.py +222 -0
  23. howler/api/v1/help.py +28 -0
  24. howler/api/v1/hit.py +1181 -0
  25. howler/api/v1/notebook.py +82 -0
  26. howler/api/v1/overview.py +191 -0
  27. howler/api/v1/search.py +788 -0
  28. howler/api/v1/template.py +206 -0
  29. howler/api/v1/tool.py +183 -0
  30. howler/api/v1/user.py +416 -0
  31. howler/api/v1/utils/__init__.py +0 -0
  32. howler/api/v1/utils/etag.py +84 -0
  33. howler/api/v1/view.py +288 -0
  34. howler/app.py +235 -0
  35. howler/common/README.md +125 -0
  36. howler/common/__init__.py +0 -0
  37. howler/common/classification.py +979 -0
  38. howler/common/classification.yml +107 -0
  39. howler/common/exceptions.py +167 -0
  40. howler/common/loader.py +154 -0
  41. howler/common/logging/__init__.py +241 -0
  42. howler/common/logging/audit.py +138 -0
  43. howler/common/logging/format.py +38 -0
  44. howler/common/net.py +79 -0
  45. howler/common/net_static.py +1494 -0
  46. howler/common/random_user.py +316 -0
  47. howler/common/swagger.py +117 -0
  48. howler/config.py +64 -0
  49. howler/cronjobs/__init__.py +29 -0
  50. howler/cronjobs/retention.py +61 -0
  51. howler/cronjobs/rules.py +274 -0
  52. howler/cronjobs/view_cleanup.py +88 -0
  53. howler/datastore/README.md +112 -0
  54. howler/datastore/__init__.py +0 -0
  55. howler/datastore/bulk.py +72 -0
  56. howler/datastore/collection.py +2342 -0
  57. howler/datastore/constants.py +119 -0
  58. howler/datastore/exceptions.py +41 -0
  59. howler/datastore/howler_store.py +105 -0
  60. howler/datastore/migrations/fix_process.py +41 -0
  61. howler/datastore/operations.py +130 -0
  62. howler/datastore/schemas.py +90 -0
  63. howler/datastore/store.py +231 -0
  64. howler/datastore/support/__init__.py +0 -0
  65. howler/datastore/support/build.py +215 -0
  66. howler/datastore/support/schemas.py +90 -0
  67. howler/datastore/types.py +22 -0
  68. howler/error.py +91 -0
  69. howler/external/__init__.py +0 -0
  70. howler/external/generate_mitre.py +96 -0
  71. howler/external/generate_sigma_rules.py +31 -0
  72. howler/external/generate_tlds.py +47 -0
  73. howler/external/reindex_data.py +66 -0
  74. howler/external/wipe_databases.py +58 -0
  75. howler/gunicorn_config.py +25 -0
  76. howler/healthz.py +47 -0
  77. howler/helper/__init__.py +0 -0
  78. howler/helper/azure.py +50 -0
  79. howler/helper/discover.py +59 -0
  80. howler/helper/hit.py +236 -0
  81. howler/helper/oauth.py +247 -0
  82. howler/helper/search.py +92 -0
  83. howler/helper/workflow.py +110 -0
  84. howler/helper/ws.py +378 -0
  85. howler/odm/README.md +102 -0
  86. howler/odm/__init__.py +1 -0
  87. howler/odm/base.py +1543 -0
  88. howler/odm/charter.txt +146 -0
  89. howler/odm/helper.py +416 -0
  90. howler/odm/howler_enum.py +25 -0
  91. howler/odm/models/__init__.py +0 -0
  92. howler/odm/models/action.py +33 -0
  93. howler/odm/models/analytic.py +90 -0
  94. howler/odm/models/assemblyline.py +48 -0
  95. howler/odm/models/aws.py +23 -0
  96. howler/odm/models/azure.py +16 -0
  97. howler/odm/models/cbs.py +44 -0
  98. howler/odm/models/config.py +558 -0
  99. howler/odm/models/dossier.py +33 -0
  100. howler/odm/models/ecs/__init__.py +0 -0
  101. howler/odm/models/ecs/agent.py +17 -0
  102. howler/odm/models/ecs/autonomous_system.py +16 -0
  103. howler/odm/models/ecs/client.py +149 -0
  104. howler/odm/models/ecs/cloud.py +141 -0
  105. howler/odm/models/ecs/code_signature.py +27 -0
  106. howler/odm/models/ecs/container.py +32 -0
  107. howler/odm/models/ecs/dns.py +62 -0
  108. howler/odm/models/ecs/egress.py +10 -0
  109. howler/odm/models/ecs/elf.py +74 -0
  110. howler/odm/models/ecs/email.py +122 -0
  111. howler/odm/models/ecs/error.py +14 -0
  112. howler/odm/models/ecs/event.py +140 -0
  113. howler/odm/models/ecs/faas.py +24 -0
  114. howler/odm/models/ecs/file.py +84 -0
  115. howler/odm/models/ecs/geo.py +30 -0
  116. howler/odm/models/ecs/group.py +18 -0
  117. howler/odm/models/ecs/hash.py +16 -0
  118. howler/odm/models/ecs/host.py +17 -0
  119. howler/odm/models/ecs/http.py +37 -0
  120. howler/odm/models/ecs/ingress.py +12 -0
  121. howler/odm/models/ecs/interface.py +21 -0
  122. howler/odm/models/ecs/network.py +30 -0
  123. howler/odm/models/ecs/observer.py +45 -0
  124. howler/odm/models/ecs/organization.py +12 -0
  125. howler/odm/models/ecs/os.py +21 -0
  126. howler/odm/models/ecs/pe.py +17 -0
  127. howler/odm/models/ecs/process.py +216 -0
  128. howler/odm/models/ecs/registry.py +26 -0
  129. howler/odm/models/ecs/related.py +45 -0
  130. howler/odm/models/ecs/rule.py +51 -0
  131. howler/odm/models/ecs/server.py +24 -0
  132. howler/odm/models/ecs/threat.py +247 -0
  133. howler/odm/models/ecs/tls.py +58 -0
  134. howler/odm/models/ecs/url.py +51 -0
  135. howler/odm/models/ecs/user.py +57 -0
  136. howler/odm/models/ecs/user_agent.py +20 -0
  137. howler/odm/models/ecs/vulnerability.py +41 -0
  138. howler/odm/models/gcp.py +16 -0
  139. howler/odm/models/hit.py +356 -0
  140. howler/odm/models/howler_data.py +328 -0
  141. howler/odm/models/lead.py +24 -0
  142. howler/odm/models/localized_label.py +13 -0
  143. howler/odm/models/overview.py +16 -0
  144. howler/odm/models/pivot.py +40 -0
  145. howler/odm/models/template.py +24 -0
  146. howler/odm/models/user.py +83 -0
  147. howler/odm/models/view.py +34 -0
  148. howler/odm/random_data.py +888 -0
  149. howler/odm/randomizer.py +609 -0
  150. howler/patched.py +5 -0
  151. howler/plugins/__init__.py +25 -0
  152. howler/plugins/config.py +123 -0
  153. howler/remote/__init__.py +0 -0
  154. howler/remote/datatypes/README.md +355 -0
  155. howler/remote/datatypes/__init__.py +98 -0
  156. howler/remote/datatypes/counters.py +63 -0
  157. howler/remote/datatypes/events.py +66 -0
  158. howler/remote/datatypes/hash.py +206 -0
  159. howler/remote/datatypes/lock.py +42 -0
  160. howler/remote/datatypes/queues/__init__.py +0 -0
  161. howler/remote/datatypes/queues/comms.py +59 -0
  162. howler/remote/datatypes/queues/multi.py +32 -0
  163. howler/remote/datatypes/queues/named.py +93 -0
  164. howler/remote/datatypes/queues/priority.py +215 -0
  165. howler/remote/datatypes/set.py +118 -0
  166. howler/remote/datatypes/user_quota_tracker.py +54 -0
  167. howler/security/__init__.py +253 -0
  168. howler/security/socket.py +108 -0
  169. howler/security/utils.py +185 -0
  170. howler/services/__init__.py +0 -0
  171. howler/services/action_service.py +111 -0
  172. howler/services/analytic_service.py +128 -0
  173. howler/services/auth_service.py +323 -0
  174. howler/services/config_service.py +128 -0
  175. howler/services/dossier_service.py +252 -0
  176. howler/services/event_service.py +93 -0
  177. howler/services/hit_service.py +893 -0
  178. howler/services/jwt_service.py +158 -0
  179. howler/services/lucene_service.py +286 -0
  180. howler/services/notebook_service.py +119 -0
  181. howler/services/overview_service.py +44 -0
  182. howler/services/template_service.py +45 -0
  183. howler/services/user_service.py +331 -0
  184. howler/utils/__init__.py +0 -0
  185. howler/utils/annotations.py +28 -0
  186. howler/utils/chunk.py +38 -0
  187. howler/utils/dict_utils.py +200 -0
  188. howler/utils/isotime.py +17 -0
  189. howler/utils/list_utils.py +11 -0
  190. howler/utils/lucene.py +77 -0
  191. howler/utils/path.py +27 -0
  192. howler/utils/socket_utils.py +61 -0
  193. howler/utils/str_utils.py +256 -0
  194. howler/utils/uid.py +47 -0
  195. howler_api-3.0.0.dev374.dist-info/METADATA +71 -0
  196. howler_api-3.0.0.dev374.dist-info/RECORD +198 -0
  197. howler_api-3.0.0.dev374.dist-info/WHEEL +4 -0
  198. howler_api-3.0.0.dev374.dist-info/entry_points.txt +8 -0
@@ -0,0 +1,609 @@
1
+ import datetime
2
+ import math
3
+ import os
4
+ import random
5
+ import time
6
+ from typing import Any as _Any
7
+ from typing import Optional as _Optional
8
+
9
+ from howler import odm
10
+ from howler.common.exceptions import HowlerValueError
11
+ from howler.odm import (
12
+ IP,
13
+ MAC,
14
+ MD5,
15
+ SHA1,
16
+ SHA256,
17
+ URI,
18
+ UUID,
19
+ Any,
20
+ Boolean,
21
+ Classification,
22
+ ClassificationString,
23
+ Compound,
24
+ Date,
25
+ Domain,
26
+ Email,
27
+ EmptyableKeyword,
28
+ Enum,
29
+ Float,
30
+ HowlerHash,
31
+ Integer,
32
+ Json,
33
+ Keyword,
34
+ List,
35
+ Long,
36
+ LowerKeyword,
37
+ Mapping,
38
+ Model,
39
+ Optional,
40
+ PhoneNumber,
41
+ Platform,
42
+ Processor,
43
+ SSDeepHash,
44
+ Text,
45
+ UpperKeyword,
46
+ URIPath,
47
+ )
48
+ from howler.odm.base import _Field
49
+ from howler.utils.uid import get_random_id
50
+
51
+ ALPHA = "ABCDEFGHIJKLMNOPQRSTUPVXYZabcdefghijklmnopqrstuvwxyz"
52
+ HASH_ALPHA = "abcdef0123456789"
53
+ SSDEEP_ALPHA = f"{ALPHA}0123456789"
54
+ WORDS = """The Cyber Centre stays on the cutting edge of technology by working with commercial vendors of cyber security
55
+ technology to support their development of enhanced cyber defence tools To do this our experts survey the cyber
56
+ security market evaluate emerging technologies in order to determine their potential to improve cyber security
57
+ across the country The Cyber Centre supports innovation by collaborating with all levels of government private
58
+ industry academia to examine complex problems in cyber security We are constantly engaging partners to promote
59
+ an open innovative environment We invite partners to work with us but also promote other Government of Canada
60
+ innovation programs One of our key partnerships is with the Government of Canada Build in Canada Innovation Program
61
+ BCIP The BCIP helps Canadian companies of all sizes transition their state of the art goods services from the
62
+ laboratory to the marketplace For certain cyber security innovations the Cyber Centre performs the role of technical
63
+ authority We evaluate participating companies new technology provide feedback in order to assist them in bringing
64
+ their product to market To learn more about selling testing an innovation visit the BCIP website""".split()
65
+ WORDS = list(set(WORDS))
66
+
67
+
68
+ with open(os.path.join(os.path.dirname(__file__), "charter.txt"), "r") as f:
69
+ SENTENCES = [line.strip() for line in f.readlines()]
70
+
71
+ MAPPING_KEYS = ["key_a", "key_b", "key_c", "key_d", "key_e", "key_f"]
72
+ EXT = [".jpg", ".doc", ".exe", ".pdf", ".xls", ".lnk", ".gif", ".ppt"]
73
+
74
+ DOM = [".com", ".ca", ".biz", ".edu"]
75
+
76
+ USERS = ["admin", "user", "shawnh"]
77
+
78
+ GROUPS = ["USERS", "ADMINS", "ANALYSTS"]
79
+
80
+ F_TYPES = [
81
+ "image/png",
82
+ "executable/windows",
83
+ "document/pdf",
84
+ "document/office",
85
+ "document/xml",
86
+ "code/javascript",
87
+ "code/vb",
88
+ ]
89
+
90
+ DEPARTMENTS = [
91
+ ("Accessibility Standards Canada", "ASC"),
92
+ ("Administrative Tribunals Support Service of Canada", "ATSSC"),
93
+ ("Agriculture and Agri-Food Canada", "AAFC"),
94
+ ("Atlantic Canada Opportunities Agency", "ACOA"),
95
+ ("Atlantic Pilotage Authority Canada", "APA"),
96
+ ("Atomic Energy of Canada Limited", "AECL"),
97
+ ("Auditor General of Canada (Office of the)", "OAG"),
98
+ ("Blue Water Bridge Canada", "BWB"),
99
+ ("Business Development Bank of Canada", "BDC"),
100
+ ("Canada Agricultural Review Tribunal", "CART"),
101
+ ("Canada Border Services Agency", "CBSA"),
102
+ ("Canada Deposit Insurance Corporation", "CDIC"),
103
+ ("Canada Development Investment Corporation", "CDEV"),
104
+ ("Canada Economic Development for Quebec Regions", "CED"),
105
+ ("Canada Employment Insurance Commission", "CEIC"),
106
+ ("Canada Energy Regulator", "CER"),
107
+ ("Canada Firearms Centre", "CAFC"),
108
+ ("Canada Industrial Relations Board", "CIRB"),
109
+ ("Canada Infrastructure Bank", "CIB"),
110
+ ("Canada Lands Company Limited", "CLC"),
111
+ ("Canada Mortgage and Housing Corporation", "CMHC"),
112
+ ("Canada Pension Plan Investment Board", "CPPIB"),
113
+ ("Canada Post", "CPC"),
114
+ ("Canada Revenue Agency", "CRA"),
115
+ ("Canada School of Public Service", "CSPS"),
116
+ ("Canadian Air Transport Security Authority", "CATSA"),
117
+ ("Canadian Army", "CA"),
118
+ ("Canadian Centre for Occupational Health and Safety", "CCOHS"),
119
+ ("Canadian Coast Guard", "CCG"),
120
+ ("Canadian Commercial Corporation", "CCC"),
121
+ ("Canadian Conservation Institute", "CCI"),
122
+ ("Canadian Dairy Commission", "CDC"),
123
+ ("Canadian Food Inspection Agency", "CFIA"),
124
+ ("Canadian Forces Housing Agency", "CFHA"),
125
+ ("Canadian Grain Commission", "CGC"),
126
+ ("Canadian Heritage", "PCH"),
127
+ ("Canadian Heritage Information Network", "CHIN"),
128
+ ("Canadian Human Rights Commission", "CHRC"),
129
+ ("Canadian Institutes of Health Research", "CIHR"),
130
+ ("Canadian Intellectual Property Office", "CIPO"),
131
+ ("Canadian Intergovernmental Conference Secretariat", "CICS"),
132
+ ("Canadian International Trade Tribunal", "CITT"),
133
+ ("Canadian Judicial Council", "CJC"),
134
+ ("Canadian Museum for Human Rights", "CMHR"),
135
+ ("Canadian Museum of History", "CMH"),
136
+ ("Canadian Museum of Immigration at Pier 21", "CMIP"),
137
+ ("Canadian Museum of Nature", "CMN"),
138
+ ("Canadian Northern Economic Development Agency", "CanNor"),
139
+ ("Canadian Nuclear Safety Commission", "CNSC"),
140
+ ("Canadian Pari-Mutuel Agency", "CPMA"),
141
+ ("Canadian Police College", "CPC"),
142
+ ("Canadian Race Relations Foundation", "CRRF"),
143
+ ("Canadian Radio-Television and Telecommunications Commission", "CRTC"),
144
+ ("Canadian Security Intelligence Service", "CSIS"),
145
+ ("Canadian Space Agency", "CSA"),
146
+ ("Canadian Special Operations Forces Command", "CANSOFCOM"),
147
+ ("Canadian Trade Commissioner Service", "TCS"),
148
+ ("Canadian Transportation Agency", "CTA"),
149
+ ("CBC/Radio-Canada", "CBC"),
150
+ ("Civilian Review and Complaints Commission for the RCMP", "CRCC"),
151
+ ("Commissioner for Federal Judicial Affairs Canada (Office of the)", "FJA"),
152
+ ("Commissioner of Lobbying of Canada (Office of the)", "OCL"),
153
+ ("Commissioner of Official Languages (Office of the)", "OCOL"),
154
+ ("Communications Research Centre Canada", "CRC"),
155
+ ("Communications Security Establishment Canada", "CSEC"),
156
+ ("Competition Bureau Canada", "COBU"),
157
+ ("Conflict of Interest and Ethics Commissioner (Office of the)", "CIEC"),
158
+ ("Copyright Board Canada", "CB"),
159
+ ("Correctional Investigator Canada", "OCI"),
160
+ ("Correctional Service Canada", "CSC"),
161
+ ("Court Martial Appeal Court of Canada", "CMAC"),
162
+ ("Courts Administration Service", "CAS"),
163
+ ("Crown-Indigenous Relations and Northern Affairs Canada", "CIRNAC"),
164
+ ("Defence Construction Canada", "DCC"),
165
+ ("Defence Research and Development Canada", "DRDC"),
166
+ ("Destination Canada", "DC"),
167
+ ("Elections Canada", "Elections"),
168
+ ("Employment and Social Development Canada", "ESDC"),
169
+ ("Environment and Climate Change Canada", "ECCC"),
170
+ ("Environmental Protection Review Canada", "EPRC"),
171
+ ("Export Development Canada", "EDC"),
172
+ ("Farm Credit Canada", "FCC"),
173
+ ("Farm Products Council of Canada", "FPCC"),
174
+ ("Federal Bridge Corporation", "FBCL"),
175
+ ("Federal Court of Appeal", "FCA"),
176
+ ("Federal Court of Canada", "FC"),
177
+ ("Federal Economic Development Agency for Southern Ontario", "FedDev Ontario"),
178
+ ("Federal Ombudsman for Victims Of Crime (Office of the)", "OFOVC"),
179
+ ("Finance Canada (Department of)", "FIN"),
180
+ ("Financial Consumer Agency of Canada", "FCAC"),
181
+ ("Financial Transactions and Reports Analysis Centre of Canada", "FINTRAC"),
182
+ ("Fisheries and Oceans Canada", "DFO"),
183
+ ("Freshwater Fish Marketing Corporation", "FFMC"),
184
+ ("Global Affairs Canada", "GAC"),
185
+ ("Governor General of Canada", "OSGG"),
186
+ ("Great Lakes Pilotage Authority Canada", "GLPA"),
187
+ ("Health Canada", "HC"),
188
+ ("Historic Sites and Monuments Board of Canada", "HSMBC"),
189
+ ("Human Rights Tribunal of Canada", "HRTC"),
190
+ ("Immigration and Refugee Board of Canada", "IRB"),
191
+ ("Immigration, Refugees and Citizenship Canada", "IRCC"),
192
+ ("Impact Assessment Agency of Canada", "IAAC"),
193
+ ("Independent Review Panel for Defence Acquisition", "IRPDA"),
194
+ ("Indigenous and Northern Affairs Canada", "INAC"),
195
+ ("Indigenous Services Canada", "ISC"),
196
+ ("Industrial Technologies Office", "ITO"),
197
+ ("Information Commissioner (Office of the)", "OIC"),
198
+ ("Infrastructure Canada", "INFC"),
199
+ ("Innovation, Science and Economic Development Canada", "ISED"),
200
+ ("Intergovernmental Affairs", "IGA"),
201
+ ("International Development Research Centre", "IDRC"),
202
+ ("Jacques Cartier and Champlain Bridges", "JCCBI"),
203
+ ("Justice Canada (Department of)", "JUS"),
204
+ ("Laurentian Pilotage Authority Canada", "LPA"),
205
+ ("Library and Archives Canada", "LAC"),
206
+ ("Marine Atlantic", "MarineAtlantic"),
207
+ ("Measurement Canada", "MC"),
208
+ ("Military Grievances External Review Committee", "MGERC"),
209
+ ("Military Police Complaints Commission of Canada", "MPCC"),
210
+ ("National Arts Centre", "NAC"),
211
+ ("National Battlefields Commission", "NBC"),
212
+ ("National Capital Commission", "NCC"),
213
+ ("National Defence", "DND"),
214
+ ("National Film Board", "NFB"),
215
+ ("National Gallery of Canada", "NGC"),
216
+ ("National Research Council Canada", "NRC"),
217
+ ("National Security and Intelligence Review Agency", "NSIRA"),
218
+ ("Natural Resources Canada", "NRCan"),
219
+ ("Natural Sciences and Engineering Research Canada", "NSERC"),
220
+ ("Northern Pipeline Agency Canada", "NPA"),
221
+ ("Occupational Health and Safety Tribunal Canada", "OHSTC"),
222
+ ("Office of the Chief Military Judge", "OCMJ"),
223
+ ("Pacific Economic Development Canada", "PacifiCan"),
224
+ ("Pacific Pilotage Authority Canada", "PPA"),
225
+ ("Parks Canada", "PC"),
226
+ ("Parole Board of Canada", "PBC"),
227
+ ("Patented Medicine Prices Review Board Canada", "PMPRB"),
228
+ ("Polar Knowledge Canada", "POLAR"),
229
+ ("Prairies Economic Development Canada", "PrairiesCan"),
230
+ ("Privacy Commissioner of Canada (Office of the)", "OPC"),
231
+ ("Privy Council Office", "PCO"),
232
+ ("Procurement Ombudsman (Office of the)", "OPO"),
233
+ ("Public Health Agency of Canada", "PHAC"),
234
+ ("Public Prosecution Service of Canada", "PPSC"),
235
+ ("Public Safety Canada", "PS"),
236
+ ("Public Sector Integrity Commissioner of Canada (Office of the)", "PSIC"),
237
+ ("Public Sector Pension Investment Board", "PSP Investments"),
238
+ ("Public Servants Disclosure Protection Tribunal Canada", "PSDPTC"),
239
+ ("Public Service Commission of Canada", "PSC"),
240
+ ("Public Service Labour Relations and Employment Board", "PSLREB"),
241
+ ("Public Services and Procurement Canada", "PSPC"),
242
+ ("Registry of the Specific Claims Tribunal of Canada", "SCT"),
243
+ ("Royal Canadian Air Force", "RCAF"),
244
+ ("Royal Canadian Mint", "Mint"),
245
+ ("Royal Canadian Mounted Police", "RCMP"),
246
+ ("Royal Canadian Mounted Police External Review Committee", "ERC"),
247
+ ("Royal Canadian Navy", "RCN"),
248
+ ("Royal Military College of Canada", "RMCC"),
249
+ (
250
+ "Secretariat of the National Security and Intelligence Committee of Parliamentarians",
251
+ "NSICOP",
252
+ ),
253
+ ("Service Canada", "ServCan"),
254
+ ("Shared Services Canada", "SSC"),
255
+ ("Social Sciences and Humanities Research Council of Canada", "SSHRC"),
256
+ ("Social Security Tribunal of Canada", "SST"),
257
+ ("Standards Council of Canada", "SCC-CCN"),
258
+ ("Statistics Canada", "StatCan"),
259
+ ("Superintendent of Bankruptcy Canada (Office of the)", "OSB"),
260
+ ("Superintendent of Financial Institutions Canada (Office of the)", "OSFI"),
261
+ ("Supreme Court of Canada", "SCC"),
262
+ ("Tax Court of Canada", "TCC"),
263
+ ("Taxpayers' Ombudsperson (Office of the)", "OTO"),
264
+ ("Transport Canada", "TC"),
265
+ ("Transportation Appeal Tribunal of Canada", "TATC"),
266
+ ("Transportation Safety Board of Canada", "TSB"),
267
+ ("Treasury Board of Canada Secretariat", "TBS"),
268
+ ("Veterans Affairs Canada", "VAC"),
269
+ ("Veterans Review and Appeal Board", "VRAB"),
270
+ ("VIA Rail Canada", "VIA Rail"),
271
+ ("Virtual Museum of Canada", "VMC"),
272
+ ("Windsor-Detroit Bridge Authority", "WDBA"),
273
+ ("Women and Gender Equality Canada", "WAGE"),
274
+ ("Youth", "YOUTH"),
275
+ ]
276
+
277
+
278
+ def get_random_file_type() -> str:
279
+ """Get a random file type"""
280
+ return random.choice(F_TYPES)
281
+
282
+
283
+ def get_random_word() -> str:
284
+ """Get a random word"""
285
+ return random.choice(WORDS)
286
+
287
+
288
+ def get_random_phrase() -> str:
289
+ """Get a random phrase"""
290
+ return random.choice(SENTENCES)
291
+
292
+
293
+ def get_random_hash(hash_len: int) -> str:
294
+ """Get a random hash"""
295
+ return "".join([random.choice(HASH_ALPHA) for _ in range(hash_len)])
296
+
297
+
298
+ def get_random_label() -> str:
299
+ """Get a random label"""
300
+ return get_random_word().upper()
301
+
302
+
303
+ def get_random_user() -> str:
304
+ """Get a random user"""
305
+ return random.choice(USERS)
306
+
307
+
308
+ def get_random_groups() -> str:
309
+ """Get a random groups"""
310
+ return random.choice(GROUPS)
311
+
312
+
313
+ def get_random_filename(smin: int = 1, smax: int = 3) -> str:
314
+ """Get a random filename"""
315
+ return "_".join([get_random_word().lower() for _ in range(random.randint(smin, smax))]) + random.choice(EXT)
316
+
317
+
318
+ def get_random_directory(smin: int = 2, smax: int = 6) -> str:
319
+ """Get a random directory"""
320
+ return "/".join([get_random_word().lower() for _ in range(random.randint(smin, smax))])
321
+
322
+
323
+ def get_random_string(smin: int = 4, smax: int = 24) -> str:
324
+ """Get a random string"""
325
+ return "".join([random.choice(ALPHA) for _ in range(random.randint(smin, smax))])
326
+
327
+
328
+ def get_random_email() -> str:
329
+ """Get a random email"""
330
+ return f"{get_random_word()}@{get_random_word()}{random.choice(DOM)}"
331
+
332
+
333
+ def get_random_host() -> str:
334
+ """Get a random host"""
335
+ return get_random_word().lower() + random.choice(DOM)
336
+
337
+
338
+ def get_random_ip() -> str:
339
+ """Get a random ip"""
340
+ return ".".join([str(random.randint(1, 254)) for _ in range(4)])
341
+
342
+
343
+ def get_random_lat_lng():
344
+ "Get a random lat/lng"
345
+ pi = math.pi
346
+ cf = 180.0 / pi # radians to degrees Correction Factor
347
+
348
+ # get a random Gaussian 3D vector:
349
+ gx = random.gauss(0.0, 1.0)
350
+ gy = random.gauss(0.0, 1.0)
351
+ gz = random.gauss(0.0, 1.0)
352
+
353
+ # normalize to an equidistributed (x,y,z) point on the unit sphere:
354
+ norm2 = gx * gx + gy * gy + gz * gz
355
+ norm1 = 1.0 / math.sqrt(norm2)
356
+ x = gx * norm1
357
+ y = gy * norm1
358
+ z = gz * norm1
359
+
360
+ rad_lat = math.asin(z) # latitude in radians
361
+ rad_lon = math.atan2(y, x) # longitude in radians
362
+
363
+ return (round(cf * rad_lat, 5), round(cf * rad_lon, 5))
364
+
365
+
366
+ def get_random_iso_date(epoch: _Optional[float] = None) -> str:
367
+ """Get a random ISO formatted date"""
368
+ if epoch is None:
369
+ epoch = time.time() + random.randint(-3000000, 0)
370
+
371
+ return datetime.datetime.fromtimestamp(epoch).isoformat() + "Z"
372
+
373
+
374
+ def get_random_mapping(field: _Field) -> dict[str, _Any]:
375
+ """Get a random mapping"""
376
+ return {MAPPING_KEYS[i]: random_data_for_field(field, MAPPING_KEYS[i]) for i in range(random.randint(1, 5))}
377
+
378
+
379
+ def get_random_phone() -> str:
380
+ """Get a random phone"""
381
+ return (
382
+ f'{random.choice(["", "+1 "])}{"-".join([str(random.randint(100, 999)) for _ in range(3)])}'
383
+ f"{str(random.randint(0, 9))}"
384
+ )
385
+
386
+
387
+ def get_random_mac() -> str:
388
+ """Get a random mac"""
389
+ return ":".join([get_random_hash(2) for _ in range(6)])
390
+
391
+
392
+ def get_random_uri_path() -> str:
393
+ """Get a random uri_path"""
394
+ return f"/{'/'.join([get_random_word() for _ in range(random.randint(2, 6))])}"
395
+
396
+
397
+ def get_random_uri() -> str:
398
+ """Get a random uri"""
399
+ return f"{random.choice(['http', 'https', 'ftp'])}://{get_random_host()}{get_random_uri_path()}"
400
+
401
+
402
+ def get_random_ssdeep() -> str:
403
+ """Get a random ssdeep"""
404
+ return (
405
+ f"{str(random.randint(30, 99999))}"
406
+ f":{''.join([random.choice(SSDEEP_ALPHA) for _ in range(random.randint(20, 64))])}"
407
+ f":{''.join([random.choice(SSDEEP_ALPHA) for _ in range(random.randint(20, 64))])}"
408
+ )
409
+
410
+
411
+ def get_random_platform() -> str:
412
+ """Get a random platform"""
413
+ return random.choice(["Windows", "Linux", "MacOS", "Android", "iOS"])
414
+
415
+
416
+ def get_random_processor() -> str:
417
+ """Get a random processor"""
418
+ return random.choice(["x86", "x64"])
419
+
420
+
421
+ def get_random_version() -> str:
422
+ """Get a random version"""
423
+ return f"{random.randint(4, 8)}.{random.randint(0, 5)}.{random.randint(0, 9)}"
424
+
425
+
426
+ def get_random_user_agent() -> str:
427
+ """Get a random user_agent"""
428
+ return random.choice(
429
+ [
430
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0",
431
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 "
432
+ "Safari/537.36 Edg/108.0.1462.46",
433
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 "
434
+ "Safari/537.36",
435
+ "Mozilla/5.0 (Linux; Android 10; SM-G980F Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like "
436
+ "Gecko) Version/4.0 Chrome/78.0.3904.96 Mobile Safari/537.36",
437
+ "Mozilla/5.0 (iPhone9,4; U; CPU iPhone OS 10_0_1 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) "
438
+ "Version/10.0 Mobile/14A403 Safari/602.1",
439
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 "
440
+ "Safari/601.3.9",
441
+ ]
442
+ )
443
+
444
+
445
+ def random_department():
446
+ """Generate a random department"""
447
+ return random.choice(DEPARTMENTS)[1], random.randint(1, len(DEPARTMENTS))
448
+
449
+
450
+ # noinspection PyProtectedMember
451
+ def random_data_for_field(field: _Field, name: str, minimal: bool = False) -> _Any:
452
+ """Get random data for any given field type"""
453
+ if isinstance(field, Boolean):
454
+ return random.choice([True, False])
455
+ elif isinstance(field, Classification):
456
+ if field.engine.enforce:
457
+ possible_classifications = list(field.engine._classification_cache)
458
+ possible_classifications.extend([field.engine.UNRESTRICTED, field.engine.RESTRICTED])
459
+ else:
460
+ possible_classifications = [field.engine.UNRESTRICTED]
461
+ return random.choice(possible_classifications)
462
+ elif isinstance(field, ClassificationString):
463
+ possible_classifications = [field.engine.UNRESTRICTED]
464
+ return random.choice(possible_classifications)
465
+ elif isinstance(field, Enum):
466
+ return random.choice([x for x in field.values if x is not None])
467
+ elif isinstance(field, List):
468
+ return [
469
+ (
470
+ random_data_for_field(field.child_type, name)
471
+ if not isinstance(field.child_type, Model)
472
+ else random_model_obj(field.child_type, as_json=True)
473
+ )
474
+ for _ in range(random.randint(1, 4))
475
+ ]
476
+ elif isinstance(field, Compound):
477
+ if minimal:
478
+ return random_minimal_obj(field.child_type, as_json=True)
479
+ else:
480
+ return random_model_obj(field.child_type, as_json=True)
481
+ elif isinstance(field, Mapping):
482
+ return get_random_mapping(field.child_type)
483
+ elif isinstance(field, Optional):
484
+ if not minimal:
485
+ return random_data_for_field(field.child_type, name)
486
+ else:
487
+ return field.child_type.default
488
+ elif isinstance(field, UUID):
489
+ return get_random_id()
490
+ elif isinstance(field, Date):
491
+ return get_random_iso_date()
492
+ elif isinstance(field, Integer):
493
+ return random.randint(128, 4096)
494
+ elif isinstance(field, Long):
495
+ return random.randint(1, 223372036854775807)
496
+ elif isinstance(field, Float):
497
+ return random.randint(12800, 409600) / 100.0
498
+ elif isinstance(field, MD5):
499
+ return get_random_hash(32)
500
+ elif isinstance(field, SHA1):
501
+ return get_random_hash(40)
502
+ elif isinstance(field, SHA256):
503
+ return get_random_hash(64)
504
+ elif isinstance(field, HowlerHash):
505
+ return get_random_hash(random.randint(1, 64))
506
+ elif isinstance(field, SSDeepHash):
507
+ return get_random_ssdeep()
508
+ elif isinstance(field, URI):
509
+ return get_random_uri()
510
+ elif isinstance(field, URIPath):
511
+ return get_random_uri_path()
512
+ elif isinstance(field, MAC):
513
+ return get_random_mac()
514
+ elif isinstance(field, PhoneNumber):
515
+ return get_random_phone()
516
+ elif isinstance(field, IP):
517
+ return get_random_ip()
518
+ elif isinstance(field, Domain):
519
+ return get_random_host()
520
+ elif isinstance(field, Email):
521
+ return get_random_email()
522
+ elif isinstance(field, Platform):
523
+ return get_random_platform()
524
+ elif isinstance(field, Processor):
525
+ return get_random_processor()
526
+ elif isinstance(field, Json):
527
+ return random.choice(
528
+ [
529
+ get_random_word(), # string
530
+ random.choice([True, False]), # boolean
531
+ [get_random_word() for _ in range(random.randint(2, 4))], # list
532
+ random.randint(0, 100),
533
+ ] # number
534
+ )
535
+ elif isinstance(field, UpperKeyword):
536
+ return get_random_word().upper()
537
+ elif isinstance(field, LowerKeyword):
538
+ return get_random_word().lower()
539
+ elif isinstance(field, Keyword) or isinstance(field, EmptyableKeyword):
540
+ if name:
541
+ if "sha256" in name:
542
+ return get_random_hash(64)
543
+ elif "filetype" in name:
544
+ return get_random_file_type()
545
+ elif "label" in name:
546
+ return get_random_label()
547
+ elif "group" in name:
548
+ return get_random_groups()
549
+ elif "user" in name or "uname" in name or "username" in name:
550
+ return get_random_user()
551
+ elif name.startswith("id_") or name.endswith("_id"):
552
+ return get_random_id()
553
+ elif "mac" in name:
554
+ return get_random_hash(12).upper()
555
+ elif "sha1" in name:
556
+ return get_random_hash(40)
557
+ elif "sha384" in name:
558
+ return get_random_hash(96)
559
+ elif "sha512" in name:
560
+ return get_random_hash(128)
561
+ elif "md5" in name:
562
+ return get_random_hash(32)
563
+ elif "host" in name or "node" in name or "domain" in name:
564
+ return get_random_host()
565
+ elif name.endswith("ip") or name.startswith("ip_"):
566
+ return get_random_ip()
567
+ elif "file" in name:
568
+ return get_random_filename()
569
+ elif "name" in name:
570
+ return get_random_filename()
571
+ elif "directory" in name:
572
+ return get_random_directory()
573
+ elif "version" in name:
574
+ return get_random_version()
575
+
576
+ return get_random_word()
577
+ elif isinstance(field, Text):
578
+ return get_random_phrase()
579
+ elif isinstance(field, Any):
580
+ return get_random_word()
581
+ else:
582
+ raise HowlerValueError(f"Unknown field type {field.__class__}")
583
+
584
+
585
+ # noinspection PyProtectedMember
586
+ def random_model_obj(model: odm.Model, as_json: bool = False) -> _Any:
587
+ """Create a random valid instance for the given model"""
588
+ data = {}
589
+ for f_name, f_value in model.fields().items():
590
+ data[f_name] = random_data_for_field(f_value, f_name)
591
+
592
+ if as_json:
593
+ return data
594
+ else:
595
+ return model(data) # type: ignore[operator]
596
+
597
+
598
+ # noinspection PyProtectedMember
599
+ def random_minimal_obj(model: odm.Model, as_json: bool = False) -> _Any:
600
+ """Create a minimal valid instance for the given model"""
601
+ data = {}
602
+ for f_name, f_value in model.fields().items():
603
+ if not f_value.default_set:
604
+ data[f_name] = random_data_for_field(f_value, f_name, minimal=True)
605
+
606
+ if as_json:
607
+ return data
608
+ else:
609
+ return model(data) # type: ignore[operator]
howler/patched.py ADDED
@@ -0,0 +1,5 @@
1
+ from gevent.monkey import patch_all
2
+
3
+ patch_all()
4
+
5
+ from howler.app import app
@@ -0,0 +1,25 @@
1
+ import importlib
2
+ from typing import Optional
3
+
4
+ from howler.common.logging import get_logger
5
+ from howler.config import config as _config # Python gets BIG mad if we don't alias this
6
+ from howler.plugins.config import BasePluginConfig
7
+
8
+ logger = get_logger(__file__)
9
+
10
+ PLUGINS: dict[str, Optional[BasePluginConfig]] = {}
11
+
12
+
13
+ def get_plugins() -> list[BasePluginConfig]:
14
+ "Get a set of plugin configurations based on the howler settings."
15
+ for plugin in _config.core.plugins:
16
+ if plugin in PLUGINS:
17
+ continue
18
+
19
+ try:
20
+ PLUGINS[plugin] = importlib.import_module(f"{plugin}.config").config
21
+ except (ImportError, ModuleNotFoundError):
22
+ logger.exception("Exception when loading plugin %s", plugin)
23
+ PLUGINS[plugin] = None
24
+
25
+ return [plugin for plugin in PLUGINS.values() if plugin]