assemblyline 4.7.0.dev16__tar.gz → 4.7.0.dev45__tar.gz

{assemblyline-4.7.0.dev16/assemblyline.egg-info → assemblyline-4.7.0.dev45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.0.dev16
+Version: 4.7.0.dev45
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team
@@ -20,7 +20,7 @@ License-File: LICENCE.md
 Requires-Dist: arrow
 Requires-Dist: aiohttp
 Requires-Dist: lark
-Requires-Dist: urllib3
+Requires-Dist: urllib3>=2.6.0
 Requires-Dist: python-baseconv
 Requires-Dist: boto3
 Requires-Dist: pysftp

assemblyline-4.7.0.dev45/assemblyline/VERSION

@@ -0,0 +1 @@
+4.7.0.dev45

{assemblyline-4.7.0.dev16 → assemblyline-4.7.0.dev45}/assemblyline/common/bundling.py

@@ -268,6 +268,7 @@ def import_bundle(
     identify=None,
     reclassification=None,
     to_ingest=False,
+    dtl=None
 ):
     with forge.get_datastore(archive_access=True) as datastore:
         current_working_dir = os.path.join(working_dir, get_random_id())
@@ -352,6 +353,13 @@ def import_bundle(
                     submission['metadata']['bundle.loaded'] = now_as_iso()
                     submission['metadata']['bundle.classification'] = original_classification
                     submission['metadata'].pop('replay', None)
+                    if dtl:
+                        if dtl <= 0:
+                            # Submission should never expire
+                            submission['expiry_ts'] = None
+                        else:
+                            # Otherwise set the new expiry time
+                            submission['expiry_ts'] = now_as_iso(dtl * 24 * 60 * 60)
                     submission.update(Classification.get_access_control_parts(submission['classification']))
 
                     if not rescan_services:
@@ -362,7 +370,15 @@ def import_bundle(
                     with forge.get_filestore() as filestore:
                         for f, f_data in files['infos'].items():
                             check_classification(f_data)
-                            datastore.save_or_freshen_file(f, f_data, f_data['expiry_ts'], f_data['classification'],
+                            expiry_ts = f_data.get('expiry_ts', None)
+                            if dtl is not None:
+                                if dtl <= 0:
+                                    # File should never expire
+                                    expiry_ts = None
+                                else:
+                                    # Otherwise set the new expiry time
+                                    expiry_ts = now_as_iso(dtl * 24 * 60 * 60)
+                            datastore.save_or_freshen_file(f, f_data, expiry_ts, f_data['classification'],
                                                            cl_engine=Classification)
                             try:
                                 filestore.upload(os.path.join(current_working_dir, f), f)
@@ -376,6 +392,13 @@ def import_bundle(
                                     config.submission.emptyresult_dtl * 24 * 60 * 60)})
                             else:
                                 check_classification(res)
+                                if dtl is not None:
+                                    if dtl <= 0:
+                                        # Result should never expire
+                                        res.pop('expiry_ts', None)
+                                    else:
+                                        # Otherwise set the new expiry time
+                                        res['expiry_ts'] = now_as_iso(dtl * 24 * 60 * 60)
                                 datastore.result.save(key, res)
 
                         # Make sure errors meet minimum classification and save the errors

{assemblyline-4.7.0.dev16 → assemblyline-4.7.0.dev45}/assemblyline/common/custom.yara

@@ -382,52 +382,39 @@ rule code_html_component {
 document/email
 */
 
-rule document_email_1 {
+rule document_email {
 
     meta:
         type = "document/email"
         score = 15
 
     strings:
-        // This is a common JavaScript key
-        $rec = "From:"
-        $subrec1 = "Bcc:"
-        // This is a common JavaScript key
-        $subrec2 = "To:"
-        $subrec3 = "Date:"
-        // This is a common JavaScript key
-        $opt1 = ">Subject:"
-        $opt2 = "Received: from"
-        $opt3 = "MIME-Version:"
-        $opt4 = "Content-Type:"
-        $opt5 = "Sent on:"
+        $from = /(^|\n)From: /
+        $to = /(^|\n)To: /
+        $subject = /(^|\n)Subject: /
+        $date = /(^|\n)Date: /
+        $mime_version = /(^|\n)MIME-Version: /
+        $multipart_content_type = /(^|\n)Content-Type: multipart\/(alternative|byteranges|digest|form-data|mixed|parallel|related);\s{0,20}boundary=/
+        $html_content_type = /(^|\n)Content-Type: text\/html;\s{0,20}charset=/
+        $html_tag = /<html/i
 
     condition:
-        // This is a relatively* trusted mime for identifying JavaScript that could be mis-identified as emails
-        mime != "application/javascript"
-        and
-        (
-            all of ($rec*)
-            and 1 of ($subrec*)
-            and 1 of ($opt*)
+        $from
+        and (
+            $mime_version
+            or (
+                $to and $subject and $date
+            )
+        ) and (
+            $multipart_content_type
+            or (
+                $html_content_type
+                and $html_tag
+                and @html_content_type[1] < @html_tag[1]
+            )
         )
 }
 
-rule document_email_2 {
-
-    meta:
-        type = "document/email"
-        score = 10
-
-    strings:
-        $ = /(^|\n)From: /
-        $ = /(^|\n)MIME-Version: /
-        $ = /(^|\n)Content-Type: multipart\/mixed;\s*boundary=/
-
-    condition:
-        all of them
-}
-
 
 /*
 log/vipermonkey

{assemblyline-4.7.0.dev16 → assemblyline-4.7.0.dev45}/assemblyline/odm/models/config.py

@@ -247,46 +247,6 @@ class OAuthProvider(odm.Model):
         default="groups", description="Name of the field in the id token that contains the list of groups."
     )
 
-
-DEFAULT_OAUTH_PROVIDER_AZURE = {
-    "access_token_url": 'https://login.microsoftonline.com/common/oauth2/token',
-    "api_base_url": 'https://login.microsoft.com/common/',
-    "authorize_url": 'https://login.microsoftonline.com/common/oauth2/authorize',
-    "client_id": None,
-    "client_secret": None,
-    "client_kwargs": {"scope": "openid email profile"},
-    "jwks_uri": "https://login.microsoftonline.com/common/discovery/v2.0/keys",
-    "user_get": "openid/userinfo"
-}
-
-DEFAULT_OAUTH_PROVIDER_GOOGLE = {
-    "access_token_url": 'https://oauth2.googleapis.com/token',
-    "api_base_url": 'https://openidconnect.googleapis.com/',
-    "authorize_url": 'https://accounts.google.com/o/oauth2/v2/auth',
-    "client_id": None,
-    "client_secret": None,
-    "client_kwargs": {"scope": "openid email profile"},
-    "jwks_uri": "https://www.googleapis.com/oauth2/v3/certs",
-    "user_get": "v1/userinfo"
-}
-
-DEFAULT_OAUTH_PROVIDER_AUTH_ZERO = {
-    "access_token_url": 'https://{TENANT}.auth0.com/oauth/token',
-    "api_base_url": 'https://{TENANT}.auth0.com/',
-    "authorize_url": 'https://{TENANT}.auth0.com/authorize',
-    "client_id": None,
-    "client_secret": None,
-    "client_kwargs": {"scope": "openid email profile"},
-    "jwks_uri": "https://{TENANT}.auth0.com/.well-known/jwks.json",
-    "user_get": "userinfo"
-}
-
-DEFAULT_OAUTH_PROVIDERS = {
-    'auth0': DEFAULT_OAUTH_PROVIDER_AUTH_ZERO,
-    'azure_ad': DEFAULT_OAUTH_PROVIDER_AZURE,
-    'google': DEFAULT_OAUTH_PROVIDER_GOOGLE,
-}
-
 OPEN_ID_CONFIGURATION_TO_OAUTH_PROVIDER_MAP = {
     "authorization_endpoint": "authorize_url",
     "issuer": "api_base_url",
@@ -300,14 +260,14 @@ class OAuth(odm.Model):
     enabled: bool = odm.Boolean(description="Enable use of OAuth?")
     gravatar_enabled: bool = odm.Boolean(description="Enable gravatar?")
     providers: Dict[str, OAuthProvider] = odm.Mapping(odm.Compound(OAuthProvider),
-                                                      default=DEFAULT_OAUTH_PROVIDERS,
+                                                      default={},
                                                       description="OAuth provider configuration")
 
 
 DEFAULT_OAUTH = {
     "enabled": False,
     "gravatar_enabled": True,
-    "providers": DEFAULT_OAUTH_PROVIDERS
+    "providers": {}
 }
 
 
@@ -457,6 +417,10 @@ class SAMLAttributes(odm.Model):
     email_attribute: str = odm.Keyword(description="SAML attribute name for a user's email address ", default="email")
     fullname_attribute: str = odm.Keyword(description="SAML attribute name for a user's first name", default="name")
     groups_attribute: str = odm.Keyword(description="SAML attribute name for the groups", default="groups")
+    classification_attribute: str = odm.Keyword(
+        description="SAML attribute name for cliassification", default="classification"
+    )
+    dn_attribute: str = odm.Keyword(description="SAML attribute name for user's LDAP DN", default="dn")
     roles_attribute: str = odm.Keyword(description="SAML attribute name for the roles", default="roles")
     group_type_mapping: Dict[str, str] = odm.Mapping(
         odm.Keyword(), description="SAML group to role mapping", default={})

{assemblyline-4.7.0.dev16 → assemblyline-4.7.0.dev45}/assemblyline/remote/datatypes/daily_quota_tracker.py

@@ -51,3 +51,17 @@ class DailyQuotaTracker(object):
 
     def get_submission(self, user):
         return int(self._get(user, 'submission'))
+
+    def _reset(self, user, type):
+        counter = self._counter_name(user, type)
+        with self.c.pipeline() as pipe:
+            pipe.set(counter, 0)
+            pipe.expire(counter, self.ttl, nx=True)
+
+            val, _ = retry_call(pipe.execute)
+
+    def reset_api(self, user):
+        self._reset(user, "api")
+
+    def reset_submission(self, user):
+        self._reset(user, "submission")

{assemblyline-4.7.0.dev16 → assemblyline-4.7.0.dev45}/assemblyline/remote/datatypes/user_quota_tracker.py

@@ -1,4 +1,3 @@
-
 import redis
 from assemblyline.remote.datatypes import get_client, retry_call
 
@@ -52,3 +51,9 @@ class UserQuotaTracker(object):
                 retry_call(self.c.zpopmin, self._queue_name(user))
             else:
                 raise
+
+    def reset(self, user):
+        retry_call(self.c.delete, self._queue_name(user))
+
+    def get_count(self, user):
+        return retry_call(self.c.zcard, self._queue_name(user))

{assemblyline-4.7.0.dev16 → assemblyline-4.7.0.dev45/assemblyline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.0.dev16
+Version: 4.7.0.dev45
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team
@@ -20,7 +20,7 @@ License-File: LICENCE.md
 Requires-Dist: arrow
 Requires-Dist: aiohttp
 Requires-Dist: lark
-Requires-Dist: urllib3
+Requires-Dist: urllib3>=2.6.0
 Requires-Dist: python-baseconv
 Requires-Dist: boto3
 Requires-Dist: pysftp

{assemblyline-4.7.0.dev16 → assemblyline-4.7.0.dev45}/assemblyline.egg-info/requires.txt

@@ -1,7 +1,7 @@
 arrow
 aiohttp
 lark
-urllib3
+urllib3>=2.6.0
 python-baseconv
 boto3
 pysftp

{assemblyline-4.7.0.dev16 → assemblyline-4.7.0.dev45}/setup.py

@@ -47,7 +47,7 @@ setup(
         'arrow',
         'aiohttp',
         'lark',
-        'urllib3',
+        'urllib3>=2.6.0',
         'python-baseconv',
         'boto3',
         'pysftp',

assemblyline-4.7.0.dev16/assemblyline/VERSION

@@ -1 +0,0 @@
-4.7.0.dev16