stix2arango 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

stix2arango/services/arangodb_service.py

@@ -146,14 +146,12 @@ class ArangoDBService:
             if isinstance(ret, arango.exceptions.DocumentInsertError):
                 if ret.error_code == 1210:
                     existing_objects[f'{obj["id"]};{obj["_record_md5_hash"]}'] = collection_name + '/' + re.search(r'conflicting key: (.*)', ret.message).group(1)
-                    if 'relationship--7a1682a3-fec3-53ed-8dd5-68f54b1d6f7a' in ret.message:
-                        print(1)
                 else:
                     raise ret
         return [obj["id"] for obj in new_insertions], existing_objects
 
     def insert_several_objects_chunked(
-        self, objects, collection_name, chunk_size=1000, remove_duplicates=True
+        self, objects, collection_name, chunk_size=5000, remove_duplicates=True
     ):
         if remove_duplicates:
             original_length = len(objects)
@@ -183,7 +181,7 @@ class ArangoDBService:
         relationships: list[dict[str, Any]],
         id_to_key_map: dict[str, str],
         collection_name: str,
-        chunk_size=1200,
+        chunk_size=5000,
     ):
         for relationship in relationships:
             source_key = id_to_key_map.get(relationship["source_ref"])
@@ -244,7 +242,7 @@ class ArangoDBService:
             progress_bar.update(len(chunk))
 
         logging.info(
-            f"Updating relationship's _is_latest for {len(deprecated_key_ids)} items"
+            f"Deprecating _is_latest for {len(deprecated_key_ids)} items"
         )
         self.deprecate_relationships(deprecated_key_ids, edge_collection)
         return deprecated_key_ids
@@ -253,11 +251,19 @@ class ArangoDBService:
         self, deprecated_key_ids: list, edge_collection: str, chunk_size=5000
     ):
         keys = self.get_relationships_to_deprecate(deprecated_key_ids, edge_collection)
-        self.db.collection(edge_collection).update_many(
-            tuple(dict(_key=_key, _is_latest=False) for _key in keys),
-            silent=True,
-            raise_on_document_error=True,
+
+        progress_bar = tqdm(
+            utils.chunked(keys, chunk_size),
+            total=len(keys),
+            desc="deprecate_relationships",
         )
+        for chunk in progress_bar:
+            self.db.collection(edge_collection).update_many(
+                tuple(dict(_key=_key, _is_latest=False) for _key in chunk),
+                silent=True,
+                raise_on_document_error=True,
+            )
+            progress_bar.update(len(chunk))
         return len(keys)
 
     def get_relationships_to_deprecate(

stix2arango/stix2arango/stix2arango.py

@@ -451,15 +451,20 @@ class Stix2Arango:
         )
 
         self.alter_objects(objects)
-        with self.arango.transactional(exclusive=[self.core_collection_edge, self.core_collection_vertex]):
-            inserted_object_ids, existing_objects = (
-                self.arango.insert_relationships_chunked(
-                    objects, self.object_key_mapping, self.core_collection_edge
+        inserted_object_ids = []
+        existing_objects = {}
+        for chunk in utils.chunked(objects, 20_000):
+            with self.arango.transactional(exclusive=[self.core_collection_edge, self.core_collection_vertex]):
+                inserted, existing = (
+                    self.arango.insert_relationships_chunked(
+                        chunk, self.object_key_mapping, self.core_collection_edge
+                    )
                 )
-            )
-            self.arango.update_is_latest_several_chunked(
-                inserted_object_ids, self.core_collection_edge, self.core_collection_edge
-            )
+                inserted_object_ids.extend(inserted)
+                existing_objects.update(existing)
+        self.arango.update_is_latest_several_chunked(
+            inserted_object_ids, self.core_collection_edge, self.core_collection_edge
+        )
         return inserted_object_ids, existing_objects
 
     def import_default_objects(self):

{stix2arango-1.0.0.dist-info → stix2arango-1.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stix2arango
-Version: 1.0.0
+Version: 1.0.2
 Summary: stix2arango is a command line tool that takes a group of STIX 2.1 objects in a bundle and inserts them into ArangoDB. It can also handle updates to existing objects in ArangoDB imported in a bundle.
 Project-URL: Homepage, https://github.com/muchdogesec/stix2arango
 Project-URL: Issues, https://github.com/muchdogesec/stix2arango/issues

{stix2arango-1.0.0.dist-info → stix2arango-1.0.2.dist-info}/RECORD

@@ -3,14 +3,14 @@ stix2arango/__main__.py,sha256=wbR_iO70Vld2NYiml6Kz4rH396uOiNwTtjNBl4AHZEg,1987
 stix2arango/config.py,sha256=NZFrcnEfz-0QBrut2Rh7xMF78v0bk6U6y2TY_7mHxSs,1407
 stix2arango/utils.py,sha256=heln_kXBSTLo0R4n1IrfCHbRnCHTQf9Y2URhvwQ0Smc,4845
 stix2arango/services/__init__.py,sha256=E87fB-dxI4mPxMVs00jdLhjp9jFhkVfjhMKIqGLRJlY,45
-stix2arango/services/arangodb_service.py,sha256=ipuWmhttAoZl1SyCqA2U2jpmdNxkpyqGQ4fUBzA5u1M,11585
+stix2arango/services/arangodb_service.py,sha256=mXBQUeKR9IHErmvuZ73AiASZKawZOpuRiJQXWaN8bx8,11706
 stix2arango/services/version_annotator.py,sha256=Sd1MIaXzK0fpNopNxRoB_3etodzAjX5D_p3uGQSWzOI,2946
 stix2arango/stix2arango/__init__.py,sha256=OqxWEEsHqR1QQpznM5DbFJ5bO5numKYtoYhjXYJMEyg,36
 stix2arango/stix2arango/bundle_loader.py,sha256=qi-0E_bMIMPZXzISvjhrWX8K-f7iFv9vOekldOGVczU,4603
-stix2arango/stix2arango/stix2arango.py,sha256=AWObgUYS1rc1XpR1qgH1PhBMO_kQg2JcqlVzw08_fzY,20559
+stix2arango/stix2arango/stix2arango.py,sha256=UVv1OVy_BzqXYN9cWzjKK3QUSRuS-7jf3n_IgGQwx_8,20769
 stix2arango/templates/marking-definition.json,sha256=0q9y35mUmiF6xIWSLpkATL4JTHGSCNyLbejqZiQ0AuE,3113
-stix2arango-1.0.0.dist-info/METADATA,sha256=NLfyjCW0z4ERKXj4-vQBLmFyj7_CmVLDKZRmcPuUW7M,6873
-stix2arango-1.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-stix2arango-1.0.0.dist-info/entry_points.txt,sha256=k2WnxMsHFLoyC6rqfvjhIMS1zwtWin51-MbNCGmSMYE,58
-stix2arango-1.0.0.dist-info/licenses/LICENSE,sha256=BK8Ppqlc4pdgnNzIxnxde0taoQ1BgicdyqmBvMiNYgY,11364
-stix2arango-1.0.0.dist-info/RECORD,,
+stix2arango-1.0.2.dist-info/METADATA,sha256=uuH8tlqansoJvxNkqCxf4h_h1MteaRUQIlSQQG2DNOA,6873
+stix2arango-1.0.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+stix2arango-1.0.2.dist-info/entry_points.txt,sha256=k2WnxMsHFLoyC6rqfvjhIMS1zwtWin51-MbNCGmSMYE,58
+stix2arango-1.0.2.dist-info/licenses/LICENSE,sha256=BK8Ppqlc4pdgnNzIxnxde0taoQ1BgicdyqmBvMiNYgY,11364
+stix2arango-1.0.2.dist-info/RECORD,,