synapse 2.179.0__py311-none-any.whl → 2.180.0__py311-none-any.whl

synapse/lib/schemas.py

@@ -493,3 +493,61 @@ driveDataVersSchema = {
     'additionalProperties': False,
 }
 reqValidDriveDataVers = s_config.getJsValidator(driveDataVersSchema)
+
+stixIngestConfigSchema = {
+    'type': 'object',
+    'properties': {
+        'bundle': {
+            'type': ['object', 'null'],
+            'properties': {'storm': {'type': 'string'}},
+        },
+        'objects': {
+            'type': 'object',
+            'properties': {'storm': {'type': 'string'}},
+        },
+        'relationships': {
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                    'type': {
+                        'type': 'array',
+                        'items': {
+                            'type': ['string', 'null'],
+                            'minItems': 3,
+                            'maxItems': 3,
+                        },
+                    },
+                    'storm': {'type': 'string'},
+                },
+                'required': ['type'],
+            },
+        },
+    },
+    'required': ['bundle', 'objects'],
+}
+reqValidStixIngestConfig = s_config.getJsValidator(stixIngestConfigSchema)
+
+stixIngestBundleSchema = {
+    'type': 'object',
+    'properties': {
+        'objects': {
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                    'id': {'type': 'string'},
+                    'type': {'type': 'string'},
+                    'object_refs': {'type': 'array', 'items': {'type': 'string'}},
+                    'relationship_type': {'type': 'string'},
+                    'source_ref': {'type': 'string'},
+                    'target_ref': {'type': 'string'},
+                },
+                'required': ['id', 'type'],
+                'if': {'properties': {'type': {'const': 'relationship'}}},
+                'then': {'required': ['source_ref', 'target_ref']},
+            }
+        },
+    },
+}
+reqValidStixIngestBundle = s_config.getJsValidator(stixIngestBundleSchema)

synapse/lib/snap.py

@@ -736,6 +736,7 @@ class Snap(s_base.Base):
 
         dorepr = False
         dopath = False
+        dolink = False
 
         show_storage = False
 
@@ -754,6 +755,7 @@ class Snap(s_base.Base):
         if opts is not None:
             dorepr = opts.get('repr', False)
             dopath = opts.get('path', False)
+            dolink = opts.get('links', False)
             show_storage = opts.get('show:storage', False)
 
         async for node, path in self.storm(text, opts=opts, user=user):
@@ -761,6 +763,9 @@ class Snap(s_base.Base):
             pode = node.pack(dorepr=dorepr)
             pode[1]['path'] = await path.pack(path=dopath)
 
+            if dolink:
+                pode[1]['links'] = path.links
+
             if show_storage:
                 pode[1]['storage'] = await node.getStorNodes()
 
@@ -1612,18 +1617,28 @@ class Snap(s_base.Base):
             last = verb
             yield verb
 
-    async def getNdefRefs(self, buid):
+    async def _getLayrNdefProp(self, layr, buid):
+        async for refsbuid, refsabrv in layr.getNdefRefs(buid):
+            yield refsbuid, layr.getAbrvProp(refsabrv)
+
+    async def getNdefRefs(self, buid, props=False):
         last = None
-        gens = [layr.getNdefRefs(buid) for layr in self.layers]
+        if props:
+            gens = [self._getLayrNdefProp(layr, buid) for layr in self.layers]
+        else:
+            gens = [layr.getNdefRefs(buid) for layr in self.layers]
 
-        async for refsbuid, _ in s_common.merggenr2(gens):
+        async for refsbuid, xtra in s_common.merggenr2(gens):
             if refsbuid == last:
                 continue
 
             await asyncio.sleep(0)
             last = refsbuid
 
-            yield refsbuid
+            if props:
+                yield refsbuid, xtra[1]
+            else:
+                yield refsbuid
 
     async def hasNodeData(self, buid, name):
         '''

synapse/lib/storm.py

@@ -5377,7 +5377,7 @@ class TeeCmd(Cmd):
 
                     outq = asyncio.Queue(maxsize=outq_size)
                     for subr in runts:
-                        subg = s_common.agen((node, path.fork(node)))
+                        subg = s_common.agen((node, path.fork(node, None)))
                         self.runt.snap.schedCoro(self.pipeline(subr, outq, genr=subg))
 
                     exited = 0
@@ -5399,7 +5399,7 @@ class TeeCmd(Cmd):
                 else:
 
                     for subr in runts:
-                        subg = s_common.agen((node, path.fork(node)))
+                        subg = s_common.agen((node, path.fork(node, None)))
                         async for subitem in subr.execute(genr=subg):
                             yield subitem
 
@@ -5553,6 +5553,7 @@ class ScrapeCmd(Cmd):
             if not todo:
                 todo = list(node.props.values())
 
+            link = {'type': 'scrape'}
             for text in todo:
 
                 text = str(text)
@@ -5562,7 +5563,7 @@ class ScrapeCmd(Cmd):
                         continue
 
                     nnode = await node.snap.addNode(form, valu)
-                    npath = path.fork(nnode)
+                    npath = path.fork(nnode, link)
 
                     if refs:
                         if node.form.isrunt:
@@ -5666,8 +5667,9 @@ class LiftByVerb(Cmd):
 
                     yield _node, _path
 
+                    link = {'type': 'runtime'}
                     async for node in self.iterEdgeNodes(verb, idenset, n2):
-                        yield node, _path.fork(node)
+                        yield node, _path.fork(node, link)
 
 class EdgesDelCmd(Cmd):
     '''

synapse/lib/stormlib/stix.py

@@ -11,6 +11,7 @@ import synapse.common as s_common
 import synapse.lib.coro as s_coro
 import synapse.lib.node as s_node
 import synapse.lib.msgpack as s_msgpack
+import synapse.lib.schemas as s_schemas
 import synapse.lib.stormctrl as s_stormctrl
 import synapse.lib.stormtypes as s_stormtypes
 
@@ -909,6 +910,9 @@ class LibStixImport(s_stormtypes.Lib):
 
         self.runt.layerConfirm(('node', 'edge', 'add', 'refs'))
 
+        config = s_schemas.reqValidStixIngestConfig(config)
+        bundle = s_schemas.reqValidStixIngestBundle(bundle)
+
         try:
             nodesbyid = await self._ingestObjects(bundle, config, rellook)
         except Exception as e:

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 179, 0)
+version = (2, 180, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = '1c51c58523e9012655d8d18bfa1ad149e5536c25'
+commit = 'aecbb8c0839f3ad763076dd7acae968fae6ba8b7'

synapse/models/auth.py

@@ -45,6 +45,8 @@ class AuthModule(s_module.CoreModule):
                     ('web:acct', ('inet:web:acct', {}), {
                         'doc': 'The web account that the credentials allow access to.',
                     }),
+                    ('service:account', ('inet:service:account', {}), {
+                        'doc': 'The service account that the credentials allow access to.'}),
                     # TODO x509, rfid, mat:item locks/keys
                 )),
 

synapse/models/geopol.py

@@ -139,6 +139,7 @@ class PolModule(s_module.CoreModule):
                             'doc': 'The date of the election.'}),
                     )),
                     # TODO jurisdiction / districts
+                    # TODO oversight authority
                     ('pol:race', {}, (
                         ('election', ('pol:election', {}), {
                             'doc': 'The election that includes the race.'}),
@@ -175,6 +176,8 @@ class PolModule(s_module.CoreModule):
                             'doc': 'The political party of the person who held office during the term.'}),
                     )),
                     ('pol:candidate', {}, (
+                        ('id', ('str', {'strip': True}), {
+                            'doc': 'A unique ID for the candidate issued by an election authority.'}),
                         ('contact', ('ps:contact', {}), {
                             'doc': 'The contact information of the candidate.'}),
                         ('race', ('pol:race', {}), {

synapse/models/inet.py

@@ -1363,6 +1363,7 @@ class InetModule(s_module.CoreModule):
                     }),
 
                     ('inet:search:query', ('guid', {}), {
+                        'interfaces': ('inet:service:action',),
                         'doc': 'An instance of a search query issued to a search engine.',
                     }),
 

synapse/models/infotech.py

@@ -757,30 +757,37 @@ class ItModule(s_module.CoreModule):
                     'doc': 'A developer selected label.',
                 }),
                 ('it:dev:repo', ('guid', {}), {
+                    'interfaces': ('inet:service:object',),
                     'doc': 'A version control system instance.',
                 }),
                 ('it:dev:repo:remote', ('guid', {}), {
                     'doc': 'A remote repo that is tracked for changes/branches/etc.',
                 }),
                 ('it:dev:repo:branch', ('guid', {}), {
+                    'interfaces': ('inet:service:object',),
                     'doc': 'A branch in a version control system instance.',
                 }),
                 ('it:dev:repo:commit', ('guid', {}), {
+                    'interfaces': ('inet:service:object',),
                     'doc': 'A commit to a repository.',
                 }),
                 ('it:dev:repo:diff', ('guid', {}), {
                     'doc': 'A diff of a file being applied in a single commit.',
                 }),
                 ('it:dev:repo:issue:label', ('guid', {}), {
+                    'interfaces': ('inet:service:object',),
                     'doc': 'A label applied to a repository issue.',
                 }),
                 ('it:dev:repo:issue', ('guid', {}), {
+                    'interfaces': ('inet:service:object',),
                     'doc': 'An issue raised in a repository.',
                 }),
                 ('it:dev:repo:issue:comment', ('guid', {}), {
+                    'interfaces': ('inet:service:object',),
                     'doc': 'A comment on an issue in a repository.',
                 }),
                 ('it:dev:repo:diff:comment', ('guid', {}), {
+                    'interfaces': ('inet:service:object',),
                     'doc': 'A comment on a diff in a repository.',
                 }),
                 ('it:prod:soft', ('guid', {}), {
@@ -1800,21 +1807,22 @@ class ItModule(s_module.CoreModule):
                     }),
                     ('desc', ('str', {}), {
                         'disp': {'hint': 'text'},
-                        'doc': 'A free-form description of the repository.',
-                    }),
+                        'doc': 'A free-form description of the repository.'}),
+
                     ('created', ('time', {}), {
-                        'doc': 'When the repository was created.',
-                    }),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use :period.'}),
+
                     ('url', ('inet:url', {}), {
-                        'doc': 'A URL where the repository is hosted.',
-                    }),
+                        'doc': 'The URL where the repository is hosted.'}),
+
                     ('type', ('it:dev:repo:type:taxonomy', {}), {
                         'doc': 'The type of the version control system used.',
-                        'ex': 'svn'
-                    }),
+                        'ex': 'svn'}),
+
                     ('submodules', ('array', {'type': 'it:dev:repo:commit'}), {
-                        'doc': "An array of other repos that this repo has as submodules, pinned at specific commits.",
-                    }),
+                        'doc': "An array of other repos that this repo has as submodules, pinned at specific commits."}),
+
                 )),
 
                 ('it:dev:repo:remote', {}, (
@@ -1834,120 +1842,127 @@ class ItModule(s_module.CoreModule):
                 )),
 
                 ('it:dev:repo:branch', {}, (
+
                     ('parent', ('it:dev:repo:branch', {}), {
-                        'doc': 'The branch this branch was branched from.',
-                    }),
+                        'doc': 'The branch this branch was branched from.'}),
+
                     ('start', ('it:dev:repo:commit', {}), {
-                        'doc': 'The commit in the parent branch this branch was created at.'
-                    }),
+                        'doc': 'The commit in the parent branch this branch was created at.'}),
+
                     ('name', ('str', {'strip': True}), {
-                        'doc': 'The name of the branch.',
-                    }),
+                        'doc': 'The name of the branch.'}),
+
                     ('url', ('inet:url', {}), {
-                        'doc': 'The URL where the branch is hosted.',
-                    }),
+                        'doc': 'The URL where the branch is hosted.'}),
+
                     ('created', ('time', {}), {
-                        'doc': 'The time this branch was created',
-                    }),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use :period.'}),
+
                     ('merged', ('time', {}), {
-                        'doc': 'The time this branch was merged back into its parent.',
-                    }),
+                        'doc': 'The time this branch was merged back into its parent.'}),
+
                     ('deleted', ('time', {}), {
-                        'doc': 'The time this branch was deleted.',
-                    }),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use :period.'}),
                 )),
 
                 ('it:dev:repo:commit', {}, (
                     ('repo', ('it:dev:repo', {}), {
-                        'doc': 'The repository the commit lives in.',
-                    }),
+                        'doc': 'The repository the commit lives in.'}),
+
                     ('parents', ('array', {'type': 'it:dev:repo:commit'}), {
-                        'doc': 'The commit or commits this commit is immediately based on.',
-                    }),
+                        'doc': 'The commit or commits this commit is immediately based on.'}),
+
                     ('branch', ('it:dev:repo:branch', {}), {
-                        'doc': 'The name of the branch the commit was made to.',
-                    }),
+                        'doc': 'The name of the branch the commit was made to.'}),
+
                     ('mesg', ('str', {}), {
                         'disp': {'hint': 'text'},
-                        'doc': 'The commit message describing the changes in the commit.',
-                    }),
+                        'doc': 'The commit message describing the changes in the commit.'}),
+
                     ('id', ('str', {}), {
-                        'doc': 'The version control system specific commit identifier.',
-                    }),
+                        'doc': 'The version control system specific commit identifier.'}),
+
                     ('created', ('time', {}), {
-                        'doc': 'The time the commit was made.',
-                    }),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use :period.'}),
+
                     ('url', ('inet:url', {}), {
-                        'doc': 'The URL where the commit is hosted.',
-                    }),
+                        'doc': 'The URL where the commit is hosted.'}),
                 )),
 
                 ('it:dev:repo:diff', {}, (
+
                     ('commit', ('it:dev:repo:commit', {}), {
-                        'doc': 'The commit that produced this diff.',
-                    }),
+                        'doc': 'The commit that produced this diff.'}),
+
                     ('file', ('file:bytes', {}), {
-                        'doc': 'The file after the commit has been applied',
-                    }),
+                        'doc': 'The file after the commit has been applied'}),
+
                     ('path', ('file:path', {}), {
-                        'doc': 'The path to the file in the repo that the diff is being applied to.',
-                    }),
+                        'doc': 'The path to the file in the repo that the diff is being applied to.'}),
+
                     ('url', ('inet:url', {}), {
-                        'doc': 'The URL where the diff is hosted.',
-                    }),
+                        'doc': 'The URL where the diff is hosted.'}),
                 )),
 
                 ('it:dev:repo:issue', {}, (
+
                     ('repo', ('it:dev:repo', {}), {
-                        'doc': 'The repo where the issue was logged.',
-                    }),
+                        'doc': 'The repo where the issue was logged.'}),
+
                     ('title', ('str', {'lower': True, 'strip': True}), {
-                        'doc': 'The title of the issue.'
-                    }),
+                        'doc': 'The title of the issue.'}),
+
                     ('desc', ('str', {}), {
                         'disp': {'hint': 'text'},
-                        'doc': 'The text describing the issue.'
-                    }),
+                        'doc': 'The text describing the issue.'}),
+
                     ('created', ('time', {}), {
-                        'doc': 'The time the issue was created.',
-                    }),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use :period.'}),
+
                     ('updated', ('time', {}), {
-                        'doc': 'The time the issue was updated.',
-                    }),
+                        'doc': 'The time the issue was updated.'}),
+
                     ('url', ('inet:url', {}), {
-                        'doc': 'The URL where the issue is hosted.',
-                    }),
+                        'doc': 'The URL where the issue is hosted.'}),
+
                     ('id', ('str', {'strip': True}), {
-                        'doc': 'The ID of the issue in the repository system.',
-                    }),
+                        'doc': 'The ID of the issue in the repository system.'}),
                 )),
 
                 ('it:dev:repo:label', {}, (
+
                     ('id', ('str', {'strip': True}), {
-                        'doc': 'The ID of the label.',
-                    }),
+                        'doc': 'The ID of the label.'}),
+
                     ('title', ('str', {'lower': True, 'strip': True}), {
-                        'doc': 'The human friendly name of the label.',
-                    }),
+                        'doc': 'The human friendly name of the label.'}),
+
                     ('desc', ('str', {}), {
                         'disp': {'hint': 'text'},
-                        'doc': 'The description of the label.',
-                    }),
+                        'doc': 'The description of the label.'}),
+
                 )),
 
                 ('it:dev:repo:issue:label', {}, (
+
                     ('issue', ('it:dev:repo:issue', {}), {
-                        'doc': 'The issue the label was applied to.',
-                    }),
+                        'doc': 'The issue the label was applied to.'}),
+
                     ('label', ('it:dev:repo:label', {}), {
-                        'doc': 'The label that was applied to the issue.',
-                    }),
+                        'doc': 'The label that was applied to the issue.'}),
+
                     ('applied', ('time', {}), {
-                        'doc': 'The time the label was applied.',
-                    }),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use :period.'}),
+
                     ('removed', ('time', {}), {
-                        'doc': 'The time the label was removed.',
-                    }),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use :period.'}),
+
                 )),
 
                 ('it:dev:repo:issue:comment', {}, (
@@ -1965,7 +1980,8 @@ class ItModule(s_module.CoreModule):
                         'doc': 'The URL where the comment is hosted.',
                     }),
                     ('created', ('time', {}), {
-                        'doc': 'The time the comment was created.',
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use :period.',
                     }),
                     ('updated', ('time', {}), {
                         'doc': 'The time the comment was updated.',
@@ -1973,31 +1989,33 @@ class ItModule(s_module.CoreModule):
                 )),
 
                 ('it:dev:repo:diff:comment', {}, (
+
                     ('diff', ('it:dev:repo:diff', {}), {
-                        'doc': 'The diff the comment is being added to.',
-                    }),
+                        'doc': 'The diff the comment is being added to.'}),
+
                     ('text', ('str', {}), {
                         'disp': {'hint': 'text'},
-                        'doc': 'The body of the comment.',
-                    }),
+                        'doc': 'The body of the comment.'}),
+
                     ('replyto', ('it:dev:repo:diff:comment', {}), {
-                        'doc': 'The comment that this comment is replying to.',
-                    }),
+                        'doc': 'The comment that this comment is replying to.'}),
+
                     ('line', ('int', {}), {
-                        'doc': 'The line in the file that is being commented on.',
-                    }),
+                        'doc': 'The line in the file that is being commented on.'}),
+
                     ('offset', ('int', {}), {
-                        'doc': 'The offset in the line in the file that is being commented on.',
-                    }),
+                        'doc': 'The offset in the line in the file that is being commented on.'}),
+
                     ('url', ('inet:url', {}), {
-                        'doc': 'The URL where the comment is hosted.',
-                    }),
+                        'doc': 'The URL where the comment is hosted.'}),
+
                     ('created', ('time', {}), {
-                        'doc': 'The time the comment was created.',
-                    }),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use :period.'}),
+
                     ('updated', ('time', {}), {
-                        'doc': 'The time the comment was updated.',
-                    }),
+                        'doc': 'The time the comment was updated.'}),
+
                 )),
 
                 ('it:prod:hardwaretype', {}, ()),

synapse/models/person.py

@@ -371,8 +371,11 @@ class PsModule(s_module.CoreModule):
                         'doc': 'The listed org/company FQDN for this contact.',
                     }),
                     ('user', ('inet:user', {}), {
-                        'doc': 'The username or handle for this contact.',
-                    }),
+                        'doc': 'The username or handle for this contact.'}),
+
+                    ('service:accounts', ('array', {'type': 'inet:service:account', 'sorted': True, 'uniq': True}), {
+                        'doc': 'The service accounts associated with this contact.'}),
+
                     ('web:acct', ('inet:web:acct', {}), {
                         'doc': 'The social media account for this contact.',
                     }),

synapse/models/telco.py

@@ -378,6 +378,9 @@ class TelcoModule(s_module.CoreModule):
                     ('email', ('inet:email', {}), {}),
                     ('acct', ('inet:web:acct', {}), {}),
 
+                    ('account', ('inet:service:account', {}), {
+                        'doc': 'The service account which is associated with the tracked device.'}),
+
                     # reporting related data
                     ('app', ('it:prod:softver', {}), {}),