synapse 2.179.0__py311-none-any.whl → 2.180.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/cortex.py +4 -4
- synapse/lib/ast.py +84 -54
- synapse/lib/layer.py +1 -1
- synapse/lib/lmdbslab.py +2 -0
- synapse/lib/modelrev.py +5 -1
- synapse/lib/node.py +14 -4
- synapse/lib/schemas.py +58 -0
- synapse/lib/snap.py +19 -4
- synapse/lib/storm.py +6 -4
- synapse/lib/stormlib/stix.py +4 -0
- synapse/lib/version.py +2 -2
- synapse/models/auth.py +2 -0
- synapse/models/geopol.py +3 -0
- synapse/models/inet.py +1 -0
- synapse/models/infotech.py +109 -91
- synapse/models/person.py +5 -2
- synapse/models/telco.py +3 -0
- synapse/tests/test_lib_ast.py +231 -0
- synapse/tests/test_lib_cell.py +1 -0
- synapse/tests/test_lib_layer.py +22 -0
- synapse/tests/test_lib_modelrev.py +7 -0
- synapse/tests/test_lib_node.py +12 -1
- synapse/tests/test_lib_storm.py +32 -7
- synapse/tests/test_lib_stormlib_stix.py +1 -1
- synapse/tests/test_model_geopol.py +2 -0
- synapse/tests/test_model_inet.py +10 -1
- synapse/tests/test_model_person.py +2 -0
- synapse/tests/test_model_telco.py +2 -1
- {synapse-2.179.0.dist-info → synapse-2.180.0.dist-info}/METADATA +1 -1
- {synapse-2.179.0.dist-info → synapse-2.180.0.dist-info}/RECORD +33 -33
- {synapse-2.179.0.dist-info → synapse-2.180.0.dist-info}/WHEEL +1 -1
- {synapse-2.179.0.dist-info → synapse-2.180.0.dist-info}/LICENSE +0 -0
- {synapse-2.179.0.dist-info → synapse-2.180.0.dist-info}/top_level.txt +0 -0
synapse/cortex.py
CHANGED
|
@@ -5730,7 +5730,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5730
5730
|
|
|
5731
5731
|
if (nexsoffs := opts.get('nexsoffs')) is not None:
|
|
5732
5732
|
if not await self.waitNexsOffs(nexsoffs, timeout=opts.get('nexstimeout')):
|
|
5733
|
-
raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs}
|
|
5733
|
+
raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs} in count()')
|
|
5734
5734
|
|
|
5735
5735
|
view = self._viewFromOpts(opts)
|
|
5736
5736
|
|
|
@@ -5794,7 +5794,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5794
5794
|
|
|
5795
5795
|
if (nexsoffs := opts.get('nexsoffs')) is not None:
|
|
5796
5796
|
if not await self.waitNexsOffs(nexsoffs, timeout=opts.get('nexstimeout')):
|
|
5797
|
-
raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs}.')
|
|
5797
|
+
raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs} in storm().')
|
|
5798
5798
|
|
|
5799
5799
|
view = self._viewFromOpts(opts)
|
|
5800
5800
|
async for mesg in view.storm(text, opts=opts):
|
|
@@ -5821,7 +5821,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5821
5821
|
|
|
5822
5822
|
if (nexsoffs := opts.get('nexsoffs')) is not None:
|
|
5823
5823
|
if not await self.waitNexsOffs(nexsoffs, timeout=opts.get('nexstimeout')):
|
|
5824
|
-
raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs}.')
|
|
5824
|
+
raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs} in callStorm().')
|
|
5825
5825
|
|
|
5826
5826
|
view = self._viewFromOpts(opts)
|
|
5827
5827
|
return await view.callStorm(text, opts=opts)
|
|
@@ -5849,7 +5849,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5849
5849
|
|
|
5850
5850
|
if (nexsoffs := opts.get('nexsoffs')) is not None:
|
|
5851
5851
|
if not await self.waitNexsOffs(nexsoffs, timeout=opts.get('nexstimeout')):
|
|
5852
|
-
raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs}.')
|
|
5852
|
+
raise s_exc.TimeOut(mesg=f'Timeout waiting for nexus offset {nexsoffs} in exportStorm().')
|
|
5853
5853
|
|
|
5854
5854
|
user = self._userFromOpts(opts)
|
|
5855
5855
|
view = self._viewFromOpts(opts)
|
synapse/lib/ast.py
CHANGED
|
@@ -419,7 +419,8 @@ class SubGraph:
|
|
|
419
419
|
await asyncio.sleep(0)
|
|
420
420
|
continue
|
|
421
421
|
|
|
422
|
-
|
|
422
|
+
link = {'type': 'prop', 'prop': propname}
|
|
423
|
+
yield (pivonode, path.fork(pivonode, link), link)
|
|
423
424
|
|
|
424
425
|
for iden in existing:
|
|
425
426
|
buid = s_common.uhex(iden)
|
|
@@ -1436,12 +1437,13 @@ class LiftOper(Oper):
|
|
|
1436
1437
|
|
|
1437
1438
|
return
|
|
1438
1439
|
|
|
1440
|
+
link = {'type': 'runtime'}
|
|
1439
1441
|
async for node, path in genr:
|
|
1440
1442
|
|
|
1441
1443
|
yield node, path
|
|
1442
1444
|
|
|
1443
1445
|
async for subn in self.lift(runt, path):
|
|
1444
|
-
yield subn, path.fork(subn)
|
|
1446
|
+
yield subn, path.fork(subn, link)
|
|
1445
1447
|
|
|
1446
1448
|
async def lift(self, runt, path): # pragma: no cover
|
|
1447
1449
|
raise NotImplementedError('Must define lift(runt, path)')
|
|
@@ -1913,8 +1915,9 @@ class PivotOut(PivotOper):
|
|
|
1913
1915
|
# <syn:tag> -> * is "from tags to nodes with tags"
|
|
1914
1916
|
if node.form.name == 'syn:tag':
|
|
1915
1917
|
|
|
1918
|
+
link = {'type': 'tag', 'tag': node.ndef[1], 'reverse': True}
|
|
1916
1919
|
async for pivo in runt.snap.nodesByTag(node.ndef[1]):
|
|
1917
|
-
yield pivo, path.fork(pivo)
|
|
1920
|
+
yield pivo, path.fork(pivo, link)
|
|
1918
1921
|
|
|
1919
1922
|
return
|
|
1920
1923
|
|
|
@@ -1925,7 +1928,7 @@ class PivotOut(PivotOper):
|
|
|
1925
1928
|
logger.warning(f'Missing node corresponding to ndef {n2def} on edge')
|
|
1926
1929
|
return
|
|
1927
1930
|
|
|
1928
|
-
yield pivo, path.fork(pivo)
|
|
1931
|
+
yield pivo, path.fork(pivo, {'type': 'prop', 'prop': 'n2'})
|
|
1929
1932
|
return
|
|
1930
1933
|
|
|
1931
1934
|
for name, prop in node.form.props.items():
|
|
@@ -1934,27 +1937,28 @@ class PivotOut(PivotOper):
|
|
|
1934
1937
|
if valu is None:
|
|
1935
1938
|
continue
|
|
1936
1939
|
|
|
1940
|
+
link = {'type': 'prop', 'prop': prop.name}
|
|
1937
1941
|
# if the outbound prop is an ndef...
|
|
1938
1942
|
if isinstance(prop.type, s_types.Ndef):
|
|
1939
1943
|
pivo = await runt.snap.getNodeByNdef(valu)
|
|
1940
1944
|
if pivo is None:
|
|
1941
1945
|
continue
|
|
1942
1946
|
|
|
1943
|
-
yield pivo, path.fork(pivo)
|
|
1947
|
+
yield pivo, path.fork(pivo, link)
|
|
1944
1948
|
continue
|
|
1945
1949
|
|
|
1946
1950
|
if isinstance(prop.type, s_types.Array):
|
|
1947
1951
|
if isinstance(prop.type.arraytype, s_types.Ndef):
|
|
1948
1952
|
for item in valu:
|
|
1949
1953
|
if (pivo := await runt.snap.getNodeByNdef(item)) is not None:
|
|
1950
|
-
yield pivo, path.fork(pivo)
|
|
1954
|
+
yield pivo, path.fork(pivo, link)
|
|
1951
1955
|
continue
|
|
1952
1956
|
|
|
1953
1957
|
typename = prop.type.opts.get('type')
|
|
1954
1958
|
if runt.model.forms.get(typename) is not None:
|
|
1955
1959
|
for item in valu:
|
|
1956
1960
|
async for pivo in runt.snap.nodesByPropValu(typename, '=', item, norm=False):
|
|
1957
|
-
yield pivo, path.fork(pivo)
|
|
1961
|
+
yield pivo, path.fork(pivo, link)
|
|
1958
1962
|
|
|
1959
1963
|
form = runt.model.forms.get(prop.type.name)
|
|
1960
1964
|
if form is None:
|
|
@@ -1962,7 +1966,7 @@ class PivotOut(PivotOper):
|
|
|
1962
1966
|
|
|
1963
1967
|
if prop.isrunt:
|
|
1964
1968
|
async for pivo in runt.snap.nodesByPropValu(form.name, '=', valu):
|
|
1965
|
-
yield pivo, path.fork(pivo)
|
|
1969
|
+
yield pivo, path.fork(pivo, link)
|
|
1966
1970
|
continue
|
|
1967
1971
|
|
|
1968
1972
|
pivo = await runt.snap.getNodeByNdef((form.name, valu))
|
|
@@ -1973,7 +1977,7 @@ class PivotOut(PivotOper):
|
|
|
1973
1977
|
if pivo.buid == node.buid:
|
|
1974
1978
|
continue
|
|
1975
1979
|
|
|
1976
|
-
yield pivo, path.fork(pivo)
|
|
1980
|
+
yield pivo, path.fork(pivo, link)
|
|
1977
1981
|
|
|
1978
1982
|
class N1WalkNPivo(PivotOut):
|
|
1979
1983
|
|
|
@@ -1990,7 +1994,7 @@ class N1WalkNPivo(PivotOut):
|
|
|
1990
1994
|
async for (verb, iden) in node.iterEdgesN1():
|
|
1991
1995
|
wnode = await runt.snap.getNodeByBuid(s_common.uhex(iden))
|
|
1992
1996
|
if wnode is not None:
|
|
1993
|
-
yield wnode, path.fork(wnode)
|
|
1997
|
+
yield wnode, path.fork(wnode, {'type': 'edge', 'verb': verb})
|
|
1994
1998
|
|
|
1995
1999
|
class PivotToTags(PivotOper):
|
|
1996
2000
|
'''
|
|
@@ -2057,7 +2061,7 @@ class PivotToTags(PivotOper):
|
|
|
2057
2061
|
if pivo is None:
|
|
2058
2062
|
continue
|
|
2059
2063
|
|
|
2060
|
-
yield pivo, path.fork(pivo)
|
|
2064
|
+
yield pivo, path.fork(pivo, {'type': 'tag', 'tag': name})
|
|
2061
2065
|
|
|
2062
2066
|
class PivotIn(PivotOper):
|
|
2063
2067
|
'''
|
|
@@ -2083,25 +2087,27 @@ class PivotIn(PivotOper):
|
|
|
2083
2087
|
|
|
2084
2088
|
pivo = await runt.snap.getNodeByNdef(ndef)
|
|
2085
2089
|
if pivo is not None:
|
|
2086
|
-
yield pivo, path.fork(pivo)
|
|
2090
|
+
yield pivo, path.fork(pivo, {'type': 'prop', 'prop': 'n1', 'reverse': True})
|
|
2087
2091
|
|
|
2088
2092
|
return
|
|
2089
2093
|
|
|
2090
2094
|
name, valu = node.ndef
|
|
2091
2095
|
|
|
2092
2096
|
for prop in runt.model.getPropsByType(name):
|
|
2097
|
+
link = {'type': 'prop', 'prop': prop.name, 'reverse': True}
|
|
2093
2098
|
norm = node.form.typehash is not prop.typehash
|
|
2094
2099
|
async for pivo in runt.snap.nodesByPropValu(prop.full, '=', valu, norm=norm):
|
|
2095
|
-
yield pivo, path.fork(pivo)
|
|
2100
|
+
yield pivo, path.fork(pivo, link)
|
|
2096
2101
|
|
|
2097
2102
|
for prop in runt.model.getArrayPropsByType(name):
|
|
2098
2103
|
norm = node.form.typehash is not prop.arraytypehash
|
|
2104
|
+
link = {'type': 'prop', 'prop': prop.name, 'reverse': True}
|
|
2099
2105
|
async for pivo in runt.snap.nodesByPropArray(prop.full, '=', valu, norm=norm):
|
|
2100
|
-
yield pivo, path.fork(pivo)
|
|
2106
|
+
yield pivo, path.fork(pivo, link)
|
|
2101
2107
|
|
|
2102
|
-
async for refsbuid in runt.snap.getNdefRefs(node.buid):
|
|
2108
|
+
async for refsbuid, prop in runt.snap.getNdefRefs(node.buid, props=True):
|
|
2103
2109
|
pivo = await runt.snap.getNodeByBuid(refsbuid)
|
|
2104
|
-
yield pivo, path.fork(pivo)
|
|
2110
|
+
yield pivo, path.fork(pivo, {'type': 'prop', 'prop': prop, 'reverse': True})
|
|
2105
2111
|
|
|
2106
2112
|
class N2WalkNPivo(PivotIn):
|
|
2107
2113
|
|
|
@@ -2118,7 +2124,7 @@ class N2WalkNPivo(PivotIn):
|
|
|
2118
2124
|
async for (verb, iden) in node.iterEdgesN2():
|
|
2119
2125
|
wnode = await runt.snap.getNodeByBuid(s_common.uhex(iden))
|
|
2120
2126
|
if wnode is not None:
|
|
2121
|
-
yield wnode, path.fork(wnode)
|
|
2127
|
+
yield wnode, path.fork(wnode, {'type': 'edge', 'verb': verb, 'reverse': True})
|
|
2122
2128
|
|
|
2123
2129
|
class PivotInFrom(PivotOper):
|
|
2124
2130
|
'''
|
|
@@ -2137,18 +2143,19 @@ class PivotInFrom(PivotOper):
|
|
|
2137
2143
|
if isinstance(form.type, s_types.Edge):
|
|
2138
2144
|
|
|
2139
2145
|
full = form.name + ':n2'
|
|
2140
|
-
|
|
2146
|
+
link = {'type': 'prop', 'prop': 'n2', 'reverse': True}
|
|
2141
2147
|
async for node, path in genr:
|
|
2142
2148
|
|
|
2143
2149
|
if self.isjoin:
|
|
2144
2150
|
yield node, path
|
|
2145
2151
|
|
|
2146
2152
|
async for pivo in runt.snap.nodesByPropValu(full, '=', node.ndef, norm=False):
|
|
2147
|
-
yield pivo, path.fork(pivo)
|
|
2153
|
+
yield pivo, path.fork(pivo, link)
|
|
2148
2154
|
|
|
2149
2155
|
return
|
|
2150
2156
|
|
|
2151
2157
|
# edge <- form
|
|
2158
|
+
link = {'type': 'prop', 'prop': 'n1', 'reverse': True}
|
|
2152
2159
|
async for node, path in genr:
|
|
2153
2160
|
|
|
2154
2161
|
if self.isjoin:
|
|
@@ -2168,7 +2175,7 @@ class PivotInFrom(PivotOper):
|
|
|
2168
2175
|
if pivo is None:
|
|
2169
2176
|
continue
|
|
2170
2177
|
|
|
2171
|
-
yield pivo, path.fork(pivo)
|
|
2178
|
+
yield pivo, path.fork(pivo, link)
|
|
2172
2179
|
|
|
2173
2180
|
class FormPivot(PivotOper):
|
|
2174
2181
|
'''
|
|
@@ -2181,8 +2188,9 @@ class FormPivot(PivotOper):
|
|
|
2181
2188
|
if isinstance(prop.type, s_types.Ndef):
|
|
2182
2189
|
|
|
2183
2190
|
async def pgenr(node, strict=True):
|
|
2191
|
+
link = {'type': 'prop', 'prop': prop.name, 'reverse': True}
|
|
2184
2192
|
async for pivo in runt.snap.nodesByPropValu(prop.full, '=', node.ndef, norm=False):
|
|
2185
|
-
yield pivo
|
|
2193
|
+
yield pivo, link
|
|
2186
2194
|
|
|
2187
2195
|
elif not prop.isform:
|
|
2188
2196
|
|
|
@@ -2200,8 +2208,9 @@ class FormPivot(PivotOper):
|
|
|
2200
2208
|
norm = prop.typehash is not node.form.typehash
|
|
2201
2209
|
ngenr = runt.snap.nodesByPropValu(prop.full, '=', node.ndef[1], norm=norm)
|
|
2202
2210
|
|
|
2211
|
+
link = {'type': 'prop', 'prop': prop.name, 'reverse': True}
|
|
2203
2212
|
async for pivo in ngenr:
|
|
2204
|
-
yield pivo
|
|
2213
|
+
yield pivo, link
|
|
2205
2214
|
|
|
2206
2215
|
# if dest form is a subtype of a graph "edge", use N1 automatically
|
|
2207
2216
|
elif isinstance(prop.type, s_types.Edge):
|
|
@@ -2209,8 +2218,9 @@ class FormPivot(PivotOper):
|
|
|
2209
2218
|
full = prop.name + ':n1'
|
|
2210
2219
|
|
|
2211
2220
|
async def pgenr(node, strict=True):
|
|
2221
|
+
link = {'type': 'prop', 'prop': 'n1', 'reverse': True}
|
|
2212
2222
|
async for pivo in runt.snap.nodesByPropValu(full, '=', node.ndef, norm=False):
|
|
2213
|
-
yield pivo
|
|
2223
|
+
yield pivo, link
|
|
2214
2224
|
|
|
2215
2225
|
else:
|
|
2216
2226
|
# form -> form pivot is nonsensical. Lets help out...
|
|
@@ -2222,8 +2232,9 @@ class FormPivot(PivotOper):
|
|
|
2222
2232
|
|
|
2223
2233
|
# <syn:tag> -> <form> is "from tags to nodes" pivot
|
|
2224
2234
|
if node.form.name == 'syn:tag' and prop.isform:
|
|
2235
|
+
link = {'type': 'tag', 'tag': node.ndef[1], 'reverse': True}
|
|
2225
2236
|
async for pivo in runt.snap.nodesByTag(node.ndef[1], form=prop.name):
|
|
2226
|
-
yield pivo
|
|
2237
|
+
yield pivo, link
|
|
2227
2238
|
|
|
2228
2239
|
return
|
|
2229
2240
|
|
|
@@ -2236,7 +2247,7 @@ class FormPivot(PivotOper):
|
|
|
2236
2247
|
|
|
2237
2248
|
pivo = await runt.snap.getNodeByNdef(node.get('n2'))
|
|
2238
2249
|
if pivo:
|
|
2239
|
-
yield pivo
|
|
2250
|
+
yield pivo, {'type': 'prop', 'prop': 'n2'}
|
|
2240
2251
|
|
|
2241
2252
|
return
|
|
2242
2253
|
|
|
@@ -2254,8 +2265,9 @@ class FormPivot(PivotOper):
|
|
|
2254
2265
|
|
|
2255
2266
|
refsvalu = node.get(refsname)
|
|
2256
2267
|
if refsvalu is not None:
|
|
2268
|
+
link = {'type': 'prop', 'prop': refsname}
|
|
2257
2269
|
async for pivo in runt.snap.nodesByPropValu(refsform, '=', refsvalu, norm=False):
|
|
2258
|
-
yield pivo
|
|
2270
|
+
yield pivo, link
|
|
2259
2271
|
|
|
2260
2272
|
for refsname, refsform in refs.get('array'):
|
|
2261
2273
|
|
|
@@ -2266,9 +2278,10 @@ class FormPivot(PivotOper):
|
|
|
2266
2278
|
|
|
2267
2279
|
refsvalu = node.get(refsname)
|
|
2268
2280
|
if refsvalu is not None:
|
|
2281
|
+
link = {'type': 'prop', 'prop': refsname}
|
|
2269
2282
|
for refselem in refsvalu:
|
|
2270
2283
|
async for pivo in runt.snap.nodesByPropValu(destform.name, '=', refselem, norm=False):
|
|
2271
|
-
yield pivo
|
|
2284
|
+
yield pivo, link
|
|
2272
2285
|
|
|
2273
2286
|
for refsname in refs.get('ndef'):
|
|
2274
2287
|
|
|
@@ -2278,17 +2291,18 @@ class FormPivot(PivotOper):
|
|
|
2278
2291
|
if refsvalu is not None and refsvalu[0] == destform.name:
|
|
2279
2292
|
pivo = await runt.snap.getNodeByNdef(refsvalu)
|
|
2280
2293
|
if pivo is not None:
|
|
2281
|
-
yield pivo
|
|
2294
|
+
yield pivo, {'type': 'prop', 'prop': refsname}
|
|
2282
2295
|
|
|
2283
2296
|
for refsname in refs.get('ndefarray'):
|
|
2284
2297
|
|
|
2285
2298
|
found = True
|
|
2286
2299
|
|
|
2287
2300
|
if (refsvalu := node.get(refsname)) is not None:
|
|
2301
|
+
link = {'type': 'prop', 'prop': refsname}
|
|
2288
2302
|
for aval in refsvalu:
|
|
2289
2303
|
if aval[0] == destform.name:
|
|
2290
2304
|
if (pivo := await runt.snap.getNodeByNdef(aval)) is not None:
|
|
2291
|
-
yield pivo
|
|
2305
|
+
yield pivo, link
|
|
2292
2306
|
|
|
2293
2307
|
#########################################################################
|
|
2294
2308
|
# reverse "-> form" pivots (ie inet:fqdn -> inet:dns:a)
|
|
@@ -2303,8 +2317,9 @@ class FormPivot(PivotOper):
|
|
|
2303
2317
|
found = True
|
|
2304
2318
|
|
|
2305
2319
|
refsprop = destform.props.get(refsname)
|
|
2320
|
+
link = {'type': 'prop', 'prop': refsname, 'reverse': True}
|
|
2306
2321
|
async for pivo in runt.snap.nodesByPropValu(refsprop.full, '=', node.ndef[1], norm=False):
|
|
2307
|
-
yield pivo
|
|
2322
|
+
yield pivo, link
|
|
2308
2323
|
|
|
2309
2324
|
# "reverse" array references...
|
|
2310
2325
|
for refsname, refsform in refs.get('array'):
|
|
@@ -2315,8 +2330,9 @@ class FormPivot(PivotOper):
|
|
|
2315
2330
|
found = True
|
|
2316
2331
|
|
|
2317
2332
|
destprop = destform.props.get(refsname)
|
|
2333
|
+
link = {'type': 'prop', 'prop': refsname, 'reverse': True}
|
|
2318
2334
|
async for pivo in runt.snap.nodesByPropArray(destprop.full, '=', node.ndef[1], norm=False):
|
|
2319
|
-
yield pivo
|
|
2335
|
+
yield pivo, link
|
|
2320
2336
|
|
|
2321
2337
|
# "reverse" ndef references...
|
|
2322
2338
|
for refsname in refs.get('ndef'):
|
|
@@ -2324,16 +2340,18 @@ class FormPivot(PivotOper):
|
|
|
2324
2340
|
found = True
|
|
2325
2341
|
|
|
2326
2342
|
refsprop = destform.props.get(refsname)
|
|
2343
|
+
link = {'type': 'prop', 'prop': refsname, 'reverse': True}
|
|
2327
2344
|
async for pivo in runt.snap.nodesByPropValu(refsprop.full, '=', node.ndef, norm=False):
|
|
2328
|
-
yield pivo
|
|
2345
|
+
yield pivo, link
|
|
2329
2346
|
|
|
2330
2347
|
for refsname in refs.get('ndefarray'):
|
|
2331
2348
|
|
|
2332
2349
|
found = True
|
|
2333
2350
|
|
|
2334
2351
|
refsprop = destform.props.get(refsname)
|
|
2352
|
+
link = {'type': 'prop', 'prop': refsname, 'reverse': True}
|
|
2335
2353
|
async for pivo in runt.snap.nodesByPropArray(refsprop.full, '=', node.ndef, norm=False):
|
|
2336
|
-
yield pivo
|
|
2354
|
+
yield pivo, link
|
|
2337
2355
|
|
|
2338
2356
|
if strict and not found:
|
|
2339
2357
|
mesg = f'No pivot found for {node.form.name} -> {destform.name}.'
|
|
@@ -2361,8 +2379,8 @@ class FormPivot(PivotOper):
|
|
|
2361
2379
|
|
|
2362
2380
|
async def listpivot(node):
|
|
2363
2381
|
for pgenr in pgenrs:
|
|
2364
|
-
async for pivo in pgenr(node, strict=False):
|
|
2365
|
-
yield pivo
|
|
2382
|
+
async for pivo, valu in pgenr(node, strict=False):
|
|
2383
|
+
yield pivo, valu
|
|
2366
2384
|
|
|
2367
2385
|
return listpivot
|
|
2368
2386
|
|
|
@@ -2383,8 +2401,8 @@ class FormPivot(PivotOper):
|
|
|
2383
2401
|
yield node, path
|
|
2384
2402
|
|
|
2385
2403
|
try:
|
|
2386
|
-
async for pivo in pgenr(node):
|
|
2387
|
-
yield pivo, path.fork(pivo)
|
|
2404
|
+
async for pivo, link in pgenr(node):
|
|
2405
|
+
yield pivo, path.fork(pivo, link)
|
|
2388
2406
|
except (s_exc.BadTypeValu, s_exc.BadLiftValu) as e:
|
|
2389
2407
|
if not warned:
|
|
2390
2408
|
logger.warning(f'Caught error during pivot: {e.items()}')
|
|
@@ -2420,11 +2438,12 @@ class PropPivotOut(PivotOper):
|
|
|
2420
2438
|
await asyncio.sleep(0)
|
|
2421
2439
|
continue
|
|
2422
2440
|
|
|
2441
|
+
link = {'type': 'prop', 'prop': prop.name}
|
|
2423
2442
|
if prop.type.isarray:
|
|
2424
2443
|
if isinstance(prop.type.arraytype, s_types.Ndef):
|
|
2425
2444
|
for item in valu:
|
|
2426
2445
|
if (pivo := await runt.snap.getNodeByNdef(item)) is not None:
|
|
2427
|
-
yield pivo, path.fork(pivo)
|
|
2446
|
+
yield pivo, path.fork(pivo, link)
|
|
2428
2447
|
continue
|
|
2429
2448
|
|
|
2430
2449
|
fname = prop.type.arraytype.name
|
|
@@ -2437,7 +2456,7 @@ class PropPivotOut(PivotOper):
|
|
|
2437
2456
|
|
|
2438
2457
|
for item in valu:
|
|
2439
2458
|
async for pivo in runt.snap.nodesByPropValu(fname, '=', item, norm=False):
|
|
2440
|
-
yield pivo, path.fork(pivo)
|
|
2459
|
+
yield pivo, path.fork(pivo, link)
|
|
2441
2460
|
|
|
2442
2461
|
continue
|
|
2443
2462
|
|
|
@@ -2448,7 +2467,7 @@ class PropPivotOut(PivotOper):
|
|
|
2448
2467
|
if pivo is None:
|
|
2449
2468
|
logger.warning(f'Missing node corresponding to ndef {valu}')
|
|
2450
2469
|
continue
|
|
2451
|
-
yield pivo, path.fork(pivo)
|
|
2470
|
+
yield pivo, path.fork(pivo, link)
|
|
2452
2471
|
continue
|
|
2453
2472
|
|
|
2454
2473
|
# :prop -> *
|
|
@@ -2465,7 +2484,7 @@ class PropPivotOut(PivotOper):
|
|
|
2465
2484
|
# A node explicitly deleted in the graph or missing from a underlying layer
|
|
2466
2485
|
# could cause this lift to return None.
|
|
2467
2486
|
if pivo:
|
|
2468
|
-
yield pivo, path.fork(pivo)
|
|
2487
|
+
yield pivo, path.fork(pivo, link)
|
|
2469
2488
|
|
|
2470
2489
|
|
|
2471
2490
|
class PropPivot(PivotOper):
|
|
@@ -2477,6 +2496,9 @@ class PropPivot(PivotOper):
|
|
|
2477
2496
|
|
|
2478
2497
|
async def pgenr(node, srcprop, valu, strict=True):
|
|
2479
2498
|
|
|
2499
|
+
link = {'type': 'prop', 'prop': srcprop.name}
|
|
2500
|
+
if not prop.isform:
|
|
2501
|
+
link['dest'] = prop.full
|
|
2480
2502
|
# pivoting from an array prop to a non-array prop needs an extra loop
|
|
2481
2503
|
if srcprop.type.isarray and not prop.type.isarray:
|
|
2482
2504
|
if isinstance(srcprop.type.arraytype, s_types.Ndef) and prop.isform:
|
|
@@ -2485,13 +2507,13 @@ class PropPivot(PivotOper):
|
|
|
2485
2507
|
continue
|
|
2486
2508
|
|
|
2487
2509
|
if (pivo := await runt.snap.getNodeByNdef(aval)) is not None:
|
|
2488
|
-
yield pivo
|
|
2510
|
+
yield pivo, link
|
|
2489
2511
|
return
|
|
2490
2512
|
|
|
2491
2513
|
norm = srcprop.arraytypehash is not prop.typehash
|
|
2492
2514
|
for arrayval in valu:
|
|
2493
2515
|
async for pivo in runt.snap.nodesByPropValu(prop.full, '=', arrayval, norm=norm):
|
|
2494
|
-
yield pivo
|
|
2516
|
+
yield pivo, link
|
|
2495
2517
|
|
|
2496
2518
|
return
|
|
2497
2519
|
|
|
@@ -2503,7 +2525,7 @@ class PropPivot(PivotOper):
|
|
|
2503
2525
|
if pivo is None:
|
|
2504
2526
|
await runt.snap.warn(f'Missing node corresponding to ndef {valu}', log=False, ndef=valu)
|
|
2505
2527
|
return
|
|
2506
|
-
yield pivo
|
|
2528
|
+
yield pivo, link
|
|
2507
2529
|
|
|
2508
2530
|
return
|
|
2509
2531
|
|
|
@@ -2515,7 +2537,7 @@ class PropPivot(PivotOper):
|
|
|
2515
2537
|
genr = runt.snap.nodesByPropValu(prop.full, '=', valu, norm=norm)
|
|
2516
2538
|
|
|
2517
2539
|
async for pivo in genr:
|
|
2518
|
-
yield pivo
|
|
2540
|
+
yield pivo, link
|
|
2519
2541
|
|
|
2520
2542
|
return pgenr
|
|
2521
2543
|
|
|
@@ -2569,8 +2591,8 @@ class PropPivot(PivotOper):
|
|
|
2569
2591
|
continue
|
|
2570
2592
|
|
|
2571
2593
|
try:
|
|
2572
|
-
async for pivo in pgenr(node, srcprop, valu):
|
|
2573
|
-
yield pivo, path.fork(pivo)
|
|
2594
|
+
async for pivo, link in pgenr(node, srcprop, valu):
|
|
2595
|
+
yield pivo, path.fork(pivo, link)
|
|
2574
2596
|
|
|
2575
2597
|
except (s_exc.BadTypeValu, s_exc.BadLiftValu) as e:
|
|
2576
2598
|
if not warned:
|
|
@@ -4218,19 +4240,20 @@ class EditUnivDel(Edit):
|
|
|
4218
4240
|
|
|
4219
4241
|
class N1Walk(Oper):
|
|
4220
4242
|
|
|
4221
|
-
def __init__(self, astinfo, kids=(), isjoin=False):
|
|
4243
|
+
def __init__(self, astinfo, kids=(), isjoin=False, reverse=False):
|
|
4222
4244
|
Oper.__init__(self, astinfo, kids=kids)
|
|
4223
4245
|
self.isjoin = isjoin
|
|
4246
|
+
self.reverse = reverse
|
|
4224
4247
|
|
|
4225
4248
|
def repr(self):
|
|
4226
4249
|
return f'{self.__class__.__name__}: {self.kids}, isjoin={self.isjoin}'
|
|
4227
4250
|
|
|
4228
4251
|
async def walkNodeEdges(self, runt, node, verb=None):
|
|
4229
|
-
async for
|
|
4252
|
+
async for verb, iden in node.iterEdgesN1(verb=verb):
|
|
4230
4253
|
buid = s_common.uhex(iden)
|
|
4231
4254
|
walknode = await runt.snap.getNodeByBuid(buid)
|
|
4232
4255
|
if walknode is not None:
|
|
4233
|
-
yield walknode
|
|
4256
|
+
yield verb, walknode
|
|
4234
4257
|
|
|
4235
4258
|
def buildfilter(self, runt, destforms, cmpr):
|
|
4236
4259
|
|
|
@@ -4330,21 +4353,28 @@ class N1Walk(Oper):
|
|
|
4330
4353
|
if verb == '*':
|
|
4331
4354
|
verb = None
|
|
4332
4355
|
|
|
4333
|
-
async for walknode in self.walkNodeEdges(runt, node, verb=verb):
|
|
4356
|
+
async for verbname, walknode in self.walkNodeEdges(runt, node, verb=verb):
|
|
4334
4357
|
|
|
4335
4358
|
if destfilt and not await destfilt(walknode, path, cmprvalu):
|
|
4336
4359
|
continue
|
|
4337
4360
|
|
|
4338
|
-
|
|
4361
|
+
link = {'type': 'edge', 'verb': verbname}
|
|
4362
|
+
if self.reverse:
|
|
4363
|
+
link['reverse'] = True
|
|
4364
|
+
|
|
4365
|
+
yield walknode, path.fork(walknode, link)
|
|
4339
4366
|
|
|
4340
4367
|
class N2Walk(N1Walk):
|
|
4341
4368
|
|
|
4369
|
+
def __init__(self, astinfo, kids=(), isjoin=False):
|
|
4370
|
+
N1Walk.__init__(self, astinfo, kids=kids, isjoin=isjoin, reverse=True)
|
|
4371
|
+
|
|
4342
4372
|
async def walkNodeEdges(self, runt, node, verb=None):
|
|
4343
|
-
async for
|
|
4373
|
+
async for verb, iden in node.iterEdgesN2(verb=verb):
|
|
4344
4374
|
buid = s_common.uhex(iden)
|
|
4345
4375
|
walknode = await runt.snap.getNodeByBuid(buid)
|
|
4346
4376
|
if walknode is not None:
|
|
4347
|
-
yield walknode
|
|
4377
|
+
yield verb, walknode
|
|
4348
4378
|
|
|
4349
4379
|
class EditEdgeAdd(Edit):
|
|
4350
4380
|
|
synapse/lib/layer.py
CHANGED
synapse/lib/lmdbslab.py
CHANGED
|
@@ -834,6 +834,7 @@ class Slab(s_base.Base):
|
|
|
834
834
|
'xactops': len(slab.xactops),
|
|
835
835
|
'mapsize': slab.mapsize,
|
|
836
836
|
'readonly': slab.readonly,
|
|
837
|
+
'readahead': slab.readahead,
|
|
837
838
|
'lockmemory': slab.lockmemory,
|
|
838
839
|
'recovering': slab.recovering,
|
|
839
840
|
'maxsize': slab.maxsize,
|
|
@@ -889,6 +890,7 @@ class Slab(s_base.Base):
|
|
|
889
890
|
self.growsize = opts.pop('growsize', self.DEFAULT_GROWSIZE)
|
|
890
891
|
|
|
891
892
|
self.readonly = opts.get('readonly', False)
|
|
893
|
+
self.readahead = opts.get('readahead', True)
|
|
892
894
|
self.lockmemory = opts.pop('lockmemory', False)
|
|
893
895
|
|
|
894
896
|
if self.lockmemory:
|
synapse/lib/modelrev.py
CHANGED
|
@@ -8,7 +8,7 @@ import synapse.lib.layer as s_layer
|
|
|
8
8
|
|
|
9
9
|
logger = logging.getLogger(__name__)
|
|
10
10
|
|
|
11
|
-
maxvers = (0, 2,
|
|
11
|
+
maxvers = (0, 2, 27)
|
|
12
12
|
|
|
13
13
|
class ModelRev:
|
|
14
14
|
|
|
@@ -40,6 +40,7 @@ class ModelRev:
|
|
|
40
40
|
((0, 2, 24), self.revModel_0_2_24),
|
|
41
41
|
((0, 2, 25), self.revModel_0_2_25),
|
|
42
42
|
((0, 2, 26), self.revModel_0_2_26),
|
|
43
|
+
((0, 2, 27), self.revModel_0_2_27),
|
|
43
44
|
)
|
|
44
45
|
|
|
45
46
|
async def _uniqSortArray(self, todoprops, layers):
|
|
@@ -783,6 +784,9 @@ class ModelRev:
|
|
|
783
784
|
logger.info(f'Updating ndef indexing for {name}')
|
|
784
785
|
await self._updatePropStortype(layers, prop.full)
|
|
785
786
|
|
|
787
|
+
async def revModel_0_2_27(self, layers):
|
|
788
|
+
await self._normPropValu(layers, 'it:dev:repo:commit:id')
|
|
789
|
+
|
|
786
790
|
async def runStorm(self, text, opts=None):
|
|
787
791
|
'''
|
|
788
792
|
Run storm code in a schedcoro and log the output messages.
|
synapse/lib/node.py
CHANGED
|
@@ -708,11 +708,16 @@ class Path:
|
|
|
708
708
|
'''
|
|
709
709
|
A path context tracked through the storm runtime.
|
|
710
710
|
'''
|
|
711
|
-
def __init__(self, vars, nodes):
|
|
711
|
+
def __init__(self, vars, nodes, links=None):
|
|
712
712
|
|
|
713
713
|
self.node = None
|
|
714
714
|
self.nodes = nodes
|
|
715
715
|
|
|
716
|
+
if links is not None:
|
|
717
|
+
self.links = links
|
|
718
|
+
else:
|
|
719
|
+
self.links = []
|
|
720
|
+
|
|
716
721
|
if len(nodes):
|
|
717
722
|
self.node = nodes[-1]
|
|
718
723
|
|
|
@@ -765,19 +770,24 @@ class Path:
|
|
|
765
770
|
info = await s_stormtypes.toprim(dict(self.metadata))
|
|
766
771
|
if path:
|
|
767
772
|
info['nodes'] = [node.iden() for node in self.nodes]
|
|
773
|
+
|
|
768
774
|
return info
|
|
769
775
|
|
|
770
|
-
def fork(self, node):
|
|
776
|
+
def fork(self, node, link):
|
|
777
|
+
|
|
778
|
+
links = list(self.links)
|
|
779
|
+
if self.node is not None and link is not None:
|
|
780
|
+
links.append((self.node.iden(), link))
|
|
771
781
|
|
|
772
782
|
nodes = list(self.nodes)
|
|
773
783
|
nodes.append(node)
|
|
774
784
|
|
|
775
|
-
path = Path(self.vars.copy(), nodes)
|
|
785
|
+
path = Path(self.vars.copy(), nodes, links=links)
|
|
776
786
|
|
|
777
787
|
return path
|
|
778
788
|
|
|
779
789
|
def clone(self):
|
|
780
|
-
path = Path(copy.copy(self.vars), copy.copy(self.nodes))
|
|
790
|
+
path = Path(copy.copy(self.vars), copy.copy(self.nodes), copy.copy(self.links))
|
|
781
791
|
path.frames = [v.copy() for v in self.frames]
|
|
782
792
|
return path
|
|
783
793
|
|