ricgraph 3.4__tar.gz → 3.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ricgraph-3.4/ricgraph.egg-info → ricgraph-3.5}/PKG-INFO +8 -8
- {ricgraph-3.4 → ricgraph-3.5}/README-PyPI.md +7 -7
- {ricgraph-3.4 → ricgraph-3.5}/README.md +7 -8
- {ricgraph-3.4 → ricgraph-3.5}/pyproject.toml +1 -1
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph/ricgraph.py +14 -14
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph/ricgraph_cache.py +70 -72
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph/ricgraph_constants.py +67 -14
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph/ricgraph_cypher.py +146 -38
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph/ricgraph_file.py +25 -19
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph/ricgraph_graphdb.py +68 -20
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph/ricgraph_harvest.py +25 -25
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph/ricgraph_researchinfo.py +5 -3
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph/ricgraph_restapi.py +39 -5
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph/ricgraph_utils.py +113 -13
- {ricgraph-3.4 → ricgraph-3.5/ricgraph.egg-info}/PKG-INFO +8 -8
- {ricgraph-3.4 → ricgraph-3.5}/LICENSE +0 -0
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph/__init__.py +0 -0
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph.egg-info/SOURCES.txt +0 -0
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph.egg-info/dependency_links.txt +0 -0
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph.egg-info/requires.txt +0 -0
- {ricgraph-3.4 → ricgraph-3.5}/ricgraph.egg-info/top_level.txt +0 -0
- {ricgraph-3.4 → ricgraph-3.5}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ricgraph
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.5
|
|
4
4
|
Summary: Ricgraph - Research in context graph
|
|
5
5
|
Author: Rik D.T. Janssen
|
|
6
6
|
License-Expression: MIT
|
|
@@ -67,19 +67,19 @@ adapted to new application areas.
|
|
|
67
67
|
|
|
68
68
|
Ricgraph is unique because it allows you to:
|
|
69
69
|
|
|
70
|
-
* Explore collaborations between (sub-)organizations
|
|
71
|
-
* Explore
|
|
72
|
-
* Enhance research information systems
|
|
70
|
+
* Explore collaborations between (sub-)organizations;
|
|
71
|
+
* Explore open science;
|
|
72
|
+
* Enhance research information systems.
|
|
73
73
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
74
|
+
One of the other strengths of Ricgraph is that anyone with
|
|
75
|
+
Python knowledge can easily extend it, allowing you to test
|
|
76
|
+
your own ideas or create your own applications.
|
|
77
77
|
|
|
78
78
|
You can use Ricgraph either as a standalone tool,
|
|
79
79
|
or as a component that provides a graph based data structure with
|
|
80
80
|
a number of handy tools.
|
|
81
81
|
Ricgraph has been developed for the application area
|
|
82
|
-
research information
|
|
82
|
+
research information,
|
|
83
83
|
and it can be used for other application areas that require
|
|
84
84
|
relations between items from various source systems.
|
|
85
85
|
|
|
@@ -39,19 +39,19 @@ adapted to new application areas.
|
|
|
39
39
|
|
|
40
40
|
Ricgraph is unique because it allows you to:
|
|
41
41
|
|
|
42
|
-
* Explore collaborations between (sub-)organizations
|
|
43
|
-
* Explore
|
|
44
|
-
* Enhance research information systems
|
|
42
|
+
* Explore collaborations between (sub-)organizations;
|
|
43
|
+
* Explore open science;
|
|
44
|
+
* Enhance research information systems.
|
|
45
45
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
46
|
+
One of the other strengths of Ricgraph is that anyone with
|
|
47
|
+
Python knowledge can easily extend it, allowing you to test
|
|
48
|
+
your own ideas or create your own applications.
|
|
49
49
|
|
|
50
50
|
You can use Ricgraph either as a standalone tool,
|
|
51
51
|
or as a component that provides a graph based data structure with
|
|
52
52
|
a number of handy tools.
|
|
53
53
|
Ricgraph has been developed for the application area
|
|
54
|
-
research information
|
|
54
|
+
research information,
|
|
55
55
|
and it can be used for other application areas that require
|
|
56
56
|
relations between items from various source systems.
|
|
57
57
|
|
|
@@ -59,14 +59,13 @@ adapted to new application areas.
|
|
|
59
59
|
|
|
60
60
|
Ricgraph is unique because it allows you to:
|
|
61
61
|
|
|
62
|
-
* [Explore collaborations between (sub-)organizations
|
|
63
|
-
|
|
64
|
-
* [
|
|
65
|
-
* [Enhance research information systems with Ricgraph](https://www.ricgraph.eu/enhancing-with-ricgraph.html).
|
|
62
|
+
* [Explore collaborations between (sub-)organizations](https://www.ricgraph.eu/collaborations-with-ricgraph.html).
|
|
63
|
+
* [Explore open science](https://www.ricgraph.eu/open-science-with-ricgraph.html).
|
|
64
|
+
* [Enhance research information systems](https://www.ricgraph.eu/enhancing-with-ricgraph.html).
|
|
66
65
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
66
|
+
One of the other strengths of Ricgraph is that anyone with
|
|
67
|
+
Python knowledge can easily extend it, allowing you to test
|
|
68
|
+
your own ideas or create your own applications.
|
|
70
69
|
|
|
71
70
|
You can use Ricgraph either as a standalone tool,
|
|
72
71
|
or as a component that provides a graph based data structure with
|
|
@@ -177,4 +176,4 @@ please let me know by creating a GitHub *Issue* at the top left of that page.
|
|
|
177
176
|
<!-- Do not change the lines below. The version number will be
|
|
178
177
|
modified by script maintenance/update_version.bash -->
|
|
179
178
|
This README.md file is part of the
|
|
180
|
-
documentation for version 3.
|
|
179
|
+
documentation for version 3.5 of Ricgraph - Research in context graph.
|
|
@@ -120,7 +120,7 @@ __package__ = 'Ricgraph'
|
|
|
120
120
|
# __version__ should be placed here instead of in __init__.py.
|
|
121
121
|
# If not here, it cannot be found when executing something like
|
|
122
122
|
# PYTHONPATH=../ricgraph ../bin/python ricgraph_explorer.py
|
|
123
|
-
__version__ = '3.
|
|
123
|
+
__version__ = '3.5'
|
|
124
124
|
|
|
125
125
|
|
|
126
126
|
# ##############################################################################
|
|
@@ -147,9 +147,9 @@ def update_nodes_df(nodes: DataFrame) -> None:
|
|
|
147
147
|
"""
|
|
148
148
|
nodes_clean = nodes.copy(deep=True)
|
|
149
149
|
# Ensure that all '' values are NaN, so that those rows can be easily removed with dropna()
|
|
150
|
-
nodes_clean.replace('', nan
|
|
151
|
-
nodes_clean.dropna(axis=0, how='any'
|
|
152
|
-
nodes_clean.drop_duplicates(keep='first',
|
|
150
|
+
nodes_clean = nodes_clean.replace('', nan)
|
|
151
|
+
nodes_clean = nodes_clean.dropna(axis=0, how='any')
|
|
152
|
+
nodes_clean = nodes_clean.drop_duplicates(keep='first', ignore_index=True)
|
|
153
153
|
|
|
154
154
|
print('There are ' + str(len(nodes_clean)) + ' nodes ('
|
|
155
155
|
+ timestamp() + '), updating node:')
|
|
@@ -248,7 +248,7 @@ def create_nodepairs_and_edges_params(name1: dict | str, category1: dict | str,
|
|
|
248
248
|
index=[0])
|
|
249
249
|
else:
|
|
250
250
|
left_and_right_nodepairs = DataFrame(data=value1).copy(deep=True)
|
|
251
|
-
left_and_right_nodepairs.rename(columns={value1.name: 'value1'}
|
|
251
|
+
left_and_right_nodepairs = left_and_right_nodepairs.rename(columns={value1.name: 'value1'})
|
|
252
252
|
|
|
253
253
|
# This makes programming more efficient
|
|
254
254
|
allowed_properties = get_ricgraph_properties_additional() + get_ricgraph_properties_standard()
|
|
@@ -268,14 +268,14 @@ def create_nodepairs_and_edges_params(name1: dict | str, category1: dict | str,
|
|
|
268
268
|
left_and_right_nodepairs[prop_name + '1'] = other_value
|
|
269
269
|
else:
|
|
270
270
|
left_and_right_nodepairs = concat([left_and_right_nodepairs, other_value], axis='columns')
|
|
271
|
-
left_and_right_nodepairs.rename(columns={other_value.name: prop_name + '1'}
|
|
271
|
+
left_and_right_nodepairs = left_and_right_nodepairs.rename(columns={other_value.name: prop_name + '1'})
|
|
272
272
|
|
|
273
273
|
if prop_name + '2' == other_name:
|
|
274
274
|
if isinstance(other_value, str):
|
|
275
275
|
left_and_right_nodepairs[prop_name + '2'] = other_value
|
|
276
276
|
else:
|
|
277
277
|
left_and_right_nodepairs = concat([left_and_right_nodepairs, other_value], axis='columns')
|
|
278
|
-
left_and_right_nodepairs.rename(columns={other_value.name: prop_name + '2'}
|
|
278
|
+
left_and_right_nodepairs = left_and_right_nodepairs.rename(columns={other_value.name: prop_name + '2'})
|
|
279
279
|
|
|
280
280
|
# Reorder a bit, starting with the last, only the name-category-value property, since they will surely exist
|
|
281
281
|
left_and_right_nodepairs.insert(0, 'value2', left_and_right_nodepairs.pop('value2'))
|
|
@@ -314,10 +314,10 @@ def unify_personal_identifiers(personal_identifiers: DataFrame,
|
|
|
314
314
|
identifiers = personal_identifiers[[column1, column2]].copy(deep=True)
|
|
315
315
|
|
|
316
316
|
# Ensure that all '' values are NaN, so that those rows can be easily removed with dropna()
|
|
317
|
-
identifiers.replace('', nan
|
|
317
|
+
identifiers = identifiers.replace('', nan)
|
|
318
318
|
# dropna(how='any'): drop row if one or more row values contain NaN
|
|
319
|
-
identifiers.dropna(axis=0, how='any'
|
|
320
|
-
identifiers.drop_duplicates(keep='first',
|
|
319
|
+
identifiers = identifiers.dropna(axis=0, how='any')
|
|
320
|
+
identifiers = identifiers.drop_duplicates(keep='first', ignore_index=True)
|
|
321
321
|
|
|
322
322
|
if get_ricgraph_nodeadd_mode() == 'strict':
|
|
323
323
|
# Remove rows which have a cell value that occurs more than once.
|
|
@@ -342,11 +342,11 @@ def unify_personal_identifiers(personal_identifiers: DataFrame,
|
|
|
342
342
|
# For more explanation, see file docs/ricgraph_install_configure.md,
|
|
343
343
|
# section RICGRAPH_NODEADD_MODE.
|
|
344
344
|
if column1 != 'FULL_NAME' and column2 != 'FULL_NAME':
|
|
345
|
-
identifiers.drop_duplicates(subset=column1, keep=False,
|
|
346
|
-
identifiers.drop_duplicates(subset=column2, keep=False,
|
|
345
|
+
identifiers = identifiers.drop_duplicates(subset=column1, keep=False, ignore_index=True)
|
|
346
|
+
identifiers = identifiers.drop_duplicates(subset=column2, keep=False, ignore_index=True)
|
|
347
347
|
|
|
348
|
-
identifiers.rename(columns={column1: 'value1',
|
|
349
|
-
|
|
348
|
+
identifiers = identifiers.rename(columns={column1: 'value1',
|
|
349
|
+
column2: 'value2'})
|
|
350
350
|
new_identifiers_columns = {'name1': column1,
|
|
351
351
|
'category1': PERSON_CATEGORY_PERSON,
|
|
352
352
|
'name2': column2,
|
|
@@ -37,31 +37,34 @@
|
|
|
37
37
|
# Original version Rik D.T. Janssen, December 2022.
|
|
38
38
|
# Updated Rik D.T. Janssen, February, March, September to December 2023.
|
|
39
39
|
# Updated Rik D.T. Janssen, February to June, September to December 2024.
|
|
40
|
-
# Updated Rik D.T. Janssen, January to June, October 2025.
|
|
40
|
+
# Updated Rik D.T. Janssen, January to June, October 2025, July 2026.
|
|
41
41
|
#
|
|
42
42
|
# ########################################################################
|
|
43
43
|
|
|
44
44
|
|
|
45
45
|
from sys import getsizeof
|
|
46
|
-
from typing import Optional, Tuple
|
|
47
|
-
from pymemcache.client.base import
|
|
48
|
-
from .ricgraph_constants import MAX_NODES_CACHE_KEY_ID
|
|
46
|
+
from typing import Optional, Tuple, Any
|
|
47
|
+
from pymemcache.client.base import PooledClient
|
|
48
|
+
from .ricgraph_constants import MAX_NODES_CACHE_KEY_ID, MEMCACHED_MAX_POOL_SIZE
|
|
49
49
|
from .ricgraph_utils import (get_configfile_key_memcached_parameters,
|
|
50
50
|
serialize_value, deserialize_value)
|
|
51
51
|
|
|
52
52
|
|
|
53
|
-
# This dict is used as a cache
|
|
53
|
+
# This dict is used as a cache.
|
|
54
|
+
# It may be used for node ids. If we have a node id, we can
|
|
54
55
|
# do a direct lookup for a node in O(1) in the graph database,
|
|
55
56
|
# instead of a search in O(log n).
|
|
56
|
-
#
|
|
57
|
-
|
|
57
|
+
# For nodes, the dict has the format: [Ricgraph _key]: [Node element_id].
|
|
58
|
+
# For all other types, it uses JSON, and it has the
|
|
59
|
+
# format: [key]: [value].
|
|
60
|
+
_ricgraph_cache = {}
|
|
58
61
|
|
|
59
62
|
# Global indicating whether Memcached is available.
|
|
60
63
|
_memcached_available = False
|
|
61
64
|
|
|
62
65
|
# Global for connection to Memcached.
|
|
63
66
|
# Type hint necessary to avoid PyCharm warning.
|
|
64
|
-
_memcached_client: Optional[
|
|
67
|
+
_memcached_client: Optional[PooledClient] = None
|
|
65
68
|
|
|
66
69
|
|
|
67
70
|
def memcached_open_connection() -> None:
|
|
@@ -84,8 +87,10 @@ def memcached_open_connection() -> None:
|
|
|
84
87
|
return
|
|
85
88
|
|
|
86
89
|
try:
|
|
87
|
-
|
|
88
|
-
|
|
90
|
+
# Do not use Client(), it is not thread safe.
|
|
91
|
+
memcached_client = PooledClient(server=(memcached_host, memcached_port),
|
|
92
|
+
max_pool_size=MEMCACHED_MAX_POOL_SIZE,
|
|
93
|
+
allow_unicode_keys=True)
|
|
89
94
|
|
|
90
95
|
# Test connection by setting and deleting a test key.
|
|
91
96
|
memcached_client.set(key='test_key', value=b'test', expire=1)
|
|
@@ -118,125 +123,119 @@ def memcached_check_available() -> bool:
|
|
|
118
123
|
return True
|
|
119
124
|
|
|
120
125
|
|
|
121
|
-
def
|
|
122
|
-
"""Create an entry in the cache
|
|
123
|
-
'id' in this sentence is the id assigned by de graph database.
|
|
126
|
+
def ricgraph_cache_item_create(key: str, value: Any) -> None:
|
|
127
|
+
"""Create an entry in the cache.
|
|
124
128
|
|
|
125
|
-
:param key:
|
|
126
|
-
:param
|
|
129
|
+
:param key: key of a cache item.
|
|
130
|
+
:param value: value of a cache item.
|
|
127
131
|
:return: None.
|
|
128
132
|
"""
|
|
129
|
-
global
|
|
133
|
+
global _ricgraph_cache, _memcached_available, _memcached_client
|
|
130
134
|
|
|
131
|
-
if key == '' or
|
|
135
|
+
if key == '' or value == '':
|
|
132
136
|
return
|
|
133
137
|
|
|
138
|
+
key_clean = key.replace(' ', '_')
|
|
139
|
+
value_serialized = serialize_value(value=value)
|
|
134
140
|
if memcached_check_available():
|
|
135
141
|
if _memcached_client is None:
|
|
136
142
|
return
|
|
137
|
-
# The docs are unclear whether I need to serialize the key if I
|
|
138
|
-
# use flag allow_unicode_keys=True in Client() above.
|
|
139
|
-
# So I do serialize to be sure. Since keys cannot have spaces,
|
|
140
|
-
# these are replaced first.
|
|
141
|
-
key_serialized = serialize_value(value=key.replace(' ', '_'))
|
|
142
|
-
elementid_serialized = serialize_value(value=elementid)
|
|
143
143
|
try:
|
|
144
|
-
_memcached_client.set(key=
|
|
145
|
-
value=
|
|
144
|
+
_memcached_client.set(key=key_clean,
|
|
145
|
+
value=value_serialized,
|
|
146
146
|
expire=0)
|
|
147
147
|
except:
|
|
148
|
-
print('
|
|
148
|
+
print('ricgraph_cache_item_create(): Warning, connection to Memcached lost, continuing...')
|
|
149
149
|
# Continue, hopefully the connection will come back soon.
|
|
150
150
|
return
|
|
151
151
|
|
|
152
|
-
if len(
|
|
153
|
-
|
|
152
|
+
if len(_ricgraph_cache) > MAX_NODES_CACHE_KEY_ID:
|
|
153
|
+
ricgraph_cache_empty()
|
|
154
154
|
|
|
155
155
|
# We use a 'dict', which does not allow for duplicates, so we
|
|
156
156
|
# do not need to check for duplicates.
|
|
157
157
|
# https://docs.python.org/3/library/stdtypes.html#mapping-types-dict
|
|
158
158
|
# tells that Python dict keys and values can have _almost_ any type.
|
|
159
|
-
#
|
|
160
|
-
|
|
161
|
-
|
|
159
|
+
# We serialize for symmetry with Memcached.
|
|
160
|
+
# print('Create: ' + str(key_clean) + ' -- ' + str(value_serialized))
|
|
161
|
+
_ricgraph_cache[key_clean] = value_serialized
|
|
162
162
|
return
|
|
163
163
|
|
|
164
164
|
|
|
165
|
-
def
|
|
166
|
-
"""Read an entry from the cache
|
|
167
|
-
'id' in this sentence is the id assigned by de graph database.
|
|
165
|
+
def ricgraph_cache_item_read(key: str) -> Any:
|
|
166
|
+
"""Read an entry from the cache.
|
|
168
167
|
|
|
169
|
-
:param key:
|
|
170
|
-
:return: The
|
|
168
|
+
:param key: key of a cache item.
|
|
169
|
+
:return: The value of the cache item, or '' if not present.
|
|
171
170
|
"""
|
|
172
|
-
global
|
|
171
|
+
global _ricgraph_cache, _memcached_available, _memcached_client
|
|
173
172
|
|
|
174
173
|
if key == '':
|
|
175
174
|
return ''
|
|
176
175
|
|
|
176
|
+
key_clean = key.replace(' ', '_')
|
|
177
177
|
if memcached_check_available():
|
|
178
178
|
if _memcached_client is None:
|
|
179
179
|
return ''
|
|
180
|
-
key_serialized = serialize_value(value=key.replace(' ', '_'))
|
|
181
180
|
try:
|
|
182
|
-
|
|
181
|
+
value_serialized = _memcached_client.get(key=key_clean)
|
|
183
182
|
except:
|
|
184
|
-
print('
|
|
183
|
+
print('ricgraph_cache_item_read(): Warning, connection to Memcached lost, continuing...')
|
|
185
184
|
# Continue, hopefully the connection will come back soon.
|
|
186
185
|
return ''
|
|
187
186
|
|
|
188
|
-
if
|
|
187
|
+
if value_serialized is None:
|
|
189
188
|
# Key not found.
|
|
190
189
|
return ''
|
|
191
190
|
else:
|
|
192
|
-
|
|
193
|
-
return
|
|
194
|
-
|
|
195
|
-
if
|
|
196
|
-
|
|
197
|
-
|
|
191
|
+
value = deserialize_value(serialized=value_serialized)
|
|
192
|
+
return value
|
|
193
|
+
|
|
194
|
+
if key_clean in _ricgraph_cache:
|
|
195
|
+
value_serialized = _ricgraph_cache[key_clean]
|
|
196
|
+
# print('Read: ' + str(key_clean) + ' -- ' + str(value_serialized))
|
|
197
|
+
value = deserialize_value(serialized=value_serialized)
|
|
198
|
+
return value
|
|
198
199
|
return ''
|
|
199
200
|
|
|
200
201
|
|
|
201
|
-
def
|
|
202
|
-
"""Delete a key 'key' from the cache
|
|
203
|
-
'id' in this sentence is the id assigned by de graph database.
|
|
202
|
+
def ricgraph_cache_item_delete_key(key: str) -> None:
|
|
203
|
+
"""Delete a key 'key' from the cache.
|
|
204
204
|
|
|
205
|
-
:param key:
|
|
205
|
+
:param key: key of a cache item.
|
|
206
206
|
:return: None.
|
|
207
207
|
"""
|
|
208
|
-
global
|
|
208
|
+
global _ricgraph_cache, _memcached_available, _memcached_client
|
|
209
209
|
|
|
210
210
|
if key == '':
|
|
211
211
|
return
|
|
212
212
|
|
|
213
|
+
key_clean = key.replace(' ', '_')
|
|
213
214
|
if memcached_check_available():
|
|
214
215
|
if _memcached_client is None:
|
|
215
216
|
return
|
|
216
|
-
key_serialized = serialize_value(value=key.replace(' ', '_'))
|
|
217
217
|
try:
|
|
218
218
|
# Works regardless whether the key is present or not.
|
|
219
|
-
_memcached_client.delete(key=
|
|
219
|
+
_memcached_client.delete(key=key_clean)
|
|
220
220
|
except:
|
|
221
|
-
print('
|
|
221
|
+
print('nodes_cache_item_delete_key(): Error, connection to Memcached lost, exiting...')
|
|
222
222
|
# Exit because the cache gets in an unexpected state:
|
|
223
223
|
# an element remains in the cache while it should not.
|
|
224
224
|
exit(1)
|
|
225
225
|
return
|
|
226
226
|
|
|
227
|
-
if
|
|
228
|
-
# print('Delete: ' + str(
|
|
229
|
-
|
|
227
|
+
if key_clean in _ricgraph_cache:
|
|
228
|
+
# print('Delete: ' + str(key_clean) + ' -- ' + _ricgraph_cache[key_clean])
|
|
229
|
+
_ricgraph_cache.pop(key_clean)
|
|
230
230
|
return
|
|
231
231
|
|
|
232
232
|
|
|
233
|
-
def
|
|
234
|
-
"""Empty the cache
|
|
235
|
-
'id' in this sentence is the id assigned by de graph database.
|
|
233
|
+
def ricgraph_cache_empty() -> None:
|
|
234
|
+
"""Empty the cache.
|
|
236
235
|
|
|
237
236
|
:return: None.
|
|
238
237
|
"""
|
|
239
|
-
global
|
|
238
|
+
global _ricgraph_cache, _memcached_available, _memcached_client
|
|
240
239
|
|
|
241
240
|
if memcached_check_available():
|
|
242
241
|
if _memcached_client is None:
|
|
@@ -244,24 +243,23 @@ def nodes_cache_key_id_empty() -> None:
|
|
|
244
243
|
try:
|
|
245
244
|
_memcached_client.flush_all()
|
|
246
245
|
except:
|
|
247
|
-
print('
|
|
246
|
+
print('ricgraph_cache_empty(): Error, connection to Memcached lost, exiting...')
|
|
248
247
|
# Exit because the cache gets in an unexpected state:
|
|
249
248
|
# it is not emptied while it should have been.
|
|
250
249
|
exit(1)
|
|
251
250
|
return
|
|
252
251
|
|
|
253
252
|
# print('Clear cache.')
|
|
254
|
-
|
|
253
|
+
_ricgraph_cache.clear()
|
|
255
254
|
return
|
|
256
255
|
|
|
257
256
|
|
|
258
|
-
def
|
|
259
|
-
"""Return the size of the cache
|
|
260
|
-
'id' in this sentence is the id assigned by de graph database.
|
|
257
|
+
def ricgraph_cache_size() -> Tuple[int, float]:
|
|
258
|
+
"""Return the size of the cache.
|
|
261
259
|
|
|
262
260
|
:return: a tuple with number of items and size in kB.
|
|
263
261
|
"""
|
|
264
|
-
global
|
|
262
|
+
global _ricgraph_cache, _memcached_client
|
|
265
263
|
|
|
266
264
|
if memcached_check_available():
|
|
267
265
|
if _memcached_client is None:
|
|
@@ -277,18 +275,18 @@ def nodes_cache_key_id_size() -> Tuple[int, float]:
|
|
|
277
275
|
nr_items = 0
|
|
278
276
|
size_kb = 0.0
|
|
279
277
|
else:
|
|
280
|
-
nr_items = len(
|
|
281
|
-
size_kb = round(getsizeof(
|
|
278
|
+
nr_items = len(_ricgraph_cache)
|
|
279
|
+
size_kb = round(getsizeof(_ricgraph_cache) / 1000, 1)
|
|
282
280
|
return nr_items, size_kb
|
|
283
281
|
|
|
284
282
|
|
|
285
|
-
def
|
|
283
|
+
def ricgraph_cache_size_text() -> str:
|
|
286
284
|
"""Return the size of the cache for node id's.
|
|
287
285
|
'id' in this sentence is the id assigned by de graph database.
|
|
288
286
|
|
|
289
287
|
:return: a sentence with the length of the cache and its size.
|
|
290
288
|
"""
|
|
291
|
-
nr_items, size_kb =
|
|
289
|
+
nr_items, size_kb = ricgraph_cache_size()
|
|
292
290
|
if memcached_check_available():
|
|
293
291
|
result = 'Ricgraph uses Memcached as cache. '
|
|
294
292
|
else:
|
|
@@ -69,6 +69,8 @@ class QueryParams(TypedDict):
|
|
|
69
69
|
start_orgs: str
|
|
70
70
|
collab_orgs: str
|
|
71
71
|
max_nr_items: int
|
|
72
|
+
skip_nr_nodes: int
|
|
73
|
+
|
|
72
74
|
|
|
73
75
|
# This TypedDict is filled in ricgraph_utils.py/create_empty_page_params(),
|
|
74
76
|
# and in ricgraph_explorer_utils.py/get_url_page_params().
|
|
@@ -84,6 +86,29 @@ class PageParams(TypedDict):
|
|
|
84
86
|
view_mode: str
|
|
85
87
|
|
|
86
88
|
|
|
89
|
+
# This TypedDict is a buffer/cache for the elements retrieved
|
|
90
|
+
# from the graph database. Since retrieving elements may be
|
|
91
|
+
# expensive, we get a batch_size of them and then return them
|
|
92
|
+
# in pages of page_size. This is useful for the e.g. REST API.
|
|
93
|
+
# graph_function_name is the function we call, and it has
|
|
94
|
+
# graph_function_kwargs as parameters.
|
|
95
|
+
# first_element and last_element are the 'real' indexes of
|
|
96
|
+
# elements. E.g., if elements has a length of 100, and first_element
|
|
97
|
+
# is 200, then the first element of elements is the 200-th element
|
|
98
|
+
# returned from graph_function_name.
|
|
99
|
+
# If exhausted is True, there are no more elements to retrieve
|
|
100
|
+
# from graph_function_name, so it is not necessary to recall it.
|
|
101
|
+
class GraphResultCache(TypedDict):
|
|
102
|
+
first_element: int # Counts from 0, not from 1.
|
|
103
|
+
last_element: int # So with 10 elems this will be 9.
|
|
104
|
+
elements: list[dict] # In the dict, the elements are 0..9.
|
|
105
|
+
exhausted: bool
|
|
106
|
+
page_size: int
|
|
107
|
+
batch_size: int
|
|
108
|
+
graph_function_name: str
|
|
109
|
+
graph_function_kwargs: dict
|
|
110
|
+
|
|
111
|
+
|
|
87
112
|
RICGRAPH_INI_FILENAME = 'ricgraph.ini'
|
|
88
113
|
|
|
89
114
|
# This one separates value & name in property _key of a node.
|
|
@@ -114,12 +139,17 @@ HTTP_RESPONSE_OK = 200
|
|
|
114
139
|
HTTP_RESPONSE_NOTHING_FOUND = 250
|
|
115
140
|
HTTP_RESPONSE_INVALID_SEARCH = 251
|
|
116
141
|
|
|
117
|
-
# The dict '
|
|
142
|
+
# The dict '_ricgraph_cache' is used to cache IDs to nodes. This is the cache size.
|
|
118
143
|
# The dict itself is defined in ricgraph_cache.py.
|
|
119
144
|
# This dict will be emptied if it reaches this number of elements.
|
|
120
145
|
# A cache entry is approx. 20 - 30 bytes. 30 x 2000000 ~ 60MB.
|
|
121
146
|
MAX_NODES_CACHE_KEY_ID = 2000000
|
|
122
147
|
|
|
148
|
+
# This is the maximum pool size for Memcached PooledClient().
|
|
149
|
+
# max_pool_size is a per-worker setting, not a global one.
|
|
150
|
+
# Various sources suggest that value 2 seems fine.
|
|
151
|
+
MEMCACHED_MAX_POOL_SIZE = 2
|
|
152
|
+
|
|
123
153
|
# In merge_two_nodes(node_merge_from, node_merge_to), we are constructing a list of
|
|
124
154
|
# all the nodes that are to be merged with node_merge_to, to add to property _history
|
|
125
155
|
# in that node. This value restricts the length of that list (to prevent property
|
|
@@ -444,29 +474,51 @@ NODELABELS_CATEGORY = [
|
|
|
444
474
|
# ########################################################################
|
|
445
475
|
# License related constants.
|
|
446
476
|
# ########################################################################
|
|
447
|
-
# For
|
|
448
|
-
#
|
|
477
|
+
# For the full SPDX license list, see https://spdx.org/licenses.
|
|
478
|
+
# Their names and spellings are used.
|
|
479
|
+
# For software licenses: https://opensource.org/licenses.
|
|
480
|
+
|
|
481
|
+
# This one should have '-only' or '-or-later' added, but
|
|
482
|
+
# the source systems do not specify.
|
|
449
483
|
LICENSE_AGPL_30 = 'AGPL-3.0'
|
|
484
|
+
|
|
450
485
|
LICENSE_APACHE_20 = 'Apache-2.0'
|
|
451
486
|
LICENSE_BSD_2 = 'BSD-2-Clause'
|
|
452
487
|
LICENSE_BSD_3 = 'BSD-3-Clause'
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
488
|
+
LICENSE_CC0_10 = 'CC0-1.0'
|
|
489
|
+
|
|
490
|
+
# A distinction is made between CC licenses that specify a
|
|
491
|
+
# version number and CC licenses that do not.
|
|
492
|
+
LICENSE_CC_BY = 'CC-BY'
|
|
493
|
+
LICENSE_CC_BY_40 = 'CC-BY-4.0'
|
|
494
|
+
LICENSE_CC_BY_NC = 'CC-BY-NC'
|
|
495
|
+
LICENSE_CC_BY_NC_ND = 'CC-BY-NC-ND'
|
|
496
|
+
LICENSE_CC_BY_NC_SA = 'CC-BY-NC-SA'
|
|
497
|
+
LICENSE_CC_BY_ND = 'CC-BY-ND'
|
|
498
|
+
LICENSE_CC_BY_SA = 'CC-BY-SA'
|
|
499
|
+
|
|
500
|
+
# These should have '-only' or '-or-later' added, but
|
|
501
|
+
# the source systems do not specify.
|
|
461
502
|
LICENSE_GPL_20 = 'GPL-2.0'
|
|
462
503
|
LICENSE_GPL_30 = 'GPL-3.0'
|
|
463
504
|
LICENSE_LGPL_21 = 'LGPL-2.1'
|
|
464
505
|
LICENSE_LGPL_30 = 'LGPL-3.0'
|
|
506
|
+
|
|
465
507
|
LICENSE_MIT = 'MIT'
|
|
466
|
-
|
|
508
|
+
|
|
509
|
+
# This one is not an official SPDX license name, but it
|
|
510
|
+
# appears in our source systems.
|
|
511
|
+
LICENSE_MULTIPLE = 'multiple'
|
|
512
|
+
|
|
513
|
+
LICENSE_ODBL_10 = 'ODbL-1.0'
|
|
467
514
|
LICENSE_ODC_BY_10 = 'ODC-By-1.0'
|
|
468
|
-
|
|
515
|
+
|
|
516
|
+
# These are not official SPDX license names, but they
|
|
517
|
+
# appear in our source systems.
|
|
518
|
+
LICENSE_OPEN_ACCESS = 'open access'
|
|
469
519
|
LICENSE_TAVERNE = 'Taverne (not for re-use)'
|
|
520
|
+
|
|
521
|
+
# Fall back if no license is obtained from a source system.
|
|
470
522
|
LICENSE_UNKNOWN = RICGRAPH_UNKNOWN
|
|
471
523
|
|
|
472
524
|
LICENSE_ALL = [
|
|
@@ -474,7 +526,7 @@ LICENSE_ALL = [
|
|
|
474
526
|
LICENSE_APACHE_20,
|
|
475
527
|
LICENSE_BSD_2,
|
|
476
528
|
LICENSE_BSD_3,
|
|
477
|
-
|
|
529
|
+
LICENSE_CC0_10,
|
|
478
530
|
LICENSE_CC_BY,
|
|
479
531
|
LICENSE_CC_BY_40,
|
|
480
532
|
LICENSE_CC_BY_NC,
|
|
@@ -488,6 +540,7 @@ LICENSE_ALL = [
|
|
|
488
540
|
LICENSE_LGPL_30,
|
|
489
541
|
LICENSE_MIT,
|
|
490
542
|
LICENSE_MULTIPLE,
|
|
543
|
+
LICENSE_ODBL_10,
|
|
491
544
|
LICENSE_ODC_BY_10,
|
|
492
545
|
LICENSE_OPEN_ACCESS,
|
|
493
546
|
LICENSE_TAVERNE,
|