ricgraph 2.11__tar.gz → 2.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ricgraph-2.11 → ricgraph-2.13}/LICENSE +1 -1
- {ricgraph-2.11/ricgraph.egg-info → ricgraph-2.13}/PKG-INFO +3 -2
- {ricgraph-2.11 → ricgraph-2.13}/README.md +1 -1
- {ricgraph-2.11 → ricgraph-2.13}/pyproject.toml +4 -3
- {ricgraph-2.11 → ricgraph-2.13}/ricgraph/__init__.py +20 -5
- ricgraph-2.13/ricgraph/ricgraph.py +359 -0
- ricgraph-2.13/ricgraph/ricgraph_cache.py +197 -0
- ricgraph-2.13/ricgraph/ricgraph_constants.py +192 -0
- ricgraph-2.13/ricgraph/ricgraph_cypher.py +1067 -0
- ricgraph-2.13/ricgraph/ricgraph_file.py +192 -0
- ricgraph-2.13/ricgraph/ricgraph_graphdb.py +987 -0
- ricgraph-2.13/ricgraph/ricgraph_harvest.py +176 -0
- ricgraph-2.13/ricgraph/ricgraph_researchinfo.py +284 -0
- ricgraph-2.13/ricgraph/ricgraph_restapi.py +76 -0
- ricgraph-2.13/ricgraph/ricgraph_utils.py +404 -0
- {ricgraph-2.11 → ricgraph-2.13/ricgraph.egg-info}/PKG-INFO +3 -2
- ricgraph-2.13/ricgraph.egg-info/SOURCES.txt +20 -0
- {ricgraph-2.11 → ricgraph-2.13}/ricgraph.egg-info/requires.txt +1 -0
- ricgraph-2.11/ricgraph/ricgraph.py +0 -3159
- ricgraph-2.11/ricgraph.egg-info/SOURCES.txt +0 -11
- {ricgraph-2.11 → ricgraph-2.13}/README-PyPI.md +0 -0
- {ricgraph-2.11 → ricgraph-2.13}/ricgraph.egg-info/dependency_links.txt +0 -0
- {ricgraph-2.11 → ricgraph-2.13}/ricgraph.egg-info/top_level.txt +0 -0
- {ricgraph-2.11 → ricgraph-2.13}/setup.cfg +0 -0
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ricgraph
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.13
|
|
4
4
|
Summary: Ricgraph - Research in context graph
|
|
5
5
|
Author: Rik D.T. Janssen
|
|
6
|
+
License-Expression: MIT
|
|
6
7
|
Project-URL: Website, https://www.ricgraph.eu
|
|
7
8
|
Project-URL: Documentation, https://docs.ricgraph.eu
|
|
8
9
|
Project-URL: GitHub, https://github.com/UtrechtUniversity/ricgraph
|
|
@@ -11,7 +12,6 @@ Project-URL: Download, https://github.com/UtrechtUniversity/ricgraph/releases
|
|
|
11
12
|
Project-URL: Release Notes, https://github.com/UtrechtUniversity/ricgraph/releases
|
|
12
13
|
Keywords: Research in context graph,Ricgraph,Ricgraph Explorer,Ricgraph REST API,Data enrichment,Data harvesting,Data linking,Enrichment,Graph,Graph database,Harvest,Harvest data,Harvester,Knowledge graph,Linked data,Metadata,Utrecht University,Visualization
|
|
13
14
|
Classifier: Development Status :: 5 - Production/Stable
|
|
14
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
15
15
|
Classifier: Operating System :: OS Independent
|
|
16
16
|
Classifier: Programming Language :: Python :: 3
|
|
17
17
|
Requires-Python: >=3.9
|
|
@@ -21,6 +21,7 @@ Requires-Dist: neo4j>=5.8
|
|
|
21
21
|
Requires-Dist: numpy
|
|
22
22
|
Requires-Dist: pandas
|
|
23
23
|
Requires-Dist: requests
|
|
24
|
+
Requires-Dist: unidecode
|
|
24
25
|
Dynamic: license-file
|
|
25
26
|
|
|
26
27
|
[](https://doi.org/10.5281/zenodo.7524314)
|
|
@@ -390,4 +390,4 @@ please let me know by creating a GitHub *Issue* at the top left of that page.
|
|
|
390
390
|
<!-- Do not change the lines below. The version number will be
|
|
391
391
|
modified by script maintenance/update_version.bash -->
|
|
392
392
|
This README.md file is part of the
|
|
393
|
-
documentation for version 2.
|
|
393
|
+
documentation for version 2.13 of Ricgraph - Research in context graph.
|
|
@@ -7,7 +7,7 @@ packages = ["ricgraph"]
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "ricgraph"
|
|
10
|
-
version = "2.
|
|
10
|
+
version = "2.13"
|
|
11
11
|
authors = [
|
|
12
12
|
{ name="Rik D.T. Janssen" },
|
|
13
13
|
]
|
|
@@ -16,20 +16,21 @@ readme = "README-PyPI.md"
|
|
|
16
16
|
requires-python = ">=3.9"
|
|
17
17
|
classifiers = [
|
|
18
18
|
"Development Status :: 5 - Production/Stable",
|
|
19
|
-
"License :: OSI Approved :: MIT License",
|
|
20
19
|
"Operating System :: OS Independent",
|
|
21
20
|
"Programming Language :: Python :: 3",
|
|
22
21
|
]
|
|
22
|
+
license = "MIT"
|
|
23
23
|
keywords = ["Research in context graph", "Ricgraph", "Ricgraph Explorer",
|
|
24
24
|
"Ricgraph REST API", "Data enrichment", "Data harvesting",
|
|
25
25
|
"Data linking", "Enrichment", "Graph", "Graph database",
|
|
26
26
|
"Harvest", "Harvest data", "Harvester", "Knowledge graph",
|
|
27
27
|
"Linked data", "Metadata", "Utrecht University", "Visualization"]
|
|
28
28
|
dependencies = [
|
|
29
|
-
"neo4j
|
|
29
|
+
"neo4j>=5.8",
|
|
30
30
|
"numpy",
|
|
31
31
|
"pandas",
|
|
32
32
|
"requests",
|
|
33
|
+
"unidecode",
|
|
33
34
|
]
|
|
34
35
|
|
|
35
36
|
[project.urls]
|
|
@@ -5,19 +5,19 @@
|
|
|
5
5
|
# ########################################################################
|
|
6
6
|
#
|
|
7
7
|
# MIT License
|
|
8
|
-
#
|
|
9
|
-
# Copyright (c) 2023 Rik D.T. Janssen
|
|
10
|
-
#
|
|
8
|
+
#
|
|
9
|
+
# Copyright (c) 2023 - 2025 Rik D.T. Janssen
|
|
10
|
+
#
|
|
11
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
13
13
|
# in the Software without restriction, including without limitation the rights
|
|
14
14
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
15
15
|
# copies of the Software, and to permit persons to whom the Software is
|
|
16
16
|
# furnished to do so, subject to the following conditions:
|
|
17
|
-
#
|
|
17
|
+
#
|
|
18
18
|
# The above copyright notice and this permission notice shall be included in all
|
|
19
19
|
# copies or substantial portions of the Software.
|
|
20
|
-
#
|
|
20
|
+
#
|
|
21
21
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
22
22
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
23
23
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
@@ -29,8 +29,23 @@
|
|
|
29
29
|
# ########################################################################
|
|
30
30
|
#
|
|
31
31
|
# This file is the __init__ file for Ricgraph.
|
|
32
|
+
# For more information about Ricgraph and Ricgraph Explorer,
|
|
33
|
+
# go to https://www.ricgraph.eu and https://docs.ricgraph.eu.
|
|
34
|
+
#
|
|
35
|
+
# ########################################################################
|
|
36
|
+
#
|
|
32
37
|
# Original version Rik D.T. Janssen, December 2022.
|
|
38
|
+
# Updated Rik D.T. Janssen, May 2025.
|
|
33
39
|
#
|
|
34
40
|
# ########################################################################
|
|
35
41
|
|
|
42
|
+
from .ricgraph_constants import *
|
|
43
|
+
from .ricgraph_cache import *
|
|
44
|
+
from .ricgraph_utils import *
|
|
45
|
+
from .ricgraph_file import *
|
|
46
|
+
from .ricgraph_cypher import *
|
|
47
|
+
from .ricgraph_graphdb import *
|
|
48
|
+
from .ricgraph_researchinfo import *
|
|
49
|
+
from .ricgraph_harvest import *
|
|
50
|
+
from .ricgraph_restapi import *
|
|
36
51
|
from .ricgraph import *
|
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
# ########################################################################
|
|
2
|
+
#
|
|
3
|
+
# Ricgraph - Research in context graph
|
|
4
|
+
#
|
|
5
|
+
# ########################################################################
|
|
6
|
+
#
|
|
7
|
+
# MIT License
|
|
8
|
+
#
|
|
9
|
+
# Copyright (c) 2023, 2024, 2025 Rik D.T. Janssen
|
|
10
|
+
#
|
|
11
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
13
|
+
# in the Software without restriction, including without limitation the rights
|
|
14
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
15
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
16
|
+
# furnished to do so, subject to the following conditions:
|
|
17
|
+
#
|
|
18
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
19
|
+
# copies or substantial portions of the Software.
|
|
20
|
+
#
|
|
21
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
22
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
23
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
24
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
25
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
26
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
27
|
+
# SOFTWARE.
|
|
28
|
+
#
|
|
29
|
+
# ########################################################################
|
|
30
|
+
#
|
|
31
|
+
# This file is the main file for Ricgraph.
|
|
32
|
+
# Original version Rik D.T. Janssen, December 2022.
|
|
33
|
+
# Updated Rik D.T. Janssen, February, March, September to December 2023.
|
|
34
|
+
# Updated Rik D.T. Janssen, February to June, September to December 2024.
|
|
35
|
+
# Updated Rik D.T. Janssen, January to June 2025.
|
|
36
|
+
#
|
|
37
|
+
# ########################################################################
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
"""Ricgraph - Research in context graph.
|
|
41
|
+
|
|
42
|
+
This is Ricgraph - Research in context graph, a graph structure with nodes and edges
|
|
43
|
+
to represent information (as properties in the nodes)
|
|
44
|
+
and their relations (the edges). You can find the code and documentation on
|
|
45
|
+
https://github.com/UtrechtUniversity/ricgraph/.
|
|
46
|
+
|
|
47
|
+
Ricgraph uses Neo4j as graph database, https://neo4j.com.
|
|
48
|
+
Neo4j only allows directed graphs, so Ricgraph is a directed graph.
|
|
49
|
+
They have several products. Their free products are:
|
|
50
|
+
|
|
51
|
+
- Neo4j Desktop (https://neo4j.com/download-center/#desktop);
|
|
52
|
+
|
|
53
|
+
- Neo4j Community Edition (https://neo4j.com/download-center/#community).
|
|
54
|
+
|
|
55
|
+
Neo4j Desktop includes Bloom, which allows exploring the graph in a more
|
|
56
|
+
intuitive way. Community Edition allows only Cypher queries.
|
|
57
|
+
|
|
58
|
+
Ricgraph can be extended for other brands of graph databases
|
|
59
|
+
by changing minor bits of the code.
|
|
60
|
+
|
|
61
|
+
All nodes have the properties in RICGRAPH_PROPERTIES_STANDARD and
|
|
62
|
+
RICGRAPH_PROPERTIES_HIDDEN (see ricgraph.ini):
|
|
63
|
+
|
|
64
|
+
In RICGRAPH_PROPERTIES_STANDARD:
|
|
65
|
+
|
|
66
|
+
- name: name of the node, e.g. ISNI, ORCID, DOI, etc.
|
|
67
|
+
|
|
68
|
+
- category: category of the node,
|
|
69
|
+
e.g. person, person-root, book, journal article, data set, etc.
|
|
70
|
+
|
|
71
|
+
- value: value of the node.
|
|
72
|
+
|
|
73
|
+
In RICGRAPH_PROPERTIES_HIDDEN:
|
|
74
|
+
|
|
75
|
+
- _key: key value of the node, not to be modified by the user.
|
|
76
|
+
|
|
77
|
+
- _source: sorted list of sources a record has been harvested from,
|
|
78
|
+
not to be modified by the user
|
|
79
|
+
|
|
80
|
+
- _history: list of history events of the node, not to be modified by the user.
|
|
81
|
+
|
|
82
|
+
Additional properties for nodes can be added by changing entry RICGRAPH_PROPERTIES_ADDITIONAL
|
|
83
|
+
in ricgraph.ini (make sure 'history_event' is last).
|
|
84
|
+
In the default configuration of Ricgraph, the following properties are included:
|
|
85
|
+
|
|
86
|
+
- comment: comment for a node, can be anything that is thought to be useful.
|
|
87
|
+
|
|
88
|
+
- url_main: main URL for a node, pointing to e.g. the corresponding ISNI, ORCID or DOI
|
|
89
|
+
record on the web.
|
|
90
|
+
|
|
91
|
+
- url_other: other URL for a node, pointing to e.g. the originating record in a source system.
|
|
92
|
+
|
|
93
|
+
- year: year of a research output.
|
|
94
|
+
|
|
95
|
+
- source_event: an event to be added to the _source list.
|
|
96
|
+
|
|
97
|
+
- history_event: an event to be added to the _history list.
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
from numpy import nan
|
|
102
|
+
from pandas import DataFrame, concat, set_option
|
|
103
|
+
from typing import Union
|
|
104
|
+
from .ricgraph_graphdb import (timestamp,
|
|
105
|
+
create_update_node, create_two_nodes_and_edge,
|
|
106
|
+
RICGRAPH_NODEADD_MODE,
|
|
107
|
+
RICGRAPH_PROPERTIES_STANDARD, RICGRAPH_PROPERTIES_ADDITIONAL)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
__author__ = 'Rik D.T. Janssen'
|
|
111
|
+
__copyright__ = 'Copyright (c) 2023, 2024, 2025 Rik D.T. Janssen'
|
|
112
|
+
__email__ = ''
|
|
113
|
+
__license__ = 'MIT License'
|
|
114
|
+
__package__ = 'Ricgraph'
|
|
115
|
+
|
|
116
|
+
# __version__ should be placed here instead of in __init__.py.
|
|
117
|
+
# If not here, it cannot be found when executing something like
|
|
118
|
+
# PYTHONPATH=../ricgraph ../bin/python ricgraph_explorer.py
|
|
119
|
+
__version__ = '2.13'
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# ##############################################################################
|
|
123
|
+
# General Ricgraph functions.
|
|
124
|
+
# ##############################################################################
|
|
125
|
+
def get_ricgraph_version() -> str:
|
|
126
|
+
"""Get the version Ricgraph.
|
|
127
|
+
|
|
128
|
+
:return: the version string.
|
|
129
|
+
"""
|
|
130
|
+
return __version__
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
# Note the similarity with create_nodepairs_and_edges_df().
|
|
134
|
+
def update_nodes_df(nodes: DataFrame) -> None:
|
|
135
|
+
"""Update the values in a node, but only if values have been changed.
|
|
136
|
+
This is done for all the nodes passed in the DataFrame.
|
|
137
|
+
|
|
138
|
+
The column names specify the property in a node (i.e. 'name', 'category', 'value').
|
|
139
|
+
The row value of that column specify the property value of that property.
|
|
140
|
+
|
|
141
|
+
:param nodes: the nodes in a DataFrame.
|
|
142
|
+
:return: None.
|
|
143
|
+
"""
|
|
144
|
+
nodes_clean = nodes.copy(deep=True)
|
|
145
|
+
# Ensure that all '' values are NaN, so that those rows can be easily removed with dropna()
|
|
146
|
+
nodes_clean.replace('', nan, inplace=True)
|
|
147
|
+
nodes_clean.dropna(axis=0, how='any', inplace=True)
|
|
148
|
+
nodes_clean.drop_duplicates(keep='first', inplace=True, ignore_index=True)
|
|
149
|
+
|
|
150
|
+
print('There are ' + str(len(nodes_clean)) + ' nodes ('
|
|
151
|
+
+ timestamp() + '), updating node: 0 ', end='')
|
|
152
|
+
count = 0
|
|
153
|
+
columns = nodes_clean.columns
|
|
154
|
+
for row in nodes_clean.itertuples():
|
|
155
|
+
count += 1
|
|
156
|
+
if count % 250 == 0:
|
|
157
|
+
print(count, ' ', end='', flush=True)
|
|
158
|
+
if count % 2500 == 0:
|
|
159
|
+
print('(' + timestamp() + ')\n', end='', flush=True)
|
|
160
|
+
|
|
161
|
+
node_properties = {}
|
|
162
|
+
for prop_name in RICGRAPH_PROPERTIES_ADDITIONAL:
|
|
163
|
+
for other_name in columns:
|
|
164
|
+
if prop_name == other_name:
|
|
165
|
+
node_properties[prop_name] = getattr(row, other_name)
|
|
166
|
+
|
|
167
|
+
create_update_node(name=row.name, category=row.category, value=row.value,
|
|
168
|
+
other_properties=node_properties)
|
|
169
|
+
|
|
170
|
+
print(count, '(' + timestamp() + ')\n', end='', flush=True)
|
|
171
|
+
return
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def create_nodepairs_and_edges_df(left_and_right_nodepairs: DataFrame) -> None:
|
|
175
|
+
"""Create two nodes (called 'node pairs') and two directed edges between those nodes.
|
|
176
|
+
This is done for all the nodes passed. The nodes are specified in a DataFrame.
|
|
177
|
+
One row in the DataFrame contains two nodes which are to be connected.
|
|
178
|
+
|
|
179
|
+
The column names specify the property in a node (i.e. 'name1', 'category1', 'value1').
|
|
180
|
+
The row value of that column specify the property value of that property.
|
|
181
|
+
All 'left' nodes end with '1', all right nodes end with '2'.
|
|
182
|
+
|
|
183
|
+
:param left_and_right_nodepairs: the node pairs in a DataFrame.
|
|
184
|
+
:return: None.
|
|
185
|
+
"""
|
|
186
|
+
print('There are ' + str(len(left_and_right_nodepairs)) + ' rows ('
|
|
187
|
+
+ timestamp() + '), creating nodes and edges for row: 0 ', end='')
|
|
188
|
+
count = 0
|
|
189
|
+
columns = left_and_right_nodepairs.columns
|
|
190
|
+
for row in left_and_right_nodepairs.itertuples():
|
|
191
|
+
count += 1
|
|
192
|
+
if count % 250 == 0:
|
|
193
|
+
print(count, ' ', end='', flush=True)
|
|
194
|
+
if count % 2500 == 0:
|
|
195
|
+
print('(' + timestamp() + ')\n', end='', flush=True)
|
|
196
|
+
|
|
197
|
+
node_properties = {}
|
|
198
|
+
for prop_name in RICGRAPH_PROPERTIES_ADDITIONAL:
|
|
199
|
+
for other_name in columns:
|
|
200
|
+
if prop_name + '1' == other_name:
|
|
201
|
+
node_properties[prop_name + '1'] = getattr(row, other_name)
|
|
202
|
+
for other_name in columns:
|
|
203
|
+
if prop_name + '2' == other_name:
|
|
204
|
+
node_properties[prop_name + '2'] = getattr(row, other_name)
|
|
205
|
+
|
|
206
|
+
create_two_nodes_and_edge(name1=row.name1, category1=row.category1, value1=row.value1,
|
|
207
|
+
name2=row.name2, category2=row.category2, value2=row.value2,
|
|
208
|
+
**node_properties)
|
|
209
|
+
print(count, '(' + timestamp() + ')\n', end='', flush=True)
|
|
210
|
+
return
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def create_nodepairs_and_edges_params(name1: Union[dict, str], category1: Union[dict, str], value1: Union[dict, str],
|
|
214
|
+
name2: Union[dict, str], category2: Union[dict, str], value2: Union[dict, str],
|
|
215
|
+
**other_properties: Union[dict, str]) -> None:
|
|
216
|
+
"""Create two nodes (called 'node pairs') and two directed edges between those nodes.
|
|
217
|
+
This is done for all the nodes passed. The nodes are specified by values passed to this function.
|
|
218
|
+
Each value can be either a string or a dict containing strings.
|
|
219
|
+
All 'left' nodes end with '1', all right nodes end with '2'.
|
|
220
|
+
|
|
221
|
+
If it is a string (like 'ORCID') then that value will be used for every
|
|
222
|
+
property value for that property. That is, every node pair will have the same value for that property.
|
|
223
|
+
|
|
224
|
+
If it is a dict containing strings, then every node pair will have a different value for that property.
|
|
225
|
+
|
|
226
|
+
:param name1: 'name' property of left node.
|
|
227
|
+
:param category1: idem.
|
|
228
|
+
:param value1: idem.
|
|
229
|
+
:param name2: 'name' property of right node.
|
|
230
|
+
:param category2: idem.
|
|
231
|
+
:param value2: idem.
|
|
232
|
+
:param other_properties: a dictionary of all the other properties.
|
|
233
|
+
Names ending with '1' belong to a 'left' node,
|
|
234
|
+
names ending with '2' to a 'right' node.
|
|
235
|
+
:return: None.
|
|
236
|
+
"""
|
|
237
|
+
if isinstance(value1, str):
|
|
238
|
+
left_and_right_nodepairs = DataFrame(data={'value1': value1},
|
|
239
|
+
index=[0])
|
|
240
|
+
else:
|
|
241
|
+
left_and_right_nodepairs = DataFrame(data=value1).copy(deep=True)
|
|
242
|
+
left_and_right_nodepairs.rename(columns={value1.name: 'value1'}, inplace=True)
|
|
243
|
+
|
|
244
|
+
# This makes programming more efficient
|
|
245
|
+
allowed_properties = RICGRAPH_PROPERTIES_ADDITIONAL + RICGRAPH_PROPERTIES_STANDARD
|
|
246
|
+
ext_other_properties = other_properties.copy()
|
|
247
|
+
ext_other_properties['name1'] = name1
|
|
248
|
+
ext_other_properties['name2'] = name2
|
|
249
|
+
ext_other_properties['category1'] = category1
|
|
250
|
+
ext_other_properties['category2'] = category2
|
|
251
|
+
# 'value1' has been included above
|
|
252
|
+
ext_other_properties['value2'] = value2
|
|
253
|
+
|
|
254
|
+
for prop_name in allowed_properties:
|
|
255
|
+
for other_name in ext_other_properties:
|
|
256
|
+
other_value = ext_other_properties[other_name]
|
|
257
|
+
if prop_name + '1' == other_name:
|
|
258
|
+
if isinstance(other_value, str):
|
|
259
|
+
left_and_right_nodepairs[prop_name + '1'] = other_value
|
|
260
|
+
else:
|
|
261
|
+
left_and_right_nodepairs = concat([left_and_right_nodepairs, other_value], axis='columns')
|
|
262
|
+
left_and_right_nodepairs.rename(columns={other_value.name: prop_name + '1'}, inplace=True)
|
|
263
|
+
|
|
264
|
+
if prop_name + '2' == other_name:
|
|
265
|
+
if isinstance(other_value, str):
|
|
266
|
+
left_and_right_nodepairs[prop_name + '2'] = other_value
|
|
267
|
+
else:
|
|
268
|
+
left_and_right_nodepairs = concat([left_and_right_nodepairs, other_value], axis='columns')
|
|
269
|
+
left_and_right_nodepairs.rename(columns={other_value.name: prop_name + '2'}, inplace=True)
|
|
270
|
+
|
|
271
|
+
# Reorder a bit, starting with the last, only the name-category-value property, since they will surely exist
|
|
272
|
+
left_and_right_nodepairs.insert(0, 'value2', left_and_right_nodepairs.pop('value2'))
|
|
273
|
+
left_and_right_nodepairs.insert(0, 'category2', left_and_right_nodepairs.pop('category2'))
|
|
274
|
+
left_and_right_nodepairs.insert(0, 'name2', left_and_right_nodepairs.pop('name2'))
|
|
275
|
+
left_and_right_nodepairs.insert(0, 'value1', left_and_right_nodepairs.pop('value1'))
|
|
276
|
+
left_and_right_nodepairs.insert(0, 'category1', left_and_right_nodepairs.pop('category1'))
|
|
277
|
+
left_and_right_nodepairs.insert(0, 'name1', left_and_right_nodepairs.pop('name1'))
|
|
278
|
+
|
|
279
|
+
create_nodepairs_and_edges_df(left_and_right_nodepairs=left_and_right_nodepairs)
|
|
280
|
+
return
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def unify_personal_identifiers(personal_identifiers: DataFrame,
|
|
284
|
+
source_event: Union[dict, str], history_event: Union[dict, str]) -> None:
|
|
285
|
+
"""Unify a collection of personal identifiers (e.g. ORCID, ISNI, etc.).
|
|
286
|
+
That means that every column is unified to every other column.
|
|
287
|
+
The identifiers are specified as columns in a DataFrame.
|
|
288
|
+
|
|
289
|
+
:param personal_identifiers: DataFrame containing personal identifiers.
|
|
290
|
+
:param source_event: an event to add to the _source list.
|
|
291
|
+
:param history_event: a history event to add.
|
|
292
|
+
:return: None.
|
|
293
|
+
"""
|
|
294
|
+
# Use i & j to make sure not to unify twice
|
|
295
|
+
i = 0
|
|
296
|
+
for column1 in personal_identifiers.columns:
|
|
297
|
+
i += 1
|
|
298
|
+
j = 0
|
|
299
|
+
for column2 in personal_identifiers.columns:
|
|
300
|
+
j += 1
|
|
301
|
+
if i >= j:
|
|
302
|
+
continue
|
|
303
|
+
print('\nUnifying personal identifiers ' + column1 + ' and ' + column2
|
|
304
|
+
+ ' at ' + timestamp() + '...')
|
|
305
|
+
identifiers = personal_identifiers[[column1, column2]].copy(deep=True)
|
|
306
|
+
|
|
307
|
+
# Ensure that all '' values are NaN, so that those rows can be easily removed with dropna()
|
|
308
|
+
identifiers.replace('', nan, inplace=True)
|
|
309
|
+
# dropna(how='any'): drop row if one or more row values contain NaN
|
|
310
|
+
identifiers.dropna(axis=0, how='any', inplace=True)
|
|
311
|
+
identifiers.drop_duplicates(keep='first', inplace=True, ignore_index=True)
|
|
312
|
+
|
|
313
|
+
if RICGRAPH_NODEADD_MODE == 'strict':
|
|
314
|
+
# Remove rows which have a cell value that occurs more than once.
|
|
315
|
+
# There are two situations: (1) nodes with unique value; (2) nodes with non-unique value.
|
|
316
|
+
# (1) We assume that in one source system, always the same ID is used to connect to
|
|
317
|
+
# research results.
|
|
318
|
+
# In principle, more than one ID could be used, but it could also be that someone made
|
|
319
|
+
# a typo and entered the ID from another person. We cannot know which one it is,
|
|
320
|
+
# therefore we assume that the same ID is used. This leads to the following,
|
|
321
|
+
# e.g. if we have a DataFrame:
|
|
322
|
+
# ISNI ORCID
|
|
323
|
+
# ISNI1 10
|
|
324
|
+
# ISNI2 11
|
|
325
|
+
# ISNI3 11
|
|
326
|
+
# then we remove both rows with ORCID 11.
|
|
327
|
+
# (2) It can very well be the same name belongs to more than one person (to more than one ID).
|
|
328
|
+
# E.g. if we have a DataFrame:
|
|
329
|
+
# FULL_NAME ORCID
|
|
330
|
+
# Name1 10
|
|
331
|
+
# Name2 10
|
|
332
|
+
# In this case we remove nothing.
|
|
333
|
+
# For more explanation, see file docs/ricgraph_install_configure.md,
|
|
334
|
+
# section RICGRAPH_NODEADD_MODE.
|
|
335
|
+
if column1 != 'FULL_NAME' and column2 != 'FULL_NAME':
|
|
336
|
+
identifiers.drop_duplicates(subset=column1, keep=False, inplace=True, ignore_index=True)
|
|
337
|
+
identifiers.drop_duplicates(subset=column2, keep=False, inplace=True, ignore_index=True)
|
|
338
|
+
|
|
339
|
+
create_nodepairs_and_edges_params(name1=column1, category1='person', value1=identifiers[column1],
|
|
340
|
+
name2=column2, category2='person', value2=identifiers[column2],
|
|
341
|
+
source_event1=source_event, source_event2=source_event,
|
|
342
|
+
history_event1=history_event, history_event2=history_event)
|
|
343
|
+
print('\nDone at ' + timestamp() + '.\n')
|
|
344
|
+
return
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
# ############################################
|
|
348
|
+
# ################### main ###################
|
|
349
|
+
# ############################################
|
|
350
|
+
# This will be executed on module initialization
|
|
351
|
+
|
|
352
|
+
# Make sure DataFrames are printed with all columns on full width
|
|
353
|
+
set_option('display.max_columns', None)
|
|
354
|
+
set_option('display.width', 500)
|
|
355
|
+
# To prevent warning
|
|
356
|
+
# "FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed
|
|
357
|
+
# in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`.
|
|
358
|
+
# To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`"
|
|
359
|
+
set_option('future.no_silent_downcasting', True)
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# ########################################################################
|
|
2
|
+
#
|
|
3
|
+
# Ricgraph - Research in context graph
|
|
4
|
+
#
|
|
5
|
+
# ########################################################################
|
|
6
|
+
#
|
|
7
|
+
# MIT License
|
|
8
|
+
#
|
|
9
|
+
# Copyright (c) 2023 - 2025 Rik D.T. Janssen
|
|
10
|
+
#
|
|
11
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
13
|
+
# in the Software without restriction, including without limitation the rights
|
|
14
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
15
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
16
|
+
# furnished to do so, subject to the following conditions:
|
|
17
|
+
#
|
|
18
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
19
|
+
# copies or substantial portions of the Software.
|
|
20
|
+
#
|
|
21
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
22
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
23
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
24
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
25
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
26
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
27
|
+
# SOFTWARE.
|
|
28
|
+
#
|
|
29
|
+
# ########################################################################
|
|
30
|
+
#
|
|
31
|
+
# Ricgraph cache functions.
|
|
32
|
+
# For more information about Ricgraph and Ricgraph Explorer,
|
|
33
|
+
# go to https://www.ricgraph.eu and https://docs.ricgraph.eu.
|
|
34
|
+
#
|
|
35
|
+
# ########################################################################
|
|
36
|
+
#
|
|
37
|
+
# Original version Rik D.T. Janssen, December 2022.
|
|
38
|
+
# Updated Rik D.T. Janssen, February, March, September to December 2023.
|
|
39
|
+
# Updated Rik D.T. Janssen, February to June, September to December 2024.
|
|
40
|
+
# Updated Rik D.T. Janssen, January to June 2025.
|
|
41
|
+
#
|
|
42
|
+
# ########################################################################
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
from sys import getsizeof
|
|
46
|
+
from typing import Union
|
|
47
|
+
from neo4j.graph import Node
|
|
48
|
+
from .ricgraph_constants import MAX_NODES_CACHE_NODELINK, MAX_NODES_CACHE_KEY_ID
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# This dict is used as a cache for node id's. If we have a node id, we can
|
|
52
|
+
# do a direct lookup for a node in O(1), instead of a search in O(log n).
|
|
53
|
+
# The dict has the format: [Ricgraph _key]: [Node element_id].
|
|
54
|
+
_nodes_cache_key_id = {}
|
|
55
|
+
|
|
56
|
+
# The dict '_nodes_cache_nodelink' is used to be able to pass nodes (i.e. links to Node)
|
|
57
|
+
# between the pages of Ricgraph Explorer. If we would not do this, we would need
|
|
58
|
+
# to search the node again every time we go to a new page. Then we would
|
|
59
|
+
# lose the advantage of using a graph.
|
|
60
|
+
# I could have used rcg.read_node() with @ricgraph_lru_cache, but I'd prefer to be
|
|
61
|
+
# able to store any node in the cache at the moment I prefer, as is done
|
|
62
|
+
# in function ricgraph_explorer.py/get_regular_table()).
|
|
63
|
+
# This dict has the format: [Ricgraph _key]: [Node link]
|
|
64
|
+
_nodes_cache_nodelink = {}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def nodes_cache_key_id_create(key: str, elementid: str) -> None:
|
|
68
|
+
"""Create an entry in the cache for node id's.
|
|
69
|
+
'id' in this sentence is the id assigned by de graph database.
|
|
70
|
+
|
|
71
|
+
:param key: _key of a node.
|
|
72
|
+
:param elementid: id of a node.
|
|
73
|
+
:return: None.
|
|
74
|
+
"""
|
|
75
|
+
global _nodes_cache_key_id
|
|
76
|
+
|
|
77
|
+
if len(_nodes_cache_key_id) > MAX_NODES_CACHE_KEY_ID:
|
|
78
|
+
nodes_cache_key_id_empty()
|
|
79
|
+
|
|
80
|
+
# We use a 'dict', which does not allow for duplicates, so we
|
|
81
|
+
# do not need to check for it.
|
|
82
|
+
_nodes_cache_key_id[key] = elementid
|
|
83
|
+
return
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def nodes_cache_key_id_read(key: str) -> str:
|
|
87
|
+
"""Read an entry from the cache for node id's.
|
|
88
|
+
'id' in this sentence is the id assigned by de graph database.
|
|
89
|
+
|
|
90
|
+
:param key: _key of a node.
|
|
91
|
+
:return: The id of the node, or '' if not present.
|
|
92
|
+
"""
|
|
93
|
+
global _nodes_cache_key_id
|
|
94
|
+
|
|
95
|
+
if key in _nodes_cache_key_id:
|
|
96
|
+
return _nodes_cache_key_id[key]
|
|
97
|
+
return ''
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def nodes_cache_key_id_delete_key(key: str) -> None:
|
|
101
|
+
"""Delete a key 'key' from the cache for node id's.
|
|
102
|
+
'id' in this sentence is the id assigned by de graph database.
|
|
103
|
+
|
|
104
|
+
:param key: _key of a node.
|
|
105
|
+
:return: None.
|
|
106
|
+
"""
|
|
107
|
+
global _nodes_cache_key_id
|
|
108
|
+
|
|
109
|
+
if key in _nodes_cache_key_id:
|
|
110
|
+
_nodes_cache_key_id.pop(key)
|
|
111
|
+
return
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def nodes_cache_key_id_delete_id(elementid: str) -> None:
|
|
115
|
+
"""Delete a value 'elementid' from the cache for node id's.
|
|
116
|
+
'id' in this sentence is the id assigned by de graph database.
|
|
117
|
+
This need to iterate over the full dict, so it is not efficient.
|
|
118
|
+
|
|
119
|
+
:param elementid: the id of a node.
|
|
120
|
+
:return: None.
|
|
121
|
+
"""
|
|
122
|
+
global _nodes_cache_key_id
|
|
123
|
+
|
|
124
|
+
key_to_delete = ''
|
|
125
|
+
for key in _nodes_cache_key_id:
|
|
126
|
+
if _nodes_cache_key_id[key] == elementid:
|
|
127
|
+
key_to_delete = key
|
|
128
|
+
break
|
|
129
|
+
if key_to_delete != '':
|
|
130
|
+
nodes_cache_key_id_delete_key(key=key_to_delete)
|
|
131
|
+
return
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def nodes_cache_key_id_empty() -> None:
|
|
135
|
+
"""Empty the cache for node id's.
|
|
136
|
+
'id' in this sentence is the id assigned by de graph database.
|
|
137
|
+
|
|
138
|
+
:return: None.
|
|
139
|
+
"""
|
|
140
|
+
global _nodes_cache_key_id
|
|
141
|
+
|
|
142
|
+
_nodes_cache_key_id.clear()
|
|
143
|
+
return
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def nodes_cache_nodelink_create(key: str, node: Node) -> None:
|
|
147
|
+
"""Create an entry in the cache for nodes.
|
|
148
|
+
|
|
149
|
+
:param key: _key of a node.
|
|
150
|
+
:param node: a node.
|
|
151
|
+
:return: None.
|
|
152
|
+
"""
|
|
153
|
+
global _nodes_cache_nodelink
|
|
154
|
+
|
|
155
|
+
if len(_nodes_cache_nodelink) > MAX_NODES_CACHE_NODELINK:
|
|
156
|
+
nodes_cache_nodelink_empty()
|
|
157
|
+
|
|
158
|
+
# We use a 'dict', which does not allow for duplicates, so we
|
|
159
|
+
# do not need to check for it.
|
|
160
|
+
_nodes_cache_nodelink[key] = node
|
|
161
|
+
return
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def nodes_cache_nodelink_read(key: str) -> Union[Node, None]:
|
|
165
|
+
"""Read an entry from the cache for nodes.
|
|
166
|
+
|
|
167
|
+
:param key: _key of a node.
|
|
168
|
+
:return: The node, or None if not present.
|
|
169
|
+
"""
|
|
170
|
+
global _nodes_cache_nodelink
|
|
171
|
+
|
|
172
|
+
if key in _nodes_cache_nodelink:
|
|
173
|
+
return _nodes_cache_nodelink[key]
|
|
174
|
+
return None
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def nodes_cache_nodelink_empty() -> None:
|
|
178
|
+
"""Empty the cache for nodes.
|
|
179
|
+
|
|
180
|
+
:return: None.
|
|
181
|
+
"""
|
|
182
|
+
global _nodes_cache_nodelink
|
|
183
|
+
|
|
184
|
+
_nodes_cache_nodelink.clear()
|
|
185
|
+
return
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def nodes_cache_nodelink_size() -> tuple[int, float]:
|
|
189
|
+
"""Return the size of the cache for nodes.
|
|
190
|
+
|
|
191
|
+
:return: the length of the dict (int) and the size in kB (float).
|
|
192
|
+
"""
|
|
193
|
+
global _nodes_cache_nodelink
|
|
194
|
+
|
|
195
|
+
length = len(_nodes_cache_nodelink)
|
|
196
|
+
size_kb = round(getsizeof(_nodes_cache_nodelink) / 1000, 1)
|
|
197
|
+
return length, size_kb
|