gsaphub 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gsaphub/__init__.py +35 -0
- gsaphub/_inception_url_helper.py +15 -0
- gsaphub/clients.py +36 -0
- gsaphub/env.py +24 -0
- gsaphub/labels.py +386 -0
- gsaphub/text_units.py +30 -0
- gsaphub-0.2.0.dist-info/METADATA +23 -0
- gsaphub-0.2.0.dist-info/RECORD +10 -0
- gsaphub-0.2.0.dist-info/WHEEL +5 -0
- gsaphub-0.2.0.dist-info/top_level.txt +1 -0
gsaphub/__init__.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from . import (
|
|
2
|
+
clients,
|
|
3
|
+
compare,
|
|
4
|
+
delete,
|
|
5
|
+
enrich,
|
|
6
|
+
env,
|
|
7
|
+
evaluate,
|
|
8
|
+
extract,
|
|
9
|
+
filter,
|
|
10
|
+
load,
|
|
11
|
+
match,
|
|
12
|
+
merge,
|
|
13
|
+
show,
|
|
14
|
+
stats,
|
|
15
|
+
transform,
|
|
16
|
+
labels,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"compare",
|
|
21
|
+
"delete",
|
|
22
|
+
"enrich",
|
|
23
|
+
"evaluate",
|
|
24
|
+
"extract",
|
|
25
|
+
"filter",
|
|
26
|
+
"load",
|
|
27
|
+
"match",
|
|
28
|
+
"merge",
|
|
29
|
+
"show",
|
|
30
|
+
"stats",
|
|
31
|
+
"transform",
|
|
32
|
+
"clients",
|
|
33
|
+
"env",
|
|
34
|
+
"labels",
|
|
35
|
+
]
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
def ent_to_url(ent):
|
|
2
|
+
info = ent["inception_meta"]
|
|
3
|
+
if info.get("line_idx") is None:
|
|
4
|
+
return
|
|
5
|
+
info["annotator"] = ent["annotator"]
|
|
6
|
+
return info_to_url(info)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def info_to_url(info):
|
|
10
|
+
url_base = "https://multiweb.gesis.org/berd-nfdi/p/"
|
|
11
|
+
url = (
|
|
12
|
+
f"{url_base}{info['project_id']}/annotate"
|
|
13
|
+
f"?3#!d={info['document_id']}&f={info['line_idx']}&u={info['annotator']}"
|
|
14
|
+
)
|
|
15
|
+
return url
|
gsaphub/clients.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from gql import Client, gql
|
|
2
|
+
from gql.transport.aiohttp import AIOHTTPTransport
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
async def graphql(url, email, password):
|
|
6
|
+
client = _create_graphql_client(url)
|
|
7
|
+
bearer = await _get_graphql_bearer(client, email, password)
|
|
8
|
+
client = _create_graphql_client(url, bearer)
|
|
9
|
+
return client
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _create_graphql_client(url, bearer=None):
|
|
13
|
+
transport_params = dict(url=url)
|
|
14
|
+
if bearer is not None:
|
|
15
|
+
transport_params["headers"] = dict(Authorization=f"Bearer {bearer}")
|
|
16
|
+
|
|
17
|
+
transport = AIOHTTPTransport(**transport_params)
|
|
18
|
+
client = Client(transport=transport, fetch_schema_from_transport=True)
|
|
19
|
+
return client
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
async def _get_graphql_bearer(client, email, password):
|
|
23
|
+
query_login = f"""
|
|
24
|
+
mutation login {{
|
|
25
|
+
authenticate(
|
|
26
|
+
input: {{email: "{email}", password: "{password}"}}
|
|
27
|
+
) {{
|
|
28
|
+
clientMutationId
|
|
29
|
+
jwtToken
|
|
30
|
+
}}
|
|
31
|
+
}}"""
|
|
32
|
+
result_login = await client.execute_async(gql(query_login))
|
|
33
|
+
bearer = result_login["authenticate"]["jwtToken"]
|
|
34
|
+
if not bearer:
|
|
35
|
+
raise Exception("AuthentificationException: Wrong Password?")
|
|
36
|
+
return bearer
|
gsaphub/env.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from dotenv import load_dotenv
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class EnvPathsAccess:
|
|
8
|
+
annotations: str
|
|
9
|
+
annotations_plmarker: str
|
|
10
|
+
predictions: str
|
|
11
|
+
raw_docs: str
|
|
12
|
+
|
|
13
|
+
def __getattr__(self, name):
|
|
14
|
+
load_dotenv()
|
|
15
|
+
name_in_env = f"PATH_{name.upper()}"
|
|
16
|
+
value = os.environ.get(name_in_env)
|
|
17
|
+
if value is None:
|
|
18
|
+
raise AttributeError(
|
|
19
|
+
f"Set {name_in_env} in .env file or as environment variable."
|
|
20
|
+
)
|
|
21
|
+
return Path(value)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
paths = EnvPathsAccess()
|
gsaphub/labels.py
ADDED
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
from collections import defaultdict
|
|
2
|
+
|
|
3
|
+
ALLOWED_ENTITY_TYPES = {
|
|
4
|
+
"Method",
|
|
5
|
+
"DatasetGeneric",
|
|
6
|
+
"MLModelGeneric",
|
|
7
|
+
"ReferenceLink",
|
|
8
|
+
"ModelArchitecture",
|
|
9
|
+
"Task",
|
|
10
|
+
"Dataset",
|
|
11
|
+
"MLModel",
|
|
12
|
+
"DataSource",
|
|
13
|
+
"URL",
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
GSAP_INTENTION_MAPPING = {
|
|
17
|
+
"IntentionCreation": "creation",
|
|
18
|
+
"IntentionComparison": "comparison",
|
|
19
|
+
"IntentionUsage": "usage",
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
GSAP_SUBTYPE_MAPPING = {
|
|
24
|
+
"Method > misc": ["Method", "Misc"],
|
|
25
|
+
"Method > ML techniques": ["Method", "MLTechnique"],
|
|
26
|
+
"Method > libraries and tools": ["Method", "Tool"],
|
|
27
|
+
"Method > ML paradigm": ["Method", "MLParadigm"],
|
|
28
|
+
"Method > evaluation techniques": ["Method", "EvaluationTechnique"],
|
|
29
|
+
"Method > data processing": ["Method", "DataPreprocessing"],
|
|
30
|
+
"Method > data collection": ["Method", "DataCollection"],
|
|
31
|
+
"ModelArchitecture>TraditionalMLTechnique": [
|
|
32
|
+
"ModelArchitecture",
|
|
33
|
+
"MLTraditionalTechnique",
|
|
34
|
+
],
|
|
35
|
+
"ModelArchitecture>NNType": ["ModelArchitecture", "NeuralNetType"],
|
|
36
|
+
"ModelArchitecture>Component": ["ModelArchitecture", "MLModelComponent"],
|
|
37
|
+
"ModelArchitecture>MLModel": ["ModelArchitecture", "MLModel"],
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
GSAP_TYPES_WITH_SUBTYPES = defaultdict(set)
|
|
41
|
+
for label, label_sub in GSAP_SUBTYPE_MAPPING.values():
|
|
42
|
+
GSAP_TYPES_WITH_SUBTYPES[label].add(label_sub)
|
|
43
|
+
|
|
44
|
+
RELATION_MAPPING = {
|
|
45
|
+
("Dataset", "isBasedOn", "Datasource"): "sourcedFrom",
|
|
46
|
+
("DatasetGeneric", "isBasedOn", "Datasource"): "sourcedFrom",
|
|
47
|
+
# Data+ isBasedOn Method/MLModel+ : generatedWith
|
|
48
|
+
("Dataset", "isBasedOn", "Method"): "generatedBy",
|
|
49
|
+
("DatasetGeneric", "isBasedOn", "Method"): "generatedBy",
|
|
50
|
+
("Dataset", "isBasedOn", "MLModel"): "generatedBy",
|
|
51
|
+
("DatasetGeneric", "isBasedOn", "MLModel"): "generatedBy",
|
|
52
|
+
("Dataset", "isBasedOn", "MLModelGeneric"): "generatedBy",
|
|
53
|
+
("DatasetGeneric", "isBasedOn", "MLModelGeneric"): "generatedBy",
|
|
54
|
+
# Data+ isBasedOn Data+
|
|
55
|
+
("Dataset", "isBasedOn", "Dataset"): "transformedFrom",
|
|
56
|
+
("Dataset", "isBasedOn", "DatasetGeneric"): "transformedFrom",
|
|
57
|
+
("DatasetGeneric", "isBasedOn", "Dataset"): "transformedFrom",
|
|
58
|
+
("DatasetGeneric", "isBasedOn", "DatasetGeneric"): "transformedFrom",
|
|
59
|
+
# X hasArchitecture ModelArchitecture
|
|
60
|
+
("ModelArchitecture", "isBasedOn", "ModelArchitecture"): "architecture",
|
|
61
|
+
("MLModel", "isBasedOn", "ModelArchitecture"): "architecture",
|
|
62
|
+
("MLModelGeneric", "isBasedOn", "ModelArchitecture"): "architecture",
|
|
63
|
+
("Method", "isBasedOn", "ModelArchitecture"): "architecture",
|
|
64
|
+
# Method/MLModel+ isBasedOn Method => usedFor
|
|
65
|
+
("Method", "isBasedOn", "Method"): "usedFor",
|
|
66
|
+
("MLModel", "isBasedOn", "Method"): "usedFor",
|
|
67
|
+
("MLModelGeneric", "isBasedOn", "Method"): "usedFor",
|
|
68
|
+
("ModelArchitecture", "isBasedOn", "Method"): "usedFor", # ??? exist?
|
|
69
|
+
# appliedOn
|
|
70
|
+
# benchmarkFor
|
|
71
|
+
("Dataset", "appliedOn", "Task"): "benchmarkFor",
|
|
72
|
+
("DatasetGeneric", "appliedOn", "Task"): "benchmarkFor",
|
|
73
|
+
# appliedTo
|
|
74
|
+
("MLModel", "appliedOn", "Task"): "appliedTo",
|
|
75
|
+
("MLModelGeneric", "appliedOn", "Task"): "appliedTo",
|
|
76
|
+
("Method", "appliedOn", "Task"): "appliedTo",
|
|
77
|
+
("ModelArchitecture", "appliedOn", "Task"): "appliedTo", # ???
|
|
78
|
+
# hasProcessed
|
|
79
|
+
("MLModel", "appliedOn", "Dataset"): "processed",
|
|
80
|
+
("MLModel", "appliedOn", "DatasetGeneric"): "processed",
|
|
81
|
+
("MLModelGeneric", "appliedOn", "Dataset"): "processed",
|
|
82
|
+
("MLModelGeneric", "appliedOn", "DatasetGeneric"): "processed",
|
|
83
|
+
("Method", "appliedOn", "Dataset"): "processed",
|
|
84
|
+
("Method", "appliedOn", "DatasetGeneric"): "processed",
|
|
85
|
+
# Repair 2025-01 annotations
|
|
86
|
+
("Dataset", "derivedFrom", "Dataset"): "transformedFrom", # 11,
|
|
87
|
+
}
|
|
88
|
+
"""
|
|
89
|
+
('MLModel', 'derivedFrom', 'MLModel'): 7,
|
|
90
|
+
('MLModel', 'usedFor', 'Method'): 854,
|
|
91
|
+
('ModelArchitecture', 'usedFor', 'Method'): 19}}
|
|
92
|
+
|
|
93
|
+
('ModelArchitecture', 'appliedOn', 'DatasetGeneric'): 6,
|
|
94
|
+
|
|
95
|
+
('Dataset', 'Unknown', 'DatasetGeneric'): 1,
|
|
96
|
+
('Dataset', 'isBasedOn', 'ReferenceLink'): 1,
|
|
97
|
+
## Footnotes
|
|
98
|
+
('DatasetGeneric', 'hasFootnote', 'FootnoteLink'): 2,
|
|
99
|
+
('MLModelGeneric', 'hasFootnote', 'FootnoteLink'): 6,
|
|
100
|
+
('MLModel', 'hasFootnote', 'FootnoteLink'): 4,
|
|
101
|
+
('Method', 'hasFootnote', 'FootnoteLink'): 3,
|
|
102
|
+
|
|
103
|
+
('MLModelGeneric', 'Unknown', 'Task'): 2,
|
|
104
|
+
|
|
105
|
+
('MLModel', 'coreference', 'ModelArchitecture'): 1,
|
|
106
|
+
('MLModel', 'isComparedTo', 'ModelArchitecture'): 1,
|
|
107
|
+
('MLModelGeneric', 'isComparedTo', 'ModelArchitecture'): 1,
|
|
108
|
+
('MLModelGeneric', 'isBasedOn', 'Dataset'): 1,
|
|
109
|
+
('MLModelGeneric', 'isHyponymOf', 'MLModel'): 1,
|
|
110
|
+
('MLModelGeneric', 'usedFor', 'Method'): 4114,
|
|
111
|
+
('MLModelGeneric', 'versionOf', 'MLModel'): 1,
|
|
112
|
+
('Method', 'isBasedOn', 'Dataset'): 3,
|
|
113
|
+
('Method', 'isBasedOn', 'DatasetGeneric'): 3,
|
|
114
|
+
('Method', 'isBasedOn', 'MLModelGeneric'): 1,
|
|
115
|
+
('ModelArchitecture', 'coreference', 'MLModelGeneric'): 1,
|
|
116
|
+
('ModelArchitecture', 'evaluatedOn', 'DatasetGeneric'): 1,
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
RELATION_INVERSE = ["usedFor"]
|
|
120
|
+
|
|
121
|
+
RELATION_LEFT_TO_RIGHT = ["coreference"]
|
|
122
|
+
RELATIONS_FREE_TEXT = {
|
|
123
|
+
"usedFor": "is used for",
|
|
124
|
+
"citation": "has reference",
|
|
125
|
+
"evaluatedOn": "is evaluated on",
|
|
126
|
+
"coreference": "is identical to",
|
|
127
|
+
"architecture": "has architecture",
|
|
128
|
+
"trainedOn": "is trained on",
|
|
129
|
+
"isPartOf": "is part of",
|
|
130
|
+
"isHyponymOf": "is a",
|
|
131
|
+
"appliedTo": "is applied to",
|
|
132
|
+
"transformedTo": "is transformed to",
|
|
133
|
+
"isComparedTo": "is compared to",
|
|
134
|
+
"generatedBy": "is generated by",
|
|
135
|
+
"benchmarkFor": "is benchmarking",
|
|
136
|
+
"isBasedOn": "is based on",
|
|
137
|
+
"hasInstanceType": "has instances of type",
|
|
138
|
+
"sourcedFrom": "is sourced From",
|
|
139
|
+
"size": "has size",
|
|
140
|
+
"url": "can be found at",
|
|
141
|
+
"processed": "has processed",
|
|
142
|
+
"versionOf": "isVersionOf",
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
ENTITY_MAPPING = {
|
|
146
|
+
"Datasource": "DataSource",
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
SYMMETRIC_RELATIONS = ["coreference"]
|
|
150
|
+
|
|
151
|
+
ALLOWED_SIGNATURES = [
|
|
152
|
+
# Attributive Relations
|
|
153
|
+
# citation
|
|
154
|
+
["MLModel", "citation", "ReferenceLink"],
|
|
155
|
+
["ModelArchitecture", "citation", "ReferenceLink"],
|
|
156
|
+
["Task", "citation", "ReferenceLink"],
|
|
157
|
+
["MLModelGeneric", "citation", "ReferenceLink"],
|
|
158
|
+
["Method", "citation", "ReferenceLink"],
|
|
159
|
+
["Dataset", "citation", "ReferenceLink"],
|
|
160
|
+
["DatasetGeneric", "citation", "ReferenceLink"],
|
|
161
|
+
["DataSource", "citation", "ReferenceLink"],
|
|
162
|
+
# url
|
|
163
|
+
["MLModelGeneric", "url", "URL"],
|
|
164
|
+
["Method", "url", "URL"],
|
|
165
|
+
["DatasetGeneric", "url", "URL"],
|
|
166
|
+
["MLModel", "url", "URL"],
|
|
167
|
+
["Dataset", "url", "URL"],
|
|
168
|
+
["DataSource", "url", "URL"],
|
|
169
|
+
["ModelArchitecture", "url", "URL"],
|
|
170
|
+
#
|
|
171
|
+
# Inner Entity Group Relations
|
|
172
|
+
#
|
|
173
|
+
# isHyponymOf
|
|
174
|
+
# Model Like
|
|
175
|
+
["MLModelGeneric", "isHyponymOf", "MLModelGeneric"],
|
|
176
|
+
["MLModel", "isHyponymOf", "MLModelGeneric"],
|
|
177
|
+
["Method", "isHyponymOf", "Method"],
|
|
178
|
+
["ModelArchitecture", "isHyponymOf", "ModelArchitecture"],
|
|
179
|
+
["MLModelGeneric", "isHyponymOf", "ModelArchitecture"],
|
|
180
|
+
["MLModel", "isHyponymOf", "ModelArchitecture"],
|
|
181
|
+
["ModelArchitecture", "isHyponymOf", "MLModelGeneric"],
|
|
182
|
+
["Method", "isHyponymOf", "MLModelGeneric"],
|
|
183
|
+
# Task
|
|
184
|
+
["Task", "isHyponymOf", "Task"],
|
|
185
|
+
# Data Like
|
|
186
|
+
["DatasetGeneric", "isHyponymOf", "DatasetGeneric"],
|
|
187
|
+
["Dataset", "isHyponymOf", "DatasetGeneric"],
|
|
188
|
+
["DatasetGeneric", "isHyponymOf", "Dataset"],
|
|
189
|
+
#
|
|
190
|
+
# isPartOf
|
|
191
|
+
# Model|Method Like
|
|
192
|
+
["MLModel", "isPartOf", "MLModelGeneric"],
|
|
193
|
+
["ModelArchitecture", "isPartOf", "MLModelGeneric"],
|
|
194
|
+
["MLModelGeneric", "isPartOf", "MLModelGeneric"],
|
|
195
|
+
["ModelArchitecture", "isPartOf", "ModelArchitecture"],
|
|
196
|
+
["MLModelGeneric", "isPartOf", "Method"],
|
|
197
|
+
["Method", "isPartOf", "Method"],
|
|
198
|
+
# Task
|
|
199
|
+
["Task", "isPartOf", "Task"],
|
|
200
|
+
# Data Like
|
|
201
|
+
["Dataset", "isPartOf", "DatasetGeneric"],
|
|
202
|
+
["DatasetGeneric", "isPartOf", "Dataset"],
|
|
203
|
+
["DatasetGeneric", "isPartOf", "DatasetGeneric"],
|
|
204
|
+
["Dataset", "isPartOf", "Dataset"],
|
|
205
|
+
#
|
|
206
|
+
# coreference
|
|
207
|
+
# Model Like
|
|
208
|
+
["MLModelGeneric", "coreference", "MLModelGeneric"],
|
|
209
|
+
["MLModelGeneric", "coreference", "MLModel"],
|
|
210
|
+
["MLModel", "coreference", "MLModel"],
|
|
211
|
+
["MLModel", "coreference", "MLModelGeneric"],
|
|
212
|
+
# Method Like
|
|
213
|
+
["Method", "coreference", "Method"],
|
|
214
|
+
["ModelArchitecture", "coreference", "ModelArchitecture"],
|
|
215
|
+
# Task
|
|
216
|
+
["Task", "coreference", "Task"],
|
|
217
|
+
# Data Like
|
|
218
|
+
["Dataset", "coreference", "Dataset"],
|
|
219
|
+
["Dataset", "coreference", "DatasetGeneric"],
|
|
220
|
+
["DatasetGeneric", "coreference", "Dataset"],
|
|
221
|
+
["DatasetGeneric", "coreference", "DatasetGeneric"],
|
|
222
|
+
["DataSource", "coreference", "DataSource"],
|
|
223
|
+
#
|
|
224
|
+
# isComparedTo
|
|
225
|
+
# Model|Method Like
|
|
226
|
+
["MLModel", "isComparedTo", "MLModel"],
|
|
227
|
+
["MLModel", "isComparedTo", "MLModelGeneric"],
|
|
228
|
+
["MLModelGeneric", "isComparedTo", "MLModelGeneric"],
|
|
229
|
+
["MLModelGeneric", "isComparedTo", "MLModel"],
|
|
230
|
+
["Method", "isComparedTo", "Method"],
|
|
231
|
+
["Method", "isComparedTo", "MLModelGeneric"],
|
|
232
|
+
["Method", "isComparedTo", "MLModel"],
|
|
233
|
+
["MLModel", "isComparedTo", "Method"],
|
|
234
|
+
["MLModelGeneric", "isComparedTo", "Method"],
|
|
235
|
+
["ModelArchitecture", "isComparedTo", "ModelArchitecture"],
|
|
236
|
+
["ModelArchitecture", "isComparedTo", "MLModel"],
|
|
237
|
+
# Task
|
|
238
|
+
["Task", "isComparedTo", "Task"],
|
|
239
|
+
# Data Like
|
|
240
|
+
["DatasetGeneric", "isComparedTo", "DatasetGeneric"],
|
|
241
|
+
["Dataset", "isComparedTo", "Dataset"],
|
|
242
|
+
["DatasetGeneric", "isComparedTo", "Dataset"],
|
|
243
|
+
["Dataset", "isComparedTo", "DatasetGeneric"],
|
|
244
|
+
#
|
|
245
|
+
# versionOf
|
|
246
|
+
# MLModel
|
|
247
|
+
#["MLModel", "versionOf", "MLModel"],
|
|
248
|
+
# Dataset
|
|
249
|
+
#["Dataset", "versionOf", "Dataset"],
|
|
250
|
+
#
|
|
251
|
+
# isBasedOn
|
|
252
|
+
# Model Like
|
|
253
|
+
["MLModel", "isBasedOn", "MLModel"],
|
|
254
|
+
["MLModel", "isBasedOn", "MLModelGeneric"],
|
|
255
|
+
["MLModelGeneric", "isBasedOn", "MLModelGeneric"],
|
|
256
|
+
["MLModelGeneric", "isBasedOn", "MLModel"],
|
|
257
|
+
#
|
|
258
|
+
# transformedFrom
|
|
259
|
+
# Data Like
|
|
260
|
+
["DatasetGeneric", "transformedFrom", "DatasetGeneric"],
|
|
261
|
+
["DatasetGeneric", "transformedFrom", "Dataset"],
|
|
262
|
+
["Dataset", "transformedFrom", "DatasetGeneric"],
|
|
263
|
+
["Dataset", "transformedFrom", "Dataset"],
|
|
264
|
+
#
|
|
265
|
+
# size
|
|
266
|
+
["DatasetGeneric", "size", "DatasetGeneric"],
|
|
267
|
+
["Dataset", "size", "DatasetGeneric"],
|
|
268
|
+
#
|
|
269
|
+
# hasInstanceType
|
|
270
|
+
# Data Like
|
|
271
|
+
["Dataset", "hasInstanceType", "DatasetGeneric"],
|
|
272
|
+
["DatasetGeneric", "hasInstanceType", "DatasetGeneric"],
|
|
273
|
+
#
|
|
274
|
+
# Inter Entity Group Relations
|
|
275
|
+
# ============================
|
|
276
|
+
#
|
|
277
|
+
# --------------+
|
|
278
|
+
# ==> Data Like |
|
|
279
|
+
# --------------+
|
|
280
|
+
#
|
|
281
|
+
# Method|Model Like ==> Data Like
|
|
282
|
+
# trainedOn
|
|
283
|
+
# ---------
|
|
284
|
+
["MLModelGeneric", "trainedOn", "DatasetGeneric"],
|
|
285
|
+
["MLModel", "trainedOn", "DatasetGeneric"],
|
|
286
|
+
["Method", "trainedOn", "DatasetGeneric"],
|
|
287
|
+
["MLModelGeneric", "trainedOn", "Dataset"],
|
|
288
|
+
["MLModel", "trainedOn", "Dataset"],
|
|
289
|
+
["Method", "trainedOn", "Dataset"],
|
|
290
|
+
# evaluatedOn
|
|
291
|
+
# -----------
|
|
292
|
+
["MLModelGeneric", "evaluatedOn", "DatasetGeneric"],
|
|
293
|
+
["MLModel", "evaluatedOn", "DatasetGeneric"],
|
|
294
|
+
["MLModel", "evaluatedOn", "Dataset"],
|
|
295
|
+
["MLModelGeneric", "evaluatedOn", "Dataset"],
|
|
296
|
+
["Method", "evaluatedOn", "DatasetGeneric"],
|
|
297
|
+
["Method", "evaluatedOn", "Dataset"],
|
|
298
|
+
# processed (former appliedOn)
|
|
299
|
+
# ---------
|
|
300
|
+
#["Method", "processed", "DatasetGeneric"],
|
|
301
|
+
#["Method", "processed", "Dataset"],
|
|
302
|
+
#["MLModelGeneric", "processed", "DatasetGeneric"],
|
|
303
|
+
#["MLModel", "processed", "DatasetGeneric"],
|
|
304
|
+
#["MLModel", "processed", "Dataset"],
|
|
305
|
+
#
|
|
306
|
+
# ---------+
|
|
307
|
+
# ==> Task |
|
|
308
|
+
# ---------+
|
|
309
|
+
#
|
|
310
|
+
# Method|Model Like ==> Task
|
|
311
|
+
# appliedTo
|
|
312
|
+
# ---------
|
|
313
|
+
["ModelArchitecture", "appliedTo", "Task"],
|
|
314
|
+
["MLModelGeneric", "appliedTo", "Task"],
|
|
315
|
+
["MLModel", "appliedTo", "Task"],
|
|
316
|
+
["Method", "appliedTo", "Task"],
|
|
317
|
+
#
|
|
318
|
+
# Data Like ==> Task
|
|
319
|
+
# benchmarkFor
|
|
320
|
+
# ------------
|
|
321
|
+
["DatasetGeneric", "benchmarkFor", "Task"],
|
|
322
|
+
["Dataset", "benchmarkFor", "Task"],
|
|
323
|
+
#
|
|
324
|
+
# ----------------------+
|
|
325
|
+
# ==> Model|Method Like |
|
|
326
|
+
# ----------------------+
|
|
327
|
+
#
|
|
328
|
+
# Data Like ==> Method|Model Like
|
|
329
|
+
# generatedBy (former basedOn)
|
|
330
|
+
# -----------
|
|
331
|
+
["DatasetGeneric", "generatedBy", "Method"],
|
|
332
|
+
["DatasetGeneric", "generatedBy", "MLModelGeneric"],
|
|
333
|
+
["DatasetGeneric", "generatedBy", "MLModel"],
|
|
334
|
+
["Dataset", "generatedBy", "Method"],
|
|
335
|
+
["Dataset", "generatedBy", "MLModelGeneric"],
|
|
336
|
+
["Dataset", "generatedBy", "MLModel"],
|
|
337
|
+
#
|
|
338
|
+
# Method ==> Model|Method Like
|
|
339
|
+
# usedFor (former invers basedOn)
|
|
340
|
+
# -------
|
|
341
|
+
["Method", "usedFor", "MLModelGeneric"],
|
|
342
|
+
["Method", "usedFor", "MLModel"],
|
|
343
|
+
["Method", "usedFor", "Method"],
|
|
344
|
+
["Method", "usedFor", "ModelArchitecture"],
|
|
345
|
+
#
|
|
346
|
+
# Model|Method Like ==> ModelArchitecture
|
|
347
|
+
# architecture
|
|
348
|
+
# ------------
|
|
349
|
+
["MLModelGeneric", "architecture", "ModelArchitecture"],
|
|
350
|
+
["MLModel", "architecture", "ModelArchitecture"],
|
|
351
|
+
["ModelArchitecture", "architecture", "ModelArchitecture"],
|
|
352
|
+
["Method", "architecture", "ModelArchitecture"],
|
|
353
|
+
#
|
|
354
|
+
# ---------------+
|
|
355
|
+
# ==> DataSource +
|
|
356
|
+
# ---------------+
|
|
357
|
+
#
|
|
358
|
+
# Data Like ==> DataSource
|
|
359
|
+
# sourcedFrom
|
|
360
|
+
# -----------
|
|
361
|
+
["DatasetGeneric", "sourcedFrom", "DataSource"],
|
|
362
|
+
["Dataset", "sourcedFrom", "DataSource"],
|
|
363
|
+
]
|
|
364
|
+
ALLOWED_RELATION_TYPES = { relation_label for subject_type, relation_label, object_type in
|
|
365
|
+
ALLOWED_SIGNATURES }
|
|
366
|
+
|
|
367
|
+
RELATION_GROUPS = {
|
|
368
|
+
"Model Design":{"architecture", "usedFor", "isBasedOn"},
|
|
369
|
+
"Task Binding":{"benchmarkFor", "appliedTo"},
|
|
370
|
+
"Data Usage":{"evaluatedOn", "trainedOn"}, #, "processed"},
|
|
371
|
+
"Data Provenance":{"generatedBy", "transformedFrom", "sourcedFrom"},
|
|
372
|
+
"Data Properties":{"size", "hasInstanceType"},
|
|
373
|
+
"Peer Relating":{
|
|
374
|
+
"coreference",
|
|
375
|
+
"isHyponymOf",
|
|
376
|
+
"isPartOf",
|
|
377
|
+
"isComparedTo",
|
|
378
|
+
# "versionOf",
|
|
379
|
+
},
|
|
380
|
+
"Referencing":{"url", "citation"},
|
|
381
|
+
"_average":{"weighted", "macro", "micro"},
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
RELATION_LABEL_TO_GROUP = {
|
|
385
|
+
l: g_label for g_label, g in RELATION_GROUPS.items() for l in g
|
|
386
|
+
}
|
gsaphub/text_units.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
def text_unit_as_string(text_unit):
|
|
2
|
+
ents = text_unit["annotations"]
|
|
3
|
+
rels = text_unit["relations"]
|
|
4
|
+
text = text_unit["text"]
|
|
5
|
+
text += "/t"
|
|
6
|
+
text += relations_as_string(rels, ents, include_url=False)
|
|
7
|
+
if "url_inception" in text_unit:
|
|
8
|
+
text += f"\n\n{text_unit['url_inception']}"
|
|
9
|
+
return text
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def relations_as_string(relations, entities, include_url=True):
|
|
13
|
+
string = []
|
|
14
|
+
ents_dict = {e["id"]: e for e in entities}
|
|
15
|
+
for rel in relations:
|
|
16
|
+
subj = ents_dict.get(rel["subject_id"], {})
|
|
17
|
+
obj = ents_dict.get(rel["object_id"], {})
|
|
18
|
+
string.append(_relation_as_string(rel, subj, obj, include_url))
|
|
19
|
+
return "\n\t".join(string)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _relation_as_string(rel, subj, obj, include_url=True):
|
|
23
|
+
ent = subj if subj else obj
|
|
24
|
+
subs = f"[{subj.get('label', 'NIL')}: '{subj.get('text', '<no text>')}']"
|
|
25
|
+
rel = f"({rel.get('relation_label', 'NIL')})"
|
|
26
|
+
objs = f"[{obj.get('label', 'NIL')}: '{obj.get('text', '<no text>')}']"
|
|
27
|
+
rel = f"{subs} - {rel} - {objs}"
|
|
28
|
+
if ent and include_url:
|
|
29
|
+
rel += f"\n\t{ent.get('url_inception')}"
|
|
30
|
+
return rel
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: gsaphub
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Data transfer platform from inception to graphql and filesystem
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: aiohttp>=3.11.2
|
|
8
|
+
Requires-Dist: gql>=3.5.0
|
|
9
|
+
Requires-Dist: pandas>=2.2.3
|
|
10
|
+
Requires-Dist: pip>=25.0.1
|
|
11
|
+
Requires-Dist: pycaprio>=0.3.0
|
|
12
|
+
Requires-Dist: python-dotenv>=1.0.1
|
|
13
|
+
Requires-Dist: spacy>=3.8.5
|
|
14
|
+
|
|
15
|
+
# Migraion tool to import exported inception annotations to postgraphile graphql backend
|
|
16
|
+
|
|
17
|
+
## .env file
|
|
18
|
+
* please copy example.env to .env and add your credentials
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
## To load the spacy model used for tokenizing to create the plmarker format:
|
|
23
|
+
* `uv run -- spacy download en_core_web_sm`
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
gsaphub/__init__.py,sha256=AcvWIHmVJHlEUMTDllxFIFYoEDa7_Ajhc8yjYnlVQsc,419
|
|
2
|
+
gsaphub/_inception_url_helper.py,sha256=kCh0jOmYfvzkhBBm0G6YmxV3UivVmALUxzHiMlqV0sI,424
|
|
3
|
+
gsaphub/clients.py,sha256=WI2-LK2cBLLKMe_qdexSUdEbB-s4gcQy1wHG4ktB_Dc,1142
|
|
4
|
+
gsaphub/env.py,sha256=8Vi_HfLzYh1GugtALRmTWu83_BiHixt0kXOib4hmVK0,532
|
|
5
|
+
gsaphub/labels.py,sha256=EpfVoJbydnyMEX-rIaJlsZfeps2Lnowyqag9rIs7RW0,14128
|
|
6
|
+
gsaphub/text_units.py,sha256=_rJGZTjAaTNamMtwsLfhuOPZqYdkjS0znL7b_OmP1sA,1106
|
|
7
|
+
gsaphub-0.2.0.dist-info/METADATA,sha256=5LJIVsVTbc-his8PN_hdbrrgEtvtEsIo3UacTQWMqX8,686
|
|
8
|
+
gsaphub-0.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
9
|
+
gsaphub-0.2.0.dist-info/top_level.txt,sha256=0wEYxQI0MF-5UbphdiIYN4k5UhVU6glhfBFRHHtJJdI,8
|
|
10
|
+
gsaphub-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
gsaphub
|