OntoAligner 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- OntoAligner-0.1.0/LICENSE +21 -0
- OntoAligner-0.1.0/OntoAligner.egg-info/PKG-INFO +15 -0
- OntoAligner-0.1.0/OntoAligner.egg-info/SOURCES.txt +73 -0
- OntoAligner-0.1.0/OntoAligner.egg-info/dependency_links.txt +1 -0
- OntoAligner-0.1.0/OntoAligner.egg-info/requires.txt +17 -0
- OntoAligner-0.1.0/OntoAligner.egg-info/top_level.txt +1 -0
- OntoAligner-0.1.0/PKG-INFO +15 -0
- OntoAligner-0.1.0/README.md +1 -0
- OntoAligner-0.1.0/ontoaligner/__init__.py +32 -0
- OntoAligner-0.1.0/ontoaligner/base/__init__.py +1 -0
- OntoAligner-0.1.0/ontoaligner/base/configs.py +121 -0
- OntoAligner-0.1.0/ontoaligner/base/dataset.py +49 -0
- OntoAligner-0.1.0/ontoaligner/base/encoder.py +28 -0
- OntoAligner-0.1.0/ontoaligner/base/model.py +18 -0
- OntoAligner-0.1.0/ontoaligner/base/ontology.py +147 -0
- OntoAligner-0.1.0/ontoaligner/encoder/__init__.py +1 -0
- OntoAligner-0.1.0/ontoaligner/encoder/encoders.py +105 -0
- OntoAligner-0.1.0/ontoaligner/encoder/fewshot.py +14 -0
- OntoAligner-0.1.0/ontoaligner/encoder/lightweight.py +27 -0
- OntoAligner-0.1.0/ontoaligner/encoder/naivconvoaei.py +27 -0
- OntoAligner-0.1.0/ontoaligner/encoder/rag.py +23 -0
- OntoAligner-0.1.0/ontoaligner/evaluation/__init__.py +1 -0
- OntoAligner-0.1.0/ontoaligner/evaluation/evaluator.py +31 -0
- OntoAligner-0.1.0/ontoaligner/evaluation/metrics.py +70 -0
- OntoAligner-0.1.0/ontoaligner/ontology/__init__.py +1 -0
- OntoAligner-0.1.0/ontoaligner/ontology/anatomy.py +27 -0
- OntoAligner-0.1.0/ontoaligner/ontology/biodiv.py +133 -0
- OntoAligner-0.1.0/ontoaligner/ontology/bioml.py +201 -0
- OntoAligner-0.1.0/ontoaligner/ontology/commonkg.py +31 -0
- OntoAligner-0.1.0/ontoaligner/ontology/food.py +36 -0
- OntoAligner-0.1.0/ontoaligner/ontology/mse.py +119 -0
- OntoAligner-0.1.0/ontoaligner/ontology/phenotype.py +69 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/__init__.py +137 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/fewshot/__init__.py +0 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/fewshot/dataset.py +56 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/fewshot/fewshot.py +85 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/fewshot/models.py +110 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/icv/__init__.py +0 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/icv/icv.py +217 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/icv/models.py +127 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/icv/tasks/__init__.py +0 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/icv/tasks/base.py +350 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/icv/tasks/demo.py +36 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/icv/tasks/loader.py +133 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/icv/utils/__init__.py +0 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/icv/utils/context_manager.py +33 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/icv/utils/forward_tracer.py +138 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/icv/utils/llm_layers.py +134 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/icv/utils/pca.py +47 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/icv/utils/rng_ctx.py +66 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/lightweight/__init__.py +0 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/lightweight/lightweight.py +58 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/lightweight/models.py +30 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/llm/__init__.py +0 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/llm/llm.py +220 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/llm/models.py +79 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/rag/__init__.py +0 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/rag/dataset.py +100 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/rag/models.py +237 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/rag/rag.py +187 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/retrieval/__init__.py +1 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/retrieval/models.py +114 -0
- OntoAligner-0.1.0/ontoaligner/ontology_matchers/retrieval/retrieval.py +182 -0
- OntoAligner-0.1.0/ontoaligner/pipeline/__init__.py +1 -0
- OntoAligner-0.1.0/ontoaligner/pipeline/om_pipeline.py +40 -0
- OntoAligner-0.1.0/ontoaligner/postprocess/__init__.py +0 -0
- OntoAligner-0.1.0/ontoaligner/postprocess/process.py +216 -0
- OntoAligner-0.1.0/ontoaligner/tools/__init__.py +0 -0
- OntoAligner-0.1.0/ontoaligner/tools/workdir.py +27 -0
- OntoAligner-0.1.0/ontoaligner/utils/__init__.py +0 -0
- OntoAligner-0.1.0/ontoaligner/utils/io.py +48 -0
- OntoAligner-0.1.0/setup.cfg +4 -0
- OntoAligner-0.1.0/setup.py +38 -0
- OntoAligner-0.1.0/tests/test_base.py +0 -0
- OntoAligner-0.1.0/tests/test_encoder.py +13 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Hamed Babaei Giglou
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: OntoAligner
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A short description of your library
|
|
5
|
+
Home-page: https://github.com/HamedBabaei/OntoAligner
|
|
6
|
+
Author: Hamed Babaei Giglou
|
|
7
|
+
Author-email: hamedbabaeigiglou@gmail.com
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.8
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
|
|
15
|
+
# OntoAligner
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
setup.py
|
|
4
|
+
OntoAligner.egg-info/PKG-INFO
|
|
5
|
+
OntoAligner.egg-info/SOURCES.txt
|
|
6
|
+
OntoAligner.egg-info/dependency_links.txt
|
|
7
|
+
OntoAligner.egg-info/requires.txt
|
|
8
|
+
OntoAligner.egg-info/top_level.txt
|
|
9
|
+
ontoaligner/__init__.py
|
|
10
|
+
ontoaligner/base/__init__.py
|
|
11
|
+
ontoaligner/base/configs.py
|
|
12
|
+
ontoaligner/base/dataset.py
|
|
13
|
+
ontoaligner/base/encoder.py
|
|
14
|
+
ontoaligner/base/model.py
|
|
15
|
+
ontoaligner/base/ontology.py
|
|
16
|
+
ontoaligner/encoder/__init__.py
|
|
17
|
+
ontoaligner/encoder/encoders.py
|
|
18
|
+
ontoaligner/encoder/fewshot.py
|
|
19
|
+
ontoaligner/encoder/lightweight.py
|
|
20
|
+
ontoaligner/encoder/naivconvoaei.py
|
|
21
|
+
ontoaligner/encoder/rag.py
|
|
22
|
+
ontoaligner/evaluation/__init__.py
|
|
23
|
+
ontoaligner/evaluation/evaluator.py
|
|
24
|
+
ontoaligner/evaluation/metrics.py
|
|
25
|
+
ontoaligner/ontology/__init__.py
|
|
26
|
+
ontoaligner/ontology/anatomy.py
|
|
27
|
+
ontoaligner/ontology/biodiv.py
|
|
28
|
+
ontoaligner/ontology/bioml.py
|
|
29
|
+
ontoaligner/ontology/commonkg.py
|
|
30
|
+
ontoaligner/ontology/food.py
|
|
31
|
+
ontoaligner/ontology/mse.py
|
|
32
|
+
ontoaligner/ontology/phenotype.py
|
|
33
|
+
ontoaligner/ontology_matchers/__init__.py
|
|
34
|
+
ontoaligner/ontology_matchers/fewshot/__init__.py
|
|
35
|
+
ontoaligner/ontology_matchers/fewshot/dataset.py
|
|
36
|
+
ontoaligner/ontology_matchers/fewshot/fewshot.py
|
|
37
|
+
ontoaligner/ontology_matchers/fewshot/models.py
|
|
38
|
+
ontoaligner/ontology_matchers/icv/__init__.py
|
|
39
|
+
ontoaligner/ontology_matchers/icv/icv.py
|
|
40
|
+
ontoaligner/ontology_matchers/icv/models.py
|
|
41
|
+
ontoaligner/ontology_matchers/icv/tasks/__init__.py
|
|
42
|
+
ontoaligner/ontology_matchers/icv/tasks/base.py
|
|
43
|
+
ontoaligner/ontology_matchers/icv/tasks/demo.py
|
|
44
|
+
ontoaligner/ontology_matchers/icv/tasks/loader.py
|
|
45
|
+
ontoaligner/ontology_matchers/icv/utils/__init__.py
|
|
46
|
+
ontoaligner/ontology_matchers/icv/utils/context_manager.py
|
|
47
|
+
ontoaligner/ontology_matchers/icv/utils/forward_tracer.py
|
|
48
|
+
ontoaligner/ontology_matchers/icv/utils/llm_layers.py
|
|
49
|
+
ontoaligner/ontology_matchers/icv/utils/pca.py
|
|
50
|
+
ontoaligner/ontology_matchers/icv/utils/rng_ctx.py
|
|
51
|
+
ontoaligner/ontology_matchers/lightweight/__init__.py
|
|
52
|
+
ontoaligner/ontology_matchers/lightweight/lightweight.py
|
|
53
|
+
ontoaligner/ontology_matchers/lightweight/models.py
|
|
54
|
+
ontoaligner/ontology_matchers/llm/__init__.py
|
|
55
|
+
ontoaligner/ontology_matchers/llm/llm.py
|
|
56
|
+
ontoaligner/ontology_matchers/llm/models.py
|
|
57
|
+
ontoaligner/ontology_matchers/rag/__init__.py
|
|
58
|
+
ontoaligner/ontology_matchers/rag/dataset.py
|
|
59
|
+
ontoaligner/ontology_matchers/rag/models.py
|
|
60
|
+
ontoaligner/ontology_matchers/rag/rag.py
|
|
61
|
+
ontoaligner/ontology_matchers/retrieval/__init__.py
|
|
62
|
+
ontoaligner/ontology_matchers/retrieval/models.py
|
|
63
|
+
ontoaligner/ontology_matchers/retrieval/retrieval.py
|
|
64
|
+
ontoaligner/pipeline/__init__.py
|
|
65
|
+
ontoaligner/pipeline/om_pipeline.py
|
|
66
|
+
ontoaligner/postprocess/__init__.py
|
|
67
|
+
ontoaligner/postprocess/process.py
|
|
68
|
+
ontoaligner/tools/__init__.py
|
|
69
|
+
ontoaligner/tools/workdir.py
|
|
70
|
+
ontoaligner/utils/__init__.py
|
|
71
|
+
ontoaligner/utils/io.py
|
|
72
|
+
tests/test_base.py
|
|
73
|
+
tests/test_encoder.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ontoaligner
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: OntoAligner
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A short description of your library
|
|
5
|
+
Home-page: https://github.com/HamedBabaei/OntoAligner
|
|
6
|
+
Author: Hamed Babaei Giglou
|
|
7
|
+
Author-email: hamedbabaeigiglou@gmail.com
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.8
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
|
|
15
|
+
# OntoAligner
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# OntoAligner
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
|
|
3
|
+
__version__ = "0.1.0"
|
|
4
|
+
|
|
5
|
+
# import logging
|
|
6
|
+
#
|
|
7
|
+
# from onmatcher import base, encoder, evaluation, ontology, ontology_matchers, utils, postprocess
|
|
8
|
+
# from onmatcher.pipeline import OMPipelines
|
|
9
|
+
#
|
|
10
|
+
# __all__ = [
|
|
11
|
+
# "base",
|
|
12
|
+
# "ontology",
|
|
13
|
+
# "utils",
|
|
14
|
+
# "ontology_matchers",
|
|
15
|
+
# "encoder",
|
|
16
|
+
# "pipeline",
|
|
17
|
+
# "OMPipelines",
|
|
18
|
+
# "evaluation",
|
|
19
|
+
# "postprocess"
|
|
20
|
+
# ]
|
|
21
|
+
#
|
|
22
|
+
# # Root logger configuration
|
|
23
|
+
# logger = logging.getLogger(__name__)
|
|
24
|
+
# logger.setLevel(logging.DEBUG)
|
|
25
|
+
#
|
|
26
|
+
# stdout = logging.StreamHandler()
|
|
27
|
+
# stdout.setLevel(logging.DEBUG)
|
|
28
|
+
#
|
|
29
|
+
# formatter = logging.Formatter("%(name)s - %(levelname)s: %(message)s")
|
|
30
|
+
# stdout.setFormatter(formatter)
|
|
31
|
+
#
|
|
32
|
+
# logger.addHandler(stdout)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
BaseConfig: Data Configuration of models
|
|
4
|
+
"""
|
|
5
|
+
import argparse
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Dict, Optional
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
import openai
|
|
12
|
+
from dotenv import find_dotenv, load_dotenv
|
|
13
|
+
|
|
14
|
+
_ = load_dotenv(find_dotenv())
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class BaseConfig:
|
|
18
|
+
def __init__(self, approach: Optional[str] = "none"):
|
|
19
|
+
self.batch_size = None
|
|
20
|
+
self.device = None
|
|
21
|
+
self.root_dataset_dir = Path(__file__).parents[2]
|
|
22
|
+
self.parser = argparse.ArgumentParser()
|
|
23
|
+
self.approach = approach
|
|
24
|
+
|
|
25
|
+
def mkdir(self, path):
|
|
26
|
+
if not os.path.exists(path):
|
|
27
|
+
os.mkdir(path)
|
|
28
|
+
|
|
29
|
+
def __str__(self):
|
|
30
|
+
return "config."
|
|
31
|
+
|
|
32
|
+
def llm(self) -> Dict:
|
|
33
|
+
config = {
|
|
34
|
+
"max_token_length": 1,
|
|
35
|
+
"tokenizer_max_length": 500,
|
|
36
|
+
"num_beams": 1,
|
|
37
|
+
"device": self.device,
|
|
38
|
+
"truncation": True,
|
|
39
|
+
"top_p": 0.95,
|
|
40
|
+
"temperature": 0.8,
|
|
41
|
+
"batch_size": self.batch_size,
|
|
42
|
+
"padding": "max_length",
|
|
43
|
+
}
|
|
44
|
+
if self.approach == "naiv-conv-oaei":
|
|
45
|
+
config["tokenizer_max_length"] = 4096
|
|
46
|
+
config["max_token_length"] = 5000
|
|
47
|
+
return config
|
|
48
|
+
|
|
49
|
+
def gpt(self) -> Dict:
|
|
50
|
+
openai.api_key = os.environ["OPENAI_KEY"]
|
|
51
|
+
# due to the privacy I had to set key here,
|
|
52
|
+
# but the correct way is to set inside OPENAILLMARCH class
|
|
53
|
+
config = {
|
|
54
|
+
"sleep": 5,
|
|
55
|
+
"top_p": 0.95,
|
|
56
|
+
"temperature": 0,
|
|
57
|
+
"batch_size": self.batch_size,
|
|
58
|
+
"max_token_length": 2,
|
|
59
|
+
}
|
|
60
|
+
if self.approach == "naiv-conv-oaei":
|
|
61
|
+
config["max_token_length"] = 5000
|
|
62
|
+
return config
|
|
63
|
+
|
|
64
|
+
def fuzzy(self) -> Dict:
|
|
65
|
+
config = {"fuzzy_sm_threshold": 0.8}
|
|
66
|
+
return config
|
|
67
|
+
|
|
68
|
+
def retrieval(self) -> Dict:
|
|
69
|
+
config = {"top_k": 5, "device": self.device}
|
|
70
|
+
return config
|
|
71
|
+
|
|
72
|
+
def get_args(self, device="cpu", batch_size: int = None, nshots: int = None) -> Dict:
|
|
73
|
+
self.device = device
|
|
74
|
+
self.batch_size = batch_size
|
|
75
|
+
|
|
76
|
+
# General configurations
|
|
77
|
+
self.parser.add_argument("--root_dir", type=str, default=os.path.join(self.root_dataset_dir, "datasets"))
|
|
78
|
+
self.parser.add_argument("--experiments_dir", type=str, default=os.path.join(self.root_dataset_dir, "experiments"))
|
|
79
|
+
self.parser.add_argument("--output_dir", type=str, default=os.path.join(self.root_dataset_dir, "experiments", "outputs"))
|
|
80
|
+
self.parser.add_argument("--stats_dir", type=str, default=os.path.join(self.root_dataset_dir, "experiments", "stats"))
|
|
81
|
+
self.parser.add_argument("--openai_embedding_dir", type=str, default=os.path.join(self.root_dataset_dir, "assets", "openai-embedding"))
|
|
82
|
+
|
|
83
|
+
# LLM configurations
|
|
84
|
+
llm_config = self.llm()
|
|
85
|
+
llm_models = ["FlanT5", "LLaMA7B", "Wizard13B", "LLaMA13B", "Mistral7B"]
|
|
86
|
+
for llm_model in llm_models:
|
|
87
|
+
self.parser.add_argument("--" + llm_model, type=dict, default=llm_config)
|
|
88
|
+
self.parser.add_argument("--ChatGPT", type=dict, default=self.gpt())
|
|
89
|
+
self.parser.add_argument("--GPT4", type=dict, default=self.gpt())
|
|
90
|
+
|
|
91
|
+
# Lightweight configurations
|
|
92
|
+
fuzzy_models = ['SimpleFuzzySM', 'WeightedFuzzySM', 'TokenSetFuzzySM']
|
|
93
|
+
for fuzzy_model in fuzzy_models:
|
|
94
|
+
self.parser.add_argument("--" + fuzzy_model, type=dict, default=self.fuzzy())
|
|
95
|
+
|
|
96
|
+
# Retrieval Configurations
|
|
97
|
+
retriever_config = self.retrieval()
|
|
98
|
+
retriever_models = ["BM25Retrieval", "TFIDFRetrieval", "BERTRetrieval", "SpecterBERTRetrieval",
|
|
99
|
+
"FlanT5XLRetrieval", "FlanT5XXLRetrieval", "SVMBERTRetrieval", "AdaRetrieval"]
|
|
100
|
+
for retriever_model in retriever_models:
|
|
101
|
+
self.parser.add_argument("--" + retriever_model, type=dict, default=retriever_config)
|
|
102
|
+
|
|
103
|
+
# RAG + ICV + FewShot configurations
|
|
104
|
+
llama_rag_config = {"retriever-config": retriever_config, "llm-config": llm_config, "nshots": nshots}
|
|
105
|
+
rag_icv_models = ["LLaMA7BAdaRAG", "MistralAdaRAG", "FalconAdaRAG", "VicunaAdaRAG", "MPTAdaRAG",
|
|
106
|
+
"LLaMA7BBertRAG", "MistralBertRAG", "FalconBertRAG", "VicunaBertRAG", "MPTBertRAG",
|
|
107
|
+
"LLaMA7BAdaICV", "FalconAdaICV", "VicunaAdaICV", "MPTAdaICV",
|
|
108
|
+
"LLaMA7BBertICV", "FalconBertICV", "VicunaBertICV", "MPTBertICV",
|
|
109
|
+
"LLaMA7BAdaFewShot", "MistralAdaFewShot", "FalconAdaFewShot", "VicunaAdaFewShot", "MPTAdaFewShot",
|
|
110
|
+
"LLaMA7BBertFewShot", "MistralBertFewShot", "FalconBertFewShot", "VicunaBertFewShot", "MPTBertFewShot",
|
|
111
|
+
"MambaLLMAdaFewShot", "MambaLLMBertFewShot", "MambaLLMAdaRAG", "MambaLLMBertRAG"]
|
|
112
|
+
|
|
113
|
+
for rag_icv_model in rag_icv_models:
|
|
114
|
+
self.parser.add_argument("--" + rag_icv_model, type=dict, default=llama_rag_config)
|
|
115
|
+
|
|
116
|
+
openai_rag_config = {"retriever-config": retriever_config, "llm-config": self.gpt(), "nshots": nshots}
|
|
117
|
+
self.parser.add_argument("--ChatGPTOpenAIAdaRAG", type=dict, default=openai_rag_config)
|
|
118
|
+
self.parser.add_argument("--ChatGPTOpenAIAdaFewShot", type=dict, default=openai_rag_config)
|
|
119
|
+
|
|
120
|
+
self.parser.add_argument("-f")
|
|
121
|
+
return self.parser.parse_args()
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
import os
|
|
3
|
+
from abc import ABC
|
|
4
|
+
from typing import Any, Dict
|
|
5
|
+
|
|
6
|
+
from ontoaligner.base.ontology import BaseAlignmentsParser
|
|
7
|
+
from ontoaligner.utils import io
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class OMDataset(ABC):
|
|
12
|
+
track: str = ""
|
|
13
|
+
ontology_name: str = ""
|
|
14
|
+
|
|
15
|
+
source_ontology: Any = None
|
|
16
|
+
target_ontology: Any = None
|
|
17
|
+
|
|
18
|
+
alignments: Any = BaseAlignmentsParser()
|
|
19
|
+
|
|
20
|
+
working_dir: str = ""
|
|
21
|
+
|
|
22
|
+
def collect(self, root_dir: str) -> Dict:
|
|
23
|
+
om_root_path = os.path.join(root_dir, self.track, self.ontology_name)
|
|
24
|
+
data = {
|
|
25
|
+
"dataset-info": {"track": self.track, "ontology-name": self.ontology_name},
|
|
26
|
+
"source": self.source_ontology.parse(
|
|
27
|
+
root_dir=om_root_path, ontology_file_name="source.xml"
|
|
28
|
+
),
|
|
29
|
+
"target": self.target_ontology.parse(
|
|
30
|
+
root_dir=om_root_path, ontology_file_name="target.xml"
|
|
31
|
+
),
|
|
32
|
+
"reference": self.alignments.parse(
|
|
33
|
+
root_dir=om_root_path, reference_file_name="reference.xml"
|
|
34
|
+
),
|
|
35
|
+
}
|
|
36
|
+
return data
|
|
37
|
+
|
|
38
|
+
def load_from_json(self, root_dir: str) -> Dict:
|
|
39
|
+
json_file_path = os.path.join(
|
|
40
|
+
root_dir, self.track, self.ontology_name, "om.json"
|
|
41
|
+
)
|
|
42
|
+
json_data = io.read_json(input_path=json_file_path)
|
|
43
|
+
return json_data
|
|
44
|
+
|
|
45
|
+
def __dir__(self):
|
|
46
|
+
return os.path.join(self.track, self.ontology_name)
|
|
47
|
+
|
|
48
|
+
def __str__(self):
|
|
49
|
+
return f"{self.ontology_name}"
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
from abc import ABC, abstractmethod
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class BaseEncoder(ABC):
|
|
8
|
+
prompt_template: str = ""
|
|
9
|
+
items_in_owl: str = ""
|
|
10
|
+
|
|
11
|
+
def __str__(self):
|
|
12
|
+
return self.prompt_template
|
|
13
|
+
|
|
14
|
+
def preprocess(self, text: str) -> str:
|
|
15
|
+
text = text.replace("_", " ")
|
|
16
|
+
text = text.lower()
|
|
17
|
+
return text
|
|
18
|
+
|
|
19
|
+
@abstractmethod
|
|
20
|
+
def parse(self, **kwargs) -> Any:
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
@abstractmethod
|
|
24
|
+
def get_encoder_info(self) -> str:
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
def __call__(self, **kwargs):
|
|
28
|
+
return self.parse(**kwargs)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
from abc import ABC, abstractmethod
|
|
3
|
+
from typing import List
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class BaseOMModel(ABC):
|
|
7
|
+
path: str = ""
|
|
8
|
+
|
|
9
|
+
def __init__(self, **kwargs) -> None:
|
|
10
|
+
self.kwargs = kwargs
|
|
11
|
+
|
|
12
|
+
@abstractmethod
|
|
13
|
+
def __str__(self):
|
|
14
|
+
pass
|
|
15
|
+
|
|
16
|
+
@abstractmethod
|
|
17
|
+
def generate(self, input_data: List) -> List:
|
|
18
|
+
pass
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
import os
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from typing import Any, Dict, List
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
from owlready2 import World
|
|
8
|
+
from rdflib import Namespace, URIRef
|
|
9
|
+
from tqdm import tqdm
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class BaseOntologyParser(ABC):
|
|
13
|
+
def is_contain_label(self, owl_class: Any) -> bool:
|
|
14
|
+
try:
|
|
15
|
+
if len(owl_class.label) == 0:
|
|
16
|
+
return False
|
|
17
|
+
return True
|
|
18
|
+
except Exception as e:
|
|
19
|
+
print(f"Exception: {e}")
|
|
20
|
+
return False
|
|
21
|
+
|
|
22
|
+
def get_name(self, owl_class: Any) -> str:
|
|
23
|
+
return owl_class.name
|
|
24
|
+
|
|
25
|
+
def get_label(self, owl_class: Any) -> str:
|
|
26
|
+
return owl_class.label.first()
|
|
27
|
+
|
|
28
|
+
def get_iri(self, owl_class: Any) -> str:
|
|
29
|
+
return owl_class.iri
|
|
30
|
+
|
|
31
|
+
def get_childrens(self, owl_class: Any) -> List:
|
|
32
|
+
return self.get_owl_items(owl_class.subclasses()) # include_self = False
|
|
33
|
+
|
|
34
|
+
def get_parents(self, owl_class: Any) -> List:
|
|
35
|
+
ans = self.get_owl_items(owl_class.is_a) # include_self = False, ancestors()
|
|
36
|
+
return ans
|
|
37
|
+
|
|
38
|
+
def get_synonyms(self, owl_class: Any) -> List:
|
|
39
|
+
return self.get_owl_items(owl_class.hasRelatedSynonym)
|
|
40
|
+
|
|
41
|
+
@abstractmethod
|
|
42
|
+
def get_comments(self, owl_class: Any) -> List:
|
|
43
|
+
pass
|
|
44
|
+
|
|
45
|
+
def get_owl_items(self, owl_class: Any) -> List:
|
|
46
|
+
owl_items = []
|
|
47
|
+
for item in owl_class:
|
|
48
|
+
if self.is_contain_label(item):
|
|
49
|
+
owl_items.append(
|
|
50
|
+
{
|
|
51
|
+
"iri": self.get_iri(item),
|
|
52
|
+
"name": self.get_name(item),
|
|
53
|
+
"label": self.get_label(item),
|
|
54
|
+
}
|
|
55
|
+
)
|
|
56
|
+
return owl_items
|
|
57
|
+
|
|
58
|
+
def get_owl_classes(self, ontology: Any) -> Any:
|
|
59
|
+
return ontology.classes()
|
|
60
|
+
|
|
61
|
+
def duplicate_removals(self, owl_class_info: Dict) -> Dict:
|
|
62
|
+
def ignore_duplicates(iri: str, duplicated_list: List[Dict]) -> List:
|
|
63
|
+
new_list = []
|
|
64
|
+
for item in duplicated_list:
|
|
65
|
+
if iri != item["iri"]:
|
|
66
|
+
new_list.append(item)
|
|
67
|
+
return new_list
|
|
68
|
+
|
|
69
|
+
new_owl_class_info = {
|
|
70
|
+
"name": owl_class_info["name"],
|
|
71
|
+
"iri": owl_class_info["iri"],
|
|
72
|
+
"label": owl_class_info["label"],
|
|
73
|
+
"childrens": ignore_duplicates(
|
|
74
|
+
iri=owl_class_info["iri"], duplicated_list=owl_class_info["childrens"]
|
|
75
|
+
),
|
|
76
|
+
"parents": ignore_duplicates(
|
|
77
|
+
iri=owl_class_info["iri"], duplicated_list=owl_class_info["parents"]
|
|
78
|
+
),
|
|
79
|
+
"synonyms": owl_class_info["synonyms"],
|
|
80
|
+
"comment": owl_class_info["comment"],
|
|
81
|
+
}
|
|
82
|
+
return new_owl_class_info
|
|
83
|
+
|
|
84
|
+
def extract_data(self, ontology: Any) -> List[Dict]:
|
|
85
|
+
parsed_ontology = []
|
|
86
|
+
for owl_class in tqdm(self.get_owl_classes(ontology)):
|
|
87
|
+
if not self.is_contain_label(owl_class):
|
|
88
|
+
continue
|
|
89
|
+
owl_class_info = {
|
|
90
|
+
"name": self.get_name(owl_class),
|
|
91
|
+
"iri": self.get_iri(owl_class),
|
|
92
|
+
"label": self.get_label(owl_class),
|
|
93
|
+
"childrens": self.get_childrens(owl_class),
|
|
94
|
+
"parents": self.get_parents(owl_class),
|
|
95
|
+
"synonyms": self.get_synonyms(owl_class),
|
|
96
|
+
"comment": self.get_comments(owl_class),
|
|
97
|
+
}
|
|
98
|
+
owl_class_info = self.duplicate_removals(owl_class_info=owl_class_info)
|
|
99
|
+
parsed_ontology.append(owl_class_info)
|
|
100
|
+
return parsed_ontology
|
|
101
|
+
|
|
102
|
+
def load_ontology(self, input_file_path: str) -> Any:
|
|
103
|
+
ontology = World()
|
|
104
|
+
ontology.get_ontology(input_file_path).load()
|
|
105
|
+
return ontology
|
|
106
|
+
|
|
107
|
+
def parse(self, root_dir: str, ontology_file_name: str) -> List:
|
|
108
|
+
input_file_path = os.path.join(root_dir, ontology_file_name)
|
|
109
|
+
print(f"\t\tworking on {input_file_path}")
|
|
110
|
+
ontology = self.load_ontology(input_file_path=input_file_path)
|
|
111
|
+
return self.extract_data(ontology)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class BaseAlignmentsParser(ABC):
|
|
115
|
+
namespace: Namespace = Namespace(
|
|
116
|
+
"http://knowledgeweb.semanticweb.org/heterogeneity/alignment"
|
|
117
|
+
)
|
|
118
|
+
entity_1: URIRef = URIRef(namespace + "entity1")
|
|
119
|
+
entity_2: URIRef = URIRef(namespace + "entity2")
|
|
120
|
+
relation: URIRef = URIRef(namespace + "relation")
|
|
121
|
+
|
|
122
|
+
def extract_data(self, reference: Any) -> List[Dict]:
|
|
123
|
+
parsed_references = []
|
|
124
|
+
graph = reference.as_rdflib_graph()
|
|
125
|
+
for source, predicate, target in tqdm(graph):
|
|
126
|
+
if predicate == self.relation:
|
|
127
|
+
entity_1 = [
|
|
128
|
+
str(o) for s, p, o in graph.triples((source, self.entity_1, None))
|
|
129
|
+
][0]
|
|
130
|
+
entity_2 = [
|
|
131
|
+
str(o) for s, p, o in graph.triples((source, self.entity_2, None))
|
|
132
|
+
][0]
|
|
133
|
+
parsed_references.append(
|
|
134
|
+
{"source": entity_1, "target": entity_2, "relation": str(target)}
|
|
135
|
+
)
|
|
136
|
+
return parsed_references
|
|
137
|
+
|
|
138
|
+
def load_ontology(self, input_file_path: str) -> Any:
|
|
139
|
+
ontology = World()
|
|
140
|
+
ontology.get_ontology(input_file_path).load()
|
|
141
|
+
return ontology
|
|
142
|
+
|
|
143
|
+
def parse(self, root_dir: str, reference_file_name: str) -> List:
|
|
144
|
+
input_file_path = os.path.join(root_dir, reference_file_name)
|
|
145
|
+
print(f"\t\tworking on reference: {input_file_path}")
|
|
146
|
+
reference = self.load_ontology(input_file_path=input_file_path)
|
|
147
|
+
return self.extract_data(reference)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
from typing import Any, Dict
|
|
3
|
+
|
|
4
|
+
from ..base import BaseEncoder
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class LightweightEncoder(BaseEncoder):
|
|
8
|
+
def parse(self, **kwargs) -> Any:
|
|
9
|
+
source_onto, target_onto = kwargs["source"], kwargs["target"]
|
|
10
|
+
source_ontos = []
|
|
11
|
+
for source in source_onto:
|
|
12
|
+
encoded_source = self.get_owl_items(owl=source)
|
|
13
|
+
encoded_source["text"] = self.preprocess(encoded_source["text"])
|
|
14
|
+
source_ontos.append(encoded_source)
|
|
15
|
+
target_ontos = []
|
|
16
|
+
for target in target_onto:
|
|
17
|
+
encoded_target = self.get_owl_items(owl=target)
|
|
18
|
+
encoded_target["text"] = self.preprocess(encoded_target["text"])
|
|
19
|
+
target_ontos.append(encoded_target)
|
|
20
|
+
return [source_ontos, target_ontos]
|
|
21
|
+
|
|
22
|
+
def __str__(self):
|
|
23
|
+
return {"LightweightEncoder": self.items_in_owl}
|
|
24
|
+
|
|
25
|
+
def get_owl_items(self, owl: Dict) -> Any:
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
def get_encoder_info(self):
|
|
29
|
+
return "INPUT CONSIST OF COMBINED INFORMATION TO FUZZY STRING MATCHING"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class NaiveConvOAEIEncoder(BaseEncoder):
|
|
33
|
+
prompt_template: str = """<Problem Definition>
|
|
34
|
+
In this task, we are given two ontologies in the form of {items_in_owl}, which consist of IRI and classes.
|
|
35
|
+
|
|
36
|
+
<Ontologies-1>
|
|
37
|
+
{source}
|
|
38
|
+
|
|
39
|
+
<Ontologies-2>
|
|
40
|
+
{target}
|
|
41
|
+
|
|
42
|
+
<Objective>
|
|
43
|
+
Our objective is to provide ontology mapping for the provided ontologies based on their semantic similarities.
|
|
44
|
+
|
|
45
|
+
For a class in the ontology-1, which class in ontology-2 is the best match?
|
|
46
|
+
|
|
47
|
+
List matches per line.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
def parse(self, **kwargs) -> Any:
|
|
51
|
+
source_onto, target_onto = kwargs["source"], kwargs["target"]
|
|
52
|
+
source_text = ""
|
|
53
|
+
for source in source_onto:
|
|
54
|
+
source_text += self.get_owl_items(owl=source)
|
|
55
|
+
target_text = ""
|
|
56
|
+
for target in target_onto:
|
|
57
|
+
target_text += self.get_owl_items(owl=target)
|
|
58
|
+
prompt_sample = self.get_prefilled_prompt()
|
|
59
|
+
prompt_sample = prompt_sample.replace("{source}", source_text)
|
|
60
|
+
prompt_sample = prompt_sample.replace("{target}", target_text)
|
|
61
|
+
return [prompt_sample]
|
|
62
|
+
|
|
63
|
+
def __str__(self):
|
|
64
|
+
return {
|
|
65
|
+
"Template": super().__str__(),
|
|
66
|
+
"NaiveConvOAEIPrompting": self.items_in_owl,
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
def get_owl_items(self, owl: Dict) -> str:
|
|
70
|
+
pass
|
|
71
|
+
|
|
72
|
+
def get_prefilled_prompt(self) -> str:
|
|
73
|
+
prompt_sample = self.prompt_template
|
|
74
|
+
prompt_sample = prompt_sample.replace("{items_in_owl}", self.items_in_owl)
|
|
75
|
+
return prompt_sample
|
|
76
|
+
|
|
77
|
+
def get_encoder_info(self) -> str:
|
|
78
|
+
return "PROMPT-TEMPLATE: " + self.get_prefilled_prompt()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class RAGEncoder(BaseEncoder):
|
|
82
|
+
retrieval_encoder: Any = None
|
|
83
|
+
llm_encoder: str = None
|
|
84
|
+
|
|
85
|
+
def parse(self, **kwargs) -> Any:
|
|
86
|
+
# self.dataset_module = kwargs["dataset-module"]
|
|
87
|
+
source_onto_iri2index = {
|
|
88
|
+
source["iri"]: index for index, source in enumerate(kwargs["source"])
|
|
89
|
+
}
|
|
90
|
+
target_onto_iri2index = {
|
|
91
|
+
target["iri"]: index for index, target in enumerate(kwargs["target"])
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
"retriever-encoder": self.retrieval_encoder,
|
|
95
|
+
"llm-encoder": self.llm_encoder,
|
|
96
|
+
"task-args": kwargs,
|
|
97
|
+
"source-onto-iri2index": source_onto_iri2index,
|
|
98
|
+
"target-onto-iri2index": target_onto_iri2index,
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
def __str__(self):
|
|
102
|
+
return {"RagEncoder": self.items_in_owl}
|
|
103
|
+
|
|
104
|
+
def get_encoder_info(self) -> str:
|
|
105
|
+
return "PROMPT-TEMPLATE USES:" + self.llm_encoder + " ENCODER"
|