vfbquery 0.5.0__py3-none-any.whl → 0.5.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- test/readme_parser.py +29 -33
- test/term_info_queries_test.py +49 -34
- test/test_default_caching.py +86 -85
- test/test_examples_code.py +7 -0
- test/test_examples_diff.py +95 -172
- test/test_neurons_part_here.py +12 -13
- test/test_query_performance.py +3 -7
- vfbquery/__init__.py +47 -35
- vfbquery/cached_functions.py +772 -131
- vfbquery/owlery_client.py +1 -1
- vfbquery/solr_cache_integration.py +34 -30
- vfbquery/solr_result_cache.py +262 -99
- vfbquery/term_info_queries.py +1 -1
- vfbquery/vfb_queries.py +69 -20
- vfbquery-0.5.2.dist-info/METADATA +2806 -0
- {vfbquery-0.5.0.dist-info → vfbquery-0.5.2.dist-info}/RECORD +19 -18
- {vfbquery-0.5.0.dist-info → vfbquery-0.5.2.dist-info}/WHEEL +1 -1
- vfbquery-0.5.0.dist-info/METADATA +0 -2273
- {vfbquery-0.5.0.dist-info → vfbquery-0.5.2.dist-info}/LICENSE +0 -0
- {vfbquery-0.5.0.dist-info → vfbquery-0.5.2.dist-info}/top_level.txt +0 -0
test/readme_parser.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import re
|
|
2
2
|
import json
|
|
3
|
+
import ast
|
|
3
4
|
import os.path
|
|
4
5
|
|
|
5
6
|
def extract_code_blocks(readme_path):
|
|
@@ -24,6 +25,9 @@ def extract_code_blocks(readme_path):
|
|
|
24
25
|
# Process Python blocks to extract vfb calls
|
|
25
26
|
processed_python_blocks = []
|
|
26
27
|
for block in python_blocks:
|
|
28
|
+
# Skip blocks that contain import statements
|
|
29
|
+
if 'import' in block:
|
|
30
|
+
continue
|
|
27
31
|
# Look for vfb.* calls and extract them
|
|
28
32
|
vfb_calls = re.findall(r'(vfb\.[^)]*\))', block)
|
|
29
33
|
if vfb_calls:
|
|
@@ -32,35 +36,17 @@ def extract_code_blocks(readme_path):
|
|
|
32
36
|
# - get_templates() doesn't support force_refresh (no SOLR cache)
|
|
33
37
|
# - Performance test terms (FBbt_00003748, VFB_00101567) should use cache
|
|
34
38
|
for call in vfb_calls:
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
# Check if this call uses performance test terms - skip force_refresh for those
|
|
41
|
-
# NOTE: FBbt_00003748 (medulla) now needs force_refresh to get updated queries
|
|
42
|
-
if 'VFB_00101567' in call:
|
|
43
|
-
processed_python_blocks.append(call)
|
|
44
|
-
continue
|
|
45
|
-
|
|
46
|
-
# Check if the call already has parameters
|
|
47
|
-
if '(' in call and ')' in call:
|
|
48
|
-
# Check if force_refresh is already present
|
|
49
|
-
if 'force_refresh' in call:
|
|
50
|
-
# Already has force_refresh, use as-is
|
|
51
|
-
processed_python_blocks.append(call)
|
|
52
|
-
else:
|
|
53
|
-
# Insert force_refresh=True before the closing parenthesis
|
|
54
|
-
# Handle both cases: with and without existing parameters
|
|
55
|
-
if call.rstrip(')').endswith('('):
|
|
56
|
-
# No parameters: vfb.function()
|
|
57
|
-
modified_call = call[:-1] + 'force_refresh=True)'
|
|
58
|
-
else:
|
|
59
|
-
# Has parameters: vfb.function(param1, param2)
|
|
39
|
+
if 'FBbt_00003748' in call:
|
|
40
|
+
# Add force_refresh for medulla calls
|
|
41
|
+
if '(' in call and ')' in call:
|
|
42
|
+
if 'force_refresh' not in call:
|
|
60
43
|
modified_call = call[:-1] + ', force_refresh=True)'
|
|
61
|
-
|
|
44
|
+
processed_python_blocks.append(modified_call)
|
|
45
|
+
else:
|
|
46
|
+
processed_python_blocks.append(call)
|
|
47
|
+
else:
|
|
48
|
+
processed_python_blocks.append(call)
|
|
62
49
|
else:
|
|
63
|
-
# Shouldn't happen, but include original call if no parentheses
|
|
64
50
|
processed_python_blocks.append(call)
|
|
65
51
|
|
|
66
52
|
# Process JSON blocks
|
|
@@ -69,9 +55,6 @@ def extract_code_blocks(readme_path):
|
|
|
69
55
|
try:
|
|
70
56
|
# Clean up the JSON text
|
|
71
57
|
json_text = block.strip()
|
|
72
|
-
# Convert Python boolean literals to JSON booleans using regex
|
|
73
|
-
json_text = re.sub(r'\bTrue\b', 'true', json_text)
|
|
74
|
-
json_text = re.sub(r'\bFalse\b', 'false', json_text)
|
|
75
58
|
# Parse the JSON and add to results
|
|
76
59
|
json_obj = json.loads(json_text)
|
|
77
60
|
processed_json_blocks.append(json_obj)
|
|
@@ -95,6 +78,16 @@ def generate_python_file(python_blocks, output_path):
|
|
|
95
78
|
for block in python_blocks:
|
|
96
79
|
f.write(f'results.append({block})\n')
|
|
97
80
|
|
|
81
|
+
def generate_code_strings_file(python_blocks, output_path):
|
|
82
|
+
"""
|
|
83
|
+
Generates a Python file containing the extracted code blocks as strings in a results list.
|
|
84
|
+
"""
|
|
85
|
+
with open(output_path, 'w') as f:
|
|
86
|
+
f.write('results = [\n')
|
|
87
|
+
for block in python_blocks:
|
|
88
|
+
f.write(f' "{block}",\n')
|
|
89
|
+
f.write(']\n')
|
|
90
|
+
|
|
98
91
|
def generate_json_file(json_blocks, output_path):
|
|
99
92
|
"""
|
|
100
93
|
Generates a Python file containing the extracted JSON blocks as a Python list.
|
|
@@ -113,12 +106,13 @@ def generate_json_file(json_blocks, output_path):
|
|
|
113
106
|
|
|
114
107
|
f.write(python_list)
|
|
115
108
|
|
|
116
|
-
def process_readme(readme_path, python_output_path, json_output_path):
|
|
109
|
+
def process_readme(readme_path, python_output_path, code_strings_output_path, json_output_path):
|
|
117
110
|
"""
|
|
118
111
|
Process the README file and generate the test files.
|
|
119
112
|
"""
|
|
120
113
|
python_blocks, json_blocks = extract_code_blocks(readme_path)
|
|
121
114
|
generate_python_file(python_blocks, python_output_path)
|
|
115
|
+
generate_code_strings_file(python_blocks, code_strings_output_path)
|
|
122
116
|
generate_json_file(json_blocks, json_output_path)
|
|
123
117
|
|
|
124
118
|
return len(python_blocks), len(json_blocks)
|
|
@@ -128,10 +122,12 @@ if __name__ == "__main__":
|
|
|
128
122
|
readme_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'README.md')
|
|
129
123
|
python_blocks, json_blocks = extract_code_blocks(readme_path)
|
|
130
124
|
|
|
131
|
-
python_path = os.path.join(os.path.dirname(
|
|
132
|
-
|
|
125
|
+
python_path = os.path.join(os.path.dirname(__file__), 'test_examples.py')
|
|
126
|
+
code_strings_path = os.path.join(os.path.dirname(__file__), 'test_examples_code.py')
|
|
127
|
+
json_path = os.path.join(os.path.dirname(__file__), 'test_results.py')
|
|
133
128
|
|
|
134
129
|
generate_python_file(python_blocks, python_path)
|
|
130
|
+
generate_code_strings_file(python_blocks, code_strings_path)
|
|
135
131
|
generate_json_file(json_blocks, json_path)
|
|
136
132
|
|
|
137
133
|
print(f"Extracted {len(python_blocks)} Python blocks and {len(json_blocks)} JSON blocks")
|
test/term_info_queries_test.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import unittest
|
|
2
2
|
import time
|
|
3
|
-
from
|
|
4
|
-
from
|
|
3
|
+
from vfbquery.term_info_queries import deserialize_term_info, deserialize_term_info_from_dict, process
|
|
4
|
+
from vfbquery.solr_fetcher import SolrTermInfoFetcher
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
class TermInfoQueriesTest(unittest.TestCase):
|
|
@@ -10,6 +10,12 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
10
10
|
self.vc = SolrTermInfoFetcher()
|
|
11
11
|
self.variable = TestVariable("my_id", "my_name")
|
|
12
12
|
|
|
13
|
+
def get_term_info_or_skip(self, term_id):
|
|
14
|
+
try:
|
|
15
|
+
return self.vc.get_TermInfo([term_id], return_dataframe=False, summary=False)[0]
|
|
16
|
+
except Exception as e:
|
|
17
|
+
self.skipTest(f"SOLR server not available: {e}")
|
|
18
|
+
|
|
13
19
|
def test_term_info_deserialization(self):
|
|
14
20
|
terminfo_json = """
|
|
15
21
|
{"term": {"core": {"iri": "http://purl.obolibrary.org/obo/FBbt_00048514", "symbol": "BM-Taste", "types": ["Entity", "Adult", "Anatomy", "Cell", "Class", "Mechanosensory_system", "Nervous_system", "Neuron", "Sensory_neuron"], "short_form": "FBbt_00048514", "unique_facets": ["Adult", "Mechanosensory_system", "Nervous_system", "Sensory_neuron"], "label": "labial taste bristle mechanosensory neuron"}, "description": ["Any mechanosensory neuron (FBbt:00005919) that has sensory dendrite in some labellar taste bristle (FBbt:00004162)."], "comment": []}, "query": "Get JSON for Neuron Class", "version": "3d2a474", "parents": [{"symbol": "", "iri": "http://purl.obolibrary.org/obo/FBbt_00048508", "types": ["Entity", "Anatomy", "Cell", "Class", "Mechanosensory_system", "Nervous_system", "Neuron", "Sensory_neuron"], "short_form": "FBbt_00048508", "unique_facets": ["Mechanosensory_system", "Nervous_system", "Sensory_neuron"], "label": "mechanosensory neuron of chaeta"}, {"symbol": "", "iri": "http://purl.obolibrary.org/obo/FBbt_00051420", "types": ["Entity", "Adult", "Anatomy", "Cell", "Class", "Mechanosensory_system", "Nervous_system", "Neuron", "Sensory_neuron"], "short_form": "FBbt_00051420", "unique_facets": ["Adult", "Mechanosensory_system", "Nervous_system", "Sensory_neuron"], "label": "adult mechanosensory neuron"}, {"symbol": "", "iri": "http://purl.obolibrary.org/obo/FBbt_00048029", "types": ["Entity", "Adult", "Anatomy", "Cell", "Class", "Nervous_system", "Neuron", "Sensory_neuron"], "short_form": "FBbt_00048029", "unique_facets": ["Adult", "Nervous_system", "Sensory_neuron"], "label": "labellar taste bristle sensory neuron"}], "relationships": [{"relation": {"iri": "http://purl.obolibrary.org/obo/BFO_0000050", "label": "is part of", "type": "part_of"}, "object": {"symbol": "", "iri": "http://purl.obolibrary.org/obo/FBbt_00005892", "types": ["Entity", "Adult", "Anatomy", "Class", "Nervous_system"], "short_form": "FBbt_00005892", "unique_facets": ["Adult", "Nervous_system"], "label": "adult peripheral nervous system"}}], "xrefs": [], "anatomy_channel_image": [], "pub_syn": [{"synonym": {"scope": "has_exact_synonym", "label": "labellar taste bristle mechanosensitive neuron", "type": ""}, "pub": {"core": {"symbol": "", "iri": "http://flybase.org/reports/Unattributed", "types": ["Entity", "Individual", "pub"], "short_form": "Unattributed", "unique_facets": ["pub"], "label": ""}, "FlyBase": "", "PubMed": "", "DOI": ""}}, {"synonym": {"scope": "has_exact_synonym", "label": "labellar taste bristle mechanosensitive neuron", "type": ""}, "pub": {"core": {"symbol": "", "iri": "http://flybase.org/reports/Unattributed", "types": ["Entity", "Individual", "pub"], "short_form": "Unattributed", "unique_facets": ["pub"], "label": ""}, "FlyBase": "", "PubMed": "", "DOI": ""}}, {"synonym": {"scope": "has_exact_synonym", "label": "labial taste bristle mechanosensitive neuron", "type": ""}, "pub": {"core": {"symbol": "", "iri": "http://flybase.org/reports/Unattributed", "types": ["Entity", "Individual", "pub"], "short_form": "Unattributed", "unique_facets": ["pub"], "label": ""}, "FlyBase": "", "PubMed": "", "DOI": ""}}], "def_pubs": [{"core": {"symbol": "", "iri": "http://flybase.org/reports/FBrf0242472", "types": ["Entity", "Individual", "pub"], "short_form": "FBrf0242472", "unique_facets": ["pub"], "label": "Zhou et al., 2019, Sci. Adv. 5(5): eaaw5141"}, "FlyBase": "", "PubMed": "31131327", "DOI": "10.1126/sciadv.aaw5141"}], "targeting_splits": []}
|
|
@@ -40,7 +46,7 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
40
46
|
def test_term_info_deserialization_from_dict(self):
|
|
41
47
|
import pkg_resources
|
|
42
48
|
print("vfb_connect version:", pkg_resources.get_distribution("vfb_connect").version)
|
|
43
|
-
vfbTerm = self.
|
|
49
|
+
vfbTerm = self.get_term_info_or_skip('FBbt_00048514')
|
|
44
50
|
start_time = time.time()
|
|
45
51
|
terminfo = deserialize_term_info_from_dict(vfbTerm)
|
|
46
52
|
print("--- %s seconds ---" % (time.time() - start_time))
|
|
@@ -64,7 +70,7 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
64
70
|
|
|
65
71
|
self.assertEqual(0, len(terminfo.xrefs))
|
|
66
72
|
|
|
67
|
-
self.assertEqual(
|
|
73
|
+
self.assertEqual(8, len(terminfo.pub_syn))
|
|
68
74
|
|
|
69
75
|
# Check that we have the expected synonym labels (order-independent)
|
|
70
76
|
synonym_labels = [entry.synonym.label for entry in terminfo.pub_syn]
|
|
@@ -84,7 +90,7 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
84
90
|
self.assertEqual("33657409", labellar_hmsn_entry.pub.PubMed)
|
|
85
91
|
|
|
86
92
|
def test_term_info_serialization_individual_anatomy(self):
|
|
87
|
-
term_info_dict = self.
|
|
93
|
+
term_info_dict = self.get_term_info_or_skip('VFB_00010001')
|
|
88
94
|
print(term_info_dict)
|
|
89
95
|
start_time = time.time()
|
|
90
96
|
serialized = process(term_info_dict, self.variable)
|
|
@@ -133,7 +139,7 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
133
139
|
'reference': '[VFB_00017894,VFB_00010001]'} in serialized["thumbnail"])
|
|
134
140
|
|
|
135
141
|
def test_term_info_serialization_class(self):
|
|
136
|
-
term_info_dict = self.
|
|
142
|
+
term_info_dict = self.get_term_info_or_skip('FBbt_00048531')
|
|
137
143
|
print(term_info_dict)
|
|
138
144
|
start_time = time.time()
|
|
139
145
|
serialized = process(term_info_dict, self.variable)
|
|
@@ -145,7 +151,7 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
145
151
|
self.assertFalse("link" in serialized)
|
|
146
152
|
self.assertEqual(4, len(serialized["types"]))
|
|
147
153
|
self.assertTrue("Anatomy" in serialized["types"])
|
|
148
|
-
self.assertEqual("Cyst composed of two cyst cells following the division of a newly-formed cystoblast in the germarium. The two cells are connected by a cytoplasmic bridge.\n([King, 1970](FBrf0021038))", serialized["description"])
|
|
154
|
+
self.assertEqual("Cyst composed of two cyst cells following the division of a newly-formed cystoblast in the germarium. The two cells are connected by a cytoplasmic bridge.\n([Spradling, 1993](FBrf0064777), [King, 1970](FBrf0021038))", serialized["description"])
|
|
149
155
|
self.assertTrue("synonyms" in serialized)
|
|
150
156
|
self.assertEqual(1, len(serialized["synonyms"]))
|
|
151
157
|
self.assertEqual("has_exact_synonym: germarial 2-cell cluster ([King, 1970](FBrf0021038))", serialized["synonyms"][0])
|
|
@@ -166,17 +172,20 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
166
172
|
self.assertFalse("examples" in serialized)
|
|
167
173
|
self.assertFalse("thumbnail" in serialized)
|
|
168
174
|
self.assertTrue("references" in serialized)
|
|
169
|
-
self.assertEqual(
|
|
175
|
+
self.assertEqual(2, len(serialized["references"]))
|
|
176
|
+
self.assertEqual({'link': '[Spradling, 1993, Bate, Martinez Arias, 1993: 1--70](FBrf0064777)',
|
|
177
|
+
'refs': ['http://flybase.org/reports/FBrf0064777'],
|
|
178
|
+
'types': ' pub'}, serialized["references"][0])
|
|
170
179
|
self.assertEqual({'link': '[King, 1970, Ovarian Development in Drosophila melanogaster. ](FBrf0021038)',
|
|
171
180
|
'refs': ['http://flybase.org/reports/FBrf0021038'],
|
|
172
|
-
'types': ' pub'}, serialized["references"][
|
|
181
|
+
'types': ' pub'}, serialized["references"][1])
|
|
173
182
|
self.assertFalse("targetingSplits" in serialized)
|
|
174
183
|
self.assertFalse("targetingNeurons" in serialized)
|
|
175
184
|
|
|
176
185
|
self.assertFalse("downloads_label" in serialized)
|
|
177
186
|
|
|
178
187
|
def test_term_info_serialization_neuron_class(self):
|
|
179
|
-
term_info_dict = self.
|
|
188
|
+
term_info_dict = self.get_term_info_or_skip('FBbt_00048999')
|
|
180
189
|
print(term_info_dict)
|
|
181
190
|
start_time = time.time()
|
|
182
191
|
serialized = process(term_info_dict, self.variable)
|
|
@@ -234,7 +243,7 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
234
243
|
self.assertFalse("template" in serialized)
|
|
235
244
|
|
|
236
245
|
def test_term_info_serialization_neuron_class2(self):
|
|
237
|
-
term_info_dict = self.
|
|
246
|
+
term_info_dict = self.get_term_info_or_skip('FBbt_00047030')
|
|
238
247
|
print(term_info_dict)
|
|
239
248
|
start_time = time.time()
|
|
240
249
|
serialized = process(term_info_dict, self.variable)
|
|
@@ -303,7 +312,7 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
303
312
|
self.assertFalse("template" in serialized)
|
|
304
313
|
|
|
305
314
|
def test_term_info_serialization_split_class(self):
|
|
306
|
-
term_info_dict = self.
|
|
315
|
+
term_info_dict = self.get_term_info_or_skip('VFBexp_FBtp0124468FBtp0133404')
|
|
307
316
|
print(term_info_dict)
|
|
308
317
|
start_time = time.time()
|
|
309
318
|
serialized = process(term_info_dict, self.variable)
|
|
@@ -330,18 +339,21 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
330
339
|
|
|
331
340
|
self.assertTrue("relationships" in serialized)
|
|
332
341
|
self.assertEqual(2, len(serialized["relationships"]))
|
|
333
|
-
|
|
342
|
+
expected_rel_1 = "has hemidriver [P{VT043927-GAL4.DBD}](FBtp0124468)"
|
|
343
|
+
expected_rel_2 = "has hemidriver [P{VT017491-p65.AD}](FBtp0133404)"
|
|
344
|
+
self.assertIn(expected_rel_1, serialized["relationships"])
|
|
345
|
+
self.assertIn(expected_rel_2, serialized["relationships"])
|
|
334
346
|
|
|
335
347
|
self.assertFalse("related_individuals" in serialized)
|
|
336
348
|
self.assertTrue("xrefs" in serialized)
|
|
337
349
|
self.assertEqual(2, len(serialized["xrefs"]))
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
350
|
+
expected_xref = {'icon': 'https://www.virtualflybrain.org/data/VFB/logos/fly_light_color.png',
|
|
351
|
+
'label': '[P{VT043927-GAL4.DBD} ∩ P{VT017491-p65.AD} expression pattern on '
|
|
352
|
+
'Driver Line on the FlyLight Split-GAL4 Site]'
|
|
353
|
+
'(http://splitgal4.janelia.org/cgi-bin/view_splitgal4_imagery.cgi?line=SS50574)',
|
|
354
|
+
'site': '[FlyLightSplit]'
|
|
355
|
+
'(http://splitgal4.janelia.org/cgi-bin/view_splitgal4_imagery.cgi?line=SS50574) '}
|
|
356
|
+
self.assertIn(expected_xref, serialized["xrefs"])
|
|
345
357
|
|
|
346
358
|
self.assertTrue("examples" in serialized)
|
|
347
359
|
self.assertFalse("thumbnail" in serialized)
|
|
@@ -357,7 +369,7 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
357
369
|
self.assertFalse("template" in serialized)
|
|
358
370
|
|
|
359
371
|
def test_term_info_serialization_dataset(self):
|
|
360
|
-
term_info_dict = self.
|
|
372
|
+
term_info_dict = self.get_term_info_or_skip('Ito2013')
|
|
361
373
|
print(term_info_dict)
|
|
362
374
|
start_time = time.time()
|
|
363
375
|
serialized = process(term_info_dict, self.variable)
|
|
@@ -395,7 +407,7 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
395
407
|
self.assertTrue("clone of Ito 2013" in sample_example["name"])
|
|
396
408
|
|
|
397
409
|
def test_term_info_serialization_license(self):
|
|
398
|
-
term_info_dict = self.
|
|
410
|
+
term_info_dict = self.get_term_info_or_skip('VFBlicense_CC_BY_NC_3_0')
|
|
399
411
|
print(term_info_dict)
|
|
400
412
|
start_time = time.time()
|
|
401
413
|
serialized = process(term_info_dict, self.variable)
|
|
@@ -430,7 +442,7 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
430
442
|
self.assertFalse("template" in serialized)
|
|
431
443
|
|
|
432
444
|
def test_term_info_serialization_template(self):
|
|
433
|
-
term_info_dict = self.
|
|
445
|
+
term_info_dict = self.get_term_info_or_skip('VFB_00200000')
|
|
434
446
|
print(term_info_dict)
|
|
435
447
|
start_time = time.time()
|
|
436
448
|
serialized = process(term_info_dict, self.variable)
|
|
@@ -458,7 +470,7 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
458
470
|
self.assertFalse("examples" in serialized)
|
|
459
471
|
self.assertTrue("thumbnail" in serialized)
|
|
460
472
|
self.assertEqual(1, len(serialized["thumbnail"]))
|
|
461
|
-
self.assertEqual({'data': '
|
|
473
|
+
self.assertEqual({'data': 'https://www.virtualflybrain.org/data/VFB/i/0020/0000/VFB_00200000/thumbnailT.png',
|
|
462
474
|
'format': 'PNG',
|
|
463
475
|
'name': 'JRC2018UnisexVNC',
|
|
464
476
|
'reference': 'VFB_00200000'}, serialized["thumbnail"][0])
|
|
@@ -486,7 +498,7 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
486
498
|
self.assertEqual("[JRC2018UnisexVNC](VFB_00200000)", serialized["template"])
|
|
487
499
|
|
|
488
500
|
def test_term_info_serialization_pub(self):
|
|
489
|
-
term_info_dict = self.
|
|
501
|
+
term_info_dict = self.get_term_info_or_skip('FBrf0243986')
|
|
490
502
|
print(term_info_dict)
|
|
491
503
|
start_time = time.time()
|
|
492
504
|
serialized = process(term_info_dict, self.variable)
|
|
@@ -531,15 +543,18 @@ class TermInfoQueriesTest(unittest.TestCase):
|
|
|
531
543
|
"""
|
|
532
544
|
import vfbquery as vfb
|
|
533
545
|
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
546
|
+
try:
|
|
547
|
+
# Test performance for FBbt_00003748 (mushroom body)
|
|
548
|
+
start_time = time.time()
|
|
549
|
+
result_1 = vfb.get_term_info('FBbt_00003748')
|
|
550
|
+
duration_1 = time.time() - start_time
|
|
551
|
+
|
|
552
|
+
# Test performance for VFB_00101567 (individual anatomy)
|
|
553
|
+
start_time = time.time()
|
|
554
|
+
result_2 = vfb.get_term_info('VFB_00101567')
|
|
555
|
+
duration_2 = time.time() - start_time
|
|
556
|
+
except Exception as e:
|
|
557
|
+
self.skipTest(f"SOLR server not available: {e}")
|
|
543
558
|
|
|
544
559
|
# Print performance metrics for GitHub Actions logs
|
|
545
560
|
print(f"\n" + "="*50)
|
test/test_default_caching.py
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Test VFBquery default caching functionality.
|
|
3
3
|
|
|
4
|
-
These tests ensure that the
|
|
5
|
-
|
|
4
|
+
These tests ensure that the SOLR-based caching system works correctly
|
|
5
|
+
and provides expected performance benefits with 3-month TTL.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
import unittest
|
|
@@ -12,165 +12,166 @@ from unittest.mock import MagicMock
|
|
|
12
12
|
import sys
|
|
13
13
|
|
|
14
14
|
# Mock vispy imports before importing vfbquery
|
|
15
|
-
for module in ['vispy', 'vispy.scene', 'vispy.util', 'vispy.util.fonts',
|
|
16
|
-
'vispy.util.fonts._triage', 'vispy.util.fonts._quartz',
|
|
17
|
-
'vispy.ext', 'vispy.ext.cocoapy', 'navis', 'navis.plotting',
|
|
15
|
+
for module in ['vispy', 'vispy.scene', 'vispy.util', 'vispy.util.fonts',
|
|
16
|
+
'vispy.util.fonts._triage', 'vispy.util.fonts._quartz',
|
|
17
|
+
'vispy.ext', 'vispy.ext.cocoapy', 'navis', 'navis.plotting',
|
|
18
18
|
'navis.plotting.vispy', 'navis.plotting.vispy.viewer']:
|
|
19
19
|
sys.modules[module] = MagicMock()
|
|
20
20
|
|
|
21
21
|
# Set environment variables
|
|
22
22
|
os.environ.update({
|
|
23
23
|
'MPLBACKEND': 'Agg',
|
|
24
|
-
'VISPY_GL_LIB': 'osmesa',
|
|
24
|
+
'VISPY_GL_LIB': 'osmesa',
|
|
25
25
|
'VISPY_USE_EGL': '0',
|
|
26
26
|
'VFBQUERY_CACHE_ENABLED': 'true'
|
|
27
27
|
})
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
class TestDefaultCaching(unittest.TestCase):
|
|
31
|
-
"""Test default caching behavior in VFBquery."""
|
|
32
|
-
|
|
31
|
+
"""Test default SOLR caching behavior in VFBquery."""
|
|
32
|
+
|
|
33
33
|
def setUp(self):
|
|
34
34
|
"""Set up test environment."""
|
|
35
35
|
# Clear any existing cache before each test
|
|
36
36
|
try:
|
|
37
37
|
import vfbquery
|
|
38
|
-
if hasattr(vfbquery, '
|
|
39
|
-
|
|
38
|
+
if hasattr(vfbquery, 'clear_solr_cache'):
|
|
39
|
+
# Clear cache for a test term
|
|
40
|
+
vfbquery.clear_solr_cache('term_info', 'FBbt_00003748')
|
|
40
41
|
except ImportError:
|
|
41
42
|
pass
|
|
42
|
-
|
|
43
|
+
|
|
43
44
|
def test_caching_enabled_by_default(self):
|
|
44
|
-
"""Test that caching is automatically enabled when importing vfbquery."""
|
|
45
|
+
"""Test that SOLR caching is automatically enabled when importing vfbquery."""
|
|
45
46
|
import vfbquery
|
|
46
|
-
|
|
47
|
-
# Check that caching functions are available
|
|
48
|
-
self.assertTrue(hasattr(vfbquery, '
|
|
49
|
-
self.assertTrue(hasattr(vfbquery, '
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
stats
|
|
53
|
-
|
|
54
|
-
self.
|
|
55
|
-
self.
|
|
47
|
+
|
|
48
|
+
# Check that SOLR caching functions are available
|
|
49
|
+
self.assertTrue(hasattr(vfbquery, 'get_solr_cache'))
|
|
50
|
+
self.assertTrue(hasattr(vfbquery, 'clear_solr_cache'))
|
|
51
|
+
self.assertTrue(hasattr(vfbquery, 'get_solr_cache_stats_func'))
|
|
52
|
+
|
|
53
|
+
# Check that caching is enabled (we can't easily check SOLR stats without network calls)
|
|
54
|
+
# But we can verify the infrastructure is in place
|
|
55
|
+
self.assertTrue(hasattr(vfbquery, '__caching_available__'))
|
|
56
|
+
self.assertTrue(vfbquery.__caching_available__)
|
|
56
57
|
|
|
57
58
|
def test_cache_performance_improvement(self):
|
|
58
|
-
"""Test that caching provides performance improvement."""
|
|
59
|
+
"""Test that SOLR caching provides performance improvement."""
|
|
59
60
|
import vfbquery
|
|
60
|
-
|
|
61
|
+
|
|
61
62
|
test_term = 'FBbt_00003748' # medulla
|
|
62
|
-
|
|
63
|
+
|
|
63
64
|
# First call (cold - populates cache)
|
|
64
65
|
start_time = time.time()
|
|
65
66
|
result1 = vfbquery.get_term_info(test_term)
|
|
66
67
|
cold_time = time.time() - start_time
|
|
67
|
-
|
|
68
|
+
|
|
68
69
|
# Verify we got a result
|
|
69
70
|
self.assertIsNotNone(result1)
|
|
70
71
|
if result1 is not None:
|
|
71
72
|
self.assertIn('Name', result1)
|
|
72
|
-
|
|
73
|
+
|
|
73
74
|
# Second call (warm - should hit cache)
|
|
74
|
-
start_time = time.time()
|
|
75
|
+
start_time = time.time()
|
|
75
76
|
result2 = vfbquery.get_term_info(test_term)
|
|
76
77
|
warm_time = time.time() - start_time
|
|
77
|
-
|
|
78
|
+
|
|
78
79
|
# Verify caching is working (results should be identical)
|
|
79
80
|
self.assertIsNotNone(result2)
|
|
80
81
|
self.assertEqual(result1, result2) # Should be identical
|
|
81
|
-
|
|
82
|
+
|
|
82
83
|
# Note: Performance improvement may vary due to network conditions
|
|
83
84
|
# The main test is that caching prevents redundant computation
|
|
84
|
-
|
|
85
|
-
# Check
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
85
|
+
|
|
86
|
+
# Check SOLR cache statistics
|
|
87
|
+
solr_stats = vfbquery.get_solr_cache_stats_func()
|
|
88
|
+
self.assertIsInstance(solr_stats, dict)
|
|
89
|
+
self.assertIn('total_cache_documents', solr_stats)
|
|
89
90
|
|
|
90
91
|
def test_cache_statistics_tracking(self):
|
|
91
|
-
"""Test that cache statistics are properly tracked."""
|
|
92
|
+
"""Test that SOLR cache statistics are properly tracked."""
|
|
92
93
|
import vfbquery
|
|
93
|
-
|
|
94
|
-
#
|
|
95
|
-
vfbquery.
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
# Make a unique query that won't be cached
|
|
94
|
+
|
|
95
|
+
# Get baseline SOLR stats
|
|
96
|
+
initial_stats = vfbquery.get_solr_cache_stats_func()
|
|
97
|
+
initial_docs = initial_stats['total_cache_documents']
|
|
98
|
+
|
|
99
|
+
# Make a unique query that should populate cache
|
|
101
100
|
unique_term = 'FBbt_00005106' # Use a different term
|
|
102
101
|
result = vfbquery.get_term_info(unique_term)
|
|
103
102
|
self.assertIsNotNone(result)
|
|
104
|
-
|
|
105
|
-
# Check that stats were updated (
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
self.
|
|
111
|
-
self.assertGreaterEqual(updated_stats['memory_cache_size_mb'], 0)
|
|
103
|
+
|
|
104
|
+
# Check that SOLR stats were updated (may take time to reflect)
|
|
105
|
+
# We mainly verify the stats function works and returns reasonable data
|
|
106
|
+
updated_stats = vfbquery.get_solr_cache_stats_func()
|
|
107
|
+
self.assertIsInstance(updated_stats, dict)
|
|
108
|
+
self.assertIn('total_cache_documents', updated_stats)
|
|
109
|
+
self.assertIn('cache_efficiency', updated_stats)
|
|
112
110
|
|
|
113
111
|
def test_memory_size_tracking(self):
|
|
114
|
-
"""Test that
|
|
112
|
+
"""Test that SOLR cache size is properly tracked."""
|
|
115
113
|
import vfbquery
|
|
116
|
-
|
|
117
|
-
# Clear cache to start fresh
|
|
118
|
-
vfbquery.clear_vfbquery_cache()
|
|
119
|
-
|
|
114
|
+
|
|
120
115
|
# Cache a few different terms
|
|
121
116
|
test_terms = ['FBbt_00003748', 'VFB_00101567']
|
|
122
|
-
|
|
117
|
+
|
|
123
118
|
for term in test_terms:
|
|
124
|
-
vfbquery.get_term_info(term)
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
#
|
|
128
|
-
|
|
129
|
-
self.
|
|
119
|
+
result = vfbquery.get_term_info(term)
|
|
120
|
+
self.assertIsNotNone(result)
|
|
121
|
+
|
|
122
|
+
# Check SOLR cache stats are available
|
|
123
|
+
stats = vfbquery.get_solr_cache_stats_func()
|
|
124
|
+
self.assertIsInstance(stats, dict)
|
|
125
|
+
self.assertIn('estimated_size_mb', stats)
|
|
126
|
+
self.assertGreaterEqual(stats['estimated_size_mb'], 0)
|
|
130
127
|
|
|
131
128
|
def test_cache_ttl_configuration(self):
|
|
132
|
-
"""Test that cache TTL is properly configured."""
|
|
129
|
+
"""Test that SOLR cache TTL is properly configured."""
|
|
133
130
|
import vfbquery
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
131
|
+
|
|
132
|
+
# Get SOLR cache instance to check TTL
|
|
133
|
+
solr_cache = vfbquery.get_solr_cache()
|
|
134
|
+
self.assertIsNotNone(solr_cache)
|
|
135
|
+
|
|
136
|
+
# Check that TTL is configured (we can't easily check the exact value without accessing private attributes)
|
|
137
|
+
# But we can verify the cache object exists and has expected methods
|
|
138
|
+
self.assertTrue(hasattr(solr_cache, 'ttl_hours'))
|
|
139
|
+
self.assertTrue(hasattr(solr_cache, 'cache_result'))
|
|
140
|
+
self.assertTrue(hasattr(solr_cache, 'get_cached_result'))
|
|
140
141
|
|
|
141
142
|
def test_transparent_caching(self):
|
|
142
143
|
"""Test that regular VFBquery functions are transparently cached."""
|
|
143
144
|
import vfbquery
|
|
144
|
-
|
|
145
|
+
|
|
145
146
|
# Test that get_term_info and get_instances are using cached versions
|
|
146
147
|
test_term = 'FBbt_00003748'
|
|
147
|
-
|
|
148
|
+
|
|
148
149
|
# These should work with caching transparently
|
|
149
150
|
term_info = vfbquery.get_term_info(test_term)
|
|
150
151
|
self.assertIsNotNone(term_info)
|
|
151
|
-
|
|
152
|
+
|
|
152
153
|
instances = vfbquery.get_instances(test_term, limit=5)
|
|
153
154
|
self.assertIsNotNone(instances)
|
|
154
|
-
|
|
155
|
-
#
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
self.assertIsInstance(stats, dict)
|
|
160
|
-
self.assertIn('enabled', stats)
|
|
161
|
-
self.assertTrue(stats['enabled'])
|
|
155
|
+
|
|
156
|
+
# SOLR cache should be accessible
|
|
157
|
+
solr_stats = vfbquery.get_solr_cache_stats_func()
|
|
158
|
+
self.assertIsInstance(solr_stats, dict)
|
|
159
|
+
self.assertIn('total_cache_documents', solr_stats)
|
|
162
160
|
|
|
163
161
|
def test_cache_disable_environment_variable(self):
|
|
164
162
|
"""Test that caching can be disabled via environment variable."""
|
|
165
163
|
# This test would need to be run in a separate process to test
|
|
166
164
|
# the environment variable behavior at import time
|
|
167
165
|
# For now, just verify the current state respects the env var
|
|
168
|
-
|
|
166
|
+
|
|
169
167
|
cache_enabled = os.getenv('VFBQUERY_CACHE_ENABLED', 'true').lower()
|
|
170
168
|
if cache_enabled not in ('false', '0', 'no', 'off'):
|
|
171
169
|
import vfbquery
|
|
172
|
-
|
|
173
|
-
|
|
170
|
+
# If caching is enabled, SOLR cache should be available
|
|
171
|
+
solr_cache = vfbquery.get_solr_cache()
|
|
172
|
+
self.assertIsNotNone(solr_cache)
|
|
173
|
+
self.assertTrue(hasattr(vfbquery, '__caching_available__'))
|
|
174
|
+
self.assertTrue(vfbquery.__caching_available__)
|
|
174
175
|
|
|
175
176
|
|
|
176
177
|
if __name__ == '__main__':
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
results = [
|
|
2
|
+
"vfb.get_term_info('FBbt_00003748', force_refresh=True)",
|
|
3
|
+
"vfb.get_term_info('VFB_00000001')",
|
|
4
|
+
"vfb.get_term_info('VFB_00101567')",
|
|
5
|
+
"vfb.get_instances('FBbt_00003748', return_dataframe=False, force_refresh=True)",
|
|
6
|
+
"vfb.get_templates(return_dataframe=False)",
|
|
7
|
+
]
|