decimer 2.8.0__tar.gz → 2.8.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/__init__.py +8 -1
  2. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/decimer.py +110 -9
  3. {decimer-2.8.0 → decimer-2.8.1}/PKG-INFO +29 -1
  4. {decimer-2.8.0 → decimer-2.8.1}/README.md +28 -0
  5. {decimer-2.8.0 → decimer-2.8.1}/decimer.egg-info/PKG-INFO +29 -1
  6. {decimer-2.8.0 → decimer-2.8.1}/decimer.egg-info/SOURCES.txt +2 -1
  7. {decimer-2.8.0 → decimer-2.8.1}/setup.py +1 -1
  8. decimer-2.8.1/tests/test_lazy_models.py +317 -0
  9. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/DECIMER_EfficinetNetV2_Transfomer_Trainer.py +0 -0
  10. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/Efficient_Net_encoder.py +0 -0
  11. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/Predictor_usingCheckpoints.py +0 -0
  12. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/Repack_model.py +0 -0
  13. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/Transformer_decoder.py +0 -0
  14. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/Utils/Create_TFrecord_From_pickles.py +0 -0
  15. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/Utils/Create_image_tokenizer.py +0 -0
  16. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/config.py +0 -0
  17. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/efficientnetv2/effnetv2_configs.py +0 -0
  18. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/efficientnetv2/effnetv2_model.py +0 -0
  19. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/efficientnetv2/hparams.py +0 -0
  20. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/efficientnetv2/utils.py +0 -0
  21. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/pre_process.py +0 -0
  22. {decimer-2.8.0 → decimer-2.8.1}/DECIMER/utils.py +0 -0
  23. {decimer-2.8.0 → decimer-2.8.1}/LICENSE +0 -0
  24. {decimer-2.8.0 → decimer-2.8.1}/decimer.egg-info/dependency_links.txt +0 -0
  25. {decimer-2.8.0 → decimer-2.8.1}/decimer.egg-info/entry_points.txt +0 -0
  26. {decimer-2.8.0 → decimer-2.8.1}/decimer.egg-info/requires.txt +0 -0
  27. {decimer-2.8.0 → decimer-2.8.1}/decimer.egg-info/top_level.txt +0 -0
  28. {decimer-2.8.0 → decimer-2.8.1}/pyproject.toml +0 -0
  29. {decimer-2.8.0 → decimer-2.8.1}/setup.cfg +0 -0
  30. {decimer-2.8.0 → decimer-2.8.1}/tests/test_functions.py +0 -0
@@ -17,15 +17,22 @@ Typical usage example::
17
17
  SMILES = predict_SMILES(image_path)
18
18
  print(SMILES)
19
19
 
20
+ Importing DECIMER does not download or load any model. The models are downloaded
21
+ (if they are missing) and loaded when predict_SMILES() is first called, or earlier
22
+ by calling ``load_models()``. They are stored in ``$PYSTOW_HOME/DECIMER-V2/``
23
+ (``~/.data/DECIMER-V2/`` by default); a directory that already holds them is used
24
+ without any network access.
25
+
20
26
  For comments, bug reports, or feature ideas,
21
27
  please raise an issue on the GitHub repository.
22
28
  """
23
29
 
24
- __version__ = "2.7.2"
30
+ __version__ = "2.8.1"
25
31
 
26
32
  __all__ = [
27
33
  "DECIMER",
28
34
  ]
29
35
 
30
36
 
37
+ from .decimer import load_models
31
38
  from .decimer import predict_SMILES
@@ -2,6 +2,7 @@ import logging
2
2
  import os
3
3
  import pickle
4
4
  import sys
5
+ import threading
5
6
  from typing import List
6
7
  from typing import Tuple
7
8
 
@@ -68,20 +69,34 @@ gpus = tf.config.experimental.list_physical_devices("GPU")
68
69
  for gpu in gpus:
69
70
  tf.config.experimental.set_memory_growth(gpu, True)
70
71
 
71
- # Set path
72
- default_path = pystow.join("DECIMER-V2")
73
-
72
+ # The models are NOT downloaded or loaded when this module is imported. They are
73
+ # downloaded (if missing) and loaded the first time they are needed, either by the
74
+ # first predict_SMILES() call or by an explicit load_models(), and cached afterwards.
75
+ # They live in $PYSTOW_HOME/DECIMER-V2/ (PYSTOW_HOME defaults to ~/.data), read when
76
+ # the models are first needed. A directory that already holds them is used as it is,
77
+ # without any network access.
74
78
  model_urls = {
75
79
  "DECIMER": "https://zenodo.org/record/8300489/files/models.zip",
76
80
  "DECIMER_HandDrawn": "https://zenodo.org/records/10781330/files/DECIMER_HandDrawn_model.zip",
77
81
  }
78
82
 
79
83
 
84
+ def _get_default_path():
85
+ """Directory that holds (or will hold) the downloaded models.
86
+
87
+ Looked up when the models are first needed, so that PYSTOW_HOME can still be set
88
+ after DECIMER was imported. Nothing is created here.
89
+ """
90
+ return pystow.join("DECIMER-V2", ensure_exists=False)
91
+
92
+
80
93
  def get_models(model_urls: dict):
81
94
  """Download and load models from the provided URLs.
82
95
 
83
96
  This function downloads models from the provided URLs to a default location,
84
- then loads tokenizers and TensorFlow saved models.
97
+ then loads tokenizers and TensorFlow saved models. It loads everything at once
98
+ and does not touch the cached models of predict_SMILES(); most callers want
99
+ load_models() instead.
85
100
 
86
101
  Args:
87
102
  model_urls (dict): A dictionary containing model names as keys and their corresponding URLs as values.
@@ -93,7 +108,9 @@ def get_models(model_urls: dict):
93
108
  - DECIMER_Hand_drawn (tf.saved_model): TensorFlow saved model for DECIMER HandDrawn.
94
109
  """
95
110
  # Download models to a default location
96
- model_paths = utils.ensure_models(default_path=default_path, model_urls=model_urls)
111
+ model_paths = utils.ensure_models(
112
+ default_path=_get_default_path(), model_urls=model_urls
113
+ )
97
114
 
98
115
  # Load tokenizers
99
116
  tokenizer_path = os.path.join(
@@ -108,7 +125,86 @@ def get_models(model_urls: dict):
108
125
  return tokenizer, DECIMER_V2, DECIMER_Hand_drawn
109
126
 
110
127
 
111
- tokenizer, DECIMER_V2, DECIMER_Hand_drawn = get_models(model_urls)
128
+ def _load_tokenizer():
129
+ """Ensure the printed-structure model files are present and load its tokenizer.
130
+
131
+ The tokenizer of the printed-structure model decodes the output of both models.
132
+ """
133
+ model_paths = utils.ensure_models(
134
+ default_path=_get_default_path(),
135
+ model_urls={"DECIMER": model_urls["DECIMER"]},
136
+ )
137
+ return load_tokenizer(
138
+ os.path.join(model_paths["DECIMER"], "assets", "tokenizer_SMILES.pkl")
139
+ )
140
+
141
+
142
+ def _load_saved_model(model_name: str):
143
+ """Ensure the files of one model are present, then load its SavedModel."""
144
+ model_paths = utils.ensure_models(
145
+ default_path=_get_default_path(),
146
+ model_urls={model_name: model_urls[model_name]},
147
+ )
148
+ return tf.saved_model.load(model_paths[model_name])
149
+
150
+
151
+ # Module-level names that are filled on first use. Once loaded, an object is stored
152
+ # in this module's globals under the same name that decimer 2.8.0 and older bound at
153
+ # import, so `from DECIMER.decimer import tokenizer, DECIMER_V2` keeps working.
154
+ _lazy_loaders = {
155
+ "tokenizer": _load_tokenizer,
156
+ "DECIMER_V2": lambda: _load_saved_model("DECIMER"),
157
+ "DECIMER_Hand_drawn": lambda: _load_saved_model("DECIMER_HandDrawn"),
158
+ }
159
+ _load_lock = threading.RLock()
160
+
161
+
162
+ def _lazy(name: str):
163
+ """Return tokenizer, DECIMER_V2 or DECIMER_Hand_drawn, loading it on first use.
164
+
165
+ Downloads the files first if they are missing. The lock makes concurrent first
166
+ calls download and load once. A failed download or load is not cached, so the
167
+ next call tries again.
168
+ """
169
+ try:
170
+ return globals()[name]
171
+ except KeyError:
172
+ pass
173
+ with _load_lock:
174
+ if name not in globals():
175
+ globals()[name] = _lazy_loaders[name]()
176
+ return globals()[name]
177
+
178
+
179
+ def __getattr__(name: str):
180
+ """Load tokenizer, DECIMER_V2 and DECIMER_Hand_drawn on first attribute access."""
181
+ if name == "default_path":
182
+ return _get_default_path()
183
+ if name in _lazy_loaders:
184
+ return _lazy(name)
185
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
186
+
187
+
188
+ def load_models(printed: bool = True, hand_drawn: bool = False) -> None:
189
+ """Download (if missing) and load the models now, instead of at the first prediction.
190
+
191
+ Call this once at start-up if the first predict_SMILES() call should not pay for
192
+ the download and the loading, for example in a server or when building an image.
193
+ Models that are already loaded are not loaded again. The tokenizer, which both
194
+ models need, is always loaded.
195
+
196
+ Args:
197
+ printed (bool): Load the model for printed chemical structures (the default
198
+ model of predict_SMILES).
199
+ hand_drawn (bool): Also load the hand-drawn model. load_models(hand_drawn=True)
200
+ loads everything that decimer 2.8.0 and older loaded on import.
201
+ """
202
+ with _load_lock:
203
+ _lazy("tokenizer")
204
+ if printed:
205
+ _lazy("DECIMER_V2")
206
+ if hand_drawn:
207
+ _lazy("DECIMER_Hand_drawn")
112
208
 
113
209
 
114
210
  def detokenize_output(predicted_array: int) -> str:
@@ -121,7 +217,8 @@ def detokenize_output(predicted_array: int) -> str:
121
217
  Returns:
122
218
  (str): SMILES representation of the molecule
123
219
  """
124
- outputs = [tokenizer.index_word[i] for i in predicted_array[0].numpy()]
220
+ index_word = _lazy("tokenizer").index_word
221
+ outputs = [index_word[i] for i in predicted_array[0].numpy()]
125
222
  prediction = (
126
223
  "".join([str(elem) for elem in outputs])
127
224
  .replace("<start>", "")
@@ -145,9 +242,10 @@ def detokenize_output_add_confidence(
145
242
  Returns:
146
243
  str: SMILES string
147
244
  """
245
+ index_word = _lazy("tokenizer").index_word
148
246
  prediction_with_confidence = [
149
247
  (
150
- tokenizer.index_word[predicted_array[0].numpy()[i]],
248
+ index_word[predicted_array[0].numpy()[i]],
151
249
  confidence_array[i].numpy(),
152
250
  )
153
251
  for i in range(len(confidence_array))
@@ -177,7 +275,10 @@ def predict_SMILES(
177
275
  """
178
276
  chemical_structure = pre_process.decode_image(image_input)
179
277
 
180
- model = DECIMER_Hand_drawn if hand_drawn else DECIMER_V2
278
+ # The first call downloads the models that are missing and loads the ones it needs
279
+ # (the tokenizer, and the printed or the hand-drawn model); later calls reuse them.
280
+ _lazy("tokenizer")
281
+ model = _lazy("DECIMER_Hand_drawn" if hand_drawn else "DECIMER_V2")
181
282
  predicted_tokens, confidence_values = model(tf.constant(chemical_structure))
182
283
 
183
284
  predicted_SMILES = utils.decoder(detokenize_output(predicted_tokens))
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: decimer
3
- Version: 2.8.0
3
+ Version: 2.8.1
4
4
  Summary: DECIMER 2.7.2: Deep Learning for Chemical Image Recognition using Efficient-Net V2 + Transformer
5
5
  Home-page: https://github.com/Kohulan/DECIMER-Image_Transformer
6
6
  Author: Kohulan Rajan
@@ -139,6 +139,34 @@ SMILES = predict_SMILES(image_path)
139
139
  print(f"🎉 Decoded SMILES: {SMILES}")
140
140
  ```
141
141
 
142
+ ### 📥 Models: downloaded on first use
143
+
144
+ `import DECIMER` does not download or load anything. The model weights are not part of the pip package; they are
145
+ downloaded from Zenodo the first time they are needed and cached afterwards:
146
+
147
+ - The first `predict_SMILES(...)` call downloads the printed-structure model (about 300 MB) if it is missing, loads
148
+ it, and keeps it in memory. Later calls reuse it.
149
+ - The hand-drawn model is only downloaded and loaded by the first `predict_SMILES(..., hand_drawn=True)` call.
150
+ (Up to version 2.8.0 both models were downloaded and loaded when DECIMER was imported.)
151
+ - To pay for the download and the loading at a time of your choosing (at the start of a server, or when building an
152
+ image), call `load_models()` once. `load_models(hand_drawn=True)` loads both models, as `import DECIMER` did before.
153
+
154
+ ```python
155
+ import DECIMER
156
+
157
+ DECIMER.load_models() # optional: download if missing and load now
158
+ SMILES = DECIMER.predict_SMILES("path/to/your/chemical/masterpiece.jpg")
159
+ ```
160
+
161
+ The models are stored in `$PYSTOW_HOME/DECIMER-V2/` (`~/.data/DECIMER-V2/` by default), where `PYSTOW_HOME` is
162
+ read when the models are first needed. A directory that already holds `DECIMER_model/` (and, for hand-drawn
163
+ structures, `DECIMER_HandDrawn_model/`) is used as it is, without any network access, so an image with
164
+ pre-installed models works offline:
165
+
166
+ ```bash
167
+ export PYSTOW_HOME=/opt/decimer-models # contains DECIMER-V2/DECIMER_model/saved_model.pb, ...
168
+ ```
169
+
142
170
  ---
143
171
 
144
172
  ## ✍️ DECIMER - Hand-drawn Model
@@ -98,6 +98,34 @@ SMILES = predict_SMILES(image_path)
98
98
  print(f"🎉 Decoded SMILES: {SMILES}")
99
99
  ```
100
100
 
101
+ ### 📥 Models: downloaded on first use
102
+
103
+ `import DECIMER` does not download or load anything. The model weights are not part of the pip package; they are
104
+ downloaded from Zenodo the first time they are needed and cached afterwards:
105
+
106
+ - The first `predict_SMILES(...)` call downloads the printed-structure model (about 300 MB) if it is missing, loads
107
+ it, and keeps it in memory. Later calls reuse it.
108
+ - The hand-drawn model is only downloaded and loaded by the first `predict_SMILES(..., hand_drawn=True)` call.
109
+ (Up to version 2.8.0 both models were downloaded and loaded when DECIMER was imported.)
110
+ - To pay for the download and the loading at a time of your choosing (at the start of a server, or when building an
111
+ image), call `load_models()` once. `load_models(hand_drawn=True)` loads both models, as `import DECIMER` did before.
112
+
113
+ ```python
114
+ import DECIMER
115
+
116
+ DECIMER.load_models() # optional: download if missing and load now
117
+ SMILES = DECIMER.predict_SMILES("path/to/your/chemical/masterpiece.jpg")
118
+ ```
119
+
120
+ The models are stored in `$PYSTOW_HOME/DECIMER-V2/` (`~/.data/DECIMER-V2/` by default), where `PYSTOW_HOME` is
121
+ read when the models are first needed. A directory that already holds `DECIMER_model/` (and, for hand-drawn
122
+ structures, `DECIMER_HandDrawn_model/`) is used as it is, without any network access, so an image with
123
+ pre-installed models works offline:
124
+
125
+ ```bash
126
+ export PYSTOW_HOME=/opt/decimer-models # contains DECIMER-V2/DECIMER_model/saved_model.pb, ...
127
+ ```
128
+
101
129
  ---
102
130
 
103
131
  ## ✍️ DECIMER - Hand-drawn Model
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: decimer
3
- Version: 2.8.0
3
+ Version: 2.8.1
4
4
  Summary: DECIMER 2.7.2: Deep Learning for Chemical Image Recognition using Efficient-Net V2 + Transformer
5
5
  Home-page: https://github.com/Kohulan/DECIMER-Image_Transformer
6
6
  Author: Kohulan Rajan
@@ -139,6 +139,34 @@ SMILES = predict_SMILES(image_path)
139
139
  print(f"🎉 Decoded SMILES: {SMILES}")
140
140
  ```
141
141
 
142
+ ### 📥 Models: downloaded on first use
143
+
144
+ `import DECIMER` does not download or load anything. The model weights are not part of the pip package; they are
145
+ downloaded from Zenodo the first time they are needed and cached afterwards:
146
+
147
+ - The first `predict_SMILES(...)` call downloads the printed-structure model (about 300 MB) if it is missing, loads
148
+ it, and keeps it in memory. Later calls reuse it.
149
+ - The hand-drawn model is only downloaded and loaded by the first `predict_SMILES(..., hand_drawn=True)` call.
150
+ (Up to version 2.8.0 both models were downloaded and loaded when DECIMER was imported.)
151
+ - To pay for the download and the loading at a time of your choosing (at the start of a server, or when building an
152
+ image), call `load_models()` once. `load_models(hand_drawn=True)` loads both models, as `import DECIMER` did before.
153
+
154
+ ```python
155
+ import DECIMER
156
+
157
+ DECIMER.load_models() # optional: download if missing and load now
158
+ SMILES = DECIMER.predict_SMILES("path/to/your/chemical/masterpiece.jpg")
159
+ ```
160
+
161
+ The models are stored in `$PYSTOW_HOME/DECIMER-V2/` (`~/.data/DECIMER-V2/` by default), where `PYSTOW_HOME` is
162
+ read when the models are first needed. A directory that already holds `DECIMER_model/` (and, for hand-drawn
163
+ structures, `DECIMER_HandDrawn_model/`) is used as it is, without any network access, so an image with
164
+ pre-installed models works offline:
165
+
166
+ ```bash
167
+ export PYSTOW_HOME=/opt/decimer-models # contains DECIMER-V2/DECIMER_model/saved_model.pb, ...
168
+ ```
169
+
142
170
  ---
143
171
 
144
172
  ## ✍️ DECIMER - Hand-drawn Model
@@ -24,4 +24,5 @@ decimer.egg-info/dependency_links.txt
24
24
  decimer.egg-info/entry_points.txt
25
25
  decimer.egg-info/requires.txt
26
26
  decimer.egg-info/top_level.txt
27
- tests/test_functions.py
27
+ tests/test_functions.py
28
+ tests/test_lazy_models.py
@@ -15,7 +15,7 @@ with open("README.md", "r") as fh:
15
15
 
16
16
  setuptools.setup(
17
17
  name="decimer",
18
- version="2.8.0",
18
+ version="2.8.1",
19
19
  author="Kohulan Rajan",
20
20
  author_email="kohulan.rajan@uni-jena.de",
21
21
  maintainer="Kohulan Rajan, Otto Brinkhaus ",
@@ -0,0 +1,317 @@
1
+ """Tests for downloading and loading the models on first use instead of at import.
2
+
3
+ None of these tests needs the network or the real model files: the download function
4
+ and ``tf.saved_model.load`` are replaced by recorders, and the weights are tiny
5
+ placeholder files.
6
+ """
7
+ import os
8
+ import subprocess
9
+ import sys
10
+ import threading
11
+ import time
12
+ from pathlib import Path
13
+ from types import SimpleNamespace
14
+
15
+ import pytest
16
+ import tensorflow as tf
17
+
18
+ import DECIMER.decimer as decimer_module
19
+ import DECIMER.utils as utils
20
+ from DECIMER import load_models
21
+ from DECIMER import predict_SMILES
22
+
23
+ REPO_ROOT = Path(__file__).resolve().parents[1]
24
+ IMAGE = str(Path(__file__).resolve().parent / "caffeine.png")
25
+ LAZY_NAMES = ("tokenizer", "DECIMER_V2", "DECIMER_Hand_drawn")
26
+
27
+ # The model directory names in the Zenodo zip files, by download URL.
28
+ PRINTED_URL = decimer_module.model_urls["DECIMER"]
29
+ HAND_DRAWN_URL = decimer_module.model_urls["DECIMER_HandDrawn"]
30
+ DIRECTORY_OF_URL = {
31
+ PRINTED_URL: "DECIMER_model",
32
+ HAND_DRAWN_URL: "DECIMER_HandDrawn_model",
33
+ }
34
+
35
+
36
+ def write_model_files(models_dir, model_directory):
37
+ """Write the placeholder files that count as an installed model."""
38
+ directory = Path(models_dir) / model_directory
39
+ (directory / "assets").mkdir(parents=True, exist_ok=True)
40
+ (directory / "saved_model.pb").write_bytes(b"placeholder")
41
+ (directory / "assets" / "tokenizer_SMILES.pkl").write_bytes(b"placeholder")
42
+
43
+
44
+ class FakeModel:
45
+ """Stands in for a loaded SavedModel: <start> C O <end>, confidence 0.9 each."""
46
+
47
+ def __init__(self, path):
48
+ self.path = path
49
+
50
+ def __call__(self, image):
51
+ return tf.constant([[1, 2, 3, 4]]), tf.constant([0.9, 0.9, 0.9, 0.9])
52
+
53
+
54
+ @pytest.fixture
55
+ def recorded(tmp_path, monkeypatch):
56
+ """An empty model directory, and recorders for everything that would download or load."""
57
+ home = tmp_path / "pystow-home"
58
+ monkeypatch.setenv("PYSTOW_HOME", str(home))
59
+ record = SimpleNamespace(
60
+ home=home,
61
+ models_dir=home / "DECIMER-V2",
62
+ downloads=[],
63
+ loads=[],
64
+ tokenizer_paths=[],
65
+ download_delay=0.0,
66
+ download_error=None,
67
+ )
68
+
69
+ def fake_download(model_url, model_path, verbose=1):
70
+ record.downloads.append(model_url)
71
+ time.sleep(record.download_delay)
72
+ if record.download_error is not None:
73
+ raise record.download_error
74
+ write_model_files(model_path, DIRECTORY_OF_URL[model_url])
75
+
76
+ def fake_saved_model_load(path, *args, **kwargs):
77
+ record.loads.append(os.path.basename(path))
78
+ return FakeModel(path)
79
+
80
+ def fake_load_tokenizer(path):
81
+ record.tokenizer_paths.append(path)
82
+ return SimpleNamespace(index_word={1: "<start>", 2: "C", 3: "O", 4: "<end>"})
83
+
84
+ monkeypatch.setattr(utils, "download_trained_weights", fake_download)
85
+ monkeypatch.setattr(tf.saved_model, "load", fake_saved_model_load)
86
+ monkeypatch.setattr(decimer_module, "load_tokenizer", fake_load_tokenizer)
87
+
88
+ def forget_loaded_models():
89
+ for name in LAZY_NAMES:
90
+ vars(decimer_module).pop(name, None)
91
+
92
+ forget_loaded_models()
93
+ yield record
94
+ forget_loaded_models()
95
+
96
+
97
+ def test_import_does_no_network_and_loads_no_model(tmp_path):
98
+ """`import DECIMER` in a fresh interpreter: no network, no model, no files."""
99
+ home = tmp_path / "pystow-home"
100
+ script = """
101
+ import os
102
+ import pathlib
103
+ import socket
104
+
105
+
106
+ def refuse(*args, **kwargs):
107
+ raise AssertionError("network access while importing DECIMER")
108
+
109
+
110
+ socket.socket.connect = refuse
111
+ socket.create_connection = refuse
112
+ socket.getaddrinfo = refuse
113
+
114
+ import tensorflow as tf
115
+
116
+
117
+ def refuse_load(*args, **kwargs):
118
+ raise AssertionError("tf.saved_model.load called while importing DECIMER")
119
+
120
+
121
+ tf.saved_model.load = refuse_load
122
+
123
+ import DECIMER
124
+ import DECIMER.decimer as decimer_module
125
+
126
+ for name in ("tokenizer", "DECIMER_V2", "DECIMER_Hand_drawn"):
127
+ assert name not in vars(decimer_module), name
128
+ home = pathlib.Path(os.environ["PYSTOW_HOME"])
129
+ assert not (home / "DECIMER-V2").exists()
130
+ assert not list(home.rglob("saved_model.pb"))
131
+ print("import-is-passive")
132
+ """
133
+ env = dict(os.environ)
134
+ env["PYSTOW_HOME"] = str(home)
135
+ env["PYTHONPATH"] = str(REPO_ROOT)
136
+ env["TF_CPP_MIN_LOG_LEVEL"] = "3"
137
+ result = subprocess.run(
138
+ [sys.executable, "-c", script],
139
+ env=env,
140
+ cwd=tmp_path,
141
+ capture_output=True,
142
+ text=True,
143
+ timeout=600,
144
+ )
145
+ assert result.returncode == 0, result.stderr[-3000:]
146
+ assert "import-is-passive" in result.stdout
147
+
148
+
149
+ def test_first_call_downloads_and_loads_once_second_call_reuses(recorded):
150
+ assert predict_SMILES(IMAGE) == "CO"
151
+ assert recorded.downloads == [PRINTED_URL]
152
+ assert recorded.loads == ["DECIMER_model"]
153
+ assert len(recorded.tokenizer_paths) == 1
154
+
155
+ assert predict_SMILES(IMAGE) == "CO"
156
+ assert recorded.downloads == [PRINTED_URL]
157
+ assert recorded.loads == ["DECIMER_model"]
158
+ assert len(recorded.tokenizer_paths) == 1
159
+
160
+
161
+ def test_hand_drawn_model_is_fetched_and_loaded_only_when_asked_for(recorded):
162
+ predict_SMILES(IMAGE)
163
+ assert HAND_DRAWN_URL not in recorded.downloads
164
+ assert "DECIMER_HandDrawn_model" not in recorded.loads
165
+
166
+ assert predict_SMILES(IMAGE, hand_drawn=True) == "CO"
167
+ assert recorded.downloads == [PRINTED_URL, HAND_DRAWN_URL]
168
+ assert recorded.loads == ["DECIMER_model", "DECIMER_HandDrawn_model"]
169
+
170
+ predict_SMILES(IMAGE, hand_drawn=True)
171
+ assert recorded.downloads == [PRINTED_URL, HAND_DRAWN_URL]
172
+ assert recorded.loads == ["DECIMER_model", "DECIMER_HandDrawn_model"]
173
+
174
+
175
+ def test_hand_drawn_first_does_not_load_the_printed_model(recorded):
176
+ """The tokenizer needs the printed model's files, but not the printed model itself."""
177
+ predict_SMILES(IMAGE, hand_drawn=True)
178
+ assert recorded.loads == ["DECIMER_HandDrawn_model"]
179
+ assert PRINTED_URL in recorded.downloads
180
+
181
+
182
+ def test_confidence_output_is_unchanged(recorded):
183
+ smiles, tokens = predict_SMILES(IMAGE, confidence=True)
184
+ assert smiles == "CO"
185
+ assert [token for token, _ in tokens] == ["C", "O"]
186
+ assert [float(value) for _, value in tokens] == pytest.approx([0.9, 0.9])
187
+
188
+
189
+ def test_prepopulated_model_directory_is_used_without_network(recorded):
190
+ write_model_files(recorded.models_dir, "DECIMER_model")
191
+ write_model_files(recorded.models_dir, "DECIMER_HandDrawn_model")
192
+ recorded.download_error = AssertionError("downloaded although the files exist")
193
+
194
+ assert predict_SMILES(IMAGE) == "CO"
195
+ assert predict_SMILES(IMAGE, hand_drawn=True) == "CO"
196
+
197
+ assert recorded.downloads == []
198
+ assert recorded.loads == ["DECIMER_model", "DECIMER_HandDrawn_model"]
199
+ expected = os.path.join(
200
+ recorded.models_dir, "DECIMER_model", "assets", "tokenizer_SMILES.pkl"
201
+ )
202
+ assert recorded.tokenizer_paths == [expected]
203
+
204
+
205
+ def test_pystow_home_set_after_import_is_respected(recorded, tmp_path, monkeypatch):
206
+ """The directory is looked up at first use, so a late PYSTOW_HOME still applies."""
207
+ elsewhere = tmp_path / "baked-in-image"
208
+ write_model_files(elsewhere / "DECIMER-V2", "DECIMER_model")
209
+ monkeypatch.setenv("PYSTOW_HOME", str(elsewhere))
210
+ recorded.download_error = AssertionError("downloaded although the files exist")
211
+
212
+ assert predict_SMILES(IMAGE) == "CO"
213
+ assert recorded.downloads == []
214
+ assert not recorded.models_dir.exists()
215
+
216
+
217
+ def test_a_bad_image_fails_before_any_download(recorded, tmp_path):
218
+ # decimer raises a bare Exception for an image it cannot read
219
+ with pytest.raises(Exception):
220
+ predict_SMILES(str(tmp_path / "missing.png"))
221
+ assert recorded.downloads == []
222
+ assert recorded.loads == []
223
+
224
+
225
+ def test_load_models_loads_the_printed_model_once(recorded):
226
+ load_models()
227
+ assert recorded.downloads == [PRINTED_URL]
228
+ assert recorded.loads == ["DECIMER_model"]
229
+
230
+ load_models()
231
+ assert predict_SMILES(IMAGE) == "CO"
232
+ assert recorded.downloads == [PRINTED_URL]
233
+ assert recorded.loads == ["DECIMER_model"]
234
+
235
+
236
+ def test_load_models_with_hand_drawn_loads_everything_decimer_2_8_0_loaded_at_import(
237
+ recorded,
238
+ ):
239
+ load_models(hand_drawn=True)
240
+ assert recorded.downloads == [PRINTED_URL, HAND_DRAWN_URL]
241
+ assert sorted(recorded.loads) == ["DECIMER_HandDrawn_model", "DECIMER_model"]
242
+
243
+ load_models(hand_drawn=True)
244
+ assert len(recorded.loads) == 2
245
+
246
+
247
+ def test_load_models_without_the_printed_model(recorded):
248
+ load_models(printed=False, hand_drawn=True)
249
+ assert recorded.loads == ["DECIMER_HandDrawn_model"]
250
+
251
+
252
+ def test_failed_download_is_not_cached_and_is_retried(recorded):
253
+ recorded.download_error = OSError("zenodo is down")
254
+ with pytest.raises(OSError, match="zenodo is down"):
255
+ predict_SMILES(IMAGE)
256
+ assert recorded.loads == []
257
+
258
+ recorded.download_error = None
259
+ assert predict_SMILES(IMAGE) == "CO"
260
+ assert recorded.downloads == [PRINTED_URL, PRINTED_URL]
261
+ assert recorded.loads == ["DECIMER_model"]
262
+
263
+
264
+ def test_concurrent_first_calls_download_and_load_once(recorded):
265
+ recorded.download_delay = 0.3
266
+ threads_count = 4
267
+ barrier = threading.Barrier(threads_count)
268
+ results = []
269
+ errors = []
270
+
271
+ def first_call():
272
+ try:
273
+ barrier.wait(timeout=30)
274
+ results.append(predict_SMILES(IMAGE))
275
+ except Exception as error: # reported below, in the main thread
276
+ errors.append(error)
277
+
278
+ threads = [threading.Thread(target=first_call) for _ in range(threads_count)]
279
+ for thread in threads:
280
+ thread.start()
281
+ for thread in threads:
282
+ thread.join(timeout=120)
283
+
284
+ assert errors == []
285
+ assert results == ["CO"] * threads_count
286
+ assert recorded.downloads == [PRINTED_URL]
287
+ assert recorded.loads == ["DECIMER_model"]
288
+
289
+
290
+ def test_module_level_names_of_older_releases_still_resolve(recorded):
291
+ """`from DECIMER.decimer import tokenizer, DECIMER_V2, ...` worked with decimer <= 2.8.0."""
292
+ assert not hasattr(decimer_module, "no_such_name")
293
+ assert recorded.loads == []
294
+
295
+ from DECIMER.decimer import DECIMER_Hand_drawn
296
+ from DECIMER.decimer import DECIMER_V2
297
+ from DECIMER.decimer import tokenizer
298
+
299
+ assert isinstance(DECIMER_V2, FakeModel)
300
+ assert DECIMER_V2.path.endswith("DECIMER_model")
301
+ assert isinstance(DECIMER_Hand_drawn, FakeModel)
302
+ assert DECIMER_Hand_drawn.path.endswith("DECIMER_HandDrawn_model")
303
+ assert tokenizer.index_word[2] == "C"
304
+ assert decimer_module.default_path == recorded.models_dir
305
+ assert sorted(recorded.loads) == ["DECIMER_HandDrawn_model", "DECIMER_model"]
306
+
307
+
308
+ def test_a_model_assigned_to_the_module_is_used(recorded):
309
+ """Code that replaced decimer.DECIMER_V2 with its own model keeps working."""
310
+
311
+ class OtherModel(FakeModel):
312
+ def __call__(self, image):
313
+ return tf.constant([[1, 3, 3, 4]]), tf.constant([0.5, 0.5, 0.5, 0.5])
314
+
315
+ decimer_module.DECIMER_V2 = OtherModel("my-own-model")
316
+ assert predict_SMILES(IMAGE) == "OO"
317
+ assert recorded.loads == []
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes