dcicutils 8.8.3.1b5__tar.gz → 8.8.3.1b8__tar.gz

Sign up to get free protection for your applications and to get access to all the features.
Files changed (78) hide show
  1. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/PKG-INFO +1 -1
  2. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/progress_bar.py +39 -27
  3. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/structured_data.py +33 -4
  4. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/pyproject.toml +1 -1
  5. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/LICENSE.txt +0 -0
  6. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/README.rst +0 -0
  7. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/__init__.py +0 -0
  8. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/base.py +0 -0
  9. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/beanstalk_utils.py +0 -0
  10. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/bundle_utils.py +0 -0
  11. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/captured_output.py +0 -0
  12. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/cloudformation_utils.py +0 -0
  13. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/codebuild_utils.py +0 -0
  14. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/command_utils.py +0 -0
  15. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/common.py +0 -0
  16. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/contribution_scripts.py +0 -0
  17. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/contribution_utils.py +0 -0
  18. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/creds_utils.py +0 -0
  19. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/data_readers.py +0 -0
  20. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/data_utils.py +0 -0
  21. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/datetime_utils.py +0 -0
  22. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/deployment_utils.py +0 -0
  23. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/diff_utils.py +0 -0
  24. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/docker_utils.py +0 -0
  25. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/ecr_scripts.py +0 -0
  26. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/ecr_utils.py +0 -0
  27. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/ecs_utils.py +0 -0
  28. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/env_base.py +0 -0
  29. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/env_manager.py +0 -0
  30. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/env_scripts.py +0 -0
  31. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/env_utils.py +0 -0
  32. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/env_utils_legacy.py +0 -0
  33. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/es_utils.py +0 -0
  34. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/exceptions.py +0 -0
  35. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/ff_mocks.py +0 -0
  36. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/ff_utils.py +0 -0
  37. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/file_utils.py +0 -0
  38. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/function_cache_decorator.py +0 -0
  39. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/glacier_utils.py +0 -0
  40. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/jh_utils.py +0 -0
  41. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/kibana/dashboards.json +0 -0
  42. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/kibana/readme.md +0 -0
  43. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/lang_utils.py +0 -0
  44. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/license_policies/c4-infrastructure.jsonc +0 -0
  45. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/license_policies/c4-python-infrastructure.jsonc +0 -0
  46. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/license_policies/park-lab-common-server.jsonc +0 -0
  47. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/license_policies/park-lab-common.jsonc +0 -0
  48. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/license_policies/park-lab-gpl-pipeline.jsonc +0 -0
  49. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/license_policies/park-lab-pipeline.jsonc +0 -0
  50. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/license_utils.py +0 -0
  51. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/log_utils.py +0 -0
  52. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/misc_utils.py +0 -0
  53. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/obfuscation_utils.py +0 -0
  54. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/opensearch_utils.py +0 -0
  55. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/portal_object_utils.py +0 -0
  56. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/portal_utils.py +0 -0
  57. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/project_utils.py +0 -0
  58. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/qa_checkers.py +0 -0
  59. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/qa_utils.py +0 -0
  60. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/redis_tools.py +0 -0
  61. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/redis_utils.py +0 -0
  62. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/s3_utils.py +0 -0
  63. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/schema_utils.py +0 -0
  64. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/scripts/publish_to_pypi.py +0 -0
  65. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/scripts/run_license_checker.py +0 -0
  66. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/scripts/view_portal_object.py +0 -0
  67. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/secrets_utils.py +0 -0
  68. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/sheet_utils.py +0 -0
  69. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/snapshot_utils.py +0 -0
  70. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/ssl_certificate_utils.py +0 -0
  71. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/submitr/progress_constants.py +0 -0
  72. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/submitr/ref_lookup_strategy.py +0 -0
  73. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/task_utils.py +0 -0
  74. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/tmpfile_utils.py +0 -0
  75. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/trace_utils.py +0 -0
  76. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/validation_utils.py +0 -0
  77. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/variant_utils.py +0 -0
  78. {dcicutils-8.8.3.1b5 → dcicutils-8.8.3.1b8}/dcicutils/zip_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcicutils
3
- Version: 8.8.3.1b5
3
+ Version: 8.8.3.1b8
4
4
  Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
5
5
  Home-page: https://github.com/4dn-dcic/utils
6
6
  License: MIT
@@ -61,6 +61,8 @@ class ProgressBar:
61
61
  self._bar = None
62
62
  self._disabled = False
63
63
  self._done = False
64
+ self._really_done = False
65
+ self.foo = time.time()
64
66
  self._tidy_output_hack = (tidy_output_hack is True)
65
67
  self._started = time.time()
66
68
  self._stop_requested = False
@@ -104,11 +106,9 @@ class ProgressBar:
104
106
  return True
105
107
  return False
106
108
 
107
- def set_total(self, value: int, reset_eta: bool = False) -> None:
109
+ def set_total(self, value: int) -> None:
108
110
  if value == self._total:
109
111
  # If the total has not changed since last set then do nothing.
110
- if reset_eta and self._bar is not None:
111
- self._bar.reset()
112
112
  return
113
113
  if isinstance(value, int) and value > 0:
114
114
  self._total = value
@@ -119,16 +119,6 @@ class ProgressBar:
119
119
  self._bar.total = value
120
120
  self._bar.refresh()
121
121
 
122
- def reset_eta(self) -> None:
123
- # Since set_total does nothing if total is the same, provide
124
- # a way to reset the ETA if starting over with the same total.
125
- if self._bar is not None:
126
- progress = self._bar.n
127
- self._bar.reset()
128
- self._bar.total = self._total
129
- self._bar.n = progress
130
- self._bar.refresh()
131
-
132
122
  def set_progress(self, value: int) -> None:
133
123
  if isinstance(value, int) and value >= 0:
134
124
  if (self._bar is not None) or self._initialize():
@@ -141,17 +131,29 @@ class ProgressBar:
141
131
  self._bar.update(value)
142
132
  self._bar.refresh()
143
133
 
144
- def set_description(self, value: str) -> None:
145
- self._description = self._format_description(value)
134
+ def reset_eta(self) -> None:
135
+ # Since set_total does nothing if total is the same, provide
136
+ # a way to reset the ETA if starting over with the same total.
137
+ # But NOTE that resetting ETA will ALSO reset the ELAPSED time.
146
138
  if self._bar is not None:
147
- self._bar.set_description(self._description)
139
+ progress = self._bar.n
140
+ self._bar.reset()
141
+ self._bar.total = self._total
142
+ self._bar.n = progress
143
+ self._bar.refresh()
148
144
 
149
- def done(self) -> None:
145
+ def set_description(self, value: str) -> None:
146
+ if isinstance(value, str):
147
+ self._description = self._format_description(value)
148
+ if self._bar is not None:
149
+ self._bar.set_description(self._description)
150
+
151
+ def done(self, description: Optional[str] = None) -> None:
150
152
  if self._done or self._bar is None:
151
153
  return
152
154
  self._ended = time.time()
153
155
  self.set_progress(self.total)
154
- self._bar.set_description(self._description)
156
+ self.set_description(description)
155
157
  self._bar.refresh()
156
158
  # FYI: Do NOT do a bar.disable = True before a bar.close() or it messes up output
157
159
  # on multiple calls; found out the hard way; a couple hours will never get back :-/
@@ -202,6 +204,12 @@ class ProgressBar:
202
204
  def captured_output_for_testing(self) -> Optional[List[str]]:
203
205
  return self._captured_output_for_testing
204
206
 
207
+ @staticmethod
208
+ def format_captured_output_for_testing(description: str, total: int, progress: int) -> str:
209
+ percent = round((progress / total) * 100.0)
210
+ separator = "✓" if percent == 100 else "|"
211
+ return f"{description} {separator} {percent:>3}% ◀|### | {progress}/{total} | 0.0/s | 00:00 | ETA: 00:00"
212
+
205
213
  def _format_description(self, value: str) -> str:
206
214
  if not isinstance(value, str):
207
215
  value = ""
@@ -261,22 +269,19 @@ class ProgressBar:
261
269
  # string in the display string where the progress bar should actually go,
262
270
  # which we do in _format_description. Other minor things too; see below.
263
271
  sys_stdout_write = sys.stdout.write
264
- last_total = None ; last_progress = None ; last_text = None # noqa
272
+ last_text = None ; last_captured_output_text = None # noqa
265
273
  def tidy_stdout_write(text: str) -> None: # noqa
266
274
  nonlocal self, sys_stdout_write, sentinel_internal, spina, spini, spinn
267
- nonlocal last_total, last_progress, last_text
275
+ nonlocal last_text, last_captured_output_text
268
276
  def replace_first(value: str, match: str, replacement: str) -> str: # noqa
269
277
  return value[:i] + replacement + value[i + len(match):] if (i := value.find(match)) >= 0 else value
270
278
  def remove_extra_trailing_spaces(text: str) -> str: # noqa
271
279
  while text.endswith(" "):
272
280
  text = text[:-1]
273
281
  return text
274
- if not text:
282
+ if (not text) or (last_text == text) or self._really_done:
275
283
  return
276
- if self._bar:
277
- if ((self._bar.total == last_total) and (self._bar.n == last_progress) and (last_text == text)):
278
- return
279
- last_total = self._bar.total ; last_progress = self._bar.n ; last_text = text # noqa
284
+ last_text = text
280
285
  if (self._disabled or self._done) and sentinel_internal in text:
281
286
  # Another hack to really disable output on interrupt; in this case we set
282
287
  # tqdm.disable to True, but output can still dribble out, so if the output
@@ -293,6 +298,9 @@ class ProgressBar:
293
298
  text = replace_first(text, "s/ ", "/s ")
294
299
  sys_stdout_write(text)
295
300
  sys.stdout.flush()
301
+ if self._done:
302
+ self._really_done = True
303
+ return
296
304
  if self._captured_output_for_testing is not None:
297
305
  # For testing only we replace vacilliting values in the out like rate,
298
306
  # time elapsed, and ETA with static values; so that something like this:
@@ -300,7 +308,7 @@ class ProgressBar:
300
308
  # becomes something more static like this after calling this function:
301
309
  # > Working | 20% ◀|### | 1/5 | 0.0/s | 00:00 | ETA: 00:00
302
310
  # This function obviously has intimate knowledge of the output; better here than in tests.
303
- def replace_vacillating_values_with_static(text: str) -> str:
311
+ def replace_time_dependent_values_with_static(text: str) -> str:
304
312
  blocks = "\u2587|\u2588|\u2589|\u258a|\u258b|\u258c|\u258d|\u258e|\u258f"
305
313
  if (n := find_nth_from_end(text, "|", 5)) >= 8:
306
314
  pattern = re.compile(
@@ -311,7 +319,11 @@ class ProgressBar:
311
319
  return (text[0:n - 6].replace("\r", "") +
312
320
  match.expand(rf"\g<1>\g<2>### \g<3>\g<4>0.0\g<5>00:00\g<6>00:00"))
313
321
  return text
314
- self._captured_output_for_testing.append(replace_vacillating_values_with_static(text))
322
+ if text != "\n":
323
+ captured_output_text = replace_time_dependent_values_with_static(text)
324
+ if captured_output_text != last_captured_output_text:
325
+ self._captured_output_for_testing.append(captured_output_text)
326
+ last_captured_output_text = captured_output_text
315
327
  def restore_stdout_write() -> None: # noqa
316
328
  nonlocal sys_stdout_write
317
329
  if sys_stdout_write is not None:
@@ -154,7 +154,32 @@ class StructuredDataSet:
154
154
 
155
155
  @property
156
156
  def resolved_refs_with_uuids(self) -> List[str]:
157
- return list([{"path": resolved_ref[0], "uuid": resolved_ref[1]} for resolved_ref in self._resolved_refs])
157
+ return list([{"path": resolved_ref[0],
158
+ "uuid": resolved_ref[1] if len(resolved_ref) >= 2 else None}
159
+ for resolved_ref in self._resolved_refs])
160
+
161
+ @property
162
+ def unchecked_refs(self) -> List[str]:
163
+ """
164
+ Returns list of unchecked (for existence) references, grouped by reference path;
165
+ each object in the list has a path property and a srcs property which is a list of
166
+ src objects containing the type, column and row of the reference to the reference.
167
+ Note that this is only populated if the norefs option is specified.
168
+ """
169
+ def load_json(value: str) -> Optional[dict]:
170
+ try:
171
+ return json.loads(value)
172
+ except Exception:
173
+ return None
174
+ result = []
175
+ if self._norefs:
176
+ for ref in self._resolved_refs:
177
+ if len(ref) >= 3 and (ref_path := ref[0]) and (ref_src := load_json(ref[2])):
178
+ if existing_ref := [item for item in result if item.get("path") == ref_path]:
179
+ existing_ref[0]["srcs"].append(ref_src)
180
+ else:
181
+ result.append({"path": ref_path, "srcs": [ref_src]})
182
+ return result
158
183
 
159
184
  @property
160
185
  def upload_files(self) -> List[str]:
@@ -653,10 +678,14 @@ class Schema(SchemaBase):
653
678
  def map_ref(value: str, link_to: str, portal: Optional[Portal], src: Optional[str]) -> Any:
654
679
  nonlocal self, typeinfo
655
680
  if self._norefs:
681
+ # Here the caller has specified the (StructuredDataSet) norefs option
682
+ # which means we do not check for the existence of references at all.
656
683
  if value:
657
- # Dump the src because this cannot add dictionary to a set; note that
658
- # this is ONLY used for smaht-submitr/submit-metadata-bundle --info --refs.
659
- self._resolved_refs.add((f"/{link_to}/{value}", json.dumps(src) if isinstance(src, dict) else None))
684
+ # Dump the src as a JSON string because a dictionary cannot be added to a set; note
685
+ # that this is ONLY used for smaht-submitr/submit-metadata-bundle --info --refs.
686
+ # This info can be gotten at using StructureDataSet.unchecked_refs.
687
+ self._resolved_refs.add((f"/{link_to}/{value}", None,
688
+ json.dumps(src) if isinstance(src, dict) else None))
660
689
  return value
661
690
  if not value:
662
691
  if (column := typeinfo.get("column")) and column in self.data.get("required", []):
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "dcicutils"
3
- version = "8.8.3.1b5" # TODO: To become 8.8.4
3
+ version = "8.8.3.1b8" # TODO: To become 8.8.4
4
4
  description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
5
5
  authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
6
6
  license = "MIT"
File without changes
File without changes