scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (704) hide show
  1. scitex/__init__.py +53 -15
  2. scitex/__main__.py +72 -26
  3. scitex/__version__.py +1 -1
  4. scitex/_sh.py +145 -23
  5. scitex/ai/__init__.py +30 -16
  6. scitex/ai/_gen_ai/_Anthropic.py +5 -7
  7. scitex/ai/_gen_ai/_BaseGenAI.py +2 -2
  8. scitex/ai/_gen_ai/_DeepSeek.py +10 -2
  9. scitex/ai/_gen_ai/_Google.py +2 -2
  10. scitex/ai/_gen_ai/_Llama.py +2 -2
  11. scitex/ai/_gen_ai/_OpenAI.py +2 -2
  12. scitex/ai/_gen_ai/_PARAMS.py +51 -65
  13. scitex/ai/_gen_ai/_Perplexity.py +2 -2
  14. scitex/ai/_gen_ai/__init__.py +25 -14
  15. scitex/ai/_gen_ai/_format_output_func.py +4 -4
  16. scitex/ai/classification/{classifier_server.py → Classifier.py} +5 -5
  17. scitex/ai/classification/CrossValidationExperiment.py +374 -0
  18. scitex/ai/classification/__init__.py +43 -4
  19. scitex/ai/classification/reporters/_BaseClassificationReporter.py +281 -0
  20. scitex/ai/classification/reporters/_ClassificationReporter.py +773 -0
  21. scitex/ai/classification/reporters/_MultiClassificationReporter.py +406 -0
  22. scitex/ai/classification/reporters/_SingleClassificationReporter.py +1834 -0
  23. scitex/ai/classification/reporters/__init__.py +11 -0
  24. scitex/ai/classification/reporters/reporter_utils/_Plotter.py +1028 -0
  25. scitex/ai/classification/reporters/reporter_utils/__init__.py +80 -0
  26. scitex/ai/classification/reporters/reporter_utils/aggregation.py +457 -0
  27. scitex/ai/classification/reporters/reporter_utils/data_models.py +313 -0
  28. scitex/ai/classification/reporters/reporter_utils/reporting.py +1056 -0
  29. scitex/ai/classification/reporters/reporter_utils/storage.py +221 -0
  30. scitex/ai/classification/reporters/reporter_utils/validation.py +395 -0
  31. scitex/ai/classification/timeseries/_TimeSeriesBlockingSplit.py +568 -0
  32. scitex/ai/classification/timeseries/_TimeSeriesCalendarSplit.py +688 -0
  33. scitex/ai/classification/timeseries/_TimeSeriesMetadata.py +139 -0
  34. scitex/ai/classification/timeseries/_TimeSeriesSlidingWindowSplit.py +1716 -0
  35. scitex/ai/classification/timeseries/_TimeSeriesSlidingWindowSplit_v01-not-using-n_splits.py +1685 -0
  36. scitex/ai/classification/timeseries/_TimeSeriesStrategy.py +84 -0
  37. scitex/ai/classification/timeseries/_TimeSeriesStratifiedSplit.py +610 -0
  38. scitex/ai/classification/timeseries/__init__.py +39 -0
  39. scitex/ai/classification/timeseries/_normalize_timestamp.py +436 -0
  40. scitex/ai/clustering/_umap.py +2 -2
  41. scitex/ai/feature_extraction/vit.py +1 -0
  42. scitex/ai/feature_selection/__init__.py +30 -0
  43. scitex/ai/feature_selection/feature_selection.py +364 -0
  44. scitex/ai/loss/multi_task_loss.py +1 -1
  45. scitex/ai/metrics/__init__.py +51 -4
  46. scitex/ai/metrics/_calc_bacc.py +61 -0
  47. scitex/ai/metrics/_calc_bacc_from_conf_mat.py +38 -0
  48. scitex/ai/metrics/_calc_clf_report.py +78 -0
  49. scitex/ai/metrics/_calc_conf_mat.py +93 -0
  50. scitex/ai/metrics/_calc_feature_importance.py +183 -0
  51. scitex/ai/metrics/_calc_mcc.py +61 -0
  52. scitex/ai/metrics/_calc_pre_rec_auc.py +116 -0
  53. scitex/ai/metrics/_calc_roc_auc.py +110 -0
  54. scitex/ai/metrics/_calc_seizure_prediction_metrics.py +490 -0
  55. scitex/ai/metrics/{silhoute_score_block.py → _calc_silhouette_score.py} +15 -8
  56. scitex/ai/metrics/_normalize_labels.py +83 -0
  57. scitex/ai/plt/__init__.py +47 -8
  58. scitex/ai/plt/{_conf_mat.py → _plot_conf_mat.py} +158 -87
  59. scitex/ai/plt/_plot_feature_importance.py +323 -0
  60. scitex/ai/plt/_plot_learning_curve.py +345 -0
  61. scitex/ai/plt/_plot_optuna_study.py +225 -0
  62. scitex/ai/plt/_plot_pre_rec_curve.py +290 -0
  63. scitex/ai/plt/_plot_roc_curve.py +255 -0
  64. scitex/ai/training/{learning_curve_logger.py → _LearningCurveLogger.py} +197 -213
  65. scitex/ai/training/__init__.py +2 -2
  66. scitex/ai/utils/grid_search.py +3 -3
  67. scitex/benchmark/__init__.py +52 -0
  68. scitex/benchmark/benchmark.py +400 -0
  69. scitex/benchmark/monitor.py +370 -0
  70. scitex/benchmark/profiler.py +297 -0
  71. scitex/browser/__init__.py +48 -0
  72. scitex/browser/automation/CookieHandler.py +216 -0
  73. scitex/browser/automation/__init__.py +7 -0
  74. scitex/browser/collaboration/__init__.py +55 -0
  75. scitex/browser/collaboration/auth_helpers.py +94 -0
  76. scitex/browser/collaboration/collaborative_agent.py +136 -0
  77. scitex/browser/collaboration/credential_manager.py +188 -0
  78. scitex/browser/collaboration/interactive_panel.py +400 -0
  79. scitex/browser/collaboration/persistent_browser.py +170 -0
  80. scitex/browser/collaboration/shared_session.py +383 -0
  81. scitex/browser/collaboration/standard_interactions.py +246 -0
  82. scitex/browser/collaboration/visual_feedback.py +181 -0
  83. scitex/browser/core/BrowserMixin.py +326 -0
  84. scitex/browser/core/ChromeProfileManager.py +446 -0
  85. scitex/browser/core/__init__.py +9 -0
  86. scitex/browser/debugging/__init__.py +18 -0
  87. scitex/browser/debugging/_browser_logger.py +657 -0
  88. scitex/browser/debugging/_highlight_element.py +143 -0
  89. scitex/browser/debugging/_show_grid.py +154 -0
  90. scitex/browser/interaction/__init__.py +24 -0
  91. scitex/browser/interaction/click_center.py +149 -0
  92. scitex/browser/interaction/click_with_fallbacks.py +206 -0
  93. scitex/browser/interaction/close_popups.py +498 -0
  94. scitex/browser/interaction/fill_with_fallbacks.py +209 -0
  95. scitex/browser/pdf/__init__.py +14 -0
  96. scitex/browser/pdf/click_download_for_chrome_pdf_viewer.py +200 -0
  97. scitex/browser/pdf/detect_chrome_pdf_viewer.py +198 -0
  98. scitex/browser/remote/CaptchaHandler.py +434 -0
  99. scitex/browser/remote/ZenRowsAPIClient.py +347 -0
  100. scitex/browser/remote/ZenRowsBrowserManager.py +570 -0
  101. scitex/browser/remote/__init__.py +11 -0
  102. scitex/browser/stealth/HumanBehavior.py +344 -0
  103. scitex/browser/stealth/StealthManager.py +1008 -0
  104. scitex/browser/stealth/__init__.py +9 -0
  105. scitex/browser/template.py +122 -0
  106. scitex/capture/__init__.py +110 -0
  107. scitex/capture/__main__.py +25 -0
  108. scitex/capture/capture.py +848 -0
  109. scitex/capture/cli.py +233 -0
  110. scitex/capture/gif.py +344 -0
  111. scitex/capture/mcp_server.py +961 -0
  112. scitex/capture/session.py +70 -0
  113. scitex/capture/utils.py +705 -0
  114. scitex/cli/__init__.py +17 -0
  115. scitex/cli/cloud.py +447 -0
  116. scitex/cli/main.py +42 -0
  117. scitex/cli/scholar.py +280 -0
  118. scitex/context/_suppress_output.py +5 -3
  119. scitex/db/__init__.py +30 -3
  120. scitex/db/__main__.py +75 -0
  121. scitex/db/_check_health.py +381 -0
  122. scitex/db/_delete_duplicates.py +25 -386
  123. scitex/db/_inspect.py +335 -114
  124. scitex/db/_inspect_optimized.py +301 -0
  125. scitex/db/{_PostgreSQL.py → _postgresql/_PostgreSQL.py} +3 -3
  126. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_BackupMixin.py +1 -1
  127. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_BatchMixin.py +1 -1
  128. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_BlobMixin.py +1 -1
  129. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_ConnectionMixin.py +1 -1
  130. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_MaintenanceMixin.py +1 -1
  131. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_QueryMixin.py +1 -1
  132. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_SchemaMixin.py +1 -1
  133. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_TransactionMixin.py +1 -1
  134. scitex/db/_postgresql/__init__.py +6 -0
  135. scitex/db/_sqlite3/_SQLite3.py +210 -0
  136. scitex/db/_sqlite3/_SQLite3Mixins/_ArrayMixin.py +581 -0
  137. scitex/db/_sqlite3/_SQLite3Mixins/_ArrayMixin_v01-need-_hash-col.py +517 -0
  138. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_BatchMixin.py +1 -1
  139. scitex/db/_sqlite3/_SQLite3Mixins/_BlobMixin.py +281 -0
  140. scitex/db/_sqlite3/_SQLite3Mixins/_ColumnMixin.py +548 -0
  141. scitex/db/_sqlite3/_SQLite3Mixins/_ColumnMixin_v01-indentation-issues.py +583 -0
  142. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_ConnectionMixin.py +29 -13
  143. scitex/db/_sqlite3/_SQLite3Mixins/_GitMixin.py +583 -0
  144. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_ImportExportMixin.py +1 -1
  145. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_IndexMixin.py +1 -1
  146. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_MaintenanceMixin.py +2 -1
  147. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_QueryMixin.py +37 -10
  148. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_RowMixin.py +46 -6
  149. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_TableMixin.py +56 -10
  150. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_TransactionMixin.py +1 -1
  151. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/__init__.py +14 -2
  152. scitex/db/_sqlite3/__init__.py +7 -0
  153. scitex/db/_sqlite3/_delete_duplicates.py +274 -0
  154. scitex/decorators/__init__.py +2 -0
  155. scitex/decorators/_cache_disk.py +13 -5
  156. scitex/decorators/_cache_disk_async.py +49 -0
  157. scitex/decorators/_deprecated.py +175 -10
  158. scitex/decorators/_timeout.py +1 -1
  159. scitex/dev/_analyze_code_flow.py +2 -2
  160. scitex/dict/_DotDict.py +73 -15
  161. scitex/dict/_DotDict_v01-not-handling-recursive-instantiations.py +442 -0
  162. scitex/dict/_DotDict_v02-not-serializing-Path-object.py +446 -0
  163. scitex/dict/__init__.py +2 -0
  164. scitex/dict/_flatten.py +27 -0
  165. scitex/dsp/_crop.py +2 -2
  166. scitex/dsp/_demo_sig.py +2 -2
  167. scitex/dsp/_detect_ripples.py +2 -2
  168. scitex/dsp/_hilbert.py +2 -2
  169. scitex/dsp/_listen.py +6 -6
  170. scitex/dsp/_modulation_index.py +2 -2
  171. scitex/dsp/_pac.py +1 -1
  172. scitex/dsp/_psd.py +2 -2
  173. scitex/dsp/_resample.py +2 -1
  174. scitex/dsp/_time.py +3 -2
  175. scitex/dsp/_wavelet.py +3 -2
  176. scitex/dsp/add_noise.py +2 -2
  177. scitex/dsp/example.py +1 -0
  178. scitex/dsp/filt.py +10 -9
  179. scitex/dsp/template.py +3 -2
  180. scitex/dsp/utils/_differential_bandpass_filters.py +1 -1
  181. scitex/dsp/utils/pac.py +2 -2
  182. scitex/dt/_normalize_timestamp.py +432 -0
  183. scitex/errors.py +572 -0
  184. scitex/gen/_DimHandler.py +2 -2
  185. scitex/gen/__init__.py +37 -7
  186. scitex/gen/_deprecated_close.py +80 -0
  187. scitex/gen/_deprecated_start.py +26 -0
  188. scitex/gen/_detect_environment.py +152 -0
  189. scitex/gen/_detect_notebook_path.py +169 -0
  190. scitex/gen/_embed.py +6 -2
  191. scitex/gen/_get_notebook_path.py +257 -0
  192. scitex/gen/_less.py +1 -1
  193. scitex/gen/_list_packages.py +2 -2
  194. scitex/gen/_norm.py +44 -9
  195. scitex/gen/_norm_cache.py +269 -0
  196. scitex/gen/_src.py +3 -5
  197. scitex/gen/_title_case.py +3 -3
  198. scitex/io/__init__.py +28 -6
  199. scitex/io/_glob.py +13 -7
  200. scitex/io/_load.py +108 -21
  201. scitex/io/_load_cache.py +303 -0
  202. scitex/io/_load_configs.py +40 -15
  203. scitex/io/{_H5Explorer.py → _load_modules/_H5Explorer.py} +80 -17
  204. scitex/io/_load_modules/_ZarrExplorer.py +114 -0
  205. scitex/io/_load_modules/_bibtex.py +207 -0
  206. scitex/io/_load_modules/_hdf5.py +53 -178
  207. scitex/io/_load_modules/_json.py +5 -3
  208. scitex/io/_load_modules/_pdf.py +871 -16
  209. scitex/io/_load_modules/_sqlite3.py +15 -0
  210. scitex/io/_load_modules/_txt.py +41 -12
  211. scitex/io/_load_modules/_yaml.py +4 -3
  212. scitex/io/_load_modules/_zarr.py +126 -0
  213. scitex/io/_save.py +429 -171
  214. scitex/io/_save_modules/__init__.py +6 -0
  215. scitex/io/_save_modules/_bibtex.py +194 -0
  216. scitex/io/_save_modules/_csv.py +8 -4
  217. scitex/io/_save_modules/_excel.py +174 -15
  218. scitex/io/_save_modules/_hdf5.py +251 -226
  219. scitex/io/_save_modules/_image.py +1 -3
  220. scitex/io/_save_modules/_json.py +49 -4
  221. scitex/io/_save_modules/_listed_dfs_as_csv.py +1 -3
  222. scitex/io/_save_modules/_listed_scalars_as_csv.py +1 -3
  223. scitex/io/_save_modules/_tex.py +277 -0
  224. scitex/io/_save_modules/_yaml.py +42 -3
  225. scitex/io/_save_modules/_zarr.py +160 -0
  226. scitex/io/utils/__init__.py +20 -0
  227. scitex/io/utils/h5_to_zarr.py +616 -0
  228. scitex/linalg/_geometric_median.py +6 -2
  229. scitex/{gen/_tee.py → logging/_Tee.py} +43 -84
  230. scitex/logging/__init__.py +122 -0
  231. scitex/logging/_config.py +158 -0
  232. scitex/logging/_context.py +103 -0
  233. scitex/logging/_formatters.py +128 -0
  234. scitex/logging/_handlers.py +64 -0
  235. scitex/logging/_levels.py +35 -0
  236. scitex/logging/_logger.py +163 -0
  237. scitex/logging/_print_capture.py +95 -0
  238. scitex/ml/__init__.py +69 -0
  239. scitex/{ai/genai/anthropic.py → ml/_gen_ai/_Anthropic.py} +13 -19
  240. scitex/{ai/genai/base_genai.py → ml/_gen_ai/_BaseGenAI.py} +5 -5
  241. scitex/{ai/genai/deepseek.py → ml/_gen_ai/_DeepSeek.py} +11 -16
  242. scitex/{ai/genai/google.py → ml/_gen_ai/_Google.py} +7 -15
  243. scitex/{ai/genai/groq.py → ml/_gen_ai/_Groq.py} +1 -8
  244. scitex/{ai/genai/llama.py → ml/_gen_ai/_Llama.py} +3 -16
  245. scitex/{ai/genai/openai.py → ml/_gen_ai/_OpenAI.py} +3 -3
  246. scitex/{ai/genai/params.py → ml/_gen_ai/_PARAMS.py} +51 -65
  247. scitex/{ai/genai/perplexity.py → ml/_gen_ai/_Perplexity.py} +3 -14
  248. scitex/ml/_gen_ai/__init__.py +43 -0
  249. scitex/{ai/genai/calc_cost.py → ml/_gen_ai/_calc_cost.py} +1 -1
  250. scitex/{ai/genai/format_output_func.py → ml/_gen_ai/_format_output_func.py} +4 -4
  251. scitex/{ai/genai/genai_factory.py → ml/_gen_ai/_genai_factory.py} +8 -8
  252. scitex/ml/activation/__init__.py +8 -0
  253. scitex/ml/activation/_define.py +11 -0
  254. scitex/{ai/classifier_server.py → ml/classification/Classifier.py} +5 -5
  255. scitex/ml/classification/CrossValidationExperiment.py +374 -0
  256. scitex/ml/classification/__init__.py +46 -0
  257. scitex/ml/classification/reporters/_BaseClassificationReporter.py +281 -0
  258. scitex/ml/classification/reporters/_ClassificationReporter.py +773 -0
  259. scitex/ml/classification/reporters/_MultiClassificationReporter.py +406 -0
  260. scitex/ml/classification/reporters/_SingleClassificationReporter.py +1834 -0
  261. scitex/ml/classification/reporters/__init__.py +11 -0
  262. scitex/ml/classification/reporters/reporter_utils/_Plotter.py +1028 -0
  263. scitex/ml/classification/reporters/reporter_utils/__init__.py +80 -0
  264. scitex/ml/classification/reporters/reporter_utils/aggregation.py +457 -0
  265. scitex/ml/classification/reporters/reporter_utils/data_models.py +313 -0
  266. scitex/ml/classification/reporters/reporter_utils/reporting.py +1056 -0
  267. scitex/ml/classification/reporters/reporter_utils/storage.py +221 -0
  268. scitex/ml/classification/reporters/reporter_utils/validation.py +395 -0
  269. scitex/ml/classification/timeseries/_TimeSeriesBlockingSplit.py +568 -0
  270. scitex/ml/classification/timeseries/_TimeSeriesCalendarSplit.py +688 -0
  271. scitex/ml/classification/timeseries/_TimeSeriesMetadata.py +139 -0
  272. scitex/ml/classification/timeseries/_TimeSeriesSlidingWindowSplit.py +1716 -0
  273. scitex/ml/classification/timeseries/_TimeSeriesSlidingWindowSplit_v01-not-using-n_splits.py +1685 -0
  274. scitex/ml/classification/timeseries/_TimeSeriesStrategy.py +84 -0
  275. scitex/ml/classification/timeseries/_TimeSeriesStratifiedSplit.py +610 -0
  276. scitex/ml/classification/timeseries/__init__.py +39 -0
  277. scitex/ml/classification/timeseries/_normalize_timestamp.py +436 -0
  278. scitex/ml/clustering/__init__.py +11 -0
  279. scitex/ml/clustering/_pca.py +115 -0
  280. scitex/ml/clustering/_umap.py +376 -0
  281. scitex/ml/feature_extraction/__init__.py +56 -0
  282. scitex/ml/feature_extraction/vit.py +149 -0
  283. scitex/ml/feature_selection/__init__.py +30 -0
  284. scitex/ml/feature_selection/feature_selection.py +364 -0
  285. scitex/ml/loss/_L1L2Losses.py +34 -0
  286. scitex/ml/loss/__init__.py +12 -0
  287. scitex/ml/loss/multi_task_loss.py +47 -0
  288. scitex/ml/metrics/__init__.py +56 -0
  289. scitex/ml/metrics/_calc_bacc.py +61 -0
  290. scitex/ml/metrics/_calc_bacc_from_conf_mat.py +38 -0
  291. scitex/ml/metrics/_calc_clf_report.py +78 -0
  292. scitex/ml/metrics/_calc_conf_mat.py +93 -0
  293. scitex/ml/metrics/_calc_feature_importance.py +183 -0
  294. scitex/ml/metrics/_calc_mcc.py +61 -0
  295. scitex/ml/metrics/_calc_pre_rec_auc.py +116 -0
  296. scitex/ml/metrics/_calc_roc_auc.py +110 -0
  297. scitex/ml/metrics/_calc_seizure_prediction_metrics.py +490 -0
  298. scitex/ml/metrics/_calc_silhouette_score.py +503 -0
  299. scitex/ml/metrics/_normalize_labels.py +83 -0
  300. scitex/ml/optim/Ranger_Deep_Learning_Optimizer/__init__.py +0 -0
  301. scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/__init__.py +3 -0
  302. scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/ranger.py +207 -0
  303. scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/ranger2020.py +238 -0
  304. scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/ranger913A.py +215 -0
  305. scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/rangerqh.py +184 -0
  306. scitex/ml/optim/Ranger_Deep_Learning_Optimizer/setup.py +24 -0
  307. scitex/ml/optim/__init__.py +13 -0
  308. scitex/ml/optim/_get_set.py +31 -0
  309. scitex/ml/optim/_optimizers.py +71 -0
  310. scitex/ml/plt/__init__.py +60 -0
  311. scitex/ml/plt/_plot_conf_mat.py +663 -0
  312. scitex/ml/plt/_plot_feature_importance.py +323 -0
  313. scitex/ml/plt/_plot_learning_curve.py +345 -0
  314. scitex/ml/plt/_plot_optuna_study.py +225 -0
  315. scitex/ml/plt/_plot_pre_rec_curve.py +290 -0
  316. scitex/ml/plt/_plot_roc_curve.py +255 -0
  317. scitex/ml/sk/__init__.py +11 -0
  318. scitex/ml/sk/_clf.py +58 -0
  319. scitex/ml/sk/_to_sktime.py +100 -0
  320. scitex/ml/sklearn/__init__.py +26 -0
  321. scitex/ml/sklearn/clf.py +58 -0
  322. scitex/ml/sklearn/to_sktime.py +100 -0
  323. scitex/{ai/training/early_stopping.py → ml/training/_EarlyStopping.py} +1 -2
  324. scitex/{ai → ml/training}/_LearningCurveLogger.py +198 -242
  325. scitex/ml/training/__init__.py +7 -0
  326. scitex/ml/utils/__init__.py +22 -0
  327. scitex/ml/utils/_check_params.py +50 -0
  328. scitex/ml/utils/_default_dataset.py +46 -0
  329. scitex/ml/utils/_format_samples_for_sktime.py +26 -0
  330. scitex/ml/utils/_label_encoder.py +134 -0
  331. scitex/ml/utils/_merge_labels.py +22 -0
  332. scitex/ml/utils/_sliding_window_data_augmentation.py +11 -0
  333. scitex/ml/utils/_under_sample.py +51 -0
  334. scitex/ml/utils/_verify_n_gpus.py +16 -0
  335. scitex/ml/utils/grid_search.py +148 -0
  336. scitex/nn/_BNet.py +15 -9
  337. scitex/nn/_Filters.py +2 -2
  338. scitex/nn/_ModulationIndex.py +2 -2
  339. scitex/nn/_PAC.py +1 -1
  340. scitex/nn/_Spectrogram.py +12 -3
  341. scitex/nn/__init__.py +9 -10
  342. scitex/path/__init__.py +18 -0
  343. scitex/path/_clean.py +4 -0
  344. scitex/path/_find.py +9 -4
  345. scitex/path/_symlink.py +348 -0
  346. scitex/path/_version.py +4 -3
  347. scitex/pd/__init__.py +2 -0
  348. scitex/pd/_get_unique.py +99 -0
  349. scitex/plt/__init__.py +114 -5
  350. scitex/plt/_subplots/_AxesWrapper.py +1 -3
  351. scitex/plt/_subplots/_AxisWrapper.py +7 -3
  352. scitex/plt/_subplots/_AxisWrapperMixins/_AdjustmentMixin.py +47 -13
  353. scitex/plt/_subplots/_AxisWrapperMixins/_MatplotlibPlotMixin.py +160 -2
  354. scitex/plt/_subplots/_AxisWrapperMixins/_SeabornMixin.py +26 -4
  355. scitex/plt/_subplots/_AxisWrapperMixins/_UnitAwareMixin.py +322 -0
  356. scitex/plt/_subplots/_AxisWrapperMixins/__init__.py +1 -0
  357. scitex/plt/_subplots/_FigWrapper.py +62 -6
  358. scitex/plt/_subplots/_export_as_csv.py +43 -27
  359. scitex/plt/_subplots/_export_as_csv_formatters/__init__.py +5 -4
  360. scitex/plt/_subplots/_export_as_csv_formatters/_format_annotate.py +81 -0
  361. scitex/plt/_subplots/_export_as_csv_formatters/_format_bar.py +1 -3
  362. scitex/plt/_subplots/_export_as_csv_formatters/_format_barh.py +20 -5
  363. scitex/plt/_subplots/_export_as_csv_formatters/_format_boxplot.py +1 -3
  364. scitex/plt/_subplots/_export_as_csv_formatters/_format_contour.py +1 -3
  365. scitex/plt/_subplots/_export_as_csv_formatters/_format_errorbar.py +35 -18
  366. scitex/plt/_subplots/_export_as_csv_formatters/_format_eventplot.py +1 -3
  367. scitex/plt/_subplots/_export_as_csv_formatters/_format_fill.py +1 -3
  368. scitex/plt/_subplots/_export_as_csv_formatters/_format_fill_between.py +1 -3
  369. scitex/plt/_subplots/_export_as_csv_formatters/_format_hist.py +1 -3
  370. scitex/plt/_subplots/_export_as_csv_formatters/_format_imshow.py +1 -3
  371. scitex/plt/_subplots/_export_as_csv_formatters/_format_imshow2d.py +1 -3
  372. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot.py +15 -3
  373. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_box.py +1 -3
  374. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_conf_mat.py +1 -3
  375. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_ecdf.py +1 -3
  376. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_fillv.py +1 -3
  377. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_heatmap.py +1 -3
  378. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_image.py +1 -3
  379. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_joyplot.py +1 -3
  380. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_kde.py +1 -3
  381. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_line.py +1 -3
  382. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_mean_ci.py +1 -3
  383. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_mean_std.py +1 -3
  384. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_median_iqr.py +1 -3
  385. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_raster.py +1 -3
  386. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_rectangle.py +1 -3
  387. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_scatter.py +35 -0
  388. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_scatter_hist.py +1 -3
  389. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_shaded_line.py +1 -3
  390. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_violin.py +1 -3
  391. scitex/plt/_subplots/_export_as_csv_formatters/_format_scatter.py +6 -4
  392. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_barplot.py +1 -3
  393. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_boxplot.py +1 -3
  394. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_heatmap.py +1 -3
  395. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_histplot.py +1 -3
  396. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_jointplot.py +1 -3
  397. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_kdeplot.py +1 -3
  398. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_lineplot.py +1 -3
  399. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_pairplot.py +1 -3
  400. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_scatterplot.py +1 -3
  401. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_stripplot.py +1 -3
  402. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_swarmplot.py +1 -3
  403. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_violinplot.py +1 -3
  404. scitex/plt/_subplots/_export_as_csv_formatters/_format_text.py +60 -0
  405. scitex/plt/_subplots/_export_as_csv_formatters/_format_violin.py +1 -3
  406. scitex/plt/_subplots/_export_as_csv_formatters/_format_violinplot.py +1 -3
  407. scitex/plt/_subplots/_export_as_csv_formatters/test_formatters.py +1 -3
  408. scitex/plt/_subplots/_export_as_csv_formatters.py +56 -59
  409. scitex/plt/ax/_style/_hide_spines.py +1 -3
  410. scitex/plt/ax/_style/_rotate_labels.py +180 -76
  411. scitex/plt/ax/_style/_rotate_labels_v01.py +248 -0
  412. scitex/plt/ax/_style/_set_meta.py +11 -4
  413. scitex/plt/ax/_style/_set_supxyt.py +3 -3
  414. scitex/plt/ax/_style/_set_xyt.py +3 -3
  415. scitex/plt/ax/_style/_share_axes.py +2 -2
  416. scitex/plt/color/__init__.py +4 -4
  417. scitex/plt/color/{_get_colors_from_cmap.py → _get_colors_from_conf_matap.py} +7 -7
  418. scitex/plt/utils/_configure_mpl.py +99 -86
  419. scitex/plt/utils/_histogram_utils.py +1 -3
  420. scitex/plt/utils/_is_valid_axis.py +1 -3
  421. scitex/plt/utils/_scitex_config.py +1 -0
  422. scitex/repro/__init__.py +75 -0
  423. scitex/{reproduce → repro}/_gen_ID.py +1 -1
  424. scitex/{reproduce → repro}/_gen_timestamp.py +1 -1
  425. scitex/repro_rng/_RandomStateManager.py +590 -0
  426. scitex/repro_rng/_RandomStateManager_v01-no-verbose-options.py +414 -0
  427. scitex/repro_rng/__init__.py +39 -0
  428. scitex/reproduce/__init__.py +25 -13
  429. scitex/reproduce/_hash_array.py +22 -0
  430. scitex/resource/_get_processor_usages.py +4 -4
  431. scitex/resource/_get_specs.py +2 -2
  432. scitex/resource/_log_processor_usages.py +2 -2
  433. scitex/rng/_RandomStateManager.py +590 -0
  434. scitex/rng/_RandomStateManager_v01-no-verbose-options.py +414 -0
  435. scitex/rng/__init__.py +39 -0
  436. scitex/scholar/__init__.py +309 -19
  437. scitex/scholar/__main__.py +319 -0
  438. scitex/scholar/auth/ScholarAuthManager.py +308 -0
  439. scitex/scholar/auth/__init__.py +12 -0
  440. scitex/scholar/auth/core/AuthenticationGateway.py +473 -0
  441. scitex/scholar/auth/core/BrowserAuthenticator.py +386 -0
  442. scitex/scholar/auth/core/StrategyResolver.py +309 -0
  443. scitex/scholar/auth/core/__init__.py +16 -0
  444. scitex/scholar/auth/gateway/_OpenURLLinkFinder.py +120 -0
  445. scitex/scholar/auth/gateway/_OpenURLResolver.py +209 -0
  446. scitex/scholar/auth/gateway/__init__.py +38 -0
  447. scitex/scholar/auth/gateway/_resolve_functions.py +101 -0
  448. scitex/scholar/auth/providers/BaseAuthenticator.py +166 -0
  449. scitex/scholar/auth/providers/EZProxyAuthenticator.py +484 -0
  450. scitex/scholar/auth/providers/OpenAthensAuthenticator.py +619 -0
  451. scitex/scholar/auth/providers/ShibbolethAuthenticator.py +686 -0
  452. scitex/scholar/auth/providers/__init__.py +18 -0
  453. scitex/scholar/auth/session/AuthCacheManager.py +189 -0
  454. scitex/scholar/auth/session/SessionManager.py +159 -0
  455. scitex/scholar/auth/session/__init__.py +11 -0
  456. scitex/scholar/auth/sso/BaseSSOAutomator.py +373 -0
  457. scitex/scholar/auth/sso/OpenAthensSSOAutomator.py +378 -0
  458. scitex/scholar/auth/sso/SSOAutomator.py +180 -0
  459. scitex/scholar/auth/sso/UniversityOfMelbourneSSOAutomator.py +380 -0
  460. scitex/scholar/auth/sso/__init__.py +15 -0
  461. scitex/scholar/browser/ScholarBrowserManager.py +705 -0
  462. scitex/scholar/browser/__init__.py +38 -0
  463. scitex/scholar/browser/utils/__init__.py +13 -0
  464. scitex/scholar/browser/utils/click_and_wait.py +205 -0
  465. scitex/scholar/browser/utils/close_unwanted_pages.py +140 -0
  466. scitex/scholar/browser/utils/wait_redirects.py +732 -0
  467. scitex/scholar/config/PublisherRules.py +132 -0
  468. scitex/scholar/config/ScholarConfig.py +126 -0
  469. scitex/scholar/config/__init__.py +17 -0
  470. scitex/scholar/core/Paper.py +627 -0
  471. scitex/scholar/core/Papers.py +722 -0
  472. scitex/scholar/core/Scholar.py +1975 -0
  473. scitex/scholar/core/__init__.py +9 -0
  474. scitex/scholar/impact_factor/ImpactFactorEngine.py +204 -0
  475. scitex/scholar/impact_factor/__init__.py +20 -0
  476. scitex/scholar/impact_factor/estimation/ImpactFactorEstimationEngine.py +0 -0
  477. scitex/scholar/impact_factor/estimation/__init__.py +40 -0
  478. scitex/scholar/impact_factor/estimation/build_database.py +0 -0
  479. scitex/scholar/impact_factor/estimation/core/__init__.py +28 -0
  480. scitex/scholar/impact_factor/estimation/core/cache_manager.py +523 -0
  481. scitex/scholar/impact_factor/estimation/core/calculator.py +355 -0
  482. scitex/scholar/impact_factor/estimation/core/journal_matcher.py +428 -0
  483. scitex/scholar/integration/__init__.py +59 -0
  484. scitex/scholar/integration/base.py +502 -0
  485. scitex/scholar/integration/mendeley/__init__.py +22 -0
  486. scitex/scholar/integration/mendeley/exporter.py +166 -0
  487. scitex/scholar/integration/mendeley/importer.py +236 -0
  488. scitex/scholar/integration/mendeley/linker.py +79 -0
  489. scitex/scholar/integration/mendeley/mapper.py +212 -0
  490. scitex/scholar/integration/zotero/__init__.py +27 -0
  491. scitex/scholar/integration/zotero/__main__.py +264 -0
  492. scitex/scholar/integration/zotero/exporter.py +351 -0
  493. scitex/scholar/integration/zotero/importer.py +372 -0
  494. scitex/scholar/integration/zotero/linker.py +415 -0
  495. scitex/scholar/integration/zotero/mapper.py +286 -0
  496. scitex/scholar/metadata_engines/ScholarEngine.py +588 -0
  497. scitex/scholar/metadata_engines/__init__.py +21 -0
  498. scitex/scholar/metadata_engines/individual/ArXivEngine.py +397 -0
  499. scitex/scholar/metadata_engines/individual/CrossRefEngine.py +274 -0
  500. scitex/scholar/metadata_engines/individual/CrossRefLocalEngine.py +263 -0
  501. scitex/scholar/metadata_engines/individual/OpenAlexEngine.py +350 -0
  502. scitex/scholar/metadata_engines/individual/PubMedEngine.py +329 -0
  503. scitex/scholar/metadata_engines/individual/SemanticScholarEngine.py +438 -0
  504. scitex/scholar/metadata_engines/individual/URLDOIEngine.py +410 -0
  505. scitex/scholar/metadata_engines/individual/_BaseDOIEngine.py +487 -0
  506. scitex/scholar/metadata_engines/individual/__init__.py +7 -0
  507. scitex/scholar/metadata_engines/utils/_PubMedConverter.py +469 -0
  508. scitex/scholar/metadata_engines/utils/_URLDOIExtractor.py +283 -0
  509. scitex/scholar/metadata_engines/utils/__init__.py +30 -0
  510. scitex/scholar/metadata_engines/utils/_metadata2bibtex.py +103 -0
  511. scitex/scholar/metadata_engines/utils/_standardize_metadata.py +376 -0
  512. scitex/scholar/pdf_download/ScholarPDFDownloader.py +579 -0
  513. scitex/scholar/pdf_download/__init__.py +5 -0
  514. scitex/scholar/pdf_download/strategies/__init__.py +38 -0
  515. scitex/scholar/pdf_download/strategies/chrome_pdf_viewer.py +376 -0
  516. scitex/scholar/pdf_download/strategies/direct_download.py +131 -0
  517. scitex/scholar/pdf_download/strategies/manual_download_fallback.py +167 -0
  518. scitex/scholar/pdf_download/strategies/manual_download_utils.py +996 -0
  519. scitex/scholar/pdf_download/strategies/response_body.py +207 -0
  520. scitex/scholar/pipelines/ScholarPipelineBibTeX.py +364 -0
  521. scitex/scholar/pipelines/ScholarPipelineParallel.py +478 -0
  522. scitex/scholar/pipelines/ScholarPipelineSingle.py +767 -0
  523. scitex/scholar/pipelines/__init__.py +49 -0
  524. scitex/scholar/storage/BibTeXHandler.py +1018 -0
  525. scitex/scholar/storage/PaperIO.py +468 -0
  526. scitex/scholar/storage/ScholarLibrary.py +182 -0
  527. scitex/scholar/storage/_DeduplicationManager.py +548 -0
  528. scitex/scholar/storage/_LibraryCacheManager.py +724 -0
  529. scitex/scholar/storage/_LibraryManager.py +1835 -0
  530. scitex/scholar/storage/__init__.py +28 -0
  531. scitex/scholar/url_finder/ScholarURLFinder.py +379 -0
  532. scitex/scholar/url_finder/__init__.py +7 -0
  533. scitex/scholar/url_finder/strategies/__init__.py +33 -0
  534. scitex/scholar/url_finder/strategies/find_pdf_urls_by_direct_links.py +261 -0
  535. scitex/scholar/url_finder/strategies/find_pdf_urls_by_dropdown.py +67 -0
  536. scitex/scholar/url_finder/strategies/find_pdf_urls_by_href.py +204 -0
  537. scitex/scholar/url_finder/strategies/find_pdf_urls_by_navigation.py +256 -0
  538. scitex/scholar/url_finder/strategies/find_pdf_urls_by_publisher_patterns.py +165 -0
  539. scitex/scholar/url_finder/strategies/find_pdf_urls_by_zotero_translators.py +163 -0
  540. scitex/scholar/url_finder/strategies/find_supplementary_urls_by_href.py +70 -0
  541. scitex/scholar/utils/__init__.py +22 -0
  542. scitex/scholar/utils/bibtex/__init__.py +9 -0
  543. scitex/scholar/utils/bibtex/_parse_bibtex.py +71 -0
  544. scitex/scholar/utils/cleanup/__init__.py +8 -0
  545. scitex/scholar/utils/cleanup/_cleanup_scholar_processes.py +96 -0
  546. scitex/scholar/utils/cleanup/cleanup_old_extractions.py +117 -0
  547. scitex/scholar/utils/text/_TextNormalizer.py +407 -0
  548. scitex/scholar/utils/text/__init__.py +9 -0
  549. scitex/scholar/zotero/__init__.py +38 -0
  550. scitex/session/__init__.py +51 -0
  551. scitex/session/_lifecycle.py +736 -0
  552. scitex/session/_manager.py +102 -0
  553. scitex/session/template.py +122 -0
  554. scitex/stats/__init__.py +30 -26
  555. scitex/stats/correct/__init__.py +21 -0
  556. scitex/stats/correct/_correct_bonferroni.py +551 -0
  557. scitex/stats/correct/_correct_fdr.py +634 -0
  558. scitex/stats/correct/_correct_holm.py +548 -0
  559. scitex/stats/correct/_correct_sidak.py +499 -0
  560. scitex/stats/descriptive/__init__.py +85 -0
  561. scitex/stats/descriptive/_circular.py +540 -0
  562. scitex/stats/descriptive/_describe.py +219 -0
  563. scitex/stats/descriptive/_nan.py +518 -0
  564. scitex/stats/descriptive/_real.py +189 -0
  565. scitex/stats/effect_sizes/__init__.py +41 -0
  566. scitex/stats/effect_sizes/_cliffs_delta.py +325 -0
  567. scitex/stats/effect_sizes/_cohens_d.py +342 -0
  568. scitex/stats/effect_sizes/_epsilon_squared.py +315 -0
  569. scitex/stats/effect_sizes/_eta_squared.py +302 -0
  570. scitex/stats/effect_sizes/_prob_superiority.py +296 -0
  571. scitex/stats/posthoc/__init__.py +19 -0
  572. scitex/stats/posthoc/_dunnett.py +463 -0
  573. scitex/stats/posthoc/_games_howell.py +383 -0
  574. scitex/stats/posthoc/_tukey_hsd.py +367 -0
  575. scitex/stats/power/__init__.py +19 -0
  576. scitex/stats/power/_power.py +433 -0
  577. scitex/stats/template.py +119 -0
  578. scitex/stats/utils/__init__.py +62 -0
  579. scitex/stats/utils/_effect_size.py +985 -0
  580. scitex/stats/utils/_formatters.py +270 -0
  581. scitex/stats/utils/_normalizers.py +927 -0
  582. scitex/stats/utils/_power.py +433 -0
  583. scitex/stats_v01/_EffectSizeCalculator.py +488 -0
  584. scitex/stats_v01/_StatisticalValidator.py +411 -0
  585. scitex/stats_v01/__init__.py +60 -0
  586. scitex/stats_v01/_additional_tests.py +415 -0
  587. scitex/{stats → stats_v01}/_p2stars.py +19 -5
  588. scitex/stats_v01/_two_sample_tests.py +141 -0
  589. scitex/stats_v01/desc/__init__.py +83 -0
  590. scitex/stats_v01/desc/_circular.py +540 -0
  591. scitex/stats_v01/desc/_describe.py +219 -0
  592. scitex/stats_v01/desc/_nan.py +518 -0
  593. scitex/{stats/desc/_nan.py → stats_v01/desc/_nan_v01-20250920_145731.py} +23 -12
  594. scitex/stats_v01/desc/_real.py +189 -0
  595. scitex/stats_v01/tests/__corr_test_optimized.py +221 -0
  596. scitex/stats_v01/tests/_corr_test_optimized.py +179 -0
  597. scitex/str/__init__.py +1 -3
  598. scitex/str/_clean_path.py +6 -2
  599. scitex/str/_latex_fallback.py +267 -160
  600. scitex/str/_parse.py +44 -36
  601. scitex/str/_printc.py +1 -3
  602. scitex/template/__init__.py +87 -0
  603. scitex/template/_create_project.py +267 -0
  604. scitex/template/create_pip_project.py +80 -0
  605. scitex/template/create_research.py +80 -0
  606. scitex/template/create_singularity.py +80 -0
  607. scitex/units.py +291 -0
  608. scitex/utils/_compress_hdf5.py +14 -3
  609. scitex/utils/_email.py +21 -2
  610. scitex/utils/_grid.py +6 -4
  611. scitex/utils/_notify.py +13 -10
  612. scitex/utils/_verify_scitex_format.py +589 -0
  613. scitex/utils/_verify_scitex_format_v01.py +370 -0
  614. scitex/utils/template.py +122 -0
  615. scitex/web/_search_pubmed.py +62 -16
  616. scitex-2.1.0.dist-info/LICENSE +21 -0
  617. scitex-2.1.0.dist-info/METADATA +677 -0
  618. scitex-2.1.0.dist-info/RECORD +919 -0
  619. {scitex-2.0.0.dist-info → scitex-2.1.0.dist-info}/WHEEL +1 -1
  620. scitex-2.1.0.dist-info/entry_points.txt +3 -0
  621. scitex/ai/__Classifiers.py +0 -101
  622. scitex/ai/classification/classification_reporter.py +0 -1137
  623. scitex/ai/classification/classifiers.py +0 -101
  624. scitex/ai/classification_reporter.py +0 -1161
  625. scitex/ai/genai/__init__.py +0 -277
  626. scitex/ai/genai/anthropic_provider.py +0 -320
  627. scitex/ai/genai/anthropic_refactored.py +0 -109
  628. scitex/ai/genai/auth_manager.py +0 -200
  629. scitex/ai/genai/base_provider.py +0 -291
  630. scitex/ai/genai/chat_history.py +0 -307
  631. scitex/ai/genai/cost_tracker.py +0 -276
  632. scitex/ai/genai/deepseek_provider.py +0 -251
  633. scitex/ai/genai/google_provider.py +0 -228
  634. scitex/ai/genai/groq_provider.py +0 -248
  635. scitex/ai/genai/image_processor.py +0 -250
  636. scitex/ai/genai/llama_provider.py +0 -214
  637. scitex/ai/genai/mock_provider.py +0 -127
  638. scitex/ai/genai/model_registry.py +0 -304
  639. scitex/ai/genai/openai_provider.py +0 -293
  640. scitex/ai/genai/perplexity_provider.py +0 -205
  641. scitex/ai/genai/provider_base.py +0 -302
  642. scitex/ai/genai/provider_factory.py +0 -370
  643. scitex/ai/genai/response_handler.py +0 -235
  644. scitex/ai/layer/_Pass.py +0 -21
  645. scitex/ai/layer/__init__.py +0 -10
  646. scitex/ai/layer/_switch.py +0 -8
  647. scitex/ai/metrics/_bACC.py +0 -51
  648. scitex/ai/plt/_learning_curve.py +0 -194
  649. scitex/ai/plt/_optuna_study.py +0 -111
  650. scitex/ai/plt/aucs/__init__.py +0 -2
  651. scitex/ai/plt/aucs/example.py +0 -60
  652. scitex/ai/plt/aucs/pre_rec_auc.py +0 -223
  653. scitex/ai/plt/aucs/roc_auc.py +0 -246
  654. scitex/ai/sampling/undersample.py +0 -29
  655. scitex/db/_SQLite3.py +0 -2136
  656. scitex/db/_SQLite3Mixins/_BlobMixin.py +0 -229
  657. scitex/gen/_close.py +0 -222
  658. scitex/gen/_start.py +0 -451
  659. scitex/general/__init__.py +0 -5
  660. scitex/io/_load_modules/_db.py +0 -24
  661. scitex/life/__init__.py +0 -10
  662. scitex/life/_monitor_rain.py +0 -49
  663. scitex/reproduce/_fix_seeds.py +0 -45
  664. scitex/res/__init__.py +0 -5
  665. scitex/scholar/_local_search.py +0 -454
  666. scitex/scholar/_paper.py +0 -244
  667. scitex/scholar/_pdf_downloader.py +0 -325
  668. scitex/scholar/_search.py +0 -393
  669. scitex/scholar/_vector_search.py +0 -370
  670. scitex/scholar/_web_sources.py +0 -457
  671. scitex/stats/desc/__init__.py +0 -40
  672. scitex-2.0.0.dist-info/METADATA +0 -307
  673. scitex-2.0.0.dist-info/RECORD +0 -572
  674. scitex-2.0.0.dist-info/licenses/LICENSE +0 -7
  675. /scitex/ai/{act → activation}/__init__.py +0 -0
  676. /scitex/ai/{act → activation}/_define.py +0 -0
  677. /scitex/ai/{early_stopping.py → training/_EarlyStopping.py} +0 -0
  678. /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_ImportExportMixin.py +0 -0
  679. /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_IndexMixin.py +0 -0
  680. /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_RowMixin.py +0 -0
  681. /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_TableMixin.py +0 -0
  682. /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/__init__.py +0 -0
  683. /scitex/{stats → stats_v01}/_calc_partial_corr.py +0 -0
  684. /scitex/{stats → stats_v01}/_corr_test_multi.py +0 -0
  685. /scitex/{stats → stats_v01}/_corr_test_wrapper.py +0 -0
  686. /scitex/{stats → stats_v01}/_describe_wrapper.py +0 -0
  687. /scitex/{stats → stats_v01}/_multiple_corrections.py +0 -0
  688. /scitex/{stats → stats_v01}/_nan_stats.py +0 -0
  689. /scitex/{stats → stats_v01}/_p2stars_wrapper.py +0 -0
  690. /scitex/{stats → stats_v01}/_statistical_tests.py +0 -0
  691. /scitex/{stats/desc/_describe.py → stats_v01/desc/_describe_v01-20250920_145731.py} +0 -0
  692. /scitex/{stats/desc/_real.py → stats_v01/desc/_real_v01-20250920_145731.py} +0 -0
  693. /scitex/{stats → stats_v01}/multiple/__init__.py +0 -0
  694. /scitex/{stats → stats_v01}/multiple/_bonferroni_correction.py +0 -0
  695. /scitex/{stats → stats_v01}/multiple/_fdr_correction.py +0 -0
  696. /scitex/{stats → stats_v01}/multiple/_multicompair.py +0 -0
  697. /scitex/{stats → stats_v01}/tests/__corr_test.py +0 -0
  698. /scitex/{stats → stats_v01}/tests/__corr_test_multi.py +0 -0
  699. /scitex/{stats → stats_v01}/tests/__corr_test_single.py +0 -0
  700. /scitex/{stats → stats_v01}/tests/__init__.py +0 -0
  701. /scitex/{stats → stats_v01}/tests/_brunner_munzel_test.py +0 -0
  702. /scitex/{stats → stats_v01}/tests/_nocorrelation_test.py +0 -0
  703. /scitex/{stats → stats_v01}/tests/_smirnov_grubbs.py +0 -0
  704. {scitex-2.0.0.dist-info → scitex-2.1.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,996 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ # Timestamp: "2025-10-13 08:03:29 (ywatanabe)"
4
+ # File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/pdf_download/strategies/manual_download_utils.py
5
+ # ----------------------------------------
6
+ from __future__ import annotations
7
+ import os
8
+ __FILE__ = (
9
+ "./src/scitex/scholar/pdf_download/strategies/manual_download_utils.py"
10
+ )
11
+ __DIR__ = os.path.dirname(__FILE__)
12
+ # ----------------------------------------
13
+
14
+ """Manual Download Utilities
15
+
16
+ This module provides shared utilities for manual download workflows:
17
+ 1. FlexibleFilenameGenerator - DOI-based filename generation
18
+ 2. DownloadMonitorAndSync - Monitor downloads directory and sync to library
19
+ 3. UI button functions - Show buttons in browser for manual interaction
20
+ """
21
+
22
+ import asyncio
23
+ import re
24
+ from datetime import datetime
25
+ from pathlib import Path
26
+ from typing import Optional
27
+
28
+ from playwright.async_api import Page
29
+
30
+ from scitex.browser.debugging import browser_logger
31
+
32
+
33
+ class FlexibleFilenameGenerator:
34
+ """Generate flexible filenames for PDFs with DOI-based naming."""
35
+
36
+ @property
37
+ def name(
38
+ self,
39
+ ):
40
+ return self.__class__.__name__
41
+
42
+ @staticmethod
43
+ def sanitize_doi(doi: str) -> str:
44
+ """Convert DOI to filesystem-safe format."""
45
+ # Replace / with _ and remove other problematic characters
46
+ safe = doi.replace("/", "_").replace("\\", "_")
47
+ safe = re.sub(r'[<>:"|?*]', "", safe)
48
+ return safe
49
+
50
+ @staticmethod
51
+ def generate_filename(
52
+ doi: Optional[str] = None,
53
+ url: Optional[str] = None,
54
+ content_type: str = "main",
55
+ sequence_index: Optional[int] = None,
56
+ add_timestamp: bool = False,
57
+ ) -> str:
58
+ """
59
+ Generate flexible filename for PDF.
60
+
61
+ Args:
62
+ doi: DOI of the article (preferred identifier)
63
+ url: URL if DOI not available
64
+ content_type: Type of content ("main", "supp", "figures", etc.)
65
+ sequence_index: Index for supplements (1, 2, 3, ...)
66
+ add_timestamp: Whether to add timestamp to avoid collisions
67
+
68
+ Returns:
69
+ Filename like: 10_1016_S1474-4422_13_70075-9_main.pdf
70
+ 10_1016_S1474-4422_13_70075-9_supp_01.pdf
71
+ 10_1016_S1474-4422_13_70075-9_supp_02_20251010_082215.pdf
72
+ """
73
+ # Generate base identifier
74
+ if doi:
75
+ base = FlexibleFilenameGenerator.sanitize_doi(doi)
76
+ elif url:
77
+ # Use domain and path as fallback
78
+ from urllib.parse import urlparse
79
+
80
+ parsed = urlparse(url)
81
+ base = f"{parsed.netloc}_{parsed.path}".replace("/", "_").replace(
82
+ ".", "_"
83
+ )
84
+ base = re.sub(r'[<>:"|?*]', "", base)
85
+ else:
86
+ # Last resort: timestamp-based
87
+ base = f"article_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
88
+
89
+ # Build filename parts
90
+ parts = [base]
91
+
92
+ # Add content type if not main
93
+ if content_type != "main":
94
+ parts.append(content_type)
95
+
96
+ # Add sequence index for supplements
97
+ if sequence_index is not None:
98
+ parts.append(f"{sequence_index:02d}")
99
+
100
+ # Add timestamp if requested
101
+ if add_timestamp:
102
+ parts.append(datetime.now().strftime("%Y%m%d_%H%M%S"))
103
+
104
+ # Join parts and add extension
105
+ filename = "_".join(parts) + ".pdf"
106
+ return filename
107
+
108
+
109
+ class DownloadMonitorAndSync:
110
+ """Monitor temp downloads directory and sync files to final destination."""
111
+
112
+ def __init__(self, temp_downloads_dir: Path, final_pdfs_dir: Path):
113
+ """
114
+ Initialize monitor.
115
+
116
+ Args:
117
+ temp_downloads_dir: Temporary browser downloads directory (library/downloads/)
118
+ final_pdfs_dir: Final organized PDFs directory (library/pdfs/)
119
+ """
120
+ self.temp_dir = Path(temp_downloads_dir)
121
+ self.final_dir = Path(final_pdfs_dir)
122
+ self.baseline_files = self._get_current_files()
123
+
124
+ # Ensure directories exist
125
+ self.temp_dir.mkdir(parents=True, exist_ok=True)
126
+ self.final_dir.mkdir(parents=True, exist_ok=True)
127
+
128
+ def _get_current_files(self) -> set:
129
+ """Get set of current files in temp directory."""
130
+ if not self.temp_dir.exists():
131
+ return set()
132
+ return {f.name for f in self.temp_dir.iterdir() if f.is_file()}
133
+
134
+ def _is_pdf_file(self, file_path: Path) -> bool:
135
+ """Check if file is a valid PDF by magic number."""
136
+ if not file_path.exists() or file_path.stat().st_size == 0:
137
+ return False
138
+
139
+ try:
140
+ with open(file_path, "rb") as f:
141
+ header = f.read(5)
142
+ return header == b"%PDF-"
143
+ except Exception:
144
+ return False
145
+
146
+ def _is_file_stable(self, file_path: Path, wait_ms: int = 300) -> bool:
147
+ """Check if file has finished downloading (size unchanged)."""
148
+ import time
149
+
150
+ if not file_path.exists():
151
+ return False
152
+
153
+ size1 = file_path.stat().st_size
154
+ time.sleep(wait_ms / 1000)
155
+ size2 = file_path.stat().st_size
156
+
157
+ return size1 == size2 and size1 > 0
158
+
159
+ async def monitor_for_new_download_async(
160
+ self,
161
+ timeout_sec: float = 120,
162
+ check_interval_sec: float = 1.0,
163
+ logger_func=None,
164
+ ) -> Optional[Path]:
165
+ """
166
+ Monitor temp directory for new PDF files.
167
+
168
+ Args:
169
+ timeout_sec: Maximum time to wait for download
170
+ check_interval_sec: How often to check for new files
171
+ logger_func: Optional logging function to report progress
172
+
173
+ Returns:
174
+ Path to new PDF file, or None if timeout
175
+ """
176
+ start_time = asyncio.get_event_loop().time()
177
+ last_progress_time = start_time
178
+ progress_interval = 10.0 # Report progress every 10 seconds
179
+
180
+ if logger_func:
181
+ logger_func(
182
+ f"{self.name}: Monitoring {self.temp_dir} for new downloads (timeout: {timeout_sec}s)"
183
+ )
184
+
185
+ while True:
186
+ elapsed = asyncio.get_event_loop().time() - start_time
187
+ remaining = timeout_sec - elapsed
188
+
189
+ if elapsed > timeout_sec:
190
+ if logger_func:
191
+ logger_func(
192
+ f"{self.name}: Download monitoring timeout - no new PDF detected"
193
+ )
194
+ return None
195
+
196
+ # Report progress periodically
197
+ if (
198
+ logger_func
199
+ and (elapsed - last_progress_time) >= progress_interval
200
+ ):
201
+ current_file_count = len(self._get_current_files())
202
+ logger_func(
203
+ f"{self.name}: Still waiting for download... ({remaining:.0f}s remaining, "
204
+ f"{self.name}: {current_file_count} files in directory)"
205
+ )
206
+ last_progress_time = elapsed
207
+
208
+ # Get current files
209
+ current_files = self._get_current_files()
210
+ new_files = current_files - self.baseline_files
211
+
212
+ # Check each new file
213
+ for filename in new_files:
214
+ file_path = self.temp_dir / filename
215
+
216
+ # Log detection
217
+ if logger_func:
218
+ logger_func(
219
+ f"{self.name}: Detected new file: {filename}, checking if complete..."
220
+ )
221
+
222
+ # Check if it's a stable file first
223
+ if not self._is_file_stable(file_path):
224
+ if logger_func:
225
+ logger_func(
226
+ f"{self.name}: File still downloading, waiting..."
227
+ )
228
+ continue
229
+
230
+ # Check if it's a valid PDF (by magic number, not extension)
231
+ if self._is_pdf_file(file_path):
232
+ if logger_func:
233
+ size_mb = file_path.stat().st_size / 1e6
234
+ logger_func(
235
+ f"{self.name}: Found valid PDF: {filename} ({size_mb:.2f} MB)"
236
+ )
237
+ return file_path
238
+ else:
239
+ if logger_func:
240
+ logger_func(
241
+ f"{self.name}: File is not a PDF, skipping: {filename}"
242
+ )
243
+
244
+ # Wait before next check
245
+ await asyncio.sleep(check_interval_sec)
246
+
247
+ def sync_to_final_destination(
248
+ self,
249
+ temp_file: Path,
250
+ doi: Optional[str] = None,
251
+ url: Optional[str] = None,
252
+ content_type: str = "main",
253
+ sequence_index: Optional[int] = None,
254
+ ) -> Path:
255
+ """
256
+ Move file from temp to final destination with proper naming.
257
+
258
+ Args:
259
+ temp_file: Path to temporary downloaded file
260
+ doi: DOI for filename generation
261
+ url: URL fallback for filename generation
262
+ content_type: Type of content ("main", "supp", etc.)
263
+ sequence_index: Index for supplements
264
+
265
+ Returns:
266
+ Path to final destination file
267
+ """
268
+ # Generate proper filename
269
+ filename = FlexibleFilenameGenerator.generate_filename(
270
+ doi=doi,
271
+ url=url,
272
+ content_type=content_type,
273
+ sequence_index=sequence_index,
274
+ add_timestamp=False,
275
+ )
276
+
277
+ final_path = self.final_dir / filename
278
+
279
+ # Handle collision by adding timestamp
280
+ if final_path.exists():
281
+ filename = FlexibleFilenameGenerator.generate_filename(
282
+ doi=doi,
283
+ url=url,
284
+ content_type=content_type,
285
+ sequence_index=sequence_index,
286
+ add_timestamp=True,
287
+ )
288
+ final_path = self.final_dir / filename
289
+
290
+ # Move file
291
+ import shutil
292
+
293
+ shutil.move(str(temp_file), str(final_path))
294
+
295
+ return final_path
296
+
297
+
298
+ def get_manual_button_init_script(target_filename: str) -> str:
299
+ """Get JavaScript init script that injects manual mode button on ALL pages.
300
+
301
+ This script is added to browser context via add_init_script, so it runs
302
+ on EVERY page load, including redirects and new tabs.
303
+
304
+ Args:
305
+ target_filename: Target filename to display
306
+
307
+ Returns:
308
+ JavaScript code to inject the button
309
+ """
310
+ return f"""
311
+ (() => {{
312
+ // Wait for DOM to be ready
313
+ if (document.readyState === 'loading') {{
314
+ document.addEventListener('DOMContentLoaded', injectManualButton);
315
+ }} else {{
316
+ injectManualButton();
317
+ }}
318
+
319
+ function injectManualButton() {{
320
+ // Remove any existing button
321
+ document.getElementById('scitex-manual-button')?.remove();
322
+
323
+ // Create button
324
+ const button = document.createElement('button');
325
+ button.id = 'scitex-manual-button';
326
+ button.setAttribute('data-scitex-no-auto-click', 'true');
327
+ button.style.cssText = `
328
+ position: fixed !important;
329
+ top: 50% !important;
330
+ left: 20px !important;
331
+ transform: translateY(-50%) !important;
332
+ z-index: 2147483647 !important;
333
+ background: linear-gradient(135deg, #506b7a 0%, #6c8ba0 30%, #8fa4b0 60%, #b5c7d1 100%) !important;
334
+ color: #1a2332 !important;
335
+ padding: 24px 36px !important;
336
+ border: 3px solid #34495e !important;
337
+ border-radius: 8px !important;
338
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif !important;
339
+ font-size: 17px !important;
340
+ font-weight: 700 !important;
341
+ cursor: pointer !important;
342
+ box-shadow: 0 8px 24px rgba(26, 35, 50, 0.4) !important;
343
+ display: block !important;
344
+ visibility: visible !important;
345
+ text-align: center !important;
346
+ line-height: 1.5 !important;
347
+ min-width: 220px !important;
348
+ text-shadow: 0 1px 2px rgba(255,255,255,0.5) !important;
349
+ `;
350
+
351
+ button.innerHTML = 'PRESS \\'M\\'<br>FOR MANUAL';
352
+
353
+ if (document.documentElement) {{
354
+ document.documentElement.appendChild(button);
355
+ }} else if (document.body) {{
356
+ document.body.appendChild(button);
357
+ }}
358
+
359
+ // Keyboard handler - Press 'M' key
360
+ document.addEventListener('keydown', (e) => {{
361
+ if ((e.key === 'm' || e.key === 'M') && !e.ctrlKey && !e.altKey && !e.metaKey) {{
362
+ if (button.getAttribute('data-scitex-clicked') !== 'true') {{
363
+ console.log('SciTeX: Key M pressed - Manual mode activated!');
364
+ button.setAttribute('data-scitex-clicked', 'true');
365
+ button.innerHTML = 'MANUAL MODE<br>ACTIVATED';
366
+ button.style.background = 'linear-gradient(135deg, #1a2332 0%, #2d3748 50%, #34495e 100%)';
367
+ button.style.border = '3px solid #8fa4b0';
368
+ window._scitexManualModeActivated = true;
369
+ }}
370
+ }}
371
+ }});
372
+
373
+ // Click shows reminder
374
+ button.addEventListener('click', () => {{
375
+ if (button.getAttribute('data-scitex-clicked') !== 'true') {{
376
+ button.innerHTML = 'PRESS M KEY!';
377
+ button.style.background = 'linear-gradient(135deg, #6c8ba0 0%, #8fa4b0 100%)';
378
+ setTimeout(() => {{
379
+ if (button.getAttribute('data-scitex-clicked') !== 'true') {{
380
+ button.innerHTML = 'PRESS \\'M\\'<br>FOR MANUAL';
381
+ button.style.background = 'linear-gradient(135deg, #506b7a 0%, #6c8ba0 30%, #8fa4b0 60%, #b5c7d1 100%)';
382
+ }}
383
+ }}, 500);
384
+ }}
385
+ }});
386
+
387
+ console.log('SciTeX: Manual mode button injected on page!');
388
+ }}
389
+ }})();
390
+ """
391
+
392
+
393
+ async def wait_for_manual_mode_activation_async(
394
+ page: Page,
395
+ stop_event: "asyncio.Event",
396
+ ) -> None:
397
+ """Wait for user to press 'M' key to activate manual mode.
398
+
399
+ Args:
400
+ page: Playwright page
401
+ stop_event: Event to set when manual mode is activated
402
+ """
403
+ try:
404
+ # Wait for the button to be activated (data-scitex-clicked='true')
405
+ await page.wait_for_selector(
406
+ "#scitex-manual-button[data-scitex-clicked='true']",
407
+ timeout=0, # No timeout - wait forever
408
+ )
409
+
410
+ # Set the stop event
411
+ stop_event.set()
412
+
413
+ # Update button to monitoring state
414
+ await page.evaluate(
415
+ """
416
+ () => {
417
+ const button = document.getElementById('scitex-manual-button');
418
+ if (button) {
419
+ button.innerHTML = 'MONITORING<br>DOWNLOADS';
420
+ button.style.background = 'linear-gradient(135deg, #6b8fb3 0%, #7a9fc3 100%)';
421
+ button.style.border = '3px solid #506b7a';
422
+ button.style.cursor = 'default';
423
+ }
424
+ }
425
+ """
426
+ )
427
+
428
+ except Exception as e:
429
+ pass
430
+
431
+
432
+ async def show_stop_automation_button_async(
433
+ page: Page,
434
+ stop_event: "asyncio.Event",
435
+ target_filename: str,
436
+ ) -> None:
437
+ """
438
+ Show 'Download Manually' button that user can click ANYTIME to skip automation.
439
+
440
+ This button is shown IMMEDIATELY when PDF page opens and allows users
441
+ to bypass all automation attempts and go straight to manual download mode.
442
+
443
+ Args:
444
+ page: Playwright page
445
+ stop_event: Event to signal automation stop
446
+ target_filename: Target filename to display
447
+ """
448
+ try:
449
+ # Log that we're about to show the button
450
+ from scitex import logging
451
+
452
+ logger = logging.getLogger(__name__)
453
+ logger.info(
454
+ f"show_stop_automation_button_async: Injecting manual download button on page"
455
+ )
456
+
457
+ # Wait a moment for page to be ready
458
+ await page.wait_for_timeout(500)
459
+
460
+ # Inject button overlay - wait for body to be ready first
461
+ await page.evaluate(
462
+ f"""
463
+ () => {{
464
+ console.log('SciTeX: Injecting manual download controls...');
465
+
466
+ // Wait for body to exist
467
+ if (!document.body) {{
468
+ console.error('SciTeX: document.body not found!');
469
+ return;
470
+ }}
471
+
472
+ // Remove any existing button
473
+ document.getElementById('scitex-manual-button')?.remove();
474
+
475
+ // ===== MIDDLE-RIGHT FLOATING BUTTON (SciTeX branded) =====
476
+ const button = document.createElement('button');
477
+ button.id = 'scitex-manual-button';
478
+ button.setAttribute('data-scitex-no-auto-click', 'true');
479
+ button.style.cssText = `
480
+ position: fixed !important;
481
+ top: 50% !important;
482
+ left: 20px !important;
483
+ transform: translateY(-50%) !important;
484
+ z-index: 2147483647 !important;
485
+ background: linear-gradient(135deg, #506b7a 0%, #6c8ba0 30%, #8fa4b0 60%, #b5c7d1 100%) !important;
486
+ color: #1a2332 !important;
487
+ padding: 24px 36px !important;
488
+ border: 3px solid #34495e !important;
489
+ border-radius: 8px !important;
490
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif !important;
491
+ font-size: 17px !important;
492
+ font-weight: 700 !important;
493
+ cursor: pointer !important;
494
+ box-shadow: 0 8px 24px rgba(26, 35, 50, 0.4) !important;
495
+ display: block !important;
496
+ visibility: visible !important;
497
+ text-align: center !important;
498
+ line-height: 1.5 !important;
499
+ min-width: 220px !important;
500
+ text-shadow: 0 1px 2px rgba(255,255,255,0.5) !important;
501
+ `;
502
+
503
+ button.innerHTML = `
504
+ PRESS 'M'<br>FOR MANUAL
505
+ `;
506
+
507
+ document.documentElement.appendChild(button);
508
+
509
+ // Hover effects
510
+ button.addEventListener('mouseenter', () => {{
511
+ button.style.background = 'linear-gradient(135deg, #34495e 0%, #506b7a 30%, #6c8ba0 60%, #8fa4b0 100%)';
512
+ button.style.transform = 'translateY(-50%) scale(1.08)';
513
+ button.style.boxShadow = '0 12px 32px rgba(26, 35, 50, 0.5)';
514
+ }});
515
+ button.addEventListener('mouseleave', () => {{
516
+ button.style.background = 'linear-gradient(135deg, #506b7a 0%, #6c8ba0 30%, #8fa4b0 60%, #b5c7d1 100%)';
517
+ button.style.transform = 'translateY(-50%)';
518
+ button.style.boxShadow = '0 8px 24px rgba(26, 35, 50, 0.4)';
519
+ }});
520
+
521
+ // KEYBOARD handler - Press 'M' key (auto-clickers can't do this!)
522
+ document.addEventListener('keydown', (e) => {{
523
+ if (e.key === 'm' || e.key === 'M') {{
524
+ if (button.getAttribute('data-scitex-clicked') !== 'true') {{
525
+ console.log('SciTeX: Key M pressed - Manual mode activated!');
526
+ button.setAttribute('data-scitex-clicked', 'true');
527
+ button.innerHTML = 'MANUAL MODE<br>ACTIVATED';
528
+ button.style.background = 'linear-gradient(135deg, #1a2332 0%, #2d3748 50%, #34495e 100%)';
529
+ button.style.border = '3px solid #8fa4b0';
530
+ }}
531
+ }}
532
+ }}, {{ capture: true }});
533
+
534
+ // Click handler just for feedback (doesn't activate)
535
+ button.addEventListener('click', (e) => {{
536
+ if (button.getAttribute('data-scitex-clicked') !== 'true') {{
537
+ button.innerHTML = 'PRESS M KEY!';
538
+ button.style.background = 'linear-gradient(135deg, #6c8ba0 0%, #8fa4b0 100%)';
539
+ setTimeout(() => {{
540
+ if (button.getAttribute('data-scitex-clicked') !== 'true') {{
541
+ button.innerHTML = 'PRESS \\'M\\'<br>FOR MANUAL';
542
+ button.style.background = 'linear-gradient(135deg, #506b7a 0%, #6c8ba0 30%, #8fa4b0 60%, #b5c7d1 100%)';
543
+ }}
544
+ }}, 500);
545
+ }}
546
+ }}, {{ capture: true }});
547
+
548
+ // Periodically ensure button stays visible and on top
549
+ setInterval(() => {{
550
+ const existing = document.getElementById('scitex-manual-button');
551
+ if (!existing) {{
552
+ document.documentElement.appendChild(button);
553
+ }} else if (existing.parentElement) {{
554
+ existing.parentElement.appendChild(existing);
555
+ }}
556
+ }}, 2000);
557
+
558
+ console.log('SciTeX: Manual mode button injected at MIDDLE-RIGHT!');
559
+ }}
560
+ """
561
+ )
562
+
563
+ logger.info(
564
+ f"show_stop_automation_button_async: Button injected, waiting for user click..."
565
+ )
566
+
567
+ # Show browser notification that button is ready
568
+ await browser_logger.info(
569
+ page,
570
+ "MANUAL DOWNLOAD BUTTON: Check lower-right corner to skip automation!",
571
+ )
572
+
573
+ except Exception as e:
574
+ logger.error(
575
+ f"show_stop_automation_button_async: Failed to inject button: {e}"
576
+ )
577
+ return
578
+
579
+ # Wait for DOUBLE-CLICK (no timeout - always available)
580
+ try:
581
+ await page.wait_for_selector(
582
+ "#scitex-manual-button",
583
+ state="attached",
584
+ )
585
+
586
+ # Wait for the data-scitex-clicked attribute to be set by double-click
587
+ await page.wait_for_selector(
588
+ "#scitex-manual-button[data-scitex-clicked='true']",
589
+ timeout=0, # No timeout - wait forever
590
+ )
591
+
592
+ # Set the stop event
593
+ stop_event.set()
594
+
595
+ # Update button to show monitoring state
596
+ await page.evaluate(
597
+ """
598
+ () => {
599
+ const button = document.getElementById('scitex-manual-button');
600
+ if (button) {
601
+ button.innerHTML = 'MONITORING<br>DOWNLOADS';
602
+ button.style.background = 'linear-gradient(135deg, #6b8fb3 0%, #7a9fc3 100%)';
603
+ button.style.border = '3px solid #506b7a';
604
+ button.style.cursor = 'default';
605
+ }
606
+ }
607
+ """
608
+ )
609
+
610
+ except Exception as e:
611
+ # Button was removed or page closed
612
+ pass
613
+
614
+
615
+ async def show_manual_download_button_async(
616
+ page: Page,
617
+ target_filename: str,
618
+ timeout_sec: float = 300,
619
+ ) -> bool:
620
+ """
621
+ Show manual download button overlay and wait for user click.
622
+
623
+ Args:
624
+ page: Playwright page
625
+ target_filename: Target filename to display
626
+ timeout_sec: How long to wait for user click
627
+
628
+ Returns:
629
+ True if button clicked, False if timeout
630
+ """
631
+ # Inject button overlay
632
+ await page.evaluate(
633
+ f"""
634
+ () => {{
635
+ // Create overlay container
636
+ const overlay = document.createElement('div');
637
+ overlay.id = 'manual-download-overlay';
638
+ overlay.style.cssText = `
639
+ position: fixed;
640
+ top: 20px;
641
+ right: 20px;
642
+ z-index: 999999;
643
+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
644
+ padding: 20px;
645
+ border-radius: 12px;
646
+ box-shadow: 0 8px 32px rgba(0, 0, 0, 0.3);
647
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
648
+ color: white;
649
+ max-width: 350px;
650
+ `;
651
+
652
+ overlay.innerHTML = `
653
+ <div style="font-size: 16px; font-weight: 600; margin-bottom: 8px;">
654
+ 📥 Manual Download Mode
655
+ </div>
656
+ <div style="font-size: 13px; margin-bottom: 12px; opacity: 0.9;">
657
+ Target: <code style="background: rgba(255,255,255,0.2); padding: 2px 6px; border-radius: 4px;">{target_filename}</code>
658
+ </div>
659
+ <div style="font-size: 12px; margin-bottom: 12px; opacity: 0.8;">
660
+ Please download the PDF manually, then click below to continue.
661
+ </div>
662
+ <button id='manual-download-confirm' style="
663
+ width: 100%;
664
+ padding: 12px;
665
+ background: white;
666
+ color: #667eea;
667
+ border: none;
668
+ border-radius: 8px;
669
+ font-size: 14px;
670
+ font-weight: 600;
671
+ cursor: pointer;
672
+ transition: all 0.2s;
673
+ ">
674
+ ✓ I've Downloaded the PDF
675
+ </button>
676
+ `;
677
+
678
+ document.body.appendChild(overlay);
679
+
680
+ // Add hover effect
681
+ const button = overlay.querySelector('#manual-download-confirm');
682
+ button.addEventListener('mouseenter', () => {{
683
+ button.style.background = '#f0f0f0';
684
+ button.style.transform = 'scale(1.02)';
685
+ }});
686
+ button.addEventListener('mouseleave', () => {{
687
+ button.style.background = 'white';
688
+ button.style.transform = 'scale(1)';
689
+ }});
690
+ }}
691
+ """
692
+ )
693
+
694
+ # Wait for button click with timeout
695
+ try:
696
+ await page.wait_for_selector(
697
+ "#manual-download-confirm",
698
+ state="attached",
699
+ timeout=timeout_sec * 1000,
700
+ )
701
+ await page.click("#manual-download-confirm")
702
+
703
+ # Remove overlay
704
+ await page.evaluate(
705
+ "() => { document.getElementById('manual-download-overlay')?.remove(); }"
706
+ )
707
+
708
+ return True
709
+ except Exception:
710
+ # Timeout or error
711
+ await page.evaluate(
712
+ "() => { document.getElementById('manual-download-overlay')?.remove(); }"
713
+ )
714
+ return False
715
+
716
+
717
+ async def complete_manual_download_workflow_async(
718
+ page: Page,
719
+ temp_downloads_dir: Path,
720
+ final_pdfs_dir: Path,
721
+ doi: Optional[str] = None,
722
+ url: Optional[str] = None,
723
+ content_type: str = "main",
724
+ sequence_index: Optional[int] = None,
725
+ button_timeout_sec: float = 300,
726
+ download_timeout_sec: float = 120,
727
+ ) -> Optional[Path]:
728
+ """
729
+ Complete manual download workflow with monitoring and syncing.
730
+
731
+ Workflow:
732
+ 1. Show manual download button with target filename
733
+ 2. Wait for user to click (button_timeout_sec)
734
+ 3. START monitoring temp downloads directory
735
+ 4. DETECT new PDF file (download_timeout_sec)
736
+ 5. SYNC to final destination with proper naming
737
+ 6. CLEANUP and confirm
738
+
739
+ Args:
740
+ page: Playwright page
741
+ temp_downloads_dir: Temporary downloads directory (library/downloads/)
742
+ final_pdfs_dir: Final PDFs directory (library/pdfs/)
743
+ doi: DOI of article
744
+ url: URL of article
745
+ content_type: Type of content ("main", "supp", etc.)
746
+ sequence_index: Index for supplements
747
+ button_timeout_sec: How long to wait for button click
748
+ download_timeout_sec: How long to wait for download
749
+
750
+ Returns:
751
+ Path to final PDF file, or None if failed
752
+ """
753
+ # Generate target filename for display
754
+ target_filename = FlexibleFilenameGenerator.generate_filename(
755
+ doi=doi,
756
+ url=url,
757
+ content_type=content_type,
758
+ sequence_index=sequence_index,
759
+ )
760
+
761
+ # Step 1: Show button and wait for click
762
+ await browser_logger.info(
763
+ page,
764
+ f"Showing manual download button (target: {target_filename})",
765
+ func_name="complete_manual_download_workflow",
766
+ )
767
+
768
+ button_clicked = await show_manual_download_button_async(
769
+ page,
770
+ target_filename,
771
+ timeout_sec=button_timeout_sec,
772
+ )
773
+
774
+ if not button_clicked:
775
+ await browser_logger.warning(
776
+ page,
777
+ "Manual download button timeout - user did not click",
778
+ func_name="complete_manual_download_workflow",
779
+ )
780
+ return None
781
+
782
+ # Step 2: Start monitoring
783
+ await browser_logger.info(
784
+ page,
785
+ "User confirmed download - monitoring temp directory",
786
+ func_name="complete_manual_download_workflow",
787
+ )
788
+
789
+ monitor = DownloadMonitorAndSync(temp_downloads_dir, final_pdfs_dir)
790
+
791
+ # Step 3: Detect new download
792
+ temp_file = await monitor.monitor_for_new_download_async(
793
+ timeout_sec=download_timeout_sec,
794
+ )
795
+
796
+ if not temp_file:
797
+ await browser_logger.error(
798
+ page,
799
+ f"No new PDF detected in {download_timeout_sec}s",
800
+ func_name="complete_manual_download_workflow",
801
+ )
802
+ return None
803
+
804
+ await browser_logger.info(
805
+ page,
806
+ f"Detected new PDF: {temp_file.name} ({temp_file.stat().st_size / 1e6:.1f} MB)",
807
+ func_name="complete_manual_download_workflow",
808
+ )
809
+
810
+ # Step 4: Sync to final destination
811
+ final_path = monitor.sync_to_final_destination(
812
+ temp_file,
813
+ doi=doi,
814
+ url=url,
815
+ content_type=content_type,
816
+ sequence_index=sequence_index,
817
+ )
818
+
819
+ await browser_logger.info(
820
+ page,
821
+ f"Synced to library: {final_path.name}",
822
+ func_name="complete_manual_download_workflow",
823
+ )
824
+
825
+ return final_path
826
+
827
+
828
+ def get_manual_button_init_script(target_filename: str) -> str:
829
+ """Get JavaScript init script to inject manual mode button on ALL pages.
830
+
831
+ This script runs on EVERY page load (including redirects) to ensure
832
+ the manual mode button is always available.
833
+
834
+ Args:
835
+ target_filename: Target filename to display
836
+
837
+ Returns:
838
+ JavaScript code as string
839
+ """
840
+ return f"""
841
+ (() => {{
842
+ // Wait for DOM to be ready
843
+ if (document.readyState === 'loading') {{
844
+ document.addEventListener('DOMContentLoaded', injectManualButton);
845
+ }} else {{
846
+ injectManualButton();
847
+ }}
848
+
849
+ function injectManualButton() {{
850
+ // Remove existing button
851
+ document.getElementById('scitex-manual-button')?.remove();
852
+
853
+ // Create button
854
+ const button = document.createElement('button');
855
+ button.id = 'scitex-manual-button';
856
+ button.setAttribute('data-scitex-no-auto-click', 'true');
857
+ button.style.cssText = `
858
+ position: fixed !important;
859
+ top: 50% !important;
860
+ left: 20px !important;
861
+ transform: translateY(-50%) !important;
862
+ z-index: 2147483647 !important;
863
+ background: linear-gradient(135deg, #506b7a 0%, #6c8ba0 30%, #8fa4b0 60%, #b5c7d1 100%) !important;
864
+ color: #1a2332 !important;
865
+ padding: 24px 36px !important;
866
+ border: 3px solid #34495e !important;
867
+ border-radius: 8px !important;
868
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif !important;
869
+ font-size: 17px !important;
870
+ font-weight: 700 !important;
871
+ cursor: pointer !important;
872
+ box-shadow: 0 8px 24px rgba(26, 35, 50, 0.4) !important;
873
+ display: block !important;
874
+ visibility: visible !important;
875
+ text-align: center !important;
876
+ line-height: 1.5 !important;
877
+ min-width: 220px !important;
878
+ text-shadow: 0 1px 2px rgba(255,255,255,0.5) !important;
879
+ `;
880
+
881
+ button.innerHTML = `SciTeX<br>Press for Manual Mode`;
882
+
883
+ (document.documentElement || document.body).appendChild(button);
884
+
885
+ // Click handler - ONLY way to activate
886
+ button.addEventListener('click', (e) => {{
887
+ e.preventDefault();
888
+ e.stopPropagation();
889
+
890
+ if (button.getAttribute('data-activated') !== 'true') {{
891
+ console.log('SciTeX: Manual mode activated!');
892
+ button.setAttribute('data-activated', 'true');
893
+ button.innerHTML = 'SciTeX<br>Manual Mode Active';
894
+ button.style.background = 'linear-gradient(135deg, #1a2332 0%, #2d3748 50%, #34495e 100%)';
895
+ button.style.border = '3px solid #8fa4b0';
896
+ button.style.cursor = 'default';
897
+
898
+ // Close all browser_logger popups
899
+ const popupContainer = document.getElementById('_scitex_popup_container');
900
+ if (popupContainer) {{
901
+ popupContainer.remove();
902
+ }}
903
+ }}
904
+ }}, {{ capture: true }});
905
+
906
+ // Hover effects
907
+ button.addEventListener('mouseenter', () => {{
908
+ if (button.getAttribute('data-activated') !== 'true') {{
909
+ button.style.background = 'linear-gradient(135deg, #34495e 0%, #506b7a 30%, #6c8ba0 60%, #8fa4b0 100%)';
910
+ button.style.transform = 'translateY(-50%) scale(1.08)';
911
+ }}
912
+ }});
913
+ button.addEventListener('mouseleave', () => {{
914
+ if (button.getAttribute('data-activated') !== 'true') {{
915
+ button.style.background = 'linear-gradient(135deg, #506b7a 0%, #6c8ba0 30%, #8fa4b0 60%, #b5c7d1 100%)';
916
+ button.style.transform = 'translateY(-50%)';
917
+ }}
918
+ }});
919
+
920
+ console.log('SciTeX: Manual mode button injected via init script (appears on ALL pages)');
921
+ }}
922
+ }})();
923
+ """
924
+
925
+
926
+ async def wait_for_manual_mode_activation_async(
927
+ page: Page,
928
+ stop_event: asyncio.Event,
929
+ timeout_sec: float = 0, # 0 = wait forever
930
+ ) -> None:
931
+ """Wait for user to click the manual mode button.
932
+
933
+ Monitors the button's data-activated attribute which gets set when
934
+ user clicks the button.
935
+
936
+ Args:
937
+ page: Playwright page
938
+ stop_event: Event to set when manual mode is activated
939
+ timeout_sec: Timeout in seconds (0 = wait forever)
940
+ """
941
+ try:
942
+ from scitex import logging
943
+
944
+ logger = logging.getLogger(__name__)
945
+ logger.info(
946
+ "wait_for_manual_mode_activation_async: Waiting for button click..."
947
+ )
948
+
949
+ # Wait for button to be activated (clicked)
950
+ timeout_ms = timeout_sec * 1000 if timeout_sec > 0 else 0
951
+ await page.wait_for_selector(
952
+ "#scitex-manual-button[data-activated='true']",
953
+ timeout=timeout_ms,
954
+ )
955
+
956
+ logger.info(
957
+ "wait_for_manual_mode_activation_async: Button clicked! Setting stop event..."
958
+ )
959
+
960
+ # Set stop event
961
+ stop_event.set()
962
+
963
+ logger.info(
964
+ f"wait_for_manual_mode_activation_async: stop_event.is_set() = {stop_event.is_set()}"
965
+ )
966
+
967
+ # Close all browser_logger popups
968
+ await page.evaluate(
969
+ """
970
+ () => {
971
+ const popupContainer = document.getElementById('_scitex_popup_container');
972
+ if (popupContainer) {
973
+ popupContainer.remove();
974
+ }
975
+ }
976
+ """
977
+ )
978
+
979
+ # Update button to show monitoring
980
+ await page.evaluate(
981
+ """
982
+ () => {
983
+ const button = document.getElementById('scitex-manual-button');
984
+ if (button) {
985
+ button.innerHTML = 'SciTeX<br>Monitoring Downloads...';
986
+ button.style.background = 'linear-gradient(135deg, #6b8fb3 0%, #7a9fc3 100%)';
987
+ button.style.border = '3px solid #506b7a';
988
+ }
989
+ }
990
+ """
991
+ )
992
+
993
+ except Exception as e:
994
+ pass
995
+
996
+ # EOF