scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (704) hide show
  1. scitex/__init__.py +53 -15
  2. scitex/__main__.py +72 -26
  3. scitex/__version__.py +1 -1
  4. scitex/_sh.py +145 -23
  5. scitex/ai/__init__.py +30 -16
  6. scitex/ai/_gen_ai/_Anthropic.py +5 -7
  7. scitex/ai/_gen_ai/_BaseGenAI.py +2 -2
  8. scitex/ai/_gen_ai/_DeepSeek.py +10 -2
  9. scitex/ai/_gen_ai/_Google.py +2 -2
  10. scitex/ai/_gen_ai/_Llama.py +2 -2
  11. scitex/ai/_gen_ai/_OpenAI.py +2 -2
  12. scitex/ai/_gen_ai/_PARAMS.py +51 -65
  13. scitex/ai/_gen_ai/_Perplexity.py +2 -2
  14. scitex/ai/_gen_ai/__init__.py +25 -14
  15. scitex/ai/_gen_ai/_format_output_func.py +4 -4
  16. scitex/ai/classification/{classifier_server.py → Classifier.py} +5 -5
  17. scitex/ai/classification/CrossValidationExperiment.py +374 -0
  18. scitex/ai/classification/__init__.py +43 -4
  19. scitex/ai/classification/reporters/_BaseClassificationReporter.py +281 -0
  20. scitex/ai/classification/reporters/_ClassificationReporter.py +773 -0
  21. scitex/ai/classification/reporters/_MultiClassificationReporter.py +406 -0
  22. scitex/ai/classification/reporters/_SingleClassificationReporter.py +1834 -0
  23. scitex/ai/classification/reporters/__init__.py +11 -0
  24. scitex/ai/classification/reporters/reporter_utils/_Plotter.py +1028 -0
  25. scitex/ai/classification/reporters/reporter_utils/__init__.py +80 -0
  26. scitex/ai/classification/reporters/reporter_utils/aggregation.py +457 -0
  27. scitex/ai/classification/reporters/reporter_utils/data_models.py +313 -0
  28. scitex/ai/classification/reporters/reporter_utils/reporting.py +1056 -0
  29. scitex/ai/classification/reporters/reporter_utils/storage.py +221 -0
  30. scitex/ai/classification/reporters/reporter_utils/validation.py +395 -0
  31. scitex/ai/classification/timeseries/_TimeSeriesBlockingSplit.py +568 -0
  32. scitex/ai/classification/timeseries/_TimeSeriesCalendarSplit.py +688 -0
  33. scitex/ai/classification/timeseries/_TimeSeriesMetadata.py +139 -0
  34. scitex/ai/classification/timeseries/_TimeSeriesSlidingWindowSplit.py +1716 -0
  35. scitex/ai/classification/timeseries/_TimeSeriesSlidingWindowSplit_v01-not-using-n_splits.py +1685 -0
  36. scitex/ai/classification/timeseries/_TimeSeriesStrategy.py +84 -0
  37. scitex/ai/classification/timeseries/_TimeSeriesStratifiedSplit.py +610 -0
  38. scitex/ai/classification/timeseries/__init__.py +39 -0
  39. scitex/ai/classification/timeseries/_normalize_timestamp.py +436 -0
  40. scitex/ai/clustering/_umap.py +2 -2
  41. scitex/ai/feature_extraction/vit.py +1 -0
  42. scitex/ai/feature_selection/__init__.py +30 -0
  43. scitex/ai/feature_selection/feature_selection.py +364 -0
  44. scitex/ai/loss/multi_task_loss.py +1 -1
  45. scitex/ai/metrics/__init__.py +51 -4
  46. scitex/ai/metrics/_calc_bacc.py +61 -0
  47. scitex/ai/metrics/_calc_bacc_from_conf_mat.py +38 -0
  48. scitex/ai/metrics/_calc_clf_report.py +78 -0
  49. scitex/ai/metrics/_calc_conf_mat.py +93 -0
  50. scitex/ai/metrics/_calc_feature_importance.py +183 -0
  51. scitex/ai/metrics/_calc_mcc.py +61 -0
  52. scitex/ai/metrics/_calc_pre_rec_auc.py +116 -0
  53. scitex/ai/metrics/_calc_roc_auc.py +110 -0
  54. scitex/ai/metrics/_calc_seizure_prediction_metrics.py +490 -0
  55. scitex/ai/metrics/{silhoute_score_block.py → _calc_silhouette_score.py} +15 -8
  56. scitex/ai/metrics/_normalize_labels.py +83 -0
  57. scitex/ai/plt/__init__.py +47 -8
  58. scitex/ai/plt/{_conf_mat.py → _plot_conf_mat.py} +158 -87
  59. scitex/ai/plt/_plot_feature_importance.py +323 -0
  60. scitex/ai/plt/_plot_learning_curve.py +345 -0
  61. scitex/ai/plt/_plot_optuna_study.py +225 -0
  62. scitex/ai/plt/_plot_pre_rec_curve.py +290 -0
  63. scitex/ai/plt/_plot_roc_curve.py +255 -0
  64. scitex/ai/training/{learning_curve_logger.py → _LearningCurveLogger.py} +197 -213
  65. scitex/ai/training/__init__.py +2 -2
  66. scitex/ai/utils/grid_search.py +3 -3
  67. scitex/benchmark/__init__.py +52 -0
  68. scitex/benchmark/benchmark.py +400 -0
  69. scitex/benchmark/monitor.py +370 -0
  70. scitex/benchmark/profiler.py +297 -0
  71. scitex/browser/__init__.py +48 -0
  72. scitex/browser/automation/CookieHandler.py +216 -0
  73. scitex/browser/automation/__init__.py +7 -0
  74. scitex/browser/collaboration/__init__.py +55 -0
  75. scitex/browser/collaboration/auth_helpers.py +94 -0
  76. scitex/browser/collaboration/collaborative_agent.py +136 -0
  77. scitex/browser/collaboration/credential_manager.py +188 -0
  78. scitex/browser/collaboration/interactive_panel.py +400 -0
  79. scitex/browser/collaboration/persistent_browser.py +170 -0
  80. scitex/browser/collaboration/shared_session.py +383 -0
  81. scitex/browser/collaboration/standard_interactions.py +246 -0
  82. scitex/browser/collaboration/visual_feedback.py +181 -0
  83. scitex/browser/core/BrowserMixin.py +326 -0
  84. scitex/browser/core/ChromeProfileManager.py +446 -0
  85. scitex/browser/core/__init__.py +9 -0
  86. scitex/browser/debugging/__init__.py +18 -0
  87. scitex/browser/debugging/_browser_logger.py +657 -0
  88. scitex/browser/debugging/_highlight_element.py +143 -0
  89. scitex/browser/debugging/_show_grid.py +154 -0
  90. scitex/browser/interaction/__init__.py +24 -0
  91. scitex/browser/interaction/click_center.py +149 -0
  92. scitex/browser/interaction/click_with_fallbacks.py +206 -0
  93. scitex/browser/interaction/close_popups.py +498 -0
  94. scitex/browser/interaction/fill_with_fallbacks.py +209 -0
  95. scitex/browser/pdf/__init__.py +14 -0
  96. scitex/browser/pdf/click_download_for_chrome_pdf_viewer.py +200 -0
  97. scitex/browser/pdf/detect_chrome_pdf_viewer.py +198 -0
  98. scitex/browser/remote/CaptchaHandler.py +434 -0
  99. scitex/browser/remote/ZenRowsAPIClient.py +347 -0
  100. scitex/browser/remote/ZenRowsBrowserManager.py +570 -0
  101. scitex/browser/remote/__init__.py +11 -0
  102. scitex/browser/stealth/HumanBehavior.py +344 -0
  103. scitex/browser/stealth/StealthManager.py +1008 -0
  104. scitex/browser/stealth/__init__.py +9 -0
  105. scitex/browser/template.py +122 -0
  106. scitex/capture/__init__.py +110 -0
  107. scitex/capture/__main__.py +25 -0
  108. scitex/capture/capture.py +848 -0
  109. scitex/capture/cli.py +233 -0
  110. scitex/capture/gif.py +344 -0
  111. scitex/capture/mcp_server.py +961 -0
  112. scitex/capture/session.py +70 -0
  113. scitex/capture/utils.py +705 -0
  114. scitex/cli/__init__.py +17 -0
  115. scitex/cli/cloud.py +447 -0
  116. scitex/cli/main.py +42 -0
  117. scitex/cli/scholar.py +280 -0
  118. scitex/context/_suppress_output.py +5 -3
  119. scitex/db/__init__.py +30 -3
  120. scitex/db/__main__.py +75 -0
  121. scitex/db/_check_health.py +381 -0
  122. scitex/db/_delete_duplicates.py +25 -386
  123. scitex/db/_inspect.py +335 -114
  124. scitex/db/_inspect_optimized.py +301 -0
  125. scitex/db/{_PostgreSQL.py → _postgresql/_PostgreSQL.py} +3 -3
  126. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_BackupMixin.py +1 -1
  127. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_BatchMixin.py +1 -1
  128. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_BlobMixin.py +1 -1
  129. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_ConnectionMixin.py +1 -1
  130. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_MaintenanceMixin.py +1 -1
  131. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_QueryMixin.py +1 -1
  132. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_SchemaMixin.py +1 -1
  133. scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_TransactionMixin.py +1 -1
  134. scitex/db/_postgresql/__init__.py +6 -0
  135. scitex/db/_sqlite3/_SQLite3.py +210 -0
  136. scitex/db/_sqlite3/_SQLite3Mixins/_ArrayMixin.py +581 -0
  137. scitex/db/_sqlite3/_SQLite3Mixins/_ArrayMixin_v01-need-_hash-col.py +517 -0
  138. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_BatchMixin.py +1 -1
  139. scitex/db/_sqlite3/_SQLite3Mixins/_BlobMixin.py +281 -0
  140. scitex/db/_sqlite3/_SQLite3Mixins/_ColumnMixin.py +548 -0
  141. scitex/db/_sqlite3/_SQLite3Mixins/_ColumnMixin_v01-indentation-issues.py +583 -0
  142. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_ConnectionMixin.py +29 -13
  143. scitex/db/_sqlite3/_SQLite3Mixins/_GitMixin.py +583 -0
  144. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_ImportExportMixin.py +1 -1
  145. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_IndexMixin.py +1 -1
  146. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_MaintenanceMixin.py +2 -1
  147. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_QueryMixin.py +37 -10
  148. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_RowMixin.py +46 -6
  149. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_TableMixin.py +56 -10
  150. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_TransactionMixin.py +1 -1
  151. scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/__init__.py +14 -2
  152. scitex/db/_sqlite3/__init__.py +7 -0
  153. scitex/db/_sqlite3/_delete_duplicates.py +274 -0
  154. scitex/decorators/__init__.py +2 -0
  155. scitex/decorators/_cache_disk.py +13 -5
  156. scitex/decorators/_cache_disk_async.py +49 -0
  157. scitex/decorators/_deprecated.py +175 -10
  158. scitex/decorators/_timeout.py +1 -1
  159. scitex/dev/_analyze_code_flow.py +2 -2
  160. scitex/dict/_DotDict.py +73 -15
  161. scitex/dict/_DotDict_v01-not-handling-recursive-instantiations.py +442 -0
  162. scitex/dict/_DotDict_v02-not-serializing-Path-object.py +446 -0
  163. scitex/dict/__init__.py +2 -0
  164. scitex/dict/_flatten.py +27 -0
  165. scitex/dsp/_crop.py +2 -2
  166. scitex/dsp/_demo_sig.py +2 -2
  167. scitex/dsp/_detect_ripples.py +2 -2
  168. scitex/dsp/_hilbert.py +2 -2
  169. scitex/dsp/_listen.py +6 -6
  170. scitex/dsp/_modulation_index.py +2 -2
  171. scitex/dsp/_pac.py +1 -1
  172. scitex/dsp/_psd.py +2 -2
  173. scitex/dsp/_resample.py +2 -1
  174. scitex/dsp/_time.py +3 -2
  175. scitex/dsp/_wavelet.py +3 -2
  176. scitex/dsp/add_noise.py +2 -2
  177. scitex/dsp/example.py +1 -0
  178. scitex/dsp/filt.py +10 -9
  179. scitex/dsp/template.py +3 -2
  180. scitex/dsp/utils/_differential_bandpass_filters.py +1 -1
  181. scitex/dsp/utils/pac.py +2 -2
  182. scitex/dt/_normalize_timestamp.py +432 -0
  183. scitex/errors.py +572 -0
  184. scitex/gen/_DimHandler.py +2 -2
  185. scitex/gen/__init__.py +37 -7
  186. scitex/gen/_deprecated_close.py +80 -0
  187. scitex/gen/_deprecated_start.py +26 -0
  188. scitex/gen/_detect_environment.py +152 -0
  189. scitex/gen/_detect_notebook_path.py +169 -0
  190. scitex/gen/_embed.py +6 -2
  191. scitex/gen/_get_notebook_path.py +257 -0
  192. scitex/gen/_less.py +1 -1
  193. scitex/gen/_list_packages.py +2 -2
  194. scitex/gen/_norm.py +44 -9
  195. scitex/gen/_norm_cache.py +269 -0
  196. scitex/gen/_src.py +3 -5
  197. scitex/gen/_title_case.py +3 -3
  198. scitex/io/__init__.py +28 -6
  199. scitex/io/_glob.py +13 -7
  200. scitex/io/_load.py +108 -21
  201. scitex/io/_load_cache.py +303 -0
  202. scitex/io/_load_configs.py +40 -15
  203. scitex/io/{_H5Explorer.py → _load_modules/_H5Explorer.py} +80 -17
  204. scitex/io/_load_modules/_ZarrExplorer.py +114 -0
  205. scitex/io/_load_modules/_bibtex.py +207 -0
  206. scitex/io/_load_modules/_hdf5.py +53 -178
  207. scitex/io/_load_modules/_json.py +5 -3
  208. scitex/io/_load_modules/_pdf.py +871 -16
  209. scitex/io/_load_modules/_sqlite3.py +15 -0
  210. scitex/io/_load_modules/_txt.py +41 -12
  211. scitex/io/_load_modules/_yaml.py +4 -3
  212. scitex/io/_load_modules/_zarr.py +126 -0
  213. scitex/io/_save.py +429 -171
  214. scitex/io/_save_modules/__init__.py +6 -0
  215. scitex/io/_save_modules/_bibtex.py +194 -0
  216. scitex/io/_save_modules/_csv.py +8 -4
  217. scitex/io/_save_modules/_excel.py +174 -15
  218. scitex/io/_save_modules/_hdf5.py +251 -226
  219. scitex/io/_save_modules/_image.py +1 -3
  220. scitex/io/_save_modules/_json.py +49 -4
  221. scitex/io/_save_modules/_listed_dfs_as_csv.py +1 -3
  222. scitex/io/_save_modules/_listed_scalars_as_csv.py +1 -3
  223. scitex/io/_save_modules/_tex.py +277 -0
  224. scitex/io/_save_modules/_yaml.py +42 -3
  225. scitex/io/_save_modules/_zarr.py +160 -0
  226. scitex/io/utils/__init__.py +20 -0
  227. scitex/io/utils/h5_to_zarr.py +616 -0
  228. scitex/linalg/_geometric_median.py +6 -2
  229. scitex/{gen/_tee.py → logging/_Tee.py} +43 -84
  230. scitex/logging/__init__.py +122 -0
  231. scitex/logging/_config.py +158 -0
  232. scitex/logging/_context.py +103 -0
  233. scitex/logging/_formatters.py +128 -0
  234. scitex/logging/_handlers.py +64 -0
  235. scitex/logging/_levels.py +35 -0
  236. scitex/logging/_logger.py +163 -0
  237. scitex/logging/_print_capture.py +95 -0
  238. scitex/ml/__init__.py +69 -0
  239. scitex/{ai/genai/anthropic.py → ml/_gen_ai/_Anthropic.py} +13 -19
  240. scitex/{ai/genai/base_genai.py → ml/_gen_ai/_BaseGenAI.py} +5 -5
  241. scitex/{ai/genai/deepseek.py → ml/_gen_ai/_DeepSeek.py} +11 -16
  242. scitex/{ai/genai/google.py → ml/_gen_ai/_Google.py} +7 -15
  243. scitex/{ai/genai/groq.py → ml/_gen_ai/_Groq.py} +1 -8
  244. scitex/{ai/genai/llama.py → ml/_gen_ai/_Llama.py} +3 -16
  245. scitex/{ai/genai/openai.py → ml/_gen_ai/_OpenAI.py} +3 -3
  246. scitex/{ai/genai/params.py → ml/_gen_ai/_PARAMS.py} +51 -65
  247. scitex/{ai/genai/perplexity.py → ml/_gen_ai/_Perplexity.py} +3 -14
  248. scitex/ml/_gen_ai/__init__.py +43 -0
  249. scitex/{ai/genai/calc_cost.py → ml/_gen_ai/_calc_cost.py} +1 -1
  250. scitex/{ai/genai/format_output_func.py → ml/_gen_ai/_format_output_func.py} +4 -4
  251. scitex/{ai/genai/genai_factory.py → ml/_gen_ai/_genai_factory.py} +8 -8
  252. scitex/ml/activation/__init__.py +8 -0
  253. scitex/ml/activation/_define.py +11 -0
  254. scitex/{ai/classifier_server.py → ml/classification/Classifier.py} +5 -5
  255. scitex/ml/classification/CrossValidationExperiment.py +374 -0
  256. scitex/ml/classification/__init__.py +46 -0
  257. scitex/ml/classification/reporters/_BaseClassificationReporter.py +281 -0
  258. scitex/ml/classification/reporters/_ClassificationReporter.py +773 -0
  259. scitex/ml/classification/reporters/_MultiClassificationReporter.py +406 -0
  260. scitex/ml/classification/reporters/_SingleClassificationReporter.py +1834 -0
  261. scitex/ml/classification/reporters/__init__.py +11 -0
  262. scitex/ml/classification/reporters/reporter_utils/_Plotter.py +1028 -0
  263. scitex/ml/classification/reporters/reporter_utils/__init__.py +80 -0
  264. scitex/ml/classification/reporters/reporter_utils/aggregation.py +457 -0
  265. scitex/ml/classification/reporters/reporter_utils/data_models.py +313 -0
  266. scitex/ml/classification/reporters/reporter_utils/reporting.py +1056 -0
  267. scitex/ml/classification/reporters/reporter_utils/storage.py +221 -0
  268. scitex/ml/classification/reporters/reporter_utils/validation.py +395 -0
  269. scitex/ml/classification/timeseries/_TimeSeriesBlockingSplit.py +568 -0
  270. scitex/ml/classification/timeseries/_TimeSeriesCalendarSplit.py +688 -0
  271. scitex/ml/classification/timeseries/_TimeSeriesMetadata.py +139 -0
  272. scitex/ml/classification/timeseries/_TimeSeriesSlidingWindowSplit.py +1716 -0
  273. scitex/ml/classification/timeseries/_TimeSeriesSlidingWindowSplit_v01-not-using-n_splits.py +1685 -0
  274. scitex/ml/classification/timeseries/_TimeSeriesStrategy.py +84 -0
  275. scitex/ml/classification/timeseries/_TimeSeriesStratifiedSplit.py +610 -0
  276. scitex/ml/classification/timeseries/__init__.py +39 -0
  277. scitex/ml/classification/timeseries/_normalize_timestamp.py +436 -0
  278. scitex/ml/clustering/__init__.py +11 -0
  279. scitex/ml/clustering/_pca.py +115 -0
  280. scitex/ml/clustering/_umap.py +376 -0
  281. scitex/ml/feature_extraction/__init__.py +56 -0
  282. scitex/ml/feature_extraction/vit.py +149 -0
  283. scitex/ml/feature_selection/__init__.py +30 -0
  284. scitex/ml/feature_selection/feature_selection.py +364 -0
  285. scitex/ml/loss/_L1L2Losses.py +34 -0
  286. scitex/ml/loss/__init__.py +12 -0
  287. scitex/ml/loss/multi_task_loss.py +47 -0
  288. scitex/ml/metrics/__init__.py +56 -0
  289. scitex/ml/metrics/_calc_bacc.py +61 -0
  290. scitex/ml/metrics/_calc_bacc_from_conf_mat.py +38 -0
  291. scitex/ml/metrics/_calc_clf_report.py +78 -0
  292. scitex/ml/metrics/_calc_conf_mat.py +93 -0
  293. scitex/ml/metrics/_calc_feature_importance.py +183 -0
  294. scitex/ml/metrics/_calc_mcc.py +61 -0
  295. scitex/ml/metrics/_calc_pre_rec_auc.py +116 -0
  296. scitex/ml/metrics/_calc_roc_auc.py +110 -0
  297. scitex/ml/metrics/_calc_seizure_prediction_metrics.py +490 -0
  298. scitex/ml/metrics/_calc_silhouette_score.py +503 -0
  299. scitex/ml/metrics/_normalize_labels.py +83 -0
  300. scitex/ml/optim/Ranger_Deep_Learning_Optimizer/__init__.py +0 -0
  301. scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/__init__.py +3 -0
  302. scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/ranger.py +207 -0
  303. scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/ranger2020.py +238 -0
  304. scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/ranger913A.py +215 -0
  305. scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/rangerqh.py +184 -0
  306. scitex/ml/optim/Ranger_Deep_Learning_Optimizer/setup.py +24 -0
  307. scitex/ml/optim/__init__.py +13 -0
  308. scitex/ml/optim/_get_set.py +31 -0
  309. scitex/ml/optim/_optimizers.py +71 -0
  310. scitex/ml/plt/__init__.py +60 -0
  311. scitex/ml/plt/_plot_conf_mat.py +663 -0
  312. scitex/ml/plt/_plot_feature_importance.py +323 -0
  313. scitex/ml/plt/_plot_learning_curve.py +345 -0
  314. scitex/ml/plt/_plot_optuna_study.py +225 -0
  315. scitex/ml/plt/_plot_pre_rec_curve.py +290 -0
  316. scitex/ml/plt/_plot_roc_curve.py +255 -0
  317. scitex/ml/sk/__init__.py +11 -0
  318. scitex/ml/sk/_clf.py +58 -0
  319. scitex/ml/sk/_to_sktime.py +100 -0
  320. scitex/ml/sklearn/__init__.py +26 -0
  321. scitex/ml/sklearn/clf.py +58 -0
  322. scitex/ml/sklearn/to_sktime.py +100 -0
  323. scitex/{ai/training/early_stopping.py → ml/training/_EarlyStopping.py} +1 -2
  324. scitex/{ai → ml/training}/_LearningCurveLogger.py +198 -242
  325. scitex/ml/training/__init__.py +7 -0
  326. scitex/ml/utils/__init__.py +22 -0
  327. scitex/ml/utils/_check_params.py +50 -0
  328. scitex/ml/utils/_default_dataset.py +46 -0
  329. scitex/ml/utils/_format_samples_for_sktime.py +26 -0
  330. scitex/ml/utils/_label_encoder.py +134 -0
  331. scitex/ml/utils/_merge_labels.py +22 -0
  332. scitex/ml/utils/_sliding_window_data_augmentation.py +11 -0
  333. scitex/ml/utils/_under_sample.py +51 -0
  334. scitex/ml/utils/_verify_n_gpus.py +16 -0
  335. scitex/ml/utils/grid_search.py +148 -0
  336. scitex/nn/_BNet.py +15 -9
  337. scitex/nn/_Filters.py +2 -2
  338. scitex/nn/_ModulationIndex.py +2 -2
  339. scitex/nn/_PAC.py +1 -1
  340. scitex/nn/_Spectrogram.py +12 -3
  341. scitex/nn/__init__.py +9 -10
  342. scitex/path/__init__.py +18 -0
  343. scitex/path/_clean.py +4 -0
  344. scitex/path/_find.py +9 -4
  345. scitex/path/_symlink.py +348 -0
  346. scitex/path/_version.py +4 -3
  347. scitex/pd/__init__.py +2 -0
  348. scitex/pd/_get_unique.py +99 -0
  349. scitex/plt/__init__.py +114 -5
  350. scitex/plt/_subplots/_AxesWrapper.py +1 -3
  351. scitex/plt/_subplots/_AxisWrapper.py +7 -3
  352. scitex/plt/_subplots/_AxisWrapperMixins/_AdjustmentMixin.py +47 -13
  353. scitex/plt/_subplots/_AxisWrapperMixins/_MatplotlibPlotMixin.py +160 -2
  354. scitex/plt/_subplots/_AxisWrapperMixins/_SeabornMixin.py +26 -4
  355. scitex/plt/_subplots/_AxisWrapperMixins/_UnitAwareMixin.py +322 -0
  356. scitex/plt/_subplots/_AxisWrapperMixins/__init__.py +1 -0
  357. scitex/plt/_subplots/_FigWrapper.py +62 -6
  358. scitex/plt/_subplots/_export_as_csv.py +43 -27
  359. scitex/plt/_subplots/_export_as_csv_formatters/__init__.py +5 -4
  360. scitex/plt/_subplots/_export_as_csv_formatters/_format_annotate.py +81 -0
  361. scitex/plt/_subplots/_export_as_csv_formatters/_format_bar.py +1 -3
  362. scitex/plt/_subplots/_export_as_csv_formatters/_format_barh.py +20 -5
  363. scitex/plt/_subplots/_export_as_csv_formatters/_format_boxplot.py +1 -3
  364. scitex/plt/_subplots/_export_as_csv_formatters/_format_contour.py +1 -3
  365. scitex/plt/_subplots/_export_as_csv_formatters/_format_errorbar.py +35 -18
  366. scitex/plt/_subplots/_export_as_csv_formatters/_format_eventplot.py +1 -3
  367. scitex/plt/_subplots/_export_as_csv_formatters/_format_fill.py +1 -3
  368. scitex/plt/_subplots/_export_as_csv_formatters/_format_fill_between.py +1 -3
  369. scitex/plt/_subplots/_export_as_csv_formatters/_format_hist.py +1 -3
  370. scitex/plt/_subplots/_export_as_csv_formatters/_format_imshow.py +1 -3
  371. scitex/plt/_subplots/_export_as_csv_formatters/_format_imshow2d.py +1 -3
  372. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot.py +15 -3
  373. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_box.py +1 -3
  374. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_conf_mat.py +1 -3
  375. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_ecdf.py +1 -3
  376. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_fillv.py +1 -3
  377. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_heatmap.py +1 -3
  378. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_image.py +1 -3
  379. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_joyplot.py +1 -3
  380. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_kde.py +1 -3
  381. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_line.py +1 -3
  382. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_mean_ci.py +1 -3
  383. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_mean_std.py +1 -3
  384. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_median_iqr.py +1 -3
  385. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_raster.py +1 -3
  386. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_rectangle.py +1 -3
  387. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_scatter.py +35 -0
  388. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_scatter_hist.py +1 -3
  389. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_shaded_line.py +1 -3
  390. scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_violin.py +1 -3
  391. scitex/plt/_subplots/_export_as_csv_formatters/_format_scatter.py +6 -4
  392. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_barplot.py +1 -3
  393. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_boxplot.py +1 -3
  394. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_heatmap.py +1 -3
  395. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_histplot.py +1 -3
  396. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_jointplot.py +1 -3
  397. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_kdeplot.py +1 -3
  398. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_lineplot.py +1 -3
  399. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_pairplot.py +1 -3
  400. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_scatterplot.py +1 -3
  401. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_stripplot.py +1 -3
  402. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_swarmplot.py +1 -3
  403. scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_violinplot.py +1 -3
  404. scitex/plt/_subplots/_export_as_csv_formatters/_format_text.py +60 -0
  405. scitex/plt/_subplots/_export_as_csv_formatters/_format_violin.py +1 -3
  406. scitex/plt/_subplots/_export_as_csv_formatters/_format_violinplot.py +1 -3
  407. scitex/plt/_subplots/_export_as_csv_formatters/test_formatters.py +1 -3
  408. scitex/plt/_subplots/_export_as_csv_formatters.py +56 -59
  409. scitex/plt/ax/_style/_hide_spines.py +1 -3
  410. scitex/plt/ax/_style/_rotate_labels.py +180 -76
  411. scitex/plt/ax/_style/_rotate_labels_v01.py +248 -0
  412. scitex/plt/ax/_style/_set_meta.py +11 -4
  413. scitex/plt/ax/_style/_set_supxyt.py +3 -3
  414. scitex/plt/ax/_style/_set_xyt.py +3 -3
  415. scitex/plt/ax/_style/_share_axes.py +2 -2
  416. scitex/plt/color/__init__.py +4 -4
  417. scitex/plt/color/{_get_colors_from_cmap.py → _get_colors_from_conf_matap.py} +7 -7
  418. scitex/plt/utils/_configure_mpl.py +99 -86
  419. scitex/plt/utils/_histogram_utils.py +1 -3
  420. scitex/plt/utils/_is_valid_axis.py +1 -3
  421. scitex/plt/utils/_scitex_config.py +1 -0
  422. scitex/repro/__init__.py +75 -0
  423. scitex/{reproduce → repro}/_gen_ID.py +1 -1
  424. scitex/{reproduce → repro}/_gen_timestamp.py +1 -1
  425. scitex/repro_rng/_RandomStateManager.py +590 -0
  426. scitex/repro_rng/_RandomStateManager_v01-no-verbose-options.py +414 -0
  427. scitex/repro_rng/__init__.py +39 -0
  428. scitex/reproduce/__init__.py +25 -13
  429. scitex/reproduce/_hash_array.py +22 -0
  430. scitex/resource/_get_processor_usages.py +4 -4
  431. scitex/resource/_get_specs.py +2 -2
  432. scitex/resource/_log_processor_usages.py +2 -2
  433. scitex/rng/_RandomStateManager.py +590 -0
  434. scitex/rng/_RandomStateManager_v01-no-verbose-options.py +414 -0
  435. scitex/rng/__init__.py +39 -0
  436. scitex/scholar/__init__.py +309 -19
  437. scitex/scholar/__main__.py +319 -0
  438. scitex/scholar/auth/ScholarAuthManager.py +308 -0
  439. scitex/scholar/auth/__init__.py +12 -0
  440. scitex/scholar/auth/core/AuthenticationGateway.py +473 -0
  441. scitex/scholar/auth/core/BrowserAuthenticator.py +386 -0
  442. scitex/scholar/auth/core/StrategyResolver.py +309 -0
  443. scitex/scholar/auth/core/__init__.py +16 -0
  444. scitex/scholar/auth/gateway/_OpenURLLinkFinder.py +120 -0
  445. scitex/scholar/auth/gateway/_OpenURLResolver.py +209 -0
  446. scitex/scholar/auth/gateway/__init__.py +38 -0
  447. scitex/scholar/auth/gateway/_resolve_functions.py +101 -0
  448. scitex/scholar/auth/providers/BaseAuthenticator.py +166 -0
  449. scitex/scholar/auth/providers/EZProxyAuthenticator.py +484 -0
  450. scitex/scholar/auth/providers/OpenAthensAuthenticator.py +619 -0
  451. scitex/scholar/auth/providers/ShibbolethAuthenticator.py +686 -0
  452. scitex/scholar/auth/providers/__init__.py +18 -0
  453. scitex/scholar/auth/session/AuthCacheManager.py +189 -0
  454. scitex/scholar/auth/session/SessionManager.py +159 -0
  455. scitex/scholar/auth/session/__init__.py +11 -0
  456. scitex/scholar/auth/sso/BaseSSOAutomator.py +373 -0
  457. scitex/scholar/auth/sso/OpenAthensSSOAutomator.py +378 -0
  458. scitex/scholar/auth/sso/SSOAutomator.py +180 -0
  459. scitex/scholar/auth/sso/UniversityOfMelbourneSSOAutomator.py +380 -0
  460. scitex/scholar/auth/sso/__init__.py +15 -0
  461. scitex/scholar/browser/ScholarBrowserManager.py +705 -0
  462. scitex/scholar/browser/__init__.py +38 -0
  463. scitex/scholar/browser/utils/__init__.py +13 -0
  464. scitex/scholar/browser/utils/click_and_wait.py +205 -0
  465. scitex/scholar/browser/utils/close_unwanted_pages.py +140 -0
  466. scitex/scholar/browser/utils/wait_redirects.py +732 -0
  467. scitex/scholar/config/PublisherRules.py +132 -0
  468. scitex/scholar/config/ScholarConfig.py +126 -0
  469. scitex/scholar/config/__init__.py +17 -0
  470. scitex/scholar/core/Paper.py +627 -0
  471. scitex/scholar/core/Papers.py +722 -0
  472. scitex/scholar/core/Scholar.py +1975 -0
  473. scitex/scholar/core/__init__.py +9 -0
  474. scitex/scholar/impact_factor/ImpactFactorEngine.py +204 -0
  475. scitex/scholar/impact_factor/__init__.py +20 -0
  476. scitex/scholar/impact_factor/estimation/ImpactFactorEstimationEngine.py +0 -0
  477. scitex/scholar/impact_factor/estimation/__init__.py +40 -0
  478. scitex/scholar/impact_factor/estimation/build_database.py +0 -0
  479. scitex/scholar/impact_factor/estimation/core/__init__.py +28 -0
  480. scitex/scholar/impact_factor/estimation/core/cache_manager.py +523 -0
  481. scitex/scholar/impact_factor/estimation/core/calculator.py +355 -0
  482. scitex/scholar/impact_factor/estimation/core/journal_matcher.py +428 -0
  483. scitex/scholar/integration/__init__.py +59 -0
  484. scitex/scholar/integration/base.py +502 -0
  485. scitex/scholar/integration/mendeley/__init__.py +22 -0
  486. scitex/scholar/integration/mendeley/exporter.py +166 -0
  487. scitex/scholar/integration/mendeley/importer.py +236 -0
  488. scitex/scholar/integration/mendeley/linker.py +79 -0
  489. scitex/scholar/integration/mendeley/mapper.py +212 -0
  490. scitex/scholar/integration/zotero/__init__.py +27 -0
  491. scitex/scholar/integration/zotero/__main__.py +264 -0
  492. scitex/scholar/integration/zotero/exporter.py +351 -0
  493. scitex/scholar/integration/zotero/importer.py +372 -0
  494. scitex/scholar/integration/zotero/linker.py +415 -0
  495. scitex/scholar/integration/zotero/mapper.py +286 -0
  496. scitex/scholar/metadata_engines/ScholarEngine.py +588 -0
  497. scitex/scholar/metadata_engines/__init__.py +21 -0
  498. scitex/scholar/metadata_engines/individual/ArXivEngine.py +397 -0
  499. scitex/scholar/metadata_engines/individual/CrossRefEngine.py +274 -0
  500. scitex/scholar/metadata_engines/individual/CrossRefLocalEngine.py +263 -0
  501. scitex/scholar/metadata_engines/individual/OpenAlexEngine.py +350 -0
  502. scitex/scholar/metadata_engines/individual/PubMedEngine.py +329 -0
  503. scitex/scholar/metadata_engines/individual/SemanticScholarEngine.py +438 -0
  504. scitex/scholar/metadata_engines/individual/URLDOIEngine.py +410 -0
  505. scitex/scholar/metadata_engines/individual/_BaseDOIEngine.py +487 -0
  506. scitex/scholar/metadata_engines/individual/__init__.py +7 -0
  507. scitex/scholar/metadata_engines/utils/_PubMedConverter.py +469 -0
  508. scitex/scholar/metadata_engines/utils/_URLDOIExtractor.py +283 -0
  509. scitex/scholar/metadata_engines/utils/__init__.py +30 -0
  510. scitex/scholar/metadata_engines/utils/_metadata2bibtex.py +103 -0
  511. scitex/scholar/metadata_engines/utils/_standardize_metadata.py +376 -0
  512. scitex/scholar/pdf_download/ScholarPDFDownloader.py +579 -0
  513. scitex/scholar/pdf_download/__init__.py +5 -0
  514. scitex/scholar/pdf_download/strategies/__init__.py +38 -0
  515. scitex/scholar/pdf_download/strategies/chrome_pdf_viewer.py +376 -0
  516. scitex/scholar/pdf_download/strategies/direct_download.py +131 -0
  517. scitex/scholar/pdf_download/strategies/manual_download_fallback.py +167 -0
  518. scitex/scholar/pdf_download/strategies/manual_download_utils.py +996 -0
  519. scitex/scholar/pdf_download/strategies/response_body.py +207 -0
  520. scitex/scholar/pipelines/ScholarPipelineBibTeX.py +364 -0
  521. scitex/scholar/pipelines/ScholarPipelineParallel.py +478 -0
  522. scitex/scholar/pipelines/ScholarPipelineSingle.py +767 -0
  523. scitex/scholar/pipelines/__init__.py +49 -0
  524. scitex/scholar/storage/BibTeXHandler.py +1018 -0
  525. scitex/scholar/storage/PaperIO.py +468 -0
  526. scitex/scholar/storage/ScholarLibrary.py +182 -0
  527. scitex/scholar/storage/_DeduplicationManager.py +548 -0
  528. scitex/scholar/storage/_LibraryCacheManager.py +724 -0
  529. scitex/scholar/storage/_LibraryManager.py +1835 -0
  530. scitex/scholar/storage/__init__.py +28 -0
  531. scitex/scholar/url_finder/ScholarURLFinder.py +379 -0
  532. scitex/scholar/url_finder/__init__.py +7 -0
  533. scitex/scholar/url_finder/strategies/__init__.py +33 -0
  534. scitex/scholar/url_finder/strategies/find_pdf_urls_by_direct_links.py +261 -0
  535. scitex/scholar/url_finder/strategies/find_pdf_urls_by_dropdown.py +67 -0
  536. scitex/scholar/url_finder/strategies/find_pdf_urls_by_href.py +204 -0
  537. scitex/scholar/url_finder/strategies/find_pdf_urls_by_navigation.py +256 -0
  538. scitex/scholar/url_finder/strategies/find_pdf_urls_by_publisher_patterns.py +165 -0
  539. scitex/scholar/url_finder/strategies/find_pdf_urls_by_zotero_translators.py +163 -0
  540. scitex/scholar/url_finder/strategies/find_supplementary_urls_by_href.py +70 -0
  541. scitex/scholar/utils/__init__.py +22 -0
  542. scitex/scholar/utils/bibtex/__init__.py +9 -0
  543. scitex/scholar/utils/bibtex/_parse_bibtex.py +71 -0
  544. scitex/scholar/utils/cleanup/__init__.py +8 -0
  545. scitex/scholar/utils/cleanup/_cleanup_scholar_processes.py +96 -0
  546. scitex/scholar/utils/cleanup/cleanup_old_extractions.py +117 -0
  547. scitex/scholar/utils/text/_TextNormalizer.py +407 -0
  548. scitex/scholar/utils/text/__init__.py +9 -0
  549. scitex/scholar/zotero/__init__.py +38 -0
  550. scitex/session/__init__.py +51 -0
  551. scitex/session/_lifecycle.py +736 -0
  552. scitex/session/_manager.py +102 -0
  553. scitex/session/template.py +122 -0
  554. scitex/stats/__init__.py +30 -26
  555. scitex/stats/correct/__init__.py +21 -0
  556. scitex/stats/correct/_correct_bonferroni.py +551 -0
  557. scitex/stats/correct/_correct_fdr.py +634 -0
  558. scitex/stats/correct/_correct_holm.py +548 -0
  559. scitex/stats/correct/_correct_sidak.py +499 -0
  560. scitex/stats/descriptive/__init__.py +85 -0
  561. scitex/stats/descriptive/_circular.py +540 -0
  562. scitex/stats/descriptive/_describe.py +219 -0
  563. scitex/stats/descriptive/_nan.py +518 -0
  564. scitex/stats/descriptive/_real.py +189 -0
  565. scitex/stats/effect_sizes/__init__.py +41 -0
  566. scitex/stats/effect_sizes/_cliffs_delta.py +325 -0
  567. scitex/stats/effect_sizes/_cohens_d.py +342 -0
  568. scitex/stats/effect_sizes/_epsilon_squared.py +315 -0
  569. scitex/stats/effect_sizes/_eta_squared.py +302 -0
  570. scitex/stats/effect_sizes/_prob_superiority.py +296 -0
  571. scitex/stats/posthoc/__init__.py +19 -0
  572. scitex/stats/posthoc/_dunnett.py +463 -0
  573. scitex/stats/posthoc/_games_howell.py +383 -0
  574. scitex/stats/posthoc/_tukey_hsd.py +367 -0
  575. scitex/stats/power/__init__.py +19 -0
  576. scitex/stats/power/_power.py +433 -0
  577. scitex/stats/template.py +119 -0
  578. scitex/stats/utils/__init__.py +62 -0
  579. scitex/stats/utils/_effect_size.py +985 -0
  580. scitex/stats/utils/_formatters.py +270 -0
  581. scitex/stats/utils/_normalizers.py +927 -0
  582. scitex/stats/utils/_power.py +433 -0
  583. scitex/stats_v01/_EffectSizeCalculator.py +488 -0
  584. scitex/stats_v01/_StatisticalValidator.py +411 -0
  585. scitex/stats_v01/__init__.py +60 -0
  586. scitex/stats_v01/_additional_tests.py +415 -0
  587. scitex/{stats → stats_v01}/_p2stars.py +19 -5
  588. scitex/stats_v01/_two_sample_tests.py +141 -0
  589. scitex/stats_v01/desc/__init__.py +83 -0
  590. scitex/stats_v01/desc/_circular.py +540 -0
  591. scitex/stats_v01/desc/_describe.py +219 -0
  592. scitex/stats_v01/desc/_nan.py +518 -0
  593. scitex/{stats/desc/_nan.py → stats_v01/desc/_nan_v01-20250920_145731.py} +23 -12
  594. scitex/stats_v01/desc/_real.py +189 -0
  595. scitex/stats_v01/tests/__corr_test_optimized.py +221 -0
  596. scitex/stats_v01/tests/_corr_test_optimized.py +179 -0
  597. scitex/str/__init__.py +1 -3
  598. scitex/str/_clean_path.py +6 -2
  599. scitex/str/_latex_fallback.py +267 -160
  600. scitex/str/_parse.py +44 -36
  601. scitex/str/_printc.py +1 -3
  602. scitex/template/__init__.py +87 -0
  603. scitex/template/_create_project.py +267 -0
  604. scitex/template/create_pip_project.py +80 -0
  605. scitex/template/create_research.py +80 -0
  606. scitex/template/create_singularity.py +80 -0
  607. scitex/units.py +291 -0
  608. scitex/utils/_compress_hdf5.py +14 -3
  609. scitex/utils/_email.py +21 -2
  610. scitex/utils/_grid.py +6 -4
  611. scitex/utils/_notify.py +13 -10
  612. scitex/utils/_verify_scitex_format.py +589 -0
  613. scitex/utils/_verify_scitex_format_v01.py +370 -0
  614. scitex/utils/template.py +122 -0
  615. scitex/web/_search_pubmed.py +62 -16
  616. scitex-2.1.0.dist-info/LICENSE +21 -0
  617. scitex-2.1.0.dist-info/METADATA +677 -0
  618. scitex-2.1.0.dist-info/RECORD +919 -0
  619. {scitex-2.0.0.dist-info → scitex-2.1.0.dist-info}/WHEEL +1 -1
  620. scitex-2.1.0.dist-info/entry_points.txt +3 -0
  621. scitex/ai/__Classifiers.py +0 -101
  622. scitex/ai/classification/classification_reporter.py +0 -1137
  623. scitex/ai/classification/classifiers.py +0 -101
  624. scitex/ai/classification_reporter.py +0 -1161
  625. scitex/ai/genai/__init__.py +0 -277
  626. scitex/ai/genai/anthropic_provider.py +0 -320
  627. scitex/ai/genai/anthropic_refactored.py +0 -109
  628. scitex/ai/genai/auth_manager.py +0 -200
  629. scitex/ai/genai/base_provider.py +0 -291
  630. scitex/ai/genai/chat_history.py +0 -307
  631. scitex/ai/genai/cost_tracker.py +0 -276
  632. scitex/ai/genai/deepseek_provider.py +0 -251
  633. scitex/ai/genai/google_provider.py +0 -228
  634. scitex/ai/genai/groq_provider.py +0 -248
  635. scitex/ai/genai/image_processor.py +0 -250
  636. scitex/ai/genai/llama_provider.py +0 -214
  637. scitex/ai/genai/mock_provider.py +0 -127
  638. scitex/ai/genai/model_registry.py +0 -304
  639. scitex/ai/genai/openai_provider.py +0 -293
  640. scitex/ai/genai/perplexity_provider.py +0 -205
  641. scitex/ai/genai/provider_base.py +0 -302
  642. scitex/ai/genai/provider_factory.py +0 -370
  643. scitex/ai/genai/response_handler.py +0 -235
  644. scitex/ai/layer/_Pass.py +0 -21
  645. scitex/ai/layer/__init__.py +0 -10
  646. scitex/ai/layer/_switch.py +0 -8
  647. scitex/ai/metrics/_bACC.py +0 -51
  648. scitex/ai/plt/_learning_curve.py +0 -194
  649. scitex/ai/plt/_optuna_study.py +0 -111
  650. scitex/ai/plt/aucs/__init__.py +0 -2
  651. scitex/ai/plt/aucs/example.py +0 -60
  652. scitex/ai/plt/aucs/pre_rec_auc.py +0 -223
  653. scitex/ai/plt/aucs/roc_auc.py +0 -246
  654. scitex/ai/sampling/undersample.py +0 -29
  655. scitex/db/_SQLite3.py +0 -2136
  656. scitex/db/_SQLite3Mixins/_BlobMixin.py +0 -229
  657. scitex/gen/_close.py +0 -222
  658. scitex/gen/_start.py +0 -451
  659. scitex/general/__init__.py +0 -5
  660. scitex/io/_load_modules/_db.py +0 -24
  661. scitex/life/__init__.py +0 -10
  662. scitex/life/_monitor_rain.py +0 -49
  663. scitex/reproduce/_fix_seeds.py +0 -45
  664. scitex/res/__init__.py +0 -5
  665. scitex/scholar/_local_search.py +0 -454
  666. scitex/scholar/_paper.py +0 -244
  667. scitex/scholar/_pdf_downloader.py +0 -325
  668. scitex/scholar/_search.py +0 -393
  669. scitex/scholar/_vector_search.py +0 -370
  670. scitex/scholar/_web_sources.py +0 -457
  671. scitex/stats/desc/__init__.py +0 -40
  672. scitex-2.0.0.dist-info/METADATA +0 -307
  673. scitex-2.0.0.dist-info/RECORD +0 -572
  674. scitex-2.0.0.dist-info/licenses/LICENSE +0 -7
  675. /scitex/ai/{act → activation}/__init__.py +0 -0
  676. /scitex/ai/{act → activation}/_define.py +0 -0
  677. /scitex/ai/{early_stopping.py → training/_EarlyStopping.py} +0 -0
  678. /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_ImportExportMixin.py +0 -0
  679. /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_IndexMixin.py +0 -0
  680. /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_RowMixin.py +0 -0
  681. /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_TableMixin.py +0 -0
  682. /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/__init__.py +0 -0
  683. /scitex/{stats → stats_v01}/_calc_partial_corr.py +0 -0
  684. /scitex/{stats → stats_v01}/_corr_test_multi.py +0 -0
  685. /scitex/{stats → stats_v01}/_corr_test_wrapper.py +0 -0
  686. /scitex/{stats → stats_v01}/_describe_wrapper.py +0 -0
  687. /scitex/{stats → stats_v01}/_multiple_corrections.py +0 -0
  688. /scitex/{stats → stats_v01}/_nan_stats.py +0 -0
  689. /scitex/{stats → stats_v01}/_p2stars_wrapper.py +0 -0
  690. /scitex/{stats → stats_v01}/_statistical_tests.py +0 -0
  691. /scitex/{stats/desc/_describe.py → stats_v01/desc/_describe_v01-20250920_145731.py} +0 -0
  692. /scitex/{stats/desc/_real.py → stats_v01/desc/_real_v01-20250920_145731.py} +0 -0
  693. /scitex/{stats → stats_v01}/multiple/__init__.py +0 -0
  694. /scitex/{stats → stats_v01}/multiple/_bonferroni_correction.py +0 -0
  695. /scitex/{stats → stats_v01}/multiple/_fdr_correction.py +0 -0
  696. /scitex/{stats → stats_v01}/multiple/_multicompair.py +0 -0
  697. /scitex/{stats → stats_v01}/tests/__corr_test.py +0 -0
  698. /scitex/{stats → stats_v01}/tests/__corr_test_multi.py +0 -0
  699. /scitex/{stats → stats_v01}/tests/__corr_test_single.py +0 -0
  700. /scitex/{stats → stats_v01}/tests/__init__.py +0 -0
  701. /scitex/{stats → stats_v01}/tests/_brunner_munzel_test.py +0 -0
  702. /scitex/{stats → stats_v01}/tests/_nocorrelation_test.py +0 -0
  703. /scitex/{stats → stats_v01}/tests/_smirnov_grubbs.py +0 -0
  704. {scitex-2.0.0.dist-info → scitex-2.1.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,1008 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ # Timestamp: "2025-10-10 00:50:44 (ywatanabe)"
4
+ # File: /home/ywatanabe/proj/scitex_repo/src/scitex/browser/stealth/StealthManager.py
5
+ # ----------------------------------------
6
+ from __future__ import annotations
7
+ import os
8
+ __FILE__ = (
9
+ "./src/scitex/browser/stealth/StealthManager.py"
10
+ )
11
+ __DIR__ = os.path.dirname(__FILE__)
12
+ # ----------------------------------------
13
+
14
+ __FILE__ = __file__
15
+
16
+ import asyncio
17
+ import random
18
+
19
+ from playwright.async_api import Browser, BrowserContext, Page
20
+
21
+ from scitex import logging
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+ # User Agent(Old) Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36
26
+ # WebDriver(New) missing (passed)
27
+ # WebDriver Advanced passed
28
+ # Chrome(New) present (passed)
29
+ # Permissions(New) prompt
30
+ # Plugins Length(Old) 5
31
+ # Plugins is of type PluginArray passed
32
+ # Languages(Old) en-US,en,ja
33
+ # WebGL Vendor Google Inc. (AMD)
34
+ # WebGL Renderer ANGLE (AMD, AMD Radeon(TM) Graphics (0x00001636) Direct3D11 vs_5_0 ps_5_0, D3D11)
35
+ # Broken Image Dimensions 16x16
36
+
37
+
38
+ class StealthManager:
39
+ def __init__(
40
+ self,
41
+ viewport_size: tuple = None,
42
+ spoof_dimension: bool = False,
43
+ ):
44
+ self.name = self.__class__.__name__
45
+ self.viewport_size = viewport_size
46
+ self.spoof_dimension = spoof_dimension
47
+
48
+ def get_random_user_agent(self) -> str:
49
+ user_agents = [
50
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
51
+ # "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
52
+ # "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
53
+ # "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
54
+ # "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
55
+ # "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
56
+ # "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
57
+ # "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
58
+ ]
59
+ user_agent = random.choice(user_agents)
60
+ logger.debug(
61
+ f"{self.name}: User Agent randomly selected: {user_agent}"
62
+ )
63
+ return user_agent
64
+
65
+ def get_random_viewport(self) -> dict:
66
+ if self.viewport_size:
67
+ viewport = {
68
+ "width": self.viewport_size[0],
69
+ "height": self.viewport_size[1],
70
+ }
71
+ logger.debug(
72
+ f"{self.name}: Viewport defined as specified in Stealth Manager initiation: {viewport}"
73
+ )
74
+ return viewport
75
+
76
+ if self.spoof_dimension:
77
+ # viewport = {"width": 1, "height": 1}
78
+ viewport = {"width": 1920, "height": 1080}
79
+ logger.debug(
80
+ f"{self.name}: Viewport defined as spoof_dimension passed during Stealth Manager initiation: {viewport}"
81
+ )
82
+ return viewport
83
+
84
+ else:
85
+ viewport = random.choice(
86
+ [
87
+ {"width": 1920, "height": 1080},
88
+ {"width": 1366, "height": 768},
89
+ {"width": 1440, "height": 900},
90
+ {"width": 1280, "height": 720},
91
+ ]
92
+ )
93
+ logger.debug(
94
+ f"{self.name}: Viewport randomly selected: {viewport}"
95
+ )
96
+ return viewport
97
+
98
+ def get_stealth_options(self) -> dict:
99
+ return {
100
+ "viewport": self.get_random_viewport(),
101
+ "user_agent": self.get_random_user_agent(),
102
+ "extra_http_headers": {
103
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
104
+ "Accept-Language": "en-US,en;q=0.9",
105
+ "Accept-Encoding": "gzip, deflate, br, zstd",
106
+ "Cache-Control": "max-age=0",
107
+ "Sec-Ch-Ua": '"Google Chrome";v="132", "Chromium";v="132", "Not_A Brand";v="24"',
108
+ "Sec-Ch-Ua-Mobile": "?0",
109
+ "Sec-Ch-Ua-Platform": '"Windows"',
110
+ "Sec-Fetch-Dest": "document",
111
+ "Sec-Fetch-Mode": "navigate",
112
+ "Sec-Fetch-Site": "none",
113
+ "Sec-Fetch-User": "?1",
114
+ "Upgrade-Insecure-Requests": "1",
115
+ "Referer": "https://www.google.com/",
116
+ },
117
+ "ignore_https_errors": True,
118
+ "java_script_enabled": True,
119
+ }
120
+
121
+ def get_stealth_options_additional(self) -> list:
122
+ stealth_args = [
123
+ # Core security and sandbox
124
+ "--no-sandbox",
125
+ "--disable-setuid-sandbox",
126
+ "--disable-dev-shm-usage",
127
+ # Critical automation detection bypass
128
+ "--disable-blink-features=AutomationControlled",
129
+ "--disable-features=UserAgentClientHint",
130
+ "--disable-features=WebRtcHideLocalIpsWithMdns",
131
+ "--disable-features=VizDisplayCompositor",
132
+ "--disable-features=TranslateUI",
133
+ "--disable-features=Translate",
134
+ "--disable-features=MediaRouter",
135
+ "--disable-features=OptimizationHints",
136
+ "--disable-features=AudioServiceOutOfProcess",
137
+ "--disable-features=VizServiceSharingEnabled",
138
+ # Enhanced fingerprinting resistance
139
+ "--disable-web-security",
140
+ "--disable-site-isolation-trials",
141
+ "--disable-cross-domain-blocking",
142
+ "--disable-features=CrossOriginOpenerPolicy",
143
+ "--disable-features=DocumentPolicy",
144
+ "--disable-features=OriginPolicy",
145
+ # Network and connectivity
146
+ "--disable-background-networking",
147
+ "--disable-client-side-phishing-detection",
148
+ "--disable-component-update",
149
+ "--disable-domain-reliability",
150
+ "--disable-background-mode",
151
+ "--disable-ipc-flooding-protection",
152
+ # Browser behavior normalization
153
+ "--disable-sync",
154
+ "--disable-translate",
155
+ "--disable-default-apps",
156
+ "--enable-extensions",
157
+ "--no-first-run",
158
+ "--no-default-browser-check",
159
+ "--disable-infobars",
160
+ "--disable-notifications",
161
+ # POPUP BLOCKING - Add these lines
162
+ "--block-new-web-contents",
163
+ "--disable-popup-blocking",
164
+ "--suppress-message-center-popups",
165
+ "--disable-session-crashed-bubble",
166
+ "--disable-features=UserAgentClientHint,TranslateSubFrames,AutofillServerCommunication",
167
+ # Performance and timing
168
+ "--disable-background-timer-throttling",
169
+ "--disable-backgrounding-occluded-windows",
170
+ "--disable-renderer-backgrounding",
171
+ "--disable-hang-monitor",
172
+ "--disable-plugins-discovery",
173
+ "--disable-field-trial-config",
174
+ # Media and hardware access
175
+ "--use-fake-ui-for-media-stream",
176
+ "--use-fake-device-for-media-stream",
177
+ "--autoplay-policy=user-gesture-required",
178
+ "--disable-audio-output",
179
+ # Logging and debugging
180
+ "--disable-logging",
181
+ "--disable-gpu-logging",
182
+ "--disable-dev-shm-usage",
183
+ "--disable-renderer-code-integrity",
184
+ # Memory optimization
185
+ "--memory-pressure-off",
186
+ "--max_old_space_size=4096",
187
+ "--disable-low-res-tiling",
188
+ "--disable-partial-raster",
189
+ "--disable-checker-imaging",
190
+ # TLS/SSL improvements
191
+ "--ignore-certificate-errors",
192
+ "--ignore-ssl-errors",
193
+ "--ignore-certificate-errors-spki-list",
194
+ "--disable-web-security",
195
+ # Additional anti-detection
196
+ "--disable-features=VizHitTestSurfaceLayer",
197
+ "--disable-features=TranslateSubFrames",
198
+ "--disable-search-engine-choice-screen",
199
+ "--disable-features=PrivacySandboxSettings4",
200
+ "--disable-features=AutofillServerCommunication",
201
+ "--enable-extensions",
202
+ ]
203
+
204
+ # Apply window size and position based on mode
205
+ if self.spoof_dimension:
206
+ # 1x1 window completely off-screen for true invisibility
207
+ stealth_args.extend(["--window-size=1,1", "--window-position=0,0"])
208
+ logger.debug(
209
+ f"{self.name}: Invisible mode: Window set to 1x1 at position 0,0 (off-screen)"
210
+ )
211
+ else:
212
+ # Standard window or custom size
213
+ if self.viewport_size:
214
+ stealth_args.append(
215
+ f"--window-size={self.viewport_size[0]},{self.viewport_size[1]}"
216
+ )
217
+ else:
218
+ stealth_args.append("--window-size=1920,1080")
219
+
220
+ if self.spoof_dimension:
221
+ config_desc = "Invisible (1x1)"
222
+ elif self.viewport_size:
223
+ config_desc = f"{self.viewport_size[0]}x{self.viewport_size[1]}"
224
+ else:
225
+ config_desc = "Default (1920x1080)"
226
+
227
+ logger.debug(
228
+ f"{self.name}: Browser window configuration: {config_desc}"
229
+ )
230
+ return stealth_args
231
+
232
+ def get_network_evasion_headers(self) -> dict:
233
+ """Generate realistic HTTP headers to avoid network-level detection."""
234
+ return {
235
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
236
+ "Accept-Encoding": "gzip, deflate, br, zstd",
237
+ "Accept-Language": "en-US,en;q=0.9",
238
+ "Cache-Control": "max-age=0",
239
+ "DNT": "1",
240
+ "Sec-Ch-Ua": '"Google Chrome";v="132", "Chromium";v="132", "Not_A Brand";v="24"',
241
+ "Sec-Ch-Ua-Mobile": "?0",
242
+ "Sec-Ch-Ua-Platform": '"Linux"',
243
+ "Sec-Ch-Ua-Platform-Version": '"5.15.0"',
244
+ "Sec-Fetch-Dest": "document",
245
+ "Sec-Fetch-Mode": "navigate",
246
+ "Sec-Fetch-Site": "none",
247
+ "Sec-Fetch-User": "?1",
248
+ "Upgrade-Insecure-Requests": "1",
249
+ "X-Forwarded-For": f"{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}",
250
+ "X-Real-IP": f"{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}",
251
+ }
252
+
253
+ async def add_human_behavior_async(self, page: Page):
254
+ """Add human-like behavior patterns to avoid detection."""
255
+ # Random delay before starting interactions
256
+ delay = random.uniform(2, 5)
257
+ logger.debug(
258
+ f"{self.name}: Adding human behavior delay: {delay:.2f} seconds"
259
+ )
260
+ await asyncio.sleep(delay)
261
+
262
+ # Simulate scrolling behavior
263
+ try:
264
+ await page.evaluate(
265
+ """
266
+ window.scrollTo({
267
+ top: Math.random() * 500,
268
+ behavior: 'smooth'
269
+ });
270
+ """
271
+ )
272
+ await asyncio.sleep(random.uniform(1, 3))
273
+ except Exception as e:
274
+ logger.debug(f"{self.name}: Human behavior simulation failed: {e}")
275
+
276
+ async def handle_cloudflare_challenge_async(
277
+ self, page: Page, max_wait: int = 45
278
+ ):
279
+ """Enhanced Cloudflare challenge detection and handling."""
280
+ logger.debug(f"{self.name}: Checking for Cloudflare challenge...")
281
+
282
+ cloudflare_indicators = [
283
+ "Just a moment",
284
+ "Checking your browser",
285
+ "DDoS protection by Cloudflare",
286
+ "Cloudflare Ray ID",
287
+ "cf-browser-verification",
288
+ "Please wait while we verify you're a human",
289
+ "Verify you are human",
290
+ "Security check",
291
+ "Browser verification",
292
+ "cf-challenge-running",
293
+ ]
294
+
295
+ try:
296
+ # First check if we're on a Cloudflare challenge page
297
+ page_content = await page.content()
298
+ title = await page.title()
299
+
300
+ is_challenge = any(
301
+ indicator.lower() in page_content.lower()
302
+ or indicator.lower() in title.lower()
303
+ for indicator in cloudflare_indicators
304
+ )
305
+
306
+ if not is_challenge:
307
+ logger.debug(f"{self.name}: No Cloudflare challenge detected")
308
+ return True
309
+
310
+ logger.debug(
311
+ f"{self.name}: Cloudflare challenge detected, waiting for completion..."
312
+ )
313
+
314
+ # Add human-like behavior during challenge
315
+ await self.add_human_behavior_async(page)
316
+
317
+ # Wait for challenge completion with multiple conditions
318
+ await page.wait_for_function(
319
+ """
320
+ () => {
321
+ const content = document.documentElement.innerText.toLowerCase();
322
+ const title = document.title.toLowerCase();
323
+
324
+ // Challenge completion indicators
325
+ const challengeComplete = !content.includes('just a moment') &&
326
+ !content.includes('checking your browser') &&
327
+ !content.includes('ddos protection') &&
328
+ !content.includes('please wait') &&
329
+ !content.includes('verify you are human') &&
330
+ !title.includes('just a moment');
331
+
332
+ // Also check if we've been redirected or if the URL changed
333
+ const urlChanged = window.location.href !== window.initialUrl;
334
+ window.initialUrl = window.initialUrl || window.location.href;
335
+
336
+ return challengeComplete || urlChanged;
337
+ }
338
+ """,
339
+ timeout=max_wait * 1000,
340
+ )
341
+
342
+ # Additional wait to ensure page is fully loaded
343
+ await asyncio.sleep(random.uniform(2, 4))
344
+
345
+ logger.debug(
346
+ f"{self.name}: Cloudflare challenge passed successfully"
347
+ )
348
+ return True
349
+
350
+ except Exception as e:
351
+ logger.warning(
352
+ f"{self.name}: Cloudflare challenge handling timeout or error: {e}"
353
+ )
354
+
355
+ # Try to detect if we're still on challenge page
356
+ try:
357
+ final_content = await page.content()
358
+ still_challenged = any(
359
+ indicator.lower() in final_content.lower()
360
+ for indicator in cloudflare_indicators
361
+ )
362
+
363
+ if still_challenged:
364
+ logger.error(
365
+ f"{self.name}: Still on Cloudflare challenge page after timeout"
366
+ )
367
+ return False
368
+ else:
369
+ logger.debug(
370
+ f"{self.name}: Challenge may have completed despite timeout"
371
+ )
372
+ return True
373
+
374
+ except:
375
+ return False
376
+
377
+ def get_init_script(self) -> str:
378
+ return """
379
+ (() => {
380
+ 'use strict';
381
+
382
+ // === CORE WEBDRIVER DETECTION REMOVAL ===
383
+ Object.defineProperty(navigator, 'webdriver', {
384
+ get: () => undefined,
385
+ configurable: true
386
+ });
387
+
388
+ // Remove all automation-related properties
389
+ const automationProps = [
390
+ 'webdriver', '__driver_evaluate', '__webdriver_evaluate', '__selenium_evaluate',
391
+ '__fxdriver_evaluate', '__driver_unwrapped', '__webdriver_unwrapped',
392
+ '__selenium_unwrapped', '__fxdriver_unwrapped', '__webdriver_script_function',
393
+ '__webdriver_script_func', '__webdriver_script_fn', '__fxdriver_script_fn',
394
+ '__selenium_script_fn', '__webdriver_func', '__webdriver_fn', '__$webdriverAsyncExecutor',
395
+ '__lastWatirAlert', '__lastWatirConfirm', '__lastWatirPrompt', '_WEBDRIVER_ELEM_CACHE',
396
+ 'ChromeDriverw', 'driver-evaluate', 'webdriver-evaluate', 'selenium-evaluate',
397
+ 'webdriverCommand', 'webdriver-evaluate-response', '__webdriverFunc', '__webdriver_script_func',
398
+ '__$webdriverAsyncExecutor', '$chrome_asyncScriptInfo', '$cdc_asdjflasutopfhvcZLmcfl_'
399
+ ];
400
+
401
+ automationProps.forEach(prop => {
402
+ try {
403
+ delete window[prop];
404
+ delete document[prop];
405
+ delete navigator[prop];
406
+ } catch (e) {}
407
+ });
408
+
409
+ // === NAVIGATOR PROPERTIES ===
410
+ // Mock realistic Chrome object with proper methods
411
+ if (!window.chrome || Object.getPrototypeOf(window.chrome) === Object.prototype) {
412
+ window.chrome = {
413
+ app: {
414
+ isInstalled: false,
415
+ InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
416
+ RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' }
417
+ },
418
+ runtime: {
419
+ onConnect: null,
420
+ onMessage: null,
421
+ onConnectExternal: null,
422
+ onMessageExternal: null,
423
+ connect: () => {},
424
+ sendMessage: () => {},
425
+ getManifest: () => ({ name: 'Chrome', version: '132.0.6834.59' }),
426
+ getURL: (path) => 'chrome-extension://invalid/' + path
427
+ },
428
+ webstore: {
429
+ onInstallStageChanged: null,
430
+ onDownloadProgress: null,
431
+ install: () => {}
432
+ },
433
+ csi: () => ({ pageT: Math.random() * 1000, startE: Math.random() * 1000 }),
434
+ loadTimes: () => ({
435
+ requestTime: performance.now() / 1000,
436
+ startLoadTime: performance.now() / 1000,
437
+ commitLoadTime: performance.now() / 1000,
438
+ finishDocumentLoadTime: performance.now() / 1000,
439
+ finishLoadTime: performance.now() / 1000,
440
+ firstPaintTime: performance.now() / 1000,
441
+ firstPaintAfterLoadTime: 0,
442
+ navigationType: 'Other'
443
+ })
444
+ };
445
+
446
+ // Make chrome object non-enumerable to match real Chrome
447
+ Object.defineProperty(window, 'chrome', {
448
+ value: window.chrome,
449
+ writable: false,
450
+ enumerable: false,
451
+ configurable: false
452
+ });
453
+ }
454
+
455
+ // === LANGUAGE AND LOCALE ===
456
+ Object.defineProperty(navigator, 'languages', {
457
+ get: () => ['en-US', 'en'],
458
+ configurable: true
459
+ });
460
+
461
+ Object.defineProperty(navigator, 'language', {
462
+ get: () => 'en-US',
463
+ configurable: true
464
+ });
465
+
466
+ // === REALISTIC PLUGINS ===
467
+ const mockPlugins = [
468
+ {
469
+ 0: { type: "application/x-google-chrome-pdf", suffixes: "pdf", description: "Portable Document Format", enabledPlugin: null },
470
+ description: "Portable Document Format",
471
+ filename: "internal-pdf-viewer",
472
+ length: 1,
473
+ name: "Chrome PDF Plugin"
474
+ },
475
+ {
476
+ 0: { type: "application/pdf", suffixes: "pdf", description: "Portable Document Format", enabledPlugin: null },
477
+ description: "Portable Document Format",
478
+ filename: "mhjfbmdgcfjbbpaeojofohoefgiehjai",
479
+ length: 1,
480
+ name: "Chrome PDF Viewer"
481
+ },
482
+ {
483
+ 0: { type: "application/x-nacl", suffixes: "", description: "Native Client Executable", enabledPlugin: null },
484
+ 1: { type: "application/x-pnacl", suffixes: "", description: "Portable Native Client Executable", enabledPlugin: null },
485
+ description: "Native Client",
486
+ filename: "internal-nacl-plugin",
487
+ length: 2,
488
+ name: "Native Client"
489
+ }
490
+ ];
491
+
492
+ Object.defineProperty(navigator, 'plugins', {
493
+ get: () => mockPlugins,
494
+ configurable: true
495
+ });
496
+
497
+ // === HARDWARE CONCURRENCY ===
498
+ Object.defineProperty(navigator, 'hardwareConcurrency', {
499
+ get: () => Math.max(2, Math.min(16, Math.floor(Math.random() * 8) + 4)),
500
+ configurable: true
501
+ });
502
+
503
+ // === PERMISSIONS API ===
504
+ if (navigator.permissions && navigator.permissions.query) {
505
+ const originalQuery = navigator.permissions.query.bind(navigator.permissions);
506
+ navigator.permissions.query = async (parameters) => {
507
+ const permission = parameters.name;
508
+ if (permission === 'notifications') {
509
+ return Promise.resolve({ state: 'default', onchange: null });
510
+ }
511
+ if (permission === 'geolocation') {
512
+ return Promise.resolve({ state: 'prompt', onchange: null });
513
+ }
514
+ try {
515
+ return await originalQuery(parameters);
516
+ } catch (e) {
517
+ return Promise.resolve({ state: 'prompt', onchange: null });
518
+ }
519
+ };
520
+ }
521
+
522
+ // === Camvas FINGERPRINTING PROTECTION ===
523
+ const getImageData = HTMLCanvasElement.prototype.toDataURL;
524
+ HTMLCanvasElement.prototype.toDataURL = function(type) {
525
+ const shift = Math.floor(Math.random() * 10) - 5;
526
+ const originalImageData = getImageData.apply(this, arguments);
527
+ return originalImageData;
528
+ };
529
+
530
+ // === WEBGL FINGERPRINTING PROTECTION ===
531
+ const glContexts = ['webgl', 'webgl2', 'experimental-webgl', 'experimental-webgl2'];
532
+
533
+ glContexts.forEach(contextType => {
534
+ try {
535
+ const canvas = document.createElement('canvas');
536
+ const gl = canvas.getContext(contextType);
537
+ if (gl) {
538
+ const getParameter = gl.getParameter.bind(gl);
539
+ gl.getParameter = function(parameter) {
540
+ // Vendor and renderer spoofing
541
+ if (parameter === gl.VENDOR || parameter === 37445) {
542
+ return 'Intel Inc.';
543
+ }
544
+ if (parameter === gl.RENDERER || parameter === 37446) {
545
+ return 'Intel Iris OpenGL Engine';
546
+ }
547
+ if (parameter === gl.VERSION) {
548
+ return 'OpenGL ES 2.0 (ANGLE 2.1.0.c8ea8ca4eb1a)';
549
+ }
550
+ if (parameter === gl.SHADING_LANGUAGE_VERSION) {
551
+ return 'OpenGL ES GLSL ES 1.0 (ANGLE 2.1.0.c8ea8ca4eb1a)';
552
+ }
553
+ return getParameter(parameter);
554
+ };
555
+ }
556
+ } catch (e) {}
557
+ });
558
+
559
+ // === CANVAS FINGERPRINTING PROTECTION ===
560
+ const getContext = HTMLCanvasElement.prototype.getContext;
561
+ HTMLCanvasElement.prototype.getContext = function(contextType, ...args) {
562
+ if (contextType === '2d') {
563
+ const context = getContext.call(this, contextType, ...args);
564
+ if (context) {
565
+ const originalFillText = context.fillText;
566
+ const originalStrokeText = context.strokeText;
567
+
568
+ context.fillText = function(...args) {
569
+ // Add slight noise to prevent consistent fingerprints
570
+ if (args.length >= 3) {
571
+ args[1] += Math.random() * 0.01 - 0.005;
572
+ args[2] += Math.random() * 0.01 - 0.005;
573
+ }
574
+ return originalFillText.apply(this, args);
575
+ };
576
+
577
+ context.strokeText = function(...args) {
578
+ if (args.length >= 3) {
579
+ args[1] += Math.random() * 0.01 - 0.005;
580
+ args[2] += Math.random() * 0.01 - 0.005;
581
+ }
582
+ return originalStrokeText.apply(this, args);
583
+ };
584
+ }
585
+ return context;
586
+ }
587
+ return getContext.call(this, contextType, ...args);
588
+ };
589
+
590
+ // === TIMEZONE AND LOCALE CONSISTENCY ===
591
+ if (Intl && Intl.DateTimeFormat) {
592
+ const originalResolvedOptions = Intl.DateTimeFormat.prototype.resolvedOptions;
593
+ Intl.DateTimeFormat.prototype.resolvedOptions = function() {
594
+ const options = originalResolvedOptions.call(this);
595
+ options.timeZone = 'America/New_York'; // Consistent timezone
596
+ return options;
597
+ };
598
+ }
599
+
600
+ // === SCREEN PROPERTIES ===
601
+ const screenProps = {
602
+ width: 1920,
603
+ height: 1080,
604
+ availWidth: 1920,
605
+ availHeight: 1040,
606
+ colorDepth: 24,
607
+ pixelDepth: 24,
608
+ orientation: {
609
+ angle: 0,
610
+ type: 'landscape-primary'
611
+ }
612
+ };
613
+
614
+ Object.keys(screenProps).forEach(prop => {
615
+ if (prop !== 'orientation') {
616
+ Object.defineProperty(screen, prop, {
617
+ get: () => screenProps[prop],
618
+ configurable: true
619
+ });
620
+ }
621
+ });
622
+
623
+ // === PREVENT TIMING ATTACKS ===
624
+ const originalNow = performance.now;
625
+ performance.now = function() {
626
+ return originalNow.call(this) + Math.random() * 0.1;
627
+ };
628
+
629
+ // === MEMORY INFO SPOOFING ===
630
+ if (performance.memory) {
631
+ Object.defineProperty(performance, 'memory', {
632
+ get: () => ({
633
+ jsHeapSizeLimit: 4294705152,
634
+ totalJSHeapSize: Math.floor(Math.random() * 50000000) + 10000000,
635
+ usedJSHeapSize: Math.floor(Math.random() * 30000000) + 5000000
636
+ }),
637
+ configurable: true
638
+ });
639
+ }
640
+
641
+ // === PREVENT IFRAME DETECTION ===
642
+ Object.defineProperty(window, 'top', {
643
+ get: () => window,
644
+ configurable: true
645
+ });
646
+
647
+ Object.defineProperty(window, 'parent', {
648
+ get: () => window,
649
+ configurable: true
650
+ });
651
+
652
+ // === CONSOLE CLEANING ===
653
+ const originalConsole = { ...console };
654
+ const cleanMethods = ['debug', 'log', 'info', 'warn', 'error'];
655
+ cleanMethods.forEach(method => {
656
+ console[method] = function(...args) {
657
+ const text = args.join(' ').toLowerCase();
658
+ if (text.includes('devtools') || text.includes('automation') ||
659
+ text.includes('webdriver') || text.includes('selenium') ||
660
+ text.includes('playwright') || text.includes('puppeteer')) {
661
+ return;
662
+ }
663
+ return originalConsole[method](...args);
664
+ };
665
+ });
666
+
667
+ // === MOUSE MOVEMENT SIMULATION ===
668
+ let mouseActivity = Date.now();
669
+ document.addEventListener('mousemove', () => {
670
+ mouseActivity = Date.now();
671
+ }, true);
672
+
673
+ // Simulate natural mouse movements
674
+ setInterval(() => {
675
+ if (Date.now() - mouseActivity > 30000) {
676
+ const event = new MouseEvent('mousemove', {
677
+ view: window,
678
+ bubbles: true,
679
+ cancelable: true,
680
+ clientX: Math.random() * window.innerWidth,
681
+ clientY: Math.random() * window.innerHeight
682
+ });
683
+ document.dispatchEvent(event);
684
+ }
685
+ }, 30000 + Math.random() * 15000);
686
+
687
+ // === FINAL CLEANUP ===
688
+ // Remove any remaining automation traces
689
+ delete window.cdc_adoQpoasnfa76pfcZLmcfl_;
690
+ delete window.$cdc_asdjflasutopfhvcZLmcfl_;
691
+ delete window.$chrome_asyncScriptInfo;
692
+ delete window.__$webdriverAsyncExecutor;
693
+
694
+ // Freeze important objects to prevent modification
695
+ try {
696
+ Object.freeze(navigator);
697
+ Object.freeze(screen);
698
+ } catch (e) {}
699
+ })();
700
+ """
701
+
702
+ def get_dimension_spoofing_script(self) -> str:
703
+ """
704
+ Generate comprehensive JavaScript dimension spoofing script for invisible browser mode.
705
+
706
+ This creates a dual-layer window configuration:
707
+ - Physical window: 1x1 pixel (invisible to user)
708
+ - Reported dimensions: 1920x1080 (natural desktop size for bot detection)
709
+
710
+ The script is bulletproof and handles all dimension-related APIs that
711
+ bot detectors commonly check.
712
+ """
713
+ logger.debug(
714
+ f"{self.name}: stealth_manager.get_dimension_spoofing_script called."
715
+ )
716
+ if not self.spoof_dimension:
717
+ return ""
718
+
719
+ return """
720
+ (() => {
721
+ // Target dimensions to report to JavaScript (natural desktop)
722
+ const TARGET_WINDOW_WIDTH = 1920;
723
+ const TARGET_WINDOW_HEIGHT = 1080;
724
+ const TARGET_SCREEN_WIDTH = 1920;
725
+ const TARGET_SCREEN_HEIGHT = 1080;
726
+ const TARGET_AVAILABLE_WIDTH = 1920;
727
+ const TARGET_AVAILABLE_HEIGHT = 1040; // Account for taskbar
728
+
729
+ // === WINDOW DIMENSIONS ===
730
+ // Override all window size properties
731
+ Object.defineProperty(window, 'innerWidth', {
732
+ get: () => TARGET_WINDOW_WIDTH,
733
+ configurable: true
734
+ });
735
+
736
+ Object.defineProperty(window, 'innerHeight', {
737
+ get: () => TARGET_WINDOW_HEIGHT,
738
+ configurable: true
739
+ });
740
+
741
+ Object.defineProperty(window, 'outerWidth', {
742
+ get: () => TARGET_WINDOW_WIDTH,
743
+ configurable: true
744
+ });
745
+
746
+ Object.defineProperty(window, 'outerHeight', {
747
+ get: () => TARGET_WINDOW_HEIGHT + 100, // Account for browser chrome
748
+ configurable: true
749
+ });
750
+
751
+ // Override client dimensions (commonly checked by bot detectors)
752
+ if (document.documentElement) {
753
+ Object.defineProperty(document.documentElement, 'clientWidth', {
754
+ get: () => TARGET_WINDOW_WIDTH,
755
+ configurable: true
756
+ });
757
+
758
+ Object.defineProperty(document.documentElement, 'clientHeight', {
759
+ get: () => TARGET_WINDOW_HEIGHT,
760
+ configurable: true
761
+ });
762
+ }
763
+
764
+ // === SCREEN DIMENSIONS ===
765
+ // Override all screen properties
766
+ Object.defineProperty(window.screen, 'width', {
767
+ get: () => TARGET_SCREEN_WIDTH,
768
+ configurable: true
769
+ });
770
+
771
+ Object.defineProperty(window.screen, 'height', {
772
+ get: () => TARGET_SCREEN_HEIGHT,
773
+ configurable: true
774
+ });
775
+
776
+ Object.defineProperty(window.screen, 'availWidth', {
777
+ get: () => TARGET_AVAILABLE_WIDTH,
778
+ configurable: true
779
+ });
780
+
781
+ Object.defineProperty(window.screen, 'availHeight', {
782
+ get: () => TARGET_AVAILABLE_HEIGHT,
783
+ configurable: true
784
+ });
785
+
786
+ // === VIEWPORT AND VISUAL DIMENSIONS ===
787
+ // Override visual viewport (modern API)
788
+ if (window.visualViewport) {
789
+ Object.defineProperty(window.visualViewport, 'width', {
790
+ get: () => TARGET_WINDOW_WIDTH,
791
+ configurable: true
792
+ });
793
+
794
+ Object.defineProperty(window.visualViewport, 'height', {
795
+ get: () => TARGET_WINDOW_HEIGHT,
796
+ configurable: true
797
+ });
798
+ }
799
+
800
+ // === DOCUMENT DIMENSIONS ===
801
+ // Override document element dimensions (wait for DOM to be ready)
802
+ const overrideDocumentDimensions = () => {
803
+ if (document.documentElement) {
804
+ Object.defineProperty(document.documentElement, 'clientWidth', {
805
+ get: () => TARGET_WINDOW_WIDTH,
806
+ configurable: true
807
+ });
808
+
809
+ Object.defineProperty(document.documentElement, 'clientHeight', {
810
+ get: () => TARGET_WINDOW_HEIGHT,
811
+ configurable: true
812
+ });
813
+
814
+ Object.defineProperty(document.documentElement, 'offsetWidth', {
815
+ get: () => TARGET_WINDOW_WIDTH,
816
+ configurable: true
817
+ });
818
+
819
+ Object.defineProperty(document.documentElement, 'offsetHeight', {
820
+ get: () => TARGET_WINDOW_HEIGHT,
821
+ configurable: true
822
+ });
823
+
824
+ Object.defineProperty(document.documentElement, 'scrollWidth', {
825
+ get: () => TARGET_WINDOW_WIDTH,
826
+ configurable: true
827
+ });
828
+
829
+ Object.defineProperty(document.documentElement, 'scrollHeight', {
830
+ get: () => TARGET_WINDOW_HEIGHT,
831
+ configurable: true
832
+ });
833
+ }
834
+
835
+ if (document.body) {
836
+ Object.defineProperty(document.body, 'clientWidth', {
837
+ get: () => TARGET_WINDOW_WIDTH,
838
+ configurable: true
839
+ });
840
+
841
+ Object.defineProperty(document.body, 'clientHeight', {
842
+ get: () => TARGET_WINDOW_HEIGHT,
843
+ configurable: true
844
+ });
845
+ }
846
+ };
847
+
848
+ // Apply immediately if DOM is ready, otherwise wait
849
+ if (document.readyState === 'loading') {
850
+ document.addEventListener('DOMContentLoaded', overrideDocumentDimensions);
851
+ } else {
852
+ overrideDocumentDimensions();
853
+ }
854
+
855
+ // === MEDIA QUERIES ===
856
+ // Override matchMedia for responsive design queries
857
+ const originalMatchMedia = window.matchMedia;
858
+ window.matchMedia = function(query) {
859
+ const result = originalMatchMedia.call(this, query);
860
+
861
+ // Override common responsive breakpoints based on our spoofed dimensions
862
+ if (query.includes('max-width')) {
863
+ const maxWidth = parseInt(query.match(/max-width:\\s*(\d+)px/)?.[1] || '0');
864
+ if (maxWidth < TARGET_WINDOW_WIDTH) {
865
+ Object.defineProperty(result, 'matches', { get: () => false });
866
+ }
867
+ }
868
+
869
+ if (query.includes('min-width')) {
870
+ const minWidth = parseInt(query.match(/min-width:\\s*(\d+)px/)?.[1] || '0');
871
+ if (minWidth <= TARGET_WINDOW_WIDTH) {
872
+ Object.defineProperty(result, 'matches', { get: () => true });
873
+ }
874
+ }
875
+
876
+ return result;
877
+ };
878
+
879
+ // === EVENT HANDLING ===
880
+ // Override resize events to maintain consistency
881
+ const originalAddEventListener = window.addEventListener;
882
+ window.addEventListener = function(type, listener, options) {
883
+ if (type === 'resize') {
884
+ // Intercept resize events and provide spoofed dimensions
885
+ const wrappedListener = function(event) {
886
+ // Create a mock resize event with spoofed dimensions
887
+ const mockEvent = new Event('resize');
888
+ Object.defineProperty(mockEvent, 'target', {
889
+ value: {
890
+ innerWidth: TARGET_WINDOW_WIDTH,
891
+ innerHeight: TARGET_WINDOW_HEIGHT
892
+ }
893
+ });
894
+ return listener.call(this, mockEvent);
895
+ };
896
+ return originalAddEventListener.call(this, type, wrappedListener, options);
897
+ }
898
+ return originalAddEventListener.call(this, type, listener, options);
899
+ };
900
+ })();
901
+ """
902
+
903
+ async def human_delay_async(self, min_ms: int = 1000, max_ms: int = 3000):
904
+ delay = random.randint(min_ms, max_ms)
905
+ await asyncio.sleep(delay / 1000)
906
+
907
+ async def human_click_async(self, page: Page, element):
908
+ await element.hover()
909
+ await self.human_delay_async(200, 500)
910
+ await element.click()
911
+
912
+ async def human_mouse_move_async(self, page: Page):
913
+ await page.mouse.move(
914
+ random.randint(100, 800), random.randint(100, 600)
915
+ )
916
+
917
+ async def human_scroll_async(self, page: Page):
918
+ scroll_distance = random.randint(300, 800)
919
+ await page.evaluate(f"window.scrollBy(0, {scroll_distance})")
920
+ await self.human_delay_async(500, 1500)
921
+
922
+ async def human_type_async(self, page: Page, selector: str, text: str):
923
+ element = page.locator(selector)
924
+ await element.click()
925
+ for char in text:
926
+ await element.type(char)
927
+ await self.human_delay_async(50, 200)
928
+
929
+
930
+ def main(args):
931
+ """Demonstrate StealthManager functionality."""
932
+ import asyncio
933
+
934
+ from playwright.async_api import async_playwright
935
+
936
+ async def demo():
937
+ stealth_manager = StealthManager()
938
+
939
+ async with async_playwright() as p:
940
+ browser = await p.chromium.launch(
941
+ headless=False,
942
+ args=stealth_manager.get_stealth_options_additional(),
943
+ )
944
+
945
+ stealth_options = stealth_manager.get_stealth_options()
946
+ context = await browser.new_context(**stealth_options)
947
+ await context.add_init_script(stealth_manager.get_init_script())
948
+
949
+ page = await context.new_page()
950
+ await page.goto("https://bot.sannysoft.com/", timeout=30000)
951
+ await stealth_manager.human_delay_async(2000, 3000)
952
+
953
+ await page.screenshot(path="/tmp/stealth_test.png")
954
+ print("✓ Stealth test complete: /tmp/stealth_test.png")
955
+
956
+ await browser.close()
957
+
958
+ asyncio.run(demo())
959
+ return 0
960
+
961
+
962
+ def parse_args():
963
+ """Parse command line arguments."""
964
+ import argparse
965
+
966
+ parser = argparse.ArgumentParser(description="StealthManager demo")
967
+ return parser.parse_args()
968
+
969
+
970
+ def run_main() -> None:
971
+ """Initialize scitex framework, run main function, and cleanup."""
972
+ global CONFIG, CC, sys, plt, rng
973
+
974
+ import sys
975
+
976
+ import matplotlib.pyplot as plt
977
+
978
+ import scitex as stx
979
+
980
+ args = parse_args()
981
+
982
+ CONFIG, sys.stdout, sys.stderr, plt, CC, rng = stx.session.start(
983
+ sys,
984
+ plt,
985
+ args=args,
986
+ file=__FILE__,
987
+ sdir_suffix=None,
988
+ verbose=False,
989
+ agg=True,
990
+ )
991
+
992
+ exit_status = main(args)
993
+
994
+ stx.session.close(
995
+ CONFIG,
996
+ verbose=False,
997
+ notify=False,
998
+ message="",
999
+ exit_status=exit_status,
1000
+ )
1001
+
1002
+
1003
+ if __name__ == "__main__":
1004
+ run_main()
1005
+
1006
+ # python -m scitex.browser.stealth.StealthManager
1007
+
1008
+ # EOF