scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scitex/__init__.py +53 -15
- scitex/__main__.py +72 -26
- scitex/__version__.py +1 -1
- scitex/_sh.py +145 -23
- scitex/ai/__init__.py +30 -16
- scitex/ai/_gen_ai/_Anthropic.py +5 -7
- scitex/ai/_gen_ai/_BaseGenAI.py +2 -2
- scitex/ai/_gen_ai/_DeepSeek.py +10 -2
- scitex/ai/_gen_ai/_Google.py +2 -2
- scitex/ai/_gen_ai/_Llama.py +2 -2
- scitex/ai/_gen_ai/_OpenAI.py +2 -2
- scitex/ai/_gen_ai/_PARAMS.py +51 -65
- scitex/ai/_gen_ai/_Perplexity.py +2 -2
- scitex/ai/_gen_ai/__init__.py +25 -14
- scitex/ai/_gen_ai/_format_output_func.py +4 -4
- scitex/ai/classification/{classifier_server.py → Classifier.py} +5 -5
- scitex/ai/classification/CrossValidationExperiment.py +374 -0
- scitex/ai/classification/__init__.py +43 -4
- scitex/ai/classification/reporters/_BaseClassificationReporter.py +281 -0
- scitex/ai/classification/reporters/_ClassificationReporter.py +773 -0
- scitex/ai/classification/reporters/_MultiClassificationReporter.py +406 -0
- scitex/ai/classification/reporters/_SingleClassificationReporter.py +1834 -0
- scitex/ai/classification/reporters/__init__.py +11 -0
- scitex/ai/classification/reporters/reporter_utils/_Plotter.py +1028 -0
- scitex/ai/classification/reporters/reporter_utils/__init__.py +80 -0
- scitex/ai/classification/reporters/reporter_utils/aggregation.py +457 -0
- scitex/ai/classification/reporters/reporter_utils/data_models.py +313 -0
- scitex/ai/classification/reporters/reporter_utils/reporting.py +1056 -0
- scitex/ai/classification/reporters/reporter_utils/storage.py +221 -0
- scitex/ai/classification/reporters/reporter_utils/validation.py +395 -0
- scitex/ai/classification/timeseries/_TimeSeriesBlockingSplit.py +568 -0
- scitex/ai/classification/timeseries/_TimeSeriesCalendarSplit.py +688 -0
- scitex/ai/classification/timeseries/_TimeSeriesMetadata.py +139 -0
- scitex/ai/classification/timeseries/_TimeSeriesSlidingWindowSplit.py +1716 -0
- scitex/ai/classification/timeseries/_TimeSeriesSlidingWindowSplit_v01-not-using-n_splits.py +1685 -0
- scitex/ai/classification/timeseries/_TimeSeriesStrategy.py +84 -0
- scitex/ai/classification/timeseries/_TimeSeriesStratifiedSplit.py +610 -0
- scitex/ai/classification/timeseries/__init__.py +39 -0
- scitex/ai/classification/timeseries/_normalize_timestamp.py +436 -0
- scitex/ai/clustering/_umap.py +2 -2
- scitex/ai/feature_extraction/vit.py +1 -0
- scitex/ai/feature_selection/__init__.py +30 -0
- scitex/ai/feature_selection/feature_selection.py +364 -0
- scitex/ai/loss/multi_task_loss.py +1 -1
- scitex/ai/metrics/__init__.py +51 -4
- scitex/ai/metrics/_calc_bacc.py +61 -0
- scitex/ai/metrics/_calc_bacc_from_conf_mat.py +38 -0
- scitex/ai/metrics/_calc_clf_report.py +78 -0
- scitex/ai/metrics/_calc_conf_mat.py +93 -0
- scitex/ai/metrics/_calc_feature_importance.py +183 -0
- scitex/ai/metrics/_calc_mcc.py +61 -0
- scitex/ai/metrics/_calc_pre_rec_auc.py +116 -0
- scitex/ai/metrics/_calc_roc_auc.py +110 -0
- scitex/ai/metrics/_calc_seizure_prediction_metrics.py +490 -0
- scitex/ai/metrics/{silhoute_score_block.py → _calc_silhouette_score.py} +15 -8
- scitex/ai/metrics/_normalize_labels.py +83 -0
- scitex/ai/plt/__init__.py +47 -8
- scitex/ai/plt/{_conf_mat.py → _plot_conf_mat.py} +158 -87
- scitex/ai/plt/_plot_feature_importance.py +323 -0
- scitex/ai/plt/_plot_learning_curve.py +345 -0
- scitex/ai/plt/_plot_optuna_study.py +225 -0
- scitex/ai/plt/_plot_pre_rec_curve.py +290 -0
- scitex/ai/plt/_plot_roc_curve.py +255 -0
- scitex/ai/training/{learning_curve_logger.py → _LearningCurveLogger.py} +197 -213
- scitex/ai/training/__init__.py +2 -2
- scitex/ai/utils/grid_search.py +3 -3
- scitex/benchmark/__init__.py +52 -0
- scitex/benchmark/benchmark.py +400 -0
- scitex/benchmark/monitor.py +370 -0
- scitex/benchmark/profiler.py +297 -0
- scitex/browser/__init__.py +48 -0
- scitex/browser/automation/CookieHandler.py +216 -0
- scitex/browser/automation/__init__.py +7 -0
- scitex/browser/collaboration/__init__.py +55 -0
- scitex/browser/collaboration/auth_helpers.py +94 -0
- scitex/browser/collaboration/collaborative_agent.py +136 -0
- scitex/browser/collaboration/credential_manager.py +188 -0
- scitex/browser/collaboration/interactive_panel.py +400 -0
- scitex/browser/collaboration/persistent_browser.py +170 -0
- scitex/browser/collaboration/shared_session.py +383 -0
- scitex/browser/collaboration/standard_interactions.py +246 -0
- scitex/browser/collaboration/visual_feedback.py +181 -0
- scitex/browser/core/BrowserMixin.py +326 -0
- scitex/browser/core/ChromeProfileManager.py +446 -0
- scitex/browser/core/__init__.py +9 -0
- scitex/browser/debugging/__init__.py +18 -0
- scitex/browser/debugging/_browser_logger.py +657 -0
- scitex/browser/debugging/_highlight_element.py +143 -0
- scitex/browser/debugging/_show_grid.py +154 -0
- scitex/browser/interaction/__init__.py +24 -0
- scitex/browser/interaction/click_center.py +149 -0
- scitex/browser/interaction/click_with_fallbacks.py +206 -0
- scitex/browser/interaction/close_popups.py +498 -0
- scitex/browser/interaction/fill_with_fallbacks.py +209 -0
- scitex/browser/pdf/__init__.py +14 -0
- scitex/browser/pdf/click_download_for_chrome_pdf_viewer.py +200 -0
- scitex/browser/pdf/detect_chrome_pdf_viewer.py +198 -0
- scitex/browser/remote/CaptchaHandler.py +434 -0
- scitex/browser/remote/ZenRowsAPIClient.py +347 -0
- scitex/browser/remote/ZenRowsBrowserManager.py +570 -0
- scitex/browser/remote/__init__.py +11 -0
- scitex/browser/stealth/HumanBehavior.py +344 -0
- scitex/browser/stealth/StealthManager.py +1008 -0
- scitex/browser/stealth/__init__.py +9 -0
- scitex/browser/template.py +122 -0
- scitex/capture/__init__.py +110 -0
- scitex/capture/__main__.py +25 -0
- scitex/capture/capture.py +848 -0
- scitex/capture/cli.py +233 -0
- scitex/capture/gif.py +344 -0
- scitex/capture/mcp_server.py +961 -0
- scitex/capture/session.py +70 -0
- scitex/capture/utils.py +705 -0
- scitex/cli/__init__.py +17 -0
- scitex/cli/cloud.py +447 -0
- scitex/cli/main.py +42 -0
- scitex/cli/scholar.py +280 -0
- scitex/context/_suppress_output.py +5 -3
- scitex/db/__init__.py +30 -3
- scitex/db/__main__.py +75 -0
- scitex/db/_check_health.py +381 -0
- scitex/db/_delete_duplicates.py +25 -386
- scitex/db/_inspect.py +335 -114
- scitex/db/_inspect_optimized.py +301 -0
- scitex/db/{_PostgreSQL.py → _postgresql/_PostgreSQL.py} +3 -3
- scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_BackupMixin.py +1 -1
- scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_BatchMixin.py +1 -1
- scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_BlobMixin.py +1 -1
- scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_ConnectionMixin.py +1 -1
- scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_MaintenanceMixin.py +1 -1
- scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_QueryMixin.py +1 -1
- scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_SchemaMixin.py +1 -1
- scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_TransactionMixin.py +1 -1
- scitex/db/_postgresql/__init__.py +6 -0
- scitex/db/_sqlite3/_SQLite3.py +210 -0
- scitex/db/_sqlite3/_SQLite3Mixins/_ArrayMixin.py +581 -0
- scitex/db/_sqlite3/_SQLite3Mixins/_ArrayMixin_v01-need-_hash-col.py +517 -0
- scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_BatchMixin.py +1 -1
- scitex/db/_sqlite3/_SQLite3Mixins/_BlobMixin.py +281 -0
- scitex/db/_sqlite3/_SQLite3Mixins/_ColumnMixin.py +548 -0
- scitex/db/_sqlite3/_SQLite3Mixins/_ColumnMixin_v01-indentation-issues.py +583 -0
- scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_ConnectionMixin.py +29 -13
- scitex/db/_sqlite3/_SQLite3Mixins/_GitMixin.py +583 -0
- scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_ImportExportMixin.py +1 -1
- scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_IndexMixin.py +1 -1
- scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_MaintenanceMixin.py +2 -1
- scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_QueryMixin.py +37 -10
- scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_RowMixin.py +46 -6
- scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_TableMixin.py +56 -10
- scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/_TransactionMixin.py +1 -1
- scitex/db/{_SQLite3Mixins → _sqlite3/_SQLite3Mixins}/__init__.py +14 -2
- scitex/db/_sqlite3/__init__.py +7 -0
- scitex/db/_sqlite3/_delete_duplicates.py +274 -0
- scitex/decorators/__init__.py +2 -0
- scitex/decorators/_cache_disk.py +13 -5
- scitex/decorators/_cache_disk_async.py +49 -0
- scitex/decorators/_deprecated.py +175 -10
- scitex/decorators/_timeout.py +1 -1
- scitex/dev/_analyze_code_flow.py +2 -2
- scitex/dict/_DotDict.py +73 -15
- scitex/dict/_DotDict_v01-not-handling-recursive-instantiations.py +442 -0
- scitex/dict/_DotDict_v02-not-serializing-Path-object.py +446 -0
- scitex/dict/__init__.py +2 -0
- scitex/dict/_flatten.py +27 -0
- scitex/dsp/_crop.py +2 -2
- scitex/dsp/_demo_sig.py +2 -2
- scitex/dsp/_detect_ripples.py +2 -2
- scitex/dsp/_hilbert.py +2 -2
- scitex/dsp/_listen.py +6 -6
- scitex/dsp/_modulation_index.py +2 -2
- scitex/dsp/_pac.py +1 -1
- scitex/dsp/_psd.py +2 -2
- scitex/dsp/_resample.py +2 -1
- scitex/dsp/_time.py +3 -2
- scitex/dsp/_wavelet.py +3 -2
- scitex/dsp/add_noise.py +2 -2
- scitex/dsp/example.py +1 -0
- scitex/dsp/filt.py +10 -9
- scitex/dsp/template.py +3 -2
- scitex/dsp/utils/_differential_bandpass_filters.py +1 -1
- scitex/dsp/utils/pac.py +2 -2
- scitex/dt/_normalize_timestamp.py +432 -0
- scitex/errors.py +572 -0
- scitex/gen/_DimHandler.py +2 -2
- scitex/gen/__init__.py +37 -7
- scitex/gen/_deprecated_close.py +80 -0
- scitex/gen/_deprecated_start.py +26 -0
- scitex/gen/_detect_environment.py +152 -0
- scitex/gen/_detect_notebook_path.py +169 -0
- scitex/gen/_embed.py +6 -2
- scitex/gen/_get_notebook_path.py +257 -0
- scitex/gen/_less.py +1 -1
- scitex/gen/_list_packages.py +2 -2
- scitex/gen/_norm.py +44 -9
- scitex/gen/_norm_cache.py +269 -0
- scitex/gen/_src.py +3 -5
- scitex/gen/_title_case.py +3 -3
- scitex/io/__init__.py +28 -6
- scitex/io/_glob.py +13 -7
- scitex/io/_load.py +108 -21
- scitex/io/_load_cache.py +303 -0
- scitex/io/_load_configs.py +40 -15
- scitex/io/{_H5Explorer.py → _load_modules/_H5Explorer.py} +80 -17
- scitex/io/_load_modules/_ZarrExplorer.py +114 -0
- scitex/io/_load_modules/_bibtex.py +207 -0
- scitex/io/_load_modules/_hdf5.py +53 -178
- scitex/io/_load_modules/_json.py +5 -3
- scitex/io/_load_modules/_pdf.py +871 -16
- scitex/io/_load_modules/_sqlite3.py +15 -0
- scitex/io/_load_modules/_txt.py +41 -12
- scitex/io/_load_modules/_yaml.py +4 -3
- scitex/io/_load_modules/_zarr.py +126 -0
- scitex/io/_save.py +429 -171
- scitex/io/_save_modules/__init__.py +6 -0
- scitex/io/_save_modules/_bibtex.py +194 -0
- scitex/io/_save_modules/_csv.py +8 -4
- scitex/io/_save_modules/_excel.py +174 -15
- scitex/io/_save_modules/_hdf5.py +251 -226
- scitex/io/_save_modules/_image.py +1 -3
- scitex/io/_save_modules/_json.py +49 -4
- scitex/io/_save_modules/_listed_dfs_as_csv.py +1 -3
- scitex/io/_save_modules/_listed_scalars_as_csv.py +1 -3
- scitex/io/_save_modules/_tex.py +277 -0
- scitex/io/_save_modules/_yaml.py +42 -3
- scitex/io/_save_modules/_zarr.py +160 -0
- scitex/io/utils/__init__.py +20 -0
- scitex/io/utils/h5_to_zarr.py +616 -0
- scitex/linalg/_geometric_median.py +6 -2
- scitex/{gen/_tee.py → logging/_Tee.py} +43 -84
- scitex/logging/__init__.py +122 -0
- scitex/logging/_config.py +158 -0
- scitex/logging/_context.py +103 -0
- scitex/logging/_formatters.py +128 -0
- scitex/logging/_handlers.py +64 -0
- scitex/logging/_levels.py +35 -0
- scitex/logging/_logger.py +163 -0
- scitex/logging/_print_capture.py +95 -0
- scitex/ml/__init__.py +69 -0
- scitex/{ai/genai/anthropic.py → ml/_gen_ai/_Anthropic.py} +13 -19
- scitex/{ai/genai/base_genai.py → ml/_gen_ai/_BaseGenAI.py} +5 -5
- scitex/{ai/genai/deepseek.py → ml/_gen_ai/_DeepSeek.py} +11 -16
- scitex/{ai/genai/google.py → ml/_gen_ai/_Google.py} +7 -15
- scitex/{ai/genai/groq.py → ml/_gen_ai/_Groq.py} +1 -8
- scitex/{ai/genai/llama.py → ml/_gen_ai/_Llama.py} +3 -16
- scitex/{ai/genai/openai.py → ml/_gen_ai/_OpenAI.py} +3 -3
- scitex/{ai/genai/params.py → ml/_gen_ai/_PARAMS.py} +51 -65
- scitex/{ai/genai/perplexity.py → ml/_gen_ai/_Perplexity.py} +3 -14
- scitex/ml/_gen_ai/__init__.py +43 -0
- scitex/{ai/genai/calc_cost.py → ml/_gen_ai/_calc_cost.py} +1 -1
- scitex/{ai/genai/format_output_func.py → ml/_gen_ai/_format_output_func.py} +4 -4
- scitex/{ai/genai/genai_factory.py → ml/_gen_ai/_genai_factory.py} +8 -8
- scitex/ml/activation/__init__.py +8 -0
- scitex/ml/activation/_define.py +11 -0
- scitex/{ai/classifier_server.py → ml/classification/Classifier.py} +5 -5
- scitex/ml/classification/CrossValidationExperiment.py +374 -0
- scitex/ml/classification/__init__.py +46 -0
- scitex/ml/classification/reporters/_BaseClassificationReporter.py +281 -0
- scitex/ml/classification/reporters/_ClassificationReporter.py +773 -0
- scitex/ml/classification/reporters/_MultiClassificationReporter.py +406 -0
- scitex/ml/classification/reporters/_SingleClassificationReporter.py +1834 -0
- scitex/ml/classification/reporters/__init__.py +11 -0
- scitex/ml/classification/reporters/reporter_utils/_Plotter.py +1028 -0
- scitex/ml/classification/reporters/reporter_utils/__init__.py +80 -0
- scitex/ml/classification/reporters/reporter_utils/aggregation.py +457 -0
- scitex/ml/classification/reporters/reporter_utils/data_models.py +313 -0
- scitex/ml/classification/reporters/reporter_utils/reporting.py +1056 -0
- scitex/ml/classification/reporters/reporter_utils/storage.py +221 -0
- scitex/ml/classification/reporters/reporter_utils/validation.py +395 -0
- scitex/ml/classification/timeseries/_TimeSeriesBlockingSplit.py +568 -0
- scitex/ml/classification/timeseries/_TimeSeriesCalendarSplit.py +688 -0
- scitex/ml/classification/timeseries/_TimeSeriesMetadata.py +139 -0
- scitex/ml/classification/timeseries/_TimeSeriesSlidingWindowSplit.py +1716 -0
- scitex/ml/classification/timeseries/_TimeSeriesSlidingWindowSplit_v01-not-using-n_splits.py +1685 -0
- scitex/ml/classification/timeseries/_TimeSeriesStrategy.py +84 -0
- scitex/ml/classification/timeseries/_TimeSeriesStratifiedSplit.py +610 -0
- scitex/ml/classification/timeseries/__init__.py +39 -0
- scitex/ml/classification/timeseries/_normalize_timestamp.py +436 -0
- scitex/ml/clustering/__init__.py +11 -0
- scitex/ml/clustering/_pca.py +115 -0
- scitex/ml/clustering/_umap.py +376 -0
- scitex/ml/feature_extraction/__init__.py +56 -0
- scitex/ml/feature_extraction/vit.py +149 -0
- scitex/ml/feature_selection/__init__.py +30 -0
- scitex/ml/feature_selection/feature_selection.py +364 -0
- scitex/ml/loss/_L1L2Losses.py +34 -0
- scitex/ml/loss/__init__.py +12 -0
- scitex/ml/loss/multi_task_loss.py +47 -0
- scitex/ml/metrics/__init__.py +56 -0
- scitex/ml/metrics/_calc_bacc.py +61 -0
- scitex/ml/metrics/_calc_bacc_from_conf_mat.py +38 -0
- scitex/ml/metrics/_calc_clf_report.py +78 -0
- scitex/ml/metrics/_calc_conf_mat.py +93 -0
- scitex/ml/metrics/_calc_feature_importance.py +183 -0
- scitex/ml/metrics/_calc_mcc.py +61 -0
- scitex/ml/metrics/_calc_pre_rec_auc.py +116 -0
- scitex/ml/metrics/_calc_roc_auc.py +110 -0
- scitex/ml/metrics/_calc_seizure_prediction_metrics.py +490 -0
- scitex/ml/metrics/_calc_silhouette_score.py +503 -0
- scitex/ml/metrics/_normalize_labels.py +83 -0
- scitex/ml/optim/Ranger_Deep_Learning_Optimizer/__init__.py +0 -0
- scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/__init__.py +3 -0
- scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/ranger.py +207 -0
- scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/ranger2020.py +238 -0
- scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/ranger913A.py +215 -0
- scitex/ml/optim/Ranger_Deep_Learning_Optimizer/ranger/rangerqh.py +184 -0
- scitex/ml/optim/Ranger_Deep_Learning_Optimizer/setup.py +24 -0
- scitex/ml/optim/__init__.py +13 -0
- scitex/ml/optim/_get_set.py +31 -0
- scitex/ml/optim/_optimizers.py +71 -0
- scitex/ml/plt/__init__.py +60 -0
- scitex/ml/plt/_plot_conf_mat.py +663 -0
- scitex/ml/plt/_plot_feature_importance.py +323 -0
- scitex/ml/plt/_plot_learning_curve.py +345 -0
- scitex/ml/plt/_plot_optuna_study.py +225 -0
- scitex/ml/plt/_plot_pre_rec_curve.py +290 -0
- scitex/ml/plt/_plot_roc_curve.py +255 -0
- scitex/ml/sk/__init__.py +11 -0
- scitex/ml/sk/_clf.py +58 -0
- scitex/ml/sk/_to_sktime.py +100 -0
- scitex/ml/sklearn/__init__.py +26 -0
- scitex/ml/sklearn/clf.py +58 -0
- scitex/ml/sklearn/to_sktime.py +100 -0
- scitex/{ai/training/early_stopping.py → ml/training/_EarlyStopping.py} +1 -2
- scitex/{ai → ml/training}/_LearningCurveLogger.py +198 -242
- scitex/ml/training/__init__.py +7 -0
- scitex/ml/utils/__init__.py +22 -0
- scitex/ml/utils/_check_params.py +50 -0
- scitex/ml/utils/_default_dataset.py +46 -0
- scitex/ml/utils/_format_samples_for_sktime.py +26 -0
- scitex/ml/utils/_label_encoder.py +134 -0
- scitex/ml/utils/_merge_labels.py +22 -0
- scitex/ml/utils/_sliding_window_data_augmentation.py +11 -0
- scitex/ml/utils/_under_sample.py +51 -0
- scitex/ml/utils/_verify_n_gpus.py +16 -0
- scitex/ml/utils/grid_search.py +148 -0
- scitex/nn/_BNet.py +15 -9
- scitex/nn/_Filters.py +2 -2
- scitex/nn/_ModulationIndex.py +2 -2
- scitex/nn/_PAC.py +1 -1
- scitex/nn/_Spectrogram.py +12 -3
- scitex/nn/__init__.py +9 -10
- scitex/path/__init__.py +18 -0
- scitex/path/_clean.py +4 -0
- scitex/path/_find.py +9 -4
- scitex/path/_symlink.py +348 -0
- scitex/path/_version.py +4 -3
- scitex/pd/__init__.py +2 -0
- scitex/pd/_get_unique.py +99 -0
- scitex/plt/__init__.py +114 -5
- scitex/plt/_subplots/_AxesWrapper.py +1 -3
- scitex/plt/_subplots/_AxisWrapper.py +7 -3
- scitex/plt/_subplots/_AxisWrapperMixins/_AdjustmentMixin.py +47 -13
- scitex/plt/_subplots/_AxisWrapperMixins/_MatplotlibPlotMixin.py +160 -2
- scitex/plt/_subplots/_AxisWrapperMixins/_SeabornMixin.py +26 -4
- scitex/plt/_subplots/_AxisWrapperMixins/_UnitAwareMixin.py +322 -0
- scitex/plt/_subplots/_AxisWrapperMixins/__init__.py +1 -0
- scitex/plt/_subplots/_FigWrapper.py +62 -6
- scitex/plt/_subplots/_export_as_csv.py +43 -27
- scitex/plt/_subplots/_export_as_csv_formatters/__init__.py +5 -4
- scitex/plt/_subplots/_export_as_csv_formatters/_format_annotate.py +81 -0
- scitex/plt/_subplots/_export_as_csv_formatters/_format_bar.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_barh.py +20 -5
- scitex/plt/_subplots/_export_as_csv_formatters/_format_boxplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_contour.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_errorbar.py +35 -18
- scitex/plt/_subplots/_export_as_csv_formatters/_format_eventplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_fill.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_fill_between.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_hist.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_imshow.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_imshow2d.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot.py +15 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_box.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_conf_mat.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_ecdf.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_fillv.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_heatmap.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_image.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_joyplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_kde.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_line.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_mean_ci.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_mean_std.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_median_iqr.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_raster.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_rectangle.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_scatter.py +35 -0
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_scatter_hist.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_shaded_line.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_violin.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_scatter.py +6 -4
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_barplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_boxplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_heatmap.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_histplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_jointplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_kdeplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_lineplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_pairplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_scatterplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_stripplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_swarmplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_violinplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_text.py +60 -0
- scitex/plt/_subplots/_export_as_csv_formatters/_format_violin.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_violinplot.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters/test_formatters.py +1 -3
- scitex/plt/_subplots/_export_as_csv_formatters.py +56 -59
- scitex/plt/ax/_style/_hide_spines.py +1 -3
- scitex/plt/ax/_style/_rotate_labels.py +180 -76
- scitex/plt/ax/_style/_rotate_labels_v01.py +248 -0
- scitex/plt/ax/_style/_set_meta.py +11 -4
- scitex/plt/ax/_style/_set_supxyt.py +3 -3
- scitex/plt/ax/_style/_set_xyt.py +3 -3
- scitex/plt/ax/_style/_share_axes.py +2 -2
- scitex/plt/color/__init__.py +4 -4
- scitex/plt/color/{_get_colors_from_cmap.py → _get_colors_from_conf_matap.py} +7 -7
- scitex/plt/utils/_configure_mpl.py +99 -86
- scitex/plt/utils/_histogram_utils.py +1 -3
- scitex/plt/utils/_is_valid_axis.py +1 -3
- scitex/plt/utils/_scitex_config.py +1 -0
- scitex/repro/__init__.py +75 -0
- scitex/{reproduce → repro}/_gen_ID.py +1 -1
- scitex/{reproduce → repro}/_gen_timestamp.py +1 -1
- scitex/repro_rng/_RandomStateManager.py +590 -0
- scitex/repro_rng/_RandomStateManager_v01-no-verbose-options.py +414 -0
- scitex/repro_rng/__init__.py +39 -0
- scitex/reproduce/__init__.py +25 -13
- scitex/reproduce/_hash_array.py +22 -0
- scitex/resource/_get_processor_usages.py +4 -4
- scitex/resource/_get_specs.py +2 -2
- scitex/resource/_log_processor_usages.py +2 -2
- scitex/rng/_RandomStateManager.py +590 -0
- scitex/rng/_RandomStateManager_v01-no-verbose-options.py +414 -0
- scitex/rng/__init__.py +39 -0
- scitex/scholar/__init__.py +309 -19
- scitex/scholar/__main__.py +319 -0
- scitex/scholar/auth/ScholarAuthManager.py +308 -0
- scitex/scholar/auth/__init__.py +12 -0
- scitex/scholar/auth/core/AuthenticationGateway.py +473 -0
- scitex/scholar/auth/core/BrowserAuthenticator.py +386 -0
- scitex/scholar/auth/core/StrategyResolver.py +309 -0
- scitex/scholar/auth/core/__init__.py +16 -0
- scitex/scholar/auth/gateway/_OpenURLLinkFinder.py +120 -0
- scitex/scholar/auth/gateway/_OpenURLResolver.py +209 -0
- scitex/scholar/auth/gateway/__init__.py +38 -0
- scitex/scholar/auth/gateway/_resolve_functions.py +101 -0
- scitex/scholar/auth/providers/BaseAuthenticator.py +166 -0
- scitex/scholar/auth/providers/EZProxyAuthenticator.py +484 -0
- scitex/scholar/auth/providers/OpenAthensAuthenticator.py +619 -0
- scitex/scholar/auth/providers/ShibbolethAuthenticator.py +686 -0
- scitex/scholar/auth/providers/__init__.py +18 -0
- scitex/scholar/auth/session/AuthCacheManager.py +189 -0
- scitex/scholar/auth/session/SessionManager.py +159 -0
- scitex/scholar/auth/session/__init__.py +11 -0
- scitex/scholar/auth/sso/BaseSSOAutomator.py +373 -0
- scitex/scholar/auth/sso/OpenAthensSSOAutomator.py +378 -0
- scitex/scholar/auth/sso/SSOAutomator.py +180 -0
- scitex/scholar/auth/sso/UniversityOfMelbourneSSOAutomator.py +380 -0
- scitex/scholar/auth/sso/__init__.py +15 -0
- scitex/scholar/browser/ScholarBrowserManager.py +705 -0
- scitex/scholar/browser/__init__.py +38 -0
- scitex/scholar/browser/utils/__init__.py +13 -0
- scitex/scholar/browser/utils/click_and_wait.py +205 -0
- scitex/scholar/browser/utils/close_unwanted_pages.py +140 -0
- scitex/scholar/browser/utils/wait_redirects.py +732 -0
- scitex/scholar/config/PublisherRules.py +132 -0
- scitex/scholar/config/ScholarConfig.py +126 -0
- scitex/scholar/config/__init__.py +17 -0
- scitex/scholar/core/Paper.py +627 -0
- scitex/scholar/core/Papers.py +722 -0
- scitex/scholar/core/Scholar.py +1975 -0
- scitex/scholar/core/__init__.py +9 -0
- scitex/scholar/impact_factor/ImpactFactorEngine.py +204 -0
- scitex/scholar/impact_factor/__init__.py +20 -0
- scitex/scholar/impact_factor/estimation/ImpactFactorEstimationEngine.py +0 -0
- scitex/scholar/impact_factor/estimation/__init__.py +40 -0
- scitex/scholar/impact_factor/estimation/build_database.py +0 -0
- scitex/scholar/impact_factor/estimation/core/__init__.py +28 -0
- scitex/scholar/impact_factor/estimation/core/cache_manager.py +523 -0
- scitex/scholar/impact_factor/estimation/core/calculator.py +355 -0
- scitex/scholar/impact_factor/estimation/core/journal_matcher.py +428 -0
- scitex/scholar/integration/__init__.py +59 -0
- scitex/scholar/integration/base.py +502 -0
- scitex/scholar/integration/mendeley/__init__.py +22 -0
- scitex/scholar/integration/mendeley/exporter.py +166 -0
- scitex/scholar/integration/mendeley/importer.py +236 -0
- scitex/scholar/integration/mendeley/linker.py +79 -0
- scitex/scholar/integration/mendeley/mapper.py +212 -0
- scitex/scholar/integration/zotero/__init__.py +27 -0
- scitex/scholar/integration/zotero/__main__.py +264 -0
- scitex/scholar/integration/zotero/exporter.py +351 -0
- scitex/scholar/integration/zotero/importer.py +372 -0
- scitex/scholar/integration/zotero/linker.py +415 -0
- scitex/scholar/integration/zotero/mapper.py +286 -0
- scitex/scholar/metadata_engines/ScholarEngine.py +588 -0
- scitex/scholar/metadata_engines/__init__.py +21 -0
- scitex/scholar/metadata_engines/individual/ArXivEngine.py +397 -0
- scitex/scholar/metadata_engines/individual/CrossRefEngine.py +274 -0
- scitex/scholar/metadata_engines/individual/CrossRefLocalEngine.py +263 -0
- scitex/scholar/metadata_engines/individual/OpenAlexEngine.py +350 -0
- scitex/scholar/metadata_engines/individual/PubMedEngine.py +329 -0
- scitex/scholar/metadata_engines/individual/SemanticScholarEngine.py +438 -0
- scitex/scholar/metadata_engines/individual/URLDOIEngine.py +410 -0
- scitex/scholar/metadata_engines/individual/_BaseDOIEngine.py +487 -0
- scitex/scholar/metadata_engines/individual/__init__.py +7 -0
- scitex/scholar/metadata_engines/utils/_PubMedConverter.py +469 -0
- scitex/scholar/metadata_engines/utils/_URLDOIExtractor.py +283 -0
- scitex/scholar/metadata_engines/utils/__init__.py +30 -0
- scitex/scholar/metadata_engines/utils/_metadata2bibtex.py +103 -0
- scitex/scholar/metadata_engines/utils/_standardize_metadata.py +376 -0
- scitex/scholar/pdf_download/ScholarPDFDownloader.py +579 -0
- scitex/scholar/pdf_download/__init__.py +5 -0
- scitex/scholar/pdf_download/strategies/__init__.py +38 -0
- scitex/scholar/pdf_download/strategies/chrome_pdf_viewer.py +376 -0
- scitex/scholar/pdf_download/strategies/direct_download.py +131 -0
- scitex/scholar/pdf_download/strategies/manual_download_fallback.py +167 -0
- scitex/scholar/pdf_download/strategies/manual_download_utils.py +996 -0
- scitex/scholar/pdf_download/strategies/response_body.py +207 -0
- scitex/scholar/pipelines/ScholarPipelineBibTeX.py +364 -0
- scitex/scholar/pipelines/ScholarPipelineParallel.py +478 -0
- scitex/scholar/pipelines/ScholarPipelineSingle.py +767 -0
- scitex/scholar/pipelines/__init__.py +49 -0
- scitex/scholar/storage/BibTeXHandler.py +1018 -0
- scitex/scholar/storage/PaperIO.py +468 -0
- scitex/scholar/storage/ScholarLibrary.py +182 -0
- scitex/scholar/storage/_DeduplicationManager.py +548 -0
- scitex/scholar/storage/_LibraryCacheManager.py +724 -0
- scitex/scholar/storage/_LibraryManager.py +1835 -0
- scitex/scholar/storage/__init__.py +28 -0
- scitex/scholar/url_finder/ScholarURLFinder.py +379 -0
- scitex/scholar/url_finder/__init__.py +7 -0
- scitex/scholar/url_finder/strategies/__init__.py +33 -0
- scitex/scholar/url_finder/strategies/find_pdf_urls_by_direct_links.py +261 -0
- scitex/scholar/url_finder/strategies/find_pdf_urls_by_dropdown.py +67 -0
- scitex/scholar/url_finder/strategies/find_pdf_urls_by_href.py +204 -0
- scitex/scholar/url_finder/strategies/find_pdf_urls_by_navigation.py +256 -0
- scitex/scholar/url_finder/strategies/find_pdf_urls_by_publisher_patterns.py +165 -0
- scitex/scholar/url_finder/strategies/find_pdf_urls_by_zotero_translators.py +163 -0
- scitex/scholar/url_finder/strategies/find_supplementary_urls_by_href.py +70 -0
- scitex/scholar/utils/__init__.py +22 -0
- scitex/scholar/utils/bibtex/__init__.py +9 -0
- scitex/scholar/utils/bibtex/_parse_bibtex.py +71 -0
- scitex/scholar/utils/cleanup/__init__.py +8 -0
- scitex/scholar/utils/cleanup/_cleanup_scholar_processes.py +96 -0
- scitex/scholar/utils/cleanup/cleanup_old_extractions.py +117 -0
- scitex/scholar/utils/text/_TextNormalizer.py +407 -0
- scitex/scholar/utils/text/__init__.py +9 -0
- scitex/scholar/zotero/__init__.py +38 -0
- scitex/session/__init__.py +51 -0
- scitex/session/_lifecycle.py +736 -0
- scitex/session/_manager.py +102 -0
- scitex/session/template.py +122 -0
- scitex/stats/__init__.py +30 -26
- scitex/stats/correct/__init__.py +21 -0
- scitex/stats/correct/_correct_bonferroni.py +551 -0
- scitex/stats/correct/_correct_fdr.py +634 -0
- scitex/stats/correct/_correct_holm.py +548 -0
- scitex/stats/correct/_correct_sidak.py +499 -0
- scitex/stats/descriptive/__init__.py +85 -0
- scitex/stats/descriptive/_circular.py +540 -0
- scitex/stats/descriptive/_describe.py +219 -0
- scitex/stats/descriptive/_nan.py +518 -0
- scitex/stats/descriptive/_real.py +189 -0
- scitex/stats/effect_sizes/__init__.py +41 -0
- scitex/stats/effect_sizes/_cliffs_delta.py +325 -0
- scitex/stats/effect_sizes/_cohens_d.py +342 -0
- scitex/stats/effect_sizes/_epsilon_squared.py +315 -0
- scitex/stats/effect_sizes/_eta_squared.py +302 -0
- scitex/stats/effect_sizes/_prob_superiority.py +296 -0
- scitex/stats/posthoc/__init__.py +19 -0
- scitex/stats/posthoc/_dunnett.py +463 -0
- scitex/stats/posthoc/_games_howell.py +383 -0
- scitex/stats/posthoc/_tukey_hsd.py +367 -0
- scitex/stats/power/__init__.py +19 -0
- scitex/stats/power/_power.py +433 -0
- scitex/stats/template.py +119 -0
- scitex/stats/utils/__init__.py +62 -0
- scitex/stats/utils/_effect_size.py +985 -0
- scitex/stats/utils/_formatters.py +270 -0
- scitex/stats/utils/_normalizers.py +927 -0
- scitex/stats/utils/_power.py +433 -0
- scitex/stats_v01/_EffectSizeCalculator.py +488 -0
- scitex/stats_v01/_StatisticalValidator.py +411 -0
- scitex/stats_v01/__init__.py +60 -0
- scitex/stats_v01/_additional_tests.py +415 -0
- scitex/{stats → stats_v01}/_p2stars.py +19 -5
- scitex/stats_v01/_two_sample_tests.py +141 -0
- scitex/stats_v01/desc/__init__.py +83 -0
- scitex/stats_v01/desc/_circular.py +540 -0
- scitex/stats_v01/desc/_describe.py +219 -0
- scitex/stats_v01/desc/_nan.py +518 -0
- scitex/{stats/desc/_nan.py → stats_v01/desc/_nan_v01-20250920_145731.py} +23 -12
- scitex/stats_v01/desc/_real.py +189 -0
- scitex/stats_v01/tests/__corr_test_optimized.py +221 -0
- scitex/stats_v01/tests/_corr_test_optimized.py +179 -0
- scitex/str/__init__.py +1 -3
- scitex/str/_clean_path.py +6 -2
- scitex/str/_latex_fallback.py +267 -160
- scitex/str/_parse.py +44 -36
- scitex/str/_printc.py +1 -3
- scitex/template/__init__.py +87 -0
- scitex/template/_create_project.py +267 -0
- scitex/template/create_pip_project.py +80 -0
- scitex/template/create_research.py +80 -0
- scitex/template/create_singularity.py +80 -0
- scitex/units.py +291 -0
- scitex/utils/_compress_hdf5.py +14 -3
- scitex/utils/_email.py +21 -2
- scitex/utils/_grid.py +6 -4
- scitex/utils/_notify.py +13 -10
- scitex/utils/_verify_scitex_format.py +589 -0
- scitex/utils/_verify_scitex_format_v01.py +370 -0
- scitex/utils/template.py +122 -0
- scitex/web/_search_pubmed.py +62 -16
- scitex-2.1.0.dist-info/LICENSE +21 -0
- scitex-2.1.0.dist-info/METADATA +677 -0
- scitex-2.1.0.dist-info/RECORD +919 -0
- {scitex-2.0.0.dist-info → scitex-2.1.0.dist-info}/WHEEL +1 -1
- scitex-2.1.0.dist-info/entry_points.txt +3 -0
- scitex/ai/__Classifiers.py +0 -101
- scitex/ai/classification/classification_reporter.py +0 -1137
- scitex/ai/classification/classifiers.py +0 -101
- scitex/ai/classification_reporter.py +0 -1161
- scitex/ai/genai/__init__.py +0 -277
- scitex/ai/genai/anthropic_provider.py +0 -320
- scitex/ai/genai/anthropic_refactored.py +0 -109
- scitex/ai/genai/auth_manager.py +0 -200
- scitex/ai/genai/base_provider.py +0 -291
- scitex/ai/genai/chat_history.py +0 -307
- scitex/ai/genai/cost_tracker.py +0 -276
- scitex/ai/genai/deepseek_provider.py +0 -251
- scitex/ai/genai/google_provider.py +0 -228
- scitex/ai/genai/groq_provider.py +0 -248
- scitex/ai/genai/image_processor.py +0 -250
- scitex/ai/genai/llama_provider.py +0 -214
- scitex/ai/genai/mock_provider.py +0 -127
- scitex/ai/genai/model_registry.py +0 -304
- scitex/ai/genai/openai_provider.py +0 -293
- scitex/ai/genai/perplexity_provider.py +0 -205
- scitex/ai/genai/provider_base.py +0 -302
- scitex/ai/genai/provider_factory.py +0 -370
- scitex/ai/genai/response_handler.py +0 -235
- scitex/ai/layer/_Pass.py +0 -21
- scitex/ai/layer/__init__.py +0 -10
- scitex/ai/layer/_switch.py +0 -8
- scitex/ai/metrics/_bACC.py +0 -51
- scitex/ai/plt/_learning_curve.py +0 -194
- scitex/ai/plt/_optuna_study.py +0 -111
- scitex/ai/plt/aucs/__init__.py +0 -2
- scitex/ai/plt/aucs/example.py +0 -60
- scitex/ai/plt/aucs/pre_rec_auc.py +0 -223
- scitex/ai/plt/aucs/roc_auc.py +0 -246
- scitex/ai/sampling/undersample.py +0 -29
- scitex/db/_SQLite3.py +0 -2136
- scitex/db/_SQLite3Mixins/_BlobMixin.py +0 -229
- scitex/gen/_close.py +0 -222
- scitex/gen/_start.py +0 -451
- scitex/general/__init__.py +0 -5
- scitex/io/_load_modules/_db.py +0 -24
- scitex/life/__init__.py +0 -10
- scitex/life/_monitor_rain.py +0 -49
- scitex/reproduce/_fix_seeds.py +0 -45
- scitex/res/__init__.py +0 -5
- scitex/scholar/_local_search.py +0 -454
- scitex/scholar/_paper.py +0 -244
- scitex/scholar/_pdf_downloader.py +0 -325
- scitex/scholar/_search.py +0 -393
- scitex/scholar/_vector_search.py +0 -370
- scitex/scholar/_web_sources.py +0 -457
- scitex/stats/desc/__init__.py +0 -40
- scitex-2.0.0.dist-info/METADATA +0 -307
- scitex-2.0.0.dist-info/RECORD +0 -572
- scitex-2.0.0.dist-info/licenses/LICENSE +0 -7
- /scitex/ai/{act → activation}/__init__.py +0 -0
- /scitex/ai/{act → activation}/_define.py +0 -0
- /scitex/ai/{early_stopping.py → training/_EarlyStopping.py} +0 -0
- /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_ImportExportMixin.py +0 -0
- /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_IndexMixin.py +0 -0
- /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_RowMixin.py +0 -0
- /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_TableMixin.py +0 -0
- /scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/__init__.py +0 -0
- /scitex/{stats → stats_v01}/_calc_partial_corr.py +0 -0
- /scitex/{stats → stats_v01}/_corr_test_multi.py +0 -0
- /scitex/{stats → stats_v01}/_corr_test_wrapper.py +0 -0
- /scitex/{stats → stats_v01}/_describe_wrapper.py +0 -0
- /scitex/{stats → stats_v01}/_multiple_corrections.py +0 -0
- /scitex/{stats → stats_v01}/_nan_stats.py +0 -0
- /scitex/{stats → stats_v01}/_p2stars_wrapper.py +0 -0
- /scitex/{stats → stats_v01}/_statistical_tests.py +0 -0
- /scitex/{stats/desc/_describe.py → stats_v01/desc/_describe_v01-20250920_145731.py} +0 -0
- /scitex/{stats/desc/_real.py → stats_v01/desc/_real_v01-20250920_145731.py} +0 -0
- /scitex/{stats → stats_v01}/multiple/__init__.py +0 -0
- /scitex/{stats → stats_v01}/multiple/_bonferroni_correction.py +0 -0
- /scitex/{stats → stats_v01}/multiple/_fdr_correction.py +0 -0
- /scitex/{stats → stats_v01}/multiple/_multicompair.py +0 -0
- /scitex/{stats → stats_v01}/tests/__corr_test.py +0 -0
- /scitex/{stats → stats_v01}/tests/__corr_test_multi.py +0 -0
- /scitex/{stats → stats_v01}/tests/__corr_test_single.py +0 -0
- /scitex/{stats → stats_v01}/tests/__init__.py +0 -0
- /scitex/{stats → stats_v01}/tests/_brunner_munzel_test.py +0 -0
- /scitex/{stats → stats_v01}/tests/_nocorrelation_test.py +0 -0
- /scitex/{stats → stats_v01}/tests/_smirnov_grubbs.py +0 -0
- {scitex-2.0.0.dist-info → scitex-2.1.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,1008 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
# Timestamp: "2025-10-10 00:50:44 (ywatanabe)"
|
|
4
|
+
# File: /home/ywatanabe/proj/scitex_repo/src/scitex/browser/stealth/StealthManager.py
|
|
5
|
+
# ----------------------------------------
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
import os
|
|
8
|
+
__FILE__ = (
|
|
9
|
+
"./src/scitex/browser/stealth/StealthManager.py"
|
|
10
|
+
)
|
|
11
|
+
__DIR__ = os.path.dirname(__FILE__)
|
|
12
|
+
# ----------------------------------------
|
|
13
|
+
|
|
14
|
+
__FILE__ = __file__
|
|
15
|
+
|
|
16
|
+
import asyncio
|
|
17
|
+
import random
|
|
18
|
+
|
|
19
|
+
from playwright.async_api import Browser, BrowserContext, Page
|
|
20
|
+
|
|
21
|
+
from scitex import logging
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
# User Agent(Old) Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36
|
|
26
|
+
# WebDriver(New) missing (passed)
|
|
27
|
+
# WebDriver Advanced passed
|
|
28
|
+
# Chrome(New) present (passed)
|
|
29
|
+
# Permissions(New) prompt
|
|
30
|
+
# Plugins Length(Old) 5
|
|
31
|
+
# Plugins is of type PluginArray passed
|
|
32
|
+
# Languages(Old) en-US,en,ja
|
|
33
|
+
# WebGL Vendor Google Inc. (AMD)
|
|
34
|
+
# WebGL Renderer ANGLE (AMD, AMD Radeon(TM) Graphics (0x00001636) Direct3D11 vs_5_0 ps_5_0, D3D11)
|
|
35
|
+
# Broken Image Dimensions 16x16
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class StealthManager:
|
|
39
|
+
def __init__(
|
|
40
|
+
self,
|
|
41
|
+
viewport_size: tuple = None,
|
|
42
|
+
spoof_dimension: bool = False,
|
|
43
|
+
):
|
|
44
|
+
self.name = self.__class__.__name__
|
|
45
|
+
self.viewport_size = viewport_size
|
|
46
|
+
self.spoof_dimension = spoof_dimension
|
|
47
|
+
|
|
48
|
+
def get_random_user_agent(self) -> str:
|
|
49
|
+
user_agents = [
|
|
50
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
|
51
|
+
# "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
|
|
52
|
+
# "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
|
|
53
|
+
# "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
|
|
54
|
+
# "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
|
|
55
|
+
# "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
56
|
+
# "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
57
|
+
# "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
|
58
|
+
]
|
|
59
|
+
user_agent = random.choice(user_agents)
|
|
60
|
+
logger.debug(
|
|
61
|
+
f"{self.name}: User Agent randomly selected: {user_agent}"
|
|
62
|
+
)
|
|
63
|
+
return user_agent
|
|
64
|
+
|
|
65
|
+
def get_random_viewport(self) -> dict:
|
|
66
|
+
if self.viewport_size:
|
|
67
|
+
viewport = {
|
|
68
|
+
"width": self.viewport_size[0],
|
|
69
|
+
"height": self.viewport_size[1],
|
|
70
|
+
}
|
|
71
|
+
logger.debug(
|
|
72
|
+
f"{self.name}: Viewport defined as specified in Stealth Manager initiation: {viewport}"
|
|
73
|
+
)
|
|
74
|
+
return viewport
|
|
75
|
+
|
|
76
|
+
if self.spoof_dimension:
|
|
77
|
+
# viewport = {"width": 1, "height": 1}
|
|
78
|
+
viewport = {"width": 1920, "height": 1080}
|
|
79
|
+
logger.debug(
|
|
80
|
+
f"{self.name}: Viewport defined as spoof_dimension passed during Stealth Manager initiation: {viewport}"
|
|
81
|
+
)
|
|
82
|
+
return viewport
|
|
83
|
+
|
|
84
|
+
else:
|
|
85
|
+
viewport = random.choice(
|
|
86
|
+
[
|
|
87
|
+
{"width": 1920, "height": 1080},
|
|
88
|
+
{"width": 1366, "height": 768},
|
|
89
|
+
{"width": 1440, "height": 900},
|
|
90
|
+
{"width": 1280, "height": 720},
|
|
91
|
+
]
|
|
92
|
+
)
|
|
93
|
+
logger.debug(
|
|
94
|
+
f"{self.name}: Viewport randomly selected: {viewport}"
|
|
95
|
+
)
|
|
96
|
+
return viewport
|
|
97
|
+
|
|
98
|
+
def get_stealth_options(self) -> dict:
|
|
99
|
+
return {
|
|
100
|
+
"viewport": self.get_random_viewport(),
|
|
101
|
+
"user_agent": self.get_random_user_agent(),
|
|
102
|
+
"extra_http_headers": {
|
|
103
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
104
|
+
"Accept-Language": "en-US,en;q=0.9",
|
|
105
|
+
"Accept-Encoding": "gzip, deflate, br, zstd",
|
|
106
|
+
"Cache-Control": "max-age=0",
|
|
107
|
+
"Sec-Ch-Ua": '"Google Chrome";v="132", "Chromium";v="132", "Not_A Brand";v="24"',
|
|
108
|
+
"Sec-Ch-Ua-Mobile": "?0",
|
|
109
|
+
"Sec-Ch-Ua-Platform": '"Windows"',
|
|
110
|
+
"Sec-Fetch-Dest": "document",
|
|
111
|
+
"Sec-Fetch-Mode": "navigate",
|
|
112
|
+
"Sec-Fetch-Site": "none",
|
|
113
|
+
"Sec-Fetch-User": "?1",
|
|
114
|
+
"Upgrade-Insecure-Requests": "1",
|
|
115
|
+
"Referer": "https://www.google.com/",
|
|
116
|
+
},
|
|
117
|
+
"ignore_https_errors": True,
|
|
118
|
+
"java_script_enabled": True,
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
def get_stealth_options_additional(self) -> list:
|
|
122
|
+
stealth_args = [
|
|
123
|
+
# Core security and sandbox
|
|
124
|
+
"--no-sandbox",
|
|
125
|
+
"--disable-setuid-sandbox",
|
|
126
|
+
"--disable-dev-shm-usage",
|
|
127
|
+
# Critical automation detection bypass
|
|
128
|
+
"--disable-blink-features=AutomationControlled",
|
|
129
|
+
"--disable-features=UserAgentClientHint",
|
|
130
|
+
"--disable-features=WebRtcHideLocalIpsWithMdns",
|
|
131
|
+
"--disable-features=VizDisplayCompositor",
|
|
132
|
+
"--disable-features=TranslateUI",
|
|
133
|
+
"--disable-features=Translate",
|
|
134
|
+
"--disable-features=MediaRouter",
|
|
135
|
+
"--disable-features=OptimizationHints",
|
|
136
|
+
"--disable-features=AudioServiceOutOfProcess",
|
|
137
|
+
"--disable-features=VizServiceSharingEnabled",
|
|
138
|
+
# Enhanced fingerprinting resistance
|
|
139
|
+
"--disable-web-security",
|
|
140
|
+
"--disable-site-isolation-trials",
|
|
141
|
+
"--disable-cross-domain-blocking",
|
|
142
|
+
"--disable-features=CrossOriginOpenerPolicy",
|
|
143
|
+
"--disable-features=DocumentPolicy",
|
|
144
|
+
"--disable-features=OriginPolicy",
|
|
145
|
+
# Network and connectivity
|
|
146
|
+
"--disable-background-networking",
|
|
147
|
+
"--disable-client-side-phishing-detection",
|
|
148
|
+
"--disable-component-update",
|
|
149
|
+
"--disable-domain-reliability",
|
|
150
|
+
"--disable-background-mode",
|
|
151
|
+
"--disable-ipc-flooding-protection",
|
|
152
|
+
# Browser behavior normalization
|
|
153
|
+
"--disable-sync",
|
|
154
|
+
"--disable-translate",
|
|
155
|
+
"--disable-default-apps",
|
|
156
|
+
"--enable-extensions",
|
|
157
|
+
"--no-first-run",
|
|
158
|
+
"--no-default-browser-check",
|
|
159
|
+
"--disable-infobars",
|
|
160
|
+
"--disable-notifications",
|
|
161
|
+
# POPUP BLOCKING - Add these lines
|
|
162
|
+
"--block-new-web-contents",
|
|
163
|
+
"--disable-popup-blocking",
|
|
164
|
+
"--suppress-message-center-popups",
|
|
165
|
+
"--disable-session-crashed-bubble",
|
|
166
|
+
"--disable-features=UserAgentClientHint,TranslateSubFrames,AutofillServerCommunication",
|
|
167
|
+
# Performance and timing
|
|
168
|
+
"--disable-background-timer-throttling",
|
|
169
|
+
"--disable-backgrounding-occluded-windows",
|
|
170
|
+
"--disable-renderer-backgrounding",
|
|
171
|
+
"--disable-hang-monitor",
|
|
172
|
+
"--disable-plugins-discovery",
|
|
173
|
+
"--disable-field-trial-config",
|
|
174
|
+
# Media and hardware access
|
|
175
|
+
"--use-fake-ui-for-media-stream",
|
|
176
|
+
"--use-fake-device-for-media-stream",
|
|
177
|
+
"--autoplay-policy=user-gesture-required",
|
|
178
|
+
"--disable-audio-output",
|
|
179
|
+
# Logging and debugging
|
|
180
|
+
"--disable-logging",
|
|
181
|
+
"--disable-gpu-logging",
|
|
182
|
+
"--disable-dev-shm-usage",
|
|
183
|
+
"--disable-renderer-code-integrity",
|
|
184
|
+
# Memory optimization
|
|
185
|
+
"--memory-pressure-off",
|
|
186
|
+
"--max_old_space_size=4096",
|
|
187
|
+
"--disable-low-res-tiling",
|
|
188
|
+
"--disable-partial-raster",
|
|
189
|
+
"--disable-checker-imaging",
|
|
190
|
+
# TLS/SSL improvements
|
|
191
|
+
"--ignore-certificate-errors",
|
|
192
|
+
"--ignore-ssl-errors",
|
|
193
|
+
"--ignore-certificate-errors-spki-list",
|
|
194
|
+
"--disable-web-security",
|
|
195
|
+
# Additional anti-detection
|
|
196
|
+
"--disable-features=VizHitTestSurfaceLayer",
|
|
197
|
+
"--disable-features=TranslateSubFrames",
|
|
198
|
+
"--disable-search-engine-choice-screen",
|
|
199
|
+
"--disable-features=PrivacySandboxSettings4",
|
|
200
|
+
"--disable-features=AutofillServerCommunication",
|
|
201
|
+
"--enable-extensions",
|
|
202
|
+
]
|
|
203
|
+
|
|
204
|
+
# Apply window size and position based on mode
|
|
205
|
+
if self.spoof_dimension:
|
|
206
|
+
# 1x1 window completely off-screen for true invisibility
|
|
207
|
+
stealth_args.extend(["--window-size=1,1", "--window-position=0,0"])
|
|
208
|
+
logger.debug(
|
|
209
|
+
f"{self.name}: Invisible mode: Window set to 1x1 at position 0,0 (off-screen)"
|
|
210
|
+
)
|
|
211
|
+
else:
|
|
212
|
+
# Standard window or custom size
|
|
213
|
+
if self.viewport_size:
|
|
214
|
+
stealth_args.append(
|
|
215
|
+
f"--window-size={self.viewport_size[0]},{self.viewport_size[1]}"
|
|
216
|
+
)
|
|
217
|
+
else:
|
|
218
|
+
stealth_args.append("--window-size=1920,1080")
|
|
219
|
+
|
|
220
|
+
if self.spoof_dimension:
|
|
221
|
+
config_desc = "Invisible (1x1)"
|
|
222
|
+
elif self.viewport_size:
|
|
223
|
+
config_desc = f"{self.viewport_size[0]}x{self.viewport_size[1]}"
|
|
224
|
+
else:
|
|
225
|
+
config_desc = "Default (1920x1080)"
|
|
226
|
+
|
|
227
|
+
logger.debug(
|
|
228
|
+
f"{self.name}: Browser window configuration: {config_desc}"
|
|
229
|
+
)
|
|
230
|
+
return stealth_args
|
|
231
|
+
|
|
232
|
+
def get_network_evasion_headers(self) -> dict:
|
|
233
|
+
"""Generate realistic HTTP headers to avoid network-level detection."""
|
|
234
|
+
return {
|
|
235
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
236
|
+
"Accept-Encoding": "gzip, deflate, br, zstd",
|
|
237
|
+
"Accept-Language": "en-US,en;q=0.9",
|
|
238
|
+
"Cache-Control": "max-age=0",
|
|
239
|
+
"DNT": "1",
|
|
240
|
+
"Sec-Ch-Ua": '"Google Chrome";v="132", "Chromium";v="132", "Not_A Brand";v="24"',
|
|
241
|
+
"Sec-Ch-Ua-Mobile": "?0",
|
|
242
|
+
"Sec-Ch-Ua-Platform": '"Linux"',
|
|
243
|
+
"Sec-Ch-Ua-Platform-Version": '"5.15.0"',
|
|
244
|
+
"Sec-Fetch-Dest": "document",
|
|
245
|
+
"Sec-Fetch-Mode": "navigate",
|
|
246
|
+
"Sec-Fetch-Site": "none",
|
|
247
|
+
"Sec-Fetch-User": "?1",
|
|
248
|
+
"Upgrade-Insecure-Requests": "1",
|
|
249
|
+
"X-Forwarded-For": f"{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}",
|
|
250
|
+
"X-Real-IP": f"{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}",
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
async def add_human_behavior_async(self, page: Page):
|
|
254
|
+
"""Add human-like behavior patterns to avoid detection."""
|
|
255
|
+
# Random delay before starting interactions
|
|
256
|
+
delay = random.uniform(2, 5)
|
|
257
|
+
logger.debug(
|
|
258
|
+
f"{self.name}: Adding human behavior delay: {delay:.2f} seconds"
|
|
259
|
+
)
|
|
260
|
+
await asyncio.sleep(delay)
|
|
261
|
+
|
|
262
|
+
# Simulate scrolling behavior
|
|
263
|
+
try:
|
|
264
|
+
await page.evaluate(
|
|
265
|
+
"""
|
|
266
|
+
window.scrollTo({
|
|
267
|
+
top: Math.random() * 500,
|
|
268
|
+
behavior: 'smooth'
|
|
269
|
+
});
|
|
270
|
+
"""
|
|
271
|
+
)
|
|
272
|
+
await asyncio.sleep(random.uniform(1, 3))
|
|
273
|
+
except Exception as e:
|
|
274
|
+
logger.debug(f"{self.name}: Human behavior simulation failed: {e}")
|
|
275
|
+
|
|
276
|
+
async def handle_cloudflare_challenge_async(
|
|
277
|
+
self, page: Page, max_wait: int = 45
|
|
278
|
+
):
|
|
279
|
+
"""Enhanced Cloudflare challenge detection and handling."""
|
|
280
|
+
logger.debug(f"{self.name}: Checking for Cloudflare challenge...")
|
|
281
|
+
|
|
282
|
+
cloudflare_indicators = [
|
|
283
|
+
"Just a moment",
|
|
284
|
+
"Checking your browser",
|
|
285
|
+
"DDoS protection by Cloudflare",
|
|
286
|
+
"Cloudflare Ray ID",
|
|
287
|
+
"cf-browser-verification",
|
|
288
|
+
"Please wait while we verify you're a human",
|
|
289
|
+
"Verify you are human",
|
|
290
|
+
"Security check",
|
|
291
|
+
"Browser verification",
|
|
292
|
+
"cf-challenge-running",
|
|
293
|
+
]
|
|
294
|
+
|
|
295
|
+
try:
|
|
296
|
+
# First check if we're on a Cloudflare challenge page
|
|
297
|
+
page_content = await page.content()
|
|
298
|
+
title = await page.title()
|
|
299
|
+
|
|
300
|
+
is_challenge = any(
|
|
301
|
+
indicator.lower() in page_content.lower()
|
|
302
|
+
or indicator.lower() in title.lower()
|
|
303
|
+
for indicator in cloudflare_indicators
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
if not is_challenge:
|
|
307
|
+
logger.debug(f"{self.name}: No Cloudflare challenge detected")
|
|
308
|
+
return True
|
|
309
|
+
|
|
310
|
+
logger.debug(
|
|
311
|
+
f"{self.name}: Cloudflare challenge detected, waiting for completion..."
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
# Add human-like behavior during challenge
|
|
315
|
+
await self.add_human_behavior_async(page)
|
|
316
|
+
|
|
317
|
+
# Wait for challenge completion with multiple conditions
|
|
318
|
+
await page.wait_for_function(
|
|
319
|
+
"""
|
|
320
|
+
() => {
|
|
321
|
+
const content = document.documentElement.innerText.toLowerCase();
|
|
322
|
+
const title = document.title.toLowerCase();
|
|
323
|
+
|
|
324
|
+
// Challenge completion indicators
|
|
325
|
+
const challengeComplete = !content.includes('just a moment') &&
|
|
326
|
+
!content.includes('checking your browser') &&
|
|
327
|
+
!content.includes('ddos protection') &&
|
|
328
|
+
!content.includes('please wait') &&
|
|
329
|
+
!content.includes('verify you are human') &&
|
|
330
|
+
!title.includes('just a moment');
|
|
331
|
+
|
|
332
|
+
// Also check if we've been redirected or if the URL changed
|
|
333
|
+
const urlChanged = window.location.href !== window.initialUrl;
|
|
334
|
+
window.initialUrl = window.initialUrl || window.location.href;
|
|
335
|
+
|
|
336
|
+
return challengeComplete || urlChanged;
|
|
337
|
+
}
|
|
338
|
+
""",
|
|
339
|
+
timeout=max_wait * 1000,
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
# Additional wait to ensure page is fully loaded
|
|
343
|
+
await asyncio.sleep(random.uniform(2, 4))
|
|
344
|
+
|
|
345
|
+
logger.debug(
|
|
346
|
+
f"{self.name}: Cloudflare challenge passed successfully"
|
|
347
|
+
)
|
|
348
|
+
return True
|
|
349
|
+
|
|
350
|
+
except Exception as e:
|
|
351
|
+
logger.warning(
|
|
352
|
+
f"{self.name}: Cloudflare challenge handling timeout or error: {e}"
|
|
353
|
+
)
|
|
354
|
+
|
|
355
|
+
# Try to detect if we're still on challenge page
|
|
356
|
+
try:
|
|
357
|
+
final_content = await page.content()
|
|
358
|
+
still_challenged = any(
|
|
359
|
+
indicator.lower() in final_content.lower()
|
|
360
|
+
for indicator in cloudflare_indicators
|
|
361
|
+
)
|
|
362
|
+
|
|
363
|
+
if still_challenged:
|
|
364
|
+
logger.error(
|
|
365
|
+
f"{self.name}: Still on Cloudflare challenge page after timeout"
|
|
366
|
+
)
|
|
367
|
+
return False
|
|
368
|
+
else:
|
|
369
|
+
logger.debug(
|
|
370
|
+
f"{self.name}: Challenge may have completed despite timeout"
|
|
371
|
+
)
|
|
372
|
+
return True
|
|
373
|
+
|
|
374
|
+
except:
|
|
375
|
+
return False
|
|
376
|
+
|
|
377
|
+
def get_init_script(self) -> str:
|
|
378
|
+
return """
|
|
379
|
+
(() => {
|
|
380
|
+
'use strict';
|
|
381
|
+
|
|
382
|
+
// === CORE WEBDRIVER DETECTION REMOVAL ===
|
|
383
|
+
Object.defineProperty(navigator, 'webdriver', {
|
|
384
|
+
get: () => undefined,
|
|
385
|
+
configurable: true
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
// Remove all automation-related properties
|
|
389
|
+
const automationProps = [
|
|
390
|
+
'webdriver', '__driver_evaluate', '__webdriver_evaluate', '__selenium_evaluate',
|
|
391
|
+
'__fxdriver_evaluate', '__driver_unwrapped', '__webdriver_unwrapped',
|
|
392
|
+
'__selenium_unwrapped', '__fxdriver_unwrapped', '__webdriver_script_function',
|
|
393
|
+
'__webdriver_script_func', '__webdriver_script_fn', '__fxdriver_script_fn',
|
|
394
|
+
'__selenium_script_fn', '__webdriver_func', '__webdriver_fn', '__$webdriverAsyncExecutor',
|
|
395
|
+
'__lastWatirAlert', '__lastWatirConfirm', '__lastWatirPrompt', '_WEBDRIVER_ELEM_CACHE',
|
|
396
|
+
'ChromeDriverw', 'driver-evaluate', 'webdriver-evaluate', 'selenium-evaluate',
|
|
397
|
+
'webdriverCommand', 'webdriver-evaluate-response', '__webdriverFunc', '__webdriver_script_func',
|
|
398
|
+
'__$webdriverAsyncExecutor', '$chrome_asyncScriptInfo', '$cdc_asdjflasutopfhvcZLmcfl_'
|
|
399
|
+
];
|
|
400
|
+
|
|
401
|
+
automationProps.forEach(prop => {
|
|
402
|
+
try {
|
|
403
|
+
delete window[prop];
|
|
404
|
+
delete document[prop];
|
|
405
|
+
delete navigator[prop];
|
|
406
|
+
} catch (e) {}
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
// === NAVIGATOR PROPERTIES ===
|
|
410
|
+
// Mock realistic Chrome object with proper methods
|
|
411
|
+
if (!window.chrome || Object.getPrototypeOf(window.chrome) === Object.prototype) {
|
|
412
|
+
window.chrome = {
|
|
413
|
+
app: {
|
|
414
|
+
isInstalled: false,
|
|
415
|
+
InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
|
|
416
|
+
RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' }
|
|
417
|
+
},
|
|
418
|
+
runtime: {
|
|
419
|
+
onConnect: null,
|
|
420
|
+
onMessage: null,
|
|
421
|
+
onConnectExternal: null,
|
|
422
|
+
onMessageExternal: null,
|
|
423
|
+
connect: () => {},
|
|
424
|
+
sendMessage: () => {},
|
|
425
|
+
getManifest: () => ({ name: 'Chrome', version: '132.0.6834.59' }),
|
|
426
|
+
getURL: (path) => 'chrome-extension://invalid/' + path
|
|
427
|
+
},
|
|
428
|
+
webstore: {
|
|
429
|
+
onInstallStageChanged: null,
|
|
430
|
+
onDownloadProgress: null,
|
|
431
|
+
install: () => {}
|
|
432
|
+
},
|
|
433
|
+
csi: () => ({ pageT: Math.random() * 1000, startE: Math.random() * 1000 }),
|
|
434
|
+
loadTimes: () => ({
|
|
435
|
+
requestTime: performance.now() / 1000,
|
|
436
|
+
startLoadTime: performance.now() / 1000,
|
|
437
|
+
commitLoadTime: performance.now() / 1000,
|
|
438
|
+
finishDocumentLoadTime: performance.now() / 1000,
|
|
439
|
+
finishLoadTime: performance.now() / 1000,
|
|
440
|
+
firstPaintTime: performance.now() / 1000,
|
|
441
|
+
firstPaintAfterLoadTime: 0,
|
|
442
|
+
navigationType: 'Other'
|
|
443
|
+
})
|
|
444
|
+
};
|
|
445
|
+
|
|
446
|
+
// Make chrome object non-enumerable to match real Chrome
|
|
447
|
+
Object.defineProperty(window, 'chrome', {
|
|
448
|
+
value: window.chrome,
|
|
449
|
+
writable: false,
|
|
450
|
+
enumerable: false,
|
|
451
|
+
configurable: false
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
// === LANGUAGE AND LOCALE ===
|
|
456
|
+
Object.defineProperty(navigator, 'languages', {
|
|
457
|
+
get: () => ['en-US', 'en'],
|
|
458
|
+
configurable: true
|
|
459
|
+
});
|
|
460
|
+
|
|
461
|
+
Object.defineProperty(navigator, 'language', {
|
|
462
|
+
get: () => 'en-US',
|
|
463
|
+
configurable: true
|
|
464
|
+
});
|
|
465
|
+
|
|
466
|
+
// === REALISTIC PLUGINS ===
|
|
467
|
+
const mockPlugins = [
|
|
468
|
+
{
|
|
469
|
+
0: { type: "application/x-google-chrome-pdf", suffixes: "pdf", description: "Portable Document Format", enabledPlugin: null },
|
|
470
|
+
description: "Portable Document Format",
|
|
471
|
+
filename: "internal-pdf-viewer",
|
|
472
|
+
length: 1,
|
|
473
|
+
name: "Chrome PDF Plugin"
|
|
474
|
+
},
|
|
475
|
+
{
|
|
476
|
+
0: { type: "application/pdf", suffixes: "pdf", description: "Portable Document Format", enabledPlugin: null },
|
|
477
|
+
description: "Portable Document Format",
|
|
478
|
+
filename: "mhjfbmdgcfjbbpaeojofohoefgiehjai",
|
|
479
|
+
length: 1,
|
|
480
|
+
name: "Chrome PDF Viewer"
|
|
481
|
+
},
|
|
482
|
+
{
|
|
483
|
+
0: { type: "application/x-nacl", suffixes: "", description: "Native Client Executable", enabledPlugin: null },
|
|
484
|
+
1: { type: "application/x-pnacl", suffixes: "", description: "Portable Native Client Executable", enabledPlugin: null },
|
|
485
|
+
description: "Native Client",
|
|
486
|
+
filename: "internal-nacl-plugin",
|
|
487
|
+
length: 2,
|
|
488
|
+
name: "Native Client"
|
|
489
|
+
}
|
|
490
|
+
];
|
|
491
|
+
|
|
492
|
+
Object.defineProperty(navigator, 'plugins', {
|
|
493
|
+
get: () => mockPlugins,
|
|
494
|
+
configurable: true
|
|
495
|
+
});
|
|
496
|
+
|
|
497
|
+
// === HARDWARE CONCURRENCY ===
|
|
498
|
+
Object.defineProperty(navigator, 'hardwareConcurrency', {
|
|
499
|
+
get: () => Math.max(2, Math.min(16, Math.floor(Math.random() * 8) + 4)),
|
|
500
|
+
configurable: true
|
|
501
|
+
});
|
|
502
|
+
|
|
503
|
+
// === PERMISSIONS API ===
|
|
504
|
+
if (navigator.permissions && navigator.permissions.query) {
|
|
505
|
+
const originalQuery = navigator.permissions.query.bind(navigator.permissions);
|
|
506
|
+
navigator.permissions.query = async (parameters) => {
|
|
507
|
+
const permission = parameters.name;
|
|
508
|
+
if (permission === 'notifications') {
|
|
509
|
+
return Promise.resolve({ state: 'default', onchange: null });
|
|
510
|
+
}
|
|
511
|
+
if (permission === 'geolocation') {
|
|
512
|
+
return Promise.resolve({ state: 'prompt', onchange: null });
|
|
513
|
+
}
|
|
514
|
+
try {
|
|
515
|
+
return await originalQuery(parameters);
|
|
516
|
+
} catch (e) {
|
|
517
|
+
return Promise.resolve({ state: 'prompt', onchange: null });
|
|
518
|
+
}
|
|
519
|
+
};
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
// === Camvas FINGERPRINTING PROTECTION ===
|
|
523
|
+
const getImageData = HTMLCanvasElement.prototype.toDataURL;
|
|
524
|
+
HTMLCanvasElement.prototype.toDataURL = function(type) {
|
|
525
|
+
const shift = Math.floor(Math.random() * 10) - 5;
|
|
526
|
+
const originalImageData = getImageData.apply(this, arguments);
|
|
527
|
+
return originalImageData;
|
|
528
|
+
};
|
|
529
|
+
|
|
530
|
+
// === WEBGL FINGERPRINTING PROTECTION ===
|
|
531
|
+
const glContexts = ['webgl', 'webgl2', 'experimental-webgl', 'experimental-webgl2'];
|
|
532
|
+
|
|
533
|
+
glContexts.forEach(contextType => {
|
|
534
|
+
try {
|
|
535
|
+
const canvas = document.createElement('canvas');
|
|
536
|
+
const gl = canvas.getContext(contextType);
|
|
537
|
+
if (gl) {
|
|
538
|
+
const getParameter = gl.getParameter.bind(gl);
|
|
539
|
+
gl.getParameter = function(parameter) {
|
|
540
|
+
// Vendor and renderer spoofing
|
|
541
|
+
if (parameter === gl.VENDOR || parameter === 37445) {
|
|
542
|
+
return 'Intel Inc.';
|
|
543
|
+
}
|
|
544
|
+
if (parameter === gl.RENDERER || parameter === 37446) {
|
|
545
|
+
return 'Intel Iris OpenGL Engine';
|
|
546
|
+
}
|
|
547
|
+
if (parameter === gl.VERSION) {
|
|
548
|
+
return 'OpenGL ES 2.0 (ANGLE 2.1.0.c8ea8ca4eb1a)';
|
|
549
|
+
}
|
|
550
|
+
if (parameter === gl.SHADING_LANGUAGE_VERSION) {
|
|
551
|
+
return 'OpenGL ES GLSL ES 1.0 (ANGLE 2.1.0.c8ea8ca4eb1a)';
|
|
552
|
+
}
|
|
553
|
+
return getParameter(parameter);
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
} catch (e) {}
|
|
557
|
+
});
|
|
558
|
+
|
|
559
|
+
// === CANVAS FINGERPRINTING PROTECTION ===
|
|
560
|
+
const getContext = HTMLCanvasElement.prototype.getContext;
|
|
561
|
+
HTMLCanvasElement.prototype.getContext = function(contextType, ...args) {
|
|
562
|
+
if (contextType === '2d') {
|
|
563
|
+
const context = getContext.call(this, contextType, ...args);
|
|
564
|
+
if (context) {
|
|
565
|
+
const originalFillText = context.fillText;
|
|
566
|
+
const originalStrokeText = context.strokeText;
|
|
567
|
+
|
|
568
|
+
context.fillText = function(...args) {
|
|
569
|
+
// Add slight noise to prevent consistent fingerprints
|
|
570
|
+
if (args.length >= 3) {
|
|
571
|
+
args[1] += Math.random() * 0.01 - 0.005;
|
|
572
|
+
args[2] += Math.random() * 0.01 - 0.005;
|
|
573
|
+
}
|
|
574
|
+
return originalFillText.apply(this, args);
|
|
575
|
+
};
|
|
576
|
+
|
|
577
|
+
context.strokeText = function(...args) {
|
|
578
|
+
if (args.length >= 3) {
|
|
579
|
+
args[1] += Math.random() * 0.01 - 0.005;
|
|
580
|
+
args[2] += Math.random() * 0.01 - 0.005;
|
|
581
|
+
}
|
|
582
|
+
return originalStrokeText.apply(this, args);
|
|
583
|
+
};
|
|
584
|
+
}
|
|
585
|
+
return context;
|
|
586
|
+
}
|
|
587
|
+
return getContext.call(this, contextType, ...args);
|
|
588
|
+
};
|
|
589
|
+
|
|
590
|
+
// === TIMEZONE AND LOCALE CONSISTENCY ===
|
|
591
|
+
if (Intl && Intl.DateTimeFormat) {
|
|
592
|
+
const originalResolvedOptions = Intl.DateTimeFormat.prototype.resolvedOptions;
|
|
593
|
+
Intl.DateTimeFormat.prototype.resolvedOptions = function() {
|
|
594
|
+
const options = originalResolvedOptions.call(this);
|
|
595
|
+
options.timeZone = 'America/New_York'; // Consistent timezone
|
|
596
|
+
return options;
|
|
597
|
+
};
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
// === SCREEN PROPERTIES ===
|
|
601
|
+
const screenProps = {
|
|
602
|
+
width: 1920,
|
|
603
|
+
height: 1080,
|
|
604
|
+
availWidth: 1920,
|
|
605
|
+
availHeight: 1040,
|
|
606
|
+
colorDepth: 24,
|
|
607
|
+
pixelDepth: 24,
|
|
608
|
+
orientation: {
|
|
609
|
+
angle: 0,
|
|
610
|
+
type: 'landscape-primary'
|
|
611
|
+
}
|
|
612
|
+
};
|
|
613
|
+
|
|
614
|
+
Object.keys(screenProps).forEach(prop => {
|
|
615
|
+
if (prop !== 'orientation') {
|
|
616
|
+
Object.defineProperty(screen, prop, {
|
|
617
|
+
get: () => screenProps[prop],
|
|
618
|
+
configurable: true
|
|
619
|
+
});
|
|
620
|
+
}
|
|
621
|
+
});
|
|
622
|
+
|
|
623
|
+
// === PREVENT TIMING ATTACKS ===
|
|
624
|
+
const originalNow = performance.now;
|
|
625
|
+
performance.now = function() {
|
|
626
|
+
return originalNow.call(this) + Math.random() * 0.1;
|
|
627
|
+
};
|
|
628
|
+
|
|
629
|
+
// === MEMORY INFO SPOOFING ===
|
|
630
|
+
if (performance.memory) {
|
|
631
|
+
Object.defineProperty(performance, 'memory', {
|
|
632
|
+
get: () => ({
|
|
633
|
+
jsHeapSizeLimit: 4294705152,
|
|
634
|
+
totalJSHeapSize: Math.floor(Math.random() * 50000000) + 10000000,
|
|
635
|
+
usedJSHeapSize: Math.floor(Math.random() * 30000000) + 5000000
|
|
636
|
+
}),
|
|
637
|
+
configurable: true
|
|
638
|
+
});
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
// === PREVENT IFRAME DETECTION ===
|
|
642
|
+
Object.defineProperty(window, 'top', {
|
|
643
|
+
get: () => window,
|
|
644
|
+
configurable: true
|
|
645
|
+
});
|
|
646
|
+
|
|
647
|
+
Object.defineProperty(window, 'parent', {
|
|
648
|
+
get: () => window,
|
|
649
|
+
configurable: true
|
|
650
|
+
});
|
|
651
|
+
|
|
652
|
+
// === CONSOLE CLEANING ===
|
|
653
|
+
const originalConsole = { ...console };
|
|
654
|
+
const cleanMethods = ['debug', 'log', 'info', 'warn', 'error'];
|
|
655
|
+
cleanMethods.forEach(method => {
|
|
656
|
+
console[method] = function(...args) {
|
|
657
|
+
const text = args.join(' ').toLowerCase();
|
|
658
|
+
if (text.includes('devtools') || text.includes('automation') ||
|
|
659
|
+
text.includes('webdriver') || text.includes('selenium') ||
|
|
660
|
+
text.includes('playwright') || text.includes('puppeteer')) {
|
|
661
|
+
return;
|
|
662
|
+
}
|
|
663
|
+
return originalConsole[method](...args);
|
|
664
|
+
};
|
|
665
|
+
});
|
|
666
|
+
|
|
667
|
+
// === MOUSE MOVEMENT SIMULATION ===
|
|
668
|
+
let mouseActivity = Date.now();
|
|
669
|
+
document.addEventListener('mousemove', () => {
|
|
670
|
+
mouseActivity = Date.now();
|
|
671
|
+
}, true);
|
|
672
|
+
|
|
673
|
+
// Simulate natural mouse movements
|
|
674
|
+
setInterval(() => {
|
|
675
|
+
if (Date.now() - mouseActivity > 30000) {
|
|
676
|
+
const event = new MouseEvent('mousemove', {
|
|
677
|
+
view: window,
|
|
678
|
+
bubbles: true,
|
|
679
|
+
cancelable: true,
|
|
680
|
+
clientX: Math.random() * window.innerWidth,
|
|
681
|
+
clientY: Math.random() * window.innerHeight
|
|
682
|
+
});
|
|
683
|
+
document.dispatchEvent(event);
|
|
684
|
+
}
|
|
685
|
+
}, 30000 + Math.random() * 15000);
|
|
686
|
+
|
|
687
|
+
// === FINAL CLEANUP ===
|
|
688
|
+
// Remove any remaining automation traces
|
|
689
|
+
delete window.cdc_adoQpoasnfa76pfcZLmcfl_;
|
|
690
|
+
delete window.$cdc_asdjflasutopfhvcZLmcfl_;
|
|
691
|
+
delete window.$chrome_asyncScriptInfo;
|
|
692
|
+
delete window.__$webdriverAsyncExecutor;
|
|
693
|
+
|
|
694
|
+
// Freeze important objects to prevent modification
|
|
695
|
+
try {
|
|
696
|
+
Object.freeze(navigator);
|
|
697
|
+
Object.freeze(screen);
|
|
698
|
+
} catch (e) {}
|
|
699
|
+
})();
|
|
700
|
+
"""
|
|
701
|
+
|
|
702
|
+
def get_dimension_spoofing_script(self) -> str:
|
|
703
|
+
"""
|
|
704
|
+
Generate comprehensive JavaScript dimension spoofing script for invisible browser mode.
|
|
705
|
+
|
|
706
|
+
This creates a dual-layer window configuration:
|
|
707
|
+
- Physical window: 1x1 pixel (invisible to user)
|
|
708
|
+
- Reported dimensions: 1920x1080 (natural desktop size for bot detection)
|
|
709
|
+
|
|
710
|
+
The script is bulletproof and handles all dimension-related APIs that
|
|
711
|
+
bot detectors commonly check.
|
|
712
|
+
"""
|
|
713
|
+
logger.debug(
|
|
714
|
+
f"{self.name}: stealth_manager.get_dimension_spoofing_script called."
|
|
715
|
+
)
|
|
716
|
+
if not self.spoof_dimension:
|
|
717
|
+
return ""
|
|
718
|
+
|
|
719
|
+
return """
|
|
720
|
+
(() => {
|
|
721
|
+
// Target dimensions to report to JavaScript (natural desktop)
|
|
722
|
+
const TARGET_WINDOW_WIDTH = 1920;
|
|
723
|
+
const TARGET_WINDOW_HEIGHT = 1080;
|
|
724
|
+
const TARGET_SCREEN_WIDTH = 1920;
|
|
725
|
+
const TARGET_SCREEN_HEIGHT = 1080;
|
|
726
|
+
const TARGET_AVAILABLE_WIDTH = 1920;
|
|
727
|
+
const TARGET_AVAILABLE_HEIGHT = 1040; // Account for taskbar
|
|
728
|
+
|
|
729
|
+
// === WINDOW DIMENSIONS ===
|
|
730
|
+
// Override all window size properties
|
|
731
|
+
Object.defineProperty(window, 'innerWidth', {
|
|
732
|
+
get: () => TARGET_WINDOW_WIDTH,
|
|
733
|
+
configurable: true
|
|
734
|
+
});
|
|
735
|
+
|
|
736
|
+
Object.defineProperty(window, 'innerHeight', {
|
|
737
|
+
get: () => TARGET_WINDOW_HEIGHT,
|
|
738
|
+
configurable: true
|
|
739
|
+
});
|
|
740
|
+
|
|
741
|
+
Object.defineProperty(window, 'outerWidth', {
|
|
742
|
+
get: () => TARGET_WINDOW_WIDTH,
|
|
743
|
+
configurable: true
|
|
744
|
+
});
|
|
745
|
+
|
|
746
|
+
Object.defineProperty(window, 'outerHeight', {
|
|
747
|
+
get: () => TARGET_WINDOW_HEIGHT + 100, // Account for browser chrome
|
|
748
|
+
configurable: true
|
|
749
|
+
});
|
|
750
|
+
|
|
751
|
+
// Override client dimensions (commonly checked by bot detectors)
|
|
752
|
+
if (document.documentElement) {
|
|
753
|
+
Object.defineProperty(document.documentElement, 'clientWidth', {
|
|
754
|
+
get: () => TARGET_WINDOW_WIDTH,
|
|
755
|
+
configurable: true
|
|
756
|
+
});
|
|
757
|
+
|
|
758
|
+
Object.defineProperty(document.documentElement, 'clientHeight', {
|
|
759
|
+
get: () => TARGET_WINDOW_HEIGHT,
|
|
760
|
+
configurable: true
|
|
761
|
+
});
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
// === SCREEN DIMENSIONS ===
|
|
765
|
+
// Override all screen properties
|
|
766
|
+
Object.defineProperty(window.screen, 'width', {
|
|
767
|
+
get: () => TARGET_SCREEN_WIDTH,
|
|
768
|
+
configurable: true
|
|
769
|
+
});
|
|
770
|
+
|
|
771
|
+
Object.defineProperty(window.screen, 'height', {
|
|
772
|
+
get: () => TARGET_SCREEN_HEIGHT,
|
|
773
|
+
configurable: true
|
|
774
|
+
});
|
|
775
|
+
|
|
776
|
+
Object.defineProperty(window.screen, 'availWidth', {
|
|
777
|
+
get: () => TARGET_AVAILABLE_WIDTH,
|
|
778
|
+
configurable: true
|
|
779
|
+
});
|
|
780
|
+
|
|
781
|
+
Object.defineProperty(window.screen, 'availHeight', {
|
|
782
|
+
get: () => TARGET_AVAILABLE_HEIGHT,
|
|
783
|
+
configurable: true
|
|
784
|
+
});
|
|
785
|
+
|
|
786
|
+
// === VIEWPORT AND VISUAL DIMENSIONS ===
|
|
787
|
+
// Override visual viewport (modern API)
|
|
788
|
+
if (window.visualViewport) {
|
|
789
|
+
Object.defineProperty(window.visualViewport, 'width', {
|
|
790
|
+
get: () => TARGET_WINDOW_WIDTH,
|
|
791
|
+
configurable: true
|
|
792
|
+
});
|
|
793
|
+
|
|
794
|
+
Object.defineProperty(window.visualViewport, 'height', {
|
|
795
|
+
get: () => TARGET_WINDOW_HEIGHT,
|
|
796
|
+
configurable: true
|
|
797
|
+
});
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
// === DOCUMENT DIMENSIONS ===
|
|
801
|
+
// Override document element dimensions (wait for DOM to be ready)
|
|
802
|
+
const overrideDocumentDimensions = () => {
|
|
803
|
+
if (document.documentElement) {
|
|
804
|
+
Object.defineProperty(document.documentElement, 'clientWidth', {
|
|
805
|
+
get: () => TARGET_WINDOW_WIDTH,
|
|
806
|
+
configurable: true
|
|
807
|
+
});
|
|
808
|
+
|
|
809
|
+
Object.defineProperty(document.documentElement, 'clientHeight', {
|
|
810
|
+
get: () => TARGET_WINDOW_HEIGHT,
|
|
811
|
+
configurable: true
|
|
812
|
+
});
|
|
813
|
+
|
|
814
|
+
Object.defineProperty(document.documentElement, 'offsetWidth', {
|
|
815
|
+
get: () => TARGET_WINDOW_WIDTH,
|
|
816
|
+
configurable: true
|
|
817
|
+
});
|
|
818
|
+
|
|
819
|
+
Object.defineProperty(document.documentElement, 'offsetHeight', {
|
|
820
|
+
get: () => TARGET_WINDOW_HEIGHT,
|
|
821
|
+
configurable: true
|
|
822
|
+
});
|
|
823
|
+
|
|
824
|
+
Object.defineProperty(document.documentElement, 'scrollWidth', {
|
|
825
|
+
get: () => TARGET_WINDOW_WIDTH,
|
|
826
|
+
configurable: true
|
|
827
|
+
});
|
|
828
|
+
|
|
829
|
+
Object.defineProperty(document.documentElement, 'scrollHeight', {
|
|
830
|
+
get: () => TARGET_WINDOW_HEIGHT,
|
|
831
|
+
configurable: true
|
|
832
|
+
});
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
if (document.body) {
|
|
836
|
+
Object.defineProperty(document.body, 'clientWidth', {
|
|
837
|
+
get: () => TARGET_WINDOW_WIDTH,
|
|
838
|
+
configurable: true
|
|
839
|
+
});
|
|
840
|
+
|
|
841
|
+
Object.defineProperty(document.body, 'clientHeight', {
|
|
842
|
+
get: () => TARGET_WINDOW_HEIGHT,
|
|
843
|
+
configurable: true
|
|
844
|
+
});
|
|
845
|
+
}
|
|
846
|
+
};
|
|
847
|
+
|
|
848
|
+
// Apply immediately if DOM is ready, otherwise wait
|
|
849
|
+
if (document.readyState === 'loading') {
|
|
850
|
+
document.addEventListener('DOMContentLoaded', overrideDocumentDimensions);
|
|
851
|
+
} else {
|
|
852
|
+
overrideDocumentDimensions();
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
// === MEDIA QUERIES ===
|
|
856
|
+
// Override matchMedia for responsive design queries
|
|
857
|
+
const originalMatchMedia = window.matchMedia;
|
|
858
|
+
window.matchMedia = function(query) {
|
|
859
|
+
const result = originalMatchMedia.call(this, query);
|
|
860
|
+
|
|
861
|
+
// Override common responsive breakpoints based on our spoofed dimensions
|
|
862
|
+
if (query.includes('max-width')) {
|
|
863
|
+
const maxWidth = parseInt(query.match(/max-width:\\s*(\d+)px/)?.[1] || '0');
|
|
864
|
+
if (maxWidth < TARGET_WINDOW_WIDTH) {
|
|
865
|
+
Object.defineProperty(result, 'matches', { get: () => false });
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
if (query.includes('min-width')) {
|
|
870
|
+
const minWidth = parseInt(query.match(/min-width:\\s*(\d+)px/)?.[1] || '0');
|
|
871
|
+
if (minWidth <= TARGET_WINDOW_WIDTH) {
|
|
872
|
+
Object.defineProperty(result, 'matches', { get: () => true });
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
return result;
|
|
877
|
+
};
|
|
878
|
+
|
|
879
|
+
// === EVENT HANDLING ===
|
|
880
|
+
// Override resize events to maintain consistency
|
|
881
|
+
const originalAddEventListener = window.addEventListener;
|
|
882
|
+
window.addEventListener = function(type, listener, options) {
|
|
883
|
+
if (type === 'resize') {
|
|
884
|
+
// Intercept resize events and provide spoofed dimensions
|
|
885
|
+
const wrappedListener = function(event) {
|
|
886
|
+
// Create a mock resize event with spoofed dimensions
|
|
887
|
+
const mockEvent = new Event('resize');
|
|
888
|
+
Object.defineProperty(mockEvent, 'target', {
|
|
889
|
+
value: {
|
|
890
|
+
innerWidth: TARGET_WINDOW_WIDTH,
|
|
891
|
+
innerHeight: TARGET_WINDOW_HEIGHT
|
|
892
|
+
}
|
|
893
|
+
});
|
|
894
|
+
return listener.call(this, mockEvent);
|
|
895
|
+
};
|
|
896
|
+
return originalAddEventListener.call(this, type, wrappedListener, options);
|
|
897
|
+
}
|
|
898
|
+
return originalAddEventListener.call(this, type, listener, options);
|
|
899
|
+
};
|
|
900
|
+
})();
|
|
901
|
+
"""
|
|
902
|
+
|
|
903
|
+
async def human_delay_async(self, min_ms: int = 1000, max_ms: int = 3000):
|
|
904
|
+
delay = random.randint(min_ms, max_ms)
|
|
905
|
+
await asyncio.sleep(delay / 1000)
|
|
906
|
+
|
|
907
|
+
async def human_click_async(self, page: Page, element):
|
|
908
|
+
await element.hover()
|
|
909
|
+
await self.human_delay_async(200, 500)
|
|
910
|
+
await element.click()
|
|
911
|
+
|
|
912
|
+
async def human_mouse_move_async(self, page: Page):
|
|
913
|
+
await page.mouse.move(
|
|
914
|
+
random.randint(100, 800), random.randint(100, 600)
|
|
915
|
+
)
|
|
916
|
+
|
|
917
|
+
async def human_scroll_async(self, page: Page):
|
|
918
|
+
scroll_distance = random.randint(300, 800)
|
|
919
|
+
await page.evaluate(f"window.scrollBy(0, {scroll_distance})")
|
|
920
|
+
await self.human_delay_async(500, 1500)
|
|
921
|
+
|
|
922
|
+
async def human_type_async(self, page: Page, selector: str, text: str):
|
|
923
|
+
element = page.locator(selector)
|
|
924
|
+
await element.click()
|
|
925
|
+
for char in text:
|
|
926
|
+
await element.type(char)
|
|
927
|
+
await self.human_delay_async(50, 200)
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
def main(args):
|
|
931
|
+
"""Demonstrate StealthManager functionality."""
|
|
932
|
+
import asyncio
|
|
933
|
+
|
|
934
|
+
from playwright.async_api import async_playwright
|
|
935
|
+
|
|
936
|
+
async def demo():
|
|
937
|
+
stealth_manager = StealthManager()
|
|
938
|
+
|
|
939
|
+
async with async_playwright() as p:
|
|
940
|
+
browser = await p.chromium.launch(
|
|
941
|
+
headless=False,
|
|
942
|
+
args=stealth_manager.get_stealth_options_additional(),
|
|
943
|
+
)
|
|
944
|
+
|
|
945
|
+
stealth_options = stealth_manager.get_stealth_options()
|
|
946
|
+
context = await browser.new_context(**stealth_options)
|
|
947
|
+
await context.add_init_script(stealth_manager.get_init_script())
|
|
948
|
+
|
|
949
|
+
page = await context.new_page()
|
|
950
|
+
await page.goto("https://bot.sannysoft.com/", timeout=30000)
|
|
951
|
+
await stealth_manager.human_delay_async(2000, 3000)
|
|
952
|
+
|
|
953
|
+
await page.screenshot(path="/tmp/stealth_test.png")
|
|
954
|
+
print("✓ Stealth test complete: /tmp/stealth_test.png")
|
|
955
|
+
|
|
956
|
+
await browser.close()
|
|
957
|
+
|
|
958
|
+
asyncio.run(demo())
|
|
959
|
+
return 0
|
|
960
|
+
|
|
961
|
+
|
|
962
|
+
def parse_args():
|
|
963
|
+
"""Parse command line arguments."""
|
|
964
|
+
import argparse
|
|
965
|
+
|
|
966
|
+
parser = argparse.ArgumentParser(description="StealthManager demo")
|
|
967
|
+
return parser.parse_args()
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def run_main() -> None:
|
|
971
|
+
"""Initialize scitex framework, run main function, and cleanup."""
|
|
972
|
+
global CONFIG, CC, sys, plt, rng
|
|
973
|
+
|
|
974
|
+
import sys
|
|
975
|
+
|
|
976
|
+
import matplotlib.pyplot as plt
|
|
977
|
+
|
|
978
|
+
import scitex as stx
|
|
979
|
+
|
|
980
|
+
args = parse_args()
|
|
981
|
+
|
|
982
|
+
CONFIG, sys.stdout, sys.stderr, plt, CC, rng = stx.session.start(
|
|
983
|
+
sys,
|
|
984
|
+
plt,
|
|
985
|
+
args=args,
|
|
986
|
+
file=__FILE__,
|
|
987
|
+
sdir_suffix=None,
|
|
988
|
+
verbose=False,
|
|
989
|
+
agg=True,
|
|
990
|
+
)
|
|
991
|
+
|
|
992
|
+
exit_status = main(args)
|
|
993
|
+
|
|
994
|
+
stx.session.close(
|
|
995
|
+
CONFIG,
|
|
996
|
+
verbose=False,
|
|
997
|
+
notify=False,
|
|
998
|
+
message="",
|
|
999
|
+
exit_status=exit_status,
|
|
1000
|
+
)
|
|
1001
|
+
|
|
1002
|
+
|
|
1003
|
+
if __name__ == "__main__":
|
|
1004
|
+
run_main()
|
|
1005
|
+
|
|
1006
|
+
# python -m scitex.browser.stealth.StealthManager
|
|
1007
|
+
|
|
1008
|
+
# EOF
|