scitex 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scitex/__init__.py +19 -8
- scitex/__main__.py +2 -1
- scitex/__version__.py +1 -1
- scitex/_optional_deps.py +13 -20
- scitex/ai/__init__.py +5 -0
- scitex/ai/_gen_ai/_Anthropic.py +3 -1
- scitex/ai/_gen_ai/_BaseGenAI.py +3 -2
- scitex/ai/_gen_ai/_DeepSeek.py +1 -1
- scitex/ai/_gen_ai/_Google.py +3 -2
- scitex/ai/_gen_ai/_Llama.py +4 -2
- scitex/ai/_gen_ai/_OpenAI.py +3 -1
- scitex/ai/_gen_ai/_PARAMS.py +1 -0
- scitex/ai/_gen_ai/_Perplexity.py +3 -1
- scitex/ai/_gen_ai/__init__.py +1 -0
- scitex/ai/_gen_ai/_format_output_func.py +3 -1
- scitex/ai/classification/CrossValidationExperiment.py +8 -14
- scitex/ai/classification/examples/timeseries_cv_demo.py +128 -112
- scitex/ai/classification/reporters/_BaseClassificationReporter.py +2 -0
- scitex/ai/classification/reporters/_ClassificationReporter.py +30 -45
- scitex/ai/classification/reporters/_MultiClassificationReporter.py +8 -11
- scitex/ai/classification/reporters/_SingleClassificationReporter.py +126 -182
- scitex/ai/classification/reporters/__init__.py +1 -1
- scitex/ai/classification/reporters/reporter_utils/_Plotter.py +213 -119
- scitex/ai/classification/reporters/reporter_utils/__init__.py +28 -36
- scitex/ai/classification/reporters/reporter_utils/aggregation.py +125 -143
- scitex/ai/classification/reporters/reporter_utils/data_models.py +128 -120
- scitex/ai/classification/reporters/reporter_utils/reporting.py +507 -340
- scitex/ai/classification/reporters/reporter_utils/storage.py +4 -1
- scitex/ai/classification/reporters/reporter_utils/validation.py +141 -154
- scitex/ai/classification/timeseries/_TimeSeriesBlockingSplit.py +204 -129
- scitex/ai/classification/timeseries/_TimeSeriesCalendarSplit.py +215 -171
- scitex/ai/classification/timeseries/_TimeSeriesMetadata.py +17 -17
- scitex/ai/classification/timeseries/_TimeSeriesSlidingWindowSplit.py +67 -143
- scitex/ai/classification/timeseries/_TimeSeriesSlidingWindowSplit_v01-not-using-n_splits.py +67 -143
- scitex/ai/classification/timeseries/_TimeSeriesStrategy.py +12 -13
- scitex/ai/classification/timeseries/_TimeSeriesStratifiedSplit.py +231 -144
- scitex/ai/classification/timeseries/__init__.py +2 -4
- scitex/ai/classification/timeseries/_normalize_timestamp.py +3 -0
- scitex/ai/clustering/_pca.py +0 -1
- scitex/ai/clustering/_umap.py +1 -2
- scitex/ai/feature_extraction/__init__.py +10 -8
- scitex/ai/feature_extraction/vit.py +0 -1
- scitex/ai/feature_selection/feature_selection.py +3 -8
- scitex/ai/metrics/_calc_conf_mat.py +2 -0
- scitex/ai/metrics/_calc_feature_importance.py +3 -7
- scitex/ai/metrics/_calc_pre_rec_auc.py +5 -5
- scitex/ai/metrics/_calc_roc_auc.py +4 -2
- scitex/ai/metrics/_calc_seizure_prediction_metrics.py +35 -20
- scitex/ai/metrics/_calc_silhouette_score.py +1 -3
- scitex/ai/optim/Ranger_Deep_Learning_Optimizer/ranger/ranger.py +0 -3
- scitex/ai/optim/Ranger_Deep_Learning_Optimizer/ranger/ranger2020.py +0 -3
- scitex/ai/optim/Ranger_Deep_Learning_Optimizer/ranger/ranger913A.py +0 -3
- scitex/ai/optim/_optimizers.py +1 -1
- scitex/ai/plt/__init__.py +6 -1
- scitex/ai/plt/_plot_feature_importance.py +1 -3
- scitex/ai/plt/_plot_learning_curve.py +9 -24
- scitex/ai/plt/_plot_optuna_study.py +4 -3
- scitex/ai/plt/_plot_pre_rec_curve.py +9 -15
- scitex/ai/plt/_plot_roc_curve.py +6 -8
- scitex/ai/plt/_stx_conf_mat.py +121 -122
- scitex/ai/sampling/undersample.py +3 -2
- scitex/ai/sklearn/__init__.py +2 -2
- scitex/ai/training/_LearningCurveLogger.py +23 -10
- scitex/ai/utils/_check_params.py +0 -1
- scitex/benchmark/__init__.py +15 -25
- scitex/benchmark/benchmark.py +124 -117
- scitex/benchmark/monitor.py +117 -107
- scitex/benchmark/profiler.py +61 -58
- scitex/bridge/__init__.py +110 -0
- scitex/bridge/_helpers.py +149 -0
- scitex/bridge/_plt_vis.py +529 -0
- scitex/bridge/_protocol.py +283 -0
- scitex/bridge/_stats_plt.py +261 -0
- scitex/bridge/_stats_vis.py +265 -0
- scitex/browser/__init__.py +0 -2
- scitex/browser/auth/__init__.py +0 -0
- scitex/browser/auth/google.py +16 -11
- scitex/browser/automation/CookieHandler.py +2 -3
- scitex/browser/collaboration/__init__.py +3 -0
- scitex/browser/collaboration/auth_helpers.py +3 -1
- scitex/browser/collaboration/collaborative_agent.py +2 -0
- scitex/browser/collaboration/interactive_panel.py +2 -2
- scitex/browser/collaboration/shared_session.py +20 -11
- scitex/browser/collaboration/standard_interactions.py +1 -0
- scitex/browser/core/BrowserMixin.py +12 -30
- scitex/browser/core/ChromeProfileManager.py +9 -24
- scitex/browser/debugging/_browser_logger.py +15 -25
- scitex/browser/debugging/_failure_capture.py +9 -2
- scitex/browser/debugging/_highlight_element.py +15 -6
- scitex/browser/debugging/_show_grid.py +5 -6
- scitex/browser/debugging/_sync_session.py +4 -3
- scitex/browser/debugging/_test_monitor.py +14 -5
- scitex/browser/debugging/_visual_cursor.py +46 -35
- scitex/browser/interaction/click_center.py +4 -3
- scitex/browser/interaction/click_with_fallbacks.py +7 -10
- scitex/browser/interaction/close_popups.py +79 -66
- scitex/browser/interaction/fill_with_fallbacks.py +8 -8
- scitex/browser/pdf/__init__.py +3 -1
- scitex/browser/pdf/click_download_for_chrome_pdf_viewer.py +11 -10
- scitex/browser/pdf/detect_chrome_pdf_viewer.py +3 -6
- scitex/browser/remote/CaptchaHandler.py +109 -96
- scitex/browser/remote/ZenRowsAPIClient.py +91 -97
- scitex/browser/remote/ZenRowsBrowserManager.py +138 -112
- scitex/browser/stealth/HumanBehavior.py +4 -9
- scitex/browser/stealth/StealthManager.py +11 -26
- scitex/capture/__init__.py +17 -17
- scitex/capture/__main__.py +2 -3
- scitex/capture/capture.py +23 -51
- scitex/capture/cli.py +14 -39
- scitex/capture/gif.py +5 -9
- scitex/capture/mcp_server.py +7 -20
- scitex/capture/session.py +4 -3
- scitex/capture/utils.py +18 -53
- scitex/cli/__init__.py +1 -1
- scitex/cli/cloud.py +158 -116
- scitex/cli/config.py +224 -0
- scitex/cli/main.py +41 -40
- scitex/cli/scholar.py +60 -27
- scitex/cli/security.py +14 -20
- scitex/cli/web.py +87 -90
- scitex/cli/writer.py +51 -45
- scitex/cloud/__init__.py +14 -11
- scitex/cloud/_matplotlib_hook.py +6 -6
- scitex/config/README.md +313 -0
- scitex/config/{PriorityConfig.py → _PriorityConfig.py} +114 -17
- scitex/config/_ScitexConfig.py +319 -0
- scitex/config/__init__.py +41 -9
- scitex/config/_paths.py +325 -0
- scitex/config/default.yaml +81 -0
- scitex/context/_suppress_output.py +2 -3
- scitex/db/_BaseMixins/_BaseBackupMixin.py +3 -1
- scitex/db/_BaseMixins/_BaseBatchMixin.py +3 -1
- scitex/db/_BaseMixins/_BaseBlobMixin.py +3 -1
- scitex/db/_BaseMixins/_BaseImportExportMixin.py +1 -3
- scitex/db/_BaseMixins/_BaseIndexMixin.py +3 -1
- scitex/db/_BaseMixins/_BaseMaintenanceMixin.py +1 -3
- scitex/db/_BaseMixins/_BaseQueryMixin.py +3 -1
- scitex/db/_BaseMixins/_BaseRowMixin.py +3 -1
- scitex/db/_BaseMixins/_BaseTableMixin.py +3 -1
- scitex/db/_BaseMixins/_BaseTransactionMixin.py +1 -3
- scitex/db/_BaseMixins/__init__.py +1 -1
- scitex/db/__init__.py +9 -1
- scitex/db/__main__.py +8 -21
- scitex/db/_check_health.py +15 -31
- scitex/db/_delete_duplicates.py +7 -4
- scitex/db/_inspect.py +22 -38
- scitex/db/_inspect_optimized.py +89 -85
- scitex/db/_postgresql/_PostgreSQL.py +0 -1
- scitex/db/_postgresql/_PostgreSQLMixins/_BlobMixin.py +3 -1
- scitex/db/_postgresql/_PostgreSQLMixins/_ConnectionMixin.py +1 -3
- scitex/db/_postgresql/_PostgreSQLMixins/_ImportExportMixin.py +1 -3
- scitex/db/_postgresql/_PostgreSQLMixins/_MaintenanceMixin.py +1 -4
- scitex/db/_postgresql/_PostgreSQLMixins/_QueryMixin.py +3 -3
- scitex/db/_postgresql/_PostgreSQLMixins/_RowMixin.py +3 -1
- scitex/db/_postgresql/_PostgreSQLMixins/_TransactionMixin.py +1 -3
- scitex/db/_postgresql/__init__.py +1 -1
- scitex/db/_sqlite3/_SQLite3.py +2 -4
- scitex/db/_sqlite3/_SQLite3Mixins/_ArrayMixin.py +11 -12
- scitex/db/_sqlite3/_SQLite3Mixins/_ArrayMixin_v01-need-_hash-col.py +19 -14
- scitex/db/_sqlite3/_SQLite3Mixins/_BatchMixin.py +3 -1
- scitex/db/_sqlite3/_SQLite3Mixins/_BlobMixin.py +7 -7
- scitex/db/_sqlite3/_SQLite3Mixins/_ColumnMixin.py +118 -111
- scitex/db/_sqlite3/_SQLite3Mixins/_ConnectionMixin.py +8 -10
- scitex/db/_sqlite3/_SQLite3Mixins/_GitMixin.py +17 -45
- scitex/db/_sqlite3/_SQLite3Mixins/_ImportExportMixin.py +1 -3
- scitex/db/_sqlite3/_SQLite3Mixins/_IndexMixin.py +3 -1
- scitex/db/_sqlite3/_SQLite3Mixins/_QueryMixin.py +3 -4
- scitex/db/_sqlite3/_SQLite3Mixins/_RowMixin.py +9 -9
- scitex/db/_sqlite3/_SQLite3Mixins/_TableMixin.py +18 -11
- scitex/db/_sqlite3/_SQLite3Mixins/__init__.py +1 -0
- scitex/db/_sqlite3/__init__.py +1 -1
- scitex/db/_sqlite3/_delete_duplicates.py +13 -11
- scitex/decorators/__init__.py +29 -4
- scitex/decorators/_auto_order.py +43 -43
- scitex/decorators/_batch_fn.py +12 -6
- scitex/decorators/_cache_disk.py +8 -9
- scitex/decorators/_cache_disk_async.py +8 -7
- scitex/decorators/_combined.py +19 -13
- scitex/decorators/_converters.py +16 -3
- scitex/decorators/_deprecated.py +32 -22
- scitex/decorators/_numpy_fn.py +18 -4
- scitex/decorators/_pandas_fn.py +17 -5
- scitex/decorators/_signal_fn.py +17 -3
- scitex/decorators/_torch_fn.py +32 -15
- scitex/decorators/_xarray_fn.py +23 -9
- scitex/dev/_analyze_code_flow.py +0 -2
- scitex/dict/_DotDict.py +15 -19
- scitex/dict/_flatten.py +1 -0
- scitex/dict/_listed_dict.py +1 -0
- scitex/dict/_pop_keys.py +1 -0
- scitex/dict/_replace.py +1 -0
- scitex/dict/_safe_merge.py +1 -0
- scitex/dict/_to_str.py +2 -3
- scitex/dsp/__init__.py +13 -4
- scitex/dsp/_crop.py +3 -1
- scitex/dsp/_detect_ripples.py +3 -1
- scitex/dsp/_modulation_index.py +3 -1
- scitex/dsp/_time.py +3 -1
- scitex/dsp/_wavelet.py +0 -1
- scitex/dsp/example.py +0 -5
- scitex/dsp/filt.py +4 -0
- scitex/dsp/utils/__init__.py +4 -1
- scitex/dsp/utils/pac.py +3 -3
- scitex/dt/_normalize_timestamp.py +4 -1
- scitex/errors.py +3 -6
- scitex/etc/__init__.py +1 -1
- scitex/gen/_DimHandler.py +6 -6
- scitex/gen/__init__.py +5 -1
- scitex/gen/_deprecated_close.py +1 -0
- scitex/gen/_deprecated_start.py +5 -3
- scitex/gen/_detect_environment.py +44 -41
- scitex/gen/_detect_notebook_path.py +51 -47
- scitex/gen/_embed.py +1 -1
- scitex/gen/_get_notebook_path.py +81 -62
- scitex/gen/_inspect_module.py +0 -1
- scitex/gen/_norm.py +16 -7
- scitex/gen/_norm_cache.py +78 -65
- scitex/gen/_print_config.py +0 -3
- scitex/gen/_src.py +2 -3
- scitex/gen/_title_case.py +3 -2
- scitex/gen/_to_even.py +8 -8
- scitex/gen/_transpose.py +3 -3
- scitex/gen/misc.py +0 -3
- scitex/gists/_SigMacro_processFigure_S.py +2 -2
- scitex/gists/_SigMacro_toBlue.py +2 -2
- scitex/gists/__init__.py +4 -1
- scitex/git/_branch.py +19 -11
- scitex/git/_clone.py +23 -15
- scitex/git/_commit.py +10 -12
- scitex/git/_init.py +15 -38
- scitex/git/_remote.py +9 -3
- scitex/git/_result.py +3 -0
- scitex/git/_retry.py +2 -5
- scitex/git/_types.py +4 -0
- scitex/git/_validation.py +8 -8
- scitex/git/_workflow.py +4 -4
- scitex/io/__init__.py +2 -1
- scitex/io/_glob.py +2 -2
- scitex/io/_json2md.py +3 -3
- scitex/io/_load.py +6 -8
- scitex/io/_load_cache.py +71 -71
- scitex/io/_load_configs.py +2 -3
- scitex/io/_load_modules/_H5Explorer.py +6 -12
- scitex/io/_load_modules/_ZarrExplorer.py +3 -3
- scitex/io/_load_modules/_bibtex.py +62 -63
- scitex/io/_load_modules/_canvas.py +4 -9
- scitex/io/_load_modules/_catboost.py +7 -2
- scitex/io/_load_modules/_hdf5.py +2 -0
- scitex/io/_load_modules/_image.py +5 -1
- scitex/io/_load_modules/_matlab.py +3 -1
- scitex/io/_load_modules/_optuna.py +0 -1
- scitex/io/_load_modules/_pdf.py +38 -29
- scitex/io/_load_modules/_sqlite3.py +1 -0
- scitex/io/_load_modules/_txt.py +2 -0
- scitex/io/_load_modules/_xml.py +9 -9
- scitex/io/_load_modules/_zarr.py +12 -10
- scitex/io/_metadata.py +76 -37
- scitex/io/_qr_utils.py +18 -13
- scitex/io/_save.py +220 -63
- scitex/io/_save_modules/__init__.py +7 -2
- scitex/io/_save_modules/_bibtex.py +66 -61
- scitex/io/_save_modules/_canvas.py +5 -6
- scitex/io/_save_modules/_catboost.py +2 -2
- scitex/io/_save_modules/_csv.py +4 -4
- scitex/io/_save_modules/_excel.py +5 -9
- scitex/io/_save_modules/_hdf5.py +9 -21
- scitex/io/_save_modules/_html.py +5 -5
- scitex/io/_save_modules/_image.py +105 -8
- scitex/io/_save_modules/_joblib.py +2 -2
- scitex/io/_save_modules/_json.py +51 -6
- scitex/io/_save_modules/_listed_dfs_as_csv.py +2 -1
- scitex/io/_save_modules/_listed_scalars_as_csv.py +2 -1
- scitex/io/_save_modules/_matlab.py +2 -2
- scitex/io/_save_modules/_numpy.py +6 -8
- scitex/io/_save_modules/_pickle.py +4 -4
- scitex/io/_save_modules/_plotly.py +3 -3
- scitex/io/_save_modules/_tex.py +23 -25
- scitex/io/_save_modules/_text.py +2 -2
- scitex/io/_save_modules/_yaml.py +9 -9
- scitex/io/_save_modules/_zarr.py +15 -15
- scitex/io/utils/__init__.py +2 -1
- scitex/io/utils/h5_to_zarr.py +173 -155
- scitex/linalg/__init__.py +1 -1
- scitex/linalg/_geometric_median.py +4 -3
- scitex/logging/_Tee.py +5 -7
- scitex/logging/__init__.py +18 -19
- scitex/logging/_config.py +4 -1
- scitex/logging/_context.py +6 -5
- scitex/logging/_formatters.py +2 -3
- scitex/logging/_handlers.py +19 -20
- scitex/logging/_levels.py +9 -17
- scitex/logging/_logger.py +74 -15
- scitex/logging/_print_capture.py +17 -17
- scitex/nn/_BNet.py +1 -3
- scitex/nn/_Filters.py +6 -2
- scitex/nn/_ModulationIndex.py +3 -1
- scitex/nn/_PAC.py +3 -2
- scitex/nn/_PSD.py +0 -1
- scitex/nn/__init__.py +16 -3
- scitex/path/_clean.py +10 -8
- scitex/path/_find.py +1 -1
- scitex/path/_get_spath.py +1 -2
- scitex/path/_mk_spath.py +1 -1
- scitex/path/_symlink.py +5 -10
- scitex/pd/__init__.py +4 -1
- scitex/pd/_force_df.py +24 -24
- scitex/pd/_get_unique.py +1 -0
- scitex/pd/_merge_columns.py +1 -1
- scitex/pd/_round.py +11 -7
- scitex/pd/_to_xy.py +0 -1
- scitex/plt/REQUESTS.md +191 -0
- scitex/plt/__init__.py +185 -87
- scitex/plt/_subplots/_AxesWrapper.py +22 -6
- scitex/plt/_subplots/_AxisWrapper.py +100 -39
- scitex/plt/_subplots/_AxisWrapperMixins/_AdjustmentMixin.py +74 -52
- scitex/plt/_subplots/_AxisWrapperMixins/_MatplotlibPlotMixin.py +183 -73
- scitex/plt/_subplots/_AxisWrapperMixins/_SeabornMixin.py +61 -45
- scitex/plt/_subplots/_AxisWrapperMixins/_TrackingMixin.py +26 -14
- scitex/plt/_subplots/_AxisWrapperMixins/_UnitAwareMixin.py +80 -73
- scitex/plt/_subplots/_FigWrapper.py +93 -60
- scitex/plt/_subplots/_SubplotsWrapper.py +135 -68
- scitex/plt/_subplots/__init__.py +10 -0
- scitex/plt/_subplots/_export_as_csv.py +89 -47
- scitex/plt/_subplots/_export_as_csv_formatters/__init__.py +1 -0
- scitex/plt/_subplots/_export_as_csv_formatters/_format_annotate.py +6 -4
- scitex/plt/_subplots/_export_as_csv_formatters/_format_bar.py +88 -38
- scitex/plt/_subplots/_export_as_csv_formatters/_format_barh.py +25 -31
- scitex/plt/_subplots/_export_as_csv_formatters/_format_boxplot.py +53 -23
- scitex/plt/_subplots/_export_as_csv_formatters/_format_contour.py +38 -25
- scitex/plt/_subplots/_export_as_csv_formatters/_format_contourf.py +17 -9
- scitex/plt/_subplots/_export_as_csv_formatters/_format_errorbar.py +70 -124
- scitex/plt/_subplots/_export_as_csv_formatters/_format_eventplot.py +12 -10
- scitex/plt/_subplots/_export_as_csv_formatters/_format_fill.py +31 -17
- scitex/plt/_subplots/_export_as_csv_formatters/_format_fill_between.py +33 -21
- scitex/plt/_subplots/_export_as_csv_formatters/_format_hexbin.py +14 -4
- scitex/plt/_subplots/_export_as_csv_formatters/_format_hist.py +43 -29
- scitex/plt/_subplots/_export_as_csv_formatters/_format_hist2d.py +14 -4
- scitex/plt/_subplots/_export_as_csv_formatters/_format_imshow.py +27 -11
- scitex/plt/_subplots/_export_as_csv_formatters/_format_imshow2d.py +7 -5
- scitex/plt/_subplots/_export_as_csv_formatters/_format_matshow.py +9 -7
- scitex/plt/_subplots/_export_as_csv_formatters/_format_pie.py +15 -6
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot.py +85 -46
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_box.py +52 -27
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_imshow.py +1 -0
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_kde.py +16 -17
- scitex/plt/_subplots/_export_as_csv_formatters/_format_plot_scatter.py +7 -5
- scitex/plt/_subplots/_export_as_csv_formatters/_format_quiver.py +10 -8
- scitex/plt/_subplots/_export_as_csv_formatters/_format_scatter.py +17 -6
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_barplot.py +43 -26
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_boxplot.py +68 -47
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_heatmap.py +52 -64
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_histplot.py +55 -50
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_jointplot.py +9 -11
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_kdeplot.py +63 -29
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_lineplot.py +4 -4
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_pairplot.py +6 -4
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_scatterplot.py +44 -40
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_stripplot.py +46 -39
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_swarmplot.py +46 -39
- scitex/plt/_subplots/_export_as_csv_formatters/_format_sns_violinplot.py +75 -94
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stem.py +12 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_step.py +12 -3
- scitex/plt/_subplots/_export_as_csv_formatters/_format_streamplot.py +10 -8
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_conf_mat.py +17 -15
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_ecdf.py +10 -9
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_fillv.py +35 -31
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_heatmap.py +18 -18
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_image.py +24 -18
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_joyplot.py +9 -7
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_line.py +34 -23
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_mean_ci.py +15 -13
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_mean_std.py +12 -10
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_median_iqr.py +15 -13
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_raster.py +11 -9
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_rectangle.py +84 -56
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_scatter_hist.py +35 -32
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_shaded_line.py +46 -30
- scitex/plt/_subplots/_export_as_csv_formatters/_format_stx_violin.py +51 -51
- scitex/plt/_subplots/_export_as_csv_formatters/_format_text.py +32 -31
- scitex/plt/_subplots/_export_as_csv_formatters/_format_violin.py +34 -31
- scitex/plt/_subplots/_export_as_csv_formatters/_format_violinplot.py +44 -37
- scitex/plt/_subplots/_export_as_csv_formatters/verify_formatters.py +91 -74
- scitex/plt/_tpl.py +6 -5
- scitex/plt/ax/_plot/__init__.py +24 -0
- scitex/plt/ax/_plot/_add_fitted_line.py +12 -11
- scitex/plt/ax/_plot/_plot_circular_hist.py +3 -1
- scitex/plt/ax/_plot/_plot_statistical_shaded_line.py +25 -19
- scitex/plt/ax/_plot/_stx_conf_mat.py +6 -3
- scitex/plt/ax/_plot/_stx_ecdf.py +5 -3
- scitex/plt/ax/_plot/_stx_fillv.py +4 -2
- scitex/plt/ax/_plot/_stx_heatmap.py +7 -4
- scitex/plt/ax/_plot/_stx_image.py +7 -5
- scitex/plt/ax/_plot/_stx_joyplot.py +32 -10
- scitex/plt/ax/_plot/_stx_raster.py +26 -11
- scitex/plt/ax/_plot/_stx_rectangle.py +2 -2
- scitex/plt/ax/_plot/_stx_shaded_line.py +15 -11
- scitex/plt/ax/_plot/_stx_violin.py +3 -1
- scitex/plt/ax/_style/_add_marginal_ax.py +6 -4
- scitex/plt/ax/_style/_auto_scale_axis.py +14 -10
- scitex/plt/ax/_style/_extend.py +3 -1
- scitex/plt/ax/_style/_force_aspect.py +5 -3
- scitex/plt/ax/_style/_format_units.py +2 -2
- scitex/plt/ax/_style/_hide_spines.py +5 -1
- scitex/plt/ax/_style/_map_ticks.py +5 -3
- scitex/plt/ax/_style/_rotate_labels.py +5 -4
- scitex/plt/ax/_style/_rotate_labels_v01.py +73 -63
- scitex/plt/ax/_style/_set_log_scale.py +120 -85
- scitex/plt/ax/_style/_set_meta.py +99 -76
- scitex/plt/ax/_style/_set_supxyt.py +33 -16
- scitex/plt/ax/_style/_set_xyt.py +27 -18
- scitex/plt/ax/_style/_share_axes.py +15 -5
- scitex/plt/ax/_style/_show_spines.py +58 -57
- scitex/plt/ax/_style/_style_barplot.py +1 -1
- scitex/plt/ax/_style/_style_boxplot.py +25 -14
- scitex/plt/ax/_style/_style_errorbar.py +0 -0
- scitex/plt/ax/_style/_style_scatter.py +1 -1
- scitex/plt/ax/_style/_style_suptitles.py +3 -3
- scitex/plt/ax/_style/_style_violinplot.py +8 -2
- scitex/plt/color/__init__.py +34 -2
- scitex/plt/color/_add_hue_col.py +1 -0
- scitex/plt/color/_colors.py +0 -1
- scitex/plt/color/_get_colors_from_conf_matap.py +3 -1
- scitex/plt/color/_vizualize_colors.py +0 -1
- scitex/plt/docs/FIGURE_ARCHITECTURE.md +155 -97
- scitex/plt/gallery/README.md +75 -0
- scitex/plt/gallery/__init__.py +29 -0
- scitex/plt/gallery/_generate.py +153 -0
- scitex/plt/gallery/_plots.py +594 -0
- scitex/plt/gallery/_registry.py +153 -0
- scitex/plt/styles/__init__.py +9 -9
- scitex/plt/styles/_plot_defaults.py +62 -61
- scitex/plt/styles/_plot_postprocess.py +126 -77
- scitex/plt/styles/_style_loader.py +0 -0
- scitex/plt/styles/presets.py +43 -18
- scitex/plt/templates/research-master/scitex/vis/gallery/area/fill_between.json +110 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/area/fill_betweenx.json +88 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/area/stx_fill_between.json +103 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/area/stx_fillv.json +106 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/categorical/bar.json +92 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/categorical/barh.json +92 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/categorical/boxplot.json +92 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/categorical/stx_bar.json +84 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/categorical/stx_barh.json +84 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/categorical/stx_box.json +83 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/categorical/stx_boxplot.json +93 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/categorical/stx_violin.json +91 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/categorical/stx_violinplot.json +91 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/categorical/violinplot.json +91 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/contour/contour.json +97 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/contour/contourf.json +98 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/contour/stx_contour.json +84 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/distribution/hist.json +101 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/distribution/hist2d.json +96 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/distribution/stx_ecdf.json +95 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/distribution/stx_joyplot.json +95 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/distribution/stx_kde.json +93 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/grid/imshow.json +95 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/grid/matshow.json +95 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/grid/stx_conf_mat.json +83 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/grid/stx_heatmap.json +92 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/grid/stx_image.json +121 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/grid/stx_imshow.json +84 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/line/plot.json +110 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/line/step.json +92 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/line/stx_line.json +95 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/line/stx_shaded_line.json +96 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/scatter/hexbin.json +95 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/scatter/scatter.json +95 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/scatter/stem.json +92 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/scatter/stx_scatter.json +84 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/special/pie.json +94 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/special/stx_raster.json +109 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/special/stx_rectangle.json +108 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/statistical/errorbar.json +93 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/statistical/stx_errorbar.json +84 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/statistical/stx_mean_ci.json +96 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/statistical/stx_mean_std.json +96 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/statistical/stx_median_iqr.json +96 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/vector/quiver.json +99 -0
- scitex/plt/templates/research-master/scitex/vis/gallery/vector/streamplot.json +100 -0
- scitex/plt/utils/__init__.py +29 -2
- scitex/plt/utils/_close.py +8 -3
- scitex/plt/utils/_collect_figure_metadata.py +3031 -265
- scitex/plt/utils/_colorbar.py +15 -17
- scitex/plt/utils/_configure_mpl.py +22 -14
- scitex/plt/utils/_crop.py +60 -27
- scitex/plt/utils/_csv_column_naming.py +123 -72
- scitex/plt/utils/_dimension_viewer.py +7 -19
- scitex/plt/utils/_figure_from_axes_mm.py +70 -16
- scitex/plt/utils/_figure_mm.py +3 -2
- scitex/plt/utils/_get_actual_font.py +5 -4
- scitex/plt/utils/_histogram_utils.py +52 -48
- scitex/plt/utils/_is_valid_axis.py +19 -13
- scitex/plt/utils/_mk_colorbar.py +3 -3
- scitex/plt/utils/_scientific_captions.py +202 -139
- scitex/plt/utils/_scitex_config.py +98 -98
- scitex/plt/utils/_units.py +0 -0
- scitex/plt/utils/metadata/__init__.py +36 -0
- scitex/plt/utils/metadata/_artist_extraction.py +119 -0
- scitex/plt/utils/metadata/_axes_metadata.py +93 -0
- scitex/plt/utils/metadata/_collection_artists.py +292 -0
- scitex/plt/utils/metadata/_core.py +208 -0
- scitex/plt/utils/metadata/_csv_column_extraction.py +186 -0
- scitex/plt/utils/metadata/_csv_hash.py +115 -0
- scitex/plt/utils/metadata/_csv_verification.py +95 -0
- scitex/plt/utils/metadata/_data_linkage.py +263 -0
- scitex/plt/utils/metadata/_dimensions.py +239 -0
- scitex/plt/utils/metadata/_figure_metadata.py +58 -0
- scitex/plt/utils/metadata/_image_text_artists.py +168 -0
- scitex/plt/utils/metadata/_label_parsing.py +82 -0
- scitex/plt/utils/metadata/_legend_extraction.py +120 -0
- scitex/plt/utils/metadata/_line_artists.py +367 -0
- scitex/plt/utils/metadata/_line_semantic_handling.py +173 -0
- scitex/plt/utils/metadata/_patch_artists.py +211 -0
- scitex/plt/utils/metadata/_plot_content.py +26 -0
- scitex/plt/utils/metadata/_plot_type_detection.py +184 -0
- scitex/plt/utils/metadata/_precision.py +134 -0
- scitex/plt/utils/metadata/_precision_config.py +68 -0
- scitex/plt/utils/metadata/_precision_sections.py +211 -0
- scitex/plt/utils/metadata/_recipe_extraction.py +267 -0
- scitex/plt/utils/metadata/_style_parsing.py +174 -0
- scitex/repro/_RandomStateManager.py +33 -38
- scitex/repro/__init__.py +16 -7
- scitex/repro/_gen_ID.py +7 -9
- scitex/repro/_gen_timestamp.py +7 -6
- scitex/repro/_hash_array.py +8 -12
- scitex/reproduce/__init__.py +1 -1
- scitex/resource/_get_processor_usages.py +3 -1
- scitex/resource/_log_processor_usages.py +3 -1
- scitex/rng/__init__.py +1 -1
- scitex/schema/README.md +178 -0
- scitex/schema/__init__.py +144 -0
- scitex/schema/_canvas.py +444 -0
- scitex/schema/_stats.py +762 -0
- scitex/schema/_validation.py +590 -0
- scitex/scholar/.legacy/Scholar.py +5 -12
- scitex/scholar/.legacy/_Scholar.py +66 -99
- scitex/scholar/.legacy/_ScholarAPI.py +75 -66
- scitex/scholar/.legacy/_tmp/search_engine/_BaseSearchEngine.py +3 -3
- scitex/scholar/.legacy/_tmp/search_engine/_UnifiedSearcher.py +4 -9
- scitex/scholar/.legacy/_tmp/search_engine/__init__.py +14 -21
- scitex/scholar/.legacy/_tmp/search_engine/local/_LocalSearchEngine.py +40 -37
- scitex/scholar/.legacy/_tmp/search_engine/local/_VectorSearchEngine.py +31 -28
- scitex/scholar/.legacy/_tmp/search_engine/web/_ArxivSearchEngine.py +74 -65
- scitex/scholar/.legacy/_tmp/search_engine/web/_CrossRefSearchEngine.py +122 -116
- scitex/scholar/.legacy/_tmp/search_engine/web/_GoogleScholarSearchEngine.py +65 -59
- scitex/scholar/.legacy/_tmp/search_engine/web/_PubMedSearchEngine.py +121 -107
- scitex/scholar/.legacy/_tmp/search_engine/web/_SemanticScholarSearchEngine.py +5 -12
- scitex/scholar/.legacy/database/_DatabaseEntry.py +49 -45
- scitex/scholar/.legacy/database/_DatabaseIndex.py +131 -94
- scitex/scholar/.legacy/database/_LibraryManager.py +65 -63
- scitex/scholar/.legacy/database/_PaperDatabase.py +138 -124
- scitex/scholar/.legacy/database/_ScholarDatabaseIntegration.py +14 -36
- scitex/scholar/.legacy/database/_StorageIntegratedDB.py +192 -156
- scitex/scholar/.legacy/database/_ZoteroCompatibleDB.py +300 -237
- scitex/scholar/.legacy/database/__init__.py +2 -1
- scitex/scholar/.legacy/database/manage.py +92 -84
- scitex/scholar/.legacy/lookup/_LookupIndex.py +157 -101
- scitex/scholar/.legacy/lookup/__init__.py +2 -1
- scitex/scholar/.legacy/metadata/doi/batch/_MetadataHandlerForBatchDOIResolution.py +4 -9
- scitex/scholar/.legacy/metadata/doi/batch/_ProgressManagerForBatchDOIResolution.py +10 -23
- scitex/scholar/.legacy/metadata/doi/batch/_SourceStatsManagerForBatchDOIResolution.py +4 -9
- scitex/scholar/.legacy/metadata/doi/batch/__init__.py +3 -1
- scitex/scholar/.legacy/metadata/doi/resolvers/_BatchDOIResolver.py +10 -25
- scitex/scholar/.legacy/metadata/doi/resolvers/_BibTeXDOIResolver.py +19 -49
- scitex/scholar/.legacy/metadata/doi/resolvers/_DOIResolver.py +1 -0
- scitex/scholar/.legacy/metadata/doi/resolvers/_SingleDOIResolver.py +8 -20
- scitex/scholar/.legacy/metadata/doi/sources/.combined-SemanticScholarSource/_SemanticScholarSource.py +37 -35
- scitex/scholar/.legacy/metadata/doi/sources/.combined-SemanticScholarSource/_SemanticScholarSourceEnhanced.py +49 -37
- scitex/scholar/.legacy/metadata/doi/sources/_ArXivSource.py +11 -30
- scitex/scholar/.legacy/metadata/doi/sources/_BaseDOISource.py +19 -47
- scitex/scholar/.legacy/metadata/doi/sources/_CrossRefLocalSource.py +1 -0
- scitex/scholar/.legacy/metadata/doi/sources/_CrossRefSource.py +12 -33
- scitex/scholar/.legacy/metadata/doi/sources/_OpenAlexSource.py +8 -20
- scitex/scholar/.legacy/metadata/doi/sources/_PubMedSource.py +10 -27
- scitex/scholar/.legacy/metadata/doi/sources/_SemanticScholarSource.py +11 -29
- scitex/scholar/.legacy/metadata/doi/sources/_SourceManager.py +8 -21
- scitex/scholar/.legacy/metadata/doi/sources/_SourceResolutionStrategy.py +24 -55
- scitex/scholar/.legacy/metadata/doi/sources/_SourceRotationManager.py +8 -21
- scitex/scholar/.legacy/metadata/doi/sources/_URLDOISource.py +9 -16
- scitex/scholar/.legacy/metadata/doi/sources/_UnifiedSource.py +8 -22
- scitex/scholar/.legacy/metadata/doi/sources/__init__.py +1 -0
- scitex/scholar/.legacy/metadata/doi/utils/_PubMedConverter.py +4 -8
- scitex/scholar/.legacy/metadata/doi/utils/_RateLimitHandler.py +17 -43
- scitex/scholar/.legacy/metadata/doi/utils/_TextNormalizer.py +8 -18
- scitex/scholar/.legacy/metadata/doi/utils/_URLDOIExtractor.py +4 -8
- scitex/scholar/.legacy/metadata/doi/utils/__init__.py +1 -0
- scitex/scholar/.legacy/metadata/doi/utils/_to_complete_metadata_structure.py +1 -0
- scitex/scholar/.legacy/metadata/enrichment/_LibraryEnricher.py +2 -3
- scitex/scholar/.legacy/metadata/enrichment/enrichers/_ImpactFactorEnricher.py +6 -12
- scitex/scholar/.legacy/metadata/enrichment/enrichers/_SmartEnricher.py +5 -10
- scitex/scholar/.legacy/metadata/enrichment/sources/_UnifiedMetadataSource.py +4 -5
- scitex/scholar/.legacy/metadata/query_to_full_meta_json.py +8 -12
- scitex/scholar/.legacy/metadata/urls/_URLMetadataHandler.py +3 -3
- scitex/scholar/.legacy/metadata/urls/_ZoteroTranslatorRunner.py +15 -21
- scitex/scholar/.legacy/metadata/urls/__init__.py +3 -3
- scitex/scholar/.legacy/metadata/urls/_finder.py +4 -6
- scitex/scholar/.legacy/metadata/urls/_handler.py +7 -15
- scitex/scholar/.legacy/metadata/urls/_resolver.py +6 -12
- scitex/scholar/.legacy/search/_Embedder.py +74 -69
- scitex/scholar/.legacy/search/_SemanticSearch.py +91 -90
- scitex/scholar/.legacy/search/_SemanticSearchEngine.py +104 -109
- scitex/scholar/.legacy/search/_UnifiedSearcher.py +530 -471
- scitex/scholar/.legacy/search/_VectorDatabase.py +111 -92
- scitex/scholar/.legacy/search/__init__.py +1 -0
- scitex/scholar/.legacy/storage/_EnhancedStorageManager.py +182 -154
- scitex/scholar/.legacy/storage/__init__.py +2 -1
- scitex/scholar/__init__.py +0 -2
- scitex/scholar/__main__.py +1 -3
- scitex/scholar/auth/ScholarAuthManager.py +13 -36
- scitex/scholar/auth/core/AuthenticationGateway.py +15 -29
- scitex/scholar/auth/core/BrowserAuthenticator.py +22 -57
- scitex/scholar/auth/core/StrategyResolver.py +10 -27
- scitex/scholar/auth/core/__init__.py +5 -1
- scitex/scholar/auth/gateway/_OpenURLLinkFinder.py +11 -21
- scitex/scholar/auth/gateway/_OpenURLResolver.py +10 -18
- scitex/scholar/auth/gateway/_resolve_functions.py +3 -3
- scitex/scholar/auth/providers/BaseAuthenticator.py +1 -0
- scitex/scholar/auth/providers/EZProxyAuthenticator.py +7 -14
- scitex/scholar/auth/providers/OpenAthensAuthenticator.py +29 -57
- scitex/scholar/auth/providers/ShibbolethAuthenticator.py +87 -73
- scitex/scholar/auth/session/AuthCacheManager.py +12 -22
- scitex/scholar/auth/session/SessionManager.py +4 -6
- scitex/scholar/auth/sso/BaseSSOAutomator.py +13 -19
- scitex/scholar/auth/sso/OpenAthensSSOAutomator.py +16 -45
- scitex/scholar/auth/sso/SSOAutomator.py +8 -15
- scitex/scholar/auth/sso/UniversityOfMelbourneSSOAutomator.py +13 -23
- scitex/scholar/browser/ScholarBrowserManager.py +31 -56
- scitex/scholar/browser/__init__.py +1 -0
- scitex/scholar/browser/utils/click_and_wait.py +3 -4
- scitex/scholar/browser/utils/close_unwanted_pages.py +4 -7
- scitex/scholar/browser/utils/wait_redirects.py +15 -40
- scitex/scholar/citation_graph/__init__.py +0 -0
- scitex/scholar/citation_graph/builder.py +3 -7
- scitex/scholar/citation_graph/database.py +4 -11
- scitex/scholar/citation_graph/example.py +5 -10
- scitex/scholar/citation_graph/models.py +0 -0
- scitex/scholar/cli/_url_utils.py +1 -1
- scitex/scholar/cli/chrome.py +5 -3
- scitex/scholar/cli/download_pdf.py +13 -14
- scitex/scholar/cli/handlers/bibtex_handler.py +4 -12
- scitex/scholar/cli/handlers/doi_handler.py +1 -3
- scitex/scholar/cli/handlers/project_handler.py +6 -20
- scitex/scholar/cli/open_browser.py +41 -39
- scitex/scholar/cli/open_browser_auto.py +31 -39
- scitex/scholar/cli/open_browser_monitored.py +27 -24
- scitex/scholar/config/ScholarConfig.py +5 -8
- scitex/scholar/config/__init__.py +1 -0
- scitex/scholar/config/core/_CascadeConfig.py +3 -3
- scitex/scholar/config/core/_PathManager.py +16 -28
- scitex/scholar/core/Paper.py +79 -78
- scitex/scholar/core/Papers.py +16 -27
- scitex/scholar/core/Scholar.py +98 -229
- scitex/scholar/core/journal_normalizer.py +52 -49
- scitex/scholar/core/oa_cache.py +27 -23
- scitex/scholar/core/open_access.py +17 -8
- scitex/scholar/docs/template.py +4 -3
- scitex/scholar/docs/to_claude/examples/example-python-project-scitex/scripts/mnist/clf_svm.py +0 -0
- scitex/scholar/docs/to_claude/examples/example-python-project-scitex/scripts/mnist/download.py +0 -0
- scitex/scholar/docs/to_claude/examples/example-python-project-scitex/scripts/mnist/plot_conf_mat.py +0 -0
- scitex/scholar/docs/to_claude/examples/example-python-project-scitex/scripts/mnist/plot_digits.py +0 -0
- scitex/scholar/docs/to_claude/examples/example-python-project-scitex/scripts/mnist/plot_umap_space.py +0 -0
- scitex/scholar/examples/00_config.py +10 -9
- scitex/scholar/examples/01_auth.py +3 -0
- scitex/scholar/examples/02_browser.py +14 -10
- scitex/scholar/examples/03_01-engine.py +3 -0
- scitex/scholar/examples/03_02-engine-for-bibtex.py +4 -3
- scitex/scholar/examples/04_01-url.py +9 -9
- scitex/scholar/examples/04_02-url-for-bibtex.py +7 -3
- scitex/scholar/examples/04_02-url-for-dois.py +87 -97
- scitex/scholar/examples/05_download_pdf.py +10 -4
- scitex/scholar/examples/06_find_and_download.py +6 -6
- scitex/scholar/examples/06_parse_bibtex.py +17 -17
- scitex/scholar/examples/07_storage_integration.py +6 -9
- scitex/scholar/examples/99_fullpipeline-for-bibtex.py +14 -15
- scitex/scholar/examples/99_fullpipeline-for-one-entry.py +31 -23
- scitex/scholar/examples/99_maintenance.py +3 -0
- scitex/scholar/examples/dev.py +2 -3
- scitex/scholar/examples/zotero_integration.py +11 -18
- scitex/scholar/impact_factor/ImpactFactorEngine.py +7 -9
- scitex/scholar/impact_factor/estimation/__init__.py +4 -4
- scitex/scholar/impact_factor/estimation/core/__init__.py +3 -7
- scitex/scholar/impact_factor/estimation/core/cache_manager.py +223 -211
- scitex/scholar/impact_factor/estimation/core/calculator.py +165 -131
- scitex/scholar/impact_factor/estimation/core/journal_matcher.py +217 -172
- scitex/scholar/impact_factor/jcr/ImpactFactorJCREngine.py +6 -14
- scitex/scholar/impact_factor/jcr/build_database.py +4 -3
- scitex/scholar/integration/base.py +9 -17
- scitex/scholar/integration/mendeley/exporter.py +2 -4
- scitex/scholar/integration/mendeley/importer.py +3 -3
- scitex/scholar/integration/mendeley/linker.py +3 -3
- scitex/scholar/integration/mendeley/mapper.py +9 -6
- scitex/scholar/integration/zotero/__main__.py +26 -43
- scitex/scholar/integration/zotero/exporter.py +15 -11
- scitex/scholar/integration/zotero/importer.py +12 -10
- scitex/scholar/integration/zotero/linker.py +8 -12
- scitex/scholar/integration/zotero/mapper.py +17 -12
- scitex/scholar/metadata_engines/.combined-SemanticScholarSource/_SemanticScholarSource.py +37 -35
- scitex/scholar/metadata_engines/.combined-SemanticScholarSource/_SemanticScholarSourceEnhanced.py +47 -35
- scitex/scholar/metadata_engines/ScholarEngine.py +21 -43
- scitex/scholar/metadata_engines/__init__.py +1 -0
- scitex/scholar/metadata_engines/individual/ArXivEngine.py +15 -37
- scitex/scholar/metadata_engines/individual/CrossRefEngine.py +15 -42
- scitex/scholar/metadata_engines/individual/CrossRefLocalEngine.py +24 -45
- scitex/scholar/metadata_engines/individual/OpenAlexEngine.py +11 -21
- scitex/scholar/metadata_engines/individual/PubMedEngine.py +10 -27
- scitex/scholar/metadata_engines/individual/SemanticScholarEngine.py +28 -35
- scitex/scholar/metadata_engines/individual/URLDOIEngine.py +11 -22
- scitex/scholar/metadata_engines/individual/_BaseDOIEngine.py +20 -49
- scitex/scholar/metadata_engines/utils/_PubMedConverter.py +4 -8
- scitex/scholar/metadata_engines/utils/_URLDOIExtractor.py +5 -10
- scitex/scholar/metadata_engines/utils/__init__.py +2 -0
- scitex/scholar/metadata_engines/utils/_metadata2bibtex.py +3 -0
- scitex/scholar/metadata_engines/utils/_standardize_metadata.py +2 -3
- scitex/scholar/pdf_download/ScholarPDFDownloader.py +25 -37
- scitex/scholar/pdf_download/strategies/chrome_pdf_viewer.py +11 -19
- scitex/scholar/pdf_download/strategies/direct_download.py +5 -9
- scitex/scholar/pdf_download/strategies/manual_download_fallback.py +3 -3
- scitex/scholar/pdf_download/strategies/manual_download_utils.py +6 -13
- scitex/scholar/pdf_download/strategies/open_access_download.py +49 -31
- scitex/scholar/pdf_download/strategies/response_body.py +8 -19
- scitex/scholar/pipelines/ScholarPipelineBibTeX.py +9 -18
- scitex/scholar/pipelines/ScholarPipelineMetadataParallel.py +25 -26
- scitex/scholar/pipelines/ScholarPipelineMetadataSingle.py +62 -23
- scitex/scholar/pipelines/ScholarPipelineParallel.py +13 -30
- scitex/scholar/pipelines/ScholarPipelineSearchParallel.py +299 -220
- scitex/scholar/pipelines/ScholarPipelineSearchSingle.py +202 -165
- scitex/scholar/pipelines/ScholarPipelineSingle.py +25 -51
- scitex/scholar/pipelines/SearchQueryParser.py +55 -55
- scitex/scholar/search_engines/ScholarSearchEngine.py +31 -27
- scitex/scholar/search_engines/_BaseSearchEngine.py +20 -23
- scitex/scholar/search_engines/individual/ArXivSearchEngine.py +53 -35
- scitex/scholar/search_engines/individual/CrossRefSearchEngine.py +47 -40
- scitex/scholar/search_engines/individual/OpenAlexSearchEngine.py +55 -50
- scitex/scholar/search_engines/individual/PubMedSearchEngine.py +8 -10
- scitex/scholar/search_engines/individual/SemanticScholarSearchEngine.py +55 -49
- scitex/scholar/storage/BibTeXHandler.py +150 -95
- scitex/scholar/storage/PaperIO.py +3 -6
- scitex/scholar/storage/ScholarLibrary.py +70 -49
- scitex/scholar/storage/_DeduplicationManager.py +52 -25
- scitex/scholar/storage/_LibraryCacheManager.py +19 -46
- scitex/scholar/storage/_LibraryManager.py +65 -175
- scitex/scholar/url_finder/ScholarURLFinder.py +9 -25
- scitex/scholar/url_finder/strategies/find_pdf_urls_by_direct_links.py +1 -1
- scitex/scholar/url_finder/strategies/find_pdf_urls_by_href.py +6 -10
- scitex/scholar/url_finder/strategies/find_pdf_urls_by_navigation.py +4 -6
- scitex/scholar/url_finder/strategies/find_pdf_urls_by_publisher_patterns.py +8 -15
- scitex/scholar/url_finder/strategies/find_pdf_urls_by_zotero_translators.py +3 -3
- scitex/scholar/url_finder/strategies/find_supplementary_urls_by_href.py +3 -3
- scitex/scholar/url_finder/translators/core/patterns.py +6 -4
- scitex/scholar/url_finder/translators/core/registry.py +6 -9
- scitex/scholar/url_finder/translators/individual/BOFiP_Impots.py +60 -52
- scitex/scholar/url_finder/translators/individual/Baidu_Scholar.py +54 -62
- scitex/scholar/url_finder/translators/individual/Bangkok_Post.py +38 -44
- scitex/scholar/url_finder/translators/individual/Baruch_Foundation.py +43 -47
- scitex/scholar/url_finder/translators/individual/Beobachter.py +46 -50
- scitex/scholar/url_finder/translators/individual/Bezneng_Gajit.py +37 -41
- scitex/scholar/url_finder/translators/individual/BibLaTeX.py +59 -52
- scitex/scholar/url_finder/translators/individual/BibTeX.py +83 -79
- scitex/scholar/url_finder/translators/individual/Biblio_com.py +48 -51
- scitex/scholar/url_finder/translators/individual/Bibliontology_RDF.py +58 -56
- scitex/scholar/url_finder/translators/individual/Camara_Brasileira_do_Livro_ISBN.py +102 -99
- scitex/scholar/url_finder/translators/individual/CanLII.py +49 -43
- scitex/scholar/url_finder/translators/individual/Canada_com.py +36 -40
- scitex/scholar/url_finder/translators/individual/Canadian_Letters_and_Images.py +43 -43
- scitex/scholar/url_finder/translators/individual/Canadiana_ca.py +77 -66
- scitex/scholar/url_finder/translators/individual/Cascadilla_Proceedings_Project.py +68 -62
- scitex/scholar/url_finder/translators/individual/Central_and_Eastern_European_Online_Library_Journals.py +60 -60
- scitex/scholar/url_finder/translators/individual/Champlain_Society_Collection.py +63 -61
- scitex/scholar/url_finder/translators/individual/Chicago_Journal_of_Theoretical_Computer_Science.py +74 -58
- scitex/scholar/url_finder/translators/individual/Christian_Science_Monitor.py +32 -38
- scitex/scholar/url_finder/translators/individual/Columbia_University_Press.py +51 -47
- scitex/scholar/url_finder/translators/individual/Common_Place.py +66 -57
- scitex/scholar/url_finder/translators/individual/Cornell_LII.py +66 -62
- scitex/scholar/url_finder/translators/individual/Cornell_University_Press.py +38 -45
- scitex/scholar/url_finder/translators/individual/CourtListener.py +52 -56
- scitex/scholar/url_finder/translators/individual/DAI_Zenon.py +53 -54
- scitex/scholar/url_finder/translators/individual/access_medicine.py +27 -33
- scitex/scholar/url_finder/translators/individual/acm.py +1 -1
- scitex/scholar/url_finder/translators/individual/acm_digital_library.py +93 -63
- scitex/scholar/url_finder/translators/individual/airiti.py +3 -1
- scitex/scholar/url_finder/translators/individual/aosic.py +3 -1
- scitex/scholar/url_finder/translators/individual/archive_ouverte_aosic.py +3 -1
- scitex/scholar/url_finder/translators/individual/archive_ouverte_en_sciences_de_l_information_et_de_la_communication___aosic_.py +6 -2
- scitex/scholar/url_finder/translators/individual/artforum.py +35 -27
- scitex/scholar/url_finder/translators/individual/arxiv.py +1 -1
- scitex/scholar/url_finder/translators/individual/arxiv_org.py +8 -4
- scitex/scholar/url_finder/translators/individual/atlanta_journal_constitution.py +22 -18
- scitex/scholar/url_finder/translators/individual/atypon_journals.py +19 -11
- scitex/scholar/url_finder/translators/individual/austlii_and_nzlii.py +48 -44
- scitex/scholar/url_finder/translators/individual/australian_dictionary_of_biography.py +21 -17
- scitex/scholar/url_finder/translators/individual/bailii.py +22 -19
- scitex/scholar/url_finder/translators/individual/bbc.py +46 -42
- scitex/scholar/url_finder/translators/individual/bbc_genome.py +37 -25
- scitex/scholar/url_finder/translators/individual/biblioteca_nacional_de_maestros.py +24 -20
- scitex/scholar/url_finder/translators/individual/bibliotheque_archives_nationale_quebec_pistard.py +42 -43
- scitex/scholar/url_finder/translators/individual/bibliotheque_archives_nationales_quebec.py +87 -81
- scitex/scholar/url_finder/translators/individual/bibliotheque_nationale_france.py +39 -37
- scitex/scholar/url_finder/translators/individual/bibsys.py +32 -28
- scitex/scholar/url_finder/translators/individual/bioconductor.py +58 -52
- scitex/scholar/url_finder/translators/individual/biomed_central.py +23 -15
- scitex/scholar/url_finder/translators/individual/biorxiv.py +26 -13
- scitex/scholar/url_finder/translators/individual/blogger.py +39 -43
- scitex/scholar/url_finder/translators/individual/bloomberg.py +48 -52
- scitex/scholar/url_finder/translators/individual/bloomsbury_food_library.py +37 -37
- scitex/scholar/url_finder/translators/individual/bluesky.py +30 -28
- scitex/scholar/url_finder/translators/individual/bnf_isbn.py +1 -1
- scitex/scholar/url_finder/translators/individual/bocc.py +66 -60
- scitex/scholar/url_finder/translators/individual/boe.py +52 -52
- scitex/scholar/url_finder/translators/individual/brill.py +3 -1
- scitex/scholar/url_finder/translators/individual/business_standard.py +36 -38
- scitex/scholar/url_finder/translators/individual/cabi_cab_abstracts.py +39 -41
- scitex/scholar/url_finder/translators/individual/cambridge.py +3 -1
- scitex/scholar/url_finder/translators/individual/cambridge_core.py +30 -24
- scitex/scholar/url_finder/translators/individual/caod.py +50 -46
- scitex/scholar/url_finder/translators/individual/cbc.py +91 -67
- scitex/scholar/url_finder/translators/individual/ccfr_bnf.py +49 -53
- scitex/scholar/url_finder/translators/individual/cia_world_factbook.py +43 -33
- scitex/scholar/url_finder/translators/individual/crossref_rest.py +208 -174
- scitex/scholar/url_finder/translators/individual/current_affairs.py +29 -35
- scitex/scholar/url_finder/translators/individual/dabi.py +70 -66
- scitex/scholar/url_finder/translators/individual/dagens_nyheter.py +3 -1
- scitex/scholar/url_finder/translators/individual/dagstuhl.py +10 -15
- scitex/scholar/url_finder/translators/individual/dar_almandumah.py +13 -9
- scitex/scholar/url_finder/translators/individual/dart_europe.py +19 -22
- scitex/scholar/url_finder/translators/individual/data_gov.py +2 -2
- scitex/scholar/url_finder/translators/individual/databrary.py +27 -28
- scitex/scholar/url_finder/translators/individual/datacite_json.py +152 -137
- scitex/scholar/url_finder/translators/individual/dataverse.py +68 -64
- scitex/scholar/url_finder/translators/individual/daum_news.py +38 -38
- scitex/scholar/url_finder/translators/individual/dblp.py +4 -8
- scitex/scholar/url_finder/translators/individual/dblp_computer_science_bibliography.py +8 -3
- scitex/scholar/url_finder/translators/individual/dbpia.py +5 -3
- scitex/scholar/url_finder/translators/individual/defense_technical_information_center.py +30 -28
- scitex/scholar/url_finder/translators/individual/delpher.py +102 -79
- scitex/scholar/url_finder/translators/individual/demographic_research.py +35 -31
- scitex/scholar/url_finder/translators/individual/denik_cz.py +58 -54
- scitex/scholar/url_finder/translators/individual/depatisnet.py +7 -10
- scitex/scholar/url_finder/translators/individual/der_freitag.py +81 -66
- scitex/scholar/url_finder/translators/individual/der_spiegel.py +56 -54
- scitex/scholar/url_finder/translators/individual/digibib_net.py +3 -1
- scitex/scholar/url_finder/translators/individual/digizeitschriften.py +3 -1
- scitex/scholar/url_finder/translators/individual/dpla.py +13 -14
- scitex/scholar/url_finder/translators/individual/dspace.py +2 -2
- scitex/scholar/url_finder/translators/individual/ebrary.py +3 -1
- scitex/scholar/url_finder/translators/individual/ebscohost.py +3 -1
- scitex/scholar/url_finder/translators/individual/electronic_colloquium_on_computational_complexity.py +3 -1
- scitex/scholar/url_finder/translators/individual/elife.py +3 -1
- scitex/scholar/url_finder/translators/individual/elsevier_health_journals.py +3 -1
- scitex/scholar/url_finder/translators/individual/emerald.py +3 -1
- scitex/scholar/url_finder/translators/individual/emerald_insight.py +3 -1
- scitex/scholar/url_finder/translators/individual/epicurious.py +3 -1
- scitex/scholar/url_finder/translators/individual/eurogamerusgamer.py +3 -1
- scitex/scholar/url_finder/translators/individual/fachportal_padagogik.py +3 -1
- scitex/scholar/url_finder/translators/individual/frontiers.py +1 -1
- scitex/scholar/url_finder/translators/individual/gale_databases.py +3 -1
- scitex/scholar/url_finder/translators/individual/gms_german_medical_science.py +6 -2
- scitex/scholar/url_finder/translators/individual/ieee_computer_society.py +6 -2
- scitex/scholar/url_finder/translators/individual/ieee_xplore.py +41 -35
- scitex/scholar/url_finder/translators/individual/inter_research_science_center.py +6 -2
- scitex/scholar/url_finder/translators/individual/jisc_historical_texts.py +3 -1
- scitex/scholar/url_finder/translators/individual/jstor.py +14 -12
- scitex/scholar/url_finder/translators/individual/korean_national_library.py +3 -1
- scitex/scholar/url_finder/translators/individual/la_times.py +3 -1
- scitex/scholar/url_finder/translators/individual/landesbibliographie_baden_wurttemberg.py +3 -1
- scitex/scholar/url_finder/translators/individual/legislative_insight.py +3 -1
- scitex/scholar/url_finder/translators/individual/libraries_tasmania.py +3 -1
- scitex/scholar/url_finder/translators/individual/library_catalog__koha_.py +3 -1
- scitex/scholar/url_finder/translators/individual/lingbuzz.py +2 -2
- scitex/scholar/url_finder/translators/individual/max_planck_institute_for_the_history_of_science_virtual_laboratory_library.py +3 -1
- scitex/scholar/url_finder/translators/individual/mdpi.py +12 -6
- scitex/scholar/url_finder/translators/individual/microbiology_society_journals.py +3 -1
- scitex/scholar/url_finder/translators/individual/midas_journals.py +3 -1
- scitex/scholar/url_finder/translators/individual/nagoya_university_opac.py +3 -1
- scitex/scholar/url_finder/translators/individual/nature_publishing_group.py +32 -19
- scitex/scholar/url_finder/translators/individual/ntsb_accident_reports.py +3 -1
- scitex/scholar/url_finder/translators/individual/openedition_journals.py +8 -4
- scitex/scholar/url_finder/translators/individual/orcid.py +16 -15
- scitex/scholar/url_finder/translators/individual/oxford.py +25 -19
- scitex/scholar/url_finder/translators/individual/oxford_dictionaries_premium.py +3 -1
- scitex/scholar/url_finder/translators/individual/ozon_ru.py +3 -1
- scitex/scholar/url_finder/translators/individual/plos.py +9 -12
- scitex/scholar/url_finder/translators/individual/polygon.py +3 -1
- scitex/scholar/url_finder/translators/individual/primo.py +3 -1
- scitex/scholar/url_finder/translators/individual/project_muse.py +3 -1
- scitex/scholar/url_finder/translators/individual/pubfactory_journals.py +3 -1
- scitex/scholar/url_finder/translators/individual/pubmed.py +71 -65
- scitex/scholar/url_finder/translators/individual/pubmed_central.py +8 -6
- scitex/scholar/url_finder/translators/individual/rechtspraak_nl.py +3 -1
- scitex/scholar/url_finder/translators/individual/sage_journals.py +25 -17
- scitex/scholar/url_finder/translators/individual/sciencedirect.py +36 -17
- scitex/scholar/url_finder/translators/individual/semantics_visual_library.py +3 -1
- scitex/scholar/url_finder/translators/individual/silverchair.py +70 -52
- scitex/scholar/url_finder/translators/individual/sora.py +3 -1
- scitex/scholar/url_finder/translators/individual/springer.py +15 -11
- scitex/scholar/url_finder/translators/individual/ssrn.py +3 -3
- scitex/scholar/url_finder/translators/individual/stanford_encyclopedia_of_philosophy.py +3 -1
- scitex/scholar/url_finder/translators/individual/superlib.py +3 -1
- scitex/scholar/url_finder/translators/individual/treesearch.py +3 -1
- scitex/scholar/url_finder/translators/individual/university_of_chicago_press_books.py +3 -1
- scitex/scholar/url_finder/translators/individual/vlex.py +3 -1
- scitex/scholar/url_finder/translators/individual/web_of_science.py +3 -1
- scitex/scholar/url_finder/translators/individual/web_of_science_nextgen.py +3 -1
- scitex/scholar/url_finder/translators/individual/wiley.py +31 -25
- scitex/scholar/url_finder/translators/individual/wilson_center_digital_archive.py +3 -1
- scitex/scholar/utils/bibtex/_parse_bibtex.py +3 -3
- scitex/scholar/utils/cleanup/_cleanup_scholar_processes.py +5 -9
- scitex/scholar/utils/text/_TextNormalizer.py +249 -176
- scitex/scholar/utils/validation/DOIValidator.py +31 -28
- scitex/scholar/utils/validation/__init__.py +0 -0
- scitex/scholar/utils/validation/validate_library_dois.py +61 -57
- scitex/scholar/zotero/__init__.py +1 -1
- scitex/security/cli.py +7 -20
- scitex/security/github.py +45 -32
- scitex/session/__init__.py +8 -9
- scitex/session/_decorator.py +49 -42
- scitex/session/_lifecycle.py +39 -39
- scitex/session/_manager.py +24 -20
- scitex/sh/__init__.py +4 -3
- scitex/sh/_execute.py +10 -7
- scitex/sh/_security.py +3 -3
- scitex/sh/_types.py +2 -3
- scitex/stats/__init__.py +57 -6
- scitex/stats/_schema.py +42 -569
- scitex/stats/auto/__init__.py +188 -0
- scitex/stats/auto/_context.py +331 -0
- scitex/stats/auto/_formatting.py +679 -0
- scitex/stats/auto/_rules.py +901 -0
- scitex/stats/auto/_selector.py +554 -0
- scitex/stats/auto/_styles.py +721 -0
- scitex/stats/correct/__init__.py +4 -4
- scitex/stats/correct/_correct_bonferroni.py +43 -34
- scitex/stats/correct/_correct_fdr.py +14 -40
- scitex/stats/correct/_correct_fdr_.py +39 -46
- scitex/stats/correct/_correct_holm.py +14 -32
- scitex/stats/correct/_correct_sidak.py +36 -21
- scitex/stats/descriptive/_circular.py +20 -21
- scitex/stats/descriptive/_describe.py +19 -5
- scitex/stats/descriptive/_nan.py +5 -7
- scitex/stats/descriptive/_real.py +4 -3
- scitex/stats/effect_sizes/__init__.py +10 -11
- scitex/stats/effect_sizes/_cliffs_delta.py +35 -32
- scitex/stats/effect_sizes/_cohens_d.py +30 -31
- scitex/stats/effect_sizes/_epsilon_squared.py +19 -22
- scitex/stats/effect_sizes/_eta_squared.py +23 -27
- scitex/stats/effect_sizes/_prob_superiority.py +18 -21
- scitex/stats/posthoc/__init__.py +3 -3
- scitex/stats/posthoc/_dunnett.py +75 -55
- scitex/stats/posthoc/_games_howell.py +61 -43
- scitex/stats/posthoc/_tukey_hsd.py +42 -34
- scitex/stats/power/__init__.py +2 -2
- scitex/stats/power/_power.py +56 -56
- scitex/stats/tests/__init__.py +1 -1
- scitex/stats/tests/correlation/__init__.py +1 -1
- scitex/stats/tests/correlation/_test_pearson.py +28 -38
- scitex/stats/utils/__init__.py +14 -17
- scitex/stats/utils/_effect_size.py +85 -78
- scitex/stats/utils/_formatters.py +49 -43
- scitex/stats/utils/_normalizers.py +7 -14
- scitex/stats/utils/_power.py +56 -56
- scitex/str/__init__.py +1 -0
- scitex/str/_clean_path.py +3 -3
- scitex/str/_factor_out_digits.py +86 -58
- scitex/str/_format_plot_text.py +180 -111
- scitex/str/_latex.py +19 -19
- scitex/str/_latex_fallback.py +9 -10
- scitex/str/_parse.py +3 -6
- scitex/str/_print_debug.py +13 -13
- scitex/str/_printc.py +2 -0
- scitex/str/_search.py +3 -3
- scitex/template/.legacy/_clone_project.py +9 -13
- scitex/template/__init__.py +10 -2
- scitex/template/_clone_project.py +7 -2
- scitex/template/_copy.py +1 -0
- scitex/template/_customize.py +3 -6
- scitex/template/_git_strategy.py +2 -3
- scitex/template/_rename.py +1 -0
- scitex/template/clone_pip_project.py +6 -7
- scitex/template/clone_research.py +7 -10
- scitex/template/clone_singularity.py +6 -7
- scitex/template/clone_writer_directory.py +6 -7
- scitex/tex/_preview.py +26 -11
- scitex/tex/_to_vec.py +10 -7
- scitex/torch/__init__.py +11 -1
- scitex/types/_ArrayLike.py +2 -0
- scitex/types/_is_listed_X.py +3 -3
- scitex/units.py +110 -77
- scitex/utils/_compress_hdf5.py +3 -3
- scitex/utils/_email.py +8 -4
- scitex/utils/_notify.py +14 -8
- scitex/utils/_search.py +6 -6
- scitex/utils/_verify_scitex_format.py +17 -42
- scitex/utils/_verify_scitex_format_v01.py +12 -34
- scitex/utils/template.py +4 -3
- scitex/vis/__init__.py +0 -0
- scitex/vis/backend/__init__.py +3 -3
- scitex/vis/backend/{export.py → _export.py} +1 -1
- scitex/vis/backend/{parser.py → _parser.py} +1 -3
- scitex/vis/backend/{render.py → _render.py} +1 -1
- scitex/vis/canvas.py +15 -3
- scitex/vis/editor/__init__.py +0 -0
- scitex/vis/editor/_dearpygui_editor.py +450 -304
- scitex/vis/editor/_defaults.py +114 -123
- scitex/vis/editor/_edit.py +38 -26
- scitex/vis/editor/_flask_editor.py +8 -8
- scitex/vis/editor/_mpl_editor.py +63 -48
- scitex/vis/editor/_qt_editor.py +210 -159
- scitex/vis/editor/_tkinter_editor.py +146 -89
- scitex/vis/editor/flask_editor/__init__.py +10 -10
- scitex/vis/editor/flask_editor/_bbox.py +529 -0
- scitex/vis/editor/flask_editor/{core.py → _core.py} +45 -29
- scitex/vis/editor/flask_editor/_plotter.py +567 -0
- scitex/vis/editor/flask_editor/_renderer.py +393 -0
- scitex/vis/editor/flask_editor/{utils.py → _utils.py} +13 -14
- scitex/vis/editor/flask_editor/templates/__init__.py +5 -5
- scitex/vis/editor/flask_editor/templates/{html.py → _html.py} +234 -16
- scitex/vis/editor/flask_editor/templates/_scripts.py +1261 -0
- scitex/vis/editor/flask_editor/templates/{styles.py → _styles.py} +192 -2
- scitex/vis/io/__init__.py +5 -5
- scitex/vis/io/{canvas.py → _canvas.py} +8 -4
- scitex/vis/io/{data.py → _data.py} +13 -9
- scitex/vis/io/{directory.py → _directory.py} +7 -4
- scitex/vis/io/{export.py → _export.py} +15 -12
- scitex/vis/io/{load.py → _load.py} +1 -1
- scitex/vis/io/{panel.py → _panel.py} +21 -13
- scitex/vis/io/{save.py → _save.py} +0 -0
- scitex/vis/model/__init__.py +7 -7
- scitex/vis/model/{annotations.py → _annotations.py} +2 -4
- scitex/vis/model/{axes.py → _axes.py} +1 -1
- scitex/vis/model/{figure.py → _figure.py} +0 -0
- scitex/vis/model/{guides.py → _guides.py} +1 -1
- scitex/vis/model/{plot.py → _plot.py} +2 -4
- scitex/vis/model/{plot_types.py → _plot_types.py} +0 -0
- scitex/vis/model/{styles.py → _styles.py} +0 -0
- scitex/vis/utils/__init__.py +2 -2
- scitex/vis/utils/{defaults.py → _defaults.py} +1 -2
- scitex/vis/utils/{validate.py → _validate.py} +3 -9
- scitex/web/__init__.py +7 -1
- scitex/web/_scraping.py +54 -38
- scitex/web/_search_pubmed.py +30 -14
- scitex/writer/.legacy/Writer_v01-refactored.py +4 -4
- scitex/writer/.legacy/_compile.py +18 -28
- scitex/writer/Writer.py +8 -21
- scitex/writer/__init__.py +11 -11
- scitex/writer/_clone_writer_project.py +2 -6
- scitex/writer/_compile/__init__.py +1 -0
- scitex/writer/_compile/_parser.py +1 -0
- scitex/writer/_compile/_runner.py +35 -38
- scitex/writer/_compile/_validator.py +1 -0
- scitex/writer/_compile/manuscript.py +1 -0
- scitex/writer/_compile/revision.py +1 -0
- scitex/writer/_compile/supplementary.py +1 -0
- scitex/writer/_compile_async.py +5 -12
- scitex/writer/_project/__init__.py +1 -0
- scitex/writer/_project/_create.py +10 -25
- scitex/writer/_project/_trees.py +4 -9
- scitex/writer/_project/_validate.py +2 -3
- scitex/writer/_validate_tree_structures.py +7 -18
- scitex/writer/dataclasses/__init__.py +8 -10
- scitex/writer/dataclasses/config/_CONSTANTS.py +2 -3
- scitex/writer/dataclasses/config/_WriterConfig.py +4 -9
- scitex/writer/dataclasses/contents/_ManuscriptContents.py +14 -25
- scitex/writer/dataclasses/contents/_RevisionContents.py +21 -16
- scitex/writer/dataclasses/contents/_SupplementaryContents.py +21 -24
- scitex/writer/dataclasses/core/_Document.py +2 -3
- scitex/writer/dataclasses/core/_DocumentSection.py +8 -23
- scitex/writer/dataclasses/results/_CompilationResult.py +2 -3
- scitex/writer/dataclasses/results/_LaTeXIssue.py +3 -6
- scitex/writer/dataclasses/results/_SaveSectionsResponse.py +20 -9
- scitex/writer/dataclasses/results/_SectionReadResponse.py +24 -10
- scitex/writer/dataclasses/tree/_ConfigTree.py +7 -4
- scitex/writer/dataclasses/tree/_ManuscriptTree.py +10 -13
- scitex/writer/dataclasses/tree/_RevisionTree.py +16 -17
- scitex/writer/dataclasses/tree/_ScriptsTree.py +10 -5
- scitex/writer/dataclasses/tree/_SharedTree.py +10 -13
- scitex/writer/dataclasses/tree/_SupplementaryTree.py +15 -14
- scitex/writer/utils/.legacy_git_retry.py +3 -8
- scitex/writer/utils/_parse_latex_logs.py +2 -3
- scitex/writer/utils/_parse_script_args.py +20 -23
- scitex/writer/utils/_watch.py +5 -5
- {scitex-2.5.0.dist-info → scitex-2.7.0.dist-info}/METADATA +4 -10
- {scitex-2.5.0.dist-info → scitex-2.7.0.dist-info}/RECORD +1071 -975
- scitex/db/_sqlite3/_SQLite3Mixins/_ColumnMixin_v01-indentation-issues.py +0 -583
- scitex/plt/_subplots/_export_as_csv_formatters.py +0 -112
- scitex/vis/editor/flask_editor/bbox.py +0 -216
- scitex/vis/editor/flask_editor/plotter.py +0 -130
- scitex/vis/editor/flask_editor/renderer.py +0 -184
- scitex/vis/editor/flask_editor/templates/scripts.py +0 -614
- {scitex-2.5.0.dist-info → scitex-2.7.0.dist-info}/WHEEL +0 -0
- {scitex-2.5.0.dist-info → scitex-2.7.0.dist-info}/entry_points.txt +0 -0
- {scitex-2.5.0.dist-info → scitex-2.7.0.dist-info}/licenses/LICENSE +0 -0
scitex/scholar/core/Scholar.py
CHANGED
|
@@ -5,9 +5,8 @@
|
|
|
5
5
|
# ----------------------------------------
|
|
6
6
|
from __future__ import annotations
|
|
7
7
|
import os
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
)
|
|
8
|
+
|
|
9
|
+
__FILE__ = "./src/scitex/scholar/core/Scholar.py"
|
|
11
10
|
__DIR__ = os.path.dirname(__FILE__)
|
|
12
11
|
# ----------------------------------------
|
|
13
12
|
|
|
@@ -136,15 +135,11 @@ class Scholar:
|
|
|
136
135
|
|
|
137
136
|
# Initialize service components (lazy loading for better performance)
|
|
138
137
|
# Use mangled names for private properties
|
|
139
|
-
self._Scholar__scholar_engine =
|
|
140
|
-
None # Replaces DOIResolver and LibraryEnricher
|
|
141
|
-
)
|
|
138
|
+
self._Scholar__scholar_engine = None # Replaces DOIResolver and LibraryEnricher
|
|
142
139
|
self._Scholar__auth_manager = None
|
|
143
140
|
self._Scholar__browser_manager = None
|
|
144
141
|
self._Scholar__library_manager = None
|
|
145
|
-
self._Scholar__library =
|
|
146
|
-
None # ScholarLibrary for high-level operations
|
|
147
|
-
)
|
|
142
|
+
self._Scholar__library = None # ScholarLibrary for high-level operations
|
|
148
143
|
|
|
149
144
|
# Show user-friendly initialization message with library location
|
|
150
145
|
library_path = self.config.get_library_project_dir()
|
|
@@ -154,9 +149,7 @@ class Scholar:
|
|
|
154
149
|
f"Scholar initialized with project '{project}' at {project_path}"
|
|
155
150
|
)
|
|
156
151
|
else:
|
|
157
|
-
logger.info(
|
|
158
|
-
f"{self.name}: Scholar initialized (library: {library_path})"
|
|
159
|
-
)
|
|
152
|
+
logger.info(f"{self.name}: Scholar initialized (library: {library_path})")
|
|
160
153
|
|
|
161
154
|
# ----------------------------------------
|
|
162
155
|
# Enrichers
|
|
@@ -196,9 +189,7 @@ class Scholar:
|
|
|
196
189
|
logger.warning(
|
|
197
190
|
f"{self.name}: Failed to enrich paper '{title[:50]}...': {e}"
|
|
198
191
|
)
|
|
199
|
-
enriched_list.append(
|
|
200
|
-
paper
|
|
201
|
-
) # Keep original if enrichment fails
|
|
192
|
+
enriched_list.append(paper) # Keep original if enrichment fails
|
|
202
193
|
|
|
203
194
|
enriched_papers = Papers(enriched_list, project=self.project)
|
|
204
195
|
|
|
@@ -256,9 +247,7 @@ class Scholar:
|
|
|
256
247
|
logger.warning(
|
|
257
248
|
f"{self.name}: Failed to enrich paper '{title[:50]}...': {e}"
|
|
258
249
|
)
|
|
259
|
-
enriched_list.append(
|
|
260
|
-
paper
|
|
261
|
-
) # Keep original if enrichment fails
|
|
250
|
+
enriched_list.append(paper) # Keep original if enrichment fails
|
|
262
251
|
|
|
263
252
|
enriched_papers = Papers(enriched_list, project=self.project)
|
|
264
253
|
|
|
@@ -309,10 +298,7 @@ class Scholar:
|
|
|
309
298
|
enriched.metadata.set_doi(results["id"]["doi"])
|
|
310
299
|
if results["id"].get("pmid") and not enriched.metadata.id.pmid:
|
|
311
300
|
enriched.metadata.id.pmid = results["id"]["pmid"]
|
|
312
|
-
if (
|
|
313
|
-
results["id"].get("arxiv_id")
|
|
314
|
-
and not enriched.metadata.id.arxiv_id
|
|
315
|
-
):
|
|
301
|
+
if results["id"].get("arxiv_id") and not enriched.metadata.id.arxiv_id:
|
|
316
302
|
enriched.metadata.id.arxiv_id = results["id"]["arxiv_id"]
|
|
317
303
|
# Note: corpus_id, semantic_id, ieee_id are in results but not in Paper dataclass
|
|
318
304
|
|
|
@@ -330,17 +316,11 @@ class Scholar:
|
|
|
330
316
|
enriched.metadata.basic.title = new_title
|
|
331
317
|
|
|
332
318
|
# Update authors if found
|
|
333
|
-
if (
|
|
334
|
-
results["basic"].get("authors")
|
|
335
|
-
and not enriched.metadata.basic.authors
|
|
336
|
-
):
|
|
319
|
+
if results["basic"].get("authors") and not enriched.metadata.basic.authors:
|
|
337
320
|
enriched.metadata.basic.authors = results["basic"]["authors"]
|
|
338
321
|
|
|
339
322
|
# Update year if found
|
|
340
|
-
if (
|
|
341
|
-
results["basic"].get("year")
|
|
342
|
-
and not enriched.metadata.basic.year
|
|
343
|
-
):
|
|
323
|
+
if results["basic"].get("year") and not enriched.metadata.basic.year:
|
|
344
324
|
enriched.metadata.basic.year = results["basic"]["year"]
|
|
345
325
|
|
|
346
326
|
# Update keywords if found
|
|
@@ -363,30 +343,24 @@ class Scholar:
|
|
|
363
343
|
results["publication"].get("publisher")
|
|
364
344
|
and not enriched.metadata.publication.publisher
|
|
365
345
|
):
|
|
366
|
-
enriched.metadata.publication.publisher = results[
|
|
367
|
-
"
|
|
368
|
-
]
|
|
346
|
+
enriched.metadata.publication.publisher = results["publication"][
|
|
347
|
+
"publisher"
|
|
348
|
+
]
|
|
369
349
|
if (
|
|
370
350
|
results["publication"].get("volume")
|
|
371
351
|
and not enriched.metadata.publication.volume
|
|
372
352
|
):
|
|
373
|
-
enriched.metadata.publication.volume = results["publication"][
|
|
374
|
-
"volume"
|
|
375
|
-
]
|
|
353
|
+
enriched.metadata.publication.volume = results["publication"]["volume"]
|
|
376
354
|
if (
|
|
377
355
|
results["publication"].get("issue")
|
|
378
356
|
and not enriched.metadata.publication.issue
|
|
379
357
|
):
|
|
380
|
-
enriched.metadata.publication.issue = results["publication"][
|
|
381
|
-
"issue"
|
|
382
|
-
]
|
|
358
|
+
enriched.metadata.publication.issue = results["publication"]["issue"]
|
|
383
359
|
if (
|
|
384
360
|
results["publication"].get("pages")
|
|
385
361
|
and not enriched.metadata.publication.pages
|
|
386
362
|
):
|
|
387
|
-
enriched.metadata.publication.pages = results["publication"][
|
|
388
|
-
"pages"
|
|
389
|
-
]
|
|
363
|
+
enriched.metadata.publication.pages = results["publication"]["pages"]
|
|
390
364
|
|
|
391
365
|
# Citation metadata
|
|
392
366
|
if "citation_count" in results:
|
|
@@ -406,16 +380,11 @@ class Scholar:
|
|
|
406
380
|
if results["url"].get("pdf"):
|
|
407
381
|
# Check if this PDF is not already in the list
|
|
408
382
|
pdf_url = results["url"]["pdf"]
|
|
409
|
-
if not any(
|
|
410
|
-
p.get("url") == pdf_url for p in enriched.metadata.url.pdfs
|
|
411
|
-
):
|
|
383
|
+
if not any(p.get("url") == pdf_url for p in enriched.metadata.url.pdfs):
|
|
412
384
|
enriched.metadata.url.pdfs.append(
|
|
413
385
|
{"url": pdf_url, "source": "enrichment"}
|
|
414
386
|
)
|
|
415
|
-
if (
|
|
416
|
-
results["url"].get("url")
|
|
417
|
-
and not enriched.metadata.url.publisher
|
|
418
|
-
):
|
|
387
|
+
if results["url"].get("url") and not enriched.metadata.url.publisher:
|
|
419
388
|
enriched.metadata.url.publisher = results["url"]["url"]
|
|
420
389
|
|
|
421
390
|
# Note: Metrics section (journal_impact_factor, h_index) not stored in Paper dataclass
|
|
@@ -446,8 +415,7 @@ class Scholar:
|
|
|
446
415
|
enriched_count = sum(
|
|
447
416
|
1
|
|
448
417
|
for i, p in enumerate(enriched_papers)
|
|
449
|
-
if p.abstract
|
|
450
|
-
and not papers[i].abstract # Check if abstract was added
|
|
418
|
+
if p.abstract and not papers[i].abstract # Check if abstract was added
|
|
451
419
|
)
|
|
452
420
|
|
|
453
421
|
# Save enriched papers back to library
|
|
@@ -463,9 +431,7 @@ class Scholar:
|
|
|
463
431
|
# ----------------------------------------
|
|
464
432
|
# URL Finding (Orchestration)
|
|
465
433
|
# ----------------------------------------
|
|
466
|
-
async def _find_urls_for_doi_async(
|
|
467
|
-
self, doi: str, context
|
|
468
|
-
) -> Dict[str, Any]:
|
|
434
|
+
async def _find_urls_for_doi_async(self, doi: str, context) -> Dict[str, Any]:
|
|
469
435
|
"""Find all URLs for a DOI (orchestration layer).
|
|
470
436
|
|
|
471
437
|
Workflow:
|
|
@@ -495,10 +461,8 @@ class Scholar:
|
|
|
495
461
|
# Step 1: Resolve publisher URL
|
|
496
462
|
page = await context.new_page()
|
|
497
463
|
try:
|
|
498
|
-
url_publisher = (
|
|
499
|
-
|
|
500
|
-
doi, page
|
|
501
|
-
)
|
|
464
|
+
url_publisher = await resolve_publisher_url_by_navigating_to_doi_page(
|
|
465
|
+
doi, page
|
|
502
466
|
)
|
|
503
467
|
urls["url_publisher"] = url_publisher
|
|
504
468
|
finally:
|
|
@@ -516,22 +480,16 @@ class Scholar:
|
|
|
516
480
|
openurl_resolver = OpenURLResolver(config=self.config)
|
|
517
481
|
page = await context.new_page()
|
|
518
482
|
try:
|
|
519
|
-
url_openurl_resolved = await openurl_resolver.resolve_doi(
|
|
520
|
-
doi, page
|
|
521
|
-
)
|
|
483
|
+
url_openurl_resolved = await openurl_resolver.resolve_doi(doi, page)
|
|
522
484
|
urls["url_openurl_resolved"] = url_openurl_resolved
|
|
523
485
|
|
|
524
486
|
if url_openurl_resolved and url_openurl_resolved != "skipped":
|
|
525
|
-
urls_pdf = await url_finder.find_pdf_urls(
|
|
526
|
-
url_openurl_resolved
|
|
527
|
-
)
|
|
487
|
+
urls_pdf = await url_finder.find_pdf_urls(url_openurl_resolved)
|
|
528
488
|
finally:
|
|
529
489
|
await page.close()
|
|
530
490
|
|
|
531
491
|
# Deduplicate and store
|
|
532
|
-
urls["urls_pdf"] = (
|
|
533
|
-
self._deduplicate_pdf_urls(urls_pdf) if urls_pdf else []
|
|
534
|
-
)
|
|
492
|
+
urls["urls_pdf"] = self._deduplicate_pdf_urls(urls_pdf) if urls_pdf else []
|
|
535
493
|
|
|
536
494
|
return urls
|
|
537
495
|
|
|
@@ -576,9 +534,10 @@ class Scholar:
|
|
|
576
534
|
return {"downloaded": 0, "failed": 0, "errors": 0}
|
|
577
535
|
|
|
578
536
|
# Get authenticated browser context
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
537
|
+
(
|
|
538
|
+
browser,
|
|
539
|
+
context,
|
|
540
|
+
) = await self._browser_manager.get_authenticated_browser_and_context_async()
|
|
582
541
|
|
|
583
542
|
try:
|
|
584
543
|
# Initialize PDF downloader with browser context
|
|
@@ -613,14 +572,14 @@ class Scholar:
|
|
|
613
572
|
temp_pdf_path = downloaded_paths[0]
|
|
614
573
|
|
|
615
574
|
# Generate paper ID and create storage
|
|
616
|
-
paper_id = self.config.path_manager._generate_paper_id(
|
|
617
|
-
doi=doi
|
|
618
|
-
)
|
|
575
|
+
paper_id = self.config.path_manager._generate_paper_id(doi=doi)
|
|
619
576
|
storage_path = master_dir / paper_id
|
|
620
577
|
storage_path.mkdir(parents=True, exist_ok=True)
|
|
621
578
|
|
|
622
579
|
# Move PDF to MASTER library
|
|
623
|
-
pdf_filename =
|
|
580
|
+
pdf_filename = (
|
|
581
|
+
f"DOI_{doi.replace('/', '_').replace(':', '_')}.pdf"
|
|
582
|
+
)
|
|
624
583
|
master_pdf_path = storage_path / pdf_filename
|
|
625
584
|
shutil.move(str(temp_pdf_path), str(master_pdf_path))
|
|
626
585
|
|
|
@@ -641,18 +600,12 @@ class Scholar:
|
|
|
641
600
|
metadata["pdf_path"] = str(
|
|
642
601
|
master_pdf_path.relative_to(library_dir)
|
|
643
602
|
)
|
|
644
|
-
metadata["pdf_downloaded_at"] = (
|
|
645
|
-
|
|
646
|
-
)
|
|
647
|
-
metadata["pdf_size_bytes"] = (
|
|
648
|
-
master_pdf_path.stat().st_size
|
|
649
|
-
)
|
|
603
|
+
metadata["pdf_downloaded_at"] = datetime.now().isoformat()
|
|
604
|
+
metadata["pdf_size_bytes"] = master_pdf_path.stat().st_size
|
|
650
605
|
metadata["updated_at"] = datetime.now().isoformat()
|
|
651
606
|
|
|
652
607
|
with open(metadata_file, "w") as f:
|
|
653
|
-
json.dump(
|
|
654
|
-
metadata, f, indent=2, ensure_ascii=False
|
|
655
|
-
)
|
|
608
|
+
json.dump(metadata, f, indent=2, ensure_ascii=False)
|
|
656
609
|
|
|
657
610
|
# Update symlink using LibraryManager
|
|
658
611
|
if self.project not in ["master", "MASTER"]:
|
|
@@ -666,15 +619,11 @@ class Scholar:
|
|
|
666
619
|
)
|
|
667
620
|
stats["downloaded"] += 1
|
|
668
621
|
else:
|
|
669
|
-
logger.warning(
|
|
670
|
-
f"{self.name}: No PDF downloaded for DOI: {doi}"
|
|
671
|
-
)
|
|
622
|
+
logger.warning(f"{self.name}: No PDF downloaded for DOI: {doi}")
|
|
672
623
|
stats["failed"] += 1
|
|
673
624
|
|
|
674
625
|
except Exception as e:
|
|
675
|
-
logger.error(
|
|
676
|
-
f"{self.name}: Failed to organize PDF for {doi}: {e}"
|
|
677
|
-
)
|
|
626
|
+
logger.error(f"{self.name}: Failed to organize PDF for {doi}: {e}")
|
|
678
627
|
stats["errors"] += 1
|
|
679
628
|
stats["failed"] += 1
|
|
680
629
|
|
|
@@ -691,9 +640,10 @@ class Scholar:
|
|
|
691
640
|
results = {"downloaded": 0, "failed": 0, "errors": 0}
|
|
692
641
|
|
|
693
642
|
# Get authenticated browser context
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
643
|
+
(
|
|
644
|
+
browser,
|
|
645
|
+
context,
|
|
646
|
+
) = await self._browser_manager.get_authenticated_browser_and_context_async()
|
|
697
647
|
|
|
698
648
|
# Initialize authentication gateway (NEW)
|
|
699
649
|
auth_gateway = AuthenticationGateway(
|
|
@@ -731,9 +681,7 @@ class Scholar:
|
|
|
731
681
|
pdf_urls = urls.get("urls_pdf", [])
|
|
732
682
|
|
|
733
683
|
if not pdf_urls:
|
|
734
|
-
logger.warning(
|
|
735
|
-
f"{self.name}: No PDF URLs found for DOI: {doi}"
|
|
736
|
-
)
|
|
684
|
+
logger.warning(f"{self.name}: No PDF URLs found for DOI: {doi}")
|
|
737
685
|
results["failed"] += 1
|
|
738
686
|
continue
|
|
739
687
|
|
|
@@ -752,8 +700,7 @@ class Scholar:
|
|
|
752
700
|
|
|
753
701
|
# Download to temp location first
|
|
754
702
|
temp_output = (
|
|
755
|
-
Path("/tmp")
|
|
756
|
-
/ f"{doi.replace('/', '_').replace(':', '_')}.pdf"
|
|
703
|
+
Path("/tmp") / f"{doi.replace('/', '_').replace(':', '_')}.pdf"
|
|
757
704
|
)
|
|
758
705
|
|
|
759
706
|
# Download PDF using simple downloader
|
|
@@ -769,9 +716,7 @@ class Scholar:
|
|
|
769
716
|
# Step 4: Store PDF in MASTER library with proper organization
|
|
770
717
|
|
|
771
718
|
# Generate unique ID from DOI using PathManager
|
|
772
|
-
paper_id = self.config.path_manager._generate_paper_id(
|
|
773
|
-
doi=doi
|
|
774
|
-
)
|
|
719
|
+
paper_id = self.config.path_manager._generate_paper_id(doi=doi)
|
|
775
720
|
|
|
776
721
|
# Create MASTER storage directory
|
|
777
722
|
storage_path = master_dir / paper_id
|
|
@@ -817,23 +762,17 @@ class Scholar:
|
|
|
817
762
|
journal_clean = "Unknown"
|
|
818
763
|
|
|
819
764
|
# Format: Author-Year-Journal
|
|
820
|
-
readable_name =
|
|
821
|
-
f"{first_author}-{year_str}-{journal_clean}"
|
|
822
|
-
)
|
|
765
|
+
readable_name = f"{first_author}-{year_str}-{journal_clean}"
|
|
823
766
|
except:
|
|
824
767
|
pass
|
|
825
768
|
|
|
826
769
|
# Fallback to DOI if metadata extraction failed
|
|
827
770
|
if not readable_name:
|
|
828
|
-
readable_name = (
|
|
829
|
-
f"DOI_{doi.replace('/', '_').replace(':', '_')}"
|
|
830
|
-
)
|
|
771
|
+
readable_name = f"DOI_{doi.replace('/', '_').replace(':', '_')}"
|
|
831
772
|
|
|
832
773
|
# Copy PDF to MASTER storage with ORIGINAL filename to track how downloaded
|
|
833
774
|
# The PDF filename preserves the DOI format for tracking
|
|
834
|
-
pdf_filename = (
|
|
835
|
-
f"DOI_{doi.replace('/', '_').replace(':', '_')}.pdf"
|
|
836
|
-
)
|
|
775
|
+
pdf_filename = f"DOI_{doi.replace('/', '_').replace(':', '_')}.pdf"
|
|
837
776
|
master_pdf_path = storage_path / pdf_filename
|
|
838
777
|
shutil.copy2(downloaded_path, master_pdf_path)
|
|
839
778
|
|
|
@@ -868,9 +807,7 @@ class Scholar:
|
|
|
868
807
|
metadata[key] = value
|
|
869
808
|
|
|
870
809
|
# Add PDF information
|
|
871
|
-
metadata["pdf_path"] = str(
|
|
872
|
-
master_pdf_path.relative_to(library_dir)
|
|
873
|
-
)
|
|
810
|
+
metadata["pdf_path"] = str(master_pdf_path.relative_to(library_dir))
|
|
874
811
|
metadata["pdf_downloaded_at"] = datetime.now().isoformat()
|
|
875
812
|
metadata["pdf_size_bytes"] = master_pdf_path.stat().st_size
|
|
876
813
|
metadata["updated_at"] = datetime.now().isoformat()
|
|
@@ -922,9 +859,7 @@ class Scholar:
|
|
|
922
859
|
"""
|
|
923
860
|
import asyncio
|
|
924
861
|
|
|
925
|
-
return asyncio.run(
|
|
926
|
-
self.download_pdfs_from_dois_async(dois, output_dir)
|
|
927
|
-
)
|
|
862
|
+
return asyncio.run(self.download_pdfs_from_dois_async(dois, output_dir))
|
|
928
863
|
|
|
929
864
|
def download_pdfs_from_bibtex(
|
|
930
865
|
self,
|
|
@@ -947,14 +882,10 @@ class Scholar:
|
|
|
947
882
|
papers = self.load_bibtex(bibtex_input)
|
|
948
883
|
|
|
949
884
|
# Extract DOIs from papers
|
|
950
|
-
dois = [
|
|
951
|
-
paper.metadata.id.doi for paper in papers if paper.metadata.id.doi
|
|
952
|
-
]
|
|
885
|
+
dois = [paper.metadata.id.doi for paper in papers if paper.metadata.id.doi]
|
|
953
886
|
|
|
954
887
|
if not dois:
|
|
955
|
-
logger.warning(
|
|
956
|
-
f"{self.name}: No papers with DOIs found in BibTeX input"
|
|
957
|
-
)
|
|
888
|
+
logger.warning(f"{self.name}: No papers with DOIs found in BibTeX input")
|
|
958
889
|
return {"downloaded": 0, "failed": 0, "errors": 0}
|
|
959
890
|
|
|
960
891
|
logger.info(
|
|
@@ -985,9 +916,7 @@ class Scholar:
|
|
|
985
916
|
from ..core.Paper import Paper
|
|
986
917
|
import json
|
|
987
918
|
|
|
988
|
-
logger.info(
|
|
989
|
-
f"{self.name}: Loading papers from project: {project_name}"
|
|
990
|
-
)
|
|
919
|
+
logger.info(f"{self.name}: Loading papers from project: {project_name}")
|
|
991
920
|
|
|
992
921
|
library_dir = self.config.get_library_project_dir()
|
|
993
922
|
project_dir = library_dir / project_name
|
|
@@ -1044,21 +973,15 @@ class Scholar:
|
|
|
1044
973
|
# Convert to Papers collection
|
|
1045
974
|
from .Papers import Papers
|
|
1046
975
|
|
|
1047
|
-
papers_collection = Papers(
|
|
1048
|
-
|
|
1049
|
-
)
|
|
1050
|
-
papers_collection.library = (
|
|
1051
|
-
self._library
|
|
1052
|
-
) # Attach library for save operations
|
|
976
|
+
papers_collection = Papers(papers, config=self.config, project=self.project)
|
|
977
|
+
papers_collection.library = self._library # Attach library for save operations
|
|
1053
978
|
|
|
1054
979
|
return papers_collection
|
|
1055
980
|
|
|
1056
981
|
# ----------------------------------------
|
|
1057
982
|
# Searchers
|
|
1058
983
|
# ----------------------------------------
|
|
1059
|
-
def search_library(
|
|
1060
|
-
self, query: str, project: Optional[str] = None
|
|
1061
|
-
) -> Papers:
|
|
984
|
+
def search_library(self, query: str, project: Optional[str] = None) -> Papers:
|
|
1062
985
|
"""
|
|
1063
986
|
Search papers in local library.
|
|
1064
987
|
|
|
@@ -1114,9 +1037,7 @@ class Scholar:
|
|
|
1114
1037
|
]
|
|
1115
1038
|
all_papers.extend(matching_papers)
|
|
1116
1039
|
except Exception as e:
|
|
1117
|
-
logger.debug(
|
|
1118
|
-
f"{self.name}: Failed to search project {project}: {e}"
|
|
1119
|
-
)
|
|
1040
|
+
logger.debug(f"{self.name}: Failed to search project {project}: {e}")
|
|
1120
1041
|
|
|
1121
1042
|
return Papers(all_papers, config=self.config, project="search_results")
|
|
1122
1043
|
|
|
@@ -1159,9 +1080,7 @@ class Scholar:
|
|
|
1159
1080
|
"""
|
|
1160
1081
|
from ..storage.BibTeXHandler import BibTeXHandler
|
|
1161
1082
|
|
|
1162
|
-
bibtex_handler = BibTeXHandler(
|
|
1163
|
-
project=self.project, config=self.config
|
|
1164
|
-
)
|
|
1083
|
+
bibtex_handler = BibTeXHandler(project=self.project, config=self.config)
|
|
1165
1084
|
return bibtex_handler.papers_to_bibtex(papers, output_path)
|
|
1166
1085
|
|
|
1167
1086
|
# ----------------------------------------
|
|
@@ -1185,25 +1104,19 @@ class Scholar:
|
|
|
1185
1104
|
# Create project and info directories
|
|
1186
1105
|
if not project_dir.exists():
|
|
1187
1106
|
project_dir.mkdir(parents=True, exist_ok=True)
|
|
1188
|
-
logger.info(
|
|
1189
|
-
f"{self.name}: Auto-created project directory: {project}"
|
|
1190
|
-
)
|
|
1107
|
+
logger.info(f"{self.name}: Auto-created project directory: {project}")
|
|
1191
1108
|
|
|
1192
1109
|
# Ensure info directory exists
|
|
1193
1110
|
info_dir.mkdir(parents=True, exist_ok=True)
|
|
1194
1111
|
|
|
1195
1112
|
# Create/move metadata file to info directory
|
|
1196
|
-
old_metadata_file =
|
|
1197
|
-
project_dir / "project_metadata.json"
|
|
1198
|
-
) # Old location
|
|
1113
|
+
old_metadata_file = project_dir / "project_metadata.json" # Old location
|
|
1199
1114
|
metadata_file = info_dir / "project_metadata.json" # New location
|
|
1200
1115
|
|
|
1201
1116
|
# Move existing metadata file if it exists in old location
|
|
1202
1117
|
if old_metadata_file.exists() and not metadata_file.exists():
|
|
1203
1118
|
shutil.move(str(old_metadata_file), str(metadata_file))
|
|
1204
|
-
logger.info(
|
|
1205
|
-
f"{self.name}: Moved project metadata to info directory"
|
|
1206
|
-
)
|
|
1119
|
+
logger.info(f"{self.name}: Moved project metadata to info directory")
|
|
1207
1120
|
|
|
1208
1121
|
# Create metadata file if it doesn't exist
|
|
1209
1122
|
if not metadata_file.exists():
|
|
@@ -1268,9 +1181,7 @@ class Scholar:
|
|
|
1268
1181
|
metadata = json.load(f)
|
|
1269
1182
|
project_info.update(metadata)
|
|
1270
1183
|
except Exception as e:
|
|
1271
|
-
logger.debug(
|
|
1272
|
-
f"Failed to load metadata for {item.name}: {e}"
|
|
1273
|
-
)
|
|
1184
|
+
logger.debug(f"Failed to load metadata for {item.name}: {e}")
|
|
1274
1185
|
|
|
1275
1186
|
projects.append(project_info)
|
|
1276
1187
|
|
|
@@ -1399,13 +1310,13 @@ class Scholar:
|
|
|
1399
1310
|
|
|
1400
1311
|
project = project or self.project
|
|
1401
1312
|
|
|
1402
|
-
logger.info(f"{'='*60}")
|
|
1313
|
+
logger.info(f"{'=' * 60}")
|
|
1403
1314
|
logger.info(f"Processing paper")
|
|
1404
1315
|
if title:
|
|
1405
1316
|
logger.info(f"Title: {title[:50]}...")
|
|
1406
1317
|
if doi:
|
|
1407
1318
|
logger.info(f"DOI: {doi}")
|
|
1408
|
-
logger.info(f"{'='*60}")
|
|
1319
|
+
logger.info(f"{'=' * 60}")
|
|
1409
1320
|
|
|
1410
1321
|
# Stage 0: Resolve DOI from title (if needed)
|
|
1411
1322
|
if not doi and title:
|
|
@@ -1452,9 +1363,10 @@ class Scholar:
|
|
|
1452
1363
|
logger.info(f"\nStage 2: Checking/finding PDF URLs...")
|
|
1453
1364
|
if not self._library_manager.has_urls(paper_id):
|
|
1454
1365
|
logger.info(f"Finding PDF URLs for DOI: {doi}")
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1366
|
+
(
|
|
1367
|
+
browser,
|
|
1368
|
+
context,
|
|
1369
|
+
) = await self._browser_manager.get_authenticated_browser_and_context_async()
|
|
1458
1370
|
try:
|
|
1459
1371
|
url_finder = ScholarURLFinder(context, config=self.config)
|
|
1460
1372
|
urls = await url_finder.find_pdf_urls(doi)
|
|
@@ -1474,13 +1386,12 @@ class Scholar:
|
|
|
1474
1386
|
if not self._library_manager.has_pdf(paper_id):
|
|
1475
1387
|
logger.info(f"Downloading PDF...")
|
|
1476
1388
|
if paper.metadata.url.pdfs:
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1389
|
+
(
|
|
1390
|
+
browser,
|
|
1391
|
+
context,
|
|
1392
|
+
) = await self._browser_manager.get_authenticated_browser_and_context_async()
|
|
1480
1393
|
try:
|
|
1481
|
-
downloader = ScholarPDFDownloader(
|
|
1482
|
-
context, config=self.config
|
|
1483
|
-
)
|
|
1394
|
+
downloader = ScholarPDFDownloader(context, config=self.config)
|
|
1484
1395
|
|
|
1485
1396
|
pdf_url = (
|
|
1486
1397
|
paper.metadata.url.pdfs[0]["url"]
|
|
@@ -1494,39 +1405,29 @@ class Scholar:
|
|
|
1494
1405
|
)
|
|
1495
1406
|
if result and result.exists():
|
|
1496
1407
|
paper.metadata.path.pdfs.append(str(result))
|
|
1497
|
-
self._library_manager.save_paper_incremental(
|
|
1498
|
-
|
|
1499
|
-
)
|
|
1500
|
-
logger.success(
|
|
1501
|
-
f"{self.name}: Downloaded PDF, saved to storage"
|
|
1502
|
-
)
|
|
1408
|
+
self._library_manager.save_paper_incremental(paper_id, paper)
|
|
1409
|
+
logger.success(f"{self.name}: Downloaded PDF, saved to storage")
|
|
1503
1410
|
else:
|
|
1504
1411
|
logger.warning(f"{self.name}: Failed to download PDF")
|
|
1505
1412
|
finally:
|
|
1506
1413
|
await self._browser_manager.close()
|
|
1507
1414
|
else:
|
|
1508
|
-
logger.warning(
|
|
1509
|
-
f"{self.name}: No PDF URLs available for download"
|
|
1510
|
-
)
|
|
1415
|
+
logger.warning(f"{self.name}: No PDF URLs available for download")
|
|
1511
1416
|
else:
|
|
1512
1417
|
logger.info(f"{self.name}: PDF already in storage")
|
|
1513
1418
|
|
|
1514
1419
|
# Stage 4: Update project symlinks
|
|
1515
1420
|
if project and project not in ["master", "MASTER"]:
|
|
1516
|
-
logger.info(
|
|
1517
|
-
f"{self.name}: \nStage 4: Updating project symlinks..."
|
|
1518
|
-
)
|
|
1421
|
+
logger.info(f"{self.name}: \nStage 4: Updating project symlinks...")
|
|
1519
1422
|
self._library_manager.update_symlink(
|
|
1520
1423
|
master_storage_path=storage_path,
|
|
1521
1424
|
project=project,
|
|
1522
1425
|
)
|
|
1523
|
-
logger.success(
|
|
1524
|
-
f"{self.name}: Updated symlink in project: {project}"
|
|
1525
|
-
)
|
|
1426
|
+
logger.success(f"{self.name}: Updated symlink in project: {project}")
|
|
1526
1427
|
|
|
1527
|
-
logger.info(f"\n{'='*60}")
|
|
1428
|
+
logger.info(f"\n{'=' * 60}")
|
|
1528
1429
|
logger.success(f"{self.name}: Paper processing complete")
|
|
1529
|
-
logger.info(f"{'='*60}\n")
|
|
1430
|
+
logger.info(f"{'=' * 60}\n")
|
|
1530
1431
|
|
|
1531
1432
|
return paper
|
|
1532
1433
|
|
|
@@ -1601,12 +1502,12 @@ class Scholar:
|
|
|
1601
1502
|
papers = Papers(papers_list, project=project, config=self.config)
|
|
1602
1503
|
|
|
1603
1504
|
total = len(papers)
|
|
1604
|
-
logger.info(f"{self.name}: \n{'='*60}")
|
|
1505
|
+
logger.info(f"{self.name}: \n{'=' * 60}")
|
|
1605
1506
|
logger.info(
|
|
1606
1507
|
f"{self.name}: Processing {total} papers (max_concurrent={max_concurrent})"
|
|
1607
1508
|
)
|
|
1608
1509
|
logger.info(f"{self.name}: Project: {project}")
|
|
1609
|
-
logger.info(f"{self.name}: {'='*60}\n")
|
|
1510
|
+
logger.info(f"{self.name}: {'=' * 60}\n")
|
|
1610
1511
|
|
|
1611
1512
|
# Use semaphore for controlled parallelism
|
|
1612
1513
|
semaphore = asyncio.Semaphore(max_concurrent)
|
|
@@ -1614,9 +1515,7 @@ class Scholar:
|
|
|
1614
1515
|
async def process_with_semaphore(paper, index):
|
|
1615
1516
|
"""Process one paper with semaphore control."""
|
|
1616
1517
|
async with semaphore:
|
|
1617
|
-
logger.info(
|
|
1618
|
-
f"{self.name}: \n[{index}/{total}] Starting paper..."
|
|
1619
|
-
)
|
|
1518
|
+
logger.info(f"{self.name}: \n[{index}/{total}] Starting paper...")
|
|
1620
1519
|
try:
|
|
1621
1520
|
result = await self.process_paper_async(
|
|
1622
1521
|
title=paper.metadata.basic.title,
|
|
@@ -1630,10 +1529,7 @@ class Scholar:
|
|
|
1630
1529
|
return None
|
|
1631
1530
|
|
|
1632
1531
|
# Create tasks for all papers
|
|
1633
|
-
tasks = [
|
|
1634
|
-
process_with_semaphore(paper, i + 1)
|
|
1635
|
-
for i, paper in enumerate(papers)
|
|
1636
|
-
]
|
|
1532
|
+
tasks = [process_with_semaphore(paper, i + 1) for i, paper in enumerate(papers)]
|
|
1637
1533
|
|
|
1638
1534
|
# Process with controlled parallelism
|
|
1639
1535
|
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
@@ -1643,21 +1539,19 @@ class Scholar:
|
|
|
1643
1539
|
errors = 0
|
|
1644
1540
|
for i, result in enumerate(results):
|
|
1645
1541
|
if isinstance(result, Exception):
|
|
1646
|
-
logger.error(
|
|
1647
|
-
f"{self.name}: Paper {i+1} raised exception: {result}"
|
|
1648
|
-
)
|
|
1542
|
+
logger.error(f"{self.name}: Paper {i + 1} raised exception: {result}")
|
|
1649
1543
|
errors += 1
|
|
1650
1544
|
elif result is not None:
|
|
1651
1545
|
processed_papers.append(result)
|
|
1652
1546
|
|
|
1653
1547
|
# Summary
|
|
1654
|
-
logger.info(f"{self.name}: \n{'='*60}")
|
|
1548
|
+
logger.info(f"{self.name}: \n{'=' * 60}")
|
|
1655
1549
|
logger.info(f"{self.name}: Batch Processing Complete")
|
|
1656
1550
|
logger.info(f"{self.name}: Total: {total}")
|
|
1657
1551
|
logger.info(f"{self.name}: Successful: {len(processed_papers)}")
|
|
1658
1552
|
logger.info(f"{self.name}: Failed: {total - len(processed_papers)}")
|
|
1659
1553
|
logger.info(f"{self.name}: Errors: {errors}")
|
|
1660
|
-
logger.info(f"{self.name}: {'='*60}\n")
|
|
1554
|
+
logger.info(f"{self.name}: {'=' * 60}\n")
|
|
1661
1555
|
|
|
1662
1556
|
return Papers(processed_papers, project=project, config=self.config)
|
|
1663
1557
|
|
|
@@ -1697,10 +1591,7 @@ class Scholar:
|
|
|
1697
1591
|
@property
|
|
1698
1592
|
def _scholar_engine(self) -> ScholarEngine:
|
|
1699
1593
|
"""Get Scholar engine for search and enrichment (PRIVATE)."""
|
|
1700
|
-
if (
|
|
1701
|
-
not hasattr(self, "__scholar_engine")
|
|
1702
|
-
or self.__scholar_engine is None
|
|
1703
|
-
):
|
|
1594
|
+
if not hasattr(self, "__scholar_engine") or self.__scholar_engine is None:
|
|
1704
1595
|
self.__scholar_engine = ScholarEngine(config=self.config)
|
|
1705
1596
|
return self.__scholar_engine
|
|
1706
1597
|
|
|
@@ -1714,10 +1605,7 @@ class Scholar:
|
|
|
1714
1605
|
@property
|
|
1715
1606
|
def _browser_manager(self) -> ScholarBrowserManager:
|
|
1716
1607
|
"""Get browser manager service (PRIVATE)."""
|
|
1717
|
-
if (
|
|
1718
|
-
not hasattr(self, "__browser_manager")
|
|
1719
|
-
or self.__browser_manager is None
|
|
1720
|
-
):
|
|
1608
|
+
if not hasattr(self, "__browser_manager") or self.__browser_manager is None:
|
|
1721
1609
|
self.__browser_manager = ScholarBrowserManager(
|
|
1722
1610
|
auth_manager=self._auth_manager,
|
|
1723
1611
|
chrome_profile_name="system",
|
|
@@ -1728,10 +1616,7 @@ class Scholar:
|
|
|
1728
1616
|
@property
|
|
1729
1617
|
def _library_manager(self) -> LibraryManager:
|
|
1730
1618
|
"""Get library manager service - low-level operations (PRIVATE)."""
|
|
1731
|
-
if (
|
|
1732
|
-
not hasattr(self, "__library_manager")
|
|
1733
|
-
or self.__library_manager is None
|
|
1734
|
-
):
|
|
1619
|
+
if not hasattr(self, "__library_manager") or self.__library_manager is None:
|
|
1735
1620
|
self.__library_manager = LibraryManager(
|
|
1736
1621
|
project=self.project, config=self.config
|
|
1737
1622
|
)
|
|
@@ -1741,9 +1626,7 @@ class Scholar:
|
|
|
1741
1626
|
def _library(self) -> ScholarLibrary:
|
|
1742
1627
|
"""Get Scholar library service - high-level operations (PRIVATE)."""
|
|
1743
1628
|
if not hasattr(self, "__library") or self.__library is None:
|
|
1744
|
-
self.__library = ScholarLibrary(
|
|
1745
|
-
project=self.project, config=self.config
|
|
1746
|
-
)
|
|
1629
|
+
self.__library = ScholarLibrary(project=self.project, config=self.config)
|
|
1747
1630
|
return self.__library
|
|
1748
1631
|
|
|
1749
1632
|
|
|
@@ -1817,9 +1700,7 @@ if __name__ == "__main__":
|
|
|
1817
1700
|
|
|
1818
1701
|
# Create some sample papers with Pydantic structure
|
|
1819
1702
|
p1 = Paper()
|
|
1820
|
-
p1.metadata.basic.title =
|
|
1821
|
-
"Vision Transformer: An Image Is Worth 16x16 Words"
|
|
1822
|
-
)
|
|
1703
|
+
p1.metadata.basic.title = "Vision Transformer: An Image Is Worth 16x16 Words"
|
|
1823
1704
|
p1.metadata.basic.authors = ["Dosovitskiy, Alexey", "Beyer, Lucas"]
|
|
1824
1705
|
p1.metadata.basic.year = 2021
|
|
1825
1706
|
p1.metadata.basic.keywords = [
|
|
@@ -1881,9 +1762,7 @@ if __name__ == "__main__":
|
|
|
1881
1762
|
f" 📂 Loaded {len(project_papers)} papers from current project"
|
|
1882
1763
|
)
|
|
1883
1764
|
except:
|
|
1884
|
-
print(
|
|
1885
|
-
f" 📂 Current project is empty or doesn't exist yet"
|
|
1886
|
-
)
|
|
1765
|
+
print(f" 📂 Current project is empty or doesn't exist yet")
|
|
1887
1766
|
|
|
1888
1767
|
except Exception as e:
|
|
1889
1768
|
print(f" ⚠️ Workflow demo partially skipped: {e}")
|
|
@@ -1900,9 +1779,7 @@ if __name__ == "__main__":
|
|
|
1900
1779
|
|
|
1901
1780
|
# Search in current library (existing papers)
|
|
1902
1781
|
library_search = scholar.search_library("vision")
|
|
1903
|
-
print(
|
|
1904
|
-
f" 🔍 Library search for 'vision': {len(library_search)} results"
|
|
1905
|
-
)
|
|
1782
|
+
print(f" 🔍 Library search for 'vision': {len(library_search)} results")
|
|
1906
1783
|
|
|
1907
1784
|
except Exception as e:
|
|
1908
1785
|
print(f" ⚠️ Search demo skipped: {e}")
|
|
@@ -1911,9 +1788,7 @@ if __name__ == "__main__":
|
|
|
1911
1788
|
# Demonstrate configuration access
|
|
1912
1789
|
print("7. Configuration Management:")
|
|
1913
1790
|
print(f" ⚙️ Scholar directory: {scholar.config.paths.scholar_dir}")
|
|
1914
|
-
print(
|
|
1915
|
-
f" ⚙️ Library directory: {scholar.config.get_library_project_dir()}"
|
|
1916
|
-
)
|
|
1791
|
+
print(f" ⚙️ Library directory: {scholar.config.get_library_project_dir()}")
|
|
1917
1792
|
print(
|
|
1918
1793
|
f" ⚙️ Debug mode: {scholar.config.resolve('debug_mode', default=False)}"
|
|
1919
1794
|
)
|
|
@@ -1921,16 +1796,10 @@ if __name__ == "__main__":
|
|
|
1921
1796
|
|
|
1922
1797
|
# Demonstrate service access (internal components)
|
|
1923
1798
|
print("8. Service Components (Internal):")
|
|
1924
|
-
print(
|
|
1925
|
-
f" 🔧 Scholar Engine: {type(scholar._scholar_engine).__name__}"
|
|
1926
|
-
)
|
|
1799
|
+
print(f" 🔧 Scholar Engine: {type(scholar._scholar_engine).__name__}")
|
|
1927
1800
|
print(f" 🔧 Auth Manager: {type(scholar._auth_manager).__name__}")
|
|
1928
|
-
print(
|
|
1929
|
-
|
|
1930
|
-
)
|
|
1931
|
-
print(
|
|
1932
|
-
f" 🔧 Library Manager: {type(scholar._library_manager).__name__}"
|
|
1933
|
-
)
|
|
1801
|
+
print(f" 🔧 Browser Manager: {type(scholar._browser_manager).__name__}")
|
|
1802
|
+
print(f" 🔧 Library Manager: {type(scholar._library_manager).__name__}")
|
|
1934
1803
|
print()
|
|
1935
1804
|
|
|
1936
1805
|
# Demonstrate backup capabilities
|