snowflake-snowpark-python 1.57.0__tar.gz → 1.58.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/CHANGELOG.md +23 -0
- {snowflake_snowpark_python-1.57.0/src/snowflake_snowpark_python.egg-info → snowflake_snowpark_python-1.58.0}/PKG-INFO +1 -1
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/document_extract_proc.py +130 -32
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/document_reader.py +36 -2
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/document_reader_options.py +38 -2
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/proto/generated/ast_pb2.py +210 -208
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/proto/generated/ast_pb2.pyi +54 -6
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/dataframe_reader.py +19 -5
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/functions.py +65 -17
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_functions.py +21 -4
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/session.py +38 -4
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/version.py +1 -1
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0/src/snowflake_snowpark_python.egg-info}/PKG-INFO +1 -1
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/LICENSE.txt +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/MANIFEST.in +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/README.md +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/pyproject.toml +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/setup.cfg +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/setup.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_functions/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_functions/general_functions.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_functions/scalar_functions.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/analyzer.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/analyzer_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/binary_expression.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/binary_plan_node.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/datatype_mapper.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/expression.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/grouping_set.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/metadata_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/query_plan_analysis_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/schema_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/select_statement.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/snowflake_plan_node.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/sort_expression.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/table_function.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/table_merge_expression.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/unary_expression.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/unary_plan_node.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/analyzer/window_expression.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/ast/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/ast/batch.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/ast/builder.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/ast/utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/code_generation.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/compiler/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/compiler/cte_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/compiler/large_query_breakdown.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/compiler/plan_compiler.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/compiler/query_generator.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/compiler/repeated_subquery_elimination.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/compiler/telemetry_constants.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/compiler/utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/datasource_partitioner.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/datasource_reader.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/datasource_typing.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/dbms_dialects/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/dbms_dialects/base_dialect.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/dbms_dialects/databricks_dialect.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/dbms_dialects/mysql_dialect.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/dbms_dialects/oracledb_dialect.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/dbms_dialects/postgresql_dialect.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/dbms_dialects/sqlite3_dialect.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/dbms_dialects/sqlserver_dialect.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/drivers/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/drivers/base_driver.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/drivers/databricks_driver.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/drivers/oracledb_driver.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/drivers/psycopg2_driver.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/drivers/pymsql_driver.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/drivers/pyodbc_driver.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/drivers/sqlite_driver.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/jdbc.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/data_source/utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/debug_config.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/debug_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/df_trace.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/error_message.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/event_table_telemetry.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/open_telemetry.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/options.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/packaging_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/pdf_reader.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/polars_backend.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/proto/generated/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/select_projection_complexity_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/server_connection.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/snowpark_profiler.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/telemetry.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/temp_table_auto_cleaner.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/type_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/udf_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/xml_reader.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/xml_schema_inference.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/_internal/xpath_handlers.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/async_job.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/catalog.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/column.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/context.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/dataframe.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/dataframe_ai_functions.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/dataframe_analytics_functions.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/dataframe_na_functions.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/dataframe_profiler.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/dataframe_stat_functions.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/dataframe_writer.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/exceptions.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/file_operation.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/files.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/lineage.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_analyzer.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_connection.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_constants.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_nop_analyzer.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_nop_connection.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_nop_plan.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_options.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_pandas_util.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_plan.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_plan_builder.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_secret_detector.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_select_statement.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_snowflake_data_type.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_snowflake_to_pandas_converter.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_stage_registry.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_stored_procedure.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_telemetry.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_udaf.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_udf.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_udf_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_udtf.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_util.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/_window_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/mock/exceptions.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/config/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/config/__main__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/config/envvars.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/config/pubsub.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/conftest.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/aggregation_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/align_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/apply_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/binary_op_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/concat_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/cumulative_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/cut_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/frame.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/generator_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/get_dummies_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/groupby_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/indexing_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/io_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/isin_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/join_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/ordered_dataframe.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/pivot_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/resample_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/row_count_estimation.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/session.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/snowpark_pandas_types.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/statement_params_constants.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/telemetry.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/timestamp_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/transpose_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/type_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/unpivot_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/where_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_internal/window_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/_typing.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/compiler/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/compiler/doc_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/compiler/ray_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/compiler/snowflake_query_compiler.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/docstrings/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/docstrings/base.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/docstrings/dataframe.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/docstrings/datetime_index.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/docstrings/general.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/docstrings/groupby.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/docstrings/index.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/docstrings/io.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/docstrings/resample.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/docstrings/series.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/docstrings/series_utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/docstrings/shared_docs.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/docstrings/timedelta_index.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/docstrings/window.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/base_overrides.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/dataframe_extensions.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/dataframe_groupby_overrides.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/dataframe_overrides.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/datetime_index.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/general_overrides.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/index.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/indexing_overrides.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/io_overrides.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/pd_extensions.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/resample_overrides.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/resampler_groupby_overrides.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/rolling_groupby_overrides.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/series_extensions.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/series_groupby_overrides.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/series_overrides.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/snow_partition_iterator.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/timedelta_index.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/extensions/window_overrides.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/io/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/io/factories.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/io/snow_io.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/utils/__init__.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/utils/error_message.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/utils/exceptions.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/utils/frontend_constants.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/utils/numpy_to_pandas.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/plugin/utils/warning_message.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/modin/utils.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/py.typed +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/query_history.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/relational_grouped_dataframe.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/row.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/secrets.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/stored_procedure.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/stored_procedure_profiler.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/table.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/table_function.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/testing.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/types.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/udaf.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/udf.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/udf_profiler.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/udtf.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake/snowpark/window.py +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake_snowpark_python.egg-info/SOURCES.txt +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake_snowpark_python.egg-info/dependency_links.txt +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake_snowpark_python.egg-info/namespace_packages.txt +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake_snowpark_python.egg-info/not-zip-safe +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake_snowpark_python.egg-info/requires.txt +0 -0
- {snowflake_snowpark_python-1.57.0 → snowflake_snowpark_python-1.58.0}/src/snowflake_snowpark_python.egg-info/top_level.txt +0 -0
|
@@ -1,5 +1,28 @@
|
|
|
1
1
|
# Release History
|
|
2
2
|
|
|
3
|
+
## 1.58.0 (2026-11-19)
|
|
4
|
+
|
|
5
|
+
### Snowpark Python API Updates
|
|
6
|
+
|
|
7
|
+
#### New Features
|
|
8
|
+
|
|
9
|
+
- `DataFrameReader.documents()` (Private Preview) reads with a `schema` or a `prompt` now return the `AI_COMPLETE` token usage of each document in `INPUT_TOKENS` and `OUTPUT_TOKENS` columns. A schema field can no longer be named like one of these columns.
|
|
10
|
+
|
|
11
|
+
#### Improvements
|
|
12
|
+
|
|
13
|
+
- `to_timestamp`, `to_char`/`to_varchar`, `to_binary` and `try_to_binary` now accept the format as a `Column` or a literal `str`. `to_binary` and `try_to_binary` now pass an empty-string format to Snowflake instead of silently ignoring it.
|
|
14
|
+
|
|
15
|
+
#### Bug Fixes
|
|
16
|
+
|
|
17
|
+
- Fixed a bug where `to_binary` and `try_to_binary` raised `TypeError` when the format was a `Column`.
|
|
18
|
+
- Fixed a bug where `DataFrameReader.documents()` (Private Preview) reads with a schema rejected `.jpg`, `.jpeg`, `.png`, `.gif` and `.webp` images with a "file type is not supported" error.
|
|
19
|
+
|
|
20
|
+
### Snowpark Local Testing Updates
|
|
21
|
+
|
|
22
|
+
#### Bug Fixes
|
|
23
|
+
|
|
24
|
+
- Fixed a bug where `to_binary` in local testing treated an empty-string format as `HEX` and failed with an unclear error for an unknown format, instead of raising an invalid binary format error as Snowflake does.
|
|
25
|
+
|
|
3
26
|
## 1.57.0 (2026-11-12)
|
|
4
27
|
|
|
5
28
|
### Snowpark Python API Updates
|
|
@@ -18,7 +18,8 @@ the steps of ``DocumentExtraction`` just above it, in order:
|
|
|
18
18
|
(``is_throttle_error``). If most documents are failing, no further rounds are
|
|
19
19
|
submitted (``should_abort``): the documents still waiting get an error and the
|
|
20
20
|
finished rows are returned.
|
|
21
|
-
4. back in ``run``: merge each file's parts, in page order, into one row
|
|
21
|
+
4. back in ``run``: merge each file's parts, in page order, into one row, with the
|
|
22
|
+
token usage of every call made for the file summed (``FileState.usage``).
|
|
22
23
|
|
|
23
24
|
Calls go to the first of the candidate models. A call AI_COMPLETE reports as an
|
|
24
25
|
unknown or unavailable model is not a document failure: ``model_missing`` moves the read
|
|
@@ -234,14 +235,32 @@ def is_throttle_error(error: str) -> bool:
|
|
|
234
235
|
# rejected 15.19 MB with a bare "malformed request"); parts stay well under that.
|
|
235
236
|
MAX_FILE_BYTES = 12_000_000
|
|
236
237
|
|
|
237
|
-
# Every file is read whole into the procedure's memory before it is cut into parts, so a
|
|
238
|
-
# file larger than this (by the size LIST reports) is not read: it gets its own error
|
|
239
|
-
# row instead of exhausting the procedure's memory.
|
|
240
238
|
# File types AI_COMPLETE accepts for Claude models. Other files are rejected before their
|
|
241
|
-
# bytes are read, so they are never sent to the model.
|
|
239
|
+
# bytes are read, so they are never sent to the model. Images are not PDFs, so each is
|
|
240
|
+
# sent whole in one call; AI_COMPLETE reports an image over its size limit (3.75 MB for
|
|
241
|
+
# Claude) as that file's error.
|
|
242
242
|
SUPPORTED_FILE_EXTENSIONS = frozenset(
|
|
243
|
-
{
|
|
243
|
+
{
|
|
244
|
+
".txt",
|
|
245
|
+
".md",
|
|
246
|
+
".pdf",
|
|
247
|
+
".doc",
|
|
248
|
+
".docx",
|
|
249
|
+
".xls",
|
|
250
|
+
".xlsx",
|
|
251
|
+
".csv",
|
|
252
|
+
".xhtml",
|
|
253
|
+
".jpg",
|
|
254
|
+
".jpeg",
|
|
255
|
+
".png",
|
|
256
|
+
".gif",
|
|
257
|
+
".webp",
|
|
258
|
+
}
|
|
244
259
|
)
|
|
260
|
+
|
|
261
|
+
# Every file is read whole into the procedure's memory before it is cut into parts, so a
|
|
262
|
+
# file larger than this (by the size LIST reports) is not read: it gets its own error
|
|
263
|
+
# row instead of exhausting the procedure's memory.
|
|
245
264
|
MAX_FILE_READ_BYTES = 300_000_000
|
|
246
265
|
|
|
247
266
|
|
|
@@ -957,11 +976,23 @@ def qualify_listed_name(path: str, listed_name: str) -> str:
|
|
|
957
976
|
|
|
958
977
|
|
|
959
978
|
# claude-sonnet-5-5 rejects response_format, so the schema travels in the predicate and
|
|
960
|
-
# the answer comes back as text.
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
"
|
|
964
|
-
|
|
979
|
+
# the answer comes back as text. show_details adds the call's token usage beside it
|
|
980
|
+
# (see split_details); with the session's token switch off the call goes without it.
|
|
981
|
+
def ai_complete_sql(show_details: bool) -> str:
|
|
982
|
+
"""The AI_COMPLETE statement each call runs, with or without show_details."""
|
|
983
|
+
return (
|
|
984
|
+
"SELECT AI_COMPLETE(model => ?, predicate => ?, file => TO_FILE(?), "
|
|
985
|
+
"model_parameters => PARSE_JSON(?), "
|
|
986
|
+
+ ("show_details => TRUE, " if show_details else "")
|
|
987
|
+
+ "return_error_details => TRUE)"
|
|
988
|
+
)
|
|
989
|
+
|
|
990
|
+
|
|
991
|
+
# The token counts of a show_details ``usage`` object, summed per file over every call
|
|
992
|
+
# that reported them (``FileState.usage``) and returned in this order (``result_row``),
|
|
993
|
+
# as the output columns ``USAGE_COLUMNS`` (TOKEN_COLUMNS in document_reader_options.py).
|
|
994
|
+
USAGE_FIELDS = ("prompt_tokens", "completion_tokens")
|
|
995
|
+
USAGE_COLUMNS = ("INPUT_TOKENS", "OUTPUT_TOKENS")
|
|
965
996
|
|
|
966
997
|
|
|
967
998
|
_SCORES_INSTRUCTION = (
|
|
@@ -1168,6 +1199,54 @@ def _from_variant_text(field: Any) -> Any:
|
|
|
1168
1199
|
return decoded if isinstance(decoded, str) else field
|
|
1169
1200
|
|
|
1170
1201
|
|
|
1202
|
+
def _details_text(details: Any) -> Optional[str]:
|
|
1203
|
+
"""The answer text of a show_details object, ``{"choices": [{"messages": <text>}],
|
|
1204
|
+
"usage": {...}, ...}``, or None when ``details`` is not one."""
|
|
1205
|
+
if not isinstance(details, dict) or not isinstance(details.get("choices"), list):
|
|
1206
|
+
return None
|
|
1207
|
+
first = details["choices"][0] if details["choices"] else None
|
|
1208
|
+
return first.get("messages") if isinstance(first, dict) else None
|
|
1209
|
+
|
|
1210
|
+
|
|
1211
|
+
def split_details(raw: Any) -> Tuple[Any, Optional[Dict[str, Any]]]:
|
|
1212
|
+
"""``(raw, usage)`` from one ``AI_COMPLETE(..., show_details => TRUE,
|
|
1213
|
+
return_error_details => TRUE)`` result: ``raw`` in the shape ``read_answer`` takes,
|
|
1214
|
+
with the answer text in place of the show_details object around it, and that
|
|
1215
|
+
object's token ``usage``. In the native Arrow form (see ``read_answer``) the object
|
|
1216
|
+
arrives as JSON text. A result without one, such as a failed call's, is returned as
|
|
1217
|
+
it is, with no usage."""
|
|
1218
|
+
if isinstance(raw, str):
|
|
1219
|
+
result = json.loads(raw)
|
|
1220
|
+
details = result.get("value")
|
|
1221
|
+
text = _details_text(details)
|
|
1222
|
+
if text is None:
|
|
1223
|
+
return raw, None
|
|
1224
|
+
return json.dumps({"value": text, "error": result.get("error")}), details.get(
|
|
1225
|
+
"usage"
|
|
1226
|
+
)
|
|
1227
|
+
details = raw.get("value")
|
|
1228
|
+
if isinstance(details, str):
|
|
1229
|
+
try:
|
|
1230
|
+
details = json.loads(details)
|
|
1231
|
+
except json.JSONDecodeError:
|
|
1232
|
+
return raw, None
|
|
1233
|
+
text = _details_text(details)
|
|
1234
|
+
if text is None:
|
|
1235
|
+
return raw, None
|
|
1236
|
+
return {"value": json.dumps(text), "error": raw.get("error")}, details.get("usage")
|
|
1237
|
+
|
|
1238
|
+
|
|
1239
|
+
def add_usage(total: Dict[str, int], usage: Any) -> None:
|
|
1240
|
+
"""Add a show_details ``usage`` object's ``USAGE_FIELDS`` into ``total``. A missing
|
|
1241
|
+
or non-integer count is skipped, so it does not hide the rest."""
|
|
1242
|
+
if not isinstance(usage, dict):
|
|
1243
|
+
return
|
|
1244
|
+
for name in USAGE_FIELDS:
|
|
1245
|
+
count = usage.get(name)
|
|
1246
|
+
if isinstance(count, int) and not isinstance(count, bool):
|
|
1247
|
+
total[name] = total.get(name, 0) + count
|
|
1248
|
+
|
|
1249
|
+
|
|
1171
1250
|
def read_answer(raw: Any) -> Tuple[Any, Optional[str], Optional[Dict[str, Any]]]:
|
|
1172
1251
|
"""(value, error, salvaged status) from one ``AI_COMPLETE(..., return_error_details =>
|
|
1173
1252
|
TRUE)`` result. The status is salvaged only from a text answer that does not parse.
|
|
@@ -1195,7 +1274,8 @@ class FileState:
|
|
|
1195
1274
|
(``splittable`` is False). The reader is dropped once the file's batch is done, to
|
|
1196
1275
|
free its bytes; ``splittable`` still says whether it was a PDF. ``failed`` marks a
|
|
1197
1276
|
file whose bytes could not be read at all: it has its error, no tasks, and still
|
|
1198
|
-
gets an output row.
|
|
1277
|
+
gets an output row. ``usage`` sums the token counts of every call made for the file,
|
|
1278
|
+
planning calls and calls sent again included (``add_usage``)."""
|
|
1199
1279
|
|
|
1200
1280
|
path: str
|
|
1201
1281
|
reader: Any
|
|
@@ -1206,6 +1286,7 @@ class FileState:
|
|
|
1206
1286
|
parts: List[Tuple[int, Dict[str, Any]]] = field(default_factory=list)
|
|
1207
1287
|
errors: List[str] = field(default_factory=list)
|
|
1208
1288
|
failed: bool = False
|
|
1289
|
+
usage: Dict[str, int] = field(default_factory=dict)
|
|
1209
1290
|
splittable: bool = field(init=False)
|
|
1210
1291
|
|
|
1211
1292
|
def __post_init__(self) -> None:
|
|
@@ -1237,10 +1318,12 @@ class Submitted:
|
|
|
1237
1318
|
|
|
1238
1319
|
|
|
1239
1320
|
def result_row(file: FileState, fields: Optional[List[str]] = None) -> List[Any]:
|
|
1240
|
-
"""The output row ``[STAGE_PATH, VALUE, ERROR, SCORES]``
|
|
1241
|
-
in page order, so a table split across ranges comes
|
|
1242
|
-
per-field confidence scores
|
|
1243
|
-
|
|
1321
|
+
"""The output row ``[STAGE_PATH, VALUE, ERROR, SCORES, INPUT_TOKENS, OUTPUT_TOKENS]``
|
|
1322
|
+
of one file: its parts merged in page order, so a table split across ranges comes
|
|
1323
|
+
back as one table, the per-field confidence scores
|
|
1324
|
+
merged by ``merge_scores`` (None when there are none), and its summed token counts
|
|
1325
|
+
(each None when no call reported it). A failure while merging is that file's error,
|
|
1326
|
+
not the read's."""
|
|
1244
1327
|
errors = list(file.errors)
|
|
1245
1328
|
value = scores = None
|
|
1246
1329
|
parts = [part for _, part in sorted(file.parts, key=lambda item: item[0])]
|
|
@@ -1252,7 +1335,8 @@ def result_row(file: FileState, fields: Optional[List[str]] = None) -> List[Any]
|
|
|
1252
1335
|
except Exception as exc:
|
|
1253
1336
|
value = scores = None
|
|
1254
1337
|
errors.append(f"could not merge the extracted parts: {exc}")
|
|
1255
|
-
|
|
1338
|
+
tokens = [file.usage.get(name) for name in USAGE_FIELDS]
|
|
1339
|
+
return [file.path, value, "; ".join(errors) if errors else None, scores, *tokens]
|
|
1256
1340
|
|
|
1257
1341
|
|
|
1258
1342
|
def document_outcomes(
|
|
@@ -1284,12 +1368,15 @@ class DocumentExtraction:
|
|
|
1284
1368
|
models: List[str],
|
|
1285
1369
|
budgets: Dict[str, Any],
|
|
1286
1370
|
work_stage: str,
|
|
1371
|
+
show_details: bool = True,
|
|
1287
1372
|
) -> None:
|
|
1288
1373
|
"""Precompute the schema sent to the model (with the status field added and
|
|
1289
1374
|
every scalar made nullable) and its instruction text. ``models`` are the
|
|
1290
1375
|
candidates in order of preference; calls use the first until AI_COMPLETE reports
|
|
1291
|
-
it unknown (see ``model_missing``).
|
|
1376
|
+
it unknown (see ``model_missing``). ``show_details`` says whether calls ask for
|
|
1377
|
+
their token usage (``ai_complete_sql``)."""
|
|
1292
1378
|
self.session = session
|
|
1379
|
+
self.call_sql = ai_complete_sql(show_details)
|
|
1293
1380
|
self.schema = response_format["schema"]
|
|
1294
1381
|
self.sent_schema = with_status(response_format)["schema"]
|
|
1295
1382
|
self.instruction = "\n\n" + schema_instruction(self.sent_schema)
|
|
@@ -1481,19 +1568,22 @@ class DocumentExtraction:
|
|
|
1481
1568
|
file.calls += 1
|
|
1482
1569
|
model = self.model
|
|
1483
1570
|
job = self.session.sql(
|
|
1484
|
-
|
|
1571
|
+
self.call_sql,
|
|
1485
1572
|
params=[model, predicate, staged, json.dumps({"max_tokens": max_tokens})],
|
|
1486
1573
|
).collect_nowait()
|
|
1487
1574
|
return Submitted(job, model)
|
|
1488
1575
|
|
|
1489
1576
|
@staticmethod
|
|
1490
1577
|
def collect(
|
|
1491
|
-
submitted: Submitted
|
|
1578
|
+
file: FileState, submitted: Submitted
|
|
1492
1579
|
) -> Tuple[Any, Optional[str], Optional[Dict[str, Any]]]:
|
|
1493
1580
|
"""The call's answer via ``read_answer``, or ``(None, str(exc), None)`` if it
|
|
1494
|
-
raised.
|
|
1581
|
+
raised. Its token usage, when it reports any, is added to ``file``'s whatever
|
|
1582
|
+
the answer: a call cut off by max_tokens is billed like one that fit."""
|
|
1495
1583
|
try:
|
|
1496
|
-
|
|
1584
|
+
raw, usage = split_details(submitted.job.result()[0][0])
|
|
1585
|
+
add_usage(file.usage, usage)
|
|
1586
|
+
return read_answer(raw)
|
|
1497
1587
|
except Exception as exc:
|
|
1498
1588
|
return None, str(exc), None
|
|
1499
1589
|
|
|
@@ -1599,7 +1689,7 @@ class DocumentExtraction:
|
|
|
1599
1689
|
throttled, or sent to a model AI_COMPLETE did not know."""
|
|
1600
1690
|
again = []
|
|
1601
1691
|
for file, a, b, submitted in jobs:
|
|
1602
|
-
value, error, _ = self.collect(submitted)
|
|
1692
|
+
value, error, _ = self.collect(file, submitted)
|
|
1603
1693
|
if self.model_missing(file, submitted, error):
|
|
1604
1694
|
again.append((file, a, b)) # plan it again with the new model
|
|
1605
1695
|
continue
|
|
@@ -1737,7 +1827,7 @@ class DocumentExtraction:
|
|
|
1737
1827
|
"""Record one call's answer and return the tasks it leaves. A call whose model
|
|
1738
1828
|
AI_COMPLETE does not know, or that was throttled, leaves its own task, to be sent
|
|
1739
1829
|
again."""
|
|
1740
|
-
value, error, salvaged = self.collect(submitted)
|
|
1830
|
+
value, error, salvaged = self.collect(file, submitted)
|
|
1741
1831
|
if self.model_missing(file, submitted, error):
|
|
1742
1832
|
return [(file, start, end, budget_name)]
|
|
1743
1833
|
if self.throttled(file, (id(file), start, end), error):
|
|
@@ -1785,6 +1875,7 @@ def run(
|
|
|
1785
1875
|
call_budgets: str,
|
|
1786
1876
|
max_calls_per_file: int,
|
|
1787
1877
|
work_stage: str,
|
|
1878
|
+
show_details: bool = True,
|
|
1788
1879
|
) -> DataFrame:
|
|
1789
1880
|
"""Extract the schema's fields from every file under ``path``; one output row per
|
|
1790
1881
|
file, however many AI_COMPLETE calls it took.
|
|
@@ -1801,6 +1892,8 @@ def run(
|
|
|
1801
1892
|
max_calls_per_file: starting cap on calls per file; raised for files whose
|
|
1802
1893
|
answers turn out large, up to HARD_MAX_CALLS_PER_FILE.
|
|
1803
1894
|
work_stage: stage where page-range part files are written (the session stage).
|
|
1895
|
+
show_details: whether calls ask AI_COMPLETE for their token usage; False (the
|
|
1896
|
+
session's token switch off) leaves the token columns None.
|
|
1804
1897
|
|
|
1805
1898
|
Raises:
|
|
1806
1899
|
NoModelAvailable (a RuntimeError): AI_COMPLETE knows none of ``models`` (not
|
|
@@ -1808,15 +1901,18 @@ def run(
|
|
|
1808
1901
|
|
|
1809
1902
|
Returns:
|
|
1810
1903
|
STAGE_PATH, VALUE (the merged answer as JSON, or None when no part succeeded),
|
|
1811
|
-
ERROR (every failure for the file, joined with "; ", or None)
|
|
1812
|
-
per-field confidence scores as JSON, see ``merge_scores``, or None)
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1904
|
+
ERROR (every failure for the file, joined with "; ", or None), SCORES (the
|
|
1905
|
+
per-field confidence scores as JSON, see ``merge_scores``, or None), and
|
|
1906
|
+
INPUT_TOKENS and OUTPUT_TOKENS (AI_COMPLETE's prompt_tokens and
|
|
1907
|
+
completion_tokens summed over every call made for the file, see
|
|
1908
|
+
``FileState.usage``, or None when no call reported them).
|
|
1909
|
+
A failure reading, staging, submitting or merging is a file's own ERROR and
|
|
1910
|
+
does not stop the read. When more than ``MAX_FAILED_FRACTION`` of the readable
|
|
1911
|
+
PDFs finished so far failed (once ``MIN_DOCUMENTS_BEFORE_ABORT`` have), the read
|
|
1912
|
+
stops submitting: files with page ranges still waiting, and files of batches not
|
|
1913
|
+
yet read, say so in their ERROR, and every finished row is returned as it is.
|
|
1818
1914
|
"""
|
|
1819
|
-
from snowflake.snowpark.types import StringType, StructField, StructType
|
|
1915
|
+
from snowflake.snowpark.types import LongType, StringType, StructField, StructType
|
|
1820
1916
|
|
|
1821
1917
|
extraction = DocumentExtraction(
|
|
1822
1918
|
session,
|
|
@@ -1825,6 +1921,7 @@ def run(
|
|
|
1825
1921
|
json.loads(models),
|
|
1826
1922
|
json.loads(call_budgets),
|
|
1827
1923
|
work_stage,
|
|
1924
|
+
show_details,
|
|
1828
1925
|
)
|
|
1829
1926
|
# 1. List the files under the path, with their sizes.
|
|
1830
1927
|
listed = extraction.list_files(path)
|
|
@@ -1841,6 +1938,7 @@ def run(
|
|
|
1841
1938
|
StructField("VALUE", StringType()),
|
|
1842
1939
|
StructField("ERROR", StringType()),
|
|
1843
1940
|
StructField("SCORES", StringType()),
|
|
1941
|
+
*[StructField(name, LongType()) for name in USAGE_COLUMNS],
|
|
1844
1942
|
]
|
|
1845
1943
|
)
|
|
1846
1944
|
return session.create_dataframe(rows, schema=schema)
|
|
@@ -20,6 +20,8 @@ from snowflake.snowpark._internal.document_reader_options import (
|
|
|
20
20
|
PAGE_INDEX_COLUMN,
|
|
21
21
|
RESPONSE_COLUMN,
|
|
22
22
|
SOURCE_FILE_COLUMN,
|
|
23
|
+
TOKEN_COLUMNS,
|
|
24
|
+
TOKEN_USAGE_FIELDS,
|
|
23
25
|
TOTAL_PAGES_COLUMN,
|
|
24
26
|
_EXTRACT_ERROR_COLUMN,
|
|
25
27
|
_PARSE_ERROR_COLUMN,
|
|
@@ -57,7 +59,9 @@ from snowflake.snowpark.functions import (
|
|
|
57
59
|
when,
|
|
58
60
|
)
|
|
59
61
|
from snowflake.snowpark.types import (
|
|
62
|
+
BooleanType,
|
|
60
63
|
IntegerType,
|
|
64
|
+
LongType,
|
|
61
65
|
StringType,
|
|
62
66
|
StructField,
|
|
63
67
|
StructType,
|
|
@@ -167,6 +171,8 @@ def output_columns(options: DocumentReaderOptions) -> List[str]:
|
|
|
167
171
|
names.append(EXTRACTION_SCORES_COLUMN)
|
|
168
172
|
elif options.complete_enabled:
|
|
169
173
|
names.append(RESPONSE_COLUMN)
|
|
174
|
+
if options.tokens_enabled:
|
|
175
|
+
names.extend(TOKEN_COLUMNS)
|
|
170
176
|
if options.has_corrupt_record_column:
|
|
171
177
|
names.append(quote_name_without_upper_casing(options.corrupt_record_column))
|
|
172
178
|
return names
|
|
@@ -547,10 +553,13 @@ def build_ai_complete_call(
|
|
|
547
553
|
# single-string variant. Parse didn't run: input_col is SOURCE_FILE, passed to
|
|
548
554
|
# AI_COMPLETE's file variant instead -- whose prompt argument must then be a plain
|
|
549
555
|
# string, not a Column, so there is no CONTENT to fold in. With no schema there is
|
|
550
|
-
# no response_format and AI_COMPLETE answers the prompt as free text.
|
|
556
|
+
# no response_format and AI_COMPLETE answers the prompt as free text. show_details
|
|
557
|
+
# adds the call's token usage beside the answer (see extract_with_ai_complete);
|
|
558
|
+
# with the session's token switch off it is not sent at all.
|
|
551
559
|
response_format = (
|
|
552
560
|
options.extraction.ai_complete_format if options.extract_enabled else None
|
|
553
561
|
)
|
|
562
|
+
show_details = True if options.tokens_enabled else None
|
|
554
563
|
if options.parse_enabled:
|
|
555
564
|
prompt_text = (
|
|
556
565
|
input_col
|
|
@@ -562,6 +571,7 @@ def build_ai_complete_call(
|
|
|
562
571
|
prompt_text,
|
|
563
572
|
model_parameters={"max_tokens": _MAX_OUTPUT_TOKENS},
|
|
564
573
|
response_format=response_format,
|
|
574
|
+
show_details=show_details,
|
|
565
575
|
return_error_details=True,
|
|
566
576
|
)
|
|
567
577
|
return ai_complete(
|
|
@@ -570,10 +580,26 @@ def build_ai_complete_call(
|
|
|
570
580
|
file=input_col,
|
|
571
581
|
model_parameters={"max_tokens": _MAX_OUTPUT_TOKENS},
|
|
572
582
|
response_format=response_format,
|
|
583
|
+
show_details=show_details,
|
|
573
584
|
return_error_details=True,
|
|
574
585
|
)
|
|
575
586
|
|
|
576
587
|
|
|
588
|
+
def details_answer(value: Column, structured: bool) -> Column:
|
|
589
|
+
"""The answer inside an ``AI_COMPLETE(..., show_details => TRUE)`` value: the
|
|
590
|
+
``structured_output`` object when a response_format was sent, else the text of the
|
|
591
|
+
one ``choices`` entry."""
|
|
592
|
+
if structured:
|
|
593
|
+
return value["structured_output"][0]["raw_message"]
|
|
594
|
+
return value["choices"][0]["messages"]
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
def usage_columns(usage: Column) -> List[Column]:
|
|
598
|
+
"""``TOKEN_COLUMNS``' values from a ``show_details`` ``usage`` object, NULL when
|
|
599
|
+
the call failed and reported none."""
|
|
600
|
+
return [usage[TOKEN_USAGE_FIELDS[name]].cast(LongType()) for name in TOKEN_COLUMNS]
|
|
601
|
+
|
|
602
|
+
|
|
577
603
|
def extract_with_ai_complete(
|
|
578
604
|
df: "DataFrame", input_col: Column, options: DocumentReaderOptions
|
|
579
605
|
) -> "DataFrame":
|
|
@@ -581,10 +607,14 @@ def extract_with_ai_complete(
|
|
|
581
607
|
# Unlike AI_PARSE_DOCUMENT/AI_EXTRACT, AI_COMPLETE(..., return_error_details=True)
|
|
582
608
|
# returns a native OBJECT rather than a JSON string -- CAST(... AS VARCHAR), which
|
|
583
609
|
# as_variant() always applies, fails to compile for it. Bracket access works
|
|
584
|
-
# directly on the raw result, so it skips as_variant() here.
|
|
610
|
+
# directly on the raw result, so it skips as_variant() here. With show_details the
|
|
611
|
+
# answer is inside `value`, beside its token `usage`.
|
|
585
612
|
df = df.with_column(_EXTRACTED_COLUMN, call)
|
|
586
613
|
extracted = col(_EXTRACTED_COLUMN)
|
|
587
614
|
value = extracted["value"]
|
|
615
|
+
if options.tokens_enabled:
|
|
616
|
+
df = df.with_columns(TOKEN_COLUMNS, usage_columns(value["usage"]))
|
|
617
|
+
value = details_answer(value, structured=options.extract_enabled)
|
|
588
618
|
extraction = options.extraction
|
|
589
619
|
if extraction is None:
|
|
590
620
|
return df.with_columns(
|
|
@@ -630,6 +660,7 @@ def extract_with_split_procedure(
|
|
|
630
660
|
json.dumps(_CALL_BUDGETS),
|
|
631
661
|
_MAX_CALLS_PER_FILE,
|
|
632
662
|
session.get_session_stage(),
|
|
663
|
+
options.tokens_enabled,
|
|
633
664
|
]
|
|
634
665
|
df = session.sql(
|
|
635
666
|
f"CALL {proc_name}({', '.join(['?'] * len(args))}) {usage_comment(options)}",
|
|
@@ -639,6 +670,7 @@ def extract_with_split_procedure(
|
|
|
639
670
|
try_parse_json(col("VALUE")).alias(_EXTRACTED_COLUMN),
|
|
640
671
|
col("ERROR").alias(_EXTRACT_ERROR_COLUMN),
|
|
641
672
|
try_parse_json(col("SCORES")).alias(EXTRACTION_SCORES_COLUMN),
|
|
673
|
+
*([col(name) for name in TOKEN_COLUMNS] if options.tokens_enabled else []),
|
|
642
674
|
)
|
|
643
675
|
return project_extracted_fields(
|
|
644
676
|
df, col(_EXTRACTED_COLUMN), col(_EXTRACT_ERROR_COLUMN), options.extraction
|
|
@@ -781,6 +813,7 @@ def register_extract_proc(session: "Session") -> str:
|
|
|
781
813
|
StructField("VALUE", StringType()),
|
|
782
814
|
StructField("ERROR", StringType()),
|
|
783
815
|
StructField("SCORES", StringType()),
|
|
816
|
+
*[StructField(name, LongType()) for name in TOKEN_COLUMNS],
|
|
784
817
|
]
|
|
785
818
|
),
|
|
786
819
|
# In the order of run()'s parameters after the session, and of the args
|
|
@@ -793,6 +826,7 @@ def register_extract_proc(session: "Session") -> str:
|
|
|
793
826
|
StringType(), # call_budgets: as JSON
|
|
794
827
|
IntegerType(), # max_calls_per_file
|
|
795
828
|
StringType(), # work_stage: where page ranges are staged
|
|
829
|
+
BooleanType(), # show_details: ask AI_COMPLETE for each call's token usage
|
|
796
830
|
],
|
|
797
831
|
name=name,
|
|
798
832
|
packages=["snowflake-snowpark-python", "pypdf"],
|
|
@@ -39,6 +39,16 @@ CONTENT_COLUMN = "CONTENT"
|
|
|
39
39
|
IMAGES_COLUMN = "IMAGES"
|
|
40
40
|
EXTRACTION_SCORES_COLUMN = "EXTRACTION_SCORES"
|
|
41
41
|
RESPONSE_COLUMN = "RESPONSE"
|
|
42
|
+
# AI_COMPLETE's token usage per document, named like the billing views
|
|
43
|
+
# (CORTEX_AI_FUNCTIONS_USAGE_HISTORY's input/output); each is read from the AI_COMPLETE
|
|
44
|
+
# `usage` field in TOKEN_USAGE_FIELDS.
|
|
45
|
+
INPUT_TOKENS_COLUMN = "INPUT_TOKENS"
|
|
46
|
+
OUTPUT_TOKENS_COLUMN = "OUTPUT_TOKENS"
|
|
47
|
+
TOKEN_COLUMNS = [INPUT_TOKENS_COLUMN, OUTPUT_TOKENS_COLUMN]
|
|
48
|
+
TOKEN_USAGE_FIELDS = {
|
|
49
|
+
INPUT_TOKENS_COLUMN: "prompt_tokens",
|
|
50
|
+
OUTPUT_TOKENS_COLUMN: "completion_tokens",
|
|
51
|
+
}
|
|
42
52
|
|
|
43
53
|
_PARSE_ERROR_COLUMN = "_DOC_PARSE_ERROR"
|
|
44
54
|
_EXTRACT_ERROR_COLUMN = "_DOC_EXTRACT_ERROR"
|
|
@@ -56,6 +66,7 @@ _RESERVED_OUTPUT_COLUMNS = frozenset(
|
|
|
56
66
|
IMAGES_COLUMN,
|
|
57
67
|
EXTRACTION_SCORES_COLUMN,
|
|
58
68
|
RESPONSE_COLUMN,
|
|
69
|
+
*TOKEN_COLUMNS,
|
|
59
70
|
_EXTRACTION_STATUS_FIELD,
|
|
60
71
|
}
|
|
61
72
|
)
|
|
@@ -163,6 +174,10 @@ class DocumentReaderOptions:
|
|
|
163
174
|
extraction_engine: str = "ai_complete"
|
|
164
175
|
extraction: Optional[ExtractionSpec] = None
|
|
165
176
|
prompt: Optional[str] = None
|
|
177
|
+
# The session's PYTHON_SNOWPARK_DOCUMENTS_TOKEN_USAGE switch: False reads without
|
|
178
|
+
# asking AI_COMPLETE for its usage, with no TOKEN_COLUMNS, as reads did before they
|
|
179
|
+
# were added.
|
|
180
|
+
token_usage: bool = True
|
|
166
181
|
|
|
167
182
|
@property
|
|
168
183
|
def parse_enabled(self) -> bool:
|
|
@@ -185,6 +200,17 @@ class DocumentReaderOptions:
|
|
|
185
200
|
def complete_enabled(self) -> bool:
|
|
186
201
|
return self.extraction is not None or self.prompt is not None
|
|
187
202
|
|
|
203
|
+
@property
|
|
204
|
+
def tokens_enabled(self) -> bool:
|
|
205
|
+
"""Whether the read asks AI_COMPLETE for its token usage and returns it in
|
|
206
|
+
TOKEN_COLUMNS: an AI_COMPLETE read does unless the session's switch turns it
|
|
207
|
+
off; AI_EXTRACT reports none, and a parse-only read is billed per page."""
|
|
208
|
+
return (
|
|
209
|
+
self.token_usage
|
|
210
|
+
and self.complete_enabled
|
|
211
|
+
and self.extraction_engine == "ai_complete"
|
|
212
|
+
)
|
|
213
|
+
|
|
188
214
|
@property
|
|
189
215
|
def read_type(self) -> str:
|
|
190
216
|
"""The read shape, for usage tracking (the SQL comment marker in
|
|
@@ -218,15 +244,18 @@ class DocumentReaderOptions:
|
|
|
218
244
|
|
|
219
245
|
@classmethod
|
|
220
246
|
def from_reader_options(
|
|
221
|
-
cls, cur_options: Dict[str, Any]
|
|
247
|
+
cls, cur_options: Dict[str, Any], token_usage: Any = True
|
|
222
248
|
) -> "DocumentReaderOptions":
|
|
223
249
|
"""The user options are SCHEMA and PROMPT; any other key is ignored, because
|
|
224
|
-
everything else is chosen by the reader.
|
|
250
|
+
everything else is chosen by the reader. ``token_usage`` is the session's
|
|
251
|
+
PYTHON_SNOWPARK_DOCUMENTS_TOKEN_USAGE value, an internal switch, not a user
|
|
252
|
+
option (see ``_token_usage_on``)."""
|
|
225
253
|
options = cls(
|
|
226
254
|
extraction=ExtractionSpec.from_response_format(
|
|
227
255
|
cls._validated_schema(cur_options.get("SCHEMA"))
|
|
228
256
|
),
|
|
229
257
|
prompt=cls._validated_prompt(cur_options),
|
|
258
|
+
token_usage=cls._token_usage_on(token_usage),
|
|
230
259
|
)
|
|
231
260
|
# With a schema or prompt the document goes to AI_COMPLETE as a FILE: parsing
|
|
232
261
|
# flattens layout (table columns, printed section numbers) that the page image keeps.
|
|
@@ -236,6 +265,13 @@ class DocumentReaderOptions:
|
|
|
236
265
|
options.validate()
|
|
237
266
|
return options
|
|
238
267
|
|
|
268
|
+
@staticmethod
|
|
269
|
+
def _token_usage_on(value: Any) -> bool:
|
|
270
|
+
"""Whether the session's PYTHON_SNOWPARK_DOCUMENTS_TOKEN_USAGE value leaves the
|
|
271
|
+
token columns on: anything but False, or the string "false" in any case, as a
|
|
272
|
+
session parameter given in the connection config can arrive as text."""
|
|
273
|
+
return value is not False and str(value).strip().lower() != "false"
|
|
274
|
+
|
|
239
275
|
@staticmethod
|
|
240
276
|
def _validated_schema(value: Any) -> Any:
|
|
241
277
|
"""The SCHEMA option, unchanged. A given schema that is not a dict or a list
|