langfun 0.1.2.dev202512050805__tar.gz → 0.1.2.dev202512280805__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of langfun might be problematic. Click here for more details.

Files changed (222) hide show
  1. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/PKG-INFO +1 -1
  2. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/agentic/action.py +2 -3
  3. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/__init__.py +1 -0
  4. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/checkpointing.py +73 -41
  5. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/evaluation.py +3 -2
  6. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/experiment.py +16 -4
  7. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/progress.py +16 -11
  8. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/reporting.py +3 -4
  9. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/runners/base.py +2 -0
  10. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/__init__.py +6 -0
  11. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/anthropic.py +59 -0
  12. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/gemini.py +53 -1
  13. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/google_genai.py +24 -0
  14. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/vertexai.py +33 -0
  15. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/modalities/mime.py +14 -1
  16. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/modalities/mime_test.py +48 -0
  17. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun.egg-info/PKG-INFO +1 -1
  18. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/LICENSE +0 -0
  19. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/README.md +0 -0
  20. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/__init__.py +0 -0
  21. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/assistant/capabilities/gui/__init__.py +0 -0
  22. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/assistant/capabilities/gui/bounding_box_parser.py +0 -0
  23. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/assistant/capabilities/gui/bounding_box_parser_test.py +0 -0
  24. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/assistant/capabilities/gui/drawing.py +0 -0
  25. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/assistant/capabilities/gui/drawing_test.py +0 -0
  26. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/assistant/capabilities/gui/location.py +0 -0
  27. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/assistant/capabilities/gui/location_test.py +0 -0
  28. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/__init__.py +0 -0
  29. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/agentic/__init__.py +0 -0
  30. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/agentic/action_eval.py +0 -0
  31. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/agentic/action_eval_test.py +0 -0
  32. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/agentic/action_test.py +0 -0
  33. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/async_support.py +0 -0
  34. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/async_support_test.py +0 -0
  35. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/coding/__init__.py +0 -0
  36. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/coding/python/__init__.py +0 -0
  37. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/coding/python/correction.py +0 -0
  38. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/coding/python/correction_test.py +0 -0
  39. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/coding/python/execution.py +0 -0
  40. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/coding/python/execution_test.py +0 -0
  41. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/coding/python/generation.py +0 -0
  42. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/coding/python/generation_test.py +0 -0
  43. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/coding/python/parsing.py +0 -0
  44. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/coding/python/parsing_test.py +0 -0
  45. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/coding/python/sandboxing.py +0 -0
  46. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/coding/python/sandboxing_test.py +0 -0
  47. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/component.py +0 -0
  48. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/component_test.py +0 -0
  49. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/concurrent.py +0 -0
  50. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/concurrent_test.py +0 -0
  51. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/console.py +0 -0
  52. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/console_test.py +0 -0
  53. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/data/__init__.py +0 -0
  54. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/data/conversion/__init__.py +0 -0
  55. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/data/conversion/anthropic.py +0 -0
  56. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/data/conversion/anthropic_test.py +0 -0
  57. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/data/conversion/gemini.py +0 -0
  58. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/data/conversion/gemini_test.py +0 -0
  59. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/data/conversion/openai.py +0 -0
  60. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/data/conversion/openai_test.py +0 -0
  61. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/__init__.py +0 -0
  62. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/base.py +0 -0
  63. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/base_test.py +0 -0
  64. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/matching.py +0 -0
  65. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/matching_test.py +0 -0
  66. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/patching.py +0 -0
  67. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/patching_test.py +0 -0
  68. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/scoring.py +0 -0
  69. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/scoring_test.py +0 -0
  70. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/checkpointing_test.py +0 -0
  71. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/config_saver.py +0 -0
  72. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/config_saver_test.py +0 -0
  73. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/eval_test_helper.py +0 -0
  74. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/evaluation_test.py +0 -0
  75. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/example.py +0 -0
  76. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/example_test.py +0 -0
  77. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/experiment_test.py +0 -0
  78. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/metric_values.py +0 -0
  79. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/metric_values_test.py +0 -0
  80. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/metrics.py +0 -0
  81. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/metrics_test.py +0 -0
  82. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/progress_test.py +0 -0
  83. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/progress_tracking.py +0 -0
  84. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/progress_tracking_test.py +0 -0
  85. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/reporting_test.py +0 -0
  86. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/runners/__init__.py +0 -0
  87. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/runners/beam.py +0 -0
  88. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/runners/beam_test.py +0 -0
  89. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/runners/ckpt_monitor.py +0 -0
  90. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/runners/ckpt_monitor_test.py +0 -0
  91. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/runners/debug.py +0 -0
  92. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/runners/debug_test.py +0 -0
  93. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/runners/parallel.py +0 -0
  94. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/runners/parallel_test.py +0 -0
  95. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/runners/sequential.py +0 -0
  96. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/eval/v2/runners/sequential_test.py +0 -0
  97. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/langfunc.py +0 -0
  98. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/langfunc_test.py +0 -0
  99. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/language_model.py +0 -0
  100. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/language_model_test.py +0 -0
  101. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/anthropic_test.py +0 -0
  102. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/azure_openai.py +0 -0
  103. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/azure_openai_test.py +0 -0
  104. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/cache/__init__.py +0 -0
  105. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/cache/base.py +0 -0
  106. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/cache/in_memory.py +0 -0
  107. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/cache/in_memory_test.py +0 -0
  108. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/compositional.py +0 -0
  109. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/compositional_test.py +0 -0
  110. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/deepseek.py +0 -0
  111. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/deepseek_test.py +0 -0
  112. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/fake.py +0 -0
  113. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/fake_test.py +0 -0
  114. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/gemini_test.py +0 -0
  115. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/google_genai_test.py +0 -0
  116. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/groq.py +0 -0
  117. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/groq_test.py +0 -0
  118. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/llama_cpp.py +0 -0
  119. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/llama_cpp_test.py +0 -0
  120. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/openai.py +0 -0
  121. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/openai_compatible.py +0 -0
  122. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/openai_compatible_test.py +0 -0
  123. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/openai_test.py +0 -0
  124. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/rest.py +0 -0
  125. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/rest_test.py +0 -0
  126. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/llms/vertexai_test.py +0 -0
  127. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/logging.py +0 -0
  128. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/logging_test.py +0 -0
  129. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/mcp/__init__.py +0 -0
  130. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/mcp/client.py +0 -0
  131. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/mcp/client_test.py +0 -0
  132. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/mcp/session.py +0 -0
  133. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/mcp/session_test.py +0 -0
  134. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/mcp/testing/simple_mcp_client.py +0 -0
  135. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/mcp/testing/simple_mcp_server.py +0 -0
  136. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/mcp/tool.py +0 -0
  137. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/mcp/tool_test.py +0 -0
  138. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/memories/__init__.py +0 -0
  139. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/memories/conversation_history.py +0 -0
  140. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/memories/conversation_history_test.py +0 -0
  141. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/memory.py +0 -0
  142. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/message.py +0 -0
  143. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/message_test.py +0 -0
  144. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/modalities/__init__.py +0 -0
  145. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/modalities/audio.py +0 -0
  146. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/modalities/audio_test.py +0 -0
  147. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/modalities/image.py +0 -0
  148. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/modalities/image_test.py +0 -0
  149. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/modalities/pdf.py +0 -0
  150. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/modalities/pdf_test.py +0 -0
  151. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/modalities/video.py +0 -0
  152. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/modalities/video_test.py +0 -0
  153. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/modality.py +0 -0
  154. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/modality_test.py +0 -0
  155. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/natural_language.py +0 -0
  156. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/natural_language_test.py +0 -0
  157. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/sampling.py +0 -0
  158. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/sampling_test.py +0 -0
  159. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/__init__.py +0 -0
  160. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/completion.py +0 -0
  161. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/completion_test.py +0 -0
  162. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/description.py +0 -0
  163. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/description_test.py +0 -0
  164. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/function_generation.py +0 -0
  165. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/function_generation_test.py +0 -0
  166. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/mapping.py +0 -0
  167. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/mapping_test.py +0 -0
  168. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/parsing.py +0 -0
  169. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/parsing_test.py +0 -0
  170. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/querying.py +0 -0
  171. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/querying_test.py +0 -0
  172. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/schema/__init__.py +0 -0
  173. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/schema/base.py +0 -0
  174. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/schema/base_test.py +0 -0
  175. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/schema/json.py +0 -0
  176. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/schema/json_test.py +0 -0
  177. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/schema/python.py +0 -0
  178. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/schema/python_test.py +0 -0
  179. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/schema_generation.py +0 -0
  180. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/schema_generation_test.py +0 -0
  181. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/scoring.py +0 -0
  182. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/scoring_test.py +0 -0
  183. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/tokenization.py +0 -0
  184. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/structured/tokenization_test.py +0 -0
  185. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/subscription.py +0 -0
  186. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/subscription_test.py +0 -0
  187. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/template.py +0 -0
  188. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/template_test.py +0 -0
  189. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/templates/__init__.py +0 -0
  190. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/templates/completion.py +0 -0
  191. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/templates/completion_test.py +0 -0
  192. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/templates/conversation.py +0 -0
  193. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/templates/conversation_test.py +0 -0
  194. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/templates/demonstration.py +0 -0
  195. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/templates/demonstration_test.py +0 -0
  196. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/templates/selfplay.py +0 -0
  197. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/core/templates/selfplay_test.py +0 -0
  198. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/__init__.py +0 -0
  199. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/base_environment.py +0 -0
  200. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/base_environment_test.py +0 -0
  201. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/base_feature.py +0 -0
  202. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/base_feature_test.py +0 -0
  203. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/base_sandbox.py +0 -0
  204. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/base_sandbox_test.py +0 -0
  205. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/event_handlers/__init__.py +0 -0
  206. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/event_handlers/chain.py +0 -0
  207. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/event_handlers/chain_test.py +0 -0
  208. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/event_handlers/event_logger.py +0 -0
  209. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/event_handlers/event_logger_test.py +0 -0
  210. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/event_handlers/metric_writer.py +0 -0
  211. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/event_handlers/metric_writer_test.py +0 -0
  212. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/interface.py +0 -0
  213. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/interface_test.py +0 -0
  214. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/load_balancers.py +0 -0
  215. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/load_balancers_test.py +0 -0
  216. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun/env/test_utils.py +0 -0
  217. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun.egg-info/SOURCES.txt +0 -0
  218. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun.egg-info/dependency_links.txt +0 -0
  219. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun.egg-info/requires.txt +0 -0
  220. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/langfun.egg-info/top_level.txt +0 -0
  221. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/setup.cfg +0 -0
  222. {langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512280805}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: langfun
3
- Version: 0.1.2.dev202512050805
3
+ Version: 0.1.2.dev202512280805
4
4
  Summary: Langfun: Language as Functions.
5
5
  Home-page: https://github.com/google/langfun
6
6
  Author: Langfun Authors
@@ -288,6 +288,8 @@ class Action(pg.Object):
288
288
 
289
289
  with session.track_action(self, max_execution_time=max_execution_time):
290
290
  try:
291
+ # Early terminate the action if the execution time is exceeded.
292
+ session.check_execution_time()
291
293
  result = self.call(session=session, **kwargs)
292
294
  self._invocation.end(result)
293
295
  except BaseException as e:
@@ -2005,9 +2007,6 @@ class Session(pg.Object, pg.views.html.HtmlTreeView.Extension):
2005
2007
  'signal the start and end of the session.'
2006
2008
  )
2007
2009
 
2008
- # Early terminate the action if the execution time is exceeded.
2009
- self.check_execution_time()
2010
-
2011
2010
  invocation = ActionInvocation(
2012
2011
  pg.maybe_ref(action),
2013
2012
  max_execution_time=self._child_max_execution_time(max_execution_time)
@@ -41,6 +41,7 @@ from langfun.core.eval.v2.checkpointing import PerExampleCheckpointer
41
41
  from langfun.core.eval.v2.reporting import HtmlReporter
42
42
  from langfun.core.eval.v2.reporting import ExampleHtmlGenerator
43
43
 
44
+ # Google-internal imports.
44
45
 
45
46
  # pylint: enable=g-bad-import-order
46
47
  # pylint: enable=g-importing-member
@@ -38,7 +38,7 @@ class Checkpointer(experiment_lib.Plugin):
38
38
  later. When an experiment starts, the checkpointer loads any previously saved
39
39
  examples from an earlier run (or a warm-start run) into `experiment.state`,
40
40
  so the runner can skip processing them again.
41
- Subclasses should implement `_list_checkpoint_filenames` to identify
41
+ Subclasses should implement `_list_checkpoint_files` to identify
42
42
  checkpoint files to load, and `_save_example` to save a newly processed
43
43
  example.
44
44
  """
@@ -122,7 +122,7 @@ class Checkpointer(experiment_lib.Plugin):
122
122
  example: Example,
123
123
  ) -> None:
124
124
  """Saves the example to the checkpoint file."""
125
- if example.newly_processed:
125
+ if example.newly_processed or runner.current_run.force_recompute_metrics:
126
126
  self._save_example(runner, experiment, example)
127
127
 
128
128
  def _load_experiment(
@@ -131,7 +131,7 @@ class Checkpointer(experiment_lib.Plugin):
131
131
  experiment: Experiment,
132
132
  ) -> None:
133
133
  """Creates the checkpoint file."""
134
- ckpt_files = self._list_checkpoint_filenames(runner, experiment)
134
+ ckpt_files = self._list_checkpoint_files(runner, experiment)
135
135
  experiment.info(f'Found {len(ckpt_files)} checkpoint files to load.')
136
136
 
137
137
  # Load the checkpoint files in parallel.
@@ -141,18 +141,18 @@ class Checkpointer(experiment_lib.Plugin):
141
141
  experiment
142
142
  )
143
143
  context = dict(counter=0, counter_lock=threading.Lock())
144
- copy_ckpt = current_run.input_root != current_run.output_root
145
144
 
146
145
  def _load_state(ckpt_file):
147
146
  error = None
148
147
  with pg.timeit() as t:
149
148
  try:
150
- experiment.load_state(
151
- current_run.input_path_for(experiment, ckpt_file),
149
+ loaded_examples = experiment.load_state(
150
+ ckpt_file,
152
151
  filter=lambda x: x.id in examples_to_load,
153
152
  load_example_metadata=lambda x: x.id in examples_to_load_metadata,
154
153
  )
155
154
  except BaseException as e: # pylint: disable=broad-except
155
+ loaded_examples = []
156
156
  error = e
157
157
  finally:
158
158
  with context['counter_lock']:
@@ -170,22 +170,18 @@ class Checkpointer(experiment_lib.Plugin):
170
170
  f'Skipping the file. ({progress_str})'
171
171
  )
172
172
 
173
- if not copy_ckpt:
174
- return
175
-
176
- # Copy the checkpoint records to the output directory.
177
- try:
178
- with pg.io.open_sequence(
179
- current_run.output_path_for(experiment, ckpt_file), 'w'
180
- ) as o, pg.io.open_sequence(
181
- current_run.input_path_for(experiment, ckpt_file), 'r'
182
- ) as i:
183
- for x in i:
184
- o.add(x)
185
- except BaseException as e: # pylint: disable=broad-except
186
- experiment.warning(
187
- f'Failed to copy checkpoint {ckpt_file!r}: {e}.'
188
- )
173
+ output_ckpt_file = current_run.output_path_for(
174
+ experiment, os.path.basename(ckpt_file)
175
+ )
176
+ if (not runner.current_run.force_recompute_metrics
177
+ and ckpt_file != output_ckpt_file
178
+ and any(e for e in loaded_examples if not e.has_error)):
179
+ # Write the error-free warm-start examples to the output checkpoint
180
+ # file.
181
+ with SequenceWriter(output_ckpt_file) as writer:
182
+ for example in loaded_examples:
183
+ if not example.has_error:
184
+ writer.add(example)
189
185
 
190
186
  _ = list(
191
187
  lf.concurrent_map(
@@ -197,10 +193,10 @@ class Checkpointer(experiment_lib.Plugin):
197
193
  )
198
194
 
199
195
  @abc.abstractmethod
200
- def _list_checkpoint_filenames(
196
+ def _list_checkpoint_files(
201
197
  self, runner: Runner, experiment: Experiment
202
198
  ) -> list[str]:
203
- """Lists the checkpoint filenames to restore."""
199
+ """Lists the checkpoint file paths to restore."""
204
200
 
205
201
  @abc.abstractmethod
206
202
  def _save_example(
@@ -226,22 +222,41 @@ class PerExampleCheckpointer(Checkpointer):
226
222
  self._checkpoint_file_prefix = prefix
227
223
  self._checkpoint_file_ext = ext
228
224
 
229
- def _list_checkpoint_filenames(
225
+ def _list_checkpoint_files(
230
226
  self, runner: Runner, experiment: Experiment
231
227
  ) -> list[str]:
232
- experiment_dir = runner.current_run.input_dir(experiment)
233
- filenames = []
228
+
229
+ def _list_checkpoints_from(ckpt_dir: str, examples_to_load: set[int]):
230
+ ckpt_files = []
231
+ if pg.io.path_exists(ckpt_dir):
232
+ regex = re.compile(
233
+ f'{self._checkpoint_file_prefix}_(\\d+){self._checkpoint_file_ext}'
234
+ .replace('.', '\\.')
235
+ )
236
+ for filename in pg.io.listdir(ckpt_dir):
237
+ match = regex.match(filename)
238
+ if match and int(match.group(1)) in examples_to_load:
239
+ examples_to_load.remove(int(match.group(1)))
240
+ ckpt_files.append(os.path.join(ckpt_dir, filename))
241
+ return ckpt_files
242
+
234
243
  examples_to_load = runner.current_run.examples_to_load(experiment)
235
- if pg.io.path_exists(experiment_dir):
236
- regex = re.compile(
237
- f'{self._checkpoint_file_prefix}_(\\d+){self._checkpoint_file_ext}'
238
- .replace('.', '\\.')
244
+
245
+ # Take output directory as the first priority to checkpoints processed in
246
+ # this run.
247
+ ckpt_files = _list_checkpoints_from(
248
+ runner.current_run.output_dir(experiment), examples_to_load
249
+ )
250
+ # If the input and output directories are different, also load from the
251
+ # input directory.
252
+ if (examples_to_load
253
+ and runner.current_run.input_root != runner.current_run.output_root):
254
+ ckpt_files.extend(
255
+ _list_checkpoints_from(
256
+ runner.current_run.input_dir(experiment), examples_to_load
257
+ )
239
258
  )
240
- for filename in pg.io.listdir(experiment_dir):
241
- match = regex.match(filename)
242
- if match and int(match.group(1)) in examples_to_load:
243
- filenames.append(filename)
244
- return filenames
259
+ return ckpt_files
245
260
 
246
261
  def _save_example(
247
262
  self,
@@ -341,13 +356,23 @@ class BulkCheckpointer(Checkpointer):
341
356
  if self._sequence_writer is not None:
342
357
  self._sequence_writer[experiment.id] = sequence_writer
343
358
 
344
- def _list_checkpoint_filenames(
359
+ def _list_checkpoint_files(
345
360
  self, runner: Runner, experiment: Experiment
346
361
  ) -> list[str]:
347
- if pg.io.path_exists(
348
- runner.current_run.input_path_for(experiment, self.checkpoint_filename)
349
- ):
350
- return [self.checkpoint_filename]
362
+ # Always honor the output directory if it's present, as it contains both
363
+ # the warm-started examples and newly processed examples.
364
+ output_ckpt_file = runner.current_run.output_path_for(
365
+ experiment, self.checkpoint_filename
366
+ )
367
+ if pg.io.path_exists(output_ckpt_file):
368
+ return [output_ckpt_file]
369
+
370
+ if runner.current_run.input_root != runner.current_run.output_root:
371
+ input_ckpt_file = runner.current_run.input_path_for(
372
+ experiment, self.checkpoint_filename
373
+ )
374
+ if pg.io.path_exists(input_ckpt_file):
375
+ return [input_ckpt_file]
351
376
  return []
352
377
 
353
378
  def on_experiment_complete(
@@ -441,5 +466,12 @@ class SequenceWriter:
441
466
  self._sequence_writer = None
442
467
  pg.io.rename(self._tmp_path, self._path)
443
468
 
469
+ def __enter__(self):
470
+ return self
471
+
472
+ def __exit__(self, *args, **kwargs):
473
+ del args, kwargs
474
+ self.close()
475
+
444
476
  def __del__(self):
445
477
  self.close()
@@ -386,10 +386,10 @@ class Evaluation(experiment_lib.Experiment):
386
386
  load_example_metadata: bool = True,
387
387
  filter: Callable[[example_lib.Example], bool] | None = None, # pylint: disable=redefined-builtin
388
388
  raise_if_not_exist: bool = False
389
- ) -> None:
389
+ ) -> list[example_lib.Example]:
390
390
  """Loads saved state from a sequence IO file."""
391
391
  if pg.io.path_exists(state_file):
392
- self._state.load(
392
+ return self._state.load(
393
393
  state_file,
394
394
  example_input_by_id=self.example_input_by_id,
395
395
  load_example_metadata=load_example_metadata,
@@ -397,6 +397,7 @@ class Evaluation(experiment_lib.Experiment):
397
397
  )
398
398
  elif raise_if_not_exist:
399
399
  raise ValueError(f'State file {state_file} does not exist.')
400
+ return []
400
401
 
401
402
  def _reset(self) -> None:
402
403
  """Resets the state of the evaluation."""
@@ -376,23 +376,24 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
376
376
  def run(
377
377
  self,
378
378
  root_dir: str,
379
- id: str | None = None, # pylint: disable=redefined-builtin
379
+ id: str | None = None, # pylint: disable=redefined-builtin
380
380
  *,
381
381
  runner: str = 'parallel',
382
382
  warm_start_from: str | None = None,
383
- filter: Callable[['Experiment'], bool] | None = None, # pylint: disable=redefined-builtin
383
+ filter: Callable[['Experiment'], bool] | None = None, # pylint: disable=redefined-builtin
384
384
  example_ids: list[int] | None = None,
385
385
  shuffle_inputs: bool = False,
386
386
  raise_if_has_error: bool = False,
387
387
  reevaluate_upon_previous_errors: bool = True,
388
388
  reprocess: bool | list[int] = False,
389
+ force_recompute_metrics: bool = False,
389
390
  generate_example_html: Literal['new', 'all', 'no'] | list[int] = 'new',
390
391
  process_timeout: int | None = None,
391
392
  use_cache: Literal['global', 'per_dataset', 'no'] = 'per_dataset',
392
393
  note: str | None = None,
393
394
  tags: list[str] | None = None,
394
395
  plugins: list['Plugin'] | None = None,
395
- **kwargs
396
+ **kwargs,
396
397
  ) -> 'Run':
397
398
  """Runs the experiment.
398
399
 
@@ -445,6 +446,8 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
445
446
  meaning that existing checkpoints will be ignored. If a list of
446
447
  example IDs, it indicates that only the specified examples will be
447
448
  reprocessed.
449
+ force_recompute_metrics: If True, it will recompute the metrics for all
450
+ examples, even if the previous checkpoints have metric metadata.
448
451
  generate_example_html: Among 'new', 'all', 'no' or a list of example IDs.
449
452
  If 'new', generate HTML files for all newly processed examples, and
450
453
  keep/copy existing HTML files for unchanged examples.
@@ -481,6 +484,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
481
484
  raise_if_has_error=raise_if_has_error,
482
485
  reevaluate_upon_previous_errors=reevaluate_upon_previous_errors,
483
486
  reprocess=reprocess,
487
+ force_recompute_metrics=force_recompute_metrics,
484
488
  generate_example_html=generate_example_html,
485
489
  use_cache=use_cache,
486
490
  process_timeout=process_timeout,
@@ -884,6 +888,14 @@ class Run(pg.Object, pg.views.html.HtmlTreeView.Extension):
884
888
  )
885
889
  ] = True
886
890
 
891
+ force_recompute_metrics: Annotated[
892
+ bool,
893
+ (
894
+ 'If True, force recompute the metrics even if metric metadata is '
895
+ 'already present from previous checkpoint.'
896
+ )
897
+ ] = False
898
+
887
899
  note: Annotated[
888
900
  str | None,
889
901
  'The user note for the current run.'
@@ -1003,7 +1015,7 @@ class Run(pg.Object, pg.views.html.HtmlTreeView.Extension):
1003
1015
  load_metadata_ids = set()
1004
1016
  if isinstance(self.generate_example_html, list):
1005
1017
  load_metadata_ids = set(self.generate_example_html)
1006
- elif self.generate_example_html == 'all':
1018
+ elif self.generate_example_html == 'all' or self.force_recompute_metrics:
1007
1019
  load_metadata_ids = self.examples_to_evaluate(experiment)
1008
1020
  load_metadata_ids -= self.examples_to_reprocess(experiment)
1009
1021
  return load_metadata_ids
@@ -227,7 +227,11 @@ class Progress(pg.Object, pg.views.HtmlTreeView.Extension):
227
227
 
228
228
  def merge_from(self, other: 'Progress') -> None:
229
229
  """Merges the progress from another progress."""
230
- with pg.notify_on_change(False), pg.allow_writable_accessors(True):
230
+ with (
231
+ self._lock,
232
+ pg.notify_on_change(False),
233
+ pg.allow_writable_accessors(True)
234
+ ):
231
235
  if other.start_time is not None and (
232
236
  self.start_time is None or self.start_time > other.start_time):
233
237
  self.start_time = other.start_time
@@ -268,16 +272,17 @@ class Progress(pg.Object, pg.views.HtmlTreeView.Extension):
268
272
  stop_time=self.stop_time_str,
269
273
  )
270
274
  if self.execution_summary:
271
- time_info['execution'] = pg.Dict(
272
- {
273
- k: pg.Dict(
274
- num_started=v.num_started,
275
- num_ended=v.num_ended,
276
- num_failed=v.num_failed,
277
- avg_duration=round(v.avg_duration, 2),
278
- ) for k, v in self.execution_summary.breakdown.items()
279
- }
280
- )
275
+ with self._lock:
276
+ time_info['execution'] = pg.Dict(
277
+ {
278
+ k: pg.Dict(
279
+ num_started=v.num_started,
280
+ num_ended=v.num_ended,
281
+ num_failed=v.num_failed,
282
+ avg_duration=round(v.avg_duration, 2),
283
+ ) for k, v in self.execution_summary.breakdown.items()
284
+ }
285
+ )
281
286
  return pg.format(time_info, verbose=False)
282
287
 
283
288
  def _html_tree_view(
@@ -86,10 +86,8 @@ class ExampleHtmlGenerator(experiment_lib.Plugin):
86
86
  return
87
87
 
88
88
  try:
89
- with pg.timeit() as t, pg.io.open(src_file, 'r') as src:
90
- content = src.read()
91
- with pg.io.open(dest_file, 'w') as dest:
92
- dest.write(content)
89
+ with pg.timeit() as t:
90
+ pg.io.copy(src_file, dest_file)
93
91
  experiment.info(
94
92
  f'\'{example.id}.html\' copied in {t.elapse:.2f} seconds.'
95
93
  )
@@ -101,6 +99,7 @@ class ExampleHtmlGenerator(experiment_lib.Plugin):
101
99
 
102
100
  generate_example_html = current_run.generate_example_html
103
101
  if (generate_example_html == 'all'
102
+ or runner.current_run.force_recompute_metrics
104
103
  or (generate_example_html == 'new' and example.newly_processed)
105
104
  or (isinstance(generate_example_html, list)
106
105
  and example.id in generate_example_html)):
@@ -139,6 +139,7 @@ class RunnerBase(Runner):
139
139
  self.current_run.examples_to_evaluate(experiment)
140
140
  )
141
141
  experiment.progress.start(total=num_examples_to_evaluate)
142
+ pg.io.mkdirs(self.current_run.output_dir(experiment))
142
143
  else:
143
144
  experiment.progress.start(total=len(experiment.leaf_nodes))
144
145
 
@@ -395,6 +396,7 @@ class RunnerBase(Runner):
395
396
  item,
396
397
  raise_if_has_error=self.current_run.raise_if_has_error,
397
398
  reevaluate_upon_previous_errors=self.current_run.reevaluate_upon_previous_errors,
399
+ force_recompute_metrics=self.current_run.force_recompute_metrics,
398
400
  )
399
401
  self.on_example_complete(evaluation, item)
400
402
  return item
@@ -43,6 +43,7 @@ from langfun.core.llms.azure_openai import AzureOpenAI
43
43
  # Gemini models.
44
44
  from langfun.core.llms.google_genai import GenAI
45
45
  from langfun.core.llms.google_genai import Gemini3ProPreview
46
+ from langfun.core.llms.google_genai import Gemini3FlashPreview
46
47
  from langfun.core.llms.google_genai import Gemini25Pro
47
48
  from langfun.core.llms.google_genai import Gemini25Flash
48
49
  from langfun.core.llms.google_genai import Gemini25ProPreview_20250605
@@ -65,6 +66,7 @@ from langfun.core.llms.google_genai import Gemini2ProExp_20250205
65
66
  from langfun.core.llms.google_genai import Gemini2FlashThinkingExp_20250121
66
67
  from langfun.core.llms.google_genai import GeminiExp_20241206
67
68
  from langfun.core.llms.google_genai import Gemini25FlashImagePreview
69
+ from langfun.core.llms.google_genai import Gemini3ProImagePreview
68
70
 
69
71
  from langfun.core.llms.vertexai import VertexAIGemini
70
72
  from langfun.core.llms.vertexai import VertexAIGemini2Flash
@@ -92,6 +94,8 @@ from langfun.core.llms.vertexai import VertexAIGemini25Pro
92
94
  from langfun.core.llms.vertexai import VertexAIGemini25Flash
93
95
  from langfun.core.llms.vertexai import VertexAIGemini25FlashImagePreview
94
96
  from langfun.core.llms.vertexai import VertexAIGemini3ProPreview
97
+ from langfun.core.llms.vertexai import VertexAIGemini3ProImagePreview
98
+ from langfun.core.llms.vertexai import VertexAIGemini3FlashPreview
95
99
 
96
100
  # For backward compatibility.
97
101
  GeminiPro1_5 = Gemini15Pro
@@ -158,6 +162,7 @@ from langfun.core.llms.openai import Gpt35
158
162
  from langfun.core.llms.anthropic import Claude45
159
163
  from langfun.core.llms.anthropic import Claude45Haiku_20251001
160
164
  from langfun.core.llms.anthropic import Claude45Sonnet_20250929
165
+ from langfun.core.llms.anthropic import Claude45Opus_20251101
161
166
  from langfun.core.llms.anthropic import Claude4
162
167
  from langfun.core.llms.anthropic import Claude4Sonnet_20250514
163
168
  from langfun.core.llms.anthropic import Claude4Opus_20250514
@@ -177,6 +182,7 @@ from langfun.core.llms.anthropic import Claude3Haiku_20240307
177
182
  from langfun.core.llms.vertexai import VertexAIAnthropic
178
183
  from langfun.core.llms.vertexai import VertexAIClaude45Haiku_20251001
179
184
  from langfun.core.llms.vertexai import VertexAIClaude45Sonnet_20250929
185
+ from langfun.core.llms.vertexai import VertexAIClaude45Opus_20251101
180
186
  from langfun.core.llms.vertexai import VertexAIClaude4Opus_20250514
181
187
  from langfun.core.llms.vertexai import VertexAIClaude4Sonnet_20250514
182
188
  from langfun.core.llms.vertexai import VertexAIClaude37Sonnet_20250219
@@ -113,6 +113,32 @@ SUPPORTED_MODELS = [
113
113
  max_output_tokens_per_minute=400_000,
114
114
  ),
115
115
  ),
116
+ AnthropicModelInfo(
117
+ model_id='claude-opus-4-5-20251101',
118
+ provider='Anthropic',
119
+ in_service=True,
120
+ description='Claude 4.5 Opus model (11/01/2025).',
121
+ release_date=datetime.datetime(2025, 11, 1),
122
+ input_modalities=(
123
+ AnthropicModelInfo.INPUT_IMAGE_TYPES
124
+ + AnthropicModelInfo.INPUT_DOC_TYPES
125
+ ),
126
+ context_length=lf.ModelInfo.ContextLength(
127
+ max_input_tokens=200_000,
128
+ max_output_tokens=64_000,
129
+ ),
130
+ pricing=lf.ModelInfo.Pricing(
131
+ cost_per_1m_cached_input_tokens=0.5,
132
+ cost_per_1m_input_tokens=5,
133
+ cost_per_1m_output_tokens=25,
134
+ ),
135
+ rate_limits=AnthropicModelInfo.RateLimits(
136
+ # Tier 4 rate limits
137
+ max_requests_per_minute=2000,
138
+ max_input_tokens_per_minute=1_000_000,
139
+ max_output_tokens_per_minute=400_000,
140
+ ),
141
+ ),
116
142
  AnthropicModelInfo(
117
143
  model_id='claude-4-opus-20250514',
118
144
  provider='Anthropic',
@@ -300,6 +326,32 @@ SUPPORTED_MODELS = [
300
326
  max_output_tokens_per_minute=0,
301
327
  ),
302
328
  ),
329
+ AnthropicModelInfo(
330
+ model_id='claude-opus-4-5@20251101',
331
+ alias_for='claude-opus-4-5-20251101',
332
+ provider='VertexAI',
333
+ in_service=True,
334
+ description='Claude 4.5 Opus model served on VertexAI (11/01/2025).',
335
+ release_date=datetime.datetime(2025, 11, 1),
336
+ input_modalities=(
337
+ AnthropicModelInfo.INPUT_IMAGE_TYPES
338
+ + AnthropicModelInfo.INPUT_DOC_TYPES
339
+ ),
340
+ context_length=lf.ModelInfo.ContextLength(
341
+ max_input_tokens=200_000,
342
+ max_output_tokens=64_000,
343
+ ),
344
+ pricing=lf.ModelInfo.Pricing(
345
+ cost_per_1m_cached_input_tokens=0.5,
346
+ cost_per_1m_input_tokens=5,
347
+ cost_per_1m_output_tokens=25,
348
+ ),
349
+ rate_limits=AnthropicModelInfo.RateLimits(
350
+ max_requests_per_minute=100,
351
+ max_input_tokens_per_minute=1_000_000,
352
+ max_output_tokens_per_minute=80_000,
353
+ ),
354
+ ),
303
355
  AnthropicModelInfo(
304
356
  model_id='claude-opus-4@20250514',
305
357
  alias_for='claude-opus-4-20250514',
@@ -834,6 +886,13 @@ class Claude45Sonnet_20250929(Claude45):
834
886
  model = 'claude-sonnet-4-5-20250929'
835
887
 
836
888
 
889
+ # pylint: disable=invalid-name
890
+ class Claude45Opus_20251101(Claude45):
891
+ """Claude 4.5 Opus model 20251101."""
892
+
893
+ model = 'claude-opus-4-5-20251101'
894
+
895
+
837
896
  class Claude4(Anthropic):
838
897
  """Base class for Claude 4 models."""
839
898
 
@@ -177,6 +177,55 @@ SUPPORTED_MODELS = [
177
177
  max_tokens_per_minute=4_000_000,
178
178
  ),
179
179
  ),
180
+ # Gemini 3 Pro Image Preview
181
+ GeminiModelInfo(
182
+ model_id='gemini-3-pro-image-preview',
183
+ in_service=True,
184
+ experimental=True,
185
+ provider=pg.oneof(['Google GenAI', 'VertexAI']),
186
+ model_type='instruction-tuned',
187
+ description=(
188
+ 'Gemini 3 Pro Image Preview for high-fidelity image generation,'
189
+ ' editing, and visual reasoning.'
190
+ ),
191
+ release_date=datetime.datetime(2025, 12, 9),
192
+ input_modalities=GeminiModelInfo.INPUT_IMAGE_TYPES
193
+ + GeminiModelInfo.INPUT_DOC_TYPES,
194
+ context_length=lf.ModelInfo.ContextLength(
195
+ max_input_tokens=65_536,
196
+ max_output_tokens=32_768,
197
+ ),
198
+ rate_limits=lf.ModelInfo.RateLimits(
199
+ max_requests_per_minute=200,
200
+ max_tokens_per_minute=1_000_000,
201
+ ),
202
+ ),
203
+ # Gemini 3 Flash Preview
204
+ GeminiModelInfo(
205
+ model_id='gemini-3-flash-preview',
206
+ in_service=True,
207
+ provider=pg.oneof(['Google GenAI', 'VertexAI']),
208
+ model_type='instruction-tuned',
209
+ description=(
210
+ 'Gemini 3 Flash Preview: High-efficiency, low-latency multimodal'
211
+ ' model optimized for agentic workflows.'
212
+ ),
213
+ release_date=datetime.datetime(2025, 12, 17),
214
+ input_modalities=GeminiModelInfo.ALL_SUPPORTED_INPUT_TYPES,
215
+ context_length=lf.ModelInfo.ContextLength(
216
+ max_input_tokens=1_048_576,
217
+ max_output_tokens=65_536,
218
+ ),
219
+ pricing=GeminiModelInfo.Pricing(
220
+ cost_per_1m_cached_input_tokens=0.05,
221
+ cost_per_1m_input_tokens=0.50,
222
+ cost_per_1m_output_tokens=3.00,
223
+ ),
224
+ rate_limits=lf.ModelInfo.RateLimits(
225
+ max_requests_per_minute=2_000,
226
+ max_tokens_per_minute=4_000_000,
227
+ ),
228
+ ),
180
229
  # Gemini 2.5 Flash
181
230
  GeminiModelInfo(
182
231
  model_id='gemini-2.5-flash',
@@ -834,7 +883,10 @@ class Gemini(rest.REST):
834
883
  config['thinkingConfig'] = thinking_config_data
835
884
 
836
885
  # This is the new feature since Gemini 3.
837
- if self.model_id.startswith('gemini-3'):
886
+ # Skip for image generation models as they don't support mediaResolution.
887
+ if self.model_id.startswith('gemini-3') and not (
888
+ self.response_modalities and 'IMAGE' in self.response_modalities
889
+ ):
838
890
  config['mediaResolution'] = 'MEDIA_RESOLUTION_HIGH'
839
891
 
840
892
  if self.response_modalities:
@@ -125,6 +125,30 @@ class Gemini3ProPreview(GenAI):
125
125
  model = 'gemini-3-pro-preview'
126
126
 
127
127
 
128
+ class Gemini3ProImagePreview(GenAI):
129
+ """Gemini 3 Pro Image Preview model for high-fidelity image generation.
130
+
131
+ This model supports:
132
+ - Text-to-image generation
133
+ - Image editing (multimodal input)
134
+ - Visual reasoning
135
+
136
+ Key Requirements:
137
+ - responseModalities must include 'IMAGE'
138
+ - Supported aspect ratios: 1:1, 16:9, 9:16, 4:3, 3:4
139
+ - Image sizes: 1K (default), 2K, 4K
140
+ """
141
+
142
+ model = 'gemini-3-pro-image-preview'
143
+ response_modalities = ['TEXT', 'IMAGE']
144
+
145
+
146
+ class Gemini3FlashPreview(GenAI):
147
+ """Gemini 3 Flash Preview model."""
148
+
149
+ model = 'gemini-3-flash-preview'
150
+
151
+
128
152
  class Gemini25FlashImagePreview(GenAI):
129
153
  """Gemini 2.5 Flash Image Preview model."""
130
154
  model = 'gemini-2.5-flash-image-preview'
@@ -220,6 +220,33 @@ class VertexAIGemini3ProPreview(VertexAIGemini): # pylint: disable=invalid-name
220
220
  location = 'global'
221
221
 
222
222
 
223
+ class VertexAIGemini3ProImagePreview(VertexAIGemini): # pylint: disable=invalid-name
224
+ """Gemini 3 Pro Image Preview model for high-fidelity image generation.
225
+
226
+ This model supports:
227
+ - Text-to-image generation
228
+ - Image editing (multimodal input)
229
+ - Visual reasoning
230
+
231
+ Key Requirements:
232
+ - Uses v1beta1 API endpoint
233
+ - responseModalities must include 'IMAGE'
234
+ - Supported aspect ratios: 1:1, 16:9, 9:16, 4:3, 3:4
235
+ - Image sizes: 1K (default), 2K, 4K
236
+ """
237
+
238
+ model = 'gemini-3-pro-image-preview'
239
+ location = 'global'
240
+ response_modalities = ['TEXT', 'IMAGE']
241
+
242
+
243
+ class VertexAIGemini3FlashPreview(VertexAIGemini): # pylint: disable=invalid-name
244
+ """Gemini 3 Flash Preview model launched on 12/17/2025."""
245
+
246
+ model = 'gemini-3-flash-preview'
247
+ location = 'global'
248
+
249
+
223
250
  class VertexAIGemini25Pro(VertexAIGemini): # pylint: disable=invalid-name
224
251
  """Gemini 2.5 Pro GA model launched on 06/17/2025."""
225
252
 
@@ -419,6 +446,12 @@ class VertexAIClaude45Sonnet_20250929(VertexAIAnthropic):
419
446
  model = 'claude-sonnet-4-5@20250929'
420
447
 
421
448
 
449
+ class VertexAIClaude45Opus_20251101(VertexAIAnthropic):
450
+ """Anthropic's Claude 4.5 Opus model on VertexAI."""
451
+
452
+ model = 'claude-opus-4-5@20251101'
453
+
454
+
422
455
  class VertexAIClaude4Opus_20250514(VertexAIAnthropic):
423
456
  """Anthropic's Claude 4 Opus model on VertexAI."""
424
457
  model = 'claude-opus-4@20250514'