langfun 0.1.2.dev202509120804__py3-none-any.whl → 0.1.2.dev202512150805__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. langfun/__init__.py +1 -1
  2. langfun/core/__init__.py +7 -1
  3. langfun/core/agentic/__init__.py +8 -1
  4. langfun/core/agentic/action.py +740 -112
  5. langfun/core/agentic/action_eval.py +9 -2
  6. langfun/core/agentic/action_test.py +189 -24
  7. langfun/core/async_support.py +104 -5
  8. langfun/core/async_support_test.py +23 -0
  9. langfun/core/coding/python/correction.py +19 -9
  10. langfun/core/coding/python/execution.py +14 -12
  11. langfun/core/coding/python/generation.py +21 -16
  12. langfun/core/coding/python/sandboxing.py +23 -3
  13. langfun/core/component.py +42 -3
  14. langfun/core/concurrent.py +70 -6
  15. langfun/core/concurrent_test.py +9 -2
  16. langfun/core/console.py +1 -1
  17. langfun/core/data/conversion/anthropic.py +12 -3
  18. langfun/core/data/conversion/anthropic_test.py +8 -6
  19. langfun/core/data/conversion/gemini.py +11 -2
  20. langfun/core/data/conversion/gemini_test.py +48 -9
  21. langfun/core/data/conversion/openai.py +145 -31
  22. langfun/core/data/conversion/openai_test.py +161 -17
  23. langfun/core/eval/base.py +48 -44
  24. langfun/core/eval/base_test.py +5 -5
  25. langfun/core/eval/matching.py +5 -2
  26. langfun/core/eval/patching.py +3 -3
  27. langfun/core/eval/scoring.py +4 -3
  28. langfun/core/eval/v2/__init__.py +3 -0
  29. langfun/core/eval/v2/checkpointing.py +148 -46
  30. langfun/core/eval/v2/checkpointing_test.py +9 -2
  31. langfun/core/eval/v2/config_saver.py +37 -0
  32. langfun/core/eval/v2/config_saver_test.py +36 -0
  33. langfun/core/eval/v2/eval_test_helper.py +104 -3
  34. langfun/core/eval/v2/evaluation.py +102 -19
  35. langfun/core/eval/v2/evaluation_test.py +9 -3
  36. langfun/core/eval/v2/example.py +50 -40
  37. langfun/core/eval/v2/example_test.py +16 -8
  38. langfun/core/eval/v2/experiment.py +95 -20
  39. langfun/core/eval/v2/experiment_test.py +19 -0
  40. langfun/core/eval/v2/metric_values.py +31 -3
  41. langfun/core/eval/v2/metric_values_test.py +32 -0
  42. langfun/core/eval/v2/metrics.py +157 -44
  43. langfun/core/eval/v2/metrics_test.py +39 -18
  44. langfun/core/eval/v2/progress.py +31 -1
  45. langfun/core/eval/v2/progress_test.py +27 -0
  46. langfun/core/eval/v2/progress_tracking.py +13 -5
  47. langfun/core/eval/v2/progress_tracking_test.py +9 -1
  48. langfun/core/eval/v2/reporting.py +88 -71
  49. langfun/core/eval/v2/reporting_test.py +24 -6
  50. langfun/core/eval/v2/runners/__init__.py +30 -0
  51. langfun/core/eval/v2/{runners.py → runners/base.py} +73 -180
  52. langfun/core/eval/v2/runners/beam.py +354 -0
  53. langfun/core/eval/v2/runners/beam_test.py +153 -0
  54. langfun/core/eval/v2/runners/ckpt_monitor.py +350 -0
  55. langfun/core/eval/v2/runners/ckpt_monitor_test.py +213 -0
  56. langfun/core/eval/v2/runners/debug.py +40 -0
  57. langfun/core/eval/v2/runners/debug_test.py +76 -0
  58. langfun/core/eval/v2/runners/parallel.py +243 -0
  59. langfun/core/eval/v2/runners/parallel_test.py +182 -0
  60. langfun/core/eval/v2/runners/sequential.py +47 -0
  61. langfun/core/eval/v2/runners/sequential_test.py +169 -0
  62. langfun/core/langfunc.py +45 -130
  63. langfun/core/langfunc_test.py +7 -5
  64. langfun/core/language_model.py +189 -36
  65. langfun/core/language_model_test.py +54 -3
  66. langfun/core/llms/__init__.py +14 -1
  67. langfun/core/llms/anthropic.py +157 -2
  68. langfun/core/llms/azure_openai.py +29 -17
  69. langfun/core/llms/cache/base.py +25 -3
  70. langfun/core/llms/cache/in_memory.py +48 -7
  71. langfun/core/llms/cache/in_memory_test.py +14 -4
  72. langfun/core/llms/compositional.py +25 -1
  73. langfun/core/llms/deepseek.py +30 -2
  74. langfun/core/llms/fake.py +32 -1
  75. langfun/core/llms/gemini.py +90 -12
  76. langfun/core/llms/gemini_test.py +110 -0
  77. langfun/core/llms/google_genai.py +52 -1
  78. langfun/core/llms/groq.py +28 -3
  79. langfun/core/llms/llama_cpp.py +23 -4
  80. langfun/core/llms/openai.py +120 -3
  81. langfun/core/llms/openai_compatible.py +148 -27
  82. langfun/core/llms/openai_compatible_test.py +207 -20
  83. langfun/core/llms/openai_test.py +0 -2
  84. langfun/core/llms/rest.py +16 -1
  85. langfun/core/llms/vertexai.py +78 -8
  86. langfun/core/logging.py +1 -1
  87. langfun/core/mcp/__init__.py +10 -0
  88. langfun/core/mcp/client.py +177 -0
  89. langfun/core/mcp/client_test.py +71 -0
  90. langfun/core/mcp/session.py +241 -0
  91. langfun/core/mcp/session_test.py +54 -0
  92. langfun/core/mcp/testing/simple_mcp_client.py +33 -0
  93. langfun/core/mcp/testing/simple_mcp_server.py +33 -0
  94. langfun/core/mcp/tool.py +254 -0
  95. langfun/core/mcp/tool_test.py +197 -0
  96. langfun/core/memory.py +1 -0
  97. langfun/core/message.py +160 -55
  98. langfun/core/message_test.py +65 -81
  99. langfun/core/modalities/__init__.py +8 -0
  100. langfun/core/modalities/audio.py +21 -1
  101. langfun/core/modalities/image.py +73 -3
  102. langfun/core/modalities/image_test.py +116 -0
  103. langfun/core/modalities/mime.py +78 -4
  104. langfun/core/modalities/mime_test.py +59 -0
  105. langfun/core/modalities/pdf.py +19 -1
  106. langfun/core/modalities/video.py +21 -1
  107. langfun/core/modality.py +167 -29
  108. langfun/core/modality_test.py +42 -12
  109. langfun/core/natural_language.py +1 -1
  110. langfun/core/sampling.py +4 -4
  111. langfun/core/sampling_test.py +20 -4
  112. langfun/core/structured/__init__.py +2 -24
  113. langfun/core/structured/completion.py +34 -44
  114. langfun/core/structured/completion_test.py +23 -43
  115. langfun/core/structured/description.py +54 -50
  116. langfun/core/structured/function_generation.py +29 -12
  117. langfun/core/structured/mapping.py +81 -37
  118. langfun/core/structured/parsing.py +95 -79
  119. langfun/core/structured/parsing_test.py +0 -3
  120. langfun/core/structured/querying.py +230 -154
  121. langfun/core/structured/querying_test.py +69 -33
  122. langfun/core/structured/schema/__init__.py +49 -0
  123. langfun/core/structured/schema/base.py +664 -0
  124. langfun/core/structured/schema/base_test.py +531 -0
  125. langfun/core/structured/schema/json.py +174 -0
  126. langfun/core/structured/schema/json_test.py +121 -0
  127. langfun/core/structured/schema/python.py +316 -0
  128. langfun/core/structured/schema/python_test.py +410 -0
  129. langfun/core/structured/schema_generation.py +33 -14
  130. langfun/core/structured/scoring.py +47 -36
  131. langfun/core/structured/tokenization.py +26 -11
  132. langfun/core/subscription.py +2 -2
  133. langfun/core/template.py +175 -50
  134. langfun/core/template_test.py +123 -17
  135. langfun/env/__init__.py +43 -0
  136. langfun/env/base_environment.py +827 -0
  137. langfun/env/base_environment_test.py +473 -0
  138. langfun/env/base_feature.py +304 -0
  139. langfun/env/base_feature_test.py +228 -0
  140. langfun/env/base_sandbox.py +842 -0
  141. langfun/env/base_sandbox_test.py +1235 -0
  142. langfun/env/event_handlers/__init__.py +14 -0
  143. langfun/env/event_handlers/chain.py +233 -0
  144. langfun/env/event_handlers/chain_test.py +253 -0
  145. langfun/env/event_handlers/event_logger.py +472 -0
  146. langfun/env/event_handlers/event_logger_test.py +304 -0
  147. langfun/env/event_handlers/metric_writer.py +726 -0
  148. langfun/env/event_handlers/metric_writer_test.py +214 -0
  149. langfun/env/interface.py +1640 -0
  150. langfun/env/interface_test.py +153 -0
  151. langfun/env/load_balancers.py +59 -0
  152. langfun/env/load_balancers_test.py +141 -0
  153. langfun/env/test_utils.py +507 -0
  154. {langfun-0.1.2.dev202509120804.dist-info → langfun-0.1.2.dev202512150805.dist-info}/METADATA +7 -3
  155. langfun-0.1.2.dev202512150805.dist-info/RECORD +217 -0
  156. langfun/core/eval/v2/runners_test.py +0 -343
  157. langfun/core/structured/schema.py +0 -987
  158. langfun/core/structured/schema_test.py +0 -982
  159. langfun-0.1.2.dev202509120804.dist-info/RECORD +0 -172
  160. {langfun-0.1.2.dev202509120804.dist-info → langfun-0.1.2.dev202512150805.dist-info}/WHEEL +0 -0
  161. {langfun-0.1.2.dev202509120804.dist-info → langfun-0.1.2.dev202512150805.dist-info}/licenses/LICENSE +0 -0
  162. {langfun-0.1.2.dev202509120804.dist-info → langfun-0.1.2.dev202512150805.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,217 @@
1
+ langfun/__init__.py,sha256=Kx-9_ODBpqqXb3cY5oSS-Sr-ZfuyM59W511ca9VYdRY,2663
2
+ langfun/assistant/capabilities/gui/__init__.py,sha256=7-fWINsczHkEAT1hS4vVHnTW3JC_4PZW0E4HfjClOD8,1290
3
+ langfun/assistant/capabilities/gui/bounding_box_parser.py,sha256=wJSEJvt2WFH0o9C3z9uW_vK0qRAJWbkgX5gBAJU7w6k,5832
4
+ langfun/assistant/capabilities/gui/bounding_box_parser_test.py,sha256=9yBvt7fXT_BFVcOXUSm4K0mLdKRYwk2Y9zaCpYYgBnQ,10934
5
+ langfun/assistant/capabilities/gui/drawing.py,sha256=8wgol61P7HovLg5EaevRmDPXTFuIpsfTqhOuksK-1aI,10422
6
+ langfun/assistant/capabilities/gui/drawing_test.py,sha256=d6LQ1ctG78YRi58UVBdndwyEqTC_ITdk191oA3tASxM,3421
7
+ langfun/assistant/capabilities/gui/location.py,sha256=QYJlY5kUNEwiZFiPYRyFAO7bD2ez4jL5hYn1_AK1ulc,8643
8
+ langfun/assistant/capabilities/gui/location_test.py,sha256=pQUOH1sKuAwjTTYKu615RUnecc_lNrddfvkyxf9297w,9051
9
+ langfun/core/__init__.py,sha256=HeTypJn85zOOzRb8hiIoC1vOE69xKtvUSz7TmeogLQo,5117
10
+ langfun/core/async_support.py,sha256=TZA5toE-dEm7NvmggvsE7dPUGF9SG6_nXdyVPLTXMzs,4142
11
+ langfun/core/async_support_test.py,sha256=fMz1ulGrfUCuHp7RtYktuBwpbRN3kCBKnB4LFvaXSAA,1754
12
+ langfun/core/component.py,sha256=OApzlPc3jOsYJdW29SBNs9D5NMFItf-cGWv8hLbogy4,3866
13
+ langfun/core/component_test.py,sha256=0CxTgjAud3aj8wBauFhG2FHDqrxCTl4OI4gzQTad-40,9254
14
+ langfun/core/concurrent.py,sha256=AIJ93eZLVSkyxNm-ovaWYahVTkPFnmdVfb6u9b5J4AY,34391
15
+ langfun/core/concurrent_test.py,sha256=Da4oqfu_YWH-B8H2Bv6e8tWIkR0ubkW40bccQ3N1mMg,17855
16
+ langfun/core/console.py,sha256=g3jczlZoMpIcbYyMB4cSoEHEGg5OxzQLMqAR-u8RSgE,2684
17
+ langfun/core/console_test.py,sha256=pBOcuNMJdVELywvroptfcRtJMsegMm3wSlHAL2TdxVk,1679
18
+ langfun/core/langfunc.py,sha256=bqYwgjAl56k7v1TLe2LrOj_cdfJ-9WQbbXPcW_f0a_c,8628
19
+ langfun/core/langfunc_test.py,sha256=QDtGA5pa2qt9F3YbXZs6jSpvU98PxvqezGvzFNc27NI,8982
20
+ langfun/core/language_model.py,sha256=TLrhJE4902YcUKDQsuuUxIFobVfSIQJunJh-QGTUvSk,55883
21
+ langfun/core/language_model_test.py,sha256=-1qjrgafNXa2WbicmqJyP-G8Ev-9ZmlFuE375foiWzM,40165
22
+ langfun/core/logging.py,sha256=xUmEsKLNmeZLZd3Ar8ROdv7kK2GjOkuEAgKRcAyo0AM,9138
23
+ langfun/core/logging_test.py,sha256=vbVGOQxwMmVSiFfbt2897gUt-8nqDpV64jCAeUG_q5U,6924
24
+ langfun/core/memory.py,sha256=tTZ7hWcFGG4Rl0SsWb7_82L1w-V46sA9BiOaDBUCaVI,2099
25
+ langfun/core/message.py,sha256=nkMNiPj6xYX7SuKUwknp-jRpfLdp3wUKE82uWY3wdH0,36514
26
+ langfun/core/message_test.py,sha256=cZFvGyoDzacF7yyNW1GEMzisc5dWb_vE6Xbj4UM5w1Q,38941
27
+ langfun/core/modality.py,sha256=8fKgyKDeSHIdSIQDQyz9gqGIujfQlg9_jxVaOGevtzs,8766
28
+ langfun/core/modality_test.py,sha256=0MQz6e39XOd2UjAIc5Vq9Eems1IUgOxD7Skf0gTMGTg,3665
29
+ langfun/core/natural_language.py,sha256=rNK-7Sa53igslbYE9S94R1NERD4z3vYKcJ80bW7379w,1178
30
+ langfun/core/natural_language_test.py,sha256=LHGU_1ytbkGuSZQFIFP7vP3dBlcY4-A12fT6dbjUA0E,1424
31
+ langfun/core/sampling.py,sha256=z9_yeeEtLYpIxurZ0ai9aqcQgcyYRq3Gv0fsjDr74-E,5884
32
+ langfun/core/sampling_test.py,sha256=kzoWYRiAOFFMYDraqnEi7lOWKfhW6EqGBgFRTHxPsaw,3958
33
+ langfun/core/subscription.py,sha256=1VzAX6aaESy0l8B5luNTcOqGRwAFNmA6jG_nLSfFDLI,10926
34
+ langfun/core/subscription_test.py,sha256=Y4ZdbZEwm83YNZBxHff0QR4QUa4rdaNXA3_jfIcArBo,8717
35
+ langfun/core/template.py,sha256=KIRLEhijPf5UP5auJKf9x6HKKW2E1Ki83Dcs9W8eKs8,29571
36
+ langfun/core/template_test.py,sha256=K7gx1CsDlau2CCvrE4BeKV26n0GyovhbsadWf8pDMek,20400
37
+ langfun/core/agentic/__init__.py,sha256=ajI1SGcQWXfBp2MFH13Fr9OkSN4slSKDlJSHPDp4P_c,1573
38
+ langfun/core/agentic/action.py,sha256=mgRUU6_dQVntZP3OmCoQCbrnNRHA_KVnOcKSy9GCQQM,72375
39
+ langfun/core/agentic/action_eval.py,sha256=Mjk5QBFInjIK3VlDR2RT__pugmtAW-iv-SMtH5GMcNo,5258
40
+ langfun/core/agentic/action_eval_test.py,sha256=7AkOwNbUX-ZgR1R0a7bvUZ5abNTUV7blf_8Mnrwb-II,2811
41
+ langfun/core/agentic/action_test.py,sha256=sKaU8IZ9whgBeV7oMB-0s8TdQ6PqDkI_eT9Yw3EZSYU,24123
42
+ langfun/core/coding/__init__.py,sha256=5utju_fwEsImaiftx4oXKl9FAM8p281k8-Esdh_-m1w,835
43
+ langfun/core/coding/python/__init__.py,sha256=yTXm92oLpQb4A-fZ2qy-bzfhPYND7B-oidtbv1PNaX0,1678
44
+ langfun/core/coding/python/correction.py,sha256=KbIJMNLmBv6QHEx8ja_ZweIqffzF95FdEssahy4AFhI,7702
45
+ langfun/core/coding/python/correction_test.py,sha256=sie88lAbsV15bvkRcYC88pgToybZYXI32Xmg_ym5V1A,4175
46
+ langfun/core/coding/python/execution.py,sha256=l-9IzmKyyhhUgc1hdaYLizAsrFarCfcfF7qmt3qjk04,4507
47
+ langfun/core/coding/python/execution_test.py,sha256=NynCGNP0gZ08fqmmQIJjJu6HmhkNOSkgWUXUCNQ0_yg,5990
48
+ langfun/core/coding/python/generation.py,sha256=gtSWo_Ax4-Gk5DJn6EgpCX67GQ0VmiZCheaK1Y-AdqM,8863
49
+ langfun/core/coding/python/generation_test.py,sha256=dcF5ef1UApzRfTpvICiChpynkzZ1mDsE0DvH0iMpTvg,2743
50
+ langfun/core/coding/python/parsing.py,sha256=jvGDIwoaY3mdGXeFhjP27w0ukO0TtdCC7G4ODVNp8S4,4554
51
+ langfun/core/coding/python/parsing_test.py,sha256=PIexYpSEhgNaSd4T6QYWzWHzm3sL4VhQJ4dhdvJAQk8,5005
52
+ langfun/core/coding/python/sandboxing.py,sha256=XoEYOYLRazdd6rIz0_tUnmXdNnfD6etnXQxRlJfo_VU,5089
53
+ langfun/core/coding/python/sandboxing_test.py,sha256=H_0_pd-_uS-ci5yYhmDTR6-hyzosAFkExziAHndfdDo,2023
54
+ langfun/core/data/__init__.py,sha256=qllw9ST1vveZv-1E0VM5hezn1YH-OcqGI-yFqQYnWgI,732
55
+ langfun/core/data/conversion/__init__.py,sha256=ZcGntBruvvZSfESO-Tha1nzHfgWEK7I1u78Jw8gsoVU,883
56
+ langfun/core/data/conversion/anthropic.py,sha256=ns224RBsKsVmiv72N3dukPuyOqY31zu92XciY9_iDQw,4886
57
+ langfun/core/data/conversion/anthropic_test.py,sha256=BMIyLgFOtFuJYJAkSLR1nwG7UOwtoT61R5xAXtJfSMY,8624
58
+ langfun/core/data/conversion/gemini.py,sha256=eCM3t0FEyrNfMzWpM-Awp1pwPz21lq_2mD_EsEcIn4w,6291
59
+ langfun/core/data/conversion/gemini_test.py,sha256=yl0xBo5sd9cW9a-iQJUyocUBcozmm-QuueZlexF33xs,8399
60
+ langfun/core/data/conversion/openai.py,sha256=P_HPlS9DIH6s-LTVq9I-UM_GGKSK9ZsPgiR2CldRnpM,7863
61
+ langfun/core/data/conversion/openai_test.py,sha256=7OrB_i4HF2zUxkdeFp35p4pyzwnpY6uN-ACIqrPgUVI,9835
62
+ langfun/core/eval/__init__.py,sha256=OEXr1ZRuvLuhJJfuQ1ZWQ-SvYzjyrtiAAEogYaB7E6o,1933
63
+ langfun/core/eval/base.py,sha256=A7jMpvvEej37GAnVWjt9FLP2Yiuuac0AP9GjzDSZDRs,76158
64
+ langfun/core/eval/base_test.py,sha256=yiZARvXldmIdpAj0hD_FM5D40DEZuYvwhnEM-x5kjso,27197
65
+ langfun/core/eval/matching.py,sha256=IXih9N3tvLuF1EmdmqldLBR0kyQsXuh2dOz2J3RC12E,9458
66
+ langfun/core/eval/matching_test.py,sha256=2xtwsTi-UzLTt0QnXl3u_eAG3fFjCG2tsae7YkcQTB0,5312
67
+ langfun/core/eval/patching.py,sha256=wJqqML_z_hXQQ65f9oJpdtiNEkUvwWWdNgGiIcV1Jq4,3871
68
+ langfun/core/eval/patching_test.py,sha256=8kCd54Egjju22FMgtJuxEsrXkW8ifs-UUBHtrCG1L6w,4775
69
+ langfun/core/eval/scoring.py,sha256=1C7e7gR8Wai7M9oBXRZifntxy5HEik5qjVo9gY8B7KI,6423
70
+ langfun/core/eval/scoring_test.py,sha256=UcBH0R6vAovZ0A4yM22s5cBHL1qVKASubrbu1t8dYBw,4529
71
+ langfun/core/eval/v2/__init__.py,sha256=XbkBqoyJBH_khtAS01gP6_V4KnWLY3bFJ7D0rtHa1BU,1878
72
+ langfun/core/eval/v2/checkpointing.py,sha256=ui4kOwOo_yu_ONzOho9Ri36NJOmYGqD1gYa6o1U7L9o,15463
73
+ langfun/core/eval/v2/checkpointing_test.py,sha256=s_E94dOPNO1zYzXyQI37wvCF3suez-r4Nls9popN58w,9787
74
+ langfun/core/eval/v2/config_saver.py,sha256=nsuG0pqTikIlsL-Mij6swteUBif-zxJUdGxTHZsOVeQ,1205
75
+ langfun/core/eval/v2/config_saver_test.py,sha256=OD0zl26YHjNibFD67YxwrZ7-zT9V7p-3zLDItWBAgic,1261
76
+ langfun/core/eval/v2/eval_test_helper.py,sha256=baew3-cqomy1p7mF1_Xw7AvEWUwCimi3J7-8Ay3eEPo,6539
77
+ langfun/core/eval/v2/evaluation.py,sha256=1T0lxTu9gy329Mq4ii16ktARbtvbBGY9IUtsUIkNXeY,30740
78
+ langfun/core/eval/v2/evaluation_test.py,sha256=gurFzSfPECZ_FMQOnf3bzKOHmQ7C4IUxEfbyZy50bjM,7966
79
+ langfun/core/eval/v2/example.py,sha256=VZeBqMWnfEtn1mmdPW2w2u2XbAWVll1q1-50qL8DjS8,11606
80
+ langfun/core/eval/v2/example_test.py,sha256=RwtBcUumPBWynA8BLMoZetSHdgvFywlHXuyvInf1y_s,3576
81
+ langfun/core/eval/v2/experiment.py,sha256=NpVRkMRi4IXt1qx9b3k_hwHfVLkBrvtYRlMH3ID8FBA,36758
82
+ langfun/core/eval/v2/experiment_test.py,sha256=7prE4ASKlbwQIXiLzEqjgaF4yQDL7KjxX-dBUPT84VA,14145
83
+ langfun/core/eval/v2/metric_values.py,sha256=WAL1BdHaU_oq7d_k1KyjhiQDK32dNLSyn1L2yEkz0o4,6040
84
+ langfun/core/eval/v2/metric_values_test.py,sha256=5ffwnqrbLIBh1hdUl3L9mpJlUvsmd2VQ8UWPOJcQj4s,3630
85
+ langfun/core/eval/v2/metrics.py,sha256=cdFqrhRlxqpBk_04Mmhk21NcOD0kor5H0iFX54_rO4s,14486
86
+ langfun/core/eval/v2/metrics_test.py,sha256=gf8hT5V5OeM-Ah-Wa4aLtgrYZmlMStKPjEhCTS0VMHQ,6812
87
+ langfun/core/eval/v2/progress.py,sha256=Cd79j8fhumW5QOuISiSXOJKOZ5-I9IkmGLgvqRmoULA,11677
88
+ langfun/core/eval/v2/progress_test.py,sha256=MzJ7wa65XYZ0chArA-lSg1eRSvQ_TzZJIHMk85Kwz7o,3208
89
+ langfun/core/eval/v2/progress_tracking.py,sha256=yMYlOMJF8M4FUhyjGRkM6O6TXiMwKPsEn3wbpftxcss,6376
90
+ langfun/core/eval/v2/progress_tracking_test.py,sha256=37v42y4kh2GfDXBrkugEupW6IRAzA774wwPJaOyefUs,2597
91
+ langfun/core/eval/v2/reporting.py,sha256=Z_tt_EfApPa-AcfYmfZ2818fk8eWK-EGl1fYlgxpCAk,8895
92
+ langfun/core/eval/v2/reporting_test.py,sha256=q3LBfPk7jvEWXB3sdk2CycbMKqNRyXhs5z6BokfwDIE,6096
93
+ langfun/core/eval/v2/runners/__init__.py,sha256=2TcCLW32OsmXQINcVKa2ZJY8Ca7j3NnT0yy9hXYUDn8,1115
94
+ langfun/core/eval/v2/runners/base.py,sha256=_ixOIxGxrrNKDLBxJlfjLHCzlkjxKUkJY_MO3CmzM14,14072
95
+ langfun/core/eval/v2/runners/beam.py,sha256=LQK9bZCFJR9j9DJ-mAudhphumItGwXc5bbGwadl9kxY,11782
96
+ langfun/core/eval/v2/runners/beam_test.py,sha256=cI5WaQQObnRrPnGjED3OFT3JXYOE3thQ640H08TG_dw,5306
97
+ langfun/core/eval/v2/runners/ckpt_monitor.py,sha256=KaaDYvHNOewUrJqJ4FHjdMeS7okpX7FYdjCx558joPU,12071
98
+ langfun/core/eval/v2/runners/ckpt_monitor_test.py,sha256=Xqd30PF0XIOrqBSZ53_7ozxYR3Wc3SiIaKuwwj1AXQ8,7176
99
+ langfun/core/eval/v2/runners/debug.py,sha256=ExsBcAvmhFsaaS3VLjxE70HImHe2YVs0IpoefM01onY,1442
100
+ langfun/core/eval/v2/runners/debug_test.py,sha256=kDWs4Fu7itzBxbRwFc-UKEP2hAV0iVFp2wWkEuZNEcg,2577
101
+ langfun/core/eval/v2/runners/parallel.py,sha256=PSdOY3i2ot94TWVCZY0iJSWFAT0CCxa1wxk7KpI_GfI,7794
102
+ langfun/core/eval/v2/runners/parallel_test.py,sha256=8M8OTpsDd-wQYZRRSPCYGkwjt7gUvkgze8NMCTKydUw,6146
103
+ langfun/core/eval/v2/runners/sequential.py,sha256=hebMZd6EVraY9zAwariT9WfsWQyX5AYuRsFdRo-knKU,1631
104
+ langfun/core/eval/v2/runners/sequential_test.py,sha256=apbNC0-Pi6r17_OQlHqqOZM0OVo1mZlaPk2B4vUteRg,6064
105
+ langfun/core/llms/__init__.py,sha256=KU00R0906yLWjSg_tquCna1CU_6z4XOIKMhLzzGE-Zc,10489
106
+ langfun/core/llms/anthropic.py,sha256=6uE1EC9YWtbiFwZNNPEFv-QzeGQQ7G27kheTTE15Ewg,31175
107
+ langfun/core/llms/anthropic_test.py,sha256=qA9vByp_cwwXNlXzcwHpPWFnO9lfFo8NKfDi5nBNqgI,9052
108
+ langfun/core/llms/azure_openai.py,sha256=LEc7-ay2fOOCwwL3SfxDr3KCdH8-2i1EtD-PBvr4kfk,2777
109
+ langfun/core/llms/azure_openai_test.py,sha256=lkMZkQdJBV97fTM4C4z8qNfvr6spgiN5G4hvVUIVr0M,1735
110
+ langfun/core/llms/compositional.py,sha256=dOGGMIidM31N3L-a5MXc10JL1euHkolqULIRDZQYpsk,3579
111
+ langfun/core/llms/compositional_test.py,sha256=4eTnOer-DncRKGaIJW2ZQQMLnt5r2R0UIx_DYOvGAQo,2027
112
+ langfun/core/llms/deepseek.py,sha256=jQsotTUk4161EJIcoQOV7iOWBZfQ3Ukh9GOh31A0HYU,5770
113
+ langfun/core/llms/deepseek_test.py,sha256=DvROWPlDuow5E1lfoSkhyGt_ELA19JoQoDsTnRgDtTg,1847
114
+ langfun/core/llms/fake.py,sha256=NH8Zlezmx3eacao4D7wihrZjRuyBJuHR5rdyp94PrAw,4409
115
+ langfun/core/llms/fake_test.py,sha256=lC-C2TpEsnf2kmZpa3OiH2H944I4hMWTAaHEXzRj1DU,7855
116
+ langfun/core/llms/gemini.py,sha256=_GMcbkfaSWkMGiK1d8DfpQzRiSCZrd092VhBMfRZ9H0,33243
117
+ langfun/core/llms/gemini_test.py,sha256=bv-Ulv3vjGhxd8nJD_UDhWDMK3K3TM7b5powBcYrv1c,10844
118
+ langfun/core/llms/google_genai.py,sha256=hodpibBtcxg8pU-XrEsPBkhzGsjSYrEUHyz0w9RWwCc,6986
119
+ langfun/core/llms/google_genai_test.py,sha256=NKNtpebArQ9ZR7Qsnhd2prFIpMjleojy6o6VMXkJ1zY,1502
120
+ langfun/core/llms/groq.py,sha256=O-kv2_R_IkC8wGIT086xin8jYi7QnsakPCGVLR58lMw,12517
121
+ langfun/core/llms/groq_test.py,sha256=P4EgexCqsh4K2x11w0UL_vz-YYNaPdQU0WsDAdnTRQ8,2045
122
+ langfun/core/llms/llama_cpp.py,sha256=SsAj9_40Px50_bNBE92x9akgj2O0FobJ00gd3AwZN1I,1907
123
+ langfun/core/llms/llama_cpp_test.py,sha256=wfTO7nmUwL65U2kK9P9fcMt92JjNDuVia4G1E7znf_4,1086
124
+ langfun/core/llms/openai.py,sha256=T2TGcORPJ0f7zKFUalRwRqxzDprkQntGwnlOVem28Wo,45689
125
+ langfun/core/llms/openai_compatible.py,sha256=scmILhSxno9JS6eX6K9dWKLZXUOtIqfWWdz4Su7DGRI,9803
126
+ langfun/core/llms/openai_compatible_test.py,sha256=8yr_jGmHCDyMwp-VcJwThFgh7B_56h69Fmw97XZxAZw,23133
127
+ langfun/core/llms/openai_test.py,sha256=1o5rxiHZj-UEgugWN8JmfJtznhUmDywy6dU3Euax-Ts,2639
128
+ langfun/core/llms/rest.py,sha256=eR-M1st5ZnzuitICyYfxSRcmQWmy_eeOoe2bHLalzN0,5351
129
+ langfun/core/llms/rest_test.py,sha256=_zM7nV8DEVyoXNiQOnuwJ917mWjki0614H88rNmDboE,5020
130
+ langfun/core/llms/vertexai.py,sha256=KjiMrEjWgoJct9QQTQKQ_8fzZ5SbpVNDyZpvHgHZj3g,22134
131
+ langfun/core/llms/vertexai_test.py,sha256=_e-acnNBAf9C3WO6i1b2J_mhRzdDdYQTorD9hIVZKOg,5034
132
+ langfun/core/llms/cache/__init__.py,sha256=QAo3InUMDM_YpteNnVCSejI4zOsnjSMWKJKzkb3VY64,993
133
+ langfun/core/llms/cache/base.py,sha256=qLGlEMi5cfsDxRTsOWrmwbxjvvwUaq4Y8MxlXr69wpw,5060
134
+ langfun/core/llms/cache/in_memory.py,sha256=yqEBJW0C_ymqT4UNiE7aB81VlmTlkvTRXJvtUfv3FTE,6633
135
+ langfun/core/llms/cache/in_memory_test.py,sha256=AG9OGdxfyW9LdteFRF-aOIhxiH5Kna5U-pQk91ljHCQ,10538
136
+ langfun/core/mcp/__init__.py,sha256=cyVP_YTjOmbjhYg8BE7-RnE4Txt8wDukYC0anhHKpuo,286
137
+ langfun/core/mcp/client.py,sha256=WoLB1LyXeOKOGS5MBSY8KpIb2iFIA3IIDcW_FhabFdY,5459
138
+ langfun/core/mcp/client_test.py,sha256=ytplbrnFDqnHkDQuf5UGJaFf-rXUtwONCJcmNZ_rNUY,2221
139
+ langfun/core/mcp/session.py,sha256=7derNfIpRaUoFUcJMXUvDnrhJJpFRvr2pD1C1WQoDnw,7366
140
+ langfun/core/mcp/session_test.py,sha256=V4UVH9uVDL10RUWUr4jldcZqn4HYB75WgkjAnNg6cVw,1952
141
+ langfun/core/mcp/tool.py,sha256=oeDY6gK77mxKXdoCQ5KVeTdfIvDSoXDFTT5Uy1Gdff8,8262
142
+ langfun/core/mcp/tool_test.py,sha256=oKsIpkyb-2Pr7_Fyqn1WNX8B07_izfdUyo4fIAp7x3o,6372
143
+ langfun/core/mcp/testing/simple_mcp_client.py,sha256=U5pUXa0SnB2-DdRI2vPhrdVUv14dX_OeS4mPLFpllMc,924
144
+ langfun/core/mcp/testing/simple_mcp_server.py,sha256=dw4t6ERWMdmi6kDE38RU5oYu5MQbEX-GJ6CMxGcV-WE,994
145
+ langfun/core/memories/__init__.py,sha256=HpghfZ-w1NQqzJXBx8Lz0daRhB2rcy2r9Xm491SBhC4,773
146
+ langfun/core/memories/conversation_history.py,sha256=KR78PurXTSeqsRK9QG9xM7-f245rs20EvWO7Mm2n2Vw,1827
147
+ langfun/core/memories/conversation_history_test.py,sha256=2kzAq2pUrbR01Z9jhxviIao52JK4JVjr5iGP8pwGxlU,2156
148
+ langfun/core/modalities/__init__.py,sha256=2_T5Hsc4iEqmJZuYaqwRNgNM5XpmuTVcnZUMe5cAQDk,1521
149
+ langfun/core/modalities/audio.py,sha256=cb95FzDE-IIQf7kXy7D4AAXtziQF0FYkZUe4pw5EEJc,1502
150
+ langfun/core/modalities/audio_test.py,sha256=tW1vEy-Cumhf-HgDgCxlSNZqgJb2HTgqOixGWLiwOmw,2065
151
+ langfun/core/modalities/image.py,sha256=SS_FSrYSjb1uL0IAVyUu2TZX2-FcI_x9ZTB8im5Amzk,4030
152
+ langfun/core/modalities/image_test.py,sha256=91LpEOvr_v6SGdtzSuCw3ks62L8vxJVIZwgAKxk7UmY,8476
153
+ langfun/core/modalities/mime.py,sha256=9YK-uRGYN6YG3ux7zSYl5XGZEDLBiXyTax1cLMhissY,11255
154
+ langfun/core/modalities/mime_test.py,sha256=e6p-XW47yNXbvPS2R4-0afZd84bOqCc6DnIzqZCkPZk,8391
155
+ langfun/core/modalities/pdf.py,sha256=rc-uIKRVkTTa0j7jC6WRwKM9WqiS5NxF-H6PPunVeXM,1231
156
+ langfun/core/modalities/pdf_test.py,sha256=ulZ0FbnlsU0wkrdckJ4ONZPTYRyMPO9Aob1UO6FXygk,1950
157
+ langfun/core/modalities/video.py,sha256=ZopyDf-8bi0V-QZDAg-_8S3HkMNiEQL9aWmGuI6Fkrs,1506
158
+ langfun/core/modalities/video_test.py,sha256=7OXZoohKMYjt7vrJUdPb553HLyl1oBOKRgzBePFv68Q,2042
159
+ langfun/core/structured/__init__.py,sha256=rFudkJOw5jZy4KV0_n-vbK8TQ90YniCeVaNT7FPQCtg,2686
160
+ langfun/core/structured/completion.py,sha256=Avr6QRM2GFB4wl33BQyc8aabIJMtWNu70Rnc017dYoI,8848
161
+ langfun/core/structured/completion_test.py,sha256=OoyxtbiaAdy_QfJeTNKfvijPiC0H91gIjuIiWvGUd4o,19776
162
+ langfun/core/structured/description.py,sha256=eHB4w5Qu_hkYTMLlTSK5efoQ2o4Bc9_SOGO_I0oFiwE,5328
163
+ langfun/core/structured/description_test.py,sha256=UxaXnKKP7TnyPDPUyf3U-zPE0TvLlIP6DGr8thjcePw,7365
164
+ langfun/core/structured/function_generation.py,sha256=tx_GtKTIR713LrFuJzBJ0m-MLUnx3776GKbxyGNTMoA,8987
165
+ langfun/core/structured/function_generation_test.py,sha256=LaXYDXf9GlqUrR6v_gtmK_H4kxzonmU7SYbn7XXMgjU,12128
166
+ langfun/core/structured/mapping.py,sha256=BZMVd_irVDzL6Q8su4IBimPR3FiKXVv4stHjuOTg8ds,16070
167
+ langfun/core/structured/mapping_test.py,sha256=OntYvfDitAf0tAnzQty3YS90vyEn6FY1Mi93r_ViEk8,9594
168
+ langfun/core/structured/parsing.py,sha256=0vyUItnKGaR-qePbt_15bzuuUrBXB8qJuGA7xQ9xl0A,14582
169
+ langfun/core/structured/parsing_test.py,sha256=qr7Mqh4bgw8ZuJIA_9rnPLa4QzfTQnX53kmDeQDmD6Q,22627
170
+ langfun/core/structured/querying.py,sha256=BHq6Do736kpOvg2181wV5secCPWcKkJidSldoZ4z26I,42886
171
+ langfun/core/structured/querying_test.py,sha256=zjUEtBPcD0xAPKVmnIwZLzoGpRCMk5pIDtMDG2K5Ps4,51728
172
+ langfun/core/structured/schema_generation.py,sha256=BZxhTk35Hxix4ddfBe9Xkqpu3XN3JwLqMokvV6Y4EB4,6099
173
+ langfun/core/structured/schema_generation_test.py,sha256=RM9s71kMNg2jTePwInkiW9fK1ACN37eyPeF8OII-0zw,2950
174
+ langfun/core/structured/scoring.py,sha256=BQSme22gV3XcTyALhXNrMR0fkuWhFi0HozSdBRuE0yo,6822
175
+ langfun/core/structured/scoring_test.py,sha256=msxtZX1MWgoLTWkmM-McbUo-qGev0uDdlXLm4kPiIiE,2473
176
+ langfun/core/structured/tokenization.py,sha256=xaOztsBaFj2b3bwny5-pdAutPV9YeAiQTdP81VVwhM4,3436
177
+ langfun/core/structured/tokenization_test.py,sha256=8IXndRokZmlLPZD_jIitfhgcRxBjmG09PhT-WLHe9dw,1499
178
+ langfun/core/structured/schema/__init__.py,sha256=QOKreIMT4QftjHTx764rrB0z0fnw98gA1mzBYVVv6WQ,2256
179
+ langfun/core/structured/schema/base.py,sha256=kpsu729IIIC8r6EsdT6ur2FpwyFoxCrz6WwSCWBkZG4,19488
180
+ langfun/core/structured/schema/base_test.py,sha256=_j9U4GwO9BLpj_ENqw_4XwLopfHG1HgR0ek-VUoXkfE,15039
181
+ langfun/core/structured/schema/json.py,sha256=gWs6Pe99WsZNiv5vNU3VFXKS6GhYpBxuaJMuiU7agxY,5047
182
+ langfun/core/structured/schema/json_test.py,sha256=PnXo36FfS7yWHeohlRGqYjpzx5bpVgEAOKOa0x9H2ow,3411
183
+ langfun/core/structured/schema/python.py,sha256=xKigOxZGdiOhmiWRC7GtkY_PZyl0x2AbE-4iZCbmhaM,9564
184
+ langfun/core/structured/schema/python_test.py,sha256=nV1_J5J5kisUmbd1IB4yliQkmy_z5vgtPgyPT6L9uOA,9555
185
+ langfun/core/templates/__init__.py,sha256=bO0eMsVJbi7sxEB2YlInKRQ2EVP-RyyKUwcD-8msuN4,927
186
+ langfun/core/templates/completion.py,sha256=mUqZHOEV3ag6-A08XghpeEltcrBvCDxXP004eDDfeag,1931
187
+ langfun/core/templates/completion_test.py,sha256=vGnjnM38UHyVDUyaUYtmp20s9KBGOdbPVsX-H-ET11E,1636
188
+ langfun/core/templates/conversation.py,sha256=iURikG7JEvXGsFSZfdzj6PyGQCAPAEejrs8hXEh7Sc0,2868
189
+ langfun/core/templates/conversation_test.py,sha256=HCD5f1sgmSGqL3OGe7hEctjRdcFjcmkz3NV1ArexNYI,3942
190
+ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fikKhwhzwhpKI,1460
191
+ langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
192
+ langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
193
+ langfun/core/templates/selfplay_test.py,sha256=Ot__1P1M8oJfoTp-M9-PQ6HUXqZKyMwvZ5f7yQ3yfyM,2326
194
+ langfun/env/__init__.py,sha256=Iwl5JQk6bGuScJcjx8NqKUCaZw1r8oRiuqFadu8x_Ss,1680
195
+ langfun/env/base_environment.py,sha256=pR0ytF6ZY_Hvd920Pb6j_PGg1XjfFWqYRpo6UqC8PTs,27017
196
+ langfun/env/base_environment_test.py,sha256=JltE2uPrruCpj0V50T3NGG3cf_8WDss67dEMk-REPRA,15457
197
+ langfun/env/base_feature.py,sha256=ohotob1Cnjd9M1BzHUS2ezGZgifcvXs6Ce0bOTGTkf8,8473
198
+ langfun/env/base_feature_test.py,sha256=n1IrU6tHRwHjkyD7yHJtfAQ8yrSSAteVdIqfTK49JI8,7094
199
+ langfun/env/base_sandbox.py,sha256=hTomEzky4Jfunt2Kse7uaboBYCCGAUf_xTY9kcPZDcY,27942
200
+ langfun/env/base_sandbox_test.py,sha256=j1WugtZBjvI9zo2_1tvuBhORMFENp8TanQwGfXwEUKE,60308
201
+ langfun/env/interface.py,sha256=tZGJ1RLmcsctz62pyBdravibUQL439Lkc4O40l7FtzY,51760
202
+ langfun/env/interface_test.py,sha256=OJMI8OAntMG7XpFya5m97nfLanuNUQVrCbbmqt1F688,4232
203
+ langfun/env/load_balancers.py,sha256=qRhCthqzjZIQBwta8qC1C0s0J-VQAVomJQqI7Nqv-r4,1948
204
+ langfun/env/load_balancers_test.py,sha256=_uurvp9-Gk6J6AtmWI4APztujNxLp8BPjsYZrO-xIdE,4145
205
+ langfun/env/test_utils.py,sha256=DUfHvfU4IF93grPqMF3qrLGu6_8ghNk_r_ikuyo1ePE,15046
206
+ langfun/env/event_handlers/__init__.py,sha256=EE3pV0BlhvYfTG3njhVTMqj88IO_gwjLqZDGSGL95DE,449
207
+ langfun/env/event_handlers/chain.py,sha256=MrAF7BTVJxQyYEFrBdyGCb0b3RotBVInJm0CHySYXNo,6946
208
+ langfun/env/event_handlers/chain_test.py,sha256=92xBFgL6NitbH8HSKZWQzK9jOm_SWyZwVywFyrxG6fc,8129
209
+ langfun/env/event_handlers/event_logger.py,sha256=ga8RN8qjwtAOCnV_MnhNPTktN8EJ-x1qw4Z_MsnvI5A,12554
210
+ langfun/env/event_handlers/event_logger_test.py,sha256=qSAcirtRz00H-1RL9ShELBiZKiPxsk_v6cVA6XdAk4k,9274
211
+ langfun/env/event_handlers/metric_writer.py,sha256=7ZrUp0rYvs7TfNpQ16Xbxg8vp-6ZbjuJ-qrhVSbhv2I,21085
212
+ langfun/env/event_handlers/metric_writer_test.py,sha256=bjdYXoXMPWpWz_-HUPM6vFP1ez5G386u0fmPfe-SR_M,5952
213
+ langfun-0.1.2.dev202512150805.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
214
+ langfun-0.1.2.dev202512150805.dist-info/METADATA,sha256=AEsimQbtMKxj8Kja2fIIgEXhoBsmqLwq-1PF3i_WlFg,7522
215
+ langfun-0.1.2.dev202512150805.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
216
+ langfun-0.1.2.dev202512150805.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
217
+ langfun-0.1.2.dev202512150805.dist-info/RECORD,,
@@ -1,343 +0,0 @@
1
- # Copyright 2024 The Langfun Authors
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
- import os
15
- import tempfile
16
- import threading
17
- import time
18
- from typing import Any
19
- import unittest
20
-
21
- from langfun.core.eval.v2 import eval_test_helper
22
- from langfun.core.eval.v2 import example as example_lib
23
- from langfun.core.eval.v2 import experiment as experiment_lib
24
- from langfun.core.eval.v2 import runners as runners_lib # pylint: disable=unused-import
25
-
26
- import pyglove as pg
27
-
28
-
29
- Runner = experiment_lib.Runner
30
- Example = example_lib.Example
31
- Experiment = experiment_lib.Experiment
32
- Suite = experiment_lib.Suite
33
- Plugin = experiment_lib.Plugin
34
-
35
-
36
- class TestPlugin(Plugin):
37
- started_experiments: list[Experiment] = []
38
- completed_experiments: list[Experiment] = []
39
- skipped_experiments: list[Experiment] = []
40
- started_example_ids: list[int] = []
41
- completed_example_ids: list[int] = []
42
- skipped_example_ids: list[int] = []
43
- start_time: float | None = None
44
- complete_time: float | None = None
45
-
46
- def _on_bound(self):
47
- super()._on_bound()
48
- self._lock = threading.Lock()
49
-
50
- def on_run_start(self, runner: Runner, root: Experiment):
51
- del root
52
- with pg.notify_on_change(False), pg.allow_writable_accessors(True):
53
- self.start_time = time.time()
54
-
55
- def on_run_complete(self, runner: Runner, root: Experiment):
56
- del root
57
- with pg.notify_on_change(False), pg.allow_writable_accessors(True):
58
- self.complete_time = time.time()
59
-
60
- def on_experiment_start(self, runner: Runner, experiment: Experiment):
61
- del runner
62
- with pg.notify_on_change(False), self._lock:
63
- self.started_experiments.append(pg.Ref(experiment))
64
-
65
- def on_experiment_skipped(self, runner: Runner, experiment: Experiment):
66
- del runner
67
- with pg.notify_on_change(False), self._lock:
68
- self.skipped_experiments.append(pg.Ref(experiment))
69
-
70
- def on_experiment_complete(self, runner: Runner, experiment: Experiment):
71
- del runner
72
- with pg.notify_on_change(False), self._lock:
73
- self.completed_experiments.append(pg.Ref(experiment))
74
-
75
- def on_example_start(
76
- self, runner: Runner, experiment: Experiment, example: Example):
77
- del runner, experiment
78
- with pg.notify_on_change(False), self._lock:
79
- self.started_example_ids.append(example.id)
80
-
81
- def on_example_skipped(
82
- self, runner: Runner, experiment: Experiment, example: Example):
83
- del runner, experiment
84
- with pg.notify_on_change(False), self._lock:
85
- self.skipped_example_ids.append(example.id)
86
-
87
- def on_example_complete(
88
- self, runner: Runner, experiment: Experiment, example: Example):
89
- del runner, experiment
90
- with pg.notify_on_change(False), self._lock:
91
- self.completed_example_ids.append(example.id)
92
-
93
-
94
- class RunnerTest(unittest.TestCase):
95
-
96
- def assert_same_list(self, actual: list[Any], expected: list[Any]):
97
- self.assertEqual(len(actual), len(expected))
98
- for i, (x, y) in enumerate(zip(actual, expected)):
99
- if x is not y:
100
- print(i, pg.diff(x, y))
101
- self.assertIs(x, y)
102
-
103
- def test_basic(self):
104
- plugin = TestPlugin()
105
- exp = eval_test_helper.test_experiment()
106
- root_dir = os.path.join(tempfile.mkdtemp(), 'test_sequential_runner')
107
- run = exp.run(root_dir, runner='sequential', plugins=[plugin])
108
-
109
- self.assertIsNotNone(plugin.start_time)
110
- self.assertIsNotNone(plugin.complete_time)
111
- self.assertGreater(plugin.complete_time, plugin.start_time)
112
-
113
- self.assert_same_list(
114
- plugin.started_experiments,
115
- exp.nonleaf_nodes + exp.leaf_nodes
116
- )
117
- self.assert_same_list(
118
- plugin.completed_experiments,
119
- exp.leaf_nodes + list(reversed(exp.nonleaf_nodes))
120
- )
121
- self.assert_same_list(
122
- plugin.started_example_ids, list(range(1, 11)) * 6
123
- )
124
- self.assert_same_list(
125
- plugin.completed_example_ids, list(range(1, 11)) * 6
126
- )
127
- self.assert_same_list(plugin.skipped_experiments, [])
128
- self.assert_same_list(plugin.skipped_example_ids, [])
129
- self.assertTrue(
130
- pg.io.path_exists(os.path.join(run.output_root, 'run.json'))
131
- )
132
-
133
- for node in exp.nodes:
134
- self.assertTrue(node.progress.is_started)
135
- self.assertTrue(node.progress.is_completed)
136
- if node.is_leaf:
137
- self.assertEqual(node.progress.num_skipped, 0)
138
- self.assertEqual(node.progress.num_completed, 10)
139
- self.assertEqual(node.progress.num_failed, 1)
140
- else:
141
- self.assertEqual(node.progress.num_skipped, 0)
142
- self.assertEqual(node.progress.num_failed, 0)
143
- self.assertEqual(node.progress.num_processed, node.progress.num_total)
144
-
145
- def test_raise_if_has_error(self):
146
- root_dir = os.path.join(tempfile.mkdtemp(), 'test_raise_if_has_error')
147
- exp = eval_test_helper.TestEvaluation()
148
- with self.assertRaisesRegex(ValueError, 'x should not be 5'):
149
- exp.run(
150
- root_dir, runner='sequential', plugins=[], raise_if_has_error=True
151
- )
152
-
153
- with self.assertRaisesRegex(ValueError, 'x should not be 5'):
154
- exp.run(root_dir, runner='parallel', plugins=[], raise_if_has_error=True)
155
-
156
- def test_example_ids(self):
157
- root_dir = os.path.join(tempfile.mkdtemp(), 'test_example_ids')
158
- exp = eval_test_helper.test_experiment()
159
- plugin = TestPlugin()
160
- _ = exp.run(
161
- root_dir, runner='sequential', plugins=[plugin], example_ids=[5, 7, 9]
162
- )
163
- self.assertEqual(plugin.started_example_ids, [5, 7, 9] * 6)
164
- self.assertEqual(plugin.completed_example_ids, [5, 7, 9] * 6)
165
-
166
- def test_shuffle_inputs(self):
167
- root_dir = os.path.join(tempfile.mkdtemp(), 'test_shuffle_inputs')
168
- exp = eval_test_helper.test_experiment()
169
- plugin = TestPlugin()
170
- run = exp.run(
171
- root_dir, runner='sequential', plugins=[plugin], shuffle_inputs=True
172
- )
173
- self.assertTrue(run.shuffle_inputs)
174
-
175
- def test_filter(self):
176
- plugin = TestPlugin()
177
- exp = eval_test_helper.test_experiment()
178
- root_dir = os.path.join(tempfile.mkdtemp(), 'test_filter')
179
-
180
- _ = exp.run(
181
- root_dir, runner='sequential', plugins=[plugin],
182
- filter=lambda e: e.lm.offset != 0
183
- )
184
- self.assert_same_list(
185
- plugin.started_experiments,
186
- exp.nonleaf_nodes + exp.leaf_nodes[2:]
187
- )
188
- self.assert_same_list(
189
- plugin.skipped_experiments, exp.leaf_nodes[:2]
190
- )
191
- self.assert_same_list(
192
- plugin.completed_experiments,
193
- exp.leaf_nodes[2:] + [exp.children[1], exp]
194
- )
195
-
196
- def test_use_cache(self):
197
- @pg.functor()
198
- def test_inputs(num_examples: int = 10):
199
- return [
200
- pg.Dict(
201
- x=i // 2, y=(i // 2) ** 2,
202
- groundtruth=(i // 2 + (i // 2) ** 2)
203
- ) for i in range(num_examples)
204
- ]
205
-
206
- exp = eval_test_helper.TestEvaluation(
207
- inputs=test_inputs(num_examples=pg.oneof([2, 4]))
208
- )
209
- # Global cache.
210
- root_dir = os.path.join(tempfile.mkdtemp(), 'global_cache')
211
- run = exp.run(
212
- root_dir, 'new', runner='sequential', use_cache='global', plugins=[]
213
- )
214
- self.assertTrue(pg.io.path_exists(run.output_path_for(exp, 'cache.json')))
215
- self.assertEqual(exp.usage_summary.cached.total.num_requests, 4)
216
- self.assertEqual(exp.usage_summary.uncached.total.num_requests, 2)
217
-
218
- # Per-dataset cache.
219
- root_dir = os.path.join(tempfile.mkdtemp(), 'per_dataset')
220
- run = exp.run(
221
- root_dir, 'new', runner='sequential',
222
- use_cache='per_dataset', plugins=[]
223
- )
224
- for leaf in exp.leaf_nodes:
225
- self.assertTrue(
226
- pg.io.path_exists(run.output_path_for(leaf, 'cache.json'))
227
- )
228
- self.assertEqual(exp.usage_summary.cached.total.num_requests, 3)
229
- self.assertEqual(exp.usage_summary.uncached.total.num_requests, 3)
230
-
231
- # No cache.
232
- root_dir = os.path.join(tempfile.mkdtemp(), 'no')
233
- run = exp.run(root_dir, runner='sequential', use_cache='no', plugins=[])
234
- self.assertFalse(pg.io.path_exists(run.output_path_for(exp, 'cache.json')))
235
- for leaf in exp.leaf_nodes:
236
- self.assertFalse(
237
- pg.io.path_exists(run.output_path_for(leaf, 'cache.json'))
238
- )
239
- self.assertEqual(exp.usage_summary.cached.total.num_requests, 0)
240
- self.assertEqual(exp.usage_summary.uncached.total.num_requests, 6)
241
-
242
-
243
- class ParallelRunnerTest(RunnerTest):
244
-
245
- def test_parallel_runner(self):
246
- plugin = TestPlugin()
247
- exp = eval_test_helper.test_experiment()
248
- root_dir = os.path.join(tempfile.mkdtemp(), 'test_parallel_runner')
249
- run = exp.run(root_dir, runner='parallel', plugins=[plugin])
250
-
251
- self.assertIsNotNone(plugin.start_time)
252
- self.assertIsNotNone(plugin.complete_time)
253
- self.assertGreater(plugin.complete_time, plugin.start_time)
254
-
255
- self.assertEqual(
256
- len(plugin.started_experiments), len(exp.nodes)
257
- )
258
- self.assertEqual(
259
- len(plugin.completed_experiments), len(exp.nodes)
260
- )
261
- self.assertEqual(
262
- len(plugin.started_example_ids), 6 * 10
263
- )
264
- self.assertEqual(
265
- len(plugin.completed_example_ids), 6 * 10
266
- )
267
- self.assert_same_list(plugin.skipped_experiments, [])
268
- self.assert_same_list(plugin.skipped_example_ids, [])
269
- self.assertTrue(
270
- pg.io.path_exists(os.path.join(run.output_root, 'run.json'))
271
- )
272
-
273
- for node in exp.nodes:
274
- self.assertTrue(node.progress.is_started)
275
- self.assertTrue(node.progress.is_completed)
276
- if node.is_leaf:
277
- self.assertEqual(node.progress.num_skipped, 0)
278
- self.assertEqual(node.progress.num_completed, 10)
279
- self.assertEqual(node.progress.num_failed, 1)
280
- else:
281
- self.assertEqual(node.progress.num_skipped, 0)
282
- self.assertEqual(node.progress.num_failed, 0)
283
- self.assertEqual(node.progress.num_processed, node.progress.num_total)
284
-
285
- def test_concurrent_startup_delay(self):
286
- plugin = TestPlugin()
287
- exp = eval_test_helper.test_experiment()
288
- root_dir = os.path.join(
289
- tempfile.mkdtemp(), 'test_concurrent_startup_delay'
290
- )
291
- _ = exp.run(
292
- root_dir,
293
- runner='parallel',
294
- plugins=[plugin],
295
- concurrent_startup_delay=(0, 5),
296
- )
297
-
298
-
299
- class DebugRunnerTest(RunnerTest):
300
-
301
- def test_debug_runner(self):
302
- plugin = TestPlugin()
303
- exp = eval_test_helper.test_experiment()
304
- root_dir = os.path.join(tempfile.mkdtemp(), 'test_debug_runner')
305
- run = exp.run(root_dir, runner='debug', plugins=[plugin])
306
-
307
- self.assertIsNotNone(plugin.start_time)
308
- self.assertIsNotNone(plugin.complete_time)
309
- self.assertGreater(plugin.complete_time, plugin.start_time)
310
-
311
- self.assertEqual(
312
- len(plugin.started_experiments), len(exp.nodes)
313
- )
314
- self.assertEqual(
315
- len(plugin.completed_experiments), len(exp.nodes)
316
- )
317
- self.assertEqual(
318
- len(plugin.started_example_ids), 6 * 1
319
- )
320
- self.assertEqual(
321
- len(plugin.completed_example_ids), 6 * 1
322
- )
323
- self.assert_same_list(plugin.skipped_experiments, [])
324
- self.assert_same_list(plugin.skipped_example_ids, [])
325
- self.assertFalse(
326
- pg.io.path_exists(os.path.join(run.output_root, 'run.json'))
327
- )
328
-
329
- for node in exp.nodes:
330
- self.assertTrue(node.progress.is_started)
331
- self.assertTrue(node.progress.is_completed)
332
- if node.is_leaf:
333
- self.assertEqual(node.progress.num_skipped, 0)
334
- self.assertEqual(node.progress.num_completed, 1)
335
- self.assertEqual(node.progress.num_failed, 0)
336
- else:
337
- self.assertEqual(node.progress.num_skipped, 0)
338
- self.assertEqual(node.progress.num_failed, 0)
339
- self.assertEqual(node.progress.num_processed, node.progress.num_total)
340
-
341
-
342
- if __name__ == '__main__':
343
- unittest.main()