ag2 0.9.1__py3-none-any.whl → 0.9.1.post0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ag2 might be problematic. Click here for more details.
- {ag2-0.9.1.dist-info → ag2-0.9.1.post0.dist-info}/METADATA +264 -73
- ag2-0.9.1.post0.dist-info/RECORD +392 -0
- {ag2-0.9.1.dist-info → ag2-0.9.1.post0.dist-info}/WHEEL +1 -2
- autogen/__init__.py +89 -0
- autogen/_website/__init__.py +3 -0
- autogen/_website/generate_api_references.py +427 -0
- autogen/_website/generate_mkdocs.py +1174 -0
- autogen/_website/notebook_processor.py +476 -0
- autogen/_website/process_notebooks.py +656 -0
- autogen/_website/utils.py +412 -0
- autogen/agentchat/__init__.py +44 -0
- autogen/agentchat/agent.py +182 -0
- autogen/agentchat/assistant_agent.py +85 -0
- autogen/agentchat/chat.py +309 -0
- autogen/agentchat/contrib/__init__.py +5 -0
- autogen/agentchat/contrib/agent_eval/README.md +7 -0
- autogen/agentchat/contrib/agent_eval/agent_eval.py +108 -0
- autogen/agentchat/contrib/agent_eval/criterion.py +43 -0
- autogen/agentchat/contrib/agent_eval/critic_agent.py +44 -0
- autogen/agentchat/contrib/agent_eval/quantifier_agent.py +39 -0
- autogen/agentchat/contrib/agent_eval/subcritic_agent.py +45 -0
- autogen/agentchat/contrib/agent_eval/task.py +42 -0
- autogen/agentchat/contrib/agent_optimizer.py +429 -0
- autogen/agentchat/contrib/capabilities/__init__.py +5 -0
- autogen/agentchat/contrib/capabilities/agent_capability.py +20 -0
- autogen/agentchat/contrib/capabilities/generate_images.py +301 -0
- autogen/agentchat/contrib/capabilities/teachability.py +393 -0
- autogen/agentchat/contrib/capabilities/text_compressors.py +66 -0
- autogen/agentchat/contrib/capabilities/tools_capability.py +22 -0
- autogen/agentchat/contrib/capabilities/transform_messages.py +93 -0
- autogen/agentchat/contrib/capabilities/transforms.py +566 -0
- autogen/agentchat/contrib/capabilities/transforms_util.py +122 -0
- autogen/agentchat/contrib/capabilities/vision_capability.py +214 -0
- autogen/agentchat/contrib/captainagent/__init__.py +9 -0
- autogen/agentchat/contrib/captainagent/agent_builder.py +790 -0
- autogen/agentchat/contrib/captainagent/captainagent.py +512 -0
- autogen/agentchat/contrib/captainagent/tool_retriever.py +335 -0
- autogen/agentchat/contrib/captainagent/tools/README.md +44 -0
- autogen/agentchat/contrib/captainagent/tools/__init__.py +5 -0
- autogen/agentchat/contrib/captainagent/tools/data_analysis/calculate_correlation.py +40 -0
- autogen/agentchat/contrib/captainagent/tools/data_analysis/calculate_skewness_and_kurtosis.py +28 -0
- autogen/agentchat/contrib/captainagent/tools/data_analysis/detect_outlier_iqr.py +28 -0
- autogen/agentchat/contrib/captainagent/tools/data_analysis/detect_outlier_zscore.py +28 -0
- autogen/agentchat/contrib/captainagent/tools/data_analysis/explore_csv.py +21 -0
- autogen/agentchat/contrib/captainagent/tools/data_analysis/shapiro_wilk_test.py +30 -0
- autogen/agentchat/contrib/captainagent/tools/information_retrieval/arxiv_download.py +27 -0
- autogen/agentchat/contrib/captainagent/tools/information_retrieval/arxiv_search.py +53 -0
- autogen/agentchat/contrib/captainagent/tools/information_retrieval/extract_pdf_image.py +53 -0
- autogen/agentchat/contrib/captainagent/tools/information_retrieval/extract_pdf_text.py +38 -0
- autogen/agentchat/contrib/captainagent/tools/information_retrieval/get_wikipedia_text.py +21 -0
- autogen/agentchat/contrib/captainagent/tools/information_retrieval/get_youtube_caption.py +34 -0
- autogen/agentchat/contrib/captainagent/tools/information_retrieval/image_qa.py +60 -0
- autogen/agentchat/contrib/captainagent/tools/information_retrieval/optical_character_recognition.py +61 -0
- autogen/agentchat/contrib/captainagent/tools/information_retrieval/perform_web_search.py +47 -0
- autogen/agentchat/contrib/captainagent/tools/information_retrieval/scrape_wikipedia_tables.py +33 -0
- autogen/agentchat/contrib/captainagent/tools/information_retrieval/transcribe_audio_file.py +21 -0
- autogen/agentchat/contrib/captainagent/tools/information_retrieval/youtube_download.py +35 -0
- autogen/agentchat/contrib/captainagent/tools/math/calculate_circle_area_from_diameter.py +21 -0
- autogen/agentchat/contrib/captainagent/tools/math/calculate_day_of_the_week.py +18 -0
- autogen/agentchat/contrib/captainagent/tools/math/calculate_fraction_sum.py +28 -0
- autogen/agentchat/contrib/captainagent/tools/math/calculate_matrix_power.py +31 -0
- autogen/agentchat/contrib/captainagent/tools/math/calculate_reflected_point.py +16 -0
- autogen/agentchat/contrib/captainagent/tools/math/complex_numbers_product.py +25 -0
- autogen/agentchat/contrib/captainagent/tools/math/compute_currency_conversion.py +23 -0
- autogen/agentchat/contrib/captainagent/tools/math/count_distinct_permutations.py +27 -0
- autogen/agentchat/contrib/captainagent/tools/math/evaluate_expression.py +28 -0
- autogen/agentchat/contrib/captainagent/tools/math/find_continuity_point.py +34 -0
- autogen/agentchat/contrib/captainagent/tools/math/fraction_to_mixed_numbers.py +39 -0
- autogen/agentchat/contrib/captainagent/tools/math/modular_inverse_sum.py +23 -0
- autogen/agentchat/contrib/captainagent/tools/math/simplify_mixed_numbers.py +36 -0
- autogen/agentchat/contrib/captainagent/tools/math/sum_of_digit_factorials.py +15 -0
- autogen/agentchat/contrib/captainagent/tools/math/sum_of_primes_below.py +15 -0
- autogen/agentchat/contrib/captainagent/tools/requirements.txt +10 -0
- autogen/agentchat/contrib/captainagent/tools/tool_description.tsv +34 -0
- autogen/agentchat/contrib/gpt_assistant_agent.py +526 -0
- autogen/agentchat/contrib/graph_rag/__init__.py +9 -0
- autogen/agentchat/contrib/graph_rag/document.py +29 -0
- autogen/agentchat/contrib/graph_rag/falkor_graph_query_engine.py +170 -0
- autogen/agentchat/contrib/graph_rag/falkor_graph_rag_capability.py +103 -0
- autogen/agentchat/contrib/graph_rag/graph_query_engine.py +53 -0
- autogen/agentchat/contrib/graph_rag/graph_rag_capability.py +63 -0
- autogen/agentchat/contrib/graph_rag/neo4j_graph_query_engine.py +268 -0
- autogen/agentchat/contrib/graph_rag/neo4j_graph_rag_capability.py +83 -0
- autogen/agentchat/contrib/graph_rag/neo4j_native_graph_query_engine.py +210 -0
- autogen/agentchat/contrib/graph_rag/neo4j_native_graph_rag_capability.py +93 -0
- autogen/agentchat/contrib/img_utils.py +397 -0
- autogen/agentchat/contrib/llamaindex_conversable_agent.py +117 -0
- autogen/agentchat/contrib/llava_agent.py +187 -0
- autogen/agentchat/contrib/math_user_proxy_agent.py +464 -0
- autogen/agentchat/contrib/multimodal_conversable_agent.py +125 -0
- autogen/agentchat/contrib/qdrant_retrieve_user_proxy_agent.py +324 -0
- autogen/agentchat/contrib/rag/__init__.py +10 -0
- autogen/agentchat/contrib/rag/chromadb_query_engine.py +272 -0
- autogen/agentchat/contrib/rag/llamaindex_query_engine.py +198 -0
- autogen/agentchat/contrib/rag/mongodb_query_engine.py +329 -0
- autogen/agentchat/contrib/rag/query_engine.py +74 -0
- autogen/agentchat/contrib/retrieve_assistant_agent.py +56 -0
- autogen/agentchat/contrib/retrieve_user_proxy_agent.py +703 -0
- autogen/agentchat/contrib/society_of_mind_agent.py +199 -0
- autogen/agentchat/contrib/swarm_agent.py +1425 -0
- autogen/agentchat/contrib/text_analyzer_agent.py +79 -0
- autogen/agentchat/contrib/vectordb/__init__.py +5 -0
- autogen/agentchat/contrib/vectordb/base.py +232 -0
- autogen/agentchat/contrib/vectordb/chromadb.py +315 -0
- autogen/agentchat/contrib/vectordb/couchbase.py +407 -0
- autogen/agentchat/contrib/vectordb/mongodb.py +550 -0
- autogen/agentchat/contrib/vectordb/pgvectordb.py +928 -0
- autogen/agentchat/contrib/vectordb/qdrant.py +320 -0
- autogen/agentchat/contrib/vectordb/utils.py +126 -0
- autogen/agentchat/contrib/web_surfer.py +303 -0
- autogen/agentchat/conversable_agent.py +4020 -0
- autogen/agentchat/group/__init__.py +64 -0
- autogen/agentchat/group/available_condition.py +91 -0
- autogen/agentchat/group/context_condition.py +77 -0
- autogen/agentchat/group/context_expression.py +238 -0
- autogen/agentchat/group/context_str.py +41 -0
- autogen/agentchat/group/context_variables.py +192 -0
- autogen/agentchat/group/group_tool_executor.py +202 -0
- autogen/agentchat/group/group_utils.py +591 -0
- autogen/agentchat/group/handoffs.py +244 -0
- autogen/agentchat/group/llm_condition.py +93 -0
- autogen/agentchat/group/multi_agent_chat.py +237 -0
- autogen/agentchat/group/on_condition.py +58 -0
- autogen/agentchat/group/on_context_condition.py +54 -0
- autogen/agentchat/group/patterns/__init__.py +18 -0
- autogen/agentchat/group/patterns/auto.py +159 -0
- autogen/agentchat/group/patterns/manual.py +176 -0
- autogen/agentchat/group/patterns/pattern.py +288 -0
- autogen/agentchat/group/patterns/random.py +106 -0
- autogen/agentchat/group/patterns/round_robin.py +117 -0
- autogen/agentchat/group/reply_result.py +26 -0
- autogen/agentchat/group/speaker_selection_result.py +41 -0
- autogen/agentchat/group/targets/__init__.py +4 -0
- autogen/agentchat/group/targets/group_chat_target.py +132 -0
- autogen/agentchat/group/targets/group_manager_target.py +151 -0
- autogen/agentchat/group/targets/transition_target.py +413 -0
- autogen/agentchat/group/targets/transition_utils.py +6 -0
- autogen/agentchat/groupchat.py +1694 -0
- autogen/agentchat/realtime/__init__.py +3 -0
- autogen/agentchat/realtime/experimental/__init__.py +20 -0
- autogen/agentchat/realtime/experimental/audio_adapters/__init__.py +8 -0
- autogen/agentchat/realtime/experimental/audio_adapters/twilio_audio_adapter.py +148 -0
- autogen/agentchat/realtime/experimental/audio_adapters/websocket_audio_adapter.py +139 -0
- autogen/agentchat/realtime/experimental/audio_observer.py +42 -0
- autogen/agentchat/realtime/experimental/clients/__init__.py +15 -0
- autogen/agentchat/realtime/experimental/clients/gemini/__init__.py +7 -0
- autogen/agentchat/realtime/experimental/clients/gemini/client.py +274 -0
- autogen/agentchat/realtime/experimental/clients/oai/__init__.py +8 -0
- autogen/agentchat/realtime/experimental/clients/oai/base_client.py +220 -0
- autogen/agentchat/realtime/experimental/clients/oai/rtc_client.py +243 -0
- autogen/agentchat/realtime/experimental/clients/oai/utils.py +48 -0
- autogen/agentchat/realtime/experimental/clients/realtime_client.py +190 -0
- autogen/agentchat/realtime/experimental/function_observer.py +85 -0
- autogen/agentchat/realtime/experimental/realtime_agent.py +158 -0
- autogen/agentchat/realtime/experimental/realtime_events.py +42 -0
- autogen/agentchat/realtime/experimental/realtime_observer.py +100 -0
- autogen/agentchat/realtime/experimental/realtime_swarm.py +475 -0
- autogen/agentchat/realtime/experimental/websockets.py +21 -0
- autogen/agentchat/realtime_agent/__init__.py +21 -0
- autogen/agentchat/user_proxy_agent.py +111 -0
- autogen/agentchat/utils.py +206 -0
- autogen/agents/__init__.py +3 -0
- autogen/agents/contrib/__init__.py +10 -0
- autogen/agents/contrib/time/__init__.py +8 -0
- autogen/agents/contrib/time/time_reply_agent.py +73 -0
- autogen/agents/contrib/time/time_tool_agent.py +51 -0
- autogen/agents/experimental/__init__.py +27 -0
- autogen/agents/experimental/deep_research/__init__.py +7 -0
- autogen/agents/experimental/deep_research/deep_research.py +52 -0
- autogen/agents/experimental/discord/__init__.py +7 -0
- autogen/agents/experimental/discord/discord.py +66 -0
- autogen/agents/experimental/document_agent/__init__.py +19 -0
- autogen/agents/experimental/document_agent/chroma_query_engine.py +316 -0
- autogen/agents/experimental/document_agent/docling_doc_ingest_agent.py +118 -0
- autogen/agents/experimental/document_agent/document_agent.py +461 -0
- autogen/agents/experimental/document_agent/document_conditions.py +50 -0
- autogen/agents/experimental/document_agent/document_utils.py +380 -0
- autogen/agents/experimental/document_agent/inmemory_query_engine.py +220 -0
- autogen/agents/experimental/document_agent/parser_utils.py +130 -0
- autogen/agents/experimental/document_agent/url_utils.py +426 -0
- autogen/agents/experimental/reasoning/__init__.py +7 -0
- autogen/agents/experimental/reasoning/reasoning_agent.py +1178 -0
- autogen/agents/experimental/slack/__init__.py +7 -0
- autogen/agents/experimental/slack/slack.py +73 -0
- autogen/agents/experimental/telegram/__init__.py +7 -0
- autogen/agents/experimental/telegram/telegram.py +77 -0
- autogen/agents/experimental/websurfer/__init__.py +7 -0
- autogen/agents/experimental/websurfer/websurfer.py +62 -0
- autogen/agents/experimental/wikipedia/__init__.py +7 -0
- autogen/agents/experimental/wikipedia/wikipedia.py +90 -0
- autogen/browser_utils.py +309 -0
- autogen/cache/__init__.py +10 -0
- autogen/cache/abstract_cache_base.py +75 -0
- autogen/cache/cache.py +203 -0
- autogen/cache/cache_factory.py +88 -0
- autogen/cache/cosmos_db_cache.py +144 -0
- autogen/cache/disk_cache.py +102 -0
- autogen/cache/in_memory_cache.py +58 -0
- autogen/cache/redis_cache.py +123 -0
- autogen/code_utils.py +596 -0
- autogen/coding/__init__.py +22 -0
- autogen/coding/base.py +119 -0
- autogen/coding/docker_commandline_code_executor.py +268 -0
- autogen/coding/factory.py +47 -0
- autogen/coding/func_with_reqs.py +202 -0
- autogen/coding/jupyter/__init__.py +23 -0
- autogen/coding/jupyter/base.py +36 -0
- autogen/coding/jupyter/docker_jupyter_server.py +167 -0
- autogen/coding/jupyter/embedded_ipython_code_executor.py +182 -0
- autogen/coding/jupyter/import_utils.py +82 -0
- autogen/coding/jupyter/jupyter_client.py +231 -0
- autogen/coding/jupyter/jupyter_code_executor.py +160 -0
- autogen/coding/jupyter/local_jupyter_server.py +172 -0
- autogen/coding/local_commandline_code_executor.py +405 -0
- autogen/coding/markdown_code_extractor.py +45 -0
- autogen/coding/utils.py +56 -0
- autogen/doc_utils.py +34 -0
- autogen/events/__init__.py +7 -0
- autogen/events/agent_events.py +1010 -0
- autogen/events/base_event.py +99 -0
- autogen/events/client_events.py +167 -0
- autogen/events/helpers.py +36 -0
- autogen/events/print_event.py +46 -0
- autogen/exception_utils.py +73 -0
- autogen/extensions/__init__.py +5 -0
- autogen/fast_depends/__init__.py +16 -0
- autogen/fast_depends/_compat.py +80 -0
- autogen/fast_depends/core/__init__.py +14 -0
- autogen/fast_depends/core/build.py +225 -0
- autogen/fast_depends/core/model.py +576 -0
- autogen/fast_depends/dependencies/__init__.py +15 -0
- autogen/fast_depends/dependencies/model.py +29 -0
- autogen/fast_depends/dependencies/provider.py +39 -0
- autogen/fast_depends/library/__init__.py +10 -0
- autogen/fast_depends/library/model.py +46 -0
- autogen/fast_depends/py.typed +6 -0
- autogen/fast_depends/schema.py +66 -0
- autogen/fast_depends/use.py +280 -0
- autogen/fast_depends/utils.py +187 -0
- autogen/formatting_utils.py +83 -0
- autogen/function_utils.py +13 -0
- autogen/graph_utils.py +178 -0
- autogen/import_utils.py +526 -0
- autogen/interop/__init__.py +22 -0
- autogen/interop/crewai/__init__.py +7 -0
- autogen/interop/crewai/crewai.py +88 -0
- autogen/interop/interoperability.py +71 -0
- autogen/interop/interoperable.py +46 -0
- autogen/interop/langchain/__init__.py +8 -0
- autogen/interop/langchain/langchain_chat_model_factory.py +155 -0
- autogen/interop/langchain/langchain_tool.py +82 -0
- autogen/interop/litellm/__init__.py +7 -0
- autogen/interop/litellm/litellm_config_factory.py +113 -0
- autogen/interop/pydantic_ai/__init__.py +7 -0
- autogen/interop/pydantic_ai/pydantic_ai.py +168 -0
- autogen/interop/registry.py +69 -0
- autogen/io/__init__.py +15 -0
- autogen/io/base.py +151 -0
- autogen/io/console.py +56 -0
- autogen/io/processors/__init__.py +12 -0
- autogen/io/processors/base.py +21 -0
- autogen/io/processors/console_event_processor.py +56 -0
- autogen/io/run_response.py +293 -0
- autogen/io/thread_io_stream.py +63 -0
- autogen/io/websockets.py +213 -0
- autogen/json_utils.py +43 -0
- autogen/llm_config.py +379 -0
- autogen/logger/__init__.py +11 -0
- autogen/logger/base_logger.py +128 -0
- autogen/logger/file_logger.py +261 -0
- autogen/logger/logger_factory.py +42 -0
- autogen/logger/logger_utils.py +57 -0
- autogen/logger/sqlite_logger.py +523 -0
- autogen/math_utils.py +339 -0
- autogen/mcp/__init__.py +7 -0
- autogen/mcp/mcp_client.py +208 -0
- autogen/messages/__init__.py +7 -0
- autogen/messages/agent_messages.py +948 -0
- autogen/messages/base_message.py +107 -0
- autogen/messages/client_messages.py +171 -0
- autogen/messages/print_message.py +49 -0
- autogen/oai/__init__.py +53 -0
- autogen/oai/anthropic.py +714 -0
- autogen/oai/bedrock.py +628 -0
- autogen/oai/cerebras.py +299 -0
- autogen/oai/client.py +1435 -0
- autogen/oai/client_utils.py +169 -0
- autogen/oai/cohere.py +479 -0
- autogen/oai/gemini.py +990 -0
- autogen/oai/gemini_types.py +129 -0
- autogen/oai/groq.py +305 -0
- autogen/oai/mistral.py +303 -0
- autogen/oai/oai_models/__init__.py +11 -0
- autogen/oai/oai_models/_models.py +16 -0
- autogen/oai/oai_models/chat_completion.py +87 -0
- autogen/oai/oai_models/chat_completion_audio.py +32 -0
- autogen/oai/oai_models/chat_completion_message.py +86 -0
- autogen/oai/oai_models/chat_completion_message_tool_call.py +37 -0
- autogen/oai/oai_models/chat_completion_token_logprob.py +63 -0
- autogen/oai/oai_models/completion_usage.py +60 -0
- autogen/oai/ollama.py +643 -0
- autogen/oai/openai_utils.py +881 -0
- autogen/oai/together.py +370 -0
- autogen/retrieve_utils.py +491 -0
- autogen/runtime_logging.py +160 -0
- autogen/token_count_utils.py +267 -0
- autogen/tools/__init__.py +20 -0
- autogen/tools/contrib/__init__.py +9 -0
- autogen/tools/contrib/time/__init__.py +7 -0
- autogen/tools/contrib/time/time.py +41 -0
- autogen/tools/dependency_injection.py +254 -0
- autogen/tools/experimental/__init__.py +43 -0
- autogen/tools/experimental/browser_use/__init__.py +7 -0
- autogen/tools/experimental/browser_use/browser_use.py +161 -0
- autogen/tools/experimental/crawl4ai/__init__.py +7 -0
- autogen/tools/experimental/crawl4ai/crawl4ai.py +153 -0
- autogen/tools/experimental/deep_research/__init__.py +7 -0
- autogen/tools/experimental/deep_research/deep_research.py +328 -0
- autogen/tools/experimental/duckduckgo/__init__.py +7 -0
- autogen/tools/experimental/duckduckgo/duckduckgo_search.py +109 -0
- autogen/tools/experimental/google/__init__.py +14 -0
- autogen/tools/experimental/google/authentication/__init__.py +11 -0
- autogen/tools/experimental/google/authentication/credentials_hosted_provider.py +43 -0
- autogen/tools/experimental/google/authentication/credentials_local_provider.py +91 -0
- autogen/tools/experimental/google/authentication/credentials_provider.py +35 -0
- autogen/tools/experimental/google/drive/__init__.py +9 -0
- autogen/tools/experimental/google/drive/drive_functions.py +124 -0
- autogen/tools/experimental/google/drive/toolkit.py +88 -0
- autogen/tools/experimental/google/model.py +17 -0
- autogen/tools/experimental/google/toolkit_protocol.py +19 -0
- autogen/tools/experimental/google_search/__init__.py +8 -0
- autogen/tools/experimental/google_search/google_search.py +93 -0
- autogen/tools/experimental/google_search/youtube_search.py +181 -0
- autogen/tools/experimental/messageplatform/__init__.py +17 -0
- autogen/tools/experimental/messageplatform/discord/__init__.py +7 -0
- autogen/tools/experimental/messageplatform/discord/discord.py +288 -0
- autogen/tools/experimental/messageplatform/slack/__init__.py +7 -0
- autogen/tools/experimental/messageplatform/slack/slack.py +391 -0
- autogen/tools/experimental/messageplatform/telegram/__init__.py +7 -0
- autogen/tools/experimental/messageplatform/telegram/telegram.py +275 -0
- autogen/tools/experimental/perplexity/__init__.py +7 -0
- autogen/tools/experimental/perplexity/perplexity_search.py +260 -0
- autogen/tools/experimental/tavily/__init__.py +7 -0
- autogen/tools/experimental/tavily/tavily_search.py +183 -0
- autogen/tools/experimental/web_search_preview/__init__.py +7 -0
- autogen/tools/experimental/web_search_preview/web_search_preview.py +114 -0
- autogen/tools/experimental/wikipedia/__init__.py +7 -0
- autogen/tools/experimental/wikipedia/wikipedia.py +287 -0
- autogen/tools/function_utils.py +411 -0
- autogen/tools/tool.py +187 -0
- autogen/tools/toolkit.py +86 -0
- autogen/types.py +29 -0
- autogen/version.py +7 -0
- ag2-0.9.1.dist-info/RECORD +0 -6
- ag2-0.9.1.dist-info/top_level.txt +0 -1
- {ag2-0.9.1.dist-info → ag2-0.9.1.post0.dist-info/licenses}/LICENSE +0 -0
- {ag2-0.9.1.dist-info → ag2-0.9.1.post0.dist-info/licenses}/NOTICE.md +0 -0
|
@@ -0,0 +1,656 @@
|
|
|
1
|
+
# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
#
|
|
5
|
+
# Portions derived from https://github.com/microsoft/autogen are under the MIT License.
|
|
6
|
+
# SPDX-License-Identifier: MIT
|
|
7
|
+
# !/usr/bin/env python
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
import shutil
|
|
14
|
+
import sys
|
|
15
|
+
from copy import deepcopy
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Sequence, Union
|
|
18
|
+
|
|
19
|
+
from ..import_utils import optional_import_block, require_optional_import
|
|
20
|
+
from .notebook_processor import (
|
|
21
|
+
create_base_argument_parser,
|
|
22
|
+
process_notebooks_core,
|
|
23
|
+
)
|
|
24
|
+
from .utils import (
|
|
25
|
+
NavigationGroup,
|
|
26
|
+
add_authors_and_social_preview,
|
|
27
|
+
ensure_edit_url,
|
|
28
|
+
get_authors_info,
|
|
29
|
+
remove_marker_blocks,
|
|
30
|
+
sort_files_by_date,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
with optional_import_block():
|
|
34
|
+
import yaml
|
|
35
|
+
from jinja2 import Template
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def notebooks_target_dir(website_build_directory: Path) -> Path:
|
|
39
|
+
"""Return the target directory for notebooks."""
|
|
40
|
+
return website_build_directory / "docs" / "use-cases" / "notebooks" / "notebooks"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def add_front_matter_to_metadata_mdx(
|
|
44
|
+
front_matter: dict[str, Union[str, list[str], None]], website_build_directory: Path, rendered_mdx: Path
|
|
45
|
+
) -> None:
|
|
46
|
+
source = front_matter.get("source_notebook")
|
|
47
|
+
if isinstance(source, str) and source.startswith("/website/docs/"):
|
|
48
|
+
return
|
|
49
|
+
|
|
50
|
+
metadata_mdx = website_build_directory / "snippets" / "data" / "NotebooksMetadata.mdx"
|
|
51
|
+
|
|
52
|
+
if not metadata_mdx.exists():
|
|
53
|
+
with open(metadata_mdx, "w", encoding="utf-8") as f:
|
|
54
|
+
f.write(
|
|
55
|
+
"{/*\nAuto-generated file - DO NOT EDIT\nPlease edit the add_front_matter_to_metadata_mdx function in process_notebooks.py\n*/}\n\n"
|
|
56
|
+
)
|
|
57
|
+
f.write("export const notebooksMetadata = [];\n")
|
|
58
|
+
|
|
59
|
+
metadata = []
|
|
60
|
+
with open(metadata_mdx, encoding="utf-8") as f:
|
|
61
|
+
content = f.read()
|
|
62
|
+
if content:
|
|
63
|
+
start = content.find("export const notebooksMetadata = [")
|
|
64
|
+
end = content.rfind("]")
|
|
65
|
+
if start != -1 and end != -1:
|
|
66
|
+
metadata = json.loads(content[start + 32 : end + 1])
|
|
67
|
+
|
|
68
|
+
# Create new entry for current notebook
|
|
69
|
+
entry = {
|
|
70
|
+
"title": front_matter.get("title", ""),
|
|
71
|
+
"link": f"/docs/use-cases/notebooks/notebooks/{rendered_mdx.stem}",
|
|
72
|
+
"description": front_matter.get("description", ""),
|
|
73
|
+
"image": front_matter.get("image"),
|
|
74
|
+
"tags": front_matter.get("tags", []),
|
|
75
|
+
"source": source,
|
|
76
|
+
}
|
|
77
|
+
# Update metadata list
|
|
78
|
+
existing_entry = next((item for item in metadata if item["link"] == entry["link"]), None)
|
|
79
|
+
if existing_entry:
|
|
80
|
+
metadata[metadata.index(existing_entry)] = entry
|
|
81
|
+
else:
|
|
82
|
+
metadata.append(entry)
|
|
83
|
+
|
|
84
|
+
# Write metadata back to file
|
|
85
|
+
with open(metadata_mdx, "w", encoding="utf-8") as f:
|
|
86
|
+
f.write(
|
|
87
|
+
"{/*\nAuto-generated file - DO NOT EDIT\nPlease edit the add_front_matter_to_metadata_mdx function in process_notebooks.py\n*/}\n\n"
|
|
88
|
+
)
|
|
89
|
+
f.write("export const notebooksMetadata = ")
|
|
90
|
+
f.write(json.dumps(metadata, indent=4))
|
|
91
|
+
f.write(";\n")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def convert_callout_blocks(content: str) -> str:
|
|
95
|
+
"""Converts callout blocks in the following formats:
|
|
96
|
+
1) Plain callout blocks using ::: syntax.
|
|
97
|
+
2) Blocks using 3-4 backticks + (mdx-code-block or {=mdx}) + ::: syntax.
|
|
98
|
+
Transforms them into custom HTML/component syntax.
|
|
99
|
+
"""
|
|
100
|
+
callout_types = {
|
|
101
|
+
"tip": "Tip",
|
|
102
|
+
"note": "Note",
|
|
103
|
+
"warning": "Warning",
|
|
104
|
+
"info": "Info",
|
|
105
|
+
"info Requirements": "Info",
|
|
106
|
+
"check": "Check",
|
|
107
|
+
"danger": "Warning",
|
|
108
|
+
"tabs": "Tabs",
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
# Regex explanation (using alternation):
|
|
112
|
+
#
|
|
113
|
+
# -- Alternative #1: Backticks + mdx-code-block/{=mdx} --
|
|
114
|
+
#
|
|
115
|
+
# ^(?P<backticks>`{3,4})(?:mdx-code-block|\{=mdx\})[ \t]*\n
|
|
116
|
+
# - Matches opening backticks and optional mdx markers.
|
|
117
|
+
# :::(?P<callout_type_backtick>...)
|
|
118
|
+
# - Captures the callout type.
|
|
119
|
+
# (.*?)
|
|
120
|
+
# - Captures the content inside the callout.
|
|
121
|
+
# ^:::[ \t]*\n
|
|
122
|
+
# - Matches the closing ::: line.
|
|
123
|
+
# (?P=backticks)
|
|
124
|
+
# - Ensures the same number of backticks close the block.
|
|
125
|
+
#
|
|
126
|
+
# -- Alternative #2: Plain ::: callout --
|
|
127
|
+
#
|
|
128
|
+
# ^:::(?P<callout_type_no_backtick>...)
|
|
129
|
+
# - Captures the callout type after :::.
|
|
130
|
+
# (.*?)
|
|
131
|
+
# - Captures the content inside the callout.
|
|
132
|
+
# ^:::
|
|
133
|
+
# - Matches the closing ::: line.
|
|
134
|
+
#
|
|
135
|
+
# (?s)(?m): DOTALL + MULTILINE flags.
|
|
136
|
+
# - DOTALL (`.` matches everything, including newlines).
|
|
137
|
+
# - MULTILINE (`^` and `$` work at the start/end of each line).
|
|
138
|
+
|
|
139
|
+
pattern = re.compile(
|
|
140
|
+
r"(?s)(?m)"
|
|
141
|
+
r"(?:"
|
|
142
|
+
# Alternative #1: Backticks + mdx-code-block/{=mdx}
|
|
143
|
+
r"^(?P<backticks>`{3,4})(?:mdx-code-block|\{=mdx\})[ \t]*\n"
|
|
144
|
+
r":::(?P<callout_type_backtick>\w+(?:\s+\w+)?)[ \t]*\n"
|
|
145
|
+
r"(?P<inner_backtick>.*?)"
|
|
146
|
+
r"^:::[ \t]*\n"
|
|
147
|
+
r"(?P=backticks)" # Closing backticks must match the opening count.
|
|
148
|
+
r")"
|
|
149
|
+
r"|"
|
|
150
|
+
# Alternative #2: Plain ::: callout
|
|
151
|
+
r"(?:"
|
|
152
|
+
r"^:::(?P<callout_type_no_backtick>\w+(?:\s+\w+)?)[ \t]*\n"
|
|
153
|
+
r"(?P<inner_no_backtick>.*?)"
|
|
154
|
+
r"^:::[ \t]*(?:\n|$)"
|
|
155
|
+
r")"
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
def replace_callout(m: re.Match[str]) -> str:
|
|
159
|
+
# Determine the matched alternative and extract the corresponding groups.
|
|
160
|
+
ctype = m.group("callout_type_backtick") or m.group("callout_type_no_backtick")
|
|
161
|
+
inner = m.group("inner_backtick") or m.group("inner_no_backtick") or ""
|
|
162
|
+
|
|
163
|
+
# Map the callout type to its standard representation or fallback to the original type.
|
|
164
|
+
mapped_type = callout_types.get(ctype, ctype)
|
|
165
|
+
|
|
166
|
+
# Return the formatted HTML block.
|
|
167
|
+
return f"""
|
|
168
|
+
<div class="{ctype}">
|
|
169
|
+
<{mapped_type}>
|
|
170
|
+
{inner.strip()}
|
|
171
|
+
</{mapped_type}>
|
|
172
|
+
</div>
|
|
173
|
+
"""
|
|
174
|
+
|
|
175
|
+
# Apply the regex pattern and replace matched callouts with the custom HTML structure.
|
|
176
|
+
return pattern.sub(replace_callout, content)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def convert_mdx_image_blocks(content: str, rendered_mdx: Path, website_build_directory: Path) -> str:
|
|
180
|
+
"""Converts MDX code block image syntax to regular markdown image syntax.
|
|
181
|
+
|
|
182
|
+
Args:
|
|
183
|
+
content (str): The markdown content containing mdx-code-block image syntax
|
|
184
|
+
rendered_mdx (Path): The path to the rendered mdx file
|
|
185
|
+
website_build_directory (Path): The path to the website build directory
|
|
186
|
+
|
|
187
|
+
Returns:
|
|
188
|
+
str: The converted markdown content with standard image syntax
|
|
189
|
+
"""
|
|
190
|
+
|
|
191
|
+
def resolve_path(match: re.Match[str]) -> str:
|
|
192
|
+
img_pattern = r"!\[(.*?)\]\((.*?)\)"
|
|
193
|
+
img_match = re.search(img_pattern, match.group(1))
|
|
194
|
+
if not img_match:
|
|
195
|
+
return match.group(0)
|
|
196
|
+
|
|
197
|
+
alt, rel_path = img_match.groups()
|
|
198
|
+
abs_path = (rendered_mdx.parent / Path(rel_path)).resolve().relative_to(website_build_directory)
|
|
199
|
+
return f""
|
|
200
|
+
|
|
201
|
+
pattern = r"````mdx-code-block\n(!\[.*?\]\(.*?\))\n````"
|
|
202
|
+
return re.sub(pattern, resolve_path, content)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def extract_img_tag_from_figure_tag(content: str, img_rel_path: Path) -> str:
|
|
206
|
+
"""Extracts the img tag from the figure tag and modifies local image path.
|
|
207
|
+
|
|
208
|
+
Quarto converts the markdown images syntax to <figure> tag while rendering and converting the notebook to mdx file.
|
|
209
|
+
Mintlify could not able to process the <figure> tags properly and does not shows these images in the documentation.
|
|
210
|
+
As a fix, the <img> tag present inside the <figure> is extracted and saved to the mdx file.
|
|
211
|
+
|
|
212
|
+
Args:
|
|
213
|
+
content (str): Content of the file
|
|
214
|
+
img_rel_path (Path): Relative path to the image directory
|
|
215
|
+
|
|
216
|
+
Returns:
|
|
217
|
+
str: Content of the file with <img> tag extracted from <figure> tag
|
|
218
|
+
"""
|
|
219
|
+
|
|
220
|
+
def replace_local_path(match: re.Match[str]) -> str:
|
|
221
|
+
img_tag = match.group(1)
|
|
222
|
+
# Find src attribute value
|
|
223
|
+
src_match = re.search(r'src="([^"]+)"', img_tag)
|
|
224
|
+
if src_match:
|
|
225
|
+
src = src_match.group(1)
|
|
226
|
+
# If src doesn't start with http/https, it's a local image
|
|
227
|
+
if not src.startswith(("http://", "https://")):
|
|
228
|
+
# Replace old src with new prefixed path
|
|
229
|
+
img_tag = img_tag.replace(f'src="{src}"', f'src="/{str(img_rel_path.as_posix())}/{src}"')
|
|
230
|
+
return img_tag
|
|
231
|
+
|
|
232
|
+
pattern = r"<figure>\s*(<img\s+.*?/>)\s*<figcaption[^>]*>.*?</figcaption>\s*</figure>"
|
|
233
|
+
content = re.sub(pattern, replace_local_path, content, flags=re.DOTALL)
|
|
234
|
+
return content
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
# rendered_notebook is the final mdx file
|
|
238
|
+
@require_optional_import("yaml", "docs")
|
|
239
|
+
def post_process_mdx(
|
|
240
|
+
rendered_mdx: Path,
|
|
241
|
+
source_notebooks: Path,
|
|
242
|
+
front_matter: dict[str, Union[str, list[str], None]],
|
|
243
|
+
website_build_directory: Path,
|
|
244
|
+
) -> None:
|
|
245
|
+
with open(rendered_mdx, encoding="utf-8") as f:
|
|
246
|
+
content = f.read()
|
|
247
|
+
|
|
248
|
+
# If there is front matter in the mdx file, we need to remove it
|
|
249
|
+
if content.startswith("---"):
|
|
250
|
+
front_matter_end = content.find("---", 3)
|
|
251
|
+
mdx_front_matter = yaml.safe_load(content[4:front_matter_end])
|
|
252
|
+
# Merge while preserving original values
|
|
253
|
+
front_matter = {**front_matter, **mdx_front_matter}
|
|
254
|
+
content = content[front_matter_end + 3 :]
|
|
255
|
+
|
|
256
|
+
# Clean heading IDs using regex - matches from # to the end of ID block
|
|
257
|
+
content = re.sub(r"(#{1,6}[^{]+){#[^}]+}", r"\1", content)
|
|
258
|
+
|
|
259
|
+
# Each intermediate path needs to be resolved for this to work reliably
|
|
260
|
+
repo_root = Path(__file__).resolve().parents[2]
|
|
261
|
+
repo_relative_notebook = source_notebooks.resolve().relative_to(repo_root)
|
|
262
|
+
front_matter["source_notebook"] = f"/{repo_relative_notebook}"
|
|
263
|
+
front_matter["custom_edit_url"] = f"https://github.com/ag2ai/ag2/edit/main/{repo_relative_notebook}"
|
|
264
|
+
|
|
265
|
+
# Is there a title on the content? Only search up until the first code cell
|
|
266
|
+
# first_code_cell = content.find("```")
|
|
267
|
+
# if first_code_cell != -1:
|
|
268
|
+
# title_search_content = content[:first_code_cell]
|
|
269
|
+
# else:
|
|
270
|
+
# title_search_content = content
|
|
271
|
+
|
|
272
|
+
# title_exists = title_search_content.find("\n# ") != -1
|
|
273
|
+
# if not title_exists:
|
|
274
|
+
# content = f"# {front_matter['title']}\n{content}"
|
|
275
|
+
# inject in content directly after the markdown title the word done
|
|
276
|
+
# Find the end of the line with the title
|
|
277
|
+
# title_end = content.find("\n", content.find("#"))
|
|
278
|
+
|
|
279
|
+
# Extract page title
|
|
280
|
+
# title = content[content.find("#") + 1 : content.find("\n", content.find("#"))].strip()
|
|
281
|
+
# If there is a { in the title we trim off the { and everything after it
|
|
282
|
+
# if "{" in title:
|
|
283
|
+
# title = title[: title.find("{")].strip()
|
|
284
|
+
|
|
285
|
+
github_link = f"https://github.com/ag2ai/ag2/blob/main/{repo_relative_notebook}"
|
|
286
|
+
content = (
|
|
287
|
+
f'\n<a href="{github_link}" class="github-badge" target="_blank">'
|
|
288
|
+
+ """<img noZoom src="https://img.shields.io/badge/Open%20on%20GitHub-grey?logo=github" alt="Open on GitHub" />"""
|
|
289
|
+
+ "</a>"
|
|
290
|
+
+ content
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
# If no colab link is present, insert one
|
|
294
|
+
if "colab-badge.svg" not in content:
|
|
295
|
+
colab_link = f"https://colab.research.google.com/github/ag2ai/ag2/blob/main/{repo_relative_notebook}"
|
|
296
|
+
content = (
|
|
297
|
+
f'\n<a href="{colab_link}" class="colab-badge" target="_blank">'
|
|
298
|
+
+ """<img noZoom src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab" />"""
|
|
299
|
+
+ "</a>"
|
|
300
|
+
+ content
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
# Create the front matter metadata js file for examples by notebook section
|
|
304
|
+
add_front_matter_to_metadata_mdx(front_matter, website_build_directory, rendered_mdx)
|
|
305
|
+
|
|
306
|
+
# Dump front_matter to yaml
|
|
307
|
+
front_matter_str = yaml.dump(front_matter, default_flow_style=False)
|
|
308
|
+
|
|
309
|
+
# Convert callout blocks
|
|
310
|
+
content = convert_callout_blocks(content)
|
|
311
|
+
|
|
312
|
+
# Convert mdx image syntax to mintly image syntax
|
|
313
|
+
content = convert_mdx_image_blocks(content, rendered_mdx, website_build_directory)
|
|
314
|
+
|
|
315
|
+
# ensure editUrl is present
|
|
316
|
+
content = ensure_edit_url(content, repo_relative_notebook)
|
|
317
|
+
|
|
318
|
+
# convert figure tag to img tag
|
|
319
|
+
img_rel_path = rendered_mdx.parent.relative_to(website_build_directory)
|
|
320
|
+
content = extract_img_tag_from_figure_tag(content, img_rel_path)
|
|
321
|
+
|
|
322
|
+
# Rewrite the content as
|
|
323
|
+
# ---
|
|
324
|
+
# front_matter_str
|
|
325
|
+
# ---
|
|
326
|
+
# content
|
|
327
|
+
new_content = f"---\n{front_matter_str}---\n{content}"
|
|
328
|
+
with open(rendered_mdx, "w", encoding="utf-8") as f:
|
|
329
|
+
f.write(new_content)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def get_sorted_files(input_dir: Path, prefix: str) -> list[str]:
|
|
333
|
+
"""Get sorted list of files with prefix prepended."""
|
|
334
|
+
if not input_dir.exists():
|
|
335
|
+
raise FileNotFoundError(f"Directory not found: {input_dir}")
|
|
336
|
+
|
|
337
|
+
files = sorted(input_dir.glob("**/index.mdx"), key=sort_files_by_date)
|
|
338
|
+
reversed_files = files[::-1]
|
|
339
|
+
|
|
340
|
+
return [f"{prefix}/{f.parent.relative_to(input_dir)}/index".replace("\\", "/") for f in reversed_files]
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def generate_nav_group(input_dir: Path, group_header: str, prefix: str) -> dict[str, Union[str, list[str]]]:
|
|
344
|
+
"""Generate navigation group for a directory.
|
|
345
|
+
|
|
346
|
+
Args:
|
|
347
|
+
input_dir (Path): Directory to process
|
|
348
|
+
group_header (str): Group header
|
|
349
|
+
prefix (str): Prefix to prepend to file paths
|
|
350
|
+
"""
|
|
351
|
+
sorted_dir_files = get_sorted_files(input_dir, prefix)
|
|
352
|
+
|
|
353
|
+
return {"group": group_header, "pages": sorted_dir_files}
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def extract_example_group(metadata_path: Path) -> list[str]:
|
|
357
|
+
# Read NotebooksMetadata.mdx and extract metadata links
|
|
358
|
+
with open(metadata_path, encoding="utf-8") as f:
|
|
359
|
+
content = f.read()
|
|
360
|
+
# Extract the array between the brackets
|
|
361
|
+
start = content.find("export const notebooksMetadata = [")
|
|
362
|
+
end = content.rfind("]")
|
|
363
|
+
if start == -1 or end == -1:
|
|
364
|
+
print("Could not find notebooksMetadata in the file")
|
|
365
|
+
return []
|
|
366
|
+
metadata_str = content[start + 32 : end + 1]
|
|
367
|
+
notebooks_metadata = json.loads(metadata_str)
|
|
368
|
+
|
|
369
|
+
# Create notebooks entry
|
|
370
|
+
notebooks = ["docs/use-cases/notebooks/Notebooks"] + [
|
|
371
|
+
Path(item["source"])
|
|
372
|
+
.with_suffix("")
|
|
373
|
+
.as_posix()
|
|
374
|
+
.replace("/website/", "/")
|
|
375
|
+
.replace("/notebook/", "docs/use-cases/notebooks/notebooks/")
|
|
376
|
+
for item in notebooks_metadata
|
|
377
|
+
if not item["source"].startswith("/website/build/docs/")
|
|
378
|
+
]
|
|
379
|
+
|
|
380
|
+
return notebooks
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def update_group_pages(
|
|
384
|
+
mint_navigation: list[dict[str, Union[str, list[Union[str, dict[str, Union[str, list[str]]]]]]]],
|
|
385
|
+
target_group: str,
|
|
386
|
+
new_value: Sequence[Union[str, dict[str, Union[str, Sequence[str]]]]],
|
|
387
|
+
) -> list[dict[str, Union[str, list[Union[str, dict[str, Union[str, list[str]]]]]]]]:
|
|
388
|
+
"""Update mint.json navigation group with new pages."""
|
|
389
|
+
nav_copy = deepcopy(mint_navigation)
|
|
390
|
+
|
|
391
|
+
def update_recursively(
|
|
392
|
+
items: list[dict[str, Union[str, list[Union[str, dict[str, Union[str, list[str]]]]]]]],
|
|
393
|
+
) -> None:
|
|
394
|
+
for item in items:
|
|
395
|
+
if isinstance(item, dict):
|
|
396
|
+
if item.get("group") == target_group:
|
|
397
|
+
item["pages"] = new_value.copy() # type: ignore [attr-defined]
|
|
398
|
+
return
|
|
399
|
+
if isinstance(item.get("pages"), list):
|
|
400
|
+
update_recursively(item["pages"]) # type: ignore [arg-type]
|
|
401
|
+
elif isinstance(item, list):
|
|
402
|
+
update_recursively(item)
|
|
403
|
+
|
|
404
|
+
update_recursively(nav_copy)
|
|
405
|
+
return nav_copy
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def add_notebooks_blogs_and_user_stories_to_nav(website_build_directory: Path) -> None:
|
|
409
|
+
"""Updates mint.json navigation to include notebooks, blogs, and user stories.
|
|
410
|
+
|
|
411
|
+
Args:
|
|
412
|
+
website_build_directory (Path): Build directory of the website
|
|
413
|
+
"""
|
|
414
|
+
mint_json_path = (website_build_directory / "mint.json").resolve()
|
|
415
|
+
metadata_path = (website_build_directory / "snippets" / "data" / "NotebooksMetadata.mdx").resolve()
|
|
416
|
+
|
|
417
|
+
if not mint_json_path.exists():
|
|
418
|
+
print(f"mint.json not found at {mint_json_path}")
|
|
419
|
+
return
|
|
420
|
+
|
|
421
|
+
if not metadata_path.exists():
|
|
422
|
+
print(f"NotebooksMetadata.mdx not found at {metadata_path}")
|
|
423
|
+
return
|
|
424
|
+
|
|
425
|
+
# Read mint.json
|
|
426
|
+
with open(mint_json_path, encoding="utf-8") as f:
|
|
427
|
+
mint_config = json.load(f)
|
|
428
|
+
|
|
429
|
+
# add talks to navigation
|
|
430
|
+
# talks_dir = website_build_directory / "talks"
|
|
431
|
+
# talks_section = generate_nav_group(talks_dir, "Talks", "talks")
|
|
432
|
+
# talks_section_pages = (
|
|
433
|
+
# [talks_section["pages"]] if isinstance(talks_section["pages"], str) else talks_section["pages"]
|
|
434
|
+
# )
|
|
435
|
+
|
|
436
|
+
# Add "talks/future_talks/index" item at the beginning of the list
|
|
437
|
+
# future_talks_index = talks_section_pages.pop()
|
|
438
|
+
# talks_section_pages.insert(0, future_talks_index)
|
|
439
|
+
# mint_config["navigation"].append(talks_section)
|
|
440
|
+
|
|
441
|
+
# add user_stories to navigation
|
|
442
|
+
user_stories_dir = website_build_directory / "docs" / "user-stories"
|
|
443
|
+
user_stories_section = {
|
|
444
|
+
"group": "User Stories",
|
|
445
|
+
"pages": [generate_nav_group(user_stories_dir, "User Stories", "docs/user-stories")],
|
|
446
|
+
}
|
|
447
|
+
mint_config["navigation"].append(user_stories_section)
|
|
448
|
+
|
|
449
|
+
# add blogs to navigation
|
|
450
|
+
blogs_dir = website_build_directory / "docs" / "_blogs"
|
|
451
|
+
blog_section = {"group": "Blog", "pages": [generate_nav_group(blogs_dir, "Recent posts", "docs/blog")]}
|
|
452
|
+
mint_config["navigation"].append(blog_section)
|
|
453
|
+
|
|
454
|
+
# Add examples to navigation
|
|
455
|
+
notebooks_navigation = extract_example_group(metadata_path)
|
|
456
|
+
updated_navigation = update_group_pages(mint_config["navigation"], "Notebooks", notebooks_navigation)
|
|
457
|
+
mint_config["navigation"] = updated_navigation
|
|
458
|
+
|
|
459
|
+
# Write back to mint.json
|
|
460
|
+
with open(mint_json_path, "w", encoding="utf-8") as f:
|
|
461
|
+
json.dump(mint_config, f, indent=2)
|
|
462
|
+
f.write("\n")
|
|
463
|
+
|
|
464
|
+
print(f"Updated navigation in {mint_json_path}")
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def fix_internal_references(content: str, root_path: Path, current_file_path: Path) -> str:
|
|
468
|
+
"""Resolves internal markdown references relative to root_dir and returns fixed content.
|
|
469
|
+
|
|
470
|
+
Args:
|
|
471
|
+
content: Markdown content to fix
|
|
472
|
+
root_path: Root directory for resolving paths
|
|
473
|
+
current_file_path: Path of the current file being processed
|
|
474
|
+
"""
|
|
475
|
+
|
|
476
|
+
def resolve_link(match: re.Match[str]) -> str:
|
|
477
|
+
display_text, raw_path = match.groups()
|
|
478
|
+
try:
|
|
479
|
+
path_parts = raw_path.split("#")
|
|
480
|
+
rel_path = path_parts[0]
|
|
481
|
+
anchor = f"#{path_parts[1]}" if len(path_parts) > 1 else ""
|
|
482
|
+
|
|
483
|
+
resolved = (current_file_path.parent / rel_path).resolve()
|
|
484
|
+
final_path = (resolved.relative_to(root_path.resolve())).with_suffix("")
|
|
485
|
+
|
|
486
|
+
return f"[{display_text}](/{final_path}{anchor})"
|
|
487
|
+
except Exception:
|
|
488
|
+
return match.group(0)
|
|
489
|
+
|
|
490
|
+
pattern = r"\[([^\]]+)\]\(((?:\.\./|\./)?\w+(?:/[\w-]+)*\.md(?:#[\w-]+)?)\)"
|
|
491
|
+
return re.sub(pattern, resolve_link, content)
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def fix_internal_references_in_mdx_files(website_build_directory: Path) -> None:
|
|
495
|
+
"""Process all MDX files in directory to fix internal references."""
|
|
496
|
+
for file_path in website_build_directory.glob("**/*.mdx"):
|
|
497
|
+
try:
|
|
498
|
+
with open(file_path, encoding="utf-8") as f:
|
|
499
|
+
content = f.read()
|
|
500
|
+
|
|
501
|
+
fixed_content = fix_internal_references(content, website_build_directory, file_path)
|
|
502
|
+
|
|
503
|
+
if content != fixed_content:
|
|
504
|
+
with open(file_path, "w", encoding="utf-8") as f:
|
|
505
|
+
f.write(fixed_content)
|
|
506
|
+
|
|
507
|
+
except Exception:
|
|
508
|
+
print(f"Error: {file_path}")
|
|
509
|
+
sys.exit(1)
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def add_authors_and_social_img_to_blog_and_user_stories(website_build_directory: Path) -> None:
|
|
513
|
+
"""Add authors info to blog posts and user stories.
|
|
514
|
+
|
|
515
|
+
Args:
|
|
516
|
+
website_build_directory (Path): Build directory of the website
|
|
517
|
+
"""
|
|
518
|
+
blog_dir = website_build_directory / "docs" / "_blogs"
|
|
519
|
+
generated_blog_dir = website_build_directory / "docs" / "blog"
|
|
520
|
+
|
|
521
|
+
authors_yml = website_build_directory / "blogs_and_user_stories_authors.yml"
|
|
522
|
+
all_authors_info = get_authors_info(authors_yml)
|
|
523
|
+
|
|
524
|
+
# Remove existing generated directory if it exists
|
|
525
|
+
if generated_blog_dir.exists():
|
|
526
|
+
shutil.rmtree(generated_blog_dir)
|
|
527
|
+
|
|
528
|
+
# Copy entire blog directory structure to generated_blog
|
|
529
|
+
shutil.copytree(blog_dir, generated_blog_dir)
|
|
530
|
+
add_authors_and_social_preview(website_build_directory, generated_blog_dir, all_authors_info)
|
|
531
|
+
|
|
532
|
+
user_stories_dir = website_build_directory / "docs" / "user-stories"
|
|
533
|
+
add_authors_and_social_preview(website_build_directory, user_stories_dir, all_authors_info)
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def ensure_mint_json_exists(website_build_directory: Path) -> None:
|
|
537
|
+
mint_json_path = website_build_directory / "mint.json"
|
|
538
|
+
if not mint_json_path.exists():
|
|
539
|
+
print(f"mint.json not found at {mint_json_path}")
|
|
540
|
+
print(
|
|
541
|
+
"You can either run the 'generate_api_references.py' script before running this script or simply run the scripts/docs_build.sh script which will execute both 'generate_api_references.py' and 'process_notebooks.py' scripts in correct order."
|
|
542
|
+
)
|
|
543
|
+
sys.exit(1)
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def cleanup_tmp_dirs(website_build_directory: Path, re_generate_notebooks: bool) -> None:
|
|
547
|
+
"""Remove the temporary notebooks directory if NotebooksMetadata.mdx is not found.
|
|
548
|
+
|
|
549
|
+
This is to ensure a clean build and generate the metadata file as well as to
|
|
550
|
+
update the navigation with correct entries.
|
|
551
|
+
"""
|
|
552
|
+
delete_tmp_dir = re_generate_notebooks
|
|
553
|
+
metadata_mdx = website_build_directory / "snippets" / "data" / "NotebooksMetadata.mdx"
|
|
554
|
+
if not metadata_mdx.exists():
|
|
555
|
+
print(f"NotebooksMetadata.mdx not found at {metadata_mdx}")
|
|
556
|
+
delete_tmp_dir = True
|
|
557
|
+
|
|
558
|
+
if delete_tmp_dir:
|
|
559
|
+
notebooks_dir = notebooks_target_dir(website_build_directory)
|
|
560
|
+
print(f"Removing the {notebooks_dir} and to ensure a clean build.")
|
|
561
|
+
shutil.rmtree(notebooks_dir, ignore_errors=True)
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def get_files_path_from_navigation(navigation: list[NavigationGroup]) -> list[Path]:
|
|
565
|
+
"""Extract all file paths from the navigation structure.
|
|
566
|
+
|
|
567
|
+
Args:
|
|
568
|
+
navigation: list of navigation groups containing nested pages and groups
|
|
569
|
+
|
|
570
|
+
Returns:
|
|
571
|
+
list of file paths found in the navigation structure
|
|
572
|
+
"""
|
|
573
|
+
file_paths = []
|
|
574
|
+
|
|
575
|
+
def extract_paths(items: Union[Sequence[Union[str, NavigationGroup]]]) -> None:
|
|
576
|
+
for item in items:
|
|
577
|
+
if isinstance(item, str):
|
|
578
|
+
file_paths.append(Path(item))
|
|
579
|
+
elif isinstance(item, dict) and "pages" in item:
|
|
580
|
+
extract_paths(item["pages"])
|
|
581
|
+
|
|
582
|
+
extract_paths(navigation)
|
|
583
|
+
return file_paths
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
@require_optional_import("jinja2", "docs")
|
|
587
|
+
def add_edit_urls_and_remove_mkdocs_markers(website_build_directory: Path) -> None:
|
|
588
|
+
"""Add edit links to the non generated mdx files and remove mkdocs specific markers from the file.
|
|
589
|
+
|
|
590
|
+
For the generated mdx files i.e. mdx files of _blogs and notebooks, it is added in their respective post processing functions.
|
|
591
|
+
"""
|
|
592
|
+
mint_json_template_path = website_build_directory / "mint-json-template.json.jinja"
|
|
593
|
+
|
|
594
|
+
mint_json_template_content = Template(mint_json_template_path.read_text(encoding="utf-8")).render()
|
|
595
|
+
mint_json_data = json.loads(mint_json_template_content)
|
|
596
|
+
|
|
597
|
+
mdx_files = get_files_path_from_navigation(mint_json_data["navigation"])
|
|
598
|
+
mdx_files_with_prefix = [Path(f"{website_build_directory}/{str(file_path)}.mdx") for file_path in mdx_files]
|
|
599
|
+
|
|
600
|
+
for mdx_file_path in mdx_files_with_prefix:
|
|
601
|
+
rel_path = str(mdx_file_path.relative_to(website_build_directory.parent)).replace("build/", "website/")
|
|
602
|
+
content = mdx_file_path.read_text(encoding="utf-8")
|
|
603
|
+
content_with_edit_url = ensure_edit_url(content, Path(rel_path))
|
|
604
|
+
|
|
605
|
+
# Remove mkdocs markers before building the docs
|
|
606
|
+
content_without_mkdocs_marker = remove_marker_blocks(content_with_edit_url, "DELETE-ME-WHILE-BUILDING-MINTLIFY")
|
|
607
|
+
mdx_file_path.write_text(content_without_mkdocs_marker, encoding="utf-8")
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def copy_images_from_notebooks_dir_to_target_dir(notebook_directory: Path, target_notebooks_dir: Path) -> None:
|
|
611
|
+
"""Copy images from notebooks directory to the target directory."""
|
|
612
|
+
# Define supported image extensions
|
|
613
|
+
supported_img_extensions = {".png", ".jpg"}
|
|
614
|
+
|
|
615
|
+
# Single loop through directory contents
|
|
616
|
+
for image_path in notebook_directory.iterdir():
|
|
617
|
+
if image_path.is_file() and image_path.suffix.lower() in supported_img_extensions:
|
|
618
|
+
target_image = target_notebooks_dir / image_path.name
|
|
619
|
+
shutil.copy(image_path, target_image)
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def main() -> None:
|
|
623
|
+
root_dir = Path(__file__).resolve().parents[2]
|
|
624
|
+
website_dir = root_dir / "website"
|
|
625
|
+
website_build_dir = website_dir / "build"
|
|
626
|
+
parser = create_base_argument_parser()
|
|
627
|
+
|
|
628
|
+
if not website_build_dir.exists():
|
|
629
|
+
website_build_dir.mkdir()
|
|
630
|
+
shutil.copytree(website_dir, website_build_dir, dirs_exist_ok=True)
|
|
631
|
+
|
|
632
|
+
args = parser.parse_args()
|
|
633
|
+
if args.subcommand is None:
|
|
634
|
+
print("No subcommand specified")
|
|
635
|
+
sys.exit(1)
|
|
636
|
+
|
|
637
|
+
if args.website_build_directory is None:
|
|
638
|
+
args.website_build_directory = website_build_dir
|
|
639
|
+
|
|
640
|
+
if args.notebook_directory is None:
|
|
641
|
+
args.notebook_directory = website_build_dir / "../../notebook"
|
|
642
|
+
|
|
643
|
+
ensure_mint_json_exists(args.website_build_directory)
|
|
644
|
+
cleanup_tmp_dirs(args.website_build_directory, args.force)
|
|
645
|
+
|
|
646
|
+
# Process notebooks using core logic
|
|
647
|
+
process_notebooks_core(args, post_process_mdx, notebooks_target_dir)
|
|
648
|
+
|
|
649
|
+
# Post-processing steps after all notebooks are handled
|
|
650
|
+
if not args.dry_run:
|
|
651
|
+
target_notebooks_dir = notebooks_target_dir(args.website_build_directory)
|
|
652
|
+
copy_images_from_notebooks_dir_to_target_dir(args.notebook_directory, target_notebooks_dir)
|
|
653
|
+
add_notebooks_blogs_and_user_stories_to_nav(args.website_build_directory)
|
|
654
|
+
fix_internal_references_in_mdx_files(args.website_build_directory)
|
|
655
|
+
add_authors_and_social_img_to_blog_and_user_stories(args.website_build_directory)
|
|
656
|
+
add_edit_urls_and_remove_mkdocs_markers(args.website_build_directory)
|