qwen-sci 0.2.2__py3-none-any.whl

src/agents/survey_agent/Dockerfile

@@ -0,0 +1,54 @@
+# Use the official vllm image for gpu with Ampere architecture and above (Compute Capability>=8.0)
+# Compute Capability version query (https://developer.nvidia.com/cuda-gpus)
+FROM vllm/vllm-openai:v0.10.1.1
+
+# Use the official vllm image for gpu with Turing architecture and below (Compute Capability<8.0)
+# FROM vllm/vllm-openai:v0.10.2
+
+# Install libgl for opencv support & Noto fonts for Chinese characters
+RUN apt-get update && \
+    apt-get install -y \
+        fonts-noto-core \
+        fonts-noto-cjk \
+        fontconfig \
+        libgl1 && \
+    fc-cache -fv && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install mineru latest
+RUN python3 -m pip install -U 'mineru[core]' --break-system-packages && \
+    python3 -m pip cache purge
+
+# Install some other packages
+RUN python3 -m pip install -U \
+        hydra-core \
+        asciinet \
+        tenacity \
+        trafilatura \
+        readability-lxml \
+        html2text
+
+# Install java
+RUN wget https://mirrors.huaweicloud.com/openjdk/25/openjdk-25_linux-x64_bin.tar.gz -O /tmp/jdk-25.tar.gz && \
+    mkdir -p /opt/jdk-25 && \
+    tar -xzf /tmp/jdk-25.tar.gz -C /opt/jdk-25 --strip-components=1 && \
+    rm /tmp/jdk-25.tar.gz
+
+ENV JAVA_HOME=/opt/jdk-25
+ENV PATH=$JAVA_HOME/bin:$PATH
+
+# for deepsurvey
+RUN pip install hydra-core && \
+    pip install rich && \
+    pip install loguru && \
+    pip install tenacity && \
+    pip install -U sentence-transformers && \
+    pip install hdbscan && \
+
+ENV HF_ENDPOINT=https://hf-mirror.com
+# Download models and update the configuration file
+RUN /bin/bash -c "mineru-models-download -s huggingface -m all"
+
+# Set the entry point to activate the virtual environment and run the command line tool
+ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"]

src/agents/survey_agent/README.md

@@ -0,0 +1,350 @@
+# Deep Survey
+
+Deep Survey is an automated academic survey generation pipeline. Given a research topic, it collects related papers, expands the paper set through references and citations, reads and analyzes papers with LLMs, clusters related work, optionally builds relation/code analysis artifacts, generates a survey draft, revises it, refines citations, and optionally evaluates the final survey.
+
+## Repository layout
+
+```text
+.
+├── Dockerfile                    # Container environment for Deep Survey and MinerU
+├── config/                       # Hydra/YAML runtime configurations
+│   ├── evaluation/               # Evaluation experiment configs
+│   └── *.yaml                    # Default and task-specific configs
+├── scripts/                      # Main runnable entry points
+├── modules/                      # Core pipeline modules
+├── utils/                        # API, logging, file, PDF, markdown and helper utilities
+├── topics/                       # Topic lists and benchmark topic definitions
+├── baselines/                    # Baseline systems and baseline outputs
+├── tests/                        # Development tests and scratch files
+├── logs/                         # Runtime logs, ignored by git
+├── outputs/                      # Generated surveys and artifacts, ignored by git
+└── database/                     # Local paper/cache/database artifacts, ignored by git
+```
+
+`logs/`, `outputs/`, and `database/` are generated/runtime directories and are ignored by git. The `logs/` directory can usually be ignored when reading the codebase.
+
+## Pipeline overview
+
+The main Deep Survey pipeline follows these stages:
+
+1. Related work collection
+   - Searches seed papers for the target topic.
+   - Expands papers through references and citations.
+   - Can use local graph/cache data when configured.
+2. Paper database construction
+   - Builds an embedding/database view over collected papers.
+3. Paper comprehension
+   - Reads papers or abstracts.
+   - Writes reusable paper keynotes into cache.
+4. Clustering and analysis
+   - Clusters related papers.
+   - Optionally performs intra-cluster and inter-cluster analysis.
+   - Optionally generates relation graphs and relation tables.
+5. Optional code/environment report generation
+   - Collects and analyzes code repositories linked to papers.
+   - Generates code and environment reports when enabled.
+6. Survey generation
+   - Generates an outline.
+   - Drafts sections/subsections.
+   - Reviews and revises the draft.
+   - Refines final paper text and references.
+7. Optional evaluation
+   - Evaluates generated survey quality and citation quality when enabled.
+
+## Core modules
+
+- `modules/work_collector.py`: collects seed papers and expands them via references/citations.
+- `modules/database.py`: builds retrieval databases over collected papers.
+- `modules/work_analyzer.py`: reads papers, extracts keynotes, clusters papers, and performs relation/cluster analysis.
+- `modules/survey_generator.py`: generates outlines, drafts survey sections, reviews/revises drafts, and saves final survey outputs.
+- `modules/judge.py`: evaluates survey outputs when evaluation is enabled.
+- `modules/code_collector.py`: collects code repository information.
+- `modules/code_report_generator.py`: generates code/environment reports from collected repositories.
+- `modules/paper_graph_retriever.py`: retrieves papers from local graph/database resources.
+
+## Important scripts
+
+### Single-topic Hydra entry point
+
+```bash
+python3 scripts/run_deep_survey.py
+```
+
+This uses Hydra with the default config declared in the script:
+
+```text
+config/deep_survey_fast.yaml
+```
+
+You can override Hydra config values from the command line, for example:
+
+```bash
+python3 scripts/run_deep_survey.py BasicInfo.topic="Graph Neural Networks"
+```
+
+### Batch entry point with explicit YAML config
+
+```bash
+python3 scripts/run_deep_survey_batch_arg.py --config ./config/personal/deep_survey_batch_0514.yaml
+```
+
+This is the most explicit batch runner. It reads a YAML file, loads topics according to `BasicInfo`, and runs the pipeline for each topic. For example, `config/personal/deep_survey_batch_0514.yaml` uses:
+
+- `BasicInfo.user_defiend_benchmarks: True`
+- `BasicInfo.topic_path: ./topics/human_compare_topics.txt`
+- `BasicInfo.output_base_dir: ./outputs/0514_mimo_pro_human_test`
+- `BasicInfo.skip_exist: True`
+
+### Adapter entry point
+
+```bash
+python3 scripts/run_deep_survey_adapter.py --workspace /workspace --config /workspace/config/runtime.yaml
+```
+
+The adapter is designed for external orchestration. It writes standardized artifacts such as:
+
+```text
+/workspace/survey/output/survey.md
+/workspace/survey/output/survey.json
+/workspace/survey/output/evaluation.txt
+/workspace/logs/relation_graph.json
+/workspace/logs/relation_table.json
+/workspace/logs/clustering_result.json
+/workspace/logs/draft.json
+/workspace/logs/deep_survey.log
+```
+
+### Baseline evaluation
+
+```bash
+python3 scripts/baseline_evaluation.py
+```
+
+This script evaluates outputs from Deep Survey and baseline systems such as AutoSurvey, SurveyForge, LiRA, and human-written surveys according to the configured benchmark/evaluation settings.
+
+### Other utility scripts
+
+- `scripts/run_keynotes_gen.py`: pre-generate paper keynotes.
+- `scripts/run_work_clusters.py`: run work clustering utilities.
+- `scripts/memory_monitor.sh`: monitor memory usage during long-running jobs.
+- `scripts/run_model_ablation.sh`: run model ablation experiments.
+
+## Configuration
+
+Configurations live under `config/`. The common top-level sections are:
+
+```yaml
+BasicInfo:
+  topic: ""
+  topic_path: ./topics/human_compare_topics.txt
+  output_base_dir: ./outputs/example
+  cache_path: ./database
+  topic_max_retry: 5
+  skip_exist: True
+
+APIInfo:
+  llm_api_key: "..."
+  llm_api_base_url: "..."
+  llm_model_name: "..."
+  # Required for OpenAlex literature retrieval; do not commit the literal key.
+  openalex_api_key: "${oc.env:OPENALEX_API_KEY,''}"
+  batch_chat_agent_worker: 2
+  chat_timeout: 3600
+  batch_chat_timeout: 1800
+
+ModuleInfo:
+  WorkCollector:
+    max_seed_paper_num: 15
+    reference_graph_depth: 1
+  WorkAnalyzer:
+    abstract_only_mode: False
+    paper_reading_max_retry: 6
+  SurveyGenerator:
+    include_initial_analysis: True
+    include_relation_graph: False
+    include_relation_table: False
+    include_code_report: False
+    enable_review_and_revise: True
+  Judge:
+    skip_evaluation: True
+```
+
+Before running experiments, check at least these fields:
+
+- `BasicInfo.topic` or `BasicInfo.topic_path`
+- `BasicInfo.output_base_dir`
+- `BasicInfo.cache_path`
+- `APIInfo.llm_api_key`
+- `APIInfo.llm_api_base_url`
+- `APIInfo.llm_model_name`
+- `APIInfo.openalex_api_key` (or the `OPENALEX_API_KEY` environment variable)
+- `APIInfo.batch_chat_agent_worker`
+- `ModuleInfo.Judge.skip_evaluation`
+
+Avoid committing real API keys. Prefer using environment variables or local-only config files for secrets.
+
+## Topics
+
+Topic lists are stored in `topics/`. Batch runs can use:
+
+```yaml
+BasicInfo:
+  user_defiend_benchmarks: True
+  topic_path: ./topics/human_compare_topics.txt
+```
+
+Benchmark topic dictionaries are defined in:
+
+```text
+topics/Benchmark_topics.py
+```
+
+Relevant switches include:
+
+- `BasicInfo.AutoSurvey_benchmark`
+- `BasicInfo.SurveyGen_benchmark`
+- `BasicInfo.sub_benchmark_test`
+- `BasicInfo.user_defiend_benchmarks`
+
+## Outputs
+
+Generated files are typically written under `BasicInfo.output_base_dir`. A batch run stores per-domain/per-topic results and may produce:
+
+- Markdown survey files
+- JSON survey files with paper text and references
+- Evaluation result files
+- Analysis artifacts such as clustering results, relation graphs, relation tables, draft files, code reports, and environment reports
+
+The exact outputs depend on `ModuleInfo.SurveyGenerator` and `ModuleInfo.Judge` switches.
+
+## Docker environment
+
+The repository includes a Dockerfile at `Dockerfile`. It is the intended environment configuration for Deep Survey and PDF parsing utilities.
+
+The image is based on:
+
+```dockerfile
+FROM vllm/vllm-openai:v0.10.1.1
+```
+
+It installs:
+
+- Noto fonts and fontconfig for Chinese text rendering
+- `libgl1` for OpenCV support
+- `mineru[core]` for PDF parsing
+- `hydra-core`, `asciinet`, `tenacity`, `trafilatura`, `readability-lxml`, `html2text`
+- OpenJDK 25
+- `rich`, `loguru`, `sentence-transformers`, `hdbscan`
+
+It also sets:
+
+```dockerfile
+ENV HF_ENDPOINT=https://hf-mirror.com
+ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"]
+```
+
+and downloads MinerU models during image build:
+
+```bash
+mineru-models-download -s huggingface -m all
+```
+
+Build example:
+
+```bash
+docker build -t deep-survey:latest .
+```
+
+Run example:
+
+```bash
+docker run --gpus all --rm -it \
+  -v /path/to/deep-survey:/workspace/deep-survey \
+  -w /workspace/deep-survey \
+  deep-survey:latest \
+  python3 scripts/run_deep_survey_batch_arg.py --config ./config/personal/deep_survey_batch_0514.yaml
+```
+
+If building fails around the final Python package installation block, check the trailing shell continuation after `pip install hdbscan`. The last command in a chained `RUN` block should not leave a dangling `&& \` before the next Docker instruction.
+
+## PDF and MinerU utilities
+
+PDF/markdown related utilities are under `utils/`, especially:
+
+```text
+utils/mineru_utils.py
+utils/convert_to_md.py
+utils/html_utils.py
+```
+
+`utils/mineru_utils.py` contains MinerU-based parsing helpers for converting PDFs into markdown-like outputs. The Dockerfile installs MinerU and pre-downloads its models so PDF parsing can run with local MinerU model sources.
+
+## Recommended run patterns
+
+### Foreground run
+
+```bash
+CUDA_VISIBLE_DEVICES=0 python3 scripts/run_deep_survey_batch_arg.py \
+  --config ./config/personal/deep_survey_batch_0514.yaml
+```
+
+### Background run with log capture
+
+```bash
+mkdir -p logs
+nohup bash -lc 'CUDA_VISIBLE_DEVICES=0 python3 scripts/run_deep_survey_batch_arg.py --config ./config/personal/deep_survey_batch_0514.yaml' \
+  > ./logs/deep_survey_batch.log 2>&1 &
+```
+
+### Memory monitoring
+
+```bash
+bash scripts/memory_monitor.sh
+```
+
+For long experiments on remote/HPC machines, prefer using `tmux`, `screen`, or a job scheduler so the process is not interrupted by SSH session disconnection.
+
+## Troubleshooting
+
+### `Killed`
+
+`Killed` can be caused by OOM, scheduler limits, or external signals. Check:
+
+- system/job scheduler logs
+- shell/session disconnection
+- memory logs
+- GPU/CPU memory usage
+- custom signal traps if enabled
+
+### `ProxyError` or API timeout
+
+In this project, repeated `ProxyError` can also indicate upstream API timeout, connection reset, gateway failure, or too much concurrent traffic. Consider reducing:
+
+```yaml
+APIInfo:
+  batch_chat_agent_worker: 1
+  low_flow_mode: True
+  low_flow_latency: 2
+  exponential_backoff: True
+```
+
+### Hugging Face network issues
+
+The Dockerfile sets:
+
+```bash
+HF_ENDPOINT=https://hf-mirror.com
+```
+
+If the environment cannot access Hugging Face, use a mirror, pre-download models, or configure offline cache paths.
+
+### Logs directory
+
+`logs/` is runtime-only and ignored by git. It is useful for debugging local runs but should not be treated as source code.
+
+## Development notes
+
+- The codebase is Python-based and uses Hydra/OmegaConf-style YAML configs.
+- Most modules are initialized from the selected config, so behavior is usually controlled by YAML rather than command-line flags.
+- Long-running batch jobs depend heavily on API stability, retry settings, and concurrency settings.
+- Generated caches under `database/` can speed up later runs but are not source files.

src/agents/survey_agent/config/deep_survey.yaml

@@ -0,0 +1,217 @@
+BasicInfo:
+  topic:  Multimodal Large Language Models
+  base_dir: ./outputs
+  cache_path: ./database
+  save_path: ./outputs/test/survey.md
+  save_json_path: ./outputs/test/survey.json
+  evaluation_save_path: ./outputs/test/eval.txt
+  debug: True
+  error_conservatism_mode: False
+  skip_exist: True
+  adapter_mode: False
+APIInfo:
+  llm_provider: qwen
+  llm_api_key: "${oc.env:DASHSCOPE_API_KEY,''}"
+  llm_api_base_url: "${oc.env:DASHSCOPE_BASE_URL,'https://dashscope.aliyuncs.com/compatible-mode/v1'}"
+  llm_model_name: qwen3.6-flash
+  llm_max_context_length: 880000
+  use_stream_mode: False
+  batch_chat_agent_worker: 4
+  chat_timeout: 600
+  batch_chat_timeout: 1800
+  openalex_api_key: "${oc.env:OPENALEX_API_KEY,''}"
+  semantic_scholar_api_key:
+  low_flow_mode: False
+  low_flow_latency: 0
+  exponential_backoff: True
+  exponential_backoff_time: 1
+  exponential_backoff_max_time: 60
+  arxiv_api_max_retry: 3
+  semantic_scholar_api_max_retry: 3
+ModuleInfo:
+  WorkCollector:
+    download_safe_mode: False
+    download_timeout: 60
+    download_in_parallel: True
+    download_parallel_workers: 4
+    pdf_parse_batch_size: 1
+    fulltext_per_host_concurrency: 2
+    fulltext_access_context_generation: "anonymous-v1"
+    fulltext_provenance_enabled: true
+    cache_enabled: True
+    # OpenAlex is the literature-discovery provider; do not emit arXiv lanes.
+    enable_arxiv_discovery: false
+    max_seed_paper_num: 18
+    use_seed_filter_LLM: True
+    LLM_seed_threshold: 4
+    reference_graph_depth: 1
+    sentence_transformer_model: sentence-transformers/all-MiniLM-L6-v2
+    sentence_transformer_batch_size: 32
+    expand_in_local_paper_graph: True
+    use_ds_when_graph_fail: True
+    advanced_filter_in_local_paper_graph_expansion: True
+    related_work_top_k: 30  # for debug
+    log_related_work_num: -1 # set to -1 to log all
+    related_work_threshold: 0.5
+    related_work_threshold_for_llm: 4 # 1-5 scale
+    relatedness_temperature: 1.0
+    RAG_source_use_embedding_filter: True
+    RAG_source_use_LLM_filter: False
+    RAG_source_downloadable_only: False
+    enable_fulltext_expanded_promotion: true
+    expanded_fulltext_promotion_candidate_multiplier: 2
+    expanded_fulltext_promotion_max_candidates_per_sh: 40
+  Database:
+    default_top_k: 20
+  PaperGraphRetriever:
+    db_path: data/processed/graph.db
+  WorkAnalyzer:
+    abstract_only_mode: False
+    cache_enabled: True
+    # Citation-expanded papers can be promoted only after the complete-section
+    # reader succeeds; local graph keynotes cannot prove that condition.
+    use_local_paper_graph_keynotes: false
+    use_ds_keynotes_when_graph_fail: True
+    graph_keynote_extraction_batch_retry: 7
+    graph_graph_keynote_extraction_temperature: 0.1
+    paper_reading_temperature: 1.0
+    paper_reading_max_retry: 5
+    llm_max_context_overhead_length_in_paper_reading: 20000
+    clustering_batch_size: 16
+    clustering_temperature: 0.5
+    clustering_in_steps: True
+    clustering_batch_size_in_creation: 6
+    clustering_batch_size_in_assignment: 8
+    cluster_assign_fast_mode: True
+    paper_clustering_max_retry: 10
+    paper_clustering_creation_max_retry: 3
+    paper_clustering_assignment_max_retry: 10
+    propose_question_temperature: 1.0
+    anwer_question_temperature: 1.0
+    intra_cluster_analysis_max_retry: 10
+    intra_cluster_analysis_temperature: 1.0
+    intra_clustering_probelm_proposing_max_retry: 10
+  SurveyGenerator:
+    include_initial_analysis: False
+    include_relation_graph: True
+    include_relation_table: True
+    include_code_report: False
+    include_env_report: False
+    omit_error_preserve_retry_time: 2
+    outline_generation_in_steps: True
+    outline_generation_draft_max_retry: 3
+    outline_generation_draft_batch_size: 20
+    outline_generation_draft_max_iterations: 5
+    outline_generation_draft_empty_keynotes_iteration: 1
+    outline_draft_RAG_topk: 60
+    outline_generation_assign_max_retry: 10
+    outline_generation_assign_batch_size: 10
+    outline_assign_RAG_topk: 20
+    outline_assign_fast_mode: True
+    outline_generation_temperature: 0.5
+    # Outline prompts use a compact SH-plan projection and must fit before a
+    # request is sent; never rely on request-layer prefix truncation.
+    outline_prompt_max_input_tokens: 160000
+    outline_max_output_tokens: 16000
+    outline_evidence_plan_max_input_tokens: 100000
+    outline_current_outline_max_input_tokens: 20000
+    outline_keynotes_max_input_tokens: 70000
+    outline_analysis_max_input_tokens: 12000
+    outline_rag_max_input_tokens: 12000
+    outline_repair_previous_response_max_input_tokens: 8000
+    outline_repair_current_outline_max_input_tokens: 8000
+    # The outline sees a SH-covering representative set.  Assignment still
+    # considers every admitted paper after the outline shape is fixed.
+    outline_representative_papers_per_sh: 4
+    outline_representative_max_papers: 30
+    outline_representative_include_context_papers: true
+    # Whole-survey contract. Per-section/subsection prompts are derived from
+    # the accepted outline; the 33.3k/40k total budget permits up to 2k words
+    # per subsection for the target outline.
+    survey_target_words: 32000
+    survey_max_words: 40000
+    writing_max_papers_per_sh: 20
+    outline_min_sections: 5
+    outline_target_sections: 6
+    outline_max_sections: 7
+    outline_min_subsections_per_section: 1
+    outline_target_subsections_per_section: 3
+    outline_max_subsections_per_section: 3
+    subsection_target_min_words: 875
+    subsection_target_max_words: 2000
+    subsection_target_citations: 3
+    subsection_max_citations: 5
+    section_preamble_target_words: 250
+    section_preamble_max_words: 400
+    section_preamble_target_citations: 1
+    section_preamble_max_citations: 2
+    outline_generation_batch_size: 16
+    outline_generation_max_retry: 3
+    outline_generation_max_retry_in_generation_loop: 6
+    subsection_draft_temperature: 1.0
+    subsection_draft_max_retry: 5
+    # Temporary operational bypass: keep visible citation checks, but do not
+    # block drafting on LLM-generated claim-trace metadata.
+    claim_trace_validation_enabled: false
+    claim_trace_repair_max_attempts: 2
+    claim_trace_repair_max_output_tokens: 2048
+    section_draft_max_retry: 5
+    always_omit_error_in_draft_validation: True
+    section_draft_temperature: 1.0
+    draft_refinement_temperature: 1.0
+    section_draft_max_error_info_length: 500
+    subsection_draft_max_error_info_length: 500
+    outline_max_error_info_length: 500
+    llm_max_context_overhead_length_generation: 20000
+    llm_max_context_overhead_length_outline_generation: 10000
+    # Legacy keys retained for external overrides; new drafting uses the
+    # whole-survey budget above rather than these per-unit hard minima.
+    subsection_least_words: 875
+    subsection_least_citations: 3
+    section_least_words: 4250
+    section_preamble_least_words: 250
+    section_preamble_least_citations: 3
+    draft_length_relax_ratio: 0.1
+    use_full_text_in_survey_generation: False
+    include_other_relevant_papers_RAG_in_outline: True
+    outline_RAG_topk: 200
+    include_other_relevant_papers_RAG: True
+    section_RAG_topk: 60
+    subsection_RAG_topk: 60
+    use_title_in_draft: True
+    enable_review_and_revise: True
+    agentic_refine_section: True
+    section_review_temperature: 1.0
+    section_review_retry: 3
+    reviewer_max_suggestions: 5
+    section_revision_RAG_topk: 30
+    section_revise_temperature: 0.5
+    section_revise_retry: 5
+    no_suggestion_run_each_iteration: 1
+    max_review_revise_iterations: 5
+    revise_section_in_parallel: 1
+    draft_refinement_in_parts: True
+    refine_in_parts_mode: 2
+    draft_refinement_max_retry: 7
+    review_and_revise_whole_survey_in_refinement: True
+    agentic_refine_survey: True
+    review_and_revise_whole_survey_max_iteration: 5
+    valid_title_min_similarity: 0.5
+  CodeAnalysis:
+    PseudoCreator:
+      planner_temperature: 0.5
+      planner_max_retry: 3
+  Judge:
+    skip_evaluation: False
+    use_different_api_for_judge: False
+    model: qwen3.7-plus
+    judge_llm_api_key: 
+    judge_llm_api_base_url: 
+    nli_temperature: 0.1
+    citation_quality_threshold: 0.5
+    remove_failed_citation_in_eval: True
+    rubrics_eval_10_dimensions: True
+    rubrics_eval_4_dimensions: False
+    rubrics_deep_survey_eval: False
+    explanation_in_rubric: True
+    citation_eval: False