surveycore-py 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- surveycore/__init__.py +44 -0
- surveycore/_question_metas.py +59 -0
- surveycore/answers/__init__.py +5 -0
- surveycore/answers/strategies.py +23 -0
- surveycore/app/__init__.py +1 -0
- surveycore/app/config.py +28 -0
- surveycore/app/runtime_paths.py +16 -0
- surveycore/app/user_paths.py +2 -0
- surveycore/client.py +187 -0
- surveycore/config.py +166 -0
- surveycore/core/__init__.py +0 -0
- surveycore/core/ai/__init__.py +3 -0
- surveycore/core/ai/batch_runtime.py +42 -0
- surveycore/core/ai/runtime.py +76 -0
- surveycore/core/config/__init__.py +0 -0
- surveycore/core/engine/__init__.py +0 -0
- surveycore/core/engine/async_events.py +90 -0
- surveycore/core/engine/async_scheduler.py +96 -0
- surveycore/core/engine/async_status_bus.py +50 -0
- surveycore/core/engine/async_wait.py +58 -0
- surveycore/core/engine/provider_common.py +150 -0
- surveycore/core/engine/runtime_actions.py +63 -0
- surveycore/core/engine/stop_signal.py +14 -0
- surveycore/core/modes/__init__.py +1 -0
- surveycore/core/modes/duration_control.py +206 -0
- surveycore/core/persona/__init__.py +0 -0
- surveycore/core/persona/context.py +139 -0
- surveycore/core/persona/generator.py +221 -0
- surveycore/core/psychometrics/__init__.py +62 -0
- surveycore/core/psychometrics/joint_optimizer.py +624 -0
- surveycore/core/psychometrics/ordinal_options.py +188 -0
- surveycore/core/psychometrics/orientation.py +190 -0
- surveycore/core/psychometrics/psychometric.py +334 -0
- surveycore/core/psychometrics/utils.py +159 -0
- surveycore/core/questions/__init__.py +0 -0
- surveycore/core/questions/config.py +33 -0
- surveycore/core/questions/consistency.py +337 -0
- surveycore/core/questions/default_builder.py +383 -0
- surveycore/core/questions/distribution.py +148 -0
- surveycore/core/questions/meta_helpers.py +165 -0
- surveycore/core/questions/normalization.py +393 -0
- surveycore/core/questions/reliability_mode.py +38 -0
- surveycore/core/questions/schema.py +104 -0
- surveycore/core/questions/strict_ratio.py +152 -0
- surveycore/core/questions/tendency.py +352 -0
- surveycore/core/questions/text_shared.py +3 -0
- surveycore/core/questions/text_values.py +136 -0
- surveycore/core/questions/types/__init__.py +0 -0
- surveycore/core/questions/utils.py +465 -0
- surveycore/core/questions/validation.py +213 -0
- surveycore/core/reverse_fill/__init__.py +49 -0
- surveycore/core/reverse_fill/parser.py +295 -0
- surveycore/core/reverse_fill/runtime.py +43 -0
- surveycore/core/reverse_fill/schema.py +151 -0
- surveycore/core/task/__init__.py +11 -0
- surveycore/core/task/distribution_state.py +235 -0
- surveycore/core/task/progress_state.py +246 -0
- surveycore/core/task/reverse_fill_state.py +189 -0
- surveycore/core/task/task_context.py +280 -0
- surveycore/core/user_agent.py +11 -0
- surveycore/credamo/__init__.py +0 -0
- surveycore/credamo/provider/__init__.py +0 -0
- surveycore/credamo/provider/answering_builders.py +185 -0
- surveycore/credamo/provider/http_runtime.py +902 -0
- surveycore/credamo/provider/parser.py +566 -0
- surveycore/data/area_codes_2022.json +1680 -0
- surveycore/errors.py +22 -0
- surveycore/io/__init__.py +0 -0
- surveycore/io/spreadsheets/__init__.py +0 -0
- surveycore/io/spreadsheets/wjx_excel.py +149 -0
- surveycore/logging/__init__.py +0 -0
- surveycore/logging/log_utils.py +43 -0
- surveycore/models.py +134 -0
- surveycore/network/__init__.py +1 -0
- surveycore/network/http/__init__.py +63 -0
- surveycore/network/session_policy.py +25 -0
- surveycore/providers/__init__.py +0 -0
- surveycore/providers/adapter_base.py +158 -0
- surveycore/providers/answering/__init__.py +7 -0
- surveycore/providers/answering/actions.py +49 -0
- surveycore/providers/answering/option_fill.py +71 -0
- surveycore/providers/answering/recording.py +55 -0
- surveycore/providers/answering/selection.py +108 -0
- surveycore/providers/common.py +174 -0
- surveycore/providers/contracts.py +435 -0
- surveycore/providers/errors.py +40 -0
- surveycore/providers/hooks.py +66 -0
- surveycore/providers/http_logic.py +285 -0
- surveycore/providers/http_progress.py +51 -0
- surveycore/providers/match_utils.py +54 -0
- surveycore/providers/registry.py +261 -0
- surveycore/providers/runtime_state.py +107 -0
- surveycore/py.typed +0 -0
- surveycore/runtime/__init__.py +5 -0
- surveycore/runtime/scheduler.py +30 -0
- surveycore/tencent/__init__.py +0 -0
- surveycore/tencent/provider/__init__.py +0 -0
- surveycore/tencent/provider/answering_builders.py +609 -0
- surveycore/tencent/provider/answering_rules.py +61 -0
- surveycore/tencent/provider/http_runtime.py +670 -0
- surveycore/tencent/provider/parser.py +1006 -0
- surveycore/transport.py +12 -0
- surveycore/wjx/__init__.py +1 -0
- surveycore/wjx/provider/__init__.py +0 -0
- surveycore/wjx/provider/answering_builders.py +727 -0
- surveycore/wjx/provider/html_parser.py +356 -0
- surveycore/wjx/provider/html_parser_choice.py +481 -0
- surveycore/wjx/provider/html_parser_common.py +586 -0
- surveycore/wjx/provider/html_parser_matrix.py +370 -0
- surveycore/wjx/provider/html_parser_rules.py +394 -0
- surveycore/wjx/provider/http_runtime.py +586 -0
- surveycore/wjx/provider/parser.py +227 -0
- surveycore/wjx/provider/questions/__init__.py +0 -0
- surveycore/wjx/provider/questions/_multiple_limit_patterns.py +130 -0
- surveycore/wjx/provider/questions/multiple_limits.py +237 -0
- surveycore/wjx/provider/questions/multiple_rules.py +115 -0
- surveycore/wjx/provider/regexes.py +164 -0
- surveycore_py-0.1.0.dist-info/METADATA +145 -0
- surveycore_py-0.1.0.dist-info/RECORD +121 -0
- surveycore_py-0.1.0.dist-info/WHEEL +4 -0
- surveycore_py-0.1.0.dist-info/licenses/LICENSE +674 -0
surveycore/__init__.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from importlib.metadata import version as _distribution_version
|
|
2
|
+
|
|
3
|
+
from .client import SurveyClient
|
|
4
|
+
from .config import deserialize_config, dumps_config, loads_config, serialize_config
|
|
5
|
+
from .errors import (
|
|
6
|
+
ConfigError,
|
|
7
|
+
SubmissionError,
|
|
8
|
+
SurveyCoreError,
|
|
9
|
+
SurveyParseError,
|
|
10
|
+
SurveyUnavailableError,
|
|
11
|
+
UnsupportedSurveyError,
|
|
12
|
+
)
|
|
13
|
+
from .models import (
|
|
14
|
+
AnswerGenerator,
|
|
15
|
+
QuestionConfig,
|
|
16
|
+
RunConfig,
|
|
17
|
+
SubmissionRequest,
|
|
18
|
+
SubmissionResult,
|
|
19
|
+
SurveyDefinition,
|
|
20
|
+
SurveyQuestion,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
__version__ = _distribution_version("surveycore-py")
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"AnswerGenerator",
|
|
27
|
+
"ConfigError",
|
|
28
|
+
"QuestionConfig",
|
|
29
|
+
"RunConfig",
|
|
30
|
+
"SubmissionError",
|
|
31
|
+
"SubmissionRequest",
|
|
32
|
+
"SubmissionResult",
|
|
33
|
+
"SurveyClient",
|
|
34
|
+
"SurveyCoreError",
|
|
35
|
+
"SurveyDefinition",
|
|
36
|
+
"SurveyParseError",
|
|
37
|
+
"SurveyQuestion",
|
|
38
|
+
"SurveyUnavailableError",
|
|
39
|
+
"UnsupportedSurveyError",
|
|
40
|
+
"deserialize_config",
|
|
41
|
+
"dumps_config",
|
|
42
|
+
"loads_config",
|
|
43
|
+
"serialize_config",
|
|
44
|
+
]
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Iterable
|
|
4
|
+
|
|
5
|
+
from .models import SurveyQuestion
|
|
6
|
+
from .providers.contracts import SurveyQuestionMeta
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def to_question_metas(questions: Iterable[SurveyQuestion]) -> list[SurveyQuestionMeta]:
|
|
10
|
+
return [
|
|
11
|
+
SurveyQuestionMeta(
|
|
12
|
+
num=int(question.num),
|
|
13
|
+
title=str(question.title),
|
|
14
|
+
type_code=str(question.type_code),
|
|
15
|
+
options=int(question.options),
|
|
16
|
+
rows=int(question.rows),
|
|
17
|
+
option_texts=list(question.option_texts),
|
|
18
|
+
row_texts=list(question.row_texts),
|
|
19
|
+
page=int(question.page),
|
|
20
|
+
provider_question_id=str(question.provider_question_id),
|
|
21
|
+
provider_page_id=str(question.provider_page_id),
|
|
22
|
+
required=bool(question.required),
|
|
23
|
+
display_num=question.display_num,
|
|
24
|
+
description=question.description,
|
|
25
|
+
forced_option_index=question.forced_option_index,
|
|
26
|
+
forced_option_text=question.forced_option_text,
|
|
27
|
+
forced_texts=list(question.forced_texts),
|
|
28
|
+
fillable_options=list(question.fillable_options),
|
|
29
|
+
attached_option_selects=[dict(item) for item in question.attached_option_selects],
|
|
30
|
+
has_attached_option_select=question.has_attached_option_select,
|
|
31
|
+
rating_max=question.rating_max,
|
|
32
|
+
text_inputs=question.text_inputs,
|
|
33
|
+
text_input_labels=list(question.text_input_labels),
|
|
34
|
+
is_multi_text=question.is_multi_text,
|
|
35
|
+
question_media=[dict(item) for item in question.question_media],
|
|
36
|
+
slider_min=question.slider_min,
|
|
37
|
+
slider_max=question.slider_max,
|
|
38
|
+
slider_step=question.slider_step,
|
|
39
|
+
multi_min_limit=question.multi_min_limit,
|
|
40
|
+
multi_max_limit=question.multi_max_limit,
|
|
41
|
+
provider_type=question.provider_type,
|
|
42
|
+
provider_page_raw=question.provider_page_raw,
|
|
43
|
+
is_text_like=bool(question.is_text_like),
|
|
44
|
+
is_location=bool(question.is_location),
|
|
45
|
+
is_description=bool(question.is_description),
|
|
46
|
+
is_rating=bool(question.is_rating),
|
|
47
|
+
is_slider_matrix=bool(question.is_slider_matrix),
|
|
48
|
+
has_jump=bool(question.has_jump),
|
|
49
|
+
jump_rules=[dict(item) for item in question.jump_rules],
|
|
50
|
+
has_display_condition=bool(question.has_display_condition),
|
|
51
|
+
display_conditions=[dict(item) for item in question.display_conditions],
|
|
52
|
+
has_dependent_display_logic=bool(question.has_dependent_display_logic),
|
|
53
|
+
controls_display_targets=[dict(item) for item in question.controls_display_targets],
|
|
54
|
+
logic_parse_status=str(question.logic_parse_status),
|
|
55
|
+
unsupported=bool(question.unsupported),
|
|
56
|
+
unsupported_reason=str(question.unsupported_reason),
|
|
57
|
+
)
|
|
58
|
+
for question in questions
|
|
59
|
+
]
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import random
|
|
4
|
+
from collections.abc import Sequence
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def choose_weighted(weights: Sequence[float], *, rng: random.Random | None = None) -> int:
|
|
8
|
+
"""Return an index using non-negative relative weights."""
|
|
9
|
+
values = [max(0.0, float(value)) for value in weights]
|
|
10
|
+
if not values or sum(values) <= 0:
|
|
11
|
+
raise ValueError("至少需要一个正权重")
|
|
12
|
+
picker = rng or random
|
|
13
|
+
return picker.choices(range(len(values)), weights=values, k=1)[0]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def limit_multiple_selection(indices: Sequence[int], *, minimum: int = 1, maximum: int | None = None) -> tuple[int, ...]:
|
|
17
|
+
"""Normalize multiple-choice selections to a bounded, duplicate-free tuple."""
|
|
18
|
+
unique = tuple(dict.fromkeys(int(index) for index in indices))
|
|
19
|
+
if len(unique) < minimum:
|
|
20
|
+
raise ValueError("多选答案少于最小选择数")
|
|
21
|
+
if maximum is not None and len(unique) > maximum:
|
|
22
|
+
return unique[:maximum]
|
|
23
|
+
return unique
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Small compatibility surface used by the migrated provider core."""
|
surveycore/app/config.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
USER_AGENT_PRESETS = {
|
|
6
|
+
"pc_web": {
|
|
7
|
+
"label": "电脑网页端",
|
|
8
|
+
"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
|
|
9
|
+
},
|
|
10
|
+
"mobile_android": {
|
|
11
|
+
"label": "安卓手机浏览器",
|
|
12
|
+
"ua": "Mozilla/5.0 (Linux; Android 16; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Mobile Safari/537.36",
|
|
13
|
+
},
|
|
14
|
+
"wechat_android": {
|
|
15
|
+
"label": "安卓微信端",
|
|
16
|
+
"ua": "Mozilla/5.0 (Linux; Android 16; Pixel 8; wv) AppleWebKit/537.36 Version/4.0 Chrome/121.0.0.0 Mobile Safari/537.36 MicroMessenger/8.0.43",
|
|
17
|
+
},
|
|
18
|
+
}
|
|
19
|
+
DEFAULT_USER_AGENT = USER_AGENT_PRESETS["pc_web"]["ua"]
|
|
20
|
+
DEFAULT_HTTP_HEADERS = {
|
|
21
|
+
"User-Agent": DEFAULT_USER_AGENT,
|
|
22
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
23
|
+
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
24
|
+
}
|
|
25
|
+
DEFAULT_FILL_TEXT = "无"
|
|
26
|
+
DIMENSION_UNGROUPED = "未分组"
|
|
27
|
+
_HTML_SPACE_RE = re.compile(r"\s+")
|
|
28
|
+
STOP_FORCE_WAIT_SECONDS = 0.3
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from importlib.resources import files
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_resource_path(relative_path: str) -> str:
|
|
7
|
+
"""Return a packaged resource path for legacy helpers."""
|
|
8
|
+
path = files("surveycore").joinpath(str(relative_path).replace("\\", "/"))
|
|
9
|
+
return str(path)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def get_assets_directory() -> str:
|
|
13
|
+
return str(files("surveycore").joinpath("data"))
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
__all__ = ["get_resource_path", "get_assets_directory"]
|
surveycore/client.py
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import random
|
|
5
|
+
import time
|
|
6
|
+
from collections.abc import AsyncIterator
|
|
7
|
+
from dataclasses import replace
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
|
|
12
|
+
from .core.questions.config import (
|
|
13
|
+
QuestionEntry,
|
|
14
|
+
build_default_question_entries,
|
|
15
|
+
configure_probabilities,
|
|
16
|
+
)
|
|
17
|
+
from .core.task import ExecutionConfig, ExecutionState
|
|
18
|
+
from .errors import SurveyParseError, SurveyUnavailableError, UnsupportedSurveyError
|
|
19
|
+
from .models import (
|
|
20
|
+
RunConfig,
|
|
21
|
+
SubmissionRequest,
|
|
22
|
+
SubmissionResult,
|
|
23
|
+
SurveyDefinition,
|
|
24
|
+
SurveyQuestion,
|
|
25
|
+
)
|
|
26
|
+
from .network.http import bind
|
|
27
|
+
from .providers.common import (
|
|
28
|
+
detect_survey_provider,
|
|
29
|
+
normalize_survey_parse_url,
|
|
30
|
+
)
|
|
31
|
+
from .providers.registry import fill_survey_http, parse_survey
|
|
32
|
+
from .runtime import completed_results
|
|
33
|
+
from .transport import require_async_client
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _question_from_meta(item: Any) -> SurveyQuestion:
|
|
37
|
+
def dicts(value: Any) -> tuple[dict[str, Any], ...]:
|
|
38
|
+
return tuple(dict(entry) for entry in list(value or []) if isinstance(entry, dict))
|
|
39
|
+
|
|
40
|
+
return SurveyQuestion(
|
|
41
|
+
num=int(getattr(item, "num", 0) or 0),
|
|
42
|
+
title=str(getattr(item, "title", "") or ""),
|
|
43
|
+
type_code=str(getattr(item, "type_code", "0") or "0"),
|
|
44
|
+
provider_question_id=str(getattr(item, "provider_question_id", "") or ""),
|
|
45
|
+
provider_page_id=str(getattr(item, "provider_page_id", "") or ""),
|
|
46
|
+
options=int(getattr(item, "options", 0) or 0),
|
|
47
|
+
rows=int(getattr(item, "rows", 1) or 1),
|
|
48
|
+
option_texts=tuple(str(value or "") for value in list(getattr(item, "option_texts", []) or [])),
|
|
49
|
+
row_texts=tuple(str(value or "") for value in list(getattr(item, "row_texts", []) or [])),
|
|
50
|
+
page=int(getattr(item, "page", 1) or 1),
|
|
51
|
+
required=bool(getattr(item, "required", False)),
|
|
52
|
+
display_num=getattr(item, "display_num", None),
|
|
53
|
+
description=str(getattr(item, "description", "") or ""),
|
|
54
|
+
forced_option_index=getattr(item, "forced_option_index", None),
|
|
55
|
+
forced_option_text=str(getattr(item, "forced_option_text", "") or ""),
|
|
56
|
+
forced_texts=tuple(str(v or "") for v in (getattr(item, "forced_texts", []) or [])),
|
|
57
|
+
fillable_options=tuple(int(v) for v in (getattr(item, "fillable_options", []) or [])),
|
|
58
|
+
attached_option_selects=dicts(getattr(item, "attached_option_selects", [])),
|
|
59
|
+
has_attached_option_select=bool(getattr(item, "has_attached_option_select", False)),
|
|
60
|
+
rating_max=int(getattr(item, "rating_max", 0) or 0),
|
|
61
|
+
text_inputs=int(getattr(item, "text_inputs", 0) or 0),
|
|
62
|
+
text_input_labels=tuple(str(v or "") for v in (getattr(item, "text_input_labels", []) or [])),
|
|
63
|
+
is_multi_text=bool(getattr(item, "is_multi_text", False)),
|
|
64
|
+
question_media=dicts(getattr(item, "question_media", [])),
|
|
65
|
+
slider_min=getattr(item, "slider_min", None),
|
|
66
|
+
slider_max=getattr(item, "slider_max", None),
|
|
67
|
+
slider_step=getattr(item, "slider_step", None),
|
|
68
|
+
multi_min_limit=getattr(item, "multi_min_limit", None),
|
|
69
|
+
multi_max_limit=getattr(item, "multi_max_limit", None),
|
|
70
|
+
provider_type=str(getattr(item, "provider_type", "") or ""),
|
|
71
|
+
provider_page_raw=getattr(item, "provider_page_raw", None),
|
|
72
|
+
is_text_like=bool(getattr(item, "is_text_like", False)),
|
|
73
|
+
is_location=bool(getattr(item, "is_location", False)),
|
|
74
|
+
is_description=bool(getattr(item, "is_description", False)),
|
|
75
|
+
is_rating=bool(getattr(item, "is_rating", False)),
|
|
76
|
+
is_slider_matrix=bool(getattr(item, "is_slider_matrix", False)),
|
|
77
|
+
has_jump=bool(getattr(item, "has_jump", False)),
|
|
78
|
+
jump_rules=dicts(getattr(item, "jump_rules", [])),
|
|
79
|
+
has_display_condition=bool(getattr(item, "has_display_condition", False)),
|
|
80
|
+
display_conditions=dicts(getattr(item, "display_conditions", [])),
|
|
81
|
+
has_dependent_display_logic=bool(getattr(item, "has_dependent_display_logic", False)),
|
|
82
|
+
controls_display_targets=dicts(getattr(item, "controls_display_targets", [])),
|
|
83
|
+
logic_parse_status=str(getattr(item, "logic_parse_status", "unknown") or "unknown"),
|
|
84
|
+
unsupported=bool(getattr(item, "unsupported", False)),
|
|
85
|
+
unsupported_reason=str(getattr(item, "unsupported_reason", "") or ""),
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class SurveyClient:
|
|
90
|
+
"""Async survey parser and HTTP submitter using a caller-owned client."""
|
|
91
|
+
|
|
92
|
+
def __init__(self, http_client: httpx.AsyncClient) -> None:
|
|
93
|
+
self.http_client = require_async_client(http_client)
|
|
94
|
+
|
|
95
|
+
@staticmethod
|
|
96
|
+
def _resolve_provider(config: RunConfig) -> str:
|
|
97
|
+
detected = detect_survey_provider(config.url, default="")
|
|
98
|
+
provider = config.provider or detected
|
|
99
|
+
if provider not in {"wjx", "qq", "credamo"}:
|
|
100
|
+
raise UnsupportedSurveyError(f"不支持的问卷平台:{provider or config.url}")
|
|
101
|
+
if config.provider and detected and config.provider != detected:
|
|
102
|
+
raise UnsupportedSurveyError(f"问卷链接平台与配置不一致:{config.provider} != {detected}")
|
|
103
|
+
return provider
|
|
104
|
+
|
|
105
|
+
async def parse(self, url: str) -> SurveyDefinition:
|
|
106
|
+
normalized = normalize_survey_parse_url(url)
|
|
107
|
+
provider = detect_survey_provider(normalized, default="")
|
|
108
|
+
if not provider:
|
|
109
|
+
raise UnsupportedSurveyError(f"不支持的问卷链接:{url}")
|
|
110
|
+
try:
|
|
111
|
+
async with bind(self.http_client):
|
|
112
|
+
definition = await parse_survey(normalized)
|
|
113
|
+
except (UnsupportedSurveyError, SurveyUnavailableError):
|
|
114
|
+
raise
|
|
115
|
+
except Exception as exc:
|
|
116
|
+
message = str(exc) or "问卷解析失败"
|
|
117
|
+
if any(marker in message for marker in ("未开放", "已停止", "已暂停", "到期")):
|
|
118
|
+
raise SurveyUnavailableError(message) from exc
|
|
119
|
+
raise SurveyParseError(message) from exc
|
|
120
|
+
questions = tuple(_question_from_meta(item) for item in definition.questions)
|
|
121
|
+
return SurveyDefinition(provider=definition.provider, title=definition.title, url=normalized, questions=questions)
|
|
122
|
+
|
|
123
|
+
def _execution_config(self, config: RunConfig, survey: SurveyDefinition | None) -> ExecutionConfig:
|
|
124
|
+
provider = self._resolve_provider(config)
|
|
125
|
+
from ._question_metas import to_question_metas
|
|
126
|
+
legacy_questions = to_question_metas(survey.questions if survey else ())
|
|
127
|
+
entries: list[QuestionEntry] = [QuestionEntry(**{
|
|
128
|
+
key: getattr(item, key) for key in QuestionEntry.__dataclass_fields__
|
|
129
|
+
if hasattr(item, key)
|
|
130
|
+
}) for item in config.questions]
|
|
131
|
+
if not entries and legacy_questions:
|
|
132
|
+
entries = list(build_default_question_entries(legacy_questions, survey_url=config.url, existing_entries=[]))
|
|
133
|
+
execution = ExecutionConfig(
|
|
134
|
+
url=str(config.url or ""),
|
|
135
|
+
survey_provider=provider,
|
|
136
|
+
target_num=max(1, int(config.target)),
|
|
137
|
+
num_threads=max(1, int(config.concurrency)),
|
|
138
|
+
answer_duration_range_seconds=(config.answer_duration[0], config.answer_duration[1]),
|
|
139
|
+
submit_interval_range_seconds=(config.submit_interval[0], config.submit_interval[1]),
|
|
140
|
+
random_user_agent_enabled=bool(config.random_user_agent),
|
|
141
|
+
user_agent_ratios=dict(config.user_agent_ratios),
|
|
142
|
+
answer_rules=list(config.answer_rules),
|
|
143
|
+
psycho_target_alpha=float(config.target_alpha),
|
|
144
|
+
stop_on_fail_enabled=False,
|
|
145
|
+
pause_on_aliyun_captcha=False,
|
|
146
|
+
)
|
|
147
|
+
execution.answer_generator = config.answer_generator # type: ignore[attr-defined]
|
|
148
|
+
execution.question_entries = entries # type: ignore[attr-defined]
|
|
149
|
+
execution.questions_metadata = {int(item.num): item for item in legacy_questions if int(getattr(item, "num", 0) or 0) > 0}
|
|
150
|
+
configure_probabilities(entries, execution, reliability_mode_enabled=bool(config.reliability_mode))
|
|
151
|
+
return execution
|
|
152
|
+
|
|
153
|
+
async def submit(self, request: SubmissionRequest | RunConfig) -> SubmissionResult:
|
|
154
|
+
config = request.config if isinstance(request, SubmissionRequest) else request
|
|
155
|
+
survey = request.survey if isinstance(request, SubmissionRequest) else None
|
|
156
|
+
started = time.perf_counter()
|
|
157
|
+
provider = self._resolve_provider(config)
|
|
158
|
+
execution = self._execution_config(config, survey)
|
|
159
|
+
state = ExecutionState(config=execution)
|
|
160
|
+
try:
|
|
161
|
+
async with bind(self.http_client):
|
|
162
|
+
success = await fill_survey_http(
|
|
163
|
+
execution,
|
|
164
|
+
state,
|
|
165
|
+
stop_signal=asyncio.Event(),
|
|
166
|
+
thread_name="surveycore",
|
|
167
|
+
provider=provider,
|
|
168
|
+
)
|
|
169
|
+
except Exception as exc: # noqa: BLE001 - one attempt must become one failed result
|
|
170
|
+
return SubmissionResult(0, provider, False, time.perf_counter() - started, str(exc) or "提交失败")
|
|
171
|
+
return SubmissionResult(0, provider, bool(success), time.perf_counter() - started, None if success else "平台未确认提交成功")
|
|
172
|
+
|
|
173
|
+
async def run(self, config: RunConfig) -> AsyncIterator[SubmissionResult]:
|
|
174
|
+
survey = await self.parse(config.url)
|
|
175
|
+
async def attempt(index: int) -> SubmissionResult:
|
|
176
|
+
if index > 1:
|
|
177
|
+
low, high = config.submit_interval
|
|
178
|
+
if high > 0:
|
|
179
|
+
await asyncio.sleep(random.uniform(low, high))
|
|
180
|
+
result = await self.submit(SubmissionRequest(config=replace(config, target=1), survey=survey))
|
|
181
|
+
return replace(result, attempt=index)
|
|
182
|
+
|
|
183
|
+
async for result in completed_results(config.target, config.concurrency, attempt):
|
|
184
|
+
yield result
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
__all__ = ["SurveyClient"]
|
surveycore/config.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from dataclasses import fields
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from .errors import ConfigError
|
|
8
|
+
from .models import QuestionConfig, RunConfig
|
|
9
|
+
|
|
10
|
+
SCHEMA_VERSION = 1
|
|
11
|
+
_RUN_FIELDS = {field.name for field in fields(RunConfig) if field.name != "answer_generator"}
|
|
12
|
+
_QUESTION_FIELDS = {
|
|
13
|
+
field.name for field in fields(QuestionConfig)
|
|
14
|
+
if field.name != "ai_enabled"
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _reject_unknown(payload: dict[str, Any], allowed: set[str], label: str) -> None:
|
|
19
|
+
unknown = sorted(set(payload) - allowed)
|
|
20
|
+
if unknown:
|
|
21
|
+
raise ConfigError(f"{label}包含未知字段:{', '.join(unknown)}")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _range(value: Any, label: str) -> tuple[int, int]:
|
|
25
|
+
if not isinstance(value, list) or len(value) != 2:
|
|
26
|
+
raise ConfigError(f"{label}必须是两个整数组成的数组")
|
|
27
|
+
if any(isinstance(item, bool) or not isinstance(item, int) for item in value):
|
|
28
|
+
raise ConfigError(f"{label}必须是整数")
|
|
29
|
+
low, high = value
|
|
30
|
+
if low < 0 or high < low:
|
|
31
|
+
raise ConfigError(f"{label}范围无效")
|
|
32
|
+
return low, high
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _string_range(value: Any, label: str) -> tuple[str, str]:
|
|
36
|
+
if not isinstance(value, list) or len(value) != 2 or any(not isinstance(item, str) for item in value):
|
|
37
|
+
raise ConfigError(f"{label}必须是两个字符串组成的数组")
|
|
38
|
+
return value[0], value[1]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _strict_bool(value: Any, label: str) -> bool:
|
|
42
|
+
if not isinstance(value, bool):
|
|
43
|
+
raise ConfigError(f"{label}必须是布尔值")
|
|
44
|
+
return value
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _strict_float(value: Any, label: str) -> float:
|
|
48
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
49
|
+
raise ConfigError(f"{label}必须是数字")
|
|
50
|
+
return float(value)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _mapping(value: Any, label: str) -> dict[str, int]:
|
|
54
|
+
if not isinstance(value, dict) or any(not isinstance(key, str) or isinstance(item, bool) or not isinstance(item, int) for key, item in value.items()):
|
|
55
|
+
raise ConfigError(f"{label}必须是字符串到整数的对象")
|
|
56
|
+
return dict(value)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _list_of_dicts(value: Any, label: str) -> list[dict[str, Any]]:
|
|
60
|
+
if not isinstance(value, list) or any(not isinstance(item, dict) for item in value):
|
|
61
|
+
raise ConfigError(f"{label}必须是对象数组")
|
|
62
|
+
return [dict(item) for item in value]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def serialize_config(config: RunConfig) -> dict[str, Any]:
|
|
66
|
+
if not isinstance(config, RunConfig):
|
|
67
|
+
raise ConfigError("只支持序列化 RunConfig")
|
|
68
|
+
return {
|
|
69
|
+
"schema_version": SCHEMA_VERSION,
|
|
70
|
+
"url": str(config.url),
|
|
71
|
+
"questions": [{key: getattr(item, key) for key in sorted(_QUESTION_FIELDS)} for item in config.questions],
|
|
72
|
+
"target": int(config.target),
|
|
73
|
+
"concurrency": int(config.concurrency),
|
|
74
|
+
"provider": config.provider,
|
|
75
|
+
"answer_duration": list(config.answer_duration),
|
|
76
|
+
"submit_interval": list(config.submit_interval),
|
|
77
|
+
"answer_datetime_window": list(config.answer_datetime_window),
|
|
78
|
+
"random_user_agent": bool(config.random_user_agent),
|
|
79
|
+
"user_agent_ratios": dict(config.user_agent_ratios),
|
|
80
|
+
"reliability_mode": bool(config.reliability_mode),
|
|
81
|
+
"target_alpha": float(config.target_alpha),
|
|
82
|
+
"answer_rules": list(config.answer_rules),
|
|
83
|
+
"reverse_fill_enabled": bool(config.reverse_fill_enabled),
|
|
84
|
+
"reverse_fill_source_path": str(config.reverse_fill_source_path),
|
|
85
|
+
"reverse_fill_format": str(config.reverse_fill_format),
|
|
86
|
+
"reverse_fill_start_row": int(config.reverse_fill_start_row),
|
|
87
|
+
"reverse_fill_threads": int(config.reverse_fill_threads),
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def deserialize_config(payload: Any) -> RunConfig:
|
|
92
|
+
if not isinstance(payload, dict):
|
|
93
|
+
raise ConfigError("配置顶层必须是对象")
|
|
94
|
+
version = payload.get("schema_version")
|
|
95
|
+
if version != SCHEMA_VERSION:
|
|
96
|
+
raise ConfigError(f"不支持的配置 schema 版本:{version!r}")
|
|
97
|
+
_reject_unknown(payload, {"schema_version", *_RUN_FIELDS}, "配置")
|
|
98
|
+
raw_questions = payload.get("questions", [])
|
|
99
|
+
if not isinstance(raw_questions, list):
|
|
100
|
+
raise ConfigError("questions 必须是数组")
|
|
101
|
+
questions: list[QuestionConfig] = []
|
|
102
|
+
for index, raw in enumerate(raw_questions, start=1):
|
|
103
|
+
if not isinstance(raw, dict):
|
|
104
|
+
raise ConfigError(f"questions[{index}] 必须是对象")
|
|
105
|
+
_reject_unknown(raw, _QUESTION_FIELDS, f"questions[{index}]")
|
|
106
|
+
if "question_type" not in raw:
|
|
107
|
+
raise ConfigError(f"questions[{index}] 缺少 question_type")
|
|
108
|
+
if not isinstance(raw["question_type"], str) or not raw["question_type"].strip():
|
|
109
|
+
raise ConfigError(f"questions[{index}].question_type 必须是非空字符串")
|
|
110
|
+
if "probabilities" in raw and raw["probabilities"] is not None and not isinstance(raw["probabilities"], (list, int, float)):
|
|
111
|
+
raise ConfigError(f"questions[{index}].probabilities 类型无效")
|
|
112
|
+
questions.append(QuestionConfig(**raw))
|
|
113
|
+
target = payload.get("target", 1)
|
|
114
|
+
concurrency = payload.get("concurrency", 1)
|
|
115
|
+
if any(isinstance(value, bool) or not isinstance(value, int) for value in (target, concurrency)):
|
|
116
|
+
raise ConfigError("target 和 concurrency 必须是整数")
|
|
117
|
+
if target < 1 or concurrency < 1:
|
|
118
|
+
raise ConfigError("target 和 concurrency 必须大于 0")
|
|
119
|
+
provider = payload.get("provider")
|
|
120
|
+
if provider is not None and (not isinstance(provider, str) or provider not in {"wjx", "qq", "credamo"}):
|
|
121
|
+
raise ConfigError("provider 必须是 wjx、qq 或 credamo")
|
|
122
|
+
reverse_fill_start_row = payload.get("reverse_fill_start_row", 1)
|
|
123
|
+
reverse_fill_threads = payload.get("reverse_fill_threads", 1)
|
|
124
|
+
if any(isinstance(value, bool) or not isinstance(value, int) or value < 1 for value in (reverse_fill_start_row, reverse_fill_threads)):
|
|
125
|
+
raise ConfigError("反填行号和线程数必须是正整数")
|
|
126
|
+
return RunConfig(
|
|
127
|
+
url=_required_string(payload.get("url", ""), "url"),
|
|
128
|
+
questions=questions,
|
|
129
|
+
target=target,
|
|
130
|
+
concurrency=concurrency,
|
|
131
|
+
provider=provider,
|
|
132
|
+
answer_duration=_range(payload.get("answer_duration", [60, 120]), "answer_duration"),
|
|
133
|
+
submit_interval=_range(payload.get("submit_interval", [0, 0]), "submit_interval"),
|
|
134
|
+
answer_datetime_window=_string_range(payload.get("answer_datetime_window", ["", ""]), "answer_datetime_window"),
|
|
135
|
+
random_user_agent=_strict_bool(payload.get("random_user_agent", False), "random_user_agent"),
|
|
136
|
+
user_agent_ratios=_mapping(payload.get("user_agent_ratios", {"wechat": 33, "mobile": 33, "pc": 34}), "user_agent_ratios"),
|
|
137
|
+
reliability_mode=_strict_bool(payload.get("reliability_mode", True), "reliability_mode"),
|
|
138
|
+
target_alpha=_strict_float(payload.get("target_alpha", 0.85), "target_alpha"),
|
|
139
|
+
answer_rules=_list_of_dicts(payload.get("answer_rules", []), "answer_rules"),
|
|
140
|
+
reverse_fill_enabled=_strict_bool(payload.get("reverse_fill_enabled", False), "reverse_fill_enabled"),
|
|
141
|
+
reverse_fill_source_path=_required_string(payload.get("reverse_fill_source_path", ""), "reverse_fill_source_path", allow_empty=True),
|
|
142
|
+
reverse_fill_format=_required_string(payload.get("reverse_fill_format", "auto"), "reverse_fill_format"),
|
|
143
|
+
reverse_fill_start_row=reverse_fill_start_row,
|
|
144
|
+
reverse_fill_threads=reverse_fill_threads,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _required_string(value: Any, label: str, *, allow_empty: bool = False) -> str:
|
|
149
|
+
if not isinstance(value, str) or (not allow_empty and not value.strip()):
|
|
150
|
+
raise ConfigError(f"{label}必须是字符串")
|
|
151
|
+
return value.strip()
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def dumps_config(config: RunConfig, *, indent: int = 2) -> str:
|
|
155
|
+
return json.dumps(serialize_config(config), ensure_ascii=False, indent=indent)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def loads_config(value: str) -> RunConfig:
|
|
159
|
+
try:
|
|
160
|
+
payload = json.loads(value)
|
|
161
|
+
except json.JSONDecodeError as exc:
|
|
162
|
+
raise ConfigError(f"配置 JSON 无效:{exc}") from exc
|
|
163
|
+
return deserialize_config(payload)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
__all__ = ["SCHEMA_VERSION", "deserialize_config", "dumps_config", "loads_config", "serialize_config"]
|
|
File without changes
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any, Iterable
|
|
4
|
+
|
|
5
|
+
from .runtime import (
|
|
6
|
+
AIRuntimeError,
|
|
7
|
+
agenerate_ai_answer,
|
|
8
|
+
is_free_ai_option_fill_placeholder,
|
|
9
|
+
is_free_ai_text_placeholder,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
async def prefill_free_ai_answers_for_questions(questions: Iterable[Any], actions: Iterable[Any], ctx: Any, *, thread_name: str = "") -> None:
|
|
14
|
+
del questions
|
|
15
|
+
for action in actions:
|
|
16
|
+
if not any(is_free_ai_text_placeholder(item) for item in tuple(getattr(action, "text_values", ()) or ())):
|
|
17
|
+
continue
|
|
18
|
+
question = next((q for q in list(getattr(ctx.config, "questions_metadata", {}).values()) if int(getattr(q, "num", 0) or 0) == int(getattr(action, "question_num", 0) or 0)), None)
|
|
19
|
+
if question is None:
|
|
20
|
+
continue
|
|
21
|
+
answer = await agenerate_ai_answer(str(getattr(question, "title", "")), question_type="multi_fill_blank" if len(action.text_values) > 1 else "fill_blank", blank_count=len(action.text_values), question_number=action.question_num, ctx=ctx)
|
|
22
|
+
values = tuple(answer if isinstance(answer, list) else [answer])
|
|
23
|
+
ctx.set_free_ai_prefill_answers(thread_name, {int(action.question_num): values})
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def assert_no_free_ai_placeholders_in_actions(actions: Iterable[Any], *, provider_label: str = "问卷") -> None:
|
|
27
|
+
for action in actions:
|
|
28
|
+
text_values = tuple(getattr(action, "text_values", ()) or ())
|
|
29
|
+
raw_option_fill_pairs: Any = getattr(action, "option_fill_texts", ()) or ()
|
|
30
|
+
option_fill_values = tuple(
|
|
31
|
+
pair[1]
|
|
32
|
+
for pair in tuple(raw_option_fill_pairs)
|
|
33
|
+
if isinstance(pair, (list, tuple)) and len(pair) >= 2
|
|
34
|
+
)
|
|
35
|
+
if any(
|
|
36
|
+
is_free_ai_text_placeholder(item) or is_free_ai_option_fill_placeholder(item)
|
|
37
|
+
for item in text_values + option_fill_values
|
|
38
|
+
):
|
|
39
|
+
raise AIRuntimeError(f"{provider_label}存在未处理的答案生成占位符")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
__all__ = ["AIRuntimeError", "assert_no_free_ai_placeholders_in_actions", "prefill_free_ai_answers_for_questions"]
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
import re
|
|
5
|
+
from typing import Any, List, Optional
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class AIRuntimeError(RuntimeError):
|
|
9
|
+
pass
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def build_ai_question_prompt(question_title: str, *, description: str = "", question_number: int = 0) -> str:
|
|
13
|
+
title = re.sub(r"^\s*\*?\s*\d+[.、]?\s*", "", str(question_title or "").strip())
|
|
14
|
+
if not title:
|
|
15
|
+
title = f"第{int(question_number)}题" if int(question_number or 0) > 0 else ""
|
|
16
|
+
description = str(description or "").strip()
|
|
17
|
+
return f"{title}\n补充说明:{description}" if title and description else title
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def build_ai_option_fill_prompt(*, question_title: str, question_number: int, option_text: str = "") -> str:
|
|
21
|
+
prompt = build_ai_question_prompt(question_title, question_number=question_number)
|
|
22
|
+
return f"{prompt}\n当前选择项:{str(option_text or '').strip()}".strip()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def build_free_ai_text_placeholder(question_num: int, blank_index: int) -> str:
|
|
26
|
+
return f"__FREE_AI_TEXT__{int(question_num)}_{int(blank_index)}"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def build_free_ai_option_fill_placeholder(question_num: int, option_index: int) -> str:
|
|
30
|
+
return f"__FREE_AI_OPTION_FILL__{int(question_num)}_{int(option_index)}"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def is_free_ai_text_placeholder(value: object) -> bool:
|
|
34
|
+
return str(value or "").startswith("__FREE_AI_TEXT__")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def is_free_ai_option_fill_placeholder(value: object) -> bool:
|
|
38
|
+
return str(value or "").startswith("__FREE_AI_OPTION_FILL__")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
async def agenerate_ai_answer(
|
|
42
|
+
question_title: str,
|
|
43
|
+
*,
|
|
44
|
+
question_type: str = "fill_blank",
|
|
45
|
+
blank_count: Optional[int] = None,
|
|
46
|
+
description: str = "",
|
|
47
|
+
question_number: int = 0,
|
|
48
|
+
ctx: Any = None,
|
|
49
|
+
) -> str | List[str]:
|
|
50
|
+
callback = getattr(getattr(ctx, "config", None), "answer_generator", None)
|
|
51
|
+
if not callable(callback):
|
|
52
|
+
raise AIRuntimeError("文本题需要通过 answer_generator 注入生成器")
|
|
53
|
+
prompt = build_ai_question_prompt(
|
|
54
|
+
question_title,
|
|
55
|
+
description=description,
|
|
56
|
+
question_number=question_number,
|
|
57
|
+
)
|
|
58
|
+
try:
|
|
59
|
+
value = callback(prompt, question_type=question_type, blank_count=blank_count, question_number=question_number)
|
|
60
|
+
if inspect.isawaitable(value):
|
|
61
|
+
value = await value
|
|
62
|
+
except Exception as exc:
|
|
63
|
+
raise AIRuntimeError(f"答案生成器调用失败:{exc}") from exc
|
|
64
|
+
if question_type == "multi_fill_blank":
|
|
65
|
+
values = value if isinstance(value, list) else [part.strip() for part in str(value or "").split("||")]
|
|
66
|
+
values = [str(item or "").strip() for item in values if str(item or "").strip()]
|
|
67
|
+
if blank_count and len(values) != int(blank_count):
|
|
68
|
+
raise AIRuntimeError(f"答案生成器返回 {len(values)} 项,期望 {int(blank_count)} 项")
|
|
69
|
+
if not values:
|
|
70
|
+
raise AIRuntimeError("答案生成器返回空内容")
|
|
71
|
+
return values
|
|
72
|
+
text = str(value or "").strip()
|
|
73
|
+
if not text:
|
|
74
|
+
raise AIRuntimeError("答案生成器返回空内容")
|
|
75
|
+
return text
|
|
76
|
+
|