mindforge-memory 6.4.1__py3-none-any.whl

mindforge/modules/connectors.py

@@ -0,0 +1,331 @@
+# -*- coding: utf-8 -*-
+"""数据连接器框架（v5.7.7 新增）
+
+插件式 Connector 接口：统一「连接 → 摄取 → 返回统计」契约。
+内置 File / Json / Csv / Markdown / Url 五类连接器，可注册自定义实现。
+
+与既有导入能力的关系：连接器是统一入口与扩展点，
+JSON / CSV 摄取复用 facade 的 import_json / import_csv（不重复实现），
+Markdown / 文本按段落摄取，URL 摄取带基础 SSRF 防护。
+"""
+
+from __future__ import annotations
+
+import http.client
+import re
+import socket
+import ipaddress
+import ssl
+import urllib.parse
+import urllib.request
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Type
+
+
+class BaseConnector(ABC):
+    """连接器基类：实现 name / description / ingest 即可注册。"""
+
+    name: str = ""
+    description: str = ""
+
+    @abstractmethod
+    def ingest(self, mf: Any, source: str, **kwargs: Any) -> Dict[str, Any]:
+        """摄取入口。
+
+        Args:
+            mf: MindForge 门面实例（用于调用导入能力）
+            source: 数据源（文件路径 / URL 等）
+            **kwargs: 连接器特定参数
+
+        Returns:
+            统计结果 dict（至少含 imported / skipped / failed / deduped）
+        """
+
+
+_CONNECTORS: Dict[str, Type[BaseConnector]] = {}
+
+
+def register_connector(cls: Type[BaseConnector]) -> Type[BaseConnector]:
+    """注册连接器（可作装饰器使用）。"""
+    if not getattr(cls, "name", ""):
+        raise ValueError("连接器必须定义 name")
+    _CONNECTORS[cls.name.lower()] = cls
+    return cls
+
+
+def get_connector(name: str) -> Optional[Type[BaseConnector]]:
+    """按名称查找连接器（大小写不敏感）。"""
+    return _CONNECTORS.get(str(name or "").lower())
+
+
+def list_connector_names() -> List[str]:
+    return sorted(_CONNECTORS.keys())
+
+
+def list_connectors() -> List[Dict[str, str]]:
+    return [{"name": n, "description": _CONNECTORS[n].description}
+            for n in list_connector_names()]
+
+
+# ---------------- 内置连接器 ----------------
+
+@register_connector
+class JsonConnector(BaseConnector):
+    name = "json"
+    description = "从 JSON 文件导入记忆（复用 import_json，支持去重/目标层级）"
+
+    def ingest(self, mf, source, **kwargs):
+        # import_json 不支持 category/source_agent，过滤后再透传
+        kwargs.pop("category", None)
+        kwargs.pop("source_agent", None)
+        return mf.import_json(source, **kwargs)
+
+
+@register_connector
+class CsvConnector(BaseConnector):
+    name = "csv"
+    description = "从 CSV 文件导入记忆（复用 import_csv）"
+
+    def ingest(self, mf, source, **kwargs):
+        kwargs.pop("category", None)
+        kwargs.pop("source_agent", None)
+        return mf.import_csv(source, **kwargs)
+
+
+@register_connector
+class MarkdownConnector(BaseConnector):
+    name = "markdown"
+    description = "从 Markdown / 文本文件按段落导入记忆"
+
+    def ingest(self, mf, source, **kwargs):
+        import os
+        path = os.path.abspath(source)
+        if not os.path.isfile(path):
+            raise FileNotFoundError(f"文件不存在: {path}")
+        with open(path, "r", encoding="utf-8") as f:
+            text = f.read()
+        target_layer = kwargs.get("target_layer")
+        category = kwargs.get("category") or "import"
+        source_agent = kwargs.get("source_agent") or "connector:markdown"
+        stats = {"imported": 0, "skipped": 0, "failed": 0, "deduped": 0}
+        for para in [p.strip() for p in text.split("\n\n") if p.strip()]:
+            if len(para) < 3:
+                continue
+            try:
+                mf.add(para, category=category, source_agent=source_agent,
+                       layer=target_layer)
+                stats["imported"] += 1
+            except Exception:
+                stats["failed"] += 1
+        return stats
+
+
+@register_connector
+class FileConnector(BaseConnector):
+    name = "file"
+    description = "按扩展名自动分派（.json / .csv / .md / .txt）"
+
+    def ingest(self, mf, source, **kwargs):
+        import os
+        ext = os.path.splitext(str(source))[1].lower()
+        mapping = {".json": "json", ".csv": "csv",
+                   ".md": "markdown", ".txt": "markdown"}
+        name = mapping.get(ext)
+        if not name:
+            raise ValueError(
+                f"不支持的文件类型: {ext}（支持 .json / .csv / .md / .txt）")
+        return _CONNECTORS[name]().ingest(mf, source, **kwargs)
+
+
+def _is_private_ip(host: str) -> bool:
+    """SSRF 防护：拒绝内网/回环/链路本地/保留/组播/未指定地址（v5.8.3 P1-1）。
+
+    用 ipaddress 标准属性判定替代字符串前缀黑名单，覆盖此前可稳定绕过的：
+      - 十进制整数 / 八进制编码 IP（getaddrinfo 解析后即真实内网地址）
+      - IPv4-mapped IPv6（::ffff:a9fe:a9fe 解包后按 IPv4 判定）
+      - CGNAT 100.64.0.0/10（Python<3.13 的 is_private 不含该段）
+      - 0.0.0.0（Linux 上 connect 它即访问 127.0.0.1）
+      - IPv6 链路本地 fe80::/10 与 ULA fc00::/7（除 ::1 外的内网 IPv6）
+    """
+    try:
+        infos = socket.getaddrinfo(host, None)
+    except Exception:
+        return True  # 解析失败按不安全处理（fail-closed）
+    for info in infos:
+        raw = info[4][0]
+        try:
+            ip = ipaddress.ip_address(raw)
+        except ValueError:
+            return True
+        # IPv4-mapped IPv6：解包后按 IPv4 判定（::ffff:169.254.169.254）
+        if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
+            ip = ip.ipv4_mapped
+        if (ip.is_private or ip.is_loopback or ip.is_link_local
+                or ip.is_multicast or ip.is_reserved or ip.is_unspecified):
+            return True
+        # CGNAT 100.64.0.0/10：低版本 is_private 不含，显式补判
+        if ip in ipaddress.ip_network("100.64.0.0/10"):
+            return True
+    return False
+
+
+def _read_limited(resp, max_bytes: int = 5 * 1024 * 1024) -> bytes:
+    """流式分块读取远端响应，累计超过 max_bytes 即中止（v5.8.3 P2-1）。
+
+    此前 resp.read() 无上限：攻击者可返回超大 body 造成进程内存膨胀，
+    入库截断（100000 字符）发生在完整读入内存之后。现在超限直接拒绝。
+    """
+    chunks = []
+    total = 0
+    while True:
+        chunk = resp.read(65536)
+        if not chunk:
+            break
+        total += len(chunk)
+        if total > max_bytes:
+            raise ValueError(
+                f"响应体超过 {max_bytes} 字节上限，已中止读取（防内存放大）")
+        chunks.append(chunk)
+    return b"".join(chunks)
+
+
+class _FixedIPHTTPConnection(http.client.HTTPConnection):
+    """connect 时用预解析并校验过的 IP 直连，Host header 仍用原域名（v5.8.9 防 DNS rebinding）。"""
+    def __init__(self, host, port=None, timeout=15, fixed_ip=None, **kw):
+        super().__init__(host, port=port, timeout=timeout, **kw)
+        self._fixed_ip = fixed_ip
+
+    def connect(self):
+        sock = socket.create_connection((self._fixed_ip or self.host, self.port),
+                                       timeout=self.timeout)
+        self.sock = sock
+        if self._tunnel_host:
+            self._tunnel()
+
+
+class _FixedIPHTTPSConnection(http.client.HTTPSConnection):
+    """HTTPS 固定 IP 直连：TCP 连 IP，TLS SNI/证书校验仍用原域名。"""
+    def __init__(self, host, port=None, timeout=15, fixed_ip=None, **kw):
+        super().__init__(host, port=port, timeout=timeout, **kw)
+        self._fixed_ip = fixed_ip
+
+    def connect(self):
+        sock = socket.create_connection((self._fixed_ip or self.host, self.port),
+                                       timeout=self.timeout)
+        if self._tunnel_host:
+            self.sock = sock
+            self._tunnel()
+        else:
+            ctx = getattr(self, "_context", None) or ssl.create_default_context()
+            self.sock = ctx.wrap_socket(sock, server_hostname=self.host)
+
+
+class _FixedIPHTTPHandler(urllib.request.HTTPHandler):
+    def __init__(self, fixed_ip):
+        super().__init__()
+        self._fixed_ip = fixed_ip
+
+    def http_open(self, req):
+        return self.do_open(
+            lambda host, **kw: _FixedIPHTTPConnection(host, fixed_ip=self._fixed_ip, **kw), req)
+
+
+class _FixedIPHTTPSHandler(urllib.request.HTTPSHandler):
+    def __init__(self, fixed_ip):
+        super().__init__()
+        self._fixed_ip = fixed_ip
+
+    def https_open(self, req):
+        return self.do_open(
+            lambda host, **kw: _FixedIPHTTPSConnection(host, fixed_ip=self._fixed_ip, **kw), req)
+
+
+def _resolve_safe_ip(host: str) -> Optional[str]:
+    """解析 host，校验所有 A/AAAA 记录均为公网，返回第一个可用 IP。"""
+    try:
+        infos = socket.getaddrinfo(host, None)
+    except Exception:
+        return None
+    for info in infos:
+        raw = info[4][0]
+        try:
+            ip = ipaddress.ip_address(raw)
+        except ValueError:
+            return None
+        if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
+            ip = ip.ipv4_mapped
+        if (ip.is_private or ip.is_loopback or ip.is_link_local
+                or ip.is_multicast or ip.is_reserved or ip.is_unspecified):
+            return None
+        if ip in ipaddress.ip_network("100.64.0.0/10"):
+            return None
+        return raw
+    return None
+
+
+class _SafeRedirectHandler(urllib.request.HTTPRedirectHandler):
+    """重定向逐跳 SSRF 复核（v5.8.3 P1-2）。
+
+    默认 HTTPRedirectHandler 自动跟随 3xx 且不重新校验目标地址，攻击者
+    可用公网 URL 302 到内网（如 http://169.254.169.254/latest/meta-data/）。
+    本 handler 对每个跳转目标重新做 scheme/host 校验，并限制最大跳转数。
+    """
+
+    max_redirections = 3
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        parsed = urllib.parse.urlparse(newurl)
+        if parsed.scheme not in ("http", "https"):
+            return None  # 拒绝跟随到非 http(s) 协议
+        newhost = parsed.hostname or ""
+        if _is_private_ip(newhost):
+            raise ValueError(
+                f"拒绝重定向到内网/回环地址: {newhost}（SSRF 防护）")
+        return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
+@register_connector
+class UrlConnector(BaseConnector):
+    name = "url"
+    description = ("从网页 URL 抓取文本导入（基础 SSRF 防护；"
+                   "生产环境建议使用 CLI import-url 的严格 DNS 校验）")
+
+    def ingest(self, mf, source, **kwargs):
+        import urllib.request
+        parsed = urllib.parse.urlparse(source)
+        if parsed.scheme not in ("http", "https"):
+            raise ValueError(f"仅支持 http/https URL，收到: {parsed.scheme}")
+        host = parsed.hostname or ""
+        if _is_private_ip(host):
+            raise ValueError(f"拒绝访问内网/回环地址: {host}（SSRF 防护）")
+        # v5.8.9：解析一次、校验全部 IP、用校验过的 IP 直连，消除检查与连接之间的 DNS rebinding TOCTOU
+        fixed_ip = _resolve_safe_ip(host)
+        if fixed_ip is None:
+            raise ValueError(f"无法解析或解析到内网地址: {host}（SSRF 防护）")
+        opener = urllib.request.build_opener(
+            _SafeRedirectHandler(),
+            _FixedIPHTTPHandler(fixed_ip),
+            _FixedIPHTTPSHandler(fixed_ip),
+        )
+        req = urllib.request.Request(
+            source, headers={"User-Agent": "MindForge-Connector/5.8.9"})
+        with opener.open(req, timeout=kwargs.get("timeout", 15)) as resp:
+            content_type = resp.headers.get("Content-Type", "")
+            data = _read_limited(resp)
+        if "json" in content_type or source.rstrip("/").endswith(".json"):
+            raise ValueError(
+                "URL 返回 JSON：请先下载后使用 json 连接器导入")
+        text = data.decode("utf-8", "replace")
+        # 粗略去除 HTML 标签与脚本样式
+        text = re.sub(r"<script.*?</script>", " ", text, flags=re.S | re.I)
+        text = re.sub(r"<style.*?</style>", " ", text, flags=re.S | re.I)
+        text = re.sub(r"<[^>]+>", " ", text)
+        text = re.sub(r"\s+", " ", text).strip()
+        stats = {"imported": 0, "skipped": 0, "failed": 0, "deduped": 0}
+        if len(text) < 3:
+            return stats
+        mf.add(text[:100000] if len(text) > 100000 else text,
+               category=kwargs.get("category") or "url",
+               source_agent=kwargs.get("source_agent") or f"connector:url:{host}")
+        stats["imported"] = 1
+        return stats

mindforge/modules/context_window.py

@@ -0,0 +1,337 @@
+"""
+MindForge v6.1.5 新增：自适应上下文窗口 (Adaptive Context Window)
+
+跨会话上下文连续性引擎：
+  - 追踪会话中的查询模式（query pattern），预测下一步可能需要的高频记忆
+  - 维护一个「预热候选池」，后续搜索时优先召回已预热条目
+  - 基于记忆距离（时间衰减 × 访问间隔 × 图谱关联深度）动态调整 min_relevance
+  - 自动管理：会话结束清空，异常恢复
+
+设计约束：
+  - 纯本地统计——不依赖外部模型
+  - 零侵入：通过 _warm_pool 附加分实现，不改变原有检索逻辑
+  - 内存安全：候选池上限 200 条，超限按分数淘汰
+"""
+
+import logging
+import re
+import time
+from collections import OrderedDict
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Any, Set, Tuple
+
+logger = logging.getLogger(__name__)
+
+# 候选池配置
+MAX_POOL_SIZE = 200
+WARM_BOOST = 0.15  # 预热条目额外加分
+PATTERN_WINDOW = 30  # 保留最近 N 次查询模式
+
+
+@dataclass
+class QueryPattern:
+    """单次查询的模式记录"""
+    query: str
+    timestamp: float
+    result_ids: List[str]
+    categories_hit: List[str]
+    layers_hit: List[str]
+
+
+@dataclass
+class ContextWindow:
+    """会话级上下文窗口"""
+    session_id: str
+    patterns: List[QueryPattern] = field(default_factory=list)
+    warm_pool: "OrderedDict[str, float]" = field(
+        default_factory=lambda: OrderedDict()
+    )
+    _created_at: float = field(default_factory=time.time)
+    _last_active: float = field(default_factory=time.time)
+
+    @property
+    def age_seconds(self) -> float:
+        return time.time() - self._created_at
+
+    @property
+    def idle_seconds(self) -> float:
+        return time.time() - self._last_active
+
+    def touch(self):
+        self._last_active = time.time()
+
+
+class AdaptiveContextEngine:
+    """自适应上下文窗口引擎
+
+    使用方式：
+        ctx = AdaptiveContextEngine(storage, index)
+        # 每次搜索后更新模式
+        ctx.record_search(session_id="s1", query="数据库", results=chunks)
+        # 下次搜索时获取 boost 加分
+        boosts = ctx.get_warm_boosts(session_id="s1", candidates=candidate_ids)
+        # 自适应阈值
+        threshold = ctx.adaptive_min_relevance(session_id="s1", base=0.3)
+    """
+
+    def __init__(self, storage, index):
+        """
+        Args:
+            storage: StorageEngine 实例
+            index: IndexEngine 实例
+        """
+        self.storage = storage
+        self.index = index
+        self._windows: Dict[str, ContextWindow] = {}
+
+    def get_or_create_window(self, session_id: str) -> ContextWindow:
+        """获取或创建会话的上下文窗口"""
+        if session_id not in self._windows:
+            self._windows[session_id] = ContextWindow(session_id=session_id)
+        window = self._windows[session_id]
+        window.touch()
+        return window
+
+    def record_search(
+        self,
+        session_id: str,
+        query: str,
+        results: List[Any],
+        categories: Optional[List[str]] = None,
+        layers: Optional[List[str]] = None,
+    ):
+        """记录一次搜索的结果模式，更新候选池。
+
+        Args:
+            session_id: 会话 ID
+            query: 原始查询词
+            results: MemoryChunk 列表或任何含 memory_id 的对象列表
+            categories: 指定过滤类别
+            layers: 指定过滤层级
+        """
+        window = self.get_or_create_window(session_id)
+
+        # Extract result metadata
+        result_ids: List[str] = []
+        cats_hit: Set[str] = set()
+        layers_hit: Set[str] = set()
+
+        for r in results:
+            mid = getattr(r, "memory_id", None) or (
+                r.get("memory_id") if isinstance(r, dict) else None
+            )
+            if mid:
+                result_ids.append(str(mid))
+            cat = getattr(r, "category", None) or (
+                r.get("category") if isinstance(r, dict) else None
+            )
+            if cat:
+                cats_hit.add(str(cat))
+            layer = getattr(r, "layer", None) or (
+                r.get("layer") if isinstance(r, dict) else None
+            )
+            if layer:
+                layers_hit.add(str(layer))
+
+        pattern = QueryPattern(
+            query=query,
+            timestamp=time.time(),
+            result_ids=result_ids,
+            categories_hit=sorted(cats_hit),
+            layers_hit=sorted(layers_hit),
+        )
+
+        window.patterns.append(pattern)
+        # Trim to window size
+        if len(window.patterns) > PATTERN_WINDOW:
+            window.patterns = window.patterns[-PATTERN_WINDOW:]
+
+        # Update warm pool: results from this search get boosted
+        self._update_warm_pool(window, result_ids)
+
+    def get_warm_boosts(
+        self, session_id: str, candidate_ids: List[str]
+    ) -> Dict[str, float]:
+        """获取候选记忆的预热加分
+
+        Args:
+            session_id: 会话 ID
+            candidate_ids: 本次搜索候选记忆 ID
+
+        Returns:
+            {memory_id: boost_score}，只含命中候选池的条目
+        """
+        if session_id not in self._windows:
+            return {}
+        window = self._windows[session_id]
+        if not window.warm_pool:
+            return {}
+
+        candidate_set = set(candidate_ids)
+        boosts: Dict[str, float] = {}
+        for mid in candidate_set:
+            if mid in window.warm_pool:
+                boosts[mid] = window.warm_pool[mid] * WARM_BOOST
+        return boosts
+
+    def adaptive_min_relevance(
+        self,
+        session_id: str,
+        base: float = 0.3,
+        min_floor: float = 0.15,
+        max_ceiling: float = 0.5,
+    ) -> float:
+        """根据会话活跃度自适应调整最低相关度阈值。
+
+        策略：
+          - 刚创建窗口（pattern < 2）：用 base
+          - 连续快速查询（density >= 5/30s）：降低阈值（用户正在探索，宁可多召回）
+          - 长时间空闲后重新查询：提高阈值（聚焦性查询）
+
+        Args:
+            session_id: 会话 ID
+            base: 基准阈值
+            min_floor: 最低允许阈值
+            max_ceiling: 最高允许阈值
+
+        Returns:
+            调整后的 min_relevance
+        """
+        if session_id not in self._windows:
+            return base
+
+        window = self._windows[session_id]
+
+        if len(window.patterns) < 2:
+            return base
+
+        # Calculate query density (queries in last 30 seconds)
+        recent_cutoff = time.time() - 30
+        recent_count = sum(1 for p in window.patterns if p.timestamp > recent_cutoff)
+
+        if recent_count >= 5:
+            # Dense exploration mode: lower threshold to recall more
+            return max(min_floor, base - 0.08)
+        elif recent_count <= 1 and window.idle_seconds > 60:
+            # Cold-start after idle: raise threshold to focus
+            return min(max_ceiling, base + 0.05)
+        else:
+            return base
+
+    def predict_next(
+        self,
+        session_id: str,
+        current_query: str,
+        top_n: int = 5,
+    ) -> List[str]:
+        """基于历史模式预测下一次搜索最可能需要哪些记忆。
+
+        策略：
+          1. 找到与当前查询语义相似的过往模式
+          2. 收集「紧随相似模式之后」的查询结果
+          3. 按出现频率排序返回
+
+        Args:
+            session_id: 会话 ID
+            current_query: 当前查询词
+            top_n: 返回预测条数
+
+        Returns:
+            预测的 memory_id 列表
+        """
+        if session_id not in self._windows:
+            return []
+        window = self._windows[session_id]
+        if len(window.patterns) < 3:
+            return []
+
+        # Tokenize for simple overlap similarity
+        current_tokens: Set[str] = set(self._tokenize(current_query))
+        if not current_tokens:
+            return []
+
+        # Find similar past patterns (exclude last one)
+        similar_follow_up_ids: Dict[str, int] = {}
+
+        for i, pattern in enumerate(window.patterns[:-1]):
+            past_tokens = set(self._tokenize(pattern.query))
+            if not past_tokens:
+                continue
+            overlap = len(current_tokens & past_tokens) / max(
+                len(current_tokens | past_tokens), 1
+            )
+            if overlap >= 0.3:
+                # Collect results from the NEXT pattern
+                next_pattern = window.patterns[i + 1]
+                for mid in next_pattern.result_ids:
+                    similar_follow_up_ids[mid] = similar_follow_up_ids.get(mid, 0) + 1
+
+        # Sort by frequency
+        sorted_ids = sorted(
+            similar_follow_up_ids.items(), key=lambda x: x[1], reverse=True
+        )
+        return [mid for mid, _ in sorted_ids[:top_n]]
+
+    def close_session(self, session_id: str) -> Dict[str, Any]:
+        """关闭并清理会话的上下文窗口"""
+        window = self._windows.pop(session_id, None)
+        if window:
+            return {
+                "closed": True,
+                "patterns_count": len(window.patterns),
+                "warm_pool_size": len(window.warm_pool),
+                "session_age_seconds": round(window.age_seconds, 1),
+            }
+        return {"closed": False, "reason": "session not found"}
+
+    def session_stats(self, session_id: str) -> Optional[Dict[str, Any]]:
+        """查看会话状态"""
+        window = self._windows.get(session_id)
+        if not window:
+            return None
+        return {
+            "session_id": session_id,
+            "patterns": len(window.patterns),
+            "warm_pool_size": len(window.warm_pool),
+            "age_seconds": round(window.age_seconds, 1),
+            "idle_seconds": round(window.idle_seconds, 1),
+            "top_categories": self._get_top_categories(window),
+        }
+
+    # ===================================================================
+    # Private
+    # ===================================================================
+
+    def _update_warm_pool(self, window: ContextWindow, result_ids: List[str]):
+        """更新候选池：新命中的条目分数递增，最旧的淘汰"""
+        for mid in result_ids:
+            current = window.warm_pool.get(mid, 0.0)
+            window.warm_pool[mid] = current + 1.0
+            # Move to end (most recent access)
+            window.warm_pool.move_to_end(mid)
+
+        # Evict oldest entries over capacity
+        while len(window.warm_pool) > MAX_POOL_SIZE:
+            window.warm_pool.popitem(last=False)
+
+    def _get_top_categories(self, window: ContextWindow) -> List[str]:
+        cats: Dict[str, int] = {}
+        for p in window.patterns:
+            for c in p.categories_hit:
+                cats[c] = cats.get(c, 0) + 1
+        return sorted(cats, key=cats.get, reverse=True)[:5] if cats else []
+
+    @staticmethod
+    def _tokenize(text: str) -> List[str]:
+        """简单分词：中文按 bigram，英文按单词"""
+        tokens: List[str] = []
+        for seg in re.findall(r'[\u4e00-\u9fff]+|[a-zA-Z0-9_]+', text.lower()):
+            if re.match(r'[\u4e00-\u9fff]', seg):
+                if len(seg) <= 2:
+                    tokens.append(seg)
+                else:
+                    for i in range(len(seg) - 1):
+                        tokens.append(seg[i:i + 2])
+            elif len(seg) > 1:
+                tokens.append(seg)
+        return tokens