hy-memory 2.0.5.dev2__py3-none-any.whl → 2.0.5.dev4__py3-none-any.whl

hy_memory/client.py

@@ -2631,6 +2631,7 @@ class HyMemoryClient:
         branch: Optional[str] = None,
         request_id: Optional[str] = None,
         meta: Optional[Dict[str, Any]] = None,
+        _existing_raw_ids: Optional[List[str]] = None,
     ) -> Dict[str, Any]:
         """写入记忆（异步）
 
@@ -2799,6 +2800,7 @@ class HyMemoryClient:
             # 落库 memory_ops / WRITE_SUMMARY / pipeline_logs 的单一真相源。
             request_id=request_id,
             client_meta=meta,
+            existing_raw_ids=list(_existing_raw_ids or []),
         )
 
         # mode → writer:
@@ -2847,7 +2849,10 @@ class HyMemoryClient:
             logger.info(f"[add] done: memory_id={result.memory_id} elapsed={result.elapsed_ms:.0f}ms")
 
             # 记录历史（非阻塞，失败仅 warning）
-            if self._history_store is not None:
+            if (
+                self._history_store is not None
+                and not _existing_raw_ids
+            ):
                 try:
                     isolation_key = self._build_isolation_key(uid, agent_id)
                     # 从 content 或 messages 中提取用于记录的文本
@@ -2979,6 +2984,35 @@ class HyMemoryClient:
             include_runtime_diag=True,
         )
 
+    def process_raw(
+        self,
+        raw_ids: List[str],
+        *,
+        user_id: str,
+        agent_id: str = "default_agent",
+        session_id: str = "default_session",
+        source: str = "default",
+        scene: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+        branch: Optional[str] = None,
+        request_id: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Extract/reconcile existing L1 raw records without inserting raw again."""
+        return self._loop_thread.run(
+            self.async_process_raw(
+                raw_ids,
+                user_id=user_id,
+                agent_id=agent_id,
+                session_id=session_id,
+                source=source,
+                scene=scene,
+                workspace_id=workspace_id,
+                branch=branch,
+                request_id=request_id,
+            ),
+            include_runtime_diag=True,
+        )
+
     def get_raw(self, raw_id: str) -> Optional[Dict[str, Any]]:
         """Read one SDK raw record through the explicit raw API."""
         return self._loop_thread.run(self.async_get_raw(raw_id))
@@ -3266,6 +3300,148 @@ class HyMemoryClient:
             raise RuntimeError("raw import changed the requested identity")
         return {"success": True, "raw_id": node_id, "request_id": request_id}
 
+    @_ensure_internal_loop
+    async def async_process_raw(
+        self,
+        raw_ids: List[str],
+        *,
+        user_id: str,
+        agent_id: str = "default_agent",
+        session_id: str = "default_session",
+        source: str = "default",
+        scene: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+        branch: Optional[str] = None,
+        request_id: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Process exact raw records through extraction/reconciliation in caller order."""
+        from .models.memory import MemoryLayer
+        from .utils.source_filter import normalize_source
+
+        if (
+            not isinstance(raw_ids, list)
+            or not raw_ids
+            or len(raw_ids) > 100
+            or any(not isinstance(raw_id, str) or not raw_id for raw_id in raw_ids)
+            or len(set(raw_ids)) != len(raw_ids)
+        ):
+            raise ValueError("raw_ids must contain 1-100 unique non-empty ids")
+        if not user_id:
+            raise ValueError("user_id is required")
+        normalized_source = normalize_source(source)
+        request_id = request_id or str(uuid.uuid4())
+        replay = await self._claim_write_outcome(
+            request_id=request_id,
+            user_id=user_id,
+            agent_id=agent_id,
+            session_id=session_id,
+            operation="process_raw",
+        )
+        if replay is not None:
+            return replay
+        pipeline_invoked = False
+        try:
+            nodes = []
+            for raw_id in raw_ids:
+                node = await self._vector_store.get_by_id(raw_id)
+                if node is None or node.layer != MemoryLayer.L1_RAW:
+                    raise ValueError(f"raw record is unavailable: {raw_id}")
+                if (
+                    node.user_id != user_id
+                    or node.agent_id != agent_id
+                    or node.session_id != session_id
+                    or node.source != normalized_source
+                ):
+                    raise ValueError("raw record is outside the requested write scope")
+                nodes.append(node)
+            source_orders = [(node.custom or {}).get("source_order") for node in nodes]
+            numeric_orders = [value for value in source_orders if type(value) is int]
+            if numeric_orders and (
+                len(numeric_orders) != len(nodes)
+                or numeric_orders != sorted(numeric_orders)
+                or len(set(numeric_orders)) != len(numeric_orders)
+            ):
+                raise ValueError("raw_ids are not in stable source order")
+            roles = [str((node.custom or {}).get("role") or "user") for node in nodes]
+            if any(role not in {"user", "assistant", "system", "tool"} for role in roles):
+                raise ValueError("raw record has an unsupported message role")
+            messages = [
+                {
+                    "role": role,
+                    "content": node.content,
+                    **(
+                        {"name": str((node.custom or {})["name"])}
+                        if (node.custom or {}).get("name")
+                        else {}
+                    ),
+                    **(
+                        {"tool_call_id": str((node.custom or {})["tool_call_id"])}
+                        if (node.custom or {}).get("tool_call_id")
+                        else {}
+                    ),
+                    **(
+                        {"tool_name": str((node.custom or {})["tool_name"])}
+                        if (node.custom or {}).get("tool_name")
+                        else {}
+                    ),
+                }
+                for node, role in zip(nodes, roles)
+            ]
+            pipeline_invoked = True
+            result = await self._async_add_impl(
+                messages,
+                user_id=user_id,
+                agent_id=agent_id,
+                session_id=session_id,
+                source=normalized_source,
+                _existing_raw_ids=list(raw_ids),
+                memory_at=max(
+                    (node.memory_at for node in nodes if node.memory_at is not None),
+                    default=None,
+                ),
+                scene=scene,
+                workspace_id=workspace_id,
+                branch=branch,
+                request_id=request_id,
+            )
+            memory_ids = list(
+                dict.fromkeys(
+                    result.get("memory_ids")
+                    or [
+                        *(result.get("experience_ids") or []),
+                        *(result.get("fact_ids") or []),
+                    ]
+                )
+            )
+            result = {
+                **result,
+                "raw_ids": list(raw_ids),
+                "memory_ids": memory_ids,
+                "request_id": request_id,
+            }
+        except BaseException:
+            if pipeline_invoked:
+                await self._mark_write_outcome_unknown(request_id)
+            else:
+                await self._finish_write_outcome(
+                    request_id,
+                    "failed",
+                    {
+                        "success": False,
+                        "request_id": request_id,
+                        "raw_ids": list(raw_ids),
+                        "memory_ids": [],
+                        "error_code": "process_raw_invalid",
+                    },
+                )
+            raise
+        await self._finish_write_outcome(
+            request_id,
+            "succeeded" if result.get("success") is True else "failed",
+            result,
+        )
+        return result
+
     @_ensure_internal_loop
     async def async_import_memory_exact(
         self,
@@ -3469,6 +3645,66 @@ class HyMemoryClient:
             "offset": offset,
         }
 
+    async def _search_explicit_raw_layer(
+        self,
+        query: str,
+        *,
+        user_ids: List[str],
+        agent_ids: List[str],
+        session_ids: List[str],
+        include_sources: List[str],
+        exclude_sources: List[str],
+        limit: int,
+        min_score: float,
+        created_after: Optional[float],
+        request_id: str,
+        t0: float,
+        query_embedding: Optional[List[float]],
+    ) -> Dict[str, Any]:
+        """Implement `search(..., layers=['l1_raw'])` without a second API."""
+        from .models.memory import MemoryLayer, MemoryStatus
+
+        embedding = query_embedding or await self._embed_service.embed(query)
+        fetch = min(max(limit * (10 if session_ids else 2), limit), 10_000)
+        hits = await self._vector_store.search(
+            query_embedding=embedding,
+            user_ids=user_ids,
+            agent_ids=agent_ids,
+            include_sources=include_sources,
+            exclude_sources=exclude_sources,
+            layers=[MemoryLayer.L1_RAW],
+            limit=fetch,
+            score_threshold=min_score,
+            status_filter=[MemoryStatus.ACTIVE],
+            only_latest=False,
+            created_after=created_after,
+        )
+        sessions = set(session_ids)
+        rows: List[Dict[str, Any]] = []
+        for hit in hits:
+            node = hit.get("node")
+            if node is None or node.layer != MemoryLayer.L1_RAW:
+                continue
+            if sessions and node.session_id not in sessions:
+                continue
+            row = self._raw_node_to_public(node)
+            rows.append(
+                {
+                    "memory_id": row.pop("raw_id"),
+                    **row,
+                    "layer": MemoryLayer.L1_RAW.value,
+                    "status": MemoryStatus.ACTIVE.value,
+                    "score": round(float(hit.get("score", 0.0)), 4),
+                }
+            )
+            if len(rows) >= limit:
+                break
+        return {
+            "request_id": request_id,
+            "memories": {"schema": [], "proactive": [], "normal": rows},
+            "elapsed_ms": round((time.perf_counter() - t0) * 1000, 2),
+        }
+
     @staticmethod
     def _public_search_views(
         result_memories: List[Dict[str, Any]],
@@ -4129,6 +4365,22 @@ class HyMemoryClient:
         effective_search_layers = [MemoryLayer.L1_RAW] if lite_mode else layers
         effective_schema_limit = 0 if lite_mode else schema_limit
         effective_intention_limit = 0 if lite_mode else intention_limit
+        normalized_search_layers = self._normalize_layers(effective_search_layers)
+        if normalized_search_layers == [MemoryLayer.L1_RAW]:
+            return await self._search_explicit_raw_layer(
+                effective_target_query,
+                user_ids=effective_user_ids,
+                agent_ids=effective_agent_ids,
+                session_ids=effective_session_ids,
+                include_sources=effective_include_sources,
+                exclude_sources=effective_exclude_sources,
+                limit=limit,
+                min_score=min_score,
+                created_after=effective_created_after,
+                request_id=request_id,
+                t0=t0,
+                query_embedding=query_embedding,
+            )
 
         read_req = ReadRequest(
             query=effective_target_query,
@@ -4149,7 +4401,7 @@ class HyMemoryClient:
             layers=(
                 [
                     layer.value
-                    for layer in self._normalize_layers(effective_search_layers)
+                    for layer in normalized_search_layers
                 ]
                 if effective_search_layers else None
             ),
@@ -4174,7 +4426,7 @@ class HyMemoryClient:
         _wanted_layers = (
             {
                 layer.value
-                for layer in self._normalize_layers(effective_search_layers)
+                for layer in normalized_search_layers
             }
             if effective_search_layers else None
         )

hy_memory/pipelines/base.py

@@ -231,6 +231,10 @@ class WriteRequest:
     # 已有记忆 (V1 合并检测用)
     existing_memories: Optional[List[Dict[str, Any]]] = None
 
+    # Public client process_raw() only: reuse these exact L1 nodes as input.
+    # Ordinary add() never populates this field.
+    existing_raw_ids: List[str] = field(default_factory=list)
+
     # 记忆时间戳（不传则用当前时间 datetime.now()）
     memory_at: Optional[datetime] = None
 

hy_memory/pipelines/writer.py

@@ -280,6 +280,27 @@ class MemoryWriter(WritePipeline):
             custom["source_raw_memory_ids"] = source_ids
         return custom
 
+    async def _attach_existing_raw_ids(
+        self,
+        vector_store,
+        memory_ids: List[str],
+        raw_ids: List[str],
+    ) -> None:
+        """Attach every reused raw record to canonical outputs."""
+        for memory_id in memory_ids:
+            node = await vector_store.get_by_id(memory_id)
+            if node is None:
+                continue
+            custom = dict(node.custom or {})
+            source_ids = list(custom.get("source_raw_memory_ids") or [])
+            legacy_source = str(getattr(node, "source_raw_memory_id", "") or "")
+            for raw_id in (legacy_source, *raw_ids):
+                if raw_id and raw_id not in source_ids:
+                    source_ids.append(raw_id)
+            if source_ids:
+                custom["source_raw_memory_ids"] = source_ids
+                await vector_store.update_payload(memory_id, {"custom": custom})
+
     @staticmethod
     def _attach_client_meta(custom: Dict[str, Any], request: Optional[WriteRequest]) -> None:
         """给新建节点挂上调用方 meta（单份 list）。
@@ -2596,13 +2617,23 @@ class MemoryWriter(WritePipeline):
             from ..metrics import MetricsCollector
             MetricsCollector.get().sys1_start()
 
-            # 2. 向量化 + 持久化（L1_RAW）
+            existing_raw_ids = request.existing_raw_ids
+            if (
+                not isinstance(existing_raw_ids, list)
+                or any(not isinstance(raw_id, str) or not raw_id for raw_id in existing_raw_ids)
+            ):
+                raise ValueError("_existing_raw_ids must be a list of non-empty ids")
+
+            # 2. 向量化（处理已有 raw 时复用已存节点，不再次 embed）
             _t_embed = datetime.now()
-            with tracer.span("embed") as s:
-                embedding, embed_meta = await self.embed_service.embed_queued_with_meta(
-                    request.content
-                )
-                s.set_output({"dims": len(embedding), "content_len": len(request.content)})
+            if existing_raw_ids:
+                embedding, embed_meta = [], {"reused_raw": True}
+            else:
+                with tracer.span("embed") as s:
+                    embedding, embed_meta = await self.embed_service.embed_queued_with_meta(
+                        request.content
+                    )
+                    s.set_output({"dims": len(embedding), "content_len": len(request.content)})
             _embed_ms = (datetime.now() - _t_embed).total_seconds() * 1000
             response.extra["embedding"] = embedding
             await self._emit_pipeline_step(
@@ -2625,27 +2656,43 @@ class MemoryWriter(WritePipeline):
             with tracer.span("qdrant_upsert") as s:
                 try:
                     vector_store = await self._get_vector_store()
-                    layer_enum = MemoryLayer.from_string(suggested_layer)
-                    _raw_custom: Dict[str, Any] = {}
-                    self._attach_client_meta(_raw_custom, request)
-                    mem_node = MemoryNode(
-                        user_id=request.user_id,
-                        agent_id=request.agent_id or "default_agent",
-                        session_id=request.session_id or "default_session",
-                        source=request.source,
-                        layer=layer_enum,
-                        content=request.content,
-                        source_type=SourceType.EXPLICIT,
-                        status=MemoryStatus.ACTIVE,
-                        is_latest=True,
-                        embedding=embedding,
-                        memory_at=request.memory_at,
-                        custom=_raw_custom,
-                    )
-                    memory_id = await vector_store.upsert(mem_node)
+                    if existing_raw_ids:
+                        memory_id = existing_raw_ids[-1]
+                        mem_node = await vector_store.get_by_id(memory_id)
+                        if (
+                            mem_node is None
+                            or mem_node.layer != MemoryLayer.L1_RAW
+                            or mem_node.user_id != request.user_id
+                            or mem_node.agent_id != (request.agent_id or "default_agent")
+                            or mem_node.session_id != (request.session_id or "default_session")
+                            or mem_node.source != request.source
+                        ):
+                            raise ValueError("existing raw record is outside the write scope")
+                    else:
+                        layer_enum = MemoryLayer.from_string(suggested_layer)
+                        _raw_custom: Dict[str, Any] = {}
+                        self._attach_client_meta(_raw_custom, request)
+                        mem_node = MemoryNode(
+                            user_id=request.user_id,
+                            agent_id=request.agent_id or "default_agent",
+                            session_id=request.session_id or "default_session",
+                            source=request.source,
+                            layer=layer_enum,
+                            content=request.content,
+                            source_type=SourceType.EXPLICIT,
+                            status=MemoryStatus.ACTIVE,
+                            is_latest=True,
+                            embedding=embedding,
+                            memory_at=request.memory_at,
+                            custom=_raw_custom,
+                        )
+                        memory_id = await vector_store.upsert(mem_node)
                     response.memory_id = memory_id
                     s.set_output({"memory_id": memory_id, "layer": suggested_layer})
-                    logger.debug(f"[write] persisted: id={memory_id} layer={suggested_layer}")
+                    logger.debug(
+                        f"[write] {'reused' if existing_raw_ids else 'persisted'}: "
+                        f"id={memory_id} layer={suggested_layer}"
+                    )
                 except Exception as persist_err:
                     _l1_error = str(persist_err)
                     s.set_error(_l1_error)
@@ -2709,6 +2756,12 @@ class MemoryWriter(WritePipeline):
                         recent_history_count=_recent_history_count,
                     )
                     total_tokens += response.extra.get("_agent_tokens", 0)
+                if existing_raw_ids and response.extra.get("agent_stored_ids"):
+                    await self._attach_existing_raw_ids(
+                        vector_store,
+                        list(response.extra["agent_stored_ids"]),
+                        existing_raw_ids,
+                    )
                 if response.extra.get("agent_status") == "failed":
                     await self._emit_pipeline_step(
                         request,
@@ -2718,12 +2771,13 @@ class MemoryWriter(WritePipeline):
                             "error": response.extra.get("agent_error", ""),
                         },
                     )
-                    await self._rollback_failed_l1(
-                        request=request,
-                        response=response,
-                        vector_store=vector_store,
-                        memory_id=memory_id,
-                    )
+                    if not existing_raw_ids:
+                        await self._rollback_failed_l1(
+                            request=request,
+                            response=response,
+                            vector_store=vector_store,
+                            memory_id=memory_id,
+                        )
                     response.tokens_used = total_tokens
                     _failed_workflow_ms = (
                         datetime.now() - _t_workflow

{hy_memory-2.0.5.dev2.dist-info → hy_memory-2.0.5.dev4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hy-memory
-Version: 2.0.5.dev2
+Version: 2.0.5.dev4
 Summary: HY Memory - Industrial-grade dual-system cognitive memory framework
 Author: alvinfei
 License-Expression: Apache-2.0

{hy_memory-2.0.5.dev2.dist-info → hy_memory-2.0.5.dev4.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hy_memory/__init__.py,sha256=idVBktG5xzToAXl6WDsAmZbBKX3_Q5D5iUtkKcdfmm0,4031
-hy_memory/client.py,sha256=2posxO_sWtNXPO5w2ixBr2pSXv58JLVdiqEHMK7fTf8,301567
+hy_memory/client.py,sha256=bfSMfaWG8oi8NYshkGiThqq5-qxYwK18ERmX2Bgl2Bw,311202
 hy_memory/config.py,sha256=J051sHyMnTysxWajFORNnGhy6SRKq8n6ZVXSS44lhGA,67809
 hy_memory/inspector.py,sha256=Uj-mrxi_Piluq4DGVHCHzMfA1QSJLEOMBpiotDjqJSs,18380
 hy_memory/metrics.py,sha256=sGhhV_aDEuws7ks4c-pI_z5o2rayKkJcNC6J1b9sPQY,16648
@@ -60,7 +60,7 @@ hy_memory/models/memory.py,sha256=5x62z7QiCoFd7-Km9Gp_GzhBsvgGLeRA-Aa7LOu867I,47
 hy_memory/models/requests.py,sha256=jtSZ6GyaUXMf9RnHIDjX5T1D-w03ufkff1y0qIC4So8,20396
 hy_memory/pipelines/__init__.py,sha256=XcLhI93wqLYqYkHPbmE1C5RW8f8X0b2TBGY68ozd_3s,1604
 hy_memory/pipelines/balanced_catalog.py,sha256=Ww1Tf-waupn5PweWvZi8IqavS4W2FMvoJBJ85Wimn2M,39677
-hy_memory/pipelines/base.py,sha256=9D_4zn7y0KQLD8bc7yaEd7qRccpuSWuTMueK5iSfp8A,18556
+hy_memory/pipelines/base.py,sha256=HReILG4-DEfEx71d6YTokKMpXfxVKi71kVBDcRCTH10,18745
 hy_memory/pipelines/cross_domain_sweeper.py,sha256=wL5PXuas38mO5G43WDhpQEUs82akc9M9YnJhwvsxEmg,29527
 hy_memory/pipelines/reader.py,sha256=xYxdInMPH9Jb7Z9ZIUo0sNclrrpMOeHfeZVpDYKZYiY,3523
 hy_memory/pipelines/reader_base.py,sha256=up9Z9hIw9cV3Z7x39cZtdge8ULmzZXI7WEHGCrEWA40,177951
@@ -72,7 +72,7 @@ hy_memory/pipelines/system2_agent.py,sha256=oV6wZ49SgDZwgtbNQ8cGcE3X4H7cjT8_MN2c
 hy_memory/pipelines/system2_tools.py,sha256=mt-IY9Tbe6zUo5WGQR6uSjuUnF6s1nlrIUmEayvGhCg,42575
 hy_memory/pipelines/system2_writer.py,sha256=qKc5wP5LfH-o1P0BtDKuKjGBU52nakm1RUKHVR9Ot-I,54224
 hy_memory/pipelines/user_profile_scope.py,sha256=lSCYyocAE7kDOS5775s1YfCCkhUMZug32nChCuIspdg,635
-hy_memory/pipelines/writer.py,sha256=XWvvwUXNzWNwNQ6OXb5a7p2IPFxKDieWoj9UjhNcZHw,174822
+hy_memory/pipelines/writer.py,sha256=azd0m7v4EQefWiSBtiHyWs2RaZ_eqn8Uuk-olXLbM8g,177553
 hy_memory/pipelines/_graph/__init__.py,sha256=R9RRj8veFgg7n6bl5IpvFp1rlLX4QtPSsyhw0sGJWFc,1441
 hy_memory/pipelines/_graph/projector.py,sha256=WE-pFT_FT2jgO5hSpRPesW0Fh9ATvg-tvSIpYqBDAaA,27667
 hy_memory/pipelines/_graph/rerank.py,sha256=IW0DS9v4y99lwGKq6rz_ns9wo5bq1INVM63mlYhBkHU,40583
@@ -128,8 +128,8 @@ hy_memory/utils/time_window.py,sha256=MGAAWLyxHHwLrNthmYDdnH5PA-pQP9ZdcoQUfpzJof
 hy_memory/utils/token_count.py,sha256=WibfD9KNuERMrcEPNFZTNo0JCpE72XPaUSlD48Wz1mk,2388
 hy_memory/utils/trace_file_writer.py,sha256=bS_Kbzzy49eLr5G7HwBKLZqU9j3P6vGjtBWUWa8MIiY,217
 hy_memory/utils/tracer.py,sha256=dVAwue7zwMax7KQFOfXpD-4pdfrVfiFFxbLhTrqu5XU,12595
-hy_memory-2.0.5.dev2.dist-info/licenses/LICENSE,sha256=ywv3qL-z2K0y7z9CI4IMqXf6eLeIKEWewwrzNfsrYN0,9650
-hy_memory-2.0.5.dev2.dist-info/METADATA,sha256=W0pOsgjOKa9BsoJ2qbi55MkS__CvV-QReHwqKyKIC0k,18340
-hy_memory-2.0.5.dev2.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
-hy_memory-2.0.5.dev2.dist-info/top_level.txt,sha256=YipheDT9yI2XTaR3lQcsXb3P9MOkd3uN2JHGtlUbnFU,10
-hy_memory-2.0.5.dev2.dist-info/RECORD,,
+hy_memory-2.0.5.dev4.dist-info/licenses/LICENSE,sha256=ywv3qL-z2K0y7z9CI4IMqXf6eLeIKEWewwrzNfsrYN0,9650
+hy_memory-2.0.5.dev4.dist-info/METADATA,sha256=qbFy82UX7latN-6mVTk-2sbG7n_dQLd35aLQdZQ4A4s,18340
+hy_memory-2.0.5.dev4.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+hy_memory-2.0.5.dev4.dist-info/top_level.txt,sha256=YipheDT9yI2XTaR3lQcsXb3P9MOkd3uN2JHGtlUbnFU,10
+hy_memory-2.0.5.dev4.dist-info/RECORD,,