davidkhala.ai 0.2.0__py3-none-any.whl → 0.2.2__py3-none-any.whl

davidkhala/ai/agent/dify/api/knowledge.py

@@ -1,191 +0,0 @@
-from __future__ import annotations
-
-import os
-from pathlib import Path
-from typing import Iterable, TypedDict, Optional
-from urllib.parse import urlparse
-
-import requests
-
-from davidkhala.ai.agent.dify.api import API, Iterator
-from davidkhala.ai.agent.dify.model import Document as DocumentBase
-
-
-class DatasetDict(TypedDict):
-    id: str
-    name: str
-    description: str
-    provider: str
-    permission: str
-    data_source_type: str
-    indexing_technique: str
-    doc_form: str
-    runtime_mode: str
-    is_published: bool
-    enable_api: bool
-    # stats
-    app_count: int
-    document_count: int
-    word_count: int
-    total_documents: int
-    total_available_documents: int
-    # embedding
-    embedding_available: bool
-    embedding_model: str
-    embedding_model_provider: str
-    retrieval_model_dict: dict
-    external_retrieval_model: dict
-    external_knowledge_info: dict
-
-
-class Document(DocumentBase):
-    data_source_info: dict[str, str]
-    data_source_detail_dict: dict[str, dict]
-    dataset_process_rule_id: str
-    created_from: str
-    created_by: str
-    created_at: int
-    tokens: int
-    archived: bool
-    display_status: str
-    word_count: int
-    hit_count: int
-    doc_form: str
-    doc_metadata: dict
-    disabled_at: int
-    disabled_by: str
-
-
-class Dataset(API):
-    def __init__(self, api_key: str, base_url="https://api.dify.ai/v1"):
-        super().__init__(api_key, f"{base_url}/datasets")
-
-    def paginate_datasets(self, page=1, size=20):
-        r = self.request(self.base_url, "GET", params={
-            'page': page,
-            'limit': size,
-        })
-        return r
-
-    def list_datasets(self) -> Iterable[list[DatasetDict]]:
-        return Iterator(self.paginate_datasets, None)
-
-    @property
-    def ids(self):
-        for sub_list in self.list_datasets():
-            for dataset in sub_list:
-                yield dataset['id']
-
-    class Instance(API):
-        def __init__(self, d: Dataset, dataset_id: str):
-            super().__init__(d.api_key, f"{d.base_url}/{dataset_id}")
-
-        def get(self):
-            return self.request(self.base_url, "GET")
-
-        def upload(self, filename, *, path=None, url=None, document_id=None):
-            """
-            don't work for .html
-            work for .md
-            """
-            files = {}
-            if path:
-                with open(path, 'rb') as f:
-                    content = f.read()
-                if not filename:
-                    filename = os.path.basename(path)
-            elif url:
-                r = requests.get(url)
-                r.raise_for_status()
-                if not filename:
-                    parsed_url = urlparse(url)
-                    filename = Path(parsed_url.path).name
-                content = r.content
-            files['file'] = (filename, content)
-            if document_id:
-                # don't work for html
-                r = requests.post(f"{self.base_url}/documents/{document_id}/update-by-file", files=files,
-                                  **self.options)
-            else:
-                r = requests.post(f"{self.base_url}/document/create-by-file", files=files, **self.options)
-            r = self.on_response(r)
-            return r['document']
-
-        def paginate_documents(self, page=1, size=20):
-            return self.request(f"{self.base_url}/documents", "GET", params={
-                'page': page,
-                'limit': size
-            })
-
-        def list_documents(self) -> Iterable[Document]:
-            for document_batch in Iterator(self.paginate_documents, None):
-                for document in document_batch:
-                    yield Document(**document)
-
-        def has_document(self, name) -> bool:
-            return any(name == item['name'] for row in self.list_documents() for item in row)
-
-
-class ChunkDict(TypedDict):
-    id: str
-    position: int
-    document_id: str
-    content: str
-    sign_content: str  # trimmed version of content
-    answer: Optional[str]  # only used in QA chunk
-    word_count: int
-    tokens: int
-    keywords: Optional[list[str]]
-    index_node_id: str  # chunk 在向量索引中的节点 ID
-    index_node_hash: str  # hash of sign_content
-    hit_count: int
-    enabled: bool
-    status: str  # 'completed'
-    created_at: int  # timestamp
-    updated_at: int  # timestamp
-    completed_at: int  # timestamp
-    created_by: str  # user id
-    child_chunks: list
-    error: Optional
-    stopped_at: Optional[int]  # timestamp
-    disabled_at: Optional[int]  # timestamp
-
-
-class Document(API):
-    def __init__(self, d: Dataset.Instance, document_id: str):
-        super().__init__(d.api_key, f"{d.base_url}/documents/{document_id}")
-
-    def exist(self):
-        try:
-            self.get()
-            return True
-        except requests.exceptions.HTTPError as e:
-            if e.response.status_code == 404:
-                return False
-            else:
-                raise e
-
-    def get(self):
-        return self.request(self.base_url, "GET")
-
-    def paginate_chunks(self, page=1, size=20):
-        return self.request(f"{self.base_url}/segments", "GET", params={
-            'page': page,
-            'limit': size
-        })
-
-    def list_chunks(self) -> Iterable[ChunkDict]:
-        for chunk_batch in Iterator(self.paginate_chunks, None):
-            for chunk in chunk_batch:
-                yield chunk
-
-    def delete(self):
-        if self.exist():
-            self.request(self.base_url, "DELETE")
-class Chunk(API):
-    def __init__(self, d: Document, segment_id: str):
-        super().__init__(d.api_key, f"{d.base_url}/segments/{segment_id}")
-    def get(self):
-        r=  self.request(self.base_url, "GET")
-        assert r['doc_form'] # optional value text_model
-        return r['data']

davidkhala/ai/agent/dify/ops/__init__.py

@@ -1 +0,0 @@
-

davidkhala/ai/agent/dify/ops/console/knowledge.py

@@ -1,158 +0,0 @@
-from time import sleep
-
-from davidkhala.utils.http_request.stream import as_sse, Request as StreamRequest
-from pydantic import BaseModel
-
-from davidkhala.ai.agent.dify.interface import IndexingError
-from davidkhala.ai.agent.dify.model import Document, Dataset
-from davidkhala.ai.agent.dify.const import IndexingStatus
-from davidkhala.ai.agent.dify.ops.console import API
-from davidkhala.ai.agent.dify.ops.console.session import ConsoleUser
-from davidkhala.ai.agent.dify.ops.db.orm import Node
-
-
-class ConsoleKnowledge(API):
-    def __init__(self, context: ConsoleUser):
-        super().__init__()
-        self.base_url = context.base_url
-        self.session.cookies = context.session.cookies
-        self.options = context.options
-
-
-class Datasource(ConsoleKnowledge):
-    """step 1: Choose a Data Source"""
-
-    class FirecrawlOutput(BaseModel):
-        source_url: str
-        description: str
-        title: str
-        credential_id: str
-        content: str
-
-    def run_firecrawl(self, pipeline: str, node: Node,
-                      *,
-                      inputs: dict,
-                      credential_id: str
-                      ):
-
-        url = f"{self.base_url}/rag/pipelines/{pipeline}/workflows/published/datasource/nodes/{node.id}/run"
-
-        stream_request = StreamRequest(self)
-        response = stream_request.request(url, 'POST', json={
-            'inputs': inputs,
-            'datasource_type': node.datasource_type,
-            'credential_id': credential_id,
-            "response_mode": "streaming"
-        })
-
-        for data in as_sse(response):
-            event = data['event']
-            if event == 'datasource_completed':
-                return data['data']
-            else:
-                assert event == 'datasource_processing'
-                print(data)
-        return None
-
-    def upload(self):
-        "http://localhost/console/api/files/upload?source=datasets"
-        # TODO
-        "form data"
-        {
-            "file": "body"
-        }
-        r = {
-            "id": "3898db5b-eb72-4f11-b507-628ad5d28887",
-            "name": "Professional Diploma Meister Power Electrical Engineering - Technological and Higher Education Institute of Hong Kong.html",
-            "size": 254362,
-            "extension": "html",
-            "mime_type": "text\/html",
-            "created_by": "dbd0b38b-5ef1-4123-8c3f-0c82eb1feacd",
-            "created_at": 1764943811,
-            "source_url": "\/files\/3898db5b-eb72-4f11-b507-628ad5d28887\/file-preview?timestamp=1764943811&nonce=43b0ff5a13372415be79de4cc7ef398c&sign=7OJ2wiVYc4tygl7yvM1sPn7s0WXDlhHxgX76bsGTD94%3D"
-        }
-
-
-class Operation(ConsoleKnowledge):
-    def website_sync(self, dataset: str, document: str, *, wait_until=True):
-        """
-        cannot be used towards a pipeline dataset. Otherwise, you will see error "no website import info found"
-        """
-        doc_url = f"{self.base_url}/datasets/{dataset}/documents/{document}"
-
-        r = self.request(f"{doc_url}/website-sync", "GET")
-        assert r == {"result": "success"}
-        if wait_until:
-            return self.wait_until(dataset, document)
-        return None
-
-    def retry(self, dataset: str, *documents: str, wait_until=True):
-        """
-        It cannot trigger rerun on success documents
-        """
-        url = f"{self.base_url}/datasets/{dataset}/retry"
-        self.request(url, "POST", json={
-            'document_ids': documents,
-        })
-        # response status code will be 204
-        if wait_until:
-            return [self.wait_until(dataset, document) for document in documents]
-        return None
-
-    def rerun(self, dataset: str, *documents: str):
-        for document in documents:
-            try:
-                self.website_sync(dataset, document)
-                assert False, "expect IndexingError"
-            except IndexingError:
-                pass
-        return self.retry(dataset, *documents)
-
-    def wait_until(self, dataset: str, document: str, *,
-                   expect_status=None,
-                   from_status=None,
-                   interval=1
-                   ):
-        if not expect_status:
-            expect_status = [IndexingStatus.FAILED, IndexingStatus.COMPLETED]
-        url = f"{self.base_url}/datasets/{dataset}/documents/{document}/indexing-status"
-        if from_status is None:
-            from_status = [IndexingStatus.WAITING, IndexingStatus.PARSING]
-        r = self.request(url, "GET")
-        status = r['indexing_status']
-        assert status in from_status, f"current status: {status}, expect: {from_status}"
-        while status not in expect_status:
-            sleep(interval)
-            r = self.request(url, "GET")
-            status = r['indexing_status']
-        if status == IndexingStatus.FAILED: raise IndexingError(r['error'])
-        return r
-
-
-class DatasetResult(Dataset):
-    chunk_structure: str
-
-class RunResult(BaseModel):
-    batch: str
-    dataset: DatasetResult
-    documents: list[Document]
-
-class Load(ConsoleKnowledge):
-    """
-    Processing Documents
-    """
-
-    def async_run(self, pipeline: str, node: Node, inputs: dict, datasource_info_list: list[dict])->RunResult:
-        """Ingest new document"""
-        url = f"{self.base_url}/rag/pipelines/{pipeline}/workflows/published/run"
-        r = self.request(url, "POST", json={
-            'inputs': inputs,
-            'start_node_id': node.id,
-            'is_preview': False,
-            'response_mode': "blocking",
-            "datasource_info_list": datasource_info_list,
-            'datasource_type': node.datasource_type
-        })
-        return RunResult(**r)
-
-

davidkhala/ai/agent/dify/ops/console/session.py

@@ -1,32 +0,0 @@
-from base64 import b64encode
-
-from davidkhala.ai.agent.dify.ops.console import API
-
-
-class ConsoleUser(API):
-    def login(self, email, password,
-              *,
-              remember_me=True,
-              language="en-US"
-              ):
-        url = f"{self.base_url}/login"
-
-        r = self.request(url, "POST", json={
-            'email': email,
-            'password': b64encode(password.encode()).decode(), # use base64 from dify 1.11
-            'remember_me': remember_me,
-            'language': language,
-        })
-        assert r == {"result": "success"}
-        self.options['headers']['x-csrf-token'] = self.session.cookies.get("csrf_token")
-        return self.session.cookies
-
-    @property
-    def me(self) -> dict:
-        url = f"{self.base_url}/account/profile"
-        return self.request(url, "GET")
-
-    @property
-    def workspace(self) -> dict:
-        url = f"{self.base_url}/features"
-        return self.request(url, "GET")

davidkhala/ai/huggingface/BAAI.py

@@ -1,10 +0,0 @@
-import os
-
-from davidkhala.ai.huggingface import clone
-from pathlib import Path
-
-def bge_m3_path(git_dir: os.PathLike):
-    model_dir = clone(git_dir, repo_id="BAAI/bge-m3",allow_patterns=["onnx/*"])
-    onnx_path = Path(model_dir) / "onnx" / "model.onnx"
-    assert onnx_path.is_file() and onnx_path.exists()
-    return onnx_path

davidkhala/ai/huggingface/__init__.py

@@ -1,21 +0,0 @@
-import os
-from typing import Optional
-
-from huggingface_hub import snapshot_download
-
-
-def clone(git_dir: os.PathLike,
-          *,
-          owner: Optional[str] = None,
-          repository: Optional[str] = None,
-          repo_id: Optional[str] = None,
-          **kwargs
-          ) -> str:
-    if not repo_id:
-        repo_id = f"{owner}/{repository}"
-    return snapshot_download(
-        repo_id=repo_id,
-        local_dir=git_dir,
-        local_dir_use_symlinks=False,
-        **kwargs
-    )

davidkhala/ai/huggingface/inference.py

@@ -1,13 +0,0 @@
-from huggingface_hub import InferenceApi
-
-
-class API:
-    def __init__(self, token):
-        self.inference = None
-        self.token = token
-
-    def as_model(self, repo_id):
-        self.inference = InferenceApi(repo_id=repo_id, token=self.token)
-
-    def call(self, **kwargs):
-        return self.inference(**kwargs)

davidkhala/ai/model.py

@@ -1,28 +0,0 @@
-from abc import ABC
-from typing import Optional
-
-
-class AbstractClient(ABC):
-    api_key: str
-    base_url: str
-    model: Optional[str]
-    messages = []
-
-    def as_chat(self, model: str, sys_prompt: str = None):
-        self.model = model
-        if sys_prompt is not None:
-            self.messages = [{"role": "system", "content": sys_prompt}]
-
-    def as_embeddings(self, model: str):
-        self.model = model
-
-    def chat(self, *user_prompt, **kwargs):
-        ...
-
-    def encode(self, *_input: str) -> list[list[float]]:
-        ...
-    def connect(self):
-        ...
-
-    def disconnect(self):
-        ...

davidkhala_ai-0.2.0.dist-info/RECORD

@@ -1,48 +0,0 @@
-davidkhala/ai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/model.py,sha256=1wcXC8X8oqerMatlcPbZmuxZ-nJWdJKmaDSDgiGlUGw,647
-davidkhala/ai/opik.py,sha256=YU1XuweMUAzUkhpjxhltt-SBBDBkR3z-PCNo0DqzBRs,39
-davidkhala/ai/agent/README.md,sha256=kIPsx3gOjrpOw7w2qhNEALuCEQkuh4nYp6uBnijdvHE,178
-davidkhala/ai/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/agent/langgraph.py,sha256=jrc_Yvgo7eJjd3y5UJn0t1FzpnObDGYscwgsuVl2O_I,1052
-davidkhala/ai/agent/ragflow.py,sha256=UaK31us6V0NhAPCthGo07rQsm72vlR-McmihC_NDe1g,273
-davidkhala/ai/agent/dify/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/agent/dify/const.py,sha256=gU4lPBe4U2taakN2jhdPMRWXkqlyCg-YRE8JJmtsblo,218
-davidkhala/ai/agent/dify/interface.py,sha256=bTOI38ZjtkgoSw-ysgFwBZ1QkKVAa92gjOnERDoagQA,118
-davidkhala/ai/agent/dify/model.py,sha256=1LEwKWWkFNmhbBWABEu7I45DRZ_BFGDP5uTHDrvldoo,641
-davidkhala/ai/agent/dify/api/__init__.py,sha256=9-8OesuXF_wPmPrh_gEZpEZP51dcZxb0i6ixOBYKcwQ,876
-davidkhala/ai/agent/dify/api/app.py,sha256=y1mILC-fvQpeH50ASbFBluD9tFAwXu_IWwtwucMV5jM,3801
-davidkhala/ai/agent/dify/api/knowledge.py,sha256=5ePqvzjBHNtQ64Dzt39wBWedYVeQJc23syNe9LFnGw8,5960
-davidkhala/ai/agent/dify/ops/__init__.py,sha256=frcCV1k9oG9oKj3dpUqdJg1PxRT2RSN_XKdLCPjaYaY,2
-davidkhala/ai/agent/dify/ops/console/__init__.py,sha256=-a81jgCJ3s2B3i1GQ7ge1aZRfbvlALwGDHVu_GEET-A,237
-davidkhala/ai/agent/dify/ops/console/knowledge.py,sha256=I1v0iE_b4VPc2Zsyt4ci_oX080Qbgn3oXObP4uVEphg,5788
-davidkhala/ai/agent/dify/ops/console/plugin.py,sha256=iJxC0xLBgSIbiumcYc35mKVMuxG6t5gsNUFOtk52ilY,2356
-davidkhala/ai/agent/dify/ops/console/session.py,sha256=9IIdQqtTuYrc7QI9XpZxrCMb-K3m6QWH5FlsckSGCgg,991
-davidkhala/ai/agent/dify/ops/db/__init__.py,sha256=HYfJEnoFAoJJck2xvTDYx8zpw9Qao7sHXOGvW0diPqw,517
-davidkhala/ai/agent/dify/ops/db/app.py,sha256=IRiSiR0v387p4p3J7M9xEkJ7pfQyO5DL6chpx7Z2IzA,1319
-davidkhala/ai/agent/dify/ops/db/knowledge.py,sha256=GVaK5QmU_VxB8fDxV60uiYiIeR3JEn3IXJTlJHLiT5U,2917
-davidkhala/ai/agent/dify/ops/db/orm.py,sha256=CnZj8mV2RZhw_7hF1YICTUjROQ66hR5_8OCMQvtujnY,4575
-davidkhala/ai/agent/dify/ops/db/sys.py,sha256=U_qqopUMlgsilhHaG_ids6gtd-pNiR_Jm0kAr9hIL7M,188
-davidkhala/ai/agent/dify/plugins/__init__.py,sha256=iTWvutlkN9bXgptesi05M447nTeF5hKFAIfn4EviFj0,183
-davidkhala/ai/agent/dify/plugins/file.py,sha256=o-HjHSFwRTNIYs8IxqZUSnBbh-xr8f-xMUM3iU9wCCQ,390
-davidkhala/ai/agent/dify/plugins/firecrawl.py,sha256=lB_f8W_bdg-7PeBKmF0-HdwYyakV_0D3nET5iT-Z1KM,460
-davidkhala/ai/agent/dify/plugins/jina.py,sha256=dQ5iJxDLWtChXb1IjCtsHctgUtgjOiDfWOuR2u0aUIM,190
-davidkhala/ai/agent/dify/plugins/popular.py,sha256=QseXF_R7-6jFmHmuXgScoCjhrOS_xiC0DTqW8oOoMGM,731
-davidkhala/ai/ali/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/ali/agentbay.py,sha256=O5t71GGwtDgBE1zUXJDYe5djMVwSaNOwn5k8zg1xa18,1200
-davidkhala/ai/ali/dashscope.py,sha256=SZIzRhVHlLx3s5I2RNUh2-u8OoSdrbvoN5e1k8Mh8N0,1943
-davidkhala/ai/api/__init__.py,sha256=q2Ro5nhW5kJx2CYR1MRVamjTT5tTexPZwhrS2hwAvFM,1319
-davidkhala/ai/api/openrouter.py,sha256=khccJr5cBnudFy6Jc2O3A1TNCuHH_5W6Q2tXrkwlUYE,2308
-davidkhala/ai/api/siliconflow.py,sha256=JbnOSv8LJLtwYSNNB8_SMBMQzOgHDtQYZKA9A2BC4sY,2139
-davidkhala/ai/google/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-davidkhala/ai/google/adk.py,sha256=QwxYoOzT2Hol03V4NM0PF_HAzUGb4fB18VUAYacYbAY,657
-davidkhala/ai/google/gemini.py,sha256=Xf4HDOOcK4-jEBERzuLnQNFsU61P2fFx4K0z-ijvNHE,214
-davidkhala/ai/huggingface/BAAI.py,sha256=LZ9kp5Gfql4UzuTn4osyekI6VV1H3RIfED2IolXFj5c,341
-davidkhala/ai/huggingface/__init__.py,sha256=FJyU8eOfWQWKAvkIa5qwubF9ghsSQ8C0e6p6DKyomgs,521
-davidkhala/ai/huggingface/inference.py,sha256=bYN0PtLF2CaIHzdTP4LaTALJhcawvuLnLR7rhMVqwDE,333
-davidkhala/ai/openai/__init__.py,sha256=GXzWaw2ER3YFGHG6TPD9SmAHV6Tpsnqxj6tXlaWsrko,1897
-davidkhala/ai/openai/azure.py,sha256=WmWSz9pKlUrQLSH25m1jE1l-mNWw9QQARj8uliOv8VU,1138
-davidkhala/ai/openai/native.py,sha256=MB0nDnzCOj_M42RMhdK3HTMVnxGnwpLT2GeLwSrepwI,704
-davidkhala/ai/openrouter/__init__.py,sha256=P8UvolZihN_CVBQ7BT1Fb6mSMFEQLyLY9G5bBDZhC0o,1037
-davidkhala_ai-0.2.0.dist-info/METADATA,sha256=jWA9caQqfyxxVWCj8i9nAw8Ni4Fkf7-kKjjDZvjr0r4,1607
-davidkhala_ai-0.2.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-davidkhala_ai-0.2.0.dist-info/RECORD,,