koi-net-slack-sensor-node 0.1.0__tar.gz

koi_net_slack_sensor_node-0.1.0/.github/workflows/publish-to-pypi.yml

@@ -0,0 +1,81 @@
+name: Publish Python package to PyPI
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+
+jobs:
+  build:
+    name: Build distro
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        persist-credentials: false
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.x"
+    - name: Install pypa/build
+      run: python3 -m pip install build --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v5
+      with:
+        name: python-package-distributions
+        path: dist/
+  
+  publish-to-pypi:
+    name: Publish Python distribution to PyPI
+    if: startsWith(github.ref, 'refs/tags/')
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/koi-net-slack-sensor-node
+    permissions:
+      id-token: write
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v6
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distro to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: >-
+      Sign the Python distribution with Sigstore
+      and upload them to GitHub Release
+    needs:
+    - publish-to-pypi
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+      id-token: write
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Sign the dists with Sigstore
+      uses: sigstore/gh-action-sigstore-python@v3.0.0
+      with:
+        inputs: >-
+          ./dist/*.tar.gz
+          ./dist/*.whl
+    - name: Upload artifact signatures to GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: >-
+        gh release upload
+        "$GITHUB_REF_NAME" dist/**
+        --repo "$GITHUB_REPOSITORY"

koi_net_slack_sensor_node-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+rid_cache
+identity.json
+events_queue.json
+venv
+.env
+*.json
+__pycache__
+config.yaml
+priv_key.pem
+*.ndjson

koi_net_slack_sensor_node-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 BlockScience
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

koi_net_slack_sensor_node-0.1.0/PKG-INFO

@@ -0,0 +1,39 @@
+Metadata-Version: 2.4
+Name: koi-net-slack-sensor-node
+Version: 0.1.0
+Summary: KOI-net Slack sensor node
+Project-URL: Homepage, https://github.com/BlockScience/koi-net-slack-sensor-node/
+Author-email: Luke Miller <luke@block.science>
+License: MIT License
+        
+        Copyright (c) 2025 BlockScience
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: aiohttp>=3.13.2
+Requires-Dist: koi-net~=2.0
+Requires-Dist: slack-bolt>=1.26.0
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: twine>=6.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# koi-net-slack-sensor-node
+Slack sensor node implementation for BlockScience's KOI-net

koi_net_slack_sensor_node-0.1.0/README.md

@@ -0,0 +1,2 @@
+# koi-net-slack-sensor-node
+Slack sensor node implementation for BlockScience's KOI-net

koi_net_slack_sensor_node-0.1.0/koi-net-slack-sensor-node.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=KOI-net Slack Sensor Node Service
+After=network.target
+
+[Service]
+WorkingDirectory=/home/dev/koi-net-slack-sensor-node
+ExecStart=/home/dev/koi-net-slack-sensor-node/.venv/bin/python3 -m koi_net_slack_sensor_node
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

koi_net_slack_sensor_node-0.1.0/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "koi-net-slack-sensor-node"
+version = "0.1.0"
+description = "KOI-net Slack sensor node"
+authors = [
+    {name = "Luke Miller", email = "luke@block.science"}
+]
+readme = "README.md"
+requires-python = ">=3.10"
+license = {file = "LICENSE"}
+dependencies = [
+    "koi-net~=2.0",
+    "aiohttp>=3.13.2",
+    "slack_bolt>=1.26.0"
+]
+
+[project.optional-dependencies]
+dev = ["twine>=6.0", "build"]
+
+[project.entry-points."koi_net.node"]
+slack_sensor = "koi_net_slack_sensor_node"
+
+[project.urls]
+Homepage = "https://github.com/BlockScience/koi-net-slack-sensor-node/"

koi_net_slack_sensor_node-0.1.0/src/koi_net_slack_sensor_node/__main__.py

@@ -0,0 +1,3 @@
+from .core import SlackSensorNode
+
+SlackSensorNode().run()

koi_net_slack_sensor_node-0.1.0/src/koi_net_slack_sensor_node/backfiller.py

@@ -0,0 +1,157 @@
+import asyncio
+import threading
+from logging import Logger
+from dataclasses import dataclass, field
+
+from koi_net.core import KobjQueue
+from koi_net.components.interfaces import ThreadedComponent
+from slack_bolt.async_app import AsyncApp
+from slack_sdk.errors import SlackApiError
+from rid_lib.ext import Bundle
+from rid_lib.types import SlackMessage
+
+from .config import SlackSensorNodeConfig
+
+
+@dataclass
+class Backfiller(ThreadedComponent):
+    log: Logger
+    slack_app: AsyncApp
+    config: SlackSensorNodeConfig
+    kobj_queue: KobjQueue
+    should_exit: threading.Event = field(init=False, default_factory=threading.Event)
+        
+    def start(self):
+        self.should_exit.clear()
+        super().start()
+        
+    def stop(self):
+        self.should_exit.set()
+        super().stop()
+
+    async def auto_retry(self, function, **kwargs):
+        try:
+            return await function(**kwargs)
+        except SlackApiError as e:
+            if e.response["error"] == "ratelimited":
+                retry_after = int(e.response.headers["Retry-After"])
+                self.log.info(f"timed out, waiting {retry_after} seconds")
+                await asyncio.sleep(retry_after)
+                return await function(**kwargs)
+            elif e.response["error"] == "not_in_channel":
+                self.log.info(f"not in channel {kwargs['channel']}, attempting to join")
+                await self.slack_app.client.conversations_join(channel=kwargs["channel"])
+                return await function(**kwargs)
+            else:
+                self.log.warning("unknown error", e)
+                
+    def run(self):
+        asyncio.run(self.backfill_messages())
+
+    async def backfill_messages(self):
+        resp = await self.slack_app.client.team_info()
+        team = resp.data["team"]
+        team_id = team["id"]
+
+        channels = [{"id": cid} for cid in self.config.slack.allowed_channels]
+        
+        self.log.info("Scanning for channels")
+        
+        # get list of channels
+        channel_cursor = None
+        while (not channels or channel_cursor) and not self.should_exit.is_set():
+            resp = await self.slack_app.client.conversations_list(cursor=channel_cursor)
+            result = resp.data
+            channels.extend(result["channels"])
+            self.log.info(f"Found {len(result['channels'])} channels")
+            channel_cursor = result.get("response_metadata", {}).get("next_cursor")
+
+        self.log.info(f"Scanning {len(channels)} channels for messages")
+        for channel in channels:
+            channel_id = channel["id"]
+            
+            self.log.info(f"Scanning {channel_id}...")
+
+            # get list of messages in channel
+            message_cursor = None
+            messages = []
+            while (not messages or message_cursor) and not self.should_exit.is_set():
+                result = await self.auto_retry(self.slack_app.client.conversations_history,
+                    channel=channel_id,
+                    limit=500,
+                    cursor=message_cursor,
+                    oldest=self.config.slack.last_processed_ts
+                )
+                
+                if not result["messages"]: break
+                
+                messages.extend(result["messages"])
+                self.log.info(f"Found {len(result['messages'])} messages")
+                if result["has_more"]:
+                    message_cursor = result["response_metadata"]["next_cursor"]
+                else:
+                    message_cursor = None
+
+            self.log.info(f"Scanning {len(messages)} messages")
+            messages.reverse()
+            for message in messages:
+                if self.should_exit.is_set():
+                    return
+                
+                message_rid = SlackMessage(team_id, channel_id, message["ts"])
+                
+                if message.get("subtype") is None:
+                    
+                    message_bundle = Bundle.generate(
+                        rid=message_rid,
+                        contents=message
+                    )
+                    self.log.info(f"{message_rid}")
+                    self.kobj_queue.push(bundle=message_bundle)
+                
+                thread_ts = message.get("thread_ts")
+                
+                # ignore threaded messages sent to channel (double counted within thread)
+                if thread_ts and (thread_ts != message["ts"]):
+                    continue
+
+                if thread_ts:
+                    threaded_message_cursor = None
+                    threaded_messages = []
+                    while (not threaded_messages or threaded_message_cursor) and not self.should_exit.is_set():
+                        result = await self.auto_retry(self.slack_app.client.conversations_replies,
+                            channel=channel_id,
+                            ts=thread_ts,
+                            limit=500,
+                            cursor=threaded_message_cursor
+                        )
+                                
+                        threaded_messages.extend(result["messages"])
+                        
+                        if result["has_more"]:
+                            threaded_message_cursor = result["response_metadata"]["next_cursor"]
+                        else:
+                            threaded_message_cursor = None
+                            
+                    self.log.info(f"{message_rid} thread with {len(threaded_messages)} messages")
+                    
+                    # don't double count thread parent message
+                    for threaded_message in threaded_messages[1:]:
+                        if self.should_exit.is_set():
+                            return
+                        
+                        threaded_message_rid = SlackMessage(
+                            team_id, 
+                            channel_id, 
+                            threaded_message["ts"]
+                        )
+                        if threaded_message.get("subtype") is None:
+                            threaded_message_bundle = Bundle.generate(
+                                rid=threaded_message_rid,
+                                contents=threaded_message
+                            )
+                            
+                            self.log.info(f"{threaded_message_rid}")
+                            self.kobj_queue.push(bundle=threaded_message_bundle)
+
+        self.log.info("done")

koi_net_slack_sensor_node-0.1.0/src/koi_net_slack_sensor_node/config.py

@@ -0,0 +1,44 @@
+from pydantic import BaseModel, Field
+from rid_lib.types import (
+    SlackMessage, 
+    SlackChannel, 
+    SlackUser, 
+    SlackWorkspace
+)
+from koi_net.config import (
+    FullNodeConfig, 
+    KoiNetConfig, 
+    FullNodeProfile, 
+    NodeProvides,
+    EnvConfig
+)
+
+
+class SlackEnvConfig(EnvConfig):
+    slack_bot_token: str
+    slack_signing_secret: str
+    slack_app_token: str
+    
+class SlackConfig(BaseModel):
+    allowed_channels: list[str] = []
+    last_processed_ts: str = "0"
+
+class SlackSensorNodeConfig(FullNodeConfig):
+    koi_net: KoiNetConfig = KoiNetConfig(
+        node_name="slack-sensor",
+        node_profile=FullNodeProfile(
+            provides=NodeProvides(
+                event=[
+                    SlackMessage
+                ],
+                state=[
+                    SlackMessage, 
+                    SlackUser, 
+                    SlackChannel, 
+                    SlackWorkspace
+                ]
+            )
+        )
+    )
+    env: SlackEnvConfig = Field(default_factory=SlackEnvConfig)
+    slack: SlackConfig = SlackConfig()

koi_net_slack_sensor_node-0.1.0/src/koi_net_slack_sensor_node/core.py

@@ -0,0 +1,23 @@
+from koi_net.core import FullNode
+from slack_bolt.async_app import AsyncApp
+
+from .backfiller import Backfiller
+from .server import SlackSensorNodeServer
+from .config import SlackSensorNodeConfig
+from .last_processed_ts_handler import LastProcessedTSHandler
+from .slack_event_handler import SlackEventHandler
+
+
+class SlackSensorNode(FullNode):
+    config_schema = SlackSensorNodeConfig
+    
+    slack_app = lambda config: AsyncApp(
+        token=config.env.slack_bot_token,
+        signing_secret=config.env.slack_signing_secret
+    )
+    
+    server = SlackSensorNodeServer
+    backfiller = Backfiller
+    slack_event_handler = SlackEventHandler
+    
+    last_processed_ts_handler = LastProcessedTSHandler

koi_net_slack_sensor_node-0.1.0/src/koi_net_slack_sensor_node/dereference.py

@@ -0,0 +1,65 @@
+from rid_lib import RID
+from rid_lib.types import SlackMessage, SlackChannel, SlackUser, SlackWorkspace
+from rid_lib.ext import Bundle
+from koi_net.protocol.api.models import BundlesPayload
+from slack_bolt.async_app import AsyncApp
+
+
+class Dereferencer:
+    def __init__(self, slack_app: AsyncApp):
+        self.slack_app = slack_app
+        
+    async def fetch_missing(self, payload: BundlesPayload):
+        found_bundles: list[Bundle] = []
+        for rid in payload.not_found:
+            if type(rid) not in (
+                SlackMessage, SlackChannel, SlackUser, SlackWorkspace
+            ): continue
+            
+            bundle = await self.dereference(rid)
+            if bundle:
+                found_bundles.append(bundle)
+        
+        for bundle in found_bundles:
+            payload.not_found.remove(bundle.rid)
+            payload.bundles.append(bundle)
+        
+        return payload
+
+    async def dereference(self, rid: RID):
+        if type(rid) == SlackMessage:
+            resp = await self.slack_app.client.conversations_replies(
+                channel=rid.channel_id,
+                ts=rid.ts
+            )
+            message = resp["messages"][0]
+            
+            return Bundle.generate(rid, message)
+            
+        elif type(rid) == SlackChannel:
+            resp = await self.slack_app.client.conversations_info(
+                channel=rid.channel_id
+            )
+            channel = resp["channel"]
+            
+            return Bundle.generate(rid, channel)
+            
+        elif type(rid) == SlackUser:
+            profile_resp = await self.slack_app.client.users_profile_get(user=rid.user_id)
+            profile = profile_resp["profile"]
+            
+            user_resp = await self.slack_app.client.users_info(user=rid.user_id)
+            user = user_resp["user"]
+            
+            user["profile"] = profile
+            
+            return Bundle.generate(rid, user)
+        
+        elif type(rid) == SlackWorkspace:
+            resp = await self.slack_app.client.team_info(team=rid.team_id)
+            workspace = resp["team"]
+            
+            return Bundle.generate(rid, workspace)
+        
+        else:
+            raise TypeError(f"RID of type {type(rid)!r} is not allowed")

koi_net_slack_sensor_node-0.1.0/src/koi_net_slack_sensor_node/last_processed_ts_handler.py

@@ -0,0 +1,24 @@
+from typing import cast
+from dataclasses import dataclass
+
+from rid_lib.types import SlackMessage
+from koi_net.components.interfaces import HandlerType, KnowledgeHandler
+from koi_net.protocol import KnowledgeObject
+
+from .config import SlackSensorNodeConfig
+
+
+@dataclass
+class LastProcessedTSHandler(KnowledgeHandler):
+    config: SlackSensorNodeConfig
+    
+    handler_type = HandlerType.RID
+    rid_types = (SlackMessage,)
+    
+    def handle(self, kobj: KnowledgeObject):
+        msg_rid = cast(SlackMessage, kobj.rid)
+        if float(msg_rid.ts) < float(self.config.slack.last_processed_ts):
+            return
+        
+        self.config.slack.last_processed_ts = msg_rid.ts
+        self.config.save_to_yaml()

koi_net_slack_sensor_node-0.1.0/src/koi_net_slack_sensor_node/server.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+
+from fastapi import Request
+from slack_bolt.async_app import AsyncApp
+from slack_bolt.adapter.fastapi.async_handler import AsyncSlackRequestHandler
+from koi_net.components import NodeServer
+
+
+@dataclass
+class SlackSensorNodeServer(NodeServer):
+    slack_app: AsyncApp
+    
+    def __post_init__(self):
+        super().__post_init__()
+        
+        @self.app.post("/slack-event-listener")
+        async def slack_listener(request: Request):
+            return await AsyncSlackRequestHandler(self.slack_app).handle(request)

koi_net_slack_sensor_node-0.1.0/src/koi_net_slack_sensor_node/slack_event_handler.py

@@ -0,0 +1,87 @@
+from dataclasses import dataclass
+from logging import Logger
+
+from slack_bolt.async_app import AsyncApp
+from rid_lib.ext import Bundle
+from rid_lib.types import SlackMessage
+from koi_net.core import KobjQueue
+from koi_net.protocol.event import EventType
+
+from .config import SlackSensorNodeConfig
+
+
+@dataclass
+class SlackEventHandler:
+    log: Logger
+    slack_app: AsyncApp
+    config: SlackSensorNodeConfig
+    kobj_queue: KobjQueue
+    
+    def __post_init__(self):
+        self.register_handlers()
+        
+    def register_handlers(self):
+        @self.slack_app.event("message")
+        async def handle_message_event(event):
+            subtype = event.get("subtype")
+            # new message
+            if not subtype:
+                message_rid = SlackMessage(
+                    team_id=event["team"],
+                    channel_id=event["channel"],
+                    ts=event["ts"]
+                )
+                
+                if message_rid.channel_id not in self.config.slack.allowed_channels:
+                    return
+                
+                # normalize to non event message structure
+                data = event
+                del data["channel"]
+                del data["event_ts"]
+                del data["channel_type"]
+                
+                msg_bundle = Bundle.generate(
+                    rid=message_rid,
+                    contents=data
+                )
+                
+                self.log.info(f"Handling new Slack message {message_rid!r}")
+                
+                self.kobj_queue.push(bundle=msg_bundle)
+                
+            
+            elif subtype == "message_changed":
+                message_rid = SlackMessage(
+                    team_id=event["message"]["team"],
+                    channel_id=event["channel"],
+                    ts=event["message"]["ts"]
+                )
+                # normalize to non event message structure
+                data = event["message"]
+                data.pop("source_team", None)
+                data.pop("user_team", None)
+                
+                msg_bundle = Bundle.generate(
+                    rid=message_rid,
+                    contents=data
+                )
+                
+                self.log.info(f"Handling updated Slack message {message_rid!r}")
+                
+                self.kobj_queue.push(bundle=msg_bundle)
+            
+            elif subtype == "message_deleted":
+                message_rid = SlackMessage(
+                    team_id=event["previous_message"]["team"],
+                    channel_id=event["channel"],
+                    ts=event["previous_message"]["ts"]
+                )
+                
+                self.log.info(f"Handling deleted Slack message {message_rid!r}")
+                
+                self.kobj_queue.push(rid=message_rid, event_type=EventType.FORGET)
+            
+            else:
+                self.log.info(f"Ignoring unsupported Slack message subtype {subtype}")
+                return