web-queue2 0.2.0__tar.gz → 0.3.0__tar.gz

{web_queue2-0.2.0 → web_queue2-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: web-queue2
-Version: 0.2.0
+Version: 0.3.0
 Summary: Get web content from queue.
 License: MIT
 License-File: LICENSE
@@ -17,6 +17,7 @@ Requires-Dist: bs4
 Requires-Dist: cachetic
 Requires-Dist: dictpress
 Requires-Dist: fastapi
+Requires-Dist: html-to-markdown
 Requires-Dist: httpx
 Requires-Dist: huey
 Requires-Dist: logfire[redis]
@@ -28,6 +29,7 @@ Requires-Dist: pydantic (>=2)
 Requires-Dist: pydantic-settings
 Requires-Dist: str-or-none
 Requires-Dist: tiktoken
+Requires-Dist: typer
 Requires-Dist: yarl
 Project-URL: Homepage, https://github.com/allen2c/web-queue
 Project-URL: PyPI, https://pypi.org/project/web-queue/

{web_queue2-0.2.0 → web_queue2-0.3.0}/pyproject.toml

@@ -5,6 +5,7 @@ dependencies = [
   "cachetic",
   "dictpress",
   "fastapi",
+  "html-to-markdown",
   "httpx",
   "huey",
   "logfire[redis]",
@@ -16,6 +17,7 @@ dependencies = [
   "pydantic-settings",
   "str-or-none",
   "tiktoken",
+  "typer",
   "yarl",
 ]
 description = "Get web content from queue."
@@ -23,7 +25,7 @@ license = { text = "MIT" }
 name = "web-queue2"
 readme = "README.md"
 requires-python = ">=3.11,<4"
-version = "0.2.0"
+version = "0.3.0"
 
 [project.urls]
 Homepage = "https://github.com/allen2c/web-queue"

web_queue2-0.3.0/web_queue/VERSION

@@ -0,0 +1 @@
+0.3.0

web_queue2-0.3.0/web_queue/app.py

@@ -0,0 +1,127 @@
+import asyncio
+import logging
+import typing
+
+import fastapi
+import huey
+import huey.exceptions
+import logfire
+import logging_bullet_train as lbt
+from huey.api import Task
+
+from web_queue.client import Settings, WebQueueClient
+from web_queue.types.fetch_html_message import FetchHTMLMessage
+from web_queue.types.html_content import HTMLContent
+from web_queue.types.message import MessageStatus, MessageUpdate
+from web_queue.types.model_var import ModelVar
+
+lbt.set_logger("web_queue")
+
+logfire.configure()
+logfire.instrument_openai()
+
+logger = logging.getLogger(__name__)
+
+logger.info("Web queue app starting...")
+
+wq_settings = Settings()
+wq_client = WebQueueClient(wq_settings)
+logger.info(f"Web queue connecting to redis: {wq_settings.web_queue_safe_url}")
+
+huey_app = huey.RedisExpireHuey(
+    wq_settings.WEB_QUEUE_NAME,
+    url=wq_settings.WEB_QUEUE_URL.get_secret_value(),
+    expire_time=24 * 60 * 60,  # 24 hours
+)
+
+
+def retrieve_result(task_id: str) -> typing.Optional[typing.Text]:
+    try:
+        result: str | None = huey_app.result(
+            task_id,
+            blocking=True,
+            timeout=2,
+        )  # type: ignore
+    except huey.exceptions.ResultTimeout:
+        logger.error(f"Timeout waiting for result for task {task_id}")
+        return None
+    if result is None:
+        logger.error(f"No result found for task {task_id}")
+    return result
+
+
+def retrieve_result_as(task_id: str, model: typing.Type[ModelVar]) -> ModelVar:
+    result = retrieve_result(task_id)
+    if result is None:
+        raise fastapi.HTTPException(
+            status_code=404, detail=f"No result found for task {task_id}"
+        )
+    return model.model_validate_json(result)
+
+
+@huey_app.task(
+    retries=1,
+    retry_delay=8,
+    expires=24 * 60 * 60,
+    context=True,
+)
+def fetch_html(
+    message: typing.Union["FetchHTMLMessage", str, bytes], task: Task
+) -> typing.Optional[typing.Text]:
+    from web_queue.types.fetch_html_message import FetchHTMLMessage
+
+    global wq_client
+
+    message = FetchHTMLMessage.from_any(message)
+    message.id = task.id
+    message.status = MessageStatus.RUNNING
+
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+
+    update_message_func = wq_client.messages.wrap_update_message(message.id, message)
+
+    try:
+        logger.info(f"Fetching HTML with parameters: {message.data.model_dump_json()}")
+        update_message_func(
+            MessageUpdate(
+                total_steps=100,
+                completed_steps=0,
+                status=MessageStatus.RUNNING,
+                message_text="Starting to fetch HTML...",
+            )
+        )
+
+        html_content: "HTMLContent" = loop.run_until_complete(
+            wq_client.fetch(
+                **message.data.model_dump(), step_callback=update_message_func
+            )
+        )
+
+        update_message_func(
+            MessageUpdate(
+                total_steps=100,
+                completed_steps=100,
+                status=MessageStatus.COMPLETED,
+                message_text="Finished fetching HTML",
+            )
+        )
+
+        return html_content.model_dump_json()
+
+    except Exception as e:
+        logger.exception(e)
+        logger.error(f"Failed to fetch HTML: {e}")
+        update_message_func(
+            MessageUpdate(
+                total_steps=100,
+                completed_steps=100,
+                status=MessageStatus.FAILED,
+                message_text=f"Failed to fetch HTML: {e}",
+            )
+        )
+
+    finally:
+        loop.close()
+
+    return None

web_queue2-0.3.0/web_queue/client/__init__.py

@@ -0,0 +1,4 @@
+from web_queue.client._client import WebQueueClient
+from web_queue.client.config import Settings
+
+__all__ = ["WebQueueClient", "Settings"]

{web_queue2-0.2.0 → web_queue2-0.3.0}/web_queue/client/_client.py

@@ -8,9 +8,10 @@ if typing.TYPE_CHECKING:
     from web_queue.client.ai import AI
     from web_queue.client.clean import Clean
     from web_queue.client.config import Settings
+    from web_queue.client.messages import Messages
     from web_queue.client.web import Web
     from web_queue.types.html_content import HTMLContent
-    from web_queue.types.step_callback import StepCallbackType
+    from web_queue.types.message import MessageUpdate
 
 
 class WebQueueClient:
@@ -37,6 +38,12 @@ class WebQueueClient:
 
         return AI(self)
 
+    @functools.cached_property
+    def messages(self) -> "Messages":
+        from web_queue.client.messages import Messages
+
+        return Messages(self)
+
     async def fetch(
         self,
         url: yarl.URL | httpx.URL | str,
@@ -47,7 +54,7 @@ class WebQueueClient:
         scrolling_times: int = 3,
         human_delay_base_delay: float = 1.2,
         dynamic_content_loading_delay: float = 2.0,
-        step_callback: typing.Optional["StepCallbackType"] = None,
+        step_callback: typing.Optional[typing.Callable[["MessageUpdate"], None]] = None,
     ) -> "HTMLContent":
         from web_queue.types.html_content import HTMLContent
         from web_queue.utils.html_to_str import htmls_to_str

{web_queue2-0.2.0 → web_queue2-0.3.0}/web_queue/client/ai/_ai.py

@@ -11,7 +11,7 @@ from rich.pretty import pretty_repr
 
 from web_queue.client import WebQueueClient
 from web_queue.types.html_metadata_response import HTMLMetadataResponse
-from web_queue.types.step_callback import StepCallbackType
+from web_queue.types.message import MessageUpdate
 from web_queue.utils.compression import compress, decompress
 
 if typing.TYPE_CHECKING:
@@ -28,7 +28,7 @@ class AI:
     async def as_html_metadata(
         self,
         html: typing.Union["bs4.BeautifulSoup", typing.Text],
-        step_callback: typing.Optional[StepCallbackType] = None,
+        step_callback: typing.Optional[typing.Callable[["MessageUpdate"], None]] = None,
     ) -> typing.Optional[HTMLMetadataResponse]:
         """Extract content metadata and CSS selector from HTML.
 
@@ -110,7 +110,13 @@ class AI:
         ).strip()
 
         if step_callback:
-            step_callback(100, 75, "Starting to extract content metadata...")
+            step_callback(
+                MessageUpdate(
+                    total_steps=100,
+                    completed_steps=75,
+                    message_text="Starting to extract content metadata...",
+                )
+            )
 
         try:
             parsed_cmpl = await openai_client.chat.completions.parse(
@@ -139,7 +145,13 @@ class AI:
                 )
 
                 if step_callback:
-                    step_callback(100, 90, "Finished extracting content metadata.")
+                    step_callback(
+                        MessageUpdate(
+                            total_steps=100,
+                            completed_steps=90,
+                            message_text="Finished extracting content metadata",
+                        )
+                    )
 
                 return output
 

{web_queue2-0.2.0 → web_queue2-0.3.0}/web_queue/client/config.py

@@ -6,9 +6,16 @@ import cachetic
 import openai
 import pydantic as pydantic
 import pydantic_settings
+from str_or_none import str_or_none
 
 
 class Settings(pydantic_settings.BaseSettings):
+    # Core
+    WEB_QUEUE_NAME: str = pydantic.Field(default="web-queue")
+    WEB_QUEUE_URL: pydantic.SecretStr = pydantic.SecretStr("")
+    MESSAGE_CACHE_EXPIRE_SECONDS: int = pydantic.Field(default=60 * 60 * 24)  # 1 day
+
+    # AI
     OPENAI_MODEL: str = pydantic.Field(default="gpt-4.1-nano")
     OPENAI_API_KEY: pydantic.SecretStr = pydantic.SecretStr("")
 
@@ -24,6 +31,24 @@ class Settings(pydantic_settings.BaseSettings):
         default=60 * 60 * 24
     )  # 1 day
 
+    @pydantic.model_validator(mode="after")
+    def validate_values(self) -> typing.Self:
+        if str_or_none(self.WEB_QUEUE_NAME) is None:
+            raise ValueError("WEB_QUEUE_NAME is required")
+        if str_or_none(self.WEB_QUEUE_URL.get_secret_value()) is None:
+            raise ValueError("WEB_QUEUE_URL is required")
+        return self
+
+    @functools.cached_property
+    def message_cache(self) -> "cachetic.Cachetic[typing.Text]":
+        import redis
+
+        return cachetic.Cachetic(
+            object_type=pydantic.TypeAdapter(typing.Text),
+            cache_url=redis.from_url(self.WEB_QUEUE_URL.get_secret_value()),
+            default_ttl=self.MESSAGE_CACHE_EXPIRE_SECONDS,
+        )
+
     @functools.cached_property
     def openai_client(self) -> openai.AsyncOpenAI:
         return openai.AsyncOpenAI(api_key=self.OPENAI_API_KEY.get_secret_value())
@@ -48,6 +73,12 @@ class Settings(pydantic_settings.BaseSettings):
             default_ttl=self.COMPRESSED_BASE64_CACHE_EXPIRE_SECONDS,
         )
 
+    @property
+    def web_queue_safe_url(self) -> str:
+        import yarl
+
+        return str(yarl.URL(self.WEB_QUEUE_URL.get_secret_value()).with_password("***"))
+
     @property
     def web_screenshot_path(self) -> pathlib.Path:
         _path = pathlib.Path(self.WEB_SCREENSHOT_PATH)

web_queue2-0.3.0/web_queue/client/messages/__init__.py

@@ -0,0 +1,3 @@
+from web_queue.client.messages._messages import Messages
+
+__all__ = ["Messages"]

web_queue2-0.3.0/web_queue/client/messages/_messages.py

@@ -0,0 +1,116 @@
+import time
+import typing
+
+import fastapi
+
+from web_queue.client import WebQueueClient
+from web_queue.types.message import Message, MessageStatus, MessageUpdate, MessageVar
+
+
+class Messages:
+    def __init__(self, client: WebQueueClient):
+        self.client = client
+
+    def get(self, message_id: str, *, timeout: float = 10.0) -> typing.Optional[str]:
+        ts = time.perf_counter()
+        while time.perf_counter() - ts < timeout:
+            message_cache_key = self.get_cache_key(message_id)
+            maybe_json = self.client.settings.message_cache.get(message_cache_key)
+            if maybe_json is not None:
+                break
+            else:
+                time.sleep(0.1)
+
+        if maybe_json is None:
+            return None
+        return maybe_json
+
+    def retrieve(self, message_id: str, *, timeout: float = 10.0) -> str:
+        json_data = self.get(message_id, timeout=timeout)
+        if json_data is None:
+            raise fastapi.HTTPException(status_code=404, detail="Message not found")
+        return json_data
+
+    def retrieve_as(
+        self, message_id: str, model: typing.Type[MessageVar], *, timeout: float = 10.0
+    ) -> MessageVar:
+        json_data = self.retrieve(message_id, timeout=timeout)
+        return model.model_validate_json(json_data)
+
+    def set(self, message_id: str, message: Message) -> None:
+        message_cache_key = self.get_cache_key(message_id)
+        self.client.settings.message_cache.set(
+            message_cache_key, message.model_dump_json()
+        )
+
+    def update(
+        self,
+        message_id: str,
+        message_update: MessageUpdate,
+    ) -> Message:
+        message = self.retrieve_as(message_id, Message)
+        if message_update.message_text is not None:
+            message.message_text = message_update.message_text
+        if message_update.data is not None:
+            message.data = message_update.data
+        if message_update.status is not None:
+            message.status = message_update.status
+        if message_update.total_steps is not None:
+            message.total_steps = message_update.total_steps
+        if message_update.completed_steps is not None:
+            message.completed_steps = message_update.completed_steps
+        if message_update.error is not None:
+            message.error = message_update.error
+        self.set(message_id, message)
+
+        return message
+
+    def wrap_update_message(
+        self, message_id: str, message: Message
+    ) -> typing.Callable[[MessageUpdate], None]:
+        def _update(message_update: MessageUpdate) -> None:
+            if message_update.message_text is not None:
+                message.message_text = message_update.message_text
+            if message_update.data is not None:
+                message.data = message_update.data
+            if message_update.status is not None:
+                message.status = message_update.status
+            if message_update.total_steps is not None:
+                message.total_steps = message_update.total_steps
+            if message_update.completed_steps is not None:
+                message.completed_steps = message_update.completed_steps
+            if message_update.error is not None:
+                message.error = message_update.error
+            self.set(message_id, message)
+
+        return _update
+
+    def poll_util_done(
+        self,
+        message_id: str,
+        *,
+        timeout: float = 60.0,
+        model: typing.Type[MessageVar],
+        delay: float = 0.2,
+    ) -> MessageVar:
+        ts = time.perf_counter()
+        msg: MessageVar | None = None
+
+        while is_timeout := (time.perf_counter() - ts < timeout):
+            msg = self.retrieve_as(message_id, model)
+            if msg.status in [MessageStatus.COMPLETED, MessageStatus.FAILED]:
+                break
+            time.sleep(delay)
+
+        if msg is None:
+            raise fastapi.HTTPException(status_code=404, detail="Message not found")
+
+        if is_timeout:
+            raise fastapi.HTTPException(
+                status_code=408, detail="Timeout waiting for message to be done"
+            )
+
+        return msg
+
+    def get_cache_key(self, message_id: str) -> str:
+        return f"{self.client.settings.WEB_QUEUE_NAME}:message:{message_id}"

{web_queue2-0.2.0 → web_queue2-0.3.0}/web_queue/client/web/_web.py

@@ -14,7 +14,7 @@ from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
 from str_or_none import str_or_none
 
 from web_queue.client import WebQueueClient
-from web_queue.types.step_callback import StepCallbackType
+from web_queue.types.message import MessageUpdate
 from web_queue.utils.compression import compress, decompress
 from web_queue.utils.human_delay import human_delay
 from web_queue.utils.page_with_init_script import page_with_init_script
@@ -49,7 +49,7 @@ class Web:
         scrolling_times: int = 3,
         human_delay_base_delay: float = 1.2,
         dynamic_content_loading_delay: float = 2.0,
-        step_callback: typing.Optional[StepCallbackType] = None,
+        step_callback: typing.Optional[typing.Callable[["MessageUpdate"], None]] = None,
     ) -> bs4.BeautifulSoup:
         _url = str_or_none(str(url))
         if not _url:
@@ -80,7 +80,13 @@ class Web:
                 ],
             )
             if step_callback:
-                step_callback(100, 15, "Launching browser...")
+                step_callback(
+                    MessageUpdate(
+                        total_steps=100,
+                        completed_steps=15,
+                        message_text="Launching browser...",
+                    )
+                )
 
             # Create context
             _viewport_size = secrets.choice(self.VIEWPORT_SIZES)
@@ -106,7 +112,13 @@ class Web:
             page = await page_with_init_script(page)
 
             if step_callback:
-                step_callback(100, 30, "Navigating to URL...")
+                step_callback(
+                    MessageUpdate(
+                        total_steps=100,
+                        completed_steps=30,
+                        message_text="Navigating to URL...",
+                    )
+                )
 
             try:
                 # Navigate to URL
@@ -125,7 +137,13 @@ class Web:
                 await human_delay(h_delay)
 
                 if step_callback:
-                    step_callback(100, 45, "Waiting for full page load...")
+                    step_callback(
+                        MessageUpdate(
+                            total_steps=100,
+                            completed_steps=45,
+                            message_text="Waiting for full page load...",
+                        )
+                    )
 
                 # Simulate smooth mouse circling three times
                 start_position = None
@@ -156,7 +174,13 @@ class Web:
                 )
 
                 if step_callback:
-                    step_callback(100, 60, "Finished fetching HTML content.")
+                    step_callback(
+                        MessageUpdate(
+                            total_steps=100,
+                            completed_steps=60,
+                            message_text="Finished fetching HTML content",
+                        )
+                    )
 
                 # Screenshot and PDF
                 snapshot_filename = f"{int(time.time()*1E3)}_{secrets.token_hex(2)}"

{web_queue2-0.2.0 → web_queue2-0.3.0}/web_queue/types/message.py

@@ -14,7 +14,7 @@ class MessageStatus(enum.StrEnum):
 
 class Message(pydantic.BaseModel):
     id: str | None = None
-    message: str = ""
+    message_text: str = ""
     data: typing.Any
     status: MessageStatus = pydantic.Field(default=MessageStatus.PENDING)
     total_steps: int = pydantic.Field(default=100)
@@ -33,3 +33,15 @@ class Message(pydantic.BaseModel):
             return cls.model_validate_json(any)
         else:
             raise ValueError(f"Invalid type: {type(any)}")
+
+
+MessageVar = typing.TypeVar("MessageVar", bound=Message)
+
+
+class MessageUpdate(pydantic.BaseModel):
+    message_text: typing.Optional[str] = None
+    data: typing.Optional[typing.Any] = None
+    status: typing.Optional[MessageStatus] = None
+    total_steps: typing.Optional[int] = None
+    completed_steps: typing.Optional[int] = None
+    error: typing.Optional[str] = None

web_queue2-0.3.0/web_queue/types/model_var.py

@@ -0,0 +1,5 @@
+import typing
+
+import pydantic
+
+ModelVar = typing.TypeVar("ModelVar", bound=pydantic.BaseModel)

{web_queue2-0.2.0 → web_queue2-0.3.0}/web_queue/utils/html_to_str.py

@@ -1,18 +1,13 @@
 import typing
 
 import bs4
+import html_to_markdown
 
 
 def html_to_str(html: bs4.BeautifulSoup | bs4.Tag | str) -> str:
     html = bs4.BeautifulSoup(html, "html.parser") if isinstance(html, str) else html
-
-    full_text = ""
-    for p in html.find_all("p"):
-        content = p.get_text(separator="\n", strip=True)
-        full_text += content
-        full_text += "\n"
-
-    return full_text.strip()
+    content = html_to_markdown.convert(str(html)).strip()
+    return "\n".join(line.rstrip() for line in content.splitlines())
 
 
 def htmls_to_str(

web_queue2-0.2.0/web_queue/VERSION

@@ -1 +0,0 @@
-0.2.0

web_queue2-0.2.0/web_queue/app.py

@@ -1,91 +0,0 @@
-import asyncio
-import logging
-import typing
-
-import huey
-import logfire
-import logging_bullet_train as lbt
-from huey.api import Task
-
-import web_queue.config
-from web_queue.types.message import MessageStatus
-
-if typing.TYPE_CHECKING:
-    from web_queue.client import WebQueueClient
-    from web_queue.types.fetch_html_message import FetchHTMLMessage
-    from web_queue.types.html_content import HTMLContent
-
-lbt.set_logger("web_queue")
-
-logfire.configure()
-logfire.instrument_openai()
-
-logger = logging.getLogger(__name__)
-
-logger.info("Web queue app starting...")
-
-web_queue_settings = web_queue.config.Settings()
-logger.info(f"Web queue connecting to redis: {web_queue_settings.web_queue_safe_url}")
-
-huey_app = huey.RedisExpireHuey(
-    web_queue_settings.WEB_QUEUE_NAME,
-    url=web_queue_settings.WEB_QUEUE_URL.get_secret_value(),
-    expire_time=24 * 60 * 60,  # 24 hours
-)
-
-
-@huey_app.task(
-    retries=1,
-    retry_delay=8,
-    expires=24 * 60 * 60,
-    context=True,
-)
-def fetch_html(
-    message: typing.Union["FetchHTMLMessage", str, bytes], task: Task
-) -> typing.Optional[typing.Text]:
-    from web_queue.types.fetch_html_message import FetchHTMLMessage
-
-    message = FetchHTMLMessage.from_any(message)
-    message.id = task.id
-    message.status = MessageStatus.RUNNING
-
-    wq_cache_key = web_queue_settings.get_message_cache_key(message.id)
-
-    def update_message_cache(
-        total_steps: int | None = None,
-        completed_steps: int | None = None,
-        message_text: str | None = None,
-    ):
-        if total_steps is not None:
-            message.total_steps = total_steps
-        if completed_steps is not None:
-            message.completed_steps = completed_steps
-        if message_text is not None:
-            message.message = message_text
-        web_queue_settings.message_cache.set(wq_cache_key, message.model_dump_json())
-
-    logger.info(f"Fetching HTML with parameters: {message.data.model_dump_json()}")
-    update_message_cache(message_text="Starting to fetch HTML...")
-
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
-
-    try:
-        wq_client: "WebQueueClient" = web_queue_settings.web_queue_client
-        html_content: "HTMLContent" = loop.run_until_complete(
-            wq_client.fetch(
-                **message.data.model_dump(), step_callback=update_message_cache
-            )
-        )
-        update_message_cache(100, 100, "Finished fetching HTML.")
-        return html_content.model_dump_json()
-
-    except Exception as e:
-        logger.exception(e)
-        logger.error(f"Failed to fetch HTML: {e}")
-        update_message_cache(message_text=f"Failed to fetch HTML: {e}")
-
-    finally:
-        loop.close()
-
-    return None

web_queue2-0.2.0/web_queue/client/__init__.py

@@ -1,3 +0,0 @@
-from web_queue.client._client import WebQueueClient
-
-__all__ = ["WebQueueClient"]

web_queue2-0.2.0/web_queue/config.py

@@ -1,47 +0,0 @@
-import functools
-import typing
-
-import cachetic
-import pydantic
-import pydantic_settings
-import redis
-import yarl
-from str_or_none import str_or_none
-
-if typing.TYPE_CHECKING:
-    from web_queue.client import WebQueueClient
-
-
-class Settings(pydantic_settings.BaseSettings):
-    WEB_QUEUE_NAME: str = pydantic.Field(default="web-queue")
-    WEB_QUEUE_URL: pydantic.SecretStr = pydantic.SecretStr("")
-    MESSAGE_CACHE_EXPIRE_SECONDS: int = pydantic.Field(default=60 * 60 * 24)  # 1 day
-
-    @pydantic.model_validator(mode="after")
-    def validate_values(self) -> typing.Self:
-        if str_or_none(self.WEB_QUEUE_NAME) is None:
-            raise ValueError("WEB_QUEUE_NAME is required")
-        if str_or_none(self.WEB_QUEUE_URL.get_secret_value()) is None:
-            raise ValueError("WEB_QUEUE_URL is required")
-        return self
-
-    @functools.cached_property
-    def web_queue_client(self) -> "WebQueueClient":
-        from web_queue.client import WebQueueClient
-
-        return WebQueueClient()
-
-    @functools.cached_property
-    def message_cache(self) -> "cachetic.Cachetic[typing.Text]":
-        return cachetic.Cachetic(
-            object_type=pydantic.TypeAdapter(typing.Text),
-            cache_url=redis.from_url(self.WEB_QUEUE_URL.get_secret_value()),
-            default_ttl=self.MESSAGE_CACHE_EXPIRE_SECONDS,
-        )
-
-    @property
-    def web_queue_safe_url(self) -> str:
-        return str(yarl.URL(self.WEB_QUEUE_URL.get_secret_value()).with_password("***"))
-
-    def get_message_cache_key(self, message_id: str) -> str:
-        return f"{self.WEB_QUEUE_NAME}:message:{message_id}"

web_queue2-0.2.0/web_queue/types/step_callback.py

@@ -1,10 +0,0 @@
-import typing
-
-StepCallbackType: typing.TypeAlias = typing.Callable[
-    [
-        typing.Annotated[int, "total_steps"],
-        typing.Annotated[int, "completed_steps"],
-        typing.Annotated[str, "message"],
-    ],
-    None,
-]