web2cli 0.2.0__py3-none-any.whl

web2cli/adapters/discord.com/web2cli.yaml

@@ -0,0 +1,476 @@
+meta:
+  spec_version: "0.2"
+  name: discord
+  domain: discord.com
+  base_url: https://discord.com/api/v9
+  version: 0.2.0
+  description: "Discord — servers, channels, messages, DMs"
+  author: web2cli-core
+  transport: http
+  impersonate: chrome
+  aliases:
+    - dc
+    - discord
+  default_headers:
+    User-Agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+    Accept: "application/json"
+
+auth:
+  methods:
+    - type: token
+      env_var: WEB2CLI_DISCORD_TOKEN
+      inject:
+        target: header
+        key: Authorization
+      capture:
+        from: request.header
+        key: Authorization
+        match:
+          host: discord.com
+          path_regex: "^/api/"
+
+resources:
+  guilds:
+    cache:
+      key: guilds
+      ttl: 3600
+    request:
+      method: GET
+      url: /users/@me/guilds
+    response:
+      format: json
+      extract: "$[*]"
+      fields:
+        - name: id
+          from: "$.id"
+        - name: name
+          from: "$.name"
+
+  guild_channels:
+    cache:
+      key: "guild_channels_{{steps.guild.id}}"
+      ttl: 3600
+    request:
+      method: GET
+      url: /guilds/{{steps.guild.id}}/channels
+    response:
+      format: json
+      extract: "$[*]"
+      fields:
+        - name: id
+          from: "$.id"
+        - name: name
+          from: "$.name"
+        - name: type
+          from: "$.type"
+        - name: topic
+          from: "$.topic"
+        - name: position
+          from: "$.position"
+        - name: parent_id
+          from: "$.parent_id"
+
+  dm_channels:
+    cache:
+      key: dm_channels
+      ttl: 300
+    request:
+      method: GET
+      url: /users/@me/channels
+    response:
+      format: json
+      extract: "$[*]"
+      fields:
+        - name: id
+          from: "$.id"
+        - name: name
+          from:
+            coalesce:
+              - "$.recipients[0].global_name"
+              - "$.recipients[0].username"
+        - name: recipients
+          from: "$.recipients[*].username"
+          ops:
+            - join:
+                sep: ", "
+        - name: type
+          from: "$.type"
+
+commands:
+  me:
+    description: "Show current user info"
+    pipeline:
+      - request:
+          name: fetch
+          method: GET
+          url: /users/@me
+      - parse:
+          name: parsed
+          from: fetch
+          format: json
+          extract: "$"
+          fields:
+            - name: id
+              from: "$.id"
+            - name: username
+              from: "$.username"
+            - name: global_name
+              from: "$.global_name"
+            - name: email
+              from: "$.email"
+            - name: phone
+              from: "$.phone"
+            - name: mfa
+              from: "$.mfa_enabled"
+    output:
+      from_step: parsed
+      default_fields: [id, username, global_name, email]
+      default_format: table
+
+  servers:
+    description: "List your Discord servers (guilds)"
+    pipeline:
+      - request:
+          name: fetch
+          method: GET
+          url: /users/@me/guilds
+      - parse:
+          name: parsed
+          from: fetch
+          format: json
+          extract: "$[*]"
+          fields:
+            - name: id
+              from: "$.id"
+            - name: name
+              from: "$.name"
+            - name: owner
+              from: "$.owner"
+            - name: permissions
+              from: "$.permissions"
+    output:
+      from_step: parsed
+      default_fields: [id, name, owner]
+      default_format: table
+
+  channels:
+    description: "List channels in a server"
+    args:
+      server:
+        type: string
+        required: true
+        description: "Server name"
+    pipeline:
+      - resolve:
+          name: guild
+          resource: guilds
+          input: "{{args.server}}"
+          by: name
+          value: id
+          match: ci_equals
+      - request:
+          name: fetch
+          method: GET
+          url: /guilds/{{steps.guild.id}}/channels
+      - parse:
+          name: parsed
+          from: fetch
+          format: json
+          extract: "$[*]"
+          fields:
+            - name: id
+              from: "$.id"
+            - name: name
+              from: "$.name"
+            - name: type
+              from: "$.type"
+            - name: topic
+              from: "$.topic"
+            - name: position
+              from: "$.position"
+            - name: parent_id
+              from: "$.parent_id"
+      - transform:
+          name: sorted
+          from: parsed
+          ops:
+            - sort:
+                by: position
+                order: asc
+    output:
+      from_step: sorted
+      default_fields: [id, name, type, topic]
+      default_format: table
+
+  messages:
+    description: "Get messages from a channel"
+    args:
+      server:
+        type: string
+        required: true
+        description: "Server name"
+      channel:
+        type: string
+        required: true
+        description: "Channel name"
+      limit:
+        type: int
+        required: false
+        default: 20
+        max: 100
+        description: "Number of messages to fetch"
+    pipeline:
+      - resolve:
+          name: guild
+          resource: guilds
+          input: "{{args.server}}"
+          by: name
+          value: id
+          match: ci_equals
+      - resolve:
+          name: channel
+          resource: guild_channels
+          input: "{{args.channel}}"
+          by: name
+          value: id
+          match: ci_equals
+      - request:
+          name: fetch
+          method: GET
+          url: /channels/{{steps.channel.id}}/messages
+          params:
+            limit: "{{args.limit}}"
+      - parse:
+          name: parsed
+          from: fetch
+          format: json
+          extract: "$[*]"
+          fields:
+            - name: author
+              from:
+                coalesce:
+                  - "$.author.global_name"
+                  - "$.author.username"
+            - name: content
+              from: "$.content"
+              ops:
+                - append_urls:
+                    path: "$.attachments[*].url"
+                    sep: " "
+            - name: timestamp
+              from: "$.timestamp"
+              transform: timestamp
+            - name: reactions
+              from: "$.reactions[*].emoji.name"
+              ops:
+                - join:
+                    sep: " "
+            - name: type
+              from: "$.type"
+            - name: id
+              from: "$.id"
+          post_ops:
+            - reverse
+    output:
+      from_step: parsed
+      default_fields: [author, content, timestamp, reactions]
+      default_format: table
+
+  send:
+    description: "Send a message to a channel"
+    args:
+      server:
+        type: string
+        required: true
+        description: "Server name"
+      channel:
+        type: string
+        required: true
+        description: "Channel name"
+      message:
+        type: string
+        required: true
+        source: [arg, stdin]
+        description: "Message content"
+    pipeline:
+      - resolve:
+          name: guild
+          resource: guilds
+          input: "{{args.server}}"
+          by: name
+          value: id
+          match: ci_equals
+      - resolve:
+          name: channel
+          resource: guild_channels
+          input: "{{args.channel}}"
+          by: name
+          value: id
+          match: ci_equals
+      - request:
+          name: send
+          method: POST
+          url: /channels/{{steps.channel.id}}/messages
+          body:
+            encoding: json
+            template:
+              content: "{{args.message}}"
+      - parse:
+          name: parsed
+          from: send
+          format: json
+          extract: "$"
+          fields:
+            - name: id
+              from: "$.id"
+            - name: content
+              from: "$.content"
+            - name: timestamp
+              from: "$.timestamp"
+              transform: timestamp
+            - name: channel_id
+              from: "$.channel_id"
+    output:
+      from_step: parsed
+      default_fields: [id, content, timestamp]
+      default_format: table
+
+  dm:
+    description: "List DM conversations"
+    pipeline:
+      - request:
+          name: fetch
+          method: GET
+          url: /users/@me/channels
+      - parse:
+          name: parsed
+          from: fetch
+          format: json
+          extract: "$[*]"
+          fields:
+            - name: id
+              from: "$.id"
+            - name: recipients
+              # from: "$.recipients[*].username"
+              from:
+                coalesce:
+                  - "$.recipients[*].global_name"
+                  - "$.recipients[*].username"
+              ops:
+                - join:
+                    sep: ", "
+            - name: type
+              from: "$.type"
+            - name: last_message_id
+              from: "$.last_message_id"
+    output:
+      from_step: parsed
+      default_fields: [recipients, type, last_message_id]
+      default_format: table
+
+  dm-messages:
+    description: "Get messages from a DM conversation"
+    args:
+      user:
+        type: string
+        required: true
+        description: "User display name"
+      limit:
+        type: int
+        required: false
+        default: 20
+        max: 100
+        description: "Number of messages to fetch"
+    pipeline:
+      - resolve:
+          name: dm_channel
+          resource: dm_channels
+          input: "{{args.user}}"
+          by: name
+          value: id
+          match: ci_equals
+      - request:
+          name: fetch
+          method: GET
+          url: /channels/{{steps.dm_channel.id}}/messages
+          params:
+            limit: "{{args.limit}}"
+      - parse:
+          name: parsed
+          from: fetch
+          format: json
+          extract: "$[*]"
+          fields:
+            - name: author
+              from:
+                coalesce:
+                  - "$.author.global_name"
+                  - "$.author.username"
+            - name: content
+              from: "$.content"
+              ops:
+                - append_urls:
+                    path: "$.attachments[*].url"
+                    sep: " "
+            - name: timestamp
+              from: "$.timestamp"
+              transform: timestamp
+            - name: reactions
+              from: "$.reactions[*].emoji.name"
+              ops:
+                - join:
+                    sep: " "
+          post_ops:
+            - reverse
+    output:
+      from_step: parsed
+      default_fields: [author, content, timestamp, reactions]
+      default_format: table
+
+  dm-send:
+    description: "Send a DM to a user"
+    args:
+      user:
+        type: string
+        required: true
+        description: "User display name"
+      message:
+        type: string
+        required: true
+        source: [arg, stdin]
+        description: "Message content"
+    pipeline:
+      - resolve:
+          name: dm_channel
+          resource: dm_channels
+          input: "{{args.user}}"
+          by: name
+          value: id
+          match: ci_equals
+      - request:
+          name: send
+          method: POST
+          url: /channels/{{steps.dm_channel.id}}/messages
+          body:
+            encoding: json
+            template:
+              content: "{{args.message}}"
+      - parse:
+          name: parsed
+          from: send
+          format: json
+          extract: "$"
+          fields:
+            - name: id
+              from: "$.id"
+            - name: content
+              from: "$.content"
+            - name: timestamp
+              from: "$.timestamp"
+              transform: timestamp
+            - name: channel_id
+              from: "$.channel_id"
+    output:
+      from_step: parsed
+      default_fields: [id, content, timestamp]
+      default_format: table

web2cli/adapters/mail.google.com/parsers/inbox.py

@@ -0,0 +1,200 @@
+"""Custom parser for Gmail inbox HTML response.
+
+Gmail embeds inbox thread data as a double-escaped JSON string inside the
+HTML/JS response under the key "sils". This parser extracts, unescapes,
+and parses that structure into a flat list of thread records.
+
+Thread data structure (after unescaping):
+  data[0][0] = list of thread entries
+  Each thread entry:
+    [0] null
+    [1] "thread-f:<id>"
+    [2] sort key (descending timestamp complement)
+    [3] subject
+    [4] inner data array:
+      [0] subject (duplicate)
+      [1] snippet
+      [2] timestamp (ms)
+      [3] thread ref
+      [4] messages array, each message:
+        [0] "msg-f:<id>"
+        [1] [type, email, display_name]
+        [6] timestamp (ms)
+        [9] snippet
+        [10] labels (e.g. ["^all", "^i", "^u"])
+"""
+
+import json
+from datetime import datetime, timezone
+
+
+# Well-known Gmail label mappings
+_LABEL_MAP = {
+    "i": "inbox",
+    "u": "unread",
+    "all": "all",
+    "st": "starred",
+    "t": "trash",
+    "s": "spam",
+    "sm": "sent",
+    "f": "draft",
+    "imp": "important",
+    "nt": "notes",
+    "cff": "scheduled",
+    "unsub": "unsubscribe",
+    "oc_unsub": "one-click-unsub",
+    "p_mtunsub": "mute-unsub",
+    "fnas": "auto-classified",
+    "ndpp": "not-displayed-in-promo",
+    "sq_ig_i_personal": "personal",
+}
+
+
+def _find_sils_string(body: str) -> str | None:
+    """Extract the double-escaped JSON string from the sils key."""
+    marker = '"sils",null,"'
+    idx = body.find(marker)
+    if idx < 0:
+        return None
+
+    start = idx + len('"sils",null,')  # keep opening quote for json.loads
+    pos = start + 1
+    escape_count = 0
+
+    while pos < len(body):
+        c = body[pos]
+        if c == "\\":
+            escape_count += 1
+        elif c == '"':
+            if escape_count % 2 == 0:
+                break
+            escape_count = 0
+        else:
+            escape_count = 0
+        pos += 1
+
+    if pos >= len(body):
+        return None
+
+    raw = body[start : pos + 1]
+    return json.loads(raw)
+
+
+def _safe_get(arr, idx, default=None):
+    """Safely index into a list."""
+    if isinstance(arr, list) and len(arr) > idx:
+        return arr[idx]
+    return default
+
+
+def _format_ts(timestamp_ms):
+    """Convert ms timestamp to YYYY-MM-DD HH:MM string."""
+    if not timestamp_ms or not isinstance(timestamp_ms, (int, float)):
+        return ""
+    try:
+        dt = datetime.fromtimestamp(timestamp_ms / 1000, tz=timezone.utc)
+        return dt.strftime("%Y-%m-%d %H:%M")
+    except (OSError, ValueError):
+        return ""
+
+
+def _clean_labels(raw_labels: list[str] | None) -> list[str]:
+    """Strip ^ prefix and map known labels to human-readable names."""
+    if not raw_labels:
+        return []
+    out = []
+    for label in raw_labels:
+        key = label.lstrip("^")
+        mapped = _LABEL_MAP.get(key, key)
+        out.append(mapped)
+    return out
+
+
+def _extract_threads(data: list) -> list[dict]:
+    """Walk the parsed JSON structure and extract thread records."""
+    threads_list = _safe_get(_safe_get(data, 0), 0)
+    if not isinstance(threads_list, list):
+        return []
+
+    records = []
+    for entry in threads_list:
+        if not isinstance(entry, list) or len(entry) < 5:
+            continue
+
+        raw_thread_id = _safe_get(entry, 1, "")
+        if not isinstance(raw_thread_id, str) or not raw_thread_id.startswith("thread-f:"):
+            continue
+
+        thread_id = raw_thread_id.replace("thread-f:", "")
+        subject = _safe_get(entry, 3, "")
+        inner = _safe_get(entry, 4)
+
+        if not isinstance(inner, list):
+            continue
+
+        snippet = _safe_get(inner, 1, "")
+        timestamp_ms = _safe_get(inner, 2, 0)
+        messages = _safe_get(inner, 4, [])
+
+        # Extract sender and labels from the first (most recent) message
+        sender_email = ""
+        sender_name = ""
+        labels_raw = []
+        message_id = ""
+
+        if isinstance(messages, list) and messages:
+            first_msg = messages[0]
+            if isinstance(first_msg, list):
+                raw_msg_id = _safe_get(first_msg, 0, "")
+                if isinstance(raw_msg_id, str):
+                    message_id = raw_msg_id.replace("msg-f:", "")
+
+                sender_info = _safe_get(first_msg, 1)
+                if isinstance(sender_info, list):
+                    sender_email = _safe_get(sender_info, 1, "")
+                    sender_name = _safe_get(sender_info, 2, "")
+
+                labels_raw = _safe_get(first_msg, 10, [])
+
+        if not isinstance(labels_raw, list):
+            labels_raw = []
+
+        unread = "^u" in labels_raw
+        labels = _clean_labels(labels_raw)
+        date = _format_ts(timestamp_ms)
+        message_count = len(messages) if isinstance(messages, list) else 0
+
+        records.append({
+            "thread_id": thread_id,
+            "message_id": message_id,
+            "subject": subject or "",
+            "snippet": snippet or "",
+            "sender_email": sender_email or "",
+            "sender_name": sender_name or "",
+            "date": date,
+            "timestamp": timestamp_ms,
+            "unread": unread,
+            "labels": ", ".join(labels),
+            "message_count": message_count,
+        })
+
+    # Sort by timestamp descending (newest first)
+    records.sort(key=lambda r: r.get("timestamp", 0), reverse=True)
+    return records
+
+
+def parse(status_code: int, headers: dict, body: str, args: dict) -> list[dict]:
+    """Entry point called by the web2cli custom parser loader."""
+    if status_code >= 400:
+        return []
+
+    unescaped = _find_sils_string(body)
+    if not unescaped:
+        return []
+
+    try:
+        data = json.loads(unescaped)
+    except json.JSONDecodeError:
+        return []
+
+    return _extract_threads(data)