toapi 2.1.3__tar.gz → 2.2.0__tar.gz

toapi-2.2.0/.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: CI
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+      - name: Set up Python
+        run: uv python install ${{ matrix.python }}
+      - name: Install dependencies
+        run: uv sync --all-extras --dev
+      - name: Lint
+        run: |
+          uv run ruff check .
+          uv run ruff format --check .
+      - name: Test
+        run: uv run pytest --cov=toapi

toapi-2.2.0/.gitignore

@@ -0,0 +1,105 @@
+.idea/
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+.html/
+# C extensions
+*.so
+.pytest_cache/
+# Distribution / packaging
+.Python
+env/
+env27/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+/tests/data.sqlite
+/examples/toapi-pic/data.sqlite

toapi-2.2.0/.omc/project-memory.json

@@ -0,0 +1,174 @@
+{
+  "version": "1.0.0",
+  "lastScanned": 1779434703584,
+  "projectRoot": "/Users/elliot/Documents/Code/toapi",
+  "techStack": {
+    "languages": [
+      {
+        "name": "Python",
+        "version": "^3.8",
+        "confidence": "high",
+        "markers": [
+          "pyproject.toml"
+        ]
+      }
+    ],
+    "frameworks": [],
+    "packageManager": "poetry",
+    "runtime": null
+  },
+  "build": {
+    "buildCommand": null,
+    "testCommand": "pytest",
+    "lintCommand": "ruff check",
+    "devCommand": null,
+    "scripts": {}
+  },
+  "conventions": {
+    "namingStyle": null,
+    "importStyle": null,
+    "testPattern": null,
+    "fileOrganization": null
+  },
+  "structure": {
+    "isMonorepo": false,
+    "workspaces": [],
+    "mainDirectories": [
+      "docs",
+      "examples",
+      "tests"
+    ],
+    "gitBranches": {
+      "defaultBranch": "master",
+      "branchingStrategy": null
+    }
+  },
+  "customNotes": [],
+  "directoryMap": {
+    "docs": {
+      "path": "docs",
+      "purpose": "Documentation",
+      "fileCount": 5,
+      "lastAccessed": 1779434703569,
+      "keyFiles": [
+        "CNAME",
+        "diagram.png",
+        "index.md",
+        "logo.png",
+        "quickstart.md"
+      ]
+    },
+    "examples": {
+      "path": "examples",
+      "purpose": "Example code",
+      "fileCount": 1,
+      "lastAccessed": 1779434703569,
+      "keyFiles": [
+        "hackernews_page.py"
+      ]
+    },
+    "tests": {
+      "path": "tests",
+      "purpose": "Test files",
+      "fileCount": 1,
+      "lastAccessed": 1779434703569,
+      "keyFiles": [
+        "test_toapi.py"
+      ]
+    },
+    "toapi": {
+      "path": "toapi",
+      "purpose": null,
+      "fileCount": 5,
+      "lastAccessed": 1779434703570,
+      "keyFiles": [
+        "__init__.py",
+        "api.py",
+        "cli.py",
+        "item.py",
+        "log.py"
+      ]
+    }
+  },
+  "hotPaths": [
+    {
+      "path": "pyproject.toml",
+      "accessCount": 7,
+      "lastAccessed": 1779435734082,
+      "type": "file"
+    },
+    {
+      "path": "toapi/api.py",
+      "accessCount": 4,
+      "lastAccessed": 1779435005253,
+      "type": "file"
+    },
+    {
+      "path": "toapi/__init__.py",
+      "accessCount": 4,
+      "lastAccessed": 1779435605203,
+      "type": "file"
+    },
+    {
+      "path": ".pre-commit-config.yaml",
+      "accessCount": 3,
+      "lastAccessed": 1779434999306,
+      "type": "file"
+    },
+    {
+      "path": "toapi/item.py",
+      "accessCount": 3,
+      "lastAccessed": 1779435568476,
+      "type": "file"
+    },
+    {
+      "path": ".travis.yml",
+      "accessCount": 2,
+      "lastAccessed": 1779434902331,
+      "type": "file"
+    },
+    {
+      "path": ".flake8",
+      "accessCount": 2,
+      "lastAccessed": 1779434903048,
+      "type": "file"
+    },
+    {
+      "path": "toapi/cli.py",
+      "accessCount": 2,
+      "lastAccessed": 1779435596581,
+      "type": "file"
+    },
+    {
+      "path": "README.md",
+      "accessCount": 1,
+      "lastAccessed": 1779434789416,
+      "type": "file"
+    },
+    {
+      "path": "toapi/log.py",
+      "accessCount": 1,
+      "lastAccessed": 1779434808975,
+      "type": "file"
+    },
+    {
+      "path": "poetry.lock",
+      "accessCount": 1,
+      "lastAccessed": 1779434809861,
+      "type": "file"
+    },
+    {
+      "path": "tests/test_toapi.py",
+      "accessCount": 1,
+      "lastAccessed": 1779434815385,
+      "type": "file"
+    },
+    {
+      "path": "LICENSE",
+      "accessCount": 1,
+      "lastAccessed": 1779434833764,
+      "type": "file"
+    }
+  ],
+  "userDirectives": []
+}

toapi-2.2.0/.omc/state/agent-replay-86181004-c476-471f-90d2-1c64e40fb749.jsonl

@@ -0,0 +1,4 @@
+{"t":0,"agent":"aae2d16","agent_type":"Explore","event":"agent_start","parent_mode":"none"}
+{"t":0,"agent":"a324b3e","agent_type":"Explore","event":"agent_start","parent_mode":"none"}
+{"t":0,"agent":"a324b3e","agent_type":"Explore","event":"agent_stop","success":true,"duration_ms":87042}
+{"t":0,"agent":"aae2d16","agent_type":"Explore","event":"agent_stop","success":true,"duration_ms":105179}

toapi-2.2.0/.omc/state/hud-stdin-cache.json

@@ -0,0 +1 @@
+{"session_id":"86181004-c476-471f-90d2-1c64e40fb749","transcript_path":"/Users/elliot/.claude/projects/-Users-elliot-Documents-Code-toapi/86181004-c476-471f-90d2-1c64e40fb749.jsonl","cwd":"/Users/elliot/Documents/Code/toapi","effort":{"level":"xhigh"},"session_name":"modernize-toapi-dependencies","model":{"id":"claude-opus-4-7[1m]","display_name":"Opus 4.7 (1M context)"},"workspace":{"current_dir":"/Users/elliot/Documents/Code/toapi","project_dir":"/Users/elliot/Documents/Code/toapi","added_dirs":[]},"version":"2.1.144","output_style":{"name":"default"},"cost":{"total_cost_usd":3.1557453,"total_duration_ms":1033890,"total_api_duration_ms":569612,"total_lines_added":260,"total_lines_removed":99},"context_window":{"total_input_tokens":82733,"total_output_tokens":961,"context_window_size":1000000,"current_usage":{"input_tokens":6,"output_tokens":961,"cache_creation_input_tokens":1867,"cache_read_input_tokens":80860},"used_percentage":8,"remaining_percentage":92},"exceeds_200k_tokens":false,"fast_mode":false,"thinking":{"enabled":true},"rate_limits":{"five_hour":{"used_percentage":34,"resets_at":1779440400},"seven_day":{"used_percentage":80,"resets_at":1779570000}}}

toapi-2.2.0/.omc/state/idle-notif-cooldown.json

@@ -0,0 +1,5 @@
+{
+  "lastSentAt": "2026-05-22T07:40:42.602Z",
+  "repoSignature": "{\"repo\":\"elliotgao2/toapi\",\"headSha\":\"6ae043cab28d16beb0be1bd9b1cd0fdc9c19baa6\",\"dirty\":true,\"openPrNumbers\":[127,135,138],\"openIssueNumbers\":[129,130,131,132,134,139],\"failingRunIds\":[24379413758,25783406210]}",
+  "backlogZero": false
+}

toapi-2.2.0/.omc/state/last-tool-error.json

@@ -0,0 +1,7 @@
+{
+  "tool_name": "Bash",
+  "tool_input_preview": "{\"command\":\"ls /opt/homebrew/bin/python3* /usr/local/bin/python3* /opt/nanobrew/prefix/bin/python3* 2>/dev/null; echo \\\"---\\\"; ls /opt/homebrew/opt 2>/dev/null | grep -i python; echo \\\"---\\\"; which py...",
+  "error": "Exit code 1\n/opt/homebrew/bin/python3\n/opt/homebrew/bin/python3-config\n/opt/homebrew/bin/python3.13\n/opt/homebrew/bin/python3.13-config\n/opt/homebrew/bin/python3.14\n/opt/homebrew/bin/python3.14-config\n/opt/nanobrew/prefix/bin/python3\n/opt/nanobrew/prefix/bin/python3-config\n/opt/nanobrew/prefix/bin/python3.13\n/opt/nanobrew/prefix/bin/python3.13-config\n/opt/nanobrew/prefix/bin/python3.14\n/opt/nanobrew/prefix/bin/python3.14-config\n/usr/local/bin/python3\n/usr/local/bin/python3-config\n/usr/local/bin/...",
+  "timestamp": "2026-05-22T07:30:24.107Z",
+  "retry_count": 2
+}

toapi-2.2.0/.omc/state/mission-state.json

@@ -0,0 +1,79 @@
+{
+  "updatedAt": "2026-05-22T07:27:37.778Z",
+  "missions": [
+    {
+      "id": "session:86181004-c476-471f-90d2-1c64e40fb749:none",
+      "source": "session",
+      "name": "none",
+      "objective": "Session mission",
+      "createdAt": "2026-05-22T07:25:52.599Z",
+      "updatedAt": "2026-05-22T07:27:37.778Z",
+      "status": "done",
+      "workerCount": 2,
+      "taskCounts": {
+        "total": 2,
+        "pending": 0,
+        "blocked": 0,
+        "inProgress": 0,
+        "completed": 2,
+        "failed": 0
+      },
+      "agents": [
+        {
+          "name": "Explore:aae2d16",
+          "role": "Explore",
+          "ownership": "aae2d161a0ef7e8e0",
+          "status": "done",
+          "currentStep": null,
+          "latestUpdate": "completed",
+          "completedSummary": null,
+          "updatedAt": "2026-05-22T07:27:37.778Z"
+        },
+        {
+          "name": "Explore:a324b3e",
+          "role": "Explore",
+          "ownership": "a324b3eaed6783abe",
+          "status": "done",
+          "currentStep": null,
+          "latestUpdate": "completed",
+          "completedSummary": null,
+          "updatedAt": "2026-05-22T07:27:37.526Z"
+        }
+      ],
+      "timeline": [
+        {
+          "id": "session-start:aae2d161a0ef7e8e0:2026-05-22T07:25:52.599Z",
+          "at": "2026-05-22T07:25:52.599Z",
+          "kind": "update",
+          "agent": "Explore:aae2d16",
+          "detail": "started Explore:aae2d16",
+          "sourceKey": "session-start:aae2d161a0ef7e8e0"
+        },
+        {
+          "id": "session-start:a324b3eaed6783abe:2026-05-22T07:26:10.484Z",
+          "at": "2026-05-22T07:26:10.484Z",
+          "kind": "update",
+          "agent": "Explore:a324b3e",
+          "detail": "started Explore:a324b3e",
+          "sourceKey": "session-start:a324b3eaed6783abe"
+        },
+        {
+          "id": "session-stop:a324b3eaed6783abe:2026-05-22T07:27:37.526Z",
+          "at": "2026-05-22T07:27:37.526Z",
+          "kind": "completion",
+          "agent": "Explore:a324b3e",
+          "detail": "completed",
+          "sourceKey": "session-stop:a324b3eaed6783abe"
+        },
+        {
+          "id": "session-stop:aae2d161a0ef7e8e0:2026-05-22T07:27:37.778Z",
+          "at": "2026-05-22T07:27:37.778Z",
+          "kind": "completion",
+          "agent": "Explore:aae2d16",
+          "detail": "completed",
+          "sourceKey": "session-stop:aae2d161a0ef7e8e0"
+        }
+      ]
+    }
+  ]
+}

toapi-2.2.0/.omc/state/sessions/86181004-c476-471f-90d2-1c64e40fb749/hud-state.json

@@ -0,0 +1,6 @@
+{
+  "timestamp": "2026-05-22T07:25:44.780Z",
+  "backgroundTasks": [],
+  "sessionStartTimestamp": "2026-05-22T07:25:03.600Z",
+  "sessionId": "86181004-c476-471f-90d2-1c64e40fb749"
+}

toapi-2.2.0/.omc/state/subagent-tracking.json

@@ -0,0 +1,26 @@
+{
+  "agents": [
+    {
+      "agent_id": "aae2d161a0ef7e8e0",
+      "agent_type": "Explore",
+      "started_at": "2026-05-22T07:25:52.599Z",
+      "parent_mode": "none",
+      "status": "completed",
+      "completed_at": "2026-05-22T07:27:37.778Z",
+      "duration_ms": 105179
+    },
+    {
+      "agent_id": "a324b3eaed6783abe",
+      "agent_type": "Explore",
+      "started_at": "2026-05-22T07:26:10.484Z",
+      "parent_mode": "none",
+      "status": "completed",
+      "completed_at": "2026-05-22T07:27:37.526Z",
+      "duration_ms": 87042
+    }
+  ],
+  "total_spawned": 2,
+  "total_completed": 2,
+  "total_failed": 0,
+  "last_updated": "2026-05-22T07:27:37.880Z"
+}

toapi-2.2.0/.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.9
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format

toapi-2.2.0/PKG-INFO

@@ -0,0 +1,117 @@
+Metadata-Version: 2.4
+Name: toapi
+Version: 2.2.0
+Summary: Every web site provides APIs.
+Project-URL: homepage, https://github.com/gaojiuli/toapi
+Project-URL: repository, https://github.com/gaojiuli/toapi
+Project-URL: documentation, https://gaojiuli.github.io/toapi/
+Author-email: Elliot Gao <gaojiuli@gmail.com>
+License: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: charset-normalizer>=3.3
+Requires-Dist: click>=8.1
+Requires-Dist: colorama>=0.4.6
+Requires-Dist: cssselect>=1.2
+Requires-Dist: flask>=3.0
+Requires-Dist: htmlfetcher>=0.0.6
+Requires-Dist: htmlparsing>=0.1.5
+Requires-Dist: requests>=2.32
+Description-Content-Type: text/markdown
+
+# Toapi
+
+[![Build](https://travis-ci.org/gaojiuli/toapi.svg?branch=master)](https://travis-ci.org/gaojiuli/toapi)
+[![Coverage](https://codecov.io/gh/gaojiuli/toapi/branch/master/graph/badge.svg)](https://codecov.io/gh/gaojiuli/toapi)
+[![Python](https://img.shields.io/pypi/pyversions/toapi.svg)](https://pypi.python.org/pypi/toapi/)
+[![Version](https://img.shields.io/pypi/v/toapi.svg)](https://pypi.python.org/pypi/toapi/)
+[![License](https://img.shields.io/pypi/l/toapi.svg)](https://pypi.python.org/pypi/toapi/)
+
+## Overview
+
+Toapi give you the ability to make every web site provides APIs.
+
+- v1.0.0 Documentation: [http://www.toapi.org](http://www.toapi.org)
+- Awesome: [https://github.com/toapi/awesome-toapi](https://github.com/toapi/awesome-toapi)
+- Organization: [https://github.com/toapi](https://github.com/toapi)
+
+## Features
+
+- Automatic converting HTML web site to API service.
+- Automatic caching every page of source site.
+- Automatic caching every request.
+- Support merging multiple web sites into one API service. 
+
+## Get Started
+
+### Installation
+
+```text
+$ pip install toapi
+```
+
+### Usage
+
+create `app.py` and copy the code:
+
+```python
+from flask import request
+from htmlparsing import Attr, Text
+from toapi import Api, Item
+
+api = Api()
+
+
+@api.site('https://news.ycombinator.com')
+@api.list('.athing')
+@api.route('/posts?page={page}', '/news?p={page}')
+@api.route('/posts', '/news?p=1')
+class Post(Item):
+    url = Attr('.storylink', 'href')
+    title = Text('.storylink')
+
+
+@api.site('https://news.ycombinator.com')
+@api.route('/posts?page={page}', '/news?p={page}')
+@api.route('/posts', '/news?p=1')
+class Page(Item):
+    next_page = Attr('.morelink', 'href')
+
+    def clean_next_page(self, value):
+        return api.convert_string('/' + value, '/news?p={page}', request.host_url.strip('/') + '/posts?page={page}')
+
+
+api.run(debug=True, host='0.0.0.0', port=5000)
+```
+
+run `python app.py`
+
+then open your browser and visit `http://127.0.0.1:5000/posts?page=1` 
+
+you will get the result like:
+
+```json
+{
+  "Page": {
+    "next_page": "http://127.0.0.1:5000/posts?page=2"
+  }, 
+  "Post": [
+    {
+      "title": "Mathematicians Crack the Cursed Curve", 
+      "url": "https://www.quantamagazine.org/mathematicians-crack-the-cursed-curve-20171207/"
+    }, 
+    {
+      "title": "Stuffing a Tesla Drivetrain into a 1981 Honda Accord", 
+      "url": "https://jalopnik.com/this-glorious-madman-stuffed-a-p85-tesla-drivetrain-int-1823461909"
+    }
+  ]
+}
+```
+
+
+## Contributing
+
+Write code and test code and pull request.
+
+
+

toapi-2.2.0/README.md

@@ -0,0 +1,96 @@
+# Toapi
+
+[![Build](https://travis-ci.org/gaojiuli/toapi.svg?branch=master)](https://travis-ci.org/gaojiuli/toapi)
+[![Coverage](https://codecov.io/gh/gaojiuli/toapi/branch/master/graph/badge.svg)](https://codecov.io/gh/gaojiuli/toapi)
+[![Python](https://img.shields.io/pypi/pyversions/toapi.svg)](https://pypi.python.org/pypi/toapi/)
+[![Version](https://img.shields.io/pypi/v/toapi.svg)](https://pypi.python.org/pypi/toapi/)
+[![License](https://img.shields.io/pypi/l/toapi.svg)](https://pypi.python.org/pypi/toapi/)
+
+## Overview
+
+Toapi give you the ability to make every web site provides APIs.
+
+- v1.0.0 Documentation: [http://www.toapi.org](http://www.toapi.org)
+- Awesome: [https://github.com/toapi/awesome-toapi](https://github.com/toapi/awesome-toapi)
+- Organization: [https://github.com/toapi](https://github.com/toapi)
+
+## Features
+
+- Automatic converting HTML web site to API service.
+- Automatic caching every page of source site.
+- Automatic caching every request.
+- Support merging multiple web sites into one API service. 
+
+## Get Started
+
+### Installation
+
+```text
+$ pip install toapi
+```
+
+### Usage
+
+create `app.py` and copy the code:
+
+```python
+from flask import request
+from htmlparsing import Attr, Text
+from toapi import Api, Item
+
+api = Api()
+
+
+@api.site('https://news.ycombinator.com')
+@api.list('.athing')
+@api.route('/posts?page={page}', '/news?p={page}')
+@api.route('/posts', '/news?p=1')
+class Post(Item):
+    url = Attr('.storylink', 'href')
+    title = Text('.storylink')
+
+
+@api.site('https://news.ycombinator.com')
+@api.route('/posts?page={page}', '/news?p={page}')
+@api.route('/posts', '/news?p=1')
+class Page(Item):
+    next_page = Attr('.morelink', 'href')
+
+    def clean_next_page(self, value):
+        return api.convert_string('/' + value, '/news?p={page}', request.host_url.strip('/') + '/posts?page={page}')
+
+
+api.run(debug=True, host='0.0.0.0', port=5000)
+```
+
+run `python app.py`
+
+then open your browser and visit `http://127.0.0.1:5000/posts?page=1` 
+
+you will get the result like:
+
+```json
+{
+  "Page": {
+    "next_page": "http://127.0.0.1:5000/posts?page=2"
+  }, 
+  "Post": [
+    {
+      "title": "Mathematicians Crack the Cursed Curve", 
+      "url": "https://www.quantamagazine.org/mathematicians-crack-the-cursed-curve-20171207/"
+    }, 
+    {
+      "title": "Stuffing a Tesla Drivetrain into a 1981 Honda Accord", 
+      "url": "https://jalopnik.com/this-glorious-madman-stuffed-a-p85-tesla-drivetrain-int-1823461909"
+    }
+  ]
+}
+```
+
+
+## Contributing
+
+Write code and test code and pull request.
+
+
+

toapi-2.2.0/docs/CNAME

@@ -0,0 +1 @@
+www.toapi.org