async-httpd-data-collector 2.0.2__tar.gz → 2.0.3__tar.gz

async_httpd_data_collector-2.0.3/.github/workflows/docs.yml

@@ -0,0 +1,40 @@
+---
+name: Docs
+
+on:
+  push:
+    branches: [master]
+
+permissions:
+  pages: write
+  id-token: write
+
+jobs:
+  build-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install mkdocs-material
+        run: pip install mkdocs-material
+
+      - name: Build docs
+        run: mkdocs build
+
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: site/
+
+  deploy:
+    needs: build-docs
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/deploy-pages@v4
+        id: deployment

{async_httpd_data_collector-2.0.2 → async_httpd_data_collector-2.0.3}/.github/workflows/publish.yml

@@ -1,17 +1,13 @@
 ---
-# Builds, relases and publishes to PyPi
+# Builds, releases and publishes to PyPI
 # .github/workflows/publish.yml
 name: CD
 
-# triggered when testing is finished
 on:
-  workflow_run:
-    workflows: ["CI"]
-    branches: [master]
-    types:
-      - completed
+  push:
+    tags:
+      - 'v*'
 
-# add contents: write for the release
 permissions:
   contents: write
 
@@ -23,10 +19,10 @@ jobs:
     steps:
 
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
 
@@ -34,20 +30,18 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install build twine
-          pip install -r requirements.txt
 
-      - name: Build the application
+      - name: Build the package
         run: |
           python -m build
 
       - name: Release
         uses: softprops/action-gh-release@v2
-        if: startsWith(github.ref, 'refs/tags/')
         with:
           files: |
             dist/**
 
-      - name: Publish to PyPi
+      - name: Publish to PyPI
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}

{async_httpd_data_collector-2.0.2 → async_httpd_data_collector-2.0.3}/.github/workflows/test.yml

@@ -28,7 +28,7 @@ jobs:
     steps:
 
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       # install InfluxDB CLI to authenticate access to the database
       - name: Install InfluxDB CLI
@@ -56,8 +56,8 @@ jobs:
             --force
 
           # save the data required for the token generation and testing
-          echo "::set-output name=org::${INFLUX_ORG}"
-          echo "::set-output name=bucket::${INFLUX_BUCKET}"
+          echo "org=${INFLUX_ORG}" >> $GITHUB_OUTPUT
+          echo "bucket=${INFLUX_BUCKET}" >> $GITHUB_OUTPUT
 
       - name: Generate token for further authentication
         id: auth
@@ -72,10 +72,10 @@ jobs:
             --json | jq '.token')
 
           # save the token
-          echo "::set-output name=token::${INFLUX_TOKEN}"
+          echo "token=${INFLUX_TOKEN}" >> $GITHUB_OUTPUT
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: '3.11'
 
@@ -92,7 +92,7 @@ jobs:
       - name: Test
         run: |-
 
-          # run the falsk dev_server
+          # run the flask dev_server
           export FLASK_APP="tests/dev_server"
           flask run --host=0.0.0.0 --port=9000 &
           sleep 5

async_httpd_data_collector-2.0.3/.gitignore

@@ -0,0 +1,45 @@
+# project specific
+secrets
+*env
+test-data
+
+# byte-compiled / optimized
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+
+# distribution / packaging
+build/
+dist/
+*.egg-info/
+*.egg
+MANIFEST
+
+# unit test / coverage
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+.pytest_cache/
+coverage.xml
+
+# environments
+.env
+.venv
+venv/
+
+# mkdocs
+/site
+docs/analysis/img/
+
+# mypy
+.mypy_cache/
+
+# ruff
+.ruff_cache/
+
+# logs
+*.log

{async_httpd_data_collector-2.0.2 → async_httpd_data_collector-2.0.3}/PKG-INFO

@@ -1,10 +1,12 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: async-httpd-data-collector
-Version: 2.0.2
-Summary: Gateway facilitating asyncronous communication between sensory data-emitting devices, InfluxDB and the user.
+Version: 2.0.3
+Summary: Gateway facilitating asynchronous communication between sensory data-emitting devices, InfluxDB and the user.
+Project-URL: Documentation, https://ahttpdc-docs.codextechnologies.org/mkdocs
 Project-URL: Repository, https://github.com/straightchlorine/async-httpd-data-collector
 Project-URL: Issues, https://github.com/straightchlorine/async-httpd-data-collector/issues
 Author-email: Piotr Krzysztof Lis <piotrlis555@gmail.com>
+Maintainer-email: Piotr Krzysztof Lis <piotrlis555@gmail.com>
 License:                     GNU GENERAL PUBLIC LICENSE
                                Version 3, 29 June 2007
         
@@ -689,322 +691,135 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Requires-Dist: aiocsv
 Requires-Dist: aiohttp
-Requires-Dist: aiosignal
-Requires-Dist: attrs
-Requires-Dist: certifi
-Requires-Dist: frozenlist
-Requires-Dist: idna
 Requires-Dist: influxdb-client[async]
-Requires-Dist: multidict
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: python-dateutil
 Requires-Dist: pytz
 Requires-Dist: reactivex
-Requires-Dist: setuptools
-Requires-Dist: six
-Requires-Dist: typing-extensions
-Requires-Dist: tzdata
-Requires-Dist: urllib3
-Requires-Dist: yarl
+Provides-Extra: docs
+Requires-Dist: mkdocs-material; extra == 'docs'
 Provides-Extra: test
-Requires-Dist: blinker; extra == 'test'
-Requires-Dist: charset-normalizer; extra == 'test'
-Requires-Dist: click; extra == 'test'
 Requires-Dist: dash; extra == 'test'
-Requires-Dist: dash-core-components; extra == 'test'
-Requires-Dist: dash-html-components; extra == 'test'
-Requires-Dist: dash-table; extra == 'test'
 Requires-Dist: flask; extra == 'test'
-Requires-Dist: importlib-metadata; extra == 'test'
-Requires-Dist: iniconfig; extra == 'test'
-Requires-Dist: itsdangerous; extra == 'test'
-Requires-Dist: jinja2; extra == 'test'
-Requires-Dist: markupsafe; extra == 'test'
-Requires-Dist: nest-asyncio; extra == 'test'
-Requires-Dist: packaging; extra == 'test'
 Requires-Dist: plotly; extra == 'test'
-Requires-Dist: pluggy; extra == 'test'
 Requires-Dist: pytest; extra == 'test'
 Requires-Dist: pytest-asyncio; extra == 'test'
 Requires-Dist: requests; extra == 'test'
-Requires-Dist: retrying; extra == 'test'
-Requires-Dist: tenacity; extra == 'test'
-Requires-Dist: werkzeug; extra == 'test'
-Requires-Dist: zipp; extra == 'test'
 Description-Content-Type: text/markdown
 
-# async-httpd-data-collector
-
-Interface handling the communication between sensory data-emitting devices, InfluxDB and the user.
-
-The most important object that a user would use is `DatabseInterface` within `ahttpdc.reads.interface` module.
-This class facilitates the communication between the fetcher and the querying apis of InfluxDB.
+<div align="center">
 
-In order to control fetching, there are two methods:
+[![PyPI version](https://badge.fury.io/py/async-httpd-data-collector.svg)](https://pypi.org/project/async-httpd-data-collector/)
+[![Total Downloads](https://static.pepy.tech/badge/async-httpd-data-collector)](https://pepy.tech/project/async-httpd-data-collector)
+[![PyPI - Downloads](https://img.shields.io/pypi/dm/async-httpd-data-collector)](https://pypi.org/project/async-httpd-data-collector/)
+</div>
 
-* `interface.daemon.enableg()`;
-* `interface.daemon.disable()`.
+# async-httpd-data-collector
 
-Those methods control the thread within which fetching process is contained.
+> **Note:** This is an older university project and is not actively maintained.
+> It works as-is, but is not extensively tested.
 
-You can query data from the database using methods with `query_` prefix. For now there are three:
+A Python library that acts as an asynchronous gateway between IoT sensor
+devices and InfluxDB. It fetches JSON readings from a device (like a
+NodeMCU/ESP8266) over HTTP, parses them, and stores them as time-series data.
+You can then query the data back as pandas DataFrames.
 
-* `interface.query_latest()`, which queries the lastest measurement;
-* `interface.query_historical()`, which queries data from a given time range or relative time (eg. -3h);
-* `interface.query()`, which can takes user's given query as an argument.
+Started as a university project to get hands-on with async Python,
+`asyncio`, `aiohttp`, and time-series databases. The hardware side runs on
+an [arduino-air-state-server](https://github.com/straightchlorine/arduino-air-state-server) -
+a NodeMCU board with air quality sensors (MQ135, BMP180, DHT22, DS18B20)
+that exposes readings at an HTTP endpoint.
 
-Some examples will be presented below:
+## Installation
 
-# 1.1 Connecting to the database
+```bash
+pip install async-httpd-data-collector
+```
 
+## Quick Start
 
 ```python
-import json
-from ahttpdc.reads.interface import DatabaseInterface
-
-# load the secrets
-with open('../../../secrets/secrets.json', 'r') as f:
-    secrets = json.load(f)
+from ahttpdc.read.database_interface import DatabaseInterface
 
-# define sensors
+# define which sensors and parameters to track
 sensors = {
-    'bmp180': ['altitude', 'pressure', 'seaLevelPressure'],
+    'bmp180': ['altitude', 'pressure', 'seaLevelPressure', 'temperature'],
     'mq135': ['aceton', 'alcohol', 'co', 'co2', 'nh4', 'toulen'],
     'ds18b20': ['temperature'],
-    'dht22': ['humidity'],
+    'dht22': ['humidity', 'temperature'],
 }
 
-# define the interface to the database
+# connect to InfluxDB and the device
 interface = DatabaseInterface(
-    secrets['host'],
-    secrets['port'],
-    secrets['token'],
-    secrets['organization'],
-    secrets['bucket'],
     sensors,
-    secrets['dev_ip'],
-    80,
-    secrets['handle'],
+    db_host='localhost',
+    db_port=8086,
+    db_token='your-influxdb-token',
+    db_org='your-org',
+    db_bucket='your-bucket',
+    srv_ip='192.168.1.100',  # device IP
+    srv_port=80,
+    handle='circumstances',  # HTTP endpoint on the device
 )
-```
 
-### 1.2 Extracting the dataframe from the database
+# start the background daemon - fetches and stores data continuously
+interface.daemon.enable()
 
+# query the last 30 days of data as a DataFrame
+df = interface.query_historical('-30d')
+print(df.head())
 
-```python
-import pandas as pd
-import asyncio
-from datetime import datetime, timedelta
-from pathlib import Path
-
-# if there is readings.csv file, load it
-# if not - create it
-readings_path = Path('../data/readings.csv')
-if readings_path.is_file():
-    sensor = pd.read_csv(readings_path)
-else:
-    sensor = await interface.query_historical('-30d')
-    sensor.to_csv(readings_path)
+# stop the daemon when done
+interface.daemon.disable()
 ```
 
+## How It Works
 
-```python
-sensor
+```
+NodeMCU device          async-httpd-data-collector            InfluxDB
+  (sensors)                                                  (storage)
+     |                                                          |
+     |--- HTTP GET /circumstances -->  AsyncFetcher             |
+     |                                     |                    |
+     |                              JSONInfluxParser            |
+     |                                     |                    |
+     |                              AsyncCollector  -------->   |
+     |                                                          |
+     |                              AsyncQuery     <--------    |
+     |                                     |                    |
+     |                               DataParser                 |
+     |                                     |                    |
+     |                              pandas DataFrame            |
 ```
 
+The `DatabaseInterface` is the main entry point. It manages two things:
 
+- **DataDaemon** - a background process (via `multiprocessing`) that
+  periodically fetches sensor data from the device and stores it in InfluxDB.
+- **AsyncQuery** - queries InfluxDB and returns results as pandas DataFrames
+  with local timezone-adjusted timestamps.
 
+### Querying
 
-<div>
-<table border="1" class="dataframe">
-  <thead>
-    <tr style="text-align: right;">
-      <th></th>
-      <th>time</th>
-      <th>aceton</th>
-      <th>alcohol</th>
-      <th>altitude</th>
-      <th>co</th>
-      <th>co2</th>
-      <th>humidity</th>
-      <th>nh4</th>
-      <th>pressure</th>
-      <th>seaLevelPressure</th>
-      <th>temperature</th>
-      <th>toulen</th>
-    </tr>
-  </thead>
-  <tbody>
-    <tr>
-      <th>0</th>
-      <td>2024-05-16 17:43:59.196399+00:00</td>
-      <td>0.41</td>
-      <td>1.17</td>
-      <td>149.92</td>
-      <td>3.38</td>
-      <td>402.54</td>
-      <td>37.4</td>
-      <td>3.93</td>
-      <td>999.35</td>
-      <td>1017.31</td>
-      <td>24.40</td>
-      <td>0.48</td>
-    </tr>
-    <tr>
-      <th>1</th>
-      <td>2024-05-16 17:44:01.768738+00:00</td>
-      <td>0.47</td>
-      <td>1.32</td>
-      <td>149.76</td>
-      <td>3.94</td>
-      <td>402.84</td>
-      <td>30.5</td>
-      <td>4.33</td>
-      <td>997.61</td>
-      <td>1015.56</td>
-      <td>24.03</td>
-      <td>0.55</td>
-    </tr>
-    <tr>
-      <th>2</th>
-      <td>2024-05-16 17:44:03.255309+00:00</td>
-      <td>0.96</td>
-      <td>2.62</td>
-      <td>149.54</td>
-      <td>9.16</td>
-      <td>405.25</td>
-      <td>49.1</td>
-      <td>7.35</td>
-      <td>999.14</td>
-      <td>1017.08</td>
-      <td>23.16</td>
-      <td>1.15</td>
-    </tr>
-    <tr>
-      <th>3</th>
-      <td>2024-05-16 17:44:04.618203+00:00</td>
-      <td>0.30</td>
-      <td>0.86</td>
-      <td>149.38</td>
-      <td>2.32</td>
-      <td>401.94</td>
-      <td>32.9</td>
-      <td>3.10</td>
-      <td>999.09</td>
-      <td>1017.02</td>
-      <td>23.05</td>
-      <td>0.35</td>
-    </tr>
-    <tr>
-      <th>4</th>
-      <td>2024-05-16 17:44:05.954714+00:00</td>
-      <td>1.31</td>
-      <td>3.50</td>
-      <td>149.37</td>
-      <td>13.13</td>
-      <td>406.82</td>
-      <td>48.8</td>
-      <td>9.21</td>
-      <td>998.04</td>
-      <td>1015.93</td>
-      <td>23.92</td>
-      <td>1.57</td>
-    </tr>
-    <tr>
-      <th>...</th>
-      <td>...</td>
-      <td>...</td>
-      <td>...</td>
-      <td>...</td>
-      <td>...</td>
-      <td>...</td>
-      <td>...</td>
-      <td>...</td>
-      <td>...</td>
-      <td>...</td>
-      <td>...</td>
-      <td>...</td>
-    </tr>
-    <tr>
-      <th>284122</th>
-      <td>2024-05-21 14:42:57.894312+00:00</td>
-      <td>1.35</td>
-      <td>3.62</td>
-      <td>150.08</td>
-      <td>13.68</td>
-      <td>407.03</td>
-      <td>47.6</td>
-      <td>9.46</td>
-      <td>998.85</td>
-      <td>1016.81</td>
-      <td>24.35</td>
-      <td>1.63</td>
-    </tr>
-    <tr>
-      <th>284123</th>
-      <td>2024-05-21 14:42:59.277937+00:00</td>
-      <td>1.08</td>
-      <td>2.92</td>
-      <td>149.87</td>
-      <td>10.48</td>
-      <td>405.79</td>
-      <td>49.3</td>
-      <td>8.00</td>
-      <td>998.58</td>
-      <td>1016.53</td>
-      <td>23.41</td>
-      <td>1.29</td>
-    </tr>
-    <tr>
-      <th>284124</th>
-      <td>2024-05-21 14:43:00.594968+00:00</td>
-      <td>0.38</td>
-      <td>1.09</td>
-      <td>149.97</td>
-      <td>3.09</td>
-      <td>402.38</td>
-      <td>33.8</td>
-      <td>3.71</td>
-      <td>999.59</td>
-      <td>1017.54</td>
-      <td>24.88</td>
-      <td>0.44</td>
-    </tr>
-    <tr>
-      <th>284125</th>
-      <td>2024-05-21 14:43:01.918239+00:00</td>
-      <td>1.41</td>
-      <td>3.77</td>
-      <td>150.13</td>
-      <td>14.38</td>
-      <td>407.29</td>
-      <td>44.4</td>
-      <td>9.76</td>
-      <td>998.51</td>
-      <td>1016.48</td>
-      <td>23.54</td>
-      <td>1.70</td>
-    </tr>
-    <tr>
-      <th>284126</th>
-      <td>2024-05-21 14:43:03.248095+00:00</td>
-      <td>1.24</td>
-      <td>3.32</td>
-      <td>150.50</td>
-      <td>12.29</td>
-      <td>406.50</td>
-      <td>48.8</td>
-      <td>8.84</td>
-      <td>998.85</td>
-      <td>1016.85</td>
-      <td>22.44</td>
-      <td>1.49</td>
-    </tr>
-  </tbody>
-</table>
-<p>284127 rows × 12 columns</p>
-</div>
+```python
+# latest reading
+df = interface.query_latest()
+
+# last 3 hours
+df = interface.query_historical('-3h')
 
+# specific time range
+df = interface.query_historical('2024-05-16T00:00:00Z', '2024-05-21T00:00:00Z')
+
+# custom Flux query
+df = interface.query_custom_sync('from(bucket:"my-bucket") |> range(start: -1d) |> last()')
+```
 
+## Related Projects
 
+- [arduino-air-state-server](https://github.com/straightchlorine/arduino-air-state-server) -
+  the NodeMCU firmware that collects sensor readings and serves them over HTTP
+- [air-quality-data-analysis](https://github.com/straightchlorine/air-quality-data-analysis) -
+  Jupyter notebooks with data analysis (heatmaps, correlations, anomaly detection)
+  and SARIMAX time-series forecasting on the collected data