async-httpd-data-collector 0.1__tar.gz → 0.2__tar.gz

async_httpd_data_collector-0.2/.github/workflows/publish.yml

@@ -0,0 +1,116 @@
+name: Python CI
+
+# publish on push to master
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    services:
+      influxdb:
+        image: influxdb:2
+        ports:
+          - 8086:8086
+
+      flask:
+        image: python:3.9
+        ports:
+          - 5000:5000
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+
+      - name: Install InfluxDB CLI
+        run: |
+          wget https://download.influxdata.com/influxdb/releases/influxdb2-client-2.7.5-linux-amd64.tar.gz
+          tar xvzf ./influxdb2-client-2.7.5-linux-amd64.tar.gz
+          sudo cp ./influx /usr/local/bin/
+
+      - name: Set up InfluxDB
+        id: setup-influx
+        run: |
+
+          # passes to test database with no data
+          INFLUX_USERNAME="test-user"
+          INFLUX_PASSWORD="test-password"
+          INFLUX_ORG="test-org"
+          INFLUX_BUCKET="test-bucket"
+
+          # setting up the database
+          influx setup \
+            --username "$INFLUX_USERNAME" \
+            --password "$INFLUX_PASSWORD" \
+            --org "$INFLUX_ORG" \
+            --bucket "$INFLUX_BUCKET" \
+            --force
+
+          # saving information required for connection later
+          echo "::set-output name=org::${INFLUX_ORG}"
+          echo "::set-output name=bucket::${INFLUX_BUCKET}"
+
+      - name: Authorization for InfluxDB
+        id: auth
+        run: |
+
+          # authenticating the connection
+          INFLUX_TOKEN=$(influx auth create \
+            --user "$INFLUX_USERNAME" \
+            --org "$INFLUX_ORG" \
+            --all-access \
+            --description "testing token" \
+            --skip-verify \
+            --json | jq '.token')
+          
+          # saving the token for the connection
+          echo "::set-output name=token::${INFLUX_TOKEN}"
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+          pip install ruff
+          pip install twine
+          pip install -r requirements.txt
+
+      - name: Lint with ruff
+        run: |
+          ruff format
+          ruff check
+
+      - name: Test with pytest
+        run: |
+
+          # starting the flask server, supplying some sample data
+          export FLASK_APP="tests/dev_server"
+          flask run --host=0.0.0.0 --port=9000 &
+          sleep 5
+
+          # setting the connection variables as for the pytest to use
+          export PYTHONPATH="$PYTHONPATH:$(pwd)"
+          export INFLUXDB_HOST="localhost"
+          export INFLUXDB_PORT=8086
+          export INFLUXDB_TOKEN=${{steps.auth.outputs.token}}
+          export INFLUXDB_ORG=${{steps.setup-influx.outputs.org}}
+          export INFLUXDB_BUCKET=${{steps.setup-influx.outputs.bucket}}
+          pytest 
+
+      - name: Build the application
+        run: |
+          python3 -m build
+
+      - name: Publish to the PyPi
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          python3 -m twine upload dist/*

async_httpd_data_collector-0.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 InfluxData
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

async_httpd_data_collector-0.2/PKG-INFO

@@ -0,0 +1,152 @@
+Metadata-Version: 2.3
+Name: async-httpd-data-collector
+Version: 0.2
+Summary: Gateway facilitating asyncronous communication between sensory data-emitting devices, InfluxDB and the user.
+Project-URL: Repository, https://github.com/straightchlorine/async-httpd-data-collector
+Project-URL: Issues, https://github.com/straightchlorine/async-httpd-data-collector/issues
+Author-email: Piotr Krzysztof Lis <piotrlis555@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2021 InfluxData
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: arduino,async,collector,data,http,influxdb,json,nodemcu,sensors
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.8
+Requires-Dist: aiocsv
+Requires-Dist: aiohttp
+Requires-Dist: aiosignal
+Requires-Dist: attrs
+Requires-Dist: certifi
+Requires-Dist: frozenlist
+Requires-Dist: idna
+Requires-Dist: influxdb-client[async]
+Requires-Dist: multidict
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: python-dateutil
+Requires-Dist: pytz
+Requires-Dist: reactivex
+Requires-Dist: setuptools
+Requires-Dist: six
+Requires-Dist: typing-extensions
+Requires-Dist: tzdata
+Requires-Dist: urllib3
+Requires-Dist: yarl
+Provides-Extra: test
+Requires-Dist: blinker; extra == 'test'
+Requires-Dist: charset-normalizer; extra == 'test'
+Requires-Dist: click; extra == 'test'
+Requires-Dist: dash; extra == 'test'
+Requires-Dist: dash-core-components; extra == 'test'
+Requires-Dist: dash-html-components; extra == 'test'
+Requires-Dist: dash-table; extra == 'test'
+Requires-Dist: flask; extra == 'test'
+Requires-Dist: importlib-metadata; extra == 'test'
+Requires-Dist: iniconfig; extra == 'test'
+Requires-Dist: itsdangerous; extra == 'test'
+Requires-Dist: jinja2; extra == 'test'
+Requires-Dist: markupsafe; extra == 'test'
+Requires-Dist: nest-asyncio; extra == 'test'
+Requires-Dist: packaging; extra == 'test'
+Requires-Dist: plotly; extra == 'test'
+Requires-Dist: pluggy; extra == 'test'
+Requires-Dist: pytest; extra == 'test'
+Requires-Dist: pytest-asyncio; extra == 'test'
+Requires-Dist: requests; extra == 'test'
+Requires-Dist: retrying; extra == 'test'
+Requires-Dist: tenacity; extra == 'test'
+Requires-Dist: werkzeug; extra == 'test'
+Requires-Dist: zipp; extra == 'test'
+Description-Content-Type: text/markdown
+
+# influx-cli
+
+CLI for managing resources in InfluxDB v2
+
+## Motivation
+
+This repository decouples the `influx` CLI from the OSS `influxdb` codebase. Our goals are to:
+1. Make it easier to keep the CLI up-to-date with InfluxDB Cloud API changes
+2. Enable faster turn-around on fixes/features that only affect the CLI
+3. Allow the CLI to be built & released for a wider range of platforms than the server can support
+
+## Building the CLI
+
+Follow these steps to build the CLI. If you're updating your CLI build, see *Updating openapi* below.
+1. Clone this repo (influx-cli) and change to your _influx-cli_ directory.
+
+   ```
+   git clone git@github.com:influxdata/influx-cli.git
+   cd influx-cli
+   ```
+   
+2. Build the CLI. The `make` and `make influx` commands write the new binary to `bin/$(GOOS)/influx`.
+   
+   ```
+   make
+   ```
+   
+### Updating openapi
+
+If you change or update your branch, you may also need to update `influx-cli/openapi` and regenerate the client code.
+`influx-cli/openapi` is a Git submodule that contains the underlying API contracts and client used by the CLI.
+We use [`OpenAPITools/openapi-generator`](https://github.com/OpenAPITools/openapi-generator) to generate
+the HTTP client.
+
+To update, run the following commands in your `influx-cli` repo:
+
+1. Update the _openapi_ Git submodule. The following command pulls the latest commits for the branch and all submodules.
+
+   `git pull --recurse-submodules`
+   
+2. With [Docker](https://docs.docker.com/get-docker/) running locally, regenerate _openapi_.
+
+   `make openapi`
+   
+3. Rebuild the CLI
+
+   `make`
+ 
+## Running the CLI
+
+After building, use `influx -h` to see the list of available commands.
+
+### Enabling Completions
+
+The CLI supports generating completions for `bash`, `zsh`, and `powershell`. To enable completions for a
+single shell session, run one of these commands:
+```
+# For bash:
+source <(influx completion bash)
+# For zsh:
+source <(influx completion zsh)
+# For pwsh:
+Invoke-Expression ((influx completion powershell) -join "`n`")
+```
+To enable completions across sessions, add the appropriate line to your shell's login profile (i.e. `~/.bash_profile`).
+
+## Testing
+
+Run `make test` to run unit tests.

async_httpd_data_collector-0.2/README.md

@@ -0,0 +1,69 @@
+# influx-cli
+
+CLI for managing resources in InfluxDB v2
+
+## Motivation
+
+This repository decouples the `influx` CLI from the OSS `influxdb` codebase. Our goals are to:
+1. Make it easier to keep the CLI up-to-date with InfluxDB Cloud API changes
+2. Enable faster turn-around on fixes/features that only affect the CLI
+3. Allow the CLI to be built & released for a wider range of platforms than the server can support
+
+## Building the CLI
+
+Follow these steps to build the CLI. If you're updating your CLI build, see *Updating openapi* below.
+1. Clone this repo (influx-cli) and change to your _influx-cli_ directory.
+
+   ```
+   git clone git@github.com:influxdata/influx-cli.git
+   cd influx-cli
+   ```
+   
+2. Build the CLI. The `make` and `make influx` commands write the new binary to `bin/$(GOOS)/influx`.
+   
+   ```
+   make
+   ```
+   
+### Updating openapi
+
+If you change or update your branch, you may also need to update `influx-cli/openapi` and regenerate the client code.
+`influx-cli/openapi` is a Git submodule that contains the underlying API contracts and client used by the CLI.
+We use [`OpenAPITools/openapi-generator`](https://github.com/OpenAPITools/openapi-generator) to generate
+the HTTP client.
+
+To update, run the following commands in your `influx-cli` repo:
+
+1. Update the _openapi_ Git submodule. The following command pulls the latest commits for the branch and all submodules.
+
+   `git pull --recurse-submodules`
+   
+2. With [Docker](https://docs.docker.com/get-docker/) running locally, regenerate _openapi_.
+
+   `make openapi`
+   
+3. Rebuild the CLI
+
+   `make`
+ 
+## Running the CLI
+
+After building, use `influx -h` to see the list of available commands.
+
+### Enabling Completions
+
+The CLI supports generating completions for `bash`, `zsh`, and `powershell`. To enable completions for a
+single shell session, run one of these commands:
+```
+# For bash:
+source <(influx completion bash)
+# For zsh:
+source <(influx completion zsh)
+# For pwsh:
+Invoke-Expression ((influx completion powershell) -join "`n`")
+```
+To enable completions across sessions, add the appropriate line to your shell's login profile (i.e. `~/.bash_profile`).
+
+## Testing
+
+Run `make test` to run unit tests.

async_httpd_data_collector-0.2/ahttpdc/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.2'

{async_httpd_data_collector-0.1 → async_httpd_data_collector-0.2}/ahttpdc/reads/fetch/async_fetch.py

@@ -12,7 +12,7 @@ import aiohttp
 from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync
 from influxdb_client.client.write.point import Point
 
-__all__ = ["AsyncReadFetcher"]
+__all__ = ['AsyncReadFetcher']
 
 
 class AsyncReadFetcher:
@@ -25,7 +25,7 @@ class AsyncReadFetcher:
         _influxdb_port (int): The port of the InfluxDB instance.
         _influxdb_token (str): The token to authenticate with InfluxDB.
         _influxdb_organization (str): The organization to use within InfluxDB.
-        _influxdb_bucket (str): Bucket within InfluxDB where the data will be stored.
+        _influxdb_bucket (str): InfluxDB bucket where the data will be stored.
         _device_ip (str): The IP address of device providing sensor readings.
         _device_port (str): The port of the device providing the readings.
         _http_handle (str): The http handle to access the data.
@@ -48,7 +48,7 @@ class AsyncReadFetcher:
     _db_url: str  # address of the influxdb instance
 
     def __init__(
-        self, host, port, token, org, bucket, sensors, dev_ip, dev_port, handle=""
+        self, host, port, token, org, bucket, sensors, dev_ip, dev_port, handle=''
     ):
         """
         Initialize the fetcher with the required information.
@@ -78,12 +78,12 @@ class AsyncReadFetcher:
         self._dev_handle = handle
 
         # device and database URLs
-        self._dev_url = f"http://{self._dev_ip}:{self._dev_port}/{self._dev_handle}"
-        self._db_url = f"http://{self._influxdb_host}:{self._influxdb_port}"
+        self._dev_url = f'http://{self._dev_ip}:{self._dev_port}/{self._dev_handle}'
+        self._db_url = f'http://{self._influxdb_host}:{self._influxdb_port}'
 
         self._sensors = sensors
 
-    def _get_reads(self, data) -> dict[str, float]:
+    def _get_reads(self, data, device) -> dict[str, float]:
         """
         Based on sensors specified in sensors attribute fill the fields
         with appropriate key-value pairs for InfluxDB storage.
@@ -95,10 +95,19 @@ class AsyncReadFetcher:
         fields = {}
         for sensor in self._sensors:
             for param in self._sensors[sensor]:
-                fields[param] = float(data["nodemcu"][sensor][param])
+                if param not in fields:
+                    fields[param] = float(data[device][sensor][param])
+                else:
+                    # if the measurement already has been recorded, calculate
+                    # the average of the measurements
+                    previous = fields[param]
+                    current = float(data[device][sensor][param])
+                    average = (previous + current) / 2
+                    fields[param] = average
+
         return fields
 
-    def _parse_into_records(self, data, device_name="nodemcu"):
+    def _parse_into_records(self, data):
         """
         Parse raw json file into records for InfluxDB.
 
@@ -107,13 +116,14 @@ class AsyncReadFetcher:
             device_name (str): The name of the device (default is 'nodemcu').
         """
 
+        device = list(data.keys())[0]
         records = {
-            "measurement": "sensor_data",
-            "tags": {"device": device_name},
-            "timestamp": str(datetime.datetime.now()),
+            'measurement': 'sensor_data',
+            'tags': {'device': device},
+            'timestamp': str(datetime.datetime.now()),
         }
 
-        records["fields"] = self._get_reads(data)
+        records['fields'] = self._get_reads(data, device)
         return records
 
     async def _write_to_db(self, client, record):
@@ -125,9 +135,9 @@ class AsyncReadFetcher:
             records (dict): The sensor readings as records for InfluxDB.
         """
 
-        print("<.> writing new read into database...")
+        print('<.> writing new read into database...')
         write_api = client.write_api()
-        point = Point.from_dict(record, write_precision="ns")
+        point = Point.from_dict(record, write_precision='ns')
         await write_api.write(
             bucket=self._influxdb_bucket, org=self._influxdb_organization, record=point
         )
@@ -157,7 +167,7 @@ class AsyncReadFetcher:
 
         async with session.get(self._dev_url) as response:
             if response.status != 200:
-                print(f"Error fetching data: {response.status}")
+                print(f'Error fetching data: {response.status}')
             else:
                 read = await response.json()
                 return read

{async_httpd_data_collector-0.1 → async_httpd_data_collector-0.2}/ahttpdc/reads/interface.py

@@ -12,7 +12,7 @@ import pandas as pd
 from ahttpdc.reads.fetch.async_fetch import AsyncReadFetcher
 from ahttpdc.reads.query.async_query import AsyncQuery
 
-__all__ = ["DatabaseInterface"]
+__all__ = ['DatabaseInterface']
 
 
 class DatabaseInterface:
@@ -47,7 +47,7 @@ class DatabaseInterface:
     _db_url: str  # address of the influxdb instance
 
     def __init__(
-        self, host, port, token, org, bucket, sensors, dev_ip, dev_port, handle=""
+        self, host, port, token, org, bucket, sensors, dev_ip, dev_port, handle=''
     ):
         """
         Initialize the fetcher with the required information.
@@ -74,8 +74,8 @@ class DatabaseInterface:
         self._dev_port = dev_port
         self._dev_handle = handle
 
-        self._dev_url = f"http://{self._dev_ip}:{self._dev_port}/{self._dev_handle}"
-        self._db_url = f"http://{self._influxdb_host}:{self._influxdb_port}"
+        self._dev_url = f'http://{self._dev_ip}:{self._dev_port}/{self._dev_handle}'
+        self._db_url = f'http://{self._influxdb_host}:{self._influxdb_port}'
 
         self.sensors = sensors
 
@@ -100,6 +100,13 @@ class DatabaseInterface:
             self.sensors,
         )
 
+        def _start_fetching():
+            asyncio.run(self._fetcher.schedule_fetcher())
+
+        self.fetching_process = multiprocessing.Process(
+            target=_start_fetching, name='asyncfetcher'
+        )
+
     def enable_fetching(self):
         """
         Enable fetching from the device specified by dev_ip, dev_port and handle.
@@ -108,13 +115,15 @@ class DatabaseInterface:
         in order to avoid blocking the main thread.
         """
 
-        def _start_fetching():
-            asyncio.run(self._fetcher.schedule_fetcher())
+        self.fetching_process.start()
 
-        fetching_process = multiprocessing.Process(
-            target=_start_fetching, name="asyncfetcher"
-        )
-        fetching_process.start()
+    def disable_fetching(self):
+        """
+        Terminate fetching process.
+        """
+
+        self.fetching_process.terminate()
+        self.fetching_process.join()
 
     async def query_latest(self) -> pd.DataFrame:
         """
@@ -124,7 +133,7 @@ class DatabaseInterface:
             pd.DataFrame: The latest measurement.
         """
 
-        print("<.> querying latest measurement...")
+        print('<.> querying latest measurement...')
         query_task = asyncio.create_task(self.query_interface.latest())
         await query_task
         result = query_task.result()
@@ -142,7 +151,7 @@ class DatabaseInterface:
             pd.DataFrame: Historical data within the specified time range.
         """
 
-        print("<.> querying historical data...")
+        print('<.> querying historical data...')
         query_task = asyncio.create_task(
             self.query_interface.historical_data(start, end)
         )
@@ -161,7 +170,7 @@ class DatabaseInterface:
             pd.DataFrame: The result of the custom query.
         """
 
-        print("<.> custom query...")
+        print('<.> custom query...')
         query_task = asyncio.create_task(self.query_interface.query(query))
         await query_task
         result = query_task.result()

{async_httpd_data_collector-0.1 → async_httpd_data_collector-0.2}/ahttpdc/reads/query/async_query.py

@@ -10,7 +10,7 @@ from influxdb_client.client.exceptions import InfluxDBError
 from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync
 import pandas as pd
 
-__all__ = ["AsyncQuery"]
+__all__ = ['AsyncQuery']
 
 
 class AsyncQuery:
@@ -56,12 +56,13 @@ class AsyncQuery:
         self._influxdb_organization = org
         self._influxdb_bucket = bucket
 
-        self._db_url = f"http://{self._influxdb_host}:{self._influxdb_port}"
+        self._db_url = f'http://{self._influxdb_host}:{self._influxdb_port}'
 
         self.sensors = sensors
 
     async def _get_InfluxDB_client(self) -> InfluxDBClientAsync:
         """Returns an InfluxDB client."""
+
         return InfluxDBClientAsync(
             url=self._db_url,
             token=self._influxdb_token,
@@ -101,7 +102,7 @@ class AsyncQuery:
         Returns:
             pd.DataFrame: procured measurements as a DataFrame.
         """
-
+        print('into dataframe')
         read: dict = {}
         timestamps = set()
 
@@ -122,12 +123,12 @@ class AsyncQuery:
         local_timestamps = self._convert_to_local_time(timestamps)
 
         # if there is no time key, create one
-        if "time" not in read:
-            read["time"] = []
+        if 'time' not in read:
+            read['time'] = []
 
         # add the timestamps to the data dict
         for timestamp in local_timestamps:
-            read["time"].append(pd.to_datetime(timestamp))
+            read['time'].append(pd.to_datetime(timestamp))
 
         # return the data as a DataFrame
         return pd.DataFrame(read)
@@ -151,7 +152,7 @@ class AsyncQuery:
         try:
             tables = await query_api.query(query)
         except InfluxDBError as e:
-            print(f"Exception caught while querying the database:\n\n {e.message}")
+            print(f'Exception caught while querying the database:\n\n {e.message}')
 
         # close the connection
         await client.close()
@@ -185,7 +186,7 @@ class AsyncQuery:
         try:
             tables = await query_api.query(query)
         except InfluxDBError as e:
-            print(f"Exception caught while querying the database:\n\n {e.message}")
+            print(f'Exception caught while querying the database:\n\n {e.message}')
 
         await client.close()
 
@@ -212,7 +213,7 @@ class AsyncQuery:
         try:
             tables = await query_api.query(query)
         except InfluxDBError as e:
-            print(f"Exception caught while querying the database:\n\n {e.message}")
+            print(f'Exception caught while querying the database:\n\n {e.message}')
 
         await client.close()