async-httpd-data-collector 0.1__py3-none-any.whl

ahttpdc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1"

ahttpdc/reads/fetch/async_fetch.py

@@ -0,0 +1,194 @@
+"""
+Module asyncronously fetches the data from given device and writes the readings
+into InfluxDB as records.
+
+Author: Piotr Krzysztof Lis - github.com/straightchlorine
+"""
+
+import asyncio
+import datetime
+
+import aiohttp
+from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync
+from influxdb_client.client.write.point import Point
+
+__all__ = ["AsyncReadFetcher"]
+
+
+class AsyncReadFetcher:
+    """
+    Manage fetching the readings from the sensor asyncronously via aiohttp and
+    handling received data so that it can be stored within InfluxDB.
+
+    Attributes:
+        _influxdb_host (str): The host of the InfluxDB instance.
+        _influxdb_port (int): The port of the InfluxDB instance.
+        _influxdb_token (str): The token to authenticate with InfluxDB.
+        _influxdb_organization (str): The organization to use within InfluxDB.
+        _influxdb_bucket (str): Bucket within InfluxDB where the data will be stored.
+        _device_ip (str): The IP address of device providing sensor readings.
+        _device_port (str): The port of the device providing the readings.
+        _http_handle (str): The http handle to access the data.
+        _device_address (str): The address of the device in the network.
+        _database_address (str): The address of the InfluxDB instance.
+        _sensors (dict): Sensors attached to the device.
+    """
+
+    _influxdb_host: str  # host of the influxdb instance
+    _influxdb_port: int  # port of the influxdb instance
+    _influxdb_token: str  # token to authenticate with influxdb
+    _influxdb_organization: str  # organization to use within influxdb
+    _influxdb_bucket: str  # bucket to save the data into
+
+    _dev_ip: str  # ip of the device sending the data
+    _dev_port: int  # port of the device sending the data
+    _dev_handle: str  # handle to access the data
+
+    _dev_url: str  # address of the device in the network
+    _db_url: str  # address of the influxdb instance
+
+    def __init__(
+        self, host, port, token, org, bucket, sensors, dev_ip, dev_port, handle=""
+    ):
+        """
+        Initialize the fetcher with the required information.
+
+        Args:
+            host (str): Host of the InfluxDB instance.
+            port (int): Port of the InfluxDB instance.
+            token (str): Token to authenticate with InfluxDB.
+            org (str): Organization to use within InfluxDB.
+            bucket (str): Bucket within InfluxDB where the data will be stored.
+            dev_ip (str):IP address of device providing sensor readings.
+            dev_port (str): Port of the device providing the readings.
+            handle (str): Http handle to access the data ("" by default).
+            sensors (dict): Which sensors device has and what do they measure.
+        """
+
+        # InfluxDB authentication data
+        self._influxdb_host = host
+        self._influxdb_port = port
+        self._influxdb_token = token
+        self._influxdb_organization = org
+        self._influxdb_bucket = bucket
+
+        # device identification
+        self._dev_ip = dev_ip
+        self._dev_port = dev_port
+        self._dev_handle = handle
+
+        # device and database URLs
+        self._dev_url = f"http://{self._dev_ip}:{self._dev_port}/{self._dev_handle}"
+        self._db_url = f"http://{self._influxdb_host}:{self._influxdb_port}"
+
+        self._sensors = sensors
+
+    def _get_reads(self, data) -> dict[str, float]:
+        """
+        Based on sensors specified in sensors attribute fill the fields
+        with appropriate key-value pairs for InfluxDB storage.
+
+        Args:
+            data (dict): The sensor readings to parse.
+        """
+
+        fields = {}
+        for sensor in self._sensors:
+            for param in self._sensors[sensor]:
+                fields[param] = float(data["nodemcu"][sensor][param])
+        return fields
+
+    def _parse_into_records(self, data, device_name="nodemcu"):
+        """
+        Parse raw json file into records for InfluxDB.
+
+        Args:
+            data (dict): The sensor readings to parse.
+            device_name (str): The name of the device (default is 'nodemcu').
+        """
+
+        records = {
+            "measurement": "sensor_data",
+            "tags": {"device": device_name},
+            "timestamp": str(datetime.datetime.now()),
+        }
+
+        records["fields"] = self._get_reads(data)
+        return records
+
+    async def _write_to_db(self, client, record):
+        """
+        Write the sensor readings to InfluxDB.
+
+        Args:
+            client (InfluxDBClientAsync): The InfluxDB client to write to.
+            records (dict): The sensor readings as records for InfluxDB.
+        """
+
+        print("<.> writing new read into database...")
+        write_api = client.write_api()
+        point = Point.from_dict(record, write_precision="ns")
+        await write_api.write(
+            bucket=self._influxdb_bucket, org=self._influxdb_organization, record=point
+        )
+
+    async def _store_sensor_readings(self, record):
+        """
+        Store sensor readings within InfluxDB.
+
+        Args:
+            records (dict): The sensor readings in the form of InfluxDB records.
+        """
+
+        async with InfluxDBClientAsync(
+            url=self._db_url,
+            token=self._influxdb_token,
+            org=self._influxdb_organization,
+        ) as client:
+            await self._write_to_db(client, record)
+
+    async def _request_sensor_readings(self, session):
+        """
+        Fetch the sensor readings from the device via http request.
+
+        Args:
+            session: The aiohttp session to use for the request.
+        """
+
+        async with session.get(self._dev_url) as response:
+            if response.status != 200:
+                print(f"Error fetching data: {response.status}")
+            else:
+                read = await response.json()
+                return read
+
+    async def _request_and_store(self):
+        """
+        Request sensor reading via aiohttp and store them in InfluxDB.
+        """
+
+        async with aiohttp.ClientSession() as session:
+            json = await self._request_sensor_readings(session)
+            await self._store_sensor_readings(self._parse_into_records(json))
+
+    async def _fetching_loop(self):
+        """
+        Main fetcher loop to request and store sensor readings.
+
+        Loop is infinite and receives readings every second, meant to run in
+        the background.
+        """
+
+        while True:
+            await asyncio.sleep(1)
+            await self._request_and_store()
+
+    async def schedule_fetcher(self):
+        """
+        Create a task group managing the fetching loop.
+
+        Tested using asyncio.run()
+        """
+
+        async with asyncio.TaskGroup() as tg:
+            await tg.create_task(self._fetching_loop())

ahttpdc/reads/interface.py

@@ -0,0 +1,168 @@
+"""
+Interface that control both fetching, writing and querying from the database.
+
+Author: Piotr Krzysztof Lis - github.com/straightchlorine
+"""
+
+import asyncio
+import multiprocessing
+
+import pandas as pd
+
+from ahttpdc.reads.fetch.async_fetch import AsyncReadFetcher
+from ahttpdc.reads.query.async_query import AsyncQuery
+
+__all__ = ["DatabaseInterface"]
+
+
+class DatabaseInterface:
+    """
+    Interface to control fetching, writing and querying from the database.
+
+    Attributes:
+        _influxdb_host (str): The host of the InfluxDB instance.
+        _influxdb_port (int): The port of the InfluxDB instance.
+        _influxdb_token (str): The token to authenticate with InfluxDB.
+        _influxdb_organization (str): The organization to use within InfluxDB.
+        _influxdb_bucket (str): Bucket within InfluxDB where the data will be stored.
+        _dev_ip (str): The IP address of device providing sensor readings.
+        _dev_port (str): The port of the device providing the readings.
+        _dev_handle (str): The http handle to access the data.
+        _dev_url (str): The address of the device in the network.
+        _db_url (str): The address of the influxdb instance.
+        sensors (dict): The sensors and their parameters to read.
+    """
+
+    _influxdb_host: str  # host of the influxdb instance
+    _influxdb_port: int  # port of the influxdb instance
+    _influxdb_token: str  # token to authenticate with influxdb
+    _influxdb_organization: str  # organization to use within influxdb
+    _influxdb_bucket: str  # bucket to save the data into
+
+    _dev_ip: str  # ip of the device sending the data
+    _dev_port: int  # port of the device sending the data
+    _dev_handle: str  # handle to access the data
+
+    _dev_url: str  # address of the device in the network
+    _db_url: str  # address of the influxdb instance
+
+    def __init__(
+        self, host, port, token, org, bucket, sensors, dev_ip, dev_port, handle=""
+    ):
+        """
+        Initialize the fetcher with the required information.
+
+        Args:
+            host (str): The host of the InfluxDB instance.
+            port (int): The port of the InfluxDB instance.
+            token (str): The token to authenticate with InfluxDB.
+            org (str): The organization to use within InfluxDB.
+            bucket (str): Bucket within InfluxDB where the data will be stored.
+            dev_ip (str): The IP address of device providing sensor readings.
+            dev_port (str): The port of the device providing the readings.
+            handle (str): The http handle to access the data ("" by default).
+            sensors (dict): The sensors and their parameters to read.
+        """
+
+        self._influxdb_host = host
+        self._influxdb_port = port
+        self._influxdb_token = token
+        self._influxdb_organization = org
+        self._influxdb_bucket = bucket
+
+        self._dev_ip = dev_ip
+        self._dev_port = dev_port
+        self._dev_handle = handle
+
+        self._dev_url = f"http://{self._dev_ip}:{self._dev_port}/{self._dev_handle}"
+        self._db_url = f"http://{self._influxdb_host}:{self._influxdb_port}"
+
+        self.sensors = sensors
+
+        self._fetcher = AsyncReadFetcher(
+            self._influxdb_host,
+            self._influxdb_port,
+            self._influxdb_token,
+            self._influxdb_organization,
+            self._influxdb_bucket,
+            self.sensors,
+            self._dev_ip,
+            self._dev_port,
+            self._dev_handle,
+        )
+
+        self.query_interface = AsyncQuery(
+            self._influxdb_host,
+            self._influxdb_port,
+            self._influxdb_token,
+            self._influxdb_organization,
+            self._influxdb_bucket,
+            self.sensors,
+        )
+
+    def enable_fetching(self):
+        """
+        Enable fetching from the device specified by dev_ip, dev_port and handle.
+
+        Starts the fetching task in the background, thus should be invoked last
+        in order to avoid blocking the main thread.
+        """
+
+        def _start_fetching():
+            asyncio.run(self._fetcher.schedule_fetcher())
+
+        fetching_process = multiprocessing.Process(
+            target=_start_fetching, name="asyncfetcher"
+        )
+        fetching_process.start()
+
+    async def query_latest(self) -> pd.DataFrame:
+        """
+        Query the latest measurement from the InfluxDB.
+
+        Returns:
+            pd.DataFrame: The latest measurement.
+        """
+
+        print("<.> querying latest measurement...")
+        query_task = asyncio.create_task(self.query_interface.latest())
+        await query_task
+        result = query_task.result()
+        return result
+
+    async def query_historical(self, start: str, end: str) -> pd.DataFrame:
+        """
+        Query historical data from the database.
+
+        Args:
+            start (str): Start time of the query (e.g., '2024-01-01T00:00:00Z').
+            end (str): End time of the query (e.g., '2024-01-02T00:00:00Z').
+
+        Returns:
+            pd.DataFrame: Historical data within the specified time range.
+        """
+
+        print("<.> querying historical data...")
+        query_task = asyncio.create_task(
+            self.query_interface.historical_data(start, end)
+        )
+        await query_task
+        result = query_task.result()
+        return result
+
+    async def query(self, query: str) -> pd.DataFrame:
+        """
+        Perform a custom query on the database.
+
+        Args:
+            query (str): The InfluxDB query to execute.
+
+        Returns:
+            pd.DataFrame: The result of the custom query.
+        """
+
+        print("<.> custom query...")
+        query_task = asyncio.create_task(self.query_interface.query(query))
+        await query_task
+        result = query_task.result()
+        return result

ahttpdc/reads/query/async_query.py

@@ -0,0 +1,222 @@
+"""
+Interface for passing queries to the database.
+
+Author: Piotr Krzysztof Lis - github.com/straightchlorine
+"""
+
+from datetime import datetime, timedelta
+
+from influxdb_client.client.exceptions import InfluxDBError
+from influxdb_client.client.influxdb_client_async import InfluxDBClientAsync
+import pandas as pd
+
+__all__ = ["AsyncQuery"]
+
+
+class AsyncQuery:
+    """
+    Interface to query the InfluxDB.
+
+    Attributes:
+        _influxdb_host (str): The host of the InfluxDB instance.
+        _influxdb_port (int): The port of the InfluxDB instance.
+        _influxdb_token (str): The token to authenticate with InfluxDB.
+        _influxdb_organization (str): The organization to use within InfluxDB.
+        _influxdb_bucket (str): Bucket within InfluxDB where the data will be stored.
+        _db_url (str): The URL of the InfluxDB instance.
+        sensors_and_params (dict): The sensors and their parameters to read.
+    """
+
+    _influxdb_host: str  # host of the influxdb instance
+    _influxdb_port: int  # port of the influxdb instance
+    _influxdb_token: str  # token to authenticate with influxdb
+    _influxdb_organization: str  # organization to use within influxdb
+    _influxdb_bucket: str  # bucket to save the data into
+
+    _db_url: str  # address of the influxdb instance
+
+    sensors: dict  # sensors and their parameters to read
+
+    def __init__(self, host, port, token, org, bucket, sensors):
+        """
+        Initialize the fetcher with the required information.
+
+        Args:
+            host (str): The host of the InfluxDB instance.
+            port (int): The port of the InfluxDB instance.
+            token (str): The token to authenticate with InfluxDB.
+            org (str): The organization to use within InfluxDB.
+            bucket (str): Bucket within InfluxDB where the data will be stored.
+            sensors (dict): The sensors and their parameters to read.
+        """
+
+        self._influxdb_host = host
+        self._influxdb_port = port
+        self._influxdb_token = token
+        self._influxdb_organization = org
+        self._influxdb_bucket = bucket
+
+        self._db_url = f"http://{self._influxdb_host}:{self._influxdb_port}"
+
+        self.sensors = sensors
+
+    async def _get_InfluxDB_client(self) -> InfluxDBClientAsync:
+        """Returns an InfluxDB client."""
+        return InfluxDBClientAsync(
+            url=self._db_url,
+            token=self._influxdb_token,
+            org=self._influxdb_organization,
+        )
+
+    def _convert_to_local_time(self, timestamps):
+        """
+        Convert a collection of UTC timestamps to local time.
+
+        Args:
+            timestamps (list): A list of UTC timestamps.
+        Returns:
+            list: A list of timestamps in local time.
+        """
+
+        local_timestamps = []
+
+        # get the local offset from UTC
+        local_offset = timedelta(
+            seconds=datetime.now().astimezone().utcoffset().total_seconds()
+        )
+
+        # add the offset to the timestamps
+        for timestamp in timestamps:
+            utc_time = pd.to_datetime(timestamp)
+            local_time = utc_time + local_offset
+            local_timestamps.append(local_time)
+
+        return local_timestamps
+
+    def _into_dataframe(self, tables) -> pd.DataFrame:
+        """
+        Turns the tables into a pandas DataFrame.
+        Args:
+            tables (list): The tables to turn into a DataFrame.
+        Returns:
+            pd.DataFrame: procured measurements as a DataFrame.
+        """
+
+        read: dict = {}
+        timestamps = set()
+
+        # unpacking the table
+        for table in tables:
+            for record in table.records:
+                # get the measurements
+                parameter = record.get_field()
+                measurement = record.get_value()
+                timestamps.add(record.get_time())
+
+                # ensure every parameter is present in the dict
+                if parameter not in read:
+                    read[parameter] = []
+                read[parameter].append(float(measurement))
+
+        # convert timestamps to local time
+        local_timestamps = self._convert_to_local_time(timestamps)
+
+        # if there is no time key, create one
+        if "time" not in read:
+            read["time"] = []
+
+        # add the timestamps to the data dict
+        for timestamp in local_timestamps:
+            read["time"].append(pd.to_datetime(timestamp))
+
+        # return the data as a DataFrame
+        return pd.DataFrame(read)
+
+    async def latest(self) -> pd.DataFrame:
+        """
+        Query the database for the latest measurement.
+
+        Returns:
+            pd.DataFrame: The latest measurement of every parameter.
+        """
+
+        # get the connection to the database via query api
+        client = await self._get_InfluxDB_client()
+        query_api = client.query_api()
+
+        # query the latest measurement
+        query = f'from(bucket:"{self._influxdb_bucket}") |> range(start: -1h) |> last()'
+
+        tables = None
+        try:
+            tables = await query_api.query(query)
+        except InfluxDBError as e:
+            print(f"Exception caught while querying the database:\n\n {e.message}")
+
+        # close the connection
+        await client.close()
+
+        # turn the tables into a DataFrame and return it
+        if tables is not None:
+            return self._into_dataframe(tables)
+        else:
+            return pd.DataFrame()
+
+    async def historical_data(self, start: str, end: str) -> pd.DataFrame:
+        """
+        Query historical data from the database.
+
+        Args:
+            start (str): Start time of the query (e.g., '2024-01-01T00:00:00Z').
+            end (str): End time of the query (e.g., '2024-01-02T00:00:00Z').
+
+        Returns:
+            pd.DataFrame: Historical data within the specified time range.
+        """
+
+        # get the connection to the database via query api
+        client = await self._get_InfluxDB_client()
+        query_api = client.query_api()
+
+        # query the data from specified start and finish
+        query = f'from(bucket:"{self._influxdb_bucket}") |> range(start: {start}, stop: {end})'
+
+        tables = None
+        try:
+            tables = await query_api.query(query)
+        except InfluxDBError as e:
+            print(f"Exception caught while querying the database:\n\n {e.message}")
+
+        await client.close()
+
+        if tables is not None:
+            return self._into_dataframe(tables)
+        else:
+            return pd.DataFrame()
+
+    async def query(self, query: str) -> pd.DataFrame:
+        """
+        Perform a custom query on the database.
+
+        Args:
+            query (str): The InfluxDB query to execute.
+
+        Returns:
+            pd.DataFrame: The result of the custom query.
+        """
+        client = await self._get_InfluxDB_client()
+        query_api = client.query_api()
+
+        tables = None
+
+        try:
+            tables = await query_api.query(query)
+        except InfluxDBError as e:
+            print(f"Exception caught while querying the database:\n\n {e.message}")
+
+        await client.close()
+
+        if tables is not None:
+            return self._into_dataframe(tables)
+        else:
+            return pd.DataFrame()