miza-datahub 1.0.0__tar.gz

miza_datahub-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 chanelcolgate
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

miza_datahub-1.0.0/MANIFEST.in

@@ -0,0 +1,2 @@
+graft src
+recursive-exclude __pycache__ *.py[cod]

miza_datahub-1.0.0/PKG-INFO

@@ -0,0 +1,42 @@
+Metadata-Version: 2.4
+Name: miza-datahub
+Version: 1.0.0
+Summary: This package does amazing things.
+Home-page: http://github.com/chanelcolgate/miza-datahub
+Author: chanelcolgate
+Author-email: "chanelcolgate" <github@chanelcolgate.com>
+License: MIT
+Classifier: License :: OSI Approved :: MIT License
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: requests
+Requires-Dist: openpyxl
+Dynamic: license-file
+
+## Miza DataHub Library
+### How to install
+- Cài pip
+```bash
+pip install --upgrade pip setuptools wheel
+```
+- Cách khác
+```bash
+python -m ensurepip --upgrade
+```
+- Cách cài đặt và chạy
+```
+pyproject-build
+python -m pip install -e .
+```
+- Cách test
+```
+python -m pip install pytest
+python -m pytest
+python -m pip install mypy
+python -m mypy
+python -m pip install tox
+tox -e typecheck 
+tox -e lint
+```

miza_datahub-1.0.0/README.md

@@ -0,0 +1,25 @@
+## Miza DataHub Library
+### How to install
+- Cài pip
+```bash
+pip install --upgrade pip setuptools wheel
+```
+- Cách khác
+```bash
+python -m ensurepip --upgrade
+```
+- Cách cài đặt và chạy
+```
+pyproject-build
+python -m pip install -e .
+```
+- Cách test
+```
+python -m pip install pytest
+python -m pytest
+python -m pip install mypy
+python -m mypy
+python -m pip install tox
+tox -e typecheck 
+tox -e lint
+```

miza_datahub-1.0.0/pyproject.toml

@@ -0,0 +1,16 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.black]
+line-length = 80
+target-version = ['py311']
+
+[dependency-groups]
+dev = [
+    "numpy>=2.4.6",
+]
+
+[[tool.mypy.overrides]]
+module = ["pandas.*", "numpy.*", "requests"]
+ignore_missing_imports = true

miza_datahub-1.0.0/setup.cfg

@@ -0,0 +1,73 @@
+[metadata]
+name = miza-datahub
+version = 1.0.0
+url = http://github.com/chanelcolgate/miza-datahub
+author = chanelcolgate
+author_email = "chanelcolgate" <github@chanelcolgate.com>
+description = This package does amazing things.
+long_description = file: README.md
+long_description_content_type = text/markdown
+license = MIT
+license_files = LICENSE
+classifiers = 
+	License :: OSI Approved :: MIT License
+
+[options]
+package_dir = 
+	=src
+packages = find:
+include_package_data = True
+install_requires = 
+	numpy
+	pandas
+	requests
+	openpyxl
+
+[options.packages.find]
+where = src
+exclude = 
+	test*
+
+[tox:tox]
+isolated_build = True
+envlist = py311
+
+[flake8]
+max-line-length = 80
+
+[testenv]
+deps = 
+	pytest
+	pytest-cov
+commands = 
+	pytest {posargs}
+
+[testenv:typecheck]
+deps = 
+	black
+	mypy
+	pytest
+	types-termcolor
+commands = 
+	black {posargs:--check --diff src test}
+	mypy src test
+
+[testenv:format]
+skip_install = True
+deps = 
+	black
+commands = 
+	black {posargs:--check --diff src test}
+
+[testenv:lint]
+skip_install = True
+deps = 
+	flake8
+	flake8-bugbear
+commands = 
+	flake8 {posargs:--ignore=E203,W503 src test}
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

miza_datahub-1.0.0/src/logic.egg-info/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: logic
+Version: 0.0.1
+Summary: This package does amazing things.
+Home-page: http://github.com/chanelcolgate/miza-datahub
+Author: chanelcolgate
+Author-email: "chanelcolgate" <github@chanelcolgate.com>
+License: MIT
+Classifier: License :: OSI Approved :: MIT License
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file

miza_datahub-1.0.0/src/logic.egg-info/SOURCES.txt

@@ -0,0 +1,17 @@
+LICENSE
+MANIFEST.in
+README.md
+pyproject.toml
+setup.cfg
+src/logic.egg-info/PKG-INFO
+src/logic.egg-info/SOURCES.txt
+src/logic.egg-info/dependency_links.txt
+src/logic.egg-info/top_level.txt
+src/miza_datahub/__init__.py
+src/miza_datahub/common/__init__.py
+src/miza_datahub/common/config_const.py
+src/miza_datahub/common/config_util.py
+src/miza_datahub/common/singleton.py
+src/miza_datahub/readers/__init__.py
+src/miza_datahub/readers/base_reader.py
+src/miza_datahub/readers/finished_product_quality_analyzer.py

miza_datahub-1.0.0/src/logic.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

miza_datahub-1.0.0/src/logic.egg-info/top_level.txt

@@ -0,0 +1 @@
+miza_datahub

miza_datahub-1.0.0/src/miza_datahub/common/config_const.py

@@ -0,0 +1,14 @@
+EXCEL = "Excel"
+DATA_PATH = "data_path"
+
+TELEGRAM = "Telegram"
+TOKEN = "token"
+URL_GET_FILE_PATH = "https://api.telegram.org/bot{token}/getFile"
+URL_GET_FILE_CONTENT = "https://api.telegram.org/file/bot{token}/{file_path}"
+
+INFLUX = "Influx"
+INFLUX_HOST = "host"
+INFLUX_PORT = "port"
+INFLUX_DB = "db"
+
+DEFAULT_CONFIG_FILE_NAME = "./config/miza_datahub_config.props"

miza_datahub-1.0.0/src/miza_datahub/common/config_util.py

@@ -0,0 +1,74 @@
+import os
+import configparser
+import logging
+
+from miza_datahub.common.singleton import Singleton
+from miza_datahub.common import config_const as ConfigConst
+
+
+class ConfigUtil(metaclass=Singleton):
+    config_file = ConfigConst.DEFAULT_CONFIG_FILE_NAME
+    config_parser = configparser.ConfigParser()
+    is_loaded = False
+
+    def __init__(self, config_file: str | None = None) -> None:
+        if config_file is not None:
+            self.config_file = config_file
+
+        self._load_config()
+        logging.info(f"Creted instance of ConfigUtil: {self}")
+
+    def _load_config(self):
+        if os.path.exists(self.config_file):
+            logging.info(f"Loading config: {self.config_file}")
+            self.config_parser.read(self.config_file)
+            self.is_loaded = True
+        else:
+            logging.info(f"""Can't load {self.config_file}.
+                Trying default: {ConfigConst.DEFAULT_CONFIG_FILE_NAME}""")
+            self.config_file = ConfigConst.DEFAULT_CONFIG_FILE_NAME
+            self.config_parser.read(self.config_file)
+            self.is_loaded = True
+
+        logging.debug(f"Config: {self.config_parser.sections()}")
+
+    def _get_config(
+        self, force_reload: bool = False
+    ) -> configparser.ConfigParser:
+        if not self.is_loaded or force_reload:
+            self._load_config()
+
+        return self.config_parser
+
+    def get_property(
+        self,
+        section: str,
+        key: str,
+        default_val: str | None = None,
+        force_reload: bool = False,
+    ):
+        return self._get_config(force_reload).get(
+            section, key, fallback=default_val
+        )
+
+    def get_float(
+        self,
+        section: str,
+        key: str,
+        default_val: float = 0.0,
+        force_reload: bool = False,
+    ):
+        return self._get_config(force_reload).getfloat(
+            section, key, fallback=default_val
+        )
+
+    def get_int(
+        self,
+        section: str,
+        key: str,
+        default_val: int = 0,
+        force_reload: bool = False,
+    ):
+        return self._get_config(force_reload).getint(
+            section, key, fallback=default_val
+        )

miza_datahub-1.0.0/src/miza_datahub/common/singleton.py

@@ -0,0 +1,11 @@
+from typing import Any, Dict
+
+
+class Singleton(type):
+    _instances: Dict[type, Any] = {}
+
+    def __call__(c, *args, **kwargs):
+        if c not in c._instances:
+            c._instances[c] = super().__call__(*args, **kwargs)
+
+        return c._instances[c]

miza_datahub-1.0.0/src/miza_datahub/influxdb/influx_repository.py

@@ -0,0 +1,15 @@
+class InfluxRepository:
+    def __init__(self, client):
+        self.client = client
+
+    def query(self, sql):
+        return self.client.query(sql)
+
+    def write(self, line_protocol):
+        return self.client.write(line_protocol)
+
+    @staticmethod
+    def escape_string(string):
+        return string.translate(
+            string.maketrans({",": r"\,", " ": r"\ ", "=": r"\="})
+        )

miza_datahub-1.0.0/src/miza_datahub/influxdb/influx_rest_client.py

@@ -0,0 +1,26 @@
+import requests
+
+
+class InfluxRestClient:
+    def __init__(self, host: str, port: int, database: str):
+        self.database = database
+        self.query_url = f"http://{host}:{port}/query"
+        self.write_url = f"http://{host}:{port}/write"
+
+    def query(self, sql: str):
+        response = requests.get(
+            self.query_url, params={"db": self.database, "q": sql}
+        )
+
+        response.raise_for_status()
+        return response.json()
+
+    def write(self, payload: str):
+        response = requests.post(
+            self.write_url,
+            params={"db": self.database, "precision": "s"},
+            data=payload,
+        )
+
+        response.raise_for_status()
+        return response.text

miza_datahub-1.0.0/src/miza_datahub/influxdb/queries/paper_machine_query.py

@@ -0,0 +1,110 @@
+from miza_datahub.influxdb.influx_repository import InfluxRepository
+
+
+class PaperMachineRepository(InfluxRepository):
+    def get_actual_production(self, start_time: str, end_time: str) -> float:
+        query = f"""
+        SELECT integral("ton_per_min",1m)
+        FROM (
+            SELECT
+            (
+                mean("reel_speed")
+                * 4.8
+                * mean("basic_weight")
+                * 0.9
+            ) / 1000000 AS ton_per_min
+            FROM miza_realtime
+            WHERE
+                line='Máy giấy PM3'
+                AND machine='Scanner'
+                AND time >= '{start_time}'
+                AND time <= '{end_time}'
+            GROUP BY time(1m)
+        )
+        """
+
+        result = self.query(query)
+        print(result)
+        return result["results"][0]["series"][0]["values"][0][1]
+
+    def get_plan_production(self, start_time: str, end_time: str) -> float:
+        query = f"""
+        SELECT integral("ton_per_min",1m)
+        FROM (
+            SELECT
+            (
+                mean("nominal_speed")
+                * 4.8
+                * mean("nominal_basic_weight")
+                * mean("nominal_efficiency")
+            ) / 100000000 AS ton_per_min
+            FROM miza_realtime
+            WHERE
+                line='Máy giấy PM3'
+                AND machine='Scanner'
+                AND time >= '{start_time}'
+                AND time <= '{end_time}'
+            GROUP BY time(1m)
+        )
+        """
+        result = self.query(query)
+        return result["results"][0]["series"][0]["values"][0][1]
+
+    def get_actual_production_daily(self, start_time: str, end_time: str):
+        query = f"""
+        SELECT integral("ton_per_min",1m)
+        FROM (
+            SELECT
+            (
+                mean("reel_speed")
+                * 4.8
+                * mean("basic_weight")
+                * 0.9
+            ) / 1000000 AS ton_per_min
+            FROM miza_realtime
+            WHERE
+                line='Máy giấy PM3'
+                AND machine='Scanner'
+                AND time >= '{start_time}'
+                AND time <= '{end_time}'
+            GROUP BY time(1m)
+        )
+        GROUP BY time(1d, 6h)
+        fill(none)
+        TZ('Asia/Ho_Chi_Minh')
+        """
+
+        result = self.query(query)
+        # {'results': [{'statement_id': 0}]}
+        if "series" in result["results"][0]:
+            return result["results"][0]["series"][0]["values"]
+        return []
+
+    def get_plan_production_daily(self, start_time: str, end_time: str):
+        query = f"""
+        SELECT integral("ton_per_min",1m)
+        FROM (
+            SELECT
+            (
+                mean("nominal_speed")
+                * 4.8
+                * mean("nominal_basic_weight")
+                * mean("nominal_efficiency")
+            ) / 100000000 AS ton_per_min
+            FROM miza_realtime
+            WHERE
+                line='Máy giấy PM3'
+                AND machine='Scanner'
+                AND time >= '{start_time}'
+                AND time <= '{end_time}'
+            GROUP BY time(1m)
+        )
+        GROUP BY time(1d, 6h)
+        fill(none)
+        TZ('Asia/Ho_Chi_Minh')
+        """
+
+        result = self.query(query)
+        if "series" in result["results"][0]:
+            return result["results"][0]["series"][0]["values"]
+        return []

miza_datahub-1.0.0/src/miza_datahub/influxdb/queries/paper_oee_query.py

@@ -0,0 +1,97 @@
+from calendar import monthrange
+
+from miza_datahub.influxdb.influx_repository import InfluxRepository
+
+
+class PaperOEEQuery(InfluxRepository):
+    def get_daily_availability(self, date: str):
+        query = f"""
+        SELECT mean("A") FROM (
+          SELECT mean("b_status") * 100 as "A"
+          FROM "miza_realtime"
+          WHERE
+            ("machine"::tag = 'Động cơ D31')
+            AND time >= '{date}T06:00:00+07:00'
+            AND time <= '{date}T06:00:00+07:00' + 1d
+          GROUP BY time(1m) fill(null)
+        )
+        GROUP BY time(1d, 6h)
+        fill(none)
+        TZ('Asia/Ho_Chi_Minh')
+        """
+        result = self.query(query)
+        return result["results"][0]["series"][0]["values"]
+
+    def get_daily_performance_quality(self, date: str):
+        query = f"""
+        SELECT mean("P"), mean("Q")
+        FROM "paper_daily_oee"
+        WHERE
+            time >= '{date}T06:00:00+07:00'
+            AND time <= '{date}T06:00:00+07:00' + 1d
+        GROUP BY time(1m) fill(none)
+        """
+        result = self.query(query)
+        return result["results"][0]["series"][0]["values"]
+
+    def get_monthly_availability(self, year: int, month: int):
+        last_day = monthrange(year, month)[1]
+        query = f"""
+        SELECT mean("A") FROM (
+          SELECT mean("b_status") * 100 as "A"
+          FROM "miza_realtime"
+          WHERE
+            ("machine"::tag = 'Động cơ D31')
+            AND time >= '{year}-{month:02d}-01T06:00:00+07:00'
+            AND time <= '{year}-{month:02d}-{last_day}T06:00:00+07:00' + 1d
+          GROUP BY time(1m) fill(null)
+        )
+        GROUP BY time(4w, 6h)
+        fill(none)
+        TZ('Asia/Ho_Chi_Minh')
+        """
+        result = self.query(query)
+        return result["results"][0]["series"][0]["values"]
+
+    def get_monthly_performance_quality(self, year: int, month: int):
+        last_day = monthrange(year, month)[1]
+        query = f"""
+        SELECT mean("P"), mean("Q")
+        FROM "paper_daily_oee"
+        WHERE
+            time >= '{year}-{month:02d}-01T06:00:00+07:00'
+            AND time <= '{year}-{month:02d}-{last_day}T06:00:00+07:00' + 1d
+        TZ('Asia/Ho_Chi_Minh')
+        """
+        result = self.query(query)
+        return result["results"][0]["series"][0]["values"]
+
+    def get_yearly_availability(self, year: int):
+        query = f"""
+        SELECT mean("A") FROM (
+          SELECT mean("b_status") * 100 as "A"
+          FROM "miza_realtime"
+          WHERE
+            ("machine"::tag = 'Động cơ D31')
+            AND time >= '{year}-01-01T06:00:00+07:00'
+            AND time <= '{year+1}-01-01T06:00:00+07:00'
+          GROUP BY time(1m) fill(null)
+        )
+        GROUP BY time(52w, 6h)
+        fill(none)
+        TZ('Asia/Ho_Chi_Minh')
+        """
+        result = self.query(query)
+        return result["results"][0]["series"][0]["values"]
+
+    def get_yearly_performance_quality(self, year: int):
+        query = f"""
+        SELECT mean("P"), mean("Q")
+        FROM "paper_daily_oee"
+        WHERE
+            time >= '{year}-01-01T06:00:00+07:00'
+            AND time <= '{year+1}-01-01T06:00:00+07:00'
+        TZ('Asia/Ho_Chi_Minh')
+        """
+        result = self.query(query)
+        return result["results"][0]["series"][0]["values"]

miza_datahub-1.0.0/src/miza_datahub/influxdb/queries/production_time_query.py

@@ -0,0 +1,51 @@
+from miza_datahub.influxdb.influx_repository import InfluxRepository
+
+
+class ProductionTimeQuery(InfluxRepository):
+    def get_production_run_time(self, date: str, group_by: str):
+        query = f"""
+        SELECT sum("cnt")
+        FROM (
+            SELECT
+                last("b_run_2") / 2 AS "cnt"
+            FROM "miza_realtime"
+            WHERE
+                "machine" = 'Động cơ D31'
+                AND time >= '{date}T06:00:00+07:00'
+                AND time <= '{date}T06:00:00+07:00' + 1d
+            GROUP BY time(1m)
+            fill(0)
+        )
+        GROUP BY time({group_by}, 6h)
+        fill(none)
+        TZ('Asia/Ho_Chi_Minh')
+        """
+        result = self.query(query)
+        return result["results"][0]["series"][0]["values"]
+
+    def get_production_down_time(self, date: str, group_by: str):
+        product_time = ""
+        if group_by == "1d":
+            product_time = "1440"
+        elif group_by == "1h":
+            product_time = "60"
+
+        query = f"""
+        SELECT {product_time} - sum("cnt")
+        FROM (
+            SELECT
+                last("b_run_2") / 2 AS "cnt"
+            FROM "miza_realtime"
+            WHERE
+                "machine" = 'Động cơ D31'
+                AND time >= '{date}T06:00:00+07:00'
+                AND time <= '{date}T06:00:00+07:00' + 1d
+            GROUP BY time(1m)
+            fill(0)
+        )
+        GROUP BY time({group_by}, 6h)
+        fill(none)
+        TZ('Asia/Ho_Chi_Minh')
+        """
+        result = self.query(query)
+        return result["results"][0]["series"]["values"]

miza_datahub-1.0.0/src/miza_datahub/influxdb/writers/paper_daily_oee_writer.py

@@ -0,0 +1,80 @@
+import pandas as pd
+
+from miza_datahub.influxdb.influx_repository import InfluxRepository
+
+
+class PaperDailyOEEWriter(InfluxRepository):
+    MEASUREMENT = "paper_daily_oee"
+
+    def write_apq(self, df):
+        lines = []
+        mapping = {
+            "Sản lượng": "actual",
+            "plan": "plan",
+            "Hàng lỗi": "defect",
+            "run_time": "run_time",
+            "ĐG + DM": "down_time",
+            "% Hiệu suất": "A",
+            "% SL": "P",
+            "% Chất lượng": "Q",
+            "OEE": "OEE",
+        }
+
+        for _, row in df.iterrows():
+            timestamp = int(row["production_day"].timestamp())
+
+            tags = [
+                f"factory={PaperDailyOEEWriter.escape_string('MIZA Nghi Sơn')}",
+                "line=PM3",
+                "machine=Scanner",
+            ]
+
+            values = [
+                f"{influx_field}={row[df_field]}"
+                for df_field, influx_field in mapping.items()
+                if pd.notna(row[df_field])
+            ]
+
+            line = (
+                f'{self.MEASUREMENT},{",".join(tags)} '
+                f'{",".join(values)} '
+                f"{timestamp}"
+            )
+
+            lines.append(line)
+
+        self.client.write("\n".join(lines))
+
+    def write_pq(self, df):
+        lines = []
+        mapping = {
+            "actual": "actual",
+            "plan": "plan",
+            "B": "defect",
+            "P": "P",
+            "Q": "Q",
+        }
+
+        for _, row in df.iterrows():
+            timestamp = int(row["production_day"].timestamp())
+
+            tags = [
+                f"factory={PaperDailyOEEWriter.escape_string('MIZA Nghi Sơn')}",
+                "line=PM3",
+                "machine=Scanner",
+            ]
+
+            values = [
+                f"{influx_field}={row[df_field]}"
+                for df_field, influx_field in mapping.items()
+                if pd.notna(row[df_field])
+            ]
+
+            line = (
+                f'{self.MEASUREMENT},{",".join(tags)} '
+                f'{",".join(values)} '
+                f"{timestamp}"
+            )
+            lines.append(line)
+
+        self.client.write("\n".join(lines))

miza_datahub-1.0.0/src/miza_datahub/readers/base_reader.py

@@ -0,0 +1,72 @@
+from abc import ABC, abstractmethod
+
+import requests
+
+from miza_datahub.common.config_util import ConfigUtil
+from miza_datahub.common import config_const as ConfigConst
+from miza_datahub.influxdb.influx_rest_client import InfluxRestClient
+
+
+class BaseReader(ABC):
+    def __init__(self, file_id=None, file_path=None):
+        config_util = ConfigUtil()
+
+        # Telegram
+        if file_id:
+            token = config_util.get_property(
+                section=ConfigConst.TELEGRAM, key=ConfigConst.TOKEN
+            )
+
+            r = requests.get(
+                ConfigConst.URL_GET_FILE_PATH.format(token=token),
+                params={"file_id": file_id},
+            ).json()
+            if not r.get("ok"):
+                raise ValueError(f"Telegram error: {r.get('description')}")
+
+            file_path = r["result"]["file_path"]
+
+            self.file_bytes = requests.get(
+                ConfigConst.URL_GET_FILE_CONTENT.format(
+                    token=token, file_path=file_path
+                )
+            ).content
+
+        # file local
+        elif file_path:
+            with open(file_path, "rb") as f:
+                self.file_bytes = f.read()
+
+        # InfluxDB
+        influx_host = config_util.get_property(
+            section=ConfigConst.INFLUX,
+            key=ConfigConst.INFLUX_HOST,
+            default_val="192.168.10.2",
+        )
+        influx_port = config_util.get_int(
+            section=ConfigConst.INFLUX,
+            key=ConfigConst.INFLUX_PORT,
+            default_val=8090,
+        )
+        influx_db = config_util.get_property(
+            section=ConfigConst.INFLUX,
+            key=ConfigConst.INFLUX_DB,
+            default_val="miza_new",
+        )
+        self.influx = InfluxRestClient(influx_host, influx_port, influx_db)
+
+    @abstractmethod
+    def load(self):
+        pass
+
+    @abstractmethod
+    def write(self):
+        pass
+
+
+class TempReader(BaseReader):
+    def load(self):
+        pass
+
+    def write(self):
+        pass

miza_datahub-1.0.0/src/miza_datahub/readers/oee_analyzer.py

@@ -0,0 +1,71 @@
+from io import BytesIO
+
+import numpy as np
+import pandas as pd
+
+from miza_datahub.readers.base_reader import BaseReader
+from miza_datahub.influxdb.writers.paper_daily_oee_writer import (
+    PaperDailyOEEWriter,
+)
+
+
+class OEEAnalyzer(BaseReader):
+    def load(self):
+        excel_file = pd.ExcelFile(BytesIO(self.file_bytes))
+        df_raw = pd.read_excel(
+            BytesIO(self.file_bytes), sheet_name="OEE", header=None
+        )
+
+        sheet_name = excel_file.sheet_names[0]
+        month_str, year_str = sheet_name.split(".")
+        month = int(month_str)
+        year = 2000 + int(year_str)
+
+        time_markers = df_raw.iloc[0, 2:].tolist()
+
+        timeline = []
+        for marker in time_markers:
+            current_day = int(marker)
+            full_timestamp = pd.to_datetime(
+                f"{current_day}/{month}/{year} 06:00", dayfirst=True
+            )
+            timeline.append(full_timestamp)
+
+        df_matrix = df_raw.iloc[1:10, 2:]
+        error_list = ["#DIV/0!", "#ERROR!", "#VALUE!", "#REF!", "#NAME?"]
+        df_matrix = df_matrix.replace(error_list, np.nan)
+
+        row_8_data = df_matrix.iloc[8]
+        clean_columns_condition = row_8_data.notna()
+
+        df_clean = df_matrix.loc[:, clean_columns_condition]
+        df_final = df_clean.T
+
+        df_final.index = timeline[: len(df_final.index)]
+        df_final.columns = [
+            "Sản lượng",
+            "Hàng lỗi",
+            "% Hàng lỗi",
+            "ĐG + DM",
+            "Rỗng",
+            "% SL",
+            "% Chất lượng",
+            "% Hiệu suất",
+            "OEE",
+        ]
+        df_final["plan"] = np.where(
+            df_final["% SL"] > 0,
+            df_final["Sản lượng"] / df_final["% SL"],
+            np.nan,
+        )
+        df_final["run_time"] = 1440 - df_final["ĐG + DM"]
+        df_final = df_final.reset_index().rename(
+            columns={"index": "production_day"}
+        )
+        self.df = df_final
+
+    def write(self):
+        self.load()
+        paper_oee = PaperDailyOEEWriter(self.influx)
+        paper_oee.write_apq(self.df)
+        return True

miza_datahub-1.0.0/src/miza_datahub/readers/quality_analyzer.py

@@ -0,0 +1,140 @@
+from io import BytesIO
+
+# import numpy as np
+import pandas as pd
+
+from miza_datahub.readers.base_reader import BaseReader
+from miza_datahub.influxdb.queries.paper_machine_query import (
+    PaperMachineRepository,
+)
+from miza_datahub.influxdb.writers.paper_daily_oee_writer import (
+    PaperDailyOEEWriter,
+)
+from miza_datahub.services.oee_service import OEEService
+
+
+class QualityAnalyzer(BaseReader):
+    def load(self):
+        df = pd.read_excel(
+            BytesIO(self.file_bytes),
+            sheet_name="Chi_Tiet_Chat_Luong",
+            skiprows=5,
+            usecols="C:F,H:O,AA:AF,AO:AZ,BB:BF",
+            names=[
+                "Ngày SX",
+                "Loại Giấy",
+                "Khách hàng",
+                "Trưởng ca",
+                "Mã cuộn",
+                "Ngày cắt cuộn",
+                "Giờ ra quả Xeo",
+                "Khổ giấy (cm)",
+                "Tổng khổ (cm)",
+                "Định lượng YC (g/m2)",
+                "Định lượng TB (g/m2)",
+                "Định lượng TT (g/m2)",
+                "Số vết nối TT",
+                "Số vết nối YC",
+                "Độ bục TB",
+                "Cường độ bục TT",
+                "Độ bục YC",
+                "Cường độ bục YC",
+                "Độ nhẵn (s)",
+                "Chống thấm YC",
+                "Chiều dài cuộn (m)",
+                "Chiều dài cuộn theo CT",
+                "Đánh giá chiều dài cuộn",
+                "Trọng lượng (kg)",
+                "Người đóng gói",
+                "Nhân viên QC",
+                "Độ dương khổ (mm)",
+                "Đường kính quả",
+                "Ghi Chú",
+                "Mã quả Xeo",
+                "Loại CL",
+                "Chỉ số màu L*",
+                "Chỉ số màu a*",
+                "Chỉ số màu b*",
+                "Loại SP",
+            ],
+        )
+
+        # 1. Chuẩn hóa giờ: đổi ; thành :
+        df["Giờ ra quả Xeo"] = (
+            df["Giờ ra quả Xeo"].astype(str).str.replace(";", ":", regex=False)
+        )
+
+        # 2. Loại bỏ NaN / NaT / chuỗi lỗi
+        df = df[df["Giờ ra quả Xeo"].notna()]
+        df = df[df["Giờ ra quả Xeo"].str.lower() != "nat"]
+
+        # 3. Tạo datetime
+        df["Thời điểm ra quả xeo"] = pd.to_datetime(
+            df["Ngày SX"].astype(str) + " " + df["Giờ ra quả Xeo"],
+            errors="coerce",
+        )
+
+        # 4. Loại bỏ dòng parse lỗi lần cuối
+        df = df.dropna(subset=["Thời điểm ra quả xeo"])
+
+        df = df.set_index(["Thời điểm ra quả xeo", "Mã cuộn", "Ngày cắt cuộn"])
+        # df["Mã quả Xeo"] = df["Mã quả Xeo"].astype(np.uint32)
+        self.df = df
+
+    def write(self):
+        self.load()
+        df_quality = self.calculate_defect_weight()
+
+        paper = PaperMachineRepository(self.influx)
+        paper_oee = PaperDailyOEEWriter(self.influx)
+
+        actual = paper.get_actual_production_daily(
+            start_time=(df_quality["production_day"].min()).strftime(
+                "%Y-%m-%dT00:00:00Z"
+            ),
+            end_time=(
+                df_quality["production_day"].max() + pd.Timedelta(days=1)
+            ).strftime("%Y-%m-%dT00:00:00Z"),
+        )
+
+        plan = paper.get_plan_production_daily(
+            start_time=(df_quality["production_day"].min()).strftime(
+                "%Y-%m-%dT00:00:00Z"
+            ),
+            end_time=(
+                df_quality["production_day"].max() + pd.Timedelta(days=1)
+            ).strftime("%Y-%m-%dT00:00:00Z"),
+        )
+        df_final = OEEService.calculate_pq(actual, plan, df_quality)
+        paper_oee.write_pq(df_final)
+        return True
+
+    def calculate_defect_weight(self):
+        time_idx = self.df.index.get_level_values("Thời điểm ra quả xeo")
+        quality_df = (
+            self.df.assign(
+                production_day=(time_idx - pd.Timedelta(hours=6)).floor("D")
+            )
+            .groupby(["production_day", "Loại CL"])["Trọng lượng (kg)"]
+            .sum()
+            .unstack(fill_value=0)
+        )
+
+        quality_df = quality_df.reset_index()
+        # quality_df["scrap_kg"] = quality_df.get("B", 0)
+        # quality_df["good_kg"] = quality_df.get("A", 0)
+        # print(quality_df)
+        return quality_df
+
+    def summarize_columns(self):
+        print(f"{'Column':35} {'Type':15} {'Unique':>10} {'Memory(MB)':>12}")
+        print("-" * 75)
+
+        for c in self.df.columns:
+            print(
+                f"{c:35} "
+                f"{str(self.df[c].dtype):15} "
+                f"{self.df[c].nunique(dropna=False):>10,} "
+                f"{self.df[c].memory_usage(deep=True)/1024**2:>12.2f}"
+            )
+        self.df.info(memory_usage="deep")

miza_datahub-1.0.0/src/miza_datahub/services/oee_service.py

@@ -0,0 +1,44 @@
+import pandas as pd
+import numpy as np
+
+
+class OEEService:
+    @staticmethod
+    def production_dataframe(data: list, value_column: str) -> pd.DataFrame:
+        if not data:
+            return pd.DataFrame(columns=["production_day", value_column])
+        df = pd.DataFrame(data, columns=["time", value_column])
+        df["production_day"] = (
+            pd.to_datetime(df["time"]).dt.tz_localize(None).dt.normalize()
+        )
+        return df[["production_day", value_column]]
+
+    @staticmethod
+    def calculate_pq(actual, plan, df_quality):
+        df_actual = OEEService.production_dataframe(actual, "actual")
+        df_plan = OEEService.production_dataframe(plan, "plan")
+
+        if df_quality is not None:
+            df_quality = pd.DataFrame(columns=["production_day", "B"])
+
+        df_final = df_actual.merge(
+            df_quality[["production_day", "B"]],
+            on="production_day",
+            how="outer",
+        ).merge(df_plan, on="production_day", how="left")
+
+        if df_final.empty:
+            return pd.DataFrame(
+                columns=["production_day", "actual", "plan", "B", "P", "Q"]
+            )
+
+        df_final["P"] = (df_final["actual"] / df_final["plan"] * 100).round(2)
+        df_final["Q"] = (
+            (df_final["actual"] - df_final["B"] / 1000)
+            / df_final["actual"]
+            * 100
+        ).round(2)
+        df_final = df_final.replace([np.inf, -np.inf], np.nan)
+        df_final = df_final.astype(object)
+        df_final = df_final.where(pd.notnull(df_final), None)
+        return df_final

miza_datahub-1.0.0/src/miza_datahub/services/reader_factory.py

@@ -0,0 +1,36 @@
+from io import BytesIO
+
+import pandas as pd
+
+from miza_datahub.readers.base_reader import TempReader
+from miza_datahub.readers.oee_analyzer import OEEAnalyzer
+from miza_datahub.readers.quality_analyzer import (
+    QualityAnalyzer,
+)
+
+
+def detect_file_types(file_bytes):
+    excel = pd.ExcelFile(BytesIO(file_bytes))
+    sheets = excel.sheet_names
+
+    if "OEE" in sheets:
+        return "oee"
+
+    if "Chi_Tiet_Chat_Luong" in sheets:
+        return "quality"
+
+    return ValueError(f"Don't read file. Sheets={sheets}")
+
+
+class ReaderFactory:
+    @staticmethod
+    def create(file_id=None, file_path=None):
+        base = TempReader(file_id=file_id, file_path=file_path)
+
+        file_type = detect_file_types(base.file_bytes)
+
+        if file_type == "oee":
+            return OEEAnalyzer(file_id=file_id, file_path=file_path)
+
+        if file_type == "quality":
+            return QualityAnalyzer(file_id=file_id, file_path=file_path)

miza_datahub-1.0.0/src/miza_datahub.egg-info/PKG-INFO

@@ -0,0 +1,42 @@
+Metadata-Version: 2.4
+Name: miza-datahub
+Version: 1.0.0
+Summary: This package does amazing things.
+Home-page: http://github.com/chanelcolgate/miza-datahub
+Author: chanelcolgate
+Author-email: "chanelcolgate" <github@chanelcolgate.com>
+License: MIT
+Classifier: License :: OSI Approved :: MIT License
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: requests
+Requires-Dist: openpyxl
+Dynamic: license-file
+
+## Miza DataHub Library
+### How to install
+- Cài pip
+```bash
+pip install --upgrade pip setuptools wheel
+```
+- Cách khác
+```bash
+python -m ensurepip --upgrade
+```
+- Cách cài đặt và chạy
+```
+pyproject-build
+python -m pip install -e .
+```
+- Cách test
+```
+python -m pip install pytest
+python -m pytest
+python -m pip install mypy
+python -m mypy
+python -m pip install tox
+tox -e typecheck 
+tox -e lint
+```

miza_datahub-1.0.0/src/miza_datahub.egg-info/SOURCES.txt

@@ -0,0 +1,56 @@
+LICENSE
+MANIFEST.in
+README.md
+pyproject.toml
+setup.cfg
+src/logic.egg-info/PKG-INFO
+src/logic.egg-info/SOURCES.txt
+src/logic.egg-info/dependency_links.txt
+src/logic.egg-info/top_level.txt
+src/miza_datahub/__init__.py
+src/miza_datahub.egg-info/PKG-INFO
+src/miza_datahub.egg-info/SOURCES.txt
+src/miza_datahub.egg-info/dependency_links.txt
+src/miza_datahub.egg-info/requires.txt
+src/miza_datahub.egg-info/top_level.txt
+src/miza_datahub/__pycache__/__init__.cpython-311.pyc
+src/miza_datahub/common/__init__.py
+src/miza_datahub/common/config_const.py
+src/miza_datahub/common/config_util.py
+src/miza_datahub/common/singleton.py
+src/miza_datahub/common/__pycache__/__init__.cpython-311.pyc
+src/miza_datahub/common/__pycache__/config_const.cpython-311.pyc
+src/miza_datahub/common/__pycache__/config_util.cpython-311.pyc
+src/miza_datahub/common/__pycache__/singleton.cpython-311.pyc
+src/miza_datahub/influxdb/__init__.py
+src/miza_datahub/influxdb/influx_repository.py
+src/miza_datahub/influxdb/influx_rest_client.py
+src/miza_datahub/influxdb/__pycache__/__init__.cpython-311.pyc
+src/miza_datahub/influxdb/__pycache__/influx_repository.cpython-311.pyc
+src/miza_datahub/influxdb/__pycache__/influx_rest_client.cpython-311.pyc
+src/miza_datahub/influxdb/queries/__init__.py
+src/miza_datahub/influxdb/queries/paper_machine_query.py
+src/miza_datahub/influxdb/queries/paper_oee_query.py
+src/miza_datahub/influxdb/queries/production_time_query.py
+src/miza_datahub/influxdb/queries/__pycache__/__init__.cpython-311.pyc
+src/miza_datahub/influxdb/queries/__pycache__/paper_machine_query.cpython-311.pyc
+src/miza_datahub/influxdb/queries/__pycache__/paper_oee_query.cpython-311.pyc
+src/miza_datahub/influxdb/writers/__init__.py
+src/miza_datahub/influxdb/writers/paper_daily_oee_writer.py
+src/miza_datahub/influxdb/writers/__pycache__/__init__.cpython-311.pyc
+src/miza_datahub/influxdb/writers/__pycache__/paper_daily_oee_writer.cpython-311.pyc
+src/miza_datahub/readers/__init__.py
+src/miza_datahub/readers/base_reader.py
+src/miza_datahub/readers/oee_analyzer.py
+src/miza_datahub/readers/quality_analyzer.py
+src/miza_datahub/readers/__pycache__/__init__.cpython-311.pyc
+src/miza_datahub/readers/__pycache__/base_reader.cpython-311.pyc
+src/miza_datahub/readers/__pycache__/finished_product_quality_analyzer.cpython-311.pyc
+src/miza_datahub/readers/__pycache__/oee_analyzer.cpython-311.pyc
+src/miza_datahub/readers/__pycache__/quality_analyzer.cpython-311.pyc
+src/miza_datahub/services/__init__.py
+src/miza_datahub/services/oee_service.py
+src/miza_datahub/services/reader_factory.py
+src/miza_datahub/services/__pycache__/__init__.cpython-311.pyc
+src/miza_datahub/services/__pycache__/oee_service.cpython-311.pyc
+src/miza_datahub/services/__pycache__/reader_factory.cpython-311.pyc

miza_datahub-1.0.0/src/miza_datahub.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

miza_datahub-1.0.0/src/miza_datahub.egg-info/requires.txt

@@ -0,0 +1,4 @@
+numpy
+pandas
+requests
+openpyxl

miza_datahub-1.0.0/src/miza_datahub.egg-info/top_level.txt

@@ -0,0 +1 @@
+miza_datahub