sciveo 0.1.25__tar.gz → 0.1.27__tar.gz

{sciveo-0.1.25 → sciveo-0.1.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.25
+Version: 0.1.27
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

{sciveo-0.1.25/sciveo/media/ml → sciveo-0.1.27/sciveo}/__init__.py

@@ -20,8 +20,6 @@ try:
   from sciveo.tools.daemon import TasksDaemon, __upload_content__
   from sciveo.content.runner import ProjectRunner
   from sciveo.content.dataset import Dataset
-  from sciveo.monitoring.start import MonitorStart
-  from sciveo.network.tools import NetworkTools
   from sciveo.version import __version__
 
 
@@ -46,10 +44,12 @@ try:
 
   # Monitoring start
   def monitor(**kwargs):
+    from sciveo.monitoring.start import MonitorStart
     MonitorStart(**kwargs)()
 
   # Network tools
   def network(**kwargs):
+    from sciveo.network.tools import NetworkTools
     return NetworkTools(**kwargs)
 
 except ImportError as e:

{sciveo-0.1.25 → sciveo-0.1.27}/sciveo/cli.py

@@ -14,9 +14,6 @@ import os
 import argparse
 
 from sciveo.tools.logger import *
-from sciveo.tools.timers import Timer
-from sciveo.monitoring.start import MonitorStart
-from sciveo.network.tools import NetworkTools
 from sciveo.tools.configuration import GlobalConfiguration
 
 
@@ -35,8 +32,10 @@ def main():
   args = parser.parse_args()
 
   if args.command == 'monitor':
+    from sciveo.monitoring.start import MonitorStart
     MonitorStart(period=args.period, block=args.block)()
   elif args.command == 'scan':
+    from sciveo.network.tools import NetworkTools
     NetworkTools(timeout=args.timeout, localhost=args.localhost).scan_port(port=args.port, network=args.net)
   elif args.command == 'init':
     home = os.path.expanduser('~')

sciveo-0.1.27/sciveo/media/pipelines/processors/sci/time_series/predictor.py

@@ -0,0 +1,108 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import os
+import boto3
+
+from sciveo.tools.logger import *
+from sciveo.media.pipelines.processors.sci.base import *
+from sciveo.media.ml.time_series.predictor import *
+
+
+class TimeSeriesPredictorProcessor(SciBaseProcessor):
+  def __init__(self, processor_config, max_progress) -> None:
+    super().__init__(processor_config, max_progress)
+
+    self.default.update({
+      "time_column": "Date Time",
+      "time_format": '%d.%m.%Y %H:%M:%S'
+    })
+
+  def run(self, job, input):
+    progress_per_media = self.max_progress / max(1, len(input))
+    debug("run", job["id"], progress_per_media, "input", input)
+    next_media = []
+    try:
+      if job['content_type'] == "project":
+        project_path = os.path.join(self.base_tmp_path, "projects", job['content_id'])
+        mkdirs(project_path)
+
+        self.download_content(job, project_path)
+        MediaJobState.queue().inc_progress(job["id"], 10)
+
+        list_content = {}
+
+        dataset_df = []
+        for content in job['content']:
+          list_content.setdefault(content["content_type"], []).append(content)
+
+          if content["content_type"] == "file":
+            if content['key'].split(".")[-1] == "timeseries":
+              list_content.setdefault("timeseries", []).append(content)
+
+          if content["content_type"] == "dataset":
+            # Datasets with S3 located files
+            if "bucket" in content and "key" in content:
+              remote_path, local_path, local_file_name = self.content_path(content, project_path)
+              file_extension = local_file_name.split(".")[-1]
+
+              if file_extension == "csv":
+                df = pd.read_csv(local_path)
+                x_col = content['data'].get("dataset", {}).get("x_col", self["time_column"])
+                df.set_index(x_col, inplace=True)
+                dataset_df.append(df)
+
+        dataset_df = pd.concat(dataset_df).drop_duplicates(keep='first').sort_index().reset_index()
+
+        content_project = list_content["project"][0]
+
+        MediaJobState.queue().inc_progress(job["id"], 5)
+
+        if "timeseries" in list_content:
+          content_predictor = self.new_content(content_project, "prediction", name=f"Prediction on {dataset_df.shape}")
+          next_media.append(content_predictor)
+
+          text_content = self.content_text(content_predictor, "timeseries predictions plots", f"""Time series predictions.\n\n{self.describe_df(dataset_df)}""")
+          next_media.append(text_content)
+
+          content = list_content["timeseries"][0]
+          remote_path, local_path, local_file_name = self.content_path(content, project_path)
+          predictor = TimeSeriesPredictor(local_path)
+          ds = TimeSeriesDataSet(dataset_df, self["time_column"], format=self["time_format"])
+          predictions, X, x_plot = predictor.predict(ds.data)
+
+          MediaJobState.queue().inc_progress(job["id"], 5)
+
+          columns = predictor.model_data["columns"][:11] # TODO: Should be configurable in the Predictor, so plot only few columns
+          plot_progress_inc = 20 / len(columns)
+          for i, y_col in enumerate(columns):
+            image_content = self.content_image(text_content, y_col, project_path)
+            predictor.plot(predictions, X, x_plot, i, image_local_path=image_content["local_path"])
+            image_content["data"] = {
+              "info": {
+                "size": os.path.getsize(image_content["local_path"]),
+                "plot": y_col
+              }
+            }
+            next_media.append(image_content)
+            MediaJobState.queue().inc_progress(job["id"], plot_progress_inc)
+      MediaJobState.queue().inc_progress(job["id"], 10)
+
+      if self["output"]:
+        job["output"] += next_media
+      if self["append-content"]:
+        job["append-content"] += next_media
+    except Exception as e:
+      exception(e)
+    return next_media
+
+  def name(self):
+    return "sci-timeseries-predictor"

sciveo-0.1.27/sciveo/media/pipelines/processors/sci/time_series/trainer.py

@@ -0,0 +1,185 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import os
+import pandas as pd
+import tensorflow as tf
+from tensorflow.keras.callbacks import Callback
+
+from sciveo.tools.logger import *
+from sciveo.media.pipelines.processors.sci.base import *
+from sciveo.media.ml.time_series.predictor import *
+from sciveo.media.ml.time_series.trainer import *
+
+
+class TrainerProgressCallback(Callback):
+  def __init__(self, job, epochs, max_progress):
+    self.job = job
+    self.progress_per_epoch = 2 * max_progress / epochs
+
+  def on_epoch_end(self, epoch, logs=None):
+    debug("epoch", epoch, "finished")
+    MediaJobState.queue().inc_progress(self.job["id"], self.progress_per_epoch)
+
+
+class TimeSeriesTrainerProcessor(SciBaseProcessor):
+  def __init__(self, processor_config, max_progress) -> None:
+    super().__init__(processor_config, max_progress)
+
+    self.default.update({
+      "model_name": "none",
+      "time_column": "Date Time",
+      "time_format": '%d.%m.%Y %H:%M:%S',
+      "ratios": [["train", 0.70], ["val", 0.20], ["test", 0.10]],
+      "max_epochs": 20, "patience": 2,
+      "window_size": 24,
+      "test_plot_from": 200, "test_plots": 20, "test_plot_column_id": 1
+    })
+
+  def run(self, job, input):
+    progress_per_media = self.max_progress / max(1, len(input))
+    debug("run", job["id"], progress_per_media, "input", input)
+    next_media = []
+    try:
+      if job['content_type'] == "project":
+        project_path = os.path.join(self.base_tmp_path, "projects", job['content_id'])
+        mkdirs(project_path)
+
+        self.download_content(job, project_path)
+        MediaJobState.queue().inc_progress(job["id"], 10)
+
+        list_content = {}
+
+        dataset_df = []
+        for content in job['content']:
+          list_content.setdefault(content["content_type"], []).append(content)
+
+          if content["content_type"] == "file":
+            if content['key'].split(".")[-1] == "timeseries":
+              list_content.setdefault("timeseries", []).append(content)
+
+          if content["content_type"] == "dataset":
+            # Datasets with S3 located files
+            if "bucket" in content and "key" in content:
+              remote_path, local_path, local_file_name = self.content_path(content, project_path)
+              file_extension = local_file_name.split(".")[-1]
+
+              if file_extension == "csv":
+                df = pd.read_csv(local_path)
+                x_col = content['data'].get("dataset", {}).get("x_col", df.keys()[0])
+                df.set_index(x_col, inplace=True)
+                dataset_df.append(df)
+
+        dataset_df = pd.concat(dataset_df).drop_duplicates(keep='first').sort_index().reset_index()
+
+        content_project = list_content["project"][0]
+        MediaJobState.queue().inc_progress(job["id"], 5)
+
+        if len(dataset_df) > 0:
+          content_trainer = self.new_content(content_project, "training", name=f"Training on {dataset_df.shape}")
+          next_media.append(content_trainer)
+
+          ds = TimeSeriesDataSet(dataset_df, self["time_column"], format=self["time_format"])
+          ds.normalize()
+          ds.split(ratios=self["ratios"])
+
+          trainer = TimeSeriesTrainer(ds, self["window_size"], self["window_size"], self["window_size"])
+          trainer.create()
+          progress_callback = TrainerProgressCallback(job, self["max_epochs"], 50)
+          history = trainer.train(self["max_epochs"], self["patience"], progress_callback)
+          trainer_eval = trainer.evaluate()
+
+          model_name, model_path = trainer.save(project_path, self["model_name"])
+
+          model_data = {
+            "eval": trainer_eval,
+            "history": {
+              "loss": history.history["loss"],
+              "val_loss": history.history["val_loss"]
+            }
+          }
+
+          model_content = self.content_file(content_trainer, model_name, model_path, data=model_data)
+          next_media.append(model_content)
+
+          text = f"""
+            Time series model.
+            ....
+            ....
+            model name {model_name}
+            model size {os.path.getsize(model_path)}
+
+            eval
+            {self.df_to_html(pd.DataFrame(trainer_eval))}
+
+            train loss
+            {self.df_to_html(pd.DataFrame({"loss": history.history["loss"], "val_loss": history.history["val_loss"]}))}
+          """
+          text_content = self.content_text(content_trainer, "Train timeseries model", text)
+          next_media.append(text_content)
+
+          # TODO: Create numeric from history.history["loss"], history.history["val_loss"]
+
+          ds.denormalize("test")
+          predictor = TimeSeriesPredictor(model_path)
+
+          # Plot some test predictions
+          columns = predictor.model_data["columns"]
+          plot_progress_inc = 10 / self["test_plots"]
+          y_col = columns[self["test_plot_column_id"]]
+          for i in range(self["test_plot_from"], self["test_plot_from"] + self["test_plots"]):
+            image_content = self.content_image(text_content, y_col, project_path)
+            self.plot_chunk(ds.dataset["test"], predictor, i, self["test_plot_column_id"], image_content["local_path"])
+            image_content["data"] = {
+              "info": {
+                "size": os.path.getsize(image_content["local_path"]),
+                "plot": y_col
+              }
+            }
+            next_media.append(image_content)
+            MediaJobState.queue().inc_progress(job["id"], plot_progress_inc)
+
+      MediaJobState.queue().inc_progress(job["id"], 10)
+
+      if self["output"]:
+        job["output"] += next_media
+      if self["append-content"]:
+        job["append-content"] += next_media
+    except Exception as e:
+      exception(e)
+    return next_media
+
+  def plot_chunk(self, df, predictor, k, i, image_local_path):
+    k += 1
+    L = predictor.model_data["window"]["input_width"]
+    X = df[-(k + 1)*L:-k*L]
+
+    predictions, X, x_plot = predictor.predict(X)
+
+    idx_from = -k * L
+    idx_to = -(k - 1) * L
+    if idx_to != 0:
+      labels = df[idx_from:idx_to]
+    else:
+      labels = df[idx_from:]
+    labels = labels.values.reshape(X.shape).astype('float32')
+    labels = tf.convert_to_tensor(labels)
+
+    predictor.plot(
+        predictions, X, x_plot, i, labels,
+        image_local_path
+    )
+
+  def content_type(self):
+    return None
+
+  def name(self):
+    return "sci-timeseries-trainer"

sciveo-0.1.27/sciveo/ml/evaluation/object_detection.py

@@ -0,0 +1,33 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import math
+import numpy as np
+
+from sciveo.ml.images.object_detection import *
+
+
+"""
+
+Object Detection Evaluation
+
+"""
+
+
+class EvaluateObjectDetection:
+  def __init__(self, X, Y_true, Y_predicted):
+    self.X = X
+    self.Y_true = Y_true
+    self.Y_predicted = Y_predicted
+
+  def evaluate(self):
+    pass
+

sciveo-0.1.27/sciveo/ml/images/object_detection.py

@@ -0,0 +1,90 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import math
+import numpy as np
+
+from sciveo.ml.images.tools import *
+
+
+"""
+Object Detection Bounding Boxes (bbox) of type [x, y, w, h]
+
+If need to use [x1, y1, x2, y2] need to use the inverted convertor bbox_convert_inverted().
+
+IoU between 2 object detections: iou(bbox1, bbox2)
+
+"""
+
+# convert from [x, y, w, h] -> [x1, y1, x2, y2]
+def bbox_convert(bbox):
+  x1 = bbox[0]
+  y1 = bbox[1]
+  x2 = x1 + bbox[2]
+  y2 = y1 + bbox[3]
+  return [x1, y1, x2, y2]
+
+# convert from [x1, y1, x2, y2] -> [x, y, w, h]
+def bbox_convert_inverted(bbox):
+  x = bbox[0]
+  y = bbox[1]
+  w = bbox[2] - x
+  h = bbox[3] - y
+  return [x, y, w, h]
+
+def bbox_norm(bbox, w, h):
+  return (bbox[0] / w, bbox[1] / h, bbox[2] / w, bbox[3] / h)
+
+def bbox_denorm(bbox, w, h):
+  return (int(bbox[0] * w), int(bbox[1] * h), int(bbox[2] * w), int(bbox[3] * h))
+
+def bbox_center(bbox):
+  return (int(bbox[0] + bbox[2] / 2), int(bbox[1] + bbox[3] / 2))
+
+def bbox_area(bbox):
+  return bbox[2] * bbox[3]
+
+def iou(bbox1, bbox2):
+  x1 = max(bbox1[0], bbox2[0])
+  y1 = max(bbox1[1], bbox2[1])
+  x2 = min(bbox1[0] + bbox1[2], bbox2[0] + bbox2[2])
+  y2 = min(bbox1[1] + bbox1[3], bbox2[1] + bbox2[3])
+
+  if x1 < x2 and y1 < y2:
+    a = (x2 - x1) * (y2 - y1)
+  else:
+    a = 0
+
+  a1 = bbox_area(bbox1)
+  a2 = bbox_area(bbox2)
+  return a / (a1 + a2 - a)
+
+def bbox_distance(bbox1, bbox2):
+  return points_distance(bbox_center(bbox1), bbox_center(bbox2))
+
+
+"""
+
+Simple Draw object detectios helpers
+
+"""
+def image_shape(image):
+  return image.shape[1], image.shape[0]
+
+# Draw label bounding boxes of type [x, y, w, h], if [x1, y1, x2, y2] then set convert=False
+def draw_label_bboxes(image, bboxes, color, convert=True):
+  w, h = image_shape(image)
+  for bbox in bboxes:
+    if convert:
+      bbox = bbox_convert(bbox)
+    bbox = bbox_denorm(bbox, w, h)
+    cv2.rectangle(image, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color, 2, 1)
+  return image

sciveo-0.1.27/sciveo/ml/images/tools.py

@@ -0,0 +1,83 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import math
+import numpy as np
+
+
+def points_distance(p1, p2):
+  p1 = np.array(p1)
+  p2 = np.array(p2)
+  return np.linalg.norm(p1 - p2)
+
+def validate_point(p, w, h):
+  x = max(p[0], 0)
+  x = min(x, w)
+  y = max(p[1], 0)
+  y = min(y, h)
+  return (x, y)
+
+# Calc diameter point from a circle point and circle center
+def diameter_point(p1, p2):
+  x = 2 * p2[0] - p1[0]
+  y = 2 * p2[1] - p1[1]
+  return (x, y)
+
+# Center point for a line section (with some ratio for general purpose)
+def line_section_center_point(p1, p2, ratio=0.5):
+  x = int(p1[0] + ratio * (p2[0] - p1[0]))
+  y = int(p1[1] + ratio * (p2[1] - p1[1]))
+  return (x, y)
+
+# Line equation ax + by + c = 0 coefficients
+def line_coeff(l):
+  a = l[0][1] - l[1][1]
+  b = l[1][0] - l[0][0]
+  c = l[0][0] * l[1][1] - l[1][0] * l[0][1]
+  return a, b, c
+
+# 2 Lines intersection point (None, None) when parallel
+def lines_intersection(l1, l2):
+  a1, b1, c1 = line_coeff(l1)
+  a2, b2, c2 = line_coeff(l2)
+
+  d = (a1 * b2 - a2 * b1)
+  if d != 0.0:
+    x = (b1 * c2 - b2 * c1) / d
+    y = (c1 * a2 - c2 * a1) / d
+  else:
+    x, y = None, None
+  return x, y
+
+# Check if p is between p1 and p2 on a line
+def line_point_between(p, p1, p2):
+  return (
+        p[0] >= min(p1[0], p2[0]) and p[0] <= max(p1[0], p2[0])
+    and p[1] >= min(p1[1], p2[1]) and p[1] <= max(p1[1], p2[1])
+  )
+
+# Check a point p is intersection for 2 line segments l1 and l2
+def is_point_segments_intersection(p, l1, l2):
+  return line_point_between(p, l1[0], l1[1]) and line_point_between(p, l2[0], l2[1])
+
+# Angle from radians to degrees convertor
+def angle_rad2deg(angle):
+  return math.fabs(angle * 180 / math.pi)
+
+# Angle between lines in radians
+def lines_angle(l1, l2):
+  v1 = (l1[0][0] - l1[1][0], l1[0][1] - l1[1][1])
+  v2 = (l2[0][0] - l2[1][0], l2[0][1] - l2[1][1])
+  return math.atan2(v1[0], v1[1]) - math.atan2(v2[0], v2[1])
+
+# Angle between lines in degrees
+def lines_angle_deg(l1, l2):
+  return angle_rad2deg(lines_angle(l1, l2))