sciveo 0.1.25__tar.gz → 0.1.26__tar.gz

{sciveo-0.1.25 → sciveo-0.1.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.25
+Version: 0.1.26
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

sciveo-0.1.26/sciveo/media/pipelines/processors/sci/time_series/predictor.py

@@ -0,0 +1,108 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import os
+import boto3
+
+from sciveo.tools.logger import *
+from sciveo.media.pipelines.processors.sci.base import *
+from sciveo.media.ml.time_series.predictor import *
+
+
+class TimeSeriesPredictorProcessor(SciBaseProcessor):
+  def __init__(self, processor_config, max_progress) -> None:
+    super().__init__(processor_config, max_progress)
+
+    self.default.update({
+      "time_column": "Date Time",
+      "time_format": '%d.%m.%Y %H:%M:%S'
+    })
+
+  def run(self, job, input):
+    progress_per_media = self.max_progress / max(1, len(input))
+    debug("run", job["id"], progress_per_media, "input", input)
+    next_media = []
+    try:
+      if job['content_type'] == "project":
+        project_path = os.path.join(self.base_tmp_path, "projects", job['content_id'])
+        mkdirs(project_path)
+
+        self.download_content(job, project_path)
+        MediaJobState.queue().inc_progress(job["id"], 10)
+
+        list_content = {}
+
+        dataset_df = []
+        for content in job['content']:
+          list_content.setdefault(content["content_type"], []).append(content)
+
+          if content["content_type"] == "file":
+            if content['key'].split(".")[-1] == "timeseries":
+              list_content.setdefault("timeseries", []).append(content)
+
+          if content["content_type"] == "dataset":
+            # Datasets with S3 located files
+            if "bucket" in content and "key" in content:
+              remote_path, local_path, local_file_name = self.content_path(content, project_path)
+              file_extension = local_file_name.split(".")[-1]
+
+              if file_extension == "csv":
+                df = pd.read_csv(local_path)
+                x_col = content['data'].get("dataset", {}).get("x_col", self["time_column"])
+                df.set_index(x_col, inplace=True)
+                dataset_df.append(df)
+
+        dataset_df = pd.concat(dataset_df).drop_duplicates(keep='first').sort_index().reset_index()
+
+        content_project = list_content["project"][0]
+
+        MediaJobState.queue().inc_progress(job["id"], 5)
+
+        if "timeseries" in list_content:
+          content_predictor = self.new_content(content_project, "prediction", name=f"Prediction on {dataset_df.shape}")
+          next_media.append(content_predictor)
+
+          text_content = self.content_text(content_predictor, "timeseries predictions plots", f"""Time series predictions.\n\n{self.describe_df(dataset_df)}""")
+          next_media.append(text_content)
+
+          content = list_content["timeseries"][0]
+          remote_path, local_path, local_file_name = self.content_path(content, project_path)
+          predictor = TimeSeriesPredictor(local_path)
+          ds = TimeSeriesDataSet(dataset_df, self["time_column"], format=self["time_format"])
+          predictions, X, x_plot = predictor.predict(ds.data)
+
+          MediaJobState.queue().inc_progress(job["id"], 5)
+
+          columns = predictor.model_data["columns"][:11] # TODO: Should be configurable in the Predictor, so plot only few columns
+          plot_progress_inc = 20 / len(columns)
+          for i, y_col in enumerate(columns):
+            image_content = self.content_image(text_content, y_col, project_path)
+            predictor.plot(predictions, X, x_plot, i, image_local_path=image_content["local_path"])
+            image_content["data"] = {
+              "info": {
+                "size": os.path.getsize(image_content["local_path"]),
+                "plot": y_col
+              }
+            }
+            next_media.append(image_content)
+            MediaJobState.queue().inc_progress(job["id"], plot_progress_inc)
+      MediaJobState.queue().inc_progress(job["id"], 10)
+
+      if self["output"]:
+        job["output"] += next_media
+      if self["append-content"]:
+        job["append-content"] += next_media
+    except Exception as e:
+      exception(e)
+    return next_media
+
+  def name(self):
+    return "sci-timeseries-predictor"

sciveo-0.1.26/sciveo/media/pipelines/processors/sci/time_series/trainer.py

@@ -0,0 +1,185 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import os
+import pandas as pd
+import tensorflow as tf
+from tensorflow.keras.callbacks import Callback
+
+from sciveo.tools.logger import *
+from sciveo.media.pipelines.processors.sci.base import *
+from sciveo.media.ml.time_series.predictor import *
+from sciveo.media.ml.time_series.trainer import *
+
+
+class TrainerProgressCallback(Callback):
+  def __init__(self, job, epochs, max_progress):
+    self.job = job
+    self.progress_per_epoch = 2 * max_progress / epochs
+
+  def on_epoch_end(self, epoch, logs=None):
+    debug("epoch", epoch, "finished")
+    MediaJobState.queue().inc_progress(self.job["id"], self.progress_per_epoch)
+
+
+class TimeSeriesTrainerProcessor(SciBaseProcessor):
+  def __init__(self, processor_config, max_progress) -> None:
+    super().__init__(processor_config, max_progress)
+
+    self.default.update({
+      "model_name": "none",
+      "time_column": "Date Time",
+      "time_format": '%d.%m.%Y %H:%M:%S',
+      "ratios": [["train", 0.70], ["val", 0.20], ["test", 0.10]],
+      "max_epochs": 20, "patience": 2,
+      "window_size": 24,
+      "test_plot_from": 200, "test_plots": 20, "test_plot_column_id": 1
+    })
+
+  def run(self, job, input):
+    progress_per_media = self.max_progress / max(1, len(input))
+    debug("run", job["id"], progress_per_media, "input", input)
+    next_media = []
+    try:
+      if job['content_type'] == "project":
+        project_path = os.path.join(self.base_tmp_path, "projects", job['content_id'])
+        mkdirs(project_path)
+
+        self.download_content(job, project_path)
+        MediaJobState.queue().inc_progress(job["id"], 10)
+
+        list_content = {}
+
+        dataset_df = []
+        for content in job['content']:
+          list_content.setdefault(content["content_type"], []).append(content)
+
+          if content["content_type"] == "file":
+            if content['key'].split(".")[-1] == "timeseries":
+              list_content.setdefault("timeseries", []).append(content)
+
+          if content["content_type"] == "dataset":
+            # Datasets with S3 located files
+            if "bucket" in content and "key" in content:
+              remote_path, local_path, local_file_name = self.content_path(content, project_path)
+              file_extension = local_file_name.split(".")[-1]
+
+              if file_extension == "csv":
+                df = pd.read_csv(local_path)
+                x_col = content['data'].get("dataset", {}).get("x_col", df.keys()[0])
+                df.set_index(x_col, inplace=True)
+                dataset_df.append(df)
+
+        dataset_df = pd.concat(dataset_df).drop_duplicates(keep='first').sort_index().reset_index()
+
+        content_project = list_content["project"][0]
+        MediaJobState.queue().inc_progress(job["id"], 5)
+
+        if len(dataset_df) > 0:
+          content_trainer = self.new_content(content_project, "training", name=f"Training on {dataset_df.shape}")
+          next_media.append(content_trainer)
+
+          ds = TimeSeriesDataSet(dataset_df, self["time_column"], format=self["time_format"])
+          ds.normalize()
+          ds.split(ratios=self["ratios"])
+
+          trainer = TimeSeriesTrainer(ds, self["window_size"], self["window_size"], self["window_size"])
+          trainer.create()
+          progress_callback = TrainerProgressCallback(job, self["max_epochs"], 50)
+          history = trainer.train(self["max_epochs"], self["patience"], progress_callback)
+          trainer_eval = trainer.evaluate()
+
+          model_name, model_path = trainer.save(project_path, self["model_name"])
+
+          model_data = {
+            "eval": trainer_eval,
+            "history": {
+              "loss": history.history["loss"],
+              "val_loss": history.history["val_loss"]
+            }
+          }
+
+          model_content = self.content_file(content_trainer, model_name, model_path, data=model_data)
+          next_media.append(model_content)
+
+          text = f"""
+            Time series model.
+            ....
+            ....
+            model name {model_name}
+            model size {os.path.getsize(model_path)}
+
+            eval
+            {self.df_to_html(pd.DataFrame(trainer_eval))}
+
+            train loss
+            {self.df_to_html(pd.DataFrame({"loss": history.history["loss"], "val_loss": history.history["val_loss"]}))}
+          """
+          text_content = self.content_text(content_trainer, "Train timeseries model", text)
+          next_media.append(text_content)
+
+          # TODO: Create numeric from history.history["loss"], history.history["val_loss"]
+
+          ds.denormalize("test")
+          predictor = TimeSeriesPredictor(model_path)
+
+          # Plot some test predictions
+          columns = predictor.model_data["columns"]
+          plot_progress_inc = 10 / self["test_plots"]
+          y_col = columns[self["test_plot_column_id"]]
+          for i in range(self["test_plot_from"], self["test_plot_from"] + self["test_plots"]):
+            image_content = self.content_image(text_content, y_col, project_path)
+            self.plot_chunk(ds.dataset["test"], predictor, i, self["test_plot_column_id"], image_content["local_path"])
+            image_content["data"] = {
+              "info": {
+                "size": os.path.getsize(image_content["local_path"]),
+                "plot": y_col
+              }
+            }
+            next_media.append(image_content)
+            MediaJobState.queue().inc_progress(job["id"], plot_progress_inc)
+
+      MediaJobState.queue().inc_progress(job["id"], 10)
+
+      if self["output"]:
+        job["output"] += next_media
+      if self["append-content"]:
+        job["append-content"] += next_media
+    except Exception as e:
+      exception(e)
+    return next_media
+
+  def plot_chunk(self, df, predictor, k, i, image_local_path):
+    k += 1
+    L = predictor.model_data["window"]["input_width"]
+    X = df[-(k + 1)*L:-k*L]
+
+    predictions, X, x_plot = predictor.predict(X)
+
+    idx_from = -k * L
+    idx_to = -(k - 1) * L
+    if idx_to != 0:
+      labels = df[idx_from:idx_to]
+    else:
+      labels = df[idx_from:]
+    labels = labels.values.reshape(X.shape).astype('float32')
+    labels = tf.convert_to_tensor(labels)
+
+    predictor.plot(
+        predictions, X, x_plot, i, labels,
+        image_local_path
+    )
+
+  def content_type(self):
+    return None
+
+  def name(self):
+    return "sci-timeseries-trainer"

sciveo-0.1.26/sciveo/tools/aws/__init__.py

@@ -0,0 +1,56 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2023
+#
+
+from builtins import ImportError
+
+try:
+
+  import os
+  import time
+
+  from sciveo.tools.logger import *
+  from sciveo.tools.daemon import TasksDaemon, __upload_content__
+  from sciveo.content.runner import ProjectRunner
+  from sciveo.content.dataset import Dataset
+  from sciveo.monitoring.start import MonitorStart
+  from sciveo.network.tools import NetworkTools
+  from sciveo.version import __version__
+
+
+  TasksDaemon.current = TasksDaemon(num_threads=int(os.environ.get("SCIVEO_TASKS_NUM_THREADS", 1)))
+
+
+  # New Experiment
+  def open():
+    if ProjectRunner.current is not None:
+      return ProjectRunner.current.project
+    else:
+      error("there is no started project")
+
+  def start(project, function, configuration={}, **kwargs):
+    TasksDaemon.current.start()
+    ProjectRunner.current = ProjectRunner(project=project, function=function, configuration=configuration, **kwargs)
+    ProjectRunner.current.run()
+
+  # Dataset info
+  def dataset(info={}):
+    return Dataset.get(info)
+
+  # Monitoring start
+  def monitor(**kwargs):
+    MonitorStart(**kwargs)()
+
+  # Network tools
+  def network(**kwargs):
+    return NetworkTools(**kwargs)
+
+except ImportError as e:
+  pass

sciveo-0.1.26/sciveo/version.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.1.26'

{sciveo-0.1.25 → sciveo-0.1.26}/sciveo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.25
+Version: 0.1.26
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

{sciveo-0.1.25 → sciveo-0.1.26}/sciveo.egg-info/SOURCES.txt

@@ -74,6 +74,9 @@ sciveo/media/pipelines/processors/nlp/address.py
 sciveo/media/pipelines/processors/sci/__init__.py
 sciveo/media/pipelines/processors/sci/base.py
 sciveo/media/pipelines/processors/sci/dataset.py
+sciveo/media/pipelines/processors/sci/time_series/__init__.py
+sciveo/media/pipelines/processors/sci/time_series/predictor.py
+sciveo/media/pipelines/processors/sci/time_series/trainer.py
 sciveo/media/pipelines/processors/video/__init__.py
 sciveo/media/pipelines/processors/video/generators.py
 sciveo/media/pipelines/processors/video/motion_detection.py

sciveo-0.1.25/sciveo/version.py

@@ -1,2 +0,0 @@
-
-__version__ = '0.1.25'