sciveo 0.1.24__tar.gz → 0.1.25__tar.gz

{sciveo-0.1.24 → sciveo-0.1.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sciveo
-Version: 0.1.24
+Version: 0.1.25
 Description-Content-Type: text/markdown
 Provides-Extra: mon
 Provides-Extra: net

sciveo-0.1.25/sciveo/media/__init__.py

@@ -0,0 +1,56 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2023
+#
+
+from builtins import ImportError
+
+try:
+
+  import os
+  import time
+
+  from sciveo.tools.logger import *
+  from sciveo.tools.daemon import TasksDaemon, __upload_content__
+  from sciveo.content.runner import ProjectRunner
+  from sciveo.content.dataset import Dataset
+  from sciveo.monitoring.start import MonitorStart
+  from sciveo.network.tools import NetworkTools
+  from sciveo.version import __version__
+
+
+  TasksDaemon.current = TasksDaemon(num_threads=int(os.environ.get("SCIVEO_TASKS_NUM_THREADS", 1)))
+
+
+  # New Experiment
+  def open():
+    if ProjectRunner.current is not None:
+      return ProjectRunner.current.project
+    else:
+      error("there is no started project")
+
+  def start(project, function, configuration={}, **kwargs):
+    TasksDaemon.current.start()
+    ProjectRunner.current = ProjectRunner(project=project, function=function, configuration=configuration, **kwargs)
+    ProjectRunner.current.run()
+
+  # Dataset info
+  def dataset(info={}):
+    return Dataset.get(info)
+
+  # Monitoring start
+  def monitor(**kwargs):
+    MonitorStart(**kwargs)()
+
+  # Network tools
+  def network(**kwargs):
+    return NetworkTools(**kwargs)
+
+except ImportError as e:
+  pass

sciveo-0.1.25/sciveo/media/ml/__init__.py

@@ -0,0 +1,56 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2023
+#
+
+from builtins import ImportError
+
+try:
+
+  import os
+  import time
+
+  from sciveo.tools.logger import *
+  from sciveo.tools.daemon import TasksDaemon, __upload_content__
+  from sciveo.content.runner import ProjectRunner
+  from sciveo.content.dataset import Dataset
+  from sciveo.monitoring.start import MonitorStart
+  from sciveo.network.tools import NetworkTools
+  from sciveo.version import __version__
+
+
+  TasksDaemon.current = TasksDaemon(num_threads=int(os.environ.get("SCIVEO_TASKS_NUM_THREADS", 1)))
+
+
+  # New Experiment
+  def open():
+    if ProjectRunner.current is not None:
+      return ProjectRunner.current.project
+    else:
+      error("there is no started project")
+
+  def start(project, function, configuration={}, **kwargs):
+    TasksDaemon.current.start()
+    ProjectRunner.current = ProjectRunner(project=project, function=function, configuration=configuration, **kwargs)
+    ProjectRunner.current.run()
+
+  # Dataset info
+  def dataset(info={}):
+    return Dataset.get(info)
+
+  # Monitoring start
+  def monitor(**kwargs):
+    MonitorStart(**kwargs)()
+
+  # Network tools
+  def network(**kwargs):
+    return NetworkTools(**kwargs)
+
+except ImportError as e:
+  pass

sciveo-0.1.25/sciveo/media/ml/base.py

@@ -0,0 +1,48 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import matplotlib.pyplot as plt
+import matplotlib.dates as mdates
+
+import numpy as np
+import pandas as pd
+
+from sciveo.tools.logger import *
+from sciveo.tools.array import *
+from sciveo.media.ml.encoders.normalizer import *
+
+
+class BaseDataSet:
+  def __init__(self, data, columns=None):
+    self.data = data
+    self.normalizer = Normalizer()
+
+    if columns is not None:
+      self.data = self.data[columns]
+    self.columns = self.data.columns
+
+  def normalize(self):
+    self.data = self.normalizer.transform(self.data)
+  def denormalize(self, key):
+    self.dataset[key] = self.normalizer.inverse(self.dataset[key])
+
+  # Split dataset
+  def split(self, ratios=[["train", 0.85], ["val", 0.10], ["test", 0.05]]):
+    self.ratios = ratios
+    self.dataset = {}
+    prev_idx = 0
+    for k, v in ratios:
+      next_idx = prev_idx + int(self.data.shape[0] * v)
+      self.dataset[k] = self.data[prev_idx:next_idx]
+      prev_idx = next_idx
+
+  def summary(self):
+    return self.data.describe().transpose()[["count", "min", "max", "mean", "std", "25%", "50%", "75%"]]

sciveo-0.1.25/sciveo/media/ml/encoders/__init__.py

@@ -0,0 +1,56 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2023
+#
+
+from builtins import ImportError
+
+try:
+
+  import os
+  import time
+
+  from sciveo.tools.logger import *
+  from sciveo.tools.daemon import TasksDaemon, __upload_content__
+  from sciveo.content.runner import ProjectRunner
+  from sciveo.content.dataset import Dataset
+  from sciveo.monitoring.start import MonitorStart
+  from sciveo.network.tools import NetworkTools
+  from sciveo.version import __version__
+
+
+  TasksDaemon.current = TasksDaemon(num_threads=int(os.environ.get("SCIVEO_TASKS_NUM_THREADS", 1)))
+
+
+  # New Experiment
+  def open():
+    if ProjectRunner.current is not None:
+      return ProjectRunner.current.project
+    else:
+      error("there is no started project")
+
+  def start(project, function, configuration={}, **kwargs):
+    TasksDaemon.current.start()
+    ProjectRunner.current = ProjectRunner(project=project, function=function, configuration=configuration, **kwargs)
+    ProjectRunner.current.run()
+
+  # Dataset info
+  def dataset(info={}):
+    return Dataset.get(info)
+
+  # Monitoring start
+  def monitor(**kwargs):
+    MonitorStart(**kwargs)()
+
+  # Network tools
+  def network(**kwargs):
+    return NetworkTools(**kwargs)
+
+except ImportError as e:
+  pass

sciveo-0.1.25/sciveo/media/ml/encoders/base.py

@@ -0,0 +1,24 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+
+class BaseEncoder:
+  def fit(self, X):
+    pass
+
+  def transform(self, X):
+    pass
+
+  def fit_transform(self, X):
+    pass
+
+  def inverse(self, X):
+    pass

sciveo-0.1.25/sciveo/media/ml/encoders/normalizer.py

@@ -0,0 +1,32 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+from sciveo.media.ml.encoders.base import *
+
+
+class Normalizer:
+  def __init(self):
+    self.mean = 0
+    self.std = 0
+
+  def fit(self, X):
+    self.mean = X.mean(axis=0)
+    self.std = X.std(axis=0)
+
+  def transform(self, X):
+    return (X - self.mean) / self.std
+
+  def fit_transform(self, X):
+    self.fit(X)
+    return self.transform(X)
+
+  def inverse(self, X):
+    return X * self.std + self.mean

sciveo-0.1.25/sciveo/media/ml/nlp/__init__.py

@@ -0,0 +1,56 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2023
+#
+
+from builtins import ImportError
+
+try:
+
+  import os
+  import time
+
+  from sciveo.tools.logger import *
+  from sciveo.tools.daemon import TasksDaemon, __upload_content__
+  from sciveo.content.runner import ProjectRunner
+  from sciveo.content.dataset import Dataset
+  from sciveo.monitoring.start import MonitorStart
+  from sciveo.network.tools import NetworkTools
+  from sciveo.version import __version__
+
+
+  TasksDaemon.current = TasksDaemon(num_threads=int(os.environ.get("SCIVEO_TASKS_NUM_THREADS", 1)))
+
+
+  # New Experiment
+  def open():
+    if ProjectRunner.current is not None:
+      return ProjectRunner.current.project
+    else:
+      error("there is no started project")
+
+  def start(project, function, configuration={}, **kwargs):
+    TasksDaemon.current.start()
+    ProjectRunner.current = ProjectRunner(project=project, function=function, configuration=configuration, **kwargs)
+    ProjectRunner.current.run()
+
+  # Dataset info
+  def dataset(info={}):
+    return Dataset.get(info)
+
+  # Monitoring start
+  def monitor(**kwargs):
+    MonitorStart(**kwargs)()
+
+  # Network tools
+  def network(**kwargs):
+    return NetworkTools(**kwargs)
+
+except ImportError as e:
+  pass

sciveo-0.1.25/sciveo/media/ml/nlp/search.py

@@ -0,0 +1,70 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import os
+import requests
+
+import numpy as np
+import pandas as pd
+
+import torch
+from transformers import pipeline, AutoTokenizer, AutoModel
+from transformers import AutoModelForSeq2SeqLM
+
+from annoy import AnnoyIndex
+from sciveo.tools.logger import *
+
+from sciveo.media.pipelines.base import ApiContent
+
+
+class SearchTrainer:
+  def __init__(self, name) -> None:
+    self.name = name
+    self.api = ApiContent()
+
+    dataset_path = os.environ.get("MEDIA_DATASETS_BASE_PATH", "data")
+    self.db_path = os.path.join(dataset_path, f"{self.name}.vdb")
+    self.data_path = os.path.join(dataset_path, f"{self.name}.csv")
+
+  def init(self):
+    cache_dir = os.path.join(os.environ['MEDIA_MODELS_BASE_PATH'], "models/")
+    self.pipe = AutoModel.from_pretrained(
+      'softel/mpnet-base-v0.3',
+      cache_dir=cache_dir, resume_download=True
+    )
+    self.tokenizer = AutoTokenizer.from_pretrained(
+      'softel/mpnet-base-v0.3',
+      cache_dir=cache_dir, resume_download=True
+    )
+    self.vdb = AnnoyIndex(768, 'euclidean')
+
+  def api_load(self):
+    fields = "fields=['guid','content_type,'owner','description','name','parent','processors']"
+    url_postfix = f"content_type=image&content_type=video&~description=&{fields}"
+    self.data = self.api.read(url_postfix, limit=10)
+    url_postfix = f"content_type=image&processors=image-diffusion&{fields}"
+    self.data += self.api.read(url_postfix, limit=10)
+
+  def embed(self, text):
+    inputs = self.tokenizer.encode_plus(text, return_tensors='pt')
+    outputs = self.pipe(**inputs)
+    sequence_output = outputs[0]
+    input_mask_expanded = inputs['attention_mask'].unsqueeze(-1).expand(sequence_output.size()).float()
+    embeddings = torch.sum(sequence_output * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+    return embeddings.detach().numpy()[0]
+
+  def train(self):
+    self.api_load()
+
+
+if __name__ == '__main__':
+  trainer = SearchTrainer("content_search")
+  trainer.api_load()

sciveo-0.1.25/sciveo/media/ml/time_series/__init__.py

@@ -0,0 +1,56 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2023
+#
+
+from builtins import ImportError
+
+try:
+
+  import os
+  import time
+
+  from sciveo.tools.logger import *
+  from sciveo.tools.daemon import TasksDaemon, __upload_content__
+  from sciveo.content.runner import ProjectRunner
+  from sciveo.content.dataset import Dataset
+  from sciveo.monitoring.start import MonitorStart
+  from sciveo.network.tools import NetworkTools
+  from sciveo.version import __version__
+
+
+  TasksDaemon.current = TasksDaemon(num_threads=int(os.environ.get("SCIVEO_TASKS_NUM_THREADS", 1)))
+
+
+  # New Experiment
+  def open():
+    if ProjectRunner.current is not None:
+      return ProjectRunner.current.project
+    else:
+      error("there is no started project")
+
+  def start(project, function, configuration={}, **kwargs):
+    TasksDaemon.current.start()
+    ProjectRunner.current = ProjectRunner(project=project, function=function, configuration=configuration, **kwargs)
+    ProjectRunner.current.run()
+
+  # Dataset info
+  def dataset(info={}):
+    return Dataset.get(info)
+
+  # Monitoring start
+  def monitor(**kwargs):
+    MonitorStart(**kwargs)()
+
+  # Network tools
+  def network(**kwargs):
+    return NetworkTools(**kwargs)
+
+except ImportError as e:
+  pass

sciveo-0.1.25/sciveo/media/ml/time_series/dataset.py

@@ -0,0 +1,67 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import matplotlib.pyplot as plt
+import matplotlib.dates as mdates
+
+import numpy as np
+import pandas as pd
+
+import tensorflow as tf
+from tensorflow import keras
+
+from sciveo.tools.logger import *
+from sciveo.tools.array import *
+from sciveo.media.ml.encoders.normalizer import *
+from sciveo.media.ml.time_series.window_generator import *
+from sciveo.media.ml.base import BaseDataSet
+
+
+class TimeSeriesDataSet(BaseDataSet):
+  def __init__(self, data, time_col, format='%Y-%m-%d %H24:%M:%S', columns=None, new_period_columns=True):
+    super().__init__(data, columns)
+
+    debug(f"time_col [{time_col}] format[{format}]")
+    debug("columns", self.data.columns)
+
+    self.time_col = time_col
+    if "datetime" not in str(self.data[time_col].dtype):
+      self.data[time_col] = pd.to_datetime(self.data[time_col], format=format)
+    self.data = self.data.set_index(time_col).sort_index()
+    self.date_time = self.data.index
+
+    if new_period_columns:
+      timestamp_s = self.date_time.map(pd.Timestamp.timestamp)
+      hour = 60 * 60
+      day = 24 * hour
+      periods = [
+        hour,
+        day,
+        91.310625 * day,
+        365.2425 * day
+      ]
+      for period in periods:
+        self.data[f"period-sin-{period / day}"] = np.sin(timestamp_s * (2 * np.pi / period))
+        # self.data[f"period-cos-{period / day}"] = np.cos(timestamp_s * (2 * np.pi / period))
+
+    self.columns = self.data.columns
+
+    self.normalizer.fit(self.data)
+
+  def plots(self, max_columns=3, max_points=120):
+    plot_cols = list(self.data.columns)[:max_columns]
+    plot_features = self.data[plot_cols]
+    plot_features.index = self.date_time
+    _ = plot_features.plot(subplots=True)
+
+    plot_features = self.data[plot_cols][:max_points]
+    plot_features.index = self.date_time[:max_points]
+    _ = plot_features.plot(subplots=True)

sciveo-0.1.25/sciveo/media/ml/time_series/predictor.py

@@ -0,0 +1,105 @@
+#
+# Pavlin Georgiev, Softel Labs
+#
+# This is a proprietary file and may not be copied,
+# distributed, or modified without express permission
+# from the owner. For licensing inquiries, please
+# contact pavlin@softel.bg.
+#
+# 2024
+#
+
+import os
+import joblib
+
+import matplotlib.pyplot as plt
+import matplotlib.dates as mdates
+
+import numpy as np
+import pandas as pd
+
+import tensorflow as tf
+from tensorflow import keras
+
+from sciveo.tools.logger import *
+from sciveo.tools.array import *
+from sciveo.media.ml.encoders.normalizer import *
+from sciveo.media.ml.time_series.window_generator import *
+from sciveo.media.ml.time_series.dataset import *
+
+
+class TimeSeriesPredictor:
+  def __init__(self, model_path):
+    self.model_path = model_path
+
+    self.model_data = joblib.load(self.model_path)
+
+    self.window_generator = WindowGenerator(
+      input_width=self.model_data["window"]["input_width"],
+      label_width=0,
+      shift=self.model_data["window"]["shift"]
+    )
+
+  def predict(self, df, x_expand_factor=2):
+    normalizer = self.model_data["normalizer"]
+    L = self.model_data["window"]["input_width"]
+    F = len(self.model_data["columns"])
+    predict_df = df[-L:][self.model_data["columns"]]
+    predict_df.index = pd.to_datetime(predict_df.index)
+
+    x_plot = list(predict_df.index)
+    x_plot = self.expand_dates(x_plot, x_expand_factor)
+
+    predict_df = normalizer.transform(predict_df)
+
+    X = predict_df.values.reshape((1, L, F)).astype('float32')
+    X = tf.convert_to_tensor(X)
+
+    predictions = self.model_data["model"](X)
+    predictions = normalizer.inverse(predictions)
+    X = normalizer.inverse(X)
+    debug(f"predict {L} shapes: predict_df[{predict_df.shape}] x_plot[{len(x_plot)}] X[{X.shape}] predictions[{predictions.shape}]")
+    return predictions, X, x_plot
+
+  def expand_dates(self, list_dates, expand_factor):
+    n = len(list_dates) * expand_factor
+    delta = (list_dates[1] - list_dates[0]) # TODO: create more robust delta computation, currently assume equidistant
+    expanded_dates = [list_dates[0] + i * delta for i in range(n)]
+    return expanded_dates
+
+  def plot(self, predictions, X, x_plot, plot_col_index, labels=None, image_local_path=None, dpi=100, width=640, height=480):
+    L = X.shape[1]
+    plot_col = self.model_data["columns"][plot_col_index]
+    plt.figure(figsize=(12, 8), dpi=dpi)
+
+    plt.subplot(1, 1, 1)
+    plt.ylabel(f'{plot_col}')
+
+    plt.plot(
+      x_plot[:L], X[0, :, plot_col_index],
+      label='X', marker='.', zorder=-10
+    )
+
+    plt.scatter(
+      x_plot[L:], predictions[0, :, plot_col_index],
+      marker='X', edgecolors='k', label='Predictions',
+      c='#ff7f0e', s=64
+    )
+
+    if labels is not None:
+      plt.scatter(
+        x_plot[L:], labels[0, :, plot_col_index],
+        edgecolors='k', label='Labels', c='#2ca02c', s=64
+      )
+
+    plt.xticks(rotation=45)
+    plt.tight_layout()
+    plt.legend(title=plot_col, loc='upper left')
+    plt.xlabel("date time")
+
+    if image_local_path is not None:
+      plt.savefig(image_local_path, format='png', dpi=dpi, bbox_inches='tight', pad_inches=0)
+      debug("plot savefig", image_local_path)
+    else:
+      plt.show()
+    plt.close()