flowcept 0.8.5__py3-none-any.whl → 0.8.7__py3-none-any.whl

flowcept/commons/daos/docdb_dao/mongodb_dao.py

@@ -61,9 +61,22 @@ class MongoDBDAO(DocumentDBDAO):
         self.logger = FlowceptLogger()
 
         if MONGO_URI is not None:
-            self._client = MongoClient(MONGO_URI)
+            self._client = MongoClient(
+                MONGO_URI,
+                maxPoolSize=1000,  # TODO: conf file
+                socketTimeoutMS=60000,
+                connectTimeoutMS=60000,
+                serverSelectionTimeoutMS=60000,
+            )
         else:
-            self._client = MongoClient(MONGO_HOST, MONGO_PORT)
+            self._client = MongoClient(
+                MONGO_HOST,
+                MONGO_PORT,
+                maxPoolSize=1000,
+                socketTimeoutMS=60000,
+                connectTimeoutMS=60000,
+                serverSelectionTimeoutMS=60000,
+            )
         self._db = self._client[MONGO_DB]
 
         self._tasks_collection = self._db["tasks"]

flowcept/commons/daos/mq_dao/mq_dao_mofka.py

@@ -65,7 +65,7 @@ class MQDaoMofka(MQDao):
 
     def _bulk_publish(self, buffer, channel=MQ_CHANNEL, serializer=msgpack.dumps):
         try:
-            self.logger.debug(f"Going to send Message:\n\t[BEGIN_MSG]{buffer}\n[END_MSG]\t")
+            # self.logger.debug(f"Going to send Message:\n\t[BEGIN_MSG]{buffer}\n[END_MSG]\t")
             for m in buffer:
                 self.producer.push(m)
 
@@ -75,14 +75,14 @@ class MQDaoMofka(MQDao):
             self.logger.error(f"Message that caused error: {buffer}")
         try:
             self.producer.flush()
-            self.logger.info(f"Flushed {len(buffer)} msgs to MQ!")
+            # self.logger.info(f"Flushed {len(buffer)} msgs to MQ!")
         except Exception as e:
             self.logger.exception(e)
 
     def _bulk_publish_timed(self, buffer, channel=MQ_CHANNEL, serializer=msgpack.dumps):
         total = 0
         try:
-            self.logger.debug(f"Going to send Message:\n\t[BEGIN_MSG]{buffer}\n[END_MSG]\t")
+            # self.logger.debug(f"Going to send Message:\n\t[BEGIN_MSG]{buffer}\n[END_MSG]\t")
 
             for m in buffer:
                 self.producer.push(m)
@@ -97,7 +97,7 @@ class MQDaoMofka(MQDao):
             self.producer.flush()
             t2 = time()
             self._flush_events.append(["bulk", t1, t2, t2 - t1, total])
-            self.logger.info(f"Flushed {len(buffer)} msgs to MQ!")
+            # self.logger.info(f"Flushed {len(buffer)} msgs to MQ!")
         except Exception as e:
             self.logger.exception(e)
 

flowcept/commons/flowcept_dataclasses/workflow_object.py

@@ -1,4 +1,4 @@
-"""Workflow module."""
+"""Workflow Object module."""
 
 from typing import Dict, AnyStr, List
 import msgpack

flowcept/configs.py

@@ -146,7 +146,7 @@ PERF_LOG = settings["project"].get("performance_logging", False)
 JSON_SERIALIZER = settings["project"].get("json_serializer", "default")
 REPLACE_NON_JSON_SERIALIZABLE = settings["project"].get("replace_non_json_serializable", True)
 ENRICH_MESSAGES = settings["project"].get("enrich_messages", True)
-REGISTER_WORKFLOW = settings["project"].get("register_workflow", True)
+
 
 TELEMETRY_CAPTURE = settings.get("telemetry_capture", None)
 

flowcept/flowceptor/adapters/base_interceptor.py

@@ -9,6 +9,7 @@ from flowcept.commons.flowcept_dataclasses.workflow_object import (
 )
 from flowcept.configs import (
     ENRICH_MESSAGES,
+    INSTRUMENTATION,
 )
 from flowcept.commons.flowcept_logger import FlowceptLogger
 from flowcept.commons.daos.mq_dao.mq_dao_base import MQDao
@@ -49,15 +50,23 @@ class BaseInterceptor(object):
         elif kind in "dask":
             # This is dask's client interceptor. We essentially use it to store the dask workflow.
             # That's why we don't need another special interceptor and we can reuse the instrumentation one.
-            from flowcept.flowceptor.adapters.instrumentation_interceptor import InstrumentationInterceptor
-
-            return InstrumentationInterceptor.get_instance()
+            return BaseInterceptor._build_instrumentation_interceptor()
         elif kind == "instrumentation":
+            return BaseInterceptor._build_instrumentation_interceptor()
+        else:
+            raise NotImplementedError
+
+    @staticmethod
+    def _build_instrumentation_interceptor():
+        # By using singleton, we lose the thread safety for the Interceptor, particularly, its MQ buffer.
+        # Since some use cases need threads, this allows disabling the singleton for more thread safety.
+        is_singleton = INSTRUMENTATION.get("singleton", True)
+        if is_singleton:
             from flowcept.flowceptor.adapters.instrumentation_interceptor import InstrumentationInterceptor
 
             return InstrumentationInterceptor.get_instance()
         else:
-            raise NotImplementedError
+            return BaseInterceptor(kind="instrumentation")
 
     def __init__(self, plugin_key=None, kind=None):
         self.logger = FlowceptLogger()

flowcept/flowceptor/consumers/document_inserter.py

@@ -211,10 +211,14 @@ class DocumentInserter:
             return True
 
     def stop(self, bundle_exec_id=None):
-        """Stop it."""
+        """Stop document inserter."""
         if self.check_safe_stops:
             trial = 0
             while not self._mq_dao.all_time_based_threads_ended(bundle_exec_id):
+                self.logger.debug(
+                    f"# time_based_threads for bundle_exec_id {bundle_exec_id} is"
+                    f"{self._mq_dao._keyvalue_dao.set_count(bundle_exec_id)}"
+                )
                 trial += 1
                 self.logger.info(
                     f"Doc Inserter {id(self)}: It's still not safe to stop DocInserter. "

flowcept/flowceptor/telemetry_capture.py

@@ -168,7 +168,7 @@ class GPUCapture:
             }
 
         if "id" in gpu_conf:
-            flowcept_gpu_info["id"] = (amdsmi_get_gpu_device_uuid(device),)
+            flowcept_gpu_info["id"] = amdsmi_get_gpu_device_uuid(device)
 
         return flowcept_gpu_info
 

flowcept/instrumentation/flowcept_torch.py

@@ -17,7 +17,6 @@ from flowcept.commons.flowcept_dataclasses.workflow_object import (
 )
 from flowcept.commons.vocabulary import Status
 from flowcept.configs import (
-    REGISTER_WORKFLOW,
     INSTRUMENTATION,
     TELEMETRY_CAPTURE,
     REPLACE_NON_JSON_SERIALIZABLE,
@@ -30,6 +29,8 @@ from flowcept.instrumentation.flowcept_task import get_current_context_task_id
 
 TORCH_CONFIG = INSTRUMENTATION.get("torch")
 
+REGISTER_WORKFLOW = TORCH_CONFIG.get("register_workflow", True)
+
 
 def flowcept_torch(cls):
     """

flowcept/instrumentation/task_capture.py

@@ -49,9 +49,6 @@ class FlowceptTask(object):
     are no-ops, and no data is captured.
     """
 
-    if INSTRUMENTATION_ENABLED:
-        _interceptor = InstrumentationInterceptor.get_instance()
-
     def __init__(
         self,
         task_id: str = None,
@@ -60,12 +57,18 @@ class FlowceptTask(object):
         activity_id: str = None,
         used: Dict = None,
         custom_metadata: Dict = None,
+        flowcept: "Flowcept" = None,
     ):
         if not INSTRUMENTATION_ENABLED:
             self._ended = True
             return
+        if flowcept is not None and flowcept._interceptor_instances[0].kind == "instrumentation":
+            self._interceptor = flowcept._interceptor_instances[0]
+        else:
+            self._interceptor = InstrumentationInterceptor.get_instance()
+
         self._task = TaskObject()
-        self._task.telemetry_at_start = FlowceptTask._interceptor.telemetry_capture.capture()
+        self._task.telemetry_at_start = self._interceptor.telemetry_capture.capture()
         self._task.activity_id = activity_id
         self._task.started_at = time()
         self._task.task_id = task_id or str(self._task.started_at)
@@ -117,11 +120,11 @@ class FlowceptTask(object):
         """
         if not INSTRUMENTATION_ENABLED:
             return
-        self._task.telemetry_at_end = FlowceptTask._interceptor.telemetry_capture.capture()
+        self._task.telemetry_at_end = self._interceptor.telemetry_capture.capture()
         self._task.ended_at = ended_at or time()
         self._task.status = status
         self._task.stderr = stderr
         self._task.stdout = stdout
         self._task.generated = generated
-        FlowceptTask._interceptor.intercept(self._task.to_dict())
+        self._interceptor.intercept(self._task.to_dict())
         self._ended = True

flowcept/version.py

@@ -4,4 +4,4 @@
 # The expected format is: <Major>.<Minor>.<Patch>
 # This file is supposed to be automatically modified by the CI Bot.
 # See .github/workflows/version_bumper.py
-__version__ = "0.8.5"
+__version__ = "0.8.7"

{flowcept-0.8.5.dist-info → flowcept-0.8.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.8.5
+Version: 0.8.7
 Summary: Capture and query workflow provenance data using data observability
 Project-URL: GitHub, https://github.com/ORNL/flowcept
 Author: Oak Ridge National Laboratory
@@ -96,20 +96,34 @@ Description-Content-Type: text/markdown
 
 # Flowcept
 
+## Table of Contents
+
+- [Overview](#overview)
+- [Features](#features)
+- [Installation](#installation)
+- [Setup and the Settings File](#setup)
+- [Running with Containers](#running-with-containers)
+- [Examples](#examples)
+- [Data Persistence](#data-persistence)
+- [Performance Tuning](#performance-tuning-for-performance-evaluation)
+- [AMD GPU Setup](#install-amd-gpu-lib)
+
+## Overview
+
 Flowcept is a runtime data integration system that captures and queries workflow provenance with minimal or no code changes. It unifies data across diverse workflows and tools, enabling integrated analysis and insights, especially in federated environments. Designed for scenarios involving critical data from multiple workflows, Flowcept seamlessly integrates data at runtime, providing a unified view for end-to-end monitoring and analysis, and enhanced support for Machine Learning (ML) workflows.
 
-Other capabilities include:
-
-- Automatic multi-workflow provenance data capture;
-- Data observability, enabling minimal intrusion to user workflows;
-- Explicit user workflow instrumentation, if this is preferred over implicit data observability;
-- ML data capture in various levels of details: workflow, model fitting or evaluation task, epoch iteration, layer forwarding;
-- ML model management;
-- Adapter-based, loosely-coupled system architecture, making it easy to plug and play with different data processing systems and backend database (e.g., MongoDB) or MQ services (e.g., Redis, Kafka);
-- Low-overhead focused system architecture, to avoid adding performance overhead particularly to workloads that run on HPC machines;
-- Telemetry data capture (e.g., CPU, GPU, Memory consumption) linked to the application dataflow;
-- Highly customizable to multiple use cases, enabling easy toggle between settings (e.g., with/without provenance capture; with/without telemetry and which telemetry type to capture; which adapters or backend services to run with); 
-- [W3C PROV](https://www.w3.org/TR/prov-overview/) adherence;
+## Features
+
+- Automatic workflow provenance data capture from heterogeneous workflows
+- Data observability with no or minimal intrusion to application workflows
+- Explicit application instrumentation, if this is preferred over data observability
+- ML data capture in various levels of details: workflow, model fitting or evaluation task, epoch iteration, layer forwarding
+- ML model management (e.g., model storage and retrieval, along with their metadata and provenance)
+- Adapter-based, loosely-coupled system architecture, making it easy to plug and play with different data processing systems and backend database (e.g., MongoDB) or MQ services (e.g., Redis, Kafka)
+- Low-overhead focused system architecture, to avoid adding performance overhead particularly to workloads that run on HPC machines
+- Telemetry data capture (e.g., CPU, GPU, Memory consumption) linked to the application dataflow
+- Highly customizable to multiple use cases, enabling easy toggle between settings (e.g., with/without provenance capture; with/without telemetry and which telemetry type to capture; which adapters or backend services to run with) 
+- [W3C PROV](https://www.w3.org/TR/prov-overview/) adherence
  
 Notes:
 
@@ -192,7 +206,8 @@ To use Flowcept, one needs to start a MQ system `$> make services`. This will st
 
 ### Flowcept Settings File
 
-Flowcept requires a settings file for configuration. You can find an example [here](resources/sample_settings.yaml).
+Flowcept requires a settings file for configuration. 
+You can find an example configuration file [here](resources/sample_settings.yaml), with documentation for each parameter provided as inline comments.
 
 #### What You Can Configure:
 
@@ -214,7 +229,6 @@ export FLOWCEPT_SETTINGS_PATH=/absolute/path/to/your/settings.yaml
 
 If this variable is not set, Flowcept will use the default values from the [example](resources/sample_settings.yaml) file.
 
-
 # Running with Containers
 
 To use containers instead of installing Flowcept's dependencies on your host system, we provide a [Dockerfile](deployment/Dockerfile) alongside a [docker-compose.yml](deployment/compose.yml) for dependent services (e.g., Redis, MongoDB).  

{flowcept-0.8.5.dist-info → flowcept-0.8.7.dist-info}/RECORD

@@ -1,6 +1,6 @@
 flowcept/__init__.py,sha256=CukmdzTUvm6Y_plTKPq4kKn7w9LdR36j7V_C_UQyjhU,2011
-flowcept/configs.py,sha256=_-jhoI_HGKjzymjYTlDuysbM38Gr2aunc0Q-Stlmcwk,7511
-flowcept/version.py,sha256=RW_aTLB2vWTDjrpIPcWytUXxQhZLynom14B2UHfVVcU,306
+flowcept/configs.py,sha256=NDUAqqoKfztt6Qjwxy95eTQU71AovVJWXalI1x3HJ7Y,7441
+flowcept/version.py,sha256=VOEzbBIlAwCTu8Yc20ihIzsk7fgYS5oHS79aAfnDY98,306
 flowcept/analytics/__init__.py,sha256=46q-7vsHq_ddPNrzNnDgEOiRgvlx-5Ggu2ocyROMV0w,641
 flowcept/analytics/analytics_utils.py,sha256=FRJdBtQa7Hrk2oR_FFhmhmMf3X6YyZ4nbH5RIYh7KL4,8753
 flowcept/analytics/data_augmentation.py,sha256=Dyr5x316Zf-k1e8rVoQMCpFOrklYVHjfejRPrtoycmc,1641
@@ -17,17 +17,17 @@ flowcept/commons/daos/keyvalue_dao.py,sha256=03xHhQIfZas0LQLP1DbGJ5DoskXyZNXQKIN
 flowcept/commons/daos/docdb_dao/__init__.py,sha256=qRvXREeUJ4mkhxdC9bzpOsVX6M2FB5hDyLFxhMxTGhs,30
 flowcept/commons/daos/docdb_dao/docdb_dao_base.py,sha256=YbfSVJPwZGK2GBYkeapRC83HkmP0c6Msv5TriD88RcI,11812
 flowcept/commons/daos/docdb_dao/lmdb_dao.py,sha256=dJOLgCx_lwdz6MKiMpM_UE4rm0angDCPaVz_WU5KqIA,10407
-flowcept/commons/daos/docdb_dao/mongodb_dao.py,sha256=-Kxjep1FbjKiGjvzyvePVHDf-Q1lOIce1EzBURSKubc,38037
+flowcept/commons/daos/docdb_dao/mongodb_dao.py,sha256=0y9RiL54e1GxSTkRHFlMrLFAHWuB3YyNS2zLsnBPtxg,38456
 flowcept/commons/daos/mq_dao/__init__.py,sha256=Xxm4FmbBUZDQ7XIAmSFbeKE_AdHsbgFmSuftvMWSykQ,21
 flowcept/commons/daos/mq_dao/mq_dao_base.py,sha256=EAqOhy7Q8V29JFDG8C50nRK34KsPxEICkG4elk4ZfX8,9020
 flowcept/commons/daos/mq_dao/mq_dao_kafka.py,sha256=bf-bZvWw9JJk8Kdfzx2UkAnQC95rSrKXDEyYkrcncOk,4400
-flowcept/commons/daos/mq_dao/mq_dao_mofka.py,sha256=aZ810wN5Wkjk7oRUxDWJWOIREUsmq57oI4AxY1bWBuk,3940
+flowcept/commons/daos/mq_dao/mq_dao_mofka.py,sha256=Q_mgZ3C_4gTTvnuJ2ZLmJgJfbAOopeSR9jvznI4JRuo,3948
 flowcept/commons/daos/mq_dao/mq_dao_redis.py,sha256=Br97SoDIkt4dHH937Yjg3wtkn1xGT-x9t-8E3VD5TeU,4277
 flowcept/commons/flowcept_dataclasses/__init__.py,sha256=8KkiJh0WSRAB50waVluxCSI8Tb9X1L9nup4c8RN3ulc,30
 flowcept/commons/flowcept_dataclasses/base_settings_dataclasses.py,sha256=Cjw2PGYtZDfnwecz6G3S42Ncmxj7AIZVEBx05bsxRUo,399
 flowcept/commons/flowcept_dataclasses/task_object.py,sha256=3DD5ZNMz7EVILS9PRkQ3khboav7lIKoUC5W6sKMFauQ,4694
 flowcept/commons/flowcept_dataclasses/telemetry.py,sha256=9_5ONCo-06r5nKHXmi5HfIhiZSuPgmTECiq_u9MlxXM,2822
-flowcept/commons/flowcept_dataclasses/workflow_object.py,sha256=FBpel5PulrN3mCCk3hrQhoYiFqd-4aNV4tT03bCV3DE,4376
+flowcept/commons/flowcept_dataclasses/workflow_object.py,sha256=f8aB0b3xcUr3KQTlloF7R_P6xQejzDPOm-s6dLhGMeA,4383
 flowcept/flowcept_api/__init__.py,sha256=T1ty86YlocQ5Z18l5fUqHj_CC6Unq_iBv0lFyiI7Ao8,22
 flowcept/flowcept_api/db_api.py,sha256=hKXep-n50rp9cAzV0ljk2QVEF8O64yxi3ujXv5_Ibac,9723
 flowcept/flowcept_api/flowcept_controller.py,sha256=lkHR7O0zAAfbGtVa4o9tjZMdZquYN7vdnymRKzc4B8s,11933
@@ -38,9 +38,9 @@ flowcept/flowcept_webserver/resources/__init__.py,sha256=XOk5yhLeLU6JmVXxbl3TY2z
 flowcept/flowcept_webserver/resources/query_rsrc.py,sha256=Mk1XDC_wVYkMk0eaazqWWrTC07gQU9U0toKfip0ihZE,1353
 flowcept/flowcept_webserver/resources/task_messages_rsrc.py,sha256=0u68it2W-9NzUUx5fWOZCqvRKe5EsLI8oyvto9634Ng,666
 flowcept/flowceptor/__init__.py,sha256=wVxRXUv07iNx6SMRRma2vqhR_GIcRl0re_WCYG65PUs,29
-flowcept/flowceptor/telemetry_capture.py,sha256=FlWGLpzb6pBJOKVi349kyZKzAHeTsL86BCQd4Wtxpig,13746
+flowcept/flowceptor/telemetry_capture.py,sha256=wSXyQJ-vPVzeldD4KqoLQA2rg7V0EOQo_11ErJE5oQQ,13743
 flowcept/flowceptor/adapters/__init__.py,sha256=SuZbSZVVQeBJ9zXW-M9jF09dw3XIjre3lSGrUO1Y8Po,27
-flowcept/flowceptor/adapters/base_interceptor.py,sha256=99a_Ipnj6g8qZMHWLBEYJh0Cox033ADxOKPFrivr9gw,6056
+flowcept/flowceptor/adapters/base_interceptor.py,sha256=a2CX7COCpYzIpQeVulrLJTSVIw453U-S2gmrMlouO5A,6487
 flowcept/flowceptor/adapters/instrumentation_interceptor.py,sha256=DhK2bBnpghqPSeA62BUqRg6pl8zxuYrP33dK4x6PhRE,733
 flowcept/flowceptor/adapters/interceptor_state_manager.py,sha256=xRzmi5YFKBEqNtX8F5s6XlMTRe27ml4BmQtBO4WtG2c,919
 flowcept/flowceptor/adapters/dask/__init__.py,sha256=GKreb5L_nliD2BEckyB943zOQ-b6Gn1fLDj81FqSK2Y,23
@@ -60,14 +60,14 @@ flowcept/flowceptor/adapters/zambeze/zambeze_dataclasses.py,sha256=nn9MxvcdzgmOa
 flowcept/flowceptor/adapters/zambeze/zambeze_interceptor.py,sha256=Bjyi48JW0DXJLJuvwPxaD8zxxsSoEFgSoXl8YcbwFWk,3782
 flowcept/flowceptor/consumers/__init__.py,sha256=foxtVEb2ZEe9g1slfYIKM4tIFv-He1l7XS--SYs7nlQ,28
 flowcept/flowceptor/consumers/consumer_utils.py,sha256=7bvFJWusJkfA4j0gwZLDIIsIOyfk9wRq6s5liS3JAV0,5665
-flowcept/flowceptor/consumers/document_inserter.py,sha256=rAK3rs3VNW5a6koesE05scQ1mR_4BhuxLurP10ipURs,9339
+flowcept/flowceptor/consumers/document_inserter.py,sha256=fNPLa25oNhr3Y6-pRvzRp1zO4j3WBg7YXRnSHyDaaCo,9568
 flowcept/instrumentation/__init__.py,sha256=M5bTmg80E4QyN91gUX3qfw_nbtJSXwGWcKxdZP3vJz0,34
 flowcept/instrumentation/flowcept_loop.py,sha256=RvETm3Pn37dIw_a1RXigyh2U7MCBHqi46dPmbrz3RMQ,12171
 flowcept/instrumentation/flowcept_task.py,sha256=l_BAYEUZ_SeBt8QJN_E9D9QcZVYRnW9qO_XRnqvmePE,5993
-flowcept/instrumentation/flowcept_torch.py,sha256=KXA1HBwz8l5Qp7PkZ7nsbYlM8IcwWD_u04NxaAcZPzM,23395
-flowcept/instrumentation/task_capture.py,sha256=u82r_SgzoVKyb6_SWtfB-meBUZgjrXvF5dxkH9vnMDs,4776
-resources/sample_settings.yaml,sha256=aZRAZRkgCe52i-8czQvZsEIAz8dGau-OF2YClUF3QGs,3427
-flowcept-0.8.5.dist-info/METADATA,sha256=uZzz3Hz-Pee3zpeyNYLk5K-JbS_miCTMXIXlqtMBXX0,17543
-flowcept-0.8.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-flowcept-0.8.5.dist-info/licenses/LICENSE,sha256=r5-2P6tFTuRGWT5TiX32s1y0tnp4cIqBEC1QjTaXe2k,1086
-flowcept-0.8.5.dist-info/RECORD,,
+flowcept/instrumentation/flowcept_torch.py,sha256=mH4sI2FMtBpGk4hN3U6MUwqd6sOPER8TbigUkexfhDY,23437
+flowcept/instrumentation/task_capture.py,sha256=fbTAhf4y69pRCpnaH8r0dczSmPyNINSpljMrVyUnp0U,4945
+resources/sample_settings.yaml,sha256=aKeHf8895vrHIbi0QS1w2WT5n8ZNI9Ep5PVPF5Y5MEQ,4957
+flowcept-0.8.7.dist-info/METADATA,sha256=xRNvugAeW4JZkzUDzpYc96qOnxyu0-abqV6p6RArHZA,18086
+flowcept-0.8.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+flowcept-0.8.7.dist-info/licenses/LICENSE,sha256=r5-2P6tFTuRGWT5TiX32s1y0tnp4cIqBEC1QjTaXe2k,1086
+flowcept-0.8.7.dist-info/RECORD,,

resources/sample_settings.yaml

@@ -1,18 +1,19 @@
+flowcept_version: 0.8.0 # Version of the Flowcept package. This setting file is compatible with this version.
+
 project:
-  debug: true
-  json_serializer: default # or complex. If "complex", Flowcept will deal with complex python dicts that may contain JSON unserializable values
-  replace_non_json_serializable: true
-  performance_logging: false
-  register_workflow: true
-  enrich_messages: true
-  db_flush_mode: online   # or offline
+  debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.
+  json_serializer: default # JSON serialization mode: default or complex. If "complex", Flowcept will deal with complex python dicts that may contain JSON unserializable values
+  replace_non_json_serializable: true # Replace values that can't be JSON serialized
+  performance_logging: false # Enable performance logging if true. Particularly useful for MQ flushes.
+  enrich_messages: true # Add extra metadata to task messages, such as IP addresses and UTC timestamps.
+  db_flush_mode: online # Mode for flushing DB entries: "online" or "offline". If online, flushes to the DB will happen before the workflow ends.
 
 log:
-  log_path: "default"
-  log_file_level: error # use 'disable' to disable logs
-  log_stream_level: error
+  log_path: "default" # Path for log file output; "default" will write the log in the directory where the main executable is running from.
+  log_file_level: error # Logging level (error, debug, info, critical) for file logs; use "disable" to turn off.
+  log_stream_level: error # Logging level (error, debug, info, critical) for console/stream logs; use "disable" to turn off.
 
-telemetry_capture:
+telemetry_capture: # This toggles each individual type of telemetry capture. GPU capture is treated different depending on the vendor (AMD or NVIDIA).
   gpu: ~ # ~ means None. This is a list with GPU metrics. AMD=[activity,used,power,temperature,others,id]; NVIDIA=[used,temperature,power,name,id]
   cpu: true
   per_cpu: true
@@ -23,17 +24,18 @@ telemetry_capture:
   machine_info: true
 
 instrumentation:
-  enabled: true
+  enabled: true # This toggles data capture for instrumentation.
+  singleton: true # Use a single instrumentation instance per process. Defaults to true
   torch:
-    what: parent_and_children # parent_only, parent_and_children, ~
-    children_mode: telemetry_and_tensor_inspection   # tensor_inspection, telemetry, telemetry_and_tensor_inspection
+    what: parent_and_children # Scope of instrumentation: "parent_only" -- will capture only at the main model level, "parent_and_children" -- will capture the inner layers, or ~ (disable).
+    children_mode: telemetry_and_tensor_inspection   # What to capture if parent_and_children is chosen in the scope. Possible values: "tensor_inspection" (i.e., tensor metadata), "telemetry", "telemetry_and_tensor_inspection"
     epoch_loop: lightweight # lightweight, ~ (disable), or default (default will use the default telemetry capture method)
     batch_loop: lightweight # lightweight, ~ (disable), or default (default will use the default telemetry capture method)
-    capture_epochs_at_every: 1 #epochs; please use a value that is multiple of #epochs
-    # enable to set between train, evaluate, and test
+    capture_epochs_at_every: 1 # Will capture data at every N epochs; please use a value that is multiple of the total number of #epochs.
+    register_workflow: true # Will store the parent model forward as a workflow itself in the database.
 
 experiment:
-  user: root
+  user: root  # Optionally identify the user running the experiment. The logged username will be captured anyways.
 
 mq:
   type: redis  # or kafka or mofka; Please adjust the port (kafka's default is 9092; redis is 6379). If mofka, adjust the group_file.