dftracer 2.1.1.post48.dev0__tar.gz → 2.2.0.post1.dev0__tar.gz

{dftracer-2.1.1.post48.dev0 → dftracer-2.2.0.post1.dev0}/CMakeLists.txt

@@ -1040,6 +1040,7 @@ else()
                 ${CMAKE_CURRENT_SOURCE_DIR}/include/dftracer/dftracer.h
                 ${CMAKE_CURRENT_SOURCE_DIR}/include/dftracer/core/common/constants.h
                 ${CMAKE_CURRENT_SOURCE_DIR}/include/dftracer/core/common/typedef.h
+                ${CMAKE_CURRENT_SOURCE_DIR}/include/dftracer/core/common/entity.h
                 ${CMAKE_CURRENT_SOURCE_DIR}/include/dftracer/core/common/cpp_typedefs.h
                 ${CMAKE_CURRENT_SOURCE_DIR}/include/dftracer/core/common/enumeration.h)
         set(DFTRACER_CORE_PRIVATE_INCLUDE

{dftracer-2.1.1.post48.dev0/python/dftracer.egg-info → dftracer-2.2.0.post1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dftracer
-Version: 2.1.1.post48.dev0
+Version: 2.2.0.post1.dev0
 Summary: I/O profiler for deep learning python apps. Specifically for dlio_benchmark.
 Author-email: "Hariharan Devarajan (Hari)" <hariharandev1@llnl.gov>
 License-Expression: MIT

{dftracer-2.1.1.post48.dev0 → dftracer-2.2.0.post1.dev0}/docs/index.rst

@@ -26,6 +26,7 @@ DFTracer: is a library for profiling I/O calls and application functions.
    api
    models
    trace_format
+   provenance
    gpu_tracing
    ai_logging
 

dftracer-2.2.0.post1.dev0/docs/provenance.rst

@@ -0,0 +1,213 @@
+================================
+Entities, Relations & Provenance
+================================
+
+DFTracer can record **what data an application works with and how each piece
+of data came to be**, alongside its usual timing and I/O events. The same API
+serves provenance (lineage of scientific artifacts) and performance analysis
+(which events touched which data instances).
+
+.. contents::
+   :local:
+   :depth: 2
+
+----------------------------------------
+What it captures
+----------------------------------------
+
+**Entity**
+   A typed instance of something the application works with: an input file, a
+   dataset shard, a checkpoint, a model, a predicted protein structure, a GPU
+   buffer. Identified by ``(type, key)``: the *type* is an application-defined
+   class name, the *key* is any application identifier for the instance (a
+   path, a sample id, ``step=120``). The key is hashed into a 64-bit
+   ``EntityID``; it is never stored, so keys may be long or sensitive.
+
+**Entity type**
+   Described once per process with a **role** (input, output, intermediate,
+   parameter, reference) and a short description of what it represents.
+   Roles let users ask "what went in", "what came out", "what happened
+   between".
+
+**Event relation**
+   An event (a traced function or region) relates to entities: it **used**
+   (consumed), **generated** (produced), **invalidated** (deleted or ended) or
+   **updated** (modified in place) them. Any event may carry relations, in any
+   category -- a C++ function, a Python region, an MPI call.
+
+**Entity relation**
+   Two entities relate directly: **derived_from**, **revision_of**,
+   **contains** (has-a), **part_of** (is-in), **specialization_of**
+   (inherits / is-a), **alternate_of**, **depends_on**.
+
+Together these form a graph: entities and events as nodes, relations as edges.
+Walking ``generated`` edges backwards from an artifact gives its lineage;
+walking ``used`` edges forwards gives the impact of an input.
+
+Design rules
+   * Enums wherever possible; strings only where unavoidable, and then
+     fixed-length and JSON-safe.
+   * Identity is computed by the core (FNV-1a-64 of ``type`` 0x1f ``key``), so
+     the same ``(type, key)`` gets the same ``EntityID`` in C, C++ and Python,
+     in every process and on every node.
+   * Each entity and each entity type is recorded **once per process**; later
+     declarations cost a set lookup. The first declaration's store and uri win.
+   * Relations are the event's content and are written even when
+     ``DFTRACER_INC_METADATA`` is off (other optional event args still need it).
+
+----------------------------------------
+Types and enums
+----------------------------------------
+
+Defined in ``dftracer/core/common/entity.h`` (C-compatible) and
+``dftracer/core/common/typedef.h``.
+
+======================== ==================================================
+Type                      Meaning
+======================== ==================================================
+``EntityID``              ``uint64_t``; 16 lowercase hex characters in traces
+``ConstEntityTypeName``   type name, at most ``DFT_ENTITY_TYPE_LEN - 1`` (31)
+``ConstEntityKey``        instance key, hashed into the id, not stored
+``ConstEntityURI``        locator unique within the app, at most 255 characters
+``ConstEntityDescription`` type description, at most 255 characters
+``EntityStore``           where the instance lives (enum below)
+``EntityRole``            role of an entity type (enum below)
+``EntityRelation``        event and entity relations (enum below)
+======================== ==================================================
+
+Strings are truncated to their capacity and restricted to ``A-Z a-z 0-9 _ - .
+: / @ + = , ~`` and space; any other character (quotes, backslash, ``|``,
+control characters) becomes ``_``.
+
+.. list-table::
+   :header-rows: 1
+
+   * - Enum
+     - Values (as written in traces)
+   * - ``EntityStore``
+     - 0 memory, 1 gpu_memory, 2 local_disk, 3 parallel_fs, 4 burst_buffer,
+       5 object_store, 6 database, 7 network, 8 other
+   * - ``EntityRole``
+     - 0 unknown, 1 input, 2 output, 3 intermediate, 4 parameter, 5 reference
+   * - ``EntityRelation`` (event)
+     - 0 used, 1 generated, 2 invalidated, 3 updated
+   * - ``EntityRelation`` (entity)
+     - 16 derived_from, 17 revision_of, 18 contains, 19 part_of,
+       20 specialization_of, 21 alternate_of, 22 depends_on
+
+Enum values are part of the trace format and are append-only.
+
+----------------------------------------
+C API
+----------------------------------------
+
+.. code-block:: c
+
+    #include <dftracer/dftracer.h>
+
+    DFTRACER_C_ENTITY_TYPE("sequence", DFT_ROLE_INPUT, "A query protein sequence");
+
+    void fold(const char* seq_id, const char* out_path) {
+      DFTRACER_C_FUNCTION_START();
+      DFTRACER_C_FUNCTION_USES("sequence", seq_id);            /* store: memory */
+      EntityID s = update_relation_entity(data_fn, DFT_REL_GENERATED,
+                                          "protein_structure", seq_id,
+                                          DFT_STORE_PARALLEL_FS, out_path);
+      DFTRACER_C_FUNCTION_END();
+    }
+
+    EntityID tar = DFTRACER_C_ENTITY("structure_tar", seq_id,
+                                     DFT_STORE_PARALLEL_FS, tar_path);
+    DFTRACER_C_ENTITY_RELATE(DFT_REL_CONTAINS, tar, s);
+
+Functions: ``dftracer_declare_entity``, ``dftracer_declare_entity_type``,
+``dftracer_relate_entities``, ``update_relation``, ``update_relation_entity``.
+Macros: ``DFTRACER_C_ENTITY``, ``DFTRACER_C_ENTITY_TYPE``,
+``DFTRACER_C_ENTITY_RELATE``, ``DFTRACER_C_FUNCTION_RELATE``,
+``DFTRACER_C_FUNCTION_USES``, ``DFTRACER_C_FUNCTION_GENERATES``,
+``DFTRACER_C_REGION_RELATE``.
+
+----------------------------------------
+C++ API
+----------------------------------------
+
+.. code-block:: cpp
+
+    #include <dftracer/dftracer.h>
+
+    void relax(const std::string& seq, const std::string& path) {
+      DFTRACER_CPP_FUNCTION();
+      EntityID raw = profiler_dft_fn.uses("unrelaxed_structure", seq.c_str());
+      EntityID out = profiler_dft_fn.generates("relaxed_structure", seq.c_str(),
+                                               DFT_STORE_BURST_BUFFER, path.c_str());
+      DFTRACER_CPP_ENTITY_RELATE(DFT_REL_REVISION_OF, out, raw);
+    }
+
+``DFTracer`` gains ``relate(EntityRelation, EntityID)``,
+``relate(EntityRelation, type, key, store, uri)``, ``uses(...)`` and
+``generates(...)``; macros ``DFTRACER_CPP_ENTITY``, ``DFTRACER_CPP_ENTITY_TYPE``,
+``DFTRACER_CPP_ENTITY_RELATE``, ``DFTRACER_CPP_FUNCTION_RELATE``,
+``DFTRACER_CPP_FUNCTION_USES``, ``DFTRACER_CPP_FUNCTION_GENERATES``,
+``DFTRACER_CPP_REGION_RELATE``.
+
+----------------------------------------
+Python API
+----------------------------------------
+
+.. code-block:: python
+
+    from dftracer.python import (dftracer, dft_fn, EntityStore, EntityRole,
+                                 EntityRelation)
+
+    log = dftracer.initialize_log(logfile=None, data_dir=None, process_id=-1)
+    log.declare_entity_type("filtered_msa", EntityRole.OUTPUT,
+                            "Diversity-filtered multiple sequence alignment")
+
+    with dft_fn("APP", name="filter_msa") as f:
+        merged = f.uses("merged_msa", seq_id, EntityStore.BURST_BUFFER, merged_path)
+        filtered = f.generates("filtered_msa", seq_id, EntityStore.BURST_BUFFER, out_path)
+        f.relate_entity(EntityRelation.INVALIDATED, "merged_msa", seq_id)  # deleted
+
+    log.relate_entities(EntityRelation.DERIVED_FROM, filtered, merged)
+
+``dftracer`` gains ``declare_entity`` (returns the id as an ``int``, 0 when
+tracing is off), ``declare_entity_type`` and ``relate_entities``; ``dft_fn``
+gains ``uses``, ``generates``, ``relate_entity`` and ``relate``. The native
+module also exposes ``entity_hash(type, key)`` and ``entity_hex(id)`` to compute
+ids without tracing.
+
+----------------------------------------
+Trace records
+----------------------------------------
+
+Entities, entity types and entity relations are metadata records
+(``"ph":4``, ``"cat":"dftracer"``, ``"type":1``) with named fields:
+
+.. code-block:: json
+
+    {"name":"EH","cat":"dftracer","type":1,"pid":1,"tid":1,"ph":4,
+     "args":{"hhash":"…","id":"8f0de68c04eea0bf","type":"protein_structure",
+             "store":3,"uri":"/p/out/a.pdb"}}
+    {"name":"ET","cat":"dftracer","type":1,"pid":1,"tid":1,"ph":4,
+     "args":{"hhash":"…","type":"protein_structure","role":2,
+             "description":"Predicted 3D structure"}}
+    {"name":"ER","cat":"dftracer","type":1,"pid":1,"tid":1,"ph":4,
+     "args":{"hhash":"…","relation":18,"subject":"6e6d650dd131e83b",
+             "object":"8f0de68c04eea0bf"}}
+
+An event's relations are one ``relations`` object in its ``args``: relation
+name to an array of entity ids. Nesting them keeps relation names from
+colliding with other event args (the lifecycle ``end`` event has its own
+``used`` object, for example):
+
+.. code-block:: json
+
+    {"id":2,"name":"fold","cat":"CPP_APP","type":9,"pid":1,"tid":1,
+     "ts":1790877632269434,"dur":32,"ph":1,
+     "args":{"hhash":"…","relations":{"used":["d824b0bb91487885"],
+                                       "generated":["8f0de68c04eea0bf"]}}}
+
+A reader joins events to entities by id; ids from different processes and
+languages join without coordination. ``dftracer-utils`` builds the graph
+(``trace::provenance::extract_provenance_graph``, ``GET /api/prov/graph`` and
+the *Provenance* tab of ``dftracer_server``).

{dftracer-2.1.1.post48.dev0 → dftracer-2.2.0.post1.dev0}/docs/trace_format.rst

@@ -226,6 +226,22 @@ An example of such event is below.
 
 Here, the "name" "PR" represents a metadata event for process and the args contain the metadata name and its value.
 
+.. _`Entity Records`:
+
+Entity records (``EH``, ``ET``, ``ER``) describe the entities an application
+works with, their types, and relations between entities; events relate to
+entities through a ``relations`` object in their ``args`` (``used`` /
+``generated`` / ``invalidated`` / ``updated`` arrays of entity ids). Unlike hash and custom metadata, these
+records use named fields rather than ``name``/``value``:
+
+.. code-block:: bash
+
+    {"name":"EH","cat":"dftracer","type":1,"pid":1,"tid":1,"ph":4,"args":{"hhash":"...","id":"8f0de68c04eea0bf","type":"protein_structure","store":3,"uri":"/p/out/a.pdb"}}
+    {"name":"ET","cat":"dftracer","type":1,"pid":1,"tid":1,"ph":4,"args":{"hhash":"...","type":"protein_structure","role":2,"description":"Predicted 3D structure"}}
+    {"name":"ER","cat":"dftracer","type":1,"pid":1,"tid":1,"ph":4,"args":{"hhash":"...","relation":18,"subject":"6e6d650dd131e83b","object":"8f0de68c04eea0bf"}}
+
+See :doc:`provenance` for the API, the enum values and what each record means.
+
 ----------------------------------------
 Lifecycle Events: "start" and "end"
 ----------------------------------------

{dftracer-2.1.1.post48.dev0 → dftracer-2.2.0.post1.dev0}/examples/dfanalyzer/dfanalyzer-ideas.ipynb

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
-name: dftracer-2.1.1.post48.dev0/examples/dfanalyzer/dfanalyzer-ideas.ipynb
+name: dftracer-2.2.0.post1.dev0/examples/dfanalyzer/dfanalyzer-ideas.ipynb
 size: 52418033 bytes
 sha256: e4b283c4e2e7dd17e9f5452b2fb101d1d9f32d4dd13fa56742629faea6700b55

dftracer-2.2.0.post1.dev0/include/dftracer/core/common/entity.h

@@ -0,0 +1,192 @@
+#ifndef DFTRACER_CORE_COMMON_ENTITY_H
+#define DFTRACER_CORE_COMMON_ENTITY_H
+/*
+ * Entities and relations -- shared definitions for the C, C++ and Python APIs.
+ *
+ * An entity is a typed instance of something an application works with (a
+ * file, a dataset shard, a checkpoint, a protein structure, a GPU buffer).
+ *
+ *   identity   EntityID         64-bit FNV-1a of (type, key); serialized as
+ *                               exactly 16 lowercase hex characters.
+ *   type       fixed-length string (DFT_ENTITY_TYPE_LEN), application-defined
+ *   store      EntityStore enum   -- where the instance lives
+ *   uri        fixed-length string (DFT_ENTITY_URI_LEN), unique within the app
+ *   role       EntityRole enum     -- per entity TYPE (input/output/...)
+ *   description fixed-length string (DFT_ENTITY_DESC_LEN), per entity TYPE
+ *   relation   EntityRelation enum -- event->entity and entity->entity
+ *
+ * Enums are serialized as integers. Strings are copied into fixed buffers,
+ * truncated, and restricted to a JSON-safe character set (dft_entity_sanitize)
+ * so records never need escaping.
+ *
+ * Trace records (metadata, like the FH/SH file/string hash records):
+ *   EH  args: {"id":"<16-hex>","type":"<type>","store":<int>,"uri":"<uri>"}
+ *   ET  args: {"type":"<type>","role":<int>,"description":"<text>"}
+ *   ER  args: {"relation":<int>,"subject":"<16-hex>","object":"<16-hex>"}
+ * Event->entity relations are event args keyed by dft_relation_name(rel)
+ * whose value is a JSON array of 16-hex ids.
+ */
+#include <dftracer/core/common/typedef.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Fixed string capacities, including the terminating NUL. */
+#define DFT_ENTITY_TYPE_LEN 32
+#define DFT_ENTITY_URI_LEN 256
+#define DFT_ENTITY_DESC_LEN 256
+/* 16 hex digits + NUL. */
+#define DFT_ENTITY_HEX_LEN 17
+
+#define DFT_ENTITY_NONE ((EntityID)0)
+
+/* Where an entity instance lives. */
+typedef enum {
+  DFT_STORE_MEMORY = 0,
+  DFT_STORE_GPU_MEMORY = 1,
+  DFT_STORE_LOCAL_DISK = 2,
+  DFT_STORE_PARALLEL_FS = 3,
+  DFT_STORE_BURST_BUFFER = 4,
+  DFT_STORE_OBJECT_STORE = 5,
+  DFT_STORE_DATABASE = 6,
+  DFT_STORE_NETWORK = 7,
+  DFT_STORE_OTHER = 8
+} EntityStore;
+
+/* Role of an entity TYPE in the workflow. */
+typedef enum {
+  DFT_ROLE_UNKNOWN = 0,
+  DFT_ROLE_INPUT = 1,        /* enters the workflow from outside */
+  DFT_ROLE_OUTPUT = 2,       /* a deliverable / scientific artifact */
+  DFT_ROLE_INTERMEDIATE = 3, /* produced and consumed inside the workflow */
+  DFT_ROLE_PARAMETER = 4,    /* configuration, seeds, model settings */
+  DFT_ROLE_REFERENCE = 5     /* versioned reference data (databases, weights) */
+} EntityRole;
+
+/* Relations. Values < 16 relate an EVENT to an entity (event args); values
+ * >= 16 relate two ENTITIES (ER records). */
+typedef enum {
+  /* event -> entity (W3C PROV style) */
+  DFT_REL_USED = 0,        /* the event consumed the entity (cause) */
+  DFT_REL_GENERATED = 1,   /* the event produced the entity (effect) */
+  DFT_REL_INVALIDATED = 2, /* the event ended the entity (delete, consume) */
+  DFT_REL_UPDATED = 3,     /* the event modified the entity in place */
+  /* entity -> entity (class-style) */
+  DFT_REL_DERIVED_FROM = 16,      /* subject was derived from object */
+  DFT_REL_REVISION_OF = 17,       /* subject is a newer version of object */
+  DFT_REL_CONTAINS = 18,          /* subject has object as a member (has-a) */
+  DFT_REL_PART_OF = 19,           /* subject is part of object (is-in) */
+  DFT_REL_SPECIALIZATION_OF = 20, /* subject is a kind of object (inherits) */
+  DFT_REL_ALTERNATE_OF = 21,      /* same thing, another representation */
+  DFT_REL_DEPENDS_ON = 22         /* subject requires object */
+} EntityRelation;
+
+static inline int dft_relation_is_event(EntityRelation r) { return r < 16; }
+
+/* Event arg holding an event's relations:
+ *   "relations": {"used":["<16-hex>",...], "generated":[...], ...}
+ * One nested object, so relation names never collide with other event
+ * args (the lifecycle "end" event has its own "used" object). */
+#define DFT_RELATIONS_ARG "relations"
+
+/* Stable relation names (part of the trace format). */
+static inline const char* dft_relation_name(EntityRelation r) {
+  switch (r) {
+    case DFT_REL_USED:
+      return "used";
+    case DFT_REL_GENERATED:
+      return "generated";
+    case DFT_REL_INVALIDATED:
+      return "invalidated";
+    case DFT_REL_UPDATED:
+      return "updated";
+    case DFT_REL_DERIVED_FROM:
+      return "derived_from";
+    case DFT_REL_REVISION_OF:
+      return "revision_of";
+    case DFT_REL_CONTAINS:
+      return "contains";
+    case DFT_REL_PART_OF:
+      return "part_of";
+    case DFT_REL_SPECIALIZATION_OF:
+      return "specialization_of";
+    case DFT_REL_ALTERNATE_OF:
+      return "alternate_of";
+    case DFT_REL_DEPENDS_ON:
+      return "depends_on";
+  }
+  return "related";
+}
+
+/* JSON-safe character set; everything else (quotes, backslash, control
+ * characters, '|', ...) becomes '_', so fields never need escaping. */
+static inline int dft_entity_safe_char(char c) {
+  if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+      (c >= '0' && c <= '9'))
+    return 1;
+  switch (c) {
+    case '_':
+    case '-':
+    case '.':
+    case ':':
+    case '/':
+    case '@':
+    case '+':
+    case '=':
+    case ',':
+    case '~':
+    case ' ':
+      return 1;
+    default:
+      return 0;
+  }
+}
+
+/* Copy src into dst[cap] (truncating) with the safe character set. */
+static inline void dft_entity_sanitize(char* dst, size_t cap, const char* src) {
+  size_t i = 0;
+  if (cap == 0) return;
+  if (src != NULL)
+    for (; src[i] != '\0' && i + 1 < cap; ++i)
+      dst[i] = dft_entity_safe_char(src[i]) ? src[i] : '_';
+  dst[i] = '\0';
+}
+
+/* 64-bit FNV-1a of type, a 0x1f separator, and key. Same value in every
+ * language and process; 0 is reserved for DFT_ENTITY_NONE. */
+static inline EntityID dft_entity_hash(const char* type, const char* key) {
+  uint64_t h = 1469598103934665603ULL;
+  const unsigned char* p;
+  for (p = (const unsigned char*)(type ? type : ""); *p; ++p) {
+    h ^= *p;
+    h *= 1099511628211ULL;
+  }
+  h ^= 0x1fu;
+  h *= 1099511628211ULL;
+  for (p = (const unsigned char*)(key ? key : ""); *p; ++p) {
+    h ^= *p;
+    h *= 1099511628211ULL;
+  }
+  return h == DFT_ENTITY_NONE ? 1 : h;
+}
+
+/* Render an id as exactly 16 lowercase hex digits into out[DFT_ENTITY_HEX_LEN].
+ */
+static inline void dft_entity_hex(EntityID id, char* out) {
+  static const char digits[] = "0123456789abcdef";
+  int i;
+  for (i = 15; i >= 0; --i) {
+    out[i] = digits[id & 0xfu];
+    id >>= 4;
+  }
+  out[16] = '\0';
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* DFTRACER_CORE_COMMON_ENTITY_H */

{dftracer-2.1.1.post48.dev0 → dftracer-2.2.0.post1.dev0}/include/dftracer/core/common/enumeration.h

@@ -259,5 +259,12 @@ inline double time_metric_units_per_second(const TimeMetricType& type) {
 #define METADATA_NAME_FILE_HASH "FH"
 #define METADATA_NAME_HOSTNAME_HASH "HH"
 #define METADATA_NAME_STRING_HASH "SH"
+// Entities (see dftracer/core/common/entity.h):
+//   EH  args: {"id":"<16-hex>","type":"<type>","store":<int>,"uri":"<uri>"}
+//   ET  args: {"type":"<type>","role":<int>,"description":"<text>"}
+//   ER  args: {"relation":<int>,"subject":"<16-hex>","object":"<16-hex>"}
+#define METADATA_NAME_ENTITY_HASH "EH"
+#define METADATA_NAME_ENTITY_TYPE "ET"
+#define METADATA_NAME_ENTITY_RELATION "ER"
 #define CUSTOM_METADATA "CM"
 #endif  // DFTRACER_ENUMERATION_H

dftracer-2.2.0.post1.dev0/include/dftracer/core/common/typedef.h

@@ -0,0 +1,22 @@
+//
+// Created by haridev on 10/7/23.
+//
+
+#ifndef DFTRACER_TYPEDEF_H
+#define DFTRACER_TYPEDEF_H
+#include <stdint.h>
+typedef unsigned long long int TimeResolution;
+typedef unsigned long int ThreadID;
+typedef int ProcessID;
+typedef char* EventNameType;
+typedef const char* ConstEventNameType;
+typedef char* HashType;
+// Entities (see core/common/entity.h). Ids are 64-bit and rendered as 16 hex
+// characters only when written; strings are bounded by the DFT_ENTITY_*_LEN
+// capacities and sanitized to a JSON-safe character set.
+typedef uint64_t EntityID;
+typedef const char* ConstEntityTypeName;     // <= DFT_ENTITY_TYPE_LEN
+typedef const char* ConstEntityKey;          // hashed into EntityID, not stored
+typedef const char* ConstEntityURI;          // <= DFT_ENTITY_URI_LEN
+typedef const char* ConstEntityDescription;  // <= DFT_ENTITY_DESC_LEN
+#endif                                       // DFTRACER_TYPEDEF_H

{dftracer-2.1.1.post48.dev0 → dftracer-2.2.0.post1.dev0}/include/dftracer/dftracer.h

@@ -9,6 +9,7 @@
  * Common to both C and CPP
  */
 #include <dftracer/core/common/constants.h>
+#include <dftracer/core/common/entity.h>
 #include <dftracer/core/common/typedef.h>
 #define DF_DATA_EVENT 0
 #define DF_METADATA_EVENT 1
@@ -38,6 +39,25 @@ void set_app_metadata_string(const char* key, const char* value);
 // log as PYTHON, so each calls this with its own name to tell them apart.
 // Idempotent; safe to call on every invocation.
 void mark_used(const char* name);
+
+// ---------------------------------------------------------------------------
+// Entities and relations (types, enums and limits: core/common/entity.h).
+//
+// Declare an entity instance: (type, key) -> 64-bit id, recorded once per
+// process as an EH record. `type` is application-defined (truncated to
+// DFT_ENTITY_TYPE_LEN), `key` is any application identifier (hashed, never
+// stored), `uri` is an optional locator unique within the app (truncated to
+// DFT_ENTITY_URI_LEN). Returns DFT_ENTITY_NONE when tracing is off.
+EntityID dftracer_declare_entity(ConstEntityTypeName type, ConstEntityKey key,
+                                 EntityStore store, ConstEntityURI uri);
+// Describe an entity type once per process: its role and what it represents
+// (description truncated to DFT_ENTITY_DESC_LEN). One ET record.
+void dftracer_declare_entity_type(ConstEntityTypeName type, EntityRole role,
+                                  ConstEntityDescription description);
+// Relate two entities (entity -> entity relations, e.g. DFT_REL_CONTAINS,
+// DFT_REL_DERIVED_FROM). One ER record.
+void dftracer_relate_entities(EntityRelation relation, EntityID subject,
+                              EntityID object);
 #ifdef __cplusplus
 }
 #endif
@@ -62,6 +82,7 @@ class DFTracer {
   TraceEventType type;
   TimeResolution start_time;
   dftracer::Metadata* metadata;
+  void* relations;  // relation name -> entity hashes; see relate()
 
  public:
   DFTracer(ConstEventNameType _name, ConstEventNameType _cat,
@@ -74,6 +95,25 @@ class DFTracer {
   void update(const char* key, const char* value,
               MetadataType type = MetadataType::MT_KEY);
 
+  // Relate this event to an entity under an event relation (DFT_REL_USED,
+  // DFT_REL_GENERATED, DFT_REL_INVALIDATED, DFT_REL_UPDATED). Ids are kept
+  // as integers and rendered to hex only when the event is written.
+  void relate(EntityRelation relation, EntityID entity);
+  // Declare (type, key) and relate it in one step; returns the entity id.
+  EntityID relate(EntityRelation relation, ConstEntityTypeName type,
+                  ConstEntityKey key, EntityStore store = DFT_STORE_MEMORY,
+                  ConstEntityURI uri = nullptr);
+  EntityID uses(ConstEntityTypeName type, ConstEntityKey key,
+                EntityStore store = DFT_STORE_MEMORY,
+                ConstEntityURI uri = nullptr) {
+    return relate(DFT_REL_USED, type, key, store, uri);
+  }
+  EntityID generates(ConstEntityTypeName type, ConstEntityKey key,
+                     EntityStore store = DFT_STORE_MEMORY,
+                     ConstEntityURI uri = nullptr) {
+    return relate(DFT_REL_GENERATED, type, key, store, uri);
+  }
+
   void finalize();
 
   ~DFTracer();
@@ -125,6 +165,23 @@ class DFTracer {
 #define DFTRACER_CPP_REGION_DYN_UPDATE_TYPE(name, key, val, type) \
   profiler_##name->update(key, val, type);
 
+// Entities and relations (see dftracer_declare_entity, core/common/entity.h).
+#define DFTRACER_CPP_ENTITY(type, key, store, uri) \
+  dftracer_declare_entity(type, key, store, uri)
+#define DFTRACER_CPP_ENTITY_TYPE(type, role, description) \
+  dftracer_declare_entity_type(type, role, description);
+#define DFTRACER_CPP_ENTITY_RELATE(relation, subject, object) \
+  dftracer_relate_entities(relation, subject, object);
+#define DFTRACER_CPP_FUNCTION_RELATE(relation, entity) \
+  profiler_dft_fn.relate(relation, entity);
+#define DFTRACER_CPP_FUNCTION_USES(type, key) profiler_dft_fn.uses(type, key);
+#define DFTRACER_CPP_FUNCTION_GENERATES(type, key) \
+  profiler_dft_fn.generates(type, key);
+#define DFTRACER_CPP_REGION_RELATE(name, relation, entity) \
+  profiler_##name.relate(relation, entity);
+#define DFTRACER_CPP_REGION_DYN_RELATE(name, relation, entity) \
+  profiler_##name->relate(relation, entity);
+
 extern "C" {
 #endif
 // C APIs
@@ -147,6 +204,15 @@ void update_metadata_int_type(struct DFTracerData* data, const char* key,
 void update_metadata_string_type(struct DFTracerData* data, const char* key,
                                  const char* value, int type);
 
+// Relate a region to an entity under an event relation, or declare
+// (type, key) and relate it in one call (returns the entity id).
+void update_relation(struct DFTracerData* data, EntityRelation relation,
+                     EntityID entity);
+EntityID update_relation_entity(struct DFTracerData* data,
+                                EntityRelation relation,
+                                ConstEntityTypeName type, ConstEntityKey key,
+                                EntityStore store, ConstEntityURI uri);
+
 #define DFTRACER_C_INIT(log_file, data_dirs, process_id) \
   initialize_main(log_file, data_dirs, process_id);      \
   mark_used("C_APP");
@@ -212,6 +278,24 @@ void update_metadata_string_type(struct DFTracerData* data, const char* key,
 #define DFTRACER_C_REGION_UPDATE_STR_TYPE(name, key, val, type) \
   update_metadata_string_type(data_##name, key, val, type);
 
+// Entities and relations (see dftracer_declare_entity, core/common/entity.h).
+#define DFTRACER_C_ENTITY(type, key, store, uri) \
+  dftracer_declare_entity(type, key, store, uri)
+#define DFTRACER_C_ENTITY_TYPE(type, role, description) \
+  dftracer_declare_entity_type(type, role, description);
+#define DFTRACER_C_ENTITY_RELATE(relation, subject, object) \
+  dftracer_relate_entities(relation, subject, object);
+#define DFTRACER_C_FUNCTION_RELATE(relation, entity) \
+  update_relation(data_fn, relation, entity);
+#define DFTRACER_C_FUNCTION_USES(type, key)                                  \
+  update_relation_entity(data_fn, DFT_REL_USED, type, key, DFT_STORE_MEMORY, \
+                         NULL);
+#define DFTRACER_C_FUNCTION_GENERATES(type, key)                \
+  update_relation_entity(data_fn, DFT_REL_GENERATED, type, key, \
+                         DFT_STORE_MEMORY, NULL);
+#define DFTRACER_C_REGION_RELATE(name, relation, entity) \
+  update_relation(data_##name, relation, entity);
+
 #ifdef __cplusplus
 }
 #endif

{dftracer-2.1.1.post48.dev0 → dftracer-2.2.0.post1.dev0}/python/dftracer/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '2.1.1.post48.dev0'
-__version_tuple__ = version_tuple = (2, 1, 1, 'post48', 'dev0')
+__version__ = version = '2.2.0.post1.dev0'
+__version_tuple__ = version_tuple = (2, 2, 0, 'post1', 'dev0')
 
 __commit_id__ = commit_id = None

{dftracer-2.1.1.post48.dev0 → dftracer-2.2.0.post1.dev0/python/dftracer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dftracer
-Version: 2.1.1.post48.dev0
+Version: 2.2.0.post1.dev0
 Summary: I/O profiler for deep learning python apps. Specifically for dlio_benchmark.
 Author-email: "Hariharan Devarajan (Hari)" <hariharandev1@llnl.gov>
 License-Expression: MIT