dftracer-utils 0.0.12.post135.dev0__tar.gz → 0.0.13.post1.dev0__tar.gz

{dftracer_utils-0.0.12.post135.dev0 → dftracer_utils-0.0.13.post1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dftracer-utils
-Version: 0.0.12.post135.dev0
+Version: 0.0.13.post1.dev0
 Summary: Python binding for dftracer-utils
 Author-Email: Ray Andrew Sinurat <raydreww@gmail.com>, "Hariharan Devarajan (Hari)" <hariharandev1@llnl.gov>
 License: MIT

dftracer_utils-0.0.13.post1.dev0/include/dftracer/utils/trace/provenance/provenance_graph.h

@@ -0,0 +1,94 @@
+#ifndef DFTRACER_UTILS_TRACE_PROVENANCE_PROVENANCE_GRAPH_H
+#define DFTRACER_UTILS_TRACE_PROVENANCE_PROVENANCE_GRAPH_H
+
+// Provenance graph extraction from dftracer traces.
+//
+// dftracer's provenance mode (dftracer_prov.h / dftracer_prov.py) records:
+//   * one metadata record per entity instance per process:
+//       args.name  = "prov_entity:<16-hex hash>"
+//       args.value = "<type>|<id>|<store>|<uri>"
+//     where hash = FNV-1a-64(type + 0x1f + id), identical in every language.
+//   * one event per activity, cat "PROV", whose args carry the edges:
+//       prov_aid       unique activity instance id
+//       prov_activity  activity type
+//       prov_used      comma-separated entity hashes consumed  (cause)
+//       prov_generated comma-separated entity hashes produced  (effect)
+//       prov_nused / prov_ngen   expected counts (detects truncation)
+//   * cat "PROV_CONT" events carrying overflowed list chunks, sharing prov_aid.
+//
+// extract_provenance_graph() runs one indexed, parallel scan through a
+// trace::views::View (the engine behind TraceViewer / dftracer_view),
+// dedupes entities across processes and folds PROV_CONT chunks into their
+// activity, so callers get the graph rather than raw events.
+
+#include <dftracer/utils/core/coro/task.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace dftracer::utils::trace::views {
+class View;
+}
+
+namespace dftracer::utils::trace::provenance {
+
+/// Query selecting every record provenance needs: activity events, their
+/// overflow chunks, and metadata (where entity declarations live).
+inline constexpr const char* kProvenanceQuery =
+    "cat == \"PROV\" or cat == \"PROV_CONT\" or ph == 4 or ph == \"M\"";
+
+struct ProvenanceEntity {
+    std::string hash;   // 16 hex digits
+    std::string type;   // entity type, e.g. "relaxed_structure"
+    std::string id;     // instance id, e.g. "MGYP001796749278/model_1_ptm"
+    std::string store;  // memory | file | db | object | network | ...
+    std::string uri;
+    std::vector<std::int64_t> pids;  // processes that declared it
+};
+
+struct ProvenanceActivity {
+    std::string aid;       // unique activity instance id
+    std::string name;      // event name
+    std::string activity;  // activity type
+    std::int64_t pid = 0;
+    std::uint64_t ts = 0;
+    std::uint64_t dur = 0;
+    std::vector<std::string> used;       // cause entity hashes
+    std::vector<std::string> generated;  // effect entity hashes
+    std::int64_t nused = -1;  // expected counts as reported (-1: absent)
+    std::int64_t ngen = -1;
+    bool has_main = false;  // false: only PROV_CONT chunks were seen
+};
+
+struct ProvenanceStats {
+    std::uint64_t entity_records = 0;  // prov_entity metadata records scanned
+    std::uint64_t prov_events = 0;
+    std::uint64_t cont_events = 0;
+    std::uint64_t dangling_hashes = 0;    // referenced, never declared
+    std::uint64_t isolated_entities = 0;  // declared, never referenced
+    std::uint64_t truncated_activities = 0;
+    std::uint64_t orphan_chunks = 0;  // PROV_CONT without its PROV event
+};
+
+struct ProvenanceGraph {
+    std::vector<ProvenanceEntity> entities;
+    std::vector<ProvenanceActivity> activities;
+    ProvenanceStats stats;
+
+    /// {"entities":[...],"activities":[...],"stats":{...}}
+    std::string to_json() const;
+};
+
+/// Scan `view` for provenance records and assemble the graph.
+///
+/// The view's existing scoping (files, time range, pid filters) is kept;
+/// this adds phase(Any) and the kProvenanceQuery filter. `num_slots` is the
+/// worker parallelism (0 -> 1).
+coro::CoroTask<ProvenanceGraph> extract_provenance_graph(
+    const views::View& view, std::size_t num_slots);
+
+}  // namespace dftracer::utils::trace::provenance
+
+#endif  // DFTRACER_UTILS_TRACE_PROVENANCE_PROVENANCE_GRAPH_H

{dftracer_utils-0.0.12.post135.dev0 → dftracer_utils-0.0.13.post1.dev0}/python/dftracer/utils/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.0.12.post135.dev0'
-__version_tuple__ = version_tuple = (0, 0, 12, 'post135', 'dev0')
+__version__ = version = '0.0.13.post1.dev0'
+__version_tuple__ = version_tuple = (0, 0, 13, 'post1', 'dev0')
 
 __commit_id__ = commit_id = None

{dftracer_utils-0.0.12.post135.dev0 → dftracer_utils-0.0.13.post1.dev0}/src/CMakeLists.txt

@@ -179,6 +179,7 @@ list(APPEND DFTRACER_UTILS_UTILITIES_SOURCES
     ${CMAKE_CURRENT_SOURCE_DIR}/dftracer/utils/trace/views/view_scanner_utility.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/dftracer/utils/trace/views/view_planner_utility.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/dftracer/utils/trace/views/view.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/dftracer/utils/trace/provenance/provenance_graph.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/dftracer/utils/trace/views/view_source.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/dftracer/utils/trace/views/view_plan_ops.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/dftracer/utils/trace/views/view_aggregate.cpp
@@ -280,6 +281,7 @@ list(APPEND DFTRACER_UTILS_UTILITIES_SOURCES
     ${CMAKE_CURRENT_SOURCE_DIR}/dftracer/utils/server/viz/histogram.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/dftracer/utils/server/viz/misc.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/dftracer/utils/server/viz/proctree.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/dftracer/utils/server/viz/prov.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/dftracer/utils/server/viz/stats.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/dftracer/utils/server/viz/summary_build.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/dftracer/utils/server/http_parser.cpp

{dftracer_utils-0.0.12.post135.dev0 → dftracer_utils-0.0.13.post1.dev0}/src/dftracer/utils/server/viz/api.cpp

@@ -73,6 +73,19 @@ void register_viz_api(Router& router, TraceIndex& index) {
             R"({"nodes":[{"pid":100,"parent":-1,"host":"node01","rank":"0",)"
             R"("bytes":16384,"io_ops":4,"io_busy":600.0}]})"});
 
+    router.get(
+        "/api/prov/graph", bind_index<handle_prov_graph>(index),
+        RouteDoc{"Provenance graph (entities, activities, cause/effect edges) "
+                 "assembled from dftracer provenance-mode records.",
+                 "Provenance",
+                 {{"file", "Limit to one trace file", false, ""}},
+                 R"({"entities":[{"hash":"f0f014be4d558b8f","type":"structure",)"
+                 R"("id":"out.pdb","store":"file","uri":"out.pdb","pids":[1]}],)"
+                 R"("activities":[{"aid":"1-a-1","name":"fold","activity":)"
+                 R"("compute","pid":1,"ts":0,"dur":5,"used":[],"generated":)"
+                 R"(["f0f014be4d558b8f"]}],"stats":{"entities":1,)"
+                 R"("activities":1,"dangling_hashes":0}})"});
+
     router.get(
         "/api/viz/counters", bind_index<handle_viz_counters>(index),
         RouteDoc{"Read/write bytes and I/O op counts per time bucket.",

{dftracer_utils-0.0.12.post135.dev0 → dftracer_utils-0.0.13.post1.dev0}/src/dftracer/utils/server/viz/handlers.h

@@ -46,6 +46,9 @@ coro::CoroTask<HttpResponse> handle_viz_layers(const HttpRequest& req,
 coro::CoroTask<HttpResponse> handle_viz_columns(const HttpRequest& req,
                                                 const QueryParams& params,
                                                 TraceIndex& index);
+coro::CoroTask<HttpResponse> handle_prov_graph(const HttpRequest& req,
+                                               const QueryParams& params,
+                                               TraceIndex& index);
 coro::CoroTask<HttpResponse> handle_viz_breaks(const HttpRequest& req,
                                                const QueryParams& params,
                                                TraceIndex& index);

dftracer_utils-0.0.13.post1.dev0/src/dftracer/utils/server/viz/prov.cpp

@@ -0,0 +1,32 @@
+// GET /api/prov/graph -- the provenance graph of the loaded trace, built by
+// trace::provenance::extract_provenance_graph over the server's indexed
+// file set (bloom-cache chunk skipping, max_concurrent worker slots).
+// Honors the same file-scoping parameters as the viz endpoints (?file=, ?pid=).
+#include <dftracer/utils/server/http_request.h>
+#include <dftracer/utils/server/http_response.h>
+#include <dftracer/utils/server/router.h>
+#include <dftracer/utils/server/trace_index.h>
+#include <dftracer/utils/server/viz/handlers.h>
+#include <dftracer/utils/server/viz/internal.h>
+#include <dftracer/utils/server/viz/scan.h>
+#include <dftracer/utils/trace/provenance/provenance_graph.h>
+#include <dftracer/utils/trace/views/view.h>
+
+#include <algorithm>
+
+namespace dftracer::utils::server {
+
+coro::CoroTask<HttpResponse> handle_prov_graph(const HttpRequest& req,
+                                               const QueryParams& params,
+                                               TraceIndex& index) {
+    auto files = collect_candidate_files(index, params);
+    std::size_t slots = std::max<std::size_t>(1, index.max_concurrent());
+    trace::views::View v =
+        trace::views::View::from_files(to_view_files(files), &index.bloom_cache())
+            .cancel_when([&req]() { return req.cancel_token.cancelled(); });
+    auto graph =
+        co_await trace::provenance::extract_provenance_graph(v, slots);
+    co_return HttpResponse::ok(graph.to_json());
+}
+
+}  // namespace dftracer::utils::server

dftracer_utils-0.0.13.post1.dev0/src/dftracer/utils/trace/provenance/provenance_graph.cpp

@@ -0,0 +1,303 @@
+#include <dftracer/utils/query/query.h>
+#include <dftracer/utils/trace/provenance/provenance_graph.h>
+#include <dftracer/utils/trace/views/view.h>
+#include <simdjson.h>
+
+#include <algorithm>
+#include <cstdlib>
+#include <string_view>
+#include <unordered_map>
+#include <tuple>
+#include <unordered_set>
+#include <utility>
+
+namespace dftracer::utils::trace::provenance {
+
+namespace {
+
+constexpr std::string_view kEntityPrefix = "prov_entity:";
+
+// Per-slot partial: keyed for O(1) merge, flattened into vectors at the end.
+struct Acc {
+    std::unordered_map<std::string, ProvenanceEntity> entities;
+    std::unordered_map<std::string, ProvenanceActivity> activities;
+    ProvenanceStats stats;
+};
+
+void split_hashes(std::string_view s, std::vector<std::string>& out) {
+    while (!s.empty()) {
+        auto c = s.find(',');
+        auto tok = s.substr(0, c);
+        if (!tok.empty()) out.emplace_back(tok);
+        if (c == std::string_view::npos) break;
+        s.remove_prefix(c + 1);
+    }
+}
+
+std::string_view str_field(simdjson::dom::object o, std::string_view k) {
+    auto v = o[k];
+    if (v.error() || !v.is_string()) return {};
+    return v.get_string().value_unsafe();
+}
+
+std::int64_t int_field(simdjson::dom::object o, std::string_view k,
+                       std::int64_t dflt) {
+    auto v = o[k];
+    if (v.error()) return dflt;
+    if (v.is_int64()) return v.get_int64().value_unsafe();
+    if (v.is_uint64())
+        return static_cast<std::int64_t>(v.get_uint64().value_unsafe());
+    if (v.is_string())
+        return std::strtoll(std::string(v.get_string().value_unsafe()).c_str(),
+                            nullptr, 10);
+    return dflt;
+}
+
+void add_pid(std::vector<std::int64_t>& v, std::int64_t pid) {
+    if (std::find(v.begin(), v.end(), pid) == v.end()) v.push_back(pid);
+}
+
+void ingest(Acc& a, std::string_view line) {
+    thread_local simdjson::dom::parser parser;
+    auto doc = parser.parse(line.data(), line.size());
+    if (doc.error() || !doc.value_unsafe().is_object()) return;
+    auto root = doc.value_unsafe().get_object().value_unsafe();
+    auto args_r = root["args"];
+    if (args_r.error() || !args_r.is_object()) return;
+    auto args = args_r.get_object().value_unsafe();
+    std::int64_t pid = int_field(root, "pid", 0);
+
+    // Entity declaration: a metadata record keyed prov_entity:<hash>.
+    auto key = str_field(args, "name");
+    if (key.size() > kEntityPrefix.size() &&
+        key.substr(0, kEntityPrefix.size()) == kEntityPrefix) {
+        ++a.stats.entity_records;
+        std::string hash(key.substr(kEntityPrefix.size()));
+        auto& e = a.entities[hash];
+        if (e.hash.empty()) {
+            e.hash = hash;
+            std::string_view val = str_field(args, "value");
+            std::string_view parts[4];
+            for (auto& part : parts) {
+                auto bar = val.find('|');
+                part = val.substr(0, bar);
+                val = bar == std::string_view::npos ? std::string_view{}
+                                                    : val.substr(bar + 1);
+            }
+            e.type = parts[0];
+            e.id = parts[1];
+            e.store = parts[2].empty() ? "memory" : std::string(parts[2]);
+            e.uri = parts[3];
+        }
+        add_pid(e.pids, pid);
+        return;
+    }
+
+    auto cat = str_field(root, "cat");
+    if (cat != "PROV" && cat != "PROV_CONT") return;
+    auto aid = str_field(args, "prov_aid");
+    if (aid.empty()) return;
+    auto& act = a.activities[std::string(aid)];
+    act.aid = aid;
+    split_hashes(str_field(args, "prov_used"), act.used);
+    split_hashes(str_field(args, "prov_generated"), act.generated);
+    if (cat == "PROV_CONT") {
+        ++a.stats.cont_events;
+        return;
+    }
+    ++a.stats.prov_events;
+    act.has_main = true;
+    act.name = str_field(root, "name");
+    act.activity = str_field(args, "prov_activity");
+    act.pid = pid;
+    act.ts = static_cast<std::uint64_t>(int_field(root, "ts", 0));
+    act.dur = static_cast<std::uint64_t>(int_field(root, "dur", 0));
+    act.nused = int_field(args, "prov_nused", -1);
+    act.ngen = int_field(args, "prov_ngen", -1);
+}
+
+Acc merge(Acc&& x, Acc&& y) {
+    for (auto& [h, e] : y.entities) {
+        auto it = x.entities.find(h);
+        if (it == x.entities.end()) {
+            x.entities.emplace(h, std::move(e));
+        } else {
+            for (auto p : e.pids) add_pid(it->second.pids, p);
+        }
+    }
+    for (auto& [aid, act] : y.activities) {
+        auto it = x.activities.find(aid);
+        if (it == x.activities.end()) {
+            x.activities.emplace(aid, std::move(act));
+            continue;
+        }
+        auto& t = it->second;
+        t.used.insert(t.used.end(), act.used.begin(), act.used.end());
+        t.generated.insert(t.generated.end(), act.generated.begin(),
+                           act.generated.end());
+        if (act.has_main) {
+            t.has_main = true;
+            t.name = std::move(act.name);
+            t.activity = std::move(act.activity);
+            t.pid = act.pid;
+            t.ts = act.ts;
+            t.dur = act.dur;
+            t.nused = act.nused;
+            t.ngen = act.ngen;
+        }
+    }
+    x.stats.entity_records += y.stats.entity_records;
+    x.stats.prov_events += y.stats.prov_events;
+    x.stats.cont_events += y.stats.cont_events;
+    return std::move(x);
+}
+
+ProvenanceGraph finish(Acc&& a) {
+    ProvenanceGraph g;
+    g.stats = a.stats;
+    std::unordered_set<std::string_view> referenced;
+    for (const auto& [aid, act] : a.activities) {
+        if (!act.has_main) ++g.stats.orphan_chunks;
+        if ((act.nused >= 0 &&
+             static_cast<std::size_t>(act.nused) != act.used.size()) ||
+            (act.ngen >= 0 &&
+             static_cast<std::size_t>(act.ngen) != act.generated.size()))
+            ++g.stats.truncated_activities;
+        for (const auto& h : act.used) referenced.insert(h);
+        for (const auto& h : act.generated) referenced.insert(h);
+    }
+    for (const auto& [h, e] : a.entities)
+        if (!referenced.count(h)) ++g.stats.isolated_entities;
+    for (auto h : referenced)
+        if (!a.entities.count(std::string(h))) ++g.stats.dangling_hashes;
+
+    g.entities.reserve(a.entities.size());
+    for (auto& [h, e] : a.entities) g.entities.push_back(std::move(e));
+    g.activities.reserve(a.activities.size());
+    for (auto& [aid, act] : a.activities)
+        g.activities.push_back(std::move(act));
+    // Deterministic output order.
+    std::sort(g.entities.begin(), g.entities.end(),
+              [](const auto& x, const auto& y) {
+                  return std::tie(x.type, x.id) < std::tie(y.type, y.id);
+              });
+    std::sort(g.activities.begin(), g.activities.end(),
+              [](const auto& x, const auto& y) {
+                  return std::tie(x.ts, x.aid) < std::tie(y.ts, y.aid);
+              });
+    return g;
+}
+
+void str_array(simdjson::builder::string_builder& b,
+               const std::vector<std::string>& v) {
+    b.start_array();
+    for (std::size_t i = 0; i < v.size(); ++i) {
+        if (i) b.append_comma();
+        b.escape_and_append_with_quotes(v[i]);
+    }
+    b.end_array();
+}
+
+}  // namespace
+
+std::string ProvenanceGraph::to_json() const {
+    simdjson::builder::string_builder b;
+    b.start_object();
+    b.escape_and_append_with_quotes("entities");
+    b.append_colon();
+    b.start_array();
+    for (std::size_t i = 0; i < entities.size(); ++i) {
+        const auto& e = entities[i];
+        if (i) b.append_comma();
+        b.start_object();
+        b.append_key_value("hash", e.hash);
+        b.append_comma();
+        b.append_key_value("type", e.type);
+        b.append_comma();
+        b.append_key_value("id", e.id);
+        b.append_comma();
+        b.append_key_value("store", e.store);
+        b.append_comma();
+        b.append_key_value("uri", e.uri);
+        b.append_comma();
+        b.escape_and_append_with_quotes("pids");
+        b.append_colon();
+        b.start_array();
+        for (std::size_t k = 0; k < e.pids.size(); ++k) {
+            if (k) b.append_comma();
+            b.append(e.pids[k]);
+        }
+        b.end_array();
+        b.end_object();
+    }
+    b.end_array();
+    b.append_comma();
+    b.escape_and_append_with_quotes("activities");
+    b.append_colon();
+    b.start_array();
+    for (std::size_t i = 0; i < activities.size(); ++i) {
+        const auto& a = activities[i];
+        if (i) b.append_comma();
+        b.start_object();
+        b.append_key_value("aid", a.aid);
+        b.append_comma();
+        b.append_key_value("name", a.name);
+        b.append_comma();
+        b.append_key_value("activity", a.activity);
+        b.append_comma();
+        b.append_key_value("pid", a.pid);
+        b.append_comma();
+        b.append_key_value("ts", a.ts);
+        b.append_comma();
+        b.append_key_value("dur", a.dur);
+        b.append_comma();
+        b.escape_and_append_with_quotes("used");
+        b.append_colon();
+        str_array(b, a.used);
+        b.append_comma();
+        b.escape_and_append_with_quotes("generated");
+        b.append_colon();
+        str_array(b, a.generated);
+        b.end_object();
+    }
+    b.end_array();
+    b.append_comma();
+    b.escape_and_append_with_quotes("stats");
+    b.append_colon();
+    b.start_object();
+    b.append_key_value("entities", entities.size());
+    b.append_comma();
+    b.append_key_value("activities", activities.size());
+    b.append_comma();
+    b.append_key_value("entity_records", stats.entity_records);
+    b.append_comma();
+    b.append_key_value("prov_events", stats.prov_events);
+    b.append_comma();
+    b.append_key_value("cont_events", stats.cont_events);
+    b.append_comma();
+    b.append_key_value("dangling_hashes", stats.dangling_hashes);
+    b.append_comma();
+    b.append_key_value("isolated_entities", stats.isolated_entities);
+    b.append_comma();
+    b.append_key_value("truncated_activities", stats.truncated_activities);
+    b.append_comma();
+    b.append_key_value("orphan_chunks", stats.orphan_chunks);
+    b.end_object();
+    b.end_object();
+    return std::string(b);
+}
+
+coro::CoroTask<ProvenanceGraph> extract_provenance_graph(
+    const views::View& view, std::size_t num_slots) {
+    views::View v = view.phase(views::Phase::Any)
+                        .filter(query::parse_or_throw(kProvenanceQuery));
+    auto scan = co_await v.map_batches<Acc>(
+        [](Acc& a, const std::vector<std::string_view>& events) {
+            for (auto ev : events) ingest(a, ev);
+        },
+        [](Acc&& x, Acc&& y) { return merge(std::move(x), std::move(y)); },
+        num_slots == 0 ? 1 : num_slots, 0);
+    co_return finish(std::move(scan.value));
+}
+
+}  // namespace dftracer::utils::trace::provenance