Refactor storing submissions into FrameSpan class, reduce AntiLag thread contention

5 files changed, 59 insertions, 65 deletions

diff --git a/src/strategies/low_latency2/device_strategy.cc b/src/strategies/low_latency2/device_strategy.cc
index 32ff981..227c385 100644
--- a/src/strategies/low_latency2/device_strategy.cc
+++ b/src/strategies/low_latency2/device_strategy.cc
@@ -68,8 +68,8 @@ void LowLatency2DeviceStrategy::submit_swapchain_present_id(
     // Iterate through all queues and grab any work that's associated with this
     // present_id. Turn it into a vector of work that we give to our swapchain
     // monitor.
-    auto work = [&]() -> std::vector<std::deque<std::unique_ptr<Submission>>> {
-        auto work = std::vector<std::deque<std::unique_ptr<Submission>>>{};
+    auto work = [&]() -> std::vector<std::unique_ptr<FrameSpan>> {
+        auto work = std::vector<std::unique_ptr<FrameSpan>>{};
         const auto lock = std::scoped_lock{this->device.mutex};
         for (const auto& queue_iter : this->device.queues) {
             const auto& queue = queue_iter.second;
@@ -84,14 +84,14 @@ void LowLatency2DeviceStrategy::submit_swapchain_present_id(
 
             // Need the lock now - we're modifying it.
             const auto strategy_lock = std::unique_lock{strategy->mutex};
-            const auto iter = strategy->present_id_submissions.find(present_id);
-            if (iter == std::end(strategy->present_id_submissions)) {
+            const auto iter = strategy->frame_spans.find(present_id);
+            if (iter == std::end(strategy->frame_spans)) {
                 continue;
             }
 
             // Make sure we clean it up from the present as well.
             work.push_back(std::move(iter->second));
-            strategy->present_id_submissions.erase(iter);
+            strategy->frame_spans.erase(iter);
         }
         return work;
     }();
diff --git a/src/strategies/low_latency2/queue_strategy.cc b/src/strategies/low_latency2/queue_strategy.cc
index 9a68b78..a020c0d 100644
--- a/src/strategies/low_latency2/queue_strategy.cc
+++ b/src/strategies/low_latency2/queue_strategy.cc
@@ -16,50 +16,48 @@ LowLatency2QueueStrategy::~LowLatency2QueueStrategy() {}
 template <typename T>
 static void notify_submit_impl(LowLatency2QueueStrategy& strategy,
                                const T& submit,
-                               std::unique_ptr<Submission> submission) {
+                               std::shared_ptr<TimestampPool::Handle> handle) {
 
     // It's actually not a requirement that we have this present id.
-    const auto lspi = find_next<VkLatencySubmissionPresentIdNV>(
-        &submit, VK_STRUCTURE_TYPE_LATENCY_SUBMISSION_PRESENT_ID_NV);
-    const auto present_id = lspi ? lspi->presentID : 0;
+    const auto present_id = [&]() -> std::uint64_t {
+        const auto lspi = find_next<VkLatencySubmissionPresentIdNV>(
+            &submit, VK_STRUCTURE_TYPE_LATENCY_SUBMISSION_PRESENT_ID_NV);
+        return lspi ? lspi->presentID : 0;
+    }();
 
     const auto lock = std::scoped_lock{strategy.mutex};
-    const auto [iter, inserted] =
-        strategy.present_id_submissions.try_emplace(present_id);
-    iter->second.push_back(std::move(submission));
-
-    // Remove stale submissions if we're presenting a lot to the same
-    // present_id. This doesn't affect anything because we're waiting on the
-    // last. It begs the question: should we should just store the last only?
-    if (std::size(iter->second) >=
-        LowLatency2QueueStrategy::MAX_TRACKED_OBJECTS) {
-
-        iter->second.pop_front();
+    const auto [iter, inserted] = strategy.frame_spans.try_emplace(present_id);
+    if (inserted) {
+        iter->second = std::make_unique<FrameSpan>(std::move(handle));
+    } else {
+        iter->second->update(std::move(handle));
     }
 
     // Add our present_id to our ring tracking if it's non-zero.
     if (inserted && present_id) {
-        strategy.present_id_ring.push_back(present_id);
+        strategy.stale_present_ids.push_back(present_id);
     }
 
     // Remove stale present_id's if they weren't presented to.
-    if (std::size(strategy.present_id_ring) >
-        LowLatency2QueueStrategy::MAX_TRACKED_OBJECTS) {
+    if (std::size(strategy.stale_present_ids) >
+        LowLatency2QueueStrategy::MAX_TRACKED_PRESENTS) {
 
-        const auto to_remove = strategy.present_id_ring.front();
-        strategy.present_id_ring.pop_front();
-        strategy.present_id_submissions.erase(to_remove);
+        const auto stale_present_id = strategy.stale_present_ids.front();
+        strategy.stale_present_ids.pop_front();
+        strategy.frame_spans.erase(stale_present_id);
     }
 }
 
 void LowLatency2QueueStrategy::notify_submit(
-    const VkSubmitInfo& submit, std::unique_ptr<Submission> submission) {
+    const VkSubmitInfo& submit,
+    std::shared_ptr<TimestampPool::Handle> submission) {
 
     notify_submit_impl(*this, submit, std::move(submission));
 }
 
 void LowLatency2QueueStrategy::notify_submit(
-    const VkSubmitInfo2& submit, std::unique_ptr<Submission> submission) {
+    const VkSubmitInfo2& submit,
+    std::shared_ptr<TimestampPool::Handle> submission) {
 
     notify_submit_impl(*this, submit, std::move(submission));
 }
diff --git a/src/strategies/low_latency2/queue_strategy.hh b/src/strategies/low_latency2/queue_strategy.hh
index a090e1b..6d41027 100644
--- a/src/strategies/low_latency2/queue_strategy.hh
+++ b/src/strategies/low_latency2/queue_strategy.hh
@@ -1,8 +1,8 @@
 #ifndef STRATEGIES_LOW_LATENCY2_QUEUE_STRATEGY_HH_
 #define STRATEGIES_LOW_LATENCY2_QUEUE_STRATEGY_HH_
 
+#include "frame_span.hh"
 #include "strategies/queue_strategy.hh"
-#include "submission.hh"
 
 #include <atomic>
 #include <deque>
@@ -16,26 +16,27 @@ class QueueContext;
 
 class LowLatency2QueueStrategy final : public QueueStrategy {
   public:
-    static constexpr auto MAX_TRACKED_OBJECTS = 50;
+    static constexpr auto MAX_TRACKED_PRESENTS = 50;
 
     // Mapping of present_id's to submissions. Grabbed later by the device
     // strategy when we present and actually can associate them to some
     // vkSwapchainKHR.
     std::mutex mutex{};
-    std::unordered_map<std::uint64_t, std::deque<std::unique_ptr<Submission>>>
-        present_id_submissions{};
-    std::deque<std::uint64_t> present_id_ring{};
-    std::atomic<bool> is_out_of_band{}; // atomic so we don't need a lock check
+    std::unordered_map<std::uint64_t, std::unique_ptr<FrameSpan>> frame_spans{};
+    std::deque<std::uint64_t> stale_present_ids{};
+    std::atomic<bool> is_out_of_band{}; // atomic to avoid lock
 
   public:
     LowLatency2QueueStrategy(QueueContext& queue);
     virtual ~LowLatency2QueueStrategy();
 
   public:
-    virtual void notify_submit(const VkSubmitInfo& submit,
-                               std::unique_ptr<Submission> submission) override;
-    virtual void notify_submit(const VkSubmitInfo2& submit,
-                               std::unique_ptr<Submission> submission) override;
+    virtual void
+    notify_submit(const VkSubmitInfo& submit,
+                  std::shared_ptr<TimestampPool::Handle> handle) override;
+    virtual void
+    notify_submit(const VkSubmitInfo2& submit,
+                  std::shared_ptr<TimestampPool::Handle> handle) override;
     virtual void notify_present(const VkPresentInfoKHR& present) override;
 
   public:
diff --git a/src/strategies/low_latency2/swapchain_monitor.cc b/src/strategies/low_latency2/swapchain_monitor.cc
index b6d4dd0..a70fa6c 100644
--- a/src/strategies/low_latency2/swapchain_monitor.cc
+++ b/src/strategies/low_latency2/swapchain_monitor.cc
@@ -35,32 +35,29 @@ void SwapchainMonitor::do_monitor(const std::stop_token stoken) {
     for (;;) {
         auto lock = std::unique_lock{this->mutex};
         this->cv.wait(lock, stoken,
-                      [&]() { return this->semaphore_submission.has_value(); });
+                      [&]() { return this->semaphore_spans.has_value(); });
 
         // Stop only if we're stopped and we have nothing to signal.
-        if (stoken.stop_requested() &&
-            !this->semaphore_submission.has_value()) {
+        if (stoken.stop_requested() && !this->semaphore_spans.has_value()) {
             break;
         }
 
         // Grab the most recent semaphore. When work completes, signal it.
-        const auto semaphore_submission =
-            std::move(*this->semaphore_submission);
-        this->semaphore_submission.reset();
+        const auto semaphore_span = std::move(*this->semaphore_spans);
+        this->semaphore_spans.reset();
 
         // If we're stopping, signal the semaphore and don't worry about work
         // actually completing.
         if (stoken.stop_requested()) {
-            semaphore_submission.wakeup_semaphore.signal(this->device);
+            semaphore_span.wakeup_semaphore.signal(this->device);
             break;
         }
 
         // Unlock, wait for work to finish, lock again.
         lock.unlock();
-        for (const auto& submission : semaphore_submission.submissions) {
-            if (!submission.empty()) {
-                const auto& last = submission.back();
-                last->handle->await_end_time();
+        for (const auto& frame_span : semaphore_span.frame_spans) {
+            if (frame_span) {
+                frame_span->await_completed();
             }
         }
 
@@ -78,7 +75,7 @@ void SwapchainMonitor::do_monitor(const std::stop_token stoken) {
         }
         lock.unlock();
 
-        semaphore_submission.wakeup_semaphore.signal(this->device);
+        semaphore_span.wakeup_semaphore.signal(this->device);
     }
 }
 
@@ -97,29 +94,29 @@ void SwapchainMonitor::notify_semaphore(const VkSemaphore& timeline_semaphore,
     }
 
     // Signal immediately if we have no outstanding work.
-    if (this->pending_submissions.empty()) {
+    if (this->pending_frame_spans.empty()) {
         wakeup_semaphore.signal(this->device);
         return;
     }
 
-    this->semaphore_submission.emplace(SemaphoreSubmissions{
+    this->semaphore_spans.emplace(SemaphoreSpans{
         .wakeup_semaphore = wakeup_semaphore,
-        .submissions = std::move(this->pending_submissions),
+        .frame_spans = std::move(this->pending_frame_spans),
     });
-    this->pending_submissions.clear();
+    this->pending_frame_spans.clear();
 
     lock.unlock();
     this->cv.notify_one();
 }
 
 void SwapchainMonitor::attach_work(
-    std::vector<std::deque<std::unique_ptr<Submission>>> submissions) {
+    std::vector<std::unique_ptr<FrameSpan>> frame_spans) {
 
     const auto lock = std::scoped_lock{this->mutex};
     if (!this->was_low_latency_requested) {
         return;
     }
-    this->pending_submissions = std::move(submissions);
+    this->pending_frame_spans = std::move(frame_spans);
 }
 
 } // namespace low_latency
\ No newline at end of file
diff --git a/src/strategies/low_latency2/swapchain_monitor.hh b/src/strategies/low_latency2/swapchain_monitor.hh
index 47c3a75..837f8e4 100644
--- a/src/strategies/low_latency2/swapchain_monitor.hh
+++ b/src/strategies/low_latency2/swapchain_monitor.hh
@@ -2,17 +2,16 @@
 #ifndef SWAPCHAIN_MONITOR_HH_
 #define SWAPCHAIN_MONITOR_HH_
 
+#include "frame_span.hh"
+
 #include <vulkan/vulkan.h>
 
 #include <chrono>
 #include <condition_variable>
-#include <deque>
 #include <memory>
 #include <mutex>
 #include <thread>
 
-#include "submission.hh"
-
 namespace low_latency {
 
 class DeviceContext;
@@ -28,15 +27,15 @@ class SwapchainMonitor final {
     };
 
     // An empty vector here represents our 'no work' state.
-    std::vector<std::deque<std::unique_ptr<Submission>>> pending_submissions{};
+    std::vector<std::unique_ptr<FrameSpan>> pending_frame_spans{};
 
     // A pairing of semaphore -> submissions.
     // If the Submissions completes then signal the bundled semaphore.
-    struct SemaphoreSubmissions {
+    struct SemaphoreSpans {
         WakeupSemaphore wakeup_semaphore{};
-        std::vector<std::deque<std::unique_ptr<Submission>>> submissions{};
+        std::vector<std::unique_ptr<FrameSpan>> frame_spans{};
     };
-    std::optional<SemaphoreSubmissions> semaphore_submission{};
+    std::optional<SemaphoreSpans> semaphore_spans{};
 
   protected:
     const DeviceContext& device;
@@ -67,8 +66,7 @@ class SwapchainMonitor final {
     void notify_semaphore(const VkSemaphore& timeline_semaphore,
                           const std::uint64_t& value);
 
-    void attach_work(
-        std::vector<std::deque<std::unique_ptr<Submission>>> submissions);
+    void attach_work(std::vector<std::unique_ptr<FrameSpan>> submissions);
 };
 
 } // namespace low_latency