1 files changed, 344 insertions, 18 deletions

diff --git a/src/virtio/vulkan/vn_common.c b/src/virtio/vulkan/vn_common.c
index e728a7971d2..4d22f184487 100644
--- a/src/virtio/vulkan/vn_common.c
+++ b/src/virtio/vulkan/vn_common.c
@@ -12,32 +12,86 @@
 
 #include <stdarg.h>
 
-#include "util/debug.h"
 #include "util/log.h"
 #include "util/os_misc.h"
+#include "util/u_debug.h"
+#include "venus-protocol/vn_protocol_driver_info.h"
 #include "vk_enum_to_str.h"
 
+#include "vn_instance.h"
+#include "vn_ring.h"
+
 static const struct debug_control vn_debug_options[] = {
+   /* clang-format off */
    { "init", VN_DEBUG_INIT },
    { "result", VN_DEBUG_RESULT },
    { "vtest", VN_DEBUG_VTEST },
    { "wsi", VN_DEBUG_WSI },
+   { "no_abort", VN_DEBUG_NO_ABORT },
+   { "log_ctx_info", VN_DEBUG_LOG_CTX_INFO },
+   { "cache", VN_DEBUG_CACHE },
+   { "no_sparse", VN_DEBUG_NO_SPARSE },
+   { "no_gpl", VN_DEBUG_NO_GPL },
+   { NULL, 0 },
+   /* clang-format on */
+};
+
+static const struct debug_control vn_perf_options[] = {
+   /* clang-format off */
+   { "no_async_set_alloc", VN_PERF_NO_ASYNC_SET_ALLOC },
+   { "no_async_buffer_create", VN_PERF_NO_ASYNC_BUFFER_CREATE },
+   { "no_async_queue_submit", VN_PERF_NO_ASYNC_QUEUE_SUBMIT },
+   { "no_event_feedback", VN_PERF_NO_EVENT_FEEDBACK },
+   { "no_fence_feedback", VN_PERF_NO_FENCE_FEEDBACK },
+   { "no_memory_suballoc", VN_PERF_NO_MEMORY_SUBALLOC },
+   { "no_cmd_batching", VN_PERF_NO_CMD_BATCHING },
+   { "no_semaphore_feedback", VN_PERF_NO_SEMAPHORE_FEEDBACK },
+   { "no_query_feedback", VN_PERF_NO_QUERY_FEEDBACK },
+   { "no_async_mem_alloc", VN_PERF_NO_ASYNC_MEM_ALLOC },
+   { "no_tiled_wsi_image", VN_PERF_NO_TILED_WSI_IMAGE },
+   { "no_multi_ring", VN_PERF_NO_MULTI_RING },
+   { "no_async_image_create", VN_PERF_NO_ASYNC_IMAGE_CREATE },
+   { "no_async_image_format", VN_PERF_NO_ASYNC_IMAGE_FORMAT },
    { NULL, 0 },
+   /* clang-format on */
 };
 
-uint64_t vn_debug;
+uint64_t vn_next_obj_id = 1;
+struct vn_env vn_env;
 
 static void
-vn_debug_init_once(void)
+vn_env_init_once(void)
 {
-   vn_debug = parse_debug_string(os_get_option("VN_DEBUG"), vn_debug_options);
+   vn_env.debug =
+      parse_debug_string(os_get_option("VN_DEBUG"), vn_debug_options);
+   vn_env.perf =
+      parse_debug_string(os_get_option("VN_PERF"), vn_perf_options);
 }
 
 void
-vn_debug_init(void)
+vn_env_init(void)
 {
    static once_flag once = ONCE_FLAG_INIT;
-   call_once(&once, vn_debug_init_once);
+   call_once(&once, vn_env_init_once);
+
+   /* log per VkInstance creation */
+   if (VN_DEBUG(INIT)) {
+      vn_log(NULL,
+             "vn_env is as below:"
+             "\n\tdebug = 0x%" PRIx64 ""
+             "\n\tperf = 0x%" PRIx64 "",
+             vn_env.debug, vn_env.perf);
+   }
+}
+
+void
+vn_trace_init(void)
+{
+#if DETECT_OS_ANDROID
+   atrace_init();
+#else
+   util_cpu_trace_init();
+#endif
 }
 
 void
@@ -61,29 +115,301 @@ vn_log_result(struct vn_instance *instance,
    return result;
 }
 
+uint32_t
+vn_extension_get_spec_version(const char *name)
+{
+   const int32_t index = vn_info_extension_index(name);
+   return index >= 0 ? vn_info_extension_get(index)->spec_version : 0;
+}
+
+static inline bool
+vn_watchdog_timeout(const struct vn_watchdog *watchdog)
+{
+   return !watchdog->alive;
+}
+
+static inline void
+vn_watchdog_release(struct vn_watchdog *watchdog)
+{
+   if (vn_gettid() == watchdog->tid) {
+      watchdog->tid = 0;
+      mtx_unlock(&watchdog->mutex);
+   }
+}
+
+static bool
+vn_watchdog_acquire(struct vn_watchdog *watchdog, bool alive)
+{
+   pid_t tid = vn_gettid();
+   if (!watchdog->tid && tid != watchdog->tid &&
+       mtx_trylock(&watchdog->mutex) == thrd_success) {
+      /* register as the only waiting thread that monitors the ring. */
+      watchdog->tid = tid;
+   }
+
+   if (tid != watchdog->tid)
+      return false;
+
+   watchdog->alive = alive;
+   return true;
+}
+
 void
-vn_relax(uint32_t *iter, const char *reason)
+vn_relax_fini(struct vn_relax_state *state)
+{
+   vn_watchdog_release(&state->instance->ring.watchdog);
+}
+
+static inline const char *
+vn_relax_reason_string(enum vn_relax_reason reason)
 {
-   /* Yield for the first 2^busy_wait_order times and then sleep for
-    * base_sleep_us microseconds for the same number of times.  After that,
-    * keep doubling both sleep length and count.
+   /* deliberately avoid default case for -Wswitch to catch upon compile */
+   switch (reason) {
+   case VN_RELAX_REASON_RING_SEQNO:
+      return "ring seqno";
+   case VN_RELAX_REASON_TLS_RING_SEQNO:
+      return "tls ring seqno";
+   case VN_RELAX_REASON_RING_SPACE:
+      return "ring space";
+   case VN_RELAX_REASON_FENCE:
+      return "fence";
+   case VN_RELAX_REASON_SEMAPHORE:
+      return "semaphore";
+   case VN_RELAX_REASON_QUERY:
+      return "query";
+   }
+   return "";
+}
+
+static struct vn_relax_profile
+vn_relax_get_profile(enum vn_relax_reason reason)
+{
+   /* This is the helper to map a vn_relax_reason to a profile. For new
+    * profiles added, we MUST also update the pre-calculated "first_warn_time"
+    * in vn_watchdog_init() if the "first warn" comes sooner.
     */
-   const uint32_t busy_wait_order = 4;
-   const uint32_t base_sleep_us = 10;
-   const uint32_t warn_order = 12;
 
+   /* deliberately avoid default case for -Wswitch to catch upon compile */
+   switch (reason) {
+   case VN_RELAX_REASON_RING_SEQNO:
+      /* warn every 4096 iters after having already slept ~3.5s:
+       *   (yielded 255 times)
+       *   stuck in wait with iter at 4096  (3.5s slept already)
+       *   stuck in wait with iter at 8192  (14s slept already)
+       *   stuck in wait with iter at 12288 (35s slept already)
+       *   ...
+       *   aborting after 895s
+       */
+      return (struct vn_relax_profile){
+         .base_sleep_us = 160,
+         .busy_wait_order = 8,
+         .warn_order = 12,
+         .abort_order = 16,
+      };
+   case VN_RELAX_REASON_TLS_RING_SEQNO:
+   case VN_RELAX_REASON_RING_SPACE:
+   case VN_RELAX_REASON_FENCE:
+   case VN_RELAX_REASON_SEMAPHORE:
+   case VN_RELAX_REASON_QUERY:
+      /* warn every 1024 iters after having already slept ~3.5s:
+       *   (yielded 15 times)
+       *   stuck in wait with iter at 1024 (3.5s slept already)
+       *   stuck in wait with iter at 2048 (14s slept already)
+       *   stuck in wait with iter at 3072 (35s slept already)
+       *   ...
+       *   aborting after 895s
+       */
+      return (struct vn_relax_profile){
+         .base_sleep_us = 160,
+         .busy_wait_order = 4,
+         .warn_order = 10,
+         .abort_order = 14,
+      };
+   }
+
+   unreachable("unhandled vn_relax_reason");
+}
+
+struct vn_relax_state
+vn_relax_init(struct vn_instance *instance, enum vn_relax_reason reason)
+{
+   struct vn_ring *ring = instance->ring.ring;
+   struct vn_watchdog *watchdog = &instance->ring.watchdog;
+   if (vn_watchdog_acquire(watchdog, true))
+      vn_ring_unset_status_bits(ring, VK_RING_STATUS_ALIVE_BIT_MESA);
+
+   return (struct vn_relax_state){
+      .instance = instance,
+      .iter = 0,
+      .profile = vn_relax_get_profile(reason),
+      .reason_str = vn_relax_reason_string(reason),
+   };
+}
+
+void
+vn_relax(struct vn_relax_state *state)
+{
+   const uint32_t base_sleep_us = state->profile.base_sleep_us;
+   const uint32_t busy_wait_order = state->profile.busy_wait_order;
+   const uint32_t warn_order = state->profile.warn_order;
+   const uint32_t abort_order = state->profile.abort_order;
+
+   uint32_t *iter = &state->iter;
    (*iter)++;
    if (*iter < (1 << busy_wait_order)) {
       thrd_yield();
       return;
    }
 
-   /* warn occasionally if we have slept at least 1.28ms for 2048 times (plus
-    * another 2047 shorter sleeps)
-    */
-   if (unlikely(*iter % (1 << warn_order) == 0))
-      vn_log(NULL, "stuck in %s wait with iter at %d", reason, *iter);
+   if (unlikely(*iter % (1 << warn_order) == 0)) {
+      struct vn_instance *instance = state->instance;
+      vn_log(instance, "stuck in %s wait with iter at %d", state->reason_str,
+             *iter);
+
+      struct vn_ring *ring = instance->ring.ring;
+      const uint32_t status = vn_ring_load_status(ring);
+      if (status & VK_RING_STATUS_FATAL_BIT_MESA) {
+         vn_log(instance, "aborting on ring fatal error at iter %d", *iter);
+         abort();
+      }
+
+      struct vn_watchdog *watchdog = &instance->ring.watchdog;
+      const bool alive = status & VK_RING_STATUS_ALIVE_BIT_MESA;
+      if (vn_watchdog_acquire(watchdog, alive))
+         vn_ring_unset_status_bits(ring, VK_RING_STATUS_ALIVE_BIT_MESA);
+
+      if (vn_watchdog_timeout(watchdog) && !VN_DEBUG(NO_ABORT)) {
+         vn_log(instance, "aborting on expired ring alive status at iter %d",
+                *iter);
+         abort();
+      }
+
+      if (*iter >= (1 << abort_order) && !VN_DEBUG(NO_ABORT)) {
+         vn_log(instance, "aborting");
+         abort();
+      }
+   }
 
    const uint32_t shift = util_last_bit(*iter) - busy_wait_order - 1;
    os_time_sleep(base_sleep_us << shift);
 }
+
+struct vn_ring *
+vn_tls_get_ring(struct vn_instance *instance)
+{
+   if (VN_PERF(NO_MULTI_RING))
+      return instance->ring.ring;
+
+   struct vn_tls *tls = vn_tls_get();
+   if (unlikely(!tls)) {
+      /* only allow to fallback on missing tls */
+      return instance->ring.ring;
+   }
+
+   /* look up tls_ring owned by instance */
+   list_for_each_entry(struct vn_tls_ring, tls_ring, &tls->tls_rings,
+                       tls_head) {
+      mtx_lock(&tls_ring->mutex);
+      if (tls_ring->instance == instance) {
+         mtx_unlock(&tls_ring->mutex);
+         assert(tls_ring->ring);
+         return tls_ring->ring;
+      }
+      mtx_unlock(&tls_ring->mutex);
+   }
+
+   struct vn_tls_ring *tls_ring = calloc(1, sizeof(*tls_ring));
+   if (!tls_ring)
+      return NULL;
+
+   /* keep the extra for potential roundtrip sync on tls ring */
+   static const size_t extra_size = sizeof(uint32_t);
+
+   /* only need a small ring for synchronous cmds on tls ring */
+   static const size_t buf_size = 16 * 1024;
+
+   /* single cmd can use the entire ring shmem on tls ring */
+   static const uint8_t direct_order = 0;
+
+   struct vn_ring_layout layout;
+   vn_ring_get_layout(buf_size, extra_size, &layout);
+
+   tls_ring->ring = vn_ring_create(instance, &layout, direct_order);
+   if (!tls_ring->ring) {
+      free(tls_ring);
+      return NULL;
+   }
+
+   mtx_init(&tls_ring->mutex, mtx_plain);
+   tls_ring->instance = instance;
+   list_add(&tls_ring->tls_head, &tls->tls_rings);
+   list_add(&tls_ring->vk_head, &instance->ring.tls_rings);
+
+   return tls_ring->ring;
+}
+
+void
+vn_tls_destroy_ring(struct vn_tls_ring *tls_ring)
+{
+   mtx_lock(&tls_ring->mutex);
+   if (tls_ring->ring) {
+      vn_ring_destroy(tls_ring->ring);
+      tls_ring->ring = NULL;
+      tls_ring->instance = NULL;
+      mtx_unlock(&tls_ring->mutex);
+   } else {
+      mtx_unlock(&tls_ring->mutex);
+      mtx_destroy(&tls_ring->mutex);
+      free(tls_ring);
+   }
+}
+
+static void
+vn_tls_free(void *tls)
+{
+   if (tls) {
+      list_for_each_entry_safe(struct vn_tls_ring, tls_ring,
+                               &((struct vn_tls *)tls)->tls_rings, tls_head)
+         vn_tls_destroy_ring(tls_ring);
+   }
+   free(tls);
+}
+
+static tss_t vn_tls_key;
+static bool vn_tls_key_valid;
+
+static void
+vn_tls_key_create_once(void)
+{
+   vn_tls_key_valid = tss_create(&vn_tls_key, vn_tls_free) == thrd_success;
+   if (!vn_tls_key_valid && VN_DEBUG(INIT))
+      vn_log(NULL, "WARNING: failed to create vn_tls_key");
+}
+
+struct vn_tls *
+vn_tls_get(void)
+{
+   static once_flag once = ONCE_FLAG_INIT;
+   call_once(&once, vn_tls_key_create_once);
+   if (unlikely(!vn_tls_key_valid))
+      return NULL;
+
+   struct vn_tls *tls = tss_get(vn_tls_key);
+   if (likely(tls))
+      return tls;
+
+   tls = calloc(1, sizeof(*tls));
+   if (!tls)
+      return NULL;
+
+   /* initialize tls */
+   tls->async_pipeline_create = false;
+   list_inithead(&tls->tls_rings);
+
+   if (tss_set(vn_tls_key, tls) != thrd_success) {
+      free(tls);
+      return NULL;
+   }
+
+   return tls;
+}