svga: enable DRM mks-stats via hooking to the corresponding DRM ioctls

SVGA DRM stat calls were situated but did not actually register with the mks-stats
mechanism due to absence of corresponding ioctls. The employed new ioctls in vmwgfx
are DRM_VMW_MKSSTAT_ADD and DRM_VMW_MKSSTAT_REMOVE, subject to version check.

Reviewed-by: Charmaine Lee <charmainel@vmware.com>
(cherry picked from commit be47c077cc927c27a8c36342b47697aa81719677)
(cherry picked from commit 0388afc67b830f6ab916d0839c33eb1d91d6353f)

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12873>

5 files changed, 539 insertions, 7 deletions

diff --git a/src/gallium/winsys/svga/drm/vmw_screen.c b/src/gallium/winsys/svga/drm/vmw_screen.c
index e97f1621ed4..60051bbdf07 100644
--- a/src/gallium/winsys/svga/drm/vmw_screen.c
+++ b/src/gallium/winsys/svga/drm/vmw_screen.c
@@ -27,6 +27,8 @@
 #include "vmw_screen.h"
 #include "vmw_fence.h"
 #include "vmw_context.h"
+#include "vmwgfx_drm.h"
+#include "xf86drm.h"
 
 #include "util/os_file.h"
 #include "util/u_memory.h"
@@ -41,6 +43,8 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
 
 static struct hash_table *dev_hash = NULL;
 
@@ -55,6 +59,61 @@ static uint32_t vmw_dev_hash(const void *key)
    return (major(*(dev_t *) key) << 16) | minor(*(dev_t *) key);
 }
 
+#ifdef VMX86_STATS
+/**
+ * Initializes mksstat TLS store.
+ */
+static void
+vmw_winsys_screen_init_mksstat(struct vmw_winsys_screen *vws)
+{
+   size_t i;
+
+   for (i = 0; i < ARRAY_SIZE(vws->mksstat_tls); ++i) {
+      vws->mksstat_tls[i].stat_pages = NULL;
+      vws->mksstat_tls[i].stat_id = -1UL;
+      vws->mksstat_tls[i].pid = 0;
+   }
+}
+
+/**
+ * Deinits mksstat TLS store.
+ */
+static void
+vmw_winsys_screen_deinit_mksstat(struct vmw_winsys_screen *vws)
+{
+   size_t i;
+
+   for (i = 0; i < ARRAY_SIZE(vws->mksstat_tls); ++i) {
+      uint32_t expected = __atomic_load_n(&vws->mksstat_tls[i].pid, __ATOMIC_ACQUIRE);
+
+      if (expected == -1U) {
+         fprintf(stderr, "%s encountered locked mksstat TLS entry at index %lu.\n", __FUNCTION__, i);
+         continue;
+      }
+
+      if (expected == 0)
+         continue;
+
+      if (__atomic_compare_exchange_n(&vws->mksstat_tls[i].pid, &expected, 0, false, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)) {
+         struct drm_vmw_mksstat_remove_arg arg = {
+            .id = vws->mksstat_tls[i].stat_id
+         };
+
+         assert(vws->mksstat_tls[i].stat_pages);
+         assert(vws->mksstat_tls[i].stat_id != -1UL);
+
+         if (drmCommandWrite(vws->ioctl.drm_fd, DRM_VMW_MKSSTAT_REMOVE, &arg, sizeof(arg))) {
+            fprintf(stderr, "%s could not ioctl: %s\n", __FUNCTION__, strerror(errno));
+         } else if (munmap(vws->mksstat_tls[i].stat_pages, vmw_svga_winsys_stats_len())) {
+            fprintf(stderr, "%s could not munmap: %s\n", __FUNCTION__, strerror(errno));
+         }
+      } else {
+         fprintf(stderr, "%s encountered volatile mksstat TLS entry at index %lu.\n", __FUNCTION__, i);
+      }
+   }
+}
+
+#endif
 /* Called from vmw_drm_create_screen(), creates and initializes the
  * vmw_winsys_screen structure, which is the main entity in this
  * module.
@@ -112,6 +171,9 @@ vmw_winsys_create( int fd )
    if (!vmw_winsys_screen_init_svga(vws))
       goto out_no_svga;
 
+#ifdef VMX86_STATS
+   vmw_winsys_screen_init_mksstat(vws);
+#endif
    _mesa_hash_table_insert(dev_hash, &vws->device, vws);
 
    cnd_init(&vws->cs_cond);
@@ -139,6 +201,9 @@ vmw_winsys_destroy(struct vmw_winsys_screen *vws)
       vmw_pools_cleanup(vws);
       vws->fence_ops->destroy(vws->fence_ops);
       vmw_ioctl_cleanup(vws);
+#ifdef VMX86_STATS
+      vmw_winsys_screen_deinit_mksstat(vws);
+#endif
       close(vws->ioctl.drm_fd);
       mtx_destroy(&vws->cs_mutex);
       cnd_destroy(&vws->cs_cond);
diff --git a/src/gallium/winsys/svga/drm/vmw_screen.h b/src/gallium/winsys/svga/drm/vmw_screen.h
index 73f04435a7e..1708921302d 100644
--- a/src/gallium/winsys/svga/drm/vmw_screen.h
+++ b/src/gallium/winsys/svga/drm/vmw_screen.h
@@ -81,6 +81,7 @@ struct vmw_winsys_screen
       boolean have_drm_2_16;
       boolean have_drm_2_17;
       boolean have_drm_2_18;
+      boolean have_drm_2_19;
    } ioctl;
 
    struct {
@@ -99,6 +100,17 @@ struct vmw_winsys_screen
 
    struct pb_fence_ops *fence_ops;
 
+#ifdef VMX86_STATS
+   /*
+    * mksGuestStats TLS array; length must be power of two
+    */
+   struct {
+      void *     stat_pages;
+      uint64_t   stat_id;
+      uint32_t   pid;
+   } mksstat_tls[64];
+
+#endif
    /*
     * Screen instances
     */
@@ -257,4 +269,7 @@ void
 vmw_svga_winsys_shader_destroy(struct svga_winsys_screen *sws,
 			       struct svga_winsys_gb_shader *shader);
 
+size_t
+vmw_svga_winsys_stats_len(void);
+
 #endif /* VMW_SCREEN_H_ */
diff --git a/src/gallium/winsys/svga/drm/vmw_screen_ioctl.c b/src/gallium/winsys/svga/drm/vmw_screen_ioctl.c
index a44c22c7595..c470b7ba572 100644
--- a/src/gallium/winsys/svga/drm/vmw_screen_ioctl.c
+++ b/src/gallium/winsys/svga/drm/vmw_screen_ioctl.c
@@ -1003,6 +1003,8 @@ vmw_ioctl_init(struct vmw_winsys_screen *vws)
       (version->version_major == 2 && version->version_minor > 16);
    vws->ioctl.have_drm_2_18 = version->version_major > 2 ||
       (version->version_major == 2 && version->version_minor > 17);
+   vws->ioctl.have_drm_2_19 = version->version_major > 2 ||
+      (version->version_major == 2 && version->version_minor > 18);
 
    vws->ioctl.drm_execbuf_version = vws->ioctl.have_drm_2_9 ? 2 : 1;
 
diff --git a/src/gallium/winsys/svga/drm/vmw_screen_svga.c b/src/gallium/winsys/svga/drm/vmw_screen_svga.c
index 9cf5ad609d5..547d6ea6d4c 100644
--- a/src/gallium/winsys/svga/drm/vmw_screen_svga.c
+++ b/src/gallium/winsys/svga/drm/vmw_screen_svga.c
@@ -33,6 +33,9 @@
  */
 
 #include <libsync.h>
+#include <stdint.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
 
 #include "svga_cmd.h"
 #include "svga3d_caps.h"
@@ -52,7 +55,9 @@
 #include "vmw_msg.h"
 #include "vmw_shader.h"
 #include "vmw_query.h"
+#include "vmwgfx_drm.h"
 #include "svga3d_surfacedefs.h"
+#include "xf86drm.h"
 
 /**
  * Try to get a surface backing buffer from the cache
@@ -60,6 +65,304 @@
  */
 #define VMW_TRY_CACHED_SIZE (2*1024*1024)
 
+#ifdef VMX86_STATS
+static const char* const vmw_svga_winsys_stats_count_names[] = {
+   SVGA_STATS_COUNT_NAMES
+};
+
+static const char* const vmw_svga_winsys_stats_time_names[] = {
+   SVGA_STATS_TIME_NAMES
+};
+
+/*
+ * It's imperative that the above two arrays are const, so that the next
+ * function can be optimized to a constant.
+ */
+static inline size_t
+vmw_svga_winsys_stats_names_len(void)
+{
+   size_t i, res = 0;
+   for (i = 0; i < ARRAY_SIZE(vmw_svga_winsys_stats_count_names); ++i)
+      res += strlen(vmw_svga_winsys_stats_count_names[i]) + 1;
+   for (i = 0; i < ARRAY_SIZE(vmw_svga_winsys_stats_time_names); ++i)
+      res += strlen(vmw_svga_winsys_stats_time_names[i]) + 1;
+   return res;
+}
+
+typedef struct Atomic_uint64 {
+   uint64_t value;
+} Atomic_uint64;
+
+typedef struct MKSGuestStatCounter {
+   Atomic_uint64 count;
+} MKSGuestStatCounter;
+
+typedef struct MKSGuestStatCounterTime {
+   MKSGuestStatCounter counter;
+   Atomic_uint64 selfCycles;
+   Atomic_uint64 totalCycles;
+} MKSGuestStatCounterTime;
+
+#define MKS_GUEST_STAT_FLAG_NONE    0
+#define MKS_GUEST_STAT_FLAG_TIME    (1U << 0)
+
+typedef __attribute__((aligned(32))) struct MKSGuestStatInfoEntry {
+   union {
+      const char *s;
+      uint64_t u;
+   } name;
+   union {
+      const char *s;
+      uint64_t u;
+   } description;
+   uint64_t flags;
+   union {
+      MKSGuestStatCounter *counter;
+      MKSGuestStatCounterTime *counterTime;
+      uint64_t u;
+   } stat;
+} MKSGuestStatInfoEntry;
+
+static __thread struct svga_winsys_stats_timeframe *mksstat_tls_global = NULL;
+
+#define ALIGN(x, power_of_two) (((x) + (power_of_two) - 1) & ~((power_of_two) - 1))
+
+static const size_t mksstat_area_size_info = sizeof(MKSGuestStatInfoEntry) * (SVGA_STATS_COUNT_MAX + SVGA_STATS_TIME_MAX);
+static const size_t mksstat_area_size_stat = sizeof(MKSGuestStatCounter) * SVGA_STATS_COUNT_MAX +
+                                             sizeof(MKSGuestStatCounterTime) * SVGA_STATS_TIME_MAX;
+
+size_t
+vmw_svga_winsys_stats_len(void)
+{
+   const size_t pg_size = getpagesize();
+   const size_t area_size_stat_pg = ALIGN(mksstat_area_size_stat, pg_size);
+   const size_t area_size_info_pg = ALIGN(mksstat_area_size_info, pg_size);
+   const size_t area_size_strs = vmw_svga_winsys_stats_names_len();
+   const size_t area_size = area_size_stat_pg + area_size_info_pg + area_size_strs;
+
+   return area_size;
+}
+
+/**
+ * vmw_mksstat_get_pstat: Computes the address of the MKSGuestStatCounter
+ * array from the address of the base page.
+ *
+ * @page_addr: Pointer to the base page.
+ * @page_size: Size of page.
+ * Return: Pointer to the MKSGuestStatCounter array.
+ */
+
+static inline MKSGuestStatCounter *
+vmw_mksstat_get_pstat(uint8_t *page_addr, size_t page_size)
+{
+   return (MKSGuestStatCounter *)page_addr;
+}
+
+/**
+ * vmw_mksstat_get_pstat_time: Computes the address of the MKSGuestStatCounterTime
+ * array from the address of the base page.
+ *
+ * @page_addr: Pointer to the base page.
+ * @page_size: Size of page.
+ * Return: Pointer to the MKSGuestStatCounterTime array.
+ */
+
+static inline MKSGuestStatCounterTime *
+vmw_mksstat_get_pstat_time(uint8_t *page_addr, size_t page_size)
+{
+   return (MKSGuestStatCounterTime *)(page_addr + sizeof(MKSGuestStatCounter) * SVGA_STATS_COUNT_MAX);
+}
+
+/**
+ * vmw_mksstat_get_pinfo: Computes the address of the MKSGuestStatInfoEntry
+ * array from the address of the base page.
+ *
+ * @page_addr: Pointer to the base page.
+ * @page_size: Size of page.
+ * Return: Pointer to the MKSGuestStatInfoEntry array.
+ */
+
+static inline MKSGuestStatInfoEntry *
+vmw_mksstat_get_pinfo(uint8_t *page_addr, size_t page_size)
+{
+   const size_t area_size_stat_pg = ALIGN(mksstat_area_size_stat, page_size);
+   return (MKSGuestStatInfoEntry *)(page_addr + area_size_stat_pg);
+}
+
+/**
+ * vmw_mksstat_get_pstrs: Computes the address of the mksGuestStat strings
+ * sequence from the address of the base page.
+ *
+ * @page_addr: Pointer to the base page.
+ * @page_size: Size of page.
+ * Return: Pointer to the mksGuestStat strings sequence.
+ */
+
+static inline char *
+vmw_mksstat_get_pstrs(uint8_t *page_addr, const size_t page_size)
+{
+   const size_t area_size_info_pg = ALIGN(mksstat_area_size_info, page_size);
+   const size_t area_size_stat_pg = ALIGN(mksstat_area_size_stat, page_size);
+   return (char *)(page_addr + area_size_info_pg + area_size_stat_pg);
+}
+
+/**
+ * Add all known mksGuestStats counters for tracking by the host.
+ */
+static int
+vmw_svga_winsys_add_stats(struct vmw_winsys_screen *vws, int slot)
+{
+   const size_t pg_size = getpagesize();
+   const size_t area_size = vmw_svga_winsys_stats_len();
+
+   MKSGuestStatInfoEntry *pinfo;
+   MKSGuestStatCounter *pstat;
+   MKSGuestStatCounterTime *pstatTime;
+   char *pstrs;
+   uint64_t id;
+   size_t i;
+
+   /* Allocate a contiguous area of pages for all info entries, counters and strings. */
+   void *area = mmap(NULL, area_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_SHARED | MAP_NORESERVE, -1, 0);
+
+   if (area == MAP_FAILED) {
+      fprintf(stderr, "%s could not mmap memory: %s\n", __FUNCTION__, strerror(errno));
+      return -1;
+   }
+
+   pinfo = vmw_mksstat_get_pinfo(area, pg_size);
+   pstat = vmw_mksstat_get_pstat(area, pg_size);
+   pstrs = vmw_mksstat_get_pstrs(area, pg_size);
+   pstatTime = vmw_mksstat_get_pstat_time(area, pg_size);
+
+   if (mlock(area, area_size)) {
+      fprintf(stderr, "%s could not mlock memory: %s\n", __FUNCTION__, strerror(errno));
+      goto error;
+   }
+
+   /* Suppress pages copy-on-write; for MAP_SHARED this should not really matter; it would if we go MAP_PRIVATE */
+   if (madvise(area, area_size, MADV_DONTFORK)) {
+      fprintf(stderr, "%s could not madvise memory: %s\n", __FUNCTION__, strerror(errno));
+      goto error;
+   }
+
+   /* Set up regular counters first */
+   for (i = 0; i < SVGA_STATS_COUNT_MAX; ++i) {
+      pinfo->name.s = pstrs;
+      pinfo->description.s = pstrs;
+      pinfo->flags = MKS_GUEST_STAT_FLAG_NONE;
+      pinfo->stat.counter = pstat + i;
+      pinfo++;
+
+      memcpy(pstrs, vmw_svga_winsys_stats_count_names[i], strlen(vmw_svga_winsys_stats_count_names[i]));
+      pstrs += strlen(vmw_svga_winsys_stats_count_names[i]) + 1;
+   }
+
+   /* Set up time counters second */
+   for (i = 0; i < SVGA_STATS_TIME_MAX; ++i) {
+      pinfo->name.s = pstrs;
+      pinfo->description.s = pstrs;
+      pinfo->flags = MKS_GUEST_STAT_FLAG_TIME;
+      pinfo->stat.counterTime = pstatTime + i;
+      pinfo++;
+
+      memcpy(pstrs, vmw_svga_winsys_stats_time_names[i], strlen(vmw_svga_winsys_stats_time_names[i]));
+      pstrs += strlen(vmw_svga_winsys_stats_time_names[i]) + 1;
+   }
+
+   { /* ioctl(DRM_VMW_MKSSTAT_ADD) */
+      char desc[64];
+      snprintf(desc, sizeof(desc) - 1, "vmw_winsys_screen=%p pid=%d", vws, gettid());
+
+      struct drm_vmw_mksstat_add_arg arg = {
+         .stat = (uintptr_t)pstat,
+         .info = (uintptr_t)vmw_mksstat_get_pinfo(area, pg_size),
+         .strs = (uintptr_t)vmw_mksstat_get_pstrs(area, pg_size),
+         .stat_len = mksstat_area_size_stat,
+         .info_len = mksstat_area_size_info,
+         .strs_len = vmw_svga_winsys_stats_names_len(),
+         .description = (uintptr_t)desc,
+         .id = -1U
+      };
+      if (drmCommandWriteRead(vws->ioctl.drm_fd, DRM_VMW_MKSSTAT_ADD, &arg, sizeof(arg))) {
+         fprintf(stderr, "%s could not ioctl: %s\n", __FUNCTION__, strerror(errno));
+         goto error;
+      }
+      id = arg.id;
+   }
+
+   vws->mksstat_tls[slot].stat_pages = area;
+   vws->mksstat_tls[slot].stat_id = id;
+   /* Don't update vws->mksstat_tls[].pid as it's reserved. */
+   return 0;
+
+error:
+   munmap(area, area_size);
+   return -1;
+}
+
+/**
+ * Acquire a mksstat TLS slot making it immutable by other parties.
+ */
+static inline int
+vmw_winsys_screen_mksstat_acq_slot(struct vmw_winsys_screen *vws)
+{
+   const pid_t pid = gettid();
+   const size_t base = (size_t)pid % ARRAY_SIZE(vws->mksstat_tls);
+   size_t i;
+
+   if (mksstat_tls_global && vmw_winsys_screen(mksstat_tls_global->sws) == vws) {
+      const size_t slot = mksstat_tls_global->slot;
+      uint32_t expecpid = pid;
+      if (__atomic_compare_exchange_n(&vws->mksstat_tls[slot].pid, &expecpid, -1U, false, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE))
+         return (int)slot;
+   }
+
+   for (i = 0; i < ARRAY_SIZE(vws->mksstat_tls); ++i) {
+      const size_t slot = (i + base) % ARRAY_SIZE(vws->mksstat_tls);
+      uint32_t expecpid = pid;
+      uint32_t expected = 0;
+
+      /* Check if pid is already present */
+      if (__atomic_compare_exchange_n(&vws->mksstat_tls[slot].pid, &expecpid, -1U, false, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE))
+         return (int)slot;
+
+      /* Try to set up a new mksstat for this pid */
+      if (__atomic_compare_exchange_n(&vws->mksstat_tls[slot].pid, &expected, -1U, false, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)) {
+         const int ret = vmw_svga_winsys_add_stats(vws, slot);
+
+         if (!ret)
+            return (int)slot;
+
+         __atomic_store_n(&vws->mksstat_tls[slot].pid, 0, __ATOMIC_RELEASE);
+         return ret;
+      }
+   }
+
+   return -1;
+}
+
+/**
+ * Release a mksstat TLS slot -- caller still owns the slot but now it is erasable by other parties.
+ */
+static inline void
+vmw_winsys_screen_mksstat_rel_slot(struct vmw_winsys_screen *vws, int slot)
+{
+   assert(slot < ARRAY_SIZE(vws->mksstat_tls));
+
+   __atomic_store_n(&vws->mksstat_tls[slot].pid, gettid(), __ATOMIC_RELEASE);
+}
+
+static inline uint64_t
+rdtsc(void)
+{
+   uint32_t hi, lo;
+   __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi));
+   return (uint64_t)lo | ((uint64_t)hi << 32);
+}
+
+#endif /* VMX86_STATS */
+
 static struct svga_winsys_buffer *
 vmw_svga_winsys_buffer_create(struct svga_winsys_screen *sws,
                               unsigned alignment,
@@ -465,20 +768,110 @@ vmw_svga_winsys_shader_destroy(struct svga_winsys_screen *sws,
    vmw_svga_winsys_shader_reference(&d_shader, NULL);
 }
 
+#ifdef VMX86_STATS
 static void
-vmw_svga_winsys_stats_inc(enum svga_stats_count index)
+vmw_svga_winsys_stats_inc(struct svga_winsys_screen *sws,
+                          enum svga_stats_count index)
 {
+   struct vmw_winsys_screen *const vws = vmw_winsys_screen(sws);
+   const int slot = vmw_winsys_screen_mksstat_acq_slot(vws);
+   assert(index < SVGA_STATS_COUNT_MAX);
+
+   if (slot >= 0) {
+      MKSGuestStatCounter *pstat;
+      assert(vws->mksstat_tls[slot].stat_pages);
+      assert(vws->mksstat_tls[slot].stat_id != -1UL);
+
+      pstat = vmw_mksstat_get_pstat(vws->mksstat_tls[slot].stat_pages, getpagesize());
+
+      __atomic_fetch_add(&pstat[index].count.value, 1, __ATOMIC_ACQ_REL);
+
+      vmw_winsys_screen_mksstat_rel_slot(vws, slot);
+   }
 }
 
 static void
-vmw_svga_winsys_stats_time_push(enum svga_stats_time index,
+vmw_svga_winsys_stats_time_push(struct svga_winsys_screen *sws,
+                                enum svga_stats_time index,
                                 struct svga_winsys_stats_timeframe *tf)
 {
+   struct vmw_winsys_screen *const vws = vmw_winsys_screen(sws);
+   const int slot = vmw_winsys_screen_mksstat_acq_slot(vws);
+
+   if (slot < 0)
+      return;
+
+   assert(vws->mksstat_tls[slot].stat_pages);
+   assert(vws->mksstat_tls[slot].stat_id != -1UL);
+
+   tf->counterTime = vmw_mksstat_get_pstat_time(vws->mksstat_tls[slot].stat_pages, getpagesize()) + index;
+
+   vmw_winsys_screen_mksstat_rel_slot(vws, slot);
+
+   tf->startTime = rdtsc();
+   tf->enclosing = mksstat_tls_global;
+   tf->sws = sws;
+   tf->slot = slot;
+
+   mksstat_tls_global = tf;
+}
+
+static void
+vmw_svga_winsys_stats_time_pop(struct svga_winsys_screen *sws)
+{
+   struct svga_winsys_stats_timeframe *const tf = mksstat_tls_global;
+   struct vmw_winsys_screen *const vws = vmw_winsys_screen(sws);
+   const int slot = tf->slot;
+   uint32_t expected = gettid();
+
+   mksstat_tls_global = tf->enclosing;
+
+   if (slot < 0)
+      return;
+
+   if (__atomic_compare_exchange_n(&vws->mksstat_tls[slot].pid, &expected, -1U, false, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)) {
+      const uint64_t dt = rdtsc() - tf->startTime;
+      MKSGuestStatCounterTime *const counterTime = tf->counterTime;
+
+      assert(vws->mksstat_tls[slot].stat_pages);
+      assert(vws->mksstat_tls[slot].stat_id != -1UL);
+
+      __atomic_fetch_add(&counterTime->counter.count.value, 1, __ATOMIC_ACQ_REL);
+      __atomic_fetch_add(&counterTime->selfCycles.value, dt, __ATOMIC_ACQ_REL);
+      __atomic_fetch_add(&counterTime->totalCycles.value, dt, __ATOMIC_ACQ_REL);
+
+      if (tf->enclosing) {
+         MKSGuestStatCounterTime *const counterTime = tf->enclosing->counterTime;
+
+         assert(counterTime);
+
+         __atomic_fetch_sub(&counterTime->selfCycles.value, dt, __ATOMIC_ACQ_REL);
+      }
+
+      __atomic_store_n(&vws->mksstat_tls[slot].pid, expected, __ATOMIC_RELEASE);
+   }
+}
+
+#endif /* VMX86_STATS */
+static void
+vmw_svga_winsys_stats_inc_noop(struct svga_winsys_screen *sws,
+                               enum svga_stats_count index)
+{
+   /* noop */
 }
 
 static void
-vmw_svga_winsys_stats_time_pop()
+vmw_svga_winsys_stats_time_push_noop(struct svga_winsys_screen *sws,
+                                     enum svga_stats_time index,
+                                     struct svga_winsys_stats_timeframe *tf)
 {
+   /* noop */
+}
+
+static void
+vmw_svga_winsys_stats_time_pop_noop(struct svga_winsys_screen *sws)
+{
+   /* noop */
 }
 
 boolean
@@ -511,10 +904,23 @@ vmw_winsys_screen_init_svga(struct vmw_winsys_screen *vws)
    vws->base.query_destroy = vmw_svga_winsys_query_destroy;
    vws->base.query_get_result = vmw_svga_winsys_query_get_result;
 
-   vws->base.stats_inc = vmw_svga_winsys_stats_inc;
-   vws->base.stats_time_push = vmw_svga_winsys_stats_time_push;
-   vws->base.stats_time_pop = vmw_svga_winsys_stats_time_pop;
+#ifdef VMX86_STATS
+   if (vws->ioctl.have_drm_2_19) {
+      vws->base.stats_inc = vmw_svga_winsys_stats_inc;
+      vws->base.stats_time_push = vmw_svga_winsys_stats_time_push;
+      vws->base.stats_time_pop = vmw_svga_winsys_stats_time_pop;
+   } else {
+      vws->base.stats_inc = vmw_svga_winsys_stats_inc_noop;
+      vws->base.stats_time_push = vmw_svga_winsys_stats_time_push_noop;
+      vws->base.stats_time_pop = vmw_svga_winsys_stats_time_pop_noop;
+   }
+
+#else
+   vws->base.stats_inc = vmw_svga_winsys_stats_inc_noop;
+   vws->base.stats_time_push = vmw_svga_winsys_stats_time_push_noop;
+   vws->base.stats_time_pop = vmw_svga_winsys_stats_time_pop_noop;
 
+#endif
    vws->base.host_log = vmw_svga_winsys_host_log;
 
    return TRUE;
diff --git a/src/gallium/winsys/svga/drm/vmwgfx_drm.h b/src/gallium/winsys/svga/drm/vmwgfx_drm.h
index 9b422e5515b..9078775feb5 100644
--- a/src/gallium/winsys/svga/drm/vmwgfx_drm.h
+++ b/src/gallium/winsys/svga/drm/vmwgfx_drm.h
@@ -72,6 +72,9 @@ extern "C" {
 #define DRM_VMW_GB_SURFACE_CREATE_EXT   27
 #define DRM_VMW_GB_SURFACE_REF_EXT      28
 #define DRM_VMW_MSG                     29
+#define DRM_VMW_MKSSTAT_RESET           30
+#define DRM_VMW_MKSSTAT_ADD             31
+#define DRM_VMW_MKSSTAT_REMOVE          32
 
 /*************************************************************************/
 /**
@@ -86,6 +89,9 @@ extern "C" {
  *
  * DRM_VMW_PARAM_SM4_1
  * SM4_1 support is enabled.
+ *
+ * DRM_VMW_PARAM_SM5
+ * SM5 support is enabled.
  */
 
 #define DRM_VMW_PARAM_NUM_STREAMS      0
@@ -1134,7 +1140,7 @@ struct drm_vmw_handle_close_arg {
  * svga3d surface flags split into 2, upper half and lower half.
  */
 enum drm_vmw_surface_version {
-	drm_vmw_gb_surface_v1
+	drm_vmw_gb_surface_v1,
 };
 
 /**
@@ -1233,6 +1239,44 @@ struct drm_vmw_msg_arg {
 	__u32 receive_len;
 };
 
+/**
+ * struct drm_vmw_mksstat_add_arg
+ *
+ * @stat: Pointer to user-space stat-counters array, page-aligned.
+ * @info: Pointer to user-space counter-infos array, page-aligned.
+ * @strs: Pointer to user-space stat strings, page-aligned.
+ * @stat_len: Length in bytes of stat-counters array.
+ * @info_len: Length in bytes of counter-infos array.
+ * @strs_len: Length in bytes of the stat strings, terminators included.
+ * @description: Pointer to instance descriptor string; will be truncated
+ *               to MKS_GUEST_STAT_INSTANCE_DESC_LENGTH chars.
+ * @id: Output identifier of the produced record; -1 if error.
+ *
+ * Argument to the DRM_VMW_MKSSTAT_ADD ioctl.
+ */
+struct drm_vmw_mksstat_add_arg {
+	__u64 stat;
+	__u64 info;
+	__u64 strs;
+	__u64 stat_len;
+	__u64 info_len;
+	__u64 strs_len;
+	__u64 description;
+	__u64 id;
+};
+
+/**
+ * struct drm_vmw_mksstat_remove_arg
+ *
+ * @id: Identifier of the record being disposed, originally obtained through
+ *      DRM_VMW_MKSSTAT_ADD ioctl.
+ *
+ * Argument to the DRM_VMW_MKSSTAT_REMOVE ioctl.
+ */
+struct drm_vmw_mksstat_remove_arg {
+	__u64 id;
+};
+
 #if defined(__cplusplus)
 }
 #endif