summaryrefslogtreecommitdiff
path: root/src/gallium/drivers/freedreno/drm/msm_ringbuffer.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/gallium/drivers/freedreno/drm/msm_ringbuffer.c')
-rw-r--r--src/gallium/drivers/freedreno/drm/msm_ringbuffer.c721
1 files changed, 721 insertions, 0 deletions
diff --git a/src/gallium/drivers/freedreno/drm/msm_ringbuffer.c b/src/gallium/drivers/freedreno/drm/msm_ringbuffer.c
new file mode 100644
index 00000000000..f1e96740231
--- /dev/null
+++ b/src/gallium/drivers/freedreno/drm/msm_ringbuffer.c
@@ -0,0 +1,721 @@
+/*
+ * Copyright (C) 2012-2018 Rob Clark <robclark@freedesktop.org>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ * Rob Clark <robclark@freedesktop.org>
+ */
+
+#include <assert.h>
+#include <inttypes.h>
+
+#include "util/hash_table.h"
+#include "util/set.h"
+#include "util/slab.h"
+
+#include "drm/freedreno_ringbuffer.h"
+#include "msm_priv.h"
+
+/* The legacy implementation of submit/ringbuffer, which still does the
+ * traditional reloc and cmd tracking
+ */
+
+
+#define INIT_SIZE 0x1000
+
+static pthread_mutex_t idx_lock = PTHREAD_MUTEX_INITIALIZER;
+
+
+struct msm_submit {
+ struct fd_submit base;
+
+ DECLARE_ARRAY(struct drm_msm_gem_submit_bo, submit_bos);
+ DECLARE_ARRAY(struct fd_bo *, bos);
+
+ unsigned seqno;
+
+ /* maps fd_bo to idx in bos table: */
+ struct hash_table *bo_table;
+
+ struct slab_mempool ring_pool;
+
+ /* hash-set of associated rings: */
+ struct set *ring_set;
+
+ struct fd_ringbuffer *primary;
+
+ /* Allow for sub-allocation of stateobj ring buffers (ie. sharing
+ * the same underlying bo)..
+ *
+ * We also rely on previous stateobj having been fully constructed
+ * so we can reclaim extra space at it's end.
+ */
+ struct fd_ringbuffer *suballoc_ring;
+};
+FD_DEFINE_CAST(fd_submit, msm_submit);
+
+/* for FD_RINGBUFFER_GROWABLE rb's, tracks the 'finalized' cmdstream buffers
+ * and sizes. Ie. a finalized buffer can have no more commands appended to
+ * it.
+ */
+struct msm_cmd {
+ struct fd_bo *ring_bo;
+ unsigned size;
+ DECLARE_ARRAY(struct drm_msm_gem_submit_reloc, relocs);
+};
+
+static struct msm_cmd *
+cmd_new(struct fd_bo *ring_bo)
+{
+ struct msm_cmd *cmd = malloc(sizeof(*cmd));
+ cmd->ring_bo = fd_bo_ref(ring_bo);
+ cmd->size = 0;
+ cmd->nr_relocs = cmd->max_relocs = 0;
+ cmd->relocs = NULL;
+ return cmd;
+}
+
+static void
+cmd_free(struct msm_cmd *cmd)
+{
+ fd_bo_del(cmd->ring_bo);
+ free(cmd);
+}
+
+/* for _FD_RINGBUFFER_OBJECT rb's we need to track the bo's and flags to
+ * later copy into the submit when the stateobj rb is later referenced by
+ * a regular rb:
+ */
+struct msm_reloc_bo {
+ struct fd_bo *bo;
+ unsigned flags;
+};
+
+struct msm_ringbuffer {
+ struct fd_ringbuffer base;
+
+ /* for FD_RINGBUFFER_STREAMING rb's which are sub-allocated */
+ unsigned offset;
+
+ union {
+ /* for _FD_RINGBUFFER_OBJECT case: */
+ struct {
+ struct fd_pipe *pipe;
+ DECLARE_ARRAY(struct msm_reloc_bo, reloc_bos);
+ struct set *ring_set;
+ };
+ /* for other cases: */
+ struct {
+ struct fd_submit *submit;
+ DECLARE_ARRAY(struct msm_cmd *, cmds);
+ };
+ } u;
+
+ struct msm_cmd *cmd; /* current cmd */
+ struct fd_bo *ring_bo;
+};
+FD_DEFINE_CAST(fd_ringbuffer, msm_ringbuffer);
+
+static void finalize_current_cmd(struct fd_ringbuffer *ring);
+static struct fd_ringbuffer * msm_ringbuffer_init(
+ struct msm_ringbuffer *msm_ring,
+ uint32_t size, enum fd_ringbuffer_flags flags);
+
+/* add (if needed) bo to submit and return index: */
+static uint32_t
+append_bo(struct msm_submit *submit, struct fd_bo *bo, uint32_t flags)
+{
+ struct msm_bo *msm_bo = to_msm_bo(bo);
+ uint32_t idx;
+ pthread_mutex_lock(&idx_lock);
+ if (likely(msm_bo->current_submit_seqno == submit->seqno)) {
+ idx = msm_bo->idx;
+ } else {
+ uint32_t hash = _mesa_hash_pointer(bo);
+ struct hash_entry *entry;
+
+ entry = _mesa_hash_table_search_pre_hashed(submit->bo_table, hash, bo);
+ if (entry) {
+ /* found */
+ idx = (uint32_t)(uintptr_t)entry->data;
+ } else {
+ idx = APPEND(submit, submit_bos);
+ idx = APPEND(submit, bos);
+
+ submit->submit_bos[idx].flags = 0;
+ submit->submit_bos[idx].handle = bo->handle;
+ submit->submit_bos[idx].presumed = 0;
+
+ submit->bos[idx] = fd_bo_ref(bo);
+
+ _mesa_hash_table_insert_pre_hashed(submit->bo_table, hash, bo,
+ (void *)(uintptr_t)idx);
+ }
+ msm_bo->current_submit_seqno = submit->seqno;
+ msm_bo->idx = idx;
+ }
+ pthread_mutex_unlock(&idx_lock);
+ if (flags & FD_RELOC_READ)
+ submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_READ;
+ if (flags & FD_RELOC_WRITE)
+ submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_WRITE;
+ return idx;
+}
+
+static void
+append_ring(struct set *set, struct fd_ringbuffer *ring)
+{
+ uint32_t hash = _mesa_hash_pointer(ring);
+
+ if (!_mesa_set_search_pre_hashed(set, hash, ring)) {
+ fd_ringbuffer_ref(ring);
+ _mesa_set_add_pre_hashed(set, hash, ring);
+ }
+}
+
+static void
+msm_submit_suballoc_ring_bo(struct fd_submit *submit,
+ struct msm_ringbuffer *msm_ring, uint32_t size)
+{
+ struct msm_submit *msm_submit = to_msm_submit(submit);
+ unsigned suballoc_offset = 0;
+ struct fd_bo *suballoc_bo = NULL;
+
+ if (msm_submit->suballoc_ring) {
+ struct msm_ringbuffer *suballoc_ring =
+ to_msm_ringbuffer(msm_submit->suballoc_ring);
+
+ suballoc_bo = suballoc_ring->ring_bo;
+ suballoc_offset = fd_ringbuffer_size(msm_submit->suballoc_ring) +
+ suballoc_ring->offset;
+
+ suballoc_offset = align(suballoc_offset, 0x10);
+
+ if ((size + suballoc_offset) > suballoc_bo->size) {
+ suballoc_bo = NULL;
+ }
+ }
+
+ if (!suballoc_bo) {
+ // TODO possibly larger size for streaming bo?
+ msm_ring->ring_bo = fd_bo_new_ring(
+ submit->pipe->dev, 0x8000, 0);
+ msm_ring->offset = 0;
+ } else {
+ msm_ring->ring_bo = fd_bo_ref(suballoc_bo);
+ msm_ring->offset = suballoc_offset;
+ }
+
+ struct fd_ringbuffer *old_suballoc_ring = msm_submit->suballoc_ring;
+
+ msm_submit->suballoc_ring = fd_ringbuffer_ref(&msm_ring->base);
+
+ if (old_suballoc_ring)
+ fd_ringbuffer_del(old_suballoc_ring);
+}
+
+static struct fd_ringbuffer *
+msm_submit_new_ringbuffer(struct fd_submit *submit, uint32_t size,
+ enum fd_ringbuffer_flags flags)
+{
+ struct msm_submit *msm_submit = to_msm_submit(submit);
+ struct msm_ringbuffer *msm_ring;
+
+ msm_ring = slab_alloc_st(&msm_submit->ring_pool);
+
+ msm_ring->u.submit = submit;
+
+ /* NOTE: needs to be before _suballoc_ring_bo() since it could
+ * increment the refcnt of the current ring
+ */
+ msm_ring->base.refcnt = 1;
+
+ if (flags & FD_RINGBUFFER_STREAMING) {
+ msm_submit_suballoc_ring_bo(submit, msm_ring, size);
+ } else {
+ if (flags & FD_RINGBUFFER_GROWABLE)
+ size = INIT_SIZE;
+
+ msm_ring->offset = 0;
+ msm_ring->ring_bo = fd_bo_new_ring(submit->pipe->dev, size, 0);
+ }
+
+ if (!msm_ringbuffer_init(msm_ring, size, flags))
+ return NULL;
+
+ if (flags & FD_RINGBUFFER_PRIMARY) {
+ debug_assert(!msm_submit->primary);
+ msm_submit->primary = fd_ringbuffer_ref(&msm_ring->base);
+ }
+
+ return &msm_ring->base;
+}
+
+static struct drm_msm_gem_submit_reloc *
+handle_stateobj_relocs(struct msm_submit *submit, struct msm_ringbuffer *ring)
+{
+ struct msm_cmd *cmd = ring->cmd;
+ struct drm_msm_gem_submit_reloc *relocs;
+
+ relocs = malloc(cmd->nr_relocs * sizeof(*relocs));
+
+ for (unsigned i = 0; i < cmd->nr_relocs; i++) {
+ unsigned idx = cmd->relocs[i].reloc_idx;
+ struct fd_bo *bo = ring->u.reloc_bos[idx].bo;
+ unsigned flags = 0;
+
+ if (ring->u.reloc_bos[idx].flags & MSM_SUBMIT_BO_READ)
+ flags |= FD_RELOC_READ;
+ if (ring->u.reloc_bos[idx].flags & MSM_SUBMIT_BO_WRITE)
+ flags |= FD_RELOC_WRITE;
+
+ relocs[i] = cmd->relocs[i];
+ relocs[i].reloc_idx = append_bo(submit, bo, flags);
+ }
+
+ return relocs;
+}
+
+static int
+msm_submit_flush(struct fd_submit *submit, int in_fence_fd,
+ int *out_fence_fd, uint32_t *out_fence)
+{
+ struct msm_submit *msm_submit = to_msm_submit(submit);
+ struct msm_pipe *msm_pipe = to_msm_pipe(submit->pipe);
+ struct drm_msm_gem_submit req = {
+ .flags = msm_pipe->pipe,
+ .queueid = msm_pipe->queue_id,
+ };
+ int ret;
+
+ debug_assert(msm_submit->primary);
+
+ finalize_current_cmd(msm_submit->primary);
+ append_ring(msm_submit->ring_set, msm_submit->primary);
+
+ struct set_entry *entry;
+ unsigned nr_cmds = 0;
+ unsigned nr_objs = 0;
+
+ set_foreach(msm_submit->ring_set, entry) {
+ struct fd_ringbuffer *ring = (void *)entry->key;
+ if (ring->flags & _FD_RINGBUFFER_OBJECT) {
+ nr_cmds += 1;
+ nr_objs += 1;
+ } else {
+ if (ring != msm_submit->primary)
+ finalize_current_cmd(ring);
+ nr_cmds += to_msm_ringbuffer(ring)->u.nr_cmds;
+ }
+ }
+
+ void *obj_relocs[nr_objs];
+ struct drm_msm_gem_submit_cmd cmds[nr_cmds];
+ unsigned i = 0, o = 0;
+
+ set_foreach(msm_submit->ring_set, entry) {
+ struct fd_ringbuffer *ring = (void *)entry->key;
+ struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
+
+ debug_assert(i < nr_cmds);
+
+ // TODO handle relocs:
+ if (ring->flags & _FD_RINGBUFFER_OBJECT) {
+
+ debug_assert(o < nr_objs);
+
+ void *relocs = handle_stateobj_relocs(msm_submit, msm_ring);
+ obj_relocs[o++] = relocs;
+
+ cmds[i].type = MSM_SUBMIT_CMD_IB_TARGET_BUF;
+ cmds[i].submit_idx =
+ append_bo(msm_submit, msm_ring->ring_bo, FD_RELOC_READ);
+ cmds[i].submit_offset = msm_ring->offset;
+ cmds[i].size = offset_bytes(ring->cur, ring->start);
+ cmds[i].pad = 0;
+ cmds[i].nr_relocs = msm_ring->cmd->nr_relocs;
+ cmds[i].relocs = VOID2U64(relocs);
+
+ i++;
+ } else {
+ for (unsigned j = 0; j < msm_ring->u.nr_cmds; j++) {
+ if (ring->flags & FD_RINGBUFFER_PRIMARY) {
+ cmds[i].type = MSM_SUBMIT_CMD_BUF;
+ } else {
+ cmds[i].type = MSM_SUBMIT_CMD_IB_TARGET_BUF;
+ }
+ cmds[i].submit_idx = append_bo(msm_submit,
+ msm_ring->u.cmds[j]->ring_bo, FD_RELOC_READ);
+ cmds[i].submit_offset = msm_ring->offset;
+ cmds[i].size = msm_ring->u.cmds[j]->size;
+ cmds[i].pad = 0;
+ cmds[i].nr_relocs = msm_ring->u.cmds[j]->nr_relocs;
+ cmds[i].relocs = VOID2U64(msm_ring->u.cmds[j]->relocs);
+
+ i++;
+ }
+ }
+ }
+
+ if (in_fence_fd != -1) {
+ req.flags |= MSM_SUBMIT_FENCE_FD_IN | MSM_SUBMIT_NO_IMPLICIT;
+ req.fence_fd = in_fence_fd;
+ }
+
+ if (out_fence_fd) {
+ req.flags |= MSM_SUBMIT_FENCE_FD_OUT;
+ }
+
+ /* needs to be after get_cmd() as that could create bos/cmds table: */
+ req.bos = VOID2U64(msm_submit->submit_bos),
+ req.nr_bos = msm_submit->nr_submit_bos;
+ req.cmds = VOID2U64(cmds),
+ req.nr_cmds = nr_cmds;
+
+ DEBUG_MSG("nr_cmds=%u, nr_bos=%u", req.nr_cmds, req.nr_bos);
+
+ ret = drmCommandWriteRead(submit->pipe->dev->fd, DRM_MSM_GEM_SUBMIT,
+ &req, sizeof(req));
+ if (ret) {
+ ERROR_MSG("submit failed: %d (%s)", ret, strerror(errno));
+ msm_dump_submit(&req);
+ } else if (!ret) {
+ if (out_fence)
+ *out_fence = req.fence;
+
+ if (out_fence_fd)
+ *out_fence_fd = req.fence_fd;
+ }
+
+ for (unsigned o = 0; o < nr_objs; o++)
+ free(obj_relocs[o]);
+
+ return ret;
+}
+
+static void
+unref_rings(struct set_entry *entry)
+{
+ struct fd_ringbuffer *ring = (void *)entry->key;
+ fd_ringbuffer_del(ring);
+}
+
+static void
+msm_submit_destroy(struct fd_submit *submit)
+{
+ struct msm_submit *msm_submit = to_msm_submit(submit);
+
+ if (msm_submit->primary)
+ fd_ringbuffer_del(msm_submit->primary);
+ if (msm_submit->suballoc_ring)
+ fd_ringbuffer_del(msm_submit->suballoc_ring);
+
+ _mesa_hash_table_destroy(msm_submit->bo_table, NULL);
+ _mesa_set_destroy(msm_submit->ring_set, unref_rings);
+
+ // TODO it would be nice to have a way to debug_assert() if all
+ // rb's haven't been free'd back to the slab, because that is
+ // an indication that we are leaking bo's
+ slab_destroy(&msm_submit->ring_pool);
+
+ for (unsigned i = 0; i < msm_submit->nr_bos; i++)
+ fd_bo_del(msm_submit->bos[i]);
+
+ free(msm_submit->submit_bos);
+ free(msm_submit->bos);
+ free(msm_submit);
+}
+
+static const struct fd_submit_funcs submit_funcs = {
+ .new_ringbuffer = msm_submit_new_ringbuffer,
+ .flush = msm_submit_flush,
+ .destroy = msm_submit_destroy,
+};
+
+struct fd_submit *
+msm_submit_new(struct fd_pipe *pipe)
+{
+ struct msm_submit *msm_submit = calloc(1, sizeof(*msm_submit));
+ struct fd_submit *submit;
+ static unsigned submit_cnt = 0;
+
+ msm_submit->seqno = ++submit_cnt;
+ msm_submit->bo_table = _mesa_hash_table_create(NULL,
+ _mesa_hash_pointer, _mesa_key_pointer_equal);
+ msm_submit->ring_set = _mesa_set_create(NULL,
+ _mesa_hash_pointer, _mesa_key_pointer_equal);
+ // TODO tune size:
+ slab_create(&msm_submit->ring_pool, sizeof(struct msm_ringbuffer), 16);
+
+ submit = &msm_submit->base;
+ submit->pipe = pipe;
+ submit->funcs = &submit_funcs;
+
+ return submit;
+}
+
+
+static void
+finalize_current_cmd(struct fd_ringbuffer *ring)
+{
+ struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
+
+ debug_assert(!(ring->flags & _FD_RINGBUFFER_OBJECT));
+
+ if (!msm_ring->cmd)
+ return;
+
+ debug_assert(msm_ring->cmd->ring_bo == msm_ring->ring_bo);
+
+ unsigned idx = APPEND(&msm_ring->u, cmds);
+
+ msm_ring->u.cmds[idx] = msm_ring->cmd;
+ msm_ring->cmd = NULL;
+
+ msm_ring->u.cmds[idx]->size = offset_bytes(ring->cur, ring->start);
+}
+
+static void
+msm_ringbuffer_grow(struct fd_ringbuffer *ring, uint32_t size)
+{
+ struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
+ struct fd_pipe *pipe = msm_ring->u.submit->pipe;
+
+ debug_assert(ring->flags & FD_RINGBUFFER_GROWABLE);
+
+ finalize_current_cmd(ring);
+
+ fd_bo_del(msm_ring->ring_bo);
+ msm_ring->ring_bo = fd_bo_new_ring(pipe->dev, size, 0);
+ msm_ring->cmd = cmd_new(msm_ring->ring_bo);
+
+ ring->start = fd_bo_map(msm_ring->ring_bo);
+ ring->end = &(ring->start[size/4]);
+ ring->cur = ring->start;
+ ring->size = size;
+}
+
+static void
+msm_ringbuffer_emit_reloc(struct fd_ringbuffer *ring,
+ const struct fd_reloc *reloc)
+{
+ struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
+ struct fd_pipe *pipe;
+ unsigned reloc_idx;
+
+ if (ring->flags & _FD_RINGBUFFER_OBJECT) {
+ unsigned idx = APPEND(&msm_ring->u, reloc_bos);
+
+ msm_ring->u.reloc_bos[idx].bo = fd_bo_ref(reloc->bo);
+ msm_ring->u.reloc_bos[idx].flags = reloc->flags;
+
+ /* this gets fixed up at submit->flush() time, since this state-
+ * object rb can be used with many different submits
+ */
+ reloc_idx = idx;
+
+ pipe = msm_ring->u.pipe;
+ } else {
+ struct msm_submit *msm_submit =
+ to_msm_submit(msm_ring->u.submit);
+
+ reloc_idx = append_bo(msm_submit, reloc->bo, reloc->flags);
+
+ pipe = msm_ring->u.submit->pipe;
+ }
+
+ struct drm_msm_gem_submit_reloc *r;
+ unsigned idx = APPEND(msm_ring->cmd, relocs);
+
+ r = &msm_ring->cmd->relocs[idx];
+
+ r->reloc_idx = reloc_idx;
+ r->reloc_offset = reloc->offset;
+ r->or = reloc->or;
+ r->shift = reloc->shift;
+ r->submit_offset = offset_bytes(ring->cur, ring->start) +
+ msm_ring->offset;
+
+ ring->cur++;
+
+ if (pipe->gpu_id >= 500) {
+ idx = APPEND(msm_ring->cmd, relocs);
+ r = &msm_ring->cmd->relocs[idx];
+
+ r->reloc_idx = reloc_idx;
+ r->reloc_offset = reloc->offset;
+ r->or = reloc->orhi;
+ r->shift = reloc->shift - 32;
+ r->submit_offset = offset_bytes(ring->cur, ring->start) +
+ msm_ring->offset;
+
+ ring->cur++;
+ }
+}
+
+static void
+append_stateobj_rings(struct msm_submit *submit, struct fd_ringbuffer *target)
+{
+ struct msm_ringbuffer *msm_target = to_msm_ringbuffer(target);
+
+ debug_assert(target->flags & _FD_RINGBUFFER_OBJECT);
+
+ struct set_entry *entry;
+ set_foreach(msm_target->u.ring_set, entry) {
+ struct fd_ringbuffer *ring = (void *)entry->key;
+
+ append_ring(submit->ring_set, ring);
+
+ if (ring->flags & _FD_RINGBUFFER_OBJECT) {
+ append_stateobj_rings(submit, ring);
+ }
+ }
+}
+
+static uint32_t
+msm_ringbuffer_emit_reloc_ring(struct fd_ringbuffer *ring,
+ struct fd_ringbuffer *target, uint32_t cmd_idx)
+{
+ struct msm_ringbuffer *msm_target = to_msm_ringbuffer(target);
+ struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
+ struct fd_bo *bo;
+ uint32_t size;
+
+ if ((target->flags & FD_RINGBUFFER_GROWABLE) &&
+ (cmd_idx < msm_target->u.nr_cmds)) {
+ bo = msm_target->u.cmds[cmd_idx]->ring_bo;
+ size = msm_target->u.cmds[cmd_idx]->size;
+ } else {
+ bo = msm_target->ring_bo;
+ size = offset_bytes(target->cur, target->start);
+ }
+
+ msm_ringbuffer_emit_reloc(ring, &(struct fd_reloc){
+ .bo = bo,
+ .flags = FD_RELOC_READ,
+ .offset = msm_target->offset,
+ });
+
+ if ((target->flags & _FD_RINGBUFFER_OBJECT) &&
+ !(ring->flags & _FD_RINGBUFFER_OBJECT)) {
+ struct msm_submit *msm_submit = to_msm_submit(msm_ring->u.submit);
+
+ append_stateobj_rings(msm_submit, target);
+ }
+
+ if (ring->flags & _FD_RINGBUFFER_OBJECT) {
+ append_ring(msm_ring->u.ring_set, target);
+ } else {
+ struct msm_submit *msm_submit = to_msm_submit(msm_ring->u.submit);
+ append_ring(msm_submit->ring_set, target);
+ }
+
+ return size;
+}
+
+static uint32_t
+msm_ringbuffer_cmd_count(struct fd_ringbuffer *ring)
+{
+ if (ring->flags & FD_RINGBUFFER_GROWABLE)
+ return to_msm_ringbuffer(ring)->u.nr_cmds + 1;
+ return 1;
+}
+
+static void
+msm_ringbuffer_destroy(struct fd_ringbuffer *ring)
+{
+ struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
+
+ fd_bo_del(msm_ring->ring_bo);
+ if (msm_ring->cmd)
+ cmd_free(msm_ring->cmd);
+
+ if (ring->flags & _FD_RINGBUFFER_OBJECT) {
+ for (unsigned i = 0; i < msm_ring->u.nr_reloc_bos; i++) {
+ fd_bo_del(msm_ring->u.reloc_bos[i].bo);
+ }
+
+ _mesa_set_destroy(msm_ring->u.ring_set, unref_rings);
+
+ free(msm_ring);
+ } else {
+ struct fd_submit *submit = msm_ring->u.submit;
+
+ for (unsigned i = 0; i < msm_ring->u.nr_cmds; i++) {
+ cmd_free(msm_ring->u.cmds[i]);
+ }
+
+ slab_free_st(&to_msm_submit(submit)->ring_pool, msm_ring);
+ }
+}
+
+static const struct fd_ringbuffer_funcs ring_funcs = {
+ .grow = msm_ringbuffer_grow,
+ .emit_reloc = msm_ringbuffer_emit_reloc,
+ .emit_reloc_ring = msm_ringbuffer_emit_reloc_ring,
+ .cmd_count = msm_ringbuffer_cmd_count,
+ .destroy = msm_ringbuffer_destroy,
+};
+
+static inline struct fd_ringbuffer *
+msm_ringbuffer_init(struct msm_ringbuffer *msm_ring, uint32_t size,
+ enum fd_ringbuffer_flags flags)
+{
+ struct fd_ringbuffer *ring = &msm_ring->base;
+
+ debug_assert(msm_ring->ring_bo);
+
+ uint8_t *base = fd_bo_map(msm_ring->ring_bo);
+ ring->start = (void *)(base + msm_ring->offset);
+ ring->end = &(ring->start[size/4]);
+ ring->cur = ring->start;
+
+ ring->size = size;
+ ring->flags = flags;
+
+ ring->funcs = &ring_funcs;
+
+ msm_ring->u.cmds = NULL;
+ msm_ring->u.nr_cmds = msm_ring->u.max_cmds = 0;
+
+ msm_ring->cmd = cmd_new(msm_ring->ring_bo);
+
+ return ring;
+}
+
+struct fd_ringbuffer *
+msm_ringbuffer_new_object(struct fd_pipe *pipe, uint32_t size)
+{
+ struct msm_ringbuffer *msm_ring = malloc(sizeof(*msm_ring));
+
+ msm_ring->u.pipe = pipe;
+ msm_ring->offset = 0;
+ msm_ring->ring_bo = fd_bo_new_ring(pipe->dev, size, 0);
+ msm_ring->base.refcnt = 1;
+
+ msm_ring->u.reloc_bos = NULL;
+ msm_ring->u.nr_reloc_bos = msm_ring->u.max_reloc_bos = 0;
+
+ msm_ring->u.ring_set = _mesa_set_create(NULL,
+ _mesa_hash_pointer, _mesa_key_pointer_equal);
+
+ return msm_ringbuffer_init(msm_ring, size, _FD_RINGBUFFER_OBJECT);
+}