gallium: move count_from_stream_output into pipe_draw_indirect_info

This removes some overhead from tc_draw_vbo and increases the maximum number
of draws per batch from 153 to 192 in u_threaded_context.

Reviewed-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/7441>

9 files changed, 28 insertions, 26 deletions

diff --git a/src/gallium/auxiliary/cso_cache/cso_context.c b/src/gallium/auxiliary/cso_cache/cso_context.c
index 3911ee73d30..2fdea6cc35f 100644
--- a/src/gallium/auxiliary/cso_cache/cso_context.c
+++ b/src/gallium/auxiliary/cso_cache/cso_context.c
@@ -1727,10 +1727,14 @@ cso_draw_vbo(struct cso_context *cso,
    struct u_vbuf *vbuf = cso->vbuf_current;
 
    /* We can't have both indirect drawing and SO-vertex-count drawing */
-   assert(info->indirect == NULL || info->count_from_stream_output == NULL);
+   assert(!info->indirect ||
+          info->indirect->buffer == NULL ||
+          info->indirect->count_from_stream_output == NULL);
 
    /* We can't have SO-vertex-count drawing with an index buffer */
-   assert(info->count_from_stream_output == NULL || info->index_size == 0);
+   assert(info->index_size == 0 ||
+          !info->indirect ||
+          info->indirect->count_from_stream_output == NULL);
 
    if (vbuf) {
       u_vbuf_draw_vbo(vbuf, info);
diff --git a/src/gallium/auxiliary/draw/draw_pt.c b/src/gallium/auxiliary/draw/draw_pt.c
index 0ea1f14c1e0..a1626c7171f 100644
--- a/src/gallium/auxiliary/draw/draw_pt.c
+++ b/src/gallium/auxiliary/draw/draw_pt.c
@@ -445,9 +445,9 @@ resolve_draw_info(const struct pipe_draw_info *raw_info,
 {
    memcpy(info, raw_info, sizeof(struct pipe_draw_info));
 
-   if (raw_info->count_from_stream_output) {
+   if (raw_info->indirect && raw_info->indirect->count_from_stream_output) {
       struct draw_so_target *target =
-         (struct draw_so_target *)info->count_from_stream_output;
+         (struct draw_so_target *)info->indirect->count_from_stream_output;
       assert(vertex_buffer != NULL);
       info->count = vertex_buffer->stride == 0 ? 0 :
                        target->internal_offset / vertex_buffer->stride;
@@ -455,6 +455,7 @@ resolve_draw_info(const struct pipe_draw_info *raw_info,
       /* Stream output draw can not be indexed */
       debug_assert(!info->index_size);
       info->max_index = info->count - 1;
+      info->indirect = NULL;
    }
 }
 
diff --git a/src/gallium/auxiliary/driver_ddebug/dd_draw.c b/src/gallium/auxiliary/driver_ddebug/dd_draw.c
index b93890f4f99..e9e3b493d89 100644
--- a/src/gallium/auxiliary/driver_ddebug/dd_draw.c
+++ b/src/gallium/auxiliary/driver_ddebug/dd_draw.c
@@ -356,13 +356,14 @@ dd_dump_draw_vbo(struct dd_draw_state *dstate, struct pipe_draw_info *info, FILE
    int sh, i;
 
    DUMP(draw_info, info);
-   if (info->count_from_stream_output)
-      DUMP_M(stream_output_target, info,
-             count_from_stream_output);
    if (info->indirect) {
-      DUMP_M(resource, info, indirect->buffer);
+      if (info->indirect->buffer)
+         DUMP_M(resource, info, indirect->buffer);
       if (info->indirect->indirect_draw_count)
          DUMP_M(resource, info, indirect->indirect_draw_count);
+      if (info->indirect->count_from_stream_output)
+         DUMP_M(stream_output_target, info,
+                indirect->count_from_stream_output);
    }
 
    fprintf(f, "\n");
@@ -705,7 +706,7 @@ dd_unreference_copy_of_call(struct dd_call *dst)
    case CALL_FLUSH:
       break;
    case CALL_DRAW_VBO:
-      pipe_so_target_reference(&dst->info.draw_vbo.draw.count_from_stream_output, NULL);
+      pipe_so_target_reference(&dst->info.draw_vbo.indirect.count_from_stream_output, NULL);
       pipe_resource_reference(&dst->info.draw_vbo.indirect.buffer, NULL);
       pipe_resource_reference(&dst->info.draw_vbo.indirect.indirect_draw_count, NULL);
       if (dst->info.draw_vbo.draw.index_size &&
@@ -1305,9 +1306,6 @@ dd_context_draw_vbo(struct pipe_context *_pipe,
 
    record->call.type = CALL_DRAW_VBO;
    record->call.info.draw_vbo.draw = *info;
-   record->call.info.draw_vbo.draw.count_from_stream_output = NULL;
-   pipe_so_target_reference(&record->call.info.draw_vbo.draw.count_from_stream_output,
-                            info->count_from_stream_output);
    if (info->index_size && !info->has_user_indices) {
       record->call.info.draw_vbo.draw.index.resource = NULL;
       pipe_resource_reference(&record->call.info.draw_vbo.draw.index.resource,
@@ -1324,6 +1322,9 @@ dd_context_draw_vbo(struct pipe_context *_pipe,
       record->call.info.draw_vbo.indirect.indirect_draw_count = NULL;
       pipe_resource_reference(&record->call.info.draw_vbo.indirect.indirect_draw_count,
                               info->indirect->indirect_draw_count);
+      record->call.info.draw_vbo.indirect.count_from_stream_output = NULL;
+      pipe_so_target_reference(&record->call.info.draw_vbo.indirect.count_from_stream_output,
+                               info->indirect->count_from_stream_output);
    } else {
       memset(&record->call.info.draw_vbo.indirect, 0, sizeof(*info->indirect));
    }
diff --git a/src/gallium/auxiliary/driver_trace/tr_dump_state.c b/src/gallium/auxiliary/driver_trace/tr_dump_state.c
index 49d2109101a..c63c373f923 100644
--- a/src/gallium/auxiliary/driver_trace/tr_dump_state.c
+++ b/src/gallium/auxiliary/driver_trace/tr_dump_state.c
@@ -793,7 +793,6 @@ void trace_dump_draw_info(const struct pipe_draw_info *state)
    trace_dump_member(uint, state, restart_index);
 
    trace_dump_member(ptr, state, index.resource);
-   trace_dump_member(ptr, state, count_from_stream_output);
 
    if (!state->indirect) {
       trace_dump_member(ptr, state, indirect);
@@ -804,6 +803,7 @@ void trace_dump_draw_info(const struct pipe_draw_info *state)
       trace_dump_member(uint, state, indirect->indirect_draw_count_offset);
       trace_dump_member(ptr, state, indirect->buffer);
       trace_dump_member(ptr, state, indirect->indirect_draw_count);
+      trace_dump_member(ptr, state, indirect->count_from_stream_output);
    }
 
    trace_dump_struct_end();
diff --git a/src/gallium/auxiliary/util/u_draw.c b/src/gallium/auxiliary/util/u_draw.c
index 90d01297c49..a6835204c72 100644
--- a/src/gallium/auxiliary/util/u_draw.c
+++ b/src/gallium/auxiliary/util/u_draw.c
@@ -139,7 +139,7 @@ util_draw_indirect(struct pipe_context *pipe,
    unsigned num_params = info_in->index_size ? 5 : 4;
 
    assert(info_in->indirect);
-   assert(!info_in->count_from_stream_output);
+   assert(!info_in->indirect->count_from_stream_output);
 
    memcpy(&info, info_in, sizeof(info));
 
diff --git a/src/gallium/auxiliary/util/u_dump_state.c b/src/gallium/auxiliary/util/u_dump_state.c
index f57a5923ad0..3cbd779bfd1 100644
--- a/src/gallium/auxiliary/util/u_dump_state.c
+++ b/src/gallium/auxiliary/util/u_dump_state.c
@@ -941,7 +941,6 @@ util_dump_draw_info(FILE *stream, const struct pipe_draw_info *state)
       else
          util_dump_member(stream, ptr, state, index.resource);
    }
-   util_dump_member(stream, ptr, state, count_from_stream_output);
 
    if (!state->indirect) {
       util_dump_member(stream, ptr, state, indirect);
@@ -952,6 +951,7 @@ util_dump_draw_info(FILE *stream, const struct pipe_draw_info *state)
       util_dump_member(stream, uint, state, indirect->indirect_draw_count_offset);
       util_dump_member(stream, ptr, state, indirect->buffer);
       util_dump_member(stream, ptr, state, indirect->indirect_draw_count);
+      util_dump_member(stream, ptr, state, indirect->count_from_stream_output);
    }
 
    util_dump_struct_end(stream);
diff --git a/src/gallium/auxiliary/util/u_prim_restart.c b/src/gallium/auxiliary/util/u_prim_restart.c
index eef2b4c622c..f7b34f9ebc8 100644
--- a/src/gallium/auxiliary/util/u_prim_restart.c
+++ b/src/gallium/auxiliary/util/u_prim_restart.c
@@ -114,7 +114,7 @@ util_translate_prim_restart_ib(struct pipe_context *context,
    dst_index_size = MAX2(2, info->index_size);
    assert(dst_index_size == 2 || dst_index_size == 4);
 
-   if (info->indirect) {
+   if (info->indirect && info->indirect->buffer) {
       indirect = read_indirect_elements(context, info->indirect);
       count = indirect.count;
       start = indirect.firstIndex;
@@ -235,7 +235,7 @@ util_draw_vbo_without_prim_restart(struct pipe_context *context,
    assert(info->index_size);
    assert(info->primitive_restart);
 
-   if (info->indirect) {
+   if (info->indirect && info->indirect->buffer) {
       indirect = read_indirect_elements(context, info->indirect);
       info_count = indirect.count;
       info_start = indirect.firstIndex;
diff --git a/src/gallium/auxiliary/util/u_threaded_context.c b/src/gallium/auxiliary/util/u_threaded_context.c
index 5da9a177fa8..f66280da6f6 100644
--- a/src/gallium/auxiliary/util/u_threaded_context.c
+++ b/src/gallium/auxiliary/util/u_threaded_context.c
@@ -126,7 +126,6 @@ tc_batch_execute(void *job, UNUSED int thread_index)
          if (next != last && next->call_id == TC_CALL_draw_vbo &&
              first_info->draw.drawid == 0 &&
              !first_info->draw.indirect &&
-             !first_info->draw.count_from_stream_output &&
              is_next_call_a_mergeable_draw(first_info, next, &next_info)) {
             /* Merge up to 256 draw calls. */
             struct pipe_draw_start_count multi[256];
@@ -2199,12 +2198,12 @@ tc_call_draw_vbo(struct pipe_context *pipe, union tc_payload *payload)
    struct tc_full_draw_info *info = (struct tc_full_draw_info*)payload;
 
    pipe->draw_vbo(pipe, &info->draw);
-   pipe_so_target_reference(&info->draw.count_from_stream_output, NULL);
    if (info->draw.index_size)
       pipe_resource_reference(&info->draw.index.resource, NULL);
    if (info->draw.indirect) {
       pipe_resource_reference(&info->indirect.buffer, NULL);
       pipe_resource_reference(&info->indirect.indirect_draw_count, NULL);
+      pipe_so_target_reference(&info->indirect.count_from_stream_output, NULL);
    }
 }
 
@@ -2243,9 +2242,6 @@ tc_draw_vbo(struct pipe_context *_pipe, const struct pipe_draw_info *info)
          return;
 
       struct tc_full_draw_info *p = tc_add_draw_vbo(_pipe, false);
-      p->draw.count_from_stream_output = NULL;
-      pipe_so_target_reference(&p->draw.count_from_stream_output,
-                               info->count_from_stream_output);
       memcpy(&p->draw, info, sizeof(*info));
       p->draw.has_user_indices = false;
       p->draw.index.resource = buffer;
@@ -2253,9 +2249,6 @@ tc_draw_vbo(struct pipe_context *_pipe, const struct pipe_draw_info *info)
    } else {
       /* Non-indexed call or indexed with a real index buffer. */
       struct tc_full_draw_info *p = tc_add_draw_vbo(_pipe, indirect != NULL);
-      p->draw.count_from_stream_output = NULL;
-      pipe_so_target_reference(&p->draw.count_from_stream_output,
-                               info->count_from_stream_output);
       if (index_size) {
          tc_set_resource_reference(&p->draw.index.resource,
                                    info->index.resource);
@@ -2266,6 +2259,9 @@ tc_draw_vbo(struct pipe_context *_pipe, const struct pipe_draw_info *info)
          tc_set_resource_reference(&p->draw.indirect->buffer, indirect->buffer);
          tc_set_resource_reference(&p->indirect.indirect_draw_count,
                                    indirect->indirect_draw_count);
+         p->indirect.count_from_stream_output = NULL;
+         pipe_so_target_reference(&p->indirect.count_from_stream_output,
+                                  indirect->count_from_stream_output);
          memcpy(&p->indirect, indirect, sizeof(*indirect));
          p->draw.indirect = &p->indirect;
       }
diff --git a/src/gallium/auxiliary/util/u_vbuf.c b/src/gallium/auxiliary/util/u_vbuf.c
index 511a97dd806..19898dc7dc5 100644
--- a/src/gallium/auxiliary/util/u_vbuf.c
+++ b/src/gallium/auxiliary/util/u_vbuf.c
@@ -1306,7 +1306,7 @@ void u_vbuf_draw_vbo(struct u_vbuf *mgr, const struct pipe_draw_info *info)
    new_info = *info;
 
    /* Handle indirect (multi)draws. */
-   if (new_info.indirect) {
+   if (new_info.indirect && new_info.indirect->buffer) {
       const struct pipe_draw_indirect_info *indirect = new_info.indirect;
       unsigned draw_count = 0;