Implement clEnqueueCopyBufferRect

6 files changed, 332 insertions, 100 deletions

diff --git a/src/api/api_enqueue.cpp b/src/api/api_enqueue.cpp
index f6c1c26..8f5e550 100644
--- a/src/api/api_enqueue.cpp
+++ b/src/api/api_enqueue.cpp
@@ -179,6 +179,44 @@ clEnqueueWriteBufferRect(cl_command_queue    command_queue,
 }
 
 cl_int
+clEnqueueCopyBufferRect(cl_command_queue    command_queue,
+                        cl_mem              src_buffer,
+                        cl_mem              dst_buffer,
+                        const size_t *      src_origin,
+                        const size_t *      dst_origin,
+                        const size_t *      region,
+                        size_t              src_row_pitch,
+                        size_t              src_slice_pitch,
+                        size_t              dst_row_pitch,
+                        size_t              dst_slice_pitch,
+                        cl_uint             num_events_in_wait_list,
+                        const cl_event *    event_wait_list,
+                        cl_event *          event)
+{
+    cl_int rs = CL_SUCCESS;
+
+    if (!command_queue)
+        return CL_INVALID_COMMAND_QUEUE;
+
+    Coal::CopyBufferRectEvent *command = new Coal::CopyBufferRectEvent(
+        (Coal::CommandQueue *)command_queue,
+        (Coal::MemObject *)src_buffer,
+        (Coal::MemObject *)dst_buffer,
+        src_origin, dst_origin, region, src_row_pitch, src_slice_pitch,
+        dst_row_pitch, dst_slice_pitch,
+        num_events_in_wait_list, (const Coal::Event **)event_wait_list, &rs
+    );
+
+    if (rs != CL_SUCCESS)
+    {
+        delete command;
+        return rs;
+    }
+
+    return queueEvent(command_queue, command, event, false);
+}
+
+cl_int
 clEnqueueCopyBuffer(cl_command_queue    command_queue,
                     cl_mem              src_buffer,
                     cl_mem              dst_buffer,
diff --git a/src/core/cpu/worker.cpp b/src/core/cpu/worker.cpp
index 576caa3..595f5a1 100644
--- a/src/core/cpu/worker.cpp
+++ b/src/core/cpu/worker.cpp
@@ -88,40 +88,59 @@ void *worker(void *data)
             }
             case Event::ReadBufferRect:
             case Event::WriteBufferRect:
+            case Event::CopyBufferRect:
             {
-                ReadWriteBufferRectEvent *e = (ReadWriteBufferRectEvent *)event;
-                CPUBuffer *buf = (CPUBuffer *)e->buffer()->deviceBuffer(device);
+                // src = buffer and dst = mem if note copy
+                ReadWriteCopyBufferRectEvent *e = (ReadWriteCopyBufferRectEvent *)event;
+                CPUBuffer *src_buf = (CPUBuffer *)e->source()->deviceBuffer(device);
+
+                unsigned char *src = (unsigned char *)src_buf->data();
+                unsigned char *dst;
+
+                if (t == Event::CopyBufferRect)
+                {
+                    CopyBufferRectEvent *cbre = (CopyBufferRectEvent *)e;
+                    CPUBuffer *dst_buf =
+                        (CPUBuffer *)cbre->destination()->deviceBuffer(device);
+
+                    dst = (unsigned char *)dst_buf->data();
+                }
+                else
+                {
+                    // dst = host memory location
+                    ReadWriteBufferRectEvent *rwbre = (ReadWriteBufferRectEvent *)e;
 
-                unsigned char *host = (unsigned char *)e->ptr();
-                unsigned char *mem = (unsigned char *)buf->data();
+                    dst = (unsigned char *)rwbre->ptr();
+                }
 
                 // Iterate over the lines to copy and use memcpy
                 for (size_t z=0; z<e->region(2); ++z)
                 {
                     for (size_t y=0; y<e->region(1); ++y)
                     {
-                        unsigned char *h;
-                        unsigned char *b;
-
-                        h = imageData(host,
-                                      e->host_origin(0),
-                                      y + e->host_origin(1),
-                                      z + e->host_origin(2),
-                                      e->host_row_pitch(),
-                                      e->host_slice_pitch(),
+                        unsigned char *s;
+                        unsigned char *d;
+
+                        d = imageData(dst,
+                                      e->dst_origin(0),
+                                      y + e->dst_origin(1),
+                                      z + e->dst_origin(2),
+                                      e->dst_row_pitch(),
+                                      e->dst_slice_pitch(),
                                       1);
 
-                        b = imageData(mem,
-                                      e->buffer_origin(0),
-                                      y + e->buffer_origin(1),
-                                      z + e->buffer_origin(2),
-                                      e->buffer_row_pitch(),
-                                      e->buffer_slice_pitch(),
+                        s = imageData(src,
+                                      e->src_origin(0),
+                                      y + e->src_origin(1),
+                                      z + e->src_origin(2),
+                                      e->src_row_pitch(),
+                                      e->src_slice_pitch(),
                                       1);
-                        if (t == Event::ReadBufferRect)
-                            std::memcpy(h, b, e->region(0));
+
+                        if (t == Event::WriteBufferRect)
+                            std::memcpy(s, d, e->region(0)); // Write dest (memory) in src
                         else
-                            std::memcpy(b, h, e->region(0));
+                            std::memcpy(d, s, e->region(0)); // Write src (buffer) in dest (memory), or copy the buffers
                     }
                 }
 
diff --git a/src/core/events.cpp b/src/core/events.cpp
index 024c588..70264a7 100644
--- a/src/core/events.cpp
+++ b/src/core/events.cpp
@@ -32,7 +32,7 @@ BufferEvent::BufferEvent(CommandQueue *parent,
     Context *ctx = 0;
     *errcode_ret = parent->info(CL_QUEUE_CONTEXT, sizeof(Context *), &ctx, 0);
 
-    if (errcode_ret != CL_SUCCESS) return;
+    if (*errcode_ret != CL_SUCCESS) return;
 
     if (buffer->context() != ctx)
     {
@@ -45,7 +45,7 @@ BufferEvent::BufferEvent(CommandQueue *parent,
     *errcode_ret = parent->info(CL_QUEUE_DEVICE, sizeof(DeviceInterface *),
                                 &device, 0);
 
-    if (errcode_ret != CL_SUCCESS)
+    if (*errcode_ret != CL_SUCCESS)
         return;
 
     if (!isSubBufferAligned(buffer, device))
@@ -87,7 +87,7 @@ bool BufferEvent::isSubBufferAligned(const MemObject *buffer,
     for (int i=0; i<align; ++i)
         mask = 1 | (mask << 1);
 
-    if (((SubBuffer *)buffer)->offset() | mask)
+    if (((SubBuffer *)buffer)->offset() & mask)
         return false;
 
     return true;
@@ -288,7 +288,7 @@ CopyBufferEvent::CopyBufferEvent(CommandQueue *parent,
     *errcode_ret = parent->info(CL_QUEUE_DEVICE, sizeof(DeviceInterface *),
                                 &device, 0);
 
-    if (errcode_ret != CL_SUCCESS)
+    if (*errcode_ret != CL_SUCCESS)
         return;
 
     if (!isSubBufferAligned(destination, device))
@@ -753,31 +753,24 @@ void UserEvent::flushQueues()
 /*
  * ReadWriteBufferRectEvent
  */
-ReadWriteBufferRectEvent::ReadWriteBufferRectEvent(CommandQueue *parent,
-                                                   MemObject *buffer,
-                                                   const size_t buffer_origin[3],
-                                                   const size_t host_origin[3],
-                                                   const size_t region[3],
-                                                   size_t buffer_row_pitch,
-                                                   size_t buffer_slice_pitch,
-                                                   size_t host_row_pitch,
-                                                   size_t host_slice_pitch,
-                                                   void *ptr,
-                                                   cl_uint num_events_in_wait_list,
-                                                   const Event **event_wait_list,
-                                                   cl_int *errcode_ret)
-: BufferEvent (parent, buffer, num_events_in_wait_list, event_wait_list,
-               errcode_ret), p_ptr(ptr)
+ReadWriteCopyBufferRectEvent::ReadWriteCopyBufferRectEvent(CommandQueue *parent,
+                                                           MemObject *source,
+                                                           const size_t src_origin[3],
+                                                           const size_t dst_origin[3],
+                                                           const size_t region[3],
+                                                           size_t src_row_pitch,
+                                                           size_t src_slice_pitch,
+                                                           size_t dst_row_pitch,
+                                                           size_t dst_slice_pitch,
+                                                           cl_uint num_events_in_wait_list,
+                                                           const Event **event_wait_list,
+                                                           cl_int *errcode_ret)
+: BufferEvent (parent, source, num_events_in_wait_list, event_wait_list,
+               errcode_ret)
 {
-    if (!ptr)
-    {
-        *errcode_ret = CL_INVALID_VALUE;
-        return;
-    }
-
     // Copy the vectors
-    std::memcpy(&p_buffer_origin, buffer_origin, 3 * sizeof(size_t));
-    std::memcpy(&p_host_origin, host_origin, 3 * sizeof(size_t));
+    std::memcpy(&p_src_origin, src_origin, 3 * sizeof(size_t));
+    std::memcpy(&p_dst_origin, dst_origin, 3 * sizeof(size_t));
 
     for (unsigned int i=0; i<3; ++i)
     {
@@ -790,101 +783,206 @@ ReadWriteBufferRectEvent::ReadWriteBufferRectEvent(CommandQueue *parent,
         p_region[i] = region[i];
     }
     // Compute the pitches
-    p_buffer_row_pitch = region[0];
+    p_src_row_pitch = region[0];
 
-    if (buffer_row_pitch)
+    if (src_row_pitch)
     {
-        if (buffer_row_pitch < p_buffer_row_pitch)
+        if (src_row_pitch < p_src_row_pitch)
         {
             *errcode_ret = CL_INVALID_VALUE;
             return;
         }
 
-        p_buffer_row_pitch = buffer_row_pitch;
+        p_src_row_pitch = src_row_pitch;
     }
 
-    p_buffer_slice_pitch = region[1] * p_buffer_row_pitch;
+    p_src_slice_pitch = region[1] * p_src_row_pitch;
 
-    if (buffer_slice_pitch)
+    if (src_slice_pitch)
     {
-        if (buffer_slice_pitch < p_buffer_slice_pitch)
+        if (src_slice_pitch < p_src_slice_pitch)
         {
             *errcode_ret = CL_INVALID_VALUE;
             return;
         }
 
-        p_buffer_slice_pitch = buffer_slice_pitch;
+        p_src_slice_pitch = src_slice_pitch;
     }
 
-    p_host_row_pitch = region[0];
+    p_dst_row_pitch = region[0];
 
-    if (host_row_pitch)
+    if (dst_row_pitch)
     {
-        if (host_row_pitch < p_host_row_pitch)
+        if (dst_row_pitch < p_dst_row_pitch)
         {
             *errcode_ret = CL_INVALID_VALUE;
             return;
         }
 
-        p_host_row_pitch = host_row_pitch;
+        p_dst_row_pitch = dst_row_pitch;
     }
 
-    p_host_slice_pitch = region[1] * p_host_row_pitch;
+    p_dst_slice_pitch = region[1] * p_dst_row_pitch;
 
-    if (host_slice_pitch)
+    if (dst_slice_pitch)
     {
-        if (host_slice_pitch < p_host_slice_pitch)
+        if (dst_slice_pitch < p_dst_slice_pitch)
         {
             *errcode_ret = CL_INVALID_VALUE;
             return;
         }
 
-        p_host_slice_pitch = host_slice_pitch;
-    }
-
-    // Check for out-of-bounds
-    if (buffer_origin[0] + region[0] > p_buffer_row_pitch ||
-        (buffer_origin[1] + region[1]) * p_buffer_row_pitch > p_buffer_slice_pitch ||
-        (buffer_origin[2] + region[2]) * p_buffer_slice_pitch > buffer->size())
-    {
-        *errcode_ret = CL_INVALID_VALUE;
-        return;
+        p_dst_slice_pitch = dst_slice_pitch;
     }
 }
 
-size_t ReadWriteBufferRectEvent::buffer_origin(unsigned int index) const
+size_t ReadWriteCopyBufferRectEvent::src_origin(unsigned int index) const
 {
-    return p_buffer_origin[index];
+    return p_src_origin[index];
 }
 
-size_t ReadWriteBufferRectEvent::host_origin(unsigned int index) const
+size_t ReadWriteCopyBufferRectEvent::dst_origin(unsigned int index) const
 {
-    return p_host_origin[index];
+    return p_dst_origin[index];
 }
 
-size_t ReadWriteBufferRectEvent::region(unsigned int index) const
+size_t ReadWriteCopyBufferRectEvent::region(unsigned int index) const
 {
     return p_region[index];
 }
 
-size_t ReadWriteBufferRectEvent::buffer_row_pitch() const
+size_t ReadWriteCopyBufferRectEvent::src_row_pitch() const
+{
+    return p_src_row_pitch;
+}
+
+size_t ReadWriteCopyBufferRectEvent::src_slice_pitch() const
+{
+    return p_src_slice_pitch;
+}
+
+size_t ReadWriteCopyBufferRectEvent::dst_row_pitch() const
+{
+    return p_dst_row_pitch;
+}
+
+size_t ReadWriteCopyBufferRectEvent::dst_slice_pitch() const
+{
+    return p_dst_slice_pitch;
+}
+
+MemObject *ReadWriteCopyBufferRectEvent::source() const
 {
-    return p_buffer_row_pitch;
+    return buffer();
+}
+
+CopyBufferRectEvent::CopyBufferRectEvent(CommandQueue *parent,
+                                         MemObject *source,
+                                         MemObject *destination,
+                                         const size_t src_origin[3],
+                                         const size_t dst_origin[3],
+                                         const size_t region[3],
+                                         size_t src_row_pitch,
+                                         size_t src_slice_pitch,
+                                         size_t dst_row_pitch,
+                                         size_t dst_slice_pitch,
+                                         cl_uint num_events_in_wait_list,
+                                         const Event **event_wait_list,
+                                         cl_int *errcode_ret)
+: ReadWriteCopyBufferRectEvent(parent, source, src_origin, dst_origin, region,
+                               src_row_pitch, src_slice_pitch, dst_row_pitch,
+                               dst_slice_pitch, num_events_in_wait_list,
+                               event_wait_list, errcode_ret),
+  p_destination(destination)
+{
+    if (!destination)
+    {
+        *errcode_ret = CL_INVALID_MEM_OBJECT;
+        return;
+    }
+
+    // Check for out-of-bounds
+    if (src_origin[0] + region[0] > this->src_row_pitch() ||
+        (src_origin[1] + region[1]) * this->src_row_pitch() > this->src_slice_pitch() ||
+        (src_origin[2] + region[2]) * this->src_slice_pitch() > source->size())
+    {
+        *errcode_ret = CL_INVALID_VALUE;
+        return;
+    }
+
+    if (dst_origin[0] + region[0] > this->dst_row_pitch() ||
+        (dst_origin[1] + region[1]) * this->dst_row_pitch() > this->dst_slice_pitch() ||
+        (dst_origin[2] + region[2]) * this->dst_slice_pitch() > destination->size())
+    {
+        *errcode_ret = CL_INVALID_VALUE;
+        return;
+    }
+
+    // Check alignment of destination (source already checked by BufferEvent)
+    DeviceInterface *device = 0;
+    *errcode_ret = parent->info(CL_QUEUE_DEVICE, sizeof(DeviceInterface *),
+                                &device, 0);
+
+    if (*errcode_ret != CL_SUCCESS)
+        return;
+
+    if (!isSubBufferAligned(destination, device))
+    {
+        *errcode_ret = CL_MISALIGNED_SUB_BUFFER_OFFSET;
+        return;
+    }
+
+    // Allocate the buffer for the device
+    if (!destination->allocate(device))
+    {
+        *errcode_ret = CL_MEM_OBJECT_ALLOCATION_FAILURE;
+        return;
+    }
 }
 
-size_t ReadWriteBufferRectEvent::buffer_slice_pitch() const
+Event::Type CopyBufferRectEvent::type() const
 {
-    return p_buffer_slice_pitch;
+    return Event::CopyBufferRect;
 }
 
-size_t ReadWriteBufferRectEvent::host_row_pitch() const
+MemObject *CopyBufferRectEvent::destination() const
 {
-    return p_host_row_pitch;
+    return p_destination;
 }
 
-size_t ReadWriteBufferRectEvent::host_slice_pitch() const
+ReadWriteBufferRectEvent::ReadWriteBufferRectEvent(CommandQueue *parent,
+                                                   MemObject *buffer,
+                                                   const size_t buffer_origin[3],
+                                                   const size_t host_origin[3],
+                                                   const size_t region[3],
+                                                   size_t buffer_row_pitch,
+                                                   size_t buffer_slice_pitch,
+                                                   size_t host_row_pitch,
+                                                   size_t host_slice_pitch,
+                                                   void *ptr,
+                                                   cl_uint num_events_in_wait_list,
+                                                   const Event **event_wait_list,
+                                                   cl_int *errcode_ret)
+: ReadWriteCopyBufferRectEvent(parent, buffer, buffer_origin, host_origin, region,
+                               buffer_row_pitch, buffer_slice_pitch,
+                               host_row_pitch, host_slice_pitch,
+                               num_events_in_wait_list, event_wait_list, errcode_ret),
+  p_ptr(ptr)
 {
-    return p_host_slice_pitch;
+    if (!ptr)
+    {
+        *errcode_ret = CL_INVALID_VALUE;
+        return;
+    }
+
+    // Check for out-of-bounds
+    if (buffer_origin[0] + region[0] > src_row_pitch() ||
+        (buffer_origin[1] + region[1]) * src_row_pitch() > src_slice_pitch() ||
+        (buffer_origin[2] + region[2]) * src_slice_pitch() > buffer->size())
+    {
+        *errcode_ret = CL_INVALID_VALUE;
+        return;
+    }
 }
 
 void *ReadWriteBufferRectEvent::ptr() const
diff --git a/src/core/events.h b/src/core/events.h
index c08680c..0376007 100644
--- a/src/core/events.h
+++ b/src/core/events.h
@@ -152,7 +152,62 @@ class CopyBufferEvent : public BufferEvent
         size_t p_src_offset, p_dst_offset, p_cb;
 };
 
-class ReadWriteBufferRectEvent : public BufferEvent
+class ReadWriteCopyBufferRectEvent : public BufferEvent
+{
+    public:
+        ReadWriteCopyBufferRectEvent(CommandQueue *parent,
+                                     MemObject *source,
+                                     const size_t src_origin[3],
+                                     const size_t dst_origin[3],
+                                     const size_t region[3],
+                                     size_t src_row_pitch,
+                                     size_t src_slice_pitch,
+                                     size_t dst_row_pitch,
+                                     size_t dst_slice_pitch,
+                                     cl_uint num_events_in_wait_list,
+                                     const Event **event_wait_list,
+                                     cl_int *errcode_ret);
+
+        size_t src_origin(unsigned int index) const;
+        size_t dst_origin(unsigned int index) const;
+        size_t region(unsigned int index) const;
+        size_t src_row_pitch() const;
+        size_t src_slice_pitch() const;
+        size_t dst_row_pitch() const;
+        size_t dst_slice_pitch() const;
+        MemObject *source() const;
+
+    private:
+        size_t p_src_origin[3], p_dst_origin[3], p_region[3];
+        size_t p_src_row_pitch, p_src_slice_pitch;
+        size_t p_dst_row_pitch, p_dst_slice_pitch;
+};
+
+class CopyBufferRectEvent : public ReadWriteCopyBufferRectEvent
+{
+    public:
+        CopyBufferRectEvent(CommandQueue *parent,
+                            MemObject *source,
+                            MemObject *destination,
+                            const size_t src_origin[3],
+                            const size_t dst_origin[3],
+                            const size_t region[3],
+                            size_t src_row_pitch,
+                            size_t src_slice_pitch,
+                            size_t dst_row_pitch,
+                            size_t dst_slice_pitch,
+                            cl_uint num_events_in_wait_list,
+                            const Event **event_wait_list,
+                            cl_int *errcode_ret);
+
+        Type type() const;
+        MemObject *destination() const;
+
+    private:
+        MemObject *p_destination;
+};
+
+class ReadWriteBufferRectEvent : public ReadWriteCopyBufferRectEvent
 {
     public:
         ReadWriteBufferRectEvent(CommandQueue *parent,
@@ -169,19 +224,9 @@ class ReadWriteBufferRectEvent : public BufferEvent
                                  const Event **event_wait_list,
                                  cl_int *errcode_ret);
 
-        size_t buffer_origin(unsigned int index) const;
-        size_t host_origin(unsigned int index) const;
-        size_t region(unsigned int index) const;
-        size_t buffer_row_pitch() const;
-        size_t buffer_slice_pitch() const;
-        size_t host_row_pitch() const;
-        size_t host_slice_pitch() const;
         void *ptr() const;
 
     private:
-        size_t p_buffer_origin[3], p_host_origin[3], p_region[3];
-        size_t p_buffer_row_pitch, p_buffer_slice_pitch;
-        size_t p_host_row_pitch, p_host_slice_pitch;
         void *p_ptr;
 };
 
diff --git a/tests/test_commandqueue.cpp b/tests/test_commandqueue.cpp
index cbf7c1f..ea27497 100644
--- a/tests/test_commandqueue.cpp
+++ b/tests/test_commandqueue.cpp
@@ -1,4 +1,5 @@
 #include <cstring>
+#include <cstdlib>
 #include <iostream>
 
 #include "test_commandqueue.h"
@@ -352,7 +353,7 @@ START_TEST (test_read_write_rect)
     cl_context ctx;
     cl_command_queue queue;
     cl_int result;
-    cl_mem buf;
+    cl_mem buf, buf_part;
 
     // Grid xyz = (5 x 7 x 2)
     unsigned char grid[70] = {
@@ -414,6 +415,13 @@ START_TEST (test_read_write_rect)
         "cannot create a valid CL_MEM_USE_HOST_PTR read-write buffer"
     );
 
+    buf_part = clCreateBuffer(ctx, CL_MEM_WRITE_ONLY | CL_MEM_USE_HOST_PTR,
+                              sizeof(buffer_part), buffer_part, &result);
+    fail_if(
+        result != CL_SUCCESS,
+        "cannot create a buffer for the part that will be read"
+    );
+
     // Write grid into buffer
     result = clEnqueueWriteBufferRect(queue, buf, 1, buf_origin, host_origin,
                                       region, 0, 0, 0, 0, grid, 0, 0, 0);
@@ -446,6 +454,29 @@ START_TEST (test_read_write_rect)
         "the part of the buffer was not correctly read"
     );
 
+    // Clear the temporary region and re-read into it using buf_part
+    std::memset(buffer_part, 0, sizeof(buffer_part));
+    cl_event event;
+
+    result = clEnqueueCopyBufferRect(queue, buf, buf_part, buf_origin,
+                                     host_origin, region, 5, 5*7, 0, 0, 0, 0, &event);
+    fail_if(
+        result != CL_SUCCESS,
+        "unable to queue a copy buffer rect event"
+    );
+
+    result = clWaitForEvents(1, &event);
+    fail_if(
+        result != CL_SUCCESS,
+        "unable to wait for the event"
+    );
+
+    fail_if(
+        std::memcmp(buffer_part, part, sizeof(part)) != 0,
+        "the part of the buffer was not correctly read using a buffer"
+    );
+
+    clReleaseMemObject(buf_part);
     clReleaseMemObject(buf);
     clReleaseCommandQueue(queue);
     clReleaseContext(ctx);
diff --git a/tests/test_mem.cpp b/tests/test_mem.cpp
index b1fd477..cbb37e2 100644
--- a/tests/test_mem.cpp
+++ b/tests/test_mem.cpp
@@ -1,7 +1,8 @@
+#include <iostream>
+
 #include "test_mem.h"
 #include "CL/cl.h"
 
-
 START_TEST (test_create_buffer)
 {
     cl_context ctx;