summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorThomas Hellstrom <thellstrom@vmware.com>2009-08-17 17:17:24 +0200
committerThomas Hellstrom <thellstrom@vmware.com>2009-08-17 17:17:24 +0200
commit3e8e650f25e5e40fd30b038ee3abf175523d42a6 (patch)
tree9672e7072483658ef81ab4d85bd020e2dcb7a59d
parent375a9b16472031d603fd0d2f3cbea3e1ed9bc6c7 (diff)
libwsbm: Big update for libwsbm 2.0.
Prepare for per-device libs, Split out command submission, Rework driver interface, Bump major.
-rw-r--r--configure.ac2
-rw-r--r--src/Makefile.am15
-rw-r--r--src/wsbm_cs.c531
-rw-r--r--src/wsbm_cs.h69
-rw-r--r--src/wsbm_driver.h6
-rw-r--r--src/wsbm_fencemgr.c108
-rw-r--r--src/wsbm_fencemgr.h11
-rw-r--r--src/wsbm_mallocpool.c9
-rw-r--r--src/wsbm_manager.c723
-rw-r--r--src/wsbm_manager.h104
-rw-r--r--src/wsbm_pool.h36
-rw-r--r--src/wsbm_slabpool.c239
-rw-r--r--src/wsbm_ttmpool.c518
-rw-r--r--src/wsbm_userpool.c102
14 files changed, 950 insertions, 1523 deletions
diff --git a/configure.ac b/configure.ac
index 24191f2..63feff7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,5 +1,5 @@
AC_PREREQ(2.57)
-AC_INIT([libwsbm], 1.1.0, [thomas@tungstengraphics.com], libwsbm)
+AC_INIT([libwsbm], 2.0.0, [thomas@vmware.com], libwsbm)
AC_CONFIG_SRCDIR([Makefile.am])
AM_INIT_AUTOMAKE([dist-bzip2])
diff --git a/src/Makefile.am b/src/Makefile.am
index 4778f37..b9be786 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1,13 +1,14 @@
-
lib_LTLIBRARIES = libwsbm.la
-libwsbm_la_CFLAGS = @libdrm_CFLAGS@ -Wall
-libwsbm_la_LDFLAGS = -version-number 1:1:0 -no-undefined @libdrm_LIBS@
+libwsbm_la_CFLAGS = -Wall @libdrm_CFLAGS@
+libwsbm_la_LDFLAGS = -version-number 2:0:0 -no-undefined
libwsbm_la_SOURCES = \
wsbm_fencemgr.c \
wsbm_fencemgr.h \
wsbm_manager.c \
wsbm_manager.h \
+ wsbm_cs.c \
+ wsbm_cs.h \
wsbm_mm.c \
wsbm_mm.h \
wsbm_pool.h \
@@ -15,13 +16,11 @@ libwsbm_la_SOURCES = \
wsbm_mallocpool.c \
wsbm_driver.h \
wsbm_driver.c \
- wsbm_ttmpool.c \
wsbm_slabpool.c \
wsbm_userpool.c \
- wsbm_priv.h
+ wsbm_priv.h \
+ wsbm_atomic.h
libwsbmincludedir = ${includedir}/wsbm
-libwsbminclude_HEADERS = wsbm_manager.h wsbm_pool.h wsbm_driver.h \
- wsbm_fencemgr.h wsbm_util.h wsbm_atomic.h
-
+libwsbminclude_HEADERS = wsbm_manager.h wsbm_fencemgr.h
diff --git a/src/wsbm_cs.c b/src/wsbm_cs.c
new file mode 100644
index 0000000..177bcb8
--- /dev/null
+++ b/src/wsbm_cs.c
@@ -0,0 +1,531 @@
+/**************************************************************************
+ *
+ * Copyright 2006-2009 Vmware, Inc., Palo Alto, CA., USA
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+/*
+ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "wsbm_cs.h"
+#include "wsbm_pool.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <errno.h>
+
+extern struct _WsbmBufStorage *
+wsbmBOStorage(struct _WsbmBufferObject *buf);
+
+struct _ValidateList
+{
+ const struct _WsbmDriver *driver;
+ unsigned numTarget;
+ unsigned numCurrent;
+ unsigned numOnList;
+ unsigned hashSize;
+ uint32_t hashMask;
+ struct _WsbmListHead list;
+ struct _WsbmListHead free;
+ struct _WsbmListHead *hashTable;
+};
+
+struct _WsbmBufferList
+{
+ int hasKernelBuffers;
+
+ struct _ValidateList kernelBuffers; /* List of kernel buffers needing validation */
+ struct _ValidateList userBuffers; /* List of user-space buffers needing validation */
+};
+
+static struct _ValidateNode *
+validateListAddNode(struct _ValidateList *list, void *item,
+ uint32_t hash, uint64_t flags, uint64_t mask)
+{
+ struct _ValidateNode *node;
+ struct _WsbmListHead *l;
+ struct _WsbmListHead *hashHead;
+
+ l = list->free.next;
+ if (l == &list->free) {
+ node = list->driver->alloc(list->driver);
+ if (!node) {
+ return NULL;
+ }
+ list->numCurrent++;
+ } else {
+ WSBMLISTDEL(l);
+ node = WSBMLISTENTRY(l, struct _ValidateNode, head);
+ }
+ node->buf = item;
+ node->set_flags = flags & mask;
+ node->clr_flags = (~flags) & mask;
+ node->listItem = list->numOnList;
+ WSBMLISTADDTAIL(&node->head, &list->list);
+ list->numOnList++;
+ hashHead = list->hashTable + hash;
+ WSBMLISTADDTAIL(&node->hashHead, hashHead);
+
+ return node;
+}
+
+static uint32_t
+wsbmHashFunc(uint8_t * key, uint32_t len, uint32_t mask)
+{
+ uint32_t hash, i;
+
+ for (hash = 0, i = 0; i < len; ++i) {
+ hash += *key++;
+ hash += (hash << 10);
+ hash ^= (hash >> 6);
+ }
+
+ hash += (hash << 3);
+ hash ^= (hash >> 11);
+ hash += (hash << 15);
+
+ return hash & mask;
+}
+
+static void
+validateFreeList(struct _ValidateList *list)
+{
+ struct _ValidateNode *node;
+ struct _WsbmListHead *l;
+
+ l = list->list.next;
+ while (l != &list->list) {
+ WSBMLISTDEL(l);
+ node = WSBMLISTENTRY(l, struct _ValidateNode, head);
+
+ WSBMLISTDEL(&node->hashHead);
+ node->driver->free(node);
+ l = list->list.next;
+ list->numCurrent--;
+ list->numOnList--;
+ }
+
+ l = list->free.next;
+ while (l != &list->free) {
+ WSBMLISTDEL(l);
+ node = WSBMLISTENTRY(l, struct _ValidateNode, head);
+
+ node->driver->free(node);
+ l = list->free.next;
+ list->numCurrent--;
+ }
+ free(list->hashTable);
+}
+
+static int
+validateListAdjustNodes(struct _ValidateList *list)
+{
+ struct _ValidateNode *node;
+ struct _WsbmListHead *l;
+ int ret = 0;
+
+ while (list->numCurrent < list->numTarget) {
+ node = list->driver->alloc(list->driver);
+ if (!node) {
+ ret = -ENOMEM;
+ break;
+ }
+ list->numCurrent++;
+ WSBMLISTADD(&node->head, &list->free);
+ }
+
+ while (list->numCurrent > list->numTarget) {
+ l = list->free.next;
+ if (l == &list->free)
+ break;
+ WSBMLISTDEL(l);
+ node = WSBMLISTENTRY(l, struct _ValidateNode, head);
+
+ node->driver->free(node);
+ list->numCurrent--;
+ }
+ return ret;
+}
+
+static inline int
+wsbmPot(unsigned int val)
+{
+ unsigned int shift = 0;
+ while(val > (1 << shift))
+ shift++;
+
+ return shift;
+}
+
+static int
+validateCreateList(int numTarget, struct _ValidateList *list,
+ const struct _WsbmDriver *driver)
+{
+ int i;
+ unsigned int shift = wsbmPot(numTarget);
+ int ret;
+
+ list->hashSize = (1 << shift);
+ list->hashMask = list->hashSize - 1;
+
+ list->hashTable = malloc(list->hashSize * sizeof(*list->hashTable));
+ if (!list->hashTable)
+ return -ENOMEM;
+
+ for (i = 0; i < list->hashSize; ++i)
+ WSBMINITLISTHEAD(&list->hashTable[i]);
+
+ WSBMINITLISTHEAD(&list->list);
+ WSBMINITLISTHEAD(&list->free);
+ list->numTarget = numTarget;
+ list->numCurrent = 0;
+ list->numOnList = 0;
+ list->driver = driver;
+ ret = validateListAdjustNodes(list);
+ if (ret != 0)
+ free(list->hashTable);
+
+ return ret;
+}
+
+static int
+validateResetList(struct _ValidateList *list)
+{
+ struct _WsbmListHead *l;
+ struct _ValidateNode *node;
+ int ret;
+
+ ret = validateListAdjustNodes(list);
+ if (ret)
+ return ret;
+
+ l = list->list.next;
+ while (l != &list->list) {
+ WSBMLISTDEL(l);
+ node = WSBMLISTENTRY(l, struct _ValidateNode, head);
+
+ WSBMLISTDEL(&node->hashHead);
+ WSBMLISTADD(l, &list->free);
+ list->numOnList--;
+ l = list->list.next;
+ }
+ return validateListAdjustNodes(list);
+}
+
+
+/*
+ * Note that lists are per-context and don't need mutex protection.
+ */
+
+struct _WsbmBufferList *
+wsbmBOCreateList(int target, int hasKernelBuffers,
+ const struct _WsbmDriver *kernelDriver,
+ const struct _WsbmDriver *userDriver)
+{
+ struct _WsbmBufferList *list = calloc(sizeof(*list), 1);
+ int ret;
+
+ list->hasKernelBuffers = hasKernelBuffers;
+ if (hasKernelBuffers) {
+ ret = validateCreateList(target, &list->kernelBuffers, kernelDriver);
+ if (ret)
+ return NULL;
+ }
+
+ ret = validateCreateList(target, &list->userBuffers, userDriver);
+ if (ret) {
+ validateFreeList(&list->kernelBuffers);
+ return NULL;
+ }
+
+ return list;
+}
+
+int
+wsbmBOResetList(struct _WsbmBufferList *list)
+{
+ int ret;
+
+ if (list->hasKernelBuffers) {
+ ret = validateResetList(&list->kernelBuffers);
+ if (ret)
+ return ret;
+ }
+ ret = validateResetList(&list->userBuffers);
+ return ret;
+}
+
+void
+wsbmBOFreeList(struct _WsbmBufferList *list)
+{
+ if (list->hasKernelBuffers)
+ validateFreeList(&list->kernelBuffers);
+ validateFreeList(&list->userBuffers);
+ free(list);
+}
+
+static int
+wsbmAddValidateItem(struct _ValidateList *list, void *buf, uint64_t flags,
+ uint64_t mask, int *itemLoc,
+ struct _ValidateNode **pnode, int *newItem)
+{
+ struct _ValidateNode *node, *cur;
+ struct _WsbmListHead *l;
+ struct _WsbmListHead *hashHead;
+ uint32_t hash;
+ uint32_t count = 0;
+ uint32_t key = (unsigned long) buf;
+
+ cur = NULL;
+ hash = wsbmHashFunc((uint8_t *) &key, 4, list->hashMask);
+ hashHead = list->hashTable + hash;
+ *newItem = 0;
+
+ for (l = hashHead->next; l != hashHead; l = l->next) {
+ count++;
+ node = WSBMLISTENTRY(l, struct _ValidateNode, hashHead);
+
+ if (node->buf == buf) {
+ cur = node;
+ break;
+ }
+ }
+
+ if (!cur) {
+ cur = validateListAddNode(list, buf, hash, flags, mask);
+ if (!cur)
+ return -ENOMEM;
+ *newItem = 1;
+ cur->driver->clear(cur);
+ } else {
+ uint64_t set_flags = flags & mask;
+ uint64_t clr_flags = (~flags) & mask;
+
+ if (((cur->clr_flags | clr_flags) & WSBM_PLACEMENT_MASK) ==
+ WSBM_PLACEMENT_MASK) {
+ /*
+ * No available memory type left. Bail.
+ */
+ return -EINVAL;
+ }
+
+ if ((cur->set_flags | set_flags) &
+ (cur->clr_flags | clr_flags) & ~WSBM_PLACEMENT_MASK) {
+ /*
+ * Conflicting flags. Bail.
+ */
+ return -EINVAL;
+ }
+
+ cur->set_flags |= set_flags;
+ cur->clr_flags |= clr_flags;
+ cur->set_flags &= ~(cur->clr_flags);
+ }
+ *itemLoc = cur->listItem;
+ if (pnode)
+ *pnode = cur;
+ return 0;
+}
+
+
+int
+wsbmBOAddListItem(struct _WsbmBufferList *list,
+ struct _WsbmBufferObject *buf,
+ uint64_t flags, uint64_t mask, int *itemLoc,
+ struct _ValidateNode **node)
+{
+ int newItem;
+ struct _WsbmBufStorage *storage = wsbmBOStorage(buf);
+ int ret;
+ int dummy;
+ struct _ValidateNode *dummyNode;
+
+ if (list->hasKernelBuffers) {
+ ret = wsbmAddValidateItem(&list->kernelBuffers,
+ storage->pool->kernel(storage),
+ flags, mask, itemLoc, node, &dummy);
+ if (ret)
+ goto out_unlock;
+ } else {
+ *node = NULL;
+ *itemLoc = -1000;
+ }
+
+ ret = wsbmAddValidateItem(&list->userBuffers, storage,
+ flags, mask, &dummy, &dummyNode, &newItem);
+ if (ret)
+ goto out_unlock;
+
+ if (newItem) {
+ wsbmAtomicInc(&storage->refCount);
+ wsbmAtomicInc(&storage->onList);
+ }
+
+ out_unlock:
+ return ret;
+}
+
+int
+wsbmBOUnrefUserList(struct _WsbmBufferList *list)
+{
+ struct _WsbmBufStorage *storage;
+ void *curBuf;
+
+ curBuf = validateListIterator(&list->userBuffers);
+
+ while (curBuf) {
+ storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf);
+ wsbmAtomicDec(&storage->onList);
+ wsbmBufStorageUnref(&storage);
+ curBuf = validateListNext(&list->userBuffers, curBuf);
+ }
+
+ return wsbmBOResetList(list);
+}
+
+
+int
+wsbmBOFenceUserList(struct _WsbmBufferList *list,
+ struct _WsbmFenceObject *fence)
+{
+ struct _WsbmBufStorage *storage;
+ void *curBuf;
+
+ curBuf = validateListIterator(&list->userBuffers);
+
+ /*
+ * User-space fencing callbacks.
+ */
+
+ while (curBuf) {
+ storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf);
+
+ storage->pool->fence(storage, fence);
+ wsbmAtomicDec(&storage->onList);
+ wsbmBufStorageUnref(&storage);
+ curBuf = validateListNext(&list->userBuffers, curBuf);
+ }
+
+ return wsbmBOResetList(list);
+}
+
+int
+wsbmBOValidateUserList(struct _WsbmBufferList *list)
+{
+ void *curBuf;
+ struct _WsbmBufStorage *storage;
+ struct _ValidateNode *node;
+ int ret;
+
+ curBuf = validateListIterator(&list->userBuffers);
+
+ /*
+ * User-space validation callbacks.
+ */
+
+ while (curBuf) {
+ node = validateListNode(curBuf);
+ storage = (struct _WsbmBufStorage *)node->buf;
+ if (storage->pool->validate) {
+ ret = storage->pool->validate(storage, node->set_flags,
+ node->clr_flags);
+ if (ret)
+ return ret;
+ }
+ curBuf = validateListNext(&list->userBuffers, curBuf);
+ }
+ return 0;
+}
+
+int wsbmBOUnvalidateUserList(struct _WsbmBufferList *list)
+{
+ void *curBuf;
+ struct _WsbmBufStorage *storage;
+ struct _ValidateNode *node;
+
+ curBuf = validateListIterator(&list->userBuffers);
+
+ /*
+ * User-space validation callbacks.
+ */
+
+ while (curBuf) {
+ node = validateListNode(curBuf);
+ storage = (struct _WsbmBufStorage *)node->buf;
+ if (storage->pool->unvalidate) {
+ storage->pool->unvalidate(storage);
+ }
+ wsbmAtomicDec(&storage->onList);
+ wsbmBufStorageUnref(&storage);
+ curBuf = validateListNext(&list->userBuffers, curBuf);
+ }
+ return wsbmBOResetList(list);
+}
+
+struct _ValidateList *
+wsbmGetKernelValidateList(struct _WsbmBufferList *list)
+{
+ return (list->hasKernelBuffers) ? &list->kernelBuffers : NULL;
+}
+
+struct _ValidateList *
+wsbmGetUserValidateList(struct _WsbmBufferList *list)
+{
+ return &list->userBuffers;
+}
+
+struct _ValidateNode *
+validateListNode(void *iterator)
+{
+ struct _WsbmListHead *l = (struct _WsbmListHead *)iterator;
+
+ return WSBMLISTENTRY(l, struct _ValidateNode, head);
+}
+
+void *
+validateListIterator(struct _ValidateList *list)
+{
+ void *ret = list->list.next;
+
+ if (ret == &list->list)
+ return NULL;
+ return ret;
+}
+
+void *
+validateListNext(struct _ValidateList *list, void *iterator)
+{
+ void *ret;
+
+ struct _WsbmListHead *l = (struct _WsbmListHead *)iterator;
+
+ ret = l->next;
+ if (ret == &list->list)
+ return NULL;
+ return ret;
+}
diff --git a/src/wsbm_cs.h b/src/wsbm_cs.h
new file mode 100644
index 0000000..511edce
--- /dev/null
+++ b/src/wsbm_cs.h
@@ -0,0 +1,69 @@
+/**************************************************************************
+ *
+ * Copyright 2006-2009 Vmware, Inc., Palo Alto, CA., USA
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+/*
+ * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
+ */
+
+#ifndef _WSBM_CS_H_
+#define _WSBM_CS_H_
+#include "wsbm_driver.h"
+
+struct _WsbmFenceObject;
+struct _WsbmBufferObject;
+struct _ValidateList;
+
+#define WSBM_PLACEMENT_MASK ((uint64_t) 0xFFFFFFFFULL)
+
+extern struct _WsbmBufferList *wsbmBOCreateList(int target,
+ int hasKernelBuffers,
+ const struct _WsbmDriver *kernelDriver,
+ const struct _WsbmDriver *userDriver);
+
+extern int wsbmBOResetList(struct _WsbmBufferList *list);
+extern int wsbmBOAddListItem(struct _WsbmBufferList *list,
+ struct _WsbmBufferObject *buf,
+ uint64_t flags, uint64_t mask, int *itemLoc,
+ struct _ValidateNode **node);
+
+extern void wsbmBOFreeList(struct _WsbmBufferList *list);
+extern int wsbmBOFenceUserList(struct _WsbmBufferList *list,
+ struct _WsbmFenceObject *fence);
+
+extern int wsbmBOUnrefUserList(struct _WsbmBufferList *list);
+extern int wsbmBOValidateUserList(struct _WsbmBufferList *list);
+extern int wsbmBOUnvalidateUserList(struct _WsbmBufferList *list);
+
+extern struct _ValidateList *wsbmGetKernelValidateList(struct _WsbmBufferList
+ *list);
+extern struct _ValidateList *wsbmGetUserValidateList(struct _WsbmBufferList
+ *list);
+
+extern struct _ValidateNode *validateListNode(void *iterator);
+extern void *validateListIterator(struct _ValidateList *list);
+extern void *validateListNext(struct _ValidateList *list, void *iterator);
+
+#endif
diff --git a/src/wsbm_driver.h b/src/wsbm_driver.h
index d7da44e..b78549b 100644
--- a/src/wsbm_driver.h
+++ b/src/wsbm_driver.h
@@ -87,9 +87,9 @@ extern struct _WsbmThreadFuncs *wsbmCurThreadFunc;
#define WSBM_COND_BROADCAST(_cond) \
(_cond)->func->condBroadcast(_cond);
-struct _WsbmVNodeFuncs
+struct _WsbmDriver
{
- struct _ValidateNode *(*alloc) (struct _WsbmVNodeFuncs *, int);
+ struct _ValidateNode *(*alloc) (const struct _WsbmDriver *);
void (*free) (struct _ValidateNode *);
void (*clear) (struct _ValidateNode *);
};
@@ -109,7 +109,7 @@ struct _ValidateNode
uint64_t set_flags;
uint64_t clr_flags;
void *buf;
- struct _WsbmVNodeFuncs *func;
+ const struct _WsbmDriver *driver;
};
static inline struct _WsbmVNodeFuncs *
diff --git a/src/wsbm_fencemgr.c b/src/wsbm_fencemgr.c
index 0e7a302..6765382 100644
--- a/src/wsbm_fencemgr.c
+++ b/src/wsbm_fencemgr.c
@@ -37,8 +37,6 @@
#include "wsbm_fencemgr.h"
#include "wsbm_pool.h"
#include "wsbm_manager.h"
-#include <xf86drm.h>
-#include <ttm/ttm_fence_user.h>
#include <string.h>
#include <unistd.h>
@@ -142,6 +140,7 @@ wsbmFenceMgrCreate(const struct _WsbmFenceMgrCreateInfo *info)
}
}
wsbmAtomicSet(&tmp->count, 0);
+ tmp->private = info->private;
return tmp;
@@ -349,98 +348,6 @@ wsbmFenceCreate(struct _WsbmFenceMgr *mgr, uint32_t fence_class,
private_size);
}
-struct _WsbmTTMFenceMgrPriv
-{
- int fd;
- unsigned int devOffset;
-};
-
-static int
-tSignaled(struct _WsbmFenceMgr *mgr, void *private, uint32_t flush_type,
- uint32_t * signaled_type)
-{
- struct _WsbmTTMFenceMgrPriv *priv =
- (struct _WsbmTTMFenceMgrPriv *)mgr->private;
- union ttm_fence_signaled_arg arg;
- int ret;
-
- arg.req.handle = (unsigned long)private;
- arg.req.fence_type = flush_type;
- arg.req.flush = 1;
- *signaled_type = 0;
-
- ret = drmCommandWriteRead(priv->fd, priv->devOffset + TTM_FENCE_SIGNALED,
- &arg, sizeof(arg));
- if (ret)
- return ret;
-
- *signaled_type = arg.rep.signaled_types;
- return 0;
-}
-
-static int
-tFinish(struct _WsbmFenceMgr *mgr, void *private, uint32_t fence_type,
- int lazy_hint)
-{
- struct _WsbmTTMFenceMgrPriv *priv =
- (struct _WsbmTTMFenceMgrPriv *)mgr->private;
- union ttm_fence_finish_arg arg =
- {.req = {.handle = (unsigned long)private,
- .fence_type = fence_type,
- .mode = (lazy_hint) ? TTM_FENCE_FINISH_MODE_LAZY : 0}
- };
- int ret;
-
- do {
- ret = drmCommandWriteRead(priv->fd, priv->devOffset + TTM_FENCE_FINISH,
- &arg, sizeof(arg));
- } while (ret == -EAGAIN || ret == -ERESTART);
-
- return ret;
-}
-
-static int
-tUnref(struct _WsbmFenceMgr *mgr, void **private)
-{
- struct _WsbmTTMFenceMgrPriv *priv =
- (struct _WsbmTTMFenceMgrPriv *)mgr->private;
- struct ttm_fence_unref_arg arg = {.handle = (unsigned long)*private };
-
- *private = NULL;
-
- return drmCommandWrite(priv->fd, priv->devOffset + TTM_FENCE_UNREF,
- &arg, sizeof(arg));
-}
-
-struct _WsbmFenceMgr *
-wsbmFenceMgrTTMInit(int fd, unsigned int numClass, unsigned int devOffset)
-{
- struct _WsbmFenceMgrCreateInfo info;
- struct _WsbmFenceMgr *mgr;
- struct _WsbmTTMFenceMgrPriv *priv = malloc(sizeof(*priv));
-
- if (!priv)
- return NULL;
-
- priv->fd = fd;
- priv->devOffset = devOffset;
-
- info.flags = WSBM_FENCE_CLASS_ORDERED;
- info.num_classes = numClass;
- info.signaled = tSignaled;
- info.finish = tFinish;
- info.unreference = tUnref;
-
- mgr = wsbmFenceMgrCreate(&info);
- if (mgr == NULL) {
- free(priv);
- return NULL;
- }
-
- mgr->private = (void *)priv;
- return mgr;
-}
-
void
wsbmFenceCmdLock(struct _WsbmFenceMgr *mgr, uint32_t fence_class)
{
@@ -454,21 +361,24 @@ wsbmFenceCmdUnlock(struct _WsbmFenceMgr *mgr, uint32_t fence_class)
}
void
-wsbmFenceMgrTTMTakedown(struct _WsbmFenceMgr *mgr)
+wsbmFenceMgrTakedown(struct _WsbmFenceMgr *mgr)
{
int i;
if (!mgr)
return;
- if (mgr->private)
- free(mgr->private);
-
for (i = 0; i < mgr->info.num_classes; ++i) {
WSBM_MUTEX_FREE(&mgr->classes[i].mutex);
WSBM_MUTEX_FREE(&mgr->classes[i].cmd_mutex);
}
+ free(mgr->classes);
free(mgr);
+}
- return;
+extern void *wsbmFenceMgrPrivate(struct _WsbmFenceMgr *mgr)
+{
+ if (!mgr)
+ return NULL;
+ return mgr->private;
}
diff --git a/src/wsbm_fencemgr.h b/src/wsbm_fencemgr.h
index cb10ae2..f6dff8f 100644
--- a/src/wsbm_fencemgr.h
+++ b/src/wsbm_fencemgr.h
@@ -136,6 +136,7 @@ extern uint32_t wsbmFenceType(struct _WsbmFenceObject *fence);
struct _WsbmFenceMgrCreateInfo
{
+ void *private;
uint32_t flags;
uint32_t num_classes;
int (*signaled) (struct _WsbmFenceMgr * mgr, void *private,
@@ -151,12 +152,6 @@ extern struct _WsbmFenceMgr *wsbmFenceMgrCreate(const struct
extern void wsbmFenceCmdLock(struct _WsbmFenceMgr *mgr, uint32_t fence_class);
extern void wsbmFenceCmdUnlock(struct _WsbmFenceMgr *mgr,
uint32_t fence_class);
-/*
- * Builtin drivers.
- */
-
-extern struct _WsbmFenceMgr *wsbmFenceMgrTTMInit(int fd,
- unsigned int numClass,
- unsigned int devOffset);
-extern void wsbmFenceMgrTTMTakedown(struct _WsbmFenceMgr *mgr);
+extern void wsbmFenceMgrTakedown(struct _WsbmFenceMgr *mgr);
+extern void *wsbmFenceMgrPrivate(struct _WsbmFenceMgr *mgr);
#endif
diff --git a/src/wsbm_mallocpool.c b/src/wsbm_mallocpool.c
index b5252d6..d36aee9 100644
--- a/src/wsbm_mallocpool.c
+++ b/src/wsbm_mallocpool.c
@@ -54,7 +54,8 @@ mallocBuf(struct _WsbmBufStorage *buf)
static struct _WsbmBufStorage *
pool_create(struct _WsbmBufferPool *pool,
- unsigned long size, uint32_t placement, unsigned alignment)
+ unsigned long size, uint32_t placement, unsigned alignment,
+ int share, int pin)
{
struct _WsbmMallocBuffer *mBuf = malloc(size + sizeof(*mBuf) + 16);
@@ -64,8 +65,6 @@ pool_create(struct _WsbmBufferPool *pool,
wsbmBufStorageInit(&mBuf->buf, pool);
mBuf->size = size;
mBuf->mem = (void *)((unsigned long)mBuf + sizeof(*mBuf));
- if ((placement & WSBM_PL_MASK_MEM) != WSBM_PL_FLAG_SYSTEM)
- abort();
return &mBuf->buf;
}
@@ -97,7 +96,7 @@ pool_unmap(struct _WsbmBufStorage *buf)
}
static int
-pool_syncforcpu(struct _WsbmBufStorage *buf, unsigned mode)
+pool_syncforcpu(struct _WsbmBufStorage *buf, unsigned mode, int noBlock)
{
return 0;
}
@@ -130,7 +129,7 @@ pool_poolOffset(struct _WsbmBufStorage *buf)
static uint32_t
pool_placement(struct _WsbmBufStorage *buf)
{
- return WSBM_PL_FLAG_SYSTEM | WSBM_PL_FLAG_CACHED;
+ return 0;
}
static unsigned long
diff --git a/src/wsbm_manager.c b/src/wsbm_manager.c
index e40571d..83a801f 100644
--- a/src/wsbm_manager.c
+++ b/src/wsbm_manager.c
@@ -53,305 +53,67 @@
#define WSBM_BUFFER_SIMPLE 1
#define WSBM_BUFFER_REF 2
-struct _ValidateList
-{
- unsigned numTarget;
- unsigned numCurrent;
- unsigned numOnList;
- unsigned hashSize;
- uint32_t hashMask;
- int driverData;
- struct _WsbmListHead list;
- struct _WsbmListHead free;
- struct _WsbmListHead *hashTable;
-};
-
struct _WsbmBufferObject
{
/* Left to the client to protect this data for now. */
struct _WsbmAtomic refCount;
struct _WsbmBufStorage *storage;
-
- uint32_t placement;
- unsigned alignment;
- unsigned bufferType;
+ struct _WsbmAttr attr;
struct _WsbmBufferPool *pool;
+ unsigned bufferType;
+ int hasAttr;
};
-struct _WsbmBufferList
-{
- int hasKernelBuffers;
-
- struct _ValidateList kernelBuffers; /* List of kernel buffers needing validation */
- struct _ValidateList userBuffers; /* List of user-space buffers needing validation */
-};
-
-static struct _WsbmMutex bmMutex;
-static struct _WsbmCond bmCond;
static int initialized = 0;
-static void *commonData = NULL;
-
-static int kernelReaders = 0;
-static int kernelLocked = 0;
+static struct _WsbmCommon *commonData = NULL;
int
-wsbmInit(struct _WsbmThreadFuncs *tf, struct _WsbmVNodeFuncs *vf)
+wsbmInit(struct _WsbmThreadFuncs *tf)
{
- int ret;
-
wsbmCurThreadFunc = tf;
- wsbmCurVNodeFunc = vf;
-
- ret = WSBM_MUTEX_INIT(&bmMutex);
- if (ret)
- return -ENOMEM;
- ret = WSBM_COND_INIT(&bmCond);
- if (ret) {
- WSBM_MUTEX_FREE(&bmMutex);
- return -ENOMEM;
- }
-
initialized = 1;
return 0;
}
void
-wsbmCommonDataSet(void *d)
+wsbmCommonDataSet(struct _WsbmCommon *common)
{
- commonData = d;
+ commonData = common;
}
-void *
+struct _WsbmCommon *
wsbmCommonDataGet(void)
{
+ if (commonData == NULL)
+ return NULL;
+ ++commonData->refCount;
return commonData;
}
-int
-wsbmIsInitialized(void)
-{
- return initialized;
-}
-
void
-wsbmTakedown(void)
+wsbmCommonDataPut(void)
{
- initialized = 0;
- commonData = NULL;
- WSBM_COND_FREE(&bmCond);
- WSBM_MUTEX_FREE(&bmMutex);
-}
+ if (commonData == NULL)
+ abort();
-static struct _ValidateNode *
-validateListAddNode(struct _ValidateList *list, void *item,
- uint32_t hash, uint64_t flags, uint64_t mask)
-{
- struct _ValidateNode *node;
- struct _WsbmListHead *l;
- struct _WsbmListHead *hashHead;
-
- l = list->free.next;
- if (l == &list->free) {
- node = wsbmVNodeFuncs()->alloc(wsbmVNodeFuncs(), 0);
- if (!node) {
- return NULL;
- }
- list->numCurrent++;
- } else {
- WSBMLISTDEL(l);
- node = WSBMLISTENTRY(l, struct _ValidateNode, head);
+ if (--commonData->refCount == 0) {
+ commonData->destroy(commonData);
+ commonData = NULL;
}
- node->buf = item;
- node->set_flags = flags & mask;
- node->clr_flags = (~flags) & mask;
- node->listItem = list->numOnList;
- WSBMLISTADDTAIL(&node->head, &list->list);
- list->numOnList++;
- hashHead = list->hashTable + hash;
- WSBMLISTADDTAIL(&node->hashHead, hashHead);
-
- return node;
}
-static uint32_t
-wsbmHashFunc(uint8_t * key, uint32_t len, uint32_t mask)
-{
- uint32_t hash, i;
-
- for (hash = 0, i = 0; i < len; ++i) {
- hash += *key++;
- hash += (hash << 10);
- hash ^= (hash >> 6);
- }
-
- hash += (hash << 3);
- hash ^= (hash >> 11);
- hash += (hash << 15);
-
- return hash & mask;
-}
-
-static void
-validateFreeList(struct _ValidateList *list)
-{
- struct _ValidateNode *node;
- struct _WsbmListHead *l;
-
- l = list->list.next;
- while (l != &list->list) {
- WSBMLISTDEL(l);
- node = WSBMLISTENTRY(l, struct _ValidateNode, head);
-
- WSBMLISTDEL(&node->hashHead);
- node->func->free(node);
- l = list->list.next;
- list->numCurrent--;
- list->numOnList--;
- }
-
- l = list->free.next;
- while (l != &list->free) {
- WSBMLISTDEL(l);
- node = WSBMLISTENTRY(l, struct _ValidateNode, head);
-
- node->func->free(node);
- l = list->free.next;
- list->numCurrent--;
- }
- free(list->hashTable);
-}
-
-static int
-validateListAdjustNodes(struct _ValidateList *list)
-{
- struct _ValidateNode *node;
- struct _WsbmListHead *l;
- int ret = 0;
-
- while (list->numCurrent < list->numTarget) {
- node = wsbmVNodeFuncs()->alloc(wsbmVNodeFuncs(), list->driverData);
- if (!node) {
- ret = -ENOMEM;
- break;
- }
- list->numCurrent++;
- WSBMLISTADD(&node->head, &list->free);
- }
-
- while (list->numCurrent > list->numTarget) {
- l = list->free.next;
- if (l == &list->free)
- break;
- WSBMLISTDEL(l);
- node = WSBMLISTENTRY(l, struct _ValidateNode, head);
-
- node->func->free(node);
- list->numCurrent--;
- }
- return ret;
-}
-
-static inline int
-wsbmPot(unsigned int val)
-{
- unsigned int shift = 0;
- while(val > (1 << shift))
- shift++;
-
- return shift;
-}
-
-
-
-static int
-validateCreateList(int numTarget, struct _ValidateList *list, int driverData)
-{
- int i;
- unsigned int shift = wsbmPot(numTarget);
- int ret;
-
- list->hashSize = (1 << shift);
- list->hashMask = list->hashSize - 1;
-
- list->hashTable = malloc(list->hashSize * sizeof(*list->hashTable));
- if (!list->hashTable)
- return -ENOMEM;
-
- for (i = 0; i < list->hashSize; ++i)
- WSBMINITLISTHEAD(&list->hashTable[i]);
-
- WSBMINITLISTHEAD(&list->list);
- WSBMINITLISTHEAD(&list->free);
- list->numTarget = numTarget;
- list->numCurrent = 0;
- list->numOnList = 0;
- list->driverData = driverData;
- ret = validateListAdjustNodes(list);
- if (ret != 0)
- free(list->hashTable);
-
- return ret;
-}
-
-static int
-validateResetList(struct _ValidateList *list)
-{
- struct _WsbmListHead *l;
- struct _ValidateNode *node;
- int ret;
-
- ret = validateListAdjustNodes(list);
- if (ret)
- return ret;
-
- l = list->list.next;
- while (l != &list->list) {
- WSBMLISTDEL(l);
- node = WSBMLISTENTRY(l, struct _ValidateNode, head);
-
- WSBMLISTDEL(&node->hashHead);
- WSBMLISTADD(l, &list->free);
- list->numOnList--;
- l = list->list.next;
- }
- return validateListAdjustNodes(list);
-}
-
-void
-wsbmWriteLockKernelBO(void)
-{
- WSBM_MUTEX_LOCK(&bmMutex);
- while (kernelReaders != 0)
- WSBM_COND_WAIT(&bmCond, &bmMutex);
- kernelLocked = 1;
-}
-
-void
-wsbmWriteUnlockKernelBO(void)
-{
- kernelLocked = 0;
- WSBM_MUTEX_UNLOCK(&bmMutex);
-}
-
-void
-wsbmReadLockKernelBO(void)
+int
+wsbmIsInitialized(void)
{
- WSBM_MUTEX_LOCK(&bmMutex);
- if (kernelReaders++ == 0)
- kernelLocked = 1;
- WSBM_MUTEX_UNLOCK(&bmMutex);
+ return initialized;
}
void
-wsbmReadUnlockKernelBO(void)
+wsbmTakedown(void)
{
- WSBM_MUTEX_LOCK(&bmMutex);
- if (--kernelReaders == 0) {
- kernelLocked = 0;
- WSBM_COND_BROADCAST(&bmCond);
- }
- WSBM_MUTEX_UNLOCK(&bmMutex);
+ initialized = 0;
+ commonData = NULL;
}
void
@@ -387,11 +149,12 @@ wsbmBOUnmap(struct _WsbmBufferObject *buf)
}
int
-wsbmBOSyncForCpu(struct _WsbmBufferObject *buf, unsigned mode)
+wsbmBOSyncForCpu(struct _WsbmBufferObject *buf, unsigned mode,
+ int noBlock)
{
struct _WsbmBufStorage *storage = buf->storage;
- return storage->pool->syncforcpu(storage, mode);
+ return storage->pool->syncforcpu(storage, mode, noBlock);
}
void
@@ -440,18 +203,23 @@ wsbmBOReference(struct _WsbmBufferObject *buf)
}
int
-wsbmBOSetStatus(struct _WsbmBufferObject *buf,
- uint32_t setFlags, uint32_t clrFlags)
+wsbmBOSetAttr(struct _WsbmBufferObject *buf,
+ const struct _WsbmAttr *attr)
{
struct _WsbmBufStorage *storage = buf->storage;
if (!storage)
return 0;
- if (storage->pool->setStatus == NULL)
+ if (storage->pool->setAttr == NULL)
return -EINVAL;
- return storage->pool->setStatus(storage, setFlags, clrFlags);
+ buf->attr = *attr;
+ return storage->pool->setAttr(storage,
+ attr->setPlacement,
+ attr->alignment,
+ attr->share,
+ attr->pin);
}
void
@@ -477,18 +245,30 @@ wsbmBOUnreference(struct _WsbmBufferObject **p_buf)
}
}
+static inline int
+wsbmAttrDiff(const struct _WsbmAttr *attr1,
+ const struct _WsbmAttr *attr2)
+{
+ return (attr1->setPlacement != attr2->setPlacement) ||
+ (attr1->clrPlacement != attr2->clrPlacement) ||
+ (attr1->alignment != attr2->alignment) ||
+ (attr1->share != attr2->share) ||
+ (attr1->pin != attr2->pin);
+}
+
int
wsbmBOData(struct _WsbmBufferObject *buf,
unsigned size, const void *data,
- struct _WsbmBufferPool *newPool, uint32_t placement)
+ struct _WsbmBufferPool *newPool,
+ const struct _WsbmAttr *attr)
{
void *virtual = NULL;
int newBuffer;
int retval = 0;
struct _WsbmBufStorage *storage;
int synced = 0;
- uint32_t placement_diff;
struct _WsbmBufferPool *curPool;
+ const struct _WsbmAttr *tmpAttr = attr;
if (buf->bufferType == WSBM_BUFFER_SIMPLE)
return -EINVAL;
@@ -506,8 +286,12 @@ wsbmBOData(struct _WsbmBufferObject *buf,
storage->pool->size(storage) >
size + WSBM_BODATA_SIZE_ACCEPT);
- if (!placement)
- placement = buf->placement;
+ if (!tmpAttr) {
+ if (buf->hasAttr)
+ tmpAttr = &buf->attr;
+ else
+ return -EINVAL;
+ }
if (newBuffer) {
if (buf->bufferType == WSBM_BUFFER_REF)
@@ -517,23 +301,27 @@ wsbmBOData(struct _WsbmBufferObject *buf,
if (size == 0) {
buf->pool = newPool;
- buf->placement = placement;
+ buf->attr = *tmpAttr;
+ buf->hasAttr = 1;
retval = 0;
goto out;
}
buf->storage =
- newPool->create(newPool, size, placement, buf->alignment);
+ newPool->create(newPool, size, tmpAttr->setPlacement,
+ tmpAttr->alignment,
+ tmpAttr->share,
+ tmpAttr->pin);
if (!buf->storage) {
retval = -ENOMEM;
goto out;
}
- buf->placement = placement;
+ buf->attr = *tmpAttr;
+ buf->hasAttr = 1;
buf->pool = newPool;
} else if (wsbmAtomicRead(&storage->onList) ||
- 0 != storage->pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE |
- WSBM_SYNCCPU_DONT_BLOCK)) {
+ 0 != storage->pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE, 1)) {
/*
* Buffer is busy. need to create a new one.
*/
@@ -543,14 +331,17 @@ wsbmBOData(struct _WsbmBufferObject *buf,
curPool = storage->pool;
tmp_storage =
- curPool->create(curPool, size, placement, buf->alignment);
+ curPool->create(curPool, size, tmpAttr->setPlacement,
+ tmpAttr->alignment,
+ tmpAttr->share,
+ tmpAttr->pin);
if (tmp_storage) {
wsbmBufStorageUnref(&buf->storage);
buf->storage = tmp_storage;
- buf->placement = placement;
+ buf->attr = *tmpAttr;
} else {
- retval = curPool->syncforcpu(storage, WSBM_SYNCCPU_WRITE);
+ retval = curPool->syncforcpu(storage, WSBM_SYNCCPU_WRITE, 0);
if (retval)
goto out;
synced = 1;
@@ -558,8 +349,6 @@ wsbmBOData(struct _WsbmBufferObject *buf,
} else
synced = 1;
- placement_diff = placement ^ buf->placement;
-
/*
* We might need to change buffer placement.
*/
@@ -567,21 +356,24 @@ wsbmBOData(struct _WsbmBufferObject *buf,
storage = buf->storage;
curPool = storage->pool;
- if (placement_diff) {
- assert(curPool->setStatus != NULL);
- curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE);
- retval = curPool->setStatus(storage,
- placement_diff & placement,
- placement_diff & ~placement);
+ if (wsbmAttrDiff(&buf->attr, tmpAttr)) {
+ assert(curPool->setAttr != NULL);
+ if (synced) {
+ curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE);
+ synced = 0;
+ }
+ retval = curPool->setAttr(storage,
+ tmpAttr->setPlacement,
+ tmpAttr->alignment,
+ tmpAttr->share,
+ tmpAttr->pin);
if (retval)
goto out;
-
- buf->placement = placement;
-
+ buf->attr = *tmpAttr;
}
- if (!synced) {
- retval = curPool->syncforcpu(buf->storage, WSBM_SYNCCPU_WRITE);
+ if (!synced && data) {
+ retval = curPool->syncforcpu(buf->storage, WSBM_SYNCCPU_WRITE, 0);
if (retval)
goto out;
@@ -612,9 +404,13 @@ wsbmStorageClone(struct _WsbmBufferObject *buf)
{
struct _WsbmBufStorage *storage = buf->storage;
struct _WsbmBufferPool *pool = storage->pool;
+ struct _WsbmAttr *attr = &buf->attr;
- return pool->create(pool, pool->size(storage), buf->placement,
- buf->alignment);
+ return pool->create(pool, pool->size(storage),
+ attr->setPlacement,
+ attr->alignment,
+ attr->share,
+ attr->pin);
}
struct _WsbmBufferObject *
@@ -643,7 +439,7 @@ wsbmBOClone(struct _WsbmBufferObject *buf,
void *virtual;
void *nVirtual;
- ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ);
+ ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ, 0);
if (ret)
goto out_err1;
ret = pool->map(storage, WSBM_ACCESS_READ, &virtual);
@@ -687,7 +483,7 @@ wsbmBOSubData(struct _WsbmBufferObject *buf,
struct _WsbmBufStorage *storage = buf->storage;
struct _WsbmBufferPool *pool = storage->pool;
- ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE);
+ ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE, 0);
if (ret)
goto out;
@@ -720,7 +516,7 @@ wsbmBOSubData(struct _WsbmBufferObject *buf,
pool = storage->pool;
}
- ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE);
+ ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE, 0);
if (ret)
goto out;
}
@@ -750,7 +546,7 @@ wsbmBOGetSubData(struct _WsbmBufferObject *buf,
struct _WsbmBufStorage *storage = buf->storage;
struct _WsbmBufferPool *pool = storage->pool;
- ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ);
+ ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ, 0);
if (ret)
goto out;
ret = pool->map(storage, WSBM_ACCESS_READ, &virtual);
@@ -795,8 +591,8 @@ wsbmBOFreeSimple(void *ptr)
struct _WsbmBufferObject *
wsbmBOCreateSimple(struct _WsbmBufferPool *pool,
unsigned long size,
- uint32_t placement,
- unsigned alignment, size_t extra_size, size_t * offset)
+ const struct _WsbmAttr *attr,
+ size_t extra_size, size_t * offset)
{
struct _WsbmBufferObject *buf;
struct _WsbmBufStorage *storage;
@@ -811,7 +607,11 @@ wsbmBOCreateSimple(struct _WsbmBufferPool *pool,
if (!buf)
return NULL;
- storage = pool->create(pool, size, placement, alignment);
+ storage = pool->create(pool, size,
+ attr->setPlacement,
+ attr->alignment,
+ attr->share,
+ attr->pin);
if (!storage)
goto out_err0;
@@ -819,10 +619,10 @@ wsbmBOCreateSimple(struct _WsbmBufferPool *pool,
storage->destroyArg = buf;
buf->storage = storage;
- buf->alignment = alignment;
buf->pool = pool;
- buf->placement = placement;
buf->bufferType = WSBM_BUFFER_SIMPLE;
+ buf->attr = *attr;
+ buf->hasAttr = 1;
return buf;
@@ -835,22 +635,21 @@ int
wsbmGenBuffers(struct _WsbmBufferPool *pool,
unsigned n,
struct _WsbmBufferObject *buffers[],
- unsigned alignment, uint32_t placement)
+ const struct _WsbmAttr *attr)
{
struct _WsbmBufferObject *buf;
int i;
- placement = (placement) ? placement :
- WSBM_PL_FLAG_SYSTEM | WSBM_PL_FLAG_CACHED;
-
for (i = 0; i < n; ++i) {
buf = (struct _WsbmBufferObject *)calloc(1, sizeof(*buf));
if (!buf)
return -ENOMEM;
wsbmAtomicSet(&buf->refCount, 1);
- buf->placement = placement;
- buf->alignment = alignment;
+ if (attr) {
+ buf->hasAttr = 1;
+ memcpy(&buf->attr, attr, sizeof(*attr));
+ }
buf->pool = pool;
buf->bufferType = WSBM_BUFFER_COMPLEX;
buffers[i] = buf;
@@ -868,155 +667,6 @@ wsbmDeleteBuffers(unsigned n, struct _WsbmBufferObject *buffers[])
}
}
-/*
- * Note that lists are per-context and don't need mutex protection.
- */
-
-struct _WsbmBufferList *
-wsbmBOCreateList(int target, int hasKernelBuffers)
-{
- struct _WsbmBufferList *list = calloc(sizeof(*list), 1);
- int ret;
-
- list->hasKernelBuffers = hasKernelBuffers;
- if (hasKernelBuffers) {
- ret = validateCreateList(target, &list->kernelBuffers, 0);
- if (ret)
- return NULL;
- }
-
- ret = validateCreateList(target, &list->userBuffers, 1);
- if (ret) {
- validateFreeList(&list->kernelBuffers);
- return NULL;
- }
-
- return list;
-}
-
-int
-wsbmBOResetList(struct _WsbmBufferList *list)
-{
- int ret;
-
- if (list->hasKernelBuffers) {
- ret = validateResetList(&list->kernelBuffers);
- if (ret)
- return ret;
- }
- ret = validateResetList(&list->userBuffers);
- return ret;
-}
-
-void
-wsbmBOFreeList(struct _WsbmBufferList *list)
-{
- if (list->hasKernelBuffers)
- validateFreeList(&list->kernelBuffers);
- validateFreeList(&list->userBuffers);
- free(list);
-}
-
-static int
-wsbmAddValidateItem(struct _ValidateList *list, void *buf, uint64_t flags,
- uint64_t mask, int *itemLoc,
- struct _ValidateNode **pnode, int *newItem)
-{
- struct _ValidateNode *node, *cur;
- struct _WsbmListHead *l;
- struct _WsbmListHead *hashHead;
- uint32_t hash;
- uint32_t count = 0;
- uint32_t key = (unsigned long) buf;
-
- cur = NULL;
- hash = wsbmHashFunc((uint8_t *) &key, 4, list->hashMask);
- hashHead = list->hashTable + hash;
- *newItem = 0;
-
- for (l = hashHead->next; l != hashHead; l = l->next) {
- count++;
- node = WSBMLISTENTRY(l, struct _ValidateNode, hashHead);
-
- if (node->buf == buf) {
- cur = node;
- break;
- }
- }
-
- if (!cur) {
- cur = validateListAddNode(list, buf, hash, flags, mask);
- if (!cur)
- return -ENOMEM;
- *newItem = 1;
- cur->func->clear(cur);
- } else {
- uint64_t set_flags = flags & mask;
- uint64_t clr_flags = (~flags) & mask;
-
- if (((cur->clr_flags | clr_flags) & WSBM_PL_MASK_MEM) ==
- WSBM_PL_MASK_MEM) {
- /*
- * No available memory type left. Bail.
- */
- return -EINVAL;
- }
-
- if ((cur->set_flags | set_flags) &
- (cur->clr_flags | clr_flags) & ~WSBM_PL_MASK_MEM) {
- /*
- * Conflicting flags. Bail.
- */
- return -EINVAL;
- }
-
- cur->set_flags &= ~(clr_flags & WSBM_PL_MASK_MEM);
- cur->set_flags |= (set_flags & ~WSBM_PL_MASK_MEM);
- cur->clr_flags |= clr_flags;
- }
- *itemLoc = cur->listItem;
- if (pnode)
- *pnode = cur;
- return 0;
-}
-
-int
-wsbmBOAddListItem(struct _WsbmBufferList *list,
- struct _WsbmBufferObject *buf,
- uint64_t flags, uint64_t mask, int *itemLoc,
- struct _ValidateNode **node)
-{
- int newItem;
- struct _WsbmBufStorage *storage = buf->storage;
- int ret;
- int dummy;
- struct _ValidateNode *dummyNode;
-
- if (list->hasKernelBuffers) {
- ret = wsbmAddValidateItem(&list->kernelBuffers,
- storage->pool->kernel(storage),
- flags, mask, itemLoc, node, &dummy);
- if (ret)
- goto out_unlock;
- } else {
- *node = NULL;
- *itemLoc = -1000;
- }
-
- ret = wsbmAddValidateItem(&list->userBuffers, storage,
- flags, mask, &dummy, &dummyNode, &newItem);
- if (ret)
- goto out_unlock;
-
- if (newItem) {
- wsbmAtomicInc(&storage->refCount);
- wsbmAtomicInc(&storage->onList);
- }
-
- out_unlock:
- return ret;
-}
-
void
wsbmBOFence(struct _WsbmBufferObject *buf, struct _WsbmFenceObject *fence)
{
@@ -1036,104 +686,6 @@ wsbmBOOnList(const struct _WsbmBufferObject *buf)
return wsbmAtomicRead(&buf->storage->onList);
}
-int
-wsbmBOUnrefUserList(struct _WsbmBufferList *list)
-{
- struct _WsbmBufStorage *storage;
- void *curBuf;
-
- curBuf = validateListIterator(&list->userBuffers);
-
- while (curBuf) {
- storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf);
- wsbmAtomicDec(&storage->onList);
- wsbmBufStorageUnref(&storage);
- curBuf = validateListNext(&list->userBuffers, curBuf);
- }
-
- return wsbmBOResetList(list);
-}
-
-
-int
-wsbmBOFenceUserList(struct _WsbmBufferList *list,
- struct _WsbmFenceObject *fence)
-{
- struct _WsbmBufStorage *storage;
- void *curBuf;
-
- curBuf = validateListIterator(&list->userBuffers);
-
- /*
- * User-space fencing callbacks.
- */
-
- while (curBuf) {
- storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf);
-
- storage->pool->fence(storage, fence);
- wsbmAtomicDec(&storage->onList);
- wsbmBufStorageUnref(&storage);
- curBuf = validateListNext(&list->userBuffers, curBuf);
- }
-
- return wsbmBOResetList(list);
-}
-
-int
-wsbmBOValidateUserList(struct _WsbmBufferList *list)
-{
- void *curBuf;
- struct _WsbmBufStorage *storage;
- struct _ValidateNode *node;
- int ret;
-
- curBuf = validateListIterator(&list->userBuffers);
-
- /*
- * User-space validation callbacks.
- */
-
- while (curBuf) {
- node = validateListNode(curBuf);
- storage = (struct _WsbmBufStorage *)node->buf;
- if (storage->pool->validate) {
- ret = storage->pool->validate(storage, node->set_flags,
- node->clr_flags);
- if (ret)
- return ret;
- }
- curBuf = validateListNext(&list->userBuffers, curBuf);
- }
- return 0;
-}
-
-int
-wsbmBOUnvalidateUserList(struct _WsbmBufferList *list)
-{
- void *curBuf;
- struct _WsbmBufStorage *storage;
- struct _ValidateNode *node;
-
- curBuf = validateListIterator(&list->userBuffers);
-
- /*
- * User-space validation callbacks.
- */
-
- while (curBuf) {
- node = validateListNode(curBuf);
- storage = (struct _WsbmBufStorage *)node->buf;
- if (storage->pool->unvalidate) {
- storage->pool->unvalidate(storage);
- }
- wsbmAtomicDec(&storage->onList);
- wsbmBufStorageUnref(&storage);
- curBuf = validateListNext(&list->userBuffers, curBuf);
- }
- return wsbmBOResetList(list);
-}
-
void
wsbmPoolTakeDown(struct _WsbmBufferPool *pool)
{
@@ -1154,56 +706,7 @@ wsbmBOSize(struct _WsbmBufferObject *buf)
}
-struct _ValidateList *
-wsbmGetKernelValidateList(struct _WsbmBufferList *list)
-{
- return (list->hasKernelBuffers) ? &list->kernelBuffers : NULL;
-}
-
-struct _ValidateList *
-wsbmGetUserValidateList(struct _WsbmBufferList *list)
-{
- return &list->userBuffers;
-}
-
-struct _ValidateNode *
-validateListNode(void *iterator)
-{
- struct _WsbmListHead *l = (struct _WsbmListHead *)iterator;
-
- return WSBMLISTENTRY(l, struct _ValidateNode, head);
-}
-
-void *
-validateListIterator(struct _ValidateList *list)
-{
- void *ret = list->list.next;
-
- if (ret == &list->list)
- return NULL;
- return ret;
-}
-
-void *
-validateListNext(struct _ValidateList *list, void *iterator)
-{
- void *ret;
-
- struct _WsbmListHead *l = (struct _WsbmListHead *)iterator;
-
- ret = l->next;
- if (ret == &list->list)
- return NULL;
- return ret;
-}
-
-uint32_t
-wsbmKBufHandle(const struct _WsbmKernelBuf * kBuf)
-{
- return kBuf->handle;
-}
-
-extern void
+void
wsbmUpdateKBuf(struct _WsbmKernelBuf *kBuf,
uint64_t gpuOffset, uint32_t placement,
uint32_t fence_type_mask)
@@ -1213,10 +716,16 @@ wsbmUpdateKBuf(struct _WsbmKernelBuf *kBuf,
kBuf->fence_type_mask = fence_type_mask;
}
-extern struct _WsbmKernelBuf *
+struct _WsbmKernelBuf *
wsbmKBuf(const struct _WsbmBufferObject *buf)
{
struct _WsbmBufStorage *storage = buf->storage;
return storage->pool->kernel(storage);
}
+
+struct _WsbmBufStorage *
+wsbmBOStorage(struct _WsbmBufferObject *buf)
+{
+ return buf->storage;
+}
diff --git a/src/wsbm_manager.h b/src/wsbm_manager.h
index ee0636f..9f0f8af 100644
--- a/src/wsbm_manager.h
+++ b/src/wsbm_manager.h
@@ -44,40 +44,47 @@
struct _WsbmFenceObject;
struct _WsbmBufferObject;
struct _WsbmBufferPool;
-struct _WsbmBufferList;
-/*
- * These flags mimics the TTM closely, but since
- * this library is not dependant on TTM, we need to
- * replicate them here, and if there is a discrepancy,
- * that needs to be resolved in the buffer pool using
- * the TTM flags.
- */
-
-#define WSBM_PL_MASK_MEM 0x0000FFFF
-
-#define WSBM_PL_FLAG_SYSTEM (1 << 0)
-#define WSBM_PL_FLAG_TT (1 << 1)
-#define WSBM_PL_FLAG_VRAM (1 << 2)
-#define WSBM_PL_FLAG_PRIV0 (1 << 3)
-#define WSBM_PL_FLAG_SWAPPED (1 << 15)
-#define WSBM_PL_FLAG_CACHED (1 << 16)
-#define WSBM_PL_FLAG_UNCACHED (1 << 17)
-#define WSBM_PL_FLAG_WC (1 << 18)
-#define WSBM_PL_FLAG_SHARED (1 << 20)
-#define WSBM_PL_FLAG_NO_EVICT (1 << 21)
#define WSBM_ACCESS_READ (1 << 0)
#define WSBM_ACCESS_WRITE (1 << 1)
#define WSBM_SYNCCPU_READ WSBM_ACCESS_READ
#define WSBM_SYNCCPU_WRITE WSBM_ACCESS_WRITE
-#define WSBM_SYNCCPU_DONT_BLOCK (1 << 2)
-#define WSBM_SYNCCPU_TRY_CACHED (1 << 3)
+
+struct _WsbmAttr {
+ uint32_t setPlacement;
+ uint32_t clrPlacement;
+ unsigned int alignment;
+ int share;
+ int pin;
+};
+
+struct _WsbmCommon {
+ uint32_t refCount;
+ void (*destroy) (struct _WsbmCommon *);
+};
+
+static inline const struct _WsbmAttr *
+wsbmInitAttr(struct _WsbmAttr *attr,
+ uint32_t setPlacement,
+ uint32_t clrPlacement,
+ unsigned int alignment,
+ int share,
+ int pin)
+{
+ attr->setPlacement = setPlacement;
+ attr->clrPlacement = clrPlacement;
+ attr->alignment = alignment;
+ attr->share = share;
+ attr->pin = pin;
+ return attr;
+}
extern void *wsbmBOMap(struct _WsbmBufferObject *buf, unsigned mode);
extern void wsbmBOUnmap(struct _WsbmBufferObject *buf);
-extern int wsbmBOSyncForCpu(struct _WsbmBufferObject *buf, unsigned mode);
+extern int wsbmBOSyncForCpu(struct _WsbmBufferObject *buf, unsigned mode,
+ int noBlock);
extern void wsbmBOReleaseFromCpu(struct _WsbmBufferObject *buf,
unsigned mode);
@@ -91,9 +98,10 @@ extern void wsbmBOUnreference(struct _WsbmBufferObject **p_buf);
extern int wsbmBOData(struct _WsbmBufferObject *r_buf,
unsigned size, const void *data,
- struct _WsbmBufferPool *pool, uint32_t placement);
-extern int wsbmBOSetStatus(struct _WsbmBufferObject *buf,
- uint32_t setPlacement, uint32_t clrPlacement);
+ struct _WsbmBufferPool *pool,
+ const struct _WsbmAttr *attr);
+extern int wsbmBOSetAttr(struct _WsbmBufferObject *buf,
+ const struct _WsbmAttr *attr);
extern int wsbmBOSubData(struct _WsbmBufferObject *buf,
unsigned long offset, unsigned long size,
const void *data,
@@ -113,32 +121,16 @@ extern int wsbmBOGetSubData(struct _WsbmBufferObject *buf,
extern int wsbmGenBuffers(struct _WsbmBufferPool *pool,
unsigned n,
struct _WsbmBufferObject *buffers[],
- unsigned alignment, uint32_t placement);
+ const struct _WsbmAttr *attr);
struct _WsbmBufferObject *wsbmBOCreateSimple(struct _WsbmBufferPool *pool,
unsigned long size,
- uint32_t placement,
- unsigned alignment,
+ const struct _WsbmAttr *attr,
size_t extra_size,
size_t * offset);
extern void wsbmDeleteBuffers(unsigned n,
struct _WsbmBufferObject *buffers[]);
-extern struct _WsbmBufferList *wsbmBOCreateList(int target,
- int hasKernelBuffers);
-extern int wsbmBOResetList(struct _WsbmBufferList *list);
-extern int wsbmBOAddListItem(struct _WsbmBufferList *list,
- struct _WsbmBufferObject *buf,
- uint64_t flags, uint64_t mask, int *itemLoc,
- struct _ValidateNode **node);
-
-extern void wsbmBOFreeList(struct _WsbmBufferList *list);
-extern int wsbmBOFenceUserList(struct _WsbmBufferList *list,
- struct _WsbmFenceObject *fence);
-
-extern int wsbmBOUnrefUserList(struct _WsbmBufferList *list);
-extern int wsbmBOValidateUserList(struct _WsbmBufferList *list);
-extern int wsbmBOUnvalidateUserList(struct _WsbmBufferList *list);
extern void wsbmBOFence(struct _WsbmBufferObject *buf,
struct _WsbmFenceObject *fence);
@@ -152,26 +144,12 @@ extern int wsbmBOOnList(const struct _WsbmBufferObject *buf);
extern void wsbmPoolTakeDown(struct _WsbmBufferPool *pool);
-extern void wsbmReadLockKernelBO(void);
-extern void wsbmReadUnlockKernelBO(void);
-extern void wsbmWriteLockKernelBO(void);
-extern void wsbmWriteUnlockKernelBO(void);
-
-extern int wsbmInit(struct _WsbmThreadFuncs *tf, struct _WsbmVNodeFuncs *vf);
+extern int wsbmInit(struct _WsbmThreadFuncs *tf);
extern void wsbmTakedown(void);
extern int wsbmIsInitialized(void);
-extern void wsbmCommonDataSet(void *d);
-extern void *wsbmCommonDataGet(void);
-
-extern struct _ValidateList *wsbmGetKernelValidateList(struct _WsbmBufferList
- *list);
-extern struct _ValidateList *wsbmGetUserValidateList(struct _WsbmBufferList
- *list);
-
-extern struct _ValidateNode *validateListNode(void *iterator);
-extern void *validateListIterator(struct _ValidateList *list);
-extern void *validateListNext(struct _ValidateList *list, void *iterator);
-
+extern void wsbmCommonDataSet(struct _WsbmCommon *common);
+extern struct _WsbmCommon *wsbmCommonDataGet(void);
+extern void wsbmCommonDataPut(void);
extern uint32_t wsbmKBufHandle(const struct _WsbmKernelBuf *);
extern void wsbmUpdateKBuf(struct _WsbmKernelBuf *,
uint64_t gpuOffset,
diff --git a/src/wsbm_pool.h b/src/wsbm_pool.h
index 1d07523..0563a99 100644
--- a/src/wsbm_pool.h
+++ b/src/wsbm_pool.h
@@ -50,6 +50,8 @@ struct _WsbmBufStorage
void (*destroyContainer) (void *);
};
+
+
struct _WsbmKernelBuf;
struct _WsbmBufferPool
@@ -57,18 +59,21 @@ struct _WsbmBufferPool
int fd;
int (*map) (struct _WsbmBufStorage * buf, unsigned mode, void **virtual);
void (*unmap) (struct _WsbmBufStorage * buf);
- int (*syncforcpu) (struct _WsbmBufStorage * buf, unsigned mode);
+ int (*syncforcpu) (struct _WsbmBufStorage * buf, unsigned mode,
+ int noBlock);
void (*releasefromcpu) (struct _WsbmBufStorage * buf, unsigned mode);
void (*destroy) (struct _WsbmBufStorage ** buf);
unsigned long (*offset) (struct _WsbmBufStorage * buf);
unsigned long (*poolOffset) (struct _WsbmBufStorage * buf);
- uint32_t(*placement) (struct _WsbmBufStorage * buf);
+ uint32_t(*placement) (struct _WsbmBufStorage * buf);
unsigned long (*size) (struct _WsbmBufStorage * buf);
struct _WsbmKernelBuf *(*kernel) (struct _WsbmBufStorage * buf);
struct _WsbmBufStorage *(*create) (struct _WsbmBufferPool * pool,
unsigned long size,
uint32_t placement,
- unsigned alignment);
+ unsigned alignment,
+ int share,
+ int pin);
struct _WsbmBufStorage *(*createByReference) (struct _WsbmBufferPool *
pool, uint32_t handle);
void (*fence) (struct _WsbmBufStorage * buf,
@@ -77,8 +82,11 @@ struct _WsbmBufferPool
int (*validate) (struct _WsbmBufStorage * buf, uint64_t set_flags,
uint64_t clr_flags);
int (*waitIdle) (struct _WsbmBufStorage * buf, int lazy);
- int (*setStatus) (struct _WsbmBufStorage * buf,
- uint32_t set_placement, uint32_t clr_placement);
+ int (*setAttr) (struct _WsbmBufStorage * buf,
+ uint32_t placement,
+ unsigned alignment,
+ int share,
+ int pin);
void (*takeDown) (struct _WsbmBufferPool * pool);
};
@@ -124,24 +132,19 @@ wsbmBufStorageUnref(struct _WsbmBufStorage **pStorage)
* Builtin pools.
*/
-/*
- * Kernel buffer objects. Size in multiples of page size. Page size aligned.
- */
-
-extern struct _WsbmBufferPool *wsbmTTMPoolInit(int fd,
- unsigned int devOffset);
extern struct _WsbmBufferPool *wsbmMallocPoolInit(void);
+struct _WsbmAttr;
struct _WsbmSlabCache;
-extern struct _WsbmBufferPool *wsbmSlabPoolInit(int fd, uint32_t devOffset,
- uint32_t placement,
- uint32_t validMask,
+extern struct _WsbmBufferPool *wsbmSlabPoolInit(int fd,
+ struct _WsbmBufferPool *basePool,
+ const struct _WsbmAttr *attr,
uint32_t smallestSize,
uint32_t numSizes,
uint32_t desiredNumBuffers,
uint32_t maxSlabSize,
- uint32_t pageAlignment,
struct _WsbmSlabCache *cache);
+
extern struct _WsbmSlabCache *wsbmSlabCacheInit(uint32_t checkIntervalMsec,
uint32_t slabTimeoutMsec);
extern void wsbmSlabCacheFinish(struct _WsbmSlabCache *cache);
@@ -152,6 +155,9 @@ extern struct _WsbmBufferPool *wsbmUserPoolInit(void *vramAddr,
void *agpAddr,
unsigned long agpStart,
unsigned long agpSize,
+ uint32_t system_flag,
+ uint32_t vram_flag,
+ uint32_t agp_flag,
uint32_t(*fenceTypes)
(uint64_t set_flags));
diff --git a/src/wsbm_slabpool.c b/src/wsbm_slabpool.c
index 99f8d22..018d70d 100644
--- a/src/wsbm_slabpool.c
+++ b/src/wsbm_slabpool.c
@@ -34,7 +34,6 @@
#include "config.h"
#endif
-#include <ttm/ttm_placement_user.h>
#include <stdint.h>
#include <sys/time.h>
#include <errno.h>
@@ -42,23 +41,12 @@
#include <assert.h>
#include <string.h>
#include <sys/mman.h>
-#include <xf86drm.h>
#include "wsbm_pool.h"
#include "wsbm_fencemgr.h"
#include "wsbm_priv.h"
#include "wsbm_manager.h"
#define WSBM_SLABPOOL_ALLOC_RETRIES 100
-#define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \
- do { \
- (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \
- } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \
-
-#define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \
- do { \
- (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \
- } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \
-
#ifdef DEBUG_FENCESIGNALED
static int createbuffer = 0;
@@ -69,7 +57,6 @@ struct _WsbmSlab;
struct _WsbmSlabBuffer
{
- struct _WsbmKernelBuf kBuf;
struct _WsbmBufStorage storage;
struct _WsbmCond event;
@@ -78,11 +65,11 @@ struct _WsbmSlabBuffer
*/
int isSlabBuffer;
+ struct _WsbmBufStorage *bStorage;
struct _WsbmSlab *parent;
uint32_t start;
void *virtual;
unsigned long requestedSize;
- uint64_t mapHandle;
/*
* Protected by struct _WsbmSlabSizeHeader::mutex
@@ -108,21 +95,20 @@ struct _WsbmSlabKernelBO
* Constant at creation
*/
- struct _WsbmKernelBuf kBuf;
+ struct _WsbmBufStorage *bStorage;
uint32_t pageAlignment;
void *virtual;
- unsigned long actualSize;
- uint64_t mapHandle;
/*
* Protected by struct _WsbmSlabCache::mutex
*/
struct _WsbmSlabPool *slabPool;
- uint32_t proposedPlacement;
+ struct _WsbmAttr attr;
struct _WsbmListHead timeoutHead;
struct _WsbmListHead head;
struct timeval timeFreed;
+ unsigned long actual_size;
};
struct _WsbmSlab
@@ -169,16 +155,15 @@ struct _WsbmSlabCache
struct _WsbmSlabPool
{
struct _WsbmBufferPool pool;
+ struct _WsbmBufferPool *basePool;
/*
* The data of this structure remains constant after
* initialization and thus needs no mutex protection.
*/
- unsigned int devOffset;
struct _WsbmSlabCache *cache;
- uint32_t proposedPlacement;
- uint32_t validMask;
+ struct _WsbmAttr attr;
uint32_t *bucketSizes;
uint32_t numBuckets;
uint32_t pageSize;
@@ -234,19 +219,10 @@ wsbmTimeAdd(struct timeval *arg, struct timeval *add)
static void
wsbmFreeKernelBO(struct _WsbmSlabKernelBO *kbo)
{
- struct ttm_pl_reference_req arg;
- struct _WsbmSlabPool *slabPool;
-
if (!kbo)
return;
- slabPool = kbo->slabPool;
- memset(&arg, 0, sizeof(arg));
- arg.handle = kbo->kBuf.handle;
- (void)munmap(kbo->virtual, kbo->actualSize);
- (void)drmCommandWrite(slabPool->pool.fd,
- slabPool->devOffset + TTM_PL_UNREF, &arg,
- sizeof(arg));
+ kbo->bStorage->pool->destroy(&kbo->bStorage);
free(kbo);
}
@@ -293,17 +269,24 @@ wsbmSetKernelBOFree(struct _WsbmSlabCache *cache,
wsbmTimeAdd(&timeFreed, &cache->slabTimeout);
kbo->timeFreed = timeFreed;
- if (kbo->kBuf.placement & TTM_PL_FLAG_CACHED)
- WSBMLISTADD(&kbo->head, &cache->cached);
- else
- WSBMLISTADD(&kbo->head, &cache->unCached);
-
+ WSBMLISTADD(&kbo->head, &cache->unCached);
WSBMLISTADDTAIL(&kbo->timeoutHead, &cache->timeoutList);
wsbmFreeTimeoutKBOsLocked(cache, &time);
WSBM_MUTEX_UNLOCK(&cache->mutex);
}
+static inline int
+wsbmAttrDiff(const struct _WsbmAttr *attr1,
+ const struct _WsbmAttr *attr2)
+{
+ return (attr1->setPlacement != attr2->setPlacement) ||
+ (attr1->clrPlacement != attr2->clrPlacement) ||
+ (attr1->alignment != attr2->alignment) ||
+ (attr1->share != attr2->share) ||
+ (attr1->pin != attr2->pin);
+}
+
/*
* Get a _SlabKernelBO for us to use as storage for a slab.
*/
@@ -317,7 +300,7 @@ wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header)
uint32_t size = header->bufSize * slabPool->desiredNumBuffers;
struct _WsbmSlabKernelBO *kbo;
struct _WsbmSlabKernelBO *kboTmp;
- int ret;
+ int ret = 0;
/*
* FIXME: We should perhaps allow some variation in slabsize in order
@@ -333,13 +316,12 @@ wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header)
kbo = NULL;
retry:
- head = (slabPool->proposedPlacement & TTM_PL_FLAG_CACHED) ?
- &cache->cached : &cache->unCached;
+ head = &cache->unCached;
WSBMLISTFOREACH(list, head) {
kboTmp = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, head);
- if ((kboTmp->actualSize == size) &&
+ if ((kboTmp->actual_size == size) &&
(slabPool->pageAlignment == 0 ||
(kboTmp->pageAlignment != 0 &&
(kboTmp->pageAlignment % slabPool->pageAlignment) == 0))) {
@@ -347,9 +329,8 @@ wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header)
if (!kbo)
kbo = kboTmp;
- if ((kbo->proposedPlacement ^ slabPool->proposedPlacement) == 0)
+ if (!wsbmAttrDiff(&kbo->attr, &slabPool->attr))
break;
-
}
}
@@ -361,27 +342,15 @@ wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header)
WSBM_MUTEX_UNLOCK(&cache->mutex);
if (kbo) {
- uint32_t new_mask =
- kbo->proposedPlacement ^ slabPool->proposedPlacement;
-
- ret = 0;
- if (new_mask) {
- union ttm_pl_setstatus_arg arg;
- struct ttm_pl_setstatus_req *req = &arg.req;
- struct ttm_pl_rep *rep = &arg.rep;
-
- memset(&arg, 0, sizeof(arg));
- req->handle = kbo->kBuf.handle;
- req->set_placement = slabPool->proposedPlacement & new_mask;
- req->clr_placement = ~slabPool->proposedPlacement & new_mask;
- DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd,
- slabPool->devOffset + TTM_PL_SETSTATUS,
- arg, ret);
- if (ret == 0) {
- kbo->kBuf.gpuOffset = rep->gpu_offset;
- kbo->kBuf.placement = rep->placement;
- }
- kbo->proposedPlacement = slabPool->proposedPlacement;
+ if (wsbmAttrDiff(&kbo->attr, &slabPool->attr)) {
+ struct _WsbmBufStorage *storage = kbo->bStorage;
+ struct _WsbmAttr *attr = &slabPool->attr;
+ ret = storage->pool->setAttr(storage,
+ attr->setPlacement,
+ attr->alignment,
+ attr->share,
+ attr->pin);
+ kbo->attr = slabPool->attr;
}
if (ret == 0)
@@ -397,53 +366,37 @@ wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header)
return NULL;
{
- union ttm_pl_create_arg arg;
+ size_t actual_size =
+ (size + slabPool->pageSize - 1) & ~(slabPool->pageSize - 1);
+ const struct _WsbmAttr *attr = &slabPool->attr;
kbo->slabPool = slabPool;
WSBMINITLISTHEAD(&kbo->head);
WSBMINITLISTHEAD(&kbo->timeoutHead);
-
- memset(&arg, 0, sizeof(arg));
- arg.req.size = size;
- arg.req.placement = slabPool->proposedPlacement;
- arg.req.page_alignment = slabPool->pageAlignment;
-
- DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd,
- slabPool->devOffset + TTM_PL_CREATE,
- arg, ret);
+ kbo->bStorage = slabPool->basePool->create(slabPool->basePool,
+ actual_size,
+ attr->setPlacement,
+ attr->alignment,
+ attr->share,
+ attr->pin);
if (ret)
goto out_err0;
- kbo->kBuf.gpuOffset = arg.rep.gpu_offset;
- kbo->kBuf.placement = arg.rep.placement;
- kbo->kBuf.handle = arg.rep.handle;
-
- kbo->actualSize = arg.rep.bo_size;
- kbo->mapHandle = arg.rep.map_handle;
- kbo->proposedPlacement = slabPool->proposedPlacement;
+ kbo->attr = slabPool->attr;
+ kbo->actual_size = actual_size;
}
- kbo->virtual = mmap(0, kbo->actualSize,
- PROT_READ | PROT_WRITE, MAP_SHARED,
- slabPool->pool.fd, kbo->mapHandle);
+ ret = kbo->bStorage->pool->map(kbo->bStorage,
+ WSBM_ACCESS_READ |
+ WSBM_ACCESS_WRITE, &kbo->virtual);
- if (kbo->virtual == MAP_FAILED) {
- ret = -errno;
+ if (ret)
goto out_err1;
- }
return kbo;
out_err1:
- {
- struct ttm_pl_reference_req arg;
-
- memset(&arg, 0, sizeof(arg));
- arg.handle = kbo->kBuf.handle;
- (void)drmCommandWrite(slabPool->pool.fd,
- slabPool->devOffset + TTM_PL_UNREF,
- &arg, sizeof(arg));
- }
+ kbo->bStorage->pool->destroy(&kbo->bStorage);
out_err0:
free(kbo);
return NULL;
@@ -468,7 +421,7 @@ wsbmAllocSlab(struct _WsbmSlabSizeHeader *header)
goto out_err0;
}
- numBuffers = slab->kbo->actualSize / header->bufSize;
+ numBuffers = slab->kbo->actual_size / header->bufSize;
slab->buffers = calloc(numBuffers, sizeof(*slab->buffers));
if (!slab->buffers) {
@@ -697,7 +650,8 @@ wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader *header)
static struct _WsbmBufStorage *
pool_create(struct _WsbmBufferPool *pool, unsigned long size,
- uint32_t placement, unsigned alignment)
+ uint32_t placement, unsigned alignment,
+ int share, int pin)
{
struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool);
struct _WsbmSlabSizeHeader *header;
@@ -749,44 +703,31 @@ pool_create(struct _WsbmBufferPool *pool, unsigned long size,
goto out_err1;
{
- union ttm_pl_create_arg arg;
-
- memset(&arg, 0, sizeof(arg));
- arg.req.placement = placement;
- arg.req.page_alignment = alignment / slabPool->pageSize;
-
- DRMRESTARTCOMMANDWRITEREAD(pool->fd,
- slabPool->devOffset + TTM_PL_CREATE,
- arg, ret);
-
- if (ret)
+ sBuf->bStorage = slabPool->basePool->create(slabPool->basePool,
+ size,
+ placement,
+ alignment,
+ share,
+ pin);
+
+ if (!sBuf->bStorage) {
+ ret = -ENOMEM;
goto out_err2;
+ }
- sBuf->kBuf.gpuOffset = arg.rep.gpu_offset;
- sBuf->kBuf.placement = arg.rep.placement;
- sBuf->kBuf.handle = arg.rep.handle;
- sBuf->mapHandle = arg.rep.map_handle;
sBuf->requestedSize = size;
+ ret = sBuf->bStorage->pool->map(sBuf->bStorage,
+ WSBM_ACCESS_READ |
+ WSBM_ACCESS_WRITE, &sBuf->virtual);
- sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED,
- pool->fd, sBuf->mapHandle);
-
- if (sBuf->virtual == MAP_FAILED)
+ if (ret != 0)
goto out_err3;
}
wsbmAtomicSet(&sBuf->writers, 0);
return &sBuf->storage;
out_err3:
- {
- struct ttm_pl_reference_req arg;
-
- memset(&arg, 0, sizeof(arg));
- arg.handle = sBuf->kBuf.handle;
- (void)drmCommandWriteRead(pool->fd,
- slabPool->devOffset + TTM_PL_UNREF,
- &arg, sizeof(arg));
- }
+ sBuf->bStorage->pool->destroy(&sBuf->bStorage);
out_err2:
WSBM_COND_FREE(&sBuf->event);
out_err1:
@@ -807,20 +748,12 @@ pool_destroy(struct _WsbmBufStorage **p_buf)
*p_buf = NULL;
if (!sBuf->isSlabBuffer) {
- struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf);
- struct ttm_pl_reference_req arg;
-
if (sBuf->virtual != NULL) {
- (void)munmap(sBuf->virtual, sBuf->requestedSize);
+ sBuf->bStorage->pool->unmap(sBuf->bStorage);
sBuf->virtual = NULL;
}
- memset(&arg, 0, sizeof(arg));
- arg.handle = sBuf->kBuf.handle;
- (void)drmCommandWrite(slabPool->pool.fd,
- slabPool->devOffset + TTM_PL_UNREF,
- &arg, sizeof(arg));
-
+ sBuf->bStorage->pool->destroy(&sBuf->bStorage);
WSBM_COND_FREE(&sBuf->event);
wsbmBufStorageTakedown(&sBuf->storage);
free(sBuf);
@@ -912,13 +845,14 @@ pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode)
}
static int
-pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode)
+pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode,
+ int noBlock)
{
struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
int ret = 0;
WSBM_MUTEX_LOCK(&buf->mutex);
- if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) {
+ if (noBlock) {
int signaled;
if (sBuf->unFenced) {
@@ -977,8 +911,9 @@ static struct _WsbmKernelBuf *
pool_kernel(struct _WsbmBufStorage *buf)
{
struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
-
- return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf;
+ struct _WsbmBufStorage *bStorage =
+ (sBuf->isSlabBuffer) ? sBuf->parent->kbo->bStorage : sBuf->bStorage;
+ return bStorage->pool->kernel(bStorage);
}
static unsigned long
@@ -1146,14 +1081,13 @@ pool_takedown(struct _WsbmBufferPool *pool)
struct _WsbmBufferPool *
wsbmSlabPoolInit(int fd,
- uint32_t devOffset,
- uint32_t placement,
- uint32_t validMask,
+ struct _WsbmBufferPool *basePool,
+ const struct _WsbmAttr *attr,
uint32_t smallestSize,
uint32_t numSizes,
uint32_t desiredNumBuffers,
uint32_t maxSlabSize,
- uint32_t pageAlignment, struct _WsbmSlabCache *cache)
+ struct _WsbmSlabCache *cache)
{
struct _WsbmBufferPool *pool;
struct _WsbmSlabPool *slabPool;
@@ -1173,13 +1107,24 @@ wsbmSlabPoolInit(int fd,
if (!slabPool->headers)
goto out_err1;
- slabPool->devOffset = devOffset;
+ slabPool->basePool = basePool;
slabPool->cache = cache;
- slabPool->proposedPlacement = placement;
- slabPool->validMask = validMask;
+ slabPool->attr = *attr;
slabPool->numBuckets = numSizes;
slabPool->pageSize = getpagesize();
- slabPool->pageAlignment = pageAlignment;
+ if (attr->alignment != 0) {
+ if (attr->alignment < slabPool->pageSize) {
+ if (slabPool->pageSize % attr->alignment != 0)
+ goto out_err1;
+ slabPool->pageAlignment = 1;
+ } else {
+ if (attr->alignment % slabPool->pageSize != 0)
+ goto out_err1;
+ slabPool->pageAlignment = attr->alignment / slabPool->pageSize;
+ }
+ } else
+ slabPool->pageAlignment = 0;
+
slabPool->maxSlabSize = maxSlabSize;
slabPool->desiredNumBuffers = desiredNumBuffers;
diff --git a/src/wsbm_ttmpool.c b/src/wsbm_ttmpool.c
deleted file mode 100644
index 52bdb35..0000000
--- a/src/wsbm_ttmpool.c
+++ /dev/null
@@ -1,518 +0,0 @@
-/**************************************************************************
- *
- * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, Tx., USA
- * All Rights Reserved.
- * Copyright 2009 VMware, Inc., Palo Alto, CA., USA
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sub license, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
- * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
- * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
- * USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- **************************************************************************/
-/*
- * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
- */
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <xf86drm.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
-#include <sys/mman.h>
-#include "wsbm_pool.h"
-#include "assert.h"
-#include "wsbm_priv.h"
-#include "wsbm_manager.h"
-#include "ttm/ttm_placement_user.h"
-
-#define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \
- do { \
- (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \
- } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \
-
-#define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \
- do { \
- (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \
- } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \
-
-/*
- * Buffer pool implementation using DRM buffer objects as wsbm buffer objects.
- */
-
-struct _TTMBuffer
-{
- struct _WsbmBufStorage buf;
- struct _WsbmCond event;
-
- /*
- * Remains constant after creation.
- */
-
- uint64_t requestedSize;
- uint64_t mapHandle;
- uint64_t realSize;
-
- /*
- * Protected by the kernel lock.
- */
-
- struct _WsbmKernelBuf kBuf;
-
- /*
- * Protected by the mutex.
- */
-
- void *virtual;
- int syncInProgress;
- unsigned readers;
- unsigned writers;
-};
-
-struct _TTMPool
-{
- struct _WsbmBufferPool pool;
- unsigned int pageSize;
- unsigned int devOffset;
-};
-
-static inline struct _TTMPool *
-ttmGetPool(struct _TTMBuffer *dBuf)
-{
- return containerOf(dBuf->buf.pool, struct _TTMPool, pool);
-}
-
-static inline struct _TTMBuffer *
-ttmBuffer(struct _WsbmBufStorage *buf)
-{
- return containerOf(buf, struct _TTMBuffer, buf);
-}
-
-static struct _WsbmBufStorage *
-pool_create(struct _WsbmBufferPool *pool,
- unsigned long size, uint32_t placement, unsigned alignment)
-{
- struct _TTMBuffer *dBuf = (struct _TTMBuffer *)
- calloc(1, sizeof(*dBuf));
- struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool);
- int ret;
- unsigned pageSize = ttmPool->pageSize;
- union ttm_pl_create_arg arg;
-
- if (!dBuf)
- return NULL;
-
- if ((alignment > pageSize) && (alignment % pageSize))
- goto out_err0;
-
- ret = wsbmBufStorageInit(&dBuf->buf, pool);
- if (ret)
- goto out_err0;
-
- ret = WSBM_COND_INIT(&dBuf->event);
- if (ret)
- goto out_err1;
-
- memset(&arg, 0, sizeof(arg));
- arg.req.size = size;
- arg.req.placement = placement;
- arg.req.page_alignment = alignment / pageSize;
-
- DRMRESTARTCOMMANDWRITEREAD(pool->fd, ttmPool->devOffset + TTM_PL_CREATE,
- arg, ret);
-
- if (ret)
- goto out_err2;
-
- dBuf->requestedSize = size;
- dBuf->kBuf.gpuOffset = arg.rep.gpu_offset;
- dBuf->mapHandle = arg.rep.map_handle;
- dBuf->realSize = arg.rep.bo_size;
- dBuf->kBuf.placement = arg.rep.placement;
- dBuf->kBuf.handle = arg.rep.handle;
-
- return &dBuf->buf;
-
- out_err2:
- WSBM_COND_FREE(&dBuf->event);
- out_err1:
- wsbmBufStorageTakedown(&dBuf->buf);
- out_err0:
- free(dBuf);
- return NULL;
-}
-
-static struct _WsbmBufStorage *
-pool_reference(struct _WsbmBufferPool *pool, unsigned handle)
-{
- struct _TTMBuffer *dBuf = (struct _TTMBuffer *)calloc(1, sizeof(*dBuf));
- struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool);
- union ttm_pl_reference_arg arg;
- int ret;
-
- if (!dBuf)
- return NULL;
-
- ret = wsbmBufStorageInit(&dBuf->buf, pool);
- if (ret)
- goto out_err0;
-
- ret = WSBM_COND_INIT(&dBuf->event);
- if (ret)
- goto out_err1;
-
- memset(&arg, 0, sizeof(arg));
- arg.req.handle = handle;
- ret = drmCommandWriteRead(pool->fd, ttmPool->devOffset + TTM_PL_REFERENCE,
- &arg, sizeof(arg));
-
- if (ret)
- goto out_err2;
-
- dBuf->requestedSize = arg.rep.bo_size;
- dBuf->kBuf.gpuOffset = arg.rep.gpu_offset;
- dBuf->mapHandle = arg.rep.map_handle;
- dBuf->realSize = arg.rep.bo_size;
- dBuf->kBuf.placement = arg.rep.placement;
- dBuf->kBuf.handle = arg.rep.handle;
- dBuf->kBuf.fence_type_mask = arg.rep.sync_object_arg;
-
- return &dBuf->buf;
-
- out_err2:
- WSBM_COND_FREE(&dBuf->event);
- out_err1:
- wsbmBufStorageTakedown(&dBuf->buf);
- out_err0:
- free(dBuf);
- return NULL;
-}
-
-static void
-pool_destroy(struct _WsbmBufStorage **buf)
-{
- struct _TTMBuffer *dBuf = ttmBuffer(*buf);
- struct _TTMPool *ttmPool = ttmGetPool(dBuf);
- struct ttm_pl_reference_req arg;
-
- *buf = NULL;
- if (dBuf->virtual != NULL) {
- (void)munmap(dBuf->virtual, dBuf->requestedSize);
- dBuf->virtual = NULL;
- }
-
- memset(&arg, 0, sizeof(arg));
- arg.handle = dBuf->kBuf.handle;
- (void)drmCommandWrite(dBuf->buf.pool->fd,
- ttmPool->devOffset + TTM_PL_UNREF,
- &arg, sizeof(arg));
-
- WSBM_COND_FREE(&dBuf->event);
- wsbmBufStorageTakedown(&dBuf->buf);
- free(dBuf);
-}
-
-static int
-syncforcpu_locked(struct _WsbmBufStorage *buf, unsigned mode)
-{
- uint32_t kmode = 0;
- struct _TTMBuffer *dBuf = ttmBuffer(buf);
- struct _TTMPool *ttmPool = ttmGetPool(dBuf);
- unsigned int readers;
- unsigned int writers;
- int ret = 0;
-
- while (dBuf->syncInProgress)
- WSBM_COND_WAIT(&dBuf->event, &buf->mutex);
-
- readers = dBuf->readers;
- writers = dBuf->writers;
-
- if ((mode & WSBM_SYNCCPU_READ) && (++dBuf->readers == 1))
- kmode |= TTM_PL_SYNCCPU_MODE_READ;
-
- if ((mode & WSBM_SYNCCPU_WRITE) && (++dBuf->writers == 1))
- kmode |= TTM_PL_SYNCCPU_MODE_WRITE;
-
- if (kmode) {
- struct ttm_pl_synccpu_arg arg;
-
- if (mode & WSBM_SYNCCPU_DONT_BLOCK)
- kmode |= TTM_PL_SYNCCPU_MODE_NO_BLOCK;
-
- dBuf->syncInProgress = 1;
-
- /*
- * This might be a lengthy wait, so
- * release the mutex.
- */
-
- WSBM_MUTEX_UNLOCK(&buf->mutex);
-
- memset(&arg, 0, sizeof(arg));
- arg.handle = dBuf->kBuf.handle;
- arg.access_mode = kmode;
- arg.op = TTM_PL_SYNCCPU_OP_GRAB;
-
- DRMRESTARTCOMMANDWRITE(dBuf->buf.pool->fd,
- ttmPool->devOffset + TTM_PL_SYNCCPU, arg, ret);
-
- WSBM_MUTEX_LOCK(&buf->mutex);
- dBuf->syncInProgress = 0;
- WSBM_COND_BROADCAST(&dBuf->event);
-
- if (ret) {
- dBuf->readers = readers;
- dBuf->writers = writers;
- }
- }
-
- return ret;
-}
-
-static int
-releasefromcpu_locked(struct _WsbmBufStorage *buf, unsigned mode)
-{
- uint32_t kmode = 0;
- struct _TTMBuffer *dBuf = ttmBuffer(buf);
- struct _TTMPool *ttmPool = ttmGetPool(dBuf);
- int ret = 0;
-
- while (dBuf->syncInProgress)
- WSBM_COND_WAIT(&dBuf->event, &buf->mutex);
-
- if ((mode & WSBM_SYNCCPU_READ) && (--dBuf->readers == 0))
- kmode |= TTM_PL_SYNCCPU_MODE_READ;
-
- if ((mode & WSBM_SYNCCPU_WRITE) && (--dBuf->writers == 0))
- kmode |= TTM_PL_SYNCCPU_MODE_WRITE;
-
- if (kmode) {
- struct ttm_pl_synccpu_arg arg;
-
- memset(&arg, 0, sizeof(arg));
- arg.handle = dBuf->kBuf.handle;
- arg.access_mode = kmode;
- arg.op = TTM_PL_SYNCCPU_OP_RELEASE;
-
- DRMRESTARTCOMMANDWRITE(dBuf->buf.pool->fd,
- ttmPool->devOffset + TTM_PL_SYNCCPU, arg, ret);
-
- }
-
- return ret;
-}
-
-static int
-pool_syncforcpu(struct _WsbmBufStorage *buf, unsigned mode)
-{
- int ret;
-
- WSBM_MUTEX_LOCK(&buf->mutex);
- ret = syncforcpu_locked(buf, mode);
- WSBM_MUTEX_UNLOCK(&buf->mutex);
- return ret;
-}
-
-static void
-pool_releasefromcpu(struct _WsbmBufStorage *buf, unsigned mode)
-{
- WSBM_MUTEX_LOCK(&buf->mutex);
- (void)releasefromcpu_locked(buf, mode);
- WSBM_MUTEX_UNLOCK(&buf->mutex);
-}
-
-static int
-pool_map(struct _WsbmBufStorage *buf, unsigned mode, void **virtual)
-{
- struct _TTMBuffer *dBuf = ttmBuffer(buf);
- void *virt;
- int ret = 0;
-
- WSBM_MUTEX_LOCK(&buf->mutex);
-
- /*
- * mmaps are expensive, so we only really unmap if
- * we destroy the buffer.
- */
-
- if (dBuf->virtual == NULL) {
- virt = mmap(0, dBuf->requestedSize,
- PROT_READ | PROT_WRITE, MAP_SHARED,
- buf->pool->fd, dBuf->mapHandle);
- if (virt == MAP_FAILED) {
- ret = -errno;
- goto out_unlock;
- }
- dBuf->virtual = virt;
- }
-
- *virtual = dBuf->virtual;
- out_unlock:
-
- WSBM_MUTEX_UNLOCK(&buf->mutex);
-
- return ret;
-}
-
-static void
-pool_unmap(struct _WsbmBufStorage *buf)
-{
- ;
-}
-
-static unsigned long
-pool_offset(struct _WsbmBufStorage *buf)
-{
- struct _TTMBuffer *dBuf = ttmBuffer(buf);
-
- return dBuf->kBuf.gpuOffset;
-}
-
-static unsigned long
-pool_poolOffset(struct _WsbmBufStorage *buf)
-{
- return 0;
-}
-
-static uint32_t
-pool_placement(struct _WsbmBufStorage *buf)
-{
- struct _TTMBuffer *dBuf = ttmBuffer(buf);
-
- return dBuf->kBuf.placement;
-}
-
-static unsigned long
-pool_size(struct _WsbmBufStorage *buf)
-{
- struct _TTMBuffer *dBuf = ttmBuffer(buf);
-
- return dBuf->realSize;
-}
-
-static void
-pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence)
-{
- /*
- * Noop. The kernel handles all fencing.
- */
-}
-
-static int
-pool_waitIdle(struct _WsbmBufStorage *buf, int lazy)
-{
- struct _TTMBuffer *dBuf = ttmBuffer(buf);
- struct _TTMPool *ttmPool = ttmGetPool(dBuf);
- struct ttm_pl_waitidle_arg req;
- struct _WsbmBufferPool *pool = buf->pool;
- int ret;
-
- memset(&req, 0, sizeof(req));
- req.handle = dBuf->kBuf.handle;
- req.mode = (lazy) ? TTM_PL_WAITIDLE_MODE_LAZY : 0;
-
- DRMRESTARTCOMMANDWRITE(pool->fd, ttmPool->devOffset + TTM_PL_WAITIDLE,
- req, ret);
-
- return ret;
-}
-
-static void
-pool_takedown(struct _WsbmBufferPool *pool)
-{
- struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool);
-
- free(ttmPool);
-}
-
-static int
-pool_setStatus(struct _WsbmBufStorage *buf, uint32_t set_placement,
- uint32_t clr_placement)
-{
- struct _TTMBuffer *dBuf = ttmBuffer(buf);
- struct _TTMPool *ttmPool = ttmGetPool(dBuf);
- union ttm_pl_setstatus_arg arg;
- struct ttm_pl_setstatus_req *req = &arg.req;
- struct ttm_pl_rep *rep = &arg.rep;
- struct _WsbmBufferPool *pool = buf->pool;
- int ret;
-
- memset(&arg, 0, sizeof(arg));
- req->handle = dBuf->kBuf.handle;
- req->set_placement = set_placement;
- req->clr_placement = clr_placement;
-
- DRMRESTARTCOMMANDWRITEREAD(pool->fd,
- ttmPool->devOffset + TTM_PL_SETSTATUS,
- arg, ret);
-
- if (!ret) {
- dBuf->kBuf.gpuOffset = rep->gpu_offset;
- dBuf->kBuf.placement = rep->placement;
- }
-
- return ret;
-}
-
-static struct _WsbmKernelBuf *
-pool_kernel(struct _WsbmBufStorage *buf)
-{
- return (void *)&ttmBuffer(buf)->kBuf;
-}
-
-struct _WsbmBufferPool *
-wsbmTTMPoolInit(int fd, unsigned int devOffset)
-{
- struct _TTMPool *ttmPool;
- struct _WsbmBufferPool *pool;
-
- ttmPool = (struct _TTMPool *)calloc(1, sizeof(*ttmPool));
-
- if (!ttmPool)
- return NULL;
-
- ttmPool->pageSize = getpagesize();
- ttmPool->devOffset = devOffset;
- pool = &ttmPool->pool;
-
- pool->fd = fd;
- pool->map = &pool_map;
- pool->unmap = &pool_unmap;
- pool->syncforcpu = &pool_syncforcpu;
- pool->releasefromcpu = &pool_releasefromcpu;
- pool->destroy = &pool_destroy;
- pool->offset = &pool_offset;
- pool->poolOffset = &pool_poolOffset;
- pool->placement = &pool_placement;
- pool->size = &pool_size;
- pool->create = &pool_create;
- pool->fence = &pool_fence;
- pool->kernel = &pool_kernel;
- pool->validate = NULL;
- pool->unvalidate = NULL;
- pool->waitIdle = &pool_waitIdle;
- pool->takeDown = &pool_takedown;
- pool->createByReference = &pool_reference;
- pool->setStatus = &pool_setStatus;
- return pool;
-}
diff --git a/src/wsbm_userpool.c b/src/wsbm_userpool.c
index d945129..c07cbc0 100644
--- a/src/wsbm_userpool.c
+++ b/src/wsbm_userpool.c
@@ -59,6 +59,7 @@ struct _WsbmUserBuffer
{
struct _WsbmBufStorage buf;
struct _WsbmKernelBuf kBuf;
+
/* Protected by the pool mutex */
@@ -102,7 +103,11 @@ struct _WsbmUserPool
struct _WsbmListHead agpLRU;
struct _WsbmMM vramMM;
struct _WsbmMM agpMM;
- uint32_t(*fenceTypes) (uint64_t);
+ uint32_t(*fenceTypes) (uint64_t);
+ uint32_t system_flag;
+ uint32_t vram_flag;
+ uint32_t agp_flag;
+ uint32_t mem_mask;
};
static inline struct _WsbmUserPool *
@@ -187,7 +192,7 @@ evict_lru(struct _WsbmListHead *lru)
memcpy(WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem), vBuf->map, vBuf->size);
WSBMLISTDELINIT(&vBuf->lru);
- vBuf->kBuf.placement = WSBM_PL_FLAG_SYSTEM;
+ vBuf->kBuf.placement = p->system_flag;
vBuf->map = WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem);
/*
@@ -202,23 +207,16 @@ evict_lru(struct _WsbmListHead *lru)
static struct _WsbmBufStorage *
pool_create(struct _WsbmBufferPool *pool,
- unsigned long size, uint32_t placement, unsigned alignment)
+ unsigned long size, uint32_t placement, unsigned alignment,
+ int share, int pin)
{
struct _WsbmUserPool *p = containerOf(pool, struct _WsbmUserPool, pool);
struct _WsbmUserBuffer *vBuf = calloc(1, sizeof(*vBuf));
- int ret;
if (!vBuf)
return NULL;
- ret = wsbmBufStorageInit(&vBuf->buf, pool);
- if (ret)
- goto out_err0;
-
- ret = WSBM_COND_INIT(&vBuf->event);
- if (ret)
- goto out_err1;
-
+ wsbmBufStorageInit(&vBuf->buf, pool);
vBuf->sysmem = NULL;
vBuf->proposedPlacement = placement;
vBuf->size = size;
@@ -228,13 +226,13 @@ pool_create(struct _WsbmBufferPool *pool,
WSBMINITLISTHEAD(&vBuf->delayed);
WSBM_MUTEX_LOCK(&p->mutex);
- if (placement & WSBM_PL_FLAG_TT) {
+ if (placement & p->agp_flag) {
vBuf->node = wsbmMMSearchFree(&p->agpMM, size, alignment, 1);
if (vBuf->node)
vBuf->node = wsbmMMGetBlock(vBuf->node, size, alignment);
if (vBuf->node) {
- vBuf->kBuf.placement = WSBM_PL_FLAG_TT;
+ vBuf->kBuf.placement = p->agp_flag;
vBuf->kBuf.gpuOffset = p->agpOffset + vBuf->node->start;
vBuf->map = (void *)(p->agpMap + vBuf->node->start);
WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU);
@@ -242,13 +240,13 @@ pool_create(struct _WsbmBufferPool *pool,
}
}
- if (placement & WSBM_PL_FLAG_VRAM) {
+ if (placement & p->vram_flag) {
vBuf->node = wsbmMMSearchFree(&p->vramMM, size, alignment, 1);
if (vBuf->node)
vBuf->node = wsbmMMGetBlock(vBuf->node, size, alignment);
if (vBuf->node) {
- vBuf->kBuf.placement = WSBM_PL_FLAG_VRAM;
+ vBuf->kBuf.placement = p->vram_flag;
vBuf->kBuf.gpuOffset = p->vramOffset + vBuf->node->start;
vBuf->map = (void *)(p->vramMap + vBuf->node->start);
WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU);
@@ -256,28 +254,22 @@ pool_create(struct _WsbmBufferPool *pool,
}
}
- if ((placement & WSBM_PL_FLAG_NO_EVICT)
- && !(placement & WSBM_PL_FLAG_SYSTEM)) {
+ if (pin && !(placement & p->system_flag)) {
WSBM_MUTEX_UNLOCK(&p->mutex);
- goto out_err2;
+ goto out_err;
}
vBuf->sysmem = malloc(size + WSBM_USER_ALIGN_ADD);
- vBuf->kBuf.placement = WSBM_PL_FLAG_SYSTEM;
+ vBuf->kBuf.placement = p->system_flag;
vBuf->map = WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem);
have_mem:
WSBM_MUTEX_UNLOCK(&p->mutex);
if (vBuf->sysmem != NULL
- || (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM)))
+ || (!(vBuf->kBuf.placement & p->system_flag)))
return &vBuf->buf;
-
- out_err2:
- WSBM_COND_FREE(&vBuf->event);
- out_err1:
- wsbmBufStorageTakedown(&vBuf->buf);
- out_err0:
+ out_err:
free(vBuf);
return NULL;
}
@@ -303,8 +295,10 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags,
vBuf->proposedPlacement =
(vBuf->proposedPlacement | set_flags) & ~clr_flags;
- if ((vBuf->proposedPlacement & vBuf->kBuf.placement & WSBM_PL_MASK_MEM) ==
- vBuf->kBuf.placement) {
+ if ((vBuf->proposedPlacement & vBuf->kBuf.placement & p->mem_mask) ==
+ vBuf->kBuf.placement &&
+ (vBuf->alignment == 0 ||
+ (vBuf->kBuf.gpuOffset % vBuf->alignment == 0))) {
err = 0;
goto have_mem;
}
@@ -314,7 +308,7 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags,
* do a sw copy to the other region.
*/
- if (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM)) {
+ if (!(vBuf->kBuf.placement & p->system_flag)) {
struct _WsbmListHead tmpLRU;
WSBMINITLISTHEAD(&tmpLRU);
@@ -324,7 +318,7 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags,
goto have_mem;
}
- if (vBuf->proposedPlacement & WSBM_PL_FLAG_TT) {
+ if (vBuf->proposedPlacement & p->agp_flag) {
do {
vBuf->node =
wsbmMMSearchFree(&p->agpMM, vBuf->size, vBuf->alignment, 1);
@@ -333,7 +327,7 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags,
wsbmMMGetBlock(vBuf->node, vBuf->size, vBuf->alignment);
if (vBuf->node) {
- vBuf->kBuf.placement = WSBM_PL_FLAG_TT;
+ vBuf->kBuf.placement = p->agp_flag;
vBuf->kBuf.gpuOffset = p->agpOffset + vBuf->node->start;
vBuf->map = (void *)(p->agpMap + vBuf->node->start);
memcpy(vBuf->map, WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem),
@@ -344,7 +338,7 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags,
} while (evict_lru(&p->agpLRU) == 0);
}
- if (vBuf->proposedPlacement & WSBM_PL_FLAG_VRAM) {
+ if (vBuf->proposedPlacement & p->vram_flag) {
do {
vBuf->node =
wsbmMMSearchFree(&p->vramMM, vBuf->size, vBuf->alignment, 1);
@@ -353,7 +347,7 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags,
wsbmMMGetBlock(vBuf->node, vBuf->size, vBuf->alignment);
if (!err) {
- vBuf->kBuf.placement = WSBM_PL_FLAG_VRAM;
+ vBuf->kBuf.placement = p->vram_flag;
vBuf->kBuf.gpuOffset = p->vramOffset + vBuf->node->start;
vBuf->map = (void *)(p->vramMap + vBuf->node->start);
memcpy(vBuf->map, WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem),
@@ -364,7 +358,7 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags,
} while (evict_lru(&p->vramLRU) == 0);
}
- if (vBuf->proposedPlacement & WSBM_PL_FLAG_SYSTEM)
+ if (vBuf->proposedPlacement & p->system_flag)
goto have_mem;
err = -ENOMEM;
@@ -377,13 +371,17 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags,
}
static int
-pool_setStatus(struct _WsbmBufStorage *buf,
- uint32_t set_placement, uint32_t clr_placement)
+pool_setAttr(struct _WsbmBufStorage *buf,
+ uint32_t placement,
+ unsigned alignment,
+ int share,
+ int pin)
{
struct _WsbmUserBuffer *vBuf = userBuf(buf);
int ret;
- ret = pool_validate(buf, set_placement, clr_placement);
+ vBuf->alignment = alignment;
+ ret = pool_validate(buf, placement, ~placement);
vBuf->unFenced = 0;
return ret;
}
@@ -413,7 +411,7 @@ release_delayed_buffers(struct _WsbmUserPool *p)
WSBMLISTDEL(&vBuf->delayed);
WSBMLISTDEL(&vBuf->lru);
- if ((vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM) == 0)
+ if ((vBuf->kBuf.placement & p->system_flag) == 0)
wsbmMMPutBlock(vBuf->node);
else
free(vBuf->sysmem);
@@ -451,13 +449,11 @@ pool_destroy(struct _WsbmBufStorage **buf)
WSBMLISTDEL(&vBuf->lru);
WSBM_MUTEX_UNLOCK(&p->mutex);
- if (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM))
+ if (!(vBuf->kBuf.placement & p->system_flag))
wsbmMMPutBlock(vBuf->node);
else
free(vBuf->sysmem);
- WSBM_COND_FREE(&vBuf->event);
-
free(vBuf);
return;
}
@@ -488,13 +484,14 @@ pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode)
}
static int
-pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode)
+pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode,
+ int noBlock)
{
struct _WsbmUserBuffer *vBuf = userBuf(buf);
int ret = 0;
WSBM_MUTEX_LOCK(&buf->mutex);
- if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) {
+ if (noBlock) {
if (vBuf->unFenced) {
ret = -EBUSY;
@@ -553,9 +550,9 @@ pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence)
WSBM_COND_BROADCAST(&vBuf->event);
WSBM_MUTEX_LOCK(&p->mutex);
- if (vBuf->kBuf.placement & WSBM_PL_FLAG_VRAM)
+ if (vBuf->kBuf.placement & p->vram_flag)
WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU);
- else if (vBuf->kBuf.placement & WSBM_PL_FLAG_TT)
+ else if (vBuf->kBuf.placement & p->agp_flag)
WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU);
WSBM_MUTEX_UNLOCK(&p->mutex);
WSBM_MUTEX_UNLOCK(&buf->mutex);
@@ -575,9 +572,9 @@ pool_unvalidate(struct _WsbmBufStorage *buf)
vBuf->unFenced = 0;
WSBM_COND_BROADCAST(&vBuf->event);
WSBM_MUTEX_LOCK(&p->mutex);
- if (vBuf->kBuf.placement & WSBM_PL_FLAG_VRAM)
+ if (vBuf->kBuf.placement & p->vram_flag)
WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU);
- else if (vBuf->kBuf.placement & WSBM_PL_FLAG_TT)
+ else if (vBuf->kBuf.placement & p->agp_flag)
WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU);
WSBM_MUTEX_UNLOCK(&p->mutex);
@@ -641,6 +638,9 @@ wsbmUserPoolInit(void *vramAddr,
unsigned long vramStart, unsigned long vramSize,
void *agpAddr, unsigned long agpStart,
unsigned long agpSize,
+ uint32_t system_flag,
+ uint32_t vram_flag,
+ uint32_t agp_flag,
uint32_t(*fenceTypes) (uint64_t set_flags))
{
struct _WsbmBufferPool *pool;
@@ -672,6 +672,10 @@ wsbmUserPoolInit(void *vramAddr,
uPool->vramOffset = vramStart;
uPool->vramMap = (unsigned long)vramAddr;
uPool->fenceTypes = fenceTypes;
+ uPool->system_flag = system_flag;
+ uPool->vram_flag = vram_flag;
+ uPool->agp_flag = agp_flag;
+ uPool->mem_mask = uPool->system_flag | uPool->vram_flag | uPool->agp_flag;
pool = &uPool->pool;
pool->map = &pool_map;
@@ -687,7 +691,7 @@ wsbmUserPoolInit(void *vramAddr,
pool->validate = &pool_validate;
pool->waitIdle = &pool_waitIdle;
pool->takeDown = &pool_takedown;
- pool->setStatus = &pool_setStatus;
+ pool->setAttr = &pool_setAttr;
pool->syncforcpu = &pool_syncForCpu;
pool->releasefromcpu = &pool_releaseFromCpu;