diff options
author | Thomas Hellstrom <thellstrom@vmware.com> | 2009-08-17 17:17:24 +0200 |
---|---|---|
committer | Thomas Hellstrom <thellstrom@vmware.com> | 2009-08-17 17:17:24 +0200 |
commit | 3e8e650f25e5e40fd30b038ee3abf175523d42a6 (patch) | |
tree | 9672e7072483658ef81ab4d85bd020e2dcb7a59d | |
parent | 375a9b16472031d603fd0d2f3cbea3e1ed9bc6c7 (diff) |
libwsbm: Big update for libwsbm 2.0.
Prepare for per-device libs,
Split out command submission,
Rework driver interface,
Bump major.
-rw-r--r-- | configure.ac | 2 | ||||
-rw-r--r-- | src/Makefile.am | 15 | ||||
-rw-r--r-- | src/wsbm_cs.c | 531 | ||||
-rw-r--r-- | src/wsbm_cs.h | 69 | ||||
-rw-r--r-- | src/wsbm_driver.h | 6 | ||||
-rw-r--r-- | src/wsbm_fencemgr.c | 108 | ||||
-rw-r--r-- | src/wsbm_fencemgr.h | 11 | ||||
-rw-r--r-- | src/wsbm_mallocpool.c | 9 | ||||
-rw-r--r-- | src/wsbm_manager.c | 723 | ||||
-rw-r--r-- | src/wsbm_manager.h | 104 | ||||
-rw-r--r-- | src/wsbm_pool.h | 36 | ||||
-rw-r--r-- | src/wsbm_slabpool.c | 239 | ||||
-rw-r--r-- | src/wsbm_ttmpool.c | 518 | ||||
-rw-r--r-- | src/wsbm_userpool.c | 102 |
14 files changed, 950 insertions, 1523 deletions
diff --git a/configure.ac b/configure.ac index 24191f2..63feff7 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_PREREQ(2.57) -AC_INIT([libwsbm], 1.1.0, [thomas@tungstengraphics.com], libwsbm) +AC_INIT([libwsbm], 2.0.0, [thomas@vmware.com], libwsbm) AC_CONFIG_SRCDIR([Makefile.am]) AM_INIT_AUTOMAKE([dist-bzip2]) diff --git a/src/Makefile.am b/src/Makefile.am index 4778f37..b9be786 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,13 +1,14 @@ - lib_LTLIBRARIES = libwsbm.la -libwsbm_la_CFLAGS = @libdrm_CFLAGS@ -Wall -libwsbm_la_LDFLAGS = -version-number 1:1:0 -no-undefined @libdrm_LIBS@ +libwsbm_la_CFLAGS = -Wall @libdrm_CFLAGS@ +libwsbm_la_LDFLAGS = -version-number 2:0:0 -no-undefined libwsbm_la_SOURCES = \ wsbm_fencemgr.c \ wsbm_fencemgr.h \ wsbm_manager.c \ wsbm_manager.h \ + wsbm_cs.c \ + wsbm_cs.h \ wsbm_mm.c \ wsbm_mm.h \ wsbm_pool.h \ @@ -15,13 +16,11 @@ libwsbm_la_SOURCES = \ wsbm_mallocpool.c \ wsbm_driver.h \ wsbm_driver.c \ - wsbm_ttmpool.c \ wsbm_slabpool.c \ wsbm_userpool.c \ - wsbm_priv.h + wsbm_priv.h \ + wsbm_atomic.h libwsbmincludedir = ${includedir}/wsbm -libwsbminclude_HEADERS = wsbm_manager.h wsbm_pool.h wsbm_driver.h \ - wsbm_fencemgr.h wsbm_util.h wsbm_atomic.h - +libwsbminclude_HEADERS = wsbm_manager.h wsbm_fencemgr.h diff --git a/src/wsbm_cs.c b/src/wsbm_cs.c new file mode 100644 index 0000000..177bcb8 --- /dev/null +++ b/src/wsbm_cs.c @@ -0,0 +1,531 @@ +/************************************************************************** + * + * Copyright 2006-2009 Vmware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "wsbm_cs.h" +#include "wsbm_pool.h" + +#include <stdint.h> +#include <stdlib.h> +#include <errno.h> + +extern struct _WsbmBufStorage * +wsbmBOStorage(struct _WsbmBufferObject *buf); + +struct _ValidateList +{ + const struct _WsbmDriver *driver; + unsigned numTarget; + unsigned numCurrent; + unsigned numOnList; + unsigned hashSize; + uint32_t hashMask; + struct _WsbmListHead list; + struct _WsbmListHead free; + struct _WsbmListHead *hashTable; +}; + +struct _WsbmBufferList +{ + int hasKernelBuffers; + + struct _ValidateList kernelBuffers; /* List of kernel buffers needing validation */ + struct _ValidateList userBuffers; /* List of user-space buffers needing validation */ +}; + +static struct _ValidateNode * +validateListAddNode(struct _ValidateList *list, void *item, + uint32_t hash, uint64_t flags, uint64_t mask) +{ + struct _ValidateNode *node; + struct _WsbmListHead *l; + struct _WsbmListHead *hashHead; + + l = list->free.next; + if (l == &list->free) { + node = list->driver->alloc(list->driver); + if (!node) { + return NULL; + } + list->numCurrent++; + } else { + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + } + node->buf = item; + node->set_flags = flags & mask; + node->clr_flags = (~flags) & mask; + node->listItem = list->numOnList; + WSBMLISTADDTAIL(&node->head, &list->list); + list->numOnList++; + hashHead = list->hashTable + hash; + WSBMLISTADDTAIL(&node->hashHead, hashHead); + + return node; +} + +static uint32_t +wsbmHashFunc(uint8_t * key, uint32_t len, uint32_t mask) +{ + uint32_t hash, i; + + for (hash = 0, i = 0; i < len; ++i) { + hash += *key++; + hash += (hash << 10); + hash ^= (hash >> 6); + } + + hash += (hash << 3); + hash ^= (hash >> 11); + hash += (hash << 15); + + return hash & mask; +} + +static void +validateFreeList(struct _ValidateList *list) +{ + struct _ValidateNode *node; + struct _WsbmListHead *l; + + l = list->list.next; + while (l != &list->list) { + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + + WSBMLISTDEL(&node->hashHead); + node->driver->free(node); + l = list->list.next; + list->numCurrent--; + list->numOnList--; + } + + l = list->free.next; + while (l != &list->free) { + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + + node->driver->free(node); + l = list->free.next; + list->numCurrent--; + } + free(list->hashTable); +} + +static int +validateListAdjustNodes(struct _ValidateList *list) +{ + struct _ValidateNode *node; + struct _WsbmListHead *l; + int ret = 0; + + while (list->numCurrent < list->numTarget) { + node = list->driver->alloc(list->driver); + if (!node) { + ret = -ENOMEM; + break; + } + list->numCurrent++; + WSBMLISTADD(&node->head, &list->free); + } + + while (list->numCurrent > list->numTarget) { + l = list->free.next; + if (l == &list->free) + break; + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + + node->driver->free(node); + list->numCurrent--; + } + return ret; +} + +static inline int +wsbmPot(unsigned int val) +{ + unsigned int shift = 0; + while(val > (1 << shift)) + shift++; + + return shift; +} + +static int +validateCreateList(int numTarget, struct _ValidateList *list, + const struct _WsbmDriver *driver) +{ + int i; + unsigned int shift = wsbmPot(numTarget); + int ret; + + list->hashSize = (1 << shift); + list->hashMask = list->hashSize - 1; + + list->hashTable = malloc(list->hashSize * sizeof(*list->hashTable)); + if (!list->hashTable) + return -ENOMEM; + + for (i = 0; i < list->hashSize; ++i) + WSBMINITLISTHEAD(&list->hashTable[i]); + + WSBMINITLISTHEAD(&list->list); + WSBMINITLISTHEAD(&list->free); + list->numTarget = numTarget; + list->numCurrent = 0; + list->numOnList = 0; + list->driver = driver; + ret = validateListAdjustNodes(list); + if (ret != 0) + free(list->hashTable); + + return ret; +} + +static int +validateResetList(struct _ValidateList *list) +{ + struct _WsbmListHead *l; + struct _ValidateNode *node; + int ret; + + ret = validateListAdjustNodes(list); + if (ret) + return ret; + + l = list->list.next; + while (l != &list->list) { + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + + WSBMLISTDEL(&node->hashHead); + WSBMLISTADD(l, &list->free); + list->numOnList--; + l = list->list.next; + } + return validateListAdjustNodes(list); +} + + +/* + * Note that lists are per-context and don't need mutex protection. + */ + +struct _WsbmBufferList * +wsbmBOCreateList(int target, int hasKernelBuffers, + const struct _WsbmDriver *kernelDriver, + const struct _WsbmDriver *userDriver) +{ + struct _WsbmBufferList *list = calloc(sizeof(*list), 1); + int ret; + + list->hasKernelBuffers = hasKernelBuffers; + if (hasKernelBuffers) { + ret = validateCreateList(target, &list->kernelBuffers, kernelDriver); + if (ret) + return NULL; + } + + ret = validateCreateList(target, &list->userBuffers, userDriver); + if (ret) { + validateFreeList(&list->kernelBuffers); + return NULL; + } + + return list; +} + +int +wsbmBOResetList(struct _WsbmBufferList *list) +{ + int ret; + + if (list->hasKernelBuffers) { + ret = validateResetList(&list->kernelBuffers); + if (ret) + return ret; + } + ret = validateResetList(&list->userBuffers); + return ret; +} + +void +wsbmBOFreeList(struct _WsbmBufferList *list) +{ + if (list->hasKernelBuffers) + validateFreeList(&list->kernelBuffers); + validateFreeList(&list->userBuffers); + free(list); +} + +static int +wsbmAddValidateItem(struct _ValidateList *list, void *buf, uint64_t flags, + uint64_t mask, int *itemLoc, + struct _ValidateNode **pnode, int *newItem) +{ + struct _ValidateNode *node, *cur; + struct _WsbmListHead *l; + struct _WsbmListHead *hashHead; + uint32_t hash; + uint32_t count = 0; + uint32_t key = (unsigned long) buf; + + cur = NULL; + hash = wsbmHashFunc((uint8_t *) &key, 4, list->hashMask); + hashHead = list->hashTable + hash; + *newItem = 0; + + for (l = hashHead->next; l != hashHead; l = l->next) { + count++; + node = WSBMLISTENTRY(l, struct _ValidateNode, hashHead); + + if (node->buf == buf) { + cur = node; + break; + } + } + + if (!cur) { + cur = validateListAddNode(list, buf, hash, flags, mask); + if (!cur) + return -ENOMEM; + *newItem = 1; + cur->driver->clear(cur); + } else { + uint64_t set_flags = flags & mask; + uint64_t clr_flags = (~flags) & mask; + + if (((cur->clr_flags | clr_flags) & WSBM_PLACEMENT_MASK) == + WSBM_PLACEMENT_MASK) { + /* + * No available memory type left. Bail. + */ + return -EINVAL; + } + + if ((cur->set_flags | set_flags) & + (cur->clr_flags | clr_flags) & ~WSBM_PLACEMENT_MASK) { + /* + * Conflicting flags. Bail. + */ + return -EINVAL; + } + + cur->set_flags |= set_flags; + cur->clr_flags |= clr_flags; + cur->set_flags &= ~(cur->clr_flags); + } + *itemLoc = cur->listItem; + if (pnode) + *pnode = cur; + return 0; +} + + +int +wsbmBOAddListItem(struct _WsbmBufferList *list, + struct _WsbmBufferObject *buf, + uint64_t flags, uint64_t mask, int *itemLoc, + struct _ValidateNode **node) +{ + int newItem; + struct _WsbmBufStorage *storage = wsbmBOStorage(buf); + int ret; + int dummy; + struct _ValidateNode *dummyNode; + + if (list->hasKernelBuffers) { + ret = wsbmAddValidateItem(&list->kernelBuffers, + storage->pool->kernel(storage), + flags, mask, itemLoc, node, &dummy); + if (ret) + goto out_unlock; + } else { + *node = NULL; + *itemLoc = -1000; + } + + ret = wsbmAddValidateItem(&list->userBuffers, storage, + flags, mask, &dummy, &dummyNode, &newItem); + if (ret) + goto out_unlock; + + if (newItem) { + wsbmAtomicInc(&storage->refCount); + wsbmAtomicInc(&storage->onList); + } + + out_unlock: + return ret; +} + +int +wsbmBOUnrefUserList(struct _WsbmBufferList *list) +{ + struct _WsbmBufStorage *storage; + void *curBuf; + + curBuf = validateListIterator(&list->userBuffers); + + while (curBuf) { + storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf); + wsbmAtomicDec(&storage->onList); + wsbmBufStorageUnref(&storage); + curBuf = validateListNext(&list->userBuffers, curBuf); + } + + return wsbmBOResetList(list); +} + + +int +wsbmBOFenceUserList(struct _WsbmBufferList *list, + struct _WsbmFenceObject *fence) +{ + struct _WsbmBufStorage *storage; + void *curBuf; + + curBuf = validateListIterator(&list->userBuffers); + + /* + * User-space fencing callbacks. + */ + + while (curBuf) { + storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf); + + storage->pool->fence(storage, fence); + wsbmAtomicDec(&storage->onList); + wsbmBufStorageUnref(&storage); + curBuf = validateListNext(&list->userBuffers, curBuf); + } + + return wsbmBOResetList(list); +} + +int +wsbmBOValidateUserList(struct _WsbmBufferList *list) +{ + void *curBuf; + struct _WsbmBufStorage *storage; + struct _ValidateNode *node; + int ret; + + curBuf = validateListIterator(&list->userBuffers); + + /* + * User-space validation callbacks. + */ + + while (curBuf) { + node = validateListNode(curBuf); + storage = (struct _WsbmBufStorage *)node->buf; + if (storage->pool->validate) { + ret = storage->pool->validate(storage, node->set_flags, + node->clr_flags); + if (ret) + return ret; + } + curBuf = validateListNext(&list->userBuffers, curBuf); + } + return 0; +} + +int wsbmBOUnvalidateUserList(struct _WsbmBufferList *list) +{ + void *curBuf; + struct _WsbmBufStorage *storage; + struct _ValidateNode *node; + + curBuf = validateListIterator(&list->userBuffers); + + /* + * User-space validation callbacks. + */ + + while (curBuf) { + node = validateListNode(curBuf); + storage = (struct _WsbmBufStorage *)node->buf; + if (storage->pool->unvalidate) { + storage->pool->unvalidate(storage); + } + wsbmAtomicDec(&storage->onList); + wsbmBufStorageUnref(&storage); + curBuf = validateListNext(&list->userBuffers, curBuf); + } + return wsbmBOResetList(list); +} + +struct _ValidateList * +wsbmGetKernelValidateList(struct _WsbmBufferList *list) +{ + return (list->hasKernelBuffers) ? &list->kernelBuffers : NULL; +} + +struct _ValidateList * +wsbmGetUserValidateList(struct _WsbmBufferList *list) +{ + return &list->userBuffers; +} + +struct _ValidateNode * +validateListNode(void *iterator) +{ + struct _WsbmListHead *l = (struct _WsbmListHead *)iterator; + + return WSBMLISTENTRY(l, struct _ValidateNode, head); +} + +void * +validateListIterator(struct _ValidateList *list) +{ + void *ret = list->list.next; + + if (ret == &list->list) + return NULL; + return ret; +} + +void * +validateListNext(struct _ValidateList *list, void *iterator) +{ + void *ret; + + struct _WsbmListHead *l = (struct _WsbmListHead *)iterator; + + ret = l->next; + if (ret == &list->list) + return NULL; + return ret; +} diff --git a/src/wsbm_cs.h b/src/wsbm_cs.h new file mode 100644 index 0000000..511edce --- /dev/null +++ b/src/wsbm_cs.h @@ -0,0 +1,69 @@ +/************************************************************************** + * + * Copyright 2006-2009 Vmware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> + */ + +#ifndef _WSBM_CS_H_ +#define _WSBM_CS_H_ +#include "wsbm_driver.h" + +struct _WsbmFenceObject; +struct _WsbmBufferObject; +struct _ValidateList; + +#define WSBM_PLACEMENT_MASK ((uint64_t) 0xFFFFFFFFULL) + +extern struct _WsbmBufferList *wsbmBOCreateList(int target, + int hasKernelBuffers, + const struct _WsbmDriver *kernelDriver, + const struct _WsbmDriver *userDriver); + +extern int wsbmBOResetList(struct _WsbmBufferList *list); +extern int wsbmBOAddListItem(struct _WsbmBufferList *list, + struct _WsbmBufferObject *buf, + uint64_t flags, uint64_t mask, int *itemLoc, + struct _ValidateNode **node); + +extern void wsbmBOFreeList(struct _WsbmBufferList *list); +extern int wsbmBOFenceUserList(struct _WsbmBufferList *list, + struct _WsbmFenceObject *fence); + +extern int wsbmBOUnrefUserList(struct _WsbmBufferList *list); +extern int wsbmBOValidateUserList(struct _WsbmBufferList *list); +extern int wsbmBOUnvalidateUserList(struct _WsbmBufferList *list); + +extern struct _ValidateList *wsbmGetKernelValidateList(struct _WsbmBufferList + *list); +extern struct _ValidateList *wsbmGetUserValidateList(struct _WsbmBufferList + *list); + +extern struct _ValidateNode *validateListNode(void *iterator); +extern void *validateListIterator(struct _ValidateList *list); +extern void *validateListNext(struct _ValidateList *list, void *iterator); + +#endif diff --git a/src/wsbm_driver.h b/src/wsbm_driver.h index d7da44e..b78549b 100644 --- a/src/wsbm_driver.h +++ b/src/wsbm_driver.h @@ -87,9 +87,9 @@ extern struct _WsbmThreadFuncs *wsbmCurThreadFunc; #define WSBM_COND_BROADCAST(_cond) \ (_cond)->func->condBroadcast(_cond); -struct _WsbmVNodeFuncs +struct _WsbmDriver { - struct _ValidateNode *(*alloc) (struct _WsbmVNodeFuncs *, int); + struct _ValidateNode *(*alloc) (const struct _WsbmDriver *); void (*free) (struct _ValidateNode *); void (*clear) (struct _ValidateNode *); }; @@ -109,7 +109,7 @@ struct _ValidateNode uint64_t set_flags; uint64_t clr_flags; void *buf; - struct _WsbmVNodeFuncs *func; + const struct _WsbmDriver *driver; }; static inline struct _WsbmVNodeFuncs * diff --git a/src/wsbm_fencemgr.c b/src/wsbm_fencemgr.c index 0e7a302..6765382 100644 --- a/src/wsbm_fencemgr.c +++ b/src/wsbm_fencemgr.c @@ -37,8 +37,6 @@ #include "wsbm_fencemgr.h" #include "wsbm_pool.h" #include "wsbm_manager.h" -#include <xf86drm.h> -#include <ttm/ttm_fence_user.h> #include <string.h> #include <unistd.h> @@ -142,6 +140,7 @@ wsbmFenceMgrCreate(const struct _WsbmFenceMgrCreateInfo *info) } } wsbmAtomicSet(&tmp->count, 0); + tmp->private = info->private; return tmp; @@ -349,98 +348,6 @@ wsbmFenceCreate(struct _WsbmFenceMgr *mgr, uint32_t fence_class, private_size); } -struct _WsbmTTMFenceMgrPriv -{ - int fd; - unsigned int devOffset; -}; - -static int -tSignaled(struct _WsbmFenceMgr *mgr, void *private, uint32_t flush_type, - uint32_t * signaled_type) -{ - struct _WsbmTTMFenceMgrPriv *priv = - (struct _WsbmTTMFenceMgrPriv *)mgr->private; - union ttm_fence_signaled_arg arg; - int ret; - - arg.req.handle = (unsigned long)private; - arg.req.fence_type = flush_type; - arg.req.flush = 1; - *signaled_type = 0; - - ret = drmCommandWriteRead(priv->fd, priv->devOffset + TTM_FENCE_SIGNALED, - &arg, sizeof(arg)); - if (ret) - return ret; - - *signaled_type = arg.rep.signaled_types; - return 0; -} - -static int -tFinish(struct _WsbmFenceMgr *mgr, void *private, uint32_t fence_type, - int lazy_hint) -{ - struct _WsbmTTMFenceMgrPriv *priv = - (struct _WsbmTTMFenceMgrPriv *)mgr->private; - union ttm_fence_finish_arg arg = - {.req = {.handle = (unsigned long)private, - .fence_type = fence_type, - .mode = (lazy_hint) ? TTM_FENCE_FINISH_MODE_LAZY : 0} - }; - int ret; - - do { - ret = drmCommandWriteRead(priv->fd, priv->devOffset + TTM_FENCE_FINISH, - &arg, sizeof(arg)); - } while (ret == -EAGAIN || ret == -ERESTART); - - return ret; -} - -static int -tUnref(struct _WsbmFenceMgr *mgr, void **private) -{ - struct _WsbmTTMFenceMgrPriv *priv = - (struct _WsbmTTMFenceMgrPriv *)mgr->private; - struct ttm_fence_unref_arg arg = {.handle = (unsigned long)*private }; - - *private = NULL; - - return drmCommandWrite(priv->fd, priv->devOffset + TTM_FENCE_UNREF, - &arg, sizeof(arg)); -} - -struct _WsbmFenceMgr * -wsbmFenceMgrTTMInit(int fd, unsigned int numClass, unsigned int devOffset) -{ - struct _WsbmFenceMgrCreateInfo info; - struct _WsbmFenceMgr *mgr; - struct _WsbmTTMFenceMgrPriv *priv = malloc(sizeof(*priv)); - - if (!priv) - return NULL; - - priv->fd = fd; - priv->devOffset = devOffset; - - info.flags = WSBM_FENCE_CLASS_ORDERED; - info.num_classes = numClass; - info.signaled = tSignaled; - info.finish = tFinish; - info.unreference = tUnref; - - mgr = wsbmFenceMgrCreate(&info); - if (mgr == NULL) { - free(priv); - return NULL; - } - - mgr->private = (void *)priv; - return mgr; -} - void wsbmFenceCmdLock(struct _WsbmFenceMgr *mgr, uint32_t fence_class) { @@ -454,21 +361,24 @@ wsbmFenceCmdUnlock(struct _WsbmFenceMgr *mgr, uint32_t fence_class) } void -wsbmFenceMgrTTMTakedown(struct _WsbmFenceMgr *mgr) +wsbmFenceMgrTakedown(struct _WsbmFenceMgr *mgr) { int i; if (!mgr) return; - if (mgr->private) - free(mgr->private); - for (i = 0; i < mgr->info.num_classes; ++i) { WSBM_MUTEX_FREE(&mgr->classes[i].mutex); WSBM_MUTEX_FREE(&mgr->classes[i].cmd_mutex); } + free(mgr->classes); free(mgr); +} - return; +extern void *wsbmFenceMgrPrivate(struct _WsbmFenceMgr *mgr) +{ + if (!mgr) + return NULL; + return mgr->private; } diff --git a/src/wsbm_fencemgr.h b/src/wsbm_fencemgr.h index cb10ae2..f6dff8f 100644 --- a/src/wsbm_fencemgr.h +++ b/src/wsbm_fencemgr.h @@ -136,6 +136,7 @@ extern uint32_t wsbmFenceType(struct _WsbmFenceObject *fence); struct _WsbmFenceMgrCreateInfo { + void *private; uint32_t flags; uint32_t num_classes; int (*signaled) (struct _WsbmFenceMgr * mgr, void *private, @@ -151,12 +152,6 @@ extern struct _WsbmFenceMgr *wsbmFenceMgrCreate(const struct extern void wsbmFenceCmdLock(struct _WsbmFenceMgr *mgr, uint32_t fence_class); extern void wsbmFenceCmdUnlock(struct _WsbmFenceMgr *mgr, uint32_t fence_class); -/* - * Builtin drivers. - */ - -extern struct _WsbmFenceMgr *wsbmFenceMgrTTMInit(int fd, - unsigned int numClass, - unsigned int devOffset); -extern void wsbmFenceMgrTTMTakedown(struct _WsbmFenceMgr *mgr); +extern void wsbmFenceMgrTakedown(struct _WsbmFenceMgr *mgr); +extern void *wsbmFenceMgrPrivate(struct _WsbmFenceMgr *mgr); #endif diff --git a/src/wsbm_mallocpool.c b/src/wsbm_mallocpool.c index b5252d6..d36aee9 100644 --- a/src/wsbm_mallocpool.c +++ b/src/wsbm_mallocpool.c @@ -54,7 +54,8 @@ mallocBuf(struct _WsbmBufStorage *buf) static struct _WsbmBufStorage * pool_create(struct _WsbmBufferPool *pool, - unsigned long size, uint32_t placement, unsigned alignment) + unsigned long size, uint32_t placement, unsigned alignment, + int share, int pin) { struct _WsbmMallocBuffer *mBuf = malloc(size + sizeof(*mBuf) + 16); @@ -64,8 +65,6 @@ pool_create(struct _WsbmBufferPool *pool, wsbmBufStorageInit(&mBuf->buf, pool); mBuf->size = size; mBuf->mem = (void *)((unsigned long)mBuf + sizeof(*mBuf)); - if ((placement & WSBM_PL_MASK_MEM) != WSBM_PL_FLAG_SYSTEM) - abort(); return &mBuf->buf; } @@ -97,7 +96,7 @@ pool_unmap(struct _WsbmBufStorage *buf) } static int -pool_syncforcpu(struct _WsbmBufStorage *buf, unsigned mode) +pool_syncforcpu(struct _WsbmBufStorage *buf, unsigned mode, int noBlock) { return 0; } @@ -130,7 +129,7 @@ pool_poolOffset(struct _WsbmBufStorage *buf) static uint32_t pool_placement(struct _WsbmBufStorage *buf) { - return WSBM_PL_FLAG_SYSTEM | WSBM_PL_FLAG_CACHED; + return 0; } static unsigned long diff --git a/src/wsbm_manager.c b/src/wsbm_manager.c index e40571d..83a801f 100644 --- a/src/wsbm_manager.c +++ b/src/wsbm_manager.c @@ -53,305 +53,67 @@ #define WSBM_BUFFER_SIMPLE 1 #define WSBM_BUFFER_REF 2 -struct _ValidateList -{ - unsigned numTarget; - unsigned numCurrent; - unsigned numOnList; - unsigned hashSize; - uint32_t hashMask; - int driverData; - struct _WsbmListHead list; - struct _WsbmListHead free; - struct _WsbmListHead *hashTable; -}; - struct _WsbmBufferObject { /* Left to the client to protect this data for now. */ struct _WsbmAtomic refCount; struct _WsbmBufStorage *storage; - - uint32_t placement; - unsigned alignment; - unsigned bufferType; + struct _WsbmAttr attr; struct _WsbmBufferPool *pool; + unsigned bufferType; + int hasAttr; }; -struct _WsbmBufferList -{ - int hasKernelBuffers; - - struct _ValidateList kernelBuffers; /* List of kernel buffers needing validation */ - struct _ValidateList userBuffers; /* List of user-space buffers needing validation */ -}; - -static struct _WsbmMutex bmMutex; -static struct _WsbmCond bmCond; static int initialized = 0; -static void *commonData = NULL; - -static int kernelReaders = 0; -static int kernelLocked = 0; +static struct _WsbmCommon *commonData = NULL; int -wsbmInit(struct _WsbmThreadFuncs *tf, struct _WsbmVNodeFuncs *vf) +wsbmInit(struct _WsbmThreadFuncs *tf) { - int ret; - wsbmCurThreadFunc = tf; - wsbmCurVNodeFunc = vf; - - ret = WSBM_MUTEX_INIT(&bmMutex); - if (ret) - return -ENOMEM; - ret = WSBM_COND_INIT(&bmCond); - if (ret) { - WSBM_MUTEX_FREE(&bmMutex); - return -ENOMEM; - } - initialized = 1; return 0; } void -wsbmCommonDataSet(void *d) +wsbmCommonDataSet(struct _WsbmCommon *common) { - commonData = d; + commonData = common; } -void * +struct _WsbmCommon * wsbmCommonDataGet(void) { + if (commonData == NULL) + return NULL; + ++commonData->refCount; return commonData; } -int -wsbmIsInitialized(void) -{ - return initialized; -} - void -wsbmTakedown(void) +wsbmCommonDataPut(void) { - initialized = 0; - commonData = NULL; - WSBM_COND_FREE(&bmCond); - WSBM_MUTEX_FREE(&bmMutex); -} + if (commonData == NULL) + abort(); -static struct _ValidateNode * -validateListAddNode(struct _ValidateList *list, void *item, - uint32_t hash, uint64_t flags, uint64_t mask) -{ - struct _ValidateNode *node; - struct _WsbmListHead *l; - struct _WsbmListHead *hashHead; - - l = list->free.next; - if (l == &list->free) { - node = wsbmVNodeFuncs()->alloc(wsbmVNodeFuncs(), 0); - if (!node) { - return NULL; - } - list->numCurrent++; - } else { - WSBMLISTDEL(l); - node = WSBMLISTENTRY(l, struct _ValidateNode, head); + if (--commonData->refCount == 0) { + commonData->destroy(commonData); + commonData = NULL; } - node->buf = item; - node->set_flags = flags & mask; - node->clr_flags = (~flags) & mask; - node->listItem = list->numOnList; - WSBMLISTADDTAIL(&node->head, &list->list); - list->numOnList++; - hashHead = list->hashTable + hash; - WSBMLISTADDTAIL(&node->hashHead, hashHead); - - return node; } -static uint32_t -wsbmHashFunc(uint8_t * key, uint32_t len, uint32_t mask) -{ - uint32_t hash, i; - - for (hash = 0, i = 0; i < len; ++i) { - hash += *key++; - hash += (hash << 10); - hash ^= (hash >> 6); - } - - hash += (hash << 3); - hash ^= (hash >> 11); - hash += (hash << 15); - - return hash & mask; -} - -static void -validateFreeList(struct _ValidateList *list) -{ - struct _ValidateNode *node; - struct _WsbmListHead *l; - - l = list->list.next; - while (l != &list->list) { - WSBMLISTDEL(l); - node = WSBMLISTENTRY(l, struct _ValidateNode, head); - - WSBMLISTDEL(&node->hashHead); - node->func->free(node); - l = list->list.next; - list->numCurrent--; - list->numOnList--; - } - - l = list->free.next; - while (l != &list->free) { - WSBMLISTDEL(l); - node = WSBMLISTENTRY(l, struct _ValidateNode, head); - - node->func->free(node); - l = list->free.next; - list->numCurrent--; - } - free(list->hashTable); -} - -static int -validateListAdjustNodes(struct _ValidateList *list) -{ - struct _ValidateNode *node; - struct _WsbmListHead *l; - int ret = 0; - - while (list->numCurrent < list->numTarget) { - node = wsbmVNodeFuncs()->alloc(wsbmVNodeFuncs(), list->driverData); - if (!node) { - ret = -ENOMEM; - break; - } - list->numCurrent++; - WSBMLISTADD(&node->head, &list->free); - } - - while (list->numCurrent > list->numTarget) { - l = list->free.next; - if (l == &list->free) - break; - WSBMLISTDEL(l); - node = WSBMLISTENTRY(l, struct _ValidateNode, head); - - node->func->free(node); - list->numCurrent--; - } - return ret; -} - -static inline int -wsbmPot(unsigned int val) -{ - unsigned int shift = 0; - while(val > (1 << shift)) - shift++; - - return shift; -} - - - -static int -validateCreateList(int numTarget, struct _ValidateList *list, int driverData) -{ - int i; - unsigned int shift = wsbmPot(numTarget); - int ret; - - list->hashSize = (1 << shift); - list->hashMask = list->hashSize - 1; - - list->hashTable = malloc(list->hashSize * sizeof(*list->hashTable)); - if (!list->hashTable) - return -ENOMEM; - - for (i = 0; i < list->hashSize; ++i) - WSBMINITLISTHEAD(&list->hashTable[i]); - - WSBMINITLISTHEAD(&list->list); - WSBMINITLISTHEAD(&list->free); - list->numTarget = numTarget; - list->numCurrent = 0; - list->numOnList = 0; - list->driverData = driverData; - ret = validateListAdjustNodes(list); - if (ret != 0) - free(list->hashTable); - - return ret; -} - -static int -validateResetList(struct _ValidateList *list) -{ - struct _WsbmListHead *l; - struct _ValidateNode *node; - int ret; - - ret = validateListAdjustNodes(list); - if (ret) - return ret; - - l = list->list.next; - while (l != &list->list) { - WSBMLISTDEL(l); - node = WSBMLISTENTRY(l, struct _ValidateNode, head); - - WSBMLISTDEL(&node->hashHead); - WSBMLISTADD(l, &list->free); - list->numOnList--; - l = list->list.next; - } - return validateListAdjustNodes(list); -} - -void -wsbmWriteLockKernelBO(void) -{ - WSBM_MUTEX_LOCK(&bmMutex); - while (kernelReaders != 0) - WSBM_COND_WAIT(&bmCond, &bmMutex); - kernelLocked = 1; -} - -void -wsbmWriteUnlockKernelBO(void) -{ - kernelLocked = 0; - WSBM_MUTEX_UNLOCK(&bmMutex); -} - -void -wsbmReadLockKernelBO(void) +int +wsbmIsInitialized(void) { - WSBM_MUTEX_LOCK(&bmMutex); - if (kernelReaders++ == 0) - kernelLocked = 1; - WSBM_MUTEX_UNLOCK(&bmMutex); + return initialized; } void -wsbmReadUnlockKernelBO(void) +wsbmTakedown(void) { - WSBM_MUTEX_LOCK(&bmMutex); - if (--kernelReaders == 0) { - kernelLocked = 0; - WSBM_COND_BROADCAST(&bmCond); - } - WSBM_MUTEX_UNLOCK(&bmMutex); + initialized = 0; + commonData = NULL; } void @@ -387,11 +149,12 @@ wsbmBOUnmap(struct _WsbmBufferObject *buf) } int -wsbmBOSyncForCpu(struct _WsbmBufferObject *buf, unsigned mode) +wsbmBOSyncForCpu(struct _WsbmBufferObject *buf, unsigned mode, + int noBlock) { struct _WsbmBufStorage *storage = buf->storage; - return storage->pool->syncforcpu(storage, mode); + return storage->pool->syncforcpu(storage, mode, noBlock); } void @@ -440,18 +203,23 @@ wsbmBOReference(struct _WsbmBufferObject *buf) } int -wsbmBOSetStatus(struct _WsbmBufferObject *buf, - uint32_t setFlags, uint32_t clrFlags) +wsbmBOSetAttr(struct _WsbmBufferObject *buf, + const struct _WsbmAttr *attr) { struct _WsbmBufStorage *storage = buf->storage; if (!storage) return 0; - if (storage->pool->setStatus == NULL) + if (storage->pool->setAttr == NULL) return -EINVAL; - return storage->pool->setStatus(storage, setFlags, clrFlags); + buf->attr = *attr; + return storage->pool->setAttr(storage, + attr->setPlacement, + attr->alignment, + attr->share, + attr->pin); } void @@ -477,18 +245,30 @@ wsbmBOUnreference(struct _WsbmBufferObject **p_buf) } } +static inline int +wsbmAttrDiff(const struct _WsbmAttr *attr1, + const struct _WsbmAttr *attr2) +{ + return (attr1->setPlacement != attr2->setPlacement) || + (attr1->clrPlacement != attr2->clrPlacement) || + (attr1->alignment != attr2->alignment) || + (attr1->share != attr2->share) || + (attr1->pin != attr2->pin); +} + int wsbmBOData(struct _WsbmBufferObject *buf, unsigned size, const void *data, - struct _WsbmBufferPool *newPool, uint32_t placement) + struct _WsbmBufferPool *newPool, + const struct _WsbmAttr *attr) { void *virtual = NULL; int newBuffer; int retval = 0; struct _WsbmBufStorage *storage; int synced = 0; - uint32_t placement_diff; struct _WsbmBufferPool *curPool; + const struct _WsbmAttr *tmpAttr = attr; if (buf->bufferType == WSBM_BUFFER_SIMPLE) return -EINVAL; @@ -506,8 +286,12 @@ wsbmBOData(struct _WsbmBufferObject *buf, storage->pool->size(storage) > size + WSBM_BODATA_SIZE_ACCEPT); - if (!placement) - placement = buf->placement; + if (!tmpAttr) { + if (buf->hasAttr) + tmpAttr = &buf->attr; + else + return -EINVAL; + } if (newBuffer) { if (buf->bufferType == WSBM_BUFFER_REF) @@ -517,23 +301,27 @@ wsbmBOData(struct _WsbmBufferObject *buf, if (size == 0) { buf->pool = newPool; - buf->placement = placement; + buf->attr = *tmpAttr; + buf->hasAttr = 1; retval = 0; goto out; } buf->storage = - newPool->create(newPool, size, placement, buf->alignment); + newPool->create(newPool, size, tmpAttr->setPlacement, + tmpAttr->alignment, + tmpAttr->share, + tmpAttr->pin); if (!buf->storage) { retval = -ENOMEM; goto out; } - buf->placement = placement; + buf->attr = *tmpAttr; + buf->hasAttr = 1; buf->pool = newPool; } else if (wsbmAtomicRead(&storage->onList) || - 0 != storage->pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE | - WSBM_SYNCCPU_DONT_BLOCK)) { + 0 != storage->pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE, 1)) { /* * Buffer is busy. need to create a new one. */ @@ -543,14 +331,17 @@ wsbmBOData(struct _WsbmBufferObject *buf, curPool = storage->pool; tmp_storage = - curPool->create(curPool, size, placement, buf->alignment); + curPool->create(curPool, size, tmpAttr->setPlacement, + tmpAttr->alignment, + tmpAttr->share, + tmpAttr->pin); if (tmp_storage) { wsbmBufStorageUnref(&buf->storage); buf->storage = tmp_storage; - buf->placement = placement; + buf->attr = *tmpAttr; } else { - retval = curPool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); + retval = curPool->syncforcpu(storage, WSBM_SYNCCPU_WRITE, 0); if (retval) goto out; synced = 1; @@ -558,8 +349,6 @@ wsbmBOData(struct _WsbmBufferObject *buf, } else synced = 1; - placement_diff = placement ^ buf->placement; - /* * We might need to change buffer placement. */ @@ -567,21 +356,24 @@ wsbmBOData(struct _WsbmBufferObject *buf, storage = buf->storage; curPool = storage->pool; - if (placement_diff) { - assert(curPool->setStatus != NULL); - curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); - retval = curPool->setStatus(storage, - placement_diff & placement, - placement_diff & ~placement); + if (wsbmAttrDiff(&buf->attr, tmpAttr)) { + assert(curPool->setAttr != NULL); + if (synced) { + curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + synced = 0; + } + retval = curPool->setAttr(storage, + tmpAttr->setPlacement, + tmpAttr->alignment, + tmpAttr->share, + tmpAttr->pin); if (retval) goto out; - - buf->placement = placement; - + buf->attr = *tmpAttr; } - if (!synced) { - retval = curPool->syncforcpu(buf->storage, WSBM_SYNCCPU_WRITE); + if (!synced && data) { + retval = curPool->syncforcpu(buf->storage, WSBM_SYNCCPU_WRITE, 0); if (retval) goto out; @@ -612,9 +404,13 @@ wsbmStorageClone(struct _WsbmBufferObject *buf) { struct _WsbmBufStorage *storage = buf->storage; struct _WsbmBufferPool *pool = storage->pool; + struct _WsbmAttr *attr = &buf->attr; - return pool->create(pool, pool->size(storage), buf->placement, - buf->alignment); + return pool->create(pool, pool->size(storage), + attr->setPlacement, + attr->alignment, + attr->share, + attr->pin); } struct _WsbmBufferObject * @@ -643,7 +439,7 @@ wsbmBOClone(struct _WsbmBufferObject *buf, void *virtual; void *nVirtual; - ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ); + ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ, 0); if (ret) goto out_err1; ret = pool->map(storage, WSBM_ACCESS_READ, &virtual); @@ -687,7 +483,7 @@ wsbmBOSubData(struct _WsbmBufferObject *buf, struct _WsbmBufStorage *storage = buf->storage; struct _WsbmBufferPool *pool = storage->pool; - ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); + ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE, 0); if (ret) goto out; @@ -720,7 +516,7 @@ wsbmBOSubData(struct _WsbmBufferObject *buf, pool = storage->pool; } - ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); + ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE, 0); if (ret) goto out; } @@ -750,7 +546,7 @@ wsbmBOGetSubData(struct _WsbmBufferObject *buf, struct _WsbmBufStorage *storage = buf->storage; struct _WsbmBufferPool *pool = storage->pool; - ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ); + ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ, 0); if (ret) goto out; ret = pool->map(storage, WSBM_ACCESS_READ, &virtual); @@ -795,8 +591,8 @@ wsbmBOFreeSimple(void *ptr) struct _WsbmBufferObject * wsbmBOCreateSimple(struct _WsbmBufferPool *pool, unsigned long size, - uint32_t placement, - unsigned alignment, size_t extra_size, size_t * offset) + const struct _WsbmAttr *attr, + size_t extra_size, size_t * offset) { struct _WsbmBufferObject *buf; struct _WsbmBufStorage *storage; @@ -811,7 +607,11 @@ wsbmBOCreateSimple(struct _WsbmBufferPool *pool, if (!buf) return NULL; - storage = pool->create(pool, size, placement, alignment); + storage = pool->create(pool, size, + attr->setPlacement, + attr->alignment, + attr->share, + attr->pin); if (!storage) goto out_err0; @@ -819,10 +619,10 @@ wsbmBOCreateSimple(struct _WsbmBufferPool *pool, storage->destroyArg = buf; buf->storage = storage; - buf->alignment = alignment; buf->pool = pool; - buf->placement = placement; buf->bufferType = WSBM_BUFFER_SIMPLE; + buf->attr = *attr; + buf->hasAttr = 1; return buf; @@ -835,22 +635,21 @@ int wsbmGenBuffers(struct _WsbmBufferPool *pool, unsigned n, struct _WsbmBufferObject *buffers[], - unsigned alignment, uint32_t placement) + const struct _WsbmAttr *attr) { struct _WsbmBufferObject *buf; int i; - placement = (placement) ? placement : - WSBM_PL_FLAG_SYSTEM | WSBM_PL_FLAG_CACHED; - for (i = 0; i < n; ++i) { buf = (struct _WsbmBufferObject *)calloc(1, sizeof(*buf)); if (!buf) return -ENOMEM; wsbmAtomicSet(&buf->refCount, 1); - buf->placement = placement; - buf->alignment = alignment; + if (attr) { + buf->hasAttr = 1; + memcpy(&buf->attr, attr, sizeof(*attr)); + } buf->pool = pool; buf->bufferType = WSBM_BUFFER_COMPLEX; buffers[i] = buf; @@ -868,155 +667,6 @@ wsbmDeleteBuffers(unsigned n, struct _WsbmBufferObject *buffers[]) } } -/* - * Note that lists are per-context and don't need mutex protection. - */ - -struct _WsbmBufferList * -wsbmBOCreateList(int target, int hasKernelBuffers) -{ - struct _WsbmBufferList *list = calloc(sizeof(*list), 1); - int ret; - - list->hasKernelBuffers = hasKernelBuffers; - if (hasKernelBuffers) { - ret = validateCreateList(target, &list->kernelBuffers, 0); - if (ret) - return NULL; - } - - ret = validateCreateList(target, &list->userBuffers, 1); - if (ret) { - validateFreeList(&list->kernelBuffers); - return NULL; - } - - return list; -} - -int -wsbmBOResetList(struct _WsbmBufferList *list) -{ - int ret; - - if (list->hasKernelBuffers) { - ret = validateResetList(&list->kernelBuffers); - if (ret) - return ret; - } - ret = validateResetList(&list->userBuffers); - return ret; -} - -void -wsbmBOFreeList(struct _WsbmBufferList *list) -{ - if (list->hasKernelBuffers) - validateFreeList(&list->kernelBuffers); - validateFreeList(&list->userBuffers); - free(list); -} - -static int -wsbmAddValidateItem(struct _ValidateList *list, void *buf, uint64_t flags, - uint64_t mask, int *itemLoc, - struct _ValidateNode **pnode, int *newItem) -{ - struct _ValidateNode *node, *cur; - struct _WsbmListHead *l; - struct _WsbmListHead *hashHead; - uint32_t hash; - uint32_t count = 0; - uint32_t key = (unsigned long) buf; - - cur = NULL; - hash = wsbmHashFunc((uint8_t *) &key, 4, list->hashMask); - hashHead = list->hashTable + hash; - *newItem = 0; - - for (l = hashHead->next; l != hashHead; l = l->next) { - count++; - node = WSBMLISTENTRY(l, struct _ValidateNode, hashHead); - - if (node->buf == buf) { - cur = node; - break; - } - } - - if (!cur) { - cur = validateListAddNode(list, buf, hash, flags, mask); - if (!cur) - return -ENOMEM; - *newItem = 1; - cur->func->clear(cur); - } else { - uint64_t set_flags = flags & mask; - uint64_t clr_flags = (~flags) & mask; - - if (((cur->clr_flags | clr_flags) & WSBM_PL_MASK_MEM) == - WSBM_PL_MASK_MEM) { - /* - * No available memory type left. Bail. - */ - return -EINVAL; - } - - if ((cur->set_flags | set_flags) & - (cur->clr_flags | clr_flags) & ~WSBM_PL_MASK_MEM) { - /* - * Conflicting flags. Bail. - */ - return -EINVAL; - } - - cur->set_flags &= ~(clr_flags & WSBM_PL_MASK_MEM); - cur->set_flags |= (set_flags & ~WSBM_PL_MASK_MEM); - cur->clr_flags |= clr_flags; - } - *itemLoc = cur->listItem; - if (pnode) - *pnode = cur; - return 0; -} - -int -wsbmBOAddListItem(struct _WsbmBufferList *list, - struct _WsbmBufferObject *buf, - uint64_t flags, uint64_t mask, int *itemLoc, - struct _ValidateNode **node) -{ - int newItem; - struct _WsbmBufStorage *storage = buf->storage; - int ret; - int dummy; - struct _ValidateNode *dummyNode; - - if (list->hasKernelBuffers) { - ret = wsbmAddValidateItem(&list->kernelBuffers, - storage->pool->kernel(storage), - flags, mask, itemLoc, node, &dummy); - if (ret) - goto out_unlock; - } else { - *node = NULL; - *itemLoc = -1000; - } - - ret = wsbmAddValidateItem(&list->userBuffers, storage, - flags, mask, &dummy, &dummyNode, &newItem); - if (ret) - goto out_unlock; - - if (newItem) { - wsbmAtomicInc(&storage->refCount); - wsbmAtomicInc(&storage->onList); - } - - out_unlock: - return ret; -} - void wsbmBOFence(struct _WsbmBufferObject *buf, struct _WsbmFenceObject *fence) { @@ -1036,104 +686,6 @@ wsbmBOOnList(const struct _WsbmBufferObject *buf) return wsbmAtomicRead(&buf->storage->onList); } -int -wsbmBOUnrefUserList(struct _WsbmBufferList *list) -{ - struct _WsbmBufStorage *storage; - void *curBuf; - - curBuf = validateListIterator(&list->userBuffers); - - while (curBuf) { - storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf); - wsbmAtomicDec(&storage->onList); - wsbmBufStorageUnref(&storage); - curBuf = validateListNext(&list->userBuffers, curBuf); - } - - return wsbmBOResetList(list); -} - - -int -wsbmBOFenceUserList(struct _WsbmBufferList *list, - struct _WsbmFenceObject *fence) -{ - struct _WsbmBufStorage *storage; - void *curBuf; - - curBuf = validateListIterator(&list->userBuffers); - - /* - * User-space fencing callbacks. - */ - - while (curBuf) { - storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf); - - storage->pool->fence(storage, fence); - wsbmAtomicDec(&storage->onList); - wsbmBufStorageUnref(&storage); - curBuf = validateListNext(&list->userBuffers, curBuf); - } - - return wsbmBOResetList(list); -} - -int -wsbmBOValidateUserList(struct _WsbmBufferList *list) -{ - void *curBuf; - struct _WsbmBufStorage *storage; - struct _ValidateNode *node; - int ret; - - curBuf = validateListIterator(&list->userBuffers); - - /* - * User-space validation callbacks. - */ - - while (curBuf) { - node = validateListNode(curBuf); - storage = (struct _WsbmBufStorage *)node->buf; - if (storage->pool->validate) { - ret = storage->pool->validate(storage, node->set_flags, - node->clr_flags); - if (ret) - return ret; - } - curBuf = validateListNext(&list->userBuffers, curBuf); - } - return 0; -} - -int -wsbmBOUnvalidateUserList(struct _WsbmBufferList *list) -{ - void *curBuf; - struct _WsbmBufStorage *storage; - struct _ValidateNode *node; - - curBuf = validateListIterator(&list->userBuffers); - - /* - * User-space validation callbacks. - */ - - while (curBuf) { - node = validateListNode(curBuf); - storage = (struct _WsbmBufStorage *)node->buf; - if (storage->pool->unvalidate) { - storage->pool->unvalidate(storage); - } - wsbmAtomicDec(&storage->onList); - wsbmBufStorageUnref(&storage); - curBuf = validateListNext(&list->userBuffers, curBuf); - } - return wsbmBOResetList(list); -} - void wsbmPoolTakeDown(struct _WsbmBufferPool *pool) { @@ -1154,56 +706,7 @@ wsbmBOSize(struct _WsbmBufferObject *buf) } -struct _ValidateList * -wsbmGetKernelValidateList(struct _WsbmBufferList *list) -{ - return (list->hasKernelBuffers) ? &list->kernelBuffers : NULL; -} - -struct _ValidateList * -wsbmGetUserValidateList(struct _WsbmBufferList *list) -{ - return &list->userBuffers; -} - -struct _ValidateNode * -validateListNode(void *iterator) -{ - struct _WsbmListHead *l = (struct _WsbmListHead *)iterator; - - return WSBMLISTENTRY(l, struct _ValidateNode, head); -} - -void * -validateListIterator(struct _ValidateList *list) -{ - void *ret = list->list.next; - - if (ret == &list->list) - return NULL; - return ret; -} - -void * -validateListNext(struct _ValidateList *list, void *iterator) -{ - void *ret; - - struct _WsbmListHead *l = (struct _WsbmListHead *)iterator; - - ret = l->next; - if (ret == &list->list) - return NULL; - return ret; -} - -uint32_t -wsbmKBufHandle(const struct _WsbmKernelBuf * kBuf) -{ - return kBuf->handle; -} - -extern void +void wsbmUpdateKBuf(struct _WsbmKernelBuf *kBuf, uint64_t gpuOffset, uint32_t placement, uint32_t fence_type_mask) @@ -1213,10 +716,16 @@ wsbmUpdateKBuf(struct _WsbmKernelBuf *kBuf, kBuf->fence_type_mask = fence_type_mask; } -extern struct _WsbmKernelBuf * +struct _WsbmKernelBuf * wsbmKBuf(const struct _WsbmBufferObject *buf) { struct _WsbmBufStorage *storage = buf->storage; return storage->pool->kernel(storage); } + +struct _WsbmBufStorage * +wsbmBOStorage(struct _WsbmBufferObject *buf) +{ + return buf->storage; +} diff --git a/src/wsbm_manager.h b/src/wsbm_manager.h index ee0636f..9f0f8af 100644 --- a/src/wsbm_manager.h +++ b/src/wsbm_manager.h @@ -44,40 +44,47 @@ struct _WsbmFenceObject; struct _WsbmBufferObject; struct _WsbmBufferPool; -struct _WsbmBufferList; -/* - * These flags mimics the TTM closely, but since - * this library is not dependant on TTM, we need to - * replicate them here, and if there is a discrepancy, - * that needs to be resolved in the buffer pool using - * the TTM flags. - */ - -#define WSBM_PL_MASK_MEM 0x0000FFFF - -#define WSBM_PL_FLAG_SYSTEM (1 << 0) -#define WSBM_PL_FLAG_TT (1 << 1) -#define WSBM_PL_FLAG_VRAM (1 << 2) -#define WSBM_PL_FLAG_PRIV0 (1 << 3) -#define WSBM_PL_FLAG_SWAPPED (1 << 15) -#define WSBM_PL_FLAG_CACHED (1 << 16) -#define WSBM_PL_FLAG_UNCACHED (1 << 17) -#define WSBM_PL_FLAG_WC (1 << 18) -#define WSBM_PL_FLAG_SHARED (1 << 20) -#define WSBM_PL_FLAG_NO_EVICT (1 << 21) #define WSBM_ACCESS_READ (1 << 0) #define WSBM_ACCESS_WRITE (1 << 1) #define WSBM_SYNCCPU_READ WSBM_ACCESS_READ #define WSBM_SYNCCPU_WRITE WSBM_ACCESS_WRITE -#define WSBM_SYNCCPU_DONT_BLOCK (1 << 2) -#define WSBM_SYNCCPU_TRY_CACHED (1 << 3) + +struct _WsbmAttr { + uint32_t setPlacement; + uint32_t clrPlacement; + unsigned int alignment; + int share; + int pin; +}; + +struct _WsbmCommon { + uint32_t refCount; + void (*destroy) (struct _WsbmCommon *); +}; + +static inline const struct _WsbmAttr * +wsbmInitAttr(struct _WsbmAttr *attr, + uint32_t setPlacement, + uint32_t clrPlacement, + unsigned int alignment, + int share, + int pin) +{ + attr->setPlacement = setPlacement; + attr->clrPlacement = clrPlacement; + attr->alignment = alignment; + attr->share = share; + attr->pin = pin; + return attr; +} extern void *wsbmBOMap(struct _WsbmBufferObject *buf, unsigned mode); extern void wsbmBOUnmap(struct _WsbmBufferObject *buf); -extern int wsbmBOSyncForCpu(struct _WsbmBufferObject *buf, unsigned mode); +extern int wsbmBOSyncForCpu(struct _WsbmBufferObject *buf, unsigned mode, + int noBlock); extern void wsbmBOReleaseFromCpu(struct _WsbmBufferObject *buf, unsigned mode); @@ -91,9 +98,10 @@ extern void wsbmBOUnreference(struct _WsbmBufferObject **p_buf); extern int wsbmBOData(struct _WsbmBufferObject *r_buf, unsigned size, const void *data, - struct _WsbmBufferPool *pool, uint32_t placement); -extern int wsbmBOSetStatus(struct _WsbmBufferObject *buf, - uint32_t setPlacement, uint32_t clrPlacement); + struct _WsbmBufferPool *pool, + const struct _WsbmAttr *attr); +extern int wsbmBOSetAttr(struct _WsbmBufferObject *buf, + const struct _WsbmAttr *attr); extern int wsbmBOSubData(struct _WsbmBufferObject *buf, unsigned long offset, unsigned long size, const void *data, @@ -113,32 +121,16 @@ extern int wsbmBOGetSubData(struct _WsbmBufferObject *buf, extern int wsbmGenBuffers(struct _WsbmBufferPool *pool, unsigned n, struct _WsbmBufferObject *buffers[], - unsigned alignment, uint32_t placement); + const struct _WsbmAttr *attr); struct _WsbmBufferObject *wsbmBOCreateSimple(struct _WsbmBufferPool *pool, unsigned long size, - uint32_t placement, - unsigned alignment, + const struct _WsbmAttr *attr, size_t extra_size, size_t * offset); extern void wsbmDeleteBuffers(unsigned n, struct _WsbmBufferObject *buffers[]); -extern struct _WsbmBufferList *wsbmBOCreateList(int target, - int hasKernelBuffers); -extern int wsbmBOResetList(struct _WsbmBufferList *list); -extern int wsbmBOAddListItem(struct _WsbmBufferList *list, - struct _WsbmBufferObject *buf, - uint64_t flags, uint64_t mask, int *itemLoc, - struct _ValidateNode **node); - -extern void wsbmBOFreeList(struct _WsbmBufferList *list); -extern int wsbmBOFenceUserList(struct _WsbmBufferList *list, - struct _WsbmFenceObject *fence); - -extern int wsbmBOUnrefUserList(struct _WsbmBufferList *list); -extern int wsbmBOValidateUserList(struct _WsbmBufferList *list); -extern int wsbmBOUnvalidateUserList(struct _WsbmBufferList *list); extern void wsbmBOFence(struct _WsbmBufferObject *buf, struct _WsbmFenceObject *fence); @@ -152,26 +144,12 @@ extern int wsbmBOOnList(const struct _WsbmBufferObject *buf); extern void wsbmPoolTakeDown(struct _WsbmBufferPool *pool); -extern void wsbmReadLockKernelBO(void); -extern void wsbmReadUnlockKernelBO(void); -extern void wsbmWriteLockKernelBO(void); -extern void wsbmWriteUnlockKernelBO(void); - -extern int wsbmInit(struct _WsbmThreadFuncs *tf, struct _WsbmVNodeFuncs *vf); +extern int wsbmInit(struct _WsbmThreadFuncs *tf); extern void wsbmTakedown(void); extern int wsbmIsInitialized(void); -extern void wsbmCommonDataSet(void *d); -extern void *wsbmCommonDataGet(void); - -extern struct _ValidateList *wsbmGetKernelValidateList(struct _WsbmBufferList - *list); -extern struct _ValidateList *wsbmGetUserValidateList(struct _WsbmBufferList - *list); - -extern struct _ValidateNode *validateListNode(void *iterator); -extern void *validateListIterator(struct _ValidateList *list); -extern void *validateListNext(struct _ValidateList *list, void *iterator); - +extern void wsbmCommonDataSet(struct _WsbmCommon *common); +extern struct _WsbmCommon *wsbmCommonDataGet(void); +extern void wsbmCommonDataPut(void); extern uint32_t wsbmKBufHandle(const struct _WsbmKernelBuf *); extern void wsbmUpdateKBuf(struct _WsbmKernelBuf *, uint64_t gpuOffset, diff --git a/src/wsbm_pool.h b/src/wsbm_pool.h index 1d07523..0563a99 100644 --- a/src/wsbm_pool.h +++ b/src/wsbm_pool.h @@ -50,6 +50,8 @@ struct _WsbmBufStorage void (*destroyContainer) (void *); }; + + struct _WsbmKernelBuf; struct _WsbmBufferPool @@ -57,18 +59,21 @@ struct _WsbmBufferPool int fd; int (*map) (struct _WsbmBufStorage * buf, unsigned mode, void **virtual); void (*unmap) (struct _WsbmBufStorage * buf); - int (*syncforcpu) (struct _WsbmBufStorage * buf, unsigned mode); + int (*syncforcpu) (struct _WsbmBufStorage * buf, unsigned mode, + int noBlock); void (*releasefromcpu) (struct _WsbmBufStorage * buf, unsigned mode); void (*destroy) (struct _WsbmBufStorage ** buf); unsigned long (*offset) (struct _WsbmBufStorage * buf); unsigned long (*poolOffset) (struct _WsbmBufStorage * buf); - uint32_t(*placement) (struct _WsbmBufStorage * buf); + uint32_t(*placement) (struct _WsbmBufStorage * buf); unsigned long (*size) (struct _WsbmBufStorage * buf); struct _WsbmKernelBuf *(*kernel) (struct _WsbmBufStorage * buf); struct _WsbmBufStorage *(*create) (struct _WsbmBufferPool * pool, unsigned long size, uint32_t placement, - unsigned alignment); + unsigned alignment, + int share, + int pin); struct _WsbmBufStorage *(*createByReference) (struct _WsbmBufferPool * pool, uint32_t handle); void (*fence) (struct _WsbmBufStorage * buf, @@ -77,8 +82,11 @@ struct _WsbmBufferPool int (*validate) (struct _WsbmBufStorage * buf, uint64_t set_flags, uint64_t clr_flags); int (*waitIdle) (struct _WsbmBufStorage * buf, int lazy); - int (*setStatus) (struct _WsbmBufStorage * buf, - uint32_t set_placement, uint32_t clr_placement); + int (*setAttr) (struct _WsbmBufStorage * buf, + uint32_t placement, + unsigned alignment, + int share, + int pin); void (*takeDown) (struct _WsbmBufferPool * pool); }; @@ -124,24 +132,19 @@ wsbmBufStorageUnref(struct _WsbmBufStorage **pStorage) * Builtin pools. */ -/* - * Kernel buffer objects. Size in multiples of page size. Page size aligned. - */ - -extern struct _WsbmBufferPool *wsbmTTMPoolInit(int fd, - unsigned int devOffset); extern struct _WsbmBufferPool *wsbmMallocPoolInit(void); +struct _WsbmAttr; struct _WsbmSlabCache; -extern struct _WsbmBufferPool *wsbmSlabPoolInit(int fd, uint32_t devOffset, - uint32_t placement, - uint32_t validMask, +extern struct _WsbmBufferPool *wsbmSlabPoolInit(int fd, + struct _WsbmBufferPool *basePool, + const struct _WsbmAttr *attr, uint32_t smallestSize, uint32_t numSizes, uint32_t desiredNumBuffers, uint32_t maxSlabSize, - uint32_t pageAlignment, struct _WsbmSlabCache *cache); + extern struct _WsbmSlabCache *wsbmSlabCacheInit(uint32_t checkIntervalMsec, uint32_t slabTimeoutMsec); extern void wsbmSlabCacheFinish(struct _WsbmSlabCache *cache); @@ -152,6 +155,9 @@ extern struct _WsbmBufferPool *wsbmUserPoolInit(void *vramAddr, void *agpAddr, unsigned long agpStart, unsigned long agpSize, + uint32_t system_flag, + uint32_t vram_flag, + uint32_t agp_flag, uint32_t(*fenceTypes) (uint64_t set_flags)); diff --git a/src/wsbm_slabpool.c b/src/wsbm_slabpool.c index 99f8d22..018d70d 100644 --- a/src/wsbm_slabpool.c +++ b/src/wsbm_slabpool.c @@ -34,7 +34,6 @@ #include "config.h" #endif -#include <ttm/ttm_placement_user.h> #include <stdint.h> #include <sys/time.h> #include <errno.h> @@ -42,23 +41,12 @@ #include <assert.h> #include <string.h> #include <sys/mman.h> -#include <xf86drm.h> #include "wsbm_pool.h" #include "wsbm_fencemgr.h" #include "wsbm_priv.h" #include "wsbm_manager.h" #define WSBM_SLABPOOL_ALLOC_RETRIES 100 -#define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \ - do { \ - (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \ - } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ - -#define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \ - do { \ - (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \ - } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ - #ifdef DEBUG_FENCESIGNALED static int createbuffer = 0; @@ -69,7 +57,6 @@ struct _WsbmSlab; struct _WsbmSlabBuffer { - struct _WsbmKernelBuf kBuf; struct _WsbmBufStorage storage; struct _WsbmCond event; @@ -78,11 +65,11 @@ struct _WsbmSlabBuffer */ int isSlabBuffer; + struct _WsbmBufStorage *bStorage; struct _WsbmSlab *parent; uint32_t start; void *virtual; unsigned long requestedSize; - uint64_t mapHandle; /* * Protected by struct _WsbmSlabSizeHeader::mutex @@ -108,21 +95,20 @@ struct _WsbmSlabKernelBO * Constant at creation */ - struct _WsbmKernelBuf kBuf; + struct _WsbmBufStorage *bStorage; uint32_t pageAlignment; void *virtual; - unsigned long actualSize; - uint64_t mapHandle; /* * Protected by struct _WsbmSlabCache::mutex */ struct _WsbmSlabPool *slabPool; - uint32_t proposedPlacement; + struct _WsbmAttr attr; struct _WsbmListHead timeoutHead; struct _WsbmListHead head; struct timeval timeFreed; + unsigned long actual_size; }; struct _WsbmSlab @@ -169,16 +155,15 @@ struct _WsbmSlabCache struct _WsbmSlabPool { struct _WsbmBufferPool pool; + struct _WsbmBufferPool *basePool; /* * The data of this structure remains constant after * initialization and thus needs no mutex protection. */ - unsigned int devOffset; struct _WsbmSlabCache *cache; - uint32_t proposedPlacement; - uint32_t validMask; + struct _WsbmAttr attr; uint32_t *bucketSizes; uint32_t numBuckets; uint32_t pageSize; @@ -234,19 +219,10 @@ wsbmTimeAdd(struct timeval *arg, struct timeval *add) static void wsbmFreeKernelBO(struct _WsbmSlabKernelBO *kbo) { - struct ttm_pl_reference_req arg; - struct _WsbmSlabPool *slabPool; - if (!kbo) return; - slabPool = kbo->slabPool; - memset(&arg, 0, sizeof(arg)); - arg.handle = kbo->kBuf.handle; - (void)munmap(kbo->virtual, kbo->actualSize); - (void)drmCommandWrite(slabPool->pool.fd, - slabPool->devOffset + TTM_PL_UNREF, &arg, - sizeof(arg)); + kbo->bStorage->pool->destroy(&kbo->bStorage); free(kbo); } @@ -293,17 +269,24 @@ wsbmSetKernelBOFree(struct _WsbmSlabCache *cache, wsbmTimeAdd(&timeFreed, &cache->slabTimeout); kbo->timeFreed = timeFreed; - if (kbo->kBuf.placement & TTM_PL_FLAG_CACHED) - WSBMLISTADD(&kbo->head, &cache->cached); - else - WSBMLISTADD(&kbo->head, &cache->unCached); - + WSBMLISTADD(&kbo->head, &cache->unCached); WSBMLISTADDTAIL(&kbo->timeoutHead, &cache->timeoutList); wsbmFreeTimeoutKBOsLocked(cache, &time); WSBM_MUTEX_UNLOCK(&cache->mutex); } +static inline int +wsbmAttrDiff(const struct _WsbmAttr *attr1, + const struct _WsbmAttr *attr2) +{ + return (attr1->setPlacement != attr2->setPlacement) || + (attr1->clrPlacement != attr2->clrPlacement) || + (attr1->alignment != attr2->alignment) || + (attr1->share != attr2->share) || + (attr1->pin != attr2->pin); +} + /* * Get a _SlabKernelBO for us to use as storage for a slab. */ @@ -317,7 +300,7 @@ wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header) uint32_t size = header->bufSize * slabPool->desiredNumBuffers; struct _WsbmSlabKernelBO *kbo; struct _WsbmSlabKernelBO *kboTmp; - int ret; + int ret = 0; /* * FIXME: We should perhaps allow some variation in slabsize in order @@ -333,13 +316,12 @@ wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header) kbo = NULL; retry: - head = (slabPool->proposedPlacement & TTM_PL_FLAG_CACHED) ? - &cache->cached : &cache->unCached; + head = &cache->unCached; WSBMLISTFOREACH(list, head) { kboTmp = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, head); - if ((kboTmp->actualSize == size) && + if ((kboTmp->actual_size == size) && (slabPool->pageAlignment == 0 || (kboTmp->pageAlignment != 0 && (kboTmp->pageAlignment % slabPool->pageAlignment) == 0))) { @@ -347,9 +329,8 @@ wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header) if (!kbo) kbo = kboTmp; - if ((kbo->proposedPlacement ^ slabPool->proposedPlacement) == 0) + if (!wsbmAttrDiff(&kbo->attr, &slabPool->attr)) break; - } } @@ -361,27 +342,15 @@ wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header) WSBM_MUTEX_UNLOCK(&cache->mutex); if (kbo) { - uint32_t new_mask = - kbo->proposedPlacement ^ slabPool->proposedPlacement; - - ret = 0; - if (new_mask) { - union ttm_pl_setstatus_arg arg; - struct ttm_pl_setstatus_req *req = &arg.req; - struct ttm_pl_rep *rep = &arg.rep; - - memset(&arg, 0, sizeof(arg)); - req->handle = kbo->kBuf.handle; - req->set_placement = slabPool->proposedPlacement & new_mask; - req->clr_placement = ~slabPool->proposedPlacement & new_mask; - DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd, - slabPool->devOffset + TTM_PL_SETSTATUS, - arg, ret); - if (ret == 0) { - kbo->kBuf.gpuOffset = rep->gpu_offset; - kbo->kBuf.placement = rep->placement; - } - kbo->proposedPlacement = slabPool->proposedPlacement; + if (wsbmAttrDiff(&kbo->attr, &slabPool->attr)) { + struct _WsbmBufStorage *storage = kbo->bStorage; + struct _WsbmAttr *attr = &slabPool->attr; + ret = storage->pool->setAttr(storage, + attr->setPlacement, + attr->alignment, + attr->share, + attr->pin); + kbo->attr = slabPool->attr; } if (ret == 0) @@ -397,53 +366,37 @@ wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header) return NULL; { - union ttm_pl_create_arg arg; + size_t actual_size = + (size + slabPool->pageSize - 1) & ~(slabPool->pageSize - 1); + const struct _WsbmAttr *attr = &slabPool->attr; kbo->slabPool = slabPool; WSBMINITLISTHEAD(&kbo->head); WSBMINITLISTHEAD(&kbo->timeoutHead); - - memset(&arg, 0, sizeof(arg)); - arg.req.size = size; - arg.req.placement = slabPool->proposedPlacement; - arg.req.page_alignment = slabPool->pageAlignment; - - DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd, - slabPool->devOffset + TTM_PL_CREATE, - arg, ret); + kbo->bStorage = slabPool->basePool->create(slabPool->basePool, + actual_size, + attr->setPlacement, + attr->alignment, + attr->share, + attr->pin); if (ret) goto out_err0; - kbo->kBuf.gpuOffset = arg.rep.gpu_offset; - kbo->kBuf.placement = arg.rep.placement; - kbo->kBuf.handle = arg.rep.handle; - - kbo->actualSize = arg.rep.bo_size; - kbo->mapHandle = arg.rep.map_handle; - kbo->proposedPlacement = slabPool->proposedPlacement; + kbo->attr = slabPool->attr; + kbo->actual_size = actual_size; } - kbo->virtual = mmap(0, kbo->actualSize, - PROT_READ | PROT_WRITE, MAP_SHARED, - slabPool->pool.fd, kbo->mapHandle); + ret = kbo->bStorage->pool->map(kbo->bStorage, + WSBM_ACCESS_READ | + WSBM_ACCESS_WRITE, &kbo->virtual); - if (kbo->virtual == MAP_FAILED) { - ret = -errno; + if (ret) goto out_err1; - } return kbo; out_err1: - { - struct ttm_pl_reference_req arg; - - memset(&arg, 0, sizeof(arg)); - arg.handle = kbo->kBuf.handle; - (void)drmCommandWrite(slabPool->pool.fd, - slabPool->devOffset + TTM_PL_UNREF, - &arg, sizeof(arg)); - } + kbo->bStorage->pool->destroy(&kbo->bStorage); out_err0: free(kbo); return NULL; @@ -468,7 +421,7 @@ wsbmAllocSlab(struct _WsbmSlabSizeHeader *header) goto out_err0; } - numBuffers = slab->kbo->actualSize / header->bufSize; + numBuffers = slab->kbo->actual_size / header->bufSize; slab->buffers = calloc(numBuffers, sizeof(*slab->buffers)); if (!slab->buffers) { @@ -697,7 +650,8 @@ wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader *header) static struct _WsbmBufStorage * pool_create(struct _WsbmBufferPool *pool, unsigned long size, - uint32_t placement, unsigned alignment) + uint32_t placement, unsigned alignment, + int share, int pin) { struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool); struct _WsbmSlabSizeHeader *header; @@ -749,44 +703,31 @@ pool_create(struct _WsbmBufferPool *pool, unsigned long size, goto out_err1; { - union ttm_pl_create_arg arg; - - memset(&arg, 0, sizeof(arg)); - arg.req.placement = placement; - arg.req.page_alignment = alignment / slabPool->pageSize; - - DRMRESTARTCOMMANDWRITEREAD(pool->fd, - slabPool->devOffset + TTM_PL_CREATE, - arg, ret); - - if (ret) + sBuf->bStorage = slabPool->basePool->create(slabPool->basePool, + size, + placement, + alignment, + share, + pin); + + if (!sBuf->bStorage) { + ret = -ENOMEM; goto out_err2; + } - sBuf->kBuf.gpuOffset = arg.rep.gpu_offset; - sBuf->kBuf.placement = arg.rep.placement; - sBuf->kBuf.handle = arg.rep.handle; - sBuf->mapHandle = arg.rep.map_handle; sBuf->requestedSize = size; + ret = sBuf->bStorage->pool->map(sBuf->bStorage, + WSBM_ACCESS_READ | + WSBM_ACCESS_WRITE, &sBuf->virtual); - sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, - pool->fd, sBuf->mapHandle); - - if (sBuf->virtual == MAP_FAILED) + if (ret != 0) goto out_err3; } wsbmAtomicSet(&sBuf->writers, 0); return &sBuf->storage; out_err3: - { - struct ttm_pl_reference_req arg; - - memset(&arg, 0, sizeof(arg)); - arg.handle = sBuf->kBuf.handle; - (void)drmCommandWriteRead(pool->fd, - slabPool->devOffset + TTM_PL_UNREF, - &arg, sizeof(arg)); - } + sBuf->bStorage->pool->destroy(&sBuf->bStorage); out_err2: WSBM_COND_FREE(&sBuf->event); out_err1: @@ -807,20 +748,12 @@ pool_destroy(struct _WsbmBufStorage **p_buf) *p_buf = NULL; if (!sBuf->isSlabBuffer) { - struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf); - struct ttm_pl_reference_req arg; - if (sBuf->virtual != NULL) { - (void)munmap(sBuf->virtual, sBuf->requestedSize); + sBuf->bStorage->pool->unmap(sBuf->bStorage); sBuf->virtual = NULL; } - memset(&arg, 0, sizeof(arg)); - arg.handle = sBuf->kBuf.handle; - (void)drmCommandWrite(slabPool->pool.fd, - slabPool->devOffset + TTM_PL_UNREF, - &arg, sizeof(arg)); - + sBuf->bStorage->pool->destroy(&sBuf->bStorage); WSBM_COND_FREE(&sBuf->event); wsbmBufStorageTakedown(&sBuf->storage); free(sBuf); @@ -912,13 +845,14 @@ pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode) } static int -pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode) +pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode, + int noBlock) { struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); int ret = 0; WSBM_MUTEX_LOCK(&buf->mutex); - if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) { + if (noBlock) { int signaled; if (sBuf->unFenced) { @@ -977,8 +911,9 @@ static struct _WsbmKernelBuf * pool_kernel(struct _WsbmBufStorage *buf) { struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); - - return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf; + struct _WsbmBufStorage *bStorage = + (sBuf->isSlabBuffer) ? sBuf->parent->kbo->bStorage : sBuf->bStorage; + return bStorage->pool->kernel(bStorage); } static unsigned long @@ -1146,14 +1081,13 @@ pool_takedown(struct _WsbmBufferPool *pool) struct _WsbmBufferPool * wsbmSlabPoolInit(int fd, - uint32_t devOffset, - uint32_t placement, - uint32_t validMask, + struct _WsbmBufferPool *basePool, + const struct _WsbmAttr *attr, uint32_t smallestSize, uint32_t numSizes, uint32_t desiredNumBuffers, uint32_t maxSlabSize, - uint32_t pageAlignment, struct _WsbmSlabCache *cache) + struct _WsbmSlabCache *cache) { struct _WsbmBufferPool *pool; struct _WsbmSlabPool *slabPool; @@ -1173,13 +1107,24 @@ wsbmSlabPoolInit(int fd, if (!slabPool->headers) goto out_err1; - slabPool->devOffset = devOffset; + slabPool->basePool = basePool; slabPool->cache = cache; - slabPool->proposedPlacement = placement; - slabPool->validMask = validMask; + slabPool->attr = *attr; slabPool->numBuckets = numSizes; slabPool->pageSize = getpagesize(); - slabPool->pageAlignment = pageAlignment; + if (attr->alignment != 0) { + if (attr->alignment < slabPool->pageSize) { + if (slabPool->pageSize % attr->alignment != 0) + goto out_err1; + slabPool->pageAlignment = 1; + } else { + if (attr->alignment % slabPool->pageSize != 0) + goto out_err1; + slabPool->pageAlignment = attr->alignment / slabPool->pageSize; + } + } else + slabPool->pageAlignment = 0; + slabPool->maxSlabSize = maxSlabSize; slabPool->desiredNumBuffers = desiredNumBuffers; diff --git a/src/wsbm_ttmpool.c b/src/wsbm_ttmpool.c deleted file mode 100644 index 52bdb35..0000000 --- a/src/wsbm_ttmpool.c +++ /dev/null @@ -1,518 +0,0 @@ -/************************************************************************** - * - * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, Tx., USA - * All Rights Reserved. - * Copyright 2009 VMware, Inc., Palo Alto, CA., USA - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sub license, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, - * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - * USE OR OTHER DEALINGS IN THE SOFTWARE. - * - **************************************************************************/ -/* - * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com> - */ -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include <xf86drm.h> -#include <stdlib.h> -#include <unistd.h> -#include <errno.h> -#include <string.h> -#include <sys/mman.h> -#include "wsbm_pool.h" -#include "assert.h" -#include "wsbm_priv.h" -#include "wsbm_manager.h" -#include "ttm/ttm_placement_user.h" - -#define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \ - do { \ - (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \ - } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ - -#define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \ - do { \ - (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \ - } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ - -/* - * Buffer pool implementation using DRM buffer objects as wsbm buffer objects. - */ - -struct _TTMBuffer -{ - struct _WsbmBufStorage buf; - struct _WsbmCond event; - - /* - * Remains constant after creation. - */ - - uint64_t requestedSize; - uint64_t mapHandle; - uint64_t realSize; - - /* - * Protected by the kernel lock. - */ - - struct _WsbmKernelBuf kBuf; - - /* - * Protected by the mutex. - */ - - void *virtual; - int syncInProgress; - unsigned readers; - unsigned writers; -}; - -struct _TTMPool -{ - struct _WsbmBufferPool pool; - unsigned int pageSize; - unsigned int devOffset; -}; - -static inline struct _TTMPool * -ttmGetPool(struct _TTMBuffer *dBuf) -{ - return containerOf(dBuf->buf.pool, struct _TTMPool, pool); -} - -static inline struct _TTMBuffer * -ttmBuffer(struct _WsbmBufStorage *buf) -{ - return containerOf(buf, struct _TTMBuffer, buf); -} - -static struct _WsbmBufStorage * -pool_create(struct _WsbmBufferPool *pool, - unsigned long size, uint32_t placement, unsigned alignment) -{ - struct _TTMBuffer *dBuf = (struct _TTMBuffer *) - calloc(1, sizeof(*dBuf)); - struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); - int ret; - unsigned pageSize = ttmPool->pageSize; - union ttm_pl_create_arg arg; - - if (!dBuf) - return NULL; - - if ((alignment > pageSize) && (alignment % pageSize)) - goto out_err0; - - ret = wsbmBufStorageInit(&dBuf->buf, pool); - if (ret) - goto out_err0; - - ret = WSBM_COND_INIT(&dBuf->event); - if (ret) - goto out_err1; - - memset(&arg, 0, sizeof(arg)); - arg.req.size = size; - arg.req.placement = placement; - arg.req.page_alignment = alignment / pageSize; - - DRMRESTARTCOMMANDWRITEREAD(pool->fd, ttmPool->devOffset + TTM_PL_CREATE, - arg, ret); - - if (ret) - goto out_err2; - - dBuf->requestedSize = size; - dBuf->kBuf.gpuOffset = arg.rep.gpu_offset; - dBuf->mapHandle = arg.rep.map_handle; - dBuf->realSize = arg.rep.bo_size; - dBuf->kBuf.placement = arg.rep.placement; - dBuf->kBuf.handle = arg.rep.handle; - - return &dBuf->buf; - - out_err2: - WSBM_COND_FREE(&dBuf->event); - out_err1: - wsbmBufStorageTakedown(&dBuf->buf); - out_err0: - free(dBuf); - return NULL; -} - -static struct _WsbmBufStorage * -pool_reference(struct _WsbmBufferPool *pool, unsigned handle) -{ - struct _TTMBuffer *dBuf = (struct _TTMBuffer *)calloc(1, sizeof(*dBuf)); - struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); - union ttm_pl_reference_arg arg; - int ret; - - if (!dBuf) - return NULL; - - ret = wsbmBufStorageInit(&dBuf->buf, pool); - if (ret) - goto out_err0; - - ret = WSBM_COND_INIT(&dBuf->event); - if (ret) - goto out_err1; - - memset(&arg, 0, sizeof(arg)); - arg.req.handle = handle; - ret = drmCommandWriteRead(pool->fd, ttmPool->devOffset + TTM_PL_REFERENCE, - &arg, sizeof(arg)); - - if (ret) - goto out_err2; - - dBuf->requestedSize = arg.rep.bo_size; - dBuf->kBuf.gpuOffset = arg.rep.gpu_offset; - dBuf->mapHandle = arg.rep.map_handle; - dBuf->realSize = arg.rep.bo_size; - dBuf->kBuf.placement = arg.rep.placement; - dBuf->kBuf.handle = arg.rep.handle; - dBuf->kBuf.fence_type_mask = arg.rep.sync_object_arg; - - return &dBuf->buf; - - out_err2: - WSBM_COND_FREE(&dBuf->event); - out_err1: - wsbmBufStorageTakedown(&dBuf->buf); - out_err0: - free(dBuf); - return NULL; -} - -static void -pool_destroy(struct _WsbmBufStorage **buf) -{ - struct _TTMBuffer *dBuf = ttmBuffer(*buf); - struct _TTMPool *ttmPool = ttmGetPool(dBuf); - struct ttm_pl_reference_req arg; - - *buf = NULL; - if (dBuf->virtual != NULL) { - (void)munmap(dBuf->virtual, dBuf->requestedSize); - dBuf->virtual = NULL; - } - - memset(&arg, 0, sizeof(arg)); - arg.handle = dBuf->kBuf.handle; - (void)drmCommandWrite(dBuf->buf.pool->fd, - ttmPool->devOffset + TTM_PL_UNREF, - &arg, sizeof(arg)); - - WSBM_COND_FREE(&dBuf->event); - wsbmBufStorageTakedown(&dBuf->buf); - free(dBuf); -} - -static int -syncforcpu_locked(struct _WsbmBufStorage *buf, unsigned mode) -{ - uint32_t kmode = 0; - struct _TTMBuffer *dBuf = ttmBuffer(buf); - struct _TTMPool *ttmPool = ttmGetPool(dBuf); - unsigned int readers; - unsigned int writers; - int ret = 0; - - while (dBuf->syncInProgress) - WSBM_COND_WAIT(&dBuf->event, &buf->mutex); - - readers = dBuf->readers; - writers = dBuf->writers; - - if ((mode & WSBM_SYNCCPU_READ) && (++dBuf->readers == 1)) - kmode |= TTM_PL_SYNCCPU_MODE_READ; - - if ((mode & WSBM_SYNCCPU_WRITE) && (++dBuf->writers == 1)) - kmode |= TTM_PL_SYNCCPU_MODE_WRITE; - - if (kmode) { - struct ttm_pl_synccpu_arg arg; - - if (mode & WSBM_SYNCCPU_DONT_BLOCK) - kmode |= TTM_PL_SYNCCPU_MODE_NO_BLOCK; - - dBuf->syncInProgress = 1; - - /* - * This might be a lengthy wait, so - * release the mutex. - */ - - WSBM_MUTEX_UNLOCK(&buf->mutex); - - memset(&arg, 0, sizeof(arg)); - arg.handle = dBuf->kBuf.handle; - arg.access_mode = kmode; - arg.op = TTM_PL_SYNCCPU_OP_GRAB; - - DRMRESTARTCOMMANDWRITE(dBuf->buf.pool->fd, - ttmPool->devOffset + TTM_PL_SYNCCPU, arg, ret); - - WSBM_MUTEX_LOCK(&buf->mutex); - dBuf->syncInProgress = 0; - WSBM_COND_BROADCAST(&dBuf->event); - - if (ret) { - dBuf->readers = readers; - dBuf->writers = writers; - } - } - - return ret; -} - -static int -releasefromcpu_locked(struct _WsbmBufStorage *buf, unsigned mode) -{ - uint32_t kmode = 0; - struct _TTMBuffer *dBuf = ttmBuffer(buf); - struct _TTMPool *ttmPool = ttmGetPool(dBuf); - int ret = 0; - - while (dBuf->syncInProgress) - WSBM_COND_WAIT(&dBuf->event, &buf->mutex); - - if ((mode & WSBM_SYNCCPU_READ) && (--dBuf->readers == 0)) - kmode |= TTM_PL_SYNCCPU_MODE_READ; - - if ((mode & WSBM_SYNCCPU_WRITE) && (--dBuf->writers == 0)) - kmode |= TTM_PL_SYNCCPU_MODE_WRITE; - - if (kmode) { - struct ttm_pl_synccpu_arg arg; - - memset(&arg, 0, sizeof(arg)); - arg.handle = dBuf->kBuf.handle; - arg.access_mode = kmode; - arg.op = TTM_PL_SYNCCPU_OP_RELEASE; - - DRMRESTARTCOMMANDWRITE(dBuf->buf.pool->fd, - ttmPool->devOffset + TTM_PL_SYNCCPU, arg, ret); - - } - - return ret; -} - -static int -pool_syncforcpu(struct _WsbmBufStorage *buf, unsigned mode) -{ - int ret; - - WSBM_MUTEX_LOCK(&buf->mutex); - ret = syncforcpu_locked(buf, mode); - WSBM_MUTEX_UNLOCK(&buf->mutex); - return ret; -} - -static void -pool_releasefromcpu(struct _WsbmBufStorage *buf, unsigned mode) -{ - WSBM_MUTEX_LOCK(&buf->mutex); - (void)releasefromcpu_locked(buf, mode); - WSBM_MUTEX_UNLOCK(&buf->mutex); -} - -static int -pool_map(struct _WsbmBufStorage *buf, unsigned mode, void **virtual) -{ - struct _TTMBuffer *dBuf = ttmBuffer(buf); - void *virt; - int ret = 0; - - WSBM_MUTEX_LOCK(&buf->mutex); - - /* - * mmaps are expensive, so we only really unmap if - * we destroy the buffer. - */ - - if (dBuf->virtual == NULL) { - virt = mmap(0, dBuf->requestedSize, - PROT_READ | PROT_WRITE, MAP_SHARED, - buf->pool->fd, dBuf->mapHandle); - if (virt == MAP_FAILED) { - ret = -errno; - goto out_unlock; - } - dBuf->virtual = virt; - } - - *virtual = dBuf->virtual; - out_unlock: - - WSBM_MUTEX_UNLOCK(&buf->mutex); - - return ret; -} - -static void -pool_unmap(struct _WsbmBufStorage *buf) -{ - ; -} - -static unsigned long -pool_offset(struct _WsbmBufStorage *buf) -{ - struct _TTMBuffer *dBuf = ttmBuffer(buf); - - return dBuf->kBuf.gpuOffset; -} - -static unsigned long -pool_poolOffset(struct _WsbmBufStorage *buf) -{ - return 0; -} - -static uint32_t -pool_placement(struct _WsbmBufStorage *buf) -{ - struct _TTMBuffer *dBuf = ttmBuffer(buf); - - return dBuf->kBuf.placement; -} - -static unsigned long -pool_size(struct _WsbmBufStorage *buf) -{ - struct _TTMBuffer *dBuf = ttmBuffer(buf); - - return dBuf->realSize; -} - -static void -pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) -{ - /* - * Noop. The kernel handles all fencing. - */ -} - -static int -pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) -{ - struct _TTMBuffer *dBuf = ttmBuffer(buf); - struct _TTMPool *ttmPool = ttmGetPool(dBuf); - struct ttm_pl_waitidle_arg req; - struct _WsbmBufferPool *pool = buf->pool; - int ret; - - memset(&req, 0, sizeof(req)); - req.handle = dBuf->kBuf.handle; - req.mode = (lazy) ? TTM_PL_WAITIDLE_MODE_LAZY : 0; - - DRMRESTARTCOMMANDWRITE(pool->fd, ttmPool->devOffset + TTM_PL_WAITIDLE, - req, ret); - - return ret; -} - -static void -pool_takedown(struct _WsbmBufferPool *pool) -{ - struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); - - free(ttmPool); -} - -static int -pool_setStatus(struct _WsbmBufStorage *buf, uint32_t set_placement, - uint32_t clr_placement) -{ - struct _TTMBuffer *dBuf = ttmBuffer(buf); - struct _TTMPool *ttmPool = ttmGetPool(dBuf); - union ttm_pl_setstatus_arg arg; - struct ttm_pl_setstatus_req *req = &arg.req; - struct ttm_pl_rep *rep = &arg.rep; - struct _WsbmBufferPool *pool = buf->pool; - int ret; - - memset(&arg, 0, sizeof(arg)); - req->handle = dBuf->kBuf.handle; - req->set_placement = set_placement; - req->clr_placement = clr_placement; - - DRMRESTARTCOMMANDWRITEREAD(pool->fd, - ttmPool->devOffset + TTM_PL_SETSTATUS, - arg, ret); - - if (!ret) { - dBuf->kBuf.gpuOffset = rep->gpu_offset; - dBuf->kBuf.placement = rep->placement; - } - - return ret; -} - -static struct _WsbmKernelBuf * -pool_kernel(struct _WsbmBufStorage *buf) -{ - return (void *)&ttmBuffer(buf)->kBuf; -} - -struct _WsbmBufferPool * -wsbmTTMPoolInit(int fd, unsigned int devOffset) -{ - struct _TTMPool *ttmPool; - struct _WsbmBufferPool *pool; - - ttmPool = (struct _TTMPool *)calloc(1, sizeof(*ttmPool)); - - if (!ttmPool) - return NULL; - - ttmPool->pageSize = getpagesize(); - ttmPool->devOffset = devOffset; - pool = &ttmPool->pool; - - pool->fd = fd; - pool->map = &pool_map; - pool->unmap = &pool_unmap; - pool->syncforcpu = &pool_syncforcpu; - pool->releasefromcpu = &pool_releasefromcpu; - pool->destroy = &pool_destroy; - pool->offset = &pool_offset; - pool->poolOffset = &pool_poolOffset; - pool->placement = &pool_placement; - pool->size = &pool_size; - pool->create = &pool_create; - pool->fence = &pool_fence; - pool->kernel = &pool_kernel; - pool->validate = NULL; - pool->unvalidate = NULL; - pool->waitIdle = &pool_waitIdle; - pool->takeDown = &pool_takedown; - pool->createByReference = &pool_reference; - pool->setStatus = &pool_setStatus; - return pool; -} diff --git a/src/wsbm_userpool.c b/src/wsbm_userpool.c index d945129..c07cbc0 100644 --- a/src/wsbm_userpool.c +++ b/src/wsbm_userpool.c @@ -59,6 +59,7 @@ struct _WsbmUserBuffer { struct _WsbmBufStorage buf; struct _WsbmKernelBuf kBuf; + /* Protected by the pool mutex */ @@ -102,7 +103,11 @@ struct _WsbmUserPool struct _WsbmListHead agpLRU; struct _WsbmMM vramMM; struct _WsbmMM agpMM; - uint32_t(*fenceTypes) (uint64_t); + uint32_t(*fenceTypes) (uint64_t); + uint32_t system_flag; + uint32_t vram_flag; + uint32_t agp_flag; + uint32_t mem_mask; }; static inline struct _WsbmUserPool * @@ -187,7 +192,7 @@ evict_lru(struct _WsbmListHead *lru) memcpy(WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem), vBuf->map, vBuf->size); WSBMLISTDELINIT(&vBuf->lru); - vBuf->kBuf.placement = WSBM_PL_FLAG_SYSTEM; + vBuf->kBuf.placement = p->system_flag; vBuf->map = WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem); /* @@ -202,23 +207,16 @@ evict_lru(struct _WsbmListHead *lru) static struct _WsbmBufStorage * pool_create(struct _WsbmBufferPool *pool, - unsigned long size, uint32_t placement, unsigned alignment) + unsigned long size, uint32_t placement, unsigned alignment, + int share, int pin) { struct _WsbmUserPool *p = containerOf(pool, struct _WsbmUserPool, pool); struct _WsbmUserBuffer *vBuf = calloc(1, sizeof(*vBuf)); - int ret; if (!vBuf) return NULL; - ret = wsbmBufStorageInit(&vBuf->buf, pool); - if (ret) - goto out_err0; - - ret = WSBM_COND_INIT(&vBuf->event); - if (ret) - goto out_err1; - + wsbmBufStorageInit(&vBuf->buf, pool); vBuf->sysmem = NULL; vBuf->proposedPlacement = placement; vBuf->size = size; @@ -228,13 +226,13 @@ pool_create(struct _WsbmBufferPool *pool, WSBMINITLISTHEAD(&vBuf->delayed); WSBM_MUTEX_LOCK(&p->mutex); - if (placement & WSBM_PL_FLAG_TT) { + if (placement & p->agp_flag) { vBuf->node = wsbmMMSearchFree(&p->agpMM, size, alignment, 1); if (vBuf->node) vBuf->node = wsbmMMGetBlock(vBuf->node, size, alignment); if (vBuf->node) { - vBuf->kBuf.placement = WSBM_PL_FLAG_TT; + vBuf->kBuf.placement = p->agp_flag; vBuf->kBuf.gpuOffset = p->agpOffset + vBuf->node->start; vBuf->map = (void *)(p->agpMap + vBuf->node->start); WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU); @@ -242,13 +240,13 @@ pool_create(struct _WsbmBufferPool *pool, } } - if (placement & WSBM_PL_FLAG_VRAM) { + if (placement & p->vram_flag) { vBuf->node = wsbmMMSearchFree(&p->vramMM, size, alignment, 1); if (vBuf->node) vBuf->node = wsbmMMGetBlock(vBuf->node, size, alignment); if (vBuf->node) { - vBuf->kBuf.placement = WSBM_PL_FLAG_VRAM; + vBuf->kBuf.placement = p->vram_flag; vBuf->kBuf.gpuOffset = p->vramOffset + vBuf->node->start; vBuf->map = (void *)(p->vramMap + vBuf->node->start); WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU); @@ -256,28 +254,22 @@ pool_create(struct _WsbmBufferPool *pool, } } - if ((placement & WSBM_PL_FLAG_NO_EVICT) - && !(placement & WSBM_PL_FLAG_SYSTEM)) { + if (pin && !(placement & p->system_flag)) { WSBM_MUTEX_UNLOCK(&p->mutex); - goto out_err2; + goto out_err; } vBuf->sysmem = malloc(size + WSBM_USER_ALIGN_ADD); - vBuf->kBuf.placement = WSBM_PL_FLAG_SYSTEM; + vBuf->kBuf.placement = p->system_flag; vBuf->map = WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem); have_mem: WSBM_MUTEX_UNLOCK(&p->mutex); if (vBuf->sysmem != NULL - || (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM))) + || (!(vBuf->kBuf.placement & p->system_flag))) return &vBuf->buf; - - out_err2: - WSBM_COND_FREE(&vBuf->event); - out_err1: - wsbmBufStorageTakedown(&vBuf->buf); - out_err0: + out_err: free(vBuf); return NULL; } @@ -303,8 +295,10 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags, vBuf->proposedPlacement = (vBuf->proposedPlacement | set_flags) & ~clr_flags; - if ((vBuf->proposedPlacement & vBuf->kBuf.placement & WSBM_PL_MASK_MEM) == - vBuf->kBuf.placement) { + if ((vBuf->proposedPlacement & vBuf->kBuf.placement & p->mem_mask) == + vBuf->kBuf.placement && + (vBuf->alignment == 0 || + (vBuf->kBuf.gpuOffset % vBuf->alignment == 0))) { err = 0; goto have_mem; } @@ -314,7 +308,7 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags, * do a sw copy to the other region. */ - if (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM)) { + if (!(vBuf->kBuf.placement & p->system_flag)) { struct _WsbmListHead tmpLRU; WSBMINITLISTHEAD(&tmpLRU); @@ -324,7 +318,7 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags, goto have_mem; } - if (vBuf->proposedPlacement & WSBM_PL_FLAG_TT) { + if (vBuf->proposedPlacement & p->agp_flag) { do { vBuf->node = wsbmMMSearchFree(&p->agpMM, vBuf->size, vBuf->alignment, 1); @@ -333,7 +327,7 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags, wsbmMMGetBlock(vBuf->node, vBuf->size, vBuf->alignment); if (vBuf->node) { - vBuf->kBuf.placement = WSBM_PL_FLAG_TT; + vBuf->kBuf.placement = p->agp_flag; vBuf->kBuf.gpuOffset = p->agpOffset + vBuf->node->start; vBuf->map = (void *)(p->agpMap + vBuf->node->start); memcpy(vBuf->map, WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem), @@ -344,7 +338,7 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags, } while (evict_lru(&p->agpLRU) == 0); } - if (vBuf->proposedPlacement & WSBM_PL_FLAG_VRAM) { + if (vBuf->proposedPlacement & p->vram_flag) { do { vBuf->node = wsbmMMSearchFree(&p->vramMM, vBuf->size, vBuf->alignment, 1); @@ -353,7 +347,7 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags, wsbmMMGetBlock(vBuf->node, vBuf->size, vBuf->alignment); if (!err) { - vBuf->kBuf.placement = WSBM_PL_FLAG_VRAM; + vBuf->kBuf.placement = p->vram_flag; vBuf->kBuf.gpuOffset = p->vramOffset + vBuf->node->start; vBuf->map = (void *)(p->vramMap + vBuf->node->start); memcpy(vBuf->map, WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem), @@ -364,7 +358,7 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags, } while (evict_lru(&p->vramLRU) == 0); } - if (vBuf->proposedPlacement & WSBM_PL_FLAG_SYSTEM) + if (vBuf->proposedPlacement & p->system_flag) goto have_mem; err = -ENOMEM; @@ -377,13 +371,17 @@ pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags, } static int -pool_setStatus(struct _WsbmBufStorage *buf, - uint32_t set_placement, uint32_t clr_placement) +pool_setAttr(struct _WsbmBufStorage *buf, + uint32_t placement, + unsigned alignment, + int share, + int pin) { struct _WsbmUserBuffer *vBuf = userBuf(buf); int ret; - ret = pool_validate(buf, set_placement, clr_placement); + vBuf->alignment = alignment; + ret = pool_validate(buf, placement, ~placement); vBuf->unFenced = 0; return ret; } @@ -413,7 +411,7 @@ release_delayed_buffers(struct _WsbmUserPool *p) WSBMLISTDEL(&vBuf->delayed); WSBMLISTDEL(&vBuf->lru); - if ((vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM) == 0) + if ((vBuf->kBuf.placement & p->system_flag) == 0) wsbmMMPutBlock(vBuf->node); else free(vBuf->sysmem); @@ -451,13 +449,11 @@ pool_destroy(struct _WsbmBufStorage **buf) WSBMLISTDEL(&vBuf->lru); WSBM_MUTEX_UNLOCK(&p->mutex); - if (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM)) + if (!(vBuf->kBuf.placement & p->system_flag)) wsbmMMPutBlock(vBuf->node); else free(vBuf->sysmem); - WSBM_COND_FREE(&vBuf->event); - free(vBuf); return; } @@ -488,13 +484,14 @@ pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode) } static int -pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode) +pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode, + int noBlock) { struct _WsbmUserBuffer *vBuf = userBuf(buf); int ret = 0; WSBM_MUTEX_LOCK(&buf->mutex); - if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) { + if (noBlock) { if (vBuf->unFenced) { ret = -EBUSY; @@ -553,9 +550,9 @@ pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) WSBM_COND_BROADCAST(&vBuf->event); WSBM_MUTEX_LOCK(&p->mutex); - if (vBuf->kBuf.placement & WSBM_PL_FLAG_VRAM) + if (vBuf->kBuf.placement & p->vram_flag) WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU); - else if (vBuf->kBuf.placement & WSBM_PL_FLAG_TT) + else if (vBuf->kBuf.placement & p->agp_flag) WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU); WSBM_MUTEX_UNLOCK(&p->mutex); WSBM_MUTEX_UNLOCK(&buf->mutex); @@ -575,9 +572,9 @@ pool_unvalidate(struct _WsbmBufStorage *buf) vBuf->unFenced = 0; WSBM_COND_BROADCAST(&vBuf->event); WSBM_MUTEX_LOCK(&p->mutex); - if (vBuf->kBuf.placement & WSBM_PL_FLAG_VRAM) + if (vBuf->kBuf.placement & p->vram_flag) WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU); - else if (vBuf->kBuf.placement & WSBM_PL_FLAG_TT) + else if (vBuf->kBuf.placement & p->agp_flag) WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU); WSBM_MUTEX_UNLOCK(&p->mutex); @@ -641,6 +638,9 @@ wsbmUserPoolInit(void *vramAddr, unsigned long vramStart, unsigned long vramSize, void *agpAddr, unsigned long agpStart, unsigned long agpSize, + uint32_t system_flag, + uint32_t vram_flag, + uint32_t agp_flag, uint32_t(*fenceTypes) (uint64_t set_flags)) { struct _WsbmBufferPool *pool; @@ -672,6 +672,10 @@ wsbmUserPoolInit(void *vramAddr, uPool->vramOffset = vramStart; uPool->vramMap = (unsigned long)vramAddr; uPool->fenceTypes = fenceTypes; + uPool->system_flag = system_flag; + uPool->vram_flag = vram_flag; + uPool->agp_flag = agp_flag; + uPool->mem_mask = uPool->system_flag | uPool->vram_flag | uPool->agp_flag; pool = &uPool->pool; pool->map = &pool_map; @@ -687,7 +691,7 @@ wsbmUserPoolInit(void *vramAddr, pool->validate = &pool_validate; pool->waitIdle = &pool_waitIdle; pool->takeDown = &pool_takedown; - pool->setStatus = &pool_setStatus; + pool->setAttr = &pool_setAttr; pool->syncforcpu = &pool_syncForCpu; pool->releasefromcpu = &pool_releaseFromCpu; |