diff options
Diffstat (limited to 'src/wsbm_slabpool_new.c')
| -rw-r--r-- | src/wsbm_slabpool_new.c | 1213 |
1 files changed, 1213 insertions, 0 deletions
diff --git a/src/wsbm_slabpool_new.c b/src/wsbm_slabpool_new.c new file mode 100644 index 0000000..92ffb17 --- /dev/null +++ b/src/wsbm_slabpool_new.c @@ -0,0 +1,1213 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include <ttm/ttm_placement_user.h> +#include <stdint.h> +#include <sys/time.h> +#include <errno.h> +#include <unistd.h> +#include <assert.h> +#include <string.h> +#include <sys/mman.h> +#include <xf86drm.h> +#include "wsbm_pool.h" +#include "wsbm_fencemgr.h" +#include "wsbm_priv.h" +#include "wsbm_manager.h" +#include <stdio.h> + + +#define WSBM_SLABPOOL_ALLOC_RETRIES 100 +#define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \ + do { \ + (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \ + } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ + +#define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \ + do { \ + (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \ + } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ + + +#ifdef DEBUG_FENCESIGNALED +static int createbuffer = 0; +static int fencesignaled = 0; +#endif + +struct _WsbmSlab; + +struct _WsbmSlabBuffer { + struct _WsbmKernelBuf kBuf; + struct _WsbmBufStorage storage; + struct _WsbmCond event; + + /* + * Remains constant after creation. + */ + + int isSlabBuffer; + struct _WsbmSlab *parent; + uint32_t start; + void *virtual; + unsigned long requestedSize; + uint64_t mapHandle; + + /* + * Protected by struct _WsbmSlabSizeHeader::mutex + */ + + struct _WsbmListHead head; + + /* + * Protected by this::mutex + */ + + struct _WsbmFenceObject *fence; + uint32_t fenceType; + struct _WsbmAtomic writers; /* (Only upping) */ + int unFenced; +}; + +struct _WsbmSlabPool; +struct _WsbmSlabKernelBO { + + /* + * Constant at creation + */ + + struct _WsbmKernelBuf kBuf; + uint32_t pageAlignment; + void *virtual; + unsigned long actualSize; + uint64_t mapHandle; + + /* + * Protected by struct _WsbmSlabCache::mutex + */ + + struct _WsbmSlabPool *slabPool; + uint32_t proposedPlacement; + struct _WsbmListHead timeoutHead; + struct _WsbmListHead head; + struct timeval timeFreed; +}; + +struct _WsbmSlab{ + struct _WsbmListHead head; + struct _WsbmListHead freeBuffers; + uint32_t numBuffers; + uint32_t numFree; + struct _WsbmSlabBuffer *buffers; + struct _WsbmSlabSizeHeader *header; + struct _WsbmSlabKernelBO *kbo; +}; + + +struct _WsbmSlabSizeHeader { + /* + * Constant at creation. + */ + struct _WsbmSlabPool *slabPool; + uint32_t bufSize; + + /* + * Protected by this::mutex + */ + + struct _WsbmListHead slabs; + struct _WsbmListHead freeSlabs; + struct _WsbmListHead delayedBuffers; + uint32_t numDelayed; + struct _WsbmMutex mutex; +}; + +struct _WsbmSlabCache { + struct timeval slabTimeout; + struct timeval checkInterval; + struct timeval nextCheck; + struct _WsbmListHead timeoutList; + struct _WsbmListHead unCached; + struct _WsbmListHead cached; + struct _WsbmMutex mutex; +}; + + +struct _WsbmSlabPool { + struct _WsbmBufferPool pool; + + /* + * The data of this structure remains constant after + * initialization and thus needs no mutex protection. + */ + + unsigned int devOffset; + struct _WsbmSlabCache *cache; + uint32_t proposedPlacement; + uint32_t validMask; + uint32_t *bucketSizes; + uint32_t numBuckets; + uint32_t pageSize; + int pageAlignment; + int maxSlabSize; + int desiredNumBuffers; + struct _WsbmSlabSizeHeader *headers; +}; + +static inline struct _WsbmSlabPool * +slabPoolFromPool(struct _WsbmBufferPool *pool) +{ + return containerOf(pool, struct _WsbmSlabPool , pool); +} + +static inline struct _WsbmSlabPool * +slabPoolFromBuf(struct _WsbmSlabBuffer *sBuf) +{ + return slabPoolFromPool(sBuf->storage.pool); +} + +static inline struct _WsbmSlabBuffer * +slabBuffer(struct _WsbmBufStorage *buf) +{ + return containerOf(buf, struct _WsbmSlabBuffer, storage); +} + + +/* + * FIXME: Perhaps arrange timeout slabs in size buckets for fast + * retreival?? + */ + + +static inline int +wsbmTimeAfterEq(struct timeval *arg1, struct timeval *arg2) +{ + return ((arg1->tv_sec > arg2->tv_sec) || + ((arg1->tv_sec == arg2->tv_sec) && + (arg1->tv_usec > arg2->tv_usec))); +} + +static inline void +wsbmTimeAdd(struct timeval *arg, struct timeval *add) +{ + unsigned int sec; + + arg->tv_sec += add->tv_sec; + arg->tv_usec += add->tv_usec; + sec = arg->tv_usec / 1000000; + arg->tv_sec += sec; + arg->tv_usec -= sec*1000000; +} + +#include <stdio.h> + +static void +wsbmFreeKernelBO(struct _WsbmSlabKernelBO *kbo) +{ + struct ttm_pl_reference_req arg; + struct _WsbmSlabPool *slabPool; + + if (!kbo) + return; + + fprintf(stderr, "Free kbo size %d\n", kbo->actualSize); + slabPool = kbo->slabPool; + arg.handle = kbo->kBuf.handle; + (void) munmap(kbo->virtual, kbo->actualSize); + (void) drmCommandWrite(slabPool->pool.fd, slabPool->devOffset + TTM_PL_UNREF, + &arg, sizeof(arg)); + free(kbo); +} + + +static void +wsbmFreeTimeoutKBOsLocked(struct _WsbmSlabCache *cache, + struct timeval *time) +{ + struct _WsbmListHead *list, *next; + struct _WsbmSlabKernelBO *kbo; + + if (!wsbmTimeAfterEq(time, &cache->nextCheck)) + return; + + WSBMLISTFOREACHSAFE(list, next, &cache->timeoutList) { + kbo = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, timeoutHead); + + if (!wsbmTimeAfterEq(time, &kbo->timeFreed)) + break; + + WSBMLISTDELINIT(&kbo->timeoutHead); + WSBMLISTDELINIT(&kbo->head); + wsbmFreeKernelBO(kbo); + } + + cache->nextCheck = *time; + wsbmTimeAdd(&cache->nextCheck, &cache->checkInterval); +} + + +/* + * Add a _SlabKernelBO to the free slab manager. + * This means that it is available for reuse, but if it's not + * reused in a while, it will be freed. + */ + +static void +wsbmSetKernelBOFree(struct _WsbmSlabCache *cache, + struct _WsbmSlabKernelBO *kbo) +{ + struct timeval time; + struct timeval timeFreed; + + gettimeofday(&time, NULL); + timeFreed = time; + WSBM_MUTEX_LOCK(&cache->mutex); + wsbmTimeAdd(&timeFreed, &cache->slabTimeout); + kbo->timeFreed = timeFreed; + + if (kbo->kBuf.placement & TTM_PL_FLAG_CACHED) + WSBMLISTADD(&kbo->head, &cache->cached); + else + WSBMLISTADD(&kbo->head, &cache->unCached); + + WSBMLISTADDTAIL(&kbo->timeoutHead, &cache->timeoutList); + wsbmFreeTimeoutKBOsLocked(cache, &time); + + WSBM_MUTEX_UNLOCK(&cache->mutex); +} + +/* + * Get a _SlabKernelBO for us to use as storage for a slab. + */ + + +static struct _WsbmSlabKernelBO * +wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header) + +{ + struct _WsbmSlabPool *slabPool = header->slabPool; + struct _WsbmSlabCache *cache = slabPool->cache; + struct _WsbmListHead *list, *head; + uint32_t size = header->bufSize * slabPool->desiredNumBuffers; + struct _WsbmSlabKernelBO *kbo; + struct _WsbmSlabKernelBO *kboTmp; + int ret; + + /* + * FIXME: We should perhaps allow some variation in slabsize in order + * to efficiently reuse slabs. + */ + + + size = (size <= slabPool->maxSlabSize) ? size : slabPool->maxSlabSize; + if (size < header->bufSize) + size = header->bufSize; + size = (size + slabPool->pageSize - 1) & ~(slabPool->pageSize - 1); + WSBM_MUTEX_LOCK(&cache->mutex); + + kbo = NULL; + + retry: + head = (slabPool->proposedPlacement & TTM_PL_FLAG_CACHED) ? + &cache->cached : &cache->unCached; + + WSBMLISTFOREACH(list, head) { + kboTmp = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, head); + if ((kboTmp->actualSize == size) && + (slabPool->pageAlignment == 0 || + (kboTmp->pageAlignment % slabPool->pageAlignment) == 0)) { + + if (!kbo) + kbo = kboTmp; + + if ((kbo->proposedPlacement ^ slabPool->proposedPlacement) == 0) + break; + + } + } + + if (kbo) { + WSBMLISTDELINIT(&kbo->head); + WSBMLISTDELINIT(&kbo->timeoutHead); + } + + WSBM_MUTEX_UNLOCK(&cache->mutex); + + if (kbo) { + uint32_t new_mask = kbo->proposedPlacement ^ slabPool->proposedPlacement; + + ret = 0; + if (new_mask) { + union ttm_pl_setstatus_arg arg; + struct ttm_pl_setstatus_req *req = &arg.req; + struct ttm_pl_rep *rep = &arg.rep; + + req->handle = kbo->kBuf.handle; + req->set_placement = slabPool->proposedPlacement & new_mask; + req->clr_placement = ~slabPool->proposedPlacement & new_mask; + DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd, + slabPool->devOffset + TTM_PL_SETSTATUS, + arg, ret); + if (ret == 0) { + kbo->kBuf.gpuOffset = rep->gpu_offset; + kbo->kBuf.placement = rep->placement; + } + kbo->proposedPlacement = slabPool->proposedPlacement; + } + + if (ret == 0) + return kbo; + + wsbmFreeKernelBO(kbo); + kbo = NULL; + goto retry; + } + + fprintf(stderr, "New kbo 0x%08x size %d\n", + slabPool->proposedPlacement, size); + kbo = calloc(1, sizeof(*kbo)); + if (!kbo) + return NULL; + + { + union ttm_pl_create_arg arg; + + kbo->slabPool = slabPool; + WSBMINITLISTHEAD(&kbo->head); + WSBMINITLISTHEAD(&kbo->timeoutHead); + + arg.req.size = size; + arg.req.placement = slabPool->proposedPlacement; + arg.req.page_alignment = slabPool->pageAlignment; + + DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd, + slabPool->devOffset + TTM_PL_CREATE, + arg, ret); + if (ret) + goto out_err0; + + kbo->kBuf.gpuOffset = arg.rep.gpu_offset; + kbo->kBuf.placement = arg.rep.placement; + kbo->kBuf.handle = arg.rep.handle; + + kbo->actualSize = arg.rep.bo_size; + kbo->mapHandle = arg.rep.map_handle; + kbo->proposedPlacement = slabPool->proposedPlacement; + } + + kbo->virtual = mmap(0, kbo->actualSize, + PROT_READ | PROT_WRITE, MAP_SHARED, + slabPool->pool.fd, kbo->mapHandle); + + if (kbo->virtual == MAP_FAILED) { + ret = -errno; + goto out_err1; + } + + return kbo; + + out_err1: + { + struct ttm_pl_reference_req arg = + {.handle = kbo->kBuf.handle}; + + (void) drmCommandWrite(slabPool->pool.fd, + slabPool->devOffset + TTM_PL_UNREF, + &arg, sizeof(arg)); + } + out_err0: + free(kbo); + return NULL; +} + + +static int +wsbmAllocSlab(struct _WsbmSlabSizeHeader *header) +{ + struct _WsbmSlab *slab; + struct _WsbmSlabBuffer *sBuf; + uint32_t numBuffers; + int ret; + int i; + + slab = calloc(1, sizeof(*slab)); + if (!slab) + return -ENOMEM; + + slab->kbo = wsbmAllocKernelBO(header); + if (!slab->kbo) { + ret = -ENOMEM; + goto out_err0; + } + + numBuffers = slab->kbo->actualSize / header->bufSize; + + slab->buffers = calloc(numBuffers, sizeof(*slab->buffers)); + if (!slab->buffers) { + ret = -ENOMEM; + goto out_err1; + } + + WSBMINITLISTHEAD(&slab->head); + WSBMINITLISTHEAD(&slab->freeBuffers); + slab->numBuffers = numBuffers; + slab->numFree = 0; + slab->header = header; + + sBuf = slab->buffers; + for (i=0; i < numBuffers; ++i) { + ret = wsbmBufStorageInit(&sBuf->storage, &header->slabPool->pool); + if (ret) + goto out_err2; + sBuf->parent = slab; + sBuf->start = i* header->bufSize; + sBuf->virtual = (void *) ((uint8_t *) slab->kbo->virtual + + sBuf->start); + wsbmAtomicSet(&sBuf->writers, 0); + sBuf->isSlabBuffer = 1; + WSBM_COND_INIT(&sBuf->event); + WSBMLISTADDTAIL(&sBuf->head, &slab->freeBuffers); + slab->numFree++; + sBuf++; + } + + WSBMLISTADDTAIL(&slab->head, &header->slabs); + + return 0; + + out_err2: + sBuf = slab->buffers; + for (i=0; i < numBuffers; ++i) { + if (sBuf->parent == slab) { + WSBM_COND_FREE(&sBuf->event); + wsbmBufStorageTakedown(&sBuf->storage); + } + sBuf++; + } + out_err1: + wsbmSetKernelBOFree(header->slabPool->cache, slab->kbo); + free(slab->buffers); + out_err0: + free(slab); + return ret; +} + +/* + * Delete a buffer from the slab header delayed list and put + * it on the slab free list. + */ + +static void +wsbmSlabFreeBufferLocked(struct _WsbmSlabBuffer *buf) +{ + struct _WsbmSlab *slab = buf->parent; + struct _WsbmSlabSizeHeader *header = slab->header; + struct _WsbmListHead *list = &buf->head; + + WSBMLISTDEL(list); + WSBMLISTADDTAIL(list, &slab->freeBuffers); + slab->numFree++; + + if (slab->head.next == &slab->head) + WSBMLISTADDTAIL(&slab->head, &header->slabs); + + if (slab->numFree == slab->numBuffers) { + list = &slab->head; + WSBMLISTDEL(list); + WSBMLISTADDTAIL(list, &header->freeSlabs); + } + + if (header->slabs.next == &header->slabs || + slab->numFree != slab->numBuffers) { + + struct _WsbmListHead *next; + struct _WsbmSlabCache *cache = header->slabPool->cache; + + WSBMLISTFOREACHSAFE(list, next, &header->freeSlabs) { + int i; + struct _WsbmSlabBuffer *sBuf; + + slab = WSBMLISTENTRY(list, struct _WsbmSlab, head); + WSBMLISTDELINIT(list); + + sBuf = slab->buffers; + for (i=0; i < slab->numBuffers; ++i) { + if (sBuf->parent == slab) { + WSBM_COND_FREE(&sBuf->event); + wsbmBufStorageTakedown(&sBuf->storage); + } + sBuf++; + } + wsbmSetKernelBOFree(cache, slab->kbo); + free(slab->buffers); + free(slab); + } + } +} + +static void +wsbmSlabCheckFreeLocked(struct _WsbmSlabSizeHeader *header, int wait) +{ + struct _WsbmListHead *list, *prev, *first; + struct _WsbmSlabBuffer *sBuf; + struct _WsbmSlab *slab; + int firstWasSignaled = 1; + int signaled; + int i; + int ret; + + /* + * Rerun the freeing test if the youngest tested buffer + * was signaled, since there might be more idle buffers + * in the delay list. + */ + + while (firstWasSignaled) { + firstWasSignaled = 0; + signaled = 0; + first = header->delayedBuffers.next; + + /* Only examine the oldest 1/3 of delayed buffers: + */ + if (header->numDelayed > 3) { + for (i = 0; i < header->numDelayed; i += 3) { + first = first->next; + } + } + + /* + * No need to take the buffer mutex for each buffer we loop + * through since we're currently the only user. + */ + + + WSBMLISTFOREACHPREVSAFE(list, prev, first->next) { + + if (list == &header->delayedBuffers) + break; + + sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); + slab = sBuf->parent; + + if (!signaled) { + if (wait) { + ret = wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0); + if (ret) + break; + signaled = 1; + wait = 0; + } else { + signaled = wsbmFenceSignaled(sBuf->fence, sBuf->fenceType); +#ifdef DEBUG_FENCESIGNALED + fencesignaled++; +#endif + } + if (signaled) { + if (list == first) + firstWasSignaled = 1; + wsbmFenceUnReference(&sBuf->fence); + header->numDelayed--; + wsbmSlabFreeBufferLocked(sBuf); + } else + break; + } else if (wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) { + wsbmFenceUnReference(&sBuf->fence); + header->numDelayed--; + wsbmSlabFreeBufferLocked(sBuf); + } + } + } +} + + +static struct _WsbmSlabBuffer * +wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader *header) +{ + static struct _WsbmSlabBuffer *buf; + struct _WsbmSlab *slab; + struct _WsbmListHead *list; + int count = WSBM_SLABPOOL_ALLOC_RETRIES; + + WSBM_MUTEX_LOCK(&header->mutex); + while(header->slabs.next == &header->slabs && count > 0) { + wsbmSlabCheckFreeLocked(header, 0); + if (header->slabs.next != &header->slabs) + break; + + WSBM_MUTEX_UNLOCK(&header->mutex); + if (count != WSBM_SLABPOOL_ALLOC_RETRIES) + usleep(1000); + WSBM_MUTEX_LOCK(&header->mutex); + (void) wsbmAllocSlab(header); + count--; + } + + list = header->slabs.next; + if (list == &header->slabs) { + WSBM_MUTEX_UNLOCK(&header->mutex); + return NULL; + } + slab = WSBMLISTENTRY(list, struct _WsbmSlab, head); + if (--slab->numFree == 0) + WSBMLISTDELINIT(list); + + list = slab->freeBuffers.next; + WSBMLISTDELINIT(list); + + WSBM_MUTEX_UNLOCK(&header->mutex); + buf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); + +#ifdef DEBUG_FENCESIGNALED + createbuffer++; + if ((createbuffer % 1000) == 0) + _mesa_printf("Createbuffer %d fencesignaled %d\n", + createbuffer, fencesignaled); +#endif + return buf; +} + +static struct _WsbmBufStorage * +pool_create(struct _WsbmBufferPool *pool, unsigned long size, + uint32_t placement, unsigned alignment) +{ + struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool); + struct _WsbmSlabSizeHeader *header; + struct _WsbmSlabBuffer *sBuf; + int i; + int ret; + + /* + * FIXME: Check for compatibility. + */ + + header = slabPool->headers; + for (i=0; i<slabPool->numBuckets; ++i) { + if (header->bufSize >= size) + break; + header++; + } + + if (i < slabPool->numBuckets) { + sBuf = wsbmSlabAllocBuffer(header); + return ((sBuf) ? &sBuf->storage : NULL); + } + + + /* + * Fall back to allocate a buffer object directly from DRM. + * and wrap it in a wsbmBO structure. + */ + + sBuf = calloc(1, sizeof(*sBuf)); + + if (!sBuf) + return NULL; + + if (alignment) { + if ((alignment < slabPool->pageSize) && (slabPool->pageSize % alignment)) + goto out_err0; + if ((alignment > slabPool->pageSize) && (alignment % slabPool->pageSize)) + goto out_err0; + } + + ret = wsbmBufStorageInit(&sBuf->storage, &header->slabPool->pool); + if (ret) + goto out_err0; + + ret = WSBM_COND_INIT(&sBuf->event); + if (ret) + goto out_err1; + + { + union ttm_pl_create_arg arg; + + arg.req.size = size; + arg.req.placement = placement; + arg.req.page_alignment = alignment / slabPool->pageSize; + + DRMRESTARTCOMMANDWRITEREAD(pool->fd, + slabPool->devOffset + TTM_PL_CREATE, + arg, ret); + + if (ret) + goto out_err2; + + sBuf->kBuf.gpuOffset = arg.rep.gpu_offset; + sBuf->kBuf.placement = arg.rep.placement; + sBuf->kBuf.handle = arg.rep.handle; + sBuf->mapHandle = arg.rep.map_handle; + sBuf->requestedSize = size; + + sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, + pool->fd, sBuf->mapHandle); + + if (sBuf->virtual == MAP_FAILED) + goto out_err3; + + } + + wsbmAtomicSet(&sBuf->writers, 0); + return &sBuf->storage; + out_err3: + { + struct ttm_pl_reference_req arg; + + arg.handle = sBuf->kBuf.handle; + (void) drmCommandWriteRead(pool->fd, + slabPool->devOffset + TTM_PL_UNREF, + &arg, sizeof(arg)); + } + out_err2: + WSBM_COND_FREE(&sBuf->event); + out_err1: + wsbmBufStorageTakedown(&sBuf->storage); + out_err0: + free(sBuf); + return NULL; +} + +static void +pool_destroy(struct _WsbmBufStorage **p_buf) +{ + struct _WsbmBufStorage *buf = *p_buf; + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + struct _WsbmSlab *slab; + struct _WsbmSlabSizeHeader *header; + + *p_buf = NULL; + + if (!sBuf->isSlabBuffer) { + struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf); + struct ttm_pl_reference_req arg; + + if (sBuf->virtual != NULL) { + (void) munmap(sBuf->virtual, sBuf->requestedSize); + sBuf->virtual = NULL; + } + + arg.handle = sBuf->kBuf.handle; + (void) drmCommandWrite(slabPool->pool.fd, + slabPool->devOffset + TTM_PL_UNREF, + &arg, sizeof(arg)); + + WSBM_COND_FREE(&sBuf->event); + wsbmBufStorageTakedown(&sBuf->storage); + free(sBuf); + return; + } + + slab = sBuf->parent; + header = slab->header; + + /* + * No need to take the buffer mutex below since we're the only user. + */ + + WSBM_MUTEX_LOCK(&header->mutex); + sBuf->unFenced = 0; + wsbmAtomicSet(&sBuf->writers, 0); + wsbmAtomicSet(&sBuf->storage.refCount, 1); + + if (sBuf->fence && !wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) { + WSBMLISTADDTAIL(&sBuf->head, &header->delayedBuffers); + header->numDelayed++; + } else { + if (sBuf->fence) + wsbmFenceUnReference(&sBuf->fence); + wsbmSlabFreeBufferLocked(sBuf); + } + + WSBM_MUTEX_UNLOCK(&header->mutex); +} + + +static void +waitIdleLocked(struct _WsbmSlabBuffer *sBuf, int lazy) +{ + struct _WsbmBufStorage *storage = &sBuf->storage; + + while(sBuf->unFenced || sBuf->fence != NULL) { + + if (sBuf->unFenced) + WSBM_COND_WAIT(&sBuf->event, &storage->mutex); + + if (sBuf->fence != NULL) { + if (!wsbmFenceSignaled(sBuf->fence, sBuf->fenceType)) { + struct _WsbmFenceObject *fence = + wsbmFenceReference(sBuf->fence); + + WSBM_MUTEX_UNLOCK(&storage->mutex); + (void) wsbmFenceFinish(fence, sBuf->fenceType, lazy); + WSBM_MUTEX_LOCK(&storage->mutex); + if (sBuf->fence == fence) + wsbmFenceUnReference(&sBuf->fence); + + wsbmFenceUnReference(&fence); + } else { + wsbmFenceUnReference(&sBuf->fence); + } + } + } +} + +static int +pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + WSBM_MUTEX_LOCK(&buf->mutex); + waitIdleLocked(sBuf, lazy); + WSBM_MUTEX_UNLOCK(&buf->mutex); + + return 0; +} + +static int +pool_map(struct _WsbmBufStorage *buf, unsigned mode, void **virtual) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + *virtual = sBuf->virtual; + + return 0; +} + +static void +pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + if (wsbmAtomicDecZero(&sBuf->writers)) + WSBM_COND_BROADCAST(&sBuf->event); +} + +static int +pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + int ret = 0; + + WSBM_MUTEX_LOCK(&buf->mutex); + if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) { + int signaled; + + if (sBuf->unFenced) { + ret = -EBUSY; + goto out_unlock; + } + + if (sBuf->isSlabBuffer) + signaled = (sBuf->fence == NULL) || + wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType); + else + signaled = (sBuf->fence == NULL) || + wsbmFenceSignaled(sBuf->fence, sBuf->fenceType); + + ret = 0; + if (signaled) { + wsbmFenceUnReference(&sBuf->fence); + wsbmAtomicInc(&sBuf->writers); + } else + ret = -EBUSY; + goto out_unlock; + } + waitIdleLocked(sBuf, 0); + wsbmAtomicInc(&sBuf->writers); + out_unlock: + WSBM_MUTEX_UNLOCK(&buf->mutex); + return ret; +} + +static void +pool_unmap(struct _WsbmBufStorage *buf) +{ + ; +} + +static unsigned long +pool_poolOffset(struct _WsbmBufStorage *buf) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + return sBuf->start; +} + +static unsigned long +pool_size(struct _WsbmBufStorage *buf) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + if (!sBuf->isSlabBuffer) + return sBuf->requestedSize; + + return sBuf->parent->header->bufSize; +} + +static struct _WsbmKernelBuf * +pool_kernel(struct _WsbmBufStorage *buf) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf; +} + + +static unsigned long +pool_offset(struct _WsbmBufStorage *buf) +{ + return pool_kernel(buf)->gpuOffset + pool_poolOffset(buf); +} + + +static void +pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + struct _WsbmKernelBuf *kBuf; + + WSBM_MUTEX_LOCK(&buf->mutex); + if (sBuf->fence) + wsbmFenceUnReference(&sBuf->fence); + + sBuf->fence = wsbmFenceReference(fence); + kBuf = pool_kernel(buf); + sBuf->fenceType = kBuf->fence_type_mask; + sBuf->unFenced = 0; + WSBM_COND_BROADCAST(&sBuf->event); + WSBM_MUTEX_UNLOCK(&buf->mutex); +} + +static int +pool_validate(struct _WsbmBufStorage *buf, + uint64_t set_flags, + uint64_t clr_flags) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + WSBM_MUTEX_LOCK(&buf->mutex); + while(wsbmAtomicRead(&sBuf->writers) != 0) { + WSBM_COND_WAIT(&sBuf->event, &buf->mutex); + } + + sBuf->unFenced = 1; + WSBM_MUTEX_UNLOCK(&buf->mutex); + return 0; +} + +static void +pool_unvalidate(struct _WsbmBufStorage *buf) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + WSBM_MUTEX_LOCK(&buf->mutex); + if (sBuf->unFenced) { + sBuf->unFenced = 0; + WSBM_COND_BROADCAST(&sBuf->event); + } + WSBM_MUTEX_UNLOCK(&buf->mutex); +} + +struct _WsbmSlabCache * +wsbmSlabCacheInit(uint32_t checkIntervalMsec, uint32_t slabTimeoutMsec) +{ + struct _WsbmSlabCache *tmp; + + tmp = calloc(1, sizeof(*tmp)); + if (!tmp) + return NULL; + + WSBM_MUTEX_INIT(&tmp->mutex); + WSBM_MUTEX_LOCK(&tmp->mutex); + tmp->slabTimeout.tv_usec = slabTimeoutMsec*1000; + tmp->slabTimeout.tv_sec = tmp->slabTimeout.tv_usec / 1000000; + tmp->slabTimeout.tv_usec -= tmp->slabTimeout.tv_sec*1000000; + + tmp->checkInterval.tv_usec = checkIntervalMsec*1000; + tmp->checkInterval.tv_sec = tmp->checkInterval.tv_usec / 1000000; + tmp->checkInterval.tv_usec -= tmp->checkInterval.tv_sec*1000000; + + gettimeofday(&tmp->nextCheck, NULL); + wsbmTimeAdd(&tmp->nextCheck, &tmp->checkInterval); + WSBMINITLISTHEAD(&tmp->timeoutList); + WSBMINITLISTHEAD(&tmp->unCached); + WSBMINITLISTHEAD(&tmp->cached); + WSBM_MUTEX_UNLOCK(&tmp->mutex); + + return tmp; +} + +void +wsbmSlabCacheFinish(struct _WsbmSlabCache *cache) +{ + struct timeval time; + + time = cache->nextCheck; + WSBM_MUTEX_LOCK(&cache->mutex); + wsbmTimeAdd(&time, &cache->checkInterval); + wsbmFreeTimeoutKBOsLocked(cache, &time); + WSBM_MUTEX_UNLOCK(&cache->mutex); + + assert(cache->timeoutList.next == &cache->timeoutList); + assert(cache->unCached.next == &cache->unCached); + assert(cache->cached.next == &cache->cached); + + WSBM_MUTEX_FREE(&cache->mutex); + free(cache); +} + +static void +wsbmInitSizeHeader(struct _WsbmSlabPool *slabPool, uint32_t size, + struct _WsbmSlabSizeHeader *header) +{ + WSBM_MUTEX_INIT(&header->mutex); + WSBM_MUTEX_LOCK(&header->mutex); + + WSBMINITLISTHEAD(&header->slabs); + WSBMINITLISTHEAD(&header->freeSlabs); + WSBMINITLISTHEAD(&header->delayedBuffers); + + header->numDelayed = 0; + header->slabPool = slabPool; + header->bufSize = size; + + WSBM_MUTEX_UNLOCK(&header->mutex); +} + +static void +wsbmFinishSizeHeader(struct _WsbmSlabSizeHeader *header) +{ + struct _WsbmListHead *list, *next; + struct _WsbmSlabBuffer *sBuf; + + WSBM_MUTEX_LOCK(&header->mutex); + WSBMLISTFOREACHSAFE(list, next, &header->delayedBuffers) { + sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); + if (sBuf->fence) { + (void) wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0); + wsbmFenceUnReference(&sBuf->fence); + } + header->numDelayed--; + wsbmSlabFreeBufferLocked(sBuf); + } + WSBM_MUTEX_UNLOCK(&header->mutex); + WSBM_MUTEX_FREE(&header->mutex); +} + + +static void +pool_takedown(struct _WsbmBufferPool *pool) +{ + struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool); + int i; + + for (i=0; i<slabPool->numBuckets; ++i) { + wsbmFinishSizeHeader(&slabPool->headers[i]); + } + + free(slabPool->headers); + free(slabPool->bucketSizes); + free(slabPool); +} + +struct _WsbmBufferPool * +wsbmSlabPoolInit(int fd, + uint32_t devOffset, + uint32_t placement, + uint32_t validMask, + uint32_t smallestSize, + uint32_t numSizes, + uint32_t desiredNumBuffers, + uint32_t maxSlabSize, + uint32_t pageAlignment, + struct _WsbmSlabCache *cache) +{ + struct _WsbmBufferPool *pool; + struct _WsbmSlabPool *slabPool; + uint32_t i; + + slabPool = calloc(1, sizeof(*slabPool)); + if (!slabPool) + return NULL; + + pool = &slabPool->pool; + + slabPool->bucketSizes = calloc(numSizes, sizeof(*slabPool->bucketSizes)); + if (!slabPool->bucketSizes) + goto out_err0; + + slabPool->headers = calloc(numSizes, sizeof(*slabPool->headers)); + if (!slabPool->headers) + goto out_err1; + + slabPool->devOffset = devOffset; + slabPool->cache = cache; + slabPool->proposedPlacement = placement; + slabPool->validMask = validMask; + slabPool->numBuckets = numSizes; + slabPool->pageSize = getpagesize(); + slabPool->pageAlignment = pageAlignment; + slabPool->maxSlabSize = maxSlabSize; + slabPool->desiredNumBuffers = desiredNumBuffers; + + for (i=0; i<slabPool->numBuckets; ++i) { + slabPool->bucketSizes[i] = (smallestSize << i); + wsbmInitSizeHeader(slabPool, slabPool->bucketSizes[i], + &slabPool->headers[i]); + } + + pool->fd = fd; + pool->map = &pool_map; + pool->unmap = &pool_unmap; + pool->destroy = &pool_destroy; + pool->offset = &pool_offset; + pool->poolOffset = &pool_poolOffset; + pool->size = &pool_size; + pool->create = &pool_create; + pool->fence = &pool_fence; + pool->kernel = &pool_kernel; + pool->validate = &pool_validate; + pool->unvalidate = &pool_unvalidate; + pool->waitIdle = &pool_waitIdle; + pool->takeDown = &pool_takedown; + pool->releasefromcpu = &pool_releaseFromCpu; + pool->syncforcpu = &pool_syncForCpu; + + return pool; + + out_err1: + free(slabPool->bucketSizes); + out_err0: + free(slabPool); + + return NULL; +} |