path: root/linux-core/ttm/ttm_fence.c

/**************************************************************************
 *
 * Copyright (c) 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA
 * All Rights Reserved.
 * Copyright (c) 2009 VMware, Inc., Palo Alto, CA., USA
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 **************************************************************************/
/*
 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
 */

#include "ttm/ttm_fence_api.h"
#include "ttm/ttm_fence_driver.h"
#include <linux/wait.h>
#include <linux/sched.h>

#include "drmP.h"

/*
 * Simple implementation for now.
 */

static void ttm_fence_lockup(struct ttm_fence_object *fence, uint32_t mask)
{
	struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);

	printk(KERN_ERR "GPU lockup dectected on engine %u "
	       "fence type 0x%08x\n",
	       (unsigned int)fence->fence_class, (unsigned int)mask);
	/*
	 * Give engines some time to idle?
	 */

	write_lock(&fc->lock);
	ttm_fence_handler(fence->fdev, fence->fence_class,
			  fence->sequence, mask, -EBUSY);
	write_unlock(&fc->lock);
}

/*
 * Convenience function to be called by fence::wait methods that
 * need polling.
 */

int ttm_fence_wait_polling(struct ttm_fence_object *fence, bool lazy,
			   bool interruptible, uint32_t mask)
{
	struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
	const struct ttm_fence_driver *driver = ttm_fence_driver(fence);
	uint32_t count = 0;
	int ret;
	unsigned long end_jiffies = fence->timeout_jiffies;

	DECLARE_WAITQUEUE(entry, current);
	add_wait_queue(&fc->fence_queue, &entry);

	ret = 0;

	for (;;) {
		__set_current_state((interruptible) ?
				    TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
		if (ttm_fence_object_signaled(fence, mask))
			break;
		if (time_after_eq(jiffies, end_jiffies)) {
			if (driver->lockup)
				driver->lockup(fence, mask);
			else
				ttm_fence_lockup(fence, mask);
			continue;
		}
		if (lazy)
			schedule_timeout(1);
		else if ((++count & 0x0F) == 0) {
			__set_current_state(TASK_RUNNING);
			schedule();
			__set_current_state((interruptible) ?
					    TASK_INTERRUPTIBLE :
					    TASK_UNINTERRUPTIBLE);
		}
		if (interruptible && signal_pending(current)) {
			ret = -ERESTART;
			break;
		}
	}
	__set_current_state(TASK_RUNNING);
	remove_wait_queue(&fc->fence_queue, &entry);
	return ret;
}

/*
 * Typically called by the IRQ handler.
 */

void ttm_fence_handler(struct ttm_fence_device *fdev, uint32_t fence_class,
		       uint32_t sequence, uint32_t type, uint32_t error)
{
	int wake = 0;
	uint32_t diff;
	uint32_t relevant_type;
	uint32_t new_type;
	struct ttm_fence_class_manager *fc = &fdev->fence_class[fence_class];
	const struct ttm_fence_driver *driver = ttm_fence_driver_from_dev(fdev);
	struct list_head *head;
	struct ttm_fence_object *fence, *next;
	bool found = false;

	if (list_empty(&fc->ring))
		return;

	list_for_each_entry(fence, &fc->ring, ring) {
		diff = (sequence - fence->sequence) & fc->sequence_mask;
		if (diff > fc->wrap_diff) {
			found = true;
			break;
		}
	}

	fc->waiting_types &= ~type;
	head = (found) ? &fence->ring : &fc->ring;

	list_for_each_entry_safe_reverse(fence, next, head, ring) {
		if (&fence->ring == &fc->ring)
			break;

		DRM_DEBUG("Fence 0x%08lx, sequence 0x%08x, type 0x%08x\n",
			  (unsigned long)fence, fence->sequence,
			  fence->fence_type);

		if (error) {
			fence->info.error = error;
			fence->info.signaled_types = fence->fence_type;
			list_del_init(&fence->ring);
			wake = 1;
			break;
		}

		relevant_type = type & fence->fence_type;
		new_type = (fence->info.signaled_types | relevant_type) ^
		    fence->info.signaled_types;

		if (new_type) {
			fence->info.signaled_types |= new_type;
			DRM_DEBUG("Fence 0x%08lx signaled 0x%08x\n",
				  (unsigned long)fence,
				  fence->info.signaled_types);

			if (unlikely(driver->signaled))
				driver->signaled(fence);

			if (driver->needed_flush)
				fc->pending_flush |=
				    driver->needed_flush(fence);

			if (new_type & fence->waiting_types)
				wake = 1;
		}

		fc->waiting_types |=
		    fence->waiting_types & ~fence->info.signaled_types;

		if (!(fence->fence_type & ~fence->info.signaled_types)) {
			DRM_DEBUG("Fence completely signaled 0x%08lx\n",
				  (unsigned long)fence);
			list_del_init(&fence->ring);
		}
	}

	/*
	 * Reinstate lost waiting types.
	 */

	if ((fc->waiting_types & type) != type) {
		head = head->prev;
		list_for_each_entry(fence, head, ring) {
			if (&fence->ring == &fc->ring)
				break;
			diff =
			    (fc->highest_waiting_sequence -
			     fence->sequence) & fc->sequence_mask;
			if (diff > fc->wrap_diff)
				break;

			fc->waiting_types |=
			    fence->waiting_types & ~fence->info.signaled_types;
		}
	}

	if (wake)
		wake_up_all(&fc->fence_queue);
}

static void ttm_fence_unring(struct ttm_fence_object *fence)
{
	struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
	unsigned long irq_flags;

	write_lock_irqsave(&fc->lock, irq_flags);
	list_del_init(&fence->ring);
	write_unlock_irqrestore(&fc->lock, irq_flags);
}

bool ttm_fence_object_signaled(struct ttm_fence_object *fence, uint32_t mask)
{
	unsigned long flags;
	bool signaled;
	const struct ttm_fence_driver *driver = ttm_fence_driver(fence);
	struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);

	mask &= fence->fence_type;
	read_lock_irqsave(&fc->lock, flags);
	signaled = (mask & fence->info.signaled_types) == mask;
	read_unlock_irqrestore(&fc->lock, flags);
	if (!signaled && driver->poll) {
		write_lock_irqsave(&fc->lock, flags);
		driver->poll(fence->fdev, fence->fence_class, mask);
		signaled = (mask & fence->info.signaled_types) == mask;
		write_unlock_irqrestore(&fc->lock, flags);
	}
	return signaled;
}

int ttm_fence_object_flush(struct ttm_fence_object *fence, uint32_t type)
{
	const struct ttm_fence_driver *driver = ttm_fence_driver(fence);
	struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
	unsigned long irq_flags;
	uint32_t saved_pending_flush;
	uint32_t diff;
	bool call_flush;

	if (type & ~fence->fence_type) {
		DRM_ERROR("Flush trying to extend fence type, "
			  "0x%x, 0x%x\n", type, fence->fence_type);
		return -EINVAL;
	}

	write_lock_irqsave(&fc->lock, irq_flags);
	fence->waiting_types |= type;
	fc->waiting_types |= fence->waiting_types;
	diff = (fence->sequence - fc->highest_waiting_sequence) &
	    fc->sequence_mask;

	if (diff < fc->wrap_diff)
		fc->highest_waiting_sequence = fence->sequence;

	/*
	 * fence->waiting_types has changed. Determine whether
	 * we need to initiate some kind of flush as a result of this.
	 */

	saved_pending_flush = fc->pending_flush;
	if (driver->needed_flush)
		fc->pending_flush |= driver->needed_flush(fence);

	if (driver->poll)
		driver->poll(fence->fdev, fence->fence_class,
			     fence->waiting_types);

	call_flush = (fc->pending_flush != 0);
	write_unlock_irqrestore(&fc->lock, irq_flags);

	if (call_flush && driver->flush)
		driver->flush(fence->fdev, fence->fence_class);

	return 0;
}

/*
 * Make sure old fence objects are signaled before their fence sequences are
 * wrapped around and reused.
 */

void ttm_fence_flush_old(struct ttm_fence_device *fdev,
			 uint32_t fence_class, uint32_t sequence)
{
	struct ttm_fence_class_manager *fc = &fdev->fence_class[fence_class];
	struct ttm_fence_object *fence;
	unsigned long irq_flags;
	const struct ttm_fence_driver *driver = fdev->driver;
	bool call_flush;

	uint32_t diff;

	write_lock_irqsave(&fc->lock, irq_flags);

	list_for_each_entry_reverse(fence, &fc->ring, ring) {
		diff = (sequence - fence->sequence) & fc->sequence_mask;
		if (diff <= fc->flush_diff)
			break;

		fence->waiting_types = fence->fence_type;
		fc->waiting_types |= fence->fence_type;

		if (driver->needed_flush)
			fc->pending_flush |= driver->needed_flush(fence);
	}

	if (driver->poll)
		driver->poll(fdev, fence_class, fc->waiting_types);

	call_flush = (fc->pending_flush != 0);
	write_unlock_irqrestore(&fc->lock, irq_flags);

	if (call_flush && driver->flush)
		driver->flush(fdev, fence->fence_class);

	/*
	 * FIXME: Shold we implement a wait here for really old fences?
	 */

}

int ttm_fence_object_wait(struct ttm_fence_object *fence,
			  bool lazy, bool interruptible, uint32_t mask)
{
	const struct ttm_fence_driver *driver = ttm_fence_driver(fence);
	struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
	int ret = 0;
	unsigned long timeout;
	unsigned long cur_jiffies;
	unsigned long to_jiffies;

	if (mask & ~fence->fence_type) {
		DRM_ERROR("Wait trying to extend fence type"
			  " 0x%08x 0x%08x\n", mask, fence->fence_type);
		BUG();
		return -EINVAL;
	}

	if (driver->wait)
		return driver->wait(fence, lazy, interruptible, mask);

	ttm_fence_object_flush(fence, mask);
      retry:
	if (!driver->has_irq ||
	    driver->has_irq(fence->fdev, fence->fence_class, mask)) {

		cur_jiffies = jiffies;
		to_jiffies = fence->timeout_jiffies;

		timeout = (time_after(to_jiffies, cur_jiffies)) ?
		    to_jiffies - cur_jiffies : 1;

		if (interruptible)
			ret = wait_event_interruptible_timeout
			    (fc->fence_queue,
			     ttm_fence_object_signaled(fence, mask), timeout);
		else
			ret = wait_event_timeout
			    (fc->fence_queue,
			     ttm_fence_object_signaled(fence, mask), timeout);

		if (unlikely(ret == -ERESTARTSYS))
			return -ERESTART;

		if (unlikely(ret == 0)) {
			if (driver->lockup)
				driver->lockup(fence, mask);
			else
				ttm_fence_lockup(fence, mask);
			goto retry;
		}

		return 0;
	}

	return ttm_fence_wait_polling(fence, lazy, interruptible, mask);
}

int ttm_fence_object_emit(struct ttm_fence_object *fence, uint32_t fence_flags,
			  uint32_t fence_class, uint32_t type)
{
	const struct ttm_fence_driver *driver = ttm_fence_driver(fence);
	struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
	unsigned long flags;
	uint32_t sequence;
	unsigned long timeout;
	int ret;

	ttm_fence_unring(fence);
	ret = driver->emit(fence->fdev,
			   fence_class, fence_flags, &sequence, &timeout);
	if (ret)
		return ret;

	write_lock_irqsave(&fc->lock, flags);
	fence->fence_class = fence_class;
	fence->fence_type = type;
	fence->waiting_types = 0;
	fence->info.signaled_types = 0;
	fence->info.error = 0;
	fence->sequence = sequence;
	fence->timeout_jiffies = timeout;
	if (list_empty(&fc->ring))
		fc->highest_waiting_sequence = sequence - 1;
	list_add_tail(&fence->ring, &fc->ring);
	fc->latest_queued_sequence = sequence;
	write_unlock_irqrestore(&fc->lock, flags);
	return 0;
}

int ttm_fence_object_init(struct ttm_fence_device *fdev,
			  uint32_t fence_class,
			  uint32_t type,
			  uint32_t create_flags,
			  void (*destroy) (struct ttm_fence_object *),
			  struct ttm_fence_object *fence)
{
	int ret = 0;

	kref_init(&fence->kref);
	fence->fence_class = fence_class;
	fence->fence_type = type;
	fence->info.signaled_types = 0;
	fence->waiting_types = 0;
	fence->sequence = 0;
	fence->info.error = 0;
	fence->fdev = fdev;
	fence->destroy = destroy;
	INIT_LIST_HEAD(&fence->ring);
	atomic_inc(&fdev->count);

	if (create_flags & TTM_FENCE_FLAG_EMIT) {
		ret = ttm_fence_object_emit(fence, create_flags,
					    fence->fence_class, type);
	}

	return ret;
}

int ttm_fence_object_create(struct ttm_fence_device *fdev,
			    uint32_t fence_class,
			    uint32_t type,
			    uint32_t create_flags,
			    struct ttm_fence_object **c_fence)
{
	struct ttm_fence_object *fence;
	int ret;

	ret = ttm_mem_global_alloc(fdev->mem_glob, sizeof(*fence), false, false, false);
	if (unlikely(ret != 0)) {
		printk(KERN_ERR "Out of memory creating fence object\n");
		return ret;
	}

	fence = kmalloc(sizeof(*fence), GFP_KERNEL);
	if (!fence) {
		printk(KERN_ERR "Out of memory creating fence object\n");
		ttm_mem_global_free(fdev->mem_glob, sizeof(*fence), false);
		return -ENOMEM;
	}

	ret = ttm_fence_object_init(fdev, fence_class, type,
				    create_flags, NULL, fence);
	if (ret) {
		ttm_fence_object_unref(&fence);
		return ret;
	}
	*c_fence = fence;

	return 0;
}

static void ttm_fence_object_destroy(struct kref *kref)
{
	struct ttm_fence_object *fence =
	    container_of(kref, struct ttm_fence_object, kref);
	struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
	unsigned long irq_flags;

	write_lock_irqsave(&fc->lock, irq_flags);
	list_del_init(&fence->ring);
	write_unlock_irqrestore(&fc->lock, irq_flags);

	atomic_dec(&fence->fdev->count);
	if (fence->destroy)
		fence->destroy(fence);
	else {
		ttm_mem_global_free(fence->fdev->mem_glob, sizeof(*fence), false);
		kfree(fence);
	}
}

void ttm_fence_device_release(struct ttm_fence_device *fdev)
{
	kfree(fdev->fence_class);
}

int
ttm_fence_device_init(int num_classes,
		      struct ttm_mem_global *mem_glob,
		      struct ttm_fence_device *fdev,
		      const struct ttm_fence_class_init *init,
		      bool replicate_init, const struct ttm_fence_driver *driver)
{
	struct ttm_fence_class_manager *fc;
	const struct ttm_fence_class_init *fci;
	int i;

	fdev->mem_glob = mem_glob;
	fdev->fence_class = kzalloc(num_classes *
				    sizeof(*fdev->fence_class), GFP_KERNEL);

	if (unlikely(!fdev->fence_class))
		return -ENOMEM;

	fdev->num_classes = num_classes;
	atomic_set(&fdev->count, 0);
	fdev->driver = driver;

	for (i = 0; i < fdev->num_classes; ++i) {
		fc = &fdev->fence_class[i];
		fci = &init[(replicate_init) ? 0 : i];

		fc->wrap_diff = fci->wrap_diff;
		fc->flush_diff = fci->flush_diff;
		fc->sequence_mask = fci->sequence_mask;

		rwlock_init(&fc->lock);
		INIT_LIST_HEAD(&fc->ring);
		init_waitqueue_head(&fc->fence_queue);
	}

	return 0;
}

struct ttm_fence_info ttm_fence_get_info(struct ttm_fence_object *fence)
{
	struct ttm_fence_class_manager *fc = ttm_fence_fc(fence);
	struct ttm_fence_info tmp;
	unsigned long irq_flags;

	read_lock_irqsave(&fc->lock, irq_flags);
	tmp = fence->info;
	read_unlock_irqrestore(&fc->lock, irq_flags);

	return tmp;
}

void ttm_fence_object_unref(struct ttm_fence_object **p_fence)
{
	struct ttm_fence_object *fence = *p_fence;

	*p_fence = NULL;
	(void)kref_put(&fence->kref, &ttm_fence_object_destroy);
}

/*
 * Placement / BO sync object glue.
 */

bool ttm_fence_sync_obj_signaled(void *sync_obj, void *sync_arg)
{
	struct ttm_fence_object *fence = (struct ttm_fence_object *)sync_obj;
	uint32_t fence_types = (uint32_t) (unsigned long)sync_arg;

	return ttm_fence_object_signaled(fence, fence_types);
}

int ttm_fence_sync_obj_wait(void *sync_obj, void *sync_arg,
			    bool lazy, bool interruptible)
{
	struct ttm_fence_object *fence = (struct ttm_fence_object *)sync_obj;
	uint32_t fence_types = (uint32_t) (unsigned long)sync_arg;

	return ttm_fence_object_wait(fence, lazy, interruptible, fence_types);
}

int ttm_fence_sync_obj_flush(void *sync_obj, void *sync_arg)
{
	struct ttm_fence_object *fence = (struct ttm_fence_object *)sync_obj;
	uint32_t fence_types = (uint32_t) (unsigned long)sync_arg;

	return ttm_fence_object_flush(fence, fence_types);
}

void ttm_fence_sync_obj_unref(void **sync_obj)
{
	ttm_fence_object_unref((struct ttm_fence_object **)sync_obj);
}

void *ttm_fence_sync_obj_ref(void *sync_obj)
{
	return (void *)
	    ttm_fence_object_ref((struct ttm_fence_object *)sync_obj);
}