From owner-svn-src-head@FreeBSD.ORG Tue Mar 5 09:49:35 2013 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.FreeBSD.org [8.8.178.115]) by hub.freebsd.org (Postfix) with ESMTP id 9CCF7BC4; Tue, 5 Mar 2013 09:49:35 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id 8F5D111A; Tue, 5 Mar 2013 09:49:35 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.6/8.14.6) with ESMTP id r259nZ5X020427; Tue, 5 Mar 2013 09:49:35 GMT (envelope-from kib@svn.freebsd.org) Received: (from kib@localhost) by svn.freebsd.org (8.14.6/8.14.5/Submit) id r259nYRn020421; Tue, 5 Mar 2013 09:49:34 GMT (envelope-from kib@svn.freebsd.org) Message-Id: <201303050949.r259nYRn020421@svn.freebsd.org> From: Konstantin Belousov Date: Tue, 5 Mar 2013 09:49:34 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r247835 - in head/sys: dev/drm2 dev/drm2/ttm modules/drm2/drm2 X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 05 Mar 2013 09:49:35 -0000 Author: kib Date: Tue Mar 5 09:49:34 2013 New Revision: 247835 URL: http://svnweb.freebsd.org/changeset/base/247835 Log: Import the preliminary port of the TTM. The early commit is done to facilitate the off-tree work on the porting of the Radeon driver. Sponsored by: The FreeBSD Foundation Debugged and tested by: dumbbell MFC after: 1 month Added: head/sys/dev/drm2/ttm/ head/sys/dev/drm2/ttm/ttm_agp_backend.c (contents, props changed) head/sys/dev/drm2/ttm/ttm_bo.c (contents, props changed) head/sys/dev/drm2/ttm/ttm_bo_api.h (contents, props changed) head/sys/dev/drm2/ttm/ttm_bo_driver.h (contents, props changed) head/sys/dev/drm2/ttm/ttm_bo_manager.c (contents, props changed) head/sys/dev/drm2/ttm/ttm_bo_util.c (contents, props changed) head/sys/dev/drm2/ttm/ttm_bo_vm.c (contents, props changed) head/sys/dev/drm2/ttm/ttm_execbuf_util.c (contents, props changed) head/sys/dev/drm2/ttm/ttm_execbuf_util.h (contents, props changed) head/sys/dev/drm2/ttm/ttm_lock.c (contents, props changed) head/sys/dev/drm2/ttm/ttm_lock.h (contents, props changed) head/sys/dev/drm2/ttm/ttm_memory.c (contents, props changed) head/sys/dev/drm2/ttm/ttm_memory.h (contents, props changed) head/sys/dev/drm2/ttm/ttm_module.h (contents, props changed) head/sys/dev/drm2/ttm/ttm_object.c (contents, props changed) head/sys/dev/drm2/ttm/ttm_object.h (contents, props changed) head/sys/dev/drm2/ttm/ttm_page_alloc.c (contents, props changed) head/sys/dev/drm2/ttm/ttm_page_alloc.h (contents, props changed) head/sys/dev/drm2/ttm/ttm_page_alloc_dma.c (contents, props changed) head/sys/dev/drm2/ttm/ttm_placement.h (contents, props changed) head/sys/dev/drm2/ttm/ttm_tt.c (contents, props changed) Modified: head/sys/dev/drm2/drmP.h head/sys/dev/drm2/drm_drv.c head/sys/dev/drm2/drm_gem.c head/sys/modules/drm2/drm2/Makefile Modified: head/sys/dev/drm2/drmP.h ============================================================================== --- head/sys/dev/drm2/drmP.h Tue Mar 5 09:27:21 2013 (r247834) +++ head/sys/dev/drm2/drmP.h Tue Mar 5 09:49:34 2013 (r247835) @@ -906,6 +906,7 @@ struct drm_device { struct drm_minor *control; /**< Control node for card */ struct drm_minor *primary; /**< render type primary screen head */ + void *drm_ttm_bo; struct unrhdr *drw_unrhdr; /* RB tree of drawable infos */ RB_HEAD(drawable_tree, bsd_drm_drawable_info) drw_head; @@ -1302,10 +1303,14 @@ void drm_gem_release(struct drm_device * int drm_gem_create_mmap_offset(struct drm_gem_object *obj); void drm_gem_free_mmap_offset(struct drm_gem_object *obj); -int drm_gem_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t size, - struct vm_object **obj_res, int nprot); +int drm_gem_mmap_single(struct drm_device *dev, vm_ooffset_t *offset, + vm_size_t size, struct vm_object **obj_res, int nprot); void drm_gem_pager_dtr(void *obj); +struct ttm_bo_device; +int ttm_bo_mmap_single(struct ttm_bo_device *bdev, vm_ooffset_t *offset, + vm_size_t size, struct vm_object **obj_res, int nprot); + void drm_device_lock_mtx(struct drm_device *dev); void drm_device_unlock_mtx(struct drm_device *dev); int drm_device_sleep_mtx(struct drm_device *dev, void *chan, int flags, Modified: head/sys/dev/drm2/drm_drv.c ============================================================================== --- head/sys/dev/drm2/drm_drv.c Tue Mar 5 09:27:21 2013 (r247834) +++ head/sys/dev/drm2/drm_drv.c Tue Mar 5 09:49:34 2013 (r247835) @@ -58,6 +58,8 @@ static int drm_load(struct drm_device *d static void drm_unload(struct drm_device *dev); static drm_pci_id_list_t *drm_find_description(int vendor, int device, drm_pci_id_list_t *idlist); +static int drm_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, + vm_size_t size, struct vm_object **obj_res, int nprot); static int drm_modevent(module_t mod, int type, void *data) @@ -187,7 +189,7 @@ static struct cdevsw drm_cdevsw = { .d_ioctl = drm_ioctl, .d_poll = drm_poll, .d_mmap = drm_mmap, - .d_mmap_single = drm_gem_mmap_single, + .d_mmap_single = drm_mmap_single, .d_name = "drm", .d_flags = D_TRACKCLOSE }; @@ -955,6 +957,23 @@ drm_add_busid_modesetting(struct drm_dev return (0); } +static int +drm_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t size, + struct vm_object **obj_res, int nprot) +{ + struct drm_device *dev; + + dev = drm_get_device_from_kdev(kdev); + if ((dev->driver->driver_features & DRIVER_GEM) != 0) { + return (drm_gem_mmap_single(dev, offset, size, obj_res, nprot)); + } else if (dev->drm_ttm_bo != NULL) { + return (ttm_bo_mmap_single(dev->drm_ttm_bo, offset, size, + obj_res, nprot)); + } else { + return (ENODEV); + } +} + #if DRM_LINUX #include Modified: head/sys/dev/drm2/drm_gem.c ============================================================================== --- head/sys/dev/drm2/drm_gem.c Tue Mar 5 09:27:21 2013 (r247834) +++ head/sys/dev/drm2/drm_gem.c Tue Mar 5 09:49:34 2013 (r247835) @@ -441,16 +441,12 @@ drm_gem_free_mmap_offset(struct drm_gem_ } int -drm_gem_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t size, +drm_gem_mmap_single(struct drm_device *dev, vm_ooffset_t *offset, vm_size_t size, struct vm_object **obj_res, int nprot) { - struct drm_device *dev; struct drm_gem_object *gem_obj; struct vm_object *vm_obj; - dev = drm_get_device_from_kdev(kdev); - if ((dev->driver->driver_features & DRIVER_GEM) == 0) - return (ENODEV); DRM_LOCK(dev); gem_obj = drm_gem_object_from_offset(dev, *offset); if (gem_obj == NULL) { Added: head/sys/dev/drm2/ttm/ttm_agp_backend.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/dev/drm2/ttm/ttm_agp_backend.c Tue Mar 5 09:49:34 2013 (r247835) @@ -0,0 +1,145 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + * Keith Packard. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#ifdef TTM_HAS_AGP +#include + +struct ttm_agp_backend { + struct ttm_tt ttm; + struct agp_memory *mem; + device_t bridge; +}; + +MALLOC_DEFINE(M_TTM_AGP, "ttm_agp", "TTM AGP Backend"); + +static int ttm_agp_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + struct drm_mm_node *node = bo_mem->mm_node; + struct agp_memory *mem; + int ret, cached = (bo_mem->placement & TTM_PL_FLAG_CACHED); + unsigned i; + + mem = agp_alloc_memory(agp_be->bridge, AGP_USER_MEMORY, ttm->num_pages); + if (unlikely(mem == NULL)) + return -ENOMEM; + + mem->page_count = 0; + for (i = 0; i < ttm->num_pages; i++) { + vm_page_t page = ttm->pages[i]; + + if (!page) + page = ttm->dummy_read_page; + + mem->pages[mem->page_count++] = page; + } + agp_be->mem = mem; + + mem->is_flushed = 1; + mem->type = (cached) ? AGP_USER_CACHED_MEMORY : AGP_USER_MEMORY; + + ret = agp_bind_memory(mem, node->start); + if (ret) + pr_err("AGP Bind memory failed\n"); + + return ret; +} + +static int ttm_agp_unbind(struct ttm_tt *ttm) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + + if (agp_be->mem) { + if (agp_be->mem->is_bound) + return agp_unbind_memory(agp_be->mem); + agp_free_memory(agp_be->mem); + agp_be->mem = NULL; + } + return 0; +} + +static void ttm_agp_destroy(struct ttm_tt *ttm) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + + if (agp_be->mem) + ttm_agp_unbind(ttm); + ttm_tt_fini(ttm); + free(agp_be, M_TTM_AGP); +} + +static struct ttm_backend_func ttm_agp_func = { + .bind = ttm_agp_bind, + .unbind = ttm_agp_unbind, + .destroy = ttm_agp_destroy, +}; + +struct ttm_tt *ttm_agp_tt_create(struct ttm_bo_device *bdev, + device_t bridge, + unsigned long size, uint32_t page_flags, + vm_page_t dummy_read_page) +{ + struct ttm_agp_backend *agp_be; + + agp_be = malloc(sizeof(*agp_be), M_TTM_AGP, M_WAITOK | M_ZERO); + + agp_be->mem = NULL; + agp_be->bridge = bridge; + agp_be->ttm.func = &ttm_agp_func; + + if (ttm_tt_init(&agp_be->ttm, bdev, size, page_flags, dummy_read_page)) { + return NULL; + } + + return &agp_be->ttm; +} + +int ttm_agp_tt_populate(struct ttm_tt *ttm) +{ + if (ttm->state != tt_unpopulated) + return 0; + + return ttm_pool_populate(ttm); +} + +void ttm_agp_tt_unpopulate(struct ttm_tt *ttm) +{ + ttm_pool_unpopulate(ttm); +} + +#endif Added: head/sys/dev/drm2/ttm/ttm_bo.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/dev/drm2/ttm/ttm_bo.c Tue Mar 5 09:49:34 2013 (r247835) @@ -0,0 +1,1820 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include + +#define TTM_ASSERT_LOCKED(param) +#define TTM_DEBUG(fmt, arg...) +#define TTM_BO_HASH_ORDER 13 + +static int ttm_bo_setup_vm(struct ttm_buffer_object *bo); +static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); +static void ttm_bo_global_kobj_release(struct ttm_bo_global *glob); + +MALLOC_DEFINE(M_TTM_BO, "ttm_bo", "TTM Buffer Objects"); + +static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type) +{ + int i; + + for (i = 0; i <= TTM_PL_PRIV5; i++) + if (flags & (1 << i)) { + *mem_type = i; + return 0; + } + return -EINVAL; +} + +static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) +{ + struct ttm_mem_type_manager *man = &bdev->man[mem_type]; + + printf(" has_type: %d\n", man->has_type); + printf(" use_type: %d\n", man->use_type); + printf(" flags: 0x%08X\n", man->flags); + printf(" gpu_offset: 0x%08lX\n", man->gpu_offset); + printf(" size: %ju\n", (uintmax_t)man->size); + printf(" available_caching: 0x%08X\n", man->available_caching); + printf(" default_caching: 0x%08X\n", man->default_caching); + if (mem_type != TTM_PL_SYSTEM) + (*man->func->debug)(man, TTM_PFX); +} + +static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, + struct ttm_placement *placement) +{ + int i, ret, mem_type; + + printf("No space for %p (%lu pages, %luK, %luM)\n", + bo, bo->mem.num_pages, bo->mem.size >> 10, + bo->mem.size >> 20); + for (i = 0; i < placement->num_placement; i++) { + ret = ttm_mem_type_from_flags(placement->placement[i], + &mem_type); + if (ret) + return; + printf(" placement[%d]=0x%08X (%d)\n", + i, placement->placement[i], mem_type); + ttm_mem_type_debug(bo->bdev, mem_type); + } +} + +#if 0 +static ssize_t ttm_bo_global_show(struct ttm_bo_global *glob, + char *buffer) +{ + + return snprintf(buffer, PAGE_SIZE, "%lu\n", + (unsigned long) atomic_read(&glob->bo_count)); +} +#endif + +static inline uint32_t ttm_bo_type_flags(unsigned type) +{ + return 1 << (type); +} + +static void ttm_bo_release_list(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + size_t acc_size = bo->acc_size; + + MPASS(atomic_read(&bo->list_kref) == 0); + MPASS(atomic_read(&bo->kref) == 0); + MPASS(atomic_read(&bo->cpu_writers) == 0); + MPASS(bo->sync_obj == NULL); + MPASS(bo->mem.mm_node == NULL); + MPASS(list_empty(&bo->lru)); + MPASS(list_empty(&bo->ddestroy)); + + if (bo->ttm) + ttm_tt_destroy(bo->ttm); + atomic_dec(&bo->glob->bo_count); + if (bo->destroy) + bo->destroy(bo); + else { + free(bo, M_TTM_BO); + } + ttm_mem_global_free(bdev->glob->mem_glob, acc_size); +} + +int +ttm_bo_wait_unreserved_locked(struct ttm_buffer_object *bo, bool interruptible) +{ + const char *wmsg; + int flags, ret; + + ret = 0; + if (interruptible) { + flags = PCATCH; + wmsg = "ttbowi"; + } else { + flags = 0; + wmsg = "ttbowu"; + } + while (!ttm_bo_is_reserved(bo)) { + ret = -msleep(bo, &bo->glob->lru_lock, flags, wmsg, 0); + if (ret != 0) + break; + } + return (ret); +} + +void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_mem_type_manager *man; + + MPASS(ttm_bo_is_reserved(bo)); + + if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { + + MPASS(list_empty(&bo->lru)); + + man = &bdev->man[bo->mem.mem_type]; + list_add_tail(&bo->lru, &man->lru); + refcount_acquire(&bo->list_kref); + + if (bo->ttm != NULL) { + list_add_tail(&bo->swap, &bo->glob->swap_lru); + refcount_acquire(&bo->list_kref); + } + } +} + +int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) +{ + int put_count = 0; + + if (!list_empty(&bo->swap)) { + list_del_init(&bo->swap); + ++put_count; + } + if (!list_empty(&bo->lru)) { + list_del_init(&bo->lru); + ++put_count; + } + + /* + * TODO: Add a driver hook to delete from + * driver-specific LRU's here. + */ + + return put_count; +} + +int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait, bool use_sequence, uint32_t sequence) +{ + int ret; + + while (unlikely(atomic_read(&bo->reserved) != 0)) { + /** + * Deadlock avoidance for multi-bo reserving. + */ + if (use_sequence && bo->seq_valid) { + /** + * We've already reserved this one. + */ + if (unlikely(sequence == bo->val_seq)) + return -EDEADLK; + /** + * Already reserved by a thread that will not back + * off for us. We need to back off. + */ + if (unlikely(sequence - bo->val_seq < (1 << 31))) + return -EAGAIN; + } + + if (no_wait) + return -EBUSY; + + ret = ttm_bo_wait_unreserved_locked(bo, interruptible); + if (unlikely(ret)) + return ret; + } + + atomic_set(&bo->reserved, 1); + if (use_sequence) { + /** + * Wake up waiters that may need to recheck for deadlock, + * if we decreased the sequence number. + */ + if (unlikely((bo->val_seq - sequence < (1 << 31)) + || !bo->seq_valid)) + wakeup(bo); + + bo->val_seq = sequence; + bo->seq_valid = true; + } else { + bo->seq_valid = false; + } + + return 0; +} + +void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, + bool never_free) +{ + u_int old; + + old = atomic_fetchadd_int(&bo->list_kref, -count); + if (old <= count) { + if (never_free) + panic("ttm_bo_ref_buf"); + ttm_bo_release_list(bo); + } +} + +int ttm_bo_reserve(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait, bool use_sequence, uint32_t sequence) +{ + struct ttm_bo_global *glob = bo->glob; + int put_count = 0; + int ret; + + mtx_lock(&glob->lru_lock); + ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence, + sequence); + if (likely(ret == 0)) + put_count = ttm_bo_del_from_lru(bo); + mtx_unlock(&glob->lru_lock); + + ttm_bo_list_ref_sub(bo, put_count, true); + + return ret; +} + +void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo) +{ + ttm_bo_add_to_lru(bo); + atomic_set(&bo->reserved, 0); + wakeup(bo); +} + +void ttm_bo_unreserve(struct ttm_buffer_object *bo) +{ + struct ttm_bo_global *glob = bo->glob; + + mtx_lock(&glob->lru_lock); + ttm_bo_unreserve_locked(bo); + mtx_unlock(&glob->lru_lock); +} + +/* + * Call bo->mutex locked. + */ +static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_bo_global *glob = bo->glob; + int ret = 0; + uint32_t page_flags = 0; + + TTM_ASSERT_LOCKED(&bo->mutex); + bo->ttm = NULL; + + if (bdev->need_dma32) + page_flags |= TTM_PAGE_FLAG_DMA32; + + switch (bo->type) { + case ttm_bo_type_device: + if (zero_alloc) + page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; + case ttm_bo_type_kernel: + bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, + page_flags, glob->dummy_read_page); + if (unlikely(bo->ttm == NULL)) + ret = -ENOMEM; + break; + case ttm_bo_type_sg: + bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, + page_flags | TTM_PAGE_FLAG_SG, + glob->dummy_read_page); + if (unlikely(bo->ttm == NULL)) { + ret = -ENOMEM; + break; + } + bo->ttm->sg = bo->sg; + break; + default: + printf("[TTM] Illegal buffer object type\n"); + ret = -EINVAL; + break; + } + + return ret; +} + +static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, + struct ttm_mem_reg *mem, + bool evict, bool interruptible, + bool no_wait_gpu) +{ + struct ttm_bo_device *bdev = bo->bdev; + bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); + bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); + struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; + struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; + int ret = 0; + + if (old_is_pci || new_is_pci || + ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { + ret = ttm_mem_io_lock(old_man, true); + if (unlikely(ret != 0)) + goto out_err; + ttm_bo_unmap_virtual_locked(bo); + ttm_mem_io_unlock(old_man); + } + + /* + * Create and bind a ttm if required. + */ + + if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { + if (bo->ttm == NULL) { + bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); + ret = ttm_bo_add_ttm(bo, zero); + if (ret) + goto out_err; + } + + ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); + if (ret) + goto out_err; + + if (mem->mem_type != TTM_PL_SYSTEM) { + ret = ttm_tt_bind(bo->ttm, mem); + if (ret) + goto out_err; + } + + if (bo->mem.mem_type == TTM_PL_SYSTEM) { + if (bdev->driver->move_notify) + bdev->driver->move_notify(bo, mem); + bo->mem = *mem; + mem->mm_node = NULL; + goto moved; + } + } + + if (bdev->driver->move_notify) + bdev->driver->move_notify(bo, mem); + + if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && + !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) + ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); + else if (bdev->driver->move) + ret = bdev->driver->move(bo, evict, interruptible, + no_wait_gpu, mem); + else + ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); + + if (ret) { + if (bdev->driver->move_notify) { + struct ttm_mem_reg tmp_mem = *mem; + *mem = bo->mem; + bo->mem = tmp_mem; + bdev->driver->move_notify(bo, mem); + bo->mem = *mem; + } + + goto out_err; + } + +moved: + if (bo->evicted) { + ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); + if (ret) + printf("[TTM] Can not flush read caches\n"); + bo->evicted = false; + } + + if (bo->mem.mm_node) { + bo->offset = (bo->mem.start << PAGE_SHIFT) + + bdev->man[bo->mem.mem_type].gpu_offset; + bo->cur_placement = bo->mem.placement; + } else + bo->offset = 0; + + return 0; + +out_err: + new_man = &bdev->man[bo->mem.mem_type]; + if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { + ttm_tt_unbind(bo->ttm); + ttm_tt_destroy(bo->ttm); + bo->ttm = NULL; + } + + return ret; +} + +/** + * Call bo::reserved. + * Will release GPU memory type usage on destruction. + * This is the place to put in driver specific hooks to release + * driver private resources. + * Will release the bo::reserved lock. + */ + +static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) +{ + if (bo->bdev->driver->move_notify) + bo->bdev->driver->move_notify(bo, NULL); + + if (bo->ttm) { + ttm_tt_unbind(bo->ttm); + ttm_tt_destroy(bo->ttm); + bo->ttm = NULL; + } + ttm_bo_mem_put(bo, &bo->mem); + + atomic_set(&bo->reserved, 0); + wakeup(&bo); + + /* + * Since the final reference to this bo may not be dropped by + * the current task we have to put a memory barrier here to make + * sure the changes done in this function are always visible. + * + * This function only needs protection against the final kref_put. + */ + mb(); +} + +static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_bo_global *glob = bo->glob; + struct ttm_bo_driver *driver = bdev->driver; + void *sync_obj = NULL; + int put_count; + int ret; + + mtx_lock(&glob->lru_lock); + ret = ttm_bo_reserve_locked(bo, false, true, false, 0); + + mtx_lock(&bdev->fence_lock); + (void) ttm_bo_wait(bo, false, false, true); + if (!ret && !bo->sync_obj) { + mtx_unlock(&bdev->fence_lock); + put_count = ttm_bo_del_from_lru(bo); + + mtx_unlock(&glob->lru_lock); + ttm_bo_cleanup_memtype_use(bo); + + ttm_bo_list_ref_sub(bo, put_count, true); + + return; + } + if (bo->sync_obj) + sync_obj = driver->sync_obj_ref(bo->sync_obj); + mtx_unlock(&bdev->fence_lock); + + if (!ret) { + atomic_set(&bo->reserved, 0); + wakeup(bo); + } + + refcount_acquire(&bo->list_kref); + list_add_tail(&bo->ddestroy, &bdev->ddestroy); + mtx_unlock(&glob->lru_lock); + + if (sync_obj) { + driver->sync_obj_flush(sync_obj); + driver->sync_obj_unref(&sync_obj); + } + taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, + ((hz / 100) < 1) ? 1 : hz / 100); +} + +/** + * function ttm_bo_cleanup_refs_and_unlock + * If bo idle, remove from delayed- and lru lists, and unref. + * If not idle, do nothing. + * + * Must be called with lru_lock and reservation held, this function + * will drop both before returning. + * + * @interruptible Any sleeps should occur interruptibly. + * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. + */ + +static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait_gpu) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_bo_driver *driver = bdev->driver; + struct ttm_bo_global *glob = bo->glob; + int put_count; + int ret; + + mtx_lock(&bdev->fence_lock); + ret = ttm_bo_wait(bo, false, false, true); + + if (ret && !no_wait_gpu) { + void *sync_obj; + + /* + * Take a reference to the fence and unreserve, + * at this point the buffer should be dead, so + * no new sync objects can be attached. + */ + sync_obj = driver->sync_obj_ref(bo->sync_obj); + mtx_unlock(&bdev->fence_lock); + + atomic_set(&bo->reserved, 0); + wakeup(bo); + mtx_unlock(&glob->lru_lock); + + ret = driver->sync_obj_wait(sync_obj, false, interruptible); + driver->sync_obj_unref(&sync_obj); + if (ret) + return ret; + + /* + * remove sync_obj with ttm_bo_wait, the wait should be + * finished, and no new wait object should have been added. + */ + mtx_lock(&bdev->fence_lock); + ret = ttm_bo_wait(bo, false, false, true); + mtx_unlock(&bdev->fence_lock); + if (ret) + return ret; + + mtx_lock(&glob->lru_lock); + ret = ttm_bo_reserve_locked(bo, false, true, false, 0); + + /* + * We raced, and lost, someone else holds the reservation now, + * and is probably busy in ttm_bo_cleanup_memtype_use. + * + * Even if it's not the case, because we finished waiting any + * delayed destruction would succeed, so just return success + * here. + */ + if (ret) { + mtx_unlock(&glob->lru_lock); + return 0; + } + } else + mtx_unlock(&bdev->fence_lock); + + if (ret || unlikely(list_empty(&bo->ddestroy))) { + atomic_set(&bo->reserved, 0); + wakeup(bo); + mtx_unlock(&glob->lru_lock); + return ret; + } + + put_count = ttm_bo_del_from_lru(bo); + list_del_init(&bo->ddestroy); + ++put_count; + + mtx_unlock(&glob->lru_lock); + ttm_bo_cleanup_memtype_use(bo); + + ttm_bo_list_ref_sub(bo, put_count, true); + + return 0; +} + +/** + * Traverse the delayed list, and call ttm_bo_cleanup_refs on all + * encountered buffers. + */ + +static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) +{ + struct ttm_bo_global *glob = bdev->glob; + struct ttm_buffer_object *entry = NULL; + int ret = 0; + + mtx_lock(&glob->lru_lock); + if (list_empty(&bdev->ddestroy)) + goto out_unlock; + + entry = list_first_entry(&bdev->ddestroy, + struct ttm_buffer_object, ddestroy); + refcount_acquire(&entry->list_kref); + + for (;;) { + struct ttm_buffer_object *nentry = NULL; + + if (entry->ddestroy.next != &bdev->ddestroy) { + nentry = list_first_entry(&entry->ddestroy, + struct ttm_buffer_object, ddestroy); + refcount_acquire(&nentry->list_kref); + } + + ret = ttm_bo_reserve_locked(entry, false, !remove_all, false, 0); + if (!ret) + ret = ttm_bo_cleanup_refs_and_unlock(entry, false, + !remove_all); + else + mtx_unlock(&glob->lru_lock); + + if (refcount_release(&entry->list_kref)) + ttm_bo_release_list(entry); + entry = nentry; + + if (ret || !entry) + goto out; + + mtx_lock(&glob->lru_lock); + if (list_empty(&entry->ddestroy)) + break; + } + +out_unlock: + mtx_unlock(&glob->lru_lock); +out: + if (entry && refcount_release(&entry->list_kref)) + ttm_bo_release_list(entry); + return ret; +} + +static void ttm_bo_delayed_workqueue(void *arg, int pending __unused) +{ + struct ttm_bo_device *bdev = arg; + + if (ttm_bo_delayed_delete(bdev, false)) { + taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, + ((hz / 100) < 1) ? 1 : hz / 100); + } +} + +static void ttm_bo_release(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; + + rw_wlock(&bdev->vm_lock); + if (likely(bo->vm_node != NULL)) { + RB_REMOVE(ttm_bo_device_buffer_objects, + &bdev->addr_space_rb, bo); + drm_mm_put_block(bo->vm_node); + bo->vm_node = NULL; + } + rw_wunlock(&bdev->vm_lock); + ttm_mem_io_lock(man, false); + ttm_mem_io_free_vm(bo); + ttm_mem_io_unlock(man); + ttm_bo_cleanup_refs_or_queue(bo); + if (refcount_release(&bo->list_kref)) + ttm_bo_release_list(bo); +} + +void ttm_bo_unref(struct ttm_buffer_object **p_bo) +{ + struct ttm_buffer_object *bo = *p_bo; + + *p_bo = NULL; + if (refcount_release(&bo->kref)) + ttm_bo_release(bo); +} + +int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) +{ + int pending; + + taskqueue_cancel_timeout(taskqueue_thread, &bdev->wq, &pending); + if (pending) + taskqueue_drain_timeout(taskqueue_thread, &bdev->wq); + return (pending); +} + +void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) +{ + if (resched) { + taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, + ((hz / 100) < 1) ? 1 : hz / 100); + } +} + +static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, + bool no_wait_gpu) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_mem_reg evict_mem; + struct ttm_placement placement; + int ret = 0; + + mtx_lock(&bdev->fence_lock); + ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); + mtx_unlock(&bdev->fence_lock); + + if (unlikely(ret != 0)) { + if (ret != -ERESTART) { + printf("[TTM] Failed to expire sync object before buffer eviction\n"); + } + goto out; + } + + MPASS(ttm_bo_is_reserved(bo)); + + evict_mem = bo->mem; + evict_mem.mm_node = NULL; + evict_mem.bus.io_reserved_vm = false; + evict_mem.bus.io_reserved_count = 0; + + placement.fpfn = 0; *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***