Back to index

libdrm  2.4.37
intel_bufmgr_gem.c
Go to the documentation of this file.
00001 /**************************************************************************
00002  *
00003  * Copyright © 2007 Red Hat Inc.
00004  * Copyright © 2007-2012 Intel Corporation
00005  * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
00006  * All Rights Reserved.
00007  *
00008  * Permission is hereby granted, free of charge, to any person obtaining a
00009  * copy of this software and associated documentation files (the
00010  * "Software"), to deal in the Software without restriction, including
00011  * without limitation the rights to use, copy, modify, merge, publish,
00012  * distribute, sub license, and/or sell copies of the Software, and to
00013  * permit persons to whom the Software is furnished to do so, subject to
00014  * the following conditions:
00015  *
00016  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
00017  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
00018  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
00019  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
00020  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
00021  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
00022  * USE OR OTHER DEALINGS IN THE SOFTWARE.
00023  *
00024  * The above copyright notice and this permission notice (including the
00025  * next paragraph) shall be included in all copies or substantial portions
00026  * of the Software.
00027  *
00028  *
00029  **************************************************************************/
00030 /*
00031  * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
00032  *          Keith Whitwell <keithw-at-tungstengraphics-dot-com>
00033  *         Eric Anholt <eric@anholt.net>
00034  *         Dave Airlie <airlied@linux.ie>
00035  */
00036 
00037 #ifdef HAVE_CONFIG_H
00038 #include "config.h"
00039 #endif
00040 
00041 #include <xf86drm.h>
00042 #include <xf86atomic.h>
00043 #include <fcntl.h>
00044 #include <stdio.h>
00045 #include <stdlib.h>
00046 #include <string.h>
00047 #include <unistd.h>
00048 #include <assert.h>
00049 #include <pthread.h>
00050 #include <sys/ioctl.h>
00051 #include <sys/mman.h>
00052 #include <sys/stat.h>
00053 #include <sys/types.h>
00054 #include <stdbool.h>
00055 
00056 #include "errno.h"
00057 #include "libdrm_lists.h"
00058 #include "intel_bufmgr.h"
00059 #include "intel_bufmgr_priv.h"
00060 #include "intel_chipset.h"
00061 #include "intel_aub.h"
00062 #include "string.h"
00063 
00064 #include "i915_drm.h"
00065 
00066 #ifdef HAVE_VALGRIND
00067 #include <valgrind.h>
00068 #include <memcheck.h>
00069 #define VG(x) x
00070 #else
00071 #define VG(x)
00072 #endif
00073 
00074 #define VG_CLEAR(s) VG(memset(&s, 0, sizeof(s)))
00075 
00076 #define DBG(...) do {                                   \
00077        if (bufmgr_gem->bufmgr.debug)                    \
00078               fprintf(stderr, __VA_ARGS__);             \
00079 } while (0)
00080 
00081 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
00082 
00083 typedef struct _drm_intel_bo_gem drm_intel_bo_gem;
00084 
00085 struct drm_intel_gem_bo_bucket {
00086        drmMMListHead head;
00087        unsigned long size;
00088 };
00089 
00090 typedef struct _drm_intel_bufmgr_gem {
00091        drm_intel_bufmgr bufmgr;
00092 
00093        int fd;
00094 
00095        int max_relocs;
00096 
00097        pthread_mutex_t lock;
00098 
00099        struct drm_i915_gem_exec_object *exec_objects;
00100        struct drm_i915_gem_exec_object2 *exec2_objects;
00101        drm_intel_bo **exec_bos;
00102        int exec_size;
00103        int exec_count;
00104 
00106        struct drm_intel_gem_bo_bucket cache_bucket[14 * 4];
00107        int num_buckets;
00108        time_t time;
00109 
00110        drmMMListHead named;
00111        drmMMListHead vma_cache;
00112        int vma_count, vma_open, vma_max;
00113 
00114        uint64_t gtt_size;
00115        int available_fences;
00116        int pci_device;
00117        int gen;
00118        unsigned int has_bsd : 1;
00119        unsigned int has_blt : 1;
00120        unsigned int has_relaxed_fencing : 1;
00121        unsigned int has_llc : 1;
00122        unsigned int has_wait_timeout : 1;
00123        unsigned int bo_reuse : 1;
00124        unsigned int no_exec : 1;
00125        bool fenced_relocs;
00126 
00127        FILE *aub_file;
00128        uint32_t aub_offset;
00129 } drm_intel_bufmgr_gem;
00130 
00131 #define DRM_INTEL_RELOC_FENCE (1<<0)
00132 
00133 typedef struct _drm_intel_reloc_target_info {
00134        drm_intel_bo *bo;
00135        int flags;
00136 } drm_intel_reloc_target;
00137 
00138 struct _drm_intel_bo_gem {
00139        drm_intel_bo bo;
00140 
00141        atomic_t refcount;
00142        uint32_t gem_handle;
00143        const char *name;
00144 
00148        unsigned int global_name;
00149        drmMMListHead name_list;
00150 
00155        int validate_index;
00156 
00160        uint32_t tiling_mode;
00161        uint32_t swizzle_mode;
00162        unsigned long stride;
00163 
00164        time_t free_time;
00165 
00167        struct drm_i915_gem_relocation_entry *relocs;
00171        drm_intel_reloc_target *reloc_target_info;
00173        int reloc_count;
00175        void *mem_virtual;
00177        void *gtt_virtual;
00178        int map_count;
00179        drmMMListHead vma_list;
00180 
00182        drmMMListHead head;
00183 
00188        bool included_in_check_aperture;
00189 
00195        bool used_as_reloc_target;
00196 
00200        bool has_error;
00201 
00205        bool reusable;
00206 
00213        int reloc_tree_size;
00214 
00219        int reloc_tree_fences;
00220 
00222        bool mapped_cpu_write;
00223 
00224        uint32_t aub_offset;
00225 
00226        drm_intel_aub_annotation *aub_annotations;
00227        unsigned aub_annotation_count;
00228 };
00229 
00230 static unsigned int
00231 drm_intel_gem_estimate_batch_space(drm_intel_bo ** bo_array, int count);
00232 
00233 static unsigned int
00234 drm_intel_gem_compute_batch_space(drm_intel_bo ** bo_array, int count);
00235 
00236 static int
00237 drm_intel_gem_bo_get_tiling(drm_intel_bo *bo, uint32_t * tiling_mode,
00238                          uint32_t * swizzle_mode);
00239 
00240 static int
00241 drm_intel_gem_bo_set_tiling_internal(drm_intel_bo *bo,
00242                                  uint32_t tiling_mode,
00243                                  uint32_t stride);
00244 
00245 static void drm_intel_gem_bo_unreference_locked_timed(drm_intel_bo *bo,
00246                                                 time_t time);
00247 
00248 static void drm_intel_gem_bo_unreference(drm_intel_bo *bo);
00249 
00250 static void drm_intel_gem_bo_free(drm_intel_bo *bo);
00251 
00252 static unsigned long
00253 drm_intel_gem_bo_tile_size(drm_intel_bufmgr_gem *bufmgr_gem, unsigned long size,
00254                         uint32_t *tiling_mode)
00255 {
00256        unsigned long min_size, max_size;
00257        unsigned long i;
00258 
00259        if (*tiling_mode == I915_TILING_NONE)
00260               return size;
00261 
00262        /* 965+ just need multiples of page size for tiling */
00263        if (bufmgr_gem->gen >= 4)
00264               return ROUND_UP_TO(size, 4096);
00265 
00266        /* Older chips need powers of two, of at least 512k or 1M */
00267        if (bufmgr_gem->gen == 3) {
00268               min_size = 1024*1024;
00269               max_size = 128*1024*1024;
00270        } else {
00271               min_size = 512*1024;
00272               max_size = 64*1024*1024;
00273        }
00274 
00275        if (size > max_size) {
00276               *tiling_mode = I915_TILING_NONE;
00277               return size;
00278        }
00279 
00280        /* Do we need to allocate every page for the fence? */
00281        if (bufmgr_gem->has_relaxed_fencing)
00282               return ROUND_UP_TO(size, 4096);
00283 
00284        for (i = min_size; i < size; i <<= 1)
00285               ;
00286 
00287        return i;
00288 }
00289 
00290 /*
00291  * Round a given pitch up to the minimum required for X tiling on a
00292  * given chip.  We use 512 as the minimum to allow for a later tiling
00293  * change.
00294  */
00295 static unsigned long
00296 drm_intel_gem_bo_tile_pitch(drm_intel_bufmgr_gem *bufmgr_gem,
00297                          unsigned long pitch, uint32_t *tiling_mode)
00298 {
00299        unsigned long tile_width;
00300        unsigned long i;
00301 
00302        /* If untiled, then just align it so that we can do rendering
00303         * to it with the 3D engine.
00304         */
00305        if (*tiling_mode == I915_TILING_NONE)
00306               return ALIGN(pitch, 64);
00307 
00308        if (*tiling_mode == I915_TILING_X
00309                      || (IS_915(bufmgr_gem->pci_device)
00310                          && *tiling_mode == I915_TILING_Y))
00311               tile_width = 512;
00312        else
00313               tile_width = 128;
00314 
00315        /* 965 is flexible */
00316        if (bufmgr_gem->gen >= 4)
00317               return ROUND_UP_TO(pitch, tile_width);
00318 
00319        /* The older hardware has a maximum pitch of 8192 with tiled
00320         * surfaces, so fallback to untiled if it's too large.
00321         */
00322        if (pitch > 8192) {
00323               *tiling_mode = I915_TILING_NONE;
00324               return ALIGN(pitch, 64);
00325        }
00326 
00327        /* Pre-965 needs power of two tile width */
00328        for (i = tile_width; i < pitch; i <<= 1)
00329               ;
00330 
00331        return i;
00332 }
00333 
00334 static struct drm_intel_gem_bo_bucket *
00335 drm_intel_gem_bo_bucket_for_size(drm_intel_bufmgr_gem *bufmgr_gem,
00336                              unsigned long size)
00337 {
00338        int i;
00339 
00340        for (i = 0; i < bufmgr_gem->num_buckets; i++) {
00341               struct drm_intel_gem_bo_bucket *bucket =
00342                   &bufmgr_gem->cache_bucket[i];
00343               if (bucket->size >= size) {
00344                      return bucket;
00345               }
00346        }
00347 
00348        return NULL;
00349 }
00350 
00351 static void
00352 drm_intel_gem_dump_validation_list(drm_intel_bufmgr_gem *bufmgr_gem)
00353 {
00354        int i, j;
00355 
00356        for (i = 0; i < bufmgr_gem->exec_count; i++) {
00357               drm_intel_bo *bo = bufmgr_gem->exec_bos[i];
00358               drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
00359 
00360               if (bo_gem->relocs == NULL) {
00361                      DBG("%2d: %d (%s)\n", i, bo_gem->gem_handle,
00362                          bo_gem->name);
00363                      continue;
00364               }
00365 
00366               for (j = 0; j < bo_gem->reloc_count; j++) {
00367                      drm_intel_bo *target_bo = bo_gem->reloc_target_info[j].bo;
00368                      drm_intel_bo_gem *target_gem =
00369                          (drm_intel_bo_gem *) target_bo;
00370 
00371                      DBG("%2d: %d (%s)@0x%08llx -> "
00372                          "%d (%s)@0x%08lx + 0x%08x\n",
00373                          i,
00374                          bo_gem->gem_handle, bo_gem->name,
00375                          (unsigned long long)bo_gem->relocs[j].offset,
00376                          target_gem->gem_handle,
00377                          target_gem->name,
00378                          target_bo->offset,
00379                          bo_gem->relocs[j].delta);
00380               }
00381        }
00382 }
00383 
00384 static inline void
00385 drm_intel_gem_bo_reference(drm_intel_bo *bo)
00386 {
00387        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
00388 
00389        atomic_inc(&bo_gem->refcount);
00390 }
00391 
00400 static void
00401 drm_intel_add_validate_buffer(drm_intel_bo *bo)
00402 {
00403        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
00404        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
00405        int index;
00406 
00407        if (bo_gem->validate_index != -1)
00408               return;
00409 
00410        /* Extend the array of validation entries as necessary. */
00411        if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
00412               int new_size = bufmgr_gem->exec_size * 2;
00413 
00414               if (new_size == 0)
00415                      new_size = 5;
00416 
00417               bufmgr_gem->exec_objects =
00418                   realloc(bufmgr_gem->exec_objects,
00419                          sizeof(*bufmgr_gem->exec_objects) * new_size);
00420               bufmgr_gem->exec_bos =
00421                   realloc(bufmgr_gem->exec_bos,
00422                          sizeof(*bufmgr_gem->exec_bos) * new_size);
00423               bufmgr_gem->exec_size = new_size;
00424        }
00425 
00426        index = bufmgr_gem->exec_count;
00427        bo_gem->validate_index = index;
00428        /* Fill in array entry */
00429        bufmgr_gem->exec_objects[index].handle = bo_gem->gem_handle;
00430        bufmgr_gem->exec_objects[index].relocation_count = bo_gem->reloc_count;
00431        bufmgr_gem->exec_objects[index].relocs_ptr = (uintptr_t) bo_gem->relocs;
00432        bufmgr_gem->exec_objects[index].alignment = 0;
00433        bufmgr_gem->exec_objects[index].offset = 0;
00434        bufmgr_gem->exec_bos[index] = bo;
00435        bufmgr_gem->exec_count++;
00436 }
00437 
00438 static void
00439 drm_intel_add_validate_buffer2(drm_intel_bo *bo, int need_fence)
00440 {
00441        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *)bo->bufmgr;
00442        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *)bo;
00443        int index;
00444 
00445        if (bo_gem->validate_index != -1) {
00446               if (need_fence)
00447                      bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |=
00448                             EXEC_OBJECT_NEEDS_FENCE;
00449               return;
00450        }
00451 
00452        /* Extend the array of validation entries as necessary. */
00453        if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
00454               int new_size = bufmgr_gem->exec_size * 2;
00455 
00456               if (new_size == 0)
00457                      new_size = 5;
00458 
00459               bufmgr_gem->exec2_objects =
00460                      realloc(bufmgr_gem->exec2_objects,
00461                             sizeof(*bufmgr_gem->exec2_objects) * new_size);
00462               bufmgr_gem->exec_bos =
00463                      realloc(bufmgr_gem->exec_bos,
00464                             sizeof(*bufmgr_gem->exec_bos) * new_size);
00465               bufmgr_gem->exec_size = new_size;
00466        }
00467 
00468        index = bufmgr_gem->exec_count;
00469        bo_gem->validate_index = index;
00470        /* Fill in array entry */
00471        bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;
00472        bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;
00473        bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
00474        bufmgr_gem->exec2_objects[index].alignment = 0;
00475        bufmgr_gem->exec2_objects[index].offset = 0;
00476        bufmgr_gem->exec_bos[index] = bo;
00477        bufmgr_gem->exec2_objects[index].flags = 0;
00478        bufmgr_gem->exec2_objects[index].rsvd1 = 0;
00479        bufmgr_gem->exec2_objects[index].rsvd2 = 0;
00480        if (need_fence) {
00481               bufmgr_gem->exec2_objects[index].flags |=
00482                      EXEC_OBJECT_NEEDS_FENCE;
00483        }
00484        bufmgr_gem->exec_count++;
00485 }
00486 
00487 #define RELOC_BUF_SIZE(x) ((I915_RELOC_HEADER + x * I915_RELOC0_STRIDE) * \
00488        sizeof(uint32_t))
00489 
00490 static void
00491 drm_intel_bo_gem_set_in_aperture_size(drm_intel_bufmgr_gem *bufmgr_gem,
00492                                   drm_intel_bo_gem *bo_gem)
00493 {
00494        int size;
00495 
00496        assert(!bo_gem->used_as_reloc_target);
00497 
00498        /* The older chipsets are far-less flexible in terms of tiling,
00499         * and require tiled buffer to be size aligned in the aperture.
00500         * This means that in the worst possible case we will need a hole
00501         * twice as large as the object in order for it to fit into the
00502         * aperture. Optimal packing is for wimps.
00503         */
00504        size = bo_gem->bo.size;
00505        if (bufmgr_gem->gen < 4 && bo_gem->tiling_mode != I915_TILING_NONE) {
00506               int min_size;
00507 
00508               if (bufmgr_gem->has_relaxed_fencing) {
00509                      if (bufmgr_gem->gen == 3)
00510                             min_size = 1024*1024;
00511                      else
00512                             min_size = 512*1024;
00513 
00514                      while (min_size < size)
00515                             min_size *= 2;
00516               } else
00517                      min_size = size;
00518 
00519               /* Account for worst-case alignment. */
00520               size = 2 * min_size;
00521        }
00522 
00523        bo_gem->reloc_tree_size = size;
00524 }
00525 
00526 static int
00527 drm_intel_setup_reloc_list(drm_intel_bo *bo)
00528 {
00529        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
00530        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
00531        unsigned int max_relocs = bufmgr_gem->max_relocs;
00532 
00533        if (bo->size / 4 < max_relocs)
00534               max_relocs = bo->size / 4;
00535 
00536        bo_gem->relocs = malloc(max_relocs *
00537                             sizeof(struct drm_i915_gem_relocation_entry));
00538        bo_gem->reloc_target_info = malloc(max_relocs *
00539                                       sizeof(drm_intel_reloc_target));
00540        if (bo_gem->relocs == NULL || bo_gem->reloc_target_info == NULL) {
00541               bo_gem->has_error = true;
00542 
00543               free (bo_gem->relocs);
00544               bo_gem->relocs = NULL;
00545 
00546               free (bo_gem->reloc_target_info);
00547               bo_gem->reloc_target_info = NULL;
00548 
00549               return 1;
00550        }
00551 
00552        return 0;
00553 }
00554 
00555 static int
00556 drm_intel_gem_bo_busy(drm_intel_bo *bo)
00557 {
00558        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
00559        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
00560        struct drm_i915_gem_busy busy;
00561        int ret;
00562 
00563        VG_CLEAR(busy);
00564        busy.handle = bo_gem->gem_handle;
00565 
00566        ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
00567 
00568        return (ret == 0 && busy.busy);
00569 }
00570 
00571 static int
00572 drm_intel_gem_bo_madvise_internal(drm_intel_bufmgr_gem *bufmgr_gem,
00573                               drm_intel_bo_gem *bo_gem, int state)
00574 {
00575        struct drm_i915_gem_madvise madv;
00576 
00577        VG_CLEAR(madv);
00578        madv.handle = bo_gem->gem_handle;
00579        madv.madv = state;
00580        madv.retained = 1;
00581        drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv);
00582 
00583        return madv.retained;
00584 }
00585 
00586 static int
00587 drm_intel_gem_bo_madvise(drm_intel_bo *bo, int madv)
00588 {
00589        return drm_intel_gem_bo_madvise_internal
00590               ((drm_intel_bufmgr_gem *) bo->bufmgr,
00591                (drm_intel_bo_gem *) bo,
00592                madv);
00593 }
00594 
00595 /* drop the oldest entries that have been purged by the kernel */
00596 static void
00597 drm_intel_gem_bo_cache_purge_bucket(drm_intel_bufmgr_gem *bufmgr_gem,
00598                                 struct drm_intel_gem_bo_bucket *bucket)
00599 {
00600        while (!DRMLISTEMPTY(&bucket->head)) {
00601               drm_intel_bo_gem *bo_gem;
00602 
00603               bo_gem = DRMLISTENTRY(drm_intel_bo_gem,
00604                                   bucket->head.next, head);
00605               if (drm_intel_gem_bo_madvise_internal
00606                   (bufmgr_gem, bo_gem, I915_MADV_DONTNEED))
00607                      break;
00608 
00609               DRMLISTDEL(&bo_gem->head);
00610               drm_intel_gem_bo_free(&bo_gem->bo);
00611        }
00612 }
00613 
00614 static drm_intel_bo *
00615 drm_intel_gem_bo_alloc_internal(drm_intel_bufmgr *bufmgr,
00616                             const char *name,
00617                             unsigned long size,
00618                             unsigned long flags,
00619                             uint32_t tiling_mode,
00620                             unsigned long stride)
00621 {
00622        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bufmgr;
00623        drm_intel_bo_gem *bo_gem;
00624        unsigned int page_size = getpagesize();
00625        int ret;
00626        struct drm_intel_gem_bo_bucket *bucket;
00627        bool alloc_from_cache;
00628        unsigned long bo_size;
00629        bool for_render = false;
00630 
00631        if (flags & BO_ALLOC_FOR_RENDER)
00632               for_render = true;
00633 
00634        /* Round the allocated size up to a power of two number of pages. */
00635        bucket = drm_intel_gem_bo_bucket_for_size(bufmgr_gem, size);
00636 
00637        /* If we don't have caching at this size, don't actually round the
00638         * allocation up.
00639         */
00640        if (bucket == NULL) {
00641               bo_size = size;
00642               if (bo_size < page_size)
00643                      bo_size = page_size;
00644        } else {
00645               bo_size = bucket->size;
00646        }
00647 
00648        pthread_mutex_lock(&bufmgr_gem->lock);
00649        /* Get a buffer out of the cache if available */
00650 retry:
00651        alloc_from_cache = false;
00652        if (bucket != NULL && !DRMLISTEMPTY(&bucket->head)) {
00653               if (for_render) {
00654                      /* Allocate new render-target BOs from the tail (MRU)
00655                       * of the list, as it will likely be hot in the GPU
00656                       * cache and in the aperture for us.
00657                       */
00658                      bo_gem = DRMLISTENTRY(drm_intel_bo_gem,
00659                                          bucket->head.prev, head);
00660                      DRMLISTDEL(&bo_gem->head);
00661                      alloc_from_cache = true;
00662               } else {
00663                      /* For non-render-target BOs (where we're probably
00664                       * going to map it first thing in order to fill it
00665                       * with data), check if the last BO in the cache is
00666                       * unbusy, and only reuse in that case. Otherwise,
00667                       * allocating a new buffer is probably faster than
00668                       * waiting for the GPU to finish.
00669                       */
00670                      bo_gem = DRMLISTENTRY(drm_intel_bo_gem,
00671                                          bucket->head.next, head);
00672                      if (!drm_intel_gem_bo_busy(&bo_gem->bo)) {
00673                             alloc_from_cache = true;
00674                             DRMLISTDEL(&bo_gem->head);
00675                      }
00676               }
00677 
00678               if (alloc_from_cache) {
00679                      if (!drm_intel_gem_bo_madvise_internal
00680                          (bufmgr_gem, bo_gem, I915_MADV_WILLNEED)) {
00681                             drm_intel_gem_bo_free(&bo_gem->bo);
00682                             drm_intel_gem_bo_cache_purge_bucket(bufmgr_gem,
00683                                                             bucket);
00684                             goto retry;
00685                      }
00686 
00687                      if (drm_intel_gem_bo_set_tiling_internal(&bo_gem->bo,
00688                                                          tiling_mode,
00689                                                          stride)) {
00690                             drm_intel_gem_bo_free(&bo_gem->bo);
00691                             goto retry;
00692                      }
00693               }
00694        }
00695        pthread_mutex_unlock(&bufmgr_gem->lock);
00696 
00697        if (!alloc_from_cache) {
00698               struct drm_i915_gem_create create;
00699 
00700               bo_gem = calloc(1, sizeof(*bo_gem));
00701               if (!bo_gem)
00702                      return NULL;
00703 
00704               bo_gem->bo.size = bo_size;
00705 
00706               VG_CLEAR(create);
00707               create.size = bo_size;
00708 
00709               ret = drmIoctl(bufmgr_gem->fd,
00710                             DRM_IOCTL_I915_GEM_CREATE,
00711                             &create);
00712               bo_gem->gem_handle = create.handle;
00713               bo_gem->bo.handle = bo_gem->gem_handle;
00714               if (ret != 0) {
00715                      free(bo_gem);
00716                      return NULL;
00717               }
00718               bo_gem->bo.bufmgr = bufmgr;
00719 
00720               bo_gem->tiling_mode = I915_TILING_NONE;
00721               bo_gem->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
00722               bo_gem->stride = 0;
00723 
00724               if (drm_intel_gem_bo_set_tiling_internal(&bo_gem->bo,
00725                                                   tiling_mode,
00726                                                   stride)) {
00727                   drm_intel_gem_bo_free(&bo_gem->bo);
00728                   return NULL;
00729               }
00730 
00731               DRMINITLISTHEAD(&bo_gem->name_list);
00732               DRMINITLISTHEAD(&bo_gem->vma_list);
00733        }
00734 
00735        bo_gem->name = name;
00736        atomic_set(&bo_gem->refcount, 1);
00737        bo_gem->validate_index = -1;
00738        bo_gem->reloc_tree_fences = 0;
00739        bo_gem->used_as_reloc_target = false;
00740        bo_gem->has_error = false;
00741        bo_gem->reusable = true;
00742        bo_gem->aub_annotations = NULL;
00743        bo_gem->aub_annotation_count = 0;
00744 
00745        drm_intel_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem);
00746 
00747        DBG("bo_create: buf %d (%s) %ldb\n",
00748            bo_gem->gem_handle, bo_gem->name, size);
00749 
00750        return &bo_gem->bo;
00751 }
00752 
00753 static drm_intel_bo *
00754 drm_intel_gem_bo_alloc_for_render(drm_intel_bufmgr *bufmgr,
00755                               const char *name,
00756                               unsigned long size,
00757                               unsigned int alignment)
00758 {
00759        return drm_intel_gem_bo_alloc_internal(bufmgr, name, size,
00760                                           BO_ALLOC_FOR_RENDER,
00761                                           I915_TILING_NONE, 0);
00762 }
00763 
00764 static drm_intel_bo *
00765 drm_intel_gem_bo_alloc(drm_intel_bufmgr *bufmgr,
00766                      const char *name,
00767                      unsigned long size,
00768                      unsigned int alignment)
00769 {
00770        return drm_intel_gem_bo_alloc_internal(bufmgr, name, size, 0,
00771                                           I915_TILING_NONE, 0);
00772 }
00773 
00774 static drm_intel_bo *
00775 drm_intel_gem_bo_alloc_tiled(drm_intel_bufmgr *bufmgr, const char *name,
00776                           int x, int y, int cpp, uint32_t *tiling_mode,
00777                           unsigned long *pitch, unsigned long flags)
00778 {
00779        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *)bufmgr;
00780        unsigned long size, stride;
00781        uint32_t tiling;
00782 
00783        do {
00784               unsigned long aligned_y, height_alignment;
00785 
00786               tiling = *tiling_mode;
00787 
00788               /* If we're tiled, our allocations are in 8 or 32-row blocks,
00789                * so failure to align our height means that we won't allocate
00790                * enough pages.
00791                *
00792                * If we're untiled, we still have to align to 2 rows high
00793                * because the data port accesses 2x2 blocks even if the
00794                * bottom row isn't to be rendered, so failure to align means
00795                * we could walk off the end of the GTT and fault.  This is
00796                * documented on 965, and may be the case on older chipsets
00797                * too so we try to be careful.
00798                */
00799               aligned_y = y;
00800               height_alignment = 2;
00801 
00802               if ((bufmgr_gem->gen == 2) && tiling != I915_TILING_NONE)
00803                      height_alignment = 16;
00804               else if (tiling == I915_TILING_X
00805                      || (IS_915(bufmgr_gem->pci_device)
00806                          && tiling == I915_TILING_Y))
00807                      height_alignment = 8;
00808               else if (tiling == I915_TILING_Y)
00809                      height_alignment = 32;
00810               aligned_y = ALIGN(y, height_alignment);
00811 
00812               stride = x * cpp;
00813               stride = drm_intel_gem_bo_tile_pitch(bufmgr_gem, stride, tiling_mode);
00814               size = stride * aligned_y;
00815               size = drm_intel_gem_bo_tile_size(bufmgr_gem, size, tiling_mode);
00816        } while (*tiling_mode != tiling);
00817        *pitch = stride;
00818 
00819        if (tiling == I915_TILING_NONE)
00820               stride = 0;
00821 
00822        return drm_intel_gem_bo_alloc_internal(bufmgr, name, size, flags,
00823                                           tiling, stride);
00824 }
00825 
00832 drm_intel_bo *
00833 drm_intel_bo_gem_create_from_name(drm_intel_bufmgr *bufmgr,
00834                               const char *name,
00835                               unsigned int handle)
00836 {
00837        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bufmgr;
00838        drm_intel_bo_gem *bo_gem;
00839        int ret;
00840        struct drm_gem_open open_arg;
00841        struct drm_i915_gem_get_tiling get_tiling;
00842        drmMMListHead *list;
00843 
00844        /* At the moment most applications only have a few named bo.
00845         * For instance, in a DRI client only the render buffers passed
00846         * between X and the client are named. And since X returns the
00847         * alternating names for the front/back buffer a linear search
00848         * provides a sufficiently fast match.
00849         */
00850        for (list = bufmgr_gem->named.next;
00851             list != &bufmgr_gem->named;
00852             list = list->next) {
00853               bo_gem = DRMLISTENTRY(drm_intel_bo_gem, list, name_list);
00854               if (bo_gem->global_name == handle) {
00855                      drm_intel_gem_bo_reference(&bo_gem->bo);
00856                      return &bo_gem->bo;
00857               }
00858        }
00859 
00860        bo_gem = calloc(1, sizeof(*bo_gem));
00861        if (!bo_gem)
00862               return NULL;
00863 
00864        VG_CLEAR(open_arg);
00865        open_arg.name = handle;
00866        ret = drmIoctl(bufmgr_gem->fd,
00867                      DRM_IOCTL_GEM_OPEN,
00868                      &open_arg);
00869        if (ret != 0) {
00870               DBG("Couldn't reference %s handle 0x%08x: %s\n",
00871                   name, handle, strerror(errno));
00872               free(bo_gem);
00873               return NULL;
00874        }
00875        bo_gem->bo.size = open_arg.size;
00876        bo_gem->bo.offset = 0;
00877        bo_gem->bo.virtual = NULL;
00878        bo_gem->bo.bufmgr = bufmgr;
00879        bo_gem->name = name;
00880        atomic_set(&bo_gem->refcount, 1);
00881        bo_gem->validate_index = -1;
00882        bo_gem->gem_handle = open_arg.handle;
00883        bo_gem->bo.handle = open_arg.handle;
00884        bo_gem->global_name = handle;
00885        bo_gem->reusable = false;
00886 
00887        VG_CLEAR(get_tiling);
00888        get_tiling.handle = bo_gem->gem_handle;
00889        ret = drmIoctl(bufmgr_gem->fd,
00890                      DRM_IOCTL_I915_GEM_GET_TILING,
00891                      &get_tiling);
00892        if (ret != 0) {
00893               drm_intel_gem_bo_unreference(&bo_gem->bo);
00894               return NULL;
00895        }
00896        bo_gem->tiling_mode = get_tiling.tiling_mode;
00897        bo_gem->swizzle_mode = get_tiling.swizzle_mode;
00898        /* XXX stride is unknown */
00899        drm_intel_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem);
00900 
00901        DRMINITLISTHEAD(&bo_gem->vma_list);
00902        DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
00903        DBG("bo_create_from_handle: %d (%s)\n", handle, bo_gem->name);
00904 
00905        return &bo_gem->bo;
00906 }
00907 
00908 static void
00909 drm_intel_gem_bo_free(drm_intel_bo *bo)
00910 {
00911        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
00912        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
00913        struct drm_gem_close close;
00914        int ret;
00915 
00916        DRMLISTDEL(&bo_gem->vma_list);
00917        if (bo_gem->mem_virtual) {
00918               VG(VALGRIND_FREELIKE_BLOCK(bo_gem->mem_virtual, 0));
00919               munmap(bo_gem->mem_virtual, bo_gem->bo.size);
00920               bufmgr_gem->vma_count--;
00921        }
00922        if (bo_gem->gtt_virtual) {
00923               munmap(bo_gem->gtt_virtual, bo_gem->bo.size);
00924               bufmgr_gem->vma_count--;
00925        }
00926 
00927        /* Close this object */
00928        VG_CLEAR(close);
00929        close.handle = bo_gem->gem_handle;
00930        ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_CLOSE, &close);
00931        if (ret != 0) {
00932               DBG("DRM_IOCTL_GEM_CLOSE %d failed (%s): %s\n",
00933                   bo_gem->gem_handle, bo_gem->name, strerror(errno));
00934        }
00935        free(bo_gem->aub_annotations);
00936        free(bo);
00937 }
00938 
00939 static void
00940 drm_intel_gem_bo_mark_mmaps_incoherent(drm_intel_bo *bo)
00941 {
00942 #if HAVE_VALGRIND
00943        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
00944 
00945        if (bo_gem->mem_virtual)
00946               VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_virtual, bo->size);
00947 
00948        if (bo_gem->gtt_virtual)
00949               VALGRIND_MAKE_MEM_NOACCESS(bo_gem->gtt_virtual, bo->size);
00950 #endif
00951 }
00952 
00954 static void
00955 drm_intel_gem_cleanup_bo_cache(drm_intel_bufmgr_gem *bufmgr_gem, time_t time)
00956 {
00957        int i;
00958 
00959        if (bufmgr_gem->time == time)
00960               return;
00961 
00962        for (i = 0; i < bufmgr_gem->num_buckets; i++) {
00963               struct drm_intel_gem_bo_bucket *bucket =
00964                   &bufmgr_gem->cache_bucket[i];
00965 
00966               while (!DRMLISTEMPTY(&bucket->head)) {
00967                      drm_intel_bo_gem *bo_gem;
00968 
00969                      bo_gem = DRMLISTENTRY(drm_intel_bo_gem,
00970                                          bucket->head.next, head);
00971                      if (time - bo_gem->free_time <= 1)
00972                             break;
00973 
00974                      DRMLISTDEL(&bo_gem->head);
00975 
00976                      drm_intel_gem_bo_free(&bo_gem->bo);
00977               }
00978        }
00979 
00980        bufmgr_gem->time = time;
00981 }
00982 
00983 static void drm_intel_gem_bo_purge_vma_cache(drm_intel_bufmgr_gem *bufmgr_gem)
00984 {
00985        int limit;
00986 
00987        DBG("%s: cached=%d, open=%d, limit=%d\n", __FUNCTION__,
00988            bufmgr_gem->vma_count, bufmgr_gem->vma_open, bufmgr_gem->vma_max);
00989 
00990        if (bufmgr_gem->vma_max < 0)
00991               return;
00992 
00993        /* We may need to evict a few entries in order to create new mmaps */
00994        limit = bufmgr_gem->vma_max - 2*bufmgr_gem->vma_open;
00995        if (limit < 0)
00996               limit = 0;
00997 
00998        while (bufmgr_gem->vma_count > limit) {
00999               drm_intel_bo_gem *bo_gem;
01000 
01001               bo_gem = DRMLISTENTRY(drm_intel_bo_gem,
01002                                   bufmgr_gem->vma_cache.next,
01003                                   vma_list);
01004               assert(bo_gem->map_count == 0);
01005               DRMLISTDELINIT(&bo_gem->vma_list);
01006 
01007               if (bo_gem->mem_virtual) {
01008                      munmap(bo_gem->mem_virtual, bo_gem->bo.size);
01009                      bo_gem->mem_virtual = NULL;
01010                      bufmgr_gem->vma_count--;
01011               }
01012               if (bo_gem->gtt_virtual) {
01013                      munmap(bo_gem->gtt_virtual, bo_gem->bo.size);
01014                      bo_gem->gtt_virtual = NULL;
01015                      bufmgr_gem->vma_count--;
01016               }
01017        }
01018 }
01019 
01020 static void drm_intel_gem_bo_close_vma(drm_intel_bufmgr_gem *bufmgr_gem,
01021                                    drm_intel_bo_gem *bo_gem)
01022 {
01023        bufmgr_gem->vma_open--;
01024        DRMLISTADDTAIL(&bo_gem->vma_list, &bufmgr_gem->vma_cache);
01025        if (bo_gem->mem_virtual)
01026               bufmgr_gem->vma_count++;
01027        if (bo_gem->gtt_virtual)
01028               bufmgr_gem->vma_count++;
01029        drm_intel_gem_bo_purge_vma_cache(bufmgr_gem);
01030 }
01031 
01032 static void drm_intel_gem_bo_open_vma(drm_intel_bufmgr_gem *bufmgr_gem,
01033                                   drm_intel_bo_gem *bo_gem)
01034 {
01035        bufmgr_gem->vma_open++;
01036        DRMLISTDEL(&bo_gem->vma_list);
01037        if (bo_gem->mem_virtual)
01038               bufmgr_gem->vma_count--;
01039        if (bo_gem->gtt_virtual)
01040               bufmgr_gem->vma_count--;
01041        drm_intel_gem_bo_purge_vma_cache(bufmgr_gem);
01042 }
01043 
01044 static void
01045 drm_intel_gem_bo_unreference_final(drm_intel_bo *bo, time_t time)
01046 {
01047        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
01048        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01049        struct drm_intel_gem_bo_bucket *bucket;
01050        int i;
01051 
01052        /* Unreference all the target buffers */
01053        for (i = 0; i < bo_gem->reloc_count; i++) {
01054               if (bo_gem->reloc_target_info[i].bo != bo) {
01055                      drm_intel_gem_bo_unreference_locked_timed(bo_gem->
01056                                                           reloc_target_info[i].bo,
01057                                                           time);
01058               }
01059        }
01060        bo_gem->reloc_count = 0;
01061        bo_gem->used_as_reloc_target = false;
01062 
01063        DBG("bo_unreference final: %d (%s)\n",
01064            bo_gem->gem_handle, bo_gem->name);
01065 
01066        /* release memory associated with this object */
01067        if (bo_gem->reloc_target_info) {
01068               free(bo_gem->reloc_target_info);
01069               bo_gem->reloc_target_info = NULL;
01070        }
01071        if (bo_gem->relocs) {
01072               free(bo_gem->relocs);
01073               bo_gem->relocs = NULL;
01074        }
01075 
01076        /* Clear any left-over mappings */
01077        if (bo_gem->map_count) {
01078               DBG("bo freed with non-zero map-count %d\n", bo_gem->map_count);
01079               bo_gem->map_count = 0;
01080               drm_intel_gem_bo_close_vma(bufmgr_gem, bo_gem);
01081               drm_intel_gem_bo_mark_mmaps_incoherent(bo);
01082        }
01083 
01084        DRMLISTDEL(&bo_gem->name_list);
01085 
01086        bucket = drm_intel_gem_bo_bucket_for_size(bufmgr_gem, bo->size);
01087        /* Put the buffer into our internal cache for reuse if we can. */
01088        if (bufmgr_gem->bo_reuse && bo_gem->reusable && bucket != NULL &&
01089            drm_intel_gem_bo_madvise_internal(bufmgr_gem, bo_gem,
01090                                          I915_MADV_DONTNEED)) {
01091               bo_gem->free_time = time;
01092 
01093               bo_gem->name = NULL;
01094               bo_gem->validate_index = -1;
01095 
01096               DRMLISTADDTAIL(&bo_gem->head, &bucket->head);
01097        } else {
01098               drm_intel_gem_bo_free(bo);
01099        }
01100 }
01101 
01102 static void drm_intel_gem_bo_unreference_locked_timed(drm_intel_bo *bo,
01103                                                 time_t time)
01104 {
01105        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01106 
01107        assert(atomic_read(&bo_gem->refcount) > 0);
01108        if (atomic_dec_and_test(&bo_gem->refcount))
01109               drm_intel_gem_bo_unreference_final(bo, time);
01110 }
01111 
01112 static void drm_intel_gem_bo_unreference(drm_intel_bo *bo)
01113 {
01114        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01115 
01116        assert(atomic_read(&bo_gem->refcount) > 0);
01117        if (atomic_dec_and_test(&bo_gem->refcount)) {
01118               drm_intel_bufmgr_gem *bufmgr_gem =
01119                   (drm_intel_bufmgr_gem *) bo->bufmgr;
01120               struct timespec time;
01121 
01122               clock_gettime(CLOCK_MONOTONIC, &time);
01123 
01124               pthread_mutex_lock(&bufmgr_gem->lock);
01125               drm_intel_gem_bo_unreference_final(bo, time.tv_sec);
01126               drm_intel_gem_cleanup_bo_cache(bufmgr_gem, time.tv_sec);
01127               pthread_mutex_unlock(&bufmgr_gem->lock);
01128        }
01129 }
01130 
01131 static int drm_intel_gem_bo_map(drm_intel_bo *bo, int write_enable)
01132 {
01133        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
01134        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01135        struct drm_i915_gem_set_domain set_domain;
01136        int ret;
01137 
01138        pthread_mutex_lock(&bufmgr_gem->lock);
01139 
01140        if (bo_gem->map_count++ == 0)
01141               drm_intel_gem_bo_open_vma(bufmgr_gem, bo_gem);
01142 
01143        if (!bo_gem->mem_virtual) {
01144               struct drm_i915_gem_mmap mmap_arg;
01145 
01146               DBG("bo_map: %d (%s), map_count=%d\n",
01147                   bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
01148 
01149               VG_CLEAR(mmap_arg);
01150               mmap_arg.handle = bo_gem->gem_handle;
01151               mmap_arg.offset = 0;
01152               mmap_arg.size = bo->size;
01153               ret = drmIoctl(bufmgr_gem->fd,
01154                             DRM_IOCTL_I915_GEM_MMAP,
01155                             &mmap_arg);
01156               if (ret != 0) {
01157                      ret = -errno;
01158                      DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
01159                          __FILE__, __LINE__, bo_gem->gem_handle,
01160                          bo_gem->name, strerror(errno));
01161                      if (--bo_gem->map_count == 0)
01162                             drm_intel_gem_bo_close_vma(bufmgr_gem, bo_gem);
01163                      pthread_mutex_unlock(&bufmgr_gem->lock);
01164                      return ret;
01165               }
01166               VG(VALGRIND_MALLOCLIKE_BLOCK(mmap_arg.addr_ptr, mmap_arg.size, 0, 1));
01167               bo_gem->mem_virtual = (void *)(uintptr_t) mmap_arg.addr_ptr;
01168        }
01169        DBG("bo_map: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
01170            bo_gem->mem_virtual);
01171        bo->virtual = bo_gem->mem_virtual;
01172 
01173        VG_CLEAR(set_domain);
01174        set_domain.handle = bo_gem->gem_handle;
01175        set_domain.read_domains = I915_GEM_DOMAIN_CPU;
01176        if (write_enable)
01177               set_domain.write_domain = I915_GEM_DOMAIN_CPU;
01178        else
01179               set_domain.write_domain = 0;
01180        ret = drmIoctl(bufmgr_gem->fd,
01181                      DRM_IOCTL_I915_GEM_SET_DOMAIN,
01182                      &set_domain);
01183        if (ret != 0) {
01184               DBG("%s:%d: Error setting to CPU domain %d: %s\n",
01185                   __FILE__, __LINE__, bo_gem->gem_handle,
01186                   strerror(errno));
01187        }
01188 
01189        if (write_enable)
01190               bo_gem->mapped_cpu_write = true;
01191 
01192        drm_intel_gem_bo_mark_mmaps_incoherent(bo);
01193        VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->mem_virtual, bo->size));
01194        pthread_mutex_unlock(&bufmgr_gem->lock);
01195 
01196        return 0;
01197 }
01198 
01199 static int
01200 map_gtt(drm_intel_bo *bo)
01201 {
01202        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
01203        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01204        int ret;
01205 
01206        if (bo_gem->map_count++ == 0)
01207               drm_intel_gem_bo_open_vma(bufmgr_gem, bo_gem);
01208 
01209        /* Get a mapping of the buffer if we haven't before. */
01210        if (bo_gem->gtt_virtual == NULL) {
01211               struct drm_i915_gem_mmap_gtt mmap_arg;
01212 
01213               DBG("bo_map_gtt: mmap %d (%s), map_count=%d\n",
01214                   bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
01215 
01216               VG_CLEAR(mmap_arg);
01217               mmap_arg.handle = bo_gem->gem_handle;
01218 
01219               /* Get the fake offset back... */
01220               ret = drmIoctl(bufmgr_gem->fd,
01221                             DRM_IOCTL_I915_GEM_MMAP_GTT,
01222                             &mmap_arg);
01223               if (ret != 0) {
01224                      ret = -errno;
01225                      DBG("%s:%d: Error preparing buffer map %d (%s): %s .\n",
01226                          __FILE__, __LINE__,
01227                          bo_gem->gem_handle, bo_gem->name,
01228                          strerror(errno));
01229                      if (--bo_gem->map_count == 0)
01230                             drm_intel_gem_bo_close_vma(bufmgr_gem, bo_gem);
01231                      return ret;
01232               }
01233 
01234               /* and mmap it */
01235               bo_gem->gtt_virtual = mmap(0, bo->size, PROT_READ | PROT_WRITE,
01236                                       MAP_SHARED, bufmgr_gem->fd,
01237                                       mmap_arg.offset);
01238               if (bo_gem->gtt_virtual == MAP_FAILED) {
01239                      bo_gem->gtt_virtual = NULL;
01240                      ret = -errno;
01241                      DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
01242                          __FILE__, __LINE__,
01243                          bo_gem->gem_handle, bo_gem->name,
01244                          strerror(errno));
01245                      if (--bo_gem->map_count == 0)
01246                             drm_intel_gem_bo_close_vma(bufmgr_gem, bo_gem);
01247                      return ret;
01248               }
01249        }
01250 
01251        bo->virtual = bo_gem->gtt_virtual;
01252 
01253        DBG("bo_map_gtt: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
01254            bo_gem->gtt_virtual);
01255 
01256        return 0;
01257 }
01258 
01259 int drm_intel_gem_bo_map_gtt(drm_intel_bo *bo)
01260 {
01261        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
01262        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01263        struct drm_i915_gem_set_domain set_domain;
01264        int ret;
01265 
01266        pthread_mutex_lock(&bufmgr_gem->lock);
01267 
01268        ret = map_gtt(bo);
01269        if (ret) {
01270               pthread_mutex_unlock(&bufmgr_gem->lock);
01271               return ret;
01272        }
01273 
01274        /* Now move it to the GTT domain so that the GPU and CPU
01275         * caches are flushed and the GPU isn't actively using the
01276         * buffer.
01277         *
01278         * The pagefault handler does this domain change for us when
01279         * it has unbound the BO from the GTT, but it's up to us to
01280         * tell it when we're about to use things if we had done
01281         * rendering and it still happens to be bound to the GTT.
01282         */
01283        VG_CLEAR(set_domain);
01284        set_domain.handle = bo_gem->gem_handle;
01285        set_domain.read_domains = I915_GEM_DOMAIN_GTT;
01286        set_domain.write_domain = I915_GEM_DOMAIN_GTT;
01287        ret = drmIoctl(bufmgr_gem->fd,
01288                      DRM_IOCTL_I915_GEM_SET_DOMAIN,
01289                      &set_domain);
01290        if (ret != 0) {
01291               DBG("%s:%d: Error setting domain %d: %s\n",
01292                   __FILE__, __LINE__, bo_gem->gem_handle,
01293                   strerror(errno));
01294        }
01295 
01296        drm_intel_gem_bo_mark_mmaps_incoherent(bo);
01297        VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->gtt_virtual, bo->size));
01298        pthread_mutex_unlock(&bufmgr_gem->lock);
01299 
01300        return 0;
01301 }
01302 
01317 int drm_intel_gem_bo_map_unsynchronized(drm_intel_bo *bo)
01318 {
01319        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
01320        int ret;
01321 
01322        /* If the CPU cache isn't coherent with the GTT, then use a
01323         * regular synchronized mapping.  The problem is that we don't
01324         * track where the buffer was last used on the CPU side in
01325         * terms of drm_intel_bo_map vs drm_intel_gem_bo_map_gtt, so
01326         * we would potentially corrupt the buffer even when the user
01327         * does reasonable things.
01328         */
01329        if (!bufmgr_gem->has_llc)
01330               return drm_intel_gem_bo_map_gtt(bo);
01331 
01332        pthread_mutex_lock(&bufmgr_gem->lock);
01333        ret = map_gtt(bo);
01334        pthread_mutex_unlock(&bufmgr_gem->lock);
01335 
01336        return ret;
01337 }
01338 
01339 static int drm_intel_gem_bo_unmap(drm_intel_bo *bo)
01340 {
01341        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
01342        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01343        int ret = 0;
01344 
01345        if (bo == NULL)
01346               return 0;
01347 
01348        pthread_mutex_lock(&bufmgr_gem->lock);
01349 
01350        if (bo_gem->map_count <= 0) {
01351               DBG("attempted to unmap an unmapped bo\n");
01352               pthread_mutex_unlock(&bufmgr_gem->lock);
01353               /* Preserve the old behaviour of just treating this as a
01354                * no-op rather than reporting the error.
01355                */
01356               return 0;
01357        }
01358 
01359        if (bo_gem->mapped_cpu_write) {
01360               struct drm_i915_gem_sw_finish sw_finish;
01361 
01362               /* Cause a flush to happen if the buffer's pinned for
01363                * scanout, so the results show up in a timely manner.
01364                * Unlike GTT set domains, this only does work if the
01365                * buffer should be scanout-related.
01366                */
01367               VG_CLEAR(sw_finish);
01368               sw_finish.handle = bo_gem->gem_handle;
01369               ret = drmIoctl(bufmgr_gem->fd,
01370                             DRM_IOCTL_I915_GEM_SW_FINISH,
01371                             &sw_finish);
01372               ret = ret == -1 ? -errno : 0;
01373 
01374               bo_gem->mapped_cpu_write = false;
01375        }
01376 
01377        /* We need to unmap after every innovation as we cannot track
01378         * an open vma for every bo as that will exhaasut the system
01379         * limits and cause later failures.
01380         */
01381        if (--bo_gem->map_count == 0) {
01382               drm_intel_gem_bo_close_vma(bufmgr_gem, bo_gem);
01383               drm_intel_gem_bo_mark_mmaps_incoherent(bo);
01384               bo->virtual = NULL;
01385        }
01386        pthread_mutex_unlock(&bufmgr_gem->lock);
01387 
01388        return ret;
01389 }
01390 
01391 int drm_intel_gem_bo_unmap_gtt(drm_intel_bo *bo)
01392 {
01393        return drm_intel_gem_bo_unmap(bo);
01394 }
01395 
01396 static int
01397 drm_intel_gem_bo_subdata(drm_intel_bo *bo, unsigned long offset,
01398                       unsigned long size, const void *data)
01399 {
01400        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
01401        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01402        struct drm_i915_gem_pwrite pwrite;
01403        int ret;
01404 
01405        VG_CLEAR(pwrite);
01406        pwrite.handle = bo_gem->gem_handle;
01407        pwrite.offset = offset;
01408        pwrite.size = size;
01409        pwrite.data_ptr = (uint64_t) (uintptr_t) data;
01410        ret = drmIoctl(bufmgr_gem->fd,
01411                      DRM_IOCTL_I915_GEM_PWRITE,
01412                      &pwrite);
01413        if (ret != 0) {
01414               ret = -errno;
01415               DBG("%s:%d: Error writing data to buffer %d: (%d %d) %s .\n",
01416                   __FILE__, __LINE__, bo_gem->gem_handle, (int)offset,
01417                   (int)size, strerror(errno));
01418        }
01419 
01420        return ret;
01421 }
01422 
01423 static int
01424 drm_intel_gem_get_pipe_from_crtc_id(drm_intel_bufmgr *bufmgr, int crtc_id)
01425 {
01426        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bufmgr;
01427        struct drm_i915_get_pipe_from_crtc_id get_pipe_from_crtc_id;
01428        int ret;
01429 
01430        VG_CLEAR(get_pipe_from_crtc_id);
01431        get_pipe_from_crtc_id.crtc_id = crtc_id;
01432        ret = drmIoctl(bufmgr_gem->fd,
01433                      DRM_IOCTL_I915_GET_PIPE_FROM_CRTC_ID,
01434                      &get_pipe_from_crtc_id);
01435        if (ret != 0) {
01436               /* We return -1 here to signal that we don't
01437                * know which pipe is associated with this crtc.
01438                * This lets the caller know that this information
01439                * isn't available; using the wrong pipe for
01440                * vblank waiting can cause the chipset to lock up
01441                */
01442               return -1;
01443        }
01444 
01445        return get_pipe_from_crtc_id.pipe;
01446 }
01447 
01448 static int
01449 drm_intel_gem_bo_get_subdata(drm_intel_bo *bo, unsigned long offset,
01450                           unsigned long size, void *data)
01451 {
01452        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
01453        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01454        struct drm_i915_gem_pread pread;
01455        int ret;
01456 
01457        VG_CLEAR(pread);
01458        pread.handle = bo_gem->gem_handle;
01459        pread.offset = offset;
01460        pread.size = size;
01461        pread.data_ptr = (uint64_t) (uintptr_t) data;
01462        ret = drmIoctl(bufmgr_gem->fd,
01463                      DRM_IOCTL_I915_GEM_PREAD,
01464                      &pread);
01465        if (ret != 0) {
01466               ret = -errno;
01467               DBG("%s:%d: Error reading data from buffer %d: (%d %d) %s .\n",
01468                   __FILE__, __LINE__, bo_gem->gem_handle, (int)offset,
01469                   (int)size, strerror(errno));
01470        }
01471 
01472        return ret;
01473 }
01474 
01476 static void
01477 drm_intel_gem_bo_wait_rendering(drm_intel_bo *bo)
01478 {
01479        drm_intel_gem_bo_start_gtt_access(bo, 1);
01480 }
01481 
01506 int drm_intel_gem_bo_wait(drm_intel_bo *bo, int64_t timeout_ns)
01507 {
01508        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
01509        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01510        struct drm_i915_gem_wait wait;
01511        int ret;
01512 
01513        if (!bufmgr_gem->has_wait_timeout) {
01514               DBG("%s:%d: Timed wait is not supported. Falling back to "
01515                   "infinite wait\n", __FILE__, __LINE__);
01516               if (timeout_ns) {
01517                      drm_intel_gem_bo_wait_rendering(bo);
01518                      return 0;
01519               } else {
01520                      return drm_intel_gem_bo_busy(bo) ? -ETIME : 0;
01521               }
01522        }
01523 
01524        wait.bo_handle = bo_gem->gem_handle;
01525        wait.timeout_ns = timeout_ns;
01526        wait.flags = 0;
01527        ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
01528        if (ret == -1)
01529               return -errno;
01530 
01531        return ret;
01532 }
01533 
01541 void
01542 drm_intel_gem_bo_start_gtt_access(drm_intel_bo *bo, int write_enable)
01543 {
01544        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
01545        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01546        struct drm_i915_gem_set_domain set_domain;
01547        int ret;
01548 
01549        VG_CLEAR(set_domain);
01550        set_domain.handle = bo_gem->gem_handle;
01551        set_domain.read_domains = I915_GEM_DOMAIN_GTT;
01552        set_domain.write_domain = write_enable ? I915_GEM_DOMAIN_GTT : 0;
01553        ret = drmIoctl(bufmgr_gem->fd,
01554                      DRM_IOCTL_I915_GEM_SET_DOMAIN,
01555                      &set_domain);
01556        if (ret != 0) {
01557               DBG("%s:%d: Error setting memory domains %d (%08x %08x): %s .\n",
01558                   __FILE__, __LINE__, bo_gem->gem_handle,
01559                   set_domain.read_domains, set_domain.write_domain,
01560                   strerror(errno));
01561        }
01562 }
01563 
01564 static void
01565 drm_intel_bufmgr_gem_destroy(drm_intel_bufmgr *bufmgr)
01566 {
01567        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bufmgr;
01568        int i;
01569 
01570        free(bufmgr_gem->exec2_objects);
01571        free(bufmgr_gem->exec_objects);
01572        free(bufmgr_gem->exec_bos);
01573 
01574        pthread_mutex_destroy(&bufmgr_gem->lock);
01575 
01576        /* Free any cached buffer objects we were going to reuse */
01577        for (i = 0; i < bufmgr_gem->num_buckets; i++) {
01578               struct drm_intel_gem_bo_bucket *bucket =
01579                   &bufmgr_gem->cache_bucket[i];
01580               drm_intel_bo_gem *bo_gem;
01581 
01582               while (!DRMLISTEMPTY(&bucket->head)) {
01583                      bo_gem = DRMLISTENTRY(drm_intel_bo_gem,
01584                                          bucket->head.next, head);
01585                      DRMLISTDEL(&bo_gem->head);
01586 
01587                      drm_intel_gem_bo_free(&bo_gem->bo);
01588               }
01589        }
01590 
01591        free(bufmgr);
01592 }
01593 
01603 static int
01604 do_bo_emit_reloc(drm_intel_bo *bo, uint32_t offset,
01605                drm_intel_bo *target_bo, uint32_t target_offset,
01606                uint32_t read_domains, uint32_t write_domain,
01607                bool need_fence)
01608 {
01609        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
01610        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01611        drm_intel_bo_gem *target_bo_gem = (drm_intel_bo_gem *) target_bo;
01612        bool fenced_command;
01613 
01614        if (bo_gem->has_error)
01615               return -ENOMEM;
01616 
01617        if (target_bo_gem->has_error) {
01618               bo_gem->has_error = true;
01619               return -ENOMEM;
01620        }
01621 
01622        /* We never use HW fences for rendering on 965+ */
01623        if (bufmgr_gem->gen >= 4)
01624               need_fence = false;
01625 
01626        fenced_command = need_fence;
01627        if (target_bo_gem->tiling_mode == I915_TILING_NONE)
01628               need_fence = false;
01629 
01630        /* Create a new relocation list if needed */
01631        if (bo_gem->relocs == NULL && drm_intel_setup_reloc_list(bo))
01632               return -ENOMEM;
01633 
01634        /* Check overflow */
01635        assert(bo_gem->reloc_count < bufmgr_gem->max_relocs);
01636 
01637        /* Check args */
01638        assert(offset <= bo->size - 4);
01639        assert((write_domain & (write_domain - 1)) == 0);
01640 
01641        /* Make sure that we're not adding a reloc to something whose size has
01642         * already been accounted for.
01643         */
01644        assert(!bo_gem->used_as_reloc_target);
01645        if (target_bo_gem != bo_gem) {
01646               target_bo_gem->used_as_reloc_target = true;
01647               bo_gem->reloc_tree_size += target_bo_gem->reloc_tree_size;
01648        }
01649        /* An object needing a fence is a tiled buffer, so it won't have
01650         * relocs to other buffers.
01651         */
01652        if (need_fence)
01653               target_bo_gem->reloc_tree_fences = 1;
01654        bo_gem->reloc_tree_fences += target_bo_gem->reloc_tree_fences;
01655 
01656        bo_gem->relocs[bo_gem->reloc_count].offset = offset;
01657        bo_gem->relocs[bo_gem->reloc_count].delta = target_offset;
01658        bo_gem->relocs[bo_gem->reloc_count].target_handle =
01659            target_bo_gem->gem_handle;
01660        bo_gem->relocs[bo_gem->reloc_count].read_domains = read_domains;
01661        bo_gem->relocs[bo_gem->reloc_count].write_domain = write_domain;
01662        bo_gem->relocs[bo_gem->reloc_count].presumed_offset = target_bo->offset;
01663 
01664        bo_gem->reloc_target_info[bo_gem->reloc_count].bo = target_bo;
01665        if (target_bo != bo)
01666               drm_intel_gem_bo_reference(target_bo);
01667        if (fenced_command)
01668               bo_gem->reloc_target_info[bo_gem->reloc_count].flags =
01669                      DRM_INTEL_RELOC_FENCE;
01670        else
01671               bo_gem->reloc_target_info[bo_gem->reloc_count].flags = 0;
01672 
01673        bo_gem->reloc_count++;
01674 
01675        return 0;
01676 }
01677 
01678 static int
01679 drm_intel_gem_bo_emit_reloc(drm_intel_bo *bo, uint32_t offset,
01680                          drm_intel_bo *target_bo, uint32_t target_offset,
01681                          uint32_t read_domains, uint32_t write_domain)
01682 {
01683        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *)bo->bufmgr;
01684 
01685        return do_bo_emit_reloc(bo, offset, target_bo, target_offset,
01686                             read_domains, write_domain,
01687                             !bufmgr_gem->fenced_relocs);
01688 }
01689 
01690 static int
01691 drm_intel_gem_bo_emit_reloc_fence(drm_intel_bo *bo, uint32_t offset,
01692                               drm_intel_bo *target_bo,
01693                               uint32_t target_offset,
01694                               uint32_t read_domains, uint32_t write_domain)
01695 {
01696        return do_bo_emit_reloc(bo, offset, target_bo, target_offset,
01697                             read_domains, write_domain, true);
01698 }
01699 
01700 int
01701 drm_intel_gem_bo_get_reloc_count(drm_intel_bo *bo)
01702 {
01703        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01704 
01705        return bo_gem->reloc_count;
01706 }
01707 
01721 void
01722 drm_intel_gem_bo_clear_relocs(drm_intel_bo *bo, int start)
01723 {
01724        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01725        int i;
01726        struct timespec time;
01727 
01728        clock_gettime(CLOCK_MONOTONIC, &time);
01729 
01730        assert(bo_gem->reloc_count >= start);
01731        /* Unreference the cleared target buffers */
01732        for (i = start; i < bo_gem->reloc_count; i++) {
01733               if (bo_gem->reloc_target_info[i].bo != bo) {
01734                      drm_intel_gem_bo_unreference_locked_timed(bo_gem->
01735                                                           reloc_target_info[i].bo,
01736                                                           time.tv_sec);
01737               }
01738        }
01739        bo_gem->reloc_count = start;
01740 }
01741 
01747 static void
01748 drm_intel_gem_bo_process_reloc(drm_intel_bo *bo)
01749 {
01750        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01751        int i;
01752 
01753        if (bo_gem->relocs == NULL)
01754               return;
01755 
01756        for (i = 0; i < bo_gem->reloc_count; i++) {
01757               drm_intel_bo *target_bo = bo_gem->reloc_target_info[i].bo;
01758 
01759               if (target_bo == bo)
01760                      continue;
01761 
01762               drm_intel_gem_bo_mark_mmaps_incoherent(bo);
01763 
01764               /* Continue walking the tree depth-first. */
01765               drm_intel_gem_bo_process_reloc(target_bo);
01766 
01767               /* Add the target to the validate list */
01768               drm_intel_add_validate_buffer(target_bo);
01769        }
01770 }
01771 
01772 static void
01773 drm_intel_gem_bo_process_reloc2(drm_intel_bo *bo)
01774 {
01775        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *)bo;
01776        int i;
01777 
01778        if (bo_gem->relocs == NULL)
01779               return;
01780 
01781        for (i = 0; i < bo_gem->reloc_count; i++) {
01782               drm_intel_bo *target_bo = bo_gem->reloc_target_info[i].bo;
01783               int need_fence;
01784 
01785               if (target_bo == bo)
01786                      continue;
01787 
01788               drm_intel_gem_bo_mark_mmaps_incoherent(bo);
01789 
01790               /* Continue walking the tree depth-first. */
01791               drm_intel_gem_bo_process_reloc2(target_bo);
01792 
01793               need_fence = (bo_gem->reloc_target_info[i].flags &
01794                            DRM_INTEL_RELOC_FENCE);
01795 
01796               /* Add the target to the validate list */
01797               drm_intel_add_validate_buffer2(target_bo, need_fence);
01798        }
01799 }
01800 
01801 
01802 static void
01803 drm_intel_update_buffer_offsets(drm_intel_bufmgr_gem *bufmgr_gem)
01804 {
01805        int i;
01806 
01807        for (i = 0; i < bufmgr_gem->exec_count; i++) {
01808               drm_intel_bo *bo = bufmgr_gem->exec_bos[i];
01809               drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01810 
01811               /* Update the buffer offset */
01812               if (bufmgr_gem->exec_objects[i].offset != bo->offset) {
01813                      DBG("BO %d (%s) migrated: 0x%08lx -> 0x%08llx\n",
01814                          bo_gem->gem_handle, bo_gem->name, bo->offset,
01815                          (unsigned long long)bufmgr_gem->exec_objects[i].
01816                          offset);
01817                      bo->offset = bufmgr_gem->exec_objects[i].offset;
01818               }
01819        }
01820 }
01821 
01822 static void
01823 drm_intel_update_buffer_offsets2 (drm_intel_bufmgr_gem *bufmgr_gem)
01824 {
01825        int i;
01826 
01827        for (i = 0; i < bufmgr_gem->exec_count; i++) {
01828               drm_intel_bo *bo = bufmgr_gem->exec_bos[i];
01829               drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *)bo;
01830 
01831               /* Update the buffer offset */
01832               if (bufmgr_gem->exec2_objects[i].offset != bo->offset) {
01833                      DBG("BO %d (%s) migrated: 0x%08lx -> 0x%08llx\n",
01834                          bo_gem->gem_handle, bo_gem->name, bo->offset,
01835                          (unsigned long long)bufmgr_gem->exec2_objects[i].offset);
01836                      bo->offset = bufmgr_gem->exec2_objects[i].offset;
01837               }
01838        }
01839 }
01840 
01841 static void
01842 aub_out(drm_intel_bufmgr_gem *bufmgr_gem, uint32_t data)
01843 {
01844        fwrite(&data, 1, 4, bufmgr_gem->aub_file);
01845 }
01846 
01847 static void
01848 aub_out_data(drm_intel_bufmgr_gem *bufmgr_gem, void *data, size_t size)
01849 {
01850        fwrite(data, 1, size, bufmgr_gem->aub_file);
01851 }
01852 
01853 static void
01854 aub_write_bo_data(drm_intel_bo *bo, uint32_t offset, uint32_t size)
01855 {
01856        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
01857        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01858        uint32_t *data;
01859        unsigned int i;
01860 
01861        data = malloc(bo->size);
01862        drm_intel_bo_get_subdata(bo, offset, size, data);
01863 
01864        /* Easy mode: write out bo with no relocations */
01865        if (!bo_gem->reloc_count) {
01866               aub_out_data(bufmgr_gem, data, size);
01867               free(data);
01868               return;
01869        }
01870 
01871        /* Otherwise, handle the relocations while writing. */
01872        for (i = 0; i < size / 4; i++) {
01873               int r;
01874               for (r = 0; r < bo_gem->reloc_count; r++) {
01875                      struct drm_i915_gem_relocation_entry *reloc;
01876                      drm_intel_reloc_target *info;
01877 
01878                      reloc = &bo_gem->relocs[r];
01879                      info = &bo_gem->reloc_target_info[r];
01880 
01881                      if (reloc->offset == offset + i * 4) {
01882                             drm_intel_bo_gem *target_gem;
01883                             uint32_t val;
01884 
01885                             target_gem = (drm_intel_bo_gem *)info->bo;
01886 
01887                             val = reloc->delta;
01888                             val += target_gem->aub_offset;
01889 
01890                             aub_out(bufmgr_gem, val);
01891                             data[i] = val;
01892                             break;
01893                      }
01894               }
01895               if (r == bo_gem->reloc_count) {
01896                      /* no relocation, just the data */
01897                      aub_out(bufmgr_gem, data[i]);
01898               }
01899        }
01900 
01901        free(data);
01902 }
01903 
01904 static void
01905 aub_bo_get_address(drm_intel_bo *bo)
01906 {
01907        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
01908        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01909 
01910        /* Give the object a graphics address in the AUB file.  We
01911         * don't just use the GEM object address because we do AUB
01912         * dumping before execution -- we want to successfully log
01913         * when the hardware might hang, and we might even want to aub
01914         * capture for a driver trying to execute on a different
01915         * generation of hardware by disabling the actual kernel exec
01916         * call.
01917         */
01918        bo_gem->aub_offset = bufmgr_gem->aub_offset;
01919        bufmgr_gem->aub_offset += bo->size;
01920        /* XXX: Handle aperture overflow. */
01921        assert(bufmgr_gem->aub_offset < 256 * 1024 * 1024);
01922 }
01923 
01924 static void
01925 aub_write_trace_block(drm_intel_bo *bo, uint32_t type, uint32_t subtype,
01926                     uint32_t offset, uint32_t size)
01927 {
01928        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
01929        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01930 
01931        aub_out(bufmgr_gem,
01932               CMD_AUB_TRACE_HEADER_BLOCK |
01933               (5 - 2));
01934        aub_out(bufmgr_gem,
01935               AUB_TRACE_MEMTYPE_GTT | type | AUB_TRACE_OP_DATA_WRITE);
01936        aub_out(bufmgr_gem, subtype);
01937        aub_out(bufmgr_gem, bo_gem->aub_offset + offset);
01938        aub_out(bufmgr_gem, size);
01939        aub_write_bo_data(bo, offset, size);
01940 }
01941 
01947 static void
01948 aub_write_large_trace_block(drm_intel_bo *bo, uint32_t type, uint32_t subtype,
01949                          uint32_t offset, uint32_t size)
01950 {
01951        uint32_t block_size;
01952        uint32_t sub_offset;
01953 
01954        for (sub_offset = 0; sub_offset < size; sub_offset += block_size) {
01955               block_size = size - sub_offset;
01956 
01957               if (block_size > 8 * 4096)
01958                      block_size = 8 * 4096;
01959 
01960               aub_write_trace_block(bo, type, subtype, offset + sub_offset,
01961                                   block_size);
01962        }
01963 }
01964 
01965 static void
01966 aub_write_bo(drm_intel_bo *bo)
01967 {
01968        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
01969        uint32_t offset = 0;
01970        unsigned i;
01971 
01972        aub_bo_get_address(bo);
01973 
01974        /* Write out each annotated section separately. */
01975        for (i = 0; i < bo_gem->aub_annotation_count; ++i) {
01976               drm_intel_aub_annotation *annotation =
01977                      &bo_gem->aub_annotations[i];
01978               uint32_t ending_offset = annotation->ending_offset;
01979               if (ending_offset > bo->size)
01980                      ending_offset = bo->size;
01981               if (ending_offset > offset) {
01982                      aub_write_large_trace_block(bo, annotation->type,
01983                                               annotation->subtype,
01984                                               offset,
01985                                               ending_offset - offset);
01986                      offset = ending_offset;
01987               }
01988        }
01989 
01990        /* Write out any remaining unannotated data */
01991        if (offset < bo->size) {
01992               aub_write_large_trace_block(bo, AUB_TRACE_TYPE_NOTYPE, 0,
01993                                        offset, bo->size - offset);
01994        }
01995 }
01996 
01997 /*
01998  * Make a ringbuffer on fly and dump it
01999  */
02000 static void
02001 aub_build_dump_ringbuffer(drm_intel_bufmgr_gem *bufmgr_gem,
02002                        uint32_t batch_buffer, int ring_flag)
02003 {
02004        uint32_t ringbuffer[4096];
02005        int ring = AUB_TRACE_TYPE_RING_PRB0; /* The default ring */
02006        int ring_count = 0;
02007 
02008        if (ring_flag == I915_EXEC_BSD)
02009               ring = AUB_TRACE_TYPE_RING_PRB1;
02010 
02011        /* Make a ring buffer to execute our batchbuffer. */
02012        memset(ringbuffer, 0, sizeof(ringbuffer));
02013        ringbuffer[ring_count++] = AUB_MI_BATCH_BUFFER_START;
02014        ringbuffer[ring_count++] = batch_buffer;
02015 
02016        /* Write out the ring.  This appears to trigger execution of
02017         * the ring in the simulator.
02018         */
02019        aub_out(bufmgr_gem,
02020               CMD_AUB_TRACE_HEADER_BLOCK |
02021               (5 - 2));
02022        aub_out(bufmgr_gem,
02023               AUB_TRACE_MEMTYPE_GTT | ring | AUB_TRACE_OP_COMMAND_WRITE);
02024        aub_out(bufmgr_gem, 0); /* general/surface subtype */
02025        aub_out(bufmgr_gem, bufmgr_gem->aub_offset);
02026        aub_out(bufmgr_gem, ring_count * 4);
02027 
02028        /* FIXME: Need some flush operations here? */
02029        aub_out_data(bufmgr_gem, ringbuffer, ring_count * 4);
02030 
02031        /* Update offset pointer */
02032        bufmgr_gem->aub_offset += 4096;
02033 }
02034 
02035 void
02036 drm_intel_gem_bo_aub_dump_bmp(drm_intel_bo *bo,
02037                            int x1, int y1, int width, int height,
02038                            enum aub_dump_bmp_format format,
02039                            int pitch, int offset)
02040 {
02041        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
02042        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *)bo;
02043        uint32_t cpp;
02044 
02045        switch (format) {
02046        case AUB_DUMP_BMP_FORMAT_8BIT:
02047               cpp = 1;
02048               break;
02049        case AUB_DUMP_BMP_FORMAT_ARGB_4444:
02050               cpp = 2;
02051               break;
02052        case AUB_DUMP_BMP_FORMAT_ARGB_0888:
02053        case AUB_DUMP_BMP_FORMAT_ARGB_8888:
02054               cpp = 4;
02055               break;
02056        default:
02057               printf("Unknown AUB dump format %d\n", format);
02058               return;
02059        }
02060 
02061        if (!bufmgr_gem->aub_file)
02062               return;
02063 
02064        aub_out(bufmgr_gem, CMD_AUB_DUMP_BMP | 4);
02065        aub_out(bufmgr_gem, (y1 << 16) | x1);
02066        aub_out(bufmgr_gem,
02067               (format << 24) |
02068               (cpp << 19) |
02069               pitch / 4);
02070        aub_out(bufmgr_gem, (height << 16) | width);
02071        aub_out(bufmgr_gem, bo_gem->aub_offset + offset);
02072        aub_out(bufmgr_gem,
02073               ((bo_gem->tiling_mode != I915_TILING_NONE) ? (1 << 2) : 0) |
02074               ((bo_gem->tiling_mode == I915_TILING_Y) ? (1 << 3) : 0));
02075 }
02076 
02077 static void
02078 aub_exec(drm_intel_bo *bo, int ring_flag, int used)
02079 {
02080        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
02081        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02082        int i;
02083        bool batch_buffer_needs_annotations;
02084 
02085        if (!bufmgr_gem->aub_file)
02086               return;
02087 
02088        /* If batch buffer is not annotated, annotate it the best we
02089         * can.
02090         */
02091        batch_buffer_needs_annotations = bo_gem->aub_annotation_count == 0;
02092        if (batch_buffer_needs_annotations) {
02093               drm_intel_aub_annotation annotations[2] = {
02094                      { AUB_TRACE_TYPE_BATCH, 0, used },
02095                      { AUB_TRACE_TYPE_NOTYPE, 0, bo->size }
02096               };
02097               drm_intel_bufmgr_gem_set_aub_annotations(bo, annotations, 2);
02098        }
02099 
02100        /* Write out all buffers to AUB memory */
02101        for (i = 0; i < bufmgr_gem->exec_count; i++) {
02102               aub_write_bo(bufmgr_gem->exec_bos[i]);
02103        }
02104 
02105        /* Remove any annotations we added */
02106        if (batch_buffer_needs_annotations)
02107               drm_intel_bufmgr_gem_set_aub_annotations(bo, NULL, 0);
02108 
02109        /* Dump ring buffer */
02110        aub_build_dump_ringbuffer(bufmgr_gem, bo_gem->aub_offset, ring_flag);
02111 
02112        fflush(bufmgr_gem->aub_file);
02113 
02114        /*
02115         * One frame has been dumped. So reset the aub_offset for the next frame.
02116         *
02117         * FIXME: Can we do this?
02118         */
02119        bufmgr_gem->aub_offset = 0x10000;
02120 }
02121 
02122 static int
02123 drm_intel_gem_bo_exec(drm_intel_bo *bo, int used,
02124                     drm_clip_rect_t * cliprects, int num_cliprects, int DR4)
02125 {
02126        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
02127        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02128        struct drm_i915_gem_execbuffer execbuf;
02129        int ret, i;
02130 
02131        if (bo_gem->has_error)
02132               return -ENOMEM;
02133 
02134        pthread_mutex_lock(&bufmgr_gem->lock);
02135        /* Update indices and set up the validate list. */
02136        drm_intel_gem_bo_process_reloc(bo);
02137 
02138        /* Add the batch buffer to the validation list.  There are no
02139         * relocations pointing to it.
02140         */
02141        drm_intel_add_validate_buffer(bo);
02142 
02143        VG_CLEAR(execbuf);
02144        execbuf.buffers_ptr = (uintptr_t) bufmgr_gem->exec_objects;
02145        execbuf.buffer_count = bufmgr_gem->exec_count;
02146        execbuf.batch_start_offset = 0;
02147        execbuf.batch_len = used;
02148        execbuf.cliprects_ptr = (uintptr_t) cliprects;
02149        execbuf.num_cliprects = num_cliprects;
02150        execbuf.DR1 = 0;
02151        execbuf.DR4 = DR4;
02152 
02153        ret = drmIoctl(bufmgr_gem->fd,
02154                      DRM_IOCTL_I915_GEM_EXECBUFFER,
02155                      &execbuf);
02156        if (ret != 0) {
02157               ret = -errno;
02158               if (errno == ENOSPC) {
02159                      DBG("Execbuffer fails to pin. "
02160                          "Estimate: %u. Actual: %u. Available: %u\n",
02161                          drm_intel_gem_estimate_batch_space(bufmgr_gem->exec_bos,
02162                                                         bufmgr_gem->
02163                                                         exec_count),
02164                          drm_intel_gem_compute_batch_space(bufmgr_gem->exec_bos,
02165                                                        bufmgr_gem->
02166                                                        exec_count),
02167                          (unsigned int)bufmgr_gem->gtt_size);
02168               }
02169        }
02170        drm_intel_update_buffer_offsets(bufmgr_gem);
02171 
02172        if (bufmgr_gem->bufmgr.debug)
02173               drm_intel_gem_dump_validation_list(bufmgr_gem);
02174 
02175        for (i = 0; i < bufmgr_gem->exec_count; i++) {
02176               drm_intel_bo *bo = bufmgr_gem->exec_bos[i];
02177               drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02178 
02179               /* Disconnect the buffer from the validate list */
02180               bo_gem->validate_index = -1;
02181               bufmgr_gem->exec_bos[i] = NULL;
02182        }
02183        bufmgr_gem->exec_count = 0;
02184        pthread_mutex_unlock(&bufmgr_gem->lock);
02185 
02186        return ret;
02187 }
02188 
02189 static int
02190 do_exec2(drm_intel_bo *bo, int used, drm_intel_context *ctx,
02191         drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
02192         unsigned int flags)
02193 {
02194        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *)bo->bufmgr;
02195        struct drm_i915_gem_execbuffer2 execbuf;
02196        int ret = 0;
02197        int i;
02198 
02199        switch (flags & 0x7) {
02200        default:
02201               return -EINVAL;
02202        case I915_EXEC_BLT:
02203               if (!bufmgr_gem->has_blt)
02204                      return -EINVAL;
02205               break;
02206        case I915_EXEC_BSD:
02207               if (!bufmgr_gem->has_bsd)
02208                      return -EINVAL;
02209               break;
02210        case I915_EXEC_RENDER:
02211        case I915_EXEC_DEFAULT:
02212               break;
02213        }
02214 
02215        pthread_mutex_lock(&bufmgr_gem->lock);
02216        /* Update indices and set up the validate list. */
02217        drm_intel_gem_bo_process_reloc2(bo);
02218 
02219        /* Add the batch buffer to the validation list.  There are no relocations
02220         * pointing to it.
02221         */
02222        drm_intel_add_validate_buffer2(bo, 0);
02223 
02224        VG_CLEAR(execbuf);
02225        execbuf.buffers_ptr = (uintptr_t)bufmgr_gem->exec2_objects;
02226        execbuf.buffer_count = bufmgr_gem->exec_count;
02227        execbuf.batch_start_offset = 0;
02228        execbuf.batch_len = used;
02229        execbuf.cliprects_ptr = (uintptr_t)cliprects;
02230        execbuf.num_cliprects = num_cliprects;
02231        execbuf.DR1 = 0;
02232        execbuf.DR4 = DR4;
02233        execbuf.flags = flags;
02234        if (ctx == NULL)
02235               i915_execbuffer2_set_context_id(execbuf, 0);
02236        else
02237               i915_execbuffer2_set_context_id(execbuf, ctx->ctx_id);
02238        execbuf.rsvd2 = 0;
02239 
02240        aub_exec(bo, flags, used);
02241 
02242        if (bufmgr_gem->no_exec)
02243               goto skip_execution;
02244 
02245        ret = drmIoctl(bufmgr_gem->fd,
02246                      DRM_IOCTL_I915_GEM_EXECBUFFER2,
02247                      &execbuf);
02248        if (ret != 0) {
02249               ret = -errno;
02250               if (ret == -ENOSPC) {
02251                      DBG("Execbuffer fails to pin. "
02252                          "Estimate: %u. Actual: %u. Available: %u\n",
02253                          drm_intel_gem_estimate_batch_space(bufmgr_gem->exec_bos,
02254                                                         bufmgr_gem->exec_count),
02255                          drm_intel_gem_compute_batch_space(bufmgr_gem->exec_bos,
02256                                                        bufmgr_gem->exec_count),
02257                          (unsigned int) bufmgr_gem->gtt_size);
02258               }
02259        }
02260        drm_intel_update_buffer_offsets2(bufmgr_gem);
02261 
02262 skip_execution:
02263        if (bufmgr_gem->bufmgr.debug)
02264               drm_intel_gem_dump_validation_list(bufmgr_gem);
02265 
02266        for (i = 0; i < bufmgr_gem->exec_count; i++) {
02267               drm_intel_bo *bo = bufmgr_gem->exec_bos[i];
02268               drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *)bo;
02269 
02270               /* Disconnect the buffer from the validate list */
02271               bo_gem->validate_index = -1;
02272               bufmgr_gem->exec_bos[i] = NULL;
02273        }
02274        bufmgr_gem->exec_count = 0;
02275        pthread_mutex_unlock(&bufmgr_gem->lock);
02276 
02277        return ret;
02278 }
02279 
02280 static int
02281 drm_intel_gem_bo_exec2(drm_intel_bo *bo, int used,
02282                      drm_clip_rect_t *cliprects, int num_cliprects,
02283                      int DR4)
02284 {
02285        return do_exec2(bo, used, NULL, cliprects, num_cliprects, DR4,
02286                      I915_EXEC_RENDER);
02287 }
02288 
02289 static int
02290 drm_intel_gem_bo_mrb_exec2(drm_intel_bo *bo, int used,
02291                      drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
02292                      unsigned int flags)
02293 {
02294        return do_exec2(bo, used, NULL, cliprects, num_cliprects, DR4,
02295                      flags);
02296 }
02297 
02298 int
02299 drm_intel_gem_bo_context_exec(drm_intel_bo *bo, drm_intel_context *ctx,
02300                            int used, unsigned int flags)
02301 {
02302        return do_exec2(bo, used, ctx, NULL, 0, 0, flags);
02303 }
02304 
02305 static int
02306 drm_intel_gem_bo_pin(drm_intel_bo *bo, uint32_t alignment)
02307 {
02308        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
02309        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02310        struct drm_i915_gem_pin pin;
02311        int ret;
02312 
02313        VG_CLEAR(pin);
02314        pin.handle = bo_gem->gem_handle;
02315        pin.alignment = alignment;
02316 
02317        ret = drmIoctl(bufmgr_gem->fd,
02318                      DRM_IOCTL_I915_GEM_PIN,
02319                      &pin);
02320        if (ret != 0)
02321               return -errno;
02322 
02323        bo->offset = pin.offset;
02324        return 0;
02325 }
02326 
02327 static int
02328 drm_intel_gem_bo_unpin(drm_intel_bo *bo)
02329 {
02330        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
02331        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02332        struct drm_i915_gem_unpin unpin;
02333        int ret;
02334 
02335        VG_CLEAR(unpin);
02336        unpin.handle = bo_gem->gem_handle;
02337 
02338        ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_UNPIN, &unpin);
02339        if (ret != 0)
02340               return -errno;
02341 
02342        return 0;
02343 }
02344 
02345 static int
02346 drm_intel_gem_bo_set_tiling_internal(drm_intel_bo *bo,
02347                                  uint32_t tiling_mode,
02348                                  uint32_t stride)
02349 {
02350        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
02351        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02352        struct drm_i915_gem_set_tiling set_tiling;
02353        int ret;
02354 
02355        if (bo_gem->global_name == 0 &&
02356            tiling_mode == bo_gem->tiling_mode &&
02357            stride == bo_gem->stride)
02358               return 0;
02359 
02360        memset(&set_tiling, 0, sizeof(set_tiling));
02361        do {
02362               /* set_tiling is slightly broken and overwrites the
02363                * input on the error path, so we have to open code
02364                * rmIoctl.
02365                */
02366               set_tiling.handle = bo_gem->gem_handle;
02367               set_tiling.tiling_mode = tiling_mode;
02368               set_tiling.stride = stride;
02369 
02370               ret = ioctl(bufmgr_gem->fd,
02371                          DRM_IOCTL_I915_GEM_SET_TILING,
02372                          &set_tiling);
02373        } while (ret == -1 && (errno == EINTR || errno == EAGAIN));
02374        if (ret == -1)
02375               return -errno;
02376 
02377        bo_gem->tiling_mode = set_tiling.tiling_mode;
02378        bo_gem->swizzle_mode = set_tiling.swizzle_mode;
02379        bo_gem->stride = set_tiling.stride;
02380        return 0;
02381 }
02382 
02383 static int
02384 drm_intel_gem_bo_set_tiling(drm_intel_bo *bo, uint32_t * tiling_mode,
02385                          uint32_t stride)
02386 {
02387        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
02388        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02389        int ret;
02390 
02391        /* Linear buffers have no stride. By ensuring that we only ever use
02392         * stride 0 with linear buffers, we simplify our code.
02393         */
02394        if (*tiling_mode == I915_TILING_NONE)
02395               stride = 0;
02396 
02397        ret = drm_intel_gem_bo_set_tiling_internal(bo, *tiling_mode, stride);
02398        if (ret == 0)
02399               drm_intel_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem);
02400 
02401        *tiling_mode = bo_gem->tiling_mode;
02402        return ret;
02403 }
02404 
02405 static int
02406 drm_intel_gem_bo_get_tiling(drm_intel_bo *bo, uint32_t * tiling_mode,
02407                          uint32_t * swizzle_mode)
02408 {
02409        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02410 
02411        *tiling_mode = bo_gem->tiling_mode;
02412        *swizzle_mode = bo_gem->swizzle_mode;
02413        return 0;
02414 }
02415 
02416 static int
02417 drm_intel_gem_bo_flink(drm_intel_bo *bo, uint32_t * name)
02418 {
02419        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
02420        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02421        int ret;
02422 
02423        if (!bo_gem->global_name) {
02424               struct drm_gem_flink flink;
02425 
02426               VG_CLEAR(flink);
02427               flink.handle = bo_gem->gem_handle;
02428 
02429               ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_FLINK, &flink);
02430               if (ret != 0)
02431                      return -errno;
02432 
02433               bo_gem->global_name = flink.name;
02434               bo_gem->reusable = false;
02435 
02436               DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
02437        }
02438 
02439        *name = bo_gem->global_name;
02440        return 0;
02441 }
02442 
02450 void
02451 drm_intel_bufmgr_gem_enable_reuse(drm_intel_bufmgr *bufmgr)
02452 {
02453        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bufmgr;
02454 
02455        bufmgr_gem->bo_reuse = true;
02456 }
02457 
02465 void
02466 drm_intel_bufmgr_gem_enable_fenced_relocs(drm_intel_bufmgr *bufmgr)
02467 {
02468        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *)bufmgr;
02469 
02470        if (bufmgr_gem->bufmgr.bo_exec == drm_intel_gem_bo_exec2)
02471               bufmgr_gem->fenced_relocs = true;
02472 }
02473 
02478 static int
02479 drm_intel_gem_bo_get_aperture_space(drm_intel_bo *bo)
02480 {
02481        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02482        int i;
02483        int total = 0;
02484 
02485        if (bo == NULL || bo_gem->included_in_check_aperture)
02486               return 0;
02487 
02488        total += bo->size;
02489        bo_gem->included_in_check_aperture = true;
02490 
02491        for (i = 0; i < bo_gem->reloc_count; i++)
02492               total +=
02493                   drm_intel_gem_bo_get_aperture_space(bo_gem->
02494                                                  reloc_target_info[i].bo);
02495 
02496        return total;
02497 }
02498 
02507 static unsigned int
02508 drm_intel_gem_total_fences(drm_intel_bo ** bo_array, int count)
02509 {
02510        int i;
02511        unsigned int total = 0;
02512 
02513        for (i = 0; i < count; i++) {
02514               drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo_array[i];
02515 
02516               if (bo_gem == NULL)
02517                      continue;
02518 
02519               total += bo_gem->reloc_tree_fences;
02520        }
02521        return total;
02522 }
02523 
02528 static void
02529 drm_intel_gem_bo_clear_aperture_space_flag(drm_intel_bo *bo)
02530 {
02531        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02532        int i;
02533 
02534        if (bo == NULL || !bo_gem->included_in_check_aperture)
02535               return;
02536 
02537        bo_gem->included_in_check_aperture = false;
02538 
02539        for (i = 0; i < bo_gem->reloc_count; i++)
02540               drm_intel_gem_bo_clear_aperture_space_flag(bo_gem->
02541                                                     reloc_target_info[i].bo);
02542 }
02543 
02548 static unsigned int
02549 drm_intel_gem_estimate_batch_space(drm_intel_bo **bo_array, int count)
02550 {
02551        int i;
02552        unsigned int total = 0;
02553 
02554        for (i = 0; i < count; i++) {
02555               drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo_array[i];
02556               if (bo_gem != NULL)
02557                      total += bo_gem->reloc_tree_size;
02558        }
02559        return total;
02560 }
02561 
02567 static unsigned int
02568 drm_intel_gem_compute_batch_space(drm_intel_bo **bo_array, int count)
02569 {
02570        int i;
02571        unsigned int total = 0;
02572 
02573        for (i = 0; i < count; i++) {
02574               total += drm_intel_gem_bo_get_aperture_space(bo_array[i]);
02575               /* For the first buffer object in the array, we get an
02576                * accurate count back for its reloc_tree size (since nothing
02577                * had been flagged as being counted yet).  We can save that
02578                * value out as a more conservative reloc_tree_size that
02579                * avoids double-counting target buffers.  Since the first
02580                * buffer happens to usually be the batch buffer in our
02581                * callers, this can pull us back from doing the tree
02582                * walk on every new batch emit.
02583                */
02584               if (i == 0) {
02585                      drm_intel_bo_gem *bo_gem =
02586                          (drm_intel_bo_gem *) bo_array[i];
02587                      bo_gem->reloc_tree_size = total;
02588               }
02589        }
02590 
02591        for (i = 0; i < count; i++)
02592               drm_intel_gem_bo_clear_aperture_space_flag(bo_array[i]);
02593        return total;
02594 }
02595 
02612 static int
02613 drm_intel_gem_check_aperture_space(drm_intel_bo **bo_array, int count)
02614 {
02615        drm_intel_bufmgr_gem *bufmgr_gem =
02616            (drm_intel_bufmgr_gem *) bo_array[0]->bufmgr;
02617        unsigned int total = 0;
02618        unsigned int threshold = bufmgr_gem->gtt_size * 3 / 4;
02619        int total_fences;
02620 
02621        /* Check for fence reg constraints if necessary */
02622        if (bufmgr_gem->available_fences) {
02623               total_fences = drm_intel_gem_total_fences(bo_array, count);
02624               if (total_fences > bufmgr_gem->available_fences)
02625                      return -ENOSPC;
02626        }
02627 
02628        total = drm_intel_gem_estimate_batch_space(bo_array, count);
02629 
02630        if (total > threshold)
02631               total = drm_intel_gem_compute_batch_space(bo_array, count);
02632 
02633        if (total > threshold) {
02634               DBG("check_space: overflowed available aperture, "
02635                   "%dkb vs %dkb\n",
02636                   total / 1024, (int)bufmgr_gem->gtt_size / 1024);
02637               return -ENOSPC;
02638        } else {
02639               DBG("drm_check_space: total %dkb vs bufgr %dkb\n", total / 1024,
02640                   (int)bufmgr_gem->gtt_size / 1024);
02641               return 0;
02642        }
02643 }
02644 
02645 /*
02646  * Disable buffer reuse for objects which are shared with the kernel
02647  * as scanout buffers
02648  */
02649 static int
02650 drm_intel_gem_bo_disable_reuse(drm_intel_bo *bo)
02651 {
02652        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02653 
02654        bo_gem->reusable = false;
02655        return 0;
02656 }
02657 
02658 static int
02659 drm_intel_gem_bo_is_reusable(drm_intel_bo *bo)
02660 {
02661        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02662 
02663        return bo_gem->reusable;
02664 }
02665 
02666 static int
02667 _drm_intel_gem_bo_references(drm_intel_bo *bo, drm_intel_bo *target_bo)
02668 {
02669        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02670        int i;
02671 
02672        for (i = 0; i < bo_gem->reloc_count; i++) {
02673               if (bo_gem->reloc_target_info[i].bo == target_bo)
02674                      return 1;
02675               if (bo == bo_gem->reloc_target_info[i].bo)
02676                      continue;
02677               if (_drm_intel_gem_bo_references(bo_gem->reloc_target_info[i].bo,
02678                                           target_bo))
02679                      return 1;
02680        }
02681 
02682        return 0;
02683 }
02684 
02686 static int
02687 drm_intel_gem_bo_references(drm_intel_bo *bo, drm_intel_bo *target_bo)
02688 {
02689        drm_intel_bo_gem *target_bo_gem = (drm_intel_bo_gem *) target_bo;
02690 
02691        if (bo == NULL || target_bo == NULL)
02692               return 0;
02693        if (target_bo_gem->used_as_reloc_target)
02694               return _drm_intel_gem_bo_references(bo, target_bo);
02695        return 0;
02696 }
02697 
02698 static void
02699 add_bucket(drm_intel_bufmgr_gem *bufmgr_gem, int size)
02700 {
02701        unsigned int i = bufmgr_gem->num_buckets;
02702 
02703        assert(i < ARRAY_SIZE(bufmgr_gem->cache_bucket));
02704 
02705        DRMINITLISTHEAD(&bufmgr_gem->cache_bucket[i].head);
02706        bufmgr_gem->cache_bucket[i].size = size;
02707        bufmgr_gem->num_buckets++;
02708 }
02709 
02710 static void
02711 init_cache_buckets(drm_intel_bufmgr_gem *bufmgr_gem)
02712 {
02713        unsigned long size, cache_max_size = 64 * 1024 * 1024;
02714 
02715        /* OK, so power of two buckets was too wasteful of memory.
02716         * Give 3 other sizes between each power of two, to hopefully
02717         * cover things accurately enough.  (The alternative is
02718         * probably to just go for exact matching of sizes, and assume
02719         * that for things like composited window resize the tiled
02720         * width/height alignment and rounding of sizes to pages will
02721         * get us useful cache hit rates anyway)
02722         */
02723        add_bucket(bufmgr_gem, 4096);
02724        add_bucket(bufmgr_gem, 4096 * 2);
02725        add_bucket(bufmgr_gem, 4096 * 3);
02726 
02727        /* Initialize the linked lists for BO reuse cache. */
02728        for (size = 4 * 4096; size <= cache_max_size; size *= 2) {
02729               add_bucket(bufmgr_gem, size);
02730 
02731               add_bucket(bufmgr_gem, size + size * 1 / 4);
02732               add_bucket(bufmgr_gem, size + size * 2 / 4);
02733               add_bucket(bufmgr_gem, size + size * 3 / 4);
02734        }
02735 }
02736 
02737 void
02738 drm_intel_bufmgr_gem_set_vma_cache_size(drm_intel_bufmgr *bufmgr, int limit)
02739 {
02740        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *)bufmgr;
02741 
02742        bufmgr_gem->vma_max = limit;
02743 
02744        drm_intel_gem_bo_purge_vma_cache(bufmgr_gem);
02745 }
02746 
02751 static int
02752 get_pci_device_id(drm_intel_bufmgr_gem *bufmgr_gem)
02753 {
02754        char *devid_override;
02755        int devid;
02756        int ret;
02757        drm_i915_getparam_t gp;
02758 
02759        if (geteuid() == getuid()) {
02760               devid_override = getenv("INTEL_DEVID_OVERRIDE");
02761               if (devid_override) {
02762                      bufmgr_gem->no_exec = true;
02763                      return strtod(devid_override, NULL);
02764               }
02765        }
02766 
02767        VG_CLEAR(devid);
02768        VG_CLEAR(gp);
02769        gp.param = I915_PARAM_CHIPSET_ID;
02770        gp.value = &devid;
02771        ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
02772        if (ret) {
02773               fprintf(stderr, "get chip id failed: %d [%d]\n", ret, errno);
02774               fprintf(stderr, "param: %d, val: %d\n", gp.param, *gp.value);
02775        }
02776        return devid;
02777 }
02778 
02779 int
02780 drm_intel_bufmgr_gem_get_devid(drm_intel_bufmgr *bufmgr)
02781 {
02782        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *)bufmgr;
02783 
02784        return bufmgr_gem->pci_device;
02785 }
02786 
02795 void
02796 drm_intel_bufmgr_gem_set_aub_dump(drm_intel_bufmgr *bufmgr, int enable)
02797 {
02798        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *)bufmgr;
02799        int entry = 0x200003;
02800        int i;
02801        int gtt_size = 0x10000;
02802 
02803        if (!enable) {
02804               if (bufmgr_gem->aub_file) {
02805                      fclose(bufmgr_gem->aub_file);
02806                      bufmgr_gem->aub_file = NULL;
02807               }
02808        }
02809 
02810        if (geteuid() != getuid())
02811               return;
02812 
02813        bufmgr_gem->aub_file = fopen("intel.aub", "w+");
02814        if (!bufmgr_gem->aub_file)
02815               return;
02816 
02817        /* Start allocating objects from just after the GTT. */
02818        bufmgr_gem->aub_offset = gtt_size;
02819 
02820        /* Start with a (required) version packet. */
02821        aub_out(bufmgr_gem, CMD_AUB_HEADER | (13 - 2));
02822        aub_out(bufmgr_gem,
02823               (4 << AUB_HEADER_MAJOR_SHIFT) |
02824               (0 << AUB_HEADER_MINOR_SHIFT));
02825        for (i = 0; i < 8; i++) {
02826               aub_out(bufmgr_gem, 0); /* app name */
02827        }
02828        aub_out(bufmgr_gem, 0); /* timestamp */
02829        aub_out(bufmgr_gem, 0); /* timestamp */
02830        aub_out(bufmgr_gem, 0); /* comment len */
02831 
02832        /* Set up the GTT. The max we can handle is 256M */
02833        aub_out(bufmgr_gem, CMD_AUB_TRACE_HEADER_BLOCK | (5 - 2));
02834        aub_out(bufmgr_gem, AUB_TRACE_MEMTYPE_NONLOCAL | 0 | AUB_TRACE_OP_DATA_WRITE);
02835        aub_out(bufmgr_gem, 0); /* subtype */
02836        aub_out(bufmgr_gem, 0); /* offset */
02837        aub_out(bufmgr_gem, gtt_size); /* size */
02838        for (i = 0x000; i < gtt_size; i += 4, entry += 0x1000) {
02839               aub_out(bufmgr_gem, entry);
02840        }
02841 }
02842 
02843 drm_intel_context *
02844 drm_intel_gem_context_create(drm_intel_bufmgr *bufmgr)
02845 {
02846        drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *)bufmgr;
02847        struct drm_i915_gem_context_create create;
02848        drm_i915_getparam_t gp;
02849        drm_intel_context *context = NULL;
02850        int tmp = 0, ret;
02851 
02852        ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE, &create);
02853        if (ret != 0) {
02854               fprintf(stderr, "DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n",
02855                      strerror(errno));
02856               return NULL;
02857        }
02858 
02859        context = calloc(1, sizeof(*context));
02860        context->ctx_id = create.ctx_id;
02861        context->bufmgr = bufmgr;
02862 
02863        return context;
02864 }
02865 
02866 void
02867 drm_intel_gem_context_destroy(drm_intel_context *ctx)
02868 {
02869        drm_intel_bufmgr_gem *bufmgr_gem;
02870        struct drm_i915_gem_context_destroy destroy;
02871        int ret;
02872 
02873        if (ctx == NULL)
02874               return;
02875 
02876        bufmgr_gem = (drm_intel_bufmgr_gem *)ctx->bufmgr;
02877        destroy.ctx_id = ctx->ctx_id;
02878        ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_DESTROY,
02879                      &destroy);
02880        if (ret != 0)
02881               fprintf(stderr, "DRM_IOCTL_I915_GEM_CONTEXT_DESTROY failed: %s\n",
02882                      strerror(errno));
02883 
02884        free(ctx);
02885 }
02886 
02887 
02909 void
02910 drm_intel_bufmgr_gem_set_aub_annotations(drm_intel_bo *bo,
02911                                     drm_intel_aub_annotation *annotations,
02912                                     unsigned count)
02913 {
02914        drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;
02915        unsigned size = sizeof(*annotations) * count;
02916        drm_intel_aub_annotation *new_annotations =
02917               count > 0 ? realloc(bo_gem->aub_annotations, size) : NULL;
02918        if (new_annotations == NULL) {
02919               free(bo_gem->aub_annotations);
02920               bo_gem->aub_annotations = NULL;
02921               bo_gem->aub_annotation_count = 0;
02922               return;
02923        }
02924        memcpy(new_annotations, annotations, size);
02925        bo_gem->aub_annotations = new_annotations;
02926        bo_gem->aub_annotation_count = count;
02927 }
02928 
02935 drm_intel_bufmgr *
02936 drm_intel_bufmgr_gem_init(int fd, int batch_size)
02937 {
02938        drm_intel_bufmgr_gem *bufmgr_gem;
02939        struct drm_i915_gem_get_aperture aperture;
02940        drm_i915_getparam_t gp;
02941        int ret, tmp;
02942        bool exec2 = false;
02943 
02944        bufmgr_gem = calloc(1, sizeof(*bufmgr_gem));
02945        if (bufmgr_gem == NULL)
02946               return NULL;
02947 
02948        bufmgr_gem->fd = fd;
02949 
02950        if (pthread_mutex_init(&bufmgr_gem->lock, NULL) != 0) {
02951               free(bufmgr_gem);
02952               return NULL;
02953        }
02954 
02955        ret = drmIoctl(bufmgr_gem->fd,
02956                      DRM_IOCTL_I915_GEM_GET_APERTURE,
02957                      &aperture);
02958 
02959        if (ret == 0)
02960               bufmgr_gem->gtt_size = aperture.aper_available_size;
02961        else {
02962               fprintf(stderr, "DRM_IOCTL_I915_GEM_APERTURE failed: %s\n",
02963                      strerror(errno));
02964               bufmgr_gem->gtt_size = 128 * 1024 * 1024;
02965               fprintf(stderr, "Assuming %dkB available aperture size.\n"
02966                      "May lead to reduced performance or incorrect "
02967                      "rendering.\n",
02968                      (int)bufmgr_gem->gtt_size / 1024);
02969        }
02970 
02971        bufmgr_gem->pci_device = get_pci_device_id(bufmgr_gem);
02972 
02973        if (IS_GEN2(bufmgr_gem->pci_device))
02974               bufmgr_gem->gen = 2;
02975        else if (IS_GEN3(bufmgr_gem->pci_device))
02976               bufmgr_gem->gen = 3;
02977        else if (IS_GEN4(bufmgr_gem->pci_device))
02978               bufmgr_gem->gen = 4;
02979        else if (IS_GEN5(bufmgr_gem->pci_device))
02980               bufmgr_gem->gen = 5;
02981        else if (IS_GEN6(bufmgr_gem->pci_device))
02982               bufmgr_gem->gen = 6;
02983        else if (IS_GEN7(bufmgr_gem->pci_device))
02984               bufmgr_gem->gen = 7;
02985        else
02986               assert(0);
02987 
02988        if (IS_GEN3(bufmgr_gem->pci_device) &&
02989            bufmgr_gem->gtt_size > 256*1024*1024) {
02990               /* The unmappable part of gtt on gen 3 (i.e. above 256MB) can't
02991                * be used for tiled blits. To simplify the accounting, just
02992                * substract the unmappable part (fixed to 256MB on all known
02993                * gen3 devices) if the kernel advertises it. */
02994               bufmgr_gem->gtt_size -= 256*1024*1024;
02995        }
02996 
02997        VG_CLEAR(gp);
02998        gp.value = &tmp;
02999 
03000        gp.param = I915_PARAM_HAS_EXECBUF2;
03001        ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
03002        if (!ret)
03003               exec2 = true;
03004 
03005        gp.param = I915_PARAM_HAS_BSD;
03006        ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
03007        bufmgr_gem->has_bsd = ret == 0;
03008 
03009        gp.param = I915_PARAM_HAS_BLT;
03010        ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
03011        bufmgr_gem->has_blt = ret == 0;
03012 
03013        gp.param = I915_PARAM_HAS_RELAXED_FENCING;
03014        ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
03015        bufmgr_gem->has_relaxed_fencing = ret == 0;
03016 
03017        gp.param = I915_PARAM_HAS_WAIT_TIMEOUT;
03018        ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
03019        bufmgr_gem->has_wait_timeout = ret == 0;
03020 
03021        gp.param = I915_PARAM_HAS_LLC;
03022        ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
03023        if (ret != 0) {
03024               /* Kernel does not supports HAS_LLC query, fallback to GPU
03025                * generation detection and assume that we have LLC on GEN6/7
03026                */
03027               bufmgr_gem->has_llc = (IS_GEN6(bufmgr_gem->pci_device) |
03028                             IS_GEN7(bufmgr_gem->pci_device));
03029        } else
03030               bufmgr_gem->has_llc = ret == 0;
03031 
03032        if (bufmgr_gem->gen < 4) {
03033               gp.param = I915_PARAM_NUM_FENCES_AVAIL;
03034               gp.value = &bufmgr_gem->available_fences;
03035               ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
03036               if (ret) {
03037                      fprintf(stderr, "get fences failed: %d [%d]\n", ret,
03038                             errno);
03039                      fprintf(stderr, "param: %d, val: %d\n", gp.param,
03040                             *gp.value);
03041                      bufmgr_gem->available_fences = 0;
03042               } else {
03043                      /* XXX The kernel reports the total number of fences,
03044                       * including any that may be pinned.
03045                       *
03046                       * We presume that there will be at least one pinned
03047                       * fence for the scanout buffer, but there may be more
03048                       * than one scanout and the user may be manually
03049                       * pinning buffers. Let's move to execbuffer2 and
03050                       * thereby forget the insanity of using fences...
03051                       */
03052                      bufmgr_gem->available_fences -= 2;
03053                      if (bufmgr_gem->available_fences < 0)
03054                             bufmgr_gem->available_fences = 0;
03055               }
03056        }
03057 
03058        /* Let's go with one relocation per every 2 dwords (but round down a bit
03059         * since a power of two will mean an extra page allocation for the reloc
03060         * buffer).
03061         *
03062         * Every 4 was too few for the blender benchmark.
03063         */
03064        bufmgr_gem->max_relocs = batch_size / sizeof(uint32_t) / 2 - 2;
03065 
03066        bufmgr_gem->bufmgr.bo_alloc = drm_intel_gem_bo_alloc;
03067        bufmgr_gem->bufmgr.bo_alloc_for_render =
03068            drm_intel_gem_bo_alloc_for_render;
03069        bufmgr_gem->bufmgr.bo_alloc_tiled = drm_intel_gem_bo_alloc_tiled;
03070        bufmgr_gem->bufmgr.bo_reference = drm_intel_gem_bo_reference;
03071        bufmgr_gem->bufmgr.bo_unreference = drm_intel_gem_bo_unreference;
03072        bufmgr_gem->bufmgr.bo_map = drm_intel_gem_bo_map;
03073        bufmgr_gem->bufmgr.bo_unmap = drm_intel_gem_bo_unmap;
03074        bufmgr_gem->bufmgr.bo_subdata = drm_intel_gem_bo_subdata;
03075        bufmgr_gem->bufmgr.bo_get_subdata = drm_intel_gem_bo_get_subdata;
03076        bufmgr_gem->bufmgr.bo_wait_rendering = drm_intel_gem_bo_wait_rendering;
03077        bufmgr_gem->bufmgr.bo_emit_reloc = drm_intel_gem_bo_emit_reloc;
03078        bufmgr_gem->bufmgr.bo_emit_reloc_fence = drm_intel_gem_bo_emit_reloc_fence;
03079        bufmgr_gem->bufmgr.bo_pin = drm_intel_gem_bo_pin;
03080        bufmgr_gem->bufmgr.bo_unpin = drm_intel_gem_bo_unpin;
03081        bufmgr_gem->bufmgr.bo_get_tiling = drm_intel_gem_bo_get_tiling;
03082        bufmgr_gem->bufmgr.bo_set_tiling = drm_intel_gem_bo_set_tiling;
03083        bufmgr_gem->bufmgr.bo_flink = drm_intel_gem_bo_flink;
03084        /* Use the new one if available */
03085        if (exec2) {
03086               bufmgr_gem->bufmgr.bo_exec = drm_intel_gem_bo_exec2;
03087               bufmgr_gem->bufmgr.bo_mrb_exec = drm_intel_gem_bo_mrb_exec2;
03088        } else
03089               bufmgr_gem->bufmgr.bo_exec = drm_intel_gem_bo_exec;
03090        bufmgr_gem->bufmgr.bo_busy = drm_intel_gem_bo_busy;
03091        bufmgr_gem->bufmgr.bo_madvise = drm_intel_gem_bo_madvise;
03092        bufmgr_gem->bufmgr.destroy = drm_intel_bufmgr_gem_destroy;
03093        bufmgr_gem->bufmgr.debug = 0;
03094        bufmgr_gem->bufmgr.check_aperture_space =
03095            drm_intel_gem_check_aperture_space;
03096        bufmgr_gem->bufmgr.bo_disable_reuse = drm_intel_gem_bo_disable_reuse;
03097        bufmgr_gem->bufmgr.bo_is_reusable = drm_intel_gem_bo_is_reusable;
03098        bufmgr_gem->bufmgr.get_pipe_from_crtc_id =
03099            drm_intel_gem_get_pipe_from_crtc_id;
03100        bufmgr_gem->bufmgr.bo_references = drm_intel_gem_bo_references;
03101 
03102        DRMINITLISTHEAD(&bufmgr_gem->named);
03103        init_cache_buckets(bufmgr_gem);
03104 
03105        DRMINITLISTHEAD(&bufmgr_gem->vma_cache);
03106        bufmgr_gem->vma_max = -1; /* unlimited by default */
03107 
03108        return &bufmgr_gem->bufmgr;
03109 }