Back to index

libdrm  2.4.37
pushbuf.c
Go to the documentation of this file.
00001 /*
00002  * Copyright 2012 Red Hat Inc.
00003  *
00004  * Permission is hereby granted, free of charge, to any person obtaining a
00005  * copy of this software and associated documentation files (the "Software"),
00006  * to deal in the Software without restriction, including without limitation
00007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
00008  * and/or sell copies of the Software, and to permit persons to whom the
00009  * Software is furnished to do so, subject to the following conditions:
00010  *
00011  * The above copyright notice and this permission notice shall be included in
00012  * all copies or substantial portions of the Software.
00013  *
00014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
00015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
00016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
00017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
00018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
00019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
00020  * OTHER DEALINGS IN THE SOFTWARE.
00021  *
00022  * Authors: Ben Skeggs
00023  */
00024 
00025 #ifdef HAVE_CONFIG_H
00026 #include <config.h>
00027 #endif
00028 
00029 #include <stdio.h>
00030 #include <stdlib.h>
00031 #include <stdint.h>
00032 #include <stdbool.h>
00033 #include <string.h>
00034 #include <assert.h>
00035 #include <errno.h>
00036 
00037 #include <xf86drm.h>
00038 #include <xf86atomic.h>
00039 #include "libdrm_lists.h"
00040 #include "nouveau_drm.h"
00041 
00042 #include "nouveau.h"
00043 #include "private.h"
00044 
00045 struct nouveau_pushbuf_krec {
00046        struct nouveau_pushbuf_krec *next;
00047        struct drm_nouveau_gem_pushbuf_bo buffer[NOUVEAU_GEM_MAX_BUFFERS];
00048        struct drm_nouveau_gem_pushbuf_reloc reloc[NOUVEAU_GEM_MAX_RELOCS];
00049        struct drm_nouveau_gem_pushbuf_push push[NOUVEAU_GEM_MAX_PUSH];
00050        int nr_buffer;
00051        int nr_reloc;
00052        int nr_push;
00053        uint64_t vram_used;
00054        uint64_t gart_used;
00055 };
00056 
00057 struct nouveau_pushbuf_priv {
00058        struct nouveau_pushbuf base;
00059        struct nouveau_pushbuf_krec *list;
00060        struct nouveau_pushbuf_krec *krec;
00061        struct nouveau_list bctx_list;
00062        struct nouveau_bo *bo;
00063        uint32_t type;
00064        uint32_t suffix0;
00065        uint32_t suffix1;
00066        uint32_t *ptr;
00067        uint32_t *bgn;
00068        int bo_next;
00069        int bo_nr;
00070        struct nouveau_bo *bos[];
00071 };
00072 
00073 static inline struct nouveau_pushbuf_priv *
00074 nouveau_pushbuf(struct nouveau_pushbuf *push)
00075 {
00076        return (struct nouveau_pushbuf_priv *)push;
00077 }
00078 
00079 static int pushbuf_validate(struct nouveau_pushbuf *, bool);
00080 static int pushbuf_flush(struct nouveau_pushbuf *);
00081 
00082 static bool
00083 pushbuf_kref_fits(struct nouveau_pushbuf *push, struct nouveau_bo *bo,
00084                 uint32_t *domains)
00085 {
00086        struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
00087        struct nouveau_pushbuf_krec *krec = nvpb->krec;
00088        struct nouveau_device *dev = push->client->device;
00089        struct nouveau_bo *kbo;
00090        struct drm_nouveau_gem_pushbuf_bo *kref;
00091        int i;
00092 
00093        /* VRAM is the only valid domain.  GART and VRAM|GART buffers
00094         * are all accounted to GART, so if this doesn't fit in VRAM
00095         * straight up, a flush is needed.
00096         */
00097        if (*domains == NOUVEAU_GEM_DOMAIN_VRAM) {
00098               if (krec->vram_used + bo->size > dev->vram_limit)
00099                      return false;
00100               krec->vram_used += bo->size;
00101               return true;
00102        }
00103 
00104        /* GART or VRAM|GART buffer.  Account both of these buffer types
00105         * to GART only for the moment, which simplifies things.  If the
00106         * buffer can fit already, we're done here.
00107         */
00108        if (krec->gart_used + bo->size <= dev->gart_limit) {
00109               krec->gart_used += bo->size;
00110               return true;
00111        }
00112 
00113        /* Ran out of GART space, if it's a VRAM|GART buffer and it'll
00114         * fit into available VRAM, turn it into a VRAM buffer
00115         */
00116        if ((*domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
00117            krec->vram_used + bo->size <= dev->vram_limit) {
00118               *domains &= NOUVEAU_GEM_DOMAIN_VRAM;
00119               krec->vram_used += bo->size;
00120               return true;
00121        }
00122 
00123        /* Still couldn't fit the buffer in anywhere, so as a last resort;
00124         * scan the buffer list for VRAM|GART buffers and turn them into
00125         * VRAM buffers until we have enough space in GART for this one
00126         */
00127        kref = krec->buffer;
00128        for (i = 0; i < krec->nr_buffer; i++, kref++) {
00129               if (!(kref->valid_domains & NOUVEAU_GEM_DOMAIN_GART))
00130                      continue;
00131 
00132               kbo = (void *)(unsigned long)kref->user_priv;
00133               if (!(kref->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) ||
00134                   krec->vram_used + kbo->size > dev->vram_limit)
00135                      continue;
00136 
00137               kref->valid_domains &= NOUVEAU_GEM_DOMAIN_VRAM;
00138               krec->gart_used -= kbo->size;
00139               krec->vram_used += kbo->size;
00140               if (krec->gart_used + bo->size <= dev->gart_limit) {
00141                      krec->gart_used += bo->size;
00142                      return true;
00143               }
00144        }
00145 
00146        /* Couldn't resolve a placement, need to force a flush */
00147        return false;
00148 }
00149 
00150 static struct drm_nouveau_gem_pushbuf_bo *
00151 pushbuf_kref(struct nouveau_pushbuf *push, struct nouveau_bo *bo,
00152             uint32_t flags)
00153 {
00154        struct nouveau_device *dev = push->client->device;
00155        struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
00156        struct nouveau_pushbuf_krec *krec = nvpb->krec;
00157        struct nouveau_pushbuf *fpush;
00158        struct drm_nouveau_gem_pushbuf_bo *kref;
00159        uint32_t domains, domains_wr, domains_rd;
00160 
00161        domains = 0;
00162        if (flags & NOUVEAU_BO_VRAM)
00163               domains |= NOUVEAU_GEM_DOMAIN_VRAM;
00164        if (flags & NOUVEAU_BO_GART)
00165               domains |= NOUVEAU_GEM_DOMAIN_GART;
00166        domains_wr = domains * !!(flags & NOUVEAU_BO_WR);
00167        domains_rd = domains * !!(flags & NOUVEAU_BO_RD);
00168 
00169        /* if buffer is referenced on another pushbuf that is owned by the
00170         * same client, we need to flush the other pushbuf first to ensure
00171         * the correct ordering of commands
00172         */
00173        fpush = cli_push_get(push->client, bo);
00174        if (fpush && fpush != push)
00175               pushbuf_flush(fpush);
00176 
00177        kref = cli_kref_get(push->client, bo);
00178        if (kref) {
00179               /* possible conflict in memory types - flush and retry */
00180               if (!(kref->valid_domains & domains))
00181                      return NULL;
00182 
00183               /* VRAM|GART buffer turning into a VRAM buffer.  Make sure
00184                * it'll fit in VRAM and force a flush if not.
00185                */
00186               if ((kref->valid_domains  & NOUVEAU_GEM_DOMAIN_GART) &&
00187                   (            domains == NOUVEAU_GEM_DOMAIN_VRAM)) {
00188                      if (krec->vram_used + bo->size > dev->vram_limit)
00189                             return NULL;
00190                      krec->vram_used += bo->size;
00191                      krec->gart_used -= bo->size;
00192               }
00193 
00194               kref->valid_domains &= domains;
00195               kref->write_domains |= domains_wr;
00196               kref->read_domains  |= domains_rd;
00197        } else {
00198               if (krec->nr_buffer == NOUVEAU_GEM_MAX_BUFFERS ||
00199                   !pushbuf_kref_fits(push, bo, &domains))
00200                      return NULL;
00201 
00202               kref = &krec->buffer[krec->nr_buffer++];
00203               kref->user_priv = (unsigned long)bo;
00204               kref->handle = bo->handle;
00205               kref->valid_domains = domains;
00206               kref->write_domains = domains_wr;
00207               kref->read_domains = domains_rd;
00208               kref->presumed.valid = 1;
00209               kref->presumed.offset = bo->offset;
00210               if (bo->flags & NOUVEAU_BO_VRAM)
00211                      kref->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
00212               else
00213                      kref->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
00214 
00215               cli_kref_set(push->client, bo, kref, push);
00216               atomic_inc(&nouveau_bo(bo)->refcnt);
00217        }
00218 
00219        return kref;
00220 }
00221 
00222 static uint32_t
00223 pushbuf_krel(struct nouveau_pushbuf *push, struct nouveau_bo *bo,
00224             uint32_t data, uint32_t flags, uint32_t vor, uint32_t tor)
00225 {
00226        struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
00227        struct nouveau_pushbuf_krec *krec = nvpb->krec;
00228        struct drm_nouveau_gem_pushbuf_reloc *krel;
00229        struct drm_nouveau_gem_pushbuf_bo *pkref;
00230        struct drm_nouveau_gem_pushbuf_bo *bkref;
00231        uint32_t reloc = data;
00232 
00233        pkref = cli_kref_get(push->client, nvpb->bo);
00234        bkref = cli_kref_get(push->client, bo);
00235        krel  = &krec->reloc[krec->nr_reloc++];
00236 
00237        krel->reloc_bo_index = pkref - krec->buffer;
00238        krel->reloc_bo_offset = (push->cur - nvpb->ptr) * 4;
00239        krel->bo_index = bkref - krec->buffer;
00240        krel->flags = 0;
00241        krel->data = data;
00242        krel->vor = vor;
00243        krel->tor = tor;
00244 
00245        if (flags & NOUVEAU_BO_LOW) {
00246               reloc = (bkref->presumed.offset + data);
00247               krel->flags |= NOUVEAU_GEM_RELOC_LOW;
00248        } else
00249        if (flags & NOUVEAU_BO_HIGH) {
00250               reloc = (bkref->presumed.offset + data) >> 32;
00251               krel->flags |= NOUVEAU_GEM_RELOC_HIGH;
00252        }
00253        if (flags & NOUVEAU_BO_OR) {
00254               if (bkref->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM)
00255                      reloc |= vor;
00256               else
00257                      reloc |= tor;
00258               krel->flags |= NOUVEAU_GEM_RELOC_OR;
00259        }
00260 
00261        return reloc;
00262 }
00263 
00264 static void
00265 pushbuf_dump(struct nouveau_pushbuf_krec *krec, int krec_id, int chid)
00266 {
00267        struct drm_nouveau_gem_pushbuf_reloc *krel;
00268        struct drm_nouveau_gem_pushbuf_push *kpsh;
00269        struct drm_nouveau_gem_pushbuf_bo *kref;
00270        struct nouveau_bo *bo;
00271        uint32_t *bgn, *end;
00272        int i;
00273 
00274        err("ch%d: krec %d pushes %d bufs %d relocs %d\n", chid,
00275            krec_id, krec->nr_push, krec->nr_buffer, krec->nr_reloc);
00276 
00277        kref = krec->buffer;
00278        for (i = 0; i < krec->nr_buffer; i++, kref++) {
00279               err("ch%d: buf %08x %08x %08x %08x %08x\n", chid, i,
00280                   kref->handle, kref->valid_domains,
00281                   kref->read_domains, kref->write_domains);
00282        }
00283 
00284        krel = krec->reloc;
00285        for (i = 0; i < krec->nr_reloc; i++, krel++) {
00286               err("ch%d: rel %08x %08x %08x %08x %08x %08x %08x\n",
00287                   chid, krel->reloc_bo_index, krel->reloc_bo_offset,
00288                   krel->bo_index, krel->flags, krel->data,
00289                   krel->vor, krel->tor);
00290        }
00291 
00292        kpsh = krec->push;
00293        for (i = 0; i < krec->nr_push; i++, kpsh++) {
00294               kref = krec->buffer + kpsh->bo_index;
00295               bo = (void *)(unsigned long)kref->user_priv;
00296               bgn = (uint32_t *)((char *)bo->map + kpsh->offset);
00297               end = bgn + (kpsh->length /4);
00298 
00299               err("ch%d: psh %08x %010llx %010llx\n", chid, kpsh->bo_index,
00300                   (unsigned long long)kpsh->offset,
00301                   (unsigned long long)(kpsh->offset + kpsh->length));
00302               while (bgn < end)
00303                      err("\t0x%08x\n", *bgn++);
00304        }
00305 }
00306 
00307 static int
00308 pushbuf_submit(struct nouveau_pushbuf *push, struct nouveau_object *chan)
00309 {
00310        struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
00311        struct nouveau_pushbuf_krec *krec = nvpb->list;
00312        struct nouveau_device *dev = push->client->device;
00313        struct drm_nouveau_gem_pushbuf_bo_presumed *info;
00314        struct drm_nouveau_gem_pushbuf_bo *kref;
00315        struct drm_nouveau_gem_pushbuf req;
00316        struct nouveau_fifo *fifo = chan->data;
00317        struct nouveau_bo *bo;
00318        int krec_id = 0;
00319        int ret = 0, i;
00320 
00321        if (chan->oclass != NOUVEAU_FIFO_CHANNEL_CLASS)
00322               return -EINVAL;
00323 
00324        if (push->kick_notify)
00325               push->kick_notify(push);
00326 
00327        nouveau_pushbuf_data(push, NULL, 0, 0);
00328 
00329        while (krec && krec->nr_push) {
00330               req.channel = fifo->channel;
00331               req.nr_buffers = krec->nr_buffer;
00332               req.buffers = (uint64_t)(unsigned long)krec->buffer;
00333               req.nr_relocs = krec->nr_reloc;
00334               req.nr_push = krec->nr_push;
00335               req.relocs = (uint64_t)(unsigned long)krec->reloc;
00336               req.push = (uint64_t)(unsigned long)krec->push;
00337               req.suffix0 = nvpb->suffix0;
00338               req.suffix1 = nvpb->suffix1;
00339               req.vram_available = 0; /* for valgrind */
00340               req.gart_available = 0;
00341 
00342               if (dbg_on(0))
00343                      pushbuf_dump(krec, krec_id++, fifo->channel);
00344 
00345 #ifndef SIMULATE
00346               ret = drmCommandWriteRead(dev->fd, DRM_NOUVEAU_GEM_PUSHBUF,
00347                                      &req, sizeof(req));
00348               nvpb->suffix0 = req.suffix0;
00349               nvpb->suffix1 = req.suffix1;
00350               dev->vram_limit = (req.vram_available * 80) / 100;
00351               dev->gart_limit = (req.gart_available * 80) / 100;
00352 #else
00353               if (dbg_on(31))
00354                      ret = -EINVAL;
00355 #endif
00356 
00357               if (ret) {
00358                      err("kernel rejected pushbuf: %s\n", strerror(-ret));
00359                      pushbuf_dump(krec, krec_id++, fifo->channel);
00360                      break;
00361               }
00362 
00363               kref = krec->buffer;
00364               for (i = 0; i < krec->nr_buffer; i++, kref++) {
00365                      bo = (void *)(unsigned long)kref->user_priv;
00366 
00367                      info = &kref->presumed;
00368                      if (!info->valid) {
00369                             bo->flags &= ~NOUVEAU_BO_APER;
00370                             if (info->domain == NOUVEAU_GEM_DOMAIN_VRAM)
00371                                    bo->flags |= NOUVEAU_BO_VRAM;
00372                             else
00373                                    bo->flags |= NOUVEAU_BO_GART;
00374                             bo->offset = info->offset;
00375                      }
00376 
00377                      if (kref->write_domains)
00378                             nouveau_bo(bo)->access |= NOUVEAU_BO_WR;
00379                      if (kref->read_domains)
00380                             nouveau_bo(bo)->access |= NOUVEAU_BO_RD;
00381               }
00382 
00383               krec = krec->next;
00384        }
00385 
00386        return ret;
00387 }
00388 
00389 static int
00390 pushbuf_flush(struct nouveau_pushbuf *push)
00391 {
00392        struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
00393        struct nouveau_pushbuf_krec *krec = nvpb->krec;
00394        struct drm_nouveau_gem_pushbuf_bo *kref;
00395        struct nouveau_bufctx *bctx, *btmp;
00396        struct nouveau_bo *bo;
00397        int ret = 0, i;
00398 
00399        if (push->channel) {
00400               ret = pushbuf_submit(push, push->channel);
00401        } else {
00402               nouveau_pushbuf_data(push, NULL, 0, 0);
00403               krec->next = malloc(sizeof(*krec));
00404               nvpb->krec = krec->next;
00405        }
00406 
00407        kref = krec->buffer;
00408        for (i = 0; i < krec->nr_buffer; i++, kref++) {
00409               bo = (void *)(unsigned long)kref->user_priv;
00410               cli_kref_set(push->client, bo, NULL, NULL);
00411               if (push->channel)
00412                      nouveau_bo_ref(NULL, &bo);
00413        }
00414 
00415        krec = nvpb->krec;
00416        krec->vram_used = 0;
00417        krec->gart_used = 0;
00418        krec->nr_buffer = 0;
00419        krec->nr_reloc = 0;
00420        krec->nr_push = 0;
00421 
00422        DRMLISTFOREACHENTRYSAFE(bctx, btmp, &nvpb->bctx_list, head) {
00423               DRMLISTJOIN(&bctx->current, &bctx->pending);
00424               DRMINITLISTHEAD(&bctx->current);
00425               DRMLISTDELINIT(&bctx->head);
00426        }
00427 
00428        return ret;
00429 }
00430 
00431 static void
00432 pushbuf_refn_fail(struct nouveau_pushbuf *push, int sref, int srel)
00433 {
00434        struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
00435        struct nouveau_pushbuf_krec *krec = nvpb->krec;
00436        struct drm_nouveau_gem_pushbuf_bo *kref;
00437 
00438        kref = krec->buffer + sref;
00439        while (krec->nr_buffer-- > sref) {
00440               struct nouveau_bo *bo = (void *)(unsigned long)kref->user_priv;
00441               cli_kref_set(push->client, bo, NULL, NULL);
00442               nouveau_bo_ref(NULL, &bo);
00443               kref++;
00444        }
00445        krec->nr_buffer = sref;
00446        krec->nr_reloc = srel;
00447 }
00448 
00449 static int
00450 pushbuf_refn(struct nouveau_pushbuf *push, bool retry,
00451             struct nouveau_pushbuf_refn *refs, int nr)
00452 {
00453        struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
00454        struct nouveau_pushbuf_krec *krec = nvpb->krec;
00455        struct drm_nouveau_gem_pushbuf_bo *kref;
00456        int sref = krec->nr_buffer;
00457        int ret = 0, i;
00458 
00459        for (i = 0; i < nr; i++) {
00460               kref = pushbuf_kref(push, refs[i].bo, refs[i].flags);
00461               if (!kref) {
00462                      ret = -ENOSPC;
00463                      break;
00464               }
00465        }
00466 
00467        if (ret) {
00468               pushbuf_refn_fail(push, sref, krec->nr_reloc);
00469               if (retry) {
00470                      pushbuf_flush(push);
00471                      nouveau_pushbuf_space(push, 0, 0, 0);
00472                      return pushbuf_refn(push, false, refs, nr);
00473               }
00474        }
00475 
00476        return ret;
00477 }
00478 
00479 static int
00480 pushbuf_validate(struct nouveau_pushbuf *push, bool retry)
00481 {
00482        struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
00483        struct nouveau_pushbuf_krec *krec = nvpb->krec;
00484        struct drm_nouveau_gem_pushbuf_bo *kref;
00485        struct nouveau_bufctx *bctx = push->bufctx;
00486        struct nouveau_bufref *bref;
00487        int relocs = bctx ? bctx->relocs * 2: 0;
00488        int sref, srel, ret;
00489 
00490        ret = nouveau_pushbuf_space(push, relocs, relocs, 0);
00491        if (ret || bctx == NULL)
00492               return ret;
00493 
00494        sref = krec->nr_buffer;
00495        srel = krec->nr_reloc;
00496 
00497        DRMLISTDEL(&bctx->head);
00498        DRMLISTADD(&bctx->head, &nvpb->bctx_list);
00499 
00500        DRMLISTFOREACHENTRY(bref, &bctx->pending, thead) {
00501               kref = pushbuf_kref(push, bref->bo, bref->flags);
00502               if (!kref) {
00503                      ret = -ENOSPC;
00504                      break;
00505               }
00506 
00507               if (bref->packet) {
00508                      pushbuf_krel(push, bref->bo, bref->packet, 0, 0, 0);
00509                      *push->cur++ = 0;
00510                      pushbuf_krel(push, bref->bo, bref->data, bref->flags,
00511                                       bref->vor, bref->tor);
00512                      *push->cur++ = 0;
00513               }
00514        }
00515 
00516        DRMLISTJOIN(&bctx->pending, &bctx->current);
00517        DRMINITLISTHEAD(&bctx->pending);
00518 
00519        if (ret) {
00520               pushbuf_refn_fail(push, sref, srel);
00521               if (retry) {
00522                      pushbuf_flush(push);
00523                      return pushbuf_validate(push, false);
00524               }
00525        }
00526 
00527        return 0;
00528 }
00529 
00530 int
00531 nouveau_pushbuf_new(struct nouveau_client *client, struct nouveau_object *chan,
00532                   int nr, uint32_t size, bool immediate,
00533                   struct nouveau_pushbuf **ppush)
00534 {
00535        struct nouveau_device *dev = client->device;
00536        struct nouveau_fifo *fifo = chan->data;
00537        struct nouveau_pushbuf_priv *nvpb;
00538        struct nouveau_pushbuf *push;
00539        struct drm_nouveau_gem_pushbuf req = {};
00540        int ret;
00541 
00542        if (chan->oclass != NOUVEAU_FIFO_CHANNEL_CLASS)
00543               return -EINVAL;
00544 
00545        /* nop pushbuf call, to get the current "return to main" sequence
00546         * we need to append to the pushbuf on early chipsets
00547         */
00548        req.channel = fifo->channel;
00549        req.nr_push = 0;
00550        ret = drmCommandWriteRead(dev->fd, DRM_NOUVEAU_GEM_PUSHBUF,
00551                               &req, sizeof(req));
00552        if (ret)
00553               return ret;
00554 
00555        nvpb = calloc(1, sizeof(*nvpb) + nr * sizeof(*nvpb->bos));
00556        if (!nvpb)
00557               return -ENOMEM;
00558 
00559 #ifndef SIMULATE
00560        nvpb->suffix0 = req.suffix0;
00561        nvpb->suffix1 = req.suffix1;
00562 #else
00563        nvpb->suffix0 = 0xffffffff;
00564        nvpb->suffix1 = 0xffffffff;
00565 #endif
00566        nvpb->krec = calloc(1, sizeof(*nvpb->krec));
00567        nvpb->list = nvpb->krec;
00568        if (!nvpb->krec) {
00569               free(nvpb);
00570               return -ENOMEM;
00571        }
00572 
00573        push = &nvpb->base;
00574        push->client = client;
00575        push->channel = immediate ? chan : NULL;
00576        push->flags = NOUVEAU_BO_RD;
00577        if (fifo->pushbuf & NOUVEAU_GEM_DOMAIN_VRAM) {
00578               push->flags |= NOUVEAU_BO_VRAM;
00579               nvpb->type   = NOUVEAU_BO_VRAM;
00580        }
00581        if (fifo->pushbuf & NOUVEAU_GEM_DOMAIN_GART) {
00582               push->flags |= NOUVEAU_BO_GART;
00583               nvpb->type   = NOUVEAU_BO_GART;
00584        }
00585        nvpb->type |= NOUVEAU_BO_MAP;
00586 
00587        for (nvpb->bo_nr = 0; nvpb->bo_nr < nr; nvpb->bo_nr++) {
00588               ret = nouveau_bo_new(client->device, nvpb->type, 0, size,
00589                                  NULL, &nvpb->bos[nvpb->bo_nr]);
00590               if (ret) {
00591                      nouveau_pushbuf_del(&push);
00592                      return ret;
00593               }
00594        }
00595 
00596        DRMINITLISTHEAD(&nvpb->bctx_list);
00597        *ppush = push;
00598        return 0;
00599 }
00600 
00601 void
00602 nouveau_pushbuf_del(struct nouveau_pushbuf **ppush)
00603 {
00604        struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(*ppush);
00605        if (nvpb) {
00606               struct drm_nouveau_gem_pushbuf_bo *kref;
00607               struct nouveau_pushbuf_krec *krec;
00608               while ((krec = nvpb->list)) {
00609                      kref = krec->buffer;
00610                      while (krec->nr_buffer--) {
00611                             unsigned long priv = kref++->user_priv;
00612                             struct nouveau_bo *bo = (void *)priv;
00613                             cli_kref_set(nvpb->base.client, bo, NULL, NULL);
00614                             nouveau_bo_ref(NULL, &bo);
00615                      }
00616                      nvpb->list = krec->next;
00617                      free(krec);
00618               }
00619               while (nvpb->bo_nr--)
00620                      nouveau_bo_ref(NULL, &nvpb->bos[nvpb->bo_nr]);
00621               nouveau_bo_ref(NULL, &nvpb->bo);
00622               free(nvpb);
00623        }
00624        *ppush = NULL;
00625 }
00626 
00627 struct nouveau_bufctx *
00628 nouveau_pushbuf_bufctx(struct nouveau_pushbuf *push, struct nouveau_bufctx *ctx)
00629 {
00630        struct nouveau_bufctx *prev = push->bufctx;
00631        push->bufctx = ctx;
00632        return prev;
00633 }
00634 
00635 int
00636 nouveau_pushbuf_space(struct nouveau_pushbuf *push,
00637                     uint32_t dwords, uint32_t relocs, uint32_t pushes)
00638 {
00639        struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
00640        struct nouveau_pushbuf_krec *krec = nvpb->krec;
00641        struct nouveau_client *client = push->client;
00642        struct nouveau_bo *bo = NULL;
00643        bool flushed = false;
00644        int ret = 0;
00645 
00646        /* switch to next buffer if insufficient space in the current one */
00647        if (push->cur + dwords >= push->end) {
00648               if (nvpb->bo_next < nvpb->bo_nr) {
00649                      nouveau_bo_ref(nvpb->bos[nvpb->bo_next++], &bo);
00650                      if (nvpb->bo_next == nvpb->bo_nr && push->channel)
00651                             nvpb->bo_next = 0;
00652               } else {
00653                      ret = nouveau_bo_new(client->device, nvpb->type, 0,
00654                                         nvpb->bos[0]->size, NULL, &bo);
00655                      if (ret)
00656                             return ret;
00657               }
00658        }
00659 
00660        /* make sure there's always enough space to queue up the pending
00661         * data in the pushbuf proper
00662         */
00663        pushes++;
00664 
00665        /* need to flush if we've run out of space on an immediate pushbuf,
00666         * if the new buffer won't fit, or if the kernel push/reloc limits
00667         * have been hit
00668         */
00669        if ((bo && ( push->channel ||
00670                   !pushbuf_kref(push, bo, push->flags))) ||
00671            krec->nr_reloc + relocs >= NOUVEAU_GEM_MAX_RELOCS ||
00672            krec->nr_push + pushes >= NOUVEAU_GEM_MAX_PUSH) {
00673               if (nvpb->bo && krec->nr_buffer)
00674                      pushbuf_flush(push);
00675               flushed = true;
00676        }
00677 
00678        /* if necessary, switch to new buffer */
00679        if (bo) {
00680               ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, push->client);
00681               if (ret)
00682                      return ret;
00683 
00684               nouveau_pushbuf_data(push, NULL, 0, 0);
00685               nouveau_bo_ref(bo, &nvpb->bo);
00686               nouveau_bo_ref(NULL, &bo);
00687 
00688               nvpb->bgn = nvpb->bo->map;
00689               nvpb->ptr = nvpb->bgn;
00690               push->cur = nvpb->bgn;
00691               push->end = push->cur + (nvpb->bo->size / 4);
00692               push->end -= 2 + push->rsvd_kick; /* space for suffix */
00693        }
00694 
00695        pushbuf_kref(push, nvpb->bo, push->flags);
00696        return flushed ? pushbuf_validate(push, false) : 0;
00697 }
00698 
00699 void
00700 nouveau_pushbuf_data(struct nouveau_pushbuf *push, struct nouveau_bo *bo,
00701                    uint64_t offset, uint64_t length)
00702 {
00703        struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
00704        struct nouveau_pushbuf_krec *krec = nvpb->krec;
00705        struct drm_nouveau_gem_pushbuf_push *kpsh;
00706        struct drm_nouveau_gem_pushbuf_bo *kref;
00707 
00708        if (bo != nvpb->bo && nvpb->bgn != push->cur) {
00709               if (nvpb->suffix0 || nvpb->suffix1) {
00710                      *push->cur++ = nvpb->suffix0;
00711                      *push->cur++ = nvpb->suffix1;
00712               }
00713 
00714               nouveau_pushbuf_data(push, nvpb->bo,
00715                                  (nvpb->bgn - nvpb->ptr) * 4,
00716                                  (push->cur - nvpb->bgn) * 4);
00717               nvpb->bgn = push->cur;
00718        }
00719 
00720        if (bo) {
00721               kref = cli_kref_get(push->client, bo);
00722               kpsh = &krec->push[krec->nr_push++];
00723               kpsh->bo_index = kref - krec->buffer;
00724               kpsh->offset   = offset;
00725               kpsh->length   = length;
00726        }
00727 }
00728 
00729 int
00730 nouveau_pushbuf_refn(struct nouveau_pushbuf *push,
00731                    struct nouveau_pushbuf_refn *refs, int nr)
00732 {
00733        return pushbuf_refn(push, true, refs, nr);
00734 }
00735 
00736 void
00737 nouveau_pushbuf_reloc(struct nouveau_pushbuf *push, struct nouveau_bo *bo,
00738                     uint32_t data, uint32_t flags, uint32_t vor, uint32_t tor)
00739 {
00740        *push->cur++ = pushbuf_krel(push, bo, data, flags, vor, tor);
00741 }
00742 
00743 int
00744 nouveau_pushbuf_validate(struct nouveau_pushbuf *push)
00745 {
00746        return pushbuf_validate(push, true);
00747 }
00748 
00749 uint32_t
00750 nouveau_pushbuf_refd(struct nouveau_pushbuf *push, struct nouveau_bo *bo)
00751 {
00752        struct drm_nouveau_gem_pushbuf_bo *kref;
00753        uint32_t flags = 0;
00754 
00755        if (cli_push_get(push->client, bo) == push) {
00756               kref = cli_kref_get(push->client, bo);
00757               if (kref->read_domains)
00758                      flags |= NOUVEAU_BO_RD;
00759               if (kref->write_domains)
00760                      flags |= NOUVEAU_BO_WR;
00761        }
00762 
00763        return flags;
00764 }
00765 
00766 int
00767 nouveau_pushbuf_kick(struct nouveau_pushbuf *push, struct nouveau_object *chan)
00768 {
00769        if (!push->channel)
00770               return pushbuf_submit(push, chan);
00771        pushbuf_flush(push);
00772        return pushbuf_validate(push, false);
00773 }