Back to index

avfs  1.0.1
zread.c
Go to the documentation of this file.
00001 /*
00002     AVFS: A Virtual File System Library
00003     Copyright (C) 2000-2001  Miklos Szeredi <miklos@szeredi.hu>
00004 
00005     This program can be distributed under the terms of the GNU GPL.
00006     See the file COPYING.
00007 */
00008 
00009 #include "config.h"
00010 #include "zfile.h"
00011 #include "zlib.h"
00012 #include "oper.h"
00013 
00014 #include <stdlib.h>
00015 #include <fcntl.h>
00016 
00017 #define INDEXDISTANCE 1048576
00018 
00019 #define INBUFSIZE 16384
00020 #define OUTBUFSIZE 32768
00021 
00022 /* This is the 'cost' of the restoration from the index cache */
00023 #define ZCACHE_EXTRA_DIST 45000
00024 
00025 /* It is not worth it to compress the state better */
00026 #define STATE_COMPRESS_LEVEL 1
00027 
00028 struct streamcache {
00029     int id;
00030     z_stream s;
00031     int calccrc;
00032     int iseof;
00033 };
00034 
00035 static struct streamcache scache;
00036 static int zread_nextid;
00037 static AV_LOCK_DECL(zread_lock);
00038 
00039 struct zindex {
00040     avoff_t offset;          /* The number of output bytes */
00041     avoff_t indexoffset;     /* Offset in the indexfile */
00042     avsize_t indexsize;      /* Size of state record */
00043     struct zindex *next;
00044 };
00045 
00046 struct zcache {
00047     char *indexfile;
00048     avoff_t filesize;
00049     avoff_t nextindex;
00050     avoff_t size;
00051     int id;
00052     struct zindex *indexes;
00053     avmutex lock;
00054     int crc_ok;
00055 };
00056 
00057 struct zfile {
00058     z_stream s;
00059     int iseof;
00060     int iserror;
00061     int id; /* Hack: the id of the last used zcache */
00062     int calccrc;
00063     avuint crc;
00064     
00065     vfile *infile;
00066     avoff_t dataoff;
00067     char inbuf[INBUFSIZE];
00068 };
00069 
00070 #ifndef USE_SYSTEM_ZLIB
00071 static int zfile_compress_state(char *state, int statelen, char **resp)
00072 {
00073     int res;
00074     z_stream s;
00075     int bufsize = sizeof(int) + statelen + statelen / 1000 + 1 + 12;
00076     char *cstate = av_malloc(bufsize);
00077     
00078     s.next_in = (Bytef*)state;
00079     s.avail_in = statelen;
00080     s.next_out = (Bytef*)( cstate + sizeof(int) );
00081     s.avail_out = bufsize - sizeof(int);
00082     s.zalloc = NULL;
00083     s.zfree = NULL;
00084 
00085     ((int *) cstate)[0] = statelen;
00086 
00087     res = deflateInit(&s, STATE_COMPRESS_LEVEL);
00088     if(res != Z_OK) {
00089         av_log(AVLOG_ERROR, "ZFILE: compress state failed");
00090         av_free(cstate);
00091         return -EIO;
00092     }
00093 
00094     res = deflate(&s, Z_FINISH);
00095     if(res != Z_STREAM_END) {
00096         av_log(AVLOG_ERROR, "ZFILE: compress state failed");
00097         av_free(cstate);
00098         return -EIO;
00099     }
00100     
00101     res = deflateEnd(&s);
00102     if(res != Z_OK) {
00103         av_log(AVLOG_ERROR, "ZFILE: compress state failed");
00104         av_free(cstate);
00105         return -EIO;
00106     }
00107     
00108     *resp = cstate;
00109     return sizeof(int) + s.total_out;
00110 }
00111 
00112 static int zfile_uncompress_state(char *cstate, int cstatelen, char **resp)
00113 {
00114     int res;
00115     z_stream s;
00116     int statelen = ((int *) cstate)[0];
00117     char *state = av_malloc(statelen);
00118     
00119     s.next_in = (Bytef*)( cstate + sizeof(int) );
00120     s.avail_in = cstatelen - sizeof(int);
00121     s.next_out = (Bytef*)state;
00122     s.avail_out = statelen;
00123     s.zalloc = NULL;
00124     s.zfree = NULL;
00125 
00126     res = inflateInit(&s);
00127     if(res != Z_OK) {
00128         av_log(AVLOG_ERROR, "ZFILE: compress state failed");
00129         av_free(state);
00130         return -EIO;
00131     }
00132 
00133     res = inflate(&s, Z_FINISH);
00134     if(res != Z_STREAM_END) {
00135         av_log(AVLOG_ERROR, "ZFILE: compress state failed");
00136         av_free(state);
00137         return -EIO;
00138     }
00139     
00140     res = inflateEnd(&s);
00141     if(res != Z_OK) {
00142         av_log(AVLOG_ERROR, "ZFILE: compress state failed");
00143         av_free(state);
00144         return -EIO;
00145     }
00146     
00147     *resp = state;
00148     return 0;
00149 }
00150 #endif
00151 
00152 static void zfile_scache_cleanup(void)
00153 {
00154     inflateEnd(&scache.s);
00155 }
00156 
00157 static void zfile_scache_save(int id, z_stream *s, int calccrc, int iseof)
00158 {
00159     int res;
00160 
00161     if(id == 0 || iseof) {
00162         res = inflateEnd(s);
00163         if(res != Z_OK) {
00164             av_log(AVLOG_ERROR, "ZFILE: inflateEnd: %s (%i)",
00165                    s->msg == NULL ? "" : s->msg, res);
00166         }
00167         return;
00168     }
00169 
00170     if(scache.id != 0) {
00171         res = inflateEnd(&scache.s);
00172         if(res != Z_OK) {
00173             av_log(AVLOG_ERROR, "ZFILE: inflateEnd: %s (%i)",
00174                    scache.s.msg == NULL ? "" : scache.s.msg, res);
00175         }
00176     }
00177     if(scache.id == 0)
00178         atexit(zfile_scache_cleanup);
00179 
00180     scache.id = id;
00181     scache.s = *s;
00182     scache.calccrc = calccrc;
00183     scache.iseof = iseof;
00184 }
00185 
00186 static int zfile_reset(struct zfile *fil)
00187 {
00188     int res;
00189 
00190     /* FIXME: Is it a good idea to save the previous state or not? */
00191     zfile_scache_save(fil->id, &fil->s, fil->calccrc, fil->iseof);
00192     memset(&fil->s, 0, sizeof(z_stream));
00193     res = inflateInit2(&fil->s, -MAX_WBITS);
00194     if(res != Z_OK) {
00195         av_log(AVLOG_ERROR, "ZFILE: inflateInit: %s (%i)",
00196                fil->s.msg == NULL ? "" : fil->s.msg, res);
00197         return -EIO;
00198     }
00199     fil->s.adler = 0;
00200     fil->iseof = 0;
00201     fil->calccrc = 0;
00202 
00203     return 0;
00204 }
00205 
00206 #ifndef USE_SYSTEM_ZLIB
00207 static int zfile_save_state(struct zcache *zc, char *state, int statesize,
00208                             avoff_t offset)
00209 {
00210     int fd;
00211     int res;
00212     struct zindex **zp;
00213     struct zindex *zi;
00214 
00215     for(zp = &zc->indexes; *zp != NULL; zp = &(*zp)->next);
00216 
00217     fd = open(zc->indexfile, O_WRONLY | O_CREAT, 0600);
00218     if(fd == -1) {
00219         av_log(AVLOG_ERROR, "ZFILE: Error opening indexfile %s: %s",
00220                zc->indexfile, strerror(errno));
00221         return -EIO;
00222     }
00223     
00224     lseek(fd, zc->filesize, SEEK_SET);
00225     
00226     res = write(fd, state, statesize);
00227     close(fd);
00228     if(res == -1) {
00229         av_log(AVLOG_ERROR, "ZFILE: Error writing indexfile %s: %s",
00230                zc->indexfile, strerror(errno));
00231         return -EIO;
00232     }
00233 
00234     AV_NEW(zi);
00235     zi->offset = offset;
00236     zi->indexoffset = zc->filesize;
00237     zi->indexsize = statesize;
00238     zi->next = NULL;
00239     
00240     *zp = zi;
00241 
00242     zc->nextindex += INDEXDISTANCE;
00243     zc->filesize += statesize;
00244     
00245     return 0;
00246 }
00247 
00248 
00249 static int zfile_save_index(struct zfile *fil, struct zcache *zc)
00250 {
00251     int res;
00252     char *state;
00253     char *cstate;
00254 
00255     res = inflateSave(&fil->s, &state);
00256     if(res < 0) {
00257         av_log(AVLOG_ERROR, "ZFILE: inflateSave: (%i)", res);
00258         return -EIO;
00259     }
00260     
00261     res = zfile_compress_state(state, res, &cstate);
00262     free(state);
00263     if(res < 0)
00264         return res;
00265 
00266     res = zfile_save_state(zc, cstate, res, fil->s.total_out);
00267     av_free(cstate);
00268 
00269     return res;
00270 }
00271 
00272 static int zfile_seek_index(struct zfile *fil, struct zcache *zc, 
00273                             struct zindex *zi)
00274 {
00275     int fd;
00276     int res;
00277     char *cstate;
00278     char *state;
00279 
00280     /* FIXME: Is it a good idea to save the previous state or not? */
00281     zfile_scache_save(fil->id, &fil->s, fil->calccrc, fil->iseof);
00282     memset(&fil->s, 0, sizeof(z_stream));
00283 
00284     fd = open(zc->indexfile, O_RDONLY, 0);
00285     if(fd == -1) {
00286         av_log(AVLOG_ERROR, "ZFILE: Error opening indexfile %s: %s",
00287                zc->indexfile, strerror(errno));
00288         return -EIO;
00289     }
00290     
00291     lseek(fd, zi->indexoffset, SEEK_SET);
00292 
00293     cstate = av_malloc(zi->indexsize);
00294     res = read(fd, cstate, zi->indexsize);
00295     close(fd);
00296     if(res != zi->indexsize) {
00297         av_free(cstate);
00298         av_log(AVLOG_ERROR, "ZFILE: Error in indexfile %s", zc->indexfile);
00299         return -EIO;
00300     }
00301 
00302     res = zfile_uncompress_state(cstate, zi->indexsize, &state);
00303     av_free(cstate);
00304     if(res < 0)
00305         return res;
00306 
00307     res = inflateRestore(&fil->s, state);
00308     av_free(state);
00309     if(res != Z_OK) {
00310         av_log(AVLOG_ERROR, "ZFILE: inflateRestore: (%i)", res);
00311         return -EIO;
00312     }
00313     fil->iseof = 0;
00314     fil->calccrc = 0;
00315 
00316     return 0;
00317 }
00318 
00319 static struct zindex *zcache_find_index(struct zcache *zc, avoff_t offset)
00320 {
00321     struct zindex *prevzi;
00322     struct zindex *zi;
00323     
00324     prevzi = NULL;
00325     for(zi = zc->indexes; zi != NULL; zi = zi->next) {
00326         if(zi->offset > offset)
00327             break;
00328         prevzi = zi;
00329     }
00330 
00331     return prevzi;
00332 }
00333 #endif
00334 
00335 static int zfile_fill_inbuf(struct zfile *fil)
00336 {
00337     avssize_t res;
00338 
00339     res = av_pread(fil->infile, fil->inbuf, INBUFSIZE,
00340                    fil->s.total_in + fil->dataoff);
00341     if(res < 0)
00342         return res;
00343     
00344     fil->s.next_in = (Bytef*)( fil->inbuf );
00345     fil->s.avail_in = res;
00346 
00347     return 0;
00348 }
00349 
00350 static int zfile_inflate(struct zfile *fil, struct zcache *zc)
00351 {
00352     int res;
00353     unsigned char *start;
00354 
00355     if(fil->s.avail_in == 0) {
00356         res = zfile_fill_inbuf(fil);
00357         if(res < 0)
00358             return res;
00359     }
00360 
00361     start = fil->s.next_out;
00362     res = inflate(&fil->s, Z_NO_FLUSH);
00363     if(fil->calccrc) {
00364         AV_LOCK(zread_lock);
00365         if(zc->crc_ok)
00366             fil->calccrc = 0;
00367         AV_UNLOCK(zread_lock);
00368 
00369         if(fil->calccrc)
00370             fil->s.adler = crc32(fil->s.adler, start, fil->s.next_out - start);
00371     }
00372     if(res == Z_STREAM_END) {
00373         fil->iseof = 1;
00374         if(fil->calccrc && fil->s.adler != fil->crc) {
00375             av_log(AVLOG_ERROR, "ZFILE: CRC error");
00376             return -EIO;
00377         }
00378         AV_LOCK(zread_lock);
00379         if(fil->calccrc)
00380             zc->crc_ok = 1;
00381         zc->size = fil->s.total_out;
00382         AV_UNLOCK(zread_lock);
00383         return 0;
00384     }
00385     if(res != Z_OK) {
00386         av_log(AVLOG_ERROR, "ZFILE: inflate: %s (%i)",
00387                fil->s.msg == NULL ? "" : fil->s.msg, res);
00388         return -EIO;
00389     }
00390     
00391     AV_LOCK(zread_lock);
00392 #ifdef USE_SYSTEM_ZLIB
00393     res = 0;
00394 #else
00395     if(fil->s.total_out >= zc->nextindex)
00396         res = zfile_save_index(fil, zc);
00397     else
00398         res = 0;
00399 #endif
00400     AV_UNLOCK(zread_lock);
00401     if(res < 0)
00402         return res;
00403 
00404     return 0;
00405 }
00406 
00407 
00408 static int zfile_read(struct zfile *fil, struct zcache *zc, char *buf,
00409                       avsize_t nbyte)
00410 {
00411     int res;
00412 
00413     fil->s.next_out = (Bytef*)buf;
00414     fil->s.avail_out = nbyte;
00415     while(fil->s.avail_out != 0 && !fil->iseof) {
00416         res = zfile_inflate(fil, zc);
00417         if(res < 0)
00418             return res;
00419     }
00420 
00421     return nbyte - fil->s.avail_out;
00422 }
00423 
00424 static int zfile_skip_to(struct zfile *fil, struct zcache *zc, avoff_t offset)
00425 {
00426     int res;
00427     char outbuf[OUTBUFSIZE];
00428     
00429     while(fil->s.total_out < offset && !fil->iseof) {
00430         /* FIXME: Maybe cache some data as well */
00431         fil->s.next_out = (Bytef*)outbuf;
00432         fil->s.avail_out = AV_MIN(OUTBUFSIZE, offset - fil->s.total_out);
00433 
00434         res = zfile_inflate(fil, zc);
00435         if(res < 0)
00436             return res;
00437     }
00438 
00439     return 0;
00440 }
00441 
00442 #ifndef USE_SYSTEM_ZLIB
00443 static int zfile_seek(struct zfile *fil, struct zcache *zc, avoff_t offset)
00444 {
00445     struct zindex *zi;
00446     avoff_t curroff = fil->s.total_out;
00447     avoff_t zcdist;
00448     avoff_t scdist;
00449     avoff_t dist;
00450 
00451     if(offset >= curroff)
00452         dist = offset - curroff;
00453     else
00454         dist = -1;
00455 
00456     zi = zcache_find_index(zc, offset);
00457     if(zi != NULL)
00458         zcdist = offset - zi->offset + ZCACHE_EXTRA_DIST;
00459     else
00460         zcdist = offset;
00461 
00462     if(scache.id == zc->id && offset >= scache.s.total_out) {
00463         scdist = offset - scache.s.total_out;
00464         if((dist == -1 || scdist < dist) && scdist < zcdist) {
00465             z_stream tmp = fil->s;
00466             int tmpcc = fil->calccrc;
00467             int tmpiseof = fil->iseof;
00468             fil->s = scache.s;
00469             fil->s.avail_in = 0;
00470             fil->calccrc = scache.calccrc;
00471             fil->iseof = scache.iseof;
00472             scache.s = tmp;
00473             scache.calccrc = tmpcc;
00474             scache.iseof = tmpiseof;
00475             return 0;
00476         }
00477     }
00478 
00479     if(dist == -1 || zcdist < dist) {
00480         if(zi == NULL)
00481             return zfile_reset(fil);
00482         else
00483             return zfile_seek_index(fil, zc, zi);
00484     }
00485     
00486     return 0;
00487 }
00488 #endif
00489 
00490 static int zfile_goto(struct zfile *fil, struct zcache *zc, avoff_t offset)
00491 {
00492     int res;
00493 
00494     AV_LOCK(zc->lock);
00495     AV_LOCK(zread_lock);
00496 #ifdef USE_SYSTEM_ZLIB
00497     if ( offset < fil->s.total_out ) {
00498         res = zfile_reset(fil);
00499     } else res = 0;
00500 #else
00501     res = zfile_seek(fil, zc, offset);
00502 #endif
00503     AV_UNLOCK(zread_lock);
00504     if(res == 0)
00505         res = zfile_skip_to(fil, zc, offset);
00506     AV_UNLOCK(zc->lock);
00507 
00508     return res;
00509 }
00510 
00511 static avssize_t av_zfile_do_pread(struct zfile *fil, struct zcache *zc,
00512                                    char *buf, avsize_t nbyte, avoff_t offset)
00513 {
00514     avssize_t res;
00515 
00516     fil->id = zc->id;
00517 
00518     if(offset != fil->s.total_out) {
00519         res = zfile_goto(fil, zc, offset);
00520         if(res < 0)
00521             return res;
00522     }
00523 
00524     res = zfile_read(fil, zc, buf, nbyte);
00525     
00526     return res;
00527 }
00528 
00529 avssize_t av_zfile_pread(struct zfile *fil, struct zcache *zc, char *buf,
00530                          avsize_t nbyte, avoff_t offset)
00531 {
00532     avssize_t res;
00533 
00534     if(fil->iserror)
00535         return -EIO;
00536 
00537     res = av_zfile_do_pread(fil, zc, buf, nbyte, offset);
00538     if(res < 0)
00539         fil->iserror = 1;
00540 
00541     return res;
00542 }
00543 
00544 int av_zfile_size(struct zfile *fil, struct zcache *zc, avoff_t *sizep)
00545 {
00546     int res;
00547     avoff_t size;
00548 
00549     AV_LOCK(zread_lock);
00550     size = zc->size;
00551     AV_UNLOCK(zread_lock);
00552 
00553     if(size != -1 || fil == NULL) {
00554         *sizep = size;
00555         return 0;
00556     }
00557 
00558     fil->id = zc->id;
00559 
00560     res  = zfile_goto(fil, zc, AV_MAXOFF);
00561     if(res < 0)
00562         return res;
00563     
00564     AV_LOCK(zread_lock);
00565     size = zc->size;
00566     AV_UNLOCK(zread_lock);
00567     
00568     if(size == -1) {
00569         av_log(AVLOG_ERROR, "ZFILE: Internal error: could not find size");
00570         return -EIO;
00571     }
00572     
00573     *sizep = size;
00574     return 0;
00575 
00576 }
00577 
00578 static void zfile_destroy(struct zfile *fil)
00579 {
00580     AV_LOCK(zread_lock);
00581     zfile_scache_save(fil->id, &fil->s, fil->calccrc, fil->iseof);
00582     AV_UNLOCK(zread_lock);
00583 }
00584 
00585 struct zfile *av_zfile_new(vfile *vf, avoff_t dataoff, avuint crc, int calccrc)
00586 {
00587     int res;
00588     struct zfile *fil;
00589 
00590     AV_NEW_OBJ(fil, zfile_destroy);
00591     fil->iseof = 0;
00592     fil->iserror = 0;
00593     fil->infile = vf;
00594     fil->dataoff = dataoff;
00595     fil->id = 0;
00596     fil->crc = crc;
00597     fil->calccrc = calccrc;
00598 
00599     memset(&fil->s, 0, sizeof(z_stream));
00600     res = inflateInit2(&fil->s, -MAX_WBITS);
00601     if(res != Z_OK) {
00602         av_log(AVLOG_ERROR, "ZFILE: inflateInit: %s (%i)",
00603                fil->s.msg == NULL ? "" : fil->s.msg, res);
00604         fil->iserror = 1;
00605     }
00606     fil->s.adler = 0;
00607 
00608     return fil;
00609 }
00610 
00611 static void zcache_destroy(struct zcache *zc)
00612 {
00613     struct zindex *zi;
00614     struct zindex *nextzi;
00615 
00616     AV_FREELOCK(zc->lock);
00617     av_del_tmpfile(zc->indexfile);
00618     
00619     for(zi = zc->indexes; zi != NULL; zi = nextzi) {
00620         nextzi = zi->next;
00621         av_free(zi);
00622     }
00623 }
00624 
00625 struct zcache *av_zcache_new()
00626 {
00627     struct zcache *zc;
00628 
00629     AV_NEW_OBJ(zc, zcache_destroy);
00630     zc->indexfile = NULL;
00631     zc->nextindex = INDEXDISTANCE;
00632     zc->indexes = NULL;
00633     zc->filesize = 0;
00634     zc->size = -1;
00635     zc->crc_ok = 0;
00636     AV_INITLOCK(zc->lock);
00637 
00638     AV_LOCK(zread_lock);
00639     if(zread_nextid == 0)
00640         zread_nextid = 1;
00641 
00642     zc->id = zread_nextid ++;
00643     AV_UNLOCK(zread_lock);
00644     
00645     av_get_tmpfile(&zc->indexfile);
00646     
00647     return zc;
00648 }
00649 
00650 avoff_t av_zcache_size(struct zcache *zc)
00651 {
00652     return zc->filesize;
00653 }