Back to index

openldap  2.4.31
tools.c
Go to the documentation of this file.
00001 /* tools.c - tools for slap tools */
00002 /* $OpenLDAP$ */
00003 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
00004  *
00005  * Copyright 2011-2012 The OpenLDAP Foundation.
00006  * All rights reserved.
00007  *
00008  * Redistribution and use in source and binary forms, with or without
00009  * modification, are permitted only as authorized by the OpenLDAP
00010  * Public License.
00011  *
00012  * A copy of this license is available in the file LICENSE in the
00013  * top-level directory of the distribution or, alternatively, at
00014  * <http://www.OpenLDAP.org/license.html>.
00015  */
00016 
00017 #include "portable.h"
00018 
00019 #include <stdio.h>
00020 #include <ac/string.h>
00021 #include <ac/errno.h>
00022 
00023 #define AVL_INTERNAL
00024 #include "back-mdb.h"
00025 #include "idl.h"
00026 
00027 #ifdef MDB_TOOL_IDL_CACHING
00028 static int mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn );
00029 
00030 #define       IDBLOCK       1024
00031 
00032 typedef struct mdb_tool_idl_cache_entry {
00033        struct mdb_tool_idl_cache_entry *next;
00034        ID ids[IDBLOCK];
00035 } mdb_tool_idl_cache_entry;
00036 
00037 typedef struct mdb_tool_idl_cache {
00038        struct berval kstr;
00039        mdb_tool_idl_cache_entry *head, *tail;
00040        ID first, last;
00041        int count;
00042        short offset;
00043        short flags;
00044 } mdb_tool_idl_cache;
00045 #define WAS_FOUND    0x01
00046 #define WAS_RANGE    0x02
00047 
00048 #define MDB_TOOL_IDL_FLUSH(be, txn)       mdb_tool_idl_flush(be, txn)
00049 #else
00050 #define MDB_TOOL_IDL_FLUSH(be, txn)
00051 #endif /* MDB_TOOL_IDL_CACHING */
00052 
00053 static MDB_txn *txn = NULL, *txi = NULL;
00054 static MDB_cursor *cursor = NULL, *idcursor = NULL;
00055 static MDB_cursor *mcp = NULL, *mcd = NULL;
00056 static MDB_val key, data;
00057 static ID previd = NOID;
00058 
00059 typedef struct dn_id {
00060        ID id;
00061        struct berval dn;
00062 } dn_id;
00063 
00064 #define       HOLE_SIZE     4096
00065 static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
00066 static unsigned nhmax = HOLE_SIZE;
00067 static unsigned nholes;
00068 
00069 static struct berval *tool_base;
00070 static int           tool_scope;
00071 static Filter        *tool_filter;
00072 static Entry         *tool_next_entry;
00073 
00074 static ID mdb_tool_ix_id;
00075 static Operation *mdb_tool_ix_op;
00076 static MDB_txn *mdb_tool_ix_txn;
00077 static int mdb_tool_index_tcount, mdb_tool_threads;
00078 static IndexRec *mdb_tool_index_rec;
00079 static struct mdb_info *mdb_tool_info;
00080 static ldap_pvt_thread_mutex_t mdb_tool_index_mutex;
00081 static ldap_pvt_thread_cond_t mdb_tool_index_cond_main;
00082 static ldap_pvt_thread_cond_t mdb_tool_index_cond_work;
00083 static void * mdb_tool_index_task( void *ctx, void *ptr );
00084 
00085 static int    mdb_writes, mdb_writes_per_commit;
00086 
00087 static int
00088 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep );
00089 
00090 int mdb_tool_entry_open(
00091        BackendDB *be, int mode )
00092 {
00093        /* In Quick mode, commit once per 1000 entries */
00094        mdb_writes = 0;
00095        if ( slapMode & SLAP_TOOL_QUICK )
00096               mdb_writes_per_commit = 1000;
00097        else
00098               mdb_writes_per_commit = 1;
00099 
00100        /* Set up for threaded slapindex */
00101        if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) {
00102               if ( !mdb_tool_info ) {
00103                      struct mdb_info *mdb = (struct mdb_info *) be->be_private;
00104                      ldap_pvt_thread_mutex_init( &mdb_tool_index_mutex );
00105                      ldap_pvt_thread_cond_init( &mdb_tool_index_cond_main );
00106                      ldap_pvt_thread_cond_init( &mdb_tool_index_cond_work );
00107                      if ( mdb->mi_nattrs ) {
00108                             int i;
00109                             mdb_tool_threads = slap_tool_thread_max - 1;
00110                             if ( mdb_tool_threads > 1 ) {
00111                                    mdb_tool_index_rec = ch_calloc( mdb->mi_nattrs, sizeof( IndexRec ));
00112                                    mdb_tool_index_tcount = mdb_tool_threads - 1;
00113                                    for (i=1; i<mdb_tool_threads; i++) {
00114                                           int *ptr = ch_malloc( sizeof( int ));
00115                                           *ptr = i;
00116                                           ldap_pvt_thread_pool_submit( &connection_pool,
00117                                                  mdb_tool_index_task, ptr );
00118                                    }
00119                                    mdb_tool_info = mdb;
00120                             }
00121                      }
00122               }
00123        }
00124 
00125        return 0;
00126 }
00127 
00128 int mdb_tool_entry_close(
00129        BackendDB *be )
00130 {
00131        if ( mdb_tool_info ) {
00132               slapd_shutdown = 1;
00133               ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
00134 
00135               /* There might still be some threads starting */
00136               while ( mdb_tool_index_tcount > 0 ) {
00137                      ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
00138                                    &mdb_tool_index_mutex );
00139               }
00140 
00141               mdb_tool_index_tcount = mdb_tool_threads - 1;
00142               ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
00143 
00144               /* Make sure all threads are stopped */
00145               while ( mdb_tool_index_tcount > 0 ) {
00146                      ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
00147                             &mdb_tool_index_mutex );
00148               }
00149               ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
00150 
00151               mdb_tool_info = NULL;
00152               slapd_shutdown = 0;
00153               ch_free( mdb_tool_index_rec );
00154               mdb_tool_index_tcount = mdb_tool_threads - 1;
00155        }
00156 
00157        if( idcursor ) {
00158               mdb_cursor_close( idcursor );
00159               idcursor = NULL;
00160        }
00161        if( cursor ) {
00162               mdb_cursor_close( cursor );
00163               cursor = NULL;
00164        }
00165        if( txn ) {
00166               MDB_TOOL_IDL_FLUSH( be, txn );
00167               if ( mdb_txn_commit( txn ))
00168                      return -1;
00169               txn = NULL;
00170        }
00171 
00172        if( nholes ) {
00173               unsigned i;
00174               fprintf( stderr, "Error, entries missing!\n");
00175               for (i=0; i<nholes; i++) {
00176                      fprintf(stderr, "  entry %ld: %s\n",
00177                             holes[i].id, holes[i].dn.bv_val);
00178               }
00179               nholes = 0;
00180               return -1;
00181        }
00182 
00183        return 0;
00184 }
00185 
00186 ID
00187 mdb_tool_entry_first_x(
00188        BackendDB *be,
00189        struct berval *base,
00190        int scope,
00191        Filter *f )
00192 {
00193        tool_base = base;
00194        tool_scope = scope;
00195        tool_filter = f;
00196 
00197        return mdb_tool_entry_next( be );
00198 }
00199 
00200 ID mdb_tool_entry_next(
00201        BackendDB *be )
00202 {
00203        int rc;
00204        ID id;
00205        struct mdb_info *mdb;
00206 
00207        assert( be != NULL );
00208        assert( slapMode & SLAP_TOOL_MODE );
00209 
00210        mdb = (struct mdb_info *) be->be_private;
00211        assert( mdb != NULL );
00212 
00213        if ( !txn ) {
00214               rc = mdb_txn_begin( mdb->mi_dbenv, NULL, MDB_RDONLY, &txn );
00215               if ( rc )
00216                      return NOID;
00217               rc = mdb_cursor_open( txn, mdb->mi_id2entry, &cursor );
00218               if ( rc ) {
00219                      mdb_txn_abort( txn );
00220                      return NOID;
00221               }
00222        }
00223 
00224 next:;
00225        rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT );
00226 
00227        if( rc ) {
00228               return NOID;
00229        }
00230 
00231        previd = *(ID *)key.mv_data;
00232        id = previd;
00233 
00234        if ( tool_filter || tool_base ) {
00235               static Operation op = {0};
00236               static Opheader ohdr = {0};
00237 
00238               op.o_hdr = &ohdr;
00239               op.o_bd = be;
00240               op.o_tmpmemctx = NULL;
00241               op.o_tmpmfuncs = &ch_mfuncs;
00242 
00243               if ( tool_next_entry ) {
00244                      mdb_entry_release( &op, tool_next_entry, 0 );
00245                      tool_next_entry = NULL;
00246               }
00247 
00248               rc = mdb_tool_entry_get_int( be, id, &tool_next_entry );
00249               if ( rc == LDAP_NO_SUCH_OBJECT ) {
00250                      goto next;
00251               }
00252 
00253               assert( tool_next_entry != NULL );
00254 
00255               if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE )
00256               {
00257                      mdb_entry_release( &op, tool_next_entry, 0 );
00258                      tool_next_entry = NULL;
00259                      goto next;
00260               }
00261        }
00262 
00263        return id;
00264 }
00265 
00266 ID mdb_tool_dn2id_get(
00267        Backend *be,
00268        struct berval *dn
00269 )
00270 {
00271        struct mdb_info *mdb;
00272        Operation op = {0};
00273        Opheader ohdr = {0};
00274        ID id;
00275        int rc;
00276 
00277        if ( BER_BVISEMPTY(dn) )
00278               return 0;
00279 
00280        mdb = (struct mdb_info *) be->be_private;
00281 
00282        if ( !txn ) {
00283               rc = mdb_txn_begin( mdb->mi_dbenv, NULL, (slapMode & SLAP_TOOL_READONLY) != 0 ?
00284                      MDB_RDONLY : 0, &txn );
00285               if ( rc )
00286                      return NOID;
00287        }
00288 
00289        op.o_hdr = &ohdr;
00290        op.o_bd = be;
00291        op.o_tmpmemctx = NULL;
00292        op.o_tmpmfuncs = &ch_mfuncs;
00293 
00294        rc = mdb_dn2id( &op, txn, NULL, dn, &id, NULL, NULL );
00295        if ( rc == MDB_NOTFOUND )
00296               return NOID;
00297 
00298        return id;
00299 }
00300 
00301 static int
00302 mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep )
00303 {
00304        Operation op = {0};
00305        Opheader ohdr = {0};
00306 
00307        Entry *e = NULL;
00308        struct berval dn = BER_BVNULL, ndn = BER_BVNULL;
00309        int rc;
00310 
00311        assert( be != NULL );
00312        assert( slapMode & SLAP_TOOL_MODE );
00313 
00314        if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) {
00315               *ep = tool_next_entry;
00316               tool_next_entry = NULL;
00317               return LDAP_SUCCESS;
00318        }
00319 
00320        if ( id != previd ) {
00321               key.mv_size = sizeof(ID);
00322               key.mv_data = &id;
00323               rc = mdb_cursor_get( cursor, &key, &data, MDB_SET );
00324               if ( rc ) {
00325                      rc = LDAP_OTHER;
00326                      goto done;
00327               }
00328        }
00329 
00330        op.o_hdr = &ohdr;
00331        op.o_bd = be;
00332        op.o_tmpmemctx = NULL;
00333        op.o_tmpmfuncs = &ch_mfuncs;
00334        if ( slapMode & SLAP_TOOL_READONLY ) {
00335               rc = mdb_id2name( &op, txn, &idcursor, id, &dn, &ndn );
00336               if ( rc  ) {
00337                      rc = LDAP_OTHER;
00338                      if ( e ) {
00339                             mdb_entry_return( &op, e );
00340                             e = NULL;
00341                      }
00342                      goto done;
00343               }
00344               if ( tool_base != NULL ) {
00345                      if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) {
00346                             ch_free( dn.bv_val );
00347                             ch_free( ndn.bv_val );
00348                             rc = LDAP_NO_SUCH_OBJECT;
00349                      }
00350               }
00351        }
00352        rc = mdb_entry_decode( &op, &data, &e );
00353        e->e_id = id;
00354        if ( !BER_BVISNULL( &dn )) {
00355               e->e_name = dn;
00356               e->e_nname = ndn;
00357        } else {
00358               e->e_name.bv_val = NULL;
00359               e->e_nname.bv_val = NULL;
00360        }
00361 
00362 done:
00363        if ( e != NULL ) {
00364               *ep = e;
00365        }
00366 
00367        return rc;
00368 }
00369 
00370 Entry*
00371 mdb_tool_entry_get( BackendDB *be, ID id )
00372 {
00373        Entry *e = NULL;
00374        int rc;
00375 
00376        if ( !txn ) {
00377               struct mdb_info *mdb = (struct mdb_info *) be->be_private;
00378               rc = mdb_txn_begin( mdb->mi_dbenv, NULL,
00379                      (slapMode & SLAP_TOOL_READONLY) ? MDB_RDONLY : 0, &txn );
00380               if ( rc )
00381                      return NULL;
00382        }
00383        if ( !cursor ) {
00384               struct mdb_info *mdb = (struct mdb_info *) be->be_private;
00385               rc = mdb_cursor_open( txn, mdb->mi_id2entry, &cursor );
00386               if ( rc ) {
00387                      mdb_txn_abort( txn );
00388                      txn = NULL;
00389                      return NULL;
00390               }
00391        }
00392        (void)mdb_tool_entry_get_int( be, id, &e );
00393        return e;
00394 }
00395 
00396 static int mdb_tool_next_id(
00397        Operation *op,
00398        MDB_txn *tid,
00399        Entry *e,
00400        struct berval *text,
00401        int hole )
00402 {
00403        struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
00404        struct berval dn = e->e_name;
00405        struct berval ndn = e->e_nname;
00406        struct berval pdn, npdn, nmatched;
00407        ID id, pid = 0;
00408        int rc;
00409 
00410        if (ndn.bv_len == 0) {
00411               e->e_id = 0;
00412               return 0;
00413        }
00414 
00415        rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, &nmatched );
00416        if ( rc == MDB_NOTFOUND ) {
00417               if ( !be_issuffix( op->o_bd, &ndn ) ) {
00418                      ID eid = e->e_id;
00419                      dnParent( &ndn, &npdn );
00420                      if ( nmatched.bv_len != npdn.bv_len ) {
00421                             dnParent( &dn, &pdn );
00422                             e->e_name = pdn;
00423                             e->e_nname = npdn;
00424                             rc = mdb_tool_next_id( op, tid, e, text, 1 );
00425                             e->e_name = dn;
00426                             e->e_nname = ndn;
00427                             if ( rc ) {
00428                                    return rc;
00429                             }
00430                             /* If parent didn't exist, it was created just now
00431                              * and its ID is now in e->e_id. Make sure the current
00432                              * entry gets added under the new parent ID.
00433                              */
00434                             if ( eid != e->e_id ) {
00435                                    pid = e->e_id;
00436                             }
00437                      } else {
00438                             pid = id;
00439                      }
00440               }
00441               rc = mdb_next_id( op->o_bd, idcursor, &e->e_id );
00442               if ( rc ) {
00443                      snprintf( text->bv_val, text->bv_len,
00444                             "next_id failed: %s (%d)",
00445                             mdb_strerror(rc), rc );
00446               Debug( LDAP_DEBUG_ANY,
00447                      "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
00448                      return rc;
00449               }
00450               rc = mdb_dn2id_add( op, mcp, mcd, pid, e );
00451               if ( rc ) {
00452                      snprintf( text->bv_val, text->bv_len,
00453                             "dn2id_add failed: %s (%d)",
00454                             mdb_strerror(rc), rc );
00455                      Debug( LDAP_DEBUG_ANY,
00456                             "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
00457               } else if ( hole ) {
00458                      MDB_val key, data;
00459                      if ( nholes == nhmax - 1 ) {
00460                             if ( holes == hbuf ) {
00461                                    holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
00462                                    AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
00463                             } else {
00464                                    holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
00465                             }
00466                             nhmax *= 2;
00467                      }
00468                      ber_dupbv( &holes[nholes].dn, &ndn );
00469                      holes[nholes++].id = e->e_id;
00470                      key.mv_size = sizeof(ID);
00471                      key.mv_data = &e->e_id;
00472                      data.mv_size = 0;
00473                      data.mv_data = NULL;
00474                      rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE );
00475                      if ( rc == MDB_KEYEXIST )
00476                             rc = 0;
00477                      if ( rc ) {
00478                             snprintf( text->bv_val, text->bv_len,
00479                                    "dummy id2entry add failed: %s (%d)",
00480                                    mdb_strerror(rc), rc );
00481                             Debug( LDAP_DEBUG_ANY,
00482                                    "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
00483                      }
00484               }
00485        } else if ( !hole ) {
00486               unsigned i, j;
00487 
00488               e->e_id = id;
00489 
00490               for ( i=0; i<nholes; i++) {
00491                      if ( holes[i].id == e->e_id ) {
00492                             free(holes[i].dn.bv_val);
00493                             for (j=i;j<nholes;j++) holes[j] = holes[j+1];
00494                             holes[j].id = 0;
00495                             nholes--;
00496                             break;
00497                      } else if ( holes[i].id > e->e_id ) {
00498                             break;
00499                      }
00500               }
00501        }
00502        return rc;
00503 }
00504 
00505 static int
00506 mdb_tool_index_add(
00507        Operation *op,
00508        MDB_txn *txn,
00509        Entry *e )
00510 {
00511        struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
00512 
00513        if ( !mdb->mi_nattrs )
00514               return 0;
00515 
00516        if ( mdb_tool_threads > 1 ) {
00517               IndexRec *ir;
00518               int i, rc;
00519               Attribute *a;
00520 
00521               ir = mdb_tool_index_rec;
00522               for (i=0; i<mdb->mi_nattrs; i++)
00523                      ir[i].ir_attrs = NULL;
00524 
00525               for ( a = e->e_attrs; a != NULL; a = a->a_next ) {
00526                      rc = mdb_index_recset( mdb, a, a->a_desc->ad_type,
00527                             &a->a_desc->ad_tags, ir );
00528                      if ( rc )
00529                             return rc;
00530               }
00531               for (i=0; i<mdb->mi_nattrs; i++) {
00532                      if ( !ir[i].ir_ai )
00533                             break;
00534                      rc = mdb_cursor_open( txn, ir[i].ir_ai->ai_dbi,
00535                              &ir[i].ir_ai->ai_cursor );
00536                      if ( rc )
00537                             return rc;
00538               }
00539               mdb_tool_ix_id = e->e_id;
00540               mdb_tool_ix_op = op;
00541               mdb_tool_ix_txn = txn;
00542               ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
00543               /* Wait for all threads to be ready */
00544               while ( mdb_tool_index_tcount ) {
00545                      ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
00546                             &mdb_tool_index_mutex );
00547               }
00548 
00549               for ( i=1; i<mdb_tool_threads; i++ )
00550                      mdb_tool_index_rec[i].ir_i = LDAP_BUSY;
00551               mdb_tool_index_tcount = mdb_tool_threads - 1;
00552               ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
00553               ldap_pvt_thread_cond_broadcast( &mdb_tool_index_cond_work );
00554 
00555               rc = mdb_index_recrun( op, txn, mdb, ir, e->e_id, 0 );
00556               if ( rc )
00557                      return rc;
00558               ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
00559               for ( i=1; i<mdb_tool_threads; i++ ) {
00560                      if ( mdb_tool_index_rec[i].ir_i == LDAP_BUSY ) {
00561                             ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_main,
00562                                    &mdb_tool_index_mutex );
00563                             i--;
00564                             continue;
00565                      }
00566                      if ( mdb_tool_index_rec[i].ir_i ) {
00567                             rc = mdb_tool_index_rec[i].ir_i;
00568                             break;
00569                      }
00570               }
00571               ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
00572               return rc;
00573        } else
00574        {
00575               return mdb_index_entry_add( op, txn, e );
00576        }
00577 }
00578 
00579 ID mdb_tool_entry_put(
00580        BackendDB *be,
00581        Entry *e,
00582        struct berval *text )
00583 {
00584        int rc;
00585        struct mdb_info *mdb;
00586        Operation op = {0};
00587        Opheader ohdr = {0};
00588 
00589        assert( be != NULL );
00590        assert( slapMode & SLAP_TOOL_MODE );
00591 
00592        assert( text != NULL );
00593        assert( text->bv_val != NULL );
00594        assert( text->bv_val[0] == '\0' ); /* overconservative? */
00595 
00596        Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_tool_entry_put)
00597               "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 );
00598 
00599        mdb = (struct mdb_info *) be->be_private;
00600 
00601        if ( !txn ) {
00602               rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn );
00603               if( rc != 0 ) {
00604                      snprintf( text->bv_val, text->bv_len,
00605                             "txn_begin failed: %s (%d)",
00606                             mdb_strerror(rc), rc );
00607                      Debug( LDAP_DEBUG_ANY,
00608                             "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
00609                              text->bv_val, 0, 0 );
00610                      return NOID;
00611               }
00612               rc = mdb_cursor_open( txn, mdb->mi_id2entry, &idcursor );
00613               if( rc != 0 ) {
00614                      snprintf( text->bv_val, text->bv_len,
00615                             "cursor_open failed: %s (%d)",
00616                             mdb_strerror(rc), rc );
00617                      Debug( LDAP_DEBUG_ANY,
00618                             "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
00619                              text->bv_val, 0, 0 );
00620                      return NOID;
00621               }
00622               rc = mdb_cursor_open( txn, mdb->mi_dn2id, &mcp );
00623               if( rc != 0 ) {
00624                      snprintf( text->bv_val, text->bv_len,
00625                             "cursor_open failed: %s (%d)",
00626                             mdb_strerror(rc), rc );
00627                      Debug( LDAP_DEBUG_ANY,
00628                             "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
00629                              text->bv_val, 0, 0 );
00630                      return NOID;
00631               }
00632               rc = mdb_cursor_open( txn, mdb->mi_dn2id, &mcd );
00633               if( rc != 0 ) {
00634                      snprintf( text->bv_val, text->bv_len,
00635                             "cursor_open failed: %s (%d)",
00636                             mdb_strerror(rc), rc );
00637                      Debug( LDAP_DEBUG_ANY,
00638                             "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
00639                              text->bv_val, 0, 0 );
00640                      return NOID;
00641               }
00642        }
00643 
00644        op.o_hdr = &ohdr;
00645        op.o_bd = be;
00646        op.o_tmpmemctx = NULL;
00647        op.o_tmpmfuncs = &ch_mfuncs;
00648 
00649        /* add dn2id indices */
00650        rc = mdb_tool_next_id( &op, txn, e, text, 0 );
00651        if( rc != 0 ) {
00652               goto done;
00653        }
00654 
00655        rc = mdb_tool_index_add( &op, txn, e );
00656        if( rc != 0 ) {
00657               snprintf( text->bv_val, text->bv_len,
00658                             "index_entry_add failed: err=%d", rc );
00659               Debug( LDAP_DEBUG_ANY,
00660                      "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
00661                      text->bv_val, 0, 0 );
00662               goto done;
00663        }
00664 
00665 
00666        /* id2entry index */
00667        rc = mdb_id2entry_add( &op, txn, idcursor, e );
00668        if( rc != 0 ) {
00669               snprintf( text->bv_val, text->bv_len,
00670                             "id2entry_add failed: err=%d", rc );
00671               Debug( LDAP_DEBUG_ANY,
00672                      "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
00673                      text->bv_val, 0, 0 );
00674               goto done;
00675        }
00676 
00677 done:
00678        if( rc == 0 ) {
00679               mdb_writes++;
00680               if ( mdb_writes >= mdb_writes_per_commit ) {
00681                      unsigned i;
00682                      MDB_TOOL_IDL_FLUSH( be, txn );
00683                      rc = mdb_txn_commit( txn );
00684                      for ( i=0; i<mdb->mi_nattrs; i++ )
00685                             mdb->mi_attrs[i]->ai_cursor = NULL;
00686                      mdb_writes = 0;
00687                      txn = NULL;
00688                      idcursor = NULL;
00689                      if( rc != 0 ) {
00690                             snprintf( text->bv_val, text->bv_len,
00691                                           "txn_commit failed: %s (%d)",
00692                                           mdb_strerror(rc), rc );
00693                             Debug( LDAP_DEBUG_ANY,
00694                                    "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
00695                                    text->bv_val, 0, 0 );
00696                             e->e_id = NOID;
00697                      }
00698               }
00699 
00700        } else {
00701               unsigned i;
00702               mdb_txn_abort( txn );
00703               txn = NULL;
00704               idcursor = NULL;
00705               for ( i=0; i<mdb->mi_nattrs; i++ )
00706                      mdb->mi_attrs[i]->ai_cursor = NULL;
00707               mdb_writes = 0;
00708               snprintf( text->bv_val, text->bv_len,
00709                      "txn_aborted! %s (%d)",
00710                      rc == LDAP_OTHER ? "Internal error" :
00711                      mdb_strerror(rc), rc );
00712               Debug( LDAP_DEBUG_ANY,
00713                      "=> " LDAP_XSTRING(mdb_tool_entry_put) ": %s\n",
00714                      text->bv_val, 0, 0 );
00715               e->e_id = NOID;
00716        }
00717 
00718        return e->e_id;
00719 }
00720 
00721 int mdb_tool_entry_reindex(
00722        BackendDB *be,
00723        ID id,
00724        AttributeDescription **adv )
00725 {
00726        struct mdb_info *mi = (struct mdb_info *) be->be_private;
00727        int rc;
00728        Entry *e;
00729        Operation op = {0};
00730        Opheader ohdr = {0};
00731 
00732        Debug( LDAP_DEBUG_ARGS,
00733               "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
00734               (long) id, 0, 0 );
00735        assert( tool_base == NULL );
00736        assert( tool_filter == NULL );
00737 
00738        /* No indexes configured, nothing to do. Could return an
00739         * error here to shortcut things.
00740         */
00741        if (!mi->mi_attrs) {
00742               return 0;
00743        }
00744 
00745        /* Check for explicit list of attrs to index */
00746        if ( adv ) {
00747               int i, j, n;
00748 
00749               if ( mi->mi_attrs[0]->ai_desc != adv[0] ) {
00750                      /* count */
00751                      for ( n = 0; adv[n]; n++ ) ;
00752 
00753                      /* insertion sort */
00754                      for ( i = 0; i < n; i++ ) {
00755                             AttributeDescription *ad = adv[i];
00756                             for ( j = i-1; j>=0; j--) {
00757                                    if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break;
00758                                    adv[j+1] = adv[j];
00759                             }
00760                             adv[j+1] = ad;
00761                      }
00762               }
00763 
00764               for ( i = 0; adv[i]; i++ ) {
00765                      if ( mi->mi_attrs[i]->ai_desc != adv[i] ) {
00766                             for ( j = i+1; j < mi->mi_nattrs; j++ ) {
00767                                    if ( mi->mi_attrs[j]->ai_desc == adv[i] ) {
00768                                           AttrInfo *ai = mi->mi_attrs[i];
00769                                           mi->mi_attrs[i] = mi->mi_attrs[j];
00770                                           mi->mi_attrs[j] = ai;
00771                                           break;
00772                                    }
00773                             }
00774                             if ( j == mi->mi_nattrs ) {
00775                                    Debug( LDAP_DEBUG_ANY,
00776                                           LDAP_XSTRING(mdb_tool_entry_reindex)
00777                                           ": no index configured for %s\n",
00778                                           adv[i]->ad_cname.bv_val, 0, 0 );
00779                                    return -1;
00780                             }
00781                      }
00782               }
00783               mi->mi_nattrs = i;
00784        }
00785 
00786        e = mdb_tool_entry_get( be, id );
00787 
00788        if( e == NULL ) {
00789               Debug( LDAP_DEBUG_ANY,
00790                      LDAP_XSTRING(mdb_tool_entry_reindex)
00791                      ": could not locate id=%ld\n",
00792                      (long) id, 0, 0 );
00793               return -1;
00794        }
00795 
00796        if ( !txi ) {
00797               rc = mdb_txn_begin( mi->mi_dbenv, NULL, 0, &txi );
00798               if( rc != 0 ) {
00799                      Debug( LDAP_DEBUG_ANY,
00800                             "=> " LDAP_XSTRING(mdb_tool_entry_reindex) ": "
00801                             "txn_begin failed: %s (%d)\n",
00802                             mdb_strerror(rc), rc, 0 );
00803                      goto done;
00804               }
00805        }
00806 
00807        if ( slapMode & SLAP_TRUNCATE_MODE ) {
00808               int i;
00809               for ( i=0; i < mi->mi_nattrs; i++ ) {
00810                      rc = mdb_drop( txi, mi->mi_attrs[i]->ai_dbi, 0 );
00811                      if ( rc ) {
00812                             Debug( LDAP_DEBUG_ANY,
00813                                    LDAP_XSTRING(mdb_tool_entry_reindex)
00814                                    ": (Truncate) mdb_drop(%s) failed: %s (%d)\n",
00815                                    mi->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val,
00816                                    mdb_strerror(rc), rc );
00817                             return -1;
00818                      }
00819               }
00820               slapMode ^= SLAP_TRUNCATE_MODE;
00821        }
00822 
00823        /*
00824         * just (re)add them for now
00825         * Use truncate mode to empty/reset index databases
00826         */
00827 
00828        Debug( LDAP_DEBUG_TRACE,
00829               "=> " LDAP_XSTRING(mdb_tool_entry_reindex) "( %ld )\n",
00830               (long) id, 0, 0 );
00831 
00832        op.o_hdr = &ohdr;
00833        op.o_bd = be;
00834        op.o_tmpmemctx = NULL;
00835        op.o_tmpmfuncs = &ch_mfuncs;
00836 
00837        rc = mdb_tool_index_add( &op, txi, e );
00838 
00839 done:
00840        if( rc == 0 ) {
00841               mdb_writes++;
00842               if ( mdb_writes >= mdb_writes_per_commit ) {
00843                      unsigned i;
00844                      MDB_TOOL_IDL_FLUSH( be, txi );
00845                      rc = mdb_txn_commit( txi );
00846                      mdb_writes = 0;
00847                      for ( i=0; i<mi->mi_nattrs; i++ )
00848                             mi->mi_attrs[i]->ai_cursor = NULL;
00849                      if( rc != 0 ) {
00850                             Debug( LDAP_DEBUG_ANY,
00851                                    "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
00852                                    ": txn_commit failed: %s (%d)\n",
00853                                    mdb_strerror(rc), rc, 0 );
00854                             e->e_id = NOID;
00855                      }
00856                      txi = NULL;
00857               }
00858 
00859        } else {
00860               unsigned i;
00861               mdb_writes = 0;
00862               mdb_txn_abort( txi );
00863               for ( i=0; i<mi->mi_nattrs; i++ )
00864                      mi->mi_attrs[i]->ai_cursor = NULL;
00865               Debug( LDAP_DEBUG_ANY,
00866                      "=> " LDAP_XSTRING(mdb_tool_entry_reindex)
00867                      ": txn_aborted! err=%d\n",
00868                      rc, 0, 0 );
00869               e->e_id = NOID;
00870               txi = NULL;
00871        }
00872        mdb_entry_release( &op, e, 0 );
00873 
00874        return rc;
00875 }
00876 
00877 ID mdb_tool_entry_modify(
00878        BackendDB *be,
00879        Entry *e,
00880        struct berval *text )
00881 {
00882        int rc;
00883        struct mdb_info *mdb;
00884        MDB_txn *tid;
00885        Operation op = {0};
00886        Opheader ohdr = {0};
00887 
00888        assert( be != NULL );
00889        assert( slapMode & SLAP_TOOL_MODE );
00890 
00891        assert( text != NULL );
00892        assert( text->bv_val != NULL );
00893        assert( text->bv_val[0] == '\0' ); /* overconservative? */
00894 
00895        assert ( e->e_id != NOID );
00896 
00897        Debug( LDAP_DEBUG_TRACE,
00898               "=> " LDAP_XSTRING(mdb_tool_entry_modify) "( %ld, \"%s\" )\n",
00899               (long) e->e_id, e->e_dn, 0 );
00900 
00901        mdb = (struct mdb_info *) be->be_private;
00902 
00903        if( cursor ) {
00904               mdb_cursor_close( cursor );
00905               cursor = NULL;
00906        }
00907        rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &tid );
00908        if( rc != 0 ) {
00909               snprintf( text->bv_val, text->bv_len,
00910                      "txn_begin failed: %s (%d)",
00911                      mdb_strerror(rc), rc );
00912               Debug( LDAP_DEBUG_ANY,
00913                      "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
00914                       text->bv_val, 0, 0 );
00915               return NOID;
00916        }
00917 
00918        op.o_hdr = &ohdr;
00919        op.o_bd = be;
00920        op.o_tmpmemctx = NULL;
00921        op.o_tmpmfuncs = &ch_mfuncs;
00922 
00923        /* id2entry index */
00924        rc = mdb_id2entry_update( &op, tid, NULL, e );
00925        if( rc != 0 ) {
00926               snprintf( text->bv_val, text->bv_len,
00927                             "id2entry_update failed: err=%d", rc );
00928               Debug( LDAP_DEBUG_ANY,
00929                      "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
00930                      text->bv_val, 0, 0 );
00931               goto done;
00932        }
00933 
00934 done:
00935        if( rc == 0 ) {
00936               rc = mdb_txn_commit( tid );
00937               if( rc != 0 ) {
00938                      snprintf( text->bv_val, text->bv_len,
00939                                    "txn_commit failed: %s (%d)",
00940                                    mdb_strerror(rc), rc );
00941                      Debug( LDAP_DEBUG_ANY,
00942                             "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": "
00943                             "%s\n", text->bv_val, 0, 0 );
00944                      e->e_id = NOID;
00945               }
00946 
00947        } else {
00948               mdb_txn_abort( tid );
00949               snprintf( text->bv_val, text->bv_len,
00950                      "txn_aborted! %s (%d)",
00951                      mdb_strerror(rc), rc );
00952               Debug( LDAP_DEBUG_ANY,
00953                      "=> " LDAP_XSTRING(mdb_tool_entry_modify) ": %s\n",
00954                      text->bv_val, 0, 0 );
00955               e->e_id = NOID;
00956        }
00957 
00958        return e->e_id;
00959 }
00960 
00961 static void *
00962 mdb_tool_index_task( void *ctx, void *ptr )
00963 {
00964        int base = *(int *)ptr;
00965 
00966        free( ptr );
00967        while ( 1 ) {
00968               ldap_pvt_thread_mutex_lock( &mdb_tool_index_mutex );
00969               mdb_tool_index_tcount--;
00970               if ( !mdb_tool_index_tcount )
00971                      ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
00972               ldap_pvt_thread_cond_wait( &mdb_tool_index_cond_work,
00973                      &mdb_tool_index_mutex );
00974               if ( slapd_shutdown ) {
00975                      mdb_tool_index_tcount--;
00976                      if ( !mdb_tool_index_tcount )
00977                             ldap_pvt_thread_cond_signal( &mdb_tool_index_cond_main );
00978                      ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
00979                      break;
00980               }
00981               ldap_pvt_thread_mutex_unlock( &mdb_tool_index_mutex );
00982               mdb_tool_index_rec[base].ir_i = mdb_index_recrun( mdb_tool_ix_op,
00983                      mdb_tool_ix_txn,
00984                      mdb_tool_info, mdb_tool_index_rec, mdb_tool_ix_id, base );
00985        }
00986 
00987        return NULL;
00988 }
00989 
00990 #ifdef MDB_TOOL_IDL_CACHING
00991 static int
00992 mdb_tool_idl_cmp( const void *v1, const void *v2 )
00993 {
00994        const mdb_tool_idl_cache *c1 = v1, *c2 = v2;
00995        int rc;
00996 
00997        if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc;
00998        return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len );
00999 }
01000 
01001 static int
01002 mdb_tool_idl_flush_one( MDB_cursor *mc, AttrInfo *ai, mdb_tool_idl_cache *ic )
01003 {
01004        mdb_tool_idl_cache_entry *ice;
01005        MDB_val key, data[2];
01006        int i, rc;
01007        ID id, nid;
01008 
01009        /* Freshly allocated, ignore it */
01010        if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) {
01011               return 0;
01012        }
01013 
01014        key.mv_data = ic->kstr.bv_val;
01015        key.mv_size = ic->kstr.bv_len;
01016 
01017        if ( ic->count > MDB_IDL_DB_SIZE ) {
01018               while ( ic->flags & WAS_FOUND ) {
01019                      rc = mdb_cursor_get( mc, &key, data, MDB_SET );
01020                      if ( rc ) {
01021                             /* FIXME: find out why this happens */
01022                             ic->flags = 0;
01023                             break;
01024                      }
01025                      if ( ic->flags & WAS_RANGE ) {
01026                             /* Skip lo */
01027                             rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
01028 
01029                             /* Get hi */
01030                             rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
01031 
01032                             /* Store range hi */
01033                             data[0].mv_data = &ic->last;
01034                             rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT );
01035                      } else {
01036                             /* Delete old data, replace with range */
01037                             ic->first = *(ID *)data[0].mv_data;
01038                             mdb_cursor_del( mc, MDB_NODUPDATA );
01039                      }
01040                      break;
01041               }
01042               if ( !(ic->flags & WAS_RANGE)) {
01043                      /* range, didn't exist before */
01044                      nid = 0;
01045                      data[0].mv_size = sizeof(ID);
01046                      data[0].mv_data = &nid;
01047                      rc = mdb_cursor_put( mc, &key, data, 0 );
01048                      if ( rc == 0 ) {
01049                             data[0].mv_data = &ic->first;
01050                             rc = mdb_cursor_put( mc, &key, data, 0 );
01051                             if ( rc == 0 ) {
01052                                    data[0].mv_data = &ic->last;
01053                                    rc = mdb_cursor_put( mc, &key, data, 0 );
01054                             }
01055                      }
01056                      if ( rc ) {
01057                             rc = -1;
01058                      }
01059               }
01060        } else {
01061               /* Normal write */
01062               int n;
01063 
01064               data[0].mv_size = sizeof(ID);
01065               rc = 0;
01066               i = ic->offset;
01067               for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
01068                      int end;
01069                      if ( ice->next ) {
01070                             end = IDBLOCK;
01071                      } else {
01072                             end = ic->count & (IDBLOCK-1);
01073                             if ( !end )
01074                                    end = IDBLOCK;
01075                      }
01076                      data[1].mv_size = end - i;
01077                      data[0].mv_data = &ice->ids[i];
01078                      i = 0;
01079                      rc = mdb_cursor_put( mc, &key, data, MDB_NODUPDATA|MDB_APPEND|MDB_MULTIPLE );
01080                      if ( rc ) {
01081                             if ( rc == MDB_KEYEXIST ) {
01082                                    rc = 0;
01083                                    continue;
01084                             }
01085                             rc = -1;
01086                             break;
01087                      }
01088               }
01089               if ( ic->head ) {
01090                      ic->tail->next = ai->ai_flist;
01091                      ai->ai_flist = ic->head;
01092               }
01093        }
01094        ic->head = ai->ai_clist;
01095        ai->ai_clist = ic;
01096        return rc;
01097 }
01098 
01099 static int
01100 mdb_tool_idl_flush_db( MDB_txn *txn, AttrInfo *ai )
01101 {
01102        MDB_cursor *mc;
01103        Avlnode *root;
01104        int rc;
01105 
01106        mdb_cursor_open( txn, ai->ai_dbi, &mc );
01107        root = tavl_end( ai->ai_root, TAVL_DIR_LEFT );
01108        do {
01109               rc = mdb_tool_idl_flush_one( mc, ai, root->avl_data );
01110               if ( rc != -1 )
01111                      rc = 0;
01112        } while ((root = tavl_next(root, TAVL_DIR_RIGHT)));
01113        mdb_cursor_close( mc );
01114 
01115        return rc;
01116 }
01117 
01118 static int
01119 mdb_tool_idl_flush( BackendDB *be, MDB_txn *txn )
01120 {
01121        struct mdb_info *mdb = (struct mdb_info *) be->be_private;
01122        int rc = 0;
01123        unsigned int i, dbi;
01124 
01125        for ( i=0; i < mdb->mi_nattrs; i++ ) {
01126               if ( !mdb->mi_attrs[i]->ai_root ) continue;
01127               rc = mdb_tool_idl_flush_db( txn, mdb->mi_attrs[i] );
01128               tavl_free(mdb->mi_attrs[i]->ai_root, NULL);
01129               mdb->mi_attrs[i]->ai_root = NULL;
01130               if ( rc )
01131                      break;
01132        }
01133        return rc;
01134 }
01135 
01136 int mdb_tool_idl_add(
01137        MDB_cursor *mc,
01138        struct berval *keys,
01139        ID id )
01140 {
01141        MDB_dbi dbi;
01142        mdb_tool_idl_cache *ic, itmp;
01143        mdb_tool_idl_cache_entry *ice;
01144        int i, rc, lcount;
01145        AttrInfo *ai = (AttrInfo *)mc;
01146        mc = ai->ai_cursor;
01147 
01148        dbi = ai->ai_dbi;
01149        for (i=0; keys[i].bv_val; i++) {
01150        itmp.kstr = keys[i];
01151        ic = tavl_find( (Avlnode *)ai->ai_root, &itmp, mdb_tool_idl_cmp );
01152 
01153        /* No entry yet, create one */
01154        if ( !ic ) {
01155               MDB_val key, data;
01156               ID nid;
01157               int rc;
01158 
01159               if ( ai->ai_clist ) {
01160                      ic = ai->ai_clist;
01161                      ai->ai_clist = ic->head;
01162               } else {
01163                      ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 );
01164               }
01165               ic->kstr.bv_len = itmp.kstr.bv_len;
01166               ic->kstr.bv_val = (char *)(ic+1);
01167               memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
01168               ic->head = ic->tail = NULL;
01169               ic->last = 0;
01170               ic->count = 0;
01171               ic->offset = 0;
01172               ic->flags = 0;
01173               tavl_insert( (Avlnode **)&ai->ai_root, ic, mdb_tool_idl_cmp,
01174                      avl_dup_error );
01175 
01176               /* load existing key count here */
01177               key.mv_size = keys[i].bv_len;
01178               key.mv_data = keys[i].bv_val;
01179               rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
01180               if ( rc == 0 ) {
01181                      ic->flags |= WAS_FOUND;
01182                      nid = *(ID *)data.mv_data;
01183                      if ( nid == 0 ) {
01184                             ic->count = MDB_IDL_DB_SIZE+1;
01185                             ic->flags |= WAS_RANGE;
01186                      } else {
01187                             size_t count;
01188 
01189                             mdb_cursor_count( mc, &count );
01190                             ic->count = count;
01191                             ic->first = nid;
01192                             ic->offset = count & (IDBLOCK-1);
01193                      }
01194               }
01195        }
01196        /* are we a range already? */
01197        if ( ic->count > MDB_IDL_DB_SIZE ) {
01198               ic->last = id;
01199               continue;
01200        /* Are we at the limit, and converting to a range? */
01201        } else if ( ic->count == MDB_IDL_DB_SIZE ) {
01202               if ( ic->head ) {
01203                      ic->tail->next = ai->ai_flist;
01204                      ai->ai_flist = ic->head;
01205               }
01206               ic->head = ic->tail = NULL;
01207               ic->last = id;
01208               ic->count++;
01209               continue;
01210        }
01211        /* No free block, create that too */
01212        lcount = ic->count & (IDBLOCK-1);
01213        if ( !ic->tail || lcount == 0) {
01214               if ( ai->ai_flist ) {
01215                      ice = ai->ai_flist;
01216                      ai->ai_flist = ice->next;
01217               } else {
01218                      ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry ));
01219               }
01220               ice->next = NULL;
01221               if ( !ic->head ) {
01222                      ic->head = ice;
01223               } else {
01224                      ic->tail->next = ice;
01225               }
01226               ic->tail = ice;
01227               if ( lcount )
01228                      ice->ids[lcount-1] = 0;
01229               if ( !ic->count )
01230                      ic->first = id;
01231        }
01232        ice = ic->tail;
01233        if (!lcount || ice->ids[lcount-1] != id)
01234               ice->ids[lcount] = id;
01235        ic->count++;
01236        }
01237 
01238        return 0;
01239 }
01240 #endif /* MDB_TOOL_IDL_CACHING */