Back to index

obnam  1.1
clientmetadatatree_tests.py
Go to the documentation of this file.
00001 # Copyright 2010  Lars Wirzenius
00002 # 
00003 # This program is free software: you can redistribute it and/or modify
00004 # it under the terms of the GNU General Public License as published by
00005 # the Free Software Foundation, either version 3 of the License, or
00006 # (at your option) any later version.
00007 # 
00008 # This program is distributed in the hope that it will be useful,
00009 # but WITHOUT ANY WARRANTY; without even the implied warranty of
00010 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00011 # GNU General Public License for more details.
00012 # 
00013 # You should have received a copy of the GNU General Public License
00014 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
00015 
00016 
00017 import shutil
00018 import stat
00019 import tempfile
00020 import time
00021 import unittest
00022 
00023 import obnamlib
00024 
00025 
00026 class ClientMetadataTreeTests(unittest.TestCase):
00027 
00028     def current_time(self):
00029         return time.time() if self.now is None else self.now
00030 
00031     def setUp(self):
00032         self.now = None
00033         self.tempdir = tempfile.mkdtemp()
00034         fs = obnamlib.LocalFS(self.tempdir)
00035         self.hooks = obnamlib.HookManager()
00036         self.hooks.new('repository-toplevel-init')
00037         self.client = obnamlib.ClientMetadataTree(fs, 'clientid',
00038                                    obnamlib.DEFAULT_NODE_SIZE,
00039                                    obnamlib.DEFAULT_UPLOAD_QUEUE_SIZE,
00040                                    obnamlib.DEFAULT_LRU_SIZE, self)
00041         self.file_size = 123
00042         self.file_metadata = obnamlib.Metadata(st_mode=stat.S_IFREG | 0666,
00043                                                st_size=self.file_size)
00044         self.file_encoded = obnamlib.encode_metadata(self.file_metadata)
00045         
00046     def tearDown(self):
00047         shutil.rmtree(self.tempdir)
00048 
00049     def test_has_not_current_generation_initially(self):
00050         self.assertEqual(self.client.tree, None)
00051     
00052     def test_lists_no_generations_initially(self):
00053         self.assertEqual(self.client.list_generations(), [])
00054 
00055     def test_starts_generation(self):
00056         self.now = 12765
00057         self.client.start_generation()
00058         self.assertNotEqual(self.client.tree, None)
00059         
00060         def lookup(x):
00061             key = self.client.genkey(x)
00062             return self.client._lookup_int(self.client.tree, key)
00063 
00064         genid = self.client.get_generation_id(self.client.tree)
00065         self.assertEqual(lookup(self.client.GEN_ID), genid)
00066         self.assertEqual(lookup(self.client.GEN_STARTED), 12765)
00067         self.assertFalse(self.client.get_is_checkpoint(genid))
00068 
00069     def test_starts_second_generation(self):
00070         self.now = 1
00071         self.client.start_generation()
00072         genid1 = self.client.get_generation_id(self.client.tree)
00073         self.client.commit()
00074         self.assertEqual(self.client.tree, None)
00075         self.now = 2
00076         self.client.start_generation()
00077         self.assertNotEqual(self.client.tree, None)
00078         
00079         def lookup(x):
00080             key = self.client.genkey(x)
00081             return self.client._lookup_int(self.client.tree, key)
00082 
00083         genid2 = self.client.get_generation_id(self.client.tree)
00084         self.assertEqual(lookup(self.client.GEN_ID), genid2)
00085         self.assertNotEqual(genid1, genid2)
00086         self.assertEqual(lookup(self.client.GEN_STARTED), 2)
00087         self.assertFalse(self.client.get_is_checkpoint(genid2))
00088         self.assertEqual(self.client.list_generations(), [genid1, genid2])
00089 
00090     def test_sets_is_checkpoint(self):
00091         self.client.start_generation()
00092         genid = self.client.get_generation_id(self.client.tree)
00093         self.client.set_current_generation_is_checkpoint(True)
00094         self.assert_(self.client.get_is_checkpoint(genid))
00095 
00096     def test_unsets_is_checkpoint(self):
00097         self.client.start_generation()
00098         genid = self.client.get_generation_id(self.client.tree)
00099         self.client.set_current_generation_is_checkpoint(True)
00100         self.client.set_current_generation_is_checkpoint(False)
00101         self.assertFalse(self.client.get_is_checkpoint(genid))
00102 
00103     def test_removes_generation(self):
00104         self.client.start_generation()
00105         self.client.commit()
00106         genid = self.client.list_generations()[0]
00107         self.client.remove_generation(genid)
00108         self.assertEqual(self.client.list_generations(), [])
00109 
00110     def test_removes_started_generation(self):
00111         self.client.start_generation()
00112         self.client.remove_generation(self.client.list_generations()[0])
00113         self.assertEqual(self.client.list_generations(), [])
00114         self.assertEqual(self.client.tree, None)
00115 
00116     def test_started_generation_has_start_time(self):
00117         self.now = 1
00118         self.client.start_generation()
00119         genid = self.client.get_generation_id(self.client.tree)
00120         self.assertEqual(self.client.get_generation_times(genid), (1, None))
00121 
00122     def test_committed_generation_has_times(self):
00123         self.now = 1
00124         self.client.start_generation()
00125         genid = self.client.get_generation_id(self.client.tree)
00126         self.now = 2
00127         self.client.commit()
00128         self.assertEqual(self.client.get_generation_times(genid), (1, 2))
00129 
00130     def test_single_empty_generation_counts_zero_files(self):
00131         self.client.start_generation()
00132         genid = self.client.get_generation_id(self.client.tree)
00133         self.client.commit()
00134         self.assertEqual(self.client.get_generation_file_count(genid), 0)
00135 
00136     def test_counts_files_in_first_generation(self):
00137         self.client.start_generation()
00138         genid = self.client.get_generation_id(self.client.tree)
00139         self.client.create('/foo', self.file_encoded)
00140         self.client.commit()
00141         self.assertEqual(self.client.get_generation_file_count(genid), 1)
00142 
00143     def test_counts_new_files_in_second_generation(self):
00144         self.client.start_generation()
00145         self.client.create('/foo', self.file_encoded)
00146         self.client.commit()
00147 
00148         self.client.start_generation()
00149         genid = self.client.get_generation_id(self.client.tree)
00150         self.client.create('/bar', self.file_encoded)
00151         self.client.commit()
00152 
00153         self.assertEqual(self.client.get_generation_file_count(genid), 2)
00154 
00155     def test_discounts_deleted_files_in_second_generation(self):
00156         self.client.start_generation()
00157         self.client.create('/foo', self.file_encoded)
00158         self.client.commit()
00159 
00160         self.client.start_generation()
00161         genid = self.client.get_generation_id(self.client.tree)
00162         self.client.remove('/foo')
00163         self.client.commit()
00164 
00165         self.assertEqual(self.client.get_generation_file_count(genid), 0)
00166 
00167     def test_does_not_increment_count_for_recreated_files(self):
00168         self.client.start_generation()
00169         self.client.create('/foo', self.file_encoded)
00170         self.client.commit()
00171 
00172         self.client.start_generation()
00173         genid = self.client.get_generation_id(self.client.tree)
00174         self.client.create('/foo', self.file_encoded)
00175         self.client.commit()
00176 
00177         self.assertEqual(self.client.get_generation_file_count(genid), 1)
00178 
00179     def test_single_empty_generation_has_no_data(self):
00180         self.client.start_generation()
00181         genid = self.client.get_generation_id(self.client.tree)
00182         self.client.commit()
00183         self.assertEqual(self.client.get_generation_data(genid), 0)
00184 
00185     def test_has_data_in_first_generation(self):
00186         self.client.start_generation()
00187         genid = self.client.get_generation_id(self.client.tree)
00188         self.client.create('/foo', self.file_encoded)
00189         self.client.commit()
00190         self.assertEqual(self.client.get_generation_data(genid),
00191                          self.file_size)
00192 
00193     def test_counts_new_files_in_second_generation(self):
00194         self.client.start_generation()
00195         self.client.create('/foo', self.file_encoded)
00196         self.client.commit()
00197 
00198         self.client.start_generation()
00199         genid = self.client.get_generation_id(self.client.tree)
00200         self.client.create('/bar', self.file_encoded)
00201         self.client.commit()
00202 
00203         self.assertEqual(self.client.get_generation_data(genid),
00204                          2 * self.file_size)
00205 
00206     def test_counts_replaced_data_in_second_generation(self):
00207         self.client.start_generation()
00208         self.client.create('/foo', self.file_encoded)
00209         self.client.commit()
00210 
00211         self.client.start_generation()
00212         genid = self.client.get_generation_id(self.client.tree)
00213         self.client.create('/foo', self.file_encoded)
00214         self.client.commit()
00215 
00216         self.assertEqual(self.client.get_generation_data(genid),
00217                          self.file_size)
00218 
00219     def test_discounts_deleted_data_in_second_generation(self):
00220         self.client.start_generation()
00221         self.client.create('/foo', self.file_encoded)
00222         self.client.commit()
00223 
00224         self.client.start_generation()
00225         genid = self.client.get_generation_id(self.client.tree)
00226         self.client.remove('/foo')
00227         self.client.commit()
00228 
00229         self.assertEqual(self.client.get_generation_data(genid), 0)
00230 
00231     def test_does_not_increment_data_for_recreated_files(self):
00232         self.client.start_generation()
00233         self.client.create('/foo', self.file_encoded)
00234         self.client.commit()
00235 
00236         self.client.start_generation()
00237         genid = self.client.get_generation_id(self.client.tree)
00238         self.client.create('/foo', self.file_encoded)
00239         self.client.commit()
00240 
00241         self.assertEqual(self.client.get_generation_data(genid), 
00242                          self.file_size)
00243 
00244     def test_finds_generation_the_first_time(self):
00245         self.client.start_generation()
00246         tree = self.client.tree
00247         genid = self.client.get_generation_id(tree)
00248         self.client.commit()
00249         self.assertEqual(self.client.find_generation(genid), tree)
00250 
00251     def test_finds_generation_the_second_time(self):
00252         self.client.start_generation()
00253         tree = self.client.tree
00254         genid = self.client.get_generation_id(tree)
00255         self.client.commit()
00256         self.client.find_generation(genid)
00257         self.assertEqual(self.client.find_generation(genid), tree)
00258 
00259     def test_find_generation_raises_keyerror_for_empty_forest(self):
00260         self.client.init_forest()
00261         self.assertRaises(KeyError, self.client.find_generation, 0)
00262 
00263     def test_find_generation_raises_keyerror_for_unknown_generation(self):
00264         self.assertRaises(KeyError, self.client.find_generation, 0)
00265 
00266 
00267 class ClientMetadataTreeFileOpsTests(unittest.TestCase):
00268 
00269     def current_time(self):
00270         return time.time() if self.now is None else self.now
00271 
00272     def setUp(self):
00273         self.now = None
00274         self.tempdir = tempfile.mkdtemp()
00275         fs = obnamlib.LocalFS(self.tempdir)
00276         self.hooks = obnamlib.HookManager()
00277         self.hooks.new('repository-toplevel-init')
00278         self.client = obnamlib.ClientMetadataTree(fs, 'clientid',
00279                                             obnamlib.DEFAULT_NODE_SIZE,
00280                                             obnamlib.DEFAULT_UPLOAD_QUEUE_SIZE,
00281                                             obnamlib.DEFAULT_LRU_SIZE, 
00282                                             self)
00283         # Force use of filename hash collisions.
00284         self.client.default_file_id = self.client._bad_default_file_id
00285         self.client.start_generation()
00286         self.clientid = self.client.get_generation_id(self.client.tree)
00287         self.file_metadata = obnamlib.Metadata(st_mode=stat.S_IFREG | 0666)
00288         self.file_encoded = obnamlib.encode_metadata(self.file_metadata)
00289         self.dir_metadata = obnamlib.Metadata(st_mode=stat.S_IFDIR | 0777)
00290         self.dir_encoded = obnamlib.encode_metadata(self.dir_metadata)
00291         
00292     def tearDown(self):
00293         shutil.rmtree(self.tempdir)
00294 
00295     def test_has_empty_root_initially(self):
00296         self.assertEqual(self.client.listdir(self.clientid, '/'), [])
00297 
00298     def test_has_no_metadata_initially(self):
00299         self.assertRaises(KeyError, self.client.get_metadata, self.clientid, 
00300                           '/foo')
00301 
00302     def test_sets_metadata(self):
00303         self.client.set_metadata('/foo', self.file_encoded)
00304         self.assertEqual(self.client.get_metadata(self.clientid, '/foo'), 
00305                          self.file_encoded)
00306 
00307     def test_creates_file_at_root(self):
00308         self.client.create('/foo', self.file_encoded)
00309         self.assertEqual(self.client.listdir(self.clientid, '/'), ['foo'])
00310         self.assertEqual(self.client.get_metadata(self.clientid, '/foo'),
00311                          self.file_encoded)
00312 
00313     def test_removes_file_at_root(self):
00314         self.client.create('/foo', self.file_encoded)
00315         self.client.remove('/foo')
00316         self.assertEqual(self.client.listdir(self.clientid, '/'), [])
00317         self.assertRaises(KeyError, self.client.get_metadata, 
00318                           self.clientid, '/foo')
00319 
00320     def test_creates_directory_at_root(self):
00321         self.client.create('/foo', self.dir_encoded)
00322         self.assertEqual(self.client.listdir(self.clientid, '/'), ['foo'])
00323         self.assertEqual(self.client.get_metadata(self.clientid, '/foo'), 
00324                          self.dir_encoded)
00325 
00326     def test_removes_directory_at_root(self):
00327         self.client.create('/foo', self.dir_encoded)
00328         self.client.remove('/foo')
00329         self.assertEqual(self.client.listdir(self.clientid, '/'), [])
00330         self.assertRaises(KeyError, self.client.get_metadata, 
00331                           self.clientid, '/foo')
00332 
00333     def test_creates_directory_and_files_and_subdirs(self):
00334         self.client.create('/foo', self.dir_encoded)
00335         self.client.create('/foo/foobar', self.file_encoded)
00336         self.client.create('/foo/bar', self.dir_encoded)
00337         self.client.create('/foo/bar/baz', self.file_encoded)
00338         self.assertEqual(self.client.listdir(self.clientid, '/'), ['foo'])
00339         self.assertEqual(sorted(self.client.listdir(self.clientid, '/foo')), 
00340                          ['bar', 'foobar'])
00341         self.assertEqual(self.client.listdir(self.clientid, '/foo/bar'), 
00342                          ['baz'])
00343         self.assertEqual(self.client.get_metadata(self.clientid, '/foo'), 
00344                          self.dir_encoded)
00345         self.assertEqual(self.client.get_metadata(self.clientid, '/foo/bar'), 
00346                          self.dir_encoded)
00347         self.assertEqual(self.client.get_metadata(self.clientid, '/foo/foobar'), 
00348                          self.file_encoded)
00349         self.assertEqual(self.client.get_metadata(self.clientid, 
00350                                                   '/foo/bar/baz'), 
00351                          self.file_encoded)
00352 
00353     def test_removes_directory_and_files_and_subdirs(self):
00354         self.client.create('/foo', self.dir_encoded)
00355         self.client.create('/foo/foobar', self.file_encoded)
00356         self.client.create('/foo/bar', self.dir_encoded)
00357         self.client.create('/foo/bar/baz', self.file_encoded)
00358         self.client.remove('/foo')
00359         self.assertEqual(self.client.listdir(self.clientid, '/'), [])
00360         self.assertRaises(KeyError, self.client.get_metadata, 
00361                           self.clientid, '/foo')
00362         self.assertRaises(KeyError, self.client.get_metadata, 
00363                           self.clientid, '/foo/foobar')
00364         self.assertRaises(KeyError, self.client.get_metadata, 
00365                           self.clientid, '/foo/bar')
00366         self.assertRaises(KeyError, self.client.get_metadata, 
00367                           self.clientid, '/foo/bar/baz')
00368 
00369     def test_has_no_file_chunks_initially(self):
00370         self.assertEqual(self.client.get_file_chunks(self.clientid, '/foo'), [])
00371 
00372     def test_sets_file_chunks(self):
00373         self.client.set_file_chunks('/foo', [1, 2, 3])
00374         self.assertEqual(self.client.get_file_chunks(self.clientid, '/foo'), 
00375                          [1, 2, 3])
00376 
00377     def test_appends_file_chunks_to_empty_list(self):
00378         self.client.append_file_chunks('/foo', [1, 2, 3])
00379         self.assertEqual(self.client.get_file_chunks(self.clientid, '/foo'), 
00380                          [1, 2, 3])
00381 
00382     def test_appends_file_chunks_to_nonempty_list(self):
00383         self.client.set_file_chunks('/foo', [1, 2, 3])
00384         self.client.append_file_chunks('/foo', [4, 5, 6])
00385         self.assertEqual(self.client.get_file_chunks(self.clientid, '/foo'), 
00386                          [1, 2, 3, 4, 5, 6])
00387                          
00388     def test_generation_has_no_chunk_refs_initially(self):
00389         minkey = self.client.chunk_key(0, 0)
00390         maxkey = self.client.chunk_key(obnamlib.MAX_ID, obnamlib.MAX_ID)
00391         self.assertEqual(list(self.client.tree.lookup_range(minkey, maxkey)), 
00392                          [])
00393                          
00394     def test_generation_has_no_chunk_refs_initially(self):
00395         minkey = self.client.chunk_key(0, 0)
00396         maxkey = self.client.chunk_key(obnamlib.MAX_ID, obnamlib.MAX_ID)
00397         self.assertEqual(list(self.client.tree.lookup_range(minkey, maxkey)), 
00398                          [])
00399 
00400     def test_sets_file_chunks(self):
00401         self.client.set_file_chunks('/foo', [1, 2, 3])
00402         self.assertEqual(self.client.get_file_chunks(self.clientid, '/foo'), 
00403                          [1, 2, 3])
00404                          
00405     def test_generation_has_no_chunk_refs_initially(self):
00406         minkey = self.client.chunk_key(0, 0)
00407         maxkey = self.client.chunk_key(obnamlib.MAX_ID, obnamlib.MAX_ID)
00408         self.assertEqual(list(self.client.tree.lookup_range(minkey, maxkey)), 
00409                          [])
00410 
00411     def test_set_file_chunks_adds_chunk_refs(self):
00412         self.client.set_file_chunks('/foo', [1, 2])
00413         file_id = self.client.get_file_id(self.client.tree, '/foo')
00414         minkey = self.client.chunk_key(0, 0)
00415         maxkey = self.client.chunk_key(obnamlib.MAX_ID, obnamlib.MAX_ID)
00416         self.assertEqual(set(self.client.tree.lookup_range(minkey, maxkey)), 
00417                          set([(self.client.chunk_key(1, file_id), ''),
00418                               (self.client.chunk_key(2, file_id), '')]))
00419 
00420     def test_set_file_chunks_removes_now_unused_chunk_refs(self):
00421         self.client.set_file_chunks('/foo', [1, 2])
00422         self.client.set_file_chunks('/foo', [1])
00423         file_id = self.client.get_file_id(self.client.tree, '/foo')
00424         minkey = self.client.chunk_key(0, 0)
00425         maxkey = self.client.chunk_key(obnamlib.MAX_ID, obnamlib.MAX_ID)
00426         self.assertEqual(list(self.client.tree.lookup_range(minkey, maxkey)), 
00427                          [(self.client.chunk_key(1, file_id), '')])
00428 
00429     def test_remove_removes_chunk_refs(self):
00430         self.client.set_file_chunks('/foo', [1, 2])
00431         self.client.remove('/foo')
00432         minkey = self.client.chunk_key(0, 0)
00433         maxkey = self.client.chunk_key(obnamlib.MAX_ID, obnamlib.MAX_ID)
00434         self.assertEqual(list(self.client.tree.lookup_range(minkey, maxkey)), 
00435                          [])
00436         
00437     def test_report_chunk_not_in_use_initially(self):
00438         gen_id = self.client.get_generation_id(self.client.tree)
00439         self.assertFalse(self.client.chunk_in_use(gen_id, 0))
00440         
00441     def test_report_chunk_in_use_after_it_is(self):
00442         gen_id = self.client.get_generation_id(self.client.tree)
00443         self.client.set_file_chunks('/foo', [0])
00444         self.assertTrue(self.client.chunk_in_use(gen_id, 0))
00445 
00446     def test_lists_no_chunks_in_generation_initially(self):
00447         gen_id = self.client.get_generation_id(self.client.tree)
00448         self.assertEqual(self.client.list_chunks_in_generation(gen_id), [])
00449 
00450     def test_lists_used_chunks_in_generation(self):
00451         gen_id = self.client.get_generation_id(self.client.tree)
00452         self.client.set_file_chunks('/foo', [0])
00453         self.client.set_file_chunks('/bar', [1])
00454         self.assertEqual(set(self.client.list_chunks_in_generation(gen_id)), 
00455                          set([0, 1]))
00456 
00457     def test_lists_chunks_in_generation_only_once(self):
00458         gen_id = self.client.get_generation_id(self.client.tree)
00459         self.client.set_file_chunks('/foo', [0])
00460         self.client.set_file_chunks('/bar', [0])
00461         self.assertEqual(self.client.list_chunks_in_generation(gen_id), [0])
00462