repository.py 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230
  1. import os
  2. import shutil
  3. import tempfile
  4. from attic.hashindex import NSIndex
  5. from attic.helpers import Location, IntegrityError
  6. from attic.remote import RemoteRepository
  7. from attic.repository import Repository
  8. from attic.testsuite import AtticTestCase
  9. class RepositoryTestCase(AtticTestCase):
  10. def open(self, create=False):
  11. return Repository(os.path.join(self.tmppath, 'repository'), create=create)
  12. def setUp(self):
  13. self.tmppath = tempfile.mkdtemp()
  14. self.repository = self.open(create=True)
  15. def tearDown(self):
  16. self.repository.close()
  17. shutil.rmtree(self.tmppath)
  18. def test1(self):
  19. for x in range(100):
  20. self.repository.put(('%-32d' % x).encode('ascii'), b'SOMEDATA')
  21. key50 = ('%-32d' % 50).encode('ascii')
  22. self.assert_equal(self.repository.get(key50), b'SOMEDATA')
  23. self.repository.delete(key50)
  24. self.assert_raises(Repository.DoesNotExist, lambda: self.repository.get(key50))
  25. self.repository.commit()
  26. self.repository.close()
  27. repository2 = self.open()
  28. self.assert_raises(Repository.DoesNotExist, lambda: repository2.get(key50))
  29. for x in range(100):
  30. if x == 50:
  31. continue
  32. self.assert_equal(repository2.get(('%-32d' % x).encode('ascii')), b'SOMEDATA')
  33. repository2.close()
  34. def test2(self):
  35. """Test multiple sequential transactions
  36. """
  37. self.repository.put(b'00000000000000000000000000000000', b'foo')
  38. self.repository.put(b'00000000000000000000000000000001', b'foo')
  39. self.repository.commit()
  40. self.repository.delete(b'00000000000000000000000000000000')
  41. self.repository.put(b'00000000000000000000000000000001', b'bar')
  42. self.repository.commit()
  43. self.assert_equal(self.repository.get(b'00000000000000000000000000000001'), b'bar')
  44. def test_consistency(self):
  45. """Test cache consistency
  46. """
  47. self.repository.put(b'00000000000000000000000000000000', b'foo')
  48. self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
  49. self.repository.put(b'00000000000000000000000000000000', b'foo2')
  50. self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2')
  51. self.repository.put(b'00000000000000000000000000000000', b'bar')
  52. self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'bar')
  53. self.repository.delete(b'00000000000000000000000000000000')
  54. self.assert_raises(Repository.DoesNotExist, lambda: self.repository.get(b'00000000000000000000000000000000'))
  55. def test_consistency2(self):
  56. """Test cache consistency2
  57. """
  58. self.repository.put(b'00000000000000000000000000000000', b'foo')
  59. self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
  60. self.repository.commit()
  61. self.repository.put(b'00000000000000000000000000000000', b'foo2')
  62. self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2')
  63. self.repository.rollback()
  64. self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
  65. def test_single_kind_transactions(self):
  66. # put
  67. self.repository.put(b'00000000000000000000000000000000', b'foo')
  68. self.repository.commit()
  69. self.repository.close()
  70. # replace
  71. self.repository = self.open()
  72. self.repository.put(b'00000000000000000000000000000000', b'bar')
  73. self.repository.commit()
  74. self.repository.close()
  75. # delete
  76. self.repository = self.open()
  77. self.repository.delete(b'00000000000000000000000000000000')
  78. self.repository.commit()
  79. class RepositoryCheckTestCase(AtticTestCase):
  80. def open(self, create=False):
  81. return Repository(os.path.join(self.tmppath, 'repository'), create=create)
  82. def reopen(self):
  83. if self.repository:
  84. self.repository.close()
  85. self.repository = self.open()
  86. def setUp(self):
  87. self.tmppath = tempfile.mkdtemp()
  88. self.repository = self.open(create=True)
  89. def tearDown(self):
  90. self.repository.close()
  91. shutil.rmtree(self.tmppath)
  92. def get_objects(self, *ids):
  93. for id_ in ids:
  94. self.repository.get(('%032d' % id_).encode('ascii'))
  95. def add_objects(self, segments):
  96. for ids in segments:
  97. for id_ in ids:
  98. self.repository.put(('%032d' % id_).encode('ascii'), b'data')
  99. self.repository.commit()
  100. def get_head(self):
  101. return sorted(int(n) for n in os.listdir(os.path.join(self.tmppath, 'repository', 'data', '0')) if n.isdigit())[-1]
  102. def open_index(self):
  103. return NSIndex(os.path.join(self.tmppath, 'repository', 'index.{}'.format(self.get_head())))
  104. def corrupt_object(self, id_):
  105. idx = self.open_index()
  106. segment, offset = idx[('%032d' % id_).encode('ascii')]
  107. with open(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)), 'r+b') as fd:
  108. fd.seek(offset)
  109. fd.write(b'BOOM')
  110. def delete_segment(self, segment):
  111. os.unlink(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)))
  112. def delete_index(self):
  113. os.unlink(os.path.join(self.tmppath, 'repository', 'index.{}'.format(self.get_head())))
  114. def rename_index(self, new_name):
  115. os.rename(os.path.join(self.tmppath, 'repository', 'index.{}'.format(self.get_head())),
  116. os.path.join(self.tmppath, 'repository', new_name))
  117. def list_objects(self):
  118. return set((int(key) for key, _ in list(self.open_index().iteritems())))
  119. def test_repair_corrupted_segment(self):
  120. self.add_objects([[1, 2, 3], [4, 5, 6]])
  121. self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
  122. self.assert_equal(True, self.repository.check())
  123. self.corrupt_object(5)
  124. self.assert_raises(IntegrityError, lambda: self.get_objects(5))
  125. self.repository.rollback()
  126. # Make sure a regular check does not repair anything
  127. self.assert_equal(False, self.repository.check())
  128. self.assert_equal(False, self.repository.check())
  129. # Make sure a repair actually repairs the repo
  130. self.assert_equal(True, self.repository.check(repair=True))
  131. self.get_objects(4)
  132. self.assert_equal(True, self.repository.check())
  133. self.assert_equal(set([1, 2, 3, 4, 6]), self.list_objects())
  134. def test_repair_missing_segment(self):
  135. self.add_objects([[1, 2, 3], [4, 5, 6]])
  136. self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
  137. self.assert_equal(True, self.repository.check())
  138. self.delete_segment(1)
  139. self.repository.rollback()
  140. self.assert_equal(True, self.repository.check(repair=True))
  141. self.assert_equal(set([1, 2, 3]), self.list_objects())
  142. def test_repair_missing_commit_segment(self):
  143. self.add_objects([[1, 2, 3], [4, 5, 6]])
  144. self.delete_segment(1)
  145. self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
  146. self.assert_equal(False, self.repository.check())
  147. self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
  148. self.assert_equal(True, self.repository.check(repair=True))
  149. self.assert_raises(Repository.DoesNotExist, lambda: self.get_objects(4))
  150. self.assert_equal(set([1, 2, 3]), self.list_objects())
  151. def test_repair_corrupted_commit_segment(self):
  152. self.add_objects([[1, 2, 3], [4, 5, 6]])
  153. with open(os.path.join(self.tmppath, 'repository', 'data', '0', '1'), 'ab') as fd:
  154. fd.write(b'X')
  155. self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
  156. self.assert_equal(False, self.repository.check())
  157. self.assert_equal(True, self.repository.check(repair=True))
  158. self.get_objects(4)
  159. self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
  160. def test_repair_missing_index(self):
  161. self.add_objects([[1, 2, 3], [4, 5, 6]])
  162. self.delete_index()
  163. self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
  164. self.assert_equal(False, self.repository.check())
  165. self.assert_equal(True, self.repository.check(repair=True))
  166. self.assert_equal(True, self.repository.check())
  167. self.get_objects(4)
  168. self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
  169. def test_repair_index_too_old(self):
  170. self.add_objects([[1, 2, 3], [4, 5, 6]])
  171. self.rename_index('index.0')
  172. self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
  173. self.assert_equal(False, self.repository.check())
  174. self.assert_equal(True, self.repository.check(repair=True))
  175. self.assert_equal(True, self.repository.check())
  176. self.get_objects(4)
  177. self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
  178. def test_repair_index_too_new(self):
  179. self.add_objects([[1, 2, 3], [4, 5, 6]])
  180. self.rename_index('index.100')
  181. self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
  182. self.assert_equal(False, self.repository.check())
  183. self.assert_equal(True, self.repository.check(repair=True))
  184. self.assert_equal(True, self.repository.check())
  185. self.get_objects(4)
  186. self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
  187. class RemoteRepositoryTestCase(RepositoryTestCase):
  188. def open(self, create=False):
  189. return RemoteRepository(Location('__testsuite__:' + os.path.join(self.tmppath, 'repository')), create=create)
  190. class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase):
  191. def open(self, create=False):
  192. return RemoteRepository(Location('__testsuite__:' + os.path.join(self.tmppath, 'repository')), create=create)