2
0

archiver.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304
  1. import os
  2. from io import StringIO
  3. import stat
  4. import subprocess
  5. import sys
  6. import shutil
  7. import tempfile
  8. import time
  9. import unittest
  10. from hashlib import sha256
  11. from attic import xattr
  12. from attic.archiver import Archiver
  13. from attic.repository import Repository
  14. from attic.testsuite import AtticTestCase
  15. from attic.crypto import bytes_to_long, num_aes_blocks
  16. try:
  17. import llfuse
  18. has_llfuse = True
  19. except ImportError:
  20. has_llfuse = False
  21. src_dir = os.path.join(os.getcwd(), os.path.dirname(__file__), '..', '..')
  22. class changedir:
  23. def __init__(self, dir):
  24. self.dir = dir
  25. def __enter__(self):
  26. self.old = os.getcwd()
  27. os.chdir(self.dir)
  28. def __exit__(self, *args, **kw):
  29. os.chdir(self.old)
  30. class ArchiverTestCase(AtticTestCase):
  31. prefix = ''
  32. def setUp(self):
  33. self.archiver = Archiver()
  34. self.tmpdir = tempfile.mkdtemp()
  35. self.repository_path = os.path.join(self.tmpdir, 'repository')
  36. self.repository_location = self.prefix + self.repository_path
  37. self.input_path = os.path.join(self.tmpdir, 'input')
  38. self.output_path = os.path.join(self.tmpdir, 'output')
  39. self.keys_path = os.path.join(self.tmpdir, 'keys')
  40. self.cache_path = os.path.join(self.tmpdir, 'cache')
  41. self.exclude_file_path = os.path.join(self.tmpdir, 'excludes')
  42. os.environ['ATTIC_KEYS_DIR'] = self.keys_path
  43. os.environ['ATTIC_CACHE_DIR'] = self.cache_path
  44. os.mkdir(self.input_path)
  45. os.mkdir(self.output_path)
  46. os.mkdir(self.keys_path)
  47. os.mkdir(self.cache_path)
  48. with open(self.exclude_file_path, 'wb') as fd:
  49. fd.write(b'input/file2\n# A commment line, then a blank line\n\n')
  50. self._old_wd = os.getcwd()
  51. os.chdir(self.tmpdir)
  52. def tearDown(self):
  53. shutil.rmtree(self.tmpdir)
  54. os.chdir(self._old_wd)
  55. def attic(self, *args, **kw):
  56. exit_code = kw.get('exit_code', 0)
  57. fork = kw.get('fork', False)
  58. if fork:
  59. try:
  60. output = subprocess.check_output((sys.executable, '-m', 'attic.archiver') + args)
  61. ret = 0
  62. except subprocess.CalledProcessError as e:
  63. output = e.output
  64. ret = e.returncode
  65. output = os.fsdecode(output)
  66. if ret != exit_code:
  67. print(output)
  68. self.assert_equal(exit_code, ret)
  69. return output
  70. args = list(args)
  71. stdout, stderr = sys.stdout, sys.stderr
  72. try:
  73. output = StringIO()
  74. sys.stdout = sys.stderr = output
  75. ret = self.archiver.run(args)
  76. sys.stdout, sys.stderr = stdout, stderr
  77. if ret != exit_code:
  78. print(output.getvalue())
  79. self.assert_equal(exit_code, ret)
  80. return output.getvalue()
  81. finally:
  82. sys.stdout, sys.stderr = stdout, stderr
  83. def create_src_archive(self, name):
  84. self.attic('create', self.repository_location + '::' + name, src_dir)
  85. def create_regual_file(self, name, size=0):
  86. filename = os.path.join(self.input_path, name)
  87. if not os.path.exists(os.path.dirname(filename)):
  88. os.makedirs(os.path.dirname(filename))
  89. with open(filename, 'wb') as fd:
  90. fd.write(b'X' * size)
  91. def create_test_files(self):
  92. """Create a minimal test case including all supported file types
  93. """
  94. # File
  95. self.create_regual_file('empty', size=0)
  96. self.create_regual_file('file1', size=1024 * 80)
  97. # Directory
  98. self.create_regual_file('dir2/file2', size=1024 * 80)
  99. # File owner
  100. os.chown('input/file1', 100, 200)
  101. # File mode
  102. os.chmod('input/file1', 0o7755)
  103. os.chmod('input/dir2', 0o555)
  104. # Block device
  105. os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20))
  106. # Char device
  107. os.mknod('input/cdev', 0o600 | stat.S_IFCHR, os.makedev(30, 40))
  108. if xattr.is_enabled():
  109. xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo', b'bar')
  110. # Hard link
  111. os.link(os.path.join(self.input_path, 'file1'),
  112. os.path.join(self.input_path, 'hardlink'))
  113. # Symlink
  114. os.symlink('somewhere', os.path.join(self.input_path, 'link1'))
  115. # FIFO node
  116. os.mkfifo(os.path.join(self.input_path, 'fifo1'))
  117. def test_basic_functionality(self):
  118. self.create_test_files()
  119. self.attic('init', self.repository_location)
  120. self.attic('create', self.repository_location + '::test', 'input')
  121. self.attic('create', self.repository_location + '::test.2', 'input')
  122. with changedir('output'):
  123. self.attic('extract', self.repository_location + '::test')
  124. self.assert_equal(len(self.attic('list', self.repository_location).splitlines()), 2)
  125. self.assert_equal(len(self.attic('list', self.repository_location + '::test').splitlines()), 10)
  126. self.assert_dirs_equal('input', 'output/input')
  127. info_output = self.attic('info', self.repository_location + '::test')
  128. shutil.rmtree(self.cache_path)
  129. info_output2 = self.attic('info', self.repository_location + '::test')
  130. # info_output2 starts with some "initializing cache" text but should
  131. # end the same way as info_output
  132. assert info_output2.endswith(info_output)
  133. def test_extract_include_exclude(self):
  134. self.attic('init', self.repository_location)
  135. self.create_regual_file('file1', size=1024 * 80)
  136. self.create_regual_file('file2', size=1024 * 80)
  137. self.create_regual_file('file3', size=1024 * 80)
  138. self.create_regual_file('file4', size=1024 * 80)
  139. self.attic('create', '--exclude=input/file4', self.repository_location + '::test', 'input')
  140. with changedir('output'):
  141. self.attic('extract', self.repository_location + '::test', 'input/file1', )
  142. self.assert_equal(sorted(os.listdir('output/input')), ['file1'])
  143. with changedir('output'):
  144. self.attic('extract', '--exclude=input/file2', self.repository_location + '::test')
  145. self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file3'])
  146. with changedir('output'):
  147. self.attic('extract', '--exclude-from=' + self.exclude_file_path, self.repository_location + '::test')
  148. self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file3'])
  149. def test_path_normalization(self):
  150. self.attic('init', self.repository_location)
  151. self.create_regual_file('dir1/dir2/file', size=1024 * 80)
  152. with changedir('input/dir1/dir2'):
  153. self.attic('create', self.repository_location + '::test', '../../../input/dir1/../dir1/dir2/..')
  154. output = self.attic('list', self.repository_location + '::test')
  155. self.assert_not_in('..', output)
  156. self.assert_in(' input/dir1/dir2/file', output)
  157. def test_overwrite(self):
  158. self.create_regual_file('file1', size=1024 * 80)
  159. self.create_regual_file('dir2/file2', size=1024 * 80)
  160. self.attic('init', self.repository_location)
  161. self.attic('create', self.repository_location + '::test', 'input')
  162. # Overwriting regular files and directories should be supported
  163. os.mkdir('output/input')
  164. os.mkdir('output/input/file1')
  165. os.mkdir('output/input/dir2')
  166. with changedir('output'):
  167. self.attic('extract', self.repository_location + '::test')
  168. self.assert_dirs_equal('input', 'output/input')
  169. # But non-empty dirs should fail
  170. os.unlink('output/input/file1')
  171. os.mkdir('output/input/file1')
  172. os.mkdir('output/input/file1/dir')
  173. with changedir('output'):
  174. self.attic('extract', self.repository_location + '::test', exit_code=1)
  175. def test_delete(self):
  176. self.create_regual_file('file1', size=1024 * 80)
  177. self.create_regual_file('dir2/file2', size=1024 * 80)
  178. self.attic('init', self.repository_location)
  179. self.attic('create', self.repository_location + '::test', 'input')
  180. self.attic('create', self.repository_location + '::test.2', 'input')
  181. self.attic('verify', self.repository_location + '::test')
  182. self.attic('verify', self.repository_location + '::test.2')
  183. self.attic('delete', self.repository_location + '::test')
  184. self.attic('verify', self.repository_location + '::test.2')
  185. self.attic('delete', self.repository_location + '::test.2')
  186. # Make sure all data except the manifest has been deleted
  187. repository = Repository(self.repository_path)
  188. self.assert_equal(len(repository), 1)
  189. def test_corrupted_repository(self):
  190. self.attic('init', self.repository_location)
  191. self.create_src_archive('test')
  192. self.attic('verify', self.repository_location + '::test')
  193. self.attic('check', self.repository_location)
  194. name = sorted(os.listdir(os.path.join(self.tmpdir, 'repository', 'data', '0')), reverse=True)[0]
  195. fd = open(os.path.join(self.tmpdir, 'repository', 'data', '0', name), 'r+')
  196. fd.seek(100)
  197. fd.write('XXXX')
  198. fd.close()
  199. self.attic('verify', self.repository_location + '::test', exit_code=1)
  200. self.attic('check', self.repository_location, exit_code=1)
  201. def test_readonly_repository(self):
  202. self.attic('init', self.repository_location)
  203. self.create_src_archive('test')
  204. os.system('chmod -R ugo-w ' + self.repository_path)
  205. try:
  206. self.attic('verify', self.repository_location + '::test')
  207. finally:
  208. # Restore permissions so shutil.rmtree is able to delete it
  209. os.system('chmod -R u+w ' + self.repository_path)
  210. def test_prune_repository(self):
  211. self.attic('init', self.repository_location)
  212. self.attic('create', self.repository_location + '::test1', src_dir)
  213. self.attic('create', self.repository_location + '::test2', src_dir)
  214. self.attic('prune', self.repository_location, '--daily=2')
  215. output = self.attic('list', self.repository_location)
  216. assert 'test1' not in output
  217. assert 'test2' in output
  218. def test_usage(self):
  219. self.assert_raises(SystemExit, lambda: self.attic())
  220. self.assert_raises(SystemExit, lambda: self.attic('-h'))
  221. @unittest.skipUnless(has_llfuse, 'llfuse not installed')
  222. def test_mount(self):
  223. mountpoint = os.path.join(self.tmpdir, 'mountpoint')
  224. os.mkdir(mountpoint)
  225. self.attic('init', self.repository_location)
  226. self.create_test_files()
  227. self.attic('create', self.repository_location + '::archive', 'input')
  228. try:
  229. self.attic('mount', self.repository_location + '::archive', mountpoint, fork=True)
  230. self.wait_for_mount(mountpoint)
  231. self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'input'))
  232. finally:
  233. if sys.platform.startswith('linux'):
  234. os.system('fusermount -u ' + mountpoint)
  235. else:
  236. os.system('umount ' + mountpoint)
  237. os.rmdir(mountpoint)
  238. # Give the daemon some time to exit
  239. time.sleep(.2)
  240. def verify_aes_counter_uniqueness(self, method):
  241. seen = set() # Chunks already seen
  242. used = set() # counter values already used
  243. def verify_uniqueness():
  244. repository = Repository(self.repository_path)
  245. for key, _ in repository.get_read_only_index(repository.get_transaction_id()).iteritems():
  246. data = repository.get(key)
  247. hash = sha256(data).digest()
  248. if not hash in seen:
  249. seen.add(hash)
  250. num_blocks = num_aes_blocks(len(data) - 41)
  251. nonce = bytes_to_long(data[33:41])
  252. for counter in range(nonce, nonce + num_blocks):
  253. self.assert_not_in(counter, used)
  254. used.add(counter)
  255. self.create_test_files()
  256. os.environ['ATTIC_PASSPHRASE'] = 'passphrase'
  257. self.attic('init', '--encryption=' + method, self.repository_location)
  258. verify_uniqueness()
  259. self.attic('create', self.repository_location + '::test', 'input')
  260. verify_uniqueness()
  261. self.attic('create', self.repository_location + '::test.2', 'input')
  262. verify_uniqueness()
  263. self.attic('delete', self.repository_location + '::test.2')
  264. verify_uniqueness()
  265. self.assert_equal(used, set(range(len(used))))
  266. def test_aes_counter_uniqueness_keyfile(self):
  267. self.verify_aes_counter_uniqueness('keyfile')
  268. def test_aes_counter_uniqueness_passphrase(self):
  269. self.verify_aes_counter_uniqueness('passphrase')
  270. class RemoteArchiverTestCase(ArchiverTestCase):
  271. prefix = '__testsuite__:'