setup.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844
  1. # -*- encoding: utf-8 *-*
  2. import os
  3. import io
  4. import re
  5. import sys
  6. from collections import OrderedDict
  7. from datetime import datetime
  8. from glob import glob
  9. from distutils.command.build import build
  10. from distutils.core import Command
  11. import textwrap
  12. import setup_lz4
  13. import setup_zstd
  14. import setup_b2
  15. # True: use the shared liblz4 (>= 1.7.0 / r129) from the system, False: use the bundled lz4 code
  16. prefer_system_liblz4 = True
  17. # True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code
  18. prefer_system_libzstd = True
  19. # True: use the shared libb2 from the system, False: use the bundled blake2 code
  20. prefer_system_libb2 = True
  21. min_python = (3, 4)
  22. my_python = sys.version_info
  23. if my_python < min_python:
  24. print("Borg requires Python %d.%d or later" % min_python)
  25. sys.exit(1)
  26. # Are we building on ReadTheDocs?
  27. on_rtd = os.environ.get('READTHEDOCS')
  28. # msgpack pure python data corruption was fixed in 0.4.6.
  29. # Also, we might use some rather recent API features.
  30. install_requires = ['msgpack-python>=0.4.6', ]
  31. # note for package maintainers: if you package borgbackup for distribution,
  32. # please add llfuse as a *requirement* on all platforms that have a working
  33. # llfuse package. "borg mount" needs llfuse to work.
  34. # if you do not have llfuse, do not require it, most of borgbackup will work.
  35. extras_require = {
  36. # llfuse 0.40 (tested, proven, ok), needs FUSE version >= 2.8.0
  37. # llfuse 0.41 (tested shortly, looks ok), needs FUSE version >= 2.8.0
  38. # llfuse 0.41.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0
  39. # llfuse 0.42 (tested shortly, looks ok), needs FUSE version >= 2.8.0
  40. # llfuse 1.0 (tested shortly, looks ok), needs FUSE version >= 2.8.0
  41. # llfuse 1.1.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0
  42. # llfuse 1.2 (tested shortly, looks ok), needs FUSE version >= 2.8.0
  43. # llfuse 1.3 (tested shortly, looks ok), needs FUSE version >= 2.8.0
  44. # llfuse 2.0 will break API
  45. 'fuse': ['llfuse<2.0', ],
  46. }
  47. if sys.platform.startswith('freebsd'):
  48. # llfuse was frequently broken / did not build on freebsd
  49. # llfuse 0.41.1, 1.1 are ok
  50. extras_require['fuse'] = ['llfuse <2.0, !=0.42.*, !=0.43, !=1.0', ]
  51. from setuptools import setup, find_packages, Extension
  52. from setuptools.command.sdist import sdist
  53. from distutils.command.clean import clean
  54. compress_source = 'src/borg/compress.pyx'
  55. crypto_ll_source = 'src/borg/crypto/low_level.pyx'
  56. chunker_source = 'src/borg/chunker.pyx'
  57. hashindex_source = 'src/borg/hashindex.pyx'
  58. item_source = 'src/borg/item.pyx'
  59. checksums_source = 'src/borg/algorithms/checksums.pyx'
  60. platform_posix_source = 'src/borg/platform/posix.pyx'
  61. platform_linux_source = 'src/borg/platform/linux.pyx'
  62. platform_darwin_source = 'src/borg/platform/darwin.pyx'
  63. platform_freebsd_source = 'src/borg/platform/freebsd.pyx'
  64. cython_sources = [
  65. compress_source,
  66. crypto_ll_source,
  67. chunker_source,
  68. hashindex_source,
  69. item_source,
  70. checksums_source,
  71. platform_posix_source,
  72. platform_linux_source,
  73. platform_freebsd_source,
  74. platform_darwin_source,
  75. ]
  76. try:
  77. from Cython.Distutils import build_ext
  78. import Cython.Compiler.Main as cython_compiler
  79. class Sdist(sdist):
  80. def __init__(self, *args, **kwargs):
  81. for src in cython_sources:
  82. cython_compiler.compile(src, cython_compiler.default_options)
  83. super().__init__(*args, **kwargs)
  84. def make_distribution(self):
  85. self.filelist.extend([
  86. 'src/borg/compress.c',
  87. 'src/borg/crypto/low_level.c',
  88. 'src/borg/chunker.c', 'src/borg/_chunker.c',
  89. 'src/borg/hashindex.c', 'src/borg/_hashindex.c',
  90. 'src/borg/cache_sync/cache_sync.c', 'src/borg/cache_sync/sysdep.h', 'src/borg/cache_sync/unpack.h',
  91. 'src/borg/cache_sync/unpack_define.h', 'src/borg/cache_sync/unpack_template.h',
  92. 'src/borg/item.c',
  93. 'src/borg/algorithms/checksums.c',
  94. 'src/borg/algorithms/crc32_dispatch.c', 'src/borg/algorithms/crc32_clmul.c', 'src/borg/algorithms/crc32_slice_by_8.c',
  95. 'src/borg/algorithms/xxh64/xxhash.h', 'src/borg/algorithms/xxh64/xxhash.c',
  96. 'src/borg/platform/posix.c',
  97. 'src/borg/platform/linux.c',
  98. 'src/borg/platform/freebsd.c',
  99. 'src/borg/platform/darwin.c',
  100. ])
  101. super().make_distribution()
  102. except ImportError:
  103. class Sdist(sdist):
  104. def __init__(self, *args, **kwargs):
  105. raise Exception('Cython is required to run sdist')
  106. compress_source = compress_source.replace('.pyx', '.c')
  107. crypto_ll_source = crypto_ll_source.replace('.pyx', '.c')
  108. chunker_source = chunker_source.replace('.pyx', '.c')
  109. hashindex_source = hashindex_source.replace('.pyx', '.c')
  110. item_source = item_source.replace('.pyx', '.c')
  111. checksums_source = checksums_source.replace('.pyx', '.c')
  112. platform_posix_source = platform_posix_source.replace('.pyx', '.c')
  113. platform_linux_source = platform_linux_source.replace('.pyx', '.c')
  114. platform_freebsd_source = platform_freebsd_source.replace('.pyx', '.c')
  115. platform_darwin_source = platform_darwin_source.replace('.pyx', '.c')
  116. from distutils.command.build_ext import build_ext
  117. if not on_rtd and not all(os.path.exists(path) for path in [
  118. compress_source, crypto_ll_source, chunker_source, hashindex_source, item_source, checksums_source,
  119. platform_posix_source, platform_linux_source, platform_freebsd_source, platform_darwin_source]):
  120. raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')
  121. def detect_openssl(prefixes):
  122. for prefix in prefixes:
  123. filename = os.path.join(prefix, 'include', 'openssl', 'evp.h')
  124. if os.path.exists(filename):
  125. with open(filename, 'r') as fd:
  126. if 'PKCS5_PBKDF2_HMAC(' in fd.read():
  127. return prefix
  128. include_dirs = []
  129. library_dirs = []
  130. define_macros = []
  131. possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl',
  132. '/usr/local/borg', '/opt/local', '/opt/pkg', ]
  133. if os.environ.get('BORG_OPENSSL_PREFIX'):
  134. possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX'))
  135. ssl_prefix = detect_openssl(possible_openssl_prefixes)
  136. if not ssl_prefix:
  137. raise Exception('Unable to find OpenSSL >= 1.0 headers. (Looked here: {})'.format(', '.join(possible_openssl_prefixes)))
  138. include_dirs.append(os.path.join(ssl_prefix, 'include'))
  139. library_dirs.append(os.path.join(ssl_prefix, 'lib'))
  140. possible_liblz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4',
  141. '/usr/local/borg', '/opt/local', '/opt/pkg', ]
  142. if os.environ.get('BORG_LIBLZ4_PREFIX'):
  143. possible_liblz4_prefixes.insert(0, os.environ.get('BORG_LIBLZ4_PREFIX'))
  144. liblz4_prefix = setup_lz4.lz4_system_prefix(possible_liblz4_prefixes)
  145. if prefer_system_liblz4 and liblz4_prefix:
  146. print('Detected and preferring liblz4 over bundled LZ4')
  147. define_macros.append(('BORG_USE_LIBLZ4', 'YES'))
  148. liblz4_system = True
  149. else:
  150. liblz4_system = False
  151. possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/local/libb2',
  152. '/usr/local/borg', '/opt/local', '/opt/pkg', ]
  153. if os.environ.get('BORG_LIBB2_PREFIX'):
  154. possible_libb2_prefixes.insert(0, os.environ.get('BORG_LIBB2_PREFIX'))
  155. libb2_prefix = setup_b2.b2_system_prefix(possible_libb2_prefixes)
  156. if prefer_system_libb2 and libb2_prefix:
  157. print('Detected and preferring libb2 over bundled BLAKE2')
  158. define_macros.append(('BORG_USE_LIBB2', 'YES'))
  159. libb2_system = True
  160. else:
  161. libb2_system = False
  162. possible_libzstd_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libzstd', '/usr/local/libzstd',
  163. '/usr/local/borg', '/opt/local', '/opt/pkg', ]
  164. if os.environ.get('BORG_LIBZSTD_PREFIX'):
  165. possible_libzstd_prefixes.insert(0, os.environ.get('BORG_LIBZSTD_PREFIX'))
  166. libzstd_prefix = setup_zstd.zstd_system_prefix(possible_libzstd_prefixes)
  167. if prefer_system_libzstd and libzstd_prefix:
  168. print('Detected and preferring libzstd over bundled ZSTD')
  169. define_macros.append(('BORG_USE_LIBZSTD', 'YES'))
  170. libzstd_system = True
  171. else:
  172. libzstd_system = False
  173. with open('README.rst', 'r') as fd:
  174. long_description = fd.read()
  175. # remove badges
  176. long_description = re.compile(r'^\.\. start-badges.*^\.\. end-badges', re.M | re.S).sub('', long_description)
  177. # remove |substitutions|
  178. long_description = re.compile(r'\|screencast\|').sub('', long_description)
  179. # remove unknown directives
  180. long_description = re.compile(r'^\.\. highlight:: \w+$', re.M).sub('', long_description)
  181. def format_metavar(option):
  182. if option.nargs in ('*', '...'):
  183. return '[%s...]' % option.metavar
  184. elif option.nargs == '?':
  185. return '[%s]' % option.metavar
  186. elif option.nargs is None:
  187. return option.metavar
  188. else:
  189. raise ValueError('Can\'t format metavar %s, unknown nargs %s!' % (option.metavar, option.nargs))
  190. class build_usage(Command):
  191. description = "generate usage for each command"
  192. user_options = [
  193. ('output=', 'O', 'output directory'),
  194. ]
  195. def initialize_options(self):
  196. pass
  197. def finalize_options(self):
  198. pass
  199. def run(self):
  200. print('generating usage docs')
  201. import borg
  202. borg.doc_mode = 'build_man'
  203. if not os.path.exists('docs/usage'):
  204. os.mkdir('docs/usage')
  205. # allows us to build docs without the C modules fully loaded during help generation
  206. from borg.archiver import Archiver
  207. parser = Archiver(prog='borg').build_parser()
  208. # borgfs has a separate man page to satisfy debian's "every program from a package
  209. # must have a man page" requirement, but it doesn't need a separate HTML docs page
  210. #borgfs_parser = Archiver(prog='borgfs').build_parser()
  211. self.generate_level("", parser, Archiver)
  212. def generate_level(self, prefix, parser, Archiver, extra_choices=None):
  213. is_subcommand = False
  214. choices = {}
  215. for action in parser._actions:
  216. if action.choices is not None and 'SubParsersAction' in str(action.__class__):
  217. is_subcommand = True
  218. for cmd, parser in action.choices.items():
  219. choices[prefix + cmd] = parser
  220. if extra_choices is not None:
  221. choices.update(extra_choices)
  222. if prefix and not choices:
  223. return
  224. print('found commands: %s' % list(choices.keys()))
  225. for command, parser in sorted(choices.items()):
  226. if command.startswith('debug'):
  227. print('skipping', command)
  228. continue
  229. print('generating help for %s' % command)
  230. if self.generate_level(command + " ", parser, Archiver):
  231. continue
  232. with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc:
  233. doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n")
  234. if command == 'help':
  235. for topic in Archiver.helptext:
  236. params = {"topic": topic,
  237. "underline": '~' * len('borg help ' + topic)}
  238. doc.write(".. _borg_{topic}:\n\n".format(**params))
  239. doc.write("borg help {topic}\n{underline}\n\n".format(**params))
  240. doc.write(Archiver.helptext[topic])
  241. else:
  242. params = {"command": command,
  243. "command_": command.replace(' ', '_'),
  244. "underline": '-' * len('borg ' + command)}
  245. doc.write(".. _borg_{command_}:\n\n".format(**params))
  246. doc.write("borg {command}\n{underline}\n.. code-block:: none\n\n borg [common options] {command}".format(**params))
  247. self.write_usage(parser, doc)
  248. epilog = parser.epilog
  249. parser.epilog = None
  250. self.write_options(parser, doc)
  251. doc.write("\n\nDescription\n~~~~~~~~~~~\n")
  252. doc.write(epilog)
  253. if 'create' in choices:
  254. common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0]
  255. with open('docs/usage/common-options.rst.inc', 'w') as doc:
  256. self.write_options_group(common_options, doc, False, base_indent=0)
  257. return is_subcommand
  258. def write_usage(self, parser, fp):
  259. if any(len(o.option_strings) for o in parser._actions):
  260. fp.write(' [options]')
  261. for option in parser._actions:
  262. if option.option_strings:
  263. continue
  264. fp.write(' ' + format_metavar(option))
  265. fp.write('\n\n')
  266. def write_options(self, parser, fp):
  267. def is_positional_group(group):
  268. return any(not o.option_strings for o in group._group_actions)
  269. # HTML output:
  270. # A table using some column-spans
  271. def html_write(s):
  272. for line in s.splitlines():
  273. fp.write(' ' + line + '\n')
  274. rows = []
  275. for group in parser._action_groups:
  276. if group.title == 'Common options':
  277. # (no of columns used, columns, ...)
  278. rows.append((1, '.. class:: borg-common-opt-ref\n\n:ref:`common_options`'))
  279. else:
  280. if not group._group_actions:
  281. continue
  282. group_header = '**%s**' % group.title
  283. if group.description:
  284. group_header += ' — ' + group.description
  285. rows.append((1, group_header))
  286. if is_positional_group(group):
  287. for option in group._group_actions:
  288. rows.append((3, '', '``%s``' % option.metavar, option.help or ''))
  289. else:
  290. for option in group._group_actions:
  291. if option.metavar:
  292. option_fmt = '``%s ' + option.metavar + '``'
  293. else:
  294. option_fmt = '``%s``'
  295. option_str = ', '.join(option_fmt % s for s in option.option_strings)
  296. option_desc = textwrap.dedent((option.help or '') % option.__dict__)
  297. rows.append((3, '', option_str, option_desc))
  298. fp.write('.. only:: html\n\n')
  299. table = io.StringIO()
  300. table.write('.. class:: borg-options-table\n\n')
  301. self.rows_to_table(rows, table.write)
  302. fp.write(textwrap.indent(table.getvalue(), ' ' * 4))
  303. # LaTeX output:
  304. # Regular rST option lists (irregular column widths)
  305. latex_options = io.StringIO()
  306. for group in parser._action_groups:
  307. if group.title == 'Common options':
  308. latex_options.write('\n\n:ref:`common_options`\n')
  309. latex_options.write(' |')
  310. else:
  311. self.write_options_group(group, latex_options)
  312. fp.write('\n.. only:: latex\n\n')
  313. fp.write(textwrap.indent(latex_options.getvalue(), ' ' * 4))
  314. def rows_to_table(self, rows, write):
  315. def write_row_separator():
  316. write('+')
  317. for column_width in column_widths:
  318. write('-' * (column_width + 1))
  319. write('+')
  320. write('\n')
  321. # Find column count and width
  322. column_count = max(columns for columns, *_ in rows)
  323. column_widths = [0] * column_count
  324. for columns, *cells in rows:
  325. for i in range(columns):
  326. # "+ 1" because we want a space between the cell contents and the delimiting "|" in the output
  327. column_widths[i] = max(column_widths[i], len(cells[i]) + 1)
  328. for columns, *original_cells in rows:
  329. write_row_separator()
  330. # If a cell contains newlines, then the row must be split up in individual rows
  331. # where each cell contains no newline.
  332. rowspanning_cells = []
  333. original_cells = list(original_cells)
  334. while any('\n' in cell for cell in original_cells):
  335. cell_bloc = []
  336. for i, cell in enumerate(original_cells):
  337. pre, _, original_cells[i] = cell.partition('\n')
  338. cell_bloc.append(pre)
  339. rowspanning_cells.append(cell_bloc)
  340. rowspanning_cells.append(original_cells)
  341. for cells in rowspanning_cells:
  342. for i, column_width in enumerate(column_widths):
  343. if i < columns:
  344. write('| ')
  345. write(cells[i].ljust(column_width))
  346. else:
  347. write(' ')
  348. write(''.ljust(column_width))
  349. write('|\n')
  350. write_row_separator()
  351. # This bit of JavaScript kills the <colgroup> that is invariably inserted by docutils,
  352. # but does absolutely no good here. It sets bogus column widths which cannot be overridden
  353. # with CSS alone.
  354. # Since this is HTML-only output, it would be possible to just generate a <table> directly,
  355. # but then we'd lose rST formatting.
  356. write(textwrap.dedent("""
  357. .. raw:: html
  358. <script type='text/javascript'>
  359. $(document).ready(function () {
  360. $('.borg-options-table colgroup').remove();
  361. })
  362. </script>
  363. """))
  364. def write_options_group(self, group, fp, with_title=True, base_indent=4):
  365. def is_positional_group(group):
  366. return any(not o.option_strings for o in group._group_actions)
  367. indent = ' ' * base_indent
  368. if is_positional_group(group):
  369. for option in group._group_actions:
  370. fp.write(option.metavar + '\n')
  371. fp.write(textwrap.indent(option.help or '', ' ' * base_indent) + '\n')
  372. return
  373. if not group._group_actions:
  374. return
  375. if with_title:
  376. fp.write('\n\n')
  377. fp.write(group.title + '\n')
  378. opts = OrderedDict()
  379. for option in group._group_actions:
  380. if option.metavar:
  381. option_fmt = '%s ' + option.metavar
  382. else:
  383. option_fmt = '%s'
  384. option_str = ', '.join(option_fmt % s for s in option.option_strings)
  385. option_desc = textwrap.dedent((option.help or '') % option.__dict__)
  386. opts[option_str] = textwrap.indent(option_desc, ' ' * 4)
  387. padding = len(max(opts)) + 1
  388. for option, desc in opts.items():
  389. fp.write(indent + option.ljust(padding) + desc + '\n')
  390. class build_man(Command):
  391. description = 'build man pages'
  392. user_options = []
  393. see_also = {
  394. 'create': ('delete', 'prune', 'check', 'patterns', 'placeholders', 'compression'),
  395. 'recreate': ('patterns', 'placeholders', 'compression'),
  396. 'list': ('info', 'diff', 'prune', 'patterns'),
  397. 'info': ('list', 'diff'),
  398. 'init': ('create', 'delete', 'check', 'list', 'key-import', 'key-export', 'key-change-passphrase'),
  399. 'key-import': ('key-export', ),
  400. 'key-export': ('key-import', ),
  401. 'mount': ('umount', 'extract'), # Would be cooler if these two were on the same page
  402. 'umount': ('mount', ),
  403. 'extract': ('mount', ),
  404. }
  405. rst_prelude = textwrap.dedent("""
  406. .. role:: ref(title)
  407. .. |project_name| replace:: Borg
  408. """)
  409. usage_group = {
  410. 'break-lock': 'lock',
  411. 'with-lock': 'lock',
  412. 'change-passphrase': 'key',
  413. 'key_change-passphrase': 'key',
  414. 'key_export': 'key',
  415. 'key_import': 'key',
  416. 'key_migrate-to-repokey': 'key',
  417. 'export-tar': 'tar',
  418. 'benchmark_crud': 'benchmark',
  419. 'umount': 'mount',
  420. }
  421. def initialize_options(self):
  422. pass
  423. def finalize_options(self):
  424. pass
  425. def run(self):
  426. print('building man pages (in docs/man)', file=sys.stderr)
  427. import borg
  428. borg.doc_mode = 'build_man'
  429. os.makedirs('docs/man', exist_ok=True)
  430. # allows us to build docs without the C modules fully loaded during help generation
  431. from borg.archiver import Archiver
  432. parser = Archiver(prog='borg').build_parser()
  433. borgfs_parser = Archiver(prog='borgfs').build_parser()
  434. self.generate_level('', parser, Archiver, {'borgfs': borgfs_parser})
  435. self.build_topic_pages(Archiver)
  436. self.build_intro_page()
  437. def generate_level(self, prefix, parser, Archiver, extra_choices=None):
  438. is_subcommand = False
  439. choices = {}
  440. for action in parser._actions:
  441. if action.choices is not None and 'SubParsersAction' in str(action.__class__):
  442. is_subcommand = True
  443. for cmd, parser in action.choices.items():
  444. choices[prefix + cmd] = parser
  445. if extra_choices is not None:
  446. choices.update(extra_choices)
  447. if prefix and not choices:
  448. return
  449. for command, parser in sorted(choices.items()):
  450. if command.startswith('debug') or command == 'help':
  451. continue
  452. if command == "borgfs":
  453. man_title = command
  454. else:
  455. man_title = 'borg-' + command.replace(' ', '-')
  456. print('building man page', man_title + '(1)', file=sys.stderr)
  457. is_intermediary = self.generate_level(command + ' ', parser, Archiver)
  458. doc, write = self.new_doc()
  459. self.write_man_header(write, man_title, parser.description)
  460. self.write_heading(write, 'SYNOPSIS')
  461. if is_intermediary:
  462. subparsers = [action for action in parser._actions if 'SubParsersAction' in str(action.__class__)][0]
  463. for subcommand in subparsers.choices:
  464. write('| borg', '[common options]', command, subcommand, '...')
  465. self.see_also.setdefault(command, []).append('%s-%s' % (command, subcommand))
  466. else:
  467. if command == "borgfs":
  468. write(command, end='')
  469. else:
  470. write('borg', '[common options]', command, end='')
  471. self.write_usage(write, parser)
  472. write('\n')
  473. description, _, notes = parser.epilog.partition('\n.. man NOTES')
  474. if description:
  475. self.write_heading(write, 'DESCRIPTION')
  476. write(description)
  477. if not is_intermediary:
  478. self.write_heading(write, 'OPTIONS')
  479. write('See `borg-common(1)` for common options of Borg commands.')
  480. write()
  481. self.write_options(write, parser)
  482. self.write_examples(write, command)
  483. if notes:
  484. self.write_heading(write, 'NOTES')
  485. write(notes)
  486. self.write_see_also(write, man_title)
  487. self.gen_man_page(man_title, doc.getvalue())
  488. # Generate the borg-common(1) man page with the common options.
  489. if 'create' in choices:
  490. doc, write = self.new_doc()
  491. man_title = 'borg-common'
  492. self.write_man_header(write, man_title, 'Common options of Borg commands')
  493. common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0]
  494. self.write_heading(write, 'SYNOPSIS')
  495. self.write_options_group(write, common_options)
  496. self.write_see_also(write, man_title)
  497. self.gen_man_page(man_title, doc.getvalue())
  498. return is_subcommand
  499. def build_topic_pages(self, Archiver):
  500. for topic, text in Archiver.helptext.items():
  501. doc, write = self.new_doc()
  502. man_title = 'borg-' + topic
  503. print('building man page', man_title + '(1)', file=sys.stderr)
  504. self.write_man_header(write, man_title, 'Details regarding ' + topic)
  505. self.write_heading(write, 'DESCRIPTION')
  506. write(text)
  507. self.gen_man_page(man_title, doc.getvalue())
  508. def build_intro_page(self):
  509. print('building man page borg(1)', file=sys.stderr)
  510. with open('docs/man_intro.rst') as fd:
  511. man_intro = fd.read()
  512. self.gen_man_page('borg', self.rst_prelude + man_intro)
  513. def new_doc(self):
  514. doc = io.StringIO(self.rst_prelude)
  515. doc.read()
  516. write = self.printer(doc)
  517. return doc, write
  518. def printer(self, fd):
  519. def write(*args, **kwargs):
  520. print(*args, file=fd, **kwargs)
  521. return write
  522. def write_heading(self, write, header, char='-', double_sided=False):
  523. write()
  524. if double_sided:
  525. write(char * len(header))
  526. write(header)
  527. write(char * len(header))
  528. write()
  529. def write_man_header(self, write, title, description):
  530. self.write_heading(write, title, '=', double_sided=True)
  531. self.write_heading(write, description, double_sided=True)
  532. # man page metadata
  533. write(':Author: The Borg Collective')
  534. write(':Date:', datetime.utcnow().date().isoformat())
  535. write(':Manual section: 1')
  536. write(':Manual group: borg backup tool')
  537. write()
  538. def write_examples(self, write, command):
  539. command = command.replace(' ', '_')
  540. with open('docs/usage/%s.rst' % self.usage_group.get(command, command)) as fd:
  541. usage = fd.read()
  542. usage_include = '.. include:: %s.rst.inc' % command
  543. begin = usage.find(usage_include)
  544. end = usage.find('.. include', begin + 1)
  545. # If a command has a dedicated anchor, it will occur before the command's include.
  546. if 0 < usage.find('.. _', begin + 1) < end:
  547. end = usage.find('.. _', begin + 1)
  548. examples = usage[begin:end]
  549. examples = examples.replace(usage_include, '')
  550. examples = examples.replace('Examples\n~~~~~~~~', '')
  551. examples = examples.replace('Miscellaneous Help\n------------------', '')
  552. examples = examples.replace('``docs/misc/prune-example.txt``:', '``docs/misc/prune-example.txt``.')
  553. examples = examples.replace('.. highlight:: none\n', '') # we don't support highlight
  554. examples = re.sub('^(~+)$', lambda matches: '+' * len(matches.group(0)), examples, flags=re.MULTILINE)
  555. examples = examples.strip()
  556. if examples:
  557. self.write_heading(write, 'EXAMPLES', '-')
  558. write(examples)
  559. def write_see_also(self, write, man_title):
  560. see_also = self.see_also.get(man_title.replace('borg-', ''), ())
  561. see_also = ['`borg-%s(1)`' % s for s in see_also]
  562. see_also.insert(0, '`borg-common(1)`')
  563. self.write_heading(write, 'SEE ALSO')
  564. write(', '.join(see_also))
  565. def gen_man_page(self, name, rst):
  566. from docutils.writers import manpage
  567. from docutils.core import publish_string
  568. from docutils.nodes import inline
  569. from docutils.parsers.rst import roles
  570. def issue(name, rawtext, text, lineno, inliner, options={}, content=[]):
  571. return [inline(rawtext, '#' + text)], []
  572. roles.register_local_role('issue', issue)
  573. # We give the source_path so that docutils can find relative includes
  574. # as-if the document where located in the docs/ directory.
  575. man_page = publish_string(source=rst, source_path='docs/%s.rst' % name, writer=manpage.Writer())
  576. with open('docs/man/%s.1' % name, 'wb') as fd:
  577. fd.write(man_page)
  578. def write_usage(self, write, parser):
  579. if any(len(o.option_strings) for o in parser._actions):
  580. write(' [options] ', end='')
  581. for option in parser._actions:
  582. if option.option_strings:
  583. continue
  584. write(format_metavar(option), end=' ')
  585. def write_options(self, write, parser):
  586. for group in parser._action_groups:
  587. if group.title == 'Common options' or not group._group_actions:
  588. continue
  589. title = 'arguments' if group.title == 'positional arguments' else group.title
  590. self.write_heading(write, title, '+')
  591. self.write_options_group(write, group)
  592. def write_options_group(self, write, group):
  593. def is_positional_group(group):
  594. return any(not o.option_strings for o in group._group_actions)
  595. if is_positional_group(group):
  596. for option in group._group_actions:
  597. write(option.metavar)
  598. write(textwrap.indent(option.help or '', ' ' * 4))
  599. return
  600. opts = OrderedDict()
  601. for option in group._group_actions:
  602. if option.metavar:
  603. option_fmt = '%s ' + option.metavar
  604. else:
  605. option_fmt = '%s'
  606. option_str = ', '.join(option_fmt % s for s in option.option_strings)
  607. option_desc = textwrap.dedent((option.help or '') % option.__dict__)
  608. opts[option_str] = textwrap.indent(option_desc, ' ' * 4)
  609. padding = len(max(opts)) + 1
  610. for option, desc in opts.items():
  611. write(option.ljust(padding), desc)
  612. def rm(file):
  613. try:
  614. os.unlink(file)
  615. print('rm', file)
  616. except FileNotFoundError:
  617. pass
  618. class Clean(clean):
  619. def run(self):
  620. super().run()
  621. for source in cython_sources:
  622. genc = source.replace('.pyx', '.c')
  623. rm(genc)
  624. compiled_glob = source.replace('.pyx', '.cpython*')
  625. for compiled in sorted(glob(compiled_glob)):
  626. rm(compiled)
  627. cmdclass = {
  628. 'build_ext': build_ext,
  629. 'build_usage': build_usage,
  630. 'build_man': build_man,
  631. 'sdist': Sdist,
  632. 'clean': Clean,
  633. }
  634. ext_modules = []
  635. if not on_rtd:
  636. compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs,
  637. define_macros=define_macros)
  638. compress_ext_kwargs = setup_lz4.lz4_ext_kwargs(bundled_path='src/borg/algorithms/lz4',
  639. system_prefix=liblz4_prefix, system=liblz4_system,
  640. **compress_ext_kwargs)
  641. compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd',
  642. system_prefix=libzstd_prefix, system=libzstd_system,
  643. multithreaded=False, legacy=False, **compress_ext_kwargs)
  644. crypto_ext_kwargs = dict(sources=[crypto_ll_source], libraries=['crypto'],
  645. include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros)
  646. crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2',
  647. system_prefix=libb2_prefix, system=libb2_system,
  648. **crypto_ext_kwargs)
  649. ext_modules += [
  650. Extension('borg.compress', **compress_ext_kwargs),
  651. Extension('borg.crypto.low_level', **crypto_ext_kwargs),
  652. Extension('borg.hashindex', [hashindex_source]),
  653. Extension('borg.item', [item_source]),
  654. Extension('borg.chunker', [chunker_source]),
  655. Extension('borg.algorithms.checksums', [checksums_source]),
  656. ]
  657. if not sys.platform.startswith(('win32', )):
  658. ext_modules.append(Extension('borg.platform.posix', [platform_posix_source]))
  659. if sys.platform == 'linux':
  660. ext_modules.append(Extension('borg.platform.linux', [platform_linux_source], libraries=['acl']))
  661. elif sys.platform.startswith('freebsd'):
  662. ext_modules.append(Extension('borg.platform.freebsd', [platform_freebsd_source]))
  663. elif sys.platform == 'darwin':
  664. ext_modules.append(Extension('borg.platform.darwin', [platform_darwin_source]))
  665. setup(
  666. name='borgbackup',
  667. use_scm_version={
  668. 'write_to': 'src/borg/_version.py',
  669. },
  670. author='The Borg Collective (see AUTHORS file)',
  671. author_email='borgbackup@python.org',
  672. url='https://borgbackup.readthedocs.io/',
  673. description='Deduplicated, encrypted, authenticated and compressed backups',
  674. long_description=long_description,
  675. license='BSD',
  676. platforms=['Linux', 'MacOS X', 'FreeBSD', 'OpenBSD', 'NetBSD', ],
  677. classifiers=[
  678. 'Development Status :: 4 - Beta',
  679. 'Environment :: Console',
  680. 'Intended Audience :: System Administrators',
  681. 'License :: OSI Approved :: BSD License',
  682. 'Operating System :: POSIX :: BSD :: FreeBSD',
  683. 'Operating System :: POSIX :: BSD :: OpenBSD',
  684. 'Operating System :: POSIX :: BSD :: NetBSD',
  685. 'Operating System :: MacOS :: MacOS X',
  686. 'Operating System :: POSIX :: Linux',
  687. 'Programming Language :: Python',
  688. 'Programming Language :: Python :: 3',
  689. 'Programming Language :: Python :: 3.4',
  690. 'Programming Language :: Python :: 3.5',
  691. 'Programming Language :: Python :: 3.6',
  692. 'Topic :: Security :: Cryptography',
  693. 'Topic :: System :: Archiving :: Backup',
  694. ],
  695. packages=find_packages('src'),
  696. package_dir={'': 'src'},
  697. zip_safe=False,
  698. entry_points={
  699. 'console_scripts': [
  700. 'borg = borg.archiver:main',
  701. 'borgfs = borg.archiver:main',
  702. ]
  703. },
  704. # See also the MANIFEST.in file.
  705. # We want to install all the files in the package directories...
  706. include_package_data=True,
  707. # ...except the source files which have been compiled (C extensions):
  708. exclude_package_data={
  709. '': ['*.c', '*.h', '*.pyx', ],
  710. },
  711. cmdclass=cmdclass,
  712. ext_modules=ext_modules,
  713. setup_requires=['setuptools_scm>=1.7'],
  714. install_requires=install_requires,
  715. extras_require=extras_require,
  716. )