Vagrantfile 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395
  1. # -*- mode: ruby -*-
  2. # vi: set ft=ruby :
  3. # Automated creation of testing environments / binaries on misc. platforms
  4. $cpus = Integer(ENV.fetch('VMCPUS', '4')) # create VMs with that many cpus
  5. $xdistn = Integer(ENV.fetch('XDISTN', '4')) # dispatch tests to that many pytest workers
  6. $wmem = $xdistn * 256 # give the VM additional memory for workers [MB]
  7. def packages_debianoid(user)
  8. return <<-EOF
  9. export DEBIAN_FRONTEND=noninteractive
  10. # this is to avoid grub asking about which device it should install to:
  11. echo "set grub-pc/install_devices /dev/sda" | debconf-communicate
  12. apt-get -y -qq update
  13. apt-get -y -qq dist-upgrade
  14. # for building borgbackup and dependencies:
  15. apt install -y libssl-dev libacl1-dev liblz4-dev libzstd-dev libfuse-dev fuse pkg-config
  16. usermod -a -G fuse #{user}
  17. chgrp fuse /dev/fuse
  18. chmod 666 /dev/fuse
  19. apt install -y fakeroot build-essential git curl
  20. apt install -y python3-dev python3-setuptools virtualenv
  21. # for building python:
  22. apt install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev libffi-dev
  23. EOF
  24. end
  25. def packages_arch
  26. return <<-EOF
  27. echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
  28. locale-gen
  29. localectl set-locale LANG=en_US.UTF-8
  30. chown vagrant.vagrant /vagrant
  31. pacman --sync --noconfirm --refresh python-virtualenv python-pip
  32. EOF
  33. end
  34. def packages_freebsd
  35. return <<-EOF
  36. # in case the VM has no hostname set
  37. hostname freebsd
  38. # install all the (security and other) updates, base system
  39. freebsd-update --not-running-from-cron fetch install
  40. # for building borgbackup and dependencies:
  41. pkg install -y liblz4 zstd fusefs-libs pkgconf
  42. pkg install -y git bash # fakeroot causes lots of troubles on freebsd
  43. # for building python:
  44. pkg install -y python37 py37-sqlite3 py37-virtualenv py37-pip
  45. # make sure there is a python3 command
  46. ln -sf /usr/local/bin/python3.7 /usr/local/bin/python3
  47. # make bash default / work:
  48. chsh -s bash vagrant
  49. mount -t fdescfs fdesc /dev/fd
  50. echo 'fdesc /dev/fd fdescfs rw 0 0' >> /etc/fstab
  51. # make FUSE work
  52. echo 'fuse_load="YES"' >> /boot/loader.conf
  53. echo 'vfs.usermount=1' >> /etc/sysctl.conf
  54. kldload fuse
  55. sysctl vfs.usermount=1
  56. pw groupmod operator -M vagrant
  57. # /dev/fuse has group operator
  58. chmod 666 /dev/fuse
  59. # install all the (security and other) updates, packages
  60. pkg update
  61. yes | pkg upgrade
  62. echo 'export BORG_OPENSSL_PREFIX=/usr' >> ~vagrant/.bash_profile
  63. EOF
  64. end
  65. def packages_openbsd
  66. return <<-EOF
  67. pkg_add bash
  68. chsh -s /usr/local/bin/bash vagrant
  69. pkg_add lz4
  70. pkg_add zstd
  71. pkg_add git # no fakeroot
  72. pkg_add py3-pip
  73. pkg_add py3-virtualenv
  74. ln -sf /usr/local/bin/virtualenv-3 /usr/local/bin/virtualenv
  75. EOF
  76. end
  77. def packages_darwin
  78. return <<-EOF
  79. # install all the (security and other) updates
  80. sudo softwareupdate --ignore iTunesX
  81. sudo softwareupdate --ignore iTunes
  82. sudo softwareupdate --ignore Safari
  83. sudo softwareupdate --ignore "Install macOS High Sierra"
  84. sudo softwareupdate --install --all
  85. which brew || CI=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
  86. brew update > /dev/null
  87. brew install pkg-config readline openssl@1.1 zstd lz4 xz fakeroot git
  88. brew tap homebrew/cask
  89. brew cask install osxfuse
  90. brew upgrade # upgrade everything
  91. echo 'export PKG_CONFIG_PATH=/usr/local/opt/openssl@1.1/lib/pkgconfig' >> ~vagrant/.bash_profile
  92. EOF
  93. end
  94. def packages_openindiana
  95. return <<-EOF
  96. # needs separate provisioning step + reboot:
  97. #pkg update
  98. pkg install python-37 clang-40 lz4 zstd git
  99. ln -sf /usr/bin/python3.7 /usr/bin/python3
  100. python3 -m ensurepip
  101. ln -sf /usr/bin/pip3.7 /usr/bin/pip3
  102. pip3 install virtualenv
  103. EOF
  104. end
  105. def install_pyenv(boxname)
  106. return <<-EOF
  107. curl -s -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash
  108. echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bash_profile
  109. echo 'eval "$(pyenv init -)"' >> ~/.bash_profile
  110. echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile
  111. echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile
  112. EOF
  113. end
  114. def fix_pyenv_darwin(boxname)
  115. return <<-EOF
  116. echo 'export PYTHON_CONFIGURE_OPTS="--enable-framework"' >> ~/.bash_profile
  117. EOF
  118. end
  119. def install_pythons(boxname)
  120. return <<-EOF
  121. . ~/.bash_profile
  122. pyenv install 3.8.0 # tests, version supporting openssl 1.1
  123. pyenv install 3.7.0 # tests, version supporting openssl 1.1
  124. pyenv install 3.6.9 # binary build, tests, version supporting openssl 1.1
  125. pyenv rehash
  126. EOF
  127. end
  128. def build_sys_venv(boxname)
  129. return <<-EOF
  130. . ~/.bash_profile
  131. cd /vagrant/borg
  132. virtualenv --python=python3 borg-env
  133. EOF
  134. end
  135. def build_pyenv_venv(boxname)
  136. return <<-EOF
  137. . ~/.bash_profile
  138. cd /vagrant/borg
  139. # use the latest 3.6 release
  140. pyenv global 3.6.9
  141. pyenv virtualenv 3.6.9 borg-env
  142. ln -s ~/.pyenv/versions/borg-env .
  143. EOF
  144. end
  145. def install_borg(fuse)
  146. script = <<-EOF
  147. . ~/.bash_profile
  148. cd /vagrant/borg
  149. . borg-env/bin/activate
  150. pip install -U wheel # upgrade wheel, too old for 3.5
  151. cd borg
  152. pip install -r requirements.d/development.txt
  153. python setup.py clean
  154. EOF
  155. if fuse
  156. script += <<-EOF
  157. # by using [fuse], setup.py can handle different FUSE requirements:
  158. pip install -e .[fuse]
  159. EOF
  160. else
  161. script += <<-EOF
  162. pip install -e .
  163. # do not install llfuse into the virtualenvs built by tox:
  164. sed -i.bak '/fuse.txt/d' tox.ini
  165. EOF
  166. end
  167. return script
  168. end
  169. def install_pyinstaller()
  170. return <<-EOF
  171. . ~/.bash_profile
  172. cd /vagrant/borg
  173. . borg-env/bin/activate
  174. git clone https://github.com/thomaswaldmann/pyinstaller.git
  175. cd pyinstaller
  176. git checkout v3.5-maint
  177. python setup.py install
  178. EOF
  179. end
  180. def build_binary_with_pyinstaller(boxname)
  181. return <<-EOF
  182. . ~/.bash_profile
  183. cd /vagrant/borg
  184. . borg-env/bin/activate
  185. cd borg
  186. pyinstaller --clean --distpath=/vagrant/borg scripts/borg.exe.spec
  187. echo 'export PATH="/vagrant/borg:$PATH"' >> ~/.bash_profile
  188. cd .. && tar -czvf borg.tgz borg-dir
  189. EOF
  190. end
  191. def run_tests(boxname)
  192. return <<-EOF
  193. . ~/.bash_profile
  194. cd /vagrant/borg/borg
  195. . ../borg-env/bin/activate
  196. if which pyenv 2> /dev/null; then
  197. # for testing, use the earliest point releases of the supported python versions:
  198. pyenv global 3.6.9 3.7.0 3.8.0
  199. pyenv local 3.6.9 3.7.0 3.8.0
  200. fi
  201. # otherwise: just use the system python
  202. if which fakeroot 2> /dev/null; then
  203. echo "Running tox WITH fakeroot -u"
  204. fakeroot -u tox --skip-missing-interpreters -e py36,py37,py38
  205. else
  206. echo "Running tox WITHOUT fakeroot -u"
  207. tox --skip-missing-interpreters -e py36,py37,py38
  208. fi
  209. EOF
  210. end
  211. def fs_init(user)
  212. return <<-EOF
  213. # clean up (wrong/outdated) stuff we likely got via rsync:
  214. rm -rf /vagrant/borg/borg/.tox 2> /dev/null
  215. rm -rf /vagrant/borg/borg/borgbackup.egg-info 2> /dev/null
  216. rm -rf /vagrant/borg/borg/__pycache__ 2> /dev/null
  217. find /vagrant/borg/borg/src -name '__pycache__' -exec rm -rf {} \\; 2> /dev/null
  218. chown -R #{user} /vagrant/borg
  219. touch ~#{user}/.bash_profile ; chown #{user} ~#{user}/.bash_profile
  220. echo 'export LANG=en_US.UTF-8' >> ~#{user}/.bash_profile
  221. echo 'export LC_CTYPE=en_US.UTF-8' >> ~#{user}/.bash_profile
  222. echo 'export XDISTN=#{$xdistn}' >> ~#{user}/.bash_profile
  223. EOF
  224. end
  225. Vagrant.configure(2) do |config|
  226. # use rsync to copy content to the folder
  227. config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "--exclude", ".python-version"], :rsync__chown => false
  228. # do not let the VM access . on the host machine via the default shared folder!
  229. config.vm.synced_folder ".", "/vagrant", disabled: true
  230. config.vm.provider :virtualbox do |v|
  231. #v.gui = true
  232. v.cpus = $cpus
  233. end
  234. config.vm.define "focal64" do |b|
  235. b.vm.box = "ubuntu/focal64"
  236. b.vm.provider :virtualbox do |v|
  237. v.memory = 1024 + $wmem
  238. end
  239. b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
  240. b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
  241. b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("focal64")
  242. b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
  243. b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("focal64")
  244. end
  245. config.vm.define "bionic64" do |b|
  246. b.vm.box = "ubuntu/bionic64"
  247. b.vm.provider :virtualbox do |v|
  248. v.memory = 1024 + $wmem
  249. end
  250. b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
  251. b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
  252. b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("bionic64")
  253. b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
  254. b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("bionic64")
  255. end
  256. config.vm.define "buster64" do |b|
  257. b.vm.box = "debian/buster64"
  258. b.vm.provider :virtualbox do |v|
  259. v.memory = 1024 + $wmem
  260. end
  261. b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
  262. b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
  263. b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("buster64")
  264. b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("buster64")
  265. b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("buster64")
  266. b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
  267. b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
  268. b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("buster64")
  269. b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("buster64")
  270. end
  271. config.vm.define "stretch64" do |b|
  272. b.vm.box = "debian/stretch64"
  273. b.vm.provider :virtualbox do |v|
  274. v.memory = 1024 + $wmem
  275. end
  276. b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
  277. b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
  278. b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("stretch64")
  279. b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("stretch64")
  280. b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("stretch64")
  281. b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
  282. b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
  283. b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("stretch64")
  284. b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("stretch64")
  285. end
  286. config.vm.define "arch64" do |b|
  287. b.vm.box = "terrywang/archlinux"
  288. b.vm.provider :virtualbox do |v|
  289. v.memory = 1024 + $wmem
  290. end
  291. b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
  292. b.vm.provision "packages arch", :type => :shell, :privileged => true, :inline => packages_arch
  293. b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("arch64")
  294. b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
  295. b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("arch64")
  296. end
  297. config.vm.define "freebsd64" do |b|
  298. b.vm.box = "freebsd121-64"
  299. b.vm.provider :virtualbox do |v|
  300. v.memory = 1024 + $wmem
  301. end
  302. b.ssh.shell = "sh"
  303. b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
  304. b.vm.provision "packages freebsd", :type => :shell, :inline => packages_freebsd
  305. b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("freebsd64")
  306. b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("freebsd64")
  307. b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("freebsd64")
  308. b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
  309. b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
  310. b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd64")
  311. b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd64")
  312. end
  313. config.vm.define "openbsd64" do |b|
  314. b.vm.box = "openbsd64-64"
  315. b.vm.provider :virtualbox do |v|
  316. v.memory = 1024 + $wmem
  317. end
  318. b.ssh.shell = "sh"
  319. b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
  320. b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd
  321. b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64")
  322. b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false)
  323. b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd64")
  324. end
  325. config.vm.define "darwin64" do |b|
  326. b.vm.box = "macos-sierra"
  327. b.vm.provider :virtualbox do |v|
  328. v.memory = 2048 + $wmem
  329. v.customize ['modifyvm', :id, '--ostype', 'MacOS_64']
  330. v.customize ['modifyvm', :id, '--paravirtprovider', 'default']
  331. v.customize ['modifyvm', :id, '--nested-hw-virt', 'on']
  332. # Adjust CPU settings according to
  333. # https://github.com/geerlingguy/macos-virtualbox-vm
  334. v.customize ['modifyvm', :id, '--cpuidset',
  335. '00000001', '000306a9', '00020800', '80000201', '178bfbff']
  336. # Disable USB variant requiring Virtualbox proprietary extension pack
  337. v.customize ["modifyvm", :id, '--usbehci', 'off', '--usbxhci', 'off']
  338. end
  339. b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
  340. b.vm.provision "packages darwin", :type => :shell, :privileged => false, :inline => packages_darwin
  341. b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("darwin64")
  342. b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_darwin("darwin64")
  343. b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("darwin64")
  344. b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("darwin64")
  345. b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
  346. b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
  347. b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("darwin64")
  348. b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("darwin64")
  349. end
  350. # rsync on openindiana has troubles, does not set correct owner for /vagrant/borg and thus gives lots of
  351. # permission errors. can be manually fixed in the VM by: sudo chown -R vagrant /vagrant/borg ; then rsync again.
  352. config.vm.define "openindiana64" do |b|
  353. b.vm.box = "openindiana"
  354. b.vm.provider :virtualbox do |v|
  355. v.memory = 1536 + $wmem
  356. end
  357. b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
  358. b.vm.provision "packages openindiana", :type => :shell, :inline => packages_openindiana
  359. b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openindiana64")
  360. b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false)
  361. b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openindiana64")
  362. end
  363. # TODO: create more VMs with python 3.6+ and openssl 1.1.
  364. # See branch 1.1-maint for a better equipped Vagrantfile (but still on py34 and openssl 1.0).
  365. end