functions.sh 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349
  1. # This file is supposed to be sourced by each Recipe
  2. # that wants to use the functions contained herein
  3. # like so:
  4. # wget -q https://github.com/AppImage/AppImages/raw/${PKG2AICOMMIT}/functions.sh -O ./functions.sh
  5. # . ./functions.sh
  6. # RECIPE=$(realpath "$0")
  7. # Specify a certain commit if you do not want to use master
  8. # by using:
  9. # export PKG2AICOMMIT=<git sha>
  10. if [ -z "$PKG2AICOMMIT" ] ; then
  11. PKG2AICOMMIT=master
  12. fi
  13. # Options for apt-get to use local files rather than the system ones
  14. OPTIONS="-o Debug::NoLocking=1
  15. -o APT::Cache-Limit=125829120
  16. -o Dir::Etc::sourcelist=./sources.list
  17. -o Dir::State=./tmp
  18. -o Dir::Cache=./tmp
  19. -o Dir::State::status=./status
  20. -o Dir::Etc::sourceparts=-
  21. -o APT::Get::List-Cleanup=0
  22. -o APT::Get::AllowUnauthenticated=1
  23. -o Debug::pkgProblemResolver=true
  24. -o Debug::pkgDepCache::AutoInstall=true
  25. -o APT::Install-Recommends=0
  26. -o APT::Install-Suggests=0
  27. "
  28. # Detect if we are running inside Docker
  29. grep docker /proc/1/cgroup >/dev/null && export DOCKER_BUILD=1 || true
  30. # Detect system architecture to know which binaries of AppImage tools
  31. # should be downloaded and used.
  32. case "$(uname -i)" in
  33. x86_64|amd64)
  34. # echo "x86-64 system architecture"
  35. SYSTEM_ARCH="x86_64";;
  36. i?86)
  37. # echo "x86 system architecture"
  38. SYSTEM_ARCH="i686";;
  39. # arm*)
  40. # echo "ARM system architecture"
  41. # SYSTEM_ARCH="";;
  42. unknown|AuthenticAMD|GenuineIntel)
  43. # uname -i not answer on debian, then:
  44. case "$(uname -m)" in
  45. x86_64|amd64)
  46. # echo "x86-64 system architecture"
  47. SYSTEM_ARCH="x86_64";;
  48. i?86)
  49. # echo "x86 system architecture"
  50. SYSTEM_ARCH="i686";;
  51. esac ;;
  52. *)
  53. echo "Unsupported system architecture"
  54. exit 1;;
  55. esac
  56. # Either get the file from remote or from a static place.
  57. # critical for builds without network access like in Open Build Service
  58. cat_file_from_url()
  59. {
  60. cat_excludelist="wget -q $1 -O -"
  61. [ -e "$STATIC_FILES/${1##*/}" ] && cat_excludelist="cat $STATIC_FILES/${1##*/}"
  62. $cat_excludelist
  63. }
  64. git_pull_rebase_helper()
  65. {
  66. git reset --hard HEAD
  67. git pull
  68. }
  69. # Patch /usr to ././ in ./usr
  70. # to make the contents of usr/ relocateable
  71. # (this requires us to cd ./usr before running the application; AppRun does that)
  72. patch_usr()
  73. {
  74. find usr/ -type f -executable -exec sed -i -e "s|/usr|././|g" {} \;
  75. }
  76. # Download AppRun and make it executable
  77. get_apprun()
  78. {
  79. cp ${HERE}/AppRun .
  80. chmod a+x AppRun
  81. }
  82. # Copy the library dependencies of all exectuable files in the current directory
  83. # (it can be beneficial to run this multiple times)
  84. copy_deps()
  85. {
  86. PWD=$(readlink -f .)
  87. FILES=$(find . -type f -executable -or -name *.so.* -or -name *.so | sort | uniq )
  88. for FILE in $FILES ; do
  89. ldd "${FILE}" | grep "=>" | awk '{print $3}' | xargs -I '{}' echo '{}' >> DEPSFILE
  90. done
  91. DEPS=$(cat DEPSFILE | sort | uniq)
  92. for FILE in $DEPS ; do
  93. if [ -e $FILE ] && [[ $(readlink -f $FILE)/ != $PWD/* ]] ; then
  94. cp -v --parents -rfL $FILE ./ || true
  95. fi
  96. done
  97. rm -f DEPSFILE
  98. }
  99. # Move ./lib/ tree to ./usr/lib/
  100. move_lib()
  101. {
  102. mkdir -p ./usr/lib ./lib && find ./lib/ -exec cp -v --parents -rfL {} ./usr/ \; && rm -rf ./lib
  103. mkdir -p ./usr/lib ./lib64 && find ./lib64/ -exec cp -v --parents -rfL {} ./usr/ \; && rm -rf ./lib64
  104. }
  105. # Delete blacklisted files
  106. delete_blacklisted()
  107. {
  108. BLACKLISTED_FILES=$(cat_file_from_url https://github.com/AppImage/pkg2appimage/raw/${PKG2AICOMMIT}/excludelist | sed 's|#.*||g')
  109. echo $BLACKLISTED_FILES
  110. for FILE in $BLACKLISTED_FILES ; do
  111. FILES="$(find . -name "${FILE}" -not -path "./usr/optional/*")"
  112. for FOUND in $FILES ; do
  113. rm -vf "$FOUND" "$(readlink -f "$FOUND")"
  114. done
  115. done
  116. # Do not bundle developer stuff
  117. rm -rf usr/include || true
  118. rm -rf usr/lib/cmake || true
  119. rm -rf usr/lib/pkgconfig || true
  120. find . -name '*.la' | xargs -i rm {}
  121. }
  122. # Echo highest glibc version needed by the executable files in the current directory
  123. glibc_needed()
  124. {
  125. find . -name *.so -or -name *.so.* -or -type f -executable -exec strings {} \; | grep ^GLIBC_2 | sed s/GLIBC_//g | sort --version-sort | uniq | tail -n 1
  126. # find . -name *.so -or -name *.so.* -or -type f -executable -exec readelf -s '{}' 2>/dev/null \; | sed -n 's/.*@GLIBC_//p'| awk '{print $1}' | sort --version-sort | tail -n 1
  127. }
  128. # Add desktop integration
  129. # Usage: get_desktopintegration name_of_desktop_file_and_exectuable
  130. get_desktopintegration()
  131. {
  132. # REALBIN=$(grep -o "^Exec=.*" *.desktop | sed -e 's|Exec=||g' | cut -d " " -f 1 | head -n 1)
  133. # cat_file_from_url https://raw.githubusercontent.com/AppImage/AppImageKit/deprecated/AppImageAssistant/desktopintegration > ./usr/bin/$REALBIN.wrapper
  134. # chmod a+x ./usr/bin/$REALBIN.wrapper
  135. echo "The desktopintegration script is deprecated. Please advise users to use https://github.com/AppImage/appimaged instead."
  136. # sed -i -e "s|^Exec=$REALBIN|Exec=$REALBIN.wrapper|g" $1.desktop
  137. }
  138. # Generate AppImage; this expects $ARCH, $APP and $VERSION to be set
  139. generate_appimage()
  140. {
  141. # Download AppImageAssistant
  142. URL="https://github.com/AppImage/AppImageKit/releases/download/6/AppImageAssistant_6-${SYSTEM_ARCH}.AppImage"
  143. wget -c "$URL" -O AppImageAssistant
  144. chmod a+x ./AppImageAssistant
  145. # if [[ "$RECIPE" == *ecipe ]] ; then
  146. # echo "#!/bin/bash -ex" > ./$APP.AppDir/Recipe
  147. # echo "# This recipe was used to generate this AppImage." >> ./$APP.AppDir/Recipe
  148. # echo "# See http://appimage.org for more information." >> ./$APP.AppDir/Recipe
  149. # echo "" >> ./$APP.AppDir/Recipe
  150. # cat $RECIPE >> ./$APP.AppDir/Recipe
  151. # fi
  152. #
  153. # Detect the architecture of what we are packaging.
  154. # The main binary could be a script, so let's use a .so library
  155. BIN=$(find . -name *.so* -type f | head -n 1)
  156. INFO=$(file "$BIN")
  157. if [ -z $ARCH ] ; then
  158. if [[ $INFO == *"x86-64"* ]] ; then
  159. ARCH=x86_64
  160. elif [[ $INFO == *"i686"* ]] ; then
  161. ARCH=i686
  162. elif [[ $INFO == *"armv6l"* ]] ; then
  163. ARCH=armhf
  164. else
  165. echo "Could not automatically detect the architecture."
  166. echo "Please set the \$ARCH environment variable."
  167. exit 1
  168. fi
  169. fi
  170. mkdir -p ../out || true
  171. rm ../out/$APP"-"$VERSION".glibc"$GLIBC_NEEDED"-"$ARCH".AppImage" 2>/dev/null || true
  172. GLIBC_NEEDED=$(glibc_needed)
  173. ./AppImageAssistant ./$APP.AppDir/ ../out/$APP"-"$VERSION".glibc"$GLIBC_NEEDED"-"$ARCH".AppImage"
  174. }
  175. # Generate AppImage type 2
  176. # Additional parameters given to this routine will be passed on to appimagetool
  177. #
  178. # If the environment variable NO_GLIBC_VERSION is set, the required glibc version
  179. # will not be added to the AppImage filename
  180. generate_type2_appimage()
  181. {
  182. # Get the ID of the last successful build on Travis CI
  183. # ID=$(wget -q https://api.travis-ci.org/repos/AppImage/appimagetool/builds -O - | head -n 1 | sed -e 's|}|\n|g' | grep '"result":0' | head -n 1 | sed -e 's|,|\n|g' | grep '"id"' | cut -d ":" -f 2)
  184. # Get the transfer.sh URL from the logfile of the last successful build on Travis CI
  185. # Only Travis knows why build ID and job ID don't match and why the above doesn't give both...
  186. # URL=$(wget -q "https://s3.amazonaws.com/archive.travis-ci.org/jobs/$((ID+1))/log.txt" -O - | grep "https://transfer.sh/.*/appimagetool" | tail -n 1 | sed -e 's|\r||g')
  187. # if [ -z "$URL" ] ; then
  188. # URL=$(wget -q "https://s3.amazonaws.com/archive.travis-ci.org/jobs/$((ID+2))/log.txt" -O - | grep "https://transfer.sh/.*/appimagetool" | tail -n 1 | sed -e 's|\r||g')
  189. # fi
  190. if [ -z "$(which appimagetool)" ] ; then
  191. URL="https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-${SYSTEM_ARCH}.AppImage"
  192. wget -c "$URL" -O appimagetool
  193. chmod a+x ./appimagetool
  194. appimagetool=$(readlink -f appimagetool)
  195. else
  196. appimagetool=$(which appimagetool)
  197. fi
  198. if [ "$DOCKER_BUILD" ]; then
  199. appimagetool_tempdir=$(mktemp -d)
  200. mv appimagetool "$appimagetool_tempdir"
  201. pushd "$appimagetool_tempdir" &>/dev/null
  202. ls -al
  203. ./appimagetool --appimage-extract
  204. rm appimagetool
  205. appimagetool=$(readlink -f squashfs-root/AppRun)
  206. popd &>/dev/null
  207. _appimagetool_cleanup() { [ -d "$appimagetool_tempdir" ] && rm -r "$appimagetool_tempdir"; }
  208. trap _appimagetool_cleanup EXIT
  209. fi
  210. if [ -z ${NO_GLIBC_VERSION+true} ]; then
  211. GLIBC_NEEDED=$(glibc_needed)
  212. VERSION_EXPANDED=$VERSION.glibc$GLIBC_NEEDED
  213. else
  214. VERSION_EXPANDED=$VERSION
  215. fi
  216. set +x
  217. GLIBC_NEEDED=$(glibc_needed)
  218. if ( [ ! -z "$KEY" ] ) && ( ! -z "$TRAVIS" ) ; then
  219. wget https://github.com/AppImage/AppImageKit/files/584665/data.zip -O data.tar.gz.gpg
  220. ( set +x ; echo $KEY | gpg2 --batch --passphrase-fd 0 --no-tty --skip-verify --output data.tar.gz --decrypt data.tar.gz.gpg )
  221. tar xf data.tar.gz
  222. sudo chown -R $USER .gnu*
  223. mv $HOME/.gnu* $HOME/.gnu_old ; mv .gnu* $HOME/
  224. VERSION=$VERSION_EXPANDED "$appimagetool" $@ -n -s --bintray-user $BINTRAY_USER --bintray-repo $BINTRAY_REPO -v ./$APP.AppDir/ -u "gh-releases-zsync|VSCodium|vscodium|latest|*.AppImage.zsync"
  225. else
  226. VERSION=$VERSION_EXPANDED "$appimagetool" $@ -n --bintray-user $BINTRAY_USER --bintray-repo $BINTRAY_REPO -v ./$APP.AppDir/ -u "gh-releases-zsync|VSCodium|vscodium|latest|*.AppImage.zsync"
  227. fi
  228. set -x
  229. mkdir -p ../out/ || true
  230. mv *.AppImage* ../out/
  231. }
  232. # Generate status file for use by apt-get; assuming that the recipe uses no newer
  233. # ingredients than what would require more recent dependencies than what we assume
  234. # to be part of the base system
  235. generate_status()
  236. {
  237. mkdir -p ./tmp/archives/
  238. mkdir -p ./tmp/lists/partial
  239. touch tmp/pkgcache.bin tmp/srcpkgcache.bin
  240. if [ -e "${HERE}/usr/share/pkg2appimage/excludedeblist" ] ; then
  241. EXCLUDEDEBLIST="${HERE}/usr/share/pkg2appimage/excludedeblist"
  242. else
  243. wget -q -c "https://github.com/AppImage/AppImages/raw/${PKG2AICOMMIT}/excludedeblist"
  244. EXCLUDEDEBLIST=excludedeblist
  245. fi
  246. rm status 2>/dev/null || true
  247. for PACKAGE in $(cat excludedeblist | cut -d "#" -f 1) ; do
  248. printf "Package: $PACKAGE\nStatus: install ok installed\nArchitecture: all\nVersion: 9:999.999.999\n\n" >> status
  249. done
  250. }
  251. # Find the desktop file and copy it to the AppDir
  252. get_desktop()
  253. {
  254. find usr/share/applications -iname "*${LOWERAPP}.desktop" -exec cp {} . \; || true
  255. }
  256. fix_desktop() {
  257. # fix trailing semicolons
  258. for key in Actions Categories Implements Keywords MimeType NotShowIn OnlyShowIn; do
  259. sed -i '/'"$key"'.*[^;]$/s/$/;/' $1
  260. done
  261. }
  262. # Find the icon file and copy it to the AppDir
  263. get_icon()
  264. {
  265. find ./usr/share/pixmaps/$LOWERAPP.png -exec cp {} . \; 2>/dev/null || true
  266. find ./usr/share/icons -path *64* -name $LOWERAPP.png -exec cp {} . \; 2>/dev/null || true
  267. find ./usr/share/icons -path *128* -name $LOWERAPP.png -exec cp {} . \; 2>/dev/null || true
  268. find ./usr/share/icons -path *512* -name $LOWERAPP.png -exec cp {} . \; 2>/dev/null || true
  269. find ./usr/share/icons -path *256* -name $LOWERAPP.png -exec cp {} . \; 2>/dev/null || true
  270. ls -lh $LOWERAPP.png || true
  271. }
  272. # Find out the version
  273. get_version()
  274. {
  275. THEDEB=$(find ../*.deb -name $LOWERAPP"_*" | head -n 1)
  276. if [ -z "$THEDEB" ] ; then
  277. echo "Version could not be determined from the .deb; you need to determine it manually"
  278. fi
  279. VERSION=$(echo $THEDEB | cut -d "~" -f 1 | cut -d "_" -f 2 | cut -d "-" -f 1 | sed -e 's|1%3a||g' | sed -e 's|.dfsg||g' )
  280. echo $VERSION
  281. }
  282. # transfer.sh
  283. transfer() { if [ $# -eq 0 ]; then echo "No arguments specified. Usage:\necho transfer /tmp/test.md\ncat /tmp/test.md | transfer test.md"; return 1; fi
  284. tmpfile=$( mktemp -t transferXXX ); if tty -s; then basefile=$(basename "$1" | sed -e 's/[^a-zA-Z0-9._-]/-/g'); curl --progress-bar --upload-file "$1" "https://transfer.sh/$basefile" >> $tmpfile; else curl --progress-bar --upload-file "-" "https://transfer.sh/$1" >> $tmpfile ; fi; cat $tmpfile; rm -f $tmpfile; }
  285. # Patch binary files; fill with padding if replacement is shorter than original
  286. # http://everydaywithlinux.blogspot.de/2012/11/patch-strings-in-binary-files-with-sed.html
  287. # Example: patch_strings_in_file foo "/usr/local/lib/foo" "/usr/lib/foo"
  288. patch_strings_in_file() {
  289. local FILE="$1"
  290. local PATTERN="$2"
  291. local REPLACEMENT="$3"
  292. # Find all unique strings in FILE that contain the pattern
  293. STRINGS=$(strings ${FILE} | grep ${PATTERN} | sort -u -r)
  294. if [ "${STRINGS}" != "" ] ; then
  295. echo "File '${FILE}' contain strings with '${PATTERN}' in them:"
  296. for OLD_STRING in ${STRINGS} ; do
  297. # Create the new string with a simple bash-replacement
  298. NEW_STRING=${OLD_STRING//${PATTERN}/${REPLACEMENT}}
  299. # Create null terminated ASCII HEX representations of the strings
  300. OLD_STRING_HEX="$(echo -n ${OLD_STRING} | xxd -g 0 -u -ps -c 256)00"
  301. NEW_STRING_HEX="$(echo -n ${NEW_STRING} | xxd -g 0 -u -ps -c 256)00"
  302. if [ ${#NEW_STRING_HEX} -le ${#OLD_STRING_HEX} ] ; then
  303. # Pad the replacement string with null terminations so the
  304. # length matches the original string
  305. while [ ${#NEW_STRING_HEX} -lt ${#OLD_STRING_HEX} ] ; do
  306. NEW_STRING_HEX="${NEW_STRING_HEX}00"
  307. done
  308. # Now, replace every occurrence of OLD_STRING with NEW_STRING
  309. echo -n "Replacing ${OLD_STRING} with ${NEW_STRING}... "
  310. hexdump -ve '1/1 "%.2X"' ${FILE} | \
  311. sed "s/${OLD_STRING_HEX}/${NEW_STRING_HEX}/g" | \
  312. xxd -r -p > ${FILE}.tmp
  313. chmod --reference ${FILE} ${FILE}.tmp
  314. mv ${FILE}.tmp ${FILE}
  315. echo "Done!"
  316. else
  317. echo "New string '${NEW_STRING}' is longer than old" \
  318. "string '${OLD_STRING}'. Skipping."
  319. fi
  320. done
  321. fi
  322. }