Merge remote-tracking branch 'origin/master' into doc/9398-irc-blocking-tor

.gitmodules

 [submodule "submodules/pythonlib"]
 	path = submodules/pythonlib
 	url = https://git-tails.immerda.ch/pythonlib
+[submodule "submodules/jenkins-tools"]
+	path = submodules/jenkins-tools
+	url = https://git-tails.immerda.ch/jenkins-tools

auto/build

@@ -46,12 +46,6 @@ rm -rf cache/stages_rootfs
 # get LB_ARCHITECTURE and LB_DISTRIBUTION
 . config/bootstrap
 
-# build the doc wiki
-./build-wiki
-
-# refresh translations of our programs
-./refresh-translations || fatal "refresh-translations failed ($?)."
-
 # save variables that are needed by chroot_local-hooks
 echo "LB_DISTRIBUTION=${LB_DISTRIBUTION}" >> config/chroot_local-includes/usr/share/amnesia/build/variables
 echo "POTFILES_DOT_IN='$(
@@ -133,6 +127,12 @@ if [ -n "$JENKINS_URL" ] && [ -z "$GIT_TAG" ] \
     BUILD_BASENAME="${BUILD_BASENAME}@${GIT_BASE_BRANCH_SHORT_ID}"
 fi
 
+# build the doc wiki
+./build-wiki
+
+# refresh translations of our programs
+./refresh-translations || fatal "refresh-translations failed ($?)."
+
 case "$LB_BINARY_IMAGES" in
    iso)
       BUILD_FILENAME_EXT=iso

auto/config

@@ -14,6 +14,8 @@ if [ -e config/amnesia.local ] ; then
    . config/amnesia.local
 fi
 
+export LB_BOOTSTRAP_INCLUDE='eatmydata'
+
 # init variables
 RUN_LB_CONFIG="lb config noauto"
 

config/base_branch

-testing
+stable

config/binary_local-hooks/40-include_syslinux_in_ISO_filesystem

@@ -21,7 +21,7 @@ Set_defaults
 # Seems like we'll have work to do
 Echo_message 'including syslinux in the ISO filesystem'
 
-# Variables
+### Variables
 LINUX_BINARY_UTILS_DIR='binary/utils/linux'
 WIN32_BINARY_UTILS_DIR='binary/utils/win32'
 BINARY_MBR_DIR='binary/utils/mbr'
@@ -29,13 +29,13 @@ CHROOT_SYSLINUX_BIN='chroot/usr/bin/syslinux'
 CHROOT_SYSLINUX_MBR='chroot/usr/lib/SYSLINUX/gptmbr.bin'
 CHROOT_TEMP_APT_SOURCES='chroot/etc/apt/sources.list.d/tmp-deb-src.list'
 
-# Functions
+### Functions
 
 syslinux_deb_version_in_chroot () {
    chroot chroot dpkg-query -W -f='${Version}\n' syslinux
 }
 
-# Main
+### Main
 mkdir -p "$LINUX_BINARY_UTILS_DIR" "$WIN32_BINARY_UTILS_DIR" "$BINARY_MBR_DIR"
 cp "$CHROOT_SYSLINUX_BIN" "$LINUX_BINARY_UTILS_DIR/"
 cp "$CHROOT_SYSLINUX_MBR" "$BINARY_MBR_DIR/mbr.bin"

config/binary_local-hooks/50-grub-efi-ia32

+#! /bin/sh
+
+# Some of this file was adapted from the Debian Installer's
+# build/util/efi-image, which is:
+#
+# Copyright (C) 2010, 2011 Canonical Ltd.
+# Author: Colin Watson <cjwatson@ubuntu.com>
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+set -e
+set -x
+
+platform="i386-efi"
+outdir="binary/EFI/BOOT/grub/$platform"
+efi_name="ia32"
+
+grub_cpmodules () {
+	if [ -z "$1" ] || [ -z "$2" ]; then
+		echo "usage: $0 OUTPUT-DIRECTORY GRUB-PLATFORM"
+		return 1
+	fi
+
+	outdir="$1"
+	platform="$2"
+
+	# Copy over GRUB modules, except for those already built in.
+	cp -a "chroot/usr/lib/grub/$platform"/*.lst "$outdir/"
+	for x in "chroot/usr/lib/grub/$platform"/*.mod; do
+		# Some of these exclusions are based on knowledge of module
+		# dependencies.
+		case $(basename "$x" .mod) in
+			configfile|search|search_fs_file|search_fs_uuid|search_label|tar|part_gpt|linux|gzio)
+			# included in boot image
+			;;
+		    affs|afs|afs_be|befs|befs_be|minix|nilfs2|sfs|zfs|zfsinfo)
+			# unnecessary filesystem modules
+			;;
+		    example_functional_test|functional_test|hello)
+			# other cruft
+			;;
+		    *)
+			cp -a "$x" "$outdir/"
+			;;
+		esac
+	done
+}
+
+# Including common functions
+. "${LB_BASE:-/usr/share/live/build}"/scripts/build.sh
+
+# Setting static variables
+DESCRIPTION="$(Echo 'including GRUB EFI for ia32 in the ISO filesystem')"
+HELP=""
+USAGE="${PROGRAM}"
+
+# Reading configuration files
+Read_conffiles config/all config/bootstrap config/common config/binary
+Set_defaults
+
+# Safeguards
+[ "${LB_ARCHITECTURE}" = "i386" ] || exit 0
+
+# Seems like we'll have work to do
+Echo_message 'including GRUB EFI for ia32 in the ISO filesystem'
+
+# Build the core image
+Chroot chroot grub-mkimage -O "$platform" \
+	-o "/tmp/boot$efi_name.efi" -p "/efi/boot/grub" \
+	search configfile normal tar fat part_gpt linux \
+	gzio
+mv "chroot/tmp/boot$efi_name.efi" "binary/EFI/BOOT/boot$efi_name.efi"
+
+mkdir -p "$outdir"
+grub_cpmodules "$outdir" "$platform"

config/binary_local-includes/EFI/BOOT/grub/grub.cfg

+function load_video {
+  if [ x$feature_all_video_module = xy ]; then
+    insmod all_video
+  else
+    insmod efi_gop
+    insmod efi_uga
+    insmod ieee1275_fb
+    insmod vbe
+    insmod vga
+    insmod video_bochs
+    insmod video_cirrus
+  fi
+}
+
+set linux_gfx_mode=
+export linux_gfx_mode
+load_video
+
+insmod syslinuxcfg
+insmod cpuid
+echo "Loading syslinux configuration..."
+syslinux_configfile /efi/boot/syslinux.cfg

config/chroot_apt/preferences

@@ -26,6 +26,14 @@ Package: cryptsetup-bin
 Pin: release o=Debian Backports,n=wheezy-backports
 Pin-Priority: 999
 
+Package: eatmydata
+Pin: release o=Debian Backports,n=wheezy-backports
+Pin-Priority: 999
+
+Package: libeatmydata1
+Pin: release o=Debian Backports,n=wheezy-backports
+Pin-Priority: 999
+
 Package: electrum
 Pin: release o=Debian Backports,n=wheezy-backports
 Pin-Priority: 999
@@ -130,68 +138,76 @@ Package: libcryptsetup4
 Pin: release o=Debian Backports,n=wheezy-backports
 Pin-Priority: 999
 
+Package: libestr0
+Pin: release o=Debian Backports,n=wheezy-backports
+Pin-Priority: 999
+
+Package: libotr5
+Pin: release o=Debian Backports,n=wheezy-backports
+Pin-Priority: 999
+
 Package: linux-base
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-compiler-gcc-4.8-x86
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-headers-586
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-headers-686-pae
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-headers-amd64
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-headers-3.16.0-4-common
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-headers-3.16.0-4-586
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-headers-3.16.0-4-686-pae
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-headers-3.16.0-4-amd64
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-image-586
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-image-686-pae
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-image-amd64
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-image-3.16.0-4-586
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-image-3.16.0-4-686-pae
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-image-3.16.0-4-amd64
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: linux-kbuild-3.16
-Pin: release o=Debian,a=testing
+Pin: release o=Debian,n=jessie
 Pin-Priority: 999
 
 Package: mat
@@ -206,6 +222,10 @@ Package: obfs4proxy
 Pin: release o=TorProject,n=obfs4proxy
 Pin-Priority: 990
 
+Package: pidgin-otr
+Pin: release o=Debian Backports,n=wheezy-backports
+Pin-Priority: 999
+
 Package: python-six
 Pin: release o=Debian Backports,n=wheezy-backports
 Pin-Priority: 999
@@ -222,6 +242,10 @@ Package: python-electrum
 Pin: release o=Debian Backports,n=wheezy-backports
 Pin-Priority: 999
 
+Package: rsyslog
+Pin: release o=Debian Backports,n=wheezy-backports
+Pin-Priority: 999
+
 Package: scdaemon
 Pin: release o=Debian Backports,n=wheezy-backports
 Pin-Priority: 999
@@ -267,6 +291,10 @@ Package: *
 Pin: release o=Debian,n=wheezy-updates
 Pin-Priority: 990
 
+Package: *
+Pin: release o=Debian,n=jessie-updates
+Pin-Priority: 500
+
 Package: *
 Pin: release o=Debian,n=wheezy
 Pin-Priority: 990

config/chroot_local-hooks/11-localize_browser

@@ -17,13 +17,15 @@ echo "Localize each supported browser locale"
 . /etc/amnesia/environment
 
 TBB_DEFAULT_SEARCHPLUGINS_DIR="${TBB_INSTALL}/browser/searchplugins"
-TBB_LOCALIZED_SEACHPLUGINS_DIR="${TBB_INSTALL}/distribution/searchplugins/locale/"
+TBB_LOCALIZED_SEARCHPLUGINS_DIR="${TBB_INSTALL}/distribution/searchplugins/locale/"
 BROWSER_LOCALIZATION_DIR="/usr/share/tails/browser-localization"
 DESCRIPTIONS_FILE="${BROWSER_LOCALIZATION_DIR}/descriptions"
 BRANDING_TEMPLATE_FILE="${BROWSER_LOCALIZATION_DIR}/amnesia.properties-template"
 BRANDING_DIR="/usr/local/share/tor-browser-extensions/branding@amnesia.boum.org/"
 NO_SPELLCHECKER_LOCALES="ko nl pl tr zh"
 
+apt-get install imagemagick
+
 # Sanity check that each supported Tor Browser locale has a
 # description for how to localize it further.
 BROKEN_LOCALES=""
@@ -59,7 +61,7 @@ while IFS=: read MOZILLA_LOCALE LOCATION LOCALIZED_LANG STARTPAGE_LANG STARTPAGE
     # locale format, since Firefox isn't very consistent in it.
     NORMAL_LOCALE="$(echo "${MOZILLA_LOCALE}" | tr - _)"
     LANG_CODE="$(language_code_from_locale "${NORMAL_LOCALE}")"
-    TARGET_SEARCHPLUGINS_DIR="${TBB_LOCALIZED_SEACHPLUGINS_DIR}/${MOZILLA_LOCALE}"
+    TARGET_SEARCHPLUGINS_DIR="${TBB_LOCALIZED_SEARCHPLUGINS_DIR}/${MOZILLA_LOCALE}"
     mkdir -p "${TARGET_SEARCHPLUGINS_DIR}"
 
     if [ -z "${STARTPAGE_LANG_UI}" ]; then
@@ -77,6 +79,30 @@ while IFS=: read MOZILLA_LOCALE LOCATION LOCALIZED_LANG STARTPAGE_LANG STARTPAGE
         "${BROWSER_LOCALIZATION_DIR}/disconnect.xml-template" > \
         "${DISCONNECT_PLUGIN}"
 
+    # We generate a Wikipedia plugin with localized icons since we
+    # want to provide both English and the locale's plugin, and
+    # Firefox' new search bar only shows icons; the description (which
+    # is localized) is only shown in a pop-up nowdays, so it's easy to
+    # mix them up.
+    CAPITALIZED_LANG_CODE="$(echo "${LANG_CODE}" | tr 'a-z' 'A-Z')"
+    LOCALIZED_WIKIPEDIA_ICON_PATH="/tmp/wikipedia-icon-${LANG_CODE}.png"
+    WIKIPEDIA_SEARCH_ICON_BASE64_PATH="#{LOCALIZED_WIKIPEDIA_ICON_PATH}.base64"
+    WIKIPEDIA_ICON_TEMPLATE="${BROWSER_LOCALIZATION_DIR}/Wikipedia-icon.png"
+    convert "${WIKIPEDIA_ICON_TEMPLATE}" \
+            -gravity SouthEast -pointsize 130 -font Liberation-Sans-Bold \
+            -fill black -annotate 0 "${CAPITALIZED_LANG_CODE}" \
+            +set date:create +set date:modify -define png:exclude-chunk=time \
+            -resize 16x16 "${LOCALIZED_WIKIPEDIA_ICON_PATH}"
+    base64 "${LOCALIZED_WIKIPEDIA_ICON_PATH}" | tr -d "\n" > \
+           "${WIKIPEDIA_SEARCH_ICON_BASE64_PATH}"
+    sed -e "s/\${LANG_CODE}/${LANG_CODE}/" \
+        -e "/\${BASE64_PNG_16x16}/ r ${WIKIPEDIA_SEARCH_ICON_BASE64_PATH}" \
+        -e "/\${BASE64_PNG_16x16}/d" \
+        "${BROWSER_LOCALIZATION_DIR}/wikipedia.xml-template" > \
+        "${TARGET_SEARCHPLUGINS_DIR}/wikipedia-${MOZILLA_LOCALE}.xml"
+    rm "${LOCALIZED_WIKIPEDIA_ICON_PATH}" \
+       "${WIKIPEDIA_SEARCH_ICON_BASE64_PATH}"
+
     # We use the branding@amnesia.org extension to set some per-locale
     # default prefs that set the appropriate localization options.
     TARGET_BRANDING_DIR="${BRANDING_DIR}/chrome/locale/${MOZILLA_LOCALE}"
@@ -134,7 +160,8 @@ while IFS=: read MOZILLA_LOCALE LOCATION LOCALIZED_LANG STARTPAGE_LANG STARTPAGE
         rm -f "${DEB_PATH_TO_SEARCHPLUGINS}"/amazon*.xml \
               "${DEB_PATH_TO_SEARCHPLUGINS}"/bing*.xml \
               "${DEB_PATH_TO_SEARCHPLUGINS}"/eBay*.xml \
-              "${DEB_PATH_TO_SEARCHPLUGINS}"/yahoo*.xml
+              "${DEB_PATH_TO_SEARCHPLUGINS}"/yahoo*.xml \
+              "${DEB_PATH_TO_SEARCHPLUGINS}"/wikipedia*.xml
         cp "${DEB_PATH_TO_SEARCHPLUGINS}"/* "${TARGET_SEARCHPLUGINS_DIR}"
         cd /
         rm -r "${TMP}"
@@ -144,11 +171,21 @@ done < "${DESCRIPTIONS_FILE}"
 # This directory is not needed after build time.
 rm -r "${BROWSER_LOCALIZATION_DIR}"
 
-# All generated files must be world-readable.
-chmod -R a+rX "${TBB_LOCALIZED_SEACHPLUGINS_DIR}" "${BRANDING_DIR}"
-
 # Remove unwanted browser search plugins bundled in the Tor Browser.
 rm "${TBB_DEFAULT_SEARCHPLUGINS_DIR}"/yahoo*.xml
 # We generate localized versions of the following:
 rm "${TBB_DEFAULT_SEARCHPLUGINS_DIR}"/disconnect*.xml
 rm "${TBB_DEFAULT_SEARCHPLUGINS_DIR}"/startpage*.xml
+rm "${TBB_DEFAULT_SEARCHPLUGINS_DIR}"/wikipedia*.xml
+
+# We want our localized English Wikipedia plugin to be available in
+# all locales.
+mv "${TBB_LOCALIZED_SEARCHPLUGINS_DIR}/en-US/wikipedia-en-US.xml" \
+   "${TBB_DEFAULT_SEARCHPLUGINS_DIR}/"
+
+# All generated files must be world-readable.
+chmod -R a+rX "${TBB_LOCALIZED_SEARCHPLUGINS_DIR}" \
+              "${TBB_DEFAULT_SEARCHPLUGINS_DIR}" \
+              "${BRANDING_DIR}"
+
+apt-get --yes purge imagemagick

config/chroot_local-hooks/44-remove-unused-AppArmor-profiles

+#!/bin/sh
+
+set -e
+
+echo "Deleting unused AppArmor profiles"
+
+(
+   cd /etc/apparmor.d
+   rm \
+       apache2.d/phpsysinfo     \
+       sbin.klogd               \
+       sbin.syslogd             \
+       sbin.syslog-ng           \
+       usr.bin.chromium-browser \
+       usr.lib.dovecot.*        \
+       usr.sbin.dnsmasq         \
+       usr.sbin.dovecot         \
+       usr.sbin.identd          \
+       usr.sbin.mdnsd           \
+       usr.sbin.nmbd            \
+       usr.sbin.ntpd            \
+       usr.sbin.nscd            \
+       usr.sbin.smb*
+)

config/chroot_local-hooks/52-update-rc.d

@@ -18,6 +18,7 @@ alsa-utils
 gdomap
 haveged
 hdparm
+hwclock.sh
 i2p
 kexec
 kexec-load

config/chroot_local-hooks/98-remove_unwanted_files

@@ -17,3 +17,13 @@ rm $POTFILES_DOT_IN
 # These files are not needed after the Tor Browser has been installed
 # (by the 10-tbb hook)
 rm /usr/share/tails/tbb-*.txt
+
+# Remove the snakeoil SSL key pair generated by ssl-cert
+find /etc/ssl/certs /etc/ssl/private |
+    while read f; do
+       if [ "$(readlink -f "$f")" = "/etc/ssl/certs/ssl-cert-snakeoil.pem" ] || \
+          [ "$(readlink -f "$f")" = "/etc/ssl/private/ssl-cert-snakeoil.key" ]; then
+	   rm "${f}"
+       fi
+    done
+update-ca-certificates

config/chroot_local-includes/etc/apparmor.d/tunables/home.d/tails

+@{HOMEDIRS}+=/lib/live/mount/overlay/home/

config/chroot_local-includes/etc/tor-browser/profile/chrome/userChrome.css

 /* Required, do not remove */
 @namespace url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul");
 
+/* Hide Firefox Sync options. Sync hasn't been audited by the
+   Tor Browser developers yet (Tor bug #10368), and it doesn't seem to
+   work any way (Tor bug #13279). Weak passwords would be a pretty
+   serious issue too. */
+#BrowserPreferences radio[pane="paneSync"],
+#sync-button,
+#sync-menu-button,
+#sync-setup,
+#sync-setup-appmenu,
+#sync-status-button,
+#sync-syncnowitem-appmenu,
+#wrapper-sync-button,
+
+/* Hide the Tools -> Apps link to the Firefox Marketplace. It doesn't
+   seem to work in the Tor Browser, and may have privacy issues. */
+#menu_openApps,
+
+/* Hide the "Share this page" button in the Tool bar, which encourages
+   the use of social (= tracking) networks. Note that this one likely
+   will be removed upstream in the final Tor Browser 5.0 release. */
+#social-share-button,
+
 /* Hide HTTPS Everywhere button in the toolbar */
 #https-everywhere-button { display: none; }

config/chroot_local-includes/etc/tor-browser/profile/preferences/0000tails.js

@@ -78,4 +78,18 @@ pref("extensions.update.enabled", false);
 pref("layout.spellcheckDefault", 0);
 pref("network.dns.disableIPv6", true);
 pref("security.warn_submit_insecure", true);
-pref("network.proxy.no_proxies_on", "10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16");
+
+// Disable fetching of the new tab page's Tiles links/ads. Ads are
+// generally unwanted, and also the fetching is a "phone home" type of
+// feature that generates traffic at least the first time the browser
+// is started.
+pref("browser.newtabpage.directory.source", "");
+pref("browser.newtabpage.directory.ping", "");
+// ... and disable the explanation shown the first time
+pref("browser.newtabpage.introShown", true);
+
+// Don't use geographically specific search prefs, like
+// browser.search.*.US for US locales. Our generated amnesia branding
+// add-on localizes search-engines in an incompatible but equivalent
+// way.
+pref("browser.search.geoSpecificDefaults", false);

config/chroot_local-includes/usr/local/bin/tor-browser

@@ -21,6 +21,11 @@ PROFILE="${HOME}/.tor-browser/profile.default"
 . /usr/local/lib/tails-shell-library/tor-browser.sh
 
 ask_for_confirmation() {
+    # Skip dialog if user is already running Tor Browser:
+    if pgrep -u amnesia -f "${TBB_INSTALL}/firefox" ; then
+        return
+    fi
+
     local dialog_title="`gettext \"Tor is not ready\"`"
     local dialog_text="`gettext \"Tor is not ready. Start Tor Browser anyway?\"`"
     local dialog_start="`gettext \"Start Tor Browser\"`"
@@ -59,6 +64,10 @@ start_browser() {
         /usr/local/bin/generate-tor-browser-profile
     fi
 
+    TMPDIR="${PROFILE}/tmp"
+    mkdir --mode=0700 -p "$TMPDIR"
+    export TMPDIR
+
     configure_best_tor_browser_locale "${PROFILE}"
 
     # Workaround bug #8036

config/chroot_local-includes/usr/local/sbin/tails-spoof-mac

@@ -80,7 +80,7 @@ spoof_mac() {
     set -e
     if [ "${ret}" != 0 ]; then
         log "macchanger failed for NIC ${1}, returned ${ret} and said: ${msg}"
-        exit 1
+        return 1
     fi
 }
 

config/chroot_local-includes/usr/share/amnesia/build/mksquashfs-excludes

@@ -15,4 +15,5 @@ var/cache/apt/archives/*.deb
 var/cache/apt/archives/partial/*.deb
 var/cache/apt/pkgcache.bin
 var/cache/apt/srcpkgcache.bin
+var/cache/man/*
 var/lib/apt/lists/*