Merge remote-tracking branch 'origin/master' into doc/17334-rescue-broken-tails

README.md

@@ -17,7 +17,7 @@ There are many ways [you can contribute to Tails](https://tails.boum.org/contrib
 
 Find out how you can make a difference in Tails: https://tails.boum.org/contribute/.
 
-### How to get started with Gitlab
+### How to get started with GitLab
 
 https://tails.boum.org/contribute/working_together/GitLab/
 

bin/generate-changelog

+#! /usr/bin/python3
+
+import datetime
+import email.utils
+import functools
+import gitlab
+import jinja2
+import logging
+import os
+import pprint
+import re
+
+from pathlib import Path
+
+PYTHON_GITLAB_CONFIG_FILE = os.getenv('PYTHON_GITLAB_CONFIG_FILE',
+                                      default=Path.home() /
+                                      '.python-gitlab.cfg')
+
+PYTHON_GITLAB_NAME = os.getenv('GITLAB_NAME', default='Tails')
+
+GROUP_NAME = 'tails'
+
+# Only changes in these projects are considered
+PROJECTS = [
+    GROUP_NAME + '/' + project for project in [
+        'chutney',
+        'installer',
+        'tails',
+        'whisperback',
+        'workarounds',
+    ]
+]
+
+# Merge requests that modify only files whose path match one of IGNORED_PATHS
+# are ignored.
+# Patterns will be passed to re.fullmatch().
+IGNORED_PATHS = [
+    r'po/.*',
+    r'wiki/.*',
+]
+
+LOG_FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
+log = logging.getLogger()
+
+
+class GitLabWrapper(gitlab.Gitlab):
+    @functools.lru_cache
+    def project(self, project_id):
+        return self.projects.get(project_id)
+
+    @functools.lru_cache
+    def project_path_with_namespace(self, project_id):
+        return self.project(project_id).path_with_namespace
+
+
+class ChangelogGenerator(object):
+    def __init__(self, gl, group, version: str):
+        self.gl = gl
+        self.group = group
+        self.version = version
+
+    def merge_requests(self, milestone) -> list:
+        mrs = []
+
+        gl_mrs = [
+            mr for mr in milestone.merge_requests(all=True)
+            if mr.state == 'merged' \
+            and self.gl.project_path_with_namespace(
+                mr.target_project_id) in PROJECTS
+        ]
+
+        for mr in gl_mrs:
+            project = self.gl.project(mr.target_project_id)
+            mr = project.mergerequests.get(mr.iid)
+            if ignore_merge_request(mr):
+                continue
+            # yapf: disable
+            mrs.append({
+                "ref": mr.references['full'],
+                "title": mr.title,
+                "web_url": mr.web_url,
+                "closes_issues": [
+                    {
+                        "ref": project.issues.get(issue.iid).references['full'],
+                        "title": issue.title,
+                    }
+                    for issue in mr.closes_issues()
+                ],
+                "commit_messages": [
+                    commit.title for commit in mr.commits()
+                    # Ignore merge commits
+                    if len(project.commits.get(commit.id).parent_ids) == 1
+                ],
+            })
+            # yapf: enable
+
+        return mrs
+
+    def changes(self) -> dict:
+        milestone_title = "Tails_" + self.version
+        milestone = [
+            m for m in self.group.milestones.list(search=milestone_title)
+            # Disambiguate between milestones whose names share a common prefix
+            if m.title == milestone_title
+        ][0]
+        assert isinstance(milestone, gitlab.v4.objects.GroupMilestone)
+
+        return {
+            "merge_requests": self.merge_requests(milestone),
+            "issues": {},  # Let's see if we really need this; probably not.
+        }
+
+
+def ignore_merge_request(mr) -> bool:
+    for change in mr.changes()['changes']:
+        for path in [change['old_path'], change['new_path']]:
+            if all([
+                    re.fullmatch(ignored_path, path) is None
+                    for ignored_path in IGNORED_PATHS
+            ]):
+                log.debug("Returning false")
+                return False
+    log.debug("Returning true")
+    return True
+
+
+def changelog_entry(version: str, date: datetime, changes: dict):
+    jinja2_env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader('config/release_management/templates'),
+        trim_blocks=True,
+        lstrip_blocks=True,
+    )
+
+    return jinja2_env.get_template('changelog.jinja2').render(
+        merge_requests=changes['merge_requests'],
+        issues=changes['issues'],
+        date=email.utils.format_datetime(date),
+        version=version,
+    )
+
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--version', required=True)
+    parser.add_argument("--debug", action="store_true", help="debug output")
+    args = parser.parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
+    else:
+        logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
+
+    gl = GitLabWrapper.from_config(PYTHON_GITLAB_NAME,
+                                   config_files=[PYTHON_GITLAB_CONFIG_FILE])
+    gl.auth()
+
+    group = gl.groups.list(search=GROUP_NAME)[0]
+    assert isinstance(group, gitlab.v4.objects.Group)
+
+    changes = ChangelogGenerator(gl, group, args.version).changes()
+
+    log.debug(pprint.PrettyPrinter().pformat(changes))
+
+    print(
+        changelog_entry(version=args.version,
+                        date=datetime.datetime.now(datetime.timezone.utc),
+                        changes=changes))

bin/update-changelog

+#! /usr/bin/python3
+
+import argparse
+import debian.changelog
+import subprocess
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--version', required=True)
+args = parser.parse_args()
+
+with open('debian/changelog', 'r') as f:
+    changelog = debian.changelog.Changelog(f)
+    # Remove the placeholder entry for the next release
+    changelog._blocks = changelog._blocks[1:]
+    changelog_minus_first_entry = changelog
+
+with open('debian/changelog', 'w') as f:
+    f.write(subprocess.check_output(
+        ['bin/generate-changelog', '--version', args.version]
+    ).decode('utf-8'))
+    changelog_minus_first_entry.write_to_open_file(f)

config/release_management/templates/changelog.jinja2

+{# Jinja2 template used by bin/generate-changelog -#}
+
+tails ({{version}}) unstable; urgency=medium
+
+{% for mr in merge_requests %}
+  {% set mr_desc = mr.title + " (" + mr.ref + ")" %}
+  * {{ mr_desc|wordwrap(wrapstring="\n    ")}}
+    {% if mr.closes_issues %}
+
+    Closes issues:
+      {% for issue in mr.closes_issues %}
+	{% set issue_desc = issue.title + " (" + issue.ref + ")" %}
+      - {{ issue_desc|wordwrap(wrapstring="\n        ") }}
+      {% endfor %}
+    {% endif %}
+
+    Commits:
+      {% for commit_message in mr.commit_messages %}
+      - {{ commit_message|wordwrap(wrapstring="\n        ") }}
+      {% endfor %}
+
+{% endfor %}
+ -- Tails developers <tails@boum.org>  {{date}}

release

-#!/bin/sh
-
-# Usage: ./release NEW_VERSION [ dev | SINCE_COMMITISH ]
-# 
-# If "dev" is supplied as the second argument, a development snapshot
-# is done rather than a real release, i.e.:
-#    * the --snapshot --auto options are passed to gbp-dch
-#    * no commit or tag is created
-# else, the second argument is passed to gbp-dch's --since option.
-
-### source the configuration files
-
-. config/amnesia
-if [ -e config/amnesia.local ] ; then
-   . config/amnesia.local
-fi
-
-### init variables
-
-NEW_VERSION="$1"
-if [ "$2" = dev ]; then
-   SNAPSHOT=yes
-else
-   SNAPSHOT=no
-   SINCE="$2"
-fi
-
-### helper functions
-
-fatal () {
-   echo "Fatal: $*" >&2
-   exit 2
-}
-
-### sanity checks
-
-[ -n "${NEW_VERSION}" ] \
-   || fatal "the new version must be supplied on the command-line."
-[ -n "${AMNESIA_DEV_FULLNAME}" ] \
-   || fatal "AMNESIA_DEV_FULLNAME must be set in config/amnesia"
-[ -n "${AMNESIA_DEV_EMAIL}" ] \
-   || fatal "AMNESIA_DEV_EMAIL must be set in config/amnesia"
-[ -n "${AMNESIA_DEV_KEYID}" ] \
-   || fatal "AMNESIA_DEV_KEYID must be set in config/amnesia"
-[ -x "`which git`" ] \
-   || fatal "could not find git, please apt-get install git-core"
-[ -x "`which gbp`" ] \
-   || fatal "could not find gbp, please apt-get install git-buildpackage"
-
-### main
-
-export DEBFULLNAME="${AMNESIA_DEV_FULLNAME}"
-export DEBEMAIL="${AMNESIA_DEV_EMAIL}"
-
-# update the Changelog
-echo "Updating debian/changelog from Git history..."
-gbp dch \
-   `if [ ${SNAPSHOT} = yes ]; then echo '--snapshot --auto' ; fi` \
-   `if [ ${SNAPSHOT} = no -a -n "${SINCE}" ]; then echo "--since=${SINCE}" ; fi` \
-   `if [ ${SNAPSHOT} = no -a -z ${SINCE} ]; then echo "--auto" ; fi` \
-   --new-version="${NEW_VERSION}" \
-   --ignore-branch \
-   --dch-opt=--force-bad-version \
-   -- '*' ':!po' ':!wiki' \
-   || fatal "gbp dch failed."
-
-# cleanup some parts of the changelog
-perl -pi'' -e 's/\A  \[ IkiWiki::Plugin::po::change \]\n//' debian/changelog
-perl -pi'' -e 's/\A  \* update[d]? PO file[s]?[.]?\n//' debian/changelog
-perl -pi'' -e 's/\A  \* \n//' debian/changelog
-perl -pi'' -e 's/\A  \[ 127\.0\.0\.1 \]\n//' debian/changelog
-perl -pi'' -e 's/\A  \[ amnesia \]\n//' debian/changelog
-perl -pi'' -e 's/\A  \[ T\(A\)ILS developers \]\n//' debian/changelog
-perl -pi'' -e 's/\A  \[ Tails developers \]\n//' debian/changelog
-perl -pi'' -e 's/\A  \[ Tails \]\n//' debian/changelog
-perl -pi'' -e 's/\A  \* Added a comment\n//' debian/changelog
-perl -pi'' -e 's/\A  \* Added a comment:.*\n//' debian/changelog
-perl -pi'' -e 's/\A  \* Remove spam\.\n//' debian/changelog
-perl -pi'' -e 's/\A  \* todo\+\+\n//i' debian/changelog
-perl -pi'' -e 's/\A  \* todo--\n//i' debian/changelog
-perl -pi'' -e 's/\A  \* TODO update[.]?\n//i' debian/changelog
-perl -pi'' -e 's/\A  \* Update ticket[.]?\n//i' debian/changelog
-perl -pi'' -e 's/\A  \* Now pending[.]?\n//i' debian/changelog
-perl -pi'' -e 's/\A  \* Done[.]?\n//i' debian/changelog
-perl -pi'' -e 's/\A  \* Upcoming release\n//' debian/changelog
-
-# commit and tag the release
-# if [ "${SNAPSHOT}" = no ]; then
-#    echo "Commit'ing debian/changelog..."
-#    git commit -m "releasing version ${NEW_VERSION}" debian/changelog \
-#       || fatal "failed to commit debian/changelog"
-#    echo "Tagging new version..."
-#    git tag -u "${AMNESIA_DEV_KEYID}" -m "tagging version ${NEW_VERSION}" "${NEW_VERSION}"
-# fi
-
-echo "done."

wiki/src/about.es.po

@@ -8,7 +8,7 @@ msgstr ""
 "Project-Id-Version: \n"
 "Report-Msgid-Bugs-To: tails-l10n@boum.org\n"
 "POT-Creation-Date: 2020-07-23 01:14+0000\n"
-"PO-Revision-Date: 2020-08-26 11:29+0000\n"
+"PO-Revision-Date: 2020-09-06 07:29+0000\n"
 "Last-Translator: emmapeel <emma.peel@riseup.net>\n"
 "Language-Team: \n"
 "Language: es\n"
@@ -16,7 +16,7 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
-"X-Generator: Weblate 3.5.1\n"
+"X-Generator: Weblate 3.8\n"
 
 #. type: Content of: outside any tag (error?)
 msgid ""
@@ -601,11 +601,11 @@ msgstr "<a href=\"https://www.torproject.org/\">El Proyecto Tor</a>"
 
 #. type: Content of: <section><div><div><ul><li>
 msgid "<a href=\"https://debian.org/\">Debian GNU/Linux</a>"
-msgstr ""
+msgstr "<a href=\"https://debian.org/\">Debian GNU/Linux</a>"
 
 #. type: Content of: <section><div><div><ul><li>
 msgid "<a href=\"https://www.gnome.org/\">GNOME</a>"
-msgstr ""
+msgstr "<a href=\"https://www.gnome.org/\">GNOME</a>"
 
 #. type: Content of: <section><div><div><ul><li>
 msgid "[[Contact]]"

wiki/src/blueprint/l10n_Spanish.mdwn

@@ -10,7 +10,7 @@ Si quiero traducir al español ¿cómo puedo empezar?
    - <https://www.autistici.org/mailman/listinfo/tails-l10n>
    - <https://www.autistici.org/mailman/listinfo/tails-l10n-spanish>
 
-2. Puedes traducir con git (un poco más complicado), o hacer una cuenta en la plataforma de traducción en pruebas (Ver abajo)
+2. Crea una cuenta en la plataforma de traducción (Ver abajo)
 
 <a name="decisiones">
 
@@ -36,6 +36,8 @@ Estilo y decisiones anteriores
 * Somos respetuosos con la ortografía: acentos, mayúsculas...
 * Los espacios y saltos de línea deben ser mantenidos para que la página se componga bien.
 
+Por favor lee este [documento de GNOME sobre traducción al castellano](https://wiki.gnome.org/TranslationProject/SpanishTeam?action=AttachFile&do=get&target=GNOME_l10n_es.pdf).
+
 ¿Cómo se traduce...?
 --------------------
 
@@ -60,7 +62,7 @@ Y lo puedo verificar porque ese archivo existe en el repositorio de Tails, en la
 Plataforma de traducción
 ------------------------
 
-Estamos testeando una plataforma de traducción del website con weblate, una aplicación web de traducción:
+El website se traduce con weblate, una aplicación web de traducción:
 
 <https://translate.tails.boum.org/>
 

wiki/src/blueprint/monthly_report/report_2020_08.mdwn

@@ -23,6 +23,24 @@ XXX: Copy the "Changes" section of the release notes, and compact a bit:
 Code
 ====
 
+- Added support for older TREZOR firmware ([[!tails_mr 142]]).
+
+- Started working on the upgrade to Thunderbird 78 ESR,
+  and in particular its new OpenPGP support ([[!tails_ticket 17148]]).
+
+- Automated generation of our Changelog ([[!tails_ticket 17886]].
+
+- Automated generation of the security advisory boilerplate from a template that
+  e.g. technical writers can improve ([[!tails_mr 128]]).
+
+- Hidden broken "Turn on Wi-Fi Hotspot" feature in GNOME Wi-Fi settings
+  ([[!tails_ticket 17887]]).
+
+- Fixed sorting Intel GPUs last in the "Error starting GDM" message
+  ([[!tails_ticket 17903]]).
+
+- Upgraded Linux to 5.7.17-1 ([[!tails_ticket 17895]]).
+
 XXX: If you feel like it and developers, foundation team, and RMs don't do it themselves,
      list important code work that is not covered already by the
      Release section (for example, the changes being worked on for
@@ -31,17 +49,77 @@ XXX: If you feel like it and developers, foundation team, and RMs don't do it th
 Documentation and website
 =========================
 
-XXX: If you feel like it and technical writers don't do it
-     themselves, explore the Git history:
+- Changed massively our instructions on how to start Tails:
+
+  1. Documented [[Shift + Restart on Windows|install/win/usb#start-tails]]. ([[!tails_ticket 16456]])
+
+     We're not telling Windows users to use the Boot Menu key anymore.
+
+     It's still the default for [[Linux|install/linux#start-tails]].
+
+  2. Wrote dedicated pages for starting Tails on [[PC|doc/first_steps/start/pc]] and
+     [[Mac|doc/first_steps/start/mac]].
+
+  3. Simplified troubleshooting sections.
+
+     We simplified and made consistent the [[troubleshooting instructions|install/win/usb#troubleshooting]]
+     in all installation scenarios.
+
+  4. Removed instructions to report to Help Desk from the installation
+     scenarios but kept them on [[doc/first_steps/start|doc/first_steps/start/pc/#help-desk]].
+
+  We believe that all these changes will lower your workload and make
+  more people start Tails at the same time.
 
-         git log --patch --since='1 October' --until='1 November' origin/master -- "doc**.*m*" "about**.*m*" "support**.*m*" "install**.*m*" "upgrade**.*m*"
+- Rephrased our [[FAQ about VPN|support/faq#vpn]]. ([[!tails_ticket 15783]])
+
+- Wrote an [[FAQ about 32-bit computers|support/faq#32-bit]]. ([[!tails_ticket 17866]])
+
+- Fixed all outdated links to torproject.org. ([[!tails_ticket 17717]])
+
+- Used ikiwiki's automatic referencing of HTML ids in table of content
+  on the pages that have the longest tables of content. ([[!tails_ticket 17844]])
+
+  Now you can copy links directly from the table of content to point to:
+
+  <https://tails.boum.org/support/faq/#javascript>
 
 User experience
 ===============
 
-XXX: If you feel like it and the UX team does not do it
-     themselves, check the archives of tails-ux:
-     <https://lists.autistici.org/list/tails-XXX.html>
+- Analyzed the responses of the [[OpenPGP and Pidgin survey|blueprint/user_survey#openpgp-pidgin]]. ([[!tails_ticket 17821]])
+
+  It's pretty exciting because we barely have quantitative data about
+  our user base at large.
+
+  Highlights:
+
+  * Our users find Tails relatively easy to use compared to an industry
+    average.
+
+  * Linux users are still the biggest share of the respondents and this
+    hasn't changed much since 2017.
+
+  * The top 5 wishes of our users are, in rough order:
+
+    1. Do nothing because Tails is great already :)
+
+    2. More persistent settings: security level in Tor Browser,
+       background, keyboard, and language
+
+    3. Messaging applications and voice calls
+
+    4. Make Tor less painful for web browsing
+
+    5. Better upgrades
+
+  * OpenPGP is much more popular outside Thunderbird: 49% vs. 16%.
+
+  * Enigmail users are more technical than OpenPGP users overall.
+
+  * Electrum (28%) is more used than Thunderbird (27%), which is more used than XMPP (17%).
+
+  * IRC is used by a non-negligible 8% of respondents.
 
 Hot topics on our help desk
 ===========================
@@ -62,21 +140,86 @@ can count on for this difficult times. Take care!
    read [[the page shown on the error message|https://tails.boum.org/gdm]]
    before mailing us, because there might be a solution for your graphics card.
 
-1. Some users had problems to do automatic upgrades to Tails 4.10. We are
-   looking onto [[!tails_ticket 17908 desc="this problem"]].
-
+1. Some users [[!tails_ticket 17908 desc="had problems"]] to do automatic
+   upgrades to Tails 4.10. Update: this was fixed.
 
 Infrastructure
 ==============
 
+- Set up infrastructure that will allow Release Managers to upload to
+  `git-annex` faster ([[!tails_gitlab tails/sysadmin/-/issues/17687]]).
+
+- Started coordination for requiring Buster or newer for running our test suite
+  ([[!tails_ticket 17842]]).
+
+- Started upgrading our Jenkins "isotester" workers to Buster.
+
+- [[Summed up|blueprint/CI_usability]] the findings from "Gather usability data
+  about our current CI" ([[!tails_ticket 16959]]).
+
 Funding
 =======
 
-XXX: The fundraising team should look at the fundraising Git.
+Grants
+------
+
+- Our grant to [Reset](https://reset.tech/) from May 1 was rejected.
+
+- We applied to [RIPE NCC](https://www.ripe.net/support/cpf/call-for-applications)
+  to work on usability improvements for censorship circumvention.
+
+Donations
+---------
+
+- We [[surveyed "lapsed donors"|blueprint/donation_process#survey]] who haven't
+  donated since 2019 and analyzed 121 responses. Conclusions:
+
+  * We're doing good! Negative comments were few exceptions in a mass of
+    love: the work that we do is very important and we do it well.
 
-       git log --patch --since='1 December' --until='1 January' origin/master
+  * We should communicate a bit more and better with past donors:
 
-XXX: The fundraising and accounting teams should look at the archives of <tails-fundraising@boum.org> and <tails-accounting@boum.org>.
+    - The main reason for lapsed donors not to renew their donation is
+      lack of awareness.
+    - We should write a newsletter to donors half-way through the year
+      or when we have major improvements.
+    - Adding an opt-in mechanism for our newsletter would allow us to be
+      less shy to write them.
+    - LimeSurvey is more convenient to send mass emails to donors.
+    - We should communicate more about the impact of donations and how
+      we spend the money.
+
+  * About our donors and their interests:
+
+    - Donors are mostly users (74%) but they are really into helping
+      others (70%).
+
+    - Donors mostly talk about:
+
+      * Privacy (24)
+      * Protection from oppressive governments and persecution (18)
+      * Liberty and freedom of speech (16)
+
+- We merged a first batch of improvements to our [[Donate|donate]] page:
+
+  * To better explain our mission:
+
+    - We wrote new slogans.
+    - We improved the text on the left.
+
+  * To better explain how we spend our money:
+
+    - We simplified the chart and added short text to explain each of
+      maintenance, improvements, user experience, and administration.
+    - We added a section about our impact with numbers of daily users.
+
+  * To increase trust:
+
+    - We duplicated the recommendations from the Home page.
+    - We improved the color scheme of the charts.
+
+Partners
+--------
 
 - We added Freiheitsfoo as [[partner|partners]].
 
@@ -104,6 +247,8 @@ XXX: Copy content from [[!tails_gitweb wiki/src/press/media_appearances_2020.mdw
 Translations
 ============
 
+- We now import all reviewed translations from Transifex ([[!tails_mr 113]]).
+
 XXX: Add the output of (adjust month!):
 
     sudo apt-get install intltool
@@ -114,8 +259,6 @@ XXX: Add the output of (adjust month!):
 Metrics
 =======
 
-* Tails has been started more than BOOTS/MONTH times this month. This makes BOOTS/DAY boots a day on average.
+* Tails has been started more than 939&#8239;540 times this month. This makes 30&#8239;308 boots a day on average.
 
 [[How do we know this?|support/faq#boot-statistics]]
-
-XXX: Ask <tails@boum.org> for this number.

wiki/src/blueprint/user_survey.mdwn

@@ -4,6 +4,8 @@
 
 [[!toc levels=2]]
 
+<a id="openpgp-pidgin"></a>
+
 OpenPGP and Pidgin survey ([[!tails_ticket 17821]])
 ===================================================
 

wiki/src/contribute/how/documentation/release_notes.mdwn

@@ -8,11 +8,28 @@
     - Tails changelog
       - [[!tails_gitlab tails/tails/-/blob/stable/debian/changelog]]
       - [[!tails_gitlab tails/tails/-/blob/testing/debian/changelog]]
-        - If the changelog of the final release has not been written
-          yet, either book some time to update the release notes on the
-          day of the release or go through:
-           - the [[!tails_gitlab tails/tails/-/merge_requests desc="Merge Requests (MRs)"]]
-           - the issues [[!tails_gitlab groups/tails/-/milestones desc="fixed for the upcoming release"]]
+      - If the changelog of the final release has not been written
+        yet, either book some time to update the release notes on the
+        day of the release, or generate the changelog entry yourself:
+
+        1. `sudo apt install python3-debian python3-gitlab python3-jinja2`
+
+        2. Ensure `~/.python-gitlab.cfg` has at least this content:
+