Commit d854c55f authored by kytv's avatar kytv
Browse files

Merge branch 'devel' into feature/5663-return-to-icedove

parents 733371c5 4c16d919
......@@ -5,12 +5,12 @@
# Run only when the interface is not "lo":
if [ $1 = "lo" ]; then
exit 0
exit 0
fi
# Run whenever an interface gets "up", not otherwise:
if [ $2 != "up" ]; then
exit 0
exit 0
fi
# Import tor_control_setconf(), TOR_LOG
......@@ -35,26 +35,41 @@ rm -f "${TOR_LOG}"
# a HTTP proxy or allowed firewall ports won't get the sandboxing, but
# much better than nothing.
if [ "$(tails_netconf)" = "direct" ]; then
tor_set_in_torrc Sandbox 1
tor_set_in_torrc Sandbox 1
fi
# A SIGHUP should be enough but there's a bug in Tor. Details:
# We would like Tor to be started during init time, even before the
# network is up, and then send it a SIGHUP here to make it start
# bootstrapping swiftly, but it doesn't work because of a bug in
# Tor. Details:
# * https://trac.torproject.org/projects/tor/ticket/1247
# * https://tails.boum.org/bugs/tor_vs_networkmanager/
restart-tor
# To work around this we restart Tor, in various ways, no matter the
# case below.
if [ "$(tails_netconf)" = "obstacle" ]; then
# When using a bridge Tor reports TLS cert lifetime errors
# (e.g. when the system clock is way off) with severity "info", but
# when no bridge is used the severity is "warn". tordate/20-time.sh
# depends on grepping these error messages, so we temporarily
# increase Tor's logging severity.
tor_control_setconf "Log=\"info file ${TOR_LOG}\""
# Enable the transports we support. We cannot do this in general,
# when bridge mode is not enabled, since we then use seccomp
# sandboxing.
tor_control_setconf 'ClientTransportPlugin="obfs2,obfs3,obfs4 exec /usr/bin/obfs4proxy managed"'
/usr/local/sbin/tails-tor-launcher &
# We do not use retart-tor since it validates that bootstraping
# succeeds. That cannot happen until Tor Launcher has started
# (below) and the user is done configuring it.
service tor restart
# When using a bridge Tor reports TLS cert lifetime errors
# (e.g. when the system clock is way off) with severity "info", but
# when no bridge is used the severity is "warn". tordate/20-time.sh
# depends on grepping these error messages, so we temporarily
# increase Tor's logging severity.
tor_control_setconf "Log=\"info file ${TOR_LOG}\""
# Enable the transports we support. We cannot do this in general,
# when bridge mode is not enabled, since we then use seccomp
# sandboxing.
tor_control_setconf 'ClientTransportPlugin="obfs2,obfs3,obfs4 exec /usr/bin/obfs4proxy managed"'
/usr/local/sbin/tails-tor-launcher &
# Wait until the user has done the Tor Launcher configuration.
until [ "$(tor_control_getconf DisableNetwork)" = 0 ]; do
sleep 1
done
else
( restart-tor ) &
fi
......@@ -70,7 +70,7 @@ has_only_unverified_consensus() {
wait_for_tor_consensus_helper() {
tries=0
while ! has_consensus && [ $tries -lt 5 ]; do
while ! has_consensus && [ $tries -lt 10 ]; do
inotifywait -q -t 30 -e close_write -e moved_to ${TOR_DIR} || log "timeout"
tries=$(expr $tries + 1)
done
......@@ -81,10 +81,6 @@ wait_for_tor_consensus_helper() {
wait_for_tor_consensus() {
log "Waiting for a Tor consensus file to contain a valid time interval"
if ! has_consensus && ! wait_for_tor_consensus_helper; then
log "Unsuccessfully waited for Tor consensus, restarting Tor and retrying."
restart-tor
fi
if ! has_consensus && ! wait_for_tor_consensus_helper; then
log "Unsuccessfully retried waiting for Tor consensus, aborting."
fi
......@@ -175,7 +171,7 @@ maybe_set_time_from_tor_consensus() {
date -us "${vmid}" 1>/dev/null
# Tor is unreliable with picking a circuit after time change
restart-tor
service tor restart
}
tor_cert_valid_after() {
......@@ -219,15 +215,6 @@ start_notification_helper() {
### Main
# When the network is obstacled (e.g. we need a bridge) we wait until
# Tor Launcher has unset DisableNetwork, since Tor's bootstrapping
# won't start until then.
if [ "$(tails_netconf)" = "obstacle" ]; then
until [ "$(tor_control_getconf DisableNetwork)" = 0 ]; do
sleep 1
done
fi
start_notification_helper
# Delegate time setting to other daemons if Tor connections work
......
......@@ -2,13 +2,58 @@
set -e
# Import try_for()
. /usr/local/lib/tails-shell-library/common.sh
# Import tor_bootstrap_progress()
. /usr/local/lib/tails-shell-library/tor.sh
# Import log()
. /usr/local/lib/tails-shell-library/log.sh
_LOG_TAG="$(basename $0)"
# The Tor log is removed to ensure `tor_bootstrap_progress`'s output will be
# accurate.
clear_tor_log() {
rm -f /var/log/tor/log
}
clear_tor_log
service tor restart
# The main point of this script is to make sure that if vidalia is
# running, and Tor is restarted, then we also restart Vidalia. This is
# because Vidalia doesn't re-connect to Tor automatically, so the user
# has to restart it to be able to control Tor again. Also, any options
# set by Vidalia will be lost since they weren't written to torrc.
# There are two main points to this script:
# * restarting Tor if bootstrapping stalls for more than 20 seconds
# * making sure that if vidalia is running it is restarted if Tor is restarted.
# This is needed because Vidalia doesn't re-connect to Tor automatically,
# so the user has to restart it to be able to control Tor again. Also, any
# options set by Vidalia will be lost since they weren't written to torrc.
bootstrap_progress=0
last_bootstrap_change=$(date +%s)
maybe_restart_tor() {
local new_bootstrap_progress=$(tor_bootstrap_progress)
if [ $new_bootstrap_progress -eq 100 ]; then
log "Tor has successfully bootstrapped."
return 0
elif [ $new_bootstrap_progress -gt $bootstrap_progress ]; then
bootstrap_progress=$new_bootstrap_progress
last_bootstrap_change=$(date +%s)
return 1
elif [ $(expr $(date +%s) - $last_bootstrap_change) -ge 20 ]; then
log "Tor seems to have stalled while bootstrapping. Restarting Tor."
clear_tor_log
service tor restart
bootstrap_progress=0
last_bootstrap_change=$(date +%s)
return 1
else
return 1
fi
}
try_for 270 maybe_restart_tor
if pgrep "\<vidalia\>" >/dev/null; then
killall -SIGKILL vidalia
# Since Tor just restarted we wait for a while until the
......
......@@ -2,7 +2,7 @@ tails (1.7) UNRELEASED; urgency=medium
* Dummy entry for next major release.
-- intrigeri <intrigeri@debian.org> Sat, 08 Aug 2015 13:35:23 +0200
-- intrigeri <intrigeri@debian.org> Sat, 22 Sep 2015 18:54:32 +0200
tails (1.6) unstable; urgency=medium
......
......@@ -63,11 +63,9 @@ def restore_background
if @vm.execute("service tor status").success?
@vm.execute("service tor stop")
@vm.execute("rm -f /var/log/tor/log")
@vm.execute("killall vidalia")
@vm.host_to_guest_time_sync
@vm.execute("service tor start")
@vm.spawn("restart-tor")
wait_until_tor_is_working
@vm.spawn("restart-vidalia")
end
else
@vm.host_to_guest_time_sync
......
......@@ -87,6 +87,7 @@ module ExtraFormatters
def debug_log(message)
@io.puts(format_string(message, :blue))
@io.flush
end
end
......
......@@ -103,6 +103,14 @@ end
def wait_until_tor_is_working
try_for(270) { @vm.execute(
'. /usr/local/lib/tails-shell-library/tor.sh; tor_is_working').success? }
rescue Timeout::Error => e
c = @vm.execute("grep restart-tor /var/log/syslog")
if c.success?
debug_log("From syslog:\n" + c.stdout.sub(/^/, " "))
else
debug_log("Nothing was syslog:ed about 'restart-tor'")
end
raise e
end
def convert_bytes_mod(unit)
......
[[!meta title="Porting Tails to Debian Stretch"]]
This is about the effort to create [Tails
3.0](https://labs.riseup.net/code/projects/tails/issues?query_id=198),
based on Debian Stretch.
[[!toc levels=2]]
# Big picture
When porting to Wheezy, our main problem has been that we started the
work too late, which made us discover Debian bugs too late (after the
Debian Wheezy freeze, or even after its release), and in the end we
had to workaround lots of problems on our side.
So we started much earlier the porting work to Jessie, which indeed
essentially avoided the aforementioned problem. But we didn't allocate
enough focused resources to this effort, and as a result the total
duration of the porting to Jessie work is lasting too much, and we're
spending too much time just keeping our feature/jessie branch
up-to-date wrt. the changes we were making in our Wheezy
production branches.
For Stretch we'd like to avoid both problems. We want to start early,
in order to fix problems directly in Debian Stretch before it's
released. And, we want the porting work to fit into a shorter time
span, so as soon as we start we'll allocate more resources to it, and
in a better organized, team-based and focused way.
Additionally, we would like to use this process as an opportunity to
evaluate the idea of basing Tails on snapshots of Debian testing.
# Schedule
* 2016Q1 — Tails 2.0 is out
* April or May 2016 — start working on Tails 3.0 (1 week sprint with
all involved people) :intrigeri:anonym:kytv:
- get feature/stretch to build and boot
- update the automated test suite to test Tails/Stretch ISO images
* May 2016 to April 2017 — have one dedicated half-time, 1-week sprint
every 6 weeks.
- Schedule these sprints in advance, announce them
publicly, and invite other contributors (e.g. doc writers).
- Most of these sprints will probably be attended remotely, but at
least some could happen face-to-face.
- At the beginning of each Stretch sprint, we import a new snapshot
of Debian testing into our freezable APT repository, so that:
* during the sprint we update our stuff as required by changes
that happened in Debian since the last sprint;
* our stuff is not broken by changes in Debian neither during
sprints, nor between them;
* we get a feeling of how "being based on snapshots of Debian
testing" would work.
* December 5 2016 — Debian Stretch freeze starts
* April 2017 (???) — Debian Stretch is released
* April-June 2017 — Tails 3.0 is released
# Let's go rolling
Let's use this porting cycle to evaluate how being based on snapshots
of Debian testing would feel like. During the entire cycle:
* Keep the automated test suite up-to-date on feature/stretch :kytv:
* Keep the documentation up-to-date on feature/stretch :doc_writer:
* Take notes of issues we see from the perspective of "how would it go
if we were based on testing already?". E.g.:
- How to deal with ongoing transitions that block migration of
security updates from sid to testing?
Note that during half of the cycle, we'll be based on a frozen Debian
testing, so the changes rate coming from Debian won't be crazy.
Of course there will be changes coming from our own porting efforts.
And then, once Tails 3.0 is out: we're not lagging behind anymore, and
are thus in a position to decide whether we want to base Tails on
snapshots of Debian testing.
......@@ -11,6 +11,9 @@
a desktop file whose base name matches the application id, then your
notification will not show up". We'll see.
* intrigeri has a working Perl example with `add_button`. Next step is
to pass parameters to the notification.
* If it's deemed overkill, or just too painful, to port every single
script that needs to send notifications with actions to
"GApplication + D-Bus activation + actions support" properly,
......@@ -40,7 +43,6 @@
use Gtk3;
use Glib::Object::Introspection;
use Try::Tiny;
Glib::Object::Introspection->setup(
basename => 'Gio',
......@@ -58,8 +60,7 @@
# If gedit is already running, this fails with:
# GLib-GIO-CRITICAL **: g_application_send_notification:
# assertion '!g_application_get_is_remote (application)' failed
$app->send_notification("my-notification-id", $notification)
or die "Could not send notification";
$app->send_notification("my-notification-id", $notification);
* private D-Bus API (`org.gtk.Notifications`):
<https://wiki.gnome.org/Projects/GLib/GNotification> ... maybe an
......@@ -87,3 +88,10 @@
{ label => "New document", action => "app.new-document"},
],
});
* GNOME Shell 3.16: non-default action buttons are only displayed when
the notification pops up initially, _not_ in the [Message
List](https://wiki.gnome.org/Design/OS/Notifications/#Message_List).
- Is it the same on Jessie?
- Can we get away with only a default action that opens the URL we
want in the use cases we have in practice?
......@@ -9,9 +9,40 @@ If I had to mention the ideal design goals for such changes, I would say
that the more straightforward would be the better for implementation and
also for maintainability.
[[!toc levels=2]]
[[!toc levels=3]]
# Using DNS
# The plan
## Big picture
We decided to implement a two-way strategy for this feature:
* Use JavaScript to modify the link on the download page, so that each
user is pointed to random mirror.
- Vanilla JS (no frameworks)
- Store the JS code and its configuration file in two dedicated
ikiwiki overlays, for finer-grained access control possibilities
to it (e.g. we may want to let people who don't have commit access
to Git maintain the mirrors pool).
- Configuration for the JS is loaded from JSON file. It attaches
a weight to each mirror. Weight 0 means that the mirror is
currently disabled, and will never be redirected to. We're using
JSON and not YAML to avoid the need to use a third-party parser.
* Keep using DNS to point to 3-5 fast and reliable mirrors. This will
be the fallback for people who do not use JS. So we still need a DNS
dynamic update system; we can simply re-purpose the one we already
have (`dl.amnesia.boum.org`).
* The ISO verification extension also needs to use mirrors. So, we'll
be providing library code for it to do the same, internally, as
a web browser would do when visiting the Tails download page, that
is replacing the hostname, in a ISO download URL, with a suitable
mirror's hostname (using the JSON mirror pool configuration file).
# Initial research
## Using DNS
Using DNS seems to be an easy way to do some round robin in low level. It
allows some kind of transparency to the upper layers protocols and
......@@ -24,7 +55,7 @@ The following ways are available to implement it:
* NS Hacks
* Modified DNS servers
## CNAME Hacks
### CNAME Hacks
As mention by ToBeFree something that can be done is to have different
pools of servers like:
......@@ -55,7 +86,7 @@ that has not been ported to bind 9. Neither NSD nor PowerDNS seem to
support it, and their is no actual data about how resolvers would
handle this case, so I don't think it is the best option.
## NS Hacks
### NS Hacks
Following the same idea the dl amnesia.boum.org could be delegated to a
few different DNS servers, and those servers may have different versions
......@@ -82,7 +113,7 @@ CNAME hacks, almost as the NS servers will not receive 50% of requests
because of [[!rfc 5452]]). However, I am not sure that playing with DNS
inconsistency will be a so good idea, for example for maintainability :)
## Using modified DNS servers
### Using modified DNS servers
Interestingly Tails is not the first project to be looking how to use DNS
for load distribution. People already wrote some DNS software designed to
......@@ -102,7 +133,7 @@ Deploying such software would solve the problem in a more elegant way than
CNAME or NS hacks. It would require a bit of system administration that
maybe can be done using some puppet templates in a few Virtal Machines.
# Using HTTP(s)
## Using HTTP(s)
DNS is not the only way to do some load balancing. It is mostly used for
low level protocols that don't allow redirects (for example: NTP). As
......@@ -147,13 +178,13 @@ Thus, if I may, I would like to recommend considering the HTTP(s) option,
even if it means that I have to write the PHP script by myself or to create
an easy task entry on the ticket tracker and follow it :)
# Proof of concept: JavaScript + multiple DNS pools / named mirrors
## Proof of concept: JavaScript + multiple DNS pools / named mirrors
This method can either be used with multiple DNS pools (dl1.amnesia.boum.org, dl2.amnesia.boum.org etc.) or with named mirrors (freiwuppertal.dl.amnesia.boum.org, othermirror.dl.amnesia.boum.org, ...). Using named mirrors allows you to use a huge, unlimited list of completely equally used mirrors; using multiple DNS pools leads to effects described under "CNAME hacks".
These POCs should be 1:1 usable on the Tails [[download]] page. All that would be needed is setting up the DNS pools and/or named mirrors, and telling the mirror owners to configure their servers to respond to \*.amnesia.boum.org (the wildcard is important).
## JavaScript POC (multiple DNS pools)
### JavaScript POC (multiple DNS pools)
<script src="//code.jquery.com/jquery.min.js"></script>
<script type="text/javascript">//<![CDATA[
......@@ -171,12 +202,12 @@ For this to work and to be flexible, mirrors need to respond to \*.amnesia.boum.
At least nginx is unable to use a wildcard like dl\*.amnesia.boum.org, so \*.amnesia.boum.org has to be used. This is more flexible anyway.
### Example webpage (see the webpage source there too)
#### Example webpage (see the webpage source there too)
<http://freiwuppertal.de/tails-mirror-example-dns.htm>
## JavaScript POC (named mirrors)
### JavaScript POC (named mirrors)
<script src="//code.jquery.com/jquery.min.js"></script>
<script type="text/javascript">//<![CDATA[
......@@ -192,16 +223,108 @@ At least nginx is unable to use a wildcard like dl\*.amnesia.boum.org, so \*.amn
For this to work and to be flexible, mirrors need to respond to \*.amnesia.boum.org. Just responding to a fixed name would make this an unflexible solution, so the wildcard is needed.
### Example webpage (see the webpage source there too)
#### Example webpage (see the webpage source there too)
<http://freiwuppertal.de/tails-mirror-example-named.htm>
### Giving mirrors higher or lower weight
#### Giving mirrors higher or lower weight
Using this approach, giving one mirror more weight than others is very easy: Simply add it's name multiple times to the array of mirrors. :D
### Vanilla JavaScript POC and JSON
<a href="http://dl.amnesia.boum.org/tails/stable/tails-i386-1.6/tails-i386-1.6.iso" id="dllink">download link</a>
<script type="text/javascript">
function fetchJSONdata(path, callback) {
var xhr = new XMLHttpRequest();
xhr.onreadystatechange = function() {
if (xhr.readyState === 4) {
if (xhr.status === 200 || xhr.status === 0) {
var data = JSON.parse(xhr.responseText);
if (callback) callback(data);
} else {
console.log( "Error: " + xhr.statusText);
}
}
};
xhr.open('GET', path, true);
xhr.send();
}
function getRandomInt(min, max) {
return Math.floor(Math.random() * (max - min +1)) + min;
}
function isJSON(str) {
try {
JSON.parse(str);
} catch (e) {
return false;
}
return true;
}
function replaceDownloadURL(updatedURL) {
var URLMarker = "/tails/stable";
// todo check that url is a correct url
var linkDOMElem = document.getElementById('dllink');
var linkHREF = linkDOMElem.href.split( '//' );
var linkToISO = linkHREF[1].split( URLMarker );
// fixme http or https
linkDOMElem.href = '//' + updatedURL + URLMarker + linkToISO[1];
return true;
}
fetchJSONdata('./mirrors.json', function(data){
//console.log(data);
if( data == "undefined" ) {
console.log( "Error: mirror data not loaded.");
} else if( !isJSON( JSON.stringify(data) ) ) {
console.log( "Error: mirror data is not JSON.");
} else {
//console.log(data.mirrors);
// todo delete all mirrors with weight 0 before choosing one
if(data.mirrors.length > 0 ) {
var activeMirrors = new Array();
for ( i = 0; i < data.mirrors.length; i++ ) {
if ( data.mirrors[i].weight != 0 ) {
// add mirror as many times as its weight, max weight is 5
if ( parseInt(data.mirrors[i].weight ) > 5) {
var max_weight = 5;
} else {
var max_weight = parseInt( data.mirrors[i].weight );
}
for ( w = 0; w < max_weight; w++ ) {
activeMirrors.push( data.mirrors[i] );
}
}
}
console.log(activeMirrors);
var randomMirror = getRandomInt(0, activeMirrors.length-1);
//console.log(randomMirror);
//console.log(data.mirrors[randomMirror]);
replaceDownloadURL(activeMirrors[randomMirror].url);
}
}
});
</script>
The mirrors.json file contains:
<pre>
{
"mirrors": [
{ "url": "1.dl.amnesia.boum.org", "weight": "10" },
{ "url": "5.dl.amnesia.boum.org", "weight": "5" },
{ "url": "6.dl.amnesia.boum.org", "weight": "6" },
{ "url": "3.dl.amnesia.boum.org", "weight": "0" }
]
}
</pre>
# PHP: first draft
## PHP: first draft
// http://stackoverflow.com/questions/4233407/get-random-item-from-array
......
......@@ -117,19 +117,18 @@ Basic configuration & integration
* Don't display the "Adblock Plus installation complete" tab.
* Don't prompt whether one wants to report usage and performance
information to Mozilla.
* Disable FoxyProxy.
* Enable "Only use secure protocols" by default (one may still
uncheck it when needed).
* Don't check updates for Add-ons.
* Add launcher to the GNOME panel.
* More generally: have a look at our Iceweasel prefs and copy all
* More generally: have a look at our Tor Browser prefs and copy all
those that exist and make sense for Icedove.
* The [[security/IP_address_leak_with_icedove]] can be fixed by
setting `mail.smtpserver.default.hello_argument` to "localhost".
See [this Tor wiki
entry](https://trac.torproject.org/projects/tor/wiki/TheOnionRouter/TorifyHOWTO/EMail#ExperimentalSuggestionsforpossiblymakingthunderbirdandorclawsstopleakinginfoExperimental)
for other goodies. By applying those configurations I think both
claws and icedove comes to an equal level security-wise.
claws and icedove comes to an equal level security-wise. (Note: This will be set by TorBirdy)
* Disable by default the indexer from
`Preferences -> Advanced -> General -> Enable Global Search and Indexer`.
Otherwise pinentry dialogs can appear while checking email in the
......
......@@ -137,9 +137,9 @@ decide to still use them, then we probably have to wait for
Proposal for a first iteration:
* For green test suite run: keep the test logs (Jenkins natively do
that) and video captures
* For red test suite run: keep the screen and video captures and the
logs.
that).
* For red test suite run: keep the screen and video captures, the