From 2593ac8509d19c927779d38447dc37d971271d4e Mon Sep 17 00:00:00 2001 From: sajolida Date: Tue, 16 Jun 2020 21:08:35 +0000 Subject: [PATCH 1/4] Optimize timing to the schedule of workers (#17755) --- manifests/check_mirrors.pp | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/manifests/check_mirrors.pp b/manifests/check_mirrors.pp index 2cabd5c7..f3d0a4c2 100644 --- a/manifests/check_mirrors.pp +++ b/manifests/check_mirrors.pp @@ -70,8 +70,9 @@ class tails::check_mirrors ( ], } + # Full run cron { 'tails_check_mirrors': - command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb\"", + command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb --ignore-failures failures.json\"", user => $user, hour => 0, minute => 16, @@ -89,4 +90,37 @@ class tails::check_mirrors ( environment => [ 'TMPDIR=/tmp' ], } + # Fast run #1: optimized for European morning + cron { 'tails_check_mirrors': + command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb --store-failures failures.json\"", + user => $user, + hour => 4, + minute => 16, + require => [Vcsrepo[$repo_checkout], Package[$needed_packages], + Postfix::Mailalias[$user], Exec['Import Tails signing key']], + environment => [ "TMPDIR=/tmp" ], + } + + # Fast run #2: optimized for American morning + cron { 'tails_check_mirrors': + command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb --store-failures failures.json\"", + user => $user, + hour => 12, + minute => 16, + require => [Vcsrepo[$repo_checkout], Package[$needed_packages], + Postfix::Mailalias[$user], Exec['Import Tails signing key']], + environment => [ "TMPDIR=/tmp" ], + } + + # Fast run #3: to complete the 3 × 8 shifts + cron { 'tails_check_mirrors': + command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb --store-failures failures.json\"", + user => $user, + hour => 20, + minute => 16, + require => [Vcsrepo[$repo_checkout], Package[$needed_packages], + Postfix::Mailalias[$user], Exec['Import Tails signing key']], + environment => [ "TMPDIR=/tmp" ], + } + } -- GitLab From 55169b12a2d3e25010a7080d0c5085aa38fd3080 Mon Sep 17 00:00:00 2001 From: intrigeri Date: Sat, 27 Jun 2020 11:49:08 +0000 Subject: [PATCH 2/4] check_mirrors: add missing --fast argument for "fast" runs refs: tails/tails#17755 --- manifests/check_mirrors.pp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manifests/check_mirrors.pp b/manifests/check_mirrors.pp index f3d0a4c2..42d858a1 100644 --- a/manifests/check_mirrors.pp +++ b/manifests/check_mirrors.pp @@ -92,7 +92,7 @@ class tails::check_mirrors ( # Fast run #1: optimized for European morning cron { 'tails_check_mirrors': - command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb --store-failures failures.json\"", + command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb --fast --store-failures failures.json\"", user => $user, hour => 4, minute => 16, @@ -103,7 +103,7 @@ class tails::check_mirrors ( # Fast run #2: optimized for American morning cron { 'tails_check_mirrors': - command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb --store-failures failures.json\"", + command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb --fast --store-failures failures.json\"", user => $user, hour => 12, minute => 16, @@ -114,7 +114,7 @@ class tails::check_mirrors ( # Fast run #3: to complete the 3 × 8 shifts cron { 'tails_check_mirrors': - command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb --store-failures failures.json\"", + command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb --fast --store-failures failures.json\"", user => $user, hour => 20, minute => 16, -- GitLab From 9c6eecae6d449ca2e4c5c9726b0df4deb7f1c421 Mon Sep 17 00:00:00 2001 From: intrigeri Date: Sat, 27 Jun 2020 12:05:42 +0000 Subject: [PATCH 3/4] check_mirrors: refactoring (factorize) and fix Puppet syntax mistakes refs: tails/tails#17755 --- manifests/check_mirrors.pp | 61 ++++++++---------------------- manifests/check_mirrors/cronjob.pp | 35 +++++++++++++++++ 2 files changed, 51 insertions(+), 45 deletions(-) create mode 100644 manifests/check_mirrors/cronjob.pp diff --git a/manifests/check_mirrors.pp b/manifests/check_mirrors.pp index 42d858a1..10f265cf 100644 --- a/manifests/check_mirrors.pp +++ b/manifests/check_mirrors.pp @@ -70,57 +70,28 @@ class tails::check_mirrors ( ], } - # Full run - cron { 'tails_check_mirrors': - command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb --ignore-failures failures.json\"", - user => $user, - hour => 0, - minute => 16, - require => [Vcsrepo[$repo_checkout], Package[$needed_packages], - Postfix::Mailalias[$user], Exec['Import Tails signing key']], - # By default, check-mirrors.rb would create its per-run temporary - # directory in $CWD, i.e. in this context $HOME, which is never - # cleaned up automatically; so interrupted runs would leave - # temporary files behind them forever. Let's ask check-mirrors.rb - # to instead create its per-run temporary directory in a place - # that is cleaned up on boot. This requires ~2.5 GiB free space - # in /tmp so if we ever need to include this class on a system - # that has a smaller tmpfs mounted there, we'll need to make - # this configurable. - environment => [ 'TMPDIR=/tmp' ], + tails::check_mirrors::cronjob { 'Full run': + args => ['--ignore-failures', 'failures.json'], + hour => 0, + minute => 16, } - # Fast run #1: optimized for European morning - cron { 'tails_check_mirrors': - command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb --fast --store-failures failures.json\"", - user => $user, - hour => 4, - minute => 16, - require => [Vcsrepo[$repo_checkout], Package[$needed_packages], - Postfix::Mailalias[$user], Exec['Import Tails signing key']], - environment => [ "TMPDIR=/tmp" ], + tails::check_mirrors::cronjob { 'Fast run #1: optimized for European morning': + args => ['--fast', '--store-failures', 'failures.json'], + hour => 4, + minute => 16, } - # Fast run #2: optimized for American morning - cron { 'tails_check_mirrors': - command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb --fast --store-failures failures.json\"", - user => $user, - hour => 12, - minute => 16, - require => [Vcsrepo[$repo_checkout], Package[$needed_packages], - Postfix::Mailalias[$user], Exec['Import Tails signing key']], - environment => [ "TMPDIR=/tmp" ], + tails::check_mirrors::cronjob { 'Fast run #2: optimized for American morning': + args => ['--fast', '--store-failures', 'failures.json'], + hour => 12, + minute => 16, } - # Fast run #3: to complete the 3 × 8 shifts - cron { 'tails_check_mirrors': - command => "sleep \$(( \$( > 5 )) && \"${repo_checkout}/check-mirrors.rb --fast --store-failures failures.json\"", - user => $user, - hour => 20, - minute => 16, - require => [Vcsrepo[$repo_checkout], Package[$needed_packages], - Postfix::Mailalias[$user], Exec['Import Tails signing key']], - environment => [ "TMPDIR=/tmp" ], + tails::check_mirrors::cronjob { 'Fast run #3: to complete the 3 × 8 shifts': + args => ['--fast', '--store-failures', 'failures.json'], + hour => 20, + minute => 16, } } diff --git a/manifests/check_mirrors/cronjob.pp b/manifests/check_mirrors/cronjob.pp new file mode 100644 index 00000000..00182c20 --- /dev/null +++ b/manifests/check_mirrors/cronjob.pp @@ -0,0 +1,35 @@ +# Manage a cronjob that runs check-mirrors.rb +define tails::check_mirrors::cronjob ( + Integer[0, 23] $hour, + Integer[0, 59] $minute, + Array[String] $args = [], +) { + + assert_private() + + $args_str = join($args, ' ') + + cron { "tails_check_mirrors ${title}": + command => "sleep \$(( \$( > 5 )) && \"${tails::check_mirrors::repo_checkout}/check-mirrors.rb\" ${args_str}", # lint:ignore:140chars -- command + user => $tails::check_mirrors::user, + hour => $hour, + minute => $minute, + require => [ + Vcsrepo[$tails::check_mirrors::repo_checkout], + Package[$tails::check_mirrors::needed_packages], + Postfix::Mailalias[$tails::check_mirrors::user], + Exec['Import Tails signing key'], + ], + # By default, check-mirrors.rb would create its per-run temporary + # directory in $CWD, i.e. in this context $HOME, which is never + # cleaned up automatically; so interrupted runs would leave + # temporary files behind them forever. Let's ask check-mirrors.rb + # to instead create its per-run temporary directory in a place + # that is cleaned up on boot. This requires ~2.5 GiB free space + # in /tmp so if we ever need to include this class on a system + # that has a smaller tmpfs mounted there, we'll need to make + # this configurable. + environment => [ 'TMPDIR=/tmp' ], + } + +} -- GitLab From 9dbdc6f5a6806cb2a2865d8f16080c42f0e803e0 Mon Sep 17 00:00:00 2001 From: intrigeri Date: Sat, 27 Jun 2020 12:13:08 +0000 Subject: [PATCH 4/4] Ensure only one instance of check-mirrors.rb runs concurrently Now that we're running check-mirrors.rb 4 times more often, there's greater chances that 2 of them end up running at the same time. Let's avoid having to reason about how check-mirrors.rb manages shared resources such as its failures.json state file. refs: tails/tails#17755 --- manifests/check_mirrors/cronjob.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/check_mirrors/cronjob.pp b/manifests/check_mirrors/cronjob.pp index 00182c20..e0204a4d 100644 --- a/manifests/check_mirrors/cronjob.pp +++ b/manifests/check_mirrors/cronjob.pp @@ -10,7 +10,7 @@ define tails::check_mirrors::cronjob ( $args_str = join($args, ' ') cron { "tails_check_mirrors ${title}": - command => "sleep \$(( \$( > 5 )) && \"${tails::check_mirrors::repo_checkout}/check-mirrors.rb\" ${args_str}", # lint:ignore:140chars -- command + command => "sleep \$(( \$( > 5 )) && flock --nonblock check-mirrors.lock \"${tails::check_mirrors::repo_checkout}/check-mirrors.rb\" ${args_str}", # lint:ignore:140chars -- command user => $tails::check_mirrors::user, hour => $hour, minute => $minute, -- GitLab