From bb10fdfb30948fe096f746fa1b8c1020ab306fe9 Mon Sep 17 00:00:00 2001 From: Sean Whitton Date: Sat, 15 Jul 2017 08:12:42 -0700 Subject: add Apt.proxy, Apt.useLocalCacher --- src/Propellor/Property/Apt.hs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'src/Propellor') diff --git a/src/Propellor/Property/Apt.hs b/src/Propellor/Property/Apt.hs index 686ddb6c..07164df6 100644 --- a/src/Propellor/Property/Apt.hs +++ b/src/Propellor/Property/Apt.hs @@ -20,6 +20,9 @@ import Propellor.Types.Info data HostMirror = HostMirror Url deriving (Eq, Show, Typeable) +data HostAptProxy = HostAptProxy Url + deriving (Eq, Show, Typeable) + -- | Indicate host's preferred apt mirror (e.g. an apt cacher on the host's LAN) mirror :: Url -> Property (HasInfo + UnixLike) mirror u = pureInfoProperty (u ++ " apt mirror selected") @@ -493,3 +496,19 @@ suitePinBlock p suite pin = dpkgStatus :: FilePath dpkgStatus = "/var/lib/dpkg/status" + +-- | Set apt's proxy +proxy :: Url -> Property (HasInfo + DebianLike) +proxy u = tightenTargets $ + proxyInfo `before` proxyConfig `describe` desc + where + proxyInfo = pureInfoProperty desc (InfoVal (HostAptProxy u)) + proxyConfig = "/etc/apt/apt.conf.d/20proxy" `File.hasContent` + [ "Acquire::HTTP::Proxy \"" ++ u ++ "\";" ] + desc = (u ++ " apt proxy selected") + +-- | Cause apt to proxy downloads via an apt cacher on localhost +useLocalCacher :: Property (HasInfo + DebianLike) +useLocalCacher = proxy "http://localhost:3142" + `requires` serviceInstalledRunning "apt-cacher-ng" + -- cgit v1.2.3 From 4be2ef464c066600a4551975aee7efe61f739393 Mon Sep 17 00:00:00 2001 From: Sean Whitton Date: Sat, 15 Jul 2017 08:12:53 -0700 Subject: drop suggestion to use Apt.mirror for a LAN apt cacher It is probably best to use Apt.proxy for this. --- src/Propellor/Property/Apt.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/Propellor') diff --git a/src/Propellor/Property/Apt.hs b/src/Propellor/Property/Apt.hs index 07164df6..2e4757f8 100644 --- a/src/Propellor/Property/Apt.hs +++ b/src/Propellor/Property/Apt.hs @@ -23,7 +23,7 @@ data HostMirror = HostMirror Url data HostAptProxy = HostAptProxy Url deriving (Eq, Show, Typeable) --- | Indicate host's preferred apt mirror (e.g. an apt cacher on the host's LAN) +-- | Indicate host's preferred apt mirror mirror :: Url -> Property (HasInfo + UnixLike) mirror u = pureInfoProperty (u ++ " apt mirror selected") (InfoVal (HostMirror u)) -- cgit v1.2.3 From 1fe45d18e676e930a383d24dad314d8b30612bff Mon Sep 17 00:00:00 2001 From: Sean Whitton Date: Sat, 15 Jul 2017 09:11:59 -0700 Subject: drop Apt.piupartsConf, Apt.piupartsConfFor, Apt.shareAptCache Migrating to use apt-cacher-ng instead of bind mounting the apt cache. This will permit multiple builds to run simultaneously. --- src/Propellor/Property/Sbuild.hs | 107 +++++---------------------------------- 1 file changed, 14 insertions(+), 93 deletions(-) (limited to 'src/Propellor') diff --git a/src/Propellor/Property/Sbuild.hs b/src/Propellor/Property/Sbuild.hs index 460d0b16..12720592 100644 --- a/src/Propellor/Property/Sbuild.hs +++ b/src/Propellor/Property/Sbuild.hs @@ -22,10 +22,8 @@ Suggested usage in @config.hs@: > & Apt.installed ["piuparts", "autopkgtest"] > & Sbuild.builtFor (System (Debian Linux Unstable) X86_32) Sbuild.UseCcache -> & Sbuild.piupartsConfFor (System (Debian Linux Unstable) X86_32) > & Sbuild.updatedFor (System (Debian Linux Unstable) X86_32) `period` Weekly 1 > & Sbuild.usableBy (User "spwhitton") -> & Sbuild.shareAptCache > & Schroot.overlaysInTmpfs If you are using sbuild older than 0.70.0, you also need: @@ -37,7 +35,8 @@ In @~/.sbuildrc@ (sbuild 0.71.0 or newer): > $run_piuparts = 1; > $piuparts_opts = [ > '--schroot', -> '%r-%a-piuparts', +> '--no-eatmydata', +> '%r-%a-sbuild', > '--fail-if-inadequate', > '--fail-on-broken-symlinks', > ]; @@ -74,16 +73,13 @@ module Propellor.Property.Sbuild ( UseCcache(..), built, updated, - piupartsConf, builtFor, updatedFor, - piupartsConfFor, -- * Global sbuild configuration -- blockNetwork, installed, keypairGenerated, keypairInsecurelyGenerated, - shareAptCache, usableBy, ) where @@ -147,7 +143,8 @@ built s@(SbuildSchroot suite arch) mirror cc = ((go `before` enhancedConf) `requires` ccacheMaybePrepared cc `requires` installed - `requires` overlaysKernel) + `requires` overlaysKernel + `requires` cleanupOldConfig) deleted where go :: Property DebianLike @@ -218,6 +215,16 @@ built s@(SbuildSchroot suite arch) mirror cc = Reboot.toKernelNewerThan "3.18" else noChange + -- clean up config from earlier versions of this module + cleanupOldConfig :: Property UnixLike + cleanupOldConfig = property' "old sbuild module config cleaned up" $ \w -> do + void $ ensureProperty w $ + check (doesFileExist fstab) (File.lacksLine fstab aptCacheLine) + liftIO $ removeDirectoryRecursive "/etc/schroot/piuparts" + makeChange $ nukeFile (schrootPiupartsConf s) + where + fstab = "/etc/schroot/sbuild/fstab" + -- A failed debootstrap run will leave a debootstrap directory; -- recover by deleting it and trying again. ispartial = ifM (doesDirectoryExist (schrootRoot s "debootstrap")) @@ -299,92 +306,6 @@ fixConfFile s@(SbuildSchroot suite arch) = tempPrefix = dir suite ++ "-" ++ architectureToDebianArchString arch ++ "-propellor-" munge = replace "-propellor]" "-sbuild]" --- | Create a corresponding schroot config file for use with piuparts --- --- This function is a convenience wrapper around 'piupartsConf', allowing the --- user to identify the schroot using the 'System' type. See that function's --- documentation for why you might want to use this property, and sample config. -piupartsConfFor :: System -> Property DebianLike -piupartsConfFor sys = property' ("piuparts schroot conf for " ++ show sys) $ - \w -> case schrootFromSystem sys of - Just s -> ensureProperty w $ piupartsConf s - _ -> errorMessage - ("don't know how to debootstrap " ++ show sys) - --- | Create a corresponding schroot config file for use with piuparts --- --- This is useful because: --- --- - piuparts will clear out the apt cache which makes 'shareAptCache' much less --- useful --- --- - piuparts itself invokes eatmydata, so the command-prefix setting in our --- regular schroot config would force the user to pass @--no-eatmydata@ to --- piuparts in their @~/.sbuildrc@, which is inconvenient. --- --- To make use of this new schroot config, you can put something like this in --- your ~/.sbuildrc (sbuild 0.71.0 or newer): --- --- > $run_piuparts = 1; --- > $piuparts_opts = [ --- > '--schroot', --- > '%r-%a-piuparts', --- > '--fail-if-inadequate', --- > '--fail-on-broken-symlinks', --- > ]; --- --- This property has no effect if the corresponding sbuild schroot does not --- exist (i.e. you also need 'Sbuild.built' or 'Sbuild.builtFor'). -piupartsConf :: SbuildSchroot -> Property DebianLike -piupartsConf s@(SbuildSchroot _ arch) = - check (doesFileExist (schrootConf s)) go - `requires` installed - where - go :: Property DebianLike - go = property' desc $ \w -> do - aliases <- aliasesLine - ensureProperty w $ combineProperties desc $ props - & check (not <$> doesFileExist f) - (File.basedOn f (schrootConf s, map munge)) - & ConfFile.containsIniSetting f - (sec, "profile", "piuparts") - & ConfFile.containsIniSetting f - (sec, "aliases", aliases) - & ConfFile.containsIniSetting f - (sec, "command-prefix", "") - & File.dirExists dir - & File.isSymlinkedTo (dir "copyfiles") - (File.LinkTarget $ orig "copyfiles") - & File.isSymlinkedTo (dir "nssdatabases") - (File.LinkTarget $ orig "nssdatabases") - & File.basedOn (dir "fstab") - (orig "fstab", filter (/= aptCacheLine)) - - orig = "/etc/schroot/sbuild" - dir = "/etc/schroot/piuparts" - sec = val s ++ "-piuparts" - f = schrootPiupartsConf s - munge = replace "-sbuild]" "-piuparts]" - desc = "piuparts schroot conf for " ++ val s - - -- normally the piuparts schroot conf has no aliases, but we have to add - -- one, for dgit compatibility, if this is the default sid chroot - aliasesLine = sidHostArchSchroot s >>= \isSidHostArchSchroot -> - return $ if isSidHostArchSchroot - then "UNRELEASED-" - ++ architectureToDebianArchString arch - ++ "-piuparts" - else "" - --- | Bind-mount /var/cache/apt/archives in all sbuild chroots so that the host --- system and the chroot share the apt cache --- --- This speeds up builds by avoiding unnecessary downloads of build --- dependencies. -shareAptCache :: Property DebianLike -shareAptCache = File.containsLine "/etc/schroot/sbuild/fstab" aptCacheLine - `requires` installed - `describe` "sbuild schroots share host apt cache" aptCacheLine :: String aptCacheLine = "/var/cache/apt/archives /var/cache/apt/archives none rw,bind 0 0" -- cgit v1.2.3 From f173da0c9e7e7221aa77cfa3247ebfb941204e05 Mon Sep 17 00:00:00 2001 From: Sean Whitton Date: Sat, 15 Jul 2017 09:13:08 -0700 Subject: sbuild properties set up an apt cache or use existing apt proxy --- src/Propellor/Property/Sbuild.hs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'src/Propellor') diff --git a/src/Propellor/Property/Sbuild.hs b/src/Propellor/Property/Sbuild.hs index 12720592..08246d27 100644 --- a/src/Propellor/Property/Sbuild.hs +++ b/src/Propellor/Property/Sbuild.hs @@ -84,6 +84,7 @@ module Propellor.Property.Sbuild ( ) where import Propellor.Base +import Propellor.Types.Info import Propellor.Property.Debootstrap (extractSuite) import Propellor.Property.Chroot.Util import qualified Propellor.Property.Apt as Apt @@ -177,6 +178,8 @@ built s@(SbuildSchroot suite arch) mirror cc = enhancedConf = combineProperties ("enhanced schroot conf for " ++ val s) $ props & aliasesLine + -- set up an apt proxy/cacher + & proxyCacher -- enable ccache and eatmydata for speed & ConfFile.containsIniSetting (schrootConf s) ( val s ++ "-sbuild" @@ -184,6 +187,21 @@ built s@(SbuildSchroot suite arch) mirror cc = , intercalate "," commandPrefix ) + proxyCacher :: Property DebianLike + proxyCacher = property' "set schroot apt proxy" $ \w -> do + proxyInfo <- getProxyInfo + ensureProperty w $ case proxyInfo of + Just (Apt.HostAptProxy u) -> setChrootProxy u + Nothing -> (Apt.serviceInstalledRunning "apt-cacher-ng" + `before` setChrootProxy "http://localhost:3142") + where + getProxyInfo :: Propellor (Maybe Apt.HostAptProxy) + getProxyInfo = fromInfoVal <$> askInfo + setChrootProxy :: Apt.Url -> Property DebianLike + setChrootProxy u = tightenTargets $ File.hasContent + (schrootRoot s "etc/apt/apt.conf.d/20proxy") + [ "Acquire::HTTP::Proxy \"" ++ u ++ "\";" ] + -- if we're building a sid chroot, add useful aliases -- In order to avoid more than one schroot getting the same aliases, we -- only do this if the arch of the chroot equals the host arch. -- cgit v1.2.3 From 4fbee5d7bd50b75aac51f9bce4f0a232bd5e2d3e Mon Sep 17 00:00:00 2001 From: Sean Whitton Date: Sat, 15 Jul 2017 09:13:23 -0700 Subject: tweak sbuild haddock Don't suggest enabling an apt cacher by running propellor inside the schroot, since we now have Apt.proxy. --- src/Propellor/Property/Sbuild.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/Propellor') diff --git a/src/Propellor/Property/Sbuild.hs b/src/Propellor/Property/Sbuild.hs index 08246d27..a1db861f 100644 --- a/src/Propellor/Property/Sbuild.hs +++ b/src/Propellor/Property/Sbuild.hs @@ -52,9 +52,9 @@ propellor spin pulls in a lot of dependencies. This could defeat using sbuild to determine if you've included all necessary build dependencies in your source package control file. -Nevertheless, the chroot that @sbuild-createchroot(1)@ creates might -not meet your needs. For example, you might need to enable an apt -cacher. In that case you can do something like this in @config.hs@: +Nevertheless, the chroot that @sbuild-createchroot(1)@ creates might not meet +your needs. For example, you might need to enable apt's https support. In that +case you can do something like this in @config.hs@: > & Sbuild.built (System (Debian Linux Unstable) X86_32) `before` mySetup > where -- cgit v1.2.3 From fe0796afa0a8d298f347612cfa608921c8b66ddc Mon Sep 17 00:00:00 2001 From: Sean Whitton Date: Sat, 15 Jul 2017 11:53:36 -0700 Subject: comment --- src/Propellor/Property/Sbuild.hs | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src/Propellor') diff --git a/src/Propellor/Property/Sbuild.hs b/src/Propellor/Property/Sbuild.hs index a1db861f..4f3fcc26 100644 --- a/src/Propellor/Property/Sbuild.hs +++ b/src/Propellor/Property/Sbuild.hs @@ -187,6 +187,16 @@ built s@(SbuildSchroot suite arch) mirror cc = , intercalate "," commandPrefix ) + -- set the apt proxy inside the chroot. If the host has an apt proxy + -- set, assume that it does some sort of caching. Otherwise, set up a + -- local apt-cacher-ng instance + -- + -- (if we didn't assume that the apt proxy does some sort of caching, + -- we'd need to complicate the Apt.HostAptProxy type to indicate whether + -- the proxy caches, and if it doesn't, set up apt-cacher-ng as an + -- intermediary proxy between the chroot's apt and the Apt.HostAptProxy + -- proxy. This complexity is more likely to cause problems than help + -- anyone) proxyCacher :: Property DebianLike proxyCacher = property' "set schroot apt proxy" $ \w -> do proxyInfo <- getProxyInfo -- cgit v1.2.3 From 83d1197cc26d0cbeff59013ad08391df9502bc04 Mon Sep 17 00:00:00 2001 From: Sean Whitton Date: Sat, 15 Jul 2017 11:56:13 -0700 Subject: factor out a 'return' --- src/Propellor/Property/Sbuild.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/Propellor') diff --git a/src/Propellor/Property/Sbuild.hs b/src/Propellor/Property/Sbuild.hs index 4f3fcc26..1ff4876a 100644 --- a/src/Propellor/Property/Sbuild.hs +++ b/src/Propellor/Property/Sbuild.hs @@ -472,7 +472,7 @@ schrootPiupartsConf (SbuildSchroot s a) = sidHostArchSchroot :: SbuildSchroot -> Propellor Bool sidHostArchSchroot (SbuildSchroot suite arch) = do maybeOS <- getOS - case maybeOS of - Nothing -> return False + return $ case maybeOS of + Nothing -> False Just (System _ hostArch) -> - return $ suite == "unstable" && hostArch == arch + suite == "unstable" && hostArch == arch -- cgit v1.2.3 From cc021857bbfed0d728e538308b87eb00bdd63efb Mon Sep 17 00:00:00 2001 From: Sean Whitton Date: Sat, 15 Jul 2017 12:39:11 -0700 Subject: ensure that cleaning up doesn't fail if nothing to clean up --- src/Propellor/Property/Sbuild.hs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'src/Propellor') diff --git a/src/Propellor/Property/Sbuild.hs b/src/Propellor/Property/Sbuild.hs index 1ff4876a..35acb8bf 100644 --- a/src/Propellor/Property/Sbuild.hs +++ b/src/Propellor/Property/Sbuild.hs @@ -245,13 +245,16 @@ built s@(SbuildSchroot suite arch) mirror cc = -- clean up config from earlier versions of this module cleanupOldConfig :: Property UnixLike - cleanupOldConfig = property' "old sbuild module config cleaned up" $ \w -> do - void $ ensureProperty w $ - check (doesFileExist fstab) (File.lacksLine fstab aptCacheLine) - liftIO $ removeDirectoryRecursive "/etc/schroot/piuparts" - makeChange $ nukeFile (schrootPiupartsConf s) + cleanupOldConfig = + property' "old sbuild module config cleaned up" $ \w -> do + void $ ensureProperty w $ + check (doesFileExist fstab) + (File.lacksLine fstab aptCacheLine) + void $ liftIO . tryIO $ removeDirectoryRecursive profile + makeChange $ nukeFile (schrootPiupartsConf s) where fstab = "/etc/schroot/sbuild/fstab" + profile = "/etc/schroot/piuparts" -- A failed debootstrap run will leave a debootstrap directory; -- recover by deleting it and trying again. -- cgit v1.2.3 From 6d43cd37c8a4b48aaca26cd2fa5db7b05df3751a Mon Sep 17 00:00:00 2001 From: Sean Whitton Date: Sat, 15 Jul 2017 12:50:55 -0700 Subject: assume cleanup property did nothing --- src/Propellor/Property/Sbuild.hs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/Propellor') diff --git a/src/Propellor/Property/Sbuild.hs b/src/Propellor/Property/Sbuild.hs index 35acb8bf..57dead3c 100644 --- a/src/Propellor/Property/Sbuild.hs +++ b/src/Propellor/Property/Sbuild.hs @@ -246,12 +246,14 @@ built s@(SbuildSchroot suite arch) mirror cc = -- clean up config from earlier versions of this module cleanupOldConfig :: Property UnixLike cleanupOldConfig = - property' "old sbuild module config cleaned up" $ \w -> do + property' "old sbuild module config cleaned up" $ \w -> do void $ ensureProperty w $ check (doesFileExist fstab) (File.lacksLine fstab aptCacheLine) void $ liftIO . tryIO $ removeDirectoryRecursive profile - makeChange $ nukeFile (schrootPiupartsConf s) + void $ liftIO $ nukeFile (schrootPiupartsConf s) + -- assume this did nothing + noChange where fstab = "/etc/schroot/sbuild/fstab" profile = "/etc/schroot/piuparts" -- cgit v1.2.3 From 8a61b0a706dfde9f5ecef40775c9534c6a18773c Mon Sep 17 00:00:00 2001 From: Sean Whitton Date: Sat, 15 Jul 2017 12:53:56 -0700 Subject: fix ordering of arguments to piuparts in sample .sbuildrc --- src/Propellor/Property/Sbuild.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/Propellor') diff --git a/src/Propellor/Property/Sbuild.hs b/src/Propellor/Property/Sbuild.hs index 57dead3c..1b83a6dc 100644 --- a/src/Propellor/Property/Sbuild.hs +++ b/src/Propellor/Property/Sbuild.hs @@ -34,8 +34,8 @@ In @~/.sbuildrc@ (sbuild 0.71.0 or newer): > $run_piuparts = 1; > $piuparts_opts = [ -> '--schroot', > '--no-eatmydata', +> '--schroot', > '%r-%a-sbuild', > '--fail-if-inadequate', > '--fail-on-broken-symlinks', -- cgit v1.2.3 From a05b4271dc0e2e7b24fdfbe4a4358127a5df730e Mon Sep 17 00:00:00 2001 From: Sean Whitton Date: Sat, 15 Jul 2017 13:39:36 -0700 Subject: add installing lintian to sample sbuild config If the user has installed the sample .sbuildrc in /root, sbuild-createchroot will error out unless lintian is installed. --- src/Propellor/Property/Sbuild.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/Propellor') diff --git a/src/Propellor/Property/Sbuild.hs b/src/Propellor/Property/Sbuild.hs index 1b83a6dc..8d1b9807 100644 --- a/src/Propellor/Property/Sbuild.hs +++ b/src/Propellor/Property/Sbuild.hs @@ -20,7 +20,7 @@ Debian stretch, which older sbuild can't handle. Suggested usage in @config.hs@: -> & Apt.installed ["piuparts", "autopkgtest"] +> & Apt.installed ["piuparts", "autopkgtest", "lintian"] > & Sbuild.builtFor (System (Debian Linux Unstable) X86_32) Sbuild.UseCcache > & Sbuild.updatedFor (System (Debian Linux Unstable) X86_32) `period` Weekly 1 > & Sbuild.usableBy (User "spwhitton") -- cgit v1.2.3 From 56fda8c3ae6af1241ce4836a6122393871fca151 Mon Sep 17 00:00:00 2001 From: Sean Whitton Date: Sat, 15 Jul 2017 13:51:23 -0700 Subject: add a description for Apt.useLocalCacher --- src/Propellor/Property/Apt.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/Propellor') diff --git a/src/Propellor/Property/Apt.hs b/src/Propellor/Property/Apt.hs index 2e4757f8..5630d83a 100644 --- a/src/Propellor/Property/Apt.hs +++ b/src/Propellor/Property/Apt.hs @@ -511,4 +511,4 @@ proxy u = tightenTargets $ useLocalCacher :: Property (HasInfo + DebianLike) useLocalCacher = proxy "http://localhost:3142" `requires` serviceInstalledRunning "apt-cacher-ng" - + `describe` "apt uses local apt cacher" -- cgit v1.2.3