From c8a653af5dfd6f876d2c5d77ef047ba47274e3d7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 19 May 2014 17:27:21 -0400 Subject: propellor spin --- .../Property/SiteSpecific/GitAnnexBuilder.hs | 59 +++++++++++++++------- 1 file changed, 41 insertions(+), 18 deletions(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 677aa760..9754d4f7 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -18,25 +18,14 @@ gitbuilderdir = homedir "gitbuilder" builddir :: FilePath builddir = gitbuilderdir "build" -builder :: Architecture -> CronTimes -> Bool -> Property -builder arch crontimes rsyncupload = combineProperties "gitannexbuilder" - [ Apt.stdSourcesList Unstable - , Apt.buildDep ["git-annex"] - , Apt.installed ["git", "rsync", "moreutils", "ca-certificates", - "liblockfile-simple-perl", "cabal-install", "vim", "less"] +type TimeOut = String -- eg, 5h + +builder :: Architecture -> CronTimes -> TimeOut -> Bool -> Property +builder buildarch crontimes timeout rsyncupload = combineProperties "gitannexbuilder" + [ treeDeps buildarch , Apt.serviceInstalledRunning "cron" - , User.accountFor builduser - , check (not <$> doesDirectoryExist gitbuilderdir) $ userScriptProperty builduser - [ "git clone git://git.kitenet.net/gitannexbuilder " ++ gitbuilderdir - , "cd " ++ gitbuilderdir - , "git checkout " ++ arch - ] - `describe` "gitbuilder setup" - , check (not <$> doesDirectoryExist builddir) $ userScriptProperty builduser - [ "git clone git://git-annex.branchable.com/ " ++ builddir - ] - , "git-annex source build deps installed" ==> Apt.buildDepIn builddir - , Cron.niceJob "gitannexbuilder" crontimes builduser gitbuilderdir "git pull ; ./autobuild" + , Cron.niceJob "gitannexbuilder" crontimes builduser gitbuilderdir $ + "git pull ; timeout " ++ timeout ++ " ./autobuild" -- The builduser account does not have a password set, -- instead use the password privdata to hold the rsync server -- password used to upload the built image. @@ -55,3 +44,37 @@ builder arch crontimes rsyncupload = combineProperties "gitannexbuilder" , makeChange $ writeFile f "no password configured" ) ] + +treeDeps :: Architecture -> Property +treeDeps buildarch = combineProperties "gitannexbuilder" + [ Apt.stdSourcesList Unstable + , Apt.buildDep ["git-annex"] + , Apt.installed ["git", "rsync", "moreutils", "ca-certificates", + "liblockfile-simple-perl", "cabal-install", "vim", "less"] + , User.accountFor builduser + , check (not <$> doesDirectoryExist gitbuilderdir) $ userScriptProperty builduser + [ "git clone git://git.kitenet.net/gitannexbuilder " ++ gitbuilderdir + , "cd " ++ gitbuilderdir + , "git checkout " ++ buildarch + ] + `describe` "gitbuilder setup" + , check (not <$> doesDirectoryExist builddir) $ userScriptProperty builduser + [ "git clone git://git-annex.branchable.com/ " ++ builddir + ] + , "git-annex source build deps installed" ==> Apt.buildDepIn builddir + ] + +-- Installs current versions of git-annex's deps from cabal, but only +-- does so once. +cabalDeps :: Property +cabalDeps = flagFile go cabalupdated + where + go = userScriptProperty builduser ["cabal update && cabal install git-annex --only-dependencies || true"] + cabalupdated = homedir ".cabal" "packages" "hackage.haskell.org" "00-index.cache" + +-- Ensure a ssh key is set up. +sshKeyGen :: Property +sshKeyGen = flagFile gen f + where + gen = userScriptProperty builduser ["ssh-keygen -t RSA -N '' -f " ++ f] + f = homedir ".ssh" "id_rsa" -- cgit v1.2.3 From 7eeb58dab91eeb73d9491a89313bb28aa00ee833 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 20 May 2014 13:37:03 -0400 Subject: fix ssh authorizied_keys --- src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 9754d4f7..ddb61381 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -72,9 +72,15 @@ cabalDeps = flagFile go cabalupdated go = userScriptProperty builduser ["cabal update && cabal install git-annex --only-dependencies || true"] cabalupdated = homedir ".cabal" "packages" "hackage.haskell.org" "00-index.cache" --- Ensure a ssh key is set up. +-- Ensure a ssh key is set up, and allow it to be used to ssh in sshKeyGen :: Property -sshKeyGen = flagFile gen f +sshKeyGen = combineProperties "sshkeygen" + [ flagFile gen privkey + , flagFile auth authkeys + ] where gen = userScriptProperty builduser ["ssh-keygen -t RSA -N '' -f " ++ f] - f = homedir ".ssh" "id_rsa" + auth = userScriptProperty builduser ["cp " ++ pubkey ++ " " ++ authkeys] + privkey = homedir ".ssh" "id_rsa" + pubkey = privkey ++ ".pub" + authkeys = homedir ".ssh" "authorized_keys" -- cgit v1.2.3 From 9acd70eb4c2eb680eed5affda4be8eb59b7507f6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 20 May 2014 18:28:13 -0400 Subject: propellor spin --- config-joey.hs | 9 +++--- .../Property/SiteSpecific/GitAnnexBuilder.hs | 33 +++++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/config-joey.hs b/config-joey.hs index 782c579b..7f32d8b6 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -94,8 +94,8 @@ hosts = -- (o) ` & Docker.configured & Docker.docked hosts "amd64-git-annex-builder" & Docker.docked hosts "i386-git-annex-builder" - & Docker.docked hosts "armel-git-annex-builder-companion" - & Docker.docked hosts "armel-git-annex-builder" + ! Docker.docked hosts "armel-git-annex-builder-companion" + ! Docker.docked hosts "armel-git-annex-builder" & Docker.garbageCollected `period` Daily & Apt.buildDep ["git-annex"] `period` Daily @@ -201,7 +201,8 @@ hosts = -- (o) ` & Docker.volume GitAnnexBuilder.homedir & File.ownerGroup GitAnnexBuilder.homedir GitAnnexBuilder.builduser GitAnnexBuilder.builduser -- Install current versions of build deps from cabal. - & GitAnnexBuilder.treeDeps "armel" + & GitAnnexBuilder.tree "armel" + & GitAnnexBuilder.buildDepsNoHaskellLibs & GitAnnexBuilder.cabalDeps -- The armel builder can ssh to this companion, -- using $COMPANION_PORT_22_TCP_ADDR as the hostname, @@ -219,7 +220,7 @@ hosts = -- (o) ` -- (Currently have to run -- git-annex/standalone/linux/install-haskell-packages -- which is not fully automated.) - & GitAnnexBuilder.builder "armel" "1 3 * * *" "5h" True + & GitAnnexBuilder.builder' GitAnnexBuilder.buildDepsNoHaskellLibs "armel" "1 3 * * *" "5h" True ] ++ monsters standardGitAnnexBuilder :: Architecture -> Int -> GitAnnexBuilder.TimeOut -> Host diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index ddb61381..1dbdbca7 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -21,8 +21,12 @@ builddir = gitbuilderdir "build" type TimeOut = String -- eg, 5h builder :: Architecture -> CronTimes -> TimeOut -> Bool -> Property -builder buildarch crontimes timeout rsyncupload = combineProperties "gitannexbuilder" - [ treeDeps buildarch +builder = builder' buildDeps + +builder' :: Property -> Architecture -> CronTimes -> TimeOut -> Bool -> Property +builder' buildepsprop buildarch crontimes timeout rsyncupload = combineProperties "gitannexbuilder" + [ tree buildarch + , buildepsprop , Apt.serviceInstalledRunning "cron" , Cron.niceJob "gitannexbuilder" crontimes builduser gitbuilderdir $ "git pull ; timeout " ++ timeout ++ " ./autobuild" @@ -45,13 +49,10 @@ builder buildarch crontimes timeout rsyncupload = combineProperties "gitannexbui ) ] -treeDeps :: Architecture -> Property -treeDeps buildarch = combineProperties "gitannexbuilder" - [ Apt.stdSourcesList Unstable - , Apt.buildDep ["git-annex"] - , Apt.installed ["git", "rsync", "moreutils", "ca-certificates", - "liblockfile-simple-perl", "cabal-install", "vim", "less"] - , User.accountFor builduser +tree :: Architecture -> Property +tree buildarch = combineProperties "gitannexbuilder tree" + [ User.accountFor builduser + , Apt.installed ["git"] , check (not <$> doesDirectoryExist gitbuilderdir) $ userScriptProperty builduser [ "git clone git://git.kitenet.net/gitannexbuilder " ++ gitbuilderdir , "cd " ++ gitbuilderdir @@ -61,9 +62,21 @@ treeDeps buildarch = combineProperties "gitannexbuilder" , check (not <$> doesDirectoryExist builddir) $ userScriptProperty builduser [ "git clone git://git-annex.branchable.com/ " ++ builddir ] + ] + +buildDeps :: Property +buildDeps = combineProperties "gitannexbuilder build deps" + [ Apt.stdSourcesList Unstable + , Apt.buildDep ["git-annex"] + , buildDepsNoHaskellLibs , "git-annex source build deps installed" ==> Apt.buildDepIn builddir ] +buildDepsNoHaskellLibs :: Property +buildDepsNoHaskellLibs = Apt.installed ["git", "rsync", "moreutils", "ca-certificates", + "debhelper", "ghc", "curl", "openssh-client", "git-remote-gcrypt", + "liblockfile-simple-perl", "cabal-install", "vim", "less"] + -- Installs current versions of git-annex's deps from cabal, but only -- does so once. cabalDeps :: Property @@ -79,7 +92,7 @@ sshKeyGen = combineProperties "sshkeygen" , flagFile auth authkeys ] where - gen = userScriptProperty builduser ["ssh-keygen -t RSA -N '' -f " ++ f] + gen = userScriptProperty builduser ["ssh-keygen -t RSA -N '' -f " ++ privkey] auth = userScriptProperty builduser ["cp " ++ pubkey ++ " " ++ authkeys] privkey = homedir ".ssh" "id_rsa" pubkey = privkey ++ ".pub" -- cgit v1.2.3 From 8f9ab315e6460166eab17c414ddf0a581dc1250b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 20 May 2014 18:49:03 -0400 Subject: propellor spin --- config-joey.hs | 1 - src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/config-joey.hs b/config-joey.hs index 10e19a0b..9972d5db 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -199,7 +199,6 @@ hosts = -- (o) ` & Apt.unattendedUpgrades -- This volume is shared with the armel builder. & Docker.volume GitAnnexBuilder.homedir - & File.ownerGroup GitAnnexBuilder.homedir GitAnnexBuilder.builduser GitAnnexBuilder.builduser -- Install current versions of build deps from cabal. & GitAnnexBuilder.tree "armel" & GitAnnexBuilder.buildDepsNoHaskellLibs diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 1dbdbca7..0cd5d3c5 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -4,6 +4,7 @@ import Propellor import qualified Propellor.Property.Apt as Apt import qualified Propellor.Property.User as User import qualified Propellor.Property.Cron as Cron +import qualified Propellor.Property.File as File import Propellor.Property.Cron (CronTimes) builduser :: UserName @@ -52,6 +53,8 @@ builder' buildepsprop buildarch crontimes timeout rsyncupload = combinePropertie tree :: Architecture -> Property tree buildarch = combineProperties "gitannexbuilder tree" [ User.accountFor builduser + -- Home directory already exists when docker volume is used. + , File.ownerGroup homedir builduser builduser , Apt.installed ["git"] , check (not <$> doesDirectoryExist gitbuilderdir) $ userScriptProperty builduser [ "git clone git://git.kitenet.net/gitannexbuilder " ++ gitbuilderdir -- cgit v1.2.3 From eeb2d688bdf435158929bb3cbfea6fb68076de9e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 20 May 2014 19:49:07 -0400 Subject: propellor spin --- config-joey.hs | 5 ++++- src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs | 9 ++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/config-joey.hs b/config-joey.hs index 9972d5db..f495004e 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -196,6 +196,7 @@ hosts = -- (o) ` -- to have the same versions of all haskell libraries installed. , Docker.container "armel-git-annex-builder-companion" (image $ System (Debian Unstable) "amd64") + & Apt.stdSourcesList Unstable & Apt.unattendedUpgrades -- This volume is shared with the armel builder. & Docker.volume GitAnnexBuilder.homedir @@ -211,6 +212,7 @@ hosts = -- (o) ` & GitAnnexBuilder.sshKeyGen , Docker.container "armel-git-annex-builder" (image $ System (Debian Unstable) "armel") + & Apt.stdSourcesList Unstable & Apt.unattendedUpgrades & Apt.installed ["openssh-client"] & Docker.link "armel-git-annex-builder-companion" "companion" @@ -225,8 +227,9 @@ hosts = -- (o) ` standardGitAnnexBuilder :: Architecture -> Int -> GitAnnexBuilder.TimeOut -> Host standardGitAnnexBuilder arch buildminute timeout = Docker.container (arch ++ "-git-annex-builder") (image $ System (Debian Unstable) arch) - & GitAnnexBuilder.builder arch (show buildminute ++ " * * * *") timeout True + & Apt.stdSourcesList Unstable & Apt.unattendedUpgrades + & GitAnnexBuilder.builder arch (show buildminute ++ " * * * *") timeout True -- This is my standard system setup. standardSystem :: HostName -> DebianSuite -> Architecture -> Host diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 0cd5d3c5..428b3d00 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -69,8 +69,7 @@ tree buildarch = combineProperties "gitannexbuilder tree" buildDeps :: Property buildDeps = combineProperties "gitannexbuilder build deps" - [ Apt.stdSourcesList Unstable - , Apt.buildDep ["git-annex"] + [ Apt.buildDep ["git-annex"] , buildDepsNoHaskellLibs , "git-annex source build deps installed" ==> Apt.buildDepIn builddir ] @@ -78,7 +77,11 @@ buildDeps = combineProperties "gitannexbuilder build deps" buildDepsNoHaskellLibs :: Property buildDepsNoHaskellLibs = Apt.installed ["git", "rsync", "moreutils", "ca-certificates", "debhelper", "ghc", "curl", "openssh-client", "git-remote-gcrypt", - "liblockfile-simple-perl", "cabal-install", "vim", "less"] + "liblockfile-simple-perl", "cabal-install", "vim", "less", + "alex", "happy", "c2hs", + -- these haskell libs depend on C libs and don't use TH + "libghc-dbus-dev", "libghc-fdo-notify-dev", "libghc-network-protocol-xmpp-dev" + ] -- Installs current versions of git-annex's deps from cabal, but only -- does so once. -- cgit v1.2.3 From 7ac991635c0dbdb2ace19480fcac22dae36e5409 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 21 May 2014 12:58:53 -0400 Subject: propellor spin --- config-joey.hs | 10 +++---- .../Property/SiteSpecific/GitAnnexBuilder.hs | 32 ++++++++-------------- 2 files changed, 16 insertions(+), 26 deletions(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/config-joey.hs b/config-joey.hs index f495004e..386e5900 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -94,8 +94,8 @@ hosts = -- (o) ` & Docker.configured & Docker.docked hosts "amd64-git-annex-builder" & Docker.docked hosts "i386-git-annex-builder" - & Docker.docked hosts "armel-git-annex-builder-companion" - & Docker.docked hosts "armel-git-annex-builder" + ! Docker.docked hosts "armel-git-annex-builder-companion" + ! Docker.docked hosts "armel-git-annex-builder" & Docker.garbageCollected `period` Daily & Apt.buildDep ["git-annex"] `period` Daily @@ -199,7 +199,7 @@ hosts = -- (o) ` & Apt.stdSourcesList Unstable & Apt.unattendedUpgrades -- This volume is shared with the armel builder. - & Docker.volume GitAnnexBuilder.homedir + & Docker.volume GitAnnexBuilder.gitbuilderdir -- Install current versions of build deps from cabal. & GitAnnexBuilder.tree "armel" & GitAnnexBuilder.buildDepsNoHaskellLibs @@ -208,8 +208,7 @@ hosts = -- (o) ` -- using $COMPANION_PORT_22_TCP_ADDR as the hostname, & Docker.expose "22" & Apt.serviceInstalledRunning "ssh" - -- ssh key is shared via the home directory volume - & GitAnnexBuilder.sshKeyGen + & Ssh.authorizedKeys GitAnnexBuilder.builduser , Docker.container "armel-git-annex-builder" (image $ System (Debian Unstable) "armel") & Apt.stdSourcesList Unstable @@ -217,6 +216,7 @@ hosts = -- (o) ` & Apt.installed ["openssh-client"] & Docker.link "armel-git-annex-builder-companion" "companion" & Docker.volumes_from "armel-git-annex-builder-companion" + & Ssh.keyImported SshRsa GitAnnexBuilder.builduser -- TODO: automate installing haskell libs -- (Currently have to run -- git-annex/standalone/linux/install-haskell-packages diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 428b3d00..acf122af 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -53,15 +53,18 @@ builder' buildepsprop buildarch crontimes timeout rsyncupload = combinePropertie tree :: Architecture -> Property tree buildarch = combineProperties "gitannexbuilder tree" [ User.accountFor builduser - -- Home directory already exists when docker volume is used. - , File.ownerGroup homedir builduser builduser , Apt.installed ["git"] - , check (not <$> doesDirectoryExist gitbuilderdir) $ userScriptProperty builduser - [ "git clone git://git.kitenet.net/gitannexbuilder " ++ gitbuilderdir - , "cd " ++ gitbuilderdir - , "git checkout " ++ buildarch - ] - `describe` "gitbuilder setup" + -- gitbuilderdir directory already exists when docker volume is used, + -- but with wrong owner. + , File.dirExists gitbuilderdir + , File.ownerGroup gitbuilderdir builduser builduser + , check (not <$> (doesDirectoryExist (gitbuilderdir ".git"))) $ + userScriptProperty builduser + [ "git clone git://git.kitenet.net/gitannexbuilder " ++ gitbuilderdir + , "cd " ++ gitbuilderdir + , "git checkout " ++ buildarch + ] + `describe` "gitbuilder setup" , check (not <$> doesDirectoryExist builddir) $ userScriptProperty builduser [ "git clone git://git-annex.branchable.com/ " ++ builddir ] @@ -90,16 +93,3 @@ cabalDeps = flagFile go cabalupdated where go = userScriptProperty builduser ["cabal update && cabal install git-annex --only-dependencies || true"] cabalupdated = homedir ".cabal" "packages" "hackage.haskell.org" "00-index.cache" - --- Ensure a ssh key is set up, and allow it to be used to ssh in -sshKeyGen :: Property -sshKeyGen = combineProperties "sshkeygen" - [ flagFile gen privkey - , flagFile auth authkeys - ] - where - gen = userScriptProperty builduser ["ssh-keygen -t RSA -N '' -f " ++ privkey] - auth = userScriptProperty builduser ["cp " ++ pubkey ++ " " ++ authkeys] - privkey = homedir ".ssh" "id_rsa" - pubkey = privkey ++ ".pub" - authkeys = homedir ".ssh" "authorized_keys" -- cgit v1.2.3 From 424ba1229ba0a786d6090d69739defe47d0612c8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 22 May 2014 21:42:57 -0400 Subject: refactor --- config-joey.hs | 54 ++++------------------ .../Property/SiteSpecific/GitAnnexBuilder.hs | 44 ++++++++++++++++++ 2 files changed, 53 insertions(+), 45 deletions(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/config-joey.hs b/config-joey.hs index 318d5d94..fafc409c 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -189,48 +189,12 @@ hosts = -- (o) ` (Just "remotes/origin/old-kitenet.net") -- git-annex autobuilder containers - , standardGitAnnexBuilder "amd64" 15 "2h" - , standardGitAnnexBuilder "i386" 45 "2h" - -- armel builder has a companion container using amd64 that - -- runs the build first to get TH splices. They need - -- to have the same versions of all haskell libraries installed. - , Docker.container "armel-git-annex-builder-companion" - (image $ System (Debian Unstable) "amd64") - & Apt.stdSourcesList Unstable - & Apt.unattendedUpgrades - -- This volume is shared with the armel builder. - & Docker.volume GitAnnexBuilder.gitbuilderdir - -- Install current versions of build deps from cabal. - & GitAnnexBuilder.tree "armel" - & GitAnnexBuilder.buildDepsNoHaskellLibs - & GitAnnexBuilder.cabalDeps - -- The armel builder can ssh to this companion, - -- using $COMPANION_PORT_22_TCP_ADDR as the hostname, - & Docker.expose "22" - & Apt.serviceInstalledRunning "ssh" - & Ssh.authorizedKeys GitAnnexBuilder.builduser - , Docker.container "armel-git-annex-builder" - (image $ System (Debian Unstable) "armel") - & Apt.stdSourcesList Unstable - & Apt.unattendedUpgrades - & Apt.installed ["openssh-client"] - & Docker.link "armel-git-annex-builder-companion" "companion" - & Docker.volumes_from "armel-git-annex-builder-companion" - -- TODO: automate installing haskell libs - -- (Currently have to run - -- git-annex/standalone/linux/install-haskell-packages - -- which is not fully automated.) - & GitAnnexBuilder.builder' GitAnnexBuilder.buildDepsNoHaskellLibs "armel" "1 3 * * *" "5h" True - & Ssh.keyImported SshRsa GitAnnexBuilder.builduser + , GitAnnexBuilder.standardContainer dockerImage "amd64" 15 "2h" + , GitAnnexBuilder.standardContainer dockerImage "i386" 45 "2h" + , GitAnnexBuilder.armelCompanionContainer dockerImage + , GitAnnexBuilder.armelContainer dockerImage "1 3 * * *" "5h" ] ++ monsters -standardGitAnnexBuilder :: Architecture -> Int -> GitAnnexBuilder.TimeOut -> Host -standardGitAnnexBuilder arch buildminute timeout = Docker.container (arch ++ "-git-annex-builder") - (image $ System (Debian Unstable) arch) - & Apt.stdSourcesList Unstable - & Apt.unattendedUpgrades - & GitAnnexBuilder.builder arch (show buildminute ++ " * * * *") timeout True - -- This is my standard system setup. standardSystem :: HostName -> DebianSuite -> Architecture -> Host standardSystem hn suite arch = host hn @@ -257,7 +221,7 @@ standardSystem hn suite arch = host hn -- This is my standard container setup, featuring automatic upgrades. standardContainer :: Docker.ContainerName -> DebianSuite -> Architecture -> Host -standardContainer name suite arch = Docker.container name (image system) +standardContainer name suite arch = Docker.container name (dockerImage system) & os (System (Debian suite) arch) & Apt.stdSourcesList suite & Apt.unattendedUpgrades @@ -265,10 +229,10 @@ standardContainer name suite arch = Docker.container name (image system) system = System (Debian suite) arch -- Docker images I prefer to use. -image :: System -> Docker.Image -image (System (Debian Unstable) arch) = "joeyh/debian-unstable-" ++ arch -image (System (Debian Stable) arch) = "joeyh/debian-stable-" ++ arch -image _ = "debian-stable-official" -- does not currently exist! +dockerImage :: System -> Docker.Image +dockerImage (System (Debian Unstable) arch) = "joeyh/debian-unstable-" ++ arch +dockerImage (System (Debian Stable) arch) = "joeyh/debian-stable-" ++ arch +dockerImage _ = "debian-stable-official" -- does not currently exist! -- Digital Ocean does not provide any way to boot -- the kernel provided by the distribution, except using kexec. diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index acf122af..80877fa5 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -4,7 +4,9 @@ import Propellor import qualified Propellor.Property.Apt as Apt import qualified Propellor.Property.User as User import qualified Propellor.Property.Cron as Cron +import qualified Propellor.Property.Ssh as Ssh import qualified Propellor.Property.File as File +import qualified Propellor.Property.Docker as Docker import Propellor.Property.Cron (CronTimes) builduser :: UserName @@ -93,3 +95,45 @@ cabalDeps = flagFile go cabalupdated where go = userScriptProperty builduser ["cabal update && cabal install git-annex --only-dependencies || true"] cabalupdated = homedir ".cabal" "packages" "hackage.haskell.org" "00-index.cache" + +standardContainer :: (System -> Docker.Image) -> Architecture -> Int -> TimeOut -> Host +standardContainer dockerImage arch buildminute timeout = Docker.container (arch ++ "-git-annex-builder") + (dockerImage $ System (Debian Unstable) arch) + & Apt.stdSourcesList Unstable + & Apt.unattendedUpgrades + & builder arch (show buildminute ++ " * * * *") timeout True + +-- armel builder has a companion container using amd64 that +-- runs the build first to get TH splices. They need +-- to have the same versions of all haskell libraries installed. +armelCompanionContainer :: (System -> Docker.Image) -> Host +armelCompanionContainer dockerImage = Docker.container "armel-git-annex-builder-companion" + (dockerImage $ System (Debian Unstable) "amd64") + & Apt.stdSourcesList Unstable + & Apt.unattendedUpgrades + -- This volume is shared with the armel builder. + & Docker.volume gitbuilderdir + -- Install current versions of build deps from cabal. + & tree "armel" + & buildDepsNoHaskellLibs + & cabalDeps + -- The armel builder can ssh to this companion, + -- using $COMPANION_PORT_22_TCP_ADDR as the hostname, + & Docker.expose "22" + & Apt.serviceInstalledRunning "ssh" + & Ssh.authorizedKeys builduser + +armelContainer :: (System -> Docker.Image) -> Cron.CronTimes -> TimeOut -> Host +armelContainer dockerImage crontimes timeout = Docker.container "armel-git-annex-builder" + (dockerImage $ System (Debian Unstable) "armel") + & Apt.stdSourcesList Unstable + & Apt.unattendedUpgrades + & Apt.installed ["openssh-client"] + & Docker.link "armel-git-annex-builder-companion" "companion" + & Docker.volumes_from "armel-git-annex-builder-companion" + -- TODO: automate installing haskell libs + -- (Currently have to run + -- git-annex/standalone/linux/install-haskell-packages + -- which is not fully automated.) + & builder' buildDepsNoHaskellLibs "armel" crontimes timeout True + & Ssh.keyImported SshRsa builduser -- cgit v1.2.3 From 4f239d1360c87b069279ce1b100db4a057f94e48 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 23 May 2014 10:46:25 -0400 Subject: propellor spin --- src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 80877fa5..56123b60 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -117,8 +117,7 @@ armelCompanionContainer dockerImage = Docker.container "armel-git-annex-builder- & tree "armel" & buildDepsNoHaskellLibs & cabalDeps - -- The armel builder can ssh to this companion, - -- using $COMPANION_PORT_22_TCP_ADDR as the hostname, + -- The armel builder can ssh to this companion. & Docker.expose "22" & Apt.serviceInstalledRunning "ssh" & Ssh.authorizedKeys builduser @@ -137,3 +136,8 @@ armelContainer dockerImage crontimes timeout = Docker.container "armel-git-annex -- which is not fully automated.) & builder' buildDepsNoHaskellLibs "armel" crontimes timeout True & Ssh.keyImported SshRsa builduser + & trivial writecompanionaddress + where + writecompanionaddress = scriptProperty + [ "echo \"$COMPANION_PORT_22_TCP_ADDR\" > " ++ homedir "companion_address" + ] -- cgit v1.2.3 From 0ec565a590e0b5e70af4f4f922925e5b0b718376 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 23 May 2014 12:30:25 -0400 Subject: propellor spin --- config-joey.hs | 37 +++------------------- debian/changelog | 1 + propellor.cabal | 2 ++ .../Property/HostingProvider/CloudAtCost.hs | 24 ++++++++++++++ .../Property/HostingProvider/DigitalOcean.hs | 21 ++++++++++++ .../Property/SiteSpecific/GitAnnexBuilder.hs | 2 +- 6 files changed, 53 insertions(+), 34 deletions(-) create mode 100644 src/Propellor/Property/HostingProvider/CloudAtCost.hs create mode 100644 src/Propellor/Property/HostingProvider/DigitalOcean.hs (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/config-joey.hs b/config-joey.hs index fafc409c..2f84fbb0 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -21,6 +21,8 @@ import qualified Propellor.Property.Git as Git import qualified Propellor.Property.Apache as Apache import qualified Propellor.Property.Postfix as Postfix import qualified Propellor.Property.Service as Service +import qualified Propellor.Property.HostingProvider.DigitalOcean as DigitalOcean +import qualified Propellor.Property.HostingProvider.CloudAtCost as CloudAtCost import qualified Propellor.Property.SiteSpecific.GitHome as GitHome import qualified Propellor.Property.SiteSpecific.GitAnnexBuilder as GitAnnexBuilder import qualified Propellor.Property.SiteSpecific.JoeySites as JoeySites @@ -46,7 +48,7 @@ hosts = -- (o) ` & ipv4 "162.248.143.249" & ipv6 "2002:5044:5531::1" - & cleanCloudAtCost + & CloudAtCost.decruft & Apt.unattendedUpgrades & Network.ipv6to4 & Tor.isBridge @@ -103,6 +105,7 @@ hosts = -- (o) ` , standardSystem "diatom.kitenet.net" Stable "amd64" & ipv4 "107.170.31.195" + & DigitalOcean.distroKernel & Hostname.sane & Ssh.hostKey SshDsa & Ssh.hostKey SshRsa @@ -234,38 +237,6 @@ dockerImage (System (Debian Unstable) arch) = "joeyh/debian-unstable-" ++ arch dockerImage (System (Debian Stable) arch) = "joeyh/debian-stable-" ++ arch dockerImage _ = "debian-stable-official" -- does not currently exist! --- Digital Ocean does not provide any way to boot --- the kernel provided by the distribution, except using kexec. --- Without this, some old, and perhaps insecure kernel will be used. --- --- Note that this only causes the new kernel to be loaded on reboot. --- If the power is cycled, the old kernel still boots up. --- TODO: detect this and reboot immediately? -digitalOceanDistroKernel :: Property -digitalOceanDistroKernel = propertyList "digital ocean distro kernel hack" - [ Apt.installed ["grub-pc", "kexec-tools"] - , "/etc/default/kexec" `File.containsLines` - [ "LOAD_KEXEC=true" - , "USE_GRUB_CONFIG=true" - ] - ] - --- Clean up a system as installed by cloudatcost.com -cleanCloudAtCost :: Property -cleanCloudAtCost = propertyList "cloudatcost cleanup" - [ Hostname.sane - , Ssh.randomHostKeys - , "worked around grub/lvm boot bug #743126" ==> - "/etc/default/grub" `File.containsLine` "GRUB_DISABLE_LINUX_UUID=true" - `onChange` cmdProperty "update-grub" [] - `onChange` cmdProperty "update-initramfs" ["-u"] - , combineProperties "nuked cloudatcost cruft" - [ File.notPresent "/etc/rc.local" - , File.notPresent "/etc/init.d/S97-setup.sh" - , User.nuked "user" User.YesReallyDeleteHome - ] - ] - myDnsSecondary :: Property myDnsSecondary = propertyList "dns secondary for all my domains" $ map toProp [ Dns.secondary hosts "kitenet.net" diff --git a/debian/changelog b/debian/changelog index 9d1d9a1c..a3dc1032 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ propellor (0.5.3) UNRELEASED; urgency=medium * Fix unattended-upgrades config for !stable. * Ensure that kernel hostname is same as /etc/hostname when configuring hostname. + * Added modules for some hosting providers (DigitalOcean, CloudAtCost). -- Joey Hess Sun, 18 May 2014 13:44:00 -0400 diff --git a/propellor.cabal b/propellor.cabal index 9d0612e9..da9a630f 100644 --- a/propellor.cabal +++ b/propellor.cabal @@ -92,6 +92,8 @@ Library Propellor.Property.Sudo Propellor.Property.Tor Propellor.Property.User + Propellor.Property.HostingProvider.DigitalOcean + Propellor.Property.HostingProvider.CloudAtCost Propellor.Property.SiteSpecific.GitHome Propellor.Property.SiteSpecific.JoeySites Propellor.Property.SiteSpecific.GitAnnexBuilder diff --git a/src/Propellor/Property/HostingProvider/CloudAtCost.hs b/src/Propellor/Property/HostingProvider/CloudAtCost.hs new file mode 100644 index 00000000..003bd3c5 --- /dev/null +++ b/src/Propellor/Property/HostingProvider/CloudAtCost.hs @@ -0,0 +1,24 @@ +module Propellor.Property.HostingProvider.CloudAtCost where + +import Propellor +import qualified Propellor.Property.Hostname as Hostname +import qualified Propellor.Property.File as File +import qualified Propellor.Property.Ssh as Ssh +import qualified Propellor.Property.User as User + +-- Clean up a system as installed by cloudatcost.com +decruft :: Property +decruft = propertyList "cloudatcost cleanup" + [ Hostname.sane + , Ssh.randomHostKeys + , "worked around grub/lvm boot bug #743126" ==> + "/etc/default/grub" `File.containsLine` "GRUB_DISABLE_LINUX_UUID=true" + `onChange` cmdProperty "update-grub" [] + `onChange` cmdProperty "update-initramfs" ["-u"] + , combineProperties "nuked cloudatcost cruft" + [ File.notPresent "/etc/rc.local" + , File.notPresent "/etc/init.d/S97-setup.sh" + , User.nuked "user" User.YesReallyDeleteHome + ] + ] + diff --git a/src/Propellor/Property/HostingProvider/DigitalOcean.hs b/src/Propellor/Property/HostingProvider/DigitalOcean.hs new file mode 100644 index 00000000..24dfd356 --- /dev/null +++ b/src/Propellor/Property/HostingProvider/DigitalOcean.hs @@ -0,0 +1,21 @@ +module Propellor.Property.HostingProvider.DigitalOcean where + +import Propellor +import qualified Propellor.Property.Apt as Apt +import qualified Propellor.Property.File as File + +-- Digital Ocean does not provide any way to boot +-- the kernel provided by the distribution, except using kexec. +-- Without this, some old, and perhaps insecure kernel will be used. +-- +-- Note that this only causes the new kernel to be loaded on reboot. +-- If the power is cycled, the old kernel still boots up. +-- TODO: detect this and reboot immediately? +distroKernel :: Property +distroKernel = propertyList "digital ocean distro kernel hack" + [ Apt.installed ["grub-pc", "kexec-tools"] + , "/etc/default/kexec" `File.containsLines` + [ "LOAD_KEXEC=true" + , "USE_GRUB_CONFIG=true" + ] + ] diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 56123b60..8055a608 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -140,4 +140,4 @@ armelContainer dockerImage crontimes timeout = Docker.container "armel-git-annex where writecompanionaddress = scriptProperty [ "echo \"$COMPANION_PORT_22_TCP_ADDR\" > " ++ homedir "companion_address" - ] + ] `describe` "companion_address file" -- cgit v1.2.3 From 60ea130321e23dd7b36337388d6627661c116548 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 24 May 2014 00:05:47 -0400 Subject: propellor spin --- config-joey.hs | 2 ++ src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/config-joey.hs b/config-joey.hs index 2f84fbb0..1f5ce0dc 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -98,6 +98,7 @@ hosts = -- (o) ` & Docker.docked hosts "i386-git-annex-builder" & Docker.docked hosts "armel-git-annex-builder-companion" & Docker.docked hosts "armel-git-annex-builder" + & Docker.docked hosts "android-git-annex-builder" & Docker.garbageCollected `period` Daily & Apt.buildDep ["git-annex"] `period` Daily @@ -196,6 +197,7 @@ hosts = -- (o) ` , GitAnnexBuilder.standardContainer dockerImage "i386" 45 "2h" , GitAnnexBuilder.armelCompanionContainer dockerImage , GitAnnexBuilder.armelContainer dockerImage "1 3 * * *" "5h" + , GitAnnexBuilder.androidContainer dockerImage "1 1 * * *" "3h" ] ++ monsters -- This is my standard system setup. diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 8055a608..13f1ac12 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -103,6 +103,24 @@ standardContainer dockerImage arch buildminute timeout = Docker.container (arch & Apt.unattendedUpgrades & builder arch (show buildminute ++ " * * * *") timeout True +-- Android is cross-built in a Debian i386 container, using the Android NDK. +androidContainer :: (System -> Docker.Image) -> Cron.CronTimes -> TimeOut -> Host +androidContainer dockerImage crontimes timeout = Docker.container "android-git-annex-builder" + (dockerImage $ System (Debian Stable) "i386") + & Apt.stdSourcesList Stable + & Apt.unattendedUpgrades + & builder "android" crontimes timeout True + -- Use git-annex's android chroot setup script, which will install + -- ghc-android and the NDK, etc, in the home directory of the + -- builder user. + & scriptProperty + [ "cd " ++ builddir ++ " && ./standalone/android/buildchroot-inchroot" + ] + -- TODO: automate installing haskell libs + -- (Currently have to run + -- git-annex/standalone/android/install-haskell-packages + -- which is not fully automated.) + -- armel builder has a companion container using amd64 that -- runs the build first to get TH splices. They need -- to have the same versions of all haskell libraries installed. -- cgit v1.2.3 From cc46cb19421d99f93fdc78cda50761dba3556b21 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 24 May 2014 00:09:58 -0400 Subject: propellor spin --- src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 13f1ac12..7941d7b4 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -88,6 +88,9 @@ buildDepsNoHaskellLibs = Apt.installed ["git", "rsync", "moreutils", "ca-certifi "libghc-dbus-dev", "libghc-fdo-notify-dev", "libghc-network-protocol-xmpp-dev" ] +noBuildDeps :: Property +noBuildDeps = propertyList "no build deps" [] + -- Installs current versions of git-annex's deps from cabal, but only -- does so once. cabalDeps :: Property @@ -109,10 +112,10 @@ androidContainer dockerImage crontimes timeout = Docker.container "android-git-a (dockerImage $ System (Debian Stable) "i386") & Apt.stdSourcesList Stable & Apt.unattendedUpgrades - & builder "android" crontimes timeout True + & builder' noBuildDeps "android" crontimes timeout True -- Use git-annex's android chroot setup script, which will install - -- ghc-android and the NDK, etc, in the home directory of the - -- builder user. + -- ghc-android and the NDK, all build deps, etc, in the home + -- directory of the builder user. & scriptProperty [ "cd " ++ builddir ++ " && ./standalone/android/buildchroot-inchroot" ] -- cgit v1.2.3 From 321a45a1ba1800241abacaaf1e0d1530360b9204 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 24 May 2014 00:52:37 -0400 Subject: propellor spin --- src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 7941d7b4..b8729871 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -113,16 +113,18 @@ androidContainer dockerImage crontimes timeout = Docker.container "android-git-a & Apt.stdSourcesList Stable & Apt.unattendedUpgrades & builder' noBuildDeps "android" crontimes timeout True + & flagFile chrootsetup ("/chrootsetup") + -- TODO: automate installing haskell libs + -- (Currently have to run + -- git-annex/standalone/android/install-haskell-packages + -- which is not fully automated.) + where -- Use git-annex's android chroot setup script, which will install -- ghc-android and the NDK, all build deps, etc, in the home -- directory of the builder user. - & scriptProperty + chrootsetup = scriptProperty [ "cd " ++ builddir ++ " && ./standalone/android/buildchroot-inchroot" ] - -- TODO: automate installing haskell libs - -- (Currently have to run - -- git-annex/standalone/android/install-haskell-packages - -- which is not fully automated.) -- armel builder has a companion container using amd64 that -- runs the build first to get TH splices. They need -- cgit v1.2.3 From e71bed664002a577c1dab6a411fbaaaf76383de6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 26 May 2014 11:55:30 -0400 Subject: propellor spin --- src/Propellor/Property/SiteSpecific/JoeySites.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/src/Propellor/Property/SiteSpecific/JoeySites.hs b/src/Propellor/Property/SiteSpecific/JoeySites.hs index 28b3dffd..5f054eaa 100644 --- a/src/Propellor/Property/SiteSpecific/JoeySites.hs +++ b/src/Propellor/Property/SiteSpecific/JoeySites.hs @@ -141,7 +141,10 @@ gitServer hosts = propertyList "git.kitenet.net setup" `requires` Ssh.knownHost hosts "usw-s002.rsync.net" "root" `requires` Ssh.authorizedKeys "family" `requires` User.accountFor "family" - , Apt.installed ["git", "rsync", "kgb-client-git", "gitweb"] + , Apt.installed ["git", "rsync", "gitweb"] + -- backport avoids channel flooding on branch merge + , Apt.installedBackport ["kgb-client"] + -- backport supports ssh event notification , Apt.installedBackport ["git-annex"] , File.hasPrivContentExposed "/etc/kgb-bot/kgb-client.conf" , toProp $ Git.daemonRunning "/srv/git" -- cgit v1.2.3 From 9fa0393f386977637fe16e93f31b835cc457d87d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 26 May 2014 18:46:42 -0400 Subject: propellor spin --- .../Property/SiteSpecific/GitAnnexBuilder.hs | 25 ++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index b8729871..a154c926 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -75,22 +75,25 @@ tree buildarch = combineProperties "gitannexbuilder tree" buildDeps :: Property buildDeps = combineProperties "gitannexbuilder build deps" [ Apt.buildDep ["git-annex"] - , buildDepsNoHaskellLibs + , buildDepsFewHaskellLibs , "git-annex source build deps installed" ==> Apt.buildDepIn builddir ] +buildDepsFewHaskellLibs :: Property +buildDepsFewHaskellLibs = combineProperties "gitannexbuilder build deps" + [ buildDepsNoHaskellLibs + -- these haskell libs depend on C libs and don't use TH + , Apt.installed ["libghc-dbus-dev", "libghc-fdo-notify-dev", "libghc-network-protocol-xmpp-dev"] + ] + buildDepsNoHaskellLibs :: Property -buildDepsNoHaskellLibs = Apt.installed ["git", "rsync", "moreutils", "ca-certificates", +buildDepsNoHaskellLibs = Apt.installed + ["git", "rsync", "moreutils", "ca-certificates", "debhelper", "ghc", "curl", "openssh-client", "git-remote-gcrypt", "liblockfile-simple-perl", "cabal-install", "vim", "less", - "alex", "happy", "c2hs", - -- these haskell libs depend on C libs and don't use TH - "libghc-dbus-dev", "libghc-fdo-notify-dev", "libghc-network-protocol-xmpp-dev" + "alex", "happy", "c2hs" ] -noBuildDeps :: Property -noBuildDeps = propertyList "no build deps" [] - -- Installs current versions of git-annex's deps from cabal, but only -- does so once. cabalDeps :: Property @@ -112,7 +115,7 @@ androidContainer dockerImage crontimes timeout = Docker.container "android-git-a (dockerImage $ System (Debian Stable) "i386") & Apt.stdSourcesList Stable & Apt.unattendedUpgrades - & builder' noBuildDeps "android" crontimes timeout True + & builder' buildDepsNoHaskellLibs "android" crontimes timeout True & flagFile chrootsetup ("/chrootsetup") -- TODO: automate installing haskell libs -- (Currently have to run @@ -138,7 +141,7 @@ armelCompanionContainer dockerImage = Docker.container "armel-git-annex-builder- & Docker.volume gitbuilderdir -- Install current versions of build deps from cabal. & tree "armel" - & buildDepsNoHaskellLibs + & buildDepsFewHaskellLibs & cabalDeps -- The armel builder can ssh to this companion. & Docker.expose "22" @@ -157,7 +160,7 @@ armelContainer dockerImage crontimes timeout = Docker.container "armel-git-annex -- (Currently have to run -- git-annex/standalone/linux/install-haskell-packages -- which is not fully automated.) - & builder' buildDepsNoHaskellLibs "armel" crontimes timeout True + & builder' buildDepsFewHaskellLibs "armel" crontimes timeout True & Ssh.keyImported SshRsa builduser & trivial writecompanionaddress where -- cgit v1.2.3 From fe685d2660ca04226330a1955705dff458a49b2c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 26 May 2014 18:59:13 -0400 Subject: propellor spin --- src/Propellor/Property/SiteSpecific/JoeySites.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/src/Propellor/Property/SiteSpecific/JoeySites.hs b/src/Propellor/Property/SiteSpecific/JoeySites.hs index 5f054eaa..587e16af 100644 --- a/src/Propellor/Property/SiteSpecific/JoeySites.hs +++ b/src/Propellor/Property/SiteSpecific/JoeySites.hs @@ -267,9 +267,11 @@ gitAnnexDistributor :: Property gitAnnexDistributor = combineProperties "git-annex distributor, including rsync server and signer" [ Apt.installed ["rsync"] , File.hasPrivContent "/etc/rsyncd.conf" + `onChange` Service.restarted "rsync" , File.hasPrivContent "/etc/rsyncd.secrets" + `onChange` Service.restarted "rsync" , "/etc/default/rsync" `File.containsLine` "RSYNC_ENABLE=true" - `onChange` Service.running "rsync" + `onChange` Service.running "rsync" , endpoint "/srv/web/downloads.kitenet.net/git-annex/autobuild" , endpoint "/srv/web/downloads.kitenet.net/git-annex/autobuild/x86_64-apple-mavericks" -- git-annex distribution signing key -- cgit v1.2.3 From 16fbf3c6f67f1eed11a9d189e2c7c2723ef5969a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 27 May 2014 21:17:53 -0400 Subject: reorg git-annex android builder, and put a dev container on my laptop --- config-joey.hs | 13 +++++--- .../Property/SiteSpecific/GitAnnexBuilder.hs | 35 ++++++++++++---------- 2 files changed, 28 insertions(+), 20 deletions(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/config-joey.hs b/config-joey.hs index 84cc7167..6c4dff7b 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -42,6 +42,7 @@ hosts = -- (o) ` & ipv6 "2001:4830:1600:187::2" -- sixxs tunnel & Docker.configured & Apt.buildDep ["git-annex"] `period` Daily + & Docker.docked hosts "android-git-annex" -- Nothing super-important lives here. , standardSystem "clam.kitenet.net" Unstable "amd64" @@ -193,11 +194,15 @@ hosts = -- (o) ` (Just "remotes/origin/old-kitenet.net") -- git-annex autobuilder containers - , GitAnnexBuilder.standardContainer dockerImage "amd64" 15 "2h" - , GitAnnexBuilder.standardContainer dockerImage "i386" 45 "2h" + , GitAnnexBuilder.standardAutoBuilderContainer dockerImage "amd64" 15 "2h" + , GitAnnexBuilder.standardAutoBuilderContainer dockerImage "i386" 45 "2h" , GitAnnexBuilder.armelCompanionContainer dockerImage - , GitAnnexBuilder.armelContainer dockerImage "1 3 * * *" "5h" - , GitAnnexBuilder.androidContainer dockerImage "1 1 * * *" "3h" + , GitAnnexBuilder.armelAutoBuilderContainer dockerImage "1 3 * * *" "5h" + , GitAnnexBuilder.androidAutoBuilderContainer dockerImage "1 1 * * *" "3h" + + -- for development of git-annex for android + , GitAnnexBuilder.androidContainer dockerImage "android-git-annex" + & Docker.volume ("/home/joey/src/git-annex:" ++ GitAnnexBuilder.homedir "git-annex") ] ++ monsters -- This is my standard system setup. diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index a154c926..15e4e6ce 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -24,12 +24,8 @@ builddir = gitbuilderdir "build" type TimeOut = String -- eg, 5h builder :: Architecture -> CronTimes -> TimeOut -> Bool -> Property -builder = builder' buildDeps - -builder' :: Property -> Architecture -> CronTimes -> TimeOut -> Bool -> Property -builder' buildepsprop buildarch crontimes timeout rsyncupload = combineProperties "gitannexbuilder" +builder buildarch crontimes timeout rsyncupload = combineProperties "gitannexbuilder" [ tree buildarch - , buildepsprop , Apt.serviceInstalledRunning "cron" , Cron.niceJob "gitannexbuilder" crontimes builduser gitbuilderdir $ "git pull ; timeout " ++ timeout ++ " ./autobuild" @@ -72,8 +68,8 @@ tree buildarch = combineProperties "gitannexbuilder tree" ] ] -buildDeps :: Property -buildDeps = combineProperties "gitannexbuilder build deps" +buildDepsApt :: Property +buildDepsApt = combineProperties "gitannexbuilder build deps" [ Apt.buildDep ["git-annex"] , buildDepsFewHaskellLibs , "git-annex source build deps installed" ==> Apt.buildDepIn builddir @@ -102,20 +98,26 @@ cabalDeps = flagFile go cabalupdated go = userScriptProperty builduser ["cabal update && cabal install git-annex --only-dependencies || true"] cabalupdated = homedir ".cabal" "packages" "hackage.haskell.org" "00-index.cache" -standardContainer :: (System -> Docker.Image) -> Architecture -> Int -> TimeOut -> Host -standardContainer dockerImage arch buildminute timeout = Docker.container (arch ++ "-git-annex-builder") +standardAutoBuilderContainer :: (System -> Docker.Image) -> Architecture -> Int -> TimeOut -> Host +standardAutoBuilderContainer dockerImage arch buildminute timeout = Docker.container (arch ++ "-git-annex-builder") (dockerImage $ System (Debian Unstable) arch) & Apt.stdSourcesList Unstable & Apt.unattendedUpgrades + & buildDepsApt & builder arch (show buildminute ++ " * * * *") timeout True +androidAutoBuilderContainer :: (System -> Docker.Image) -> Cron.CronTimes -> TimeOut -> Host +androidAutoBuilderContainer dockerImage crontimes timeout = + androidContainer dockerImage "android-git-annex-builder" + & Apt.unattendedUpgrades + & builder "android" crontimes timeout True + -- Android is cross-built in a Debian i386 container, using the Android NDK. -androidContainer :: (System -> Docker.Image) -> Cron.CronTimes -> TimeOut -> Host -androidContainer dockerImage crontimes timeout = Docker.container "android-git-annex-builder" +androidContainer :: (System -> Docker.Image) -> Docker.ContainerName -> Host +androidContainer dockerImage name = Docker.container name (dockerImage $ System (Debian Stable) "i386") & Apt.stdSourcesList Stable - & Apt.unattendedUpgrades - & builder' buildDepsNoHaskellLibs "android" crontimes timeout True + & buildDepsNoHaskellLibs & flagFile chrootsetup ("/chrootsetup") -- TODO: automate installing haskell libs -- (Currently have to run @@ -148,8 +150,8 @@ armelCompanionContainer dockerImage = Docker.container "armel-git-annex-builder- & Apt.serviceInstalledRunning "ssh" & Ssh.authorizedKeys builduser -armelContainer :: (System -> Docker.Image) -> Cron.CronTimes -> TimeOut -> Host -armelContainer dockerImage crontimes timeout = Docker.container "armel-git-annex-builder" +armelAutoBuilderContainer :: (System -> Docker.Image) -> Cron.CronTimes -> TimeOut -> Host +armelAutoBuilderContainer dockerImage crontimes timeout = Docker.container "armel-git-annex-builder" (dockerImage $ System (Debian Unstable) "armel") & Apt.stdSourcesList Unstable & Apt.unattendedUpgrades @@ -160,7 +162,8 @@ armelContainer dockerImage crontimes timeout = Docker.container "armel-git-annex -- (Currently have to run -- git-annex/standalone/linux/install-haskell-packages -- which is not fully automated.) - & builder' buildDepsFewHaskellLibs "armel" crontimes timeout True + & buildDepsFewHaskellLibs + & builder "armel" crontimes timeout True & Ssh.keyImported SshRsa builduser & trivial writecompanionaddress where -- cgit v1.2.3 From 2066292be9490fe2d8bc433e4c21b0a1907455e6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 28 May 2014 10:38:29 -0400 Subject: propellor spin --- config-joey.hs | 8 ++++--- src/Propellor/Property.hs | 3 +++ .../Property/SiteSpecific/GitAnnexBuilder.hs | 28 ++++++++++++---------- 3 files changed, 23 insertions(+), 16 deletions(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/config-joey.hs b/config-joey.hs index e145a39b..7d0a4e80 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -197,9 +197,11 @@ hosts = -- (o) ` , GitAnnexBuilder.armelAutoBuilderContainer dockerImage "1 3 * * *" "5h" , GitAnnexBuilder.androidAutoBuilderContainer dockerImage "1 1 * * *" "3h" - -- for development of git-annex for android - , GitAnnexBuilder.androidContainer dockerImage "android-git-annex" - & Docker.volume ("/home/joey/src/git-annex:" ++ GitAnnexBuilder.homedir "git-annex") + -- for development of git-annex for android, using my git-annex + -- work tree + , let gitannexdir = GitAnnexBuilder.homedir "git-annex" + in GitAnnexBuilder.androidContainer dockerImage "android-git-annex" doNothing gitannexdir + & Docker.volume ("/home/joey/src/git-annex:" ++ gitannexdir) ] ++ monsters -- This is my standard system setup. diff --git a/src/Propellor/Property.hs b/src/Propellor/Property.hs index 24494654..0728932e 100644 --- a/src/Propellor/Property.hs +++ b/src/Propellor/Property.hs @@ -103,6 +103,9 @@ trivial p = adjustProperty p $ \satisfy -> do then return NoChange else return r +doNothing :: Property +doNothing = property "noop property" noChange + -- | Makes a property that is satisfied differently depending on the host's -- operating system. -- diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 15e4e6ce..4f086a70 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -23,10 +23,9 @@ builddir = gitbuilderdir "build" type TimeOut = String -- eg, 5h -builder :: Architecture -> CronTimes -> TimeOut -> Bool -> Property -builder buildarch crontimes timeout rsyncupload = combineProperties "gitannexbuilder" - [ tree buildarch - , Apt.serviceInstalledRunning "cron" +autobuilder :: CronTimes -> TimeOut -> Bool -> Property +autobuilder crontimes timeout rsyncupload = combineProperties "gitannexbuilder" + [ Apt.serviceInstalledRunning "cron" , Cron.niceJob "gitannexbuilder" crontimes builduser gitbuilderdir $ "git pull ; timeout " ++ timeout ++ " ./autobuild" -- The builduser account does not have a password set, @@ -50,8 +49,7 @@ builder buildarch crontimes timeout rsyncupload = combineProperties "gitannexbui tree :: Architecture -> Property tree buildarch = combineProperties "gitannexbuilder tree" - [ User.accountFor builduser - , Apt.installed ["git"] + [ Apt.installed ["git"] -- gitbuilderdir directory already exists when docker volume is used, -- but with wrong owner. , File.dirExists gitbuilderdir @@ -104,21 +102,24 @@ standardAutoBuilderContainer dockerImage arch buildminute timeout = Docker.conta & Apt.stdSourcesList Unstable & Apt.unattendedUpgrades & buildDepsApt - & builder arch (show buildminute ++ " * * * *") timeout True + & autobuilder (show buildminute ++ " * * * *") timeout True + `requires` tree arch androidAutoBuilderContainer :: (System -> Docker.Image) -> Cron.CronTimes -> TimeOut -> Host androidAutoBuilderContainer dockerImage crontimes timeout = - androidContainer dockerImage "android-git-annex-builder" + androidContainer dockerImage "android-git-annex-builder" (tree "android") builddir & Apt.unattendedUpgrades - & builder "android" crontimes timeout True + & autobuilder crontimes timeout True -- Android is cross-built in a Debian i386 container, using the Android NDK. -androidContainer :: (System -> Docker.Image) -> Docker.ContainerName -> Host -androidContainer dockerImage name = Docker.container name +androidContainer :: (System -> Docker.Image) -> Docker.ContainerName -> Property -> FilePath -> Host +androidContainer dockerImage name setupgitannexdir gitannexdir = Docker.container name (dockerImage $ System (Debian Stable) "i386") & Apt.stdSourcesList Stable + & User.accountFor builduser & buildDepsNoHaskellLibs & flagFile chrootsetup ("/chrootsetup") + `requires` setupgitannexdir -- TODO: automate installing haskell libs -- (Currently have to run -- git-annex/standalone/android/install-haskell-packages @@ -128,7 +129,7 @@ androidContainer dockerImage name = Docker.container name -- ghc-android and the NDK, all build deps, etc, in the home -- directory of the builder user. chrootsetup = scriptProperty - [ "cd " ++ builddir ++ " && ./standalone/android/buildchroot-inchroot" + [ "cd " ++ gitannexdir ++ " && ./standalone/android/buildchroot-inchroot" ] -- armel builder has a companion container using amd64 that @@ -163,7 +164,8 @@ armelAutoBuilderContainer dockerImage crontimes timeout = Docker.container "arme -- git-annex/standalone/linux/install-haskell-packages -- which is not fully automated.) & buildDepsFewHaskellLibs - & builder "armel" crontimes timeout True + & autobuilder crontimes timeout True + `requires` tree "armel" & Ssh.keyImported SshRsa builduser & trivial writecompanionaddress where -- cgit v1.2.3 From 8c1848b897d5b5591db50bbc2e62e4f4f4b220b9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 28 May 2014 11:10:55 -0400 Subject: propellor spin --- src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 4f086a70..bc3e874f 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -117,6 +117,8 @@ androidContainer dockerImage name setupgitannexdir gitannexdir = Docker.containe (dockerImage $ System (Debian Stable) "i386") & Apt.stdSourcesList Stable & User.accountFor builduser + & File.dirExists gitbuilderdir + & File.ownerGroup gitbuilderdir builduser builduser & buildDepsNoHaskellLibs & flagFile chrootsetup ("/chrootsetup") `requires` setupgitannexdir -- cgit v1.2.3 From 3c7816cac0b922205aa5331223c08105518faf9b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 28 May 2014 11:20:45 -0400 Subject: propellor spin --- src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/Propellor/Property/SiteSpecific') diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index bc3e874f..3dcafa35 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -118,7 +118,7 @@ androidContainer dockerImage name setupgitannexdir gitannexdir = Docker.containe & Apt.stdSourcesList Stable & User.accountFor builduser & File.dirExists gitbuilderdir - & File.ownerGroup gitbuilderdir builduser builduser + & File.ownerGroup homedir builduser builduser & buildDepsNoHaskellLibs & flagFile chrootsetup ("/chrootsetup") `requires` setupgitannexdir -- cgit v1.2.3